diff --git a/BedrockCommand.cpp b/BedrockCommand.cpp index 31ebbd37d..7a8405a9e 100644 --- a/BedrockCommand.cpp +++ b/BedrockCommand.cpp @@ -52,6 +52,9 @@ BedrockCommand::BedrockCommand(SQLiteCommand&& baseCommand, BedrockPlugin* plugi } } _commandCount++; + if (_plugin) { + _plugin->activeCommandCount++; + } } const string& BedrockCommand::getName() const @@ -88,6 +91,9 @@ BedrockCommand::~BedrockCommand() if (destructionCallback) { (*destructionCallback)(); } + if (_plugin) { + _plugin->activeCommandCount--; + } _commandCount--; } diff --git a/BedrockPlugin.cpp b/BedrockPlugin.cpp index ca904c520..b1be951cf 100644 --- a/BedrockPlugin.cpp +++ b/BedrockPlugin.cpp @@ -3,6 +3,8 @@ #include "BedrockServer.h" map> BedrockPlugin::g_registeredPluginList; +map BedrockPlugin::g_pluginDLHandles; +map BedrockPlugin::g_pluginPaths; BedrockPlugin::BedrockPlugin(BedrockServer& s) : server(s) { diff --git a/BedrockPlugin.h b/BedrockPlugin.h index 11d7f2975..e92373c8f 100644 --- a/BedrockPlugin.h +++ b/BedrockPlugin.h @@ -101,6 +101,13 @@ class BedrockPlugin { // Map of plugin names to functions that will return a new plugin of the given type. static map> g_registeredPluginList; + // dlopen handles and filesystem paths for dynamically loaded plugins (keyed by UPPER(name)). + static map g_pluginDLHandles; + static map g_pluginPaths; + + // Tracks the number of in-flight commands owned by this plugin instance. + atomic activeCommandCount{0}; + // Reference to the BedrockServer object that owns this plugin. BedrockServer& server; }; diff --git a/BedrockServer.cpp b/BedrockServer.cpp index d22731ed2..6c1681bfd 100644 --- a/BedrockServer.cpp +++ b/BedrockServer.cpp @@ -7,7 +7,9 @@ #include #include #include +#include #include +#include #include #include @@ -1367,6 +1369,12 @@ BedrockServer::~BedrockServer() delete p.second; } + // Close any dynamically loaded plugin shared libraries. + for (auto& h : BedrockPlugin::g_pluginDLHandles) { + dlclose(h.second); + } + BedrockPlugin::g_pluginDLHandles.clear(); + shutdownTimer.serverDestructor = chrono::steady_clock::now(); SINFO("Shutdown timing: " << "start->safeState=" << chrono::duration_cast(shutdownTimer.safeNodeState - shutdownTimer.shutdownStart) @@ -1473,10 +1481,13 @@ void BedrockServer::postPoll(fd_map& fdm, uint64_t& nextActivity) _acceptSockets(); // If any plugin timers are firing, let the plugins know. - for (auto plugin : plugins) { - for (SStopwatch* timer : plugin.second->timers) { - if (timer->ding()) { - plugin.second->timerFired(timer); + { + shared_lock pluginLock(_pluginsMutex); + for (auto plugin : plugins) { + for (SStopwatch* timer : plugin.second->timers) { + if (timer->ding()) { + plugin.second->timerFired(timer); + } } } } @@ -1516,7 +1527,15 @@ unique_ptr BedrockServer::getCommandFromPlugins(SData&& request) unique_ptr BedrockServer::getCommandFromPlugins(unique_ptr&& baseCommand) { try { + shared_lock pluginLock(_pluginsMutex); for (auto pair : plugins) { + // Reject commands for a plugin that is being hot-reloaded. + if (_pluginReloadInProgress.load() && SIEquals(pair.first, _reloadingPluginName)) { + auto errorCommand = make_unique(SQLiteCommand(move(*baseCommand)), nullptr); + errorCommand->complete = true; + errorCommand->response.methodLine = "503 Plugin reload in progress"; + return errorCommand; + } // This is a bit weird to avoid changing this signature in all the plugins. It would be more straightforward if // the plugins just accepted a `unique_ptr&&`, but this still works. auto command = pair.second->getCommand(move(*baseCommand)); @@ -1857,6 +1876,7 @@ bool BedrockServer::_isControlCommand(const unique_ptr& command) SIEquals(command->request.methodLine, "BlockWrites") || SIEquals(command->request.methodLine, "UnblockWrites") || SIEquals(command->request.methodLine, "SetMaxSocketThreads") || + SIEquals(command->request.methodLine, "ReloadPlugin") || SIEquals(command->request.methodLine, "CRASH_COMMAND") ) { return true; @@ -2019,6 +2039,321 @@ void BedrockServer::_control(unique_ptr& command) } else { response.methodLine = "401 Don't Use Zero"; } + } else if (SIEquals(command->request.methodLine, "ReloadPlugin")) { + // Hot-reload a dynamically loaded plugin (.so) without restarting Bedrock. + // Use "PluginName" header (not "Plugin") to avoid collision with the internal "plugin" + // header that _reply() uses to route responses to plugin port handlers. + string pluginKey = SToUpper(command->request["PluginName"]); + if (pluginKey.empty()) { + response.methodLine = "400 Missing PluginName header"; + return; + } + + // Phase 1: Validate + // The plugins map is keyed by plugin->getName() which may differ in case from the + // upper-cased key used in g_registeredPluginList/g_pluginDLHandles. Find it case-insensitively. + string pluginMapKey; + for (auto& p : plugins) { + if (SIEquals(p.first, pluginKey)) { + pluginMapKey = p.first; + break; + } + } + if (pluginMapKey.empty()) { + response.methodLine = "400 Plugin not found: " + pluginKey; + return; + } + auto handleIt = BedrockPlugin::g_pluginDLHandles.find(pluginKey); + if (handleIt == BedrockPlugin::g_pluginDLHandles.end()) { + response.methodLine = "400 Plugin is built-in, cannot reload: " + pluginKey; + return; + } + string pluginPath = BedrockPlugin::g_pluginPaths[pluginKey]; + if (!SFileExists(pluginPath)) { + response.methodLine = "400 Plugin .so file not found: " + pluginPath; + return; + } + + SINFO("[ReloadPlugin] Starting reload of plugin '" << pluginKey << "' from " << pluginPath); + + // Phase 2: Suppress new commands for this plugin + blockCommandPort("ReloadPlugin"); + _pluginReloadInProgress.store(true); + _reloadingPluginName = pluginMapKey; + + // Phase 3: Drain in-flight commands (120s timeout, slightly above DEFAULT_TIMEOUT of 110s) + constexpr uint64_t DRAIN_TIMEOUT_US = 120'000'000; + BedrockPlugin* oldPlugin = plugins[pluginMapKey]; + uint64_t drainStart = STimeNow(); + while (oldPlugin->activeCommandCount.load() > 0) { + if (STimeNow() - drainStart > DRAIN_TIMEOUT_US) { + SWARN("[ReloadPlugin] Drain timeout after 120s, " << oldPlugin->activeCommandCount.load() + << " commands still active. Aborting reload."); + _pluginReloadInProgress.store(false); + _reloadingPluginName.clear(); + unblockCommandPort("ReloadPlugin"); + response.methodLine = "500 Plugin reload failed: drain timeout"; + return; + } + usleep(10'000); + } + SINFO("[ReloadPlugin] All commands drained for plugin " << pluginKey); + + // Phase 4: Tear down old plugin + string oldSOPath = pluginPath; + void* oldHandle = handleIt->second; + string symbolName = "BEDROCK_PLUGIN_REGISTER_" + pluginKey; + + { + unique_lock pluginLock(_pluginsMutex); + oldPlugin->serverStopping(); + delete oldPlugin; + plugins.erase(pluginMapKey); + BedrockPlugin::g_pluginDLHandles.erase(pluginKey); + } + dlclose(oldHandle); + SINFO("[ReloadPlugin] Old plugin instance destroyed and .so closed"); + + // Phase 5: Load new plugin + // dlopen caches by path — calling dlclose + dlopen on the same path may return the + // old in-memory copy. Copy the .so to a unique temp path to force a fresh load. + string tempSOPath = pluginPath + ".reload." + to_string(STimeNow()); + if (!SFileCopy(pluginPath, tempSOPath)) { + SALERT("[ReloadPlugin] Failed to copy " << pluginPath << " to " << tempSOPath); + _pluginReloadInProgress.store(false); + _reloadingPluginName.clear(); + unblockCommandPort("ReloadPlugin"); + response.methodLine = "500 Plugin reload failed: could not create temp copy of .so"; + return; + } + void* newLib = dlopen(tempSOPath.c_str(), RTLD_NOW); + SFileDelete(tempSOPath); + if (!newLib) { + string dlErr = dlerror(); + SALERT("[ReloadPlugin] Failed to dlopen new .so: " << dlErr << ". Attempting rollback."); + // Attempt rollback with old path + void* rollbackLib = dlopen(oldSOPath.c_str(), RTLD_NOW); + if (rollbackLib) { + void* rollbackSym = dlsym(rollbackLib, symbolName.c_str()); + if (rollbackSym) { + try { + auto factory = (BedrockPlugin*(*)(BedrockServer&))rollbackSym; + BedrockPlugin* rollbackPlugin = factory(*this); + unique_lock pluginLock(_pluginsMutex); + plugins[pluginMapKey] = rollbackPlugin; + BedrockPlugin::g_pluginDLHandles[pluginKey] = rollbackLib; + BedrockPlugin::g_registeredPluginList[pluginKey] = factory; + pluginLock.unlock(); + { + size_t rbIdx = _dbPool->getIndex(); + SQLite& rbDB = _dbPool->initializeIndex(rbIdx); + rollbackPlugin->stateChanged(rbDB, getState()); + shared_ptr rbPoolCopy = _dbPool; + thread([rbPoolCopy, rbIdx, this]() { + SInitialize("ReloadPluginRollbackDBReturn"); + while (!isUpgradeComplete()) { usleep(50'000); } + usleep(100'000); + rbPoolCopy->returnToPool(rbIdx); + }).detach(); + } + SWARN("[ReloadPlugin] Rolled back to previous plugin version"); + _pluginReloadInProgress.store(false); + _reloadingPluginName.clear(); + unblockCommandPort("ReloadPlugin"); + response.methodLine = "500 Plugin reload failed: " + dlErr + ". Rolled back to previous version."; + return; + } catch (const exception& e) { + dlclose(rollbackLib); + SALERT("[ReloadPlugin] Rollback constructor failed: " << e.what()); + } + } else { + dlclose(rollbackLib); + } + } + SALERT("[ReloadPlugin] Rollback failed. Plugin " << pluginKey << " is now unavailable."); + _pluginReloadInProgress.store(false); + _reloadingPluginName.clear(); + unblockCommandPort("ReloadPlugin"); + response.methodLine = "500 Plugin reload failed: " + dlErr + ". Rollback failed. Plugin " + pluginKey + " is now unavailable."; + return; + } + + void* newSym = dlsym(newLib, symbolName.c_str()); + if (!newSym) { + string dlErr = dlerror(); + dlclose(newLib); + SALERT("[ReloadPlugin] Failed to find symbol " << symbolName << ": " << dlErr << ". Attempting rollback."); + // Rollback attempt + void* rollbackLib = dlopen(oldSOPath.c_str(), RTLD_NOW); + if (rollbackLib) { + void* rollbackSym = dlsym(rollbackLib, symbolName.c_str()); + if (rollbackSym) { + try { + auto factory = (BedrockPlugin*(*)(BedrockServer&))rollbackSym; + BedrockPlugin* rollbackPlugin = factory(*this); + unique_lock pluginLock(_pluginsMutex); + plugins[pluginMapKey] = rollbackPlugin; + BedrockPlugin::g_pluginDLHandles[pluginKey] = rollbackLib; + BedrockPlugin::g_registeredPluginList[pluginKey] = factory; + pluginLock.unlock(); + { + size_t rbIdx = _dbPool->getIndex(); + SQLite& rbDB = _dbPool->initializeIndex(rbIdx); + rollbackPlugin->stateChanged(rbDB, getState()); + shared_ptr rbPoolCopy = _dbPool; + thread([rbPoolCopy, rbIdx, this]() { + SInitialize("ReloadPluginRollbackDBReturn"); + while (!isUpgradeComplete()) { usleep(50'000); } + usleep(100'000); + rbPoolCopy->returnToPool(rbIdx); + }).detach(); + } + SWARN("[ReloadPlugin] Rolled back to previous plugin version"); + _pluginReloadInProgress.store(false); + _reloadingPluginName.clear(); + unblockCommandPort("ReloadPlugin"); + response.methodLine = "500 Plugin reload failed: " + dlErr + ". Rolled back to previous version."; + return; + } catch (const exception& e) { + dlclose(rollbackLib); + } + } else { + dlclose(rollbackLib); + } + } + _pluginReloadInProgress.store(false); + _reloadingPluginName.clear(); + unblockCommandPort("ReloadPlugin"); + response.methodLine = "500 Plugin reload failed: " + dlErr + ". Rollback failed. Plugin " + pluginKey + " is now unavailable."; + return; + } + + auto newFactory = (BedrockPlugin*(*)(BedrockServer&))newSym; + BedrockPlugin* newPlugin = nullptr; + try { + newPlugin = newFactory(*this); + } catch (const exception& e) { + dlclose(newLib); + SALERT("[ReloadPlugin] New plugin constructor threw: " << e.what() << ". Attempting rollback."); + void* rollbackLib = dlopen(oldSOPath.c_str(), RTLD_NOW); + if (rollbackLib) { + void* rollbackSym = dlsym(rollbackLib, symbolName.c_str()); + if (rollbackSym) { + try { + auto factory = (BedrockPlugin*(*)(BedrockServer&))rollbackSym; + BedrockPlugin* rollbackPlugin = factory(*this); + unique_lock pluginLock(_pluginsMutex); + plugins[pluginMapKey] = rollbackPlugin; + BedrockPlugin::g_pluginDLHandles[pluginKey] = rollbackLib; + BedrockPlugin::g_registeredPluginList[pluginKey] = factory; + pluginLock.unlock(); + { + size_t rbIdx = _dbPool->getIndex(); + SQLite& rbDB = _dbPool->initializeIndex(rbIdx); + rollbackPlugin->stateChanged(rbDB, getState()); + shared_ptr rbPoolCopy = _dbPool; + thread([rbPoolCopy, rbIdx, this]() { + SInitialize("ReloadPluginRollbackDBReturn"); + while (!isUpgradeComplete()) { usleep(50'000); } + usleep(100'000); + rbPoolCopy->returnToPool(rbIdx); + }).detach(); + } + SWARN("[ReloadPlugin] Rolled back to previous plugin version"); + _pluginReloadInProgress.store(false); + _reloadingPluginName.clear(); + unblockCommandPort("ReloadPlugin"); + response.methodLine = "500 Plugin reload failed: "s + e.what() + ". Rolled back to previous version."; + return; + } catch (const exception& e2) { + dlclose(rollbackLib); + } + } else { + dlclose(rollbackLib); + } + } + _pluginReloadInProgress.store(false); + _reloadingPluginName.clear(); + unblockCommandPort("ReloadPlugin"); + response.methodLine = "500 Plugin reload failed: "s + e.what() + ". Rollback failed. Plugin " + pluginKey + " is now unavailable."; + return; + } + + // Insert the new plugin and save the handle + { + unique_lock pluginLock(_pluginsMutex); + plugins[pluginMapKey] = newPlugin; + } + BedrockPlugin::g_pluginDLHandles[pluginKey] = newLib; + BedrockPlugin::g_pluginPaths[pluginKey] = pluginPath; + BedrockPlugin::g_registeredPluginList[pluginKey] = newFactory; + SINFO("[ReloadPlugin] New plugin instance created and registered"); + + // Phase 6: Initialize new plugin + // Auth's stateChanged() spawns a detached thread that captures `db` by reference + // and uses it after stateChanged() returns. A SQLiteScopedHandle would be destroyed + // at the end of this block, leaving the detached thread with a dangling reference. + // Instead, we acquire a pool handle and return it in a background thread after + // the plugin's async initialization completes. + if (_dbPool) { + size_t dbIndex = _dbPool->getIndex(); + SQLite& db = _dbPool->initializeIndex(dbIndex); + if (getState() == SQLiteNodeState::LEADING) { + SINFO("[ReloadPlugin] Running upgradeDatabase for " << pluginKey); + try { + db.beginTransaction(SQLite::TRANSACTION_TYPE::EXCLUSIVE); + newPlugin->upgradeDatabase(db); + if (db.getUncommittedQuery().empty()) { + db.rollback(); + } else { + SINFO("[ReloadPlugin] Schema changes detected, committing."); + db.prepare(); + db.commit(); + } + } catch (const exception& e) { + SWARN("[ReloadPlugin] upgradeDatabase failed: " << e.what()); + db.rollback(); + } + } + newPlugin->stateChanged(db, getState()); + + // Return the DB handle to the pool once the plugin's async initialization is done. + // This is necessary because Auth's stateChanged spawns a detached thread that uses + // the db reference until isUpgradeComplete() returns true. + shared_ptr dbPoolCopy = _dbPool; + thread([dbPoolCopy, dbIndex, this]() { + SInitialize("ReloadPluginDBReturn"); + while (!isUpgradeComplete()) { + usleep(50'000); + } + // Give the plugin's thread a moment to finish using the handle after upgrade completes. + usleep(100'000); + dbPoolCopy->returnToPool(dbIndex); + SINFO("[ReloadPlugin] Returned DB handle to pool after plugin initialization"); + }).detach(); + } + + // Rebuild the _version string to reflect the reloaded plugin's version. + { + vector versions = {VERSION}; + for (auto& p : plugins) { + auto info = p.second->getInfo(); + auto it = info.find("version"); + if (it != info.end()) { + versions.push_back(p.second->getName() + "_" + it->second); + } + } + sort(versions.begin(), versions.end()); + _version = SComposeList(versions, ":"); + SINFO("[ReloadPlugin] Updated version string: " << _version); + } + + // Phase 7: Resume + _pluginReloadInProgress.store(false); + _reloadingPluginName.clear(); + unblockCommandPort("ReloadPlugin"); + SINFO("[ReloadPlugin] Plugin " << pluginKey << " reloaded successfully"); + response.methodLine = "200 Plugin reloaded successfully"; } } @@ -2026,6 +2361,7 @@ bool BedrockServer::_upgradeDB(SQLite& db) { // These all get conglomerated into one big query. try { + shared_lock pluginLock(_pluginsMutex); db.beginTransaction(SQLite::TRANSACTION_TYPE::EXCLUSIVE); for (auto plugin : plugins) { plugin.second->upgradeDatabase(db); @@ -2549,6 +2885,7 @@ void BedrockServer::handleSocket(Socket&& socket, bool fromControlPort, bool fro void BedrockServer::notifyStateChangeToPlugins(SQLite& db, SQLiteNodeState newState) { + shared_lock pluginLock(_pluginsMutex); for (auto plugin : plugins) { plugin.second->stateChanged(db, newState); } diff --git a/BedrockServer.h b/BedrockServer.h index 622c73ffb..ac26734ac 100644 --- a/BedrockServer.h +++ b/BedrockServer.h @@ -1,5 +1,6 @@ #pragma once #include +#include #include #include #include @@ -437,6 +438,13 @@ class BedrockServer : public SQLiteServer { // Whether or not all plugins are detached bool _pluginsDetached; + // Plugin hot-reload support. _pluginReloadInProgress is checked by getCommandFromPlugins() to reject + // commands for the plugin being reloaded. _reloadingPluginName is the UPPER-cased name of that plugin. + // _pluginsMutex protects the plugins map during the brief swap window. + atomic _pluginReloadInProgress{false}; + string _reloadingPluginName; + shared_mutex _pluginsMutex; + // This is a snapshot of the state of the node taken at the beginning of any call to peekCommand or processCommand // so that the state can't change for the lifetime of that call, from the view of that function. static thread_local atomic _nodeStateSnapshot; diff --git a/libstuff/SLog.cpp b/libstuff/SLog.cpp index 9eb260a44..771f64982 100644 --- a/libstuff/SLog.cpp +++ b/libstuff/SLog.cpp @@ -62,6 +62,7 @@ static set PARAMS_WHITELIST = { "logParam", "message", "peer", + "PluginName", "prepareElapsed", "query", "readElapsed", diff --git a/main.cpp b/main.cpp index 0eff7efc1..97a168b3f 100644 --- a/main.cpp +++ b/main.cpp @@ -4,6 +4,7 @@ /// #include #include +#include #include #include #include @@ -145,6 +146,23 @@ set loadPlugins(SData& args) } else { // Call the plugin registration function with the same name. BedrockPlugin::g_registeredPluginList.emplace(make_pair(SToUpper(name), (BedrockPlugin * (*)(BedrockServer&)) sym)); + BedrockPlugin::g_pluginDLHandles[SToUpper(name)] = lib; + + // Resolve the full filesystem path of the loaded .so via dlinfo, since + // pluginName may be a bare filename (e.g., "auth.so") that dlopen resolved + // through the standard library search path. + string resolvedPath = pluginName; + struct link_map* lm = nullptr; + if (dlinfo(lib, RTLD_DI_LINKMAP, &lm) == 0 && lm && lm->l_name[0]) { + char* rp = realpath(lm->l_name, nullptr); + if (rp) { + resolvedPath = rp; + free(rp); + } else { + resolvedPath = lm->l_name; + } + } + BedrockPlugin::g_pluginPaths[SToUpper(name)] = resolvedPath; } } } diff --git a/test/clustertest/tests/ReloadPluginTest.cpp b/test/clustertest/tests/ReloadPluginTest.cpp new file mode 100644 index 000000000..a922dae29 --- /dev/null +++ b/test/clustertest/tests/ReloadPluginTest.cpp @@ -0,0 +1,151 @@ +#include +#include +#include + +struct ReloadPluginTest : tpunit::TestFixture { + ReloadPluginTest() + : tpunit::TestFixture("ReloadPlugin", + TEST(ReloadPluginTest::happyPathReload), + TEST(ReloadPluginTest::reloadDrainsInflightCommands), + TEST(ReloadPluginTest::reloadBuiltinPluginRejected), + TEST(ReloadPluginTest::reloadNonexistentPluginRejected), + TEST(ReloadPluginTest::reloadPreservesDBState), + TEST(ReloadPluginTest::reloadOnFollower), + TEST(ReloadPluginTest::doubleReloadSerializes)) + { + } + + // Helper to send ReloadPlugin control command + string sendReloadPlugin(BedrockTester& tester, const string& pluginName) { + SData command("ReloadPlugin"); + command["PluginName"] = pluginName; + return tester.executeWaitVerifyContent(command, "", true); + } + + void happyPathReload() { + // Start a 3-node cluster with the test plugin + BedrockClusterTester tester(ClusterSize::THREE_NODE_CLUSTER); + BedrockTester& leader = tester.getTester(0); + + // Verify plugin works before reload + SData cmd("testcommand"); + leader.executeWaitVerifyContent(cmd, "200"); + + // Reload the plugin + SData reloadCmd("ReloadPlugin"); + reloadCmd["PluginName"] = "TESTPLUGIN"; + leader.executeWaitVerifyContent(reloadCmd, "200", true); + + // Verify plugin still works after reload + SData cmd2("testcommand"); + leader.executeWaitVerifyContent(cmd2, "200"); + } + + void reloadDrainsInflightCommands() { + BedrockClusterTester tester(ClusterSize::THREE_NODE_CLUSTER); + BedrockTester& leader = tester.getTester(0); + + // Start a slow query that will take a few seconds + SData slowCmd("slowquery"); + slowCmd["size"] = "5000000"; + + // Send the slow command in a background thread + thread slowThread([&]() { + leader.executeWaitVerifyContent(slowCmd, "200"); + }); + + // Give it a moment to start processing + usleep(100'000); + + // Send reload - it should wait for the slow query to finish + SData reloadCmd("ReloadPlugin"); + reloadCmd["PluginName"] = "TESTPLUGIN"; + leader.executeWaitVerifyContent(reloadCmd, "200", true); + + slowThread.join(); + + // Verify commands work after reload + SData cmd("testcommand"); + leader.executeWaitVerifyContent(cmd, "200"); + } + + void reloadBuiltinPluginRejected() { + BedrockClusterTester tester(ClusterSize::THREE_NODE_CLUSTER); + BedrockTester& leader = tester.getTester(0); + + SData cmd("ReloadPlugin"); + cmd["PluginName"] = "DB"; + leader.executeWaitVerifyContent(cmd, "400", true); + } + + void reloadNonexistentPluginRejected() { + BedrockClusterTester tester(ClusterSize::THREE_NODE_CLUSTER); + BedrockTester& leader = tester.getTester(0); + + SData cmd("ReloadPlugin"); + cmd["PluginName"] = "FAKEPLUGIN"; + leader.executeWaitVerifyContent(cmd, "400", true); + } + + void reloadPreservesDBState() { + BedrockClusterTester tester(ClusterSize::THREE_NODE_CLUSTER); + BedrockTester& leader = tester.getTester(0); + + // Insert some data + SData insertCmd("testquery"); + insertCmd["Query"] = "INSERT INTO test (id, value) VALUES (12345, 'reload_test_data');"; + leader.executeWaitVerifyContent(insertCmd, "200"); + + // Reload the plugin + SData reloadCmd("ReloadPlugin"); + reloadCmd["PluginName"] = "TESTPLUGIN"; + leader.executeWaitVerifyContent(reloadCmd, "200", true); + + // Verify data persists after reload + SData selectCmd("testquery"); + selectCmd["Query"] = "SELECT value FROM test WHERE id = 12345;"; + leader.executeWaitVerifyContent(selectCmd, "200"); + } + + void reloadOnFollower() { + BedrockClusterTester tester(ClusterSize::THREE_NODE_CLUSTER); + BedrockTester& follower = tester.getTester(1); + + // Wait for the follower to be in FOLLOWING state + follower.waitForState("FOLLOWING"); + + // Reload on the follower + SData reloadCmd("ReloadPlugin"); + reloadCmd["PluginName"] = "TESTPLUGIN"; + follower.executeWaitVerifyContent(reloadCmd, "200", true); + + // Verify commands still work on the follower + SData cmd("testcommand"); + follower.executeWaitVerifyContent(cmd, "200"); + } + + void doubleReloadSerializes() { + BedrockClusterTester tester(ClusterSize::THREE_NODE_CLUSTER); + BedrockTester& leader = tester.getTester(0); + + // Send two reload commands concurrently + thread t1([&]() { + SData reloadCmd("ReloadPlugin"); + reloadCmd["PluginName"] = "TESTPLUGIN"; + leader.executeWaitVerifyContent(reloadCmd, "", true); + }); + thread t2([&]() { + SData reloadCmd("ReloadPlugin"); + reloadCmd["PluginName"] = "TESTPLUGIN"; + leader.executeWaitVerifyContent(reloadCmd, "", true); + }); + + t1.join(); + t2.join(); + + // Server should still be stable + SData cmd("testcommand"); + leader.executeWaitVerifyContent(cmd, "200"); + } + +} __ReloadPluginTest;