Rework memory pressure system

src/init.cpp

@@ -517,7 +517,7 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc)
     argsman.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-allowignoredconf", strprintf("For backwards compatibility, treat an unused %s file in the datadir as a warning, not an error.", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-loadblock=<file>", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
-    argsman.AddArg("-lowmem=<n>", strprintf("If system available memory falls below <n> MiB, flush caches (0 to disable, default: %s)", g_low_memory_threshold / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+    argsman.AddArg("-lowmem=<n>", "If system available memory falls below <n> MiB, flush caches (0 to disable, default: 200 on Linux/Windows, 0 otherwise)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-maxmempool=<n>", strprintf("Keep the transaction memory pool below <n> megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE_MB), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-mempoolexpiry=<n>", strprintf("Do not keep transactions in the mempool longer than <n> hours (default: %u)", DEFAULT_MEMPOOL_EXPIRY_HOURS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
@@ -1461,7 +1461,17 @@ static ChainstateLoadResult InitAndLoadChainstate(
     LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", cache_sizes.coins * (1.0 / 1024 / 1024), mempool_opts.max_size_bytes * (1.0 / 1024 / 1024));
 
     if (gArgs.IsArgSet("-lowmem")) {
-        g_low_memory_threshold = gArgs.GetIntArg("-lowmem", 0 /* not used */) * 1024 * 1024;
+        const int64_t lowmem_val = gArgs.GetIntArg("-lowmem", 0);
+        if (lowmem_val < 0) {
+            return {ChainstateLoadStatus::FAILURE_FATAL, _("-lowmem must be non-negative")};
+        }
+        const uint64_t lowmem_mib = static_cast<uint64_t>(lowmem_val);
+        const uint64_t lowmem_bytes = (lowmem_mib > SIZE_MAX / (1024 * 1024)) ? SIZE_MAX : lowmem_mib * 1024 * 1024;
+        g_low_memory_threshold = static_cast<size_t>(lowmem_bytes);
+    } else {
+#if defined(__linux__) || defined(WIN32)
+        g_low_memory_threshold = 200 * 1024 * 1024;
+#endif
     }
     if (g_low_memory_threshold > 0) {
         LogPrintf("* Flushing caches if available system memory drops below %s MiB\n", g_low_memory_threshold / 1024 / 1024);
@@ -1612,6 +1622,24 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
         }
     }, std::chrono::minutes{5});
 
+    // Check memory pressure every 30 seconds and flush caches if needed.
+    if (g_low_memory_threshold > 0) {
+        scheduler.scheduleEvery([&node]{
+            if (!SystemNeedsMemoryReleased()) return;
+            if (!node.chainman) return;
+            LogPrintf("Memory pressure detected during idle, triggering flush\n");
+            LOCK(cs_main);
+            for (Chainstate* chainstate : node.chainman->GetAll()) {
+                if (chainstate->CanFlushToDisk()) {
+                    BlockValidationState state;
+                    if (!chainstate->FlushStateToDisk(state, FlushStateMode::IF_NEEDED)) {
+                        LogWarning("Memory pressure flush failed: %s\n", state.ToString());
+                    }
+                }
+            }
+        }, std::chrono::seconds{30});
+    }
+
     if (args.GetBoolArg("-logratelimit", BCLog::DEFAULT_LOGRATELIMIT)) {
         LogInstance().SetRateLimiting(BCLog::LogRateLimiter::Create(
             [&scheduler](auto func, auto window) { scheduler.scheduleEvery(std::move(func), window); },

src/txdb.h

@@ -45,6 +45,8 @@ class CCoinsViewDB final : public CCoinsView
 public:
     explicit CCoinsViewDB(DBParams db_params, CoinsViewOptions options);
 
+    size_t GetBatchWriteBytes() const { return m_options.batch_write_bytes; }
+
     std::optional<Coin> GetCoin(const COutPoint& outpoint) const override;
     bool HaveCoin(const COutPoint &outpoint) const override;
     uint256 GetBestBlock() const override;

src/util/mempressure.cpp

@@ -7,7 +7,6 @@
 #include <util/mempressure.h>
 
 #include <logging.h>
-#include <util/byte_units.h>
 
 #ifdef HAVE_LINUX_SYSINFO
 #include <sys/sysinfo.h>
@@ -16,15 +15,60 @@
 #include <windows.h>
 #endif
 
+#include <cinttypes>
 #include <cstddef>
 #include <cstdint>
+#include <cstdio>
+#include <cstring>
 
 size_t g_low_memory_threshold{0};
 
+uint64_t GetAvailableSystemMemory()
+{
+#ifdef __linux__
+    FILE* f = fopen("/proc/meminfo", "r");
+    if (f) {
+        char line[256];
+        while (fgets(line, sizeof(line), f)) {
+            if (strncmp(line, "MemAvailable:", 13) == 0) {
+                const char* p = line + 13;
+                while (*p == ' ') ++p;
+                uint64_t val = 0;
+                bool overflow = false;
+                while (*p >= '0' && *p <= '9') {
+                    if (val > UINT64_MAX / 10) { overflow = true; break; }
+                    val = val * 10 + (*p - '0');
+                    ++p;
+                }
+                fclose(f);
+                if (overflow || val > UINT64_MAX / 1024) return UINT64_MAX;
+                return val * 1024;
+            }
+        }
+        fclose(f);
+    }
+#endif
+#ifdef HAVE_LINUX_SYSINFO
+    struct sysinfo sys_info;
+    if (!sysinfo(&sys_info)) {
+        const uint64_t free_ram = uint64_t(sys_info.freeram) * sys_info.mem_unit;
+        const uint64_t buffer_ram = uint64_t(sys_info.bufferram) * sys_info.mem_unit;
+        return free_ram + buffer_ram;
+    }
+#endif
+#ifdef WIN32
+    MEMORYSTATUSEX mem_status;
+    mem_status.dwLength = sizeof(mem_status);
+    if (GlobalMemoryStatusEx(&mem_status)) {
+        return mem_status.ullAvailPhys;
+    }
+#endif
+    return 0;
+}
+
 bool SystemNeedsMemoryReleased()
 {
-    if (g_low_memory_threshold <= 0) {
-        // Intentionally bypass other metrics when disabled entirely
+    if (g_low_memory_threshold == 0) {
         return false;
     }
 #ifdef WIN32
@@ -34,23 +78,17 @@ bool SystemNeedsMemoryReleased()
         if (mem_status.dwMemoryLoad >= 99 ||
             mem_status.ullAvailPhys < g_low_memory_threshold ||
             mem_status.ullAvailVirtual < g_low_memory_threshold) {
-            LogPrintf("%s: YES: %s%% memory load; %s available physical memory; %s available virtual memory\n", __func__, int(mem_status.dwMemoryLoad), size_t(mem_status.ullAvailPhys), size_t(mem_status.ullAvailVirtual));
+            LogPrintf("%s: YES: %s%% memory load; %" PRIu64 " available physical memory; %" PRIu64 " available virtual memory\n", __func__, int(mem_status.dwMemoryLoad), uint64_t(mem_status.ullAvailPhys), uint64_t(mem_status.ullAvailVirtual));
             return true;
         }
     }
-#endif
-#ifdef HAVE_LINUX_SYSINFO
-    struct sysinfo sys_info;
-    if (!sysinfo(&sys_info)) {
-        // Explicitly 64-bit in case of 32-bit userspace on 64-bit kernel
-        const uint64_t free_ram = uint64_t(sys_info.freeram) * sys_info.mem_unit;
-        const uint64_t buffer_ram = uint64_t(sys_info.bufferram) * sys_info.mem_unit;
-        if (free_ram + buffer_ram < g_low_memory_threshold) {
-            LogPrintf("%s: YES: %s free RAM + %s buffer RAM\n", __func__, free_ram, buffer_ram);
-            return true;
-        }
+    return false;
+#else
+    const uint64_t avail = GetAvailableSystemMemory();
+    if (avail > 0 && avail < g_low_memory_threshold) {
+        LogPrintf("%s: YES: %" PRIu64 " bytes available memory\n", __func__, avail);
+        return true;
     }
-#endif
-    // NOTE: sysconf(_SC_AVPHYS_PAGES) doesn't account for caches on at least Linux, so not safe to use here
     return false;
+#endif
 }

src/util/mempressure.h

@@ -6,9 +6,13 @@
 #define BITCOIN_UTIL_MEMPRESSURE_H
 
 #include <cstddef>
+#include <cstdint>
 
 extern size_t g_low_memory_threshold;
 
+/** Returns available system memory in bytes, or 0 if unknown. */
+uint64_t GetAvailableSystemMemory();
+
 bool SystemNeedsMemoryReleased();
 
 #endif // BITCOIN_UTIL_MEMPRESSURE_H

src/validation.cpp

@@ -3053,6 +3053,15 @@ CoinsCacheSizeState Chainstate::GetCoinsCacheSizeState(
     int64_t nTotalSpace =
         max_coins_cache_size_bytes + std::max<int64_t>(int64_t(max_mempool_size_bytes) - nMempoolUsage, 0);
 
+    //! Reserve headroom for LevelDB write batch allocations during flush.
+    //! A batch uses ~3x its nominal size in practice due to Arena/MemTable
+    //! allocations, serialization buffers, and dual write buffers.
+    //! Only applied when cache is large enough for this to matter.
+    const int64_t leveldb_flush_headroom = static_cast<int64_t>(CoinsDB().GetBatchWriteBytes()) * 3;
+    if (nTotalSpace > leveldb_flush_headroom * 2) {
+        nTotalSpace -= leveldb_flush_headroom;
+    }
+
     //! No need to periodic flush if at least this much space still available.
     static constexpr int64_t MAX_BLOCK_COINSDB_USAGE_BYTES = 10 * 1024 * 1024;  // 10MB
     int64_t large_threshold =
@@ -3115,19 +3124,15 @@ bool Chainstate::FlushStateToDisk(
         const auto nNow{NodeClock::now()};
         // The cache is large and we're within 10% and 10 MiB of the limit, but we have time now (not in the middle of a block processing).
         bool fCacheLarge = mode == FlushStateMode::PERIODIC && cache_state >= CoinsCacheSizeState::LARGE;
-        bool fCacheCritical = false;
-        if (mode == FlushStateMode::IF_NEEDED) {
-            if (cache_state >= CoinsCacheSizeState::CRITICAL) {
-                // The cache is over the limit, we have to write now.
-                fCacheCritical = true;
-            } else if (SystemNeedsMemoryReleased()) {
-                fCacheCritical = true;
-            }
+        bool fCacheCritical = mode == FlushStateMode::IF_NEEDED && cache_state >= CoinsCacheSizeState::CRITICAL;
+        bool fMemoryPressure = false;
+        if (!fCacheLarge && !fCacheCritical && (mode == FlushStateMode::IF_NEEDED || mode == FlushStateMode::PERIODIC)) {
+            fMemoryPressure = SystemNeedsMemoryReleased();
         }
         // It's been a while since we wrote the block index and chain state to disk. Do this frequently, so we don't need to redownload or reindex after a crash.
         bool fPeriodicWrite = mode == FlushStateMode::PERIODIC && nNow >= m_next_write;
         // Combine all conditions that result in a write to disk.
-        bool should_write = (mode == FlushStateMode::ALWAYS) || fCacheLarge || fCacheCritical || fPeriodicWrite || fFlushForPrune;
+        bool should_write = (mode == FlushStateMode::ALWAYS) || fCacheLarge || fCacheCritical || fMemoryPressure || fPeriodicWrite || fFlushForPrune;
         // Write blocks, block index and best chain related state to disk.
         if (should_write) {
             // Ensure we can write block index
@@ -3175,7 +3180,9 @@ bool Chainstate::FlushStateToDisk(
                 return FatalError(m_chainman.GetNotifications(), state, _("Disk space is too low!"));
             }
             // Flush the chainstate (which may refer to block index entries).
-            const auto empty_cache{(mode == FlushStateMode::ALWAYS) || fCacheLarge || fCacheCritical};
+            // Memory pressure requires Flush() (not Sync()) because the pool
+            // allocator only releases memory to the OS via ReallocateCache().
+            const auto empty_cache{(mode == FlushStateMode::ALWAYS) || fCacheLarge || fCacheCritical || fMemoryPressure};
             if (empty_cache ? !CoinsTip().Flush() : !CoinsTip().Sync()) {
                 return FatalError(m_chainman.GetNotifications(), state, _("Failed to write to coin database."));
             }