From 69a01b88f35db2a5003d42116f573207ca5c275b Mon Sep 17 00:00:00 2001 From: Tomasz Sobczyk Date: Tue, 22 Jul 2025 13:42:01 +0200 Subject: [PATCH] Use shared memory for network weights MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This enables different Stockfish processes that use the same weights to use the same memory. The approach establishes equivalence by memory content, and is compatible with NUMA replication. The benefit of sharing is reduced memory usage and a speedup thanks to improved (inter-process) caching of the network in the CPUs cache, and thus reduced bandwidth usage to main memory. Even though this change doesn't benefit a user running a single process, this helps on fishtest or e.g. for Lichess, when multiple games run concurrently, or multiple positions are analyzed in parallel. This concept was probably first introduced in the Monty engine (https://github.com/official-monty/Monty/pull/62), after a discussion in https://github.com/official-stockfish/fishtest/issues/2077 on the issue of memory pressure. Measurements based on Torch (https://github.com/user-attachments/files/21386224/verbatim.pdf) further suggested that large gains were possible. Multiple other engines have adopted this 'verbatim' format as well. The implementation here adds the flexibility needed for SF, for example, retains the ability to bundle compressed networks with the binary, to load nets by uci option, and to distribute the shared nets to the proper NUMA region. This flexibility comes with a fair amount of complexity in the implementation, such as OS specific code, and fallback code. For most users this should be transparent. However, for example, those running docker containers should ensure the `--ipc` flag is set correctly, and `--shm-size` is sufficiently large. The benefits of this patch significantly depend on hardware, with systems with many cores and a large (O(150MB), the net size) L3 cache benefitting typically most. On such systems SF speedups (as measured via nps playing games with large concurrency but just 1 thread) can be 38%, which results in master vs. patch Elo which gains about 25 Elo. ``` # PLAYER : RATING ERROR POINTS PLAYED (%) 1 shared_memoryPR : 24.8 1.9 39432.0 73728 53 2 master : 0.0 ---- 34296.0 73728 47 ``` In a multithreaded setup, where weights are already shared, that benefit is smaller, for example on the same HW as above, but with 8t for each side. ``` # PLAYER : RATING ERROR POINTS PLAYED (%) 1 shared_memoryPR : 5.2 3.5 9351.0 18432 51 2 master : 0.0 ---- 9081.0 18432 49 ``` On fishtest with a typical hardware mix of our contributors, the following was measured: STC, 60k games https://tests.stockfishchess.org/tests/view/69074a49ea4b268f1fac236c Elo: 4.69 ± 1.4 (95%) LOS: 100.0% Total: 60000 W: 16085 L: 15275 D: 28640 Ptnml(0-2): 154, 6440, 16053, 7148, 205 nElo: 9.38 ± 2.8 (95%) PairsRatio: 1.12 To verify correctness with a single process on a NUMA architecture, speedtest was used, confirming near equivalence: ``` master: Average (over 10): 296236186 shared_memory: Average (over 10): 295769332 ``` Currently, using large pages for the shared network weights is not always possible, which can lead to a small slowdown (1-2%), in case a single process is run. closes https://github.com/official-stockfish/Stockfish/pull/6173 No functional change Co-authored-by: disservin Co-authored-by: Joost VandeVondele --- src/Makefile | 16 +- src/engine.cpp | 43 +- src/engine.h | 8 +- src/evaluate.cpp | 8 +- src/main.cpp | 10 +- src/memory.cpp | 85 +-- src/memory.h | 98 +++ src/misc.h | 101 ++- src/nnue/layers/affine_transform.h | 9 + .../layers/affine_transform_sparse_input.h | 9 + src/nnue/layers/clipped_relu.h | 6 + src/nnue/layers/sqr_clipped_relu.h | 6 + src/nnue/network.cpp | 82 +-- src/nnue/network.h | 49 +- src/nnue/nnue_accumulator.h | 2 +- src/nnue/nnue_architecture.h | 22 +- src/nnue/nnue_common.h | 4 +- src/nnue/nnue_feature_transformer.h | 34 +- src/nnue/nnue_misc.cpp | 12 +- src/nnue/nnue_misc.h | 21 +- src/numa.h | 142 +++- src/search.h | 24 +- src/shm.h | 634 +++++++++++++++++ src/shm_linux.h | 658 ++++++++++++++++++ 24 files changed, 1885 insertions(+), 198 deletions(-) create mode 100644 src/shm.h create mode 100644 src/shm_linux.h diff --git a/src/Makefile b/src/Makefile index 7244f7040..b30e16c07 100644 --- a/src/Makefile +++ b/src/Makefile @@ -435,7 +435,7 @@ endif ifeq ($(COMP),gcc) comp=gcc CXX=g++ - CXXFLAGS += -pedantic -Wextra -Wshadow -Wmissing-declarations + CXXFLAGS += -pedantic -Wextra -Wshadow -Wmissing-declarations -Wstack-usage=128000 ifeq ($(arch),$(filter $(arch),armv7 armv8 riscv64)) ifeq ($(OS),Android) @@ -618,6 +618,19 @@ ifneq ($(comp),mingw) ifneq ($(KERNEL),Haiku) ifneq ($(COMP),ndk) LDFLAGS += -lpthread + + add_lrt = yes + ifeq ($(target_windows),yes) + add_lrt = no + endif + + ifeq ($(KERNEL),Darwin) + add_lrt = no + endif + + ifeq ($(add_lrt),yes) + LDFLAGS += -lrt + endif endif endif endif @@ -628,6 +641,7 @@ ifeq ($(debug),no) CXXFLAGS += -DNDEBUG else CXXFLAGS += -g + CXXFLAGS += -D_GLIBCXX_ASSERTIONS -D_GLIBCXX_DEBUG endif ### 3.2.2 Debugging with undefined behavior sanitizers diff --git a/src/engine.cpp b/src/engine.cpp index a4c0bb1eb..9edff4864 100644 --- a/src/engine.cpp +++ b/src/engine.cpp @@ -33,10 +33,12 @@ #include "misc.h" #include "nnue/network.h" #include "nnue/nnue_common.h" +#include "nnue/nnue_misc.h" #include "numa.h" #include "perft.h" #include "position.h" #include "search.h" +#include "shm.h" #include "syzygy/tbprobe.h" #include "types.h" #include "uci.h" @@ -57,11 +59,14 @@ Engine::Engine(std::optional path) : threads(), networks( numaContext, - NN::Networks( - NN::NetworkBig({EvalFileDefaultNameBig, "None", ""}, NN::EmbeddedNNUEType::BIG), - NN::NetworkSmall({EvalFileDefaultNameSmall, "None", ""}, NN::EmbeddedNNUEType::SMALL))) { - pos.set(StartFEN, false, &states->back()); + // Heap-allocate because sizeof(NN::Networks) is large + std::make_unique( + std::make_unique(NN::EvalFile{EvalFileDefaultNameBig, "None", ""}, + NN::EmbeddedNNUEType::BIG), + std::make_unique(NN::EvalFile{EvalFileDefaultNameSmall, "None", ""}, + NN::EmbeddedNNUEType::SMALL))) { + pos.set(StartFEN, false, &states->back()); options.add( // "Debug Log File", Option("", [](const Option& o) { @@ -254,6 +259,36 @@ void Engine::set_ponderhit(bool b) { threads.main_manager()->ponder = b; } void Engine::verify_networks() const { networks->big.verify(options["EvalFile"], onVerifyNetworks); networks->small.verify(options["EvalFileSmall"], onVerifyNetworks); + + auto statuses = networks.get_status_and_errors(); + for (size_t i = 0; i < statuses.size(); ++i) + { + const auto [status, error] = statuses[i]; + std::string message = "Network replica " + std::to_string(i + 1) + ": "; + if (status == SystemWideSharedConstantAllocationStatus::NoAllocation) + { + message += "No allocation."; + } + else if (status == SystemWideSharedConstantAllocationStatus::LocalMemory) + { + message += "Local memory."; + } + else if (status == SystemWideSharedConstantAllocationStatus::SharedMemory) + { + message += "Shared memory."; + } + else + { + message += "Unknown status."; + } + + if (error.has_value()) + { + message += " " + *error; + } + + onVerifyNetworks(message); + } } void Engine::load_networks() { diff --git a/src/engine.h b/src/engine.h index d26844f4c..7315b8881 100644 --- a/src/engine.h +++ b/src/engine.h @@ -115,10 +115,10 @@ class Engine { Position pos; StateListPtr states; - OptionsMap options; - ThreadPool threads; - TranspositionTable tt; - LazyNumaReplicated networks; + OptionsMap options; + ThreadPool threads; + TranspositionTable tt; + LazyNumaReplicatedSystemWide networks; Search::SearchManager::UpdateContext updateContext; std::function onVerifyNetworks; diff --git a/src/evaluate.cpp b/src/evaluate.cpp index 23f4b8c2b..23bc70d0e 100644 --- a/src/evaluate.cpp +++ b/src/evaluate.cpp @@ -98,8 +98,8 @@ std::string Eval::trace(Position& pos, const Eval::NNUE::Networks& networks) { if (pos.checkers()) return "Final evaluation: none (in check)"; - Eval::NNUE::AccumulatorStack accumulators; - auto caches = std::make_unique(networks); + auto accumulators = std::make_unique(); + auto caches = std::make_unique(networks); std::stringstream ss; ss << std::showpoint << std::noshowpos << std::fixed << std::setprecision(2); @@ -107,12 +107,12 @@ std::string Eval::trace(Position& pos, const Eval::NNUE::Networks& networks) { ss << std::showpoint << std::showpos << std::fixed << std::setprecision(2) << std::setw(15); - auto [psqt, positional] = networks.big.evaluate(pos, accumulators, &caches->big); + auto [psqt, positional] = networks.big.evaluate(pos, *accumulators, &caches->big); Value v = psqt + positional; v = pos.side_to_move() == WHITE ? v : -v; ss << "NNUE evaluation " << 0.01 * UCIEngine::to_cp(v, pos) << " (white side)\n"; - v = evaluate(networks, pos, accumulators, *caches, VALUE_ZERO); + v = evaluate(networks, pos, *accumulators, *caches, VALUE_ZERO); v = pos.side_to_move() == WHITE ? v : -v; ss << "Final evaluation " << 0.01 * UCIEngine::to_cp(v, pos) << " (white side)"; ss << " [with scaled NNUE, ...]"; diff --git a/src/main.cpp b/src/main.cpp index e262f3875..9988680aa 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -17,28 +17,28 @@ */ #include +#include #include "bitboard.h" #include "misc.h" #include "position.h" +#include "tune.h" #include "types.h" #include "uci.h" -#include "tune.h" using namespace Stockfish; int main(int argc, char* argv[]) { - std::cout << engine_info() << std::endl; Bitboards::init(); Position::init(); - UCIEngine uci(argc, argv); + auto uci = std::make_unique(argc, argv); - Tune::init(uci.engine_options()); + Tune::init(uci->engine_options()); - uci.loop(); + uci->loop(); return 0; } diff --git a/src/memory.cpp b/src/memory.cpp index 92cb650f2..f4aa8fc26 100644 --- a/src/memory.cpp +++ b/src/memory.cpp @@ -55,12 +55,6 @@ // the calls at compile time), try to load them at runtime. To do this we need // first to define the corresponding function pointers. -extern "C" { -using OpenProcessToken_t = bool (*)(HANDLE, DWORD, PHANDLE); -using LookupPrivilegeValueA_t = bool (*)(LPCSTR, LPCSTR, PLUID); -using AdjustTokenPrivileges_t = - bool (*)(HANDLE, BOOL, PTOKEN_PRIVILEGES, DWORD, PTOKEN_PRIVILEGES, PDWORD); -} #endif @@ -106,77 +100,14 @@ void std_aligned_free(void* ptr) { static void* aligned_large_pages_alloc_windows([[maybe_unused]] size_t allocSize) { - #if !defined(_WIN64) - return nullptr; - #else - - HANDLE hProcessToken{}; - LUID luid{}; - void* mem = nullptr; - - const size_t largePageSize = GetLargePageMinimum(); - if (!largePageSize) - return nullptr; - - // Dynamically link OpenProcessToken, LookupPrivilegeValue and AdjustTokenPrivileges - - HMODULE hAdvapi32 = GetModuleHandle(TEXT("advapi32.dll")); - - if (!hAdvapi32) - hAdvapi32 = LoadLibrary(TEXT("advapi32.dll")); - - auto OpenProcessToken_f = - OpenProcessToken_t((void (*)()) GetProcAddress(hAdvapi32, "OpenProcessToken")); - if (!OpenProcessToken_f) - return nullptr; - auto LookupPrivilegeValueA_f = - LookupPrivilegeValueA_t((void (*)()) GetProcAddress(hAdvapi32, "LookupPrivilegeValueA")); - if (!LookupPrivilegeValueA_f) - return nullptr; - auto AdjustTokenPrivileges_f = - AdjustTokenPrivileges_t((void (*)()) GetProcAddress(hAdvapi32, "AdjustTokenPrivileges")); - if (!AdjustTokenPrivileges_f) - return nullptr; - - // We need SeLockMemoryPrivilege, so try to enable it for the process - - if (!OpenProcessToken_f( // OpenProcessToken() - GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hProcessToken)) - return nullptr; - - if (LookupPrivilegeValueA_f(nullptr, "SeLockMemoryPrivilege", &luid)) - { - TOKEN_PRIVILEGES tp{}; - TOKEN_PRIVILEGES prevTp{}; - DWORD prevTpLen = 0; - - tp.PrivilegeCount = 1; - tp.Privileges[0].Luid = luid; - tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; - - // Try to enable SeLockMemoryPrivilege. Note that even if AdjustTokenPrivileges() - // succeeds, we still need to query GetLastError() to ensure that the privileges - // were actually obtained. - - if (AdjustTokenPrivileges_f(hProcessToken, FALSE, &tp, sizeof(TOKEN_PRIVILEGES), &prevTp, - &prevTpLen) - && GetLastError() == ERROR_SUCCESS) - { - // Round up size to full pages and allocate - allocSize = (allocSize + largePageSize - 1) & ~size_t(largePageSize - 1); - mem = VirtualAlloc(nullptr, allocSize, MEM_RESERVE | MEM_COMMIT | MEM_LARGE_PAGES, - PAGE_READWRITE); - - // Privilege no longer needed, restore previous state - AdjustTokenPrivileges_f(hProcessToken, FALSE, &prevTp, 0, nullptr, nullptr); - } - } - - CloseHandle(hProcessToken); - - return mem; - - #endif + return windows_try_with_large_page_priviliges( + [&](size_t largePageSize) { + // Round up size to full pages and allocate + allocSize = (allocSize + largePageSize - 1) & ~size_t(largePageSize - 1); + return VirtualAlloc(nullptr, allocSize, MEM_RESERVE | MEM_COMMIT | MEM_LARGE_PAGES, + PAGE_READWRITE); + }, + []() { return (void*) nullptr; }); } void* aligned_large_pages_alloc(size_t allocSize) { diff --git a/src/memory.h b/src/memory.h index e4a59fec5..b9be6f170 100644 --- a/src/memory.h +++ b/src/memory.h @@ -29,6 +29,29 @@ #include "types.h" +#if defined(_WIN64) + + #if _WIN32_WINNT < 0x0601 + #undef _WIN32_WINNT + #define _WIN32_WINNT 0x0601 // Force to include needed API prototypes + #endif + + #if !defined(NOMINMAX) + #define NOMINMAX + #endif + #include + + #include + +extern "C" { +using OpenProcessToken_t = bool (*)(HANDLE, DWORD, PHANDLE); +using LookupPrivilegeValueA_t = bool (*)(LPCSTR, LPCSTR, PLUID); +using AdjustTokenPrivileges_t = + bool (*)(HANDLE, BOOL, PTOKEN_PRIVILEGES, DWORD, PTOKEN_PRIVILEGES, PDWORD); +} +#endif + + namespace Stockfish { void* std_aligned_alloc(size_t alignment, size_t size); @@ -211,6 +234,81 @@ T* align_ptr_up(T* ptr) { reinterpret_cast((ptrint + (Alignment - 1)) / Alignment * Alignment)); } +#if defined(_WIN32) + +template +auto windows_try_with_large_page_priviliges([[maybe_unused]] FuncYesT&& fyes, FuncNoT&& fno) { + + #if !defined(_WIN64) + return fno(); + #else + + HANDLE hProcessToken{}; + LUID luid{}; + + const size_t largePageSize = GetLargePageMinimum(); + if (!largePageSize) + return fno(); + + // Dynamically link OpenProcessToken, LookupPrivilegeValue and AdjustTokenPrivileges + + HMODULE hAdvapi32 = GetModuleHandle(TEXT("advapi32.dll")); + + if (!hAdvapi32) + hAdvapi32 = LoadLibrary(TEXT("advapi32.dll")); + + auto OpenProcessToken_f = + OpenProcessToken_t((void (*)()) GetProcAddress(hAdvapi32, "OpenProcessToken")); + if (!OpenProcessToken_f) + return fno(); + auto LookupPrivilegeValueA_f = + LookupPrivilegeValueA_t((void (*)()) GetProcAddress(hAdvapi32, "LookupPrivilegeValueA")); + if (!LookupPrivilegeValueA_f) + return fno(); + auto AdjustTokenPrivileges_f = + AdjustTokenPrivileges_t((void (*)()) GetProcAddress(hAdvapi32, "AdjustTokenPrivileges")); + if (!AdjustTokenPrivileges_f) + return fno(); + + // We need SeLockMemoryPrivilege, so try to enable it for the process + + if (!OpenProcessToken_f( // OpenProcessToken() + GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hProcessToken)) + return fno(); + + if (!LookupPrivilegeValueA_f(nullptr, "SeLockMemoryPrivilege", &luid)) + return fno(); + + TOKEN_PRIVILEGES tp{}; + TOKEN_PRIVILEGES prevTp{}; + DWORD prevTpLen = 0; + + tp.PrivilegeCount = 1; + tp.Privileges[0].Luid = luid; + tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; + + // Try to enable SeLockMemoryPrivilege. Note that even if AdjustTokenPrivileges() + // succeeds, we still need to query GetLastError() to ensure that the privileges + // were actually obtained. + + if (!AdjustTokenPrivileges_f(hProcessToken, FALSE, &tp, sizeof(TOKEN_PRIVILEGES), &prevTp, + &prevTpLen) + || GetLastError() != ERROR_SUCCESS) + return fno(); + + auto&& ret = fyes(largePageSize); + + // Privilege no longer needed, restore previous state + AdjustTokenPrivileges_f(hProcessToken, FALSE, &prevTp, 0, nullptr, nullptr); + + CloseHandle(hProcessToken); + + return std::forward(ret); + + #endif +} + +#endif } // namespace Stockfish diff --git a/src/misc.h b/src/misc.h index 756433290..b1707e742 100644 --- a/src/misc.h +++ b/src/misc.h @@ -23,11 +23,15 @@ #include #include #include -#include #include #include +#include // IWYU pragma: keep +// IWYU pragma: no_include <__exception/terminate.h> +#include #include #include +#include +#include #include #include #include @@ -291,6 +295,94 @@ inline uint64_t mul_hi64(uint64_t a, uint64_t b) { } +template +inline void hash_combine(std::size_t& seed, const T& v) { + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); +} + +template<> +inline void hash_combine(std::size_t& seed, const std::size_t& v) { + seed ^= v + 0x9e3779b9 + (seed << 6) + (seed >> 2); +} + +template +inline std::size_t get_raw_data_hash(const T& value) { + return std::hash{}( + std::string_view(reinterpret_cast(&value), sizeof(value))); +} + +template +class FixedString { + public: + FixedString() : + length_(0) { + data_[0] = '\0'; + } + + FixedString(const char* str) { + size_t len = std::strlen(str); + if (len > Capacity) + std::terminate(); + std::memcpy(data_, str, len); + length_ = len; + data_[length_] = '\0'; + } + + FixedString(const std::string& str) { + if (str.size() > Capacity) + std::terminate(); + std::memcpy(data_, str.data(), str.size()); + length_ = str.size(); + data_[length_] = '\0'; + } + + std::size_t size() const { return length_; } + std::size_t capacity() const { return Capacity; } + + const char* c_str() const { return data_; } + const char* data() const { return data_; } + + char& operator[](std::size_t i) { return data_[i]; } + + const char& operator[](std::size_t i) const { return data_[i]; } + + FixedString& operator+=(const char* str) { + size_t len = std::strlen(str); + if (length_ + len > Capacity) + std::terminate(); + std::memcpy(data_ + length_, str, len); + length_ += len; + data_[length_] = '\0'; + return *this; + } + + FixedString& operator+=(const FixedString& other) { return (*this += other.c_str()); } + + operator std::string() const { return std::string(data_, length_); } + + operator std::string_view() const { return std::string_view(data_, length_); } + + template + bool operator==(const T& other) const noexcept { + return (std::string_view) (*this) == other; + } + + template + bool operator!=(const T& other) const noexcept { + return (std::string_view) (*this) != other; + } + + void clear() { + length_ = 0; + data_[0] = '\0'; + } + + private: + char data_[Capacity + 1]; // +1 for null terminator + std::size_t length_; +}; + struct CommandLine { public: CommandLine(int _argc, char** _argv) : @@ -335,4 +427,11 @@ void move_to_front(std::vector& vec, Predicate pred) { } // namespace Stockfish +template +struct std::hash> { + std::size_t operator()(const Stockfish::FixedString& fstr) const noexcept { + return std::hash{}((std::string_view) fstr); + } +}; + #endif // #ifndef MISC_H_INCLUDED diff --git a/src/nnue/layers/affine_transform.h b/src/nnue/layers/affine_transform.h index b4440c1e3..7ead09327 100644 --- a/src/nnue/layers/affine_transform.h +++ b/src/nnue/layers/affine_transform.h @@ -181,6 +181,15 @@ class AffineTransform { return !stream.fail(); } + + std::size_t get_content_hash() const { + std::size_t h = 0; + hash_combine(h, get_raw_data_hash(biases)); + hash_combine(h, get_raw_data_hash(weights)); + hash_combine(h, get_hash_value(0)); + return h; + } + // Forward propagation void propagate(const InputType* input, OutputType* output) const { diff --git a/src/nnue/layers/affine_transform_sparse_input.h b/src/nnue/layers/affine_transform_sparse_input.h index 472da834f..85c0dbfec 100644 --- a/src/nnue/layers/affine_transform_sparse_input.h +++ b/src/nnue/layers/affine_transform_sparse_input.h @@ -241,6 +241,15 @@ class AffineTransformSparseInput { return !stream.fail(); } + + std::size_t get_content_hash() const { + std::size_t h = 0; + hash_combine(h, get_raw_data_hash(biases)); + hash_combine(h, get_raw_data_hash(weights)); + hash_combine(h, get_hash_value(0)); + return h; + } + // Forward propagation void propagate(const InputType* input, OutputType* output) const { diff --git a/src/nnue/layers/clipped_relu.h b/src/nnue/layers/clipped_relu.h index 2ad5a86a1..a8679d14d 100644 --- a/src/nnue/layers/clipped_relu.h +++ b/src/nnue/layers/clipped_relu.h @@ -58,6 +58,12 @@ class ClippedReLU { // Write network parameters bool write_parameters(std::ostream&) const { return true; } + std::size_t get_content_hash() const { + std::size_t h = 0; + hash_combine(h, get_hash_value(0)); + return h; + } + // Forward propagation void propagate(const InputType* input, OutputType* output) const { diff --git a/src/nnue/layers/sqr_clipped_relu.h b/src/nnue/layers/sqr_clipped_relu.h index d14f1e0ab..4218c0fe2 100644 --- a/src/nnue/layers/sqr_clipped_relu.h +++ b/src/nnue/layers/sqr_clipped_relu.h @@ -58,6 +58,12 @@ class SqrClippedReLU { // Write network parameters bool write_parameters(std::ostream&) const { return true; } + std::size_t get_content_hash() const { + std::size_t h = 0; + hash_combine(h, get_hash_value(0)); + return h; + } + // Forward propagation void propagate(const InputType* input, OutputType* output) const { diff --git a/src/nnue/network.cpp b/src/nnue/network.cpp index 957dc7bff..48aeafdf6 100644 --- a/src/nnue/network.cpp +++ b/src/nnue/network.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -30,7 +29,6 @@ #include "../incbin/incbin.h" #include "../evaluate.h" -#include "../memory.h" #include "../misc.h" #include "../position.h" #include "../types.h" @@ -101,7 +99,7 @@ bool read_parameters(std::istream& stream, T& reference) { // Write evaluation function parameters template -bool write_parameters(std::ostream& stream, T& reference) { +bool write_parameters(std::ostream& stream, const T& reference) { write_little_endian(stream, T::get_hash_value()); return reference.write_parameters(stream); @@ -109,43 +107,6 @@ bool write_parameters(std::ostream& stream, T& reference) { } // namespace Detail -template -Network::Network(const Network& other) : - evalFile(other.evalFile), - embeddedType(other.embeddedType) { - - if (other.featureTransformer) - featureTransformer = make_unique_large_page(*other.featureTransformer); - - network = make_unique_aligned(LayerStacks); - - if (!other.network) - return; - - for (std::size_t i = 0; i < LayerStacks; ++i) - network[i] = other.network[i]; -} - -template -Network& -Network::operator=(const Network& other) { - evalFile = other.evalFile; - embeddedType = other.embeddedType; - - if (other.featureTransformer) - featureTransformer = make_unique_large_page(*other.featureTransformer); - - network = make_unique_aligned(LayerStacks); - - if (!other.network) - return *this; - - for (std::size_t i = 0; i < LayerStacks; ++i) - network[i] = other.network[i]; - - return *this; -} - template void Network::load(const std::string& rootDirectory, std::string evalfilePath) { #if defined(DEFAULT_NNUE_DIRECTORY) @@ -160,14 +121,14 @@ void Network::load(const std::string& rootDirectory, std::str for (const auto& directory : dirs) { - if (evalFile.current != evalfilePath) + if (std::string(evalFile.current) != evalfilePath) { if (directory != "") { load_user_net(directory, evalfilePath); } - if (directory == "" && evalfilePath == evalFile.defaultName) + if (directory == "" && evalfilePath == std::string(evalFile.defaultName)) { load_internal(); } @@ -185,7 +146,7 @@ bool Network::save(const std::optional& filename actualFilename = filename.value(); else { - if (evalFile.current != evalFile.defaultName) + if (std::string(evalFile.current) != std::string(evalFile.defaultName)) { msg = "Failed to export a net. " "A non-embedded net can only be saved if the filename is specified"; @@ -222,7 +183,7 @@ Network::evaluate(const Position& pos const int bucket = (pos.count() - 1) / 4; const auto psqt = - featureTransformer->transform(pos, accumulatorStack, cache, transformedFeatures, bucket); + featureTransformer.transform(pos, accumulatorStack, cache, transformedFeatures, bucket); const auto positional = network[bucket].propagate(transformedFeatures); return {static_cast(psqt / OutputScale), static_cast(positional / OutputScale)}; } @@ -234,7 +195,7 @@ void Network::verify(std::string if (evalfilePath.empty()) evalfilePath = evalFile.defaultName; - if (evalFile.current != evalfilePath) + if (std::string(evalFile.current) != evalfilePath) { if (f) { @@ -245,7 +206,7 @@ void Network::verify(std::string "including the directory name, to the network file."; std::string msg4 = "The default net can be downloaded from: " "https://tests.stockfishchess.org/api/nn/" - + evalFile.defaultName; + + std::string(evalFile.defaultName); std::string msg5 = "The engine will be terminated now."; std::string msg = "ERROR: " + msg1 + '\n' + "ERROR: " + msg2 + '\n' + "ERROR: " + msg3 @@ -259,9 +220,9 @@ void Network::verify(std::string if (f) { - size_t size = sizeof(*featureTransformer) + sizeof(Arch) * LayerStacks; + size_t size = sizeof(featureTransformer) + sizeof(Arch) * LayerStacks; f("NNUE evaluation using " + evalfilePath + " (" + std::to_string(size / (1024 * 1024)) - + "MiB, (" + std::to_string(featureTransformer->InputDimensions) + ", " + + "MiB, (" + std::to_string(featureTransformer.InputDimensions) + ", " + std::to_string(network[0].TransformedFeatureDimensions) + ", " + std::to_string(network[0].FC_0_OUTPUTS) + ", " + std::to_string(network[0].FC_1_OUTPUTS) + ", 1))"); @@ -287,7 +248,7 @@ Network::trace_evaluate(const Position& for (IndexType bucket = 0; bucket < LayerStacks; ++bucket) { const auto materialist = - featureTransformer->transform(pos, accumulatorStack, cache, transformedFeatures, bucket); + featureTransformer.transform(pos, accumulatorStack, cache, transformedFeatures, bucket); const auto positional = network[bucket].propagate(transformedFeatures); t.psqt[bucket] = static_cast(materialist / OutputScale); @@ -341,8 +302,7 @@ void Network::load_internal() { template void Network::initialize() { - featureTransformer = make_unique_large_page(); - network = make_unique_aligned(LayerStacks); + initialized = true; } @@ -366,6 +326,20 @@ std::optional Network::load(std::istream& stream } +template +std::size_t Network::get_content_hash() const { + if (!initialized) + return 0; + + std::size_t h = 0; + hash_combine(h, featureTransformer); + for (auto&& layerstack : network) + hash_combine(h, layerstack); + hash_combine(h, evalFile); + hash_combine(h, static_cast(embeddedType)); + return h; +} + // Read network header template bool Network::read_header(std::istream& stream, @@ -399,13 +373,13 @@ bool Network::write_header(std::ostream& stream, template bool Network::read_parameters(std::istream& stream, - std::string& netDescription) const { + std::string& netDescription) { std::uint32_t hashValue; if (!read_header(stream, &hashValue, &netDescription)) return false; if (hashValue != Network::hash) return false; - if (!Detail::read_parameters(stream, *featureTransformer)) + if (!Detail::read_parameters(stream, featureTransformer)) return false; for (std::size_t i = 0; i < LayerStacks; ++i) { @@ -421,7 +395,7 @@ bool Network::write_parameters(std::ostream& stream, const std::string& netDescription) const { if (!write_header(stream, Network::hash, netDescription)) return false; - if (!Detail::write_parameters(stream, *featureTransformer)) + if (!Detail::write_parameters(stream, featureTransformer)) return false; for (std::size_t i = 0; i < LayerStacks; ++i) { diff --git a/src/nnue/network.h b/src/nnue/network.h index 6855831d5..c701ac6f5 100644 --- a/src/nnue/network.h +++ b/src/nnue/network.h @@ -19,16 +19,18 @@ #ifndef NETWORK_H_INCLUDED #define NETWORK_H_INCLUDED +#include #include #include #include +#include #include #include #include #include #include -#include "../memory.h" +#include "../misc.h" #include "../types.h" #include "nnue_accumulator.h" #include "nnue_architecture.h" @@ -49,6 +51,9 @@ enum class EmbeddedNNUEType { using NetworkOutput = std::tuple; +// The network must be a trivial type, i.e. the memory must be in-line. +// This is required to allow sharing the network via shared memory, as +// there is no way to run destructors. template class Network { static constexpr IndexType FTDimensions = Arch::TransformedFeatureDimensions; @@ -58,15 +63,17 @@ class Network { evalFile(file), embeddedType(type) {} - Network(const Network& other); - Network(Network&& other) = default; + Network(const Network& other) = default; + Network(Network&& other) = default; - Network& operator=(const Network& other); - Network& operator=(Network&& other) = default; + Network& operator=(const Network& other) = default; + Network& operator=(Network&& other) = default; void load(const std::string& rootDirectory, std::string evalfilePath); bool save(const std::optional& filename) const; + std::size_t get_content_hash() const; + NetworkOutput evaluate(const Position& pos, AccumulatorStack& accumulatorStack, AccumulatorCaches::Cache* cache) const; @@ -89,18 +96,20 @@ class Network { bool read_header(std::istream&, std::uint32_t*, std::string*) const; bool write_header(std::ostream&, std::uint32_t, const std::string&) const; - bool read_parameters(std::istream&, std::string&) const; + bool read_parameters(std::istream&, std::string&); bool write_parameters(std::ostream&, const std::string&) const; // Input feature converter - LargePagePtr featureTransformer; + Transformer featureTransformer; // Evaluation function - AlignedPtr network; + Arch network[LayerStacks]; EvalFile evalFile; EmbeddedNNUEType embeddedType; + bool initialized = false; + // Hash value of evaluation function structure static constexpr std::uint32_t hash = Transformer::get_hash_value() ^ Arch::get_hash_value(); @@ -121,9 +130,9 @@ using NetworkSmall = Network; struct Networks { - Networks(NetworkBig&& nB, NetworkSmall&& nS) : - big(std::move(nB)), - small(std::move(nS)) {} + Networks(std::unique_ptr&& nB, std::unique_ptr&& nS) : + big(std::move(*nB)), + small(std::move(*nS)) {} NetworkBig big; NetworkSmall small; @@ -132,4 +141,22 @@ struct Networks { } // namespace Stockfish +template +struct std::hash> { + std::size_t operator()( + const Stockfish::Eval::NNUE::Network& network) const noexcept { + return network.get_content_hash(); + } +}; + +template<> +struct std::hash { + std::size_t operator()(const Stockfish::Eval::NNUE::Networks& networks) const noexcept { + std::size_t h = 0; + Stockfish::hash_combine(h, networks.big); + Stockfish::hash_combine(h, networks.small); + return h; + } +}; + #endif diff --git a/src/nnue/nnue_accumulator.h b/src/nnue/nnue_accumulator.h index 7ff95b6ef..95cf74fa3 100644 --- a/src/nnue/nnue_accumulator.h +++ b/src/nnue/nnue_accumulator.h @@ -87,7 +87,7 @@ struct AccumulatorCaches { void clear(const Network& network) { for (auto& entries1D : entries) for (auto& entry : entries1D) - entry.clear(network.featureTransformer->biases); + entry.clear(network.featureTransformer.biases); } std::array& operator[](Square sq) { return entries[sq]; } diff --git a/src/nnue/nnue_architecture.h b/src/nnue/nnue_architecture.h index c020ce05b..5965f24aa 100644 --- a/src/nnue/nnue_architecture.h +++ b/src/nnue/nnue_architecture.h @@ -97,7 +97,7 @@ struct NetworkArchitecture { && fc_2.write_parameters(stream); } - std::int32_t propagate(const TransformedFeatureType* transformedFeatures) { + std::int32_t propagate(const TransformedFeatureType* transformedFeatures) const { struct alignas(CacheLineSize) Buffer { alignas(CacheLineSize) typename decltype(fc_0)::OutputBuffer fc_0_out; alignas(CacheLineSize) typename decltype(ac_sqr_0)::OutputType @@ -136,8 +136,28 @@ struct NetworkArchitecture { return outputValue; } + + std::size_t get_content_hash() const { + std::size_t h = 0; + hash_combine(h, fc_0.get_content_hash()); + hash_combine(h, ac_sqr_0.get_content_hash()); + hash_combine(h, ac_0.get_content_hash()); + hash_combine(h, fc_1.get_content_hash()); + hash_combine(h, ac_1.get_content_hash()); + hash_combine(h, fc_2.get_content_hash()); + hash_combine(h, get_hash_value()); + return h; + } }; } // namespace Stockfish::Eval::NNUE +template +struct std::hash> { + std::size_t + operator()(const Stockfish::Eval::NNUE::NetworkArchitecture& arch) const noexcept { + return arch.get_content_hash(); + } +}; + #endif // #ifndef NNUE_ARCHITECTURE_H_INCLUDED diff --git a/src/nnue/nnue_common.h b/src/nnue/nnue_common.h index 550bcaad4..3617a85eb 100644 --- a/src/nnue/nnue_common.h +++ b/src/nnue/nnue_common.h @@ -258,8 +258,8 @@ inline void write_leb_128(std::ostream& stream, const IntType* values, std::size } }; - auto write = [&](std::uint8_t byte) { - buf[buf_pos++] = byte; + auto write = [&](std::uint8_t b) { + buf[buf_pos++] = b; if (buf_pos == BUF_SIZE) flush(); }; diff --git a/src/nnue/nnue_feature_transformer.h b/src/nnue/nnue_feature_transformer.h index 37634cbc6..3439bc43c 100644 --- a/src/nnue/nnue_feature_transformer.h +++ b/src/nnue/nnue_feature_transformer.h @@ -157,20 +157,28 @@ class FeatureTransformer { } // Write network parameters - bool write_parameters(std::ostream& stream) { + bool write_parameters(std::ostream& stream) const { + std::unique_ptr copy = std::make_unique(*this); - unpermute_weights(); - scale_weights(false); + copy->unpermute_weights(); + copy->scale_weights(false); - write_leb_128(stream, biases, HalfDimensions); - write_leb_128(stream, weights, HalfDimensions * InputDimensions); - write_leb_128(stream, psqtWeights, PSQTBuckets * InputDimensions); + write_leb_128(stream, copy->biases, HalfDimensions); + write_leb_128(stream, copy->weights, HalfDimensions * InputDimensions); + write_leb_128(stream, copy->psqtWeights, PSQTBuckets * InputDimensions); - permute_weights(); - scale_weights(true); return !stream.fail(); } + std::size_t get_content_hash() const { + std::size_t h = 0; + hash_combine(h, get_raw_data_hash(biases)); + hash_combine(h, get_raw_data_hash(weights)); + hash_combine(h, get_raw_data_hash(psqtWeights)); + hash_combine(h, get_hash_value()); + return h; + } + // Convert input features std::int32_t transform(const Position& pos, AccumulatorStack& accumulatorStack, @@ -309,4 +317,14 @@ class FeatureTransformer { } // namespace Stockfish::Eval::NNUE + +template +struct std::hash> { + std::size_t + operator()(const Stockfish::Eval::NNUE::FeatureTransformer& ft) + const noexcept { + return ft.get_content_hash(); + } +}; + #endif // #ifndef NNUE_FEATURE_TRANSFORMER_H_INCLUDED diff --git a/src/nnue/nnue_misc.cpp b/src/nnue/nnue_misc.cpp index c99874076..957e3453f 100644 --- a/src/nnue/nnue_misc.cpp +++ b/src/nnue/nnue_misc.cpp @@ -120,11 +120,11 @@ trace(Position& pos, const Eval::NNUE::Networks& networks, Eval::NNUE::Accumulat format_cp_compact(value, &board[y + 2][x + 2], pos); }; - AccumulatorStack accumulators; + auto accumulators = std::make_unique(); // We estimate the value of each piece by doing a differential evaluation from // the current base eval, simulating the removal of the piece from its square. - auto [psqt, positional] = networks.big.evaluate(pos, accumulators, &caches.big); + auto [psqt, positional] = networks.big.evaluate(pos, *accumulators, &caches.big); Value base = psqt + positional; base = pos.side_to_move() == WHITE ? base : -base; @@ -139,8 +139,8 @@ trace(Position& pos, const Eval::NNUE::Networks& networks, Eval::NNUE::Accumulat { pos.remove_piece(sq); - accumulators.reset(); - std::tie(psqt, positional) = networks.big.evaluate(pos, accumulators, &caches.big); + accumulators->reset(); + std::tie(psqt, positional) = networks.big.evaluate(pos, *accumulators, &caches.big); Value eval = psqt + positional; eval = pos.side_to_move() == WHITE ? eval : -eval; v = base - eval; @@ -156,8 +156,8 @@ trace(Position& pos, const Eval::NNUE::Networks& networks, Eval::NNUE::Accumulat ss << board[row] << '\n'; ss << '\n'; - accumulators.reset(); - auto t = networks.big.trace_evaluate(pos, accumulators, &caches.big); + accumulators->reset(); + auto t = networks.big.trace_evaluate(pos, *accumulators, &caches.big); ss << " NNUE network contributions " << (pos.side_to_move() == WHITE ? "(White to move)" : "(Black to move)") << std::endl diff --git a/src/nnue/nnue_misc.h b/src/nnue/nnue_misc.h index 02212160a..7ecfd58a2 100644 --- a/src/nnue/nnue_misc.h +++ b/src/nnue/nnue_misc.h @@ -20,8 +20,10 @@ #define NNUE_MISC_H_INCLUDED #include +#include #include +#include "../misc.h" #include "../types.h" #include "nnue_architecture.h" @@ -31,17 +33,17 @@ class Position; namespace Eval::NNUE { +// EvalFile uses fixed string types because it's part of the network structure which must be trivial. struct EvalFile { // Default net name, will use one of the EvalFileDefaultName* macros defined // in evaluate.h - std::string defaultName; + FixedString<256> defaultName; // Selected net name, either via uci option or default - std::string current; + FixedString<256> current; // Net description extracted from the net file - std::string netDescription; + FixedString<256> netDescription; }; - struct NnueEvalTrace { static_assert(LayerStacks == PSQTBuckets); @@ -58,4 +60,15 @@ std::string trace(Position& pos, const Networks& networks, AccumulatorCaches& ca } // namespace Stockfish::Eval::NNUE } // namespace Stockfish +template<> +struct std::hash { + std::size_t operator()(const Stockfish::Eval::NNUE::EvalFile& evalFile) const noexcept { + std::size_t h = 0; + Stockfish::hash_combine(h, evalFile.defaultName); + Stockfish::hash_combine(h, evalFile.current); + Stockfish::hash_combine(h, evalFile.netDescription); + return h; + } +}; + #endif // #ifndef NNUE_MISC_H_INCLUDED diff --git a/src/numa.h b/src/numa.h index 5cadbc726..261b6005d 100644 --- a/src/numa.h +++ b/src/numa.h @@ -37,7 +37,7 @@ #include #include -#include "memory.h" +#include "shm.h" // We support linux very well, but we explicitly do NOT support Android, // because there is no affected systems, not worth maintaining. @@ -945,10 +945,11 @@ class NumaConfig { th.join(); } - private: std::vector> nodes; std::map nodeByCpu; - CpuIndex highestCpuIndex; + + private: + CpuIndex highestCpuIndex; bool customAffinity; @@ -1263,6 +1264,141 @@ class LazyNumaReplicated: public NumaReplicatedBase { } }; +// Utilizes shared memory. +template +class LazyNumaReplicatedSystemWide: public NumaReplicatedBase { + public: + using ReplicatorFuncType = std::function; + + LazyNumaReplicatedSystemWide(NumaReplicationContext& ctx) : + NumaReplicatedBase(ctx) { + prepare_replicate_from(std::make_unique()); + } + + LazyNumaReplicatedSystemWide(NumaReplicationContext& ctx, std::unique_ptr&& source) : + NumaReplicatedBase(ctx) { + prepare_replicate_from(std::move(source)); + } + + LazyNumaReplicatedSystemWide(const LazyNumaReplicatedSystemWide&) = delete; + LazyNumaReplicatedSystemWide(LazyNumaReplicatedSystemWide&& other) noexcept : + NumaReplicatedBase(std::move(other)), + instances(std::exchange(other.instances, {})) {} + + LazyNumaReplicatedSystemWide& operator=(const LazyNumaReplicatedSystemWide&) = delete; + LazyNumaReplicatedSystemWide& operator=(LazyNumaReplicatedSystemWide&& other) noexcept { + NumaReplicatedBase::operator=(*this, std::move(other)); + instances = std::exchange(other.instances, {}); + + return *this; + } + + LazyNumaReplicatedSystemWide& operator=(std::unique_ptr&& source) { + prepare_replicate_from(std::move(source)); + + return *this; + } + + ~LazyNumaReplicatedSystemWide() override = default; + + const T& operator[](NumaReplicatedAccessToken token) const { + assert(token.get_numa_index() < instances.size()); + ensure_present(token.get_numa_index()); + return *(instances[token.get_numa_index()]); + } + + const T& operator*() const { return *(instances[0]); } + + const T* operator->() const { return &*instances[0]; } + + std::vector>> + get_status_and_errors() const { + std::vector>> + status; + status.reserve(instances.size()); + + for (const auto& instance : instances) + { + status.emplace_back(instance.get_status(), instance.get_error_message()); + } + + return status; + } + + template + void modify_and_replicate(FuncT&& f) { + auto source = std::make_unique(*instances[0]); + std::forward(f)(*source); + prepare_replicate_from(std::move(source)); + } + + void on_numa_config_changed() override { + // Use the first one as the source. It doesn't matter which one we use, + // because they all must be identical, but the first one is guaranteed to exist. + auto source = std::make_unique(*instances[0]); + prepare_replicate_from(std::move(source)); + } + + private: + mutable std::vector> instances; + mutable std::mutex mutex; + + std::size_t get_discriminator(NumaIndex idx) const { + const NumaConfig& cfg = get_numa_config(); + const NumaConfig& cfg_sys = NumaConfig::from_system(false); + // as a descriminator, locate the hardware/system numadomain this cpuindex belongs to + CpuIndex cpu = *cfg.nodes[idx].begin(); // get a CpuIndex from NumaIndex + NumaIndex sys_idx = cfg_sys.is_cpu_assigned(cpu) ? cfg_sys.nodeByCpu.at(cpu) : 0; + std::string s = cfg_sys.to_string() + "$" + std::to_string(sys_idx); + return std::hash{}(s); + } + + void ensure_present(NumaIndex idx) const { + assert(idx < instances.size()); + + if (instances[idx] != nullptr) + return; + + assert(idx != 0); + + std::unique_lock lock(mutex); + // Check again for races. + if (instances[idx] != nullptr) + return; + + const NumaConfig& cfg = get_numa_config(); + cfg.execute_on_numa_node(idx, [this, idx]() { + instances[idx] = SystemWideSharedConstant(*instances[0], get_discriminator(idx)); + }); + } + + void prepare_replicate_from(std::unique_ptr&& source) { + instances.clear(); + + const NumaConfig& cfg = get_numa_config(); + // We just need to make sure the first instance is there. + // Note that we cannot move here as we need to reallocate the data + // on the correct NUMA node. + // Even in the case of a single NUMA node we have to copy since it's shared memory. + if (cfg.requires_memory_replication()) + { + assert(cfg.num_numa_nodes() > 0); + + cfg.execute_on_numa_node(0, [this, &source]() { + instances.emplace_back(SystemWideSharedConstant(*source, get_discriminator(0))); + }); + + // Prepare others for lazy init. + instances.resize(cfg.num_numa_nodes()); + } + else + { + assert(cfg.num_numa_nodes() == 1); + instances.emplace_back(SystemWideSharedConstant(*source, get_discriminator(0))); + } + } +}; + class NumaReplicationContext { public: NumaReplicationContext(NumaConfig&& cfg) : diff --git a/src/search.h b/src/search.h index d4bbca5c6..4c4357d3e 100644 --- a/src/search.h +++ b/src/search.h @@ -133,19 +133,19 @@ struct LimitsType { // The UCI stores the uci options, thread pool, and transposition table. // This struct is used to easily forward data to the Search::Worker class. struct SharedState { - SharedState(const OptionsMap& optionsMap, - ThreadPool& threadPool, - TranspositionTable& transpositionTable, - const LazyNumaReplicated& nets) : + SharedState(const OptionsMap& optionsMap, + ThreadPool& threadPool, + TranspositionTable& transpositionTable, + const LazyNumaReplicatedSystemWide& nets) : options(optionsMap), threads(threadPool), tt(transpositionTable), networks(nets) {} - const OptionsMap& options; - ThreadPool& threads; - TranspositionTable& tt; - const LazyNumaReplicated& networks; + const OptionsMap& options; + ThreadPool& threads; + TranspositionTable& tt; + const LazyNumaReplicatedSystemWide& networks; }; class Worker; @@ -349,10 +349,10 @@ class Worker { Tablebases::Config tbConfig; - const OptionsMap& options; - ThreadPool& threads; - TranspositionTable& tt; - const LazyNumaReplicated& networks; + const OptionsMap& options; + ThreadPool& threads; + TranspositionTable& tt; + const LazyNumaReplicatedSystemWide& networks; // Used by NNUE Eval::NNUE::AccumulatorStack accumulatorStack; diff --git a/src/shm.h b/src/shm.h new file mode 100644 index 000000000..ae5429676 --- /dev/null +++ b/src/shm.h @@ -0,0 +1,634 @@ +/* + Stockfish, a UCI chess playing engine derived from Glaurung 2.1 + Copyright (C) 2004-2025 The Stockfish developers (see AUTHORS file) + + Stockfish is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Stockfish is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#ifndef SHM_H_INCLUDED +#define SHM_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(_WIN32) && !defined(__ANDROID__) + #include "shm_linux.h" +#endif + +#if defined(__ANDROID__) + #include + #define SF_MAX_SEM_NAME_LEN NAME_MAX +#endif + +#include "types.h" + +#include "memory.h" + +#if defined(_WIN32) + + #if _WIN32_WINNT < 0x0601 + #undef _WIN32_WINNT + #define _WIN32_WINNT 0x0601 // Force to include needed API prototypes + #endif + + #if !defined(NOMINMAX) + #define NOMINMAX + #endif + #include +#else + #include + #include + #include + #include + #include + #include + #include +#endif + + +#if defined(__APPLE__) + #include + #include + +#elif defined(__sun) + #include + +#elif defined(__FreeBSD__) + #include + #include + #include + +#elif defined(__NetBSD__) || defined(__DragonFly__) || defined(__linux__) + #include + #include +#endif + + +namespace Stockfish { + +// argv[0] CANNOT be used because we need to identify the executable. +// argv[0] contains the command used to invoke it, which does not involve the full path. +// Just using a path is not fully resilient either, as the executable could +// have changed if it wasn't locked by the OS. Ideally we would hash the executable +// but it's not really that important at this point. +// If the path is longer than 4095 bytes the hash will be computed from an unspecified +// amount of bytes of the path; in particular it can a hash of an empty string. + +inline std::string getExecutablePathHash() { + char executable_path[4096] = {0}; + std::size_t path_length = 0; + +#if defined(_WIN32) + path_length = GetModuleFileNameA(NULL, executable_path, sizeof(executable_path)); + +#elif defined(__APPLE__) + uint32_t size = sizeof(executable_path); + if (_NSGetExecutablePath(executable_path, &size) == 0) + { + path_length = std::strlen(executable_path); + } + +#elif defined(__sun) // Solaris + const char* path = getexecname(); + if (path) + { + std::strncpy(executable_path, path, sizeof(executable_path) - 1); + path_length = std::strlen(executable_path); + } + +#elif defined(__FreeBSD__) + size_t size = sizeof(executable_path); + int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1}; + if (sysctl(mib, 4, executable_path, &size, NULL, 0) == 0) + { + path_length = std::strlen(executable_path); + } + +#elif defined(__NetBSD__) || defined(__DragonFly__) + ssize_t len = readlink("/proc/curproc/exe", executable_path, sizeof(executable_path) - 1); + if (len >= 0) + { + executable_path[len] = '\0'; + path_length = len; + } + +#elif defined(__linux__) + ssize_t len = readlink("/proc/self/exe", executable_path, sizeof(executable_path) - 1); + if (len >= 0) + { + executable_path[len] = '\0'; + path_length = len; + } + +#endif + + // In case of any error the path will be empty. + return std::string(executable_path, path_length); +} + +enum class SystemWideSharedConstantAllocationStatus { + NoAllocation, + LocalMemory, + SharedMemory +}; + +#if defined(_WIN32) + +inline std::string GetLastErrorAsString(DWORD error) { + //Get the error message ID, if any. + DWORD errorMessageID = error; + if (errorMessageID == 0) + { + return std::string(); //No error message has been recorded + } + + LPSTR messageBuffer = nullptr; + + //Ask Win32 to give us the string version of that message ID. + //The parameters we pass in, tell Win32 to create the buffer that holds the message for us (because we don't yet know how long the message string will be). + size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM + | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, errorMessageID, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPSTR) &messageBuffer, 0, NULL); + + //Copy the error message into a std::string. + std::string message(messageBuffer, size); + + //Free the Win32's string's buffer. + LocalFree(messageBuffer); + + return message; +} + +// Utilizes shared memory to store the value. It is deduplicated system-wide (for the single user). +template +class SharedMemoryBackend { + public: + enum class Status { + Success, + LargePageAllocationError, + FileMappingError, + MapViewError, + MutexCreateError, + MutexWaitError, + MutexReleaseError, + NotInitialized + }; + + static constexpr DWORD IS_INITIALIZED_VALUE = 1; + + SharedMemoryBackend() : + status(Status::NotInitialized) {}; + + SharedMemoryBackend(const std::string& shm_name, const T& value) : + status(Status::NotInitialized) { + + initialize(shm_name, value); + } + + bool is_valid() const { return status == Status::Success; } + + std::optional get_error_message() const { + switch (status) + { + case Status::Success : + return std::nullopt; + case Status::LargePageAllocationError : + return "Failed to allocate large page memory"; + case Status::FileMappingError : + return "Failed to create file mapping: " + last_error_message; + case Status::MapViewError : + return "Failed to map view: " + last_error_message; + case Status::MutexCreateError : + return "Failed to create mutex: " + last_error_message; + case Status::MutexWaitError : + return "Failed to wait on mutex: " + last_error_message; + case Status::MutexReleaseError : + return "Failed to release mutex: " + last_error_message; + case Status::NotInitialized : + return "Not initialized"; + default : + return "Unknown error"; + } + } + + void* get() const { return is_valid() ? pMap : nullptr; } + + ~SharedMemoryBackend() { cleanup(); } + + SharedMemoryBackend(const SharedMemoryBackend&) = delete; + SharedMemoryBackend& operator=(const SharedMemoryBackend&) = delete; + + SharedMemoryBackend(SharedMemoryBackend&& other) noexcept : + pMap(other.pMap), + hMapFile(other.hMapFile), + status(other.status), + last_error_message(std::move(other.last_error_message)) { + + other.pMap = nullptr; + other.hMapFile = 0; + other.status = Status::NotInitialized; + } + + SharedMemoryBackend& operator=(SharedMemoryBackend&& other) noexcept { + if (this != &other) + { + cleanup(); + pMap = other.pMap; + hMapFile = other.hMapFile; + status = other.status; + last_error_message = std::move(other.last_error_message); + + other.pMap = nullptr; + other.hMapFile = 0; + other.status = Status::NotInitialized; + } + return *this; + } + + SystemWideSharedConstantAllocationStatus get_status() const { + return status == Status::Success ? SystemWideSharedConstantAllocationStatus::SharedMemory + : SystemWideSharedConstantAllocationStatus::NoAllocation; + } + + private: + void initialize(const std::string& shm_name, const T& value) { + const size_t total_size = sizeof(T) + sizeof(IS_INITIALIZED_VALUE); + + // Try allocating with large pages first. + hMapFile = windows_try_with_large_page_priviliges( + [&](size_t largePageSize) { + const size_t total_size_aligned = + (total_size + largePageSize - 1) / largePageSize * largePageSize; + + #if defined(_WIN64) + DWORD total_size_low = total_size_aligned & 0xFFFFFFFFu; + DWORD total_size_high = total_size_aligned >> 32u; + #else + DWORD total_size_low = total_size_aligned; + DWORD total_size_high = 0; + #endif + + return CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, + PAGE_READWRITE | SEC_COMMIT | SEC_LARGE_PAGES, + total_size_high, total_size_low, shm_name.c_str()); + }, + []() { return (void*) nullptr; }); + + // Fallback to normal allocation if no large pages available. + if (!hMapFile) + { + hMapFile = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, + static_cast(total_size), shm_name.c_str()); + } + + if (!hMapFile) + { + const DWORD err = GetLastError(); + last_error_message = GetLastErrorAsString(err); + status = Status::FileMappingError; + return; + } + + pMap = MapViewOfFile(hMapFile, FILE_MAP_ALL_ACCESS, 0, 0, total_size); + if (!pMap) + { + const DWORD err = GetLastError(); + last_error_message = GetLastErrorAsString(err); + status = Status::MapViewError; + cleanup_partial(); + return; + } + + // Use named mutex to ensure only one initializer + std::string mutex_name = shm_name + "$mutex"; + HANDLE hMutex = CreateMutexA(NULL, FALSE, mutex_name.c_str()); + if (!hMutex) + { + const DWORD err = GetLastError(); + last_error_message = GetLastErrorAsString(err); + status = Status::MutexCreateError; + cleanup_partial(); + return; + } + + DWORD wait_result = WaitForSingleObject(hMutex, INFINITE); + if (wait_result != WAIT_OBJECT_0) + { + const DWORD err = GetLastError(); + last_error_message = GetLastErrorAsString(err); + status = Status::MutexWaitError; + CloseHandle(hMutex); + cleanup_partial(); + return; + } + + // Crucially, we place the object first to ensure alignment. + volatile DWORD* is_initialized = + std::launder(reinterpret_cast(reinterpret_cast(pMap) + sizeof(T))); + T* object = std::launder(reinterpret_cast(pMap)); + + if (*is_initialized != IS_INITIALIZED_VALUE) + { + // First time initialization, message for debug purposes + new (object) T{value}; + *is_initialized = IS_INITIALIZED_VALUE; + } + + BOOL release_result = ReleaseMutex(hMutex); + CloseHandle(hMutex); + + if (!release_result) + { + const DWORD err = GetLastError(); + last_error_message = GetLastErrorAsString(err); + status = Status::MutexReleaseError; + cleanup_partial(); + return; + } + + status = Status::Success; + } + + void cleanup_partial() { + if (pMap != nullptr) + { + UnmapViewOfFile(pMap); + pMap = nullptr; + } + if (hMapFile) + { + CloseHandle(hMapFile); + hMapFile = 0; + } + } + + void cleanup() { + if (pMap != nullptr) + { + UnmapViewOfFile(pMap); + pMap = nullptr; + } + if (hMapFile) + { + CloseHandle(hMapFile); + hMapFile = 0; + } + } + + void* pMap = nullptr; + HANDLE hMapFile = 0; + Status status = Status::NotInitialized; + std::string last_error_message; +}; + +#elif !defined(__ANDROID__) + +template +class SharedMemoryBackend { + public: + SharedMemoryBackend() = default; + + SharedMemoryBackend(const std::string& shm_name, const T& value) : + shm1(shm::create_shared(shm_name, value)) {} + + void* get() const { + const T* ptr = &shm1->get(); + return reinterpret_cast(const_cast(ptr)); + } + + bool is_valid() const { return shm1 && shm1->is_open() && shm1->is_initialized(); } + + SystemWideSharedConstantAllocationStatus get_status() const { + return is_valid() ? SystemWideSharedConstantAllocationStatus::SharedMemory + : SystemWideSharedConstantAllocationStatus::NoAllocation; + } + + std::optional get_error_message() const { + if (!shm1) + return "Shared memory not initialized"; + + if (!shm1->is_open()) + return "Shared memory is not open"; + + if (!shm1->is_initialized()) + return "Not initialized"; + + return std::nullopt; + } + + private: + std::optional> shm1; +}; + +#else + +// For systems that don't have shared memory, or support is troublesome. +// The way fallback is done is that we need a dummy backend. + +template +class SharedMemoryBackend { + public: + SharedMemoryBackend() = default; + + SharedMemoryBackend(const std::string& shm_name, const T& value) {} + + void* get() const { return nullptr; } + + bool is_valid() const { return false; } + + SystemWideSharedConstantAllocationStatus get_status() const { + return SystemWideSharedConstantAllocationStatus::NoAllocation; + } + + std::optional get_error_message() const { return "Dummy SharedMemoryBackend"; } +}; + +#endif + +template +struct SharedMemoryBackendFallback { + SharedMemoryBackendFallback() = default; + + SharedMemoryBackendFallback(const std::string&, const T& value) : + fallback_object(make_unique_large_page(value)) {} + + void* get() const { return fallback_object.get(); } + + SharedMemoryBackendFallback(const SharedMemoryBackendFallback&) = delete; + SharedMemoryBackendFallback& operator=(const SharedMemoryBackendFallback&) = delete; + + SharedMemoryBackendFallback(SharedMemoryBackendFallback&& other) noexcept : + fallback_object(std::move(other.fallback_object)) {} + + SharedMemoryBackendFallback& operator=(SharedMemoryBackendFallback&& other) noexcept { + fallback_object = std::move(other.fallback_object); + return *this; + } + + SystemWideSharedConstantAllocationStatus get_status() const { + return fallback_object == nullptr ? SystemWideSharedConstantAllocationStatus::NoAllocation + : SystemWideSharedConstantAllocationStatus::LocalMemory; + } + + std::optional get_error_message() const { + if (fallback_object == nullptr) + return "Not initialized"; + + return "Shared memory not supported by the OS. Local allocation fallback."; + } + + private: + LargePagePtr fallback_object; +}; + +// Platform-independent wrapper +template +struct SystemWideSharedConstant { + private: + static std::string createHashString(const std::string& input) { + size_t hash = std::hash{}(input); + + std::stringstream ss; + ss << std::hex << std::setfill('0') << hash; + + return ss.str(); + } + + public: + // We can't run the destructor because it may be in a completely different process. + // The object stored must also be obviously in-line but we can't check for that, other than some basic checks that cover most cases. + static_assert(std::is_trivially_destructible_v); + static_assert(std::is_trivially_move_constructible_v); + static_assert(std::is_trivially_copy_constructible_v); + + SystemWideSharedConstant() = default; + + + // Content is addressed by its hash. An additional discriminator can be added to account for differences + // that are not present in the content, for example NUMA node allocation. + SystemWideSharedConstant(const T& value, std::size_t discriminator = 0) { + std::size_t content_hash = std::hash{}(value); + std::size_t executable_hash = std::hash{}(getExecutablePathHash()); + + std::string shm_name = std::string("Local\\sf_") + std::to_string(content_hash) + "$" + + std::to_string(executable_hash) + "$" + + std::to_string(discriminator); + +#if !defined(_WIN32) + // POSIX shared memory names must start with a slash + shm_name = "/sf_" + createHashString(shm_name); + + // hash name and make sure it is not longer than SF_MAX_SEM_NAME_LEN + if (shm_name.size() > SF_MAX_SEM_NAME_LEN) + { + shm_name = shm_name.substr(0, SF_MAX_SEM_NAME_LEN - 1); + } +#endif + + SharedMemoryBackend shm_backend(shm_name, value); + + if (shm_backend.is_valid()) + { + backend = std::move(shm_backend); + } + else + { + backend = SharedMemoryBackendFallback(shm_name, value); + } + } + + SystemWideSharedConstant(const SystemWideSharedConstant&) = delete; + SystemWideSharedConstant& operator=(const SystemWideSharedConstant&) = delete; + + SystemWideSharedConstant(SystemWideSharedConstant&& other) noexcept : + backend(std::move(other.backend)) {} + + SystemWideSharedConstant& operator=(SystemWideSharedConstant&& other) noexcept { + backend = std::move(other.backend); + return *this; + } + + const T& operator*() const { return *std::launder(reinterpret_cast(get_ptr())); } + + bool operator==(std::nullptr_t) const noexcept { return get_ptr() == nullptr; } + + bool operator!=(std::nullptr_t) const noexcept { return get_ptr() != nullptr; } + + SystemWideSharedConstantAllocationStatus get_status() const { + return std::visit( + [](const auto& end) -> SystemWideSharedConstantAllocationStatus { + if constexpr (std::is_same_v, std::monostate>) + { + return SystemWideSharedConstantAllocationStatus::NoAllocation; + } + else + { + return end.get_status(); + } + }, + backend); + } + + std::optional get_error_message() const { + return std::visit( + [](const auto& end) -> std::optional { + if constexpr (std::is_same_v, std::monostate>) + { + return std::nullopt; + } + else + { + return end.get_error_message(); + } + }, + backend); + } + + private: + auto get_ptr() const { + return std::visit( + [](const auto& end) -> void* { + if constexpr (std::is_same_v, std::monostate>) + { + return nullptr; + } + else + { + return end.get(); + } + }, + backend); + } + + std::variant, SharedMemoryBackendFallback> backend; +}; + + +} // namespace Stockfish + +#endif // #ifndef SHM_H_INCLUDED diff --git a/src/shm_linux.h b/src/shm_linux.h new file mode 100644 index 000000000..a8b5404b2 --- /dev/null +++ b/src/shm_linux.h @@ -0,0 +1,658 @@ +/* + Stockfish, a UCI chess playing engine derived from Glaurung 2.1 + Copyright (C) 2004-2025 The Stockfish developers (see AUTHORS file) + + Stockfish is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Stockfish is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#ifndef SHM_LINUX_H_INCLUDED +#define SHM_LINUX_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#if defined(__NetBSD__) || defined(__DragonFly__) || defined(__linux__) + #include + #define SF_MAX_SEM_NAME_LEN NAME_MAX +#elif defined(__APPLE__) + #define SF_MAX_SEM_NAME_LEN 31 +#else + #define SF_MAX_SEM_NAME_LEN 255 +#endif + + +namespace Stockfish::shm { + +namespace detail { + +struct ShmHeader { + static constexpr uint32_t SHM_MAGIC = 0xAD5F1A12; + pthread_mutex_t mutex; + std::atomic ref_count{0}; + std::atomic initialized{false}; + uint32_t magic = SHM_MAGIC; +}; + +class SharedMemoryBase { + public: + virtual ~SharedMemoryBase() = default; + virtual void close() noexcept = 0; + virtual const std::string& name() const noexcept = 0; +}; + +class SharedMemoryRegistry { + private: + static std::mutex registry_mutex_; + static std::unordered_set active_instances_; + + public: + static void register_instance(SharedMemoryBase* instance) { + std::scoped_lock lock(registry_mutex_); + active_instances_.insert(instance); + } + + static void unregister_instance(SharedMemoryBase* instance) { + std::scoped_lock lock(registry_mutex_); + active_instances_.erase(instance); + } + + static void cleanup_all() noexcept { + std::scoped_lock lock(registry_mutex_); + for (auto* instance : active_instances_) + instance->close(); + active_instances_.clear(); + } +}; + +inline std::mutex SharedMemoryRegistry::registry_mutex_; +inline std::unordered_set SharedMemoryRegistry::active_instances_; + +class CleanupHooks { + private: + static std::once_flag register_once_; + + static void handle_signal(int sig) noexcept { + SharedMemoryRegistry::cleanup_all(); + _Exit(128 + sig); + } + + static void register_signal_handlers() noexcept { + std::atexit([]() { SharedMemoryRegistry::cleanup_all(); }); + + constexpr int signals[] = {SIGHUP, SIGINT, SIGQUIT, SIGILL, SIGABRT, SIGFPE, + SIGSEGV, SIGTERM, SIGBUS, SIGSYS, SIGXCPU, SIGXFSZ}; + + struct sigaction sa; + sa.sa_handler = handle_signal; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + + for (int sig : signals) + sigaction(sig, &sa, nullptr); + } + + public: + static void ensure_registered() noexcept { + std::call_once(register_once_, register_signal_handlers); + } +}; + +inline std::once_flag CleanupHooks::register_once_; + + +inline int portable_fallocate(int fd, off_t offset, off_t length) { +#ifdef __APPLE__ + fstore_t store = {F_ALLOCATECONTIG, F_PEOFPOSMODE, offset, length, 0}; + int ret = fcntl(fd, F_PREALLOCATE, &store); + if (ret == -1) + { + store.fst_flags = F_ALLOCATEALL; + ret = fcntl(fd, F_PREALLOCATE, &store); + } + if (ret != -1) + ret = ftruncate(fd, offset + length); + return ret; +#else + return posix_fallocate(fd, offset, length); +#endif +} + +} // namespace detail + +template +class SharedMemory: public detail::SharedMemoryBase { + static_assert(std::is_trivially_copyable_v, "T must be trivially copyable"); + static_assert(!std::is_pointer_v, "T cannot be a pointer type"); + + private: + std::string name_; + int fd_ = -1; + void* mapped_ptr_ = nullptr; + T* data_ptr_ = nullptr; + detail::ShmHeader* header_ptr_ = nullptr; + size_t total_size_ = 0; + std::string sentinel_base_; + std::string sentinel_path_; + + static constexpr size_t calculate_total_size() noexcept { + return sizeof(T) + sizeof(detail::ShmHeader); + } + + static std::string make_sentinel_base(const std::string& name) { + uint64_t hash = std::hash{}(name); + char buf[32]; + std::snprintf(buf, sizeof(buf), "sfshm_%016" PRIx64, static_cast(hash)); + return buf; + } + + public: + explicit SharedMemory(const std::string& name) noexcept : + name_(name), + total_size_(calculate_total_size()), + sentinel_base_(make_sentinel_base(name)) {} + + ~SharedMemory() noexcept override { + detail::SharedMemoryRegistry::unregister_instance(this); + close(); + } + + SharedMemory(const SharedMemory&) = delete; + SharedMemory& operator=(const SharedMemory&) = delete; + + SharedMemory(SharedMemory&& other) noexcept : + name_(std::move(other.name_)), + fd_(other.fd_), + mapped_ptr_(other.mapped_ptr_), + data_ptr_(other.data_ptr_), + header_ptr_(other.header_ptr_), + total_size_(other.total_size_), + sentinel_base_(std::move(other.sentinel_base_)), + sentinel_path_(std::move(other.sentinel_path_)) { + + detail::SharedMemoryRegistry::unregister_instance(&other); + detail::SharedMemoryRegistry::register_instance(this); + other.reset(); + } + + SharedMemory& operator=(SharedMemory&& other) noexcept { + if (this != &other) + { + detail::SharedMemoryRegistry::unregister_instance(this); + close(); + + name_ = std::move(other.name_); + fd_ = other.fd_; + mapped_ptr_ = other.mapped_ptr_; + data_ptr_ = other.data_ptr_; + header_ptr_ = other.header_ptr_; + total_size_ = other.total_size_; + sentinel_base_ = std::move(other.sentinel_base_); + sentinel_path_ = std::move(other.sentinel_path_); + + detail::SharedMemoryRegistry::unregister_instance(&other); + detail::SharedMemoryRegistry::register_instance(this); + + other.reset(); + } + return *this; + } + + [[nodiscard]] bool open(const T& initial_value) noexcept { + detail::CleanupHooks::ensure_registered(); + + bool retried_stale = false; + + while (true) + { + if (is_open()) + return false; + + bool created_new = false; + fd_ = shm_open(name_.c_str(), O_CREAT | O_EXCL | O_RDWR, 0666); + + if (fd_ == -1) + { + fd_ = shm_open(name_.c_str(), O_RDWR, 0666); + if (fd_ == -1) + return false; + } + else + created_new = true; + + if (!lock_file(LOCK_EX)) + { + ::close(fd_); + reset(); + return false; + } + + bool invalid_header = false; + bool success = + created_new ? setup_new_region(initial_value) : setup_existing_region(invalid_header); + + if (!success) + { + if (created_new || invalid_header) + shm_unlink(name_.c_str()); + if (mapped_ptr_) + unmap_region(); + unlock_file(); + ::close(fd_); + reset(); + + if (!created_new && invalid_header && !retried_stale) + { + retried_stale = true; + continue; + } + return false; + } + + if (!lock_shared_mutex()) + { + if (created_new) + shm_unlink(name_.c_str()); + if (mapped_ptr_) + unmap_region(); + unlock_file(); + ::close(fd_); + reset(); + + if (!created_new && !retried_stale) + { + retried_stale = true; + continue; + } + return false; + } + + if (!create_sentinel_file_locked()) + { + unlock_shared_mutex(); + unmap_region(); + if (created_new) + shm_unlink(name_.c_str()); + unlock_file(); + ::close(fd_); + reset(); + return false; + } + + header_ptr_->ref_count.fetch_add(1, std::memory_order_acq_rel); + + unlock_shared_mutex(); + unlock_file(); + detail::SharedMemoryRegistry::register_instance(this); + return true; + } + } + + void close() noexcept override { + if (fd_ == -1 && mapped_ptr_ == nullptr) + return; + + bool remove_region = false; + bool file_locked = lock_file(LOCK_EX); + bool mutex_locked = false; + + if (file_locked && header_ptr_ != nullptr) + mutex_locked = lock_shared_mutex(); + + if (mutex_locked) + { + if (header_ptr_) + { + header_ptr_->ref_count.fetch_sub(1, std::memory_order_acq_rel); + } + remove_sentinel_file(); + remove_region = !has_other_live_sentinels_locked(); + unlock_shared_mutex(); + } + else + { + remove_sentinel_file(); + decrement_refcount_relaxed(); + } + + unmap_region(); + + if (remove_region) + shm_unlink(name_.c_str()); + + if (file_locked) + unlock_file(); + + if (fd_ != -1) + { + ::close(fd_); + fd_ = -1; + } + + reset(); + } + + const std::string& name() const noexcept override { return name_; } + + [[nodiscard]] bool is_open() const noexcept { return fd_ != -1 && mapped_ptr_ && data_ptr_; } + + [[nodiscard]] const T& get() const noexcept { return *data_ptr_; } + + [[nodiscard]] const T* operator->() const noexcept { return data_ptr_; } + + [[nodiscard]] const T& operator*() const noexcept { return *data_ptr_; } + + [[nodiscard]] uint32_t ref_count() const noexcept { + return header_ptr_ ? header_ptr_->ref_count.load(std::memory_order_acquire) : 0; + } + + [[nodiscard]] bool is_initialized() const noexcept { + return header_ptr_ ? header_ptr_->initialized.load(std::memory_order_acquire) : false; + } + + static void cleanup_all_instances() noexcept { detail::SharedMemoryRegistry::cleanup_all(); } + + private: + void reset() noexcept { + fd_ = -1; + mapped_ptr_ = nullptr; + data_ptr_ = nullptr; + header_ptr_ = nullptr; + sentinel_path_.clear(); + } + + void unmap_region() noexcept { + if (mapped_ptr_) + { + munmap(mapped_ptr_, total_size_); + mapped_ptr_ = nullptr; + data_ptr_ = nullptr; + header_ptr_ = nullptr; + } + } + + [[nodiscard]] bool lock_file(int operation) noexcept { + if (fd_ == -1) + return false; + + while (flock(fd_, operation) == -1) + { + if (errno == EINTR) + continue; + return false; + } + return true; + } + + void unlock_file() noexcept { + if (fd_ == -1) + return; + + while (flock(fd_, LOCK_UN) == -1) + { + if (errno == EINTR) + continue; + break; + } + } + + std::string sentinel_full_path(pid_t pid) const { + std::string path = "/dev/shm/"; + path += sentinel_base_; + path.push_back('.'); + path += std::to_string(pid); + return path; + } + + void decrement_refcount_relaxed() noexcept { + if (!header_ptr_) + return; + + uint32_t expected = header_ptr_->ref_count.load(std::memory_order_relaxed); + while (expected != 0 + && !header_ptr_->ref_count.compare_exchange_weak( + expected, expected - 1, std::memory_order_acq_rel, std::memory_order_relaxed)) + {} + } + + bool create_sentinel_file_locked() noexcept { + if (!header_ptr_) + return false; + + const pid_t self_pid = getpid(); + sentinel_path_ = sentinel_full_path(self_pid); + + for (int attempt = 0; attempt < 2; ++attempt) + { + int fd = ::open(sentinel_path_.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0600); + if (fd != -1) + { + ::close(fd); + return true; + } + + if (errno == EEXIST) + { + ::unlink(sentinel_path_.c_str()); + decrement_refcount_relaxed(); + continue; + } + + break; + } + + sentinel_path_.clear(); + return false; + } + + void remove_sentinel_file() noexcept { + if (!sentinel_path_.empty()) + { + ::unlink(sentinel_path_.c_str()); + sentinel_path_.clear(); + } + } + + static bool pid_is_alive(pid_t pid) noexcept { + if (pid <= 0) + return false; + + if (kill(pid, 0) == 0) + return true; + + return errno == EPERM; + } + + [[nodiscard]] bool initialize_shared_mutex() noexcept { + if (!header_ptr_) + return false; + + pthread_mutexattr_t attr; + if (pthread_mutexattr_init(&attr) != 0) + return false; + + bool success = pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED) == 0; +#ifdef PTHREAD_MUTEX_ROBUST + if (success) + success = pthread_mutexattr_setrobust(&attr, PTHREAD_MUTEX_ROBUST) == 0; +#endif + + if (success) + success = pthread_mutex_init(&header_ptr_->mutex, &attr) == 0; + + pthread_mutexattr_destroy(&attr); + return success; + } + + [[nodiscard]] bool lock_shared_mutex() noexcept { + if (!header_ptr_) + return false; + + while (true) + { + int rc = pthread_mutex_lock(&header_ptr_->mutex); + if (rc == 0) + return true; + +#ifdef PTHREAD_MUTEX_ROBUST + if (rc == EOWNERDEAD) + { + if (pthread_mutex_consistent(&header_ptr_->mutex) == 0) + return true; + return false; + } +#endif + + if (rc == EINTR) + continue; + + return false; + } + } + + void unlock_shared_mutex() noexcept { + if (header_ptr_) + pthread_mutex_unlock(&header_ptr_->mutex); + } + + bool has_other_live_sentinels_locked() const noexcept { + DIR* dir = opendir("/dev/shm"); + if (!dir) + return false; + + std::string prefix = sentinel_base_ + "."; + bool found = false; + + while (dirent* entry = readdir(dir)) + { + std::string name = entry->d_name; + if (name.rfind(prefix, 0) != 0) + continue; + + auto pid_str = name.substr(prefix.size()); + char* end = nullptr; + long value = std::strtol(pid_str.c_str(), &end, 10); + if (!end || *end != '\0') + continue; + + pid_t pid = static_cast(value); + if (pid_is_alive(pid)) + { + found = true; + break; + } + + std::string stale_path = std::string("/dev/shm/") + name; + ::unlink(stale_path.c_str()); + const_cast(this)->decrement_refcount_relaxed(); + } + + closedir(dir); + return found; + } + + [[nodiscard]] bool setup_new_region(const T& initial_value) noexcept { + if (ftruncate(fd_, static_cast(total_size_)) == -1) + return false; + + if (detail::portable_fallocate(fd_, 0, static_cast(total_size_)) != 0) + return false; + + mapped_ptr_ = mmap(nullptr, total_size_, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0); + if (mapped_ptr_ == MAP_FAILED) + { + mapped_ptr_ = nullptr; + return false; + } + + data_ptr_ = static_cast(mapped_ptr_); + header_ptr_ = + reinterpret_cast(static_cast(mapped_ptr_) + sizeof(T)); + + new (header_ptr_) detail::ShmHeader{}; + new (data_ptr_) T{initial_value}; + + if (!initialize_shared_mutex()) + return false; + + header_ptr_->ref_count.store(0, std::memory_order_release); + header_ptr_->initialized.store(true, std::memory_order_release); + return true; + } + + [[nodiscard]] bool setup_existing_region(bool& invalid_header) noexcept { + invalid_header = false; + + struct stat st; + fstat(fd_, &st); + if (static_cast(st.st_size) < total_size_) + { + invalid_header = true; + return false; + } + + mapped_ptr_ = mmap(nullptr, total_size_, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0); + if (mapped_ptr_ == MAP_FAILED) + { + mapped_ptr_ = nullptr; + return false; + } + + data_ptr_ = static_cast(mapped_ptr_); + header_ptr_ = std::launder( + reinterpret_cast(static_cast(mapped_ptr_) + sizeof(T))); + + if (!header_ptr_->initialized.load(std::memory_order_acquire) + || header_ptr_->magic != detail::ShmHeader::SHM_MAGIC) + { + invalid_header = true; + unmap_region(); + return false; + } + + return true; + } +}; + +template +[[nodiscard]] std::optional> create_shared(const std::string& name, + const T& initial_value) noexcept { + SharedMemory shm(name); + if (shm.open(initial_value)) + return shm; + return std::nullopt; +} + +} // namespace Stockfish::shm + +#endif // #ifndef SHM_LINUX_H_INCLUDED