diff --git a/src/Makefile b/src/Makefile index 7244f7040..b30e16c07 100644 --- a/src/Makefile +++ b/src/Makefile @@ -435,7 +435,7 @@ endif ifeq ($(COMP),gcc) comp=gcc CXX=g++ - CXXFLAGS += -pedantic -Wextra -Wshadow -Wmissing-declarations + CXXFLAGS += -pedantic -Wextra -Wshadow -Wmissing-declarations -Wstack-usage=128000 ifeq ($(arch),$(filter $(arch),armv7 armv8 riscv64)) ifeq ($(OS),Android) @@ -618,6 +618,19 @@ ifneq ($(comp),mingw) ifneq ($(KERNEL),Haiku) ifneq ($(COMP),ndk) LDFLAGS += -lpthread + + add_lrt = yes + ifeq ($(target_windows),yes) + add_lrt = no + endif + + ifeq ($(KERNEL),Darwin) + add_lrt = no + endif + + ifeq ($(add_lrt),yes) + LDFLAGS += -lrt + endif endif endif endif @@ -628,6 +641,7 @@ ifeq ($(debug),no) CXXFLAGS += -DNDEBUG else CXXFLAGS += -g + CXXFLAGS += -D_GLIBCXX_ASSERTIONS -D_GLIBCXX_DEBUG endif ### 3.2.2 Debugging with undefined behavior sanitizers diff --git a/src/engine.cpp b/src/engine.cpp index a4c0bb1eb..9edff4864 100644 --- a/src/engine.cpp +++ b/src/engine.cpp @@ -33,10 +33,12 @@ #include "misc.h" #include "nnue/network.h" #include "nnue/nnue_common.h" +#include "nnue/nnue_misc.h" #include "numa.h" #include "perft.h" #include "position.h" #include "search.h" +#include "shm.h" #include "syzygy/tbprobe.h" #include "types.h" #include "uci.h" @@ -57,11 +59,14 @@ Engine::Engine(std::optional path) : threads(), networks( numaContext, - NN::Networks( - NN::NetworkBig({EvalFileDefaultNameBig, "None", ""}, NN::EmbeddedNNUEType::BIG), - NN::NetworkSmall({EvalFileDefaultNameSmall, "None", ""}, NN::EmbeddedNNUEType::SMALL))) { - pos.set(StartFEN, false, &states->back()); + // Heap-allocate because sizeof(NN::Networks) is large + std::make_unique( + std::make_unique(NN::EvalFile{EvalFileDefaultNameBig, "None", ""}, + NN::EmbeddedNNUEType::BIG), + std::make_unique(NN::EvalFile{EvalFileDefaultNameSmall, "None", ""}, + NN::EmbeddedNNUEType::SMALL))) { + pos.set(StartFEN, false, &states->back()); options.add( // "Debug Log File", Option("", [](const Option& o) { @@ -254,6 +259,36 @@ void Engine::set_ponderhit(bool b) { threads.main_manager()->ponder = b; } void Engine::verify_networks() const { networks->big.verify(options["EvalFile"], onVerifyNetworks); networks->small.verify(options["EvalFileSmall"], onVerifyNetworks); + + auto statuses = networks.get_status_and_errors(); + for (size_t i = 0; i < statuses.size(); ++i) + { + const auto [status, error] = statuses[i]; + std::string message = "Network replica " + std::to_string(i + 1) + ": "; + if (status == SystemWideSharedConstantAllocationStatus::NoAllocation) + { + message += "No allocation."; + } + else if (status == SystemWideSharedConstantAllocationStatus::LocalMemory) + { + message += "Local memory."; + } + else if (status == SystemWideSharedConstantAllocationStatus::SharedMemory) + { + message += "Shared memory."; + } + else + { + message += "Unknown status."; + } + + if (error.has_value()) + { + message += " " + *error; + } + + onVerifyNetworks(message); + } } void Engine::load_networks() { diff --git a/src/engine.h b/src/engine.h index d26844f4c..7315b8881 100644 --- a/src/engine.h +++ b/src/engine.h @@ -115,10 +115,10 @@ class Engine { Position pos; StateListPtr states; - OptionsMap options; - ThreadPool threads; - TranspositionTable tt; - LazyNumaReplicated networks; + OptionsMap options; + ThreadPool threads; + TranspositionTable tt; + LazyNumaReplicatedSystemWide networks; Search::SearchManager::UpdateContext updateContext; std::function onVerifyNetworks; diff --git a/src/evaluate.cpp b/src/evaluate.cpp index 23f4b8c2b..23bc70d0e 100644 --- a/src/evaluate.cpp +++ b/src/evaluate.cpp @@ -98,8 +98,8 @@ std::string Eval::trace(Position& pos, const Eval::NNUE::Networks& networks) { if (pos.checkers()) return "Final evaluation: none (in check)"; - Eval::NNUE::AccumulatorStack accumulators; - auto caches = std::make_unique(networks); + auto accumulators = std::make_unique(); + auto caches = std::make_unique(networks); std::stringstream ss; ss << std::showpoint << std::noshowpos << std::fixed << std::setprecision(2); @@ -107,12 +107,12 @@ std::string Eval::trace(Position& pos, const Eval::NNUE::Networks& networks) { ss << std::showpoint << std::showpos << std::fixed << std::setprecision(2) << std::setw(15); - auto [psqt, positional] = networks.big.evaluate(pos, accumulators, &caches->big); + auto [psqt, positional] = networks.big.evaluate(pos, *accumulators, &caches->big); Value v = psqt + positional; v = pos.side_to_move() == WHITE ? v : -v; ss << "NNUE evaluation " << 0.01 * UCIEngine::to_cp(v, pos) << " (white side)\n"; - v = evaluate(networks, pos, accumulators, *caches, VALUE_ZERO); + v = evaluate(networks, pos, *accumulators, *caches, VALUE_ZERO); v = pos.side_to_move() == WHITE ? v : -v; ss << "Final evaluation " << 0.01 * UCIEngine::to_cp(v, pos) << " (white side)"; ss << " [with scaled NNUE, ...]"; diff --git a/src/main.cpp b/src/main.cpp index e262f3875..9988680aa 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -17,28 +17,28 @@ */ #include +#include #include "bitboard.h" #include "misc.h" #include "position.h" +#include "tune.h" #include "types.h" #include "uci.h" -#include "tune.h" using namespace Stockfish; int main(int argc, char* argv[]) { - std::cout << engine_info() << std::endl; Bitboards::init(); Position::init(); - UCIEngine uci(argc, argv); + auto uci = std::make_unique(argc, argv); - Tune::init(uci.engine_options()); + Tune::init(uci->engine_options()); - uci.loop(); + uci->loop(); return 0; } diff --git a/src/memory.cpp b/src/memory.cpp index 92cb650f2..f4aa8fc26 100644 --- a/src/memory.cpp +++ b/src/memory.cpp @@ -55,12 +55,6 @@ // the calls at compile time), try to load them at runtime. To do this we need // first to define the corresponding function pointers. -extern "C" { -using OpenProcessToken_t = bool (*)(HANDLE, DWORD, PHANDLE); -using LookupPrivilegeValueA_t = bool (*)(LPCSTR, LPCSTR, PLUID); -using AdjustTokenPrivileges_t = - bool (*)(HANDLE, BOOL, PTOKEN_PRIVILEGES, DWORD, PTOKEN_PRIVILEGES, PDWORD); -} #endif @@ -106,77 +100,14 @@ void std_aligned_free(void* ptr) { static void* aligned_large_pages_alloc_windows([[maybe_unused]] size_t allocSize) { - #if !defined(_WIN64) - return nullptr; - #else - - HANDLE hProcessToken{}; - LUID luid{}; - void* mem = nullptr; - - const size_t largePageSize = GetLargePageMinimum(); - if (!largePageSize) - return nullptr; - - // Dynamically link OpenProcessToken, LookupPrivilegeValue and AdjustTokenPrivileges - - HMODULE hAdvapi32 = GetModuleHandle(TEXT("advapi32.dll")); - - if (!hAdvapi32) - hAdvapi32 = LoadLibrary(TEXT("advapi32.dll")); - - auto OpenProcessToken_f = - OpenProcessToken_t((void (*)()) GetProcAddress(hAdvapi32, "OpenProcessToken")); - if (!OpenProcessToken_f) - return nullptr; - auto LookupPrivilegeValueA_f = - LookupPrivilegeValueA_t((void (*)()) GetProcAddress(hAdvapi32, "LookupPrivilegeValueA")); - if (!LookupPrivilegeValueA_f) - return nullptr; - auto AdjustTokenPrivileges_f = - AdjustTokenPrivileges_t((void (*)()) GetProcAddress(hAdvapi32, "AdjustTokenPrivileges")); - if (!AdjustTokenPrivileges_f) - return nullptr; - - // We need SeLockMemoryPrivilege, so try to enable it for the process - - if (!OpenProcessToken_f( // OpenProcessToken() - GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hProcessToken)) - return nullptr; - - if (LookupPrivilegeValueA_f(nullptr, "SeLockMemoryPrivilege", &luid)) - { - TOKEN_PRIVILEGES tp{}; - TOKEN_PRIVILEGES prevTp{}; - DWORD prevTpLen = 0; - - tp.PrivilegeCount = 1; - tp.Privileges[0].Luid = luid; - tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; - - // Try to enable SeLockMemoryPrivilege. Note that even if AdjustTokenPrivileges() - // succeeds, we still need to query GetLastError() to ensure that the privileges - // were actually obtained. - - if (AdjustTokenPrivileges_f(hProcessToken, FALSE, &tp, sizeof(TOKEN_PRIVILEGES), &prevTp, - &prevTpLen) - && GetLastError() == ERROR_SUCCESS) - { - // Round up size to full pages and allocate - allocSize = (allocSize + largePageSize - 1) & ~size_t(largePageSize - 1); - mem = VirtualAlloc(nullptr, allocSize, MEM_RESERVE | MEM_COMMIT | MEM_LARGE_PAGES, - PAGE_READWRITE); - - // Privilege no longer needed, restore previous state - AdjustTokenPrivileges_f(hProcessToken, FALSE, &prevTp, 0, nullptr, nullptr); - } - } - - CloseHandle(hProcessToken); - - return mem; - - #endif + return windows_try_with_large_page_priviliges( + [&](size_t largePageSize) { + // Round up size to full pages and allocate + allocSize = (allocSize + largePageSize - 1) & ~size_t(largePageSize - 1); + return VirtualAlloc(nullptr, allocSize, MEM_RESERVE | MEM_COMMIT | MEM_LARGE_PAGES, + PAGE_READWRITE); + }, + []() { return (void*) nullptr; }); } void* aligned_large_pages_alloc(size_t allocSize) { diff --git a/src/memory.h b/src/memory.h index e4a59fec5..b9be6f170 100644 --- a/src/memory.h +++ b/src/memory.h @@ -29,6 +29,29 @@ #include "types.h" +#if defined(_WIN64) + + #if _WIN32_WINNT < 0x0601 + #undef _WIN32_WINNT + #define _WIN32_WINNT 0x0601 // Force to include needed API prototypes + #endif + + #if !defined(NOMINMAX) + #define NOMINMAX + #endif + #include + + #include + +extern "C" { +using OpenProcessToken_t = bool (*)(HANDLE, DWORD, PHANDLE); +using LookupPrivilegeValueA_t = bool (*)(LPCSTR, LPCSTR, PLUID); +using AdjustTokenPrivileges_t = + bool (*)(HANDLE, BOOL, PTOKEN_PRIVILEGES, DWORD, PTOKEN_PRIVILEGES, PDWORD); +} +#endif + + namespace Stockfish { void* std_aligned_alloc(size_t alignment, size_t size); @@ -211,6 +234,81 @@ T* align_ptr_up(T* ptr) { reinterpret_cast((ptrint + (Alignment - 1)) / Alignment * Alignment)); } +#if defined(_WIN32) + +template +auto windows_try_with_large_page_priviliges([[maybe_unused]] FuncYesT&& fyes, FuncNoT&& fno) { + + #if !defined(_WIN64) + return fno(); + #else + + HANDLE hProcessToken{}; + LUID luid{}; + + const size_t largePageSize = GetLargePageMinimum(); + if (!largePageSize) + return fno(); + + // Dynamically link OpenProcessToken, LookupPrivilegeValue and AdjustTokenPrivileges + + HMODULE hAdvapi32 = GetModuleHandle(TEXT("advapi32.dll")); + + if (!hAdvapi32) + hAdvapi32 = LoadLibrary(TEXT("advapi32.dll")); + + auto OpenProcessToken_f = + OpenProcessToken_t((void (*)()) GetProcAddress(hAdvapi32, "OpenProcessToken")); + if (!OpenProcessToken_f) + return fno(); + auto LookupPrivilegeValueA_f = + LookupPrivilegeValueA_t((void (*)()) GetProcAddress(hAdvapi32, "LookupPrivilegeValueA")); + if (!LookupPrivilegeValueA_f) + return fno(); + auto AdjustTokenPrivileges_f = + AdjustTokenPrivileges_t((void (*)()) GetProcAddress(hAdvapi32, "AdjustTokenPrivileges")); + if (!AdjustTokenPrivileges_f) + return fno(); + + // We need SeLockMemoryPrivilege, so try to enable it for the process + + if (!OpenProcessToken_f( // OpenProcessToken() + GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hProcessToken)) + return fno(); + + if (!LookupPrivilegeValueA_f(nullptr, "SeLockMemoryPrivilege", &luid)) + return fno(); + + TOKEN_PRIVILEGES tp{}; + TOKEN_PRIVILEGES prevTp{}; + DWORD prevTpLen = 0; + + tp.PrivilegeCount = 1; + tp.Privileges[0].Luid = luid; + tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; + + // Try to enable SeLockMemoryPrivilege. Note that even if AdjustTokenPrivileges() + // succeeds, we still need to query GetLastError() to ensure that the privileges + // were actually obtained. + + if (!AdjustTokenPrivileges_f(hProcessToken, FALSE, &tp, sizeof(TOKEN_PRIVILEGES), &prevTp, + &prevTpLen) + || GetLastError() != ERROR_SUCCESS) + return fno(); + + auto&& ret = fyes(largePageSize); + + // Privilege no longer needed, restore previous state + AdjustTokenPrivileges_f(hProcessToken, FALSE, &prevTp, 0, nullptr, nullptr); + + CloseHandle(hProcessToken); + + return std::forward(ret); + + #endif +} + +#endif } // namespace Stockfish diff --git a/src/misc.h b/src/misc.h index 756433290..b1707e742 100644 --- a/src/misc.h +++ b/src/misc.h @@ -23,11 +23,15 @@ #include #include #include -#include #include #include +#include // IWYU pragma: keep +// IWYU pragma: no_include <__exception/terminate.h> +#include #include #include +#include +#include #include #include #include @@ -291,6 +295,94 @@ inline uint64_t mul_hi64(uint64_t a, uint64_t b) { } +template +inline void hash_combine(std::size_t& seed, const T& v) { + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); +} + +template<> +inline void hash_combine(std::size_t& seed, const std::size_t& v) { + seed ^= v + 0x9e3779b9 + (seed << 6) + (seed >> 2); +} + +template +inline std::size_t get_raw_data_hash(const T& value) { + return std::hash{}( + std::string_view(reinterpret_cast(&value), sizeof(value))); +} + +template +class FixedString { + public: + FixedString() : + length_(0) { + data_[0] = '\0'; + } + + FixedString(const char* str) { + size_t len = std::strlen(str); + if (len > Capacity) + std::terminate(); + std::memcpy(data_, str, len); + length_ = len; + data_[length_] = '\0'; + } + + FixedString(const std::string& str) { + if (str.size() > Capacity) + std::terminate(); + std::memcpy(data_, str.data(), str.size()); + length_ = str.size(); + data_[length_] = '\0'; + } + + std::size_t size() const { return length_; } + std::size_t capacity() const { return Capacity; } + + const char* c_str() const { return data_; } + const char* data() const { return data_; } + + char& operator[](std::size_t i) { return data_[i]; } + + const char& operator[](std::size_t i) const { return data_[i]; } + + FixedString& operator+=(const char* str) { + size_t len = std::strlen(str); + if (length_ + len > Capacity) + std::terminate(); + std::memcpy(data_ + length_, str, len); + length_ += len; + data_[length_] = '\0'; + return *this; + } + + FixedString& operator+=(const FixedString& other) { return (*this += other.c_str()); } + + operator std::string() const { return std::string(data_, length_); } + + operator std::string_view() const { return std::string_view(data_, length_); } + + template + bool operator==(const T& other) const noexcept { + return (std::string_view) (*this) == other; + } + + template + bool operator!=(const T& other) const noexcept { + return (std::string_view) (*this) != other; + } + + void clear() { + length_ = 0; + data_[0] = '\0'; + } + + private: + char data_[Capacity + 1]; // +1 for null terminator + std::size_t length_; +}; + struct CommandLine { public: CommandLine(int _argc, char** _argv) : @@ -335,4 +427,11 @@ void move_to_front(std::vector& vec, Predicate pred) { } // namespace Stockfish +template +struct std::hash> { + std::size_t operator()(const Stockfish::FixedString& fstr) const noexcept { + return std::hash{}((std::string_view) fstr); + } +}; + #endif // #ifndef MISC_H_INCLUDED diff --git a/src/nnue/layers/affine_transform.h b/src/nnue/layers/affine_transform.h index b4440c1e3..7ead09327 100644 --- a/src/nnue/layers/affine_transform.h +++ b/src/nnue/layers/affine_transform.h @@ -181,6 +181,15 @@ class AffineTransform { return !stream.fail(); } + + std::size_t get_content_hash() const { + std::size_t h = 0; + hash_combine(h, get_raw_data_hash(biases)); + hash_combine(h, get_raw_data_hash(weights)); + hash_combine(h, get_hash_value(0)); + return h; + } + // Forward propagation void propagate(const InputType* input, OutputType* output) const { diff --git a/src/nnue/layers/affine_transform_sparse_input.h b/src/nnue/layers/affine_transform_sparse_input.h index 472da834f..85c0dbfec 100644 --- a/src/nnue/layers/affine_transform_sparse_input.h +++ b/src/nnue/layers/affine_transform_sparse_input.h @@ -241,6 +241,15 @@ class AffineTransformSparseInput { return !stream.fail(); } + + std::size_t get_content_hash() const { + std::size_t h = 0; + hash_combine(h, get_raw_data_hash(biases)); + hash_combine(h, get_raw_data_hash(weights)); + hash_combine(h, get_hash_value(0)); + return h; + } + // Forward propagation void propagate(const InputType* input, OutputType* output) const { diff --git a/src/nnue/layers/clipped_relu.h b/src/nnue/layers/clipped_relu.h index 2ad5a86a1..a8679d14d 100644 --- a/src/nnue/layers/clipped_relu.h +++ b/src/nnue/layers/clipped_relu.h @@ -58,6 +58,12 @@ class ClippedReLU { // Write network parameters bool write_parameters(std::ostream&) const { return true; } + std::size_t get_content_hash() const { + std::size_t h = 0; + hash_combine(h, get_hash_value(0)); + return h; + } + // Forward propagation void propagate(const InputType* input, OutputType* output) const { diff --git a/src/nnue/layers/sqr_clipped_relu.h b/src/nnue/layers/sqr_clipped_relu.h index d14f1e0ab..4218c0fe2 100644 --- a/src/nnue/layers/sqr_clipped_relu.h +++ b/src/nnue/layers/sqr_clipped_relu.h @@ -58,6 +58,12 @@ class SqrClippedReLU { // Write network parameters bool write_parameters(std::ostream&) const { return true; } + std::size_t get_content_hash() const { + std::size_t h = 0; + hash_combine(h, get_hash_value(0)); + return h; + } + // Forward propagation void propagate(const InputType* input, OutputType* output) const { diff --git a/src/nnue/network.cpp b/src/nnue/network.cpp index 957dc7bff..48aeafdf6 100644 --- a/src/nnue/network.cpp +++ b/src/nnue/network.cpp @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -30,7 +29,6 @@ #include "../incbin/incbin.h" #include "../evaluate.h" -#include "../memory.h" #include "../misc.h" #include "../position.h" #include "../types.h" @@ -101,7 +99,7 @@ bool read_parameters(std::istream& stream, T& reference) { // Write evaluation function parameters template -bool write_parameters(std::ostream& stream, T& reference) { +bool write_parameters(std::ostream& stream, const T& reference) { write_little_endian(stream, T::get_hash_value()); return reference.write_parameters(stream); @@ -109,43 +107,6 @@ bool write_parameters(std::ostream& stream, T& reference) { } // namespace Detail -template -Network::Network(const Network& other) : - evalFile(other.evalFile), - embeddedType(other.embeddedType) { - - if (other.featureTransformer) - featureTransformer = make_unique_large_page(*other.featureTransformer); - - network = make_unique_aligned(LayerStacks); - - if (!other.network) - return; - - for (std::size_t i = 0; i < LayerStacks; ++i) - network[i] = other.network[i]; -} - -template -Network& -Network::operator=(const Network& other) { - evalFile = other.evalFile; - embeddedType = other.embeddedType; - - if (other.featureTransformer) - featureTransformer = make_unique_large_page(*other.featureTransformer); - - network = make_unique_aligned(LayerStacks); - - if (!other.network) - return *this; - - for (std::size_t i = 0; i < LayerStacks; ++i) - network[i] = other.network[i]; - - return *this; -} - template void Network::load(const std::string& rootDirectory, std::string evalfilePath) { #if defined(DEFAULT_NNUE_DIRECTORY) @@ -160,14 +121,14 @@ void Network::load(const std::string& rootDirectory, std::str for (const auto& directory : dirs) { - if (evalFile.current != evalfilePath) + if (std::string(evalFile.current) != evalfilePath) { if (directory != "") { load_user_net(directory, evalfilePath); } - if (directory == "" && evalfilePath == evalFile.defaultName) + if (directory == "" && evalfilePath == std::string(evalFile.defaultName)) { load_internal(); } @@ -185,7 +146,7 @@ bool Network::save(const std::optional& filename actualFilename = filename.value(); else { - if (evalFile.current != evalFile.defaultName) + if (std::string(evalFile.current) != std::string(evalFile.defaultName)) { msg = "Failed to export a net. " "A non-embedded net can only be saved if the filename is specified"; @@ -222,7 +183,7 @@ Network::evaluate(const Position& pos const int bucket = (pos.count() - 1) / 4; const auto psqt = - featureTransformer->transform(pos, accumulatorStack, cache, transformedFeatures, bucket); + featureTransformer.transform(pos, accumulatorStack, cache, transformedFeatures, bucket); const auto positional = network[bucket].propagate(transformedFeatures); return {static_cast(psqt / OutputScale), static_cast(positional / OutputScale)}; } @@ -234,7 +195,7 @@ void Network::verify(std::string if (evalfilePath.empty()) evalfilePath = evalFile.defaultName; - if (evalFile.current != evalfilePath) + if (std::string(evalFile.current) != evalfilePath) { if (f) { @@ -245,7 +206,7 @@ void Network::verify(std::string "including the directory name, to the network file."; std::string msg4 = "The default net can be downloaded from: " "https://tests.stockfishchess.org/api/nn/" - + evalFile.defaultName; + + std::string(evalFile.defaultName); std::string msg5 = "The engine will be terminated now."; std::string msg = "ERROR: " + msg1 + '\n' + "ERROR: " + msg2 + '\n' + "ERROR: " + msg3 @@ -259,9 +220,9 @@ void Network::verify(std::string if (f) { - size_t size = sizeof(*featureTransformer) + sizeof(Arch) * LayerStacks; + size_t size = sizeof(featureTransformer) + sizeof(Arch) * LayerStacks; f("NNUE evaluation using " + evalfilePath + " (" + std::to_string(size / (1024 * 1024)) - + "MiB, (" + std::to_string(featureTransformer->InputDimensions) + ", " + + "MiB, (" + std::to_string(featureTransformer.InputDimensions) + ", " + std::to_string(network[0].TransformedFeatureDimensions) + ", " + std::to_string(network[0].FC_0_OUTPUTS) + ", " + std::to_string(network[0].FC_1_OUTPUTS) + ", 1))"); @@ -287,7 +248,7 @@ Network::trace_evaluate(const Position& for (IndexType bucket = 0; bucket < LayerStacks; ++bucket) { const auto materialist = - featureTransformer->transform(pos, accumulatorStack, cache, transformedFeatures, bucket); + featureTransformer.transform(pos, accumulatorStack, cache, transformedFeatures, bucket); const auto positional = network[bucket].propagate(transformedFeatures); t.psqt[bucket] = static_cast(materialist / OutputScale); @@ -341,8 +302,7 @@ void Network::load_internal() { template void Network::initialize() { - featureTransformer = make_unique_large_page(); - network = make_unique_aligned(LayerStacks); + initialized = true; } @@ -366,6 +326,20 @@ std::optional Network::load(std::istream& stream } +template +std::size_t Network::get_content_hash() const { + if (!initialized) + return 0; + + std::size_t h = 0; + hash_combine(h, featureTransformer); + for (auto&& layerstack : network) + hash_combine(h, layerstack); + hash_combine(h, evalFile); + hash_combine(h, static_cast(embeddedType)); + return h; +} + // Read network header template bool Network::read_header(std::istream& stream, @@ -399,13 +373,13 @@ bool Network::write_header(std::ostream& stream, template bool Network::read_parameters(std::istream& stream, - std::string& netDescription) const { + std::string& netDescription) { std::uint32_t hashValue; if (!read_header(stream, &hashValue, &netDescription)) return false; if (hashValue != Network::hash) return false; - if (!Detail::read_parameters(stream, *featureTransformer)) + if (!Detail::read_parameters(stream, featureTransformer)) return false; for (std::size_t i = 0; i < LayerStacks; ++i) { @@ -421,7 +395,7 @@ bool Network::write_parameters(std::ostream& stream, const std::string& netDescription) const { if (!write_header(stream, Network::hash, netDescription)) return false; - if (!Detail::write_parameters(stream, *featureTransformer)) + if (!Detail::write_parameters(stream, featureTransformer)) return false; for (std::size_t i = 0; i < LayerStacks; ++i) { diff --git a/src/nnue/network.h b/src/nnue/network.h index 6855831d5..c701ac6f5 100644 --- a/src/nnue/network.h +++ b/src/nnue/network.h @@ -19,16 +19,18 @@ #ifndef NETWORK_H_INCLUDED #define NETWORK_H_INCLUDED +#include #include #include #include +#include #include #include #include #include #include -#include "../memory.h" +#include "../misc.h" #include "../types.h" #include "nnue_accumulator.h" #include "nnue_architecture.h" @@ -49,6 +51,9 @@ enum class EmbeddedNNUEType { using NetworkOutput = std::tuple; +// The network must be a trivial type, i.e. the memory must be in-line. +// This is required to allow sharing the network via shared memory, as +// there is no way to run destructors. template class Network { static constexpr IndexType FTDimensions = Arch::TransformedFeatureDimensions; @@ -58,15 +63,17 @@ class Network { evalFile(file), embeddedType(type) {} - Network(const Network& other); - Network(Network&& other) = default; + Network(const Network& other) = default; + Network(Network&& other) = default; - Network& operator=(const Network& other); - Network& operator=(Network&& other) = default; + Network& operator=(const Network& other) = default; + Network& operator=(Network&& other) = default; void load(const std::string& rootDirectory, std::string evalfilePath); bool save(const std::optional& filename) const; + std::size_t get_content_hash() const; + NetworkOutput evaluate(const Position& pos, AccumulatorStack& accumulatorStack, AccumulatorCaches::Cache* cache) const; @@ -89,18 +96,20 @@ class Network { bool read_header(std::istream&, std::uint32_t*, std::string*) const; bool write_header(std::ostream&, std::uint32_t, const std::string&) const; - bool read_parameters(std::istream&, std::string&) const; + bool read_parameters(std::istream&, std::string&); bool write_parameters(std::ostream&, const std::string&) const; // Input feature converter - LargePagePtr featureTransformer; + Transformer featureTransformer; // Evaluation function - AlignedPtr network; + Arch network[LayerStacks]; EvalFile evalFile; EmbeddedNNUEType embeddedType; + bool initialized = false; + // Hash value of evaluation function structure static constexpr std::uint32_t hash = Transformer::get_hash_value() ^ Arch::get_hash_value(); @@ -121,9 +130,9 @@ using NetworkSmall = Network; struct Networks { - Networks(NetworkBig&& nB, NetworkSmall&& nS) : - big(std::move(nB)), - small(std::move(nS)) {} + Networks(std::unique_ptr&& nB, std::unique_ptr&& nS) : + big(std::move(*nB)), + small(std::move(*nS)) {} NetworkBig big; NetworkSmall small; @@ -132,4 +141,22 @@ struct Networks { } // namespace Stockfish +template +struct std::hash> { + std::size_t operator()( + const Stockfish::Eval::NNUE::Network& network) const noexcept { + return network.get_content_hash(); + } +}; + +template<> +struct std::hash { + std::size_t operator()(const Stockfish::Eval::NNUE::Networks& networks) const noexcept { + std::size_t h = 0; + Stockfish::hash_combine(h, networks.big); + Stockfish::hash_combine(h, networks.small); + return h; + } +}; + #endif diff --git a/src/nnue/nnue_accumulator.h b/src/nnue/nnue_accumulator.h index 7ff95b6ef..95cf74fa3 100644 --- a/src/nnue/nnue_accumulator.h +++ b/src/nnue/nnue_accumulator.h @@ -87,7 +87,7 @@ struct AccumulatorCaches { void clear(const Network& network) { for (auto& entries1D : entries) for (auto& entry : entries1D) - entry.clear(network.featureTransformer->biases); + entry.clear(network.featureTransformer.biases); } std::array& operator[](Square sq) { return entries[sq]; } diff --git a/src/nnue/nnue_architecture.h b/src/nnue/nnue_architecture.h index c020ce05b..5965f24aa 100644 --- a/src/nnue/nnue_architecture.h +++ b/src/nnue/nnue_architecture.h @@ -97,7 +97,7 @@ struct NetworkArchitecture { && fc_2.write_parameters(stream); } - std::int32_t propagate(const TransformedFeatureType* transformedFeatures) { + std::int32_t propagate(const TransformedFeatureType* transformedFeatures) const { struct alignas(CacheLineSize) Buffer { alignas(CacheLineSize) typename decltype(fc_0)::OutputBuffer fc_0_out; alignas(CacheLineSize) typename decltype(ac_sqr_0)::OutputType @@ -136,8 +136,28 @@ struct NetworkArchitecture { return outputValue; } + + std::size_t get_content_hash() const { + std::size_t h = 0; + hash_combine(h, fc_0.get_content_hash()); + hash_combine(h, ac_sqr_0.get_content_hash()); + hash_combine(h, ac_0.get_content_hash()); + hash_combine(h, fc_1.get_content_hash()); + hash_combine(h, ac_1.get_content_hash()); + hash_combine(h, fc_2.get_content_hash()); + hash_combine(h, get_hash_value()); + return h; + } }; } // namespace Stockfish::Eval::NNUE +template +struct std::hash> { + std::size_t + operator()(const Stockfish::Eval::NNUE::NetworkArchitecture& arch) const noexcept { + return arch.get_content_hash(); + } +}; + #endif // #ifndef NNUE_ARCHITECTURE_H_INCLUDED diff --git a/src/nnue/nnue_common.h b/src/nnue/nnue_common.h index 550bcaad4..3617a85eb 100644 --- a/src/nnue/nnue_common.h +++ b/src/nnue/nnue_common.h @@ -258,8 +258,8 @@ inline void write_leb_128(std::ostream& stream, const IntType* values, std::size } }; - auto write = [&](std::uint8_t byte) { - buf[buf_pos++] = byte; + auto write = [&](std::uint8_t b) { + buf[buf_pos++] = b; if (buf_pos == BUF_SIZE) flush(); }; diff --git a/src/nnue/nnue_feature_transformer.h b/src/nnue/nnue_feature_transformer.h index 37634cbc6..3439bc43c 100644 --- a/src/nnue/nnue_feature_transformer.h +++ b/src/nnue/nnue_feature_transformer.h @@ -157,20 +157,28 @@ class FeatureTransformer { } // Write network parameters - bool write_parameters(std::ostream& stream) { + bool write_parameters(std::ostream& stream) const { + std::unique_ptr copy = std::make_unique(*this); - unpermute_weights(); - scale_weights(false); + copy->unpermute_weights(); + copy->scale_weights(false); - write_leb_128(stream, biases, HalfDimensions); - write_leb_128(stream, weights, HalfDimensions * InputDimensions); - write_leb_128(stream, psqtWeights, PSQTBuckets * InputDimensions); + write_leb_128(stream, copy->biases, HalfDimensions); + write_leb_128(stream, copy->weights, HalfDimensions * InputDimensions); + write_leb_128(stream, copy->psqtWeights, PSQTBuckets * InputDimensions); - permute_weights(); - scale_weights(true); return !stream.fail(); } + std::size_t get_content_hash() const { + std::size_t h = 0; + hash_combine(h, get_raw_data_hash(biases)); + hash_combine(h, get_raw_data_hash(weights)); + hash_combine(h, get_raw_data_hash(psqtWeights)); + hash_combine(h, get_hash_value()); + return h; + } + // Convert input features std::int32_t transform(const Position& pos, AccumulatorStack& accumulatorStack, @@ -309,4 +317,14 @@ class FeatureTransformer { } // namespace Stockfish::Eval::NNUE + +template +struct std::hash> { + std::size_t + operator()(const Stockfish::Eval::NNUE::FeatureTransformer& ft) + const noexcept { + return ft.get_content_hash(); + } +}; + #endif // #ifndef NNUE_FEATURE_TRANSFORMER_H_INCLUDED diff --git a/src/nnue/nnue_misc.cpp b/src/nnue/nnue_misc.cpp index c99874076..957e3453f 100644 --- a/src/nnue/nnue_misc.cpp +++ b/src/nnue/nnue_misc.cpp @@ -120,11 +120,11 @@ trace(Position& pos, const Eval::NNUE::Networks& networks, Eval::NNUE::Accumulat format_cp_compact(value, &board[y + 2][x + 2], pos); }; - AccumulatorStack accumulators; + auto accumulators = std::make_unique(); // We estimate the value of each piece by doing a differential evaluation from // the current base eval, simulating the removal of the piece from its square. - auto [psqt, positional] = networks.big.evaluate(pos, accumulators, &caches.big); + auto [psqt, positional] = networks.big.evaluate(pos, *accumulators, &caches.big); Value base = psqt + positional; base = pos.side_to_move() == WHITE ? base : -base; @@ -139,8 +139,8 @@ trace(Position& pos, const Eval::NNUE::Networks& networks, Eval::NNUE::Accumulat { pos.remove_piece(sq); - accumulators.reset(); - std::tie(psqt, positional) = networks.big.evaluate(pos, accumulators, &caches.big); + accumulators->reset(); + std::tie(psqt, positional) = networks.big.evaluate(pos, *accumulators, &caches.big); Value eval = psqt + positional; eval = pos.side_to_move() == WHITE ? eval : -eval; v = base - eval; @@ -156,8 +156,8 @@ trace(Position& pos, const Eval::NNUE::Networks& networks, Eval::NNUE::Accumulat ss << board[row] << '\n'; ss << '\n'; - accumulators.reset(); - auto t = networks.big.trace_evaluate(pos, accumulators, &caches.big); + accumulators->reset(); + auto t = networks.big.trace_evaluate(pos, *accumulators, &caches.big); ss << " NNUE network contributions " << (pos.side_to_move() == WHITE ? "(White to move)" : "(Black to move)") << std::endl diff --git a/src/nnue/nnue_misc.h b/src/nnue/nnue_misc.h index 02212160a..7ecfd58a2 100644 --- a/src/nnue/nnue_misc.h +++ b/src/nnue/nnue_misc.h @@ -20,8 +20,10 @@ #define NNUE_MISC_H_INCLUDED #include +#include #include +#include "../misc.h" #include "../types.h" #include "nnue_architecture.h" @@ -31,17 +33,17 @@ class Position; namespace Eval::NNUE { +// EvalFile uses fixed string types because it's part of the network structure which must be trivial. struct EvalFile { // Default net name, will use one of the EvalFileDefaultName* macros defined // in evaluate.h - std::string defaultName; + FixedString<256> defaultName; // Selected net name, either via uci option or default - std::string current; + FixedString<256> current; // Net description extracted from the net file - std::string netDescription; + FixedString<256> netDescription; }; - struct NnueEvalTrace { static_assert(LayerStacks == PSQTBuckets); @@ -58,4 +60,15 @@ std::string trace(Position& pos, const Networks& networks, AccumulatorCaches& ca } // namespace Stockfish::Eval::NNUE } // namespace Stockfish +template<> +struct std::hash { + std::size_t operator()(const Stockfish::Eval::NNUE::EvalFile& evalFile) const noexcept { + std::size_t h = 0; + Stockfish::hash_combine(h, evalFile.defaultName); + Stockfish::hash_combine(h, evalFile.current); + Stockfish::hash_combine(h, evalFile.netDescription); + return h; + } +}; + #endif // #ifndef NNUE_MISC_H_INCLUDED diff --git a/src/numa.h b/src/numa.h index 5cadbc726..261b6005d 100644 --- a/src/numa.h +++ b/src/numa.h @@ -37,7 +37,7 @@ #include #include -#include "memory.h" +#include "shm.h" // We support linux very well, but we explicitly do NOT support Android, // because there is no affected systems, not worth maintaining. @@ -945,10 +945,11 @@ class NumaConfig { th.join(); } - private: std::vector> nodes; std::map nodeByCpu; - CpuIndex highestCpuIndex; + + private: + CpuIndex highestCpuIndex; bool customAffinity; @@ -1263,6 +1264,141 @@ class LazyNumaReplicated: public NumaReplicatedBase { } }; +// Utilizes shared memory. +template +class LazyNumaReplicatedSystemWide: public NumaReplicatedBase { + public: + using ReplicatorFuncType = std::function; + + LazyNumaReplicatedSystemWide(NumaReplicationContext& ctx) : + NumaReplicatedBase(ctx) { + prepare_replicate_from(std::make_unique()); + } + + LazyNumaReplicatedSystemWide(NumaReplicationContext& ctx, std::unique_ptr&& source) : + NumaReplicatedBase(ctx) { + prepare_replicate_from(std::move(source)); + } + + LazyNumaReplicatedSystemWide(const LazyNumaReplicatedSystemWide&) = delete; + LazyNumaReplicatedSystemWide(LazyNumaReplicatedSystemWide&& other) noexcept : + NumaReplicatedBase(std::move(other)), + instances(std::exchange(other.instances, {})) {} + + LazyNumaReplicatedSystemWide& operator=(const LazyNumaReplicatedSystemWide&) = delete; + LazyNumaReplicatedSystemWide& operator=(LazyNumaReplicatedSystemWide&& other) noexcept { + NumaReplicatedBase::operator=(*this, std::move(other)); + instances = std::exchange(other.instances, {}); + + return *this; + } + + LazyNumaReplicatedSystemWide& operator=(std::unique_ptr&& source) { + prepare_replicate_from(std::move(source)); + + return *this; + } + + ~LazyNumaReplicatedSystemWide() override = default; + + const T& operator[](NumaReplicatedAccessToken token) const { + assert(token.get_numa_index() < instances.size()); + ensure_present(token.get_numa_index()); + return *(instances[token.get_numa_index()]); + } + + const T& operator*() const { return *(instances[0]); } + + const T* operator->() const { return &*instances[0]; } + + std::vector>> + get_status_and_errors() const { + std::vector>> + status; + status.reserve(instances.size()); + + for (const auto& instance : instances) + { + status.emplace_back(instance.get_status(), instance.get_error_message()); + } + + return status; + } + + template + void modify_and_replicate(FuncT&& f) { + auto source = std::make_unique(*instances[0]); + std::forward(f)(*source); + prepare_replicate_from(std::move(source)); + } + + void on_numa_config_changed() override { + // Use the first one as the source. It doesn't matter which one we use, + // because they all must be identical, but the first one is guaranteed to exist. + auto source = std::make_unique(*instances[0]); + prepare_replicate_from(std::move(source)); + } + + private: + mutable std::vector> instances; + mutable std::mutex mutex; + + std::size_t get_discriminator(NumaIndex idx) const { + const NumaConfig& cfg = get_numa_config(); + const NumaConfig& cfg_sys = NumaConfig::from_system(false); + // as a descriminator, locate the hardware/system numadomain this cpuindex belongs to + CpuIndex cpu = *cfg.nodes[idx].begin(); // get a CpuIndex from NumaIndex + NumaIndex sys_idx = cfg_sys.is_cpu_assigned(cpu) ? cfg_sys.nodeByCpu.at(cpu) : 0; + std::string s = cfg_sys.to_string() + "$" + std::to_string(sys_idx); + return std::hash{}(s); + } + + void ensure_present(NumaIndex idx) const { + assert(idx < instances.size()); + + if (instances[idx] != nullptr) + return; + + assert(idx != 0); + + std::unique_lock lock(mutex); + // Check again for races. + if (instances[idx] != nullptr) + return; + + const NumaConfig& cfg = get_numa_config(); + cfg.execute_on_numa_node(idx, [this, idx]() { + instances[idx] = SystemWideSharedConstant(*instances[0], get_discriminator(idx)); + }); + } + + void prepare_replicate_from(std::unique_ptr&& source) { + instances.clear(); + + const NumaConfig& cfg = get_numa_config(); + // We just need to make sure the first instance is there. + // Note that we cannot move here as we need to reallocate the data + // on the correct NUMA node. + // Even in the case of a single NUMA node we have to copy since it's shared memory. + if (cfg.requires_memory_replication()) + { + assert(cfg.num_numa_nodes() > 0); + + cfg.execute_on_numa_node(0, [this, &source]() { + instances.emplace_back(SystemWideSharedConstant(*source, get_discriminator(0))); + }); + + // Prepare others for lazy init. + instances.resize(cfg.num_numa_nodes()); + } + else + { + assert(cfg.num_numa_nodes() == 1); + instances.emplace_back(SystemWideSharedConstant(*source, get_discriminator(0))); + } + } +}; + class NumaReplicationContext { public: NumaReplicationContext(NumaConfig&& cfg) : diff --git a/src/search.h b/src/search.h index d4bbca5c6..4c4357d3e 100644 --- a/src/search.h +++ b/src/search.h @@ -133,19 +133,19 @@ struct LimitsType { // The UCI stores the uci options, thread pool, and transposition table. // This struct is used to easily forward data to the Search::Worker class. struct SharedState { - SharedState(const OptionsMap& optionsMap, - ThreadPool& threadPool, - TranspositionTable& transpositionTable, - const LazyNumaReplicated& nets) : + SharedState(const OptionsMap& optionsMap, + ThreadPool& threadPool, + TranspositionTable& transpositionTable, + const LazyNumaReplicatedSystemWide& nets) : options(optionsMap), threads(threadPool), tt(transpositionTable), networks(nets) {} - const OptionsMap& options; - ThreadPool& threads; - TranspositionTable& tt; - const LazyNumaReplicated& networks; + const OptionsMap& options; + ThreadPool& threads; + TranspositionTable& tt; + const LazyNumaReplicatedSystemWide& networks; }; class Worker; @@ -349,10 +349,10 @@ class Worker { Tablebases::Config tbConfig; - const OptionsMap& options; - ThreadPool& threads; - TranspositionTable& tt; - const LazyNumaReplicated& networks; + const OptionsMap& options; + ThreadPool& threads; + TranspositionTable& tt; + const LazyNumaReplicatedSystemWide& networks; // Used by NNUE Eval::NNUE::AccumulatorStack accumulatorStack; diff --git a/src/shm.h b/src/shm.h new file mode 100644 index 000000000..ae5429676 --- /dev/null +++ b/src/shm.h @@ -0,0 +1,634 @@ +/* + Stockfish, a UCI chess playing engine derived from Glaurung 2.1 + Copyright (C) 2004-2025 The Stockfish developers (see AUTHORS file) + + Stockfish is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Stockfish is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#ifndef SHM_H_INCLUDED +#define SHM_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if !defined(_WIN32) && !defined(__ANDROID__) + #include "shm_linux.h" +#endif + +#if defined(__ANDROID__) + #include + #define SF_MAX_SEM_NAME_LEN NAME_MAX +#endif + +#include "types.h" + +#include "memory.h" + +#if defined(_WIN32) + + #if _WIN32_WINNT < 0x0601 + #undef _WIN32_WINNT + #define _WIN32_WINNT 0x0601 // Force to include needed API prototypes + #endif + + #if !defined(NOMINMAX) + #define NOMINMAX + #endif + #include +#else + #include + #include + #include + #include + #include + #include + #include +#endif + + +#if defined(__APPLE__) + #include + #include + +#elif defined(__sun) + #include + +#elif defined(__FreeBSD__) + #include + #include + #include + +#elif defined(__NetBSD__) || defined(__DragonFly__) || defined(__linux__) + #include + #include +#endif + + +namespace Stockfish { + +// argv[0] CANNOT be used because we need to identify the executable. +// argv[0] contains the command used to invoke it, which does not involve the full path. +// Just using a path is not fully resilient either, as the executable could +// have changed if it wasn't locked by the OS. Ideally we would hash the executable +// but it's not really that important at this point. +// If the path is longer than 4095 bytes the hash will be computed from an unspecified +// amount of bytes of the path; in particular it can a hash of an empty string. + +inline std::string getExecutablePathHash() { + char executable_path[4096] = {0}; + std::size_t path_length = 0; + +#if defined(_WIN32) + path_length = GetModuleFileNameA(NULL, executable_path, sizeof(executable_path)); + +#elif defined(__APPLE__) + uint32_t size = sizeof(executable_path); + if (_NSGetExecutablePath(executable_path, &size) == 0) + { + path_length = std::strlen(executable_path); + } + +#elif defined(__sun) // Solaris + const char* path = getexecname(); + if (path) + { + std::strncpy(executable_path, path, sizeof(executable_path) - 1); + path_length = std::strlen(executable_path); + } + +#elif defined(__FreeBSD__) + size_t size = sizeof(executable_path); + int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1}; + if (sysctl(mib, 4, executable_path, &size, NULL, 0) == 0) + { + path_length = std::strlen(executable_path); + } + +#elif defined(__NetBSD__) || defined(__DragonFly__) + ssize_t len = readlink("/proc/curproc/exe", executable_path, sizeof(executable_path) - 1); + if (len >= 0) + { + executable_path[len] = '\0'; + path_length = len; + } + +#elif defined(__linux__) + ssize_t len = readlink("/proc/self/exe", executable_path, sizeof(executable_path) - 1); + if (len >= 0) + { + executable_path[len] = '\0'; + path_length = len; + } + +#endif + + // In case of any error the path will be empty. + return std::string(executable_path, path_length); +} + +enum class SystemWideSharedConstantAllocationStatus { + NoAllocation, + LocalMemory, + SharedMemory +}; + +#if defined(_WIN32) + +inline std::string GetLastErrorAsString(DWORD error) { + //Get the error message ID, if any. + DWORD errorMessageID = error; + if (errorMessageID == 0) + { + return std::string(); //No error message has been recorded + } + + LPSTR messageBuffer = nullptr; + + //Ask Win32 to give us the string version of that message ID. + //The parameters we pass in, tell Win32 to create the buffer that holds the message for us (because we don't yet know how long the message string will be). + size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM + | FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, errorMessageID, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPSTR) &messageBuffer, 0, NULL); + + //Copy the error message into a std::string. + std::string message(messageBuffer, size); + + //Free the Win32's string's buffer. + LocalFree(messageBuffer); + + return message; +} + +// Utilizes shared memory to store the value. It is deduplicated system-wide (for the single user). +template +class SharedMemoryBackend { + public: + enum class Status { + Success, + LargePageAllocationError, + FileMappingError, + MapViewError, + MutexCreateError, + MutexWaitError, + MutexReleaseError, + NotInitialized + }; + + static constexpr DWORD IS_INITIALIZED_VALUE = 1; + + SharedMemoryBackend() : + status(Status::NotInitialized) {}; + + SharedMemoryBackend(const std::string& shm_name, const T& value) : + status(Status::NotInitialized) { + + initialize(shm_name, value); + } + + bool is_valid() const { return status == Status::Success; } + + std::optional get_error_message() const { + switch (status) + { + case Status::Success : + return std::nullopt; + case Status::LargePageAllocationError : + return "Failed to allocate large page memory"; + case Status::FileMappingError : + return "Failed to create file mapping: " + last_error_message; + case Status::MapViewError : + return "Failed to map view: " + last_error_message; + case Status::MutexCreateError : + return "Failed to create mutex: " + last_error_message; + case Status::MutexWaitError : + return "Failed to wait on mutex: " + last_error_message; + case Status::MutexReleaseError : + return "Failed to release mutex: " + last_error_message; + case Status::NotInitialized : + return "Not initialized"; + default : + return "Unknown error"; + } + } + + void* get() const { return is_valid() ? pMap : nullptr; } + + ~SharedMemoryBackend() { cleanup(); } + + SharedMemoryBackend(const SharedMemoryBackend&) = delete; + SharedMemoryBackend& operator=(const SharedMemoryBackend&) = delete; + + SharedMemoryBackend(SharedMemoryBackend&& other) noexcept : + pMap(other.pMap), + hMapFile(other.hMapFile), + status(other.status), + last_error_message(std::move(other.last_error_message)) { + + other.pMap = nullptr; + other.hMapFile = 0; + other.status = Status::NotInitialized; + } + + SharedMemoryBackend& operator=(SharedMemoryBackend&& other) noexcept { + if (this != &other) + { + cleanup(); + pMap = other.pMap; + hMapFile = other.hMapFile; + status = other.status; + last_error_message = std::move(other.last_error_message); + + other.pMap = nullptr; + other.hMapFile = 0; + other.status = Status::NotInitialized; + } + return *this; + } + + SystemWideSharedConstantAllocationStatus get_status() const { + return status == Status::Success ? SystemWideSharedConstantAllocationStatus::SharedMemory + : SystemWideSharedConstantAllocationStatus::NoAllocation; + } + + private: + void initialize(const std::string& shm_name, const T& value) { + const size_t total_size = sizeof(T) + sizeof(IS_INITIALIZED_VALUE); + + // Try allocating with large pages first. + hMapFile = windows_try_with_large_page_priviliges( + [&](size_t largePageSize) { + const size_t total_size_aligned = + (total_size + largePageSize - 1) / largePageSize * largePageSize; + + #if defined(_WIN64) + DWORD total_size_low = total_size_aligned & 0xFFFFFFFFu; + DWORD total_size_high = total_size_aligned >> 32u; + #else + DWORD total_size_low = total_size_aligned; + DWORD total_size_high = 0; + #endif + + return CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, + PAGE_READWRITE | SEC_COMMIT | SEC_LARGE_PAGES, + total_size_high, total_size_low, shm_name.c_str()); + }, + []() { return (void*) nullptr; }); + + // Fallback to normal allocation if no large pages available. + if (!hMapFile) + { + hMapFile = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, + static_cast(total_size), shm_name.c_str()); + } + + if (!hMapFile) + { + const DWORD err = GetLastError(); + last_error_message = GetLastErrorAsString(err); + status = Status::FileMappingError; + return; + } + + pMap = MapViewOfFile(hMapFile, FILE_MAP_ALL_ACCESS, 0, 0, total_size); + if (!pMap) + { + const DWORD err = GetLastError(); + last_error_message = GetLastErrorAsString(err); + status = Status::MapViewError; + cleanup_partial(); + return; + } + + // Use named mutex to ensure only one initializer + std::string mutex_name = shm_name + "$mutex"; + HANDLE hMutex = CreateMutexA(NULL, FALSE, mutex_name.c_str()); + if (!hMutex) + { + const DWORD err = GetLastError(); + last_error_message = GetLastErrorAsString(err); + status = Status::MutexCreateError; + cleanup_partial(); + return; + } + + DWORD wait_result = WaitForSingleObject(hMutex, INFINITE); + if (wait_result != WAIT_OBJECT_0) + { + const DWORD err = GetLastError(); + last_error_message = GetLastErrorAsString(err); + status = Status::MutexWaitError; + CloseHandle(hMutex); + cleanup_partial(); + return; + } + + // Crucially, we place the object first to ensure alignment. + volatile DWORD* is_initialized = + std::launder(reinterpret_cast(reinterpret_cast(pMap) + sizeof(T))); + T* object = std::launder(reinterpret_cast(pMap)); + + if (*is_initialized != IS_INITIALIZED_VALUE) + { + // First time initialization, message for debug purposes + new (object) T{value}; + *is_initialized = IS_INITIALIZED_VALUE; + } + + BOOL release_result = ReleaseMutex(hMutex); + CloseHandle(hMutex); + + if (!release_result) + { + const DWORD err = GetLastError(); + last_error_message = GetLastErrorAsString(err); + status = Status::MutexReleaseError; + cleanup_partial(); + return; + } + + status = Status::Success; + } + + void cleanup_partial() { + if (pMap != nullptr) + { + UnmapViewOfFile(pMap); + pMap = nullptr; + } + if (hMapFile) + { + CloseHandle(hMapFile); + hMapFile = 0; + } + } + + void cleanup() { + if (pMap != nullptr) + { + UnmapViewOfFile(pMap); + pMap = nullptr; + } + if (hMapFile) + { + CloseHandle(hMapFile); + hMapFile = 0; + } + } + + void* pMap = nullptr; + HANDLE hMapFile = 0; + Status status = Status::NotInitialized; + std::string last_error_message; +}; + +#elif !defined(__ANDROID__) + +template +class SharedMemoryBackend { + public: + SharedMemoryBackend() = default; + + SharedMemoryBackend(const std::string& shm_name, const T& value) : + shm1(shm::create_shared(shm_name, value)) {} + + void* get() const { + const T* ptr = &shm1->get(); + return reinterpret_cast(const_cast(ptr)); + } + + bool is_valid() const { return shm1 && shm1->is_open() && shm1->is_initialized(); } + + SystemWideSharedConstantAllocationStatus get_status() const { + return is_valid() ? SystemWideSharedConstantAllocationStatus::SharedMemory + : SystemWideSharedConstantAllocationStatus::NoAllocation; + } + + std::optional get_error_message() const { + if (!shm1) + return "Shared memory not initialized"; + + if (!shm1->is_open()) + return "Shared memory is not open"; + + if (!shm1->is_initialized()) + return "Not initialized"; + + return std::nullopt; + } + + private: + std::optional> shm1; +}; + +#else + +// For systems that don't have shared memory, or support is troublesome. +// The way fallback is done is that we need a dummy backend. + +template +class SharedMemoryBackend { + public: + SharedMemoryBackend() = default; + + SharedMemoryBackend(const std::string& shm_name, const T& value) {} + + void* get() const { return nullptr; } + + bool is_valid() const { return false; } + + SystemWideSharedConstantAllocationStatus get_status() const { + return SystemWideSharedConstantAllocationStatus::NoAllocation; + } + + std::optional get_error_message() const { return "Dummy SharedMemoryBackend"; } +}; + +#endif + +template +struct SharedMemoryBackendFallback { + SharedMemoryBackendFallback() = default; + + SharedMemoryBackendFallback(const std::string&, const T& value) : + fallback_object(make_unique_large_page(value)) {} + + void* get() const { return fallback_object.get(); } + + SharedMemoryBackendFallback(const SharedMemoryBackendFallback&) = delete; + SharedMemoryBackendFallback& operator=(const SharedMemoryBackendFallback&) = delete; + + SharedMemoryBackendFallback(SharedMemoryBackendFallback&& other) noexcept : + fallback_object(std::move(other.fallback_object)) {} + + SharedMemoryBackendFallback& operator=(SharedMemoryBackendFallback&& other) noexcept { + fallback_object = std::move(other.fallback_object); + return *this; + } + + SystemWideSharedConstantAllocationStatus get_status() const { + return fallback_object == nullptr ? SystemWideSharedConstantAllocationStatus::NoAllocation + : SystemWideSharedConstantAllocationStatus::LocalMemory; + } + + std::optional get_error_message() const { + if (fallback_object == nullptr) + return "Not initialized"; + + return "Shared memory not supported by the OS. Local allocation fallback."; + } + + private: + LargePagePtr fallback_object; +}; + +// Platform-independent wrapper +template +struct SystemWideSharedConstant { + private: + static std::string createHashString(const std::string& input) { + size_t hash = std::hash{}(input); + + std::stringstream ss; + ss << std::hex << std::setfill('0') << hash; + + return ss.str(); + } + + public: + // We can't run the destructor because it may be in a completely different process. + // The object stored must also be obviously in-line but we can't check for that, other than some basic checks that cover most cases. + static_assert(std::is_trivially_destructible_v); + static_assert(std::is_trivially_move_constructible_v); + static_assert(std::is_trivially_copy_constructible_v); + + SystemWideSharedConstant() = default; + + + // Content is addressed by its hash. An additional discriminator can be added to account for differences + // that are not present in the content, for example NUMA node allocation. + SystemWideSharedConstant(const T& value, std::size_t discriminator = 0) { + std::size_t content_hash = std::hash{}(value); + std::size_t executable_hash = std::hash{}(getExecutablePathHash()); + + std::string shm_name = std::string("Local\\sf_") + std::to_string(content_hash) + "$" + + std::to_string(executable_hash) + "$" + + std::to_string(discriminator); + +#if !defined(_WIN32) + // POSIX shared memory names must start with a slash + shm_name = "/sf_" + createHashString(shm_name); + + // hash name and make sure it is not longer than SF_MAX_SEM_NAME_LEN + if (shm_name.size() > SF_MAX_SEM_NAME_LEN) + { + shm_name = shm_name.substr(0, SF_MAX_SEM_NAME_LEN - 1); + } +#endif + + SharedMemoryBackend shm_backend(shm_name, value); + + if (shm_backend.is_valid()) + { + backend = std::move(shm_backend); + } + else + { + backend = SharedMemoryBackendFallback(shm_name, value); + } + } + + SystemWideSharedConstant(const SystemWideSharedConstant&) = delete; + SystemWideSharedConstant& operator=(const SystemWideSharedConstant&) = delete; + + SystemWideSharedConstant(SystemWideSharedConstant&& other) noexcept : + backend(std::move(other.backend)) {} + + SystemWideSharedConstant& operator=(SystemWideSharedConstant&& other) noexcept { + backend = std::move(other.backend); + return *this; + } + + const T& operator*() const { return *std::launder(reinterpret_cast(get_ptr())); } + + bool operator==(std::nullptr_t) const noexcept { return get_ptr() == nullptr; } + + bool operator!=(std::nullptr_t) const noexcept { return get_ptr() != nullptr; } + + SystemWideSharedConstantAllocationStatus get_status() const { + return std::visit( + [](const auto& end) -> SystemWideSharedConstantAllocationStatus { + if constexpr (std::is_same_v, std::monostate>) + { + return SystemWideSharedConstantAllocationStatus::NoAllocation; + } + else + { + return end.get_status(); + } + }, + backend); + } + + std::optional get_error_message() const { + return std::visit( + [](const auto& end) -> std::optional { + if constexpr (std::is_same_v, std::monostate>) + { + return std::nullopt; + } + else + { + return end.get_error_message(); + } + }, + backend); + } + + private: + auto get_ptr() const { + return std::visit( + [](const auto& end) -> void* { + if constexpr (std::is_same_v, std::monostate>) + { + return nullptr; + } + else + { + return end.get(); + } + }, + backend); + } + + std::variant, SharedMemoryBackendFallback> backend; +}; + + +} // namespace Stockfish + +#endif // #ifndef SHM_H_INCLUDED diff --git a/src/shm_linux.h b/src/shm_linux.h new file mode 100644 index 000000000..a8b5404b2 --- /dev/null +++ b/src/shm_linux.h @@ -0,0 +1,658 @@ +/* + Stockfish, a UCI chess playing engine derived from Glaurung 2.1 + Copyright (C) 2004-2025 The Stockfish developers (see AUTHORS file) + + Stockfish is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Stockfish is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#ifndef SHM_LINUX_H_INCLUDED +#define SHM_LINUX_H_INCLUDED + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#if defined(__NetBSD__) || defined(__DragonFly__) || defined(__linux__) + #include + #define SF_MAX_SEM_NAME_LEN NAME_MAX +#elif defined(__APPLE__) + #define SF_MAX_SEM_NAME_LEN 31 +#else + #define SF_MAX_SEM_NAME_LEN 255 +#endif + + +namespace Stockfish::shm { + +namespace detail { + +struct ShmHeader { + static constexpr uint32_t SHM_MAGIC = 0xAD5F1A12; + pthread_mutex_t mutex; + std::atomic ref_count{0}; + std::atomic initialized{false}; + uint32_t magic = SHM_MAGIC; +}; + +class SharedMemoryBase { + public: + virtual ~SharedMemoryBase() = default; + virtual void close() noexcept = 0; + virtual const std::string& name() const noexcept = 0; +}; + +class SharedMemoryRegistry { + private: + static std::mutex registry_mutex_; + static std::unordered_set active_instances_; + + public: + static void register_instance(SharedMemoryBase* instance) { + std::scoped_lock lock(registry_mutex_); + active_instances_.insert(instance); + } + + static void unregister_instance(SharedMemoryBase* instance) { + std::scoped_lock lock(registry_mutex_); + active_instances_.erase(instance); + } + + static void cleanup_all() noexcept { + std::scoped_lock lock(registry_mutex_); + for (auto* instance : active_instances_) + instance->close(); + active_instances_.clear(); + } +}; + +inline std::mutex SharedMemoryRegistry::registry_mutex_; +inline std::unordered_set SharedMemoryRegistry::active_instances_; + +class CleanupHooks { + private: + static std::once_flag register_once_; + + static void handle_signal(int sig) noexcept { + SharedMemoryRegistry::cleanup_all(); + _Exit(128 + sig); + } + + static void register_signal_handlers() noexcept { + std::atexit([]() { SharedMemoryRegistry::cleanup_all(); }); + + constexpr int signals[] = {SIGHUP, SIGINT, SIGQUIT, SIGILL, SIGABRT, SIGFPE, + SIGSEGV, SIGTERM, SIGBUS, SIGSYS, SIGXCPU, SIGXFSZ}; + + struct sigaction sa; + sa.sa_handler = handle_signal; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + + for (int sig : signals) + sigaction(sig, &sa, nullptr); + } + + public: + static void ensure_registered() noexcept { + std::call_once(register_once_, register_signal_handlers); + } +}; + +inline std::once_flag CleanupHooks::register_once_; + + +inline int portable_fallocate(int fd, off_t offset, off_t length) { +#ifdef __APPLE__ + fstore_t store = {F_ALLOCATECONTIG, F_PEOFPOSMODE, offset, length, 0}; + int ret = fcntl(fd, F_PREALLOCATE, &store); + if (ret == -1) + { + store.fst_flags = F_ALLOCATEALL; + ret = fcntl(fd, F_PREALLOCATE, &store); + } + if (ret != -1) + ret = ftruncate(fd, offset + length); + return ret; +#else + return posix_fallocate(fd, offset, length); +#endif +} + +} // namespace detail + +template +class SharedMemory: public detail::SharedMemoryBase { + static_assert(std::is_trivially_copyable_v, "T must be trivially copyable"); + static_assert(!std::is_pointer_v, "T cannot be a pointer type"); + + private: + std::string name_; + int fd_ = -1; + void* mapped_ptr_ = nullptr; + T* data_ptr_ = nullptr; + detail::ShmHeader* header_ptr_ = nullptr; + size_t total_size_ = 0; + std::string sentinel_base_; + std::string sentinel_path_; + + static constexpr size_t calculate_total_size() noexcept { + return sizeof(T) + sizeof(detail::ShmHeader); + } + + static std::string make_sentinel_base(const std::string& name) { + uint64_t hash = std::hash{}(name); + char buf[32]; + std::snprintf(buf, sizeof(buf), "sfshm_%016" PRIx64, static_cast(hash)); + return buf; + } + + public: + explicit SharedMemory(const std::string& name) noexcept : + name_(name), + total_size_(calculate_total_size()), + sentinel_base_(make_sentinel_base(name)) {} + + ~SharedMemory() noexcept override { + detail::SharedMemoryRegistry::unregister_instance(this); + close(); + } + + SharedMemory(const SharedMemory&) = delete; + SharedMemory& operator=(const SharedMemory&) = delete; + + SharedMemory(SharedMemory&& other) noexcept : + name_(std::move(other.name_)), + fd_(other.fd_), + mapped_ptr_(other.mapped_ptr_), + data_ptr_(other.data_ptr_), + header_ptr_(other.header_ptr_), + total_size_(other.total_size_), + sentinel_base_(std::move(other.sentinel_base_)), + sentinel_path_(std::move(other.sentinel_path_)) { + + detail::SharedMemoryRegistry::unregister_instance(&other); + detail::SharedMemoryRegistry::register_instance(this); + other.reset(); + } + + SharedMemory& operator=(SharedMemory&& other) noexcept { + if (this != &other) + { + detail::SharedMemoryRegistry::unregister_instance(this); + close(); + + name_ = std::move(other.name_); + fd_ = other.fd_; + mapped_ptr_ = other.mapped_ptr_; + data_ptr_ = other.data_ptr_; + header_ptr_ = other.header_ptr_; + total_size_ = other.total_size_; + sentinel_base_ = std::move(other.sentinel_base_); + sentinel_path_ = std::move(other.sentinel_path_); + + detail::SharedMemoryRegistry::unregister_instance(&other); + detail::SharedMemoryRegistry::register_instance(this); + + other.reset(); + } + return *this; + } + + [[nodiscard]] bool open(const T& initial_value) noexcept { + detail::CleanupHooks::ensure_registered(); + + bool retried_stale = false; + + while (true) + { + if (is_open()) + return false; + + bool created_new = false; + fd_ = shm_open(name_.c_str(), O_CREAT | O_EXCL | O_RDWR, 0666); + + if (fd_ == -1) + { + fd_ = shm_open(name_.c_str(), O_RDWR, 0666); + if (fd_ == -1) + return false; + } + else + created_new = true; + + if (!lock_file(LOCK_EX)) + { + ::close(fd_); + reset(); + return false; + } + + bool invalid_header = false; + bool success = + created_new ? setup_new_region(initial_value) : setup_existing_region(invalid_header); + + if (!success) + { + if (created_new || invalid_header) + shm_unlink(name_.c_str()); + if (mapped_ptr_) + unmap_region(); + unlock_file(); + ::close(fd_); + reset(); + + if (!created_new && invalid_header && !retried_stale) + { + retried_stale = true; + continue; + } + return false; + } + + if (!lock_shared_mutex()) + { + if (created_new) + shm_unlink(name_.c_str()); + if (mapped_ptr_) + unmap_region(); + unlock_file(); + ::close(fd_); + reset(); + + if (!created_new && !retried_stale) + { + retried_stale = true; + continue; + } + return false; + } + + if (!create_sentinel_file_locked()) + { + unlock_shared_mutex(); + unmap_region(); + if (created_new) + shm_unlink(name_.c_str()); + unlock_file(); + ::close(fd_); + reset(); + return false; + } + + header_ptr_->ref_count.fetch_add(1, std::memory_order_acq_rel); + + unlock_shared_mutex(); + unlock_file(); + detail::SharedMemoryRegistry::register_instance(this); + return true; + } + } + + void close() noexcept override { + if (fd_ == -1 && mapped_ptr_ == nullptr) + return; + + bool remove_region = false; + bool file_locked = lock_file(LOCK_EX); + bool mutex_locked = false; + + if (file_locked && header_ptr_ != nullptr) + mutex_locked = lock_shared_mutex(); + + if (mutex_locked) + { + if (header_ptr_) + { + header_ptr_->ref_count.fetch_sub(1, std::memory_order_acq_rel); + } + remove_sentinel_file(); + remove_region = !has_other_live_sentinels_locked(); + unlock_shared_mutex(); + } + else + { + remove_sentinel_file(); + decrement_refcount_relaxed(); + } + + unmap_region(); + + if (remove_region) + shm_unlink(name_.c_str()); + + if (file_locked) + unlock_file(); + + if (fd_ != -1) + { + ::close(fd_); + fd_ = -1; + } + + reset(); + } + + const std::string& name() const noexcept override { return name_; } + + [[nodiscard]] bool is_open() const noexcept { return fd_ != -1 && mapped_ptr_ && data_ptr_; } + + [[nodiscard]] const T& get() const noexcept { return *data_ptr_; } + + [[nodiscard]] const T* operator->() const noexcept { return data_ptr_; } + + [[nodiscard]] const T& operator*() const noexcept { return *data_ptr_; } + + [[nodiscard]] uint32_t ref_count() const noexcept { + return header_ptr_ ? header_ptr_->ref_count.load(std::memory_order_acquire) : 0; + } + + [[nodiscard]] bool is_initialized() const noexcept { + return header_ptr_ ? header_ptr_->initialized.load(std::memory_order_acquire) : false; + } + + static void cleanup_all_instances() noexcept { detail::SharedMemoryRegistry::cleanup_all(); } + + private: + void reset() noexcept { + fd_ = -1; + mapped_ptr_ = nullptr; + data_ptr_ = nullptr; + header_ptr_ = nullptr; + sentinel_path_.clear(); + } + + void unmap_region() noexcept { + if (mapped_ptr_) + { + munmap(mapped_ptr_, total_size_); + mapped_ptr_ = nullptr; + data_ptr_ = nullptr; + header_ptr_ = nullptr; + } + } + + [[nodiscard]] bool lock_file(int operation) noexcept { + if (fd_ == -1) + return false; + + while (flock(fd_, operation) == -1) + { + if (errno == EINTR) + continue; + return false; + } + return true; + } + + void unlock_file() noexcept { + if (fd_ == -1) + return; + + while (flock(fd_, LOCK_UN) == -1) + { + if (errno == EINTR) + continue; + break; + } + } + + std::string sentinel_full_path(pid_t pid) const { + std::string path = "/dev/shm/"; + path += sentinel_base_; + path.push_back('.'); + path += std::to_string(pid); + return path; + } + + void decrement_refcount_relaxed() noexcept { + if (!header_ptr_) + return; + + uint32_t expected = header_ptr_->ref_count.load(std::memory_order_relaxed); + while (expected != 0 + && !header_ptr_->ref_count.compare_exchange_weak( + expected, expected - 1, std::memory_order_acq_rel, std::memory_order_relaxed)) + {} + } + + bool create_sentinel_file_locked() noexcept { + if (!header_ptr_) + return false; + + const pid_t self_pid = getpid(); + sentinel_path_ = sentinel_full_path(self_pid); + + for (int attempt = 0; attempt < 2; ++attempt) + { + int fd = ::open(sentinel_path_.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0600); + if (fd != -1) + { + ::close(fd); + return true; + } + + if (errno == EEXIST) + { + ::unlink(sentinel_path_.c_str()); + decrement_refcount_relaxed(); + continue; + } + + break; + } + + sentinel_path_.clear(); + return false; + } + + void remove_sentinel_file() noexcept { + if (!sentinel_path_.empty()) + { + ::unlink(sentinel_path_.c_str()); + sentinel_path_.clear(); + } + } + + static bool pid_is_alive(pid_t pid) noexcept { + if (pid <= 0) + return false; + + if (kill(pid, 0) == 0) + return true; + + return errno == EPERM; + } + + [[nodiscard]] bool initialize_shared_mutex() noexcept { + if (!header_ptr_) + return false; + + pthread_mutexattr_t attr; + if (pthread_mutexattr_init(&attr) != 0) + return false; + + bool success = pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED) == 0; +#ifdef PTHREAD_MUTEX_ROBUST + if (success) + success = pthread_mutexattr_setrobust(&attr, PTHREAD_MUTEX_ROBUST) == 0; +#endif + + if (success) + success = pthread_mutex_init(&header_ptr_->mutex, &attr) == 0; + + pthread_mutexattr_destroy(&attr); + return success; + } + + [[nodiscard]] bool lock_shared_mutex() noexcept { + if (!header_ptr_) + return false; + + while (true) + { + int rc = pthread_mutex_lock(&header_ptr_->mutex); + if (rc == 0) + return true; + +#ifdef PTHREAD_MUTEX_ROBUST + if (rc == EOWNERDEAD) + { + if (pthread_mutex_consistent(&header_ptr_->mutex) == 0) + return true; + return false; + } +#endif + + if (rc == EINTR) + continue; + + return false; + } + } + + void unlock_shared_mutex() noexcept { + if (header_ptr_) + pthread_mutex_unlock(&header_ptr_->mutex); + } + + bool has_other_live_sentinels_locked() const noexcept { + DIR* dir = opendir("/dev/shm"); + if (!dir) + return false; + + std::string prefix = sentinel_base_ + "."; + bool found = false; + + while (dirent* entry = readdir(dir)) + { + std::string name = entry->d_name; + if (name.rfind(prefix, 0) != 0) + continue; + + auto pid_str = name.substr(prefix.size()); + char* end = nullptr; + long value = std::strtol(pid_str.c_str(), &end, 10); + if (!end || *end != '\0') + continue; + + pid_t pid = static_cast(value); + if (pid_is_alive(pid)) + { + found = true; + break; + } + + std::string stale_path = std::string("/dev/shm/") + name; + ::unlink(stale_path.c_str()); + const_cast(this)->decrement_refcount_relaxed(); + } + + closedir(dir); + return found; + } + + [[nodiscard]] bool setup_new_region(const T& initial_value) noexcept { + if (ftruncate(fd_, static_cast(total_size_)) == -1) + return false; + + if (detail::portable_fallocate(fd_, 0, static_cast(total_size_)) != 0) + return false; + + mapped_ptr_ = mmap(nullptr, total_size_, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0); + if (mapped_ptr_ == MAP_FAILED) + { + mapped_ptr_ = nullptr; + return false; + } + + data_ptr_ = static_cast(mapped_ptr_); + header_ptr_ = + reinterpret_cast(static_cast(mapped_ptr_) + sizeof(T)); + + new (header_ptr_) detail::ShmHeader{}; + new (data_ptr_) T{initial_value}; + + if (!initialize_shared_mutex()) + return false; + + header_ptr_->ref_count.store(0, std::memory_order_release); + header_ptr_->initialized.store(true, std::memory_order_release); + return true; + } + + [[nodiscard]] bool setup_existing_region(bool& invalid_header) noexcept { + invalid_header = false; + + struct stat st; + fstat(fd_, &st); + if (static_cast(st.st_size) < total_size_) + { + invalid_header = true; + return false; + } + + mapped_ptr_ = mmap(nullptr, total_size_, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0); + if (mapped_ptr_ == MAP_FAILED) + { + mapped_ptr_ = nullptr; + return false; + } + + data_ptr_ = static_cast(mapped_ptr_); + header_ptr_ = std::launder( + reinterpret_cast(static_cast(mapped_ptr_) + sizeof(T))); + + if (!header_ptr_->initialized.load(std::memory_order_acquire) + || header_ptr_->magic != detail::ShmHeader::SHM_MAGIC) + { + invalid_header = true; + unmap_region(); + return false; + } + + return true; + } +}; + +template +[[nodiscard]] std::optional> create_shared(const std::string& name, + const T& initial_value) noexcept { + SharedMemory shm(name); + if (shm.open(initial_value)) + return shm; + return std::nullopt; +} + +} // namespace Stockfish::shm + +#endif // #ifndef SHM_LINUX_H_INCLUDED