jirayu_sri/stockfish
1
0
Fork 0
mirror of https://github.com/official-stockfish/Stockfish.git synced 2026-07-23 21:27:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit '69a01b88f35db2a5003d42116f573207ca5c275b' into cluster

Browse Source
This commit is contained in:
Steinar H. Gunderson
2025-12-25 20:16:11 +01:00
parent bfc8c24999 69a01b88f3
commit 8cc99e1314
24 changed files with 1885 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Makefile
+15 -1
View File
@@ -438,7 +438,7 @@ endif
ifeq ($(COMP),gcc)
comp=gcc
CXX=g++
CXXFLAGS += -pedantic -Wextra -Wshadow -Wmissing-declarations
CXXFLAGS += -pedantic -Wextra -Wshadow -Wmissing-declarations -Wstack-usage=128000
ifeq ($(arch),$(filter $(arch),armv7 armv8 riscv64))
ifeq ($(OS),Android)
@@ -621,6 +621,19 @@ ifneq ($(comp),mingw)
ifneq ($(KERNEL),Haiku)
ifneq ($(COMP),ndk)
LDFLAGS += -lpthread
add_lrt = yes
ifeq ($(target_windows),yes)
add_lrt = no
endif
ifeq ($(KERNEL),Darwin)
add_lrt = no
endif
ifeq ($(add_lrt),yes)
LDFLAGS += -lrt
endif
endif
endif
endif
@@ -631,6 +644,7 @@ ifeq ($(debug),no)
CXXFLAGS += -DNDEBUG
else
CXXFLAGS += -g
CXXFLAGS += -D_GLIBCXX_ASSERTIONS -D_GLIBCXX_DEBUG
endif
### 3.2.2 Debugging with undefined behavior sanitizers
src/engine.cpp
+39 -4
View File
@@ -33,10 +33,12 @@
#include "misc.h"
#include "nnue/network.h"
#include "nnue/nnue_common.h"
#include "nnue/nnue_misc.h"
#include "numa.h"
#include "perft.h"
#include "position.h"
#include "search.h"
#include "shm.h"
#include "syzygy/tbprobe.h"
#include "types.h"
#include "uci.h"
@@ -57,11 +59,14 @@ Engine::Engine(std::optional<std::string> path) :
threads(),
networks(
numaContext,
NN::Networks(
NN::NetworkBig({EvalFileDefaultNameBig, "None", ""}, NN::EmbeddedNNUEType::BIG),
NN::NetworkSmall({EvalFileDefaultNameSmall, "None", ""}, NN::EmbeddedNNUEType::SMALL))) {
pos.set(StartFEN, false, &states->back());
// Heap-allocate because sizeof(NN::Networks) is large
std::make_unique<NN::Networks>(
std::make_unique<NN::NetworkBig>(NN::EvalFile{EvalFileDefaultNameBig, "None", ""},
NN::EmbeddedNNUEType::BIG),
std::make_unique<NN::NetworkSmall>(NN::EvalFile{EvalFileDefaultNameSmall, "None", ""},
NN::EmbeddedNNUEType::SMALL))) {
pos.set(StartFEN, false, &states->back());
options.add( //
"Debug Log File", Option("", [](const Option& o) {
@@ -257,6 +262,36 @@ void Engine::set_ponderhit(bool b) { threads.main_manager()->ponder = b; }
void Engine::verify_networks() const {
networks->big.verify(options["EvalFile"], onVerifyNetworks);
networks->small.verify(options["EvalFileSmall"], onVerifyNetworks);
auto statuses = networks.get_status_and_errors();
for (size_t i = 0; i < statuses.size(); ++i)
{
const auto [status, error] = statuses[i];
std::string message = "Network replica " + std::to_string(i + 1) + ": ";
if (status == SystemWideSharedConstantAllocationStatus::NoAllocation)
{
message += "No allocation.";
}
else if (status == SystemWideSharedConstantAllocationStatus::LocalMemory)
{
message += "Local memory.";
}
else if (status == SystemWideSharedConstantAllocationStatus::SharedMemory)
{
message += "Shared memory.";
}
else
{
message += "Unknown status.";
}
if (error.has_value())
{
message += " " + *error;
}
onVerifyNetworks(message);
}
}
void Engine::load_networks() {
src/engine.h
+4 -4
View File
@@ -115,10 +115,10 @@ class Engine {
Position pos;
StateListPtr states;
OptionsMap options;
ThreadPool threads;
TranspositionTable tt;
LazyNumaReplicated<Eval::NNUE::Networks> networks;
OptionsMap options;
ThreadPool threads;
TranspositionTable tt;
LazyNumaReplicatedSystemWide<Eval::NNUE::Networks> networks;
Search::SearchManager::UpdateContext updateContext;
std::function<void(std::string_view)> onVerifyNetworks;
src/evaluate.cpp
+4 -4
View File
@@ -98,8 +98,8 @@ std::string Eval::trace(Position& pos, const Eval::NNUE::Networks& networks) {
if (pos.checkers())
return "Final evaluation: none (in check)";
Eval::NNUE::AccumulatorStack accumulators;
auto caches = std::make_unique<Eval::NNUE::AccumulatorCaches>(networks);
auto accumulators = std::make_unique<Eval::NNUE::AccumulatorStack>();
auto caches = std::make_unique<Eval::NNUE::AccumulatorCaches>(networks);
std::stringstream ss;
ss << std::showpoint << std::noshowpos << std::fixed << std::setprecision(2);
@@ -107,12 +107,12 @@ std::string Eval::trace(Position& pos, const Eval::NNUE::Networks& networks) {
ss << std::showpoint << std::showpos << std::fixed << std::setprecision(2) << std::setw(15);
auto [psqt, positional] = networks.big.evaluate(pos, accumulators, &caches->big);
auto [psqt, positional] = networks.big.evaluate(pos, *accumulators, &caches->big);
Value v = psqt + positional;
v = pos.side_to_move() == WHITE ? v : -v;
ss << "NNUE evaluation " << 0.01 * UCIEngine::to_cp(v, pos) << " (white side)\n";
v = evaluate(networks, pos, accumulators, *caches, VALUE_ZERO);
v = evaluate(networks, pos, *accumulators, *caches, VALUE_ZERO);
v = pos.side_to_move() == WHITE ? v : -v;
ss << "Final evaluation " << 0.01 * UCIEngine::to_cp(v, pos) << " (white side)";
ss << " [with scaled NNUE, ...]";
src/main.cpp
+5 -5
View File
@@ -17,18 +17,18 @@
*/
#include <iostream>
#include <memory>
#include "bitboard.h"
#include "misc.h"
#include "position.h"
#include "tune.h"
#include "types.h"
#include "uci.h"
#include "tune.h"
using namespace Stockfish;
int main(int argc, char* argv[]) {
Distributed::init();
if (Distributed::is_root())
std::cout << engine_info() << std::endl;
@@ -36,11 +36,11 @@ int main(int argc, char* argv[]) {
Bitboards::init();
Position::init();
UCIEngine uci(argc, argv);
auto uci = std::make_unique<UCIEngine>(argc, argv);
Tune::init(uci.engine_options());
Tune::init(uci->engine_options());
uci.loop();
uci->loop();
Distributed::finalize();
src/memory.cpp
+8 -77
View File
@@ -55,12 +55,6 @@
// the calls at compile time), try to load them at runtime. To do this we need
// first to define the corresponding function pointers.
extern "C" {
using OpenProcessToken_t = bool (*)(HANDLE, DWORD, PHANDLE);
using LookupPrivilegeValueA_t = bool (*)(LPCSTR, LPCSTR, PLUID);
using AdjustTokenPrivileges_t =
bool (*)(HANDLE, BOOL, PTOKEN_PRIVILEGES, DWORD, PTOKEN_PRIVILEGES, PDWORD);
}
#endif
@@ -106,77 +100,14 @@ void std_aligned_free(void* ptr) {
static void* aligned_large_pages_alloc_windows([[maybe_unused]] size_t allocSize) {
#if !defined(_WIN64)
return nullptr;
#else
HANDLE hProcessToken{};
LUID luid{};
void* mem = nullptr;
const size_t largePageSize = GetLargePageMinimum();
if (!largePageSize)
return nullptr;
// Dynamically link OpenProcessToken, LookupPrivilegeValue and AdjustTokenPrivileges
HMODULE hAdvapi32 = GetModuleHandle(TEXT("advapi32.dll"));
if (!hAdvapi32)
hAdvapi32 = LoadLibrary(TEXT("advapi32.dll"));
auto OpenProcessToken_f =
OpenProcessToken_t((void (*)()) GetProcAddress(hAdvapi32, "OpenProcessToken"));
if (!OpenProcessToken_f)
return nullptr;
auto LookupPrivilegeValueA_f =
LookupPrivilegeValueA_t((void (*)()) GetProcAddress(hAdvapi32, "LookupPrivilegeValueA"));
if (!LookupPrivilegeValueA_f)
return nullptr;
auto AdjustTokenPrivileges_f =
AdjustTokenPrivileges_t((void (*)()) GetProcAddress(hAdvapi32, "AdjustTokenPrivileges"));
if (!AdjustTokenPrivileges_f)
return nullptr;
// We need SeLockMemoryPrivilege, so try to enable it for the process
if (!OpenProcessToken_f( // OpenProcessToken()
GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hProcessToken))
return nullptr;
if (LookupPrivilegeValueA_f(nullptr, "SeLockMemoryPrivilege", &luid))
{
TOKEN_PRIVILEGES tp{};
TOKEN_PRIVILEGES prevTp{};
DWORD prevTpLen = 0;
tp.PrivilegeCount = 1;
tp.Privileges[0].Luid = luid;
tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
// Try to enable SeLockMemoryPrivilege. Note that even if AdjustTokenPrivileges()
// succeeds, we still need to query GetLastError() to ensure that the privileges
// were actually obtained.
if (AdjustTokenPrivileges_f(hProcessToken, FALSE, &tp, sizeof(TOKEN_PRIVILEGES), &prevTp,
&prevTpLen)
&& GetLastError() == ERROR_SUCCESS)
{
// Round up size to full pages and allocate
allocSize = (allocSize + largePageSize - 1) & ~size_t(largePageSize - 1);
mem = VirtualAlloc(nullptr, allocSize, MEM_RESERVE | MEM_COMMIT | MEM_LARGE_PAGES,
PAGE_READWRITE);
// Privilege no longer needed, restore previous state
AdjustTokenPrivileges_f(hProcessToken, FALSE, &prevTp, 0, nullptr, nullptr);
}
}
CloseHandle(hProcessToken);
return mem;
#endif
return windows_try_with_large_page_priviliges(
[&](size_t largePageSize) {
// Round up size to full pages and allocate
allocSize = (allocSize + largePageSize - 1) & ~size_t(largePageSize - 1);
return VirtualAlloc(nullptr, allocSize, MEM_RESERVE | MEM_COMMIT | MEM_LARGE_PAGES,
PAGE_READWRITE);
},
[]() { return (void*) nullptr; });
}
void* aligned_large_pages_alloc(size_t allocSize) {
src/memory.h
+98
View File
@@ -29,6 +29,29 @@
#include "types.h"
#if defined(_WIN64)
#if _WIN32_WINNT < 0x0601
#undef _WIN32_WINNT
#define _WIN32_WINNT 0x0601 // Force to include needed API prototypes
#endif
#if !defined(NOMINMAX)
#define NOMINMAX
#endif
#include <windows.h>
#include <psapi.h>
extern "C" {
using OpenProcessToken_t = bool (*)(HANDLE, DWORD, PHANDLE);
using LookupPrivilegeValueA_t = bool (*)(LPCSTR, LPCSTR, PLUID);
using AdjustTokenPrivileges_t =
bool (*)(HANDLE, BOOL, PTOKEN_PRIVILEGES, DWORD, PTOKEN_PRIVILEGES, PDWORD);
}
#endif
namespace Stockfish {
void* std_aligned_alloc(size_t alignment, size_t size);
@@ -211,6 +234,81 @@ T* align_ptr_up(T* ptr) {
reinterpret_cast<char*>((ptrint + (Alignment - 1)) / Alignment * Alignment));
}
#if defined(_WIN32)
template<typename FuncYesT, typename FuncNoT>
auto windows_try_with_large_page_priviliges([[maybe_unused]] FuncYesT&& fyes, FuncNoT&& fno) {
#if !defined(_WIN64)
return fno();
#else
HANDLE hProcessToken{};
LUID luid{};
const size_t largePageSize = GetLargePageMinimum();
if (!largePageSize)
return fno();
// Dynamically link OpenProcessToken, LookupPrivilegeValue and AdjustTokenPrivileges
HMODULE hAdvapi32 = GetModuleHandle(TEXT("advapi32.dll"));
if (!hAdvapi32)
hAdvapi32 = LoadLibrary(TEXT("advapi32.dll"));
auto OpenProcessToken_f =
OpenProcessToken_t((void (*)()) GetProcAddress(hAdvapi32, "OpenProcessToken"));
if (!OpenProcessToken_f)
return fno();
auto LookupPrivilegeValueA_f =
LookupPrivilegeValueA_t((void (*)()) GetProcAddress(hAdvapi32, "LookupPrivilegeValueA"));
if (!LookupPrivilegeValueA_f)
return fno();
auto AdjustTokenPrivileges_f =
AdjustTokenPrivileges_t((void (*)()) GetProcAddress(hAdvapi32, "AdjustTokenPrivileges"));
if (!AdjustTokenPrivileges_f)
return fno();
// We need SeLockMemoryPrivilege, so try to enable it for the process
if (!OpenProcessToken_f( // OpenProcessToken()
GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hProcessToken))
return fno();
if (!LookupPrivilegeValueA_f(nullptr, "SeLockMemoryPrivilege", &luid))
return fno();
TOKEN_PRIVILEGES tp{};
TOKEN_PRIVILEGES prevTp{};
DWORD prevTpLen = 0;
tp.PrivilegeCount = 1;
tp.Privileges[0].Luid = luid;
tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
// Try to enable SeLockMemoryPrivilege. Note that even if AdjustTokenPrivileges()
// succeeds, we still need to query GetLastError() to ensure that the privileges
// were actually obtained.
if (!AdjustTokenPrivileges_f(hProcessToken, FALSE, &tp, sizeof(TOKEN_PRIVILEGES), &prevTp,
&prevTpLen)
|| GetLastError() != ERROR_SUCCESS)
return fno();
auto&& ret = fyes(largePageSize);
// Privilege no longer needed, restore previous state
AdjustTokenPrivileges_f(hProcessToken, FALSE, &prevTp, 0, nullptr, nullptr);
CloseHandle(hProcessToken);
return std::forward<decltype(ret)>(ret);
#endif
}
#endif
} // namespace Stockfish
src/misc.h
+100 -1
View File
@@ -23,11 +23,15 @@
#include <array>
#include <cassert>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <exception> // IWYU pragma: keep
// IWYU pragma: no_include <__exception/terminate.h>
#include <functional>
#include <iosfwd>
#include <optional>
#include <cstring>
#include <memory>
#include <string>
#include <string_view>
#include <vector>
@@ -291,6 +295,94 @@ inline uint64_t mul_hi64(uint64_t a, uint64_t b) {
}
template<typename T>
inline void hash_combine(std::size_t& seed, const T& v) {
std::hash<T> hasher;
seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
}
template<>
inline void hash_combine(std::size_t& seed, const std::size_t& v) {
seed ^= v + 0x9e3779b9 + (seed << 6) + (seed >> 2);
}
template<typename T>
inline std::size_t get_raw_data_hash(const T& value) {
return std::hash<std::string_view>{}(
std::string_view(reinterpret_cast<const char*>(&value), sizeof(value)));
}
template<std::size_t Capacity>
class FixedString {
public:
FixedString() :
length_(0) {
data_[0] = '\0';
}
FixedString(const char* str) {
size_t len = std::strlen(str);
if (len > Capacity)
std::terminate();
std::memcpy(data_, str, len);
length_ = len;
data_[length_] = '\0';
}
FixedString(const std::string& str) {
if (str.size() > Capacity)
std::terminate();
std::memcpy(data_, str.data(), str.size());
length_ = str.size();
data_[length_] = '\0';
}
std::size_t size() const { return length_; }
std::size_t capacity() const { return Capacity; }
const char* c_str() const { return data_; }
const char* data() const { return data_; }
char& operator[](std::size_t i) { return data_[i]; }
const char& operator[](std::size_t i) const { return data_[i]; }
FixedString& operator+=(const char* str) {
size_t len = std::strlen(str);
if (length_ + len > Capacity)
std::terminate();
std::memcpy(data_ + length_, str, len);
length_ += len;
data_[length_] = '\0';
return *this;
}
FixedString& operator+=(const FixedString& other) { return (*this += other.c_str()); }
operator std::string() const { return std::string(data_, length_); }
operator std::string_view() const { return std::string_view(data_, length_); }
template<typename T>
bool operator==(const T& other) const noexcept {
return (std::string_view) (*this) == other;
}
template<typename T>
bool operator!=(const T& other) const noexcept {
return (std::string_view) (*this) != other;
}
void clear() {
length_ = 0;
data_[0] = '\0';
}
private:
char data_[Capacity + 1]; // +1 for null terminator
std::size_t length_;
};
struct CommandLine {
public:
CommandLine(int _argc, char** _argv) :
@@ -335,4 +427,11 @@ void move_to_front(std::vector<T>& vec, Predicate pred) {
} // namespace Stockfish
template<std::size_t N>
struct std::hash<Stockfish::FixedString<N>> {
std::size_t operator()(const Stockfish::FixedString<N>& fstr) const noexcept {
return std::hash<std::string_view>{}((std::string_view) fstr);
}
};
#endif // #ifndef MISC_H_INCLUDED
src/nnue/layers/affine_transform.h
+9
View File
@@ -181,6 +181,15 @@ class AffineTransform {
return !stream.fail();
}
std::size_t get_content_hash() const {
std::size_t h = 0;
hash_combine(h, get_raw_data_hash(biases));
hash_combine(h, get_raw_data_hash(weights));
hash_combine(h, get_hash_value(0));
return h;
}
// Forward propagation
void propagate(const InputType* input, OutputType* output) const {
src/nnue/layers/affine_transform_sparse_input.h
+9
View File
@@ -241,6 +241,15 @@ class AffineTransformSparseInput {
return !stream.fail();
}
std::size_t get_content_hash() const {
std::size_t h = 0;
hash_combine(h, get_raw_data_hash(biases));
hash_combine(h, get_raw_data_hash(weights));
hash_combine(h, get_hash_value(0));
return h;
}
// Forward propagation
void propagate(const InputType* input, OutputType* output) const {
src/nnue/layers/clipped_relu.h
+6
View File
@@ -58,6 +58,12 @@ class ClippedReLU {
// Write network parameters
bool write_parameters(std::ostream&) const { return true; }
std::size_t get_content_hash() const {
std::size_t h = 0;
hash_combine(h, get_hash_value(0));
return h;
}
// Forward propagation
void propagate(const InputType* input, OutputType* output) const {
src/nnue/layers/sqr_clipped_relu.h
+6
View File
@@ -58,6 +58,12 @@ class SqrClippedReLU {
// Write network parameters
bool write_parameters(std::ostream&) const { return true; }
std::size_t get_content_hash() const {
std::size_t h = 0;
hash_combine(h, get_hash_value(0));
return h;
}
// Forward propagation
void propagate(const InputType* input, OutputType* output) const {
src/nnue/network.cpp
+28 -54
View File
@@ -21,7 +21,6 @@
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <memory>
#include <optional>
#include <type_traits>
#include <vector>
@@ -31,7 +30,6 @@
#include "../cluster.h"
#include "../evaluate.h"
#include "../memory.h"
#include "../misc.h"
#include "../position.h"
#include "../types.h"
@@ -102,7 +100,7 @@ bool read_parameters(std::istream& stream, T& reference) {
// Write evaluation function parameters
template<typename T>
bool write_parameters(std::ostream& stream, T& reference) {
bool write_parameters(std::ostream& stream, const T& reference) {
write_little_endian<std::uint32_t>(stream, T::get_hash_value());
return reference.write_parameters(stream);
@@ -110,43 +108,6 @@ bool write_parameters(std::ostream& stream, T& reference) {
} // namespace Detail
template<typename Arch, typename Transformer>
Network<Arch, Transformer>::Network(const Network<Arch, Transformer>& other) :
evalFile(other.evalFile),
embeddedType(other.embeddedType) {
if (other.featureTransformer)
featureTransformer = make_unique_large_page<Transformer>(*other.featureTransformer);
network = make_unique_aligned<Arch[]>(LayerStacks);
if (!other.network)
return;
for (std::size_t i = 0; i < LayerStacks; ++i)
network[i] = other.network[i];
}
template<typename Arch, typename Transformer>
Network<Arch, Transformer>&
Network<Arch, Transformer>::operator=(const Network<Arch, Transformer>& other) {
evalFile = other.evalFile;
embeddedType = other.embeddedType;
if (other.featureTransformer)
featureTransformer = make_unique_large_page<Transformer>(*other.featureTransformer);
network = make_unique_aligned<Arch[]>(LayerStacks);
if (!other.network)
return *this;
for (std::size_t i = 0; i < LayerStacks; ++i)
network[i] = other.network[i];
return *this;
}
template<typename Arch, typename Transformer>
void Network<Arch, Transformer>::load(const std::string& rootDirectory, std::string evalfilePath) {
#if defined(DEFAULT_NNUE_DIRECTORY)
@@ -161,14 +122,14 @@ void Network<Arch, Transformer>::load(const std::string& rootDirectory, std::str
for (const auto& directory : dirs)
{
if (evalFile.current != evalfilePath)
if (std::string(evalFile.current) != evalfilePath)
{
if (directory != "<internal>")
{
load_user_net(directory, evalfilePath);
}
if (directory == "<internal>" && evalfilePath == evalFile.defaultName)
if (directory == "<internal>" && evalfilePath == std::string(evalFile.defaultName))
{
load_internal();
}
@@ -186,7 +147,7 @@ bool Network<Arch, Transformer>::save(const std::optional<std::string>& filename
actualFilename = filename.value();
else
{
if (evalFile.current != evalFile.defaultName)
if (std::string(evalFile.current) != std::string(evalFile.defaultName))
{
msg = "Failed to export a net. "
"A non-embedded net can only be saved if the filename is specified";
@@ -223,7 +184,7 @@ Network<Arch, Transformer>::evaluate(const Position& pos
const int bucket = (pos.count<ALL_PIECES>() - 1) / 4;
const auto psqt =
featureTransformer->transform(pos, accumulatorStack, cache, transformedFeatures, bucket);
featureTransformer.transform(pos, accumulatorStack, cache, transformedFeatures, bucket);
const auto positional = network[bucket].propagate(transformedFeatures);
return {static_cast<Value>(psqt / OutputScale), static_cast<Value>(positional / OutputScale)};
}
@@ -235,7 +196,7 @@ void Network<Arch, Transformer>::verify(std::string
if (evalfilePath.empty())
evalfilePath = evalFile.defaultName;
if (evalFile.current != evalfilePath)
if (std::string(evalFile.current) != evalfilePath)
{
if (f)
{
@@ -246,7 +207,7 @@ void Network<Arch, Transformer>::verify(std::string
"including the directory name, to the network file.";
std::string msg4 = "The default net can be downloaded from: "
"https://tests.stockfishchess.org/api/nn/"
+ evalFile.defaultName;
+ std::string(evalFile.defaultName);
std::string msg5 = "The engine will be terminated now.";
std::string msg = "ERROR: " + msg1 + '\n' + "ERROR: " + msg2 + '\n' + "ERROR: " + msg3
@@ -260,10 +221,10 @@ void Network<Arch, Transformer>::verify(std::string
if (f)
{
size_t size = sizeof(*featureTransformer) + sizeof(Arch) * LayerStacks;
size_t size = sizeof(featureTransformer) + sizeof(Arch) * LayerStacks;
if (Distributed::is_root())
f("NNUE evaluation using " + evalfilePath + " (" + std::to_string(size / (1024 * 1024))
+ "MiB, (" + std::to_string(featureTransformer->InputDimensions) + ", "
+ "MiB, (" + std::to_string(featureTransformer.InputDimensions) + ", "
+ std::to_string(network[0].TransformedFeatureDimensions) + ", "
+ std::to_string(network[0].FC_0_OUTPUTS) + ", "
+ std::to_string(network[0].FC_1_OUTPUTS) + ", 1))");
@@ -289,7 +250,7 @@ Network<Arch, Transformer>::trace_evaluate(const Position&
for (IndexType bucket = 0; bucket < LayerStacks; ++bucket)
{
const auto materialist =
featureTransformer->transform(pos, accumulatorStack, cache, transformedFeatures, bucket);
featureTransformer.transform(pos, accumulatorStack, cache, transformedFeatures, bucket);
const auto positional = network[bucket].propagate(transformedFeatures);
t.psqt[bucket] = static_cast<Value>(materialist / OutputScale);
@@ -343,8 +304,7 @@ void Network<Arch, Transformer>::load_internal() {
template<typename Arch, typename Transformer>
void Network<Arch, Transformer>::initialize() {
featureTransformer = make_unique_large_page<Transformer>();
network = make_unique_aligned<Arch[]>(LayerStacks);
initialized = true;
}
@@ -368,6 +328,20 @@ std::optional<std::string> Network<Arch, Transformer>::load(std::istream& stream
}
template<typename Arch, typename Transformer>
std::size_t Network<Arch, Transformer>::get_content_hash() const {
if (!initialized)
return 0;
std::size_t h = 0;
hash_combine(h, featureTransformer);
for (auto&& layerstack : network)
hash_combine(h, layerstack);
hash_combine(h, evalFile);
hash_combine(h, static_cast<int>(embeddedType));
return h;
}
// Read network header
template<typename Arch, typename Transformer>
bool Network<Arch, Transformer>::read_header(std::istream& stream,
@@ -401,13 +375,13 @@ bool Network<Arch, Transformer>::write_header(std::ostream& stream,
template<typename Arch, typename Transformer>
bool Network<Arch, Transformer>::read_parameters(std::istream& stream,
std::string& netDescription) const {
std::string& netDescription) {
std::uint32_t hashValue;
if (!read_header(stream, &hashValue, &netDescription))
return false;
if (hashValue != Network::hash)
return false;
if (!Detail::read_parameters(stream, *featureTransformer))
if (!Detail::read_parameters(stream, featureTransformer))
return false;
for (std::size_t i = 0; i < LayerStacks; ++i)
{
@@ -423,7 +397,7 @@ bool Network<Arch, Transformer>::write_parameters(std::ostream& stream,
const std::string& netDescription) const {
if (!write_header(stream, Network::hash, netDescription))
return false;
if (!Detail::write_parameters(stream, *featureTransformer))
if (!Detail::write_parameters(stream, featureTransformer))
return false;
for (std::size_t i = 0; i < LayerStacks; ++i)
{
src/nnue/network.h
+38 -11
View File
@@ -19,16 +19,18 @@
#ifndef NETWORK_H_INCLUDED
#define NETWORK_H_INCLUDED
#include <cstddef>
#include <cstdint>
#include <functional>
#include <iostream>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <tuple>
#include <utility>
#include "../memory.h"
#include "../misc.h"
#include "../types.h"
#include "nnue_accumulator.h"
#include "nnue_architecture.h"
@@ -49,6 +51,9 @@ enum class EmbeddedNNUEType {
using NetworkOutput = std::tuple<Value, Value>;
// The network must be a trivial type, i.e. the memory must be in-line.
// This is required to allow sharing the network via shared memory, as
// there is no way to run destructors.
template<typename Arch, typename Transformer>
class Network {
static constexpr IndexType FTDimensions = Arch::TransformedFeatureDimensions;
@@ -58,15 +63,17 @@ class Network {
evalFile(file),
embeddedType(type) {}
Network(const Network& other);
Network(Network&& other) = default;
Network(const Network& other) = default;
Network(Network&& other) = default;
Network& operator=(const Network& other);
Network& operator=(Network&& other) = default;
Network& operator=(const Network& other) = default;
Network& operator=(Network&& other) = default;
void load(const std::string& rootDirectory, std::string evalfilePath);
bool save(const std::optional<std::string>& filename) const;
std::size_t get_content_hash() const;
NetworkOutput evaluate(const Position& pos,
AccumulatorStack& accumulatorStack,
AccumulatorCaches::Cache<FTDimensions>* cache) const;
@@ -89,18 +96,20 @@ class Network {
bool read_header(std::istream&, std::uint32_t*, std::string*) const;
bool write_header(std::ostream&, std::uint32_t, const std::string&) const;
bool read_parameters(std::istream&, std::string&) const;
bool read_parameters(std::istream&, std::string&);
bool write_parameters(std::ostream&, const std::string&) const;
// Input feature converter
LargePagePtr<Transformer> featureTransformer;
Transformer featureTransformer;
// Evaluation function
AlignedPtr<Arch[]> network;
Arch network[LayerStacks];
EvalFile evalFile;
EmbeddedNNUEType embeddedType;
bool initialized = false;
// Hash value of evaluation function structure
static constexpr std::uint32_t hash = Transformer::get_hash_value() ^ Arch::get_hash_value();
@@ -121,9 +130,9 @@ using NetworkSmall = Network<SmallNetworkArchitecture, SmallFeatureTransformer>;
struct Networks {
Networks(NetworkBig&& nB, NetworkSmall&& nS) :
big(std::move(nB)),
small(std::move(nS)) {}
Networks(std::unique_ptr<NetworkBig>&& nB, std::unique_ptr<NetworkSmall>&& nS) :
big(std::move(*nB)),
small(std::move(*nS)) {}
NetworkBig big;
NetworkSmall small;
@@ -132,4 +141,22 @@ struct Networks {
} // namespace Stockfish
template<typename ArchT, typename FeatureTransformerT>
struct std::hash<Stockfish::Eval::NNUE::Network<ArchT, FeatureTransformerT>> {
std::size_t operator()(
const Stockfish::Eval::NNUE::Network<ArchT, FeatureTransformerT>& network) const noexcept {
return network.get_content_hash();
}
};
template<>
struct std::hash<Stockfish::Eval::NNUE::Networks> {
std::size_t operator()(const Stockfish::Eval::NNUE::Networks& networks) const noexcept {
std::size_t h = 0;
Stockfish::hash_combine(h, networks.big);
Stockfish::hash_combine(h, networks.small);
return h;
}
};
#endif
src/nnue/nnue_accumulator.h
+1 -1
View File
@@ -87,7 +87,7 @@ struct AccumulatorCaches {
void clear(const Network& network) {
for (auto& entries1D : entries)
for (auto& entry : entries1D)
entry.clear(network.featureTransformer->biases);
entry.clear(network.featureTransformer.biases);
}
std::array<Entry, COLOR_NB>& operator[](Square sq) { return entries[sq]; }
src/nnue/nnue_architecture.h
+21 -1
View File
@@ -97,7 +97,7 @@ struct NetworkArchitecture {
&& fc_2.write_parameters(stream);
}
std::int32_t propagate(const TransformedFeatureType* transformedFeatures) {
std::int32_t propagate(const TransformedFeatureType* transformedFeatures) const {
struct alignas(CacheLineSize) Buffer {
alignas(CacheLineSize) typename decltype(fc_0)::OutputBuffer fc_0_out;
alignas(CacheLineSize) typename decltype(ac_sqr_0)::OutputType
@@ -136,8 +136,28 @@ struct NetworkArchitecture {
return outputValue;
}
std::size_t get_content_hash() const {
std::size_t h = 0;
hash_combine(h, fc_0.get_content_hash());
hash_combine(h, ac_sqr_0.get_content_hash());
hash_combine(h, ac_0.get_content_hash());
hash_combine(h, fc_1.get_content_hash());
hash_combine(h, ac_1.get_content_hash());
hash_combine(h, fc_2.get_content_hash());
hash_combine(h, get_hash_value());
return h;
}
};
} // namespace Stockfish::Eval::NNUE
template<Stockfish::Eval::NNUE::IndexType L1, int L2, int L3>
struct std::hash<Stockfish::Eval::NNUE::NetworkArchitecture<L1, L2, L3>> {
std::size_t
operator()(const Stockfish::Eval::NNUE::NetworkArchitecture<L1, L2, L3>& arch) const noexcept {
return arch.get_content_hash();
}
};
#endif // #ifndef NNUE_ARCHITECTURE_H_INCLUDED
src/nnue/nnue_common.h
+2 -2
View File
@@ -258,8 +258,8 @@ inline void write_leb_128(std::ostream& stream, const IntType* values, std::size
}
};
auto write = [&](std::uint8_t byte) {
buf[buf_pos++] = byte;
auto write = [&](std::uint8_t b) {
buf[buf_pos++] = b;
if (buf_pos == BUF_SIZE)
flush();
};
src/nnue/nnue_feature_transformer.h
+26 -8
View File
@@ -157,20 +157,28 @@ class FeatureTransformer {
}
// Write network parameters
bool write_parameters(std::ostream& stream) {
bool write_parameters(std::ostream& stream) const {
std::unique_ptr<FeatureTransformer> copy = std::make_unique<FeatureTransformer>(*this);
unpermute_weights();
scale_weights(false);
copy->unpermute_weights();
copy->scale_weights(false);
write_leb_128<BiasType>(stream, biases, HalfDimensions);
write_leb_128<WeightType>(stream, weights, HalfDimensions * InputDimensions);
write_leb_128<PSQTWeightType>(stream, psqtWeights, PSQTBuckets * InputDimensions);
write_leb_128<BiasType>(stream, copy->biases, HalfDimensions);
write_leb_128<WeightType>(stream, copy->weights, HalfDimensions * InputDimensions);
write_leb_128<PSQTWeightType>(stream, copy->psqtWeights, PSQTBuckets * InputDimensions);
permute_weights();
scale_weights(true);
return !stream.fail();
}
std::size_t get_content_hash() const {
std::size_t h = 0;
hash_combine(h, get_raw_data_hash(biases));
hash_combine(h, get_raw_data_hash(weights));
hash_combine(h, get_raw_data_hash(psqtWeights));
hash_combine(h, get_hash_value());
return h;
}
// Convert input features
std::int32_t transform(const Position& pos,
AccumulatorStack& accumulatorStack,
@@ -309,4 +317,14 @@ class FeatureTransformer {
} // namespace Stockfish::Eval::NNUE
template<Stockfish::Eval::NNUE::IndexType TransformedFeatureDimensions>
struct std::hash<Stockfish::Eval::NNUE::FeatureTransformer<TransformedFeatureDimensions>> {
std::size_t
operator()(const Stockfish::Eval::NNUE::FeatureTransformer<TransformedFeatureDimensions>& ft)
const noexcept {
return ft.get_content_hash();
}
};
#endif // #ifndef NNUE_FEATURE_TRANSFORMER_H_INCLUDED
src/nnue/nnue_misc.cpp
+6 -6
View File
@@ -120,11 +120,11 @@ trace(Position& pos, const Eval::NNUE::Networks& networks, Eval::NNUE::Accumulat
format_cp_compact(value, &board[y + 2][x + 2], pos);
};
AccumulatorStack accumulators;
auto accumulators = std::make_unique<AccumulatorStack>();
// We estimate the value of each piece by doing a differential evaluation from
// the current base eval, simulating the removal of the piece from its square.
auto [psqt, positional] = networks.big.evaluate(pos, accumulators, &caches.big);
auto [psqt, positional] = networks.big.evaluate(pos, *accumulators, &caches.big);
Value base = psqt + positional;
base = pos.side_to_move() == WHITE ? base : -base;
@@ -139,8 +139,8 @@ trace(Position& pos, const Eval::NNUE::Networks& networks, Eval::NNUE::Accumulat
{
pos.remove_piece(sq);
accumulators.reset();
std::tie(psqt, positional) = networks.big.evaluate(pos, accumulators, &caches.big);
accumulators->reset();
std::tie(psqt, positional) = networks.big.evaluate(pos, *accumulators, &caches.big);
Value eval = psqt + positional;
eval = pos.side_to_move() == WHITE ? eval : -eval;
v = base - eval;
@@ -156,8 +156,8 @@ trace(Position& pos, const Eval::NNUE::Networks& networks, Eval::NNUE::Accumulat
ss << board[row] << '\n';
ss << '\n';
accumulators.reset();
auto t = networks.big.trace_evaluate(pos, accumulators, &caches.big);
accumulators->reset();
auto t = networks.big.trace_evaluate(pos, *accumulators, &caches.big);
ss << " NNUE network contributions "
<< (pos.side_to_move() == WHITE ? "(White to move)" : "(Black to move)") << std::endl
src/nnue/nnue_misc.h
+17 -4
View File
@@ -20,8 +20,10 @@
#define NNUE_MISC_H_INCLUDED
#include <cstddef>
#include <memory>
#include <string>
#include "../misc.h"
#include "../types.h"
#include "nnue_architecture.h"
@@ -31,17 +33,17 @@ class Position;
namespace Eval::NNUE {
// EvalFile uses fixed string types because it's part of the network structure which must be trivial.
struct EvalFile {
// Default net name, will use one of the EvalFileDefaultName* macros defined
// in evaluate.h
std::string defaultName;
FixedString<256> defaultName;
// Selected net name, either via uci option or default
std::string current;
FixedString<256> current;
// Net description extracted from the net file
std::string netDescription;
FixedString<256> netDescription;
};
struct NnueEvalTrace {
static_assert(LayerStacks == PSQTBuckets);
@@ -58,4 +60,15 @@ std::string trace(Position& pos, const Networks& networks, AccumulatorCaches& ca
} // namespace Stockfish::Eval::NNUE
} // namespace Stockfish
template<>
struct std::hash<Stockfish::Eval::NNUE::EvalFile> {
std::size_t operator()(const Stockfish::Eval::NNUE::EvalFile& evalFile) const noexcept {
std::size_t h = 0;
Stockfish::hash_combine(h, evalFile.defaultName);
Stockfish::hash_combine(h, evalFile.current);
Stockfish::hash_combine(h, evalFile.netDescription);
return h;
}
};
#endif // #ifndef NNUE_MISC_H_INCLUDED
src/numa.h
+139 -3
View File
@@ -37,7 +37,7 @@
#include <vector>
#include <cstring>
#include "memory.h"
#include "shm.h"
// We support linux very well, but we explicitly do NOT support Android,
// because there is no affected systems, not worth maintaining.
@@ -945,10 +945,11 @@ class NumaConfig {
th.join();
}
private:
std::vector<std::set<CpuIndex>> nodes;
std::map<CpuIndex, NumaIndex> nodeByCpu;
CpuIndex highestCpuIndex;
private:
CpuIndex highestCpuIndex;
bool customAffinity;
@@ -1263,6 +1264,141 @@ class LazyNumaReplicated: public NumaReplicatedBase {
}
};
// Utilizes shared memory.
template<typename T>
class LazyNumaReplicatedSystemWide: public NumaReplicatedBase {
public:
using ReplicatorFuncType = std::function<T(const T&)>;
LazyNumaReplicatedSystemWide(NumaReplicationContext& ctx) :
NumaReplicatedBase(ctx) {
prepare_replicate_from(std::make_unique<T>());
}
LazyNumaReplicatedSystemWide(NumaReplicationContext& ctx, std::unique_ptr<T>&& source) :
NumaReplicatedBase(ctx) {
prepare_replicate_from(std::move(source));
}
LazyNumaReplicatedSystemWide(const LazyNumaReplicatedSystemWide&) = delete;
LazyNumaReplicatedSystemWide(LazyNumaReplicatedSystemWide&& other) noexcept :
NumaReplicatedBase(std::move(other)),
instances(std::exchange(other.instances, {})) {}
LazyNumaReplicatedSystemWide& operator=(const LazyNumaReplicatedSystemWide&) = delete;
LazyNumaReplicatedSystemWide& operator=(LazyNumaReplicatedSystemWide&& other) noexcept {
NumaReplicatedBase::operator=(*this, std::move(other));
instances = std::exchange(other.instances, {});
return *this;
}
LazyNumaReplicatedSystemWide& operator=(std::unique_ptr<T>&& source) {
prepare_replicate_from(std::move(source));
return *this;
}
~LazyNumaReplicatedSystemWide() override = default;
const T& operator[](NumaReplicatedAccessToken token) const {
assert(token.get_numa_index() < instances.size());
ensure_present(token.get_numa_index());
return *(instances[token.get_numa_index()]);
}
const T& operator*() const { return *(instances[0]); }
const T* operator->() const { return &*instances[0]; }
std::vector<std::pair<SystemWideSharedConstantAllocationStatus, std::optional<std::string>>>
get_status_and_errors() const {
std::vector<std::pair<SystemWideSharedConstantAllocationStatus, std::optional<std::string>>>
status;
status.reserve(instances.size());
for (const auto& instance : instances)
{
status.emplace_back(instance.get_status(), instance.get_error_message());
}
return status;
}
template<typename FuncT>
void modify_and_replicate(FuncT&& f) {
auto source = std::make_unique<T>(*instances[0]);
std::forward<FuncT>(f)(*source);
prepare_replicate_from(std::move(source));
}
void on_numa_config_changed() override {
// Use the first one as the source. It doesn't matter which one we use,
// because they all must be identical, but the first one is guaranteed to exist.
auto source = std::make_unique<T>(*instances[0]);
prepare_replicate_from(std::move(source));
}
private:
mutable std::vector<SystemWideSharedConstant<T>> instances;
mutable std::mutex mutex;
std::size_t get_discriminator(NumaIndex idx) const {
const NumaConfig& cfg = get_numa_config();
const NumaConfig& cfg_sys = NumaConfig::from_system(false);
// as a descriminator, locate the hardware/system numadomain this cpuindex belongs to
CpuIndex cpu = *cfg.nodes[idx].begin(); // get a CpuIndex from NumaIndex
NumaIndex sys_idx = cfg_sys.is_cpu_assigned(cpu) ? cfg_sys.nodeByCpu.at(cpu) : 0;
std::string s = cfg_sys.to_string() + "$" + std::to_string(sys_idx);
return std::hash<std::string>{}(s);
}
void ensure_present(NumaIndex idx) const {
assert(idx < instances.size());
if (instances[idx] != nullptr)
return;
assert(idx != 0);
std::unique_lock<std::mutex> lock(mutex);
// Check again for races.
if (instances[idx] != nullptr)
return;
const NumaConfig& cfg = get_numa_config();
cfg.execute_on_numa_node(idx, [this, idx]() {
instances[idx] = SystemWideSharedConstant<T>(*instances[0], get_discriminator(idx));
});
}
void prepare_replicate_from(std::unique_ptr<T>&& source) {
instances.clear();
const NumaConfig& cfg = get_numa_config();
// We just need to make sure the first instance is there.
// Note that we cannot move here as we need to reallocate the data
// on the correct NUMA node.
// Even in the case of a single NUMA node we have to copy since it's shared memory.
if (cfg.requires_memory_replication())
{
assert(cfg.num_numa_nodes() > 0);
cfg.execute_on_numa_node(0, [this, &source]() {
instances.emplace_back(SystemWideSharedConstant<T>(*source, get_discriminator(0)));
});
// Prepare others for lazy init.
instances.resize(cfg.num_numa_nodes());
}
else
{
assert(cfg.num_numa_nodes() == 1);
instances.emplace_back(SystemWideSharedConstant<T>(*source, get_discriminator(0)));
}
}
};
class NumaReplicationContext {
public:
NumaReplicationContext(NumaConfig&& cfg) :
src/search.h
+12 -12
View File
@@ -137,19 +137,19 @@ struct LimitsType {
// The UCI stores the uci options, thread pool, and transposition table.
// This struct is used to easily forward data to the Search::Worker class.
struct SharedState {
SharedState(const OptionsMap& optionsMap,
ThreadPool& threadPool,
TranspositionTable& transpositionTable,
const LazyNumaReplicated<Eval::NNUE::Networks>& nets) :
SharedState(const OptionsMap& optionsMap,
ThreadPool& threadPool,
TranspositionTable& transpositionTable,
const LazyNumaReplicatedSystemWide<Eval::NNUE::Networks>& nets) :
options(optionsMap),
threads(threadPool),
tt(transpositionTable),
networks(nets) {}
const OptionsMap& options;
ThreadPool& threads;
TranspositionTable& tt;
const LazyNumaReplicated<Eval::NNUE::Networks>& networks;
const OptionsMap& options;
ThreadPool& threads;
TranspositionTable& tt;
const LazyNumaReplicatedSystemWide<Eval::NNUE::Networks>& networks;
};
class Worker;
@@ -383,10 +383,10 @@ class Worker {
Tablebases::Config tbConfig;
const OptionsMap& options;
ThreadPool& threads;
TranspositionTable& tt;
const LazyNumaReplicated<Eval::NNUE::Networks>& networks;
const OptionsMap& options;
ThreadPool& threads;
TranspositionTable& tt;
const LazyNumaReplicatedSystemWide<Eval::NNUE::Networks>& networks;
// Used by NNUE
Eval::NNUE::AccumulatorStack accumulatorStack;
src/shm.h
+634
View File
@@ -0,0 +1,634 @@
/*
Stockfish, a UCI chess playing engine derived from Glaurung 2.1
Copyright (C) 2004-2025 The Stockfish developers (see AUTHORS file)
Stockfish is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Stockfish is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef SHM_H_INCLUDED
#define SHM_H_INCLUDED
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <functional>
#include <iomanip>
#include <iostream>
#include <memory>
#include <new>
#include <optional>
#include <sstream>
#include <string>
#include <type_traits>
#include <utility>
#include <variant>
#if !defined(_WIN32) && !defined(__ANDROID__)
#include "shm_linux.h"
#endif
#if defined(__ANDROID__)
#include <limits.h>
#define SF_MAX_SEM_NAME_LEN NAME_MAX
#endif
#include "types.h"
#include "memory.h"
#if defined(_WIN32)
#if _WIN32_WINNT < 0x0601
#undef _WIN32_WINNT
#define _WIN32_WINNT 0x0601 // Force to include needed API prototypes
#endif
#if !defined(NOMINMAX)
#define NOMINMAX
#endif
#include <windows.h>
#else
#include <cstring>
#include <fcntl.h>
#include <pthread.h>
#include <semaphore.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
#endif
#if defined(__APPLE__)
#include <mach-o/dyld.h>
#include <sys/syslimits.h>
#elif defined(__sun)
#include <stdlib.h>
#elif defined(__FreeBSD__)
#include <sys/sysctl.h>
#include <sys/types.h>
#include <unistd.h>
#elif defined(__NetBSD__) || defined(__DragonFly__) || defined(__linux__)
#include <limits.h>
#include <unistd.h>
#endif
namespace Stockfish {
// argv[0] CANNOT be used because we need to identify the executable.
// argv[0] contains the command used to invoke it, which does not involve the full path.
// Just using a path is not fully resilient either, as the executable could
// have changed if it wasn't locked by the OS. Ideally we would hash the executable
// but it's not really that important at this point.
// If the path is longer than 4095 bytes the hash will be computed from an unspecified
// amount of bytes of the path; in particular it can a hash of an empty string.
inline std::string getExecutablePathHash() {
char executable_path[4096] = {0};
std::size_t path_length = 0;
#if defined(_WIN32)
path_length = GetModuleFileNameA(NULL, executable_path, sizeof(executable_path));
#elif defined(__APPLE__)
uint32_t size = sizeof(executable_path);
if (_NSGetExecutablePath(executable_path, &size) == 0)
{
path_length = std::strlen(executable_path);
}
#elif defined(__sun) // Solaris
const char* path = getexecname();
if (path)
{
std::strncpy(executable_path, path, sizeof(executable_path) - 1);
path_length = std::strlen(executable_path);
}
#elif defined(__FreeBSD__)
size_t size = sizeof(executable_path);
int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1};
if (sysctl(mib, 4, executable_path, &size, NULL, 0) == 0)
{
path_length = std::strlen(executable_path);
}
#elif defined(__NetBSD__) || defined(__DragonFly__)
ssize_t len = readlink("/proc/curproc/exe", executable_path, sizeof(executable_path) - 1);
if (len >= 0)
{
executable_path[len] = '\0';
path_length = len;
}
#elif defined(__linux__)
ssize_t len = readlink("/proc/self/exe", executable_path, sizeof(executable_path) - 1);
if (len >= 0)
{
executable_path[len] = '\0';
path_length = len;
}
#endif
// In case of any error the path will be empty.
return std::string(executable_path, path_length);
}
enum class SystemWideSharedConstantAllocationStatus {
NoAllocation,
LocalMemory,
SharedMemory
};
#if defined(_WIN32)
inline std::string GetLastErrorAsString(DWORD error) {
//Get the error message ID, if any.
DWORD errorMessageID = error;
if (errorMessageID == 0)
{
return std::string(); //No error message has been recorded
}
LPSTR messageBuffer = nullptr;
//Ask Win32 to give us the string version of that message ID.
//The parameters we pass in, tell Win32 to create the buffer that holds the message for us (because we don't yet know how long the message string will be).
size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM
| FORMAT_MESSAGE_IGNORE_INSERTS,
NULL, errorMessageID, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
(LPSTR) &messageBuffer, 0, NULL);
//Copy the error message into a std::string.
std::string message(messageBuffer, size);
//Free the Win32's string's buffer.
LocalFree(messageBuffer);
return message;
}
// Utilizes shared memory to store the value. It is deduplicated system-wide (for the single user).
template<typename T>
class SharedMemoryBackend {
public:
enum class Status {
Success,
LargePageAllocationError,
FileMappingError,
MapViewError,
MutexCreateError,
MutexWaitError,
MutexReleaseError,
NotInitialized
};
static constexpr DWORD IS_INITIALIZED_VALUE = 1;
SharedMemoryBackend() :
status(Status::NotInitialized) {};
SharedMemoryBackend(const std::string& shm_name, const T& value) :
status(Status::NotInitialized) {
initialize(shm_name, value);
}
bool is_valid() const { return status == Status::Success; }
std::optional<std::string> get_error_message() const {
switch (status)
{
case Status::Success :
return std::nullopt;
case Status::LargePageAllocationError :
return "Failed to allocate large page memory";
case Status::FileMappingError :
return "Failed to create file mapping: " + last_error_message;
case Status::MapViewError :
return "Failed to map view: " + last_error_message;
case Status::MutexCreateError :
return "Failed to create mutex: " + last_error_message;
case Status::MutexWaitError :
return "Failed to wait on mutex: " + last_error_message;
case Status::MutexReleaseError :
return "Failed to release mutex: " + last_error_message;
case Status::NotInitialized :
return "Not initialized";
default :
return "Unknown error";
}
}
void* get() const { return is_valid() ? pMap : nullptr; }
~SharedMemoryBackend() { cleanup(); }
SharedMemoryBackend(const SharedMemoryBackend&) = delete;
SharedMemoryBackend& operator=(const SharedMemoryBackend&) = delete;
SharedMemoryBackend(SharedMemoryBackend&& other) noexcept :
pMap(other.pMap),
hMapFile(other.hMapFile),
status(other.status),
last_error_message(std::move(other.last_error_message)) {
other.pMap = nullptr;
other.hMapFile = 0;
other.status = Status::NotInitialized;
}
SharedMemoryBackend& operator=(SharedMemoryBackend&& other) noexcept {
if (this != &other)
{
cleanup();
pMap = other.pMap;
hMapFile = other.hMapFile;
status = other.status;
last_error_message = std::move(other.last_error_message);
other.pMap = nullptr;
other.hMapFile = 0;
other.status = Status::NotInitialized;
}
return *this;
}
SystemWideSharedConstantAllocationStatus get_status() const {
return status == Status::Success ? SystemWideSharedConstantAllocationStatus::SharedMemory
: SystemWideSharedConstantAllocationStatus::NoAllocation;
}
private:
void initialize(const std::string& shm_name, const T& value) {
const size_t total_size = sizeof(T) + sizeof(IS_INITIALIZED_VALUE);
// Try allocating with large pages first.
hMapFile = windows_try_with_large_page_priviliges(
[&](size_t largePageSize) {
const size_t total_size_aligned =
(total_size + largePageSize - 1) / largePageSize * largePageSize;
#if defined(_WIN64)
DWORD total_size_low = total_size_aligned & 0xFFFFFFFFu;
DWORD total_size_high = total_size_aligned >> 32u;
#else
DWORD total_size_low = total_size_aligned;
DWORD total_size_high = 0;
#endif
return CreateFileMappingA(INVALID_HANDLE_VALUE, NULL,
PAGE_READWRITE | SEC_COMMIT | SEC_LARGE_PAGES,
total_size_high, total_size_low, shm_name.c_str());
},
[]() { return (void*) nullptr; });
// Fallback to normal allocation if no large pages available.
if (!hMapFile)
{
hMapFile = CreateFileMappingA(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0,
static_cast<DWORD>(total_size), shm_name.c_str());
}
if (!hMapFile)
{
const DWORD err = GetLastError();
last_error_message = GetLastErrorAsString(err);
status = Status::FileMappingError;
return;
}
pMap = MapViewOfFile(hMapFile, FILE_MAP_ALL_ACCESS, 0, 0, total_size);
if (!pMap)
{
const DWORD err = GetLastError();
last_error_message = GetLastErrorAsString(err);
status = Status::MapViewError;
cleanup_partial();
return;
}
// Use named mutex to ensure only one initializer
std::string mutex_name = shm_name + "$mutex";
HANDLE hMutex = CreateMutexA(NULL, FALSE, mutex_name.c_str());
if (!hMutex)
{
const DWORD err = GetLastError();
last_error_message = GetLastErrorAsString(err);
status = Status::MutexCreateError;
cleanup_partial();
return;
}
DWORD wait_result = WaitForSingleObject(hMutex, INFINITE);
if (wait_result != WAIT_OBJECT_0)
{
const DWORD err = GetLastError();
last_error_message = GetLastErrorAsString(err);
status = Status::MutexWaitError;
CloseHandle(hMutex);
cleanup_partial();
return;
}
// Crucially, we place the object first to ensure alignment.
volatile DWORD* is_initialized =
std::launder(reinterpret_cast<DWORD*>(reinterpret_cast<char*>(pMap) + sizeof(T)));
T* object = std::launder(reinterpret_cast<T*>(pMap));
if (*is_initialized != IS_INITIALIZED_VALUE)
{
// First time initialization, message for debug purposes
new (object) T{value};
*is_initialized = IS_INITIALIZED_VALUE;
}
BOOL release_result = ReleaseMutex(hMutex);
CloseHandle(hMutex);
if (!release_result)
{
const DWORD err = GetLastError();
last_error_message = GetLastErrorAsString(err);
status = Status::MutexReleaseError;
cleanup_partial();
return;
}
status = Status::Success;
}
void cleanup_partial() {
if (pMap != nullptr)
{
UnmapViewOfFile(pMap);
pMap = nullptr;
}
if (hMapFile)
{
CloseHandle(hMapFile);
hMapFile = 0;
}
}
void cleanup() {
if (pMap != nullptr)
{
UnmapViewOfFile(pMap);
pMap = nullptr;
}
if (hMapFile)
{
CloseHandle(hMapFile);
hMapFile = 0;
}
}
void* pMap = nullptr;
HANDLE hMapFile = 0;
Status status = Status::NotInitialized;
std::string last_error_message;
};
#elif !defined(__ANDROID__)
template<typename T>
class SharedMemoryBackend {
public:
SharedMemoryBackend() = default;
SharedMemoryBackend(const std::string& shm_name, const T& value) :
shm1(shm::create_shared<T>(shm_name, value)) {}
void* get() const {
const T* ptr = &shm1->get();
return reinterpret_cast<void*>(const_cast<T*>(ptr));
}
bool is_valid() const { return shm1 && shm1->is_open() && shm1->is_initialized(); }
SystemWideSharedConstantAllocationStatus get_status() const {
return is_valid() ? SystemWideSharedConstantAllocationStatus::SharedMemory
: SystemWideSharedConstantAllocationStatus::NoAllocation;
}
std::optional<std::string> get_error_message() const {
if (!shm1)
return "Shared memory not initialized";
if (!shm1->is_open())
return "Shared memory is not open";
if (!shm1->is_initialized())
return "Not initialized";
return std::nullopt;
}
private:
std::optional<shm::SharedMemory<T>> shm1;
};
#else
// For systems that don't have shared memory, or support is troublesome.
// The way fallback is done is that we need a dummy backend.
template<typename T>
class SharedMemoryBackend {
public:
SharedMemoryBackend() = default;
SharedMemoryBackend(const std::string& shm_name, const T& value) {}
void* get() const { return nullptr; }
bool is_valid() const { return false; }
SystemWideSharedConstantAllocationStatus get_status() const {
return SystemWideSharedConstantAllocationStatus::NoAllocation;
}
std::optional<std::string> get_error_message() const { return "Dummy SharedMemoryBackend"; }
};
#endif
template<typename T>
struct SharedMemoryBackendFallback {
SharedMemoryBackendFallback() = default;
SharedMemoryBackendFallback(const std::string&, const T& value) :
fallback_object(make_unique_large_page<T>(value)) {}
void* get() const { return fallback_object.get(); }
SharedMemoryBackendFallback(const SharedMemoryBackendFallback&) = delete;
SharedMemoryBackendFallback& operator=(const SharedMemoryBackendFallback&) = delete;
SharedMemoryBackendFallback(SharedMemoryBackendFallback&& other) noexcept :
fallback_object(std::move(other.fallback_object)) {}
SharedMemoryBackendFallback& operator=(SharedMemoryBackendFallback&& other) noexcept {
fallback_object = std::move(other.fallback_object);
return *this;
}
SystemWideSharedConstantAllocationStatus get_status() const {
return fallback_object == nullptr ? SystemWideSharedConstantAllocationStatus::NoAllocation
: SystemWideSharedConstantAllocationStatus::LocalMemory;
}
std::optional<std::string> get_error_message() const {
if (fallback_object == nullptr)
return "Not initialized";
return "Shared memory not supported by the OS. Local allocation fallback.";
}
private:
LargePagePtr<T> fallback_object;
};
// Platform-independent wrapper
template<typename T>
struct SystemWideSharedConstant {
private:
static std::string createHashString(const std::string& input) {
size_t hash = std::hash<std::string>{}(input);
std::stringstream ss;
ss << std::hex << std::setfill('0') << hash;
return ss.str();
}
public:
// We can't run the destructor because it may be in a completely different process.
// The object stored must also be obviously in-line but we can't check for that, other than some basic checks that cover most cases.
static_assert(std::is_trivially_destructible_v<T>);
static_assert(std::is_trivially_move_constructible_v<T>);
static_assert(std::is_trivially_copy_constructible_v<T>);
SystemWideSharedConstant() = default;
// Content is addressed by its hash. An additional discriminator can be added to account for differences
// that are not present in the content, for example NUMA node allocation.
SystemWideSharedConstant(const T& value, std::size_t discriminator = 0) {
std::size_t content_hash = std::hash<T>{}(value);
std::size_t executable_hash = std::hash<std::string>{}(getExecutablePathHash());
std::string shm_name = std::string("Local\\sf_") + std::to_string(content_hash) + "$"
+ std::to_string(executable_hash) + "$"
+ std::to_string(discriminator);
#if !defined(_WIN32)
// POSIX shared memory names must start with a slash
shm_name = "/sf_" + createHashString(shm_name);
// hash name and make sure it is not longer than SF_MAX_SEM_NAME_LEN
if (shm_name.size() > SF_MAX_SEM_NAME_LEN)
{
shm_name = shm_name.substr(0, SF_MAX_SEM_NAME_LEN - 1);
}
#endif
SharedMemoryBackend<T> shm_backend(shm_name, value);
if (shm_backend.is_valid())
{
backend = std::move(shm_backend);
}
else
{
backend = SharedMemoryBackendFallback<T>(shm_name, value);
}
}
SystemWideSharedConstant(const SystemWideSharedConstant&) = delete;
SystemWideSharedConstant& operator=(const SystemWideSharedConstant&) = delete;
SystemWideSharedConstant(SystemWideSharedConstant&& other) noexcept :
backend(std::move(other.backend)) {}
SystemWideSharedConstant& operator=(SystemWideSharedConstant&& other) noexcept {
backend = std::move(other.backend);
return *this;
}
const T& operator*() const { return *std::launder(reinterpret_cast<const T*>(get_ptr())); }
bool operator==(std::nullptr_t) const noexcept { return get_ptr() == nullptr; }
bool operator!=(std::nullptr_t) const noexcept { return get_ptr() != nullptr; }
SystemWideSharedConstantAllocationStatus get_status() const {
return std::visit(
[](const auto& end) -> SystemWideSharedConstantAllocationStatus {
if constexpr (std::is_same_v<std::decay_t<decltype(end)>, std::monostate>)
{
return SystemWideSharedConstantAllocationStatus::NoAllocation;
}
else
{
return end.get_status();
}
},
backend);
}
std::optional<std::string> get_error_message() const {
return std::visit(
[](const auto& end) -> std::optional<std::string> {
if constexpr (std::is_same_v<std::decay_t<decltype(end)>, std::monostate>)
{
return std::nullopt;
}
else
{
return end.get_error_message();
}
},
backend);
}
private:
auto get_ptr() const {
return std::visit(
[](const auto& end) -> void* {
if constexpr (std::is_same_v<std::decay_t<decltype(end)>, std::monostate>)
{
return nullptr;
}
else
{
return end.get();
}
},
backend);
}
std::variant<std::monostate, SharedMemoryBackend<T>, SharedMemoryBackendFallback<T>> backend;
};
} // namespace Stockfish
#endif // #ifndef SHM_H_INCLUDED
src/shm_linux.h
+658
View File
@@ -0,0 +1,658 @@
/*
Stockfish, a UCI chess playing engine derived from Glaurung 2.1
Copyright (C) 2004-2025 The Stockfish developers (see AUTHORS file)
Stockfish is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Stockfish is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef SHM_LINUX_H_INCLUDED
#define SHM_LINUX_H_INCLUDED
#include <atomic>
#include <cassert>
#include <cerrno>
#include <cstdlib>
#include <cstring>
#include <cstdio>
#include <dirent.h>
#include <mutex>
#include <new>
#include <optional>
#include <pthread.h>
#include <string>
#include <inttypes.h>
#include <type_traits>
#include <unordered_set>
#include <fcntl.h>
#include <signal.h>
#include <sys/file.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <unistd.h>
#if defined(__NetBSD__) || defined(__DragonFly__) || defined(__linux__)
#include <limits.h>
#define SF_MAX_SEM_NAME_LEN NAME_MAX
#elif defined(__APPLE__)
#define SF_MAX_SEM_NAME_LEN 31
#else
#define SF_MAX_SEM_NAME_LEN 255
#endif
namespace Stockfish::shm {
namespace detail {
struct ShmHeader {
static constexpr uint32_t SHM_MAGIC = 0xAD5F1A12;
pthread_mutex_t mutex;
std::atomic<uint32_t> ref_count{0};
std::atomic<bool> initialized{false};
uint32_t magic = SHM_MAGIC;
};
class SharedMemoryBase {
public:
virtual ~SharedMemoryBase() = default;
virtual void close() noexcept = 0;
virtual const std::string& name() const noexcept = 0;
};
class SharedMemoryRegistry {
private:
static std::mutex registry_mutex_;
static std::unordered_set<SharedMemoryBase*> active_instances_;
public:
static void register_instance(SharedMemoryBase* instance) {
std::scoped_lock lock(registry_mutex_);
active_instances_.insert(instance);
}
static void unregister_instance(SharedMemoryBase* instance) {
std::scoped_lock lock(registry_mutex_);
active_instances_.erase(instance);
}
static void cleanup_all() noexcept {
std::scoped_lock lock(registry_mutex_);
for (auto* instance : active_instances_)
instance->close();
active_instances_.clear();
}
};
inline std::mutex SharedMemoryRegistry::registry_mutex_;
inline std::unordered_set<SharedMemoryBase*> SharedMemoryRegistry::active_instances_;
class CleanupHooks {
private:
static std::once_flag register_once_;
static void handle_signal(int sig) noexcept {
SharedMemoryRegistry::cleanup_all();
_Exit(128 + sig);
}
static void register_signal_handlers() noexcept {
std::atexit([]() { SharedMemoryRegistry::cleanup_all(); });
constexpr int signals[] = {SIGHUP, SIGINT, SIGQUIT, SIGILL, SIGABRT, SIGFPE,
SIGSEGV, SIGTERM, SIGBUS, SIGSYS, SIGXCPU, SIGXFSZ};
struct sigaction sa;
sa.sa_handler = handle_signal;
sigemptyset(&sa.sa_mask);
sa.sa_flags = 0;
for (int sig : signals)
sigaction(sig, &sa, nullptr);
}
public:
static void ensure_registered() noexcept {
std::call_once(register_once_, register_signal_handlers);
}
};
inline std::once_flag CleanupHooks::register_once_;
inline int portable_fallocate(int fd, off_t offset, off_t length) {
#ifdef __APPLE__
fstore_t store = {F_ALLOCATECONTIG, F_PEOFPOSMODE, offset, length, 0};
int ret = fcntl(fd, F_PREALLOCATE, &store);
if (ret == -1)
{
store.fst_flags = F_ALLOCATEALL;
ret = fcntl(fd, F_PREALLOCATE, &store);
}
if (ret != -1)
ret = ftruncate(fd, offset + length);
return ret;
#else
return posix_fallocate(fd, offset, length);
#endif
}
} // namespace detail
template<typename T>
class SharedMemory: public detail::SharedMemoryBase {
static_assert(std::is_trivially_copyable_v<T>, "T must be trivially copyable");
static_assert(!std::is_pointer_v<T>, "T cannot be a pointer type");
private:
std::string name_;
int fd_ = -1;
void* mapped_ptr_ = nullptr;
T* data_ptr_ = nullptr;
detail::ShmHeader* header_ptr_ = nullptr;
size_t total_size_ = 0;
std::string sentinel_base_;
std::string sentinel_path_;
static constexpr size_t calculate_total_size() noexcept {
return sizeof(T) + sizeof(detail::ShmHeader);
}
static std::string make_sentinel_base(const std::string& name) {
uint64_t hash = std::hash<std::string>{}(name);
char buf[32];
std::snprintf(buf, sizeof(buf), "sfshm_%016" PRIx64, static_cast<uint64_t>(hash));
return buf;
}
public:
explicit SharedMemory(const std::string& name) noexcept :
name_(name),
total_size_(calculate_total_size()),
sentinel_base_(make_sentinel_base(name)) {}
~SharedMemory() noexcept override {
detail::SharedMemoryRegistry::unregister_instance(this);
close();
}
SharedMemory(const SharedMemory&) = delete;
SharedMemory& operator=(const SharedMemory&) = delete;
SharedMemory(SharedMemory&& other) noexcept :
name_(std::move(other.name_)),
fd_(other.fd_),
mapped_ptr_(other.mapped_ptr_),
data_ptr_(other.data_ptr_),
header_ptr_(other.header_ptr_),
total_size_(other.total_size_),
sentinel_base_(std::move(other.sentinel_base_)),
sentinel_path_(std::move(other.sentinel_path_)) {
detail::SharedMemoryRegistry::unregister_instance(&other);
detail::SharedMemoryRegistry::register_instance(this);
other.reset();
}
SharedMemory& operator=(SharedMemory&& other) noexcept {
if (this != &other)
{
detail::SharedMemoryRegistry::unregister_instance(this);
close();
name_ = std::move(other.name_);
fd_ = other.fd_;
mapped_ptr_ = other.mapped_ptr_;
data_ptr_ = other.data_ptr_;
header_ptr_ = other.header_ptr_;
total_size_ = other.total_size_;
sentinel_base_ = std::move(other.sentinel_base_);
sentinel_path_ = std::move(other.sentinel_path_);
detail::SharedMemoryRegistry::unregister_instance(&other);
detail::SharedMemoryRegistry::register_instance(this);
other.reset();
}
return *this;
}
[[nodiscard]] bool open(const T& initial_value) noexcept {
detail::CleanupHooks::ensure_registered();
bool retried_stale = false;
while (true)
{
if (is_open())
return false;
bool created_new = false;
fd_ = shm_open(name_.c_str(), O_CREAT | O_EXCL | O_RDWR, 0666);
if (fd_ == -1)
{
fd_ = shm_open(name_.c_str(), O_RDWR, 0666);
if (fd_ == -1)
return false;
}
else
created_new = true;
if (!lock_file(LOCK_EX))
{
::close(fd_);
reset();
return false;
}
bool invalid_header = false;
bool success =
created_new ? setup_new_region(initial_value) : setup_existing_region(invalid_header);
if (!success)
{
if (created_new || invalid_header)
shm_unlink(name_.c_str());
if (mapped_ptr_)
unmap_region();
unlock_file();
::close(fd_);
reset();
if (!created_new && invalid_header && !retried_stale)
{
retried_stale = true;
continue;
}
return false;
}
if (!lock_shared_mutex())
{
if (created_new)
shm_unlink(name_.c_str());
if (mapped_ptr_)
unmap_region();
unlock_file();
::close(fd_);
reset();
if (!created_new && !retried_stale)
{
retried_stale = true;
continue;
}
return false;
}
if (!create_sentinel_file_locked())
{
unlock_shared_mutex();
unmap_region();
if (created_new)
shm_unlink(name_.c_str());
unlock_file();
::close(fd_);
reset();
return false;
}
header_ptr_->ref_count.fetch_add(1, std::memory_order_acq_rel);
unlock_shared_mutex();
unlock_file();
detail::SharedMemoryRegistry::register_instance(this);
return true;
}
}
void close() noexcept override {
if (fd_ == -1 && mapped_ptr_ == nullptr)
return;
bool remove_region = false;
bool file_locked = lock_file(LOCK_EX);
bool mutex_locked = false;
if (file_locked && header_ptr_ != nullptr)
mutex_locked = lock_shared_mutex();
if (mutex_locked)
{
if (header_ptr_)
{
header_ptr_->ref_count.fetch_sub(1, std::memory_order_acq_rel);
}
remove_sentinel_file();
remove_region = !has_other_live_sentinels_locked();
unlock_shared_mutex();
}
else
{
remove_sentinel_file();
decrement_refcount_relaxed();
}
unmap_region();
if (remove_region)
shm_unlink(name_.c_str());
if (file_locked)
unlock_file();
if (fd_ != -1)
{
::close(fd_);
fd_ = -1;
}
reset();
}
const std::string& name() const noexcept override { return name_; }
[[nodiscard]] bool is_open() const noexcept { return fd_ != -1 && mapped_ptr_ && data_ptr_; }
[[nodiscard]] const T& get() const noexcept { return *data_ptr_; }
[[nodiscard]] const T* operator->() const noexcept { return data_ptr_; }
[[nodiscard]] const T& operator*() const noexcept { return *data_ptr_; }
[[nodiscard]] uint32_t ref_count() const noexcept {
return header_ptr_ ? header_ptr_->ref_count.load(std::memory_order_acquire) : 0;
}
[[nodiscard]] bool is_initialized() const noexcept {
return header_ptr_ ? header_ptr_->initialized.load(std::memory_order_acquire) : false;
}
static void cleanup_all_instances() noexcept { detail::SharedMemoryRegistry::cleanup_all(); }
private:
void reset() noexcept {
fd_ = -1;
mapped_ptr_ = nullptr;
data_ptr_ = nullptr;
header_ptr_ = nullptr;
sentinel_path_.clear();
}
void unmap_region() noexcept {
if (mapped_ptr_)
{
munmap(mapped_ptr_, total_size_);
mapped_ptr_ = nullptr;
data_ptr_ = nullptr;
header_ptr_ = nullptr;
}
}
[[nodiscard]] bool lock_file(int operation) noexcept {
if (fd_ == -1)
return false;
while (flock(fd_, operation) == -1)
{
if (errno == EINTR)
continue;
return false;
}
return true;
}
void unlock_file() noexcept {
if (fd_ == -1)
return;
while (flock(fd_, LOCK_UN) == -1)
{
if (errno == EINTR)
continue;
break;
}
}
std::string sentinel_full_path(pid_t pid) const {
std::string path = "/dev/shm/";
path += sentinel_base_;
path.push_back('.');
path += std::to_string(pid);
return path;
}
void decrement_refcount_relaxed() noexcept {
if (!header_ptr_)
return;
uint32_t expected = header_ptr_->ref_count.load(std::memory_order_relaxed);
while (expected != 0
&& !header_ptr_->ref_count.compare_exchange_weak(
expected, expected - 1, std::memory_order_acq_rel, std::memory_order_relaxed))
{}
}
bool create_sentinel_file_locked() noexcept {
if (!header_ptr_)
return false;
const pid_t self_pid = getpid();
sentinel_path_ = sentinel_full_path(self_pid);
for (int attempt = 0; attempt < 2; ++attempt)
{
int fd = ::open(sentinel_path_.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0600);
if (fd != -1)
{
::close(fd);
return true;
}
if (errno == EEXIST)
{
::unlink(sentinel_path_.c_str());
decrement_refcount_relaxed();
continue;
}
break;
}
sentinel_path_.clear();
return false;
}
void remove_sentinel_file() noexcept {
if (!sentinel_path_.empty())
{
::unlink(sentinel_path_.c_str());
sentinel_path_.clear();
}
}
static bool pid_is_alive(pid_t pid) noexcept {
if (pid <= 0)
return false;
if (kill(pid, 0) == 0)
return true;
return errno == EPERM;
}
[[nodiscard]] bool initialize_shared_mutex() noexcept {
if (!header_ptr_)
return false;
pthread_mutexattr_t attr;
if (pthread_mutexattr_init(&attr) != 0)
return false;
bool success = pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED) == 0;
#ifdef PTHREAD_MUTEX_ROBUST
if (success)
success = pthread_mutexattr_setrobust(&attr, PTHREAD_MUTEX_ROBUST) == 0;
#endif
if (success)
success = pthread_mutex_init(&header_ptr_->mutex, &attr) == 0;
pthread_mutexattr_destroy(&attr);
return success;
}
[[nodiscard]] bool lock_shared_mutex() noexcept {
if (!header_ptr_)
return false;
while (true)
{
int rc = pthread_mutex_lock(&header_ptr_->mutex);
if (rc == 0)
return true;
#ifdef PTHREAD_MUTEX_ROBUST
if (rc == EOWNERDEAD)
{
if (pthread_mutex_consistent(&header_ptr_->mutex) == 0)
return true;
return false;
}
#endif
if (rc == EINTR)
continue;
return false;
}
}
void unlock_shared_mutex() noexcept {
if (header_ptr_)
pthread_mutex_unlock(&header_ptr_->mutex);
}
bool has_other_live_sentinels_locked() const noexcept {
DIR* dir = opendir("/dev/shm");
if (!dir)
return false;
std::string prefix = sentinel_base_ + ".";
bool found = false;
while (dirent* entry = readdir(dir))
{
std::string name = entry->d_name;
if (name.rfind(prefix, 0) != 0)
continue;
auto pid_str = name.substr(prefix.size());
char* end = nullptr;
long value = std::strtol(pid_str.c_str(), &end, 10);
if (!end || *end != '\0')
continue;
pid_t pid = static_cast<pid_t>(value);
if (pid_is_alive(pid))
{
found = true;
break;
}
std::string stale_path = std::string("/dev/shm/") + name;
::unlink(stale_path.c_str());
const_cast<SharedMemory*>(this)->decrement_refcount_relaxed();
}
closedir(dir);
return found;
}
[[nodiscard]] bool setup_new_region(const T& initial_value) noexcept {
if (ftruncate(fd_, static_cast<off_t>(total_size_)) == -1)
return false;
if (detail::portable_fallocate(fd_, 0, static_cast<off_t>(total_size_)) != 0)
return false;
mapped_ptr_ = mmap(nullptr, total_size_, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0);
if (mapped_ptr_ == MAP_FAILED)
{
mapped_ptr_ = nullptr;
return false;
}
data_ptr_ = static_cast<T*>(mapped_ptr_);
header_ptr_ =
reinterpret_cast<detail::ShmHeader*>(static_cast<char*>(mapped_ptr_) + sizeof(T));
new (header_ptr_) detail::ShmHeader{};
new (data_ptr_) T{initial_value};
if (!initialize_shared_mutex())
return false;
header_ptr_->ref_count.store(0, std::memory_order_release);
header_ptr_->initialized.store(true, std::memory_order_release);
return true;
}
[[nodiscard]] bool setup_existing_region(bool& invalid_header) noexcept {
invalid_header = false;
struct stat st;
fstat(fd_, &st);
if (static_cast<size_t>(st.st_size) < total_size_)
{
invalid_header = true;
return false;
}
mapped_ptr_ = mmap(nullptr, total_size_, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0);
if (mapped_ptr_ == MAP_FAILED)
{
mapped_ptr_ = nullptr;
return false;
}
data_ptr_ = static_cast<T*>(mapped_ptr_);
header_ptr_ = std::launder(
reinterpret_cast<detail::ShmHeader*>(static_cast<char*>(mapped_ptr_) + sizeof(T)));
if (!header_ptr_->initialized.load(std::memory_order_acquire)
|| header_ptr_->magic != detail::ShmHeader::SHM_MAGIC)
{
invalid_header = true;
unmap_region();
return false;
}
return true;
}
};
template<typename T>
[[nodiscard]] std::optional<SharedMemory<T>> create_shared(const std::string& name,
const T& initial_value) noexcept {
SharedMemory<T> shm(name);
if (shm.open(initial_value))
return shm;
return std::nullopt;
}
} // namespace Stockfish::shm
#endif // #ifndef SHM_LINUX_H_INCLUDED