diff --git a/src/engine.cpp b/src/engine.cpp index 01927fc84..dfd2099d1 100644 --- a/src/engine.cpp +++ b/src/engine.cpp @@ -52,9 +52,15 @@ constexpr auto StartFEN = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq constexpr int MaxHashMB = Is64Bit ? 33554432 : 2048; int MaxThreads = std::max(1024, 4 * int(get_hardware_concurrency())); +// The default configuration will attempt to group L3 domains up to 32 threads. +// This size was found to be a good balance between the Elo gain of increased +// history sharing and the speed loss from more cross-cache accesses (see +// PR#6526). The user can always explicitly override this behavior. +constexpr NumaAutoPolicy DefaultNumaPolicy = BundledL3Policy{32}; + Engine::Engine(std::optional path) : binaryDirectory(path ? CommandLine::get_binary_directory(*path) : ""), - numaContext(NumaConfig::from_system()), + numaContext(NumaConfig::from_system(DefaultNumaPolicy)), states(new std::deque(1)), threads(), networks(numaContext, @@ -213,12 +219,12 @@ void Engine::set_position(const std::string& fen, const std::vector void Engine::set_numa_config_from_option(const std::string& o) { if (o == "auto" || o == "system") { - numaContext.set_numa_config(NumaConfig::from_system()); + numaContext.set_numa_config(NumaConfig::from_system(DefaultNumaPolicy)); } else if (o == "hardware") { // Don't respect affinity set in the system. - numaContext.set_numa_config(NumaConfig::from_system(false)); + numaContext.set_numa_config(NumaConfig::from_system(DefaultNumaPolicy, false)); } else if (o == "none") { diff --git a/src/numa.h b/src/numa.h index 99169c211..df11f0286 100644 --- a/src/numa.h +++ b/src/numa.h @@ -447,14 +447,39 @@ class NumaReplicatedAccessToken { NumaIndex n; }; +struct L3Domain { + NumaIndex systemNumaIndex{}; + std::set cpus{}; +}; + +// Use system NUMA nodes +struct SystemNumaPolicy {}; +// Use system-reported L3 domains +struct L3DomainsPolicy {}; +// Group system-reported L3 domains until they reach bundleSize +struct BundledL3Policy { + size_t bundleSize; +}; + +using NumaAutoPolicy = std::variant; + // Designed as immutable, because there is no good reason to alter an already // existing config in a way that doesn't require recreating it completely, and // it would be complex and expensive to maintain class invariants. // The CPU (processor) numbers always correspond to the actual numbering used // by the system. The NUMA node numbers MAY NOT correspond to the system's -// numbering of the NUMA nodes. In particular, empty nodes may be removed, or -// the user may create custom nodes. It is guaranteed that NUMA nodes are NOT -// empty: every node exposed by NumaConfig has at least one processor assigned. +// numbering of the NUMA nodes. In particular, by default, if the processor has +// non-uniform cache access within a NUMA node (i.e., a non-unified L3 cache structure), +// then L3 domains within a system NUMA node will be used to subdivide it +// into multiple logical NUMA nodes in the config. Additionally, empty nodes may +// be removed, or the user may create custom nodes. +// +// As a special case, when performing system-wide replication of read-only data +// (i.e., LazyNumaReplicatedSystemWide), the system NUMA node is used, rather than +// custom or L3-aware nodes. See that class's get_discriminator() function. +// +// It is guaranteed that NUMA nodes are NOT empty: every node exposed by NumaConfig +// has at least one processor assigned. // // We use startup affinities so as not to modify its own behaviour in time. // @@ -469,78 +494,19 @@ class NumaConfig { add_cpu_range_to_node(NumaIndex{0}, CpuIndex{0}, numCpus - 1); } - // This function queries the system for the mapping of processors to NUMA nodes. - // On Linux we read from standardized kernel sysfs, with a fallback to single NUMA - // node. On Windows we utilize GetNumaProcessorNodeEx, which has its quirks, see - // comment for Windows implementation of get_process_affinity. - static NumaConfig from_system([[maybe_unused]] bool respectProcessAffinity = true) { + // This function gets a NumaConfig based on the system's provided information. + // The available policies are documented above. + static NumaConfig from_system([[maybe_unused]] const NumaAutoPolicy& policy, + bool respectProcessAffinity = true) { NumaConfig cfg = empty(); -#if defined(__linux__) && !defined(__ANDROID__) +#if !((defined(__linux__) && !defined(__ANDROID__)) || defined(_WIN64)) + // Fallback for unsupported systems. + for (CpuIndex c = 0; c < SYSTEM_THREADS_NB; ++c) + cfg.add_cpu_to_node(NumaIndex{0}, c); +#else - std::set allowedCpus; - - if (respectProcessAffinity) - allowedCpus = STARTUP_PROCESSOR_AFFINITY; - - auto is_cpu_allowed = [respectProcessAffinity, &allowedCpus](CpuIndex c) { - return !respectProcessAffinity || allowedCpus.count(c) == 1; - }; - - // On Linux things are straightforward, since there's no processor groups and - // any thread can be scheduled on all processors. - // We try to gather this information from the sysfs first - // https://www.kernel.org/doc/Documentation/ABI/stable/sysfs-devices-node - - bool useFallback = false; - auto fallback = [&]() { - useFallback = true; - cfg = empty(); - }; - - // /sys/devices/system/node/online contains information about active NUMA nodes - auto nodeIdsStr = read_file_to_string("/sys/devices/system/node/online"); - if (!nodeIdsStr.has_value() || nodeIdsStr->empty()) - { - fallback(); - } - else - { - remove_whitespace(*nodeIdsStr); - for (size_t n : indices_from_shortened_string(*nodeIdsStr)) - { - // /sys/devices/system/node/node.../cpulist - std::string path = - std::string("/sys/devices/system/node/node") + std::to_string(n) + "/cpulist"; - auto cpuIdsStr = read_file_to_string(path); - // Now, we only bail if the file does not exist. Some nodes may be - // empty, that's fine. An empty node still has a file that appears - // to have some whitespace, so we need to handle that. - if (!cpuIdsStr.has_value()) - { - fallback(); - break; - } - else - { - remove_whitespace(*cpuIdsStr); - for (size_t c : indices_from_shortened_string(*cpuIdsStr)) - { - if (is_cpu_allowed(c)) - cfg.add_cpu_to_node(n, c); - } - } - } - } - - if (useFallback) - { - for (CpuIndex c = 0; c < SYSTEM_THREADS_NB; ++c) - if (is_cpu_allowed(c)) - cfg.add_cpu_to_node(NumaIndex{0}, c); - } - -#elif defined(_WIN64) + #if defined(_WIN64) std::optional> allowedCpus; @@ -555,28 +521,38 @@ class NumaConfig { return !allowedCpus.has_value() || allowedCpus->count(c) == 1; }; - WORD numProcGroups = GetActiveProcessorGroupCount(); - for (WORD procGroup = 0; procGroup < numProcGroups; ++procGroup) - { - for (BYTE number = 0; number < WIN_PROCESSOR_GROUP_SIZE; ++number) - { - PROCESSOR_NUMBER procnum; - procnum.Group = procGroup; - procnum.Number = number; - procnum.Reserved = 0; - USHORT nodeNumber; + #elif defined(__linux__) && !defined(__ANDROID__) - const BOOL status = GetNumaProcessorNodeEx(&procnum, &nodeNumber); - const CpuIndex c = static_cast(procGroup) * WIN_PROCESSOR_GROUP_SIZE - + static_cast(number); - if (status != 0 && nodeNumber != std::numeric_limits::max() - && is_cpu_allowed(c)) - { - cfg.add_cpu_to_node(nodeNumber, c); - } + std::set allowedCpus; + + if (respectProcessAffinity) + allowedCpus = STARTUP_PROCESSOR_AFFINITY; + + auto is_cpu_allowed = [respectProcessAffinity, &allowedCpus](CpuIndex c) { + return !respectProcessAffinity || allowedCpus.count(c) == 1; + }; + + #endif + + bool l3Success = false; + if (!std::holds_alternative(policy)) + { + size_t l3BundleSize = 0; + if (const auto* v = std::get_if(&policy)) + { + l3BundleSize = v->bundleSize; + } + if (auto l3Cfg = + try_get_l3_aware_config(respectProcessAffinity, l3BundleSize, is_cpu_allowed)) + { + cfg = std::move(*l3Cfg); + l3Success = true; } } + if (!l3Success) + cfg = from_system_numa(respectProcessAffinity, is_cpu_allowed); + #if defined(_WIN64) // Split the NUMA nodes to be contained within a group if necessary. // This is needed between Windows 10 Build 20348 and Windows 11, because // the new NUMA allocation behaviour was introduced while there was @@ -623,12 +599,7 @@ class NumaConfig { cfg = std::move(splitCfg); } - -#else - - // Fallback for unsupported systems. - for (CpuIndex c = 0; c < SYSTEM_THREADS_NB; ++c) - cfg.add_cpu_to_node(NumaIndex{0}, c); + #endif #endif @@ -1041,6 +1012,239 @@ class NumaConfig { return indices; } + + // This function queries the system for the mapping of processors to NUMA nodes. + // On Linux we read from standardized kernel sysfs, with a fallback to single NUMA + // node. On Windows we utilize GetNumaProcessorNodeEx, which has its quirks, see + // comment for Windows implementation of get_process_affinity. + template + static NumaConfig from_system_numa([[maybe_unused]] bool respectProcessAffinity, + [[maybe_unused]] Pred&& is_cpu_allowed) { + NumaConfig cfg = empty(); + +#if defined(__linux__) && !defined(__ANDROID__) + + // On Linux things are straightforward, since there's no processor groups and + // any thread can be scheduled on all processors. + // We try to gather this information from the sysfs first + // https://www.kernel.org/doc/Documentation/ABI/stable/sysfs-devices-node + + bool useFallback = false; + auto fallback = [&]() { + useFallback = true; + cfg = empty(); + }; + + // /sys/devices/system/node/online contains information about active NUMA nodes + auto nodeIdsStr = read_file_to_string("/sys/devices/system/node/online"); + if (!nodeIdsStr.has_value() || nodeIdsStr->empty()) + { + fallback(); + } + else + { + remove_whitespace(*nodeIdsStr); + for (size_t n : indices_from_shortened_string(*nodeIdsStr)) + { + // /sys/devices/system/node/node.../cpulist + std::string path = + std::string("/sys/devices/system/node/node") + std::to_string(n) + "/cpulist"; + auto cpuIdsStr = read_file_to_string(path); + // Now, we only bail if the file does not exist. Some nodes may be + // empty, that's fine. An empty node still has a file that appears + // to have some whitespace, so we need to handle that. + if (!cpuIdsStr.has_value()) + { + fallback(); + break; + } + else + { + remove_whitespace(*cpuIdsStr); + for (size_t c : indices_from_shortened_string(*cpuIdsStr)) + { + if (is_cpu_allowed(c)) + cfg.add_cpu_to_node(n, c); + } + } + } + } + + if (useFallback) + { + for (CpuIndex c = 0; c < SYSTEM_THREADS_NB; ++c) + if (is_cpu_allowed(c)) + cfg.add_cpu_to_node(NumaIndex{0}, c); + } + +#elif defined(_WIN64) + + WORD numProcGroups = GetActiveProcessorGroupCount(); + for (WORD procGroup = 0; procGroup < numProcGroups; ++procGroup) + { + for (BYTE number = 0; number < WIN_PROCESSOR_GROUP_SIZE; ++number) + { + PROCESSOR_NUMBER procnum; + procnum.Group = procGroup; + procnum.Number = number; + procnum.Reserved = 0; + USHORT nodeNumber; + + const BOOL status = GetNumaProcessorNodeEx(&procnum, &nodeNumber); + const CpuIndex c = static_cast(procGroup) * WIN_PROCESSOR_GROUP_SIZE + + static_cast(number); + if (status != 0 && nodeNumber != std::numeric_limits::max() + && is_cpu_allowed(c)) + { + cfg.add_cpu_to_node(nodeNumber, c); + } + } + } + +#else + + abort(); // should not reach here + +#endif + + return cfg; + } + + template + static std::optional try_get_l3_aware_config( + bool respectProcessAffinity, size_t bundleSize, [[maybe_unused]] Pred&& is_cpu_allowed) { + // Get the normal system configuration so we know to which NUMA node + // each L3 domain belongs. + NumaConfig systemConfig = + NumaConfig::from_system(SystemNumaPolicy{}, respectProcessAffinity); + std::vector l3Domains; + +#if defined(__linux__) && !defined(__ANDROID__) + + std::set seenCpus; + auto nextUnseenCpu = [&seenCpus]() { + for (CpuIndex i = 0;; ++i) + if (!seenCpus.count(i)) + return i; + }; + + while (true) + { + CpuIndex next = nextUnseenCpu(); + auto siblingsStr = + read_file_to_string("/sys/devices/system/cpu/cpu" + std::to_string(next) + + "/cache/index3/shared_cpu_list"); + + if (!siblingsStr.has_value() || siblingsStr->empty()) + { + break; // we have read all available CPUs + } + + L3Domain domain; + for (size_t c : indices_from_shortened_string(*siblingsStr)) + { + if (is_cpu_allowed(c)) + { + domain.systemNumaIndex = systemConfig.nodeByCpu.at(c); + domain.cpus.insert(c); + } + seenCpus.insert(c); + } + if (!domain.cpus.empty()) + { + l3Domains.emplace_back(std::move(domain)); + } + } + +#elif defined(_WIN64) + + DWORD bufSize = 0; + GetLogicalProcessorInformationEx(RelationCache, nullptr, &bufSize); + if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) + return std::nullopt; + + std::vector buffer(bufSize); + auto info = reinterpret_cast(buffer.data()); + if (!GetLogicalProcessorInformationEx(RelationCache, info, &bufSize)) + return std::nullopt; + + while (reinterpret_cast(info) < buffer.data() + bufSize) + { + info = std::launder(info); + if (info->Relationship == RelationCache && info->Cache.Level == 3) + { + L3Domain domain{}; + for (WORD procGroup = 0; procGroup < info->Cache.GroupCount; ++procGroup) + { + for (BYTE number = 0; number < WIN_PROCESSOR_GROUP_SIZE; ++number) + { + WORD groupNumber = info->Cache.GroupMasks[procGroup].Group; + const CpuIndex c = + static_cast(groupNumber) * WIN_PROCESSOR_GROUP_SIZE + + static_cast(number); + if (!(info->Cache.GroupMasks[procGroup].Mask & (1ULL << number)) + || !is_cpu_allowed(c)) + continue; + domain.systemNumaIndex = systemConfig.nodeByCpu.at(c); + domain.cpus.insert(c); + } + } + if (!domain.cpus.empty()) + l3Domains.push_back(std::move(domain)); + } + // Variable length data structure, advance to next + info = reinterpret_cast( + reinterpret_cast(info) + info->Size); + } + +#endif + + if (!l3Domains.empty()) + return {NumaConfig::from_l3_info(std::move(l3Domains), bundleSize)}; + + return std::nullopt; + } + + + static NumaConfig from_l3_info(std::vector&& domains, size_t bundleSize) { + assert(!domains.empty()); + + std::map> list; + for (auto& d : domains) + list[d.systemNumaIndex].emplace_back(std::move(d)); + + NumaConfig cfg = empty(); + NumaIndex n = 0; + for (auto& [_, ds] : list) + { + bool changed; + // Scan through pairs and merge them. With roughly equal L3 sizes, should give + // a decent distribution. + do + { + changed = false; + for (size_t j = 0; j + 1 < ds.size(); ++j) + { + if (ds[j].cpus.size() + ds[j + 1].cpus.size() <= bundleSize) + { + changed = true; + ds[j].cpus.merge(ds[j + 1].cpus); + ds.erase(ds.begin() + j + 1); + } + } + // ds.size() has decreased if changed is true, so this loop will terminate + } while (changed); + for (const L3Domain& d : ds) + { + const NumaIndex dn = n++; + for (CpuIndex cpu : d.cpus) + { + cfg.add_cpu_to_node(dn, cpu); + } + } + } + return cfg; + } }; class NumaReplicationContext; @@ -1345,7 +1549,7 @@ class LazyNumaReplicatedSystemWide: public NumaReplicatedBase { std::size_t get_discriminator(NumaIndex idx) const { const NumaConfig& cfg = get_numa_config(); - const NumaConfig& cfg_sys = NumaConfig::from_system(false); + const NumaConfig& cfg_sys = NumaConfig::from_system(SystemNumaPolicy{}, false); // as a discriminator, locate the hardware/system numadomain this cpuindex belongs to CpuIndex cpu = *cfg.nodes[idx].begin(); // get a CpuIndex from NumaIndex NumaIndex sys_idx = cfg_sys.is_cpu_assigned(cpu) ? cfg_sys.nodeByCpu.at(cpu) : 0;