From be9df38f271f0c69bd5adbd2ba34fd9a90b6fe1b Mon Sep 17 00:00:00 2001 From: anematode Date: Sun, 24 May 2026 14:17:54 +0200 Subject: [PATCH] Add 1GB page support on x86 Linux 1GB pages for the TT give a small performance bump (`./stockfish speedtest 256 32000`): master: Nodes/second : 230514479 patch Nodes/second : 237997728 With a large enough TT there is still dTLB thrashing with 2MB pages. 1GB pages reduces that and makes page table walks a bit faster (they're shorter, and there's fewer PTEs, which fits better in cache). `madvise` won't give you huge pages, only large pages, so we use mmap here. We guard the huge page attempt to x86 + at least 8 huge pages per NUMA node, to avoid memory oversubscription when for example running with a 1.1 GB hash. Additionally, we change the TT clearing slightly so that the pages are better distributed across NUMA nodes. Data from Torom: master: Nodes/second : 43524292 patch: Nodes/second : 44080034 Data on an 8-thread Emerald Rapids VM: ./stockfish speedtest 8 65536 Baseline speedtest (2MB huge pages): Nodes/second : 7366232 Nodes/second : 7262794 Nodes/second : 7355431 Nodes/second : 7318777 Baseline speedtest (1GB huge pages): Nodes/second : 7541027 Nodes/second : 7572720 Nodes/second : 7550268 Nodes/second : 7538925 closes https://github.com/official-stockfish/Stockfish/pull/6681 No functional change --- src/memory.cpp | 67 +++++++++++++++++++++++++++++++++++++++++++++----- src/memory.h | 3 +++ src/thread.cpp | 12 +++++++++ src/thread.h | 2 ++ src/tt.cpp | 28 ++++++++++++++++++--- 5 files changed, 103 insertions(+), 9 deletions(-) diff --git a/src/memory.cpp b/src/memory.cpp index 94a599399..a271b37c7 100644 --- a/src/memory.cpp +++ b/src/memory.cpp @@ -19,13 +19,19 @@ #include "memory.h" #include +#include // std::cerr #if __has_include("features.h") #include #endif #if defined(__linux__) && !defined(__ANDROID__) + #include #include + // IWYU pragma: no_include + #include + #include + #include #endif #if defined(__APPLE__) || defined(__ANDROID__) || defined(__OpenBSD__) \ @@ -45,9 +51,7 @@ #define NOMINMAX #endif - #include // std::hex, std::dec - #include // std::cerr - #include // std::endl + #include // std::hex, std::dec #include // The needed Windows API for processor groups could be missed from old Windows @@ -110,7 +114,7 @@ static void* aligned_large_pages_alloc_windows([[maybe_unused]] size_t allocSize []() { return (void*) nullptr; }); } -void* aligned_large_pages_alloc(size_t allocSize) { +void* aligned_large_pages_alloc_with_hint(size_t allocSize, bool) { // Try to allocate large pages void* mem = aligned_large_pages_alloc_windows(allocSize); @@ -124,7 +128,35 @@ void* aligned_large_pages_alloc(size_t allocSize) { #else -void* aligned_large_pages_alloc(size_t allocSize) { + #if defined(__linux__) && defined(MAP_HUGE_SHIFT) && defined(__x86_64__) + #define HAS_HUGE_PAGES + +static std::map huge_pages; +static std::mutex huge_pages_mtx; + +static void* try_huge_pages_alloc(size_t allocSize) { + size_t size = ((allocSize + HugePageSize - 1) / HugePageSize) * HugePageSize; + void* mem = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | (30 << MAP_HUGE_SHIFT), -1, 0); + + if (mem == MAP_FAILED) + return nullptr; + + std::lock_guard lg(huge_pages_mtx); + huge_pages[mem] = size; + return mem; +} + #endif // defined(__linux__) && defined(MAP_HUGE_SHIFT) && defined(__x86_64__) + +void* aligned_large_pages_alloc_with_hint(size_t allocSize, [[maybe_unused]] bool hugePageHint) { + #ifdef HAS_HUGE_PAGES + if (hugePageHint && allocSize >= HugePageSize) + { + void* mem = try_huge_pages_alloc(allocSize); + if (mem) + return mem; + } + #endif #if defined(__linux__) constexpr size_t alignment = 2 * 1024 * 1024; // 2MB page size assumed @@ -143,6 +175,10 @@ void* aligned_large_pages_alloc(size_t allocSize) { #endif +void* aligned_large_pages_alloc(size_t size) { + return aligned_large_pages_alloc_with_hint(size, false); +} + bool has_large_pages() { #if defined(_WIN32) @@ -193,7 +229,26 @@ void aligned_large_pages_free(void* mem) { #else -void aligned_large_pages_free(void* mem) { std_aligned_free(mem); } +void aligned_large_pages_free(void* mem) { + if (!mem) + return; + + #ifdef HAS_HUGE_PAGES + std::lock_guard lg(huge_pages_mtx); + if (auto it = huge_pages.find(mem); it != huge_pages.end()) + { + if (munmap(mem, it->second) != 0) + { + std::cerr << "munmap failed: " << strerror(errno) << std::endl; + exit(EXIT_FAILURE); + } + huge_pages.erase(it); + return; + } + #endif + + std_aligned_free(mem); +} #endif } // namespace Stockfish diff --git a/src/memory.h b/src/memory.h index 056b07c6d..f2dea0d0a 100644 --- a/src/memory.h +++ b/src/memory.h @@ -61,10 +61,13 @@ using AdjustTokenPrivileges_t = namespace Stockfish { +constexpr size_t HugePageSize = size_t(1) << 30; + void* std_aligned_alloc(size_t alignment, size_t size); void std_aligned_free(void* ptr); // Memory aligned by page size, min alignment: 4096 bytes +void* aligned_large_pages_alloc_with_hint(size_t size, bool hugePageHint); void* aligned_large_pages_alloc(size_t size); void aligned_large_pages_free(void* mem); diff --git a/src/thread.cpp b/src/thread.cpp index 311f719ef..d87bcee08 100644 --- a/src/thread.cpp +++ b/src/thread.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include "bitboard.h" @@ -427,6 +428,10 @@ void ThreadPool::wait_for_search_finished() const { th->wait_for_search_finished(); } +std::vector ThreadPool::get_bound_thread_to_numa_node() const { + return boundThreadToNumaNode; +} + std::vector ThreadPool::get_bound_thread_count_by_numa_node() const { std::vector counts; @@ -446,6 +451,13 @@ std::vector ThreadPool::get_bound_thread_count_by_numa_node() const { return counts; } +size_t ThreadPool::numa_nodes() const { + std::unordered_set seen; + for (NumaIndex n : boundThreadToNumaNode) + seen.insert(n); + return std::max(seen.size(), size_t(1)); +} + void ThreadPool::ensure_network_replicated() { for (auto&& th : threads) th->ensure_network_replicated(); diff --git a/src/thread.h b/src/thread.h index d6032d295..445826048 100644 --- a/src/thread.h +++ b/src/thread.h @@ -149,7 +149,9 @@ class ThreadPool { void start_searching(); void wait_for_search_finished() const; + std::vector get_bound_thread_to_numa_node() const; std::vector get_bound_thread_count_by_numa_node() const; + size_t numa_nodes() const; void ensure_network_replicated(); diff --git a/src/tt.cpp b/src/tt.cpp index 527096a5c..0ad076c76 100644 --- a/src/tt.cpp +++ b/src/tt.cpp @@ -18,11 +18,14 @@ #include "tt.h" +#include #include #include #include #include #include +#include +#include #include "memory.h" #include "misc.h" @@ -161,9 +164,14 @@ static_assert(sizeof(Cluster) == 32, "Suboptimal Cluster size"); void TranspositionTable::resize(size_t mbSize, ThreadPool& threads) { aligned_large_pages_free(table); - clusterCount = mbSize * 1024 * 1024 / sizeof(Cluster); + clusterCount = mbSize * 1024 * 1024 / sizeof(Cluster); + size_t ttBytes = clusterCount * sizeof(Cluster); - table = static_cast(aligned_large_pages_alloc(clusterCount * sizeof(Cluster))); + // Request 1GB pages if we'd get at least eight per NUMA node, to avoid + // memory oversubscription + bool hugePageHint = ttBytes >= threads.numa_nodes() * HugePageSize * 8; + + table = static_cast(aligned_large_pages_alloc_with_hint(ttBytes, hugePageHint)); if (!table) { @@ -181,9 +189,23 @@ void TranspositionTable::clear(ThreadPool& threads) { generation8 = 0; const size_t threadCount = threads.num_threads(); + std::vector threadToNuma = threads.get_bound_thread_to_numa_node(); + + std::vector order(threadCount); + std::iota(order.begin(), order.end(), 0); + + // To promote good NUMA distribution (esp. with huge pages), we permute threads so that + // all threads in a NUMA node clear a contiguous region of the TT. + if (threadToNuma.size() == threadCount) + { + std::stable_sort(order.begin(), order.end(), [&threadToNuma](size_t t1, size_t t2) { + return threadToNuma.at(t1) < threadToNuma.at(t2); + }); + } + for (size_t i = 0; i < threadCount; ++i) { - threads.run_on_thread(i, [this, i, threadCount]() { + threads.run_on_thread(order[i], [this, i, threadCount]() { // Each thread will zero its part of the hash table const size_t stride = clusterCount / threadCount; const size_t start = stride * i;