Add 1GB page support on x86 Linux

1GB pages for the TT give a small performance bump (`./stockfish speedtest 256 32000`):

master:
Nodes/second               : 230514479
patch
Nodes/second               : 237997728

With a large enough TT there is still dTLB thrashing with 2MB pages. 1GB pages reduces that and makes page table walks a bit faster (they're shorter, and there's fewer PTEs, which fits better in cache). `madvise` won't give you huge pages, only large pages, so we use mmap here.

We guard the huge page attempt to x86 + at least 8 huge pages per NUMA node, to avoid memory oversubscription when for example running with a 1.1 GB hash. Additionally, we change the TT clearing slightly so that the pages are better distributed across NUMA nodes.

Data from Torom:

master:
Nodes/second               : 43524292
patch:
Nodes/second               : 44080034

Data on an 8-thread Emerald Rapids VM:

./stockfish speedtest 8 65536

Baseline speedtest (2MB huge pages):

Nodes/second : 7366232
Nodes/second : 7262794
Nodes/second : 7355431
Nodes/second : 7318777

Baseline speedtest (1GB huge pages):

Nodes/second : 7541027
Nodes/second : 7572720
Nodes/second : 7550268
Nodes/second : 7538925

closes https://github.com/official-stockfish/Stockfish/pull/6681

No functional change

src/memory.cpp

@@ -19,13 +19,19 @@
 #include "memory.h"
 
 #include <cstdlib>
+#include <iostream>  // std::cerr
 
 #if __has_include("features.h")
     #include <features.h>
 #endif
 
 #if defined(__linux__) && !defined(__ANDROID__)
+    #include <errno.h>
     #include <sys/mman.h>
+    // IWYU pragma: no_include <bits/mman-map-flags-generic.h>
+    #include <cstring>
+    #include <mutex>
+    #include <map>
 #endif
 
 #if defined(__APPLE__) || defined(__ANDROID__) || defined(__OpenBSD__) \
@@ -46,8 +52,6 @@
     #endif
 
     #include <ios>  // std::hex, std::dec
-    #include <iostream>  // std::cerr
-    #include <ostream>   // std::endl
     #include <windows.h>
 
 // The needed Windows API for processor groups could be missed from old Windows
@@ -110,7 +114,7 @@ static void* aligned_large_pages_alloc_windows([[maybe_unused]] size_t allocSize
       []() { return (void*) nullptr; });
 }
 
-void* aligned_large_pages_alloc(size_t allocSize) {
+void* aligned_large_pages_alloc_with_hint(size_t allocSize, bool) {
 
     // Try to allocate large pages
     void* mem = aligned_large_pages_alloc_windows(allocSize);
@@ -124,7 +128,35 @@ void* aligned_large_pages_alloc(size_t allocSize) {
 
 #else
 
-void* aligned_large_pages_alloc(size_t allocSize) {
+    #if defined(__linux__) && defined(MAP_HUGE_SHIFT) && defined(__x86_64__)
+        #define HAS_HUGE_PAGES
+
+static std::map<void*, size_t> huge_pages;
+static std::mutex              huge_pages_mtx;
+
+static void* try_huge_pages_alloc(size_t allocSize) {
+    size_t size = ((allocSize + HugePageSize - 1) / HugePageSize) * HugePageSize;
+    void*  mem  = mmap(NULL, size, PROT_READ | PROT_WRITE,
+                       MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | (30 << MAP_HUGE_SHIFT), -1, 0);
+
+    if (mem == MAP_FAILED)
+        return nullptr;
+
+    std::lock_guard lg(huge_pages_mtx);
+    huge_pages[mem] = size;
+    return mem;
+}
+    #endif  // defined(__linux__) && defined(MAP_HUGE_SHIFT) && defined(__x86_64__)
+
+void* aligned_large_pages_alloc_with_hint(size_t allocSize, [[maybe_unused]] bool hugePageHint) {
+    #ifdef HAS_HUGE_PAGES
+    if (hugePageHint && allocSize >= HugePageSize)
+    {
+        void* mem = try_huge_pages_alloc(allocSize);
+        if (mem)
+            return mem;
+    }
+    #endif
 
     #if defined(__linux__)
     constexpr size_t alignment = 2 * 1024 * 1024;  // 2MB page size assumed
@@ -143,6 +175,10 @@ void* aligned_large_pages_alloc(size_t allocSize) {
 
 #endif
 
+void* aligned_large_pages_alloc(size_t size) {
+    return aligned_large_pages_alloc_with_hint(size, false);
+}
+
 bool has_large_pages() {
 
 #if defined(_WIN32)
@@ -193,7 +229,26 @@ void aligned_large_pages_free(void* mem) {
 
 #else
 
-void aligned_large_pages_free(void* mem) { std_aligned_free(mem); }
+void aligned_large_pages_free(void* mem) {
+    if (!mem)
+        return;
+
+    #ifdef HAS_HUGE_PAGES
+    std::lock_guard lg(huge_pages_mtx);
+    if (auto it = huge_pages.find(mem); it != huge_pages.end())
+    {
+        if (munmap(mem, it->second) != 0)
+        {
+            std::cerr << "munmap failed: " << strerror(errno) << std::endl;
+            exit(EXIT_FAILURE);
+        }
+        huge_pages.erase(it);
+        return;
+    }
+    #endif
+
+    std_aligned_free(mem);
+}
 
 #endif
 }  // namespace Stockfish

src/memory.h

@@ -61,10 +61,13 @@ using AdjustTokenPrivileges_t =
 
 namespace Stockfish {
 
+constexpr size_t HugePageSize = size_t(1) << 30;
+
 void* std_aligned_alloc(size_t alignment, size_t size);
 void  std_aligned_free(void* ptr);
 
 // Memory aligned by page size, min alignment: 4096 bytes
+void* aligned_large_pages_alloc_with_hint(size_t size, bool hugePageHint);
 void* aligned_large_pages_alloc(size_t size);
 void  aligned_large_pages_free(void* mem);
 

src/thread.cpp

@@ -27,6 +27,7 @@
 #include <memory>
 #include <string>
 #include <unordered_map>
+#include <unordered_set>
 #include <utility>
 
 #include "bitboard.h"
@@ -427,6 +428,10 @@ void ThreadPool::wait_for_search_finished() const {
             th->wait_for_search_finished();
 }
 
+std::vector<size_t> ThreadPool::get_bound_thread_to_numa_node() const {
+    return boundThreadToNumaNode;
+}
+
 std::vector<size_t> ThreadPool::get_bound_thread_count_by_numa_node() const {
     std::vector<size_t> counts;
 
@@ -446,6 +451,13 @@ std::vector<size_t> ThreadPool::get_bound_thread_count_by_numa_node() const {
     return counts;
 }
 
+size_t ThreadPool::numa_nodes() const {
+    std::unordered_set<size_t> seen;
+    for (NumaIndex n : boundThreadToNumaNode)
+        seen.insert(n);
+    return std::max(seen.size(), size_t(1));
+}
+
 void ThreadPool::ensure_network_replicated() {
     for (auto&& th : threads)
         th->ensure_network_replicated();

src/thread.h

@@ -149,7 +149,9 @@ class ThreadPool {
     void                   start_searching();
     void                   wait_for_search_finished() const;
 
+    std::vector<size_t> get_bound_thread_to_numa_node() const;
     std::vector<size_t> get_bound_thread_count_by_numa_node() const;
+    size_t              numa_nodes() const;
 
     void ensure_network_replicated();
 

src/tt.cpp

@@ -18,11 +18,14 @@
 
 #include "tt.h"
 
+#include <algorithm>
 #include <cassert>
 #include <cstdint>
 #include <cstdlib>
 #include <cstring>
 #include <iostream>
+#include <numeric>
+#include <vector>
 
 #include "memory.h"
 #include "misc.h"
@@ -162,8 +165,13 @@ void TranspositionTable::resize(size_t mbSize, ThreadPool& threads) {
     aligned_large_pages_free(table);
 
     clusterCount   = mbSize * 1024 * 1024 / sizeof(Cluster);
+    size_t ttBytes = clusterCount * sizeof(Cluster);
 
-    table = static_cast<Cluster*>(aligned_large_pages_alloc(clusterCount * sizeof(Cluster)));
+    // Request 1GB pages if we'd get at least eight per NUMA node, to avoid
+    // memory oversubscription
+    bool hugePageHint = ttBytes >= threads.numa_nodes() * HugePageSize * 8;
+
+    table = static_cast<Cluster*>(aligned_large_pages_alloc_with_hint(ttBytes, hugePageHint));
 
     if (!table)
     {
@@ -181,9 +189,23 @@ void TranspositionTable::clear(ThreadPool& threads) {
     generation8              = 0;
     const size_t threadCount = threads.num_threads();
 
+    std::vector<size_t> threadToNuma = threads.get_bound_thread_to_numa_node();
+
+    std::vector<size_t> order(threadCount);
+    std::iota(order.begin(), order.end(), 0);
+
+    // To promote good NUMA distribution (esp. with huge pages), we permute threads so that
+    // all threads in a NUMA node clear a contiguous region of the TT.
+    if (threadToNuma.size() == threadCount)
+    {
+        std::stable_sort(order.begin(), order.end(), [&threadToNuma](size_t t1, size_t t2) {
+            return threadToNuma.at(t1) < threadToNuma.at(t2);
+        });
+    }
+
     for (size_t i = 0; i < threadCount; ++i)
     {
-        threads.run_on_thread(i, [this, i, threadCount]() {
+        threads.run_on_thread(order[i], [this, i, threadCount]() {
             // Each thread will zero its part of the hash table
             const size_t stride = clusterCount / threadCount;
             const size_t start  = stride * i;