diff --git a/src/engine.cpp b/src/engine.cpp index 9edff4864..40466c8f8 100644 --- a/src/engine.cpp +++ b/src/engine.cpp @@ -240,7 +240,8 @@ void Engine::set_numa_config_from_option(const std::string& o) { void Engine::resize_threads() { threads.wait_for_search_finished(); - threads.set(numaContext.get_numa_config(), {options, threads, tt, networks}, updateContext); + threads.set(numaContext.get_numa_config(), {options, threads, tt, sharedHists, networks}, + updateContext); // Reallocate the hash with the new threadpool size set_tt_size(options["Hash"]); diff --git a/src/engine.h b/src/engine.h index 7315b8881..6fd1ce040 100644 --- a/src/engine.h +++ b/src/engine.h @@ -22,12 +22,14 @@ #include #include #include +#include #include #include #include #include #include +#include "history.h" #include "nnue/network.h" #include "numa.h" #include "position.h" @@ -122,6 +124,7 @@ class Engine { Search::SearchManager::UpdateContext updateContext; std::function onVerifyNetworks; + std::map sharedHists; }; } // namespace Stockfish diff --git a/src/history.h b/src/history.h index 87004ead9..127fb9ef3 100644 --- a/src/history.h +++ b/src/history.h @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -28,6 +29,7 @@ #include #include // IWYU pragma: keep +#include "memory.h" #include "misc.h" #include "position.h" @@ -35,56 +37,54 @@ namespace Stockfish { constexpr int PAWN_HISTORY_SIZE = 8192; // has to be a power of 2 constexpr int UINT_16_HISTORY_SIZE = std::numeric_limits::max() + 1; +constexpr int CORRHIST_BASE_SIZE = UINT_16_HISTORY_SIZE; constexpr int CORRECTION_HISTORY_LIMIT = 1024; constexpr int LOW_PLY_HISTORY_SIZE = 5; static_assert((PAWN_HISTORY_SIZE & (PAWN_HISTORY_SIZE - 1)) == 0, "PAWN_HISTORY_SIZE has to be a power of 2"); -static_assert((UINT_16_HISTORY_SIZE & (UINT_16_HISTORY_SIZE - 1)) == 0, - "CORRECTION_HISTORY_SIZE has to be a power of 2"); +static_assert((CORRHIST_BASE_SIZE & (CORRHIST_BASE_SIZE - 1)) == 0, + "CORRHIST_BASE_SIZE has to be a power of 2"); inline int pawn_history_index(const Position& pos) { return pos.pawn_key() & (PAWN_HISTORY_SIZE - 1); } -inline uint16_t pawn_correction_history_index(const Position& pos) { - return uint16_t(pos.pawn_key()); -} - -inline uint16_t minor_piece_index(const Position& pos) { return uint16_t(pos.minor_piece_key()); } - -template -inline uint16_t non_pawn_index(const Position& pos) { - return uint16_t(pos.non_pawn_key(c)); -} - // StatsEntry is the container of various numerical statistics. We use a class // instead of a naked value to directly call history update operator<<() on // the entry. The first template parameter T is the base type of the array, // and the second template parameter D limits the range of updates in [-D, D] // when we update values with the << operator -template -class StatsEntry { - +template +struct StatsEntry { static_assert(std::is_arithmetic_v, "Not an arithmetic type"); - static_assert(D <= std::numeric_limits::max(), "D overflows T"); - T entry; + private: + std::conditional_t, T> entry; public: - StatsEntry& operator=(const T& v) { - entry = v; - return *this; + void operator=(const T& v) { + if constexpr (Atomic) + entry.store(v, std::memory_order_relaxed); + else + entry = v; + } + + operator T() const { + if constexpr (Atomic) + return entry.load(std::memory_order_relaxed); + else + return entry; } - operator const T&() const { return entry; } void operator<<(int bonus) { // Make sure that bonus is in range [-D, D] int clampedBonus = std::clamp(bonus, -D, D); - entry += clampedBonus - entry * std::abs(clampedBonus) / D; + T val = *this; + *this = val + clampedBonus - val * std::abs(clampedBonus) / D; - assert(std::abs(entry) <= D); + assert(std::abs(T(*this)) <= D); } }; @@ -96,6 +96,37 @@ enum StatsType { template using Stats = MultiArray, Sizes...>; +// DynStats is a dynamically sized array of Stats, used for thread-shared histories +// which should scale with the total number of threads. The SizeMultiplier gives +// the per-thread allocation count of T. +template +struct DynStats { + explicit DynStats(size_t s) { + size = s * SizeMultiplier; + data = make_unique_large_page(size); + } + // Sets all values in the range to 0 + void clear_range(size_t start, size_t end) { + assert(start < size); + assert(end <= size); + T* fill_start = &(*this)[start]; + memset(reinterpret_cast(fill_start), 0, sizeof(T) * (end - start)); + } + size_t get_size() const { return size; } + T& operator[](size_t index) { + assert(index < size); + return data.get()[index]; + } + const T& operator[](size_t index) const { + assert(index < size); + return data.get()[index]; + } + + private: + size_t size; + LargePagePtr data; +}; + // ButterflyHistory records how often quiet moves have been successful or unsuccessful // during the current search, and is used for reduction and move ordering decisions. // It uses 2 tables (one for each color) indexed by the move's from and to squares, @@ -132,11 +163,20 @@ enum CorrHistType { Continuation, // Combined history of move pairs }; +template +struct CorrectionBundle { + StatsEntry pawn; + StatsEntry minor; + StatsEntry nonPawnWhite; + StatsEntry nonPawnBlack; +}; + namespace Detail { template struct CorrHistTypedef { - using type = Stats; + using type = + DynStats, CORRHIST_BASE_SIZE>; }; template<> @@ -151,17 +191,63 @@ struct CorrHistTypedef { template<> struct CorrHistTypedef { - using type = - Stats; + using type = DynStats, + CORRHIST_BASE_SIZE>; }; } +using UnifiedCorrectionHistory = + DynStats, COLOR_NB>, + CORRHIST_BASE_SIZE>; + template using CorrectionHistory = typename Detail::CorrHistTypedef::type; using TTMoveHistory = StatsEntry; +// Set of histories shared between groups of threads. To avoid excessive +// cross-node data transfer, histories are shared only between threads +// on a given NUMA node. The passed size must be a power of two to make +// the indexing more efficient. +struct SharedHistories { + SharedHistories(size_t threadCount) : + correctionHistory(threadCount) { + assert((threadCount & (threadCount - 1)) == 0 && threadCount != 0); + sizeMinus1 = correctionHistory.get_size() - 1; + } + + size_t get_size() const { return sizeMinus1 + 1; } + + auto& pawn_correction_entry(const Position& pos) { + return correctionHistory[pos.pawn_key() & sizeMinus1]; + } + const auto& pawn_correction_entry(const Position& pos) const { + return correctionHistory[pos.pawn_key() & sizeMinus1]; + } + + auto& minor_piece_correction_entry(const Position& pos) { + return correctionHistory[pos.minor_piece_key() & sizeMinus1]; + } + const auto& minor_piece_correction_entry(const Position& pos) const { + return correctionHistory[pos.minor_piece_key() & sizeMinus1]; + } + + template + auto& nonpawn_correction_entry(const Position& pos) { + return correctionHistory[pos.non_pawn_key(c) & sizeMinus1]; + } + template + const auto& nonpawn_correction_entry(const Position& pos) const { + return correctionHistory[pos.non_pawn_key(c) & sizeMinus1]; + } + + UnifiedCorrectionHistory correctionHistory; + + private: + size_t sizeMinus1; +}; + } // namespace Stockfish #endif // #ifndef HISTORY_H_INCLUDED diff --git a/src/position.cpp b/src/position.cpp index a4286b86a..7377b2029 100644 --- a/src/position.cpp +++ b/src/position.cpp @@ -32,6 +32,7 @@ #include #include "bitboard.h" +#include "history.h" #include "misc.h" #include "movegen.h" #include "syzygy/tbprobe.h" @@ -692,13 +693,14 @@ bool Position::gives_check(Move m) const { // to a StateInfo object. The move is assumed to be legal. Pseudo-legal // moves should be filtered out before this function is called. // If a pointer to the TT table is passed, the entry for the new position -// will be prefetched +// will be prefetched, and likewise for shared history. void Position::do_move(Move m, StateInfo& newSt, bool givesCheck, DirtyPiece& dp, DirtyThreats& dts, - const TranspositionTable* tt = nullptr) { + const TranspositionTable* tt = nullptr, + const SharedHistories* history = nullptr) { assert(m.is_ok()); assert(&newSt != st); @@ -880,6 +882,14 @@ void Position::do_move(Move m, if (tt && !checkEP) prefetch(tt->first_entry(adjust_key50(k))); + if (history) + { + prefetch(&history->pawn_correction_entry(*this)); + prefetch(&history->minor_piece_correction_entry(*this)); + prefetch(&history->nonpawn_correction_entry(*this)); + prefetch(&history->nonpawn_correction_entry(*this)); + } + // Set capture piece st->capturedPiece = captured; diff --git a/src/position.h b/src/position.h index e5d3f6d28..e49e10f96 100644 --- a/src/position.h +++ b/src/position.h @@ -33,6 +33,7 @@ namespace Stockfish { class TranspositionTable; +struct SharedHistories; // StateInfo struct stores information needed to restore a Position object to // its previous state when we retract a move. Whenever a move is made on the @@ -69,7 +70,6 @@ struct StateInfo { // elements are not invalidated upon list resizing. using StateListPtr = std::unique_ptr>; - // Position class stores information regarding the board representation as // pieces, side to move, hash keys, castling info, etc. Important methods are // do_move() and undo_move(), used by the search to update node info when @@ -140,7 +140,8 @@ class Position { bool givesCheck, DirtyPiece& dp, DirtyThreats& dts, - const TranspositionTable* tt); + const TranspositionTable* tt, + const SharedHistories* worker); void undo_move(Move m); void do_null_move(StateInfo& newSt, const TranspositionTable& tt); void undo_null_move(); @@ -403,7 +404,7 @@ inline void Position::swap_piece(Square s, Piece pc, DirtyThreats* const dts) { inline void Position::do_move(Move m, StateInfo& newSt, const TranspositionTable* tt = nullptr) { new (&scratch_dts) DirtyThreats; - do_move(m, newSt, gives_check(m), scratch_dp, scratch_dts, tt); + do_move(m, newSt, gives_check(m), scratch_dp, scratch_dts, tt, nullptr); } inline StateInfo* Position::state() const { return st; } diff --git a/src/search.cpp b/src/search.cpp index adb7985d2..7c08bb0fa 100644 --- a/src/search.cpp +++ b/src/search.cpp @@ -77,16 +77,17 @@ using SearchedList = ValueList; // optimized for require verifications at longer time controls int correction_value(const Worker& w, const Position& pos, const Stack* const ss) { - const Color us = pos.side_to_move(); - const auto m = (ss - 1)->currentMove; - const auto pcv = w.pawnCorrectionHistory[pawn_correction_history_index(pos)][us]; - const auto micv = w.minorPieceCorrectionHistory[minor_piece_index(pos)][us]; - const auto wnpcv = w.nonPawnCorrectionHistory[non_pawn_index(pos)][WHITE][us]; - const auto bnpcv = w.nonPawnCorrectionHistory[non_pawn_index(pos)][BLACK][us]; - const auto cntcv = + const Color us = pos.side_to_move(); + const auto m = (ss - 1)->currentMove; + const auto& shared = w.sharedHistory; + const int pcv = shared.pawn_correction_entry(pos).at(us).pawn; + const int micv = shared.minor_piece_correction_entry(pos).at(us).minor; + const int wnpcv = shared.nonpawn_correction_entry(pos).at(us).nonPawnWhite; + const int bnpcv = shared.nonpawn_correction_entry(pos).at(us).nonPawnBlack; + const int cntcv = m.is_ok() ? (*(ss - 2)->continuationCorrectionHistory)[pos.piece_on(m.to_sq())][m.to_sq()] + (*(ss - 4)->continuationCorrectionHistory)[pos.piece_on(m.to_sq())][m.to_sq()] - : 8; + : 8; return 10347 * pcv + 8821 * micv + 11168 * (wnpcv + bnpcv) + 7841 * cntcv; } @@ -105,13 +106,12 @@ void update_correction_history(const Position& pos, const Color us = pos.side_to_move(); constexpr int nonPawnWeight = 178; + auto& shared = workerThread.sharedHistory; - workerThread.pawnCorrectionHistory[pawn_correction_history_index(pos)][us] << bonus; - workerThread.minorPieceCorrectionHistory[minor_piece_index(pos)][us] << bonus * 156 / 128; - workerThread.nonPawnCorrectionHistory[non_pawn_index(pos)][WHITE][us] - << bonus * nonPawnWeight / 128; - workerThread.nonPawnCorrectionHistory[non_pawn_index(pos)][BLACK][us] - << bonus * nonPawnWeight / 128; + shared.pawn_correction_entry(pos).at(us).pawn << bonus; + shared.minor_piece_correction_entry(pos).at(us).minor << bonus * 156 / 128; + shared.nonpawn_correction_entry(pos).at(us).nonPawnWhite << bonus * nonPawnWeight / 128; + shared.nonpawn_correction_entry(pos).at(us).nonPawnBlack << bonus * nonPawnWeight / 128; if (m.is_ok()) { @@ -155,9 +155,14 @@ bool is_shuffling(Move move, Stack* const ss, const Position& pos) { Search::Worker::Worker(SharedState& sharedState, std::unique_ptr sm, size_t threadId, + size_t numaThreadId, + size_t numaTotalThreads, NumaReplicatedAccessToken token) : // Unpack the SharedState struct into member variables + sharedHistory(sharedState.sharedHistories.at(token.get_numa_index())), threadIdx(threadId), + numaThreadIdx(numaThreadId), + numaTotal(numaTotalThreads), numaAccessToken(token), manager(std::move(sm)), options(sharedState.options), @@ -544,7 +549,7 @@ void Search::Worker::do_move( nodes.store(nodes.load(std::memory_order_relaxed) + 1, std::memory_order_relaxed); auto [dirtyPiece, dirtyThreats] = accumulatorStack.push(); - pos.do_move(move, st, givesCheck, dirtyPiece, dirtyThreats, &tt); + pos.do_move(move, st, givesCheck, dirtyPiece, dirtyThreats, &tt, &sharedHistory); if (ss != nullptr) { @@ -576,9 +581,13 @@ void Search::Worker::clear() { mainHistory.fill(68); captureHistory.fill(-689); pawnHistory.fill(-1238); - pawnCorrectionHistory.fill(5); - minorPieceCorrectionHistory.fill(0); - nonPawnCorrectionHistory.fill(0); + + // Each thread is responsible for clearing their part of shared history + size_t len = sharedHistory.get_size() / numaTotal; + size_t start = numaThreadIdx * len; + size_t end = std::min(start + len, sharedHistory.get_size()); + + sharedHistory.correctionHistory.clear_range(start, end); ttMoveHistory = 0; diff --git a/src/search.h b/src/search.h index f3d99d415..9644f6aa5 100644 --- a/src/search.h +++ b/src/search.h @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -136,15 +137,18 @@ struct SharedState { SharedState(const OptionsMap& optionsMap, ThreadPool& threadPool, TranspositionTable& transpositionTable, + std::map& sharedHists, const LazyNumaReplicatedSystemWide& nets) : options(optionsMap), threads(threadPool), tt(transpositionTable), + sharedHistories(sharedHists), networks(nets) {} const OptionsMap& options; ThreadPool& threads; TranspositionTable& tt; + std::map& sharedHistories; const LazyNumaReplicatedSystemWide& networks; }; @@ -258,13 +262,17 @@ class NullSearchManager: public ISearchManager { void check_time(Search::Worker&) override {} }; - // Search::Worker is the class that does the actual search. // It is instantiated once per thread, and it is responsible for keeping track // of the search history, and storing data required for the search. class Worker { public: - Worker(SharedState&, std::unique_ptr, size_t, NumaReplicatedAccessToken); + Worker(SharedState&, + std::unique_ptr, + size_t, + size_t, + size_t, + NumaReplicatedAccessToken); // Called at instantiation to initialize reductions tables. // Reset histories, usually before a new game. @@ -282,16 +290,13 @@ class Worker { ButterflyHistory mainHistory; LowPlyHistory lowPlyHistory; - CapturePieceToHistory captureHistory; - ContinuationHistory continuationHistory[2][2]; - PawnHistory pawnHistory; - - CorrectionHistory pawnCorrectionHistory; - CorrectionHistory minorPieceCorrectionHistory; - CorrectionHistory nonPawnCorrectionHistory; + CapturePieceToHistory captureHistory; + ContinuationHistory continuationHistory[2][2]; + PawnHistory pawnHistory; CorrectionHistory continuationCorrectionHistory; - TTMoveHistory ttMoveHistory; + TTMoveHistory ttMoveHistory; + SharedHistories& sharedHistory; private: void iterative_deepening(); @@ -338,7 +343,7 @@ class Worker { Depth rootDepth, completedDepth; Value rootDelta; - size_t threadIdx; + size_t threadIdx, numaThreadIdx, numaTotal; NumaReplicatedAccessToken numaAccessToken; // Reductions lookup table initialized at startup diff --git a/src/thread.cpp b/src/thread.cpp index 58840a874..eaf1d09bd 100644 --- a/src/thread.cpp +++ b/src/thread.cpp @@ -21,11 +21,14 @@ #include #include #include +#include #include #include #include #include +#include "bitboard.h" +#include "history.h" #include "memory.h" #include "movegen.h" #include "search.h" @@ -42,8 +45,12 @@ namespace Stockfish { Thread::Thread(Search::SharedState& sharedState, std::unique_ptr sm, size_t n, + size_t numaN, + size_t totalNumaCount, OptionalThreadToNumaNodeBinder binder) : idx(n), + idxInNuma(numaN), + totalNuma(totalNumaCount), nthreads(sharedState.options["Threads"]), stdThread(&Thread::idle_loop, this) { @@ -54,8 +61,8 @@ Thread::Thread(Search::SharedState& sharedState, // the Worker allocation. Ideally we would also allocate the SearchManager // here, but that's minor. this->numaAccessToken = binder(); - this->worker = make_unique_large_page(sharedState, std::move(sm), n, - this->numaAccessToken); + this->worker = make_unique_large_page( + sharedState, std::move(sm), n, idxInNuma, totalNuma, this->numaAccessToken); }); wait_for_search_finished(); @@ -134,6 +141,8 @@ Search::SearchManager* ThreadPool::main_manager() { return main_thread()->worker uint64_t ThreadPool::nodes_searched() const { return accumulate(&Search::Worker::nodes); } uint64_t ThreadPool::tb_hits() const { return accumulate(&Search::Worker::tbHits); } +static size_t next_power_of_two(uint64_t count) { return count > 1 ? (2ULL << msb(count - 1)) : 1; } + // Creates/destroys threads to match the requested number. // Created and launched threads will immediately go to sleep in idle_loop. // Upon resizing, threads are recreated to allow for binding if necessary. @@ -172,10 +181,36 @@ void ThreadPool::set(const NumaConfig& numaConfig, return true; }(); + std::map counts; boundThreadToNumaNode = doBindThreads ? numaConfig.distribute_threads_among_numa_nodes(requested) : std::vector{}; + if (boundThreadToNumaNode.empty()) + counts[0] = requested; // Pretend all threads are part of numa node 0 + else + { + for (size_t i = 0; i < boundThreadToNumaNode.size(); ++i) + counts[boundThreadToNumaNode[i]]++; + } + + sharedState.sharedHistories.clear(); + for (auto pair : counts) + { + NumaIndex numaIndex = pair.first; + uint64_t count = pair.second; + auto f = [&]() { + sharedState.sharedHistories.try_emplace(numaIndex, next_power_of_two(count)); + }; + if (doBindThreads) + numaConfig.execute_on_numa_node(numaIndex, f); + else + f(); + } + + auto threadsPerNode = counts; + counts.clear(); + while (threads.size() < requested) { const size_t threadId = threads.size(); @@ -191,8 +226,9 @@ void ThreadPool::set(const NumaConfig& numaConfig, auto binder = doBindThreads ? OptionalThreadToNumaNodeBinder(numaConfig, numaId) : OptionalThreadToNumaNodeBinder(numaId); - threads.emplace_back( - std::make_unique(sharedState, std::move(manager), threadId, binder)); + threads.emplace_back(std::make_unique(sharedState, std::move(manager), threadId, + counts[numaId]++, threadsPerNode[numaId], + binder)); } clear(); diff --git a/src/thread.h b/src/thread.h index 79376b10a..2368c3069 100644 --- a/src/thread.h +++ b/src/thread.h @@ -76,6 +76,8 @@ class Thread { Thread(Search::SharedState&, std::unique_ptr, size_t, + size_t, + size_t, OptionalThreadToNumaNodeBinder); virtual ~Thread(); @@ -100,7 +102,7 @@ class Thread { private: std::mutex mutex; std::condition_variable cv; - size_t idx, nthreads; + size_t idx, idxInNuma, totalNuma, nthreads; bool exit = false, searching = true; // Set before starting std::thread NativeThread stdThread; NumaReplicatedAccessToken numaAccessToken;