Merge pull request #6497 from sesse/cluster-master-fmt

Merge cluster up to latest master

README.md

@@ -59,6 +59,33 @@ This distribution of Stockfish consists of the following files:
   * a file with the .nnue extension, storing the neural network for the NNUE
     evaluation. Binary distributions will have this file embedded.
 
+## Stockfish on distributed memory systems
+
+The cluster branch allows for running Stockfish on a cluster of servers (nodes)
+that are connected with a high-speed and low-latency network, using the message
+passing interface (MPI). In this case, one MPI process should be run per node,
+and UCI options can be used to set the number of threads/hash per node as usual.
+Typically, the engine will be invoked as
+```
+mpirun -np N /path/to/stockfish
+```
+where ```N``` stands for the number of MPI processes used (alternatives to ```mpirun```,
+include ```mpiexec```, ```srun```). Use 1 mpi rank per node, and employ threading
+according to the cores per node. To build the cluster
+branch, it is sufficient to specify ```COMPCXX=mpicxx``` (or e.g. CC depending on the name
+of the compiler providing MPI support) on the make command line, and do a clean build:
+```
+make -j ARCH=x86-64-modern clean build COMPCXX=mpicxx mpi=yes
+```
+Make sure that the MPI installation is configured to support ```MPI_THREAD_MULTIPLE```,
+this might require adding system specific compiler options to the Makefile. Stockfish employs
+non-blocking (asynchronous) communication, and benefits from an MPI
+implementation that efficiently supports this. Some MPI implentations might benefit
+from leaving 1 core/thread free for these asynchronous communications, and might require
+setting additional environment variables. ```mpirun``` should forward stdin/stdout
+to ```rank 0``` only (e.g. ```srun --input=0 --output=0```).
+Refer to your MPI documentation for more info.
+
 ## Contributing
 
 __See [Contributing Guide](CONTRIBUTING.md).__

src/Makefile

@@ -68,7 +68,7 @@ PGOBENCH = $(RUN_PREFIX) ./$(EXE) bench
 
 ### Source and object files
 SRCS = benchmark.cpp bitboard.cpp evaluate.cpp main.cpp \
-	misc.cpp movegen.cpp movepick.cpp position.cpp \
+	misc.cpp movegen.cpp movepick.cpp position.cpp cluster.cpp \
 	search.cpp thread.cpp timeman.cpp tt.cpp uci.cpp ucioption.cpp tune.cpp syzygy/tbprobe.cpp \
 	nnue/nnue_accumulator.cpp nnue/nnue_misc.cpp nnue/network.cpp \
 	nnue/features/half_ka_v2_hm.cpp nnue/features/full_threats.cpp \
@@ -80,7 +80,8 @@ HEADERS = benchmark.h bitboard.h evaluate.h misc.h movegen.h movepick.h history.
 		nnue/layers/clipped_relu.h nnue/layers/sqr_clipped_relu.h nnue/nnue_accumulator.h \
 		nnue/nnue_architecture.h nnue/nnue_common.h nnue/nnue_feature_transformer.h nnue/simd.h \
 		position.h search.h syzygy/tbprobe.h thread.h thread_win32_osx.h timeman.h \
-		tt.h tune.h types.h uci.h ucioption.h perft.h nnue/network.h engine.h score.h numa.h memory.h
+		tt.h tune.h types.h uci.h ucioption.h perft.h nnue/network.h engine.h score.h numa.h memory.h \
+		cluster.h
 
 OBJS = $(notdir $(SRCS:.cpp=.o))
 
@@ -121,6 +122,7 @@ VPATH = syzygy:nnue:nnue/features
 # dotprod = yes/no    --- -DUSE_NEON_DOTPROD --- Use ARM advanced SIMD Int8 dot product instructions
 # lsx = yes/no        --- -mlsx              --- Use Loongson SIMD eXtension
 # lasx = yes/no       --- -mlasx             --- use Loongson Advanced SIMD eXtension
+# mpi = yes/no        --- -DUSE_MPI        --- Use Message Passing Interface
 #
 # Note that Makefile is space sensitive, so when adding new architectures
 # or modifying existing flags, you have to make sure there are no extra spaces
@@ -173,6 +175,7 @@ avx512icl = no
 altivec = no
 vsx = no
 neon = no
+mpi = no
 dotprod = no
 arm_version = 0
 lsx = no
@@ -892,6 +895,15 @@ ifeq ($(OS), Android)
 	LDFLAGS += -fPIE -pie
 endif
 
+### 3.10 MPI
+ifneq (,$(findstring mpi, $(CXX)))
+	mpi = yes
+endif
+ifeq ($(mpi),yes)
+	CXXFLAGS += -DUSE_MPI -Wno-cast-qual -fexceptions
+        DEPENDFLAGS += -DUSE_MPI
+endif
+
 ### ==========================================================================
 ### Section 4. Public Targets
 ### ==========================================================================
@@ -1066,6 +1078,7 @@ config-sanity: net
 	echo "altivec: '$(altivec)'" && \
 	echo "vsx: '$(vsx)'" && \
 	echo "neon: '$(neon)'" && \
+	echo "mpi: '$(mpi)'" && \
 	echo "dotprod: '$(dotprod)'" && \
 	echo "arm_version: '$(arm_version)'" && \
 	echo "lsx: '$(lsx)'" && \

src/cluster.cpp

@@ -0,0 +1,490 @@
+/*
+  Stockfish, a UCI chess playing engine derived from Glaurung 2.1
+  Copyright (C) 2004-2026 The Stockfish developers (see AUTHORS file)
+
+  Stockfish is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  Stockfish is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifdef USE_MPI
+
+    #include <array>
+    #include <cstddef>
+    #include <cstdlib>
+    #include <iostream>
+    #include <istream>
+    #include <map>
+    #include <mpi.h>
+    #include <string>
+    #include <vector>
+
+    #include "cluster.h"
+    #include "thread.h"
+    #include "timeman.h"
+    #include "tt.h"
+    #include "search.h"
+
+namespace Stockfish {
+namespace Distributed {
+
+// Total number of ranks and rank within the communicator
+static int world_rank = MPI_PROC_NULL;
+static int world_size = 0;
+
+// Signals between ranks exchange basic info using a dedicated communicator
+static MPI_Comm    signalsComm        = MPI_COMM_NULL;
+static MPI_Request reqSignals         = MPI_REQUEST_NULL;
+static uint64_t    signalsCallCounter = 0;
+
+// Signals are the number of nodes searched, stop, table base hits, transposition table saves
+enum Signals : int {
+    SIG_NODES = 0,
+    SIG_STOP  = 1,
+    SIG_TB    = 2,
+    SIG_TTS   = 3,
+    SIG_NB    = 4
+};
+static uint64_t signalsSend[SIG_NB] = {};
+static uint64_t signalsRecv[SIG_NB] = {};
+static uint64_t nodesSearchedOthers = 0;
+static uint64_t tbHitsOthers        = 0;
+static uint64_t TTsavesOthers       = 0;
+static uint64_t stopSignalsPosted   = 0;
+
+// The UCI threads of each rank exchange use a dedicated communicator
+static MPI_Comm InputComm = MPI_COMM_NULL;
+
+// bestMove requires MoveInfo communicators and data types
+static MPI_Comm     MoveComm   = MPI_COMM_NULL;
+static MPI_Datatype MIDatatype = MPI_DATATYPE_NULL;
+
+// TT entries are communicated with a dedicated communicator.
+// The receive buffer is used to gather information from all ranks.
+// THe TTCacheCounter tracks the number of local elements that are ready to be sent.
+static MPI_Comm                                 TTComm = MPI_COMM_NULL;
+static std::array<std::vector<KeyedTTEntry>, 2> TTSendRecvBuffs;
+static std::array<MPI_Request, 2> reqsTTSendRecv = {MPI_REQUEST_NULL, MPI_REQUEST_NULL};
+static uint64_t                   sendRecvPosted = 0;
+static std::atomic<uint64_t>      TTCacheCounter = {};
+
+/// Initialize MPI and associated data types. Note that the MPI library must be configured
+/// to support MPI_THREAD_MULTIPLE, since multiple threads access MPI simultaneously.
+void init() {
+
+    int thread_support;
+    MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &thread_support);
+    if (thread_support < MPI_THREAD_MULTIPLE)
+    {
+        std::cerr << "Stockfish requires support for MPI_THREAD_MULTIPLE." << std::endl;
+        std::exit(EXIT_FAILURE);
+    }
+
+    MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &world_size);
+
+    const std::array<MPI_Aint, 5> MIdisps = {offsetof(MoveInfo, move), offsetof(MoveInfo, ponder),
+                                             offsetof(MoveInfo, depth), offsetof(MoveInfo, score),
+                                             offsetof(MoveInfo, rank)};
+    MPI_Type_create_hindexed_block(5, 1, MIdisps.data(), MPI_INT, &MIDatatype);
+    MPI_Type_commit(&MIDatatype);
+
+    MPI_Comm_dup(MPI_COMM_WORLD, &InputComm);
+    MPI_Comm_dup(MPI_COMM_WORLD, &TTComm);
+    MPI_Comm_dup(MPI_COMM_WORLD, &MoveComm);
+    MPI_Comm_dup(MPI_COMM_WORLD, &signalsComm);
+}
+
+/// Finalize MPI and free the associated data types.
+void finalize() {
+
+    MPI_Type_free(&MIDatatype);
+
+    MPI_Comm_free(&InputComm);
+    MPI_Comm_free(&TTComm);
+    MPI_Comm_free(&MoveComm);
+    MPI_Comm_free(&signalsComm);
+
+    MPI_Finalize();
+}
+
+/// Return the total number of ranks
+int size() { return world_size; }
+
+/// Return the rank (index) of the process
+int rank() { return world_rank; }
+
+/// The receive buffer depends on the number of MPI ranks and threads, resize as needed
+void ttSendRecvBuff_resize(size_t nThreads) {
+
+    for (int i : {0, 1})
+    {
+        TTSendRecvBuffs[i].resize(TTCacheSize * world_size * nThreads);
+        std::fill(TTSendRecvBuffs[i].begin(), TTSendRecvBuffs[i].end(), KeyedTTEntry());
+    }
+}
+
+/// As input is only received by the root (rank 0) of the cluster, this input must be relayed
+/// to the UCI threads of all ranks, in order to setup the position, etc. We do this with a
+/// dedicated getline implementation, where the root broadcasts to all other ranks the received
+/// information.
+bool getline(std::istream& input, std::string& str) {
+
+    int               size;
+    std::vector<char> vec;
+    int               state;
+
+    if (is_root())
+    {
+        state = static_cast<bool>(std::getline(input, str));
+        vec.assign(str.begin(), str.end());
+        size = vec.size();
+    }
+
+    // Some MPI implementations use busy-wait polling, while we need yielding as otherwise
+    // the UCI thread on the non-root ranks would be consuming resources.
+    static MPI_Request reqInput = MPI_REQUEST_NULL;
+    MPI_Ibcast(&size, 1, MPI_INT, 0, InputComm, &reqInput);
+    if (is_root())
+        MPI_Wait(&reqInput, MPI_STATUS_IGNORE);
+    else
+    {
+        while (true)
+        {
+            int flag;
+            MPI_Test(&reqInput, &flag, MPI_STATUS_IGNORE);
+            if (flag)
+                break;
+            else
+                std::this_thread::sleep_for(std::chrono::milliseconds(10));
+        }
+    }
+
+    // Broadcast received string
+    if (!is_root())
+        vec.resize(size);
+    MPI_Bcast(vec.data(), size, MPI_CHAR, 0, InputComm);
+    if (!is_root())
+        str.assign(vec.begin(), vec.end());
+    MPI_Bcast(&state, 1, MPI_INT, 0, InputComm);
+
+    return state;
+}
+
+/// Sending part of the signal communication loop
+namespace {
+void signals_send(const ThreadPool& threads) {
+
+    signalsSend[SIG_NODES] = threads.nodes_searched();
+    signalsSend[SIG_TB]    = threads.tb_hits();
+    signalsSend[SIG_TTS]   = threads.TT_saves();
+    signalsSend[SIG_STOP]  = threads.stop;
+    MPI_Iallreduce(signalsSend, signalsRecv, SIG_NB, MPI_UINT64_T, MPI_SUM, signalsComm,
+                   &reqSignals);
+    ++signalsCallCounter;
+}
+
+
+/// Processing part of the signal communication loop.
+/// For some counters (e.g. nodes) we only keep their sum on the other nodes
+/// allowing to add local counters at any time for more fine grained process,
+/// which is useful to indicate progress during early iterations, and to have
+/// node counts that exactly match the non-MPI code in the single rank case.
+/// This call also propagates the stop signal between ranks.
+void signals_process(ThreadPool& threads) {
+
+    nodesSearchedOthers = signalsRecv[SIG_NODES] - signalsSend[SIG_NODES];
+    tbHitsOthers        = signalsRecv[SIG_TB] - signalsSend[SIG_TB];
+    TTsavesOthers       = signalsRecv[SIG_TTS] - signalsSend[SIG_TTS];
+    stopSignalsPosted   = signalsRecv[SIG_STOP];
+    if (signalsRecv[SIG_STOP] > 0)
+        threads.stop = true;
+}
+
+void sendrecv_post() {
+
+    ++sendRecvPosted;
+    MPI_Irecv(TTSendRecvBuffs[sendRecvPosted % 2].data(),
+              TTSendRecvBuffs[sendRecvPosted % 2].size() * sizeof(KeyedTTEntry), MPI_BYTE,
+              (rank() + size() - 1) % size(), 42, TTComm, &reqsTTSendRecv[0]);
+    MPI_Isend(TTSendRecvBuffs[(sendRecvPosted + 1) % 2].data(),
+              TTSendRecvBuffs[(sendRecvPosted + 1) % 2].size() * sizeof(KeyedTTEntry), MPI_BYTE,
+              (rank() + 1) % size(), 42, TTComm, &reqsTTSendRecv[1]);
+}
+}
+
+/// During search, most message passing is asynchronous, but at the end of
+/// search it makes sense to bring them to a common, finalized state.
+void signals_sync(ThreadPool& threads) {
+
+    while (stopSignalsPosted < uint64_t(size()))
+        signals_poll(threads);
+
+    // Finalize outstanding messages of the signal loops.
+    // We might have issued one call less than needed on some ranks.
+    uint64_t globalCounter;
+    MPI_Allreduce(&signalsCallCounter, &globalCounter, 1, MPI_UINT64_T, MPI_MAX, MoveComm);
+    if (signalsCallCounter < globalCounter)
+    {
+        MPI_Wait(&reqSignals, MPI_STATUS_IGNORE);
+        signals_send(threads);
+    }
+    assert(signalsCallCounter == globalCounter);
+    MPI_Wait(&reqSignals, MPI_STATUS_IGNORE);
+    signals_process(threads);
+
+    // Finalize outstanding messages in the sendRecv loop
+    MPI_Allreduce(&sendRecvPosted, &globalCounter, 1, MPI_UINT64_T, MPI_MAX, MoveComm);
+    while (sendRecvPosted < globalCounter)
+    {
+        MPI_Waitall(reqsTTSendRecv.size(), reqsTTSendRecv.data(), MPI_STATUSES_IGNORE);
+        sendrecv_post();
+    }
+    assert(sendRecvPosted == globalCounter);
+    MPI_Waitall(reqsTTSendRecv.size(), reqsTTSendRecv.data(), MPI_STATUSES_IGNORE);
+}
+
+/// Initialize signal counters to zero.
+void signals_init() {
+
+    stopSignalsPosted = tbHitsOthers = TTsavesOthers = nodesSearchedOthers = 0;
+
+    signalsSend[SIG_NODES] = signalsRecv[SIG_NODES] = 0;
+    signalsSend[SIG_TB] = signalsRecv[SIG_TB] = 0;
+    signalsSend[SIG_TTS] = signalsRecv[SIG_TTS] = 0;
+    signalsSend[SIG_STOP] = signalsRecv[SIG_STOP] = 0;
+}
+
+/// Poll the signal loop, and start next round as needed.
+void signals_poll(ThreadPool& threads) {
+
+    int flag;
+    MPI_Test(&reqSignals, &flag, MPI_STATUS_IGNORE);
+    if (flag)
+    {
+        signals_process(threads);
+        signals_send(threads);
+    }
+}
+
+/// Provide basic info related the cluster performance, in particular, the number of signals send,
+/// signals per sounds (sps), the number of gathers, the number of positions gathered (per node and per second, gpps)
+/// The number of TT saves and TT saves per second. If gpps equals approximately TTSavesps the gather loop has enough bandwidth.
+void cluster_info(const ThreadPool& threads, Depth depth, TimePoint elapsed) {
+
+    // TimePoint elapsed = Time.elapsed() + 1;
+    uint64_t TTSaves = TT_saves(threads);
+
+    sync_cout << "info depth " << depth << " cluster "
+              << " signals " << signalsCallCounter << " sps " << signalsCallCounter * 1000 / elapsed
+              << " sendRecvs " << sendRecvPosted << " srpps "
+              << TTSendRecvBuffs[0].size() * sendRecvPosted * 1000 / elapsed << " TTSaves "
+              << TTSaves << " TTSavesps " << TTSaves * 1000 / elapsed << sync_endl;
+}
+
+/// When a TT entry is saved, additional steps are taken if the entry is of sufficient depth.
+/// If sufficient entries has been collected, a communication is initiated.
+/// If a communication has been completed, the received results are saved to the TT.
+void save(TranspositionTable& TT,
+          ThreadPool&         threads,
+          Search::Worker*     thread,
+          TTWriter            ttWriter,
+          Key                 k,
+          Value               v,
+          bool                PvHit,
+          Bound               b,
+          Depth               d,
+          Move                m,
+          Value               ev,
+          uint8_t             generation8) {
+
+    // Standard save to the TT
+    ttWriter.write(k, v, PvHit, b, d, m, ev, generation8);
+
+    // If the entry is of sufficient depth to be worth communicating, take action.
+    if (d > 3)
+    {
+        // count the TTsaves to information: this should be relatively similar
+        // to the number of entries we can send/recv.
+        thread->TTsaves.fetch_add(1, std::memory_order_relaxed);
+
+        // Add to thread's send buffer, the locking here avoids races when the master thread
+        // prepares the send buffer.
+        {
+            std::lock_guard<std::mutex> lk(thread->ttCache.mutex);
+            thread->ttCache.buffer.replace(KeyedTTEntry(k, TTData(m, v, ev, d, b, PvHit)));
+            ++TTCacheCounter;
+        }
+
+        size_t recvBuffPerRankSize = threads.size() * TTCacheSize;
+
+        // Communicate on main search thread, as soon the threads combined have collected
+        // sufficient data to fill the send buffers.
+        if (thread == threads.main_thread()->worker.get() && TTCacheCounter > recvBuffPerRankSize)
+        {
+            // Test communication status
+            int flag;
+            MPI_Testall(reqsTTSendRecv.size(), reqsTTSendRecv.data(), &flag, MPI_STATUSES_IGNORE);
+
+            // Current communication is complete
+            if (flag)
+            {
+                // Save all received entries to TT, and store our TTCaches, ready for the next round of communication
+                for (size_t irank = 0; irank < size_t(size()); ++irank)
+                {
+                    if (irank
+                        == size_t(
+                          rank()))  // this is our part, fill the part of the buffer for sending
+                    {
+                        // Copy from the thread caches to the right spot in the buffer
+                        size_t i = irank * recvBuffPerRankSize;
+                        for (auto&& th : threads)
+                        {
+                            std::lock_guard<std::mutex> lk(th->worker->ttCache.mutex);
+
+                            for (auto&& e : th->worker->ttCache.buffer)
+                                TTSendRecvBuffs[sendRecvPosted % 2][i++] = e;
+
+                            // Reset thread's send buffer
+                            th->worker->ttCache.buffer = {};
+                        }
+
+                        TTCacheCounter = 0;
+                    }
+                    else  // process data received from the corresponding rank.
+                        for (size_t i = irank * recvBuffPerRankSize;
+                             i < (irank + 1) * recvBuffPerRankSize; ++i)
+                        {
+                            auto&& e = TTSendRecvBuffs[sendRecvPosted % 2][i];
+                            auto [ttHit, ttData, ttWriterForRecvd] = TT.probe(e.first);
+                            ttWriterForRecvd.write(e.first, e.second.value, e.second.is_pv,
+                                                   e.second.bound, e.second.depth, e.second.move,
+                                                   e.second.eval, TT.generation());
+                        }
+                }
+
+                // Start next communication
+                sendrecv_post();
+
+                // Force check of time on the next occasion, the above actions might have taken some time.
+                thread->main_manager()->callsCnt = 0;
+            }
+        }
+    }
+}
+
+/// Picks the bestMove across ranks, and send the associated info and PV to the root of the cluster.
+/// Note that this bestMove and PV must be output by the root, the guarantee proper ordering of output.
+/// TODO update to the scheme in master.. can this use aggregation of votes?
+void pick_moves(MoveInfo& mi, std::vector<std::vector<char>>& serializedInfo) {
+
+    MoveInfo* pMoveInfo = NULL;
+    if (is_root())
+    {
+        pMoveInfo = (MoveInfo*) malloc(sizeof(MoveInfo) * size());
+    }
+    MPI_Gather(&mi, 1, MIDatatype, pMoveInfo, 1, MIDatatype, 0, MoveComm);
+
+    if (is_root())
+    {
+        std::map<int, int> votes;
+        int                minScore = pMoveInfo[0].score;
+        for (int i = 0; i < size(); ++i)
+        {
+            minScore                 = std::min(minScore, pMoveInfo[i].score);
+            votes[pMoveInfo[i].move] = 0;
+        }
+        for (int i = 0; i < size(); ++i)
+        {
+            votes[pMoveInfo[i].move] += pMoveInfo[i].score - minScore + pMoveInfo[i].depth;
+        }
+        int bestVote = votes[pMoveInfo[0].move];
+        for (int i = 0; i < size(); ++i)
+        {
+            if (votes[pMoveInfo[i].move] > bestVote)
+            {
+                bestVote = votes[pMoveInfo[i].move];
+                mi       = pMoveInfo[i];
+            }
+        }
+        free(pMoveInfo);
+    }
+
+    // Send around the final result
+    MPI_Bcast(&mi, 1, MIDatatype, 0, MoveComm);
+
+    // Send PV line to root as needed
+    if (mi.rank != 0 && mi.rank == rank())
+    {
+        int numLines = serializedInfo.size();
+        MPI_Send(&numLines, 1, MPI_INT, 0, 42, MoveComm);
+
+        for (const auto& serializedInfoOne : serializedInfo)
+        {
+            int size;
+            size = serializedInfoOne.size();
+            MPI_Send(&size, 1, MPI_INT, 0, 42, MoveComm);
+            MPI_Send(serializedInfoOne.data(), size, MPI_CHAR, 0, 42, MoveComm);
+        }
+    }
+    if (mi.rank != 0 && is_root())
+    {
+        serializedInfo.clear();
+
+        int numLines;
+        MPI_Recv(&numLines, 1, MPI_INT, mi.rank, 42, MoveComm, MPI_STATUS_IGNORE);
+
+        for (int i = 0; i < numLines; ++i)
+        {
+            int               size;
+            std::vector<char> vec;
+            MPI_Recv(&size, 1, MPI_INT, mi.rank, 42, MoveComm, MPI_STATUS_IGNORE);
+            vec.resize(size);
+            MPI_Recv(vec.data(), size, MPI_CHAR, mi.rank, 42, MoveComm, MPI_STATUS_IGNORE);
+            serializedInfo.push_back(std::move(vec));
+        }
+    }
+}
+
+/// Return nodes searched (lazily updated cluster wide in the signal loop)
+uint64_t nodes_searched(const ThreadPool& threads) {
+    return nodesSearchedOthers + threads.nodes_searched();
+}
+
+/// Return table base hits (lazily updated cluster wide in the signal loop)
+uint64_t tb_hits(const ThreadPool& threads) { return tbHitsOthers + threads.tb_hits(); }
+
+/// Return the number of saves to the TT buffers, (lazily updated cluster wide in the signal loop)
+uint64_t TT_saves(const ThreadPool& threads) { return TTsavesOthers + threads.TT_saves(); }
+
+
+}
+}
+
+#else
+
+    #include "cluster.h"
+    #include "thread.h"
+
+namespace Stockfish {
+namespace Distributed {
+
+uint64_t nodes_searched(const ThreadPool& threads) { return threads.nodes_searched(); }
+
+uint64_t tb_hits(const ThreadPool& threads) { return threads.tb_hits(); }
+
+uint64_t TT_saves(const ThreadPool& threads) { return threads.TT_saves(); }
+
+}
+}
+
+#endif  // USE_MPI

src/cluster.h

@@ -0,0 +1,158 @@
+/*
+  Stockfish, a UCI chess playing engine derived from Glaurung 2.1
+  Copyright (C) 2004-2026 The Stockfish developers (see AUTHORS file)
+
+  Stockfish is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  Stockfish is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef CLUSTER_H_INCLUDED
+#define CLUSTER_H_INCLUDED
+
+#include <algorithm>
+#include <array>
+#include <istream>
+#include <string>
+
+#include "misc.h"
+#include "tt.h"
+
+namespace Stockfish {
+class Thread;
+class ThreadPool;
+
+namespace Search {
+class Worker;
+}
+
+/// The Distributed namespace contains functionality required to run on distributed
+/// memory architectures using MPI as the message passing interface. On a high level,
+/// a 'lazy SMP'-like scheme is implemented where TT saves of sufficient depth are
+/// collected on each rank and distributed to, and used by, all other ranks,
+/// which search essentially independently.  The root (MPI rank 0) of the cluster
+/// is responsible for all I/O and time management, communicating this info to
+/// the other ranks as needed. UCI options such as Threads and Hash specify these
+/// quantities per MPI rank.  It is recommended to have one rank (MPI process) per node.
+/// For the non-MPI case, wrappers that will be compiler-optimized away are provided.
+
+namespace Distributed {
+
+/// Basic info to find the cluster-wide bestMove
+struct MoveInfo {
+    int move;
+    int ponder;
+    int depth;
+    int score;
+    int rank;
+};
+
+#ifdef USE_MPI
+
+// store the TTData with its (full) key, so it can be saved on the receiver side
+using KeyedTTEntry                = std::pair<Key, TTData>;
+constexpr std::size_t TTCacheSize = 16;
+
+// Threads locally cache their high-depth TT entries till a batch can be send by MPI
+template<std::size_t N>
+class TTCache: public std::array<KeyedTTEntry, N> {
+
+    struct Compare {
+        inline bool operator()(const KeyedTTEntry& lhs, const KeyedTTEntry& rhs) {
+            return lhs.second.depth > rhs.second.depth;
+        }
+    };
+    Compare compare;
+
+   public:
+    // Keep a heap of entries replacing low depth with high depth entries
+    bool replace(const KeyedTTEntry& value) {
+
+        if (compare(value, this->front()))
+        {
+            std::pop_heap(this->begin(), this->end(), compare);
+            this->back() = value;
+            std::push_heap(this->begin(), this->end(), compare);
+            return true;
+        }
+        return false;
+    }
+};
+
+void        init();
+void        finalize();
+bool        getline(std::istream& input, std::string& str);
+int         size();
+int         rank();
+inline bool is_root() { return rank() == 0; }
+void        save(TranspositionTable&,
+                 ThreadPool&,
+                 Search::Worker* thread,
+                 TTWriter        ttWriter,
+                 Key             k,
+                 Value           v,
+                 bool            PvHit,
+                 Bound           b,
+                 Depth           d,
+                 Move            m,
+                 Value           ev,
+                 uint8_t         generation8);
+void        pick_moves(MoveInfo& mi, std::vector<std::vector<char>>& PVLine);
+void        ttSendRecvBuff_resize(size_t nThreads);
+uint64_t    nodes_searched(const ThreadPool&);
+uint64_t    tb_hits(const ThreadPool&);
+uint64_t    TT_saves(const ThreadPool&);
+void        cluster_info(const ThreadPool&, Depth depth, TimePoint elapsed);
+void        signals_init();
+void        signals_poll(ThreadPool& threads);
+void        signals_sync(ThreadPool& threads);
+
+#else
+
+inline void init() {}
+inline void finalize() {}
+inline bool getline(std::istream& input, std::string& str) {
+    return static_cast<bool>(std::getline(input, str));
+}
+constexpr int  size() { return 1; }
+constexpr int  rank() { return 0; }
+constexpr bool is_root() { return true; }
+inline void    save(TranspositionTable&,
+                    ThreadPool&,
+                    Search::Worker*,
+                    TTWriter ttWriter,
+                    Key      k,
+                    Value    v,
+                    bool     PvHit,
+                    Bound    b,
+                    Depth    d,
+                    Move     m,
+                    Value    ev,
+                    uint8_t  generation8) {
+    ttWriter.write(k, v, PvHit, b, d, m, ev, generation8);
+}
+inline void pick_moves(MoveInfo&, std::vector<std::vector<char>>&) {}
+inline void ttSendRecvBuff_resize(size_t) {}
+uint64_t    nodes_searched(const ThreadPool&);
+uint64_t    tb_hits(const ThreadPool&);
+uint64_t    TT_saves(const ThreadPool&);
+inline void cluster_info(const ThreadPool&, Depth, TimePoint) {}
+inline void signals_init() {}
+inline void signals_poll(ThreadPool&) {}
+inline void signals_sync(ThreadPool&) {}
+
+#endif /* USE_MPI */
+
+}
+}
+
+#endif  // #ifndef CLUSTER_H_INCLUDED

src/engine.cpp

@@ -251,6 +251,9 @@ void Engine::resize_threads() {
 void Engine::set_tt_size(size_t mb) {
     wait_for_search_finished();
     tt.resize(mb, threads);
+
+    // Adjust cluster buffers
+    Distributed::ttSendRecvBuff_resize(threads.num_threads());
 }
 
 void Engine::set_ponderhit(bool b) { threads.main_manager()->ponder = b; }

src/main.cpp

@@ -28,7 +28,9 @@
 using namespace Stockfish;
 
 int main(int argc, char* argv[]) {
-    std::cout << engine_info() << std::endl;
+    Distributed::init();
+    if (Distributed::is_root())
+        std::cout << engine_info() << std::endl;
 
     Bitboards::init();
     Position::init();
@@ -39,5 +41,7 @@ int main(int argc, char* argv[]) {
 
     uci->loop();
 
+    Distributed::finalize();
+
     return 0;
 }

src/nnue/network.cpp

@@ -28,6 +28,7 @@
 #define INCBIN_SILENCE_BITCODE_WARNING
 #include "../incbin/incbin.h"
 
+#include "../cluster.h"
 #include "../evaluate.h"
 #include "../misc.h"
 #include "../position.h"
@@ -221,11 +222,12 @@ void Network<Arch, Transformer>::verify(std::string
     if (f)
     {
         size_t size = sizeof(featureTransformer) + sizeof(Arch) * LayerStacks;
-        f("NNUE evaluation using " + evalfilePath + " (" + std::to_string(size / (1024 * 1024))
+        if (Distributed::is_root())
-          + "MiB, (" + std::to_string(featureTransformer.TotalInputDimensions) + ", "
+            f("NNUE evaluation using " + evalfilePath + " (" + std::to_string(size / (1024 * 1024))
-          + std::to_string(network[0].TransformedFeatureDimensions) + ", "
+              + "MiB, (" + std::to_string(featureTransformer.TotalInputDimensions) + ", "
-          + std::to_string(network[0].FC_0_OUTPUTS) + ", " + std::to_string(network[0].FC_1_OUTPUTS)
+              + std::to_string(network[0].TransformedFeatureDimensions) + ", "
-          + ", 1))");
+              + std::to_string(network[0].FC_0_OUTPUTS) + ", "
+              + std::to_string(network[0].FC_1_OUTPUTS) + ", 1))");
     }
 }
 

src/perft.h

@@ -21,6 +21,7 @@
 
 #include <cstdint>
 
+#include "cluster.h"
 #include "movegen.h"
 #include "position.h"
 #include "types.h"
@@ -49,7 +50,7 @@ uint64_t perft(Position& pos, Depth depth) {
             nodes += cnt;
             pos.undo_move(m);
         }
-        if (Root)
+        if (Root && Distributed::is_root())
             sync_cout << UCIEngine::move(m, pos.is_chess960()) << ": " << cnt << sync_endl;
     }
     return nodes;

src/search.cpp

@@ -34,6 +34,7 @@
 #include <utility>
 
 #include "bitboard.h"
+#include "cluster.h"
 #include "evaluate.h"
 #include "history.h"
 #include "misc.h"
@@ -198,8 +199,9 @@ void Search::Worker::start_searching() {
     if (rootMoves.empty())
     {
         rootMoves.emplace_back(Move::none());
-        main_manager()->updates.onUpdateNoMoves(
+        if (Distributed::is_root())
-          {0, {rootPos.checkers() ? -VALUE_MATE : VALUE_DRAW, rootPos}});
+            main_manager()->updates.onUpdateNoMoves(
+              {0, {rootPos.checkers() ? -VALUE_MATE : VALUE_DRAW, rootPos}});
     }
     else
     {
@@ -213,19 +215,24 @@ void Search::Worker::start_searching() {
     // GUI sends a "stop" or "ponderhit" command. We therefore simply wait here
     // until the GUI sends one of those commands.
     while (!threads.stop && (main_manager()->ponder || limits.infinite))
-    {}  // Busy wait for a stop or a ponder reset
+    {
+        Distributed::signals_poll(threads);
+    }  // Busy wait for a stop or a ponder reset
 
     // Stop the threads if not already stopped (also raise the stop if
     // "ponderhit" just reset threads.ponder)
     threads.stop = true;
 
+    // Signal and synchronize all other ranks
+    Distributed::signals_sync(threads);
+
     // Wait until all threads have finished
     threads.wait_for_search_finished();
 
     // When playing in 'nodes as time' mode, subtract the searched nodes from
     // the available ones before exiting.
     if (limits.npmsec)
-        main_manager()->tm.advance_nodes_time(threads.nodes_searched()
+        main_manager()->tm.advance_nodes_time(Distributed::nodes_searched(threads)
                                               - limits.inc[rootPos.side_to_move()]);
 
     Worker* bestThread = this;
@@ -239,18 +246,53 @@ void Search::Worker::start_searching() {
     main_manager()->bestPreviousScore        = bestThread->rootMoves[0].score;
     main_manager()->bestPreviousAverageScore = bestThread->rootMoves[0].averageScore;
 
-    // Send again PV info if we have a new best thread
+    // Temporarily switch out onUpdateFull to capture the PV information that we need,
-    if (bestThread != this)
+    // so that we can exchange it through MPI. (We may end up not actually printing
-        main_manager()->pv(*bestThread, threads, tt, bestThread->completedDepth);
+    // it out.)
-
+    auto oldOnUpdateFull = std::move(main_manager()->updates.onUpdateFull);
-    std::string ponder;
+    std::vector<std::vector<char>> serializedInfo;  // One for each MultiPV.
+    main_manager()->updates.onUpdateFull = [&](const InfoFull& info) {
+        serializedInfo.push_back(info.serialize());
+    };
+    main_manager()->pv(*bestThread, threads, tt, bestThread->completedDepth);
+    assert(!serializedInfo.empty());
+    main_manager()->updates.onUpdateFull = std::move(oldOnUpdateFull);
 
+    Move bestMove   = bestThread->rootMoves[0].pv[0];
+    Move ponderMove = Move::none();
     if (bestThread->rootMoves[0].pv.size() > 1
         || bestThread->rootMoves[0].extract_ponder_from_tt(tt, rootPos))
-        ponder = UCIEngine::move(bestThread->rootMoves[0].pv[1], rootPos.is_chess960());
+        ponderMove = bestThread->rootMoves[0].pv[1];
 
-    auto bestmove = UCIEngine::move(bestThread->rootMoves[0].pv[0], rootPos.is_chess960());
+    // Exchange info as needed
-    main_manager()->updates.onBestmove(bestmove, ponder);
+    Distributed::MoveInfo mi{bestMove.raw(), ponderMove.raw(), bestThread->completedDepth,
+                             bestThread->rootMoves[0].score, Distributed::rank()};
+    Distributed::pick_moves(mi, serializedInfo);
+
+    main_manager()->bestPreviousScore = static_cast<Value>(mi.score);
+
+    if (Distributed::is_root())
+    {
+        // Send again PV info if we have a new best thread/rank
+        if (bestThread != this || mi.rank != 0)
+        {
+            for (const auto& serializedInfoOne : serializedInfo)
+            {
+                Search::InfoFull info = Search::InfoFull::unserialize(serializedInfoOne);
+                main_manager()->updates.onUpdateFull(info);
+            }
+        }
+
+        bestMove   = static_cast<Move>(mi.move);
+        ponderMove = static_cast<Move>(mi.ponder);
+
+        std::string ponder;
+        if (ponderMove != Move::none())
+            ponder = UCIEngine::move(ponderMove, rootPos.is_chess960());
+
+        auto bestmove = UCIEngine::move(bestThread->rootMoves[0].pv[0], rootPos.is_chess960());
+        main_manager()->updates.onBestmove(bestmove, ponder);
+    }
 }
 
 // Main iterative deepening loop. It calls search()
@@ -320,7 +362,7 @@ void Search::Worker::iterative_deepening() {
 
     // Iterative deepening loop until requested to stop or the target depth is reached
     while (++rootDepth < MAX_PLY && !threads.stop
-           && !(limits.depth && mainThread && rootDepth > limits.depth))
+           && !(limits.depth && mainThread && Distributed::is_root() && rootDepth > limits.depth))
     {
         // Age out PV variability metric
         if (mainThread)
@@ -391,9 +433,12 @@ void Search::Worker::iterative_deepening() {
                 // When failing high/low give some update before a re-search. To avoid
                 // excessive output that could hang GUIs like Fritz 19, only start
                 // at nodes > 10M (rather than depth N, which can be reached quickly)
-                if (mainThread && multiPV == 1 && (bestValue <= alpha || bestValue >= beta)
+                if (Distributed::is_root() && mainThread && multiPV == 1
-                    && nodes > 10000000)
+                    && (bestValue <= alpha || bestValue >= beta) && nodes > 10000000)
+                {
                     main_manager()->pv(*this, threads, tt, rootDepth);
+                    Distributed::cluster_info(threads, rootDepth, elapsed());
+                }
 
                 // In case of failing low/high increase aspiration window and re-search,
                 // otherwise exit the loop.
@@ -423,7 +468,7 @@ void Search::Worker::iterative_deepening() {
             // Sort the PV lines searched so far and update the GUI
             std::stable_sort(rootMoves.begin() + pvFirst, rootMoves.begin() + pvIdx + 1);
 
-            if (mainThread
+            if (Distributed::is_root() && mainThread
                 && (threads.stop || pvIdx + 1 == multiPV || nodes > 10000000)
                 // A thread that aborted search can have mated-in/TB-loss PV and
                 // score that cannot be trusted, i.e. it can be delayed or refuted
@@ -431,7 +476,10 @@ void Search::Worker::iterative_deepening() {
                 // we suppress this output and below pick a proven score/PV for this
                 // thread (from the previous iteration).
                 && !(threads.abortedSearch && is_loss(rootMoves[0].uciScore)))
+            {
                 main_manager()->pv(*this, threads, tt, rootDepth);
+                Distributed::cluster_info(threads, rootDepth, elapsed() + 1);
+            }
 
             if (threads.stop)
                 break;
@@ -737,8 +785,8 @@ Value Search::Worker::search(
         ss->staticEval = eval = to_corrected_static_eval(unadjustedStaticEval, correctionValue);
 
         // Static evaluation is saved as it was before adjustment by correction history
-        ttWriter.write(posKey, VALUE_NONE, ss->ttPv, BOUND_NONE, DEPTH_UNSEARCHED, Move::none(),
+        Distributed::save(tt, threads, this, ttWriter, posKey, VALUE_NONE, ss->ttPv, BOUND_NONE,
-                       unadjustedStaticEval, tt.generation());
+                          DEPTH_UNSEARCHED, Move::none(), unadjustedStaticEval, tt.generation());
     }
 
     // Set up the improving flag, which is true if current static evaluation is
@@ -835,9 +883,9 @@ Value Search::Worker::search(
 
                 if (b == BOUND_EXACT || (b == BOUND_LOWER ? value >= beta : value <= alpha))
                 {
-                    ttWriter.write(posKey, value_to_tt(value, ss->ply), ss->ttPv, b,
+                    Distributed::save(
-                                   std::min(MAX_PLY - 1, depth + 6), Move::none(), VALUE_NONE,
+                      tt, threads, this, ttWriter, posKey, value_to_tt(value, ss->ply), ss->ttPv, b,
-                                   tt.generation());
+                      std::min(MAX_PLY - 1, depth + 6), Move::none(), VALUE_NONE, tt.generation());
 
                     return value;
                 }
@@ -971,8 +1019,9 @@ Value Search::Worker::search(
             if (value >= probCutBeta)
             {
                 // Save ProbCut data into transposition table
-                ttWriter.write(posKey, value_to_tt(value, ss->ply), ss->ttPv, BOUND_LOWER,
+                Distributed::save(tt, threads, this, ttWriter, posKey, value_to_tt(value, ss->ply),
-                               probCutDepth + 1, move, unadjustedStaticEval, tt.generation());
+                                  ss->ttPv, BOUND_LOWER, probCutDepth + 1, move,
+                                  unadjustedStaticEval, tt.generation());
 
                 if (!is_decisive(value))
                     return value - (probCutBeta - beta);
@@ -1021,7 +1070,7 @@ moves_loop:  // When in check, search starts here
 
         ss->moveCount = ++moveCount;
 
-        if (rootNode && is_mainthread() && nodes > 10000000)
+        if (rootNode && Distributed::is_root() && is_mainthread() && nodes > 10000000)
         {
             main_manager()->updates.onIter(
               {depth, UCIEngine::move(move, pos.is_chess960()), moveCount + pvIdx});
@@ -1464,12 +1513,13 @@ moves_loop:  // When in check, search starts here
     // Write gathered information in transposition table. Note that the
     // static evaluation is saved as it was before correction history.
     if (!excludedMove && !(rootNode && pvIdx))
-        ttWriter.write(posKey, value_to_tt(bestValue, ss->ply), ss->ttPv,
+        Distributed::save(tt, threads, this, ttWriter, posKey, value_to_tt(bestValue, ss->ply),
-                       bestValue >= beta    ? BOUND_LOWER
+                          ss->ttPv,
-                       : PvNode && bestMove ? BOUND_EXACT
+                          bestValue >= beta    ? BOUND_LOWER
-                                            : BOUND_UPPER,
+                          : PvNode && bestMove ? BOUND_EXACT
-                       moveCount != 0 ? depth : std::min(MAX_PLY - 1, depth + 6), bestMove,
+                                               : BOUND_UPPER,
-                       unadjustedStaticEval, tt.generation());
+                          moveCount != 0 ? depth : std::min(MAX_PLY - 1, depth + 6), bestMove,
+                          unadjustedStaticEval, tt.generation());
 
     // Adjust correction history if the best move is not a capture
     // and the error direction matches whether we are above/below bounds.
@@ -1593,9 +1643,11 @@ Value Search::Worker::qsearch(Position& pos, Stack* ss, Value alpha, Value beta)
                 bestValue = (bestValue + beta) / 2;
 
             if (!ss->ttHit)
-                ttWriter.write(posKey, value_to_tt(bestValue, ss->ply), false, BOUND_LOWER,
+                Distributed::save(tt, threads, this, ttWriter, posKey,
-                               DEPTH_UNSEARCHED, Move::none(), unadjustedStaticEval,
+                                  value_to_tt(bestValue, ss->ply), false, BOUND_LOWER,
-                               tt.generation());
+                                  DEPTH_UNSEARCHED, Move::none(), unadjustedStaticEval,
+                                  tt.generation());
+
             return bestValue;
         }
 
@@ -1724,9 +1776,9 @@ Value Search::Worker::qsearch(Position& pos, Stack* ss, Value alpha, Value beta)
 
     // Save gathered info in transposition table. The static evaluation
     // is saved as it was before adjustment by correction history.
-    ttWriter.write(posKey, value_to_tt(bestValue, ss->ply), pvHit,
+    Distributed::save(tt, threads, this, ttWriter, posKey, value_to_tt(bestValue, ss->ply), pvHit,
-                   bestValue >= beta ? BOUND_LOWER : BOUND_UPPER, DEPTH_QS, bestMove,
+                      bestValue >= beta ? BOUND_LOWER : BOUND_UPPER, DEPTH_QS, bestMove,
-                   unadjustedStaticEval, tt.generation());
+                      unadjustedStaticEval, tt.generation());
 
     assert(bestValue > -VALUE_INFINITE && bestValue < VALUE_INFINITE);
 
@@ -1747,7 +1799,7 @@ Depth Search::Worker::reduction(bool i, Depth d, int mn, int delta) const {
 // This function is intended for use only when printing PV outputs, and not used
 // for making decisions within the search algorithm itself.
 TimePoint Search::Worker::elapsed() const {
-    return main_manager()->tm.elapsed([this]() { return threads.nodes_searched(); });
+    return main_manager()->tm.elapsed([this]() { return Distributed::nodes_searched(threads); });
 }
 
 TimePoint Search::Worker::elapsed_time() const { return main_manager()->tm.elapsed_time(); }
@@ -1950,8 +2002,9 @@ void SearchManager::check_time(Search::Worker& worker) {
 
     static TimePoint lastInfoTime = now();
 
-    TimePoint elapsed = tm.elapsed([&worker]() { return worker.threads.nodes_searched(); });
+    TimePoint elapsed =
-    TimePoint tick    = worker.limits.startTime + elapsed;
+      tm.elapsed([&worker]() { return Distributed::nodes_searched(worker.threads); });
+    TimePoint tick = worker.limits.startTime + elapsed;
 
     if (tick - lastInfoTime >= 1000)
     {
@@ -1959,6 +2012,9 @@ void SearchManager::check_time(Search::Worker& worker) {
         dbg_print();
     }
 
+    // poll on MPI signals
+    Distributed::signals_poll(worker.threads);
+
     // We should not stop pondering until told so by the GUI
     if (ponder)
         return;
@@ -1969,7 +2025,8 @@ void SearchManager::check_time(Search::Worker& worker) {
       worker.completedDepth >= 1
       && ((worker.limits.use_time_management() && (elapsed > tm.maximum() || stopOnPonderhit))
           || (worker.limits.movetime && elapsed >= worker.limits.movetime)
-          || (worker.limits.nodes && worker.threads.nodes_searched() >= worker.limits.nodes)))
+          || (worker.limits.nodes
+              && Distributed::nodes_searched(worker.threads) >= worker.limits.nodes)))
         worker.threads.stop = worker.threads.abortedSearch = true;
 }
 
@@ -2116,12 +2173,13 @@ void SearchManager::pv(Search::Worker&           worker,
                        const TranspositionTable& tt,
                        Depth                     depth) {
 
-    const auto nodes     = threads.nodes_searched();
+    const auto nodes     = Distributed::nodes_searched(threads);
     auto&      rootMoves = worker.rootMoves;
     auto&      pos       = worker.rootPos;
     size_t     pvIdx     = worker.pvIdx;
     size_t     multiPV   = std::min(size_t(worker.options["MultiPV"]), rootMoves.size());
-    uint64_t   tbHits    = threads.tb_hits() + (worker.tbConfig.rootInTB ? rootMoves.size() : 0);
+    uint64_t   tbHits =
+      Distributed::tb_hits(threads) + (worker.tbConfig.rootInTB ? rootMoves.size() : 0);
 
     for (size_t i = 0; i < multiPV; ++i)
     {
@@ -2207,5 +2265,75 @@ bool RootMove::extract_ponder_from_tt(const TranspositionTable& tt, Position& po
     return pv.size() > 1;
 }
 
+std::vector<char> Search::InfoFull::serialize() const {
+    std::vector<char> vec;
+    vec.resize(sizeof(*this) + 3 * sizeof(size_t) + wdl.size() + bound.size() + pv.size());
+    char* ptr = vec.data();
+
+    // The base struct.
+    memcpy(ptr, this, sizeof(*this));
+    ptr += sizeof(*this);
+
+    // All string lengths.
+    size_t wdl_len = wdl.size();
+    memcpy(ptr, &wdl_len, sizeof(wdl_len));
+    ptr += sizeof(wdl_len);
+
+    size_t bound_len = bound.size();
+    memcpy(ptr, &bound_len, sizeof(bound_len));
+    ptr += sizeof(bound_len);
+
+    size_t pv_len = pv.size();
+    memcpy(ptr, &pv_len, sizeof(pv_len));
+    ptr += sizeof(pv_len);
+
+    // The string data itself.
+    memcpy(ptr, wdl.data(), wdl_len);
+    ptr += wdl_len;
+
+    memcpy(ptr, bound.data(), bound_len);
+    ptr += bound_len;
+
+    memcpy(ptr, pv.data(), pv_len);
+    ptr += pv_len;
+
+    assert(ptr == vec.data() + vec.size());
+    return vec;
+}
+
+InfoFull Search::InfoFull::unserialize(const std::vector<char>& buf) {
+    InfoFull    info;
+    const char* ptr = buf.data();
+
+    // The base struct.
+    memcpy(&info, ptr, sizeof(info));
+    ptr += sizeof(info);
+
+    // All string lengths.
+    size_t wdl_len;
+    memcpy(&wdl_len, ptr, sizeof(wdl_len));
+    ptr += sizeof(wdl_len);
+
+    size_t bound_len;
+    memcpy(&bound_len, ptr, sizeof(bound_len));
+    ptr += sizeof(bound_len);
+
+    size_t pv_len;
+    memcpy(&pv_len, ptr, sizeof(pv_len));
+    ptr += sizeof(pv_len);
+
+    // The string data itself.
+    info.wdl = std::string_view(ptr, wdl_len);
+    ptr += wdl_len;
+
+    info.bound = std::string_view(ptr, bound_len);
+    ptr += bound_len;
+
+    info.pv = std::string_view(ptr, pv_len);
+    ptr += pv_len;
+
+    assert(ptr == buf.data() + buf.size());
+    return info;
+}
 
 }  // namespace Stockfish

src/search.h

@@ -28,10 +28,12 @@
 #include <functional>
 #include <map>
 #include <memory>
+#include <mutex>
 #include <string>
 #include <string_view>
 #include <vector>
 
+#include "cluster.h"
 #include "history.h"
 #include "misc.h"
 #include "nnue/network.h"
@@ -121,7 +123,9 @@ struct LimitsType {
         ponderMode                                  = false;
     }
 
-    bool use_time_management() const { return time[WHITE] || time[BLACK]; }
+    bool use_time_management() const {
+        return Distributed::is_root() && (time[WHITE] || time[BLACK]);
+    }
 
     std::vector<std::string> searchmoves;
     TimePoint                time[COLOR_NB], inc[COLOR_NB], npmsec, movetime, startTime;
@@ -178,6 +182,14 @@ struct InfoFull: InfoShort {
     size_t           tbHits;
     std::string_view pv;
     int              hashfull;
+
+    std::vector<char> serialize() const;
+
+    // NOTE: The returned InfoFull will contain string_views that point
+    // into the given buffer, so it must live (and not be reallocated)
+    // (and not be reallocated) for at least as long as you want to
+    // access fields in the InfoFull.
+    static InfoFull unserialize(const std::vector<char>& buf);
 };
 
 struct InfoIteration {
@@ -231,7 +243,7 @@ class SearchManager: public ISearchManager {
     };
 
 
-    SearchManager(const UpdateContext& updateContext) :
+    SearchManager(UpdateContext& updateContext) :
         updates(updateContext) {}
 
     void check_time(Search::Worker& worker) override;
@@ -254,7 +266,7 @@ class SearchManager: public ISearchManager {
 
     size_t id;
 
-    const UpdateContext& updates;
+    UpdateContext& updates;
 };
 
 class NullSearchManager: public ISearchManager {
@@ -294,6 +306,28 @@ class Worker {
     ContinuationHistory             continuationHistory[2][2];
     CorrectionHistory<Continuation> continuationCorrectionHistory;
 
+#ifdef USE_MPI
+    struct {
+        std::mutex                                     mutex;
+        Distributed::TTCache<Distributed::TTCacheSize> buffer = {};
+    } ttCache;
+#endif
+
+    std::atomic<uint64_t> TTsaves;
+
+    friend void Distributed::save(TranspositionTable&,
+                                  ThreadPool&,
+                                  Search::Worker*,
+                                  TTWriter ttWriter,
+                                  Key      k,
+                                  Value    v,
+                                  bool     PvHit,
+                                  Bound    b,
+                                  Depth    d,
+                                  Move     m,
+                                  Value    ev,
+                                  uint8_t  generation8);
+
     TTMoveHistory    ttMoveHistory;
     SharedHistories& sharedHistory;
 

src/syzygy/tbprobe.cpp

@@ -38,6 +38,7 @@
 #include <array>
 
 #include "../bitboard.h"
+#include "../cluster.h"
 #include "../misc.h"
 #include "../movegen.h"
 #include "../position.h"
@@ -494,8 +495,9 @@ class TBTables {
     }
 
     void info() const {
-        sync_cout << "info string Found " << foundWDLFiles << " WDL and " << foundDTZFiles
+        if (Distributed::is_root())
-                  << " DTZ tablebase files (up to " << MaxCardinality << "-man)." << sync_endl;
+            sync_cout << "info string Found " << foundWDLFiles << " WDL and " << foundDTZFiles
+                      << " DTZ tablebase files (up to " << MaxCardinality << "-man)." << sync_endl;
     }
 
     void add(const std::vector<PieceType>& pieces);

src/thread.cpp

@@ -28,8 +28,10 @@
 #include <utility>
 
 #include "bitboard.h"
+#include "cluster.h"
 #include "history.h"
 #include "memory.h"
+#include "misc.h"
 #include "movegen.h"
 #include "search.h"
 #include "syzygy/tbprobe.h"
@@ -140,15 +142,16 @@ Search::SearchManager* ThreadPool::main_manager() { return main_thread()->worker
 
 uint64_t ThreadPool::nodes_searched() const { return accumulate(&Search::Worker::nodes); }
 uint64_t ThreadPool::tb_hits() const { return accumulate(&Search::Worker::tbHits); }
+uint64_t ThreadPool::TT_saves() const { return accumulate(&Search::Worker::TTsaves); }
 
 static size_t next_power_of_two(uint64_t count) { return count > 1 ? (2ULL << msb(count - 1)) : 1; }
 
 // Creates/destroys threads to match the requested number.
 // Created and launched threads will immediately go to sleep in idle_loop.
 // Upon resizing, threads are recreated to allow for binding if necessary.
-void ThreadPool::set(const NumaConfig&                           numaConfig,
+void ThreadPool::set(const NumaConfig&                     numaConfig,
-                     Search::SharedState                         sharedState,
+                     Search::SharedState                   sharedState,
-                     const Search::SearchManager::UpdateContext& updateContext) {
+                     Search::SearchManager::UpdateContext& updateContext) {
 
     if (threads.size() > 0)  // destroy any existing thread(s)
     {
@@ -332,6 +335,8 @@ void ThreadPool::start_thinking(const OptionsMap&  options,
     for (auto&& th : threads)
         th->wait_for_search_finished();
 
+    Distributed::signals_init();
+
     main_thread()->start_searching();
 }
 

src/thread.h

@@ -29,6 +29,7 @@
 #include <vector>
 
 #include "memory.h"
+#include "movepick.h"
 #include "numa.h"
 #include "position.h"
 #include "search.h"
@@ -137,14 +138,14 @@ class ThreadPool {
     void   wait_on_thread(size_t threadId);
     size_t num_threads() const;
     void   clear();
-    void   set(const NumaConfig& numaConfig,
+    void
-               Search::SharedState,
+    set(const NumaConfig& numaConfig, Search::SharedState, Search::SearchManager::UpdateContext&);
-               const Search::SearchManager::UpdateContext&);
 
     Search::SearchManager* main_manager();
     Thread*                main_thread() const { return threads.front().get(); }
     uint64_t               nodes_searched() const;
     uint64_t               tb_hits() const;
+    uint64_t               TT_saves() const;
     Thread*                get_best_thread() const;
     void                   start_searching();
     void                   wait_for_search_finished() const;

src/timeman.h

@@ -21,6 +21,7 @@
 
 #include <cstdint>
 
+#include "cluster.h"
 #include "misc.h"
 
 namespace Stockfish {

src/tt.h

@@ -28,6 +28,10 @@
 
 namespace Stockfish {
 
+namespace Distributed {
+void init();
+}
+
 class ThreadPool;
 struct TTEntry;
 struct Cluster;
@@ -52,7 +56,12 @@ struct TTData {
     Bound bound;
     bool  is_pv;
 
+#ifdef USE_MPI
+    // We need this for TTCache to be constructible.
+    TTData() = default;
+#else
     TTData() = delete;
+#endif
 
     // clang-format off
     TTData(Move m, Value v, Value ev, Depth d, Bound b, bool pv) :
@@ -73,6 +82,8 @@ struct TTWriter {
 
    private:
     friend class TranspositionTable;
+    friend void Distributed::init();
+
     TTEntry* entry;
     TTWriter(TTEntry* tte);
 };
@@ -80,6 +91,8 @@ struct TTWriter {
 
 class TranspositionTable {
 
+    friend void Distributed::init();
+
    public:
     ~TranspositionTable() { aligned_large_pages_free(table); }
 

src/uci.cpp

@@ -30,6 +30,7 @@
 #include <vector>
 
 #include "benchmark.h"
+#include "cluster.h"
 #include "engine.h"
 #include "memory.h"
 #include "movegen.h"
@@ -94,7 +95,8 @@ void UCIEngine::loop() {
     do
     {
         if (cli.argc == 1
-            && !getline(std::cin, cmd))  // Wait for an input or an end-of-file (EOF) indication
+            && !Distributed::getline(std::cin,
+                                     cmd))  // Wait for an input or an end-of-file (EOF) indication
             cmd = "quit";
 
         std::istringstream is(cmd);
@@ -112,7 +114,7 @@ void UCIEngine::loop() {
         else if (token == "ponderhit")
             engine.set_ponderhit(false);
 
-        else if (token == "uci")
+        else if (token == "uci" && Distributed::is_root())
         {
             sync_cout << "id name " << engine_info(true) << "\n"
                       << engine.get_options() << sync_endl;
@@ -133,7 +135,7 @@ void UCIEngine::loop() {
             position(is);
         else if (token == "ucinewgame")
             engine.search_clear();
-        else if (token == "isready")
+        else if (token == "isready" && Distributed::is_root())
             sync_cout << "readyok" << sync_endl;
 
         // Add custom non-UCI commands, mainly for debugging purposes.
@@ -144,13 +146,13 @@ void UCIEngine::loop() {
             bench(is);
         else if (token == BenchmarkCommand)
             benchmark(is);
-        else if (token == "d")
+        else if (token == "d" && Distributed::is_root())
             sync_cout << engine.visualize() << sync_endl;
-        else if (token == "eval")
+        else if (token == "eval" && Distributed::is_root())
             engine.trace_eval();
-        else if (token == "compiler")
+        else if (token == "compiler" && Distributed::is_root())
             sync_cout << compiler_info() << sync_endl;
-        else if (token == "export_net")
+        else if (token == "export_net" && Distributed::is_root())
         {
             std::pair<std::optional<std::string>, std::string> files[2];
 
@@ -162,7 +164,9 @@ void UCIEngine::loop() {
 
             engine.save_network(files);
         }
-        else if (token == "--help" || token == "help" || token == "--license" || token == "license")
+        else if ((token == "--help" || token == "help" || token == "--license"
+                  || token == "license")
+                 && Distributed::is_root())
             sync_cout
               << "\nStockfish is a powerful chess engine for playing and analyzing."
                  "\nIt is released as free software licensed under the GNU GPLv3 License."
@@ -171,7 +175,7 @@ void UCIEngine::loop() {
                  "\nFor any further information, visit https://github.com/official-stockfish/Stockfish#readme"
                  "\nor read the corresponding README.md and Copying.txt files distributed along with this program.\n"
               << sync_endl;
-        else if (!token.empty() && token[0] != '#')
+        else if (!token.empty() && token[0] != '#' && Distributed::is_root())
             sync_cout << "Unknown command: '" << cmd << "'. Type help for more information."
                       << sync_endl;
 
@@ -252,8 +256,9 @@ void UCIEngine::bench(std::istream& args) {
 
         if (token == "go" || token == "eval")
         {
-            std::cerr << "\nPosition: " << cnt++ << '/' << num << " (" << engine.fen() << ")"
+            if (Distributed::is_root())
-                      << std::endl;
+                std::cerr << "\nPosition: " << cnt++ << '/' << num << " (" << engine.fen() << ")"
+                          << std::endl;
             if (token == "go")
             {
                 Search::LimitsType limits = parse_limits(is);
@@ -269,7 +274,7 @@ void UCIEngine::bench(std::istream& args) {
                 nodes += nodesSearched;
                 nodesSearched = 0;
             }
-            else
+            else if (Distributed::is_root())
                 engine.trace_eval();
         }
         else if (token == "setoption")
@@ -287,10 +292,11 @@ void UCIEngine::bench(std::istream& args) {
 
     dbg_print();
 
-    std::cerr << "\n==========================="    //
+    if (Distributed::is_root())
-              << "\nTotal time (ms) : " << elapsed  //
+        std::cerr << "\n==========================="    //
-              << "\nNodes searched  : " << nodes    //
+                  << "\nTotal time (ms) : " << elapsed  //
-              << "\nNodes/second    : " << 1000 * nodes / elapsed << std::endl;
+                  << "\nNodes searched  : " << nodes    //
+                  << "\nNodes/second    : " << 1000 * nodes / elapsed << std::endl;
 
     // reset callback, to not capture a dangling reference to nodesSearched
     engine.set_on_update_full([&](const auto& i) { on_update_full(i, options["UCI_ShowWDL"]); });
@@ -461,7 +467,8 @@ void UCIEngine::setoption(std::istringstream& is) {
 
 std::uint64_t UCIEngine::perft(const Search::LimitsType& limits) {
     auto nodes = engine.perft(engine.fen(), limits.perft, engine.get_options()["UCI_Chess960"]);
-    sync_cout << "\nNodes searched: " << nodes << "\n" << sync_endl;
+    if (Distributed::is_root())
+        sync_cout << "\nNodes searched: " << nodes << "\n" << sync_endl;
     return nodes;
 }
 

src/ucioption.cpp

@@ -26,6 +26,7 @@
 #include <sstream>
 #include <utility>
 
+#include "cluster.h"
 #include "misc.h"
 
 namespace Stockfish {
@@ -54,7 +55,7 @@ void OptionsMap::setoption(std::istringstream& is) {
 
     if (options_map.count(name))
         options_map[name] = value;
-    else
+    else if (Distributed::is_root())
         sync_cout << "No such option: " << name << sync_endl;
 }