Merge commit '44cddbd962c738678f407a7414efa5b93f0710d9' into cluster

This is the last commit before there are more conflicts.

.github/workflows/matetrack.yml

@@ -0,0 +1,36 @@
+# This workflow will run matetrack on the PR
+
+name: Matetrack
+on:
+  workflow_call:
+jobs:
+  Matetrack:
+    name: Matetrack
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout SF repo 
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          path: Stockfish
+
+      - name: build SF
+        working-directory: Stockfish/src
+        run: make -j profile-build
+
+      - name: Checkout matetrack repo
+        uses: actions/checkout@v4
+        with:
+          repository: vondele/matetrack
+          path: matetrack
+          ref: 20287a1a145f30a166b7ef251eddb611e4e44fbf
+
+      - name: matetrack install deps
+        working-directory: matetrack
+        run: pip install -r requirements.txt
+
+      - name: Run matetrack
+        working-directory: matetrack
+        run: |
+          python matecheck.py --engine /home/runner/work/Stockfish/Stockfish/Stockfish/src/stockfish --epdFile mates2000.epd --nodes 100000 | tee matecheckout.out
+          ! grep "issues were detected" matecheckout.out > /dev/null

.github/workflows/stockfish.yml

@@ -90,6 +90,8 @@ jobs:
     uses: ./.github/workflows/sanitizers.yml
   Tests:
     uses: ./.github/workflows/tests.yml
+  Matetrack:
+    uses: ./.github/workflows/matetrack.yml
   Binaries:
     if: github.repository == 'official-stockfish/Stockfish'
     needs: [Matrix, Prerelease, Compilation]

src/engine.cpp

@@ -44,7 +44,8 @@ namespace Stockfish {
 
 namespace NN = Eval::NNUE;
 
-constexpr auto StartFEN = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1";
+constexpr auto StartFEN  = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1";
+constexpr int  MaxHashMB = Is64Bit ? 33554432 : 2048;
 
 Engine::Engine(std::string path) :
     binaryDirectory(CommandLine::get_binary_directory(path)),
@@ -58,6 +59,58 @@ Engine::Engine(std::string path) :
         NN::NetworkSmall({EvalFileDefaultNameSmall, "None", ""}, NN::EmbeddedNNUEType::SMALL))) {
     pos.set(StartFEN, false, &states->back());
     capSq = SQ_NONE;
+
+    options["Debug Log File"] << Option("", [](const Option& o) {
+        start_logger(o);
+        return std::nullopt;
+    });
+
+    options["NumaPolicy"] << Option("auto", [this](const Option& o) {
+        set_numa_config_from_option(o);
+        return numa_config_information_as_string() + "\n" + thread_binding_information_as_string();
+    });
+
+    options["Threads"] << Option(1, 1, 1024, [this](const Option&) {
+        resize_threads();
+        return thread_binding_information_as_string();
+    });
+
+    options["Hash"] << Option(16, 1, MaxHashMB, [this](const Option& o) {
+        set_tt_size(o);
+        return std::nullopt;
+    });
+
+    options["Clear Hash"] << Option([this](const Option&) {
+        search_clear();
+        return std::nullopt;
+    });
+    options["Ponder"] << Option(false);
+    options["MultiPV"] << Option(1, 1, MAX_MOVES);
+    options["Skill Level"] << Option(20, 0, 20);
+    options["Move Overhead"] << Option(10, 0, 5000);
+    options["nodestime"] << Option(0, 0, 10000);
+    options["UCI_Chess960"] << Option(false);
+    options["UCI_LimitStrength"] << Option(false);
+    options["UCI_Elo"] << Option(1320, 1320, 3190);
+    options["UCI_ShowWDL"] << Option(false);
+    options["SyzygyPath"] << Option("<empty>", [](const Option& o) {
+        Tablebases::init(o);
+        return std::nullopt;
+    });
+    options["SyzygyProbeDepth"] << Option(1, 1, 100);
+    options["Syzygy50MoveRule"] << Option(true);
+    options["SyzygyProbeLimit"] << Option(7, 0, 7);
+    options["EvalFile"] << Option(EvalFileDefaultNameBig, [this](const Option& o) {
+        load_big_network(o);
+        return std::nullopt;
+    });
+    options["EvalFileSmall"] << Option(EvalFileDefaultNameSmall, [this](const Option& o) {
+        load_small_network(o);
+        return std::nullopt;
+    });
+
+    load_networks();
+    resize_threads();
 }
 
 std::uint64_t Engine::perft(const std::string& fen, Depth depth, bool isChess960) {
@@ -215,7 +268,8 @@ void Engine::trace_eval() const {
     sync_cout << "\n" << Eval::trace(p, *networks) << sync_endl;
 }
 
-OptionsMap& Engine::get_options() { return options; }
+const OptionsMap& Engine::get_options() const { return options; }
+OptionsMap&       Engine::get_options() { return options; }
 
 std::string Engine::fen() const { return pos.fen(); }
 
@@ -244,4 +298,30 @@ std::string Engine::get_numa_config_as_string() const {
     return numaContext.get_numa_config().to_string();
 }
 
+std::string Engine::numa_config_information_as_string() const {
+    auto cfgStr = get_numa_config_as_string();
+    return "Available Processors: " + cfgStr;
+}
+
+std::string Engine::thread_binding_information_as_string() const {
+    auto boundThreadsByNode = get_bound_thread_count_by_numa_node();
+    if (boundThreadsByNode.empty())
+        return "";
+
+    std::stringstream ss;
+    ss << "NUMA Node Thread Binding: ";
+
+    bool isFirst = true;
+
+    for (auto&& [current, total] : boundThreadsByNode)
+    {
+        if (!isFirst)
+            ss << ":";
+        ss << current << "/" << total;
+        isFirst = false;
+    }
+
+    return ss.str();
+}
+
 }

src/engine.h

@@ -29,13 +29,13 @@
 #include <vector>
 
 #include "nnue/network.h"
+#include "numa.h"
 #include "position.h"
 #include "search.h"
 #include "syzygy/tbprobe.h"  // for Stockfish::Depth
 #include "thread.h"
 #include "tt.h"
 #include "ucioption.h"
-#include "numa.h"
 
 namespace Stockfish {
 
@@ -92,13 +92,18 @@ class Engine {
 
     // utility functions
 
-    void                                   trace_eval() const;
-    OptionsMap&                            get_options();
+    void trace_eval() const;
+
+    const OptionsMap& get_options() const;
+    OptionsMap&       get_options();
+
     std::string                            fen() const;
     void                                   flip();
     std::string                            visualize() const;
     std::vector<std::pair<size_t, size_t>> get_bound_thread_count_by_numa_node() const;
     std::string                            get_numa_config_as_string() const;
+    std::string                            numa_config_information_as_string() const;
+    std::string                            thread_binding_information_as_string() const;
 
    private:
     const std::string binaryDirectory;

src/evaluate.cpp

@@ -66,14 +66,14 @@ Value Eval::evaluate(const Eval::NNUE::Networks&    networks,
     auto [psqt, positional] = smallNet ? networks.small.evaluate(pos, &caches.small)
                                        : networks.big.evaluate(pos, &caches.big);
 
-    Value nnue           = psqt + positional;
+    Value nnue           = (125 * psqt + 131 * positional) / 128;
     int   nnueComplexity = std::abs(psqt - positional);
 
     // Re-evaluate the position when higher eval accuracy is worth the time spent
     if (smallNet && (nnue * simpleEval < 0 || std::abs(nnue) < 227))
     {
         std::tie(psqt, positional) = networks.big.evaluate(pos, &caches.big);
-        nnue                       = psqt + positional;
+        nnue                       = (125 * psqt + 131 * positional) / 128;
         nnueComplexity             = std::abs(psqt - positional);
         smallNet                   = false;
     }

src/misc.cpp

@@ -371,6 +371,8 @@ std::ostream& operator<<(std::ostream& os, SyncCout sc) {
     return os;
 }
 
+void sync_cout_start() { std::cout << IO_LOCK; }
+void sync_cout_end() { std::cout << IO_UNLOCK; }
 
 // Trampoline helper to avoid moving Logger to misc.h
 void start_logger(const std::string& fname) { Logger::start(fname); }
@@ -419,6 +421,10 @@ void remove_whitespace(std::string& s) {
     s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return std::isspace(c); }), s.end());
 }
 
+bool is_whitespace(const std::string& s) {
+    return std::all_of(s.begin(), s.end(), [](char c) { return std::isspace(c); });
+}
+
 std::string CommandLine::get_binary_directory(std::string argv0) {
     std::string pathSeparator;
 

src/misc.h

@@ -101,6 +101,7 @@ inline std::vector<std::string> split(const std::string& s, const std::string& d
 }
 
 void remove_whitespace(std::string& s);
+bool is_whitespace(const std::string& s);
 
 enum SyncCout {
     IO_LOCK,
@@ -111,6 +112,9 @@ std::ostream& operator<<(std::ostream&, SyncCout);
 #define sync_cout std::cout << IO_LOCK
 #define sync_endl std::endl << IO_UNLOCK
 
+void sync_cout_start();
+void sync_cout_end();
+
 // True if and only if the binary is compiled on a little-endian machine
 static inline const union {
     uint32_t i;

src/search.cpp

@@ -576,6 +576,9 @@ Value Search::Worker::search(
     if (depth <= 0)
         return qsearch<PvNode ? PV : NonPV>(pos, ss, alpha, beta);
 
+    // Limit the depth if extensions made it too large
+    depth = std::min(depth, MAX_PLY - 1);
+
     // Check if we have an upcoming move that draws by repetition, or
     // if the opponent had an alternative move earlier to this position.
     if (!rootNode && alpha < VALUE_DRAW && pos.has_game_cycle(ss->ply))

src/tune.cpp

@@ -21,6 +21,7 @@
 #include <algorithm>
 #include <iostream>
 #include <map>
+#include <optional>
 #include <sstream>
 #include <string>
 
@@ -33,19 +34,19 @@ namespace Stockfish {
 bool          Tune::update_on_last;
 const Option* LastOption = nullptr;
 OptionsMap*   Tune::options;
-
-
 namespace {
 std::map<std::string, int> TuneResults;
 
-void on_tune(const Option& o) {
+std::optional<std::string> on_tune(const Option& o) {
 
     if (!Tune::update_on_last || LastOption == &o)
         Tune::read_options();
+
+    return std::nullopt;
+}
 }
 
-
-void make_option(OptionsMap* options, const string& n, int v, const SetRange& r) {
+void Tune::make_option(OptionsMap* opts, const string& n, int v, const SetRange& r) {
 
     // Do not generate option when there is nothing to tune (ie. min = max)
     if (r(v).first == r(v).second)
@@ -54,8 +55,8 @@ void make_option(OptionsMap* options, const string& n, int v, const SetRange& r)
     if (TuneResults.count(n))
         v = TuneResults[n];
 
-    (*options)[n] << Option(v, r(v).first, r(v).second, on_tune);
-    LastOption = &((*options)[n]);
+    (*opts)[n] << Option(v, r(v).first, r(v).second, on_tune);
+    LastOption = &((*opts)[n]);
 
     // Print formatted parameters, ready to be copy-pasted in Fishtest
     std::cout << n << ","                                  //
@@ -65,7 +66,6 @@ void make_option(OptionsMap* options, const string& n, int v, const SetRange& r)
               << (r(v).second - r(v).first) / 20.0 << ","  //
               << "0.0020" << std::endl;
 }
-}
 
 string Tune::next(string& names, bool pop) {
 

src/tune.h

@@ -145,6 +145,8 @@ class Tune {
         return add(value, (next(names), std::move(names)), args...);
     }
 
+    static void make_option(OptionsMap* options, const std::string& n, int v, const SetRange& r);
+
     std::vector<std::unique_ptr<EntryBase>> list;
 
    public:

src/uci.cpp

@@ -31,20 +31,16 @@
 #include "benchmark.h"
 #include "cluster.h"
 #include "engine.h"
-#include "evaluate.h"
 #include "movegen.h"
 #include "position.h"
 #include "score.h"
 #include "search.h"
-#include "syzygy/tbprobe.h"
 #include "types.h"
 #include "ucioption.h"
 
 namespace Stockfish {
 
-constexpr auto StartFEN  = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1";
-constexpr int  MaxHashMB = Is64Bit ? 33554432 : 2048;
-
+constexpr auto StartFEN = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1";
 template<typename... Ts>
 struct overload: Ts... {
     using Ts::operator()...;
@@ -53,59 +49,35 @@ struct overload: Ts... {
 template<typename... Ts>
 overload(Ts...) -> overload<Ts...>;
 
+void UCIEngine::print_info_string(const std::string& str) {
+    sync_cout_start();
+    for (auto& line : split(str, "\n"))
+    {
+        if (!is_whitespace(line))
+        {
+            std::cout << "info string " << line << '\n';
+        }
+    }
+    sync_cout_end();
+}
+
 UCIEngine::UCIEngine(int argc, char** argv) :
     engine(argv[0]),
     cli(argc, argv) {
 
-    auto& options = engine.get_options();
-
-    options["Debug Log File"] << Option("", [](const Option& o) { start_logger(o); });
-
-    options["NumaPolicy"] << Option("auto", [this](const Option& o) {
-        engine.set_numa_config_from_option(o);
-        print_numa_config_information();
-        print_thread_binding_information();
+    engine.get_options().add_info_listener([](const std::optional<std::string>& str) {
+        if (str.has_value())
+            print_info_string(*str);
     });
 
-    options["Threads"] << Option(1, 1, 1024, [this](const Option&) {
-        engine.resize_threads();
-        print_thread_binding_information();
-    });
-
-    options["Hash"] << Option(16, 1, MaxHashMB, [this](const Option& o) { engine.set_tt_size(o); });
-
-    options["Clear Hash"] << Option([this](const Option&) { engine.search_clear(); });
-    options["Ponder"] << Option(false);
-    options["MultiPV"] << Option(1, 1, MAX_MOVES);
-    options["Skill Level"] << Option(20, 0, 20);
-    options["Move Overhead"] << Option(10, 0, 5000);
-    options["nodestime"] << Option(0, 0, 10000);
-    options["UCI_Chess960"] << Option(false);
-    options["UCI_LimitStrength"] << Option(false);
-    options["UCI_Elo"] << Option(1320, 1320, 3190);
-    options["UCI_ShowWDL"] << Option(false);
-    options["SyzygyPath"] << Option("<empty>", [](const Option& o) { Tablebases::init(o); });
-    options["SyzygyProbeDepth"] << Option(1, 1, 100);
-    options["Syzygy50MoveRule"] << Option(true);
-    options["SyzygyProbeLimit"] << Option(7, 0, 7);
-    options["EvalFile"] << Option(EvalFileDefaultNameBig,
-                                  [this](const Option& o) { engine.load_big_network(o); });
-    options["EvalFileSmall"] << Option(EvalFileDefaultNameSmall,
-                                       [this](const Option& o) { engine.load_small_network(o); });
-
-
     engine.set_on_iter([](const auto& i) { on_iter(i); });
     engine.set_on_update_no_moves([](const auto& i) { on_update_no_moves(i); });
-    engine.set_on_update_full([&](const auto& i) { on_update_full(i, options["UCI_ShowWDL"]); });
+    engine.set_on_update_full(
+      [this](const auto& i) { on_update_full(i, engine.get_options()["UCI_ShowWDL"]); });
     engine.set_on_bestmove([](const auto& bm, const auto& p) { on_bestmove(bm, p); });
-
-    engine.load_networks();
-    engine.resize_threads();
-    engine.search_clear();  // After threads are up
 }
 
 void UCIEngine::loop() {
-
     std::string token, cmd;
 
     for (int i = 1; i < cli.argc; ++i)
@@ -138,8 +110,8 @@ void UCIEngine::loop() {
             sync_cout << "id name " << engine_info(true) << "\n"
                       << engine.get_options() << sync_endl;
 
-            print_numa_config_information();
-            print_thread_binding_information();
+            print_info_string(engine.numa_config_information_as_string());
+            print_info_string(engine.thread_binding_information_as_string());
 
             sync_cout << "uciok" << sync_endl;
         }
@@ -197,28 +169,6 @@ void UCIEngine::loop() {
     } while (token != "quit" && cli.argc == 1);  // The command-line arguments are one-shot
 }
 
-void UCIEngine::print_numa_config_information() const {
-    auto cfgStr = engine.get_numa_config_as_string();
-    sync_cout << "info string Available Processors: " << cfgStr << sync_endl;
-}
-
-void UCIEngine::print_thread_binding_information() const {
-    auto boundThreadsByNode = engine.get_bound_thread_count_by_numa_node();
-    if (!boundThreadsByNode.empty())
-    {
-        sync_cout << "info string NUMA Node Thread Binding: ";
-        bool isFirst = true;
-        for (auto&& [current, total] : boundThreadsByNode)
-        {
-            if (!isFirst)
-                std::cout << ":";
-            std::cout << current << "/" << total;
-            isFirst = false;
-        }
-        std::cout << sync_endl;
-    }
-}
-
 Search::LimitsType UCIEngine::parse_limits(std::istream& is) {
     Search::LimitsType limits;
     std::string        token;

src/uci.h

@@ -19,10 +19,10 @@
 #ifndef UCI_H_INCLUDED
 #define UCI_H_INCLUDED
 
+#include <cstdint>
 #include <iostream>
 #include <string>
 #include <string_view>
-#include <cstdint>
 
 #include "engine.h"
 #include "misc.h"
@@ -42,9 +42,6 @@ class UCIEngine {
 
     void loop();
 
-    void print_numa_config_information() const;
-    void print_thread_binding_information() const;
-
     static int         to_cp(Value v, const Position& pos);
     static std::string format_score(const Score& s);
     static std::string square(Square s);
@@ -61,6 +58,8 @@ class UCIEngine {
     Engine      engine;
     CommandLine cli;
 
+    static void print_info_string(const std::string& str);
+
     void          go(std::istringstream& is);
     void          bench(std::istream& args);
     void          position(std::istringstream& is);

src/ucioption.cpp

@@ -37,6 +37,8 @@ bool CaseInsensitiveLess::operator()(const std::string& s1, const std::string& s
       [](char c1, char c2) { return std::tolower(c1) < std::tolower(c2); });
 }
 
+void OptionsMap::add_info_listener(InfoListener&& message_func) { info = std::move(message_func); }
+
 void OptionsMap::setoption(std::istringstream& is) {
     std::string token, name, value;
 
@@ -58,13 +60,20 @@ void OptionsMap::setoption(std::istringstream& is) {
 
 Option OptionsMap::operator[](const std::string& name) const {
     auto it = options_map.find(name);
-    return it != options_map.end() ? it->second : Option();
+    return it != options_map.end() ? it->second : Option(this);
 }
 
-Option& OptionsMap::operator[](const std::string& name) { return options_map[name]; }
+Option& OptionsMap::operator[](const std::string& name) {
+    if (!options_map.count(name))
+        options_map[name] = Option(this);
+    return options_map[name];
+}
 
 std::size_t OptionsMap::count(const std::string& name) const { return options_map.count(name); }
 
+Option::Option(const OptionsMap* map) :
+    parent(map) {}
+
 Option::Option(const char* v, OnChange f) :
     type("string"),
     min(0),
@@ -128,10 +137,12 @@ void Option::operator<<(const Option& o) {
 
     static size_t insert_order = 0;
 
-    *this = o;
-    idx   = insert_order++;
-}
+    auto p = this->parent;
+    *this  = o;
 
+    this->parent = p;
+    idx          = insert_order++;
+}
 
 // Updates currentValue and triggers on_change() action. It's up to
 // the GUI to check for option's limits, but we could receive the new value
@@ -160,7 +171,12 @@ Option& Option::operator=(const std::string& v) {
         currentValue = v;
 
     if (on_change)
-        on_change(*this);
+    {
+        const auto ret = on_change(*this);
+
+        if (ret && parent != nullptr && parent->info != nullptr)
+            parent->info(ret);
+    }
 
     return *this;
 }

src/ucioption.h

@@ -23,6 +23,7 @@
 #include <functional>
 #include <iosfwd>
 #include <map>
+#include <optional>
 #include <string>
 
 namespace Stockfish {
@@ -31,31 +32,14 @@ struct CaseInsensitiveLess {
     bool operator()(const std::string&, const std::string&) const;
 };
 
-class Option;
-
-class OptionsMap {
-   public:
-    void setoption(std::istringstream&);
-
-    friend std::ostream& operator<<(std::ostream&, const OptionsMap&);
-
-    Option  operator[](const std::string&) const;
-    Option& operator[](const std::string&);
-
-    std::size_t count(const std::string&) const;
-
-   private:
-    // The options container is defined as a std::map
-    using OptionsStore = std::map<std::string, Option, CaseInsensitiveLess>;
-
-    OptionsStore options_map;
-};
+class OptionsMap;
 
 // The Option class implements each option as specified by the UCI protocol
 class Option {
    public:
-    using OnChange = std::function<void(const Option&)>;
+    using OnChange = std::function<std::optional<std::string>(const Option&)>;
 
+    Option(const OptionsMap*);
     Option(OnChange = nullptr);
     Option(bool v, OnChange = nullptr);
     Option(const char* v, OnChange = nullptr);
@@ -63,7 +47,6 @@ class Option {
     Option(const char* v, const char* cur, OnChange = nullptr);
 
     Option& operator=(const std::string&);
-    void    operator<<(const Option&);
     operator int() const;
     operator std::string() const;
     bool operator==(const char*) const;
@@ -72,10 +55,49 @@ class Option {
     friend std::ostream& operator<<(std::ostream&, const OptionsMap&);
 
    private:
-    std::string defaultValue, currentValue, type;
-    int         min, max;
-    size_t      idx;
-    OnChange    on_change;
+    friend class OptionsMap;
+    friend class Engine;
+    friend class Tune;
+
+    void operator<<(const Option&);
+
+    std::string       defaultValue, currentValue, type;
+    int               min, max;
+    size_t            idx;
+    OnChange          on_change;
+    const OptionsMap* parent = nullptr;
+};
+
+class OptionsMap {
+   public:
+    using InfoListener = std::function<void(std::optional<std::string>)>;
+
+    OptionsMap()                             = default;
+    OptionsMap(const OptionsMap&)            = delete;
+    OptionsMap(OptionsMap&&)                 = delete;
+    OptionsMap& operator=(const OptionsMap&) = delete;
+    OptionsMap& operator=(OptionsMap&&)      = delete;
+
+    void add_info_listener(InfoListener&&);
+
+    void setoption(std::istringstream&);
+
+    Option  operator[](const std::string&) const;
+    Option& operator[](const std::string&);
+
+    std::size_t count(const std::string&) const;
+
+   private:
+    friend class Engine;
+    friend class Option;
+
+    friend std::ostream& operator<<(std::ostream&, const OptionsMap&);
+
+    // The options container is defined as a std::map
+    using OptionsStore = std::map<std::string, Option, CaseInsensitiveLess>;
+
+    OptionsStore options_map;
+    InfoListener info;
 };
 
 }

tests/instrumented.sh

@@ -170,6 +170,11 @@ cat << EOF > game.exp
  expect "score mate -1"
  expect "bestmove"
 
+ send "ucinewgame\n"
+ send "position fen 7K/P1p1p1p1/2P1P1Pk/6pP/3p2P1/1P6/3P4/8 w - - 0 1\n"
+ send "go nodes 500000\n"
+ expect "bestmove"
+
  send "ucinewgame\n"
  send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
  send "go depth 18\n"