diff --git a/AUTHORS b/AUTHORS index 31ff51def..db552a3a5 100644 --- a/AUTHORS +++ b/AUTHORS @@ -200,6 +200,7 @@ Panthee Pascal Romaret Pasquale Pigazzini (ppigazzini) Patrick Jansen (mibere) +Patrick Leonhardt (Yoshie2000) Peter Schneider (pschneider1968) Peter Zsifkovits (CoffeeOne) PikaCat diff --git a/src/Makefile b/src/Makefile index b30e16c07..cc85ac78e 100644 --- a/src/Makefile +++ b/src/Makefile @@ -55,15 +55,16 @@ PGOBENCH = $(WINE_PATH) ./$(EXE) bench SRCS = benchmark.cpp bitboard.cpp evaluate.cpp main.cpp \ misc.cpp movegen.cpp movepick.cpp position.cpp \ search.cpp thread.cpp timeman.cpp tt.cpp uci.cpp ucioption.cpp tune.cpp syzygy/tbprobe.cpp \ - nnue/nnue_accumulator.cpp nnue/nnue_misc.cpp nnue/features/half_ka_v2_hm.cpp nnue/network.cpp \ + nnue/nnue_accumulator.cpp nnue/nnue_misc.cpp nnue/network.cpp \ + nnue/features/half_ka_v2_hm.cpp nnue/features/full_threats.cpp \ engine.cpp score.cpp memory.cpp HEADERS = benchmark.h bitboard.h evaluate.h misc.h movegen.h movepick.h history.h \ - nnue/nnue_misc.h nnue/features/half_ka_v2_hm.h nnue/layers/affine_transform.h \ - nnue/layers/affine_transform_sparse_input.h nnue/layers/clipped_relu.h \ - nnue/layers/sqr_clipped_relu.h nnue/nnue_accumulator.h nnue/nnue_architecture.h \ - nnue/nnue_common.h nnue/nnue_feature_transformer.h nnue/simd.h position.h \ - search.h syzygy/tbprobe.h thread.h thread_win32_osx.h timeman.h \ + nnue/nnue_misc.h nnue/features/half_ka_v2_hm.h nnue/features/full_threats.h \ + nnue/layers/affine_transform.h nnue/layers/affine_transform_sparse_input.h \ + nnue/layers/clipped_relu.h nnue/layers/sqr_clipped_relu.h nnue/nnue_accumulator.h \ + nnue/nnue_architecture.h nnue/nnue_common.h nnue/nnue_feature_transformer.h nnue/simd.h \ + position.h search.h syzygy/tbprobe.h thread.h thread_win32_osx.h timeman.h \ tt.h tune.h types.h uci.h ucioption.h perft.h nnue/network.h engine.h score.h numa.h memory.h OBJS = $(notdir $(SRCS:.cpp=.o)) diff --git a/src/benchmark.cpp b/src/benchmark.cpp index a9f70f10d..38cafaec7 100644 --- a/src/benchmark.cpp +++ b/src/benchmark.cpp @@ -65,6 +65,10 @@ const std::vector Defaults = { "3Qb1k1/1r2ppb1/pN1n2q1/Pp1Pp1Pr/4P2p/4BP2/4B1R1/1R5K b - - 11 40", "4k3/3q1r2/1N2r1b1/3ppN2/2nPP3/1B1R2n1/2R1Q3/3K4 w - - 5 1", + // Positions with high numbers of changed threats + "k7/2n1n3/1nbNbn2/2NbRBn1/1nbRQR2/2NBRBN1/3N1N2/7K w - - 0 1", + "K7/8/8/BNQNQNB1/N5N1/R1Q1q2r/n5n1/bnqnqnbk w - - 0 1", + // 5-man positions "8/8/8/8/5kp1/P7/8/1K1N4 w - - 0 1", // Kc2 - mate "8/8/8/5N2/8/p7/8/2NK3k w - - 0 1", // Na2 - mate @@ -509,4 +513,4 @@ BenchmarkSetup setup_benchmark(std::istream& is) { return setup; } -} // namespace Stockfish \ No newline at end of file +} // namespace Stockfish diff --git a/src/bitboard.cpp b/src/bitboard.cpp index 9b1d674c0..6c97d7863 100644 --- a/src/bitboard.cpp +++ b/src/bitboard.cpp @@ -31,6 +31,7 @@ uint8_t SquareDistance[SQUARE_NB][SQUARE_NB]; Bitboard LineBB[SQUARE_NB][SQUARE_NB]; Bitboard BetweenBB[SQUARE_NB][SQUARE_NB]; +Bitboard RayPassBB[SQUARE_NB][SQUARE_NB]; Bitboard PseudoAttacks[PIECE_TYPE_NB][SQUARE_NB]; alignas(64) Magic Magics[SQUARE_NB][2]; @@ -105,6 +106,8 @@ void Bitboards::init() { LineBB[s1][s2] = (attacks_bb(pt, s1, 0) & attacks_bb(pt, s2, 0)) | s1 | s2; BetweenBB[s1][s2] = (attacks_bb(pt, s1, square_bb(s2)) & attacks_bb(pt, s2, square_bb(s1))); + RayPassBB[s1][s2] = + attacks_bb(pt, s1, 0) & (attacks_bb(pt, s2, square_bb(s1)) | s2); } BetweenBB[s1][s2] |= s2; } diff --git a/src/bitboard.h b/src/bitboard.h index 27ef89c0e..1cd717b8f 100644 --- a/src/bitboard.h +++ b/src/bitboard.h @@ -61,6 +61,7 @@ extern uint8_t SquareDistance[SQUARE_NB][SQUARE_NB]; extern Bitboard BetweenBB[SQUARE_NB][SQUARE_NB]; extern Bitboard LineBB[SQUARE_NB][SQUARE_NB]; +extern Bitboard RayPassBB[SQUARE_NB][SQUARE_NB]; extern Bitboard PseudoAttacks[PIECE_TYPE_NB][SQUARE_NB]; @@ -252,6 +253,20 @@ inline Bitboard attacks_bb(PieceType pt, Square s, Bitboard occupied) { } } +inline Bitboard attacks_bb(Piece pc, Square s) { + if (type_of(pc) == PAWN) + return PseudoAttacks[color_of(pc)][s]; + + return PseudoAttacks[type_of(pc)][s]; +} + + +inline Bitboard attacks_bb(Piece pc, Square s, Bitboard occupied) { + if (type_of(pc) == PAWN) + return PseudoAttacks[color_of(pc)][s]; + + return attacks_bb(type_of(pc), s, occupied); +} // Counts the number of non-zero bits in a bitboard. inline int popcount(Bitboard b) { diff --git a/src/evaluate.h b/src/evaluate.h index 2a6c9afa9..c8dc64ace 100644 --- a/src/evaluate.h +++ b/src/evaluate.h @@ -33,7 +33,7 @@ namespace Eval { // for the build process (profile-build and fishtest) to work. Do not change the // name of the macro or the location where this macro is defined, as it is used // in the Makefile/Fishtest. -#define EvalFileDefaultNameBig "nn-1c0000000000.nnue" +#define EvalFileDefaultNameBig "nn-49c1193b131c.nnue" #define EvalFileDefaultNameSmall "nn-37f18f62d772.nnue" namespace NNUE { diff --git a/src/main.cpp b/src/main.cpp index 9988680aa..6ab3507f3 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -21,9 +21,9 @@ #include "bitboard.h" #include "misc.h" +#include "nnue/features/full_threats.h" #include "position.h" #include "tune.h" -#include "types.h" #include "uci.h" using namespace Stockfish; @@ -33,6 +33,7 @@ int main(int argc, char* argv[]) { Bitboards::init(); Position::init(); + Eval::NNUE::Features::init_threat_offsets(); auto uci = std::make_unique(argc, argv); diff --git a/src/misc.h b/src/misc.h index b1707e742..fce6f17df 100644 --- a/src/misc.h +++ b/src/misc.h @@ -134,10 +134,13 @@ class ValueList { public: std::size_t size() const { return size_; } - void push_back(const T& value) { values_[size_++] = value; } - const T* begin() const { return values_; } - const T* end() const { return values_ + size_; } - const T& operator[](int index) const { return values_[index]; } + void push_back(const T& value) { + assert(size_ < MaxSize); + values_[size_++] = value; + } + const T* begin() const { return values_; } + const T* end() const { return values_ + size_; } + const T& operator[](int index) const { return values_[index]; } private: T values_[MaxSize]; diff --git a/src/nnue/features/full_threats.cpp b/src/nnue/features/full_threats.cpp new file mode 100644 index 000000000..6fc6b00ec --- /dev/null +++ b/src/nnue/features/full_threats.cpp @@ -0,0 +1,311 @@ +/* + Stockfish, a UCI chess playing engine derived from Glaurung 2.1 + Copyright (C) 2004-2025 The Stockfish developers (see AUTHORS file) + + Stockfish is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Stockfish is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +//Definition of input features FullThreats of NNUE evaluation function + +#include "full_threats.h" + +#include +#include + +#include "../../bitboard.h" +#include "../../misc.h" +#include "../../position.h" +#include "../../types.h" +#include "../nnue_common.h" + +namespace Stockfish::Eval::NNUE::Features { + +// Lookup array for indexing threats +IndexType offsets[PIECE_NB][SQUARE_NB + 2]; + +// Information on a particular pair of pieces and whether they should be excluded +struct PiecePairData { + // Layout: bits 8..31 are the index contribution of this piece pair, bits 0 and 1 are exclusion info + uint32_t data; + PiecePairData() {} + PiecePairData(bool excluded_pair, bool semi_excluded_pair, IndexType feature_index_base) { + data = + excluded_pair << 1 | (semi_excluded_pair && !excluded_pair) | feature_index_base << 8; + } + // lsb: excluded if from < to; 2nd lsb: always excluded + uint8_t excluded_pair_info() const { return (uint8_t) data; } + IndexType feature_index_base() const { return data >> 8; } +}; + +constexpr std::array AllPieces = { + W_PAWN, W_KNIGHT, W_BISHOP, W_ROOK, W_QUEEN, W_KING, + B_PAWN, B_KNIGHT, B_BISHOP, B_ROOK, B_QUEEN, B_KING, +}; + +// The final index is calculated from summing data found in these two LUTs, as well +// as offsets[attacker][from] +PiecePairData index_lut1[PIECE_NB][PIECE_NB]; // [attacker][attacked] +uint8_t index_lut2[PIECE_NB][SQUARE_NB][SQUARE_NB]; // [attacker][from][to] + +static void init_index_luts() { + for (Piece attacker : AllPieces) + { + for (Piece attacked : AllPieces) + { + bool enemy = (attacker ^ attacked) == 8; + PieceType attackerType = type_of(attacker); + PieceType attackedType = type_of(attacked); + + int map = FullThreats::map[attackerType - 1][attackedType - 1]; + bool semi_excluded = attackerType == attackedType && (enemy || attackerType != PAWN); + IndexType feature = offsets[attacker][65] + + (color_of(attacked) * (numValidTargets[attacker] / 2) + map) + * offsets[attacker][64]; + + bool excluded = map < 0; + index_lut1[attacker][attacked] = PiecePairData(excluded, semi_excluded, feature); + } + } + + for (Piece attacker : AllPieces) + { + for (int from = 0; from < SQUARE_NB; ++from) + { + for (int to = 0; to < SQUARE_NB; ++to) + { + Bitboard attacks = attacks_bb(attacker, Square(from)); + index_lut2[attacker][from][to] = popcount((square_bb(Square(to)) - 1) & attacks); + } + } + } +} + +void init_threat_offsets() { + int cumulativeOffset = 0; + for (Piece piece : AllPieces) + { + int pieceIdx = piece; + int cumulativePieceOffset = 0; + + for (Square from = SQ_A1; from <= SQ_H8; ++from) + { + offsets[pieceIdx][from] = cumulativePieceOffset; + + if (type_of(piece) != PAWN) + { + Bitboard attacks = attacks_bb(piece, from, 0ULL); + cumulativePieceOffset += popcount(attacks); + } + + else if (from >= SQ_A2 && from <= SQ_H7) + { + Bitboard attacks = (pieceIdx < 8) ? pawn_attacks_bb(square_bb(from)) + : pawn_attacks_bb(square_bb(from)); + cumulativePieceOffset += popcount(attacks); + } + } + + offsets[pieceIdx][64] = cumulativePieceOffset; + offsets[pieceIdx][65] = cumulativeOffset; + + cumulativeOffset += numValidTargets[pieceIdx] * cumulativePieceOffset; + } + + init_index_luts(); +} + +// Index of a feature for a given king position and another piece on some square +template +IndexType +FullThreats::make_index(Piece attacker, Square from, Square to, Piece attacked, Square ksq) { + from = (Square) (int(from) ^ OrientTBL[Perspective][ksq]); + to = (Square) (int(to) ^ OrientTBL[Perspective][ksq]); + + if (Perspective == BLACK) + { + attacker = ~attacker; + attacked = ~attacked; + } + + auto piecePairData = index_lut1[attacker][attacked]; + + // Some threats imply the existence of the corresponding ones in the opposite + // direction. We filter them here to ensure only one such threat is active. + + // In the below addition, the 2nd lsb gets set iff either the pair is always excluded, + // or the pair is semi-excluded and from < to. By using an unsigned compare, the following + // sequence can use an add-with-carry instruction. + bool less_than = static_cast(from) < static_cast(to); + if ((piecePairData.excluded_pair_info() + less_than) & 2) + return Dimensions; + + IndexType index = + piecePairData.feature_index_base() + offsets[attacker][from] + index_lut2[attacker][from][to]; + + sf_assume(index != Dimensions); + return index; +} + +// Get a list of indices for active features in ascending order +template +void FullThreats::append_active_indices(const Position& pos, IndexList& active) { + static constexpr Color order[2][2] = {{WHITE, BLACK}, {BLACK, WHITE}}; + + Square ksq = pos.square(Perspective); + Bitboard occupied = pos.pieces(); + + for (Color color : {WHITE, BLACK}) + { + for (PieceType pt = PAWN; pt <= KING; ++pt) + { + Color c = order[Perspective][color]; + Piece attacker = make_piece(c, pt); + Bitboard bb = pos.pieces(c, pt); + + if (pt == PAWN) + { + auto right = (c == WHITE) ? NORTH_EAST : SOUTH_WEST; + auto left = (c == WHITE) ? NORTH_WEST : SOUTH_EAST; + auto attacks_left = + ((c == WHITE) ? shift(bb) : shift(bb)) & occupied; + auto attacks_right = + ((c == WHITE) ? shift(bb) : shift(bb)) & occupied; + + while (attacks_left) + { + Square to = pop_lsb(attacks_left); + Square from = to - right; + Piece attacked = pos.piece_on(to); + IndexType index = make_index(attacker, from, to, attacked, ksq); + + if (index < Dimensions) + active.push_back(index); + } + + while (attacks_right) + { + Square to = pop_lsb(attacks_right); + Square from = to - left; + Piece attacked = pos.piece_on(to); + IndexType index = make_index(attacker, from, to, attacked, ksq); + + if (index < Dimensions) + active.push_back(index); + } + } + else + { + while (bb) + { + Square from = pop_lsb(bb); + Bitboard attacks = (attacks_bb(pt, from, occupied)) & occupied; + + while (attacks) + { + Square to = pop_lsb(attacks); + Piece attacked = pos.piece_on(to); + IndexType index = + make_index(attacker, from, to, attacked, ksq); + + if (index < Dimensions) + active.push_back(index); + } + } + } + } + } +} + +// Explicit template instantiations +template void FullThreats::append_active_indices(const Position& pos, IndexList& active); +template void FullThreats::append_active_indices(const Position& pos, IndexList& active); +template IndexType +FullThreats::make_index(Piece attkr, Square from, Square to, Piece attkd, Square ksq); +template IndexType +FullThreats::make_index(Piece attkr, Square from, Square to, Piece attkd, Square ksq); + +// Get a list of indices for recently changed features +template +void FullThreats::append_changed_indices(Square ksq, + const DiffType& diff, + IndexList& removed, + IndexList& added, + FusedUpdateData* fusedData, + bool first) { + for (const auto dirty : diff.list) + { + auto attacker = dirty.pc(); + auto attacked = dirty.threatened_pc(); + auto from = dirty.pc_sq(); + auto to = dirty.threatened_sq(); + auto add = dirty.add(); + + if (fusedData) + { + if (from == fusedData->dp2removed) + { + if (add) + { + if (first) + { + fusedData->dp2removedOriginBoard |= square_bb(to); + continue; + } + } + else if (fusedData->dp2removedOriginBoard & square_bb(to)) + continue; + } + + if (to != SQ_NONE && to == fusedData->dp2removed) + { + if (add) + { + if (first) + { + fusedData->dp2removedTargetBoard |= square_bb(from); + continue; + } + } + else if (fusedData->dp2removedTargetBoard & square_bb(from)) + continue; + } + } + + IndexType index = make_index(attacker, from, to, attacked, ksq); + + if (index != Dimensions) + (add ? added : removed).push_back(index); + } +} + +// Explicit template instantiations +template void FullThreats::append_changed_indices(Square ksq, + const DiffType& diff, + IndexList& removed, + IndexList& added, + FusedUpdateData* fd, + bool first); +template void FullThreats::append_changed_indices(Square ksq, + const DiffType& diff, + IndexList& removed, + IndexList& added, + FusedUpdateData* fd, + bool first); + +bool FullThreats::requires_refresh(const DiffType& diff, Color perspective) { + return perspective == diff.us + && OrientTBL[diff.us][diff.ksq] != OrientTBL[diff.us][diff.prevKsq]; +} + +} // namespace Stockfish::Eval::NNUE::Features diff --git a/src/nnue/features/full_threats.h b/src/nnue/features/full_threats.h new file mode 100644 index 000000000..458b04dd1 --- /dev/null +++ b/src/nnue/features/full_threats.h @@ -0,0 +1,115 @@ +/* + Stockfish, a UCI chess playing engine derived from Glaurung 2.1 + Copyright (C) 2004-2025 The Stockfish developers (see AUTHORS file) + Stockfish is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + Stockfish is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +//Definition of input features Simplified_Threats of NNUE evaluation function + +#ifndef NNUE_FEATURES_FULL_THREATS_INCLUDED +#define NNUE_FEATURES_FULL_THREATS_INCLUDED + +#include + +#include "../../misc.h" +#include "../../types.h" +#include "../nnue_common.h" + +namespace Stockfish { +class Position; +} + +namespace Stockfish::Eval::NNUE::Features { + +static constexpr int numValidTargets[PIECE_NB] = {0, 6, 12, 10, 10, 12, 8, 0, + 0, 6, 12, 10, 10, 12, 8, 0}; +extern IndexType offsets[PIECE_NB][SQUARE_NB + 2]; +void init_threat_offsets(); + +class FullThreats { + public: + // Feature name + static constexpr const char* Name = "Full_Threats(Friend)"; + + // Hash value embedded in the evaluation file + static constexpr std::uint32_t HashValue = 0x8f234cb8u; + + // Number of feature dimensions + static constexpr IndexType Dimensions = 79856; + + // clang-format off + // Orient a square according to perspective (rotates by 180 for black) + static constexpr int OrientTBL[COLOR_NB][SQUARE_NB] = { + { SQ_A1, SQ_A1, SQ_A1, SQ_A1, SQ_H1, SQ_H1, SQ_H1, SQ_H1, + SQ_A1, SQ_A1, SQ_A1, SQ_A1, SQ_H1, SQ_H1, SQ_H1, SQ_H1, + SQ_A1, SQ_A1, SQ_A1, SQ_A1, SQ_H1, SQ_H1, SQ_H1, SQ_H1, + SQ_A1, SQ_A1, SQ_A1, SQ_A1, SQ_H1, SQ_H1, SQ_H1, SQ_H1, + SQ_A1, SQ_A1, SQ_A1, SQ_A1, SQ_H1, SQ_H1, SQ_H1, SQ_H1, + SQ_A1, SQ_A1, SQ_A1, SQ_A1, SQ_H1, SQ_H1, SQ_H1, SQ_H1, + SQ_A1, SQ_A1, SQ_A1, SQ_A1, SQ_H1, SQ_H1, SQ_H1, SQ_H1, + SQ_A1, SQ_A1, SQ_A1, SQ_A1, SQ_H1, SQ_H1, SQ_H1, SQ_H1 }, + { SQ_A8, SQ_A8, SQ_A8, SQ_A8, SQ_H8, SQ_H8, SQ_H8, SQ_H8, + SQ_A8, SQ_A8, SQ_A8, SQ_A8, SQ_H8, SQ_H8, SQ_H8, SQ_H8, + SQ_A8, SQ_A8, SQ_A8, SQ_A8, SQ_H8, SQ_H8, SQ_H8, SQ_H8, + SQ_A8, SQ_A8, SQ_A8, SQ_A8, SQ_H8, SQ_H8, SQ_H8, SQ_H8, + SQ_A8, SQ_A8, SQ_A8, SQ_A8, SQ_H8, SQ_H8, SQ_H8, SQ_H8, + SQ_A8, SQ_A8, SQ_A8, SQ_A8, SQ_H8, SQ_H8, SQ_H8, SQ_H8, + SQ_A8, SQ_A8, SQ_A8, SQ_A8, SQ_H8, SQ_H8, SQ_H8, SQ_H8, + SQ_A8, SQ_A8, SQ_A8, SQ_A8, SQ_H8, SQ_H8, SQ_H8, SQ_H8 } + }; + + static constexpr int map[PIECE_TYPE_NB-2][PIECE_TYPE_NB-2] = { + {0, 1, -1, 2, -1, -1}, + {0, 1, 2, 3, 4, 5}, + {0, 1, 2, 3, -1, 4}, + {0, 1, 2, 3, -1, 4}, + {0, 1, 2, 3, 4, 5}, + {0, 1, 2, 3, -1, -1} + }; + // clang-format on + + struct FusedUpdateData { + Bitboard dp2removedOriginBoard = 0; + Bitboard dp2removedTargetBoard = 0; + + Square dp2removed; + }; + + // Maximum number of simultaneously active features. + static constexpr IndexType MaxActiveDimensions = 128; + using IndexList = ValueList; + using DiffType = DirtyThreats; + + template + static IndexType make_index(Piece attkr, Square from, Square to, Piece attkd, Square ksq); + + // Get a list of indices for active features + template + static void append_active_indices(const Position& pos, IndexList& active); + + // Get a list of indices for recently changed features + template + static void append_changed_indices(Square ksq, + const DiffType& diff, + IndexList& removed, + IndexList& added, + FusedUpdateData* fd = nullptr, + bool first = false); + + // Returns whether the change stored in this DirtyPiece means + // that a full accumulator refresh is required. + static bool requires_refresh(const DiffType& diff, Color perspective); +}; + +} // namespace Stockfish::Eval::NNUE::Features + +#endif // #ifndef NNUE_FEATURES_FULL_THREATS_INCLUDED diff --git a/src/nnue/features/half_ka_v2_hm.cpp b/src/nnue/features/half_ka_v2_hm.cpp index 5bcfb0849..f652ba0b8 100644 --- a/src/nnue/features/half_ka_v2_hm.cpp +++ b/src/nnue/features/half_ka_v2_hm.cpp @@ -55,33 +55,33 @@ template IndexType HalfKAv2_hm::make_index(Square s, Piece pc, Square ksq // Get a list of indices for recently changed features template -void HalfKAv2_hm::append_changed_indices(Square ksq, - const DirtyPiece& dp, - IndexList& removed, - IndexList& added) { - removed.push_back(make_index(dp.from, dp.pc, ksq)); - if (dp.to != SQ_NONE) - added.push_back(make_index(dp.to, dp.pc, ksq)); +void HalfKAv2_hm::append_changed_indices(Square ksq, + const DiffType& diff, + IndexList& removed, + IndexList& added) { + removed.push_back(make_index(diff.from, diff.pc, ksq)); + if (diff.to != SQ_NONE) + added.push_back(make_index(diff.to, diff.pc, ksq)); - if (dp.remove_sq != SQ_NONE) - removed.push_back(make_index(dp.remove_sq, dp.remove_pc, ksq)); + if (diff.remove_sq != SQ_NONE) + removed.push_back(make_index(diff.remove_sq, diff.remove_pc, ksq)); - if (dp.add_sq != SQ_NONE) - added.push_back(make_index(dp.add_sq, dp.add_pc, ksq)); + if (diff.add_sq != SQ_NONE) + added.push_back(make_index(diff.add_sq, diff.add_pc, ksq)); } // Explicit template instantiations -template void HalfKAv2_hm::append_changed_indices(Square ksq, - const DirtyPiece& dp, - IndexList& removed, - IndexList& added); -template void HalfKAv2_hm::append_changed_indices(Square ksq, - const DirtyPiece& dp, - IndexList& removed, - IndexList& added); +template void HalfKAv2_hm::append_changed_indices(Square ksq, + const DiffType& dp, + IndexList& removed, + IndexList& added); +template void HalfKAv2_hm::append_changed_indices(Square ksq, + const DiffType& dp, + IndexList& removed, + IndexList& added); -bool HalfKAv2_hm::requires_refresh(const DirtyPiece& dirtyPiece, Color perspective) { - return dirtyPiece.pc == make_piece(perspective, KING); +bool HalfKAv2_hm::requires_refresh(const DiffType& diff, Color perspective) { + return diff.pc == make_piece(perspective, KING); } } // namespace Stockfish::Eval::NNUE::Features diff --git a/src/nnue/features/half_ka_v2_hm.h b/src/nnue/features/half_ka_v2_hm.h index b72bbbce4..e695b273a 100644 --- a/src/nnue/features/half_ka_v2_hm.h +++ b/src/nnue/features/half_ka_v2_hm.h @@ -104,6 +104,7 @@ class HalfKAv2_hm { // Maximum number of simultaneously active features. static constexpr IndexType MaxActiveDimensions = 32; using IndexList = ValueList; + using DiffType = DirtyPiece; // Index of a feature for a given king position and another piece on some square template @@ -116,11 +117,11 @@ class HalfKAv2_hm { // Get a list of indices for recently changed features template static void - append_changed_indices(Square ksq, const DirtyPiece& dp, IndexList& removed, IndexList& added); + append_changed_indices(Square ksq, const DiffType& diff, IndexList& removed, IndexList& added); // Returns whether the change stored in this DirtyPiece means // that a full accumulator refresh is required. - static bool requires_refresh(const DirtyPiece& dirtyPiece, Color perspective); + static bool requires_refresh(const DiffType& diff, Color perspective); }; } // namespace Stockfish::Eval::NNUE::Features diff --git a/src/nnue/network.cpp b/src/nnue/network.cpp index 48aeafdf6..b91763f06 100644 --- a/src/nnue/network.cpp +++ b/src/nnue/network.cpp @@ -222,7 +222,7 @@ void Network::verify(std::string { size_t size = sizeof(featureTransformer) + sizeof(Arch) * LayerStacks; f("NNUE evaluation using " + evalfilePath + " (" + std::to_string(size / (1024 * 1024)) - + "MiB, (" + std::to_string(featureTransformer.InputDimensions) + ", " + + "MiB, (" + std::to_string(featureTransformer.TotalInputDimensions) + ", " + std::to_string(network[0].TransformedFeatureDimensions) + ", " + std::to_string(network[0].FC_0_OUTPUTS) + ", " + std::to_string(network[0].FC_1_OUTPUTS) + ", 1))"); diff --git a/src/nnue/nnue_accumulator.cpp b/src/nnue/nnue_accumulator.cpp index b1743329f..0444b6e40 100644 --- a/src/nnue/nnue_accumulator.cpp +++ b/src/nnue/nnue_accumulator.cpp @@ -27,7 +27,9 @@ #include "../misc.h" #include "../position.h" #include "../types.h" +#include "features/half_ka_v2_hm.h" #include "nnue_architecture.h" +#include "nnue_common.h" #include "nnue_feature_transformer.h" // IWYU pragma: keep #include "simd.h" @@ -40,43 +42,90 @@ namespace { template void double_inc_update(const FeatureTransformer& featureTransformer, const Square ksq, - AccumulatorState& middle_state, - AccumulatorState& target_state, - const AccumulatorState& computed); + AccumulatorState& middle_state, + AccumulatorState& target_state, + const AccumulatorState& computed); -template +template +void double_inc_update(const FeatureTransformer& featureTransformer, + const Square ksq, + AccumulatorState& middle_state, + AccumulatorState& target_state, + const AccumulatorState& computed, + const DirtyPiece& dp2); + +template void update_accumulator_incremental( const FeatureTransformer& featureTransformer, const Square ksq, - AccumulatorState& target_state, - const AccumulatorState& computed); + AccumulatorState& target_state, + const AccumulatorState& computed); template void update_accumulator_refresh_cache(const FeatureTransformer& featureTransformer, const Position& pos, - AccumulatorState& accumulatorState, + AccumulatorState& accumulatorState, AccumulatorCaches::Cache& cache); +template +void update_threats_accumulator_full(const FeatureTransformer& featureTransformer, + const Position& pos, + AccumulatorState& accumulatorState); } -void AccumulatorState::reset(const DirtyPiece& dp) noexcept { - dirtyPiece = dp; - accumulatorBig.computed.fill(false); - accumulatorSmall.computed.fill(false); +template +const AccumulatorState& AccumulatorStack::latest() const noexcept { + return accumulators()[size - 1]; } -const AccumulatorState& AccumulatorStack::latest() const noexcept { return accumulators[size - 1]; } +// Explicit template instantiations +template const AccumulatorState& AccumulatorStack::latest() const noexcept; +template const AccumulatorState& AccumulatorStack::latest() const noexcept; -AccumulatorState& AccumulatorStack::mut_latest() noexcept { return accumulators[size - 1]; } +template +AccumulatorState& AccumulatorStack::mut_latest() noexcept { + return mut_accumulators()[size - 1]; +} + +template +const std::array, AccumulatorStack::MaxSize>& +AccumulatorStack::accumulators() const noexcept { + static_assert(std::is_same_v || std::is_same_v, + "Invalid Feature Set Type"); + + if constexpr (std::is_same_v) + return psq_accumulators; + + if constexpr (std::is_same_v) + return threat_accumulators; +} + +template +std::array, AccumulatorStack::MaxSize>& +AccumulatorStack::mut_accumulators() noexcept { + static_assert(std::is_same_v || std::is_same_v, + "Invalid Feature Set Type"); + + if constexpr (std::is_same_v) + return psq_accumulators; + + if constexpr (std::is_same_v) + return threat_accumulators; +} void AccumulatorStack::reset() noexcept { - accumulators[0].reset({}); + psq_accumulators[0].reset({}); + threat_accumulators[0].reset({}); size = 1; } -void AccumulatorStack::push(const DirtyPiece& dirtyPiece) noexcept { - assert(size < accumulators.size()); - accumulators[size].reset(dirtyPiece); +void AccumulatorStack::push(const DirtyBoardData& dirtyBoardData) noexcept { + assert(size < MaxSize); + psq_accumulators[size].reset(dirtyBoardData.dp); + threat_accumulators[size].reset(dirtyBoardData.dts); size++; } @@ -89,53 +138,71 @@ template void AccumulatorStack::evaluate(const Position& pos, const FeatureTransformer& featureTransformer, AccumulatorCaches::Cache& cache) noexcept { + constexpr bool UseThreats = (Dimensions == TransformedFeatureDimensionsBig); - evaluate_side(pos, featureTransformer, cache); - evaluate_side(pos, featureTransformer, cache); + evaluate_side(pos, featureTransformer, cache); + + if (UseThreats) + evaluate_side(pos, featureTransformer, cache); + + evaluate_side(pos, featureTransformer, cache); + + if (UseThreats) + evaluate_side(pos, featureTransformer, cache); } -template +template void AccumulatorStack::evaluate_side(const Position& pos, const FeatureTransformer& featureTransformer, AccumulatorCaches::Cache& cache) noexcept { - const auto last_usable_accum = find_last_usable_accumulator(); + const auto last_usable_accum = + find_last_usable_accumulator(); - if ((accumulators[last_usable_accum].template acc()).computed[Perspective]) - forward_update_incremental(pos, featureTransformer, last_usable_accum); + if ((accumulators()[last_usable_accum].template acc()) + .computed[Perspective]) + forward_update_incremental(pos, featureTransformer, + last_usable_accum); else { - update_accumulator_refresh_cache(featureTransformer, pos, mut_latest(), cache); - backward_update_incremental(pos, featureTransformer, last_usable_accum); + if constexpr (std::is_same_v) + update_accumulator_refresh_cache(featureTransformer, pos, + mut_latest(), cache); + else + update_threats_accumulator_full(featureTransformer, pos, + mut_latest()); + + backward_update_incremental(pos, featureTransformer, + last_usable_accum); } } // Find the earliest usable accumulator, this can either be a computed accumulator or the accumulator // state just before a change that requires full refresh. -template +template std::size_t AccumulatorStack::find_last_usable_accumulator() const noexcept { for (std::size_t curr_idx = size - 1; curr_idx > 0; curr_idx--) { - if ((accumulators[curr_idx].template acc()).computed[Perspective]) + if ((accumulators()[curr_idx].template acc()).computed[Perspective]) return curr_idx; - if (FeatureSet::requires_refresh(accumulators[curr_idx].dirtyPiece, Perspective)) + if (FeatureSet::requires_refresh(accumulators()[curr_idx].diff, Perspective)) return curr_idx; } return 0; } -template +template void AccumulatorStack::forward_update_incremental( const Position& pos, const FeatureTransformer& featureTransformer, const std::size_t begin) noexcept { - assert(begin < accumulators.size()); - assert((accumulators[begin].acc()).computed[Perspective]); + assert(begin < accumulators().size()); + assert((accumulators()[begin].template acc()).computed[Perspective]); const Square ksq = pos.square(Perspective); @@ -143,45 +210,65 @@ void AccumulatorStack::forward_update_incremental( { if (next + 1 < size) { - DirtyPiece& dp1 = accumulators[next].dirtyPiece; - DirtyPiece& dp2 = accumulators[next + 1].dirtyPiece; + DirtyPiece& dp1 = mut_accumulators()[next].diff; + DirtyPiece& dp2 = mut_accumulators()[next + 1].diff; - if (dp1.to != SQ_NONE && dp1.to == dp2.remove_sq) + auto& accumulators = mut_accumulators(); + + if constexpr (std::is_same_v) { - const Square captureSq = dp1.to; - dp1.to = dp2.remove_sq = SQ_NONE; - double_inc_update(featureTransformer, ksq, accumulators[next], - accumulators[next + 1], accumulators[next - 1]); - dp1.to = dp2.remove_sq = captureSq; + if (dp2.remove_sq != SQ_NONE + && (accumulators[next].diff.threateningSqs & square_bb(dp2.remove_sq))) + { + double_inc_update(featureTransformer, ksq, accumulators[next], + accumulators[next + 1], accumulators[next - 1], + dp2); + next++; + continue; + } + } - next++; - continue; + if constexpr (std::is_same_v) + { + if (dp1.to != SQ_NONE && dp1.to == dp2.remove_sq) + { + const Square captureSq = dp1.to; + dp1.to = dp2.remove_sq = SQ_NONE; + double_inc_update(featureTransformer, ksq, accumulators[next], + accumulators[next + 1], accumulators[next - 1]); + dp1.to = dp2.remove_sq = captureSq; + next++; + continue; + } } } - update_accumulator_incremental( - featureTransformer, ksq, accumulators[next], accumulators[next - 1]); + + update_accumulator_incremental(featureTransformer, ksq, + mut_accumulators()[next], + accumulators()[next - 1]); } - assert((latest().acc()).computed[Perspective]); + assert((latest().acc()).computed[Perspective]); } -template +template void AccumulatorStack::backward_update_incremental( const Position& pos, const FeatureTransformer& featureTransformer, const std::size_t end) noexcept { - assert(end < accumulators.size()); + assert(end < accumulators().size()); assert(end < size); - assert((latest().acc()).computed[Perspective]); + assert((latest().template acc()).computed[Perspective]); const Square ksq = pos.square(Perspective); for (std::int64_t next = std::int64_t(size) - 2; next >= std::int64_t(end); next--) - update_accumulator_incremental( - featureTransformer, ksq, accumulators[next], accumulators[next + 1]); + update_accumulator_incremental(featureTransformer, ksq, + mut_accumulators()[next], + accumulators()[next + 1]); - assert((accumulators[end].acc()).computed[Perspective]); + assert((accumulators()[end].template acc()).computed[Perspective]); } // Explicit template instantiations @@ -214,15 +301,15 @@ void fused_row_reduce(const ElementType* in, ElementType* out, const Ts* const.. vecIn[i], reinterpret_cast(rows)[i]...); } -template +template struct AccumulatorUpdateContext { const FeatureTransformer& featureTransformer; - const AccumulatorState& from; - AccumulatorState& to; + const AccumulatorState& from; + AccumulatorState& to; AccumulatorUpdateContext(const FeatureTransformer& ft, - const AccumulatorState& accF, - AccumulatorState& accT) noexcept : + const AccumulatorState& accF, + AccumulatorState& accT) noexcept : featureTransformer{ft}, from{accF}, to{accT} {} @@ -240,40 +327,169 @@ struct AccumulatorUpdateContext { }; fused_row_reduce( - (from.acc()).accumulation[Perspective], - (to.acc()).accumulation[Perspective], to_weight_vector(indices)...); + (from.template acc()).accumulation[Perspective], + (to.template acc()).accumulation[Perspective], to_weight_vector(indices)...); fused_row_reduce( - (from.acc()).psqtAccumulation[Perspective], - (to.acc()).psqtAccumulation[Perspective], to_psqt_weight_vector(indices)...); + (from.template acc()).psqtAccumulation[Perspective], + (to.template acc()).psqtAccumulation[Perspective], + to_psqt_weight_vector(indices)...); + } + + void apply(typename FeatureSet::IndexList added, typename FeatureSet::IndexList removed) { + const auto fromAcc = from.template acc().accumulation[Perspective]; + const auto toAcc = to.template acc().accumulation[Perspective]; + + const auto fromPsqtAcc = from.template acc().psqtAccumulation[Perspective]; + const auto toPsqtAcc = to.template acc().psqtAccumulation[Perspective]; + +#ifdef VECTOR + using Tiling = SIMDTiling; + vec_t acc[Tiling::NumRegs]; + psqt_vec_t psqt[Tiling::NumPsqtRegs]; + + for (IndexType j = 0; j < Dimensions / Tiling::TileHeight; ++j) + { + auto* fromTile = reinterpret_cast(&fromAcc[j * Tiling::TileHeight]); + auto* toTile = reinterpret_cast(&toAcc[j * Tiling::TileHeight]); + + for (IndexType k = 0; k < Tiling::NumRegs; ++k) + acc[k] = fromTile[k]; + + for (IndexType i = 0; i < removed.size(); ++i) + { + IndexType index = removed[i]; + const IndexType offset = Dimensions * index + j * Tiling::TileHeight; + auto* column = + reinterpret_cast(&featureTransformer.threatWeights[offset]); + + #ifdef USE_NEON + for (IndexType k = 0; k < Tiling::NumRegs; k += 2) + { + acc[k] = vec_sub_16(acc[k], vmovl_s8(vget_low_s8(column[k / 2]))); + acc[k + 1] = vec_sub_16(acc[k + 1], vmovl_high_s8(column[k / 2])); + } + #else + for (IndexType k = 0; k < Tiling::NumRegs; ++k) + acc[k] = vec_sub_16(acc[k], vec_convert_8_16(column[k])); + #endif + } + + for (IndexType i = 0; i < added.size(); ++i) + { + IndexType index = added[i]; + const IndexType offset = Dimensions * index + j * Tiling::TileHeight; + auto* column = + reinterpret_cast(&featureTransformer.threatWeights[offset]); + + #ifdef USE_NEON + for (IndexType k = 0; k < Tiling::NumRegs; k += 2) + { + acc[k] = vec_add_16(acc[k], vmovl_s8(vget_low_s8(column[k / 2]))); + acc[k + 1] = vec_add_16(acc[k + 1], vmovl_high_s8(column[k / 2])); + } + #else + for (IndexType k = 0; k < Tiling::NumRegs; ++k) + acc[k] = vec_add_16(acc[k], vec_convert_8_16(column[k])); + #endif + } + + for (IndexType k = 0; k < Tiling::NumRegs; k++) + vec_store(&toTile[k], acc[k]); + } + + for (IndexType j = 0; j < PSQTBuckets / Tiling::PsqtTileHeight; ++j) + { + auto* fromTilePsqt = + reinterpret_cast(&fromPsqtAcc[j * Tiling::PsqtTileHeight]); + auto* toTilePsqt = + reinterpret_cast(&toPsqtAcc[j * Tiling::PsqtTileHeight]); + + for (IndexType k = 0; k < Tiling::NumPsqtRegs; ++k) + psqt[k] = fromTilePsqt[k]; + + for (IndexType i = 0; i < removed.size(); ++i) + { + IndexType index = removed[i]; + const IndexType offset = PSQTBuckets * index + j * Tiling::PsqtTileHeight; + auto* columnPsqt = reinterpret_cast( + &featureTransformer.threatPsqtWeights[offset]); + + for (std::size_t k = 0; k < Tiling::NumPsqtRegs; ++k) + psqt[k] = vec_sub_psqt_32(psqt[k], columnPsqt[k]); + } + + for (IndexType i = 0; i < added.size(); ++i) + { + IndexType index = added[i]; + const IndexType offset = PSQTBuckets * index + j * Tiling::PsqtTileHeight; + auto* columnPsqt = reinterpret_cast( + &featureTransformer.threatPsqtWeights[offset]); + + for (std::size_t k = 0; k < Tiling::NumPsqtRegs; ++k) + psqt[k] = vec_add_psqt_32(psqt[k], columnPsqt[k]); + } + + for (IndexType k = 0; k < Tiling::NumPsqtRegs; ++k) + vec_store_psqt(&toTilePsqt[k], psqt[k]); + } + +#else + + std::copy_n(fromAcc, Dimensions, toAcc); + std::copy_n(fromPsqtAcc, PSQTBuckets, toPsqtAcc); + + for (const auto index : removed) + { + const IndexType offset = Dimensions * index; + + for (IndexType j = 0; j < Dimensions; ++j) + toAcc[j] -= featureTransformer.threatWeights[offset + j]; + + for (std::size_t k = 0; k < PSQTBuckets; ++k) + toPsqtAcc[k] -= featureTransformer.threatPsqtWeights[index * PSQTBuckets + k]; + } + + for (const auto index : added) + { + const IndexType offset = Dimensions * index; + + for (IndexType j = 0; j < Dimensions; ++j) + toAcc[j] += featureTransformer.threatWeights[offset + j]; + + for (std::size_t k = 0; k < PSQTBuckets; ++k) + toPsqtAcc[k] += featureTransformer.threatPsqtWeights[index * PSQTBuckets + k]; + } + +#endif } }; -template +template auto make_accumulator_update_context(const FeatureTransformer& featureTransformer, - const AccumulatorState& accumulatorFrom, - AccumulatorState& accumulatorTo) noexcept { - return AccumulatorUpdateContext{featureTransformer, accumulatorFrom, - accumulatorTo}; + const AccumulatorState& accumulatorFrom, + AccumulatorState& accumulatorTo) noexcept { + return AccumulatorUpdateContext{ + featureTransformer, accumulatorFrom, accumulatorTo}; } template void double_inc_update(const FeatureTransformer& featureTransformer, const Square ksq, - AccumulatorState& middle_state, - AccumulatorState& target_state, - const AccumulatorState& computed) { + AccumulatorState& middle_state, + AccumulatorState& target_state, + const AccumulatorState& computed) { assert(computed.acc().computed[Perspective]); assert(!middle_state.acc().computed[Perspective]); assert(!target_state.acc().computed[Perspective]); - FeatureSet::IndexList removed, added; - FeatureSet::append_changed_indices(ksq, middle_state.dirtyPiece, removed, added); + PSQFeatureSet::IndexList removed, added; + PSQFeatureSet::append_changed_indices(ksq, middle_state.diff, removed, added); // you can't capture a piece that was just involved in castling since the rook ends up // in a square that the king passed assert(added.size() < 2); - FeatureSet::append_changed_indices(ksq, target_state.dirtyPiece, removed, added); + PSQFeatureSet::append_changed_indices(ksq, target_state.diff, removed, added); assert(added.size() == 1); assert(removed.size() == 2 || removed.size() == 3); @@ -300,15 +516,48 @@ void double_inc_update(const FeatureTransformer& f target_state.acc().computed[Perspective] = true; } -template +template +void double_inc_update(const FeatureTransformer& featureTransformer, + const Square ksq, + AccumulatorState& middle_state, + AccumulatorState& target_state, + const AccumulatorState& computed, + const DirtyPiece& dp2) { + + assert(computed.acc().computed[Perspective]); + assert(!middle_state.acc().computed[Perspective]); + assert(!target_state.acc().computed[Perspective]); + + ThreatFeatureSet::FusedUpdateData fusedData; + + fusedData.dp2removed = dp2.remove_sq; + + ThreatFeatureSet::IndexList removed, added; + ThreatFeatureSet::append_changed_indices(ksq, middle_state.diff, removed, added, + &fusedData, true); + ThreatFeatureSet::append_changed_indices(ksq, target_state.diff, removed, added, + &fusedData, false); + + auto updateContext = + make_accumulator_update_context(featureTransformer, computed, target_state); + + updateContext.apply(added, removed); + + target_state.acc().computed[Perspective] = true; +} + +template void update_accumulator_incremental( const FeatureTransformer& featureTransformer, const Square ksq, - AccumulatorState& target_state, - const AccumulatorState& computed) { + AccumulatorState& target_state, + const AccumulatorState& computed) { - assert((computed.acc()).computed[Perspective]); - assert(!(target_state.acc()).computed[Perspective]); + assert((computed.template acc()).computed[Perspective]); + assert(!(target_state.template acc()).computed[Perspective]); // The size must be enough to contain the largest possible update. // That might depend on the feature set and generally relies on the @@ -316,50 +565,56 @@ void update_accumulator_incremental( // updates with more added/removed features than MaxActiveDimensions. // In this case, the maximum size of both feature addition and removal // is 2, since we are incrementally updating one move at a time. - FeatureSet::IndexList removed, added; + typename FeatureSet::IndexList removed, added; if constexpr (Forward) - FeatureSet::append_changed_indices(ksq, target_state.dirtyPiece, removed, - added); + FeatureSet::template append_changed_indices(ksq, target_state.diff, removed, + added); else - FeatureSet::append_changed_indices(ksq, computed.dirtyPiece, added, removed); - - assert(added.size() == 1 || added.size() == 2); - assert(removed.size() == 1 || removed.size() == 2); - assert((Forward && added.size() <= removed.size()) - || (!Forward && added.size() >= removed.size())); - - // Workaround compiler warning for uninitialized variables, replicated on - // profile builds on windows with gcc 14.2.0. - // TODO remove once unneeded - sf_assume(added.size() == 1 || added.size() == 2); - sf_assume(removed.size() == 1 || removed.size() == 2); + FeatureSet::template append_changed_indices(ksq, computed.diff, added, + removed); auto updateContext = make_accumulator_update_context(featureTransformer, computed, target_state); - if ((Forward && removed.size() == 1) || (!Forward && added.size() == 1)) - { - assert(added.size() == 1 && removed.size() == 1); - updateContext.template apply(added[0], removed[0]); - } - else if (Forward && added.size() == 1) - { - assert(removed.size() == 2); - updateContext.template apply(added[0], removed[0], removed[1]); - } - else if (!Forward && removed.size() == 1) - { - assert(added.size() == 2); - updateContext.template apply(added[0], added[1], removed[0]); - } + if constexpr (std::is_same_v) + updateContext.apply(added, removed); else { - assert(added.size() == 2 && removed.size() == 2); - updateContext.template apply(added[0], added[1], removed[0], - removed[1]); + assert(added.size() == 1 || added.size() == 2); + assert(removed.size() == 1 || removed.size() == 2); + assert((Forward && added.size() <= removed.size()) + || (!Forward && added.size() >= removed.size())); + + // Workaround compiler warning for uninitialized variables, replicated + // on profile builds on windows with gcc 14.2.0. + // TODO remove once unneeded + sf_assume(added.size() == 1 || added.size() == 2); + sf_assume(removed.size() == 1 || removed.size() == 2); + + if ((Forward && removed.size() == 1) || (!Forward && added.size() == 1)) + { + assert(added.size() == 1 && removed.size() == 1); + updateContext.template apply(added[0], removed[0]); + } + else if (Forward && added.size() == 1) + { + assert(removed.size() == 2); + updateContext.template apply(added[0], removed[0], removed[1]); + } + else if (!Forward && removed.size() == 1) + { + assert(added.size() == 2); + updateContext.template apply(added[0], added[1], removed[0]); + } + else + { + assert(added.size() == 2 && removed.size() == 2); + updateContext.template apply(added[0], added[1], removed[0], + removed[1]); + } } - (target_state.acc()).computed[Perspective] = true; + (target_state.template acc()).computed[Perspective] = true; } Bitboard get_changed_pieces(const Piece old[SQUARE_NB], const Piece new_[SQUARE_NB]) { @@ -388,32 +643,32 @@ Bitboard get_changed_pieces(const Piece old[SQUARE_NB], const Piece new_[SQUARE_ template void update_accumulator_refresh_cache(const FeatureTransformer& featureTransformer, const Position& pos, - AccumulatorState& accumulatorState, + AccumulatorState& accumulatorState, AccumulatorCaches::Cache& cache) { using Tiling [[maybe_unused]] = SIMDTiling; - const Square ksq = pos.square(Perspective); - auto& entry = cache[ksq][Perspective]; - FeatureSet::IndexList removed, added; + const Square ksq = pos.square(Perspective); + auto& entry = cache[ksq][Perspective]; + PSQFeatureSet::IndexList removed, added; - const Bitboard changed_bb = get_changed_pieces(entry.pieces, pos.piece_array()); + const Bitboard changed_bb = get_changed_pieces(entry.pieces, pos.piece_array().data()); Bitboard removed_bb = changed_bb & entry.pieceBB; Bitboard added_bb = changed_bb & pos.pieces(); while (removed_bb) { Square sq = pop_lsb(removed_bb); - removed.push_back(FeatureSet::make_index(sq, entry.pieces[sq], ksq)); + removed.push_back(PSQFeatureSet::make_index(sq, entry.pieces[sq], ksq)); } while (added_bb) { Square sq = pop_lsb(added_bb); - added.push_back(FeatureSet::make_index(sq, pos.piece_on(sq), ksq)); + added.push_back(PSQFeatureSet::make_index(sq, pos.piece_on(sq), ksq)); } entry.pieceBB = pos.pieces(); - std::copy_n(pos.piece_array(), SQUARE_NB, entry.pieces); + std::copy_n(pos.piece_array().begin(), SQUARE_NB, entry.pieces); auto& accumulator = accumulatorState.acc(); accumulator.computed[Perspective] = true; @@ -530,14 +785,110 @@ void update_accumulator_refresh_cache(const FeatureTransformer& feat // The accumulator of the refresh entry has been updated. // Now copy its content to the actual accumulator we were refreshing. - std::memcpy(accumulator.accumulation[Perspective], entry.accumulation, + std::memcpy(accumulator.accumulation[Perspective], entry.accumulation.data(), sizeof(BiasType) * Dimensions); - std::memcpy(accumulator.psqtAccumulation[Perspective], entry.psqtAccumulation, + std::memcpy(accumulator.psqtAccumulation[Perspective], entry.psqtAccumulation.data(), sizeof(int32_t) * PSQTBuckets); #endif } +template +void update_threats_accumulator_full(const FeatureTransformer& featureTransformer, + const Position& pos, + AccumulatorState& accumulatorState) { + using Tiling [[maybe_unused]] = SIMDTiling; + + ThreatFeatureSet::IndexList active; + ThreatFeatureSet::append_active_indices(pos, active); + + auto& accumulator = accumulatorState.acc(); + accumulator.computed[Perspective] = true; + +#ifdef VECTOR + vec_t acc[Tiling::NumRegs]; + psqt_vec_t psqt[Tiling::NumPsqtRegs]; + + for (IndexType j = 0; j < Dimensions / Tiling::TileHeight; ++j) + { + auto* accTile = + reinterpret_cast(&accumulator.accumulation[Perspective][j * Tiling::TileHeight]); + + for (IndexType k = 0; k < Tiling::NumRegs; ++k) + acc[k] = vec_zero(); + + IndexType i = 0; + + for (; i < active.size(); ++i) + { + IndexType index = active[i]; + const IndexType offset = Dimensions * index + j * Tiling::TileHeight; + auto* column = + reinterpret_cast(&featureTransformer.threatWeights[offset]); + + #ifdef USE_NEON + for (IndexType k = 0; k < Tiling::NumRegs; k += 2) + { + acc[k] = vec_add_16(acc[k], vmovl_s8(vget_low_s8(column[k / 2]))); + acc[k + 1] = vec_add_16(acc[k + 1], vmovl_high_s8(column[k / 2])); + } + #else + for (IndexType k = 0; k < Tiling::NumRegs; ++k) + acc[k] = vec_add_16(acc[k], vec_convert_8_16(column[k])); + #endif + } + + for (IndexType k = 0; k < Tiling::NumRegs; k++) + vec_store(&accTile[k], acc[k]); + } + + for (IndexType j = 0; j < PSQTBuckets / Tiling::PsqtTileHeight; ++j) + { + auto* accTilePsqt = reinterpret_cast( + &accumulator.psqtAccumulation[Perspective][j * Tiling::PsqtTileHeight]); + + for (IndexType k = 0; k < Tiling::NumPsqtRegs; ++k) + psqt[k] = vec_zero_psqt(); + + for (IndexType i = 0; i < active.size(); ++i) + { + IndexType index = active[i]; + const IndexType offset = PSQTBuckets * index + j * Tiling::PsqtTileHeight; + auto* columnPsqt = + reinterpret_cast(&featureTransformer.threatPsqtWeights[offset]); + + for (std::size_t k = 0; k < Tiling::NumPsqtRegs; ++k) + psqt[k] = vec_add_psqt_32(psqt[k], columnPsqt[k]); + } + + for (IndexType k = 0; k < Tiling::NumPsqtRegs; ++k) + vec_store_psqt(&accTilePsqt[k], psqt[k]); + } + +#else + + for (IndexType j = 0; j < Dimensions; ++j) + accumulator.accumulation[Perspective][j] = 0; + + for (std::size_t k = 0; k < PSQTBuckets; ++k) + accumulator.psqtAccumulation[Perspective][k] = 0; + + for (const auto index : active) + { + const IndexType offset = Dimensions * index; + + for (IndexType j = 0; j < Dimensions; ++j) + accumulator.accumulation[Perspective][j] += + featureTransformer.threatWeights[offset + j]; + + for (std::size_t k = 0; k < PSQTBuckets; ++k) + accumulator.psqtAccumulation[Perspective][k] += + featureTransformer.threatPsqtWeights[index * PSQTBuckets + k]; + } + +#endif +} + } } diff --git a/src/nnue/nnue_accumulator.h b/src/nnue/nnue_accumulator.h index 95cf74fa3..4c907c7e3 100644 --- a/src/nnue/nnue_accumulator.h +++ b/src/nnue/nnue_accumulator.h @@ -68,16 +68,16 @@ struct AccumulatorCaches { struct alignas(CacheLineSize) Cache { struct alignas(CacheLineSize) Entry { - BiasType accumulation[Size]; - PSQTWeightType psqtAccumulation[PSQTBuckets]; - Piece pieces[SQUARE_NB]; - Bitboard pieceBB; + std::array accumulation; + std::array psqtAccumulation; + Piece pieces[SQUARE_NB]; + Bitboard pieceBB; // To initialize a refresh entry, we set all its bitboards empty, // so we put the biases in the accumulation, without any weights on top - void clear(const BiasType* biases) { + void clear(const std::array& biases) { - std::memcpy(accumulation, biases, sizeof(accumulation)); + accumulation = biases; std::memset((uint8_t*) this + offsetof(Entry, psqtAccumulation), 0, sizeof(Entry) - offsetof(Entry, psqtAccumulation)); } @@ -106,10 +106,11 @@ struct AccumulatorCaches { }; +template struct AccumulatorState { Accumulator accumulatorBig; Accumulator accumulatorSmall; - DirtyPiece dirtyPiece; + typename FeatureSet::DiffType diff; template auto& acc() noexcept { @@ -135,16 +136,22 @@ struct AccumulatorState { return accumulatorSmall; } - void reset(const DirtyPiece& dp) noexcept; + void reset(const typename FeatureSet::DiffType& dp) noexcept { + diff = dp; + accumulatorBig.computed.fill(false); + accumulatorSmall.computed.fill(false); + } }; - class AccumulatorStack { public: - [[nodiscard]] const AccumulatorState& latest() const noexcept; + static constexpr std::size_t MaxSize = MAX_PLY + 1; + + template + [[nodiscard]] const AccumulatorState& latest() const noexcept; void reset() noexcept; - void push(const DirtyPiece& dirtyPiece) noexcept; + void push(const DirtyBoardData& dirtyBoardData) noexcept; void pop() noexcept; template @@ -153,28 +160,36 @@ class AccumulatorStack { AccumulatorCaches::Cache& cache) noexcept; private: - [[nodiscard]] AccumulatorState& mut_latest() noexcept; + template + [[nodiscard]] AccumulatorState& mut_latest() noexcept; - template + template + [[nodiscard]] const std::array, MaxSize>& accumulators() const noexcept; + + template + [[nodiscard]] std::array, MaxSize>& mut_accumulators() noexcept; + + template void evaluate_side(const Position& pos, const FeatureTransformer& featureTransformer, AccumulatorCaches::Cache& cache) noexcept; - template + template [[nodiscard]] std::size_t find_last_usable_accumulator() const noexcept; - template + template void forward_update_incremental(const Position& pos, const FeatureTransformer& featureTransformer, const std::size_t begin) noexcept; - template + template void backward_update_incremental(const Position& pos, const FeatureTransformer& featureTransformer, const std::size_t end) noexcept; - std::array accumulators; - std::size_t size = 1; + std::array, MaxSize> psq_accumulators; + std::array, MaxSize> threat_accumulators; + std::size_t size = 1; }; } // namespace Stockfish::Eval::NNUE diff --git a/src/nnue/nnue_architecture.h b/src/nnue/nnue_architecture.h index 5965f24aa..5093abdd7 100644 --- a/src/nnue/nnue_architecture.h +++ b/src/nnue/nnue_architecture.h @@ -26,6 +26,7 @@ #include #include "features/half_ka_v2_hm.h" +#include "features/full_threats.h" #include "layers/affine_transform.h" #include "layers/affine_transform_sparse_input.h" #include "layers/clipped_relu.h" @@ -35,10 +36,11 @@ namespace Stockfish::Eval::NNUE { // Input features used in evaluation function -using FeatureSet = Features::HalfKAv2_hm; +using ThreatFeatureSet = Features::FullThreats; +using PSQFeatureSet = Features::HalfKAv2_hm; // Number of input feature dimensions after conversion -constexpr IndexType TransformedFeatureDimensionsBig = 3072; +constexpr IndexType TransformedFeatureDimensionsBig = 1024; constexpr int L2Big = 15; constexpr int L3Big = 32; diff --git a/src/nnue/nnue_common.h b/src/nnue/nnue_common.h index 3617a85eb..8a877ae2b 100644 --- a/src/nnue/nnue_common.h +++ b/src/nnue/nnue_common.h @@ -48,10 +48,11 @@ namespace Stockfish::Eval::NNUE { -using BiasType = std::int16_t; -using WeightType = std::int16_t; -using PSQTWeightType = std::int32_t; -using IndexType = std::uint32_t; +using BiasType = std::int16_t; +using ThreatWeightType = std::int8_t; +using WeightType = std::int16_t; +using PSQTWeightType = std::int32_t; +using IndexType = std::uint32_t; // Version of the evaluation file constexpr std::uint32_t Version = 0x7AF32F20u; @@ -172,8 +173,8 @@ inline void write_little_endian(std::ostream& stream, const IntType* values, std // Read N signed integers from the stream s, putting them in the array out. // The stream is assumed to be compressed using the signed LEB128 format. // See https://en.wikipedia.org/wiki/LEB128 for a description of the compression scheme. -template -inline void read_leb_128(std::istream& stream, IntType* out, std::size_t count) { +template +inline void read_leb_128(std::istream& stream, std::array& out) { // Check the presence of our LEB128 magic string char leb128MagicString[Leb128MagicStringSize]; @@ -188,7 +189,7 @@ inline void read_leb_128(std::istream& stream, IntType* out, std::size_t count) auto bytes_left = read_little_endian(stream); std::uint32_t buf_pos = BUF_SIZE; - for (std::size_t i = 0; i < count; ++i) + for (std::size_t i = 0; i < Count; ++i) { IntType result = 0; size_t shift = 0; @@ -223,8 +224,8 @@ inline void read_leb_128(std::istream& stream, IntType* out, std::size_t count) // This takes N integers from array values, compresses them with // the LEB128 algorithm and writes the result on the stream s. // See https://en.wikipedia.org/wiki/LEB128 for a description of the compression scheme. -template -inline void write_leb_128(std::ostream& stream, const IntType* values, std::size_t count) { +template +inline void write_leb_128(std::ostream& stream, const std::array& values) { // Write our LEB128 magic string stream.write(Leb128MagicString, Leb128MagicStringSize); @@ -232,7 +233,7 @@ inline void write_leb_128(std::ostream& stream, const IntType* values, std::size static_assert(std::is_signed_v, "Not implemented for unsigned types"); std::uint32_t byte_count = 0; - for (std::size_t i = 0; i < count; ++i) + for (std::size_t i = 0; i < Count; ++i) { IntType value = values[i]; std::uint8_t byte; @@ -264,7 +265,7 @@ inline void write_leb_128(std::ostream& stream, const IntType* values, std::size flush(); }; - for (std::size_t i = 0; i < count; ++i) + for (std::size_t i = 0; i < Count; ++i) { IntType value = values[i]; while (true) diff --git a/src/nnue/nnue_feature_transformer.h b/src/nnue/nnue_feature_transformer.h index 3439bc43c..5ad2d3371 100644 --- a/src/nnue/nnue_feature_transformer.h +++ b/src/nnue/nnue_feature_transformer.h @@ -25,6 +25,7 @@ #include #include #include +#include #include "../position.h" #include "../types.h" @@ -48,7 +49,7 @@ invert_permutation(const std::array& order) { // Divide a byte region of size TotalSize to chunks of size // BlockSize, and permute the blocks by a given order template -void permute(T (&data)[N], const std::array& order) { +void permute(std::array& data, const std::array& order) { constexpr std::size_t TotalSize = N * sizeof(T); static_assert(TotalSize % (BlockSize * OrderSize) == 0, @@ -58,7 +59,7 @@ void permute(T (&data)[N], const std::array& order) { std::array buffer{}; - std::byte* const bytes = reinterpret_cast(data); + std::byte* const bytes = reinterpret_cast(data.data()); for (std::size_t i = 0; i < TotalSize; i += ProcessChunkSize) { @@ -79,7 +80,8 @@ void permute(T (&data)[N], const std::array& order) { // Input feature converter template class FeatureTransformer { - + static constexpr bool UseThreats = + (TransformedFeatureDimensions == TransformedFeatureDimensionsBig); // Number of output dimensions for one side static constexpr IndexType HalfDimensions = TransformedFeatureDimensions; @@ -88,7 +90,10 @@ class FeatureTransformer { using OutputType = TransformedFeatureType; // Number of input/output dimensions - static constexpr IndexType InputDimensions = FeatureSet::Dimensions; + static constexpr IndexType InputDimensions = PSQFeatureSet::Dimensions; + static constexpr IndexType ThreatInputDimensions = ThreatFeatureSet::Dimensions; + static constexpr IndexType TotalInputDimensions = + InputDimensions + (UseThreats ? ThreatInputDimensions : 0); static constexpr IndexType OutputDimensions = HalfDimensions; // Size of forward propagation buffer @@ -119,17 +124,24 @@ class FeatureTransformer { // Hash value embedded in the evaluation file static constexpr std::uint32_t get_hash_value() { - return FeatureSet::HashValue ^ (OutputDimensions * 2); + return (UseThreats ? ThreatFeatureSet::HashValue : PSQFeatureSet::HashValue) + ^ (OutputDimensions * 2); } void permute_weights() { permute<16>(biases, PackusEpi16Order); permute<16>(weights, PackusEpi16Order); + + if (UseThreats) + permute<8>(threatWeights, PackusEpi16Order); } void unpermute_weights() { permute<16>(biases, InversePackusEpi16Order); permute<16>(weights, InversePackusEpi16Order); + + if (UseThreats) + permute<8>(threatWeights, InversePackusEpi16Order); } inline void scale_weights(bool read) { @@ -145,14 +157,51 @@ class FeatureTransformer { } // Read network parameters + // TODO: This is ugly. Currently LEB128 on the entire L1 necessitates + // reading the weights into a combined array, and then splitting. bool read_parameters(std::istream& stream) { + read_leb_128(stream, biases); - read_leb_128(stream, biases, HalfDimensions); - read_leb_128(stream, weights, HalfDimensions * InputDimensions); - read_leb_128(stream, psqtWeights, PSQTBuckets * InputDimensions); + if (UseThreats) + { + auto combinedWeights = + std::make_unique>(); + auto combinedPsqtWeights = + std::make_unique>(); + + read_leb_128(stream, *combinedWeights); + + std::copy(combinedWeights->begin(), + combinedWeights->begin() + ThreatInputDimensions * HalfDimensions, + std::begin(threatWeights)); + + std::copy(combinedWeights->begin() + ThreatInputDimensions * HalfDimensions, + combinedWeights->begin() + + (ThreatInputDimensions + InputDimensions) * HalfDimensions, + std::begin(weights)); + + read_leb_128(stream, *combinedPsqtWeights); + + std::copy(combinedPsqtWeights->begin(), + combinedPsqtWeights->begin() + ThreatInputDimensions * PSQTBuckets, + std::begin(threatPsqtWeights)); + + std::copy(combinedPsqtWeights->begin() + ThreatInputDimensions * PSQTBuckets, + combinedPsqtWeights->begin() + + (ThreatInputDimensions + InputDimensions) * PSQTBuckets, + std::begin(psqtWeights)); + } + else + { + read_leb_128(stream, weights); + read_leb_128(stream, psqtWeights); + } permute_weights(); - scale_weights(true); + + if (!UseThreats) + scale_weights(true); + return !stream.fail(); } @@ -161,11 +210,44 @@ class FeatureTransformer { std::unique_ptr copy = std::make_unique(*this); copy->unpermute_weights(); - copy->scale_weights(false); - write_leb_128(stream, copy->biases, HalfDimensions); - write_leb_128(stream, copy->weights, HalfDimensions * InputDimensions); - write_leb_128(stream, copy->psqtWeights, PSQTBuckets * InputDimensions); + if (!UseThreats) + copy->scale_weights(false); + + write_leb_128(stream, copy->biases); + + if (UseThreats) + { + auto combinedWeights = + std::make_unique>(); + auto combinedPsqtWeights = + std::make_unique>(); + + std::copy(std::begin(copy->threatWeights), + std::begin(copy->threatWeights) + ThreatInputDimensions * HalfDimensions, + combinedWeights->begin()); + + std::copy(std::begin(copy->weights), + std::begin(copy->weights) + InputDimensions * HalfDimensions, + combinedWeights->begin() + ThreatInputDimensions * HalfDimensions); + + write_leb_128(stream, *combinedWeights); + + std::copy(std::begin(copy->threatPsqtWeights), + std::begin(copy->threatPsqtWeights) + ThreatInputDimensions * PSQTBuckets, + combinedPsqtWeights->begin()); + + std::copy(std::begin(copy->psqtWeights), + std::begin(copy->psqtWeights) + InputDimensions * PSQTBuckets, + combinedPsqtWeights->begin() + ThreatInputDimensions * PSQTBuckets); + + write_leb_128(stream, *combinedPsqtWeights); + } + else + { + write_leb_128(stream, copy->weights); + write_leb_128(stream, copy->psqtWeights); + } return !stream.fail(); } @@ -187,17 +269,29 @@ class FeatureTransformer { int bucket) const { using namespace SIMD; - accumulatorStack.evaluate(pos, *this, *cache); - const auto& accumulatorState = accumulatorStack.latest(); + const auto& accumulatorState = accumulatorStack.latest(); + const auto& threatAccumulatorState = accumulatorStack.latest(); const Color perspectives[2] = {pos.side_to_move(), ~pos.side_to_move()}; const auto& psqtAccumulation = (accumulatorState.acc()).psqtAccumulation; - const auto psqt = - (psqtAccumulation[perspectives[0]][bucket] - psqtAccumulation[perspectives[1]][bucket]) - / 2; + auto psqt = + (psqtAccumulation[perspectives[0]][bucket] - psqtAccumulation[perspectives[1]][bucket]); + + if (UseThreats) + { + const auto& threatPsqtAccumulation = + (threatAccumulatorState.acc()).psqtAccumulation; + psqt = (psqt + threatPsqtAccumulation[perspectives[0]][bucket] + - threatPsqtAccumulation[perspectives[1]][bucket]) + / 2; + } + else + psqt /= 2; const auto& accumulation = (accumulatorState.acc()).accumulation; + const auto& threatAccumulation = + (threatAccumulatorState.acc()).accumulation; for (IndexType p = 0; p < 2; ++p) { @@ -210,7 +304,7 @@ class FeatureTransformer { constexpr IndexType NumOutputChunks = HalfDimensions / 2 / OutputChunkSize; const vec_t Zero = vec_zero(); - const vec_t One = vec_set_16(127 * 2); + const vec_t One = vec_set_16(UseThreats ? 255 : 127 * 2); const vec_t* in0 = reinterpret_cast(&(accumulation[perspectives[p]][0])); const vec_t* in1 = @@ -276,20 +370,48 @@ class FeatureTransformer { #else 6; #endif - - for (IndexType j = 0; j < NumOutputChunks; ++j) + if (UseThreats) { - const vec_t sum0a = - vec_slli_16(vec_max_16(vec_min_16(in0[j * 2 + 0], One), Zero), shift); - const vec_t sum0b = - vec_slli_16(vec_max_16(vec_min_16(in0[j * 2 + 1], One), Zero), shift); - const vec_t sum1a = vec_min_16(in1[j * 2 + 0], One); - const vec_t sum1b = vec_min_16(in1[j * 2 + 1], One); + const vec_t* tin0 = + reinterpret_cast(&(threatAccumulation[perspectives[p]][0])); + const vec_t* tin1 = reinterpret_cast( + &(threatAccumulation[perspectives[p]][HalfDimensions / 2])); + for (IndexType j = 0; j < NumOutputChunks; ++j) + { + const vec_t acc0a = vec_add_16(in0[j * 2 + 0], tin0[j * 2 + 0]); + const vec_t acc0b = vec_add_16(in0[j * 2 + 1], tin0[j * 2 + 1]); + const vec_t acc1a = vec_add_16(in1[j * 2 + 0], tin1[j * 2 + 0]); + const vec_t acc1b = vec_add_16(in1[j * 2 + 1], tin1[j * 2 + 1]); - const vec_t pa = vec_mulhi_16(sum0a, sum1a); - const vec_t pb = vec_mulhi_16(sum0b, sum1b); + const vec_t sum0a = + vec_slli_16(vec_max_16(vec_min_16(acc0a, One), Zero), shift); + const vec_t sum0b = + vec_slli_16(vec_max_16(vec_min_16(acc0b, One), Zero), shift); + const vec_t sum1a = vec_min_16(acc1a, One); + const vec_t sum1b = vec_min_16(acc1b, One); - out[j] = vec_packus_16(pa, pb); + const vec_t pa = vec_mulhi_16(sum0a, sum1a); + const vec_t pb = vec_mulhi_16(sum0b, sum1b); + + out[j] = vec_packus_16(pa, pb); + } + } + else + { + for (IndexType j = 0; j < NumOutputChunks; ++j) + { + const vec_t sum0a = + vec_slli_16(vec_max_16(vec_min_16(in0[j * 2 + 0], One), Zero), shift); + const vec_t sum0b = + vec_slli_16(vec_max_16(vec_min_16(in0[j * 2 + 1], One), Zero), shift); + const vec_t sum1a = vec_min_16(in1[j * 2 + 0], One); + const vec_t sum1b = vec_min_16(in1[j * 2 + 1], One); + + const vec_t pa = vec_mulhi_16(sum0a, sum1a); + const vec_t pb = vec_mulhi_16(sum0b, sum1b); + + out[j] = vec_packus_16(pa, pb); + } } #else @@ -299,8 +421,21 @@ class FeatureTransformer { BiasType sum0 = accumulation[static_cast(perspectives[p])][j + 0]; BiasType sum1 = accumulation[static_cast(perspectives[p])][j + HalfDimensions / 2]; - sum0 = std::clamp(sum0, 0, 127 * 2); - sum1 = std::clamp(sum1, 0, 127 * 2); + + if (UseThreats) + { + BiasType sum0t = threatAccumulation[static_cast(perspectives[p])][j + 0]; + BiasType sum1t = + threatAccumulation[static_cast(perspectives[p])][j + HalfDimensions / 2]; + sum0 = std::clamp(sum0 + sum0t, 0, 255); + sum1 = std::clamp(sum1 + sum1t, 0, 255); + } + else + { + sum0 = std::clamp(sum0, 0, 127 * 2); + sum1 = std::clamp(sum1, 0, 127 * 2); + } + output[offset + j] = static_cast(unsigned(sum0 * sum1) / 512); } @@ -310,9 +445,15 @@ class FeatureTransformer { return psqt; } // end of function transform() - alignas(CacheLineSize) BiasType biases[HalfDimensions]; - alignas(CacheLineSize) WeightType weights[HalfDimensions * InputDimensions]; - alignas(CacheLineSize) PSQTWeightType psqtWeights[InputDimensions * PSQTBuckets]; + alignas(CacheLineSize) std::array biases; + alignas(CacheLineSize) std::array weights; + alignas(CacheLineSize) + std::array threatWeights; + alignas(CacheLineSize) std::array psqtWeights; + alignas(CacheLineSize) + std::array threatPsqtWeights; }; } // namespace Stockfish::Eval::NNUE diff --git a/src/nnue/simd.h b/src/nnue/simd.h index f37eeb930..6160221b9 100644 --- a/src/nnue/simd.h +++ b/src/nnue/simd.h @@ -47,11 +47,13 @@ namespace Stockfish::Eval::NNUE::SIMD { #ifdef USE_AVX512 using vec_t = __m512i; +using vec_i8_t = __m256i; using vec128_t = __m128i; using psqt_vec_t = __m256i; using vec_uint_t = __m512i; #define vec_load(a) _mm512_load_si512(a) #define vec_store(a, b) _mm512_store_si512(a, b) + #define vec_convert_8_16(a) _mm512_cvtepi8_epi16(a) #define vec_add_16(a, b) _mm512_add_epi16(a, b) #define vec_sub_16(a, b) _mm512_sub_epi16(a, b) #define vec_mulhi_16(a, b) _mm512_mulhi_epi16(a, b) @@ -82,11 +84,13 @@ using vec_uint_t = __m512i; #elif USE_AVX2 using vec_t = __m256i; +using vec_i8_t = __m128i; using vec128_t = __m128i; using psqt_vec_t = __m256i; using vec_uint_t = __m256i; #define vec_load(a) _mm256_load_si256(a) #define vec_store(a, b) _mm256_store_si256(a, b) + #define vec_convert_8_16(a) _mm256_cvtepi8_epi16(a) #define vec_add_16(a, b) _mm256_add_epi16(a, b) #define vec_sub_16(a, b) _mm256_sub_epi16(a, b) #define vec_mulhi_16(a, b) _mm256_mulhi_epi16(a, b) @@ -119,11 +123,12 @@ using vec_uint_t = __m256i; #define vec128_storeu(a, b) _mm_storeu_si128(a, b) #define vec128_add(a, b) _mm_add_epi16(a, b) - #define NumRegistersSIMD 16 + #define NumRegistersSIMD 12 #define MaxChunkSize 32 #elif USE_SSE2 using vec_t = __m128i; +using vec_i8_t = std::uint64_t; // for the correct size -- will be loaded into an xmm reg using vec128_t = __m128i; using psqt_vec_t = __m128i; using vec_uint_t = __m128i; @@ -149,17 +154,35 @@ using vec_uint_t = __m128i; _mm_movemask_ps(_mm_castsi128_ps(_mm_cmpgt_epi32(a, _mm_setzero_si128()))) #endif + #ifdef __i386__ +inline __m128i _mm_cvtsi64_si128(int64_t val) { + return _mm_loadl_epi64(reinterpret_cast(&val)); +} + #endif + + #ifdef USE_SSE41 + #define vec_convert_8_16(a) _mm_cvtepi8_epi16(_mm_cvtsi64_si128(static_cast(a))) + #else +// Credit: Yoshie2000 +inline __m128i vec_convert_8_16(uint64_t x) { + __m128i v8 = _mm_cvtsi64_si128(static_cast(x)); + __m128i sign = _mm_cmpgt_epi8(_mm_setzero_si128(), v8); + return _mm_unpacklo_epi8(v8, sign); +} + #endif + #define vec128_zero _mm_setzero_si128() #define vec128_set_16(a) _mm_set1_epi16(a) #define vec128_load(a) _mm_load_si128(a) #define vec128_storeu(a, b) _mm_storeu_si128(a, b) #define vec128_add(a, b) _mm_add_epi16(a, b) - #define NumRegistersSIMD (Is64Bit ? 16 : 8) + #define NumRegistersSIMD (Is64Bit ? 12 : 6) #define MaxChunkSize 16 #elif USE_NEON using vec_t = int16x8_t; +using vec_i8_t = int8x16_t; using psqt_vec_t = int32x4_t; using vec128_t = uint16x8_t; using vec_uint_t = uint32x4_t; @@ -191,6 +214,11 @@ static constexpr std::uint32_t Mask[4] = {1, 2, 4, 8}; #define NumRegistersSIMD 16 #define MaxChunkSize 16 + #ifndef __aarch64__ +// Single instruction doesn't exist on 32-bit ARM +inline int8x16_t vmovl_high_s8(int8x16_t val) { return vmovl_s8(vget_high_s8(val)); } + #endif + #else #undef VECTOR diff --git a/src/position.cpp b/src/position.cpp index 1551eb9df..a515876b6 100644 --- a/src/position.cpp +++ b/src/position.cpp @@ -48,6 +48,7 @@ Key psq[PIECE_NB][SQUARE_NB]; Key enpassant[FILE_NB]; Key castling[CASTLING_RIGHT_NB]; Key side, noPawns; + } namespace { @@ -687,10 +688,10 @@ bool Position::gives_check(Move m) const { // moves should be filtered out before this function is called. // If a pointer to the TT table is passed, the entry for the new position // will be prefetched -DirtyPiece Position::do_move(Move m, - StateInfo& newSt, - bool givesCheck, - const TranspositionTable* tt = nullptr) { +DirtyBoardData Position::do_move(Move m, + StateInfo& newSt, + bool givesCheck, + const TranspositionTable* tt = nullptr) { assert(m.is_ok()); assert(&newSt != st); @@ -724,6 +725,10 @@ DirtyPiece Position::do_move(Move m, dp.from = from; dp.to = to; dp.add_sq = SQ_NONE; + DirtyThreats dts; + dts.us = us; + dts.prevKsq = square(us); + dts.threatenedSqs = dts.threateningSqs = 0; assert(color_of(pc) == us); assert(captured == NO_PIECE || color_of(captured) == (m.type_of() != CASTLING ? them : us)); @@ -735,7 +740,7 @@ DirtyPiece Position::do_move(Move m, assert(captured == make_piece(us, ROOK)); Square rfrom, rto; - do_castling(us, from, to, rfrom, rto, &dp); + do_castling(us, from, to, rfrom, rto, &dts, &dp); k ^= Zobrist::psq[captured][rfrom] ^ Zobrist::psq[captured][rto]; st->nonPawnKey[us] ^= Zobrist::psq[captured][rfrom] ^ Zobrist::psq[captured][rto]; @@ -758,6 +763,9 @@ DirtyPiece Position::do_move(Move m, assert(relative_rank(us, to) == RANK_6); assert(piece_on(to) == NO_PIECE); assert(piece_on(capsq) == make_piece(them, PAWN)); + + // Update board and piece lists in ep case, normal captures are updated later + remove_piece(capsq, &dts); } st->pawnKey ^= Zobrist::psq[captured][capsq]; @@ -774,11 +782,9 @@ DirtyPiece Position::do_move(Move m, dp.remove_pc = captured; dp.remove_sq = capsq; - // Update board and piece lists - remove_piece(capsq); - k ^= Zobrist::psq[captured][capsq]; - st->materialKey ^= Zobrist::psq[captured][8 + pieceCount[captured]]; + st->materialKey ^= + Zobrist::psq[captured][8 + pieceCount[captured] - (m.type_of() != EN_PASSANT)]; // Reset rule 50 counter st->rule50 = 0; @@ -806,7 +812,15 @@ DirtyPiece Position::do_move(Move m, // Move the piece. The tricky Chess960 castling is handled earlier if (m.type_of() != CASTLING) - move_piece(from, to); + { + if (captured && m.type_of() != EN_PASSANT) + { + remove_piece(from, &dts); + swap_piece(to, pc, &dts); + } + else + move_piece(from, to, &dts); + } // If the moving piece is a pawn do some special extra work if (type_of(pc) == PAWN) @@ -823,8 +837,7 @@ DirtyPiece Position::do_move(Move m, assert(relative_rank(us, to) == RANK_8); assert(type_of(promotion) >= KNIGHT && type_of(promotion) <= QUEEN); - remove_piece(to); - put_piece(promotion, to); + swap_piece(to, promotion, &dts); dp.add_pc = promotion; dp.add_sq = to; @@ -949,13 +962,16 @@ DirtyPiece Position::do_move(Move m, } } + dts.ksq = square(us); + assert(pos_is_ok()); assert(dp.pc != NO_PIECE); assert(!(bool(captured) || m.type_of() == CASTLING) ^ (dp.remove_sq != SQ_NONE)); assert(dp.from != SQ_NONE); assert(!(dp.add_sq != SQ_NONE) ^ (m.type_of() == PROMOTION || m.type_of() == CASTLING)); - return dp; + + return {dp, dts}; } @@ -1021,12 +1037,113 @@ void Position::undo_move(Move m) { assert(pos_is_ok()); } +template +inline void add_dirty_threat( + DirtyThreats* const dts, Piece pc, Piece threatened, Square s, Square threatenedSq) { + if (PutPiece) + { + dts->threatenedSqs |= square_bb(threatenedSq); + dts->threateningSqs |= square_bb(s); + } + + dts->list.push_back({pc, threatened, s, threatenedSq, PutPiece}); +} + +template +void Position::update_piece_threats(Piece pc, Square s, DirtyThreats* const dts) { + // Add newly threatened pieces + Bitboard occupied = pieces(); + + Bitboard rAttacks = attacks_bb(s, occupied); + Bitboard bAttacks = attacks_bb(s, occupied); + Bitboard qAttacks = rAttacks | bAttacks; + + Bitboard threatened; + + switch (type_of(pc)) + { + case PAWN : + threatened = PseudoAttacks[color_of(pc)][s]; + break; + case BISHOP : + threatened = bAttacks; + break; + case ROOK : + threatened = rAttacks; + break; + case QUEEN : + threatened = qAttacks; + break; + + default : + threatened = PseudoAttacks[type_of(pc)][s]; + } + + threatened &= occupied; + + while (threatened) + { + Square threatened_sq = pop_lsb(threatened); + Piece threatened_pc = piece_on(threatened_sq); + + assert(threatened_sq != s); + assert(threatened_pc); + + add_dirty_threat(dts, pc, threatened_pc, s, threatened_sq); + } + + Bitboard sliders = (pieces(ROOK, QUEEN) & rAttacks) | (pieces(BISHOP, QUEEN) & bAttacks); + + Bitboard incoming_threats = (attacks_bb(s, occupied) & pieces(KNIGHT)) + | (attacks_bb(s, WHITE) & pieces(BLACK, PAWN)) + | (attacks_bb(s, BLACK) & pieces(WHITE, PAWN)) + | (attacks_bb(s, occupied) & pieces(KING)); + + while (sliders) + { + Square slider_sq = pop_lsb(sliders); + Piece slider = piece_on(slider_sq); + + Bitboard ray = RayPassBB[slider_sq][s] & ~BetweenBB[slider_sq][s]; + threatened = ray & qAttacks & occupied; + + assert(!more_than_one(threatened)); + if (ComputeRay && threatened) + { + Square threatened_sq = lsb(threatened); + + Piece threatened_pc = piece_on(threatened_sq); + add_dirty_threat(dts, slider, threatened_pc, slider_sq, threatened_sq); + } + + add_dirty_threat(dts, slider, pc, slider_sq, s); + } + + // Add threats of sliders that were already threatening s, + // sliders are already handled in the loop above + + while (incoming_threats) + { + Square src_sq = pop_lsb(incoming_threats); + Piece src_pc = piece_on(src_sq); + + assert(src_sq != s); + assert(src_pc != NO_PIECE); + + add_dirty_threat(dts, src_pc, pc, src_sq, s); + } +} // Helper used to do/undo a castling move. This is a bit // tricky in Chess960 where from/to squares can overlap. template -void Position::do_castling( - Color us, Square from, Square& to, Square& rfrom, Square& rto, DirtyPiece* const dp) { +void Position::do_castling(Color us, + Square from, + Square& to, + Square& rfrom, + Square& rto, + DirtyThreats* const dts, + DirtyPiece* const dp) { bool kingSide = to > from; rfrom = to; // Castling is encoded as "king captures friendly rook" @@ -1044,12 +1161,12 @@ void Position::do_castling( } // Remove both pieces first since squares could overlap in Chess960 - remove_piece(Do ? from : to); - remove_piece(Do ? rfrom : rto); + remove_piece(Do ? from : to, dts); + remove_piece(Do ? rfrom : rto, dts); board[Do ? from : to] = board[Do ? rfrom : rto] = NO_PIECE; // remove_piece does not do this for us - put_piece(make_piece(us, KING), Do ? to : from); - put_piece(make_piece(us, ROOK), Do ? rto : rfrom); + put_piece(make_piece(us, KING), Do ? to : from, dts); + put_piece(make_piece(us, ROOK), Do ? rto : rfrom, dts); } @@ -1349,7 +1466,7 @@ bool Position::pos_is_ok() const { for (Piece pc : Pieces) if (pieceCount[pc] != popcount(pieces(color_of(pc), type_of(pc))) - || pieceCount[pc] != std::count(board, board + SQUARE_NB, pc)) + || pieceCount[pc] != std::count(board.begin(), board.end(), pc)) assert(0 && "pos_is_ok: Pieces"); for (Color c : {WHITE, BLACK}) diff --git a/src/position.h b/src/position.h index 579121bf3..9afdb17f9 100644 --- a/src/position.h +++ b/src/position.h @@ -19,6 +19,7 @@ #ifndef POSITION_H_INCLUDED #define POSITION_H_INCLUDED +#include #include #include #include @@ -91,11 +92,11 @@ class Position { Bitboard pieces(PieceTypes... pts) const; Bitboard pieces(Color c) const; template - Bitboard pieces(Color c, PieceTypes... pts) const; - Piece piece_on(Square s) const; - const Piece* piece_array() const; - Square ep_square() const; - bool empty(Square s) const; + Bitboard pieces(Color c, PieceTypes... pts) const; + Piece piece_on(Square s) const; + const std::array& piece_array() const; + Square ep_square() const; + bool empty(Square s) const; template int count(Color c) const; template @@ -132,11 +133,11 @@ class Position { Piece captured_piece() const; // Doing and undoing moves - void do_move(Move m, StateInfo& newSt, const TranspositionTable* tt); - DirtyPiece do_move(Move m, StateInfo& newSt, bool givesCheck, const TranspositionTable* tt); - void undo_move(Move m); - void do_null_move(StateInfo& newSt, const TranspositionTable& tt); - void undo_null_move(); + void do_move(Move m, StateInfo& newSt, const TranspositionTable* tt); + DirtyBoardData do_move(Move m, StateInfo& newSt, bool givesCheck, const TranspositionTable* tt); + void undo_move(Move m); + void do_null_move(StateInfo& newSt, const TranspositionTable& tt); + void undo_null_move(); // Static Exchange Evaluation bool see_ge(Move m, int threshold = 0) const; @@ -166,8 +167,9 @@ class Position { StateInfo* state() const; - void put_piece(Piece pc, Square s); - void remove_piece(Square s); + void put_piece(Piece pc, Square s, DirtyThreats* const dts = nullptr); + void remove_piece(Square s, DirtyThreats* const dts = nullptr); + void swap_piece(Square s, Piece pc, DirtyThreats* const dts = nullptr); private: // Initialization helpers (used while setting up a position) @@ -176,20 +178,24 @@ class Position { void set_check_info() const; // Other helpers - void move_piece(Square from, Square to); + template + void update_piece_threats(Piece pc, Square s, DirtyThreats* const dts); + void move_piece(Square from, Square to, DirtyThreats* const dts = nullptr); template - void do_castling(Color us, - Square from, - Square& to, - Square& rfrom, - Square& rto, - DirtyPiece* const dp = nullptr); + void do_castling(Color us, + Square from, + Square& to, + Square& rfrom, + Square& rto, + DirtyThreats* const dts = nullptr, + DirtyPiece* const dp = nullptr); Key adjust_key50(Key k) const; // Data members - Piece board[SQUARE_NB]; - Bitboard byTypeBB[PIECE_TYPE_NB]; - Bitboard byColorBB[COLOR_NB]; + std::array board; + std::array byTypeBB; + std::array byColorBB; + int pieceCount[PIECE_NB]; int castlingRightsMask[SQUARE_NB]; Square castlingRookSquare[CASTLING_RIGHT_NB]; @@ -209,7 +215,7 @@ inline Piece Position::piece_on(Square s) const { return board[s]; } -inline const Piece* Position::piece_array() const { return board; } +inline const std::array& Position::piece_array() const { return board; } inline bool Position::empty(Square s) const { return piece_on(s) == NO_PIECE; } @@ -326,18 +332,23 @@ inline bool Position::capture_stage(Move m) const { inline Piece Position::captured_piece() const { return st->capturedPiece; } -inline void Position::put_piece(Piece pc, Square s) { - +inline void Position::put_piece(Piece pc, Square s, DirtyThreats* const dts) { board[s] = pc; byTypeBB[ALL_PIECES] |= byTypeBB[type_of(pc)] |= s; byColorBB[color_of(pc)] |= s; pieceCount[pc]++; pieceCount[make_piece(color_of(pc), ALL_PIECES)]++; + + if (dts) + update_piece_threats(pc, s, dts); } -inline void Position::remove_piece(Square s) { - +inline void Position::remove_piece(Square s, DirtyThreats* const dts) { Piece pc = board[s]; + + if (dts) + update_piece_threats(pc, s, dts); + byTypeBB[ALL_PIECES] ^= s; byTypeBB[type_of(pc)] ^= s; byColorBB[color_of(pc)] ^= s; @@ -346,15 +357,35 @@ inline void Position::remove_piece(Square s) { pieceCount[make_piece(color_of(pc), ALL_PIECES)]--; } -inline void Position::move_piece(Square from, Square to) { - +inline void Position::move_piece(Square from, Square to, DirtyThreats* const dts) { Piece pc = board[from]; Bitboard fromTo = from | to; + + if (dts) + update_piece_threats(pc, from, dts); + byTypeBB[ALL_PIECES] ^= fromTo; byTypeBB[type_of(pc)] ^= fromTo; byColorBB[color_of(pc)] ^= fromTo; board[from] = NO_PIECE; board[to] = pc; + + if (dts) + update_piece_threats(pc, to, dts); +} + +inline void Position::swap_piece(Square s, Piece pc, DirtyThreats* const dts) { + Piece old = board[s]; + + remove_piece(s); + + if (dts) + update_piece_threats(old, s, dts); + + put_piece(pc, s); + + if (dts) + update_piece_threats(pc, s, dts); } inline void Position::do_move(Move m, StateInfo& newSt, const TranspositionTable* tt = nullptr) { diff --git a/src/search.cpp b/src/search.cpp index a0c6eafe3..ce7d41008 100644 --- a/src/search.cpp +++ b/src/search.cpp @@ -46,6 +46,7 @@ #include "thread.h" #include "timeman.h" #include "tt.h" +#include "types.h" #include "uci.h" #include "ucioption.h" @@ -527,15 +528,19 @@ void Search::Worker::do_move(Position& pos, const Move move, StateInfo& st, Stac void Search::Worker::do_move( Position& pos, const Move move, StateInfo& st, const bool givesCheck, Stack* const ss) { - bool capture = pos.capture_stage(move); - DirtyPiece dp = pos.do_move(move, st, givesCheck, &tt); + bool capture = pos.capture_stage(move); nodes.fetch_add(1, std::memory_order_relaxed); - accumulatorStack.push(dp); + + DirtyBoardData dirtyBoardData = pos.do_move(move, st, givesCheck, &tt); + accumulatorStack.push(dirtyBoardData); + if (ss != nullptr) { - ss->currentMove = move; - ss->continuationHistory = &continuationHistory[ss->inCheck][capture][dp.pc][move.to_sq()]; - ss->continuationCorrectionHistory = &continuationCorrectionHistory[dp.pc][move.to_sq()]; + ss->currentMove = move; + ss->continuationHistory = + &continuationHistory[ss->inCheck][capture][dirtyBoardData.dp.pc][move.to_sq()]; + ss->continuationCorrectionHistory = + &continuationCorrectionHistory[dirtyBoardData.dp.pc][move.to_sq()]; } } diff --git a/src/types.h b/src/types.h index d40e1e292..46aa16a03 100644 --- a/src/types.h +++ b/src/types.h @@ -40,6 +40,7 @@ #include #include #include + #include "misc.h" #if defined(_MSC_VER) // Disable some silly and noisy warnings from MSVC compiler @@ -290,6 +291,48 @@ struct DirtyPiece { Piece remove_pc, add_pc; }; +// Keep track of what threats change on the board (used by NNUE) +struct DirtyThreat { + DirtyThreat() { /* don't initialize data */ } + DirtyThreat(Piece pc, Piece threatened_pc, Square pc_sq, Square threatened_sq, bool add) { + data = (add << 28) | (pc << 20) | (threatened_pc << 16) | (threatened_sq << 8) | (pc_sq); + } + + Piece pc() const { return static_cast(data >> 20 & 0xf); } + Piece threatened_pc() const { return static_cast(data >> 16 & 0xf); } + Square threatened_sq() const { return static_cast(data >> 8 & 0xff); } + Square pc_sq() const { return static_cast(data & 0xff); } + bool add() const { + uint32_t b = data >> 28; + sf_assume(b == 0 || b == 1); + return b; + } + + private: + uint32_t data; +}; + +using DirtyThreatList = ValueList; + +// A piece can be involved in at most 8 outgoing attacks and 16 incoming attacks. +// Moving a piece also can reveal at most 8 discovered attacks. +// This implies that a non-castling move can change at most (8 + 16) * 3 + 8 = 80 features. +// By similar logic, a castling move can change at most (5 + 1 + 3 + 9) * 2 = 36 features. +// Thus, 80 should work as an upper bound. + +struct DirtyThreats { + DirtyThreatList list; + Color us; + Square prevKsq, ksq; + + Bitboard threatenedSqs, threateningSqs; +}; + +struct DirtyBoardData { + DirtyPiece dp; + DirtyThreats dts; +}; + #define ENABLE_INCR_OPERATORS_ON(T) \ constexpr T& operator++(T& d) { return d = T(int(d) + 1); } \ constexpr T& operator--(T& d) { return d = T(int(d) - 1); }