mirror of
https://github.com/official-stockfish/Stockfish.git
synced 2026-07-24 21:57:14 +00:00
anematodeandJoost VandeVondele
d5bbc6b67d
Performance on Spacemit K3, thanks @edolnx for testing master: Total time (ms) : 65200 Nodes searched : 3493826 Nodes/second : 53586 riscv-scalable-port: Total time (ms) : 15834 Nodes searched : 3493826 Nodes/second : 220653 Also thanks to @camel-cdr for guidance on RVV programming, and https://cloud-v.co for supplying an RVV instance to test with passed STC: LLR: 2.81 (-2.94,2.94) <0.00,2.00> Total: 1152 W: 527 L: 108 D: 517 Ptnml(0-2): 0, 17, 167, 348, 44 https://tests.stockfishchess.org/tests/view/6a39895b3036e45021aeb368 ## Summary We've had a `riscv64` target for a while, but haven't really optimized for it, in particular the vector extension (RVV). RVV, like SVE, is based on a scalable vector system where the vector length ranges from 128 to 65536. In practice implementations are between 128 and 2048, and 256 bits is quite common (e.g. the Spacemit K3 system above). Unfortunately this doesn't fit well into the rest of our code which assumes a fixed vector length, so what I've done is bypass the `VECTOR` ifdef (which now basically means "FIXED_LENGTH_VECTOR") and just have RVV-specific paths. The ability to explicitly control `vl` makes the code quite readable, in my opinion. We use LMUL>1 in most places to take advantage of multi-vector instructions. Generally the LMULs were chosen to best support a 256-bit vlen, which is very common, but by virtue of how the vlen control works, the code works with any vlen. In a couple places, i.e., `get_changed_pieces` and `AffineTransformSparseInput::propagate`, we have separate implementations depending on the vlen, because the optimal LMUL varies a lot between implementations. One little wrinkle is that `load_as` is compiled to a sequence of byte loads, because although unaligned loads are legal in RVA23, the spec says that they *may* be extremely slow (even though they usually aren't, in actual hw), so compilers are conservative. Thus I aligned the relevant buffers and made the semantics of `load_as` that the operand is aligned, by adding a runtime assertion. ### Universal binary Adding a universal binary is pretty easy and we can just cross-compile. There are two targets: baseline rv64gc and riscv64-rva23, which is actually a smaller subset of RVA23 that also works on some older processors that don't support the full thing. We use clang because GCC, until recently, has a nasty bug with LTO and RVV. Like the universal ARM and x86 builds, we check all the builds in CI. In this case we run bench with multiple vlens, 128 through 1024. In the meantime I deleted the existing broken and unused riscv64 tests. ### Follow-ups - Optimizations - zvdot4a8i path closes https://github.com/official-stockfish/Stockfish/pull/6920 No functional change
779 lines
30 KiB
C++
779 lines
30 KiB
C++
/*
|
|
Stockfish, a UCI chess playing engine derived from Glaurung 2.1
|
|
Copyright (C) 2004-2026 The Stockfish developers (see AUTHORS file)
|
|
|
|
Stockfish is free software: you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
(at your option) any later version.
|
|
|
|
Stockfish is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
#include "nnue_accumulator.h"
|
|
|
|
#include <cassert>
|
|
#include <new>
|
|
|
|
#include "../bitboard.h"
|
|
#include "../misc.h"
|
|
#include "../position.h"
|
|
#include "../types.h"
|
|
#include "nnue_architecture.h"
|
|
#include "nnue_common.h"
|
|
#include "nnue_feature_transformer.h" // IWYU pragma: keep
|
|
#include "simd.h"
|
|
|
|
namespace Stockfish::Eval::NNUE {
|
|
|
|
using namespace SIMD;
|
|
|
|
namespace {
|
|
|
|
template<bool Forward>
|
|
void update_accumulator_incremental(Color perspective,
|
|
const FeatureTransformer& featureTransformer,
|
|
const Square ksq,
|
|
AccumulatorState& target_state,
|
|
const AccumulatorState& computed);
|
|
|
|
void update_accumulator_refresh_cache(Color perspective,
|
|
const FeatureTransformer& featureTransformer,
|
|
const Position& pos,
|
|
AccumulatorState& accumulatorState,
|
|
AccumulatorCaches& cache);
|
|
}
|
|
|
|
const AccumulatorState& AccumulatorStack::latest() const noexcept { return accumulators[size - 1]; }
|
|
|
|
AccumulatorState& AccumulatorStack::mut_latest() noexcept { return accumulators[size - 1]; }
|
|
|
|
void AccumulatorStack::reset() noexcept {
|
|
accumulators[0].dirtyPiece = {};
|
|
new (&accumulators[0].dirtyThreats) DirtyThreats;
|
|
accumulators[0].computed.fill(false);
|
|
size = 1;
|
|
}
|
|
|
|
std::pair<DirtyPiece&, DirtyThreats&> AccumulatorStack::push() noexcept {
|
|
assert(size < MaxSize);
|
|
auto& st = accumulators[size];
|
|
st.computed.fill(false);
|
|
new (&st.dirtyThreats) DirtyThreats;
|
|
size++;
|
|
return {st.dirtyPiece, st.dirtyThreats};
|
|
}
|
|
|
|
void AccumulatorStack::pop() noexcept {
|
|
assert(size > 1);
|
|
size--;
|
|
}
|
|
|
|
void AccumulatorStack::evaluate(const Position& pos,
|
|
const FeatureTransformer& featureTransformer,
|
|
// Silence spurious warning on GCC 10
|
|
[[maybe_unused]] AccumulatorCaches& cache) noexcept {
|
|
evaluate_side(WHITE, pos, featureTransformer, cache);
|
|
evaluate_side(BLACK, pos, featureTransformer, cache);
|
|
}
|
|
|
|
void AccumulatorStack::evaluate_side(Color perspective,
|
|
const Position& pos,
|
|
const FeatureTransformer& featureTransformer,
|
|
AccumulatorCaches& cache) noexcept {
|
|
|
|
const auto last_usable_accum = find_last_usable_accumulator(perspective);
|
|
|
|
if (accumulators[last_usable_accum].computed[perspective])
|
|
forward_update_incremental(perspective, pos, featureTransformer, last_usable_accum);
|
|
|
|
else
|
|
{
|
|
update_accumulator_refresh_cache(perspective, featureTransformer, pos, mut_latest(), cache);
|
|
backward_update_incremental(perspective, pos, featureTransformer, last_usable_accum);
|
|
}
|
|
}
|
|
|
|
// Find the earliest usable accumulator, this can either be a computed accumulator or the accumulator
|
|
// state just before a change that requires full refresh.
|
|
usize AccumulatorStack::find_last_usable_accumulator(Color perspective) const noexcept {
|
|
|
|
for (usize curr_idx = size - 1; curr_idx > 0; curr_idx--)
|
|
{
|
|
if (accumulators[curr_idx].computed[perspective])
|
|
return curr_idx;
|
|
|
|
// Threat feature set refreshes require a king move across the center, i.e.,
|
|
// a subset of halfka refreshes
|
|
if (PSQFeatureSet::requires_refresh(accumulators[curr_idx].dirtyPiece, perspective))
|
|
return curr_idx;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
void AccumulatorStack::forward_update_incremental(Color perspective,
|
|
const Position& pos,
|
|
const FeatureTransformer& featureTransformer,
|
|
const usize begin) noexcept {
|
|
|
|
assert(begin < accumulators.size());
|
|
assert(accumulators[begin].computed[perspective]);
|
|
|
|
const Square ksq = pos.square<KING>(perspective);
|
|
|
|
for (usize next = begin + 1; next < size; next++)
|
|
update_accumulator_incremental<true>(perspective, featureTransformer, ksq,
|
|
accumulators[next], accumulators[next - 1]);
|
|
|
|
assert(latest().computed[perspective]);
|
|
}
|
|
|
|
void AccumulatorStack::backward_update_incremental(Color perspective,
|
|
const Position& pos,
|
|
const FeatureTransformer& featureTransformer,
|
|
const usize end) noexcept {
|
|
|
|
assert(end < accumulators.size());
|
|
assert(end < size);
|
|
assert(latest().computed[perspective]);
|
|
|
|
const Square ksq = pos.square<KING>(perspective);
|
|
|
|
for (i64 next = i64(size) - 2; next >= i64(end); next--)
|
|
update_accumulator_incremental<false>(perspective, featureTransformer, ksq,
|
|
accumulators[next], accumulators[next + 1]);
|
|
|
|
assert(accumulators[end].computed[perspective]);
|
|
}
|
|
|
|
namespace {
|
|
|
|
void apply_combined(Color perspective,
|
|
const FeatureTransformer& featureTransformer,
|
|
const AccumulatorState& from,
|
|
AccumulatorState& to,
|
|
const PSQFeatureSet::IndexList& psqAdded,
|
|
const PSQFeatureSet::IndexList& psqRemoved,
|
|
const ThreatFeatureSet::IndexList& thrAdded,
|
|
const ThreatFeatureSet::IndexList& thrRemoved) {
|
|
constexpr IndexType Dimensions = FeatureTransformer::OutputDimensions;
|
|
|
|
const auto& fromAcc = from.accumulation[perspective];
|
|
auto& toAcc = to.accumulation[perspective];
|
|
|
|
const auto& fromPsqtAcc = from.psqtAccumulation[perspective];
|
|
auto& toPsqtAcc = to.psqtAccumulation[perspective];
|
|
|
|
#ifdef VECTOR
|
|
using Tiling = SIMDTiling<Dimensions, Dimensions, PSQTBuckets>;
|
|
|
|
vec_t acc[Tiling::NumRegs];
|
|
psqt_vec_t psqt[Tiling::NumPsqtRegs];
|
|
|
|
const auto* psqWeights = &featureTransformer.weights[0];
|
|
const auto* threatWeights = &featureTransformer.threatWeights[0];
|
|
|
|
for (IndexType j = 0; j < Dimensions / Tiling::TileHeight; ++j)
|
|
{
|
|
const usize tileOff = j * Tiling::TileHeight;
|
|
auto* fromTile = reinterpret_cast<const vec_t*>(&fromAcc[tileOff]);
|
|
auto* toTile = reinterpret_cast<vec_t*>(&toAcc[tileOff]);
|
|
|
|
for (IndexType k = 0; k < Tiling::NumRegs; ++k)
|
|
acc[k] = fromTile[k];
|
|
|
|
for (int i = 0; i < psqRemoved.ssize(); ++i)
|
|
{
|
|
auto* row =
|
|
reinterpret_cast<const vec_t*>(&psqWeights[psqRemoved[i] * Dimensions + tileOff]);
|
|
for (IndexType k = 0; k < Tiling::NumRegs; ++k)
|
|
acc[k] = vec_sub_16(acc[k], row[k]);
|
|
}
|
|
|
|
for (int i = 0; i < psqAdded.ssize(); ++i)
|
|
{
|
|
auto* row =
|
|
reinterpret_cast<const vec_t*>(&psqWeights[psqAdded[i] * Dimensions + tileOff]);
|
|
for (IndexType k = 0; k < Tiling::NumRegs; ++k)
|
|
acc[k] = vec_add_16(acc[k], row[k]);
|
|
}
|
|
|
|
for (int i = 0; i < thrRemoved.ssize(); ++i)
|
|
{
|
|
auto* column = reinterpret_cast<const vec_i8_t*>(
|
|
&threatWeights[thrRemoved[i] * Dimensions + tileOff]);
|
|
|
|
#ifdef USE_NEON
|
|
for (IndexType k = 0; k < Tiling::NumRegs; k += 2)
|
|
{
|
|
acc[k] = vsubw_s8(acc[k], vget_low_s8(column[k / 2]));
|
|
acc[k + 1] = vsubw_high_s8(acc[k + 1], column[k / 2]);
|
|
}
|
|
#else
|
|
for (IndexType k = 0; k < Tiling::NumRegs; ++k)
|
|
acc[k] = vec_sub_16(acc[k], vec_convert_8_16(column[k]));
|
|
#endif
|
|
}
|
|
|
|
for (int i = 0; i < thrAdded.ssize(); ++i)
|
|
{
|
|
auto* column =
|
|
reinterpret_cast<const vec_i8_t*>(&threatWeights[thrAdded[i] * Dimensions + tileOff]);
|
|
|
|
#ifdef USE_NEON
|
|
for (IndexType k = 0; k < Tiling::NumRegs; k += 2)
|
|
{
|
|
acc[k] = vaddw_s8(acc[k], vget_low_s8(column[k / 2]));
|
|
acc[k + 1] = vaddw_high_s8(acc[k + 1], column[k / 2]);
|
|
}
|
|
#else
|
|
for (IndexType k = 0; k < Tiling::NumRegs; ++k)
|
|
acc[k] = vec_add_16(acc[k], vec_convert_8_16(column[k]));
|
|
#endif
|
|
}
|
|
|
|
for (IndexType k = 0; k < Tiling::NumRegs; k++)
|
|
vec_store(&toTile[k], acc[k]);
|
|
}
|
|
|
|
for (IndexType j = 0; j < PSQTBuckets / Tiling::PsqtTileHeight; ++j)
|
|
{
|
|
const usize psqtTileOff = j * Tiling::PsqtTileHeight;
|
|
auto* fromTilePsqt = reinterpret_cast<const psqt_vec_t*>(&fromPsqtAcc[psqtTileOff]);
|
|
auto* toTilePsqt = reinterpret_cast<psqt_vec_t*>(&toPsqtAcc[psqtTileOff]);
|
|
|
|
for (IndexType k = 0; k < Tiling::NumPsqtRegs; ++k)
|
|
psqt[k] = fromTilePsqt[k];
|
|
|
|
for (int i = 0; i < psqRemoved.ssize(); ++i)
|
|
{
|
|
auto* columnPsqt = reinterpret_cast<const psqt_vec_t*>(
|
|
&featureTransformer.psqtWeights[psqRemoved[i] * PSQTBuckets + psqtTileOff]);
|
|
for (usize k = 0; k < Tiling::NumPsqtRegs; ++k)
|
|
psqt[k] = vec_sub_psqt_32(psqt[k], columnPsqt[k]);
|
|
}
|
|
|
|
for (int i = 0; i < psqAdded.ssize(); ++i)
|
|
{
|
|
auto* columnPsqt = reinterpret_cast<const psqt_vec_t*>(
|
|
&featureTransformer.psqtWeights[psqAdded[i] * PSQTBuckets + psqtTileOff]);
|
|
for (usize k = 0; k < Tiling::NumPsqtRegs; ++k)
|
|
psqt[k] = vec_add_psqt_32(psqt[k], columnPsqt[k]);
|
|
}
|
|
|
|
for (int i = 0; i < thrRemoved.ssize(); ++i)
|
|
{
|
|
auto* columnPsqt = reinterpret_cast<const psqt_vec_t*>(
|
|
&featureTransformer.threatPsqtWeights[thrRemoved[i] * PSQTBuckets + psqtTileOff]);
|
|
for (usize k = 0; k < Tiling::NumPsqtRegs; ++k)
|
|
psqt[k] = vec_sub_psqt_32(psqt[k], columnPsqt[k]);
|
|
}
|
|
|
|
for (int i = 0; i < thrAdded.ssize(); ++i)
|
|
{
|
|
auto* columnPsqt = reinterpret_cast<const psqt_vec_t*>(
|
|
&featureTransformer.threatPsqtWeights[thrAdded[i] * PSQTBuckets + psqtTileOff]);
|
|
for (usize k = 0; k < Tiling::NumPsqtRegs; ++k)
|
|
psqt[k] = vec_add_psqt_32(psqt[k], columnPsqt[k]);
|
|
}
|
|
|
|
for (IndexType k = 0; k < Tiling::NumPsqtRegs; ++k)
|
|
vec_store_psqt(&toTilePsqt[k], psqt[k]);
|
|
}
|
|
|
|
#elif defined(USE_RVV)
|
|
|
|
usize tileOffset = 0;
|
|
|
|
const auto* psqWeights = &featureTransformer.weights[0];
|
|
const auto* threatWeights = &featureTransformer.threatWeights[0];
|
|
const auto* psqtWeights = &featureTransformer.psqtWeights[0];
|
|
const auto* threatPsqtWeights = &featureTransformer.threatPsqtWeights[0];
|
|
|
|
while (tileOffset < Dimensions)
|
|
{
|
|
usize vl = __riscv_vsetvl_e16m8(Dimensions - tileOffset);
|
|
|
|
vint16m8_t accum = __riscv_vle16_v_i16m8(&fromAcc[tileOffset], vl);
|
|
for (int i : psqRemoved)
|
|
accum = __riscv_vsub_vv_i16m8(
|
|
accum, __riscv_vle16_v_i16m8(&psqWeights[i * Dimensions + tileOffset], vl), vl);
|
|
for (int i : psqAdded)
|
|
accum = __riscv_vadd_vv_i16m8(
|
|
accum, __riscv_vle16_v_i16m8(&psqWeights[i * Dimensions + tileOffset], vl), vl);
|
|
for (int i : thrRemoved)
|
|
accum = __riscv_vwsub_wv_i16m8(
|
|
accum, __riscv_vle8_v_i8m4(&threatWeights[i * Dimensions + tileOffset], vl), vl);
|
|
for (int i : thrAdded)
|
|
accum = __riscv_vwadd_wv_i16m8(
|
|
accum, __riscv_vle8_v_i8m4(&threatWeights[i * Dimensions + tileOffset], vl), vl);
|
|
__riscv_vse16_v_i16m8(&toAcc[tileOffset], accum, vl);
|
|
|
|
tileOffset += vl;
|
|
}
|
|
|
|
tileOffset = 0;
|
|
|
|
while (tileOffset < PSQTBuckets)
|
|
{
|
|
usize vl = __riscv_vsetvl_e32m1(PSQTBuckets - tileOffset);
|
|
|
|
vint32m1_t accum = __riscv_vle32_v_i32m1(&fromPsqtAcc[tileOffset], vl);
|
|
for (int i : psqRemoved)
|
|
accum = __riscv_vsub_vv_i32m1(
|
|
accum, __riscv_vle32_v_i32m1(&psqtWeights[i * PSQTBuckets + tileOffset], vl), vl);
|
|
for (int i : psqAdded)
|
|
accum = __riscv_vadd_vv_i32m1(
|
|
accum, __riscv_vle32_v_i32m1(&psqtWeights[i * PSQTBuckets + tileOffset], vl), vl);
|
|
for (int i : thrRemoved)
|
|
accum = __riscv_vsub_vv_i32m1(
|
|
accum, __riscv_vle32_v_i32m1(&threatPsqtWeights[i * PSQTBuckets + tileOffset], vl),
|
|
vl);
|
|
for (int i : thrAdded)
|
|
accum = __riscv_vadd_vv_i32m1(
|
|
accum, __riscv_vle32_v_i32m1(&threatPsqtWeights[i * PSQTBuckets + tileOffset], vl),
|
|
vl);
|
|
|
|
__riscv_vse32_v_i32m1(&toPsqtAcc[tileOffset], accum, vl);
|
|
|
|
tileOffset += vl;
|
|
}
|
|
|
|
#else
|
|
|
|
toAcc = fromAcc;
|
|
toPsqtAcc = fromPsqtAcc;
|
|
|
|
for (const auto index : psqRemoved)
|
|
{
|
|
const IndexType offset = Dimensions * index;
|
|
for (IndexType j = 0; j < Dimensions; ++j)
|
|
toAcc[j] -= featureTransformer.weights[offset + j];
|
|
for (usize k = 0; k < PSQTBuckets; ++k)
|
|
toPsqtAcc[k] -= featureTransformer.psqtWeights[index * PSQTBuckets + k];
|
|
}
|
|
|
|
for (const auto index : psqAdded)
|
|
{
|
|
const IndexType offset = Dimensions * index;
|
|
for (IndexType j = 0; j < Dimensions; ++j)
|
|
toAcc[j] += featureTransformer.weights[offset + j];
|
|
for (usize k = 0; k < PSQTBuckets; ++k)
|
|
toPsqtAcc[k] += featureTransformer.psqtWeights[index * PSQTBuckets + k];
|
|
}
|
|
|
|
for (const auto index : thrRemoved)
|
|
{
|
|
const IndexType offset = Dimensions * index;
|
|
for (IndexType j = 0; j < Dimensions; ++j)
|
|
toAcc[j] -= featureTransformer.threatWeights[offset + j];
|
|
for (usize k = 0; k < PSQTBuckets; ++k)
|
|
toPsqtAcc[k] -= featureTransformer.threatPsqtWeights[index * PSQTBuckets + k];
|
|
}
|
|
|
|
for (const auto index : thrAdded)
|
|
{
|
|
const IndexType offset = Dimensions * index;
|
|
for (IndexType j = 0; j < Dimensions; ++j)
|
|
toAcc[j] += featureTransformer.threatWeights[offset + j];
|
|
for (usize k = 0; k < PSQTBuckets; ++k)
|
|
toPsqtAcc[k] += featureTransformer.threatPsqtWeights[index * PSQTBuckets + k];
|
|
}
|
|
|
|
#endif
|
|
}
|
|
|
|
template<bool Forward>
|
|
void update_accumulator_incremental(Color perspective,
|
|
const FeatureTransformer& featureTransformer,
|
|
const Square ksq,
|
|
AccumulatorState& target_state,
|
|
const AccumulatorState& computed) {
|
|
|
|
assert(computed.computed[perspective]);
|
|
assert(!target_state.computed[perspective]);
|
|
|
|
// The size must be enough to contain the largest possible update.
|
|
// That might depend on the feature set and generally relies on the
|
|
// feature set's update cost calculation to be correct and never allow
|
|
// updates with more added/removed features than MaxActiveDimensions.
|
|
PSQFeatureSet::IndexList psqRemoved, psqAdded;
|
|
ThreatFeatureSet::IndexList thrRemoved, thrAdded;
|
|
|
|
const auto& dirtyPiece = Forward ? target_state.dirtyPiece : computed.dirtyPiece;
|
|
const auto& dirtyThreats = Forward ? target_state.dirtyThreats : computed.dirtyThreats;
|
|
|
|
const auto* pfBase = &featureTransformer.threatWeights[0];
|
|
IndexType pfStride = FeatureTransformer::OutputDimensions;
|
|
|
|
if constexpr (Forward)
|
|
{
|
|
ThreatFeatureSet::append_changed_indices(perspective, ksq, dirtyThreats, thrRemoved,
|
|
thrAdded, pfBase, pfStride);
|
|
PSQFeatureSet::append_changed_indices(perspective, ksq, dirtyPiece, psqRemoved, psqAdded);
|
|
}
|
|
else
|
|
{
|
|
ThreatFeatureSet::append_changed_indices(perspective, ksq, dirtyThreats, thrAdded,
|
|
thrRemoved, pfBase, pfStride);
|
|
PSQFeatureSet::append_changed_indices(perspective, ksq, dirtyPiece, psqAdded, psqRemoved);
|
|
}
|
|
|
|
apply_combined(perspective, featureTransformer, computed, target_state, psqAdded, psqRemoved,
|
|
thrAdded, thrRemoved);
|
|
|
|
target_state.computed[perspective] = true;
|
|
}
|
|
|
|
Bitboard get_changed_pieces(const std::array<Piece, SQUARE_NB>& oldPieces,
|
|
const std::array<Piece, SQUARE_NB>& newPieces) {
|
|
#if defined(USE_AVX2)
|
|
static_assert(sizeof(Piece) == 1);
|
|
Bitboard sameBB = 0;
|
|
|
|
for (int i = 0; i < 64; i += 32)
|
|
{
|
|
const __m256i old_v = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(&oldPieces[i]));
|
|
const __m256i new_v = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(&newPieces[i]));
|
|
const __m256i cmpEqual = _mm256_cmpeq_epi8(old_v, new_v);
|
|
const u32 equalMask = _mm256_movemask_epi8(cmpEqual);
|
|
sameBB |= static_cast<Bitboard>(equalMask) << i;
|
|
}
|
|
return ~sameBB;
|
|
#elif defined(USE_LASX)
|
|
static_assert(sizeof(Piece) == 1);
|
|
|
|
Bitboard changed = 0;
|
|
|
|
for (int i = 0; i < 64; i += 32)
|
|
{
|
|
const __m256i old_v = __lasx_xvld(reinterpret_cast<const void*>(&oldPieces[i]), 0);
|
|
const __m256i new_v = __lasx_xvld(reinterpret_cast<const void*>(&newPieces[i]), 0);
|
|
const __m256i diff = __lasx_xvxor_v(old_v, new_v);
|
|
const __m256i mask = __lasx_xvmsknz_b(diff);
|
|
const auto lo = __lasx_xvpickve2gr_d(mask, 0);
|
|
const auto hi = __lasx_xvpickve2gr_d(mask, 2);
|
|
|
|
changed |= (static_cast<Bitboard>(lo) | (static_cast<Bitboard>(hi) << 16)) << i;
|
|
}
|
|
|
|
return changed;
|
|
#elif defined(USE_LSX)
|
|
static_assert(sizeof(Piece) == 1);
|
|
|
|
Bitboard changed = 0;
|
|
|
|
for (int i = 0; i < 64; i += 16)
|
|
{
|
|
const __m128i old_v = __lsx_vld(reinterpret_cast<const void*>(&oldPieces[i]), 0);
|
|
const __m128i new_v = __lsx_vld(reinterpret_cast<const void*>(&newPieces[i]), 0);
|
|
const __m128i diff = __lsx_vxor_v(old_v, new_v);
|
|
const __m128i mask = __lsx_vmsknz_b(diff);
|
|
|
|
changed |= static_cast<Bitboard>(__lsx_vpickve2gr_d(mask, 0)) << i;
|
|
}
|
|
|
|
return changed;
|
|
#elif defined(USE_NEON)
|
|
uint8x16x4_t old_v = vld4q_u8(reinterpret_cast<const u8*>(oldPieces.data()));
|
|
uint8x16x4_t new_v = vld4q_u8(reinterpret_cast<const u8*>(newPieces.data()));
|
|
auto cmp = [=](const int i) { return vceqq_u8(old_v.val[i], new_v.val[i]); };
|
|
|
|
uint8x16_t cmp0_1 = vsriq_n_u8(cmp(1), cmp(0), 1);
|
|
uint8x16_t cmp2_3 = vsriq_n_u8(cmp(3), cmp(2), 1);
|
|
uint8x16_t merged = vsriq_n_u8(cmp2_3, cmp0_1, 2);
|
|
merged = vsriq_n_u8(merged, merged, 4);
|
|
uint8x8_t sameBB = vshrn_n_u16(vreinterpretq_u16_u8(merged), 4);
|
|
|
|
return ~vget_lane_u64(vreinterpret_u64_u8(sameBB), 0);
|
|
#elif defined(USE_SSE2)
|
|
Bitboard sameBB = 0;
|
|
|
|
for (int i = 0; i < 64; i += 16)
|
|
{
|
|
const __m128i old_v = _mm_loadu_si128(reinterpret_cast<const __m128i*>(&oldPieces[i]));
|
|
const __m128i new_v = _mm_loadu_si128(reinterpret_cast<const __m128i*>(&newPieces[i]));
|
|
const __m128i same = _mm_cmpeq_epi8(old_v, new_v);
|
|
|
|
sameBB |= static_cast<Bitboard>(_mm_movemask_epi8(same)) << i;
|
|
}
|
|
|
|
return ~sameBB;
|
|
#elif defined(USE_RVV)
|
|
|
|
#define IMPL(mx, bx) \
|
|
return __riscv_vmv_x_s_u64m1_u64(__riscv_vreinterpret_v_u8m1_u64m1( \
|
|
__riscv_vreinterpret_v_b##bx##_u8m1(__riscv_vmsne_vv_i8m##mx##_b##bx( \
|
|
__riscv_vle8_v_i8m##mx(reinterpret_cast<const i8*>(oldPieces.data()), 64), \
|
|
__riscv_vle8_v_i8m##mx(reinterpret_cast<const i8*>(newPieces.data()), 64), 64))))
|
|
|
|
|
|
usize vl = __riscv_vsetvlmax_e8m1();
|
|
if (vl >= 64)
|
|
IMPL(1, 8);
|
|
else if (vl == 32)
|
|
IMPL(2, 4);
|
|
else
|
|
IMPL(4, 2);
|
|
|
|
#undef IMPL
|
|
|
|
#else
|
|
Bitboard changed = 0;
|
|
|
|
for (Square sq = SQUARE_ZERO; sq < SQUARE_NB; ++sq)
|
|
changed |= static_cast<Bitboard>(oldPieces[sq] != newPieces[sq]) << sq;
|
|
|
|
return changed;
|
|
#endif
|
|
}
|
|
|
|
// HalfKA data comes from the Finny table entry, while the threats are built
|
|
// from the active threat features
|
|
void update_accumulator_refresh_cache(Color perspective,
|
|
const FeatureTransformer& featureTransformer,
|
|
const Position& pos,
|
|
AccumulatorState& accumulator,
|
|
AccumulatorCaches& cache) {
|
|
constexpr auto Dimensions = FeatureTransformer::OutputDimensions;
|
|
|
|
using Tiling [[maybe_unused]] = SIMDTiling<Dimensions, Dimensions, PSQTBuckets>;
|
|
|
|
const Square ksq = pos.square<KING>(perspective);
|
|
auto& entry = cache[ksq][perspective];
|
|
PSQFeatureSet::IndexList removed, added;
|
|
|
|
const Bitboard changedBB = get_changed_pieces(entry.pieces, pos.piece_array());
|
|
Bitboard removedBB = changedBB & entry.pieceBB;
|
|
Bitboard addedBB = changedBB & pos.pieces();
|
|
|
|
#if defined(USE_AVX512ICL)
|
|
PSQFeatureSet::write_indices(entry.pieces, pos.piece_array(), removedBB, addedBB, perspective,
|
|
ksq, removed, added);
|
|
#else
|
|
while (removedBB)
|
|
{
|
|
Square sq = pop_lsb(removedBB);
|
|
removed.push_back(PSQFeatureSet::make_index(perspective, sq, entry.pieces[sq], ksq));
|
|
}
|
|
while (addedBB)
|
|
{
|
|
Square sq = pop_lsb(addedBB);
|
|
added.push_back(PSQFeatureSet::make_index(perspective, sq, pos.piece_on(sq), ksq));
|
|
}
|
|
#endif
|
|
|
|
entry.pieceBB = pos.pieces();
|
|
entry.pieces = pos.piece_array();
|
|
|
|
ThreatFeatureSet::IndexList active;
|
|
ThreatFeatureSet::append_active_indices(perspective, pos, active);
|
|
|
|
accumulator.computed[perspective] = true;
|
|
|
|
#ifdef VECTOR
|
|
vec_t acc[Tiling::NumRegs];
|
|
psqt_vec_t psqt[Tiling::NumPsqtRegs];
|
|
|
|
const auto* weights = &featureTransformer.weights[0];
|
|
const auto* threatWeights = &featureTransformer.threatWeights[0];
|
|
|
|
for (IndexType j = 0; j < Dimensions / Tiling::TileHeight; ++j)
|
|
{
|
|
const usize tileOff = j * Tiling::TileHeight;
|
|
auto* accTile = reinterpret_cast<vec_t*>(&accumulator.accumulation[perspective][tileOff]);
|
|
auto* entryTile = reinterpret_cast<vec_t*>(&entry.accumulation[tileOff]);
|
|
|
|
for (IndexType k = 0; k < Tiling::NumRegs; ++k)
|
|
acc[k] = entryTile[k];
|
|
|
|
for (int i = 0; i < removed.ssize(); ++i)
|
|
{
|
|
auto* column =
|
|
reinterpret_cast<const vec_t*>(&weights[removed[i] * Dimensions + tileOff]);
|
|
for (IndexType k = 0; k < Tiling::NumRegs; ++k)
|
|
acc[k] = vec_sub_16(acc[k], column[k]);
|
|
}
|
|
for (int i = 0; i < added.ssize(); ++i)
|
|
{
|
|
auto* column =
|
|
reinterpret_cast<const vec_t*>(&weights[added[i] * Dimensions + tileOff]);
|
|
for (IndexType k = 0; k < Tiling::NumRegs; ++k)
|
|
acc[k] = vec_add_16(acc[k], column[k]);
|
|
}
|
|
|
|
for (IndexType k = 0; k < Tiling::NumRegs; k++)
|
|
vec_store(&entryTile[k], acc[k]);
|
|
|
|
for (int i = 0; i < active.ssize(); ++i)
|
|
{
|
|
auto* column =
|
|
reinterpret_cast<const vec_i8_t*>(&threatWeights[active[i] * Dimensions + tileOff]);
|
|
|
|
#ifdef USE_NEON
|
|
for (IndexType k = 0; k < Tiling::NumRegs; k += 2)
|
|
{
|
|
acc[k] = vaddw_s8(acc[k], vget_low_s8(column[k / 2]));
|
|
acc[k + 1] = vaddw_high_s8(acc[k + 1], column[k / 2]);
|
|
}
|
|
#else
|
|
for (IndexType k = 0; k < Tiling::NumRegs; ++k)
|
|
acc[k] = vec_add_16(acc[k], vec_convert_8_16(column[k]));
|
|
#endif
|
|
}
|
|
|
|
for (IndexType k = 0; k < Tiling::NumRegs; k++)
|
|
vec_store(&accTile[k], acc[k]);
|
|
}
|
|
|
|
for (IndexType j = 0; j < PSQTBuckets / Tiling::PsqtTileHeight; ++j)
|
|
{
|
|
const usize psqtTileOff = j * Tiling::PsqtTileHeight;
|
|
auto* accTilePsqt =
|
|
reinterpret_cast<psqt_vec_t*>(&accumulator.psqtAccumulation[perspective][psqtTileOff]);
|
|
auto* entryTilePsqt = reinterpret_cast<psqt_vec_t*>(&entry.psqtAccumulation[psqtTileOff]);
|
|
|
|
for (IndexType k = 0; k < Tiling::NumPsqtRegs; ++k)
|
|
psqt[k] = entryTilePsqt[k];
|
|
|
|
for (int i = 0; i < removed.ssize(); ++i)
|
|
{
|
|
auto* columnPsqt = reinterpret_cast<const psqt_vec_t*>(
|
|
&featureTransformer.psqtWeights[removed[i] * PSQTBuckets + psqtTileOff]);
|
|
for (usize k = 0; k < Tiling::NumPsqtRegs; ++k)
|
|
psqt[k] = vec_sub_psqt_32(psqt[k], columnPsqt[k]);
|
|
}
|
|
for (int i = 0; i < added.ssize(); ++i)
|
|
{
|
|
auto* columnPsqt = reinterpret_cast<const psqt_vec_t*>(
|
|
&featureTransformer.psqtWeights[added[i] * PSQTBuckets + psqtTileOff]);
|
|
for (usize k = 0; k < Tiling::NumPsqtRegs; ++k)
|
|
psqt[k] = vec_add_psqt_32(psqt[k], columnPsqt[k]);
|
|
}
|
|
|
|
for (IndexType k = 0; k < Tiling::NumPsqtRegs; ++k)
|
|
vec_store_psqt(&entryTilePsqt[k], psqt[k]);
|
|
|
|
for (int i = 0; i < active.ssize(); ++i)
|
|
{
|
|
auto* columnPsqt = reinterpret_cast<const psqt_vec_t*>(
|
|
&featureTransformer.threatPsqtWeights[active[i] * PSQTBuckets + psqtTileOff]);
|
|
for (usize k = 0; k < Tiling::NumPsqtRegs; ++k)
|
|
psqt[k] = vec_add_psqt_32(psqt[k], columnPsqt[k]);
|
|
}
|
|
|
|
for (IndexType k = 0; k < Tiling::NumPsqtRegs; ++k)
|
|
vec_store_psqt(&accTilePsqt[k], psqt[k]);
|
|
}
|
|
|
|
#elif defined(USE_RVV)
|
|
|
|
const auto* weights = &featureTransformer.weights[0];
|
|
const auto* threatWeights = &featureTransformer.threatWeights[0];
|
|
const auto* psqtWeights = &featureTransformer.psqtWeights[0];
|
|
const auto* threatPsqtWeights = &featureTransformer.threatPsqtWeights[0];
|
|
|
|
usize tileOffset = 0;
|
|
|
|
while (tileOffset < Dimensions)
|
|
{
|
|
usize vl = __riscv_vsetvl_e16m8(Dimensions - tileOffset);
|
|
|
|
vint16m8_t accum = __riscv_vle16_v_i16m8(&entry.accumulation[tileOffset], vl);
|
|
for (int i : removed)
|
|
accum = __riscv_vsub_vv_i16m8(
|
|
accum, __riscv_vle16_v_i16m8(&weights[i * Dimensions + tileOffset], vl), vl);
|
|
for (int i : added)
|
|
accum = __riscv_vadd_vv_i16m8(
|
|
accum, __riscv_vle16_v_i16m8(&weights[i * Dimensions + tileOffset], vl), vl);
|
|
|
|
__riscv_vse16_v_i16m8(&entry.accumulation[tileOffset], accum, vl);
|
|
|
|
for (int i : active)
|
|
accum = __riscv_vwadd_wv_i16m8(
|
|
accum, __riscv_vle8_v_i8m4(&threatWeights[i * Dimensions + tileOffset], vl), vl);
|
|
|
|
__riscv_vse16_v_i16m8(&accumulator.accumulation[perspective][tileOffset], accum, vl);
|
|
|
|
tileOffset += vl;
|
|
}
|
|
|
|
tileOffset = 0;
|
|
|
|
while (tileOffset < PSQTBuckets)
|
|
{
|
|
usize vl = __riscv_vsetvl_e32m1(PSQTBuckets - tileOffset);
|
|
|
|
vint32m1_t accum = __riscv_vle32_v_i32m1(&entry.psqtAccumulation[tileOffset], vl);
|
|
for (int i : removed)
|
|
accum = __riscv_vsub_vv_i32m1(
|
|
accum, __riscv_vle32_v_i32m1(&psqtWeights[i * PSQTBuckets + tileOffset], vl), vl);
|
|
for (int i : added)
|
|
accum = __riscv_vadd_vv_i32m1(
|
|
accum, __riscv_vle32_v_i32m1(&psqtWeights[i * PSQTBuckets + tileOffset], vl), vl);
|
|
|
|
__riscv_vse32_v_i32m1(&entry.psqtAccumulation[tileOffset], accum, vl);
|
|
|
|
for (int i : active)
|
|
accum = __riscv_vadd_vv_i32m1(
|
|
accum, __riscv_vle32_v_i32m1(&threatPsqtWeights[i * PSQTBuckets + tileOffset], vl),
|
|
vl);
|
|
|
|
__riscv_vse32_v_i32m1(&accumulator.psqtAccumulation[perspective][tileOffset], accum, vl);
|
|
|
|
tileOffset += vl;
|
|
}
|
|
|
|
#else
|
|
|
|
for (const auto index : removed)
|
|
{
|
|
const IndexType offset = Dimensions * index;
|
|
for (IndexType j = 0; j < Dimensions; ++j)
|
|
entry.accumulation[j] -= featureTransformer.weights[offset + j];
|
|
|
|
for (usize k = 0; k < PSQTBuckets; ++k)
|
|
entry.psqtAccumulation[k] -= featureTransformer.psqtWeights[index * PSQTBuckets + k];
|
|
}
|
|
for (const auto index : added)
|
|
{
|
|
const IndexType offset = Dimensions * index;
|
|
for (IndexType j = 0; j < Dimensions; ++j)
|
|
entry.accumulation[j] += featureTransformer.weights[offset + j];
|
|
|
|
for (usize k = 0; k < PSQTBuckets; ++k)
|
|
entry.psqtAccumulation[k] += featureTransformer.psqtWeights[index * PSQTBuckets + k];
|
|
}
|
|
|
|
// The accumulator of the refresh entry has been updated.
|
|
// Now copy its content to the actual accumulator we were refreshing.
|
|
accumulator.accumulation[perspective] = entry.accumulation;
|
|
accumulator.psqtAccumulation[perspective] = entry.psqtAccumulation;
|
|
|
|
for (const auto index : active)
|
|
{
|
|
const IndexType offset = Dimensions * index;
|
|
|
|
for (IndexType j = 0; j < Dimensions; ++j)
|
|
accumulator.accumulation[perspective][j] +=
|
|
featureTransformer.threatWeights[offset + j];
|
|
|
|
for (usize k = 0; k < PSQTBuckets; ++k)
|
|
accumulator.psqtAccumulation[perspective][k] +=
|
|
featureTransformer.threatPsqtWeights[index * PSQTBuckets + k];
|
|
}
|
|
|
|
#endif
|
|
}
|
|
|
|
}
|
|
|
|
}
|