From f614ff90e21e0898008ac8e676d134abd5f7c679 Mon Sep 17 00:00:00 2001 From: Linmiao Xu Date: Thu, 2 Apr 2026 20:22:54 +0200 Subject: [PATCH] Neon dotprod speedup in sparse-input affine transform Also apply VNNI accumulator splitting strategy to NEON dotprod. Speedup measured locally with profile-build, apple-silicon M3 Pro: ``` Result of 20 runs ================== base (...kfish-master) = 1582485 +/- 12985 test (...parse-affine) = 1605204 +/- 13801 diff = +22720 +/- 1212 speedup = +0.0144 P(speedup > 0) = 1.0000 CPU: 11 x arm Hyperthreading: off ``` Passed STC: https://tests.stockfishchess.org/tests/view/69c04c71f690a4b7f5fb0cde LLR: 2.95 (-2.94,2.94) <0.00,2.00> Total: 80576 W: 20748 L: 20391 D: 39437 Ptnml(0-2): 161, 8472, 22658, 8843, 154 closes https://github.com/official-stockfish/Stockfish/pull/6682 No functional change --- src/nnue/layers/affine_transform_sparse_input.h | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/nnue/layers/affine_transform_sparse_input.h b/src/nnue/layers/affine_transform_sparse_input.h index 059a77397..39c5166f3 100644 --- a/src/nnue/layers/affine_transform_sparse_input.h +++ b/src/nnue/layers/affine_transform_sparse_input.h @@ -288,7 +288,7 @@ class AffineTransformSparseInput { // If we're using high-latency dot product instructions, split the accumulators // to create 3 separate dependency chains and merge at the end constexpr IndexType NumRegs = - #if defined(USE_VNNI) + #if defined(USE_VNNI) || defined(USE_NEON_DOTPROD) 3 * NumAccums; #else NumAccums; @@ -309,9 +309,14 @@ class AffineTransformSparseInput { // convince GCC to not do weird pointer arithmetic in the following loop const std::int8_t* weights_cp = weights; - #if defined(USE_VNNI) + #if defined(USE_VNNI) || defined(USE_NEON_DOTPROD) + #if defined(USE_VNNI) for (IndexType k = NumAccums; k < NumRegs; ++k) acc[k] = vec_zero(); + #else + for (IndexType k = NumAccums; k < NumRegs; ++k) + acc[k] = vdupq_n_s32(0); + #endif while (start < end - 2) { @@ -337,8 +342,13 @@ class AffineTransformSparseInput { vec_add_dpbusd_32(acc[k + 2 * NumAccums], in2, col2[k]); } } + #if defined(USE_VNNI) for (IndexType k = 0; k < NumAccums; ++k) acc[k] = vec_add_32(vec_add_32(acc[k], acc[k + NumAccums]), acc[k + 2 * NumAccums]); + #else + for (IndexType k = 0; k < NumAccums; ++k) + acc[k] = vaddq_s32(vaddq_s32(acc[k], acc[k + NumAccums]), acc[k + 2 * NumAccums]); + #endif #endif while (start < end) {