diff --git a/src/nnue/layers/affine_transform.h b/src/nnue/layers/affine_transform.h index bdf6ff281..055e7ca5f 100644 --- a/src/nnue/layers/affine_transform.h +++ b/src/nnue/layers/affine_transform.h @@ -255,7 +255,7 @@ class AffineTransform { constexpr IndexType NumChunks = ceil_to_multiple(InputDimensions, 8) / 4; constexpr IndexType NumAccums = OutputDimensions / OutputSimdWidth; - #if defined(USE_VNNI) + #if defined(USE_VNNI) || defined(USE_NEON_DOTPROD) constexpr IndexType NumRegs = 2 * NumAccums; #else constexpr IndexType NumRegs = NumAccums; @@ -269,7 +269,7 @@ class AffineTransform { acc[k] = vec_set_32(0); IndexType i = 0; - #if defined(USE_VNNI) + #if defined(USE_VNNI) || defined(USE_NEON_DOTPROD) for (; i < NumChunks; i += 2) { const vec_t in0 = vec_set_32(load_as(input + i * sizeof(i32))); @@ -287,7 +287,11 @@ class AffineTransform { } for (IndexType k = 0; k < NumAccums; ++k) + #if defined(USE_NEON_DOTPROD) + acc[k] = vaddq_s32(acc[k], acc[k + NumAccums]); + #else acc[k] = vec_add_32(acc[k], acc[k + NumAccums]); + #endif #endif for (; i < NumChunks; ++i) { @@ -295,12 +299,12 @@ class AffineTransform { const auto col0 = reinterpret_cast(&weights[i * OutputDimensions * 4]); - for (IndexType k = 0; k < NumRegs; ++k) + for (IndexType k = 0; k < NumAccums; ++k) vec_add_dpbusd_32(acc[k], in0, col0[k]); } vec_t* outptr = reinterpret_cast(output); - for (IndexType k = 0; k < NumRegs; ++k) + for (IndexType k = 0; k < NumAccums; ++k) outptr[k] = acc[k]; #undef vec_set_32