Update nnue-architecture to SFNNv15

Updating nnue to a new architecture including a newly trained net using recipe https://github.com/vondele/nettest/pull/395 and trainer https://github.com/official-stockfish/nnue-pytorch/pull/480.

The recipe contains many small improvements additionally the newly relabled data. Thus this is a combined effort.

Passed STC
https://tests.stockfishchess.org/tests/view/6a429fd8f97ff95f78795110
```
LLR: 2.94 (-2.94,2.94) <0.00,2.00>
Total: 25312 W: 6725 L: 6422 D: 12165
Ptnml(0-2): 78, 2915, 6397, 3158, 108
```

Passed LTC
https://tests.stockfishchess.org/tests/view/6a434c4cf97ff95f78795235
```
LLR: 2.94 (-2.94,2.94) <0.50,2.50>
Total: 32814 W: 8655 L: 8345 D: 15814
Ptnml(0-2): 17, 3469, 9132, 3765, 24
```

closes https://github.com/official-stockfish/Stockfish/pull/6938

Bench: 2639962

Co-authored-by: anematode <anematode@users.noreply.github.com>
Co-authored-by: Joost VandeVondele <vondele@users.noreply.github.com>
Co-authored-by: xu-shawn <xu-shawn@users.noreply.github.com>

src/evaluate.h

@@ -33,7 +33,7 @@ namespace Eval {
 // for the build process (profile-build and fishtest) to work. Do not change the
 // name of the macro or the location where this macro is defined, as it is used
 // in the Makefile/Fishtest.
-#define EvalFileDefaultName "nn-af1339a6dea3.nnue"
+#define EvalFileDefaultName "nn-2ab42c9eeb3e.nnue"
 
 namespace NNUE {
 class Network;

src/nnue/nnue_architecture.h

@@ -42,7 +42,7 @@ using PSQFeatureSet    = Features::HalfKAv2_hm;
 
 // Number of input feature dimensions after conversion
 constexpr IndexType L1 = 1024;
-constexpr int       L2 = 31;
+constexpr int       L2 = 32;
 constexpr int       L3 = 32;
 
 constexpr IndexType PSQTBuckets = 8;
@@ -59,12 +59,13 @@ struct NetworkArchitecture {
     static constexpr int       FC_0_OUTPUTS                 = L2;
     static constexpr int       FC_1_OUTPUTS                 = L3;
 
-    Layers::AffineTransformSparseInput<TransformedFeatureDimensions, FC_0_OUTPUTS + 1> fc_0;
+    Layers::AffineTransformSparseInput<TransformedFeatureDimensions, FC_0_OUTPUTS> fc_0;
-    Layers::SqrClippedReLU<FC_0_OUTPUTS + 1, WeightScaleBits + 1>                      ac_sqr_0;
+    Layers::SqrClippedReLU<FC_0_OUTPUTS, WeightScaleBits + 1>                      ac_sqr_0;
-    Layers::ClippedReLU<FC_0_OUTPUTS + 1, WeightScaleBits + 1>                         ac_0;
+    Layers::ClippedReLU<FC_0_OUTPUTS, WeightScaleBits + 1>                         ac_0;
-    Layers::AffineTransform<FC_0_OUTPUTS * 2, FC_1_OUTPUTS>                            fc_1;
+    Layers::AffineTransform<FC_0_OUTPUTS * 2, FC_1_OUTPUTS>                        fc_1;
-    Layers::ClippedReLU<FC_1_OUTPUTS, WeightScaleBits>                                 ac_1;
+    Layers::SqrClippedReLU<FC_1_OUTPUTS, WeightScaleBits>                          ac_sqr_1;
-    Layers::AffineTransform<FC_1_OUTPUTS, 1>                                           fc_2;
+    Layers::ClippedReLU<FC_1_OUTPUTS, WeightScaleBits>                             ac_1;
+    Layers::AffineTransform<FC_0_OUTPUTS * 2 + FC_1_OUTPUTS * 2, 1>                fc_2;
 
     // Hash value embedded in the evaluation file
     static constexpr u32 get_hash_value() {
@@ -102,34 +103,39 @@ struct NetworkArchitecture {
         struct alignas(CacheLineSize) Buffer {
             alignas(CacheLineSize) typename decltype(fc_0)::OutputBuffer fc_0_out;
             alignas(CacheLineSize) typename decltype(ac_sqr_0)::OutputType
-              ac_sqr_0_out[ceil_to_multiple<IndexType>(FC_0_OUTPUTS * 2, 32)];
+              concat_buffer[ceil_to_multiple<IndexType>(FC_0_OUTPUTS * 2 + FC_1_OUTPUTS * 2, 32)];
             alignas(CacheLineSize) typename decltype(ac_0)::OutputBuffer ac_0_out;
             alignas(CacheLineSize) typename decltype(fc_1)::OutputBuffer fc_1_out;
             alignas(CacheLineSize) typename decltype(ac_1)::OutputBuffer ac_1_out;
             alignas(CacheLineSize) typename decltype(fc_2)::OutputBuffer fc_2_out;
 
-            Buffer() { std::memset(ac_sqr_0_out, 0, sizeof(ac_sqr_0_out)); }
+            Buffer() { std::memset(concat_buffer, 0, sizeof(concat_buffer)); }
         };
 
         Buffer buffer;
 
         fc_0.propagate(transformedFeatures, buffer.fc_0_out, nnzInfo);
-        ac_sqr_0.propagate(buffer.fc_0_out, buffer.ac_sqr_0_out);
+        ac_sqr_0.propagate(buffer.fc_0_out, buffer.concat_buffer);
         ac_0.propagate(buffer.fc_0_out, buffer.ac_0_out);
-        std::memcpy(buffer.ac_sqr_0_out + FC_0_OUTPUTS, buffer.ac_0_out,
+        std::memcpy(buffer.concat_buffer + FC_0_OUTPUTS, buffer.ac_0_out,
                     FC_0_OUTPUTS * sizeof(typename decltype(ac_0)::OutputType));
-        fc_1.propagate(buffer.ac_sqr_0_out, buffer.fc_1_out);
-        ac_1.propagate(buffer.fc_1_out, buffer.ac_1_out);
-        fc_2.propagate(buffer.ac_1_out, buffer.fc_2_out);
 
-        // max value for fwdOut is (L1 + L3) * HiddenMaxVal * WeightMaxVal
+        fc_1.propagate(buffer.concat_buffer, buffer.fc_1_out);
-        // for int8 activations and weights this is (L1 + L3) * 16129 making
+        ac_sqr_1.propagate(buffer.fc_1_out, buffer.concat_buffer + FC_0_OUTPUTS * 2);
-        // fwdOut safe from overflow until (L1 + L3) > 133,144
+        ac_1.propagate(buffer.fc_1_out, buffer.ac_1_out);
-        // first layer and last layer use WeightScaleBits + 1
+        std::memcpy(buffer.concat_buffer + FC_0_OUTPUTS * 2 + FC_1_OUTPUTS, buffer.ac_1_out,
-        i32 fwdOut = buffer.fc_2_out[0] + buffer.fc_0_out[FC_0_OUTPUTS];
+                    FC_1_OUTPUTS * sizeof(typename decltype(ac_1)::OutputType));
+
+        fc_2.propagate(buffer.concat_buffer, buffer.fc_2_out);
+
+        static_assert(FC_0_OUTPUTS >= 2);
+        i32 fwdOut = buffer.fc_2_out[0];
+        i32 skip_0 = buffer.fc_0_out[FC_0_OUTPUTS - 2] - buffer.fc_0_out[FC_0_OUTPUTS - 1];
+        fwdOut += skip_0;
+
         // fwdOut is such that 1.0 is equal to HiddenOneVal*(1<<WeightScaleBits)*2 in
         // quantized form, but we want 1.0 to be equal to 600*OutputScale
-        // to make overflow impossible we cast to i64
+        // to make overflow impossible we cast to int64_t
         constexpr i64 multiplier = 600 * OutputScale;
         constexpr i64 denominator =
           static_cast<i64>(HiddenOneVal) * static_cast<i64>(1U << WeightScaleBits) * 2;
@@ -144,6 +150,7 @@ struct NetworkArchitecture {
         hash_combine(h, ac_sqr_0.get_content_hash());
         hash_combine(h, ac_0.get_content_hash());
         hash_combine(h, fc_1.get_content_hash());
+        // hash_combine(h, ac_sqr_1.get_content_hash()); TODO
         hash_combine(h, ac_1.get_content_hash());
         hash_combine(h, fc_2.get_content_hash());
         hash_combine(h, get_hash_value());