Universal builds for Android armv8 and compatibility with older compilers

- We add a new path for supporting universal builds without #embed . `xxd` causes the compiler to run out of memory, so instead we embed it as a string literal created at compile time.
    - This file is created in `net.sh`, and only if `--embed-dir` isn't supported + we are doing a universal build.
    - This was necessary because Android NDK seems to not support embed, idk why
- A couple other Makefile tweaks were necessary for the Android path.
- While we're at it, remove the non-universal arm64 windows and android binaries.

closes https://github.com/official-stockfish/Stockfish/pull/6827

No functional change

.github/ci/arm_matrix.json

@@ -21,16 +21,10 @@
       "archive_ext": "tar"
     }
   ],
-  "binaries": ["armv8-dotprod", "armv8", "armv7", "armv7-neon"],
+  "binaries": ["armv8-universal", "armv7", "armv7-neon"],
   "exclude": [
     {
-      "binaries": "armv8-dotprod",
+      "binaries": "armv8-universal",
-      "config": {
-        "compiler": "armv7a-linux-androideabi29-clang++"
-      }
-    },
-    {
-      "binaries": "armv8",
       "config": {
         "compiler": "armv7a-linux-androideabi29-clang++"
       }

.github/ci/windows_arm_matrix.json

@@ -1,34 +0,0 @@
-{
-  "include": [
-    {
-      "config": {
-        "name": "Windows 11 Mingw-w64 Clang arm64",
-        "os": "windows-11-arm",
-        "simple_name": "windows",
-        "compiler": "clang++",
-        "comp": "clang",
-        "msys_sys": "clangarm64",
-        "msys_env": "clang-aarch64-clang",
-        "shell": "msys2 {0}",
-        "ext": ".exe",
-        "archive_ext": "zip"
-      },
-      "binaries": "armv8"
-    },
-    {
-      "config": {
-        "name": "Windows 11 Mingw-w64 Clang arm64",
-        "os": "windows-11-arm",
-        "simple_name": "windows",
-        "compiler": "clang++",
-        "comp": "clang",
-        "msys_sys": "clangarm64",
-        "msys_env": "clang-aarch64-clang",
-        "shell": "msys2 {0}",
-        "ext": ".exe",
-        "archive_ext": "zip"
-      },
-      "binaries": "armv8-dotprod"
-    }
-  ]
-}

.github/workflows/arm_compilation.yml

@@ -86,7 +86,7 @@ jobs:
           mv ./stockfish$EXT ../stockfish-android-$BINARY$EXT
 
       - name: Remove non src files
-        run: git clean -fx
+        run: git clean -fxd
 
       - name: Upload artifact for (pre)-release
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

.github/workflows/compilation.yml

@@ -82,7 +82,7 @@ jobs:
           mv ./stockfish$EXT ../stockfish-$NAME-$BINARY$EXT
 
       - name: Remove non src files
-        run: git clean -fx
+        run: git clean -fxd
 
       - name: Upload artifact for (pre)-release
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

.github/workflows/stockfish.yml

@@ -21,7 +21,7 @@ jobs:
     name: Prerelease
     if: github.repository == 'official-stockfish/Stockfish' && (github.ref == 'refs/heads/master' || (startsWith(github.ref_name, 'sf_') && github.ref_type == 'tag'))
     runs-on: ubuntu-latest
-    needs: [Matrix, Compilation, WindowsARMCompilation, ARMCompilation, UniversalCompilation]
+    needs: [Matrix, Compilation, ARMCompilation, UniversalCompilation]
     permissions:
       contents: write # For deleting/creating a prerelease
     steps:
@@ -113,7 +113,6 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       macos_matrix: ${{ steps.set-macos-matrix.outputs.macos_matrix }}
-      windows_arm_matrix: ${{ steps.set-windows-arm-matrix.outputs.windows_arm_matrix }}
       arm_matrix: ${{ steps.set-arm-matrix.outputs.arm_matrix }}
       universal_matrix: ${{ steps.set-universal-matrix.outputs.universal_matrix }}
     steps:
@@ -124,10 +123,6 @@ jobs:
         run: |
           MACOS_MATRIX=$(jq -c '.' .github/ci/macos_matrix.json)
           echo "MACOS_MATRIX=$MACOS_MATRIX" >> $GITHUB_OUTPUT
-      - id: set-windows-arm-matrix
-        run: |
-          WINDOWS_ARM_MATRIX=$(jq -c '.' .github/ci/windows_arm_matrix.json)
-          echo "WINDOWS_ARM_MATRIX=$WINDOWS_ARM_MATRIX" >> $GITHUB_OUTPUT
       - id: set-arm-matrix
         run: |
           ARM_MATRIX=$(jq -c '.' .github/ci/arm_matrix.json)
@@ -167,12 +162,6 @@ jobs:
     uses: ./.github/workflows/compilation.yml
     with:
       matrix: ${{ needs.Matrix.outputs.macos_matrix }}
-  WindowsARMCompilation:
-    name: Windows ARM builds
-    needs: [Matrix, Sanitizers, Tests, Matetrack, Games, CompilerCheck]
-    uses: ./.github/workflows/compilation.yml
-    with:
-      matrix: ${{ needs.Matrix.outputs.windows_arm_matrix }}
   ARMCompilation:
     name: Android builds
     needs: [Matrix, Sanitizers, Tests, Matetrack, Games, CompilerCheck]
@@ -194,17 +183,6 @@ jobs:
       contents: write # For deleting/creating a (pre)release
     secrets:
       token: ${{ secrets.GITHUB_TOKEN }}
-  WindowsARM_Binaries:
-    if: github.repository == 'official-stockfish/Stockfish'
-    name: Windows ARM uploads
-    needs: [Prerelease, Matrix]
-    uses: ./.github/workflows/upload_binaries.yml
-    with:
-      matrix: ${{ needs.Matrix.outputs.windows_arm_matrix }}
-    permissions:
-      contents: write # For deleting/creating a (pre)release
-    secrets:
-      token: ${{ secrets.GITHUB_TOKEN }}
   ARM_Binaries:
     if: github.repository == 'official-stockfish/Stockfish'
     name: Android uploads

.github/workflows/universal_compilation.yml

@@ -45,7 +45,7 @@ jobs:
         run: ./scripts/check_universal.sh ./stockfish-linux-x86-64-universal "$SDE_DIR/sde" "$benchref"
 
       - name: Remove non-src files
-        run: git clean -fx -- src
+        run: git clean -fxd -- src
 
       - name: Upload artifact for (pre)-release
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -102,7 +102,7 @@ jobs:
         run: ./scripts/check_universal.sh ./stockfish-windows-x86-64-universal.exe "$SDE_DIR/sde" "$benchref"
 
       - name: Remove non-src files
-        run: git clean -fx -- src
+        run: git clean -fxd -- src
 
       - name: Upload artifact for (pre)-release
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -155,7 +155,7 @@ jobs:
         run: ./scripts/check_universal_arm.sh ./stockfish-linux-arm64-universal "$benchref"
 
       - name: Remove non-src files
-        run: git clean -fx -- src
+        run: git clean -fxd -- src
 
       - name: Upload artifact for (pre)-release
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -207,7 +207,7 @@ jobs:
         run: mv src/stockfish.exe stockfish-windows-arm64-universal.exe
 
       - name: Remove non-src files
-        run: git clean -fx -- src
+        run: git clean -fxd -- src
 
       - name: Upload artifact for (pre)-release
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

.gitignore

@@ -3,6 +3,7 @@
 **/*.s
 src/.depend
 src/temp_builds
+src/universal/network_dump.inc
 .build_sha.txt
 .build_date.txt
 

scripts/net.sh

@@ -76,3 +76,9 @@ fetch_network() {
 
 fetch_network EvalFileDefaultName
 
+if [ "$1" = "0" ]; then
+    DUMP_FILE=universal/network_dump.inc
+    echo -n '"' > $DUMP_FILE
+    hexdump -v -e '"\\" "x" 1/1 "%02X"' "$(get_nnue_filename EvalFileDefaultName)" >> $DUMP_FILE
+    echo -n '"' >> $DUMP_FILE
+fi

src/Makefile

@@ -372,6 +372,14 @@ ifeq ($(ARCH),armv8-dotprod)
 	arm_version = 8
 endif
 
+ifeq ($(ARCH),armv8-universal)
+	arch = armv8
+	prefetch = yes
+	popcnt = yes
+	neon = yes
+	arm_version = 8
+endif
+
 ifeq ($(ARCH),apple-silicon)
 	arch = arm64
 	prefetch = yes
@@ -589,6 +597,7 @@ ifeq ($(COMP),ndk)
 			STRIP=llvm-strip
 		endif
 	endif
+	OBJCOPY=llvm-objcopy
 	LDFLAGS += -static-libstdc++
 endif
 
@@ -1126,6 +1135,7 @@ profileclean:
 
 universalclean:
 	@rm -rf $(TEMP_DIR)
+	@rm -f universal/network_dump.inc
 
 # evaluation network (nnue)
 net:
@@ -1281,20 +1291,6 @@ UNIVERSAL_EXE        := $(CURDIR)/$(EXE)
 
 ifeq ($(UNIVERSAL_BUILD),true)
 
-# GCC 15+ or Clang 19+ required for #embed
-CXX_MAJOR    := $(shell $(CXX) -dumpversion 2>/dev/null | cut -d. -f1)
-CXX_IS_CLANG := $(shell $(CXX) --version 2>/dev/null | head -1 | grep -ic clang)
-ifeq ($(CXX_IS_CLANG),0)
-  EMBED_MIN  := 15
-  EMBED_NAME := GCC
-else
-  EMBED_MIN  := 19
-  EMBED_NAME := Clang
-endif
-ifneq ($(shell test "$(CXX_MAJOR)" -ge "$(EMBED_MIN)" 2>/dev/null && echo ok),ok)
-$(error Universal build needs $(EMBED_NAME) $(EMBED_MIN)+ for \#embed support; CXX=$(CXX) reports version '$(CXX_MAJOR)')
-endif
-
 UNIVERSAL_SRC_DIR    := $(CURDIR)/universal
 NNUE_EMBED_OBJ       := $(TEMP_DIR)/nnue_embed.o
 NET_SENTINEL         := $(TEMP_DIR)/.net-done
@@ -1324,7 +1320,12 @@ UNIVERSAL_FINAL_FLAGS := -fno-exceptions -Os \
 			 -Wno-c++26-extensions -Wno-c23-extensions # suppress #embed warnings
 
 ifeq ($(KERNEL),Linux)
-  UNIVERSAL_FINAL_FLAGS += -static-libstdc++ -static-libgcc -lrt -lpthread
+  UNIVERSAL_FINAL_FLAGS += -static-libstdc++ -static-libgcc
+  ifeq ($(COMP),ndk)
+    UNIVERSAL_FINAL_FLAGS += -static
+  else
+    UNIVERSAL_FINAL_FLAGS += -lrt -lpthread
+  endif
 else    # windows
   UNIVERSAL_FINAL_FLAGS += -Wl,--allow-multiple-definition -static
   ifeq ($(COMP),clang)
@@ -1346,7 +1347,12 @@ else
   arch-defmain = -Wl,--defsym=main=$(call arch-mangled,$(1))
 endif
 
-arch-cxxflags   = --embed-dir=$(CURDIR) -DStockfish=$(call arch-namespace,$(1)) -DUNIVERSAL_BINARY \
+EMBED_DIR_SUPPORTED := $(shell echo 'int main(){}' | $(CXX) -x c++ --embed-dir=. -o /dev/null -c - 2>/dev/null && echo 1 || echo 0)
+ifeq ($(EMBED_DIR_SUPPORTED),1)
+  EMBED_DIR = --embed-dir=$(CURDIR)
+endif
+
+arch-cxxflags   = $(EMBED_DIR) -DStockfish=$(call arch-namespace,$(1)) -DUNIVERSAL_BINARY \
                   $(call arch-defmain,$(1)) -Wno-c++26-extensions -Wno-c23-extensions
 
 UOBJ_TGT ?= universal-object-pgo
@@ -1363,11 +1369,11 @@ $(TEMP_DIR):
 
 # Download nnue files once in src/ (reused by every sub-arch via --embed-dir).
 $(NET_SENTINEL): | $(TEMP_DIR)
-	@cd $(CURDIR) && $(SHELL) ../scripts/net.sh
+	@cd $(CURDIR) && $(SHELL) ../scripts/net.sh $(EMBED_DIR_SUPPORTED)
 	@touch $@
 
 $(NNUE_EMBED_OBJ): $(UNIVERSAL_SRC_DIR)/nnue_embed.cpp $(NET_SENTINEL) | $(TEMP_DIR)
-	$(CXX) -O2 -Wno-c++26-extensions --embed-dir=$(CURDIR) -c $< -o $@
+	$(CXX) -O2 -Wno-c++26-extensions $(EMBED_DIR) -c $< -o $@
 
 $(UNIVERSAL_ENTRY_OBJ): $(UNIVERSAL_ENTRY_SRC) | $(TEMP_DIR)
 	$(CXX) -O2 -c $< -o $@
@@ -1385,7 +1391,7 @@ $(TEMP_DIR)/%/.setup: | $(TEMP_DIR)
 $(TEMP_DIR)/%/stockfish.o: $(TEMP_DIR)/%/.setup $(NNUE_EMBED_OBJ) $(NET_SENTINEL)
 	+ENV_CXXFLAGS='$(ENV_CXXFLAGS) $(call arch-cxxflags,$*)' \
 	    $(MAKE) -C $(TEMP_DIR)/$* $(UOBJ_TGT) \
-	        ARCH=$* CXX=$(CXX) \
+	        ARCH=$* CXX=$(CXX) COMP=$(COMP) \
 	        NNUE_EMBED_OBJ=$(NNUE_EMBED_OBJ)
 # Drop COMDAT groups; clang LTO leaves them around internalized LOCAL
 # template instantiations which causes problems at the final link

src/nnue/network.cpp

@@ -51,10 +51,18 @@ INCBIN(EmbeddedNNUE, EvalFileDefaultName);
     // separate, non-LTO nnue_embed.o (with strong symbols) can override them during the LTO link,
     // (INCBIN can't deduplicate.)
     #define WEAK_SYM __attribute__((weak))
-extern const unsigned char gEmbeddedNNUEData[] WEAK_SYM = {
+extern const unsigned char gEmbeddedNNUEData[] WEAK_SYM =
+    #ifdef __has_embed
+  {
         #embed EvalFileDefaultName
 };
-extern const unsigned int gEmbeddedNNUESize WEAK_SYM = sizeof(gEmbeddedNNUEData);
+const int padding = 0;
+    #else
+        #include "../universal/network_dump.inc"
+  ;
+const int padding = 1;  // trailing NUL byte
+    #endif
+extern const unsigned int gEmbeddedNNUESize WEAK_SYM = sizeof(gEmbeddedNNUEData) - padding;
 #else
 const unsigned char gEmbeddedNNUEData[1] = {0x0};
 const unsigned int  gEmbeddedNNUESize    = 1;

src/universal/nnue_embed.cpp

@@ -2,7 +2,15 @@
 
 #include "../evaluate.h"
 
-extern const unsigned char gEmbeddedNNUEData[] = {
+extern const unsigned char gEmbeddedNNUEData[] =
+#ifdef __has_embed
+  {
     #embed EvalFileDefaultName
 };
-extern const unsigned int gEmbeddedNNUESize = sizeof(gEmbeddedNNUEData);
+const unsigned int padding = 0;
+#else
+    #include "network_dump.inc"
+  ;
+const unsigned int padding = 1;  // trailing NUL byte
+#endif
+extern const unsigned int gEmbeddedNNUESize = sizeof(gEmbeddedNNUEData) - padding;