From 24abab9f6c517ebf02b232e2adf110f2b62355c4 Mon Sep 17 00:00:00 2001 From: anematode Date: Tue, 19 May 2026 18:39:06 +0200 Subject: [PATCH] Universal builds for Android armv8 and compatibility with older compilers - We add a new path for supporting universal builds without #embed . `xxd` causes the compiler to run out of memory, so instead we embed it as a string literal created at compile time. - This file is created in `net.sh`, and only if `--embed-dir` isn't supported + we are doing a universal build. - This was necessary because Android NDK seems to not support embed, idk why - A couple other Makefile tweaks were necessary for the Android path. - While we're at it, remove the non-universal arm64 windows and android binaries. closes https://github.com/official-stockfish/Stockfish/pull/6827 No functional change --- .github/ci/arm_matrix.json | 10 +---- .github/ci/windows_arm_matrix.json | 34 ---------------- .github/workflows/arm_compilation.yml | 2 +- .github/workflows/compilation.yml | 2 +- .github/workflows/stockfish.yml | 24 +---------- .github/workflows/universal_compilation.yml | 8 ++-- .gitignore | 1 + scripts/net.sh | 6 +++ src/Makefile | 44 ++++++++++++--------- src/nnue/network.cpp | 14 +++++-- src/universal/nnue_embed.cpp | 14 +++++-- 11 files changed, 63 insertions(+), 96 deletions(-) delete mode 100644 .github/ci/windows_arm_matrix.json diff --git a/.github/ci/arm_matrix.json b/.github/ci/arm_matrix.json index b53fe03af..0661e1b29 100644 --- a/.github/ci/arm_matrix.json +++ b/.github/ci/arm_matrix.json @@ -21,16 +21,10 @@ "archive_ext": "tar" } ], - "binaries": ["armv8-dotprod", "armv8", "armv7", "armv7-neon"], + "binaries": ["armv8-universal", "armv7", "armv7-neon"], "exclude": [ { - "binaries": "armv8-dotprod", - "config": { - "compiler": "armv7a-linux-androideabi29-clang++" - } - }, - { - "binaries": "armv8", + "binaries": "armv8-universal", "config": { "compiler": "armv7a-linux-androideabi29-clang++" } diff --git a/.github/ci/windows_arm_matrix.json b/.github/ci/windows_arm_matrix.json deleted file mode 100644 index 42f9baefa..000000000 --- a/.github/ci/windows_arm_matrix.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "include": [ - { - "config": { - "name": "Windows 11 Mingw-w64 Clang arm64", - "os": "windows-11-arm", - "simple_name": "windows", - "compiler": "clang++", - "comp": "clang", - "msys_sys": "clangarm64", - "msys_env": "clang-aarch64-clang", - "shell": "msys2 {0}", - "ext": ".exe", - "archive_ext": "zip" - }, - "binaries": "armv8" - }, - { - "config": { - "name": "Windows 11 Mingw-w64 Clang arm64", - "os": "windows-11-arm", - "simple_name": "windows", - "compiler": "clang++", - "comp": "clang", - "msys_sys": "clangarm64", - "msys_env": "clang-aarch64-clang", - "shell": "msys2 {0}", - "ext": ".exe", - "archive_ext": "zip" - }, - "binaries": "armv8-dotprod" - } - ] -} diff --git a/.github/workflows/arm_compilation.yml b/.github/workflows/arm_compilation.yml index 0baf739db..ea9ffe053 100644 --- a/.github/workflows/arm_compilation.yml +++ b/.github/workflows/arm_compilation.yml @@ -86,7 +86,7 @@ jobs: mv ./stockfish$EXT ../stockfish-android-$BINARY$EXT - name: Remove non src files - run: git clean -fx + run: git clean -fxd - name: Upload artifact for (pre)-release uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/compilation.yml b/.github/workflows/compilation.yml index 2eb2db21e..ef5ecb927 100644 --- a/.github/workflows/compilation.yml +++ b/.github/workflows/compilation.yml @@ -82,7 +82,7 @@ jobs: mv ./stockfish$EXT ../stockfish-$NAME-$BINARY$EXT - name: Remove non src files - run: git clean -fx + run: git clean -fxd - name: Upload artifact for (pre)-release uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/stockfish.yml b/.github/workflows/stockfish.yml index 148333337..0065fd728 100644 --- a/.github/workflows/stockfish.yml +++ b/.github/workflows/stockfish.yml @@ -21,7 +21,7 @@ jobs: name: Prerelease if: github.repository == 'official-stockfish/Stockfish' && (github.ref == 'refs/heads/master' || (startsWith(github.ref_name, 'sf_') && github.ref_type == 'tag')) runs-on: ubuntu-latest - needs: [Matrix, Compilation, WindowsARMCompilation, ARMCompilation, UniversalCompilation] + needs: [Matrix, Compilation, ARMCompilation, UniversalCompilation] permissions: contents: write # For deleting/creating a prerelease steps: @@ -113,7 +113,6 @@ jobs: runs-on: ubuntu-latest outputs: macos_matrix: ${{ steps.set-macos-matrix.outputs.macos_matrix }} - windows_arm_matrix: ${{ steps.set-windows-arm-matrix.outputs.windows_arm_matrix }} arm_matrix: ${{ steps.set-arm-matrix.outputs.arm_matrix }} universal_matrix: ${{ steps.set-universal-matrix.outputs.universal_matrix }} steps: @@ -124,10 +123,6 @@ jobs: run: | MACOS_MATRIX=$(jq -c '.' .github/ci/macos_matrix.json) echo "MACOS_MATRIX=$MACOS_MATRIX" >> $GITHUB_OUTPUT - - id: set-windows-arm-matrix - run: | - WINDOWS_ARM_MATRIX=$(jq -c '.' .github/ci/windows_arm_matrix.json) - echo "WINDOWS_ARM_MATRIX=$WINDOWS_ARM_MATRIX" >> $GITHUB_OUTPUT - id: set-arm-matrix run: | ARM_MATRIX=$(jq -c '.' .github/ci/arm_matrix.json) @@ -167,12 +162,6 @@ jobs: uses: ./.github/workflows/compilation.yml with: matrix: ${{ needs.Matrix.outputs.macos_matrix }} - WindowsARMCompilation: - name: Windows ARM builds - needs: [Matrix, Sanitizers, Tests, Matetrack, Games, CompilerCheck] - uses: ./.github/workflows/compilation.yml - with: - matrix: ${{ needs.Matrix.outputs.windows_arm_matrix }} ARMCompilation: name: Android builds needs: [Matrix, Sanitizers, Tests, Matetrack, Games, CompilerCheck] @@ -194,17 +183,6 @@ jobs: contents: write # For deleting/creating a (pre)release secrets: token: ${{ secrets.GITHUB_TOKEN }} - WindowsARM_Binaries: - if: github.repository == 'official-stockfish/Stockfish' - name: Windows ARM uploads - needs: [Prerelease, Matrix] - uses: ./.github/workflows/upload_binaries.yml - with: - matrix: ${{ needs.Matrix.outputs.windows_arm_matrix }} - permissions: - contents: write # For deleting/creating a (pre)release - secrets: - token: ${{ secrets.GITHUB_TOKEN }} ARM_Binaries: if: github.repository == 'official-stockfish/Stockfish' name: Android uploads diff --git a/.github/workflows/universal_compilation.yml b/.github/workflows/universal_compilation.yml index 86eaf49bf..9a32b8b3c 100644 --- a/.github/workflows/universal_compilation.yml +++ b/.github/workflows/universal_compilation.yml @@ -45,7 +45,7 @@ jobs: run: ./scripts/check_universal.sh ./stockfish-linux-x86-64-universal "$SDE_DIR/sde" "$benchref" - name: Remove non-src files - run: git clean -fx -- src + run: git clean -fxd -- src - name: Upload artifact for (pre)-release uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -102,7 +102,7 @@ jobs: run: ./scripts/check_universal.sh ./stockfish-windows-x86-64-universal.exe "$SDE_DIR/sde" "$benchref" - name: Remove non-src files - run: git clean -fx -- src + run: git clean -fxd -- src - name: Upload artifact for (pre)-release uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -155,7 +155,7 @@ jobs: run: ./scripts/check_universal_arm.sh ./stockfish-linux-arm64-universal "$benchref" - name: Remove non-src files - run: git clean -fx -- src + run: git clean -fxd -- src - name: Upload artifact for (pre)-release uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -207,7 +207,7 @@ jobs: run: mv src/stockfish.exe stockfish-windows-arm64-universal.exe - name: Remove non-src files - run: git clean -fx -- src + run: git clean -fxd -- src - name: Upload artifact for (pre)-release uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.gitignore b/.gitignore index 7ed725ed8..3e136b625 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ **/*.s src/.depend src/temp_builds +src/universal/network_dump.inc .build_sha.txt .build_date.txt diff --git a/scripts/net.sh b/scripts/net.sh index d14e2be46..222073b8e 100755 --- a/scripts/net.sh +++ b/scripts/net.sh @@ -76,3 +76,9 @@ fetch_network() { fetch_network EvalFileDefaultName +if [ "$1" = "0" ]; then + DUMP_FILE=universal/network_dump.inc + echo -n '"' > $DUMP_FILE + hexdump -v -e '"\\" "x" 1/1 "%02X"' "$(get_nnue_filename EvalFileDefaultName)" >> $DUMP_FILE + echo -n '"' >> $DUMP_FILE +fi diff --git a/src/Makefile b/src/Makefile index 2809b248a..1cd017b18 100644 --- a/src/Makefile +++ b/src/Makefile @@ -372,6 +372,14 @@ ifeq ($(ARCH),armv8-dotprod) arm_version = 8 endif +ifeq ($(ARCH),armv8-universal) + arch = armv8 + prefetch = yes + popcnt = yes + neon = yes + arm_version = 8 +endif + ifeq ($(ARCH),apple-silicon) arch = arm64 prefetch = yes @@ -589,6 +597,7 @@ ifeq ($(COMP),ndk) STRIP=llvm-strip endif endif + OBJCOPY=llvm-objcopy LDFLAGS += -static-libstdc++ endif @@ -1126,6 +1135,7 @@ profileclean: universalclean: @rm -rf $(TEMP_DIR) + @rm -f universal/network_dump.inc # evaluation network (nnue) net: @@ -1281,20 +1291,6 @@ UNIVERSAL_EXE := $(CURDIR)/$(EXE) ifeq ($(UNIVERSAL_BUILD),true) -# GCC 15+ or Clang 19+ required for #embed -CXX_MAJOR := $(shell $(CXX) -dumpversion 2>/dev/null | cut -d. -f1) -CXX_IS_CLANG := $(shell $(CXX) --version 2>/dev/null | head -1 | grep -ic clang) -ifeq ($(CXX_IS_CLANG),0) - EMBED_MIN := 15 - EMBED_NAME := GCC -else - EMBED_MIN := 19 - EMBED_NAME := Clang -endif -ifneq ($(shell test "$(CXX_MAJOR)" -ge "$(EMBED_MIN)" 2>/dev/null && echo ok),ok) -$(error Universal build needs $(EMBED_NAME) $(EMBED_MIN)+ for \#embed support; CXX=$(CXX) reports version '$(CXX_MAJOR)') -endif - UNIVERSAL_SRC_DIR := $(CURDIR)/universal NNUE_EMBED_OBJ := $(TEMP_DIR)/nnue_embed.o NET_SENTINEL := $(TEMP_DIR)/.net-done @@ -1324,7 +1320,12 @@ UNIVERSAL_FINAL_FLAGS := -fno-exceptions -Os \ -Wno-c++26-extensions -Wno-c23-extensions # suppress #embed warnings ifeq ($(KERNEL),Linux) - UNIVERSAL_FINAL_FLAGS += -static-libstdc++ -static-libgcc -lrt -lpthread + UNIVERSAL_FINAL_FLAGS += -static-libstdc++ -static-libgcc + ifeq ($(COMP),ndk) + UNIVERSAL_FINAL_FLAGS += -static + else + UNIVERSAL_FINAL_FLAGS += -lrt -lpthread + endif else # windows UNIVERSAL_FINAL_FLAGS += -Wl,--allow-multiple-definition -static ifeq ($(COMP),clang) @@ -1346,7 +1347,12 @@ else arch-defmain = -Wl,--defsym=main=$(call arch-mangled,$(1)) endif -arch-cxxflags = --embed-dir=$(CURDIR) -DStockfish=$(call arch-namespace,$(1)) -DUNIVERSAL_BINARY \ +EMBED_DIR_SUPPORTED := $(shell echo 'int main(){}' | $(CXX) -x c++ --embed-dir=. -o /dev/null -c - 2>/dev/null && echo 1 || echo 0) +ifeq ($(EMBED_DIR_SUPPORTED),1) + EMBED_DIR = --embed-dir=$(CURDIR) +endif + +arch-cxxflags = $(EMBED_DIR) -DStockfish=$(call arch-namespace,$(1)) -DUNIVERSAL_BINARY \ $(call arch-defmain,$(1)) -Wno-c++26-extensions -Wno-c23-extensions UOBJ_TGT ?= universal-object-pgo @@ -1363,11 +1369,11 @@ $(TEMP_DIR): # Download nnue files once in src/ (reused by every sub-arch via --embed-dir). $(NET_SENTINEL): | $(TEMP_DIR) - @cd $(CURDIR) && $(SHELL) ../scripts/net.sh + @cd $(CURDIR) && $(SHELL) ../scripts/net.sh $(EMBED_DIR_SUPPORTED) @touch $@ $(NNUE_EMBED_OBJ): $(UNIVERSAL_SRC_DIR)/nnue_embed.cpp $(NET_SENTINEL) | $(TEMP_DIR) - $(CXX) -O2 -Wno-c++26-extensions --embed-dir=$(CURDIR) -c $< -o $@ + $(CXX) -O2 -Wno-c++26-extensions $(EMBED_DIR) -c $< -o $@ $(UNIVERSAL_ENTRY_OBJ): $(UNIVERSAL_ENTRY_SRC) | $(TEMP_DIR) $(CXX) -O2 -c $< -o $@ @@ -1385,7 +1391,7 @@ $(TEMP_DIR)/%/.setup: | $(TEMP_DIR) $(TEMP_DIR)/%/stockfish.o: $(TEMP_DIR)/%/.setup $(NNUE_EMBED_OBJ) $(NET_SENTINEL) +ENV_CXXFLAGS='$(ENV_CXXFLAGS) $(call arch-cxxflags,$*)' \ $(MAKE) -C $(TEMP_DIR)/$* $(UOBJ_TGT) \ - ARCH=$* CXX=$(CXX) \ + ARCH=$* CXX=$(CXX) COMP=$(COMP) \ NNUE_EMBED_OBJ=$(NNUE_EMBED_OBJ) # Drop COMDAT groups; clang LTO leaves them around internalized LOCAL # template instantiations which causes problems at the final link diff --git a/src/nnue/network.cpp b/src/nnue/network.cpp index ab8084924..05193aa04 100644 --- a/src/nnue/network.cpp +++ b/src/nnue/network.cpp @@ -51,10 +51,18 @@ INCBIN(EmbeddedNNUE, EvalFileDefaultName); // separate, non-LTO nnue_embed.o (with strong symbols) can override them during the LTO link, // (INCBIN can't deduplicate.) #define WEAK_SYM __attribute__((weak)) -extern const unsigned char gEmbeddedNNUEData[] WEAK_SYM = { - #embed EvalFileDefaultName +extern const unsigned char gEmbeddedNNUEData[] WEAK_SYM = + #ifdef __has_embed + { + #embed EvalFileDefaultName }; -extern const unsigned int gEmbeddedNNUESize WEAK_SYM = sizeof(gEmbeddedNNUEData); +const int padding = 0; + #else + #include "../universal/network_dump.inc" + ; +const int padding = 1; // trailing NUL byte + #endif +extern const unsigned int gEmbeddedNNUESize WEAK_SYM = sizeof(gEmbeddedNNUEData) - padding; #else const unsigned char gEmbeddedNNUEData[1] = {0x0}; const unsigned int gEmbeddedNNUESize = 1; diff --git a/src/universal/nnue_embed.cpp b/src/universal/nnue_embed.cpp index e9d5879f1..4e4829def 100644 --- a/src/universal/nnue_embed.cpp +++ b/src/universal/nnue_embed.cpp @@ -2,7 +2,15 @@ #include "../evaluate.h" -extern const unsigned char gEmbeddedNNUEData[] = { -#embed EvalFileDefaultName +extern const unsigned char gEmbeddedNNUEData[] = +#ifdef __has_embed + { + #embed EvalFileDefaultName }; -extern const unsigned int gEmbeddedNNUESize = sizeof(gEmbeddedNNUEData); +const unsigned int padding = 0; +#else + #include "network_dump.inc" + ; +const unsigned int padding = 1; // trailing NUL byte +#endif +extern const unsigned int gEmbeddedNNUESize = sizeof(gEmbeddedNNUEData) - padding;