diff --git a/.github/ci/arm_matrix.json b/.github/ci/arm_matrix.json index 5e0b289e3..b4fb5d3c0 100644 --- a/.github/ci/arm_matrix.json +++ b/.github/ci/arm_matrix.json @@ -22,12 +22,12 @@ } ], "binaries": [ - "armv8-universal", + "arm64-universal", "armv7-neon" ], "exclude": [ { - "binaries": "armv8-universal", + "binaries": "arm64-universal", "config": { "compiler": "armv7a-linux-androideabi29-clang++" } diff --git a/.github/ci/macos_matrix.json b/.github/ci/macos_matrix.json deleted file mode 100644 index 7d6d89d62..000000000 --- a/.github/ci/macos_matrix.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "include": [ - { - "config": { - "name": "macOS 15 Apple Clang", - "os": "macos-15-intel", - "simple_name": "macos", - "compiler": "clang++", - "comp": "clang", - "shell": "bash", - "archive_ext": "tar.gz" - }, - "binaries": "x86-64-bmi2" - }, - { - "config": { - "name": "macOS 15 Apple Clang M1", - "os": "macos-15", - "simple_name": "macos-m1", - "compiler": "clang++", - "comp": "clang", - "shell": "bash", - "archive_ext": "tar.gz" - }, - "binaries": "apple-silicon" - } - ] -} diff --git a/.github/ci/universal_matrix.json b/.github/ci/universal_matrix.json index 794b6384a..307f29549 100644 --- a/.github/ci/universal_matrix.json +++ b/.github/ci/universal_matrix.json @@ -37,6 +37,15 @@ "archive_ext": "zip" }, "binaries": "arm64-universal" + }, + { + "config": { + "name": "macOS 15 Apple Clang Universal", + "os": "macos-15", + "simple_name": "macos", + "archive_ext": "tar.gz" + }, + "binaries": "universal" } ] } diff --git a/.github/workflows/compilation.yml b/.github/workflows/compilation.yml deleted file mode 100644 index ef5ecb927..000000000 --- a/.github/workflows/compilation.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: Compilation -on: - workflow_call: - inputs: - matrix: - type: string - required: true -jobs: - Compilation: - name: ${{ matrix.config.name }} ${{ matrix.binaries }} - runs-on: ${{ matrix.config.os }} - env: - COMPCXX: ${{ matrix.config.compiler }} - COMP: ${{ matrix.config.comp }} - EXT: ${{ matrix.config.ext }} - NAME: ${{ matrix.config.simple_name }} - BINARY: ${{ matrix.binaries }} - SDE: ${{ matrix.config.sde }} - strategy: - fail-fast: false - matrix: ${{ fromJson(inputs.matrix) }} - defaults: - run: - working-directory: src - shell: ${{ matrix.config.shell }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Install fixed GCC on Linux - if: runner.os == 'Linux' - uses: egor-tensin/setup-gcc@a2861a8b8538f49cf2850980acccf6b05a1b2ae4 # v2.0 - with: - version: 11 - - - name: Setup msys and install required packages - if: runner.os == 'Windows' - uses: msys2/setup-msys2@e9898307ac31d1a803454791be09ab9973336e1c # v2.31.1 - with: - msystem: ${{ matrix.config.msys_sys }} - install: mingw-w64-${{ matrix.config.msys_env }} make git zip - - - name: Download SDE package - if: ${{ matrix.config.sde }} - uses: petarpetrovt/setup-sde@d230e06df0ca06ced70000e4a7fa62afc9a2001e # v4.0 - with: - environmentVariableName: SDE_DIR - sdeVersion: 9.58.0 - - - name: Download the used network from the fishtest framework - run: make net - - - name: Check compiler - run: $COMPCXX -v - - - name: Test help target - run: make help - - - name: Check git - run: git --version - - - name: Check compiler - run: $COMPCXX -v - - - name: Show compiler cpu info - run: | - if [[ "$COMPCXX" == clang* ]]; then - $COMPCXX -E - -march=native -### - else - $COMPCXX -Q -march=native --help=target - fi - - # Compile the configured release binary. - - - name: Compile ${{ matrix.config.binaries }} build - run: | - make clean - make -j4 profile-build ARCH=$BINARY COMP=$COMP RUN_PREFIX="$SDE" - make strip ARCH=$BINARY COMP=$COMP - RUN_PREFIX="$SDE" ../tests/signature.sh $benchref - mv ./stockfish$EXT ../stockfish-$NAME-$BINARY$EXT - - - name: Remove non src files - run: git clean -fxd - - - name: Upload artifact for (pre)-release - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: ${{ matrix.config.simple_name }} ${{ matrix.binaries }} - path: | - . - !.git - !.output diff --git a/.github/workflows/stockfish.yml b/.github/workflows/stockfish.yml index 0065fd728..0606d685c 100644 --- a/.github/workflows/stockfish.yml +++ b/.github/workflows/stockfish.yml @@ -21,7 +21,7 @@ jobs: name: Prerelease if: github.repository == 'official-stockfish/Stockfish' && (github.ref == 'refs/heads/master' || (startsWith(github.ref_name, 'sf_') && github.ref_type == 'tag')) runs-on: ubuntu-latest - needs: [Matrix, Compilation, ARMCompilation, UniversalCompilation] + needs: [Matrix, ARMCompilation, UniversalCompilation] permissions: contents: write # For deleting/creating a prerelease steps: @@ -112,17 +112,12 @@ jobs: name: Prepare matrices runs-on: ubuntu-latest outputs: - macos_matrix: ${{ steps.set-macos-matrix.outputs.macos_matrix }} arm_matrix: ${{ steps.set-arm-matrix.outputs.arm_matrix }} universal_matrix: ${{ steps.set-universal-matrix.outputs.universal_matrix }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - id: set-macos-matrix - run: | - MACOS_MATRIX=$(jq -c '.' .github/ci/macos_matrix.json) - echo "MACOS_MATRIX=$MACOS_MATRIX" >> $GITHUB_OUTPUT - id: set-arm-matrix run: | ARM_MATRIX=$(jq -c '.' .github/ci/arm_matrix.json) @@ -156,12 +151,6 @@ jobs: if: ${{ always() }} uses: ./.github/workflows/avx2_compilers.yml # Release Jobs - Compilation: - name: macOS builds - needs: [Matrix, Sanitizers, Tests, Matetrack, Games, CompilerCheck] - uses: ./.github/workflows/compilation.yml - with: - matrix: ${{ needs.Matrix.outputs.macos_matrix }} ARMCompilation: name: Android builds needs: [Matrix, Sanitizers, Tests, Matetrack, Games, CompilerCheck] @@ -172,17 +161,6 @@ jobs: name: "Universal builds" needs: [Matrix, Sanitizers, Tests, Matetrack, Games, CompilerCheck] uses: ./.github/workflows/universal_compilation.yml - Binaries: - if: github.repository == 'official-stockfish/Stockfish' - name: macOS uploads - needs: [Prerelease, Matrix] - uses: ./.github/workflows/upload_binaries.yml - with: - matrix: ${{ needs.Matrix.outputs.macos_matrix }} - permissions: - contents: write # For deleting/creating a (pre)release - secrets: - token: ${{ secrets.GITHUB_TOKEN }} ARM_Binaries: if: github.repository == 'official-stockfish/Stockfish' name: Android uploads diff --git a/.github/workflows/universal_compilation.yml b/.github/workflows/universal_compilation.yml index 9a32b8b3c..28909bba4 100644 --- a/.github/workflows/universal_compilation.yml +++ b/.github/workflows/universal_compilation.yml @@ -217,3 +217,40 @@ jobs: . !.git !.output + + UniversalMacOS: + name: macOS + runs-on: macos-15 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + fetch-depth: 0 + + # macos-15 is Apple silicon; Rosetta 2 is needed to exercise the x86-64 slice + - name: Install Rosetta 2 + run: softwareupdate --install-rosetta --agree-to-license + + - name: Build universal binary + run: make -j4 -C src macos-lipo COMP=clang + + - name: Rename binary + run: mv src/stockfish stockfish-macos-universal + + - name: Verify binary runs natively and under Rosetta + run: ./scripts/check_universal_macos.sh ./stockfish-macos-universal + + - name: Remove non-src files + run: git clean -fxd -- src + + - name: Upload artifact for (pre)-release + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: macos universal + path: | + . + !.git + !.output diff --git a/scripts/check_universal_macos.sh b/scripts/check_universal_macos.sh new file mode 100755 index 000000000..93c2f6f03 --- /dev/null +++ b/scripts/check_universal_macos.sh @@ -0,0 +1,62 @@ +#!/bin/sh + +# Usage: check_universal_macos.sh STOCKFISH_EXE [EXPECTED_BENCH] + +set -eu + +STOCKFISH_EXE=$1 + +extract() { + awk -F: -v lbl="$1" '$0 ~ lbl { + sub(/^[[:space:]]+/, "", $2); sub(/[[:space:]]+$/, "", $2); print $2; exit + }' +} + +FAIL=0 + +BINARY_SIZE=$(wc -c < "$STOCKFISH_EXE") +MAX_SIZE=$((150 * 1024 * 1024)) +if [ "$BINARY_SIZE" -gt "$MAX_SIZE" ]; then + printf 'check_universal_macos.sh: binary size %d bytes exceeds 150 MB limit\n' "$BINARY_SIZE" >&2 + exit 1 +fi + +native_bench=$(arch -arm64 "$STOCKFISH_EXE" bench 2>&1 | extract "Nodes searched" || true) +if [ -z "$native_bench" ]; then + echo "check_universal_macos.sh: arm64 run produced no bench" >&2 + FAIL=1 +else + printf 'native (arm64) bench %s\n' "$native_bench" >&2 +fi + +x86_bench=$(arch -x86_64 "$STOCKFISH_EXE" bench 2>&1 | extract "Nodes searched" || true) +if [ -z "$x86_bench" ]; then + echo "check_universal_macos.sh: x86_64 (Rosetta) run produced no bench (crashed?)" >&2 + FAIL=1 +else + printf 'x86_64 (Rosetta) bench %s\n' "$x86_bench" >&2 +fi + +# Should match +if [ "$native_bench" != "$x86_bench" ]; then + printf 'check_universal_macos.sh: slice bench mismatch: arm64=%s x86_64=%s\n' \ + "$native_bench" "$x86_bench" >&2 + FAIL=1 +fi + +# Check that under Rosetta, SSE41 is detected +x86_compiler=$(arch -x86_64 "$STOCKFISH_EXE" compiler 2>&1 || true) +if printf '%s\n' "$x86_compiler" | grep -q 'SSE41'; then + printf 'x86_64 (Rosetta) compiler reports SSE41 ok\n' >&2 +else + echo "check_universal_macos.sh: x86_64 compiler output missing SSE41" >&2 + printf '%s\n' "$x86_compiler" >&2 + FAIL=1 +fi + +if [ "$FAIL" != 0 ]; then + echo "check_universal_macos.sh: failed" + exit 1 +fi + +echo "check_universal_macos.sh: Good! Universal macOS binary works." diff --git a/src/Makefile b/src/Makefile index 5329ebbe4..ce2170436 100644 --- a/src/Makefile +++ b/src/Makefile @@ -148,7 +148,7 @@ ifeq ($(ARCH), $(filter $(ARCH), \ x86-64-universal x86-64-avx512icl x86-64-vnni512 x86-64-avx512 x86-64-avxvnni \ x86-64-bmi2 x86-64-avx2 x86-64-sse41-popcnt x86-64-modern x86-64-ssse3 x86-64-sse3-popcnt \ x86-64 x86-32-sse41-popcnt x86-32-sse2 x86-32 ppc-64 ppc-64-altivec ppc-64-vsx ppc-32 e2k \ - armv7 armv7-neon armv8 armv8-dotprod armv8-universal apple-silicon general-64 general-32 riscv64 \ + armv7 armv7-neon armv8 armv8-dotprod arm64-universal apple-silicon general-64 general-32 riscv64 \ loongarch64 loongarch64-lsx loongarch64-lasx)) SUPPORTED_ARCH=true else @@ -372,7 +372,7 @@ ifeq ($(ARCH),armv8-dotprod) arm_version = 8 endif -ifeq ($(ARCH),armv8-universal) +ifeq ($(ARCH),arm64-universal) arch = armv8 prefetch = yes popcnt = yes @@ -559,12 +559,12 @@ ifeq ($(COMP),clang) endif ifeq ($(KERNEL),Darwin) - CXXFLAGS += -mmacosx-version-min=10.15 - LDFLAGS += -mmacosx-version-min=10.15 + mac_target_flags := -mmacosx-version-min=10.15 ifneq ($(arch),any) - CXXFLAGS += -arch $(arch) - LDFLAGS += -arch $(arch) + mac_target_flags += -arch $(arch) endif + CXXFLAGS += $(mac_target_flags) + LDFLAGS += $(mac_target_flags) XCRUN = xcrun endif @@ -640,6 +640,7 @@ ifeq ($(COMP),gcc) ifneq ($(gccisclang),) profile_make = clang-profile-make profile_use = clang-profile-use + CXXFLAGS += -fconstexpr-steps=500000000 else CXXFLAGS += -Wstack-usage=128000 -fno-ipa-cp-clone -fconstexpr-ops-limit=500000000 endif @@ -956,6 +957,7 @@ help: echo "help > Display architecture details" && \ echo "profile-build > standard build with profile-guided optimization" && \ echo "build > skip profile-guided optimization" && \ + echo "macos-lipo > macOS x86-64 + Apple silicon universal binary" && \ echo "net > Download the default nnue nets" && \ echo "strip > Strip executable" && \ echo "install > Install executable" && \ @@ -985,7 +987,7 @@ help: echo "ppc-32 > PPC 32-bit" && \ echo "armv7 > ARMv7 32-bit" && \ echo "armv7-neon > ARMv7 32-bit with popcnt and neon" && \ - echo "armv8-universal > ARMv8 64-bit with automatic runtime selection of the best architecture" && \ + echo "arm64-universal > ARMv8 64-bit with automatic runtime selection of the best architecture" && \ echo "armv8 > ARMv8 64-bit with popcnt and neon" && \ echo "armv8-dotprod > ARMv8 64-bit with popcnt, neon and dot product support" && \ echo "e2k > Elbrus 2000" && \ @@ -1024,7 +1026,7 @@ ifneq ($(SUPPORTED_ARCH), true) endif -.PHONY: help analyze build profile-build strip install clean net \ +.PHONY: help analyze build profile-build macos-lipo strip install clean net \ objclean profileclean config-sanity \ icx-profile-use icx-profile-make \ gcc-profile-use gcc-profile-make \ @@ -1055,7 +1057,32 @@ profile-build: net config-sanity objclean profileclean $(MAKE) ARCH=$(ARCH) COMP=$(COMP) profileclean endif -ifeq ($(comp), clang) +# Default network filename, read from evaluate.h's EvalFileDefaultName. +macos-lipo: NNUE_NET = $(shell grep -oE 'nn-[0-9a-f]+\.nnue' $(CURDIR)/evaluate.h | head -1) +macos-lipo: + @echo "" + @echo "Step 1/4. Building x86-64 slice (network shared from Apple-silicon slice) ..." + CXXFLAGS="-arch x86_64" $(MAKE) build ARCH=x86-64-universal COMP=$(COMP) MACOS_X86_SLICE=1 + mv $(EXE) $(EXE)-x86_64 + @echo "" + @echo "Step 2/4. Building Apple silicon slice ..." + $(MAKE) profile-build ARCH=apple-silicon COMP=$(COMP) + mv $(EXE) $(EXE)-arm64 + @echo "" + @echo "Step 3/4. Combining slices and locating the embedded network ..." + lipo -create $(EXE)-x86_64 $(EXE)-arm64 -output $(EXE) + $(SHELL) universal/patch_x86_slice.sh $(EXE) $(EXE)-x86_64 $(NNUE_NET) + @echo "" + @echo "Step 4/4. Re-combining slices with the patched x86-64 slice ..." + lipo -create $(EXE)-x86_64 $(EXE)-arm64 -output $(EXE) + @rm -f $(EXE)-x86_64 $(EXE)-arm64 + @echo "Universal macOS binary built: $(EXE)" + +ifeq ($(KERNEL),Darwin) +# Apple uses ld64 which doesn't have --save-temps + LTO_OBJ_SUFFIX := .lto.o + SAVE_TEMPS := -Wl,-object_path_lto,stockfish.lto.o +else ifeq ($(comp), clang) LTO_OBJ_SUFFIX := .lto.o ifeq ($(target_windows),yes) SAVE_TEMPS := -Wl,--lto-emit-asm # ld.lld doesn't have a save temps functionality @@ -1077,7 +1104,7 @@ MODNAME := $(subst -,_,$(notdir $(CURDIR))) universal-object-pgo: objclean profileclean universalclean @echo "" @echo "Step 1/4. Building instrumented executable ..." - $(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_make) + $(MAKE) ARCH=$(ARCH) COMP=$(COMP) ENV_LDFLAGS='$(ENV_LDFLAGS) $(NNUE_EMBED_OBJ)' $(profile_make) @echo "" @echo "Step 2/4. Running benchmark for pgo-build ..." $(PGOBENCH) > PGOBENCH.out 2>&1 @@ -1284,7 +1311,7 @@ endif # Usage: # make -j ARCH=x86-64-universal build # non-PGO # make -j ARCH=x86-64-universal profile-build # PGO (use RUN_PREFIX for intel SDE) -# make -j ARCH=armv8-universal build +# make -j ARCH=arm64-universal build TEMP_DIR := $(CURDIR)/temp_builds UNIVERSAL_EXE := $(CURDIR)/$(EXE) @@ -1319,7 +1346,9 @@ ARCH_OBJS := $(TEMP_DIR)/$(BASELINE_ARCH)/stockfish.o \ UNIVERSAL_FINAL_FLAGS := -fno-exceptions -Os \ -Wno-c++26-extensions -Wno-c23-extensions # suppress #embed warnings -ifeq ($(KERNEL),Linux) +ifeq ($(KERNEL),Darwin) + UNIVERSAL_FINAL_FLAGS += $(mac_target_flags) +else ifeq ($(KERNEL),Linux) UNIVERSAL_FINAL_FLAGS += -static-libstdc++ -static-libgcc ifeq ($(COMP),ndk) UNIVERSAL_FINAL_FLAGS += -static @@ -1337,11 +1366,16 @@ endif arch-suffix = $(subst -,_,$(1)) arch-namespace = Stockfish_$(call arch-suffix,$(1)) # Itanium ABI mangled name for Stockfish_::main(int, char**) -arch-mangled = _ZN$(shell printf %s '$(call arch-namespace,$(1))' | wc -c)$(call arch-namespace,$(1))4mainEiPPc +arch-mangled = _ZN$(shell printf %s '$(call arch-namespace,$(1))' | wc -c | tr -d ' ')$(call arch-namespace,$(1))4mainEiPPc +# Mach-O initializer section name: strip "x86-64" or "x86-64-" and prefix with "_i_" +# (Mach-O section names are restricted to <=16 chars, hence this naming scheme) +arch-mac-section = _i_$(subst -,_,$(patsubst x86-64,,$(patsubst x86-64-%,%,$(1)))) +ifeq ($(KERNEL),Darwin) + arch-defmain = -Wl,-alias,_$(call arch-mangled,$(1)),_main # lld-link, used on Windows+clang, doesn't accept --defsym, so # we define main directly in main.cpp instead and delete it later. -ifeq ($(target_windows)+$(COMP),yes+clang) +else ifeq ($(target_windows)+$(COMP),yes+clang) arch-defmain = -DUNIVERSAL_NEEDS_MAIN_SHIM else arch-defmain = -Wl,--defsym=main=$(call arch-mangled,$(1)) @@ -1352,7 +1386,14 @@ ifeq ($(EMBED_DIR_SUPPORTED),1) EMBED_DIR = --embed-dir=$(CURDIR) endif -arch-cxxflags = $(EMBED_DIR) -DStockfish=$(call arch-namespace,$(1)) -DUNIVERSAL_BINARY \ +# macos-lipo sets MACOS_X86_SLICE=1 when building the x86-64 slice so that the +# slice reads the network out of the Apple-silicon slice instead of embedding +# its own copy (see universal/nnue_embed.cpp and universal/patch_x86_slice.sh). +ifeq ($(MACOS_X86_SLICE),1) + SLICE_DEF := -DUNIVERSAL_BINARY_MACOS_X86_SLICE +endif + +arch-cxxflags = $(EMBED_DIR) -DStockfish=$(call arch-namespace,$(1)) -DUNIVERSAL_BINARY $(SLICE_DEF) \ $(call arch-defmain,$(1)) -Wno-c++26-extensions -Wno-c23-extensions UOBJ_TGT ?= universal-object-pgo @@ -1373,10 +1414,10 @@ $(NET_SENTINEL): | $(TEMP_DIR) @touch $@ $(NNUE_EMBED_OBJ): $(UNIVERSAL_SRC_DIR)/nnue_embed.cpp $(NET_SENTINEL) | $(TEMP_DIR) - $(CXX) -O2 -Wno-c++26-extensions $(EMBED_DIR) -c $< -o $@ + $(CXX) -O2 -std=c++20 -Wno-c++26-extensions $(SLICE_DEF) $(mac_target_flags) $(EMBED_DIR) -c $< -o $@ $(UNIVERSAL_ENTRY_OBJ): $(UNIVERSAL_ENTRY_SRC) | $(TEMP_DIR) - $(CXX) -O2 -c $< -o $@ + $(CXX) -O2 -std=c++20 $(mac_target_flags) -c $< -o $@ # Symlink tracked top-level items from src/ into temp_builds//. $(TEMP_DIR)/%/.setup: | $(TEMP_DIR) @@ -1393,6 +1434,11 @@ $(TEMP_DIR)/%/stockfish.o: $(TEMP_DIR)/%/.setup $(NNUE_EMBED_OBJ) $(NET_SENTINEL $(MAKE) -C $(TEMP_DIR)/$* $(UOBJ_TGT) \ ARCH=$* CXX=$(CXX) COMP=$(COMP) \ NNUE_EMBED_OBJ=$(NNUE_EMBED_OBJ) +ifeq ($(KERNEL),Darwin) +# Rename the initializer section + ld -r -rename_section __DATA __mod_init_func __DATA $(call arch-mac-section,$*) $@ -o $@.tmp + mv $@.tmp $@ +else # Drop COMDAT groups; clang LTO leaves them around internalized LOCAL # template instantiations which causes problems at the final link $(OBJCOPY) -R .group $@ @@ -1406,6 +1452,7 @@ else # Make the array inert $(OBJCOPY) --set-section-flags $(call arch-suffix,$*)_init=alloc,data $@ endif +endif $(UNIVERSAL_EXE): $(UNIVERSAL_ENTRY_OBJ) $(NNUE_EMBED_OBJ) $(ARCH_OBJS) $(CXX) -o $@ $(UNIVERSAL_ENTRY_OBJ) $(NNUE_EMBED_OBJ) $(ARCH_OBJS) $(UNIVERSAL_FINAL_FLAGS) diff --git a/src/nnue/network.cpp b/src/nnue/network.cpp index 05193aa04..382f4f445 100644 --- a/src/nnue/network.cpp +++ b/src/nnue/network.cpp @@ -46,23 +46,13 @@ // Note that this does not work in Microsoft Visual Studio. #if !defined(UNIVERSAL_BINARY) && !defined(_MSC_VER) && !defined(NNUE_EMBEDDING_OFF) INCBIN(EmbeddedNNUE, EvalFileDefaultName); +#elif defined(UNIVERSAL_BINARY_MACOS_X86_SLICE) +// Determined at runtime, see universal/nnue_embed.cpp +extern const unsigned char* const gEmbeddedNNUEData; +extern const unsigned int gEmbeddedNNUESize; #elif defined(UNIVERSAL_BINARY) - // When building for the universal binary, use C++26 #embed with weak symbols so that a - // separate, non-LTO nnue_embed.o (with strong symbols) can override them during the LTO link, - // (INCBIN can't deduplicate.) - #define WEAK_SYM __attribute__((weak)) -extern const unsigned char gEmbeddedNNUEData[] WEAK_SYM = - #ifdef __has_embed - { - #embed EvalFileDefaultName -}; -const int padding = 0; - #else - #include "../universal/network_dump.inc" - ; -const int padding = 1; // trailing NUL byte - #endif -extern const unsigned int gEmbeddedNNUESize WEAK_SYM = sizeof(gEmbeddedNNUEData) - padding; +extern const unsigned char gEmbeddedNNUEData[]; +extern const unsigned int gEmbeddedNNUESize; #else const unsigned char gEmbeddedNNUEData[1] = {0x0}; const unsigned int gEmbeddedNNUESize = 1; @@ -257,6 +247,11 @@ void Network::load_internal() { } }; +#ifdef UNIVERSAL_BINARY_MACOS_X86_SLICE + if (gEmbeddedNNUEData == nullptr) // failed embedded load + return; +#endif + MemoryBuffer buffer(const_cast(reinterpret_cast(gEmbeddedNNUEData)), size_t(gEmbeddedNNUESize)); diff --git a/src/universal/entry_arm64.cpp b/src/universal/entry_arm64.cpp index 1616f1614..158bc64d6 100644 --- a/src/universal/entry_arm64.cpp +++ b/src/universal/entry_arm64.cpp @@ -1,3 +1,21 @@ +/* + Stockfish, a UCI chess playing engine derived from Glaurung 2.1 + Copyright (C) 2004-2026 The Stockfish developers (see AUTHORS file) + + Stockfish is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Stockfish is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + #include #if defined(_WIN32) diff --git a/src/universal/entry_x86.cpp b/src/universal/entry_x86.cpp index 54fbc9283..a69af2c14 100644 --- a/src/universal/entry_x86.cpp +++ b/src/universal/entry_x86.cpp @@ -1,18 +1,62 @@ +/* + Stockfish, a UCI chess playing engine derived from Glaurung 2.1 + Copyright (C) 2004-2026 The Stockfish developers (see AUTHORS file) + + Stockfish is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Stockfish is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + #include #include -#define DEFINE_BUILD(x) \ - namespace Stockfish_##x { \ - extern int main(int argc, char* argv[]); \ - } \ - extern "C" void (*__start_##x##_init[])(void); \ - extern "C" void (*__stop_##x##_init[])(void); \ - int entry_##x(int argc, char* argv[]) { \ - unsigned count = __stop_##x##_init - __start_##x##_init; \ - for (unsigned i = 0; i < count; i++) \ - __start_##x##_init[i](); \ - return Stockfish_##x::main(argc, argv); \ - } +#ifdef __APPLE__ + // We locate each arch's initializer pointer array at runtime via getsectiondata(). + // Example name is "_i_sse41_popcnt", baseline build is just "_i_" + #include + #include + #include + +extern "C" const struct mach_header_64 _mh_execute_header; + + #define DEFINE_BUILD(x) \ + namespace Stockfish_##x { \ + extern int main(int argc, char* argv[]); \ + } \ + int entry_##x(int argc, char* argv[]) { \ + char name[17]; \ + const char* full = #x; \ + snprintf(name, sizeof(name), "_i_%s", full[6] ? full + 7 : ""); \ + unsigned long size = 0; \ + auto** fns = reinterpret_cast( \ + getsectiondata(&_mh_execute_header, "__DATA", name, &size)); \ + for (unsigned long i = 0; i < size / sizeof(*fns); i++) \ + fns[i](); \ + return Stockfish_##x::main(argc, argv); \ + } +#else + #define DEFINE_BUILD(x) \ + namespace Stockfish_##x { \ + extern int main(int argc, char* argv[]); \ + } \ + extern "C" void (*__start_##x##_init[])(void); \ + extern "C" void (*__stop_##x##_init[])(void); \ + int entry_##x(int argc, char* argv[]) { \ + unsigned count = __stop_##x##_init - __start_##x##_init; \ + for (unsigned i = 0; i < count; i++) \ + __start_##x##_init[i](); \ + return Stockfish_##x::main(argc, argv); \ + } +#endif DEFINE_BUILD(x86_64) DEFINE_BUILD(x86_64_sse41_popcnt) @@ -108,8 +152,25 @@ static int dispatch(const CpuFeatures& f, int argc, char* argv[]) { return entry_x86_64_avx512icl(argc, argv); } +static void maybe_promote_thread_to_avx512() { +#ifdef __APPLE__ + // Intel Macs supporting AVX512 don't advertise it in xgetbv and only + // do so once at least one avx512 instruction has been executed. + // See https://github.com/apple/darwin-xnu/blob/0a798f6738bc1db01281fc08ae024145e84df927/osfmk/i386/fpu.c#L176 + + int supported = 0; + size_t len = sizeof(supported); + if (sysctlbyname("hw.optional.avx512f", &supported, &len, nullptr, 0) == 0 && supported) + { + asm volatile(".byte 0x62, 0xf1, 0x7d, 0x48, 0x6f, 0xc0"); // vmovdqa32 zmm0,zmm0 + } +#endif +} + int main(int argc, char* argv[]) { + maybe_promote_thread_to_avx512(); + __builtin_cpu_init(); CpuFeatures features = query_cpu_features(); return dispatch(features, argc, argv); -} \ No newline at end of file +} diff --git a/src/universal/nnue_embed.cpp b/src/universal/nnue_embed.cpp index 4e4829def..edabebb44 100644 --- a/src/universal/nnue_embed.cpp +++ b/src/universal/nnue_embed.cpp @@ -1,16 +1,85 @@ +/* + Stockfish, a UCI chess playing engine derived from Glaurung 2.1 + Copyright (C) 2004-2026 The Stockfish developers (see AUTHORS file) + + Stockfish is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Stockfish is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + // Standalone NNUE embedding for universal binary builds #include "../evaluate.h" +#ifdef UNIVERSAL_BINARY_MACOS_X86_SLICE + +// In a macOS universal binary the network is embedded only in the arm64 slice, +// and the x86-64 slice mmaps it from the arm64 slice. + + #include + #include + #include + #include + #include + #include + #include + +// Must be kept in sync with patch_x86_slice.sh +extern const volatile uint64_t gUniversalNNUEOffset = 0xCAFE0FF5E70FF5E7ULL; +extern const volatile uint64_t gUniversalNNUESize = 0xCAFE512ECAFE512EULL; + +static const unsigned char* map_embedded_nnue() { + char path[PATH_MAX]; + uint32_t len = sizeof(path); + if (_NSGetExecutablePath(path, &len) != 0) + return nullptr; + + char resolved[PATH_MAX]; + const char* file = realpath(path, resolved) ? resolved : path; + + int fd = open(file, O_RDONLY); + if (fd < 0) + return nullptr; + + // Align down to page size for mmap + const uint64_t pageSize = uint64_t(sysconf(_SC_PAGESIZE)); + const uint64_t base = gUniversalNNUEOffset & ~(pageSize - 1); + const uint64_t pad = gUniversalNNUEOffset - base; + + void* p = + mmap(nullptr, size_t(gUniversalNNUESize + pad), PROT_READ, MAP_PRIVATE, fd, off_t(base)); + close(fd); + if (p == MAP_FAILED) + return nullptr; + + return reinterpret_cast(p) + pad; +} + +extern const unsigned char* const gEmbeddedNNUEData = map_embedded_nnue(); +extern const unsigned int gEmbeddedNNUESize = static_cast(gUniversalNNUESize); + +#else + extern const unsigned char gEmbeddedNNUEData[] = -#ifdef __has_embed + #ifdef __has_embed { - #embed EvalFileDefaultName + #embed EvalFileDefaultName }; const unsigned int padding = 0; -#else - #include "network_dump.inc" + #else + #include "network_dump.inc" ; const unsigned int padding = 1; // trailing NUL byte -#endif + #endif extern const unsigned int gEmbeddedNNUESize = sizeof(gEmbeddedNNUEData) - padding; + +#endif diff --git a/src/universal/patch_x86_slice.sh b/src/universal/patch_x86_slice.sh new file mode 100755 index 000000000..ec076a902 --- /dev/null +++ b/src/universal/patch_x86_slice.sh @@ -0,0 +1,73 @@ +#!/bin/sh + +# Stockfish, a UCI chess playing engine derived from Glaurung 2.1 +# Copyright (C) 2004-2026 The Stockfish developers (see AUTHORS file) +# +# Stockfish is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Stockfish is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# This finds the gUniversalNNUEOffset/gUniversalNNUESize symbols in the x86 slice +# and patches them to be an offset+size into the full executable. +# +# Usage: patch_x86_slice.sh + +set -eu + +FAT="$1" +X86="$2" +NET="$3" + +# Must match volatile initializers in nnue_embed.cpp; byte order +# is reversed to match how it appears in the binary xxd +OFFSET_MAGIC=e7f50fe7f50ffeca +SIZE_MAGIC=2e51feca2e51feca + +die() { echo "patch_x86_slice: $*" >&2; exit 1; } + +# Find hex pattern in file, return byte offset +# Optional $3 = bytes to skip, $4 = max bytes to scan +find_bytes() { + file=$1; needle=$2; skip=${3:-0}; window=${4:-0} + [ "$window" -gt 0 ] && win="-l $window" || win="" + # Disassemble as hex, strip \n and search for the pattern + pos=$(xxd -s "$skip" $win -p "$file" | tr -d '\n' \ + | awk -v n="$needle" '{ p = index($0, n); if (p) { print p; exit } }') + [ -n "$pos" ] || return 1 + echo $(( skip + (pos - 1) / 2 )) +} + +# Overwrite 8 bytes at a file offset +write_u64_le() { + f="$1"; off="$2"; v="$3"; bytes=""; i=0 + while [ "$i" -lt 8 ]; do + bytes="$bytes$(printf '\\%03o' "$(( v & 255 ))")" + v=$(( v >> 8 )); i=$(( i + 1 )) + done + printf "$bytes" | dd of="$f" bs=1 seek="$off" conv=notrunc 2>/dev/null +} + +needle=$(xxd -p -l 64 "$NET" | tr -d '\n') # take first bytes of network as needle +net_size=$(wc -c < "$NET" | tr -d ' ') + +# The arm64 slice always follows the x86-64 slice in +# the fat binary, so scan from ~after the x86-64 slice. +x86_size=$(wc -c < "$X86" | tr -d ' ') +net_off=$(find_bytes "$FAT" "$needle" "$x86_size" 8000000) || die "network not found" + +off_pos=$(find_bytes "$X86" "$OFFSET_MAGIC") || die "offset sentinel not found" +size_pos=$(find_bytes "$X86" "$SIZE_MAGIC") || die "size sentinel not found" + +write_u64_le "$X86" "$off_pos" "$net_off" +write_u64_le "$X86" "$size_pos" "$net_size" + +echo "patch_x86_slice.sh: network at offset $net_off + size $net_size " diff --git a/src/universal/rewrite_asm_sections.awk b/src/universal/rewrite_asm_sections.awk index 2b34fe5e6..01e5d1049 100644 --- a/src/universal/rewrite_asm_sections.awk +++ b/src/universal/rewrite_asm_sections.awk @@ -1,3 +1,19 @@ +# Stockfish, a UCI chess playing engine derived from Glaurung 2.1 +# Copyright (C) 2004-2026 The Stockfish developers (see AUTHORS file) +# +# Stockfish is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Stockfish is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + # This is used if the universal binary is compiled on clang+Windows. # Usage (avx2 example): # awk -v MODNAME=x86_64_avx2 -f rewrite_asm_sections.awk stockfish.exe.lto.s > renamed.s