From 8cbca11a5374c1c2de43be416188bf79f1a88302 Mon Sep 17 00:00:00 2001 From: anematode Date: Fri, 29 May 2026 19:05:53 +0200 Subject: [PATCH] macOS universal binary, take 2 - Add new target `macos-lipo`. - Created by compiling a universal x86 binary (no PGO) and a standard Apple silicon binary (with PGO), then combining them into a Mach-O fat binary - To keep only one copy of the net, we add custom loading logic in the x86 section. The executable reads its own path and mmaps the net that's in the ARM section. - The offset and size (from the executable base) of the mapping is injected after compilation in `patch_x86_slice.sh` - avx512 on macOS isn't advertised in the xcr0 register by default. The simple solution I came up with is to execute a dummy AVX512 instruction, which sets up the register, before calling `__builtin_cpu_init`. Some housekeeping as well: - Rename `armv8-universal` -> `arm64-universal`. - Add standard copyright headers to the files we've added recently. Potential follow-ups: - Disservin's Makefile cleanup - Alternative ideas for the net loading. In particular, this will error out if the user strips the binary (since that'll invalidate the offset). closes https://github.com/official-stockfish/Stockfish/pull/6860 No functional change Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/ci/arm_matrix.json | 4 +- .github/ci/macos_matrix.json | 28 ------ .github/ci/universal_matrix.json | 9 ++ .github/workflows/compilation.yml | 94 --------------------- .github/workflows/stockfish.yml | 24 +----- .github/workflows/universal_compilation.yml | 37 ++++++++ scripts/check_universal_macos.sh | 62 ++++++++++++++ src/Makefile | 81 ++++++++++++++---- src/nnue/network.cpp | 27 +++--- src/universal/entry_arm64.cpp | 18 ++++ src/universal/entry_x86.cpp | 87 ++++++++++++++++--- src/universal/nnue_embed.cpp | 79 +++++++++++++++-- src/universal/patch_x86_slice.sh | 73 ++++++++++++++++ src/universal/rewrite_asm_sections.awk | 16 ++++ 14 files changed, 441 insertions(+), 198 deletions(-) delete mode 100644 .github/ci/macos_matrix.json delete mode 100644 .github/workflows/compilation.yml create mode 100755 scripts/check_universal_macos.sh create mode 100755 src/universal/patch_x86_slice.sh diff --git a/.github/ci/arm_matrix.json b/.github/ci/arm_matrix.json index 5e0b289e3..b4fb5d3c0 100644 --- a/.github/ci/arm_matrix.json +++ b/.github/ci/arm_matrix.json @@ -22,12 +22,12 @@ } ], "binaries": [ - "armv8-universal", + "arm64-universal", "armv7-neon" ], "exclude": [ { - "binaries": "armv8-universal", + "binaries": "arm64-universal", "config": { "compiler": "armv7a-linux-androideabi29-clang++" } diff --git a/.github/ci/macos_matrix.json b/.github/ci/macos_matrix.json deleted file mode 100644 index 7d6d89d62..000000000 --- a/.github/ci/macos_matrix.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "include": [ - { - "config": { - "name": "macOS 15 Apple Clang", - "os": "macos-15-intel", - "simple_name": "macos", - "compiler": "clang++", - "comp": "clang", - "shell": "bash", - "archive_ext": "tar.gz" - }, - "binaries": "x86-64-bmi2" - }, - { - "config": { - "name": "macOS 15 Apple Clang M1", - "os": "macos-15", - "simple_name": "macos-m1", - "compiler": "clang++", - "comp": "clang", - "shell": "bash", - "archive_ext": "tar.gz" - }, - "binaries": "apple-silicon" - } - ] -} diff --git a/.github/ci/universal_matrix.json b/.github/ci/universal_matrix.json index 794b6384a..307f29549 100644 --- a/.github/ci/universal_matrix.json +++ b/.github/ci/universal_matrix.json @@ -37,6 +37,15 @@ "archive_ext": "zip" }, "binaries": "arm64-universal" + }, + { + "config": { + "name": "macOS 15 Apple Clang Universal", + "os": "macos-15", + "simple_name": "macos", + "archive_ext": "tar.gz" + }, + "binaries": "universal" } ] } diff --git a/.github/workflows/compilation.yml b/.github/workflows/compilation.yml deleted file mode 100644 index ef5ecb927..000000000 --- a/.github/workflows/compilation.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: Compilation -on: - workflow_call: - inputs: - matrix: - type: string - required: true -jobs: - Compilation: - name: ${{ matrix.config.name }} ${{ matrix.binaries }} - runs-on: ${{ matrix.config.os }} - env: - COMPCXX: ${{ matrix.config.compiler }} - COMP: ${{ matrix.config.comp }} - EXT: ${{ matrix.config.ext }} - NAME: ${{ matrix.config.simple_name }} - BINARY: ${{ matrix.binaries }} - SDE: ${{ matrix.config.sde }} - strategy: - fail-fast: false - matrix: ${{ fromJson(inputs.matrix) }} - defaults: - run: - working-directory: src - shell: ${{ matrix.config.shell }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - - name: Install fixed GCC on Linux - if: runner.os == 'Linux' - uses: egor-tensin/setup-gcc@a2861a8b8538f49cf2850980acccf6b05a1b2ae4 # v2.0 - with: - version: 11 - - - name: Setup msys and install required packages - if: runner.os == 'Windows' - uses: msys2/setup-msys2@e9898307ac31d1a803454791be09ab9973336e1c # v2.31.1 - with: - msystem: ${{ matrix.config.msys_sys }} - install: mingw-w64-${{ matrix.config.msys_env }} make git zip - - - name: Download SDE package - if: ${{ matrix.config.sde }} - uses: petarpetrovt/setup-sde@d230e06df0ca06ced70000e4a7fa62afc9a2001e # v4.0 - with: - environmentVariableName: SDE_DIR - sdeVersion: 9.58.0 - - - name: Download the used network from the fishtest framework - run: make net - - - name: Check compiler - run: $COMPCXX -v - - - name: Test help target - run: make help - - - name: Check git - run: git --version - - - name: Check compiler - run: $COMPCXX -v - - - name: Show compiler cpu info - run: | - if [[ "$COMPCXX" == clang* ]]; then - $COMPCXX -E - -march=native -### - else - $COMPCXX -Q -march=native --help=target - fi - - # Compile the configured release binary. - - - name: Compile ${{ matrix.config.binaries }} build - run: | - make clean - make -j4 profile-build ARCH=$BINARY COMP=$COMP RUN_PREFIX="$SDE" - make strip ARCH=$BINARY COMP=$COMP - RUN_PREFIX="$SDE" ../tests/signature.sh $benchref - mv ./stockfish$EXT ../stockfish-$NAME-$BINARY$EXT - - - name: Remove non src files - run: git clean -fxd - - - name: Upload artifact for (pre)-release - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: ${{ matrix.config.simple_name }} ${{ matrix.binaries }} - path: | - . - !.git - !.output diff --git a/.github/workflows/stockfish.yml b/.github/workflows/stockfish.yml index 0065fd728..0606d685c 100644 --- a/.github/workflows/stockfish.yml +++ b/.github/workflows/stockfish.yml @@ -21,7 +21,7 @@ jobs: name: Prerelease if: github.repository == 'official-stockfish/Stockfish' && (github.ref == 'refs/heads/master' || (startsWith(github.ref_name, 'sf_') && github.ref_type == 'tag')) runs-on: ubuntu-latest - needs: [Matrix, Compilation, ARMCompilation, UniversalCompilation] + needs: [Matrix, ARMCompilation, UniversalCompilation] permissions: contents: write # For deleting/creating a prerelease steps: @@ -112,17 +112,12 @@ jobs: name: Prepare matrices runs-on: ubuntu-latest outputs: - macos_matrix: ${{ steps.set-macos-matrix.outputs.macos_matrix }} arm_matrix: ${{ steps.set-arm-matrix.outputs.arm_matrix }} universal_matrix: ${{ steps.set-universal-matrix.outputs.universal_matrix }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - id: set-macos-matrix - run: | - MACOS_MATRIX=$(jq -c '.' .github/ci/macos_matrix.json) - echo "MACOS_MATRIX=$MACOS_MATRIX" >> $GITHUB_OUTPUT - id: set-arm-matrix run: | ARM_MATRIX=$(jq -c '.' .github/ci/arm_matrix.json) @@ -156,12 +151,6 @@ jobs: if: ${{ always() }} uses: ./.github/workflows/avx2_compilers.yml # Release Jobs - Compilation: - name: macOS builds - needs: [Matrix, Sanitizers, Tests, Matetrack, Games, CompilerCheck] - uses: ./.github/workflows/compilation.yml - with: - matrix: ${{ needs.Matrix.outputs.macos_matrix }} ARMCompilation: name: Android builds needs: [Matrix, Sanitizers, Tests, Matetrack, Games, CompilerCheck] @@ -172,17 +161,6 @@ jobs: name: "Universal builds" needs: [Matrix, Sanitizers, Tests, Matetrack, Games, CompilerCheck] uses: ./.github/workflows/universal_compilation.yml - Binaries: - if: github.repository == 'official-stockfish/Stockfish' - name: macOS uploads - needs: [Prerelease, Matrix] - uses: ./.github/workflows/upload_binaries.yml - with: - matrix: ${{ needs.Matrix.outputs.macos_matrix }} - permissions: - contents: write # For deleting/creating a (pre)release - secrets: - token: ${{ secrets.GITHUB_TOKEN }} ARM_Binaries: if: github.repository == 'official-stockfish/Stockfish' name: Android uploads diff --git a/.github/workflows/universal_compilation.yml b/.github/workflows/universal_compilation.yml index 9a32b8b3c..28909bba4 100644 --- a/.github/workflows/universal_compilation.yml +++ b/.github/workflows/universal_compilation.yml @@ -217,3 +217,40 @@ jobs: . !.git !.output + + UniversalMacOS: + name: macOS + runs-on: macos-15 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + fetch-depth: 0 + + # macos-15 is Apple silicon; Rosetta 2 is needed to exercise the x86-64 slice + - name: Install Rosetta 2 + run: softwareupdate --install-rosetta --agree-to-license + + - name: Build universal binary + run: make -j4 -C src macos-lipo COMP=clang + + - name: Rename binary + run: mv src/stockfish stockfish-macos-universal + + - name: Verify binary runs natively and under Rosetta + run: ./scripts/check_universal_macos.sh ./stockfish-macos-universal + + - name: Remove non-src files + run: git clean -fxd -- src + + - name: Upload artifact for (pre)-release + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: macos universal + path: | + . + !.git + !.output diff --git a/scripts/check_universal_macos.sh b/scripts/check_universal_macos.sh new file mode 100755 index 000000000..93c2f6f03 --- /dev/null +++ b/scripts/check_universal_macos.sh @@ -0,0 +1,62 @@ +#!/bin/sh + +# Usage: check_universal_macos.sh STOCKFISH_EXE [EXPECTED_BENCH] + +set -eu + +STOCKFISH_EXE=$1 + +extract() { + awk -F: -v lbl="$1" '$0 ~ lbl { + sub(/^[[:space:]]+/, "", $2); sub(/[[:space:]]+$/, "", $2); print $2; exit + }' +} + +FAIL=0 + +BINARY_SIZE=$(wc -c < "$STOCKFISH_EXE") +MAX_SIZE=$((150 * 1024 * 1024)) +if [ "$BINARY_SIZE" -gt "$MAX_SIZE" ]; then + printf 'check_universal_macos.sh: binary size %d bytes exceeds 150 MB limit\n' "$BINARY_SIZE" >&2 + exit 1 +fi + +native_bench=$(arch -arm64 "$STOCKFISH_EXE" bench 2>&1 | extract "Nodes searched" || true) +if [ -z "$native_bench" ]; then + echo "check_universal_macos.sh: arm64 run produced no bench" >&2 + FAIL=1 +else + printf 'native (arm64) bench %s\n' "$native_bench" >&2 +fi + +x86_bench=$(arch -x86_64 "$STOCKFISH_EXE" bench 2>&1 | extract "Nodes searched" || true) +if [ -z "$x86_bench" ]; then + echo "check_universal_macos.sh: x86_64 (Rosetta) run produced no bench (crashed?)" >&2 + FAIL=1 +else + printf 'x86_64 (Rosetta) bench %s\n' "$x86_bench" >&2 +fi + +# Should match +if [ "$native_bench" != "$x86_bench" ]; then + printf 'check_universal_macos.sh: slice bench mismatch: arm64=%s x86_64=%s\n' \ + "$native_bench" "$x86_bench" >&2 + FAIL=1 +fi + +# Check that under Rosetta, SSE41 is detected +x86_compiler=$(arch -x86_64 "$STOCKFISH_EXE" compiler 2>&1 || true) +if printf '%s\n' "$x86_compiler" | grep -q 'SSE41'; then + printf 'x86_64 (Rosetta) compiler reports SSE41 ok\n' >&2 +else + echo "check_universal_macos.sh: x86_64 compiler output missing SSE41" >&2 + printf '%s\n' "$x86_compiler" >&2 + FAIL=1 +fi + +if [ "$FAIL" != 0 ]; then + echo "check_universal_macos.sh: failed" + exit 1 +fi + +echo "check_universal_macos.sh: Good! Universal macOS binary works." diff --git a/src/Makefile b/src/Makefile index 5329ebbe4..ce2170436 100644 --- a/src/Makefile +++ b/src/Makefile @@ -148,7 +148,7 @@ ifeq ($(ARCH), $(filter $(ARCH), \ x86-64-universal x86-64-avx512icl x86-64-vnni512 x86-64-avx512 x86-64-avxvnni \ x86-64-bmi2 x86-64-avx2 x86-64-sse41-popcnt x86-64-modern x86-64-ssse3 x86-64-sse3-popcnt \ x86-64 x86-32-sse41-popcnt x86-32-sse2 x86-32 ppc-64 ppc-64-altivec ppc-64-vsx ppc-32 e2k \ - armv7 armv7-neon armv8 armv8-dotprod armv8-universal apple-silicon general-64 general-32 riscv64 \ + armv7 armv7-neon armv8 armv8-dotprod arm64-universal apple-silicon general-64 general-32 riscv64 \ loongarch64 loongarch64-lsx loongarch64-lasx)) SUPPORTED_ARCH=true else @@ -372,7 +372,7 @@ ifeq ($(ARCH),armv8-dotprod) arm_version = 8 endif -ifeq ($(ARCH),armv8-universal) +ifeq ($(ARCH),arm64-universal) arch = armv8 prefetch = yes popcnt = yes @@ -559,12 +559,12 @@ ifeq ($(COMP),clang) endif ifeq ($(KERNEL),Darwin) - CXXFLAGS += -mmacosx-version-min=10.15 - LDFLAGS += -mmacosx-version-min=10.15 + mac_target_flags := -mmacosx-version-min=10.15 ifneq ($(arch),any) - CXXFLAGS += -arch $(arch) - LDFLAGS += -arch $(arch) + mac_target_flags += -arch $(arch) endif + CXXFLAGS += $(mac_target_flags) + LDFLAGS += $(mac_target_flags) XCRUN = xcrun endif @@ -640,6 +640,7 @@ ifeq ($(COMP),gcc) ifneq ($(gccisclang),) profile_make = clang-profile-make profile_use = clang-profile-use + CXXFLAGS += -fconstexpr-steps=500000000 else CXXFLAGS += -Wstack-usage=128000 -fno-ipa-cp-clone -fconstexpr-ops-limit=500000000 endif @@ -956,6 +957,7 @@ help: echo "help > Display architecture details" && \ echo "profile-build > standard build with profile-guided optimization" && \ echo "build > skip profile-guided optimization" && \ + echo "macos-lipo > macOS x86-64 + Apple silicon universal binary" && \ echo "net > Download the default nnue nets" && \ echo "strip > Strip executable" && \ echo "install > Install executable" && \ @@ -985,7 +987,7 @@ help: echo "ppc-32 > PPC 32-bit" && \ echo "armv7 > ARMv7 32-bit" && \ echo "armv7-neon > ARMv7 32-bit with popcnt and neon" && \ - echo "armv8-universal > ARMv8 64-bit with automatic runtime selection of the best architecture" && \ + echo "arm64-universal > ARMv8 64-bit with automatic runtime selection of the best architecture" && \ echo "armv8 > ARMv8 64-bit with popcnt and neon" && \ echo "armv8-dotprod > ARMv8 64-bit with popcnt, neon and dot product support" && \ echo "e2k > Elbrus 2000" && \ @@ -1024,7 +1026,7 @@ ifneq ($(SUPPORTED_ARCH), true) endif -.PHONY: help analyze build profile-build strip install clean net \ +.PHONY: help analyze build profile-build macos-lipo strip install clean net \ objclean profileclean config-sanity \ icx-profile-use icx-profile-make \ gcc-profile-use gcc-profile-make \ @@ -1055,7 +1057,32 @@ profile-build: net config-sanity objclean profileclean $(MAKE) ARCH=$(ARCH) COMP=$(COMP) profileclean endif -ifeq ($(comp), clang) +# Default network filename, read from evaluate.h's EvalFileDefaultName. +macos-lipo: NNUE_NET = $(shell grep -oE 'nn-[0-9a-f]+\.nnue' $(CURDIR)/evaluate.h | head -1) +macos-lipo: + @echo "" + @echo "Step 1/4. Building x86-64 slice (network shared from Apple-silicon slice) ..." + CXXFLAGS="-arch x86_64" $(MAKE) build ARCH=x86-64-universal COMP=$(COMP) MACOS_X86_SLICE=1 + mv $(EXE) $(EXE)-x86_64 + @echo "" + @echo "Step 2/4. Building Apple silicon slice ..." + $(MAKE) profile-build ARCH=apple-silicon COMP=$(COMP) + mv $(EXE) $(EXE)-arm64 + @echo "" + @echo "Step 3/4. Combining slices and locating the embedded network ..." + lipo -create $(EXE)-x86_64 $(EXE)-arm64 -output $(EXE) + $(SHELL) universal/patch_x86_slice.sh $(EXE) $(EXE)-x86_64 $(NNUE_NET) + @echo "" + @echo "Step 4/4. Re-combining slices with the patched x86-64 slice ..." + lipo -create $(EXE)-x86_64 $(EXE)-arm64 -output $(EXE) + @rm -f $(EXE)-x86_64 $(EXE)-arm64 + @echo "Universal macOS binary built: $(EXE)" + +ifeq ($(KERNEL),Darwin) +# Apple uses ld64 which doesn't have --save-temps + LTO_OBJ_SUFFIX := .lto.o + SAVE_TEMPS := -Wl,-object_path_lto,stockfish.lto.o +else ifeq ($(comp), clang) LTO_OBJ_SUFFIX := .lto.o ifeq ($(target_windows),yes) SAVE_TEMPS := -Wl,--lto-emit-asm # ld.lld doesn't have a save temps functionality @@ -1077,7 +1104,7 @@ MODNAME := $(subst -,_,$(notdir $(CURDIR))) universal-object-pgo: objclean profileclean universalclean @echo "" @echo "Step 1/4. Building instrumented executable ..." - $(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_make) + $(MAKE) ARCH=$(ARCH) COMP=$(COMP) ENV_LDFLAGS='$(ENV_LDFLAGS) $(NNUE_EMBED_OBJ)' $(profile_make) @echo "" @echo "Step 2/4. Running benchmark for pgo-build ..." $(PGOBENCH) > PGOBENCH.out 2>&1 @@ -1284,7 +1311,7 @@ endif # Usage: # make -j ARCH=x86-64-universal build # non-PGO # make -j ARCH=x86-64-universal profile-build # PGO (use RUN_PREFIX for intel SDE) -# make -j ARCH=armv8-universal build +# make -j ARCH=arm64-universal build TEMP_DIR := $(CURDIR)/temp_builds UNIVERSAL_EXE := $(CURDIR)/$(EXE) @@ -1319,7 +1346,9 @@ ARCH_OBJS := $(TEMP_DIR)/$(BASELINE_ARCH)/stockfish.o \ UNIVERSAL_FINAL_FLAGS := -fno-exceptions -Os \ -Wno-c++26-extensions -Wno-c23-extensions # suppress #embed warnings -ifeq ($(KERNEL),Linux) +ifeq ($(KERNEL),Darwin) + UNIVERSAL_FINAL_FLAGS += $(mac_target_flags) +else ifeq ($(KERNEL),Linux) UNIVERSAL_FINAL_FLAGS += -static-libstdc++ -static-libgcc ifeq ($(COMP),ndk) UNIVERSAL_FINAL_FLAGS += -static @@ -1337,11 +1366,16 @@ endif arch-suffix = $(subst -,_,$(1)) arch-namespace = Stockfish_$(call arch-suffix,$(1)) # Itanium ABI mangled name for Stockfish_::main(int, char**) -arch-mangled = _ZN$(shell printf %s '$(call arch-namespace,$(1))' | wc -c)$(call arch-namespace,$(1))4mainEiPPc +arch-mangled = _ZN$(shell printf %s '$(call arch-namespace,$(1))' | wc -c | tr -d ' ')$(call arch-namespace,$(1))4mainEiPPc +# Mach-O initializer section name: strip "x86-64" or "x86-64-" and prefix with "_i_" +# (Mach-O section names are restricted to <=16 chars, hence this naming scheme) +arch-mac-section = _i_$(subst -,_,$(patsubst x86-64,,$(patsubst x86-64-%,%,$(1)))) +ifeq ($(KERNEL),Darwin) + arch-defmain = -Wl,-alias,_$(call arch-mangled,$(1)),_main # lld-link, used on Windows+clang, doesn't accept --defsym, so # we define main directly in main.cpp instead and delete it later. -ifeq ($(target_windows)+$(COMP),yes+clang) +else ifeq ($(target_windows)+$(COMP),yes+clang) arch-defmain = -DUNIVERSAL_NEEDS_MAIN_SHIM else arch-defmain = -Wl,--defsym=main=$(call arch-mangled,$(1)) @@ -1352,7 +1386,14 @@ ifeq ($(EMBED_DIR_SUPPORTED),1) EMBED_DIR = --embed-dir=$(CURDIR) endif -arch-cxxflags = $(EMBED_DIR) -DStockfish=$(call arch-namespace,$(1)) -DUNIVERSAL_BINARY \ +# macos-lipo sets MACOS_X86_SLICE=1 when building the x86-64 slice so that the +# slice reads the network out of the Apple-silicon slice instead of embedding +# its own copy (see universal/nnue_embed.cpp and universal/patch_x86_slice.sh). +ifeq ($(MACOS_X86_SLICE),1) + SLICE_DEF := -DUNIVERSAL_BINARY_MACOS_X86_SLICE +endif + +arch-cxxflags = $(EMBED_DIR) -DStockfish=$(call arch-namespace,$(1)) -DUNIVERSAL_BINARY $(SLICE_DEF) \ $(call arch-defmain,$(1)) -Wno-c++26-extensions -Wno-c23-extensions UOBJ_TGT ?= universal-object-pgo @@ -1373,10 +1414,10 @@ $(NET_SENTINEL): | $(TEMP_DIR) @touch $@ $(NNUE_EMBED_OBJ): $(UNIVERSAL_SRC_DIR)/nnue_embed.cpp $(NET_SENTINEL) | $(TEMP_DIR) - $(CXX) -O2 -Wno-c++26-extensions $(EMBED_DIR) -c $< -o $@ + $(CXX) -O2 -std=c++20 -Wno-c++26-extensions $(SLICE_DEF) $(mac_target_flags) $(EMBED_DIR) -c $< -o $@ $(UNIVERSAL_ENTRY_OBJ): $(UNIVERSAL_ENTRY_SRC) | $(TEMP_DIR) - $(CXX) -O2 -c $< -o $@ + $(CXX) -O2 -std=c++20 $(mac_target_flags) -c $< -o $@ # Symlink tracked top-level items from src/ into temp_builds//. $(TEMP_DIR)/%/.setup: | $(TEMP_DIR) @@ -1393,6 +1434,11 @@ $(TEMP_DIR)/%/stockfish.o: $(TEMP_DIR)/%/.setup $(NNUE_EMBED_OBJ) $(NET_SENTINEL $(MAKE) -C $(TEMP_DIR)/$* $(UOBJ_TGT) \ ARCH=$* CXX=$(CXX) COMP=$(COMP) \ NNUE_EMBED_OBJ=$(NNUE_EMBED_OBJ) +ifeq ($(KERNEL),Darwin) +# Rename the initializer section + ld -r -rename_section __DATA __mod_init_func __DATA $(call arch-mac-section,$*) $@ -o $@.tmp + mv $@.tmp $@ +else # Drop COMDAT groups; clang LTO leaves them around internalized LOCAL # template instantiations which causes problems at the final link $(OBJCOPY) -R .group $@ @@ -1406,6 +1452,7 @@ else # Make the array inert $(OBJCOPY) --set-section-flags $(call arch-suffix,$*)_init=alloc,data $@ endif +endif $(UNIVERSAL_EXE): $(UNIVERSAL_ENTRY_OBJ) $(NNUE_EMBED_OBJ) $(ARCH_OBJS) $(CXX) -o $@ $(UNIVERSAL_ENTRY_OBJ) $(NNUE_EMBED_OBJ) $(ARCH_OBJS) $(UNIVERSAL_FINAL_FLAGS) diff --git a/src/nnue/network.cpp b/src/nnue/network.cpp index 05193aa04..382f4f445 100644 --- a/src/nnue/network.cpp +++ b/src/nnue/network.cpp @@ -46,23 +46,13 @@ // Note that this does not work in Microsoft Visual Studio. #if !defined(UNIVERSAL_BINARY) && !defined(_MSC_VER) && !defined(NNUE_EMBEDDING_OFF) INCBIN(EmbeddedNNUE, EvalFileDefaultName); +#elif defined(UNIVERSAL_BINARY_MACOS_X86_SLICE) +// Determined at runtime, see universal/nnue_embed.cpp +extern const unsigned char* const gEmbeddedNNUEData; +extern const unsigned int gEmbeddedNNUESize; #elif defined(UNIVERSAL_BINARY) - // When building for the universal binary, use C++26 #embed with weak symbols so that a - // separate, non-LTO nnue_embed.o (with strong symbols) can override them during the LTO link, - // (INCBIN can't deduplicate.) - #define WEAK_SYM __attribute__((weak)) -extern const unsigned char gEmbeddedNNUEData[] WEAK_SYM = - #ifdef __has_embed - { - #embed EvalFileDefaultName -}; -const int padding = 0; - #else - #include "../universal/network_dump.inc" - ; -const int padding = 1; // trailing NUL byte - #endif -extern const unsigned int gEmbeddedNNUESize WEAK_SYM = sizeof(gEmbeddedNNUEData) - padding; +extern const unsigned char gEmbeddedNNUEData[]; +extern const unsigned int gEmbeddedNNUESize; #else const unsigned char gEmbeddedNNUEData[1] = {0x0}; const unsigned int gEmbeddedNNUESize = 1; @@ -257,6 +247,11 @@ void Network::load_internal() { } }; +#ifdef UNIVERSAL_BINARY_MACOS_X86_SLICE + if (gEmbeddedNNUEData == nullptr) // failed embedded load + return; +#endif + MemoryBuffer buffer(const_cast(reinterpret_cast(gEmbeddedNNUEData)), size_t(gEmbeddedNNUESize)); diff --git a/src/universal/entry_arm64.cpp b/src/universal/entry_arm64.cpp index 1616f1614..158bc64d6 100644 --- a/src/universal/entry_arm64.cpp +++ b/src/universal/entry_arm64.cpp @@ -1,3 +1,21 @@ +/* + Stockfish, a UCI chess playing engine derived from Glaurung 2.1 + Copyright (C) 2004-2026 The Stockfish developers (see AUTHORS file) + + Stockfish is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Stockfish is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + #include #if defined(_WIN32) diff --git a/src/universal/entry_x86.cpp b/src/universal/entry_x86.cpp index 54fbc9283..a69af2c14 100644 --- a/src/universal/entry_x86.cpp +++ b/src/universal/entry_x86.cpp @@ -1,18 +1,62 @@ +/* + Stockfish, a UCI chess playing engine derived from Glaurung 2.1 + Copyright (C) 2004-2026 The Stockfish developers (see AUTHORS file) + + Stockfish is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Stockfish is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + #include #include -#define DEFINE_BUILD(x) \ - namespace Stockfish_##x { \ - extern int main(int argc, char* argv[]); \ - } \ - extern "C" void (*__start_##x##_init[])(void); \ - extern "C" void (*__stop_##x##_init[])(void); \ - int entry_##x(int argc, char* argv[]) { \ - unsigned count = __stop_##x##_init - __start_##x##_init; \ - for (unsigned i = 0; i < count; i++) \ - __start_##x##_init[i](); \ - return Stockfish_##x::main(argc, argv); \ - } +#ifdef __APPLE__ + // We locate each arch's initializer pointer array at runtime via getsectiondata(). + // Example name is "_i_sse41_popcnt", baseline build is just "_i_" + #include + #include + #include + +extern "C" const struct mach_header_64 _mh_execute_header; + + #define DEFINE_BUILD(x) \ + namespace Stockfish_##x { \ + extern int main(int argc, char* argv[]); \ + } \ + int entry_##x(int argc, char* argv[]) { \ + char name[17]; \ + const char* full = #x; \ + snprintf(name, sizeof(name), "_i_%s", full[6] ? full + 7 : ""); \ + unsigned long size = 0; \ + auto** fns = reinterpret_cast( \ + getsectiondata(&_mh_execute_header, "__DATA", name, &size)); \ + for (unsigned long i = 0; i < size / sizeof(*fns); i++) \ + fns[i](); \ + return Stockfish_##x::main(argc, argv); \ + } +#else + #define DEFINE_BUILD(x) \ + namespace Stockfish_##x { \ + extern int main(int argc, char* argv[]); \ + } \ + extern "C" void (*__start_##x##_init[])(void); \ + extern "C" void (*__stop_##x##_init[])(void); \ + int entry_##x(int argc, char* argv[]) { \ + unsigned count = __stop_##x##_init - __start_##x##_init; \ + for (unsigned i = 0; i < count; i++) \ + __start_##x##_init[i](); \ + return Stockfish_##x::main(argc, argv); \ + } +#endif DEFINE_BUILD(x86_64) DEFINE_BUILD(x86_64_sse41_popcnt) @@ -108,8 +152,25 @@ static int dispatch(const CpuFeatures& f, int argc, char* argv[]) { return entry_x86_64_avx512icl(argc, argv); } +static void maybe_promote_thread_to_avx512() { +#ifdef __APPLE__ + // Intel Macs supporting AVX512 don't advertise it in xgetbv and only + // do so once at least one avx512 instruction has been executed. + // See https://github.com/apple/darwin-xnu/blob/0a798f6738bc1db01281fc08ae024145e84df927/osfmk/i386/fpu.c#L176 + + int supported = 0; + size_t len = sizeof(supported); + if (sysctlbyname("hw.optional.avx512f", &supported, &len, nullptr, 0) == 0 && supported) + { + asm volatile(".byte 0x62, 0xf1, 0x7d, 0x48, 0x6f, 0xc0"); // vmovdqa32 zmm0,zmm0 + } +#endif +} + int main(int argc, char* argv[]) { + maybe_promote_thread_to_avx512(); + __builtin_cpu_init(); CpuFeatures features = query_cpu_features(); return dispatch(features, argc, argv); -} \ No newline at end of file +} diff --git a/src/universal/nnue_embed.cpp b/src/universal/nnue_embed.cpp index 4e4829def..edabebb44 100644 --- a/src/universal/nnue_embed.cpp +++ b/src/universal/nnue_embed.cpp @@ -1,16 +1,85 @@ +/* + Stockfish, a UCI chess playing engine derived from Glaurung 2.1 + Copyright (C) 2004-2026 The Stockfish developers (see AUTHORS file) + + Stockfish is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Stockfish is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + // Standalone NNUE embedding for universal binary builds #include "../evaluate.h" +#ifdef UNIVERSAL_BINARY_MACOS_X86_SLICE + +// In a macOS universal binary the network is embedded only in the arm64 slice, +// and the x86-64 slice mmaps it from the arm64 slice. + + #include + #include + #include + #include + #include + #include + #include + +// Must be kept in sync with patch_x86_slice.sh +extern const volatile uint64_t gUniversalNNUEOffset = 0xCAFE0FF5E70FF5E7ULL; +extern const volatile uint64_t gUniversalNNUESize = 0xCAFE512ECAFE512EULL; + +static const unsigned char* map_embedded_nnue() { + char path[PATH_MAX]; + uint32_t len = sizeof(path); + if (_NSGetExecutablePath(path, &len) != 0) + return nullptr; + + char resolved[PATH_MAX]; + const char* file = realpath(path, resolved) ? resolved : path; + + int fd = open(file, O_RDONLY); + if (fd < 0) + return nullptr; + + // Align down to page size for mmap + const uint64_t pageSize = uint64_t(sysconf(_SC_PAGESIZE)); + const uint64_t base = gUniversalNNUEOffset & ~(pageSize - 1); + const uint64_t pad = gUniversalNNUEOffset - base; + + void* p = + mmap(nullptr, size_t(gUniversalNNUESize + pad), PROT_READ, MAP_PRIVATE, fd, off_t(base)); + close(fd); + if (p == MAP_FAILED) + return nullptr; + + return reinterpret_cast(p) + pad; +} + +extern const unsigned char* const gEmbeddedNNUEData = map_embedded_nnue(); +extern const unsigned int gEmbeddedNNUESize = static_cast(gUniversalNNUESize); + +#else + extern const unsigned char gEmbeddedNNUEData[] = -#ifdef __has_embed + #ifdef __has_embed { - #embed EvalFileDefaultName + #embed EvalFileDefaultName }; const unsigned int padding = 0; -#else - #include "network_dump.inc" + #else + #include "network_dump.inc" ; const unsigned int padding = 1; // trailing NUL byte -#endif + #endif extern const unsigned int gEmbeddedNNUESize = sizeof(gEmbeddedNNUEData) - padding; + +#endif diff --git a/src/universal/patch_x86_slice.sh b/src/universal/patch_x86_slice.sh new file mode 100755 index 000000000..ec076a902 --- /dev/null +++ b/src/universal/patch_x86_slice.sh @@ -0,0 +1,73 @@ +#!/bin/sh + +# Stockfish, a UCI chess playing engine derived from Glaurung 2.1 +# Copyright (C) 2004-2026 The Stockfish developers (see AUTHORS file) +# +# Stockfish is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Stockfish is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# This finds the gUniversalNNUEOffset/gUniversalNNUESize symbols in the x86 slice +# and patches them to be an offset+size into the full executable. +# +# Usage: patch_x86_slice.sh + +set -eu + +FAT="$1" +X86="$2" +NET="$3" + +# Must match volatile initializers in nnue_embed.cpp; byte order +# is reversed to match how it appears in the binary xxd +OFFSET_MAGIC=e7f50fe7f50ffeca +SIZE_MAGIC=2e51feca2e51feca + +die() { echo "patch_x86_slice: $*" >&2; exit 1; } + +# Find hex pattern in file, return byte offset +# Optional $3 = bytes to skip, $4 = max bytes to scan +find_bytes() { + file=$1; needle=$2; skip=${3:-0}; window=${4:-0} + [ "$window" -gt 0 ] && win="-l $window" || win="" + # Disassemble as hex, strip \n and search for the pattern + pos=$(xxd -s "$skip" $win -p "$file" | tr -d '\n' \ + | awk -v n="$needle" '{ p = index($0, n); if (p) { print p; exit } }') + [ -n "$pos" ] || return 1 + echo $(( skip + (pos - 1) / 2 )) +} + +# Overwrite 8 bytes at a file offset +write_u64_le() { + f="$1"; off="$2"; v="$3"; bytes=""; i=0 + while [ "$i" -lt 8 ]; do + bytes="$bytes$(printf '\\%03o' "$(( v & 255 ))")" + v=$(( v >> 8 )); i=$(( i + 1 )) + done + printf "$bytes" | dd of="$f" bs=1 seek="$off" conv=notrunc 2>/dev/null +} + +needle=$(xxd -p -l 64 "$NET" | tr -d '\n') # take first bytes of network as needle +net_size=$(wc -c < "$NET" | tr -d ' ') + +# The arm64 slice always follows the x86-64 slice in +# the fat binary, so scan from ~after the x86-64 slice. +x86_size=$(wc -c < "$X86" | tr -d ' ') +net_off=$(find_bytes "$FAT" "$needle" "$x86_size" 8000000) || die "network not found" + +off_pos=$(find_bytes "$X86" "$OFFSET_MAGIC") || die "offset sentinel not found" +size_pos=$(find_bytes "$X86" "$SIZE_MAGIC") || die "size sentinel not found" + +write_u64_le "$X86" "$off_pos" "$net_off" +write_u64_le "$X86" "$size_pos" "$net_size" + +echo "patch_x86_slice.sh: network at offset $net_off + size $net_size " diff --git a/src/universal/rewrite_asm_sections.awk b/src/universal/rewrite_asm_sections.awk index 2b34fe5e6..01e5d1049 100644 --- a/src/universal/rewrite_asm_sections.awk +++ b/src/universal/rewrite_asm_sections.awk @@ -1,3 +1,19 @@ +# Stockfish, a UCI chess playing engine derived from Glaurung 2.1 +# Copyright (C) 2004-2026 The Stockfish developers (see AUTHORS file) +# +# Stockfish is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Stockfish is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + # This is used if the universal binary is compiled on clang+Windows. # Usage (avx2 example): # awk -v MODNAME=x86_64_avx2 -f rewrite_asm_sections.awk stockfish.exe.lto.s > renamed.s