macOS universal binary, take 2

- Add new target `macos-lipo`.
   - Created by compiling a universal x86 binary (no PGO) and a standard Apple silicon binary (with PGO), then combining them into a Mach-O fat binary
   - To keep only one copy of the net, we add custom loading logic in the x86 section. The executable reads its own path and mmaps the net that's in the ARM section.
       - The offset and size (from the executable base) of the mapping is injected after compilation in `patch_x86_slice.sh`
   - avx512 on macOS isn't advertised in the xcr0 register by default. The simple solution I came up with is to execute a dummy AVX512 instruction, which sets up the register, before calling `__builtin_cpu_init`.

Some housekeeping as well:
- Rename `armv8-universal` -> `arm64-universal`.
- Add standard copyright headers to the files we've added recently.

Potential follow-ups:
- Disservin's Makefile cleanup
- Alternative ideas for the net loading. In particular, this will error out if the user strips the binary (since that'll invalidate the offset).

closes https://github.com/official-stockfish/Stockfish/pull/6860

No functional change

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
anematode
2026-05-29 19:05:53 +02:00
committed by Joost VandeVondele
co-authored by Copilot Autofix powered by AI
parent 24a551899e
commit 8cbca11a53
14 changed files with 441 additions and 198 deletions
+18
View File
@@ -1,3 +1,21 @@
/*
Stockfish, a UCI chess playing engine derived from Glaurung 2.1
Copyright (C) 2004-2026 The Stockfish developers (see AUTHORS file)
Stockfish is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Stockfish is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stdint.h>
#if defined(_WIN32)
+74 -13
View File
@@ -1,18 +1,62 @@
/*
Stockfish, a UCI chess playing engine derived from Glaurung 2.1
Copyright (C) 2004-2026 The Stockfish developers (see AUTHORS file)
Stockfish is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Stockfish is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <cpuid.h>
#include <stdint.h>
#define DEFINE_BUILD(x) \
namespace Stockfish_##x { \
extern int main(int argc, char* argv[]); \
} \
extern "C" void (*__start_##x##_init[])(void); \
extern "C" void (*__stop_##x##_init[])(void); \
int entry_##x(int argc, char* argv[]) { \
unsigned count = __stop_##x##_init - __start_##x##_init; \
for (unsigned i = 0; i < count; i++) \
__start_##x##_init[i](); \
return Stockfish_##x::main(argc, argv); \
}
#ifdef __APPLE__
// We locate each arch's initializer pointer array at runtime via getsectiondata().
// Example name is "_i_sse41_popcnt", baseline build is just "_i_"
#include <stdio.h>
#include <mach-o/getsect.h>
#include <sys/sysctl.h>
extern "C" const struct mach_header_64 _mh_execute_header;
#define DEFINE_BUILD(x) \
namespace Stockfish_##x { \
extern int main(int argc, char* argv[]); \
} \
int entry_##x(int argc, char* argv[]) { \
char name[17]; \
const char* full = #x; \
snprintf(name, sizeof(name), "_i_%s", full[6] ? full + 7 : ""); \
unsigned long size = 0; \
auto** fns = reinterpret_cast<void (**)()>( \
getsectiondata(&_mh_execute_header, "__DATA", name, &size)); \
for (unsigned long i = 0; i < size / sizeof(*fns); i++) \
fns[i](); \
return Stockfish_##x::main(argc, argv); \
}
#else
#define DEFINE_BUILD(x) \
namespace Stockfish_##x { \
extern int main(int argc, char* argv[]); \
} \
extern "C" void (*__start_##x##_init[])(void); \
extern "C" void (*__stop_##x##_init[])(void); \
int entry_##x(int argc, char* argv[]) { \
unsigned count = __stop_##x##_init - __start_##x##_init; \
for (unsigned i = 0; i < count; i++) \
__start_##x##_init[i](); \
return Stockfish_##x::main(argc, argv); \
}
#endif
DEFINE_BUILD(x86_64)
DEFINE_BUILD(x86_64_sse41_popcnt)
@@ -108,8 +152,25 @@ static int dispatch(const CpuFeatures& f, int argc, char* argv[]) {
return entry_x86_64_avx512icl(argc, argv);
}
static void maybe_promote_thread_to_avx512() {
#ifdef __APPLE__
// Intel Macs supporting AVX512 don't advertise it in xgetbv and only
// do so once at least one avx512 instruction has been executed.
// See https://github.com/apple/darwin-xnu/blob/0a798f6738bc1db01281fc08ae024145e84df927/osfmk/i386/fpu.c#L176
int supported = 0;
size_t len = sizeof(supported);
if (sysctlbyname("hw.optional.avx512f", &supported, &len, nullptr, 0) == 0 && supported)
{
asm volatile(".byte 0x62, 0xf1, 0x7d, 0x48, 0x6f, 0xc0"); // vmovdqa32 zmm0,zmm0
}
#endif
}
int main(int argc, char* argv[]) {
maybe_promote_thread_to_avx512();
__builtin_cpu_init();
CpuFeatures features = query_cpu_features();
return dispatch(features, argc, argv);
}
}
+74 -5
View File
@@ -1,16 +1,85 @@
/*
Stockfish, a UCI chess playing engine derived from Glaurung 2.1
Copyright (C) 2004-2026 The Stockfish developers (see AUTHORS file)
Stockfish is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Stockfish is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// Standalone NNUE embedding for universal binary builds
#include "../evaluate.h"
#ifdef UNIVERSAL_BINARY_MACOS_X86_SLICE
// In a macOS universal binary the network is embedded only in the arm64 slice,
// and the x86-64 slice mmaps it from the arm64 slice.
#include <climits>
#include <cstdint>
#include <fcntl.h>
#include <mach-o/dyld.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <unistd.h>
// Must be kept in sync with patch_x86_slice.sh
extern const volatile uint64_t gUniversalNNUEOffset = 0xCAFE0FF5E70FF5E7ULL;
extern const volatile uint64_t gUniversalNNUESize = 0xCAFE512ECAFE512EULL;
static const unsigned char* map_embedded_nnue() {
char path[PATH_MAX];
uint32_t len = sizeof(path);
if (_NSGetExecutablePath(path, &len) != 0)
return nullptr;
char resolved[PATH_MAX];
const char* file = realpath(path, resolved) ? resolved : path;
int fd = open(file, O_RDONLY);
if (fd < 0)
return nullptr;
// Align down to page size for mmap
const uint64_t pageSize = uint64_t(sysconf(_SC_PAGESIZE));
const uint64_t base = gUniversalNNUEOffset & ~(pageSize - 1);
const uint64_t pad = gUniversalNNUEOffset - base;
void* p =
mmap(nullptr, size_t(gUniversalNNUESize + pad), PROT_READ, MAP_PRIVATE, fd, off_t(base));
close(fd);
if (p == MAP_FAILED)
return nullptr;
return reinterpret_cast<const unsigned char*>(p) + pad;
}
extern const unsigned char* const gEmbeddedNNUEData = map_embedded_nnue();
extern const unsigned int gEmbeddedNNUESize = static_cast<unsigned int>(gUniversalNNUESize);
#else
extern const unsigned char gEmbeddedNNUEData[] =
#ifdef __has_embed
#ifdef __has_embed
{
#embed EvalFileDefaultName
#embed EvalFileDefaultName
};
const unsigned int padding = 0;
#else
#include "network_dump.inc"
#else
#include "network_dump.inc"
;
const unsigned int padding = 1; // trailing NUL byte
#endif
#endif
extern const unsigned int gEmbeddedNNUESize = sizeof(gEmbeddedNNUEData) - padding;
#endif
+73
View File
@@ -0,0 +1,73 @@
#!/bin/sh
# Stockfish, a UCI chess playing engine derived from Glaurung 2.1
# Copyright (C) 2004-2026 The Stockfish developers (see AUTHORS file)
#
# Stockfish is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Stockfish is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# This finds the gUniversalNNUEOffset/gUniversalNNUESize symbols in the x86 slice
# and patches them to be an offset+size into the full executable.
#
# Usage: patch_x86_slice.sh <fat_binary> <x86_64_thin_slice> <nnue_file>
set -eu
FAT="$1"
X86="$2"
NET="$3"
# Must match volatile initializers in nnue_embed.cpp; byte order
# is reversed to match how it appears in the binary xxd
OFFSET_MAGIC=e7f50fe7f50ffeca
SIZE_MAGIC=2e51feca2e51feca
die() { echo "patch_x86_slice: $*" >&2; exit 1; }
# Find hex pattern in file, return byte offset
# Optional $3 = bytes to skip, $4 = max bytes to scan
find_bytes() {
file=$1; needle=$2; skip=${3:-0}; window=${4:-0}
[ "$window" -gt 0 ] && win="-l $window" || win=""
# Disassemble as hex, strip \n and search for the pattern
pos=$(xxd -s "$skip" $win -p "$file" | tr -d '\n' \
| awk -v n="$needle" '{ p = index($0, n); if (p) { print p; exit } }')
[ -n "$pos" ] || return 1
echo $(( skip + (pos - 1) / 2 ))
}
# Overwrite 8 bytes at a file offset
write_u64_le() {
f="$1"; off="$2"; v="$3"; bytes=""; i=0
while [ "$i" -lt 8 ]; do
bytes="$bytes$(printf '\\%03o' "$(( v & 255 ))")"
v=$(( v >> 8 )); i=$(( i + 1 ))
done
printf "$bytes" | dd of="$f" bs=1 seek="$off" conv=notrunc 2>/dev/null
}
needle=$(xxd -p -l 64 "$NET" | tr -d '\n') # take first bytes of network as needle
net_size=$(wc -c < "$NET" | tr -d ' ')
# The arm64 slice always follows the x86-64 slice in
# the fat binary, so scan from ~after the x86-64 slice.
x86_size=$(wc -c < "$X86" | tr -d ' ')
net_off=$(find_bytes "$FAT" "$needle" "$x86_size" 8000000) || die "network not found"
off_pos=$(find_bytes "$X86" "$OFFSET_MAGIC") || die "offset sentinel not found"
size_pos=$(find_bytes "$X86" "$SIZE_MAGIC") || die "size sentinel not found"
write_u64_le "$X86" "$off_pos" "$net_off"
write_u64_le "$X86" "$size_pos" "$net_size"
echo "patch_x86_slice.sh: network at offset $net_off + size $net_size "
+16
View File
@@ -1,3 +1,19 @@
# Stockfish, a UCI chess playing engine derived from Glaurung 2.1
# Copyright (C) 2004-2026 The Stockfish developers (see AUTHORS file)
#
# Stockfish is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Stockfish is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# This is used if the universal binary is compiled on clang+Windows.
# Usage (avx2 example):
# awk -v MODNAME=x86_64_avx2 -f rewrite_asm_sections.awk stockfish.exe.lto.s > renamed.s