diff --git a/src/universal/entry_x86.cpp b/src/universal/entry_x86.cpp index fb1d8bc93..54fbc9283 100644 --- a/src/universal/entry_x86.cpp +++ b/src/universal/entry_x86.cpp @@ -1,5 +1,4 @@ #include -#include #include #define DEFINE_BUILD(x) \ @@ -24,88 +23,93 @@ DEFINE_BUILD(x86_64_avx512) DEFINE_BUILD(x86_64_vnni512) DEFINE_BUILD(x86_64_avx512icl) -// Zen, Zen+ and Zen 2 (AMD family 17h) have microcoded pdep/pext +// AMD Zen/Zen+/Zen2 (family 17h) implement pdep/pext via microcode. static bool has_slow_bmi2() { - unsigned int eax, ebx, ecx, edx; - __cpuid(0, eax, ebx, ecx, edx); - if (ebx != 0x68747541 || edx != 0x69746e65 || ecx != 0x444d4163) // "AuthenticAMD" - return false; - __cpuid(1, eax, ebx, ecx, edx); - unsigned family = (eax >> 8) & 0xf; - if (family == 0xf) - family += (eax >> 20) & 0xff; - return family == 0x17; + return __builtin_cpu_is("amd") && (__builtin_cpu_is("znver1") || __builtin_cpu_is("znver2")); } -constexpr uint64_t XCR0_SSE_AVX_MASK = 0x06; -constexpr uint64_t XCR0_AVX512_MASK = 0xE6; +struct CpuFeatures { + bool sse41; // SSE4.1 + bool popcnt; // POPCNT + bool avx2; // AVX2 + bool bmi2; // BMI2 (may be slow on AMD Zen/Zen+/Zen2) + bool avx512f; // AVX-512 Foundation + bool avx512vl; // AVX-512 Vector Length extensions + bool avx512bw; // AVX-512 Byte and Word instructions + bool avx512vnni; // AVX-512 Vector Neural Network Instructions + bool avx512ifma; // AVX-512 Integer Fused Multiply-Add + bool avx512vbmi; // AVX-512 Vector Bit Manipulation Instructions + bool avx512vbmi2; // AVX-512 VBMI2 + bool avx512vpopcntdq; // AVX-512 VPOPCNTDQ + bool avx512bitalg; // AVX-512 BITALG + bool vpclmulqdq; // Carry-less multiplication (AVX512 variant) + bool gfni; // Galois Field instructions + bool vaes; // AES instructions (AVX512 variant) + bool avxvnni; // AVX-VNNI (non-512 dot product instructions) +}; -int main(int argc, char* argv[]) { - unsigned _; - unsigned max_leaf = __get_cpuid_max(0, &_); - if (max_leaf < 1U) - { + +static CpuFeatures query_cpu_features() { + return { + .sse41 = (bool) __builtin_cpu_supports("sse4.1"), + .popcnt = (bool) __builtin_cpu_supports("popcnt"), + .avx2 = (bool) __builtin_cpu_supports("avx2"), + .bmi2 = (bool) __builtin_cpu_supports("bmi2"), + .avx512f = (bool) __builtin_cpu_supports("avx512f"), + .avx512vl = (bool) __builtin_cpu_supports("avx512vl"), + .avx512bw = (bool) __builtin_cpu_supports("avx512bw"), + .avx512vnni = (bool) __builtin_cpu_supports("avx512vnni"), + .avx512ifma = (bool) __builtin_cpu_supports("avx512ifma"), + .avx512vbmi = (bool) __builtin_cpu_supports("avx512vbmi"), + .avx512vbmi2 = (bool) __builtin_cpu_supports("avx512vbmi2"), + .avx512vpopcntdq = (bool) __builtin_cpu_supports("avx512vpopcntdq"), + .avx512bitalg = (bool) __builtin_cpu_supports("avx512bitalg"), + .vpclmulqdq = (bool) __builtin_cpu_supports("vpclmulqdq"), + .gfni = (bool) __builtin_cpu_supports("gfni"), + .vaes = (bool) __builtin_cpu_supports("vaes"), + .avxvnni = (bool) __builtin_cpu_supports("avxvnni"), + }; +} + + +// Selects the most capable ISA variant supported by this CPU and OS +static int dispatch(const CpuFeatures& f, int argc, char* argv[]) { + if (!f.sse41 || !f.popcnt) return entry_x86_64(argc, argv); - } - unsigned int eax, ebx, ecx, edx; - __cpuid(1, eax, ebx, ecx, edx); - if (!(ecx & (1U << 19)) || !(ecx & (1U << 23))) - { // no popcnt or no sse4.1 - return entry_x86_64(argc, argv); - } - bool xgetbv_ok = (ecx & (1U << 27)) != 0; // OSXSAVE - uint64_t xcr0 = 0; - - if (xgetbv_ok) - { - uint32_t xcr0_eax, xcr0_edx; - uint32_t xcr0_ecx = 0; - asm("xgetbv" : "=a"(xcr0_eax), "=d"(xcr0_edx) : "c"(xcr0_ecx)); - xcr0 = (static_cast(xcr0_edx) << 32) | xcr0_eax; - } - - - bool leaf_supported = __get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx); - if (!leaf_supported || !(ebx & (1U << 5)) || !xgetbv_ok - || (xcr0 & XCR0_SSE_AVX_MASK) != XCR0_SSE_AVX_MASK) - { - // CPUID query not supported, no avx2, missing xgetbv, or missing OS restore for AVX regs + if (!f.avx2) return entry_x86_64_sse41_popcnt(argc, argv); - } - if (!(ebx & (1U << 8)) || has_slow_bmi2()) - { // no or slow bmi2 + if (!f.bmi2 || has_slow_bmi2()) return entry_x86_64_avx2(argc, argv); - } - if (!(ebx & (1U << 16)) || !(ebx & (1U << 31)) || !(ebx & (1U << 30)) - || (xcr0 & XCR0_AVX512_MASK) != XCR0_AVX512_MASK) + if (!f.avx512f || !f.avx512vl || !f.avx512bw) { - // no avx512f/vl/bw, or OS doesn't restore AVX-512 regs - bool leaf_supported = __get_cpuid_count(7, 1, &eax, &ebx, &ecx, &edx); - if (leaf_supported && eax & (1U << 4)) - { // avxvnni + if (f.avxvnni) return entry_x86_64_avxvnni(argc, argv); - } - else - { - return entry_x86_64_bmi2(argc, argv); - } + return entry_x86_64_bmi2(argc, argv); } - if (!(ecx & 1U << 11 /* vnni512 */)) - { + if (!f.avx512vnni) return entry_x86_64_avx512(argc, argv); - } - if (!(ebx & 1U << 21 /* ifma */) || !(ecx & 1U << 1 /* vbmi */) || !(ecx & 1U << 6 /* vbmi2 */) - || !(ecx & 1U << 14 /* vpopcntdq */) || !(ecx & 1U << 12 /* bitalg */) - || !(ecx & 1U << 10 /* vpclmulqdq */) || !(ecx & 1U << 8 /* gfni */) - || !(ecx & 1U << 9 /* vaes */)) - { + // AVX512ICL requires the full Icelake-client feature suite + if (!f.avx512ifma // + || !f.avx512vbmi // + || !f.avx512vbmi2 // + || !f.avx512vpopcntdq // + || !f.avx512bitalg // + || !f.vpclmulqdq // + || !f.gfni // + || !f.vaes // + ) return entry_x86_64_vnni512(argc, argv); - } return entry_x86_64_avx512icl(argc, argv); } + +int main(int argc, char* argv[]) { + __builtin_cpu_init(); + CpuFeatures features = query_cpu_features(); + return dispatch(features, argc, argv); +} \ No newline at end of file