Simon Jakobi pushed to branch wip/sjakobi/T25450-march-native at Glasgow Haskell Compiler / GHC Commits: 7bab9752 by Simon Jakobi at 2026-06-10T02:05:57+02:00 Vendor x86 CPU feature detection from LLVM's compiler-rt Following review feedback, replace the hand-rolled CPUID/XGETBV probe in cpu_features_x86.c with code vendored from compiler-rt's plain-C CPU model implementation: compiler-rt/lib/builtins/cpu_model/x86.c at tag llvmorg-20.1.0 This is the runtime behind clang's __builtin_cpu_supports and __builtin_cpu_is; upstream notes it is a copy of llvm/lib/TargetParser/Host.cpp (the code behind clang -march=native) and that the two files are kept in sync. The file header references the upstream source so the code can be diffed against it when debugging or updating. The vendored code is byte-for-byte upstream: enum ProcessorFeatures, the CPUID/XGETBV helpers (getX86CpuIDAndInfo, getX86CpuIDAndInfoEx, getX86XCR0) and the entire getAvailableFeatures function are verbatim. The single GHC deviation is a separate marked block that clears FEATURE_FMA again when OS AVX-state support is missing -- FMA instructions are VEX-encoded and unusable without it -- matching Host.cpp and GCC's cpuinfo.h, where upstream x86.c uses the raw CPUID bit alone. To make this possible, cpuFeatureBitLayout in GHC.Driver.CpuFeatures adopts LLVM/GCC's stable __builtin_cpu_supports feature numbering (BMI1 = upstream's FEATURE_BMI). ghc_detect_x86_cpu_features stands in for __cpu_indicator_init as the driver, returning the first 64 feature bits of that numbering as a mask (every feature GHC decodes is below bit 64; GFNI is the highest at 32). Features decoded by upstream code beyond GHC's set are simply ignored on the Haskell side, so updating the vendored code is a pure copy-paste and adding a new feature to GHC only requires a Haskell-side change. Behavioural changes from adopting LLVM's logic: * On x86_64 macOS we no longer query sysctl(hw.optional.avx512f) to work around the kernel's lazy AVX-512 XSAVE enablement. Like LLVM, we trust that Darwin will save the AVX-512 context on first use and rely on the CPUID leaf-7 bits alone (HasAVX512Save = true on __APPLE__). This also drops the sys/sysctl.h dependency. * The AVX-512 sub-features (BW/CD/DQ/VL) are each gated individually on OS support for the AVX-512 context save, instead of being nested under an AVX512F check. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> - - - - - 2 changed files: - compiler/GHC/Driver/CpuFeatures.hs - compiler/cbits/cpu_features_x86.c Changes: ===================================== compiler/GHC/Driver/CpuFeatures.hs ===================================== @@ -32,7 +32,10 @@ data X86CpuFeature -- | Decode the bitmask returned by 'ghc_detect_x86_cpu_features'. -- --- NOTE: Bit positions must match the enum in @compiler/cbits/cpu_features_x86.c@. +-- NOTE: Bit positions are LLVM\/GCC's @__builtin_cpu_supports@ feature +-- numbering, i.e. @enum ProcessorFeatures@ vendored in +-- @compiler/cbits/cpu_features_x86.c@. The C side returns the first 64 +-- feature bits of that numbering. decodeX86CpuFeatureMask :: Word64 -> [X86CpuFeature] decodeX86CpuFeatureMask mask = [ feat @@ -61,24 +64,27 @@ cachedX86CpuFeatures :: [X86CpuFeature] cachedX86CpuFeatures = unsafePerformIO detectX86CpuFeatures {-# NOINLINE cachedX86CpuFeatures #-} +-- | See the NOTE on 'decodeX86CpuFeatureMask' for where these bit positions +-- come from. The constant names on the C side are upstream's: @FEATURE_SSE2@, +-- @FEATURE_BMI@ (= 'BMI1'), etc. cpuFeatureBitLayout :: [(Int, X86CpuFeature)] cpuFeatureBitLayout = - [ (0, SSE2) - , (1, SSE3) - , (2, SSSE3) - , (3, SSE4_1) - , (4, SSE4_2) - , (5, AVX) - , (6, AVX2) - , (7, AVX512F) - , (8, AVX512BW) - , (9, AVX512CD) - , (10, AVX512DQ) - , (11, AVX512VL) - , (12, BMI1) - , (13, BMI2) - , (14, FMA) - , (15, GFNI) + [ (4, SSE2) -- FEATURE_SSE2 + , (5, SSE3) -- FEATURE_SSE3 + , (6, SSSE3) -- FEATURE_SSSE3 + , (7, SSE4_1) -- FEATURE_SSE4_1 + , (8, SSE4_2) -- FEATURE_SSE4_2 + , (9, AVX) -- FEATURE_AVX + , (10, AVX2) -- FEATURE_AVX2 + , (14, FMA) -- FEATURE_FMA + , (15, AVX512F) -- FEATURE_AVX512F + , (16, BMI1) -- FEATURE_BMI + , (17, BMI2) -- FEATURE_BMI2 + , (20, AVX512VL) -- FEATURE_AVX512VL + , (21, AVX512BW) -- FEATURE_AVX512BW + , (22, AVX512DQ) -- FEATURE_AVX512DQ + , (23, AVX512CD) -- FEATURE_AVX512CD + , (32, GFNI) -- FEATURE_GFNI ] #if !defined(javascript_HOST_ARCH) ===================================== compiler/cbits/cpu_features_x86.c ===================================== @@ -1,208 +1,586 @@ +/* Host x86 CPU feature detection, used to implement -march=native. + * + * The detection code is vendored from LLVM's compiler-rt, where it implements + * the runtime support for __builtin_cpu_supports/__builtin_cpu_is + * (__cpu_model, __cpu_indicator_init): + * + * compiler-rt/lib/builtins/cpu_model/x86.c + * at tag llvmorg-20.1.0 (LLVM 20.1.0) + * https://github.com/llvm/llvm-project/blob/llvmorg-20.1.0/compiler-rt/lib/bui... + * + * LLVM is licensed under Apache-2.0 WITH LLVM-exception. Upstream notes that + * this file is itself a copy of llvm/lib/TargetParser/Host.cpp -- the code + * behind clang's -march=native -- and that the two are kept in sync. + * + * Vendored verbatim: enum ProcessorFeatures, getX86CpuIDAndInfo, + * getX86CpuIDAndInfoEx, getX86XCR0 and getAvailableFeatures, the latter with + * a single marked GHC deviation that additionally gates FEATURE_FMA on OS + * support for saving the AVX register state. + * + * Adaptations around the vendored code: + * + * - ghc_detect_x86_cpu_features stands in for __cpu_indicator_init as the + * driver: it performs the same CPUID call sequence, but returns the first + * 64 feature bits as a mask -- which covers every feature GHC decodes, + * see GHC.Driver.CpuFeatures -- instead of filling in the __cpu_model + * globals. + * - On non-x86 hosts the driver compiles to a stub returning 0 instead of + * upstream's "#error This file is intended only for x86-based targets". + */ + #include <HsFFI.h> -#include <stdint.h> +#include <stdbool.h> -#if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) -#include <immintrin.h> -#include <intrin.h> +#if defined(__i386__) || defined(_M_IX86) || defined(__x86_64__) || \ + defined(_M_X64) +#define GHC_HOST_IS_X86 1 #endif -#if !defined(_MSC_VER) && (defined(__i386__) || defined(__x86_64__)) +#if defined(GHC_HOST_IS_X86) + +#if (defined(__GNUC__) || defined(__clang__)) && !defined(_MSC_VER) #include <cpuid.h> #endif -#if defined(__APPLE__) && (defined(__i386__) || defined(__x86_64__)) -#include <sys/sysctl.h> +#ifdef _MSC_VER +#include <intrin.h> #endif -enum { - GHC_X86_FEAT_SSE2 = 0, - GHC_X86_FEAT_SSE3, - GHC_X86_FEAT_SSSE3, - GHC_X86_FEAT_SSE4_1, - GHC_X86_FEAT_SSE4_2, - GHC_X86_FEAT_AVX, - GHC_X86_FEAT_AVX2, - GHC_X86_FEAT_AVX512F, - GHC_X86_FEAT_AVX512BW, - GHC_X86_FEAT_AVX512CD, - GHC_X86_FEAT_AVX512DQ, - GHC_X86_FEAT_AVX512VL, - GHC_X86_FEAT_BMI1, - GHC_X86_FEAT_BMI2, - GHC_X86_FEAT_FMA, - GHC_X86_FEAT_GFNI +/* NOTE: The feature bit positions below are LLVM/GCC's stable + * __builtin_cpu_supports feature numbering. cpuFeatureBitLayout in + * GHC.Driver.CpuFeatures must use the same values. */ +enum ProcessorFeatures { + FEATURE_CMOV = 0, + FEATURE_MMX, + FEATURE_POPCNT, + FEATURE_SSE, + FEATURE_SSE2, + FEATURE_SSE3, + FEATURE_SSSE3, + FEATURE_SSE4_1, + FEATURE_SSE4_2, + FEATURE_AVX, + FEATURE_AVX2, + FEATURE_SSE4_A, + FEATURE_FMA4, + FEATURE_XOP, + FEATURE_FMA, + FEATURE_AVX512F, + FEATURE_BMI, + FEATURE_BMI2, + FEATURE_AES, + FEATURE_PCLMUL, + FEATURE_AVX512VL, + FEATURE_AVX512BW, + FEATURE_AVX512DQ, + FEATURE_AVX512CD, + FEATURE_AVX512ER, + FEATURE_AVX512PF, + FEATURE_AVX512VBMI, + FEATURE_AVX512IFMA, + FEATURE_AVX5124VNNIW, + FEATURE_AVX5124FMAPS, + FEATURE_AVX512VPOPCNTDQ, + FEATURE_AVX512VBMI2, + FEATURE_GFNI, + FEATURE_VPCLMULQDQ, + FEATURE_AVX512VNNI, + FEATURE_AVX512BITALG, + FEATURE_AVX512BF16, + FEATURE_AVX512VP2INTERSECT, + // FIXME: Below Features has some missings comparing to gcc, it's because gcc + // has some not one-to-one mapped in llvm. + // FEATURE_3DNOW, + // FEATURE_3DNOWP, + FEATURE_ADX = 40, + // FEATURE_ABM, + FEATURE_CLDEMOTE = 42, + FEATURE_CLFLUSHOPT, + FEATURE_CLWB, + FEATURE_CLZERO, + FEATURE_CMPXCHG16B, + // FIXME: Not adding FEATURE_CMPXCHG8B is a workaround to make 'generic' as + // a cpu string with no X86_FEATURE_COMPAT features, which is required in + // current implementantion of cpu_specific/cpu_dispatch FMV feature. + // FEATURE_CMPXCHG8B, + FEATURE_ENQCMD = 48, + FEATURE_F16C, + FEATURE_FSGSBASE, + // FEATURE_FXSAVE, + // FEATURE_HLE, + // FEATURE_IBT, + FEATURE_LAHF_LM = 54, + FEATURE_LM, + FEATURE_LWP, + FEATURE_LZCNT, + FEATURE_MOVBE, + FEATURE_MOVDIR64B, + FEATURE_MOVDIRI, + FEATURE_MWAITX, + // FEATURE_OSXSAVE, + FEATURE_PCONFIG = 63, + FEATURE_PKU, + FEATURE_PREFETCHWT1, + FEATURE_PRFCHW, + FEATURE_PTWRITE, + FEATURE_RDPID, + FEATURE_RDRND, + FEATURE_RDSEED, + FEATURE_RTM, + FEATURE_SERIALIZE, + FEATURE_SGX, + FEATURE_SHA, + FEATURE_SHSTK, + FEATURE_TBM, + FEATURE_TSXLDTRK, + FEATURE_VAES, + FEATURE_WAITPKG, + FEATURE_WBNOINVD, + FEATURE_XSAVE, + FEATURE_XSAVEC, + FEATURE_XSAVEOPT, + FEATURE_XSAVES, + FEATURE_AMX_TILE, + FEATURE_AMX_INT8, + FEATURE_AMX_BF16, + FEATURE_UINTR, + FEATURE_HRESET, + FEATURE_KL, + // FEATURE_AESKLE, + FEATURE_WIDEKL = 92, + FEATURE_AVXVNNI, + FEATURE_AVX512FP16, + FEATURE_X86_64_BASELINE, + FEATURE_X86_64_V2, + FEATURE_X86_64_V3, + FEATURE_X86_64_V4, + FEATURE_AVXIFMA, + FEATURE_AVXVNNIINT8, + FEATURE_AVXNECONVERT, + FEATURE_CMPCCXADD, + FEATURE_AMX_FP16, + FEATURE_PREFETCHI, + FEATURE_RAOINT, + FEATURE_AMX_COMPLEX, + FEATURE_AVXVNNIINT16, + FEATURE_SM3, + FEATURE_SHA512, + FEATURE_SM4, + FEATURE_APXF, + FEATURE_USERMSR, + FEATURE_AVX10_1_256, + FEATURE_AVX10_1_512, + FEATURE_AVX10_2_256, + FEATURE_AVX10_2_512, + FEATURE_MOVRS, + CPU_FEATURE_MAX }; -#define SET_FEAT(mask, bit) ((mask) |= ((HsWord64)1ULL << (bit))) +// This code is copied from lib/Support/Host.cpp. +// Changes to either file should be mirrored in the other. -static int ghc_cpuid_count(uint32_t leaf, uint32_t subleaf, - uint32_t *a, uint32_t *b, uint32_t *c, uint32_t *d) -{ -#if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) - int regs[4]; - __cpuidex(regs, (int)leaf, (int)subleaf); - *a = (uint32_t)regs[0]; - *b = (uint32_t)regs[1]; - *c = (uint32_t)regs[2]; - *d = (uint32_t)regs[3]; - return 1; -#elif defined(__i386__) || defined(__x86_64__) - return __get_cpuid_count(leaf, subleaf, a, b, c, d); +/// getX86CpuIDAndInfo - Execute the specified cpuid and return the 4 values in +/// the specified arguments. If we can't run cpuid on the host, return true. +static bool getX86CpuIDAndInfo(unsigned value, unsigned *rEAX, unsigned *rEBX, + unsigned *rECX, unsigned *rEDX) { +#if (defined(__GNUC__) || defined(__clang__)) && !defined(_MSC_VER) + return !__get_cpuid(value, rEAX, rEBX, rECX, rEDX); +#elif defined(_MSC_VER) + // The MSVC intrinsic is portable across x86 and x64. + int registers[4]; + __cpuid(registers, value); + *rEAX = registers[0]; + *rEBX = registers[1]; + *rECX = registers[2]; + *rEDX = registers[3]; + return false; #else - (void)leaf; - (void)subleaf; - (void)a; - (void)b; - (void)c; - (void)d; - return 0; + return true; #endif } -static uint64_t ghc_xgetbv0(void) -{ -#if defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64)) - return (uint64_t)_xgetbv(0); -#elif defined(__i386__) || defined(__x86_64__) - uint32_t eax, edx; - __asm__ volatile(".byte 0x0f, 0x01, 0xd0" /* xgetbv */ - : "=a"(eax), "=d"(edx) - : "c"(0)); - return ((uint64_t)edx << 32) | (uint64_t)eax; +/// getX86CpuIDAndInfoEx - Execute the specified cpuid with subleaf and return +/// the 4 values in the specified arguments. If we can't run cpuid on the host, +/// return true. +static bool getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf, + unsigned *rEAX, unsigned *rEBX, unsigned *rECX, + unsigned *rEDX) { + // TODO(boomanaiden154): When the minimum toolchain versions for gcc and clang + // are such that __cpuidex is defined within cpuid.h for both, we can remove + // the __get_cpuid_count function and share the MSVC implementation between + // all three. +#if (defined(__GNUC__) || defined(__clang__)) && !defined(_MSC_VER) + return !__get_cpuid_count(value, subleaf, rEAX, rEBX, rECX, rEDX); +#elif defined(_MSC_VER) + int registers[4]; + __cpuidex(registers, value, subleaf); + *rEAX = registers[0]; + *rEBX = registers[1]; + *rECX = registers[2]; + *rEDX = registers[3]; + return false; #else - return 0; + return true; #endif } -#if defined(__APPLE__) && (defined(__i386__) || defined(__x86_64__)) -/* Query a macOS CPU-capability sysctl, e.g. "hw.optional.avx512f". */ -static int ghc_macos_sysctl_flag(const char *name) -{ - int result = 0; - size_t len = sizeof(result); - if (sysctlbyname(name, &result, &len, NULL, 0) != 0) { - return 0; - } - return result != 0; -} +// Read control register 0 (XCR0). Used to detect features such as AVX. +static bool getX86XCR0(unsigned *rEAX, unsigned *rEDX) { + // TODO(boomanaiden154): When the minimum toolchain versions for gcc and clang + // are such that _xgetbv is supported by both, we can unify the implementation + // with MSVC and remove all inline assembly. +#if defined(__GNUC__) || defined(__clang__) + // Check xgetbv; this uses a .byte sequence instead of the instruction + // directly because older assemblers do not include support for xgetbv and + // there is no easy way to conditionally compile based on the assembler used. + __asm__(".byte 0x0f, 0x01, 0xd0" : "=a"(*rEAX), "=d"(*rEDX) : "c"(0)); + return false; +#elif defined(_MSC_FULL_VER) && defined(_XCR_XFEATURE_ENABLED_MASK) + unsigned long long Result = _xgetbv(_XCR_XFEATURE_ENABLED_MASK); + *rEAX = Result; + *rEDX = Result >> 32; + return false; +#else + return true; #endif +} -HsWord64 ghc_detect_x86_cpu_features(void) -{ - HsWord64 feats = 0; +static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, + unsigned *Features) { + unsigned EAX = 0, EBX = 0; -#if defined(_M_IX86) || defined(_M_X64) || defined(__i386__) || defined(__x86_64__) - uint32_t a, b, c, d; - uint32_t max_basic = 0; +#define hasFeature(F) ((Features[F / 32] >> (F % 32)) & 1) +#define setFeature(F) Features[F / 32] |= 1U << (F % 32) - if (!ghc_cpuid_count(0, 0, &a, &b, &c, &d)) { - return 0; - } - max_basic = a; - if (max_basic < 1) { - return 0; - } + if ((EDX >> 15) & 1) + setFeature(FEATURE_CMOV); + if ((EDX >> 23) & 1) + setFeature(FEATURE_MMX); + if ((EDX >> 25) & 1) + setFeature(FEATURE_SSE); + if ((EDX >> 26) & 1) + setFeature(FEATURE_SSE2); - ghc_cpuid_count(1, 0, &a, &b, &c, &d); - - { - int has_sse2 = !!(d & (1u << 26)); - int has_sse3 = !!(c & (1u << 0)); - int has_ssse3 = !!(c & (1u << 9)); - int has_sse4_1 = !!(c & (1u << 19)); - int has_sse4_2 = !!(c & (1u << 20)); - int has_fma_hw = !!(c & (1u << 12)); - int has_avx_hw = !!(c & (1u << 28)); - int has_osxsave = !!(c & (1u << 27)); - - int avx_usable = 0; - int avx512_usable = 0; - - if (has_osxsave) { - uint64_t xcr0 = ghc_xgetbv0(); - avx_usable = ((xcr0 & 0x6u) == 0x6u); /* XMM + YMM state */ - avx512_usable = ((xcr0 & 0xE6u) == 0xE6u); /* XMM+YMM+opmask+ZMM */ - } + if ((ECX >> 0) & 1) + setFeature(FEATURE_SSE3); + if ((ECX >> 1) & 1) + setFeature(FEATURE_PCLMUL); + if ((ECX >> 9) & 1) + setFeature(FEATURE_SSSE3); + if ((ECX >> 12) & 1) + setFeature(FEATURE_FMA); + if ((ECX >> 13) & 1) + setFeature(FEATURE_CMPXCHG16B); + if ((ECX >> 19) & 1) + setFeature(FEATURE_SSE4_1); + if ((ECX >> 20) & 1) + setFeature(FEATURE_SSE4_2); + if ((ECX >> 22) & 1) + setFeature(FEATURE_MOVBE); + if ((ECX >> 23) & 1) + setFeature(FEATURE_POPCNT); + if ((ECX >> 25) & 1) + setFeature(FEATURE_AES); + if ((ECX >> 29) & 1) + setFeature(FEATURE_F16C); + if ((ECX >> 30) & 1) + setFeature(FEATURE_RDRND); + // If CPUID indicates support for XSAVE, XRESTORE and AVX, and XGETBV + // indicates that the AVX registers will be saved and restored on context + // switch, then we have full AVX support. + const unsigned AVXBits = (1 << 27) | (1 << 28); + bool HasAVXSave = ((ECX & AVXBits) == AVXBits) && !getX86XCR0(&EAX, &EDX) && + ((EAX & 0x6) == 0x6); #if defined(__APPLE__) - /* On x86_64 macOS the kernel enables AVX-512 XSAVE state lazily: XCR0 - reads back with the opmask/ZMM bits clear until a process first faults - on an AVX-512 instruction, so the XCR0 check above is a false negative - on AVX-512-capable Macs. Use the OS feature query instead. Checking - AVX512F alone suffices here; the AVX-512 sub-features (BW/CD/DQ/VL) are - still decoded from CPUID leaf 7 below. - - Refs: - https://zenn.dev/mod_poppo/articles/detect-processor-features-x86?locale=en#... - https://github.com/minoki/haskell-cpu-features */ - avx512_usable = ghc_macos_sysctl_flag("hw.optional.avx512f"); + // Darwin lazily saves the AVX512 context on first use: trust that the OS will + // save the AVX512 context if we use AVX512 instructions, even the bit is not + // set right now. + bool HasAVX512Save = true; +#else + // AVX512 requires additional context to be saved by the OS. + bool HasAVX512Save = HasAVXSave && ((EAX & 0xe0) == 0xe0); #endif + // AMX requires additional context to be saved by the OS. + const unsigned AMXBits = (1 << 17) | (1 << 18); + bool HasXSave = ((ECX >> 27) & 1) && !getX86XCR0(&EAX, &EDX); + bool HasAMXSave = HasXSave && ((EAX & AMXBits) == AMXBits); - if (has_sse2) { - SET_FEAT(feats, GHC_X86_FEAT_SSE2); - } - if (has_sse3) { - SET_FEAT(feats, GHC_X86_FEAT_SSE3); - } - if (has_ssse3) { - SET_FEAT(feats, GHC_X86_FEAT_SSSE3); - } - if (has_sse4_1) { - SET_FEAT(feats, GHC_X86_FEAT_SSE4_1); - } - if (has_sse4_2) { - SET_FEAT(feats, GHC_X86_FEAT_SSE4_2); - } - if (has_avx_hw && avx_usable) { - SET_FEAT(feats, GHC_X86_FEAT_AVX); + // GHC deviation from upstream: FMA instructions are VEX-encoded and + // unusable unless the OS saves the AVX register state, so FEATURE_FMA (set + // from the raw CPUID bit above) is cleared again when full AVX support is + // missing. This matches llvm/lib/TargetParser/Host.cpp ("fma") and GCC's + // gcc/common/config/i386/cpuinfo.h (FEATURE_FMA under avx_usable). + if (!HasAVXSave) + Features[FEATURE_FMA / 32] &= ~(1U << (FEATURE_FMA % 32)); + + if (HasAVXSave) + setFeature(FEATURE_AVX); + + if (((ECX >> 26) & 1) && HasAVXSave) + setFeature(FEATURE_XSAVE); + + bool HasLeaf7 = + MaxLeaf >= 0x7 && !getX86CpuIDAndInfoEx(0x7, 0x0, &EAX, &EBX, &ECX, &EDX); + + if (HasLeaf7 && ((EBX >> 0) & 1)) + setFeature(FEATURE_FSGSBASE); + if (HasLeaf7 && ((EBX >> 2) & 1)) + setFeature(FEATURE_SGX); + if (HasLeaf7 && ((EBX >> 3) & 1)) + setFeature(FEATURE_BMI); + if (HasLeaf7 && ((EBX >> 5) & 1) && HasAVXSave) + setFeature(FEATURE_AVX2); + if (HasLeaf7 && ((EBX >> 8) & 1)) + setFeature(FEATURE_BMI2); + if (HasLeaf7 && ((EBX >> 11) & 1)) + setFeature(FEATURE_RTM); + if (HasLeaf7 && ((EBX >> 16) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512F); + if (HasLeaf7 && ((EBX >> 17) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512DQ); + if (HasLeaf7 && ((EBX >> 18) & 1)) + setFeature(FEATURE_RDSEED); + if (HasLeaf7 && ((EBX >> 19) & 1)) + setFeature(FEATURE_ADX); + if (HasLeaf7 && ((EBX >> 21) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512IFMA); + if (HasLeaf7 && ((EBX >> 24) & 1)) + setFeature(FEATURE_CLWB); + if (HasLeaf7 && ((EBX >> 26) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512PF); + if (HasLeaf7 && ((EBX >> 27) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512ER); + if (HasLeaf7 && ((EBX >> 28) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512CD); + if (HasLeaf7 && ((EBX >> 29) & 1)) + setFeature(FEATURE_SHA); + if (HasLeaf7 && ((EBX >> 30) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512BW); + if (HasLeaf7 && ((EBX >> 31) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512VL); + + if (HasLeaf7 && ((ECX >> 0) & 1)) + setFeature(FEATURE_PREFETCHWT1); + if (HasLeaf7 && ((ECX >> 1) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512VBMI); + if (HasLeaf7 && ((ECX >> 4) & 1)) + setFeature(FEATURE_PKU); + if (HasLeaf7 && ((ECX >> 5) & 1)) + setFeature(FEATURE_WAITPKG); + if (HasLeaf7 && ((ECX >> 6) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512VBMI2); + if (HasLeaf7 && ((ECX >> 7) & 1)) + setFeature(FEATURE_SHSTK); + if (HasLeaf7 && ((ECX >> 8) & 1)) + setFeature(FEATURE_GFNI); + if (HasLeaf7 && ((ECX >> 9) & 1) && HasAVXSave) + setFeature(FEATURE_VAES); + if (HasLeaf7 && ((ECX >> 10) & 1) && HasAVXSave) + setFeature(FEATURE_VPCLMULQDQ); + if (HasLeaf7 && ((ECX >> 11) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512VNNI); + if (HasLeaf7 && ((ECX >> 12) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512BITALG); + if (HasLeaf7 && ((ECX >> 14) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512VPOPCNTDQ); + if (HasLeaf7 && ((ECX >> 22) & 1)) + setFeature(FEATURE_RDPID); + if (HasLeaf7 && ((ECX >> 23) & 1)) + setFeature(FEATURE_KL); + if (HasLeaf7 && ((ECX >> 25) & 1)) + setFeature(FEATURE_CLDEMOTE); + if (HasLeaf7 && ((ECX >> 27) & 1)) + setFeature(FEATURE_MOVDIRI); + if (HasLeaf7 && ((ECX >> 28) & 1)) + setFeature(FEATURE_MOVDIR64B); + if (HasLeaf7 && ((ECX >> 29) & 1)) + setFeature(FEATURE_ENQCMD); + + if (HasLeaf7 && ((EDX >> 2) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX5124VNNIW); + if (HasLeaf7 && ((EDX >> 3) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX5124FMAPS); + if (HasLeaf7 && ((EDX >> 5) & 1)) + setFeature(FEATURE_UINTR); + if (HasLeaf7 && ((EDX >> 8) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512VP2INTERSECT); + if (HasLeaf7 && ((EDX >> 14) & 1)) + setFeature(FEATURE_SERIALIZE); + if (HasLeaf7 && ((EDX >> 16) & 1)) + setFeature(FEATURE_TSXLDTRK); + if (HasLeaf7 && ((EDX >> 18) & 1)) + setFeature(FEATURE_PCONFIG); + if (HasLeaf7 && ((EDX >> 22) & 1) && HasAMXSave) + setFeature(FEATURE_AMX_BF16); + if (HasLeaf7 && ((EDX >> 23) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512FP16); + if (HasLeaf7 && ((EDX >> 24) & 1) && HasAMXSave) + setFeature(FEATURE_AMX_TILE); + if (HasLeaf7 && ((EDX >> 25) & 1) && HasAMXSave) + setFeature(FEATURE_AMX_INT8); + + // EAX from subleaf 0 is the maximum subleaf supported. Some CPUs don't + // return all 0s for invalid subleaves so check the limit. + bool HasLeaf7Subleaf1 = + HasLeaf7 && EAX >= 1 && + !getX86CpuIDAndInfoEx(0x7, 0x1, &EAX, &EBX, &ECX, &EDX); + if (HasLeaf7Subleaf1 && ((EAX >> 0) & 1)) + setFeature(FEATURE_SHA512); + if (HasLeaf7Subleaf1 && ((EAX >> 1) & 1)) + setFeature(FEATURE_SM3); + if (HasLeaf7Subleaf1 && ((EAX >> 2) & 1)) + setFeature(FEATURE_SM4); + if (HasLeaf7Subleaf1 && ((EAX >> 3) & 1)) + setFeature(FEATURE_RAOINT); + if (HasLeaf7Subleaf1 && ((EAX >> 4) & 1) && HasAVXSave) + setFeature(FEATURE_AVXVNNI); + if (HasLeaf7Subleaf1 && ((EAX >> 5) & 1) && HasAVX512Save) + setFeature(FEATURE_AVX512BF16); + if (HasLeaf7Subleaf1 && ((EAX >> 7) & 1)) + setFeature(FEATURE_CMPCCXADD); + if (HasLeaf7Subleaf1 && ((EAX >> 21) & 1) && HasAMXSave) + setFeature(FEATURE_AMX_FP16); + if (HasLeaf7Subleaf1 && ((EAX >> 22) & 1)) + setFeature(FEATURE_HRESET); + if (HasLeaf7Subleaf1 && ((EAX >> 23) & 1) && HasAVXSave) + setFeature(FEATURE_AVXIFMA); + if (HasLeaf7Subleaf1 && ((EAX >> 31) & 1)) + setFeature(FEATURE_MOVRS); + + if (HasLeaf7Subleaf1 && ((EDX >> 4) & 1) && HasAVXSave) + setFeature(FEATURE_AVXVNNIINT8); + if (HasLeaf7Subleaf1 && ((EDX >> 5) & 1) && HasAVXSave) + setFeature(FEATURE_AVXNECONVERT); + if (HasLeaf7Subleaf1 && ((EDX >> 8) & 1) && HasAMXSave) + setFeature(FEATURE_AMX_COMPLEX); + if (HasLeaf7Subleaf1 && ((EDX >> 10) & 1) && HasAVXSave) + setFeature(FEATURE_AVXVNNIINT16); + if (HasLeaf7Subleaf1 && ((EDX >> 14) & 1)) + setFeature(FEATURE_PREFETCHI); + if (HasLeaf7Subleaf1 && ((EDX >> 15) & 1)) + setFeature(FEATURE_USERMSR); + if (HasLeaf7Subleaf1 && ((EDX >> 21) & 1)) + setFeature(FEATURE_APXF); + + unsigned MaxLevel = 0; + getX86CpuIDAndInfo(0, &MaxLevel, &EBX, &ECX, &EDX); + bool HasLeafD = MaxLevel >= 0xd && + !getX86CpuIDAndInfoEx(0xd, 0x1, &EAX, &EBX, &ECX, &EDX); + if (HasLeafD && ((EAX >> 0) & 1) && HasAVXSave) + setFeature(FEATURE_XSAVEOPT); + if (HasLeafD && ((EAX >> 1) & 1) && HasAVXSave) + setFeature(FEATURE_XSAVEC); + if (HasLeafD && ((EAX >> 3) & 1) && HasAVXSave) + setFeature(FEATURE_XSAVES); + + bool HasLeaf24 = + MaxLevel >= 0x24 && !getX86CpuIDAndInfo(0x24, &EAX, &EBX, &ECX, &EDX); + if (HasLeaf7Subleaf1 && ((EDX >> 19) & 1) && HasLeaf24) { + bool Has512Len = (EBX >> 18) & 1; + int AVX10Ver = EBX & 0xff; + if (AVX10Ver >= 2) { + setFeature(FEATURE_AVX10_2_256); + if (Has512Len) + setFeature(FEATURE_AVX10_2_512); } - if (has_fma_hw && avx_usable) { - SET_FEAT(feats, GHC_X86_FEAT_FMA); + if (AVX10Ver >= 1) { + setFeature(FEATURE_AVX10_1_256); + if (Has512Len) + setFeature(FEATURE_AVX10_1_512); } + } - if (max_basic >= 7 && ghc_cpuid_count(7, 0, &a, &b, &c, &d)) { - int has_bmi1 = !!(b & (1u << 3)); - int has_avx2_hw = !!(b & (1u << 5)); - int has_bmi2 = !!(b & (1u << 8)); - int has_avx512f = !!(b & (1u << 16)); - int has_avx512dq = !!(b & (1u << 17)); - int has_avx512cd = !!(b & (1u << 28)); - int has_avx512bw = !!(b & (1u << 30)); - int has_avx512vl = !!(b & (1u << 31)); - int has_gfni = !!(c & (1u << 8)); - - if (has_bmi1) { - SET_FEAT(feats, GHC_X86_FEAT_BMI1); - } - if (has_bmi2) { - SET_FEAT(feats, GHC_X86_FEAT_BMI2); - } - if (avx_usable && has_avx2_hw) { - SET_FEAT(feats, GHC_X86_FEAT_AVX2); - } + unsigned MaxExtLevel = 0; + getX86CpuIDAndInfo(0x80000000, &MaxExtLevel, &EBX, &ECX, &EDX); - if (avx512_usable && has_avx512f) { - SET_FEAT(feats, GHC_X86_FEAT_AVX512F); - if (has_avx512bw) { - SET_FEAT(feats, GHC_X86_FEAT_AVX512BW); - } - if (has_avx512cd) { - SET_FEAT(feats, GHC_X86_FEAT_AVX512CD); - } - if (has_avx512dq) { - SET_FEAT(feats, GHC_X86_FEAT_AVX512DQ); - } - if (has_avx512vl) { - SET_FEAT(feats, GHC_X86_FEAT_AVX512VL); - } - } + bool HasExtLeaf1 = MaxExtLevel >= 0x80000001 && + !getX86CpuIDAndInfo(0x80000001, &EAX, &EBX, &ECX, &EDX); + if (HasExtLeaf1) { + if (ECX & 1) + setFeature(FEATURE_LAHF_LM); + if ((ECX >> 5) & 1) + setFeature(FEATURE_LZCNT); + if (((ECX >> 6) & 1)) + setFeature(FEATURE_SSE4_A); + if (((ECX >> 8) & 1)) + setFeature(FEATURE_PRFCHW); + if (((ECX >> 11) & 1)) + setFeature(FEATURE_XOP); + if (((ECX >> 15) & 1)) + setFeature(FEATURE_LWP); + if (((ECX >> 16) & 1)) + setFeature(FEATURE_FMA4); + if (((ECX >> 21) & 1)) + setFeature(FEATURE_TBM); + if (((ECX >> 29) & 1)) + setFeature(FEATURE_MWAITX); + + if (((EDX >> 29) & 1)) + setFeature(FEATURE_LM); + } + + bool HasExtLeaf8 = MaxExtLevel >= 0x80000008 && + !getX86CpuIDAndInfo(0x80000008, &EAX, &EBX, &ECX, &EDX); + if (HasExtLeaf8 && ((EBX >> 0) & 1)) + setFeature(FEATURE_CLZERO); + if (HasExtLeaf8 && ((EBX >> 9) & 1)) + setFeature(FEATURE_WBNOINVD); + + bool HasLeaf14 = MaxLevel >= 0x14 && + !getX86CpuIDAndInfoEx(0x14, 0x0, &EAX, &EBX, &ECX, &EDX); + if (HasLeaf14 && ((EBX >> 4) & 1)) + setFeature(FEATURE_PTWRITE); - if (has_gfni) { - SET_FEAT(feats, GHC_X86_FEAT_GFNI); + bool HasLeaf19 = + MaxLevel >= 0x19 && !getX86CpuIDAndInfo(0x19, &EAX, &EBX, &ECX, &EDX); + if (HasLeaf7 && HasLeaf19 && ((EBX >> 2) & 1)) + setFeature(FEATURE_WIDEKL); + + if (hasFeature(FEATURE_LM) && hasFeature(FEATURE_SSE2)) { + setFeature(FEATURE_X86_64_BASELINE); + if (hasFeature(FEATURE_CMPXCHG16B) && hasFeature(FEATURE_POPCNT) && + hasFeature(FEATURE_LAHF_LM) && hasFeature(FEATURE_SSE4_2)) { + setFeature(FEATURE_X86_64_V2); + if (hasFeature(FEATURE_AVX2) && hasFeature(FEATURE_BMI) && + hasFeature(FEATURE_BMI2) && hasFeature(FEATURE_F16C) && + hasFeature(FEATURE_FMA) && hasFeature(FEATURE_LZCNT) && + hasFeature(FEATURE_MOVBE)) { + setFeature(FEATURE_X86_64_V3); + if (hasFeature(FEATURE_AVX512BW) && hasFeature(FEATURE_AVX512CD) && + hasFeature(FEATURE_AVX512DQ) && hasFeature(FEATURE_AVX512VL)) + setFeature(FEATURE_X86_64_V4); } } } -#endif - return feats; +#undef hasFeature +#undef setFeature +} + +#endif /* GHC_HOST_IS_X86 */ + +/* The driver, modeled on upstream's __cpu_indicator_init: same CPUID call + * sequence, but the first 64 feature bits are returned to the caller (see + * GHC.Driver.CpuFeatures) instead of being stored in the __cpu_model + * globals. */ +HsWord64 ghc_detect_x86_cpu_features(void) +{ +#if defined(GHC_HOST_IS_X86) + unsigned EAX = 0, EBX = 0, ECX = 0, EDX = 0; + unsigned MaxLeaf = 5; + unsigned Features[(CPU_FEATURE_MAX + 31) / 32] = {0}; + + if (getX86CpuIDAndInfo(0, &MaxLeaf, &EBX, &ECX, &EDX) || MaxLeaf < 1) + return 0; + + getX86CpuIDAndInfo(1, &EAX, &EBX, &ECX, &EDX); + + // Find available features. + getAvailableFeatures(ECX, EDX, MaxLeaf, &Features[0]); + + return ((HsWord64)Features[1] << 32) | (HsWord64)Features[0]; +#else + return 0; +#endif } View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/7bab9752985efd921aa0198e2c3cd2a8... -- View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/7bab9752985efd921aa0198e2c3cd2a8... You're receiving this email because of your account on gitlab.haskell.org.