Magnus pushed to branch wip/mangoiv/ghc-9.12-bp at Glasgow Haskell Compiler / GHC

Commits:

13 changed files:

Changes:

  • compiler/GHC/CmmToLlvm.hs
    ... ... @@ -217,11 +217,15 @@ cmmMetaLlvmPrelude = do
    217 217 
                   Nothing -> [ MetaStr name ]
    218 218
    219 219 
       platform <- getPlatform
    220 
    -  cfg <- getConfig
    221 220 
       let stack_alignment_metas =
    222 221 
               case platformArch platform of
    223 
    -            ArchX86_64 | llvmCgAvxEnabled cfg -> [mkStackAlignmentMeta 32]
    224 
    -            _                                 -> []
    222 
    +            -- LLVM inserts stack realignment prologue/epilogue when it wants to place __m256 or __m512 on the stack.
    223 
    +            -- However, it reserves %rbp as the frame pointer, conflicting with our use of it.
    224 
    +            -- Therefore, we tell LLVM that the stack is already aligned to avoid stack realignment.
    225 
    +            -- See also Note [Stack Alignment on X86] and https://gitlab.haskell.org/ghc/ghc/-/issues/26595.
    226 
    +            ArchX86    -> [mkStackAlignmentMeta 64]
    227 
    +            ArchX86_64 -> [mkStackAlignmentMeta 64]
    228 
    +            _          -> []
    225 229 
       let codel_model_metas =
    226 230 
               case platformArch platform of
    227 231 
                 -- FIXME: We should not rely on LLVM

  • compiler/GHC/CmmToLlvm/Config.hs
    ... ... @@ -22,7 +22,6 @@ data LlvmCgConfig = LlvmCgConfig
    22 22 
       , llvmCgContext           :: !SDocContext  -- ^ Context for LLVM code generation
    23 23 
       , llvmCgFillUndefWithGarbage :: !Bool      -- ^ Fill undefined literals with garbage values
    24 24 
       , llvmCgSplitSection      :: !Bool         -- ^ Split sections
    25 
    -  , llvmCgAvxEnabled        :: !Bool
    26 25 
       , llvmCgBmiVersion        :: Maybe BmiVersion  -- ^ (x86) BMI instructions
    27 26 
       , llvmCgLlvmVersion       :: Maybe LlvmVersion -- ^ version of Llvm we're using
    28 27 
       , llvmCgDoWarn            :: !Bool         -- ^ True ==> warn unsupported Llvm version

  • compiler/GHC/CmmToLlvm/Mangler.hs
    ... ... @@ -38,7 +38,7 @@ llvmFixupAsm platform f1 f2 = {-# SCC "llvm_mangler" #-}
    38 38
    39 39 
     -- | These are the rewrites that the mangler will perform
    40 40 
     rewrites :: [Rewrite]
    41 
    -rewrites = [rewriteSymType, rewriteAVX, rewriteCall, rewriteJump]
    41 
    +rewrites = [rewriteSymType, rewriteCall, rewriteJump]
    42 42
    43 43 
     type Rewrite = Platform -> B.ByteString -> Maybe B.ByteString
    44 44
    ... ... @@ -85,23 +85,6 @@ rewriteSymType _ l
    85 85 
             funcType = prefix `B.cons` B.pack "function"
    86 86 
             objType  = prefix `B.cons` B.pack "object"
    87 87
    88 
    --- | This rewrites aligned AVX instructions to their unaligned counterparts on
    89 
    --- x86-64. This is necessary because the stack is not adequately aligned for
    90 
    --- aligned AVX spills, so LLVM would emit code that adjusts the stack pointer
    91 
    --- and disable tail call optimization. Both would be catastrophic here so GHC
    92 
    --- tells LLVM that the stack is 32-byte aligned (even though it isn't) and then
    93 
    --- rewrites the instructions in the mangler.
    94 
    -rewriteAVX :: Rewrite
    95 
    -rewriteAVX platform s
    96 
    -  | not isX86_64 = Nothing
    97 
    -  | isVmovdqa s  = Just $ replaceOnce (B.pack "vmovdqa") (B.pack "vmovdqu") s
    98 
    -  | isVmovap s   = Just $ replaceOnce (B.pack "vmovap") (B.pack "vmovup") s
    99 
    -  | otherwise    = Nothing
    100 
    -  where
    101 
    -    isX86_64 = platformArch platform == ArchX86_64
    102 
    -    isVmovdqa = B.isPrefixOf (B.pack "vmovdqa")
    103 
    -    isVmovap = B.isPrefixOf (B.pack "vmovap")
    104 
    -
    105 88 
     -- | This rewrites (tail) calls to avoid creating PLT entries for
    106 89 
     -- functions on riscv64. The replacement will load the address from the
    107 90 
     -- GOT, which is resolved to point to the real address of the function.

  • compiler/GHC/Driver/Config/CmmToLlvm.hs
    ... ... @@ -23,7 +23,6 @@ initLlvmCgConfig logger config_cache dflags = do
    23 23 
         , llvmCgContext              = initSDocContext dflags PprCode
    24 24 
         , llvmCgFillUndefWithGarbage = gopt Opt_LlvmFillUndefWithGarbage dflags
    25 25 
         , llvmCgSplitSection         = gopt Opt_SplitSections dflags
    26 
    -    , llvmCgAvxEnabled           = isAvxEnabled dflags
    27 26 
         , llvmCgBmiVersion           = case platformArch (targetPlatform dflags) of
    28 27 
                                           ArchX86_64 -> bmiVersion dflags
    29 28 
                                           ArchX86    -> bmiVersion dflags

  • rts/StgCRun.c
    ... ... @@ -108,19 +108,24 @@ StgFunPtr StgReturn(void)
    108 108 
     /*
    109 109 
      * Note [Stack Alignment on X86]
    110 110 
      * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    111 
    - * On X86 (both 32bit and 64bit) we keep the stack aligned on function calls at
    112 
    - * a 16-byte boundary. This is done because on a number of architectures the
    113 
    - * ABI requires this (e.g. the System V AMD64 ABI, Mac OS X 32-bit/64-bit ABIs,
    114 
    - * and the Win64 ABI) as well as interfacing with * other libraries through the
    115 
    - * FFI.
    111 
    + * On X86, we keep the stack aligned on function calls. We use 64-byte alignment
    112 
    + * because it is required for passing AVX-512 vector types (__m512 and friends)
    113 
    + * as arguments. See #26822 for a problem caused by insufficient stack alignment.
    116 114 
      *
    117 
    - * As part of this arrangement we must maintain the stack at a 16-byte boundary
    118 
    - * - word_size-bytes (so 16n - 4 for i386 and 16n - 8 for x64) on entry to a
    119 
    - * procedure since both GCC and LLVM expect this. This is because the stack
    120 
    - * should have been 16-byte boundary aligned and then a call made which pushes
    121 
    - * a return address onto the stack (so word_size more space used). In STG code
    122 
    - * we only jump to other STG procedures, so we maintain the 16n - word_size
    123 
    - * alignment for these jumps.
    115 
    + * Strictly speaking, we only need 16-byte alignment when we do not use AVX or
    116 
    + * AVX-512, but dispatching based on the available ISA extensions would
    117 
    + * complicate things. Always using 64-byte alignment is simpler.
    118 
    + *
    119 
    + * However, the C ABI only requires 16-byte alignment, which is just sufficient
    120 
    + * for SSE2 vectors. Therefore, we dynamically align the stack with
    121 
    + * `and{l,q} $-64`, and save the original stack pointer.
    122 
    + *
    123 
    + * On entry to a procedure, we must maintain the stack at a 64-byte boundary
    124 
    + * - word_size-bytes (so 64n - 4 for i386 and 64n - 8 for x64) since both GCC
    125 
    + * and LLVM expect this. This is because the stack should have been aligned to
    126 
    + * a 64-byte boundary, and then a call made which pushes a return address onto
    127 
    + * the stack (so word_size more space used). In STG code we only jump to other
    128 
    + * STG procedures, so we maintain the 64n - word_size alignment for these jumps.
    124 129 
      *
    125 130 
      * This gives us binary compatibility with LLVM and GCC as well as dealing
    126 131 
      * with the FFI. Previously we just maintained a 16n byte alignment for
    ... ... @@ -157,51 +162,84 @@ StgFunPtr StgReturn(void)
    157 162 
      * Concretely this means we must always keep the stack valid.
    158 163 
      * */
    159 164
    165 
    +/* Stack layout on x86-32:
    166 
    +  +-----------------------------+ <------ esp
    167 
    +  |        4-byte padding       |
    168 
    +  |-----------------------------| <------ esp + 4 (64-byte aligned)
    169 
    +  |                             |
    170 
    +  | RESERVED_C_STACK_BYTES ~16k |
    171 
    +  |                             |
    172 
    +  |-----------------------------| <------ 64-byte aligned
    173 
    +  |             ebx             |
    174 
    +  |-----------------------------|
    175 
    +  |             esi             |
    176 
    +  |-----------------------------|
    177 
    +  |             edi             |
    178 
    +  |-----------------------------|
    179 
    +  |             ebp             |
    180 
    +  |-----------------------------|
    181 
    +  |          saved esp          |----+
    182 
    +  |-----------------------------|    |
    183 
    +  |   (padding for alignment)   |    |
    184 
    +  |-----------------------------| <--+
    185 
    +  |  eip saved by call StgRun   |
    186 
    +  |        in schedule()        |
    187 
    +  +-----------------------------+
    188 
    +  |    the function pointer     |
    189 
    +  +-----------------------------+
    190 
    +  |           BaseReg           |
    191 
    +  +-----------------------------+
    192 
    +                ...
    193 
    +      schedule() stack frame
    194 
    +
    195 
    + Lower addresses on the top
    196 
    + */
    197 
    +
    198 
    +#if RESERVED_C_STACK_BYTES % 64 != 0
    199 
    +#error "RESERVED_C_STACK_BYTES must be a multiple of 64. If you are changing it, please make sure %esp+4 is a multiple of 64"
    200 
    +#endif
    160 201
    161 202 
     static void STG_USED
    162 203 
     StgRunIsImplementedInAssembler(void)
    163 204 
     {
    164 205 
         __asm__ volatile (
    165 206 
             STG_GLOBAL STG_RUN "\n"
    166 
    -#if !defined(mingw32_HOST_OS)
    167 207 
             STG_HIDDEN STG_RUN "\n"
    168 
    -#endif
    169 208 
             STG_RUN ":\n\t"
    170 209
    171 210 
             /*
    172 211 
              * move %esp down to reserve an area for temporary storage
    173 212 
              * during the execution of STG code.
    174 213 
              *
    175 
    -         * The stack pointer has to be aligned to a multiple of 16
    176 
    -         * bytes from here - this is a requirement of the C ABI, so
    177 
    -         * that C code can assign SSE2 registers directly to/from
    178 
    -         * stack locations.
    179 
    -         *
    180 
    -         * See Note [Windows Stack allocations]
    214 
    +         * We want the stack pointer to be aligned to a multiple of 64
    215 
    +         * bytes from here - this is a requirement of the C ABI for
    216 
    +         * AVX-512, so that C code can assign ZMM registers directly
    217 
    +         * to/from stack locations.
    181 218 
              */
    182 
    -#if defined(mingw32_HOST_OS)
    183 
    -        "movl %0, %%eax\n\t"
    184 
    -        "call ___chkstk_ms\n\t"
    185 
    -#endif
    186 
    -        "subl %0, %%esp\n\t"
    187 219
    220 
    +        /* We no longer support i386 Windows, so no need to call __chkstk_ms */
    221 
    +
    222 
    +        /* Save the original esp in eax */
    223 
    +        "movl %%esp, %%eax\n\t"
    224 
    +        "subl $20, %%esp\n\t"  /* area to save 5 registers */
    225 
    +        "andl $-64, %%esp\n\t"
    188 226 
             /*
    189 227 
              * save callee-saves registers on behalf of the STG code.
    190 228 
              */
    191 
    -        "movl %%esp, %%eax\n\t"
    192 
    -        "addl %0-16, %%eax\n\t"
    193 
    -        "movl %%ebx,0(%%eax)\n\t"
    194 
    -        "movl %%esi,4(%%eax)\n\t"
    195 
    -        "movl %%edi,8(%%eax)\n\t"
    196 
    -        "movl %%ebp,12(%%eax)\n\t"
    229 
    +        "movl %%ebx, 0(%%esp)\n\t"
    230 
    +        "movl %%esi, 4(%%esp)\n\t"
    231 
    +        "movl %%edi, 8(%%esp)\n\t"
    232 
    +        "movl %%ebp, 12(%%esp)\n\t"
    233 
    +        "movl %%eax, 16(%%esp)\n\t"
    234 
    +        "subl %0, %%esp\n\t"
    197 235 
             /*
    198 236 
              * Set BaseReg
    199 237 
              */
    200 
    -        "movl 24(%%eax),%%ebx\n\t"
    238 
    +        "movl 8(%%eax),%%ebx\n\t"
    201 239 
             /*
    202 240 
              * grab the function argument from the stack
    203 241 
              */
    204 
    -        "movl 20(%%eax),%%eax\n\t"
    242 
    +        "movl 4(%%eax),%%eax\n\t"
    205 243 
             /*
    206 244 
              * jump to it
    207 245 
              */
    ... ... @@ -216,17 +254,16 @@ StgRunIsImplementedInAssembler(void)
    216 254 
              * restore callee-saves registers.  (Don't stomp on %%eax!)
    217 255 
              */
    218 256 
             "movl %%esp, %%edx\n\t"
    219 
    -        "addl %0-16, %%edx\n\t"
    257 
    +        "addl %0, %%edx\n\t"
    220 258 
             "movl 0(%%edx),%%ebx\n\t"       /* restore the registers saved above */
    221 259 
             "movl 4(%%edx),%%esi\n\t"
    222 260 
             "movl 8(%%edx),%%edi\n\t"
    223 261 
             "movl 12(%%edx),%%ebp\n\t"
    224 
    -
    225 
    -        "addl %0, %%esp\n\t"
    262 
    +        "movl 16(%%edx),%%esp\n\t"
    226 263 
             "ret"
    227 264
    228 
    -      : : "i" (RESERVED_C_STACK_BYTES + 16)
    229 
    -        // + 16 to make room for the 4 registers we have to save
    265 
    +      : : "i" (RESERVED_C_STACK_BYTES + 4)
    266 
    +        // + 4 to mimic `calll`
    230 267 
             // See Note [Stack Alignment on X86]
    231 268 
         );
    232 269 
     }
    ... ... @@ -302,10 +339,12 @@ is as follows:
    302 339 
          C STACK "ADDRESS SPACE"   |
    303 340 
                                    v
    304 341 
       +-----------------------------+ <------ rsp
    342 
    +  |        8-byte padding       |
    343 
    +  |-----------------------------| <------ rsp + 8 (64-byte aligned)
    305 344 
       |                             |
    306 345 
       | RESERVED_C_STACK_BYTES ~16k |
    307 346 
       |                             |
    308 
    -  |-----------------------------|
    347 
    +  |-----------------------------| <------ 64-byte aligned
    309 348 
       |             rbx             ||
    310 349 
       |-----------------------------| \
    311 350 
       |             rbp             | |
    ... ... @@ -315,9 +354,13 @@ is as follows:
    315 354 
       |             r13             |   | STG_RUN_STACK_FRAME_SIZE
    316 355 
       |-----------------------------|  /
    317 356 
       |             r14             |  |
    318 
    -  |-----------------------------| /
    319 
    -  |             r15             | |
    320 
    -  |-----------------------------|/
    357 
    +  |-----------------------------|  |
    358 
    +  |             r15             | /
    359 
    +  |-----------------------------| |
    360 
    +  |          saved rsp          |-+---
    361 
    +  |-----------------------------|/   |
    362 
    +  |   (padding for alignment)   |    |
    363 
    +  |-----------------------------| <---
    321 364 
       |  rip saved by call StgRun   |
    322 365 
       |        in schedule()        |
    323 366 
       +-----------------------------+
    ... ... @@ -367,6 +410,10 @@ stack unwinding.
    367 410 
     */
    368 411
    369 412
    413 
    +#if RESERVED_C_STACK_BYTES % 64 != 0
    414 
    +#error "RESERVED_C_STACK_BYTES must be a multiple of 64. If you are changing it, please make sure %rsp+8 is a multiple of 64"
    415 
    +#endif
    416 
    +
    370 417 
     static void STG_USED
    371 418 
     StgRunIsImplementedInAssembler(void)
    372 419 
     {
    ... ... @@ -385,9 +432,22 @@ StgRunIsImplementedInAssembler(void)
    385 432 
     #if defined(mingw32_HOST_OS)
    386 433 
             "movq %1, %%rax\n\t"
    387 434 
             "addq %0, %%rax\n\t"
    435 
    +        "addq $63, %%rax\n\t"  /* extra space for 64-byte alignment */
    388 436 
             "callq ___chkstk_ms\n\t"
    389 437 
     #endif
    438 
    +        /*
    439 
    +         * Save the original rsp in r11 (caller-saved, so we don't need to
    440 
    +         * preserve it across calls). We need this to restore rsp after
    441 
    +         * the 64-byte alignment.
    442 
    +         */
    443 
    +        "movq %%rsp, %%r11\n\t"
    444 
    +        /*
    445 
    +         * Allocate space for saved registers, then align to 64-byte boundary.
    446 
    +         * The alignment is required for AVX-512 instructions.
    447 
    +         * See Note [Stack Alignment on X86].
    448 
    +         */
    390 449 
             "subq %1, %%rsp\n\t"
    450 
    +        "andq $-64, %%rsp\n\t"
    391 451 
             "movq %%rsp, %%rax\n\t"
    392 452 
             "subq %0, %%rsp\n\t"
    393 453 
             "movq %%rbx,0(%%rax)\n\t"
    ... ... @@ -396,6 +456,7 @@ StgRunIsImplementedInAssembler(void)
    396 456 
             "movq %%r13,24(%%rax)\n\t"
    397 457 
             "movq %%r14,32(%%rax)\n\t"
    398 458 
             "movq %%r15,40(%%rax)\n\t"
    459 
    +        "movq %%r11,48(%%rax)\n\t"  /* save original rsp */
    399 460 
     #if defined(mingw32_HOST_OS)
    400 461 
             /*
    401 462 
              * Additional callee saved registers on Win64. This must match
    ... ... @@ -403,25 +464,35 @@ StgRunIsImplementedInAssembler(void)
    403 464 
              * both represent the Win64 calling convention.
    404 465 
              *
    405 466 
              * Note that we must save the entire 128-bit width of the XMM
    406 
    -         * registers, as noted in #21465. Moreover, note that, due to the
    407 
    -         * presence of the return address on the stack, %rsp+8 is
    408 
    -         * 16-byte aligned. Since MOVAPS requires memory operands to be aligned
    409 
    -         * to 16-bytes, we must add a word of padding here.
    467 
    +         * registers, as noted in #21465. Since we align the stack to
    468 
    +         * 64 bytes, MOVAPS alignment requirements are satisfied.
    469 
    +         *
    470 
    +         * The Win64 calling convention says the upper YMM/ZMM parts are
    471 
    +         * volatile, so we do not need to save them. See Microsoft Learn
    472 
    +         * for the description of the calling convention:
    473 
    +         * https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention?view=msvc-170#callercallee-saved-registers
    474 
    +         *
    475 
    +         * Layout (offsets from rax):
    476 
    +         *   0-47:  rbx, rbp, r12, r13, r14, r15
    477 
    +         *   48:    original rsp
    478 
    +         *   56:    rdi
    479 
    +         *   64:    rsi
    480 
    +         *   72:    padding for 16-byte alignment of XMM registers
    481 
    +         *   80-239: xmm6-xmm15 (10 * 16 bytes)
    410 482 
              */
    411 
    -        "movq %%rdi,   48(%%rax)\n\t"
    412 
    -        "movq %%rsi,   56(%%rax)\n\t"
    413 
    -        /* 8 bytes of padding for alignment */
    414 
    -        "movaps %%xmm6,  72(%%rax)\n\t"
    415 
    -        "movaps %%xmm7,  88(%%rax)\n\t"
    416 
    -        "movaps %%xmm8, 104(%%rax)\n\t"
    417 
    -        "movaps %%xmm9, 120(%%rax)\n\t"
    418 
    -        "movaps %%xmm10,136(%%rax)\n\t"
    419 
    -        "movaps %%xmm11,152(%%rax)\n\t"
    420 
    -        "movaps %%xmm12,168(%%rax)\n\t"
    421 
    -        "movaps %%xmm13,184(%%rax)\n\t"
    422 
    -        "movaps %%xmm14,200(%%rax)\n\t"
    423 
    -        "movaps %%xmm15,216(%%rax)\n\t"
    483 
    +        "movq %%rdi,   56(%%rax)\n\t"
    484 
    +        "movq %%rsi,   64(%%rax)\n\t"
    424 485 
             /* 8 bytes of padding for alignment */
    486 
    +        "movaps %%xmm6,  80(%%rax)\n\t"
    487 
    +        "movaps %%xmm7,  96(%%rax)\n\t"
    488 
    +        "movaps %%xmm8, 112(%%rax)\n\t"
    489 
    +        "movaps %%xmm9, 128(%%rax)\n\t"
    490 
    +        "movaps %%xmm10,144(%%rax)\n\t"
    491 
    +        "movaps %%xmm11,160(%%rax)\n\t"
    492 
    +        "movaps %%xmm12,176(%%rax)\n\t"
    493 
    +        "movaps %%xmm13,192(%%rax)\n\t"
    494 
    +        "movaps %%xmm14,208(%%rax)\n\t"
    495 
    +        "movaps %%xmm15,224(%%rax)\n\t"
    425 496 
     #endif
    426 497
    427 498 
     #if defined(ENABLE_UNWINDING)
    ... ... @@ -431,6 +502,11 @@ StgRunIsImplementedInAssembler(void)
    431 502 
              *
    432 503 
              * N.B. We don't support unwinding on Darwin due to
    433 504 
              * various toolchain insanity.
    505 
    +         *
    506 
    +         * Note: Due to dynamic 64-byte alignment, the exact stack offset
    507 
    +         * varies at runtime. The original rsp is saved at offset 48 in the
    508 
    +         * register save area. We use a DWARF expression to compute the
    509 
    +         * CFA from the saved rsp value.
    434 510 
              */
    435 511 
             ".cfi_def_cfa rsp, 0\n\t"
    436 512 
             ".cfi_offset rbx, %c2\n\t"
    ... ... @@ -439,30 +515,69 @@ StgRunIsImplementedInAssembler(void)
    439 515 
             ".cfi_offset r13, %c5\n\t"
    440 516 
             ".cfi_offset r14, %c6\n\t"
    441 517 
             ".cfi_offset r15, %c7\n\t"
    442 
    -        ".cfi_offset rip, %c8\n\t"
    443 
    -        ".cfi_escape " // DW_CFA_val_expression is not expressible otherwise
    444 
    -          "0x16, " // DW_CFA_val_expression
    445 
    -          "0x07, " // register num 7 - rsp
    446 
    -          "0x04, " // block length
    447 
    -          "0x77, " // DW_OP_breg7 - signed LEB128 offset from rsp
    448 
    -#define RSP_DELTA (RESERVED_C_STACK_BYTES + STG_RUN_STACK_FRAME_SIZE + 8)
    449 
    -          "%c9" // signed LEB128 encoded delta - byte 1
    450 
    -#if (RSP_DELTA >> 7) > 0
    451 
    -          ", %c10" // signed LEB128 encoded delta - byte 2
    518 
    +        /*
    519 
    +         * The original rsp is saved at RESERVED_C_STACK_BYTES + 56 from
    520 
    +         * current rsp. We use DW_CFA_expression to indicate that rsp's
    521 
    +         * value can be found by loading from that stack location.
    522 
    +         */
    523 
    +#define RSP_DELTA (RESERVED_C_STACK_BYTES + 56)
    524 
    +        ".cfi_escape "
    525 
    +          "0x10, "  // DW_CFA_expression
    526 
    +          "0x07, "  // register num 7 - rsp
    527 
    +#if (RSP_DELTA >> 20) > 0
    528 
    +          "0x05, "  // block length = 5
    529 
    +#elif (RSP_DELTA >> 13) > 0
    530 
    +          "0x04, "  // block length = 4
    531 
    +#elif (RSP_DELTA >> 6) > 0
    532 
    +          "0x03, "  // block length = 3
    533 
    +#else
    534 
    +          "0x02, "  // block length = 2
    452 535 
     #endif
    453 
    -
    454 
    -#if (RSP_DELTA >> 14) > 0
    455 
    -          ", %c11" // signed LEB128 encoded delta - byte 3
    536 
    +          "0x77, "  // DW_OP_breg7 (rsp + offset)
    537 
    +          "%c8"     // signed LEB128 offset to saved rsp (RESERVED_C_STACK_BYTES + 56)
    538 
    +#if (RSP_DELTA >> 6) > 0
    539 
    +          ", %c9"   // signed LEB128 encoded delta - byte 2
    456 540 
     #endif
    457 
    -
    458 
    -#if (RSP_DELTA >> 21) > 0
    459 
    -          ", %c12" // signed LEB128 encoded delta - byte 4
    541 
    +#if (RSP_DELTA >> 13) > 0
    542 
    +          ", %c10"  // signed LEB128 encoded delta - byte 3
    460 543 
     #endif
    461 
    -
    462 
    -#if (RSP_DELTA >> 28) > 0
    544 
    +#if (RSP_DELTA >> 20) > 0
    545 
    +          ", %c11"  // signed LEB128 encoded delta - byte 4
    546 
    +#endif
    547 
    +#if (RSP_DELTA >> 27) > 0
    463 548 
     #error "RSP_DELTA too big"
    464 549 
     #endif
    465 550 
               "\n\t"
    551 
    +        /*
    552 
    +         * The return address (rip) is at the original rsp location.
    553 
    +         * Since original rsp is saved at offset 48 in register save area,
    554 
    +         * rip = *saved_rsp = **(rsp + 8 + RESERVED_C_STACK_BYTES + 48).
    555 
    +         */
    556 
    +        ".cfi_escape "
    557 
    +          "0x10, "  // DW_CFA_expression
    558 
    +          "0x10, "  // register num 16 - rip
    559 
    +#if (RSP_DELTA >> 20) > 0
    560 
    +          "0x06, "  // block length = 6
    561 
    +#elif (RSP_DELTA >> 13) > 0
    562 
    +          "0x05, "  // block length = 5
    563 
    +#elif (RSP_DELTA >> 6) > 0
    564 
    +          "0x04, "  // block length = 4
    565 
    +#else
    566 
    +          "0x03, "  // block length = 3
    567 
    +#endif
    568 
    +          "0x77, "  // DW_OP_breg7 (rsp + offset)
    569 
    +          "%c8"     // signed LEB128 offset to saved rsp
    570 
    +#if (RSP_DELTA >> 6) > 0
    571 
    +          ", %c9"   // signed LEB128 encoded delta - byte 2
    572 
    +#endif
    573 
    +#if (RSP_DELTA >> 13) > 0
    574 
    +          ", %c10"  // signed LEB128 encoded delta - byte 3
    575 
    +#endif
    576 
    +#if (RSP_DELTA >> 20) > 0
    577 
    +          ", %c11"  // signed LEB128 encoded delta - byte 4
    578 
    +#endif
    579 
    +          ", 0x06"  // DW_OP_deref
    580 
    +          "\n\t"
    466 581 
     #endif /* defined(ENABLE_UNWINDING) */
    467 582
    468 583 
             /*
    ... ... @@ -510,55 +625,50 @@ StgRunIsImplementedInAssembler(void)
    510 625 
             "movq 32(%%rsp),%%r14\n\t"
    511 626 
             "movq 40(%%rsp),%%r15\n\t"
    512 627 
     #if defined(mingw32_HOST_OS)
    513 
    -        "movq 48(%%rsp),%%rdi\n\t"
    514 
    -        "movq 56(%%rsp),%%rsi\n\t"
    628 
    +        "movq 56(%%rsp),%%rdi\n\t"
    629 
    +        "movq 64(%%rsp),%%rsi\n\t"
    515 630 
             /* 8 bytes of padding for alignment */
    516 
    -        "movaps  72(%%rsp),%%xmm6\n\t"
    517 
    -        "movaps  88(%%rsp),%%xmm7\n\t"
    518 
    -        "movaps 104(%%rsp),%%xmm8\n\t"
    519 
    -        "movaps 120(%%rsp),%%xmm9\n\t"
    520 
    -        "movaps 136(%%rsp),%%xmm10\n\t"
    521 
    -        "movaps 152(%%rsp),%%xmm11\n\t"
    522 
    -        "movaps 168(%%rsp),%%xmm12\n\t"
    523 
    -        "movaps 184(%%rsp),%%xmm13\n\t"
    524 
    -        "movaps 200(%%rsp),%%xmm14\n\t"
    525 
    -        "movaps 216(%%rsp),%%xmm15\n\t"
    526 
    -        /* 8 bytes of padding for alignment */
    527 
    -#endif
    528 
    -        "addq %1, %%rsp\n\t"
    631 
    +        "movaps  80(%%rsp),%%xmm6\n\t"
    632 
    +        "movaps  96(%%rsp),%%xmm7\n\t"
    633 
    +        "movaps 112(%%rsp),%%xmm8\n\t"
    634 
    +        "movaps 128(%%rsp),%%xmm9\n\t"
    635 
    +        "movaps 144(%%rsp),%%xmm10\n\t"
    636 
    +        "movaps 160(%%rsp),%%xmm11\n\t"
    637 
    +        "movaps 176(%%rsp),%%xmm12\n\t"
    638 
    +        "movaps 192(%%rsp),%%xmm13\n\t"
    639 
    +        "movaps 208(%%rsp),%%xmm14\n\t"
    640 
    +        "movaps 224(%%rsp),%%xmm15\n\t"
    641 
    +#endif
    642 
    +        "movq 48(%%rsp),%%rsp\n\t"  /* restore original rsp */
    529 643 
             "retq"
    530 644
    531 645 
             :
    532 
    -        : "i"(RESERVED_C_STACK_BYTES),
    646 
    +        : "i"(RESERVED_C_STACK_BYTES + 8),
    533 647 
               "i"(STG_RUN_STACK_FRAME_SIZE /* stack frame size */),
    534 
    -          "i"(RESERVED_C_STACK_BYTES /* rbx relative to cfa (rsp) */),
    535 
    -          "i"(RESERVED_C_STACK_BYTES + 8 /* rbp relative to cfa (rsp) */),
    536 
    -          "i"(RESERVED_C_STACK_BYTES + 16 /* r12 relative to cfa (rsp) */),
    537 
    -          "i"(RESERVED_C_STACK_BYTES + 24 /* r13 relative to cfa (rsp) */),
    538 
    -          "i"(RESERVED_C_STACK_BYTES + 32 /* r14 relative to cfa (rsp) */),
    539 
    -          "i"(RESERVED_C_STACK_BYTES + 40 /* r15 relative to cfa (rsp) */),
    540 
    -          "i"(RESERVED_C_STACK_BYTES + STG_RUN_STACK_FRAME_SIZE
    541 
    -              /* rip relative to cfa */)
    648 
    +          "i"(RESERVED_C_STACK_BYTES + 8 /* rbx relative to cfa (rsp) */),
    649 
    +          "i"(RESERVED_C_STACK_BYTES + 16 /* rbp relative to cfa (rsp) */),
    650 
    +          "i"(RESERVED_C_STACK_BYTES + 24 /* r12 relative to cfa (rsp) */),
    651 
    +          "i"(RESERVED_C_STACK_BYTES + 32 /* r13 relative to cfa (rsp) */),
    652 
    +          "i"(RESERVED_C_STACK_BYTES + 40 /* r14 relative to cfa (rsp) */),
    653 
    +          "i"(RESERVED_C_STACK_BYTES + 48 /* r15 relative to cfa (rsp) */)
    542 654
    543 655 
     #if defined(ENABLE_UNWINDING)
    544 
    -          , "i"((RSP_DELTA & 127) | (128 * ((RSP_DELTA >> 7) > 0)))
    656 
    +          /* LEB128-encoded offset to saved rsp (RESERVED_C_STACK_BYTES + 56) */
    657 
    +          , "i"((RSP_DELTA & 127) | (128 * ((RSP_DELTA >> 6) > 0)))
    545 658 
                 /* signed LEB128-encoded delta from rsp - byte 1 */
    546 
    -#if (RSP_DELTA >> 7) > 0
    547 
    -          , "i"(((RSP_DELTA >> 7) & 127) | (128 * ((RSP_DELTA >> 14) > 0)))
    659 
    +#if (RSP_DELTA >> 6) > 0
    660 
    +          , "i"(((RSP_DELTA >> 7) & 127) | (128 * ((RSP_DELTA >> 13) > 0)))
    548 661 
                 /* signed LEB128-encoded delta from rsp - byte 2 */
    549 662 
     #endif
    550 
    -
    551 
    -#if (RSP_DELTA >> 14) > 0
    552 
    -          , "i"(((RSP_DELTA >> 14) & 127) | (128 * ((RSP_DELTA >> 21) > 0)))
    663 
    +#if (RSP_DELTA >> 13) > 0
    664 
    +          , "i"(((RSP_DELTA >> 14) & 127) | (128 * ((RSP_DELTA >> 20) > 0)))
    553 665 
                 /* signed LEB128-encoded delta from rsp - byte 3 */
    554 666 
     #endif
    555 
    -
    556 
    -#if (RSP_DELTA >> 21) > 0
    557 
    -          , "i"(((RSP_DELTA >> 21) & 127) | (128 * ((RSP_DELTA >> 28) > 0)))
    667 
    +#if (RSP_DELTA >> 20) > 0
    668 
    +          , "i"(((RSP_DELTA >> 21) & 127) | (128 * ((RSP_DELTA >> 27) > 0)))
    558 669 
                 /* signed LEB128-encoded delta from rsp - byte 4 */
    559 670 
     #endif
    560 671 
     #undef RSP_DELTA
    561 
    -
    562 672 
     #endif /* defined(ENABLE_UNWINDING) */
    563 673
    564 674 
             );

  • rts/include/rts/Constants.h
    ... ... @@ -117,16 +117,14 @@
    117 117 
        How large is the stack frame saved by StgRun?
    118 118 
        world.  Used in StgCRun.c.
    119 119
    120 
    -   The size has to be enough to save the registers (see StgCRun)
    121 
    -   plus padding if the result is not 16 byte aligned.
    122 
    -   See the Note [Stack Alignment on X86] in StgCRun.c for details.
    120 
    +   The size has to be enough to save the registers (see StgCRun).
    123 121
    124 122 
        -------------------------------------------------------------------------- */
    125 123 
     #if defined(x86_64_HOST_ARCH)
    126 124 
     #  if defined(mingw32_HOST_OS)
    127 125 
     #    define STG_RUN_STACK_FRAME_SIZE 240
    128 126 
     #  else
    129 
    -#    define STG_RUN_STACK_FRAME_SIZE 48
    127 
    +#    define STG_RUN_STACK_FRAME_SIZE 56
    130 128 
     #  endif
    131 129 
     #endif
    132 130

  • testsuite/tests/simd/should_run/StackAlignment32.hs
    1 
    +{-# LANGUAGE MagicHash, UnboxedTuples, UnliftedFFITypes #-}
    2 
    +module StackAlignment32 where
    3 
    +import GHC.Exts
    4 
    +
    5 
    +foreign import ccall unsafe add10 :: DoubleX4# -> DoubleX4# -> DoubleX4# -> DoubleX4# -> DoubleX4# -> DoubleX4# -> DoubleX4# -> DoubleX4# -> DoubleX4# -> DoubleX4# -> DoubleX4#
    6 
    +
    7 
    +foo :: Double -> IO ()
    8 
    +foo (D# x) = do
    9 
    +  let a = broadcastDoubleX4# x
    10 
    +      b = packDoubleX4# (# 1.0##, 2.0##, 3.0##, 4.0## #)
    11 
    +      c = add10 a a a a a a a a a b
    12 
    +      (# c0, c1, c2, c3 #) = unpackDoubleX4# c
    13 
    +  print (D# c0, D# c1, D# c2, D# c3)
    14 
    +
    15 
    +foreign export ccall foo :: Double -> IO ()

  • testsuite/tests/simd/should_run/StackAlignment32.stdout
    1 
    +(1.1,2.1,3.1,4.1)
    2 
    +(2.1,3.1,4.1,5.1)
    3 
    +(3.2,4.2,5.2,6.2)

  • testsuite/tests/simd/should_run/StackAlignment32_main.c
    1 
    +#include "HsFFI.h"
    2 
    +#include <immintrin.h>
    3 
    +
    4 
    +extern void foo(double x);
    5 
    +
    6 
    +__m256d add10(__m256d a0, __m256d a1, __m256d a2, __m256d a3, __m256d a4, __m256d a5, __m256d a6, __m256d a7, __m256d a8, __m256d a9)
    7 
    +{
    8 
    +    // Test whether the stack is 32-byte aligned.
    9 
    +    // Under the System V ABI, the first eight parameters are passed in registers.
    10 
    +    // Here, `a8` and `a9` are passed on the stack.
    11 
    +    // We want the compiler to emit `vmovapd` so we can verify 32-byte stack alignment.
    12 
    +    // We cannot use `_mm256_add_pd(a0, a9)` because the compiler may fuse the load and
    13 
    +    // the addition and emit a `vaddpd m256` form, which permits an unaligned load of `a9`.
    14 
    +    // Therefore, we ensure that both operands are read from the stack,
    15 
    +    // so that at least one operand is loaded using `vmovapd`.
    16 
    +    return _mm256_add_pd(a8, a9);
    17 
    +}
    18 
    +
    19 
    +__attribute__((noinline))
    20 
    +void baz(void)
    21 
    +{
    22 
    +    // Make the stack pointer shift by 16 bytes on x86-64
    23 
    +    foo(2.2);
    24 
    +}
    25 
    +
    26 
    +__attribute__((noinline))
    27 
    +void bar(void)
    28 
    +{
    29 
    +    // Make the stack pointer shift by 16 bytes on x86-64
    30 
    +    foo(1.1);
    31 
    +    baz();
    32 
    +}
    33 
    +
    34 
    +int main(int argc, char *argv[])
    35 
    +{
    36 
    +    hs_init(&argc, &argv);
    37 
    +
    38 
    +    foo(0.1);
    39 
    +    bar();
    40 
    +
    41 
    +    hs_exit();
    42 
    +}

  • testsuite/tests/simd/should_run/StackAlignment64.hs
    1 
    +{-# LANGUAGE MagicHash, UnboxedTuples, UnliftedFFITypes #-}
    2 
    +module StackAlignment64 where
    3 
    +import GHC.Exts
    4 
    +
    5 
    +foreign import ccall unsafe add10 :: DoubleX8# -> DoubleX8# -> DoubleX8# -> DoubleX8# -> DoubleX8# -> DoubleX8# -> DoubleX8# -> DoubleX8# -> DoubleX8# -> DoubleX8# -> DoubleX8#
    6 
    +
    7 
    +foo :: Double -> IO ()
    8 
    +foo (D# x) = do
    9 
    +  let a = broadcastDoubleX8# x
    10 
    +      b = packDoubleX8# (# 1.0##, 2.0##, 3.0##, 4.0##, 5.0##, 6.0##, 7.0##, 8.0## #)
    11 
    +      c = add10 a a a a a a a a a b
    12 
    +      (# c0, c1, c2, c3, c4, c5, c6, c7 #) = unpackDoubleX8# c
    13 
    +  print (D# c0, D# c1, D# c2, D# c3, D# c4, D# c5, D# c6, D# c7)
    14 
    +
    15 
    +foreign export ccall foo :: Double -> IO ()

  • testsuite/tests/simd/should_run/StackAlignment64.stdout
    1 
    +(1.1,2.1,3.1,4.1,5.1,6.1,7.1,8.1)
    2 
    +(2.1,3.1,4.1,5.1,6.1,7.1,8.1,9.1)
    3 
    +(3.2,4.2,5.2,6.2,7.2,8.2,9.2,10.2)

  • testsuite/tests/simd/should_run/StackAlignment64_main.c
    1 
    +#include "HsFFI.h"
    2 
    +#include <immintrin.h>
    3 
    +
    4 
    +extern void foo(double x);
    5 
    +
    6 
    +__m512d add10(__m512d a0, __m512d a1, __m512d a2, __m512d a3, __m512d a4, __m512d a5, __m512d a6, __m512d a7, __m512d a8, __m512d a9)
    7 
    +{
    8 
    +    // Test whether the stack is 64-byte aligned.
    9 
    +    // Under the System V ABI, the first eight parameters are passed in registers.
    10 
    +    // Here, `a8` and `a9` are passed on the stack.
    11 
    +    // We want the compiler to emit `vmovapd` so we can verify 64-byte stack alignment.
    12 
    +    // We cannot use `_mm512_add_pd(a0, a9)` because the compiler may fuse the load and
    13 
    +    // the addition and emit a `vaddpd m512` form, which permits an unaligned load of `a9`.
    14 
    +    // Therefore, we ensure that both operands are read from the stack,
    15 
    +    // so that at least one operand is loaded using `vmovapd`.
    16 
    +    return _mm512_add_pd(a8, a9);
    17 
    +}
    18 
    +
    19 
    +__attribute__((noinline))
    20 
    +void baz(void)
    21 
    +{
    22 
    +    // Make the stack pointer shift by 16 bytes on x86-64
    23 
    +    foo(2.2);
    24 
    +}
    25 
    +
    26 
    +__attribute__((noinline))
    27 
    +void bar(void)
    28 
    +{
    29 
    +    // Make the stack pointer shift by 16 bytes on x86-64
    30 
    +    foo(1.1);
    31 
    +    baz();
    32 
    +}
    33 
    +
    34 
    +int main(int argc, char *argv[])
    35 
    +{
    36 
    +    hs_init(&argc, &argv);
    37 
    +
    38 
    +    foo(0.1);
    39 
    +    bar();
    40 
    +
    41 
    +    hs_exit();
    42 
    +}

  • testsuite/tests/simd/should_run/all.T
    ... ... @@ -92,7 +92,7 @@ test('T26411b', [], compile_and_run, ['-O'])
    92 92 
     # Even if the CPU we run on doesn't support *executing* those tests we should try to
    93 93 
     # compile them.
    94 94 
     # Currently even for compilation we only support 256+ bit on x86
    95 
    -only_V32_plus_compilation_support = unless(arch('x86_64'), skip)
    95 
    +only_V32_plus_compilation_support = unless(arch('x86_64') or arch('i386'), skip)
    96 96
    97 97 
     test('T25062_V32'
    98 98 
         ,   [ extra_hc_opts('-mavx2')
    ... ... @@ -116,17 +116,37 @@ test('T25486', [], compile_and_run, [''])
    116 116
    117 117 
     test('T26410_ffi'
    118 118 
         , [ only_ways(llvm_ways) # SIMD NCG TODO: support 512-bit wide vectors
    119 
    -      , unless(arch('x86_64') and have_cpu_feature('avx512f'), skip)
    119 
    +      , unless(arch('x86_64') or arch('i386'), skip)
    120 120 
           , extra_hc_opts('-mavx512f -optc -mavx512f -optlc -mcpu=penryn')
    121 
    -      , when(opsys('mingw32'), fragile(26595))
    122 121 
           ]
    123 122 
         , compile_and_run if have_cpu_feature('avx512f') else compile
    124 123 
         , ['T26410_ffi_c.c'])
    125 124
    126 125 
     test('T26410_prim'
    127 126 
         , [ only_ways(llvm_ways) # SIMD NCG TODO: support 512-bit wide vectors
    128 
    -      , unless(arch('x86_64') and have_cpu_feature('avx512f'), skip)
    127 
    +      , unless(arch('x86_64') or arch('i386'), skip)
    129 128 
           , extra_hc_opts('-mavx512f -optlc -mcpu=penryn')
    130 129 
           ]
    131 130 
         , compile_and_run if have_cpu_feature('avx512f') else compile
    132 131 
         , [''])
    132 
    +
    133 
    +test('FloatConstant', [], compile_and_run, [''])
    134 
    +test('IntConstant', [], compile_and_run, [''])
    135 
    +
    136 
    +test('StackAlignment32'
    137 
    +    , [ only_ways(llvm_ways) # SIMD NCG TODO: support 256-bit wide vectors
    138 
    +      , unless(arch('x86_64') or arch('i386'), skip)
    139 
    +      , extra_hc_opts("-mavx -optc -mavx -no-hs-main")
    140 
    +      ]
    141 
    +    , compile_and_run if have_cpu_feature('avx') else compile
    142 
    +    , ['StackAlignment32_main.c']
    143 
    +    )
    144 
    +
    145 
    +test('StackAlignment64'
    146 
    +    , [ only_ways(llvm_ways) # SIMD NCG TODO: support 512-bit wide vectors
    147 
    +      , unless(arch('x86_64') or arch('i386'), skip)
    148 
    +      , extra_hc_opts("-mavx512f -optc -mavx512f -no-hs-main")
    149 
    +      ]
    150 
    +    , compile_and_run if have_cpu_feature('avx512f') else compile
    151 
    +    , ['StackAlignment64_main.c']
    152 
    +    )

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications · Help