Simon Jakobi pushed to branch wip/sjakobi/cmmsink-touch at Glasgow Haskell Compiler / GHC
Commits:
-
5b1e1e19
by Simon Jakobi at 2026-09-08T18:52:28+02:00
-
ef34e1e3
by Simon Jakobi at 2026-09-08T19:02:41+02:00
6 changed files:
- changelog.d/cmmsink-touch-27782
- compiler/GHC/Cmm/MachOp.hs
- compiler/GHC/Cmm/Sink.hs
- + testsuite/tests/codeGen/should_gen_asm/T27782b.asm
- + testsuite/tests/codeGen/should_gen_asm/T27782b.cmm
- testsuite/tests/codeGen/should_gen_asm/all.T
Changes:
| 1 | 1 | section: codegen
|
| 2 | -synopsis: The Cmm sinking pass no longer treats ``touch#`` as clobbering
|
|
| 3 | - memory, so a load preceding a ``touch#`` can be folded into the
|
|
| 4 | - instruction that uses it, as in ``ByteString`` indexing.
|
|
| 2 | +synopsis: The Cmm sinking pass no longer treats ``touch#``, arithmetic
|
|
| 3 | + helpers such as ``quotRemInt#`` and ``timesWord2#``, bit operations, or
|
|
| 4 | + libm calls as clobbering memory, so a load preceding one of them can be
|
|
| 5 | + folded into the instruction that uses it, as in ``ByteString`` indexing.
|
|
| 5 | 6 | issues: #27782
|
| 6 | 7 | mrs: !16650 |
| ... | ... | @@ -885,9 +885,139 @@ callishMachOpHints op = case op of |
| 885 | 885 | -- in GHC.Cmm.Sink.
|
| 886 | 886 | callishMachOpClobbersMemory :: CallishMachOp -> Bool
|
| 887 | 887 | callishMachOpClobbersMemory op = case op of
|
| 888 | + -- CQ[clobber-libm]
|
|
| 889 | + -- Q: these become C calls on most backends. Why is a C call to sin/pow
|
|
| 890 | + -- safe to sink a load past, given the stmCommitNestedTransaction
|
|
| 891 | + -- story in Note [Foreign calls clobber heap]?
|
|
| 892 | + -- A~ the counterexample is about RTS functions reached via ForeignTarget,
|
|
| 893 | + -- which stays clobbering. libm/ghc-prim helpers only read their
|
|
| 894 | + -- arguments and write errno, thread-local C memory no Cmm load reads.
|
|
| 895 | + -- Loads through caller-saved STG registers are stopped by rule (C1)
|
|
| 896 | + -- anyway (foreignTargetRegs in GHC.Cmm.Node).
|
|
| 897 | + MO_F64_Pwr -> False
|
|
| 898 | + MO_F64_Sin -> False
|
|
| 899 | + MO_F64_Cos -> False
|
|
| 900 | + MO_F64_Tan -> False
|
|
| 901 | + MO_F64_Sinh -> False
|
|
| 902 | + MO_F64_Cosh -> False
|
|
| 903 | + MO_F64_Tanh -> False
|
|
| 904 | + MO_F64_Asin -> False
|
|
| 905 | + MO_F64_Acos -> False
|
|
| 906 | + MO_F64_Atan -> False
|
|
| 907 | + MO_F64_Asinh -> False
|
|
| 908 | + MO_F64_Acosh -> False
|
|
| 909 | + MO_F64_Atanh -> False
|
|
| 910 | + MO_F64_Log -> False
|
|
| 911 | + MO_F64_Log1P -> False
|
|
| 912 | + MO_F64_Exp -> False
|
|
| 913 | + MO_F64_ExpM1 -> False
|
|
| 914 | + MO_F64_Fabs -> False
|
|
| 915 | + MO_F64_Sqrt -> False
|
|
| 916 | + MO_F32_Pwr -> False
|
|
| 917 | + MO_F32_Sin -> False
|
|
| 918 | + MO_F32_Cos -> False
|
|
| 919 | + MO_F32_Tan -> False
|
|
| 920 | + MO_F32_Sinh -> False
|
|
| 921 | + MO_F32_Cosh -> False
|
|
| 922 | + MO_F32_Tanh -> False
|
|
| 923 | + MO_F32_Asin -> False
|
|
| 924 | + MO_F32_Acos -> False
|
|
| 925 | + MO_F32_Atan -> False
|
|
| 926 | + MO_F32_Asinh -> False
|
|
| 927 | + MO_F32_Acosh -> False
|
|
| 928 | + MO_F32_Atanh -> False
|
|
| 929 | + MO_F32_Log -> False
|
|
| 930 | + MO_F32_Log1P -> False
|
|
| 931 | + MO_F32_Exp -> False
|
|
| 932 | + MO_F32_ExpM1 -> False
|
|
| 933 | + MO_F32_Fabs -> False
|
|
| 934 | + MO_F32_Sqrt -> False
|
|
| 935 | + |
|
| 936 | + MO_I64_ToI -> False
|
|
| 937 | + MO_I64_FromI -> False
|
|
| 938 | + MO_W64_ToW -> False
|
|
| 939 | + MO_W64_FromW -> False
|
|
| 940 | + MO_x64_Neg -> False
|
|
| 941 | + MO_x64_Add -> False
|
|
| 942 | + MO_x64_Sub -> False
|
|
| 943 | + MO_x64_Mul -> False
|
|
| 944 | + MO_I64_Quot -> False
|
|
| 945 | + MO_I64_Rem -> False
|
|
| 946 | + MO_W64_Quot -> False
|
|
| 947 | + MO_W64_Rem -> False
|
|
| 948 | + MO_x64_And -> False
|
|
| 949 | + MO_x64_Or -> False
|
|
| 950 | + MO_x64_Xor -> False
|
|
| 951 | + MO_x64_Not -> False
|
|
| 952 | + MO_x64_Shl -> False
|
|
| 953 | + MO_I64_Shr -> False
|
|
| 954 | + MO_W64_Shr -> False
|
|
| 955 | + MO_x64_Eq -> False
|
|
| 956 | + MO_x64_Ne -> False
|
|
| 957 | + MO_I64_Ge -> False
|
|
| 958 | + MO_I64_Gt -> False
|
|
| 959 | + MO_I64_Le -> False
|
|
| 960 | + MO_I64_Lt -> False
|
|
| 961 | + MO_W64_Ge -> False
|
|
| 962 | + MO_W64_Gt -> False
|
|
| 963 | + MO_W64_Le -> False
|
|
| 964 | + MO_W64_Lt -> False
|
|
| 965 | + |
|
| 966 | + MO_UF_Conv _ -> False
|
|
| 967 | + MO_S_Mul2 _ -> False
|
|
| 968 | + MO_S_QuotRem _ -> False
|
|
| 969 | + MO_U_QuotRem _ -> False
|
|
| 970 | + MO_U_QuotRem2 _ -> False
|
|
| 971 | + MO_Add2 _ -> False
|
|
| 972 | + MO_AddWordC _ -> False
|
|
| 973 | + MO_SubWordC _ -> False
|
|
| 974 | + MO_AddIntC _ -> False
|
|
| 975 | + MO_SubIntC _ -> False
|
|
| 976 | + MO_U_Mul2 _ -> False
|
|
| 977 | + |
|
| 978 | + MO_VS_Quot _ _ -> False
|
|
| 979 | + MO_VS_Rem _ _ -> False
|
|
| 980 | + MO_VU_Quot _ _ -> False
|
|
| 981 | + MO_VU_Rem _ _ -> False
|
|
| 982 | + MO_I64X2_Min -> False
|
|
| 983 | + MO_I64X2_Max -> False
|
|
| 984 | + MO_W64X2_Min -> False
|
|
| 985 | + MO_W64X2_Max -> False
|
|
| 986 | + |
|
| 888 | 987 | MO_Touch -> False -- pure liveness marker, no instructions
|
| 889 | 988 | MO_Prefetch_Data _ -> False -- reads memory, writes nothing
|
| 890 | - _ -> True
|
|
| 989 | + |
|
| 990 | + MO_Memcpy _ -> True
|
|
| 991 | + MO_Memset _ -> True
|
|
| 992 | + MO_Memmove _ -> True
|
|
| 993 | + -- CQ[clobber-memcmp]
|
|
| 994 | + -- Q: memcmp reads both buffers and writes nothing, like prefetch.
|
|
| 995 | + -- Is there any RTS use where sinking a load past it matters?
|
|
| 996 | + MO_Memcmp _ -> False
|
|
| 997 | + |
|
| 998 | + MO_PopCnt _ -> False
|
|
| 999 | + MO_Pdep _ -> False
|
|
| 1000 | + MO_Pext _ -> False
|
|
| 1001 | + MO_Clz _ -> False
|
|
| 1002 | + MO_Ctz _ -> False
|
|
| 1003 | + MO_BSwap _ -> False
|
|
| 1004 | + MO_BRev _ -> False
|
|
| 1005 | + |
|
| 1006 | + MO_AcquireFence -> True
|
|
| 1007 | + MO_ReleaseFence -> True
|
|
| 1008 | + MO_SeqCstFence -> True
|
|
| 1009 | + MO_AtomicRMW {} -> True
|
|
| 1010 | + -- CQ[clobber-atomic-read]
|
|
| 1011 | + -- Q: an atomic read writes nothing. Why keep it clobbering?
|
|
| 1012 | + -- A~ it carries an ordering: a plain load may not move past a seq-cst
|
|
| 1013 | + -- read, and there is nothing to gain from allowing the relaxed case.
|
|
| 1014 | + MO_AtomicRead {} -> True
|
|
| 1015 | + MO_AtomicWrite {} -> True
|
|
| 1016 | + MO_Cmpxchg _ -> True
|
|
| 1017 | + MO_Xchg _ -> True
|
|
| 1018 | + |
|
| 1019 | + MO_SuspendThread -> True
|
|
| 1020 | + MO_ResumeThread -> True
|
|
| 891 | 1021 | |
| 892 | 1022 | -- | The alignment of a 'memcpy'-ish operation.
|
| 893 | 1023 | machOpMemcpyishAlign :: CallishMachOp -> Maybe Int
|
| ... | ... | @@ -884,13 +884,23 @@ data AbsMem |
| 884 | 884 | -- float accesses to heap, stack or virtual global registers stored in the
|
| 885 | 885 | -- capability (e.g. with unregisterised build, see #19237).
|
| 886 | 886 | --
|
| 887 | --- Conversely, a few CallishMachOps neither write memory nor imply a
|
|
| 887 | +-- Conversely, most CallishMachOps neither write memory nor imply a
|
|
| 888 | 888 | -- barrier, and needn't block sinking ('callishMachOpClobbersMemory' in
|
| 889 | --- GHC.Cmm.MachOp classifies them):
|
|
| 889 | +-- GHC.Cmm.MachOp classifies them). Two deserve a word:
|
|
| 890 | +-- CQ[clobber-note-scope]
|
|
| 891 | +-- Q: the Note now says "most" but only explains touch# and prefetch. Where
|
|
| 892 | +-- is the argument for the arithmetic and libm ops?
|
|
| 893 | +-- A~ at callishMachOpClobbersMemory, see CQ[clobber-libm]. The
|
|
| 894 | +-- per-constructor rationale should live next to the classification,
|
|
| 895 | +-- the Note keeps the two cases that need a safety argument.
|
|
| 896 | +-- D~ move the touch#/prefetch bullets to the classification too, and have
|
|
| 897 | +-- the Note only point there.
|
|
| 890 | 898 | --
|
| 891 | 899 | -- * MO_Touch is a pure liveness marker: no backend emits any code for it.
|
| 892 | --- Sinking is intra-block, so the Core-level touch# reordering hazard
|
|
| 893 | --- of #18040 does not arise.
|
|
| 900 | +-- The hazard of #18040, a read of the touched object delayed past a GC
|
|
| 901 | +-- safe point, cannot arise here: after stack layout the only safe points
|
|
| 902 | +-- are CmmCall nodes, which rule (C3)(d) still blocks, and
|
|
| 903 | +-- suspendThread/resumeThread, which stay clobbering.
|
|
| 894 | 904 | --
|
| 895 | 905 | -- * MO_Prefetch_Data reads memory but writes nothing.
|
| 896 | 906 |
| 1 | +loadMul2:
|
|
| 2 | + movq
|
|
| 3 | + mulq
|
|
| 4 | + addq
|
|
| 5 | + movq
|
|
| 6 | + addq
|
|
| 7 | + jmp
|
|
| 8 | +loadBSwap:
|
|
| 9 | + bswapq
|
|
| 10 | + movq
|
|
| 11 | + addq
|
|
| 12 | + jmp |
| 1 | +#include "Cmm.h"
|
|
| 2 | + |
|
| 3 | +// A load must sink past callish ops that don't clobber memory, so that it
|
|
| 4 | +// can be folded into the instruction that uses it. mul2u_64 also clobbers
|
|
| 5 | +// rax/rdx, so keeping the load live across it would cost extra moves.
|
|
| 6 | + |
|
| 7 | +loadMul2 (W_ p, W_ x, W_ y) {
|
|
| 8 | + W_ a, hi, lo;
|
|
| 9 | + a = W_[p];
|
|
| 10 | + (hi, lo) = prim %mul2u_64(x, y);
|
|
| 11 | + return (a + hi + lo);
|
|
| 12 | +}
|
|
| 13 | + |
|
| 14 | +loadBSwap (W_ p, W_ x) {
|
|
| 15 | + W_ a, b;
|
|
| 16 | + a = W_[p];
|
|
| 17 | + (b) = prim %bswap64(x);
|
|
| 18 | + return (a + b);
|
|
| 19 | +} |
| ... | ... | @@ -28,6 +28,7 @@ test('T25233', [unless(arch('x86_64'), skip), |
| 28 | 28 | test('T25233b', [unless(arch('x86_64'), skip),
|
| 29 | 29 | when(unregisterised(), skip)], compile_grep_asm, ['cmm', True, ''])
|
| 30 | 30 | test('T27782', is_amd64_codegen, compile_cmp_asm, ['cmm', ''])
|
| 31 | +test('T27782b', is_amd64_codegen, compile_cmp_asm, ['cmm', ''])
|
|
| 31 | 32 | is_aarch64_codegen = [
|
| 32 | 33 | unless(arch('aarch64'), skip),
|
| 33 | 34 | when(unregisterised(), skip),
|