[Git][ghc/ghc][master] NCG/LA64: add cmpxchg and xchg primops

1 Apr 2026


      Marge Bot pushed to branch master at Glasgow Haskell Compiler / GHC


Commits:
5ae43275 by Peng Fan at 2026-04-01T19:01:06-04:00
NCG/LA64: add cmpxchg and xchg primops

And append some new instructions for LA664 uarch.

Apply fix to cmpxchg-prim by Andreas Klebinger.

Suggestions in https://gitlab.haskell.org/ghc/ghc/-/merge_requests/15515

- - - - -


3 changed files:

- compiler/GHC/CmmToAsm/LA64/CodeGen.hs
- compiler/GHC/CmmToAsm/LA64/Instr.hs
- compiler/GHC/CmmToAsm/LA64/Ppr.hs


Changes:

=====================================
compiler/GHC/CmmToAsm/LA64/CodeGen.hs
=====================================
@@ -57,6 +57,12 @@ import Control.Monad
 import GHC.Cmm.Dataflow.Label
 import GHC.Types.Unique.DSM
 import GHC.Types.Literal.Floating
+import GHC.Unit.Types ( ghcInternalUnitId )
+
+la664Enabled :: NatM Bool
+la664Enabled = do
+  config <- getConfig
+  return (ncgLa664Enabled config)
 
 -- [General layout of an NCG]
 cmmTopCodeGen ::
@@ -1651,6 +1657,10 @@ genPrim (MO_Prefetch_Data _n) []     [_]            = return nilOL
 genPrim (MO_AtomicRead w mo)  [dst]  [addr]         = genAtomicRead w mo dst addr
 genPrim (MO_AtomicWrite w mo) []     [addr,val]     = genAtomicWrite w mo addr val
 
+genPrim (MO_AtomicRMW width amop)    [dst] [addr,n] = genLibCCall (atomicRMWLabel width amop) [dst] [addr,n]
+genPrim (MO_Cmpxchg width)   [dst]   [addr,expe,new]  = genCmpxchg width dst addr expe new
+genPrim (MO_Xchg width)      [dst]   [addr,value]     = genXchg width dst addr value
+
 genPrim mop@(MO_S_Mul2     _w) _         _          = unsupported mop
 genPrim mop@(MO_S_QuotRem  _w) _         _          = unsupported mop
 genPrim mop@(MO_U_QuotRem  _w) _         _          = unsupported mop
@@ -1674,9 +1684,6 @@ genPrim (MO_PopCnt width)    [dst]   [src]          = genLibCCall (popCntLabel w
 genPrim (MO_Pdep width)      [dst]   [src,mask]     = genLibCCall (pdepLabel width) [dst] [src,mask]
 genPrim (MO_Pext width)      [dst]   [src,mask]     = genLibCCall (pextLabel width) [dst] [src,mask]
 genPrim (MO_UF_Conv width)   [dst]   [src]          = genLibCCall (word2FloatLabel width) [dst] [src]
-genPrim (MO_AtomicRMW width amop)    [dst] [addr,n] = genLibCCall (atomicRMWLabel width amop) [dst] [addr,n]
-genPrim (MO_Cmpxchg width)   [dst]   [addr,old,new] = genLibCCall (cmpxchgLabel width) [dst] [addr,old,new]
-genPrim (MO_Xchg width)      [dst]   [addr,val]     = genLibCCall (xchgLabel width) [dst] [addr,val]
 genPrim (MO_Memcpy _align)   []   [dst,src,n]       = genLibCCall (fsLit "memcpy")  [] [dst,src,n]
 genPrim (MO_Memmove _align)  []   [dst,src,n]       = genLibCCall (fsLit "memmove") [] [dst,src,n]
 genPrim (MO_Memcmp _align)   [rst]   [dst,src,n]    = genLibCCall (fsLit "memcmp")  [rst] [dst,src,n]
@@ -1872,6 +1879,20 @@ genBitRev w dst src = do
              )
     _ -> return ( code_x `snocOL` BITREV (OpReg w dst_reg) (OpReg w reg_x))
 
+genPrimCCall
+  :: FastString
+  -> [CmmFormal]
+  -> [CmmActual]
+  -> NatM InstrBlock
+
+genPrimCCall name dsts args = do
+  config <- getConfig
+  target <-
+    cmmMakeDynamicReference config CallReference
+      $ mkCmmCodeLabel ghcInternalUnitId name
+  let cconv = ForeignConvention CCallConv [NoHint] [NoHint] CmmMayReturn
+  genCCall target cconv dsts args
+
 -- Generate C call to the given function in libc
 genLibCCall :: FastString -> [CmmFormal] -> [CmmActual] -> NatM InstrBlock
 genLibCCall name dsts args = do
@@ -1945,6 +1966,52 @@ genAtomicWrite w mo addr val = do
              )
     _ ->  panic $ "Unexpected MemOrderAcquire on an AtomicWrite" ++ show mo
 
+genCmpxchg :: Width -> LocalReg -> CmmExpr -> CmmExpr -> CmmExpr -> NatM InstrBlock
+genCmpxchg w dst addr expe new = do
+  config <- getConfig
+  let
+    platform = ncgPlatform config
+    format = intFormat w
+
+  la664Enabled >>= \case
+
+    True -> do
+      (addr_reg, _, code_addr) <- getSomeReg addr
+      (expe_reg, _, code_expe) <- getSomeReg expe
+      (new_reg, _, code_new) <- getSomeReg new
+      let dst_reg = getRegisterReg platform (CmmLocal dst)
+      return $ code_addr `appOL` code_expe `appOL` code_new `appOL` toOL
+        [
+          -- Behave like the GCC builtin CAS operation
+          AMCASDB format (OpReg w expe_reg) (OpReg w new_reg) (OpReg w addr_reg),
+          MOV (OpReg w dst_reg) (OpReg w expe_reg)
+        ]
+
+    False ->
+      genPrimCCall (cmpxchgLabel w) [dst] [addr,expe,new]
+
+genXchg :: Width -> LocalReg -> CmmExpr -> CmmExpr -> NatM InstrBlock
+genXchg w dst addr val = do
+  config <- getConfig
+  tmp <- getNewRegNat II64
+  let
+    platform = ncgPlatform config
+    format = intFormat w
+
+  la664Enabled >>= \case
+
+    True -> do
+      (addr_reg, _, code_addr) <- getSomeReg addr
+      (val_reg, _, code_val) <- getSomeReg val
+      let dst_reg = getRegisterReg platform (CmmLocal dst)
+      return $ code_addr `appOL` code_val `appOL` toOL
+        [
+          AMSWAPDB format (OpReg w tmp) (OpReg w val_reg) (OpReg w addr_reg),
+          MOV (OpReg W64 dst_reg) (OpReg W64 tmp)
+        ]
+    False ->
+      genPrimCCall (xchgLabel w) [dst] [addr,val]
+
 -- -----------------------------------------------------------------------------
 {-
 Generating C calls
@@ -1977,6 +2044,7 @@ member of a structure or union argument, or a vector/floating-point argument
 wider than FRLEN may be passed in a GAR.
 -}
 
+-- Generate C call to the given function in ghc-prim
 genCCall
   :: CmmExpr            -- address of func call
   -> ForeignConvention  -- calling convention


=====================================
compiler/GHC/CmmToAsm/LA64/Instr.hs
=====================================
@@ -150,10 +150,11 @@ regUsageOfInstr platform instr = case instr of
   -- ranges, corresponding to 2 and 1 instruction implementations respectively.
   --
   -- BCOND1 is selected by default.
-  BCOND1 _ j d t           -> usage (regTarget t ++ regOp j ++ regOp d, [])
-  BCOND _ j d t            -> usage (regTarget t ++ regOp j ++ regOp d, [])
-  BEQZ j t                 -> usage (regTarget t ++ regOp j, [])
-  BNEZ j t                 -> usage (regTarget t ++ regOp j, [])
+  BCOND1 _ j d t        -> usage (regTarget t ++ regOp j ++ regOp d, [])
+  BCOND _ j d t         -> usage (regTarget t ++ regOp j ++ regOp d, [])
+  BEQZ1 o1 o2           -> usage (regOp o1 ++ regOp o2, [])
+  BEQZ j t              -> usage (regTarget t ++ regOp j, [])
+  BNEZ j t              -> usage (regTarget t ++ regOp j, [])
   -- 5. Common Memory Access Instructions --------------------------------------
   LD _ dst src             -> usage (regOp src, regOp dst)
   LDU _ dst src            -> usage (regOp src, regOp dst)
@@ -168,7 +169,17 @@ regUsageOfInstr platform instr = case instr of
   -- LDCOND dst src1 src2     -> usage (regOp src1 ++ regOp src2, regOp dst)
   -- STCOND dst src1 src2     -> usage (regOp src1 ++ regOp src2, regOp dst)
   -- 7. Atomic Memory Access Instructions --------------------------------------
+  -- In LoongArch, if the AM* atomic memory access instruction has the same register number as rd and rj,
+  -- the execution will trigger an Instruction Non-defined Exception. Here should be avoided.
   AMSWAPDB _ dst src1 src2 -> usage (regOp src1 ++ regOp src2, regOp dst)
+  AMADDDB  _ dst src1 src2 -> usage (regOp src1 ++ regOp src2, regOp dst)
+  AMANDDB  _ dst src1 src2 -> usage (regOp src1 ++ regOp src2, regOp dst)
+  AMORDB   _ dst src1 src2 -> usage (regOp src1 ++ regOp src2, regOp dst)
+  AMXORDB  _ dst src1 src2 -> usage (regOp src1 ++ regOp src2, regOp dst)
+  --AMCASDB  _ dst src1 src2 -> usage (regOp src1 ++ regOp src2, regOp src1 ++ regOp src2 ++ regOp dst)
+  AMCASDB  _ dst src1 src2 -> usage (regOp dst ++ regOp src1 ++ regOp src2, regOp dst)
+  LL  _ dst src1 _ -> usage (regOp src1, regOp dst)
+  SC  _ dst src1 _ -> usage (regOp src1, regOp dst)
   -- 8. Barrier Instructions ---------------------------------------------------
   DBAR _hint               -> usage ([], [])
   IBAR _hint               -> usage ([], [])
@@ -330,6 +341,7 @@ patchRegsOfInstr instr env = case instr of
     TAIL36 r t     -> TAIL36 (patchOp r) (patchTarget t)
     BCOND1 c j d t -> BCOND1 c (patchOp j) (patchOp d) (patchTarget t)
     BCOND c j d t  -> BCOND c (patchOp j) (patchOp d) (patchTarget t)
+    BEQZ1 o1 o2    -> BEQZ1 (patchOp o1) (patchOp o2)
     BEQZ j t       -> BEQZ (patchOp j) (patchTarget t)
     BNEZ j t       -> BNEZ (patchOp j) (patchTarget t)
     -- 5. Common Memory Access Instructions --------------------------------------
@@ -348,6 +360,13 @@ patchRegsOfInstr instr env = case instr of
     -- STCOND o1 o2 o3       -> STCOND  (patchOp o1)  (patchOp o2)  (patchOp o3)
     -- 7. Atomic Memory Access Instructions --------------------------------------
     AMSWAPDB f o1 o2 o3 -> AMSWAPDB f (patchOp o1) (patchOp o2) (patchOp o3)
+    AMADDDB f o1 o2 o3  -> AMADDDB f (patchOp o1) (patchOp o2) (patchOp o3)
+    AMANDDB f o1 o2 o3  -> AMANDDB f (patchOp o1) (patchOp o2) (patchOp o3)
+    AMORDB f o1 o2 o3   -> AMORDB f (patchOp o1) (patchOp o2) (patchOp o3)
+    AMXORDB f o1 o2 o3  -> AMXORDB f (patchOp o1) (patchOp o2) (patchOp o3)
+    AMCASDB f o1 o2 o3  -> AMCASDB f (patchOp o1) (patchOp o2) (patchOp o3)
+    LL f o1 o2 o3  -> LL f (patchOp o1) (patchOp o2) (patchOp o3)
+    SC f o1 o2 o3  -> SC f (patchOp o1) (patchOp o2) (patchOp o3)
     -- 8. Barrier Instructions ---------------------------------------------------
     DBAR o1             -> DBAR o1
     IBAR o1             -> IBAR o1
@@ -398,6 +417,7 @@ isJumpishInstr instr = case instr of
   TAIL36 {} -> True
   BCOND1 {} -> True
   BCOND {} -> True
+  BEQZ1 {} -> True
   BEQZ {} -> True
   BNEZ {} -> True
   _ -> False
@@ -718,6 +738,7 @@ data Instr
     | TAIL36 Operand Target
     | BCOND1 Cond Operand Operand Target
     | BCOND Cond Operand Operand Target
+    | BEQZ1 Operand Operand
     | BEQZ Operand Target
     | BNEZ Operand Target
     -- 5. Common Memory Access Instructions --------------------------------------
@@ -733,6 +754,13 @@ data Instr
     -- 6. Bound Check Memory Access Instructions ---------------------------------
     -- 7. Atomic Memory Access Instructions --------------------------------------
     | AMSWAPDB Format Operand Operand Operand
+    | AMADDDB Format Operand Operand Operand
+    | AMANDDB Format Operand Operand Operand
+    | AMORDB Format Operand Operand Operand
+    | AMXORDB Format Operand Operand Operand
+    | AMCASDB Format Operand Operand Operand
+    | LL Format Operand Operand Operand
+    | SC Format Operand Operand Operand
     -- 8. Barrier Instructions ---------------------------------------------------
     | DBAR BarrierType
     | IBAR BarrierType
@@ -839,6 +867,7 @@ instrCon i =
       TAIL36{} -> "TAIL36"
       BCOND1{} -> "BCOND1"
       BCOND{} -> "BCOND"
+      BEQZ1{} -> "BEQZ1"
       BEQZ{} -> "BEQZ"
       BNEZ{} -> "BNEZ"
       LD{} -> "LD"
@@ -851,6 +880,13 @@ instrCon i =
       STPTR{} -> "STPTR"
       PRELD{} -> "PRELD"
       AMSWAPDB{} -> "AMSWAPDB"
+      AMADDDB{} -> "AMADDDB"
+      AMANDDB{} -> "AMANDDB"
+      AMORDB{} -> "AMORDB"
+      AMXORDB{} -> "AMXORDB"
+      AMCASDB{} -> "AMCASDB"
+      LL{} -> "LL"
+      SC{} -> "SC"
       DBAR{} -> "DBAR"
       IBAR{} -> "IBAR"
       FCVT{} -> "FCVT"


=====================================
compiler/GHC/CmmToAsm/LA64/Ppr.hs
=====================================
@@ -852,6 +852,7 @@ pprInstr platform instr = case instr of
       line $ text "\tbgeu" <+> pprOp platform d <> comma <+> pprOp platform j <> comma <+> pprAsmLabel platform (mkLocalBlockLabel (getUnique bid))
     UGT ->
       line $ text "\tbltu" <+> pprOp platform d <> comma <+> pprOp platform j <> comma <+> pprAsmLabel platform (mkLocalBlockLabel (getUnique bid))
+
     _ -> line $ text "\t" <> pprBcond c <+> pprOp platform j <> comma <+> pprOp platform d <> comma <+> pprAsmLabel platform (mkLocalBlockLabel (getUnique bid))
 
   BCOND1 _ _ _ (TLabel _) -> panic "LA64.ppr: BCOND1: No conditional branching to TLabel!"
@@ -916,17 +917,18 @@ pprInstr platform instr = case instr of
 
   BCOND _ _ _ (TReg _) -> panic "LA64.ppr: BCOND: No conditional branching to registers!"
 
+  BEQZ1 o1 o2 | isImmOp o2 -> op2 (text "\tbeqz") o1 o2
   BEQZ j (TBlock bid) ->
     line $ text "\tbeqz" <+> pprOp platform j <> comma <+> pprAsmLabel platform (mkLocalBlockLabel (getUnique bid))
   BEQZ j (TLabel lbl) ->
     line $ text "\tbeqz" <+> pprOp platform j <> comma <+> pprAsmLabel platform lbl
-  BEQZ _ (TReg _)     -> panic "LA64.ppr: BEQZ: No conditional branching to registers!"
+  BEQZ _ (TReg _) -> panic "LA64.ppr: BEQZ: No conditional branching to registers!"
 
   BNEZ j (TBlock bid) ->
     line $ text "\tbnez" <+> pprOp platform j <> comma <+> pprAsmLabel platform (mkLocalBlockLabel (getUnique bid))
   BNEZ j (TLabel lbl) ->
     line $ text "\tbnez" <+> pprOp platform j <> comma <+> pprAsmLabel platform lbl
-  BNEZ _ (TReg _)     -> panic "LA64.ppr: BNEZ: No conditional branching to registers!"
+  BNEZ _ (TReg _) -> panic "LA64.ppr: BNEZ: No conditional branching to registers!"
 
   -- 5. Common Memory Access Instructions --------------------------------------
     -- LD.{B[U]/H[U]/W[U]/D}, ST.{B/H/W/D}: AddrRegImm
@@ -1020,8 +1022,29 @@ pprInstr platform instr = case instr of
   AMSWAPDB II32 o1 o2 o3 -> op3 (text "\tamswap_db.w") o1 o2 o3
   AMSWAPDB II64 o1 o2 o3 -> op3 (text "\tamswap_db.d") o1 o2 o3
     -- AM.{SWAP/ADD}[_DB].{B/H}
+  AMADDDB II8  o1 o2 o3 -> op3 (text "\tamadd_db.b") o1 o2 o3
+  AMADDDB II16 o1 o2 o3 -> op3 (text "\tamadd_db.h") o1 o2 o3
+  AMADDDB II32 o1 o2 o3 -> op3 (text "\tamadd_db.w") o1 o2 o3
+  AMADDDB II64 o1 o2 o3 -> op3 (text "\tamadd_db.d") o1 o2 o3
+
+  AMANDDB II32 o1 o2 o3 -> op3 (text "\tamand_db.w") o1 o2 o3
+  AMANDDB II64 o1 o2 o3 -> op3 (text "\tamand_db.d") o1 o2 o3
+
+  AMORDB II32 o1 o2 o3 -> op3 (text "\tamor_db.w") o1 o2 o3
+  AMORDB II64 o1 o2 o3 -> op3 (text "\tamor_db.d") o1 o2 o3
+
+  AMXORDB II32 o1 o2 o3 -> op3 (text "\tamxor_db.w") o1 o2 o3
+  AMXORDB II64 o1 o2 o3 -> op3 (text "\tamxor_db.d") o1 o2 o3
     -- AMCAS[_DB].{B/H/W/D}
+  AMCASDB II8  o1 o2 o3 -> op3 (text "\tamcas_db.b") o1 o2 o3
+  AMCASDB II16 o1 o2 o3 -> op3 (text "\tamcas_db.h") o1 o2 o3
+  AMCASDB II32 o1 o2 o3 -> op3 (text "\tamcas_db.w") o1 o2 o3
+  AMCASDB II64 o1 o2 o3 -> op3 (text "\tamcas_db.d") o1 o2 o3
     -- LL.{W/D}, SC.{W/D}
+  LL II32 o1 o2 o3 -> op3 (text "\tll.w") o1 o2 o3
+  SC II32 o1 o2 o3 -> op3 (text "\tsc.w") o1 o2 o3
+  LL II64 o1 o2 o3 -> op3 (text "\tll.d") o1 o2 o3
+  SC II64 o1 o2 o3 -> op3 (text "\tsc.d") o1 o2 o3
     -- SC.Q
     -- LL.ACQ.{W/D}, SC.REL.{W/D}
   -- 8. Barrier Instructions ---------------------------------------------------



View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/5ae43275ae8fa1a6938f2efcc774c36f...

-- 
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/5ae43275ae8fa1a6938f2efcc774c36f...
You're receiving this email because of your account on gitlab.haskell.org.

    

Marge Bot (＠marge-bot)

tags

participants (1)