GitLab

Marge Bot pushed to branch master at Glasgow Haskell Compiler / GHC

Commits:

cdb74049

by Cheng Shao at 2026-01-22T14:52:36-05:00

rts: avoid Cmm loop to initialize Array#/SmallArray#

Previously, `newArray#`/`newSmallArray#` called an RTS C function to
allocate the `Array#`/`SmallArray#`, then used a Cmm loop to
initialize the elements. Cmm doesn't have native for-loop so the code
is a bit awkward, and it's less efficient than a C loop, since the C
compiler can effectively vectorize the loop with optimizations.

So this patch moves the loop that initializes the elements to the C
side. `allocateMutArrPtrs`/`allocateSmallMutArrPtrs` now takes a new
`init` argument and initializes the elements if `init` is non-NULL.

7 changed files:

rts/AllocArray.c
rts/AllocArray.h
rts/ClosureTable.c
rts/Heap.c
rts/PrimOps.cmm
rts/Threads.c
rts/Weak.c

Changes:

rts/AllocArray.c

@@ -5,6 +5,7 @@
  StgMutArrPtrs *allocateMutArrPtrs (Capability *cap,
                                     StgWord nelements,
 +                                   StgClosure *init,
                                     CostCentreStack *ccs USED_IF_PROFILING)
+ {
      /* All sizes in words */
@@ -25,6 +26,12 @@ StgMutArrPtrs *allocateMutArrPtrs (Capability *cap,
      arr->ptrs = nelements;
      arr->size = arrsize;
 +    if (init != NULL) {
 +        for (StgWord i = 0; i < nelements; ++i) {
 +            arr->payload[i] = init;
 +        }
 +    }
++
      /* Initialize the card array. Note that memset needs sizes in bytes. */
      memset(&(arr->payload[nelements]), 0, mutArrPtrsCards(nelements));
@@ -33,6 +40,7 @@ StgMutArrPtrs *allocateMutArrPtrs (Capability *cap,
  StgSmallMutArrPtrs *allocateSmallMutArrPtrs (Capability *cap,
                                               StgWord nelements,
 +                                             StgClosure *init,
                                               CostCentreStack *ccs
                                                 USED_IF_PROFILING)
+ {
@@ -47,6 +55,13 @@ StgSmallMutArrPtrs *allocateSmallMutArrPtrs (Capability *cap,
      /* No write barrier needed since this is a new allocation. */
      SET_HDR(arr, &stg_SMALL_MUT_ARR_PTRS_DIRTY_info, ccs);
      arr->ptrs = nelements;
++
 +    if (init != NULL) {
 +        for (StgWord i = 0; i < nelements; ++i) {
 +            arr->payload[i] = init;
 +        }
 +    }
++
      return arr;
+ }

rts/AllocArray.h

@@ -21,16 +21,19 @@
   */
  /* Allocate a StgMutArrPtrs for a given number of elements. It is allocated in
 - * the DIRTY state.
 + * the DIRTY state. If init is non-NULL, initialize payload elements to init.
   */
  StgMutArrPtrs *allocateMutArrPtrs (Capability *cap,
                                     StgWord nelements,
 +                                   StgClosure *init,
                                     CostCentreStack *ccs);
 -/* Allocate a StgSmallMutArrPtrs for a given number of elements.
 +/* Allocate a StgSmallMutArrPtrs for a given number of elements. If init is
 + * non-NULL, initialize payload elements to init.
   */
  StgSmallMutArrPtrs *allocateSmallMutArrPtrs (Capability *cap,
                                               StgWord nelements,
 +                                             StgClosure *init,
                                               CostCentreStack *ccs);
  /* Allocate a StgArrBytes for a given number of bytes.

rts/ClosureTable.c

@@ -46,7 +46,7 @@ bool enlargeClosureTable(Capability *cap, ClosureTable *t, int newcapacity)
      ASSERT(newcapacity > oldcapacity);
      StgMutArrPtrs *newarr;
 -    newarr = allocateMutArrPtrs(cap, newcapacity, CCS_SYSTEM_OR_NULL);
 +    newarr = allocateMutArrPtrs(cap, newcapacity, NULL, CCS_SYSTEM_OR_NULL);
      if (RTS_UNLIKELY(newarr == NULL)) return false;
      StgArrBytes *newfree;
@@ -276,4 +276,3 @@ static bool isCompactClosureTable(ClosureTable *t)
+     }
      return isCompact;
+ }
+-

rts/Heap.c

@@ -279,7 +279,7 @@ StgMutArrPtrs *heap_view_closurePtrs(Capability *cap, StgClosure *closure) {
      StgClosure **ptrs = (StgClosure **) stgMallocBytes(sizeof(StgClosure *) * size, "heap_view_closurePtrs");
      StgWord nptrs = collect_pointers(closure, ptrs);
 -    StgMutArrPtrs *arr = allocateMutArrPtrs(cap, nptrs, cap->r.rCCCS);
 +    StgMutArrPtrs *arr = allocateMutArrPtrs(cap, nptrs, NULL, cap->r.rCCCS);
      if (RTS_UNLIKELY(arr == NULL)) goto end;
      SET_INFO((StgClosure *) arr, &stg_MUT_ARR_PTRS_FROZEN_CLEAN_info);

rts/PrimOps.cmm

@@ -386,24 +386,11 @@ stg_newArrayzh ( W_ n /* words */, gcptr init )
      again: MAYBE_GC(again);
 -    ("ptr" arr) = ccall allocateMutArrPtrs(MyCapability() "ptr", n, CCCS);
 +    ("ptr" arr) = ccall allocateMutArrPtrs(MyCapability() "ptr", n, init "ptr", CCCS);
      if (arr == NULL) (likely: False) {
          jump stg_raisezh(HsIface_heapOverflow_closure(W_[ghc_hs_iface]));
+     }
 -    // Initialise all elements of the array with the value init
 -    W_ p;
 -    p = arr + SIZEOF_StgMutArrPtrs;
 -    // Avoid the shift for `WDS(n)` in the inner loop
 -    W_ limit;
 -    limit = arr + SIZEOF_StgMutArrPtrs + WDS(n);
 -  for:
 -    if (p < limit) (likely: True) {
 -        W_[p] = init;
 -        p = p + WDS(1);
 -        goto for;
 -    }
+-
      return (arr);
+ }
@@ -496,24 +483,11 @@ stg_newSmallArrayzh ( W_ n /* words */, gcptr init )
      again: MAYBE_GC(again);
 -    ("ptr" arr) = ccall allocateSmallMutArrPtrs(MyCapability() "ptr", n, CCCS);
 +    ("ptr" arr) = ccall allocateSmallMutArrPtrs(MyCapability() "ptr", n, init "ptr", CCCS);
      if (arr == NULL) (likely: False) {
          jump stg_raisezh(HsIface_heapOverflow_closure(W_[ghc_hs_iface]));
+     }
 -    // Initialise all elements of the array with the value init
 -    W_ p;
 -    p = arr + SIZEOF_StgSmallMutArrPtrs;
 -    // Avoid the shift for `WDS(n)` in the inner loop
 -    W_ limit;
 -    limit = arr + SIZEOF_StgSmallMutArrPtrs + WDS(n);
 -  for:
 -    if (p < limit) (likely: True) {
 -        W_[p] = init;
 -        p = p + WDS(1);
 -        goto for;
 -    }
+-
      return (arr);
+ }

rts/Threads.c

@@ -894,7 +894,7 @@ StgMutArrPtrs *listThreads(Capability *cap)
+     }
      // Allocate a suitably-sized array...
 -    StgMutArrPtrs *arr = allocateMutArrPtrs(cap, n_threads, cap->r.rCCCS);
 +    StgMutArrPtrs *arr = allocateMutArrPtrs(cap, n_threads, NULL, cap->r.rCCCS);
      if (RTS_UNLIKELY(arr == NULL)) goto end;
      // Populate it...

rts/Weak.c

@@ -146,7 +146,7 @@ scheduleFinalizers(Capability *cap, StgWeak *list)
      debugTrace(DEBUG_weak, "weak: batching %d finalizers", n);
 -    StgMutArrPtrs *arr = allocateMutArrPtrs(cap, n, CCS_SYSTEM_OR_NULL);
 +    StgMutArrPtrs *arr = allocateMutArrPtrs(cap, n, NULL, CCS_SYSTEM_OR_NULL);
      if (RTS_UNLIKELY(arr == NULL)) exitHeapOverflow();
      // No write barrier needed here; this array is only going to referred to by this core.
      SET_INFO((StgClosure *) arr, &stg_MUT_ARR_PTRS_FROZEN_CLEAN_info);