GitLab

Cheng Shao pushed to branch wip/ci-make-install-j at Glasgow Haskell Compiler / GHC

Commits:

1d0021cb

by Cheng Shao at 2026-07-16T15:45:19+00:00

ci: fix abi-test job on non full-ci mr pipelines

b0ada958

by Cheng Shao at 2026-07-16T15:45:19+00:00

bindist: Fix make install -j race condition on macos/freebsd

This patch fixes make install -j race condition on macos/freebsd. BSD
install fails with EEXIST when multiple install processes concurrently
create the same prefix directory. So we add an `install_dirs`
prerequisite job that sequentially creates the directories for
subsequent jobs to work with. Fixes #27499.

Co-authored-by: Codex <codex@openai.com>

06850ecf

by Cheng Shao at 2026-07-16T15:45:19+00:00

ci: run bindist make install with -j

This patch makes the ci scripts run `make install` with `-j` to reduce
wall clock time when installing the bindist, see related issue for
benchmark numbers. This only affects ghc ci logic, the user-facing
default is up to distributors and is still `-j1`. Closes #27029.

4 changed files:

.gitlab-ci.yml
.gitlab/ci.sh
+ changelog.d/fix-make-install-j
hadrian/bindist/Makefile

Changes:

.gitlab-ci.yml

@@ -499,7 +499,7 @@ hadrian-multi:
          || tar -xf ghc-x86_64-linux-deb13-release.tar.xz -C tmp
        pushd tmp/ghc-*/
        ./configure --prefix=$root
 -      make install
 +      make install -j$CPUS
        popd
        rm -Rf tmp
      - export HC=$root/bin/ghc
@@ -1005,7 +1005,7 @@ perf-nofib:
        tar -xf ../ghc-x86_64-linux-fedora43-release.tar.xz -C tmp
        pushd tmp/ghc-*/
        ./configure --prefix=$root
 -      make install
 +      make install -j$CPUS
        popd
        rm -Rf tmp
      - export PATH=$root/bin:$PATH
@@ -1047,7 +1047,7 @@ perf:
          || tar -xf ghc-x86_64-linux-deb13-release.tar.xz -C tmp
        pushd tmp/ghc-*/
        ./configure --prefix=$root
 -      make install
 +      make install -j$CPUS
        popd
        rm -Rf tmp
      - export BOOT_HC=$(which ghc)
@@ -1068,7 +1068,7 @@ perf:
  abi-test:
    stage: testing
    needs:
 -    - job: x86_64-linux-deb13-validate
 +    - job: x86_64-linux-deb13-numa-slow-validate
        optional: true
      - job: nightly-x86_64-linux-deb13-validate
        optional: true
@@ -1083,11 +1083,12 @@ abi-test:
      - root=$(pwd)/ghc
      - |
        mkdir tmp
 -      tar -xf ghc-x86_64-linux-deb13-validate.tar.xz -C tmp \
 +      tar -xf ghc-x86_64-linux-deb13-numa-slow-validate.tar.xz -C tmp \
 +        || tar -xf ghc-x86_64-linux-deb13-validate.tar.xz -C tmp \
          || tar -xf ghc-x86_64-linux-deb13-release.tar.xz -C tmp
        pushd tmp/ghc-*/
        ./configure --prefix=$root
 -      make install
 +      make install -j$CPUS
        popd
        rm -Rf tmp
      - export BOOT_HC=$(which ghc)

.gitlab/ci.sh

@@ -605,7 +605,7 @@ function make_install_destdir() {
    mkdir -p "$destdir"
    mkdir -p "$instdir"
 -  run "$MAKE" DESTDIR="$destdir" install || fail "make install failed"
 +  run "$MAKE" DESTDIR="$destdir" install -j"$cores" || fail "make install failed"
    # check for empty dir portably
    # https://superuser.com/a/667100
    if find "$instdir" -mindepth 1 -maxdepth 1 | read; then
@@ -899,8 +899,8 @@ function save_cache () {
    if [[ "${CI_JOB_NAME}" == *"darwin"* ]]; then
      # -a makes APFS behave like a COW file system
      # From man CP(1)
 -    # copy files using clonefile(2).
 -    # Note that if clonefile(2) is not supported for the target filesystem,
 +    # copy files using clonefile(2).
 +    # Note that if clonefile(2) is not supported for the target filesystem,
      # then cp will fallback to using copyfile(2) instead to ensure the copy still succeeds.
      cp -Rcf "$CABAL_DIR" "$CABAL_CACHE"
    else

changelog.d/fix-make-install-j

 +section: packaging
 +synopsis: Fix race condition in bindist make install -j on macos/freebsd
 +issues: #27499
 +mrs: !15707

hadrian/bindist/Makefile

@@ -97,12 +97,26 @@ lib/targets/default.target : config.mk default.target
  	@echo "Copying the bindist-configured default.target to lib/targets/default.target"
  	cp default.target $@
 +# sequentially create one layer of subdirectories under DESTDIR, then
 +# subsequent install_* jobs can happen concurrently without bsd
 +# install race condition. required for make install -j to work on
 +# macos/freebsd, see #27499.
 +.PHONY: install_dirs
 +install_dirs:
 +	$(INSTALL_DIR) "$(DESTDIR)$(prefix)"
 +	$(INSTALL_DIR) "$(DESTDIR)$(ActualBinsDir)"
 +	$(INSTALL_DIR) "$(DESTDIR)$(WrapperBinsDir)"
 +	$(INSTALL_DIR) "$(DESTDIR)$(ActualLibsDir)"
 +	$(INSTALL_DIR) "$(DESTDIR)$(mandir)"
 +	$(INSTALL_DIR) "$(DESTDIR)$(docdir)"
 +	$(INSTALL_DIR) "$(DESTDIR)$(datadir)"
++
  # We need to install binaries relative to libraries.
  BINARIES = $(wildcard ./bin/*)
  .PHONY: install_bin_libdir
 -install_bin_libdir:
 +install_bin_libdir: install_dirs
  	@echo "Copying binaries to $(DESTDIR)$(ActualBinsDir)"
 -	$(INSTALL_DIR) "$(DESTDIR)$(ActualBinsDir)"
++
  	for i in $(BINARIES); do \
  		if test -L "$$i"; then \
  			cp -RP "$$i" "$(DESTDIR)$(ActualBinsDir)"; \
@@ -112,15 +126,14 @@ install_bin_libdir:
  	done
  .PHONY: install_bin_direct
 -install_bin_direct:
 +install_bin_direct: install_dirs
  	@echo "Copying binaries to $(DESTDIR)$(WrapperBinsDir)"
 -	$(INSTALL_DIR) "$(DESTDIR)$(WrapperBinsDir)"
++
  	$(INSTALL_PROGRAM) ./bin/* "$(DESTDIR)$(WrapperBinsDir)/"
  .PHONY: install_lib
 -install_lib: lib/settings lib/targets/default.target
 +install_lib: install_dirs lib/settings lib/targets/default.target
  	@echo "Copying libraries to $(DESTDIR)$(ActualLibsDir)"
 -	$(INSTALL_DIR) "$(DESTDIR)$(ActualLibsDir)"
  	@dest="$(DESTDIR)$(ActualLibsDir)"; \
  	cd ./lib; \
@@ -146,9 +159,8 @@ install_lib: lib/settings lib/targets/default.target
  	done; \
  .PHONY: install_docs
 -install_docs:
 +install_docs: install_dirs
  	@echo "Copying docs to $(DESTDIR)$(docdir)"
 -	$(INSTALL_DIR) "$(DESTDIR)$(docdir)"
  	if [ -d doc ]; then \
  		cd ./doc; $(FIND) . -type f -exec sh -c \
@@ -163,9 +175,9 @@ install_docs:
  	fi
  .PHONY: install_data
 -install_data:
 +install_data: install_dirs
  	@echo "Copying data to $(DESTDIR)share"
 -	$(INSTALL_DIR) "$(DESTDIR)$(datadir)"
++
  	if [ -d share ]; then \
  		cd ./share; $(FIND) . -type f -exec sh -c \
  			'$(INSTALL_DIR) "$(DESTDIR)$(datadir)/`dirname $$1`" && \
@@ -177,18 +189,17 @@ MAN_SECTION := 1
  MAN_PAGES := manpage/ghc.1
  .PHONY: install_man
 -install_man:
 +install_man: install_dirs
  	if [ -f $(MAN_PAGES) ]; then \
 -		$(INSTALL_DIR) "$(DESTDIR)$(mandir)"; \
  		$(INSTALL_DIR) "$(DESTDIR)$(mandir)/man$(MAN_SECTION)"; \
  		$(INSTALL_MAN) $(INSTALL_OPTS) $(MAN_PAGES) "$(DESTDIR)$(mandir)/man$(MAN_SECTION)"; \
  	fi
  export SHELL
  .PHONY: install_wrappers
 -install_wrappers: install_bin_libdir install_hsc2hs_wrapper
 +install_wrappers: install_dirs install_bin_libdir install_hsc2hs_wrapper
  	@echo "Installing wrapper scripts"
 -	$(INSTALL_DIR) "$(DESTDIR)$(WrapperBinsDir)"
++
  	for p in `cd ./wrappers; $(FIND) . ! -type d`; do \
  	    mk/install_script.sh "$$p" "$(DESTDIR)/$(WrapperBinsDir)/$$p" "$(WrapperBinsDir)" "$(ActualBinsDir)" "$(ActualBinsDir)/$$p" "$(ActualLibsDir)" "$(docdir)" "$(includedir)"; \
  	done