Cheng Shao pushed to branch wip/ci-make-install-j at Glasgow Haskell Compiler / GHC Commits: 1d0021cb by Cheng Shao at 2026-07-16T15:45:19+00:00 ci: fix abi-test job on non full-ci mr pipelines - - - - - b0ada958 by Cheng Shao at 2026-07-16T15:45:19+00:00 bindist: Fix make install -j race condition on macos/freebsd This patch fixes make install -j race condition on macos/freebsd. BSD install fails with EEXIST when multiple install processes concurrently create the same prefix directory. So we add an `install_dirs` prerequisite job that sequentially creates the directories for subsequent jobs to work with. Fixes #27499. Co-authored-by: Codex <codex@openai.com> - - - - - 06850ecf by Cheng Shao at 2026-07-16T15:45:19+00:00 ci: run bindist make install with -j This patch makes the ci scripts run `make install` with `-j` to reduce wall clock time when installing the bindist, see related issue for benchmark numbers. This only affects ghc ci logic, the user-facing default is up to distributors and is still `-j1`. Closes #27029. - - - - - 4 changed files: - .gitlab-ci.yml - .gitlab/ci.sh - + changelog.d/fix-make-install-j - hadrian/bindist/Makefile Changes: ===================================== .gitlab-ci.yml ===================================== @@ -499,7 +499,7 @@ hadrian-multi: || tar -xf ghc-x86_64-linux-deb13-release.tar.xz -C tmp pushd tmp/ghc-*/ ./configure --prefix=$root - make install + make install -j$CPUS popd rm -Rf tmp - export HC=$root/bin/ghc @@ -1005,7 +1005,7 @@ perf-nofib: tar -xf ../ghc-x86_64-linux-fedora43-release.tar.xz -C tmp pushd tmp/ghc-*/ ./configure --prefix=$root - make install + make install -j$CPUS popd rm -Rf tmp - export PATH=$root/bin:$PATH @@ -1047,7 +1047,7 @@ perf: || tar -xf ghc-x86_64-linux-deb13-release.tar.xz -C tmp pushd tmp/ghc-*/ ./configure --prefix=$root - make install + make install -j$CPUS popd rm -Rf tmp - export BOOT_HC=$(which ghc) @@ -1068,7 +1068,7 @@ perf: abi-test: stage: testing needs: - - job: x86_64-linux-deb13-validate + - job: x86_64-linux-deb13-numa-slow-validate optional: true - job: nightly-x86_64-linux-deb13-validate optional: true @@ -1083,11 +1083,12 @@ abi-test: - root=$(pwd)/ghc - | mkdir tmp - tar -xf ghc-x86_64-linux-deb13-validate.tar.xz -C tmp \ + tar -xf ghc-x86_64-linux-deb13-numa-slow-validate.tar.xz -C tmp \ + || tar -xf ghc-x86_64-linux-deb13-validate.tar.xz -C tmp \ || tar -xf ghc-x86_64-linux-deb13-release.tar.xz -C tmp pushd tmp/ghc-*/ ./configure --prefix=$root - make install + make install -j$CPUS popd rm -Rf tmp - export BOOT_HC=$(which ghc) ===================================== .gitlab/ci.sh ===================================== @@ -605,7 +605,7 @@ function make_install_destdir() { mkdir -p "$destdir" mkdir -p "$instdir" - run "$MAKE" DESTDIR="$destdir" install || fail "make install failed" + run "$MAKE" DESTDIR="$destdir" install -j"$cores" || fail "make install failed" # check for empty dir portably # https://superuser.com/a/667100 if find "$instdir" -mindepth 1 -maxdepth 1 | read; then @@ -899,8 +899,8 @@ function save_cache () { if [[ "${CI_JOB_NAME}" == *"darwin"* ]]; then # -a makes APFS behave like a COW file system # From man CP(1) - # copy files using clonefile(2). - # Note that if clonefile(2) is not supported for the target filesystem, + # copy files using clonefile(2). + # Note that if clonefile(2) is not supported for the target filesystem, # then cp will fallback to using copyfile(2) instead to ensure the copy still succeeds. cp -Rcf "$CABAL_DIR" "$CABAL_CACHE" else ===================================== changelog.d/fix-make-install-j ===================================== @@ -0,0 +1,4 @@ +section: packaging +synopsis: Fix race condition in bindist make install -j on macos/freebsd +issues: #27499 +mrs: !15707 ===================================== hadrian/bindist/Makefile ===================================== @@ -97,12 +97,26 @@ lib/targets/default.target : config.mk default.target @echo "Copying the bindist-configured default.target to lib/targets/default.target" cp default.target $@ +# sequentially create one layer of subdirectories under DESTDIR, then +# subsequent install_* jobs can happen concurrently without bsd +# install race condition. required for make install -j to work on +# macos/freebsd, see #27499. +.PHONY: install_dirs +install_dirs: + $(INSTALL_DIR) "$(DESTDIR)$(prefix)" + $(INSTALL_DIR) "$(DESTDIR)$(ActualBinsDir)" + $(INSTALL_DIR) "$(DESTDIR)$(WrapperBinsDir)" + $(INSTALL_DIR) "$(DESTDIR)$(ActualLibsDir)" + $(INSTALL_DIR) "$(DESTDIR)$(mandir)" + $(INSTALL_DIR) "$(DESTDIR)$(docdir)" + $(INSTALL_DIR) "$(DESTDIR)$(datadir)" + # We need to install binaries relative to libraries. BINARIES = $(wildcard ./bin/*) .PHONY: install_bin_libdir -install_bin_libdir: +install_bin_libdir: install_dirs @echo "Copying binaries to $(DESTDIR)$(ActualBinsDir)" - $(INSTALL_DIR) "$(DESTDIR)$(ActualBinsDir)" + for i in $(BINARIES); do \ if test -L "$$i"; then \ cp -RP "$$i" "$(DESTDIR)$(ActualBinsDir)"; \ @@ -112,15 +126,14 @@ install_bin_libdir: done .PHONY: install_bin_direct -install_bin_direct: +install_bin_direct: install_dirs @echo "Copying binaries to $(DESTDIR)$(WrapperBinsDir)" - $(INSTALL_DIR) "$(DESTDIR)$(WrapperBinsDir)" + $(INSTALL_PROGRAM) ./bin/* "$(DESTDIR)$(WrapperBinsDir)/" .PHONY: install_lib -install_lib: lib/settings lib/targets/default.target +install_lib: install_dirs lib/settings lib/targets/default.target @echo "Copying libraries to $(DESTDIR)$(ActualLibsDir)" - $(INSTALL_DIR) "$(DESTDIR)$(ActualLibsDir)" @dest="$(DESTDIR)$(ActualLibsDir)"; \ cd ./lib; \ @@ -146,9 +159,8 @@ install_lib: lib/settings lib/targets/default.target done; \ .PHONY: install_docs -install_docs: +install_docs: install_dirs @echo "Copying docs to $(DESTDIR)$(docdir)" - $(INSTALL_DIR) "$(DESTDIR)$(docdir)" if [ -d doc ]; then \ cd ./doc; $(FIND) . -type f -exec sh -c \ @@ -163,9 +175,9 @@ install_docs: fi .PHONY: install_data -install_data: +install_data: install_dirs @echo "Copying data to $(DESTDIR)share" - $(INSTALL_DIR) "$(DESTDIR)$(datadir)" + if [ -d share ]; then \ cd ./share; $(FIND) . -type f -exec sh -c \ '$(INSTALL_DIR) "$(DESTDIR)$(datadir)/`dirname $$1`" && \ @@ -177,18 +189,17 @@ MAN_SECTION := 1 MAN_PAGES := manpage/ghc.1 .PHONY: install_man -install_man: +install_man: install_dirs if [ -f $(MAN_PAGES) ]; then \ - $(INSTALL_DIR) "$(DESTDIR)$(mandir)"; \ $(INSTALL_DIR) "$(DESTDIR)$(mandir)/man$(MAN_SECTION)"; \ $(INSTALL_MAN) $(INSTALL_OPTS) $(MAN_PAGES) "$(DESTDIR)$(mandir)/man$(MAN_SECTION)"; \ fi export SHELL .PHONY: install_wrappers -install_wrappers: install_bin_libdir install_hsc2hs_wrapper +install_wrappers: install_dirs install_bin_libdir install_hsc2hs_wrapper @echo "Installing wrapper scripts" - $(INSTALL_DIR) "$(DESTDIR)$(WrapperBinsDir)" + for p in `cd ./wrappers; $(FIND) . ! -type d`; do \ mk/install_script.sh "$$p" "$(DESTDIR)/$(WrapperBinsDir)/$$p" "$(WrapperBinsDir)" "$(ActualBinsDir)" "$(ActualBinsDir)/$$p" "$(ActualLibsDir)" "$(docdir)" "$(includedir)"; \ done View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/57ecdb75e2e110634c8a716dd1d986c... -- View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/57ecdb75e2e110634c8a716dd1d986c... You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications: https://gitlab.haskell.org/-/profile/notifications | Help: https://gitlab.haskell.org/help