Sven Tennie pushed to branch wip/supersven/hadrian-cross-stage3 at Glasgow Haskell Compiler / GHC Commits: d748b106 by Sven Tennie at 2026-07-26T19:42:56+02:00 hadrian: Add stage3 cross-bindist target with separate output folder Add a 'binary-dist-stage3' target that packages target executables (produced by the stage2 cross-compiler) into a separate 'bindist-stage3/' folder, distinct from the stage2 cross-compiler bindist in 'bindist/'. Key changes: - BindistConfig: add 'bindistFolder' field so each config knows its parent output folder. crossBindist uses 'bindist', targetBindist uses 'bindist-stage3'. implicitBindistConfig now considers finalStage to pick targetBindist for stage3 builds. - BinaryDist: - Drive configure/Makefile/install-file rules for both folders via forM_ instead of a single 'bindist' path; per-stage distrib directory avoids autoreconf races. - binary-dist-stage3 phony outputs to 'bindist-stage3'. - generateSettings is passed the compiler stage explicitly; ghc-pkg recache uses the executable_stage builder. - pkgToWrappers honors the supplied stage for program naming. - Generate: per-stage configure.ac templates (templateRuleForStages) with correct Build/Host platform interpolation for stage3 (target = host = build). generateSettings takes the compiler stage as a parameter rather than inferring it as predStage. - CabalReinstall: use executable_stage from implicitBindistConfig instead of hard-coded Stage2 for wrapper generation. This allows a single cross-compile to produce both a cross-compiler bindist (x86_64 host) and a target-architecture bindist (e.g. RISC-V) that can be installed and run natively on the target. - - - - - 749b92da by Sven Tennie at 2026-07-26T19:42:56+02:00 ci: Combine cross stage2 and stage3 bindists in one job Rename crossConfig to stage2CrossConfig for clarity and turn the nightly x86_64-linux-deb13-riscv-cross job into a combined stage3 job (crossStage = Just 3). With CROSS_STAGE=3, build_hadrian invokes both the 'binary-dist' and 'binary-dist-stage3' targets, then renames the resulting tarballs via BIN_DIST_NAME and BIN_DIST_NAME_STAGE3. The combined job produces two artifacts: the cross-compiler bindist (stage2, runs on x86_64) and the target bindist (stage3, native RISC-V executables) under their historical names, so downstream consumers need no changes. gen_ci.hs: - stage2CrossConfig replaces crossConfig. - binDistNameStage3 generates the backward-compatible stage3 artifact name for jobs with crossStage == Just 3. - jobVariables advertises BIN_DIST_NAME_STAGE3 for stage3 jobs. - stage3Artifacts adds the stage3 tarball to artifactPaths. - cross_jobs: riscv job now sets crossStage = Just 3. ci.sh: - CROSS_STAGE=3 builds both binary-dist and binary-dist-stage3. - After building, rename the stage3 tarball from _build/bindist-stage3/ to $BIN_DIST_NAME_STAGE3.tar.xz. - cross_prefix is empty for CROSS_STAGE=3 (target GHC runs natively on the target, no host-target distinction at install time). jobs.yaml: regenerated. The riscv job keeps its existing name, now with CROSS_STAGE=3 and the extra stage3 artifact + variable. - - - - - 8 changed files: - .gitignore - .gitlab/ci.sh - .gitlab/generate-ci/gen_ci.hs - .gitlab/jobs.yaml - distrib/configure.ac.in - hadrian/src/BindistConfig.hs - hadrian/src/Rules/BinaryDist.hs - hadrian/src/Rules/Generate.hs Changes: ===================================== .gitignore ===================================== @@ -120,7 +120,6 @@ _darcs/ /compiler/GHC/CmmToLlvm/Version/Bounds.hs /compiler/ghc.cabal /compiler/ghc.cabal.old -/distrib/configure.ac /distrib/ghc.iss /docs/index.html /docs/man ===================================== .gitlab/ci.sh ===================================== @@ -575,7 +575,15 @@ function build_hadrian() { case "${CROSS_STAGE:-2}" in 2) BINDIST_TARGET="binary-dist";; - 3) BINDIST_TARGET="binary-dist-stage3";; + # Stage2 cross-compiler bindists are (almost) a byproduct of Stage3 + # cross-compiled bindists. So, we bundle both of them when the Stage3 + # bindist is built. + 3) + BINDIST_TARGET="binary-dist binary-dist-stage3" + if [[ -z "${BIN_DIST_NAME_STAGE3:-}" ]]; then + fail "CROSS_STAGE=3 requires BIN_DIST_NAME_STAGE3 to be set" + fi + ;; *) fail "Unknown CROSS_STAGE, must be 2 or 3";; esac @@ -590,6 +598,9 @@ function build_hadrian() { *) run_hadrian test:all_deps $BINDIST_TARGET mv _build/bindist/ghc*.tar.xz "$BIN_DIST_NAME.tar.xz" + if [[ "${CROSS_STAGE:-2}" == "3" ]]; then + mv _build/bindist-stage3/ghc*.tar.xz "$BIN_DIST_NAME_STAGE3.tar.xz" + fi ;; esac fi @@ -696,6 +707,26 @@ function test_hadrian() { # --- # > main = putStrLn "hello world" run diff -w expected actual + + if [[ "${CROSS_STAGE:-2}" == "3" ]]; then + local stage3_dir + stage3_dir="$(echo _build/bindist-stage3/ghc-*/)" + local stage3_ghc="$stage3_dir/bin/ghc$exe" + + info "Smoke-testing stage3 compiler..." + file "$stage3_ghc" + run ${CROSS_EMULATOR} "$stage3_ghc" --info + + run ${CROSS_EMULATOR} "$stage3_ghc" -package ghc "$TOP/.gitlab/hello.hs" -o hello-stage3 + + if [[ "${CROSS_TARGET:-no_cross_target}" =~ "mingw" ]]; then + ${CROSS_EMULATOR:-} ./hello-stage3.exe > actual-stage3 + else + ${CROSS_EMULATOR:-} ./hello-stage3 > actual-stage3 + fi + + run diff -w expected actual-stage3 + fi elif [[ -n "${REINSTALL_GHC:-}" ]]; then run_hadrian \ test \ ===================================== .gitlab/generate-ci/gen_ci.hs ===================================== @@ -159,7 +159,7 @@ data BuildConfig , withNuma :: Bool , withZstd :: Bool , crossTarget :: Maybe String - , crossStage :: Maybe Int + , crossStage :: Maybe FinalCrossStage , crossEmulator :: CrossEmulator , configureWrapper :: Maybe String , fullyStatic :: Bool @@ -275,13 +275,26 @@ static = vanilla { fullyStatic = True } staticNativeInt :: BuildConfig staticNativeInt = static { bignumBackend = Native } +-- | The final stage for which binary distrubutions should be built +-- +-- `Stage2` builds a cross-compiler (build == host, host /= target). `Stage3` +-- implies `Stage2` and additionally builds a cross-compiled compiler (build /= +-- host, host == target). +data FinalCrossStage = Stage2 | Stage3 + deriving (Eq, Ord) + +crossStageToInt :: FinalCrossStage -> Int +crossStageToInt Stage2 = 2 +crossStageToInt Stage3 = 3 + crossConfig :: String -- ^ target triple -> CrossEmulator -- ^ emulator for testing -> Maybe String -- ^ Configure wrapper + -> FinalCrossStage -- ^ final stage to build -> BuildConfig -crossConfig triple emulator configure_wrapper = +crossConfig triple emulator configure_wrapper crossStage = vanilla { crossTarget = Just triple - , crossStage = Just 2 + , crossStage = Just crossStage , crossEmulator = emulator , configureWrapper = configure_wrapper } @@ -356,6 +369,17 @@ archName I386 = "i386" binDistName :: Arch -> Opsys -> BuildConfig -> String binDistName arch opsys bc = "ghc-" ++ testEnv arch opsys bc +-- | The bindist name for the stage3 (target) artifact produced by a combined +-- cross job (crossStage == Just 3). This preserves the naming convention +-- expected by downstream consumers. +binDistNameStage3 :: Arch -> Opsys -> BuildConfig -> String +binDistNameStage3 arch opsys bc = "ghc-" ++ intercalate "-" (concat + [ [ archName arch, opsysName opsys ] + , ["cross_"++triple | Just triple <- pure (crossTarget bc) ] + , ["stage3"] + , [flavourString (mkJobFlavour bc)] + ]) + -- | Test env should create a string which changes whenever the 'BuildConfig' changes. -- Either the change is reflected by modifying the flavourString or directly (as is -- the case for settings which affect environment variables) @@ -883,13 +907,16 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} } [ opsysVariables arch opsys , "TEST_ENV" =: testEnv arch opsys buildConfig , "BIN_DIST_NAME" =: binDistName arch opsys buildConfig + , if crossStage buildConfig == Just Stage3 + then "BIN_DIST_NAME_STAGE3" =: binDistNameStage3 arch opsys buildConfig + else mempty , "BUILD_FLAVOUR" =: flavourString jobFlavour , "BIGNUM_BACKEND" =: bignumString (bignumBackend buildConfig) , "CONFIGURE_ARGS" =: configureArgsStr buildConfig , "INSTALL_CONFIGURE_ARGS" =: "--enable-strict-ghc-toolchain-check" , maybe mempty ("CONFIGURE_WRAPPER" =:) (configureWrapper buildConfig) , maybe mempty ("CROSS_TARGET" =:) (crossTarget buildConfig) - , maybe mempty (("CROSS_STAGE" =:) . show) (crossStage buildConfig) + , maybe mempty (("CROSS_STAGE" =:) . show . crossStageToInt) (crossStage buildConfig) , case crossEmulator buildConfig of NoEmulator -- we need an emulator but it isn't set. Won't run the testsuite @@ -922,6 +949,11 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} } trim :: String -> String trim = dropWhileEnd isSpace . dropWhile isSpace + stage3Artifacts + | crossStage buildConfig == Just Stage3 = + [binDistNameStage3 arch opsys buildConfig ++ ".tar.xz"] + | otherwise = [] + -- Keep in sync with the exclude list in `function clean()` in -- `.gitlab/ci.sh`! jobArtifacts = Artifacts @@ -930,6 +962,7 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} } , artifactPaths = [binDistName arch opsys buildConfig ++ ".tar.xz" ,"junit.xml" ,"unexpected-test-output.tar.gz"] + ++ stage3Artifacts , artifactsWhen = ArtifactsAlways } @@ -1288,13 +1321,13 @@ alpine_aarch64 = [ cross_jobs :: [JobGroup Job] cross_jobs = [ -- x86 -> aarch64 - validateBuilds Amd64 (Linux Debian13) (crossConfig "aarch64-linux-gnu" (Emulator "qemu-aarch64 -L /usr/aarch64-linux-gnu") Nothing) + validateBuilds Amd64 (Linux Debian13) (crossConfig "aarch64-linux-gnu" (Emulator "qemu-aarch64 -L /usr/aarch64-linux-gnu") Nothing Stage2) - -- x86_64 -> riscv - , addValidateRule RiscV (validateBuilds Amd64 (Linux Debian13Riscv) (crossConfig "riscv64-linux-gnu" (Emulator "qemu-riscv64 -L /usr/riscv64-linux-gnu") Nothing)) + -- x86_64 (build) -> riscv64 (host/target) + , addValidateRule RiscV (validateBuilds Amd64 (Linux Debian13Riscv) (crossConfig "riscv64-linux-gnu" (Emulator "qemu-riscv64 -L /usr/riscv64-linux-gnu") Nothing Stage3)) -- x86_64 -> loongarch64 - , addValidateRule LoongArch64 (validateBuilds Amd64 (Linux Ubuntu2404LoongArch64) (crossConfig "loongarch64-linux-gnu" (Emulator "qemu-loongarch64 -L /usr/loongarch64-linux-gnu") Nothing)) + , addValidateRule LoongArch64 (validateBuilds Amd64 (Linux Ubuntu2404LoongArch64) (crossConfig "loongarch64-linux-gnu" (Emulator "qemu-loongarch64 -L /usr/loongarch64-linux-gnu") Nothing Stage2)) -- Javascript , addValidateRule JSBackend (validateBuilds Amd64 (Linux Debian11Js) javascriptConfig) @@ -1315,7 +1348,7 @@ cross_jobs = [ (validateBuilds AArch64 (Linux Debian12Wine) (winAarch64Config {llvmBootstrap = True})) ] where - javascriptConfig = (crossConfig "javascript-unknown-ghcjs" (NoEmulatorNeeded TimeoutIncrease) (Just "emconfigure")) + javascriptConfig = (crossConfig "javascript-unknown-ghcjs" (NoEmulatorNeeded TimeoutIncrease) (Just "emconfigure") Stage2) { bignumBackend = Native } makeWinArmJobs = modifyJobs @@ -1354,7 +1387,7 @@ cross_jobs = [ llvm_prefix = "/opt/llvm-mingw-linux/bin/aarch64-w64-mingw32-" cflags = "-fuse-ld=" ++ llvm_prefix ++ "ld --rtlib=compiler-rt" - winAarch64Config = (crossConfig "aarch64-unknown-mingw32" (Emulator "/opt/wine-arm64ec-msys2-deb12/bin/wine") Nothing) + winAarch64Config = (crossConfig "aarch64-unknown-mingw32" (Emulator "/opt/wine-arm64ec-msys2-deb12/bin/wine") Nothing Stage2) { bignumBackend = Native } make_wasm_jobs cfg = @@ -1367,7 +1400,7 @@ cross_jobs = [ $ addValidateRule WasmBackend $ validateBuilds Amd64 (Linux AlpineWasm) cfg wasm_build_config = - (crossConfig "wasm32-wasi" (NoEmulatorNeeded NoTimeoutIncrease) Nothing) + (crossConfig "wasm32-wasi" (NoEmulatorNeeded NoTimeoutIncrease) Nothing Stage2) { hostFullyStatic = True , buildFlavour = Release -- TODO: This needs to be validate but wasm backend doesn't pass yet , textWithSIMDUTF = True ===================================== .gitlab/jobs.yaml ===================================== @@ -2750,7 +2750,8 @@ "paths": [ "ghc-x86_64-linux-deb13-riscv-cross_riscv64-linux-gnu-validate.tar.xz", "junit.xml", - "unexpected-test-output.tar.gz" + "unexpected-test-output.tar.gz", + "ghc-x86_64-linux-deb13-riscv-cross_riscv64-linux-gnu-stage3-validate.tar.xz" ], "reports": { "junit": "junit.xml" @@ -2792,10 +2793,11 @@ "variables": { "BIGNUM_BACKEND": "gmp", "BIN_DIST_NAME": "ghc-x86_64-linux-deb13-riscv-cross_riscv64-linux-gnu-validate", + "BIN_DIST_NAME_STAGE3": "ghc-x86_64-linux-deb13-riscv-cross_riscv64-linux-gnu-stage3-validate", "BUILD_FLAVOUR": "validate", "CONFIGURE_ARGS": "--with-intree-gmp --enable-strict-ghc-toolchain-check", "CROSS_EMULATOR": "qemu-riscv64 -L /usr/riscv64-linux-gnu", - "CROSS_STAGE": "2", + "CROSS_STAGE": "3", "CROSS_TARGET": "riscv64-linux-gnu", "INSTALL_CONFIGURE_ARGS": "--enable-strict-ghc-toolchain-check", "RUNTEST_ARGS": "-e config.timeout=900", @@ -6726,7 +6728,8 @@ "paths": [ "ghc-x86_64-linux-deb13-riscv-cross_riscv64-linux-gnu-validate.tar.xz", "junit.xml", - "unexpected-test-output.tar.gz" + "unexpected-test-output.tar.gz", + "ghc-x86_64-linux-deb13-riscv-cross_riscv64-linux-gnu-stage3-validate.tar.xz" ], "reports": { "junit": "junit.xml" @@ -6768,10 +6771,11 @@ "variables": { "BIGNUM_BACKEND": "gmp", "BIN_DIST_NAME": "ghc-x86_64-linux-deb13-riscv-cross_riscv64-linux-gnu-validate", + "BIN_DIST_NAME_STAGE3": "ghc-x86_64-linux-deb13-riscv-cross_riscv64-linux-gnu-stage3-validate", "BUILD_FLAVOUR": "validate", "CONFIGURE_ARGS": "--with-intree-gmp --enable-strict-ghc-toolchain-check", "CROSS_EMULATOR": "qemu-riscv64 -L /usr/riscv64-linux-gnu", - "CROSS_STAGE": "2", + "CROSS_STAGE": "3", "CROSS_TARGET": "riscv64-linux-gnu", "INSTALL_CONFIGURE_ARGS": "--enable-strict-ghc-toolchain-check", "RUNTEST_ARGS": "-e config.timeout=900", ===================================== distrib/configure.ac.in ===================================== @@ -9,7 +9,7 @@ AC_INIT([The Glorious Glasgow Haskell Compilation System], [@ProjectVersion@], [ dnl See /configure.ac for rationale. AC_PREREQ([2.69]) -AC_CONFIG_MACRO_DIRS([../m4]) +AC_CONFIG_MACRO_DIRS([m4]) dnl-------------------------------------------------------------------- dnl * Deal with arguments telling us gmp is somewhere odd ===================================== hadrian/src/BindistConfig.hs ===================================== @@ -24,6 +24,10 @@ crossBindist = BindistConfig { library_stage = Stage2, executable_stage = Stage1 targetBindist :: BindistConfig targetBindist = BindistConfig { library_stage = Stage2, executable_stage = Stage2 } +-- | Folder relative to build root ("bindist" or "bindist-stage3") +bindistFolder :: BindistConfig -> FilePath +bindistFolder conf | executable_stage conf == Stage2 = "bindist-stage3" +bindistFolder _conf = "bindist" -- | The implicit bindist config, if we don't know any better. implicitBindistConfig :: Action BindistConfig ===================================== hadrian/src/Rules/BinaryDist.hs ===================================== @@ -6,7 +6,6 @@ import Context import Data.Either import qualified Data.Set as Set import Expression -import Hadrian.Oracles.Path (fixUnixPathsOnWindows) import Oracles.Flavour import Oracles.Setting import Packages @@ -14,66 +13,89 @@ import Rules.Generate (generateSettings) import Settings import qualified System.Directory.Extra as IO import Settings.Program (programContext) -import Target -import Utilities import BindistConfig {- Note [Binary distributions] ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Hadrian produces binary distributions under: +Hadrian produces binary distributions that run on the build host architecture +(build == host, target == host || target /= host) under: <build root>/bindist/ghc-<X>.<Y>.<Z>-<arch>-<os>.tar.xz -It is generated by creating an archive from: - <build root>/bindist/ghc-<X>.<Y>.<Z>-<arch>-<os>/ +For stage3 (build /= host, host == target) bindists, a separate parent folder +is used: + <build root>/bindist-stage3/ghc-<X>.<Y>.<Z>-<arch>-<os>.tar.xz -It does so by following the steps below. +While regular bindists are usual same-arch compilers or cross-compilers, stage3 +bindists are cross-compiled compilers. -- make sure we have a complete stage 2 compiler + haddock +Bindists are generated by creating an archive from: + <build root>/<bindist|bindist-stage3>/ghc-<X>.<Y>.<Z>-<arch>-<os>/ + +Stage2 cross-compilers are a by-product of creating stage3 cross-compiled +compilers. The additional build dir (bindist-stage3) lets us keep both, such +that we can build them in one go on CI. +Configuration files (e.g. configure script and default.host.target) differ in +this case and keeping both targets separated also saves us some headache +dealing with stale files. + +This table introduces variables to simplify the following step descriptions: + +| compiler kind | <bindist-dir> | <executable-stage-dir> | <library-stage-dir> | +|------------------|----------------|------------------------|---------------------| +| native | bindist | stage1/ | stage1/ | +| cross compiler | bindist | stage1/ | stage2/ | +| cross-compiled | bindist-stage3 | stage2/ | stage2/ | + +These are the steps to build a bindist: + +- make sure we have a complete compiler + libraries + haddock for the stage(s) + to bundle - copy the specific binaries which should be in the bindist to the bin folder and add the version suffix: - <build root>/stage1/bin/xxxx + <build root>/<executable-stage-dir>/bin/xxxx to - <build root/bindist/ghc-<X>.<Y>.<Z>-<arch>-<os>/bin/xxxx-<VER> + <build root>/<bindist-dir>/ghc-<X>.<Y>.<Z>-<arch>-<os>/bin/[<target>-]xxxx-<VER> + where the optional <target>- prefix is the cross triple for cross-compilers. - create symlink (or bash) wrapper from unversioned to versioned executable: - <build root/bindist/ghc-<X>.<Y>.<Z>-<arch>-<os>/bin/xxxx + <build root>/<bindist-dir>/ghc-<X>.<Y>.<Z>-<arch>-<os>/bin/[<target>-]xxxx points to: - <build root/bindist/ghc-<X>.<Y>.<Z>-<arch>-<os>/bin/xxxx-<VER> + <build root>/<bindist-dir>/ghc-<X>.<Y>.<Z>-<arch>-<os>/bin/[<target>-]xxxx-<VER> - copy the lib directories of the compiler we built: - <build root>/stage1/lib + <build root>/<library-stage-dir>/lib to - <build root>/bindist/ghc-<X>.<Y>.<Z>-<arch>-<os>/lib + <build root>/<bindist-dir>/ghc-<X>.<Y>.<Z>-<arch>-<os>/lib - copy the generated docs (user guide, haddocks, etc): - <build root>/docs/ + <build root>/doc/ to - <build root>/bindist/ghc-<X>.<Y>.<Z>-<arch>-<os>/docs/ + <build root>/<bindist-dir>/ghc-<X>.<Y>.<Z>-<arch>-<os>/doc/ -- use autoreconf to generate a `configure` script from - aclocal.m4 and distrib/configure.ac, that we move to: - <build root>/bindist/ghc-<X>.<Y>.<Z>-<arch>-<os>/configure +- use autoreconf to generate a staged `configure` script in + <build root>/<executable-stage>/distrib + that we move to: + <build root>/<bindist-dir>/ghc-<X>.<Y>.<Z>-<arch>-<os>/configure - write a (fixed) Makefile capable of supporting 'make install' to: - <build root>/bindist/ghc-<X>.<Y>.<Z>-<arch>-<os>/Makefile + <build root>/<bindist-dir>/ghc-<X>.<Y>.<Z>-<arch>-<os>/Makefile - write some (fixed) supporting bash code for the wrapper scripts to: - <build root>/bindist/ghc-<X>.<Y>.<Z>-<arch>-<os>/wrappers/<program> + <build root>/<bindist-dir>/ghc-<X>.<Y>.<Z>-<arch>-<os>/wrappers/<program> where <program> is the name of the executable that the bash file will help wrapping. -- copy supporting configure/make related files - (see @bindistInstallFiles@) to: - <build root>/bindist/ghc-<X>.<Y>.<Z>-<arch>-<os>/<file> +- copy supporting configure/make related files (see @bindistInstallFiles@) to: + <build root>/<bindist-dir>/ghc-<X>.<Y>.<Z>-<arch>-<os>/<file> - create a .tar.xz archive of the directory: - <build root>/bindist/ghc-<X>.<Y>.<Z>-<arch>-<os>/ + <build root>/<bindist-dir>/ghc-<X>.<Y>.<Z>-<arch>-<os>/ at - <build root>/bindist/ghc-<X>.<Y>.<Z>-<arch>-<os>.tar.xz + <build root>/<bindist-dir>/ghc-<X>.<Y>.<Z>-<arch>-<os>.tar.xz Note [Wrapper scripts and binary distributions] @@ -160,12 +182,12 @@ buildBinDistDir root conf@BindistConfig{..} = do distDir <- Context.distDir (vanillaContext library_stage rts) let ghcBuildDir = root -/- stageString library_stage - bindistFilesDir = root -/- "bindist" -/- ghcVersionPretty + bindistFilesDir = root -/- bindistFolder conf -/- ghcVersionPretty ghcVersionPretty = "ghc-" ++ version ++ "-" ++ targetPlatform rtsIncludeDir = distDir -/- "include" - -- We create the bindist directory at <root>/bindist/ghc-X.Y.Z-platform/ - -- and populate it with Stage2 build results + -- We create the bindist directory at <root>/<bindistFolder>/ghc-X.Y.Z-platform/ + -- and populate it with build results createDirectory bindistFilesDir createDirectory (bindistFilesDir -/- "bin") createDirectory (bindistFilesDir -/- "lib") @@ -231,9 +253,9 @@ buildBinDistDir root conf@BindistConfig{..} = do -- relocatable. The package DB is always at "package.conf.d" relative to -- the lib dir, matching the known bindist layout. let bindistSettings = bindistFilesDir -/- "lib" -/- "settings" - bindistContext = vanillaContext library_stage compiler + bindistContext = vanillaContext executable_stage compiler bindistSettingsContent <- interpretInContext bindistContext $ - generateSettings bindistSettings False "package.conf.d" + generateSettings bindistSettings False "package.conf.d" library_stage writeFile' bindistSettings bindistSettingsContent copyDirectory rtsIncludeDir bindistFilesDir @@ -245,10 +267,14 @@ buildBinDistDir root conf@BindistConfig{..} = do -- -- N.B. the ghc-pkg executable may be prefixed with a target triple -- (c.f. #20267). - - -- Not going to work for cross - ghcPkgName <- programName (vanillaContext Stage1 ghcPkg) - cmd_ (bindistFilesDir -/- "bin" -/- ghcPkgName) ["recache", "--package-db", bindistFilesDir -/- "lib" -/- "package.conf.d" ] + -- Recache using the stage1 ghc-pkg executable. This is the unprefixed + -- host ghc-pkg for native bindists and the target-triple-prefixed cross + -- ghc-pkg for cross bindists; both run on the build host and can handle + -- the target package database. The stage3 bindist's package DB is also + -- built by the stage1 cross compiler, so stage1 ghc-pkg is correct there + -- too. + ghcPkgPath <- programPath =<< programContext Stage1 ghcPkg + cmd_ ghcPkgPath ["recache", "--package-db", bindistFilesDir -/- "lib" -/- "package.conf.d" ] need ["docs"] @@ -344,8 +370,7 @@ bindistRules = do buildBinDistDir root cfg phony "binary-dist-dir-cross" $ buildBinDistDir root crossBindist - -- MP: Not working yet - -- phony "binary-dist-dir-stage3" $ buildBinDistDir root targetBindist + phony "binary-dist-dir-stage3" $ buildBinDistDir root targetBindist let buildBinDist compressor = do win_host <- isWinHost @@ -377,54 +402,32 @@ bindistRules = do phony (name <> "-dist-xz") $ mk_bindist Xz phony "binary-dist-cross" $ buildBinDistX "binary-dist-dir-cross" "bindist" Xz - phony "binary-dist-stage3" $ buildBinDistX "binary-dist-dir-stage3" "bindist" Xz - - -- Prepare binary distribution configure script - -- (generated under <ghc root>/distrib/configure by 'autoreconf') - root -/- "bindist" -/- "ghc-*" -/- "configure" %> \configurePath -> do - need ["distrib" -/- "configure.ac"] - ghcRoot <- topDirectory - copyFile (ghcRoot -/- "aclocal.m4") (ghcRoot -/- "distrib" -/- "aclocal.m4") - copyDirectory (ghcRoot -/- "m4") (ghcRoot -/- "distrib") - - -- Note [Autoreconf unix paths from ACLOCAL_PATH] - -- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- On Windows, autoreconf fails when the ACLOCAL_PATH env variable contains Windows- - -- style paths. This happens because MSYS2 automatically converts env variables to - -- Windows-style paths. To fix this, we convert ACLOCAL_PATH back to Unix style. - -- This is done both in the boot Python script and here when building a bindist. - win_host <- isWinHost - env <- if not win_host - then pure [] - else do - aclocalPathMay <- getEnv "ACLOCAL_PATH" - case aclocalPathMay of - Nothing -> pure [] - Just aclocalPath -> do - unixAclocalPath <- fixUnixPathsOnWindows aclocalPath - pure [AddEnv "ACLOCAL_PATH" unixAclocalPath] - - buildWithCmdOptions env $ - target (vanillaContext Stage1 ghc) (Autoreconf $ ghcRoot -/- "distrib") [] [] - -- We clean after ourselves, moving the configure script we generated in - -- our bindist dir - removeFile (ghcRoot -/- "distrib" -/- "aclocal.m4") - removeDirectory (ghcRoot -/- "distrib" -/- "m4") - - moveFile (ghcRoot -/- "distrib" -/- "configure") configurePath - - -- Generate the Makefile that enables the "make install" part - root -/- "bindist" -/- "ghc-*" -/- "Makefile" %> \makefilePath -> do - top <- topDirectory - copyFile (top -/- "hadrian" -/- "bindist" -/- "Makefile") makefilePath - - -- Copy various configure-related files needed for a working - -- './configure [...] && make install' workflow - -- (see the list of files needed in the 'binary-dist' rule above, before - -- creating the archive). - forM_ bindistInstallFiles $ \file -> - root -/- "bindist" -/- "ghc-*" -/- file %> \dest -> do - copyFile (fixup file) dest + phony "binary-dist-stage3" $ buildBinDistX "binary-dist-dir-stage3" "bindist-stage3" Xz + + forM_ [normalBindist, targetBindist] $ \bindistCfg -> do + let bindistFolderName = bindistFolder bindistCfg + stg = executable_stage bindistCfg + -- Copy the per-stage 'configure' (produced by autoreconf in Generate.hs) + -- from the build distrib dir into the bindist. Generating it there keeps + -- the autoreconf inputs (configure.ac, aclocal.m4, m4/*.m4) next to the + -- configure script and lets Shake track their changes correctly. + root -/- bindistFolderName -/- "ghc-*" -/- "configure" %> \configurePath -> do + let distribConfigure = root -/- stageString stg -/- "distrib" -/- "configure" + need [distribConfigure] + copyFile distribConfigure configurePath + + -- Generate the Makefile that enables the "make install" part + root -/- bindistFolderName -/- "ghc-*" -/- "Makefile" %> \makefilePath -> do + top <- topDirectory + copyFile (top -/- "hadrian" -/- "bindist" -/- "Makefile") makefilePath + + -- Copy various configure-related files needed for a working + -- './configure [...] && make install' workflow + -- (see the list of files needed in the 'binary-dist' rule above, before + -- creating the archive). + forM_ bindistInstallFiles $ \file -> + root -/- bindistFolderName -/- "ghc-*" -/- file %> \dest -> do + copyFile (fixup file) dest where fixup f | f `elem` ["INSTALL", "README"] = "distrib" -/- f @@ -504,7 +507,7 @@ pkgToWrappers stage pkg = do -- These are the packages which we want to expose to the user and hence -- there are wrappers installed in the bindist. | pkg `elem` [hpcBin, haddock, hp2ps, hsc2hs, ghc, ghcPkg] - -> (:[]) <$> (programName =<< programContext Stage1 pkg) + -> (:[]) <$> (programName =<< programContext stage pkg) | otherwise -> pure [] wrapper :: Stage -> FilePath -> Action String @@ -557,7 +560,11 @@ ghciScriptWrapper stage = do -- | Create a wrapper script calls the executable given as first argument createVersionWrapper :: Stage -> Package -> String -> FilePath -> Action () createVersionWrapper executable_stage pkg versioned_exe install_path = do - ghcPath <- builderPath (Ghc CompileCWithGhc (succStage executable_stage)) + -- The wrapper compiler must run on the build host. For the stage3 target + -- bindist executable_stage is Stage2, whose successor would be the + -- target-native Stage3 compiler; cap the wrapper stage at Stage2. + let wrapperGhcStage = min Stage2 (succStage executable_stage) + ghcPath <- builderPath (Ghc CompileCWithGhc wrapperGhcStage) top <- topDirectory let version_wrapper_dir = top -/- "hadrian" -/- "bindist" -/- "cwrappers" wrapper_files = [ version_wrapper_dir -/- file | file <- ["version-wrapper.c", "getLocation.c", "cwrapper.c"]] ===================================== hadrian/src/Rules/Generate.hs ===================================== @@ -9,6 +9,7 @@ import qualified Data.Set as Set import Base import qualified Context import Expression +import Hadrian.Oracles.Path (fixUnixPathsOnWindows) import Hadrian.Oracles.TextFile (lookupStageBuildConfig) import Oracles.Flag hiding (arSupportsAtFile, arSupportsDashL) import Oracles.ModuleFiles @@ -251,32 +252,51 @@ generateRules = do (root -/- "ghc-stage2") <~+ ghcWrapper Stage2 (root -/- "ghc-stage3") <~+ ghcWrapper Stage3 - forM_ allStages $ \stage -> do - let prefix = root -/- stageString stage -/- "lib" - -- For the finalStage, we generate settings for that stage. For - -- others we look at the next stage. Why? Because cross-compilers - -- require libs from the successor stage, otherwise they are - -- compiled for the host and not the target. - stage' = if stage /= finalStage then succStage stage else stage - go gen file = generate file (semiEmptyTarget stage') gen + forM_ allStages $ \buildStage -> do + let -- Two stages are in play per rule iteration: + -- + -- * @buildStage@ — loop variable; the settings file is written + -- into @_build/<buildStage>/lib/settings@ and + -- describes the compiler at @compilerStage@. + -- * @compilerStage@ — the stage whose @bin/@ holds the compiler + -- the settings file describes; also the + -- ambient 'Expr' stage passed to + -- 'generateSettings' (via 'semiEmptyTarget'), + -- so it is the value of @executableStage@ + -- inside that function. + -- + -- For a cross-compiler the libs it links against live in the + -- /successor/ stage's lib dir; @libraryStage@ (computed in the + -- rule body below) is that successor. @compilerStage@ normally + -- equals @buildStage@, but at @finalStage@ there is no successor + -- to hold its libs, so @compilerStage@ drops to the predecessor + -- (the final stage's lib dir merely hosts the predecessor + -- cross-compiler's target-arch libs). + compilerStage = if buildStage == finalStage + then predStage buildStage + else buildStage + prefix = root -/- stageString buildStage -/- "lib" + go gen file = generate file (semiEmptyTarget compilerStage) gen (prefix -/- "settings") %> \out -> do - let get_pkg_db stg = packageDbPath (PackageDbLoc stg Final) - -- For cross, LibDir points to stage' lib dir, so pkgDb must also - -- be relative to stage' lib dir. - isCross <- crossStage stage - let libStage = case stage of + -- Stage0 has no library or package DB of its own (the + -- bootstrapping compiler uses Stage1's); for any other stage the + -- package DB lives where the LibDir redirect points (this stage's + -- own lib dir, or the successor's when @buildStage@ is a cross + -- stage). + isCross <- crossStage buildStage + let libraryStage = case buildStage of Stage0 {} -> Stage1 - _ -> if isCross then stage' else stage - pkgDb <- get_pkg_db libStage + _ -> if isCross then succStage buildStage else buildStage + pkgDb <- packageDbPath (PackageDbLoc libraryStage Final) -- addTrailingPathSeparator needed: makeRelativeNoSysLink uses -- splitPath where "lib" and "lib/" are distinct components. let libTopDir = addTrailingPathSeparator $ - if isCross - then root -/- stageString stage' -/- "lib" - else prefix + if isStage0 buildStage + then prefix + else root -/- stageString libraryStage -/- "lib" relPkgDb = makeRelativeNoSysLink libTopDir pkgDb - go (generateSettings out True relPkgDb) out - (prefix -/- "targets" -/- "default.target") %> \out -> go (show <$> expr (targetStage (succStage stage))) out + go (generateSettings out True relPkgDb libraryStage) out + (prefix -/- "targets" -/- "default.target") %> \out -> go (show <$> expr (targetStage (succStage buildStage))) out where file <~+ gen = file %> \out -> generate out emptyTarget gen >> makeExecutable out @@ -410,6 +430,7 @@ templateRules = do bindistRules :: Rules () bindistRules = do + root <- buildRootRules templateRule ("mk" -/- "project.mk") $ mconcat [ interpolateSetting "ProjectName" ProjectName , interpolateSetting "ProjectVersion" ProjectVersion @@ -421,43 +442,119 @@ bindistRules = do , interpolateVar "HostOS_CPP" $ fmap cppify $ interp $ queryHost queryOS - , interpolateVar "TargetPlatform" $ getTarget targetPlatformTriple - , interpolateVar "TargetPlatform_CPP" $ cppify <$> getTarget targetPlatformTriple - , interpolateVar "TargetArch_CPP" $ cppify <$> getTarget queryArch - , interpolateVar "TargetOS_CPP" $ cppify <$> getTarget queryOS - , interpolateVar "LLVMTarget" $ getTarget tgtLlvmTarget + -- Stage2 always targets the final architecture. Thus, we can use a + -- constant stage here. + , interpolateVar "TargetPlatform" $ getTarget Stage2 targetPlatformTriple + , interpolateVar "TargetPlatform_CPP" $ cppify <$> getTarget Stage2 targetPlatformTriple + , interpolateVar "TargetArch_CPP" $ cppify <$> getTarget Stage2 queryArch + , interpolateVar "TargetOS_CPP" $ cppify <$> getTarget Stage2 queryOS + , interpolateVar "LLVMTarget" $ getTarget Stage2 tgtLlvmTarget ] - templateRule ("distrib" -/- "configure.ac") $ mconcat + forM_ [Stage1, Stage2] $ \stage -> + let crossStageInterps = Interpolations $ do + isCrossStage <- crossStage stage + targetPlatform <- setting TargetPlatformFull + -- For cross-compiled compilers we need to pretend that they were + -- build on the target. For regular commpilers we can assume that: + -- build == host == target + buildPlatform <- + if isCrossStage + then + interp $ queryBuild targetPlatformTriple + else getTarget stage targetPlatformTriple + hostPlatform <- + if isCrossStage + then + interp $ queryHost targetPlatformTriple + else getTarget stage targetPlatformTriple + baseUnitId <- pkgUnitId (if isCrossStage then succStage stage else stage) base + buildPlatformFull <- if isCrossStage then setting BuildPlatformFull else setting TargetPlatformFull + hostPlatformFull <- if isCrossStage then setting HostPlatformFull else setting TargetPlatformFull + pure + [ ("CrossCompilePrefix", if isCrossStage then targetPlatform <> "-" else "") + , ("TargetPlatformFull", targetPlatform) + , ("BuildPlatform", buildPlatform) + , ("HostPlatform", hostPlatform) + , ("BaseUnitId", baseUnitId) + , ("BuildPlatformFull", buildPlatformFull) + , ("HostPlatformFull", hostPlatformFull) + ] + in templateRuleFrom + ("distrib" -/- "configure.ac" <.> "in") + (root -/- stageString stage -/- "distrib" -/- "configure.ac") + $ mconcat [ interpolateSetting "ConfiguredEmsdkVersion" EmsdkVersion - , interpolateVar "CrossCompilePrefix" $ do - crossCompiling <- interp $ getFlag CrossCompiling - tpf <- setting TargetPlatformFull - pure $ if crossCompiling then tpf <> "-" else "" - , interpolateVar "LeadingUnderscore" $ yesNo <$> getTarget tgtSymbolsHaveLeadingUnderscore + , interpolateVar "LeadingUnderscore" $ yesNo <$> getTarget stage tgtSymbolsHaveLeadingUnderscore , interpolateSetting "LlvmMaxVersion" LlvmMaxVersion , interpolateSetting "LlvmMinVersion" LlvmMinVersion - , interpolateVar "LlvmTarget" $ getTarget tgtLlvmTarget + , interpolateVar "LlvmTarget" $ getTarget stage tgtLlvmTarget , interpolateSetting "ProjectVersion" ProjectVersion , interpolateVar "EnableDistroToolchain" $ interp (staged (lookupStageBuildConfig "settings-use-distro-mingw")) - , interpolateVar "TablesNextToCode" $ yesNo <$> getTarget tgtTablesNextToCode - , interpolateVar "TargetHasLibm" $ yesNo <$> getTarget tgtHasLibm - , interpolateVar "TargetPlatform" $ getTarget targetPlatformTriple - , interpolateVar "BuildPlatform" $ interp $ queryBuild targetPlatformTriple - , interpolateVar "HostPlatform" $ interp $ queryHost targetPlatformTriple - , interpolateVar "TargetWordBigEndian" $ getTarget isBigEndian - , interpolateVar "TargetWordSize" $ getTarget wordSize - , interpolateVar "Unregisterised" $ yesNo <$> getTarget tgtUnregisterised + , interpolateVar "TablesNextToCode" $ yesNo <$> getTarget stage tgtTablesNextToCode + , interpolateVar "TargetHasLibm" $ yesNo <$> getTarget stage tgtHasLibm + , interpolateVar "TargetPlatform" $ getTarget stage targetPlatformTriple + , interpolateVar "TargetWordBigEndian" $ getTarget stage isBigEndian + , interpolateVar "TargetWordSize" $ getTarget stage wordSize + , interpolateVar "Unregisterised" $ yesNo <$> getTarget stage tgtUnregisterised , interpolateVar "UseLibdw" $ fmap yesNo $ interp $ staged (fmap (isJust . tgtRTSWithLibdw) . targetStage) - , interpolateVar "UseLibffiForAdjustors" $ yesNo <$> getTarget tgtUseLibffiForAdjustors - , interpolateVar "BaseUnitId" $ pkgUnitId Stage1 base - , interpolateVar "GhcWithSMP" $ yesNo <$> targetSupportsSMP Stage2 - , interpolateVar "TargetPlatformFull" (setting TargetPlatformFull) - , interpolateVar "BuildPlatformFull" (setting BuildPlatformFull) - , interpolateVar "HostPlatformFull" (setting HostPlatformFull) + , interpolateVar "UseLibffiForAdjustors" $ yesNo <$> getTarget stage tgtUseLibffiForAdjustors + , interpolateVar "GhcWithSMP" $ yesNo <$> targetSupportsSMP stage + , crossStageInterps ] + + -- We can build two kinds of bindists: Regular Stage2 (including + -- cross-compilers) and fully cross-compiled Stage3. To avoid + -- race-conditions, stale files, etc. build the `configure` scripts as part + -- of the stage's _build files. This requires copying several files such that + -- they are available to the autoconf run. + forM_ [Stage1, Stage2] $ \stage -> do + let distribDir = root -/- stageString stage -/- "distrib" + + distribDir -/- "aclocal.m4" %> \out -> do + top <- topDirectory + copyFile (top -/- "aclocal.m4") out + + forM_ ["config.sub", "config.guess", "install-sh"] $ \f -> + distribDir -/- f %> \out -> do + top <- topDirectory + copyFile (top -/- f) out + + distribDir -/- "m4/*.m4" %> \out -> do + top <- topDirectory + copyFile (top -/- "m4" -/- takeFileName out) out + + distribDir -/- "configure" %> \_ -> do + top <- topDirectory + m4Files <- getDirectoryFiles (top -/- "m4") ["*.m4"] + need $ [ distribDir -/- "configure.ac" + , distribDir -/- "config.sub" + , distribDir -/- "config.guess" + , distribDir -/- "install-sh" + , distribDir -/- "aclocal.m4" + ] + ++ [ distribDir -/- "m4" -/- takeFileName f | f <- m4Files ] + + -- Note [Autoreconf unix paths from ACLOCAL_PATH] + -- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + -- On Windows, autoreconf fails when the ACLOCAL_PATH env variable + -- contains Windows-style paths. MSYS2 auto-converts env vars to + -- Windows-style, so we convert ACLOCAL_PATH back to Unix style here. + win_host <- isWinHost + env <- if not win_host + then pure [] + else do + aclocalPathMay <- getEnv "ACLOCAL_PATH" + case aclocalPathMay of + Nothing -> pure [] + Just aclocalPath -> do + unixAclocalPath <- fixUnixPathsOnWindows aclocalPath + pure [AddEnv "ACLOCAL_PATH" unixAclocalPath] + + buildWithCmdOptions env $ + target (vanillaContext stage ghc) (Autoreconf distribDir) [] [] where interp = interpretInContext (semiEmptyTarget Stage2) - getTarget = interp . queryTarget Stage2 + getTarget stage = interp . queryTarget stage -- | Given a 'String' replace characters '.' and '-' by underscores ('_') so that -- the resulting 'String' is a valid C preprocessor identifier. @@ -479,42 +576,40 @@ ghcWrapper stage = do -- | Generate settings file, optionally including @LibDir@. -- +-- Describes the compiler whose stage is the ambient 'Expr' context +-- (available here as @executableStage@ via 'getStage'). The @libraryStage@ +-- argument is the stage whose lib dir holds the libraries the described +-- compiler links against — used both for the @base@ unit-id lookup and for +-- the @LibDir@ entry. It usually equals @executableStage@ but differs when +-- the compiler links against libraries from a different stage (cross +-- compilers, or the Stage0 bootstrap compiler using Stage1's libraries). +-- -- @rel_pkg_db@: package DB path relative to the lib dir (e.g. -- "package.conf.d"). Callers supply the correct relative path. For bindists --- the layout is known statically; for in-tree builds callers compute it. For --- bindists, we omit @LibDir@ so it defaults to @topDir@ at runtime. -generateSettings :: FilePath -> Bool -> FilePath -> Expr String -generateSettings settingsFile includeLibDir rel_pkg_db = do +-- the layout is known statically; for in-tree builds callers compute it. +-- For bindists, we omit @LibDir@ so it defaults to @topDir@ at runtime. +generateSettings :: FilePath -> Bool -> FilePath -> Stage -> Expr String +generateSettings settingsFile includeLibDir rel_pkg_db libraryStage = do ctx <- getContext - stage <- getStage + executableStage <- getStage + + base_unit_id <- expr $ pkgUnitId libraryStage base - -- The unit-id of the base package which is always linked against (#25382) - base_unit_id <- expr $ do - case stage of - Stage0 {} -> error "Unable to generate settings for stage0" - Stage1 -> pkgUnitId Stage1 base - Stage2 -> pkgUnitId Stage1 base - Stage3 -> pkgUnitId Stage2 base - - -- For cross compilers, LibDir points to the succeeding stage's lib dir - -- (which contains the target architecture's libraries). For non-cross, - -- it points to the preceding stage's lib dir as usual. - let compilerStage = predStage stage -- the GHC that builds packages in this stage - isCrossLibDir <- expr $ crossStage compilerStage - let stage_dir_stage = if isCrossLibDir then stage else compilerStage - - -- addTrailingPathSeparator is needed because makeRelativeNoSysLink uses - -- splitPath internally, where "lib" and "lib/" are distinct components. - lib_topDir :: FilePath <- expr $ addTrailingPathSeparator <$> stageLibPath stage_dir_stage + lib_topDir :: FilePath <- expr $ addTrailingPathSeparator <$> stageLibPath libraryStage let rel_lib_topDir = makeRelativeNoSysLink (dropFileName settingsFile) lib_topDir settings <- traverse sequence $ - [ ("unlit command", ("$topdir/../bin/" <>) <$> expr (programName (ctx { Context.package = unlit, Context.stage = compilerStage }))) - , ("Use interpreter", expr $ yesNo <$> ghcWithInterpreter compilerStage) - -- Hard-coded as Cabal queries these to determine way support and we - -- need to always advertise all ways when bootstrapping. - -- The settings file is generated at install time when installing a bindist. - , ("RTS ways", unwords . map show . Set.toList <$> getRtsWays) + [ ("unlit command", ("$topdir/../bin/" <>) <$> expr (programName (ctx { Context.package = unlit }))) + , ("Use interpreter", expr $ yesNo <$> ghcWithInterpreter executableStage) + -- Advertise the RTS ways that will actually ship with the compiler + -- described by this settings file, i.e. the ways the @libraryStage@ + -- RTS is built with. Cabal queries this to decide which library ways + -- the compiler supports (see + -- 'Distribution.Simple.Compiler.waySupported'); under-advertising + -- causes Cabal to silently drop flags like + -- @--enable-profiling-shared@. + -- The settings file is regenerated at install time when installing a bindist. + , ("RTS ways", unwords . map show . Set.toList <$> expr (interpretInContext (vanillaContext libraryStage rts) getRtsWays)) , ("Relative Global Package DB", pure rel_pkg_db) , ("base unit-id", pure base_unit_id) ] View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/13e154cb58efb78037dbac37df3a53d... -- View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/13e154cb58efb78037dbac37df3a53d... You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications: https://gitlab.haskell.org/-/profile/notifications | Help: https://gitlab.haskell.org/help