Magnus pushed to branch wip/mangoiv/ci-stages at Glasgow Haskell Compiler / GHC Commits: d574c36c by mangoiv at 2026-10-15T17:05:01+02:00 ci: build and test stage Split up CI in a build and test stage: 1. in the build stage, just produce a bindist as well as running the stage1 tests 2. in the test stage, install the bindist, then run the testsuite with out of tree args - - - - - d7c8fa8d by mangoiv at 2026-10-15T17:05:01+02:00 ci: reorder packaging and testing stages - - - - - 3e132fcd by mangoiv at 2026-10-15T17:05:02+02:00 ci: out of tree compiler does not imply no hadrian being available - - - - - 3bb81c56 by mangoiv at 2026-10-15T17:05:02+02:00 ci: don't do msys deps check on stage1 compiler - - - - - e92e2b6b by mangoiv at 2026-10-15T17:05:02+02:00 testsuite: allow metric decreases on windows Metric Decrease: LinkableUsage01 MultiLayerModulesTH_OneShot T10547 T12234 T12425 T13035 T13253-spj T13820 T18140 T18730 T18923 T9198 TcPlugin_RewritePerf mhu-perf - - - - - f8f4c0c2 by mangoiv at 2026-10-15T17:05:55+02:00 ci: stop using ellipses in testsuite output and output env before invoking hadrian - - - - - 23 changed files: - .gitlab-ci.yml - .gitlab/ci.sh - .gitlab/common.sh - .gitlab/generate-ci/gen_ci.hs - .gitlab/jobs.yaml - hadrian/doc/cross-compile.md - hadrian/src/Base.hs - hadrian/src/Builder.hs - hadrian/src/Context.hs - hadrian/src/Flavour.hs - hadrian/src/Hadrian/Haskell/Hash.hs - hadrian/src/Hadrian/Package.hs - hadrian/src/Oracles/Flag.hs - hadrian/src/Oracles/Setting.hs - hadrian/src/Oracles/TestSettings.hs - hadrian/src/Packages.hs - hadrian/src/Rules/Generate.hs - hadrian/src/Rules/Program.hs - hadrian/src/Rules/Test.hs - hadrian/src/Settings/Builders/DeriveConstants.hs - hadrian/src/Settings/Builders/RunTest.hs - hadrian/src/Settings/Packages.hs - testsuite/driver/testlib.py Changes: ===================================== .gitlab-ci.yml ===================================== @@ -68,8 +68,8 @@ stages: - tool-lint # Source linting of the tools - quick-build # A very quick smoke-test to weed out broken commits - full-build # Build all the things - - packaging # Source distribution, etc. - testing # head.hackage correctness and compiler performance testing + - packaging # Source distribution, etc. - deploy # push documentation # Note [The CI Story] @@ -445,10 +445,7 @@ lint-submods-branch: hadrian-ghc-in-ghci: stage: quick-build extends: .lint-ghc-toolchain - needs: - - job: lint-linters - - job: lint-submods - optional: true + needs: [] variables: GHC_FLAGS: -Werror -Wwarn=unused-imports script: @@ -546,9 +543,18 @@ stack-hadrian-build: # Testing reinstallable ghc codepath #################################### +# cabal-reinstall artifacts aren't in the +# bindists so it needs to run its testsuite +# in-tree. test-cabal-reinstall-x86_64-linux-deb13: extends: nightly-x86_64-linux-deb13-validate stage: full-build + script: + - sudo chown ghc:ghc -R . + - .gitlab/ci.sh setup + - .gitlab/ci.sh configure + - .gitlab/ci.sh build_hadrian + - .gitlab/ci.sh test_hadrian variables: REINSTALL_GHC: "yes" BUILD_FLAVOUR: validate ===================================== .gitlab/ci.sh ===================================== @@ -614,12 +614,12 @@ function make_install_destdir() { fi info "merging file tree from $destdir to $instdir" cp -a "$destdir/$instdir"/* "$instdir"/ - "$instdir"/bin/${cross_prefix}ghc-pkg recache + "$instdir/bin/${cross_prefix}ghc-pkg" recache } # install the binary distribution in directory $1 to $2. function install_bindist() { - start_section install-bindist "Install bindist" + start_section install-bindist "Install bindist ($1 -> $2)" case "${CONFIGURE_WRAPPER:-}" in emconfigure) source "$EMSDK/emsdk_env.sh" ;; *) ;; @@ -631,7 +631,7 @@ function install_bindist() { case "$(uname)" in MSYS_*|MINGW*) mkdir -p "$instdir" - cp -a * "$instdir" + cp -a ./* "$instdir" ;; *) read -r -a args <<< "${INSTALL_CONFIGURE_ARGS:-}" @@ -648,39 +648,75 @@ function install_bindist() { end_section install-bindist } +function find_bindist_tar() { + local tar_files + tar_files=( *ghc*.tar.xz ) + if [[ ${#tar_files[@]} -gt 0 ]]; then + bindist_tar="${tar_files[0]}" + else + unset bindist_tar + fi +} + +# stage 1 tests -- these do not run as part +# of the test stage but already as part of the +# build stage as they don't require a bindist +function test_hadrian_stage1() { + # Ensure that statically-linked builds are actually static + if [[ "${BUILD_FLAVOUR}" = *static* ]]; then + bad_execs="" + for binary in _build/stage1/bin/*; do + if ldd "${binary}" &> /dev/null; then + warn "${binary} is not static!" + ldd "${binary}" + echo + bad_execs="$bad_execs $binary" + fi + done + if [ -n "$bad_execs" ]; then + fail "the following executables contain dynamic-object references: $bad_execs" + fi + fi + + if [[ "${CI_JOB_NAME}" != *"windows"* ]] && [ -z "${CROSS_TARGET:-}" ] && [ -z "${REINSTALL_GHC:-}" ] + + then + run_hadrian \ + test \ + --test-root-dirs=testsuite/tests/stage1 \ + --test-compiler=stage1 \ + ${TEST_WAYS[@]/#/--test-way=} \ + "runtest.opts+=${RUNTEST_ARGS:-}" || fail "hadrian stage1 test" + info "STAGE1_TEST=$?" + fi +} + function test_hadrian() { - check_msys2_deps _build/stage1/bin/ghc --version + + find_bindist_tar + dist_dir="_build/bindist" + + if [ -n "$bindist_tar" ]; then + mkdir -p "$dist_dir" + tar -xf "$bindist_tar" -C "$dist_dir" + fi + check_release_build # GitLab's log viewer renders ANSI colors, but stdout here is not a tty, # so the driver must be told to emit them. RUNTEST_ARGS="${RUNTEST_ARGS:-} --force-colors" - # Ensure that statically-linked builds are actually static - if [[ "${BUILD_FLAVOUR}" = *static* ]]; then - bad_execs="" - for binary in _build/stage1/bin/*; do - if ldd "${binary}" &> /dev/null; then - warn "${binary} is not static!" - ldd "${binary}" - echo - bad_execs="$bad_execs $binary" - fi - done - if [ -n "$bad_execs" ]; then - fail "the following executables contain dynamic-object references: $bad_execs" - fi - fi - if [[ "${CROSS_EMULATOR:-}" == "NOT_SET" ]]; then info "Cannot test cross-compiled build without CROSS_EMULATOR being set." return # If we have set CROSS_EMULATOR, then can't test using normal testsuite. elif [ -n "${CROSS_EMULATOR:-}" ] && [[ "${CROSS_TARGET:-}" != *"wasm"* ]]; then + info "Cross compiling with CROSS_EMULATOR='$CROSS_EMULATOR' and CROSS_TARGET='$CROSS_TARGET'" local instdir="$TOP/_build/install" local test_compiler="$instdir/bin/${cross_prefix}ghc$exe" - install_bindist _build/bindist/ghc-*/ "$instdir" + install_bindist $dist_dir/ghc-*/ "$instdir" echo 'main = putStrLn "hello world"' > expected run "$test_compiler" -package ghc "$TOP/.gitlab/hello.hs" -o hello @@ -701,6 +737,7 @@ function test_hadrian() { # > main = putStrLn "hello world" run diff -w expected actual elif [[ -n "${REINSTALL_GHC:-}" ]]; then + info "Running with reinstall GHC $REINSTALL_GHC" run_hadrian \ test \ --test-root-dirs=testsuite/tests/stage1 \ @@ -714,18 +751,9 @@ function test_hadrian() { else local instdir="$TOP/_build/install" local test_compiler="$instdir/bin/${cross_prefix}ghc$exe" - install_bindist _build/bindist/ghc-*/ "$instdir" + install_bindist $dist_dir/ghc-*/ "$instdir" - if [[ "${CI_JOB_NAME}" != *"windows"* ]] && [ -z "${CROSS_TARGET:-}" ] - then - run_hadrian \ - test \ - --test-root-dirs=testsuite/tests/stage1 \ - --test-compiler=stage1 \ - ${TEST_WAYS[@]/#/--test-way=} \ - "runtest.opts+=${RUNTEST_ARGS:-}" || fail "hadrian stage1 test" - info "STAGE1_TEST=$?" - fi + check_msys2_deps "$test_compiler" --version # Ensure the resulting compiler has the correct bignum-flavour, # except for cross-compilers as they may not support the interpreter @@ -735,6 +763,8 @@ function test_hadrian() { if [ $test_compiler_backend != "\"$BIGNUM_BACKEND\"" ]; then fail "Test compiler has a different BIGNUM_BACKEND ($test_compiler_backend) than requested ($BIGNUM_BACKEND)" fi + else + info "CROSS_TARGET=$CROSS_TARGET" fi # If we are doing a release job, check the compiler can build a profiled executable @@ -747,15 +777,14 @@ function test_hadrian() { run_hadrian \ test \ --summary-junit=./junit.xml \ - --test-have-intree-files \ --test-compiler="${test_compiler}" \ + --test-have-intree-files \ ${TEST_WAYS[@]/#/--test-way=} \ "runtest.opts+=${RUNTEST_ARGS:-}" \ "runtest.opts+=--unexpected-output-dir=$TOP/unexpected-test-output" \ || fail "hadrian main testsuite" info "STAGE2_TEST=$?" - fi } @@ -946,7 +975,7 @@ function run_hadrian() { if [ -n "${VERBOSE:-}" ]; then args+=("-V"); fi # Before running the compiler, unset variables gitlab env vars as these # can destabilise the performance test (see #20341) - (unset $(compgen -v | grep CI_*); + (unset $(compgen -v | grep -E '^(CI_|FF_|GITLAB_)'); run "${HADRIAN_PATH:-hadrian/build-cabal}" \ --flavour="$BUILD_FLAVOUR" \ -j"$cores" \ @@ -1131,6 +1160,10 @@ case ${1:-help} in time_it "test" test_hadrian || res=$? push_perf_notes exit $res ;; + test_hadrian_stage1) + res=0 + time_it "stage1 tests" test_hadrian_stage1 || res=$? + exit $res ;; run_hadrian) shift; run_hadrian "$@" ;; perf_test) run_perf_test ;; abi_test) abi_test ;; ===================================== .gitlab/common.sh ===================================== @@ -47,7 +47,7 @@ info() { echo_color "${LT_BLUE}" "$1"; } fail() { error "error: $1"; exit 1; } function run() { - info "Running $*..." + info "Running $*" "$@" || ( error "$* failed"; return 1; ) } ===================================== .gitlab/generate-ci/gen_ci.hs ===================================== @@ -1,6 +1,7 @@ #!/usr/bin/env cabal {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE OverloadedRecordDot #-} {-# LANGUAGE DeriveFunctor #-} {-# LANGUAGE GeneralizedNewtypeDeriving #-} {-# LANGUAGE ViewPatterns #-} @@ -10,6 +11,8 @@ {-# HLINT ignore "Use newtype instead of data" #-} {-# HLINT ignore "Use camelCase" #-} {-# LANGUAGE DerivingStrategies #-} +{-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE TypeFamilies #-} import Data.Aeson as A import qualified Data.Map as Map @@ -172,6 +175,7 @@ data BuildConfig , textWithSIMDUTF :: Bool , testsuiteUsePerf :: Bool , testsuiteWays :: [String] + , extendName :: String -> String } -- Extra arguments to pass to ./configure due to the BuildConfig @@ -242,6 +246,7 @@ vanilla = BuildConfig , textWithSIMDUTF = False , testsuiteUsePerf = False , testsuiteWays = [] + , extendName = id } splitSectionsBroken :: BuildConfig -> BuildConfig @@ -796,14 +801,17 @@ validateRuleString LoongArch64 = labelString "loongarch" -- certain platforms and so on. data Job = Job { jobStage :: String - , jobNeeds :: [String] + , jobNeeds :: [(String, Bool)] + -- ^ needs mustn't be combined with + -- dependencies; since we have needs + -- we rule out dependencies by not + -- adding it to this record , jobTags :: [String] , jobAllowFailure :: Bool , jobScript :: [String] , jobAfterScript :: [String] , jobDockerImage :: Maybe String , jobVariables :: Variables - , jobDependencies :: [String] , jobArtifacts :: Artifacts , jobCache :: Cache , jobRules :: OnOffRules @@ -818,8 +826,7 @@ instance ToJSON Job where [ "stage" A..= jobStage -- Convoluted to avoid download artifacts from ghci job -- https://docs.gitlab.com/ee/ci/yaml/#needsartifacts - , "needs" A..= map (\j -> object [ "job" A..= j, "artifacts" A..= False ]) jobNeeds - , "dependencies" A..= jobDependencies + , "needs" A..= map (\(j, a) -> object [ "job" A..= j, "artifacts" A..= a ]) jobNeeds , "image" A..= jobDockerImage , "tags" A..= jobTags , "allow_failure" A..= jobAllowFailure @@ -834,25 +841,48 @@ instance ToJSON Job where ] -- | Build a job description from the system description and 'BuildConfig' -job :: Arch -> Opsys -> BuildConfig -> NamedJob Job -job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} } +job :: Arch -> Opsys -> BuildConfig -> BuildAndTestJob Job +job arch opsys buildConfig = + MkBuildAndTestJob + { buildJob = NamedJob { name = jobName, jobInfo = Job { jobScript = jobBuildScript, jobStage = "full-build", jobNeeds = [("hadrian-ghc-in-ghci", False)], jobArtifacts = buildJobArtifacts, ..} } + , testJobs = + NamedJob { name = jobName <> "-test", jobInfo = testJobInfo } : + [ NamedJob + { name = jobName <> "-testperf" + , jobInfo = testJobInfo + { jobVariables = testJobInfo.jobVariables <> "RUNTEST_ARGS" =: "--config perf_path=perf" + , jobTags = [runnerPerfTag arch opsys] + } } + | testsuiteUsePerf buildConfig ] } where + testJobInfo = Job {jobScript = jobTestScript, jobNeeds = [(jobName, True)], jobStage = "testing", jobArtifacts = testJobArtifacts, ..} jobPlatform = (arch, opsys) jobRules = emptyRules jobName - jobName = testEnv arch opsys buildConfig + jobName = extendName buildConfig $ testEnv arch opsys buildConfig jobTags = tags arch opsys buildConfig jobDockerImage = dockerImage arch opsys - jobScript + jobTestScript + | Windows <- opsys + = [ "bash .gitlab/ci.sh setup" + , "bash .gitlab/ci.sh configure" -- configure needs to be run because hadrian gets its config like that + , "bash .gitlab/ci.sh test_hadrian" ] + | otherwise + = [ "sudo chown ghc:ghc -R ." | Linux {} <- [opsys]] ++ + [ ".gitlab/ci.sh setup" + , ".gitlab/ci.sh configure" + , ".gitlab/ci.sh test_hadrian" ] + + jobBuildScript | Windows <- opsys = [ "bash .gitlab/ci.sh setup" , "bash .gitlab/ci.sh configure" , "bash .gitlab/ci.sh build_hadrian" - , "bash .gitlab/ci.sh test_hadrian" + , "bash .gitlab/ci.sh test_hadrian_stage1" ] | otherwise = [ "find libraries -name config.sub -exec cp config.sub {} \\;" | Darwin == opsys ] ++ @@ -860,7 +890,7 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} } [ ".gitlab/ci.sh setup" , ".gitlab/ci.sh configure" , ".gitlab/ci.sh build_hadrian" - , ".gitlab/ci.sh test_hadrian" + , ".gitlab/ci.sh test_hadrian_stage1" ] jobAfterScript @@ -878,7 +908,6 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} } jobFlavour = mkJobFlavour buildConfig - jobDependencies = [] jobVariables = mconcat [ opsysVariables arch opsys , "TEST_ENV" =: testEnv arch opsys buildConfig @@ -915,7 +944,6 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} } | validateNonmovingGc buildConfig ] in "RUNTEST_ARGS" =: (trim . unwords) runtestArgs - , if testsuiteUsePerf buildConfig then "RUNTEST_ARGS" =: "--config perf_path=perf" else mempty , "TEST_WAYS" =: unwords (testsuiteWays buildConfig) ] @@ -924,7 +952,7 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} } -- Keep in sync with the exclude list in `function clean()` in -- `.gitlab/ci.sh`! - jobArtifacts = Artifacts + buildJobArtifacts = Artifacts { junitReport = "junit.xml" , expireIn = "2 weeks" , artifactPaths = [binDistName arch opsys buildConfig ++ ".tar.xz" @@ -933,6 +961,15 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} } , artifactsWhen = ArtifactsAlways } + -- don't reupload the bindist from the testsuite run + testJobArtifacts = Artifacts + { junitReport = "junit.xml" + , expireIn = "2 weeks" + , artifactPaths = ["junit.xml" + ,"unexpected-test-output.tar.gz"] + , artifactsWhen = ArtifactsAlways + } + jobCache -- N.B. We have temporarily disabled cabal-install store caching on -- Windows due to #21347. @@ -945,8 +982,6 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} } } jobAllowFailure = False - jobStage = "full-build" - jobNeeds = ["hadrian-ghc-in-ghci"] --------------------------------------------------------------------------- -- Job Modifiers @@ -997,7 +1032,7 @@ delVariable k j = j { jobVariables = MonoidalMap $ Map.delete k $ unMonoidalMap --------------------------------------------------------------------- -- | Make a normal validate CI job -validate :: Arch -> Opsys -> BuildConfig -> NamedJob Job +validate :: Arch -> Opsys -> BuildConfig -> BuildAndTestJob Job validate = job -- Nightly and release apply the FastCI configuration to all jobs so that they all run in @@ -1009,25 +1044,28 @@ releaseRule :: Job -> Job releaseRule = setJobRule ReleaseOnly -- | Make a normal nightly CI job -nightly :: Arch -> Opsys -> BuildConfig -> NamedJob Job +nightly :: Arch -> Opsys -> BuildConfig -> BuildAndTestJob Job nightly arch opsys bc = - let NamedJob n j = job arch opsys bc - in NamedJob { name = "nightly-" ++ n + let MkBuildAndTestJob {buildJob, testJobs} = job arch opsys bc { extendName = ("nightly-" <>) . bc.extendName } + mkJob NamedJob {name, jobInfo} + = NamedJob { name , jobInfo = nightlyRule . keepArtifacts "8 weeks" - . highCompression $ j + . highCompression $ jobInfo } + in MkBuildAndTestJob { buildJob = mkJob buildJob, testJobs = map mkJob testJobs} -- | Make a normal release CI job -release :: Arch -> Opsys -> BuildConfig -> NamedJob Job +release :: Arch -> Opsys -> BuildConfig -> BuildAndTestJob Job release arch opsys bc = - let NamedJob n j = job arch opsys (bc { buildFlavour = Release }) - in NamedJob { name = "release-" ++ n + let MkBuildAndTestJob {buildJob, testJobs} = job arch opsys (bc { buildFlavour = Release, extendName = ("release-" <>) . bc.extendName }) + mkJob NamedJob {name, jobInfo} = NamedJob { name , jobInfo = releaseRule . keepArtifacts "1 year" . ignorePerfFailures - . highCompression $ j + . highCompression $ jobInfo } + in MkBuildAndTestJob {buildJob = mkJob buildJob, testJobs = map mkJob testJobs} --------------------------------------------------------------------- -- Specific job modification functions --------------------------------------------------------------------- @@ -1053,11 +1091,6 @@ ignorePerfFailures = addVariable "IGNORE_PERF_FAILURES" "all" highCompression :: Job -> Job highCompression = addVariable "XZ_OPT" "-9" --- | Change the tag of the job to make sure the job is scheduled on a --- runner that has the necessary capabilties to run the job with 'perf' --- profiling counters. -perfProfilingJobTag :: Arch -> Opsys -> Job -> Job -perfProfilingJobTag arch opsys j = j { jobTags = [ runnerPerfTag arch opsys ] } -- | Mark the validate job to run in fast-ci mode -- This is default way, to enable all jobs you have to apply the `full-ci` label. @@ -1090,8 +1123,8 @@ disableValidate = modifyValidateJobs (removeValidateJobRule FastCI . removeValid data NamedJob a = NamedJob { name :: String, jobInfo :: a } deriving (Show, Functor) -renameJob :: (String -> String) -> NamedJob a -> NamedJob a -renameJob f (NamedJob n i) = NamedJob (f n) i +data BuildAndTestJob a = MkBuildAndTestJob { buildJob :: NamedJob a, testJobs :: [NamedJob a]} + deriving stock (Show, Functor) instance ToJSON a => ToJSON (NamedJob a) where toJSON nj = object @@ -1105,21 +1138,21 @@ instance ToJSON a => ToJSON (NamedJob a) where -- Jobs are grouped into either triples or pairs depending on whether the -- job is just validate and nightly, or also release. data JobGroup a - = StandardTriple { v :: Maybe (NamedJob a) - , n :: Maybe (NamedJob a) - , r :: Maybe (NamedJob a) + = StandardTriple { v :: Maybe (BuildAndTestJob a) + , n :: Maybe (BuildAndTestJob a) + , r :: Maybe (BuildAndTestJob a) } deriving (Functor, Show) instance ToJSON a => ToJSON (JobGroup a) where toJSON StandardTriple{..} = object - [ "v" A..= v - , "n" A..= n - , "r" A..= r - ] + [ "v" .= v + , "n" .= n + , "r" .= r ] -rename :: (String -> String) -> JobGroup a -> JobGroup a -rename f (StandardTriple nv nn nr) = StandardTriple (renameJob f <$> nv) (renameJob f <$> nn) (renameJob f <$> nr) +instance ToJSON a => ToJSON (BuildAndTestJob a) where + -- the test job is not relevant for the job metadata output + toJSON MkBuildAndTestJob {buildJob} = toJSON buildJob -- | Construct a 'JobGroup' which consists of a validate, nightly and release build with -- a specific config. @@ -1142,10 +1175,10 @@ validateBuilds a op bc = StandardTriple { v = Just (validate a op bc) , r = Nothing } flattenJobGroup :: JobGroup a -> [(String, a)] -flattenJobGroup (StandardTriple a b c) = map flattenNamedJob (catMaybes [a,b,c]) +flattenJobGroup (StandardTriple a b c) = concatMap flattenNamedJob (catMaybes [a,b,c]) -flattenNamedJob :: NamedJob a -> (String, a) -flattenNamedJob (NamedJob n i) = (n, i) +flattenNamedJob :: BuildAndTestJob a -> [(String, a)] +flattenNamedJob MkBuildAndTestJob {buildJob, testJobs} = (buildJob.name, buildJob.jobInfo) : map (\NamedJob {name, jobInfo} -> (name, jobInfo)) testJobs -- | Specification for all the jobs we want to build. jobs :: Map String Job @@ -1172,8 +1205,9 @@ debian_x86 = , modifyNightlyJobs allowFailure (modifyValidateJobs (allowFailure . manual) tsan_jobs) , -- Nightly allowed to fail: #22343 modifyNightlyJobs allowFailure (modifyValidateJobs manual (validateBuilds Amd64 (Linux validate_debian) noTntc)) + -- Run the 'perf' profiling nightly job in the release config. - , perfProfilingJob Amd64 (Linux Debian13) releaseConfig + , disableValidate (validateBuilds Amd64 (Linux validate_debian) (usePerfProfilingTestsuite releaseConfig)) , onlyRule LLVMBackend (validateBuilds Amd64 (Linux validate_debian) llvm) , addValidateRule TestPrimops (standardBuilds Amd64 (Linux validate_debian)) @@ -1184,12 +1218,6 @@ debian_x86 = where validate_debian = Debian13 - perfProfilingJob arch sys buildConfig = - -- Rename the job to avoid conflicts - rename (<> "-perf") - $ modifyJobs (perfProfilingJobTag arch sys) - $ disableValidate (validateBuilds arch sys $ usePerfProfilingTestsuite buildConfig) - tsan_jobs = modifyJobs ( addVariable "TSAN_OPTIONS" "suppressions=$CI_PROJECT_DIR/rts/.tsan-suppressions" @@ -1231,7 +1259,7 @@ fedora_x86 = -- validate pipeline which is built with perf. fastCI (standardBuildsWithConfig Amd64 (Linux Fedora43) releaseConfig) -- This job is only for generating head.hackage docs - , hackage_doc_job (disableValidate (standardBuildsWithConfig Amd64 (Linux Fedora43) releaseConfig)) + , hackage_doc_job , disableValidate (standardBuildsWithConfig Amd64 (Linux Fedora43) dwarf) , disableValidate (standardBuilds Amd64 (Linux Fedora43)) -- For UBSan jobs, only enable for validate/nightly pipelines. @@ -1245,7 +1273,10 @@ fedora_x86 = $ validateBuilds Amd64 (Linux Fedora43) enableUBSan ] where - hackage_doc_job = rename (<> "-hackage") . modifyJobs (addVariable "HADRIAN_ARGS" "--haddock-for-hackage") + hackage_doc_job + = modifyJobs (addVariable "HADRIAN_ARGS" "--haddock-for-hackage") $ + disableValidate (standardBuildsWithConfig Amd64 (Linux Fedora43) + releaseConfig { extendName = (<> "-hackage") . releaseConfig.extendName }) windows_x86 :: [JobGroup Job] windows_x86 = @@ -1438,7 +1469,10 @@ platform_mapping = Map.map go combined_result process sel = Map.fromListWith combine - [ (uncurry mkPlatform (jobPlatform (jobInfo j)), j) + -- for now we only support buildJob platform equals testjob platform + -- if this restriction were to be lifted, we just had to indicate that the + -- specific testjob runs on a different platform + [ (uncurry mkPlatform (jobPlatform (jobInfo (buildJob j))), j) | (sel -> Just j) <- job_groups ] @@ -1457,15 +1491,15 @@ platform_mapping = Map.map go combined_result | platform <- S.toList all_platforms ] - combine a b + combine bta@MkBuildAndTestJob {buildJob = a} btb@MkBuildAndTestJob {buildJob = b} | name a `elem` whitelist , name b `elem` whitelist , name a /= name b = error $ unlines [ "both selected, can only select one job for a specific key: " , show (name a) , show (name b) ] -- Explicitly selected - | name a `elem` whitelist = a -- Explicitly selected - | name b `elem` whitelist = b + | name a `elem` whitelist = bta -- Explicitly selected + | name b `elem` whitelist = btb | otherwise = error (show (name a) ++ show (name b)) go = fmap (BindistInfo . unwords . fromJust . mmlookup "BIN_DIST_NAME" . jobVariables) ===================================== .gitlab/jobs.yaml ===================================== The diff for this file was not included because it is too large. ===================================== hadrian/doc/cross-compile.md ===================================== @@ -1,3 +1,5 @@ +**This is severely outdated. And is here merely for historical interest** + ## Build a cross-compiling GHC In this example, our host machine is "Ubuntu 16.04.2 LTS, Linux ubuntu 4.4.0-79-generic 86_64". ===================================== hadrian/src/Base.hs ===================================== @@ -5,6 +5,7 @@ {-# LANGUAGE StandaloneDeriving #-} {-# LANGUAGE StandaloneKindSignatures #-} {-# LANGUAGE TypeOperators #-} +{-# LANGUAGE MultiWayIf #-} module Base ( -- * General utilities @@ -22,6 +23,7 @@ module Base ( module Development.Shake.Util, Vec(..), (&%>), + exe, -- * Basic data types module Hadrian.Package, @@ -57,7 +59,7 @@ import Development.Shake hiding (unit, (&%>), (*>), Normal, cmd, cmd_) #endif import qualified Development.Shake as Shake import Development.Shake.Classes -import Development.Shake.FilePath +import Development.Shake.FilePath hiding (exe) import Development.Shake.Util import Hadrian.Oracles.DirectoryContents import Hadrian.Utilities @@ -67,6 +69,13 @@ import GHC.Stack ( HasCallStack ) import Stage import Way +import GHC.Platform.ArchOS (ArchOS(..), Arch (..), OS (..)) + +exe :: ArchOS -> FilePath +exe arch + | ArchWasm32 <- archOS_arch arch = "wasm" + | OSMinGW32 <- archOS_OS arch = "exe" + | otherwise = "" -- | Hadrian lives in the 'hadrianPath' directory of the GHC tree. hadrianPath :: FilePath ===================================== hadrian/src/Builder.hs ===================================== @@ -15,7 +15,6 @@ import Control.Exception.Extra (Partial) import qualified Data.ByteString.Lazy.Char8 as BSL import Development.Shake.Classes import Development.Shake.Command -import Development.Shake.FilePath import GHC.Generics import GHC.Platform.ArchOS (ArchOS(..), Arch(..), OS(..)) import qualified Hadrian.Builder as H @@ -467,7 +466,7 @@ systemBuilderPath builder = case builder of -- Get program from a certain stage's target configuration fromStageTC stage keyname key = do - path <- prgPath . key <$> targetStage stage + path <- prgPath . key <$> perStageTarget stage validate keyname path validate keyname path = do @@ -494,7 +493,7 @@ systemBuilderPath builder = case builder of case (windowsHost, hasExtension fullPath) of (False, _ ) -> return path (True , True ) -> return $ unifyPath fullPath - (True , False) -> return $ unifyPath fullPath <.> exe + (True , False) -> return $ unifyPath fullPath <.> exe (tgtArchOs target) -- Without this function, on Windows we can observe a bad builder path -- for 'autoreconf'. If the relevant system.config field is set to ===================================== hadrian/src/Context.hs ===================================== @@ -51,7 +51,7 @@ getWay = way <$> getContext -- | Get the 'Target' configuration of the current stage getStagedTarget :: Expr Context b Target -getStagedTarget = expr . targetStage =<< getStage +getStagedTarget = expr . perStageTarget =<< getStage -- | Path to the directory containing the final artifact in a given 'Context'. libPath :: Context -> Action FilePath ===================================== hadrian/src/Flavour.hs ===================================== @@ -170,7 +170,7 @@ werror = addArgs $ do -- TODO: get rid of EFF_ altogether (#14647) and make sure -- unregisterised backend emits clean C without needing -- these hacks. - , queryTargetTarget stage tgtUnregisterised ? arg "-optc-Wno-error" + , queryPerStageTargetSpec stage tgtUnregisterised ? arg "-optc-Wno-error" ] -- N.B. We currently don't build the boot libraries' C sources with -Werror -- as this tends to be a portability nightmare. ===================================== hadrian/src/Hadrian/Haskell/Hash.hs ===================================== @@ -125,7 +125,7 @@ pkgHashOracle = void $ addOracleCache $ \(PkgHashKey (stag, pkg)) -> do flav <- flavour let flavourArgs = extraArgs flav - targetOs <- queryTargetTarget stag queryOS + targetOs <- queryPerStageTargetSpec stag queryOS projectVersion <- setting ProjectVersion let pkgHashCompilerId = "ghc-" ++ projectVersion pkgHashPlatform = targetOs ===================================== hadrian/src/Hadrian/Package.hs ===================================== @@ -23,7 +23,7 @@ module Hadrian.Package ( ) where import Development.Shake.Classes -import Development.Shake.FilePath +import Development.Shake.FilePath ((<.>)) import GHC.Generics import Hadrian.Utilities ===================================== hadrian/src/Oracles/Flag.hs ===================================== @@ -77,13 +77,13 @@ getFlag = expr . flag targetRTSLinkerOnlySupportsSharedLibs :: Stage -> Action Bool targetRTSLinkerOnlySupportsSharedLibs s = - queryTargetTarget s Toolchain.tgtRTSLinkerOnlySupportsSharedLibs + queryPerStageTargetSpec s Toolchain.tgtRTSLinkerOnlySupportsSharedLibs arSupportsDashL :: Stage -> Action Bool -arSupportsDashL stage = Toolchain.arSupportsDashL . tgtAr <$> targetStage stage +arSupportsDashL stage = Toolchain.arSupportsDashL . tgtAr <$> perStageTarget stage arSupportsAtFile :: Stage -> Action Bool -arSupportsAtFile stage = Toolchain.arSupportsAtFile . tgtAr <$> targetStage stage +arSupportsAtFile stage = Toolchain.arSupportsAtFile . tgtAr <$> perStageTarget stage targetSupportsSharedLibs :: Stage -> Action Bool targetSupportsSharedLibs stage = do @@ -101,10 +101,10 @@ targetSupportsThreadedRts stage = do -- | Does the target support the -N RTS flag? targetSupportsSMP :: Stage -> Action Bool -targetSupportsSMP stage = queryTargetTarget stage Toolchain.tgtSupportsSMP +targetSupportsSMP stage = queryPerStageTargetSpec stage Toolchain.tgtSupportsSMP targetUseLibffiForAdjustors :: Stage -> Action Bool -targetUseLibffiForAdjustors stage = queryTargetTarget stage tgtUseLibffiForAdjustors +targetUseLibffiForAdjustors stage = queryPerStageTargetSpec stage tgtUseLibffiForAdjustors useLibdw :: Stage -> Action Bool -useLibdw stage = queryTargetTarget stage (isJust . tgtRTSWithLibdw) +useLibdw stage = queryPerStageTargetSpec stage (isJust . tgtRTSWithLibdw) ===================================== hadrian/src/Oracles/Setting.hs ===================================== @@ -6,7 +6,7 @@ module Oracles.Setting ( -- * Helpers ghcCanonVersion, cmdLineLengthLimit, targetSupportsRPaths, topDirectory, - libsuf, ghcVersionStage, shPath, targetStage, crossStage, queryTarget, queryTargetTarget, + libsuf, ghcVersionStage, shPath, perStageTarget, crossStage, queryTarget, queryPerStageTargetSpec, isHostStage, -- ** Target platform things @@ -152,7 +152,7 @@ isOsxTarget :: Stage -> Action Bool isOsxTarget stage = anyTargetOs stage [OSDarwin] isArmTarget :: Stage -> Action Bool -isArmTarget stage = queryTargetTarget stage (isARM . archOS_arch . tgtArchOs) +isArmTarget stage = queryPerStageTargetSpec stage (isARM . archOS_arch . tgtArchOs) -- | Check whether the host OS setting matches one of the given strings. anyHostOs :: [OS] -> Action Bool @@ -161,15 +161,15 @@ anyHostOs oss = (`elem` oss) <$> queryHostTarget (archOS_OS . tgtArchOs) -- | Check whether the target architecture setting matches one of the given -- strings. anyTargetArch :: Stage -> [Arch] -> Action Bool -anyTargetArch stage archs = (`elem` archs) <$> queryTargetTarget stage (archOS_arch . tgtArchOs) +anyTargetArch stage archs = (`elem` archs) <$> queryPerStageTargetSpec stage (archOS_arch . tgtArchOs) -- | Check whether the target OS setting matches one of the given strings. anyTargetOs :: Stage -> [OS] -> Action Bool -anyTargetOs stage oss = (`elem` oss) <$> queryTargetTarget stage (archOS_OS . tgtArchOs) +anyTargetOs stage oss = (`elem` oss) <$> queryPerStageTargetSpec stage (archOS_OS . tgtArchOs) -- | Check whether the target OS uses the ELF object format. isElfTarget :: Stage -> Action Bool -isElfTarget stage = queryTargetTarget stage (osElfTarget . archOS_OS . tgtArchOs) +isElfTarget stage = queryPerStageTargetSpec stage (osElfTarget . archOS_OS . tgtArchOs) -- | Check whether the target OS supports the @-rpath@ linker option when -- using dynamic linking. @@ -179,7 +179,7 @@ isElfTarget stage = queryTargetTarget stage (osElfTarget . archOS_OS . tgtArchOs -- TODO: Windows supports lazy binding (but GHC doesn't currently support -- dynamic way on Windows anyways). targetSupportsRPaths :: Stage -> Action Bool -targetSupportsRPaths stage = queryTargetTarget stage +targetSupportsRPaths stage = queryPerStageTargetSpec stage (\t -> let os = archOS_OS (tgtArchOs t) in osElfTarget os || osMachOTarget os) @@ -187,7 +187,7 @@ targetSupportsRPaths stage = queryTargetTarget stage -- ARM)? targetArmVersion :: Stage -> Action (Maybe ArmISA) targetArmVersion stage = runMaybeT $ do - ArchARM isa _ _ <- lift $ queryTargetTarget stage (archOS_arch . tgtArchOs) + ArchARM isa _ _ <- lift $ queryPerStageTargetSpec stage (archOS_arch . tgtArchOs) return isa -- | Canonicalised GHC version number, used for integer version comparisons. We @@ -231,33 +231,34 @@ libsuf st way let suffix = waySuffix (removeWayUnit Dynamic way) return (suffix ++ "-ghc" ++ version ++ extension) --- | Build libraries for this `Stage` targetting this `Target` +-- | Determine the 'Target' that this 'Stage' is supposed to build libraries for. -- --- For example, we want to build RTS with stage1 for the host target as we +-- For example, we want to build RTS with stage1 for the host as we -- produce a host executable with stage1 (which cross-compiles to stage2). -targetStage :: Stage -> Action Target -targetStage Stage0 {} = getHostTarget -targetStage stage | isHostStage stage = do +perStageTarget :: Stage -> Action Target +perStageTarget Stage0 {} = getHostTarget +perStageTarget stage | isHostStage stage = do ht <- getHostTarget tt <- getTargetTarget if targetPlatformTriple ht == targetPlatformTriple tt then return tt else return ht -targetStage _ = getTargetTarget +perStageTarget _ = getTargetTarget isHostStage :: Stage -> Bool isHostStage stage | stage <= Stage1 = True isHostStage _ = False -queryTarget :: Stage -> (Target -> a) -> (Expr c b a) -queryTarget s f = expr (f <$> targetStage s) +queryTarget :: Stage -> (Target -> a) -> Expr c b a +queryTarget s f = expr (f <$> perStageTarget s) -queryTargetTarget :: Stage -> (Target -> a) -> Action a -queryTargetTarget s f = f <$> targetStage s +-- | get the conentents of the .target file for the specified stage +queryPerStageTargetSpec :: Stage -> (Target -> a) -> Action a +queryPerStageTargetSpec s f = f <$> perStageTarget s -- | A 'Stage' is a cross-stage if the produced compiler is a cross-compiler. crossStage :: Stage -> Action Bool crossStage st = do - st_target <- targetStage (succStage st) - st_host <- targetStage st + st_target <- perStageTarget (succStage st) + st_host <- perStageTarget st return (targetPlatformTriple st_target /= targetPlatformTriple st_host) ===================================== hadrian/src/Oracles/TestSettings.hs ===================================== @@ -13,6 +13,7 @@ import Hadrian.Oracles.TextFile import Oracles.Setting (topDirectory, setting, ProjectSetting(..), crossStage) import Packages import Settings.Program (programContext) +import GHC.Toolchain (Target(tgtArchOs)) testConfigFile :: Action FilePath testConfigFile = buildRoot <&> (-/- "test/ghcconfig") @@ -153,7 +154,8 @@ getTestExePath testGhc pkg = do bindir <- getBinaryDirectory testGhc compiler_path <- getCompilerPath testGhc cross <- getBooleanSetting TestCrossCompiling + tgt <- getHostTarget -- FIXME: is this right -- ask rodrigo about the targettarget and query/get versions of this function let cross_prefix = if cross then dropWhileEnd ((/=) '-') (takeFileName compiler_path) else "" -- get relative path for the given program in the given stage - liftIO $ makeAbsolute (bindir </> (cross_prefix ++ programBasename pkg) <.> exe) + liftIO $ makeAbsolute (bindir </> (cross_prefix ++ programBasename pkg) <.> exe (tgtArchOs tgt)) -- get relative path for the given program in the given stage ===================================== hadrian/src/Packages.hs ===================================== @@ -24,6 +24,8 @@ import Hadrian.Utilities import Base import Context.Type import Oracles.Setting +import GHC.Toolchain (tgtArchOs) +import qualified Development.Shake.FilePath as Shake -- | These are all GHC packages we know about. Build rules will be generated for -- all of them. However, not all of these packages will be built. For example, @@ -196,12 +198,13 @@ programPath :: Context -> Action FilePath programPath context@Context {..} = do name <- programName context path <- stageBinPath stage - return $ path -/- name <.> exe + arch <- queryPerStageTargetSpec stage tgtArchOs + return $ path -/- name <.> exe arch -- TODO: Move @timeout@ to the @util@ directory and build in a more standard -- location like other programs used only by the testsuite. timeoutPath :: FilePath -timeoutPath = "testsuite/timeout/install-inplace/bin/timeout" <.> exe +timeoutPath = "testsuite/timeout/install-inplace/bin/timeout" <.> Shake.exe -- TODO: Can we extract this information from Cabal files? -- | Some program packages should not be linked with Haskell main function. ===================================== hadrian/src/Rules/Generate.hs ===================================== @@ -277,7 +277,7 @@ generateRules = do else prefix relPkgDb = makeRelativeNoSysLink libTopDir pkgDb go (generateSettings libStage out True relPkgDb) out - (prefix -/- "targets" -/- "default.target") %> \out -> go (show <$> expr (targetStage (succStage stage))) out + (prefix -/- "targets" -/- "default.target") %> \out -> go (show <$> expr (perStageTarget (succStage stage))) out where file <~+ gen = file %> \out -> generate out emptyTarget gen >> makeExecutable out @@ -450,7 +450,7 @@ bindistRules = do , interpolateVar "TargetWordBigEndian" $ getTarget isBigEndian , interpolateVar "TargetWordSize" $ getTarget wordSize , interpolateVar "Unregisterised" $ yesNo <$> getTarget tgtUnregisterised - , interpolateVar "UseLibdw" $ fmap yesNo $ interp $ staged (fmap (isJust . tgtRTSWithLibdw) . targetStage) + , interpolateVar "UseLibdw" $ fmap yesNo $ interp $ staged (fmap (isJust . tgtRTSWithLibdw) . perStageTarget) , interpolateVar "UseLibffiForAdjustors" $ yesNo <$> getTarget tgtUseLibffiForAdjustors , interpolateVar "BaseUnitId" $ bindistUnitId base , interpolateVar "GhcUnitId" $ bindistUnitId compiler @@ -546,7 +546,7 @@ generateConfigHs :: Expr String generateConfigHs = do stage <- getStage let chooseSetting x y = case stage of { Stage0 {} -> x; _ -> y } - let queryTarget f = f <$> expr (targetStage stage) + let queryTarget f = f <$> expr (perStageTarget stage) -- Not right for stage3 buildPlatform <- chooseSetting (queryBuild targetPlatformTriple) (queryHost targetPlatformTriple) hostPlatform <- queryTarget targetPlatformTriple ===================================== hadrian/src/Rules/Program.hs ===================================== @@ -9,7 +9,7 @@ import Base import Context import Expression hiding (stage, way) import Oracles.ModuleFiles -import Oracles.Setting (topDirectory) +import Oracles.Setting (topDirectory, perStageTarget) import Packages import Settings import Settings.Default @@ -18,6 +18,7 @@ import Target import Utilities import Rules.Library import Rules.Register +import GHC.Toolchain (Target(tgtArchOs)) -- | TODO: Drop code duplication buildProgramRules :: [(Resource, Int)] -> Rules () @@ -49,13 +50,14 @@ getProgramContexts stage = do -- 'Rules', because it is an 'Action' depending on an oracle. sPackages <- filter isProgram <$> stagePackages stage tPackages <- testsuitePackages + tgt <- perStageTarget stage -- TODO: Shall we use Stage2 for testsuite packages instead? let allPackages = sPackages ++ tPackages forM allPackages $ \pkg -> do ctx <- programContext stage pkg -- TODO: see todo on programContext. name <- programName ctx - return (name <.> exe, ctx) + return (name <.> exe (tgtArchOs tgt), ctx) lookupProgramContext :: FilePath -> [(FilePath, Context)] -> Maybe Context lookupProgramContext wholePath progs = lookup (takeFileName wholePath) progs ===================================== hadrian/src/Rules/Test.hs ===================================== @@ -1,3 +1,4 @@ +{-# LANGUAGE OverloadedRecordDot #-} {-# OPTIONS_GHC -Wno-unused-top-binds #-} module Rules.Test (testRules) where @@ -20,58 +21,71 @@ import qualified System.Directory as IO import GHC.Toolchain as Toolchain import GHC.Toolchain.Program as Toolchain +import Hadrian.Oracles.TextFile (getHostTarget, getTargetTarget) +import GHC.Platform.ArchOS (ArchOS (..), Arch (..), OS (..)) -checkPprProgPath, checkPprSourcePath :: FilePath -checkPprProgPath = "test/bin/check-ppr" <.> exe +checkPprProgPath :: ArchOS -> FilePath +checkPprProgPath archos = "test/bin/check-ppr" <.> exe archos +checkPprSourcePath :: FilePath checkPprSourcePath = "utils/check-ppr/Main.hs" checkPprExtra :: [String] checkPprExtra = [] -checkExactProgPath, checkExactSourcePath :: FilePath -checkExactProgPath = "test/bin/check-exact" <.> exe +checkExactProgPath :: ArchOS -> FilePath +checkExactProgPath archos = "test/bin/check-exact" <.> exe archos +checkExactSourcePath :: FilePath checkExactSourcePath = "utils/check-exact/Main.hs" checkExactExtra :: [String] checkExactExtra = ["-iutils/check-exact"] -countDepsProgPath, countDepsSourcePath :: FilePath -countDepsProgPath = "test/bin/count-deps" <.> exe +countDepsProgPath :: ArchOS -> FilePath +countDepsProgPath archos = "test/bin/count-deps" <.> exe archos +countDepsSourcePath :: FilePath countDepsSourcePath = "utils/count-deps/Main.hs" countDepsExtra :: [String] countDepsExtra = ["-iutils/count-deps"] -dumpDeclsProgPath, dumpDeclsSourcePath :: FilePath -dumpDeclsProgPath = "test/bin/dump-decls" <.> exe +dumpDeclsProgPath :: ArchOS -> FilePath +dumpDeclsProgPath archos = "test/bin/dump-decls" <.> exe archos +dumpDeclsSourcePath :: FilePath dumpDeclsSourcePath = "utils/dump-decls/Main.hs" dumpDeclsExtra :: [String] dumpDeclsExtra = [] -noteLinterProgPath, noteLinterSourcePath :: FilePath -noteLinterProgPath = "test/bin/lint-notes" <.> exe +noteLinterProgPath :: ArchOS -> FilePath +noteLinterProgPath archos = "test/bin/lint-notes" <.> exe archos +noteLinterSourcePath :: FilePath noteLinterSourcePath = "linters/lint-notes/Main.hs" noteLinterExtra :: [String] noteLinterExtra = ["-ilinters/lint-notes"] -codeLinterProgPath, codeLinterSourcePath :: FilePath -codeLinterProgPath = "test/bin/lint-codes" <.> exe +codeLinterProgPath :: ArchOS -> FilePath +codeLinterProgPath archos = "test/bin/lint-codes" <.> exe archos +codeLinterSourcePath :: FilePath codeLinterSourcePath = "linters/lint-codes/Main.hs" codeLinterExtra :: [String] codeLinterExtra = ["-ilinters/lint-codes"] -whitespaceLinterProgPath, whitespaceLinterSourcePath :: FilePath -whitespaceLinterProgPath = "test/bin/lint-whitespace" <.> exe +whitespaceLinterProgPath :: ArchOS -> FilePath +whitespaceLinterProgPath archos = "test/bin/lint-whitespace" <.> exe archos +whitespaceLinterSourcePath :: FilePath whitespaceLinterSourcePath = "linters/lint-whitespace/Main.hs" whitespaceLinterExtra :: [String] whitespaceLinterExtra = ["-ilinters/lint-whitespace", "-ilinters/linters-common"] -changelogDProgPath, changelogDSourcePath :: FilePath -changelogDProgPath = "test/bin/changelog-d" <.> exe +changelogDProgPath :: ArchOS -> FilePath +changelogDProgPath archos = "test/bin/changelog-d" <.> exe archos +changelogDSourcePath :: FilePath changelogDSourcePath = "utils/changelog-d/ChangelogD.hs" changelogDExtra :: [String] changelogDExtra = ["-iutils/changelog-d"] data CheckProgram = CheckProgram { cp_target :: String -- ^ Name for the hadrian target - , cp_exe_path :: FilePath -- ^ Path to resulting executable + , cp_exe_path :: ArchOS -> FilePattern + -- ^ Path to resulting executable. + -- This depends on the taret the program is built for, + -- not the host that hadrian is built on , cp_src_path :: FilePath -- ^ Source to the Main.hs for the executable , cp_extra_args :: [String] -- ^ Any extra arguments to use when compiling Main.hs , cp_hadrian_pkg :: Package -- ^ How to build the executable when using in-tree compiler. @@ -81,16 +95,16 @@ data CheckProgram = checkPrograms :: [CheckProgram] checkPrograms = - [ CheckProgram "test:check-ppr" checkPprProgPath checkPprSourcePath checkPprExtra checkPpr id id - , CheckProgram "test:check-exact" checkExactProgPath checkExactSourcePath checkExactExtra checkExact id id - , CheckProgram "test:count-deps" countDepsProgPath countDepsSourcePath countDepsExtra countDeps id id - , CheckProgram "test:dump-decls" dumpDeclsProgPath dumpDeclsSourcePath dumpDeclsExtra dumpDecls id id - , CheckProgram "lint:notes" noteLinterProgPath noteLinterSourcePath noteLinterExtra lintNotes (const stage0Boot) id - , CheckProgram "lint:codes" codeLinterProgPath codeLinterSourcePath codeLinterExtra lintCodes id id - , CheckProgram "lint:whitespace" whitespaceLinterProgPath whitespaceLinterSourcePath whitespaceLinterExtra lintWhitespace (const stage0Boot) (filter (/= lintersCommon)) + [ CheckProgram { cp_target = "test:check-ppr", cp_exe_path = checkPprProgPath, cp_src_path = checkPprSourcePath, cp_extra_args = checkPprExtra, cp_hadrian_pkg = checkPpr, cp_modify_stage = id, cp_modify_deps = id } + , CheckProgram { cp_target = "test:check-exact", cp_exe_path = checkExactProgPath, cp_src_path = checkExactSourcePath, cp_extra_args = checkExactExtra, cp_hadrian_pkg = checkExact, cp_modify_stage = id, cp_modify_deps = id } + , CheckProgram { cp_target = "test:count-deps", cp_exe_path = countDepsProgPath, cp_src_path = countDepsSourcePath, cp_extra_args = countDepsExtra, cp_hadrian_pkg = countDeps, cp_modify_stage = id, cp_modify_deps = id } + , CheckProgram { cp_target = "test:dump-decls", cp_exe_path = dumpDeclsProgPath, cp_src_path = dumpDeclsSourcePath, cp_extra_args = dumpDeclsExtra, cp_hadrian_pkg = dumpDecls, cp_modify_stage = id, cp_modify_deps = id } + , CheckProgram { cp_target = "lint:notes", cp_exe_path = noteLinterProgPath, cp_src_path = noteLinterSourcePath, cp_extra_args = noteLinterExtra, cp_hadrian_pkg = lintNotes, cp_modify_stage = (const stage0Boot), cp_modify_deps = id } + , CheckProgram { cp_target = "lint:codes", cp_exe_path = codeLinterProgPath, cp_src_path = codeLinterSourcePath, cp_extra_args = codeLinterExtra, cp_hadrian_pkg = lintCodes, cp_modify_stage = id, cp_modify_deps = id } + , CheckProgram { cp_target = "lint:whitespace", cp_exe_path = whitespaceLinterProgPath, cp_src_path = whitespaceLinterSourcePath, cp_extra_args = whitespaceLinterExtra, cp_hadrian_pkg = lintWhitespace, cp_modify_stage = (const stage0Boot), cp_modify_deps = (filter (/= lintersCommon)) } -- N.B. The lint:changelog build is replicated by lint_changelog in -- .gitlab/ci.sh. Keep its package dependencies in sync with this target. - , CheckProgram "lint:changelog" changelogDProgPath changelogDSourcePath changelogDExtra changelogD (const stage0Boot) id + , CheckProgram { cp_target = "lint:changelog", cp_exe_path = changelogDProgPath, cp_src_path = changelogDSourcePath, cp_extra_args = changelogDExtra, cp_hadrian_pkg = changelogD, cp_modify_stage = (const stage0Boot), cp_modify_deps = id } ] inTreeOutTree :: (Stage -> Action b) -> Action b -> Action b @@ -134,12 +148,23 @@ testRules = do testsuiteDeps + -- the test targets will all be compiled by the test comppiler which is going + -- to produce artifacts for its target + -- Rules for building check-ppr, check-exact and -- check-ppr-annotations with the compiler we are going to test -- (in-tree or out-of-tree). - forM_ checkPrograms $ \(CheckProgram name progPath sourcePath mextra progPkg mod_stage mod_pkgs) -> do - name ~> need [root -/- progPath] - root -/- progPath %> \path -> do + forM_ checkPrograms $ \(CheckProgram name progPathForArch sourcePath mextra progPkg mod_stage mod_pkgs) -> do + name ~> do + tt <- getTargetTarget + need [root -/- progPathForArch (tgtArchOs tt)] + + -- HACK: we don't have a program path here and just a pattern + -- because we can't query the target before declaring the rule. + -- we specify target arch linux (no extension), then append a pattern + -- for any extension + let filePat = progPathForArch ArchOS { archOS_arch = ArchUnknown, archOS_OS = OSLinux } <> "*" + root -/- filePat %> \path -> do need [ sourcePath ] testGhc <- testCompiler <$> userSetting defaultTestArgs @@ -158,8 +183,14 @@ testRules = do depsPkgs <- mod_pkgs . packageDependencies <$> readPackageData progPkg bindir <- getBinaryDirectory testGhc test_args <- outOfTreeCompilerArgs + ht <- getHostTarget + tt <- getTargetTarget + targetPlatform <- setting TargetPlatformFull + let mkGhcProg prog + | targetPlatformTriple ht /= targetPlatformTriple tt = targetPlatform ++ "-" ++ prog + | otherwise = prog let dynPrograms = hasDynamic test_args - ghcProg <- exeSpawnPath (bindir </> "ghc" <.> exe) + ghcProg <- exeSpawnPath (bindir </> mkGhcProg "ghc" <.> exe (tgtArchOs ht)) -- FIXME: needs proper prefix! cmdExe ghcProg $ concatMap (\p -> ["-package", pkgName p]) depsPkgs ++ ["-o", top -/- path, top -/- sourcePath] ++ @@ -237,8 +268,12 @@ testEnv stg = do top <- topDirectory pythonPath <- builderPath Python -- MP: TODO wrong, should use the ccPath and ccFlags from the bindist we are testing. - ccPath <- queryTargetTarget stg (Toolchain.prgPath . Toolchain.ccProgram . Toolchain.tgtCCompiler) - ccFlags <- queryTargetTarget stg (unwords . Toolchain.prgFlags . Toolchain.ccProgram . Toolchain.tgtCCompiler) + tgt <- queryPerStageTargetSpec stg id + let ccPath = tgt.tgtCCompiler.ccProgram.prgPath + let ccFlags = unwords tgt.tgtCCompiler.ccProgram.prgFlags + let archos = tgt.tgtArchOs + let mkProgPath k = top -/- root -/- k archos + ghcFlags <- runTestGhcFlags stg let ghciFlags = ghcFlags ++ unwords [ "--interactive", "-v0", "-ignore-dot-ghci" @@ -256,14 +291,14 @@ testEnv stg = do , "TEST_HC_OPTS_INTERACTIVE" .= ghciFlags , "TEST_CC" .= ccPath , "TEST_CC_OPTS" .= ccFlags - , "CHECK_PPR" .= (top -/- root -/- checkPprProgPath) - , "CHECK_EXACT" .= (top -/- root -/- checkExactProgPath) - , "DUMP_DECLS" .= (top -/- root -/- dumpDeclsProgPath) - , "COUNT_DEPS" .= (top -/- root -/- countDepsProgPath) - , "LINT_NOTES" .= (top -/- root -/- noteLinterProgPath) - , "LINT_CODES" .= (top -/- root -/- codeLinterProgPath) - , "LINT_WHITESPACE" .= (top -/- root -/- whitespaceLinterProgPath) - , "CHANGELOG_D" .= (top -/- root -/- changelogDProgPath) + , "CHECK_PPR" .= mkProgPath checkPprProgPath + , "CHECK_EXACT" .= mkProgPath checkExactProgPath + , "DUMP_DECLS" .= mkProgPath dumpDeclsProgPath + , "COUNT_DEPS" .= mkProgPath countDepsProgPath + , "LINT_NOTES" .= mkProgPath noteLinterProgPath + , "LINT_CODES" .= mkProgPath codeLinterProgPath + , "LINT_WHITESPACE" .= mkProgPath whitespaceLinterProgPath + , "CHANGELOG_D" .= mkProgPath changelogDProgPath -- This lets us bypass the need to generate a config -- through Make, which happens in testsuite/mk/boilerplate.mk -- which is in turn included by all test 'Makefile's. @@ -301,12 +336,13 @@ absolutePathStage s p = -- We can always build a target with an intree compiler But we can only build -- targets with special support (checkPrograms) with arbitrary compilers. -- --- We need to build the dependencies if --test-have-intree-files is set. --- We should have built them already by this point, but +-- If @--test-have-intree-files@ is set, we would be build all dependencies of +-- 'isOkToBuild' targets. However, this flag doesn't mean that we have a +-- /_build/ directory available., so we mustn't add it as one of the sufficient +-- conditions. isOkToBuild :: TestArgs -> String -> Bool isOkToBuild args target = isInTreeCompiler (testCompiler args) - || testHasInTreeFiles args || target `elem` map cp_target checkPrograms -- | Build the timeout program. ===================================== hadrian/src/Settings/Builders/DeriveConstants.hs ===================================== @@ -42,8 +42,8 @@ includeCcArgs = do stage <- getStage rtsPath <- expr $ rtsBuildPath stage mconcat [ cWarnings - , prgFlags . ccProgram . tgtCCompiler <$> expr (targetStage stage) - , queryTargetTarget stage tgtUnregisterised ? arg "-DUSE_MINIINTERPRETER" + , prgFlags . ccProgram . tgtCCompiler <$> expr (perStageTarget stage) + , queryPerStageTargetSpec stage tgtUnregisterised ? arg "-DUSE_MINIINTERPRETER" , arg "-Irts" , arg "-Irts/include" , arg $ "-I" ++ rtsPath </> "include" ===================================== hadrian/src/Settings/Builders/RunTest.hs ===================================== @@ -1,4 +1,5 @@ {-# LANGUAGE TypeApplications #-} +{-# LANGUAGE NamedFieldPuns #-} module Settings.Builders.RunTest (runTestBuilderArgs , runTestGhcFlags , assertSameCompilerArgs @@ -27,7 +28,7 @@ import GHC.Toolchain.Target -- | Extra flags to send to the Haskell compiler to run tests. runTestGhcFlags :: Stage -> Action String runTestGhcFlags stage = do - unregisterised <- queryTargetTarget stage tgtUnregisterised + unregisterised <- queryPerStageTargetSpec stage tgtUnregisterised let ifMinGhcVer ver opt = do v <- ghcCanonVersion if ver <= v then pure opt @@ -112,10 +113,10 @@ inTreeCompilerArgs stg = do ways <- interpretInContext (vanillaContext ghcStage rts) getRtsWays return (dynamic `elem` ways, threaded `elem` ways) hasDynamic <- (wayUnit Dynamic) . Context.Type.way <$> (programContext stg ghc) - leadingUnderscore <- queryTargetTarget ghcStage tgtSymbolsHaveLeadingUnderscore + leadingUnderscore <- queryPerStageTargetSpec ghcStage tgtSymbolsHaveLeadingUnderscore withInterpreter <- ghcWithInterpreter ghcStage - unregisterised <- queryTargetTarget ghcStage tgtUnregisterised - tables_next_to_code <- queryTargetTarget ghcStage tgtTablesNextToCode + unregisterised <- queryPerStageTargetSpec ghcStage tgtUnregisterised + tables_next_to_code <- queryPerStageTargetSpec ghcStage tgtTablesNextToCode targetWithSMP <- targetSupportsSMP ghcStage interpForceDyn <- targetRTSLinkerOnlySupportsSharedLibs ghcStage @@ -124,17 +125,17 @@ inTreeCompilerArgs stg = do profiled <- ghcProfiled <$> flavour <*> pure ghcStage os <- queryHostTarget queryOS - arch <- queryTargetTarget ghcStage queryArch + arch <- queryPerStageTargetSpec ghcStage queryArch let codegen_arches = ["x86_64", "i386", "powerpc", "powerpc64", "powerpc64le", "aarch64", "wasm32", "riscv64", "loongarch64"] let withNativeCodeGen | unregisterised = False | arch `elem` codegen_arches = True | otherwise = False - platform <- queryTargetTarget ghcStage targetPlatformTriple - wordsize <- show @Int . (*8) <$> queryTargetTarget ghcStage (wordSize2Bytes . tgtWordSize) + platform <- queryPerStageTargetSpec ghcStage targetPlatformTriple + wordsize <- show @Int . (*8) <$> queryPerStageTargetSpec ghcStage (wordSize2Bytes . tgtWordSize) - llc_cmd <- queryTargetTarget ghcStage tgtLlc - llvm_as_cmd <- queryTargetTarget ghcStage tgtLlvmAs + llc_cmd <- queryPerStageTargetSpec ghcStage tgtLlc + llvm_as_cmd <- queryPerStageTargetSpec ghcStage tgtLlvmAs let have_llvm = allowHaveLLVM arch && all isJust [llc_cmd, llvm_as_cmd] top <- topDirectory @@ -231,6 +232,8 @@ runTestBuilderArgs = builder Testsuite ? do keepFiles <- expr (testKeepFiles <$> userSetting defaultTestArgs) + tgtArchOs <- expr $ queryPerStageTargetSpec stage tgtArchOs + accept <- expr (testAccept <$> userSetting defaultTestArgs) (acceptPlatform, acceptOS) <- expr . liftIO $ (,) <$> (maybe False (=="YES") <$> lookupEnv "PLATFORM") @@ -266,7 +269,7 @@ runTestBuilderArgs = builder Testsuite ? do , arg "-e", arg $ "config.accept=" ++ show accept , arg "-e", arg $ "config.accept_platform=" ++ show acceptPlatform , arg "-e", arg $ "config.accept_os=" ++ show acceptOS - , arg "-e", arg $ "config.exeext=" ++ quote (if null exe then "" else "."<>exe) + , arg "-e", arg $ "config.exeext=" ++ quote ("" <.> exe tgtArchOs) , arg "-e", arg $ "config.compiler_debugged=" ++ show debugAssertions , arg "-e", arg $ "config.debug_rts=" ++ show debugged ===================================== hadrian/src/Settings/Packages.hs ===================================== @@ -295,8 +295,8 @@ rtsPackageArgs = package rts ? do path <- getBuildPath top <- expr topDirectory useSystemFfi <- succStaged (buildFlag UseSystemFfi) - libdwIncludeDir <- staged (\s -> queryTargetTarget s (Lib.includePath <=< tgtRTSWithLibdw)) - libdwLibraryDir <- staged (\s -> queryTargetTarget s (Lib.libraryPath <=< tgtRTSWithLibdw)) + libdwIncludeDir <- staged (\s -> queryPerStageTargetSpec s (Lib.includePath <=< tgtRTSWithLibdw)) + libdwLibraryDir <- staged (\s -> queryPerStageTargetSpec s (Lib.libraryPath <=< tgtRTSWithLibdw)) libnumaIncludeDir <- staged (buildSetting LibnumaIncludeDir) libnumaLibraryDir <- staged (buildSetting LibnumaLibDir) libzstdIncludeDir <- staged (buildSetting LibZstdIncludeDir) @@ -412,7 +412,7 @@ rtsPackageArgs = package rts ? do , buildFlag UseLibnuma stage `cabalFlag` "libnuma" , buildFlag UseLibzstd stage `cabalFlag` "libzstd" , buildFlag StaticLibzstd stage `cabalFlag` "static-libzstd" - , queryTargetTarget stage tgtSymbolsHaveLeadingUnderscore `cabalFlag` "leading-underscore" + , queryPerStageTargetSpec stage tgtSymbolsHaveLeadingUnderscore `cabalFlag` "leading-underscore" , ghcUnreg `cabalFlag` "unregisterised" , ghcEnableTNC `cabalFlag` "tables-next-to-code" ] ===================================== testsuite/driver/testlib.py ===================================== @@ -1697,11 +1697,6 @@ async def test_common_work(name: TestName, opts, do_ways = [] config.hadrian_deps |= getTestOpts().hadrian_deps - # Skip tests which require hadrian dependencies if we are testing - # an out-of-tree compiler as Hadrian is unavailable. See #13897. - if not config.in_tree_compiler and getTestOpts().hadrian_deps - {'test:ghc'}: - do_ways = [] - # Run the required tests... for way in do_ways: if stopping(): View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/c4f25df1bfa28dc08be60df42fdd9f7... -- View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/c4f25df1bfa28dc08be60df42fdd9f7... You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications: https://gitlab.haskell.org/-/profile/notifications | Help: https://gitlab.haskell.org/help