Sven Tennie pushed to branch wip/supersven/hadrian-cross-stage3 at Glasgow Haskell Compiler / GHC

Commits:

6 changed files:

Changes:

  • .gitlab/ci.sh
    ... ... @@ -575,8 +575,14 @@ function build_hadrian() {
    575 575
       case "${CROSS_STAGE:-2}" in
    
    576 576
         2) BINDIST_TARGET="binary-dist";;
    
    577 577
         # Stage2 cross-compiler bindists are (almost) a byproduct of Stage3
    
    578
    -    # cross-compiled bindists. So, we bundle both of them
    
    579
    -    3) BINDIST_TARGET="binary-dist binary-dist-stage3";;
    
    578
    +    # cross-compiled bindists. So, we bundle both of them when the Stage3
    
    579
    +    # bindist is built.
    
    580
    +    3)
    
    581
    +      BINDIST_TARGET="binary-dist binary-dist-stage3"
    
    582
    +      if [[ -z "${BIN_DIST_NAME_STAGE3:-}" ]]; then
    
    583
    +        fail "CROSS_STAGE=3 requires BIN_DIST_NAME_STAGE3 to be set"
    
    584
    +      fi
    
    585
    +      ;;
    
    580 586
         *) fail "Unknown CROSS_STAGE, must be 2 or 3";;
    
    581 587
       esac
    
    582 588
     
    
    ... ... @@ -592,9 +598,6 @@ function build_hadrian() {
    592 598
               run_hadrian test:all_deps $BINDIST_TARGET
    
    593 599
               mv _build/bindist/ghc*.tar.xz "$BIN_DIST_NAME.tar.xz"
    
    594 600
               if [[ "${CROSS_STAGE:-2}" == "3" ]]; then
    
    595
    -            if [[ -z "${BIN_DIST_NAME_STAGE3:-}" ]]; then
    
    596
    -              fail "CROSS_STAGE=3 requires BIN_DIST_NAME_STAGE3 to be set"
    
    597
    -            fi
    
    598 601
                 mv _build/bindist-stage3/ghc*.tar.xz "$BIN_DIST_NAME_STAGE3.tar.xz"
    
    599 602
               fi
    
    600 603
               ;;
    
    ... ... @@ -682,11 +685,6 @@ function test_hadrian() {
    682 685
       # If we have set CROSS_EMULATOR, then can't test using normal testsuite.
    
    683 686
       elif [ -n "${CROSS_EMULATOR:-}" ] && [[ "${CROSS_TARGET:-}" != *"wasm"* ]]; then
    
    684 687
         local instdir="$TOP/_build/install"
    
    685
    -    # The stage-2 cross bindist has target-triple-prefixed binaries and runs
    
    686
    -    # natively on the host. Override the global cross_prefix (which is empty
    
    687
    -    # for CROSS_STAGE=3 because the stage-3 bindist has unprefixed binaries)
    
    688
    -    # so that we test the stage-2 cross compiler consistently.
    
    689
    -    local cross_prefix="$target_triple-"
    
    690 688
         local test_compiler="$instdir/bin/${cross_prefix}ghc$exe"
    
    691 689
         install_bindist _build/bindist/ghc-*/ "$instdir"
    
    692 690
         echo 'main = putStrLn "hello world"' > expected
    
    ... ... @@ -708,6 +706,26 @@ function test_hadrian() {
    708 706
         # ---
    
    709 707
         # > main = putStrLn "hello world"
    
    710 708
         run diff -w expected actual
    
    709
    +
    
    710
    +    if [[ "${CROSS_STAGE:-2}" == "3" ]]; then
    
    711
    +      local stage3_dir
    
    712
    +      stage3_dir="$(echo _build/bindist-stage3/ghc-*/)"
    
    713
    +      local stage3_ghc="$stage3_dir/bin/ghc$exe"
    
    714
    +
    
    715
    +      info "Smoke-testing stage3 compiler..."
    
    716
    +      file "$stage3_ghc"
    
    717
    +      run ${CROSS_EMULATOR} "$stage3_ghc" --info
    
    718
    +
    
    719
    +      run ${CROSS_EMULATOR} "$stage3_ghc" -package ghc "$TOP/.gitlab/hello.hs" -o hello-stage3
    
    720
    +
    
    721
    +      if [[ "${CROSS_TARGET:-no_cross_target}" =~ "mingw" ]]; then
    
    722
    +        ${CROSS_EMULATOR:-} ./hello-stage3.exe > actual-stage3
    
    723
    +      else
    
    724
    +        ${CROSS_EMULATOR:-} ./hello-stage3 > actual-stage3
    
    725
    +      fi
    
    726
    +
    
    727
    +      run diff -w expected actual-stage3
    
    728
    +    fi
    
    711 729
       elif [[ -n "${REINSTALL_GHC:-}" ]]; then
    
    712 730
         run_hadrian \
    
    713 731
           test \
    
    ... ... @@ -935,8 +953,7 @@ function clean() {
    935 953
       #
    
    936 954
       # The exclude list are the artifacts that we do expect to be
    
    937 955
       # uploaded. Keep in sync with `jobArtifacts` in
    
    938
    -  # `.gitlab/generate-ci/gen_ci.hs`! The `ghc-*.tar.xz` pattern covers both
    
    939
    -  # stage2/cross and stage3/target bindists.
    
    956
    +  # `.gitlab/generate-ci/gen_ci.hs`!
    
    940 957
       if [[ "${CI_DISPOSABLE_ENVIRONMENT:-}" != true ]]; then
    
    941 958
         git submodule --quiet foreach --recursive git clean -xdfq
    
    942 959
         git clean -xdfq \
    
    ... ... @@ -1046,15 +1063,12 @@ case "$(uname)" in
    1046 1063
       *) fail "uname $(uname) is not supported" ;;
    
    1047 1064
     esac
    
    1048 1065
     
    
    1049
    -cross_prefix=""
    
    1050 1066
     if [ -n "${CROSS_TARGET:-}" ]; then
    
    1051
    -  info "Cross-compiling for $CROSS_TARGET... (stage: $CROSS_STAGE)"
    
    1067
    +  info "Cross-compiling for $CROSS_TARGET..."
    
    1052 1068
       target_triple="$CROSS_TARGET"
    
    1053
    -  # Stage3 native GHC runs on the target itself, so no cross prefix.
    
    1054
    -  # CROSS_STAGE is either 2 (host != target) or 3 (host == target)
    
    1055
    -  if [ "${CROSS_STAGE:-2}" = "2" ]; then
    
    1056
    -    cross_prefix="$target_triple-"
    
    1057
    -  fi
    
    1069
    +  cross_prefix="$target_triple-"
    
    1070
    +else
    
    1071
    +  cross_prefix=""
    
    1058 1072
     fi
    
    1059 1073
     
    
    1060 1074
     echo "Branch name ${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME:-}"
    

  • .gitlab/generate-ci/gen_ci.hs
    ... ... @@ -159,7 +159,7 @@ data BuildConfig
    159 159
                     , withNuma       :: Bool
    
    160 160
                     , withZstd       :: Bool
    
    161 161
                     , crossTarget    :: Maybe String
    
    162
    -                , crossStage     :: Maybe Int
    
    162
    +                , crossStage     :: Maybe FinalCrossStage
    
    163 163
                     , crossEmulator  :: CrossEmulator
    
    164 164
                     , configureWrapper :: Maybe String
    
    165 165
                     , fullyStatic    :: Bool
    
    ... ... @@ -275,14 +275,26 @@ static = vanilla { fullyStatic = True }
    275 275
     staticNativeInt :: BuildConfig
    
    276 276
     staticNativeInt = static { bignumBackend = Native }
    
    277 277
     
    
    278
    --- | cross-compiler (build == host, host /= target)
    
    279
    -stage2CrossConfig :: String       -- ^ target triple
    
    278
    +-- | The final stage for which binary distrubutions should be built
    
    279
    +--
    
    280
    +-- `Stage2` builds a cross-compiler (build == host, host /= target). `Stage3`
    
    281
    +-- implies `Stage2` and additionally builds a cross-compiled compiler (build /=
    
    282
    +-- host, host == target).
    
    283
    +data FinalCrossStage = Stage2 | Stage3
    
    284
    +  deriving (Eq, Ord)
    
    285
    +
    
    286
    +crossStageToInt :: FinalCrossStage -> Int
    
    287
    +crossStageToInt Stage2 = 2
    
    288
    +crossStageToInt Stage3 = 3
    
    289
    +
    
    290
    +crossConfig :: String       -- ^ target triple
    
    280 291
                 -> CrossEmulator -- ^ emulator for testing
    
    281 292
                 -> Maybe String -- ^ Configure wrapper
    
    293
    +            -> FinalCrossStage -- ^ final stage to build
    
    282 294
                 -> BuildConfig
    
    283
    -stage2CrossConfig triple emulator configure_wrapper =
    
    295
    +crossConfig triple emulator configure_wrapper crossStage =
    
    284 296
         vanilla { crossTarget = Just triple
    
    285
    -            , crossStage  = Just 2
    
    297
    +            , crossStage  = Just crossStage
    
    286 298
                 , crossEmulator = emulator
    
    287 299
                 , configureWrapper = configure_wrapper
    
    288 300
                 }
    
    ... ... @@ -895,7 +907,7 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} }
    895 907
           [ opsysVariables arch opsys
    
    896 908
           , "TEST_ENV" =: testEnv arch opsys buildConfig
    
    897 909
           , "BIN_DIST_NAME" =: binDistName arch opsys buildConfig
    
    898
    -      , if crossStage buildConfig == Just 3
    
    910
    +      , if crossStage buildConfig == Just Stage3
    
    899 911
                 then "BIN_DIST_NAME_STAGE3" =: binDistNameStage3 arch opsys buildConfig
    
    900 912
                 else mempty
    
    901 913
           , "BUILD_FLAVOUR" =: flavourString jobFlavour
    
    ... ... @@ -904,7 +916,7 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} }
    904 916
           , "INSTALL_CONFIGURE_ARGS" =: "--enable-strict-ghc-toolchain-check"
    
    905 917
           , maybe mempty ("CONFIGURE_WRAPPER" =:) (configureWrapper buildConfig)
    
    906 918
           , maybe mempty ("CROSS_TARGET" =:) (crossTarget buildConfig)
    
    907
    -      , maybe mempty (("CROSS_STAGE" =:) . show) (crossStage buildConfig)
    
    919
    +      , maybe mempty (("CROSS_STAGE" =:) . show . crossStageToInt) (crossStage buildConfig)
    
    908 920
           , case crossEmulator buildConfig of
    
    909 921
               NoEmulator
    
    910 922
                 -- we need an emulator but it isn't set. Won't run the testsuite
    
    ... ... @@ -938,8 +950,8 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} }
    938 950
         trim = dropWhileEnd isSpace . dropWhile isSpace
    
    939 951
     
    
    940 952
         stage3Artifacts
    
    941
    -      | crossStage buildConfig == Just 3
    
    942
    -      = [binDistNameStage3 arch opsys buildConfig ++ ".tar.xz"]
    
    953
    +      | crossStage buildConfig == Just Stage3 =
    
    954
    +          [binDistNameStage3 arch opsys buildConfig ++ ".tar.xz"]
    
    943 955
           | otherwise = []
    
    944 956
     
    
    945 957
         -- Keep in sync with the exclude list in `function clean()` in
    
    ... ... @@ -947,9 +959,10 @@ job arch opsys buildConfig = NamedJob { name = jobName, jobInfo = Job {..} }
    947 959
         jobArtifacts = Artifacts
    
    948 960
           { junitReport = "junit.xml"
    
    949 961
           , expireIn = "2 weeks"
    
    950
    -      , artifactPaths = stage3Artifacts ++ [binDistName arch opsys buildConfig ++ ".tar.xz"
    
    962
    +      , artifactPaths = [binDistName arch opsys buildConfig ++ ".tar.xz"
    
    951 963
                             ,"junit.xml"
    
    952 964
                             ,"unexpected-test-output.tar.gz"]
    
    965
    +                        ++ stage3Artifacts  
    
    953 966
           , artifactsWhen = ArtifactsAlways
    
    954 967
           }
    
    955 968
     
    
    ... ... @@ -1308,13 +1321,13 @@ alpine_aarch64 = [
    1308 1321
     cross_jobs :: [JobGroup Job]
    
    1309 1322
     cross_jobs = [
    
    1310 1323
         -- x86 -> aarch64
    
    1311
    -    validateBuilds Amd64 (Linux Debian13) (stage2CrossConfig "aarch64-linux-gnu" (Emulator "qemu-aarch64 -L /usr/aarch64-linux-gnu") Nothing)
    
    1324
    +    validateBuilds Amd64 (Linux Debian13) (crossConfig "aarch64-linux-gnu" (Emulator "qemu-aarch64 -L /usr/aarch64-linux-gnu") Nothing Stage2)
    
    1312 1325
     
    
    1313 1326
         -- x86_64 (build) -> riscv64 (host/target)
    
    1314
    -  , addValidateRule RiscV (validateBuilds Amd64 (Linux Debian13Riscv) (stage2CrossConfig "riscv64-linux-gnu" (Emulator "qemu-riscv64 -L /usr/riscv64-linux-gnu") Nothing) { crossStage = Just 3 })
    
    1327
    +  , addValidateRule RiscV (validateBuilds Amd64 (Linux Debian13Riscv) (crossConfig "riscv64-linux-gnu" (Emulator "qemu-riscv64 -L /usr/riscv64-linux-gnu") Nothing Stage3))
    
    1315 1328
     
    
    1316 1329
         -- x86_64 -> loongarch64
    
    1317
    -  , addValidateRule LoongArch64 (validateBuilds Amd64 (Linux Ubuntu2404LoongArch64) (stage2CrossConfig "loongarch64-linux-gnu" (Emulator "qemu-loongarch64 -L /usr/loongarch64-linux-gnu") Nothing))
    
    1330
    +  , addValidateRule LoongArch64 (validateBuilds Amd64 (Linux Ubuntu2404LoongArch64) (crossConfig "loongarch64-linux-gnu" (Emulator "qemu-loongarch64 -L /usr/loongarch64-linux-gnu") Nothing Stage2))
    
    1318 1331
     
    
    1319 1332
         -- Javascript
    
    1320 1333
       , addValidateRule JSBackend (validateBuilds Amd64 (Linux Debian11Js) javascriptConfig)
    
    ... ... @@ -1335,7 +1348,7 @@ cross_jobs = [
    1335 1348
             (validateBuilds AArch64 (Linux Debian12Wine) (winAarch64Config {llvmBootstrap = True}))
    
    1336 1349
       ]
    
    1337 1350
       where
    
    1338
    -    javascriptConfig = (stage2CrossConfig "javascript-unknown-ghcjs" (NoEmulatorNeeded TimeoutIncrease) (Just "emconfigure"))
    
    1351
    +    javascriptConfig = (crossConfig "javascript-unknown-ghcjs" (NoEmulatorNeeded TimeoutIncrease) (Just "emconfigure") Stage2)
    
    1339 1352
                              { bignumBackend = Native }
    
    1340 1353
     
    
    1341 1354
         makeWinArmJobs = modifyJobs
    
    ... ... @@ -1374,7 +1387,7 @@ cross_jobs = [
    1374 1387
                 llvm_prefix = "/opt/llvm-mingw-linux/bin/aarch64-w64-mingw32-"
    
    1375 1388
                 cflags = "-fuse-ld=" ++ llvm_prefix ++ "ld --rtlib=compiler-rt"
    
    1376 1389
     
    
    1377
    -    winAarch64Config = (stage2CrossConfig "aarch64-unknown-mingw32" (Emulator "/opt/wine-arm64ec-msys2-deb12/bin/wine") Nothing)
    
    1390
    +    winAarch64Config = (crossConfig "aarch64-unknown-mingw32" (Emulator "/opt/wine-arm64ec-msys2-deb12/bin/wine") Nothing Stage2)
    
    1378 1391
                              { bignumBackend = Native }
    
    1379 1392
     
    
    1380 1393
         make_wasm_jobs cfg =
    
    ... ... @@ -1387,7 +1400,7 @@ cross_jobs = [
    1387 1400
             $ addValidateRule WasmBackend $ validateBuilds Amd64 (Linux AlpineWasm) cfg
    
    1388 1401
     
    
    1389 1402
         wasm_build_config =
    
    1390
    -      (stage2CrossConfig "wasm32-wasi" (NoEmulatorNeeded NoTimeoutIncrease) Nothing)
    
    1403
    +      (crossConfig "wasm32-wasi" (NoEmulatorNeeded NoTimeoutIncrease) Nothing Stage2)
    
    1391 1404
             { hostFullyStatic = True
    
    1392 1405
             , buildFlavour    = Release -- TODO: This needs to be validate but wasm backend doesn't pass yet
    
    1393 1406
             , textWithSIMDUTF = True
    
    ... ... @@ -1458,9 +1471,8 @@ platform_mapping = Map.map go combined_result
    1458 1471
     
    
    1459 1472
         process sel =
    
    1460 1473
           Map.fromListWith combine
    
    1461
    -      [ (mkPlatform a o, j)
    
    1474
    +      [ (uncurry mkPlatform (jobPlatform (jobInfo j)), j)
    
    1462 1475
           | (sel -> Just j) <- job_groups
    
    1463
    -      , let (a, o) = jobPlatform (jobInfo j)
    
    1464 1476
           ]
    
    1465 1477
     
    
    1466 1478
         vs = process v
    

  • .gitlab/jobs.yaml
    ... ... @@ -2748,10 +2748,10 @@
    2748 2748
         "artifacts": {
    
    2749 2749
           "expire_in": "8 weeks",
    
    2750 2750
           "paths": [
    
    2751
    -        "ghc-x86_64-linux-deb13-riscv-cross_riscv64-linux-gnu-stage3-validate.tar.xz",
    
    2752 2751
             "ghc-x86_64-linux-deb13-riscv-cross_riscv64-linux-gnu-validate.tar.xz",
    
    2753 2752
             "junit.xml",
    
    2754
    -        "unexpected-test-output.tar.gz"
    
    2753
    +        "unexpected-test-output.tar.gz",
    
    2754
    +        "ghc-x86_64-linux-deb13-riscv-cross_riscv64-linux-gnu-stage3-validate.tar.xz"
    
    2755 2755
           ],
    
    2756 2756
           "reports": {
    
    2757 2757
             "junit": "junit.xml"
    
    ... ... @@ -6726,10 +6726,10 @@
    6726 6726
         "artifacts": {
    
    6727 6727
           "expire_in": "2 weeks",
    
    6728 6728
           "paths": [
    
    6729
    -        "ghc-x86_64-linux-deb13-riscv-cross_riscv64-linux-gnu-stage3-validate.tar.xz",
    
    6730 6729
             "ghc-x86_64-linux-deb13-riscv-cross_riscv64-linux-gnu-validate.tar.xz",
    
    6731 6730
             "junit.xml",
    
    6732
    -        "unexpected-test-output.tar.gz"
    
    6731
    +        "unexpected-test-output.tar.gz",
    
    6732
    +        "ghc-x86_64-linux-deb13-riscv-cross_riscv64-linux-gnu-stage3-validate.tar.xz"
    
    6733 6733
           ],
    
    6734 6734
           "reports": {
    
    6735 6735
             "junit": "junit.xml"
    

  • hadrian/src/BindistConfig.hs
    ... ... @@ -3,11 +3,11 @@ module BindistConfig where
    3 3
     import Stage
    
    4 4
     import Oracles.Flag
    
    5 5
     import Expression
    
    6
    +
    
    6 7
     data BindistConfig = BindistConfig { library_stage :: Stage -- ^ The stage compiler which builds the libraries
    
    7 8
                                        , executable_stage :: Stage -- ^ The stage compiler which builds the executables
    
    8 9
                                        }
    
    9 10
     
    
    10
    -
    
    11 11
     -- | A bindist for when the host = target, non cross-compilation setting.
    
    12 12
     -- Both the libraries and final executables are built with stage1 compiler.
    
    13 13
     normalBindist :: BindistConfig
    

  • hadrian/src/Rules/CabalReinstall.hs
    ... ... @@ -67,14 +67,13 @@ cabalBuildRules = do
    67 67
     
    
    68 68
             let cabal_package_db = cwd -/- root -/- "stage-cabal" -/- "dist-newstyle" -/- "packagedb" -/- "ghc-" ++ version
    
    69 69
     
    
    70
    -        executableStage <- executable_stage <$> implicitBindistConfig
    
    71 70
             forM_ bin_targets $ \(bin_pkg,_bin_path) -> do
    
    72 71
                 let pgmName pkg
    
    73 72
                       | pkg == ghc    = "ghc"
    
    74 73
                       | pkg == hpcBin = "hpc"
    
    75 74
                       | otherwise     = pkgName pkg
    
    76 75
                 let cabal_bin_out = work_dir -/- "cabal-bin" -/- (pgmName bin_pkg)
    
    77
    -            needed_wrappers <- pkgToWrappers executableStage bin_pkg
    
    76
    +            needed_wrappers <- pkgToWrappers Stage2 bin_pkg
    
    78 77
                 forM_ needed_wrappers $ \wrapper_name -> do
    
    79 78
                   let wrapper_prefix = unlines
    
    80 79
                         ["#!/usr/bin/env sh"
    
    ... ... @@ -86,7 +85,7 @@ cabalBuildRules = do
    86 85
                         ,"export GHC_PACKAGE_PATH="++show cabal_package_db++":"
    
    87 86
                         ]
    
    88 87
                       output_file = outputDir -/- wrapper_name
    
    89
    -              wrapper_content <- wrapper executableStage wrapper_name
    
    88
    +              wrapper_content <- wrapper Stage2 wrapper_name
    
    90 89
                   writeFile' output_file (wrapper_prefix ++ wrapper_content)
    
    91 90
                   makeExecutable output_file
    
    92 91
                   pure ()
    

  • hadrian/src/Rules/Generate.hs
    ... ... @@ -423,7 +423,7 @@ bindistRules = do
    423 423
     
    
    424 424
         , interpolateVar "HostOS_CPP" $ fmap cppify $ interp $ queryHost queryOS
    
    425 425
     
    
    426
    -    -- Stage2 always targets the final  architecture. Thus, we can use a
    
    426
    +    -- Stage2 always targets the final architecture. Thus, we can use a
    
    427 427
         -- constant stage here.
    
    428 428
         , interpolateVar "TargetPlatform" $ getTarget Stage2 targetPlatformTriple
    
    429 429
         , interpolateVar "TargetPlatform_CPP" $ cppify <$> getTarget Stage2 targetPlatformTriple