Marge Bot pushed to branch wip/marge_bot_batch_merge_job at Glasgow Haskell Compiler / GHC

Commits:

18 changed files:

Changes:

  • .gitlab-ci.yml
    ... ... @@ -137,15 +137,37 @@ workflow:
    137 137
     .lint:
    
    138 138
       stage: tool-lint
    
    139 139
       tags:
    
    140
    -    - lint
    
    141
    -  before_script:
    
    142
    -    # Allow Git to read from the runner-owned checkout. This is needed
    
    143
    -    # for example by the lint-author job.
    
    144
    -    - git config --global --add safe.directory "$CI_PROJECT_DIR"
    
    140
    +    - lint-anyarch
    
    141
    +  image: debian:13
    
    145 142
       rules:
    
    146 143
         - *drafts-can-fail-lint
    
    147 144
         - when: always
    
    148 145
     
    
    146
    +.install-ghc-toolchain: &install-ghc-toolchain
    
    147
    +  - apt update
    
    148
    +  - apt install -qy build-essential curl libffi-dev libffi8 libgmp-dev libgmp10 libncurses-dev libncurses6 libtinfo6 pkg-config python3 cmake git happy alex autoconf automake python3 libgmp-dev libnuma-dev libncurses-dev
    
    149
    +  - curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | BOOTSTRAP_HASKELL_NONINTERACTIVE=1 BOOTSTRAP_HASKELL_MINIMAL=1 sh
    
    150
    +  - source /root/.ghcup/env
    
    151
    +  - ghcup --no-verbose install ghc --set --install-targets 'install_bin install_lib update_package_db' '9.10'
    
    152
    +  - ghcup --no-verbose install cabal --set
    
    153
    +  - ghc --info
    
    154
    +  - cabal --version
    
    155
    +
    
    156
    +# linting but we need a boot toolchain
    
    157
    +.lint-ghc-toolchain:
    
    158
    +  extends: .lint
    
    159
    +  before_script:
    
    160
    +    - *install-ghc-toolchain
    
    161
    +    - git config --global --add safe.directory "$CI_PROJECT_DIR"
    
    162
    +
    
    163
    +# linting but we need python
    
    164
    +.lint-python:
    
    165
    +  extends: .lint
    
    166
    +  before_script:
    
    167
    +    - apt update
    
    168
    +    - apt install -qy python3 mypy shellcheck git
    
    169
    +    - git config --global --add safe.directory "$CI_PROJECT_DIR"
    
    170
    +
    
    149 171
     .nightly: &nightly
    
    150 172
       variables:
    
    151 173
         XZ_OPT: "-9"
    
    ... ... @@ -217,11 +239,11 @@ not-interruptible:
    217 239
       stage: not-interruptible
    
    218 240
       script: "true"
    
    219 241
       interruptible: false
    
    220
    -  image: "debian:13"
    
    242
    +  image: debian:13
    
    221 243
       variables:
    
    222 244
         GIT_STRATEGY: none
    
    223 245
       tags:
    
    224
    -    - lint
    
    246
    +    - lint-anyarch
    
    225 247
       rules:
    
    226 248
     #   - if: '$CI_COMMIT_BRANCH == "master"'
    
    227 249
     #      when: always
    
    ... ... @@ -242,7 +264,6 @@ include: '.gitlab/jobs.yaml'
    242 264
     
    
    243 265
     ghc-linters:
    
    244 266
       stage: tool-lint
    
    245
    -  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb13:$DOCKER_REV"
    
    246 267
       extends: .lint-params
    
    247 268
       variables:
    
    248 269
         BUILD_FLAVOUR: default
    
    ... ... @@ -258,16 +279,11 @@ ghc-linters:
    258 279
     # Skipped if the MR has the ~"no-changelog" label.
    
    259 280
     lint-changelog:
    
    260 281
       stage: tool-lint
    
    261
    -  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb13:$DOCKER_REV"
    
    262
    -  extends: .lint
    
    282
    +  extends: .lint-ghc-toolchain
    
    263 283
       variables:
    
    264 284
         CHANGELOG_EXPECT_MR: "$CI_MERGE_REQUEST_IID"
    
    265 285
         GIT_DEPTH: 1
    
    266 286
         GIT_SUBMODULE_STRATEGY: none
    
    267
    -  before_script:
    
    268
    -    - export PATH="/opt/toolchain/bin:$PATH"
    
    269
    -    # workaround for docker permissions
    
    270
    -    - sudo chown ghc:ghc -R .
    
    271 287
       script:
    
    272 288
         - .gitlab/ci.sh lint_changelog
    
    273 289
       dependencies: []
    
    ... ... @@ -279,42 +295,30 @@ lint-changelog:
    279 295
     
    
    280 296
     # Run mypy Python typechecker on linter scripts.
    
    281 297
     lint-linters:
    
    282
    -  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
    
    283
    -  extends: .lint
    
    298
    +  extends: .lint-python
    
    284 299
       variables:
    
    285 300
         GIT_DEPTH: 1
    
    286 301
         GIT_SUBMODULE_STRATEGY: none
    
    287
    -  before_script:
    
    288
    -    # workaround for docker permissions
    
    289
    -    - sudo chown ghc:ghc -R .
    
    290 302
       script:
    
    291 303
         - mypy testsuite/tests/linters/regex-linters/*.py
    
    292 304
       dependencies: []
    
    293 305
     
    
    294 306
     # Check that .T files all parse by listing broken tests.
    
    295 307
     lint-testsuite:
    
    296
    -  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb13:$DOCKER_REV"
    
    297
    -  extends: .lint
    
    308
    +  extends: .lint-ghc-toolchain
    
    298 309
       variables:
    
    299 310
         GIT_DEPTH: 1
    
    300 311
         GIT_SUBMODULE_STRATEGY: none
    
    301
    -  before_script:
    
    302
    -    # workaround for docker permissions
    
    303
    -    - sudo chown ghc:ghc -R .
    
    304 312
       script:
    
    305
    -    - make -Ctestsuite list_broken TEST_HC=$GHC
    
    313
    +    - make -Ctestsuite list_broken TEST_HC=$(which ghc)
    
    306 314
       dependencies: []
    
    307 315
     
    
    308 316
     # Run mypy Python typechecker on testsuite driver
    
    309 317
     typecheck-testsuite:
    
    310
    -  image: "registry.gitlab.haskell.org/ghc/ci-images/linters:$DOCKER_REV"
    
    311
    -  extends: .lint
    
    318
    +  extends: .lint-python
    
    312 319
       variables:
    
    313 320
         GIT_DEPTH: 1
    
    314 321
         GIT_SUBMODULE_STRATEGY: none
    
    315
    -  before_script:
    
    316
    -    # workaround for docker permissions
    
    317
    -    - sudo chown ghc:ghc -R .
    
    318 322
       script:
    
    319 323
         - mypy testsuite/driver/runtests.py
    
    320 324
       dependencies: []
    
    ... ... @@ -323,7 +327,6 @@ typecheck-testsuite:
    323 327
     # accommodate, e.g., haddock changes not yet upstream) but not on `master` or
    
    324 328
     # Marge jobs.
    
    325 329
     .lint-submods:
    
    326
    -  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb13:$DOCKER_REV"
    
    327 330
       extends: .lint-params
    
    328 331
       variables:
    
    329 332
         BUILD_FLAVOUR: default
    
    ... ... @@ -341,13 +344,9 @@ typecheck-testsuite:
    341 344
     # Check that commits in an MR were not authored with the identity that CI uses
    
    342 345
     # when it needs to create commits itself. This only needs Git history.
    
    343 346
     lint-author:
    
    344
    -  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb13:$DOCKER_REV"
    
    345
    -  extends: .lint
    
    347
    +  extends: .lint-python
    
    346 348
       variables:
    
    347 349
         GIT_SUBMODULE_STRATEGY: none
    
    348
    -  before_script:
    
    349
    -    # workaround for docker permissions
    
    350
    -    - sudo chown ghc:ghc -R .
    
    351 350
       script:
    
    352 351
         - git fetch "$CI_MERGE_REQUEST_PROJECT_URL" $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
    
    353 352
         - base="$(git merge-base FETCH_HEAD $CI_COMMIT_SHA)"
    
    ... ... @@ -425,12 +424,9 @@ lint-submods-branch:
    425 424
     
    
    426 425
     .lint-params:
    
    427 426
       needs: []
    
    428
    -  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb13:$DOCKER_REV"
    
    429 427
       extends: .lint
    
    430 428
       before_script:
    
    431
    -    - export PATH="/opt/toolchain/bin:$PATH"
    
    432
    -    # workaround for docker permissions
    
    433
    -    - sudo chown ghc:ghc -R .
    
    429
    +    - *install-ghc-toolchain
    
    434 430
         - .gitlab/ci.sh setup
    
    435 431
       after_script:
    
    436 432
         - .gitlab/ci.sh save_cache
    
    ... ... @@ -448,18 +444,13 @@ lint-submods-branch:
    448 444
     
    
    449 445
     hadrian-ghc-in-ghci:
    
    450 446
       stage: quick-build
    
    447
    +  extends: .lint-ghc-toolchain
    
    451 448
       needs:
    
    452 449
         - job: lint-linters
    
    453 450
         - job: lint-submods
    
    454 451
           optional: true
    
    455
    -  image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb13:$DOCKER_REV"
    
    456
    -  before_script:
    
    457
    -    # workaround for docker permissions
    
    458
    -    - sudo chown ghc:ghc -R .
    
    459 452
       variables:
    
    460 453
         GHC_FLAGS: -Werror -Wwarn=unused-imports
    
    461
    -  tags:
    
    462
    -    - x86_64-linux
    
    463 454
       script:
    
    464 455
         - git clean -xdf && git submodule foreach git clean -xdf
    
    465 456
         - . .gitlab/ci.sh setup
    
    ... ... @@ -493,9 +484,6 @@ hadrian-multi:
    493 484
           optional: true
    
    494 485
       dependencies: null
    
    495 486
       image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb13:$DOCKER_REV"
    
    496
    -  before_script:
    
    497
    -    # workaround for docker permissions
    
    498
    -    - sudo chown ghc:ghc -R .
    
    499 487
       variables:
    
    500 488
         GHC_FLAGS: "-Wwarn=deprecations -Wwarn=unused-imports"
    
    501 489
         CONFIGURE_ARGS: --enable-bootstrap-with-devel-snapshot
    
    ... ... @@ -544,6 +532,7 @@ stack-hadrian-build:
    544 532
       extends: hadrian-ghc-in-ghci
    
    545 533
       stage: quick-build
    
    546 534
       script:
    
    535
    +    - ghcup --no-verbose install stack --set
    
    547 536
         - . .gitlab/ci.sh setup
    
    548 537
         - . .gitlab/ci.sh configure
    
    549 538
         - hadrian/build-stack --version
    
    ... ... @@ -1050,9 +1039,6 @@ perf:
    1050 1039
       image: "registry.gitlab.haskell.org/ghc/ci-images/x86_64-linux-deb13:$DOCKER_REV"
    
    1051 1040
       tags:
    
    1052 1041
         - x86_64-linux-perf
    
    1053
    -  before_script:
    
    1054
    -    # workaround for docker permissions
    
    1055
    -    - sudo chown ghc:ghc -R .
    
    1056 1042
       script:
    
    1057 1043
         - root=$(pwd)/ghc
    
    1058 1044
         - |
    

  • changelog.d/24450
    1
    +section: rts
    
    2
    +synopsis:
    
    3
    +  Fix a hang when a program is started with a standard file descriptor closed
    
    4
    +description:
    
    5
    +  A process started with stdout or stderr closed -- ``prog >&-`` from a shell
    
    6
    +  does exactly that -- left that descriptor number free for the runtime to
    
    7
    +  take for its own use.  The ``Handle`` for the stream, built later when the
    
    8
    +  program first wrote to it, then took over a descriptor belonging to the
    
    9
    +  runtime: the program wrote its output into the runtime's own plumbing and
    
    10
    +  hung, with no error reported anywhere.  The runtime now opens ``/dev/null``
    
    11
    +  onto whichever of the descriptors 0, 1 and 2 are closed when it starts, so
    
    12
    +  such a stream is connected to ``/dev/null`` instead.
    
    13
    +mrs: !16710
    
    14
    +issues: #24450

  • changelog.d/fix-parallel-gc-selector-race-27477
    1 1
     section: rts
    
    2 2
     synopsis: Fix a parallel GC race on weakly-ordered architectures (AArch64) that could crash or silently corrupt the heap
    
    3 3
     description:
    
    4
    -  On weakly-ordered architectures such as AArch64 programs using
    
    4
    +  On weakly-ordered architectures such as AArch64, programs
    
    5 5
       could crash during parallel garbage collection (segmentation fault
    
    6 6
       or ``internal error: eval_thunk_selector: strange selectee <n>``) or
    
    7 7
       continue running with silently corrupted heap data, due to a missing
    

  • changelog.d/hadrian-persist-flavour
    1
    +section: packaging
    
    2
    +synopsis: Hadrian now persists the flavour name in the build root. When
    
    3
    +  rebuilding, the previously used flavour is automatically restored, removing
    
    4
    +  the need to re-specify it on every invocation and avoiding accidental build
    
    5
    +  root invalidation.
    
    6
    +issues: #20650
    
    7
    +mrs: !7017

  • changelog.d/unit-index
    1 1
     section: compiler
    
    2 2
     synopsis: Use global ``UnitIndex`` to deduplicate ``UnitInfo``s over multiple home units
    
    3
    -issues: #27500 #26423 #27748 #27834
    
    4
    -mrs: !16115 !16598 !16743
    
    3
    +issues: #27500 #26423 #27748 #27834 #27835
    
    4
    +mrs: !16115 !16598 !16743 !16748
    
    5 5
     
    
    6 6
     description: {
    
    7 7
         The ``UnitState`` used to be duplicated for all ``HomeUnitEnv``, not sharing any of the ``UnitInfo``s.
    

  • compiler/GHC/Driver/Backpack.hs
    ... ... @@ -441,9 +441,11 @@ addInMemoryDatabase dflags u = do
    441 441
               }
    
    442 442
         let uic = hscUIC hsc_env
    
    443 443
         liftIO $ cacheExternalUnitDatabase uic newdb
    
    444
    -    -- added at the end because ordering matters
    
    444
    +    -- Ordering matters here! We prepend the new unit database, because 'packageDBFlags'
    
    445
    +    -- works like a stack, where what's on top takes precedence over what's below (same as right-to-left when given at the CLI).
    
    446
    +    -- The new db needs highest precedence to overwrite any other in-memory units that potentially occur in the dbs below it.
    
    445 447
         pure dflags
    
    446
    -          { packageDBFlags = packageDBFlags dflags ++ [PackageDB (PkgDbPath (unitDatabasePath newdb))]
    
    448
    +          { packageDBFlags = PackageDB (PkgDbPath (unitDatabasePath newdb)) : packageDBFlags dflags
    
    447 449
               }
    
    448 450
     
    
    449 451
     -- | Register a new virtual unit database containing a single unit
    

  • distrib/configure.ac.in
    ... ... @@ -103,7 +103,7 @@ AC_ARG_ENABLE(distro-toolchain,
    103 103
     AC_SUBST([EnableDistroToolchain])
    
    104 104
     
    
    105 105
     if test "$HostOS" = "mingw32" -a "$EnableDistroToolchain" = "NO"; then
    
    106
    -  FP_SETUP_WINDOWS_TOOLCHAIN([$hardtop/mingw/], [\$\$topdir/../mingw/])
    
    106
    +  FP_SETUP_WINDOWS_TOOLCHAIN([$hardtop/mingw/], [\$topdir/../mingw/])
    
    107 107
     fi
    
    108 108
     
    
    109 109
     
    

  • hadrian/doc/flavours.md
    ... ... @@ -3,7 +3,14 @@
    3 3
     Hadrian supports a few predefined _build flavours_, i.e. collections of build
    
    4 4
     settings that fully define a GHC build (see `src/Flavour.hs`). Users can add their
    
    5 5
     own build flavours if need be, as described
    
    6
    -[here](https://gitlab.haskell.org/ghc/ghc/blob/master/hadrian/doc/user-settings.md#build-flavour).
    
    6
    +[here](https://gitlab.haskell.org/ghc/ghc/blob/master/hadrian/doc/user-settings.md#build-flavour) or modify existing flavours using the flavour transformers notion seem below.
    
    7
    +
    
    8
    +The flavour to be used for a build root can be specified on the command-line
    
    9
    +via the `--flavour` flag. Moreover, this flavour name is saved in the build
    
    10
    +root; future invocations of Hadrian without a `--flavour` flag will default to
    
    11
    +the last-used flavour.
    
    12
    +
    
    13
    +See `Note [Persisting the flavour]` in `hadrian/src/Settings.hs` 
    
    7 14
     
    
    8 15
     ## Arguments
    
    9 16
     
    

  • hadrian/src/Rules.hs
    ... ... @@ -129,6 +129,7 @@ packageRules = do
    129 129
     
    
    130 130
     buildRules :: Rules ()
    
    131 131
     buildRules = do
    
    132
    +    Settings.flavourFileRules
    
    132 133
         Rules.BinaryDist.bindistRules
    
    133 134
         Rules.Generate.copyRules
    
    134 135
         Rules.Generate.generateRules
    

  • hadrian/src/Settings.hs
    1
    +{-# LANGUAGE TupleSections #-}
    
    2
    +{-# LANGUAGE TypeFamilies  #-}
    
    3
    +
    
    1 4
     module Settings (
    
    2 5
         getExtraArgs, getArgs, getLibraryWays, getRtsWays, flavour, knownPackages,
    
    3 6
         findPackageByName, unsafeFindPackageByName, unsafeFindPackageByPath,
    
    4 7
         isLibrary, stagePackages, getBignumBackend, completeSetting,
    
    5 8
         queryBuildTarget, queryHostTarget,
    
    6 9
         queryBuild, queryHost,
    
    7
    -    queryArch, queryOS, queryVendor
    
    10
    +    queryArch, queryOS, queryVendor,
    
    11
    +    flavourFileRules
    
    8 12
         ) where
    
    9 13
     
    
    10 14
     import CommandLine
    
    ... ... @@ -58,6 +62,56 @@ hadrianFlavours =
    58 62
         , ghcInGhciFlavour, validateFlavour, slowValidateFlavour
    
    59 63
         ]
    
    60 64
     
    
    65
    +{-
    
    66
    +Note [Persisting the flavour]
    
    67
    +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    
    68
    +Passing the flavour on the command-line means the user must consistently pass
    
    69
    +the same --flavour flag to all Hadrian invocations on a particular build root.
    
    70
    +Forgetting even once can result in a full rebuild of the tree.
    
    71
    +
    
    72
    +To avoid this we persist the flavour name in the build root itself
    
    73
    +(specifically, in `<build root>/flavour`). We refer to this file when looking
    
    74
    +up the active flavour, defaulting to it if the user does not specify a flavour
    
    75
    +name explicitly.
    
    76
    +-}
    
    77
    +
    
    78
    +-- | Oracle query type for the active flavour name.
    
    79
    +-- See Note [Persisting the flavour].
    
    80
    +newtype FlavourName = FlavourName ()
    
    81
    +    deriving (Binary, Eq, Hashable, NFData, Show)
    
    82
    +type instance RuleResult FlavourName = String
    
    83
    +
    
    84
    +flavourFileRules :: Rules ()
    
    85
    +flavourFileRules = do
    
    86
    +    root <- buildRootRules
    
    87
    +
    
    88
    +    -- Cache the flavour name lookup for the whole Shake run.
    
    89
    +    -- alwaysRerun ensures the oracle re-executes on every new invocation so
    
    90
    +    -- that a changed --flavour flag is never ignored due to a stale cross-run
    
    91
    +    -- cache entry (CLI args are not tracked as Shake dependencies).
    
    92
    +    -- See Note [Persisting the flavour].
    
    93
    +    void $ addOracleCache $ \(FlavourName _) -> do
    
    94
    +        alwaysRerun
    
    95
    +        let flavourFile = root -/- "flavour"
    
    96
    +        cmdFlavour >>= \case
    
    97
    +          Just nm -> do
    
    98
    +            putBuild $ "Flavour: " ++ nm
    
    99
    +            liftIO $ writeFile flavourFile nm
    
    100
    +            return nm
    
    101
    +          Nothing -> do
    
    102
    +            exists <- doesFileExist flavourFile
    
    103
    +            if exists
    
    104
    +              then do nm <- liftIO $ readFile flavourFile
    
    105
    +                      putBuild $ "Flavour: " ++ nm ++ " (cached from last build, override by passing --flavour=... explicitly)"
    
    106
    +                      return nm
    
    107
    +              else return userDefaultFlavour
    
    108
    +
    
    109
    +-- | Lookup the requested flavour name, referring to the persistent flavour
    
    110
    +-- file saved from the previous Hadrian execution if not specified.
    
    111
    +-- The result is cached for the whole Shake run via the 'FlavourName' oracle.
    
    112
    +getFlavourName :: Action String
    
    113
    +getFlavourName = askOracle (FlavourName ())
    
    114
    +
    
    61 115
     -- | This action looks up a flavour with the name given on the
    
    62 116
     --   command line with @--flavour@, defaulting to 'userDefaultFlavour'
    
    63 117
     --   when no explicit @--flavour@ is passed. It then applies any
    
    ... ... @@ -66,7 +120,7 @@ hadrianFlavours =
    66 120
     --   syntax. See Note [Hadrian settings] at the bottom of this file.
    
    67 121
     flavour :: Action Flavour
    
    68 122
     flavour = do
    
    69
    -    flavourName <- fromMaybe userDefaultFlavour <$> cmdFlavour
    
    123
    +    flavourName <- getFlavourName
    
    70 124
         when ("quickest" `isPrefixOf` flavourName) $
    
    71 125
           fail $ "The `quickest` flavour has been deprecated. Use `quick` instead.\n"
    
    72 126
               ++ "Skip building dynamic libraries with `quick+no_dynamic_libs`\n"
    

  • m4/subst_tooldir.m4
    ... ... @@ -41,5 +41,5 @@ dnl ghc-toolchain.
    41 41
     # See Note [How we configure the bundled windows toolchain]
    
    42 42
     AC_DEFUN([SUBST_TOOLDIR],
    
    43 43
     [
    
    44
    -    sed -i.bkp $1 's%'"$mingw_prefix"'%'"$mingw_install_prefix"'%g'
    
    44
    +    sed -i.bkp 's%'"$mingw_prefix"'%'"$mingw_install_prefix"'%g' $1
    
    45 45
     ])

  • rts/RtsStartup.c
    ... ... @@ -69,6 +69,10 @@
    69 69
     #if defined(HAVE_LOCALE_H)
    
    70 70
     #include <locale.h>
    
    71 71
     #endif
    
    72
    +#if !defined(mingw32_HOST_OS)
    
    73
    +#include <errno.h>
    
    74
    +#include <fcntl.h>
    
    75
    +#endif
    
    72 76
     
    
    73 77
     // Count of how many outstanding hs_init()s there have been.
    
    74 78
     static StgWord hs_init_count = 0;
    
    ... ... @@ -81,6 +85,10 @@ static int64_t __codePage = -1;
    81 85
     
    
    82 86
     static void flushStdHandles(void);
    
    83 87
     
    
    88
    +#if !defined(mingw32_HOST_OS)
    
    89
    +static void sanitiseStandardFds(void);
    
    90
    +#endif
    
    91
    +
    
    84 92
     /* -----------------------------------------------------------------------------
    
    85 93
        Initialise floating point unit on x86 (currently disabled; See Note
    
    86 94
        [x86 Floating point precision] in compiler/GHC/CmmToAsm/X86/Instr.hs)
    
    ... ... @@ -238,6 +246,82 @@ hs_init_with_rtsopts(int *argc, char **argv[])
    238 246
     
    
    239 247
     void init_ghc_hs_iface(void);
    
    240 248
     
    
    249
    +#if !defined(mingw32_HOST_OS)
    
    250
    +/* See Note [Standard file descriptors must be open]. */
    
    251
    +static void sanitiseStandardFds(void)
    
    252
    +{
    
    253
    +    // The access mode each descriptor would have had, had the process been
    
    254
    +    // started with it open.
    
    255
    +    static const int access_mode[3] = { O_RDONLY, O_WRONLY, O_WRONLY };
    
    256
    +
    
    257
    +    for (int fd = 0; fd <= 2; fd++) {
    
    258
    +        if (fcntl(fd, F_GETFD) != -1 || errno != EBADF) {
    
    259
    +            continue;           // already open
    
    260
    +        }
    
    261
    +
    
    262
    +        int nullfd;
    
    263
    +        do {
    
    264
    +            nullfd = open("/dev/null", access_mode[fd]);
    
    265
    +        } while (nullfd == -1 && errno == EINTR);
    
    266
    +
    
    267
    +        if (nullfd == -1) {
    
    268
    +            // No /dev/null to be had.  There is nothing better to do than
    
    269
    +            // carry on; the hazard the Note describes is then still there,
    
    270
    +            // but refusing to start would be worse.
    
    271
    +            return;
    
    272
    +        }
    
    273
    +        if (nullfd != fd) {
    
    274
    +            // open() hands out the lowest free descriptor, so this cannot
    
    275
    +            // happen unless something in this process is opening descriptors
    
    276
    +            // concurrently.  Do not shuffle its descriptor around.
    
    277
    +            close(nullfd);
    
    278
    +            return;
    
    279
    +        }
    
    280
    +    }
    
    281
    +}
    
    282
    +
    
    283
    +/*
    
    284
    + * Note [Standard file descriptors must be open]
    
    285
    + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    
    286
    + * A process may be started with any of file descriptors 0, 1 and 2 closed --
    
    287
    + * from a shell, `prog >&-` does exactly that.  Nothing then reserves those
    
    288
    + * numbers, and open(), pipe() and socket() hand out the lowest free
    
    289
    + * descriptor, so the next descriptor the runtime creates for itself takes one
    
    290
    + * of them: the ticker's pipe, or a capability's IO manager control pipe.
    
    291
    + *
    
    292
    + * Nothing warns anybody.  'GHC.Internal.IO.StdHandles.stdout' is a CAF, so it
    
    293
    + * is built the first time the program writes to stdout, which is long after
    
    294
    + * that has happened, and it builds a 'Handle' over descriptor 1 -- which now
    
    295
    + * belongs to the runtime.  The program then writes its output into the
    
    296
    + * runtime's own plumbing, and at exit closes it.
    
    297
    + *
    
    298
    + * What this looks like from the outside is a hang, because the descriptor the
    
    299
    + * Handle has taken over is the read end of a pipe.  It is a perfectly good
    
    300
    + * descriptor, so nothing reports an error: poll() for POLLOUT on it simply
    
    301
    + * never says yes, epoll_ctl() accepts a registration for it, and the thread
    
    302
    + * waiting to write blocks for ever.  See #24450, where this is `prog >&-`
    
    303
    + * hanging after main has finished.
    
    304
    + *
    
    305
    + * Closing those descriptors is legal, so the runtime cannot refuse them; and
    
    306
    + * moving only the descriptors the runtime opens would not help, since the
    
    307
    + * first file the program itself opens would land on the free number instead.
    
    308
    + * The fix is to leave no free number: open /dev/null onto whichever of 0, 1
    
    309
    + * and 2 are closed, before the runtime creates any descriptor of its own.
    
    310
    + * Other runtimes do the same, and for the same reason.
    
    311
    + *
    
    312
    + * A program run this way therefore finds stdout connected to /dev/null rather
    
    313
    + * than failing with EBADF on the first write.  That is the price of the fix,
    
    314
    + * and it is the behaviour such a program would get from most other language
    
    315
    + * runtimes.
    
    316
    + *
    
    317
    + * Each substitute is opened for the access the descriptor would normally have:
    
    318
    + * read-only for 0, write-only for 1 and 2.  Opening all three for both would
    
    319
    + * make the substitute something a real stdin or stdout is not, and writing to
    
    320
    + * 'stdin' or reading from 'stdout' would then quietly succeed instead of
    
    321
    + * failing with EBADF.
    
    322
    + */
    
    323
    +#endif
    
    324
    +
    
    241 325
     void
    
    242 326
     hs_init_ghc(int *argc, char **argv[], RtsConfig rts_config)
    
    243 327
     {
    
    ... ... @@ -252,6 +336,12 @@ hs_init_ghc(int *argc, char **argv[], RtsConfig rts_config)
    252 336
             stg_exit(1);
    
    253 337
         }
    
    254 338
     
    
    339
    +#if !defined(mingw32_HOST_OS)
    
    340
    +    /* Before anything here opens a file descriptor of its own.
    
    341
    +       See Note [Standard file descriptors must be open]. */
    
    342
    +    sanitiseStandardFds();
    
    343
    +#endif
    
    344
    +
    
    255 345
     #if defined(wasm32_HOST_ARCH)
    
    256 346
         char *pwd = getenv("PWD");
    
    257 347
         if (pwd != NULL) {
    

  • testsuite/tests/backpack/should_compile/T27835.bkp
    1
    +unit p where
    
    2
    +  module P where
    
    3
    +    p :: Int
    
    4
    +    p = 42
    
    5
    +
    
    6
    +unit q where
    
    7
    +  dependency p
    
    8
    +  module Q where
    
    9
    +    import P
    
    10
    +    q :: Int
    
    11
    +    q = p + 1

  • testsuite/tests/backpack/should_compile/T27835.stderr
    1
    +[1 of 2] Processing p
    
    2
    +  Instantiating p
    
    3
    +  [1 of 1] Compiling P                ( p/P.hs, T27835.out/p/P.o )
    
    4
    +[2 of 2] Processing q
    
    5
    +  Instantiating q
    
    6
    +  [1 of 1] Including p
    
    7
    +  [1 of 1] Compiling Q                ( q/Q.hs, T27835.out/q/Q.o )

  • testsuite/tests/backpack/should_compile/all.T
    ... ... @@ -61,3 +61,4 @@ test('T13250', normal, backpack_compile, [''])
    61 61
     test('T13323', normal, backpack_compile, [''])
    
    62 62
     test('T20396', normal, backpack_compile, [''])
    
    63 63
     test('T23424', [ignore_stdout, ignore_stderr], backpack_compile, ['-ddump-rn-trace -ddump-if-trace -ddump-tc-trace'])
    
    64
    +test('T27835', normal, backpack_compile, ['-clear-package-db -global-package-db'])

  • testsuite/tests/rts/T24450.hs
    1
    +{-# LANGUAGE ScopedTypeVariables #-}
    
    2
    +-- A process may be started with stdout closed -- `prog >&-` from a shell does
    
    3
    +-- exactly that.  Nothing then reserves descriptor 1, so the next descriptor
    
    4
    +-- the runtime opens for itself takes it, and the 'Handle' for stdout, built
    
    5
    +-- later when the program first writes, takes over a descriptor belonging to
    
    6
    +-- the runtime.  The write then goes into the runtime's own plumbing, and what
    
    7
    +-- this looks like from the outside is a hang.  See #24450 and
    
    8
    +-- Note [Standard file descriptors must be open] in rts/RtsStartup.c.
    
    9
    +--
    
    10
    +-- This test is run with its stdin and stdout closed, so what it has to say
    
    11
    +-- goes to stderr.  The four writes are the reproducer from the ticket and must
    
    12
    +-- simply finish.  The lines after them are what tell the two outcomes apart:
    
    13
    +-- the descriptors the runtime opens for itself are pipes, so finding a pipe on
    
    14
    +-- descriptor 0 or 1 means it was handed to the runtime.
    
    15
    +module Main (main) where
    
    16
    +
    
    17
    +import Control.Exception (IOException, try)
    
    18
    +import System.IO
    
    19
    +import System.Posix.Files (getFdStatus, isCharacterDevice, isNamedPipe)
    
    20
    +import System.Posix.Types (Fd (..))
    
    21
    +
    
    22
    +main :: IO ()
    
    23
    +main = do
    
    24
    +    hPutChar stderr 'A'     -- unbuffered
    
    25
    +    hPutChar stdout 'X'     -- buffered; this is the one that used to hang
    
    26
    +    hPutChar stderr 'Z'     -- unbuffered
    
    27
    +    hPutChar stderr '\n'    -- unbuffered
    
    28
    +    mapM_ (\fd -> hPutStrLn stderr =<< describeFd fd) [0, 1]
    
    29
    +
    
    30
    +describeFd :: Fd -> IO String
    
    31
    +describeFd fd = do
    
    32
    +    r <- try (getFdStatus fd)
    
    33
    +    return $ "fd " ++ show fd ++ ": " ++ case r of
    
    34
    +      Left (_ :: IOException) -> "closed"
    
    35
    +      Right st
    
    36
    +        -- What the fix arranges: /dev/null, opened onto the free number
    
    37
    +        -- before the runtime could take it.
    
    38
    +        | isCharacterDevice st -> "character device"
    
    39
    +        -- What #24450 is: the runtime's ticker or control pipe.
    
    40
    +        | isNamedPipe st       -> "pipe"
    
    41
    +        | otherwise            -> "something else"

  • testsuite/tests/rts/T24450.stderr
    1
    +AZ
    
    2
    +fd 0: character device
    
    3
    +fd 1: character device

  • testsuite/tests/rts/all.T
    ... ... @@ -408,6 +408,18 @@ test('T10590', [ ignore_stderr
    408 408
                    , req_process
    
    409 409
                    ], compile_and_run, [''])
    
    410 410
     
    
    411
    +# Run with stdin and stdout closed -- that is what leaves those descriptors
    
    412
    +# free for the runtime to take, which is the whole of #24450.  What the test
    
    413
    +# has to say therefore goes to stderr and is checked against T24450.stderr.
    
    414
    +test('T24450',
    
    415
    +     [ when(opsys('mingw32'), skip) # POSIX only
    
    416
    +     , js_skip # JS backend has its own runtime
    
    417
    +     # The wrapper below would close the stdout of the wasm runtime rather
    
    418
    +     # than that of the program running under it, which tests nothing.
    
    419
    +     , when(arch('wasm32'), skip)
    
    420
    +     , cmd_wrapper(lambda c: c + ' <&- >&-')
    
    421
    +     ], compile_and_run, [''])
    
    422
    +
    
    411 423
     # 20000 was easily enough to trigger the bug with 7.10
    
    412 424
     test('T10904', [ extra_run_opts('20000'), req_c ],
    
    413 425
                    compile_and_run, ['T10904lib.c'])