Marge Bot pushed to branch master at Glasgow Haskell Compiler / GHC

Commits:

6 changed files:

Changes:

  • changelog.d/hadrian-response-files.md
    1 1
     section: packaging
    
    2
    -synopsis: Add a flag to tell Hadrian to keep response files
    
    3
    -issues: #27184
    
    4
    -mrs: !15906
    
    2
    +synopsis: Improved Hadrian's use of response files
    
    3
    +issues: #27230
    
    4
    +mrs: !15906 !16134
    
    5 5
     description:
    
    6
    -  Hadrian can now be instructed to keep response files with the new
    
    7
    -  --keep-response-files command line flag. This is helpful when debugging a
    
    8
    -  build failure, as it allows re-running the failing command line invocation
    
    9
    -  without an error due to a missing response file.
    6
    +  Response files are files that contain command-line arguments. Hadrian uses
    
    7
    +  response files to shorten command-line lengths. This is important on Windows
    
    8
    +  where command-line lengths are limited.
    
    9
    +
    
    10
    +  Hadrian now supports response files when invoking GHC. In order to support
    
    11
    +  manually rerunning commands issued by Hadrian, response files are no longer
    
    12
    +  deleted. Instead they are stored under `_build/rsp`. Response files are now
    
    13
    +  only used when the corresponding command-line is too long for the host
    
    14
    +  platform. This greatly reduces the use of response files and avoids excessive
    
    15
    +  file usage. Response files are overwritten on subsequent Hadrian builds.

  • docs/users_guide/using.rst
    ... ... @@ -85,17 +85,6 @@ all files; you cannot, for example, invoke
    85 85
     ``ghc -c -O1 Foo.hs -O2 Bar.hs`` to apply different optimisation levels
    
    86 86
     to the files ``Foo.hs`` and ``Bar.hs``.
    
    87 87
     
    
    88
    -In addition to passing arguments via the command-line, arguments can be passed
    
    89
    -via GNU-style response files. For instance,
    
    90
    -
    
    91
    -.. code-block:: bash
    
    92
    -
    
    93
    -    $ cat response-file
    
    94
    -    -O1
    
    95
    -    Hello.hs
    
    96
    -    -o Hello
    
    97
    -    $ ghc @response-file
    
    98
    -
    
    99 88
     .. note::
    
    100 89
     
    
    101 90
         .. index::
    
    ... ... @@ -118,9 +107,24 @@ via GNU-style response files. For instance,
    118 107
             ``-fspecialise`` will not be enabled, since the ``-fno-specialise``
    
    119 108
             overrides the ``-fspecialise`` implied by ``-O1``.
    
    120 109
     
    
    110
    +
    
    111
    +Command-line arguments in response files
    
    112
    +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    
    113
    +
    
    114
    +In addition to passing arguments via the command-line, arguments can be passed
    
    115
    +via GNU-style response files. For instance,
    
    116
    +
    
    117
    +.. code-block:: bash
    
    118
    +
    
    119
    +    $ cat response-file
    
    120
    +    -O1
    
    121
    +    Hello.hs
    
    122
    +    -o Hello
    
    123
    +    $ ghc @response-file
    
    124
    +
    
    121 125
     .. _source-file-options:
    
    122 126
     
    
    123
    -Command line options in source files
    
    127
    +Command-line options in source files
    
    124 128
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    
    125 129
     
    
    126 130
     .. index::
    

  • hadrian/src/Builder.hs
    ... ... @@ -304,7 +304,7 @@ instance H.Builder Builder where
    304 304
                 case builder of
    
    305 305
                     Ar Pack stg -> do
    
    306 306
                         useTempFile <- arSupportsAtFile stg
    
    307
    -                    if useTempFile then runAr                path buildArgs buildInputs buildOptions
    
    307
    +                    if useTempFile then runAr output         path buildArgs buildInputs buildOptions
    
    308 308
                                        else runArWithoutTempFile path buildArgs buildInputs buildOptions
    
    309 309
     
    
    310 310
                     Ar Unpack _ -> cmd' [Cwd output] [path] buildArgs buildOptions
    
    ... ... @@ -343,7 +343,7 @@ instance H.Builder Builder where
    343 343
                         Exit _ <- cmd' [path] (buildArgs ++ [input]) buildOptions
    
    344 344
                         return ()
    
    345 345
     
    
    346
    -                Haddock BuildPackage -> runHaddock path buildArgs buildInputs
    
    346
    +                Haddock BuildPackage -> runHaddock output path buildArgs buildInputs
    
    347 347
     
    
    348 348
                     Ghc _ _ ->
    
    349 349
                       -- Use a response file for ghc invocations to avoid issues with command line
    
    ... ... @@ -351,9 +351,11 @@ instance H.Builder Builder where
    351 351
                       -- NB: we can't put the buildArgs in a response file, because some flags require
    
    352 352
                       -- empty arguments (such as the -dep-suffix flag), but that isn't supported
    
    353 353
                       -- yet due to #26560.
    
    354
    -                  withResponseFileOnWindows
    
    355
    -                    (\buildInputs' -> cmd [path] buildArgs buildInputs' buildOptions)
    
    354
    +                  withResponseFileIfLongCmd
    
    355
    +                    output
    
    356
    +                    (toCmdArgument [path] <> toCmdArgument buildArgs)
    
    356 357
                         buildInputs
    
    358
    +                    (toCmdArgument buildOptions)
    
    357 359
     
    
    358 360
                     HsCpp    -> captureStdout
    
    359 361
     
    
    ... ... @@ -389,13 +391,16 @@ instance H.Builder Builder where
    389 391
     
    
    390 392
     -- | Invoke @haddock@ given a path to it and a list of arguments. On Windows,
    
    391 393
     -- the input file arguments are passed as a response file.
    
    392
    -runHaddock :: FilePath    -- ^ path to @haddock@
    
    394
    +runHaddock :: FilePath -- ^ base name to use for response file
    
    395
    +      -> FilePath    -- ^ path to @haddock@
    
    393 396
           -> [String]
    
    394 397
           -> [FilePath]  -- ^ input file paths
    
    395 398
           -> Action ()
    
    396
    -runHaddock haddockPath flagArgs fileInputs = withResponseFileOnWindows
    
    397
    -  (cmd [haddockPath] flagArgs)
    
    399
    +runHaddock outputFilePath haddockPath flagArgs fileInputs = withResponseFileIfLongCmd
    
    400
    +  outputFilePath
    
    401
    +  (toCmdArgument [haddockPath] <> toCmdArgument flagArgs)
    
    398 402
       fileInputs
    
    403
    +  (CmdArgument [])
    
    399 404
     
    
    400 405
     -- TODO: Some builders are required only on certain platforms. For example,
    
    401 406
     -- 'Objdump' is only required on OpenBSD and AIX. Add support for platform
    

  • hadrian/src/CommandLine.hs
    ... ... @@ -3,8 +3,7 @@ module CommandLine (
    3 3
         lookupBignum,
    
    4 4
         cmdBignum, cmdProgressInfo, cmdCompleteSetting,
    
    5 5
         cmdDocsArgs, cmdUnitIdHash, lookupBuildRoot, TestArgs(..), TestSpeed(..), defaultTestArgs,
    
    6
    -    cmdPrefix, cmdChangelogVersion, DocArgs(..), defaultDocArgs,
    
    7
    -    cmdKeepResponseFiles
    
    6
    +    cmdPrefix, cmdChangelogVersion, DocArgs(..), defaultDocArgs
    
    8 7
         ) where
    
    9 8
     
    
    10 9
     import Data.Either
    
    ... ... @@ -12,7 +11,7 @@ import qualified Data.HashMap.Strict as Map
    12 11
     import Data.List.Extra
    
    13 12
     import Development.Shake hiding (Normal)
    
    14 13
     import Flavour (DocTargets, DocTarget(..))
    
    15
    -import Hadrian.Utilities hiding (buildRoot, keepResponseFiles)
    
    14
    +import Hadrian.Utilities hiding (buildRoot)
    
    16 15
     import Settings.Parser
    
    17 16
     import System.Console.GetOpt
    
    18 17
     import System.Environment
    
    ... ... @@ -37,7 +36,6 @@ data CommandLineArgs = CommandLineArgs
    37 36
         , testArgs       :: TestArgs
    
    38 37
         , docsArgs       :: DocArgs
    
    39 38
         , docTargets     :: DocTargets
    
    40
    -    , keepResponseFiles :: Bool
    
    41 39
         , prefix           :: Maybe FilePath
    
    42 40
         , changelogVersion :: Maybe String
    
    43 41
         , completeStg      :: Maybe String }
    
    ... ... @@ -58,7 +56,6 @@ defaultCommandLineArgs = CommandLineArgs
    58 56
         , testArgs       = defaultTestArgs
    
    59 57
         , docsArgs       = defaultDocArgs
    
    60 58
         , docTargets     = Set.fromList [minBound..maxBound]
    
    61
    -    , keepResponseFiles    = False
    
    62 59
         , prefix           = Nothing
    
    63 60
         , changelogVersion = Nothing
    
    64 61
         , completeStg      = Nothing }
    
    ... ... @@ -141,9 +138,6 @@ readFreeze1 = Right $ \flags -> flags { freeze1 = True }
    141 138
     readFreeze2 = Right $ \flags -> flags { freeze1 = True, freeze2 = True }
    
    142 139
     readSkipDepends = Right $ \flags -> flags { skipDepends = True }
    
    143 140
     
    
    144
    -readKeepResponseFiles :: Either String (CommandLineArgs -> CommandLineArgs)
    
    145
    -readKeepResponseFiles = Right $ \flags -> flags { keepResponseFiles = True }
    
    146
    -
    
    147 141
     readUnitIdHash :: Either String (CommandLineArgs -> CommandLineArgs)
    
    148 142
     readUnitIdHash = Right $ \flags ->
    
    149 143
       trace "--hash-unit-ids is deprecated. It is enabled by release flavour or +hash_unit_ids flavour transformer" $
    
    ... ... @@ -302,8 +296,6 @@ optDescrs =
    302 296
           "Progress info style (None, Brief, Normal or Unicorn)."
    
    303 297
         , Option [] ["docs"] (ReqArg readDocsArg "TARGET")
    
    304 298
           "Strip down docs targets (none, no-haddocks, no-sphinx[-{html, pdfs, man}]."
    
    305
    -    , Option ['r'] ["keep-response-files"] (NoArg readKeepResponseFiles)
    
    306
    -      "Keep response files created during the build (for debugging)."
    
    307 299
         , Option ['k'] ["keep-test-files"] (NoArg readTestKeepFiles)
    
    308 300
           "Keep all the files generated when running the testsuite."
    
    309 301
         , Option [] ["test-compiler"] (ReqArg readTestCompiler "TEST_COMPILER")
    
    ... ... @@ -382,7 +374,6 @@ cmdLineArgsMap = do
    382 374
     
    
    383 375
         return $ insertExtra (progressInfo               args) -- Accessed by Hadrian.Utilities
    
    384 376
                $ insertExtra (buildRoot                  args) -- Accessed by Hadrian.Utilities
    
    385
    -           $ insertExtra (KeepResponseFiles $ keepResponseFiles args) -- Accessed by Hadrian.Utilities
    
    386 377
                $ insertExtra (testArgs                   args) -- Accessed by Settings.Builders.RunTest
    
    387 378
                $ insertExtra (docsArgs                   args) -- Accessed by Rules.Documentation
    
    388 379
                $ insertExtra allSettings                       -- Accessed by Settings
    
    ... ... @@ -424,9 +415,6 @@ cmdUnitIdHash = unitIdHash <$> cmdLineArgs
    424 415
     cmdBignum :: Action (Maybe String)
    
    425 416
     cmdBignum = bignum <$> cmdLineArgs
    
    426 417
     
    
    427
    -cmdKeepResponseFiles :: Action Bool
    
    428
    -cmdKeepResponseFiles = keepResponseFiles <$> cmdLineArgs
    
    429
    -
    
    430 418
     cmdProgressInfo :: Action ProgressInfo
    
    431 419
     cmdProgressInfo = progressInfo <$> cmdLineArgs
    
    432 420
     
    

  • hadrian/src/Hadrian/Builder/Ar.hs
    ... ... @@ -35,14 +35,16 @@ instance NFData ArMode
    35 35
     -- to be archived is passed via a temporary response file. Passing arguments
    
    36 36
     -- via a response file is not supported by some versions of @ar@, in which
    
    37 37
     -- case you should use 'runArWithoutTempFile' instead.
    
    38
    -runAr :: FilePath    -- ^ path to @ar@
    
    38
    +runAr :: FilePath    -- ^ base name to use for response files
    
    39
    +      -> FilePath    -- ^ path to @ar@
    
    39 40
           -> [String]    -- ^ other arguments
    
    40 41
           -> [FilePath]  -- ^ input file paths
    
    41 42
           -> [CmdOption] -- ^ Additional options
    
    42 43
           -> Action ()
    
    43
    -runAr arPath flagArgs fileArgs buildOptions = withResponseFile $ \tmp -> do
    
    44
    -    writeFile' tmp $ unwords fileArgs
    
    45
    -    cmd [arPath] flagArgs ('@' : tmp) buildOptions
    
    44
    +runAr outputFilePath arPath flagArgs fileArgs buildOptions = do
    
    45
    +    rspFile <- responseFilePath outputFilePath
    
    46
    +    writeFile' rspFile $ unwords fileArgs
    
    47
    +    cmd [arPath] flagArgs ('@' : rspFile) buildOptions
    
    46 48
     
    
    47 49
     -- | Invoke @ar@ given a path to it and a list of arguments. Note that @ar@
    
    48 50
     -- will be called multiple times if the list of files to be archived is too
    

  • hadrian/src/Hadrian/Utilities.hs
    1
    +{-# LANGUAGE ImpredicativeTypes #-}
    
    1 2
     {-# LANGUAGE TypeFamilies #-}
    
    3
    +
    
    2 4
     module Hadrian.Utilities (
    
    3 5
         -- * List manipulation
    
    4 6
         fromSingleton, replaceEq, minusOrd, intersectOrd, lookupAll, chunksOfSize,
    
    ... ... @@ -14,7 +16,7 @@ module Hadrian.Utilities (
    14 16
     
    
    15 17
         -- * Paths
    
    16 18
         BuildRoot (..), buildRoot, buildRootRules, isGeneratedSource,
    
    17
    -    KeepResponseFiles (..), keepResponseFiles, withResponseFile, withResponseFileOnWindows,
    
    19
    +    withResponseFileIfLongCmd, responseFilePath,
    
    18 20
     
    
    19 21
         -- * File system operations
    
    20 22
         copyFile, copyFileUntracked, createFileLink, fixFile,
    
    ... ... @@ -47,11 +49,10 @@ import Data.Maybe
    47 49
     import Data.Typeable (TypeRep, typeOf)
    
    48 50
     import Development.Shake hiding (Normal)
    
    49 51
     import Development.Shake.Classes
    
    52
    +import Development.Shake.Command (CmdArgument (..), IsCmdArgument (toCmdArgument))
    
    50 53
     import Development.Shake.FilePath
    
    51 54
     import GHC.ResponseFile (escapeArgs)
    
    52 55
     import System.Environment (lookupEnv)
    
    53
    -import System.Info.Extra (isWindows)
    
    54
    -import System.IO (hClose, openTempFile)
    
    55 56
     import System.IO.Error (isPermissionError)
    
    56 57
     
    
    57 58
     import qualified Data.ByteString        as BS
    
    ... ... @@ -255,13 +256,13 @@ infix 1 %%>
    255 256
     -- library, they can reach 2MB! Some operating systems do not support command
    
    256 257
     -- lines of such length, and this function can be used to obtain a reasonable
    
    257 258
     -- approximation of the limit. On Windows, it is theoretically 32768 characters
    
    258
    --- (since Windows 7). In practice we use 31000 to leave some breathing space for
    
    259
    +-- (since Windows 7). In practice we use 30000 to leave some breathing space for
    
    259 260
     -- the builder path & name, auxiliary flags, and other overheads. On Mac OS X,
    
    260 261
     -- ARG_MAX is 262144, yet when using @xargs@ on OSX this is reduced by over
    
    261 262
     -- 20000. Hence, 200000 seems like a sensible limit. On other operating systems
    
    262 263
     -- we currently use the 4194304 setting.
    
    263 264
     cmdLineLengthLimit :: Int
    
    264
    -cmdLineLengthLimit | IO.isWindows = 31000
    
    265
    +cmdLineLengthLimit | IO.isWindows = 30000
    
    265 266
                        | IO.isMac     = 200000
    
    266 267
                        | otherwise    = 4194304
    
    267 268
     
    
    ... ... @@ -321,53 +322,35 @@ buildRootRules = do
    321 322
     isGeneratedSource :: FilePath -> Action Bool
    
    322 323
     isGeneratedSource file = buildRoot <&> (`isPrefixOf` file)
    
    323 324
     
    
    324
    -newtype KeepResponseFiles = KeepResponseFiles Bool deriving (Eq, Show)
    
    325
    -
    
    326
    --- | Whether to retain response files after the build action that created them
    
    327
    --- completes. Mainly useful for debugging.
    
    328
    -keepResponseFiles :: Action Bool
    
    329
    -keepResponseFiles = do
    
    330
    -    KeepResponseFiles keep <- userSetting (KeepResponseFiles False)
    
    331
    -    return keep
    
    332
    -
    
    333
    --- | Run an action either with command arguments direcly or by, on Windows,
    
    334
    --- placing those arguments into a response file escaped with @GHC.ResponseFile.escapeArgs@.
    
    335
    ---
    
    336
    --- With @--keep-response-files@, the file is left on disk (if used)
    
    337
    -withResponseFileOnWindows ::
    
    338
    -    ([String] -> Action a)  -- ^ Action to perform given arguments (of the form @["\@reponseFilePath"]@ on Windows)
    
    339
    -    -> [String]             -- ^ Command arguments
    
    340
    -    -> Action a
    
    341
    -withResponseFileOnWindows action commandArgs = do
    
    342
    -    if isWindows
    
    343
    -        then withResponseFile $ \tmp -> do
    
    344
    -                writeFile' tmp (escapeArgs commandArgs)
    
    345
    -                action ['@' : tmp]
    
    346
    -        else action commandArgs
    
    347
    -
    
    348
    --- | Run an action with a response file path.
    
    349
    ---
    
    350
    --- With @--keep-response-files@, the file is left on disk.
    
    351
    -withResponseFile :: (FilePath -> Action a) -> Action a
    
    352
    -withResponseFile action = do
    
    353
    -    keep <- keepResponseFiles
    
    354
    -    let putVerboseResponseFile tmp = do
    
    355
    -            verbosity <- getVerbosity
    
    356
    -            when (verbosity >= Verbose) $ do
    
    357
    -                tmpContent <- liftIO (readFile tmp)
    
    358
    -                putVerbose (tmp <> " (use hadrian flag --keep-response-files to keep this file):\n" <> tmpContent)
    
    359
    -    if keep
    
    360
    -        then do
    
    361
    -            (tmp, h) <- liftIO $ openTempFile "." "hadrian-rsp"
    
    362
    -            liftIO $ hClose h
    
    363
    -            putInfo $ "Keeping response file: " ++ tmp
    
    364
    -            result <- action tmp
    
    365
    -            putVerboseResponseFile tmp
    
    366
    -            return result
    
    367
    -        else withTempFile $ \tmp -> do
    
    368
    -            result <- action tmp
    
    369
    -            putVerboseResponseFile tmp
    
    370
    -            return result
    
    325
    +-- | Run an command with the given arguments. If the command is too long then the
    
    326
    +-- response file arguments are placed into a response file and escaped with @GHC.ResponseFile.escapeArgs@.
    
    327
    +withResponseFileIfLongCmd ::
    
    328
    +    CmdResult c
    
    329
    +    => FilePath     -- ^ Response base name. The reponse file is placed in @_build/rsp/\<Response base name\>@.
    
    330
    +    -> CmdArgument  -- ^ Command and arguments before the response file arguments.
    
    331
    +    -> [String]     -- ^ Response file aruguments.
    
    332
    +    -> CmdArgument  -- ^ Command arguments after the response file arguments.
    
    333
    +    -> Action c
    
    334
    +withResponseFileIfLongCmd outputFilePath argsPre argsResp argsPost = do
    
    335
    +    let cmdLineLengh = sum
    
    336
    +            [ 1 + length arg -- add one to account for space inbetween arguments
    
    337
    +            | let CmdArgument args = argsPre <> toCmdArgument argsResp <> argsPost
    
    338
    +            , Right arg <- args
    
    339
    +            ]
    
    340
    +    if cmdLineLengh < cmdLineLengthLimit
    
    341
    +        then cmd argsPre argsResp argsPost
    
    342
    +        else do
    
    343
    +            rspFile <- responseFilePath outputFilePath
    
    344
    +            writeFile' rspFile (escapeArgs argsResp)
    
    345
    +            cmd argsPre ['@' : rspFile] argsPost
    
    346
    +
    
    347
    +-- | Convert a command's output file path to a response file path to be used for that command.
    
    348
    +-- Response files are placed in a dedicated @rps@ directory under the build directory. This avoids
    
    349
    +-- clutering the work tree or interfearing with other build directories.
    
    350
    +responseFilePath :: FilePath -> Action FilePath
    
    351
    +responseFilePath outputFilePath = do
    
    352
    +    buildDir <- buildRoot
    
    353
    +    return $ buildDir </> "rsp" </> outputFilePath
    
    371 354
     
    
    372 355
     -- | Link a file tracking the link target. Create the target directory if
    
    373 356
     -- missing.