Marge Bot pushed to branch master at Glasgow Haskell Compiler / GHC
Commits:
-
498bb21a
by David Eichmann at 2026-06-09T18:02:39-04:00
6 changed files:
- changelog.d/hadrian-response-files.md
- docs/users_guide/using.rst
- hadrian/src/Builder.hs
- hadrian/src/CommandLine.hs
- hadrian/src/Hadrian/Builder/Ar.hs
- hadrian/src/Hadrian/Utilities.hs
Changes:
| 1 | 1 | section: packaging
|
| 2 | -synopsis: Add a flag to tell Hadrian to keep response files
|
|
| 3 | -issues: #27184
|
|
| 4 | -mrs: !15906
|
|
| 2 | +synopsis: Improved Hadrian's use of response files
|
|
| 3 | +issues: #27230
|
|
| 4 | +mrs: !15906 !16134
|
|
| 5 | 5 | description:
|
| 6 | - Hadrian can now be instructed to keep response files with the new
|
|
| 7 | - --keep-response-files command line flag. This is helpful when debugging a
|
|
| 8 | - build failure, as it allows re-running the failing command line invocation
|
|
| 9 | - without an error due to a missing response file. |
|
| 6 | + Response files are files that contain command-line arguments. Hadrian uses
|
|
| 7 | + response files to shorten command-line lengths. This is important on Windows
|
|
| 8 | + where command-line lengths are limited.
|
|
| 9 | + |
|
| 10 | + Hadrian now supports response files when invoking GHC. In order to support
|
|
| 11 | + manually rerunning commands issued by Hadrian, response files are no longer
|
|
| 12 | + deleted. Instead they are stored under `_build/rsp`. Response files are now
|
|
| 13 | + only used when the corresponding command-line is too long for the host
|
|
| 14 | + platform. This greatly reduces the use of response files and avoids excessive
|
|
| 15 | + file usage. Response files are overwritten on subsequent Hadrian builds. |
| ... | ... | @@ -85,17 +85,6 @@ all files; you cannot, for example, invoke |
| 85 | 85 | ``ghc -c -O1 Foo.hs -O2 Bar.hs`` to apply different optimisation levels
|
| 86 | 86 | to the files ``Foo.hs`` and ``Bar.hs``.
|
| 87 | 87 | |
| 88 | -In addition to passing arguments via the command-line, arguments can be passed
|
|
| 89 | -via GNU-style response files. For instance,
|
|
| 90 | - |
|
| 91 | -.. code-block:: bash
|
|
| 92 | - |
|
| 93 | - $ cat response-file
|
|
| 94 | - -O1
|
|
| 95 | - Hello.hs
|
|
| 96 | - -o Hello
|
|
| 97 | - $ ghc @response-file
|
|
| 98 | - |
|
| 99 | 88 | .. note::
|
| 100 | 89 | |
| 101 | 90 | .. index::
|
| ... | ... | @@ -118,9 +107,24 @@ via GNU-style response files. For instance, |
| 118 | 107 | ``-fspecialise`` will not be enabled, since the ``-fno-specialise``
|
| 119 | 108 | overrides the ``-fspecialise`` implied by ``-O1``.
|
| 120 | 109 | |
| 110 | + |
|
| 111 | +Command-line arguments in response files
|
|
| 112 | +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
| 113 | + |
|
| 114 | +In addition to passing arguments via the command-line, arguments can be passed
|
|
| 115 | +via GNU-style response files. For instance,
|
|
| 116 | + |
|
| 117 | +.. code-block:: bash
|
|
| 118 | + |
|
| 119 | + $ cat response-file
|
|
| 120 | + -O1
|
|
| 121 | + Hello.hs
|
|
| 122 | + -o Hello
|
|
| 123 | + $ ghc @response-file
|
|
| 124 | + |
|
| 121 | 125 | .. _source-file-options:
|
| 122 | 126 | |
| 123 | -Command line options in source files
|
|
| 127 | +Command-line options in source files
|
|
| 124 | 128 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
| 125 | 129 | |
| 126 | 130 | .. index::
|
| ... | ... | @@ -304,7 +304,7 @@ instance H.Builder Builder where |
| 304 | 304 | case builder of
|
| 305 | 305 | Ar Pack stg -> do
|
| 306 | 306 | useTempFile <- arSupportsAtFile stg
|
| 307 | - if useTempFile then runAr path buildArgs buildInputs buildOptions
|
|
| 307 | + if useTempFile then runAr output path buildArgs buildInputs buildOptions
|
|
| 308 | 308 | else runArWithoutTempFile path buildArgs buildInputs buildOptions
|
| 309 | 309 | |
| 310 | 310 | Ar Unpack _ -> cmd' [Cwd output] [path] buildArgs buildOptions
|
| ... | ... | @@ -343,7 +343,7 @@ instance H.Builder Builder where |
| 343 | 343 | Exit _ <- cmd' [path] (buildArgs ++ [input]) buildOptions
|
| 344 | 344 | return ()
|
| 345 | 345 | |
| 346 | - Haddock BuildPackage -> runHaddock path buildArgs buildInputs
|
|
| 346 | + Haddock BuildPackage -> runHaddock output path buildArgs buildInputs
|
|
| 347 | 347 | |
| 348 | 348 | Ghc _ _ ->
|
| 349 | 349 | -- Use a response file for ghc invocations to avoid issues with command line
|
| ... | ... | @@ -351,9 +351,11 @@ instance H.Builder Builder where |
| 351 | 351 | -- NB: we can't put the buildArgs in a response file, because some flags require
|
| 352 | 352 | -- empty arguments (such as the -dep-suffix flag), but that isn't supported
|
| 353 | 353 | -- yet due to #26560.
|
| 354 | - withResponseFileOnWindows
|
|
| 355 | - (\buildInputs' -> cmd [path] buildArgs buildInputs' buildOptions)
|
|
| 354 | + withResponseFileIfLongCmd
|
|
| 355 | + output
|
|
| 356 | + (toCmdArgument [path] <> toCmdArgument buildArgs)
|
|
| 356 | 357 | buildInputs
|
| 358 | + (toCmdArgument buildOptions)
|
|
| 357 | 359 | |
| 358 | 360 | HsCpp -> captureStdout
|
| 359 | 361 | |
| ... | ... | @@ -389,13 +391,16 @@ instance H.Builder Builder where |
| 389 | 391 | |
| 390 | 392 | -- | Invoke @haddock@ given a path to it and a list of arguments. On Windows,
|
| 391 | 393 | -- the input file arguments are passed as a response file.
|
| 392 | -runHaddock :: FilePath -- ^ path to @haddock@
|
|
| 394 | +runHaddock :: FilePath -- ^ base name to use for response file
|
|
| 395 | + -> FilePath -- ^ path to @haddock@
|
|
| 393 | 396 | -> [String]
|
| 394 | 397 | -> [FilePath] -- ^ input file paths
|
| 395 | 398 | -> Action ()
|
| 396 | -runHaddock haddockPath flagArgs fileInputs = withResponseFileOnWindows
|
|
| 397 | - (cmd [haddockPath] flagArgs)
|
|
| 399 | +runHaddock outputFilePath haddockPath flagArgs fileInputs = withResponseFileIfLongCmd
|
|
| 400 | + outputFilePath
|
|
| 401 | + (toCmdArgument [haddockPath] <> toCmdArgument flagArgs)
|
|
| 398 | 402 | fileInputs
|
| 403 | + (CmdArgument [])
|
|
| 399 | 404 | |
| 400 | 405 | -- TODO: Some builders are required only on certain platforms. For example,
|
| 401 | 406 | -- 'Objdump' is only required on OpenBSD and AIX. Add support for platform
|
| ... | ... | @@ -3,8 +3,7 @@ module CommandLine ( |
| 3 | 3 | lookupBignum,
|
| 4 | 4 | cmdBignum, cmdProgressInfo, cmdCompleteSetting,
|
| 5 | 5 | cmdDocsArgs, cmdUnitIdHash, lookupBuildRoot, TestArgs(..), TestSpeed(..), defaultTestArgs,
|
| 6 | - cmdPrefix, cmdChangelogVersion, DocArgs(..), defaultDocArgs,
|
|
| 7 | - cmdKeepResponseFiles
|
|
| 6 | + cmdPrefix, cmdChangelogVersion, DocArgs(..), defaultDocArgs
|
|
| 8 | 7 | ) where
|
| 9 | 8 | |
| 10 | 9 | import Data.Either
|
| ... | ... | @@ -12,7 +11,7 @@ import qualified Data.HashMap.Strict as Map |
| 12 | 11 | import Data.List.Extra
|
| 13 | 12 | import Development.Shake hiding (Normal)
|
| 14 | 13 | import Flavour (DocTargets, DocTarget(..))
|
| 15 | -import Hadrian.Utilities hiding (buildRoot, keepResponseFiles)
|
|
| 14 | +import Hadrian.Utilities hiding (buildRoot)
|
|
| 16 | 15 | import Settings.Parser
|
| 17 | 16 | import System.Console.GetOpt
|
| 18 | 17 | import System.Environment
|
| ... | ... | @@ -37,7 +36,6 @@ data CommandLineArgs = CommandLineArgs |
| 37 | 36 | , testArgs :: TestArgs
|
| 38 | 37 | , docsArgs :: DocArgs
|
| 39 | 38 | , docTargets :: DocTargets
|
| 40 | - , keepResponseFiles :: Bool
|
|
| 41 | 39 | , prefix :: Maybe FilePath
|
| 42 | 40 | , changelogVersion :: Maybe String
|
| 43 | 41 | , completeStg :: Maybe String }
|
| ... | ... | @@ -58,7 +56,6 @@ defaultCommandLineArgs = CommandLineArgs |
| 58 | 56 | , testArgs = defaultTestArgs
|
| 59 | 57 | , docsArgs = defaultDocArgs
|
| 60 | 58 | , docTargets = Set.fromList [minBound..maxBound]
|
| 61 | - , keepResponseFiles = False
|
|
| 62 | 59 | , prefix = Nothing
|
| 63 | 60 | , changelogVersion = Nothing
|
| 64 | 61 | , completeStg = Nothing }
|
| ... | ... | @@ -141,9 +138,6 @@ readFreeze1 = Right $ \flags -> flags { freeze1 = True } |
| 141 | 138 | readFreeze2 = Right $ \flags -> flags { freeze1 = True, freeze2 = True }
|
| 142 | 139 | readSkipDepends = Right $ \flags -> flags { skipDepends = True }
|
| 143 | 140 | |
| 144 | -readKeepResponseFiles :: Either String (CommandLineArgs -> CommandLineArgs)
|
|
| 145 | -readKeepResponseFiles = Right $ \flags -> flags { keepResponseFiles = True }
|
|
| 146 | - |
|
| 147 | 141 | readUnitIdHash :: Either String (CommandLineArgs -> CommandLineArgs)
|
| 148 | 142 | readUnitIdHash = Right $ \flags ->
|
| 149 | 143 | trace "--hash-unit-ids is deprecated. It is enabled by release flavour or +hash_unit_ids flavour transformer" $
|
| ... | ... | @@ -302,8 +296,6 @@ optDescrs = |
| 302 | 296 | "Progress info style (None, Brief, Normal or Unicorn)."
|
| 303 | 297 | , Option [] ["docs"] (ReqArg readDocsArg "TARGET")
|
| 304 | 298 | "Strip down docs targets (none, no-haddocks, no-sphinx[-{html, pdfs, man}]."
|
| 305 | - , Option ['r'] ["keep-response-files"] (NoArg readKeepResponseFiles)
|
|
| 306 | - "Keep response files created during the build (for debugging)."
|
|
| 307 | 299 | , Option ['k'] ["keep-test-files"] (NoArg readTestKeepFiles)
|
| 308 | 300 | "Keep all the files generated when running the testsuite."
|
| 309 | 301 | , Option [] ["test-compiler"] (ReqArg readTestCompiler "TEST_COMPILER")
|
| ... | ... | @@ -382,7 +374,6 @@ cmdLineArgsMap = do |
| 382 | 374 | |
| 383 | 375 | return $ insertExtra (progressInfo args) -- Accessed by Hadrian.Utilities
|
| 384 | 376 | $ insertExtra (buildRoot args) -- Accessed by Hadrian.Utilities
|
| 385 | - $ insertExtra (KeepResponseFiles $ keepResponseFiles args) -- Accessed by Hadrian.Utilities
|
|
| 386 | 377 | $ insertExtra (testArgs args) -- Accessed by Settings.Builders.RunTest
|
| 387 | 378 | $ insertExtra (docsArgs args) -- Accessed by Rules.Documentation
|
| 388 | 379 | $ insertExtra allSettings -- Accessed by Settings
|
| ... | ... | @@ -424,9 +415,6 @@ cmdUnitIdHash = unitIdHash <$> cmdLineArgs |
| 424 | 415 | cmdBignum :: Action (Maybe String)
|
| 425 | 416 | cmdBignum = bignum <$> cmdLineArgs
|
| 426 | 417 | |
| 427 | -cmdKeepResponseFiles :: Action Bool
|
|
| 428 | -cmdKeepResponseFiles = keepResponseFiles <$> cmdLineArgs
|
|
| 429 | - |
|
| 430 | 418 | cmdProgressInfo :: Action ProgressInfo
|
| 431 | 419 | cmdProgressInfo = progressInfo <$> cmdLineArgs
|
| 432 | 420 |
| ... | ... | @@ -35,14 +35,16 @@ instance NFData ArMode |
| 35 | 35 | -- to be archived is passed via a temporary response file. Passing arguments
|
| 36 | 36 | -- via a response file is not supported by some versions of @ar@, in which
|
| 37 | 37 | -- case you should use 'runArWithoutTempFile' instead.
|
| 38 | -runAr :: FilePath -- ^ path to @ar@
|
|
| 38 | +runAr :: FilePath -- ^ base name to use for response files
|
|
| 39 | + -> FilePath -- ^ path to @ar@
|
|
| 39 | 40 | -> [String] -- ^ other arguments
|
| 40 | 41 | -> [FilePath] -- ^ input file paths
|
| 41 | 42 | -> [CmdOption] -- ^ Additional options
|
| 42 | 43 | -> Action ()
|
| 43 | -runAr arPath flagArgs fileArgs buildOptions = withResponseFile $ \tmp -> do
|
|
| 44 | - writeFile' tmp $ unwords fileArgs
|
|
| 45 | - cmd [arPath] flagArgs ('@' : tmp) buildOptions
|
|
| 44 | +runAr outputFilePath arPath flagArgs fileArgs buildOptions = do
|
|
| 45 | + rspFile <- responseFilePath outputFilePath
|
|
| 46 | + writeFile' rspFile $ unwords fileArgs
|
|
| 47 | + cmd [arPath] flagArgs ('@' : rspFile) buildOptions
|
|
| 46 | 48 | |
| 47 | 49 | -- | Invoke @ar@ given a path to it and a list of arguments. Note that @ar@
|
| 48 | 50 | -- will be called multiple times if the list of files to be archived is too
|
| 1 | +{-# LANGUAGE ImpredicativeTypes #-}
|
|
| 1 | 2 | {-# LANGUAGE TypeFamilies #-}
|
| 3 | + |
|
| 2 | 4 | module Hadrian.Utilities (
|
| 3 | 5 | -- * List manipulation
|
| 4 | 6 | fromSingleton, replaceEq, minusOrd, intersectOrd, lookupAll, chunksOfSize,
|
| ... | ... | @@ -14,7 +16,7 @@ module Hadrian.Utilities ( |
| 14 | 16 | |
| 15 | 17 | -- * Paths
|
| 16 | 18 | BuildRoot (..), buildRoot, buildRootRules, isGeneratedSource,
|
| 17 | - KeepResponseFiles (..), keepResponseFiles, withResponseFile, withResponseFileOnWindows,
|
|
| 19 | + withResponseFileIfLongCmd, responseFilePath,
|
|
| 18 | 20 | |
| 19 | 21 | -- * File system operations
|
| 20 | 22 | copyFile, copyFileUntracked, createFileLink, fixFile,
|
| ... | ... | @@ -47,11 +49,10 @@ import Data.Maybe |
| 47 | 49 | import Data.Typeable (TypeRep, typeOf)
|
| 48 | 50 | import Development.Shake hiding (Normal)
|
| 49 | 51 | import Development.Shake.Classes
|
| 52 | +import Development.Shake.Command (CmdArgument (..), IsCmdArgument (toCmdArgument))
|
|
| 50 | 53 | import Development.Shake.FilePath
|
| 51 | 54 | import GHC.ResponseFile (escapeArgs)
|
| 52 | 55 | import System.Environment (lookupEnv)
|
| 53 | -import System.Info.Extra (isWindows)
|
|
| 54 | -import System.IO (hClose, openTempFile)
|
|
| 55 | 56 | import System.IO.Error (isPermissionError)
|
| 56 | 57 | |
| 57 | 58 | import qualified Data.ByteString as BS
|
| ... | ... | @@ -255,13 +256,13 @@ infix 1 %%> |
| 255 | 256 | -- library, they can reach 2MB! Some operating systems do not support command
|
| 256 | 257 | -- lines of such length, and this function can be used to obtain a reasonable
|
| 257 | 258 | -- approximation of the limit. On Windows, it is theoretically 32768 characters
|
| 258 | --- (since Windows 7). In practice we use 31000 to leave some breathing space for
|
|
| 259 | +-- (since Windows 7). In practice we use 30000 to leave some breathing space for
|
|
| 259 | 260 | -- the builder path & name, auxiliary flags, and other overheads. On Mac OS X,
|
| 260 | 261 | -- ARG_MAX is 262144, yet when using @xargs@ on OSX this is reduced by over
|
| 261 | 262 | -- 20000. Hence, 200000 seems like a sensible limit. On other operating systems
|
| 262 | 263 | -- we currently use the 4194304 setting.
|
| 263 | 264 | cmdLineLengthLimit :: Int
|
| 264 | -cmdLineLengthLimit | IO.isWindows = 31000
|
|
| 265 | +cmdLineLengthLimit | IO.isWindows = 30000
|
|
| 265 | 266 | | IO.isMac = 200000
|
| 266 | 267 | | otherwise = 4194304
|
| 267 | 268 | |
| ... | ... | @@ -321,53 +322,35 @@ buildRootRules = do |
| 321 | 322 | isGeneratedSource :: FilePath -> Action Bool
|
| 322 | 323 | isGeneratedSource file = buildRoot <&> (`isPrefixOf` file)
|
| 323 | 324 | |
| 324 | -newtype KeepResponseFiles = KeepResponseFiles Bool deriving (Eq, Show)
|
|
| 325 | - |
|
| 326 | --- | Whether to retain response files after the build action that created them
|
|
| 327 | --- completes. Mainly useful for debugging.
|
|
| 328 | -keepResponseFiles :: Action Bool
|
|
| 329 | -keepResponseFiles = do
|
|
| 330 | - KeepResponseFiles keep <- userSetting (KeepResponseFiles False)
|
|
| 331 | - return keep
|
|
| 332 | - |
|
| 333 | --- | Run an action either with command arguments direcly or by, on Windows,
|
|
| 334 | --- placing those arguments into a response file escaped with @GHC.ResponseFile.escapeArgs@.
|
|
| 335 | ---
|
|
| 336 | --- With @--keep-response-files@, the file is left on disk (if used)
|
|
| 337 | -withResponseFileOnWindows ::
|
|
| 338 | - ([String] -> Action a) -- ^ Action to perform given arguments (of the form @["\@reponseFilePath"]@ on Windows)
|
|
| 339 | - -> [String] -- ^ Command arguments
|
|
| 340 | - -> Action a
|
|
| 341 | -withResponseFileOnWindows action commandArgs = do
|
|
| 342 | - if isWindows
|
|
| 343 | - then withResponseFile $ \tmp -> do
|
|
| 344 | - writeFile' tmp (escapeArgs commandArgs)
|
|
| 345 | - action ['@' : tmp]
|
|
| 346 | - else action commandArgs
|
|
| 347 | - |
|
| 348 | --- | Run an action with a response file path.
|
|
| 349 | ---
|
|
| 350 | --- With @--keep-response-files@, the file is left on disk.
|
|
| 351 | -withResponseFile :: (FilePath -> Action a) -> Action a
|
|
| 352 | -withResponseFile action = do
|
|
| 353 | - keep <- keepResponseFiles
|
|
| 354 | - let putVerboseResponseFile tmp = do
|
|
| 355 | - verbosity <- getVerbosity
|
|
| 356 | - when (verbosity >= Verbose) $ do
|
|
| 357 | - tmpContent <- liftIO (readFile tmp)
|
|
| 358 | - putVerbose (tmp <> " (use hadrian flag --keep-response-files to keep this file):\n" <> tmpContent)
|
|
| 359 | - if keep
|
|
| 360 | - then do
|
|
| 361 | - (tmp, h) <- liftIO $ openTempFile "." "hadrian-rsp"
|
|
| 362 | - liftIO $ hClose h
|
|
| 363 | - putInfo $ "Keeping response file: " ++ tmp
|
|
| 364 | - result <- action tmp
|
|
| 365 | - putVerboseResponseFile tmp
|
|
| 366 | - return result
|
|
| 367 | - else withTempFile $ \tmp -> do
|
|
| 368 | - result <- action tmp
|
|
| 369 | - putVerboseResponseFile tmp
|
|
| 370 | - return result
|
|
| 325 | +-- | Run an command with the given arguments. If the command is too long then the
|
|
| 326 | +-- response file arguments are placed into a response file and escaped with @GHC.ResponseFile.escapeArgs@.
|
|
| 327 | +withResponseFileIfLongCmd ::
|
|
| 328 | + CmdResult c
|
|
| 329 | + => FilePath -- ^ Response base name. The reponse file is placed in @_build/rsp/\<Response base name\>@.
|
|
| 330 | + -> CmdArgument -- ^ Command and arguments before the response file arguments.
|
|
| 331 | + -> [String] -- ^ Response file aruguments.
|
|
| 332 | + -> CmdArgument -- ^ Command arguments after the response file arguments.
|
|
| 333 | + -> Action c
|
|
| 334 | +withResponseFileIfLongCmd outputFilePath argsPre argsResp argsPost = do
|
|
| 335 | + let cmdLineLengh = sum
|
|
| 336 | + [ 1 + length arg -- add one to account for space inbetween arguments
|
|
| 337 | + | let CmdArgument args = argsPre <> toCmdArgument argsResp <> argsPost
|
|
| 338 | + , Right arg <- args
|
|
| 339 | + ]
|
|
| 340 | + if cmdLineLengh < cmdLineLengthLimit
|
|
| 341 | + then cmd argsPre argsResp argsPost
|
|
| 342 | + else do
|
|
| 343 | + rspFile <- responseFilePath outputFilePath
|
|
| 344 | + writeFile' rspFile (escapeArgs argsResp)
|
|
| 345 | + cmd argsPre ['@' : rspFile] argsPost
|
|
| 346 | + |
|
| 347 | +-- | Convert a command's output file path to a response file path to be used for that command.
|
|
| 348 | +-- Response files are placed in a dedicated @rps@ directory under the build directory. This avoids
|
|
| 349 | +-- clutering the work tree or interfearing with other build directories.
|
|
| 350 | +responseFilePath :: FilePath -> Action FilePath
|
|
| 351 | +responseFilePath outputFilePath = do
|
|
| 352 | + buildDir <- buildRoot
|
|
| 353 | + return $ buildDir </> "rsp" </> outputFilePath
|
|
| 371 | 354 | |
| 372 | 355 | -- | Link a file tracking the link target. Create the target directory if
|
| 373 | 356 | -- missing.
|