David Eichmann pushed to branch wip/davide/hadrian_avoid_response_files_2 at Glasgow Haskell Compiler / GHC
Commits:
-
e0ab52ce
by David Eichmann at 2026-06-03T16:37:53+01:00
5 changed files:
- − changelog.d/hadrian-response-files.md
- hadrian/src/Builder.hs
- hadrian/src/CommandLine.hs
- hadrian/src/Hadrian/Builder/Ar.hs
- hadrian/src/Hadrian/Utilities.hs
Changes:
| 1 | -section: packaging
|
|
| 2 | -synopsis: Add a flag to tell Hadrian to keep response files
|
|
| 3 | -issues: #27184
|
|
| 4 | -mrs: !15906
|
|
| 5 | -description:
|
|
| 6 | - Hadrian can now be instructed to keep response files with the new
|
|
| 7 | - --keep-response-files command line flag. This is helpful when debugging a
|
|
| 8 | - build failure, as it allows re-running the failing command line invocation
|
|
| 9 | - without an error due to a missing response file. |
| ... | ... | @@ -304,7 +304,7 @@ instance H.Builder Builder where |
| 304 | 304 | case builder of
|
| 305 | 305 | Ar Pack stg -> do
|
| 306 | 306 | useTempFile <- arSupportsAtFile stg
|
| 307 | - if useTempFile then runAr path buildArgs buildInputs buildOptions
|
|
| 307 | + if useTempFile then runAr output path buildArgs buildInputs buildOptions
|
|
| 308 | 308 | else runArWithoutTempFile path buildArgs buildInputs buildOptions
|
| 309 | 309 | |
| 310 | 310 | Ar Unpack _ -> cmd' [Cwd output] [path] buildArgs buildOptions
|
| ... | ... | @@ -343,7 +343,7 @@ instance H.Builder Builder where |
| 343 | 343 | Exit _ <- cmd' [path] (buildArgs ++ [input]) buildOptions
|
| 344 | 344 | return ()
|
| 345 | 345 | |
| 346 | - Haddock BuildPackage -> runHaddock path buildArgs buildInputs
|
|
| 346 | + Haddock BuildPackage -> runHaddock output path buildArgs buildInputs
|
|
| 347 | 347 | |
| 348 | 348 | Ghc _ _ ->
|
| 349 | 349 | -- Use a response file for ghc invocations to avoid issues with command line
|
| ... | ... | @@ -352,6 +352,7 @@ instance H.Builder Builder where |
| 352 | 352 | -- empty arguments (such as the -dep-suffix flag), but that isn't supported
|
| 353 | 353 | -- yet due to #26560.
|
| 354 | 354 | withResponseFileIfLongCmd
|
| 355 | + output
|
|
| 355 | 356 | (toCmdArgument [path] <> toCmdArgument buildArgs)
|
| 356 | 357 | buildInputs
|
| 357 | 358 | (toCmdArgument buildOptions)
|
| ... | ... | @@ -390,11 +391,13 @@ instance H.Builder Builder where |
| 390 | 391 | |
| 391 | 392 | -- | Invoke @haddock@ given a path to it and a list of arguments. On Windows,
|
| 392 | 393 | -- the input file arguments are passed as a response file.
|
| 393 | -runHaddock :: FilePath -- ^ path to @haddock@
|
|
| 394 | +runHaddock :: FilePath -- ^ output file path
|
|
| 395 | + -> FilePath -- ^ path to @haddock@
|
|
| 394 | 396 | -> [String]
|
| 395 | 397 | -> [FilePath] -- ^ input file paths
|
| 396 | 398 | -> Action ()
|
| 397 | -runHaddock haddockPath flagArgs fileInputs = withResponseFileIfLongCmd
|
|
| 399 | +runHaddock outputFilePath haddockPath flagArgs fileInputs = withResponseFileIfLongCmd
|
|
| 400 | + outputFilePath
|
|
| 398 | 401 | (toCmdArgument [haddockPath] <> toCmdArgument flagArgs)
|
| 399 | 402 | fileInputs
|
| 400 | 403 | (CmdArgument [])
|
| ... | ... | @@ -3,8 +3,7 @@ module CommandLine ( |
| 3 | 3 | lookupBignum,
|
| 4 | 4 | cmdBignum, cmdProgressInfo, cmdCompleteSetting,
|
| 5 | 5 | cmdDocsArgs, cmdUnitIdHash, lookupBuildRoot, TestArgs(..), TestSpeed(..), defaultTestArgs,
|
| 6 | - cmdPrefix, cmdChangelogVersion, DocArgs(..), defaultDocArgs,
|
|
| 7 | - cmdKeepResponseFiles
|
|
| 6 | + cmdPrefix, cmdChangelogVersion, DocArgs(..), defaultDocArgs
|
|
| 8 | 7 | ) where
|
| 9 | 8 | |
| 10 | 9 | import Data.Either
|
| ... | ... | @@ -12,7 +11,7 @@ import qualified Data.HashMap.Strict as Map |
| 12 | 11 | import Data.List.Extra
|
| 13 | 12 | import Development.Shake hiding (Normal)
|
| 14 | 13 | import Flavour (DocTargets, DocTarget(..))
|
| 15 | -import Hadrian.Utilities hiding (buildRoot, keepResponseFiles)
|
|
| 14 | +import Hadrian.Utilities hiding (buildRoot)
|
|
| 16 | 15 | import Settings.Parser
|
| 17 | 16 | import System.Console.GetOpt
|
| 18 | 17 | import System.Environment
|
| ... | ... | @@ -37,7 +36,6 @@ data CommandLineArgs = CommandLineArgs |
| 37 | 36 | , testArgs :: TestArgs
|
| 38 | 37 | , docsArgs :: DocArgs
|
| 39 | 38 | , docTargets :: DocTargets
|
| 40 | - , keepResponseFiles :: Bool
|
|
| 41 | 39 | , prefix :: Maybe FilePath
|
| 42 | 40 | , changelogVersion :: Maybe String
|
| 43 | 41 | , completeStg :: Maybe String }
|
| ... | ... | @@ -58,7 +56,6 @@ defaultCommandLineArgs = CommandLineArgs |
| 58 | 56 | , testArgs = defaultTestArgs
|
| 59 | 57 | , docsArgs = defaultDocArgs
|
| 60 | 58 | , docTargets = Set.fromList [minBound..maxBound]
|
| 61 | - , keepResponseFiles = False
|
|
| 62 | 59 | , prefix = Nothing
|
| 63 | 60 | , changelogVersion = Nothing
|
| 64 | 61 | , completeStg = Nothing }
|
| ... | ... | @@ -141,9 +138,6 @@ readFreeze1 = Right $ \flags -> flags { freeze1 = True } |
| 141 | 138 | readFreeze2 = Right $ \flags -> flags { freeze1 = True, freeze2 = True }
|
| 142 | 139 | readSkipDepends = Right $ \flags -> flags { skipDepends = True }
|
| 143 | 140 | |
| 144 | -readKeepResponseFiles :: Either String (CommandLineArgs -> CommandLineArgs)
|
|
| 145 | -readKeepResponseFiles = Right $ \flags -> flags { keepResponseFiles = True }
|
|
| 146 | - |
|
| 147 | 141 | readUnitIdHash :: Either String (CommandLineArgs -> CommandLineArgs)
|
| 148 | 142 | readUnitIdHash = Right $ \flags ->
|
| 149 | 143 | trace "--hash-unit-ids is deprecated. It is enabled by release flavour or +hash_unit_ids flavour transformer" $
|
| ... | ... | @@ -302,8 +296,6 @@ optDescrs = |
| 302 | 296 | "Progress info style (None, Brief, Normal or Unicorn)."
|
| 303 | 297 | , Option [] ["docs"] (ReqArg readDocsArg "TARGET")
|
| 304 | 298 | "Strip down docs targets (none, no-haddocks, no-sphinx[-{html, pdfs, man}]."
|
| 305 | - , Option ['r'] ["keep-response-files"] (NoArg readKeepResponseFiles)
|
|
| 306 | - "Keep response files created during the build (for debugging)."
|
|
| 307 | 299 | , Option ['k'] ["keep-test-files"] (NoArg readTestKeepFiles)
|
| 308 | 300 | "Keep all the files generated when running the testsuite."
|
| 309 | 301 | , Option [] ["test-compiler"] (ReqArg readTestCompiler "TEST_COMPILER")
|
| ... | ... | @@ -382,7 +374,6 @@ cmdLineArgsMap = do |
| 382 | 374 | |
| 383 | 375 | return $ insertExtra (progressInfo args) -- Accessed by Hadrian.Utilities
|
| 384 | 376 | $ insertExtra (buildRoot args) -- Accessed by Hadrian.Utilities
|
| 385 | - $ insertExtra (KeepResponseFiles $ keepResponseFiles args) -- Accessed by Hadrian.Utilities
|
|
| 386 | 377 | $ insertExtra (testArgs args) -- Accessed by Settings.Builders.RunTest
|
| 387 | 378 | $ insertExtra (docsArgs args) -- Accessed by Rules.Documentation
|
| 388 | 379 | $ insertExtra allSettings -- Accessed by Settings
|
| ... | ... | @@ -424,9 +415,6 @@ cmdUnitIdHash = unitIdHash <$> cmdLineArgs |
| 424 | 415 | cmdBignum :: Action (Maybe String)
|
| 425 | 416 | cmdBignum = bignum <$> cmdLineArgs
|
| 426 | 417 | |
| 427 | -cmdKeepResponseFiles :: Action Bool
|
|
| 428 | -cmdKeepResponseFiles = keepResponseFiles <$> cmdLineArgs
|
|
| 429 | - |
|
| 430 | 418 | cmdProgressInfo :: Action ProgressInfo
|
| 431 | 419 | cmdProgressInfo = progressInfo <$> cmdLineArgs
|
| 432 | 420 |
| ... | ... | @@ -35,14 +35,16 @@ instance NFData ArMode |
| 35 | 35 | -- to be archived is passed via a temporary response file. Passing arguments
|
| 36 | 36 | -- via a response file is not supported by some versions of @ar@, in which
|
| 37 | 37 | -- case you should use 'runArWithoutTempFile' instead.
|
| 38 | -runAr :: FilePath -- ^ path to @ar@
|
|
| 38 | +runAr :: FilePath -- ^ output file path
|
|
| 39 | + -> FilePath -- ^ path to @ar@
|
|
| 39 | 40 | -> [String] -- ^ other arguments
|
| 40 | 41 | -> [FilePath] -- ^ input file paths
|
| 41 | 42 | -> [CmdOption] -- ^ Additional options
|
| 42 | 43 | -> Action ()
|
| 43 | -runAr arPath flagArgs fileArgs buildOptions = withResponseFile $ \tmp -> do
|
|
| 44 | - writeFile' tmp $ unwords fileArgs
|
|
| 45 | - cmd [arPath] flagArgs ('@' : tmp) buildOptions
|
|
| 44 | +runAr outputFilePath arPath flagArgs fileArgs buildOptions = do
|
|
| 45 | + rspFile <- responseFilePath outputFilePath
|
|
| 46 | + writeFile' rspFile $ unwords fileArgs
|
|
| 47 | + cmd [arPath] flagArgs ('@' : rspFile) buildOptions
|
|
| 46 | 48 | |
| 47 | 49 | -- | Invoke @ar@ given a path to it and a list of arguments. Note that @ar@
|
| 48 | 50 | -- will be called multiple times if the list of files to be archived is too
|
| ... | ... | @@ -16,7 +16,7 @@ module Hadrian.Utilities ( |
| 16 | 16 | |
| 17 | 17 | -- * Paths
|
| 18 | 18 | BuildRoot (..), buildRoot, buildRootRules, isGeneratedSource,
|
| 19 | - KeepResponseFiles (..), keepResponseFiles, withResponseFile, withResponseFileIfLongCmd,
|
|
| 19 | + withResponseFileIfLongCmd, responseFilePath,
|
|
| 20 | 20 | |
| 21 | 21 | -- * File system operations
|
| 22 | 22 | copyFile, copyFileUntracked, createFileLink, fixFile,
|
| ... | ... | @@ -49,10 +49,10 @@ import Data.Maybe |
| 49 | 49 | import Data.Typeable (TypeRep, typeOf)
|
| 50 | 50 | import Development.Shake hiding (Normal)
|
| 51 | 51 | import Development.Shake.Classes
|
| 52 | +import Development.Shake.Command (CmdArgument (..), IsCmdArgument (toCmdArgument))
|
|
| 52 | 53 | import Development.Shake.FilePath
|
| 53 | 54 | import GHC.ResponseFile (escapeArgs)
|
| 54 | 55 | import System.Environment (lookupEnv)
|
| 55 | -import System.IO (hClose, openTempFile)
|
|
| 56 | 56 | import System.IO.Error (isPermissionError)
|
| 57 | 57 | |
| 58 | 58 | import qualified Data.ByteString as BS
|
| ... | ... | @@ -62,7 +62,6 @@ import qualified System.Directory.Extra as IO |
| 62 | 62 | import qualified System.Info.Extra as IO
|
| 63 | 63 | import qualified System.IO as IO
|
| 64 | 64 | import qualified System.FilePath.Posix as Posix
|
| 65 | -import Development.Shake.Command (CmdArgument (..), IsCmdArgument (toCmdArgument))
|
|
| 66 | 65 | |
| 67 | 66 | -- | Extract a value from a singleton list, or terminate with an error message
|
| 68 | 67 | -- if the list does not contain exactly one value.
|
| ... | ... | @@ -323,59 +322,32 @@ buildRootRules = do |
| 323 | 322 | isGeneratedSource :: FilePath -> Action Bool
|
| 324 | 323 | isGeneratedSource file = buildRoot <&> (`isPrefixOf` file)
|
| 325 | 324 | |
| 326 | -newtype KeepResponseFiles = KeepResponseFiles Bool deriving (Eq, Show)
|
|
| 327 | - |
|
| 328 | --- | Whether to retain response files after the build action that created them
|
|
| 329 | --- completes. Mainly useful for debugging.
|
|
| 330 | -keepResponseFiles :: Action Bool
|
|
| 331 | -keepResponseFiles = do
|
|
| 332 | - KeepResponseFiles keep <- userSetting (KeepResponseFiles False)
|
|
| 333 | - return keep
|
|
| 334 | - |
|
| 335 | 325 | -- | Run an command with the given arguments. If the command is too long then the
|
| 336 | 326 | -- response file arguments are placed into a response file and escaped with @GHC.ResponseFile.escapeArgs@.
|
| 337 | ---
|
|
| 338 | --- With @--keep-response-files@, the file is left on disk (if used)
|
|
| 339 | 327 | withResponseFileIfLongCmd :: (CmdResult c) =>
|
| 340 | - CmdArgument -- ^ Command and arguments before the response file arguments.
|
|
| 328 | + FilePath -- ^ Command output file path. The reponse file is placed in _build/rsp/<Command output file path>.
|
|
| 329 | + -> CmdArgument -- ^ Command and arguments before the response file arguments.
|
|
| 341 | 330 | -> [String] -- ^ Response file aruguments.
|
| 342 | 331 | -> CmdArgument -- ^ Command arguments after the response file arguments.
|
| 343 | 332 | -> Action c
|
| 344 | -withResponseFileIfLongCmd argsPre argsResp argsPost = do
|
|
| 333 | +withResponseFileIfLongCmd outputFilePath argsPre argsResp argsPost = do
|
|
| 345 | 334 | let cmdLineLengh = sum
|
| 346 | 335 | [ 1 + length arg -- add one to account for space inbetween arguments
|
| 347 | 336 | | let CmdArgument args = argsPre <> toCmdArgument argsResp <> argsPost
|
| 348 | 337 | , Right arg <- args
|
| 349 | 338 | ]
|
| 350 | - if cmdLineLengh >= cmdLineLengthLimit
|
|
| 351 | - then withResponseFile $ \tmp -> do
|
|
| 352 | - writeFile' tmp (escapeArgs argsResp)
|
|
| 353 | - cmd argsPre ['@' : tmp] argsPost
|
|
| 354 | - else cmd argsPre argsResp argsPost
|
|
| 355 | - |
|
| 356 | --- | Run an action with a response file path.
|
|
| 357 | ---
|
|
| 358 | --- With @--keep-response-files@, the file is left on disk.
|
|
| 359 | -withResponseFile :: (FilePath -> Action a) -> Action a
|
|
| 360 | -withResponseFile action = do
|
|
| 361 | - keep <- keepResponseFiles
|
|
| 362 | - let putVerboseResponseFile tmp = do
|
|
| 363 | - verbosity <- getVerbosity
|
|
| 364 | - when (verbosity >= Verbose) $ do
|
|
| 365 | - tmpContent <- liftIO (readFile tmp)
|
|
| 366 | - putVerbose (tmp <> " (use hadrian flag --keep-response-files to keep this file):\n" <> tmpContent)
|
|
| 367 | - if keep
|
|
| 368 | - then do
|
|
| 369 | - (tmp, h) <- liftIO $ openTempFile "." "hadrian-rsp"
|
|
| 370 | - liftIO $ hClose h
|
|
| 371 | - putInfo $ "Keeping response file: " ++ tmp
|
|
| 372 | - result <- action tmp
|
|
| 373 | - putVerboseResponseFile tmp
|
|
| 374 | - return result
|
|
| 375 | - else withTempFile $ \tmp -> do
|
|
| 376 | - result <- action tmp
|
|
| 377 | - putVerboseResponseFile tmp
|
|
| 378 | - return result
|
|
| 339 | + if cmdLineLengh < cmdLineLengthLimit
|
|
| 340 | + then cmd argsPre argsResp argsPost
|
|
| 341 | + else do
|
|
| 342 | + rspFile <- responseFilePath outputFilePath
|
|
| 343 | + writeFile' rspFile (escapeArgs argsResp)
|
|
| 344 | + cmd argsPre ['@' : rspFile] argsPost
|
|
| 345 | + |
|
| 346 | +-- | Convert a command's output file path to a response file path to be used for that command.
|
|
| 347 | +responseFilePath :: FilePath -> Action FilePath
|
|
| 348 | +responseFilePath outputFilePath = do
|
|
| 349 | + buildDir <- buildRoot
|
|
| 350 | + return $ buildDir </> "rsp" </> outputFilePath
|
|
| 379 | 351 | |
| 380 | 352 | -- | Link a file tracking the link target. Create the target directory if
|
| 381 | 353 | -- missing.
|