Marge Bot pushed to branch master at Glasgow Haskell Compiler / GHC
Commits:
498bb21a by David Eichmann at 2026-06-09T18:02:39-04:00
Hadrian: avoid response files when command line is short enough
This replaces the logic of always using response files on Windows.
With the new condition based on command line lenght, reponse files
can be avoided in many more cases (on windows).
Now that response files are only used in a small number of cases,
response files are always kept and the -r / --keep-response-files
command line options have been removed
The response file paths are nolonger randomized. They are placed in the
`_build/rsp` directory. This ensures they are ignored by git and we
that Hadrian reuses response file paths when rebuilding rather than
leaving stale response files around.
Update user guide putting response files in its own section
- - - - -
6 changed files:
- changelog.d/hadrian-response-files.md
- docs/users_guide/using.rst
- hadrian/src/Builder.hs
- hadrian/src/CommandLine.hs
- hadrian/src/Hadrian/Builder/Ar.hs
- hadrian/src/Hadrian/Utilities.hs
Changes:
=====================================
changelog.d/hadrian-response-files.md
=====================================
@@ -1,9 +1,15 @@
section: packaging
-synopsis: Add a flag to tell Hadrian to keep response files
-issues: #27184
-mrs: !15906
+synopsis: Improved Hadrian's use of response files
+issues: #27230
+mrs: !15906 !16134
description:
- Hadrian can now be instructed to keep response files with the new
- --keep-response-files command line flag. This is helpful when debugging a
- build failure, as it allows re-running the failing command line invocation
- without an error due to a missing response file.
+ Response files are files that contain command-line arguments. Hadrian uses
+ response files to shorten command-line lengths. This is important on Windows
+ where command-line lengths are limited.
+
+ Hadrian now supports response files when invoking GHC. In order to support
+ manually rerunning commands issued by Hadrian, response files are no longer
+ deleted. Instead they are stored under `_build/rsp`. Response files are now
+ only used when the corresponding command-line is too long for the host
+ platform. This greatly reduces the use of response files and avoids excessive
+ file usage. Response files are overwritten on subsequent Hadrian builds.
=====================================
docs/users_guide/using.rst
=====================================
@@ -85,17 +85,6 @@ all files; you cannot, for example, invoke
``ghc -c -O1 Foo.hs -O2 Bar.hs`` to apply different optimisation levels
to the files ``Foo.hs`` and ``Bar.hs``.
-In addition to passing arguments via the command-line, arguments can be passed
-via GNU-style response files. For instance,
-
-.. code-block:: bash
-
- $ cat response-file
- -O1
- Hello.hs
- -o Hello
- $ ghc @response-file
-
.. note::
.. index::
@@ -118,9 +107,24 @@ via GNU-style response files. For instance,
``-fspecialise`` will not be enabled, since the ``-fno-specialise``
overrides the ``-fspecialise`` implied by ``-O1``.
+
+Command-line arguments in response files
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In addition to passing arguments via the command-line, arguments can be passed
+via GNU-style response files. For instance,
+
+.. code-block:: bash
+
+ $ cat response-file
+ -O1
+ Hello.hs
+ -o Hello
+ $ ghc @response-file
+
.. _source-file-options:
-Command line options in source files
+Command-line options in source files
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. index::
=====================================
hadrian/src/Builder.hs
=====================================
@@ -304,7 +304,7 @@ instance H.Builder Builder where
case builder of
Ar Pack stg -> do
useTempFile <- arSupportsAtFile stg
- if useTempFile then runAr path buildArgs buildInputs buildOptions
+ if useTempFile then runAr output path buildArgs buildInputs buildOptions
else runArWithoutTempFile path buildArgs buildInputs buildOptions
Ar Unpack _ -> cmd' [Cwd output] [path] buildArgs buildOptions
@@ -343,7 +343,7 @@ instance H.Builder Builder where
Exit _ <- cmd' [path] (buildArgs ++ [input]) buildOptions
return ()
- Haddock BuildPackage -> runHaddock path buildArgs buildInputs
+ Haddock BuildPackage -> runHaddock output path buildArgs buildInputs
Ghc _ _ ->
-- Use a response file for ghc invocations to avoid issues with command line
@@ -351,9 +351,11 @@ instance H.Builder Builder where
-- NB: we can't put the buildArgs in a response file, because some flags require
-- empty arguments (such as the -dep-suffix flag), but that isn't supported
-- yet due to #26560.
- withResponseFileOnWindows
- (\buildInputs' -> cmd [path] buildArgs buildInputs' buildOptions)
+ withResponseFileIfLongCmd
+ output
+ (toCmdArgument [path] <> toCmdArgument buildArgs)
buildInputs
+ (toCmdArgument buildOptions)
HsCpp -> captureStdout
@@ -389,13 +391,16 @@ instance H.Builder Builder where
-- | Invoke @haddock@ given a path to it and a list of arguments. On Windows,
-- the input file arguments are passed as a response file.
-runHaddock :: FilePath -- ^ path to @haddock@
+runHaddock :: FilePath -- ^ base name to use for response file
+ -> FilePath -- ^ path to @haddock@
-> [String]
-> [FilePath] -- ^ input file paths
-> Action ()
-runHaddock haddockPath flagArgs fileInputs = withResponseFileOnWindows
- (cmd [haddockPath] flagArgs)
+runHaddock outputFilePath haddockPath flagArgs fileInputs = withResponseFileIfLongCmd
+ outputFilePath
+ (toCmdArgument [haddockPath] <> toCmdArgument flagArgs)
fileInputs
+ (CmdArgument [])
-- TODO: Some builders are required only on certain platforms. For example,
-- 'Objdump' is only required on OpenBSD and AIX. Add support for platform
=====================================
hadrian/src/CommandLine.hs
=====================================
@@ -3,8 +3,7 @@ module CommandLine (
lookupBignum,
cmdBignum, cmdProgressInfo, cmdCompleteSetting,
cmdDocsArgs, cmdUnitIdHash, lookupBuildRoot, TestArgs(..), TestSpeed(..), defaultTestArgs,
- cmdPrefix, cmdChangelogVersion, DocArgs(..), defaultDocArgs,
- cmdKeepResponseFiles
+ cmdPrefix, cmdChangelogVersion, DocArgs(..), defaultDocArgs
) where
import Data.Either
@@ -12,7 +11,7 @@ import qualified Data.HashMap.Strict as Map
import Data.List.Extra
import Development.Shake hiding (Normal)
import Flavour (DocTargets, DocTarget(..))
-import Hadrian.Utilities hiding (buildRoot, keepResponseFiles)
+import Hadrian.Utilities hiding (buildRoot)
import Settings.Parser
import System.Console.GetOpt
import System.Environment
@@ -37,7 +36,6 @@ data CommandLineArgs = CommandLineArgs
, testArgs :: TestArgs
, docsArgs :: DocArgs
, docTargets :: DocTargets
- , keepResponseFiles :: Bool
, prefix :: Maybe FilePath
, changelogVersion :: Maybe String
, completeStg :: Maybe String }
@@ -58,7 +56,6 @@ defaultCommandLineArgs = CommandLineArgs
, testArgs = defaultTestArgs
, docsArgs = defaultDocArgs
, docTargets = Set.fromList [minBound..maxBound]
- , keepResponseFiles = False
, prefix = Nothing
, changelogVersion = Nothing
, completeStg = Nothing }
@@ -141,9 +138,6 @@ readFreeze1 = Right $ \flags -> flags { freeze1 = True }
readFreeze2 = Right $ \flags -> flags { freeze1 = True, freeze2 = True }
readSkipDepends = Right $ \flags -> flags { skipDepends = True }
-readKeepResponseFiles :: Either String (CommandLineArgs -> CommandLineArgs)
-readKeepResponseFiles = Right $ \flags -> flags { keepResponseFiles = True }
-
readUnitIdHash :: Either String (CommandLineArgs -> CommandLineArgs)
readUnitIdHash = Right $ \flags ->
trace "--hash-unit-ids is deprecated. It is enabled by release flavour or +hash_unit_ids flavour transformer" $
@@ -302,8 +296,6 @@ optDescrs =
"Progress info style (None, Brief, Normal or Unicorn)."
, Option [] ["docs"] (ReqArg readDocsArg "TARGET")
"Strip down docs targets (none, no-haddocks, no-sphinx[-{html, pdfs, man}]."
- , Option ['r'] ["keep-response-files"] (NoArg readKeepResponseFiles)
- "Keep response files created during the build (for debugging)."
, Option ['k'] ["keep-test-files"] (NoArg readTestKeepFiles)
"Keep all the files generated when running the testsuite."
, Option [] ["test-compiler"] (ReqArg readTestCompiler "TEST_COMPILER")
@@ -382,7 +374,6 @@ cmdLineArgsMap = do
return $ insertExtra (progressInfo args) -- Accessed by Hadrian.Utilities
$ insertExtra (buildRoot args) -- Accessed by Hadrian.Utilities
- $ insertExtra (KeepResponseFiles $ keepResponseFiles args) -- Accessed by Hadrian.Utilities
$ insertExtra (testArgs args) -- Accessed by Settings.Builders.RunTest
$ insertExtra (docsArgs args) -- Accessed by Rules.Documentation
$ insertExtra allSettings -- Accessed by Settings
@@ -424,9 +415,6 @@ cmdUnitIdHash = unitIdHash <$> cmdLineArgs
cmdBignum :: Action (Maybe String)
cmdBignum = bignum <$> cmdLineArgs
-cmdKeepResponseFiles :: Action Bool
-cmdKeepResponseFiles = keepResponseFiles <$> cmdLineArgs
-
cmdProgressInfo :: Action ProgressInfo
cmdProgressInfo = progressInfo <$> cmdLineArgs
=====================================
hadrian/src/Hadrian/Builder/Ar.hs
=====================================
@@ -35,14 +35,16 @@ instance NFData ArMode
-- to be archived is passed via a temporary response file. Passing arguments
-- via a response file is not supported by some versions of @ar@, in which
-- case you should use 'runArWithoutTempFile' instead.
-runAr :: FilePath -- ^ path to @ar@
+runAr :: FilePath -- ^ base name to use for response files
+ -> FilePath -- ^ path to @ar@
-> [String] -- ^ other arguments
-> [FilePath] -- ^ input file paths
-> [CmdOption] -- ^ Additional options
-> Action ()
-runAr arPath flagArgs fileArgs buildOptions = withResponseFile $ \tmp -> do
- writeFile' tmp $ unwords fileArgs
- cmd [arPath] flagArgs ('@' : tmp) buildOptions
+runAr outputFilePath arPath flagArgs fileArgs buildOptions = do
+ rspFile <- responseFilePath outputFilePath
+ writeFile' rspFile $ unwords fileArgs
+ cmd [arPath] flagArgs ('@' : rspFile) buildOptions
-- | Invoke @ar@ given a path to it and a list of arguments. Note that @ar@
-- will be called multiple times if the list of files to be archived is too
=====================================
hadrian/src/Hadrian/Utilities.hs
=====================================
@@ -1,4 +1,6 @@
+{-# LANGUAGE ImpredicativeTypes #-}
{-# LANGUAGE TypeFamilies #-}
+
module Hadrian.Utilities (
-- * List manipulation
fromSingleton, replaceEq, minusOrd, intersectOrd, lookupAll, chunksOfSize,
@@ -14,7 +16,7 @@ module Hadrian.Utilities (
-- * Paths
BuildRoot (..), buildRoot, buildRootRules, isGeneratedSource,
- KeepResponseFiles (..), keepResponseFiles, withResponseFile, withResponseFileOnWindows,
+ withResponseFileIfLongCmd, responseFilePath,
-- * File system operations
copyFile, copyFileUntracked, createFileLink, fixFile,
@@ -47,11 +49,10 @@ import Data.Maybe
import Data.Typeable (TypeRep, typeOf)
import Development.Shake hiding (Normal)
import Development.Shake.Classes
+import Development.Shake.Command (CmdArgument (..), IsCmdArgument (toCmdArgument))
import Development.Shake.FilePath
import GHC.ResponseFile (escapeArgs)
import System.Environment (lookupEnv)
-import System.Info.Extra (isWindows)
-import System.IO (hClose, openTempFile)
import System.IO.Error (isPermissionError)
import qualified Data.ByteString as BS
@@ -255,13 +256,13 @@ infix 1 %%>
-- library, they can reach 2MB! Some operating systems do not support command
-- lines of such length, and this function can be used to obtain a reasonable
-- approximation of the limit. On Windows, it is theoretically 32768 characters
--- (since Windows 7). In practice we use 31000 to leave some breathing space for
+-- (since Windows 7). In practice we use 30000 to leave some breathing space for
-- the builder path & name, auxiliary flags, and other overheads. On Mac OS X,
-- ARG_MAX is 262144, yet when using @xargs@ on OSX this is reduced by over
-- 20000. Hence, 200000 seems like a sensible limit. On other operating systems
-- we currently use the 4194304 setting.
cmdLineLengthLimit :: Int
-cmdLineLengthLimit | IO.isWindows = 31000
+cmdLineLengthLimit | IO.isWindows = 30000
| IO.isMac = 200000
| otherwise = 4194304
@@ -321,53 +322,35 @@ buildRootRules = do
isGeneratedSource :: FilePath -> Action Bool
isGeneratedSource file = buildRoot <&> (`isPrefixOf` file)
-newtype KeepResponseFiles = KeepResponseFiles Bool deriving (Eq, Show)
-
--- | Whether to retain response files after the build action that created them
--- completes. Mainly useful for debugging.
-keepResponseFiles :: Action Bool
-keepResponseFiles = do
- KeepResponseFiles keep <- userSetting (KeepResponseFiles False)
- return keep
-
--- | Run an action either with command arguments direcly or by, on Windows,
--- placing those arguments into a response file escaped with @GHC.ResponseFile.escapeArgs@.
---
--- With @--keep-response-files@, the file is left on disk (if used)
-withResponseFileOnWindows ::
- ([String] -> Action a) -- ^ Action to perform given arguments (of the form @["\@reponseFilePath"]@ on Windows)
- -> [String] -- ^ Command arguments
- -> Action a
-withResponseFileOnWindows action commandArgs = do
- if isWindows
- then withResponseFile $ \tmp -> do
- writeFile' tmp (escapeArgs commandArgs)
- action ['@' : tmp]
- else action commandArgs
-
--- | Run an action with a response file path.
---
--- With @--keep-response-files@, the file is left on disk.
-withResponseFile :: (FilePath -> Action a) -> Action a
-withResponseFile action = do
- keep <- keepResponseFiles
- let putVerboseResponseFile tmp = do
- verbosity <- getVerbosity
- when (verbosity >= Verbose) $ do
- tmpContent <- liftIO (readFile tmp)
- putVerbose (tmp <> " (use hadrian flag --keep-response-files to keep this file):\n" <> tmpContent)
- if keep
- then do
- (tmp, h) <- liftIO $ openTempFile "." "hadrian-rsp"
- liftIO $ hClose h
- putInfo $ "Keeping response file: " ++ tmp
- result <- action tmp
- putVerboseResponseFile tmp
- return result
- else withTempFile $ \tmp -> do
- result <- action tmp
- putVerboseResponseFile tmp
- return result
+-- | Run an command with the given arguments. If the command is too long then the
+-- response file arguments are placed into a response file and escaped with @GHC.ResponseFile.escapeArgs@.
+withResponseFileIfLongCmd ::
+ CmdResult c
+ => FilePath -- ^ Response base name. The reponse file is placed in @_build/rsp/\