Marge Bot pushed to branch wip/marge_bot_batch_merge_job at Glasgow Haskell Compiler / GHC
Commits:
23a13665 by David Eichmann at 2026-06-09T11:22:05-04:00
Hadrian: avoid response files when command line is short enough
This replaces the logic of always using response files on Windows.
With the new condition based on command line lenght, reponse files
can be avoided in many more cases (on windows).
Now that response files are only used in a small number of cases,
response files are always kept and the -r / --keep-response-files
command line options have been removed
The response file paths are nolonger randomized. They are placed in the
`_build/rsp` directory. This ensures they are ignored by git and we
that Hadrian reuses response file paths when rebuilding rather than
leaving stale response files around.
Update user guide putting response files in its own section
- - - - -
648aa8e5 by Simon Hengel at 2026-06-09T11:22:06-04:00
Don't use non-breaking spaces
- - - - -
5d75f13c by David Eichmann at 2026-06-09T11:22:06-04:00
Hadrian: remove unused wrapper scripts from windows bindist
These wrapper scripts are only installed on non-relocatable builds
which are not generally supported on windows.
- - - - -
14 changed files:
- changelog.d/hadrian-response-files.md
- compiler/GHC/Driver/Backend.hs
- compiler/GHC/Parser/PostProcess/Haddock.hs
- compiler/GHC/Runtime/Debugger/Breakpoints.hs
- compiler/GHC/Runtime/Eval/Types.hs
- compiler/GHC/Runtime/Heap/Inspect.hs
- compiler/GHC/Utils/Logger.hs
- docs/users_guide/using.rst
- hadrian/src/Builder.hs
- hadrian/src/CommandLine.hs
- hadrian/src/Hadrian/Builder/Ar.hs
- hadrian/src/Hadrian/Utilities.hs
- hadrian/src/Rules/BinaryDist.hs
- libraries/ghc-internal/src/GHC/Internal/Lexeme.hs
Changes:
=====================================
changelog.d/hadrian-response-files.md
=====================================
@@ -1,9 +1,15 @@
section: packaging
-synopsis: Add a flag to tell Hadrian to keep response files
-issues: #27184
-mrs: !15906
+synopsis: Improved Hadrian's use of response files
+issues: #27230
+mrs: !15906 !16134
description:
- Hadrian can now be instructed to keep response files with the new
- --keep-response-files command line flag. This is helpful when debugging a
- build failure, as it allows re-running the failing command line invocation
- without an error due to a missing response file.
+ Response files are files that contain command-line arguments. Hadrian uses
+ response files to shorten command-line lengths. This is important on Windows
+ where command-line lengths are limited.
+
+ Hadrian now supports response files when invoking GHC. In order to support
+ manually rerunning commands issued by Hadrian, response files are no longer
+ deleted. Instead they are stored under `_build/rsp`. Response files are now
+ only used when the corresponding command-line is too long for the host
+ platform. This greatly reduces the use of response files and avoids excessive
+ file usage. Response files are overwritten on subsequent Hadrian builds.
=====================================
compiler/GHC/Driver/Backend.hs
=====================================
@@ -184,7 +184,7 @@ import GHC.Platform
-- about enumerating them. Just one set of error messages has been
-- ported to have an open-world assumption: these are the error
-- messages associated with type checking of foreign imports and
--- exports. To allow other errors to be issued with an open-world
+-- exports. To allow other errors to be issued with an open-world
-- assumption, use functions `backendValidityOfCImport` and
-- `backendValidityOfCExport` as models, and have a look at how the
-- 'expected back ends' are used in modules "GHC.Tc.Gen.Foreign" and
@@ -225,11 +225,11 @@ platformJSSupported platform
| otherwise = False
--- | A value of type @Backend@ represents one of GHC's back ends.
+-- | A value of type @Backend@ represents one of GHC's back ends.
-- The set of back ends cannot be extended except by modifying the
-- definition of @Backend@ in this module.
--
--- The @Backend@ type is abstract; that is, its value constructors are
+-- The @Backend@ type is abstract; that is, its value constructors are
-- not exported. It's crucial that they not be exported, because a
-- value of type @Backend@ carries only the back end's /name/, not its
-- behavior or properties. If @Backend@ were not abstract, then code
=====================================
compiler/GHC/Parser/PostProcess/Haddock.hs
=====================================
@@ -289,7 +289,7 @@ instance HasHaddock (Located (HsModule GhcPs)) where
pure $ L l_mod $
mod { hsmodExports = hsmodExports'
, hsmodDecls = hsmodDecls'
- , hsmodExt = (hsmodExt mod) { hsmodHaddockModHeader = headerDocs } }
+ , hsmodExt = (hsmodExt mod) { hsmodHaddockModHeader = headerDocs } }
lexHsDocString :: HsDocString -> HsDoc GhcPs
lexHsDocString = lexHsDoc parseIdentifier
=====================================
compiler/GHC/Runtime/Debugger/Breakpoints.hs
=====================================
@@ -177,7 +177,7 @@ resolveFunctionBreakpoint inp = do
-- for
-- (a) this binder only (it maybe a top-level or a nested declaration)
-- (b) that do not have an enclosing breakpoint
-findBreakForBind :: String {-^ Name of bind to break at -} -> ModBreaks -> [(BreakTickIndex, RealSrcSpan)]
+findBreakForBind :: String {-^ Name of bind to break at -} -> ModBreaks -> [(BreakTickIndex, RealSrcSpan)]
findBreakForBind str_name modbreaks = filter (not . enclosed) ticks
where
ticks = [ (index, span)
@@ -226,7 +226,7 @@ getModBreak m = do
-- source breakpoint, it means all *ocurrences* of that breakpoint across
-- modules should be stopped at -- hence we keep a trie from BreakpointId to
-- the list of internal break ids using it.
--- See also Note [Breakpoint identifiers]
+-- See also Note [Breakpoint identifiers]
type BreakpointOccurrences = ModuleEnv (IntMap.IntMap [InternalBreakpointId])
-- | Lookup all InternalBreakpointIds matching the given BreakpointId
=====================================
compiler/GHC/Runtime/Eval/Types.hs
=====================================
@@ -75,7 +75,7 @@ enableGhcStepMode _ = EvalStepSingle
-- and the SrcSpan of a breakpoint we hit, return @True@ if we should stop at
-- this breakpoint.
--
--- In particular, this will always be @False@ for @'RunToCompletion'@ and
+-- In particular, this will always be @False@ for @'RunToCompletion'@ and
-- @'RunAndLogSteps'@. We'd need further information e.g. about the user
-- breakpoints to determine whether to break in those modes.
breakHere :: Bool -- ^ Was this breakpoint explicitly active (in the @BreakArray@s)?
=====================================
compiler/GHC/Runtime/Heap/Inspect.hs
=====================================
@@ -293,22 +293,22 @@ ppr_termM1 Prim{valRaw=words, ty=ty} =
return $ repPrim (tyConAppTyCon ty) words
ppr_termM1 Suspension{ty=ty, bound_to=Nothing, infoprov=mipe} =
return $ hcat $
- [ char '_'
- , whenPprDebug $
+ [ char '_'
+ , whenPprDebug $
space <>
dcolon <>
pprSigmaType ty
- ] ++
- [ whenPprDebug $
+ ] ++
+ [ whenPprDebug $
space <>
char '<' <>
text (ipSrcFile ipe) <>
char ':' <>
text (ipSrcSpan ipe) <>
char '>'
- | Just ipe <- [mipe]
+ | Just ipe <- [mipe]
, not $ null $ ipSrcFile ipe
- ]
+ ]
ppr_termM1 Suspension{ty=ty, bound_to=Just n}
| otherwise = return$ parens$ ppr n <> dcolon <> pprSigmaType ty
ppr_termM1 Term{} = panic "ppr_termM1 - Term"
=====================================
compiler/GHC/Utils/Logger.hs
=====================================
@@ -413,7 +413,7 @@ defaultLogAction = defaultLogActionWithHandles stdout stderr
-- | The default 'LogAction' parametrized over the standard output and standard error handles.
-- Allows clients to replicate the log message formatting of GHC with custom handles.
-defaultLogActionWithHandles :: Handle {-^ Handle for standard output -} -> Handle {-^ Handle for standard errors -} -> LogAction
+defaultLogActionWithHandles :: Handle {-^ Handle for standard output -} -> Handle {-^ Handle for standard errors -} -> LogAction
defaultLogActionWithHandles out err logflags msg_class srcSpan msg
| log_dopt Opt_D_dump_json logflags = jsonLogActionWithHandle out logflags msg_class srcSpan msg
| otherwise = case msg_class of
=====================================
docs/users_guide/using.rst
=====================================
@@ -85,17 +85,6 @@ all files; you cannot, for example, invoke
``ghc -c -O1 Foo.hs -O2 Bar.hs`` to apply different optimisation levels
to the files ``Foo.hs`` and ``Bar.hs``.
-In addition to passing arguments via the command-line, arguments can be passed
-via GNU-style response files. For instance,
-
-.. code-block:: bash
-
- $ cat response-file
- -O1
- Hello.hs
- -o Hello
- $ ghc @response-file
-
.. note::
.. index::
@@ -118,9 +107,24 @@ via GNU-style response files. For instance,
``-fspecialise`` will not be enabled, since the ``-fno-specialise``
overrides the ``-fspecialise`` implied by ``-O1``.
+
+Command-line arguments in response files
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In addition to passing arguments via the command-line, arguments can be passed
+via GNU-style response files. For instance,
+
+.. code-block:: bash
+
+ $ cat response-file
+ -O1
+ Hello.hs
+ -o Hello
+ $ ghc @response-file
+
.. _source-file-options:
-Command line options in source files
+Command-line options in source files
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. index::
=====================================
hadrian/src/Builder.hs
=====================================
@@ -304,7 +304,7 @@ instance H.Builder Builder where
case builder of
Ar Pack stg -> do
useTempFile <- arSupportsAtFile stg
- if useTempFile then runAr path buildArgs buildInputs buildOptions
+ if useTempFile then runAr output path buildArgs buildInputs buildOptions
else runArWithoutTempFile path buildArgs buildInputs buildOptions
Ar Unpack _ -> cmd' [Cwd output] [path] buildArgs buildOptions
@@ -343,7 +343,7 @@ instance H.Builder Builder where
Exit _ <- cmd' [path] (buildArgs ++ [input]) buildOptions
return ()
- Haddock BuildPackage -> runHaddock path buildArgs buildInputs
+ Haddock BuildPackage -> runHaddock output path buildArgs buildInputs
Ghc _ _ ->
-- Use a response file for ghc invocations to avoid issues with command line
@@ -351,9 +351,11 @@ instance H.Builder Builder where
-- NB: we can't put the buildArgs in a response file, because some flags require
-- empty arguments (such as the -dep-suffix flag), but that isn't supported
-- yet due to #26560.
- withResponseFileOnWindows
- (\buildInputs' -> cmd [path] buildArgs buildInputs' buildOptions)
+ withResponseFileIfLongCmd
+ output
+ (toCmdArgument [path] <> toCmdArgument buildArgs)
buildInputs
+ (toCmdArgument buildOptions)
HsCpp -> captureStdout
@@ -389,13 +391,16 @@ instance H.Builder Builder where
-- | Invoke @haddock@ given a path to it and a list of arguments. On Windows,
-- the input file arguments are passed as a response file.
-runHaddock :: FilePath -- ^ path to @haddock@
+runHaddock :: FilePath -- ^ base name to use for response file
+ -> FilePath -- ^ path to @haddock@
-> [String]
-> [FilePath] -- ^ input file paths
-> Action ()
-runHaddock haddockPath flagArgs fileInputs = withResponseFileOnWindows
- (cmd [haddockPath] flagArgs)
+runHaddock outputFilePath haddockPath flagArgs fileInputs = withResponseFileIfLongCmd
+ outputFilePath
+ (toCmdArgument [haddockPath] <> toCmdArgument flagArgs)
fileInputs
+ (CmdArgument [])
-- TODO: Some builders are required only on certain platforms. For example,
-- 'Objdump' is only required on OpenBSD and AIX. Add support for platform
=====================================
hadrian/src/CommandLine.hs
=====================================
@@ -3,8 +3,7 @@ module CommandLine (
lookupBignum,
cmdBignum, cmdProgressInfo, cmdCompleteSetting,
cmdDocsArgs, cmdUnitIdHash, lookupBuildRoot, TestArgs(..), TestSpeed(..), defaultTestArgs,
- cmdPrefix, cmdChangelogVersion, DocArgs(..), defaultDocArgs,
- cmdKeepResponseFiles
+ cmdPrefix, cmdChangelogVersion, DocArgs(..), defaultDocArgs
) where
import Data.Either
@@ -12,7 +11,7 @@ import qualified Data.HashMap.Strict as Map
import Data.List.Extra
import Development.Shake hiding (Normal)
import Flavour (DocTargets, DocTarget(..))
-import Hadrian.Utilities hiding (buildRoot, keepResponseFiles)
+import Hadrian.Utilities hiding (buildRoot)
import Settings.Parser
import System.Console.GetOpt
import System.Environment
@@ -37,7 +36,6 @@ data CommandLineArgs = CommandLineArgs
, testArgs :: TestArgs
, docsArgs :: DocArgs
, docTargets :: DocTargets
- , keepResponseFiles :: Bool
, prefix :: Maybe FilePath
, changelogVersion :: Maybe String
, completeStg :: Maybe String }
@@ -58,7 +56,6 @@ defaultCommandLineArgs = CommandLineArgs
, testArgs = defaultTestArgs
, docsArgs = defaultDocArgs
, docTargets = Set.fromList [minBound..maxBound]
- , keepResponseFiles = False
, prefix = Nothing
, changelogVersion = Nothing
, completeStg = Nothing }
@@ -141,9 +138,6 @@ readFreeze1 = Right $ \flags -> flags { freeze1 = True }
readFreeze2 = Right $ \flags -> flags { freeze1 = True, freeze2 = True }
readSkipDepends = Right $ \flags -> flags { skipDepends = True }
-readKeepResponseFiles :: Either String (CommandLineArgs -> CommandLineArgs)
-readKeepResponseFiles = Right $ \flags -> flags { keepResponseFiles = True }
-
readUnitIdHash :: Either String (CommandLineArgs -> CommandLineArgs)
readUnitIdHash = Right $ \flags ->
trace "--hash-unit-ids is deprecated. It is enabled by release flavour or +hash_unit_ids flavour transformer" $
@@ -302,8 +296,6 @@ optDescrs =
"Progress info style (None, Brief, Normal or Unicorn)."
, Option [] ["docs"] (ReqArg readDocsArg "TARGET")
"Strip down docs targets (none, no-haddocks, no-sphinx[-{html, pdfs, man}]."
- , Option ['r'] ["keep-response-files"] (NoArg readKeepResponseFiles)
- "Keep response files created during the build (for debugging)."
, Option ['k'] ["keep-test-files"] (NoArg readTestKeepFiles)
"Keep all the files generated when running the testsuite."
, Option [] ["test-compiler"] (ReqArg readTestCompiler "TEST_COMPILER")
@@ -382,7 +374,6 @@ cmdLineArgsMap = do
return $ insertExtra (progressInfo args) -- Accessed by Hadrian.Utilities
$ insertExtra (buildRoot args) -- Accessed by Hadrian.Utilities
- $ insertExtra (KeepResponseFiles $ keepResponseFiles args) -- Accessed by Hadrian.Utilities
$ insertExtra (testArgs args) -- Accessed by Settings.Builders.RunTest
$ insertExtra (docsArgs args) -- Accessed by Rules.Documentation
$ insertExtra allSettings -- Accessed by Settings
@@ -424,9 +415,6 @@ cmdUnitIdHash = unitIdHash <$> cmdLineArgs
cmdBignum :: Action (Maybe String)
cmdBignum = bignum <$> cmdLineArgs
-cmdKeepResponseFiles :: Action Bool
-cmdKeepResponseFiles = keepResponseFiles <$> cmdLineArgs
-
cmdProgressInfo :: Action ProgressInfo
cmdProgressInfo = progressInfo <$> cmdLineArgs
=====================================
hadrian/src/Hadrian/Builder/Ar.hs
=====================================
@@ -35,14 +35,16 @@ instance NFData ArMode
-- to be archived is passed via a temporary response file. Passing arguments
-- via a response file is not supported by some versions of @ar@, in which
-- case you should use 'runArWithoutTempFile' instead.
-runAr :: FilePath -- ^ path to @ar@
+runAr :: FilePath -- ^ base name to use for response files
+ -> FilePath -- ^ path to @ar@
-> [String] -- ^ other arguments
-> [FilePath] -- ^ input file paths
-> [CmdOption] -- ^ Additional options
-> Action ()
-runAr arPath flagArgs fileArgs buildOptions = withResponseFile $ \tmp -> do
- writeFile' tmp $ unwords fileArgs
- cmd [arPath] flagArgs ('@' : tmp) buildOptions
+runAr outputFilePath arPath flagArgs fileArgs buildOptions = do
+ rspFile <- responseFilePath outputFilePath
+ writeFile' rspFile $ unwords fileArgs
+ cmd [arPath] flagArgs ('@' : rspFile) buildOptions
-- | Invoke @ar@ given a path to it and a list of arguments. Note that @ar@
-- will be called multiple times if the list of files to be archived is too
=====================================
hadrian/src/Hadrian/Utilities.hs
=====================================
@@ -1,4 +1,6 @@
+{-# LANGUAGE ImpredicativeTypes #-}
{-# LANGUAGE TypeFamilies #-}
+
module Hadrian.Utilities (
-- * List manipulation
fromSingleton, replaceEq, minusOrd, intersectOrd, lookupAll, chunksOfSize,
@@ -14,7 +16,7 @@ module Hadrian.Utilities (
-- * Paths
BuildRoot (..), buildRoot, buildRootRules, isGeneratedSource,
- KeepResponseFiles (..), keepResponseFiles, withResponseFile, withResponseFileOnWindows,
+ withResponseFileIfLongCmd, responseFilePath,
-- * File system operations
copyFile, copyFileUntracked, createFileLink, fixFile,
@@ -47,11 +49,10 @@ import Data.Maybe
import Data.Typeable (TypeRep, typeOf)
import Development.Shake hiding (Normal)
import Development.Shake.Classes
+import Development.Shake.Command (CmdArgument (..), IsCmdArgument (toCmdArgument))
import Development.Shake.FilePath
import GHC.ResponseFile (escapeArgs)
import System.Environment (lookupEnv)
-import System.Info.Extra (isWindows)
-import System.IO (hClose, openTempFile)
import System.IO.Error (isPermissionError)
import qualified Data.ByteString as BS
@@ -255,13 +256,13 @@ infix 1 %%>
-- library, they can reach 2MB! Some operating systems do not support command
-- lines of such length, and this function can be used to obtain a reasonable
-- approximation of the limit. On Windows, it is theoretically 32768 characters
--- (since Windows 7). In practice we use 31000 to leave some breathing space for
+-- (since Windows 7). In practice we use 30000 to leave some breathing space for
-- the builder path & name, auxiliary flags, and other overheads. On Mac OS X,
-- ARG_MAX is 262144, yet when using @xargs@ on OSX this is reduced by over
-- 20000. Hence, 200000 seems like a sensible limit. On other operating systems
-- we currently use the 4194304 setting.
cmdLineLengthLimit :: Int
-cmdLineLengthLimit | IO.isWindows = 31000
+cmdLineLengthLimit | IO.isWindows = 30000
| IO.isMac = 200000
| otherwise = 4194304
@@ -321,53 +322,35 @@ buildRootRules = do
isGeneratedSource :: FilePath -> Action Bool
isGeneratedSource file = buildRoot <&> (`isPrefixOf` file)
-newtype KeepResponseFiles = KeepResponseFiles Bool deriving (Eq, Show)
-
--- | Whether to retain response files after the build action that created them
--- completes. Mainly useful for debugging.
-keepResponseFiles :: Action Bool
-keepResponseFiles = do
- KeepResponseFiles keep <- userSetting (KeepResponseFiles False)
- return keep
-
--- | Run an action either with command arguments direcly or by, on Windows,
--- placing those arguments into a response file escaped with @GHC.ResponseFile.escapeArgs@.
---
--- With @--keep-response-files@, the file is left on disk (if used)
-withResponseFileOnWindows ::
- ([String] -> Action a) -- ^ Action to perform given arguments (of the form @["\@reponseFilePath"]@ on Windows)
- -> [String] -- ^ Command arguments
- -> Action a
-withResponseFileOnWindows action commandArgs = do
- if isWindows
- then withResponseFile $ \tmp -> do
- writeFile' tmp (escapeArgs commandArgs)
- action ['@' : tmp]
- else action commandArgs
-
--- | Run an action with a response file path.
---
--- With @--keep-response-files@, the file is left on disk.
-withResponseFile :: (FilePath -> Action a) -> Action a
-withResponseFile action = do
- keep <- keepResponseFiles
- let putVerboseResponseFile tmp = do
- verbosity <- getVerbosity
- when (verbosity >= Verbose) $ do
- tmpContent <- liftIO (readFile tmp)
- putVerbose (tmp <> " (use hadrian flag --keep-response-files to keep this file):\n" <> tmpContent)
- if keep
- then do
- (tmp, h) <- liftIO $ openTempFile "." "hadrian-rsp"
- liftIO $ hClose h
- putInfo $ "Keeping response file: " ++ tmp
- result <- action tmp
- putVerboseResponseFile tmp
- return result
- else withTempFile $ \tmp -> do
- result <- action tmp
- putVerboseResponseFile tmp
- return result
+-- | Run an command with the given arguments. If the command is too long then the
+-- response file arguments are placed into a response file and escaped with @GHC.ResponseFile.escapeArgs@.
+withResponseFileIfLongCmd ::
+ CmdResult c
+ => FilePath -- ^ Response base name. The reponse file is placed in @_build/rsp/\