Marge Bot pushed to branch master at Glasgow Haskell Compiler / GHC Commits: 498bb21a by David Eichmann at 2026-06-09T18:02:39-04:00 Hadrian: avoid response files when command line is short enough This replaces the logic of always using response files on Windows. With the new condition based on command line lenght, reponse files can be avoided in many more cases (on windows). Now that response files are only used in a small number of cases, response files are always kept and the -r / --keep-response-files command line options have been removed The response file paths are nolonger randomized. They are placed in the `_build/rsp` directory. This ensures they are ignored by git and we that Hadrian reuses response file paths when rebuilding rather than leaving stale response files around. Update user guide putting response files in its own section - - - - - 6 changed files: - changelog.d/hadrian-response-files.md - docs/users_guide/using.rst - hadrian/src/Builder.hs - hadrian/src/CommandLine.hs - hadrian/src/Hadrian/Builder/Ar.hs - hadrian/src/Hadrian/Utilities.hs Changes: ===================================== changelog.d/hadrian-response-files.md ===================================== @@ -1,9 +1,15 @@ section: packaging -synopsis: Add a flag to tell Hadrian to keep response files -issues: #27184 -mrs: !15906 +synopsis: Improved Hadrian's use of response files +issues: #27230 +mrs: !15906 !16134 description: - Hadrian can now be instructed to keep response files with the new - --keep-response-files command line flag. This is helpful when debugging a - build failure, as it allows re-running the failing command line invocation - without an error due to a missing response file. + Response files are files that contain command-line arguments. Hadrian uses + response files to shorten command-line lengths. This is important on Windows + where command-line lengths are limited. + + Hadrian now supports response files when invoking GHC. In order to support + manually rerunning commands issued by Hadrian, response files are no longer + deleted. Instead they are stored under `_build/rsp`. Response files are now + only used when the corresponding command-line is too long for the host + platform. This greatly reduces the use of response files and avoids excessive + file usage. Response files are overwritten on subsequent Hadrian builds. ===================================== docs/users_guide/using.rst ===================================== @@ -85,17 +85,6 @@ all files; you cannot, for example, invoke ``ghc -c -O1 Foo.hs -O2 Bar.hs`` to apply different optimisation levels to the files ``Foo.hs`` and ``Bar.hs``. -In addition to passing arguments via the command-line, arguments can be passed -via GNU-style response files. For instance, - -.. code-block:: bash - - $ cat response-file - -O1 - Hello.hs - -o Hello - $ ghc @response-file - .. note:: .. index:: @@ -118,9 +107,24 @@ via GNU-style response files. For instance, ``-fspecialise`` will not be enabled, since the ``-fno-specialise`` overrides the ``-fspecialise`` implied by ``-O1``. + +Command-line arguments in response files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In addition to passing arguments via the command-line, arguments can be passed +via GNU-style response files. For instance, + +.. code-block:: bash + + $ cat response-file + -O1 + Hello.hs + -o Hello + $ ghc @response-file + .. _source-file-options: -Command line options in source files +Command-line options in source files ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. index:: ===================================== hadrian/src/Builder.hs ===================================== @@ -304,7 +304,7 @@ instance H.Builder Builder where case builder of Ar Pack stg -> do useTempFile <- arSupportsAtFile stg - if useTempFile then runAr path buildArgs buildInputs buildOptions + if useTempFile then runAr output path buildArgs buildInputs buildOptions else runArWithoutTempFile path buildArgs buildInputs buildOptions Ar Unpack _ -> cmd' [Cwd output] [path] buildArgs buildOptions @@ -343,7 +343,7 @@ instance H.Builder Builder where Exit _ <- cmd' [path] (buildArgs ++ [input]) buildOptions return () - Haddock BuildPackage -> runHaddock path buildArgs buildInputs + Haddock BuildPackage -> runHaddock output path buildArgs buildInputs Ghc _ _ -> -- Use a response file for ghc invocations to avoid issues with command line @@ -351,9 +351,11 @@ instance H.Builder Builder where -- NB: we can't put the buildArgs in a response file, because some flags require -- empty arguments (such as the -dep-suffix flag), but that isn't supported -- yet due to #26560. - withResponseFileOnWindows - (\buildInputs' -> cmd [path] buildArgs buildInputs' buildOptions) + withResponseFileIfLongCmd + output + (toCmdArgument [path] <> toCmdArgument buildArgs) buildInputs + (toCmdArgument buildOptions) HsCpp -> captureStdout @@ -389,13 +391,16 @@ instance H.Builder Builder where -- | Invoke @haddock@ given a path to it and a list of arguments. On Windows, -- the input file arguments are passed as a response file. -runHaddock :: FilePath -- ^ path to @haddock@ +runHaddock :: FilePath -- ^ base name to use for response file + -> FilePath -- ^ path to @haddock@ -> [String] -> [FilePath] -- ^ input file paths -> Action () -runHaddock haddockPath flagArgs fileInputs = withResponseFileOnWindows - (cmd [haddockPath] flagArgs) +runHaddock outputFilePath haddockPath flagArgs fileInputs = withResponseFileIfLongCmd + outputFilePath + (toCmdArgument [haddockPath] <> toCmdArgument flagArgs) fileInputs + (CmdArgument []) -- TODO: Some builders are required only on certain platforms. For example, -- 'Objdump' is only required on OpenBSD and AIX. Add support for platform ===================================== hadrian/src/CommandLine.hs ===================================== @@ -3,8 +3,7 @@ module CommandLine ( lookupBignum, cmdBignum, cmdProgressInfo, cmdCompleteSetting, cmdDocsArgs, cmdUnitIdHash, lookupBuildRoot, TestArgs(..), TestSpeed(..), defaultTestArgs, - cmdPrefix, cmdChangelogVersion, DocArgs(..), defaultDocArgs, - cmdKeepResponseFiles + cmdPrefix, cmdChangelogVersion, DocArgs(..), defaultDocArgs ) where import Data.Either @@ -12,7 +11,7 @@ import qualified Data.HashMap.Strict as Map import Data.List.Extra import Development.Shake hiding (Normal) import Flavour (DocTargets, DocTarget(..)) -import Hadrian.Utilities hiding (buildRoot, keepResponseFiles) +import Hadrian.Utilities hiding (buildRoot) import Settings.Parser import System.Console.GetOpt import System.Environment @@ -37,7 +36,6 @@ data CommandLineArgs = CommandLineArgs , testArgs :: TestArgs , docsArgs :: DocArgs , docTargets :: DocTargets - , keepResponseFiles :: Bool , prefix :: Maybe FilePath , changelogVersion :: Maybe String , completeStg :: Maybe String } @@ -58,7 +56,6 @@ defaultCommandLineArgs = CommandLineArgs , testArgs = defaultTestArgs , docsArgs = defaultDocArgs , docTargets = Set.fromList [minBound..maxBound] - , keepResponseFiles = False , prefix = Nothing , changelogVersion = Nothing , completeStg = Nothing } @@ -141,9 +138,6 @@ readFreeze1 = Right $ \flags -> flags { freeze1 = True } readFreeze2 = Right $ \flags -> flags { freeze1 = True, freeze2 = True } readSkipDepends = Right $ \flags -> flags { skipDepends = True } -readKeepResponseFiles :: Either String (CommandLineArgs -> CommandLineArgs) -readKeepResponseFiles = Right $ \flags -> flags { keepResponseFiles = True } - readUnitIdHash :: Either String (CommandLineArgs -> CommandLineArgs) readUnitIdHash = Right $ \flags -> trace "--hash-unit-ids is deprecated. It is enabled by release flavour or +hash_unit_ids flavour transformer" $ @@ -302,8 +296,6 @@ optDescrs = "Progress info style (None, Brief, Normal or Unicorn)." , Option [] ["docs"] (ReqArg readDocsArg "TARGET") "Strip down docs targets (none, no-haddocks, no-sphinx[-{html, pdfs, man}]." - , Option ['r'] ["keep-response-files"] (NoArg readKeepResponseFiles) - "Keep response files created during the build (for debugging)." , Option ['k'] ["keep-test-files"] (NoArg readTestKeepFiles) "Keep all the files generated when running the testsuite." , Option [] ["test-compiler"] (ReqArg readTestCompiler "TEST_COMPILER") @@ -382,7 +374,6 @@ cmdLineArgsMap = do return $ insertExtra (progressInfo args) -- Accessed by Hadrian.Utilities $ insertExtra (buildRoot args) -- Accessed by Hadrian.Utilities - $ insertExtra (KeepResponseFiles $ keepResponseFiles args) -- Accessed by Hadrian.Utilities $ insertExtra (testArgs args) -- Accessed by Settings.Builders.RunTest $ insertExtra (docsArgs args) -- Accessed by Rules.Documentation $ insertExtra allSettings -- Accessed by Settings @@ -424,9 +415,6 @@ cmdUnitIdHash = unitIdHash <$> cmdLineArgs cmdBignum :: Action (Maybe String) cmdBignum = bignum <$> cmdLineArgs -cmdKeepResponseFiles :: Action Bool -cmdKeepResponseFiles = keepResponseFiles <$> cmdLineArgs - cmdProgressInfo :: Action ProgressInfo cmdProgressInfo = progressInfo <$> cmdLineArgs ===================================== hadrian/src/Hadrian/Builder/Ar.hs ===================================== @@ -35,14 +35,16 @@ instance NFData ArMode -- to be archived is passed via a temporary response file. Passing arguments -- via a response file is not supported by some versions of @ar@, in which -- case you should use 'runArWithoutTempFile' instead. -runAr :: FilePath -- ^ path to @ar@ +runAr :: FilePath -- ^ base name to use for response files + -> FilePath -- ^ path to @ar@ -> [String] -- ^ other arguments -> [FilePath] -- ^ input file paths -> [CmdOption] -- ^ Additional options -> Action () -runAr arPath flagArgs fileArgs buildOptions = withResponseFile $ \tmp -> do - writeFile' tmp $ unwords fileArgs - cmd [arPath] flagArgs ('@' : tmp) buildOptions +runAr outputFilePath arPath flagArgs fileArgs buildOptions = do + rspFile <- responseFilePath outputFilePath + writeFile' rspFile $ unwords fileArgs + cmd [arPath] flagArgs ('@' : rspFile) buildOptions -- | Invoke @ar@ given a path to it and a list of arguments. Note that @ar@ -- will be called multiple times if the list of files to be archived is too ===================================== hadrian/src/Hadrian/Utilities.hs ===================================== @@ -1,4 +1,6 @@ +{-# LANGUAGE ImpredicativeTypes #-} {-# LANGUAGE TypeFamilies #-} + module Hadrian.Utilities ( -- * List manipulation fromSingleton, replaceEq, minusOrd, intersectOrd, lookupAll, chunksOfSize, @@ -14,7 +16,7 @@ module Hadrian.Utilities ( -- * Paths BuildRoot (..), buildRoot, buildRootRules, isGeneratedSource, - KeepResponseFiles (..), keepResponseFiles, withResponseFile, withResponseFileOnWindows, + withResponseFileIfLongCmd, responseFilePath, -- * File system operations copyFile, copyFileUntracked, createFileLink, fixFile, @@ -47,11 +49,10 @@ import Data.Maybe import Data.Typeable (TypeRep, typeOf) import Development.Shake hiding (Normal) import Development.Shake.Classes +import Development.Shake.Command (CmdArgument (..), IsCmdArgument (toCmdArgument)) import Development.Shake.FilePath import GHC.ResponseFile (escapeArgs) import System.Environment (lookupEnv) -import System.Info.Extra (isWindows) -import System.IO (hClose, openTempFile) import System.IO.Error (isPermissionError) import qualified Data.ByteString as BS @@ -255,13 +256,13 @@ infix 1 %%> -- library, they can reach 2MB! Some operating systems do not support command -- lines of such length, and this function can be used to obtain a reasonable -- approximation of the limit. On Windows, it is theoretically 32768 characters --- (since Windows 7). In practice we use 31000 to leave some breathing space for +-- (since Windows 7). In practice we use 30000 to leave some breathing space for -- the builder path & name, auxiliary flags, and other overheads. On Mac OS X, -- ARG_MAX is 262144, yet when using @xargs@ on OSX this is reduced by over -- 20000. Hence, 200000 seems like a sensible limit. On other operating systems -- we currently use the 4194304 setting. cmdLineLengthLimit :: Int -cmdLineLengthLimit | IO.isWindows = 31000 +cmdLineLengthLimit | IO.isWindows = 30000 | IO.isMac = 200000 | otherwise = 4194304 @@ -321,53 +322,35 @@ buildRootRules = do isGeneratedSource :: FilePath -> Action Bool isGeneratedSource file = buildRoot <&> (`isPrefixOf` file) -newtype KeepResponseFiles = KeepResponseFiles Bool deriving (Eq, Show) - --- | Whether to retain response files after the build action that created them --- completes. Mainly useful for debugging. -keepResponseFiles :: Action Bool -keepResponseFiles = do - KeepResponseFiles keep <- userSetting (KeepResponseFiles False) - return keep - --- | Run an action either with command arguments direcly or by, on Windows, --- placing those arguments into a response file escaped with @GHC.ResponseFile.escapeArgs@. --- --- With @--keep-response-files@, the file is left on disk (if used) -withResponseFileOnWindows :: - ([String] -> Action a) -- ^ Action to perform given arguments (of the form @["\@reponseFilePath"]@ on Windows) - -> [String] -- ^ Command arguments - -> Action a -withResponseFileOnWindows action commandArgs = do - if isWindows - then withResponseFile $ \tmp -> do - writeFile' tmp (escapeArgs commandArgs) - action ['@' : tmp] - else action commandArgs - --- | Run an action with a response file path. --- --- With @--keep-response-files@, the file is left on disk. -withResponseFile :: (FilePath -> Action a) -> Action a -withResponseFile action = do - keep <- keepResponseFiles - let putVerboseResponseFile tmp = do - verbosity <- getVerbosity - when (verbosity >= Verbose) $ do - tmpContent <- liftIO (readFile tmp) - putVerbose (tmp <> " (use hadrian flag --keep-response-files to keep this file):\n" <> tmpContent) - if keep - then do - (tmp, h) <- liftIO $ openTempFile "." "hadrian-rsp" - liftIO $ hClose h - putInfo $ "Keeping response file: " ++ tmp - result <- action tmp - putVerboseResponseFile tmp - return result - else withTempFile $ \tmp -> do - result <- action tmp - putVerboseResponseFile tmp - return result +-- | Run an command with the given arguments. If the command is too long then the +-- response file arguments are placed into a response file and escaped with @GHC.ResponseFile.escapeArgs@. +withResponseFileIfLongCmd :: + CmdResult c + => FilePath -- ^ Response base name. The reponse file is placed in @_build/rsp/\@. + -> CmdArgument -- ^ Command and arguments before the response file arguments. + -> [String] -- ^ Response file aruguments. + -> CmdArgument -- ^ Command arguments after the response file arguments. + -> Action c +withResponseFileIfLongCmd outputFilePath argsPre argsResp argsPost = do + let cmdLineLengh = sum + [ 1 + length arg -- add one to account for space inbetween arguments + | let CmdArgument args = argsPre <> toCmdArgument argsResp <> argsPost + , Right arg <- args + ] + if cmdLineLengh < cmdLineLengthLimit + then cmd argsPre argsResp argsPost + else do + rspFile <- responseFilePath outputFilePath + writeFile' rspFile (escapeArgs argsResp) + cmd argsPre ['@' : rspFile] argsPost + +-- | Convert a command's output file path to a response file path to be used for that command. +-- Response files are placed in a dedicated @rps@ directory under the build directory. This avoids +-- clutering the work tree or interfearing with other build directories. +responseFilePath :: FilePath -> Action FilePath +responseFilePath outputFilePath = do + buildDir <- buildRoot + return $ buildDir "rsp" outputFilePath -- | Link a file tracking the link target. Create the target directory if -- missing. View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/498bb21afd3852bd99d001ac5b20eeb0... -- View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/498bb21afd3852bd99d001ac5b20eeb0... You're receiving this email because of your account on gitlab.haskell.org.