Hannes Siebenhandl pushed to branch wip/fendor/dep-stdout at Glasgow Haskell Compiler / GHC Commits: 419cfed9 by fendor at 2026-10-20T14:01:13+02:00 Add support for streaming `ghc -M` to stdout Adds the flags `-dep-json-stdout` and `-dep-makefile-stdout` which streams dependency information line-by-line to stdout. Each dependency generation can be redirected to separate outputs. Providing any extra `-dep-{json,makefile}[-stdout]` disables the default generation of the `Makefile` file. Allows consumers to start processing the dependency information as it comes in and not waiting for the dependency generation to finish. `DepStream` abstracts over the possible output handles, which is either a file or the stdout handle. Writing to a file requires temporary file paths, thus `DepStream` must provide a way to perform clean up when dependency processing finished or when an error is thrown for any reason. We further decouple the rendering and writing of the result from the dependency generation, localising side effects and making the code slightly more readable. The testsuite now also handles `ghc -M` writing to stdout. We add a test for each combination of writing to stdout and file, except for writing both makefile and json. While it probably works, is really undefined behaviour. Closes #27937 - - - - - 15 changed files: - + changelog.d/make-depends-streaming-output - compiler/GHC/Driver/DynFlags.hs - compiler/GHC/Driver/MakeFile.hs - compiler/GHC/Driver/Session.hs - docs/users_guide/separate_compilation.rst - testsuite/driver/my_typing.py - testsuite/driver/testglobals.py - testsuite/driver/testlib.py - testsuite/tests/driver/mkdepend/all.T - testsuite/tests/driver/mkdepend/mkdepend_mhu01.dep-json → testsuite/tests/driver/mkdepend/mkdepend_mhu01a.dep-json - testsuite/tests/driver/mkdepend/mkdepend_mhu01.dep-makefile → testsuite/tests/driver/mkdepend/mkdepend_mhu01a.dep-makefile - + testsuite/tests/driver/mkdepend/mkdepend_mhu01b.stderr - + testsuite/tests/driver/mkdepend/mkdepend_mhu01c.stderr - + testsuite/tests/driver/mkdepend/mkdepend_mhu01d.dep-makefile - + testsuite/tests/driver/mkdepend/mkdepend_mhu01d.stderr Changes: ===================================== changelog.d/make-depends-streaming-output ===================================== @@ -0,0 +1,16 @@ +section: compiler +synopsis: Allow streaming `ghc -M` output to stdout +issues: #27937 +mrs: !16830 +description: { + Add options `-dep-makefile-stdout` and `-dep-json-stdout` which stream the dependency generation + output to stdout. + + The Makefile and json dependency generation can stream to separate targets, e.g. `-dep-makefile target.Makefile -dep-json-stdout` + streams the JSON Lines output to stdout and the Makefile to `target.Makefile`. + + If `-dep-makefile-stdout`, `-dep-json` or `-dep-json-stdout` is given, we no longer generate a Makefile by default, but only + if the user explicitly requests the Makefile dependency generation. + + The output from a particular backend can be disabled via `-no-dep-json` and `-no-dep-makefile` respectively. +} ===================================== compiler/GHC/Driver/DynFlags.hs ===================================== @@ -73,6 +73,9 @@ module GHC.Driver.DynFlags ( IncludeSpecs(..), addGlobalInclude, addQuoteInclude, flattenIncludes, addImplicitQuoteInclude, + -- * `ghc -M` specification + MkDependOutput(..), + -- * SDoc initSDocContext, initDefaultSDocContext, initPromotionTickContext, @@ -350,8 +353,8 @@ data DynFlags = DynFlags { -- ^ External plugins loaded from shared libraries -- For ghc -M - depMakefile :: Maybe FilePath, - depJson :: Maybe FilePath, + depMakefile :: MkDependOutput, + depJson :: MkDependOutput, depIncludePkgDeps :: Bool, depIncludeCppDeps :: Bool, depExcludeMods :: [ModuleName], @@ -694,8 +697,8 @@ defaultDynFlags mySettings = llvmOptLevel = 0, -- ghc -M values - depMakefile = Just "Makefile", - depJson = Nothing, + depMakefile = MkDepend_Default "Makefile", + depJson = MkDepend_None, depIncludePkgDeps = False, depIncludeCppDeps = False, depExcludeMods = [], @@ -801,6 +804,15 @@ flattenExtensionFlags ml = foldr g defaultExtensionFlags g (Off f) flags = EnumSet.delete f flags defaultExtensionFlags = EnumSet.fromList (languageExtensions ml) +-- ----------------------------------------------------------------------------- +-- -M -dep-* + +data MkDependOutput + = MkDepend_None + | MkDepend_Default !FilePath + | MkDepend_Explicit !FilePath + | MkDepend_StdOut + -- ----------------------------------------------------------------------------- -- -jN ===================================== compiler/GHC/Driver/MakeFile.hs ===================================== @@ -132,7 +132,7 @@ doMkDepend = do module_graph <- GHC.depanal excl_mods True {- Allow dup roots -} doMkDependModuleGraph dflags module_graph -doMkDependModuleGraph :: GhcMonad m => DynFlags -> ModuleGraph -> m () +doMkDependModuleGraph :: GhcMonad m => DynFlags -> ModuleGraph -> m () doMkDependModuleGraph dflags module_graph = do hsc_env <- getSession logger <- getLogger @@ -143,12 +143,14 @@ doMkDependModuleGraph dflags module_graph = do root <- liftIO Directory.getCurrentDirectory let + suffixes = fmap unsafeEncodeUtf (depSuffixes dflags) + tmpdir = tmpDir dflags backends = concat - [ [ initFileDepWriter logger root tmpfs dflags makefile_output - | Just makefile_output <- [depMakefile dflags] + [ [ initFileDepWriter logger root tmpfs tmpdir suffixes makefile_output + | Just makefile_output <- [initDepOutput (depMakefile dflags)] ] - , [ initJsonDepWriter logger tmpfs dflags json_output - | Just json_output <- [depJson dflags] + , [ initJsonDepWriter logger tmpfs tmpdir json_output + | Just json_output <- [initDepOutput (depJson dflags)] ] ] @@ -158,24 +160,14 @@ doMkDependModuleGraph dflags module_graph = do liftIO $ debugTraceMsg logger 2 (text "Module dependencies" $$ ppr sorted) liftIO $ do - - -- Setup the writers - writers <- sequenceA backends - sinks <- traverse dw_beginWriter writers - - -- Do the actual work - do + withWriters backends $ \ depSink -> do + -- Do the actual work -- Process them one by one, dumping results and complaining about cycles - mapM_ (processDeps hsc_env (UniqSet.mkUniqSet excl_mods) sinks) sorted + mapM_ (processDeps hsc_env (UniqSet.mkUniqSet excl_mods) depSink) sorted -- If -ddump-mod-cycles, show cycles in the module graph liftIO $ dumpModCycles logger module_graph - -- Tidy up - do - traverse_ dw_endWriter writers - - -- Unconditional exiting is a bad idea. If an error occurs we'll get an --exception; if that is not caught it's fine, but at least we have a --chance to find out exactly what went wrong. Uncomment the following @@ -216,15 +208,119 @@ data Dep { dep_path :: OsPath } +-- | Consumer of the dependency information generated from the 'ModuleGraph'. +-- +-- Use with 'withWriter'/'withWriters' to initialise the consumers of the dependency information +-- generation. data DependencyWriter = DependencyWriter - { dw_beginWriter :: IO DepSink + { dw_sink :: DepSink + -- ^ Sink for dependency information , dw_endWriter :: IO () + -- ^ We finished consuming the dependency information, perform final + -- tidy up operations. This may be a no-op, but could also + -- copy temporary files, close open handlers, etc... + -- + -- If 'dw_endWriter' was called, calling 'dw_onError' is unnecessary, + -- but it must be safe to call 'dw_onError' any way. + , dw_onError :: IO () + -- ^ Something crashed! Clean up any resources opened by this 'DependencyWriter'. } -data DepSink = DepSink +-- | Initialise multiple backends and combine them into one output function. +-- +-- Combines the 'DepSink' of each 'DependencyWriter' into one single consumer. +withWriters :: [IO DependencyWriter] -> (DepSink -> IO a) -> IO a +withWriters [] k = k noopSink +withWriters (dw : dws) k = + withWriter + dw + $ \s -> withWriters dws (\s' -> k (s `mappendSink` s')) + +-- | Initialise a single 'DependencyWriter', making sure any allocated resource +-- get's tidied up again. +-- +-- Accepts 'IO' 'DependencyWriter' since the initialisation may perform setup operations, +-- such as opening file handles, or intermediate data types. +withWriter :: IO DependencyWriter -> (DepSink -> IO r) -> IO r +withWriter dwIO k = + bracketOnError dwIO dw_onError $ \ dw -> do + r <- k (dw_sink dw) + -- Clean up open resources + dw_endWriter dw + pure r + +-- | Process the dependency information for one node in the 'ModuleGraph'. +newtype DepSink = DepSink { ds_writeDependency :: DepNode -> [Dep] -> IO () } +noopSink :: DepSink +noopSink = DepSink $ \ _ _ -> pure () + +mappendSink :: DepSink -> DepSink -> DepSink +mappendSink (DepSink s1) (DepSink s2) = DepSink (\ node deps -> s1 node deps >> s2 node deps) + +-- | Where should the dependency output end up? +data DepOutput + = DepStreamHandle Handle + -- ^ The dependency information should be written to this 'Handle'. + | DepFileOutput FilePath + -- ^ The dependency information should be written to this 'FilePath'. + +-- | A 'DepStream' is an abstraction over how to output single elements +-- of the dependency generation, e.g., a json object or one makefile rule. +data DepStream = DepStream + { depStream_write :: String -> IO () + , depStream_close :: IO () + , depStream_onError :: IO () + } + +initDepOutput :: MkDependOutput -> Maybe DepOutput +initDepOutput = \ case + MkDepend_None -> Nothing + MkDepend_Default fp -> Just $ DepFileOutput fp + MkDepend_Explicit fp -> Just $ DepFileOutput fp + MkDepend_StdOut -> Just $ DepStreamHandle stdout + +initMakefileDepStream :: Logger -> TmpFs -> TempDir -> DepOutput -> IO DepStream +initMakefileDepStream _ _ _ (DepStreamHandle h) = + pure $ depStreamHandle h +initMakefileDepStream logger tmpfs tmpdir (DepFileOutput fp) = do + files <- beginMkDependHS logger tmpfs tmpdir fp + pure $ + DepStream + { depStream_write = hPutStrLn (mkd_tmp_hdl files) + , depStream_onError = do + hClose $ mkd_tmp_hdl files + case mkd_make_hdl files of + Nothing -> return () + Just hdl -> hClose hdl + , depStream_close = endMkDependHS logger files + } + +initJsonDepStream :: Logger -> TmpFs -> TempDir -> DepOutput -> IO DepStream +initJsonDepStream _ _ _ (DepStreamHandle h) = + pure $ depStreamHandle h +initJsonDepStream logger tmpfs tmpdir (DepFileOutput fp) = do + files <- initJsonDepFiles logger tmpfs tmpdir fp + pure $ + DepStream + { depStream_write = hPutStrLn (json_tmp_hdl files) + , depStream_onError = hClose (json_tmp_hdl files) + , depStream_close = endJsonDepFiles logger files + } + +depStreamHandle :: Handle -> DepStream +depStreamHandle h = + DepStream + { depStream_write = + \output -> do + hPutStrLn h output + hFlush h -- We flush to ensure the output is streamed per line. + , depStream_close = pure () + , depStream_onError = pure () + } + ----------------------------------------------------------------- -- -- processDeps @@ -233,7 +329,7 @@ data DepSink = DepSink processDeps :: HscEnv -> UniqSet ModuleName -- ^ Excludes - -> [DepSink] + -> DepSink -> SCC ModuleGraphNode -> IO () -- Write suitable dependencies to handle @@ -267,7 +363,7 @@ processDeps _ _ _ (AcyclicSCC (ModuleNode _ (ModuleNodeFixed {}))) -- No dependencies needed for fixed modules (already compiled) = return () -processDeps hsc_env0 excl_mods sinks (AcyclicSCC (ModuleNode _ (ModuleNodeCompile node))) = do +processDeps hsc_env0 excl_mods sink (AcyclicSCC (ModuleNode _ (ModuleNodeCompile node))) = do pp <- preprocessor let dep_node = mkDepNode pp @@ -279,7 +375,7 @@ processDeps hsc_env0 excl_mods sinks (AcyclicSCC (ModuleNode _ (ModuleNodeCompil if null missing_dep_errs then do - traverse_ (\ sink -> ds_writeDependency sink dep_node deps) sinks + ds_writeDependency sink dep_node deps else do let sec = initSourceErrorContext (hsc_dflags local_hsc_env) throwErrors sec (mkMessages (listToBag missing_dep_errs)) @@ -380,17 +476,16 @@ find_preprocessor d = do -- ----------------------------------------------------------------- -initFileDepWriter :: Logger -> OsPath -> TmpFs -> DynFlags -> FilePath -> IO DependencyWriter -initFileDepWriter logger root tmpfs dflags makefile = do - files <- beginMkDependHS logger tmpfs dflags makefile +initFileDepWriter :: Logger -> OsPath -> TmpFs -> TempDir -> [OsPath] -> DepOutput -> IO DependencyWriter +initFileDepWriter logger root tmpfs tmpdir suffixes output = do + stream <- initMakefileDepStream logger tmpfs tmpdir output pure DependencyWriter - { dw_beginWriter = do - pure $ DepSink $ \ node deps -> do - writeDependencies root (mkd_tmp_hdl files) suffixes node deps - , dw_endWriter = endMkDependHS logger files + { dw_sink = + DepSink $ \ node deps -> do + traverse_ (depStream_write stream) (dependencyRules root suffixes node deps) + , dw_endWriter = depStream_close stream + , dw_onError = depStream_onError stream } - where - suffixes = map unsafeEncodeUtf (depSuffixes dflags) data MkDepFiles = MkDep { mkd_make_file :: FilePath, -- Name of the makefile @@ -398,11 +493,11 @@ data MkDepFiles mkd_tmp_file :: FilePath, -- Name of the temporary file mkd_tmp_hdl :: Handle } -- Handle of the open temporary file -beginMkDependHS :: Logger -> TmpFs -> DynFlags -> FilePath -> IO MkDepFiles -beginMkDependHS logger tmpfs dflags makefile = do +beginMkDependHS :: Logger -> TmpFs -> TempDir -> FilePath -> IO MkDepFiles +beginMkDependHS logger tmpfs tmpdir makefile = do -- open a new temp file in which to stuff the dependency info -- as we go along. - tmp_file <- newTempName logger tmpfs (tmpDir dflags) TFL_CurrentModule "dep" + tmp_file <- newTempName logger tmpfs tmpdir TFL_CurrentModule "dep" tmp_hdl <- openFile tmp_file WriteMode -- open the makefile @@ -443,16 +538,15 @@ beginMkDependHS logger tmpfs dflags makefile = do return (MkDep { mkd_make_file = makefile, mkd_make_hdl = mb_make_hdl, mkd_tmp_file = tmp_file, mkd_tmp_hdl = tmp_hdl}) -writeDependencies :: +dependencyRules :: OsPath -> - Handle -> [OsString] -> -- ^ Suffixes DepNode -> [Dep] -> - IO () -writeDependencies root hdl suffixes node deps = - traverse_ write tasks + [String] +dependencyRules root suffixes node deps = + map rule tasks where tasks = source_dep : boot_dep ++ concatMap import_dep deps @@ -479,7 +573,7 @@ writeDependencies root hdl suffixes node deps = DepCpp {dep_path} -> [(obj_files, dep_path)] - write (from, to) = writeDependency root hdl from to + rule (from, to) = dependencyRule root from to obj_files = suffixed dn_obj @@ -488,18 +582,18 @@ writeDependencies root hdl suffixes node deps = DepNode {dn_src, dn_obj, dn_hi, dn_mod} = node ----------------------------- -writeDependency :: OsPath -> Handle -> [OsPath] -> OsPath -> IO () --- (writeDependency r h [t1,t2] dep) writes to handle h the dependency +dependencyRule :: OsPath -> [OsPath] -> OsPath -> String +-- (dependencyRule r [t1,t2] dep) renders the dependency -- t1 t2 : dep -writeDependency root hdl targets dep - = do let -- We need to avoid making deps on - -- c:/foo/... - -- on Windows as make gets confused by the : - -- Making relative deps avoids some instances of this. - dep' = OsPath.makeRelative root dep - forOutput = escapeSpaces . reslash Forwards . unsafeDecodeUtf . OsPath.normalise - output = unwords (map forOutput targets) ++ " : " ++ forOutput dep' - hPutStrLn hdl output +dependencyRule root targets dep + = let -- We need to avoid making deps on + -- c:/foo/... + -- on Windows as make gets confused by the : + -- Making relative deps avoids some instances of this. + dep' = OsPath.makeRelative root dep + forOutput = escapeSpaces . reslash Forwards . unsafeDecodeUtf . OsPath.normalise + in + unwords (map forOutput targets) ++ " : " ++ forOutput dep' ----------------------------- insertSuffixes @@ -554,16 +648,27 @@ endMkDependHS logger -- ----------------------------------------------------------------- -initJsonDepWriter :: Logger -> TmpFs -> DynFlags -> FilePath -> IO DependencyWriter -initJsonDepWriter logger tmpfs dflags output = do - json_var <- mkJsonOutput logger tmpfs dflags output initDepJson - pure DependencyWriter - { dw_beginWriter = - pure $ DepSink $ \ node deps -> - writeHomeUnitDeps json_var node deps - , dw_endWriter = do - writeJsonOutput logger json_var - } +initJsonDepWriter :: Logger -> TmpFs -> TempDir -> DepOutput -> IO DependencyWriter +initJsonDepWriter logger tmpfs tmpdir output = do + stream <- initJsonDepStream logger tmpfs tmpdir output + json_var <- mkJsonOutput + pure + DependencyWriter + { dw_sink = + DepSink $ \node deps -> do + extUnits <- readJson json_var + let + (extUnitsDeps, payload) = homeUnitDeps node deps + newExtUnits = Set.difference extUnitsDeps (externalDeps extUnits) + -- Write new external unit deps before the module that uses them + for_ newExtUnits $ \extUnit -> do + depStream_write stream (renderJsonl extUnit) + depStream_write stream (renderJsonl payload) + -- Cache external deps we already reported in the past + updateJson json_var $ cacheExternalDepsJson newExtUnits + , dw_endWriter = depStream_close stream + , dw_onError = depStream_onError stream + } -------------------------------------------------------------------------------- -- Output interface for json dumps @@ -572,65 +677,60 @@ initJsonDepWriter logger tmpfs dflags output = do -- The flag @-dep-json@ add an additional output target for dependency -- diagnostics. -- --- Contains temporary file locations to stream output to. -data JsonOutput = - JsonOutput { - -- | This ref is updated in @processDeps@ incrementally, using a - -- flag-specific type. - json_ref :: IORef DepJson, - - -- | The output file path specified as argument to the flag. - json_path :: FilePath, - -- | Name of the temporary file - json_tmp_file :: FilePath, - -- | Handle of the open temporary file - json_tmp_hdl :: Handle +data JsonOutput + = JsonOutput + { json_ref :: IORef DepJson + -- ^ This ref is updated in @processDeps@ incrementally, using a + -- flag-specific type. } --- | Set up json generation for `ghc -M`. --- --- Creates a temporary file that we write the dependency information to. --- The temporary file is written to the final output file path in 'writeJsonOutput'. -mkJsonOutput :: - Logger -> - TmpFs -> - DynFlags -> - FilePath -> - IO (IORef DepJson) -> - IO JsonOutput -mkJsonOutput logger tmpfs dflags json_path mk_ref = do - json_ref <- mk_ref - -- open a new temp file in which to stuff the dependency info +data JsonDepFiles = JsonDepFiles + { json_path :: FilePath + -- ^ The output file path specified as argument to the flag. + , json_tmp_file :: FilePath + -- ^ Name of the temporary file + , json_tmp_hdl :: Handle + -- ^ Handle of the open temporary file + } + +initJsonDepFiles :: Logger -> TmpFs -> TempDir -> FilePath -> IO JsonDepFiles +initJsonDepFiles logger tmpfs tmpdir path = do + -- open a new temp file in which to stuff the dependency info into -- as we go along. - json_tmp_file <- newTempName logger tmpfs (tmpDir dflags) TFL_CurrentModule "dep" + json_tmp_file <- newTempName logger tmpfs tmpdir TFL_CurrentModule "dep-json" json_tmp_hdl <- openFile json_tmp_file WriteMode - - pure JsonOutput {json_ref, json_path, json_tmp_file, json_tmp_hdl} - -updateJson :: JsonOutput -> (DepJson -> DepJson) -> IO () -updateJson JsonOutput {json_ref} f = modifyIORef' json_ref f - -readJson :: JsonOutput -> IO DepJson -readJson JsonOutput {json_ref} = readIORef json_ref + pure $ JsonDepFiles + { json_path = path + , json_tmp_file = json_tmp_file + , json_tmp_hdl = json_tmp_hdl + } -- | Finalise the json generation by copying the temporary file to the final destination. -- Avoids partial writes in the case of an error. -writeJsonOutput :: - Logger -> - JsonOutput -> - IO () -writeJsonOutput logger JsonOutput {json_path, json_tmp_file, json_tmp_hdl} = do +endJsonDepFiles :: Logger -> JsonDepFiles -> IO () +endJsonDepFiles logger JsonDepFiles {json_path, json_tmp_file, json_tmp_hdl} = do hClose json_tmp_hdl -- Copy the new jsonl in place showPass logger "Installing new json output" SysTools.copyFile json_tmp_file json_path +-- | Set up json generation for `ghc -M`. +mkJsonOutput :: IO JsonOutput +mkJsonOutput = + JsonOutput <$> newIORef (DepJson Set.empty) + +updateJson :: JsonOutput -> (DepJson -> DepJson) -> IO () +updateJson JsonOutput {json_ref} f = modifyIORef' json_ref f + +readJson :: JsonOutput -> IO DepJson +readJson JsonOutput {json_ref} = readIORef json_ref + -------------------------------------------------------------------------------- -- Output helpers -writeJsonl :: ToJson a => JsonOutput -> a -> IO () -writeJsonl JsonOutput {json_tmp_hdl} output = do - hPutStrLn json_tmp_hdl (showSDocUnsafe $ renderJSON $ json output) +renderJsonl :: ToJson a => a -> String +renderJsonl output = + showSDocUnsafe $ renderJSON $ json output -------------------------------------------------------------------------------- -- Payload for -dep-json @@ -788,9 +888,6 @@ instance ToJson ExtUnitDep where where showPackageId (DPackageId (PackageId pid)) = unpackFS pid -initDepJson :: IO (IORef DepJson) -initDepJson = newIORef $ DepJson Set.empty - cacheExternalDepsJson :: Set.Set ExtUnitDep -> DepJson -> DepJson cacheExternalDepsJson extUnits (DepJson e0) = DepJson @@ -798,19 +895,10 @@ cacheExternalDepsJson extUnits (DepJson e0) = Set.union e0 extUnits } -writeHomeUnitDeps :: JsonOutput -> DepNode -> [Dep] -> IO () -writeHomeUnitDeps deps_json DepNode {..} deps = do - extUnits <- readJson deps_json - let newExtUnits = Set.difference extUnitsDeps (externalDeps extUnits) - -- Write new external unit deps before the module that uses them - for_ newExtUnits $ \ extUnit -> do - writeJsonl deps_json extUnit - writeJsonl deps_json payload - -- Cache external deps we already reported in the past - updateJson deps_json $ cacheExternalDepsJson newExtUnits +homeUnitDeps :: Foldable t => DepNode -> t Dep -> (Set.Set ExtUnitDep, HomeUnitDep) +homeUnitDeps DepNode {..} deps = + foldl' go (Set.empty, initial_node_data) deps where - (extUnitsDeps, payload) = foldl' go (Set.empty, initial_node_data) deps - initial_node_data = HomeUnitDep { unitId = dn_unitId ===================================== compiler/GHC/Driver/Session.hs ===================================== @@ -790,16 +790,44 @@ addOptCmmP f = alterToolSettings $ \s -> s } setDepMakefile :: FilePath -> DynFlags -> DynFlags -setDepMakefile f d = d { depMakefile = Just f } +setDepMakefile f d = d { depMakefile = MkDepend_Explicit f + -- depJson has no default value, we add this for symmetry with `setDepJson` + -- and if anything changes in the future, then the code is already here :) + , depJson = unsetMkDependOutputIfDefault (depJson d) + } + +setDepMakefileStdout :: DynFlags -> DynFlags +setDepMakefileStdout d = d { depMakefile = MkDepend_StdOut + -- depJson has no default value, we add this for symmetry with `setDepJson` + -- and if anything changes in the future, then the code is already here :) + , depJson = unsetMkDependOutputIfDefault (depJson d) + } unsetDepMakefile :: DynFlags -> DynFlags -unsetDepMakefile d = d { depMakefile = Nothing } +unsetDepMakefile d = d { depMakefile = MkDepend_None } setDepJson :: FilePath -> DynFlags -> DynFlags -setDepJson f d = d { depJson = Just f } +setDepJson f d = d { depJson = MkDepend_Explicit f + -- We generate a Makefile by default, but if a user requests json + -- output, then they most likely don't need a Makefile. + -- Thus, we only generate it if the user requests it. + , depMakefile = unsetMkDependOutputIfDefault (depMakefile d) + } + +setDepJsonStdout :: DynFlags -> DynFlags +setDepJsonStdout d = d { depJson = MkDepend_StdOut + -- We generate a Makefile by default, but if a user requests json + -- output, then they most likely don't need a Makefile. + -- Thus, we only generate it if the user requests it. + , depMakefile = unsetMkDependOutputIfDefault (depMakefile d) + } unsetDepJson :: DynFlags -> DynFlags -unsetDepJson d = d { depJson = Nothing } +unsetDepJson d = d { depJson = MkDepend_None } + +unsetMkDependOutputIfDefault :: MkDependOutput -> MkDependOutput +unsetMkDependOutputIfDefault (MkDepend_Default _) = MkDepend_None +unsetMkDependOutputIfDefault mkDepend_output = mkDepend_output setDepIncludeCppDeps :: Bool -> DynFlags -> DynFlags setDepIncludeCppDeps b d = d { depIncludeCppDeps = b } @@ -1272,8 +1300,10 @@ dynamic_flags_deps = [ , make_ord_flag defGhcFlag "dep-suffix" (hasArg addDepSuffix) , make_ord_flag defGhcFlag "dep-makefile" (hasArg setDepMakefile) , make_ord_flag defGhcFlag "no-dep-makefile" (noArg unsetDepMakefile) + , make_ord_flag defGhcFlag "dep-makefile-stdout" (noArg setDepMakefileStdout) , make_ord_flag defGhcFlag "dep-json" (hasArg setDepJson) , make_ord_flag defGhcFlag "no-dep-json" (noArg unsetDepJson) + , make_ord_flag defGhcFlag "dep-json-stdout" (noArg setDepJsonStdout) , make_ord_flag defGhcFlag "include-cpp-deps" (noArg (setDepIncludeCppDeps True)) , make_ord_flag defGhcFlag "include-pkg-deps" ===================================== docs/users_guide/separate_compilation.rst ===================================== @@ -1511,6 +1511,23 @@ generation are: Introduced in GHC 10.1, ``-no-dep-makefile`` disables the generation of a Makefile altogether. +.. ghc-flag:: -dep-makefile-stdout + :shortdesc: Write dependency generation to stdout using makefile format + :type: dynamic + :reverse: -no-dep-makefile + :category: redirect-output + + :since: 10.1 + + Stream the dependency generation of :ghc-flag:`-dep-makefile ⟨file⟩` to stdout instead of ⟨file⟩. + + Contrary to :ghc-flag:`-dep-makefile ⟨file⟩`, the markers + + * "``# DO NOT DELETE: Beginning of Haskell dependencies``" and + * "``# DO NOT DELETE: End of Haskell dependencies``" + + are not emitted. + .. ghc-flag:: -dep-json ⟨file⟩ :shortdesc: Write dependencies to ⟨file⟩ as a JSON Lines format :type: dynamic @@ -1540,8 +1557,19 @@ generation are: ``-no-dep-json`` disables the generation of the JSON file. - By default, :ghc-flag:`-dep-makefile ⟨file⟩` is enabled even if :ghc-flag:`-dep-json ⟨file⟩` is given. - Disable the generation of any Makefile via ``-no-dep-makefile``. + :ghc-flag:`-dep-json ⟨file⟩` and :ghc-flag:`-dep-json-stdout` disable the default generation of a Makefile. + To generate a Makefile for the dependency information, explicitly use :ghc-flag:`-dep-makefile ⟨file⟩` or + :ghc-flag:`-dep-makefile-stdout`. + +.. ghc-flag:: -dep-json-stdout + :shortdesc: Write dependency generation to stdout using JSON Lines format + :type: dynamic + :reverse: -no-dep-json + :category: redirect-output + + :since: 10.1 + + Stream the dependency generation of :ghc-flag:`-dep-json ⟨file⟩` to stdout instead of ⟨file⟩. .. ghc-flag:: -dep-suffix ⟨suffix⟩ :shortdesc: Make dependencies that declare that files with suffix ===================================== testsuite/driver/my_typing.py ===================================== @@ -10,7 +10,7 @@ The testsuite driver can be typechecked using mypy [1]. from typing import * import typing - +import enum #################################################### # Backwards compatibility shims @@ -34,3 +34,7 @@ GitHash = NewType("GitHash", str) # a Git commit hash GitRef = NewType("GitRef", str) TestEnv = NewType("TestEnv", str) MetricName = NewType("MetricName", str) + +# Used by `ghc -M` test mode +DepMode = enum.Enum('DepMode', [('NONE', 0), ('FILE', 1), ('STDOUT', 2)]) + ===================================== testsuite/driver/testglobals.py ===================================== @@ -390,8 +390,8 @@ class TestOptions: self.ignore_extension = False # make_depend tests: which outputs to write and check - self.dep_makefile = False - self.dep_json = False + self.dep_makefile: DepMode = DepMode.NONE + self.dep_json: DepMode = DepMode.NONE self.ignore_dep_outputs = False # Backpack test ===================================== testsuite/driver/testlib.py ===================================== @@ -438,14 +438,26 @@ def dep_makefile( name, opts ): For make_depend tests: write the -dep-makefile output and compare it against <name>.dep-makefile. """ - opts.dep_makefile = True + opts.dep_makefile = DepMode.FILE def dep_json( name, opts ): """ For make_depend tests: write the -dep-json output and compare it against <name>.dep-json. """ - opts.dep_json = True + opts.dep_json = DepMode.FILE + +def dep_makefile_stdout( name, opts ): + """ + For make_depend tests: write the -dep-makefile output to stdout. + """ + opts.dep_makefile = DepMode.STDOUT + +def dep_json_stdout( name, opts ): + """ + For make_depend tests: write the -dep-json output to stdout. + """ + opts.dep_json = DepMode.STDOUT def ignore_dep_outputs( name, opts ): """ @@ -2060,14 +2072,19 @@ async def do_make_depend(name: TestName, outputs = [] output_flags = [] - if opts.dep_makefile: + if opts.dep_makefile == DepMode.FILE: outputs.append(('dep-makefile', normalise_dep_makefile)) output_flags.append('-dep-makefile %s' % actual_file('dep-makefile')) - else: - output_flags.append('-no-dep-makefile') - if opts.dep_json: + elif opts.dep_makefile == DepMode.STDOUT: + outputs.append(('stdout', normalise_dep_makefile)) + output_flags.append('-dep-makefile-stdout') + + if opts.dep_json == DepMode.FILE: outputs.append(('dep-json', normalise_dep_json)) output_flags.append('-dep-json %s' % actual_file('dep-json')) + elif opts.dep_json == DepMode.STDOUT: + outputs.append(('stdout', normalise_dep_json)) + output_flags.append('-dep-json-stdout') result = await do_compile( name, way, False, None, [], units, ' '.join(output_flags + [extra_hc_opts]), @@ -2077,9 +2094,9 @@ async def do_make_depend(name: TestName, for (kind, normaliser) in outputs: output_match = await compare_outputs(way, kind, - join_normalisers(opts.extra_normaliser, - normalise_slashes_, normaliser), - find_expected_file(name, kind, way), actual_file(kind)) + join_normalisers(opts.extra_normaliser, normalise_slashes_, normaliser), + find_expected_file(name, kind, way), + actual_file(kind)) if not output_match: return failBecause(kind + ' mismatch', diff=output_match.diff) ===================================== testsuite/tests/driver/mkdepend/all.T ===================================== @@ -15,9 +15,20 @@ test('T24384b', [ *T24384_extras, when(opsys('mingw32'), skip), *json_normaliser test('T24384c', [ *T24384_extras, when(opsys('mingw32'), skip), *json_normalisers ], makefile_test, []) # Multiple home units with explicit level imports. -test('mkdepend_mhu01', +test('mkdepend_mhu01a', [ extra_files(['mhu01/']), dep_makefile, dep_json ], make_depend_multiunit, [['mhu01/unitA', 'mhu01/unitB', 'mhu01/unitC'], '']) +# test the output streaming and writing to files +test('mkdepend_mhu01b', + [ extra_files(['mhu01/']), dep_json_stdout ], + make_depend_multiunit, [['mhu01/unitA', 'mhu01/unitB', 'mhu01/unitC'], '']) +test('mkdepend_mhu01c', + [ extra_files(['mhu01/']), dep_makefile_stdout ], + make_depend_multiunit, [['mhu01/unitA', 'mhu01/unitB', 'mhu01/unitC'], '']) +test('mkdepend_mhu01d', + [ extra_files(['mhu01/']), dep_json_stdout, dep_makefile ], + make_depend_multiunit, [['mhu01/unitA', 'mhu01/unitB', 'mhu01/unitC'], '']) + # Preprocessor must not be inherited by default active home unit test('mkdepend_mhu02', [ extra_files(['mhu02/']), dep_json ===================================== testsuite/tests/driver/mkdepend/mkdepend_mhu01.dep-json → testsuite/tests/driver/mkdepend/mkdepend_mhu01a.dep-json ===================================== ===================================== testsuite/tests/driver/mkdepend/mkdepend_mhu01.dep-makefile → testsuite/tests/driver/mkdepend/mkdepend_mhu01a.dep-makefile ===================================== @@ -1,23 +1,23 @@ # DO NOT DELETE: Beginning of Haskell dependencies +mhu01/a/A/Base.o-boot : mhu01/a/A/Base.hs-boot mhu01/a/A/Base.o : mhu01/a/A/Base.hi-boot +mhu01/a/Util.o : mhu01/a/Util.hs +mhu01/a/Util.o : mhu01/a/A/Base.hi-boot mhu01/a/A/Base.o : mhu01/a/A/Base.hs mhu01/a/A/Base.o : mhu01/a/Util.hi -mhu01/a/A/Base.o-boot : mhu01/a/A/Base.hs-boot -mhu01/a/A/TH.o : mhu01/a/A/Base.hi mhu01/a/A/TH.o : mhu01/a/A/TH.hs -mhu01/a/Util.o : mhu01/a/A/Base.hi-boot -mhu01/a/Util.o : mhu01/a/Util.hs -mhu01/b/B/Normal.o : mhu01/a/A/Base.hi -mhu01/b/B/Normal.o : mhu01/b/B/Normal.hs -mhu01/b/B/Normal.o : mhu01/b/B/Splice.hi -mhu01/b/B/Splice.o : mhu01/a/A/Base.hi -mhu01/b/B/Splice.o : mhu01/a/A/TH.hi +mhu01/a/A/TH.o : mhu01/a/A/Base.hi +mhu01/b/Util.o : mhu01/b/Util.hs mhu01/b/B/Splice.o : mhu01/b/B/Splice.hs mhu01/b/B/Splice.o : mhu01/b/Util.hi -mhu01/b/Util.o : mhu01/b/Util.hs -mhu01/c/C.o : mhu01/a/Util.hi -mhu01/c/C.o : mhu01/b/B/Normal.hi +mhu01/b/B/Splice.o : mhu01/a/A/Base.hi +mhu01/b/B/Splice.o : mhu01/a/A/TH.hi +mhu01/b/B/Normal.o : mhu01/b/B/Normal.hs +mhu01/b/B/Normal.o : mhu01/b/B/Splice.hi +mhu01/b/B/Normal.o : mhu01/a/A/Base.hi +mhu01/c/C.o : mhu01/c/C.hs mhu01/c/C.o : mhu01/b/B/Splice.hi +mhu01/c/C.o : mhu01/b/B/Normal.hi mhu01/c/C.o : mhu01/b/Util.hi -mhu01/c/C.o : mhu01/c/C.hs +mhu01/c/C.o : mhu01/a/Util.hi # DO NOT DELETE: End of Haskell dependencies ===================================== testsuite/tests/driver/mkdepend/mkdepend_mhu01b.stderr ===================================== @@ -0,0 +1,8 @@ +{"module":"A.Base","unit-id":"a","boot":true,"imports":{},"package-name":"a","source":"mhu01/a/A/Base.hs-boot"} +{"module":"Util","unit-id":"a","imports":{"a":[{"module":"A.Base","boot":true}]},"package-name":"a","source":"mhu01/a/Util.hs"} +{"module":"A.Base","unit-id":"a","imports":{"a":["Util"]},"package-name":"a","source":"mhu01/a/A/Base.hs"} +{"module":"A.TH","unit-id":"a","imports":{"a":["A.Base"]},"package-name":"a","source":"mhu01/a/A/TH.hs"} +{"module":"Util","unit-id":"b","imports":{},"package-name":"b","source":"mhu01/b/Util.hs"} +{"module":"B.Splice","unit-id":"b","imports":{"a":[{"module":"A.Base","level":"quote"},{"module":"A.TH","level":"splice"}],"b":["Util"]},"package-name":"b","source":"mhu01/b/B/Splice.hs"} +{"module":"B.Normal","unit-id":"b","imports":{"a":["A.Base"],"b":["B.Splice"]},"package-name":"b","source":"mhu01/b/B/Normal.hs"} +{"module":"C","unit-id":"c","imports":{"a":[{"module":"Util","level":"splice"}],"b":["B.Normal",{"module":"B.Splice","level":"splice"},{"module":"Util","level":"quote"}]},"package-name":"c","source":"mhu01/c/C.hs"} ===================================== testsuite/tests/driver/mkdepend/mkdepend_mhu01c.stderr ===================================== @@ -0,0 +1,21 @@ +mhu01/a/A/Base.o-boot : mhu01/a/A/Base.hs-boot +mhu01/a/A/Base.o : mhu01/a/A/Base.hi-boot +mhu01/a/Util.o : mhu01/a/Util.hs +mhu01/a/Util.o : mhu01/a/A/Base.hi-boot +mhu01/a/A/Base.o : mhu01/a/A/Base.hs +mhu01/a/A/Base.o : mhu01/a/Util.hi +mhu01/a/A/TH.o : mhu01/a/A/TH.hs +mhu01/a/A/TH.o : mhu01/a/A/Base.hi +mhu01/b/Util.o : mhu01/b/Util.hs +mhu01/b/B/Splice.o : mhu01/b/B/Splice.hs +mhu01/b/B/Splice.o : mhu01/b/Util.hi +mhu01/b/B/Splice.o : mhu01/a/A/Base.hi +mhu01/b/B/Splice.o : mhu01/a/A/TH.hi +mhu01/b/B/Normal.o : mhu01/b/B/Normal.hs +mhu01/b/B/Normal.o : mhu01/b/B/Splice.hi +mhu01/b/B/Normal.o : mhu01/a/A/Base.hi +mhu01/c/C.o : mhu01/c/C.hs +mhu01/c/C.o : mhu01/b/B/Splice.hi +mhu01/c/C.o : mhu01/b/B/Normal.hi +mhu01/c/C.o : mhu01/b/Util.hi +mhu01/c/C.o : mhu01/a/Util.hi ===================================== testsuite/tests/driver/mkdepend/mkdepend_mhu01d.dep-makefile ===================================== @@ -0,0 +1,23 @@ +# DO NOT DELETE: Beginning of Haskell dependencies +mhu01/a/A/Base.o-boot : mhu01/a/A/Base.hs-boot +mhu01/a/A/Base.o : mhu01/a/A/Base.hi-boot +mhu01/a/Util.o : mhu01/a/Util.hs +mhu01/a/Util.o : mhu01/a/A/Base.hi-boot +mhu01/a/A/Base.o : mhu01/a/A/Base.hs +mhu01/a/A/Base.o : mhu01/a/Util.hi +mhu01/a/A/TH.o : mhu01/a/A/TH.hs +mhu01/a/A/TH.o : mhu01/a/A/Base.hi +mhu01/b/Util.o : mhu01/b/Util.hs +mhu01/b/B/Splice.o : mhu01/b/B/Splice.hs +mhu01/b/B/Splice.o : mhu01/b/Util.hi +mhu01/b/B/Splice.o : mhu01/a/A/Base.hi +mhu01/b/B/Splice.o : mhu01/a/A/TH.hi +mhu01/b/B/Normal.o : mhu01/b/B/Normal.hs +mhu01/b/B/Normal.o : mhu01/b/B/Splice.hi +mhu01/b/B/Normal.o : mhu01/a/A/Base.hi +mhu01/c/C.o : mhu01/c/C.hs +mhu01/c/C.o : mhu01/b/B/Splice.hi +mhu01/c/C.o : mhu01/b/B/Normal.hi +mhu01/c/C.o : mhu01/b/Util.hi +mhu01/c/C.o : mhu01/a/Util.hi +# DO NOT DELETE: End of Haskell dependencies ===================================== testsuite/tests/driver/mkdepend/mkdepend_mhu01d.stderr ===================================== @@ -0,0 +1,8 @@ +{"module":"A.Base","unit-id":"a","boot":true,"imports":{},"package-name":"a","source":"mhu01/a/A/Base.hs-boot"} +{"module":"Util","unit-id":"a","imports":{"a":[{"module":"A.Base","boot":true}]},"package-name":"a","source":"mhu01/a/Util.hs"} +{"module":"A.Base","unit-id":"a","imports":{"a":["Util"]},"package-name":"a","source":"mhu01/a/A/Base.hs"} +{"module":"A.TH","unit-id":"a","imports":{"a":["A.Base"]},"package-name":"a","source":"mhu01/a/A/TH.hs"} +{"module":"Util","unit-id":"b","imports":{},"package-name":"b","source":"mhu01/b/Util.hs"} +{"module":"B.Splice","unit-id":"b","imports":{"a":[{"module":"A.Base","level":"quote"},{"module":"A.TH","level":"splice"}],"b":["Util"]},"package-name":"b","source":"mhu01/b/B/Splice.hs"} +{"module":"B.Normal","unit-id":"b","imports":{"a":["A.Base"],"b":["B.Splice"]},"package-name":"b","source":"mhu01/b/B/Normal.hs"} +{"module":"C","unit-id":"c","imports":{"a":[{"module":"Util","level":"splice"}],"b":["B.Normal",{"module":"B.Splice","level":"splice"},{"module":"Util","level":"quote"}]},"package-name":"c","source":"mhu01/c/C.hs"} View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/419cfed9e5775cccf05d7217255ad0a8... -- View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/419cfed9e5775cccf05d7217255ad0a8... You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications: https://gitlab.haskell.org/-/profile/notifications | Help: https://gitlab.haskell.org/help
participants (1)
-
Hannes Siebenhandl (@fendor)