Hassan Al-Awwadi pushed to branch wip/haanss/depdir at Glasgow Haskell Compiler / GHC
Commits:
01d3154e by Wen Kokke at 2025-07-10T17:06:36+01:00
Fix documentation for HEAP_PROF_SAMPLE_STRING
- - - - -
ac259c48 by Wen Kokke at 2025-07-10T17:06:38+01:00
Fix documentation for HEAP_PROF_SAMPLE_COST_CENTRE
- - - - -
2b4db9ba by Pi Delport at 2025-07-11T16:40:52-04:00
(Applicative docs typo: missing "one")
- - - - -
0bfcaf03 by Hassan Al-Awwadi at 2025-07-14T18:54:33+02:00
Adds the fucnction addDependentDirectory to Q, resolving issue #26148.
This function adds a new directory to the list of things a module depends upon. That means that when the contents of the directory change, the recompilation checker will notice this and the module will be recompiled. Documentation has also been added for addDependentFunction and addDependentDirectory in the user guide.
- - - - -
25 changed files:
- compiler/GHC/HsToCore/Usage.hs
- compiler/GHC/Iface/Make.hs
- compiler/GHC/Iface/Recomp.hs
- compiler/GHC/Iface/Recomp/Types.hs
- compiler/GHC/Tc/Gen/Splice.hs
- compiler/GHC/Tc/Types.hs
- compiler/GHC/Tc/Utils/Monad.hs
- compiler/GHC/Unit/Finder.hs
- compiler/GHC/Unit/Finder/Types.hs
- compiler/GHC/Unit/Module/Deps.hs
- docs/users_guide/9.14.1-notes.rst
- docs/users_guide/eventlog-formats.rst
- docs/users_guide/separate_compilation.rst
- libraries/ghc-internal/src/GHC/Internal/Base.hs
- libraries/ghc-internal/src/GHC/Internal/TH/Syntax.hs
- libraries/ghci/GHCi/Message.hs
- libraries/ghci/GHCi/TH.hs
- libraries/template-haskell/Language/Haskell/TH/Syntax.hs
- testsuite/.gitignore
- testsuite/tests/interface-stability/template-haskell-exports.stdout
- testsuite/tests/th/Makefile
- + testsuite/tests/th/TH_Depends_Dir.hs
- + testsuite/tests/th/TH_Depends_Dir.stdout
- + testsuite/tests/th/TH_Depends_Dir_External.hs
- testsuite/tests/th/all.T
Changes:
=====================================
compiler/GHC/HsToCore/Usage.hs
=====================================
@@ -75,14 +75,15 @@ data UsageConfig = UsageConfig
mkUsageInfo :: UsageConfig -> Plugins -> FinderCache -> UnitEnv
-> Module -> ImportedMods -> [ImportUserSpec] -> NameSet
- -> [FilePath] -> [(Module, Fingerprint)] -> [Linkable] -> PkgsLoaded
+ -> [FilePath] -> [FilePath] -> [(Module, Fingerprint)] -> [Linkable] -> PkgsLoaded
-> IfG [Usage]
mkUsageInfo uc plugins fc unit_env
this_mod dir_imp_mods imp_decls used_names
- dependent_files merged needed_links needed_pkgs
+ dependent_files dependent_dirs merged needed_links needed_pkgs
= do
eps <- liftIO $ readIORef (euc_eps (ue_eps unit_env))
- hashes <- liftIO $ mapM getFileHash dependent_files
+ file_hashes <- liftIO $ mapM getFileHash dependent_files
+ dirs_hashes <- liftIO $ mapM getDirHash dependent_dirs
let hu = ue_unsafeHomeUnit unit_env
hug = ue_home_unit_graph unit_env
-- Dependencies on object files due to TH and plugins
@@ -93,7 +94,11 @@ mkUsageInfo uc plugins fc unit_env
let usages = mod_usages ++ [ UsageFile { usg_file_path = mkFastString f
, usg_file_hash = hash
, usg_file_label = Nothing }
- | (f, hash) <- zip dependent_files hashes ]
+ | (f, hash) <- zip dependent_files file_hashes ]
+ ++ [ UsageDirectory { usg_dir_path = mkFastString d
+ , usg_dir_hash = hash
+ , usg_dir_label = Nothing }
+ | (d, hash) <- zip dependent_dirs dirs_hashes]
++ [ UsageMergedRequirement
{ usg_mod = mod,
usg_mod_hash = hash
=====================================
compiler/GHC/Iface/Make.hs
=====================================
@@ -269,6 +269,7 @@ mkRecompUsageInfo hsc_env tc_result = do
else do
let used_names = mkUsedNames tc_result
dep_files <- (readIORef (tcg_dependent_files tc_result))
+ dep_dirs <- (readIORef (tcg_dependent_dirs tc_result))
(needed_links, needed_pkgs) <- readIORef (tcg_th_needed_deps tc_result)
let uc = initUsageConfig hsc_env
plugins = hsc_plugins hsc_env
@@ -289,6 +290,7 @@ mkRecompUsageInfo hsc_env tc_result = do
(tcg_import_decls tc_result)
used_names
dep_files
+ dep_dirs
(tcg_merged tc_result)
needed_links
needed_pkgs
=====================================
compiler/GHC/Iface/Recomp.hs
=====================================
@@ -194,6 +194,7 @@ data RecompReason
| ModuleChangedRaw ModuleName
| ModuleChangedIface ModuleName
| FileChanged FilePath
+ | DirChanged FilePath
| CustomReason String
| FlagsChanged
| LinkFlagsChanged
@@ -230,6 +231,7 @@ instance Outputable RecompReason where
ModuleRemoved (_st, _uid, m) -> ppr m <+> text "removed"
ModuleAdded (_st, _uid, m) -> ppr m <+> text "added"
FileChanged fp -> text fp <+> text "changed"
+ DirChanged dp -> text dp <+> text "changed"
CustomReason s -> text s
FlagsChanged -> text "Flags changed"
LinkFlagsChanged -> text "Flags changed"
@@ -815,6 +817,22 @@ checkModUsage fc UsageFile{ usg_file_path = file,
then \e -> pprTrace "UsageFile" (text (show e)) $ return recomp
else \_ -> return recomp -- if we can't find the file, just recompile, don't fail
+checkModUsage fc UsageDirectory{ usg_dir_path = dir,
+ usg_dir_hash = old_hash,
+ usg_dir_label = mlabel } =
+ liftIO $
+ handleIO handler $ do
+ new_hash <- lookupDirCache fc $ unpackFS dir
+ if (old_hash /= new_hash)
+ then return recomp
+ else return UpToDate
+ where
+ reason = DirChanged $ unpackFS dir
+ recomp = needsRecompileBecause $ fromMaybe reason $ fmap CustomReason mlabel
+ handler = if debugIsOn
+ then \e -> pprTrace "UsageDirectory" (text (show e)) $ return recomp
+ else \_ -> return recomp -- if we can't find the dir, just recompile, don't fail
+
-- | We are importing a module whose exports have changed.
-- Does this require recompilation?
--
=====================================
compiler/GHC/Iface/Recomp/Types.hs
=====================================
@@ -140,6 +140,10 @@ pprUsage usage@UsageFile{}
= hsep [text "addDependentFile",
doubleQuotes (ftext (usg_file_path usage)),
ppr (usg_file_hash usage)]
+pprUsage usage@UsageDirectory{}
+ = hsep [text "AddDependentDirectory",
+ doubleQuotes (ftext (usg_dir_path usage)),
+ ppr (usg_dir_hash usage)]
pprUsage usage@UsageMergedRequirement{}
= hsep [text "merged", ppr (usg_mod usage), ppr (usg_mod_hash usage)]
pprUsage usage@UsageHomeModuleInterface{}
=====================================
compiler/GHC/Tc/Gen/Splice.hs
=====================================
@@ -173,8 +173,6 @@ import GHC.Parser.HaddockLex (lexHsDoc)
import GHC.Parser (parseIdentifier)
import GHC.Rename.Doc (rnHsDoc)
-
-
{-
Note [Template Haskell state diagram]
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -1524,6 +1522,11 @@ instance TH.Quasi TcM where
dep_files <- readTcRef ref
writeTcRef ref (fp:dep_files)
+ qAddDependentDirectory dp = do
+ ref <- fmap tcg_dependent_dirs getGblEnv
+ dep_dirs <- readTcRef ref
+ writeTcRef ref (dp:dep_dirs)
+
qAddTempFile suffix = do
dflags <- getDynFlags
logger <- getLogger
@@ -1928,6 +1931,7 @@ handleTHMessage msg = case msg of
ReifyConStrictness nm -> wrapTHResult $ TH.qReifyConStrictness nm
GetPackageRoot -> wrapTHResult $ TH.qGetPackageRoot
AddDependentFile f -> wrapTHResult $ TH.qAddDependentFile f
+ AddDependentDirectory d -> wrapTHResult $ TH.qAddDependentDirectory d
AddTempFile s -> wrapTHResult $ TH.qAddTempFile s
AddModFinalizer r -> do
interp <- hscInterp <$> getTopEnv
=====================================
compiler/GHC/Tc/Types.hs
=====================================
@@ -603,6 +603,7 @@ data TcGblEnv
-- decls.
tcg_dependent_files :: TcRef [FilePath], -- ^ dependencies from addDependentFile
+ tcg_dependent_dirs :: TcRef [FilePath], -- ^ dependencies from addDependentDirectory
tcg_th_topdecls :: TcRef [LHsDecl GhcPs],
-- ^ Top-level declarations from addTopDecls
=====================================
compiler/GHC/Tc/Utils/Monad.hs
=====================================
@@ -55,7 +55,7 @@ module GHC.Tc.Utils.Monad(
getRdrEnvs, getImports,
getFixityEnv, extendFixityEnv,
getDeclaredDefaultTys,
- addDependentFiles,
+ addDependentFiles, addDependentDirectories,
-- * Error management
getSrcSpanM, setSrcSpan, setSrcSpanA, addLocM,
@@ -274,6 +274,7 @@ initTc hsc_env hsc_src keep_rn_syntax mod loc do_this
let { type_env_var = hsc_type_env_vars hsc_env };
dependent_files_var <- newIORef [] ;
+ dependent_dirs_var <- newIORef [] ;
static_wc_var <- newIORef emptyWC ;
cc_st_var <- newIORef newCostCentreState ;
th_topdecls_var <- newIORef [] ;
@@ -369,6 +370,7 @@ initTc hsc_env hsc_src keep_rn_syntax mod loc do_this
tcg_safe_infer = infer_var,
tcg_safe_infer_reasons = infer_reasons_var,
tcg_dependent_files = dependent_files_var,
+ tcg_dependent_dirs = dependent_dirs_var,
tcg_tc_plugin_solvers = [],
tcg_tc_plugin_rewriters = emptyUFM,
tcg_defaulting_plugins = [],
@@ -957,6 +959,12 @@ addDependentFiles fs = do
dep_files <- readTcRef ref
writeTcRef ref (fs ++ dep_files)
+addDependentDirectories :: [FilePath] -> TcRn ()
+addDependentDirectories ds = do
+ ref <- fmap tcg_dependent_dirs getGblEnv
+ dep_dirs <- readTcRef ref
+ writeTcRef ref (ds ++ dep_dirs)
+
{-
************************************************************************
* *
=====================================
compiler/GHC/Unit/Finder.hs
=====================================
@@ -31,6 +31,9 @@ module GHC.Unit.Finder (
findObjectLinkableMaybe,
findObjectLinkable,
+
+ -- important that GHC.HsToCore.Usage uses the same hashing method for usage dirs as is used here.
+ getDirHash,
) where
import GHC.Prelude
@@ -68,7 +71,9 @@ import qualified Data.Map as M
import GHC.Driver.Env
import GHC.Driver.Config.Finder
import qualified Data.Set as Set
+import qualified Data.List as L(sort)
import Data.List.NonEmpty ( NonEmpty (..) )
+import qualified System.Directory as SD
import qualified System.OsPath as OsPath
import qualified Data.List.NonEmpty as NE
@@ -107,10 +112,12 @@ initFinderCache :: IO FinderCache
initFinderCache = do
mod_cache <- newIORef emptyInstalledModuleEnv
file_cache <- newIORef M.empty
+ dir_cache <- newIORef M.empty
let flushFinderCaches :: UnitEnv -> IO ()
flushFinderCaches ue = do
atomicModifyIORef' mod_cache $ \fm -> (filterInstalledModuleEnv is_ext fm, ())
atomicModifyIORef' file_cache $ \_ -> (M.empty, ())
+ atomicModifyIORef' dir_cache $ \_ -> (M.empty, ())
where
is_ext mod _ = not (isUnitEnvInstalledModule ue mod)
@@ -137,8 +144,29 @@ initFinderCache = do
atomicModifyIORef' file_cache $ \c -> (M.insert key hash c, ())
return hash
Just fp -> return fp
+ lookupDirCache :: FilePath -> IO Fingerprint
+ lookupDirCache key = do
+ c <- readIORef dir_cache
+ case M.lookup key c of
+ Nothing -> do
+ hash <- getDirHash key
+ atomicModifyIORef' dir_cache $ \c -> (M.insert key hash c, ())
+ return hash
+ Just fp -> return fp
return FinderCache{..}
+-- | This function computes a shallow hash of a directory, so really just what files and directories are directly inside it.
+-- It does not look at the contents of the files, or the contents of the directories it contains.
+getDirHash :: FilePath -> IO Fingerprint
+getDirHash dir = do
+ contents <- SD.listDirectory dir
+ -- The documentation of Fingerprints describes this as an easy naive implementation
+ -- I wonder if we should do something more sophisticated here?
+ let hashes = fingerprintString <$> contents
+ let s_hashes = L.sort hashes
+ let hash = fingerprintFingerprints s_hashes
+ return hash
+
-- -----------------------------------------------------------------------------
-- The three external entry points
=====================================
compiler/GHC/Unit/Finder/Types.hs
=====================================
@@ -37,6 +37,7 @@ data FinderCache = FinderCache { flushFinderCaches :: UnitEnv -> IO ()
, lookupFileCache :: FilePath -> IO Fingerprint
-- ^ Look for the hash of a file in the cache. This should add it to the
-- cache. If the file doesn't exist, raise an IOException.
+ , lookupDirCache :: FilePath -> IO Fingerprint
}
data InstalledFindResult
=====================================
compiler/GHC/Unit/Module/Deps.hs
=====================================
@@ -357,6 +357,23 @@ data Usage
-- contents don't change. This previously lead to odd
-- recompilation behaviors; see #8114
}
+ | UsageDirectory {
+ usg_dir_path :: FastString,
+ -- ^ External dir dependency. From TH addDependentFile.
+ -- Should be absolute.
+ usg_dir_hash :: Fingerprint,
+ -- ^ 'Fingerprint' of the directories contents.
+
+ usg_dir_label :: Maybe String
+ -- ^ An optional string which is used in recompilation messages if
+ -- dir in question has changed.
+
+ -- Note: We do a very shallow check indeed, just what the contents of
+ -- the directory are, aka what files and directories are within it.
+ -- If those files/directories have their own contents changed...
+ -- We won't spot it here, better recursive add them to your usage
+ -- seperately.
+ }
| UsageHomeModuleInterface {
usg_mod_name :: ModuleName
-- ^ Name of the module
@@ -395,6 +412,7 @@ instance NFData Usage where
rnf (UsagePackageModule mod hash safe) = rnf mod `seq` rnf hash `seq` rnf safe `seq` ()
rnf (UsageHomeModule mod uid hash entities exports safe) = rnf mod `seq` rnf uid `seq` rnf hash `seq` rnf entities `seq` rnf exports `seq` rnf safe `seq` ()
rnf (UsageFile file hash label) = rnf file `seq` rnf hash `seq` rnf label `seq` ()
+ rnf (UsageDirectory dir hash label) = rnf dir `seq` rnf hash `seq` rnf label `seq` ()
rnf (UsageMergedRequirement mod hash) = rnf mod `seq` rnf hash `seq` ()
rnf (UsageHomeModuleInterface mod uid hash) = rnf mod `seq` rnf uid `seq` rnf hash `seq` ()
@@ -431,6 +449,12 @@ instance Binary Usage where
put_ bh (usg_unit_id usg)
put_ bh (usg_iface_hash usg)
+ put_ bh usg@UsageDirectory{} = do
+ putByte bh 5
+ put_ bh (usg_dir_path usg)
+ put_ bh (usg_dir_hash usg)
+ put_ bh (usg_dir_label usg)
+
get bh = do
h <- getByte bh
case h of
@@ -462,6 +486,12 @@ instance Binary Usage where
uid <- get bh
hash <- get bh
return UsageHomeModuleInterface { usg_mod_name = mod, usg_unit_id = uid, usg_iface_hash = hash }
+ 5 -> do
+ dp <- get bh
+ hash <- get bh
+ label <- get bh
+ return UsageDirectory { usg_dir_path = dp, usg_dir_hash = hash, usg_dir_label = label }
+
i -> error ("Binary.get(Usage): " ++ show i)
-- | Records the imports that we depend on from a home module,
=====================================
docs/users_guide/9.14.1-notes.rst
=====================================
@@ -232,6 +232,11 @@ Cmm
They are replaced, respectively, by ``SpecialiseEP``, ``pragSpecED`` and
``pragSpecInlED``.
+- We have added the ``addDependentDirectory`` function to match
+ ``addDependentFile``, which adds a directory to the list of dependencies that
+ the recompilation checker will look at to determine if a module needs to be
+ recompiled.
+
Included libraries
~~~~~~~~~~~~~~~~~~
=====================================
docs/users_guide/eventlog-formats.rst
=====================================
@@ -779,9 +779,9 @@ the total time spent profiling.
Cost-centre break-down
^^^^^^^^^^^^^^^^^^^^^^
-A variable-length packet encoding a heap profile sample broken down by,
- * cost-centre (:rts-flag:`-hc`)
-
+A variable-length packet encoding a heap profile sample.
+This event is only emitted when the heap profile type is set to :rts-flag:`-hc` or :rts-flag:`-hb`.
+Otherwise, a :event-type:`HEAP_PROF_SAMPLE_STRING` event is emitted instead.
.. event-type:: HEAP_PROF_SAMPLE_COST_CENTRE
@@ -796,11 +796,19 @@ A variable-length packet encoding a heap profile sample broken down by,
String break-down
^^^^^^^^^^^^^^^^^
-A variable-length event encoding a heap sample broken down by,
+A variable-length event encoding a heap sample.
+The content of the sample label varies depending on the heap profile type:
+
+ * :rts-flag:`-hT` The sample label contains a closure type, e.g., ``"ghc-bignum:GHC.Num.Integer.IS"``.
+ * :rts-flag:`-hm` The sample label contains a module name, e.g., ``"GHC.Num.Integer"``.
+ * :rts-flag:`-hd` The sample label contains a closure description, e.g., ``"IS"``.
+ * :rts-flag:`-hy` The sample label contains a type description, e.g., ``"Integer"``.
+ * :rts-flag:`-he` The sample label contains a stringified era, e.g., ``"1"``.
+ * :rts-flag:`-hr` The sample label contains a retainer set description, e.g., ``"(184)$stoIntegralSized1"``.
+ * :rts-flag:`-hi` The sample label contains a stringified pointer, e.g., ``"0x1008b7588"``,
+ which can be matched to an info table description emitted by the :event-type:`IPE` event.
- * type description (:rts-flag:`-hy`)
- * closure description (:rts-flag:`-hd`)
- * module (:rts-flag:`-hm`)
+If the heap profile type is set to :rts-flag:`-hc` or :rts-flag:`-hb`, a :event-type:`HEAP_PROF_SAMPLE_COST_CENTRE` event is emitted instead.
.. event-type:: HEAP_PROF_SAMPLE_STRING
@@ -808,7 +816,7 @@ A variable-length event encoding a heap sample broken down by,
:length: variable
:field Word8: profile ID
:field Word64: heap residency in bytes
- :field String: type or closure description, or module name
+ :field String: sample label
.. _time-profiler-events:
=====================================
docs/users_guide/separate_compilation.rst
=====================================
@@ -710,7 +710,7 @@ beautiful sight!
You can read about :ghc-wiki:`how all this works