Hannes Siebenhandl pushed to branch wip/fendor/external-unit-db-cache at Glasgow Haskell Compiler / GHC

Commits:

11 changed files:

Changes:

  • compiler/GHC.hs
    ... ... @@ -671,7 +671,7 @@ setUnitDynFlagsNoCheck uid dflags1 = do
    671 671
       logger <- getLogger
    
    672 672
       hsc_env <- getSession
    
    673 673
     
    
    674
    -  (unit_state,home_unit,mconstants) <- liftIO $ initUnits logger dflags1 (hscUIC hsc_env) (hscEUDC hsc_env) (hsc_all_home_unit_ids hsc_env)
    
    674
    +  (unit_state,home_unit,mconstants) <- liftIO $ initUnits logger dflags1 (hscUIC hsc_env) (hsc_all_home_unit_ids hsc_env)
    
    675 675
       updated_dflags <- liftIO $ updatePlatformConstants dflags1 mconstants
    
    676 676
     
    
    677 677
       let upd hue =
    
    ... ... @@ -760,7 +760,7 @@ setProgramDynFlags_ invalidate_needed dflags = do
    760 760
                   old_hpt = homeUnitEnv_hpt homeUnitEnv
    
    761 761
                   home_units = HUG.allUnits (ue_home_unit_graph old_unit_env)
    
    762 762
     
    
    763
    -          (unit_state,home_unit,mconstants) <- liftIO $ initUnits logger dflags (ue_uic old_unit_env) (ue_eud old_unit_env) home_units
    
    763
    +          (unit_state,home_unit,mconstants) <- liftIO $ initUnits logger dflags (ue_uic old_unit_env) home_units
    
    764 764
     
    
    765 765
               updated_dflags <- liftIO $ updatePlatformConstants dflags0 mconstants
    
    766 766
               pure HomeUnitEnv
    
    ... ... @@ -778,7 +778,6 @@ setProgramDynFlags_ invalidate_needed dflags = do
    778 778
                   , ue_current_unit    = ue_currentUnit old_unit_env
    
    779 779
                   , ue_module_graph    = ue_module_graph old_unit_env
    
    780 780
                   , ue_eps             = ue_eps old_unit_env
    
    781
    -              , ue_eud             = ue_eud old_unit_env
    
    782 781
                   , ue_uic      = ue_uic old_unit_env
    
    783 782
                   }
    
    784 783
             modifySession $ \h -> hscSetFlags dflags1 h{ hsc_unit_env = unit_env }
    
    ... ... @@ -837,7 +836,6 @@ setProgramHUG_ invalidate_needed new_hug0 = do
    837 836
                 , ue_current_unit    = ue_currentUnit unit_env0
    
    838 837
                 , ue_eps             = ue_eps unit_env0
    
    839 838
                 , ue_module_graph    = ue_module_graph unit_env0
    
    840
    -            , ue_eud             = ue_eud unit_env0
    
    841 839
                 , ue_uic      = ue_uic unit_env0
    
    842 840
                 }
    
    843 841
           modifySession $ \h ->
    
    ... ... @@ -886,7 +884,7 @@ setProgramHUG_ invalidate_needed new_hug0 = do
    886 884
               old_hpt = homeUnitEnv_hpt homeUnitEnv
    
    887 885
               home_units = HUG.allUnits (ue_home_unit_graph unit_env)
    
    888 886
     
    
    889
    -      (unit_state,home_unit,mconstants) <- liftIO $ initUnits logger dflags (ue_uic unit_env) (ue_eud unit_env) home_units
    
    887
    +      (unit_state,home_unit,mconstants) <- liftIO $ initUnits logger dflags (ue_uic unit_env) home_units
    
    890 888
     
    
    891 889
           updated_dflags <- liftIO $ updatePlatformConstants dflags mconstants
    
    892 890
           pure HomeUnitEnv
    

  • compiler/GHC/Driver/Backpack.hs
    ... ... @@ -92,7 +92,7 @@ import GHC.Types.Error (mkUnknownDiagnostic)
    92 92
     import qualified GHC.Unit.Home.Graph as HUG
    
    93 93
     import GHC.Unit.Home.ModInfo
    
    94 94
     import GHC.Unit.Home.PackageTable
    
    95
    -import GHC.Unit.External.Database (cacheExternalUnitDatabase)
    
    95
    +import GHC.Unit.External.Index (cacheExternalUnitDatabase)
    
    96 96
     
    
    97 97
     -- | Entry point to compile a Backpack file.
    
    98 98
     doBackpack :: [FilePath] -> Ghc ()
    
    ... ... @@ -442,8 +442,8 @@ addInMemoryDatabase dflags u = do
    442 442
               { unitDatabasePath  = unsafeEncodeUtf $ "(in memory " ++ showSDoc dflags (ppr (unitId u)) ++ ")"
    
    443 443
               , unitDatabaseUnits = [u]
    
    444 444
               }
    
    445
    -    let eud = hscEUDC hsc_env
    
    446
    -    liftIO $ cacheExternalUnitDatabase eud newdb
    
    445
    +    let uic = hscUIC hsc_env
    
    446
    +    liftIO $ cacheExternalUnitDatabase uic newdb
    
    447 447
         -- added at the end because ordering matters
    
    448 448
         pure dflags
    
    449 449
               { packageDBFlags = packageDBFlags dflags ++ [PackageDB (PkgDbPath (unitDatabasePath newdb))]
    
    ... ... @@ -456,11 +456,10 @@ addUnit u = do
    456 456
         logger <- getLogger
    
    457 457
         let dflags0 = hsc_dflags hsc_env
    
    458 458
         let old_unit_env = hsc_unit_env hsc_env
    
    459
    -    let eud = hscEUDC hsc_env
    
    460 459
     
    
    461 460
         dflags1 <- addInMemoryDatabase dflags0 u
    
    462 461
     
    
    463
    -    (unit_state,home_unit,mconstants) <- liftIO $ initUnits logger dflags1 (ue_uic old_unit_env) eud (hsc_all_home_unit_ids hsc_env)
    
    462
    +    (unit_state,home_unit,mconstants) <- liftIO $ initUnits logger dflags1 (ue_uic old_unit_env) (hsc_all_home_unit_ids hsc_env)
    
    464 463
     
    
    465 464
     
    
    466 465
         -- update platform constants
    
    ... ... @@ -477,7 +476,6 @@ addUnit u = do
    477 476
                         (HUG.mkHomeUnitEnv unit_state dflags (ue_hpt old_unit_env) (Just home_unit))
    
    478 477
               , ue_eps       = ue_eps old_unit_env
    
    479 478
               , ue_module_graph = ue_module_graph old_unit_env
    
    480
    -          , ue_eud       = ue_eud old_unit_env
    
    481 479
               , ue_uic = ue_uic old_unit_env
    
    482 480
               }
    
    483 481
         setSession $ hscSetFlags dflags $ hsc_env { hsc_unit_env = unit_env }
    

  • compiler/GHC/Driver/Env.hs
    ... ... @@ -14,7 +14,6 @@ module GHC.Driver.Env
    14 14
        , hsc_HUG
    
    15 15
        , hsc_all_home_unit_ids
    
    16 16
        , hscUnitIndex
    
    17
    -   , hscUIC
    
    18 17
        , hscUpdateLoggerFlags
    
    19 18
        , hscUpdateHUG
    
    20 19
        , hscInsertHPT
    
    ... ... @@ -27,7 +26,7 @@ module GHC.Driver.Env
    27 26
        , runInteractiveHsc
    
    28 27
        , hscEPS
    
    29 28
        , hscEUD
    
    30
    -   , hscEUDC
    
    29
    +   , hscUIC
    
    31 30
        , hscInterp
    
    32 31
        , prepareAnnotations
    
    33 32
        , discardIC
    
    ... ... @@ -228,10 +227,7 @@ hscEPS :: HscEnv -> IO ExternalPackageState
    228 227
     hscEPS hsc_env = readIORef (euc_eps (ue_eps (hsc_unit_env hsc_env)))
    
    229 228
     
    
    230 229
     hscEUD :: HscEnv -> IO (ExternalUnitDatabases UnitId)
    
    231
    -hscEUD = readExternalUnitDatabases . hscEUDC
    
    232
    -
    
    233
    -hscEUDC :: HscEnv -> ExternalUnitDatabaseCache UnitId
    
    234
    -hscEUDC hsc_env = ue_eud (hsc_unit_env hsc_env)
    
    230
    +hscEUD = readExternalUnitDatabases . hscUIC
    
    235 231
     
    
    236 232
     hscUnitIndex :: HscEnv -> IO UnitIndex
    
    237 233
     hscUnitIndex hsc_env = ueUI (hsc_unit_env hsc_env)
    

  • compiler/GHC/Driver/Session/Units.hs
    ... ... @@ -131,7 +131,7 @@ initMulti unitArgsFiles lintDynFlagsAndSrcs = do
    131 131
       home_unit_graph <- forM initial_home_graph $ \homeUnitEnv -> do
    
    132 132
         let hue_flags = homeUnitEnv_dflags homeUnitEnv
    
    133 133
             dflags = homeUnitEnv_dflags homeUnitEnv
    
    134
    -    (unit_state,home_unit,mconstants) <- liftIO $ State.initUnits logger hue_flags (hscUIC hsc_env) (hscEUDC hsc_env) home_units
    
    134
    +    (unit_state,home_unit,mconstants) <- liftIO $ State.initUnits logger hue_flags (hscUIC hsc_env) home_units
    
    135 135
     
    
    136 136
         updated_dflags <- liftIO $ updatePlatformConstants dflags mconstants
    
    137 137
         emptyHpt <- liftIO $ emptyHomePackageTable
    

  • compiler/GHC/Unit/Env.hs
    ... ... @@ -178,19 +178,21 @@ data UnitEnv = UnitEnv
    178 178
         , ue_namever   :: !GhcNameVersion
    
    179 179
             -- ^ GHC name/version (used for dynamic library suffix)
    
    180 180
     
    
    181
    -    , ue_eud :: {-# UNPACK #-} !(ExternalUnitDatabaseCache UnitId)
    
    182
    -        -- ^ Global cache of already read package databases
    
    183
    -
    
    184 181
         , ue_uic :: {-# UNPACK #-} !UnitIndexCache
    
    185
    -        -- ^ Index of already processed 'UnitInfo's.
    
    186
    -        -- Shares state over all 'UnitState' in the 'HomeUnitGraph'.
    
    182
    +        -- ^ Global index of already processed external units.
    
    183
    +        -- Shares state over all 'UnitState's in the 'HomeUnitGraph'.
    
    184
    +        --
    
    185
    +        -- Allows sharing of 'UnitInfo's, ensuring each individual 'UnitInfo'
    
    186
    +        -- is retained a constant number of times.
    
    187
    +        --
    
    188
    +        -- See Note [Sharing 'UnitInfo's across the 'UnitEnv'] for details.
    
    187 189
         }
    
    188 190
     
    
    189 191
     ueEPS :: UnitEnv -> IO ExternalPackageState
    
    190 192
     ueEPS = eucEPS . ue_eps
    
    191 193
     
    
    192 194
     ueEUD :: UnitEnv -> IO (ExternalUnitDatabases UnitId)
    
    193
    -ueEUD = readExternalUnitDatabases . ue_eud
    
    195
    +ueEUD = readExternalUnitDatabases . ue_uic
    
    194 196
     
    
    195 197
     ueUI :: UnitEnv -> IO UnitIndex
    
    196 198
     ueUI = readUnitIndex . ue_uic
    
    ... ... @@ -199,7 +201,6 @@ ueUI = readUnitIndex . ue_uic
    199 201
     initUnitEnv :: UnitId -> HomeUnitGraph -> GhcNameVersion -> Platform -> IO UnitEnv
    
    200 202
     initUnitEnv cur_unit hug namever platform = do
    
    201 203
       eps <- initExternalUnitCache
    
    202
    -  eud <- initExternalUnitDatabaseCache
    
    203 204
       uic <- initUnitIndexCache
    
    204 205
       return $ UnitEnv
    
    205 206
         { ue_eps             = eps
    
    ... ... @@ -208,7 +209,6 @@ initUnitEnv cur_unit hug namever platform = do
    208 209
         , ue_current_unit    = cur_unit
    
    209 210
         , ue_platform        = platform
    
    210 211
         , ue_namever         = namever
    
    211
    -    , ue_eud             = eud
    
    212 212
         , ue_uic      = uic
    
    213 213
         }
    
    214 214
     
    

  • compiler/GHC/Unit/External/Database.hs
    1 1
     module GHC.Unit.External.Database (
    
    2
    -  -- * Mutable cache for 'ExternalUnitDatabases'
    
    3
    -  ExternalUnitDatabaseCache (..),
    
    4
    -  initExternalUnitDatabaseCache,
    
    5
    -  readExternalUnitDatabases,
    
    6
    -  readExternalUnitDatabase,
    
    7
    -  cacheExternalUnitDatabase,
    
    8
    -  clearExternalUnitDatabaseCache,
    
    9 2
       -- * 'ExternalUnitDatabases'
    
    10 3
       ExternalUnitDatabases,
    
    11 4
       emptyExternalUnitDatabases,
    
    12 5
       insertExternalUnitDatabases,
    
    13 6
       deleteExternalUnitDatabases,
    
    14 7
       lookupExternalUnitDatabases,
    
    15
    -  -- * 'UnitDatabase'
    
    8
    +  -- * 'UnitDatabase' and how to merge them.
    
    16 9
       UnitDatabase (..),
    
    17 10
       mergeDatabases,
    
    18 11
       UnitPrecedenceMap,
    
    ... ... @@ -20,8 +13,6 @@ module GHC.Unit.External.Database (
    20 13
       compareByPreference,
    
    21 14
       -- * Reading packages from disk.
    
    22 15
       UnitDbConfig (..),
    
    23
    -  readOrGetUnitDatabase,
    
    24
    -  readUnitDatabases,
    
    25 16
       readUnitDatabase,
    
    26 17
       getUnitDbRefs,
    
    27 18
       resolveUnitDatabase,
    
    ... ... @@ -48,8 +39,6 @@ import GHC.Utils.Panic
    48 39
     
    
    49 40
     import Control.Monad
    
    50 41
     import Data.Char
    
    51
    -import Data.IORef
    
    52
    -import Data.IORef qualified as IORef
    
    53 42
     import Data.List (sortBy)
    
    54 43
     import Data.Map.Strict (Map)
    
    55 44
     import Data.Map.Strict qualified as Map
    
    ... ... @@ -60,40 +49,6 @@ import System.Directory
    60 49
     import System.Environment (getEnv)
    
    61 50
     import System.FilePath as FilePath
    
    62 51
     
    
    63
    --- ----------------------------------------------------------------------------
    
    64
    --- ExternalUnitDatabaseCache
    
    65
    --- ----------------------------------------------------------------------------
    
    66
    -
    
    67
    --- | Mutable wrapper around 'ExternalUnitDatabases'.
    
    68
    -newtype ExternalUnitDatabaseCache unit = ExternalUnitDatabaseCache
    
    69
    -  { eudc_databases :: IORef (ExternalUnitDatabases unit)
    
    70
    -  }
    
    71
    -
    
    72
    -initExternalUnitDatabaseCache :: IO (ExternalUnitDatabaseCache unit)
    
    73
    -initExternalUnitDatabaseCache =
    
    74
    -  ExternalUnitDatabaseCache <$> IORef.newIORef emptyExternalUnitDatabases
    
    75
    -
    
    76
    -readExternalUnitDatabases :: ExternalUnitDatabaseCache unit -> IO (ExternalUnitDatabases unit)
    
    77
    -readExternalUnitDatabases eudc =
    
    78
    -  IORef.readIORef (eudc_databases eudc)
    
    79
    -
    
    80
    -modifyExternalUnitDatabaseCache :: ExternalUnitDatabaseCache unit -> (ExternalUnitDatabases unit -> ExternalUnitDatabases unit) -> IO ()
    
    81
    -modifyExternalUnitDatabaseCache eudc f =
    
    82
    -  IORef.modifyIORef' (eudc_databases eudc) f
    
    83
    -
    
    84
    -readExternalUnitDatabase :: ExternalUnitDatabaseCache unit -> OsPath -> IO (Maybe (UnitDatabase unit))
    
    85
    -readExternalUnitDatabase eudc path = do
    
    86
    -  dbs <- readExternalUnitDatabases eudc
    
    87
    -  pure $ lookupExternalUnitDatabases path dbs
    
    88
    -
    
    89
    -cacheExternalUnitDatabase :: ExternalUnitDatabaseCache unit -> UnitDatabase unit -> IO ()
    
    90
    -cacheExternalUnitDatabase eudc db =
    
    91
    -  modifyExternalUnitDatabaseCache eudc (insertExternalUnitDatabases db)
    
    92
    -
    
    93
    -clearExternalUnitDatabaseCache :: ExternalUnitDatabaseCache unit -> IO ()
    
    94
    -clearExternalUnitDatabaseCache eudc =
    
    95
    -  modifyExternalUnitDatabaseCache eudc (const emptyExternalUnitDatabases)
    
    96
    -
    
    97 52
     -- ----------------------------------------------------------------------------
    
    98 53
     -- ExternalUnitDatabases
    
    99 54
     -- ----------------------------------------------------------------------------
    
    ... ... @@ -230,12 +185,6 @@ data UnitDbConfig = UnitDbConfig
    230 185
       , unitDbConfigGHCDir :: FilePath
    
    231 186
       }
    
    232 187
     
    
    233
    -readUnitDatabases :: Logger -> ExternalUnitDatabaseCache UnitId -> UnitDbConfig -> IO [UnitDatabase UnitId]
    
    234
    -readUnitDatabases logger db_cache cfg = do
    
    235
    -  conf_refs <- getUnitDbRefs cfg
    
    236
    -  confs     <- liftM catMaybes $ mapM (resolveUnitDatabase cfg) conf_refs
    
    237
    -  mapM (readOrGetUnitDatabase logger db_cache cfg) confs
    
    238
    -
    
    239 188
     getUnitDbRefs :: UnitDbConfig -> IO [PkgDbRef]
    
    240 189
     getUnitDbRefs cfg = do
    
    241 190
       let system_conf_refs = [UserPkgDb, GlobalPkgDb]
    
    ... ... @@ -285,17 +234,6 @@ resolveUnitDatabase cfg UserPkgDb = runMaybeT $ do
    285 234
       if exist then return (OsPath.unsafeEncodeUtf pkgconf) else mzero
    
    286 235
     resolveUnitDatabase _ (PkgDbPath name) = return $ Just name
    
    287 236
     
    
    288
    --- | Get the cached 'UnitDatabase' or read the 'UnitDatabase' at the given location.
    
    289
    -readOrGetUnitDatabase :: Logger -> ExternalUnitDatabaseCache UnitId -> UnitDbConfig -> OsPath -> IO (UnitDatabase UnitId)
    
    290
    -readOrGetUnitDatabase logger db_cache cfg conf_file =
    
    291
    -  readExternalUnitDatabase db_cache conf_file >>= \ case
    
    292
    -    Nothing -> do
    
    293
    -      new_db <- readUnitDatabase logger cfg conf_file
    
    294
    -      cacheExternalUnitDatabase db_cache new_db
    
    295
    -      pure new_db
    
    296
    -    Just db ->
    
    297
    -      pure db
    
    298
    -
    
    299 237
     -- | Read the 'UnitDatabase' at the given location.
    
    300 238
     readUnitDatabase :: Logger -> UnitDbConfig -> OsPath -> IO (UnitDatabase UnitId)
    
    301 239
     readUnitDatabase logger cfg conf_file = do
    

  • compiler/GHC/Unit/External/Index.hs
    ... ... @@ -6,12 +6,16 @@ module GHC.Unit.External.Index (
    6 6
       readUnitIndex,
    
    7 7
       modifyUnitIndexCache,
    
    8 8
       clearUnitIndexCache,
    
    9
    +  cacheExternalUnitDatabase,
    
    10
    +  readExternalUnitDatabases,
    
    11
    +  readExternalUnitDatabase,
    
    9 12
       -- * 'UnitIndex'
    
    10 13
       UnitIndex,
    
    11 14
       emptyUnitIndex,
    
    12 15
       wiringMap,
    
    13 16
       unwiringMap,
    
    14 17
       globalUnits,
    
    18
    +  externalUnitDatabases,
    
    15 19
       setWireMap,
    
    16 20
       wireMapExists,
    
    17 21
       addUnitInfoMap,
    
    ... ... @@ -32,10 +36,14 @@ module GHC.Unit.External.Index (
    32 36
       updateWiredInUnits,
    
    33 37
       updateWiredInUnitsInUnitInfo,
    
    34 38
       updateWiredInUnitIdInModule,
    
    39
    +  -- * Reading external unit databases into the 'UnitIndexCache'
    
    40
    +  readOrGetUnitDatabase,
    
    41
    +  readUnitDatabases,
    
    35 42
     ) where
    
    36 43
     
    
    37 44
     import GHC.Prelude
    
    38 45
     
    
    46
    +import GHC.Data.OsPath
    
    39 47
     import GHC.Data.ShortText qualified as ST
    
    40 48
     import GHC.Types.Unique.Map
    
    41 49
     import GHC.Unit.Database
    
    ... ... @@ -46,17 +54,36 @@ import GHC.Unit.Info
    46 54
     import GHC.Unit.Types
    
    47 55
     import GHC.Utils.Logger
    
    48 56
     
    
    57
    +import Control.Monad (liftM)
    
    49 58
     import Data.Either
    
    50 59
     import Data.IORef (IORef)
    
    51 60
     import Data.IORef qualified as IORef
    
    52 61
     import Data.Map.Strict (Map)
    
    53 62
     import Data.Map.Strict qualified as Map
    
    63
    +import Data.Maybe (catMaybes)
    
    54 64
     
    
    55 65
     -- ----------------------------------------------------------------------------
    
    56 66
     -- UnitIndex
    
    57 67
     -- ----------------------------------------------------------------------------
    
    58 68
     
    
    59 69
     -- | Mutable version of 'UnitIndex'.
    
    70
    +--
    
    71
    +-- The 'UnitIndexCache' ensures that all calls to 'initUnits' will
    
    72
    +-- share the 'UnitInfo' if it is possible.
    
    73
    +--
    
    74
    +-- To share the 'UnitInfo', it needs to be fully-resolved, i.e., its wired-in
    
    75
    +-- dependencies and modules need to be resolved.
    
    76
    +-- Thus, the 'UnitIndexCache' caches both the global 'WireMap' and the 'UnitInfoMap'.
    
    77
    +--
    
    78
    +-- The 'WireMap' is globally valid, as other parts of the compiler rely on the fact
    
    79
    +-- that only one instance of wired-in units is used.
    
    80
    +--
    
    81
    +-- Memory Invariant: The 'UnitIndexCache' is the root object for retaining fully-resolved
    
    82
    +-- 'UnitInfo'. 'UnitState' is expected to reference only 'UnitInfo's from the 'UnitIndexCache'.
    
    83
    +-- There is exactly one fully-resolved 'UnitInfo' alive for each external unit per unit database.
    
    84
    +--
    
    85
    +-- A second instance may or may not be stored in the 'externalUnitDatabases', which represent the
    
    86
    +-- in-memory cache of the on-disk unit databases.
    
    60 87
     newtype UnitIndexCache = UnitIndexCache
    
    61 88
       { uic_index :: IORef UnitIndex
    
    62 89
       }
    
    ... ... @@ -66,21 +93,41 @@ initUnitIndexCache =
    66 93
       UnitIndexCache <$> IORef.newIORef emptyUnitIndex
    
    67 94
     
    
    68 95
     readUnitIndex :: UnitIndexCache -> IO UnitIndex
    
    69
    -readUnitIndex eudc =
    
    70
    -  IORef.readIORef (uic_index eudc)
    
    96
    +readUnitIndex uic =
    
    97
    +  IORef.readIORef (uic_index uic)
    
    71 98
     
    
    72 99
     modifyUnitIndexCache :: UnitIndexCache -> (UnitIndex -> UnitIndex) -> IO ()
    
    73
    -modifyUnitIndexCache eudc f =
    
    74
    -  IORef.modifyIORef' (uic_index eudc) f
    
    100
    +modifyUnitIndexCache uic f =
    
    101
    +  IORef.modifyIORef' (uic_index uic) f
    
    75 102
     
    
    76 103
     clearUnitIndexCache :: UnitIndexCache -> IO ()
    
    77
    -clearUnitIndexCache eudc =
    
    78
    -  modifyUnitIndexCache eudc (const emptyUnitIndex)
    
    104
    +clearUnitIndexCache uic =
    
    105
    +  modifyUnitIndexCache uic (const emptyUnitIndex)
    
    106
    +
    
    107
    +cacheExternalUnitDatabase :: UnitIndexCache -> UnitDatabase UnitId -> IO ()
    
    108
    +cacheExternalUnitDatabase uic db =
    
    109
    +  modifyUnitIndexCache uic
    
    110
    +    (\ ui ->
    
    111
    +      ui
    
    112
    +        { ui_externalUnitDatabases = insertExternalUnitDatabases db (ui_externalUnitDatabases ui)
    
    113
    +        }
    
    114
    +    )
    
    115
    +
    
    116
    +readExternalUnitDatabases :: UnitIndexCache -> IO (ExternalUnitDatabases UnitId)
    
    117
    +readExternalUnitDatabases uic =
    
    118
    +  externalUnitDatabases <$> readUnitIndex uic
    
    119
    +
    
    120
    +readExternalUnitDatabase :: UnitIndexCache -> OsPath -> IO (Maybe (UnitDatabase UnitId))
    
    121
    +readExternalUnitDatabase uic path = do
    
    122
    +  dbs <- readExternalUnitDatabases uic
    
    123
    +  pure $ lookupExternalUnitDatabases path dbs
    
    79 124
     
    
    80 125
     -- | Global index for external units that can be shared across multiple 'HomeUnitEnv's.
    
    81 126
     --
    
    82 127
     -- Allows sharing of the 'WireMap' and 'UnitInfo's that are stored in the 'UnitState'
    
    83 128
     -- of each 'HomeUnitEnv'.
    
    129
    +--
    
    130
    +-- See Note [Sharing 'UnitInfo's across the 'UnitEnv'] for details about memory usage.
    
    84 131
     data UnitIndex = UnitIndex
    
    85 132
       { ui_wireMap :: !WireMap
    
    86 133
         -- ^ A mapping from database unit keys to wired in unit ids.
    
    ... ... @@ -95,10 +142,13 @@ data UnitIndex = UnitIndex
    95 142
       , ui_unitInfoMap :: !GlobalUnitInfoMap
    
    96 143
         -- ^ A global map for all fully-resolved 'UnitInfo's.
    
    97 144
         --
    
    98
    -    -- A 'UnitInfo' is fully-resolved, if its dependencies were updated to reference the
    
    99
    -    -- wired-in packages (e.g., 'wiringMap') and the wired-in packages are updated.
    
    100
    -    -- Further, the 'UnitInfo' is based on the 'ExternalUnitDatabases' results, resolving
    
    101
    -    -- variables such as @${pkgroot}@ in paths.
    
    145
    +    -- See Note [Sharing 'UnitInfo's across the 'UnitEnv'] for more details
    
    146
    +    -- what we use this for and what a fully-resolved 'UnitInfo' is.
    
    147
    +  , ui_externalUnitDatabases :: !(ExternalUnitDatabases UnitId)
    
    148
    +    -- ^ Cache the already processed unit databases in-memory.
    
    149
    +    --
    
    150
    +    -- These 'GenericUnitInfo's have their paths resolved, e.g., no @${pkgroot}@ is
    
    151
    +    -- present any more.
    
    102 152
       }
    
    103 153
     
    
    104 154
     -- | Get the 'WireMap'.
    
    ... ... @@ -115,7 +165,18 @@ wiringMap = ui_wireMap
    115 165
     unwiringMap :: UnitIndex -> UnwireMap
    
    116 166
     unwiringMap = ui_unwireMap
    
    117 167
     
    
    168
    +-- | Access the already processed unit databases.
    
    169
    +externalUnitDatabases :: UnitIndex -> ExternalUnitDatabases UnitId
    
    170
    +externalUnitDatabases = ui_externalUnitDatabases
    
    171
    +
    
    118 172
     -- | Access the global map of fully-resolved 'UnitInfo's.
    
    173
    +--
    
    174
    +-- A 'UnitInfo' is fully-resolved, if its dependencies were updated to reference the
    
    175
    +-- wired-in packages (e.g., 'wiringMap') and the wired-in packages are updated.
    
    176
    +-- Further, the 'UnitInfo' is based on the 'ExternalUnitDatabases' results, resolving
    
    177
    +-- variables such as @${pkgroot}@ in paths.
    
    178
    +--
    
    179
    +-- See Note [Sharing 'UnitInfo's across the 'UnitEnv'] for why this is helpful.
    
    119 180
     globalUnits :: UnitIndex -> GlobalUnitInfoMap
    
    120 181
     globalUnits = ui_unitInfoMap
    
    121 182
     
    
    ... ... @@ -124,6 +185,7 @@ emptyUnitIndex = UnitIndex
    124 185
       { ui_wireMap = emptyWireMap
    
    125 186
       , ui_unwireMap = emptyUnwireMap
    
    126 187
       , ui_unitInfoMap = emptyGlobalUnitInfoMap
    
    188
    +  , ui_externalUnitDatabases = emptyExternalUnitDatabases
    
    127 189
       }
    
    128 190
     
    
    129 191
     -- | Set the 'WireMap' of 'UnitIndex'.
    
    ... ... @@ -167,7 +229,8 @@ type UnitAbiHash = ST.ShortText
    167 229
     --
    
    168 230
     -- However, a user can choose a conflicting 'UnitId', causing a conflict after all.
    
    169 231
     -- We use the 'UnitAbiHash' for disambiguation. If both 'UnitId' and 'UnitAbiHash' are
    
    170
    --- identical in separate unit databases, we can assume they are the same unit.
    
    232
    +-- identical in separate unit databases, we can assume they are the same unit, according
    
    233
    +-- to the documentation of GHC.
    
    171 234
     newtype GlobalUnitInfoMap = GlobalUnitInfoMap (UniqMap UnitId (Map UnitAbiHash UnitInfo))
    
    172 235
     
    
    173 236
     -- | Lookup the 'UnitInfo' in the 'GlobalUnitInfoMap'.
    
    ... ... @@ -310,3 +373,23 @@ upd_wired_in :: WireMap -> UnitId -> UnitId
    310 373
     upd_wired_in wiredInMap key
    
    311 374
         | Just key' <- lookupWireMap key wiredInMap = key'
    
    312 375
         | otherwise = key
    
    376
    +
    
    377
    +-- -----------------------------------------------------------------------------
    
    378
    +-- Reading the unit database(s) into the 'UnitIndexCache'
    
    379
    +
    
    380
    +readUnitDatabases :: Logger -> UnitIndexCache -> UnitDbConfig -> IO [UnitDatabase UnitId]
    
    381
    +readUnitDatabases logger db_cache cfg = do
    
    382
    +  conf_refs <- getUnitDbRefs cfg
    
    383
    +  confs     <- liftM catMaybes $ mapM (resolveUnitDatabase cfg) conf_refs
    
    384
    +  mapM (readOrGetUnitDatabase logger db_cache cfg) confs
    
    385
    +
    
    386
    +-- | Get the cached 'UnitDatabase' or read the 'UnitDatabase' at the given location.
    
    387
    +readOrGetUnitDatabase :: Logger -> UnitIndexCache -> UnitDbConfig -> OsPath -> IO (UnitDatabase UnitId)
    
    388
    +readOrGetUnitDatabase logger db_cache cfg conf_file =
    
    389
    +  readExternalUnitDatabase db_cache conf_file >>= \ case
    
    390
    +    Nothing -> do
    
    391
    +      new_db <- readUnitDatabase logger cfg conf_file
    
    392
    +      cacheExternalUnitDatabase db_cache new_db
    
    393
    +      pure new_db
    
    394
    +    Just db ->
    
    395
    +      pure db

  • compiler/GHC/Unit/State.hs
    ... ... @@ -6,7 +6,6 @@ module GHC.Unit.State (
    6 6
             module GHC.Unit.Info,
    
    7 7
             -- * Reading the package config, and processing cmdline args
    
    8 8
             UnitState(..),
    
    9
    -        TrustOverlay(..),
    
    10 9
             UnitDatabase (..),
    
    11 10
             UnitErr (..),
    
    12 11
             emptyUnitState,
    
    ... ... @@ -16,6 +15,13 @@ module GHC.Unit.State (
    16 15
             getUnitDbRefs,
    
    17 16
             resolveUnitDatabase,
    
    18 17
             listUnitInfo,
    
    18
    +        -- * Overlays over the unit set
    
    19
    +        TrustOverlay,
    
    20
    +        IsTrusted(..),
    
    21
    +        lookupTrustOverlay,
    
    22
    +        distrustUnits,
    
    23
    +        trustUnits,
    
    24
    +        emptyTrustOverlay,
    
    19 25
             -- * Querying the package config
    
    20 26
             lookupUnit,
    
    21 27
             lookupUnit',
    
    ... ... @@ -252,12 +258,19 @@ data IsTrusted
    252 258
       | Distrusted
    
    253 259
       deriving ( Eq, Ord )
    
    254 260
     
    
    261
    +-- | The 'TrustOverlay' stores user overwrites of the on-disk 'unitIsTrusted' status.
    
    262
    +--
    
    263
    +-- The user can overwrite this value via flags such as @-distrust-all-packages@.
    
    264
    +-- We do not modify the 'UnitInfo' directory, but rather store this user selection
    
    265
    +-- in the 'TrustOverlay'.
    
    266
    +--
    
    267
    +-- This allows us to share the 'UnitInfo' completely and saves us memory.
    
    255 268
     newtype TrustOverlay = TrustOverlay
    
    256 269
       { trustOverlay :: UniqMap UnitId IsTrusted
    
    257 270
       }
    
    258 271
     
    
    259 272
     lookupTrustOverlay :: TrustOverlay -> UnitId -> Maybe IsTrusted
    
    260
    -lookupTrustOverlay (TrustOverlay to) = lookupUniqMap to
    
    273
    +lookupTrustOverlay to = lookupUniqMap (trustOverlay to)
    
    261 274
     
    
    262 275
     distrustUnits :: [UnitId] -> TrustOverlay -> TrustOverlay
    
    263 276
     distrustUnits elements (TrustOverlay to) = TrustOverlay $ foldl' (\ acc uid -> addToUniqMap acc uid Distrusted) to elements
    
    ... ... @@ -268,6 +281,52 @@ trustUnits elements (TrustOverlay to) = TrustOverlay $ foldl' (\ acc uid -> addT
    268 281
     emptyTrustOverlay :: TrustOverlay
    
    269 282
     emptyTrustOverlay = TrustOverlay emptyUniqMap
    
    270 283
     
    
    284
    +{-
    
    285
    +Note [Sharing 'UnitInfo's across the 'UnitEnv']
    
    286
    +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    
    287
    +The 'UnitState' and 'UnitIndex' are closely related.
    
    288
    +
    
    289
    +The 'UnitState' stores all information about the external units referenced by
    
    290
    +a single HomeUnitEnv.
    
    291
    +This includes in particular the 'unitInfoMap', an in-memory representation of
    
    292
    +the unit databases that a 'HomeUnitEnv' depends on.
    
    293
    +Multiple home units can depend on the same unit database, leading to a linear
    
    294
    +increase of 'UnitInfo's per 'HomeUnitEnv'. (It used to be quadratic even, due
    
    295
    +to accidentally retaining old 'UnitInfo's.)
    
    296
    +Thus, we want to share the 'UnitInfo' across multiple 'HomeUnitEnv's.
    
    297
    +This where the 'UnitIndex' is needed.
    
    298
    +
    
    299
    +The 'UnitIndex' stores all fully-resolved 'UnitInfo's that can be referenced
    
    300
    +by the 'UnitState'.'unitInfoMap'.
    
    301
    +We consider a 'UnitInfo' as fully-resolved, if its dependencies were updated to reference the
    
    302
    +wired-in units (e.g., 'wiringMap') and the wired-in units are updated as well.
    
    303
    +See Note [Wired-in units] for more details on wired-in units.
    
    304
    +Further, the 'UnitInfo' is based on the 'ExternalUnitDatabases' results, resolving
    
    305
    +variables such as @${pkgroot}@ in paths.
    
    306
    +
    
    307
    +As such, we can consider the 'UnitIndex' to be global data that is referenced by
    
    308
    +the 'UnitState' for better sharing of 'UnitInfo's.
    
    309
    +
    
    310
    +In fact, using the 'UnitIndex', we can impose a hard upper bound on the number
    
    311
    +of live 'UnitInfo's in a GHC session:
    
    312
    +
    
    313
    +> For each on-disk 'GenericUnitInfo', there are at most two objects alive.
    
    314
    +
    
    315
    +One instance is stored in 'ExternalUnitDatabases' where variables are resolved,
    
    316
    +but the wired-in units haven't been resolved.
    
    317
    +
    
    318
    +The second instance is the fully-resolved 'UnitInfo' stored in the 'UnitIndex'.
    
    319
    +-}
    
    320
    +
    
    321
    +-- | The 'UnitState' contains a plethora of information local to a single 'HomeUnitEnv'.
    
    322
    +--
    
    323
    +-- It stores module visibilities, unit trust for @SafeHaskell@, available units for error messages,
    
    324
    +-- explicit unit dependencies and knows how to instantiate backpack signature and holes
    
    325
    +-- on demand.
    
    326
    +--
    
    327
    +-- A 'HomeUnitEnv' should rarely/never have to look into the 'UnitIndex', all external
    
    328
    +-- unit related information is stored in the 'UnitState'.
    
    329
    +--
    
    271 330
     data UnitState = UnitState {
    
    272 331
       -- | A mapping of 'Unit' to 'UnitInfo'.  This list is adjusted
    
    273 332
       -- so that only valid units are here.  'UnitInfo' reflects
    
    ... ... @@ -276,6 +335,7 @@ data UnitState = UnitState {
    276 335
       -- may have the 'exposed' flag be 'False'.)
    
    277 336
       --
    
    278 337
       -- All values are shared with 'UnitIndex'.'globalUnits'.
    
    338
    +  -- See Note [Sharing 'UnitInfo's across the 'UnitEnv'] for details.
    
    279 339
       unitInfoMap :: UnitInfoMap,
    
    280 340
     
    
    281 341
       -- | Set of units that we trust.
    
    ... ... @@ -283,7 +343,10 @@ data UnitState = UnitState {
    283 343
       -- Local overlay of 'UnitInfo'.
    
    284 344
       -- This avoids modifying the 'UnitInfo' directly, potentially saving
    
    285 345
       -- a lot of duplication.
    
    286
    -  trustedUnits :: TrustOverlay,
    
    346
    +  --
    
    347
    +  -- We keep this in WHNF as it is relatively cheap but could easily retain
    
    348
    +  -- references to bigger structures.
    
    349
    +  trustedUnits :: !TrustOverlay,
    
    287 350
     
    
    288 351
       -- | A mapping of 'PackageName' to 'UnitId'. If several units have the same
    
    289 352
       -- package name (e.g. different instantiations), then we return one of them...
    
    ... ... @@ -333,8 +396,6 @@ emptyUnitState = UnitState {
    333 396
         unitInfoMap    = emptyUniqMap,
    
    334 397
         trustedUnits   = emptyTrustOverlay,
    
    335 398
         packageNameMap = emptyUFM,
    
    336
    -    -- wireMap        = emptyUniqMap,
    
    337
    -    -- unwireMap      = emptyUniqMap,
    
    338 399
         preloadUnits   = [],
    
    339 400
         explicitUnits  = [],
    
    340 401
         homeUnitDepends = Set.empty,
    
    ... ... @@ -439,14 +500,16 @@ isUnitInfoTrusted ue u =
    439 500
     -- 'initUnits' can be called again subsequently after updating the
    
    440 501
     -- 'packageFlags' and 'packageDBFlags' fields of the 'DynFlags', and it will
    
    441 502
     -- update the 'unitState' in 'DynFlags'.
    
    442
    -initUnits :: Logger -> DynFlags -> UnitIndexCache -> ExternalUnitDatabaseCache UnitId -> Set.Set UnitId -> IO (UnitState, HomeUnit, Maybe PlatformConstants)
    
    443
    -initUnits logger dflags unit_index cached_dbs home_units = do
    
    503
    +--
    
    504
    +-- Also, see Note [Sharing 'UnitInfo's across the 'UnitEnv'] for implementation details.
    
    505
    +initUnits :: Logger -> DynFlags -> UnitIndexCache -> Set.Set UnitId -> IO (UnitState, HomeUnit, Maybe PlatformConstants)
    
    506
    +initUnits logger dflags unit_index home_units = do
    
    444 507
     
    
    445 508
       let forceUnitInfoMap state = unitInfoMap state `seq` ()
    
    446 509
     
    
    447 510
       unit_state <- withTiming logger (text "initializing unit database")
    
    448 511
                        forceUnitInfoMap
    
    449
    -                 $ mkUnitState logger unit_index cached_dbs (initUnitConfig dflags  home_units)
    
    512
    +                 $ mkUnitState logger unit_index (initUnitConfig dflags  home_units)
    
    450 513
     
    
    451 514
       putDumpFileMaybe logger Opt_D_dump_mod_map "Module Map"
    
    452 515
         FormatText (updSDocContext (\ctx -> ctx {sdocLineLength = 200})
    
    ... ... @@ -638,10 +701,9 @@ reportCycles logger sccs = when (logVerbAtLeast logger 2) $ mapM_ report sccs
    638 701
     mkUnitState
    
    639 702
         :: Logger
    
    640 703
         -> UnitIndexCache
    
    641
    -    -> ExternalUnitDatabaseCache UnitId
    
    642 704
         -> UnitConfig
    
    643 705
         -> IO UnitState
    
    644
    -mkUnitState logger unit_index db_cache cfg = do
    
    706
    +mkUnitState logger unit_index cfg = do
    
    645 707
     {-
    
    646 708
        Plan.
    
    647 709
     
    
    ... ... @@ -695,7 +757,7 @@ mkUnitState logger unit_index db_cache cfg = do
    695 757
               we build a mapping saying what every in scope module name points to.
    
    696 758
     -}
    
    697 759
     
    
    698
    -  dbs <- readUnitDatabases logger db_cache (initUnitDbConfig cfg)
    
    760
    +  dbs <- readUnitDatabases logger unit_index (initUnitDbConfig cfg)
    
    699 761
     
    
    700 762
       -- distrust all units if the flag is set
    
    701 763
       let distrustUnitsOfDb overlay db = foldl' (\ acc ui -> distrustUnits [unitId ui] acc) overlay (unitDatabaseUnits db)
    
    ... ... @@ -713,7 +775,7 @@ mkUnitState logger unit_index db_cache cfg = do
    713 775
       debugTraceMsg logger 2 $
    
    714 776
           text "package flags" <+> ppr other_flags
    
    715 777
     
    
    716
    -  let !home_unit_deps = selectHomeUnits (unitConfigHomeUnits cfg) hpt_flags
    
    778
    +  let home_unit_deps = selectHomeUnits (unitConfigHomeUnits cfg) hpt_flags
    
    717 779
     
    
    718 780
       -- Merge databases together, without checking validity
    
    719 781
       (pkg_map1, prec_map) <- mergeDatabases logger dbs
    
    ... ... @@ -727,7 +789,7 @@ mkUnitState logger unit_index db_cache cfg = do
    727 789
     
    
    728 790
       -- Compute trust flags (these flags apply regardless of whether
    
    729 791
       -- or not packages are visible or not)
    
    730
    -  !trustUnitsOverlay <- mayThrowUnitErr
    
    792
    +  trustUnitsOverlay <- mayThrowUnitErr
    
    731 793
                 $ foldM (applyTrustFlag prec_map unusable (nonDetEltsUniqMap pkg_map2))
    
    732 794
                      distrustedUnitsOverlay (reverse (unitConfigFlagsTrusted cfg))
    
    733 795
       let pkgs1 = nonDetEltsUniqMap pkg_map2
    
    ... ... @@ -769,12 +831,12 @@ mkUnitState logger unit_index db_cache cfg = do
    769 831
                                 -- Note: we NEVER expose indefinite packages by
    
    770 832
                                 -- default, because it's almost assuredly not
    
    771 833
                                 -- what you want (no mix-in linking has occurred).
    
    772
    -                            let !x = fsPackageName p in if unitIsExposed p && unitIsDefinite (mkUnit p) && mostPreferable p
    
    834
    +                            if unitIsExposed p && unitIsDefinite (mkUnit p) && mostPreferable p
    
    773 835
                                    then addToUniqMap vm (mkUnit p)
    
    774 836
                                                    UnitVisibility {
    
    775 837
                                                      uv_expose_all = True,
    
    776 838
                                                      uv_renamings = [],
    
    777
    -                                                 uv_package_name = First (Just x),
    
    839
    +                                                 uv_package_name = First (Just $ fsPackageName p),
    
    778 840
                                                      uv_requirements = emptyUniqMap,
    
    779 841
                                                      uv_explicit = Nothing
    
    780 842
                                                    }
    

  • ghc/GHCi/UI.hs
    ... ... @@ -857,7 +857,7 @@ installInteractiveHomeUnits dflags = do
    857 857
           env <- GHC.getSession
    
    858 858
           let unit_index = hscUIC env
    
    859 859
           (unit_state,home_unit,_mconstants) <-
    
    860
    -        liftIO $ initUnits logger dflags unit_index (hscEUDC env) all_home_units
    
    860
    +        liftIO $ initUnits logger dflags unit_index all_home_units
    
    861 861
           hpt <- liftIO emptyHomePackageTable
    
    862 862
           pure (HUG.mkHomeUnitEnv unit_state dflags hpt (Just home_unit))
    
    863 863
     
    

  • testsuite/tests/driver/TUnitInfo/generic-unit-info-space-mhu.stdout
    1
    +### Heap Census
    
    2
    +There are exactly two GenericUnitInfo closures alive per on-disk package 

  • testsuite/tests/driver/TUnitInfo/generic-unit-info-space.hs
    ... ... @@ -87,7 +87,7 @@ initGhcM numOfPkgs xs = do
    87 87
           -- If this number changes without a good reason, DO NOT ACCEPT THE CHANGES, you have introduced a space leak.
    
    88 88
           -- We say less than the expected size is accepted, because in the multiple-home-units case, we don't force the second
    
    89 89
           -- UnitInfo closure enough after initial processing.
    
    90
    -      when (num <= expectedSizeInBytes) $ do
    
    90
    +      when (num > expectedSizeInBytes) $ do
    
    91 91
             putStrLn "Space leak detected by generic-unit-info-space test:"
    
    92 92
             putStrLn $ (show (num `div` genericUnitInfoSizeInBytes)) ++ " live GenericUnitInfo when <= (" ++ show expectedNumberOfUnitInfos ++ ") are expected"
    
    93 93
             readFile hpFile >>= putStrLn