| ... |
... |
@@ -112,8 +112,11 @@ import Control.Monad.Trans.Reader |
|
112
|
112
|
import qualified Data.Map.Strict as M
|
|
113
|
113
|
import Control.Monad.Trans.Class
|
|
114
|
114
|
import System.IO.Unsafe (unsafeInterleaveIO)
|
|
115
|
|
-import Data.IORef
|
|
116
|
115
|
import qualified Data.List.NonEmpty as NE
|
|
|
116
|
+import Control.Concurrent
|
|
|
117
|
+import Control.Concurrent.STM.TQueue
|
|
|
118
|
+import Control.Concurrent.STM
|
|
|
119
|
+import Control.Applicative
|
|
117
|
120
|
|
|
118
|
121
|
{-
|
|
119
|
122
|
Note [The ModuleGraph]
|
| ... |
... |
@@ -256,8 +259,8 @@ downsweep :: HscEnv |
|
256
|
259
|
-- which case there can be repeats
|
|
257
|
260
|
downsweep hsc_env diag_wrapper msg old_summaries maybe_base_graph excl_mods allow_dup_roots = do
|
|
258
|
261
|
n_jobs <- mkWorkerLimit (hsc_dflags hsc_env)
|
|
259
|
|
- summ_cache <- newIORef (mkModSummaryCache (zip old_summaries (repeat SummOld)))
|
|
260
|
|
- imps_cache <- newIORef Map.empty
|
|
|
262
|
+ summ_cache <- newMVar (mkModSummaryCache (zip old_summaries (repeat SummOld)))
|
|
|
263
|
+ imps_cache <- newMVar Map.empty
|
|
261
|
264
|
withMakeEnv n_jobs hsc_env diag_wrapper msg $ \make_env -> do
|
|
262
|
265
|
(root_errs, root_summaries) <- rootSummariesParallel n_jobs make_env (hsc_targets hsc_env)
|
|
263
|
266
|
(getRootSummary excl_mods summ_cache imps_cache)
|
| ... |
... |
@@ -341,8 +344,8 @@ downsweepThunk :: HscEnv -> ModSummary -> IO ModuleGraph |
|
341
|
344
|
downsweepThunk hsc_env mod_summary = unsafeInterleaveIO $ do
|
|
342
|
345
|
debugTraceMsg (hsc_logger hsc_env) 3 $ text "Computing Module Graph thunk..."
|
|
343
|
346
|
njobs <- mkWorkerLimit (hsc_dflags hsc_env)
|
|
344
|
|
- summs <- newIORef (mkModSummaryCache [(mod_summary,SummOld)])
|
|
345
|
|
- imps <- newIORef mempty
|
|
|
347
|
+ summs <- newMVar (mkModSummaryCache [(mod_summary,SummOld)])
|
|
|
348
|
+ imps <- newMVar mempty
|
|
346
|
349
|
withMakeEnv njobs hsc_env mkUnknownDiagnostic Nothing $ \make_env -> do
|
|
347
|
350
|
let env = DownsweepEnv
|
|
348
|
351
|
{ ds_hsc_env = hsc_env
|
| ... |
... |
@@ -386,8 +389,8 @@ downsweepInteractiveImports hsc_env ic = unsafeInterleaveIO $ do |
|
386
|
389
|
let cached_nodes = Map.fromList [ (mkNodeKey n, NSuccess n) | n <- mg_mss (hsc_mod_graph hsc_env) ]
|
|
387
|
390
|
|
|
388
|
391
|
n_jobs <- mkWorkerLimit (hsc_dflags hsc_env)
|
|
389
|
|
- summ_cache <- newIORef mempty
|
|
390
|
|
- imps_cache <- newIORef mempty
|
|
|
392
|
+ summ_cache <- newMVar mempty
|
|
|
393
|
+ imps_cache <- newMVar mempty
|
|
391
|
394
|
withMakeEnv n_jobs hsc_env mkUnknownDiagnostic Nothing $ \make_env -> do
|
|
392
|
395
|
let env = DownsweepEnv
|
|
393
|
396
|
{ ds_hsc_env = hsc_env
|
| ... |
... |
@@ -431,8 +434,8 @@ downsweepInstalledModules hsc_env mods = do |
|
431
|
434
|
|
|
432
|
435
|
njobs <- mkWorkerLimit (hsc_dflags hsc_env)
|
|
433
|
436
|
nodes <- mapM process installed_mods
|
|
434
|
|
- summs <- newIORef mempty
|
|
435
|
|
- imps <- newIORef mempty
|
|
|
437
|
+ summs <- newMVar mempty
|
|
|
438
|
+ imps <- newMVar mempty
|
|
436
|
439
|
withMakeEnv njobs hsc_env mkUnknownDiagnostic Nothing $ \make_env -> do
|
|
437
|
440
|
let env = DownsweepEnv
|
|
438
|
441
|
{ ds_hsc_env = hsc_env
|
| ... |
... |
@@ -459,8 +462,8 @@ downsweepInstalledModules hsc_env mods = do |
|
459
|
462
|
-- * Orchestrator: downsweepFromRootNodes
|
|
460
|
463
|
-----------------------------------------------------------------------------
|
|
461
|
464
|
|
|
462
|
|
-type ModSummaryCache = IORef ModSummaryCacheMap
|
|
463
|
|
-type ImportsCache = IORef ImportsCacheMap
|
|
|
465
|
+type ModSummaryCache = MVar ModSummaryCacheMap
|
|
|
466
|
+type ImportsCache = MVar ImportsCacheMap
|
|
464
|
467
|
|
|
465
|
468
|
-- | A cache from file paths to the already summarised modules. The same file
|
|
466
|
469
|
-- can be used in multiple units so the map is actually also keyed by which
|
| ... |
... |
@@ -514,7 +517,7 @@ downsweepFromRootNodes maybe_base_graph allow_dup_roots root_nodes root_uids = |
|
514
|
517
|
all_deps <- loopUnits module_deps (hscActiveUnitId ds_hsc_env) root_uids
|
|
515
|
518
|
deps' <- loopInstantiations all_deps (getHomeUnitInstantiations ds_hsc_env)
|
|
516
|
519
|
return deps'
|
|
517
|
|
- f_cache <- readIORef ds_summaries_cache
|
|
|
520
|
+ f_cache <- readMVar ds_summaries_cache
|
|
518
|
521
|
let downsweep_errs = lefts (M.elems f_cache)
|
|
519
|
522
|
downsweep_nodes = [ s | NSuccess s <- M.elems deps' ]
|
|
520
|
523
|
|
| ... |
... |
@@ -573,8 +576,8 @@ addModSummaryCache ms pr fe = upd_fe fe |
|
573
|
576
|
|
|
574
|
577
|
modifySummCache :: ModSummaryCache -> (ModSummaryCacheMap -> ModSummaryCacheMap) -> IO ()
|
|
575
|
578
|
modifyImpsCache :: ImportsCache -> (ImportsCacheMap -> ImportsCacheMap) -> IO ()
|
|
576
|
|
-modifySummCache r f = atomicModifyIORef' r (\c -> (f c, ()))
|
|
577
|
|
-modifyImpsCache r f = atomicModifyIORef' r (\c -> (f c, ()))
|
|
|
579
|
+modifySummCache r f = modifyMVar r (\c -> let !r = f c in pure (r, ()))
|
|
|
580
|
+modifyImpsCache r f = modifyMVar r (\c -> let !r = f c in pure (r, ()))
|
|
578
|
581
|
|
|
579
|
582
|
-- | A cache from a module import (in given home unit context, with a package
|
|
580
|
583
|
-- qualifier, and the imported module name (with or without SOURCE)) to the
|
| ... |
... |
@@ -597,7 +600,7 @@ loopModuleNodeInfos :: M.Map NodeKey (NodeRes ModuleGraphNode) -> [ModuleNodeInf |
|
597
|
600
|
loopUnits :: M.Map NodeKey (NodeRes ModuleGraphNode) -> UnitId -> [UnitId] -> DownsweepM (M.Map NodeKey (NodeRes ModuleGraphNode))
|
|
598
|
601
|
loopInstantiations :: M.Map NodeKey (NodeRes ModuleGraphNode) -> [(UnitId, InstantiatedUnit)] -> DownsweepM (M.Map NodeKey (NodeRes ModuleGraphNode))
|
|
599
|
602
|
loopFromInteractive :: M.Map NodeKey (NodeRes ModuleGraphNode) -> Module -> [InteractiveImport] -> DownsweepM (M.Map NodeKey (NodeRes ModuleGraphNode))
|
|
600
|
|
-loopDownsweepNodes base_map nodes = dfsBuild (Just base_map) nodes dsNodeInfoKey dsNodeExpand
|
|
|
603
|
+loopDownsweepNodes base_map nodes = parDfsBuild (Just base_map) nodes dsNodeInfoKey dsNodeExpand
|
|
601
|
604
|
loopModuleNodeInfos base_map = loopDownsweepNodes base_map . map DSMod
|
|
602
|
605
|
loopUnits base_map homud = loopDownsweepNodes base_map . map (DSUnit homud)
|
|
603
|
606
|
loopInstantiations base_map = loopDownsweepNodes base_map . map (uncurry DSInst)
|
| ... |
... |
@@ -696,7 +699,7 @@ expandModuleSummary ms = do -- Didn't work out what the imports mean yet, now do |
|
696
|
699
|
FoundHomeWithError (_uid, _e) -> return
|
|
697
|
700
|
( Nothing, [] )
|
|
698
|
701
|
-- the error @e@ is already stored in the summarisation cache,
|
|
699
|
|
- -- (the IORef in DownsweepM) and will get reported at the end.
|
|
|
702
|
+ -- (the MVar in DownsweepM) and will get reported at the end.
|
|
700
|
703
|
FoundHome s -> return
|
|
701
|
704
|
-- MP: This assumes that we can only instantiate non home units, which is probably fair enough for now.
|
|
702
|
705
|
( Just $ mkModuleEdge lvl (NodeKey_Module (mnKey s))
|
| ... |
... |
@@ -1328,7 +1331,7 @@ summariseFile |
|
1328
|
1331
|
-> IO (Either DriverMessages ModSummary)
|
|
1329
|
1332
|
|
|
1330
|
1333
|
summariseFile hsc_env' home_unit summ_cache_ref src_fn mb_phase maybe_buf
|
|
1331
|
|
- = do file_summ_cache <- readIORef summ_cache_ref
|
|
|
1334
|
+ = do file_summ_cache <- readMVar summ_cache_ref
|
|
1332
|
1335
|
case M.lookup (homeUnitId home_unit, src_fn_os) file_summ_cache of
|
|
1333
|
1336
|
Just (Right (chd_summary, SummFresh)) ->
|
|
1334
|
1337
|
-- Fresh: use it straight away
|
| ... |
... |
@@ -1508,7 +1511,7 @@ summariseModuleDispatch k hsc_env' imps_cache_ref home_unit imp excl_mods |
|
1508
|
1511
|
|
|
1509
|
1512
|
find_it :: IO SummariseResult
|
|
1510
|
1513
|
find_it = do
|
|
1511
|
|
- imps_cache <- readIORef imps_cache_ref
|
|
|
1514
|
+ imps_cache <- readMVar imps_cache_ref
|
|
1512
|
1515
|
case M.lookup cache_key imps_cache of
|
|
1513
|
1516
|
Just result -> return result
|
|
1514
|
1517
|
Nothing -> do
|
| ... |
... |
@@ -1550,7 +1553,7 @@ summariseModuleWithSource home_unit summ_cache_ref is_boot maybe_buf hsc_env loc |
|
1550
|
1553
|
-- Adjust location to point to the hs-boot source file,
|
|
1551
|
1554
|
-- hi file, object file, when is_boot says so
|
|
1552
|
1555
|
let src_fn = expectJust (ml_hs_file location)
|
|
1553
|
|
- summ_cache <- readIORef summ_cache_ref
|
|
|
1556
|
+ summ_cache <- readMVar summ_cache_ref
|
|
1554
|
1557
|
|
|
1555
|
1558
|
-- Reject the cache result if the module name doesn't match the inferred
|
|
1556
|
1559
|
-- module name based on the file name.
|
| ... |
... |
@@ -1741,7 +1744,7 @@ data NodeRes v |
|
1741
|
1744
|
-- abort.
|
|
1742
|
1745
|
| NSkip
|
|
1743
|
1746
|
|
|
1744
|
|
--- | In a depth-first order, and starting from the given roots, traverse a
|
|
|
1747
|
+-- | In a parallel depth-first order, and starting from the given roots, traverse a
|
|
1745
|
1748
|
-- graph by iteratively expanding a node into a payload and a list of children
|
|
1746
|
1749
|
-- nodes to visit next.
|
|
1747
|
1750
|
--
|
| ... |
... |
@@ -1759,13 +1762,10 @@ data NodeRes v |
|
1759
|
1762
|
-- The @expand@ function returns an 'NResult'. See the 'NResult' documentation
|
|
1760
|
1763
|
-- for more information about each result type.
|
|
1761
|
1764
|
--
|
|
1762
|
|
--- Error handling and exiting early can be achieved by selecting a @Monad m@
|
|
1763
|
|
--- accordingly, such as @Control.Monad.Except.Except@
|
|
1764
|
|
---
|
|
1765
|
1765
|
-- Example usage: @n@ is instanced to @DownsweepNode@, @k@ is @NodeKey@, and @v@ is @ModuleNodeEdge@.
|
|
1766
|
1766
|
--
|
|
1767
|
1767
|
-- See also Note [Downsweep Control Flow and Caching]
|
|
1768
|
|
-dfsBuild :: (Ord k, Monad m)
|
|
|
1768
|
+parDfsBuild :: Ord k
|
|
1769
|
1769
|
=> Maybe (Map.Map k (NodeRes v))
|
|
1770
|
1770
|
-- ^ Base map, existing results. We won't re-expand any of the nodes
|
|
1771
|
1771
|
-- already present in this map.
|
| ... |
... |
@@ -1773,29 +1773,85 @@ dfsBuild :: (Ord k, Monad m) |
|
1773
|
1773
|
-- ^ The root nodes from where to start traversal
|
|
1774
|
1774
|
-> (n -> k)
|
|
1775
|
1775
|
-- ^ Compute the key which uniquely identifies this node
|
|
1776
|
|
- -> (n -> m (NodeRes (v,[n])))
|
|
|
1776
|
+ -> (n -> DownsweepM (NodeRes (v,[n])))
|
|
1777
|
1777
|
-- ^ Expand this node into its payload result and into the list of
|
|
1778
|
1778
|
-- children nodes to visit next.
|
|
1779
|
|
- -> m (Map.Map k (NodeRes v))
|
|
|
1779
|
+ -> DownsweepM (Map.Map k (NodeRes v))
|
|
1780
|
1780
|
-- ^ The result accumulates the payload of expanding the root nodes
|
|
1781
|
1781
|
-- and all nodes transitively reachable from those roots.
|
|
1782
|
|
-dfsBuild base_map roots key expand = go roots (fromMaybe Map.empty base_map)
|
|
|
1782
|
+parDfsBuild base_map roots key expand = ReaderT $ \ds_env -> do
|
|
|
1783
|
+ exc_var <- newTVarIO Nothing -- signal this var when there's an exception
|
|
|
1784
|
+ visited_var <- newTVarIO (fromMaybe Map.empty base_map)
|
|
|
1785
|
+ pending <- newTVarIO Set.empty
|
|
|
1786
|
+ worklist <- newTQueueIO
|
|
|
1787
|
+ coord_tid <- forkIO $
|
|
|
1788
|
+ coordinator ds_env exc_var visited_var worklist pending
|
|
|
1789
|
+ `MC.catch` \(e::MC.SomeException) -> atomically (modifyTVar' exc_var (<|> Just e))
|
|
|
1790
|
+ mapM_ (atomically . writeTQueue worklist) roots
|
|
|
1791
|
+ mb_exc <- atomically $ do
|
|
|
1792
|
+ readTVar exc_var >>= \case
|
|
|
1793
|
+ Just e ->
|
|
|
1794
|
+ -- exit if there's an exception
|
|
|
1795
|
+ return (Just e)
|
|
|
1796
|
+ Nothing -> do
|
|
|
1797
|
+ -- otherwise, exit only when pending and worklist are both empty,
|
|
|
1798
|
+ -- *atomically*; else, `retry`.
|
|
|
1799
|
+ empty_worklist <- isEmptyTQueue worklist
|
|
|
1800
|
+ empty_pending <- Set.null <$> readTVar pending
|
|
|
1801
|
+ unless (empty_worklist && empty_pending) retry
|
|
|
1802
|
+ return Nothing
|
|
|
1803
|
+ killThread coord_tid
|
|
|
1804
|
+ case mb_exc of
|
|
|
1805
|
+ Just e -> throwIO e
|
|
|
1806
|
+ Nothing -> readTVarIO visited_var
|
|
1783
|
1807
|
where
|
|
1784
|
|
- go [] visited = pure visited
|
|
1785
|
|
- go (s:ss) visited
|
|
1786
|
|
- | k `Map.member` visited
|
|
1787
|
|
- = go ss visited
|
|
1788
|
|
- | otherwise
|
|
1789
|
|
- = do r <- expand s
|
|
1790
|
|
- case r of
|
|
1791
|
|
- NSkip ->
|
|
1792
|
|
- go ss
|
|
1793
|
|
- (Map.insert k NSkip visited) -- Skip!
|
|
1794
|
|
- NSuccess (v,ns) ->
|
|
1795
|
|
- go (ns ++ ss)
|
|
1796
|
|
- (Map.insert k (NSuccess v) visited)
|
|
1797
|
|
- where
|
|
1798
|
|
- k = key s
|
|
|
1808
|
+ coordinator ds_env exc_var visvar worklist pendvar = forever $ do
|
|
|
1809
|
+ mb_node_to_expand <- atomically $ do
|
|
|
1810
|
+ node <- readTQueue worklist
|
|
|
1811
|
+ let k = key node
|
|
|
1812
|
+
|
|
|
1813
|
+ visited <- readTVar visvar
|
|
|
1814
|
+ pending <- readTVar pendvar
|
|
|
1815
|
+
|
|
|
1816
|
+ if (k `Set.member` pending || k `Map.member` visited)
|
|
|
1817
|
+ then return Nothing
|
|
|
1818
|
+ else do
|
|
|
1819
|
+ -- must add to pending in the same transaction as worklist dequeue,
|
|
|
1820
|
+ -- otherwise the main thread may find both the worklist and pending
|
|
|
1821
|
+ -- lists empty and exit prematurely.
|
|
|
1822
|
+ modifyTVar' pendvar (Set.insert k)
|
|
|
1823
|
+ return (Just (k, node))
|
|
|
1824
|
+
|
|
|
1825
|
+ case mb_node_to_expand of
|
|
|
1826
|
+ Nothing -> return ()
|
|
|
1827
|
+ Just (k, node) -> do
|
|
|
1828
|
+ withLocalTmpFSMake (ds_make_env ds_env) $ \make_env ->
|
|
|
1829
|
+ forkIO $
|
|
|
1830
|
+ worker ds_env{ds_make_env = make_env} exc_var visvar
|
|
|
1831
|
+ worklist pendvar k node
|
|
|
1832
|
+ return ()
|
|
|
1833
|
+
|
|
|
1834
|
+ worker ds_env@DownsweepEnv{..} exc_var visvar worklist pendvar k node =
|
|
|
1835
|
+ withAbstractSem (compile_sem ds_make_env) $ -- acquire -j par token
|
|
|
1836
|
+ withLoggerHsc 1{- TODO: seq numb-} ds_make_env \ lcl_hsc_env -> do
|
|
|
1837
|
+
|
|
|
1838
|
+ r <- MC.try $ runDownsweepM ds_env{ds_hsc_env = lcl_hsc_env} $
|
|
|
1839
|
+ expand node -- do the main work!
|
|
|
1840
|
+
|
|
|
1841
|
+ case r of
|
|
|
1842
|
+ Left (e :: MC.SomeException) ->
|
|
|
1843
|
+ -- signal exception in this thread for the main thread to throw it
|
|
|
1844
|
+ atomically $ modifyTVar' exc_var (<|> Just e)
|
|
|
1845
|
+ Right NSkip ->
|
|
|
1846
|
+ -- write node skip; nothing new to queue
|
|
|
1847
|
+ atomically $ modifyTVar' visvar (Map.insert k NSkip)
|
|
|
1848
|
+ Right (NSuccess (v,ns)) -> do
|
|
|
1849
|
+ -- write success result; queue the next nodes
|
|
|
1850
|
+ atomically $ modifyTVar' visvar (Map.insert k (NSuccess v))
|
|
|
1851
|
+ mapM_ (atomically . writeTQueue worklist) ns
|
|
|
1852
|
+
|
|
|
1853
|
+ -- no longer pending:
|
|
|
1854
|
+ atomically $ modifyTVar' pendvar (Set.delete k)
|
|
1799
|
1855
|
|
|
1800
|
1856
|
{-
|
|
1801
|
1857
|
Note [Downsweep Control Flow and Caching]
|