| ... |
... |
@@ -179,8 +179,9 @@ incrementally constructing a ModuleGraph using the GHC API; See #27054). So |
|
179
|
179
|
`downsweep` takes a `Maybe ModuleGraph` as one of its arguments.
|
|
180
|
180
|
|
|
181
|
181
|
Downsweep iteratively *expands* each so-called 'DownsweepNode' into a list of
|
|
182
|
|
-its dependencies, and recursively traverses all reachable nodes in a
|
|
183
|
|
-depth-first order using 'dfsBuild'. A 'DownsweepNode' is *expanded* by 'dsNodeExpand':
|
|
|
182
|
+its dependencies, and recursively traverses all reachable nodes in a parallel
|
|
|
183
|
+non-det-depth-first order using 'parDfsBuild'. A 'DownsweepNode' is *expanded*
|
|
|
184
|
+by 'dsNodeExpand':
|
|
184
|
185
|
|
|
185
|
186
|
dsNodeExpand :: DownsweepNode -> DownsweepM (NodeRes (ModuleGraphNode, [DownsweepNode]))
|
|
186
|
187
|
|
| ... |
... |
@@ -576,8 +577,8 @@ addModSummaryCache ms pr fe = upd_fe fe |
|
576
|
577
|
|
|
577
|
578
|
modifySummCache :: ModSummaryCache -> (ModSummaryCacheMap -> ModSummaryCacheMap) -> IO ()
|
|
578
|
579
|
modifyImpsCache :: ImportsCache -> (ImportsCacheMap -> ImportsCacheMap) -> IO ()
|
|
579
|
|
-modifySummCache r f = modifyMVar r (\c -> let !r = f c in pure (r, ()))
|
|
580
|
|
-modifyImpsCache r f = modifyMVar r (\c -> let !r = f c in pure (r, ()))
|
|
|
580
|
+modifySummCache v f = modifyMVar v (\c -> let !r = f c in pure (r, ()))
|
|
|
581
|
+modifyImpsCache v f = modifyMVar v (\c -> let !r = f c in pure (r, ()))
|
|
581
|
582
|
|
|
582
|
583
|
-- | A cache from a module import (in given home unit context, with a package
|
|
583
|
584
|
-- qualifier, and the imported module name (with or without SOURCE)) to the
|
| ... |
... |
@@ -1728,10 +1729,10 @@ getPreprocessedImports hsc_env src_fn mb_phase maybe_buf = do |
|
1728
|
1729
|
return PreprocessedImports {..}
|
|
1729
|
1730
|
|
|
1730
|
1731
|
--------------------------------------------------------------------------------
|
|
1731
|
|
--- * Generic traversal of iteratively-built graph: dfsBuild
|
|
|
1732
|
+-- * Generic traversal of iteratively-built graph: parDfsBuild
|
|
1732
|
1733
|
--------------------------------------------------------------------------------
|
|
1733
|
1734
|
|
|
1734
|
|
--- | The result of expanding a node in 'dfsBuild'.
|
|
|
1735
|
+-- | The result of expanding a node in 'parDfsBuild'.
|
|
1735
|
1736
|
data NodeRes v
|
|
1736
|
1737
|
-- | Computed the node payload successfully
|
|
1737
|
1738
|
= NSuccess v
|
| ... |
... |
@@ -1744,7 +1745,7 @@ data NodeRes v |
|
1744
|
1745
|
-- abort.
|
|
1745
|
1746
|
| NSkip
|
|
1746
|
1747
|
|
|
1747
|
|
--- | In a parallel depth-first order, and starting from the given roots, traverse a
|
|
|
1748
|
+-- | In a parallel non-det-depth-first order, and starting from the given roots, traverse a
|
|
1748
|
1749
|
-- graph by iteratively expanding a node into a payload and a list of children
|
|
1749
|
1750
|
-- nodes to visit next.
|
|
1750
|
1751
|
--
|
| ... |
... |
@@ -1757,7 +1758,7 @@ data NodeRes v |
|
1757
|
1758
|
-- The result is a mapping from the key of every node transitively reachable
|
|
1758
|
1759
|
-- from the root nodes (inclusively) to the payload returned by expanding that
|
|
1759
|
1760
|
-- node. The result includes the previously visited nodes given in @base_map@,
|
|
1760
|
|
--- s.t. @dfsBuild base_map [] _ _ == base_map@.
|
|
|
1761
|
+-- s.t. @parDfsBuild base_map [] _ _ == base_map@.
|
|
1761
|
1762
|
--
|
|
1762
|
1763
|
-- The @expand@ function returns an 'NResult'. See the 'NResult' documentation
|
|
1763
|
1764
|
-- for more information about each result type.
|
| ... |
... |
@@ -1784,30 +1785,34 @@ parDfsBuild base_map roots key expand = ReaderT $ \ds_env -> do |
|
1784
|
1785
|
visited_var <- newTVarIO (fromMaybe Map.empty base_map)
|
|
1785
|
1786
|
pending <- newTVarIO Set.empty
|
|
1786
|
1787
|
worklist <- newTQueueIO
|
|
|
1788
|
+
|
|
1787
|
1789
|
coord_tid <- forkIO $
|
|
1788
|
1790
|
coordinator ds_env exc_var visited_var worklist pending
|
|
1789
|
1791
|
`MC.catch` \case
|
|
1790
|
1792
|
(e::MC.SomeException)
|
|
1791
|
1793
|
-- exit cleanly when killed
|
|
1792
|
|
- | Just ThreadKilled <- fromException e -> pure ()
|
|
1793
|
|
- -- otherwise signal exc_var for the main thread to throw it
|
|
|
1794
|
+ | Just ThreadKilled <- fromException e -> return ()
|
|
|
1795
|
+ -- if the coordinator somehow else crashes,
|
|
|
1796
|
+ -- signal the exc_var for the main thread to throw it
|
|
1794
|
1797
|
| otherwise -> atomically (modifyTVar' exc_var (<|> Just e))
|
|
|
1798
|
+
|
|
1795
|
1799
|
mapM_ (atomically . writeTQueue worklist) roots
|
|
|
1800
|
+
|
|
|
1801
|
+ -- this txn retries until all work is done
|
|
1796
|
1802
|
mb_exc <- atomically $ do
|
|
1797
|
1803
|
readTVar exc_var >>= \case
|
|
1798
|
|
- Just e ->
|
|
1799
|
|
- -- exit if there's an exception
|
|
1800
|
|
- return (Just e)
|
|
|
1804
|
+ Just e -> return (Just e)
|
|
1801
|
1805
|
Nothing -> do
|
|
1802
|
|
- -- or exit when pending and worklist are both empty atomically; else, `retry`.
|
|
1803
|
1806
|
empty_worklist <- isEmptyTQueue worklist
|
|
1804
|
1807
|
empty_pending <- Set.null <$> readTVar pending
|
|
1805
|
1808
|
unless (empty_worklist && empty_pending) retry
|
|
1806
|
1809
|
return Nothing
|
|
|
1810
|
+
|
|
1807
|
1811
|
killThread coord_tid
|
|
1808
|
1812
|
case mb_exc of
|
|
1809
|
1813
|
Just e -> throwIO e
|
|
1810
|
1814
|
Nothing -> readTVarIO visited_var
|
|
|
1815
|
+
|
|
1811
|
1816
|
where
|
|
1812
|
1817
|
coordinator ds_env exc_var visvar worklist pendvar = forever $ do
|
|
1813
|
1818
|
mb_node_to_expand <- atomically $ do
|
| ... |
... |
@@ -1828,40 +1833,35 @@ parDfsBuild base_map roots key expand = ReaderT $ \ds_env -> do |
|
1828
|
1833
|
|
|
1829
|
1834
|
case mb_node_to_expand of
|
|
1830
|
1835
|
Nothing -> return ()
|
|
1831
|
|
- Just (k, node) -> do
|
|
|
1836
|
+ Just (k, node) -> void $ do
|
|
|
1837
|
+ let i = 1 -- tODO: not 1
|
|
1832
|
1838
|
withLocalTmpFSMake (ds_make_env ds_env) $ \make_env ->
|
|
1833
|
1839
|
forkIO $
|
|
1834
|
|
- worker ds_env{ds_make_env = make_env} exc_var visvar
|
|
1835
|
|
- worklist pendvar k node
|
|
1836
|
|
- `MC.catch` \(e::MC.SomeException) -> atomically (modifyTVar' exc_var (<|> Just e))
|
|
1837
|
|
- return ()
|
|
|
1840
|
+ worker i ds_env{ds_make_env = make_env} visvar worklist pendvar k node
|
|
|
1841
|
+ -- write exception in the worker to the main thread
|
|
|
1842
|
+ `MC.catch` \(e::MC.SomeException) ->
|
|
|
1843
|
+ atomically (modifyTVar' exc_var (<|> Just e))
|
|
1838
|
1844
|
|
|
1839
|
|
- worker ds_env@DownsweepEnv{..} exc_var visvar worklist pendvar k node =
|
|
|
1845
|
+ worker i ds_env@DownsweepEnv{..} visvar worklist pendvar k node =
|
|
1840
|
1846
|
withAbstractSem (compile_sem ds_make_env) $ -- acquire -j par token
|
|
1841
|
|
- withLoggerHsc 1{- TODO: seq numb-} ds_make_env \ lcl_hsc_env -> do
|
|
|
1847
|
+ withLoggerHsc i ds_make_env \ lcl_hsc_env -> do
|
|
1842
|
1848
|
|
|
1843
|
|
- r <- MC.try $ runDownsweepM ds_env{ds_hsc_env = lcl_hsc_env} $
|
|
|
1849
|
+ r <- runDownsweepM ds_env{ds_hsc_env = lcl_hsc_env} $
|
|
1844
|
1850
|
expand node -- do the main work!
|
|
1845
|
1851
|
|
|
1846
|
1852
|
case r of
|
|
1847
|
|
- Left (e :: MC.SomeException) ->
|
|
1848
|
|
- -- signal exception in this thread for the main thread to throw it
|
|
1849
|
|
- atomically $ modifyTVar' exc_var (<|> Just e)
|
|
1850
|
|
- Right NSkip ->
|
|
1851
|
|
- -- write node skip; nothing new to queue
|
|
|
1853
|
+ NSkip ->
|
|
1852
|
1854
|
atomically $ modifyTVar' visvar (Map.insert k NSkip)
|
|
1853
|
|
- Right (NSuccess (v,ns)) -> do
|
|
1854
|
|
- -- write success result; queue the next nodes
|
|
|
1855
|
+ NSuccess (v,ns) -> do
|
|
1855
|
1856
|
atomically $ modifyTVar' visvar (Map.insert k (NSuccess v))
|
|
1856
|
1857
|
mapM_ (atomically . writeTQueue worklist) ns
|
|
1857
|
1858
|
|
|
1858
|
|
- -- no longer pending:
|
|
1859
|
1859
|
atomically $ modifyTVar' pendvar (Set.delete k)
|
|
1860
|
1860
|
|
|
1861
|
1861
|
{-
|
|
1862
|
1862
|
Note [Downsweep Control Flow and Caching]
|
|
1863
|
1863
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
1864
|
|
-The control flow of downsweep is extracted into a single function `dfsBuild`,
|
|
|
1864
|
+The control flow of downsweep is extracted into a single function `parDfsBuild`,
|
|
1865
|
1865
|
which takes care of iteratively expanding and traversing all nodes of the
|
|
1866
|
1866
|
in-construction module graph necessary to build a full `ModuleGraph` at the
|
|
1867
|
1867
|
end.
|
| ... |
... |
@@ -1870,7 +1870,7 @@ There are three levels of caching going on, all of which are necessary to make |
|
1870
|
1870
|
sure we don't do repeated work (notably, we NEVER summarise the same module
|
|
1871
|
1871
|
twice).
|
|
1872
|
1872
|
|
|
1873
|
|
-1. `dfsBuild` accumulates the final module graph and never revisits the
|
|
|
1873
|
+1. `parDfsBuild` accumulates the final module graph and never revisits the
|
|
1874
|
1874
|
same node of the module graph. Cache is keyed by the final
|
|
1875
|
1875
|
`ModuleGraph`s `NodeKey`s.
|
|
1876
|
1876
|
|