Rodrigo Mesquita pushed to branch wip/romes/27514 at Glasgow Haskell Compiler / GHC

Commits:

1 changed file:

Changes:

  • compiler/GHC/Driver/Downsweep.hs
    ... ... @@ -179,8 +179,9 @@ incrementally constructing a ModuleGraph using the GHC API; See #27054). So
    179 179
     `downsweep` takes a `Maybe ModuleGraph` as one of its arguments.
    
    180 180
     
    
    181 181
     Downsweep iteratively *expands* each so-called 'DownsweepNode' into a list of
    
    182
    -its dependencies, and recursively traverses all reachable nodes in a
    
    183
    -depth-first order using 'dfsBuild'. A 'DownsweepNode' is *expanded* by 'dsNodeExpand':
    
    182
    +its dependencies, and recursively traverses all reachable nodes in a parallel
    
    183
    +non-det-depth-first order using 'parDfsBuild'. A 'DownsweepNode' is *expanded*
    
    184
    +by 'dsNodeExpand':
    
    184 185
     
    
    185 186
       dsNodeExpand :: DownsweepNode -> DownsweepM (NodeRes (ModuleGraphNode, [DownsweepNode]))
    
    186 187
     
    
    ... ... @@ -576,8 +577,8 @@ addModSummaryCache ms pr fe = upd_fe fe
    576 577
     
    
    577 578
     modifySummCache :: ModSummaryCache -> (ModSummaryCacheMap -> ModSummaryCacheMap) -> IO ()
    
    578 579
     modifyImpsCache :: ImportsCache    -> (ImportsCacheMap    -> ImportsCacheMap)    -> IO ()
    
    579
    -modifySummCache r f = modifyMVar r (\c -> let !r = f c in pure (r, ()))
    
    580
    -modifyImpsCache r f = modifyMVar r (\c -> let !r = f c in pure (r, ()))
    
    580
    +modifySummCache v f = modifyMVar v (\c -> let !r = f c in pure (r, ()))
    
    581
    +modifyImpsCache v f = modifyMVar v (\c -> let !r = f c in pure (r, ()))
    
    581 582
     
    
    582 583
     -- | A cache from a module import (in given home unit context, with a package
    
    583 584
     -- qualifier, and the imported module name (with or without SOURCE)) to the
    
    ... ... @@ -1728,10 +1729,10 @@ getPreprocessedImports hsc_env src_fn mb_phase maybe_buf = do
    1728 1729
       return PreprocessedImports {..}
    
    1729 1730
     
    
    1730 1731
     --------------------------------------------------------------------------------
    
    1731
    --- * Generic traversal of iteratively-built graph: dfsBuild
    
    1732
    +-- * Generic traversal of iteratively-built graph: parDfsBuild
    
    1732 1733
     --------------------------------------------------------------------------------
    
    1733 1734
     
    
    1734
    --- | The result of expanding a node in 'dfsBuild'.
    
    1735
    +-- | The result of expanding a node in 'parDfsBuild'.
    
    1735 1736
     data NodeRes v
    
    1736 1737
       -- | Computed the node payload successfully
    
    1737 1738
       = NSuccess v
    
    ... ... @@ -1744,7 +1745,7 @@ data NodeRes v
    1744 1745
       -- abort.
    
    1745 1746
       | NSkip
    
    1746 1747
     
    
    1747
    --- | In a parallel depth-first order, and starting from the given roots, traverse a
    
    1748
    +-- | In a parallel non-det-depth-first order, and starting from the given roots, traverse a
    
    1748 1749
     -- graph by iteratively expanding a node into a payload and a list of children
    
    1749 1750
     -- nodes to visit next.
    
    1750 1751
     --
    
    ... ... @@ -1757,7 +1758,7 @@ data NodeRes v
    1757 1758
     -- The result is a mapping from the key of every node transitively reachable
    
    1758 1759
     -- from the root nodes (inclusively) to the payload returned by expanding that
    
    1759 1760
     -- node. The result includes the previously visited nodes given in @base_map@,
    
    1760
    --- s.t. @dfsBuild base_map [] _ _ == base_map@.
    
    1761
    +-- s.t. @parDfsBuild base_map [] _ _ == base_map@.
    
    1761 1762
     --
    
    1762 1763
     -- The @expand@ function returns an 'NResult'. See the 'NResult' documentation
    
    1763 1764
     -- for more information about each result type.
    
    ... ... @@ -1784,30 +1785,34 @@ parDfsBuild base_map roots key expand = ReaderT $ \ds_env -> do
    1784 1785
         visited_var <- newTVarIO (fromMaybe Map.empty base_map)
    
    1785 1786
         pending     <- newTVarIO Set.empty
    
    1786 1787
         worklist    <- newTQueueIO
    
    1788
    +
    
    1787 1789
         coord_tid   <- forkIO $
    
    1788 1790
           coordinator ds_env exc_var visited_var worklist pending
    
    1789 1791
             `MC.catch` \case
    
    1790 1792
               (e::MC.SomeException)
    
    1791 1793
                 -- exit cleanly when killed
    
    1792
    -            | Just ThreadKilled <- fromException e -> pure ()
    
    1793
    -            -- otherwise signal exc_var for the main thread to throw it
    
    1794
    +            | Just ThreadKilled <- fromException e -> return ()
    
    1795
    +            -- if the coordinator somehow else crashes,
    
    1796
    +            -- signal the exc_var for the main thread to throw it
    
    1794 1797
                 | otherwise -> atomically (modifyTVar' exc_var (<|> Just e))
    
    1798
    +
    
    1795 1799
         mapM_ (atomically . writeTQueue worklist) roots
    
    1800
    +
    
    1801
    +    -- this txn retries until all work is done
    
    1796 1802
         mb_exc <- atomically $ do
    
    1797 1803
           readTVar exc_var >>= \case
    
    1798
    -        Just e ->
    
    1799
    -          -- exit if there's an exception
    
    1800
    -          return (Just e)
    
    1804
    +        Just e -> return (Just e)
    
    1801 1805
             Nothing -> do
    
    1802
    -          -- or exit when pending and worklist are both empty atomically; else, `retry`.
    
    1803 1806
               empty_worklist <- isEmptyTQueue worklist
    
    1804 1807
               empty_pending  <- Set.null <$> readTVar pending
    
    1805 1808
               unless (empty_worklist && empty_pending) retry
    
    1806 1809
               return Nothing
    
    1810
    +
    
    1807 1811
         killThread coord_tid
    
    1808 1812
         case mb_exc of
    
    1809 1813
           Just e  -> throwIO e
    
    1810 1814
           Nothing -> readTVarIO visited_var
    
    1815
    +
    
    1811 1816
       where
    
    1812 1817
         coordinator ds_env exc_var visvar worklist pendvar = forever $ do
    
    1813 1818
           mb_node_to_expand <- atomically $ do
    
    ... ... @@ -1828,40 +1833,35 @@ parDfsBuild base_map roots key expand = ReaderT $ \ds_env -> do
    1828 1833
     
    
    1829 1834
           case mb_node_to_expand of
    
    1830 1835
             Nothing        -> return ()
    
    1831
    -        Just (k, node) -> do
    
    1836
    +        Just (k, node) -> void $ do
    
    1837
    +          let i = 1 -- tODO: not 1
    
    1832 1838
               withLocalTmpFSMake (ds_make_env ds_env) $ \make_env ->
    
    1833 1839
                 forkIO $
    
    1834
    -              worker ds_env{ds_make_env = make_env} exc_var visvar
    
    1835
    -                     worklist pendvar k node
    
    1836
    -                `MC.catch` \(e::MC.SomeException) -> atomically (modifyTVar' exc_var (<|> Just e))
    
    1837
    -          return ()
    
    1840
    +              worker i ds_env{ds_make_env = make_env} visvar worklist pendvar k node
    
    1841
    +                -- write exception in the worker to the main thread
    
    1842
    +                `MC.catch` \(e::MC.SomeException) ->
    
    1843
    +                  atomically (modifyTVar' exc_var (<|> Just e))
    
    1838 1844
     
    
    1839
    -    worker ds_env@DownsweepEnv{..} exc_var visvar worklist pendvar k node =
    
    1845
    +    worker i ds_env@DownsweepEnv{..} visvar worklist pendvar k node =
    
    1840 1846
           withAbstractSem (compile_sem ds_make_env) $ -- acquire -j par token
    
    1841
    -      withLoggerHsc 1{- TODO: seq numb-} ds_make_env \ lcl_hsc_env -> do
    
    1847
    +      withLoggerHsc i ds_make_env \ lcl_hsc_env -> do
    
    1842 1848
     
    
    1843
    -        r <- MC.try $ runDownsweepM ds_env{ds_hsc_env = lcl_hsc_env} $
    
    1849
    +        r <- runDownsweepM ds_env{ds_hsc_env = lcl_hsc_env} $
    
    1844 1850
                  expand node -- do the main work!
    
    1845 1851
     
    
    1846 1852
             case r of
    
    1847
    -          Left (e :: MC.SomeException) ->
    
    1848
    -            -- signal exception in this thread for the main thread to throw it
    
    1849
    -            atomically $ modifyTVar' exc_var (<|> Just e)
    
    1850
    -          Right NSkip ->
    
    1851
    -            -- write node skip; nothing new to queue
    
    1853
    +          NSkip ->
    
    1852 1854
                 atomically $ modifyTVar' visvar (Map.insert k NSkip)
    
    1853
    -          Right (NSuccess (v,ns)) -> do
    
    1854
    -            -- write success result; queue the next nodes
    
    1855
    +          NSuccess (v,ns) -> do
    
    1855 1856
                 atomically $ modifyTVar' visvar (Map.insert k (NSuccess v))
    
    1856 1857
                 mapM_ (atomically . writeTQueue worklist) ns
    
    1857 1858
     
    
    1858
    -        -- no longer pending:
    
    1859 1859
             atomically $ modifyTVar' pendvar (Set.delete k)
    
    1860 1860
     
    
    1861 1861
     {-
    
    1862 1862
     Note [Downsweep Control Flow and Caching]
    
    1863 1863
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    
    1864
    -The control flow of downsweep is extracted into a single function `dfsBuild`,
    
    1864
    +The control flow of downsweep is extracted into a single function `parDfsBuild`,
    
    1865 1865
     which takes care of iteratively expanding and traversing all nodes of the
    
    1866 1866
     in-construction module graph necessary to build a full `ModuleGraph` at the
    
    1867 1867
     end.
    
    ... ... @@ -1870,7 +1870,7 @@ There are three levels of caching going on, all of which are necessary to make
    1870 1870
     sure we don't do repeated work (notably, we NEVER summarise the same module
    
    1871 1871
     twice).
    
    1872 1872
     
    
    1873
    -1. `dfsBuild` accumulates the final module graph and never revisits the
    
    1873
    +1. `parDfsBuild` accumulates the final module graph and never revisits the
    
    1874 1874
        same node of the module graph. Cache is keyed by the final
    
    1875 1875
        `ModuleGraph`s `NodeKey`s.
    
    1876 1876