David Eichmann pushed to branch wip/davide/io_manager_tests at Glasgow Haskell Compiler / GHC

Commits:

6 changed files:

Changes:

  • testsuite/config/ghc
    ... ... @@ -315,6 +315,59 @@ def get_compiler_info():
    315 315
             config.ghc_th_way       = "normal"
    
    316 316
             config.ghc_th_hi_ext    = "hi"
    
    317 317
     
    
    318
    +    # I/O manager discovery.
    
    319
    +    #
    
    320
    +    # Which I/O managers (GHC's `--io-manager` RTS flag) a given RTS build
    
    321
    +    # supports depends on the target platform and on whether the RTS is the
    
    322
    +    # threaded or non-threaded flavour (see rts/IOManager.h's
    
    323
    +    # IOMGR_ENABLED_* macros). Rather than duplicate that platform/way
    
    324
    +    # dependent logic here, we ask the RTS itself: giving it a bogus
    
    325
    +    # '--io-manager=' choice makes it print, on stderr, the list of I/O
    
    326
    +    # managers it was actually built to support (see IOManager.h's
    
    327
    +    # IOMGRS_ENABLED_STR, used when reporting the error in RtsFlags.c),
    
    328
    +    # which we then parse out.
    
    329
    +    def discover_io_managers_generic(extra_flags):
    
    330
    +        import tempfile
    
    331
    +
    
    332
    +        test_src = 'main :: IO ()\nmain = return ()\n'
    
    333
    +
    
    334
    +        res = [None]
    
    335
    +
    
    336
    +        with tempfile.TemporaryDirectory() as d:
    
    337
    +            src = Path(d) / 'IOMgrProbe.hs'
    
    338
    +            src.write_text(test_src)
    
    339
    +            exe = Path(d) / ('IoMgrProbe' + config.exeext)
    
    340
    +
    
    341
    +            # -rtsopts is needed so that the resulting binary will even
    
    342
    +            # consider an --io-manager= flag (rather than immediately
    
    343
    +            # rejecting all "unsafe" RTS options).
    
    344
    +            cmd = '{} -v0 -rtsopts -o {} {} {}'.format(
    
    345
    +                                    config.compiler,
    
    346
    +                                    exe,
    
    347
    +                                    src,
    
    348
    +                                    ' '.join(extra_flags))
    
    349
    +            try:
    
    350
    +                cp = subprocess.run(
    
    351
    +                    cmd,
    
    352
    +                    shell=True,
    
    353
    +                    cwd=d,
    
    354
    +                    stdout=subprocess.DEVNULL,
    
    355
    +                    stderr=None if config.verbose >= 3 else subprocess.DEVNULL)
    
    356
    +                if cp.returncode != 0 or not exe.exists():
    
    357
    +                    raise Exception("Error in testsuite/config/ghc.discover_io_managers: failed to compile with: {}".format(cmd))
    
    358
    +                res = list(discover_io_managers(exe, d))
    
    359
    +
    
    360
    +            except Exception as err:
    
    361
    +                print("Exception thrown in testsuite/config/ghc.discover_io_managers_generic: %s", err)
    
    362
    +
    
    363
    +        return res
    
    364
    +
    
    365
    +    config.io_managers_nonthreaded = discover_io_managers_generic([])
    
    366
    +    config.io_managers_threaded = (
    
    367
    +        discover_io_managers_generic(['-threaded']) if config.ghc_with_threaded_rts
    
    368
    +        else [])
    
    369
    +    config.io_managers_all = list(set(config.io_managers_nonthreaded).union(set(config.io_managers_threaded)))
    
    370
    +
    
    318 371
         config.cmm_cpp_is_gcc = test_compile(
    
    319 372
             ['-c'],
    
    320 373
             test_filename='test.cmm',
    

  • testsuite/driver/runtests.py
    ... ... @@ -28,7 +28,7 @@ import subprocess
    28 28
     
    
    29 29
     import asyncio
    
    30 30
     
    
    31
    -from testutil import getStdout, str_warn, str_info, print_table, shorten_metric_name, str_removeprefix
    
    31
    +from testutil import discover_io_managers, getStdout, str_warn, str_info, print_table, shorten_metric_name, str_removeprefix
    
    32 32
     from testglobals import getConfig, ghc_env, TestConfig, t, \
    
    33 33
                             TestOptions, brokens, PerfMetric
    
    34 34
     from my_typing import TestName
    

  • testsuite/driver/testglobals.py
    ... ... @@ -126,6 +126,15 @@ class TestConfig:
    126 126
             self.way_flags = {}  # type: Dict[WayName, List[str]]
    
    127 127
             self.way_rts_flags = {}  # type: Dict[WayName, List[str]]
    
    128 128
     
    
    129
    +        # I/O managers (GHC's `--io-manager` RTS flag) supported by the threaded runtime.
    
    130
    +        self.io_managers_threaded = []  # type: List[str]
    
    131
    +
    
    132
    +        # I/O managers (GHC's `--io-manager` RTS flag) supported by the non-threaded runtime.
    
    133
    +        self.io_managers_nonthreaded = []  # type: List[str]
    
    134
    +
    
    135
    +        # All I/O managers (GHC's `--io-manager` RTS flag)
    
    136
    +        self.io_managers_all = []  # type: List[str]
    
    137
    +
    
    129 138
             # Do we have a functional LLVM toolchain?
    
    130 139
             self.have_llvm = False
    
    131 140
     
    
    ... ... @@ -366,6 +375,11 @@ class TestOptions:
    366 375
            # add these ways to the default set
    
    367 376
            self.extra_ways = [] # type: List[WayName]
    
    368 377
     
    
    378
    +       # IO Managers to test
    
    379
    +       # * None - only default IO Manager
    
    380
    +       # * [...] - list of io managers e.g. "mio" or "poll"
    
    381
    +       self.io_managers = [None] # type: List[str|None]
    
    382
    +
    
    369 383
            # the result we normally expect for this test
    
    370 384
            self.expect = 'pass'
    
    371 385
     
    

  • testsuite/driver/testlib.py
    ... ... @@ -23,7 +23,7 @@ import subprocess
    23 23
     
    
    24 24
     from testglobals import config, ghc_env, default_testopts, brokens, t, \
    
    25 25
                             TestRun, TestResult, TestOptions, PerfMetric
    
    26
    -from testutil import strip_quotes, lndir, link_or_copy_file, passed, \
    
    26
    +from testutil import discover_io_managers, strip_quotes, lndir, link_or_copy_file, passed, \
    
    27 27
                          failBecause, residency_testing_metrics, \
    
    28 28
                          stable_perf_counters, \
    
    29 29
                          PassFail, badResult, str_warn, str_removeprefix
    
    ... ... @@ -569,6 +569,33 @@ def extra_ways( ways: List[WayName] ):
    569 569
     
    
    570 570
         return helper
    
    571 571
     
    
    572
    +# -----
    
    573
    +
    
    574
    +def _io_managers( name: TestName, opts, io_mgrs: List[str] ) -> None:
    
    575
    +    opts.io_managers = io_mgrs
    
    576
    +
    
    577
    +def io_managers( io_mgrs: List[str] ):
    
    578
    +    """
    
    579
    +    Run this test once for each of the given I/O managers (identified by
    
    580
    +    the name used with GHC's `--io-manager` RTS flag, e.g. 'mio',
    
    581
    +    'select', 'winio'), instead of the usual set of ways. I/O managers
    
    582
    +    that are not supported on the current platform/build are silently
    
    583
    +    skipped.
    
    584
    +    """
    
    585
    +    return lambda name, opts: _io_managers(name, opts, io_mgrs)
    
    586
    +
    
    587
    +def all_io_managers( name: TestName, opts ):
    
    588
    +    """
    
    589
    +    Run this test once for each I/O manager supported by this
    
    590
    +    platform/build.
    
    591
    +    """
    
    592
    +    _io_managers(name, opts, config.io_managers_all)
    
    593
    +
    
    594
    +def all_io_managers_except(exclude: List[str]):
    
    595
    +    """
    
    596
    +    Like all_io_managers, but excluding the given I/O managers.
    
    597
    +    """
    
    598
    +    return lambda name, opts: _io_managers(name, opts, list(set(config.io_managers_all) - set(exclude)))
    
    572 599
     
    
    573 600
     # -----
    
    574 601
     
    
    ... ... @@ -1571,6 +1598,12 @@ do_not_copy = ('.hi', '.o', '.dyn_hi'
    1571 1598
                   , '.dyn_o', '.out'
    
    1572 1599
                   ,'.hi-boot', '.o-boot') # 12112
    
    1573 1600
     
    
    1601
    +def way_supports_io_manager(way: WayName, io_manager: None | str) -> bool:
    
    1602
    +    if not io_manager:
    
    1603
    +        return True
    
    1604
    +    way_is_threaded = '-threaded' in config.way_rts_flags[way]
    
    1605
    +    return io_manager in config.io_managers_threaded if way_is_threaded else io_manager in config.io_managers_nonthreaded
    
    1606
    +
    
    1574 1607
     async def test_common_work(name: TestName, opts,
    
    1575 1608
                          func, args) -> None:
    
    1576 1609
         try:
    
    ... ... @@ -1619,7 +1652,20 @@ async def test_common_work(name: TestName, opts,
    1619 1652
             # A test itself can request extra ways by setting opts.extra_ways
    
    1620 1653
             all_ways = list(OrderedDict.fromkeys(all_ways + [way for way in opts.extra_ways if way not in all_ways]))
    
    1621 1654
     
    
    1622
    -        t.total_test_cases += len(all_ways)
    
    1655
    +        # We run each way over each io manager (or just None meaning the default IO Manager)
    
    1656
    +        all_ways_and_io_managers = dict(
    
    1657
    +            (
    
    1658
    +                way,
    
    1659
    +                [io_manager
    
    1660
    +                    for io_manager in opts.io_managers
    
    1661
    +                    if way_supports_io_manager(way, io_manager)
    
    1662
    +                ]
    
    1663
    +            )
    
    1664
    +            for way in all_ways
    
    1665
    +        )
    
    1666
    +
    
    1667
    +        for io_managers in all_ways_and_io_managers.values():
    
    1668
    +            t.total_test_cases += len(io_managers)
    
    1623 1669
     
    
    1624 1670
             only_ways = getTestOpts().only_ways
    
    1625 1671
             ok_way = lambda way: \
    
    ... ... @@ -1702,17 +1748,26 @@ async def test_common_work(name: TestName, opts,
    1702 1748
     
    
    1703 1749
             # Run the required tests...
    
    1704 1750
             for way in do_ways:
    
    1705
    -            if stopping():
    
    1706
    -                break
    
    1707
    -            try:
    
    1708
    -                await do_test(name, way, func, args, files)
    
    1709
    -            except KeyboardInterrupt:
    
    1710
    -                stopNow()
    
    1711
    -            except Exception as e:
    
    1712
    -                traceback.print_exc()
    
    1713
    -                framework_fail(name, way, traceback.format_exc())
    
    1751
    +            for io_manager in all_ways_and_io_managers[way]:
    
    1752
    +                if stopping():
    
    1753
    +                    break
    
    1754
    +                try:
    
    1755
    +                    await do_test(
    
    1756
    +                        name,
    
    1757
    +                        way,
    
    1758
    +                        io_manager,
    
    1759
    +                        func,
    
    1760
    +                        args,
    
    1761
    +                        files)
    
    1762
    +                except KeyboardInterrupt:
    
    1763
    +                    stopNow()
    
    1764
    +                except Exception as e:
    
    1765
    +                    traceback.print_exc()
    
    1766
    +                    framework_fail(name, way, traceback.format_exc())
    
    1767
    +
    
    1768
    +        for skipped_way in set(all_ways) - set(do_ways):
    
    1769
    +            t.n_tests_skipped += len(all_ways_and_io_managers[skipped_way])
    
    1714 1770
     
    
    1715
    -        t.n_tests_skipped += len(set(all_ways) - set(do_ways))
    
    1716 1771
             if getTestOpts().expect == 'missing-lib': t.n_missing_libs += 1
    
    1717 1772
     
    
    1718 1773
             if config.cleanup and do_ways:
    
    ... ... @@ -1731,6 +1786,7 @@ async def test_common_work(name: TestName, opts,
    1731 1786
     
    
    1732 1787
     async def do_test(name: TestName,
    
    1733 1788
                 way: WayName,
    
    1789
    +            io_manager: None | str,
    
    1734 1790
                 func: Callable[..., Awaitable[PassFail]],
    
    1735 1791
                 args,
    
    1736 1792
                 files: Set[str]
    
    ... ... @@ -1738,6 +1794,8 @@ async def do_test(name: TestName,
    1738 1794
         opts = getTestOpts()
    
    1739 1795
     
    
    1740 1796
         full_name = name + '(' + way + ')'
    
    1797
    +    if io_manager:
    
    1798
    +        full_name += ' (IO Manager: {})'.format(io_manager)
    
    1741 1799
         test_n = len(allTestNames)
    
    1742 1800
         progress_args = [ full_name, t.total_tests, test_n,
    
    1743 1801
             [len(t.unexpected_passes),
    
    ... ... @@ -1818,7 +1876,20 @@ async def do_test(name: TestName,
    1818 1876
                 # Don't continue and try to run the test if the pre_cmd fails.
    
    1819 1877
                 return
    
    1820 1878
     
    
    1821
    -    result = await func(*[name,way] + args)
    
    1879
    +
    
    1880
    +    func_has_io_manager_param = func in [
    
    1881
    +        backpack_run,
    
    1882
    +        compile_and_run,
    
    1883
    +        ghci_multiunit_script,
    
    1884
    +        ghci_script,
    
    1885
    +        multi_compile_and_run,
    
    1886
    +        multimod_compile_and_run,
    
    1887
    +        warn_and_run
    
    1888
    +    ]
    
    1889
    +
    
    1890
    +    io_manager_arg = [io_manager] if func_has_io_manager_param else []
    
    1891
    +
    
    1892
    +    result = await func(*[name,way] + io_manager_arg + args)
    
    1822 1893
     
    
    1823 1894
         if opts.expect not in ['pass', 'fail', 'missing-lib']:
    
    1824 1895
             framework_fail(name, way, 'bad expected ' + opts.expect)
    
    ... ... @@ -1907,7 +1978,7 @@ def framework_warn(name: TestName, way: WayName, reason: str) -> None:
    1907 1978
     # run_command.
    
    1908 1979
     
    
    1909 1980
     async def run_command( name, way, cmd ):
    
    1910
    -    return await simple_run( name, WayName(''), override_options(cmd), '' )
    
    1981
    +    return await simple_run( name, WayName(''), None, override_options(cmd), '' )
    
    1911 1982
     
    
    1912 1983
     async def makefile_test( name, way, target=None ):
    
    1913 1984
         if target is None:
    
    ... ... @@ -1919,7 +1990,7 @@ async def makefile_test( name, way, target=None ):
    1919 1990
     # -----------------------------------------------------------------------------
    
    1920 1991
     # GHCi tests
    
    1921 1992
     
    
    1922
    -async def ghci_script( name, way, script):
    
    1993
    +async def ghci_script( name, way, io_manager, script):
    
    1923 1994
         flags = ' '.join(get_compiler_flags())
    
    1924 1995
         way_flags = ' '.join(config.way_flags[way])
    
    1925 1996
     
    
    ... ... @@ -1930,9 +2001,9 @@ async def ghci_script( name, way, script):
    1930 2001
           # NB: put way_flags before flags so that flags in all.T can override others
    
    1931 2002
     
    
    1932 2003
         getTestOpts().stdin = script
    
    1933
    -    return await simple_run( name, way, cmd, getTestOpts().extra_run_opts )
    
    2004
    +    return await simple_run( name, way, io_manager, cmd, getTestOpts().extra_run_opts )
    
    1934 2005
     
    
    1935
    -async def ghci_multiunit_script( name, way, units, script):
    
    2006
    +async def ghci_multiunit_script( name, way, io_manager, units, script):
    
    1936 2007
         flags = ' '.join(get_compiler_flags())
    
    1937 2008
         way_flags = ' '.join(config.way_flags[way])
    
    1938 2009
         unit_flags = ' '.join(['-unit @%s' % unit for unit in units])
    
    ... ... @@ -1944,7 +2015,7 @@ async def ghci_multiunit_script( name, way, units, script):
    1944 2015
           # NB: put way_flags before flags so that flags in all.T can override others
    
    1945 2016
     
    
    1946 2017
         getTestOpts().stdin = script
    
    1947
    -    return await simple_run( name, way, cmd, getTestOpts().extra_run_opts )
    
    2018
    +    return await simple_run( name, way, io_manager, cmd, getTestOpts().extra_run_opts )
    
    1948 2019
     
    
    1949 2020
     # -----------------------------------------------------------------------------
    
    1950 2021
     # Compile-only tests
    
    ... ... @@ -2124,6 +2195,7 @@ async def compile_grep_core(name: TestName,
    2124 2195
     
    
    2125 2196
     async def compile_and_run__(name: TestName,
    
    2126 2197
                           way: WayName,
    
    2198
    +                      io_manager: None | str,
    
    2127 2199
                           top_mod: Path | None,
    
    2128 2200
                           extra_mods: List[str],
    
    2129 2201
                           extra_hc_opts: str,
    
    ... ... @@ -2166,22 +2238,22 @@ async def compile_and_run__(name: TestName,
    2166 2238
             cmd = './' + name + extension
    
    2167 2239
     
    
    2168 2240
             # we don't check the compiler's stderr for a compile-and-run test
    
    2169
    -        return await simple_run( name, way, cmd, getTestOpts().extra_run_opts )
    
    2241
    +        return await simple_run( name, way, io_manager, cmd, getTestOpts().extra_run_opts )
    
    2170 2242
     
    
    2171
    -async def compile_and_run( name, way, extra_hc_opts ):
    
    2172
    -    return await compile_and_run__( name, way, None, [], extra_hc_opts)
    
    2243
    +async def compile_and_run( name, way, io_manager, extra_hc_opts ):
    
    2244
    +    return await compile_and_run__( name, way, io_manager, None, [], extra_hc_opts)
    
    2173 2245
     
    
    2174
    -async def backpack_run( name, way, extra_hc_opts ):
    
    2175
    -    return await compile_and_run__( name, way, None, [], extra_hc_opts, backpack=True )
    
    2246
    +async def backpack_run( name, way, io_manager, extra_hc_opts ):
    
    2247
    +    return await compile_and_run__( name, way, io_manager, None, [], extra_hc_opts, backpack=True )
    
    2176 2248
     
    
    2177
    -async def multimod_compile_and_run( name, way, top_mod, extra_hc_opts ):
    
    2178
    -    return await compile_and_run__( name, way, top_mod, [], extra_hc_opts)
    
    2249
    +async def multimod_compile_and_run( name, way, io_manager, top_mod, extra_hc_opts ):
    
    2250
    +    return await compile_and_run__( name, way, io_manager, top_mod, [], extra_hc_opts)
    
    2179 2251
     
    
    2180
    -async def multi_compile_and_run( name, way, top_mod, extra_mods, extra_hc_opts ):
    
    2181
    -    return await compile_and_run__( name, way, top_mod, extra_mods, extra_hc_opts)
    
    2252
    +async def multi_compile_and_run( name, way, io_manager, top_mod, extra_mods, extra_hc_opts ):
    
    2253
    +    return await compile_and_run__( name, way, io_manager, top_mod, extra_mods, extra_hc_opts)
    
    2182 2254
     
    
    2183
    -async def warn_and_run( name, way, extra_hc_opts ):
    
    2184
    -    return await compile_and_run__( name, way, None, [], extra_hc_opts, compile_stderr = True)
    
    2255
    +async def warn_and_run( name, way, io_manager, extra_hc_opts ):
    
    2256
    +    return await compile_and_run__( name, way, io_manager, None, [], extra_hc_opts, compile_stderr = True)
    
    2185 2257
     
    
    2186 2258
     async def static_stats( name, way ):
    
    2187 2259
         opts = getTestOpts()
    
    ... ... @@ -2388,7 +2460,7 @@ async def simple_build(name: Union[TestName, str],
    2388 2460
     # from /dev/null.  Route output to testname.run.stdout and
    
    2389 2461
     # testname.run.stderr.  Returns the exit code of the run.
    
    2390 2462
     
    
    2391
    -async def simple_run(name: TestName, way: WayName, prog: str, extra_run_opts: str) -> Any:
    
    2463
    +async def simple_run(name: TestName, way: WayName, io_manager: None | str, prog: str, extra_run_opts: str) -> Any:
    
    2392 2464
         opts = getTestOpts()
    
    2393 2465
     
    
    2394 2466
         # figure out what to use for stdin
    
    ... ... @@ -2407,6 +2479,9 @@ async def simple_run(name: TestName, way: WayName, prog: str, extra_run_opts: st
    2407 2479
     
    
    2408 2480
         my_rts_flags = rts_flags(way)
    
    2409 2481
     
    
    2482
    +    if io_manager:
    
    2483
    +        my_rts_flags += '+RTS --io-manager={0} -RTS'.format(io_manager)
    
    2484
    +
    
    2410 2485
         # Collect runtime stats if necessary:
    
    2411 2486
         # isStatsTest and not isCompilerStatsTest():
    
    2412 2487
         #   assume we are running a ghc compiled program. Collect stats.
    
    ... ... @@ -2432,6 +2507,12 @@ async def simple_run(name: TestName, way: WayName, prog: str, extra_run_opts: st
    2432 2507
     
    
    2433 2508
         output_file = perfStatsFile(False, name)
    
    2434 2509
     
    
    2510
    +    # Check that IO Manager is available
    
    2511
    +    if io_manager:
    
    2512
    +        available_io_managers = discover_io_managers(prog, opts.testdir)
    
    2513
    +        if io_manager not in available_io_managers:
    
    2514
    +            framework_fail(name, way, 'IO Manager {} unexpectedly unavailable (available IO Managers: {}). `-non-threaded`/`-threaded` is only supported through way RTS flags (not manually added via test modifiers/args) when testing over IO Managers.'.format(io_manager, available_io_managers))
    
    2515
    +
    
    2435 2516
         # run the command
    
    2436 2517
         exit_code = await runCmdPerf(opts.compiler_perf_counters, cmd, output_file,
    
    2437 2518
             working_dir=opts.testdir,
    

  • testsuite/driver/testutil.py
    1 1
     import os
    
    2
    +import re
    
    2 3
     import subprocess
    
    3 4
     import shutil
    
    4 5
     import tempfile
    
    ... ... @@ -179,3 +180,21 @@ def shorten_metric_name(name: str) -> str:
    179 180
             "binary_size/if_compression(3)": "size/3",
    
    180 181
         }
    
    181 182
         return dic.get(name, name)
    
    183
    +
    
    184
    +# I/O manager discovery for a haskell binary. Returns the set of IO Manager
    
    185
    +# strings supported by the binary
    
    186
    +def discover_io_managers(hs_exe: str | Path, cwd: str | Path) -> Set[str]:
    
    187
    +    rp = subprocess.run(
    
    188
    +        [str(hs_exe), '+RTS', '--io-manager=not-a-real-io-manager', '-RTS'],
    
    189
    +        cwd=cwd,
    
    190
    +        stdout=subprocess.DEVNULL,
    
    191
    +        stderr=subprocess.PIPE)
    
    192
    +    stderr = rp.stderr.decode('utf-8', errors='replace')
    
    193
    +
    
    194
    +    m = re.search(r'choices are:\s*(.*)', stderr)
    
    195
    +    if not m:
    
    196
    +        raise Exception("Expected stderr output of form 'choices are: ...' but got: {}".format(stderr))
    
    197
    +
    
    198
    +    # 'auto' is a meta-choice (let the RTS pick a default for us), not
    
    199
    +    # an I/O manager implementation in its own right.
    
    200
    +    return set(m.group(1).split()) - {'auto'}

  • testsuite/tests/rts/all.T
    ... ... @@ -661,8 +661,8 @@ test('T23142', [unless(debug_rts(), skip), req_interp], makefile_test, ['T23142'
    661 661
     
    
    662 662
     test('T23400', [], compile_and_run, ['-with-rtsopts -A8k'])
    
    663 663
     
    
    664
    -test('IOManager', [js_skip, when(arch('wasm32'), skip), when(opsys('mingw32'), skip),
    
    665
    -                   pre_cmd('$MAKE -s --no-print-directory IOManager.hs')],
    
    664
    +test('IOManager', [js_skip, when(arch('wasm32'), skip),
    
    665
    +                   pre_cmd('$MAKE -s --no-print-directory IOManager.hs'), all_io_managers_except('winio')],
    
    666 666
                       compile_and_run, [''])
    
    667 667
     
    
    668 668
     test('T24142', [req_target_smp], compile_and_run, ['-threaded -with-rtsopts "-N2"'])