David Eichmann pushed to branch wip/davide/io_manager_tests at Glasgow Haskell Compiler / GHC

Commits:

7 changed files:

Changes:

  • rts/RtsUtils.c
    ... ... @@ -397,6 +397,7 @@ void printRtsInfo(const RtsConfig rts_config) {
    397 397
             rts_config.rts_opts != NULL ? rts_config.rts_opts : "");
    
    398 398
         selectIOManager(); /* resolve the io-manager, accounting for flags  */
    
    399 399
         mkRtsInfoPair("I/O manager default",     showIOManager());
    
    400
    +    mkRtsInfoPair("I/O managers",            IOMGRS_ENABLED_STR + 1);
    
    400 401
         printf(" ]\n");
    
    401 402
     }
    
    402 403
     
    

  • testsuite/config/ghc
    ... ... @@ -257,7 +257,9 @@ def get_compiler_info():
    257 257
                         )
    
    258 258
     
    
    259 259
                 except Exception as err:
    
    260
    -                print("Exception thrown in testsuite/config/ghc.get_compiler_info: %s", err)
    
    260
    +                # We don't throw so that we can still discover tests before we've built
    
    261
    +                # GHC (i.e. config.compiler doesn't yet exist).
    
    262
    +                print("Exception thrown in testsuite/config/ghc.get_compiler_info: ", err)
    
    261 263
     
    
    262 264
                 else:
    
    263 265
                     res = p.returncode == 0
    
    ... ... @@ -315,6 +317,61 @@ def get_compiler_info():
    315 317
             config.ghc_th_way       = "normal"
    
    316 318
             config.ghc_th_hi_ext    = "hi"
    
    317 319
     
    
    320
    +    # I/O manager discovery.
    
    321
    +    #
    
    322
    +    # Which I/O managers (GHC's `--io-manager` RTS flag) a given RTS build
    
    323
    +    # supports depends on the target platform and on whether the RTS is the
    
    324
    +    # threaded or non-threaded flavour (see rts/IOManager.h's
    
    325
    +    # IOMGR_ENABLED_* macros). Rather than duplicate that platform/way
    
    326
    +    # dependent logic here, we ask the RTS itself: giving it a bogus
    
    327
    +    # '--io-manager=' choice makes it print, on stderr, the list of I/O
    
    328
    +    # managers it was actually built to support (see IOManager.h's
    
    329
    +    # IOMGRS_ENABLED_STR, used when reporting the error in RtsFlags.c),
    
    330
    +    # which we then parse out.
    
    331
    +    def discover_io_managers_generic(extra_flags):
    
    332
    +        import tempfile
    
    333
    +
    
    334
    +        test_src = 'main :: IO ()\nmain = return ()\n'
    
    335
    +
    
    336
    +        res = [None]
    
    337
    +
    
    338
    +        with tempfile.TemporaryDirectory() as d:
    
    339
    +            src = Path(d) / 'IOMgrProbe.hs'
    
    340
    +            src.write_text(test_src)
    
    341
    +            exe = Path(d) / ('IoMgrProbe' + config.exeext)
    
    342
    +
    
    343
    +            # -rtsopts is needed so that the resulting binary will even
    
    344
    +            # consider an --io-manager= flag (rather than immediately
    
    345
    +            # rejecting all "unsafe" RTS options).
    
    346
    +            cmd = '{} -v0 -rtsopts -o {} {} {}'.format(
    
    347
    +                                    config.compiler,
    
    348
    +                                    exe,
    
    349
    +                                    src,
    
    350
    +                                    ' '.join(extra_flags))
    
    351
    +            try:
    
    352
    +                cp = subprocess.run(
    
    353
    +                    cmd,
    
    354
    +                    shell=True,
    
    355
    +                    cwd=d,
    
    356
    +                    stdout=subprocess.DEVNULL,
    
    357
    +                    stderr=None if config.verbose >= 3 else subprocess.DEVNULL)
    
    358
    +                if cp.returncode != 0 or not exe.exists():
    
    359
    +                    raise Exception("Error in testsuite/config/ghc.discover_io_managers: failed to compile with: {}".format(cmd))
    
    360
    +                res = list(discover_io_managers(exe, d))
    
    361
    +
    
    362
    +            except Exception as err:
    
    363
    +                # We don't throw so that we can still discover tests before we've built
    
    364
    +                # GHC (i.e. config.compiler doesn't yet exist).
    
    365
    +                print("Exception thrown in testsuite/config/ghc.discover_io_managers_generic: ", err)
    
    366
    +
    
    367
    +        return res
    
    368
    +
    
    369
    +    config.io_managers_nonthreaded = discover_io_managers_generic([])
    
    370
    +    config.io_managers_threaded = (
    
    371
    +        discover_io_managers_generic(['-threaded']) if config.ghc_with_threaded_rts
    
    372
    +        else [])
    
    373
    +    config.io_managers_all = list(set(config.io_managers_nonthreaded).union(set(config.io_managers_threaded)))
    
    374
    +
    
    318 375
         config.cmm_cpp_is_gcc = test_compile(
    
    319 376
             ['-c'],
    
    320 377
             test_filename='test.cmm',
    

  • testsuite/driver/runtests.py
    ... ... @@ -28,7 +28,7 @@ import subprocess
    28 28
     
    
    29 29
     import asyncio
    
    30 30
     
    
    31
    -from testutil import getStdout, str_warn, str_info, print_table, shorten_metric_name, str_removeprefix
    
    31
    +from testutil import discover_io_managers, getStdout, str_warn, str_info, print_table, shorten_metric_name, str_removeprefix
    
    32 32
     from testglobals import getConfig, ghc_env, TestConfig, t, \
    
    33 33
                             TestOptions, brokens, PerfMetric
    
    34 34
     from my_typing import TestName
    

  • testsuite/driver/testglobals.py
    ... ... @@ -126,6 +126,15 @@ class TestConfig:
    126 126
             self.way_flags = {}  # type: Dict[WayName, List[str]]
    
    127 127
             self.way_rts_flags = {}  # type: Dict[WayName, List[str]]
    
    128 128
     
    
    129
    +        # I/O managers (GHC's `--io-manager` RTS flag) supported by the threaded runtime.
    
    130
    +        self.io_managers_threaded = []  # type: List[str]
    
    131
    +
    
    132
    +        # I/O managers (GHC's `--io-manager` RTS flag) supported by the non-threaded runtime.
    
    133
    +        self.io_managers_nonthreaded = []  # type: List[str]
    
    134
    +
    
    135
    +        # All I/O managers (GHC's `--io-manager` RTS flag)
    
    136
    +        self.io_managers_all = []  # type: List[str]
    
    137
    +
    
    129 138
             # Do we have a functional LLVM toolchain?
    
    130 139
             self.have_llvm = False
    
    131 140
     
    
    ... ... @@ -366,6 +375,14 @@ class TestOptions:
    366 375
            # add these ways to the default set
    
    367 376
            self.extra_ways = [] # type: List[WayName]
    
    368 377
     
    
    378
    +       # IO Managers to test (passed as the --io-manager RTS option)
    
    379
    +       # * [None] - only default IO Manager
    
    380
    +       # * [...] - list of io managers e.g. "mio" or "poll"
    
    381
    +       self.io_managers = [None] # type: List[Optional[str]]
    
    382
    +
    
    383
    +       # IO Manager used in the current test case. Set before each test case. None means default IO Manager
    
    384
    +       self.io_manager_current = None # type: Optional[str]
    
    385
    +
    
    369 386
            # the result we normally expect for this test
    
    370 387
            self.expect = 'pass'
    
    371 388
     
    

  • testsuite/driver/testlib.py
    ... ... @@ -23,7 +23,7 @@ import subprocess
    23 23
     
    
    24 24
     from testglobals import config, ghc_env, default_testopts, brokens, t, \
    
    25 25
                             TestRun, TestResult, TestOptions, PerfMetric
    
    26
    -from testutil import strip_quotes, lndir, link_or_copy_file, passed, \
    
    26
    +from testutil import discover_io_managers, strip_quotes, lndir, link_or_copy_file, passed, \
    
    27 27
                          failBecause, residency_testing_metrics, \
    
    28 28
                          stable_perf_counters, \
    
    29 29
                          PassFail, badResult, str_warn, str_removeprefix
    
    ... ... @@ -569,6 +569,34 @@ def extra_ways( ways: List[WayName] ):
    569 569
     
    
    570 570
         return helper
    
    571 571
     
    
    572
    +# -----
    
    573
    +
    
    574
    +def _io_managers( name: TestName, opts, io_mgrs: List[str] ) -> None:
    
    575
    +    opts.io_managers = io_mgrs
    
    576
    +
    
    577
    +def io_managers( io_mgrs: List[str] ):
    
    578
    +    """
    
    579
    +    Further split test cases into the given IO Managers if available.
    
    580
    +    Available IO Managers are determined by the test case's way (threaded/non-threads) and OS.
    
    581
    +    You'll likely want to use `extra_ways(['threaded2'])` if you want to test threaded IO Managers (e.g. mio).
    
    582
    +    """
    
    583
    +    return lambda name, opts: _io_managers(name, opts, io_mgrs)
    
    584
    +
    
    585
    +def all_available_io_managers( name: TestName, opts ):
    
    586
    +    """
    
    587
    +    Further split test cases into all available IO Managers.
    
    588
    +    Available IO Managers are determined by the test case's way (threaded/non-threads) and OS.
    
    589
    +    You'll likely want to use `extra_ways(['threaded2'])` to ensure threaded IO Managers are tested (e.g. mio).
    
    590
    +    """
    
    591
    +    _io_managers(name, opts, config.io_managers_all)
    
    592
    +
    
    593
    +def all_available_io_managers_except(exclude: List[str]):
    
    594
    +    """
    
    595
    +    Further split test cases into all available (except the given excluded) IO Managers.
    
    596
    +    Available IO Managers are determined by the test case's way (threaded/non-threads) and OS.
    
    597
    +    You'll likely want to use `extra_ways(['threaded2'])` to ensure threaded IO Managers are tested (e.g. mio).
    
    598
    +    """
    
    599
    +    return lambda name, opts: _io_managers(name, opts, list(set(config.io_managers_all) - set(exclude)))
    
    572 600
     
    
    573 601
     # -----
    
    574 602
     
    
    ... ... @@ -1571,6 +1599,15 @@ do_not_copy = ('.hi', '.o', '.dyn_hi'
    1571 1599
                   , '.dyn_o', '.out'
    
    1572 1600
                   ,'.hi-boot', '.o-boot') # 12112
    
    1573 1601
     
    
    1602
    +def way_supports_io_manager(way: WayName, io_manager: Optional[str]) -> bool:
    
    1603
    +    # All ways support a default IO Manager
    
    1604
    +    if io_manager is None:
    
    1605
    +        return True
    
    1606
    +
    
    1607
    +    way_is_threaded = '-threaded' in config.way_flags[way]
    
    1608
    +    available_io_managers = config.io_managers_threaded if way_is_threaded else config.io_managers_nonthreaded
    
    1609
    +    return io_manager in available_io_managers
    
    1610
    +
    
    1574 1611
     async def test_common_work(name: TestName, opts,
    
    1575 1612
                          func, args) -> None:
    
    1576 1613
         try:
    
    ... ... @@ -1619,7 +1656,22 @@ async def test_common_work(name: TestName, opts,
    1619 1656
             # A test itself can request extra ways by setting opts.extra_ways
    
    1620 1657
             all_ways = list(OrderedDict.fromkeys(all_ways + [way for way in opts.extra_ways if way not in all_ways]))
    
    1621 1658
     
    
    1622
    -        t.total_test_cases += len(all_ways)
    
    1659
    +        # We split each way over available io managers
    
    1660
    +        if len(opts.io_managers) == 0:
    
    1661
    +            framework_fail(name, None, 'opts.io_managers is empty')
    
    1662
    +        all_ways_and_io_managers = dict(
    
    1663
    +            (
    
    1664
    +                way,
    
    1665
    +                [io_manager
    
    1666
    +                    for io_manager in opts.io_managers
    
    1667
    +                    if way_supports_io_manager(way, io_manager)
    
    1668
    +                ]
    
    1669
    +            )
    
    1670
    +            for way in all_ways
    
    1671
    +        )
    
    1672
    +
    
    1673
    +        for io_managers in all_ways_and_io_managers.values():
    
    1674
    +            t.total_test_cases += len(io_managers)
    
    1623 1675
     
    
    1624 1676
             only_ways = getTestOpts().only_ways
    
    1625 1677
             ok_way = lambda way: \
    
    ... ... @@ -1702,17 +1754,26 @@ async def test_common_work(name: TestName, opts,
    1702 1754
     
    
    1703 1755
             # Run the required tests...
    
    1704 1756
             for way in do_ways:
    
    1705
    -            if stopping():
    
    1706
    -                break
    
    1707
    -            try:
    
    1708
    -                await do_test(name, way, func, args, files)
    
    1709
    -            except KeyboardInterrupt:
    
    1710
    -                stopNow()
    
    1711
    -            except Exception as e:
    
    1712
    -                traceback.print_exc()
    
    1713
    -                framework_fail(name, way, traceback.format_exc())
    
    1757
    +            for io_manager in all_ways_and_io_managers[way]:
    
    1758
    +                if stopping():
    
    1759
    +                    break
    
    1760
    +                try:
    
    1761
    +                    getTestOpts().io_manager_current = io_manager
    
    1762
    +                    await do_test(
    
    1763
    +                        name,
    
    1764
    +                        way,
    
    1765
    +                        func,
    
    1766
    +                        args,
    
    1767
    +                        files)
    
    1768
    +                except KeyboardInterrupt:
    
    1769
    +                    stopNow()
    
    1770
    +                except Exception as e:
    
    1771
    +                    traceback.print_exc()
    
    1772
    +                    framework_fail(name, way, traceback.format_exc())
    
    1773
    +
    
    1774
    +        for skipped_way in set(all_ways) - set(do_ways):
    
    1775
    +            t.n_tests_skipped += len(all_ways_and_io_managers[skipped_way])
    
    1714 1776
     
    
    1715
    -        t.n_tests_skipped += len(set(all_ways) - set(do_ways))
    
    1716 1777
             if getTestOpts().expect == 'missing-lib': t.n_missing_libs += 1
    
    1717 1778
     
    
    1718 1779
             if config.cleanup and do_ways:
    
    ... ... @@ -1738,6 +1799,8 @@ async def do_test(name: TestName,
    1738 1799
         opts = getTestOpts()
    
    1739 1800
     
    
    1740 1801
         full_name = name + '(' + way + ')'
    
    1802
    +    if opts.io_manager_current is not None:
    
    1803
    +        full_name += ' (IO Manager: {})'.format(opts.io_manager_current)
    
    1741 1804
         test_n = len(allTestNames)
    
    1742 1805
         progress_args = [ full_name, t.total_tests, test_n,
    
    1743 1806
             [len(t.unexpected_passes),
    
    ... ... @@ -1907,7 +1970,7 @@ def framework_warn(name: TestName, way: WayName, reason: str) -> None:
    1907 1970
     # run_command.
    
    1908 1971
     
    
    1909 1972
     async def run_command( name, way, cmd ):
    
    1910
    -    return await simple_run( name, WayName(''), override_options(cmd), '' )
    
    1973
    +    return await simple_run( name, WayName(''), None, override_options(cmd), '' )
    
    1911 1974
     
    
    1912 1975
     async def makefile_test( name, way, target=None ):
    
    1913 1976
         if target is None:
    
    ... ... @@ -2124,7 +2187,7 @@ async def compile_grep_core(name: TestName,
    2124 2187
     
    
    2125 2188
     async def compile_and_run__(name: TestName,
    
    2126 2189
                           way: WayName,
    
    2127
    -                      top_mod: Path | None,
    
    2190
    +                      top_mod: Optional[Path],
    
    2128 2191
                           extra_mods: List[str],
    
    2129 2192
                           extra_hc_opts: str,
    
    2130 2193
                           backpack: bool=False,
    
    ... ... @@ -2407,6 +2470,9 @@ async def simple_run(name: TestName, way: WayName, prog: str, extra_run_opts: st
    2407 2470
     
    
    2408 2471
         my_rts_flags = rts_flags(way)
    
    2409 2472
     
    
    2473
    +    if opts.io_manager_current is not None:
    
    2474
    +        my_rts_flags += ' +RTS --io-manager={0} -RTS '.format(opts.io_manager_current)
    
    2475
    +
    
    2410 2476
         # Collect runtime stats if necessary:
    
    2411 2477
         # isStatsTest and not isCompilerStatsTest():
    
    2412 2478
         #   assume we are running a ghc compiled program. Collect stats.
    
    ... ... @@ -2432,6 +2498,12 @@ async def simple_run(name: TestName, way: WayName, prog: str, extra_run_opts: st
    2432 2498
     
    
    2433 2499
         output_file = perfStatsFile(False, name)
    
    2434 2500
     
    
    2501
    +    # Check that IO Manager is available
    
    2502
    +    if opts.io_manager_current is not None:
    
    2503
    +        available_io_managers = discover_io_managers(prog, opts.testdir)
    
    2504
    +        if opts.io_manager_current not in available_io_managers:
    
    2505
    +            framework_fail(name, way, 'IO Manager {} unexpectedly unavailable (available IO Managers: {}). `-non-threaded`/`-threaded` is only supported through way RTS flags (not manually added via test modifiers/args) when testing over IO Managers.'.format(io_manager, available_io_managers))
    
    2506
    +
    
    2435 2507
         # run the command
    
    2436 2508
         exit_code = await runCmdPerf(opts.compiler_perf_counters, cmd, output_file,
    
    2437 2509
             working_dir=opts.testdir,
    
    ... ... @@ -2493,7 +2565,7 @@ def rts_flags(way: WayName) -> str:
    2493 2565
     async def interpreter_run(name: TestName,
    
    2494 2566
                         way: WayName,
    
    2495 2567
                         extra_hc_opts: str,
    
    2496
    -                    top_mod: Path | None
    
    2568
    +                    top_mod: Optional[Path]
    
    2497 2569
                         ) -> PassFail:
    
    2498 2570
         opts = getTestOpts()
    
    2499 2571
     
    

  • testsuite/driver/testutil.py
    1 1
     import os
    
    2
    +import re
    
    2 3
     import subprocess
    
    3 4
     import shutil
    
    4 5
     import tempfile
    
    ... ... @@ -179,3 +180,19 @@ def shorten_metric_name(name: str) -> str:
    179 180
             "binary_size/if_compression(3)": "size/3",
    
    180 181
         }
    
    181 182
         return dic.get(name, name)
    
    183
    +
    
    184
    +# I/O manager discovery for a haskell binary. Returns the set of IO Manager
    
    185
    +# strings supported by the binary
    
    186
    +def discover_io_managers(hs_exe: Union[str, Path], cwd: Union[str, Path]) -> Set[str]:
    
    187
    +    rp = subprocess.run(
    
    188
    +        [str(hs_exe), '+RTS', '--info', '-RTS'],
    
    189
    +        cwd=cwd,
    
    190
    +        stdout=subprocess.PIPE,
    
    191
    +        stderr=subprocess.DEVNULL)
    
    192
    +    stdout = rp.stdout.decode('utf-8', errors='replace')
    
    193
    +
    
    194
    +    m = re.search(r'\("I/O managers", "(.*)"\)', stdout)
    
    195
    +    if not m:
    
    196
    +        raise Exception('Expected stdout output of form \'... ("I/O managers", "select selectbis poll") ...\' but got:\n{}'.format(stdout))
    
    197
    +
    
    198
    +    return set(m.group(1).split())

  • testsuite/tests/rts/all.T
    ... ... @@ -662,8 +662,9 @@ test('T23142', [unless(debug_rts(), skip), req_interp], makefile_test, ['T23142'
    662 662
     test('T23400', [], compile_and_run, ['-with-rtsopts -A8k'])
    
    663 663
     
    
    664 664
     test('IOManager', [js_skip, when(arch('wasm32'), skip), when(opsys('mingw32'), skip),
    
    665
    -                   pre_cmd('$MAKE -s --no-print-directory IOManager.hs')],
    
    666
    -                  compile_and_run, [''])
    
    665
    +                   pre_cmd('$MAKE -s --no-print-directory IOManager.hs'),
    
    666
    +                   extra_ways(['threaded2']), all_available_io_managers_except('winio')],
    
    667
    +                   compile_and_run, [''])
    
    667 668
     
    
    668 669
     test('T24142', [req_target_smp], compile_and_run, ['-threaded -with-rtsopts "-N2"'])
    
    669 670