Marge Bot pushed to branch master at Glasgow Haskell Compiler / GHC Commits: 984ee82b by Andreas Klebinger at 2026-10-01T06:58:10-04:00 Fix MultiLayerModules timeout multiplier. We accidentally used the compile multiplier for ghci commands which use the run timeout setting to set the timeout. Fixes #26263 and #27796. - - - - - 50a9cf26 by Andreas Klebinger at 2026-10-01T06:58:10-04:00 testsuite: Include runtime in JUnit report. - - - - - 4 changed files: - testsuite/driver/junit.py - testsuite/driver/testglobals.py - testsuite/driver/testlib.py - testsuite/tests/perf/compiler/all.T Changes: ===================================== testsuite/driver/junit.py ===================================== @@ -2,10 +2,33 @@ from my_typing import * from datetime import datetime import xml.etree.ElementTree as ET -from testglobals import TestRun +from testglobals import TestRun, TestResult + + +def add_testcase(testsuite: ET.Element, test_result: TestResult) -> ET.Element: + """Adds the xml element representing the testcase.""" + testcase = ET.SubElement(testsuite, 'testcase', + classname = test_result.way, + name = '%s(%s)' % (test_result.testname, test_result.way)) + if test_result.runtime is not None: + testcase.set('time', '%.3f' % test_result.runtime) + return testcase + +def add_result(testcase: ET.Element, kind: str, res_type: str, + reason: Optional[str], text: str) -> ET.Element: + """Add a result to the testcase.""" + # GitLab ignores the message attribute and shows only the text. + result = ET.SubElement(testcase, kind, + type = res_type, + message = reason or '') + result.text = text + return result def junit(t: TestRun) -> ET.ElementTree: - testsuites = ET.Element('testsuites') + # Report time spent in testsuite. + assert t.start_time is not None + elapsed = '%.3f' % (datetime.now() - t.start_time).total_seconds() + testsuites = ET.Element('testsuites', time = elapsed) testsuite = ET.SubElement(testsuites, 'testsuite', id = "0", package = 'ghc', @@ -15,6 +38,7 @@ def junit(t: TestRun) -> ET.ElementTree: + len(t.unexpected_passes)), errors = str(len(t.framework_failures)), skipped = str(len(t.fragile_failures)), + time = elapsed, timestamp = datetime.now().isoformat()) for kind, res_type, group in [('failure', 'stat failure', t.unexpected_stat_failures), @@ -22,9 +46,7 @@ def junit(t: TestRun) -> ET.ElementTree: ('failure', 'unexpected pass', t.unexpected_passes), ('skipped', 'fragile failure', t.fragile_failures)]: for tr in group: - testcase = ET.SubElement(testsuite, 'testcase', - classname = tr.way, - name = '%s(%s)' % (tr.testname, tr.way)) + testcase = add_testcase(testsuite, tr) message = [] # type: List[str] if tr.diff: message += ['diff:', '==========', tr.diff] @@ -35,23 +57,16 @@ def junit(t: TestRun) -> ET.ElementTree: if not message: message = [tr.reason] - result = ET.SubElement(testcase, kind, - type = res_type, - message = tr.reason) - result.text = '\n'.join(message) + add_result(testcase, kind, res_type, tr.reason, '\n'.join(message)) for tr in t.framework_failures: - testcase = ET.SubElement(testsuite, 'testcase', - classname = tr.way, - name = '%s(%s)' % (tr.testname, tr.way)) - result = ET.SubElement(testcase, 'error', - type = "framework failure", - message = tr.reason) - - for tr in t.expected_passes: - testcase = ET.SubElement(testsuite, 'testcase', - classname = tr.way, - name = '%s(%s)' % (tr.testname, tr.way)) + testcase = add_testcase(testsuite, tr) + add_result(testcase, 'error', 'framework failure', tr.reason, tr.reason) + + # Expected failures (expect_broken) behaved as expected, so they count as + # passes, like in the driver's exit code. + for tr in t.expected_passes + t.expected_failures + t.fragile_passes: + add_testcase(testsuite, tr) return ET.ElementTree(testsuites) ===================================== testsuite/driver/testglobals.py ===================================== @@ -288,10 +288,10 @@ ghc_env = os.environ.copy() class TestResult: """ A result from the execution of a test. These live in the expected_passes, - framework_failures, framework_warnings, unexpected_passes, + expected_failures, framework_failures, framework_warnings, unexpected_passes, unexpected_failures, unexpected_stat_failures lists of TestRun. """ - __slots__ = 'directory', 'testname', 'reason', 'way', 'stdout', 'stderr', 'diff' + __slots__ = 'directory', 'testname', 'reason', 'way', 'stdout', 'stderr', 'diff', 'runtime' def __init__(self, directory: str, testname: TestName, @@ -299,7 +299,8 @@ class TestResult: way: WayName, stdout: Optional[str]=None, stderr: Optional[str]=None, - diff: Optional[str]=None) -> None: + diff: Optional[str]=None, + runtime: Optional[float]=None) -> None: self.directory = directory self.testname = testname self.reason = reason @@ -307,6 +308,7 @@ class TestResult: self.stdout = stdout self.stderr = stderr self.diff = diff + self.runtime = runtime # Walltime # A performance metric measured in this test run. PerfMetric = NamedTuple('PerfMetric', @@ -322,13 +324,12 @@ class TestRun: self.n_tests_skipped = 0 self.n_missing_libs = 0 - self.n_expected_passes = 0 - self.n_expected_failures = 0 self.framework_failures = [] # type: List[TestResult] self.framework_warnings = [] # type: List[TestResult] self.expected_passes = [] # type: List[TestResult] + self.expected_failures = [] # type: List[TestResult] self.unexpected_passes = [] # type: List[TestResult] self.unexpected_failures = [] # type: List[TestResult] self.unexpected_stat_failures = [] # type: List[TestResult] ===================================== testsuite/driver/testlib.py ===================================== @@ -1706,13 +1706,15 @@ async def test_common_work(name: TestName, opts, for way in do_ways: if stopping(): break + start = time.monotonic() try: await do_test(name, way, func, args, files) except KeyboardInterrupt: stopNow() except Exception as e: traceback.print_exc() - framework_fail(name, way, traceback.format_exc()) + framework_fail(name, way, traceback.format_exc(), + runtime=time.monotonic() - start) t.n_tests_skipped += len(set(all_ways) - set(do_ways)) if getTestOpts().expect == 'missing-lib': t.n_missing_libs += 1 @@ -1738,6 +1740,7 @@ async def do_test(name: TestName, files: Set[str] ) -> None: opts = getTestOpts() + start = time.monotonic() full_name = name + '(' + way + ')' test_n = len(allTestNames) @@ -1787,7 +1790,8 @@ async def do_test(name: TestName, pass else: framework_fail(name, way, - 'extra_file does not exist: ' + extra_file) + 'extra_file does not exist: ' + extra_file, + runtime=time.monotonic() - start) if func.__name__ == 'run_command' or func.__name__ == 'makefile_test' or opts.pre_cmd: # When running 'MAKE' make sure 'TOP' still points to the @@ -1811,7 +1815,8 @@ async def do_test(name: TestName, # If user used expect_broken then don't record failures of pre_cmd if exit_code != 0 and opts.expect not in ['fail']: - framework_fail(name, way, 'pre_cmd failed: {0}'.format(exit_code)) + framework_fail(name, way, 'pre_cmd failed: {0}'.format(exit_code), + runtime=time.monotonic() - start) if_verbose(1, '** pre_cmd was "{0}".'.format(override_options(opts.pre_cmd))) stderr_contents = stderr_path.read_text(encoding='UTF-8', errors='replace') stdout_contents = stdout_path.read_text(encoding='UTF-8', errors='replace') @@ -1821,45 +1826,54 @@ async def do_test(name: TestName, return result = await func(*[name,way] + args) + runtime = time.monotonic() - start if opts.expect not in ['pass', 'fail', 'missing-lib']: - framework_fail(name, way, 'bad expected ' + opts.expect) + framework_fail(name, way, 'bad expected ' + opts.expect, runtime=runtime) directory = _result_directory(opts) if way in opts.fragile_ways: if_verbose(1, '*** fragile test %s resulted in %s' % (full_name, 'pass' if result.passed else 'fail')) if result.passed: - t.fragile_passes.append(TestResult(directory, name, 'fragile', way)) + t.fragile_passes.append(TestResult(directory, name, 'fragile', way, + runtime=runtime)) else: reason = '%s (fragile)' % result.reason if result.reason else 'fragile' t.fragile_failures.append(TestResult(directory, name, reason, way, stdout=result.stdout, stderr=result.stderr, - diff=result.diff)) + diff=result.diff, + runtime=runtime)) elif result.passed: if _expect_pass(way): - t.expected_passes.append(TestResult(directory, name, "", way)) - t.n_expected_passes += 1 + t.expected_passes.append(TestResult(directory, name, "", way, + runtime=runtime)) else: if_verbose(1, '*** unexpected pass for %s' % full_name) - t.unexpected_passes.append(TestResult(directory, name, 'unexpected', way)) + t.unexpected_passes.append(TestResult(directory, name, 'unexpected', way, + runtime=runtime)) else: if _expect_pass(way): reason = result.reason tag = result.tag if tag == 'stat': if_verbose(1, '*** unexpected stat test failure for %s' % full_name) - t.unexpected_stat_failures.append(TestResult(directory, name, reason, way)) + t.unexpected_stat_failures.append(TestResult(directory, name, reason, way, + runtime=runtime)) else: if_verbose(1, '*** unexpected failure for %s' % full_name) tr = TestResult(directory, name, reason, way, stdout=result.stdout, stderr=result.stderr, - diff=result.diff) + diff=result.diff, + runtime=runtime) t.unexpected_failures.append(tr) else: - t.n_expected_failures += 1 + # Only keep the reason: nothing reports the output of expected + # failures. + t.expected_failures.append(TestResult(directory, name, result.reason, way, + runtime=runtime)) # Make is often invoked with -s, which means if it fails, we get # no feedback at all. This is annoying. So let's remove the option @@ -1873,7 +1887,8 @@ def override_options(pre_cmd): return pre_cmd -def framework_fail(name: Optional[TestName], way: Optional[WayName], reason: str) -> None: +def framework_fail(name: Optional[TestName], way: Optional[WayName], reason: str, + runtime: Optional[float]=None) -> None: opts = getTestOpts() # framework_fail can be called before testdir is initialised, # so we need to take care not to blow up with the wrong way @@ -1888,7 +1903,8 @@ def framework_fail(name: Optional[TestName], way: Optional[WayName], reason: str way2 = way if way is not None else WayName('none') if way not in opts.fragile_ways: # If the test is fragile then we rather report this as a fragile test failure - t.framework_failures.append(TestResult(directory, name2, reason, way2)) + t.framework_failures.append(TestResult(directory, name2, reason, way2, + runtime=runtime)) def framework_warn(name: TestName, way: WayName, reason: str) -> None: opts = getTestOpts() @@ -3649,9 +3665,9 @@ def summary(t: TestRun, file: TextIO, color=False, junit_path: Optional[Path]=No + repr(t.n_missing_libs).rjust(8) + ' had missing libraries\n' + '\n' - + repr(t.n_expected_passes).rjust(8) + + repr(len(t.expected_passes)).rjust(8) + ' expected passes\n' - + repr(t.n_expected_failures).rjust(8) + + repr(len(t.expected_failures)).rjust(8) + ' expected failures\n' + '\n' + repr(len(t.framework_failures)).rjust(8) ===================================== testsuite/tests/perf/compiler/all.T ===================================== @@ -363,15 +363,20 @@ test('T13379', # (though the size of the fluctuations does perhaps suggest that it # would be valuable to solve the pathology in a more general way). +# Note [MultiLayerModules* test timeout] +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# These tests are relatively sensitive to system load and cane +# take a while to run. This caused a fair number of CI failures +# in the past. So we give them some headroom. +# We need to ensure to use the right timeout for those test. In the past +# we used the compile timeout for the GHCi versions by accident. + test('MultiLayerModules', [ collect_compiler_stats('bytes allocated',2), pre_cmd('./genMultiLayerModules'), extra_files(['genMultiLayerModules']), + # See Note [MultiLayerModules* test timeout] compile_timeout_multiplier(5) - # this is _a lot_ - # but this test has been failing every now and then, - # especially on i386. Let's just give it some room - # to complete successfully reliably everywhere. ], multimod_compile, # see Note [Increased initial stack size for MultiLayerModules] @@ -382,11 +387,8 @@ test('MultiLayerModulesRecomp', [ collect_compiler_stats('bytes allocated',2), pre_cmd('$MAKE -s --no-print-directory MultiModulesRecomp'), extra_files(['genMultiLayerModules']), + # See Note [MultiLayerModules* test timeout] compile_timeout_multiplier(5) - # this is _a lot_ - # but this test has been failing every now and then, - # especially on i386. Let's just give it some room - # to complete successfully reliably everywhere. ], multimod_compile, # see Note [Increased initial stack size for MultiLayerModules] @@ -398,6 +400,7 @@ test('MultiLayerModulesTH_Make', [ collect_compiler_stats('bytes allocated',2), pre_cmd('$MAKE -s --no-print-directory MultiLayerModulesTH_Make_Prep'), extra_files(['genMultiLayerModulesTH']), + # See Note [MultiLayerModules* test timeout] compile_timeout_multiplier(5), when(arch('wasm32'), fragile(25336)), # We skip the test on darwin due to recent regression due to toolchain @@ -413,6 +416,7 @@ test('MultiLayerModulesTH_OneShot', [ collect_compiler_stats('bytes allocated',2), pre_cmd('$MAKE -s --no-print-directory MultiLayerModulesTH_OneShot_Prep'), extra_files(['genMultiLayerModulesTH']), + # See Note [MultiLayerModules* test timeout] compile_timeout_multiplier(5), when(arch('wasm32'), fragile(25336)), # We skip the test on darwin due to recent regression due to toolchain @@ -427,11 +431,8 @@ test('MultiLayerModulesDefsGhci', [ collect_compiler_residency(15), pre_cmd('./genMultiLayerModulesDefs'), extra_files(['genMultiLayerModulesDefs']), - compile_timeout_multiplier(5) - # this is _a lot_ - # but this test has been failing every now and then, - # especially on i386. Let's just give it some room - # to complete successfully reliably everywhere. + # See Note [MultiLayerModules* test timeout] + run_timeout_multiplier(5) ], ghci_script, ['MultiLayerModulesDefsGhci.script']) @@ -440,11 +441,9 @@ test('MultiLayerModulesDefsGhciWithCore', [ collect_compiler_residency(15), pre_cmd('$MAKE -s --no-print-directory MultiModulesRecompDefsWithCore'), extra_files(['genMultiLayerModulesCore', 'MultiLayerModulesDefsGhciWithCore.script']), - compile_timeout_multiplier(5), - # this is _a lot_ - # but this test has been failing every now and then, - # especially on i386. Let's just give it some room - # to complete successfully reliably everywhere. + # See Note [MultiLayerModules* test timeout] + pre_cmd_timeout_multiplier(5), + run_timeout_multiplier(5), extra_run_opts('-fwrite-if-simplified-core MultiLayerModules') ], ghci_script, @@ -454,11 +453,9 @@ test('MultiLayerModulesDefsGhciWithBytecodeFiles', [ collect_compiler_residency(15), pre_cmd('$MAKE -s --no-print-directory MultiModulesRecompDefsWithBytecodeFiles'), extra_files(['genMultiLayerModulesCore', 'MultiLayerModulesDefsGhciWithBytecodeFiles.script']), - compile_timeout_multiplier(5), - # this is _a lot_ - # but this test has been failing every now and then, - # especially on i386. Let's just give it some room - # to complete successfully reliably everywhere. + # See Note [MultiLayerModules* test timeout] + pre_cmd_timeout_multiplier(5), + run_timeout_multiplier(5), extra_run_opts('-fwrite-byte-code MultiLayerModules') ], ghci_script, @@ -468,11 +465,8 @@ test('MultiLayerModulesDefsGhcWithCore', [ collect_compiler_residency(15), pre_cmd('$MAKE -s --no-print-directory MultiModulesDefsWithCore'), extra_files(['genMultiLayerModulesCore']), + # See Note [MultiLayerModules* test timeout] compile_timeout_multiplier(5) - # this is _a lot_ - # but this test has been failing every now and then, - # especially on i386. Let's just give it some room - # to complete successfully reliably everywhere. ], multimod_compile, ['MultiLayerModules', '-v0 -fwrite-if-simplified-core']) @@ -481,11 +475,8 @@ test('MultiLayerModulesDefsGhciReload', [ collect_compiler_residency(15), pre_cmd('./genMultiLayerModulesDefs'), extra_files(['genMultiLayerModulesDefs']), - compile_timeout_multiplier(10) - # this is _a lot_ - # but this test has been failing every now and then, - # especially on i386. Let's just give it some room - # to complete successfully reliably everywhere. + # See Note [MultiLayerModules* test timeout] + run_timeout_multiplier(5) ], ghci_script, ['MultiLayerModulesDefsGhciReload.script']) @@ -504,11 +495,8 @@ test('InstanceMatching1', [ collect_compiler_stats('bytes allocated',3), pre_cmd('./genMatchingTest 1'), extra_files(['genMatchingTest']), + # See Note [MultiLayerModules* test timeout] compile_timeout_multiplier(5) - # this is _a lot_ - # but this test has been failing every now and then, - # especially on i386. Let's just give it some room - # to complete successfully reliably everywhere. ], multimod_compile, ['Defs', '-fno-code -fwrite-interface -v0']) @@ -518,11 +506,8 @@ test('MultiLayerModulesNoCode', [ collect_compiler_residency(15), pre_cmd('./genMultiLayerModulesNoCode'), extra_files(['genMultiLayerModulesNoCode']), - compile_timeout_multiplier(5) - # this is _a lot_ - # but this test has been failing every now and then, - # especially on i386. Let's just give it some room - # to complete successfully reliably everywhere. + # See Note [MultiLayerModules* test timeout] + run_timeout_multiplier(5) ], ghci_script, ['MultiLayerModulesNoCode.script']) View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/c406135cc3aa0379caa402e16e1a1ac... -- View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/c406135cc3aa0379caa402e16e1a1ac... You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications: https://gitlab.haskell.org/-/profile/notifications | Help: https://gitlab.haskell.org/help