Simon Jakobi pushed to branch wip/sjakobi/T16720 at Glasgow Haskell Compiler / GHC Commits: b4e9e1ab by Simon Jakobi at 2026-06-17T13:15:40+02:00 testsuite: Don't list unexpected failures twice in the summary Previously a failing test could appear in three places in the summary: its repeated output (=====> header), a separate 'Unexpected failures:' directory list, and the concise TEST= rerun line. Collapse this so each failing case is shown once, and place the repeated output where readers look for it: - When the output is shown (at most MAX_SUMMARY_OUTPUT_TESTS failures), the =====> header is the only per-test listing; the test's directory is folded into it and the separate directory list is dropped. - When the output is omitted (too many failures), the failures are listed in the directory list instead. The repeated failure output and the TEST= rerun line are printed directly above the SUMMARY counts: the verbose output blends with the run's inline output, and TEST= sits right where the counts begin. The TEST= line always includes the failures, so it is shown even when their output is. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> - - - - - 3979543d by Simon Jakobi at 2026-06-17T13:15:55+02:00 testsuite: Bundle identical failure output across ways A test that fails identically in several ways (e.g. normal and g1) now shares a single output block in the summary, with the ways collected in the header, rather than repeating the same captured output once per way. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> - - - - - 16d64293 by Simon Jakobi at 2026-06-17T13:34:58+02:00 testsuite: Show a stable source-relative directory in failure output The failure-output summary derived the test's directory from opts.testdir, which is relative to wherever make was invoked, so e.g. a test under codeGen/should_run would show as should_run/... when make ran from codeGen/. Record the directory as the test's source directory relative to the GHC source root instead, so it reads the same regardless of the make working directory and works for out-of-tree suites like libraries/base/tests. Drop the now-redundant run-dir leaf (it just repeated the test name) and the obsolete tempdir-stripping that the old /tmp-prefixed paths needed. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> - - - - - 1 changed file: - testsuite/driver/testlib.py Changes: ===================================== testsuite/driver/testlib.py ===================================== @@ -1493,6 +1493,14 @@ def _newTestDir(name: TestName, opts: TestOptions, tempdir, dir): opts.testdir_raw = Path(os.path.join(tempdir, testdir, name + testdir_suffix)) opts.compiler_always_flags = config.compiler_always_flags +def _result_directory(opts: TestOptions) -> str: + # The test's source directory, relative to the GHC source root, so it reads + # the same regardless of which directory `make` was invoked from. + try: + return os.path.relpath(opts.srcdir, config.top.parent) + except Exception: + return '' + # ----------------------------------------------------------------------------- # Actually doing tests @@ -1817,7 +1825,7 @@ async def do_test(name: TestName, if opts.expect not in ['pass', 'fail', 'missing-lib']: framework_fail(name, way, 'bad expected ' + opts.expect) - directory = str_removeprefix(str_removeprefix(str(opts.testdir), './'), '.\\') + directory = _result_directory(opts) if way in opts.fragile_ways: if_verbose(1, '*** fragile test %s resulted in %s' % (full_name, 'pass' if result.passed else 'fail')) @@ -1868,7 +1876,7 @@ def framework_fail(name: Optional[TestName], way: Optional[WayName], reason: str # so we need to take care not to blow up with the wrong way # and report the actual reason for the failure. try: - directory = str_removeprefix(str_removeprefix(str(opts.testdir), './'), '.\\') + directory = _result_directory(opts) except: directory = '' full_name = '%s(%s)' % (name, way) @@ -1881,7 +1889,7 @@ def framework_fail(name: Optional[TestName], way: Optional[WayName], reason: str def framework_warn(name: TestName, way: WayName, reason: str) -> None: opts = getTestOpts() - directory = str_removeprefix(str_removeprefix(str(opts.testdir), './'), '.\\') + directory = _result_directory(opts) full_name = name + '(' + way + ')' if_verbose(1, '*** framework warning for %s %s ' % (full_name, reason)) t.framework_warnings.append(TestResult(directory, name, reason, way)) @@ -3531,17 +3539,17 @@ def summary(t: TestRun, file: TextIO, color=False, junit_path: Optional[Path]=No file.write('\n') + too_many_failures = len(t.unexpected_failures) > MAX_SUMMARY_OUTPUT_TESTS + if t.unexpected_failures: - if len(t.unexpected_failures) > MAX_SUMMARY_OUTPUT_TESTS: - # junit.xml only exists when a path was requested (e.g. in CI); don't - # point at a file that a local run never wrote. - where = 'see {}'.format(junit_path) if junit_path \ - else 'rerun them individually' - file.write('Output of {} unexpected failures omitted (limit {}); {}.\n\n' - .format(len(t.unexpected_failures), MAX_SUMMARY_OUTPUT_TESTS, where)) - else: + if not too_many_failures: file.write('Output of unexpected failures:\n\n') printTestOutputSummary(file, t.unexpected_failures, color, junit_path) + else: + where = '; see {}'.format(junit_path) if junit_path else '' + file.write('Unexpected failures (output omitted, more than {}{}):\n' + .format(MAX_SUMMARY_OUTPUT_TESTS, where)) + printTestInfosSummary(file, t.unexpected_failures) printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures, @@ -3593,10 +3601,6 @@ def summary(t: TestRun, file: TextIO, color=False, junit_path: Optional[Path]=No file.write('Unexpected passes:\n') printTestInfosSummary(file, t.unexpected_passes) - if t.unexpected_failures: - file.write('Unexpected failures:\n') - printTestInfosSummary(file, t.unexpected_failures) - if t.unexpected_stat_failures: file.write('Unexpected stat failures:\n') printTestInfosSummary(file, t.unexpected_stat_failures) @@ -3634,8 +3638,15 @@ def printTestOutputSummary(file: TextIO, testInfos, color: bool=False, # Repeat failing tests' captured output in the summary, so one needn't # hunt for it earlier in a possibly very long log; see #16720. where = ', see {}'.format(junit_path) if junit_path else '' - for result in sorted(testInfos, key=lambda r: (r.testname.lower(), r.way, r.directory)): - header = '=====> {}({}) [{}]'.format(result.testname, result.way, result.reason) + # Tests that fail identically in several ways (e.g. normal and g1) share one + # output block, with the ways collected in the header. + groups = collections.OrderedDict() # type: ignore + for result in sorted(testInfos, key=lambda r: (r.testname.lower(), r.directory, r.way)): + key = (result.testname, result.directory, result.reason, result.stdout, result.stderr) + groups.setdefault(key, (result, []))[1].append(result.way) + for result, ways in groups.values(): + header = '=====> {}({}) ({}) [{}]'.format( + result.testname, ', '.join(ways), result.directory + os.sep, result.reason) if color: header = colored(Color.RED, header) file.write(header + '\n') @@ -3654,19 +3665,15 @@ def printTestOutputSummary(file: TextIO, testInfos, color: bool=False, # cannot represent; replace rather than crash (cf safe_print). enc = getattr(file, 'encoding', None) or 'utf-8' file.write(s.encode(enc, errors='replace').decode(enc)) - file.write('\n') footer = '<===== end of output of unexpected failures' if color: footer = colored(Color.RED, footer) file.write(footer + '\n\n') def printTestInfosSummary(file: TextIO, testInfos): - maxDirLen = max(len(tr.directory) for tr in testInfos) for result in sorted(testInfos, key=lambda r: (r.testname.lower(), r.way, r.directory)): - directory = result.directory.ljust(maxDirLen) - file.write(' {directory} {r.testname} [{r.reason}] ({r.way})\n'.format( - r = result, - directory = directory)) + path = os.path.join(result.directory, result.testname) + file.write(' {path} [{r.reason}] ({r.way})\n'.format(r=result, path=path)) file.write('\n') def modify_lines(s: str, f: Callable[[str], str]) -> str: View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/e538a38453c42403de99feb70206ae9... -- View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/e538a38453c42403de99feb70206ae9... You're receiving this email because of your account on gitlab.haskell.org.