Simon Jakobi pushed to branch wip/sjakobi/T18922-filecheck-poc at Glasgow Haskell Compiler / GHC Commits: 67cf5041 by Simon Jakobi at 2026-11-05T03:30:40+01:00 testsuite: Label FileCheck reports in the failure summary The summary printed every mismatch text under "Output diff (expected vs actual)", which misdescribes a FileCheck report. CompareOutput, PassFail and TestResult now carry an optional heading for the text, which run_filecheck sets. Context: #18922 Assisted-by: Claude Fable 5.1 - - - - - b8c38aa7 by Simon Jakobi at 2026-11-05T03:31:54+01:00 testsuite: Show the input around a FileCheck failure The vendored tool's report quotes the failing pattern and one input line, at a position that is often wrong (0-based, stale after CHECK-LABEL). Append the normalised input from where the failing check started looking, numbered, as a stand-in for LLVM FileCheck's annotated input dump. The anchor comes from the matcher's cursor rather than its line counter, so it stays right where the printed position is not. Also print the check file path relative to the working directory, like the rest of the summary. Context: #18922 Assisted-by: Claude Fable 5.1 - - - - - 3 changed files: - testsuite/driver/testglobals.py - testsuite/driver/testlib.py - testsuite/driver/testutil.py Changes: ===================================== testsuite/driver/testglobals.py ===================================== @@ -291,7 +291,7 @@ class TestResult: expected_failures, framework_failures, framework_warnings, unexpected_passes, unexpected_failures, unexpected_stat_failures lists of TestRun. """ - __slots__ = 'directory', 'testname', 'reason', 'way', 'stdout', 'stderr', 'diff', 'runtime' + __slots__ = 'directory', 'testname', 'reason', 'way', 'stdout', 'stderr', 'diff', 'diff_label', 'runtime' def __init__(self, directory: str, testname: TestName, @@ -300,6 +300,7 @@ class TestResult: stdout: Optional[str]=None, stderr: Optional[str]=None, diff: Optional[str]=None, + diff_label: Optional[str]=None, runtime: Optional[float]=None) -> None: self.directory = directory self.testname = testname @@ -308,6 +309,8 @@ class TestResult: self.stdout = stdout self.stderr = stderr self.diff = diff + # Heading for diff in reports, None for the default one + self.diff_label = diff_label self.runtime = runtime # Walltime # A performance metric measured in this test run. ===================================== testsuite/driver/testlib.py ===================================== @@ -1861,6 +1861,7 @@ async def do_test(name: TestName, stdout=result.stdout, stderr=result.stderr, diff=result.diff, + diff_label=result.diff_label, runtime=runtime)) elif result.passed: if _expect_pass(way): @@ -1884,6 +1885,7 @@ async def do_test(name: TestName, stdout=result.stdout, stderr=result.stderr, diff=result.diff, + diff_label=result.diff_label, runtime=runtime) t.unexpected_failures.append(tr) else: @@ -2070,7 +2072,8 @@ async def do_compile(name: TestName, "whitespace_normaliser", normalise_whitespace)) if not stderr_match: - return failBecause('stderr mismatch', diff=stderr_match.diff) + return failBecause('stderr mismatch', diff=stderr_match.diff, + diff_label=stderr_match.diff_label) opts = getTestOpts() if isGenericStatsTest(): @@ -2491,13 +2494,15 @@ async def simple_run(name: TestName, way: WayName, prog: str, extra_run_opts: st return failBecause('bad stderr', stderr=None if stderr_match.diff else read_stderr(name), stdout=read_stdout(name), - diff=stderr_match.diff) + diff=stderr_match.diff, + diff_label=stderr_match.diff_label) stdout_match = CompareOutput(True) if opts.ignore_stdout else await stdout_ok(name, way) if not stdout_match: return failBecause('bad stdout', stderr=read_stderr(name), stdout=None if stdout_match.diff else read_stdout(name), - diff=stdout_match.diff) + diff=stdout_match.diff, + diff_label=stdout_match.diff_label) check_hp = '-hT' in my_rts_flags and opts.check_hp check_prof = '-p' in my_rts_flags @@ -2610,7 +2615,8 @@ async def interpreter_run(name: TestName, return failBecause('bad stderr', stderr=None if stderr_match.diff else read_stderr(name), stdout=read_stdout(name), - diff=stderr_match.diff) + diff=stderr_match.diff, + diff_label=stderr_match.diff_label) stdout_match = CompareOutput(True) if opts.ignore_stdout else await stdout_ok(name, way) if not stdout_match: if _expect_pass(way): @@ -2618,7 +2624,8 @@ async def interpreter_run(name: TestName, return failBecause('bad stdout', stderr=read_stderr(name), stdout=None if stdout_match.diff else read_stdout(name), - diff=stdout_match.diff) + diff=stdout_match.diff, + diff_label=stdout_match.diff_label) return passed() def split_file(in_fn: Path, delimiter: str, out1_fn: Path, out2_fn: Path): @@ -2651,10 +2658,12 @@ def get_compiler_flags() -> List[str]: return flags class CompareOutput: - __slots__ = 'ok', 'diff' - def __init__(self, ok: bool, diff: Optional[str]=None) -> None: + __slots__ = 'ok', 'diff', 'diff_label' + def __init__(self, ok: bool, diff: Optional[str]=None, + diff_label: Optional[str]=None) -> None: self.ok = ok self.diff = diff + self.diff_label = diff_label def __bool__(self) -> bool: return self.ok @@ -2945,16 +2954,41 @@ def run_filecheck(way: WayName, # CQ[filecheck-platform-prefixes] # Q: WS32/WS64 are always enabled, mirroring the -ws-NN golden suffixes; # should OS/arch prefixes follow (cf. find_expected_file's platform chain)? - opts = Options(match_filename=str(in_srcdir(check_file)), + prefixes = prefixes + ['WS' + config.wordsize] + opts = Options(match_filename=os.path.relpath(in_srcdir(check_file)), input_file=str(actual_path), - check_prefixes=prefixes + ['WS' + config.wordsize]) + check_prefixes=prefixes) matcher = Matcher.from_opts(opts) matcher.stderr = io.StringIO() - ok = matcher.run() == 0 - report = matcher.stderr.getvalue() - if not ok and config.verbose >= 1 and _expect_pass(way): + if matcher.run() == 0: + return CompareOutput(True) + report = matcher.stderr.getvalue() + filecheck_input_window(matcher, prefixes) + if config.verbose >= 1 and _expect_pass(way): print(report) - return CompareOutput(ok, None if ok else report) + return CompareOutput(False, report, 'FileCheck report:') + +# The input from where the failing check started looking, numbered like the +# tool's positions; a substitute for LLVM FileCheck's annotated input dump. +def filecheck_input_window(matcher, prefixes: List[str], max_lines: int=40) -> str: + content = matcher.file.content + lines = content.split('\n') + if content.endswith('\n'): + lines.pop() + # CQ[filecheck-window-anchor] + # Q: a failing CHECK-NOT raises from inside the deferred check, leaving + # negative_matches_start set; is relying on that too fragile? + start = matcher.ctx.negative_matches_start + if start is None: + start = matcher.file.range.start + first = content.count('\n', 0, start) + shown = lines[first:first + max_lines] + out = ['Input from line {} of {}, checked with prefixes {}:'.format( + first + 1, len(lines), ', '.join(prefixes))] + out += ['{:4} | {}'.format(first + i + 1, l) for i, l in enumerate(shown)] + omitted = len(lines) - first - len(shown) + if omitted > 0: + out.append('... ({} more lines)'.format(omitted)) + return '\n'.join(out) + '\n' # Checks that each line from pattern_file is present in actual_file as # a substring or regex pattern depending on is_substring. @@ -3814,7 +3848,8 @@ def printTestOutputSummary(file: TextIO, result.testname, ', '.join(ways), result.directory + os.sep, result.reason) file.write(colored_if(color, Color.RED, header) + '\n') # See Note [Redundant output in test results] for why these don't overlap. - for label, contents in [('Output diff (expected vs actual):', diff), + diff_label = result.diff_label or 'Output diff (expected vs actual):' + for label, contents in [(diff_label, diff), ('Captured stdout:', result.stdout), ('Captured stderr:', result.stderr)]: if contents and contents.strip(): ===================================== testsuite/driver/testutil.py ===================================== @@ -15,6 +15,7 @@ PassFail = NamedTuple('PassFail', ('stderr', Optional[str]), ('stdout', Optional[str]), ('diff', Optional[str]), + ('diff_label', Optional[str]), ('hc_opts', Optional[str]), ]) @@ -28,16 +29,19 @@ def passed(hc_opts=None) -> PassFail: stderr=None, stdout=None, diff=None, + diff_label=None, hc_opts=hc_opts) def failBecause(reason: str, tag: Optional[str]=None, stderr: Optional[str]=None, stdout: Optional[str]=None, - diff: Optional[str]=None + diff: Optional[str]=None, + diff_label: Optional[str]=None ) -> PassFail: return PassFail(passed=False, reason=reason, tag=tag, - stderr=stderr, stdout=stdout, diff=diff, hc_opts=None) + stderr=stderr, stdout=stdout, diff=diff, + diff_label=diff_label, hc_opts=None) def strip_quotes(s: str) -> str: # Don't wrap commands to subprocess.call/Popen in quotes. View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/8270c6da49460ddebbe5505beb6ebb8... -- View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/8270c6da49460ddebbe5505beb6ebb8... You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications: https://gitlab.haskell.org/-/profile/notifications | Help: https://gitlab.haskell.org/help