GitLab

Simon Jakobi pushed to branch wip/sjakobi/T18922-filecheck-poc at Glasgow Haskell Compiler / GHC

Commits:

67cf5041

by Simon Jakobi at 2026-11-05T03:30:40+01:00

testsuite: Label FileCheck reports in the failure summary

The summary printed every mismatch text under "Output diff (expected vs
actual)", which misdescribes a FileCheck report. CompareOutput, PassFail
and TestResult now carry an optional heading for the text, which
run_filecheck sets.

Context: #18922

Assisted-by: Claude Fable 5.1

b8c38aa7

by Simon Jakobi at 2026-11-05T03:31:54+01:00

testsuite: Show the input around a FileCheck failure

The vendored tool's report quotes the failing pattern and one input
line, at a position that is often wrong (0-based, stale after
CHECK-LABEL). Append the normalised input from where the failing check
started looking, numbered, as a stand-in for LLVM FileCheck's annotated
input dump. The anchor comes from the matcher's cursor rather than its
line counter, so it stays right where the printed position is not.

Also print the check file path relative to the working directory, like
the rest of the summary.

Context: #18922

Assisted-by: Claude Fable 5.1

3 changed files:

testsuite/driver/testglobals.py
testsuite/driver/testlib.py
testsuite/driver/testutil.py

Changes:

testsuite/driver/testglobals.py

@@ -291,7 +291,7 @@ class TestResult:
      expected_failures, framework_failures, framework_warnings, unexpected_passes,
      unexpected_failures, unexpected_stat_failures lists of TestRun.
      """
 -    __slots__ = 'directory', 'testname', 'reason', 'way', 'stdout', 'stderr', 'diff', 'runtime'
 +    __slots__ = 'directory', 'testname', 'reason', 'way', 'stdout', 'stderr', 'diff', 'diff_label', 'runtime'
      def __init__(self,
                   directory: str,
                   testname: TestName,
@@ -300,6 +300,7 @@ class TestResult:
                   stdout: Optional[str]=None,
                   stderr: Optional[str]=None,
                   diff: Optional[str]=None,
 +                 diff_label: Optional[str]=None,
                   runtime: Optional[float]=None) -> None:
          self.directory = directory
          self.testname = testname
@@ -308,6 +309,8 @@ class TestResult:
          self.stdout = stdout
          self.stderr = stderr
          self.diff = diff
 +        # Heading for diff in reports, None for the default one
 +        self.diff_label = diff_label
          self.runtime = runtime # Walltime
  # A performance metric measured in this test run.

testsuite/driver/testlib.py

@@ -1861,6 +1861,7 @@ async def do_test(name: TestName,
                                                   stdout=result.stdout,
                                                   stderr=result.stderr,
                                                   diff=result.diff,
 +                                                 diff_label=result.diff_label,
                                                   runtime=runtime))
      elif result.passed:
          if _expect_pass(way):
@@ -1884,6 +1885,7 @@ async def do_test(name: TestName,
                                  stdout=result.stdout,
                                  stderr=result.stderr,
                                  diff=result.diff,
 +                                diff_label=result.diff_label,
                                  runtime=runtime)
                  t.unexpected_failures.append(tr)
          else:
@@ -2070,7 +2072,8 @@ async def do_compile(name: TestName,
                                                           "whitespace_normaliser",
                                                           normalise_whitespace))
          if not stderr_match:
 -            return failBecause('stderr mismatch', diff=stderr_match.diff)
 +            return failBecause('stderr mismatch', diff=stderr_match.diff,
 +                               diff_label=stderr_match.diff_label)
      opts = getTestOpts()
      if isGenericStatsTest():
@@ -2491,13 +2494,15 @@ async def simple_run(name: TestName, way: WayName, prog: str, extra_run_opts: st
          return failBecause('bad stderr',
                             stderr=None if stderr_match.diff else read_stderr(name),
                             stdout=read_stdout(name),
 -                           diff=stderr_match.diff)
 +                           diff=stderr_match.diff,
 +                           diff_label=stderr_match.diff_label)
      stdout_match = CompareOutput(True) if opts.ignore_stdout else await stdout_ok(name, way)
      if not stdout_match:
          return failBecause('bad stdout',
                             stderr=read_stderr(name),
                             stdout=None if stdout_match.diff else read_stdout(name),
 -                           diff=stdout_match.diff)
 +                           diff=stdout_match.diff,
 +                           diff_label=stdout_match.diff_label)
      check_hp = '-hT' in my_rts_flags and opts.check_hp
      check_prof = '-p' in my_rts_flags
@@ -2610,7 +2615,8 @@ async def interpreter_run(name: TestName,
          return failBecause('bad stderr',
                             stderr=None if stderr_match.diff else read_stderr(name),
                             stdout=read_stdout(name),
 -                           diff=stderr_match.diff)
 +                           diff=stderr_match.diff,
 +                           diff_label=stderr_match.diff_label)
      stdout_match = CompareOutput(True) if opts.ignore_stdout else await stdout_ok(name, way)
      if not stdout_match:
          if _expect_pass(way):
@@ -2618,7 +2624,8 @@ async def interpreter_run(name: TestName,
          return failBecause('bad stdout',
                             stderr=read_stderr(name),
                             stdout=None if stdout_match.diff else read_stdout(name),
 -                           diff=stdout_match.diff)
 +                           diff=stdout_match.diff,
 +                           diff_label=stdout_match.diff_label)
      return passed()
  def split_file(in_fn: Path, delimiter: str, out1_fn: Path, out2_fn: Path):
@@ -2651,10 +2658,12 @@ def get_compiler_flags() -> List[str]:
      return flags
  class CompareOutput:
 -    __slots__ = 'ok', 'diff'
 -    def __init__(self, ok: bool, diff: Optional[str]=None) -> None:
 +    __slots__ = 'ok', 'diff', 'diff_label'
 +    def __init__(self, ok: bool, diff: Optional[str]=None,
 +                 diff_label: Optional[str]=None) -> None:
          self.ok = ok
          self.diff = diff
 +        self.diff_label = diff_label
      def __bool__(self) -> bool:
          return self.ok
@@ -2945,16 +2954,41 @@ def run_filecheck(way: WayName,
      # CQ[filecheck-platform-prefixes]
      # Q: WS32/WS64 are always enabled, mirroring the -ws-NN golden suffixes;
      #    should OS/arch prefixes follow (cf. find_expected_file's platform chain)?
 -    opts = Options(match_filename=str(in_srcdir(check_file)),
 +    prefixes = prefixes + ['WS' + config.wordsize]
 +    opts = Options(match_filename=os.path.relpath(in_srcdir(check_file)),
                     input_file=str(actual_path),
 -                   check_prefixes=prefixes + ['WS' + config.wordsize])
 +                   check_prefixes=prefixes)
      matcher = Matcher.from_opts(opts)
      matcher.stderr = io.StringIO()
 -    ok = matcher.run() == 0
 -    report = matcher.stderr.getvalue()
 -    if not ok and config.verbose >= 1 and _expect_pass(way):
 +    if matcher.run() == 0:
 +        return CompareOutput(True)
 +    report = matcher.stderr.getvalue() + filecheck_input_window(matcher, prefixes)
 +    if config.verbose >= 1 and _expect_pass(way):
          print(report)
 -    return CompareOutput(ok, None if ok else report)
 +    return CompareOutput(False, report, 'FileCheck report:')
++
 +# The input from where the failing check started looking, numbered like the
 +# tool's positions; a substitute for LLVM FileCheck's annotated input dump.
 +def filecheck_input_window(matcher, prefixes: List[str], max_lines: int=40) -> str:
 +    content = matcher.file.content
 +    lines = content.split('\n')
 +    if content.endswith('\n'):
 +        lines.pop()
 +    # CQ[filecheck-window-anchor]
 +    # Q: a failing CHECK-NOT raises from inside the deferred check, leaving
 +    #    negative_matches_start set; is relying on that too fragile?
 +    start = matcher.ctx.negative_matches_start
 +    if start is None:
 +        start = matcher.file.range.start
 +    first = content.count('\n', 0, start)
 +    shown = lines[first:first + max_lines]
 +    out = ['Input from line {} of {}, checked with prefixes {}:'.format(
 +               first + 1, len(lines), ', '.join(prefixes))]
 +    out += ['{:4} | {}'.format(first + i + 1, l) for i, l in enumerate(shown)]
 +    omitted = len(lines) - first - len(shown)
 +    if omitted > 0:
 +        out.append('... ({} more lines)'.format(omitted))
 +    return '\n'.join(out) + '\n'
  # Checks that each line from pattern_file is present in actual_file as
  # a substring or regex pattern depending on is_substring.
@@ -3814,7 +3848,8 @@ def printTestOutputSummary(file: TextIO,
              result.testname, ', '.join(ways), result.directory + os.sep, result.reason)
          file.write(colored_if(color, Color.RED, header) + '\n')
          # See Note [Redundant output in test results] for why these don't overlap.
 -        for label, contents in [('Output diff (expected vs actual):', diff),
 +        diff_label = result.diff_label or 'Output diff (expected vs actual):'
 +        for label, contents in [(diff_label, diff),
                                  ('Captured stdout:', result.stdout),
                                  ('Captured stderr:', result.stderr)]:
              if contents and contents.strip():

testsuite/driver/testutil.py

@@ -15,6 +15,7 @@ PassFail = NamedTuple('PassFail',
                         ('stderr', Optional[str]),
                         ('stdout', Optional[str]),
                         ('diff', Optional[str]),
 +                       ('diff_label', Optional[str]),
                         ('hc_opts', Optional[str]),
                         ])
@@ -28,16 +29,19 @@ def passed(hc_opts=None) -> PassFail:
                      stderr=None,
                      stdout=None,
                      diff=None,
 +                    diff_label=None,
                      hc_opts=hc_opts)
  def failBecause(reason: str,
                  tag: Optional[str]=None,
                  stderr: Optional[str]=None,
                  stdout: Optional[str]=None,
 -                diff: Optional[str]=None
 +                diff: Optional[str]=None,
 +                diff_label: Optional[str]=None
                  ) -> PassFail:
      return PassFail(passed=False, reason=reason, tag=tag,
 -                    stderr=stderr, stdout=stdout, diff=diff, hc_opts=None)
 +                    stderr=stderr, stdout=stdout, diff=diff,
 +                    diff_label=diff_label, hc_opts=None)
  def strip_quotes(s: str) -> str:
      # Don't wrap commands to subprocess.call/Popen in quotes.