GitLab

Simon Jakobi pushed to branch wip/sjakobi/T16720 at Glasgow Haskell Compiler / GHC

Commits:

b4e9e1ab

by Simon Jakobi at 2026-06-17T13:15:40+02:00

testsuite: Don't list unexpected failures twice in the summary

Previously a failing test could appear in three places in the summary:
its repeated output (=====> header), a separate 'Unexpected failures:'
directory list, and the concise TEST= rerun line. Collapse this so each
failing case is shown once, and place the repeated output where readers
look for it:

  - When the output is shown (at most MAX_SUMMARY_OUTPUT_TESTS failures),
    the =====> header is the only per-test listing; the test's directory
    is folded into it and the separate directory list is dropped.

  - When the output is omitted (too many failures), the failures are
    listed in the directory list instead.

The repeated failure output and the TEST= rerun line are printed directly
above the SUMMARY counts: the verbose output blends with the run's inline
output, and TEST= sits right where the counts begin. The TEST= line always
includes the failures, so it is shown even when their output is.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

3979543d

by Simon Jakobi at 2026-06-17T13:15:55+02:00

testsuite: Bundle identical failure output across ways

A test that fails identically in several ways (e.g. normal and g1) now
shares a single output block in the summary, with the ways collected in
the header, rather than repeating the same captured output once per way.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

16d64293

by Simon Jakobi at 2026-06-17T13:34:58+02:00

testsuite: Show a stable source-relative directory in failure output

The failure-output summary derived the test's directory from opts.testdir,
which is relative to wherever make was invoked, so e.g. a test under
codeGen/should_run would show as should_run/... when make ran from codeGen/.

Record the directory as the test's source directory relative to the GHC
source root instead, so it reads the same regardless of the make working
directory and works for out-of-tree suites like libraries/base/tests. Drop
the now-redundant run-dir leaf (it just repeated the test name) and the
obsolete tempdir-stripping that the old /tmp-prefixed paths needed.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

1 changed file:

testsuite/driver/testlib.py

Changes:

testsuite/driver/testlib.py

@@ -1493,6 +1493,14 @@ def _newTestDir(name: TestName, opts: TestOptions, tempdir, dir):
      opts.testdir_raw = Path(os.path.join(tempdir, testdir, name + testdir_suffix))
      opts.compiler_always_flags = config.compiler_always_flags
 +def _result_directory(opts: TestOptions) -> str:
 +    # The test's source directory, relative to the GHC source root, so it reads
 +    # the same regardless of which directory `make` was invoked from.
 +    try:
 +        return os.path.relpath(opts.srcdir, config.top.parent)
 +    except Exception:
 +        return ''
++
  # -----------------------------------------------------------------------------
  # Actually doing tests
@@ -1817,7 +1825,7 @@ async def do_test(name: TestName,
      if opts.expect not in ['pass', 'fail', 'missing-lib']:
          framework_fail(name, way, 'bad expected ' + opts.expect)
 -    directory = str_removeprefix(str_removeprefix(str(opts.testdir), './'), '.\\')
 +    directory = _result_directory(opts)
      if way in opts.fragile_ways:
          if_verbose(1, '*** fragile test %s resulted in %s' % (full_name, 'pass' if result.passed else 'fail'))
@@ -1868,7 +1876,7 @@ def framework_fail(name: Optional[TestName], way: Optional[WayName], reason: str
      # so we need to take care not to blow up with the wrong way
      # and report the actual reason for the failure.
      try:
 -      directory = str_removeprefix(str_removeprefix(str(opts.testdir), './'), '.\\')
 +      directory = _result_directory(opts)
      except:
        directory = ''
      full_name = '%s(%s)' % (name, way)
@@ -1881,7 +1889,7 @@ def framework_fail(name: Optional[TestName], way: Optional[WayName], reason: str
  def framework_warn(name: TestName, way: WayName, reason: str) -> None:
      opts = getTestOpts()
 -    directory = str_removeprefix(str_removeprefix(str(opts.testdir), './'), '.\\')
 +    directory = _result_directory(opts)
      full_name = name + '(' + way + ')'
      if_verbose(1, '*** framework warning for %s %s ' % (full_name, reason))
      t.framework_warnings.append(TestResult(directory, name, reason, way))
@@ -3531,17 +3539,17 @@ def summary(t: TestRun, file: TextIO, color=False, junit_path: Optional[Path]=No
      file.write('\n')
 +    too_many_failures = len(t.unexpected_failures) > MAX_SUMMARY_OUTPUT_TESTS
++
      if t.unexpected_failures:
 -        if len(t.unexpected_failures) > MAX_SUMMARY_OUTPUT_TESTS:
 -            # junit.xml only exists when a path was requested (e.g. in CI); don't
 -            # point at a file that a local run never wrote.
 -            where = 'see {}'.format(junit_path) if junit_path \
 -                else 'rerun them individually'
 -            file.write('Output of {} unexpected failures omitted (limit {}); {}.\n\n'
 -                       .format(len(t.unexpected_failures), MAX_SUMMARY_OUTPUT_TESTS, where))
 -        else:
 +        if not too_many_failures:
              file.write('Output of unexpected failures:\n\n')
              printTestOutputSummary(file, t.unexpected_failures, color, junit_path)
 +        else:
 +            where = '; see {}'.format(junit_path) if junit_path else ''
 +            file.write('Unexpected failures (output omitted, more than {}{}):\n'
 +                       .format(MAX_SUMMARY_OUTPUT_TESTS, where))
 +            printTestInfosSummary(file, t.unexpected_failures)
      printUnexpectedTests(file,
          [t.unexpected_passes, t.unexpected_failures,
@@ -3593,10 +3601,6 @@ def summary(t: TestRun, file: TextIO, color=False, junit_path: Optional[Path]=No
          file.write('Unexpected passes:\n')
          printTestInfosSummary(file, t.unexpected_passes)
 -    if t.unexpected_failures:
 -        file.write('Unexpected failures:\n')
 -        printTestInfosSummary(file, t.unexpected_failures)
+-
      if t.unexpected_stat_failures:
          file.write('Unexpected stat failures:\n')
          printTestInfosSummary(file, t.unexpected_stat_failures)
@@ -3634,8 +3638,15 @@ def printTestOutputSummary(file: TextIO, testInfos, color: bool=False,
      # Repeat failing tests' captured output in the summary, so one needn't
      # hunt for it earlier in a possibly very long log; see #16720.
      where = ', see {}'.format(junit_path) if junit_path else ''
 -    for result in sorted(testInfos, key=lambda r: (r.testname.lower(), r.way, r.directory)):
 -        header = '=====> {}({}) [{}]'.format(result.testname, result.way, result.reason)
 +    # Tests that fail identically in several ways (e.g. normal and g1) share one
 +    # output block, with the ways collected in the header.
 +    groups = collections.OrderedDict() # type: ignore
 +    for result in sorted(testInfos, key=lambda r: (r.testname.lower(), r.directory, r.way)):
 +        key = (result.testname, result.directory, result.reason, result.stdout, result.stderr)
 +        groups.setdefault(key, (result, []))[1].append(result.way)
 +    for result, ways in groups.values():
 +        header = '=====> {}({}) ({}) [{}]'.format(
 +            result.testname, ', '.join(ways), result.directory + os.sep, result.reason)
          if color:
              header = colored(Color.RED, header)
          file.write(header + '\n')
@@ -3654,19 +3665,15 @@ def printTestOutputSummary(file: TextIO, testInfos, color: bool=False,
                  # cannot represent; replace rather than crash (cf safe_print).
                  enc = getattr(file, 'encoding', None) or 'utf-8'
                  file.write(s.encode(enc, errors='replace').decode(enc))
 -        file.write('\n')
      footer = '<===== end of output of unexpected failures'
      if color:
          footer = colored(Color.RED, footer)
      file.write(footer + '\n\n')
  def printTestInfosSummary(file: TextIO, testInfos):
 -    maxDirLen = max(len(tr.directory) for tr in testInfos)
      for result in sorted(testInfos, key=lambda r: (r.testname.lower(), r.way, r.directory)):
 -        directory = result.directory.ljust(maxDirLen)
 -        file.write('   {directory}  {r.testname} [{r.reason}] ({r.way})\n'.format(
 -            r = result,
 -            directory = directory))
 +        path = os.path.join(result.directory, result.testname)
 +        file.write('   {path} [{r.reason}] ({r.way})\n'.format(r=result, path=path))
      file.write('\n')
  def modify_lines(s: str, f: Callable[[str], str]) -> str: