GitLab

Simon Jakobi pushed to branch wip/sjakobi/T16720 at Glasgow Haskell Compiler / GHC

Commits:

1d2672c0

by Simon Jakobi at 2026-06-16T19:31:45+02:00

testsuite: Reorder and bound the repeated failure output

Address review feedback on !16166:

  - Print the repeated failure output first, so the TEST= list, counts and
    failure lists stay at the very end of the log where readers look for them.
  - Skip the output entirely above MAX_SUMMARY_OUTPUT_TESTS unexpected
    failures, where the dump would drown out the summary and junit.xml is the
    better tool.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

e538a384

by Simon Jakobi at 2026-06-16T19:31:45+02:00

DO NOT MERGE: Break some tests to exercise the failure summary

Deliberately break three tests, one per failure mode, to see the new
end-of-run failure output in CI:

* cgrun001: wrong expected stdout  -> bad stdout (diff in summary)
* T18619:   wrong expected stderr  -> bad stderr (diff in summary)
* cgrun002: program now exits 3 and writes to stderr
            -> bad exit code, exercising the newly captured
               run stdout/stderr in simple_run

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

5 changed files:

testsuite/driver/runtests.py
testsuite/driver/testlib.py
testsuite/tests/codeGen/should_run/cgrun001.stdout
testsuite/tests/codeGen/should_run/cgrun002.hs
testsuite/tests/numeric/should_run/T18619.stderr

Changes:

testsuite/driver/runtests.py

@@ -591,7 +591,7 @@ else:
          print(Perf.allow_changes_string([(m.change, m.stat) for m in t.metrics]))
          print('-' * 25)
 -    summary(t, sys.stdout, color=term_color.enable_color)
 +    summary(t, sys.stdout, color=term_color.enable_color, junit_path=args.junit)
      # Write perf stats if any exist or if a metrics file is specified.
      stats_metrics = [stat for (_, stat, __) in t.metrics] # type: List[PerfStat]

testsuite/driver/testlib.py

@@ -3527,9 +3527,22 @@ def findTFiles(roots: List[str]) -> Iterator[str]:
  # -----------------------------------------------------------------------------
  # Output a test summary to the specified file object
 -def summary(t: TestRun, file: TextIO, color=False) -> None:
 +def summary(t: TestRun, file: TextIO, color=False, junit_path: Optional[Path]=None) -> None:
      file.write('\n')
++
 +    if t.unexpected_failures:
 +        if len(t.unexpected_failures) > MAX_SUMMARY_OUTPUT_TESTS:
 +            # junit.xml only exists when a path was requested (e.g. in CI); don't
 +            # point at a file that a local run never wrote.
 +            where = 'see {}'.format(junit_path) if junit_path \
 +                else 'rerun them individually'
 +            file.write('Output of {} unexpected failures omitted (limit {}); {}.\n\n'
 +                       .format(len(t.unexpected_failures), MAX_SUMMARY_OUTPUT_TESTS, where))
 +        else:
 +            file.write('Output of unexpected failures:\n\n')
 +            printTestOutputSummary(file, t.unexpected_failures, color, junit_path)
++
      printUnexpectedTests(file,
          [t.unexpected_passes, t.unexpected_failures,
           t.unexpected_stat_failures, t.framework_failures])
@@ -3596,10 +3609,6 @@ def summary(t: TestRun, file: TextIO, color=False) -> None:
          file.write('Framework warnings:\n')
          printTestInfosSummary(file, t.framework_warnings)
 -    if t.unexpected_failures:
 -        file.write('Output of unexpected failures:\n\n')
 -        printTestOutputSummary(file, t.unexpected_failures, color)
+-
      if stopping():
          file.write('WARNING: Testsuite run was terminated early\n')
@@ -3616,9 +3625,15 @@ def printUnexpectedTests(file: TextIO, testInfoss):
  # Per-stream cap on a failing test's output repeated in the final summary.
  MAX_SUMMARY_OUTPUT_LINES = 100
 -def printTestOutputSummary(file: TextIO, testInfos, color: bool=False) -> None:
 +# Above this many unexpected failures, skip repeating output entirely: the
 +# dump would drown out the summary.
 +MAX_SUMMARY_OUTPUT_TESTS = 20
++
 +def printTestOutputSummary(file: TextIO, testInfos, color: bool=False,
 +                           junit_path: Optional[Path]=None) -> None:
      # Repeat failing tests' captured output in the summary, so one needn't
      # hunt for it earlier in a possibly very long log; see #16720.
 +    where = ', see {}'.format(junit_path) if junit_path else ''
      for result in sorted(testInfos, key=lambda r: (r.testname.lower(), r.way, r.directory)):
          header = '=====> {}({}) [{}]'.format(result.testname, result.way, result.reason)
          if color:
@@ -3633,7 +3648,7 @@ def printTestOutputSummary(file: TextIO, testInfos, color: bool=False) -> None:
                  if len(lines) > MAX_SUMMARY_OUTPUT_LINES:
                      omitted = len(lines) - MAX_SUMMARY_OUTPUT_LINES
                      lines = lines[:MAX_SUMMARY_OUTPUT_LINES] \
 -                        + ['... ({} more lines omitted, see junit.xml)'.format(omitted)]
 +                        + ['... ({} more lines omitted{})'.format(omitted, where)]
                  s = label + '\n' + ''.join(l + '\n' for l in lines)
                  # Test output can contain characters that file's encoding
                  # cannot represent; replace rather than crash (cf safe_print).

testsuite/tests/codeGen/should_run/cgrun001.stdout

1		--42
	1	+-43

testsuite/tests/codeGen/should_run/cgrun002.hs

 -main = print ((f id2) (10 + thirty_two))
 +import System.Exit
 +import System.IO
++
 +main = do
 +    print ((f id2) (10 + thirty_two))
 +    hPutStrLn stderr "deliberate breakage to test summary output"
 +    exitWith (ExitFailure 3)
    where
      f x = g x
        where

testsuite/tests/numeric/should_run/T18619.stderr

  T18619: Uncaught exception ghc-internal:GHC.Internal.Exception.Type.ArithException:
 -arithmetic overflow
 +arithmetic underflow