Simon Jakobi pushed to branch wip/sjakobi/T16720 at Glasgow Haskell Compiler / GHC Commits: 1d2672c0 by Simon Jakobi at 2026-06-16T19:31:45+02:00 testsuite: Reorder and bound the repeated failure output Address review feedback on !16166: - Print the repeated failure output first, so the TEST= list, counts and failure lists stay at the very end of the log where readers look for them. - Skip the output entirely above MAX_SUMMARY_OUTPUT_TESTS unexpected failures, where the dump would drown out the summary and junit.xml is the better tool. Co-Authored-By: Claude Opus 4.8 - - - - - e538a384 by Simon Jakobi at 2026-06-16T19:31:45+02:00 DO NOT MERGE: Break some tests to exercise the failure summary Deliberately break three tests, one per failure mode, to see the new end-of-run failure output in CI: * cgrun001: wrong expected stdout -> bad stdout (diff in summary) * T18619: wrong expected stderr -> bad stderr (diff in summary) * cgrun002: program now exits 3 and writes to stderr -> bad exit code, exercising the newly captured run stdout/stderr in simple_run Co-Authored-By: Claude Fable 5 - - - - - 5 changed files: - testsuite/driver/runtests.py - testsuite/driver/testlib.py - testsuite/tests/codeGen/should_run/cgrun001.stdout - testsuite/tests/codeGen/should_run/cgrun002.hs - testsuite/tests/numeric/should_run/T18619.stderr Changes: ===================================== testsuite/driver/runtests.py ===================================== @@ -591,7 +591,7 @@ else: print(Perf.allow_changes_string([(m.change, m.stat) for m in t.metrics])) print('-' * 25) - summary(t, sys.stdout, color=term_color.enable_color) + summary(t, sys.stdout, color=term_color.enable_color, junit_path=args.junit) # Write perf stats if any exist or if a metrics file is specified. stats_metrics = [stat for (_, stat, __) in t.metrics] # type: List[PerfStat] ===================================== testsuite/driver/testlib.py ===================================== @@ -3527,9 +3527,22 @@ def findTFiles(roots: List[str]) -> Iterator[str]: # ----------------------------------------------------------------------------- # Output a test summary to the specified file object -def summary(t: TestRun, file: TextIO, color=False) -> None: +def summary(t: TestRun, file: TextIO, color=False, junit_path: Optional[Path]=None) -> None: file.write('\n') + + if t.unexpected_failures: + if len(t.unexpected_failures) > MAX_SUMMARY_OUTPUT_TESTS: + # junit.xml only exists when a path was requested (e.g. in CI); don't + # point at a file that a local run never wrote. + where = 'see {}'.format(junit_path) if junit_path \ + else 'rerun them individually' + file.write('Output of {} unexpected failures omitted (limit {}); {}.\n\n' + .format(len(t.unexpected_failures), MAX_SUMMARY_OUTPUT_TESTS, where)) + else: + file.write('Output of unexpected failures:\n\n') + printTestOutputSummary(file, t.unexpected_failures, color, junit_path) + printUnexpectedTests(file, [t.unexpected_passes, t.unexpected_failures, t.unexpected_stat_failures, t.framework_failures]) @@ -3596,10 +3609,6 @@ def summary(t: TestRun, file: TextIO, color=False) -> None: file.write('Framework warnings:\n') printTestInfosSummary(file, t.framework_warnings) - if t.unexpected_failures: - file.write('Output of unexpected failures:\n\n') - printTestOutputSummary(file, t.unexpected_failures, color) - if stopping(): file.write('WARNING: Testsuite run was terminated early\n') @@ -3616,9 +3625,15 @@ def printUnexpectedTests(file: TextIO, testInfoss): # Per-stream cap on a failing test's output repeated in the final summary. MAX_SUMMARY_OUTPUT_LINES = 100 -def printTestOutputSummary(file: TextIO, testInfos, color: bool=False) -> None: +# Above this many unexpected failures, skip repeating output entirely: the +# dump would drown out the summary. +MAX_SUMMARY_OUTPUT_TESTS = 20 + +def printTestOutputSummary(file: TextIO, testInfos, color: bool=False, + junit_path: Optional[Path]=None) -> None: # Repeat failing tests' captured output in the summary, so one needn't # hunt for it earlier in a possibly very long log; see #16720. + where = ', see {}'.format(junit_path) if junit_path else '' for result in sorted(testInfos, key=lambda r: (r.testname.lower(), r.way, r.directory)): header = '=====> {}({}) [{}]'.format(result.testname, result.way, result.reason) if color: @@ -3633,7 +3648,7 @@ def printTestOutputSummary(file: TextIO, testInfos, color: bool=False) -> None: if len(lines) > MAX_SUMMARY_OUTPUT_LINES: omitted = len(lines) - MAX_SUMMARY_OUTPUT_LINES lines = lines[:MAX_SUMMARY_OUTPUT_LINES] \ - + ['... ({} more lines omitted, see junit.xml)'.format(omitted)] + + ['... ({} more lines omitted{})'.format(omitted, where)] s = label + '\n' + ''.join(l + '\n' for l in lines) # Test output can contain characters that file's encoding # cannot represent; replace rather than crash (cf safe_print). ===================================== testsuite/tests/codeGen/should_run/cgrun001.stdout ===================================== @@ -1 +1 @@ --42 +-43 ===================================== testsuite/tests/codeGen/should_run/cgrun002.hs ===================================== @@ -1,4 +1,10 @@ -main = print ((f id2) (10 + thirty_two)) +import System.Exit +import System.IO + +main = do + print ((f id2) (10 + thirty_two)) + hPutStrLn stderr "deliberate breakage to test summary output" + exitWith (ExitFailure 3) where f x = g x where ===================================== testsuite/tests/numeric/should_run/T18619.stderr ===================================== @@ -1,3 +1,3 @@ T18619: Uncaught exception ghc-internal:GHC.Internal.Exception.Type.ArithException: -arithmetic overflow +arithmetic underflow View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/18e8c673894f55b7eb06cd9ec2fc3f6... -- View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/18e8c673894f55b7eb06cd9ec2fc3f6... You're receiving this email because of your account on gitlab.haskell.org.