Simon Jakobi pushed to branch wip/sjakobi/T16720 at Glasgow Haskell Compiler / GHC
Commits:
1d2672c0 by Simon Jakobi at 2026-06-16T19:31:45+02:00
testsuite: Reorder and bound the repeated failure output
Address review feedback on !16166:
- Print the repeated failure output first, so the TEST= list, counts and
failure lists stay at the very end of the log where readers look for them.
- Skip the output entirely above MAX_SUMMARY_OUTPUT_TESTS unexpected
failures, where the dump would drown out the summary and junit.xml is the
better tool.
Co-Authored-By: Claude Opus 4.8
- - - - -
e538a384 by Simon Jakobi at 2026-06-16T19:31:45+02:00
DO NOT MERGE: Break some tests to exercise the failure summary
Deliberately break three tests, one per failure mode, to see the new
end-of-run failure output in CI:
* cgrun001: wrong expected stdout -> bad stdout (diff in summary)
* T18619: wrong expected stderr -> bad stderr (diff in summary)
* cgrun002: program now exits 3 and writes to stderr
-> bad exit code, exercising the newly captured
run stdout/stderr in simple_run
Co-Authored-By: Claude Fable 5
- - - - -
5 changed files:
- testsuite/driver/runtests.py
- testsuite/driver/testlib.py
- testsuite/tests/codeGen/should_run/cgrun001.stdout
- testsuite/tests/codeGen/should_run/cgrun002.hs
- testsuite/tests/numeric/should_run/T18619.stderr
Changes:
=====================================
testsuite/driver/runtests.py
=====================================
@@ -591,7 +591,7 @@ else:
print(Perf.allow_changes_string([(m.change, m.stat) for m in t.metrics]))
print('-' * 25)
- summary(t, sys.stdout, color=term_color.enable_color)
+ summary(t, sys.stdout, color=term_color.enable_color, junit_path=args.junit)
# Write perf stats if any exist or if a metrics file is specified.
stats_metrics = [stat for (_, stat, __) in t.metrics] # type: List[PerfStat]
=====================================
testsuite/driver/testlib.py
=====================================
@@ -3527,9 +3527,22 @@ def findTFiles(roots: List[str]) -> Iterator[str]:
# -----------------------------------------------------------------------------
# Output a test summary to the specified file object
-def summary(t: TestRun, file: TextIO, color=False) -> None:
+def summary(t: TestRun, file: TextIO, color=False, junit_path: Optional[Path]=None) -> None:
file.write('\n')
+
+ if t.unexpected_failures:
+ if len(t.unexpected_failures) > MAX_SUMMARY_OUTPUT_TESTS:
+ # junit.xml only exists when a path was requested (e.g. in CI); don't
+ # point at a file that a local run never wrote.
+ where = 'see {}'.format(junit_path) if junit_path \
+ else 'rerun them individually'
+ file.write('Output of {} unexpected failures omitted (limit {}); {}.\n\n'
+ .format(len(t.unexpected_failures), MAX_SUMMARY_OUTPUT_TESTS, where))
+ else:
+ file.write('Output of unexpected failures:\n\n')
+ printTestOutputSummary(file, t.unexpected_failures, color, junit_path)
+
printUnexpectedTests(file,
[t.unexpected_passes, t.unexpected_failures,
t.unexpected_stat_failures, t.framework_failures])
@@ -3596,10 +3609,6 @@ def summary(t: TestRun, file: TextIO, color=False) -> None:
file.write('Framework warnings:\n')
printTestInfosSummary(file, t.framework_warnings)
- if t.unexpected_failures:
- file.write('Output of unexpected failures:\n\n')
- printTestOutputSummary(file, t.unexpected_failures, color)
-
if stopping():
file.write('WARNING: Testsuite run was terminated early\n')
@@ -3616,9 +3625,15 @@ def printUnexpectedTests(file: TextIO, testInfoss):
# Per-stream cap on a failing test's output repeated in the final summary.
MAX_SUMMARY_OUTPUT_LINES = 100
-def printTestOutputSummary(file: TextIO, testInfos, color: bool=False) -> None:
+# Above this many unexpected failures, skip repeating output entirely: the
+# dump would drown out the summary.
+MAX_SUMMARY_OUTPUT_TESTS = 20
+
+def printTestOutputSummary(file: TextIO, testInfos, color: bool=False,
+ junit_path: Optional[Path]=None) -> None:
# Repeat failing tests' captured output in the summary, so one needn't
# hunt for it earlier in a possibly very long log; see #16720.
+ where = ', see {}'.format(junit_path) if junit_path else ''
for result in sorted(testInfos, key=lambda r: (r.testname.lower(), r.way, r.directory)):
header = '=====> {}({}) [{}]'.format(result.testname, result.way, result.reason)
if color:
@@ -3633,7 +3648,7 @@ def printTestOutputSummary(file: TextIO, testInfos, color: bool=False) -> None:
if len(lines) > MAX_SUMMARY_OUTPUT_LINES:
omitted = len(lines) - MAX_SUMMARY_OUTPUT_LINES
lines = lines[:MAX_SUMMARY_OUTPUT_LINES] \
- + ['... ({} more lines omitted, see junit.xml)'.format(omitted)]
+ + ['... ({} more lines omitted{})'.format(omitted, where)]
s = label + '\n' + ''.join(l + '\n' for l in lines)
# Test output can contain characters that file's encoding
# cannot represent; replace rather than crash (cf safe_print).
=====================================
testsuite/tests/codeGen/should_run/cgrun001.stdout
=====================================
@@ -1 +1 @@
--42
+-43
=====================================
testsuite/tests/codeGen/should_run/cgrun002.hs
=====================================
@@ -1,4 +1,10 @@
-main = print ((f id2) (10 + thirty_two))
+import System.Exit
+import System.IO
+
+main = do
+ print ((f id2) (10 + thirty_two))
+ hPutStrLn stderr "deliberate breakage to test summary output"
+ exitWith (ExitFailure 3)
where
f x = g x
where
=====================================
testsuite/tests/numeric/should_run/T18619.stderr
=====================================
@@ -1,3 +1,3 @@
T18619: Uncaught exception ghc-internal:GHC.Internal.Exception.Type.ArithException:
-arithmetic overflow
+arithmetic underflow
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/18e8c673894f55b7eb06cd9ec2fc3f6...
--
View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/18e8c673894f55b7eb06cd9ec2fc3f6...
You're receiving this email because of your account on gitlab.haskell.org.
