GitLab

Simon Jakobi pushed to branch wip/sjakobi/T16720 at Glasgow Haskell Compiler / GHC

Commits:

3d1919a5

by Simon Jakobi at 2026-06-11T08:32:19+02:00

testsuite: Colorize the failure-output summary, also in CI

Make the per-test blocks in the "Output of unexpected failures"
section easier to scan:

* the '=====> test(way) [reason]' header is red,
* the 'Captured stdout/stderr:' labels are cyan,
* the captured output itself is indented four spaces.

Colors were previously disabled in CI because the driver only emits
them when stdout is a tty, yet the GitLab log viewer renders ANSI
colors fine. Add a --force-colors driver flag and pass it from
.gitlab/ci.sh. Note that config.supports_colors stays tty-based, since
it also guards terminal-title escape sequences, which must not end up
in a CI log.

The SUMMARY header and the new section now honor summary()'s color
parameter, so the plain-text summary file no longer receives escape
codes when colors are enabled.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

3 changed files:

.gitlab/ci.sh
testsuite/driver/runtests.py
testsuite/driver/testlib.py

Changes:

.gitlab/ci.sh

@@ -644,6 +644,10 @@ function test_hadrian() {
    check_msys2_deps _build/stage1/bin/ghc --version
    check_release_build
 +  # The GitLab log viewer renders ANSI colors, but the testsuite driver only
 +  # emits them when stdout is a tty unless they are forced.
 +  RUNTEST_ARGS="${RUNTEST_ARGS:-} --force-colors"
++
    # Ensure that statically-linked builds are actually static
    if [[ "${BUILD_FLAVOUR}" = *static* ]]; then
      bad_execs=""

testsuite/driver/runtests.py

@@ -94,6 +94,8 @@ parser.add_argument("--ignore-perf-failures", choices=['increases','decreases','
                          help="Do not fail due to out-of-tolerance perf tests")
  parser.add_argument("--only-report-hadrian-deps", type=Path,
                          help="Dry run the testsuite and report all extra hadrian dependencies needed on the given file")
 +parser.add_argument("--force-colors", action="store_true",
 +                        help="emit ANSI colors even when stdout is not a tty (e.g. for CI logs)")
  args = parser.parse_args()
@@ -259,7 +261,9 @@ def supports_colors():
      return True
  config.supports_colors = supports_colors()
 -term_color.enable_color = config.supports_colors
 +# Note that config.supports_colors deliberately stays tty-based: it also
 +# guards terminal-title updates, which must not end up in a CI log.
 +term_color.enable_color = config.supports_colors or args.force_colors
  # This has to come after arg parsing as the args can change the compiler
  get_compiler_info()
@@ -587,7 +591,7 @@ else:
          print(Perf.allow_changes_string([(m.change, m.stat) for m in t.metrics]))
          print('-' * 25)
 -    summary(t, sys.stdout, color=config.supports_colors)
 +    summary(t, sys.stdout, color=term_color.enable_color)
      # Write perf stats if any exist or if a metrics file is specified.
      stats_metrics = [stat for (_, stat, __) in t.metrics] # type: List[PerfStat]

testsuite/driver/testlib.py

@@ -3543,7 +3543,8 @@ def summary(t: TestRun, file: TextIO, color=False) -> None:
          summary_color = Color.GREEN
      assert t.start_time is not None
 -    file.write(colored(summary_color, 'SUMMARY') + ' for test run started at '
 +    summary_header = colored(summary_color, 'SUMMARY') if color else 'SUMMARY'
 +    file.write(summary_header + ' for test run started at '
                 + t.start_time.strftime("%c %Z") + '\n'
                 + str(datetime.datetime.now() - t.start_time).rjust(8)
                 + ' spent to go through\n'
@@ -3597,7 +3598,7 @@ def summary(t: TestRun, file: TextIO, color=False) -> None:
      if t.unexpected_failures:
          file.write('Output of unexpected failures:\n\n')
 -        printTestOutputSummary(file, t.unexpected_failures)
 +        printTestOutputSummary(file, t.unexpected_failures, color)
      if stopping():
          file.write('WARNING: Testsuite run was terminated early\n')
@@ -3616,22 +3617,27 @@ def printUnexpectedTests(file: TextIO, testInfoss):
  # per stream in the final summary (see printTestOutputSummary).
  MAX_SUMMARY_OUTPUT_LINES = 100
 -def printTestOutputSummary(file: TextIO, testInfos) -> None:
 +def printTestOutputSummary(file: TextIO, testInfos, color: bool=False) -> None:
      # Repeat the captured output of failing tests at the end of the run, so
      # that one can see why a test failed without hunting for its output
      # earlier in the (possibly very long) log; see #16720.
 +    # The captured output is indented to set it apart from the headers.
      for result in sorted(testInfos, key=lambda r: (r.testname.lower(), r.way, r.directory)):
 -        file.write(colored(Color.RED,
 -                           '=====> {}({}) [{}]'.format(result.testname, result.way, result.reason))
 -                   + '\n')
 +        header = '=====> {}({}) [{}]'.format(result.testname, result.way, result.reason)
 +        if color:
 +            header = colored(Color.RED, header)
 +        file.write(header + '\n')
          for stream_name, contents in [('stdout', result.stdout), ('stderr', result.stderr)]:
              if contents and contents.strip():
 +                label = 'Captured {}:'.format(stream_name)
 +                if color:
 +                    label = colored(Color.CYAN, label)
                  lines = contents.rstrip('\n').split('\n')
                  if len(lines) > MAX_SUMMARY_OUTPUT_LINES:
                      omitted = len(lines) - MAX_SUMMARY_OUTPUT_LINES
                      lines = lines[:MAX_SUMMARY_OUTPUT_LINES] \
                          + ['... ({} more lines omitted, see junit.xml)'.format(omitted)]
 -                s = 'Captured {}:\n{}\n'.format(stream_name, '\n'.join(lines))
 +                s = label + '\n' + ''.join('    ' + l + '\n' for l in lines)
                  # Test output can contain characters that file's encoding
                  # cannot represent; replace rather than crash (cf safe_print).
                  enc = getattr(file, 'encoding', None) or 'utf-8'