GitLab

Simon Jakobi pushed to branch wip/sjakobi/T16720 at Glasgow Haskell Compiler / GHC

Commits:

18e8c673

by Simon Jakobi at 2026-06-13T22:01:14+02:00

testsuite: Colorize the failure-output summary, also in CI

Make the per-test blocks in the "Output of unexpected failures"
section easier to scan:

* the '=====> test(way) [reason]' header is red,
* the 'Captured stdout/stderr:' labels are cyan,
* a closing '<===== end of output of unexpected failures' marker,
  also red, fences the section off from subsequent build output.

Colors were previously disabled in CI because the driver only emits
them when stdout is a tty, yet the GitLab log viewer renders ANSI
colors fine. Add a --force-colors driver flag and pass it from
.gitlab/ci.sh. Note that config.supports_colors stays tty-based, since
it also guards terminal-title escape sequences, which must not end up
in a CI log.

The SUMMARY header and the new section now honor summary()'s color
parameter, so the plain-text summary file no longer receives escape
codes when colors are enabled.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

3 changed files:

.gitlab/ci.sh
testsuite/driver/runtests.py
testsuite/driver/testlib.py

Changes:

.gitlab/ci.sh

@@ -644,6 +644,10 @@ function test_hadrian() {
    check_msys2_deps _build/stage1/bin/ghc --version
    check_release_build
 +  # GitLab's log viewer renders ANSI colors, but stdout here is not a tty,
 +  # so the driver must be told to emit them.
 +  RUNTEST_ARGS="${RUNTEST_ARGS:-} --force-colors"
++
    # Ensure that statically-linked builds are actually static
    if [[ "${BUILD_FLAVOUR}" = *static* ]]; then
      bad_execs=""

testsuite/driver/runtests.py

@@ -94,6 +94,8 @@ parser.add_argument("--ignore-perf-failures", choices=['increases','decreases','
                          help="Do not fail due to out-of-tolerance perf tests")
  parser.add_argument("--only-report-hadrian-deps", type=Path,
                          help="Dry run the testsuite and report all extra hadrian dependencies needed on the given file")
 +parser.add_argument("--force-colors", action="store_true",
 +                        help="emit ANSI colors even when stdout is not a tty (e.g. for CI logs)")
  args = parser.parse_args()
@@ -259,7 +261,9 @@ def supports_colors():
      return True
  config.supports_colors = supports_colors()
 -term_color.enable_color = config.supports_colors
 +# config.supports_colors deliberately stays tty-based: it also guards
 +# terminal-title updates, which must not end up in a CI log.
 +term_color.enable_color = config.supports_colors or args.force_colors
  # This has to come after arg parsing as the args can change the compiler
  get_compiler_info()
@@ -587,7 +591,7 @@ else:
          print(Perf.allow_changes_string([(m.change, m.stat) for m in t.metrics]))
          print('-' * 25)
 -    summary(t, sys.stdout, color=config.supports_colors)
 +    summary(t, sys.stdout, color=term_color.enable_color)
      # Write perf stats if any exist or if a metrics file is specified.
      stats_metrics = [stat for (_, stat, __) in t.metrics] # type: List[PerfStat]

testsuite/driver/testlib.py

@@ -3543,7 +3543,8 @@ def summary(t: TestRun, file: TextIO, color=False) -> None:
          summary_color = Color.GREEN
      assert t.start_time is not None
 -    file.write(colored(summary_color, 'SUMMARY') + ' for test run started at '
 +    summary_header = colored(summary_color, 'SUMMARY') if color else 'SUMMARY'
 +    file.write(summary_header + ' for test run started at '
                 + t.start_time.strftime("%c %Z") + '\n'
                 + str(datetime.datetime.now() - t.start_time).rjust(8)
                 + ' spent to go through\n'
@@ -3597,7 +3598,7 @@ def summary(t: TestRun, file: TextIO, color=False) -> None:
      if t.unexpected_failures:
          file.write('Output of unexpected failures:\n\n')
 -        printTestOutputSummary(file, t.unexpected_failures)
 +        printTestOutputSummary(file, t.unexpected_failures, color)
      if stopping():
          file.write('WARNING: Testsuite run was terminated early\n')
@@ -3615,26 +3616,34 @@ def printUnexpectedTests(file: TextIO, testInfoss):
  # Per-stream cap on a failing test's output repeated in the final summary.
  MAX_SUMMARY_OUTPUT_LINES = 100
 -def printTestOutputSummary(file: TextIO, testInfos) -> None:
 +def printTestOutputSummary(file: TextIO, testInfos, color: bool=False) -> None:
      # Repeat failing tests' captured output in the summary, so one needn't
      # hunt for it earlier in a possibly very long log; see #16720.
      for result in sorted(testInfos, key=lambda r: (r.testname.lower(), r.way, r.directory)):
 -        file.write(colored(Color.RED,
 -                           '=====> {}({}) [{}]'.format(result.testname, result.way, result.reason))
 -                   + '\n')
 +        header = '=====> {}({}) [{}]'.format(result.testname, result.way, result.reason)
 +        if color:
 +            header = colored(Color.RED, header)
 +        file.write(header + '\n')
          for stream_name, contents in [('stdout', result.stdout), ('stderr', result.stderr)]:
              if contents and contents.strip():
 +                label = 'Captured {}:'.format(stream_name)
 +                if color:
 +                    label = colored(Color.CYAN, label)
                  lines = contents.rstrip('\n').split('\n')
                  if len(lines) > MAX_SUMMARY_OUTPUT_LINES:
                      omitted = len(lines) - MAX_SUMMARY_OUTPUT_LINES
                      lines = lines[:MAX_SUMMARY_OUTPUT_LINES] \
                          + ['... ({} more lines omitted, see junit.xml)'.format(omitted)]
 -                s = 'Captured {}:\n{}\n'.format(stream_name, '\n'.join(lines))
 +                s = label + '\n' + ''.join(l + '\n' for l in lines)
                  # Test output can contain characters that file's encoding
                  # cannot represent; replace rather than crash (cf safe_print).
                  enc = getattr(file, 'encoding', None) or 'utf-8'
                  file.write(s.encode(enc, errors='replace').decode(enc))
          file.write('\n')
 +    footer = '<===== end of output of unexpected failures'
 +    if color:
 +        footer = colored(Color.RED, footer)
 +    file.write(footer + '\n\n')
  def printTestInfosSummary(file: TextIO, testInfos):
      maxDirLen = max(len(tr.directory) for tr in testInfos)