Simon Jakobi pushed to branch wip/sjakobi/T16720 at Glasgow Haskell Compiler / GHC

Commits:

4 changed files:

Changes:

  • .gitlab/ci.sh
    ... ... @@ -652,6 +652,10 @@ function test_hadrian() {
    652 652
       check_msys2_deps _build/stage1/bin/ghc --version
    
    653 653
       check_release_build
    
    654 654
     
    
    655
    +  # GitLab's log viewer renders ANSI colors, but stdout here is not a tty,
    
    656
    +  # so the driver must be told to emit them.
    
    657
    +  RUNTEST_ARGS="${RUNTEST_ARGS:-} --force-colors"
    
    658
    +
    
    655 659
       # Ensure that statically-linked builds are actually static
    
    656 660
       if [[ "${BUILD_FLAVOUR}" = *static* ]]; then
    
    657 661
         bad_execs=""
    

  • testsuite/driver/runtests.py
    ... ... @@ -94,6 +94,8 @@ parser.add_argument("--ignore-perf-failures", choices=['increases','decreases','
    94 94
                             help="Do not fail due to out-of-tolerance perf tests")
    
    95 95
     parser.add_argument("--only-report-hadrian-deps", type=Path,
    
    96 96
                             help="Dry run the testsuite and report all extra hadrian dependencies needed on the given file")
    
    97
    +parser.add_argument("--force-colors", action="store_true",
    
    98
    +                        help="emit ANSI colors even when stdout is not a tty (e.g. for CI logs)")
    
    97 99
     
    
    98 100
     args = parser.parse_args()
    
    99 101
     
    
    ... ... @@ -259,7 +261,9 @@ def supports_colors():
    259 261
         return True
    
    260 262
     
    
    261 263
     config.supports_colors = supports_colors()
    
    262
    -term_color.enable_color = config.supports_colors
    
    264
    +# config.supports_colors deliberately stays tty-based: it also guards
    
    265
    +# terminal-title updates, which must not end up in a CI log.
    
    266
    +term_color.enable_color = config.supports_colors or args.force_colors
    
    263 267
     
    
    264 268
     # This has to come after arg parsing as the args can change the compiler
    
    265 269
     get_compiler_info()
    
    ... ... @@ -587,7 +591,7 @@ else:
    587 591
             print(Perf.allow_changes_string([(m.change, m.stat) for m in t.metrics]))
    
    588 592
             print('-' * 25)
    
    589 593
     
    
    590
    -    summary(t, sys.stdout, color=config.supports_colors)
    
    594
    +    summary(t, sys.stdout, color=term_color.enable_color, junit_path=args.junit)
    
    591 595
     
    
    592 596
         # Write perf stats if any exist or if a metrics file is specified.
    
    593 597
         stats_metrics = [stat for (_, stat, __) in t.metrics] # type: List[PerfStat]
    

  • testsuite/driver/term_color.py
    1 1
     from enum import Enum
    
    2 2
     
    
    3
    +# Whether to emit color escapes; set in runtests.py.
    
    3 4
     enable_color = True
    
    4 5
     
    
    5 6
     class Color(Enum):
    
    ... ... @@ -18,3 +19,7 @@ def colored(color: Color, s: str) -> str:
    18 19
         else:
    
    19 20
             return s
    
    20 21
     
    
    22
    +# For renderers that serve several sinks: `enabled` says whether *this* sink
    
    23
    +# takes color (the summary is written both to stdout and to a plain-text file).
    
    24
    +def colored_if(enabled: bool, color: Color, s: str) -> str:
    
    25
    +    return colored(color, s) if enabled else s

  • testsuite/driver/testlib.py
    ... ... @@ -27,7 +27,7 @@ from testutil import strip_quotes, lndir, link_or_copy_file, passed, \
    27 27
                          failBecause, testing_metrics, residency_testing_metrics, \
    
    28 28
                          stable_perf_counters, \
    
    29 29
                          PassFail, badResult, str_warn, str_removeprefix
    
    30
    -from term_color import Color, colored
    
    30
    +from term_color import Color, colored_if
    
    31 31
     import testutil
    
    32 32
     from cpu_features import have_cpu_feature
    
    33 33
     import perf_notes as Perf
    
    ... ... @@ -1497,6 +1497,19 @@ def _newTestDir(name: TestName, opts: TestOptions, tempdir, dir):
    1497 1497
         opts.testdir_raw = Path(os.path.join(tempdir, testdir, name + testdir_suffix))
    
    1498 1498
         opts.compiler_always_flags = config.compiler_always_flags
    
    1499 1499
     
    
    1500
    +def _result_directory(opts: TestOptions) -> str:
    
    1501
    +    # The test's source directory, relative to the GHC source root, so it reads
    
    1502
    +    # the same regardless of which directory `make` was invoked from.
    
    1503
    +    srcdir = opts.srcdir
    
    1504
    +    if srcdir is None:
    
    1505
    +        return ''
    
    1506
    +    try:
    
    1507
    +        return os.path.relpath(srcdir, config.top.parent)
    
    1508
    +    except ValueError:
    
    1509
    +        # No relative path exists (e.g. different Windows drives); the
    
    1510
    +        # absolute path is still more useful than nothing.
    
    1511
    +        return str(srcdir)
    
    1512
    +
    
    1500 1513
     # -----------------------------------------------------------------------------
    
    1501 1514
     # Actually doing tests
    
    1502 1515
     
    
    ... ... @@ -1821,7 +1834,7 @@ async def do_test(name: TestName,
    1821 1834
         if opts.expect not in ['pass', 'fail', 'missing-lib']:
    
    1822 1835
             framework_fail(name, way, 'bad expected ' + opts.expect)
    
    1823 1836
     
    
    1824
    -    directory = str_removeprefix(str_removeprefix(str(opts.testdir), './'), '.\\')
    
    1837
    +    directory = _result_directory(opts)
    
    1825 1838
     
    
    1826 1839
         if way in opts.fragile_ways:
    
    1827 1840
             if_verbose(1, '*** fragile test %s resulted in %s' % (full_name, 'pass' if result.passed else 'fail'))
    
    ... ... @@ -1875,7 +1888,7 @@ def framework_fail(name: Optional[TestName], way: Optional[WayName], reason: str
    1875 1888
         # so we need to take care not to blow up with the wrong way
    
    1876 1889
         # and report the actual reason for the failure.
    
    1877 1890
         try:
    
    1878
    -      directory = str_removeprefix(str_removeprefix(str(opts.testdir), './'), '.\\')
    
    1891
    +      directory = _result_directory(opts)
    
    1879 1892
         except:
    
    1880 1893
           directory = ''
    
    1881 1894
         full_name = '%s(%s)' % (name, way)
    
    ... ... @@ -1888,7 +1901,7 @@ def framework_fail(name: Optional[TestName], way: Optional[WayName], reason: str
    1888 1901
     
    
    1889 1902
     def framework_warn(name: TestName, way: WayName, reason: str) -> None:
    
    1890 1903
         opts = getTestOpts()
    
    1891
    -    directory = str_removeprefix(str_removeprefix(str(opts.testdir), './'), '.\\')
    
    1904
    +    directory = _result_directory(opts)
    
    1892 1905
         full_name = name + '(' + way + ')'
    
    1893 1906
         if_verbose(1, '*** framework warning for %s %s ' % (full_name, reason))
    
    1894 1907
         t.framework_warnings.append(TestResult(directory, name, reason, way))
    
    ... ... @@ -2443,19 +2456,23 @@ async def simple_run(name: TestName, way: WayName, prog: str, extra_run_opts: st
    2443 2456
                 dump_stdout(name)
    
    2444 2457
                 dump_stderr(name)
    
    2445 2458
             message = format_bad_exit_code_message(exit_code)
    
    2446
    -        return failBecause(message)
    
    2459
    +        return failBecause(message,
    
    2460
    +                           stderr=read_stderr(name),
    
    2461
    +                           stdout=read_stdout(name))
    
    2447 2462
     
    
    2448 2463
         stderr_match = CompareOutput(True) if (opts.ignore_stderr or opts.combined_output) else await stderr_ok(name, way)
    
    2449 2464
         if not stderr_match:
    
    2465
    +        # The diff already contains the mismatching stream; see Note [Redundant
    
    2466
    +        # output in test results].
    
    2450 2467
             return failBecause('bad stderr',
    
    2451
    -                           stderr=read_stderr(name),
    
    2468
    +                           stderr=None if stderr_match.diff else read_stderr(name),
    
    2452 2469
                                stdout=read_stdout(name),
    
    2453 2470
                                diff=stderr_match.diff)
    
    2454 2471
         stdout_match = CompareOutput(True) if opts.ignore_stdout else await stdout_ok(name, way)
    
    2455 2472
         if not stdout_match:
    
    2456 2473
             return failBecause('bad stdout',
    
    2457 2474
                                stderr=read_stderr(name),
    
    2458
    -                           stdout=read_stdout(name),
    
    2475
    +                           stdout=None if stdout_match.diff else read_stdout(name),
    
    2459 2476
                                diff=stdout_match.diff)
    
    2460 2477
     
    
    2461 2478
         check_hp = '-hT' in my_rts_flags and opts.check_hp
    
    ... ... @@ -2565,8 +2582,9 @@ async def interpreter_run(name: TestName,
    2565 2582
         if not stderr_match:
    
    2566 2583
             if _expect_pass(way):
    
    2567 2584
                 dump_stderr_for('comp', name)
    
    2585
    +        # See Note [Redundant output in test results].
    
    2568 2586
             return failBecause('bad stderr',
    
    2569
    -                           stderr=read_stderr(name),
    
    2587
    +                           stderr=None if stderr_match.diff else read_stderr(name),
    
    2570 2588
                                stdout=read_stdout(name),
    
    2571 2589
                                diff=stderr_match.diff)
    
    2572 2590
         stdout_match = CompareOutput(True) if opts.ignore_stdout else await stdout_ok(name, way)
    
    ... ... @@ -2575,7 +2593,7 @@ async def interpreter_run(name: TestName,
    2575 2593
                 dump_stderr_for('comp', name)
    
    2576 2594
             return failBecause('bad stdout',
    
    2577 2595
                                stderr=read_stderr(name),
    
    2578
    -                           stdout=read_stdout(name),
    
    2596
    +                           stdout=None if stdout_match.diff else read_stdout(name),
    
    2579 2597
                                diff=stdout_match.diff)
    
    2580 2598
         return passed()
    
    2581 2599
     
    
    ... ... @@ -2633,13 +2651,13 @@ async def stdout_ok(name: TestName, way: WayName) -> CompareOutput:
    2633 2651
     def read_stdout( name: TestName ) -> str:
    
    2634 2652
         path = in_testdir(name, 'run.stdout')
    
    2635 2653
         if path.exists():
    
    2636
    -        return path.read_text(encoding='UTF-8')
    
    2654
    +        return path.read_text(encoding='UTF-8', errors='replace')
    
    2637 2655
         else:
    
    2638 2656
             return ''
    
    2639 2657
     
    
    2640 2658
     def read_diff( diff_file: Path ) -> Optional[str]:
    
    2641 2659
         if diff_file.exists():
    
    2642
    -        diff = diff_file.read_text()
    
    2660
    +        diff = diff_file.read_text(encoding='UTF-8', errors='replace')
    
    2643 2661
             diff_file.unlink()
    
    2644 2662
             return diff or None
    
    2645 2663
         else:
    
    ... ... @@ -2663,14 +2681,14 @@ async def stderr_ok(name: TestName, way: WayName) -> CompareOutput:
    2663 2681
     def read_comp_stderr( name: TestName ) -> str:
    
    2664 2682
         path = in_testdir(name, 'comp.stderr')
    
    2665 2683
         if path.exists():
    
    2666
    -        return path.read_text(encoding='UTF-8')
    
    2684
    +        return path.read_text(encoding='UTF-8', errors='replace')
    
    2667 2685
         else:
    
    2668 2686
             return ''
    
    2669 2687
     
    
    2670 2688
     def read_stderr_for( phase: str, name: TestName ) -> str:
    
    2671 2689
         path = in_testdir(name, phase + '.stderr')
    
    2672 2690
         if path.exists():
    
    2673
    -        return path.read_text(encoding='UTF-8')
    
    2691
    +        return path.read_text(encoding='UTF-8', errors='replace')
    
    2674 2692
         else:
    
    2675 2693
             return ''
    
    2676 2694
     
    
    ... ... @@ -3569,12 +3587,50 @@ def findTFiles(roots: List[str]) -> Iterator[str]:
    3569 3587
     # -----------------------------------------------------------------------------
    
    3570 3588
     # Output a test summary to the specified file object
    
    3571 3589
     
    
    3572
    -def summary(t: TestRun, file: TextIO, color=False) -> None:
    
    3590
    +def summary(t: TestRun, file: TextIO, color=False, junit_path: Optional[Path]=None) -> None:
    
    3573 3591
     
    
    3574 3592
         file.write('\n')
    
    3593
    +
    
    3594
    +    if t.unexpected_failures:
    
    3595
    +        # Count output blocks rather than results: a test failing in many ways
    
    3596
    +        # collapses to a single block.
    
    3597
    +        groups = groupTestOutput(t.unexpected_failures)
    
    3598
    +        if len(groups) <= MAX_SUMMARY_OUTPUT_TESTS:
    
    3599
    +            printTestOutputSummary(file, groups, color, junit_path)
    
    3600
    +        else:
    
    3601
    +            where = '; see {}'.format(junit_path) if junit_path else ''
    
    3602
    +            header = ('Unexpected failures (more than {}, output omitted{}):'
    
    3603
    +                      .format(MAX_SUMMARY_OUTPUT_TESTS, where))
    
    3604
    +            file.write(colored_if(color, Color.RED, header) + '\n')
    
    3605
    +            printTestInfosSummary(file, t.unexpected_failures)
    
    3606
    +
    
    3607
    +    if t.unexpected_passes:
    
    3608
    +        header = 'Unexpected passes:'
    
    3609
    +        file.write(colored_if(color, Color.RED, header) + '\n')
    
    3610
    +        printTestInfosSummary(file, t.unexpected_passes)
    
    3611
    +
    
    3612
    +    if t.unexpected_stat_failures:
    
    3613
    +        header = 'Unexpected stat failures:'
    
    3614
    +        file.write(colored_if(color, Color.RED, header) + '\n')
    
    3615
    +        printTestInfosSummary(file, t.unexpected_stat_failures)
    
    3616
    +
    
    3617
    +    if t.framework_failures:
    
    3618
    +        header = 'Framework failures:'
    
    3619
    +        file.write(colored_if(color, Color.RED, header) + '\n')
    
    3620
    +        printTestInfosSummary(file, t.framework_failures)
    
    3621
    +
    
    3622
    +    if t.framework_warnings:
    
    3623
    +        header = 'Framework warnings:'
    
    3624
    +        file.write(colored_if(color, Color.YELLOW, header) + '\n')
    
    3625
    +        printTestInfosSummary(file, t.framework_warnings)
    
    3626
    +
    
    3627
    +    if stopping():
    
    3628
    +        warning = 'WARNING: Testsuite run was terminated early'
    
    3629
    +        file.write(colored_if(color, Color.YELLOW, warning) + '\n')
    
    3630
    +
    
    3575 3631
         printUnexpectedTests(file,
    
    3576 3632
             [t.unexpected_passes, t.unexpected_failures,
    
    3577
    -         t.unexpected_stat_failures, t.framework_failures])
    
    3633
    +         t.unexpected_stat_failures, t.framework_failures], color)
    
    3578 3634
     
    
    3579 3635
         if len(t.unexpected_failures) > 0 or \
    
    3580 3636
             len(t.unexpected_stat_failures) > 0 or \
    
    ... ... @@ -3585,7 +3641,8 @@ def summary(t: TestRun, file: TextIO, color=False) -> None:
    3585 3641
             summary_color = Color.GREEN
    
    3586 3642
     
    
    3587 3643
         assert t.start_time is not None
    
    3588
    -    file.write(colored(summary_color, 'SUMMARY') + ' for test run started at '
    
    3644
    +    summary_header = colored_if(color, summary_color, 'SUMMARY')
    
    3645
    +    file.write(summary_header + ' for test run started at '
    
    3589 3646
                    + t.start_time.strftime("%c %Z") + '\n'
    
    3590 3647
                    + str(datetime.datetime.now() - t.start_time).rjust(8)
    
    3591 3648
                    + ' spent to go through\n'
    
    ... ... @@ -3617,46 +3674,107 @@ def summary(t: TestRun, file: TextIO, color=False) -> None:
    3617 3674
                    + ' fragile tests\n'
    
    3618 3675
                    + '\n')
    
    3619 3676
     
    
    3620
    -    if t.unexpected_passes:
    
    3621
    -        file.write('Unexpected passes:\n')
    
    3622
    -        printTestInfosSummary(file, t.unexpected_passes)
    
    3623
    -
    
    3624
    -    if t.unexpected_failures:
    
    3625
    -        file.write('Unexpected failures:\n')
    
    3626
    -        printTestInfosSummary(file, t.unexpected_failures)
    
    3627
    -
    
    3628
    -    if t.unexpected_stat_failures:
    
    3629
    -        file.write('Unexpected stat failures:\n')
    
    3630
    -        printTestInfosSummary(file, t.unexpected_stat_failures)
    
    3631
    -
    
    3632
    -    if t.framework_failures:
    
    3633
    -        file.write('Framework failures:\n')
    
    3634
    -        printTestInfosSummary(file, t.framework_failures)
    
    3635
    -
    
    3636
    -    if t.framework_warnings:
    
    3637
    -        file.write('Framework warnings:\n')
    
    3638
    -        printTestInfosSummary(file, t.framework_warnings)
    
    3639
    -
    
    3640
    -    if stopping():
    
    3641
    -        file.write('WARNING: Testsuite run was terminated early\n')
    
    3642
    -
    
    3643
    -def printUnexpectedTests(file: TextIO, testInfoss):
    
    3677
    +def printUnexpectedTests(file: TextIO, testInfoss, color=False):
    
    3644 3678
         unexpected = set(result.testname
    
    3645 3679
                          for testInfos in testInfoss
    
    3646 3680
                          for result in testInfos
    
    3647 3681
                          if not result.testname.endswith('.T'))
    
    3648 3682
         if unexpected:
    
    3649
    -        file.write('Unexpected results from:\n')
    
    3683
    +        header = 'Unexpected results from:'
    
    3684
    +        file.write(colored_if(color, Color.RED, header) + '\n')
    
    3650 3685
             file.write('TEST="' + ' '.join(sorted(unexpected)) + '"\n')
    
    3651 3686
             file.write('\n')
    
    3652 3687
     
    
    3688
    +# Per-stream cap on a failing test's output repeated in the final summary.
    
    3689
    +MAX_SUMMARY_OUTPUT_LINES = 100
    
    3690
    +
    
    3691
    +# Above this many output blocks, skip repeating output entirely: the dump
    
    3692
    +# would drown out the summary.
    
    3693
    +MAX_SUMMARY_OUTPUT_TESTS = 20
    
    3694
    +
    
    3695
    +# Note [Redundant output in test results]
    
    3696
    +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    
    3697
    +# A failing test result carries up to three pieces of output: `diff`, `stdout`
    
    3698
    +# and `stderr`. For an output mismatch these overlap: the diff's `+` lines are
    
    3699
    +# the very stream that mismatched, normalised. Reporting both would print the
    
    3700
    +# same text twice, so the mismatching stream is dropped at the call sites in
    
    3701
    +# favour of the diff, which additionally shows what was expected. The *other*
    
    3702
    +# stream is kept: on a stdout mismatch, stderr is independent context.
    
    3703
    +#
    
    3704
    +# The drop is conditional on there being a diff at all: compare_outputs only
    
    3705
    +# runs `diff` when config.verbose >= 1, so under -v0 the stream is the only
    
    3706
    +# output there is.
    
    3707
    +#
    
    3708
    +# Note that, since the drop happens at result construction, it also affects the
    
    3709
    +# JUnit report (junit.py).
    
    3710
    +
    
    3711
    +def strip_diff_header(diff: Optional[str]) -> Optional[str]:
    
    3712
    +    # Drop diff(1)'s ---/+++ lines: they name normalised files in the test
    
    3713
    +    # directory and carry timestamps, which would also keep otherwise
    
    3714
    +    # identical failures from being grouped.
    
    3715
    +    if diff is None:
    
    3716
    +        return None
    
    3717
    +    lines = diff.split('\n')
    
    3718
    +    if len(lines) >= 2 and lines[0].startswith('--- ') and lines[1].startswith('+++ '):
    
    3719
    +        return '\n'.join(lines[2:])
    
    3720
    +    return diff
    
    3721
    +
    
    3722
    +def sorted_results(testInfos: List[TestResult]) -> List[TestResult]:
    
    3723
    +    return sorted(testInfos, key=lambda r: (r.testname.lower(), r.directory, r.way))
    
    3724
    +
    
    3725
    +# A failure-output block: a representative result, its header-stripped diff,
    
    3726
    +# and the ways that share it.
    
    3727
    +OutputGroup = Tuple[TestResult, Optional[str], List[WayName]]
    
    3728
    +
    
    3729
    +# Tests that fail identically in several ways (e.g. normal and g1) share one
    
    3730
    +# output block, with the ways collected in the header.
    
    3731
    +def groupTestOutput(testInfos: List[TestResult]) -> List[OutputGroup]:
    
    3732
    +    # Relies on dicts preserving insertion order.
    
    3733
    +    groups = {} # type: Dict[Tuple, OutputGroup]
    
    3734
    +    for result in sorted_results(testInfos):
    
    3735
    +        diff = strip_diff_header(result.diff)
    
    3736
    +        key = (result.testname, result.directory, result.reason,
    
    3737
    +               diff, result.stdout, result.stderr)
    
    3738
    +        groups.setdefault(key, (result, diff, []))[2].append(result.way)
    
    3739
    +    return list(groups.values())
    
    3740
    +
    
    3741
    +def printTestOutputSummary(file: TextIO,
    
    3742
    +                           groups: List[OutputGroup],
    
    3743
    +                           color: bool=False,
    
    3744
    +                           junit_path: Optional[Path]=None) -> None:
    
    3745
    +    # Repeat failing tests' captured output in the summary, so one needn't
    
    3746
    +    # hunt for it earlier in a possibly very long log; see #16720.
    
    3747
    +    header = '=====> Unexpected failures output summary'
    
    3748
    +    file.write(colored_if(color, Color.RED, header) + '\n\n')
    
    3749
    +
    
    3750
    +    where = ', see {}'.format(junit_path) if junit_path else ''
    
    3751
    +    for result, diff, ways in groups:
    
    3752
    +        header = '=====> {}({}) ({}) [{}]'.format(
    
    3753
    +            result.testname, ', '.join(ways), result.directory + os.sep, result.reason)
    
    3754
    +        file.write(colored_if(color, Color.RED, header) + '\n')
    
    3755
    +        # See Note [Redundant output in test results] for why these don't overlap.
    
    3756
    +        for label, contents in [('Output diff (expected vs actual):', diff),
    
    3757
    +                                ('Captured stdout:', result.stdout),
    
    3758
    +                                ('Captured stderr:', result.stderr)]:
    
    3759
    +            if contents and contents.strip():
    
    3760
    +                lines = contents.rstrip('\n').split('\n')
    
    3761
    +                if len(lines) > MAX_SUMMARY_OUTPUT_LINES:
    
    3762
    +                    omitted = len(lines) - MAX_SUMMARY_OUTPUT_LINES
    
    3763
    +                    lines = lines[:MAX_SUMMARY_OUTPUT_LINES] \
    
    3764
    +                        + ['... ({} more lines omitted{})'.format(omitted, where)]
    
    3765
    +                s = colored_if(color, Color.CYAN, label) + '\n' \
    
    3766
    +                    + ''.join(l + '\n' for l in lines)
    
    3767
    +                # Test output can contain characters that file's encoding
    
    3768
    +                # cannot represent; replace rather than crash (cf safe_print).
    
    3769
    +                enc = getattr(file, 'encoding', None) or 'utf-8'
    
    3770
    +                file.write(s.encode(enc, errors='replace').decode(enc))
    
    3771
    +    footer = '<===== end of unexpected failures output summary'
    
    3772
    +    file.write(colored_if(color, Color.RED, footer) + '\n\n')
    
    3773
    +
    
    3653 3774
     def printTestInfosSummary(file: TextIO, testInfos):
    
    3654
    -    maxDirLen = max(len(tr.directory) for tr in testInfos)
    
    3655
    -    for result in sorted(testInfos, key=lambda r: (r.testname.lower(), r.way, r.directory)):
    
    3656
    -        directory = result.directory.ljust(maxDirLen)
    
    3657
    -        file.write('   {directory}  {r.testname} [{r.reason}] ({r.way})\n'.format(
    
    3658
    -            r = result,
    
    3659
    -            directory = directory))
    
    3775
    +    for result in sorted_results(testInfos):
    
    3776
    +        path = os.path.join(result.directory, result.testname)
    
    3777
    +        file.write('   {path} [{r.reason}] ({r.way})\n'.format(r=result, path=path))
    
    3660 3778
         file.write('\n')
    
    3661 3779
     
    
    3662 3780
     def modify_lines(s: str, f: Callable[[str], str]) -> str: