| ... |
... |
@@ -27,7 +27,7 @@ from testutil import strip_quotes, lndir, link_or_copy_file, passed, \ |
|
27
|
27
|
failBecause, testing_metrics, residency_testing_metrics, \
|
|
28
|
28
|
stable_perf_counters, \
|
|
29
|
29
|
PassFail, badResult, str_warn, str_removeprefix
|
|
30
|
|
-from term_color import Color, colored
|
|
|
30
|
+from term_color import Color, colored_if
|
|
31
|
31
|
import testutil
|
|
32
|
32
|
from cpu_features import have_cpu_feature
|
|
33
|
33
|
import perf_notes as Perf
|
| ... |
... |
@@ -1497,6 +1497,19 @@ def _newTestDir(name: TestName, opts: TestOptions, tempdir, dir): |
|
1497
|
1497
|
opts.testdir_raw = Path(os.path.join(tempdir, testdir, name + testdir_suffix))
|
|
1498
|
1498
|
opts.compiler_always_flags = config.compiler_always_flags
|
|
1499
|
1499
|
|
|
|
1500
|
+def _result_directory(opts: TestOptions) -> str:
|
|
|
1501
|
+ # The test's source directory, relative to the GHC source root, so it reads
|
|
|
1502
|
+ # the same regardless of which directory `make` was invoked from.
|
|
|
1503
|
+ srcdir = opts.srcdir
|
|
|
1504
|
+ if srcdir is None:
|
|
|
1505
|
+ return ''
|
|
|
1506
|
+ try:
|
|
|
1507
|
+ return os.path.relpath(srcdir, config.top.parent)
|
|
|
1508
|
+ except ValueError:
|
|
|
1509
|
+ # No relative path exists (e.g. different Windows drives); the
|
|
|
1510
|
+ # absolute path is still more useful than nothing.
|
|
|
1511
|
+ return str(srcdir)
|
|
|
1512
|
+
|
|
1500
|
1513
|
# -----------------------------------------------------------------------------
|
|
1501
|
1514
|
# Actually doing tests
|
|
1502
|
1515
|
|
| ... |
... |
@@ -1821,7 +1834,7 @@ async def do_test(name: TestName, |
|
1821
|
1834
|
if opts.expect not in ['pass', 'fail', 'missing-lib']:
|
|
1822
|
1835
|
framework_fail(name, way, 'bad expected ' + opts.expect)
|
|
1823
|
1836
|
|
|
1824
|
|
- directory = str_removeprefix(str_removeprefix(str(opts.testdir), './'), '.\\')
|
|
|
1837
|
+ directory = _result_directory(opts)
|
|
1825
|
1838
|
|
|
1826
|
1839
|
if way in opts.fragile_ways:
|
|
1827
|
1840
|
if_verbose(1, '*** fragile test %s resulted in %s' % (full_name, 'pass' if result.passed else 'fail'))
|
| ... |
... |
@@ -1875,7 +1888,7 @@ def framework_fail(name: Optional[TestName], way: Optional[WayName], reason: str |
|
1875
|
1888
|
# so we need to take care not to blow up with the wrong way
|
|
1876
|
1889
|
# and report the actual reason for the failure.
|
|
1877
|
1890
|
try:
|
|
1878
|
|
- directory = str_removeprefix(str_removeprefix(str(opts.testdir), './'), '.\\')
|
|
|
1891
|
+ directory = _result_directory(opts)
|
|
1879
|
1892
|
except:
|
|
1880
|
1893
|
directory = ''
|
|
1881
|
1894
|
full_name = '%s(%s)' % (name, way)
|
| ... |
... |
@@ -1888,7 +1901,7 @@ def framework_fail(name: Optional[TestName], way: Optional[WayName], reason: str |
|
1888
|
1901
|
|
|
1889
|
1902
|
def framework_warn(name: TestName, way: WayName, reason: str) -> None:
|
|
1890
|
1903
|
opts = getTestOpts()
|
|
1891
|
|
- directory = str_removeprefix(str_removeprefix(str(opts.testdir), './'), '.\\')
|
|
|
1904
|
+ directory = _result_directory(opts)
|
|
1892
|
1905
|
full_name = name + '(' + way + ')'
|
|
1893
|
1906
|
if_verbose(1, '*** framework warning for %s %s ' % (full_name, reason))
|
|
1894
|
1907
|
t.framework_warnings.append(TestResult(directory, name, reason, way))
|
| ... |
... |
@@ -2443,19 +2456,23 @@ async def simple_run(name: TestName, way: WayName, prog: str, extra_run_opts: st |
|
2443
|
2456
|
dump_stdout(name)
|
|
2444
|
2457
|
dump_stderr(name)
|
|
2445
|
2458
|
message = format_bad_exit_code_message(exit_code)
|
|
2446
|
|
- return failBecause(message)
|
|
|
2459
|
+ return failBecause(message,
|
|
|
2460
|
+ stderr=read_stderr(name),
|
|
|
2461
|
+ stdout=read_stdout(name))
|
|
2447
|
2462
|
|
|
2448
|
2463
|
stderr_match = CompareOutput(True) if (opts.ignore_stderr or opts.combined_output) else await stderr_ok(name, way)
|
|
2449
|
2464
|
if not stderr_match:
|
|
|
2465
|
+ # The diff already contains the mismatching stream; see Note [Redundant
|
|
|
2466
|
+ # output in test results].
|
|
2450
|
2467
|
return failBecause('bad stderr',
|
|
2451
|
|
- stderr=read_stderr(name),
|
|
|
2468
|
+ stderr=None if stderr_match.diff else read_stderr(name),
|
|
2452
|
2469
|
stdout=read_stdout(name),
|
|
2453
|
2470
|
diff=stderr_match.diff)
|
|
2454
|
2471
|
stdout_match = CompareOutput(True) if opts.ignore_stdout else await stdout_ok(name, way)
|
|
2455
|
2472
|
if not stdout_match:
|
|
2456
|
2473
|
return failBecause('bad stdout',
|
|
2457
|
2474
|
stderr=read_stderr(name),
|
|
2458
|
|
- stdout=read_stdout(name),
|
|
|
2475
|
+ stdout=None if stdout_match.diff else read_stdout(name),
|
|
2459
|
2476
|
diff=stdout_match.diff)
|
|
2460
|
2477
|
|
|
2461
|
2478
|
check_hp = '-hT' in my_rts_flags and opts.check_hp
|
| ... |
... |
@@ -2565,8 +2582,9 @@ async def interpreter_run(name: TestName, |
|
2565
|
2582
|
if not stderr_match:
|
|
2566
|
2583
|
if _expect_pass(way):
|
|
2567
|
2584
|
dump_stderr_for('comp', name)
|
|
|
2585
|
+ # See Note [Redundant output in test results].
|
|
2568
|
2586
|
return failBecause('bad stderr',
|
|
2569
|
|
- stderr=read_stderr(name),
|
|
|
2587
|
+ stderr=None if stderr_match.diff else read_stderr(name),
|
|
2570
|
2588
|
stdout=read_stdout(name),
|
|
2571
|
2589
|
diff=stderr_match.diff)
|
|
2572
|
2590
|
stdout_match = CompareOutput(True) if opts.ignore_stdout else await stdout_ok(name, way)
|
| ... |
... |
@@ -2575,7 +2593,7 @@ async def interpreter_run(name: TestName, |
|
2575
|
2593
|
dump_stderr_for('comp', name)
|
|
2576
|
2594
|
return failBecause('bad stdout',
|
|
2577
|
2595
|
stderr=read_stderr(name),
|
|
2578
|
|
- stdout=read_stdout(name),
|
|
|
2596
|
+ stdout=None if stdout_match.diff else read_stdout(name),
|
|
2579
|
2597
|
diff=stdout_match.diff)
|
|
2580
|
2598
|
return passed()
|
|
2581
|
2599
|
|
| ... |
... |
@@ -2633,13 +2651,13 @@ async def stdout_ok(name: TestName, way: WayName) -> CompareOutput: |
|
2633
|
2651
|
def read_stdout( name: TestName ) -> str:
|
|
2634
|
2652
|
path = in_testdir(name, 'run.stdout')
|
|
2635
|
2653
|
if path.exists():
|
|
2636
|
|
- return path.read_text(encoding='UTF-8')
|
|
|
2654
|
+ return path.read_text(encoding='UTF-8', errors='replace')
|
|
2637
|
2655
|
else:
|
|
2638
|
2656
|
return ''
|
|
2639
|
2657
|
|
|
2640
|
2658
|
def read_diff( diff_file: Path ) -> Optional[str]:
|
|
2641
|
2659
|
if diff_file.exists():
|
|
2642
|
|
- diff = diff_file.read_text()
|
|
|
2660
|
+ diff = diff_file.read_text(encoding='UTF-8', errors='replace')
|
|
2643
|
2661
|
diff_file.unlink()
|
|
2644
|
2662
|
return diff or None
|
|
2645
|
2663
|
else:
|
| ... |
... |
@@ -2663,14 +2681,14 @@ async def stderr_ok(name: TestName, way: WayName) -> CompareOutput: |
|
2663
|
2681
|
def read_comp_stderr( name: TestName ) -> str:
|
|
2664
|
2682
|
path = in_testdir(name, 'comp.stderr')
|
|
2665
|
2683
|
if path.exists():
|
|
2666
|
|
- return path.read_text(encoding='UTF-8')
|
|
|
2684
|
+ return path.read_text(encoding='UTF-8', errors='replace')
|
|
2667
|
2685
|
else:
|
|
2668
|
2686
|
return ''
|
|
2669
|
2687
|
|
|
2670
|
2688
|
def read_stderr_for( phase: str, name: TestName ) -> str:
|
|
2671
|
2689
|
path = in_testdir(name, phase + '.stderr')
|
|
2672
|
2690
|
if path.exists():
|
|
2673
|
|
- return path.read_text(encoding='UTF-8')
|
|
|
2691
|
+ return path.read_text(encoding='UTF-8', errors='replace')
|
|
2674
|
2692
|
else:
|
|
2675
|
2693
|
return ''
|
|
2676
|
2694
|
|
| ... |
... |
@@ -3569,12 +3587,50 @@ def findTFiles(roots: List[str]) -> Iterator[str]: |
|
3569
|
3587
|
# -----------------------------------------------------------------------------
|
|
3570
|
3588
|
# Output a test summary to the specified file object
|
|
3571
|
3589
|
|
|
3572
|
|
-def summary(t: TestRun, file: TextIO, color=False) -> None:
|
|
|
3590
|
+def summary(t: TestRun, file: TextIO, color=False, junit_path: Optional[Path]=None) -> None:
|
|
3573
|
3591
|
|
|
3574
|
3592
|
file.write('\n')
|
|
|
3593
|
+
|
|
|
3594
|
+ if t.unexpected_failures:
|
|
|
3595
|
+ # Count output blocks rather than results: a test failing in many ways
|
|
|
3596
|
+ # collapses to a single block.
|
|
|
3597
|
+ groups = groupTestOutput(t.unexpected_failures)
|
|
|
3598
|
+ if len(groups) <= MAX_SUMMARY_OUTPUT_TESTS:
|
|
|
3599
|
+ printTestOutputSummary(file, groups, color, junit_path)
|
|
|
3600
|
+ else:
|
|
|
3601
|
+ where = '; see {}'.format(junit_path) if junit_path else ''
|
|
|
3602
|
+ header = ('Unexpected failures (more than {}, output omitted{}):'
|
|
|
3603
|
+ .format(MAX_SUMMARY_OUTPUT_TESTS, where))
|
|
|
3604
|
+ file.write(colored_if(color, Color.RED, header) + '\n')
|
|
|
3605
|
+ printTestInfosSummary(file, t.unexpected_failures)
|
|
|
3606
|
+
|
|
|
3607
|
+ if t.unexpected_passes:
|
|
|
3608
|
+ header = 'Unexpected passes:'
|
|
|
3609
|
+ file.write(colored_if(color, Color.RED, header) + '\n')
|
|
|
3610
|
+ printTestInfosSummary(file, t.unexpected_passes)
|
|
|
3611
|
+
|
|
|
3612
|
+ if t.unexpected_stat_failures:
|
|
|
3613
|
+ header = 'Unexpected stat failures:'
|
|
|
3614
|
+ file.write(colored_if(color, Color.RED, header) + '\n')
|
|
|
3615
|
+ printTestInfosSummary(file, t.unexpected_stat_failures)
|
|
|
3616
|
+
|
|
|
3617
|
+ if t.framework_failures:
|
|
|
3618
|
+ header = 'Framework failures:'
|
|
|
3619
|
+ file.write(colored_if(color, Color.RED, header) + '\n')
|
|
|
3620
|
+ printTestInfosSummary(file, t.framework_failures)
|
|
|
3621
|
+
|
|
|
3622
|
+ if t.framework_warnings:
|
|
|
3623
|
+ header = 'Framework warnings:'
|
|
|
3624
|
+ file.write(colored_if(color, Color.YELLOW, header) + '\n')
|
|
|
3625
|
+ printTestInfosSummary(file, t.framework_warnings)
|
|
|
3626
|
+
|
|
|
3627
|
+ if stopping():
|
|
|
3628
|
+ warning = 'WARNING: Testsuite run was terminated early'
|
|
|
3629
|
+ file.write(colored_if(color, Color.YELLOW, warning) + '\n')
|
|
|
3630
|
+
|
|
3575
|
3631
|
printUnexpectedTests(file,
|
|
3576
|
3632
|
[t.unexpected_passes, t.unexpected_failures,
|
|
3577
|
|
- t.unexpected_stat_failures, t.framework_failures])
|
|
|
3633
|
+ t.unexpected_stat_failures, t.framework_failures], color)
|
|
3578
|
3634
|
|
|
3579
|
3635
|
if len(t.unexpected_failures) > 0 or \
|
|
3580
|
3636
|
len(t.unexpected_stat_failures) > 0 or \
|
| ... |
... |
@@ -3585,7 +3641,8 @@ def summary(t: TestRun, file: TextIO, color=False) -> None: |
|
3585
|
3641
|
summary_color = Color.GREEN
|
|
3586
|
3642
|
|
|
3587
|
3643
|
assert t.start_time is not None
|
|
3588
|
|
- file.write(colored(summary_color, 'SUMMARY') + ' for test run started at '
|
|
|
3644
|
+ summary_header = colored_if(color, summary_color, 'SUMMARY')
|
|
|
3645
|
+ file.write(summary_header + ' for test run started at '
|
|
3589
|
3646
|
+ t.start_time.strftime("%c %Z") + '\n'
|
|
3590
|
3647
|
+ str(datetime.datetime.now() - t.start_time).rjust(8)
|
|
3591
|
3648
|
+ ' spent to go through\n'
|
| ... |
... |
@@ -3617,46 +3674,107 @@ def summary(t: TestRun, file: TextIO, color=False) -> None: |
|
3617
|
3674
|
+ ' fragile tests\n'
|
|
3618
|
3675
|
+ '\n')
|
|
3619
|
3676
|
|
|
3620
|
|
- if t.unexpected_passes:
|
|
3621
|
|
- file.write('Unexpected passes:\n')
|
|
3622
|
|
- printTestInfosSummary(file, t.unexpected_passes)
|
|
3623
|
|
-
|
|
3624
|
|
- if t.unexpected_failures:
|
|
3625
|
|
- file.write('Unexpected failures:\n')
|
|
3626
|
|
- printTestInfosSummary(file, t.unexpected_failures)
|
|
3627
|
|
-
|
|
3628
|
|
- if t.unexpected_stat_failures:
|
|
3629
|
|
- file.write('Unexpected stat failures:\n')
|
|
3630
|
|
- printTestInfosSummary(file, t.unexpected_stat_failures)
|
|
3631
|
|
-
|
|
3632
|
|
- if t.framework_failures:
|
|
3633
|
|
- file.write('Framework failures:\n')
|
|
3634
|
|
- printTestInfosSummary(file, t.framework_failures)
|
|
3635
|
|
-
|
|
3636
|
|
- if t.framework_warnings:
|
|
3637
|
|
- file.write('Framework warnings:\n')
|
|
3638
|
|
- printTestInfosSummary(file, t.framework_warnings)
|
|
3639
|
|
-
|
|
3640
|
|
- if stopping():
|
|
3641
|
|
- file.write('WARNING: Testsuite run was terminated early\n')
|
|
3642
|
|
-
|
|
3643
|
|
-def printUnexpectedTests(file: TextIO, testInfoss):
|
|
|
3677
|
+def printUnexpectedTests(file: TextIO, testInfoss, color=False):
|
|
3644
|
3678
|
unexpected = set(result.testname
|
|
3645
|
3679
|
for testInfos in testInfoss
|
|
3646
|
3680
|
for result in testInfos
|
|
3647
|
3681
|
if not result.testname.endswith('.T'))
|
|
3648
|
3682
|
if unexpected:
|
|
3649
|
|
- file.write('Unexpected results from:\n')
|
|
|
3683
|
+ header = 'Unexpected results from:'
|
|
|
3684
|
+ file.write(colored_if(color, Color.RED, header) + '\n')
|
|
3650
|
3685
|
file.write('TEST="' + ' '.join(sorted(unexpected)) + '"\n')
|
|
3651
|
3686
|
file.write('\n')
|
|
3652
|
3687
|
|
|
|
3688
|
+# Per-stream cap on a failing test's output repeated in the final summary.
|
|
|
3689
|
+MAX_SUMMARY_OUTPUT_LINES = 100
|
|
|
3690
|
+
|
|
|
3691
|
+# Above this many output blocks, skip repeating output entirely: the dump
|
|
|
3692
|
+# would drown out the summary.
|
|
|
3693
|
+MAX_SUMMARY_OUTPUT_TESTS = 20
|
|
|
3694
|
+
|
|
|
3695
|
+# Note [Redundant output in test results]
|
|
|
3696
|
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
|
3697
|
+# A failing test result carries up to three pieces of output: `diff`, `stdout`
|
|
|
3698
|
+# and `stderr`. For an output mismatch these overlap: the diff's `+` lines are
|
|
|
3699
|
+# the very stream that mismatched, normalised. Reporting both would print the
|
|
|
3700
|
+# same text twice, so the mismatching stream is dropped at the call sites in
|
|
|
3701
|
+# favour of the diff, which additionally shows what was expected. The *other*
|
|
|
3702
|
+# stream is kept: on a stdout mismatch, stderr is independent context.
|
|
|
3703
|
+#
|
|
|
3704
|
+# The drop is conditional on there being a diff at all: compare_outputs only
|
|
|
3705
|
+# runs `diff` when config.verbose >= 1, so under -v0 the stream is the only
|
|
|
3706
|
+# output there is.
|
|
|
3707
|
+#
|
|
|
3708
|
+# Note that, since the drop happens at result construction, it also affects the
|
|
|
3709
|
+# JUnit report (junit.py).
|
|
|
3710
|
+
|
|
|
3711
|
+def strip_diff_header(diff: Optional[str]) -> Optional[str]:
|
|
|
3712
|
+ # Drop diff(1)'s ---/+++ lines: they name normalised files in the test
|
|
|
3713
|
+ # directory and carry timestamps, which would also keep otherwise
|
|
|
3714
|
+ # identical failures from being grouped.
|
|
|
3715
|
+ if diff is None:
|
|
|
3716
|
+ return None
|
|
|
3717
|
+ lines = diff.split('\n')
|
|
|
3718
|
+ if len(lines) >= 2 and lines[0].startswith('--- ') and lines[1].startswith('+++ '):
|
|
|
3719
|
+ return '\n'.join(lines[2:])
|
|
|
3720
|
+ return diff
|
|
|
3721
|
+
|
|
|
3722
|
+def sorted_results(testInfos: List[TestResult]) -> List[TestResult]:
|
|
|
3723
|
+ return sorted(testInfos, key=lambda r: (r.testname.lower(), r.directory, r.way))
|
|
|
3724
|
+
|
|
|
3725
|
+# A failure-output block: a representative result, its header-stripped diff,
|
|
|
3726
|
+# and the ways that share it.
|
|
|
3727
|
+OutputGroup = Tuple[TestResult, Optional[str], List[WayName]]
|
|
|
3728
|
+
|
|
|
3729
|
+# Tests that fail identically in several ways (e.g. normal and g1) share one
|
|
|
3730
|
+# output block, with the ways collected in the header.
|
|
|
3731
|
+def groupTestOutput(testInfos: List[TestResult]) -> List[OutputGroup]:
|
|
|
3732
|
+ # Relies on dicts preserving insertion order.
|
|
|
3733
|
+ groups = {} # type: Dict[Tuple, OutputGroup]
|
|
|
3734
|
+ for result in sorted_results(testInfos):
|
|
|
3735
|
+ diff = strip_diff_header(result.diff)
|
|
|
3736
|
+ key = (result.testname, result.directory, result.reason,
|
|
|
3737
|
+ diff, result.stdout, result.stderr)
|
|
|
3738
|
+ groups.setdefault(key, (result, diff, []))[2].append(result.way)
|
|
|
3739
|
+ return list(groups.values())
|
|
|
3740
|
+
|
|
|
3741
|
+def printTestOutputSummary(file: TextIO,
|
|
|
3742
|
+ groups: List[OutputGroup],
|
|
|
3743
|
+ color: bool=False,
|
|
|
3744
|
+ junit_path: Optional[Path]=None) -> None:
|
|
|
3745
|
+ # Repeat failing tests' captured output in the summary, so one needn't
|
|
|
3746
|
+ # hunt for it earlier in a possibly very long log; see #16720.
|
|
|
3747
|
+ header = '=====> Unexpected failures output summary'
|
|
|
3748
|
+ file.write(colored_if(color, Color.RED, header) + '\n\n')
|
|
|
3749
|
+
|
|
|
3750
|
+ where = ', see {}'.format(junit_path) if junit_path else ''
|
|
|
3751
|
+ for result, diff, ways in groups:
|
|
|
3752
|
+ header = '=====> {}({}) ({}) [{}]'.format(
|
|
|
3753
|
+ result.testname, ', '.join(ways), result.directory + os.sep, result.reason)
|
|
|
3754
|
+ file.write(colored_if(color, Color.RED, header) + '\n')
|
|
|
3755
|
+ # See Note [Redundant output in test results] for why these don't overlap.
|
|
|
3756
|
+ for label, contents in [('Output diff (expected vs actual):', diff),
|
|
|
3757
|
+ ('Captured stdout:', result.stdout),
|
|
|
3758
|
+ ('Captured stderr:', result.stderr)]:
|
|
|
3759
|
+ if contents and contents.strip():
|
|
|
3760
|
+ lines = contents.rstrip('\n').split('\n')
|
|
|
3761
|
+ if len(lines) > MAX_SUMMARY_OUTPUT_LINES:
|
|
|
3762
|
+ omitted = len(lines) - MAX_SUMMARY_OUTPUT_LINES
|
|
|
3763
|
+ lines = lines[:MAX_SUMMARY_OUTPUT_LINES] \
|
|
|
3764
|
+ + ['... ({} more lines omitted{})'.format(omitted, where)]
|
|
|
3765
|
+ s = colored_if(color, Color.CYAN, label) + '\n' \
|
|
|
3766
|
+ + ''.join(l + '\n' for l in lines)
|
|
|
3767
|
+ # Test output can contain characters that file's encoding
|
|
|
3768
|
+ # cannot represent; replace rather than crash (cf safe_print).
|
|
|
3769
|
+ enc = getattr(file, 'encoding', None) or 'utf-8'
|
|
|
3770
|
+ file.write(s.encode(enc, errors='replace').decode(enc))
|
|
|
3771
|
+ footer = '<===== end of unexpected failures output summary'
|
|
|
3772
|
+ file.write(colored_if(color, Color.RED, footer) + '\n\n')
|
|
|
3773
|
+
|
|
3653
|
3774
|
def printTestInfosSummary(file: TextIO, testInfos):
|
|
3654
|
|
- maxDirLen = max(len(tr.directory) for tr in testInfos)
|
|
3655
|
|
- for result in sorted(testInfos, key=lambda r: (r.testname.lower(), r.way, r.directory)):
|
|
3656
|
|
- directory = result.directory.ljust(maxDirLen)
|
|
3657
|
|
- file.write(' {directory} {r.testname} [{r.reason}] ({r.way})\n'.format(
|
|
3658
|
|
- r = result,
|
|
3659
|
|
- directory = directory))
|
|
|
3775
|
+ for result in sorted_results(testInfos):
|
|
|
3776
|
+ path = os.path.join(result.directory, result.testname)
|
|
|
3777
|
+ file.write(' {path} [{r.reason}] ({r.way})\n'.format(r=result, path=path))
|
|
3660
|
3778
|
file.write('\n')
|
|
3661
|
3779
|
|
|
3662
|
3780
|
def modify_lines(s: str, f: Callable[[str], str]) -> str:
|