Simon Jakobi pushed to branch wip/sjakobi/T27602 at Glasgow Haskell Compiler / GHC Commits: 41bfc813 by Simon Jakobi at 2026-08-15T17:37:37+02:00 testsuite: Show baseline sample count and spread in perf failures A perf baseline is the mean of all samples recorded for a commit, so a single outlier can silently corrupt it. Previously, the failure output gave no hint about such outliers: the baseline printed as one number. In #27602, T27336's peak_megabytes_allocated baseline showed as 757 when the underlying samples were 605 and 909. When the baseline is averaged from more than one sample, say so in the failure message and list the samples, both in the one-line stat-failure reason and in the detail block. Single-sample baselines print exactly as before. Context: #27602 Assisted-by: Claude Fable 5 - - - - - a8a5f938 by Simon Jakobi at 2026-08-15T17:37:37+02:00 ci: Clarify comment on pushing perf notes after failures Context: #27602 Assisted-by: Claude Fable 5 - - - - - a9c078d6 by Simon Jakobi at 2026-08-15T17:37:37+02:00 DEMO: Tighten T3064 residency tolerance (do not merge) The previous demo attempts (T4830, T9675) failed for two reasons: MR pipelines pin the perf baseline to the merge-base with master (CI_MERGE_REQUEST_DIFF_BASE_SHA), and that merge-base had only one recorded sample per test env — multi-sample notes accumulate only on master commits that were the tip when a nightly pipeline ran, since nightly jobs share TEST_ENV with their regular counterparts. This branch is now rebased onto b757727a786, the newest master commit with nightly samples. Its T3064 max_bytes_used baseline on aarch64-linux-deb13-validate is bimodal (14626304 vs 13329528, mean \~13.98M), so a 1% window cannot contain either mode and the failure shows a baseline averaged from genuinely disagreeing samples. aarch64-darwin-validate and x86_64-linux-fedora43-release have two near-identical samples each, demoing the agreeing-samples output if they fail too. Context: #27602 Assisted-by: Claude Fable 5 - - - - - 3 changed files: - .gitlab/ci.sh - testsuite/driver/perf_notes.py - testsuite/tests/perf/compiler/all.T Changes: ===================================== .gitlab/ci.sh ===================================== @@ -1120,9 +1120,10 @@ case ${1:-help} in setup) setup && cleanup_submodules ;; configure) time_it "configure" configure ;; build_hadrian) time_it "build" build_hadrian ;; - # N.B. Always push notes, even if the build fails. This is okay to do as the - # testsuite driver doesn't record notes for tests that fail due to - # correctness. + # N.B. Always push notes, even if the build fails. Metrics from runs failing + # a perf stat check are deliberately recorded too — discarding them would + # bias the baseline towards whichever sample came first. Only correctness + # failures record nothing. test_hadrian) fetch_perf_notes res=0 ===================================== testsuite/driver/perf_notes.py ===================================== @@ -84,8 +84,11 @@ PerfStat = NamedTuple('PerfStat', [('test_env', TestEnv), ('value', float)]) # A baseline recovered form stored metrics. -Baseline = NamedTuple('Baseline', [('perfStat', PerfStat), - ('commit', GitHash)]) +class Baseline(NamedTuple): + perfStat: PerfStat + commit: GitHash + # The raw samples the baseline value was averaged over. + samples: List[float] = [] # The type of exceptions which are thrown when computing the current stat value # fails. @@ -465,6 +468,10 @@ def get_allowed_changes(baseline_ref: Optional[GitRef]) -> Dict[TestName, List[A # (bool , str ) -> (str , str , str , str) -> float _commit_metric_cache = {} # type: ignore +# Like _commit_metric_cache, but mapping to the list of raw sample values the +# baseline was averaged over. Filled by get_commit_metric. +_commit_samples_cache = {} # type: ignore + # Get the baseline of a test at a given commit. This is the expected value # *before* the commit is applied (i.e. on the parent commit). # This searches git notes from older commits for recorded metrics (locally and @@ -506,7 +513,8 @@ def baseline_metric(commit: GitHash, if baseline_commit is not None: current_metric = get_commit_metric(namespace, baseline_commit, test_env, name, metric, way) if current_metric is not None: - return Baseline(current_metric, baseline_commit) + return Baseline(current_metric, baseline_commit, + get_commit_samples(namespace, baseline_commit, test_env, name, metric, way)) else: return None @@ -515,7 +523,8 @@ def baseline_metric(commit: GitHash, # Check for a metric on this commit. current_metric = get_commit_metric(namespace, current_commit, test_env, name, metric, way) if current_metric is not None: - return Baseline(current_metric, current_commit) + return Baseline(current_metric, current_commit, + get_commit_samples(namespace, current_commit, test_env, name, metric, way)) # Stop if there is an expected change at this commit. In that case # metrics on ancestor commits will not be a valid baseline. @@ -598,8 +607,28 @@ def get_commit_metric(gitNoteRef, # Save baselines to the cache. _commit_metric_cache[cacheKeyA] = baseline_by_cache_key_b + _commit_samples_cache[cacheKeyA] = values_by_cache_key_b return baseline_by_cache_key_b.get(cacheKeyB) +# Get the raw sample values that get_commit_metric averages over. Uses the +# cache filled by get_commit_metric, so no extra git calls after it has run. +def get_commit_samples(gitNoteRef, + ref: Union[GitRef, GitHash], + test_env: TestEnv, + name: TestName, + metric: MetricName, + way: WayName + ) -> List[float]: + get_commit_metric(gitNoteRef, ref, test_env, name, metric, way) + cacheKeyA = (gitNoteRef, commit_hash(ref)) + cacheKeyB = (test_env, name, metric, way) + return _commit_samples_cache.get(cacheKeyA, {}).get(cacheKeyB, []) + +# Metric samples are integral in practice; '%g' would render large byte +# counts in truncated scientific notation. +def format_samples(samples: List[float]) -> str: + return ', '.join(str(int(s)) if s == int(s) else str(s) for s in samples) + def check_stats_change(actual: PerfStat, baseline: Baseline, acceptance_window: MetricAcceptanceWindow, @@ -654,9 +683,17 @@ def check_stats_change(actual: PerfStat, ' baseline @ %s' % baseline.commit print(actual.metric, error + ':') dev = 100.0 if expected_val == 0 else round(((float(actual.value) * 100) / int(expected_val)) - 100, 1) + # A multi-sample baseline is a mean; show the samples so outliers + # corrupting the baseline are visible (#27602). + if len(baseline.samples) > 1: + samples_note = ('; baseline is mean of %d samples: %s' + % (len(baseline.samples), + format_samples(baseline.samples))) + else: + samples_note = '' change_line = (f'{actual.metric} {change.value} from {baseline.perfStat.test_env} ' f'baseline @ {baseline.commit[:7]}: {expected_val} -> {actual.value} ' - f'({dev:+g}%, allowed {acceptance_window.describe()})') + f'({dev:+g}%, allowed {acceptance_window.describe()}{samples_note})') result = failBecause('stat ' + change_line, tag='stat') if not change_allowed or force_print: @@ -666,6 +703,10 @@ def check_stats_change(actual: PerfStat, print(descr, str(val).rjust(length), extra) display(' Expected ' + full_name + ' ' + actual.metric + ':', expected_val, acceptance_window.describe()) + if len(baseline.samples) > 1: + display(' Samples ' + full_name + ' ' + actual.metric + ':', + len(baseline.samples), + '(' + format_samples(baseline.samples) + ')') display(' Lower bound ' + full_name + ' ' + actual.metric + ':', lowerBound, '') display(' Upper bound ' + full_name + ' ' + actual.metric + ':', upperBound, '') display(' Actual ' + full_name + ' ' + actual.metric + ':', actual.value, '') ===================================== testsuite/tests/perf/compiler/all.T ===================================== @@ -48,7 +48,9 @@ test('T4801', ['']) test('T3064', - [collect_compiler_residency(20), + [# Demo-only: tightened from 20 so the bimodal max_bytes_used + # baseline on aarch64-linux-deb13-validate fails. + collect_compiler_residency(1), collect_compiler_runtime(2), only_ways(['normal']), ], View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/2965e0a244131ceafd4982bd0285810... -- View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/2965e0a244131ceafd4982bd0285810... You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications: https://gitlab.haskell.org/-/profile/notifications | Help: https://gitlab.haskell.org/help