Simon Jakobi pushed to branch wip/sjakobi/T27602 at Glasgow Haskell Compiler / GHC Commits: 5d69b025 by Simon Jakobi at 2026-08-15T11:15:42+02:00 testsuite: Show baseline sample count and spread in perf failures A perf baseline is the mean of all samples recorded for a commit, so a single outlier can silently corrupt it. Previously, the failure output gave no hint about such outliers: the baseline printed as one number. In #27602, T27336's peak_megabytes_allocated baseline showed as 757 when the underlying samples were 605 and 909. When the baseline is averaged from more than one sample, say so in the failure message and list the samples, both in the one-line stat-failure reason and in the detail block. Single-sample baselines print exactly as before. Context: #27602 Assisted-by: Claude Fable 5 - - - - - b222f971 by Simon Jakobi at 2026-08-15T11:19:30+02:00 ci: Clarify comment on pushing perf notes after failures Context: #27602 Assisted-by: Claude Fable 5 - - - - - 2 changed files: - .gitlab/ci.sh - testsuite/driver/perf_notes.py Changes: ===================================== .gitlab/ci.sh ===================================== @@ -1120,9 +1120,10 @@ case ${1:-help} in setup) setup && cleanup_submodules ;; configure) time_it "configure" configure ;; build_hadrian) time_it "build" build_hadrian ;; - # N.B. Always push notes, even if the build fails. This is okay to do as the - # testsuite driver doesn't record notes for tests that fail due to - # correctness. + # N.B. Always push notes, even if the build fails. Metrics from runs failing + # a perf stat check are deliberately recorded too — discarding them would + # bias the baseline towards whichever sample came first. Only correctness + # failures record nothing. test_hadrian) fetch_perf_notes res=0 ===================================== testsuite/driver/perf_notes.py ===================================== @@ -84,8 +84,11 @@ PerfStat = NamedTuple('PerfStat', [('test_env', TestEnv), ('value', float)]) # A baseline recovered form stored metrics. -Baseline = NamedTuple('Baseline', [('perfStat', PerfStat), - ('commit', GitHash)]) +class Baseline(NamedTuple): + perfStat: PerfStat + commit: GitHash + # The raw samples the baseline value was averaged over. + samples: List[float] = [] # The type of exceptions which are thrown when computing the current stat value # fails. @@ -465,6 +468,10 @@ def get_allowed_changes(baseline_ref: Optional[GitRef]) -> Dict[TestName, List[A # (bool , str ) -> (str , str , str , str) -> float _commit_metric_cache = {} # type: ignore +# Like _commit_metric_cache, but mapping to the list of raw sample values the +# baseline was averaged over. Filled by get_commit_metric. +_commit_samples_cache = {} # type: ignore + # Get the baseline of a test at a given commit. This is the expected value # *before* the commit is applied (i.e. on the parent commit). # This searches git notes from older commits for recorded metrics (locally and @@ -506,7 +513,8 @@ def baseline_metric(commit: GitHash, if baseline_commit is not None: current_metric = get_commit_metric(namespace, baseline_commit, test_env, name, metric, way) if current_metric is not None: - return Baseline(current_metric, baseline_commit) + return Baseline(current_metric, baseline_commit, + get_commit_samples(namespace, baseline_commit, test_env, name, metric, way)) else: return None @@ -515,7 +523,8 @@ def baseline_metric(commit: GitHash, # Check for a metric on this commit. current_metric = get_commit_metric(namespace, current_commit, test_env, name, metric, way) if current_metric is not None: - return Baseline(current_metric, current_commit) + return Baseline(current_metric, current_commit, + get_commit_samples(namespace, current_commit, test_env, name, metric, way)) # Stop if there is an expected change at this commit. In that case # metrics on ancestor commits will not be a valid baseline. @@ -598,8 +607,23 @@ def get_commit_metric(gitNoteRef, # Save baselines to the cache. _commit_metric_cache[cacheKeyA] = baseline_by_cache_key_b + _commit_samples_cache[cacheKeyA] = values_by_cache_key_b return baseline_by_cache_key_b.get(cacheKeyB) +# Get the raw sample values that get_commit_metric averages over. Uses the +# cache filled by get_commit_metric, so no extra git calls after it has run. +def get_commit_samples(gitNoteRef, + ref: Union[GitRef, GitHash], + test_env: TestEnv, + name: TestName, + metric: MetricName, + way: WayName + ) -> List[float]: + get_commit_metric(gitNoteRef, ref, test_env, name, metric, way) + cacheKeyA = (gitNoteRef, commit_hash(ref)) + cacheKeyB = (test_env, name, metric, way) + return _commit_samples_cache.get(cacheKeyA, {}).get(cacheKeyB, []) + def check_stats_change(actual: PerfStat, baseline: Baseline, acceptance_window: MetricAcceptanceWindow, @@ -654,9 +678,17 @@ def check_stats_change(actual: PerfStat, ' baseline @ %s' % baseline.commit print(actual.metric, error + ':') dev = 100.0 if expected_val == 0 else round(((float(actual.value) * 100) / int(expected_val)) - 100, 1) + # A multi-sample baseline is a mean; show the samples so outliers + # corrupting the baseline are visible (#27602). + if len(baseline.samples) > 1: + samples_note = ('; baseline is mean of %d samples: %s' + % (len(baseline.samples), + ', '.join('%g' % s for s in baseline.samples))) + else: + samples_note = '' change_line = (f'{actual.metric} {change.value} from {baseline.perfStat.test_env} ' f'baseline @ {baseline.commit[:7]}: {expected_val} -> {actual.value} ' - f'({dev:+g}%, allowed {acceptance_window.describe()})') + f'({dev:+g}%, allowed {acceptance_window.describe()}{samples_note})') result = failBecause('stat ' + change_line, tag='stat') if not change_allowed or force_print: @@ -666,6 +698,10 @@ def check_stats_change(actual: PerfStat, print(descr, str(val).rjust(length), extra) display(' Expected ' + full_name + ' ' + actual.metric + ':', expected_val, acceptance_window.describe()) + if len(baseline.samples) > 1: + display(' Samples ' + full_name + ' ' + actual.metric + ':', + len(baseline.samples), + '(' + ', '.join('%g' % s for s in baseline.samples) + ')') display(' Lower bound ' + full_name + ' ' + actual.metric + ':', lowerBound, '') display(' Upper bound ' + full_name + ' ' + actual.metric + ':', upperBound, '') display(' Actual ' + full_name + ' ' + actual.metric + ':', actual.value, '') View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/622d8907861a5edfdd1bde0a2ec0a7f... -- View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/compare/622d8907861a5edfdd1bde0a2ec0a7f... You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications: https://gitlab.haskell.org/-/profile/notifications | Help: https://gitlab.haskell.org/help