GitLab

Simon Jakobi pushed to branch wip/sjakobi/T27602 at Glasgow Haskell Compiler / GHC

Commits:

befe88fd

by Simon Jakobi at 2026-08-16T17:54:44+02:00

testsuite: Show baseline sample count and spread in perf failures

A perf baseline is the mean of all samples recorded for a commit, so a
single outlier can silently corrupt it. Previously, the failure output
gave no hint about such outliers: the baseline printed as one number.
In #27602, T27336's peak_megabytes_allocated baseline showed as 757
when the underlying samples were 605 and 909.

When the baseline is averaged from more than one sample, say so in the
failure message and list the samples, both in the one-line stat-failure
reason and in the detail block. Single-sample baselines print exactly
as before.

Context: #27602

Assisted-by: Claude Fable 5

d54d52ea

by Simon Jakobi at 2026-08-16T17:54:44+02:00

ci: Clarify comment on pushing perf notes after failures

Context: #27602

Assisted-by: Claude Fable 5

e17c0f57

by Simon Jakobi at 2026-08-16T17:55:13+02:00

DEMO: Tighten T3064 residency tolerance (do not merge)

Assisted-by: Claude Fable 5

3 changed files:

.gitlab/ci.sh
testsuite/driver/perf_notes.py
testsuite/tests/perf/compiler/all.T

Changes:

.gitlab/ci.sh

@@ -1120,9 +1120,10 @@ case ${1:-help} in
    setup) setup && cleanup_submodules ;;
    configure) time_it "configure" configure ;;
    build_hadrian) time_it "build" build_hadrian ;;
 -  # N.B. Always push notes, even if the build fails. This is okay to do as the
 -  # testsuite driver doesn't record notes for tests that fail due to
 -  # correctness.
 +  # N.B. Always push notes, even if the build fails. Metrics from runs failing
 +  # a perf stat check are deliberately recorded too — discarding them would
 +  # bias the baseline towards whichever sample came first. Only correctness
 +  # failures record nothing.
    test_hadrian)
      fetch_perf_notes
      res=0

testsuite/driver/perf_notes.py

@@ -84,8 +84,11 @@ PerfStat = NamedTuple('PerfStat', [('test_env', TestEnv),
                                     ('value', float)])
  # A baseline recovered form stored metrics.
 -Baseline = NamedTuple('Baseline', [('perfStat', PerfStat),
 -                                   ('commit', GitHash)])
 +class Baseline(NamedTuple):
 +    perfStat: PerfStat
 +    commit: GitHash
 +    # The raw samples the baseline value was averaged over.
 +    samples: List[float] = []
  # The type of exceptions which are thrown when computing the current stat value
  # fails.
@@ -465,6 +468,10 @@ def get_allowed_changes(baseline_ref: Optional[GitRef]) -> Dict[TestName, List[A
  # (bool          , str   ) -> (str     , str , str   , str) -> float
  _commit_metric_cache = {} # type: ignore
 +# Like _commit_metric_cache, but mapping to the list of raw sample values the
 +# baseline was averaged over. Filled by get_commit_metric.
 +_commit_samples_cache = {} # type: ignore
++
  # Get the baseline of a test at a given commit. This is the expected value
  # *before* the commit is applied (i.e. on the parent commit).
  # This searches git notes from older commits for recorded metrics (locally and
@@ -506,7 +513,8 @@ def baseline_metric(commit: GitHash,
          if baseline_commit is not None:
              current_metric = get_commit_metric(namespace, baseline_commit, test_env, name, metric, way)
              if current_metric is not None:
 -                return Baseline(current_metric, baseline_commit)
 +                return Baseline(current_metric, baseline_commit,
 +                                get_commit_samples(namespace, baseline_commit, test_env, name, metric, way))
              else:
                  return None
@@ -515,7 +523,8 @@ def baseline_metric(commit: GitHash,
              # Check for a metric on this commit.
              current_metric = get_commit_metric(namespace, current_commit, test_env, name, metric, way)
              if current_metric is not None:
 -                return Baseline(current_metric, current_commit)
 +                return Baseline(current_metric, current_commit,
 +                                get_commit_samples(namespace, current_commit, test_env, name, metric, way))
              # Stop if there is an expected change at this commit. In that case
              # metrics on ancestor commits will not be a valid baseline.
@@ -598,8 +607,26 @@ def get_commit_metric(gitNoteRef,
      # Save baselines to the cache.
      _commit_metric_cache[cacheKeyA] = baseline_by_cache_key_b
 +    _commit_samples_cache[cacheKeyA] = values_by_cache_key_b
      return baseline_by_cache_key_b.get(cacheKeyB)
 +# Get the raw sample values that get_commit_metric averages over. Uses the
 +# cache filled by get_commit_metric, so no extra git calls after it has run.
 +def get_commit_samples(gitNoteRef,
 +                       ref: Union[GitRef, GitHash],
 +                       test_env: TestEnv,
 +                       name: TestName,
 +                       metric: MetricName,
 +                       way: WayName
 +                       ) -> List[float]:
 +    get_commit_metric(gitNoteRef, ref, test_env, name, metric, way)
 +    cacheKeyA = (gitNoteRef, commit_hash(ref))
 +    cacheKeyB = (test_env, name, metric, way)
 +    return _commit_samples_cache.get(cacheKeyA, {}).get(cacheKeyB, [])
++
 +def format_samples(samples: List[float]) -> str:
 +    return ', '.join(str(int(s)) if s == int(s) else str(s) for s in samples)
++
  def check_stats_change(actual: PerfStat,
                         baseline: Baseline,
                         acceptance_window: MetricAcceptanceWindow,
@@ -654,9 +681,16 @@ def check_stats_change(actual: PerfStat,
                  ' baseline @ %s' % baseline.commit
          print(actual.metric, error + ':')
          dev = 100.0 if expected_val == 0 else round(((float(actual.value) * 100) / int(expected_val)) - 100, 1)
 +        # Show the samples so outliers become visible (#27602).
 +        if len(baseline.samples) > 1:
 +            samples_note = ('; baseline is mean of %d samples: %s'
 +                            % (len(baseline.samples),
 +                               format_samples(baseline.samples)))
 +        else:
 +            samples_note = ''
          change_line = (f'{actual.metric} {change.value} from {baseline.perfStat.test_env} '
                         f'baseline @ {baseline.commit[:7]}: {expected_val} -> {actual.value} '
 -                       f'({dev:+g}%, allowed {acceptance_window.describe()})')
 +                       f'({dev:+g}%, allowed {acceptance_window.describe()}{samples_note})')
          result = failBecause('stat ' + change_line, tag='stat')
      if not change_allowed or force_print:
@@ -666,6 +700,10 @@ def check_stats_change(actual: PerfStat,
              print(descr, str(val).rjust(length), extra)
          display('    Expected    ' + full_name + ' ' + actual.metric + ':', expected_val, acceptance_window.describe())
 +        if len(baseline.samples) > 1:
 +            display('    Samples     ' + full_name + ' ' + actual.metric + ':',
 +                    len(baseline.samples),
 +                    '(' + format_samples(baseline.samples) + ')')
          display('    Lower bound ' + full_name + ' ' + actual.metric + ':', lowerBound, '')
          display('    Upper bound ' + full_name + ' ' + actual.metric + ':', upperBound, '')
          display('    Actual      ' + full_name + ' ' + actual.metric + ':', actual.value, '')

testsuite/tests/perf/compiler/all.T

@@ -48,7 +48,9 @@ test('T4801',
       [''])
  test('T3064',
 -     [collect_compiler_residency(20),
 +     [# Demo-only: tightened from 20 so the bimodal max_bytes_used
 +      # baseline on aarch64-linux-deb13-validate fails.
 +      collect_compiler_residency(1),
        collect_compiler_runtime(2),
        only_ways(['normal']),
        ],