Marge Bot pushed to branch wip/marge_bot_batch_merge_job at Glasgow Haskell Compiler / GHC
Commits:
-
1ffe999f
by Simon Jakobi at 2026-08-20T08:07:25-04:00
-
3827f24e
by Simon Jakobi at 2026-08-20T08:07:25-04:00
-
ed4d3273
by Simon Jakobi at 2026-08-20T08:07:26-04:00
-
8bb8a88e
by Alan Zimmerman at 2026-08-20T08:07:26-04:00
12 changed files:
- .gitlab/ci.sh
- compiler/GHC/Data/BooleanFormula.hs
- compiler/GHC/Hs/Dump.hs
- compiler/GHC/Iface/Ext/Ast.hs
- compiler/GHC/Iface/Syntax.hs
- compiler/GHC/IfaceToCore.hs
- compiler/GHC/Parser.y
- compiler/GHC/Parser/Annotation.hs
- testsuite/driver/perf_notes.py
- testsuite/driver/testglobals.py
- utils/check-exact/ExactPrint.hs
- utils/haddock/haddock-api/src/Haddock/Types.hs
Changes:
| ... | ... | @@ -1120,9 +1120,10 @@ case ${1:-help} in |
| 1120 | 1120 | setup) setup && cleanup_submodules ;;
|
| 1121 | 1121 | configure) time_it "configure" configure ;;
|
| 1122 | 1122 | build_hadrian) time_it "build" build_hadrian ;;
|
| 1123 | - # N.B. Always push notes, even if the build fails. This is okay to do as the
|
|
| 1124 | - # testsuite driver doesn't record notes for tests that fail due to
|
|
| 1125 | - # correctness.
|
|
| 1123 | + # N.B. Always push notes, even if the build fails. Metrics from runs failing
|
|
| 1124 | + # a perf stat check are deliberately recorded too — discarding them would
|
|
| 1125 | + # bias the baseline towards whichever sample came first. Only correctness
|
|
| 1126 | + # failures record nothing.
|
|
| 1126 | 1127 | test_hadrian)
|
| 1127 | 1128 | fetch_perf_notes
|
| 1128 | 1129 | res=0
|
| ... | ... | @@ -25,7 +25,7 @@ import GHC.Types.Unique |
| 25 | 25 | import GHC.Types.Unique.Set
|
| 26 | 26 | import GHC.Types.SrcLoc (unLoc)
|
| 27 | 27 | import GHC.Utils.Outputable
|
| 28 | -import GHC.Parser.Annotation ( SrcSpanAnnBF )
|
|
| 28 | +import GHC.Parser.Annotation ( SrcSpanAnnA, EpToken(..) )
|
|
| 29 | 29 | import GHC.Hs.Extension (GhcPass (..), OutputableBndrId)
|
| 30 | 30 | import Language.Haskell.Syntax.Extension (Anno, LIdP, IdP,
|
| 31 | 31 | noExtField, NoExtField, DataConCantHappen,
|
| ... | ... | @@ -38,12 +38,12 @@ import Language.Haskell.Syntax.BooleanFormula |
| 38 | 38 | -- Boolean formula type and smart constructors
|
| 39 | 39 | ----------------------------------------------------------------------
|
| 40 | 40 | |
| 41 | -type instance Anno (BooleanFormula (GhcPass p)) = SrcSpanAnnBF
|
|
| 41 | +type instance Anno (BooleanFormula (GhcPass p)) = SrcSpanAnnA
|
|
| 42 | 42 | |
| 43 | 43 | type instance XBFVar (GhcPass _) = NoExtField
|
| 44 | 44 | type instance XBFAnd (GhcPass _) = NoExtField
|
| 45 | 45 | type instance XBFOr (GhcPass _) = NoExtField
|
| 46 | -type instance XBFParens (GhcPass _) = NoExtField
|
|
| 46 | +type instance XBFParens (GhcPass _) = (EpToken "(", EpToken ")")
|
|
| 47 | 47 | type instance XXBooleanFormula (GhcPass _) = DataConCantHappen
|
| 48 | 48 | |
| 49 | 49 | instance BooleanFormulaDefault (GhcPass p) where
|
| ... | ... | @@ -97,7 +97,6 @@ showAstData bs ba a0 = blankLine $$ showAstData' a0 |
| 97 | 97 | `ext2Q` located
|
| 98 | 98 | `extQ` srcSpanAnnA
|
| 99 | 99 | `extQ` srcSpanAnnN
|
| 100 | - `extQ` srcSpanAnnBF
|
|
| 101 | 100 | |
| 102 | 101 | where generic :: Data a => a -> SDoc
|
| 103 | 102 | generic t = parens $ text (showConstr (toConstr t))
|
| ... | ... | @@ -396,10 +395,6 @@ showAstData bs ba a0 = blankLine $$ showAstData' a0 |
| 396 | 395 | srcSpanAnnN :: EpAnn NameAnn -> SDoc
|
| 397 | 396 | srcSpanAnnN = locatedAnn'' (text "SrcSpanAnnN")
|
| 398 | 397 | |
| 399 | - srcSpanAnnBF :: EpAnn AnnBooleanFormula -> SDoc
|
|
| 400 | - srcSpanAnnBF = locatedAnn'' (text "SrcSpanAnnBF")
|
|
| 401 | - |
|
| 402 | - |
|
| 403 | 398 | locatedAnn'' :: forall a. (Typeable a, Data a)
|
| 404 | 399 | => SDoc -> EpAnn a -> SDoc
|
| 405 | 400 | locatedAnn'' tag ss = parens $
|
| ... | ... | @@ -2081,7 +2081,7 @@ instance ToHie PendingRnSplice where |
| 2081 | 2081 | toHie (PendingRnSplice _ e) = toHie e
|
| 2082 | 2082 | |
| 2083 | 2083 | instance (HiePass p, Data (IdGhcP p))
|
| 2084 | - => ToHie (GenLocated SrcSpanAnnBF (BooleanFormula (GhcPass p))) where
|
|
| 2084 | + => ToHie (GenLocated SrcSpanAnnA (BooleanFormula (GhcPass p))) where
|
|
| 2085 | 2085 | toHie (L span form) = concatM $ makeNode form (locA span) : case form of
|
| 2086 | 2086 | Var _ a ->
|
| 2087 | 2087 | [ toHie $ C Use a
|
| ... | ... | @@ -1330,7 +1330,7 @@ pprIfaceDecl ss decl@(IfaceClass { ifName = clas |
| 1330 | 1330 | fromIfaceBooleanFormula (IfVar nm ) = Var noExtField $ noLocA . mkUnboundName . mkVarOccFS . ifLclNameFS $ nm
|
| 1331 | 1331 | fromIfaceBooleanFormula (IfAnd bfs ) = And noExtField $ map (noLocA . fromIfaceBooleanFormula) bfs
|
| 1332 | 1332 | fromIfaceBooleanFormula (IfOr bfs ) = Or noExtField $ map (noLocA . fromIfaceBooleanFormula) bfs
|
| 1333 | - fromIfaceBooleanFormula (IfParens bf) = Parens noExtField $ (noLocA . fromIfaceBooleanFormula) bf
|
|
| 1333 | + fromIfaceBooleanFormula (IfParens bf) = Parens noAnn $ (noLocA . fromIfaceBooleanFormula) bf
|
|
| 1334 | 1334 | |
| 1335 | 1335 | |
| 1336 | 1336 | -- See Note [Suppressing binder signatures] in GHC.Iface.Type
|
| ... | ... | @@ -121,7 +121,7 @@ import GHC.Types.Tickish |
| 121 | 121 | import GHC.Types.TyThing
|
| 122 | 122 | import GHC.Types.Error
|
| 123 | 123 | |
| 124 | -import GHC.Parser.Annotation (noLocA)
|
|
| 124 | +import GHC.Parser.Annotation (noLocA, noAnn)
|
|
| 125 | 125 | |
| 126 | 126 | import GHC.Fingerprint
|
| 127 | 127 | |
| ... | ... | @@ -885,7 +885,7 @@ tc_iface_decl _parent ignore_prags |
| 885 | 885 | tc_boolean_formula :: IfaceBooleanFormula -> IfL (BooleanFormula GhcRn)
|
| 886 | 886 | tc_boolean_formula (IfAnd ibfs ) = BF.And NoExtField . map noLocA <$> traverse tc_boolean_formula ibfs
|
| 887 | 887 | tc_boolean_formula (IfOr ibfs ) = BF.Or NoExtField . map noLocA <$> traverse tc_boolean_formula ibfs
|
| 888 | - tc_boolean_formula (IfParens ibf) = BF.Parens NoExtField . noLocA <$> tc_boolean_formula ibf
|
|
| 888 | + tc_boolean_formula (IfParens ibf) = BF.Parens noAnn . noLocA <$> tc_boolean_formula ibf
|
|
| 889 | 889 | tc_boolean_formula (IfVar nm ) = BF.Var NoExtField . noLocA <$> (lookupIfaceTop . mkVarOccFS . ifLclNameFS $ nm)
|
| 890 | 890 | |
| 891 | 891 | mk_sc_doc pred = text "Superclass" <+> ppr pred
|
| ... | ... | @@ -3820,7 +3820,7 @@ name_boolformula_opt :: { LBooleanFormula GhcPs } |
| 3820 | 3820 | name_boolformula :: { LBooleanFormula GhcPs }
|
| 3821 | 3821 | : name_boolformula_and { $1 }
|
| 3822 | 3822 | | name_boolformula_and '|' name_boolformula
|
| 3823 | - {% do { h <- addTrailingVbarBF $1 (epTok $2)
|
|
| 3823 | + {% do { h <- addTrailingVbarA $1 (epTok $2)
|
|
| 3824 | 3824 | ; return (sLLa $1 $> (Or noExtField [h,$3])) } }
|
| 3825 | 3825 | |
| 3826 | 3826 | name_boolformula_and :: { LBooleanFormula GhcPs }
|
| ... | ... | @@ -3830,12 +3830,11 @@ name_boolformula_and :: { LBooleanFormula GhcPs } |
| 3830 | 3830 | name_boolformula_and_list :: { NonEmpty (LBooleanFormula GhcPs) }
|
| 3831 | 3831 | : name_boolformula_atom { NE.singleton $1 }
|
| 3832 | 3832 | | name_boolformula_atom ',' name_boolformula_and_list
|
| 3833 | - {% do { h <- addTrailingCommaBF $1 (epTok $2)
|
|
| 3833 | + {% do { h <- addTrailingCommaA $1 (epTok $2)
|
|
| 3834 | 3834 | ; return (h NE.<| $3) } }
|
| 3835 | 3835 | |
| 3836 | 3836 | name_boolformula_atom :: { LBooleanFormula GhcPs }
|
| 3837 | - : '(' name_boolformula ')' {% amsr (sLL $1 $> (Parens noExtField $2))
|
|
| 3838 | - (AnnBooleanFormula (epTok $1) (epTok $3) []) }
|
|
| 3837 | + : '(' name_boolformula ')' {% amsA' (sLL $1 $> (Parens (epTok $1, epTok $3) $2)) }
|
|
| 3839 | 3838 | | name_var { sL1a $1 (Var noExtField $1) }
|
| 3840 | 3839 | |
| 3841 | 3840 | namelist :: { Located [LocatedN RdrName] }
|
| ... | ... | @@ -4794,20 +4793,6 @@ addTrailingAnnA (L anns a) tok ta = do |
| 4794 | 4793 | |
| 4795 | 4794 | -- -------------------------------------
|
| 4796 | 4795 | |
| 4797 | -addTrailingVbarBF :: MonadP m => LocatedBF a -> EpToken "|" -> m (LocatedBF a)
|
|
| 4798 | -addTrailingVbarBF la tok = addTrailingAnnBF la (AddVbarAnn tok)
|
|
| 4799 | - |
|
| 4800 | -addTrailingCommaBF :: MonadP m => LocatedBF a -> EpToken "," -> m (LocatedBF a)
|
|
| 4801 | -addTrailingCommaBF la tok = addTrailingAnnBF la (AddCommaAnn tok)
|
|
| 4802 | - |
|
| 4803 | -addTrailingAnnBF :: MonadP m => LocatedBF a -> TrailingAnn -> m (LocatedBF a)
|
|
| 4804 | -addTrailingAnnBF (L anns a) ta = do
|
|
| 4805 | - !cs <- getCommentsFor (locA anns)
|
|
| 4806 | - let anns' = addTrailingAnnToBF ta cs anns
|
|
| 4807 | - return (L anns' a)
|
|
| 4808 | - |
|
| 4809 | --- -------------------------------------
|
|
| 4810 | - |
|
| 4811 | 4796 | -- Mostly use to add AnnComma, special case it to NOP if adding a zero-width annotation
|
| 4812 | 4797 | addTrailingCommaN :: MonadP m => LocatedN a -> SrcSpan -> m (LocatedN a)
|
| 4813 | 4798 | addTrailingCommaN (L anns a) span = do
|
| ... | ... | @@ -28,23 +28,19 @@ module GHC.Parser.Annotation ( |
| 28 | 28 | |
| 29 | 29 | -- ** Annotations in 'GenLocated'
|
| 30 | 30 | LocatedA, LocatedN, LocatedAn,
|
| 31 | - LocatedBF,
|
|
| 32 | 31 | SrcSpanAnnA, SrcSpanAnnN,
|
| 33 | - SrcSpanAnnBF,
|
|
| 34 | 32 | |
| 35 | 33 | -- ** Annotation data types used in 'GenLocated'
|
| 36 | 34 | |
| 37 | 35 | AnnList(..), AnnListBrackets(..),
|
| 38 | 36 | AnnParen(..),
|
| 39 | 37 | AnnCType(..),AnnWarningTxt(..),AnnOverlap(..),AnnAnnDecl(..),AnnPragSCC(..),
|
| 40 | - AnnBooleanFormula(..),
|
|
| 41 | 38 | NameAnn(..), NameAdornment(..),
|
| 42 | 39 | NoEpAnns(..),
|
| 43 | 40 | |
| 44 | 41 | -- ** Trailing annotations in lists
|
| 45 | 42 | TrailingAnn(..), ta_location,
|
| 46 | 43 | addTrailingAnnToA, addTrailingCommaToN,
|
| 47 | - addTrailingAnnToBF,
|
|
| 48 | 44 | noTrailingN,
|
| 49 | 45 | |
| 50 | 46 | -- ** Utilities for converting between different 'GenLocated' when
|
| ... | ... | @@ -430,8 +426,6 @@ emptyComments = EpaComments [] |
| 430 | 426 | type LocatedA = GenLocated SrcSpanAnnA
|
| 431 | 427 | type LocatedN = GenLocated SrcSpanAnnN
|
| 432 | 428 | |
| 433 | -type LocatedBF = GenLocated SrcSpanAnnBF
|
|
| 434 | - |
|
| 435 | 429 | -- | Annotation for items appearing in a list. They can have one or
|
| 436 | 430 | -- more trailing punctuations items, such as commas or semicolons.
|
| 437 | 431 | type SrcSpanAnnA = EpAnn [TrailingAnn]
|
| ... | ... | @@ -440,8 +434,6 @@ type SrcSpanAnnA = EpAnn [TrailingAnn] |
| 440 | 434 | -- on the context, such as backticks.
|
| 441 | 435 | type SrcSpanAnnN = EpAnn NameAnn
|
| 442 | 436 | |
| 443 | -type SrcSpanAnnBF = EpAnn AnnBooleanFormula
|
|
| 444 | - |
|
| 445 | 437 | -- | General representation of a 'GenLocated' type carrying a
|
| 446 | 438 | -- parameterised annotation type.
|
| 447 | 439 | type LocatedAn an = GenLocated (EpAnn an)
|
| ... | ... | @@ -551,17 +543,6 @@ data AnnParen |
| 551 | 543 | | AnnParensHash (EpToken "(#") (EpToken "#)") -- ^ '(#', '#)'
|
| 552 | 544 | deriving Data
|
| 553 | 545 | |
| 554 | --- ---------------------------------------------------------------------
|
|
| 555 | --- | Exact print annotation for the 'BooleanFormula' data type.
|
|
| 556 | - |
|
| 557 | -data AnnBooleanFormula
|
|
| 558 | - = AnnBooleanFormula {
|
|
| 559 | - abf_open :: (EpToken "("), -- ^ opening parenthesis.
|
|
| 560 | - abf_close :: (EpToken ")"), -- ^ closing parenthesis.
|
|
| 561 | - abf_trailing :: ![TrailingAnn] -- ^ items appearing after the
|
|
| 562 | - -- item, such as '|', ','
|
|
| 563 | - } deriving (Data,Eq)
|
|
| 564 | - |
|
| 565 | 546 | -- ---------------------------------------------------------------------
|
| 566 | 547 | -- Annotations for names
|
| 567 | 548 | -- ---------------------------------------------------------------------
|
| ... | ... | @@ -669,14 +650,6 @@ data AnnPragSCC |
| 669 | 650 | |
| 670 | 651 | -- ---------------------------------------------------------------------
|
| 671 | 652 | |
| 672 | -addTrailingAnnToBF :: TrailingAnn -> EpAnnComments
|
|
| 673 | - -> EpAnn AnnBooleanFormula -> EpAnn AnnBooleanFormula
|
|
| 674 | -addTrailingAnnToBF t cs n = n { anns = addTrailing (anns n)
|
|
| 675 | - , comments = comments n <> cs }
|
|
| 676 | - where
|
|
| 677 | - -- See Note [list append in addTrailing*]
|
|
| 678 | - addTrailing n = n { abf_trailing = abf_trailing n ++ [t]}
|
|
| 679 | - |
|
| 680 | 653 | -- | Helper function used in the parser to add a 'TrailingAnn' items
|
| 681 | 654 | -- to an existing annotation.
|
| 682 | 655 | addTrailingAnnToA :: TrailingAnn -> EpAnnComments
|
| ... | ... | @@ -1030,9 +1003,6 @@ instance (NoAnn ann) => NoAnn (EpAnn ann) where |
| 1030 | 1003 | instance NoAnn NoEpAnns where
|
| 1031 | 1004 | noAnn = NoEpAnns
|
| 1032 | 1005 | |
| 1033 | -instance NoAnn AnnBooleanFormula where
|
|
| 1034 | - noAnn = AnnBooleanFormula noAnn noAnn []
|
|
| 1035 | - |
|
| 1036 | 1006 | instance NoAnn AnnList where
|
| 1037 | 1007 | noAnn = AnnList Nothing ListNone noAnn
|
| 1038 | 1008 |
| ... | ... | @@ -83,9 +83,13 @@ PerfStat = NamedTuple('PerfStat', [('test_env', TestEnv), |
| 83 | 83 | ('metric', MetricName),
|
| 84 | 84 | ('value', float)])
|
| 85 | 85 | |
| 86 | -# A baseline recovered form stored metrics.
|
|
| 87 | -Baseline = NamedTuple('Baseline', [('perfStat', PerfStat),
|
|
| 88 | - ('commit', GitHash)])
|
|
| 86 | +# A test's metric recovered from a commit's git note: the raw sample values
|
|
| 87 | +# recorded there, and a PerfStat whose value is their mean. Serves as the
|
|
| 88 | +# baseline when comparing a test run against an earlier commit.
|
|
| 89 | +class CommitMetric(NamedTuple):
|
|
| 90 | + perfStat: PerfStat
|
|
| 91 | + commit: GitHash
|
|
| 92 | + samples: List[float]
|
|
| 89 | 93 | |
| 90 | 94 | # The type of exceptions which are thrown when computing the current stat value
|
| 91 | 95 | # fails.
|
| ... | ... | @@ -460,10 +464,10 @@ def get_allowed_changes(baseline_ref: Optional[GitRef]) -> Dict[TestName, List[A |
| 460 | 464 | else:
|
| 461 | 465 | return get_allowed_perf_changes()
|
| 462 | 466 | |
| 463 | -# Cache of baseline values. This is a dict of dicts indexed on:
|
|
| 464 | -# (useCiNamespace, commit) -> (test_env, test, metric, way) -> baseline
|
|
| 465 | -# (bool , str ) -> (str , str , str , str) -> float
|
|
| 466 | -_commit_metric_cache = {} # type: ignore
|
|
| 467 | +# Cache of commit metrics.
|
|
| 468 | +_commit_metric_cache: Dict[Tuple[NoteNamespace, GitHash],
|
|
| 469 | + Dict[Tuple[TestEnv, TestName, MetricName, WayName],
|
|
| 470 | + CommitMetric]] = {}
|
|
| 467 | 471 | |
| 468 | 472 | # Get the baseline of a test at a given commit. This is the expected value
|
| 469 | 473 | # *before* the commit is applied (i.e. on the parent commit).
|
| ... | ... | @@ -477,7 +481,7 @@ _commit_metric_cache = {} # type: ignore |
| 477 | 481 | # instead when looking for ci results)
|
| 478 | 482 | # metric: str - test metric
|
| 479 | 483 | # way: str - test way
|
| 480 | -# returns: the Baseline or None if no metric was found within
|
|
| 484 | +# returns: the baseline CommitMetric or None if no metric was found within
|
|
| 481 | 485 | # BaselineSearchDepth commits and since the last expected change
|
| 482 | 486 | # (ignoring any expected change in the given commit).
|
| 483 | 487 | def baseline_metric(commit: GitHash,
|
| ... | ... | @@ -486,7 +490,7 @@ def baseline_metric(commit: GitHash, |
| 486 | 490 | metric: MetricName,
|
| 487 | 491 | way: WayName,
|
| 488 | 492 | baseline_ref: Optional[GitRef]
|
| 489 | - ) -> Optional[Baseline]:
|
|
| 493 | + ) -> Optional[CommitMetric]:
|
|
| 490 | 494 | # For performance reasons (in order to avoid calling commit_hash), we assert
|
| 491 | 495 | # commit is already a commit hash.
|
| 492 | 496 | assert is_commit_hash(commit)
|
| ... | ... | @@ -502,20 +506,16 @@ def baseline_metric(commit: GitHash, |
| 502 | 506 | # Searches through previous commits trying local then ci for each commit in.
|
| 503 | 507 | def find_baseline(namespace: NoteNamespace,
|
| 504 | 508 | test_env: TestEnv
|
| 505 | - ) -> Optional[Baseline]:
|
|
| 509 | + ) -> Optional[CommitMetric]:
|
|
| 506 | 510 | if baseline_commit is not None:
|
| 507 | - current_metric = get_commit_metric(namespace, baseline_commit, test_env, name, metric, way)
|
|
| 508 | - if current_metric is not None:
|
|
| 509 | - return Baseline(current_metric, baseline_commit)
|
|
| 510 | - else:
|
|
| 511 | - return None
|
|
| 511 | + return get_commit_metric(namespace, baseline_commit, test_env, name, metric, way)
|
|
| 512 | 512 | |
| 513 | 513 | for depth, current_commit in list(enumerate(commit_hashes)):
|
| 514 | 514 | if current_commit == commit: continue
|
| 515 | 515 | # Check for a metric on this commit.
|
| 516 | 516 | current_metric = get_commit_metric(namespace, current_commit, test_env, name, metric, way)
|
| 517 | 517 | if current_metric is not None:
|
| 518 | - return Baseline(current_metric, current_commit)
|
|
| 518 | + return current_metric
|
|
| 519 | 519 | |
| 520 | 520 | # Stop if there is an expected change at this commit. In that case
|
| 521 | 521 | # metrics on ancestor commits will not be a valid baseline.
|
| ... | ... | @@ -527,7 +527,7 @@ def baseline_metric(commit: GitHash, |
| 527 | 527 | # Test environment to use when comparing against CI namespace
|
| 528 | 528 | ci_test_env = best_fit_ci_test_env()
|
| 529 | 529 | |
| 530 | - baseline = find_baseline(LocalNamespace, test_env) # type: Optional[Baseline]
|
|
| 530 | + baseline = find_baseline(LocalNamespace, test_env) # type: Optional[CommitMetric]
|
|
| 531 | 531 | if baseline is None and ci_test_env is not None:
|
| 532 | 532 | baseline = find_baseline(CiNamespace, ci_test_env)
|
| 533 | 533 | |
| ... | ... | @@ -545,23 +545,23 @@ def get_commit_metric_value_str_or_none(gitNoteRef, |
| 545 | 545 | result = get_commit_metric(gitNoteRef, commit, test_env, name, metric, way)
|
| 546 | 546 | if result is None:
|
| 547 | 547 | return None
|
| 548 | - return str(result.value)
|
|
| 548 | + return str(result.perfStat.value)
|
|
| 549 | 549 | |
| 550 | -# gets the average commit metric from git notes.
|
|
| 550 | +# gets the commit metric (average and raw samples) from git notes.
|
|
| 551 | 551 | # gitNoteRef: git notes ref space e.g. "perf" or "ci/perf"
|
| 552 | 552 | # ref: git commit
|
| 553 | 553 | # test_env: test environment
|
| 554 | 554 | # name: test name
|
| 555 | 555 | # metric: test metric
|
| 556 | 556 | # way: test way
|
| 557 | -# returns: PerfStat | None if stats don't exist for the given input
|
|
| 557 | +# returns: CommitMetric | None if stats don't exist for the given input
|
|
| 558 | 558 | def get_commit_metric(gitNoteRef,
|
| 559 | 559 | ref: Union[GitRef, GitHash],
|
| 560 | 560 | test_env: TestEnv,
|
| 561 | 561 | name: TestName,
|
| 562 | 562 | metric: MetricName,
|
| 563 | 563 | way: WayName
|
| 564 | - ) -> Optional[PerfStat]:
|
|
| 564 | + ) -> Optional[CommitMetric]:
|
|
| 565 | 565 | global _commit_metric_cache
|
| 566 | 566 | assert test_env != None
|
| 567 | 567 | commit = commit_hash(ref)
|
| ... | ... | @@ -573,9 +573,9 @@ def get_commit_metric(gitNoteRef, |
| 573 | 573 | return _commit_metric_cache[cacheKeyA].get(cacheKeyB)
|
| 574 | 574 | |
| 575 | 575 | # Cache miss.
|
| 576 | - # Calculate baselines from the current commit's git note.
|
|
| 576 | + # Calculate metrics from the current commit's git note.
|
|
| 577 | 577 | # Note that the git note may contain data for other tests. All tests'
|
| 578 | - # baselines will be collected and cached for future use.
|
|
| 578 | + # metrics will be collected and cached for future use.
|
|
| 579 | 579 | allCommitMetrics = get_perf_stats(ref, gitNoteRef)
|
| 580 | 580 | |
| 581 | 581 | # Collect recorded values by cacheKeyB.
|
| ... | ... | @@ -586,22 +586,32 @@ def get_commit_metric(gitNoteRef, |
| 586 | 586 | currentValues = values_by_cache_key_b.setdefault(currentCacheKey, [])
|
| 587 | 587 | currentValues.append(float(perfStat.value))
|
| 588 | 588 | |
| 589 | - # Calculate and baseline (average of values) by cacheKeyB.
|
|
| 590 | - baseline_by_cache_key_b = {}
|
|
| 589 | + # Calculate the metric (average of values, plus the values themselves)
|
|
| 590 | + # by cacheKeyB.
|
|
| 591 | + metric_by_cache_key_b = {}
|
|
| 591 | 592 | for currentCacheKey, currentValues in values_by_cache_key_b.items():
|
| 592 | - baseline_by_cache_key_b[currentCacheKey] = PerfStat( \
|
|
| 593 | - currentCacheKey[0],
|
|
| 594 | - currentCacheKey[1],
|
|
| 595 | - currentCacheKey[3],
|
|
| 596 | - currentCacheKey[2],
|
|
| 597 | - sum(currentValues) / len(currentValues))
|
|
| 598 | - |
|
| 599 | - # Save baselines to the cache.
|
|
| 600 | - _commit_metric_cache[cacheKeyA] = baseline_by_cache_key_b
|
|
| 601 | - return baseline_by_cache_key_b.get(cacheKeyB)
|
|
| 593 | + metric_by_cache_key_b[currentCacheKey] = CommitMetric(
|
|
| 594 | + PerfStat(
|
|
| 595 | + currentCacheKey[0],
|
|
| 596 | + currentCacheKey[1],
|
|
| 597 | + currentCacheKey[3],
|
|
| 598 | + currentCacheKey[2],
|
|
| 599 | + sum(currentValues) / len(currentValues)),
|
|
| 600 | + commit,
|
|
| 601 | + currentValues)
|
|
| 602 | + |
|
| 603 | + # Save metrics to the cache.
|
|
| 604 | + _commit_metric_cache[cacheKeyA] = metric_by_cache_key_b
|
|
| 605 | + return metric_by_cache_key_b.get(cacheKeyB)
|
|
| 606 | + |
|
| 607 | +def format_sample(s: float) -> str:
|
|
| 608 | + return str(int(s)) if s == int(s) else str(s)
|
|
| 609 | + |
|
| 610 | +def format_samples(samples: List[float]) -> str:
|
|
| 611 | + return ', '.join(format_sample(s) for s in samples)
|
|
| 602 | 612 | |
| 603 | 613 | def check_stats_change(actual: PerfStat,
|
| 604 | - baseline: Baseline,
|
|
| 614 | + baseline: CommitMetric,
|
|
| 605 | 615 | acceptance_window: MetricAcceptanceWindow,
|
| 606 | 616 | allowed_perf_changes: Dict[TestName, List[AllowedPerfChange]] = {},
|
| 607 | 617 | force_print = False
|
| ... | ... | @@ -611,8 +621,8 @@ def check_stats_change(actual: PerfStat, |
| 611 | 621 | |
| 612 | 622 | Parameters:
|
| 613 | 623 | actual: the PerfStat with actual value
|
| 614 | - baseline: the expected Baseline value (this should generally be derived
|
|
| 615 | - from baseline_metric())
|
|
| 624 | + baseline: the CommitMetric to compare against (this should generally be
|
|
| 625 | + derived from baseline_metric())
|
|
| 616 | 626 | acceptance_window: allowed deviation of the actual value from the expected
|
| 617 | 627 | value.
|
| 618 | 628 | allowed_perf_changes: allowed changes in stats. This is a dictionary as
|
| ... | ... | @@ -654,9 +664,17 @@ def check_stats_change(actual: PerfStat, |
| 654 | 664 | ' baseline @ %s' % baseline.commit
|
| 655 | 665 | print(actual.metric, error + ':')
|
| 656 | 666 | dev = 100.0 if expected_val == 0 else round(((float(actual.value) * 100) / int(expected_val)) - 100, 1)
|
| 667 | + # Show the sample spread so unreliable baselines become visible (#27602).
|
|
| 668 | + if len(baseline.samples) > 1:
|
|
| 669 | + samples_note = ('; baseline is mean of %d samples spanning %s..%s'
|
|
| 670 | + % (len(baseline.samples),
|
|
| 671 | + format_sample(min(baseline.samples)),
|
|
| 672 | + format_sample(max(baseline.samples))))
|
|
| 673 | + else:
|
|
| 674 | + samples_note = ''
|
|
| 657 | 675 | change_line = (f'{actual.metric} {change.value} from {baseline.perfStat.test_env} '
|
| 658 | 676 | f'baseline @ {baseline.commit[:7]}: {expected_val} -> {actual.value} '
|
| 659 | - f'({dev:+g}%, allowed {acceptance_window.describe()})')
|
|
| 677 | + f'({dev:+g}%, allowed {acceptance_window.describe()}{samples_note})')
|
|
| 660 | 678 | result = failBecause('stat ' + change_line, tag='stat')
|
| 661 | 679 | |
| 662 | 680 | if not change_allowed or force_print:
|
| ... | ... | @@ -666,6 +684,10 @@ def check_stats_change(actual: PerfStat, |
| 666 | 684 | print(descr, str(val).rjust(length), extra)
|
| 667 | 685 | |
| 668 | 686 | display(' Expected ' + full_name + ' ' + actual.metric + ':', expected_val, acceptance_window.describe())
|
| 687 | + if len(baseline.samples) > 1:
|
|
| 688 | + display(' Samples ' + full_name + ' ' + actual.metric + ':',
|
|
| 689 | + len(baseline.samples),
|
|
| 690 | + '(' + format_samples(baseline.samples) + ')')
|
|
| 669 | 691 | display(' Lower bound ' + full_name + ' ' + actual.metric + ':', lowerBound, '')
|
| 670 | 692 | display(' Upper bound ' + full_name + ' ' + actual.metric + ':', upperBound, '')
|
| 671 | 693 | display(' Actual ' + full_name + ' ' + actual.metric + ':', actual.value, '')
|
| ... | ... | @@ -866,7 +888,7 @@ def main() -> None: |
| 866 | 888 | # HEAD~2 21234 21234
|
| 867 | 889 | # HEAD~3 20000 20000
|
| 868 | 890 | def strMetric(x):
|
| 869 | - return '{:.2f}'.format(x.value) if x != None else ""
|
|
| 891 | + return '{:.2f}'.format(x.perfStat.value) if x != None else ""
|
|
| 870 | 892 | # Data is in column major format, so transpose and pass to print_table.
|
| 871 | 893 | T = TypeVar('T')
|
| 872 | 894 | def transpose(xss: List[List[T]]) -> List[List[T]]:
|
| ... | ... | @@ -4,7 +4,7 @@ |
| 4 | 4 | |
| 5 | 5 | from my_typing import *
|
| 6 | 6 | from pathlib import Path
|
| 7 | -from perf_notes import MetricChange, PerfStat, Baseline, GitRef
|
|
| 7 | +from perf_notes import MetricChange, PerfStat, CommitMetric, GitRef
|
|
| 8 | 8 | from datetime import datetime
|
| 9 | 9 | |
| 10 | 10 | # -----------------------------------------------------------------------------
|
| ... | ... | @@ -312,7 +312,7 @@ class TestResult: |
| 312 | 312 | PerfMetric = NamedTuple('PerfMetric',
|
| 313 | 313 | [('change', MetricChange),
|
| 314 | 314 | ('stat', PerfStat),
|
| 315 | - ('baseline', Optional[Baseline]) ])
|
|
| 315 | + ('baseline', Optional[CommitMetric]) ])
|
|
| 316 | 316 | |
| 317 | 317 | class TestRun:
|
| 318 | 318 | def __init__(self) -> None:
|
| ... | ... | @@ -353,10 +353,6 @@ instance HasTrailing (EpToken "{", EpToken "}") where |
| 353 | 353 | trailing _ = []
|
| 354 | 354 | setTrailing a _ = a
|
| 355 | 355 | |
| 356 | -instance HasTrailing (AnnBooleanFormula) where
|
|
| 357 | - trailing bf = abf_trailing bf
|
|
| 358 | - setTrailing a ts = a { abf_trailing = ts }
|
|
| 359 | - |
|
| 360 | 356 | -- ---------------------------------------------------------------------
|
| 361 | 357 | |
| 362 | 358 | fromAnn' :: (HasEntry a) => a -> Entry
|
| ... | ... | @@ -2731,9 +2727,11 @@ instance ExactPrint (BF.BooleanFormula GhcPs) where |
| 2731 | 2727 | exact (BF.And e ls) = do
|
| 2732 | 2728 | ls' <- mapM markAnnotated ls
|
| 2733 | 2729 | return (BF.And e ls')
|
| 2734 | - exact (BF.Parens e x) = do
|
|
| 2730 | + exact (BF.Parens (o,c) x) = do
|
|
| 2731 | + o' <- markEpToken o
|
|
| 2735 | 2732 | x' <- markAnnotated x
|
| 2736 | - return (BF.Parens e x')
|
|
| 2733 | + c' <- markEpToken c
|
|
| 2734 | + return (BF.Parens (o',c') x')
|
|
| 2737 | 2735 | |
| 2738 | 2736 | -- ---------------------------------------------------------------------
|
| 2739 | 2737 | |
| ... | ... | @@ -4474,17 +4472,6 @@ instance ExactPrint [LocatedA (StmtLR GhcPs GhcPs (LocatedA (HsCmd GhcPs)))] whe |
| 4474 | 4472 | stmts' <- markAnnotated stmts
|
| 4475 | 4473 | return stmts'
|
| 4476 | 4474 | |
| 4477 | -instance ExactPrint (LocatedBF (BF.BooleanFormula GhcPs)) where
|
|
| 4478 | - getAnnotationEntry = entryFromLocatedA
|
|
| 4479 | - setAnnotationAnchor = setAnchorAn
|
|
| 4480 | - exact (L an bf) = do
|
|
| 4481 | - debugM $ "LocatedCB [LBooleanFormula"
|
|
| 4482 | - let (AnnBooleanFormula op cp ta) = anns an
|
|
| 4483 | - op' <- markEpToken op
|
|
| 4484 | - bf' <- markAnnotated bf
|
|
| 4485 | - cp' <- markEpToken cp
|
|
| 4486 | - return (L (an {anns = AnnBooleanFormula op' cp' ta}) bf')
|
|
| 4487 | - |
|
| 4488 | 4475 | instance ExactPrint [Located HsDocStringChunk] where
|
| 4489 | 4476 | getAnnotationEntry _ = NoEntryVal
|
| 4490 | 4477 | setAnnotationAnchor a _ _ _ = a
|
| ... | ... | @@ -834,7 +834,7 @@ type instance Anno (HsDecl DocNameI) = SrcSpanAnnA |
| 834 | 834 | type instance Anno (FamilyResultSig DocNameI) = EpAnn NoEpAnns
|
| 835 | 835 | type instance Anno (HsOuterTyVarBndrs Specificity DocNameI) = SrcSpanAnnA
|
| 836 | 836 | type instance Anno (HsSigType DocNameI) = SrcSpanAnnA
|
| 837 | -type instance Anno (BooleanFormula DocNameI) = SrcSpanAnnBF
|
|
| 837 | +type instance Anno (BooleanFormula DocNameI) = SrcSpanAnnA
|
|
| 838 | 838 | type instance Anno (OverlapMode DocNameI) = SrcSpanAnnA
|
| 839 | 839 | type instance Anno (CType DocNameI) = SrcSpanAnnA
|
| 840 | 840 | type instance Anno (Header DocNameI) = SrcSpanAnnA
|
| ... | ... | @@ -1041,7 +1041,7 @@ type instance XXHsContextDetails DocNameI = DataConCantHappen |
| 1041 | 1041 | type instance XBFVar DocNameI = NoExtField
|
| 1042 | 1042 | type instance XBFAnd DocNameI = NoExtField
|
| 1043 | 1043 | type instance XBFOr DocNameI = NoExtField
|
| 1044 | -type instance XBFParens DocNameI = NoExtField
|
|
| 1044 | +type instance XBFParens DocNameI = (EpToken "(", EpToken ")")
|
|
| 1045 | 1045 | type instance XXBooleanFormula DocNameI = DataConCantHappen
|
| 1046 | 1046 | |
| 1047 | 1047 | -----------------------------------------------------------------------------
|