Richard Eisenberg pushed to branch wip/rae/skip-tests at Glasgow Haskell Compiler / GHC Commits: 88b53793 by Richard Eisenberg at 2026-04-16T10:54:22-04:00 Teach hadrian to --skip-test. Fixes #27188. This adds the --skip-test flag to `hadrian build`, as documented in the patch. - - - - - 8 changed files: - + changelog.d/skip-test - hadrian/doc/make.md - hadrian/doc/testsuite.md - hadrian/src/CommandLine.hs - hadrian/src/Settings/Builders/RunTest.hs - testsuite/driver/runtests.py - testsuite/driver/testglobals.py - testsuite/driver/testlib.py Changes: ===================================== changelog.d/skip-test ===================================== @@ -0,0 +1,13 @@ +section: build-tools +synopsis: The testsuite driver and Hadrian now support a ``--skip-test`` flag + (``--skip`` in the Python driver) to skip specific tests. +issues: #27188 +mrs: !15915 + +description: { + A new ``--skip-test`` flag has been added to Hadrian, passed through + as ``--skip`` to the testsuite driver. This is the complement of + ``--only``: it removes named tests from the set of tests to run. + If a test appears in both ``--only`` and ``--skip-test``, it is skipped. + Multiple ``--skip-test`` flags are unioned together. +} ===================================== hadrian/doc/make.md ===================================== @@ -133,7 +133,8 @@ time you fire up a build. This is not possible with the Make build system. As illustrated in the examples above, you can use the `TEST` environment variable, the `--only=...` flag or even both to restrict your testsuite run - to some (usually small) subset of the testsuite. + to some (usually small) subset of the testsuite. You can also skip specific + tests with `--skip-test=...` (e.g. `build test --skip-test="test1 test2"`). See [the docs for the test rules](./testsuite.md) if you want to know about all the options that hadrian supports and what they correspond to in the Make ===================================== hadrian/doc/testsuite.md ===================================== @@ -60,6 +60,29 @@ TEST="test1 test2" build test TEST="test1 test2" build test --only="test3 test4" ``` +### Skipping specific tests + +You can use the `--skip-test=...` command line argument to skip specific +tests. This is useful when some tests are known to fail or are too slow +in your environment. + +``` sh +# skip the test named 'slowtest' +build test --skip-test=slowtest + +# skip 'test1' and 'test2' +build test --skip-test="test1 test2" +``` + +`--skip-test` can be combined with `--only`: if a test appears in both, +it is skipped. This lets you start from a set of tests and then subtract +from it: + +``` sh +# run test1 and test3, but not test2 +build test --only="test1 test2 test3" --skip-test=test2 +``` + ### Whole directories of tests You can also ask Hadrian to run all the tests that live under one or ===================================== hadrian/src/CommandLine.hs ===================================== @@ -71,6 +71,7 @@ data TestArgs = TestArgs , testJUnit :: Maybe FilePath , testMetricsFile:: Maybe FilePath , testOnly :: [String] + , testSkip :: [String] , testOnlyPerf :: Bool , testSkipPerf :: Bool , testRootDirs :: [FilePath] @@ -100,6 +101,7 @@ defaultTestArgs = TestArgs , testJUnit = Nothing , testMetricsFile= Nothing , testOnly = [] + , testSkip = [] , testOnlyPerf = False , testSkipPerf = False , testRootDirs = [] @@ -191,6 +193,13 @@ readTestOnly tests = Right $ \flags -> where tests' = maybe [] words tests tests'' flags = testOnly (testArgs flags) ++ tests' +readTestSkip :: Maybe String -> Either String (CommandLineArgs -> CommandLineArgs) +readTestSkip tests = Right $ \flags -> + flags { testArgs = (testArgs flags) { testSkip = tests'' flags } } + + where tests' = maybe [] words tests + tests'' flags = testSkip (testArgs flags) ++ tests' + readTestOnlyPerf :: Either String (CommandLineArgs -> CommandLineArgs) readTestOnlyPerf = Right $ \flags -> flags { testArgs = (testArgs flags) { testOnlyPerf = True } } @@ -306,6 +315,8 @@ optDescrs = "Output testsuite performance metrics summary." , Option [] ["only"] (OptArg readTestOnly "TESTS") "Test cases to run." + , Option [] ["skip-test"] (OptArg readTestSkip "TESTS") + "Test cases to skip." , Option [] ["only-perf"] (NoArg readTestOnlyPerf) "Only run performance tests." , Option [] ["skip-perf"] (NoArg readTestSkipPerf) ===================================== hadrian/src/Settings/Builders/RunTest.hs ===================================== @@ -360,6 +360,7 @@ getTestArgs = do haveDocs <- willDocsBeBuilt let configFileArg= ["--config-file=" ++ (testConfigFile args)] testOnlyArg = map ("--only=" ++) (testOnly args ++ testEnvTargets) + testSkipArg = map ("--skip=" ++) (testSkip args) onlyPerfArg = if testOnlyPerf args then Just "--only-perf-tests" else Nothing @@ -399,7 +400,7 @@ getTestArgs = do inTreeArg = [ "-e", "config.in_tree_compiler=" ++ show (isInTreeCompiler (testCompiler args) || testHasInTreeFiles args) ] - pure $ configFileArg ++ testOnlyArg ++ speedArg + pure $ configFileArg ++ testOnlyArg ++ testSkipArg ++ speedArg ++ catMaybes [ onlyPerfArg, skipPerfArg, summaryArg , junitArg, metricsArg, verbosityArg ] ++ configArgs ++ wayArgs ++ compilerArg ++ ghcPkgArg ===================================== testsuite/driver/runtests.py ===================================== @@ -79,6 +79,7 @@ parser.add_argument("--summary-file", help="file in which to save the (human-rea parser.add_argument("--unexpected-output-dir", help="directory in which to place unexpected output") parser.add_argument("--target-wrapper", help="wrapper executable to use when executing binaries compiled for the target") parser.add_argument("--only", action="append", help="just this test (can be give multiple --only= flags)") +parser.add_argument("--skip", action="append", help="skip this test (can be given multiple --skip= flags)") parser.add_argument("--way", action="append", help="just this way") parser.add_argument("--skipway", action="append", help="skip this way") parser.add_argument("--threads", type=int, help="threads to run simultaneously") @@ -135,6 +136,9 @@ if args.only: config.only = args.only config.run_only_some_tests = True +if args.skip: + config.skip = set(args.skip) + if args.way: for way in args.way: if way not in all_ways: ===================================== testsuite/driver/testglobals.py ===================================== @@ -31,6 +31,9 @@ class TestConfig: self.run_only_some_tests = False self.only = set() + # Skip these tests + self.skip = set() + # Don't fail on out-of-tolerance stat failures self.ignore_perf_increases = False self.ignore_perf_decreases = False ===================================== testsuite/driver/testlib.py ===================================== @@ -1526,6 +1526,9 @@ def test(name: TestName, # report on any tests we couldn't find and error out. config.only.remove(name) + if name in config.skip: + return + # Make a deep copy of the default_testopts, as we need our own copy # of any dictionaries etc inside it. Otherwise, if one test modifies # them, all tests will see the modified version! View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/88b53793e20352300617a91513da3d84... -- View it on GitLab: https://gitlab.haskell.org/ghc/ghc/-/commit/88b53793e20352300617a91513da3d84... You're receiving this email because of your account on gitlab.haskell.org.