Marge Bot pushed to branch master at Glasgow Haskell Compiler / GHC

Commits:

8 changed files:

Changes:

  • changelog.d/skip-test
    1 
    +section: build-tools
    2 
    +synopsis: The testsuite driver and Hadrian now support a ``--skip-test`` flag
    3 
    +          (``--skip`` in the Python driver) to skip specific tests.
    4 
    +issues: #27188
    5 
    +mrs: !15915
    6 
    +
    7 
    +description: {
    8 
    +    A new ``--skip-test`` flag has been added to Hadrian, passed through
    9 
    +    as ``--skip`` to the testsuite driver. This is the complement of
    10 
    +    ``--only``: it removes named tests from the set of tests to run.
    11 
    +    If a test appears in both ``--only`` and ``--skip-test``, it is skipped.
    12 
    +    Multiple ``--skip-test`` flags are unioned together.
    13 
    +}

  • hadrian/doc/make.md
    ... ... @@ -133,7 +133,8 @@ time you fire up a build. This is not possible with the Make build system.
    133 133
    134 134 
       As illustrated in the examples above, you can use the `TEST` environment
    135 135 
       variable, the `--only=...` flag or even both to restrict your testsuite run
    136 
    -  to some (usually small) subset of the testsuite.
    136 
    +  to some (usually small) subset of the testsuite. You can also skip specific
    137 
    +  tests with `--skip-test=...` (e.g. `build test --skip-test="test1 test2"`).
    137 138
    138 139 
       See [the docs for the test rules](./testsuite.md) if you want to know about
    139 140 
       all the options that hadrian supports and what they correspond to in the Make

  • hadrian/doc/testsuite.md
    ... ... @@ -60,6 +60,29 @@ TEST="test1 test2" build test
    60 60 
     TEST="test1 test2" build test --only="test3 test4"
    61 61 
     ```
    62 62
    63 
    +### Skipping specific tests
    64 
    +
    65 
    +You can use the `--skip-test=...` command line argument to skip specific
    66 
    +tests. This is useful when some tests are known to fail or are too slow
    67 
    +in your environment.
    68 
    +
    69 
    +``` sh
    70 
    +# skip the test named 'slowtest'
    71 
    +build test --skip-test=slowtest
    72 
    +
    73 
    +# skip 'test1' and 'test2'
    74 
    +build test --skip-test="test1 test2"
    75 
    +```
    76 
    +
    77 
    +`--skip-test` can be combined with `--only`: if a test appears in both,
    78 
    +it is skipped. This lets you start from a set of tests and then subtract
    79 
    +from it:
    80 
    +
    81 
    +``` sh
    82 
    +# run test1 and test3, but not test2
    83 
    +build test --only="test1 test2 test3" --skip-test=test2
    84 
    +```
    85 
    +
    63 86 
     ### Whole directories of tests
    64 87
    65 88 
     You can also ask Hadrian to run all the tests that live under one or

  • hadrian/src/CommandLine.hs
    ... ... @@ -71,6 +71,7 @@ data TestArgs = TestArgs
    71 71 
         , testJUnit      :: Maybe FilePath
    72 72 
         , testMetricsFile:: Maybe FilePath
    73 73 
         , testOnly       :: [String]
    74 
    +    , testSkip       :: [String]
    74 75 
         , testOnlyPerf   :: Bool
    75 76 
         , testSkipPerf   :: Bool
    76 77 
         , testRootDirs   :: [FilePath]
    ... ... @@ -100,6 +101,7 @@ defaultTestArgs = TestArgs
    100 101 
         , testJUnit      = Nothing
    101 102 
         , testMetricsFile= Nothing
    102 103 
         , testOnly       = []
    104 
    +    , testSkip       = []
    103 105 
         , testOnlyPerf   = False
    104 106 
         , testSkipPerf   = False
    105 107 
         , testRootDirs   = []
    ... ... @@ -191,6 +193,13 @@ readTestOnly tests = Right $ \flags ->
    191 193 
       where tests' = maybe [] words tests
    192 194 
             tests'' flags = testOnly (testArgs flags) ++ tests'
    193 195
    196 
    +readTestSkip :: Maybe String -> Either String (CommandLineArgs -> CommandLineArgs)
    197 
    +readTestSkip tests = Right $ \flags ->
    198 
    +  flags { testArgs = (testArgs flags) { testSkip = tests'' flags } }
    199 
    +
    200 
    +  where tests' = maybe [] words tests
    201 
    +        tests'' flags = testSkip (testArgs flags) ++ tests'
    202 
    +
    194 203 
     readTestOnlyPerf :: Either String (CommandLineArgs -> CommandLineArgs)
    195 204 
     readTestOnlyPerf = Right $ \flags -> flags { testArgs = (testArgs flags) { testOnlyPerf = True } }
    196 205
    ... ... @@ -306,6 +315,8 @@ optDescrs =
    306 315 
           "Output testsuite performance metrics summary."
    307 316 
         , Option [] ["only"] (OptArg readTestOnly "TESTS")
    308 317 
           "Test cases to run."
    318 
    +    , Option [] ["skip-test"] (OptArg readTestSkip "TESTS")
    319 
    +      "Test cases to skip."
    309 320 
         , Option [] ["only-perf"] (NoArg readTestOnlyPerf)
    310 321 
           "Only run performance tests."
    311 322 
         , Option [] ["skip-perf"] (NoArg readTestSkipPerf)

  • hadrian/src/Settings/Builders/RunTest.hs
    ... ... @@ -360,6 +360,7 @@ getTestArgs = do
    360 360 
         haveDocs        <- willDocsBeBuilt
    361 361 
         let configFileArg= ["--config-file=" ++ (testConfigFile args)]
    362 362 
             testOnlyArg  =  map ("--only=" ++) (testOnly args ++ testEnvTargets)
    363 
    +        testSkipArg  =  map ("--skip=" ++) (testSkip args)
    363 364 
             onlyPerfArg  = if testOnlyPerf args
    364 365 
                                then Just "--only-perf-tests"
    365 366 
                                else Nothing
    ... ... @@ -399,7 +400,7 @@ getTestArgs = do
    399 400 
             inTreeArg    = [ "-e", "config.in_tree_compiler=" ++
    400 401 
               show (isInTreeCompiler (testCompiler args) || testHasInTreeFiles args) ]
    401 402
    402 
    -    pure $  configFileArg ++ testOnlyArg ++ speedArg
    403 
    +    pure $  configFileArg ++ testOnlyArg ++ testSkipArg ++ speedArg
    403 404 
              ++ catMaybes [ onlyPerfArg, skipPerfArg, summaryArg
    404 405 
                           , junitArg, metricsArg, verbosityArg  ]
    405 406 
              ++ configArgs ++ wayArgs ++  compilerArg ++ ghcPkgArg

  • testsuite/driver/runtests.py
    ... ... @@ -79,6 +79,7 @@ parser.add_argument("--summary-file", help="file in which to save the (human-rea
    79 79 
     parser.add_argument("--unexpected-output-dir", help="directory in which to place unexpected output")
    80 80 
     parser.add_argument("--target-wrapper", help="wrapper executable to use when executing binaries compiled for the target")
    81 81 
     parser.add_argument("--only", action="append", help="just this test (can be give multiple --only= flags)")
    82 
    +parser.add_argument("--skip", action="append", help="skip this test (can be given multiple --skip= flags)")
    82 83 
     parser.add_argument("--way", action="append", help="just this way")
    83 84 
     parser.add_argument("--skipway", action="append", help="skip this way")
    84 85 
     parser.add_argument("--threads", type=int, help="threads to run simultaneously")
    ... ... @@ -135,6 +136,9 @@ if args.only:
    135 136 
         config.only = args.only
    136 137 
         config.run_only_some_tests = True
    137 138
    139 
    +if args.skip:
    140 
    +    config.skip = set(args.skip)
    141 
    +
    138 142 
     if args.way:
    139 143 
         for way in args.way:
    140 144 
             if way not in all_ways:

  • testsuite/driver/testglobals.py
    ... ... @@ -31,6 +31,9 @@ class TestConfig:
    31 31 
             self.run_only_some_tests = False
    32 32 
             self.only = set()
    33 33
    34 
    +        # Skip these tests
    35 
    +        self.skip = set()
    36 
    +
    34 37 
             # Don't fail on out-of-tolerance stat failures
    35 38 
             self.ignore_perf_increases = False
    36 39 
             self.ignore_perf_decreases = False

  • testsuite/driver/testlib.py
    ... ... @@ -1526,6 +1526,9 @@ def test(name: TestName,
    1526 1526 
                 # report on any tests we couldn't find and error out.
    1527 1527 
                 config.only.remove(name)
    1528 1528
    1529 
    +    if name in config.skip:
    1530 
    +        return
    1531 
    +
    1529 1532 
         # Make a deep copy of the default_testopts, as we need our own copy
    1530 1533 
         # of any dictionaries etc inside it. Otherwise, if one test modifies
    1531 1534 
         # them, all tests will see the modified version!

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.haskell.org. Manage all notifications · Help