diff --git a/docs/markdown/Unit-tests.md b/docs/markdown/Unit-tests.md index 13f6093f2e7b..10bf1f7452af 100644 --- a/docs/markdown/Unit-tests.md +++ b/docs/markdown/Unit-tests.md @@ -206,6 +206,11 @@ name(s), the test name(s) must be contained in the suite(s). This however is redundant-- it would be more useful to specify either specific test names or suite(s). +Since version *1.7.0*, you can pass `--slice i/n` to split up the set of tests +into `n` slices and execute the `ith` such slice. This allows you to distribute +a set of long-running tests across multiple machines to decrease the overall +runtime of tests. + ### Other test options Sometimes you need to run the tests multiple times, which is done like this: diff --git a/docs/markdown/snippets/test-slicing.md b/docs/markdown/snippets/test-slicing.md new file mode 100644 index 000000000000..180b9ace513d --- /dev/null +++ b/docs/markdown/snippets/test-slicing.md @@ -0,0 +1,6 @@ +## New option to execute a slice of tests + +When tests take a long time to run a common strategy is to slice up the tests +into multiple sets, where each set is executed on a separate machine. You can +now use the `--slice i/n` argument for `meson test` to create `n` slices and +execute the `ith` slice. diff --git a/man/meson.1 b/man/meson.1 index 41917a97db30..df9818a8ae87 100644 --- a/man/meson.1 +++ b/man/meson.1 @@ -328,6 +328,9 @@ a multiplier to use for test timeout values (usually something like 100 for Valg .TP \fB\-\-setup\fR use the specified test setup +.Tp +\fB\-\-slice SLICE/NUM_SLICES\fR +Split tests into NUM_SLICES slices and execute slice number SLICE. (Since 1.7.0) .SH The wrap command diff --git a/mesonbuild/mtest.py b/mesonbuild/mtest.py index 39970e530872..4cf50c8c5f72 100644 --- a/mesonbuild/mtest.py +++ b/mesonbuild/mtest.py @@ -99,6 +99,29 @@ def uniwidth(s: str) -> int: result += UNIWIDTH_MAPPING[w] return result +def test_slice(arg: str) -> T.Tuple[int, int]: + values = arg.split('/') + if len(values) != 2: + raise argparse.ArgumentTypeError("value does not conform to format 'SLICE/NUM_SLICES'") + + try: + nrslices = int(values[1]) + except ValueError: + raise argparse.ArgumentTypeError("NUM_SLICES is not an integer") + if nrslices <= 0: + raise argparse.ArgumentTypeError("NUM_SLICES is not a positive integer") + + try: + subslice = int(values[0]) + except ValueError: + raise argparse.ArgumentTypeError("SLICE is not an integer") + if subslice <= 0: + raise argparse.ArgumentTypeError("SLICE is not a positive integer") + if subslice > nrslices: + raise argparse.ArgumentTypeError("SLICE exceeds NUM_SLICES") + + return subslice, nrslices + # Note: when adding arguments, please also add them to the completion # scripts in $MESONSRC/data/shell-completions/ def add_arguments(parser: argparse.ArgumentParser) -> None: @@ -149,12 +172,13 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: help='Arguments to pass to the specified test(s) or all tests') parser.add_argument('--max-lines', default=100, dest='max_lines', type=int, help='Maximum number of lines to show from a long test log. Since 1.5.0.') + parser.add_argument('--slice', default=None, type=test_slice, metavar='SLICE/NUM_SLICES', + help='Split tests into NUM_SLICES slices and execute slice SLICE. Since 1.7.0.') parser.add_argument('args', nargs='*', help='Optional list of test names to run. "testname" to run all tests with that name, ' '"subprojname:testname" to specifically run "testname" from "subprojname", ' '"subprojname:" to run all tests defined by "subprojname".') - def print_safe(s: str) -> None: end = '' if s[-1] == '\n' else '\n' try: @@ -1976,6 +2000,9 @@ def get_tests(self, errorfile: T.Optional[T.IO] = None) -> T.List[TestSerialisat tests = [t for t in self.tests if self.test_suitable(t)] if self.options.args: tests = list(self.tests_from_args(tests)) + if self.options.slice: + our_slice, nslices = self.options.slice + tests = tests[our_slice - 1::nslices] if not tests: print('No suitable tests defined.', file=errorfile) diff --git a/test cases/unit/124 test slice/meson.build b/test cases/unit/124 test slice/meson.build new file mode 100644 index 000000000000..a41c2f62d7ff --- /dev/null +++ b/test cases/unit/124 test slice/meson.build @@ -0,0 +1,12 @@ +project('test_slice') + +python = import('python').find_installation('python3') + +foreach i : range(10) + test('test-' + (i + 1).to_string(), + python, + args: [ + meson.current_source_dir() / 'test.py' + ], + ) +endforeach diff --git a/test cases/unit/124 test slice/test.py b/test cases/unit/124 test slice/test.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/unittests/allplatformstests.py b/unittests/allplatformstests.py index b5338b834a75..f94c47db89cc 100644 --- a/unittests/allplatformstests.py +++ b/unittests/allplatformstests.py @@ -5076,6 +5076,37 @@ def test_c_cpp_stds(self): self.setconf('-Dcpp_std=c++11,gnu++11,vc++11') self.assertEqual(self.getconf('cpp_std'), 'c++11') + def test_slice(self): + testdir = os.path.join(self.unit_test_dir, '124 test slice') + self.init(testdir) + self.build() + + for arg, expectation in { + '1/1': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + '1/2': [1, 3, 5, 7, 9], + '2/2': [2, 4, 6, 8, 10], + '1/10': [1], + '2/20': [2], + '10/20': [10], + '11/20': [ ], + }.items(): + output = self._run(self.mtest_command + ['--slice=' + arg]) + tests = sorted([ int(x[5:]) for x in re.findall(r'test-[0-9]*', output) ]) + self.assertEqual(tests, expectation) + + for arg, expectation in { + '': 'value does not conform to format \'SLICE/NUM_SLICES\'', + '0': 'value does not conform to format \'SLICE/NUM_SLICES\'', + '0/1': 'SLICE is not a positive integer', + 'a/1': 'SLICE is not an integer', + '1/0': 'NUM_SLICES is not a positive integer', + '1/a': 'NUM_SLICES is not an integer', + '2/1': 'SLICE exceeds NUM_SLICES', + }.items(): + with self.assertRaises(subprocess.CalledProcessError) as cm: + self._run(self.mtest_command + ['--slice=' + arg]) + self.assertIn('error: argument --slice: ' + expectation, cm.exception.output) + def test_rsp_support(self): env = get_fake_env() cc = detect_c_compiler(env, MachineChoice.HOST)