From 8a0c23fc4650b6c1bf18fa8139a77ae5063092c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaroslav=20Loba=C4=8Devski?= Date: Tue, 25 Feb 2025 17:25:24 +0000 Subject: [PATCH 1/2] Use one job per worker to utilize all cores --- infra/cifuzz/fuzz_target.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infra/cifuzz/fuzz_target.py b/infra/cifuzz/fuzz_target.py index 8c9789d8a7ed..7292fd43912b 100644 --- a/infra/cifuzz/fuzz_target.py +++ b/infra/cifuzz/fuzz_target.py @@ -60,7 +60,8 @@ def get_libfuzzer_parallel_options(): """Returns a list containing options to pass to libFuzzer to fuzz using all available cores.""" - return ['-jobs=' + str(multiprocessing.cpu_count())] + cpu_count = str(multiprocessing.cpu_count()) + return [f'-jobs={cpu_count}', f'-workers={cpu_count}'] class ReproduceError(Exception): From 2ec479fa43a34eeb96b10b497ab1ba2f6458de0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaroslav=20Loba=C4=8Devski?= Date: Tue, 25 Feb 2025 11:08:14 +0000 Subject: [PATCH 2/2] Allow integers in `parallel-fuzzing` to force the number of cores --- infra/cifuzz/actions/run_fuzzers/action.yml | 2 +- infra/cifuzz/config_utils.py | 2 +- .../external-actions/run_fuzzers/action.yml | 2 +- infra/cifuzz/fuzz_target.py | 18 +++++++++++++----- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/infra/cifuzz/actions/run_fuzzers/action.yml b/infra/cifuzz/actions/run_fuzzers/action.yml index 197aed2b02e4..98f01c3bd246 100644 --- a/infra/cifuzz/actions/run_fuzzers/action.yml +++ b/infra/cifuzz/actions/run_fuzzers/action.yml @@ -45,7 +45,7 @@ inputs: required: false default: False parallel-fuzzing: - description: "Whether to use all available cores for fuzzing." + description: "How many cores to use cores for fuzzing. A specific number, True - all available cores or False to run single threaded on a single core." required: false default: false output-sarif: diff --git a/infra/cifuzz/config_utils.py b/infra/cifuzz/config_utils.py index a4a3cb8f7ad9..0e9c54780d41 100644 --- a/infra/cifuzz/config_utils.py +++ b/infra/cifuzz/config_utils.py @@ -129,7 +129,7 @@ def __init__(self): self.build_integration_path = ( constants.DEFAULT_EXTERNAL_BUILD_INTEGRATION_PATH) - self.parallel_fuzzing = environment.get_bool('PARALLEL_FUZZING', False) + self.parallel_fuzzing = environment.get('PARALLEL_FUZZING', False) self.extra_environment_variables = _get_extra_environment_variables() self.output_sarif = environment.get_bool('OUTPUT_SARIF', False) diff --git a/infra/cifuzz/external-actions/run_fuzzers/action.yml b/infra/cifuzz/external-actions/run_fuzzers/action.yml index 9ca11e448a0d..7f2e132b50f8 100644 --- a/infra/cifuzz/external-actions/run_fuzzers/action.yml +++ b/infra/cifuzz/external-actions/run_fuzzers/action.yml @@ -57,7 +57,7 @@ inputs: required: false default: False parallel-fuzzing: - description: "Whether to use all available cores for fuzzing." + description: "How many cores to use cores for fuzzing. A specific number, True - all available cores or False to run single threaded on a single core." required: false default: false output-sarif: diff --git a/infra/cifuzz/fuzz_target.py b/infra/cifuzz/fuzz_target.py index 7292fd43912b..eb3d7a7ff710 100644 --- a/infra/cifuzz/fuzz_target.py +++ b/infra/cifuzz/fuzz_target.py @@ -57,10 +57,13 @@ ['testcase', 'stacktrace', 'corpus_path']) -def get_libfuzzer_parallel_options(): +def get_libfuzzer_parallel_options(option): """Returns a list containing options to pass to libFuzzer to fuzz using all - available cores.""" - cpu_count = str(multiprocessing.cpu_count()) + available or the specified number of cores.""" + if option == 'true' or (isinstance(option, bool) and option == True): + cpu_count = str(multiprocessing.cpu_count()) + else: + cpu_count = option return [f'-jobs={cpu_count}', f'-workers={cpu_count}'] @@ -190,13 +193,18 @@ def fuzz(self, batch=False) -> Optional[FuzzResult]: if not self.config.report_ooms: options.arguments.extend(LIBFUZZER_OPTIONS_NO_REPORT_OOM) - if self.config.parallel_fuzzing: + if (self.config.parallel_fuzzing == 'true' or + (isinstance(self.config.parallel_fuzzing, int) and + not isinstance(self.config.parallel_fuzzing, bool)) or + (isinstance(self.config.parallel_fuzzing, bool) and + self.config.parallel_fuzzing == True)): if self.config.sanitizer == 'memory': # TODO(https://github.com/google/oss-fuzz/issues/11915): Don't gate # this after jobs is fixed for MSAN. logging.info('Not using jobs because it breaks MSAN.') else: - options.arguments.extend(get_libfuzzer_parallel_options()) + options.arguments.extend( + get_libfuzzer_parallel_options(self.config.parallel_fuzzing)) result = engine_impl.fuzz(self.target_path, options, artifacts_dir, self.duration)