From 7c4876c9e7e9b0c4e7405121a7979a5e24a01026 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Tue, 13 Aug 2024 10:49:26 +0100 Subject: [PATCH 1/4] Added memtier_benchmark-3Mkeys-load-string-with-512B-values: Runs memtier_benchmark, for a keyspace length of 3M keys loading STRINGs in which the value has a data size of 512 Bytes, with 650 clients running sequential SET commands --- ...rk-3Mkeys-load-string-with-512B-values.yml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml new file mode 100644 index 0000000..10de3ca --- /dev/null +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml @@ -0,0 +1,30 @@ +version: 0.4 +name: memtier_benchmark-3Mkeys-load-string-with-512B-values +description: Runs memtier_benchmark, for a keyspace length of 3M keys loading STRINGs in which the value has a data size of 512 Bytes, with 650 clients running sequential SET commands. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 3g +tested-commands: +- set +redis-topologies: +- oss-standalone +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "512" --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 3000000 --test-time 180 -c 50 -t 13 --hide-histogram' + resources: + requests: + cpus: '13' + memory: 2g + +tested-groups: +- string +priority: 17 From 46ea9173739e94265172215f10b79b2abc06b21b Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Wed, 14 Aug 2024 02:35:17 +0100 Subject: [PATCH 2/4] Added tests for io-threads setups --- .../__common__/spec.py | 52 +++++++ .../__self_contained_coordinator__/docker.py | 6 +- .../__self_contained_coordinator__/runners.py | 2 +- .../self_contained_coordinator.py | 21 ++- .../setups/topologies/topologies.yml | 65 +++++++++ ...rk-3Mkeys-load-string-with-512B-values.yml | 6 + .../test-memtier-dockerhub-iothreads.yml | 30 +++++ .../test_data/test-suites/topologies.yml | 22 +++ .../tests/test_self_contained_coordinator.py | 4 + ...test_self_contained_coordinator_memtier.py | 127 ++++++++++++++++++ utils/tests/test_spec.py | 11 +- utils/tests/test_topologies.py | 15 +++ 12 files changed, 354 insertions(+), 7 deletions(-) create mode 100644 utils/tests/test_data/test-suites/test-memtier-dockerhub-iothreads.yml create mode 100644 utils/tests/test_data/test-suites/topologies.yml diff --git a/redis_benchmarks_specification/__common__/spec.py b/redis_benchmarks_specification/__common__/spec.py index 14d4c0f..67495a7 100644 --- a/redis_benchmarks_specification/__common__/spec.py +++ b/redis_benchmarks_specification/__common__/spec.py @@ -1,4 +1,56 @@ import math +import logging + + +def extract_redis_dbconfig_parameters(benchmark_config, dbconfig_keyname): + redis_configuration_parameters = {} + modules_configuration_parameters_map = {} + dataset_load_timeout_secs = 120 + dataset_name = None + dbconfig_present = False + if dbconfig_keyname in benchmark_config: + dbconfig_present = True + if type(benchmark_config[dbconfig_keyname]) == list: + for k in benchmark_config[dbconfig_keyname]: + if "configuration-parameters" in k: + cp = k["configuration-parameters"] + for item in cp: + for k, v in item.items(): + redis_configuration_parameters[k] = v + if "dataset_load_timeout_secs" in k: + dataset_load_timeout_secs = k["dataset_load_timeout_secs"] + if "dataset_name" in k: + dataset_name = k["dataset_name"] + if type(benchmark_config[dbconfig_keyname]) == dict: + if "configuration-parameters" in benchmark_config[dbconfig_keyname]: + cp = benchmark_config[dbconfig_keyname]["configuration-parameters"] + for k, v in cp.items(): + redis_configuration_parameters[k] = v + if "dataset_load_timeout_secs" in benchmark_config[dbconfig_keyname]: + dataset_load_timeout_secs = benchmark_config[dbconfig_keyname][ + "dataset_load_timeout_secs" + ] + if "dataset_name" in benchmark_config[dbconfig_keyname]: + dataset_name = benchmark_config[dbconfig_keyname]["dataset_name"] + + return ( + dbconfig_present, + dataset_name, + redis_configuration_parameters, + dataset_load_timeout_secs, + modules_configuration_parameters_map, + ) + + +def extract_redis_configuration_from_topology(topologies_map, topology_spec_name): + redis_arguments = "" + topology_spec = topologies_map[topology_spec_name] + if "redis_arguments" in topology_spec: + redis_arguments = topology_spec["redis_arguments"] + logging.info( + f"extracted redis_arguments: {redis_arguments} from topology: {topology_spec_name}" + ) + return redis_arguments def extract_client_cpu_limit(benchmark_config): diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/docker.py b/redis_benchmarks_specification/__self_contained_coordinator__/docker.py index 96bc573..ca3e9f6 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/docker.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/docker.py @@ -8,7 +8,7 @@ def generate_standalone_redis_server_args( - binary, port, dbdir, configuration_parameters=None + binary, port, dbdir, configuration_parameters=None, redis_arguments="" ): added_params = ["port", "protected-mode", "dir"] # start redis-server @@ -30,6 +30,10 @@ def generate_standalone_redis_server_args( parameter_value, ] ) + if redis_arguments != "": + redis_arguments_arr = redis_arguments.split(" ") + logging.info(f"adding redis arguments {redis_arguments_arr}") + command.extend(redis_arguments_arr) return command diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/runners.py b/redis_benchmarks_specification/__self_contained_coordinator__/runners.py index cc3f398..8bb2992 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/runners.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/runners.py @@ -32,7 +32,6 @@ from redisbench_admin.run.run import calculate_client_tool_duration_and_check from redisbench_admin.utils.benchmark_config import ( get_final_benchmark_config, - extract_redis_dbconfig_parameters, ) from redisbench_admin.utils.local import get_local_run_full_filename from redisbench_admin.utils.results import post_process_benchmark_results @@ -47,6 +46,7 @@ extract_client_cpu_limit, extract_client_tool, extract_client_container_image, + extract_redis_dbconfig_parameters, ) from redis_benchmarks_specification.__self_contained_coordinator__.artifacts import ( restore_build_artifacts_from_test_details, diff --git a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py index 696e751..3b7f48d 100644 --- a/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py +++ b/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py @@ -80,7 +80,6 @@ from redisbench_admin.run.run import calculate_client_tool_duration_and_check from redisbench_admin.utils.benchmark_config import ( get_final_benchmark_config, - extract_redis_dbconfig_parameters, get_defaults, ) from redisbench_admin.utils.local import get_local_run_full_filename @@ -95,6 +94,8 @@ extract_client_cpu_limit, extract_client_tool, extract_client_container_image, + extract_redis_dbconfig_parameters, + extract_redis_configuration_from_topology, ) from redis_benchmarks_specification.__self_contained_coordinator__.artifacts import ( restore_build_artifacts_from_test_details, @@ -756,6 +757,14 @@ def process_self_contained_coordinator_stream( ) ) for topology_spec_name in benchmark_config["redis-topologies"]: + setup_name = topology_spec_name + setup_type = "oss-standalone" + if topology_spec_name in topologies_map: + topology_spec = topologies_map[topology_spec_name] + setup_type = topology_spec["type"] + logging.info( + f"Running topology named {topology_spec_name} of type {setup_type}" + ) test_result = False redis_container = None try: @@ -763,6 +772,11 @@ def process_self_contained_coordinator_stream( ceil_db_cpu_limit = extract_db_cpu_limit( topologies_map, topology_spec_name ) + redis_arguments = ( + extract_redis_configuration_from_topology( + topologies_map, topology_spec_name + ) + ) temporary_dir = tempfile.mkdtemp(dir=home) temporary_dir_client = tempfile.mkdtemp(dir=home) logging.info( @@ -776,8 +790,6 @@ def process_self_contained_coordinator_stream( ) ) - setup_name = "oss-standalone" - setup_type = "oss-standalone" tf_triggering_env = "ci" github_actor = "{}-{}".format( tf_triggering_env, running_platform @@ -814,6 +826,7 @@ def process_self_contained_coordinator_stream( redis_proc_start_port, mnt_point, redis_configuration_parameters, + redis_arguments, ) command_str = " ".join(command) db_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus( @@ -926,7 +939,7 @@ def process_self_contained_coordinator_stream( start_time_str, git_hash, test_name, - "oss-standalone", + setup_name, ) ) logging.info( diff --git a/redis_benchmarks_specification/setups/topologies/topologies.yml b/redis_benchmarks_specification/setups/topologies/topologies.yml index 3b3dbd8..59e9d61 100644 --- a/redis_benchmarks_specification/setups/topologies/topologies.yml +++ b/redis_benchmarks_specification/setups/topologies/topologies.yml @@ -10,6 +10,71 @@ spec: cpus: "1" memory: "10g" + - name: oss-standalone-02-io-threads + type: oss-standalone + redis_topology: + primaries: 1 + replicas: 0 + redis_arguments: --io-threads 2 --io-threads-do-reads yes + resources: + requests: + cpus: "3" + memory: "10g" + + - name: oss-standalone-04-io-threads + type: oss-standalone + redis_topology: + primaries: 1 + replicas: 0 + redis_arguments: --io-threads 4 --io-threads-do-reads yes + resources: + requests: + cpus: "5" + memory: "10g" + + - name: oss-standalone-08-io-threads + type: oss-standalone + redis_topology: + primaries: 1 + replicas: 0 + redis_arguments: --io-threads 8 --io-threads-do-reads yes + resources: + requests: + cpus: "9" + memory: "10g" + + - name: oss-standalone-16-io-threads + type: oss-standalone + redis_topology: + primaries: 1 + replicas: 0 + redis_arguments: --io-threads 16 --io-threads-do-reads yes + resources: + requests: + cpus: "17" + memory: "10g" + + - name: oss-standalone-32-io-threads + type: oss-standalone + redis_topology: + primaries: 1 + replicas: 0 + redis_arguments: --io-threads 32 --io-threads-do-reads yes + resources: + requests: + cpus: "33" + memory: "10g" + + - name: oss-standalone-64-io-threads + type: oss-standalone + redis_topology: + primaries: 1 + replicas: 0 + redis_arguments: --io-threads 64 --io-threads-do-reads yes + resources: + requests: + cpus: "65" + memory: "10g" - name: oss-standalone-1replica type: oss-standalone redis_topology: diff --git a/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml b/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml index 10de3ca..5f33d54 100644 --- a/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml +++ b/redis_benchmarks_specification/test-suites/memtier_benchmark-3Mkeys-load-string-with-512B-values.yml @@ -13,6 +13,12 @@ tested-commands: - set redis-topologies: - oss-standalone +- oss-standalone-02-io-threads +- oss-standalone-04-io-threads +- oss-standalone-08-io-threads +- oss-standalone-16-io-threads +- oss-standalone-32-io-threads +- oss-standalone-64-io-threads build-variants: - gcc:8.5.0-amd64-debian-buster-default - dockerhub diff --git a/utils/tests/test_data/test-suites/test-memtier-dockerhub-iothreads.yml b/utils/tests/test_data/test-suites/test-memtier-dockerhub-iothreads.yml new file mode 100644 index 0000000..c6d5cf1 --- /dev/null +++ b/utils/tests/test_data/test-suites/test-memtier-dockerhub-iothreads.yml @@ -0,0 +1,30 @@ +version: 0.4 +name: memtier_benchmark-1Mkeys-load-string-with-10B-values +description: Runs memtier_benchmark, for a keyspace length of 1M keys loading STRINGs in which the value has a data size of 10 Bytes. +dbconfig: + configuration-parameters: + save: '""' + check: + keyspacelen: 0 + resources: + requests: + memory: 1g +tested-commands: +- set +redis-topologies: +- oss-standalone-02-io-threads +build-variants: +- gcc:8.5.0-amd64-debian-buster-default +- dockerhub +clientconfig: + run_image: redislabs/memtier_benchmark:edge + tool: memtier_benchmark + arguments: '"--data-size" "10" --ratio 1:0 --key-pattern P:P --key-minimum=1 --key-maximum 1000000 --test-time 180 -c 50 -t 4 --hide-histogram' + resources: + requests: + cpus: "1" + memory: "2g" + +tested-groups: +- string +priority: 17 diff --git a/utils/tests/test_data/test-suites/topologies.yml b/utils/tests/test_data/test-suites/topologies.yml new file mode 100644 index 0000000..0fd4393 --- /dev/null +++ b/utils/tests/test_data/test-suites/topologies.yml @@ -0,0 +1,22 @@ +spec: + setups: + - name: oss-standalone + type: oss-standalone + redis_topology: + primaries: 1 + replicas: 0 + resources: + requests: + cpus: "1" + memory: "10g" + + - name: oss-standalone-02-io-threads + type: oss-standalone + redis_topology: + primaries: 1 + replicas: 0 + redis_arguments: --io-threads 2 --io-threads-do-reads yes + resources: + requests: + cpus: "1" + memory: "10g" diff --git a/utils/tests/test_self_contained_coordinator.py b/utils/tests/test_self_contained_coordinator.py index 8855de6..a3a9f31 100644 --- a/utils/tests/test_self_contained_coordinator.py +++ b/utils/tests/test_self_contained_coordinator.py @@ -263,3 +263,7 @@ def test_self_contained_coordinator_blocking_read(): except redis.exceptions.ConnectionError: pass + + +def test_extract_redis_configuration_from_topology(): + assert False diff --git a/utils/tests/test_self_contained_coordinator_memtier.py b/utils/tests/test_self_contained_coordinator_memtier.py index 8a325bb..3f5538c 100644 --- a/utils/tests/test_self_contained_coordinator_memtier.py +++ b/utils/tests/test_self_contained_coordinator_memtier.py @@ -478,6 +478,133 @@ def test_self_contained_coordinator_dockerhub(): pass +def test_self_contained_coordinator_dockerhub_iothreads(): + try: + if run_coordinator_tests_dockerhub(): + db_port = int(os.getenv("DATASINK_PORT", "6379")) + conn = redis.StrictRedis(port=db_port) + conn.ping() + conn.flushall() + + id = "dockerhub" + redis_version = "7.4.0" + run_image = f"redis:{redis_version}" + build_arch = "amd64" + testDetails = {} + build_os = "test_build_os" + build_stream_fields, result = generate_benchmark_stream_request( + id, + conn, + run_image, + build_arch, + testDetails, + build_os, + ) + build_stream_fields["mnt_point"] = "" + if result is True: + benchmark_stream_id = conn.xadd( + STREAM_KEYNAME_NEW_BUILD_EVENTS, build_stream_fields + ) + logging.info( + "sucessfully requested a new run {}. Stream id: {}".format( + build_stream_fields, benchmark_stream_id + ) + ) + + build_variant_name = "gcc:8.5.0-amd64-debian-buster-default" + expected_datapoint_ts = None + + assert conn.exists(STREAM_KEYNAME_NEW_BUILD_EVENTS) + assert conn.xlen(STREAM_KEYNAME_NEW_BUILD_EVENTS) > 0 + running_platform = "fco-ThinkPad-T490" + + build_runners_consumer_group_create(conn, running_platform, "0") + datasink_conn = redis.StrictRedis(port=db_port) + docker_client = docker.from_env() + home = str(Path.home()) + stream_id = ">" + topologies_map = get_topologies( + "./utils/tests/test_data/test-suites/topologies.yml" + ) + # we use a benchmark spec with smaller CPU limit for client given github machines only contain 2 cores + # and we need 1 core for DB and another for CLIENT + testsuite_spec_files = [ + "./utils/tests/test_data/test-suites/test-memtier-dockerhub-iothreads.yml" + ] + defaults_filename = "./utils/tests/test_data/test-suites/defaults.yml" + ( + _, + _, + default_metrics, + _, + _, + _, + ) = get_defaults(defaults_filename) + + ( + result, + stream_id, + number_processed_streams, + num_process_test_suites, + ) = self_contained_coordinator_blocking_read( + conn, + True, + docker_client, + home, + stream_id, + datasink_conn, + testsuite_spec_files, + topologies_map, + running_platform, + False, + [], + "", + 0, + 6399, + 1, + False, + 5, + default_metrics, + "amd64", + None, + 0, + 10000, + "unstable", + "", + True, + False, + ) + + assert result == True + assert number_processed_streams == 1 + assert num_process_test_suites == 1 + by_version_key = f"ci.benchmarks.redislabs/ci/redis/redis/memtier_benchmark-1Mkeys-load-string-with-10B-values/by.version/{redis_version}/benchmark_end/oss-standalone-02-io-threads/memory_maxmemory" + assert datasink_conn.exists(by_version_key) + rts = datasink_conn.ts() + # check we have by version metrics + assert "version" in rts.info(by_version_key).labels + assert redis_version == rts.info(by_version_key).labels["version"] + + # get all keys + all_keys = datasink_conn.keys("*") + by_hash_keys = [] + for key in all_keys: + if "/by.hash/" in key.decode(): + by_hash_keys.append(key) + + # ensure we have by hash keys + assert len(by_hash_keys) > 0 + for hash_key in by_hash_keys: + # ensure we have both version and hash info on the key + assert "version" in rts.info(hash_key).labels + assert "oss-standalone-02-io-threads" in hash_key.decode() + assert "hash" in rts.info(hash_key).labels + assert redis_version == rts.info(hash_key).labels["version"] + + except redis.exceptions.ConnectionError: + pass + + def test_self_contained_coordinator_dockerhub_valkey(): try: if run_coordinator_tests_dockerhub(): diff --git a/utils/tests/test_spec.py b/utils/tests/test_spec.py index 0886083..8c640af 100644 --- a/utils/tests/test_spec.py +++ b/utils/tests/test_spec.py @@ -11,7 +11,6 @@ def test_extract_build_variant_variations(): - with open( "./utils/tests/test_data/test-suites/redis-benchmark-full-suite-1Mkeys-100B.yml", "r", @@ -27,3 +26,13 @@ def test_extract_build_variant_variations(): benchmark_config = yaml.safe_load(yml_file) build_variants = extract_build_variant_variations(benchmark_config) assert "gcc:8.5.0-amd64-debian-buster-default" in build_variants + + +def test_extract_redis_dbconfig_parameters(): + with open( + "./redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-100B-expire-use-case.yml", + "r", + ) as yml_file: + benchmark_config = yaml.safe_load(yml_file) + build_variants = extract_build_variant_variations(benchmark_config) + assert "gcc:8.5.0-amd64-debian-buster-default" in build_variants diff --git a/utils/tests/test_topologies.py b/utils/tests/test_topologies.py index 6b4aaef..9071955 100644 --- a/utils/tests/test_topologies.py +++ b/utils/tests/test_topologies.py @@ -1,3 +1,6 @@ +from redis_benchmarks_specification.__common__.spec import ( + extract_redis_configuration_from_topology, +) from redis_benchmarks_specification.__setups__.topologies import get_topologies @@ -7,3 +10,15 @@ def test_get_topologies(): ) assert "oss-standalone" in topologies_map assert topologies_map["oss-standalone"]["resources"]["requests"]["cpus"] == "1" + + +def test_extract_redis_configuration_from_topology(): + topologies_map = get_topologies( + "./redis_benchmarks_specification/setups/topologies/topologies.yml" + ) + assert "oss-standalone-04-io-threads" in topologies_map.keys() + res = extract_redis_configuration_from_topology( + topologies_map, "oss-standalone-04-io-threads" + ) + assert res != "" + assert "--io-threads 4 io-threads-do-reads yes" in res From 3e04c2455cc0625843b3caffcbe59e7a8e234ab9 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Wed, 14 Aug 2024 02:39:25 +0100 Subject: [PATCH 3/4] Removed duplicate test --- utils/tests/test_self_contained_coordinator.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/utils/tests/test_self_contained_coordinator.py b/utils/tests/test_self_contained_coordinator.py index a3a9f31..8855de6 100644 --- a/utils/tests/test_self_contained_coordinator.py +++ b/utils/tests/test_self_contained_coordinator.py @@ -263,7 +263,3 @@ def test_self_contained_coordinator_blocking_read(): except redis.exceptions.ConnectionError: pass - - -def test_extract_redis_configuration_from_topology(): - assert False From 20d61ea039e25b5c5ca7c0d11fa6e45430075911 Mon Sep 17 00:00:00 2001 From: fcosta_oliveira Date: Wed, 14 Aug 2024 02:41:11 +0100 Subject: [PATCH 4/4] Fixed missing update tests --- utils/tests/test_runner.py | 4 ++-- utils/tests/test_topologies.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/tests/test_runner.py b/utils/tests/test_runner.py index 833575f..4cb7ab7 100644 --- a/utils/tests/test_runner.py +++ b/utils/tests/test_runner.py @@ -258,7 +258,7 @@ def test_extract_testsuites(): ] ) tests = extract_testsuites(args) - assert len(tests) == 7 + assert len(tests) == 9 args = parser.parse_args( args=[ @@ -269,7 +269,7 @@ def test_extract_testsuites(): ] ) tests = extract_testsuites(args) - assert len(tests) == 7 + assert len(tests) == 9 args = parser.parse_args( args=[ diff --git a/utils/tests/test_topologies.py b/utils/tests/test_topologies.py index 9071955..4850e84 100644 --- a/utils/tests/test_topologies.py +++ b/utils/tests/test_topologies.py @@ -21,4 +21,4 @@ def test_extract_redis_configuration_from_topology(): topologies_map, "oss-standalone-04-io-threads" ) assert res != "" - assert "--io-threads 4 io-threads-do-reads yes" in res + assert "--io-threads 4 --io-threads-do-reads yes" in res