Skip to content

Commit 155c2f3

Browse files
yashtewariyashtewari
authored
Make tests more concurrent. (#374)
Add the range option to tests so they can be split up and run in concurrent parts using a test matrix.
1 parent 336e3ad commit 155c2f3

14 files changed

+258
-102
lines changed

‎.github/workflows/cloudbeat-ci.yml

+82-4
Original file line numberDiff line numberDiff line change
@@ -92,14 +92,92 @@ jobs:
9292
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
9393
9494
Test_Matrix:
95-
name: ${{ matrix.test-target }}-tests
95+
name: ${{ matrix.test-target }}-${{ matrix.range }}-tests
9696
needs: Build
9797
runs-on: ubuntu-latest
9898
timeout-minutes: 55
9999
strategy:
100100
matrix:
101-
# test-targets chosen by stabillity and runtime.Only pre_merge will be set as mandatory
102-
test-target: [pre_merge, file_system_rules, k8s_object_rules, process_scheduler_rules, process_api_server_rules, process_controller_manager_rules, process_etcd_rules, process_kubelet_rules]
101+
include:
102+
- test-target: pre_merge
103+
- test-target: file_system_rules
104+
range: '0..15'
105+
- test-target: file_system_rules
106+
range: '15..30'
107+
- test-target: file_system_rules
108+
range: '30..45'
109+
- test-target: file_system_rules
110+
range: '45..60'
111+
- test-target: file_system_rules
112+
range: '60..'
113+
- test-target: k8s_object_rules
114+
range: '0..6'
115+
- test-target: k8s_object_rules
116+
range: '6..12'
117+
- test-target: k8s_object_rules
118+
range: '12..18'
119+
- test-target: k8s_object_rules
120+
range: '18..'
121+
- test-target: process_scheduler_rules
122+
range: '0..3'
123+
- test-target: process_scheduler_rules
124+
range: '3..'
125+
- test-target: process_api_server_rules
126+
range: '0..5'
127+
- test-target: process_api_server_rules
128+
range: '5..10'
129+
- test-target: process_api_server_rules
130+
range: '10..15'
131+
- test-target: process_api_server_rules
132+
range: '15..20'
133+
- test-target: process_api_server_rules
134+
range: '20..24'
135+
- test-target: process_api_server_rules
136+
range: '24..28'
137+
- test-target: process_api_server_rules
138+
range: '28..32'
139+
- test-target: process_api_server_rules
140+
range: '32..36'
141+
- test-target: process_api_server_rules
142+
range: '36..40'
143+
- test-target: process_api_server_rules
144+
range: '40..44'
145+
- test-target: process_api_server_rules
146+
range: '44..48'
147+
- test-target: process_api_server_rules
148+
range: '48..52'
149+
- test-target: process_api_server_rules
150+
range: '52..56'
151+
- test-target: process_api_server_rules
152+
range: '56..'
153+
- test-target: process_controller_manager_rules
154+
range: '0..4'
155+
- test-target: process_controller_manager_rules
156+
range: '4..8'
157+
- test-target: process_controller_manager_rules
158+
range: '8..12'
159+
- test-target: process_controller_manager_rules
160+
range: '12..'
161+
- test-target: process_etcd_rules
162+
range: '0..4'
163+
- test-target: process_etcd_rules
164+
range: '4..8'
165+
- test-target: process_etcd_rules
166+
range: '8..12'
167+
- test-target: process_etcd_rules
168+
range: '12..'
169+
- test-target: process_kubelet_rules
170+
range: '0..4'
171+
- test-target: process_kubelet_rules
172+
range: '4..8'
173+
- test-target: process_kubelet_rules
174+
range: '8..12'
175+
- test-target: process_kubelet_rules
176+
range: '12..16'
177+
- test-target: process_kubelet_rules
178+
range: '16..20'
179+
- test-target: process_kubelet_rules
180+
range: '20..'
103181
fail-fast: false
104182
steps:
105183
- name: Check out the repo
@@ -131,7 +209,7 @@ jobs:
131209
- name: Deploy tests Helm chart
132210
id: deploy_helm
133211
run: |
134-
just deploy-tests-helm-ci ${{ matrix.test-target }}
212+
just deploy-tests-helm-ci ${{ matrix.test-target }} ${{ matrix.range }}
135213
136214
- name: Run Tests
137215
id: run_tests

‎JUSTFILE

+12-12
Original file line numberDiff line numberDiff line change
@@ -98,14 +98,14 @@ build-pytest-docker:
9898
load-pytest-kind:
9999
kind load docker-image {{TESTS_RELEASE}}:latest --name kind-mono
100100

101-
deploy-tests-helm-ci target:
102-
helm upgrade --wait --timeout={{TIMEOUT}} --install --values tests/deploy/values/ci.yml --set testData.marker={{target}} --set testData.marker={{target}} --set elasticsearch.imageTag={{VERSION}} -n {{NAMESPACE}} {{TESTS_RELEASE}} tests/deploy/k8s-cloudbeat-tests/
101+
deploy-tests-helm-ci target range='':
102+
helm upgrade --wait --timeout={{TIMEOUT}} --install --values tests/deploy/values/ci.yml --set testData.marker={{target}} --set testData.range={{range}} --set elasticsearch.imageTag={{VERSION}} -n {{NAMESPACE}} {{TESTS_RELEASE}} tests/deploy/k8s-cloudbeat-tests/
103103

104-
deploy-tests-helm-ci-agent target:
105-
helm upgrade --wait --timeout={{TIMEOUT}} --install --values tests/deploy/values/ci-sa-agent.yml --set testData.marker={{target}} --set elasticsearch.imageTag={{VERSION}} -n {{NAMESPACE}} {{TESTS_RELEASE}} tests/deploy/k8s-cloudbeat-tests/
104+
deploy-tests-helm-ci-agent target range='':
105+
helm upgrade --wait --timeout={{TIMEOUT}} --install --values tests/deploy/values/ci-sa-agent.yml --set testData.marker={{target}} --set testData.range={{range}} --set elasticsearch.imageTag={{VERSION}} -n {{NAMESPACE}} {{TESTS_RELEASE}} tests/deploy/k8s-cloudbeat-tests/
106106

107-
deploy-local-tests-helm target:
108-
helm upgrade --wait --timeout={{TIMEOUT}} --install --values tests/deploy/values/local-host.yml --set testData.marker={{target}} --set testData.marker={{target}} --set elasticsearch.imageTag={{VERSION}} -n {{NAMESPACE}} {{TESTS_RELEASE}} tests/deploy/k8s-cloudbeat-tests/
107+
deploy-local-tests-helm target range='':
108+
helm upgrade --wait --timeout={{TIMEOUT}} --install --values tests/deploy/values/local-host.yml --set testData.marker={{target}} --set testData.range={{range}} --set elasticsearch.imageTag={{VERSION}} -n {{NAMESPACE}} {{TESTS_RELEASE}} tests/deploy/k8s-cloudbeat-tests/
109109

110110
purge-pvc:
111111
kubectl delete -f tests/deploy/pvc-deleter.yaml -n {{NAMESPACE}} & kubectl apply -f tests/deploy/pvc-deleter.yaml -n {{NAMESPACE}}
@@ -136,8 +136,8 @@ build-load-run-tests: build-pytest-docker load-pytest-kind run-tests
136136
delete-local-helm-cluster:
137137
kind delete cluster --name kind-mono
138138

139-
cleanup-create-local-helm-cluster target: delete-local-helm-cluster create-kind-cluster build-cloudbeat load-cloudbeat-image
140-
just deploy-local-tests-helm {{target}}
139+
cleanup-create-local-helm-cluster target range='..': delete-local-helm-cluster create-kind-cluster build-cloudbeat load-cloudbeat-image
140+
just deploy-local-tests-helm {{target}} {{range}}
141141

142142
# TODO(DaveSys911): Move scripts out of JUSTFILE: https://github.com/elastic/security-team/issues/4291
143143
test-pod-status:
@@ -180,21 +180,21 @@ collect-logs target:
180180
rm $LOG_FILE_TMP
181181
echo 'Done collecting logs for target {{target}}.'
182182

183-
run-test-target target:
183+
run-test-target target range='..':
184184
echo 'Cleaning up cluster for running test target: {{target}}'
185-
just cleanup-create-local-helm-cluster {{target}}
185+
just cleanup-create-local-helm-cluster {{target}} {{range}}
186186

187187
echo 'Running test target: {{target}}'
188188
just build-load-run-tests &
189189

190190

191-
run-test-targets +targets='file_system_rules k8s_object_rules process_api_server_rules process_controller_manager_rules process_etcd_rules process_kubelet_rules process_scheduler_rules':
191+
run-test-targets range='..' +targets='file_system_rules k8s_object_rules process_api_server_rules process_controller_manager_rules process_etcd_rules process_kubelet_rules process_scheduler_rules':
192192
#!/usr/bin/env sh
193193

194194
echo 'Running tests: {{targets}}'
195195

196196
for TARGET in {{targets}}; do
197-
just run-test-target $TARGET
197+
just run-test-target $TARGET {{range}}
198198
just collect-logs $TARGET
199199
done
200200

‎tests/conftest.py

+8
Original file line numberDiff line numberDiff line change
@@ -54,3 +54,11 @@ def api_client():
5454
else:
5555
client = FsClient
5656
return client
57+
58+
59+
def pytest_addoption(parser):
60+
parser.addoption(
61+
'--range',
62+
default=['..'],
63+
help='range to run tests on',
64+
)

‎tests/deploy/k8s-cloudbeat-tests/templates/cloudbeat-test-configmap.yml

+7
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,11 @@ data:
1111
ES_HOST: "elasticsearch-master.kube-system"
1212
USE_DOCKER: "false"
1313
TEST_MARKER: {{ .Values.testData.marker }}
14+
{{- if .Values.testData.range }}
15+
RANGE_FLAG: "--range"
16+
TEST_RANGE: {{ .Values.testData.range }}
17+
{{- else }}
18+
RANGE_FLAG: ""
19+
TEST_RANGE: ""
20+
{{- end }}
1421
AGENT_NAME: {{ .Values.testData.agentName }}

‎tests/deploy/k8s-cloudbeat-tests/templates/tests/cloudbeat-test-pod.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ spec:
1616
- /bin/sh
1717
- -c
1818
- |
19-
pytest -rA --disable-warnings -m ${TEST_MARKER} --alluredir=/usr/src/app/tests/report
19+
pytest -rA --disable-warnings -m ${TEST_MARKER} ${RANGE_FLAG} ${TEST_RANGE} --alluredir=/usr/src/app/tests/report
2020
envFrom:
2121
- configMapRef:
2222
name: {{ .Values.serviceAccount.name }}-configmap

‎tests/product/tests/conftest.py

+32
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
from kubernetes.utils import FailToCreateError
1111
from commonlib.io_utils import get_k8s_yaml_objects
1212

13+
from product.tests.parameters import TEST_PARAMETERS
14+
1315

1416
DEPLOY_YML = "../../deploy/cloudbeat-pytest.yml"
1517
KUBE_RULES_ENV_YML = "../../deploy/mock-pod.yml"
@@ -128,3 +130,33 @@ def test_env(cloudbeat_start_stop):
128130
yield k8s, api_client, cloudbeat_agent
129131
# teardown
130132
k8s.delete_from_yaml(yaml_objects_list=k8s_resources) # stop agent
133+
134+
135+
def pytest_generate_tests(metafunc):
136+
# Only parametrize tests which are required for this run.
137+
if metafunc.definition.get_closest_marker(metafunc.config.getoption('markexpr', default=None)) is None:
138+
return
139+
140+
params = TEST_PARAMETERS.get(metafunc.function)
141+
if params is None:
142+
raise ValueError(f'Params for function {metafunc.function} are not registered.')
143+
144+
test_range = metafunc.config.getoption('range')
145+
test_range_start, test_range_end = test_range.split('..')
146+
147+
if test_range_end != '' and int(test_range_end) < len(params.argvalues):
148+
params.argvalues = params.argvalues[:int(test_range_end)]
149+
150+
if params.ids is not None:
151+
params.ids = params.ids[:int(test_range_end)]
152+
153+
if test_range_start != '':
154+
if int(test_range_start) >= len(params.argvalues):
155+
raise ValueError(f'Invalid range for test function {metafunc.function}')
156+
157+
params.argvalues = params.argvalues[int(test_range_start):]
158+
159+
if params.ids is not None:
160+
params.ids = params.ids[int(test_range_start):]
161+
162+
metafunc.parametrize(params.argnames, params.argvalues, ids=params.ids)

‎tests/product/tests/parameters.py

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
class Parameters:
2+
def __init__(self, argnames, argvalues, ids=None):
3+
self.argnames = argnames
4+
self.argvalues = argvalues
5+
self.ids = ids
6+
7+
8+
TEST_PARAMETERS = {}
9+
10+
11+
def register_params(func, params: Parameters):
12+
if func in TEST_PARAMETERS:
13+
raise KeyError(f'Parameters for test {func} are already registered')
14+
15+
TEST_PARAMETERS[func] = params

‎tests/product/tests/test_file_system_rules.py

+34-31
Original file line numberDiff line numberDiff line change
@@ -5,40 +5,12 @@
55
from datetime import datetime
66
import pytest
77
from commonlib.utils import get_ES_evaluation
8-
from commonlib.framework.reporting import skip_param_case, SkipReportData
9-
from .data.file_system import file_system_test_cases as fs_tc
8+
9+
from product.tests.data.file_system import file_system_test_cases as fs_tc
10+
from product.tests.parameters import register_params, Parameters
1011

1112

1213
@pytest.mark.file_system_rules
13-
@pytest.mark.parametrize(
14-
("rule_tag", "command", "param_value", "resource", "expected"),
15-
[*fs_tc.cis_1_1_1,
16-
*fs_tc.cis_1_1_2,
17-
*fs_tc.cis_1_1_3,
18-
*fs_tc.cis_1_1_4,
19-
*fs_tc.cis_1_1_5,
20-
*fs_tc.cis_1_1_6,
21-
*fs_tc.cis_1_1_7,
22-
*fs_tc.cis_1_1_8,
23-
*fs_tc.cis_1_1_11,
24-
*fs_tc.cis_1_1_12,
25-
*fs_tc.cis_1_1_13,
26-
*fs_tc.cis_1_1_14,
27-
*fs_tc.cis_1_1_15,
28-
*fs_tc.cis_1_1_16,
29-
*fs_tc.cis_1_1_17,
30-
*fs_tc.cis_1_1_18,
31-
*fs_tc.cis_1_1_19,
32-
*fs_tc.cis_1_1_20,
33-
*fs_tc.cis_1_1_21,
34-
*fs_tc.cis_4_1_1,
35-
*fs_tc.cis_4_1_2,
36-
*fs_tc.cis_4_1_5,
37-
*fs_tc.cis_4_1_6,
38-
*fs_tc.cis_4_1_9,
39-
*fs_tc.cis_4_1_10
40-
],
41-
)
4214
def test_file_system_configuration(elastic_client,
4315
config_node_pre_test,
4416
rule_tag,
@@ -99,3 +71,34 @@ def identifier(eval_resource):
9971
assert evaluation is not None, f"No evaluation for rule {rule_tag} could be found"
10072
assert evaluation == expected, f"Rule {rule_tag} verification failed," \
10173
f"expected: {expected}, got: {evaluation}"
74+
75+
76+
register_params(test_file_system_configuration, Parameters(
77+
("rule_tag", "command", "param_value", "resource", "expected"),
78+
[*fs_tc.cis_1_1_1,
79+
*fs_tc.cis_1_1_2,
80+
*fs_tc.cis_1_1_3,
81+
*fs_tc.cis_1_1_4,
82+
*fs_tc.cis_1_1_5,
83+
*fs_tc.cis_1_1_6,
84+
*fs_tc.cis_1_1_7,
85+
*fs_tc.cis_1_1_8,
86+
*fs_tc.cis_1_1_11,
87+
*fs_tc.cis_1_1_12,
88+
*fs_tc.cis_1_1_13,
89+
*fs_tc.cis_1_1_14,
90+
*fs_tc.cis_1_1_15,
91+
*fs_tc.cis_1_1_16,
92+
*fs_tc.cis_1_1_17,
93+
*fs_tc.cis_1_1_18,
94+
*fs_tc.cis_1_1_19,
95+
*fs_tc.cis_1_1_20,
96+
*fs_tc.cis_1_1_21,
97+
*fs_tc.cis_4_1_1,
98+
*fs_tc.cis_4_1_2,
99+
*fs_tc.cis_4_1_5,
100+
*fs_tc.cis_4_1_6,
101+
*fs_tc.cis_4_1_9,
102+
*fs_tc.cis_4_1_10
103+
],
104+
))

0 commit comments

Comments
 (0)