Mend CI with resource identifiers, increased timeout. (#415)

* Increase test timeout to mend CI.

Workaround until we figure out why some findings are taking a long
time to show up in the ES index.

* Resource identifier for process_* test suites.

* Include upstream bug fix - csp-security-policies 1.2.1

* Split file_system_rules suite into more parts.

* Disable automated test case for CIS K8S 4.2.3

Author: yashtewari

.github/workflows/cloudbeat-ci.yml

@@ -115,15 +115,35 @@ jobs:
         include:
           - test-target: pre_merge
           - test-target: file_system_rules
-            range: '0..15'
+            range: '0..5'
+          - test-target: file_system_rules
+            range: '5..10'
+          - test-target: file_system_rules
+            range: '10..15'
+          - test-target: file_system_rules
+            range: '15..20'
+          - test-target: file_system_rules
+            range: '20..25'
+          - test-target: file_system_rules
+            range: '25..30'
+          - test-target: file_system_rules
+            range: '30..35'
+          - test-target: file_system_rules
+            range: '35..40'
+          - test-target: file_system_rules
+            range: '40..45'
+          - test-target: file_system_rules
+            range: '45..50'
+          - test-target: file_system_rules
+            range: '50..55'
           - test-target: file_system_rules
-            range: '15..30'
+            range: '55..60'
           - test-target: file_system_rules
-            range: '30..45'
+            range: '60..65'
           - test-target: file_system_rules
-            range: '45..60'
+            range: '65..70'
           - test-target: file_system_rules
-            range: '60..'
+            range: '70..'
           - test-target: k8s_object_rules
             range: '0..6'
           - test-target: k8s_object_rules

go.mod

@@ -81,7 +81,7 @@ require (
 	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
 	github.com/elastic/beats/v7 v7.0.0-alpha2.0.20220726163833-e8a0da132f1f
-	github.com/elastic/csp-security-policies v1.2.0
+	github.com/elastic/csp-security-policies v1.2.1
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-ini/ini v1.66.6 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect

go.sum

@@ -537,6 +537,8 @@ github.com/elastic/csp-security-policies v1.0.8 h1:Wy13f5kFlwoJLMhnhS8O5gmP3EnGE
 github.com/elastic/csp-security-policies v1.0.8/go.mod h1:iiMuf3IRjGmWD+a88MGen0qeYgReNd2EUqDDb+LZJGc=
 github.com/elastic/csp-security-policies v1.2.0 h1:o0FP379xv8xF/HUcow4cSoswtORV6CLzbri001ioI2U=
 github.com/elastic/csp-security-policies v1.2.0/go.mod h1:3fA3X9OiyP7IRNyacOGnWqt1eHSMVZRx7p3bmZm98oE=
+github.com/elastic/csp-security-policies v1.2.1 h1:exmKOXId1rT1P/SUYweuwCkuZMaxGD0/1ex8yv3JqXs=
+github.com/elastic/csp-security-policies v1.2.1/go.mod h1:3fA3X9OiyP7IRNyacOGnWqt1eHSMVZRx7p3bmZm98oE=
 github.com/elastic/e2e-testing v1.99.2-0.20220117192005-d3365c99b9c4 h1:uYT+Krd8dsvnhnLK9pe/JHZkYtXEGPfbV4Wt1JPPol0=
 github.com/elastic/e2e-testing v1.99.2-0.20220117192005-d3365c99b9c4/go.mod h1:UcNuf4pX/qDVNQr0zybm1NL2YoWik+jKBaINZqQCA40=
 github.com/elastic/elastic-agent-autodiscover v0.2.1 h1:Nbeayh3vq2FNm6xaFo34mhUdOu0EVlpj53CqCsbU0E4=

tests/commonlib/utils.py

@@ -36,6 +36,7 @@ def get_ES_evaluation(elastic_client, timeout, rule_tag, exec_timestamp,
                 resource = event.resource.raw
                 evaluation = event.result.evaluation
             except AttributeError:
+                print('Warning: got finding with missing fields:', event)
                 continue
 
             if resource_identifier(resource):
@@ -155,3 +156,49 @@ def wait_for_cycle_completion(elastic_client, nodes: list) -> bool:
 
 def is_timeout(start_time: time, timeout: int) -> bool:
     return time.time() - start_time > timeout
+
+
+def command_contains_arguments(command, arguments_dict):
+    args = command.split()[1:]
+    args_dict = {}
+    for arg in args:
+        key, val = arg.split('=', 1)
+        args_dict[key] = val
+
+    set_dict = arguments_dict.get('set', {})
+    unset_list = arguments_dict.get('unset', [])
+
+    for key, val in set_dict.items():
+        argval = args_dict.get(key)
+        if val != argval:
+            return False
+
+    for key in unset_list:
+        if key in args_dict:
+            return False
+
+    return True
+
+
+def config_contains_arguments(config, arguments_dict):
+    set_dict = arguments_dict.get('set', {})
+    unset_list = arguments_dict.get('unset', [])
+
+    if not dict_contains(set_dict, config):
+        return False
+
+    for arg in unset_list:
+        current = config
+        arg_set = True
+
+        for arg_part in arg.split('.'):
+            if (not isinstance(current, dict)) or (arg_part not in current):
+                arg_set = False
+                break
+
+            current = current[arg_part]
+
+        if arg_set:
+            return False
+
+    return True

tests/configuration.py

@@ -11,7 +11,7 @@
 agent = Munch()
 agent.name = os.getenv('AGENT_NAME', 'elastic-agent')
 agent.namespace = os.getenv('AGENT_NAMESPACE', 'kube-system')
-agent.findings_timeout = 30
+agent.findings_timeout = 500
 
 # --- Kubernetes environment definition --------------------
 kubernetes = Munch()

tests/product/tests/data/process/process_test_cases.py

@@ -1363,18 +1363,19 @@
 ]
 
 kubelet_rules = [
-    *skip_param_case(skip_list=[*cis_4_2_1,
-                                *cis_4_2_2,
-                                *cis_4_2_3,
-                                *cis_4_2_4,
-                                *cis_4_2_5,
-                                *cis_4_2_6,
-                                *cis_4_2_7,
-                                *cis_4_2_9,
-                                ],
-                     data_to_report=SkipReportData(
-                         skip_reason="Dangling tests"
-                     )),
+    *cis_4_2_1,
+    *cis_4_2_2,
+    *skip_param_case(skip_list=[*cis_4_2_3],
+                    data_to_report=SkipReportData(
+                        skip_reason="Known issue",
+                        url_link="https://github.com/elastic/security-team/issues/5106",
+                        url_title="security-team #5106",
+                    )),
+    *cis_4_2_4,
+    *cis_4_2_5,
+    *cis_4_2_6,
+    *cis_4_2_7,
+    *cis_4_2_9,
     # *cis_4_2_8, # TODO setting is not configurable via the Kubelet config file.
     *cis_4_2_10,
     *cis_4_2_11,

tests/product/tests/test_process_api_server_rules.py

@@ -7,7 +7,7 @@
 import time
 import pytest
 
-from commonlib.utils import get_ES_evaluation
+from commonlib.utils import get_ES_evaluation, command_contains_arguments
 from product.tests.data.process.process_test_cases import api_server_rules
 from product.tests.parameters import register_params, Parameters
 
@@ -46,11 +46,15 @@ def test_process_api_server(elastic_client,
     # TODO: Implement a more optimal way of waiting
     time.sleep(60)
 
+    def identifier(eval_resource):
+        return command_contains_arguments(eval_resource.command, dictionary)
+
     evaluation = get_ES_evaluation(
         elastic_client=elastic_client,
         timeout=cloudbeat_agent.findings_timeout,
         rule_tag=rule_tag,
         exec_timestamp=datetime.utcnow(),
+        resource_identifier=identifier,
     )
 
     assert evaluation is not None, f"No evaluation for rule {rule_tag} could be found"

tests/product/tests/test_process_controller_manager_rules.py

@@ -7,7 +7,7 @@
 import time
 import pytest
 
-from commonlib.utils import get_ES_evaluation
+from commonlib.utils import get_ES_evaluation, command_contains_arguments
 from product.tests.data.process.process_test_cases import controller_manager_rules
 from product.tests.parameters import register_params, Parameters
 
@@ -46,11 +46,15 @@ def test_process_controller_manager(elastic_client,
     # TODO: Implement a more optimal way of waiting
     time.sleep(60)
 
+    def identifier(eval_resource):
+        return command_contains_arguments(eval_resource.command, dictionary)
+
     evaluation = get_ES_evaluation(
         elastic_client=elastic_client,
         timeout=cloudbeat_agent.findings_timeout,
         rule_tag=rule_tag,
         exec_timestamp=datetime.utcnow(),
+        resource_identifier=identifier,
     )
 
     assert evaluation is not None, f"No evaluation for rule {rule_tag} could be found"

tests/product/tests/test_process_etcd_rules.py

@@ -7,7 +7,7 @@
 import time
 import pytest
 
-from commonlib.utils import get_ES_evaluation
+from commonlib.utils import get_ES_evaluation, command_contains_arguments
 from product.tests.data.process.process_test_cases import etcd_rules
 from product.tests.parameters import register_params, Parameters
 
@@ -46,11 +46,15 @@ def test_process_etcd(elastic_client,
     # TODO: Implement a more optimal way of waiting
     time.sleep(60)
 
+    def identifier(eval_resource):
+        return command_contains_arguments(eval_resource.command, dictionary)
+
     evaluation = get_ES_evaluation(
         elastic_client=elastic_client,
         timeout=cloudbeat_agent.findings_timeout,
         rule_tag=rule_tag,
         exec_timestamp=datetime.utcnow(),
+        resource_identifier=identifier,
     )
 
     assert evaluation is not None, f"No evaluation for rule {rule_tag} could be found"

tests/product/tests/test_process_kubelet_rules.py

@@ -7,7 +7,7 @@
 import time
 import pytest
 
-from commonlib.utils import get_ES_evaluation
+from commonlib.utils import get_ES_evaluation, config_contains_arguments
 from product.tests.data.process.process_test_cases import kubelet_rules
 from product.tests.parameters import register_params, Parameters
 
@@ -46,11 +46,21 @@ def test_process_kubelet(elastic_client,
     # TODO: Implement a more optimal way of waiting
     time.sleep(60)
 
+    def identifier(eval_resource):
+        # Needs to be done because EKS findings are showing up in vanilla K8S tests,
+        # leading to unexpected findings structure.
+        # See: https://github.com/elastic/security-team/issues/5107
+        if 'external_data' not in eval_resource.keys():
+            return False
+
+        return config_contains_arguments(eval_resource.external_data.config, dictionary)
+
     evaluation = get_ES_evaluation(
         elastic_client=elastic_client,
         timeout=cloudbeat_agent.findings_timeout,
         rule_tag=rule_tag,
         exec_timestamp=datetime.utcnow(),
+        resource_identifier=identifier,
     )
 
     assert evaluation is not None, f"No evaluation for rule {rule_tag} could be found"

tests/product/tests/test_process_scheduler_rules.py

@@ -7,7 +7,7 @@
 import time
 import pytest
 
-from commonlib.utils import get_ES_evaluation
+from commonlib.utils import get_ES_evaluation, command_contains_arguments
 from product.tests.data.process.process_test_cases import scheduler_rules
 from product.tests.parameters import register_params, Parameters
 
@@ -46,11 +46,15 @@ def test_process_scheduler(elastic_client,
     # TODO: Implement a more optimal way of waiting
     time.sleep(60)
 
+    def identifier(eval_resource):
+        return command_contains_arguments(eval_resource.command, dictionary)
+
     evaluation = get_ES_evaluation(
         elastic_client=elastic_client,
         timeout=cloudbeat_agent.findings_timeout,
         rule_tag=rule_tag,
         exec_timestamp=datetime.utcnow(),
+        resource_identifier=identifier,
     )
 
     assert evaluation is not None, f"No evaluation for rule {rule_tag} could be found"