Refactor automated tests, add pre-commit and CI linter checks. (#480)

Co-authored-by: oren zohar <oren.zohar@elastic.co>
Co-authored-by: Oren Zohar <85433724+oren-zohar@users.noreply.github.com>

Author: yashtewari

.ci/jobs/cloudbeat-mbp.yml

@@ -49,4 +49,4 @@
           reference-repo: /var/lib/jenkins/.git-references/cloudbeat.git
         timeout: '15'
         use-author: true
-        wipe-workspace: 'True'
+        wipe-workspace: 'True'

.ci/jobs/cloudbeat.yml

@@ -2,4 +2,4 @@
 - job:
     name: cloudbeat
     description: cloudbeat
-    project-type: folder
+    project-type: folder

.ci/jobs/defaults.yml

@@ -10,4 +10,4 @@
     logrotate:
       daysToKeep: 10
       numToKeep: 30
-    node: linux
+    node: linux

.github/workflows/cloudbeat-ci.yml

@@ -18,6 +18,28 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  Lint:
+    # for more information see .pre-commit-config.yaml
+    name: Lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Init Hermit
+        run: ./bin/hermit env -r >> $GITHUB_ENV
+
+      - name: Python lints
+        run: |
+          git ls-files -- '*.py' | xargs pre-commit run --file
+        shell: bash
+
+      - name: Go lints
+        run: |
+          git ls-files -- '*.go' | xargs pre-commit run --file
+        shell: bash
+
   Build:
     name: Build
     runs-on: ubuntu-20.04
@@ -235,10 +257,7 @@ jobs:
             range: '4..8'
             values_file: tests/deploy/values/ci.yml
           - test-target: process_etcd_rules
-            range: '8..12'
-            values_file: tests/deploy/values/ci.yml
-          - test-target: process_etcd_rules
-            range: '12..'
+            range: '8..'
             values_file: tests/deploy/values/ci.yml
           - test-target: process_kubelet_rules
             range: '0..4'

.golangci.yaml

@@ -13,4 +13,4 @@ linters-settings:
           - Errorf
           - Fatalf
           - Panicf
-          - DPanicf
+          - DPanicf

.mergify.yml

@@ -192,4 +192,4 @@ pull_request_rules:
           - "8.6"
         labels:
           - "backport"
-        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"

.pre-commit-config.yaml

@@ -1,51 +1,58 @@
 repos:
+  ## General
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v4.4.0
     hooks:
       - id: check-merge-conflict
-
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
-    hooks:
       - id: trailing-whitespace
-        exclude: (^cloudbeat.*yml)$
       - id: end-of-file-fixer
-        exclude: (^cloudbeat.*yml)$
+      - id: check-added-large-files
       - id: check-yaml
         args: [--allow-multiple-documents]
-        exclude: (^cloudbeat.*yml)$
-        args: [--allow-multiple-documents]
+        exclude: (^tests/deploy/k8s-cloudbeat-tests/templates/.*|dev-tools/packaging/packages.yml)$
       - id: check-json
 
-  ## Security
+      ## Security
       - id: detect-private-key
-      - id: detect-aws-credentials
+
+  - repo: https://github.com/asottile/add-trailing-comma
+    rev: v2.3.0
+    hooks:
+      - id: add-trailing-comma
 
   ## Golang hooks
   - repo: https://github.com/dnephin/pre-commit-golang
-    rev: master
+    rev: v0.5.1
     hooks:
       - id: go-fmt
       - id: golangci-lint
 
   ## Python
   - repo: https://github.com/PyCQA/flake8
-    rev: 4.0.1
+    rev: 6.0.0
     hooks:
       - id: flake8
         name: flake8 ./tests/
         files: ^tests/
-        args: [--config, tests/setup.cfg]
 
   - repo: https://github.com/pycqa/pylint
-    rev: pylint-2.6.0
+    rev: v2.15.7
     hooks:
       - id: pylint
         args: [
-            "-rn", # Only display messages
-            "-sn", # Don't display the score
-            "--rcfile=tests/pylintrc", # Link to your config file
-          ]
+          "-rn", # Only display messages
+          "--rcfile=tests/pylintrc", # Link to your config file
+        ]
+
+  - repo: https://github.com/psf/black
+    rev: 22.10.0
+    hooks:
+      - id: black
+        args: [
+          --line-length=120,
+          --check,
+          --diff,
+        ]
 
   - repo: local
     hooks:

_meta/config/beat.docker.yml.tmpl

@@ -1,2 +1,2 @@
 {{template "header.yml.tmpl" .}}
-{{template "cloudbeat.common.yml.tmpl" .}}
+{{template "cloudbeat.common.yml.tmpl" .}}

_meta/config/beat.reference.yml.tmpl

@@ -1,2 +1,2 @@
 {{template "header.reference.yml.tmpl" .}}
-{{template "cloudbeat.common.yml.tmpl" .}}
+{{template "cloudbeat.common.yml.tmpl" .}}

_meta/config/beat.yml.tmpl

@@ -1,2 +1,2 @@
 {{template "header.yml.tmpl" .}}
-{{template "cloudbeat.common.yml.tmpl" .}}
+{{template "cloudbeat.common.yml.tmpl" .}}

bin/.golangci-lint-1.50.1.pkg

@@ -0,0 +1 @@
+hermit

bin/.pre-commit-2.20.0.pkg

@@ -0,0 +1 @@
+hermit

bin/.python3@3.10.pkg

@@ -0,0 +1 @@
+hermit

bin/activate-hermit

@@ -1,6 +1,8 @@
 #!/bin/bash
 # This file must be used with "source bin/activate-hermit" from bash or zsh.
 # You cannot run it directly
+#
+# THIS FILE IS GENERATED; DO NOT MODIFY
 
 if [ "${BASH_SOURCE-}" = "$0" ]; then
   echo "You must source this script: \$ source $0" >&2

bin/golangci-lint

@@ -0,0 +1 @@
+.golangci-lint-1.50.1.pkg

bin/pip3

@@ -0,0 +1 @@
+.python3@3.10.pkg

bin/pre-commit

@@ -0,0 +1 @@
+.pre-commit-2.20.0.pkg

bin/python3

@@ -0,0 +1 @@
+.python3@3.10.pkg

cloudbeat.reference.yml

@@ -1110,7 +1110,7 @@ output.elasticsearch:
 
   # Permissions to use for file creation. The default is 0600.
   #permissions: 0600
-  
+
   # Configure automatic file rotation on every startup. The default is true.
   #rotate_on_startup: true
 
@@ -1682,4 +1682,3 @@ logging.files:
 
 # This allows to enable 6.7 migration aliases
 #migration.6_to_7.enabled: false
-

cloudbeat.yml

@@ -313,4 +313,3 @@ processors:
 
 # This allows to enable 6.7 migration aliases
 #migration.6_to_7.enabled: true
-

deploy/cloud/README.md

@@ -16,7 +16,7 @@ Create environment
 To connect to the environment use the console ui or see the details how to connect to the environment.
 
 Delete environment
-1. `terraform destroy --auto-approve` 
+1. `terraform destroy --auto-approve`
 
 **Next Steps**
 * [Setup](https://github.com/elastic/security-team/blob/main/docs/cloud-security-posture-team/onboarding/deploy-agent-cloudbeat-on-eks.mdx) EKS cluster
@@ -26,7 +26,7 @@ Delete environment
 # Examples
 
 ## Spesific version
-To create an environment with spesific version use `terraform apply --auto-approve -var="stack_version=8.5.1"` 
+To create an environment with spesific version use `terraform apply --auto-approve -var="stack_version=8.5.1"`
 
 ## Named environment
-To give your environment a different prefix in the name use `terraform apply --auto-approve -var="deployment_name_prefix=elastic-deployment"` 
+To give your environment a different prefix in the name use `terraform apply --auto-approve -var="deployment_name_prefix=elastic-deployment"`

deploy/cloud/data/dashboard.ndjson

@@ -5,4 +5,4 @@
 {"attributes":{"actions":[{"actionRef":"preconfigured:elastic-cloud-email","actionTypeId":".email","group":"threshold met","params":{"message":"alert '{{alertName}}' is active for group '{{context.group}}':\n\n- Value: {{context.value}}\n- Conditions Met: {{context.conditions}} over {{params.timeWindowSize}}{{params.timeWindowUnit}}\n- Timestamp: {{context.date}}","subject":"{{rule.name}} - Rule is active, Total findings: {{context.value}} ","to":["change.this@elastic.co"]}},{"actionRef":"action_1","actionTypeId":".slack","group":"threshold met","params":{"message":"alert '{{alertName}}' is active for group '{{context.group}}':\n\n- Value: {{context.value}}\n- Conditions Met: {{context.conditions}} over {{params.timeWindowSize}}{{params.timeWindowUnit}}\n- Timestamp: {{context.date}}"}}],"alertTypeId":".es-query","apiKey":null,"apiKeyOwner":null,"consumer":"alerts","createdAt":"2022-08-22T12:03:34.223Z","createdBy":"elastic","enabled":false,"executionStatus":{"error":null,"lastExecutionDate":"2022-11-22T13:53:01.710Z","status":"pending","warning":null},"legacyId":null,"meta":{"versionApiKeyLastmodified":"8.5.1"},"monitoring":{"execution":{"calculated_metrics":{"p50":827.5,"p95":1176.5999999999997,"p99":1734,"success_ratio":1},"history":[{"duration":110,"success":true,"timestamp":1661169816803},{"duration":1734,"success":true,"timestamp":1661169881425},{"duration":890,"success":true,"timestamp":1661169943614},{"duration":980,"success":true,"timestamp":1661170003826},{"duration":1056,"success":true,"timestamp":1661170066775},{"duration":752,"success":true,"timestamp":1661170129466},{"duration":712,"success":true,"timestamp":1661170192433},{"duration":1102,"success":true,"timestamp":1661170255834},{"duration":998,"success":true,"timestamp":1661170318729},{"duration":1193,"success":true,"timestamp":1661170381908},{"duration":852,"success":true,"timestamp":1661170444569},{"duration":548,"success":true,"timestamp":1661170507290},{"duration":686,"success":true,"timestamp":1661170570436},{"duration":878,"success":true,"timestamp":1661170633622},{"duration":1024,"success":true,"timestamp":1661170696836},{"duration":259,"success":true,"timestamp":1661170759017},{"duration":717,"success":true,"timestamp":1661170822482},{"duration":621,"success":true,"timestamp":1661170885368},{"duration":803,"success":true,"timestamp":1661170948563},{"duration":935,"success":true,"timestamp":1661171011723},{"duration":394,"success":true,"timestamp":1661171074188},{"duration":793,"success":true,"timestamp":1661171134684},{"duration":486,"success":true,"timestamp":1661171197249},{"duration":644,"success":true,"timestamp":1661171260407},{"duration":905,"success":true,"timestamp":1661171323703},{"duration":1010,"success":true,"timestamp":1661171386867},{"duration":1111,"success":true,"timestamp":1661171450016},{"duration":1007,"success":true,"timestamp":1661171512895},{"duration":773,"success":true,"timestamp":1661171575590},{"duration":921,"success":true,"timestamp":1661171622159},{"duration":83,"success":true,"timestamp":1661175223099},{"duration":987,"success":true,"timestamp":1668587228023},{"duration":222,"success":true,"timestamp":1669125066934},{"duration":128,"success":true,"timestamp":1669125141824}]}},"muteAll":false,"mutedInstanceIds":[],"name":"Expected Findings - Above 435","notifyWhen":"onActiveAlert","params":{"esQuery":"{\r\n    \"query\":{\r\n      \"bool\": {\r\n        \"filter\": [{\r\n          \"term\": {\r\n            \"rule.benchmark.id\": \"cis_k8s\"\r\n          }\r\n        }]\r\n      }\r\n    }\r\n  }","index":["logs-cloud_security_posture.findings_latest-default"],"searchType":"esQuery","size":1,"threshold":[435],"thresholdComparator":">","timeField":"@timestamp","timeWindowSize":5,"timeWindowUnit":"h"},"schedule":{"interval":"1h"},"scheduledTaskId":null,"snoozeSchedule":[],"tags":["CSP Deployment"],"throttle":null,"updatedAt":"2022-11-22T13:51:02.337Z","updatedBy":"4034003682"},"coreMigrationVersion":"8.5.1","id":"51ed915d-e74b-4e3d-b473-8c9e7650617e","migrationVersion":{"alert":"8.5.0"},"references":[{"id":"015cccd0-6588-11ed-98ea-a7ce41bfaa36","name":"action_1","type":"action"}],"type":"alert","updated_at":"2022-11-22T13:52:21.829Z","version":"WzI1OTMsMV0="}
 {"attributes":{"actions":[{"actionRef":"preconfigured:elastic-cloud-email","actionTypeId":".email","group":"query matched","params":{"message":"Elasticsearch query alert '{{alertName}}' is active:\n\n- Value: {{context.value}}\n- Conditions Met: {{context.conditions}} over {{params.timeWindowSize}}{{params.timeWindowUnit}}\n- Timestamp: {{context.date}}\n- Link: {{context.link}}","subject":"{{rule.name}} - Rule's status has changed, failed findings: {{context.value}} ","to":["change.this@elastic.co"]}},{"actionRef":"action_1","actionTypeId":".slack","group":"query matched","params":{"message":"Elasticsearch query alert '{{alertName}}' is active:\n\n- Value: {{context.value}}\n- Conditions Met: {{context.conditions}} over {{params.timeWindowSize}}{{params.timeWindowUnit}}\n- Timestamp: {{context.date}}\n- Link: {{context.link}}"}}],"alertTypeId":".es-query","apiKey":null,"apiKeyOwner":null,"consumer":"alerts","createdAt":"2022-09-05T12:20:53.220Z","createdBy":"elastic","enabled":false,"executionStatus":{"error":null,"lastExecutionDate":"2022-11-22T13:53:01.710Z","status":"pending","warning":null},"legacyId":null,"meta":{"versionApiKeyLastmodified":"8.5.1"},"monitoring":{"execution":{"calculated_metrics":{"p50":101,"p95":1052.6499999999999,"p99":1102,"success_ratio":1},"history":[{"duration":765,"success":true,"timestamp":1662380456298},{"duration":106,"success":true,"timestamp":1662380517754},{"duration":83,"success":true,"timestamp":1662380578604},{"duration":83,"success":true,"timestamp":1662380638707},{"duration":101,"success":true,"timestamp":1662380701631},{"duration":84,"success":true,"timestamp":1662380764631},{"duration":1102,"success":true,"timestamp":1662380829255},{"duration":76,"success":true,"timestamp":1662380890609},{"duration":84,"success":true,"timestamp":1662380953618},{"duration":961,"success":true,"timestamp":1662381017773},{"duration":80,"success":true,"timestamp":1662381079634},{"duration":90,"success":true,"timestamp":1662381142639},{"duration":88,"success":true,"timestamp":1662381205644},{"duration":901,"success":true,"timestamp":1662381222520},{"duration":326,"success":true,"timestamp":1668587227361},{"duration":229,"success":true,"timestamp":1669125066938},{"duration":116,"success":true,"timestamp":1669125141811}]}},"muteAll":false,"mutedInstanceIds":[],"name":"Expected Failed Findings - Below 62","notifyWhen":"onActionGroupChange","params":{"esQuery":"{\r\n    \"query\":{\r\n      \"bool\": {\r\n        \"filter\": [{\r\n          \"term\": {\r\n            \"result.evaluation\": \"failed\"\r\n          }\r\n        }, {\r\n          \"term\": {\r\n            \"rule.benchmark.id\": \"cis_k8s\"\r\n          }\r\n        }]\r\n      }\r\n    }\r\n  }","index":["logs-cloud_security_posture.findings_latest-default"],"searchType":"esQuery","size":1,"threshold":[62],"thresholdComparator":"<","timeField":"@timestamp","timeWindowSize":5,"timeWindowUnit":"h"},"schedule":{"interval":"1h"},"scheduledTaskId":null,"snoozeSchedule":[],"tags":["CSP Deployment"],"throttle":null,"updatedAt":"2022-11-22T13:51:02.335Z","updatedBy":"4034003682"},"coreMigrationVersion":"8.5.1","id":"2f3c0860-2d15-11ed-9d27-95e61229c57c","migrationVersion":{"alert":"8.5.0"},"references":[{"id":"015cccd0-6588-11ed-98ea-a7ce41bfaa36","name":"action_1","type":"action"}],"type":"alert","updated_at":"2022-11-22T13:52:21.814Z","version":"WzI1OTIsMV0="}
 {"attributes":{"actions":[{"actionRef":"preconfigured:elastic-cloud-email","actionTypeId":".email","group":"query matched","params":{"message":"Elasticsearch query alert '{{alertName}}' is active:\n\n- Value: {{context.value}}\n- Conditions Met: {{context.conditions}} over {{params.timeWindowSize}}{{params.timeWindowUnit}}\n- Timestamp: {{context.date}}\n- Link: {{context.link}}","subject":"{{rule.name}} - Rule's status has changed, failed findings: {{context.value}} ","to":["change.this@elastic.co"]}},{"actionRef":"action_1","actionTypeId":".slack","group":"query matched","params":{"message":"Elasticsearch query alert '{{alertName}}' is active:\n\n- Value: {{context.value}}\n- Conditions Met: {{context.conditions}} over {{params.timeWindowSize}}{{params.timeWindowUnit}}\n- Timestamp: {{context.date}}\n- Link: {{context.link}}"}}],"alertTypeId":".es-query","apiKey":null,"apiKeyOwner":null,"consumer":"alerts","createdAt":"2022-09-05T12:35:51.862Z","createdBy":"elastic","enabled":false,"executionStatus":{"error":null,"lastExecutionDate":"2022-11-22T13:53:01.710Z","status":"pending","warning":null},"legacyId":null,"meta":{"versionApiKeyLastmodified":"8.5.1"},"monitoring":{"execution":{"calculated_metrics":{"p50":176.5,"p95":2531,"p99":2531,"success_ratio":1},"history":[{"duration":103,"success":true,"timestamp":1662381356063},{"duration":2531,"success":true,"timestamp":1668587226548},{"duration":235,"success":true,"timestamp":1669125066948},{"duration":118,"success":true,"timestamp":1669125138833}]}},"muteAll":false,"mutedInstanceIds":[],"name":"Expected Failed Findings - Above 62","notifyWhen":"onActionGroupChange","params":{"esQuery":"{\r\n    \"query\":{\r\n      \"bool\": {\r\n        \"filter\": [{\r\n          \"term\": {\r\n            \"result.evaluation\": \"failed\"\r\n          }\r\n        }, {\r\n          \"term\": {\r\n            \"rule.benchmark.id\": \"cis_k8s\"\r\n          }\r\n        }]\r\n      }\r\n    }\r\n  }","index":["logs-cloud_security_posture.findings_latest-default"],"searchType":"esQuery","size":1,"threshold":[62],"thresholdComparator":">","timeField":"@timestamp","timeWindowSize":5,"timeWindowUnit":"h"},"schedule":{"interval":"1h"},"scheduledTaskId":null,"snoozeSchedule":[],"tags":["CSP Deployment"],"throttle":null,"updatedAt":"2022-11-22T13:51:02.339Z","updatedBy":"4034003682"},"coreMigrationVersion":"8.5.1","id":"467bf560-2d17-11ed-9d27-95e61229c57c","migrationVersion":{"alert":"8.5.0"},"references":[{"id":"015cccd0-6588-11ed-98ea-a7ce41bfaa36","name":"action_1","type":"action"}],"type":"alert","updated_at":"2022-11-22T13:52:18.836Z","version":"WzI1ODMsMV0="}
-{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":7,"missingRefCount":0,"missingReferences":[]}
+{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":7,"missingRefCount":0,"missingReferences":[]}

deploy/cloud/data/rules.ndjson

@@ -60,4 +60,4 @@ resource "null_resource" "rules" {
   triggers = {
     dashboard_sha1 = "${sha1(file("data/rules.ndjson"))}"
   }
-}
+}

deploy/cloud/main.tf

@@ -21,4 +21,4 @@ output "kibana_url" {
 output "admin_console_url" {
   value       = module.ec_deployment.admin_console_url
   description = "The admin console URL"
-}
+}

deploy/cloud/outputs.tf

@@ -56,4 +56,4 @@ variable "security_team_repository" {
 variable "deployment_name_prefix" {
   default = "cloudbeat"
   description = "Optional set a prefix of the deployment"
-}
+}

deploy/cloud/variables.tf

@@ -22,7 +22,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
         - name: elastic-agent
-          image: 704479110758.dkr.ecr.eu-west-1.amazonaws.com/elastic-agent:csp-latest 
+          image: 704479110758.dkr.ecr.eu-west-1.amazonaws.com/elastic-agent:csp-latest
           imagePullPolicy: IfNotPresent
           env:
             - name: FLEET_ENROLL

deploy/k8s/fleet-managed-agent.yml

@@ -683,4 +683,3 @@ metadata:
   namespace: kube-system
   labels:
     k8s-app: elastic-agent
-

deploy/k8s/standalone-agent.yml

@@ -38,4 +38,4 @@ subjects:
 roleRef:
   kind: Role
   name: cloudbeat-kubeadm-config
-  apiGroup: rbac.authorization.k8s.io
+  apiGroup: rbac.authorization.k8s.io

deploy/kustomize/base/role-binding.yml

@@ -91,4 +91,4 @@ rules:
       - configmaps
     resourceNames:
       - kubeadm-config
-    verbs: ["get"]
+    verbs: ["get"]

deploy/kustomize/base/role.yml

@@ -5,4 +5,4 @@ metadata:
   name: cloudbeat
   namespace: kube-system
   labels:
-    k8s-app: cloudbeat
+    k8s-app: cloudbeat

deploy/kustomize/base/service-account.yml

@@ -20,6 +20,3 @@ configMapGenerator:
     behavior: create
     files:
       - cloudbeat.yml
-
-
-

deploy/kustomize/overlays/cloudbeat-eks/kustomization.yml

@@ -1,2 +1,2 @@
 # kustomize will read it and populate into the final manifests
-FLEET_ENROLLMENT_TOKEN
+FLEET_ENROLLMENT_TOKEN

deploy/kustomize/overlays/cloudbeat-vanilla-agent/.fleet-token.env

@@ -4,7 +4,7 @@ kind: Kustomization
 namespace: kube-system
 
 resources:
-  - manifests.yaml 
+  - manifests.yaml
 
 patches:
   - path: ./patches/patchs.yaml
@@ -24,4 +24,4 @@ secretGenerator:
 images:
   - name: docker.elastic.co/beats/elastic-agent:8.5.0-SNAPSHOT
     newName: elastic-agent
-    newTag: 8.5.0-SNAPSHOT
+    newTag: 8.5.0-SNAPSHOT

deploy/kustomize/overlays/cloudbeat-vanilla-agent/kustomization.yml

@@ -290,4 +290,4 @@ metadata:
   namespace: kube-system
   labels:
     k8s-app: elastic-agent
----
+---

deploy/kustomize/overlays/cloudbeat-vanilla-agent/manifests.yaml

@@ -14,4 +14,4 @@
   path: /spec/template/spec/containers/0/volumeMounts/-
   value:
     name: elastic-package-certs
-    mountPath: /etc/ssl/elastic-package
+    mountPath: /etc/ssl/elastic-package

deploy/kustomize/overlays/cloudbeat-vanilla-agent/patches/patchs.yaml

@@ -25,4 +25,4 @@ spec:
         - name: elastic-package-ca
           secret:
             defaultMode: 420
-            secretName: elastic-package-certs
+            secretName: elastic-package-certs

deploy/kustomize/overlays/cloudbeat-vanilla/patches/patch-add-ssl-certs.yaml

@@ -2,4 +2,4 @@ resources:
   - ../base
 
 patchesStrategicMerge:
-  - cloudbeat-ds.yml
+  - cloudbeat-ds.yml