Add csp-security-policies

Author: orestisfl

.buildkite/scripts/generate_notice.py

@@ -26,7 +26,6 @@
 notice_overrides = [
     {"name": "github.com/elastic/beats/v7", "licenceType": "Elastic"},
     {"name": "github.com/build-security/beats/v7", "licenceType": "Elastic"},
-    {"name": "github.com/elastic/csp-security-policies", "licenceType": "Elastic"},
     {"name": "github.com/golang/glog", "licenceType": "Apache-2.0"},
     {"name": "github.com/spdx/tools-golang", "licenceFile": "LICENSE.code", "licenceType": "Apache-2.0"},
 ]

.github/pull_request_template.md

@@ -19,3 +19,9 @@ If this PR adds a new feature, please add an example screenshot or data (finding
 ### Checklist
 - [ ] I have added tests that prove my fix is effective or that my feature works
 - [ ] I have added the necessary README/documentation (if appropriate)
+
+#### Introducing a new rule?
+
+- [ ] Generate rule metadata using [this script](https://github.com/elastic/cloudbeat/tree/main/security-policies/dev#generate-rules-metadata)
+- [ ] Add relevant unit tests
+- [ ] Generate relevant rule templates using [this script](https://github.com/elastic/cloudbeat/tree/main/security-policies/dev#generate-rule-templates), and open a PR in [elastic/packages/cloud_security_posture](https://github.com/elastic/integrations/tree/main/packages/cloud_security_posture)

.github/workflows/cloudbeat-ci.yml

@@ -34,6 +34,12 @@ jobs:
       - name: Init Hermit
         run: ./bin/hermit env -r >> $GITHUB_ENV
 
+      - name: Install poetry
+        run: |
+          pip3 install poetry
+          cd security-policies
+          poetry install
+
       - name: Pre-commit Hooks
         env:
           # Skipping golangci-lint as it's tested by golangci/golangci-lint-action

security-policies/.github/workflows/test-coverage.yml .github/workflows/test-opa-coverage.yml

@@ -1,4 +1,4 @@
-name: Test Coverage
+name: Test OPA Coverage
 
 on:
   push:
@@ -8,15 +8,15 @@ on:
 jobs:
   test-coverage:
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./security-policies
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Download OPA Binary
-        run: |
-          curl -L -o opa https://openpolicyagent.org/downloads/latest/opa_linux_amd64
-          chmod +x opa
-          ./opa -h
+      - name: Init Hermit
+        run: ../bin/hermit env -r >> $GITHUB_ENV
 
       - name: OPA test coverage
         run: |

.github/workflows/test-opa-policies.yml

@@ -0,0 +1,35 @@
+name: Test OPA Policies
+
+on:
+  pull_request:
+  push:
+    branches: [ main ]
+
+jobs:
+  test-rego-policies:
+    name: Test Rego Policies
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./security-policies
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Init Hermit
+        run: ../bin/hermit env -r >> $GITHUB_ENV
+
+      - name: OPA format
+        run: opa fmt ./bundle --fail=true --diff
+
+      - name: OPA format list failed files
+        run: opa fmt ./bundle --list
+
+      - name: OPA build
+        run: opa build -b ./bundle -e ./bundle/compliance
+
+      - name: OPA test
+        run: opa test -b ./bundle -v
+
+      - name: OPA check -strict
+        run: opa check --strict --bundle ./bundle

.mergify.yml

@@ -4,6 +4,9 @@ queue_rules:
       - check-success=Cloudbeat-CI / Integration Tests (pull_request)
       - check-success=Cloudbeat-CI / Build (pull_request)
       - check-success=UnitTests / Unit Tests (pull_request)
+      - check-success=Test Rego Policies (pull_request)
+      - check-success=Update rules status (pull_request)
+      - check-success=Update rules metadata (pull_request)
 pull_request_rules:
   - name: self-assign PRs
     conditions:

.pre-commit-config.yaml

@@ -12,7 +12,7 @@ repos:
       - id: check-json
       - id: check-executables-have-shebangs
       - id: trailing-whitespace
-        exclude: cloudbeat.reference.yml
+        exclude: (cloudbeat.reference.yml|security-policies/bundle/.*)
 
       ## Security
       - id: detect-private-key
@@ -44,6 +44,8 @@ repos:
             "-rn", # Only display messages
             "--rcfile=tests/pylintrc", # Link to your config file
           ]
+        # TODO: Apply pylint in security-policies
+        exclude: security-policies.*
 
   - repo: https://github.com/psf/black
     rev: 23.9.1
@@ -91,3 +93,62 @@ repos:
         language: system
         files: .*.go
         pass_filenames: false
+
+  - repo: local
+    hooks:
+      - id: opa-fmt
+        name: OPA fmt
+        description: Formats Rego policy using opa fmt
+        entry: opa fmt
+        args: [ -w ]
+        language: system
+        files: (\.rego)$
+
+      - id: opa-check
+        name: OPA check
+        description: Check syntax of staged Rego files
+        entry: opa check
+        args: [ -S, './security-policies/bundle/compliance' ]
+        pass_filenames: false
+        language: system
+        files: (\.rego)$
+
+      - id: opa-test
+        name: OPA test
+        description: Runs OPA unit tests on rego source files
+        entry: opa test
+        args: [ -b, './security-policies/bundle' ]
+        pass_filenames: false
+        language: system
+        files: (\.rego)$
+
+      - id: update-rules-status
+        name: Update rule status
+        description: Update rule status in rules_table.md
+        require_serial: true
+        entry: poetry run -C security-policies python security-policies/dev/update_rule_status.py
+        pass_filenames: false
+        language: system
+        files: (\.rego)$
+
+      - id: update-rule-metadata
+        name: Update rule metadata
+        description: Update rule status in rules_table.md
+        require_serial: true
+        entry: poetry run -C security-policies python security-policies/dev/generate_rule_metadata.py
+        pass_filenames: false
+        language: system
+        files: (\.rego)$
+
+  - repo: https://github.com/igorshubovych/markdownlint-cli
+    rev: v0.32.2
+    hooks:
+      - id: markdownlint
+        args: [ '--disable',
+                MD013,  # Line length can be ignored for now
+                MD033,  # Allow inline HTML
+                MD046,  # Allow code blocks to be fenced with backticks
+                MD041,  # Allow multiple top level headers
+                '--' ]
+        # TODO: Apply makrdownlint in other files
+        files: security-policies/.*\.(md|markdown)$

README.md

@@ -10,7 +10,7 @@ It is designed to be used as part of the [Cloud Security](https://www.elastic.co
 
 ### CSP Security Policies
 
-Cloudbeat uses security policies from the [CSP Security Policies](https://github.com/elastic/csp-security-policies) repository to evaluate cloud resources.
+Cloudbeat uses security policies from the [Security Policies](./security-policies) directory to evaluate cloud resources.
 
 ## Getting Started
 

justfile

@@ -4,7 +4,6 @@ kustomizeVanillaOverlay := "deploy/kustomize/overlays/cloudbeat-vanilla"
 kustomizeVanillaNoCertOverlay := "deploy/kustomize/overlays/cloudbeat-vanilla-nocert"
 kustomizeEksOverlay := "deploy/kustomize/overlays/cloudbeat-eks"
 kustomizeAwsOverlay := "deploy/kustomize/overlays/cloudbeat-aws"
-cspPoliciesPkg := "github.com/elastic/csp-security-policies"
 hermitActivationScript := "bin/activate-hermit"
 
 # use env var if available

magefile.go

@@ -35,7 +35,6 @@ import (
 	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
 	"github.com/elastic/e2e-testing/pkg/downloads"
 	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
@@ -50,7 +49,6 @@ import (
 	_ "github.com/elastic/beats/v7/dev-tools/mage/target/unittest"
 
 	cloudbeat "github.com/elastic/cloudbeat/scripts/mage"
-	"github.com/elastic/cloudbeat/version"
 )
 
 const (
@@ -350,57 +348,13 @@ func checkoutBranch(wt *git.Worktree, branch string) error {
 }
 
 func BuildOpaBundle() (err error) {
-	owner := "elastic"
-	repoName := "csp-security-policies"
-
 	// Override default SIGINT behaviour which does not allow deferred functions to be called
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, os.Interrupt)
 
-	cspPoliciesPkgDir, err := os.MkdirTemp("", repoName)
-	if err != nil {
-		return err
-	}
-
-	defer func() {
-		rmErr := os.RemoveAll(cspPoliciesPkgDir)
-		if rmErr != nil && err == nil {
-			err = rmErr
-		}
-		signal.Stop(c)
-	}()
-
-	repo, err := git.PlainClone(cspPoliciesPkgDir, false, &git.CloneOptions{
-		URL: fmt.Sprintf("https://github.com/%s/%s.git", owner, repoName),
-	})
-	if err != nil {
-		return err
-	}
-	err = repo.Fetch(&git.FetchOptions{
-		RefSpecs: []config.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD"},
-	})
-	if err != nil {
-		return err
-	}
-	// Check out the provided release tag commit
-	wt, err := repo.Worktree()
-	if err != nil {
-		return err
-	}
-
-	branch := getMajorMinorVersion(version.CloudbeatVersion().Version)
-	if err := checkoutBranch(wt, branch); err != nil {
-		fmt.Printf("Fallback from %s to main branch\n", branch)
-		branch = "main"
-		if err = checkoutBranch(wt, branch); err != nil {
-			return err
-		}
-	}
-
-	if err = sh.Run("bin/opa", "build", "-b", cspPoliciesPkgDir+"/bundle", "-e", cspPoliciesPkgDir+"/bundle/compliance"); err != nil {
+	if err = sh.Run("bin/opa", "build", "-b", "security-policies/bundle", "-e", "security-policies/bundle/compliance"); err != nil {
 		return err
 	}
 
-	fmt.Printf("Generated OPA bundle from %s branch at %s\n", branch, cspPoliciesPkgDir)
 	return nil
 }