POC - Adding CIS and CDR integrations to an existing deployment (#2953)

gurevichdmitry · web-flow · commit 5be7ff155305 · 2025-03-12T12:18:13.000+02:00
* add new wf

* modify wfs to support cis and cdr install

* update wf

* update wfs

* add stack version

* update action condition

* update documentation
diff --git a/.github/workflows/install-integrations.yml b/.github/workflows/install-integrations.yml
@@ -0,0 +1,78 @@
+name: Install Integrations
+run-name: Install integrations by @${{ github.actor }}
+
+on:
+  workflow_dispatch:
+    inputs:
+      deployment-name:
+        type: string
+        description: |
+          Name with letters, numbers, hyphens; start with a letter. Max 20 chars. e.g., 'my-env-123'
+        required: true
+      stack-version:
+        type: string
+        description: "The version of the stack to deploy"
+        required: true
+      kibana-url:
+        type: string
+        description: "The Kibana URL to install the integrations"
+        required: true
+      kibana-username:
+        type: string
+        description: "The Kibana username to install the integrations"
+        required: true
+      kibana-password:
+        type: string
+        description: "The Kibana password to install the integrations"
+        required: true
+      infra-type:
+        description: 'Choose an option (all, cdr, cis)'
+        required: true
+        type: choice
+        options:
+          - all
+          - cdr
+          - cis
+      docker-image-override:
+        required: false
+        description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)"
+        type: string
+
+jobs:
+  naming:
+    runs-on: ubuntu-latest
+    outputs:
+      es-password: ${{ steps.password.outputs.kbn-password }}
+    steps:
+      - name: Mask Sensitive Data
+        id: password
+        env:
+          SECRET: ${{ secrets.GPG_PASSPHRASE }}
+        run: |
+          kbn_password=$(jq -r '.inputs["kibana-password"]' $GITHUB_EVENT_PATH)
+          echo "::add-mask::$kbn_password"
+          kbn_password_encrypted=$(gpg --symmetric --batch --passphrase "$SECRET" --output - <(echo "$kbn_password") | base64 -w0)
+          echo "kbn-password=$kbn_password_encrypted" >> $GITHUB_OUTPUT
+
+  deploy:
+    needs: naming
+    uses: ./.github/workflows/test-environment.yml
+    secrets: inherit
+    # Required for the 'Deploy' job in the 'test-environment.yml' to authenticate with Google Cloud (gcloud).
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    with:
+      deployment_name: ${{ inputs.deployment-name }}
+      # For now, the region is not used because it's overridden in the tf, but it's here for future compatibility.
+      ess-region: "gcp-us-west2"
+      elk-stack-version: ${{ inputs.stack-version }}
+      serverless_mode: false
+      agentless: false
+      expiration_days: 14
+      infra-type: ${{ inputs.infra-type }}
+      deploy-stack: false
+      ext-kibana-url: ${{ inputs.kibana-url }}
+      ext-es-user: ${{ inputs.kibana-username }}
+      ext-es-password: ${{ needs.naming.outputs.es-password }}
+      docker-image-override: ${{ inputs.docker-image-override }}
diff --git a/.github/workflows/test-environment.yml b/.github/workflows/test-environment.yml
@@ -113,6 +113,27 @@ on:
         type: boolean
         required: false
         default: true
+      ext-kibana-url:
+        description: "External Kibana URL for update existing environment"
+        type: string
+        required: false
+      ext-es-url:
+        description: "External Elasticsearch URL for update existing environment"
+        type: string
+        required: false
+      ext-es-user:
+        description: "External Elasticsearch user for update existing environment"
+        type: string
+        required: false
+      ext-es-password:
+        description: "External Elasticsearch password for update existing environment"
+        type: string
+        required: false
+      deploy-stack:
+        description: "Deploy stack"
+        type: boolean
+        required: false
+        default: true
     outputs:
       s3-bucket:
         description: "Terraform state s3 bucket folder"
@@ -239,6 +260,17 @@ jobs:
             echo "INFRA_TYPE=$INPUT_INFRA_TYPE" >> $GITHUB_ENV
           fi
 
+      - name: Init Deploy Stack
+        id: init-deploy-stack
+        env:
+          INIT_DEPLOY_STACK: ${{ inputs.deploy-stack }}
+        run: |
+          if [[ "${INIT_DEPLOY_STACK}" == "true" || -z "${INIT_DEPLOY_STACK}" ]]; then
+            echo "DEPLOY_STACK=true" >> $GITHUB_ENV
+          else
+            echo "DEPLOY_STACK=false" >> $GITHUB_ENV
+          fi
+
       - name: Init Agent Based
         id: init-agent-based
         env:
@@ -304,6 +336,7 @@ jobs:
 
       - name: Deploy ELK Cloud Stack
         id: elk-stack
+        if: ${{ env.DEPLOY_STACK == 'true' }}
         uses: ./.github/actions/elk-stack
         with:
           deployment-name: ${{ env.DEPLOYMENT_NAME }}
@@ -328,7 +361,8 @@ jobs:
           python3 ../../.ci/scripts/create_env_config.py
           aws s3 cp "./env_config.json" "${S3_BUCKET}/env_config.json"
 
-      - name: Update Stack Vars
+      - name: Update Stack Vars - new Deployment
+        if: ${{ env.DEPLOY_STACK == 'true' }}
         env:
           STACK_ES_USER: ${{ steps.elk-stack.outputs.es-user }}
           STACK_ES_PASSWORD: ${{ steps.elk-stack.outputs.es-password }}
@@ -340,6 +374,23 @@ jobs:
           echo "KIBANA_URL=$STACK_KIBANA_URL" >> $GITHUB_ENV
           echo "ES_URL=$STACK_ES_URL" >> $GITHUB_ENV
 
+      - name: Update Stack Vars - existing Deployment
+        if: ${{ env.DEPLOY_STACK == 'false' }}
+        env:
+          USER_ES_USER: ${{ inputs.ext-es-user || '' }}
+          USER_ES_PASSWORD: ${{ inputs.ext-es-password }}
+          USER_KIBANA_URL: ${{ inputs.ext-kibana-url || '' }}
+          USER_ES_URL: ${{ inputs.ext-es-url || '' }}
+          SECRET: ${{ secrets.GPG_PASSPHRASE }}
+        run: |
+          echo "Using user-provided environment values..."
+          echo "ES_USER=$USER_ES_USER" >> $GITHUB_ENV
+          user_password=$(gpg --decrypt --quiet --batch --passphrase "$SECRET" --output - <(echo "$USER_ES_PASSWORD" | base64 --decode))
+          echo "::add-mask::$user_password"
+          echo "ES_PASSWORD=$user_password" >> $GITHUB_ENV
+          echo "KIBANA_URL=$USER_KIBANA_URL" >> $GITHUB_ENV
+          echo "ES_URL=$USER_ES_URL" >> $GITHUB_ENV
+
       - name: Summary
         if: success()
         run: |
@@ -357,7 +408,7 @@ jobs:
 
       - name: Deploy CDR Integrations
         id: cdr-integrations
-        if: ${{ !cancelled() && steps.elk-stack.outcome == 'success' && env.INFRA_TYPE != 'cis' }}
+        if: ${{ !cancelled() && (steps.elk-stack.outcome == 'success' || env.DEPLOY_STACK == 'false') && env.INFRA_TYPE != 'cis' }}
         uses: ./.github/actions/cdr
         with:
           deployment-name: ${{ env.DEPLOYMENT_NAME }}
@@ -372,17 +423,17 @@ jobs:
           wiz-endpoint-url: ${{ secrets.WIZ_ENDPOINT_URL }}
           wiz-token-url: ${{ secrets.WIZ_TOKEN_URL }}
           env-s3-bucket: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
-          es-user: ${{ steps.elk-stack.outputs.es-user }}
-          es-password: ${{ steps.elk-stack.outputs.es-password }}
-          kibana-url: ${{ steps.elk-stack.outputs.kibana-url }}
+          es-user: ${{ env.ES_USER }}
+          es-password: ${{ env.ES_PASSWORD }}
+          kibana-url: ${{ env.KIBANA_URL }}
           elk-stack-version: ${{ env.STACK_VERSION }}
           azure-tags: ${{ env.AZURE_DEFAULT_TAGS }}
           tag-project: ${{ github.actor }}
           tag-owner: ${{ github.actor }}
 
       - name: Deploy CIS Agent Based Integrations
         id: cis-integrations
-        if: ${{ !cancelled() && env.AGENT_BASED == 'true' && steps.elk-stack.outcome == 'success' && env.INFRA_TYPE != 'cdr' }}
+        if: ${{ !cancelled() && env.AGENT_BASED == 'true' && (steps.elk-stack.outcome == 'success' || env.DEPLOY_STACK == 'false') && env.INFRA_TYPE != 'cdr' }}
         uses: ./.github/actions/cis-agent-based
         with:
           deployment-name: ${{ env.DEPLOYMENT_NAME }}
@@ -392,23 +443,23 @@ jobs:
           cspm-azure-tags: ${{ env.AZURE_DEFAULT_TAGS }}
           stack-enrollment-token: ${{ env.ENROLLMENT_TOKEN }}
           env-s3-bucket: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
-          es-user: ${{ steps.elk-stack.outputs.es-user }}
-          es-password: ${{ steps.elk-stack.outputs.es-password }}
-          kibana-url: ${{ steps.elk-stack.outputs.kibana-url }}
+          es-user: ${{ env.ES_USER }}
+          es-password: ${{ env.ES_PASSWORD }}
+          kibana-url: ${{ env.KIBANA_URL }}
           docker-image-override: ${{ env.DOCKER_IMAGE_OVERRIDE }}
           serverless-mode: "${{ env.TF_VAR_serverless_mode }}"
           tag-project: ${{ github.actor }}
           tag-owner: ${{ github.actor }}
 
       - name: Deploy CIS Agentless Integrations
         id: cis-agentless-integrations
-        if: ${{ !cancelled() && env.AGENTLESS == 'true' && steps.elk-stack.outcome == 'success' && env.INFRA_TYPE != 'cdr' }}
+        if: ${{ !cancelled() && env.AGENTLESS == 'true' && (steps.elk-stack.outcome == 'success' || env.DEPLOY_STACK == 'false') && env.INFRA_TYPE != 'cdr' }}
         uses: ./.github/actions/cis-agentless
         with:
           cspm-azure-creds: ${{ secrets.AZURE_CREDENTIALS }}
-          es-user: ${{ steps.elk-stack.outputs.es-user }}
-          es-password: ${{ steps.elk-stack.outputs.es-password }}
-          kibana-url: ${{ steps.elk-stack.outputs.kibana-url }}
+          es-user: ${{ env.ES_USER }}
+          es-password: ${{ env.ES_PASSWORD }}
+          kibana-url: ${{ env.KIBANA_URL }}
 
       - name: Wait for agents to enroll
         id: wait-for-agents
diff --git a/dev-docs/Cloud-Env-Testing.md b/dev-docs/Cloud-Env-Testing.md
@@ -146,6 +146,21 @@ The [`Create Environment with Cloud Logs`](https://github.com/elastic/cloudbeat/
 
 The workflow requires a subset of input parameters. All required inputs are described [here](#how-to-run-the-workflow).
 
+## Install Integrations Worfklow
+
+The [`Install Integrations`](https://github.com/elastic/cloudbeat/actions/workflows/install-integrations.yml) GitHub workflow is used when the Elastic Stack is already installed, and the user wants to add `CIS` and/or `CDR` integrations.
+
+### Workflow Inputs
+
+- **`stack-version`** - The version of the stack to deploy.
+- **`kibana-url`** - The Kibana URL where the integrations will be installed.
+- **`kibana-username`** - The username for Kibana login.
+- **`kibana-password`** - The password for Kibana login.
+- **`infra-type`** - The type of integrations to install, with three allow options:
+  - **`all`** - Installs both `CIS` and `CDR` integrations.
+  - **`cis`** - Installs `CSPM`, `KSPM`, and `CNVM` integrations.
+  - **`cdr`** - Installs `Audit Logs`, `Asset Inventory`, and `Wiz` integrations.
+- **`docker-image-override`** - For build candidate versions, specifies a custom Docker image path for agent installations.
 
 ## Cleanup Procedure
 
