Merge branch 'main' into dependabot/go_modules/github.com/elastic/gosigar-0.14.3

Author: cmacknz

.agent-versions.json

@@ -1,8 +1,9 @@
 {
   "testVersions": [
-    "8.14.0-SNAPSHOT",
-    "8.13.0-SNAPSHOT",
+    "8.13.3-SNAPSHOT",
+    "8.13.2",
     "8.12.2",
-    "7.17.18"
+    "7.17.21-SNAPSHOT",
+    "7.17.20"
   ]
 }

.buildkite/hooks/pre-exit

@@ -10,11 +10,8 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" ==
 
   # Perform cleanup of integration tests resources
   echo "--- Cleaning up integration test resources"
-  if [[ "$BUILDKITE_STEP_KEY" == "serverless-integration-tests" ]]; then
-    STACK_PROVISIONER=serverless SNAPSHOT=true mage integration:clean
-  else
-    SNAPSHOT=true mage integration:clean
-  fi
+  STACK_PROVISIONER=serverless SNAPSHOT=true mage integration:clean
+  SNAPSHOT=true mage integration:clean
 fi
 
 if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then

.buildkite/misc/gce-cleanup.yml

@@ -18,7 +18,7 @@ scanners:
         filters:
           - type: "<"
             pointer: "/extra/creationTimestamp"
-            param: "${CREATION_DATE}"
+            param: "${DELETE_CREATED_AFTER_DATE}"
             converters:
               param: "date"
               value: "date"

.buildkite/pipeline.elastic-agent-package.yml

@@ -10,10 +10,10 @@ env:
 
 steps:
   - input: "Build parameters"
-    if: build.env("ManifestURL") == null
+    if: build.env("MANIFEST_URL") == null
     fields:
-      - text: "ManifestURL"
-        key: "ManifestURL"
+      - text: "MANIFEST_URL"
+        key: "MANIFEST_URL"
         default: ""
         required: true
         hint: "Link to the build manifest URL."
@@ -51,7 +51,7 @@ steps:
         hint: "If the DRA release manager script would actually publish anything or just print"
 
   - wait: ~
-    if: build.env("ManifestURL") == null
+    if: build.env("MANIFEST_URL") == null
 
   - group: ":Packaging Artefacts"
     key: "package"
@@ -63,10 +63,10 @@ steps:
           machineType: "c2-standard-16"
           diskSizeGb: 400
         command: |
-          if [[ -z "${ManifestURL}" ]]; then
-            export ManifestURL=$(buildkite-agent meta-data get ManifestURL --default "")
-            if [[ -z "${ManifestURL}" ]]; then
-              echo ":broken_heart: Missing ManifestURL variable or empty string provided"
+          if [[ -z "${MANIFEST_URL}" ]]; then
+            export MANIFEST_URL=$(buildkite-agent meta-data get MANIFEST_URL --default "")
+            if [[ -z "${MANIFEST_URL}" ]]; then
+              echo ":broken_heart: Missing MANIFEST_URL variable or empty string provided"
               exit 1
             fi
           fi
@@ -86,10 +86,10 @@ steps:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
         command: |
-          if [[ -z "${ManifestURL}" ]]; then
-            export ManifestURL=$(buildkite-agent meta-data get ManifestURL --default "")
-            if [[ -z "${ManifestURL}" ]]; then
-              echo ":broken_heart: Missing ManifestURL variable or empty string provided"
+          if [[ -z "${MANIFEST_URL}" ]]; then
+            export MANIFEST_URL=$(buildkite-agent meta-data get MANIFEST_URL --default "")
+            if [[ -z "${MANIFEST_URL}" ]]; then
+              echo ":broken_heart: Missing MANIFEST_URL variable or empty string provided"
               exit 1
             fi
           fi
@@ -134,6 +134,8 @@ steps:
     depends_on: package
     agents:
       provider: "gcp"
+      machineType: "n2-standard-8"
+      diskSizeGb: 250
     env:
       DRA_PROJECT_ID: "elastic-agent-package"
       DRA_PROJECT_ARTIFACT_ID: "agent-package"

.buildkite/pipeline.yml

@@ -231,7 +231,7 @@ steps:
       - "build/diagnostics/*"
     agents:
       provider: "gcp"
-      machineType: "n1-standard-8"    
+      machineType: "n1-standard-8"
 
   - label: "Integration tests"
     key: "integration-tests"
@@ -246,7 +246,7 @@ steps:
   - label: "Serverless Beats Tests"
     key: "serverless-beats-integration-tests"
     command: ".buildkite/scripts/steps/beats_tests.sh"
-    if: "build.env('CRON') == 'yes'"
+    # if: "build.env('CRON') == 'yes'"
     agents:
       provider: "gcp"
       machineType: "n1-standard-8"

.buildkite/scripts/steps/gce-cleanup.sh

@@ -5,11 +5,11 @@ set -euo pipefail
 export ACCOUNT_KEY_SECRET=$(vault kv get -field=client_email $VAULT_PATH)
 export ACCOUNT_SECRET=$(vault kv get -field=private_key $VAULT_PATH)
 export ACCOUNT_PROJECT_SECRET=$(vault kv get -field=project_id $VAULT_PATH)
-export CREATION_DATE=$(date -Is -d "24 hours ago")
+export DELETE_CREATED_AFTER_DATE=$(date -Is -d "5 hours ago")
 
 docker run -v $(pwd)/.buildkite/misc/gce-cleanup.yml:/etc/cloud-reaper/config.yml \
   -e ACCOUNT_SECRET="$ACCOUNT_SECRET" \
   -e ACCOUNT_KEY="$ACCOUNT_KEY_SECRET" \
   -e ACCOUNT_PROJECT=$ACCOUNT_PROJECT_SECRET \
-  -e CREATION_DATE=$CREATION_DATE \
+  -e DELETE_CREATED_AFTER_DATE=$DELETE_CREATED_AFTER_DATE \
   ${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper --config /etc/cloud-reaper/config.yml destroy --confirm

.buildkite/scripts/steps/integration_tests.sh

@@ -11,15 +11,15 @@ MAGE_SUBTARGET="${3:-""}"
 # Override the agent package version using a string with format <major>.<minor>.<patch>
 # NOTE: use only after version bump when the new version is not yet available, for example:
 # OVERRIDE_AGENT_PACKAGE_VERSION="8.10.3" otherwise OVERRIDE_AGENT_PACKAGE_VERSION="".
-OVERRIDE_AGENT_PACKAGE_VERSION=""
+OVERRIDE_AGENT_PACKAGE_VERSION="8.14.0"
 
 if [[ -n "$OVERRIDE_AGENT_PACKAGE_VERSION" ]]; then
   OVERRIDE_TEST_AGENT_VERSION=${OVERRIDE_AGENT_PACKAGE_VERSION}"-SNAPSHOT"
 else
  OVERRIDE_TEST_AGENT_VERSION=""
 fi
 # PACKAGE
-AGENT_PACKAGE_VERSION="${OVERRIDE_AGENT_PACKAGE_VERSION}" DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64,windows/amd64 PACKAGES=tar.gz,zip mage package
+AGENT_PACKAGE_VERSION="${OVERRIDE_AGENT_PACKAGE_VERSION}" DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64,windows/amd64 PACKAGES=tar.gz,zip,rpm,deb mage package
 
 # Run integration tests
 set +e

.buildkite/scripts/steps/package.sh

@@ -5,19 +5,19 @@ set -euo pipefail
 _SELF=$(dirname $0)
 source "${_SELF}/../common.sh"
 
-if test -z "${ManifestURL=:""}"; then
-  echo "Missing variable ManifestURL, export it before use."
+if test -z "${MANIFEST_URL=:""}"; then
+  echo "Missing variable MANIFEST_URL, export it before use."
   exit 2
 fi
 
 export AGENT_DROP_PATH=build/elastic-agent-drop
 mkdir -p $AGENT_DROP_PATH
 
-# Download the components from the ManifestURL and then package those downloaded into the $AGENT_DROP_PATH
-mage clean downloadManifest package ironbank fixDRADockerArtifacts
+# Download the components from the MANIFEST_URL and then package those downloaded into the $AGENT_DROP_PATH
+mage clean downloadManifest packageUsingDRA ironbank fixDRADockerArtifacts
 
 echo  "+++ Generate dependencies report"
-BEAT_VERSION_FULL=$(curl -s -XGET "${ManifestURL}" |jq '.version' -r )
+BEAT_VERSION_FULL=$(curl -s -XGET "${MANIFEST_URL}" |jq '.version' -r )
 bash "${_SELF}/../../../dev-tools/dependencies-report"
 mkdir -p build/distributions/reports
 mv dependencies.csv "build/distributions/reports/dependencies-${BEAT_VERSION_FULL}.csv"

.buildkite/scripts/steps/sync-k8s.sh

@@ -6,7 +6,14 @@ export PATH=$HOME/bin:${PATH}
 source .buildkite/scripts/install-gh.sh
 source .buildkite/scripts/common.sh
 
-export GITHUB_TOKEN=$(retry 5 vault kv get -field token kv/ci-shared/platform-ingest/github_token)
+echo "--- [Prepare env] Create required env variables"
+GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
+GITHUB_USERNAME_SECRET=$(retry 5 vault kv get -field username ${GITHUB_TOKEN_VAULT_PATH})
+export GITHUB_USERNAME_SECRET
+GITHUB_EMAIL_SECRET=$(retry 5 vault kv get -field email ${GITHUB_TOKEN_VAULT_PATH})
+export GITHUB_EMAIL_SECRET
+GITHUB_TOKEN_SECRET=$(retry 5 vault kv get -field token ${GITHUB_TOKEN_VAULT_PATH})
+export GITHUB_TOKEN_SECRET
 
 cd deploy/kubernetes
 

.github/workflows/bump-agent-versions.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+set -e
+
+version_requirements=$(mage integration:updateVersions)
+changes=$(git status -s -uno .agent-versions.json)
+if [ -z "$changes" ]
+then
+    echo "The versions file didn't change, skipping..."
+else
+    echo "The versions file changed"
+    open=$(gh pr list --repo "$GITHUB_REPOSITORY" --label="update-versions" --limit 1 --state open --base "$GITHUB_REF_NAME")
+    if [ -n "$open" ]
+    then
+        echo "Another PR for $GITHUB_REF_NAME is in review, skipping..."
+        exit 0
+    fi
+    git diff -p
+    git add ".agent-versions.json"
+
+    nl=$'\n' # otherwise the new line character is not recognized properly
+    commit_desc="This file is used for picking agent versions in integration tests.${nl}${nl}The file's content is based on responses from https://www.elastic.co/api/product_versions and https://snapshots.elastic.co${nl}${nl}The current update is generated based on the following requirements:${nl}${nl}\`\`\`json${nl}${version_requirements}${nl}\`\`\`"
+
+    git commit -m "[$GITHUB_REF_NAME][Automation] Update .agent-versions.json" -m "$commit_desc"
+    git push --set-upstream origin "update-agent-versions-$GITHUB_RUN_ID"
+    pr=$(gh pr create \
+       --base "$GITHUB_REF_NAME" \
+       --fill-first \
+       --head "update-agent-versions-$GITHUB_RUN_ID" \
+       --label 'Team:Elastic-Agent' \
+       --label 'update-versions' \
+       --label 'skip-changelog' \
+       --label 'backport-skip' \
+       --repo $GITHUB_REPOSITORY)
+    echo "pr=$pr" >> "$GITHUB_OUTPUT" # set the step output for Slack notifications
+    echo "Created a PR with the file update: $pr"
+fi

.github/workflows/bump-agent-versions.yml

@@ -9,8 +9,21 @@ jobs:
   update_versions:
     runs-on: ubuntu-latest
     steps:
+      - name: Setup Git
+        uses: elastic/apm-pipeline-library/.github/actions/setup-git@current
+
+      - uses: elastic/apm-pipeline-library/.github/actions/github-token@current
+        with:
+          url: ${{ secrets.VAULT_ADDR }}
+          roleId: ${{ secrets.VAULT_ROLE_ID }}
+          secretId: ${{ secrets.VAULT_SECRET_ID }}
+
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          # no depth limit
+          # so, we can generate snapshot versions based on release branches
+          fetch-depth: 0
 
       - name: Set up Go
         uses: actions/setup-go@v4
@@ -20,41 +33,33 @@ jobs:
       - name: Set up branch
         run: git checkout -b update-agent-versions-$GITHUB_RUN_ID
 
-      - name: Update the agent version file
+      - name: Install mage
         uses: magefile/mage-action@v3
         with:
           version: v1.13.0
-          args: integration:updateVersions
+          install-only: true
 
-      - name: Check for file changes
+      - name: Update versions file
+        id: update
         env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          changes=$(git status -s -uno .agent-versions.json)
-          if [ -z "$changes" ]
-          then
-            echo "The versions file didn't change, skipping..."
-          else
-            echo "The versions file changed"
-            open=$(gh pr list --repo "$GITHUB_REPOSITORY" --label="update-versions" --limit 1 --state open --base "$GITHUB_REF_NAME")
-            if [ -n "$open" ]
-            then
-              echo "Another PR for $GITHUB_REF_NAME is in review, skipping..."
-              exit 0
-            fi
-            git diff -p
-            git config --global user.name 'apmmachine'
-            git config --global user.email 'apmmachine@users.noreply.github.com'
-            git add ".agent-versions.json"
-            git commit -m "[$GITHUB_REF_NAME](automation) Update .agent-versions.json" -m "This file is used for picking agent versions in integration tests. It's content is based on the reponse from https://artifacts-api.elastic.co/v1/versions/"
-            git push --set-upstream origin "update-agent-versions-$GITHUB_RUN_ID"
-            gh pr create \
-              --base "$GITHUB_REF_NAME" \
-              --fill-first \
-              --head "update-agent-versions-$GITHUB_RUN_ID" \
-              --label 'Team:Elastic-Agent' \
-              --label 'update-versions' \
-              --label 'skip-changelog' \
-              --label 'backport-skip' \
-              --repo $GITHUB_REPOSITORY
-          fi
+          GH_TOKEN: ${{ env.GITHUB_TOKEN }}
+        run: ./.github/workflows/bump-agent-versions.sh
+
+      - if: ${{ failure() }}
+        uses: elastic/apm-pipeline-library/.github/actions/slack-message@current
+        with:
+          url: ${{ secrets.VAULT_ADDR }}
+          roleId: ${{ secrets.VAULT_ROLE_ID }}
+          secretId: ${{ secrets.VAULT_SECRET_ID }}
+          message: ":traffic_cone: Elastic Agent versions file update failed: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          channel: "#ingest-notifications"
+
+        # if a PR was created as a result of this job, we notify on the Slack channel
+      - if: ${{ startsWith(steps.update.outputs.pr, 'https') }}
+        uses: elastic/apm-pipeline-library/.github/actions/slack-message@current
+        with:
+          url: ${{ secrets.VAULT_ADDR }}
+          roleId: ${{ secrets.VAULT_ROLE_ID }}
+          secretId: ${{ secrets.VAULT_SECRET_ID }}
+          message: "Update for Elastic Agent versions file has been created: ${{ steps.update.outputs.pr }}"
+          channel: "#ingest-notifications"

.go-version

@@ -1 +1 @@
-1.21.8
+1.21.9

.golangci.yml

@@ -116,7 +116,7 @@ linters-settings:
 
   gosimple:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.8"
+    go: "1.21.9"
 
   nakedret:
     # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
@@ -136,17 +136,17 @@ linters-settings:
 
   staticcheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.8"
+    go: "1.21.9"
     checks: ["all"]
 
   stylecheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.8"
+    go: "1.21.9"
     checks: ["all"]
 
   unused:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.8"
+    go: "1.21.9"
 
   gosec:
     excludes:

.mergify.yml

@@ -306,3 +306,16 @@ pull_request_rules:
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: backport patches to 8.14 branch
+    conditions:
+      - merged
+      - label=backport-v8.14.0
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "8.14"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"

Dockerfile

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.21.8
+ARG GO_VERSION=1.21.9
 FROM circleci/golang:${GO_VERSION}
 
 

Dockerfile.skaffold

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.21.8
+ARG GO_VERSION=1.21.9
 ARG crossbuild_image="docker.elastic.co/beats-dev/golang-crossbuild"
 ARG AGENT_VERSION=8.9.0-SNAPSHOT
 ARG AGENT_IMAGE="docker.elastic.co/beats/elastic-agent"