Automate fetching the latest package version (#4768)

So, we don't have to manually edit the environment variable when the
snapshot becomes available.

Author: rdner

.buildkite/scripts/steps/integration_tests.sh

@@ -9,9 +9,9 @@ MAGE_SUBTARGET="${3:-""}"
 
 
 # Override the agent package version using a string with format <major>.<minor>.<patch>
-# NOTE: use only after version bump when the new version is not yet available, for example:
-# OVERRIDE_AGENT_PACKAGE_VERSION="8.10.3" otherwise OVERRIDE_AGENT_PACKAGE_VERSION="".
-OVERRIDE_AGENT_PACKAGE_VERSION="8.14.0"
+# There is a time when the snapshot is not built yet, so we cannot use the latest version automatically
+# This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready.
+OVERRIDE_AGENT_PACKAGE_VERSION="$(cat .package-version)"
 
 if [[ -n "$OVERRIDE_AGENT_PACKAGE_VERSION" ]]; then
   OVERRIDE_TEST_AGENT_VERSION=${OVERRIDE_AGENT_PACKAGE_VERSION}"-SNAPSHOT"

.github/workflows/bump-agent-versions.sh

@@ -1,26 +1,27 @@
 #!/bin/bash
 set -e
 
+package_version=$(mage integration:updatePackageVersion)
 version_requirements=$(mage integration:updateVersions)
-changes=$(git status -s -uno .agent-versions.json)
+changes=$(git status -s -uno .agent-versions.json .package-version)
 if [ -z "$changes" ]
 then
-    echo "The versions file didn't change, skipping..."
+    echo "The version files didn't change, skipping..."
 else
-    echo "The versions file changed"
+    echo "The version file(s) changed"
+    git diff -p
     open=$(gh pr list --repo "$GITHUB_REPOSITORY" --label="update-versions" --limit 1 --state open --base "$GITHUB_REF_NAME")
     if [ -n "$open" ]
     then
         echo "Another PR for $GITHUB_REF_NAME is in review, skipping..."
         exit 0
     fi
-    git diff -p
-    git add ".agent-versions.json"
+    git add .agent-versions.json .package-version
 
     nl=$'\n' # otherwise the new line character is not recognized properly
-    commit_desc="This file is used for picking agent versions in integration tests.${nl}${nl}The file's content is based on responses from https://www.elastic.co/api/product_versions and https://snapshots.elastic.co${nl}${nl}The current update is generated based on the following requirements:${nl}${nl}\`\`\`json${nl}${version_requirements}${nl}\`\`\`"
+    commit_desc="These files are used for picking agent versions in integration tests.${nl}${nl}The content is based on responses from https://www.elastic.co/api/product_versions and https://snapshots.elastic.co${nl}${nl}The current update is generated based on the following requirements:${nl}${nl}Package version: ${package_version}${nl}${nl}\`\`\`json${nl}${version_requirements}${nl}\`\`\`"
 
-    git commit -m "[$GITHUB_REF_NAME][Automation] Update .agent-versions.json" -m "$commit_desc"
+    git commit -m "[$GITHUB_REF_NAME][Automation] Update versions" -m "$commit_desc"
     git push --set-upstream origin "update-agent-versions-$GITHUB_RUN_ID"
     pr=$(gh pr create \
        --base "$GITHUB_REF_NAME" \
@@ -33,5 +34,5 @@ else
        --label 'backport-skip' \
        --repo $GITHUB_REPOSITORY)
     echo "pr=$pr" >> "$GITHUB_OUTPUT" # set the step output for Slack notifications
-    echo "Created a PR with the file update: $pr"
+    echo "Created a PR with the an update: $pr"
 fi

.github/workflows/bump-agent-versions.yml

@@ -39,7 +39,7 @@ jobs:
           version: v1.13.0
           install-only: true
 
-      - name: Update versions file
+      - name: Update versions
         id: update
         env:
           GH_TOKEN: ${{ env.GITHUB_TOKEN }}
@@ -51,7 +51,7 @@ jobs:
           url: ${{ secrets.VAULT_ADDR }}
           roleId: ${{ secrets.VAULT_ROLE_ID }}
           secretId: ${{ secrets.VAULT_SECRET_ID }}
-          message: ":traffic_cone: Elastic Agent versions file update failed: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+          message: ":traffic_cone: Elastic Agent version update failed: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           channel: "#ingest-notifications"
 
         # if a PR was created as a result of this job, we notify on the Slack channel
@@ -61,5 +61,5 @@ jobs:
           url: ${{ secrets.VAULT_ADDR }}
           roleId: ${{ secrets.VAULT_ROLE_ID }}
           secretId: ${{ secrets.VAULT_SECRET_ID }}
-          message: "Update for Elastic Agent versions file has been created: ${{ steps.update.outputs.pr }}"
+          message: "Update for Elastic Agent versions has been created: ${{ steps.update.outputs.pr }}"
           channel: "#ingest-notifications"

.package-version

@@ -0,0 +1 @@
+8.14.0

magefile.go

@@ -2045,6 +2045,34 @@ func (Integration) UpdateVersions(ctx context.Context) error {
 	return nil
 }
 
+// UpdatePackageVersion update the file that contains the latest available snapshot version
+func (Integration) UpdatePackageVersion(ctx context.Context) error {
+	const packageVersionFilename = ".package-version"
+
+	sc := snapshots.NewSnapshotsClient()
+	versions, err := sc.FindLatestSnapshots(ctx, []string{"master"})
+	if err != nil {
+		return fmt.Errorf("failed to fetch a manifest for the latest snapshot: %w", err)
+	}
+	if len(versions) != 1 {
+		return fmt.Errorf("expected a single version, got %v", versions)
+	}
+	packageVersion := versions[0].CoreVersion()
+	file, err := os.OpenFile(packageVersionFilename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to open %s for write: %w", packageVersionFilename, err)
+	}
+	defer file.Close()
+	_, err = file.WriteString(packageVersion)
+	if err != nil {
+		return fmt.Errorf("failed to write the package version file %s: %w", packageVersionFilename, err)
+	}
+
+	fmt.Println(packageVersion)
+
+	return nil
+}
+
 var stateDir = ".integration-cache"
 var stateFile = "state.yml"