[buildkite]: get version qualifier from google bucket (#2950)

* [buildkite]: get version qualifier from google bucket

* omit version qualifier on snapshot builds

* omit version qualifier on snapshot builds

* omit version qualifier on snapshot builds

* fix

Author: moukoublen

.buildkite/pipeline.yml

@@ -5,36 +5,31 @@ agents:
   image: "family/platform-ingest-beats-ubuntu-2204"
 
 steps:
-  # TODO remove && build.env('VERSION_QUALIFIER') == null
   - label: ":package: Package Cloudbeat - Snapshot"
-    if: (build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env("RUN_RELEASE") == "true") && build.env('VERSION_QUALIFIER') == null
+    if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env("RUN_RELEASE") == "true"
     env:
       WORKFLOW: "snapshot"
     key: "package-snapshot"
     command: "./.buildkite/scripts/package.sh"
     artifact_paths: "build/distributions/*"
 
-  # TODO remove && build.env('VERSION_QUALIFIER') == null
   - label: ":rocket: Publishing Snapshot DRA artifacts"
-    if: (build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env("RUN_RELEASE") == "true") && build.env('VERSION_QUALIFIER') == null
+    if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env("RUN_RELEASE") == "true"
     depends_on: "package-snapshot"
     command: "./.buildkite/scripts/publish.sh"
     env:
       WORKFLOW: "snapshot"
 
-    # Allow building staging from main when VERSION_QUALIFIER is set (allow prerelease testing)
-    # TODO remove || build.env('VERSION_QUALIFIER') != null
   - label: ":package: Package Cloudbeat - Staging"
-    if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_RELEASE") == "true" || build.env('VERSION_QUALIFIER') != null
+    if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_RELEASE") == "true"
     env:
       WORKFLOW: "staging"
     key: "package-staging"
     command: "./.buildkite/scripts/package.sh"
     artifact_paths: "build/distributions/*"
 
-  # TODO remove || build.env('VERSION_QUALIFIER') != null
   - label: ":rocket: Publishing Staging DRA artifacts"
-    if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_RELEASE") == "true" || build.env('VERSION_QUALIFIER') != null
+    if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_RELEASE") == "true"
     depends_on: "package-staging"
     command: "./.buildkite/scripts/publish.sh"
     env:

.buildkite/scripts/package.sh

@@ -8,7 +8,10 @@ source ./bin/activate-hermit
 CLOUDBEAT_VERSION=$(grep defaultBeatVersion version/version.go | cut -d'=' -f2 | tr -d '" ')
 PYTHON_BIN=./build/ve/$(go env GOOS)/bin
 PYTHON=$PYTHON_BIN/python
-VERSION_QUALIFIER="${VERSION_QUALIFIER:-""}"
+
+source .buildkite/scripts/qualifier.sh
+echo "VERSION_QUALIFIER: ${VERSION_QUALIFIER}"
+export VERSION_QUALIFIER
 
 if [ "$WORKFLOW" = "snapshot" ]; then
     export SNAPSHOT="true"
@@ -22,7 +25,7 @@ mage package
 
 CSV_FILE="build/dependencies-${CLOUDBEAT_VERSION}"
 [ -n "${SNAPSHOT+x}" ] && CSV_FILE+="-SNAPSHOT"
-if [[ -n "$VERSION_QUALIFIER" ]]; then
+if [[ -n "${VERSION_QUALIFIER}" ]]; then
     CSV_FILE+="-${VERSION_QUALIFIER}"
 fi
 

.buildkite/scripts/publish.sh

@@ -13,7 +13,9 @@ elif [ -n "$(git ls-remote --heads origin $MAJOR.x)" ]; then
 else
     BRANCH=main
 fi
-VERSION_QUALIFIER="${VERSION_QUALIFIER:-""}"
+
+source .buildkite/scripts/qualifier.sh
+echo "VERSION_QUALIFIER: ${VERSION_QUALIFIER}"
 
 # Download artifacts from other stages
 echo "Downloading artifacts..."

.buildkite/scripts/qualifier.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# this file should be sourced from inside the package and publish script.
+
+fetch_elastic_qualifier() {
+    local branch="${1}"
+    local url="https://storage.googleapis.com/dra-qualifier/${branch}"
+    local qualifier=""
+    if curl -sf -o /dev/null "${url}"; then
+        qualifier=$(curl -s "${url}")
+    fi
+    echo "${qualifier}"
+}
+
+# If this is a snapshot build, omit VERSION_QUALIFIER
+if [ "${WORKFLOW}" = "snapshot" ]; then
+    VERSION_QUALIFIER=''
+
+# If the VERSION_QUALIFIER is already set (e.g. buildkite custom run), don't modify it.
+# Else try to fetch from google bucket for the current branch
+elif [ -z "${VERSION_QUALIFIER+x}" ]; then
+    # VERSION_QUALIFIER is not set, get from bucket
+    VERSION_QUALIFIER="$(fetch_elastic_qualifier "${BUILDKITE_BRANCH}")"
+fi