Buildkite migration (#944)

Author: uri-weisman

.buildkite/README.md

@@ -0,0 +1,25 @@
+# Buildkite
+
+This README provides an overview of the Buildkite pipeline used to automate the build and publish process for Cloudbeat artifacts.
+
+## Artifacts
+
+The pipeline generates the following artifacts:
+
+- **dependencies-CLOUDBEAT_VERSION-WORKFLOW.csv**: This CSV file contains a list of dependencies for the specific Cloudbeat version being built. It helps track build dependencies.
+
+- **cloudbeat-CLOUDBEAT_VERSION-WORKFLOW-linux-ARCH.tar.gz**: This tarball includes the Cloudbeat binary and its corresponding csp-policies archive. The supported architectures for the artifacts are amd64 and arm64.
+
+## Triggering the Pipeline
+
+The pipeline is triggered in the following scenarios:
+
+- **Snapshot Builds**: A snapshot build is triggered when a pull request (PR) is merged into the 'main' branch or a version-specific branch. Additionally, if the environment variable RUN_RELEASE is set to "true", a snapshot build is also triggered.
+
+- **Staging Builds**: A staging build is triggered when a PR is merged into a version-specific branch or when the environment variable RUN_RELEASE is set to "true". Staging builds are typically used for a release build candidate.
+
+After a successful build, the pipeline publishes the generated artifacts to the Google Cloud Storage (GCS) bucket named [elastic-artifacts-snapshot/cloudbeat](https://console.cloud.google.com/storage/browser/elastic-artifacts-snapshot/cloudbeat). You can access the published artifacts in this bucket.
+
+## Pipeline Configuration
+
+To view the pipeline and its configuration, click [here](https://buildkite.com/elastic/cloudbeat).

.buildkite/pipeline.yml

@@ -0,0 +1,35 @@
+env:
+  BRANCH: "${BUILDKITE_BRANCH}"
+agents:
+  provider: "gcp" # needed for running docker commands
+
+steps:
+  - label: ":package: Package Cloudbeat - Snapshot"
+    if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_RELEASE") == "true"
+    env:
+      WORKFLOW: "snapshot"
+    key: "package-snapshot"
+    command: "./.buildkite/scripts/package.sh"
+    artifact_paths: "build/distributions/*"
+
+  - label: ":rocket: Publishing Snapshot DRA artifacts"
+    if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_RELEASE") == "true"
+    depends_on: "package-snapshot"
+    command: "./.buildkite/scripts/publish.sh"
+    env:
+      WORKFLOW: "snapshot"
+
+  - label: ":package: Package Cloudbeat - Staging"
+    if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_RELEASE") == "true"
+    env:
+      WORKFLOW: "staging"
+    key: "package-staging"
+    command: "./.buildkite/scripts/package.sh"
+    artifact_paths: "build/distributions/*"
+
+  - label: ":rocket: Publishing Staging DRA artifacts"
+    if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_RELEASE") == "true"
+    depends_on: "package-staging"
+    command: "./.buildkite/scripts/publish.sh"
+    env:
+      WORKFLOW: "staging"

.ci/scripts/package.sh .buildkite/scripts/package.sh

@@ -1,13 +1,10 @@
 #!/usr/bin/env bash
 set -euox pipefail
 
-# linux/amd64 is in the default list already, set here
-# to prevent jenkins_release.sh from adding more PLATFORMS
 export PLATFORMS="linux/amd64,linux/arm64"
 export TYPES="tar.gz"
 
-make activate-hermit
-
+source ./bin/activate-hermit
 if [ $WORKFLOW = "staging" ] ; then
     make release-manager-release
 else

.buildkite/scripts/publish.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Allow other users write access to create checksum files
+
+# The "branch" here selects which "$BRANCH.gradle" file of release manager is used
+export VERSION=$(grep defaultBeatVersion version/version.go | cut -f2 -d "\"")
+MAJOR=$(echo $VERSION | awk -F. '{ print $1 }')
+MINOR=$(echo $VERSION | awk -F. '{ print $2 }')
+if [ -n "$(git ls-remote --heads origin $MAJOR.$MINOR)" ] ; then
+    BRANCH=$MAJOR.$MINOR
+elif [ -n "$(git ls-remote --heads origin $MAJOR.x)" ] ; then
+    BRANCH=$MAJOR.x
+else
+    BRANCH=main
+fi
+
+# Download artifacts from other stages
+echo "Downloading artifacts..."
+buildkite-agent artifact download "build/distributions/*" "." --step package-"${WORKFLOW}"
+chmod -R 777 build/distributions
+
+# Shared secret path containing the dra creds for project teams
+DRA_CREDS=$(vault kv get -field=data -format=json kv/ci-shared/release/dra-role)
+
+# Run release-manager
+echo "Running release-manager container..."
+IMAGE="docker.elastic.co/infra/release-manager:latest"
+docker run --rm \
+  --name release-manager \
+  -e VAULT_ADDR=$(echo $DRA_CREDS | jq -r '.vault_addr') \
+  -e VAULT_ROLE_ID=$(echo $DRA_CREDS | jq -r '.role_id') \
+  -e VAULT_SECRET_ID=$(echo $DRA_CREDS | jq -r '.secret_id') \
+  --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \
+  "$IMAGE" \
+    cli collect \
+      --project cloudbeat \
+      --branch "${BRANCH}" \
+      --commit "${BUILDKITE_COMMIT}" \
+      --workflow "${WORKFLOW}" \
+      --version "${VERSION}" \
+      --artifact-set main \

.ci/jobs/cloudbeat-mbp.yml

@@ -29,10 +29,6 @@ indent_style = tab
 [*.mk]
 indent_style = tab
 
-[Jenkinsfile]
-indent_size = 2
-indent_style = space
-
 [Vagrantfile]
 indent_size = 2
 indent_style = space

.ci/jobs/cloudbeat.yml

@@ -4,7 +4,7 @@
 # Infra & ops related
 /deploy/ @elastic/csp-ops
 .ci/  @elastic/csp-ops
-Jenkinsfile  @elastic/csp-ops
+.buildkite/  @elastic/csp-ops
 
 # Automation tests
 /tests/ @elastic/csp-automation

.ci/jobs/defaults.yml

@@ -4,16 +4,11 @@
 CI_ELASTIC_AGENT_DOCKER_TAG?=8.7.0-SNAPSHOT
 CI_ELASTIC_AGENT_DOCKER_IMAGE?=704479110758.dkr.ecr.eu-west-2.amazonaws.com/elastic-agent
 
-# Ensure the Go version in .go_version is installed and used.
-GOROOT?=$(shell ./scripts/make/run_with_go_ver go env GOROOT)
-GO:=$(GOROOT)/bin/go
-export PATH:=$(GOROOT)/bin:$(PATH)
-
 # By default we run tests with verbose output. This may be overridden, e.g.
 # scripts may set GOTESTFLAGS=-json to format test output for processing.
 GOTESTFLAGS?=-v
 
-GOOSBUILD:=./build/$(shell $(GO) env GOOS)
+GOOSBUILD:=./build/$(shell go env GOOS)
 APPROVALS=$(GOOSBUILD)/approvals
 GENPACKAGE=$(GOOSBUILD)/genpackage
 GOIMPORTS=$(GOOSBUILD)/goimports
@@ -24,7 +19,7 @@ STATICCHECK=$(GOOSBUILD)/staticcheck
 ELASTICPACKAGE=$(GOOSBUILD)/elastic-package
 
 PYTHON_ENV?=.
-PYTHON_BIN:=$(PYTHON_ENV)/build/ve/$(shell $(GO) env GOOS)/bin
+PYTHON_BIN:=$(PYTHON_ENV)/build/ve/$(shell go env GOOS)/bin
 PYTHON=$(PYTHON_BIN)/python
 
 CLOUDBEAT_VERSION=$(shell grep defaultBeatVersion version/version.go | cut -d'=' -f2 | tr -d '" ')
@@ -67,15 +62,15 @@ cloudbeat:
 
 .PHONY: test
 test:
-	$(GO) test $(GOTESTFLAGS) ./...
+	go test $(GOTESTFLAGS) ./...
 
 .PHONY:
 clean:
 	mage clean
 
 .PHONY: PackageAgent
 PackageAgent:
-	SNAPSHOT=TRUE PLATFORMS=linux/$(shell $(GO) env GOARCH) TYPES=tar.gz mage -v $@
+	SNAPSHOT=TRUE PLATFORMS=linux/$(shell go env GOARCH) TYPES=tar.gz mage -v $@
 
 # elastic_agent_docker_image builds the Cloud Elastic Agent image
 # with the local APM Server binary injected. The image will be based
@@ -108,7 +103,7 @@ check: check-fmt check-headers
 
 .PHONY: bench
 bench:
-	@$(GO) test -benchmem -run=XXX -benchtime=100ms -bench='.*' ./...
+	@go test -benchmem -run=XXX -benchtime=100ms -bench='.*' ./...
 
 ##############################################################################
 # Rules for updating config files, etc.
@@ -123,7 +118,7 @@ config:
 
 .PHONY: go-generate
 go-generate:
-	@$(GO) generate .
+	@go generate .
 
 notice: NOTICE.txt
 NOTICE.txt: $(PYTHON) go.mod utils/go.mod
@@ -162,7 +157,7 @@ update-beats-docs: $(PYTHON)
 # Linting, style-checking, license header checks, etc.
 ##############################################################################
 
-GOLINT_TARGETS?=$(shell $(GO) list ./...)
+GOLINT_TARGETS?=$(shell go list ./...)
 GOLINT_UPSTREAM?=origin/main
 REVIEWDOG_FLAGS?=-conf=reviewdog.yml -f=golint -diff="git diff $(GOLINT_UPSTREAM)"
 GOLINT_COMMAND=$(GOLINT) ${GOLINT_TARGETS} | grep -v "should have comment" | $(REVIEWDOG) $(REVIEWDOG_FLAGS)
@@ -209,22 +204,22 @@ autopep8: $(PYTHON_BIN)
 ##############################################################################
 
 $(GOLINT): go.mod
-	$(GO) build -o $@ golang.org/x/lint/golint
+	go build -o $@ golang.org/x/lint/golint
 
 $(GOIMPORTS): go.mod
-	$(GO) build -o $@ golang.org/x/utils/cmd/goimports
+	go build -o $@ golang.org/x/utils/cmd/goimports
 
 $(STATICCHECK): utils/go.mod
-	$(GO) build -o $@ -modfile=$< honnef.co/go/utils/cmd/staticcheck
+	go build -o $@ -modfile=$< honnef.co/go/utils/cmd/staticcheck
 
 $(GOLICENSER): go.mod
-	$(GO) build -o $@ -modfile=$< github.com/elastic/go-licenser
+	go build -o $@ -modfile=$< github.com/elastic/go-licenser
 
 $(REVIEWDOG): utils/go.mod
-	$(GO) build -o $@ -modfile=$< github.com/reviewdog/reviewdog/cmd/reviewdog
+	go build -o $@ -modfile=$< github.com/reviewdog/reviewdog/cmd/reviewdog
 
 $(ELASTICPACKAGE): utils/go.mod
-	$(GO) build -o $@ -modfile=$< -ldflags '-X github.com/elastic/elastic-package/internal/version.CommitHash=anything' github.com/elastic/elastic-package
+	go build -o $@ -modfile=$< -ldflags '-X github.com/elastic/elastic-package/internal/version.CommitHash=anything' github.com/elastic/elastic-package
 
 $(PYTHON): $(PYTHON_BIN)
 $(PYTHON_BIN): $(PYTHON_BIN)/activate
@@ -234,7 +229,7 @@ $(PYTHON_BIN)/activate:
 
 .PHONY: $(APPROVALS)
 $(APPROVALS):
-	@$(GO) build -o $@ github.com/elastic/cloudbeat/approvaltest/cmd/check-approvals
+	@go build -o $@ github.com/elastic/cloudbeat/approvaltest/cmd/check-approvals
 
 ##############################################################################
 # Release manager.

.ci/scripts/intake.sh

@@ -1,4 +1,4 @@
 env = {
-  "CLOUDBEAT_VERSION": "8.8.0",
+  "CLOUDBEAT_VERSION": "8.9.0",
   "ELK_VERSION": "${CLOUDBEAT_VERSION}-SNAPSHOT",
 }

.ci/scripts/integration-tests.sh

@@ -132,10 +132,10 @@ elastic-stack-down:
   elastic-package stack down
 
 elastic-stack-connect-kind kind='kind-multi':
-  ./.ci/scripts/connect_kind.sh {{kind}}
+  ./scripts/connect_kind.sh {{kind}}
 
 elastic-stack-disconnect-kind kind='kind-multi':
-  ./.ci/scripts/connect_kind.sh {{kind}} disconnect
+  ./scripts/connect_kind.sh {{kind}} disconnect
 
 ssh-cloudbeat:
   CLOUDBEAT_POD=$( kubectl get pods -o=name -n kube-system | grep -m 1 "cloudbeat" ) && \
@@ -155,7 +155,7 @@ generate-mocks:
 
 # run to validate no mocks are missing
 validate-mocks:
-  ./.ci/scripts/validate-mocks.sh
+  ./scripts/validate-mocks.sh
 
 
 #### TESTS ####

.ci/scripts/rm-docker.sh

@@ -80,32 +80,6 @@ setup_go_path() {
   debug "GOPATH=${GOPATH}"
 }
 
-jenkins_setup() {
-  : "${HOME:?Need to set HOME to a non-empty value.}"
-  : "${WORKSPACE:?Need to set WORKSPACE to a non-empty value.}"
-
-  if [ -z ${GO_VERSION:-} ]; then
-    get_go_version
-  fi
-
-  # Setup Go.
-  export GOPATH=${WORKSPACE}
-  export PATH=${GOPATH}/bin:${PATH}
-  eval "$(gvm ${GO_VERSION})"
-
-  # Workaround for Python virtualenv path being too long.
-  export TEMP_PYTHON_ENV=$(mktemp -d)
-  export PYTHON_ENV="${TEMP_PYTHON_ENV}/python-env"
-
-  # Write cached magefile binaries to workspace to ensure
-  # each run starts from a clean slate.
-  export MAGEFILE_CACHE="${WORKSPACE}/.magefile"
-
-  # Enable verbose output for Mage,
-  # to help diagnose build failures.
-  export MAGEFILE_VERBOSE=1
-}
-
 docker_setup() {
   OS="$(uname)"
   case $OS in