[Independent Agent Release] New pipeline upload step for agent package for Independent Agent Release (#4222)

* Add alternate upload for independent agent pipeline

* use test env var for testing

* try split version

* fixing up bugs

* fix upload command

* replace + with _

* PR comments to use version parser

* use dash instead of underscore

* Use proper buildkite env var

* Removing unneeded log message

* Add comment for tag replace statement

* remove extraneous log

* Add RM list only, save commit hash

* fix typo

* temp set 8.12

* Add step key to hook

* temp set to main

* try dry run instead

* add proper dry run param

* file listing

* Add in workaround for ironbank

* debug ls commands

* Replace + in tag with . instead of -

* Cleaning up

* temp change for testing

* revert temp test code

* adding test var back

* alias pkg/errors as travis_errors

* Renaming to pkg_errors

* Adding detailed comment about saving build ID in metadata

* Replace with fmt.Errorf

* Define artifact_paths instead of CLI

* fix error linting

* better error handling

* Cleaning up script

* fix proper artifact paths

* Removing test branch assignment

Author: dwhyrock

.buildkite/hooks/pre-command

@@ -41,7 +41,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-package" ]]; then
     docker_login
   fi
 
-  if [[ "$BUILDKITE_STEP_KEY" == "dra-publish" ]]; then
+  if [[ "$BUILDKITE_STEP_KEY" == "dra-publish" || "$BUILDKITE_STEP_KEY" == "bk-api-publish-independent-agent" ]]; then
     release_manager_login
   fi
 fi

.buildkite/pipeline.elastic-agent-package.yml

@@ -101,6 +101,7 @@ steps:
           - "build/distributions/**/*"
 
   - label: ":elastic-stack: Publishing to DRA"
+    if: build.env("BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG") == null || build.env("BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG") != "independent-agent-release-staging"
     key: dra-publish
     depends_on: package
     agents:
@@ -126,3 +127,47 @@ steps:
         export DRA_VERSION
       fi
       .buildkite/scripts/steps/dra-publish.sh
+
+  - label: "Publishing via BK API for Independent Agent Release"
+    if: build.env("BUILDKITE_TRIGGERED_FROM_BUILD_PIPELINE_SLUG") == "independent-agent-release-staging"
+    key: bk-api-publish-independent-agent
+    depends_on: package
+    agents:
+      provider: "gcp"
+    env:
+      DRA_PROJECT_ID: "elastic-agent-package"
+      DRA_PROJECT_ARTIFACT_ID: "agent-package"
+    artifact_paths:
+      - "build/distributions/**/*"
+    command: |
+      echo "+++ Restoring Artifacts"
+      buildkite-agent artifact download "build/**/*" .
+      echo "+++ Changing permissions for the BK API commands"
+      sudo chown -R :1000 build/distributions/
+      echo "--- File listing"
+      ls -alR build
+      echo "--- Copy workaround for ironbank container filename"
+      .buildkite/scripts/steps/ironbank-cp-workaround.sh
+      echo "--- File listing after workaround"
+      ls -alR build
+      echo "+++ Checking artifact validity with release-manager collect dry run"
+      DRA_DRY_RUN="--dry-run"
+      export DRA_DRY_RUN
+      .buildkite/scripts/steps/dra-publish.sh
+      # Artifacts will be uploaded via the artifact_paths entry above
+      echo "+++ Set job metadata if TRIGGER_JOB_ID is properly set"
+      if [[ -z "${TRIGGER_JOB_ID}" ]]; then
+        echo "TRIGGER_JOB_ID is not set, so not setting metadata"
+      else
+        # If a pipeline that triggered this build passes in a "TRIGGER_JOB_ID" env var, that
+        # is an indicator that it will want us to set some metadata in that calling build
+        # so that it can reference this specific build ID in order to easily download
+        # artifacts saved off in this build.
+        #
+        # This is a much easier way to pull back artifacts from a triggered build than using
+        # a Buildkite token that we then have to manage via vault, etc.
+        # See: https://forum.buildkite.community/t/how-to-download-artifacts-back-from-triggered-pipeline/3480/2
+        echo "Setting metadata for job that trigger this one"
+        buildkite-agent meta-data set "triggered_build_id" "$BUILDKITE_BUILD_ID" --job $TRIGGER_JOB_ID
+        buildkite-agent meta-data set "triggered_commit_hash" "$BUILDKITE_COMMIT" --job $TRIGGER_JOB_ID
+      fi

.buildkite/scripts/steps/ironbank-cp-workaround.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# This file is a temporary workaround for the Independent Agent Release
+# workflow.  The ironbank docker image does not currently handle the
+# AGENT_PACKAGE_VERSION env var override.  This renames that file
+# to use that new version.
+#
+# We will not (at first) be using the ironbank docker image in the 
+# Independent Agent releases; however, we want to use the release-manager
+# container dry-run as a check that all the expected images exist.
+#
+# This workaround allows the check to proceed without erroring on the file
+# that we know won't be named correctly.
+#
+
+PACKAGE_VERSION="${AGENT_PACKAGE_VERSION:=""}"
+
+if [[ -z "${PACKAGE_VERSION}" ]]; then 
+    echo "AGENT_PACKAGE_VERSION is not set, exiting"
+    exit 1
+fi
+
+IRONBANK_DOCKER_BLOB_PREFIX="elastic-agent-ironbank"
+IRONBANK_DOCKER_BLOB_SUFFIX="docker-build-context.tar.gz"
+OUTPUT_DIRNAME="build/distributions"
+
+echo "--- ls ${OUTPUT_DIRNAME}"
+ls -al "${OUTPUT_DIRNAME}"
+echo "--- ironbank expected path"
+echo "${OUTPUT_DIRNAME}"/"${IRONBANK_DOCKER_BLOB_PREFIX}"*"${IRONBANK_DOCKER_BLOB_SUFFIX}"
+ls -al "${OUTPUT_DIRNAME}"/"${IRONBANK_DOCKER_BLOB_PREFIX}"*"${IRONBANK_DOCKER_BLOB_SUFFIX}" || true
+
+echo "--- looking for ironbank file to copy to new name"
+if ls "${OUTPUT_DIRNAME}"/"${IRONBANK_DOCKER_BLOB_PREFIX}"*"${IRONBANK_DOCKER_BLOB_SUFFIX}" 2>/dev/null; then 
+    # Found the ironbank file
+    echo "Found the ironbank file"
+    NEW_IRONBANK_NAME="elastic-agent-ironbank-${PACKAGE_VERSION}-docker-build-context.tar.gz"
+    echo "Copying to new path: ${OUTPUT_DIRNAME}/${NEW_IRONBANK_NAME}"
+    cp "${OUTPUT_DIRNAME}"/"${IRONBANK_DOCKER_BLOB_PREFIX}"*"${IRONBANK_DOCKER_BLOB_SUFFIX}" "${OUTPUT_DIRNAME}/${NEW_IRONBANK_NAME}"
+else
+    echo "Error: could not find ironbank file"
+    exit 1
+fi

dev-tools/mage/dockerbuilder.go

@@ -7,6 +7,7 @@ package mage
 import (
 	"bytes"
 	"compress/gzip"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -16,7 +17,6 @@ import (
 	"time"
 
 	"github.com/magefile/mage/sh"
-	"github.com/pkg/errors"
 )
 
 type dockerBuilder struct {
@@ -46,15 +46,15 @@ func newDockerBuilder(spec PackageSpec) (*dockerBuilder, error) {
 
 func (b *dockerBuilder) Build() error {
 	if err := os.RemoveAll(b.buildDir); err != nil {
-		return errors.Wrapf(err, "failed to clean existing build directory %s", b.buildDir)
+		return fmt.Errorf("failed to clean existing build directory %s: %w", b.buildDir, err)
 	}
 
 	if err := b.copyFiles(); err != nil {
 		return err
 	}
 
 	if err := b.prepareBuild(); err != nil {
-		return errors.Wrap(err, "failed to prepare build")
+		return fmt.Errorf("failed to prepare build: %w", err)
 	}
 
 	tag, err := b.dockerBuild()
@@ -67,11 +67,11 @@ func (b *dockerBuilder) Build() error {
 		tries--
 	}
 	if err != nil {
-		return errors.Wrap(err, "failed to build docker")
+		return fmt.Errorf("failed to build docker: %w", err)
 	}
 
 	if err := b.dockerSave(tag); err != nil {
-		return errors.Wrap(err, "failed to save docker as artifact")
+		return fmt.Errorf("failed to save docker as artifact: %w", err)
 	}
 
 	return nil
@@ -88,7 +88,7 @@ func (b *dockerBuilder) modulesDirs() []string {
 }
 
 func (b *dockerBuilder) exposePorts() []string {
-	if ports, _ := b.ExtraVars["expose_ports"]; ports != "" {
+	if ports := b.ExtraVars["expose_ports"]; ports != "" {
 		return strings.Split(ports, ",")
 	}
 	return nil
@@ -101,7 +101,7 @@ func (b *dockerBuilder) copyFiles() error {
 			if f.SkipOnMissing && errors.Is(err, os.ErrNotExist) {
 				continue
 			}
-			return errors.Wrapf(err, "failed to copy from %s to %s", f.Source, target)
+			return fmt.Errorf("failed to copy from %s to %s: %w", f.Source, target, err)
 		}
 	}
 	return nil
@@ -128,7 +128,7 @@ func (b *dockerBuilder) prepareBuild() error {
 
 			err = b.ExpandFile(path, target, data)
 			if err != nil {
-				return errors.Wrapf(err, "expanding template '%s' to '%s'", path, target)
+				return fmt.Errorf("expanding template '%s' to '%s': %w", path, target, err)
 			}
 		}
 		return nil
@@ -169,7 +169,7 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in
 		path := filepath.Join(templatesDir, file.source)
 		err := b.ExpandFile(path, target, data)
 		if err != nil {
-			return errors.Wrapf(err, "expanding template '%s' to '%s'", path, target)
+			return fmt.Errorf("expanding template '%s' to '%s': %w", path, target, err)
 		}
 	}
 
@@ -178,10 +178,14 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in
 
 func (b *dockerBuilder) dockerBuild() (string, error) {
 	tag := fmt.Sprintf("%s:%s", b.imageName, b.Version)
+	// For Independent Agent releases, replace the "+" with a "." since the "+" character
+	// currently isn't allowed in a tag in Docker
+	// E.g., 8.13.0+build202402191057 -> 8.13.0.build202402191057
+	tag = strings.Replace(tag, "+", ".", 1)
 	if b.Snapshot {
 		tag = tag + "-SNAPSHOT"
 	}
-	if repository, _ := b.ExtraVars["repository"]; repository != "" {
+	if repository := b.ExtraVars["repository"]; repository != "" {
 		tag = fmt.Sprintf("%s/%s", repository, tag)
 	}
 	return tag, sh.Run("docker", "build", "-t", tag, b.buildDir)
@@ -238,9 +242,14 @@ func (b *dockerBuilder) dockerSave(tag string) error {
 
 	if err = cmd.Wait(); err != nil {
 		if errmsg := strings.TrimSpace(stderr.String()); errmsg != "" {
-			err = errors.Wrap(errors.New(errmsg), err.Error())
+			err = fmt.Errorf("%w: %s", errors.New(errmsg), err.Error())
 		}
 		return err
 	}
-	return errors.Wrap(CreateSHA512File(outputFile), "failed to create .sha512 file")
+
+	err = CreateSHA512File(outputFile)
+	if err != nil {
+		return fmt.Errorf("failed to create .sha512 file: %w", err)
+	}
+	return nil
 }

dev-tools/mage/manifest/manifest.go

@@ -19,6 +19,7 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/elastic/elastic-agent/pkg/testing/tools"
+	"github.com/elastic/elastic-agent/pkg/version"
 )
 
 // A backoff schedule for when and how often to retry failed HTTP
@@ -61,7 +62,7 @@ func DownloadManifest(manifest string) (tools.Build, error) {
 	}
 	if mg.Verbose() {
 		log.Printf(">>>> Downloaded manifest %s", manifest)
-		log.Printf(">>>> Packaing version: %s, build_id: %s, manifest_version:%s", manifestResponse.Version, manifestResponse.BuildID, manifestResponse.ManifestVersion)
+		log.Printf(">>>> Packaging version: %s, build_id: %s, manifest_version:%s", manifestResponse.Version, manifestResponse.BuildID, manifestResponse.ManifestVersion)
 	}
 	return manifestResponse, nil
 }
@@ -99,6 +100,14 @@ func DownloadComponentsFromManifest(manifest string, platforms []string, platfor
 	}
 	projects := manifestResponse.Projects
 
+	parsedManifestVersion, err := version.ParseVersion(manifestResponse.Version)
+	if err != nil {
+		return fmt.Errorf("failed to parse manifest version: [%s]", manifestResponse.Version)
+	}
+
+	// For resolving manifest package name and version, just use the Major.Minor.Patch part of the version
+	majorMinorPatchVersion := fmt.Sprintf("%d.%d.%d", parsedManifestVersion.Major(), parsedManifestVersion.Minor(), parsedManifestVersion.Patch())
+
 	errGrp, downloadsCtx := errgroup.WithContext(context.Background())
 	for component, pkgs := range componentSpec {
 		for _, platform := range platforms {
@@ -111,7 +120,7 @@ func DownloadComponentsFromManifest(manifest string, platforms []string, platfor
 
 			for _, pkg := range pkgs {
 				reqPackage := platformPackages[platform]
-				pkgURL := resolveManifestPackage(projects[component], pkg, reqPackage, manifestResponse.Version)
+				pkgURL := resolveManifestPackage(projects[component], pkg, reqPackage, majorMinorPatchVersion)
 				if pkgURL != nil {
 					for _, p := range pkgURL {
 						log.Printf(">>>>>>>>> Downloading [%s] [%s] ", pkg, p)