Add a mage target that packages elastic-agent using elastic-agent-core DRA (#4403)

* Extract dependencies download to its own function

* Add PackageUsingDRA mage target

* filter elastic-agent-core artifacts by platform

* update buildkite package script with packageUsingDRA target

* Add sha512 validation for downloaded elastic-agent-core packages

* Override the elastic agent commit hash when packaging using DRA

* change Manifest URL env variable name

Author: pchila

.buildkite/pipeline.elastic-agent-package.yml

@@ -10,10 +10,10 @@ env:
 
 steps:
   - input: "Build parameters"
-    if: build.env("ManifestURL") == null
+    if: build.env("MANIFEST_URL") == null
     fields:
-      - text: "ManifestURL"
-        key: "ManifestURL"
+      - text: "MANIFEST_URL"
+        key: "MANIFEST_URL"
         default: ""
         required: true
         hint: "Link to the build manifest URL."
@@ -51,7 +51,7 @@ steps:
         hint: "If the DRA release manager script would actually publish anything or just print"
 
   - wait: ~
-    if: build.env("ManifestURL") == null
+    if: build.env("MANIFEST_URL") == null
 
   - group: ":Packaging Artefacts"
     key: "package"
@@ -63,10 +63,10 @@ steps:
           machineType: "c2-standard-16"
           diskSizeGb: 400
         command: |
-          if [[ -z "${ManifestURL}" ]]; then
-            export ManifestURL=$(buildkite-agent meta-data get ManifestURL --default "")
-            if [[ -z "${ManifestURL}" ]]; then
-              echo ":broken_heart: Missing ManifestURL variable or empty string provided"
+          if [[ -z "${MANIFEST_URL}" ]]; then
+            export MANIFEST_URL=$(buildkite-agent meta-data get MANIFEST_URL --default "")
+            if [[ -z "${MANIFEST_URL}" ]]; then
+              echo ":broken_heart: Missing MANIFEST_URL variable or empty string provided"
               exit 1
             fi
           fi
@@ -86,10 +86,10 @@ steps:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
         command: |
-          if [[ -z "${ManifestURL}" ]]; then
-            export ManifestURL=$(buildkite-agent meta-data get ManifestURL --default "")
-            if [[ -z "${ManifestURL}" ]]; then
-              echo ":broken_heart: Missing ManifestURL variable or empty string provided"
+          if [[ -z "${MANIFEST_URL}" ]]; then
+            export MANIFEST_URL=$(buildkite-agent meta-data get MANIFEST_URL --default "")
+            if [[ -z "${MANIFEST_URL}" ]]; then
+              echo ":broken_heart: Missing MANIFEST_URL variable or empty string provided"
               exit 1
             fi
           fi

.buildkite/scripts/steps/package.sh

@@ -5,19 +5,19 @@ set -euo pipefail
 _SELF=$(dirname $0)
 source "${_SELF}/../common.sh"
 
-if test -z "${ManifestURL=:""}"; then
-  echo "Missing variable ManifestURL, export it before use."
+if test -z "${MANIFEST_URL=:""}"; then
+  echo "Missing variable MANIFEST_URL, export it before use."
   exit 2
 fi
 
 export AGENT_DROP_PATH=build/elastic-agent-drop
 mkdir -p $AGENT_DROP_PATH
 
-# Download the components from the ManifestURL and then package those downloaded into the $AGENT_DROP_PATH
-mage clean downloadManifest package ironbank fixDRADockerArtifacts
+# Download the components from the MANIFEST_URL and then package those downloaded into the $AGENT_DROP_PATH
+mage clean downloadManifest packageUsingDRA ironbank fixDRADockerArtifacts
 
 echo  "+++ Generate dependencies report"
-BEAT_VERSION_FULL=$(curl -s -XGET "${ManifestURL}" |jq '.version' -r )
+BEAT_VERSION_FULL=$(curl -s -XGET "${MANIFEST_URL}" |jq '.version' -r )
 bash "${_SELF}/../../../dev-tools/dependencies-report"
 mkdir -p build/distributions/reports
 mv dependencies.csv "build/distributions/reports/dependencies-${BEAT_VERSION_FULL}.csv"

dev-tools/mage/manifest/manifest.go

@@ -130,7 +130,7 @@ func DownloadComponentsFromManifest(manifest string, platforms []string, platfor
 						downloadTarget := filepath.Join(targetPath, pkgFilename)
 						if _, err := os.Stat(downloadTarget); err != nil {
 							errGrp.Go(func(ctx context.Context, url, target string) func() error {
-								return func() error { return downloadPackage(ctx, url, target) }
+								return func() error { return DownloadPackage(ctx, url, target) }
 							}(downloadsCtx, p, downloadTarget))
 						}
 					}
@@ -150,7 +150,7 @@ func DownloadComponentsFromManifest(manifest string, platforms []string, platfor
 	return nil
 }
 
-func downloadPackage(ctx context.Context, downloadUrl string, target string) error {
+func DownloadPackage(ctx context.Context, downloadUrl string, target string) error {
 	parsedURL, errorUrl := url.Parse(downloadUrl)
 	if errorUrl != nil {
 		return errorInvalidManifestURL

dev-tools/mage/settings.go

@@ -44,6 +44,10 @@ const (
 	// Env vars
 	// agent package version
 	agentPackageVersionEnvVar = "AGENT_PACKAGE_VERSION"
+	//ManifestUrlEnvVar is the name fo the environment variable containing the Manifest URL to be used for packaging agent
+	ManifestUrlEnvVar = "MANIFEST_URL"
+	// AgentCommitHashEnvVar allows to override agent commit hash string during packaging
+	AgentCommitHashEnvVar
 
 	// Mapped functions
 	agentPackageVersionMappedFunc    = "agent_package_version"
@@ -152,7 +156,7 @@ func initGlobals() {
 
 	agentPackageVersion = EnvOr(agentPackageVersionEnvVar, "")
 
-	ManifestURL = EnvOr("ManifestURL", "")
+	ManifestURL = EnvOr(ManifestUrlEnvVar, "")
 	PackagingFromManifest = ManifestURL != ""
 }
 
@@ -281,7 +285,12 @@ var (
 func CommitHash() (string, error) {
 	var err error
 	commitHashOnce.Do(func() {
-		commitHash, err = sh.Output("git", "rev-parse", "HEAD")
+		// Check commit hash override first
+		commitHash = EnvOr(AgentCommitHashEnvVar, "")
+		if commitHash == "" {
+			// no override found, get the hash from HEAD
+			commitHash, err = sh.Output("git", "rev-parse", "HEAD")
+		}
 	})
 	return commitHash, err
 }

internal/pkg/agent/application/upgrade/artifact/download/verifier.go

@@ -11,6 +11,7 @@ import (
 	"crypto/sha512"
 	"encoding/hex"
 	"fmt"
+	"hash"
 	"io"
 	"net/http"
 	"net/url"
@@ -130,8 +131,16 @@ func getHashFileName(filename string) string {
 // exists and that the checksum in the sidecar file matches the checksum of
 // the file. It returns an error if validation fails.
 func VerifySHA512Hash(filename string) error {
+	hasher := sha512.New()
+	checksumFileName := getHashFileName(filename)
+	return VerifyChecksum(hasher, filename, checksumFileName)
+}
+
+// VerifyChecksum checks that the hash contained in checksumFileName correspond to the hash calculated for filename using
+// hasher.Sum()
+func VerifyChecksum(hasher hash.Hash, filename, checksumFileName string) error {
 	// Read expected checksum.
-	expectedHash, err := readChecksumFile(getHashFileName(filename), filepath.Base(filename))
+	expectedHash, err := readChecksumFile(checksumFileName, filepath.Base(filename))
 	if err != nil {
 		return fmt.Errorf("could not read checksum file: %w", err)
 	}
@@ -143,12 +152,11 @@ func VerifySHA512Hash(filename string) error {
 	}
 	defer f.Close()
 
-	hash := sha512.New()
-	if _, err := io.Copy(hash, f); err != nil {
+	if _, err := io.Copy(hasher, f); err != nil {
 		return fmt.Errorf("faled to read file to calculate hash")
 	}
 
-	computedHash := hex.EncodeToString(hash.Sum(nil))
+	computedHash := hex.EncodeToString(hasher.Sum(nil))
 	if computedHash != expectedHash {
 		return &ChecksumMismatchError{
 			Expected: expectedHash,