mge package

Author: narph

NOTICE.txt

@@ -319,6 +319,7 @@ func (b GolangCrossBuilder) Build() error {
 		"--env", "MAGEFILE_TIMEOUT="+EnvOr("MAGEFILE_TIMEOUT", ""),
 		"--env", fmt.Sprintf("SNAPSHOT=%v", Snapshot),
 		"--env", fmt.Sprintf("DEV=%v", DevBuild),
+		"--env", fmt.Sprintf("EXTERNAL=%v", ExternalBuild),
 		"-v", repoInfo.RootDir+":"+mountPoint,
 		"-w", workDir,
 		image,

dev-tools/mage/crossbuild.go

@@ -80,6 +80,7 @@ var (
 
 	Snapshot bool
 	DevBuild bool
+	ExternalBuild bool
 
 	versionQualified bool
 	versionQualifier string
@@ -125,6 +126,11 @@ func init() {
 		panic(errors.Wrap(err, "failed to parse DEV env value"))
 	}
 
+	ExternalBuild, err = strconv.ParseBool(EnvOr("EXTERNAL", "false"))
+	if err != nil {
+		panic(errors.Wrap(err, "failed to parse EXTERNAL env value"))
+	}
+
 	versionQualifier, versionQualified = os.LookupEnv("VERSION_QUALIFIER")
 }
 
@@ -176,6 +182,7 @@ func varMap(args ...map[string]interface{}) map[string]interface{} {
 		"BeatUser":        BeatUser,
 		"Snapshot":        Snapshot,
 		"DEV":             DevBuild,
+		"EXTERNAL":        ExternalBuild,
 		"Qualifier":       versionQualifier,
 	}
 

dev-tools/mage/settings.go

@@ -0,0 +1,14 @@
+package mage
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+
+func TestGetVersion(t *testing.T) {
+	bp, err := BeatQualifiedVersion()
+	assert.NoError(t, err)
+	_= bp
+}

dev-tools/mage/settings_test.go

@@ -5,7 +5,7 @@ import re
 import sys
 from subprocess import check_call
 
-vendored_elastic_agent = os.path.normpath("vendor/github.com/elastic/elastic-agent_poc")
+vendored_elastic_agent = os.path.normpath("vendor/github.com/elastic/elastic-agent-poc")
 
 
 goversion_var_elastic_agent = "defaultAgentVersion"		# version in agent

dev-tools/set_version

@@ -0,0 +1,8 @@
+:stack-version: 8.0.0
+:doc-branch: master
+:go-version: 1.17.5
+:release-state: unreleased
+:python: 3.7
+:docker: 1.12
+:docker-compose: 1.11
+:libpcap: 0.8

elastic-agent/Jenkinsfile.yml

@@ -26,6 +26,7 @@ import (
 	devtools "github.com/elastic/elastic-agent-poc/dev-tools/mage"
 	"github.com/elastic/elastic-agent-poc/elastic-agent/pkg/release"
 
+	downloads "github.com/elastic/e2e-testing/pkg/downloads"
 	// mage:import
 	"github.com/elastic/elastic-agent-poc/dev-tools/mage/target/common"
 	// mage:import
@@ -37,14 +38,15 @@ import (
 )
 
 const (
-	goLintRepo     = "golang.org/x/lint/golint"
-	goLicenserRepo = "github.com/elastic/go-licenser"
-	buildDir       = "build"
-	metaDir        = "_meta"
-	snapshotEnv    = "SNAPSHOT"
-	devEnv         = "DEV"
-	configFile     = "elastic-agent.yml"
-	agentDropPath  = "AGENT_DROP_PATH"
+	goLintRepo        = "golang.org/x/lint/golint"
+	goLicenserRepo    = "github.com/elastic/go-licenser"
+	buildDir          = "build"
+	metaDir           = "_meta"
+	snapshotEnv       = "SNAPSHOT"
+	devEnv            = "DEV"
+	externalArtifacts = "EXTERNAL"
+	configFile        = "elastic-agent.yml"
+	agentDropPath     = "AGENT_DROP_PATH"
 )
 
 // Aliases for commands required by master makefile
@@ -325,13 +327,15 @@ func Package() {
 
 	packageAgent(requiredPackages, devtools.UseElasticAgentPackaging)
 }
-
+func getPackageName(beat, version, pkg string) (string, string) {
+	if _, ok := os.LookupEnv(snapshotEnv); ok {
+		version += "-SNAPSHOT"
+	}
+	return version, fmt.Sprintf("%s-%s-%s", beat, version, pkg)
+}
 func requiredPackagesPresent(basePath, beat, version string, requiredPackages []string) bool {
 	for _, pkg := range requiredPackages {
-		if _, ok := os.LookupEnv(snapshotEnv); ok {
-			version += "-SNAPSHOT"
-		}
-		packageName := fmt.Sprintf("%s-%s-%s", beat, version, pkg)
+		_, packageName := getPackageName(beat, version, pkg)
 		path := filepath.Join(basePath, "build", "distributions", packageName)
 
 		if _, err := os.Stat(path); err != nil {
@@ -589,34 +593,48 @@ func packageAgent(requiredPackages []string, packagingFn func()) {
 		defer os.Unsetenv(agentDropPath)
 
 		packedBeats := []string{"filebeat", "heartbeat", "metricbeat", "osquerybeat"}
-
-		for _, b := range packedBeats {
-			pwd, err := filepath.Abs(filepath.Join("../../beats", b))
-			if err != nil {
-				panic(err)
+		if devtools.ExternalBuild == true {
+			ctx := context.Background()
+			for _, beat := range packedBeats {
+				for _, reqPackage := range requiredPackages {
+					newVersion, packageName := getPackageName(beat, version, reqPackage)
+					err := fetchBinaryFromArtifactsApi(ctx, packageName, beat, newVersion, dropPath)
+					if err != nil {
+						panic(err)
+					}
+				}
 			}
+		} else {
+			//build from local repo, will assume beats repo is located on the same root level
+			for _, b := range packedBeats {
+				pwd, err := filepath.Abs(filepath.Join("../../beats", b))
+				if err != nil {
+					panic(err)
+				}
 
-			if !requiredPackagesPresent(pwd, b, version, requiredPackages) {
-				cmd := exec.Command("mage", "package")
-				cmd.Dir = pwd
-				cmd.Stdout = os.Stdout
-				cmd.Stderr = os.Stderr
-				cmd.Env = append(os.Environ(), fmt.Sprintf("PWD=%s", pwd), "AGENT_PACKAGING=on")
-				if envVar := selectedPackageTypes(); envVar != "" {
-					cmd.Env = append(cmd.Env, envVar)
+				if !requiredPackagesPresent(pwd, b, version, requiredPackages) {
+					cmd := exec.Command("mage", "package")
+					cmd.Dir = pwd
+					cmd.Stdout = os.Stdout
+					cmd.Stderr = os.Stderr
+					cmd.Env = append(os.Environ(), fmt.Sprintf("PWD=%s", pwd), "AGENT_PACKAGING=on")
+					if envVar := selectedPackageTypes(); envVar != "" {
+						cmd.Env = append(cmd.Env, envVar)
+					}
+
+					if err := cmd.Run(); err != nil {
+						panic(err)
+					}
 				}
 
-				if err := cmd.Run(); err != nil {
+				// copy to new drop
+				sourcePath := filepath.Join(pwd, "build", "distributions")
+				if err := copyAll(sourcePath, dropPath); err != nil {
 					panic(err)
 				}
 			}
-
-			// copy to new drop
-			sourcePath := filepath.Join(pwd, "build", "distributions")
-			if err := copyAll(sourcePath, dropPath); err != nil {
-				panic(err)
-			}
 		}
+
 	}
 
 	// package agent
@@ -627,6 +645,14 @@ func packageAgent(requiredPackages []string, packagingFn func()) {
 	mg.SerialDeps(devtools.Package, TestPackages)
 }
 
+func fetchBinaryFromArtifactsApi(ctx context.Context, packageName, artifact, version, downloadPath string) error {
+	location, err := downloads.FetchBeatsBinary(ctx, packageName, artifact, version, 3, false, downloadPath, true)
+	if err != nil {
+		panic(err)
+	}
+	return nil
+}
+
 func selectedPackageTypes() string {
 	if len(devtools.SelectedPackageTypes) == 0 {
 		return ""

elastic-agent/docs/version.asciidoc

@@ -18,11 +18,11 @@
 package release
 
 import (
+	"github.com/elastic/elastic-agent-poc/elastic-agent/version"
 	"strconv"
 	"strings"
 	"time"
 
-	libbeatVersion "github.com/elastic/beats/v7/libbeat/version"
 )
 
 const (
@@ -51,7 +51,7 @@ func TrimCommit(commit string) string {
 
 // Commit returns the current build hash or unknown if it was not injected in the build process.
 func Commit() string {
-	return libbeatVersion.Commit()
+	return version.Commit()
 }
 
 // ShortCommit returns commit up to 6 characters.
@@ -61,12 +61,12 @@ func ShortCommit() string {
 
 // BuildTime returns the build time of the binaries.
 func BuildTime() time.Time {
-	return libbeatVersion.BuildTime()
+	return version.BuildTime()
 }
 
 // Version returns the version of the application.
 func Version() string {
-	return libbeatVersion.GetDefaultVersion()
+	return version.GetDefaultVersion()
 }
 
 // Snapshot returns true if binary was built as snapshot.

elastic-agent/magefile.go

@@ -4,4 +4,4 @@
 
 package version
 
-const defaultBeatVersion = "7.15.3"
+const defaultBeatVersion = "8.1.0"