Switch to using agentbeat versus all the other beats from the beats repository (#4516)

* Work on using agentbeat.

* Fix docker build.

* Fix issue with BinaryName().

* Adjust to agentbeat only.

* Add changelog.

* Fix unit tests.

* Fix issue with cloud docker build.

* Another fix.

* Forgot to create /opt/agentbeat.

* Set executable.

* Fix package version test.

* Fix binary name usage places.

Author: blakerouse

changelog/fragments/1712949156-Switch-to-agentbeat.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: Reduce the overall download and on-disk size of the Elastic Agent
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component:
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/4516
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/3364

dev-tools/mage/manifest/manifest.go

@@ -85,7 +85,7 @@ func resolveManifestPackage(project tools.Project, pkg string, reqPackage string
 func DownloadComponentsFromManifest(manifest string, platforms []string, platformPackages map[string]string, dropPath string) error {
 	componentSpec := map[string][]string{
 		"apm-server":            {"apm-server"},
-		"beats":                 {"auditbeat", "filebeat", "heartbeat", "metricbeat", "osquerybeat", "packetbeat"},
+		"beats":                 {"agentbeat"},
 		"cloud-defend":          {"cloud-defend"},
 		"cloudbeat":             {"cloudbeat"},
 		"elastic-agent-shipper": {"elastic-agent-shipper"},

dev-tools/packaging/files/linux/filebeat.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+exec /opt/agentbeat/agentbeat filebeat $@

dev-tools/packaging/files/linux/metricbeat.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+exec /opt/agentbeat/agentbeat metricbeat $@

dev-tools/packaging/packages.yml

@@ -260,12 +260,6 @@ shared:
         content: >
           {{ commit }}
         mode: 0644
-      'data/cloud_downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
-        source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
-        mode: 0755
-      'data/cloud_downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
-        source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
-        mode: 0755
 
   - &agent_docker_arm_spec
     <<: *agent_docker_spec
@@ -278,6 +272,16 @@ shared:
     extra_vars:
       image_name: '{{.BeatName}}-cloud'
       repository: 'docker.elastic.co/beats-ci'
+    files:
+      'data/cloud_downloads/filebeat.sh':
+        source: '{{ elastic_beats_dir }}/dev-tools/packaging/files/linux/filebeat.sh'
+        mode: 0755
+      'data/cloud_downloads/metricbeat.sh':
+        source: '{{ elastic_beats_dir }}/dev-tools/packaging/files/linux/metricbeat.sh'
+        mode: 0755
+      'data/cloud_downloads/agentbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
+        source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/agentbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+        mode: 0755
 
   - &agent_docker_complete_spec
     <<: *agent_docker_spec

dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl

@@ -34,11 +34,14 @@ RUN true && \
     chmod 0775 {{ $beatHome}}/{{ $modulesd }} && \
 {{- end }}
 {{- if contains .image_name "-cloud" }}
-    mkdir -p /opt/filebeat /opt/metricbeat && \
-    tar xf {{ $beatHome }}/data/cloud_downloads/metricbeat-*.tar.gz -C /opt/metricbeat --strip-components=1 && \
-    tar xf {{ $beatHome }}/data/cloud_downloads/filebeat-*.tar.gz -C /opt/filebeat --strip-components=1 && \
-{{- end }}
+    mkdir -p /opt/agentbeat /opt/filebeat /opt/metricbeat && \
+    cp -f {{ $beatHome }}/data/cloud_downloads/filebeat.sh /opt/filebeat/filebeat && \
+    chmod +x /opt/filebeat/filebeat && \
+    cp -f {{ $beatHome }}/data/cloud_downloads/metricbeat.sh /opt/metricbeat/metricbeat && \
+    chmod +x /opt/metricbeat/metricbeat && \
+    tar xf {{ $beatHome }}/data/cloud_downloads/agentbeat-*.tar.gz -C /opt/agentbeat --strip-components=1 && \
     rm -rf {{ $beatHome }}/data/cloud_downloads && \
+{{- end }}
     true
 
 FROM {{ .from }}
@@ -192,7 +195,7 @@ RUN cd {{$beatHome}}/.node \
   && chmod ugo+rwX -R $NODE_PATH \
 # Install synthetics as a regular user, installing npm deps as root odesn't work
    # fix .node .npm and .synthetics
-   && chown -R {{ .user }}:{{ .user }} $NODE_PATH 
+   && chown -R {{ .user }}:{{ .user }} $NODE_PATH
 USER {{ .user }}
 # If this fails dump the NPM logs
 RUN (npm i -g --loglevel verbose --engine-strict @elastic/synthetics@stack_release || sh -c 'tail -n +1 /root/.npm/_logs/* && exit 1') && \

internal/pkg/agent/application/monitoring/processes.go

@@ -53,7 +53,7 @@ func processesHandler(coord *coordinator.Coordinator) func(http.ResponseWriter,
 				procs = append(procs, process{
 					ID:     expectedCloudProcessID(&c.Component),
 					PID:    c.LegacyPID,
-					Binary: c.Component.InputSpec.BinaryName,
+					Binary: c.Component.BinaryName(),
 					Source: sourceFromComponentID(c.Component.ID),
 				})
 			}

internal/pkg/agent/application/monitoring/processes_cloud.go

@@ -23,7 +23,7 @@ func expectedCloudProcessID(c *component.Component) string {
 	// Ensure that this is the ID we use, in agent v2 the ID is usually "apm-default".
 	// Otherwise apm-server won't be routable/accessible in cloud.
 	// https://github.com/elastic/elastic-agent/issues/1731#issuecomment-1325862913
-	if strings.Contains(c.InputSpec.BinaryName, "apm-server") {
+	if strings.Contains(c.BinaryName(), "apm-server") {
 		// cloud understands `apm-server-default` and does not understand `apm-default`
 		return strings.Replace(c.ID, "apm-", "apm-server-", 1)
 	}
@@ -36,7 +36,7 @@ func matchesCloudProcessID(c *component.Component, id string) bool {
 	// to find the APM server address. Rather than change all of the monitoring in cloud,
 	// it is easier to just make sure the existing ID maps to the APM server component.
 	if strings.Contains(id, "apm-server") {
-		if strings.Contains(c.InputSpec.BinaryName, "apm-server") {
+		if strings.Contains(c.BinaryName(), "apm-server") {
 			return true
 		}
 	}

internal/pkg/agent/application/monitoring/v1_monitor.go

@@ -439,7 +439,7 @@ func (b *BeatsMonitor) injectLogsInput(cfg map[string]interface{}, components []
 			continue
 		}
 
-		fixedBinaryName := strings.ReplaceAll(strings.ReplaceAll(comp.InputSpec.BinaryName, "-", "_"), "/", "_") // conform with index naming policy
+		fixedBinaryName := strings.ReplaceAll(strings.ReplaceAll(comp.BinaryName(), "-", "_"), "/", "_") // conform with index naming policy
 		dataset := fmt.Sprintf("elastic_agent.%s", fixedBinaryName)
 		streams = append(streams, map[string]interface{}{
 			idKey:  fmt.Sprintf("%s-%s", monitoringFilesUnitsID, comp.ID),
@@ -475,7 +475,7 @@ func (b *BeatsMonitor) injectLogsInput(cfg map[string]interface{}, components []
 						"fields": map[string]interface{}{
 							"id":      comp.ID,
 							"type":    comp.InputSpec.InputType,
-							"binary":  comp.InputSpec.BinaryName,
+							"binary":  comp.BinaryName(),
 							"dataset": dataset,
 						},
 					},

internal/pkg/agent/application/upgrade/artifact/download/http/common_test.go

@@ -24,16 +24,16 @@ import (
 )
 
 const (
-	sourcePattern = "/downloads/beats/filebeat/"
+	sourcePattern = "/downloads/beats/agentbeat/"
 	source        = "http://artifacts.elastic.co/downloads/"
 )
 
 var (
 	version  = agtversion.NewParsedSemVer(7, 5, 1, "", "")
 	beatSpec = artifact.Artifact{
-		Name:     "filebeat",
-		Cmd:      "filebeat",
-		Artifact: "beats/filebeat",
+		Name:     "agentbeat",
+		Cmd:      "agentbeat",
+		Artifact: "beats/agentbeat",
 	}
 )
 

internal/pkg/agent/application/upgrade/artifact/download/http/downloader_test.go

@@ -121,7 +121,7 @@ func TestDownloadBodyError(t *testing.T) {
 	infoLogs := obs.FilterLevelExact(zapcore.InfoLevel).TakeAll()
 	warnLogs := obs.FilterLevelExact(zapcore.WarnLevel).TakeAll()
 
-	expectedURL := fmt.Sprintf("%s/%s-%s-%s", srv.URL, "beats/filebeat/filebeat", version, "linux-x86_64.tar.gz")
+	expectedURL := fmt.Sprintf("%s/%s-%s-%s", srv.URL, "beats/agentbeat/agentbeat", version, "linux-x86_64.tar.gz")
 	expectedMsg := fmt.Sprintf("download from %s failed at 0B @ NaNBps: unexpected EOF", expectedURL)
 	require.GreaterOrEqual(t, len(infoLogs), 1, "download error not logged at info level")
 	assert.True(t, containsMessage(infoLogs, expectedMsg))
@@ -173,7 +173,7 @@ func TestDownloadLogProgressWithLength(t *testing.T) {
 	os.Remove(artifactPath)
 	require.NoError(t, err, "Download should not have errored")
 
-	expectedURL := fmt.Sprintf("%s/%s-%s-%s", srv.URL, "beats/filebeat/filebeat", version, "linux-x86_64.tar.gz")
+	expectedURL := fmt.Sprintf("%s/%s-%s-%s", srv.URL, "beats/agentbeat/agentbeat", version, "linux-x86_64.tar.gz")
 	expectedProgressRegexp := regexp.MustCompile(
 		`^download progress from ` + expectedURL + `(.sha512)? is \S+/\S+ \(\d+\.\d{2}% complete\) @ \S+$`,
 	)
@@ -256,7 +256,7 @@ func TestDownloadLogProgressWithoutLength(t *testing.T) {
 	os.Remove(artifactPath)
 	require.NoError(t, err, "Download should not have errored")
 
-	expectedURL := fmt.Sprintf("%s/%s-%s-%s", srv.URL, "beats/filebeat/filebeat", version, "linux-x86_64.tar.gz")
+	expectedURL := fmt.Sprintf("%s/%s-%s-%s", srv.URL, "beats/agentbeat/agentbeat", version, "linux-x86_64.tar.gz")
 	expectedProgressRegexp := regexp.MustCompile(
 		`^download progress from ` + expectedURL + `(.sha512)? has fetched \S+ @ \S+$`,
 	)

internal/pkg/agent/cmd/inspect.go

@@ -186,7 +186,7 @@ func inspectConfig(ctx context.Context, cfgPath string, opts inspectConfigOpts,
 		binaryMapping := make(map[string]string)
 		for _, component := range components {
 			if spec := component.InputSpec; spec != nil {
-				binaryMapping[component.ID] = spec.BinaryName
+				binaryMapping[component.ID] = component.BinaryName()
 			}
 		}
 		monitorCfg, err := monitorFn(cfg, components, binaryMapping)

magefile.go

@@ -1012,12 +1012,7 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi
 			// https://artifacts-snapshot.elastic.co/fleet-server/latest/8.11.0-SNAPSHOT.json
 			// https://artifacts-snapshot.elastic.co/prodfiler/latest/8.11.0-SNAPSHOT.json
 			externalBinaries := map[string]string{
-				"auditbeat":             "beats",
-				"filebeat":              "beats",
-				"heartbeat":             "beats",
-				"metricbeat":            "beats",
-				"osquerybeat":           "beats",
-				"packetbeat":            "beats",
+				"agentbeat":             "beats",
 				"cloudbeat":             "cloudbeat", // only supporting linux/amd64 or linux/arm64
 				"cloud-defend":          "cloud-defend",
 				"apm-server":            "apm-server", // not supported on darwin/aarch64
@@ -1058,7 +1053,7 @@ func collectPackageDependencies(platforms []string, packageVersion string, requi
 				panic(fmt.Sprintf("No packages were successfully downloaded. You may be building against an invalid or unreleased version. version=%s. If this is an unreleased version, try SNAPSHOT=true or EXTERNAL=false", packageVersion))
 			}
 		} else {
-			packedBeats := []string{"filebeat", "heartbeat", "metricbeat", "osquerybeat"}
+			packedBeats := []string{"agentbeat"}
 			// build from local repo, will assume beats repo is located on the same root level
 			for _, b := range packedBeats {
 				pwd, err := filepath.Abs(filepath.Join("../beats/x-pack", b))

pkg/component/component.go

@@ -208,6 +208,26 @@ func (c *Component) Type() string {
 	return ""
 }
 
+// BinaryName returns the binary name used for the component.
+//
+// This can differ from the actual binary name that is on disk, when the input specification states that the
+// command has a different name.
+func (c *Component) BinaryName() string {
+	if c.InputSpec != nil {
+		if c.InputSpec.Spec.Command != nil && c.InputSpec.Spec.Command.Name != "" {
+			return c.InputSpec.Spec.Command.Name
+		}
+		return c.InputSpec.BinaryName
+	}
+	if c.ShipperSpec != nil {
+		if c.ShipperSpec.Spec.Command != nil && c.ShipperSpec.Spec.Command.Name != "" {
+			return c.ShipperSpec.Spec.Command.Name
+		}
+		return c.ShipperSpec.BinaryName
+	}
+	return ""
+}
+
 // Model is the components model with signed policy data
 // This replaces former top level []Components with the top Model that captures signed policy data.
 // The signed data is a part of the policy since 8.8.0 release and contains the signed policy fragments and the signature that can be validated.
@@ -275,9 +295,7 @@ func (r *RuntimeSpecs) ToComponents(
 		// binary name
 		binaryMapping := make(map[string]string)
 		for _, component := range components {
-			if spec := component.InputSpec; spec != nil {
-				binaryMapping[component.ID] = spec.BinaryName
-			}
+			binaryMapping[component.ID] = component.BinaryName()
 		}
 		monitoringCfg, err := monitoringInjector(policy, components, binaryMapping)
 		if err != nil {

pkg/component/runtime/command.go

@@ -497,13 +497,7 @@ func (c *commandRuntime) getSpecType() string {
 }
 
 func (c *commandRuntime) getSpecBinaryName() string {
-	if c.current.InputSpec != nil {
-		return c.current.InputSpec.BinaryName
-	}
-	if c.current.ShipperSpec != nil {
-		return c.current.ShipperSpec.BinaryName
-	}
-	return ""
+	return c.current.BinaryName()
 }
 
 func (c *commandRuntime) getSpecBinaryPath() string {

pkg/component/runtime/service.go

@@ -584,20 +584,20 @@ func (s *serviceRuntime) name() string {
 // check executes the service check command
 func (s *serviceRuntime) check(ctx context.Context) error {
 	if s.comp.InputSpec.Spec.Service.Operations.Check == nil {
-		s.log.Errorf("missing check spec for %s service", s.comp.InputSpec.BinaryName)
+		s.log.Errorf("missing check spec for %s service", s.comp.BinaryName())
 		return ErrOperationSpecUndefined
 	}
-	s.log.Debugf("check if the %s is installed", s.comp.InputSpec.BinaryName)
+	s.log.Debugf("check if the %s is installed", s.comp.BinaryName())
 	return s.executeServiceCommandImpl(ctx, s.log, s.comp.InputSpec.BinaryPath, s.comp.InputSpec.Spec.Service.Operations.Check)
 }
 
 // install executes the service install command
 func (s *serviceRuntime) install(ctx context.Context) error {
 	if s.comp.InputSpec.Spec.Service.Operations.Install == nil {
-		s.log.Errorf("missing install spec for %s service", s.comp.InputSpec.BinaryName)
+		s.log.Errorf("missing install spec for %s service", s.comp.BinaryName())
 		return ErrOperationSpecUndefined
 	}
-	s.log.Debugf("install %s service", s.comp.InputSpec.BinaryName)
+	s.log.Debugf("install %s service", s.comp.BinaryName())
 	return s.executeServiceCommandImpl(ctx, s.log, s.comp.InputSpec.BinaryPath, s.comp.InputSpec.Spec.Service.Operations.Install)
 }
 
@@ -645,7 +645,7 @@ func resolveUninstallTokenArg(uninstallSpec *component.ServiceOperationsCommandS
 
 func uninstallService(ctx context.Context, log *logger.Logger, comp component.Component, uninstallToken string, executeServiceCommandImpl executeServiceCommandFunc) error {
 	if comp.InputSpec.Spec.Service.Operations.Uninstall == nil {
-		log.Errorf("missing uninstall spec for %s service", comp.InputSpec.BinaryName)
+		log.Errorf("missing uninstall spec for %s service", comp.BinaryName())
 		return ErrOperationSpecUndefined
 	}
 
@@ -657,6 +657,6 @@ func uninstallService(ctx context.Context, log *logger.Logger, comp component.Co
 
 	uninstallSpec := resolveUninstallTokenArg(comp.InputSpec.Spec.Service.Operations.Uninstall, uninstallToken)
 
-	log.Debugf("uninstall %s service", comp.InputSpec.BinaryName)
+	log.Debugf("uninstall %s service", comp.BinaryName())
 	return executeServiceCommandImpl(ctx, log, comp.InputSpec.BinaryPath, uninstallSpec)
 }

pkg/component/spec.go

@@ -74,6 +74,7 @@ type RuntimePreventionSpec struct {
 
 // CommandSpec is the specification for an input that executes as a subprocess.
 type CommandSpec struct {
+	Name                    string             `config:"name,omitempty" yaml:"name,omitempty"`
 	Args                    []string           `config:"args,omitempty" yaml:"args,omitempty"`
 	Env                     []CommandEnvSpec   `config:"env,omitempty" yaml:"env,omitempty"`
 	Timeouts                CommandTimeoutSpec `config:"timeouts,omitempty" yaml:"timeouts,omitempty"`

testing/integration/package_version_test.go

@@ -124,25 +124,25 @@ func TestComponentBuildHashInDiagnostics(t *testing.T) {
 		5*time.Minute, 10*time.Second,
 		"agent never became healthy. Last status: %v", &stateBuff)
 
-	filebeat := "filebeat"
+	agentbeat := "agentbeat"
 	if runtime.GOOS == "windows" {
-		filebeat += ".exe"
+		agentbeat += ".exe"
 	}
 	wd := f.WorkDir()
-	glob := filepath.Join(wd, "data", "elastic-agent-*", "components", filebeat)
+	glob := filepath.Join(wd, "data", "elastic-agent-*", "components", agentbeat)
 	compPaths, err := filepath.Glob(glob)
-	require.NoErrorf(t, err, "failed to glob filebeat path pattern %q", glob)
+	require.NoErrorf(t, err, "failed to glob agentbeat path pattern %q", glob)
 	require.Lenf(t, compPaths, 1,
-		"glob pattern \"%s\": found %d paths to filebeat, can only have 1",
+		"glob pattern \"%s\": found %d paths to agentbeat, can only have 1",
 		glob, len(compPaths))
 
-	cmdVer := exec.Command(compPaths[0], "version")
+	cmdVer := exec.Command(compPaths[0], "filebeat", "version")
 	output, err = cmdVer.CombinedOutput()
 	require.NoError(t, err, "failed to get filebeat version")
 	outStr := string(output)
 
 	// version output example:
-	// filebeat version 8.13.0 (amd64), libbeat 8.13.0 [0baedd2518bd7e5b78e2280684580cbfdcab5ae8 built 2024-01-23 06:57:37 +0000 UTC
+	// filebeat version 8.14.0 (arm64), libbeat 8.14.0 [ab27a657e4f15976c181cf44c529bba6159f2c64 built 2024-04-17 18:13:16 +0000 UTC]
 	t.Log("parsing commit hash from filebeat version: ", outStr)
 	splits := strings.Split(outStr, "[")
 	require.Lenf(t, splits, 2,