Skip to content

Commit d412734

Browse files
cmacknzcmacknz
authored
Merge branch 'main' into dependabot/go_modules/github.com/elastic/elastic-agent-libs-0.9.4
2 parents 640447e + 506b695 commit d412734

34 files changed

+613
-155
lines changed

‎.buildkite/hooks/pre-exit

+2-5
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,8 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" ==
1010

1111
# Perform cleanup of integration tests resources
1212
echo "--- Cleaning up integration test resources"
13-
if [[ "$BUILDKITE_STEP_KEY" == "serverless-integration-tests" ]]; then
14-
STACK_PROVISIONER=serverless SNAPSHOT=true mage integration:clean
15-
else
16-
SNAPSHOT=true mage integration:clean
17-
fi
13+
STACK_PROVISIONER=serverless SNAPSHOT=true mage integration:clean
14+
SNAPSHOT=true mage integration:clean
1815
fi
1916

2017
if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then

‎.buildkite/misc/gce-cleanup.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ scanners:
1818
filters:
1919
- type: "<"
2020
pointer: "/extra/creationTimestamp"
21-
param: "${CREATION_DATE}"
21+
param: "${DELETE_CREATED_AFTER_DATE}"
2222
converters:
2323
param: "date"
2424
value: "date"

‎.buildkite/pipeline.elastic-agent-package.yml

+2
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,8 @@ steps:
134134
depends_on: package
135135
agents:
136136
provider: "gcp"
137+
machineType: "n2-standard-8"
138+
diskSizeGb: 250
137139
env:
138140
DRA_PROJECT_ID: "elastic-agent-package"
139141
DRA_PROJECT_ARTIFACT_ID: "agent-package"

‎.buildkite/scripts/steps/gce-cleanup.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@ set -euo pipefail
55
export ACCOUNT_KEY_SECRET=$(vault kv get -field=client_email $VAULT_PATH)
66
export ACCOUNT_SECRET=$(vault kv get -field=private_key $VAULT_PATH)
77
export ACCOUNT_PROJECT_SECRET=$(vault kv get -field=project_id $VAULT_PATH)
8-
export CREATION_DATE=$(date -Is -d "24 hours ago")
8+
export DELETE_CREATED_AFTER_DATE=$(date -Is -d "5 hours ago")
99

1010
docker run -v $(pwd)/.buildkite/misc/gce-cleanup.yml:/etc/cloud-reaper/config.yml \
1111
-e ACCOUNT_SECRET="$ACCOUNT_SECRET" \
1212
-e ACCOUNT_KEY="$ACCOUNT_KEY_SECRET" \
1313
-e ACCOUNT_PROJECT=$ACCOUNT_PROJECT_SECRET \
14-
-e CREATION_DATE=$CREATION_DATE \
14+
-e DELETE_CREATED_AFTER_DATE=$DELETE_CREATED_AFTER_DATE \
1515
${DOCKER_REGISTRY}/observability-ci/cloud-reaper:0.3.0 cloud-reaper --config /etc/cloud-reaper/config.yml destroy --confirm

‎.mergify.yml

+13
Original file line numberDiff line numberDiff line change
@@ -306,3 +306,16 @@ pull_request_rules:
306306
labels:
307307
- "backport"
308308
title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
309+
- name: backport patches to 8.14 branch
310+
conditions:
311+
- merged
312+
- label=backport-v8.14.0
313+
actions:
314+
backport:
315+
assignees:
316+
- "{{ author }}"
317+
branches:
318+
- "8.14"
319+
labels:
320+
- "backport"
321+
title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"

‎catalog-info.yaml

+3-3
Original file line numberDiff line numberDiff line change
@@ -187,10 +187,10 @@ spec:
187187
build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
188188
repository: elastic/elastic-agent
189189
schedules:
190-
Daily main:
190+
Every 4 hours:
191191
branch: main
192-
cronline: "@daily"
193-
message: Daily GCE cleanup
192+
cronline: "0 0/4 * * *" # every 4th hour
193+
message: GCE cleanup
194194
teams:
195195
ingest-fp:
196196
access_level: MANAGE_BUILD_AND_READ

‎changelog/fragments/1712737337-always-launch-more-recent-watcher.yaml

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Kind can be one of:
2+
# - breaking-change: a change to previously-documented behavior
3+
# - deprecation: functionality that is being removed in a later release
4+
# - bug-fix: fixes a problem in a previous version
5+
# - enhancement: extends functionality but does not break or fix existing behavior
6+
# - feature: new functionality
7+
# - known-issue: problems that we are aware of in a given version
8+
# - security: impacts on the security of a product or a user’s deployment.
9+
# - upgrade: important information for someone upgrading from a prior version
10+
# - other: does not fit into any of the other categories
11+
kind: bug-fix
12+
13+
# Change summary; a 80ish characters long description of the change.
14+
summary: Always select the more recent watcher during agent upgrade/downgrade
15+
16+
# Long description; in case the summary is not enough to describe the change
17+
# this field accommodate a description without length limits.
18+
# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
19+
#description:
20+
21+
# Affected component; a word indicating the component this changeset affects.
22+
component: elastic-agent
23+
24+
# PR URL; optional; the PR number that added the changeset.
25+
# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
26+
# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
27+
# Please provide it if you are adding a fragment for a different PR.
28+
pr: https://github.com/elastic/elastic-agent/pull/4491
29+
30+
# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
31+
# If not present is automatically filled by the tooling with the issue linked to the PR number.
32+
#issue: https://github.com/owner/repo/1234

‎changelog/fragments/1712949156-Switch-to-agentbeat.yaml

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Kind can be one of:
2+
# - breaking-change: a change to previously-documented behavior
3+
# - deprecation: functionality that is being removed in a later release
4+
# - bug-fix: fixes a problem in a previous version
5+
# - enhancement: extends functionality but does not break or fix existing behavior
6+
# - feature: new functionality
7+
# - known-issue: problems that we are aware of in a given version
8+
# - security: impacts on the security of a product or a user’s deployment.
9+
# - upgrade: important information for someone upgrading from a prior version
10+
# - other: does not fit into any of the other categories
11+
kind: enhancement
12+
13+
# Change summary; a 80ish characters long description of the change.
14+
summary: Reduce the overall download and on-disk size of the Elastic Agent
15+
16+
# Long description; in case the summary is not enough to describe the change
17+
# this field accommodate a description without length limits.
18+
# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
19+
#description:
20+
21+
# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
22+
component:
23+
24+
# PR URL; optional; the PR number that added the changeset.
25+
# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
26+
# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
27+
# Please provide it if you are adding a fragment for a different PR.
28+
pr: https://github.com/elastic/elastic-agent/pull/4516
29+
30+
# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
31+
# If not present is automatically filled by the tooling with the issue linked to the PR number.
32+
issue: https://github.com/elastic/elastic-agent/issues/3364

‎changelog/fragments/1713257367-Remove-beat-state-metricset-from-elastic-agent-monitoring.yaml

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Kind can be one of:
2+
# - breaking-change: a change to previously-documented behavior
3+
# - deprecation: functionality that is being removed in a later release
4+
# - bug-fix: fixes a problem in a previous version
5+
# - enhancement: extends functionality but does not break or fix existing behavior
6+
# - feature: new functionality
7+
# - known-issue: problems that we are aware of in a given version
8+
# - security: impacts on the security of a product or a user’s deployment.
9+
# - upgrade: important information for someone upgrading from a prior version
10+
# - other: does not fit into any of the other categories
11+
kind: bug-fix
12+
13+
# Change summary; a 80ish characters long description of the change.
14+
summary: Removed beat state metricset from monitoring collect metrics are unused
15+
16+
# Long description; in case the summary is not enough to describe the change
17+
# this field accommodate a description without length limits.
18+
# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
19+
#description:
20+
21+
# Affected component; a word indicating the component this changeset affects.
22+
component: elastic-agent
23+
24+
# PR URL; optional; the PR number that added the changeset.
25+
# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
26+
# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
27+
# Please provide it if you are adding a fragment for a different PR.
28+
pr: https://github.com/elastic/elastic-agent/pull/4579
29+
30+
# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
31+
# If not present is automatically filled by the tooling with the issue linked to the PR number.
32+
#issue: https://github.com/owner/repo/1234

‎changelog/fragments/1713289288-Removed-cloud-defend-from-.tar.gz-archives.yaml

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Kind can be one of:
2+
# - breaking-change: a change to previously-documented behavior
3+
# - deprecation: functionality that is being removed in a later release
4+
# - bug-fix: fixes a problem in a previous version
5+
# - enhancement: extends functionality but does not break or fix existing behavior
6+
# - feature: new functionality
7+
# - known-issue: problems that we are aware of in a given version
8+
# - security: impacts on the security of a product or a user’s deployment.
9+
# - upgrade: important information for someone upgrading from a prior version
10+
# - other: does not fit into any of the other categories
11+
kind: enhancement
12+
13+
# Change summary; a 80ish characters long description of the change.
14+
summary: Removed cloud-defend from .tar.gz archives, now it only appears in docker images
15+
16+
# Long description; in case the summary is not enough to describe the change
17+
# this field accommodate a description without length limits.
18+
# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
19+
#description:
20+
21+
# Affected component; a word indicating the component this changeset affects.
22+
component: elastic-agent
23+
24+
# PR URL; optional; the PR number that added the changeset.
25+
# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
26+
# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
27+
# Please provide it if you are adding a fragment for a different PR.
28+
pr: https://github.com/elastic/elastic-agent/pull/4584
29+
30+
# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
31+
# If not present is automatically filled by the tooling with the issue linked to the PR number.
32+
#issue: https://github.com/owner/repo/1234

‎dev-tools/mage/manifest/manifest.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ func resolveManifestPackage(project tools.Project, pkg string, reqPackage string
8585
func DownloadComponentsFromManifest(manifest string, platforms []string, platformPackages map[string]string, dropPath string) error {
8686
componentSpec := map[string][]string{
8787
"apm-server": {"apm-server"},
88-
"beats": {"auditbeat", "filebeat", "heartbeat", "metricbeat", "osquerybeat", "packetbeat"},
88+
"beats": {"agentbeat"},
8989
"cloud-defend": {"cloud-defend"},
9090
"cloudbeat": {"cloudbeat"},
9191
"elastic-agent-shipper": {"elastic-agent-shipper"},

‎dev-tools/mage/pkgtypes.go

+14-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"reflect"
1818
"regexp"
1919
"runtime"
20+
"slices"
2021
"strconv"
2122
"strings"
2223

@@ -904,15 +905,27 @@ func addFileToZip(ar *zip.Writer, baseDir string, pkgFile PackageFile) error {
904905

905906
// addFileToTar adds a file (or directory) to a tar archive.
906907
func addFileToTar(ar *tar.Writer, baseDir string, pkgFile PackageFile) error {
908+
excludedFiles := []string{
909+
"cloud-defend",
910+
"cloud-defend.spec.yml",
911+
}
912+
907913
return filepath.Walk(pkgFile.Source, func(path string, info os.FileInfo, err error) error {
908914
if err != nil {
909915
if pkgFile.SkipOnMissing && os.IsNotExist(err) {
910916
return nil
911917
}
912-
913918
return err
914919
}
915920

921+
if slices.Contains(excludedFiles, info.Name()) {
922+
// it's a file we have to exclude
923+
if mg.Verbose() {
924+
log.Printf("Skipping file %q...", path)
925+
}
926+
return nil
927+
}
928+
916929
header, err := tar.FileInfoHeader(info, info.Name())
917930
if err != nil {
918931
return err

‎dev-tools/packaging/files/linux/filebeat.sh

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/usr/bin/env bash
2+
3+
exec /opt/agentbeat/agentbeat filebeat $@

‎dev-tools/packaging/files/linux/metricbeat.sh

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/usr/bin/env bash
2+
3+
exec /opt/agentbeat/agentbeat metricbeat $@

‎dev-tools/packaging/packages.yml

+10-6
Original file line numberDiff line numberDiff line change
@@ -260,12 +260,6 @@ shared:
260260
content: >
261261
{{ commit }}
262262
mode: 0644
263-
'data/cloud_downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
264-
source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
265-
mode: 0755
266-
'data/cloud_downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
267-
source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
268-
mode: 0755
269263

270264
- &agent_docker_arm_spec
271265
<<: *agent_docker_spec
@@ -278,6 +272,16 @@ shared:
278272
extra_vars:
279273
image_name: '{{.BeatName}}-cloud'
280274
repository: 'docker.elastic.co/beats-ci'
275+
files:
276+
'data/cloud_downloads/filebeat.sh':
277+
source: '{{ elastic_beats_dir }}/dev-tools/packaging/files/linux/filebeat.sh'
278+
mode: 0755
279+
'data/cloud_downloads/metricbeat.sh':
280+
source: '{{ elastic_beats_dir }}/dev-tools/packaging/files/linux/metricbeat.sh'
281+
mode: 0755
282+
'data/cloud_downloads/agentbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
283+
source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/agentbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
284+
mode: 0755
281285

282286
- &agent_docker_complete_spec
283287
<<: *agent_docker_spec

‎dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl

+8-5
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,14 @@ RUN true && \
3434
chmod 0775 {{ $beatHome}}/{{ $modulesd }} && \
3535
{{- end }}
3636
{{- if contains .image_name "-cloud" }}
37-
mkdir -p /opt/filebeat /opt/metricbeat && \
38-
tar xf {{ $beatHome }}/data/cloud_downloads/metricbeat-*.tar.gz -C /opt/metricbeat --strip-components=1 && \
39-
tar xf {{ $beatHome }}/data/cloud_downloads/filebeat-*.tar.gz -C /opt/filebeat --strip-components=1 && \
40-
{{- end }}
37+
mkdir -p /opt/agentbeat /opt/filebeat /opt/metricbeat && \
38+
cp -f {{ $beatHome }}/data/cloud_downloads/filebeat.sh /opt/filebeat/filebeat && \
39+
chmod +x /opt/filebeat/filebeat && \
40+
cp -f {{ $beatHome }}/data/cloud_downloads/metricbeat.sh /opt/metricbeat/metricbeat && \
41+
chmod +x /opt/metricbeat/metricbeat && \
42+
tar xf {{ $beatHome }}/data/cloud_downloads/agentbeat-*.tar.gz -C /opt/agentbeat --strip-components=1 && \
4143
rm -rf {{ $beatHome }}/data/cloud_downloads && \
44+
{{- end }}
4245
true
4346

4447
FROM {{ .from }}
@@ -192,7 +195,7 @@ RUN cd {{$beatHome}}/.node \
192195
&& chmod ugo+rwX -R $NODE_PATH \
193196
# Install synthetics as a regular user, installing npm deps as root odesn't work
194197
# fix .node .npm and .synthetics
195-
&& chown -R {{ .user }}:{{ .user }} $NODE_PATH
198+
&& chown -R {{ .user }}:{{ .user }} $NODE_PATH
196199
USER {{ .user }}
197200
# If this fails dump the NPM logs
198201
RUN (npm i -g --loglevel verbose --engine-strict @elastic/synthetics@stack_release || sh -c 'tail -n +1 /root/.npm/_logs/* && exit 1') && \

‎internal/pkg/agent/application/monitoring/processes.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ func processesHandler(coord *coordinator.Coordinator) func(http.ResponseWriter,
5353
procs = append(procs, process{
5454
ID: expectedCloudProcessID(&c.Component),
5555
PID: c.LegacyPID,
56-
Binary: c.Component.InputSpec.BinaryName,
56+
Binary: c.Component.BinaryName(),
5757
Source: sourceFromComponentID(c.Component.ID),
5858
})
5959
}

‎internal/pkg/agent/application/monitoring/processes_cloud.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ func expectedCloudProcessID(c *component.Component) string {
2323
// Ensure that this is the ID we use, in agent v2 the ID is usually "apm-default".
2424
// Otherwise apm-server won't be routable/accessible in cloud.
2525
// https://github.com/elastic/elastic-agent/issues/1731#issuecomment-1325862913
26-
if strings.Contains(c.InputSpec.BinaryName, "apm-server") {
26+
if strings.Contains(c.BinaryName(), "apm-server") {
2727
// cloud understands `apm-server-default` and does not understand `apm-default`
2828
return strings.Replace(c.ID, "apm-", "apm-server-", 1)
2929
}
@@ -36,7 +36,7 @@ func matchesCloudProcessID(c *component.Component, id string) bool {
3636
// to find the APM server address. Rather than change all of the monitoring in cloud,
3737
// it is easier to just make sure the existing ID maps to the APM server component.
3838
if strings.Contains(id, "apm-server") {
39-
if strings.Contains(c.InputSpec.BinaryName, "apm-server") {
39+
if strings.Contains(c.BinaryName(), "apm-server") {
4040
return true
4141
}
4242
}

‎internal/pkg/agent/application/monitoring/v1_monitor.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -439,7 +439,7 @@ func (b *BeatsMonitor) injectLogsInput(cfg map[string]interface{}, components []
439439
continue
440440
}
441441

442-
fixedBinaryName := strings.ReplaceAll(strings.ReplaceAll(comp.InputSpec.BinaryName, "-", "_"), "/", "_") // conform with index naming policy
442+
fixedBinaryName := strings.ReplaceAll(strings.ReplaceAll(comp.BinaryName(), "-", "_"), "/", "_") // conform with index naming policy
443443
dataset := fmt.Sprintf("elastic_agent.%s", fixedBinaryName)
444444
streams = append(streams, map[string]interface{}{
445445
idKey: fmt.Sprintf("%s-%s", monitoringFilesUnitsID, comp.ID),
@@ -475,7 +475,7 @@ func (b *BeatsMonitor) injectLogsInput(cfg map[string]interface{}, components []
475475
"fields": map[string]interface{}{
476476
"id": comp.ID,
477477
"type": comp.InputSpec.InputType,
478-
"binary": comp.InputSpec.BinaryName,
478+
"binary": comp.BinaryName(),
479479
"dataset": dataset,
480480
},
481481
},
@@ -635,7 +635,7 @@ func (b *BeatsMonitor) injectMetricsInput(cfg map[string]interface{}, componentI
635635
"dataset": fmt.Sprintf("elastic_agent.%s", name),
636636
"namespace": monitoringNamespace,
637637
},
638-
"metricsets": []interface{}{"stats", "state"},
638+
"metricsets": []interface{}{"stats"},
639639
"hosts": endpoints,
640640
"period": metricsCollectionIntervalString,
641641
"index": fmt.Sprintf("metrics-elastic_agent.%s-%s", name, monitoringNamespace),

‎internal/pkg/agent/application/upgrade/artifact/download/http/common_test.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -24,16 +24,16 @@ import (
2424
)
2525

2626
const (
27-
sourcePattern = "/downloads/beats/filebeat/"
27+
sourcePattern = "/downloads/beats/agentbeat/"
2828
source = "http://artifacts.elastic.co/downloads/"
2929
)
3030

3131
var (
3232
version = agtversion.NewParsedSemVer(7, 5, 1, "", "")
3333
beatSpec = artifact.Artifact{
34-
Name: "filebeat",
35-
Cmd: "filebeat",
36-
Artifact: "beats/filebeat",
34+
Name: "agentbeat",
35+
Cmd: "agentbeat",
36+
Artifact: "beats/agentbeat",
3737
}
3838
)
3939

0 commit comments

Comments
 (0)