Merge branch 'main' into fleet-bootstrap-output

Author: michel-laterman

.agent-versions.json

@@ -1,5 +1,6 @@
 {
   "testVersions": [
+    "8.14.0-SNAPSHOT",
     "8.13.3-SNAPSHOT",
     "8.13.2",
     "8.12.2",

.ci/bump-golang.yml

@@ -123,12 +123,21 @@ targets:
         - Dockerfile
         - Dockerfile.skaffold
       matchpattern: 'ARG GO_VERSION=\d+.\d+.\d+'
-  update-gomod:
-    name: "Update go.mod"
+  update-gomod-minor-version:
+    name: "Update go.mod minor version"
     sourceid: gomod
     scmid: githubConfig
     kind: file
     spec:
       content: 'go {{ source "gomod" }}'
       file: go.mod
       matchpattern: 'go \d+.\d+'
+  update-gomod-toolchain-version:
+    name: "Update go.mod toolchain version"
+    sourceid: latestGoVersion
+    scmid: githubConfig
+    kind: file
+    spec:
+      content: 'toolchain go{{ source "latestGoVersion" }}'
+      file: go.mod
+      matchpattern: 'toolchain go\d+.\d+.\d+'

.github/PULL_REQUEST_TEMPLATE.md

@@ -36,13 +36,6 @@ List here all the items you have verified BEFORE sending this PR. Please DO NOT
 - [ ] I have added an entry in `./changelog/fragments` using the [changelog tool](https://github.com/elastic/elastic-agent#changelog)
 - [ ] I have added an integration test or an E2E test
 
-## Author's Checklist
-
-<!-- Recommended
-Add a checklist of things that are required to be reviewed in order to have the PR approved
--->
-- [ ]
-
 ## How to test this PR locally
 
 <!-- Recommended
@@ -59,32 +52,12 @@ Link related issues below. Insert the issue link or reference after the word "Cl
 - Requires #123
 - Superseds #123
 -->
-- 
-
-## Use cases
-
-<!-- Recommended
-Explain here the different behaviors that this PR introduces or modifies in this project, user roles, environment configuration, etc.
-
-If you are familiar with Gherkin test scenarios, we recommend its usage: https://cucumber.io/docs/gherkin/reference/
--->
-
-## Screenshots
-
-<!-- Optional
-Add here screenshots about how the project will be changed after the PR is applied. They could be related to web pages, terminal, etc, or any other image you consider important to be shared with the team.
--->
-
-## Logs
-
-<!-- Recommended
-Paste here output logs discovered while creating this PR, such as stack traces or integration logs, or any other output you consider important to be shared with the team.
--->
+-
 
 ## Questions to ask yourself
 
-- How are we going to support this in production? 
-- How are we going to measure its adoption? 
+- How are we going to support this in production?
+- How are we going to measure its adoption?
 - How are we going to debug this?
 - What are the metrics I should take care of?
 - ...

NOTICE.txt

@@ -692,16 +692,18 @@ spec:
       # Uncomment if using hints feature
       #initContainers:
       #  - name: k8s-templates-downloader
-      #    image: busybox:1.28
-      #    command: ['sh']
+      #    image: docker.elastic.co/beats/elastic-agent:8.15.0
+      #    command: ['bash']
       #    args:
       #      - -c
       #      - >-
-      #        mkdir -p /etc/elastic-agent/inputs.d &&
-      #        wget -O - https://github.com/elastic/elastic-agent/archive/8.15.tar.gz | tar xz -C /etc/elastic-agent/inputs.d --strip=5 "elastic-agent-8.15/deploy/kubernetes/elastic-agent-standalone/templates.d"
+      #        mkdir -p /usr/share/elastic-agent/state/inputs.d &&
+      #        curl -sL https://github.com/elastic/elastic-agent/archive/8.15.tar.gz | tar xz -C /usr/share/elastic-agent/state/inputs.d --strip=5 "elastic-agent-8.15/deploy/kubernetes/elastic-agent-standalone/templates.d"
+      #    securityContext:
+      #      runAsUser: 0
       #    volumeMounts:
-      #      - name: external-inputs
-      #        mountPath: /etc/elastic-agent/inputs.d
+      #      - name: elastic-agent-state
+      #        mountPath: /usr/share/elastic-agent/state
       containers:
         - name: elastic-agent-standalone
           image: docker.elastic.co/beats/elastic-agent:8.15.0
@@ -725,8 +727,6 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
-            - name: STATE_PATH
-              value: "/etc/elastic-agent"
             # The following ELASTIC_NETINFO:false variable will disable the netinfo.enabled option of add-host-metadata processor. This will remove fields host.ip and host.mac.  
             # For more info: https://www.elastic.co/guide/en/beats/metricbeat/current/add-host-metadata.html
             - name: ELASTIC_NETINFO
@@ -760,9 +760,6 @@ spec:
               mountPath: /etc/elastic-agent/agent.yml
               readOnly: true
               subPath: agent.yml
-            # Uncomment if using hints feature
-            #- name: external-inputs
-            #  mountPath: /etc/elastic-agent/inputs.d
             - name: proc
               mountPath: /hostfs/proc
               readOnly: true
@@ -793,9 +790,6 @@ spec:
           configMap:
             defaultMode: 0640
             name: agent-node-datastreams
-        # Uncomment if using hints feature
-        #- name: external-inputs
-        #  emptyDir: {}
         - name: proc
           hostPath:
             path: /proc

deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml

@@ -28,16 +28,18 @@ spec:
       # Uncomment if using hints feature
       #initContainers:
       #  - name: k8s-templates-downloader
-      #    image: busybox:1.28
-      #    command: ['sh']
+      #    image: docker.elastic.co/beats/elastic-agent:%VERSION%
+      #    command: ['bash']
       #    args:
       #      - -c
       #      - >-
-      #        mkdir -p /etc/elastic-agent/inputs.d &&
-      #        wget -O - https://github.com/elastic/elastic-agent/archive/%BRANCH%.tar.gz | tar xz -C /etc/elastic-agent/inputs.d --strip=5 "elastic-agent-%BRANCH%/deploy/kubernetes/elastic-agent-standalone/templates.d"
+      #        mkdir -p /usr/share/elastic-agent/state/inputs.d &&
+      #        curl -sL https://github.com/elastic/elastic-agent/archive/%BRANCH%.tar.gz | tar xz -C /usr/share/elastic-agent/state/inputs.d --strip=5 "elastic-agent-%BRANCH%/deploy/kubernetes/elastic-agent-standalone/templates.d"
+      #    securityContext:
+      #      runAsUser: 0
       #    volumeMounts:
-      #      - name: external-inputs
-      #        mountPath: /etc/elastic-agent/inputs.d
+      #      - name: elastic-agent-state
+      #        mountPath: /usr/share/elastic-agent/state
       containers:
         - name: elastic-agent-standalone
           image: docker.elastic.co/beats/elastic-agent:%VERSION%
@@ -61,8 +63,6 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.name
-            - name: STATE_PATH
-              value: "/etc/elastic-agent"
             # The following ELASTIC_NETINFO:false variable will disable the netinfo.enabled option of add-host-metadata processor. This will remove fields host.ip and host.mac.  
             # For more info: https://www.elastic.co/guide/en/beats/metricbeat/current/add-host-metadata.html
             - name: ELASTIC_NETINFO
@@ -96,9 +96,6 @@ spec:
               mountPath: /etc/elastic-agent/agent.yml
               readOnly: true
               subPath: agent.yml
-            # Uncomment if using hints feature
-            #- name: external-inputs
-            #  mountPath: /etc/elastic-agent/inputs.d
             - name: proc
               mountPath: /hostfs/proc
               readOnly: true
@@ -129,9 +126,6 @@ spec:
           configMap:
             defaultMode: 0640
             name: agent-node-datastreams
-        # Uncomment if using hints feature
-        #- name: external-inputs
-        #  emptyDir: {}
         - name: proc
           hostPath:
             path: /proc

deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml

@@ -14,7 +14,7 @@ require (
 	github.com/elastic/e2e-testing v1.1.0
 	github.com/elastic/elastic-agent-autodiscover v0.6.8
 	github.com/elastic/elastic-agent-client/v7 v7.8.1
-	github.com/elastic/elastic-agent-libs v0.9.5
+	github.com/elastic/elastic-agent-libs v0.9.6
 	github.com/elastic/elastic-agent-system-metrics v0.9.2
 	github.com/elastic/elastic-transport-go/v8 v8.5.0
 	github.com/elastic/go-elasticsearch/v8 v8.13.1
@@ -41,11 +41,6 @@ require (
 	github.com/mitchellh/hashstructure v1.1.0
 	github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c
 	github.com/oklog/ulid v1.3.1
-	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/fileexporter v0.98.0
-	github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.98.0
-	github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.98.0
-	github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.98.0
-	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver v0.98.0
 	github.com/otiai10/copy v1.14.0
 	github.com/pierrre/gotestcover v0.0.0-20160517101806-924dca7d15f0
 	github.com/pkg/errors v0.9.1
@@ -62,24 +57,6 @@ require (
 	go.elastic.co/apm/module/apmgorilla v1.15.0
 	go.elastic.co/ecszap v1.0.1
 	go.elastic.co/go-licence-detector v0.5.0
-	go.opentelemetry.io/collector/component v0.99.0
-	go.opentelemetry.io/collector/confmap v0.99.0
-	go.opentelemetry.io/collector/confmap/converter/expandconverter v0.99.0
-	go.opentelemetry.io/collector/confmap/provider/envprovider v0.99.0
-	go.opentelemetry.io/collector/confmap/provider/fileprovider v0.99.0
-	go.opentelemetry.io/collector/confmap/provider/httpprovider v0.99.0
-	go.opentelemetry.io/collector/confmap/provider/httpsprovider v0.99.0
-	go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.99.0
-	go.opentelemetry.io/collector/exporter v0.99.0
-	go.opentelemetry.io/collector/exporter/debugexporter v0.99.0
-	go.opentelemetry.io/collector/exporter/otlpexporter v0.99.0
-	go.opentelemetry.io/collector/featuregate v1.6.0
-	go.opentelemetry.io/collector/otelcol v0.99.0
-	go.opentelemetry.io/collector/processor v0.99.0
-	go.opentelemetry.io/collector/processor/batchprocessor v0.99.0
-	go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.99.0
-	go.opentelemetry.io/collector/receiver v0.99.0
-	go.opentelemetry.io/collector/receiver/otlpreceiver v0.99.0
 	go.uber.org/zap v1.27.0
 	golang.org/x/crypto v0.22.0
 	golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8
@@ -104,6 +81,35 @@ require (
 	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448
 )
 
+require (
+	// open telemetry dependencies
+	// pinned to v0.97.0 due to issue with ports binding which causes our tests to fail
+	// Do NOT upgrade the version beyond 0.97.0 until https://github.com/open-telemetry/opentelemetry-collector/issues/10031 is fixed!
+	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/fileexporter v0.97.0
+	github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.97.0
+	github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.97.0
+	github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.97.0
+	github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver v0.97.0
+	go.opentelemetry.io/collector/component v0.97.0
+	go.opentelemetry.io/collector/confmap v0.99.0
+	go.opentelemetry.io/collector/confmap/converter/expandconverter v0.97.0
+	go.opentelemetry.io/collector/confmap/provider/envprovider v0.97.0
+	go.opentelemetry.io/collector/confmap/provider/fileprovider v0.97.0
+	go.opentelemetry.io/collector/confmap/provider/httpprovider v0.97.0
+	go.opentelemetry.io/collector/confmap/provider/httpsprovider v0.97.0
+	go.opentelemetry.io/collector/confmap/provider/yamlprovider v0.97.0
+	go.opentelemetry.io/collector/exporter v0.97.0
+	go.opentelemetry.io/collector/exporter/debugexporter v0.97.0
+	go.opentelemetry.io/collector/exporter/otlpexporter v0.97.0
+	go.opentelemetry.io/collector/featuregate v1.6.0
+	go.opentelemetry.io/collector/otelcol v0.97.0
+	go.opentelemetry.io/collector/processor v0.97.0
+	go.opentelemetry.io/collector/processor/batchprocessor v0.97.0
+	go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.97.0
+	go.opentelemetry.io/collector/receiver v0.97.0
+	go.opentelemetry.io/collector/receiver/otlpreceiver v0.97.0
+)
+
 require (
 	github.com/Jeffail/gabs/v2 v2.6.0 // indirect
 	github.com/StackExchange/wmi v1.2.1 // indirect
@@ -122,7 +128,8 @@ require (
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/elastic/go-structform v0.0.10 // indirect
 	github.com/elastic/go-windows v1.0.1 // indirect
-	github.com/elastic/gosigar v0.14.2 // indirect
+	github.com/elastic/gosigar v0.14.3 // indirect
+	github.com/elastic/pkcs8 v1.0.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.10.1 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/expr-lang/expr v1.16.4 // indirect
@@ -178,12 +185,12 @@ require (
 	github.com/mostynb/go-grpc-compression v1.2.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/onsi/ginkgo/v2 v2.9.0 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.98.0 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter v0.98.0 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.98.0 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.98.0 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.98.0 // indirect
-	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.98.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.97.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter v0.97.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.97.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.97.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.97.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.97.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
@@ -202,50 +209,49 @@ require (
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/valyala/fastjson v1.6.4 // indirect
-	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.elastic.co/apm/module/apmhttp v1.15.0 // indirect
 	go.elastic.co/apm/module/apmhttp/v2 v2.0.0 // indirect
 	go.elastic.co/apm/v2 v2.0.0 // indirect
 	go.elastic.co/fastjson v1.1.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/collector v0.99.0 // indirect
-	go.opentelemetry.io/collector/config/configauth v0.99.0 // indirect
-	go.opentelemetry.io/collector/config/configcompression v1.6.0 // indirect
-	go.opentelemetry.io/collector/config/configgrpc v0.99.0 // indirect
-	go.opentelemetry.io/collector/config/confighttp v0.99.0 // indirect
-	go.opentelemetry.io/collector/config/confignet v0.99.0 // indirect
-	go.opentelemetry.io/collector/config/configopaque v1.6.0 // indirect
-	go.opentelemetry.io/collector/config/configretry v0.99.0 // indirect
-	go.opentelemetry.io/collector/config/configtelemetry v0.99.0 // indirect
-	go.opentelemetry.io/collector/config/configtls v0.99.0 // indirect
-	go.opentelemetry.io/collector/config/internal v0.99.0 // indirect
-	go.opentelemetry.io/collector/connector v0.99.0 // indirect
-	go.opentelemetry.io/collector/consumer v0.99.0 // indirect
-	go.opentelemetry.io/collector/extension v0.99.0 // indirect
-	go.opentelemetry.io/collector/extension/auth v0.99.0 // indirect
-	go.opentelemetry.io/collector/pdata v1.6.0 // indirect
-	go.opentelemetry.io/collector/semconv v0.99.0 // indirect
-	go.opentelemetry.io/collector/service v0.99.0 // indirect
-	go.opentelemetry.io/contrib/config v0.5.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.50.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.50.0 // indirect
-	go.opentelemetry.io/contrib/propagators/b3 v1.25.0 // indirect
-	go.opentelemetry.io/otel v1.25.0 // indirect
-	go.opentelemetry.io/otel/bridge/opencensus v1.25.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.25.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.25.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.25.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.25.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.25.0 // indirect
-	go.opentelemetry.io/otel/exporters/prometheus v0.47.0 // indirect
-	go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.25.0 // indirect
-	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.25.0 // indirect
-	go.opentelemetry.io/otel/metric v1.25.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.25.0 // indirect
-	go.opentelemetry.io/otel/sdk/metric v1.25.0 // indirect
-	go.opentelemetry.io/otel/trace v1.25.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.2.0 // indirect
+	go.opentelemetry.io/collector v0.97.0 // indirect
+	go.opentelemetry.io/collector/config/configauth v0.97.0 // indirect
+	go.opentelemetry.io/collector/config/configcompression v1.4.0 // indirect
+	go.opentelemetry.io/collector/config/configgrpc v0.97.0 // indirect
+	go.opentelemetry.io/collector/config/confighttp v0.97.0 // indirect
+	go.opentelemetry.io/collector/config/confignet v0.97.0 // indirect
+	go.opentelemetry.io/collector/config/configopaque v1.4.0 // indirect
+	go.opentelemetry.io/collector/config/configretry v0.97.0 // indirect
+	go.opentelemetry.io/collector/config/configtelemetry v0.97.0 // indirect
+	go.opentelemetry.io/collector/config/configtls v0.97.0 // indirect
+	go.opentelemetry.io/collector/config/internal v0.97.0 // indirect
+	go.opentelemetry.io/collector/connector v0.97.0 // indirect
+	go.opentelemetry.io/collector/consumer v0.97.0 // indirect
+	go.opentelemetry.io/collector/extension v0.97.0 // indirect
+	go.opentelemetry.io/collector/extension/auth v0.97.0 // indirect
+	go.opentelemetry.io/collector/pdata v1.4.0 // indirect
+	go.opentelemetry.io/collector/semconv v0.97.0 // indirect
+	go.opentelemetry.io/collector/service v0.97.0 // indirect
+	go.opentelemetry.io/contrib/config v0.4.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/contrib/propagators/b3 v1.24.0 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/bridge/opencensus v1.24.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.24.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.24.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 // indirect
+	go.opentelemetry.io/otel/exporters/prometheus v0.46.0 // indirect
+	go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.24.0 // indirect
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.24.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.1.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/mod v0.17.0 // indirect
 	golang.org/x/net v0.24.0 // indirect