Merge branch 'main' into dependabot/go_modules/github.com/elastic/go-elasticsearch/v8-8.11.0

Author: pierrehilbert

.buildkite/hooks/pre-command.ps1

@@ -14,8 +14,12 @@ go version
 
 $GOPATH = $(go env GOPATH)
 $env:Path = "$GOPATH\bin;" + $env:Path
+$env:GOTMPDIR = "$GOPATH\tmp"
+New-Item -ItemType Directory -Force -Path $env:GOTMPDIR
 [Environment]::SetEnvironmentVariable("GOPATH", "$GOPATH", [EnvironmentVariableTarget]::Machine)
+[Environment]::SetEnvironmentVariable("GOTMPDIR", "$GOPATH\tmp", [EnvironmentVariableTarget]::Machine)
 [Environment]::SetEnvironmentVariable("Path", "$GOPATH\bin;$env:Path", [EnvironmentVariableTarget]::Machine)
+go env
 
 # Install tools
 go install github.com/magefile/mage

.github/CODEOWNERS

@@ -4,3 +4,4 @@
 /deploy/kubernetes @elastic/obs-cloudnative-monitoring
 /dev-tools/kubernetes @elastic/obs-cloudnative-monitoring
 /internal/pkg/composable/providers/kubernetes @elastic/obs-cloudnative-monitoring
+/internal/pkg/agent/application/configuration_embed_changed_test.go @elastic/cloudbeat

.go-version

@@ -1 +1 @@
-1.21.6
+1.21.7

.golangci.yml

@@ -116,7 +116,7 @@ linters-settings:
 
   gosimple:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.6"
+    go: "1.21.7"
 
   nakedret:
     # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
@@ -136,17 +136,17 @@ linters-settings:
 
   staticcheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.6"
+    go: "1.21.7"
     checks: ["all"]
 
   stylecheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.6"
+    go: "1.21.7"
     checks: ["all"]
 
   unused:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.6"
+    go: "1.21.7"
 
   gosec:
     excludes:

Dockerfile

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.21.6
+ARG GO_VERSION=1.21.7
 FROM circleci/golang:${GO_VERSION}
 
 

Dockerfile.skaffold

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.21.6
+ARG GO_VERSION=1.21.7
 ARG crossbuild_image="docker.elastic.co/beats-dev/golang-crossbuild"
 ARG AGENT_VERSION=8.9.0-SNAPSHOT
 ARG AGENT_IMAGE="docker.elastic.co/beats/elastic-agent"

NOTICE.txt

@@ -4,6 +4,11 @@
 # and conditionals. Each provider's keys are automatically prefixed with the name
 # of the provider.
 
+# All registered providers are enabled by default.
+
+# Disable all providers by default and only enable explicitly configured providers.
+# agent.providers.initial_default: false
+
 #providers:
 
 # Agent provides information about the running agent.

_meta/config/providers.yml.tmpl

@@ -104,6 +104,8 @@ spec:
           access_level: MANAGE_BUILD_AND_READ
         release-eng:
           access_level: BUILD_AND_READ
+        endpoint-ci-admin:
+          access_level: BUILD_AND_READ
         everyone:
           access_level: READ_ONLY
 

catalog-info.yaml

@@ -11,12 +11,12 @@
 kind: security
 
 # Change summary; a 80ish characters long description of the change.
-summary: Upgrade to Go 1.21.6
+summary: Upgrade to Go 1.21.7
 
 # Long description; in case the summary is not enough to describe the change
 # this field accommodate a description without length limits.
 # NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
-# description:
+description: Removes support for Windows 8.1. See https://tip.golang.org/doc/go1.21#windows.
 
 # Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
 component: "elastic-agent"
@@ -25,7 +25,7 @@ component: "elastic-agent"
 # If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
 # NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
 # Please provide it if you are adding a fragment for a different PR.
-pr: https://github.com/elastic/elastic-agent/pull/4073
+pr: https://github.com/elastic/elastic-agent/pull/4221
 
 # Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
 # If not present is automatically filled by the tooling with the issue linked to the PR number.

changelog/fragments/1704304944-Upgrade-to-Go-1.21.6.yaml changelog/fragments/1704304944-Upgrade-to-Go-1.21.7.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: enhancement
+
+# Change summary; a 80ish characters long description of the change.
+summary: Add agent.providers.initial_default configuration flag to disable providers by default
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/4166
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+issue: https://github.com/elastic/elastic-agent/issues/4145

changelog/fragments/1707070032-providers-default-disable.yaml

@@ -134,6 +134,60 @@ when running them manually, such as `ELASTICSEARCH_HOST`, `ELASTICSEARCH_USERNAM
 
 ### Debugging tests
 
+#### Manually debugging tests on VMs
+Many of the integration tests will install the Elastic-Agent and/or
+require root to run, which makes it hard to just run them on our work
+machines, the best way to circumvent that is to debug the tests
+directly on a VM. `mage integration:DeployDebugTools` will show a menu
+to select a VM and then install the common debugging tools: Delve,
+Mage and Docker. It will also create the `~/elastic-agent` folder
+containing the Git repository (required to package from within the VM)
+and the last version of the code uploaded to the VM. This allows you
+to easily build/package the Elastic-Agent from within the VM as well
+as run any tests.
+
+After deploying the debug tools, `mage integrationDeployEnvFile` will
+create a `env.sh` and copy it to a selected VM, sourcing it will allow
+you to any test against the Cloud Stack you selected.
+
+Example of how to run a test from within the VM:
+```
+## Run a single test
+SNAPSHOT=true TEST_PLATFORMS="linux/amd64" mage integration:single TestLogIngestionFleetManaged
+## Let's suppose it has failed
+
+## Install DebugTools
+mage -v integration:DeployDebugTools
+
+## Generate and deploy env file
+mage -v integration:DeployEnvFile
+
+## SSH into the VM
+$(mage integration:SSHVM)
+
+## From inside the VM, the test needs root
+sudo su
+source ./env.sh
+cd elastic-agent
+## Any flags passed to the test binary go after the '--', they also need to
+## include the `test.` prefix if they're for `go test`
+TEST_DEFINE_PREFIX=gambiarra dlv test ./testing/integration/ --build-flags="-tags integration" -- -test.v -test.run TestLogIngestionFleetManaged
+```
+
+**A Delve trick**
+If you didn't build the Elastic-Agent directly on the machine you're
+debugging, it is very likely the location of the source code is
+different, hence delve cannot show you the code it is running. To
+solve this, once on Delve shell, run:
+``
+config substitute-path /go/src/github.com/elastic/elastic-agent /home/ubuntu/elastic-agent`
+``
+where:
+- `/go/src/github.com/elastic/elastic-agent` is the path annotated in
+  the binary you are debugging (the one Delve shows).
+- `/home/ubuntu/elastic-agent` is where Delve should read the source
+  code form.
+
 #### Auto diagnostics retrieval
 When an integration test fails the testing fixture will try its best to automatically collect the diagnostic
 information of the installed Elastic Agent. In the case that diagnostics is collected the test runner will
@@ -172,6 +226,32 @@ with `go test`. E.g.:
 TEST_DEFINE_PREFIX=gambiarra go test -v -tags integration -run TestProxyURL ./testing/integration/
 ```
 
+## Connecting to VMs and running tests
+### Connecting to VMs
+All VMs (including Windows) support connections via SSH, the framework
+generates and stores the necessary SSH keys to access the VMs, the
+easiest way to connect to them is using the SSH command returned by
+`mage integration:SSHVM`. It will list the VMs and ask to select
+one.
+
+On a Unix shell you can run `$(mage integration:SSHVM)`, the menu is
+printed to stderr and the SSH command to stdout. After selecting the
+VM you will have shell connected to it.
+
+### Credentials for cloud stack/projects
+All cloud deployments and projects can be listed with `mage
+integration:listStacks`, they can be used to manually connect to
+Kibana and Elasticsearch.
+
+If you need to manually run tests against any deployments, `mage
+integration:GenerateEnvFile` will generate a file called `env.sh` that
+exports environment variables for Unix compatible shells, you can load
+them into your shell by running `source ./env.sh`.
+
+To easily deploy the credentials to any VM, just run `mage
+integration:DeployEnvFile`. A menu will ask for the desired Stack and
+VM.
+
 ## Writing tests
 
 Write integration and E2E tests by adding them to the `testing/integration`

docs/test-framework-dev-guide.md

@@ -218,6 +218,11 @@ agent.logging.to_stderr: true
 # and conditionals. Each provider's keys are automatically prefixed with the name
 # of the provider.
 
+# All registered providers are enabled by default.
+
+# Disable all providers by default and only enable explicitly configured providers.
+# agent.providers.initial_default: false
+
 #providers:
 
 # Agent provides information about the running agent.

elastic-agent.docker.yml

@@ -271,6 +271,11 @@ agent.logging.to_stderr: true
 # and conditionals. Each provider's keys are automatically prefixed with the name
 # of the provider.
 
+# All registered providers are enabled by default.
+
+# Disable all providers by default and only enable explicitly configured providers.
+# agent.providers.initial_default: false
+
 #providers:
 
 # Agent provides information about the running agent.

elastic-agent.reference.yml

@@ -261,6 +261,11 @@ agent.logging.to_stderr: true
 # and conditionals. Each provider's keys are automatically prefixed with the name
 # of the provider.
 
+# All registered providers are enabled by default.
+
+# Disable all providers by default and only enable explicitly configured providers.
+# agent.providers.initial_default: false
+
 #providers:
 
 # Agent provides information about the running agent.

elastic-agent.yml

@@ -65,6 +65,7 @@ require (
 	go.opentelemetry.io/collector/exporter v0.93.0
 	go.opentelemetry.io/collector/exporter/debugexporter v0.93.0
 	go.opentelemetry.io/collector/exporter/otlpexporter v0.93.0
+	go.opentelemetry.io/collector/featuregate v1.0.1
 	go.opentelemetry.io/collector/otelcol v0.93.0
 	go.opentelemetry.io/collector/processor v0.93.0
 	go.opentelemetry.io/collector/processor/batchprocessor v0.93.0
@@ -214,7 +215,6 @@ require (
 	go.opentelemetry.io/collector/consumer v0.93.0 // indirect
 	go.opentelemetry.io/collector/extension v0.93.0 // indirect
 	go.opentelemetry.io/collector/extension/auth v0.93.0 // indirect
-	go.opentelemetry.io/collector/featuregate v1.0.1 // indirect
 	go.opentelemetry.io/collector/pdata v1.0.1 // indirect
 	go.opentelemetry.io/collector/semconv v0.93.0 // indirect
 	go.opentelemetry.io/collector/service v0.93.0 // indirect

go.mod

@@ -37,7 +37,7 @@ const (
 // PolicyChangeHandler is a handler for POLICY_CHANGE action.
 type PolicyChangeHandler struct {
 	log       *logger.Logger
-	agentInfo *info.AgentInfo
+	agentInfo info.Agent
 	config    *configuration.Configuration
 	store     storage.Store
 	ch        chan coordinator.ConfigChange
@@ -52,7 +52,7 @@ type PolicyChangeHandler struct {
 // NewPolicyChangeHandler creates a new PolicyChange handler.
 func NewPolicyChangeHandler(
 	log *logger.Logger,
-	agentInfo *info.AgentInfo,
+	agentInfo info.Agent,
 	config *configuration.Configuration,
 	store storage.Store,
 	ch chan coordinator.ConfigChange,
@@ -264,7 +264,7 @@ func clientEqual(k1 remote.Config, k2 remote.Config) bool {
 	return true
 }
 
-func fleetToReader(agentInfo *info.AgentInfo, cfg *configuration.Configuration) (io.Reader, error) {
+func fleetToReader(agentInfo info.Agent, cfg *configuration.Configuration) (io.Reader, error) {
 	configToStore := map[string]interface{}{
 		"fleet": cfg.Fleet,
 		"agent": map[string]interface{}{

internal/pkg/agent/application/actions/handlers/handler_action_policy_change.go

@@ -34,10 +34,7 @@ func TestPolicyChange(t *testing.T) {
 	log, _ := logger.New("", false)
 	ack := noopacker.New()
 
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	agentInfo, _ := info.NewAgentInfo(ctx, true)
+	agentInfo := &info.AgentInfo{}
 	nullStore := &storage.NullStore{}
 
 	t.Run("Receive a config change and successfully emits a raw configuration", func(t *testing.T) {
@@ -63,10 +60,8 @@ func TestPolicyChange(t *testing.T) {
 
 func TestPolicyAcked(t *testing.T) {
 	log, _ := logger.New("", false)
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
 
-	agentInfo, _ := info.NewAgentInfo(ctx, true)
+	agentInfo := &info.AgentInfo{}
 	nullStore := &storage.NullStore{}
 
 	t.Run("Config change should ACK", func(t *testing.T) {

internal/pkg/agent/application/actions/handlers/handler_action_policy_change_test.go

@@ -20,14 +20,14 @@ import (
 // Settings handles settings change coming from fleet and updates log level.
 type Settings struct {
 	log       *logger.Logger
-	agentInfo *info.AgentInfo
+	agentInfo info.Agent
 	coord     *coordinator.Coordinator
 }
 
 // NewSettings creates a new Settings handler.
 func NewSettings(
 	log *logger.Logger,
-	agentInfo *info.AgentInfo,
+	agentInfo info.Agent,
 	coord *coordinator.Coordinator,
 ) *Settings {
 	return &Settings{

internal/pkg/agent/application/actions/handlers/handler_action_settings.go

@@ -63,7 +63,8 @@ func TestUpgradeHandler(t *testing.T) {
 	defer cancel()
 
 	log, _ := logger.New("", false)
-	agentInfo, _ := info.NewAgentInfo(ctx, true)
+
+	agentInfo := &info.AgentInfo{}
 	msgChan := make(chan string)
 
 	// Create and start the coordinator
@@ -95,7 +96,8 @@ func TestUpgradeHandlerSameVersion(t *testing.T) {
 	defer cancel()
 
 	log, _ := logger.New("", false)
-	agentInfo, _ := info.NewAgentInfo(ctx, true)
+
+	agentInfo := &info.AgentInfo{}
 	msgChan := make(chan string)
 
 	// Create and start the Coordinator
@@ -129,7 +131,8 @@ func TestUpgradeHandlerNewVersion(t *testing.T) {
 	defer cancel()
 
 	log, _ := logger.New("", false)
-	agentInfo, _ := info.NewAgentInfo(ctx, true)
+
+	agentInfo := &info.AgentInfo{}
 	msgChan := make(chan string)
 
 	// Create and start the Coordinator