Merge branch 'main' into dependabot/go_modules/github.com/elastic/go-elasticsearch/v8-8.11.0

Author: pierrehilbert

.buildkite/pipeline.yml

@@ -130,7 +130,7 @@ steps:
         agents:
           provider: "gcp"
           # TODO create own image
-          image: "family/endpoint-windows-10-tester-rel"
+          image: "family/general-windows-10"
           machine_type: "n2-standard-8"
           disk_type: "pd-ssd"
         retry:
@@ -147,7 +147,7 @@ steps:
         agents:
           provider: "gcp"
           # TODO create own image
-          image: "family/endpoint-windows-11-tester-rel"
+          image: "family/general-windows-11"
           machine_type: "n2-standard-8"
           disk_type: "pd-ssd"
         retry:

.github/workflows/bump-golang.yml

@@ -4,11 +4,14 @@ name: bump-golang
 on:
   workflow_dispatch:
   schedule:
-    - cron: "0 20 * * 6"
+    - cron: "0 20 * * 1-6"
 
 permissions:
   contents: read
 
+env:
+  JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
 jobs:
   bump:
     runs-on: ubuntu-latest
@@ -21,3 +24,5 @@ jobs:
           vaultRoleId: ${{ secrets.VAULT_ROLE_ID }}
           vaultSecretId: ${{ secrets.VAULT_SECRET_ID }}
           pipeline: ./.github/updatecli-bump-golang.yml
+          notifySlackChannel: "#ingest-notifications"
+          messageIfFailure: ":traffic_cone: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, `@agent-team` please look what's going on <${{ env.JOB_URL }}|here>"

NOTICE.txt

@@ -48,9 +48,9 @@ spec:
         filter_condition: >-
           build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
       cancel_intermediate_builds: true
-      cancel_intermediate_builds_branch_filter: "!main"
+      cancel_intermediate_builds_branch_filter: '!main !7.* !8.* !9.*'
       skip_intermediate_builds: true
-      skip_intermediate_builds_branch_filter: "!main"
+      skip_intermediate_builds_branch_filter: '!main !7.* !8.* !9.*'
       env:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
         SLACK_NOTIFICATIONS_CHANNEL: "#ingest-notifications"
@@ -96,9 +96,9 @@ spec:
         filter_condition: >-
           build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
       cancel_intermediate_builds: true
-      cancel_intermediate_builds_branch_filter: "!main"
+      cancel_intermediate_builds_branch_filter: '!main !7.* !8.* !9.*'
       skip_intermediate_builds: true
-      skip_intermediate_builds_branch_filter: "!main"
+      skip_intermediate_builds_branch_filter: '!main !7.* !8.* !9.*'
       teams:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ

catalog-info.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: fleet-config-change-logging
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: Coordinator
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

changelog/fragments/1704824761-fleet-config-change-logging.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# Change summary; a 80ish characters long description of the change.
+summary: On Windows prevent uninstall from within installed directory
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description:  'On Windows, prevent uninstall from within installed directory to prevent a sharing violation.  Also for install and uninstall, print debug logs to stderr if command fails.'
+
+# Affected component; a word indicating the component this changeset affects.
+component: agent
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

changelog/fragments/1706123313-on-Windows-prevent-uninstall-from-within-installed-directory.yaml

@@ -480,6 +480,14 @@ func untar(sourceFile, destinationDir string) error {
 			if err = writer.Close(); err != nil {
 				return err
 			}
+		case tar.TypeSymlink:
+			if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+				return err
+			}
+			if err := os.Symlink(header.Linkname, path); err != nil {
+				return fmt.Errorf("error creating symlink %s pointing to %s: %w", path, header.Linkname, err)
+			}
+
 		default:
 			return fmt.Errorf("unable to untar type=%c in file=%s", header.Typeflag, path)
 		}
@@ -861,21 +869,21 @@ var parseVersionRegex = regexp.MustCompile(`(?m)^[^\d]*(?P<major>\d+)\.(?P<minor
 
 // ParseVersion extracts the major, minor, and optional patch number from a
 // version string.
-func ParseVersion(version string) (major, minor, patch int, err error) {
+func ParseVersion(version string) (int, int, int, error) {
 	names := parseVersionRegex.SubexpNames()
 	matches := parseVersionRegex.FindStringSubmatch(version)
 	if len(matches) == 0 {
-		err = fmt.Errorf("failed to parse version '%v'", version)
-		return
+		err := fmt.Errorf("failed to parse version '%v'", version)
+		return 0, 0, 0, err
 	}
 
 	data := map[string]string{}
 	for i, match := range matches {
 		data[names[i]] = match
 	}
-	major, _ = strconv.Atoi(data["major"])
-	minor, _ = strconv.Atoi(data["minor"])
-	patch, _ = strconv.Atoi(data["patch"])
+	major, _ := strconv.Atoi(data["major"])
+	minor, _ := strconv.Atoi(data["minor"])
+	patch, _ := strconv.Atoi(data["patch"])
 	return major, minor, patch, nil
 }
 

dev-tools/mage/common.go

@@ -11,8 +11,6 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
-
-	"github.com/pkg/errors"
 )
 
 // Copy copies a file or a directory (recursively) and preserves the permissions.
@@ -34,22 +32,25 @@ type CopyTask struct {
 // Execute executes the copy and returns an error of there is a failure.
 func (t *CopyTask) Execute() error {
 	if err := t.init(); err != nil {
-		return errors.Wrap(err, "copy failed")
+		return fmt.Errorf("copy failed: %w", err)
 	}
 
 	info, err := os.Stat(t.Source)
 	if err != nil {
-		return errors.Wrapf(err, "copy failed: cannot stat source file %v", t.Source)
+		return fmt.Errorf("copy failed: cannot stat source file %v: %w", t.Source, err)
 	}
 
-	return errors.Wrap(t.recursiveCopy(t.Source, t.Dest, fs.FileInfoToDirEntry(info)), "copy failed")
+	if err := t.recursiveCopy(t.Source, t.Dest, fs.FileInfoToDirEntry(info)); err != nil {
+		return fmt.Errorf("copy failed: %w", err)
+	}
+	return nil
 }
 
 func (t *CopyTask) init() error {
 	for _, excl := range t.Exclude {
 		re, err := regexp.Compile(excl)
 		if err != nil {
-			return errors.Wrapf(err, "bad exclude pattern %v", excl)
+			return fmt.Errorf("bad exclude pattern %v: %w", excl, err)
 		}
 		t.excludes = append(t.excludes, re)
 	}
@@ -89,8 +90,7 @@ func (t *CopyTask) fileCopy(src, dest string, entry fs.DirEntry) error {
 	}
 
 	if !info.Mode().IsRegular() {
-		return errors.Errorf("failed to copy source file because it is not a " +
-			"regular file")
+		return fmt.Errorf("failed to copy source file because it is not a regular file")
 	}
 
 	mode := t.Mode
@@ -126,19 +126,19 @@ func (t *CopyTask) dirCopy(src, dest string, entry fs.DirEntry) error {
 	}
 
 	if err := os.MkdirAll(dest, mode&os.ModePerm); err != nil {
-		return errors.Wrap(err, "failed creating dirs")
+		return fmt.Errorf("failed creating dirs: %w", err)
 	}
 
 	contents, err := os.ReadDir(src)
 	if err != nil {
-		return errors.Wrapf(err, "failed to read dir %v", src)
+		return fmt.Errorf("failed to read dir %v: %w", src, err)
 	}
 
 	for _, entry := range contents {
-		srcFile := filepath.Join(src, info.Name())
-		destFile := filepath.Join(dest, info.Name())
+		srcFile := filepath.Join(src, entry.Name())
+		destFile := filepath.Join(dest, entry.Name())
 		if err = t.recursiveCopy(srcFile, destFile, entry); err != nil {
-			return errors.Wrapf(err, "failed to copy %v to %v", srcFile, destFile)
+			return fmt.Errorf("failed to copy %v to %v: %w", srcFile, destFile, err)
 		}
 	}
 

dev-tools/mage/copy.go

@@ -10,6 +10,7 @@ import (
 	"go/build"
 	"log"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -19,11 +20,13 @@ import (
 
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
+	"gopkg.in/yaml.v3"
 
 	"github.com/magefile/mage/sh"
 	"golang.org/x/tools/go/vcs"
 
 	"github.com/elastic/elastic-agent/dev-tools/mage/gotool"
+	v1 "github.com/elastic/elastic-agent/pkg/api/v1"
 )
 
 const (
@@ -43,7 +46,8 @@ const (
 	agentPackageVersionEnvVar = "AGENT_PACKAGE_VERSION"
 
 	// Mapped functions
-	agentPackageVersionMappedFunc = "agent_package_version"
+	agentPackageVersionMappedFunc    = "agent_package_version"
+	agentManifestGeneratorMappedFunc = "manifest"
 )
 
 // Common settings with defaults derived from files, CWD, and environment.
@@ -91,18 +95,19 @@ var (
 	ManifestURL string
 
 	FuncMap = map[string]interface{}{
-		"beat_doc_branch":             BeatDocBranch,
-		"beat_version":                BeatQualifiedVersion,
-		"commit":                      CommitHash,
-		"commit_short":                CommitHashShort,
-		"date":                        BuildDate,
-		"elastic_beats_dir":           ElasticBeatsDir,
-		"go_version":                  GoVersion,
-		"repo":                        GetProjectRepoInfo,
-		"title":                       func(s string) string { return cases.Title(language.English, cases.NoLower).String(s) },
-		"tolower":                     strings.ToLower,
-		"contains":                    strings.Contains,
-		agentPackageVersionMappedFunc: AgentPackageVersion,
+		"beat_doc_branch":                BeatDocBranch,
+		"beat_version":                   BeatQualifiedVersion,
+		"commit":                         CommitHash,
+		"commit_short":                   CommitHashShort,
+		"date":                           BuildDate,
+		"elastic_beats_dir":              ElasticBeatsDir,
+		"go_version":                     GoVersion,
+		"repo":                           GetProjectRepoInfo,
+		"title":                          func(s string) string { return cases.Title(language.English, cases.NoLower).String(s) },
+		"tolower":                        strings.ToLower,
+		"contains":                       strings.Contains,
+		agentPackageVersionMappedFunc:    AgentPackageVersion,
+		agentManifestGeneratorMappedFunc: PackageManifest,
 	}
 )
 
@@ -296,6 +301,40 @@ func AgentPackageVersion() (string, error) {
 	return BeatQualifiedVersion()
 }
 
+func PackageManifest() (string, error) {
+	m := v1.NewManifest()
+	m.Package.Snapshot = Snapshot
+	packageVersion, err := AgentPackageVersion()
+	if err != nil {
+		return "", fmt.Errorf("retrieving agent package version: %w", err)
+	}
+	m.Package.Version = packageVersion
+	commitHashShort, err := CommitHashShort()
+	if err != nil {
+		return "", fmt.Errorf("retrieving agent commit hash: %w", err)
+	}
+
+	versionedHomePath := path.Join("data", fmt.Sprintf("%s-%s", BeatName, commitHashShort))
+	m.Package.VersionedHome = versionedHomePath
+	m.Package.PathMappings = []map[string]string{{}}
+	m.Package.PathMappings[0][versionedHomePath] = fmt.Sprintf("data/elastic-agent-%s%s-%s", m.Package.Version, SnapshotSuffix(), commitHashShort)
+	m.Package.PathMappings[0]["manifest.yaml"] = fmt.Sprintf("data/elastic-agent-%s%s-%s/manifest.yaml", m.Package.Version, SnapshotSuffix(), commitHashShort)
+	yamlBytes, err := yaml.Marshal(m)
+	if err != nil {
+		return "", fmt.Errorf("marshaling manifest: %w", err)
+
+	}
+	return string(yamlBytes), nil
+}
+
+func SnapshotSuffix() string {
+	if !Snapshot {
+		return ""
+	}
+
+	return "-SNAPSHOT"
+}
+
 var (
 	elasticBeatsDirValue string
 	elasticBeatsDirErr   error