Enable unprivileged on macos (#4362)

* create vault as unprivileged when running as non-root

* Fix AgentInfo creation

* Allow install --unprivileged on darwin

* Add --unprivileged flag to uninstall

* Move the check for unprivileged in EncryptedDiskStore defaults

Checking if we are running as root when creating the EncryptedDiskStore
removes the need for propagating an unprivileged flag from a lot of
places in the code.
Only exception is the uninstall command, since it needs to run with
administrator privileges: in there we check for existence of a FileVault
containing the agent secret to detect if we have to load the agent
configuration as unprivileged (currently relevant only on Darwin)

* Deprecate '--disable-encrypted-store' switch on elastic-agent run

* add unit test for uninstall unprivileged check

* fixup! add unit test for uninstall unprivileged check

* propagate errors when creating encrypted disk store

* fixup! propagate errors when creating encrypted disk store

* changelog

* fixup! fixup! propagate errors when creating encrypted disk store

* Fix order of overrides in EncryptedDiskStore

* fixup! Fix order of overrides in EncryptedDiskStore

* fixup! fixup! Fix order of overrides in EncryptedDiskStore

* fixup! fixup! fixup! Fix order of overrides in EncryptedDiskStore

* fixup! propagate errors when creating encrypted disk store

* fix log and error strings

Author: pchila

changelog/fragments/1710179479-Enable---unprivileged-on-Mac-OS.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: Enable --unprivileged on Mac OS, allowing elastic-agent to run as an unprivileged user
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; a word indicating the component this changeset affects.
+component:
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+#pr: https://github.com/owner/repo/1234
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

internal/pkg/agent/application/application.go

@@ -211,7 +211,10 @@ func New(
 
 func mergeFleetConfig(ctx context.Context, rawConfig *config.Config) (storage.Store, *configuration.Configuration, error) {
 	path := paths.AgentConfigFile()
-	store := storage.NewEncryptedDiskStore(ctx, path)
+	store, err := storage.NewEncryptedDiskStore(ctx, path)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error instantiating encrypted disk store: %w", err)
+	}
 
 	reader, err := store.Load()
 	if err != nil {

internal/pkg/agent/application/gateway/fleet/fleet_gateway_test.go

@@ -502,7 +502,8 @@ func newStateStore(t *testing.T, log *logger.Logger) *store.StateStore {
 	require.NoError(t, err)
 
 	filename := filepath.Join(dir, "state.enc")
-	diskStore := storage.NewDiskStore(filename)
+	diskStore, err := storage.NewDiskStore(filename)
+	require.NoError(t, err)
 	stateStore, err := store.NewStateStore(log, diskStore)
 	require.NoError(t, err)
 

internal/pkg/agent/application/info/agent_id.go

@@ -53,7 +53,10 @@ func updateLogLevel(ctx context.Context, level string) error {
 	}
 
 	agentConfigFile := paths.AgentConfigFile()
-	diskStore := storage.NewEncryptedDiskStore(ctx, agentConfigFile)
+	diskStore, err := storage.NewEncryptedDiskStore(ctx, agentConfigFile)
+	if err != nil {
+		return fmt.Errorf("error instantiating encrypted disk store: %w", err)
+	}
 
 	ai.LogLevel = level
 	return updateAgentInfo(diskStore, ai)
@@ -202,7 +205,10 @@ func loadAgentInfo(ctx context.Context, forceUpdate bool, logLevel string, creat
 	defer idLock.Unlock()
 
 	agentConfigFile := paths.AgentConfigFile()
-	diskStore := storage.NewEncryptedDiskStore(ctx, agentConfigFile)
+	diskStore, err := storage.NewEncryptedDiskStore(ctx, agentConfigFile)
+	if err != nil {
+		return nil, fmt.Errorf("error instantiating encrypted disk store: %w", err)
+	}
 
 	agentInfo, err := getInfoFromStore(diskStore, logLevel)
 	if err != nil {

internal/pkg/agent/application/secret/secret.go

@@ -16,7 +16,7 @@ import (
 	"github.com/elastic/elastic-agent/internal/pkg/agent/vault/aesgcm"
 )
 
-const agentSecretKey = "secret"
+const AgentSecretKey = "secret"
 
 // mutex for secret create calls
 var mxCreate sync.Mutex
@@ -29,7 +29,7 @@ type Secret struct {
 
 // CreateAgentSecret creates agent secret key if it doesn't exist
 func CreateAgentSecret(ctx context.Context, opts ...vault.OptionFunc) error {
-	return Create(ctx, agentSecretKey, opts...)
+	return Create(ctx, AgentSecretKey, opts...)
 }
 
 // Create creates secret and stores it in the vault under given key
@@ -69,13 +69,13 @@ func Create(ctx context.Context, key string, opts ...vault.OptionFunc) error {
 
 // GetAgentSecret read the agent secret from the vault
 func GetAgentSecret(ctx context.Context, opts ...vault.OptionFunc) (secret Secret, err error) {
-	return Get(ctx, agentSecretKey, opts...)
+	return Get(ctx, AgentSecretKey, opts...)
 }
 
 // SetAgentSecret saves the agent secret from the vault
 // This is needed for migration from 8.3.0-8.3.2 to higher versions
 func SetAgentSecret(ctx context.Context, secret Secret, opts ...vault.OptionFunc) error {
-	return Set(ctx, agentSecretKey, secret, opts...)
+	return Set(ctx, AgentSecretKey, secret, opts...)
 }
 
 // Get reads the secret key from the vault

internal/pkg/agent/cmd/enroll_cmd.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/elastic/elastic-agent-libs/transport/httpcommon"
 	"github.com/elastic/elastic-agent-libs/transport/tlscommon"
+
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/filelock"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/info"
@@ -29,6 +30,7 @@ import (
 	"github.com/elastic/elastic-agent/internal/pkg/agent/errors"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/install"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/storage"
+	"github.com/elastic/elastic-agent/internal/pkg/agent/vault"
 	"github.com/elastic/elastic-agent/internal/pkg/cli"
 	"github.com/elastic/elastic-agent/internal/pkg/config"
 	"github.com/elastic/elastic-agent/internal/pkg/core/authority"
@@ -173,10 +175,14 @@ func newEnrollCmd(
 	configPath string,
 ) (*enrollCmd, error) {
 
+	encryptedDiskStore, err := storage.NewEncryptedDiskStore(ctx, paths.AgentConfigFile())
+	if err != nil {
+		return nil, fmt.Errorf("error instantiating encrypted disk store: %w", err)
+	}
 	store := storage.NewReplaceOnSuccessStore(
 		configPath,
 		application.DefaultAgentFleetConfig,
-		storage.NewEncryptedDiskStore(ctx, paths.AgentConfigFile()),
+		encryptedDiskStore,
 	)
 
 	return newEnrollCmdWithStore(
@@ -214,9 +220,14 @@ func (c *enrollCmd) Execute(ctx context.Context, streams *cli.IOStreams) error {
 		span.End()
 	}()
 
+	hasRoot, err := utils.HasRoot()
+	if err != nil {
+		return fmt.Errorf("checking if running with root/Administrator privileges: %w", err)
+	}
+
 	// Create encryption key from the agent before touching configuration
 	if !c.options.SkipCreateSecret {
-		err = secret.CreateAgentSecret(ctx)
+		err = secret.CreateAgentSecret(ctx, vault.WithUnprivileged(!hasRoot))
 		if err != nil {
 			return err
 		}

internal/pkg/agent/cmd/inspect.go

@@ -30,6 +30,7 @@ import (
 	"github.com/elastic/elastic-agent/internal/pkg/config/operations"
 	"github.com/elastic/elastic-agent/pkg/component"
 	"github.com/elastic/elastic-agent/pkg/core/logger"
+	"github.com/elastic/elastic-agent/pkg/utils"
 )
 
 func newInspectCommandWithArgs(s []string, streams *cli.IOStreams) *cobra.Command {
@@ -107,6 +108,7 @@ variables for the configuration.
 
 			ctx, cancel := context.WithCancel(context.Background())
 			service.HandleSignals(func() {}, cancel)
+
 			if err := inspectComponents(ctx, paths.ConfigFile(), opts, streams); err != nil {
 				fmt.Fprintf(streams.Err, "Error: %v\n%s\n", err, troubleshootMessage())
 				os.Exit(1)
@@ -133,8 +135,12 @@ func inspectConfig(ctx context.Context, cfgPath string, opts inspectConfigOpts,
 		return fmt.Errorf("error creating logger: %w", err)
 	}
 
+	isAdmin, err := utils.HasRoot()
+	if err != nil {
+		return fmt.Errorf("error checking for root/Administrator privileges: %w", err)
+	}
 	if !opts.variables && !opts.includeMonitoring {
-		fullCfg, err := operations.LoadFullAgentConfig(ctx, l, cfgPath, true)
+		fullCfg, err := operations.LoadFullAgentConfig(ctx, l, cfgPath, true, !isAdmin)
 		if err != nil {
 			return fmt.Errorf("error loading agent config: %w", err)
 		}
@@ -144,7 +150,7 @@ func inspectConfig(ctx context.Context, cfgPath string, opts inspectConfigOpts,
 		}
 	}
 
-	cfg, lvl, err := getConfigWithVariables(ctx, l, cfgPath, opts.variablesWait)
+	cfg, lvl, err := getConfigWithVariables(ctx, l, cfgPath, opts.variablesWait, !isAdmin)
 	if err != nil {
 		return fmt.Errorf("error fetching config with variables: %w", err)
 	}
@@ -257,7 +263,12 @@ func inspectComponents(ctx context.Context, cfgPath string, opts inspectComponen
 		return fmt.Errorf("failed to detect inputs and outputs: %w", err)
 	}
 
-	m, lvl, err := getConfigWithVariables(ctx, l, cfgPath, opts.variablesWait)
+	isAdmin, err := utils.HasRoot()
+	if err != nil {
+		return fmt.Errorf("error checking for root/Administrator privileges: %w", err)
+	}
+
+	m, lvl, err := getConfigWithVariables(ctx, l, cfgPath, opts.variablesWait, !isAdmin)
 	if err != nil {
 		return err
 	}
@@ -358,9 +369,9 @@ func getMonitoringFn(ctx context.Context, cfg map[string]interface{}) (component
 	return monitor.MonitoringConfig, nil
 }
 
-func getConfigWithVariables(ctx context.Context, l *logger.Logger, cfgPath string, timeout time.Duration) (map[string]interface{}, logp.Level, error) {
+func getConfigWithVariables(ctx context.Context, l *logger.Logger, cfgPath string, timeout time.Duration, unprivileged bool) (map[string]interface{}, logp.Level, error) {
 
-	cfg, err := operations.LoadFullAgentConfig(ctx, l, cfgPath, true)
+	cfg, err := operations.LoadFullAgentConfig(ctx, l, cfgPath, true, unprivileged)
 	if err != nil {
 		return nil, logp.InfoLevel, err
 	}

internal/pkg/agent/cmd/install.go

@@ -70,16 +70,16 @@ func installCmd(streams *cli.IOStreams, cmd *cobra.Command) error {
 
 	isAdmin, err := utils.HasRoot()
 	if err != nil {
-		return fmt.Errorf("unable to perform install command while checking for administrator rights, %w", err)
+		return fmt.Errorf("unable to perform install command while checking for root/Administrator rights: %w", err)
 	}
 	if !isAdmin {
 		return fmt.Errorf("unable to perform install command, not executed with %s permissions", utils.PermissionUser)
 	}
 
-	// only support Linux at the moment
+	// only support Linux and MacOS at the moment
 	unprivileged, _ := cmd.Flags().GetBool(flagInstallUnprivileged)
-	if unprivileged && runtime.GOOS != "linux" {
-		return fmt.Errorf("unable to perform install command, unprivileged is currently only supported on Linux")
+	if unprivileged && (runtime.GOOS != "linux" && runtime.GOOS != "darwin") {
+		return fmt.Errorf("unable to perform install command, unprivileged is currently only supported on Linux and MacOSß")
 	}
 	if unprivileged {
 		fmt.Fprintln(streams.Out, "Unprivileged installation mode enabled; this is an experimental and currently unsupported feature.")

internal/pkg/agent/cmd/run.go

@@ -26,6 +26,7 @@ import (
 	monitoringLib "github.com/elastic/elastic-agent-libs/monitoring"
 	"github.com/elastic/elastic-agent-libs/service"
 	"github.com/elastic/elastic-agent-system-metrics/report"
+	"github.com/elastic/elastic-agent/internal/pkg/agent/vault"
 
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/coordinator"
@@ -88,7 +89,11 @@ func newRunCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command {
 	// feature of the Elastic Agent. On Mac OS root privileges are required to perform the disk
 	// store encryption, by setting this flag it disables that feature and allows the Elastic Agent to
 	// run as non-root.
+	//
+	// Deprecated: MacOS can be run/installed without root privileges
 	cmd.Flags().Bool("disable-encrypted-store", false, "Disable the encrypted disk storage (Only useful on Mac OS)")
+	_ = cmd.Flags().MarkHidden("disable-encrypted-store")
+	_ = cmd.Flags().MarkDeprecated("disable-encrypted-store", "agent on Mac OS can be run/installed without root privileges, see elastic-agent install --help")
 
 	// --testing-mode is a hidden flag that spawns the Elastic Agent in testing mode
 	// it is hidden because we really don't want users to execute Elastic Agent to run
@@ -162,6 +167,12 @@ func runElasticAgent(ctx context.Context, cancel context.CancelFunc, override cf
 
 	l.Infow("Elastic Agent started", "process.pid", os.Getpid(), "agent.version", version.GetAgentPackageVersion())
 
+	// try early to check if running as root
+	isRoot, err := utils.HasRoot()
+	if err != nil {
+		return fmt.Errorf("failed to check for root/Administrator privileges: %w", err)
+	}
+
 	cfg, err = tryDelayEnroll(ctx, l, cfg, override)
 	if err != nil {
 		err = errors.New(err, "failed to perform delayed enrollment")
@@ -170,12 +181,6 @@ func runElasticAgent(ctx context.Context, cancel context.CancelFunc, override cf
 	}
 	pathConfigFile := paths.AgentConfigFile()
 
-	// try early to check if running as root
-	isRoot, err := utils.HasRoot()
-	if err != nil {
-		return fmt.Errorf("failed to check for root permissions: %w", err)
-	}
-
 	// agent ID needs to stay empty in bootstrap mode
 	createAgentID := !runAsOtel
 	if cfg.Fleet != nil && cfg.Fleet.Server != nil && cfg.Fleet.Server.Bootstrap {
@@ -186,7 +191,7 @@ func runElasticAgent(ctx context.Context, cancel context.CancelFunc, override cf
 	// The secret is not created here if it exists already from the previous enrollment.
 	// This is needed for compatibility with agent running in standalone mode,
 	// that writes the agentID into fleet.enc (encrypted fleet.yml) before even loading the configuration.
-	err = secret.CreateAgentSecret(ctx)
+	err = secret.CreateAgentSecret(ctx, vault.WithUnprivileged(!isRoot))
 	if err != nil {
 		return fmt.Errorf("failed to read/write secrets: %w", err)
 	}
@@ -434,7 +439,10 @@ func getOverwrites(ctx context.Context, rawConfig *config.Config) error {
 		return nil
 	}
 	path := paths.AgentConfigFile()
-	store := storage.NewEncryptedDiskStore(ctx, path)
+	store, err := storage.NewEncryptedDiskStore(ctx, path)
+	if err != nil {
+		return fmt.Errorf("error instantiating encrypted disk store: %w", err)
+	}
 
 	reader, err := store.Load()
 	if err != nil && errors.Is(err, os.ErrNotExist) {

internal/pkg/agent/cmd/uninstall.go

@@ -11,6 +11,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/elastic/elastic-agent-libs/logp"
+
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/install"
 	"github.com/elastic/elastic-agent/internal/pkg/cli"