Allow installing a second agent on the same machine for development

README.md

@@ -3,21 +3,63 @@
 [![Build status](https://badge.buildkite.com/1d35bb40427cc6833979645b61ea214fc4b686a2ffe3a68bdf.svg)](https://buildkite.com/elastic/elastic-agent)
 [![Coverage](https://sonarcloud.io/api/project_badges/measure?project=elastic_elastic-agent&metric=coverage)](https://sonarcloud.io/summary/new_code?id=elastic_elastic-agent)
 
-## Architecture / internal docs
+## Architecture and Internals
 
 - [Agent architecture](docs/architecture.md)
 - [Component spec files](docs/component-specs.md)
 - [Policy configuration](docs/agent-policy.md)
 
+## Official Documentation
+
+See https://www.elastic.co/guide/en/fleet/current/index.html.
+
+The source files for the offical Elastic Agent documentation are currently stored
+in the [ingest-docs](https://github.com/elastic/ingest-docs/tree/main/docs/en/ingest-management) repository.
+
 ## Contributing
 
 See [CONTRIBUTING.md](CONTRIBUTING.md).
 
-## Developer docs
+## Developing
 
-The source files for the general Elastic Agent documentation are currently stored
-in the [ingest-docs](https://github.com/elastic/ingest-docs/tree/main/docs/en/ingest-management) repository.
-The following docs are only focused on getting developers started building code for Elastic Agent.
+The following are exclusively focused on getting developers started building code for Elastic Agent.
+
+### Development Installations
+
+> :warning: Development installations are not officially supported and are intended for Elastic Agent developers.
+
+If you are an Elastic employee, you already have an Information Security managed Elastic Agent installed on your machine for endpoint protection.
+This prevents you from installing the Elastic Agent a second time for development without using a VM or Docker container. To eliminate this point
+of friction, Elastic Agent has a development mode that permits installing the Elastic Agent on your machine a second time:
+
+```sh
+# All other arguments to the install command are still supported when --develop is specified.
+sudo ./elastic-agent install --develop
+# The run command also supports the --develop option to allow running without installing when there is another agent on the machine.
+./elastic-agent run -e --develop
+```
+
+Using the `--develop` option will install the agent in an isolated `Agent-Development` agent directory in the chosen base path.
+Development agents enrolled in Fleet will have the `Development` tag added automatically. Using the default base path on MacOS you will see:
+
+```sh
+sudo ls /Library/Elastic/
+Agent
+Agent-Development
+```
+
+The `elastic-agent` command in the shell is replaced with `elastic-development-agent` to interact with the development agent:
+
+```sh
+# For a privileged agent
+sudo elastic-development-agent status
+# For an unprivileged agent
+sudo -u elastic-agent-user elastic-development-agent status
+```
+
+The primary restriction of `--develop` installations is that you cannot run Elastic Defend a second time on the same machine. Attempting to
+install Defend twice will fail with resource conflicts. All other integrations should be usable provided conflicting configurations are
+changed ahead of time. For example two agents cannot bind to the same `agent.monitoring.http.port` to expose their monitoring servers.
 
 ### Test Framework
 

internal/pkg/agent/application/paths/common.go

@@ -321,11 +321,6 @@ func BinaryPath(baseDir, agentName string) string {
 	return filepath.Join(binaryDir(baseDir), agentName)
 }
 
-// InstallPath returns the top level directory Agent will be installed into.
-func InstallPath(basePath string) string {
-	return filepath.Join(basePath, "Elastic", "Agent")
-}
-
 // TopBinaryPath returns the path to the Elastic Agent binary that is inside the Top directory.
 //
 // This always points to the symlink that points to the latest Elastic Agent version.

internal/pkg/agent/application/paths/common_namespace.go

@@ -0,0 +1,130 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+// This file encapsulates the common paths that need to account for installation namepsaces.
+// Installation namespaces allow multiple agents on the same machine.
+package paths
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	// installDirNamespaceFmt is the format of the directory agent will be installed to within the base path when using an installation namepsace.
+	// It is $BasePath/Agent-$namespace.
+	installDir                = "Agent"
+	installDirNamespaceSep    = "-"
+	installDirNamespacePrefix = installDir + installDirNamespaceSep
+	installDirNamespaceFmt    = installDirNamespacePrefix + "%s"
+
+	// DevelopmentNamespace defines the "well known" development namespace.
+	DevelopmentNamespace = "Development"
+
+	// Service display names. Must be different from the ServiceName() on Windows.
+	serviceDisplayName             = "Elastic Agent"
+	serviceDisplayNameNamespaceFmt = "Elastic Agent - %s"
+)
+
+// installNamespace is the name of the agent's current installation namepsace.
+var installNamespace string
+
+// SetInstallNamespace sets whether the agent is currently in or is being installed in an installation namespace.
+// Removes leading and trailing whitespace
+func SetInstallNamespace(namespace string) {
+	installNamespace = strings.TrimSpace(namespace)
+}
+
+// InstallNamespace returns the name of the current installation namespace. Returns the empty string
+// for the default namespace. For installed agents, the namespace is parsed from the installation
+// directory name, since a unique directory name is required to avoid collisions between installed
+// agents in the same base path. Before installation, the installation namespace must be configured
+// using SetInstallNamespace().
+func InstallNamespace() string {
+	if installNamespace != "" {
+		return installNamespace
+	}
+
+	if RunningInstalled() {
+		// Parse the namespace from the directory once to ensure deterministic behavior from startup.
+		namespace := parseNamespaceFromDir(filepath.Base(Top()))
+		installNamespace = namespace
+	}
+
+	return ""
+}
+
+func parseNamespaceFromDir(dir string) string {
+	parts := strings.SplitAfterN(dir, "-", 2)
+	if len(parts) <= 1 {
+		return ""
+	} else if parts[0] != installDirNamespacePrefix {
+		return ""
+	}
+
+	return parts[1]
+}
+
+// InInstallNamespace returns true if the agent is being installed in an installation namespace.
+func InInstallNamespace() bool {
+	return InstallNamespace() != ""
+}
+
+// InstallDirNameForNamespace returns the installation directory name for a given namespace.
+// The installation directory name with a namespace is $BasePath/InstallDirNameForNamespace().
+func InstallDirNameForNamespace(namespace string) string {
+	if namespace == "" {
+		return installDir
+	}
+
+	return fmt.Sprintf(installDirNamespaceFmt, namespace)
+}
+
+// InstallPath returns the top level directory Agent will be installed into, accounting for any namespace.
+func InstallPath(basePath string) string {
+	namespace := InstallNamespace()
+	return filepath.Join(basePath, "Elastic", InstallDirNameForNamespace(namespace))
+}
+
+// ServiceName returns the service name accounting for any namespace.
+func ServiceName() string {
+	namespace := InstallNamespace()
+	if namespace == "" {
+		return serviceName
+	}
+
+	return fmt.Sprintf(serviceNameNamespaceFmt, namespace)
+}
+
+// ServiceDisplayName returns the service display name accounting for any namespace.
+func ServiceDisplayName() string {
+	namespace := InstallNamespace()
+	if namespace == "" {
+		return serviceDisplayName
+	}
+
+	return fmt.Sprintf(serviceDisplayNameNamespaceFmt, namespace)
+}
+
+// ShellWrapperPath returns the shell wrapper path accounting for any namespace.
+// The provided namespace is always lowercased for consistency.
+func ShellWrapperPath() string {
+	namespace := InstallNamespace()
+	if namespace == "" {
+		return shellWrapperPath
+	}
+
+	return ShellWrapperPathForNamespace(namespace)
+}
+
+// ControlSocketRunSymlink returns the shell wrapper path accounting for any namespace.
+// Does not auto detect the namespace because it is used outside of agent itself in the testing framework.
+func ControlSocketRunSymlink(namespace string) string {
+	if namespace == "" {
+		return controlSocketRunSymlink
+	}
+
+	return controlSocketRunSymlinkForNamespace(namespace)
+}

internal/pkg/agent/application/paths/common_namespace_test.go

@@ -0,0 +1,76 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package paths
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestInstallNamespace(t *testing.T) {
+	namespace := "testing"
+	basePath := filepath.Join("base", "path")
+
+	// Add whitespace to ensure it gets removed.
+	SetInstallNamespace(" " + namespace + "\t   ")
+
+	assert.Equal(t, namespace, InstallNamespace())
+	assert.True(t, InInstallNamespace())
+	assert.Equal(t, filepath.Join(basePath, "Elastic", fmt.Sprintf(installDirNamespaceFmt, namespace)), InstallPath(basePath))
+	assert.Equal(t, fmt.Sprintf(serviceNameNamespaceFmt, namespace), ServiceName())
+	assert.Equal(t, fmt.Sprintf(serviceDisplayNameNamespaceFmt, namespace), ServiceDisplayName())
+	assert.Equal(t, ShellWrapperPathForNamespace(namespace), ShellWrapperPath())
+	assert.Equal(t, controlSocketRunSymlinkForNamespace(namespace), ControlSocketRunSymlink(namespace))
+}
+
+func TestInstallNoNamespace(t *testing.T) {
+	namespace := ""
+	basePath := filepath.Join("base", "path")
+	SetInstallNamespace(namespace)
+
+	assert.Equal(t, namespace, InstallNamespace())
+	assert.False(t, InInstallNamespace())
+	assert.Equal(t, filepath.Join(basePath, "Elastic", installDir), InstallPath(basePath))
+	assert.Equal(t, serviceName, ServiceName())
+	assert.Equal(t, serviceDisplayName, ServiceDisplayName())
+	assert.Equal(t, shellWrapperPath, ShellWrapperPath())
+	assert.Equal(t, controlSocketRunSymlink, ControlSocketRunSymlink(namespace))
+}
+
+func TestParseNamespaceFromDirName(t *testing.T) {
+	testcases := []struct {
+		name      string
+		dir       string
+		namespace string
+	}{
+		{name: "empty", dir: "", namespace: ""},
+		{name: "none", dir: "Agent", namespace: ""},
+		{name: "develop", dir: "Agent-Development", namespace: "Development"},
+		{name: "dashes", dir: "Agent-With-Dashes", namespace: "With-Dashes"},
+		{name: "special", dir: "Agent-@!$%^&*()-_+=", namespace: "@!$%^&*()-_+="},
+		{name: "format", dir: "Agent-%s%d%v%t", namespace: "%s%d%v%t"},
+		{name: "spaces", dir: "Agent- Development \t", namespace: " Development \t"},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equalf(t, tc.namespace, parseNamespaceFromDir(tc.dir), "parsing %s", tc.dir)
+
+			// Special case: if the directory is empty the install dir is the default "Agent" not "Agent-"
+			wantDir := tc.dir
+			if wantDir == "" {
+				wantDir = installDir
+			}
+			assert.Equal(t, wantDir, InstallDirNameForNamespace(tc.namespace))
+		})
+	}
+}
+
+func TestParseNamespaceFromDirNameWithoutAgentPrefix(t *testing.T) {
+	assert.Equal(t, "", parseNamespaceFromDir("Beats-Development"))
+}

internal/pkg/agent/application/paths/paths_darwin.go

@@ -14,19 +14,22 @@ const (
 	// for installing Elastic Agent's files.
 	DefaultBasePath = "/Library"
 
-	// ControlSocketRunSymlink is the path to the symlink that should be
+	// controlSocketRunSymlink is the path to the symlink that should be
 	// created to the control socket when Elastic Agent is running with root.
-	ControlSocketRunSymlink = "/var/run/elastic-agent.sock"
+	controlSocketRunSymlink             = "/var/run/elastic-agent.sock"
+	controlSocketRunSymlinkNamespaceFmt = "/var/run/elastic-agent-%s.sock"
 
-	// ServiceName is the service name when installed.
-	ServiceName = "co.elastic.elastic-agent"
+	// serviceName is the service name when installed.
+	serviceName             = "co.elastic.elastic-agent"
+	serviceNameNamespaceFmt = "co.elastic.elastic-agent-%s"
 
-	// ShellWrapperPath is the path to the installed shell wrapper.
-	ShellWrapperPath = "/usr/local/bin/elastic-agent"
+	// shellWrapperPath is the path to the installed shell wrapper.
+	shellWrapperPath             = "/usr/local/bin/elastic-agent"
+	shellWrapperPathNamespaceFmt = "/usr/local/bin/elastic-%s-agent"
 
 	// ShellWrapper is the wrapper that is installed.  The %s must
 	// be substituted with the appropriate top path.
-	ShellWrapper = `#!/bin/sh
+	ShellWrapperFmt = `#!/bin/sh
 exec %s/elastic-agent $@
 `
 )

internal/pkg/agent/application/paths/paths_linux.go

@@ -14,21 +14,24 @@ const (
 	// for installing Elastic Agent's files.
 	DefaultBasePath = "/opt"
 
-	// ServiceName is the service name when installed.
-	ServiceName = "elastic-agent"
+	// serviceName is the service name when installed.
+	serviceName             = "elastic-agent"
+	serviceNameNamespaceFmt = "elastic-agent-%s"
 
-	// ShellWrapperPath is the path to the installed shell wrapper.
-	ShellWrapperPath = "/usr/bin/elastic-agent"
+	// shellWrapperPath is the path to the installed shell wrapper.
+	shellWrapperPath             = "/usr/bin/elastic-agent"
+	shellWrapperPathNamespaceFmt = "/usr/bin/elastic-%s-agent"
 
 	// ShellWrapper is the wrapper that is installed.  The %s must
 	// be substituted with the appropriate top path.
-	ShellWrapper = `#!/bin/sh
+	ShellWrapperFmt = `#!/bin/sh
 exec %s/elastic-agent $@
 `
 
-	// ControlSocketRunSymlink is the path to the symlink that should be
+	// controlSocketRunSymlink is the path to the symlink that should be
 	// created to the control socket when Elastic Agent is running with root.
-	ControlSocketRunSymlink = "/run/elastic-agent.sock"
+	controlSocketRunSymlink             = "/run/elastic-agent.sock"
+	controlSocketRunSymlinkNamespaceFmt = "/run/elastic-agent-%s.sock"
 )
 
 // ArePathsEqual determines whether paths are equal taking case sensitivity of os into account.

internal/pkg/agent/application/paths/paths_unix.go

@@ -7,10 +7,27 @@
 package paths
 
 import (
+	"fmt"
 	"path/filepath"
 	"runtime"
+	"strings"
 )
 
+const ()
+
+// shellWrapperPathForNamespace is a helper to work around not being able to use fmt.Sprintf
+// unconditionally since shellWrapperPathNamespaceFmt is empty on Windows. The provided namespace is
+// always lowercased for consistency.
+func ShellWrapperPathForNamespace(namespace string) string {
+	return fmt.Sprintf(shellWrapperPathNamespaceFmt, strings.ToLower(namespace))
+}
+
+// controlSocketRunSymlinkForNamespace is a helper to work around not being able to use fmt.Sprintf
+// unconditionally since controlSocketRunSymlinkNamespaceFmt is empty on Windows.
+func controlSocketRunSymlinkForNamespace(namespace string) string {
+	return fmt.Sprintf(controlSocketRunSymlinkNamespaceFmt, namespace)
+}
+
 func initialControlSocketPath(topPath string) string {
 	return ControlSocketFromPath(runtime.GOOS, topPath)
 }