[DO NOT MERGE] - PoC for orchestrating remote tests from inside the package using terraform

.rsync.exclude

@@ -0,0 +1,10 @@
+.git
+.ssh
+.integration-cache
+.buildkite
+.agent-testing
+.terraform
+.ogc-cache
+.vagrant
+build
+deploy

go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/elastic/go-licenser v0.4.1
 	github.com/elastic/go-sysinfo v1.14.0
 	github.com/elastic/go-ucfg v0.8.8
-	github.com/fatih/color v1.15.0
+	github.com/fatih/color v1.16.0
 	github.com/fsnotify/fsnotify v1.7.0
 	github.com/gofrs/flock v0.8.1
 	github.com/gofrs/uuid v4.4.0+incompatible
@@ -32,6 +32,8 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/mux v1.8.0
 	github.com/hashicorp/go-multierror v1.1.1
+	github.com/hashicorp/hc-install v0.7.0
+	github.com/hashicorp/terraform-exec v0.21.0
 	github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95
 	github.com/jaypipes/ghw v0.12.0
 	github.com/jedib0t/go-pretty/v6 v6.4.6
@@ -115,14 +117,17 @@ require (
 
 require (
 	github.com/Jeffail/gabs/v2 v2.6.0 // indirect
+	github.com/ProtonMail/go-crypto v1.1.0-alpha.2 // indirect
 	github.com/StackExchange/wmi v1.2.1 // indirect
 	github.com/akavel/rsrc v0.8.0 // indirect
 	github.com/alecthomas/participle/v2 v2.1.1 // indirect
+	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/armon/go-radix v1.0.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect
 	github.com/cavaliercoder/badio v0.0.0-20160213150051-ce5280129e9e // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/distribution/reference v0.5.0 // indirect
@@ -158,9 +163,11 @@ require (
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
+	github.com/hashicorp/terraform-json v0.22.1 // indirect
 	github.com/iancoleman/strcase v0.3.0 // indirect
-	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/influxdata/go-syslog/v3 v3.0.1-0.20230911200830-875f5bc594a4 // indirect
 	github.com/jaypipes/pcidb v1.0.0 // indirect
@@ -177,7 +184,7 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/markbates/pkger v0.17.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
@@ -208,14 +215,15 @@ require (
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/rs/cors v1.10.1 // indirect
 	github.com/santhosh-tekuri/jsonschema v1.2.4 // indirect
-	github.com/sergi/go-diff v1.2.0 // indirect
+	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/valyala/fastjson v1.6.4 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
+	github.com/zclconf/go-cty v1.14.4 // indirect
 	go.elastic.co/apm/module/apmhttp v1.15.0 // indirect
 	go.elastic.co/apm/module/apmhttp/v2 v2.0.0 // indirect
 	go.elastic.co/apm/v2 v2.0.0 // indirect
@@ -276,7 +284,7 @@ require (
 )
 
 require (
-	github.com/hashicorp/go-version v1.6.0 // indirect
+	github.com/hashicorp/go-version v1.6.0
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/onsi/gomega v1.27.3 // indirect
 	go.elastic.co/apm v1.15.0

testing/newexp/foo_test.go

@@ -0,0 +1,44 @@
+//go:build integration
+
+package newexp
+
+import (
+	"io"
+	"net/http"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/elastic/elastic-agent/pkg/testing/define"
+)
+
+func TestDrill(t *testing.T) {
+	// assert that values set in TestMain is available for tests
+	assert.Equal(t, "This is not a drill.", pkgVar)
+	assert.Contains(t, os.Environ(), "TEST_ENV_VAR=This is not a drill.")
+}
+
+func TestDeployment(t *testing.T) {
+	// assert that we get something usable from define directive
+	info := define.Require(t, define.Requirements{
+		Group: "Foo",
+		Stack: &define.Stack{},
+	})
+	assert.NotEmpty(t, info.ESClient, "couldn't instantiate ES client")
+	assert.NotEmpty(t, info.KibanaClient, "couldn't instantiate Kibana client")
+
+	esInfoResp, err := info.ESClient.Info()
+	assert.NoError(t, err, "error fetch ES info")
+	defer esInfoResp.Body.Close()
+	infoBytes, err := io.ReadAll(esInfoResp.Body)
+	assert.NoError(t, err, "error reading ES info response body")
+	t.Logf("ES info:\n%s\n", string(infoBytes))
+
+	kibanaResponse, err := info.KibanaClient.Send(http.MethodGet, "/api/fleet/agents", nil, nil, nil)
+	assert.NoError(t, err, "error pinging Kibana/Fleet")
+	defer kibanaResponse.Body.Close()
+	agentStatuses, err := io.ReadAll(kibanaResponse.Body)
+	assert.NoError(t, err, "error reading Fleet agents response body")
+	t.Logf("Fleet Agents:\n%s\n", string(agentStatuses))
+}

testing/newexp/main_test.go

@@ -0,0 +1,155 @@
+//go:build integration
+
+package newexp
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+// Flags consts
+const (
+	flagPrefix           = "integration."
+	skipDestroyFlag      = "skip-destroy"
+	terraformDirFlag     = "terraform-dir"
+	skipProvisioningFlag = "skip-provisioning"
+)
+
+// Flags globals
+type testOptions struct {
+	skipDestroy      bool
+	skipProvisioning bool
+	terraformWorkDir string
+}
+
+var flagSet = flag.CommandLine
+var testOpts testOptions
+
+// Simple package variable to test
+var pkgVar string
+
+func init() {
+	err := bindTestFlags(flagPrefix, flagSet, &testOpts)
+	if err != nil {
+		panic(fmt.Errorf("initializing command line flags: %w", err))
+	}
+}
+
+func bindTestFlags(prefix string, flagSet *flag.FlagSet, opts *testOptions) error {
+	cwd, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("detecting CWD: %w", err)
+	}
+
+	// flags
+	flagSet.BoolVar(&opts.skipDestroy, prefix+skipDestroyFlag, false, "Set this flag to skip destroying resources")
+	flagSet.BoolVar(&opts.skipProvisioning, prefix+skipProvisioningFlag, false, "Set this flag to run directly the tests by skipping the provisioning")
+	flagSet.StringVar(&opts.terraformWorkDir, prefix+terraformDirFlag, filepath.Join(cwd, "terraform"), "Directory containing terraform files")
+
+	return nil
+}
+
+func TestMain(m *testing.M) {
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Minute)
+	defer cancel()
+	flag.Parse()
+
+	os.Exit(innerRun(ctx, m))
+}
+
+func innerRun(ctx context.Context, m *testing.M) (returnCode int) {
+
+	log.Printf("go test args: %s\n", os.Args)
+
+	if !testOpts.skipProvisioning {
+		// "Remote execution case"
+		defer func() {
+			err := tearDown(ctx)
+			if err != nil {
+				log.Printf("error during teardown: %s", err)
+			}
+		}()
+
+		client, _, essDeployment, err := setup(ctx)
+		if err != nil {
+			log.Printf("error during setup: %s", err)
+			return 1
+		}
+
+		cmdLine := buildRemoteTestCommand(essDeployment)
+
+		log.Printf("full command to run on remote host: %q", cmdLine)
+
+		session, err := client.NewSession()
+		if err != nil {
+			log.Printf("initiating ssh session: %s", err)
+			return 1
+		}
+		defer session.Close()
+		output, err := session.CombinedOutput(cmdLine)
+		if err != nil {
+			log.Printf("error running tests on remote machine: %s", err)
+			returnCode = 1
+		}
+		log.Printf("Test run output:\n%s\n", string(output))
+	} else {
+		// Local execution case
+
+		// SMALL setup for the test (this would need to be performed where the test runs)
+		pkgVar = "This is not a drill."
+		os.Setenv("TEST_ENV_VAR", "This is not a drill.")
+
+		return m.Run()
+	}
+
+	return returnCode
+}
+
+func buildRemoteTestCommand(deployment *ESSDeployment) string {
+	sb := new(strings.Builder)
+
+	sb.WriteString("cd /src/elastic-agent && ")
+
+	// FIXME Hack to have define directives work at runtime
+	sb.WriteString("TEST_DEFINE_PREFIX=aaaaaa ")
+	sb.WriteString(" ELASTICSEARCH_HOST=")
+	sb.WriteString(deployment.ElasticsearchHost)
+	sb.WriteString(" KIBANA_HOST=")
+	sb.WriteString(deployment.KibanaHost)
+	sb.WriteString(" ELASTICSEARCH_USERNAME=")
+	sb.WriteString(deployment.ESUser)
+	sb.WriteString(" ELASTICSEARCH_PASSWORD=")
+	sb.WriteString(deployment.ESPassword)
+	sb.WriteString(" KIBANA_USERNAME=")
+	sb.WriteString(deployment.ESUser)
+	sb.WriteString(" KIBANA_PASSWORD=")
+	sb.WriteString(deployment.ESPassword)
+
+	// Start with the test command
+	sb.WriteString(" go test ")
+
+	// HACK to run the correct package
+	sb.WriteString(" github.com/elastic/elastic-agent/testing/newexp ")
+	sb.WriteString("-tags integration")
+	for _, arg := range os.Args[1:] {
+		if strings.HasPrefix(arg, "-"+flagPrefix) {
+			// that's a flag for this test main, skip it
+			continue
+		}
+		sb.WriteString(" ")
+		sb.WriteString(arg)
+	}
+
+	// Add the "no-provision" switch for the remote run
+	sb.WriteString(" -args ")
+	sb.WriteString("-" + flagPrefix + skipProvisioningFlag)
+
+	return sb.String()
+}

testing/newexp/terraform/.gitignore

@@ -0,0 +1,4 @@
+.ssh/
+.terraform.lock.hcl
+*.tfstate
+*.tfplan

testing/newexp/terraform/ess.tf

@@ -0,0 +1,50 @@
+locals {
+  # FIXME
+  deployment_name    = join("-", ["elastic-agent-integration-tests", "aaaaa"])
+  # FIXME
+  deployment_version = "8.15.0-SNAPSHOT"
+
+  ess_region             = coalesce(var.ess_region, "gcp-us-west2")
+  deployment_template_id = coalesce(var.deployment_template_id, "gcp-cpu-optimized-v7")
+}
+
+
+
+provider "ec" {
+  # FIXME
+  apikey = file("~/.config/ess/api_key.txt")
+}
+
+resource "ec_deployment" "integration-testing" {
+  name                   = local.deployment_name
+  alias                  = local.deployment_name
+  region                 = local.ess_region
+  deployment_template_id = local.deployment_template_id
+  version                = local.deployment_version
+
+  elasticsearch = {
+    autoscale = false
+
+    hot = {
+      autoscaling = {}
+      size        = "8g"
+      zone_count  = 1
+    }
+  }
+  kibana = {
+    size       = "1g"
+    zone_count = 1
+  }
+
+  integrations_server = {
+    size = "1g"
+    zone_count = 1
+  }
+
+  tags = {
+    "provisioner"    = "elastic-agent-integration-test"
+    #"environment_id" = var.environment_id
+    #"creator"        = var.creator
+    "buildkite_id"   = var.build_id
+  }
+}