[CI] Integration tests matrix pipeline (#4959)

* [CI] Integration tests matrix pipeline

* [CI] Integration tests matrix pipeline

* Removed pipeline slug for integration tests hooks

* Removed pipeline slug for integration tests hooks

* Commented Windows AMD 64 tests from support matrix due to an issue

* Commented Windows AMD 64 tests from support matrix due to an issue

Author: pazone

.buildkite/hooks/pre-command

@@ -46,7 +46,8 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-package" ]]; then
   fi
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-extended-testing" && "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
+if [[ "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
+  echo "Setting credentials"
   # Set GCP credentials
   export GOOGLE_APPLICATION_GCP_SECRET=$(retry 5 vault kv get -format=json -field=data ${CI_GCP_OBS_PATH})
   echo "${GOOGLE_APPLICATION_GCP_SECRET}" > ./gcp.json

.buildkite/hooks/pre-exit

@@ -2,7 +2,7 @@
 
 set -eo pipefail
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-extended-testing" && "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
+if [[ "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
   if [[ -z "${WORKSPACE-""}" ]]; then
       WORKSPACE=$(git rev-parse --show-toplevel)
   fi

.buildkite/pipeline.integration-test-matrix.yml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  DOCKER_REGISTRY: "docker.elastic.co"
+  VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
+
+steps:
+  - label: "Integration test matrix"
+    key: "integration-tests-matrix"
+    command: ".buildkite/scripts/steps/integration_tests.sh stateful integration:matrix"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"

pkg/testing/runner/supported.go

@@ -137,6 +137,7 @@ var (
 		},
 		Runner: WindowsRunner{},
 	}
+
 	// WindowsAMD64_2016 - Windows (amd64) Server 2016
 	WindowsAMD64_2016 = SupportedOS{
 		OS: define.OS{
@@ -177,8 +178,9 @@ var supported = []SupportedOS{
 	WindowsAMD64_2022_Core,
 	WindowsAMD64_2019,
 	WindowsAMD64_2019_Core,
-	WindowsAMD64_2016,
-	WindowsAMD64_2016_Core,
+	// https://github.com/elastic/ingest-dev/issues/3484
+	// WindowsAMD64_2016,
+	// WindowsAMD64_2016_Core,
 }
 
 // osMatch returns true when the specific OS is a match for a non-specific OS.