[8.14](backport #4906) [CI] Monorepo diff plugin for integration tests (#4922)

* [CI] Monorepo diff plugin for integration tests (#4906)

* [CI] Monorepo diff plugin for integration tests

* [CI] Monorepo diff plugin for integration tests

* [CI] Monorepo diff plugin for integration tests

* [CI] Monorepo diff plugin for integration tests

* Debug

* Debug

* Debug

* Debug

* Debug

* Debug

* Debug

* Debug

* Debug

* Debug

* Cleanup

* Cleanup

* added missing watch paths

(cherry picked from commit fd8d1ee)

# Conflicts:
#	.buildkite/pull-requests.json

* Fixed conflicts

* Fixed conflicts

---------

Co-authored-by: Pavel Zorin <pavel.zorin@elastic.co>

Author: mergify[bot]

.buildkite/pipeline.yml

@@ -232,3 +232,51 @@ steps:
                   image: "family/core-ubuntu-2204"
                 env:
                   - GH_VERSION=2.4.0
+
+  # Trigger for pull requests
+  - label: "Trigger Integration tests for Pull request"
+    if: |
+      (build.pull_request.id != null && !build.env("GITHUB_PR_LABELS") =~ /skip-it/) ||
+      build.env("GITHUB_PR_TRIGGER_COMMENT") =~ /.*extended.*/
+
+    plugins:
+      - monorepo-diff#v1.0.1:
+          diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
+          watch:
+            - path:
+                - internal/
+                - dev-tools/
+                - pkg/
+                - testing/
+                - version/
+                - specs/
+                - .agent-versions.json
+                - .go-version
+                - .package-version
+                - go.mod
+                - go.sum
+                - magefile.go
+                - main.go
+
+                - .buildkite/integration.pipeline.yml
+                - .buildkite/pipeline.yml
+                - .buildkite/scripts/
+                - .buildkite/hooks/
+
+              config:
+                trigger: "elastic-agent-extended-testing"
+                build:
+                  commit: "${BUILDKITE_COMMIT}"
+                  branch: "${BUILDKITE_BRANCH}"
+                  env:
+                    - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                    - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+                    - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
+  # Trigger for branches
+  - label: "Triggering Integration tests for branches"
+    if: build.pull_request.id == null
+    trigger: "elastic-agent-extended-testing"
+    build:
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"

.buildkite/pull-requests.json

@@ -9,11 +9,11 @@
             "set_commit_status": true,
             "build_on_commit": true,
             "build_on_comment": true,
-            "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test$",
-            "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test$",
-            "skip_ci_labels": [  ],
+            "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it|extended))|^/test\\W*(?:extended|)",
+            "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it|extended))|^/test\\W*(?:extended|)",
+            "skip_ci_labels": [ "skip-ci" ],
             "skip_target_branches": [ ],
-            "skip_ci_on_only_changed": [ ],
+            "skip_ci_on_only_changed": [ "^.ci/", "^changelog", "^docs/", "\\.md$", "^docker-compose.yml", "^.pre-commit-config.yaml", "skaffold.yaml", "^Dockerfile.skaffold", "^Dockerfile"],
             "always_require_ci_on_changed": [ ]
         },
         {