From 8c4e27651f554db8c543bc29d184d03a7e5b69e2 Mon Sep 17 00:00:00 2001
From: Pavlos Rontidis <pavlos.rontidis@gmail.com>
Date: Tue, 22 Oct 2024 18:19:17 -0400
Subject: [PATCH] feat(ci): regression detection overhaul (#21567)

* --no-edit

* remove trailing spaces

* pulled in fixes: changed timeouts to 70, fixed ignore glob
---
 .github/workflows/regression_v2.yml | 703 ++++++++++++++++++++++++++++
 1 file changed, 703 insertions(+)
 create mode 100644 .github/workflows/regression_v2.yml

diff --git a/.github/workflows/regression_v2.yml b/.github/workflows/regression_v2.yml
new file mode 100644
index 0000000000000..1b9001519bd26
--- /dev/null
+++ b/.github/workflows/regression_v2.yml
@@ -0,0 +1,703 @@
+# Regression Detection Suite
+#
+# This workflow runs under the following conditions:
+#   - Once per day, based on a schedule
+#   - On demand, triggered from https://github.com/vectordotdev/vector/actions/workflows/regression_v2.yml
+#
+# The workflow accepts two optional inputs:
+#   - The baseline SHA:
+#     - If not specified, the SHA from 24 hours ago on origin/master is used.
+#   - The comparison SHA:
+#     - If not specified, the current HEAD of origin/master is used.
+#
+# This workflow runs regression detection experiments, performing relative
+# evaluations of the baseline SHA and comparison SHA. The exact SHAs are determined
+# by how the workflow is triggered.
+#
+# The goal is to provide quick feedback on Vector's performance across a variety
+# of configurations, checking if throughput performance has degraded or become
+# more variable in the comparison SHA relative to the baseline SHA.
+#
+# Docker image tags are based on the resolved SHAs.
+
+name: Regression Detection Suite (new)
+
+on:
+  workflow_dispatch:
+    inputs:
+      baseline-sha:
+        description: "The SHA to use as the baseline (optional). If not provided, it defaults to the SHA from 24 hours ago."
+        required: false
+      comparison-sha:
+        description: "The SHA to use for comparison (optional). If not provided, it defaults to the current HEAD of the origin/master branch."
+        required: false
+  schedule:
+    - cron: '0 6 * * 1-5' # Runs at 6 AM UTC on weekdays (Monday to Friday)
+
+env:
+  SINGLE_MACHINE_PERFORMANCE_API: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_API }}
+  SMP_WARMUP_SECONDS: 70 # default is 45 seconds
+
+jobs:
+
+  resolve-inputs:
+    runs-on: ubuntu-latest
+    outputs:
+      baseline-sha: ${{ steps.set_and_validate_shas.outputs.BASELINE_SHA }}
+      comparison-sha: ${{ steps.set_and_validate_shas.outputs.COMPARISON_SHA }}
+      baseline-tag: ${{ steps.set_and_validate_shas.outputs.BASELINE_TAG }}
+      comparison-tag: ${{ steps.set_and_validate_shas.outputs.COMPARISON_TAG }}
+      smp-version: ${{ steps.experimental-meta.outputs.SMP_CRATE_VERSION }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # need to pull repository history to find merge bases
+
+      - name: Set and Validate SHAs
+        id: set_and_validate_shas
+        run: |
+          # Set baseline SHA
+          if [ -z "${{ github.event.inputs.baseline-sha }}" ]; then
+            BASELINE_SHA=$(git rev-list -n 1 --before="24 hours ago" origin/master)
+            echo "Using baseline SHA from 24 hours ago: ${BASELINE_SHA}"
+          else
+            BASELINE_SHA="${{ github.event.inputs.baseline-sha }}"
+            echo "Using provided baseline SHA: ${BASELINE_SHA}"
+          fi
+
+          # Validate baseline SHA
+          if [ -n "${BASELINE_SHA}" ] && git cat-file -e "${BASELINE_SHA}^{commit}"; then
+            echo "Baseline SHA is valid."
+          else
+            echo "Invalid baseline SHA: ${BASELINE_SHA}."
+            exit 1
+          fi
+
+          # Set comparison SHA
+          if [ -z "${{ github.event.inputs.comparison-sha }}" ]; then
+            COMPARISON_SHA=$(git rev-parse origin/master)
+            echo "Using current HEAD for comparison SHA: ${COMPARISON_SHA}"
+          else
+            COMPARISON_SHA="${{ github.event.inputs.comparison-sha }}"
+            echo "Using provided comparison SHA: ${COMPARISON_SHA}"
+          fi
+
+          # Validate comparison SHA
+          if [ -n "${COMPARISON_SHA}" ] && git cat-file -e "${COMPARISON_SHA}^{commit}"; then
+            echo "Comparison SHA is valid."
+          else
+            echo "Invalid comparison SHA: ${COMPARISON_SHA}."
+            exit 1
+          fi
+
+          # Set tags and export them
+          BASELINE_TAG="workflow_dispatch-${COMPARISON_SHA}-${BASELINE_SHA}"
+          COMPARISON_TAG="workflow_dispatch-${COMPARISON_SHA}-${COMPARISON_SHA}"
+
+          echo "BASELINE_SHA=${BASELINE_SHA}" >> $GITHUB_OUTPUT
+          echo "COMPARISON_SHA=${COMPARISON_SHA}" >> $GITHUB_OUTPUT
+
+          echo "BASELINE_TAG=${BASELINE_TAG}" >> $GITHUB_OUTPUT
+          echo "COMPARISON_TAG=${COMPARISON_TAG}" >> $GITHUB_OUTPUT
+
+      - name: Set SMP version
+        id: experimental-meta
+        run: |
+          export SMP_CRATE_VERSION="0.16.1"
+          echo "smp crate version: ${SMP_CRATE_VERSION}"
+          echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT
+
+  # Only run this workflow if files changed in areas that could possibly introduce a regression.
+  check-source-changed:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    needs: resolve-inputs
+    outputs:
+      source_changed: ${{ steps.filter.outputs.SOURCE_CHANGED }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Collect file changes
+        id: changes
+        uses: dorny/paths-filter@v3
+        with:
+          base: ${{ needs.resolve-inputs.outputs.baseline-sha }}
+          ref: ${{ needs.resolve-inputs.outputs.comparison-sha }}
+          list-files: shell
+          filters: |
+            all_changed:
+              - added|deleted|modified: "**"
+            ignore:
+              - "./.github/**/!(regression_v2.yml)"
+              - "./.gitignore"
+              - "distribution/**"
+              - "rust-doc/**"
+              - "docs/**"
+              - "rfcs/**"
+              - "testing/**"
+              - "tilt/**"
+              - "website/**"
+              - "*.md"
+              - "Tiltfile"
+              - "netlify.toml"
+              - "NOTICE"
+              - "LICENSE-3rdparty.csv"
+              - "LICENSE"
+              - "lib/codecs/tests/data/**"
+
+      # This step allows us to conservatively run the tests if we added a new
+      # file or directory for source code, but forgot to add it to this workflow.
+      # Instead, we may unnecessarily run the test on new file or dir additions that
+      # wouldn't likely introduce regressions.
+      - name: Determine if should not run due to irrelevant file changes
+        id: filter
+        if: github.event_name == 'merge_group'
+        env:
+          ALL: ${{ steps.changes.outputs.all_changed_files }}
+          IGNORE: ${{ steps.changes.outputs.ignore_files }}
+        run: |
+          echo "ALL='${{ env.ALL }}'"
+          echo "IGNORE='${{ env.IGNORE }}'"
+          export SOURCE_CHANGED=$(comm -2 -3 <(printf "%s\n" "${{ env.ALL }}") <(printf "%s\n" "${{ env.IGNORE }}"))
+          echo "SOURCE_CHANGED='${SOURCE_CHANGED}'"
+
+          if [ "${SOURCE_CHANGED}" == "" ]; then
+            export SOURCE_CHANGED="false"
+          else
+            export SOURCE_CHANGED="true"
+          fi
+
+          echo "SOURCE_CHANGED='${SOURCE_CHANGED}'"
+          echo "SOURCE_CHANGED=${SOURCE_CHANGED}" >> $GITHUB_OUTPUT
+
+  should-run-gate:
+    runs-on: ubuntu-latest
+    needs: check-source-changed
+    if: ${{ needs.check-source-changed.outputs.source_changed }}
+
+  ##
+  ## BUILD
+  ##
+
+  build-baseline:
+    name: Build baseline Vector container
+    runs-on: ubuntu-20.04-4core
+    timeout-minutes: 30
+    needs:
+      - should-run-gate
+      - resolve-inputs
+    steps:
+      - uses: colpal/actions-clean@v1
+
+      - uses: actions/checkout@v3
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ needs.resolve-inputs.outputs.baseline-sha }}
+          path: baseline-vector
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3.7.1
+
+      - name: Build 'vector' target image
+        uses: docker/build-push-action@v6.9.0
+        with:
+          context: baseline-vector/
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          file: regression/Dockerfile
+          builder: ${{ steps.buildx.outputs.name }}
+          outputs: type=docker,dest=${{ runner.temp }}/baseline-image.tar
+          tags: |
+            vector:${{ needs.resolve-inputs.outputs.baseline-tag }}
+
+      - name: Upload image as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: baseline-image
+          path: "${{ runner.temp }}/baseline-image.tar"
+
+  build-comparison:
+    name: Build comparison Vector container
+    runs-on: ubuntu-20.04-4core
+    timeout-minutes: 30
+    needs:
+      - should-run-gate
+      - resolve-inputs
+    steps:
+      - uses: colpal/actions-clean@v1
+
+      - uses: actions/checkout@v3
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ needs.resolve-inputs.outputs.comparison-sha }}
+          path: comparison-vector
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3.7.1
+
+      - name: Build 'vector' target image
+        uses: docker/build-push-action@v6.9.0
+        with:
+          context: comparison-vector/
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          file: regression/Dockerfile
+          builder: ${{ steps.buildx.outputs.name }}
+          outputs: type=docker,dest=${{ runner.temp }}/comparison-image.tar
+          tags: |
+            vector:${{ needs.resolve-inputs.outputs.comparison-tag }}
+
+      - name: Upload image as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: comparison-image
+          path: "${{ runner.temp }}/comparison-image.tar"
+
+  confirm-valid-credentials:
+    name: Confirm AWS credentials are minimally valid
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
+    needs:
+      - should-run-gate
+      - resolve-inputs
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4.0.2
+        with:
+          aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+
+      - name: Download SMP binary
+        run: |
+          aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp
+
+  ##
+  ## SUBMIT
+  ##
+
+  upload-baseline-image-to-ecr:
+    name: Upload baseline images to ECR
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
+    needs:
+      - should-run-gate
+      - resolve-inputs
+      - confirm-valid-credentials
+      - build-baseline
+    steps:
+      - name: 'Download baseline image'
+        uses: actions/download-artifact@v3
+        with:
+          name: baseline-image
+
+      - name: Load baseline image
+        run: |
+          docker load --input baseline-image.tar
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4.0.2
+        with:
+          aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Docker Login to ECR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ steps.login-ecr.outputs.registry }}
+
+      - name: Tag & push baseline image
+        run: |
+          docker tag vector:${{ needs.resolve-inputs.outputs.baseline-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.baseline-tag }}
+          docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.baseline-tag }}
+
+  upload-comparison-image-to-ecr:
+    name: Upload comparison images to ECR
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
+    needs:
+      - should-run-gate
+      - resolve-inputs
+      - confirm-valid-credentials
+      - build-comparison
+    steps:
+      - name: 'Download comparison image'
+        uses: actions/download-artifact@v3
+        with:
+          name: comparison-image
+
+      - name: Load comparison image
+        run: |
+          docker load --input comparison-image.tar
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4.0.2
+        with:
+          aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Docker Login to ECR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ steps.login-ecr.outputs.registry }}
+
+      - name: Tag & push comparison image
+        run: |
+          docker tag vector:${{ needs.resolve-inputs.outputs.comparison-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.comparison-tag }}
+          docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.comparison-tag }}
+
+  submit-job:
+    name: Submit regression job
+    runs-on: ubuntu-22.04
+    timeout-minutes: 70
+    needs:
+      - should-run-gate
+      - resolve-inputs
+      - upload-baseline-image-to-ecr
+      - upload-comparison-image-to-ecr
+    steps:
+      - name: Check status, in-progress
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \
+            -f state='pending' \
+            -f description='Experiments submitted to the Regression Detection cluster.' \
+            -f context='Regression Detection Suite / submission' \
+            -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ needs.resolve-inputs.outputs.comparison-sha }}
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4.0.2
+        with:
+          aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Download SMP binary
+        run: |
+          aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp
+
+      - name: Submit job
+        env:
+          RUST_LOG: info
+        run: |
+          chmod +x ${{ runner.temp }}/bin/smp
+
+          ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job submit \
+            --baseline-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.baseline-tag }} \
+            --comparison-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.resolve-inputs.outputs.comparison-tag }} \
+            --baseline-sha ${{ needs.resolve-inputs.outputs.baseline-sha }} \
+            --comparison-sha ${{ needs.resolve-inputs.outputs.comparison-sha }} \
+            --target-config-dir ${{ github.workspace }}/regression/ \
+            --warmup-seconds ${{ env.SMP_WARMUP_SECONDS }} \
+            --submission-metadata ${{ runner.temp }}/submission-metadata
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: vector-submission-metadata
+          path: ${{ runner.temp }}/submission-metadata
+
+      - name: Await job
+        timeout-minutes: 70
+        env:
+          RUST_LOG: info
+        run: |
+          chmod +x ${{ runner.temp }}/bin/smp
+
+          ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job status \
+            --wait \
+            --wait-delay-seconds 60 \
+            --wait-timeout-minutes 70 \
+            --submission-metadata ${{ runner.temp }}/submission-metadata
+
+      - name: Handle cancellation if necessary
+        if: ${{ cancelled() }}
+        env:
+          RUST_LOG: info
+        run: |
+          chmod +x ${{ runner.temp }}/bin/smp
+          ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job cancel \
+            --submission-metadata ${{ runner.temp }}/submission-metadata
+
+      - name: Check status, cancelled
+        if: ${{ cancelled() }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \
+            -f state='failure' \
+            -f description='Experiments submitted to the Regression Detection cluster cancelled.' \
+            -f context='Regression Detection Suite / submission' \
+            -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+      - name: Check status, success
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \
+            -f state='success' \
+            -f description='Experiments submitted to the Regression Detection cluster successfully.' \
+            -f context='Regression Detection Suite / submission' \
+            -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+      - name: Check status, failure
+        if: ${{ failure() }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \
+            -f state='success' \
+            -f description='Experiments submitted to the Regression Detection Suite failed.' \
+            -f context='Regression Detection Suite / submission' \
+            -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+  ##
+  ## ANALYZE
+  ##
+
+  detect-regression:
+    name: Determine regression status
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
+    needs:
+      - submit-job
+      - should-run-gate
+      - resolve-inputs
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4.0.2
+        with:
+          aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+
+      - name: Download SMP binary
+        run: |
+          aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp
+
+      - name: Download submission metadata
+        uses: actions/download-artifact@v3
+        with:
+          name: vector-submission-metadata
+          path: ${{ runner.temp }}/
+
+      - name: Determine SMP job result
+        env:
+          RUST_LOG: info
+        run: |
+          chmod +x ${{ runner.temp }}/bin/smp
+
+          ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job result \
+            --submission-metadata ${{ runner.temp }}/submission-metadata
+
+      - name: Check status, cancelled
+        if: ${{ cancelled() }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \
+            -f state='failure' \
+            -f description='Analyze experimental results from Regression Detection Suite cancelled.' \
+            -f context='Regression Detection Suite / detect-regression' \
+            -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+      - name: Check status, success
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \
+            -f state='success' \
+            -f description='Analyze experimental results from Regression Detection Suite succeeded.' \
+            -f context='Regression Detection Suite / detect-regression' \
+            -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+      - name: Check status, failure
+        if: ${{ failure() }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \
+            -f state='failure' \
+            -f description='Analyze experimental results from Regression Detection Suite failed.' \
+            -f context='Regression Detection Suite / detect-regression' \
+            -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+  analyze-experiment:
+    name: Download regression analysis & upload report
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
+    needs:
+      - should-run-gate
+      - submit-job
+      - resolve-inputs
+    steps:
+      - name: Check status, in-progress
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \
+            -f state='pending' \
+            -f description='Analyze experimental results from Regression Detection Suite.' \
+            -f context='Regression Detection Suite / analyze-experiment' \
+            -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ needs.resolve-inputs.outputs.comparison-sha }}
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4.0.2
+        with:
+          aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+
+      - name: Download SMP binary
+        run: |
+          aws s3 cp s3://smp-cli-releases/v${{ needs.resolve-inputs.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp
+
+      - name: Download submission metadata
+        uses: actions/download-artifact@v3
+        with:
+          name: vector-submission-metadata
+          path: ${{ runner.temp }}/
+
+      - name: Sync regression report to local system
+        env:
+          RUST_LOG: info
+        run: |
+          chmod +x ${{ runner.temp }}/bin/smp
+
+          ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job sync \
+            --submission-metadata ${{ runner.temp }}/submission-metadata \
+            --output-path "${{ runner.temp }}/outputs"
+
+      - name: Read regression report
+        id: read-analysis
+        uses: juliangruber/read-file-action@v1
+        with:
+          path: ${{ runner.temp }}/outputs/report.md
+
+      - name: Upload regression report to artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: capture-artifacts
+          path: ${{ runner.temp }}/outputs/*
+
+      - name: Check status, cancelled
+        if: ${{ cancelled() }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \
+            -f state='failure' \
+            -f description='Analyze experimental results from Regression Detection Suite cancelled.' \
+            -f context='Regression Detection Suite / analyze-experiment' \
+            -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+      - name: Check status, success
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \
+            -f state='success' \
+            -f description='Analyze experimental results from Regression Detection Suite succeeded.' \
+            -f context='Regression Detection Suite / analyze-experiment' \
+            -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+      - name: Check status, failure
+        if: ${{ failure() }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            /repos/${{ github.repository }}/statuses/${{ needs.resolve-inputs.outputs.comparison-sha }} \
+            -f state='failure' \
+            -f description='Analyze experimental results from Regression Detection Suite failed.' \
+            -f context='Regression Detection Suite / analyze-experiment' \
+            -f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+
+  regression-detection-suite:
+    name: Set final result
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    if: always()
+    needs:
+      - should-run-gate
+      - resolve-inputs
+      - build-baseline
+      - build-comparison
+      - confirm-valid-credentials
+      - upload-baseline-image-to-ecr
+      - upload-comparison-image-to-ecr
+      - submit-job
+      - detect-regression
+      - analyze-experiment
+    env:
+      FAILED: ${{ contains(needs.*.result, 'failure') }}
+    steps:
+      - name: exit
+        run: |
+          echo "failed=${{ env.FAILED }}"
+          if [[ "$FAILED" == "true" ]] ; then
+            exit 1
+          else
+            exit 0
+          fi