chore(ci): Use free OSS 4 vCPU runners (#19683)

* Update

Signed-off-by: Jesse Szwedko <jesse.szwedko@datadoghq.com>

* Clean up disk space

Signed-off-by: Jesse Szwedko <jesse.szwedko@datadoghq.com>

* Clear disk space for misc workflow

Signed-off-by: Jesse Szwedko <jesse.szwedko@datadoghq.com>

* leave node_modules

Signed-off-by: Jesse Szwedko <jesse.szwedko@datadoghq.com>

* clean more disk space

Signed-off-by: Jesse Szwedko <jesse.szwedko@datadoghq.com>

---------

Signed-off-by: Jesse Szwedko <jesse.szwedko@datadoghq.com>

Author: jszwedko

.github/workflows/ci-integration-review.yml

@@ -81,7 +81,7 @@ jobs:
 
   integration-tests:
     needs: prep-pr
-    runs-on: ubuntu-20.04-4core
+    runs-on: ubuntu-20.04
     timeout-minutes: 90
     steps:
       - uses: actions/checkout@v4

.github/workflows/integration-test.yml

@@ -30,7 +30,7 @@ env:
 
 jobs:
   test-integration:
-    runs-on: ubuntu-20.04-4core
+    runs-on: ubuntu-20.04
     timeout-minutes: 40
     if: inputs.if || github.event_name == 'workflow_dispatch'
     steps:

.github/workflows/integration.yml

@@ -48,7 +48,7 @@ jobs:
 
   integration-tests:
     name: Integration Tests
-    runs-on: ubuntu-20.04-4core
+    runs-on: ubuntu-20.04
     needs: changes
     if: always() && (
       github.event_name == 'merge_group' || (
@@ -97,6 +97,7 @@ jobs:
 
       - run: sudo npm -g install @datadog/datadog-ci
 
+      - run: sudo -E bash scripts/ci-free-disk-space.sh
       - run: docker image prune -af ; docker container prune -f
 
       - name: Determine if secrets are defined (PR author is team member).
@@ -110,6 +111,8 @@ jobs:
             echo "PR_HAS_ACCESS_TO_SECRETS=false" >> "$GITHUB_ENV"
           fi
 
+      - run: docker image prune -af ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.amqp == 'true' }}
         name: amqp
         uses: nick-fields/retry@v3
@@ -118,6 +121,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int amqp
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.appsignal == 'true') &&
           (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
         name: appsignal
@@ -127,6 +132,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  appsignal
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.aws == 'true' }}
         name: aws
         uses: nick-fields/retry@v3
@@ -135,6 +142,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  aws
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.axiom == 'true') &&
           (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
         name: axiom
@@ -144,6 +153,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  axiom
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.azure == 'true' }}
         name: azure
         uses: nick-fields/retry@v3
@@ -152,6 +163,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  azure
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.clickhouse == 'true' }}
         name: clickhouse
         uses: nick-fields/retry@v3
@@ -160,6 +173,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  clickhouse
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.databend == 'true') &&
           (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
         name: databend
@@ -169,6 +184,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  databend
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-agent == 'true') &&
           (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
         name: datadog-agent
@@ -178,6 +195,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  datadog-agent
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-logs == 'true') &&
           (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
         name: datadog-logs
@@ -198,6 +217,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  datadog-metrics
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog-traces == 'true') &&
           (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true')
         name: datadog-traces
@@ -207,6 +228,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  datadog-traces
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.dnstap == 'true' }}
         name: dnstap
         uses: nick-fields/retry@v3
@@ -215,6 +238,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  dnstap
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.docker-logs == 'true' }}
         name: docker-logs
         uses: nick-fields/retry@v3
@@ -223,6 +248,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  docker-logs
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.elasticsearch == 'true' }}
         name: elasticsearch
         uses: nick-fields/retry@v3
@@ -231,6 +258,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  elasticsearch
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.eventstoredb == 'true' }}
         name: eventstoredb
         uses: nick-fields/retry@v3
@@ -239,6 +268,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  eventstoredb
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.fluent == 'true' }}
         name: fluent
         uses: nick-fields/retry@v3
@@ -247,6 +278,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  fluent
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.gcp == 'true' }}
         name: gcp
         uses: nick-fields/retry@v3
@@ -255,6 +288,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  gcp
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.greptimedb == 'true' }}
         name: greptimedb
         uses: nick-fields/retry@v3
@@ -273,6 +308,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  humio
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.http-client == 'true' }}
         name: http-client
         uses: nick-fields/retry@v3
@@ -281,6 +318,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  http-client
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.influxdb == 'true' }}
         name: influxdb
         uses: nick-fields/retry@v3
@@ -289,6 +328,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  influxdb
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.kafka == 'true' }}
         name: kafka
         uses: nick-fields/retry@v3
@@ -297,6 +338,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  kafka
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.logstash == 'true' }}
         name: logstash
         uses: nick-fields/retry@v3
@@ -305,6 +348,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  logstash
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.loki == 'true' }}
         name: loki
         uses: nick-fields/retry@v3
@@ -313,6 +358,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  loki
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.mongodb == 'true' }}
         name: mongodb
         uses: nick-fields/retry@v3
@@ -321,6 +368,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  mongodb
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.mqtt == 'true' }}
         name: mqtt
         uses: nick-fields/retry@v3
@@ -329,6 +378,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  mqtt
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.nats == 'true' }}
         name: nats
         uses: nick-fields/retry@v3
@@ -337,6 +388,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  nats
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.nginx == 'true' }}
         name: nginx
         uses: nick-fields/retry@v3
@@ -355,6 +408,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  opentelemetry
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.postgres == 'true' }}
         name: postgres
         uses: nick-fields/retry@v3
@@ -363,6 +418,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  postgres
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.prometheus == 'true' }}
         name: prometheus
         uses: nick-fields/retry@v3
@@ -371,6 +428,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  prometheus
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.pulsar == 'true' }}
         name: pulsar
         uses: nick-fields/retry@v3
@@ -379,6 +438,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  pulsar
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.redis == 'true' }}
         name: redis
         uses: nick-fields/retry@v3
@@ -387,6 +448,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  redis
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' }}
         name: shutdown
         uses: nick-fields/retry@v3
@@ -395,6 +458,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  shutdown
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.splunk == 'true' }}
         name: splunk
         uses: nick-fields/retry@v3
@@ -403,6 +468,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  splunk
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
       - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.webhdfs == 'true' }}
         name: webhdfs
         uses: nick-fields/retry@v3
@@ -411,6 +478,8 @@ jobs:
           max_attempts: 3
           command: bash scripts/ci-int-e2e-test.sh int  webhdfs
 
+      - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f
+
   integration-test-suite:
     name: Integration Test Suite
     runs-on: ubuntu-latest

.github/workflows/k8s_e2e.yml

@@ -59,7 +59,7 @@ jobs:
 
   build-x86_64-unknown-linux-gnu:
     name: Build - x86_64-unknown-linux-gnu
-    runs-on: ubuntu-20.04-4core
+    runs-on: ubuntu-20.04
     timeout-minutes: 45
     needs: changes
     # Run this job even if `changes` job is skipped (non- pull request trigger)
@@ -176,7 +176,7 @@ jobs:
 
   test-e2e-kubernetes:
     name: K8s ${{ matrix.kubernetes_version.version }} / ${{ matrix.container_runtime }} (${{ matrix.kubernetes_version.role }})
-    runs-on: ubuntu-20.04-4core
+    runs-on: ubuntu-20.04
     timeout-minutes: 45
     needs:
       - build-x86_64-unknown-linux-gnu

.github/workflows/misc.yml

@@ -5,7 +5,7 @@ on:
 
 jobs:
   test-misc:
-    runs-on: ubuntu-20.04-4core
+    runs-on: ubuntu-20.04
     timeout-minutes: 60
     env:
       CARGO_INCREMENTAL: 0
@@ -41,6 +41,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-cargo-
 
+      - run: sudo -E bash scripts/ci-free-disk-space.sh
       - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh
       - run: bash scripts/environment/prepare.sh
       - run: echo "::add-matcher::.github/matchers/rust.json"

.github/workflows/regression.yml

@@ -184,7 +184,7 @@ jobs:
 
   build-baseline:
     name: Build baseline Vector container
-    runs-on: ubuntu-20.04-4core
+    runs-on: ubuntu-20.04
     timeout-minutes: 30
     needs:
       - should-run-gate
@@ -223,7 +223,7 @@ jobs:
 
   build-comparison:
     name: Build comparison Vector container
-    runs-on: ubuntu-20.04-4core
+    runs-on: ubuntu-20.04
     timeout-minutes: 30
     needs:
       - should-run-gate