dbt_slim_ci.yml

# .github/workflows/app.yaml
name: dbt slim ci (in beta)

on:
  workflow_dispatch:
  pull_request:
    paths-ignore:
      - 'scripts/**'
      - 'Pipfile'
      - '.gitignore'
      - '.github/**'

concurrency:
  group:  ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true


jobs:
  dbt-test:
    runs-on: [ self-hosted, linux, spellbook-trino ]
    strategy:
      matrix:
        engine: [ 'dunesql', 'spark' ]
      max-parallel: 1
    timeout-minutes: 90

    steps:
      - name: Check out repository code
        uses: actions/checkout@v3

      - name: Setup variables
        run: |
          if [[ "${{ matrix.engine }}" == "dunesql" ]]; then
            printf "Using dunesql engine\n"
            echo "PROFILE=--profiles-dir $HOME/.dbt --profile dunesql" >> $GITHUB_ENV
            echo "TAG=,tag:dunesql" >> $GITHUB_ENV
            echo "COMPILE_TAG=--select tag:dunesql" >> $GITHUB_ENV
            echo "S3_LOCATION=manifest-spellbook-dunesql" >> $GITHUB_ENV
          elif [[ "${{ matrix.engine }}" == "spark" ]]; then
            printf "Using spark engine\n"
            echo "PROFILE=--profiles-dir $HOME/.dbt --profile spark" >> $GITHUB_ENV
            echo "TAG=,tag:legacy" >> $GITHUB_ENV
            echo "COMPILE_TAG=--select tag:legacy" >> $GITHUB_ENV
            echo "S3_LOCATION=manifest-spellbook" >> $GITHUB_ENV
            echo 
          else
            echo "Unknown engine: ${{ matrix.engine }}"
            exit 1
          fi
          echo "GIT_SHA=$(echo ${{ github.sha }} | tr - _ | cut -c1-8)" >> $GITHUB_ENV

      - name: Add git_sha to schema
        run: "/runner/change_schema.sh git_${{ matrix.engine }}_$GIT_SHA"

      - name: Get latest manifest
        run: "aws s3 cp s3://$S3_LOCATION/manifest.json manifest.json"

      - name: dbt dependencies
        run: "dbt deps"

      - name: Check no common models between DuneSQL and Spark
        if: matrix.engine == 'dunesql'
        run: "./scripts/dunesql_spark_empty_intersection.sh"

      - name: Check for missing legacy models
        if: matrix.engine == 'dunesql'
        run: "./scripts/missing_legacy_model.sh"

      - name: dbt compile to create manifest to compare to
        run: "dbt compile $PROFILE $COMPILE_TAG"

      - name: check schemas
        if: matrix.engine == 'dunesql'
        run: |
          test=$(dbt --quiet --no-print ls $PROFILE --resource-type model --select  state:modified,config.schema:no_schema --output path --state .)
          [[ -z "$test" ]] && { echo "Success: All models have a custom schema"; exit 0; } || { echo "Found models without custom schema:"; echo "$test"; exit 1; }

      - name: check tags
        if: matrix.engine == 'dunesql'
        run: |
          test=$(dbt --quiet --no-print ls $PROFILE --resource-type model --select state:modified --exclude tag:legacy tag:dunesql --output path --state .)
          [[ -z "$test" ]] && { echo "Success: No models without a tag"; exit 0; } || { echo "Found models with no dunesql or legacy tag:"; echo "$test"; exit 1; }

      - name: dbt seed
        run: "dbt seed $PROFILE --select state:modified$TAG --exclude tag:prod_exclude --state ."

      - name: dbt run initial model(s)
        run: "dbt -x run $PROFILE --select state:modified$TAG --exclude tag:prod_exclude --defer --state ."

      - name: dbt test initial model(s)
        run: "dbt test $PROFILE --select state:new$TAG state:modified$TAG --exclude tag:prod_exclude --defer --state ."

      - name: Set environment variable for incremental model count
        run: |
          echo "INC_MODEL_COUNT=$(echo dbt ls $PROFILE --select state:modified,config.materialized:incremental$TAG --state . --resource-type model  | wc -l)" >> $GITHUB_ENV

      - name: dbt run incremental model(s) if applicable
        if: env.INC_MODEL_COUNT > 0
        run: "dbt run $PROFILE --select state:modified,config.materialized:incremental$TAG --exclude tag:prod_exclude --defer --state ."

      - name: dbt test incremental model(s) if applicable
        if: env.INC_MODEL_COUNT > 0
        run: "dbt test $PROFILE --select state:modified,config.materialized:incremental$TAG --exclude tag:prod_exclude --defer --state ."

      - name: Run DuneSQL Check
        if: matrix.engine != 'dunesql'
        run: "/runner/dunesql_check.py --schema test_schema --pr_schema git_$GIT_SHA"