Merge pull request #12 from casparwylie/#11-use-date-types-for-date-fields

Use DATE types for all date fields

Author: fivetran-catfritz

.buildkite/hooks/pre-command

@@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
 export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -58,7 +58,7 @@ steps:
     commands: |
       bash .buildkite/scripts/run_models.sh redshift
 
-  - label: ":bricks: Run Tests - Databricks"
+  - label: ":databricks: Run Tests - Databricks"
     key: "run_dbt_databricks"
     plugins:
       - docker#v3.13.0:
@@ -69,5 +69,6 @@ steps:
             - "CI_DATABRICKS_DBT_HOST"
             - "CI_DATABRICKS_DBT_HTTP_PATH"
             - "CI_DATABRICKS_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
     commands: |
       bash .buildkite/scripts/run_models.sh databricks

CHANGELOG.md

@@ -1,3 +1,11 @@
+# dbt_google_play_source v0.3.1
+[PR #12](https://github.com/fivetran/dbt_google_play_source/pull/12) includes the following updates:
+## 🪲 Bug fixes:
+- Updated staging models to cast `date_day` columns to the `date` datatype. This prevents downstream datatype conflicts when joining on these columns.
+
+## Contributors
+- [@casparwylie](https://github.com/casparwylie) ([#12](https://github.com/fivetran/dbt_google_play_source/pull/12))
+
 # dbt_google_play_source v0.3.0
 ## 🚨 Breaking Changes 🚨:
 [PR #11](https://github.com/fivetran/dbt_google_play_source/pull/11) includes the following changes:

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'google_play_source'
-version: '0.3.0'
+version: '0.3.1'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 vars:

docs/catalog.json

@@ -45,10 +45,10 @@ integration_tests:
       schema: google_play_source_integration_tests_1
       threads: 8
     databricks:
-      catalog: null
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
       schema: google_play_source_integration_tests_1
-      threads: 2
+      threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

docs/index.html

@@ -1,5 +1,5 @@
 name: 'google_play_source_integration_tests'
-version: '0.3.0'
+version: '0.3.1'
 profile: 'integration_tests'
 config-version: 2
 

docs/manifest.json

@@ -1,7 +1,7 @@
 
 with base as (
 
-    select * 
+    select *
     from {{ ref('stg_google_play__stats_crashes_app_version_tmp') }}
 ),
 
@@ -19,9 +19,9 @@ fields as (
 ),
 
 final as (
-    
-    select 
-        date as date_day,
+
+    select
+        cast(date as date) as date_day,
         app_version_code,
         package_name,
         sum(daily_anrs) as anrs,
@@ -30,5 +30,5 @@ final as (
     group by 1,2,3
 )
 
-select * 
+select *
 from final

docs/run_results.json

@@ -1,7 +1,7 @@
 
 with base as (
 
-    select * 
+    select *
     from {{ ref('stg_google_play__stats_crashes_os_version_tmp') }}
 ),
 
@@ -14,14 +14,14 @@ fields as (
                 staging_columns=get_stats_crashes_os_version_columns()
             )
         }}
-        
+
     from base
 ),
 
 final as (
-    
-    select 
-        date as date_day,
+
+    select
+        cast(date as date) as date_day,
         android_os_version,
         package_name,
         sum(daily_anrs) as anrs,
@@ -30,5 +30,5 @@ final as (
     group by 1,2,3
 )
 
-select * 
+select *
 from final

integration_tests/ci/sample.profiles.yml

@@ -1,7 +1,7 @@
 
 with base as (
 
-    select * 
+    select *
     from {{ ref('stg_google_play__stats_crashes_overview_tmp') }}
 ),
 
@@ -14,20 +14,20 @@ fields as (
                 staging_columns=get_stats_crashes_overview_columns()
             )
         }}
-        
+
     from base
 ),
 
 final as (
-    
+
     select
-        date as date_day,
+        cast(date as date) as date_day,
         package_name,
         daily_anrs as anrs,
         daily_crashes as crashes,
         _fivetran_synced
     from fields
 )
 
-select * 
+select *
 from final

integration_tests/dbt_project.yml

@@ -1,7 +1,7 @@
 
 with base as (
 
-    select * 
+    select *
     from {{ ref('stg_google_play__stats_installs_app_version_tmp') }}
 ),
 
@@ -14,14 +14,14 @@ fields as (
                 staging_columns=get_stats_installs_app_version_columns()
             )
         }}
-        
+
     from base
 ),
 
 final as (
-    
-    select 
-        date as date_day,
+
+    select
+        cast(date as date) as date_day,
         app_version_code,
         package_name,
         sum(active_device_installs) as active_devices_last_30_days,
@@ -37,5 +37,5 @@ final as (
     group by 1,2,3
 )
 
-select * 
+select *
 from final

models/stg_google_play__stats_crashes_app_version.sql

@@ -1,7 +1,7 @@
 
 with base as (
 
-    select * 
+    select *
     from {{ ref('stg_google_play__stats_installs_country_tmp') }}
 ),
 
@@ -14,14 +14,14 @@ fields as (
                 staging_columns=get_stats_installs_country_columns()
             )
         }}
-        
+
     from base
 ),
 
 final as (
-    
-    select 
-        date as date_day,
+
+    select
+        cast(date as date) as date_day,
         country,
         package_name,
         sum(active_device_installs) as active_devices_last_30_days,
@@ -37,5 +37,5 @@ final as (
     group by 1,2,3
 )
 
-select * 
+select *
 from final

models/stg_google_play__stats_crashes_os_version.sql

@@ -1,7 +1,7 @@
 
 with base as (
 
-    select * 
+    select *
     from {{ ref('stg_google_play__stats_installs_device_tmp') }}
 ),
 
@@ -14,14 +14,14 @@ fields as (
                 staging_columns=get_stats_installs_device_columns()
             )
         }}
-        
+
     from base
 ),
 
 final as (
-    
-    select 
-        date as date_day,
+
+    select
+        cast(date as date) as date_day,
         device, -- device_type
         package_name,
         sum(active_device_installs) as active_devices_last_30_days,
@@ -38,5 +38,5 @@ final as (
 
 )
 
-select * 
+select *
 from final

models/stg_google_play__stats_crashes_overview.sql

@@ -1,7 +1,7 @@
 
 with base as (
 
-    select * 
+    select *
     from {{ ref('stg_google_play__stats_installs_os_version_tmp') }}
 ),
 
@@ -14,14 +14,14 @@ fields as (
                 staging_columns=get_stats_installs_os_version_columns()
             )
         }}
-        
+
     from base
 ),
 
 final as (
-    
-    select 
-        date as date_day,
+
+    select
+        cast(date as date) as date_day,
         package_name,
         android_os_version,
         sum(active_device_installs) as active_devices_last_30_days,
@@ -37,5 +37,5 @@ final as (
     group by 1,2,3
 )
 
-select * 
+select *
 from final

models/stg_google_play__stats_installs_app_version.sql

@@ -1,7 +1,7 @@
 
 with base as (
 
-    select * 
+    select *
     from {{ ref('stg_google_play__stats_installs_overview_tmp') }}
 ),
 
@@ -14,14 +14,14 @@ fields as (
                 staging_columns=get_stats_installs_overview_columns()
             )
         }}
-        
+
     from base
 ),
 
 final as (
-    
-    select 
-        date as date_day,
+
+    select
+        cast(date as date) as date_day,
         package_name,
         active_device_installs as active_devices_last_30_days,
         daily_device_installs as device_installs,
@@ -36,5 +36,5 @@ final as (
     from fields
 )
 
-select * 
+select *
 from final

models/stg_google_play__stats_installs_country.sql

@@ -1,7 +1,7 @@
 
 with base as (
 
-    select * 
+    select *
     from {{ ref('stg_google_play__stats_ratings_app_version_tmp') }}
 ),
 
@@ -14,14 +14,14 @@ fields as (
                 staging_columns=get_stats_ratings_app_version_columns()
             )
         }}
-        
+
     from base
 ),
 
 final as (
-    
-    select 
-        date as date_day,
+
+    select
+        cast(date as date) as date_day,
         app_version_code,
         package_name,
         case when app_version_code is null then null else cast( nullif(cast(daily_average_rating as {{ dbt.type_string() }}), 'NA') as {{ dbt.type_float() }} ) end as average_rating,
@@ -30,5 +30,5 @@ final as (
     {{ dbt_utils.group_by(n=5) }}
 )
 
-select * 
+select *
 from final