From c1ddbc64a91087a8131345f36e8c178e74502a73 Mon Sep 17 00:00:00 2001 From: Soren Spicknall Date: Tue, 19 Mar 2024 13:06:40 -0500 Subject: [PATCH] Fuzzy Key Uniqueness Threshold for fct_vehicle_locations (#3303) --- warehouse/models/mart/gtfs/_mart_gtfs_fcts.yml | 10 +--------- warehouse/models/mart/gtfs/fct_vehicle_locations.sql | 10 ++++++++-- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/warehouse/models/mart/gtfs/_mart_gtfs_fcts.yml b/warehouse/models/mart/gtfs/_mart_gtfs_fcts.yml index 0d4ab923dc..2b1965d17d 100644 --- a/warehouse/models/mart/gtfs/_mart_gtfs_fcts.yml +++ b/warehouse/models/mart/gtfs/_mart_gtfs_fcts.yml @@ -354,15 +354,7 @@ models: level. columns: - name: key - description: | - Synthetic primary key. - tests: - - unique: - config: - where: '__rt_sampled__' - - not_null: - config: - where: '__rt_sampled__' + tests: *almost_unique_rt_key_tests - <<: *rt_trip_id tests: - not_null: diff --git a/warehouse/models/mart/gtfs/fct_vehicle_locations.sql b/warehouse/models/mart/gtfs/fct_vehicle_locations.sql index 31247df86e..0c20806731 100644 --- a/warehouse/models/mart/gtfs/fct_vehicle_locations.sql +++ b/warehouse/models/mart/gtfs/fct_vehicle_locations.sql @@ -1,8 +1,14 @@ {{ config( materialized='incremental', - unique_key = 'key', - cluster_by = ['service_date', 'base64_url'], + incremental_strategy='insert_overwrite', + partition_by = { + 'field': 'dt', + 'data_type': 'date', + 'granularity': 'day', + }, + cluster_by=['dt', 'base64_url'], + on_schema_change='append_new_columns' ) }}