diff --git a/macros/tables/snowflake/hub.sql b/macros/tables/snowflake/hub.sql index d10bde57a..69a9b48fc 100644 --- a/macros/tables/snowflake/hub.sql +++ b/macros/tables/snowflake/hub.sql @@ -28,6 +28,8 @@ {%- set source_cols = automate_dv.expand_column_list(columns=[src_pk, src_nk, src_extra_columns, src_ldts, src_source]) -%} +{%- set predicates = model.config.get('predicates', none) -%} + {%- if model.config.materialized == 'vault_insert_by_rank' %} {%- set source_cols_with_rank = source_cols + [config.get('rank_column')] -%} {%- endif %} @@ -106,8 +108,13 @@ records_to_insert AS ( {%- if automate_dv.is_any_incremental() %} LEFT JOIN {{ this }} AS d ON {{ automate_dv.multikey(src_pk, prefix=['a','d'], condition='=') }} + {%- if predicates is not none %} + {% for predicate in predicates %} + AND d.{{ predicate }} + {% endfor %} + {% endif %} WHERE {{ automate_dv.multikey(src_pk, prefix='d', condition='IS NULL') }} - {%- endif %} + {%- endif %} ) SELECT * FROM records_to_insert diff --git a/macros/tables/snowflake/sat.sql b/macros/tables/snowflake/sat.sql index 75c5b42e5..06ddec17f 100644 --- a/macros/tables/snowflake/sat.sql +++ b/macros/tables/snowflake/sat.sql @@ -29,6 +29,7 @@ {%- set window_cols = automate_dv.expand_column_list(columns=[src_pk, src_hashdiff, src_ldts]) -%} {%- set pk_cols = automate_dv.expand_column_list(columns=[src_pk]) -%} {%- set enable_ghost_record = var('enable_ghost_records', false) -%} +{%- set predicates = model.config.get('predicates', none) -%} {%- if model.config.materialized == 'vault_insert_by_rank' %} {%- set source_cols_with_rank = source_cols + [config.get('rank_column')] -%} @@ -65,6 +66,12 @@ latest_records AS ( FROM source_data ) AS source_records ON {{ automate_dv.multikey(src_pk, prefix=['current_records','source_records'], condition='=') }} + WHERE 1=1 + {%- if predicates is not none %} + {% for predicate in predicates %} + AND {{ predicate }} + {% endfor %} + {% endif %} ) AS a WHERE a.rank = 1 ),