diff --git a/dataeng/jobs/analytics/WarehouseTransforms.groovy b/dataeng/jobs/analytics/WarehouseTransforms.groovy index 5b8715f46..cad6f7768 100644 --- a/dataeng/jobs/analytics/WarehouseTransforms.groovy +++ b/dataeng/jobs/analytics/WarehouseTransforms.groovy @@ -31,6 +31,7 @@ class WarehouseTransforms{ stringParam('TEST_SOURCES_FIRST', env_config.get('TEST_SOURCES_FIRST', 'true'), 'Set to \'true\' to perform source testing first (if SKIP_TESTS is false). All other values test sources post-run.') booleanParam('CAUTIOUS_INDIRECT_SELECTION', env_config.get('CAUTIOUS_INDIRECT_SELECTION', false), 'Check this box if you want a test to run if and only if ALL the models associated with that test have been selected (see dbt docs for --indirect-selection=cautious).') stringParam('PUSH_ARTIFACTS_TO_SNOWFLAKE', env_config.get('PUSH_ARTIFACTS_TO_SNOWFLAKE', 'false'), 'Set to \'true\' to push the run results file to Snowflake for telemetry. Avoid this on frequently-running jobs.') + stringParam('TEST_PARENT_MODELS_FIRST', env_config.get('TEST_PARENT_MODELS_FIRST', 'false'), 'Set to \'true\' to run the upstream models tests first.') stringParam('NOTIFY', env_config.get('NOTIFY', allVars.get('NOTIFY','$PAGER_NOTIFY')), 'Space separated list of emails to send notifications to.') booleanParam('FULL_REFRESH_INCREMENTALS', false, '[DANGEROUS] Supply the --full-refresh flag to the `dbt run` command, and use a larger warehouse. Use when you need to re-compute an incremental table from scratch. Applies to ALL incrementals in this run.') } diff --git a/dataeng/resources/warehouse-transforms.sh b/dataeng/resources/warehouse-transforms.sh index 791c3b37b..bcd53ece6 100644 --- a/dataeng/resources/warehouse-transforms.sh +++ b/dataeng/resources/warehouse-transforms.sh @@ -66,6 +66,30 @@ then postCommandChecks "source_test" $ret ; fi +# Parent models tests *before* model-building can be enabled/disabled with this envvar. +if [ "$TEST_PARENT_MODELS_FIRST" = 'true' ] +then + # Copy the value of MODEL_SELECTOR to MODEL_SELECTOR_WITH_PARENTS and EXCLUDE_MODELS. + MODEL_SELECTOR_WITH_PARENTS="$MODEL_SELECTOR" + EXCLUDE_MODELS="$MODEL_SELECTOR" + + # Check if MODEL_SELECTOR_WITH_PARENTS doesn't start with '+', then add '+' at the beginning. + if [ ${MODEL_SELECTOR_WITH_PARENTS:0:1} != "+" ] + then + MODEL_SELECTOR_WITH_PARENTS="+$MODEL_SELECTOR_WITH_PARENTS" + fi + + # Check if EXCLUDE_MODELS starts with '+', then remove the '+' at the beginning + if [ ${EXCLUDE_MODELS:0:1} = "+" ] + then + EXCLUDE_MODELS="${EXCLUDE_MODELS:1}" + fi + + # This will only runs parents tests without running current models tests. + dbt test --models $MODEL_SELECTOR_WITH_PARENTS --exclude $EXCLUDE_MODELS --profile $DBT_PROFILE --target $DBT_TARGET --profiles-dir $WORKSPACE/analytics-secure/warehouse-transforms/ ; ret=$?; + postCommandChecks "parent_models_tests" $ret ; +fi + # Compile/build all models with this tag. dbt $DBT_COMMAND $FULL_REFRESH_ARG --models $MODEL_SELECTOR --profile $DBT_PROFILE --target $DBT_TARGET --profiles-dir $WORKSPACE/analytics-secure/warehouse-transforms/ ; ret=$?; postCommandChecks "run" $ret ;