diff --git a/dataeng/jobs/analytics/ModelTransfers.groovy b/dataeng/jobs/analytics/ModelTransfers.groovy index bef9063d7..69f199c99 100644 --- a/dataeng/jobs/analytics/ModelTransfers.groovy +++ b/dataeng/jobs/analytics/ModelTransfers.groovy @@ -45,7 +45,6 @@ class ModelTransfers{ publishers common_publishers(allVars) steps { shell(dslFactory.readFileFromWorkspace('dataeng/resources/model-transfers.sh')) - shell(dslFactory.readFileFromWorkspace('dataeng/resources/trigger-prefect-flow.sh')) } } } diff --git a/dataeng/jobs/analytics/ModelTransfersJenkinsTest.groovy b/dataeng/jobs/analytics/ModelTransfersJenkinsTest.groovy new file mode 100644 index 000000000..5a827ef00 --- /dev/null +++ b/dataeng/jobs/analytics/ModelTransfersJenkinsTest.groovy @@ -0,0 +1,54 @@ +package analytics +import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm +import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator +import static org.edx.jenkins.dsl.AnalyticsConstants.common_wrappers +import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers +import static org.edx.jenkins.dsl.AnalyticsConstants.common_triggers +import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm_parameters +import static org.edx.jenkins.dsl.AnalyticsConstants.common_authorization + +class ModelTransfersJenkinsTest{ + public static def job = { dslFactory, allVars -> + allVars.get('ENVIRONMENTS').each { environment, env_config -> + dslFactory.job("transfer-dbt-models-$environment"){ + authorization common_authorization(env_config) + logRotator common_log_rotator(allVars) + parameters secure_scm_parameters(allVars) + parameters { + stringParam('WAREHOUSE_TRANSFORMS_URL', allVars.get('WAREHOUSE_TRANSFORMS_URL'), 'URL for the Warehouse Transforms Repo.') + stringParam('WAREHOUSE_TRANSFORMS_BRANCH', allVars.get('WAREHOUSE_TRANSFORMS_BRANCH'), 'Branch of Warehouse Transforms to use.') + stringParam('DBT_PROJECT', env_config.get('DBT_PROJECT', allVars.get('DBT_PROJECT')), 'dbt project in warehouse-transforms to work on.') + stringParam('DBT_PROFILE', env_config.get('DBT_PROFILE', allVars.get('DBT_PROFILE')), 'dbt profile from analytics-secure to work on.') + stringParam('DBT_TARGET', env_config.get('DBT_TARGET', allVars.get('DBT_TARGET')), 'dbt target from analytics-secure to work on.') + stringParam('MODELS_TO_TRANSFER', env_config.get('MODELS_TO_TRANSFER'), 'Name of DBT models which should be transferred to S3 via a Snowflake stage.') + stringParam('NOTIFY', env_config.get('NOTIFY', allVars.get('NOTIFY','$PAGER_NOTIFY')), 'Space separated list of emails to send notifications to.') + } + multiscm secure_scm(allVars) << { + git { + remote { + url('$WAREHOUSE_TRANSFORMS_URL') + branch('$WAREHOUSE_TRANSFORMS_BRANCH') + credentials('1') + } + extensions { + relativeTargetDirectory('warehouse-transforms') + pruneBranches() + cleanAfterCheckout() + } + } + } + triggers common_triggers(allVars, env_config) + wrappers common_wrappers(allVars) + wrappers { + colorizeOutput('xterm') + } + publishers common_publishers(allVars) + steps { + shell(dslFactory.readFileFromWorkspace('dataeng/resources/model-transfers.sh')) + shell(dslFactory.readFileFromWorkspace('dataeng/resources/trigger-prefect-flow.sh')) + } + } + } + } +} + diff --git a/dataeng/jobs/createJobsNew.groovy b/dataeng/jobs/createJobsNew.groovy index cee6a0e79..5cd8d9208 100644 --- a/dataeng/jobs/createJobsNew.groovy +++ b/dataeng/jobs/createJobsNew.groovy @@ -4,6 +4,7 @@ import static analytics.DBTSourceFreshness.job as DBTSourceFreshnessJob import static analytics.DeployCluster.job as DeployClusterJob import static analytics.EmrCostReporter.job as EmrCostReporterJob import static analytics.ModelTransfers.job as ModelTransfersJob +import static analytics.ModelTransfersJenkinsTest.job as ModelTransfersJenkinsTestJob import static analytics.RetirementJobEdxTriggers.job as RetirementJobEdxTriggersJob import static analytics.RetirementJobs.job as RetirementJobsJob import static analytics.SnowflakeCollectMetrics.job as SnowflakeCollectMetricsJob @@ -46,6 +47,7 @@ def taskMap = [ DEPLOY_CLUSTER_JOB: DeployClusterJob, EMR_COST_REPORTER_JOB: EmrCostReporterJob, MODEL_TRANSFERS_JOB: ModelTransfersJob, + MODEL_TRANSFERS_TEST_JOB: ModelTransfersJenkinsTestJob, RETIREMENT_JOB_EDX_TRIGGERS_JOB: RetirementJobEdxTriggersJob, RETIREMENT_JOBS_JOB: RetirementJobsJob, SNOWFLAKE_COLLECT_METRICS_JOB: SnowflakeCollectMetricsJob,