diff --git a/dataeng/jobs/analytics/AmplitudeUserPropertiesBackfill.groovy b/dataeng/jobs/analytics/AmplitudeUserPropertiesBackfill.groovy deleted file mode 100644 index 63315b35d..000000000 --- a/dataeng/jobs/analytics/AmplitudeUserPropertiesBackfill.groovy +++ /dev/null @@ -1,58 +0,0 @@ -package analytics - -import static org.edx.jenkins.dsl.AnalyticsConstants.common_authorization -import static org.edx.jenkins.dsl.AnalyticsConstants.common_log_rotator -import static org.edx.jenkins.dsl.AnalyticsConstants.common_publishers -import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm_parameters -import static org.edx.jenkins.dsl.AnalyticsConstants.secure_scm - - -class AmplitudeUserPropertiesBackfill { - public static def job = { dslFactory, allVars -> - dslFactory.job("amplitude-user-properties-backfill") { - logRotator common_log_rotator(allVars) - authorization common_authorization(allVars) - parameters secure_scm_parameters(allVars) - parameters { - stringParam('ANALYTICS_TOOLS_URL', allVars.get('ANALYTICS_TOOLS_URL'), 'URL for the analytics tools repo.') - stringParam('ANALYTICS_TOOLS_BRANCH', allVars.get('ANALYTICS_TOOLS_BRANCH'), 'Branch of analytics tools repo to use.') - stringParam('NOTIFY', allVars.get('NOTIFY','$PAGER_NOTIFY'), 'Space separated list of emails to send notifications to.') - stringParam('PYTHON_VENV_VERSION', 'python3.7', 'Python virtual environment version to used.') - stringParam('AMPLITUDE_DATA_SOURCE_TABLE', '', 'Table name that has data which needs to be updated on Amplitude. It should have format like database.schema.table.') - stringParam('COLUMNS_TO_UPDATE', '', 'Columns that you want to update. Separate multiple columns with commas.') - stringParam('RESPONSE_TABLE', '', 'Output table which will store the updated data along with response from API endpoint.') - stringParam('AMPLITUDE_OPERATION_NAME', '', 'Amplitude user property operation name. e.g: set or setOnce.') - } - environmentVariables { - env('USER', allVars.get('USER')) - env('ACCOUNT', allVars.get('ACCOUNT')) - env('AMPLITUDE_VAULT_KV_PATH', allVars.get('AMPLITUDE_VAULT_KV_PATH')) - env('AMPLITUDE_VAULT_KV_VERSION', allVars.get('AMPLITUDE_VAULT_KV_VERSION')) - } - multiscm secure_scm(allVars) << { - git { - remote { - url('$ANALYTICS_TOOLS_URL') - branch('$ANALYTICS_TOOLS_BRANCH') - credentials('1') - } - extensions { - relativeTargetDirectory('analytics-tools') - pruneBranches() - cleanAfterCheckout() - } - } - } - wrappers { - timestamps() - credentialsBinding { - usernamePassword('ANALYTICS_VAULT_ROLE_ID', 'ANALYTICS_VAULT_SECRET_ID', 'analytics-vault'); - } - } - publishers common_publishers(allVars) - steps { - shell(dslFactory.readFileFromWorkspace('dataeng/resources/amplitude-properties-backfill.sh')) - } - } - } -} diff --git a/dataeng/jobs/createJobsNew.groovy b/dataeng/jobs/createJobsNew.groovy index 5a825df08..7f2653f5c 100644 --- a/dataeng/jobs/createJobsNew.groovy +++ b/dataeng/jobs/createJobsNew.groovy @@ -9,7 +9,6 @@ import static analytics.ModelTransfers.job as ModelTransfersJob import static analytics.RetirementJobEdxTriggers.job as RetirementJobEdxTriggersJob import static analytics.RetirementJobs.job as RetirementJobsJob import static analytics.SnowflakeCollectMetrics.job as SnowflakeCollectMetricsJob -import static analytics.AmplitudeUserPropertiesBackfill.job as AmplitudeUserPropertiesBackfillJob import static analytics.SnowflakeSchemaBuilder.job as SnowflakeSchemaBuilderJob import static analytics.SnowflakeUserRetirementStatusCleanup.job as SnowflakeUserRetirementStatusCleanupJob import static analytics.PrefectFlowsDeployment.job as PrefectFlowsDeploymentJob @@ -53,7 +52,6 @@ def taskMap = [ RETIREMENT_JOB_EDX_TRIGGERS_JOB: RetirementJobEdxTriggersJob, RETIREMENT_JOBS_JOB: RetirementJobsJob, SNOWFLAKE_COLLECT_METRICS_JOB: SnowflakeCollectMetricsJob, - AMPLITUDE_USER_PROPERTIES_BACKFILL_JOB: AmplitudeUserPropertiesBackfillJob, SNOWFLAKE_SCHEMA_BUILDER_JOB: SnowflakeSchemaBuilderJob, SNOWFLAKE_USER_RETIREMENT_STATUS_CLEANUP_JOB: SnowflakeUserRetirementStatusCleanupJob, PREFECT_FLOWS_DEPLOYMENT_JOB: PrefectFlowsDeploymentJob, diff --git a/dataeng/resources/amplitude-properties-backfill.sh b/dataeng/resources/amplitude-properties-backfill.sh deleted file mode 100644 index 1d7a38891..000000000 --- a/dataeng/resources/amplitude-properties-backfill.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -set -ex - -# Creating Python virtual env -PYTHON_VENV="python_venv" -virtualenv --python=$PYTHON_VENV_VERSION --clear "${PYTHON_VENV}" -source "${PYTHON_VENV}/bin/activate" - -# Setup -cd $WORKSPACE/analytics-tools/snowflake -make requirements - -# Do not print commands in this function since they may contain secrets. -set +x - -# Retrieve a vault token corresponding to the jenkins AppRole. The token is then stored in the VAULT_TOKEN variable -# which is implicitly used by subsequent vault commands within this script. -# Instructions followed: https://learn.hashicorp.com/tutorials/vault/approle#step-4-login-with-roleid-secretid -export VAULT_TOKEN=$(vault write -field=token auth/approle/login \ - role_id=${ANALYTICS_VAULT_ROLE_ID} \ - secret_id=${ANALYTICS_VAULT_SECRET_ID} - ) - -API_KEY=$( - vault kv get \ - -version=${AMPLITUDE_VAULT_KV_VERSION} \ - -field=API_KEY \ - ${AMPLITUDE_VAULT_KV_PATH} \ - - -python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_snowpipe_user.p8 -v rsa_key_snowflake_task_automation_user -python3 secrets-manager.py -w -n analytics-secure/snowflake/rsa_key_passphrase_snowpipe_user -v rsa_key_passphrase_snowflake_task_automation_user - - -python amplitude_user_properties_update.py \ - --automation_user 'SNOWFLAKE_TASK_AUTOMATION_USER' \ - --account 'edx.us-east-1' \ - --amplitude_data_source_table $AMPLITUDE_DATA_SOURCE_TABLE \ - --columns_to_update $COLUMNS_TO_UPDATE \ - --response_table $RESPONSE_TABLE \ - --amplitude_operation_name $AMPLITUDE_OPERATION_NAME \ - --amplitude_api_key $API_KEY \ - --key_file rsa_key_snowflake_task_automation_user \ - --passphrase_file rsa_key_passphrase_snowflake_task_automation_user - -rm rsa_key_snowflake_task_automation_user -rm rsa_key_passphrase_snowflake_task_automation_user