diff --git a/dataeng/resources/run-pipeline-acceptance-test.sh b/dataeng/resources/run-pipeline-acceptance-test.sh index f54932528..f800f43f2 100644 --- a/dataeng/resources/run-pipeline-acceptance-test.sh +++ b/dataeng/resources/run-pipeline-acceptance-test.sh @@ -26,6 +26,7 @@ export COURSE_EXPORTER=$EXPORTER_BIN/course-exporter # Exporter configuration destination +source secrets-manager.sh analytics-secure/analytics-exporter/task-auth.json TASK_AUTH ROOT=${WORKSPACE}/analytics-secure/analytics-exporter SECURE_HASH=`GIT_DIR=./analytics-secure/.git git rev-parse HEAD` EXPORTER_CONFIG_BUCKET=$EXPORTER_BUCKET_PATH/$SECURE_HASH diff --git a/dataeng/resources/secrets-manager.sh b/dataeng/resources/secrets-manager.sh new file mode 100755 index 000000000..880a1010f --- /dev/null +++ b/dataeng/resources/secrets-manager.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +secret_to_call="$1" +secret_name="$2" +set +x + +SECRET_JSON=$(aws secretsmanager get-secret-value --secret-id $secret_to_call --region "us-east-1" --output json) +# Check the exit status of the AWS CLI command + +echo "$SECRET_JSON" +extract_and_store_secret_value() { + + value=$(echo "$SECRET_JSON" | jq -r ".SecretString | fromjson.$secret_name" 2>/dev/null) + eval "$secret_name"='$value' +} + +if [ $? -eq 0 ]; then + # Use jq to extract the values from the JSON response + extract_and_store_secret_value $SECRET_JSON $secret_name +else + echo "AWS CLI command failed" +fi diff --git a/dataeng/resources/snowflake-collect-metrics.sh b/dataeng/resources/snowflake-collect-metrics.sh index 0f18f753a..6b3dbd677 100644 --- a/dataeng/resources/snowflake-collect-metrics.sh +++ b/dataeng/resources/snowflake-collect-metrics.sh @@ -10,6 +10,12 @@ source "${PYTHON_VENV}/bin/activate" cd $WORKSPACE/analytics-tools/snowflake make requirements +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_COLLECT_METRICS_JOB_EXTRA_VARS KEY_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_COLLECT_METRICS_JOB_EXTRA_VARS PASSPHRASE_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_COLLECT_METRICS_JOB_EXTRA_VARS USER +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_COLLECT_METRICS_JOB_EXTRA_VARS ACCOUNT +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_COLLECT_METRICS_JOB_EXTRA_VARS METRIC_NAME + python collect-metrics.py \ --metric_name $METRIC_NAME \ --key_path $WORKSPACE/analytics-secure/snowflake/rsa_key_snowflake_task_automation_user.p8 \ diff --git a/dataeng/resources/snowflake-demographics-cleanup.sh b/dataeng/resources/snowflake-demographics-cleanup.sh index 7f86c757f..aba33c64b 100644 --- a/dataeng/resources/snowflake-demographics-cleanup.sh +++ b/dataeng/resources/snowflake-demographics-cleanup.sh @@ -10,6 +10,11 @@ source "${PYTHON_VENV}/bin/activate" cd $WORKSPACE/analytics-tools/snowflake make requirements +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_DEMOGRAPHICS_CLEANUP_JOB_EXTRA_VARS KEY_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_DEMOGRAPHICS_CLEANUP_JOB_EXTRA_VARS PASSPHRASE_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_DEMOGRAPHICS_CLEANUP_JOB_EXTRA_VARS USER +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_DEMOGRAPHICS_CLEANUP_JOB_EXTRA_VARS ACCOUNT + python demographics_cleanup.py \ --key_path $WORKSPACE/analytics-secure/$KEY_PATH \ --passphrase_path $WORKSPACE/analytics-secure/$PASSPHRASE_PATH \ diff --git a/dataeng/resources/snowflake-public-grants-cleaner.sh b/dataeng/resources/snowflake-public-grants-cleaner.sh index 4fb013ff6..6e6374229 100644 --- a/dataeng/resources/snowflake-public-grants-cleaner.sh +++ b/dataeng/resources/snowflake-public-grants-cleaner.sh @@ -10,6 +10,11 @@ source "${PYTHON_VENV}/bin/activate" cd $WORKSPACE/analytics-tools/snowflake make requirements +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_PUBLIC_GRANTS_CLEANER_JOB_EXTRA_VARS KEY_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_PUBLIC_GRANTS_CLEANER_JOB_EXTRA_VARS PASSPHRASE_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_PUBLIC_GRANTS_CLEANER_JOB_EXTRA_VARS USER +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_PUBLIC_GRANTS_CLEANER_JOB_EXTRA_VARS ACCOUNT + python snowflake_public_grants_cleaner.py \ --key_path $WORKSPACE/analytics-secure/$KEY_PATH \ --passphrase_path $WORKSPACE/analytics-secure/$PASSPHRASE_PATH \ diff --git a/dataeng/resources/snowflake-refresh-snowpipe.sh b/dataeng/resources/snowflake-refresh-snowpipe.sh index d288c83e3..ff3985c7a 100644 --- a/dataeng/resources/snowflake-refresh-snowpipe.sh +++ b/dataeng/resources/snowflake-refresh-snowpipe.sh @@ -10,6 +10,17 @@ source "${PYTHON_VENV}/bin/activate" cd $WORKSPACE/analytics-tools/snowflake make requirements +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS KEY_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS PASSPHRASE_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS USER +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS ACCOUNT +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS SCHEMA +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS PIPE_NAME +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS TABLE_NAME +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS DELAY +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_REFRESH_SNOWPIPE_JOB_EXTRA_VARS LIMIT + + python refresh_snowpipe.py \ --key_path $WORKSPACE/analytics-secure/$KEY_PATH \ --passphrase_path $WORKSPACE/analytics-secure/$PASSPHRASE_PATH \ diff --git a/dataeng/resources/snowflake-user-retirement-status-cleanup.sh b/dataeng/resources/snowflake-user-retirement-status-cleanup.sh old mode 100644 new mode 100755 index 7f8c526df..b4507b93e --- a/dataeng/resources/snowflake-user-retirement-status-cleanup.sh +++ b/dataeng/resources/snowflake-user-retirement-status-cleanup.sh @@ -10,6 +10,11 @@ source "${PYTHON_VENV}/bin/activate" cd $WORKSPACE/analytics-tools/snowflake make requirements +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_USER_RETIREMENT_STATUS_CLEANUP_JOB_EXTRA_VARS KEY_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_USER_RETIREMENT_STATUS_CLEANUP_JOB_EXTRA_VARS PASSPHRASE_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_USER_RETIREMENT_STATUS_CLEANUP_JOB_EXTRA_VARS USER +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_USER_RETIREMENT_STATUS_CLEANUP_JOB_EXTRA_VARS ACCOUNT + python retirement_cleanup.py \ --key_path $WORKSPACE/analytics-secure/$KEY_PATH \ --passphrase_path $WORKSPACE/analytics-secure/$PASSPHRASE_PATH \ diff --git a/dataeng/resources/snowflake-validate-stitch.sh b/dataeng/resources/snowflake-validate-stitch.sh index 2017e1d94..5f75ec293 100644 --- a/dataeng/resources/snowflake-validate-stitch.sh +++ b/dataeng/resources/snowflake-validate-stitch.sh @@ -14,6 +14,11 @@ COMPARISON_START_TIME=$(date --utc --iso=minutes -d "${COMPARISON_END_TIME} - 15 cd $WORKSPACE/analytics-tools/snowflake make requirements +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_VALIDATE_STITCH_JOB_EXTRA_VARS KEY_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_VALIDATE_STITCH_JOB_EXTRA_VARS PASSPHRASE_PATH +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_VALIDATE_STITCH_JOB_EXTRA_VARS USER +source secrets-manager.sh analytics-secure/job-configs/SNOWFLAKE_VALIDATE_STITCH_JOB_EXTRA_VARS ACCOUNT + python stitch_vs_sqoop_validation.py \ --key_path $WORKSPACE/analytics-secure/${SNOWFLAKE_KEY_PATH} \ --passphrase_path $WORKSPACE/analytics-secure/${SNOWFLAKE_PASSPHRASE_PATH} \ diff --git a/dataeng/resources/stitch-snowflake-lag-monitor.sh b/dataeng/resources/stitch-snowflake-lag-monitor.sh index 1cdd7804e..d21581374 100644 --- a/dataeng/resources/stitch-snowflake-lag-monitor.sh +++ b/dataeng/resources/stitch-snowflake-lag-monitor.sh @@ -10,6 +10,11 @@ source "${PYTHON_VENV}/bin/activate" cd $WORKSPACE/analytics-tools/snowflake make requirements +source secrets-manager.sh analytics-secure/job-configs/STITCH_SNOWFLAKE_LAG_MONITOR_JOB_EXTRA_VARS KEY_PATH +source secrets-manager.sh analytics-secure/job-configs/STITCH_SNOWFLAKE_LAG_MONITOR_JOB_EXTRA_VARS PASSPHRASE_PATH +source secrets-manager.sh analytics-secure/job-configs/STITCH_SNOWFLAKE_LAG_MONITOR_JOB_EXTRA_VARS USER +source secrets-manager.sh analytics-secure/job-configs/STITCH_SNOWFLAKE_LAG_MONITOR_JOB_EXTRA_VARS ACCOUNT + python stitch-snowflake-monitoring.py \ --key_path $WORKSPACE/analytics-secure/$KEY_PATH \ --passphrase_path $WORKSPACE/analytics-secure/$PASSPHRASE_PATH \