From 117b8d1fb03d5c62ca3c42f8d2cb4cd9e4b9e079 Mon Sep 17 00:00:00 2001 From: Duc Nguyen <117302958+ndduc01@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:23:30 -0500 Subject: [PATCH 1/3] update query for data config --- .../sql/dataCompareDataGeneration.sql | 580 +++++++++++++++++- .../constant/MoreThanOneCountTableEnum.java | 6 + 2 files changed, 585 insertions(+), 1 deletion(-) create mode 100644 DataCompareProcessor/src/main/java/gov/cdc/datacompareprocessor/constant/MoreThanOneCountTableEnum.java diff --git a/DataCompareAPIs/src/main/resources/sql/dataCompareDataGeneration.sql b/DataCompareAPIs/src/main/resources/sql/dataCompareDataGeneration.sql index 776adab..0a70116 100644 --- a/DataCompareAPIs/src/main/resources/sql/dataCompareDataGeneration.sql +++ b/DataCompareAPIs/src/main/resources/sql/dataCompareDataGeneration.sql @@ -15,4 +15,582 @@ values ('D_PATIENT', 'RDB', 'RDB_MODERN', ', 'PATIENT_UID', 'RowNum, PATIENT_KEY, PATIENT_LAST_CHANGE_TIME, PATIENT_ADD_TIME', - 1); \ No newline at end of file + 1), + ( + 'D_ORGANIZATION', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_ORGANIZATION.*, + ROW_NUMBER() OVER (ORDER BY D_ORGANIZATION.ORGANIZATION_UID ASC) AS RowNum + FROM D_ORGANIZATION + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_ORGANIZATION;', + 'ORGANIZATION_UID', + 'RowNum, ORGANIZATION_KEY, ORGANIZATION_LAST_CHANGE_TIME, ORGANIZATION_ADD_TIME', + 1 + ), + ( + 'D_PROVIDER', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_PROVIDER.*, + ROW_NUMBER() OVER (ORDER BY D_PROVIDER.PROVIDER_UID ASC) AS RowNum + FROM D_PROVIDER + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_PROVIDER;', + 'PROVIDER_UID', + 'RowNum, PROVIDER_KEY, PROVIDER_LAST_CHANGE_TIME, PROVIDER_ADD_TIME', + 1 + ), + ( + 'RDB_DATE', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT RDB_DATE.*, + ROW_NUMBER() OVER (ORDER BY RDB_DATE.DATE_MM_DD_YYYY ASC) AS RowNum + FROM RDB_DATE + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM RDB_DATE;', + 'DATE_MM_DD_YYYY', + 'RowNum', + 1 + ), + ( + 'CONFIRMATION_METHOD_GROUP', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT CONFIRMATION_METHOD_GROUP.*, + ROW_NUMBER() OVER (ORDER BY CONFIRMATION_METHOD_GROUP.INVESTIGATION_KEY ASC) AS RowNum + FROM CONFIRMATION_METHOD_GROUP + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM CONFIRMATION_METHOD_GROUP;', + 'INVESTIGATION_KEY', + 'RowNum, CONFIRMATION_DT', + 1 + ), + ( + 'CONFIRMATION_METHOD', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT CONFIRMATION_METHOD.*, + ROW_NUMBER() OVER (ORDER BY CONFIRMATION_METHOD.CONFIRMATION_METHOD_CD ASC) AS RowNum + FROM CONFIRMATION_METHOD + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM CONFIRMATION_METHOD;', + 'CONFIRMATION_METHOD_CD', + 'RowNum', + 1 + ), + ( + 'INVESTIGATION', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT INVESTIGATION.*, + ROW_NUMBER() OVER (ORDER BY INVESTIGATION.CASE_UID ASC) AS RowNum + FROM INVESTIGATION + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM INVESTIGATION;', + 'CASE_UID', + 'RowNum, INVESTIGATION_KEY, ADD_TIME, LAST_CHG_TIME', + 1 + ), + ( + 'LDF_GROUP', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT LDF_GROUP.*, + ROW_NUMBER() OVER (ORDER BY LDF_GROUP.BUSINESS_OBJECT_UID ASC) AS RowNum + FROM LDF_GROUP + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM LDF_GROUP;', + 'BUSINESS_OBJECT_UID', + 'RowNum', + 1 + ), + ( + 'D_INV_ADMINISTRATIVE', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_ADMINISTRATIVE.*, + ROW_NUMBER() OVER (ORDER BY D_INV_ADMINISTRATIVE.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_ADMINISTRATIVE + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_ADMINISTRATIVE;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_ADMINISTRATIVE_KEY', + 1 + ), + ( + 'D_INV_CLINICAL', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_CLINICAL.*, + ROW_NUMBER() OVER (ORDER BY D_INV_CLINICAL.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_CLINICAL + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_CLINICAL;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_CLINICAL_KEY', + 1 + ), + ( + 'D_INV_COMPLICATION', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_COMPLICATION.*, + ROW_NUMBER() OVER (ORDER BY D_INV_COMPLICATION.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_COMPLICATION + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_COMPLICATION;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_COMPLICATION_KEY', + 1 + ), + ( + 'D_INV_CONTACT', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_CONTACT.*, + ROW_NUMBER() OVER (ORDER BY D_INV_CONTACT.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_CONTACT + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_CONTACT;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_CONTACT_KEY', + 1 + ), + ( + 'D_INV_DEATH', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_DEATH.*, + ROW_NUMBER() OVER (ORDER BY D_INV_DEATH.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_DEATH + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_DEATH;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_DEATH_KEY', + 1 + ), + ( + 'D_INV_EPIDEMIOLOGY', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_EPIDEMIOLOGY.*, + ROW_NUMBER() OVER (ORDER BY D_INV_EPIDEMIOLOGY.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_EPIDEMIOLOGY + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_EPIDEMIOLOGY;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_EPIDEMIOLOGY_KEY', + 1 + ), + ( + 'D_INV_HIV', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_HIV.*, + ROW_NUMBER() OVER (ORDER BY D_INV_HIV.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_HIV + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_HIV;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_HIV_KEY', + 1 + ), + ( + 'D_INV_ISOLATE_TRACKING', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_ISOLATE_TRACKING.*, + ROW_NUMBER() OVER (ORDER BY D_INV_ISOLATE_TRACKING.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_ISOLATE_TRACKING + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_ISOLATE_TRACKING;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_ISOLATE_TRACKING_KEY', + 1 + ), + ( + 'D_INV_LAB_FINDING', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_LAB_FINDING.*, + ROW_NUMBER() OVER (ORDER BY D_INV_LAB_FINDING.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_LAB_FINDING + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_LAB_FINDING;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_LAB_FINDING_KEY', + 1 + ), + ( + 'D_INV_MEDICAL_HISTORY', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_MEDICAL_HISTORY.*, + ROW_NUMBER() OVER (ORDER BY D_INV_MEDICAL_HISTORY.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_MEDICAL_HISTORY + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_MEDICAL_HISTORY;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_MEDICAL_HISTORY_KEY', + 1 + ), + ( + 'D_INV_MOTHER', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_MOTHER.*, + ROW_NUMBER() OVER (ORDER BY D_INV_MOTHER.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_MOTHER + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_MOTHER;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_MOTHER_KEY', + 1 + ), + ( + 'D_INV_OTHER', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_OTHER.*, + ROW_NUMBER() OVER (ORDER BY D_INV_OTHER.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_OTHER + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_OTHER;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_OTHER_KEY', + 1 + ), + ( + 'D_INV_PATIENT_OBS', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_PATIENT_OBS.*, + ROW_NUMBER() OVER (ORDER BY D_INV_PATIENT_OBS.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_PATIENT_OBS + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_PATIENT_OBS;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_PATIENT_OBS_KEY', + 1 + ), + ( + 'D_INV_PREGNANCY_BIRTH', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_PREGNANCY_BIRTH.*, + ROW_NUMBER() OVER (ORDER BY D_INV_PREGNANCY_BIRTH.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_PREGNANCY_BIRTH + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_PREGNANCY_BIRTH;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_PREGNANCY_BIRTH_KEY', + 1 + ), + ( + 'D_INV_RESIDENCY', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_RESIDENCY.*, + ROW_NUMBER() OVER (ORDER BY D_INV_RESIDENCY.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_RESIDENCY + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_RESIDENCY;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_RESIDENCY_KEY', + 1 + ), + ( + 'D_INV_RISK_FACTOR', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_RISK_FACTOR.*, + ROW_NUMBER() OVER (ORDER BY D_INV_RISK_FACTOR.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_RISK_FACTOR + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_RISK_FACTOR;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_RISK_FACTOR_KEY', + 1 + ), + ( + 'D_INV_SOCIAL_HISTORY', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_SOCIAL_HISTORY.*, + ROW_NUMBER() OVER (ORDER BY D_INV_SOCIAL_HISTORY.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_SOCIAL_HISTORY + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_SOCIAL_HISTORY;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_SOCIAL_HISTORY_KEY', + 1 + ), + ( + 'D_INV_SYMPTOM', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_SYMPTOM.*, + ROW_NUMBER() OVER (ORDER BY D_INV_SYMPTOM.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_SYMPTOM + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_SYMPTOM;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_SYMPTOM_KEY', + 1 + ), + ( + 'D_INV_TRAVEL', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_TRAVEL.*, + ROW_NUMBER() OVER (ORDER BY D_INV_TRAVEL.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_TRAVEL + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_TRAVEL;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_TRAVEL_KEY', + 1 + ), + ( + 'D_INV_TREATMENT', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_TREATMENT.*, + ROW_NUMBER() OVER (ORDER BY D_INV_TREATMENT.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_TREATMENT + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_TREATMENT;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_TREATMENT_KEY', + 1 + ), + ( + 'D_INV_UNDER_CONDITION', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_UNDER_CONDITION.*, + ROW_NUMBER() OVER (ORDER BY D_INV_UNDER_CONDITION.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_UNDER_CONDITION + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_UNDER_CONDITION;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_UNDER_CONDITION_KEY', + 1 + ), + ( + 'D_INV_VACCINATION', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT D_INV_VACCINATION.*, + ROW_NUMBER() OVER (ORDER BY D_INV_VACCINATION.nbs_case_answer_uid ASC) AS RowNum + FROM D_INV_VACCINATION + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM D_INV_VACCINATION;', + 'nbs_case_answer_uid', + 'RowNum, D_INV_VACCINATION_KEY', + 1 + ), + ( + 'HEPATITIS_DATAMART', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT DISTINCT HEPATITIS_DATAMART.*, + CONCAT(inv_local_id, pat_local_id, patient_uid, case_uid) AS CONCAT_KEY, + ROW_NUMBER() OVER (ORDER BY CONCAT(inv_local_id, pat_local_id, patient_uid, case_uid) ASC) AS RowNum + FROM HEPATITIS_DATAMART + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(DISTINCT CONCAT(inv_local_id, pat_local_id, patient_uid, case_uid)) + FROM HEPATITIS_DATAMART;', + 'CONCAT_KEY', + 'RowNum, REFRESH_DATETIME', + 1 + ), + ( + 'D_INVESTIGATION_REPEAT', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT DISTINCT D_INVESTIGATION_REPEAT.*, + CONCAT(page_case_uid, block_nm) AS CONCAT_KEY, + ROW_NUMBER() OVER (ORDER BY CONCAT(page_case_uid, block_nm) ASC) AS RowNum + FROM D_INVESTIGATION_REPEAT + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(DISTINCT CONCAT(page_case_uid, block_nm)) + FROM D_INVESTIGATION_REPEAT;', + 'CONCAT_KEY', + 'RowNum, D_INVESTIGATION_REPEAT_KEY', + 1 + ), + ( + 'NOTIFICATION', + 'RDB', + 'RDB_MODERN', + 'WITH PaginatedResults AS ( + SELECT DISTINCT NOTIFICATION.*, + ROW_NUMBER() OVER (ORDER BY NOTIFICATION.notification_local_id ASC) AS RowNum + FROM NOTIFICATION + ) + SELECT * + FROM PaginatedResults + WHERE RowNum BETWEEN :startRow AND :endRow;', + 'SELECT COUNT(*) + FROM NOTIFICATION;', + 'notification_local_id', + 'RowNum, NOTIFICATION_LAST_CHANGE_TIME', + 1 + ) \ No newline at end of file diff --git a/DataCompareProcessor/src/main/java/gov/cdc/datacompareprocessor/constant/MoreThanOneCountTableEnum.java b/DataCompareProcessor/src/main/java/gov/cdc/datacompareprocessor/constant/MoreThanOneCountTableEnum.java new file mode 100644 index 0000000..f61c12c --- /dev/null +++ b/DataCompareProcessor/src/main/java/gov/cdc/datacompareprocessor/constant/MoreThanOneCountTableEnum.java @@ -0,0 +1,6 @@ +package gov.cdc.datacompareprocessor.constant; + +public enum MoreThanOneCountTableEnum { + NOTIFICATION, + D_INVESTIGATION_REPEAT +} From f2aea751dd4a8e9680672352f2ccc455897c0638 Mon Sep 17 00:00:00 2001 From: Duc Nguyen <117302958+ndduc01@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:44:18 -0500 Subject: [PATCH 2/3] update with runNow --- .../controller/DataCompareController.java | 2 +- .../datacompareapis/service/S3DataService.java | 12 ++++++++++-- .../src/main/resources/application.yaml | 2 ++ .../service/S3DataPullerService.java | 16 +++++++++++++--- .../src/main/resources/application.yaml | 2 ++ 5 files changed, 28 insertions(+), 6 deletions(-) diff --git a/DataCompareAPIs/src/main/java/gov/cdc/datacompareapis/controller/DataCompareController.java b/DataCompareAPIs/src/main/java/gov/cdc/datacompareapis/controller/DataCompareController.java index 6a819db..1bfff2f 100644 --- a/DataCompareAPIs/src/main/java/gov/cdc/datacompareapis/controller/DataCompareController.java +++ b/DataCompareAPIs/src/main/java/gov/cdc/datacompareapis/controller/DataCompareController.java @@ -45,7 +45,7 @@ public DataCompareController(IDataPullerService dataPullerService) { @GetMapping(path = "/api/data-compare") public ResponseEntity dataSyncTotalRecords( @RequestHeader(name = "batchLimit", defaultValue = "1000") String batchLimit, - @RequestParam(name = "runNowMode", defaultValue = "false") boolean runNowMode) throws DataCompareException { + @RequestHeader(name = "runNowMode", defaultValue = "false") boolean runNowMode) throws DataCompareException { if (isInteger(batchLimit)) { dataPullerService.pullingData(Integer.parseInt(batchLimit), runNowMode); diff --git a/DataCompareAPIs/src/main/java/gov/cdc/datacompareapis/service/S3DataService.java b/DataCompareAPIs/src/main/java/gov/cdc/datacompareapis/service/S3DataService.java index 176d846..8ee1687 100644 --- a/DataCompareAPIs/src/main/java/gov/cdc/datacompareapis/service/S3DataService.java +++ b/DataCompareAPIs/src/main/java/gov/cdc/datacompareapis/service/S3DataService.java @@ -8,6 +8,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; @@ -39,10 +40,17 @@ public S3DataService( @Value("${aws.auth.static.access_key}") String accessKey, @Value("${aws.auth.static.token}") String token, @Value("${aws.s3.region}") String region, - @Value("${aws.auth.profile.profile_name}") String profile + @Value("${aws.auth.profile.profile_name}") String profile, + @Value("${aws.auth.iam.enabled}") boolean iamEnable ) throws DataCompareException { - if (!keyId.isEmpty() && !accessKey.isEmpty() && !token.isEmpty()) { + if (iamEnable) { + this.s3Client = S3Client.builder() + .region(Region.of(region)) + .credentialsProvider(DefaultCredentialsProvider.create()) // Automatically retrieves IAM role credentials + .build(); + } + else if (!keyId.isEmpty() && !accessKey.isEmpty() && !token.isEmpty()) { this.s3Client = S3Client.builder() .region(Region.of(region)) .credentialsProvider(StaticCredentialsProvider.create( diff --git a/DataCompareAPIs/src/main/resources/application.yaml b/DataCompareAPIs/src/main/resources/application.yaml index 239f29d..9ddcc42 100644 --- a/DataCompareAPIs/src/main/resources/application.yaml +++ b/DataCompareAPIs/src/main/resources/application.yaml @@ -37,6 +37,8 @@ aws: token: ${AWS_TOKEN:NA} profile: profile_name: ${AWS_PROFILE:NA} + iam: + enabled: ${AWS_IAM:false} s3: bucket-name: ${S3_BUCKET_NAME:NA} region: ${S3_REGION:NA} diff --git a/DataCompareProcessor/src/main/java/gov/cdc/datacompareprocessor/service/S3DataPullerService.java b/DataCompareProcessor/src/main/java/gov/cdc/datacompareprocessor/service/S3DataPullerService.java index 3b3affc..252d7d6 100644 --- a/DataCompareProcessor/src/main/java/gov/cdc/datacompareprocessor/service/S3DataPullerService.java +++ b/DataCompareProcessor/src/main/java/gov/cdc/datacompareprocessor/service/S3DataPullerService.java @@ -11,8 +11,10 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.security.core.parameters.P; import org.springframework.stereotype.Service; import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.ResponseBytes; @@ -42,10 +44,17 @@ public S3DataPullerService( @Value("${aws.auth.static.access_key}") String accessKey, @Value("${aws.auth.static.token}") String token, @Value("${aws.s3.region}") String region, - @Value("${aws.auth.profile.profile_name}") String profile + @Value("${aws.auth.profile.profile_name}") String profile, + @Value("${aws.auth.iam.enabled}") boolean iamEnable ) throws DataProcessorException { - if (!keyId.isEmpty() && !accessKey.isEmpty() && !token.isEmpty()) { + if (iamEnable) { + this.s3Client = S3Client.builder() + .region(Region.of(region)) + .credentialsProvider(DefaultCredentialsProvider.create()) // Automatically retrieves IAM role credentials + .build(); + } + else if (!keyId.isEmpty() && !accessKey.isEmpty() && !token.isEmpty()) { this.s3Client = S3Client.builder() .region(Region.of(region)) .credentialsProvider(StaticCredentialsProvider.create( @@ -57,7 +66,8 @@ public S3DataPullerService( .region(Region.of(region)) .credentialsProvider(ProfileCredentialsProvider.create(profile)) .build(); - } else { + } + else { throw new DataProcessorException("No Valid AWS Profile or Credentials found"); } this.gson = new GsonBuilder() diff --git a/DataCompareProcessor/src/main/resources/application.yaml b/DataCompareProcessor/src/main/resources/application.yaml index bdd1ce8..c425349 100644 --- a/DataCompareProcessor/src/main/resources/application.yaml +++ b/DataCompareProcessor/src/main/resources/application.yaml @@ -38,6 +38,8 @@ aws: token: ${AWS_TOKEN:NA} profile: profile_name: ${AWS_PROFILE:NA} + iam: + enabled: ${AWS_IAM:false} s3: bucket-name: ${S3_BUCKET_NAME:NA} region: ${S3_REGION:NA} From 64c638e07e7db913a21d00e6b618b1fd2d592f2c Mon Sep 17 00:00:00 2001 From: Duc Nguyen <117302958+ndduc01@users.noreply.github.com> Date: Mon, 25 Nov 2024 16:52:01 -0500 Subject: [PATCH 3/3] update --- .../cdc/datacompareprocessor/service/DataCompareService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/DataCompareProcessor/src/main/java/gov/cdc/datacompareprocessor/service/DataCompareService.java b/DataCompareProcessor/src/main/java/gov/cdc/datacompareprocessor/service/DataCompareService.java index 5a44abc..39b81f6 100644 --- a/DataCompareProcessor/src/main/java/gov/cdc/datacompareprocessor/service/DataCompareService.java +++ b/DataCompareProcessor/src/main/java/gov/cdc/datacompareprocessor/service/DataCompareService.java @@ -543,10 +543,11 @@ protected Map loadJsonAsMapFromS3(String fileName, String un Map recordMap = new HashMap<>(); try { - // Convert JsonArray into a map using the unique identifier field for (JsonElement element : jsonElement.getAsJsonArray()) { JsonObject jsonObject = element.getAsJsonObject(); - String id = jsonObject.get(uniqueIdField).getAsString(); + String id = jsonObject.has(uniqueIdField) && !jsonObject.get(uniqueIdField).isJsonNull() + ? jsonObject.get(uniqueIdField).getAsString() + : "NULL"; recordMap.put(id, jsonObject); } } catch (Exception e) {