update to support the additional query col

CDCgov · Dec 11, 2024 · 4c304c8 · 4c304c8
1 parent cd8c98b
commit 4c304c8
Show file tree

Hide file tree

Showing 5 changed files with 57 additions and 9 deletions.
diff --git a/DataCompareAPIs/src/main/java/gov/cdc/datacompareapis/controller/DataCompareController.java b/DataCompareAPIs/src/main/java/gov/cdc/datacompareapis/controller/DataCompareController.java
@@ -36,7 +36,12 @@ public DataCompareController(IDataPullerService dataPullerService) {
                             description = "Flag indicating max records per pulling process",
                             schema = @Schema(type = "Integer", defaultValue = "1000"),
                             required = true),
-                    @Parameter(in = ParameterIn.QUERY,
+                    @Parameter(in = ParameterIn.HEADER,
+                            name = "autoApply",
+                            description = "Flag whether allow the service to automatically determine when to apply the run now mode",
+                            schema = @Schema(type = "Boolean", defaultValue = "false"),
+                            required = true),
+                    @Parameter(in = ParameterIn.HEADER,
                             name = "runNowMode",
                             description = "Boolean flag to initiate the run immediately",
                             schema = @Schema(type = "Boolean", defaultValue = "false"))
@@ -45,10 +50,11 @@ public DataCompareController(IDataPullerService dataPullerService) {
     @GetMapping(path = "/api/data-compare")
     public ResponseEntity<String> dataSyncTotalRecords(
             @RequestHeader(name = "batchLimit", defaultValue = "1000") String batchLimit,
-            @RequestHeader(name = "runNowMode", defaultValue = "false") boolean runNowMode) throws DataCompareException {
+            @RequestHeader(name = "runNowMode", defaultValue = "false") boolean runNowMode,
+            @RequestHeader(name = "autoApply", defaultValue = "false") boolean autoApply) throws DataCompareException {
 
         if (isInteger(batchLimit)) {
-            dataPullerService.pullingData(Integer.parseInt(batchLimit), runNowMode);
+            dataPullerService.pullingData(Integer.parseInt(batchLimit), runNowMode, autoApply);
             return new ResponseEntity<>("OK", HttpStatus.OK);
         }
         throw new DataCompareException("Invalid Header");

diff --git a/...src/main/java/gov/cdc/datacompareapis/repository/dataCompare/model/DataCompareConfig.java b/...src/main/java/gov/cdc/datacompareapis/repository/dataCompare/model/DataCompareConfig.java
@@ -34,6 +34,14 @@ public class DataCompareConfig {
     @Column(name = "query_count")
     private String queryCount;
 
+    @Column(name = "query_rtr")
+    private String queryRtr;
+
+    @Column(name = "query_rtr_count")
+    private String queryRtrCount;
+
+
+
 
     @Column(name = "key_column_list")
     private String keyColumns;

diff --git a/DataCompareAPIs/src/main/java/gov/cdc/datacompareapis/service/DataPullerService.java b/DataCompareAPIs/src/main/java/gov/cdc/datacompareapis/service/DataPullerService.java
@@ -62,19 +62,37 @@ public DataPullerService(DataCompareConfigRepository dataCompareConfigRepository
     }
 
     @Async("defaultAsyncExecutor")
-    public void pullingData(int pullLimit, boolean runNowMode)   {
+    public void pullingData(int pullLimit, boolean runNowMode, boolean autoApply)   {
         List<DataCompareConfig>  configs = getDataConfig();
+        if (autoApply) {
+            pullDataAutomaticApplyingCondition(pullLimit, configs);
+        }
+        else {
+            if (runNowMode) {
+                pullDataRunNow(pullLimit, configs);
+            }
+            else {
+                pullDataAllTable(pullLimit, configs);
+            }
+
+        }
+
+        logger.info("PULLER IS COMPLETED FOR ALL TABLE");
+    }
 
-        if (runNowMode) {
+    protected void pullDataAutomaticApplyingCondition (int pullLimit, List<DataCompareConfig>  configs) {
+        boolean runNowOpt = configs.stream().anyMatch(DataCompareConfig::getRunNow);
+        if (runNowOpt) {
             pullDataRunNow(pullLimit, configs);
         }
         else {
             pullDataAllTable(pullLimit, configs);
         }
 
-        logger.info("PULLER IS COMPLETED FOR ALL TABLE");
     }
 
+
+
     protected void pullDataAllTable(int pullLimit, List<DataCompareConfig>  configs) {
         for(DataCompareConfig config : configs) {
             if (config.getCompare()) {
@@ -110,7 +128,15 @@ protected void pullDataLogic(int pullLimit, DataCompareConfig config ) {
         DataCompareLog dataCompareLogRdbModern =dataCompareDefaultLogBuilder(config, currentTime, "Inprogress", "RDB_MODERN");
 
         Integer rdbCount = rdbJdbcTemplate.queryForObject(config.getQueryCount(), Integer.class);
-        Integer rdbModernCount = rdbModernJdbcTemplate.queryForObject(config.getQueryCount(), Integer.class);
+        Integer rdbModernCount = 0;
+        if (config.getQueryRtrCount() != null && !config.getQueryRtrCount().isEmpty())
+        {
+            rdbModernCount = rdbModernJdbcTemplate.queryForObject(config.getQueryRtrCount(), Integer.class);
+        }
+        else
+        {
+            rdbModernCount = rdbModernJdbcTemplate.queryForObject(config.getQueryCount(), Integer.class);
+        }
         boolean errorDuringPullingDataRdb = false;
         String stackTraceRdb = null;
         boolean errorDuringPullingDataRdbModern = false;
@@ -145,7 +171,13 @@ protected void pullDataLogic(int pullLimit, DataCompareConfig config ) {
                     }
                     int startRow = i * pullLimit + 1;
                     int endRow = (i + 1) * pullLimit;
-                    String query = preparingPaginationQuery(config.getQuery(), startRow, endRow);
+                    String query = "";
+                    if (config.getQueryRtr() != null && !config.getQueryRtr().isEmpty()) {
+                        preparingPaginationQuery(config.getQueryRtr(), startRow, endRow);
+                    }
+                    else {
+                        preparingPaginationQuery(config.getQuery(), startRow, endRow);
+                    }
                     List<Map<String, Object>> returnData = executeQueryForData(query, config.getTargetDb());
                     String rawJsonData = gson.toJson(returnData);
                     s3DataService.persistToS3MultiPart(config.getTargetDb(),rawJsonData, config.getTableName(), currentTime, i);

diff --git a/...pareAPIs/src/main/java/gov/cdc/datacompareapis/service/interfaces/IDataPullerService.java b/...pareAPIs/src/main/java/gov/cdc/datacompareapis/service/interfaces/IDataPullerService.java
@@ -1,5 +1,5 @@
 package gov.cdc.datacompareapis.service.interfaces;
 
 public interface IDataPullerService {
-     void pullingData(int pullLimit, boolean runNowMode);
+     void pullingData(int pullLimit, boolean runNowMode, boolean autoApply);
 }
diff --git a/DataCompareAPIs/src/main/resources/sql/dataCompareConfig.sql b/DataCompareAPIs/src/main/resources/sql/dataCompareConfig.sql
@@ -5,7 +5,9 @@ CREATE TABLE Data_Compare_Config
     source_db VARCHAR(100),
     target_db VARCHAR(100),
     query VARCHAR(MAX),
+    query_rtr VARCHAR(MAX),
     query_count VARCHAR(MAX),
+    query_rtr_count VARCHAR(MAX),
     key_column_list VARCHAR(MAX),
     ignore_column_list VARCHAR(MAX),
     compare BIT DEFAULT 0,