Skip to content

Commit 4a6e3da

Browse files
ndduc01ndduc01
committed
added some additional logic for comparing
1 parent 2360bb1 commit 4a6e3da

File tree

3 files changed

+142
-23
lines changed

3 files changed

+142
-23
lines changed

‎DataCompareAPIs/src/main/resources/sql/dataCompareDataGenerationDummy.sql

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
INSERT INTO Data_Compare_Config
2+
(table_name, source_db, target_db, query, query_count, key_column_list, ignore_column_list, compare)
3+
VALUES ('dc_test', 'RDB', 'RDB_MODERN',
4+
'WITH PaginatedResults AS (
5+
SELECT dc_test.*,
6+
ROW_NUMBER() OVER (ORDER BY dc_test.id ASC) AS RowNum
7+
FROM dc_test
8+
)
9+
SELECT *
10+
FROM PaginatedResults
11+
WHERE RowNum BETWEEN :startRow AND :endRow;
12+
',
13+
'SELECT COUNT(*)
14+
FROM dc_test;
15+
',
16+
'id',
17+
'RowNum',
18+
1);

‎DataCompareProcessor/src/main/java/gov/cdc/datacompareprocessor/service/DataCompareService.java

Lines changed: 123 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
import java.lang.reflect.Type;
2323
import java.sql.Timestamp;
2424
import java.util.*;
25+
import java.util.function.Predicate;
26+
import java.util.stream.Collectors;
2527

2628
import static gov.cdc.datacompareprocessor.share.StackTraceUtil.getStackTraceAsString;
2729
import static gov.cdc.datacompareprocessor.share.StringHelper.convertStringToList;
@@ -125,6 +127,13 @@ public void processingData(PullerEventModel pullerEventModel) {
125127
var remainList = processingRemainingData( ignoreColList, pullerEventModel.getKeyColumn(), pullerEventModel.getFileName());
126128
differModels.addAll(remainList);
127129

130+
Predicate<DifferentModel> rdbPredicate = r -> r.getMissingColumn() != null && r.getMissingColumn().contains("is not exist in RDB");
131+
Predicate<DifferentModel> rdbModernPredicate = r -> r.getMissingColumn() != null && r.getMissingColumn().contains("is not exist in RDB_MODERN");
132+
processFirstFound(differModels, rdbPredicate);
133+
processFirstFound(differModels, rdbModernPredicate);
134+
135+
differModels.sort(Comparator.comparing(DifferentModel::getKey, Comparator.nullsFirst(Comparator.naturalOrder())));
136+
128137
var stringValue = gson.toJson(differModels);
129138
s3DataPullerService.uploadDataToS3(
130139
pullerEventModel.getFirstLayerRdbModernFolderName(),
@@ -175,6 +184,21 @@ public void processingData(PullerEventModel pullerEventModel) {
175184

176185
}
177186

187+
private void processFirstFound(List<DifferentModel> records, Predicate<DifferentModel> predicate) {
188+
Optional<DifferentModel> firstFound = records.stream()
189+
.filter(predicate)
190+
.peek(r -> r.setKey(null))
191+
.findFirst();
192+
193+
if (firstFound.isPresent()) {
194+
// Remove all that match the predicate
195+
records.removeIf(predicate);
196+
197+
// Add the first found back to the list
198+
records.add(firstFound.get());
199+
}
200+
}
201+
178202
protected Map<String, Integer> getMaxIndexWithSource(PullerEventModel pullerEventModel) {
179203
Map<String, Integer> result = new HashMap<>();
180204

@@ -273,6 +297,7 @@ protected List<DifferentModel> compareJsonFilesOnRemaining(String uniqueIdField,
273297

274298

275299
StringBuilder diffBuilder = new StringBuilder();
300+
StringBuilder missingColBuilder = new StringBuilder();
276301
List<DifferentModel> differentModels = new ArrayList<>();
277302

278303
if (mapRdb.size() < mapRdbModern.size()) {
@@ -284,6 +309,7 @@ protected List<DifferentModel> compareJsonFilesOnRemaining(String uniqueIdField,
284309
mapRdb.remove(id);
285310
mapRdbModern.remove(id);
286311
List<String> differList = new ArrayList<>();
312+
List<String> missingColList = new ArrayList<>();
287313

288314
for (String key : recordRdb.keySet()) {
289315
if (ignoreCols.contains(key)) {
@@ -308,11 +334,16 @@ protected List<DifferentModel> compareJsonFilesOnRemaining(String uniqueIdField,
308334

309335
}
310336
else {
311-
diffBuilder.setLength(0);
312-
diffBuilder.append(key).append(": ")
313-
.append("[RDB_VALUE: ").append(valueRdb).append("], ")
314-
.append("[RDB_MODERN_VALUE: COLUMN NOT EXIST]").trimToSize();
315-
differList.add(diffBuilder.toString());
337+
missingColBuilder.setLength(0);
338+
missingColBuilder.append(key).append(" is not exist in RDB_MODERN").trimToSize();
339+
missingColList.add(missingColBuilder.toString());
340+
341+
//
342+
// diffBuilder.setLength(0);
343+
// diffBuilder.append(key).append(": ")
344+
// .append("[RDB_VALUE: ").append(valueRdb).append("], ")
345+
// .append("[RDB_MODERN_VALUE: COLUMN NOT EXIST]").trimToSize();
346+
// differList.add(diffBuilder.toString());
316347
}
317348

318349
}
@@ -330,14 +361,32 @@ protected List<DifferentModel> compareJsonFilesOnRemaining(String uniqueIdField,
330361
}
331362
diffBuilder.append("]");
332363

364+
missingColBuilder.setLength(0);
365+
missingColBuilder.append("[");
366+
for (int i = 0; i < missingColList.size(); i++) {
367+
368+
missingColBuilder.append(missingColList.get(i));
369+
// Add a comma if it's not the last element
370+
if (i < missingColList.size() - 1) {
371+
missingColBuilder.append(",");
372+
}
373+
}
374+
missingColBuilder.append("]");
375+
differentModel.setMissingColumn(missingColBuilder.toString().replaceAll("\"", ""));
376+
333377
differentModel.setKey(id);
334378
differentModel.setKeyColumn(uniqueIdField);
335379
differentModel.setDifferentColumnAndValue(diffBuilder.toString().replaceAll("\"", ""));
336-
337-
if (!differentModel.getDifferentColumnAndValue().equals("[]")) {
380+
if (!differentModel.getDifferentColumnAndValue().equals("[]"))
381+
{
382+
differentModels.add(differentModel);
383+
}
384+
else if (!differentModel.getMissingColumn().equals("[]"))
385+
{
338386
differentModels.add(differentModel);
339387
}
340388

389+
341390
}
342391

343392
}
@@ -351,6 +400,7 @@ protected List<DifferentModel> compareJsonFilesOnRemaining(String uniqueIdField,
351400
mapRdb.remove(id);
352401
mapRdbModern.remove(id);
353402
List<String> differList = new ArrayList<>();
403+
List<String> missingColList = new ArrayList<>();
354404

355405
for (String key : mapRdbModern.keySet()) {
356406
if (ignoreCols.contains(key)) {
@@ -375,11 +425,15 @@ protected List<DifferentModel> compareJsonFilesOnRemaining(String uniqueIdField,
375425

376426
}
377427
else {
378-
diffBuilder.setLength(0);
379-
diffBuilder.append(key).append(": ")
380-
.append("[RDB_VALUE: COLUMN NOT EXIST")
381-
.append("[RDB_MODERN_VALUE: ").append(valueRdbModern).append("]").trimToSize();
382-
differList.add(diffBuilder.toString());
428+
missingColBuilder.setLength(0);
429+
missingColBuilder.append(key).append(" is not exist in RDB").trimToSize();
430+
missingColList.add(missingColBuilder.toString());
431+
432+
// diffBuilder.setLength(0);
433+
// diffBuilder.append(key).append(": ")
434+
// .append("[RDB_VALUE: COLUMN NOT EXIST")
435+
// .append("[RDB_MODERN_VALUE: ").append(valueRdbModern).append("]").trimToSize();
436+
// differList.add(diffBuilder.toString());
383437
}
384438

385439
}
@@ -397,13 +451,31 @@ protected List<DifferentModel> compareJsonFilesOnRemaining(String uniqueIdField,
397451
}
398452
diffBuilder.append("]");
399453

454+
missingColBuilder.setLength(0);
455+
missingColBuilder.append("[");
456+
for (int i = 0; i < missingColList.size(); i++) {
457+
458+
missingColBuilder.append(missingColList.get(i));
459+
// Add a comma if it's not the last element
460+
if (i < missingColList.size() - 1) {
461+
missingColBuilder.append(",");
462+
}
463+
}
464+
missingColBuilder.append("]");
465+
differentModel.setMissingColumn(missingColBuilder.toString().replaceAll("\"", ""));
466+
467+
400468
differentModel.setKey(id);
401469
differentModel.setKeyColumn(uniqueIdField);
402470
differentModel.setDifferentColumnAndValue(diffBuilder.toString().replaceAll("\"", ""));
403471

404472
if (!differentModel.getDifferentColumnAndValue().equals("[]")) {
405473
differentModels.add(differentModel);
406474
}
475+
else if (!differentModel.getMissingColumn().equals("[]"))
476+
{
477+
differentModels.add(differentModel);
478+
}
407479

408480
}
409481

@@ -440,6 +512,7 @@ protected String compareJsonFiles(String fileRdbPath, String fileRdbModernPath,
440512
}
441513

442514
StringBuilder diffBuilder = new StringBuilder();
515+
StringBuilder missingColBuilder = new StringBuilder();
443516
List<DifferentModel> differentModels = new ArrayList<>();
444517

445518

@@ -448,6 +521,7 @@ protected String compareJsonFiles(String fileRdbPath, String fileRdbModernPath,
448521
DifferentModel differentModel = new DifferentModel();
449522
differentModel.setTable(pullerEventModel.getFileName());
450523
List<String> differList = new ArrayList<>();
524+
List<String> missingColList = new ArrayList<>();
451525

452526
JsonObject recordRdb = mapRdb.get(id);
453527
JsonObject recordRdbModern = mapRdbModern.get(id);
@@ -482,11 +556,15 @@ protected String compareJsonFiles(String fileRdbPath, String fileRdbModernPath,
482556

483557
}
484558
else {
485-
diffBuilder.setLength(0);
486-
diffBuilder.append(key).append(": ")
487-
.append("[RDB_VALUE: ").append(valueRdb).append("], ")
488-
.append("[RDB_MODERN_VALUE: COLUMN NOT EXIST]").trimToSize();
489-
differList.add(diffBuilder.toString());
559+
// diffBuilder.setLength(0);
560+
// diffBuilder.append(key).append(": ")
561+
// .append("[RDB_VALUE: ").append(valueRdb).append("], ")
562+
// .append("[RDB_MODERN_VALUE: COLUMN NOT EXIST]").trimToSize();
563+
// differList.add(diffBuilder.toString());
564+
missingColBuilder.setLength(0);
565+
missingColBuilder.append(key).append(" is not exist in RDB_MODERN").trimToSize();
566+
missingColList.add(missingColBuilder.toString());
567+
490568
}
491569

492570
}
@@ -499,12 +577,17 @@ protected String compareJsonFiles(String fileRdbPath, String fileRdbModernPath,
499577
continue;
500578
}
501579

502-
JsonElement valueRdbModern = recordRdbModern.get(key);
503-
diffBuilder.setLength(0);
504-
diffBuilder.append(key).append(": ")
505-
.append("[RDB_VALUE: COLUMN NOT EXIST]")
506-
.append("[RDB_MODERN_VALUE: ").append(valueRdbModern).append("]").trimToSize();
507-
differList.add(diffBuilder.toString());
580+
// JsonElement valueRdbModern = recordRdbModern.get(key);
581+
// diffBuilder.setLength(0);
582+
// diffBuilder.append(key).append(": ")
583+
// .append("[RDB_VALUE: COLUMN NOT EXIST]")
584+
// .append("[RDB_MODERN_VALUE: ").append(valueRdbModern).append("]").trimToSize();
585+
// differList.add(diffBuilder.toString());
586+
587+
missingColBuilder.setLength(0);
588+
missingColBuilder.append(key).append(" is not exist in RDB").trimToSize();
589+
missingColList.add(missingColBuilder.toString());
590+
508591

509592
}
510593
}
@@ -522,12 +605,29 @@ protected String compareJsonFiles(String fileRdbPath, String fileRdbModernPath,
522605
}
523606
diffBuilder.append("]");
524607

608+
missingColBuilder.setLength(0);
609+
missingColBuilder.append("[");
610+
for (int i = 0; i < missingColList.size(); i++) {
611+
612+
missingColBuilder.append(missingColList.get(i));
613+
// Add a comma if it's not the last element
614+
if (i < missingColList.size() - 1) {
615+
missingColBuilder.append(",");
616+
}
617+
}
618+
missingColBuilder.append("]");
619+
differentModel.setMissingColumn(missingColBuilder.toString().replaceAll("\"", ""));
620+
621+
622+
525623
differentModel.setKey(id);
526624
differentModel.setKeyColumn(uniqueIdField);
527625
differentModel.setDifferentColumnAndValue(diffBuilder.toString().replaceAll("\"", ""));
528626

529627
if (!differentModel.getDifferentColumnAndValue().equals("[]")) {
530628
differentModels.add(differentModel);
629+
} else if (!differentModel.getMissingColumn().equals("[]")) {
630+
differentModels.add(differentModel);
531631
}
532632
}
533633

‎DataCompareProcessor/src/main/java/gov/cdc/datacompareprocessor/service/model/DifferentModel.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,5 +9,6 @@ public class DifferentModel {
99
public String keyColumn;
1010
public String key;
1111
public String table;
12+
public String missingColumn;
1213
public String differentColumnAndValue;
1314
}

0 commit comments

Comments
 (0)