Updated to ensure the most recent unique functions parse correctly.

NationalSecurityAgency · May 31, 2024 · 454bb9a · 454bb9a
1 parent ec3be20
commit 454bb9a
Show file tree

Hide file tree

Showing 14 changed files with 63 additions and 20 deletions.
diff --git a/warehouse/query-core/src/main/java/datawave/query/attributes/UniqueFields.java b/warehouse/query-core/src/main/java/datawave/query/attributes/UniqueFields.java
@@ -32,6 +32,7 @@ public class UniqueFields implements Serializable, Cloneable {
 
     private final TreeMultimap<String,UniqueGranularity> fieldMap = TreeMultimap.create();
     private boolean mostRecent = false;
+    private static String MOST_RECENT_UNIQUE = "_MOST_RECENT_";
 
     /**
      * Returns a new {@link UniqueFields} parsed from this string. The provided string is expected to have the format returned by
@@ -76,8 +77,12 @@ public static UniqueFields from(String string) {
             if (nextComma == -1 && nextStartBracket == -1) {
                 String field = string.substring(currentIndex);
                 if (!field.isEmpty()) {
-                    // Add the field only if its not blank. Ignore cases with consecutive trailing commas like field1[ALL],,
-                    uniqueFields.put(field, UniqueGranularity.ALL);
+                    if (field.equals(MOST_RECENT_UNIQUE)) {
+                        uniqueFields.setMostRecent(true);
+                    } else {
+                        // Add the field only if its not blank. Ignore cases with consecutive trailing commas like field1[ALL],,
+                        uniqueFields.put(field, UniqueGranularity.ALL);
+                    }
                 }
                 break; // There are no more fields to be parsed.
             } else if (nextComma != -1 && (nextStartBracket == -1 || nextComma < nextStartBracket)) {
@@ -91,8 +96,12 @@ public static UniqueFields from(String string) {
                 // Add the field with the ALL granularity.
                 String field = string.substring(currentIndex, nextComma);
                 if (!field.isEmpty()) {
-                    // Add the field only if its not blank. Ignore cases with consecutive commas like field1,,field2[DAY]
-                    uniqueFields.put(field, UniqueGranularity.ALL);
+                    if (field.equals(MOST_RECENT_UNIQUE)) {
+                        uniqueFields.setMostRecent(true);
+                    } else {
+                        // Add the field only if its not blank. Ignore cases with consecutive commas like field1,,field2[DAY]
+                        uniqueFields.put(field, UniqueGranularity.ALL);
+                    }
                 }
                 currentIndex = nextComma + 1; // Advance to the start of the next field.
             } else {
@@ -104,14 +113,18 @@ public static UniqueFields from(String string) {
                 String field = string.substring(currentIndex, nextStartBracket);
                 int nextEndBracket = string.indexOf(Constants.BRACKET_END, currentIndex);
                 if (!field.isEmpty()) {
-                    String granularityList = string.substring((nextStartBracket + 1), nextEndBracket);
-                    // An empty granularity list, e.g. field[] is equivalent to field[ALL].
-                    if (granularityList.isEmpty()) {
-                        uniqueFields.put(field, UniqueGranularity.ALL);
+                    if (field.equals(MOST_RECENT_UNIQUE)) {
+                        uniqueFields.setMostRecent(true);
                     } else {
-                        String[] granularities = StringUtils.split(granularityList, Constants.COMMA);
-                        for (String granularity : granularities) {
-                            uniqueFields.put(field, parseGranularity(granularity));
+                        String granularityList = string.substring((nextStartBracket + 1), nextEndBracket);
+                        // An empty granularity list, e.g. field[] is equivalent to field[ALL].
+                        if (granularityList.isEmpty()) {
+                            uniqueFields.put(field, UniqueGranularity.ALL);
+                        } else {
+                            String[] granularities = StringUtils.split(granularityList, Constants.COMMA);
+                            for (String granularity : granularities) {
+                                uniqueFields.put(field, parseGranularity(granularity));
+                            }
                         }
                     }
                 }
@@ -308,6 +321,10 @@ public String transformValue(String field, String value) {
     @Override
     public String toString() {
         StringBuilder sb = new StringBuilder();
+        if (mostRecent) {
+            sb.append(MOST_RECENT_UNIQUE);
+            sb.append(Constants.COMMA);
+        }
         Iterator<String> fieldIterator = fieldMap.keySet().iterator();
         while (fieldIterator.hasNext()) {
             // Write the field.

diff --git a/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java b/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java
@@ -389,7 +389,6 @@ public class ShardQueryConfiguration extends GenericQueryConfiguration implement
     private int groupFieldsBatchSize;
     private boolean accrueStats = false;
     private UniqueFields uniqueFields = new UniqueFields();
-    private boolean mostRecentUnique = false;
     private boolean cacheModel = false;
     /**
      * should the sizes of documents be tracked for this query
@@ -685,6 +684,7 @@ public ShardQueryConfiguration(ShardQueryConfiguration other) {
         this.setGroupFieldsBatchSize(other.getGroupFieldsBatchSize());
         this.setAccrueStats(other.getAccrueStats());
         this.setUniqueFields(other.getUniqueFields());
+        log.info("Checkpointing with " + getUniqueFields());
         this.setUniqueCacheBufferSize(other.getUniqueCacheBufferSize());
         this.setCacheModel(other.getCacheModel());
         this.setTrackSizes(other.isTrackSizes());

diff --git a/...ouse/query-core/src/main/java/datawave/query/jexl/functions/QueryFunctionsDescriptor.java b/...ouse/query-core/src/main/java/datawave/query/jexl/functions/QueryFunctionsDescriptor.java
@@ -205,6 +205,15 @@ private static void verify(String name, int numArgs) {
             case QueryOptionsFromQueryVisitor.UniqueFunction.UNIQUE_BY_DAY_FUNCTION:
             case QueryOptionsFromQueryVisitor.UniqueFunction.UNIQUE_BY_MONTH_FUNCTION:
             case QueryOptionsFromQueryVisitor.UniqueFunction.UNIQUE_BY_YEAR_FUNCTION:
+            case QueryFunctions.MOST_RECENT_PREFIX + QueryFunctions.UNIQUE_FUNCTION:
+            case QueryFunctions.MOST_RECENT_PREFIX + QueryOptionsFromQueryVisitor.UniqueFunction.UNIQUE_BY_MILLISECOND_FUNCTION:
+            case QueryFunctions.MOST_RECENT_PREFIX + QueryOptionsFromQueryVisitor.UniqueFunction.UNIQUE_BY_SECOND_FUNCTION:
+            case QueryFunctions.MOST_RECENT_PREFIX + QueryOptionsFromQueryVisitor.UniqueFunction.UNIQUE_BY_MINUTE_FUNCTION:
+            case QueryFunctions.MOST_RECENT_PREFIX + QueryOptionsFromQueryVisitor.UniqueFunction.UNIQUE_BY_TENTH_OF_HOUR_FUNCTION:
+            case QueryFunctions.MOST_RECENT_PREFIX + QueryOptionsFromQueryVisitor.UniqueFunction.UNIQUE_BY_HOUR_FUNCTION:
+            case QueryFunctions.MOST_RECENT_PREFIX + QueryOptionsFromQueryVisitor.UniqueFunction.UNIQUE_BY_DAY_FUNCTION:
+            case QueryFunctions.MOST_RECENT_PREFIX + QueryOptionsFromQueryVisitor.UniqueFunction.UNIQUE_BY_MONTH_FUNCTION:
+            case QueryFunctions.MOST_RECENT_PREFIX + QueryOptionsFromQueryVisitor.UniqueFunction.UNIQUE_BY_YEAR_FUNCTION:
             case QueryFunctions.GROUPBY_FUNCTION:
             case QueryFunctions.EXCERPT_FIELDS_FUNCTION:
             case QueryFunctions.MATCH_REGEX:

diff --git a/...use/query-core/src/main/java/datawave/query/language/functions/jexl/MostRecentUnique.java b/...use/query-core/src/main/java/datawave/query/language/functions/jexl/MostRecentUnique.java
@@ -66,7 +66,7 @@ public String toString() {
 
     @Override
     public QueryFunction duplicate() {
-        return new Unique();
+        return new MostRecentUnique();
     }
 
 }
diff --git a/...uery-core/src/main/java/datawave/query/language/functions/jexl/MostRecentUniqueByDay.java b/...uery-core/src/main/java/datawave/query/language/functions/jexl/MostRecentUniqueByDay.java
@@ -17,6 +17,6 @@ public MostRecentUniqueByDay() {
 
     @Override
     public QueryFunction duplicate() {
-        return new UniqueByDay();
+        return new MostRecentUniqueByDay();
     }
 }
diff --git a/...ery-core/src/main/java/datawave/query/language/functions/jexl/MostRecentUniqueByHour.java b/...ery-core/src/main/java/datawave/query/language/functions/jexl/MostRecentUniqueByHour.java
@@ -18,6 +18,6 @@ public MostRecentUniqueByHour() {
 
     @Override
     public QueryFunction duplicate() {
-        return new UniqueByHour();
+        return new MostRecentUniqueByHour();
     }
 }
diff --git a/...y-core/src/main/java/datawave/query/language/functions/jexl/MostRecentUniqueByMinute.java b/...y-core/src/main/java/datawave/query/language/functions/jexl/MostRecentUniqueByMinute.java
@@ -18,6 +18,6 @@ public MostRecentUniqueByMinute() {
 
     @Override
     public QueryFunction duplicate() {
-        return new UniqueByMinute();
+        return new MostRecentUniqueByMinute();
     }
 }
diff --git a/...ry-core/src/main/java/datawave/query/language/functions/jexl/MostRecentUniqueByMonth.java b/...ry-core/src/main/java/datawave/query/language/functions/jexl/MostRecentUniqueByMonth.java
@@ -18,6 +18,6 @@ public MostRecentUniqueByMonth() {
 
     @Override
     public QueryFunction duplicate() {
-        return new UniqueByMonth();
+        return new MostRecentUniqueByMonth();
     }
 }
diff --git a/...y-core/src/main/java/datawave/query/language/functions/jexl/MostRecentUniqueBySecond.java b/...y-core/src/main/java/datawave/query/language/functions/jexl/MostRecentUniqueBySecond.java
@@ -18,6 +18,6 @@ public MostRecentUniqueBySecond() {
 
     @Override
     public QueryFunction duplicate() {
-        return new UniqueBySecond();
+        return new MostRecentUniqueBySecond();
     }
 }
diff --git a/...e/src/main/java/datawave/query/language/functions/jexl/MostRecentUniqueByTenthOfHour.java b/...e/src/main/java/datawave/query/language/functions/jexl/MostRecentUniqueByTenthOfHour.java
@@ -18,6 +18,6 @@ public MostRecentUniqueByTenthOfHour() {
 
     @Override
     public QueryFunction duplicate() {
-        return new UniqueByTenthOfHour();
+        return new MostRecentUniqueByTenthOfHour();
     }
 }
diff --git a/...ery-core/src/main/java/datawave/query/language/functions/jexl/MostRecentUniqueByYear.java b/...ery-core/src/main/java/datawave/query/language/functions/jexl/MostRecentUniqueByYear.java
@@ -18,6 +18,6 @@ public MostRecentUniqueByYear() {
 
     @Override
     public QueryFunction duplicate() {
-        return new UniqueByYear();
+        return new MostRecentUniqueByYear();
     }
 }
diff --git a/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java b/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java
@@ -533,6 +533,7 @@ private void configureIterator(ShardQueryConfiguration config, IteratorSetting c
         addOption(cfg, QueryOptions.GROUP_FIELDS_BATCH_SIZE, config.getGroupFieldsBatchSizeAsString(), true);
         addOption(cfg, QueryOptions.UNIQUE_FIELDS, config.getUniqueFields().toString(), true);
         if (config.getUniqueFields().isMostRecent()) {
+            // this may be redundant with the uniqueFields.toString(), but other code relies on this explicitly being set
             addOption(cfg, QueryOptions.MOST_RECENT_UNIQUE, Boolean.toString(true), false);
             addOption(cfg, QueryOptions.UNIQUE_CACHE_BUFFER_SIZE, Integer.toString(config.getUniqueCacheBufferSize()), false);
         }

diff --git a/warehouse/query-core/src/main/java/datawave/query/planner/QueryOptionsSwitch.java b/warehouse/query-core/src/main/java/datawave/query/planner/QueryOptionsSwitch.java
@@ -67,6 +67,7 @@ public static void apply(Map<String,String> optionsMap, ShardQueryConfiguration
                     config.setUniqueFields(uniqueFields);
                     break;
                 case QueryParameters.MOST_RECENT_UNIQUE:
+                    log.info("Setting unique fields to be most recent");
                     config.getUniqueFields().setMostRecent(Boolean.parseBoolean(value));
                     break;
                 case QueryParameters.EXCERPT_FIELDS:

diff --git a/warehouse/query-core/src/test/java/datawave/query/UniqueTest.java b/warehouse/query-core/src/test/java/datawave/query/UniqueTest.java
@@ -373,6 +373,21 @@ public void testUniquenessWithModelAliases() throws Exception {
         runTestQueryWithUniqueness(expected, queryString, startDate, endDate, extraParameters);
     }
 
+    @Test
+    public void testRecentUniquenessWithModelAliases() throws Exception {
+        Map<String,String> extraParameters = new HashMap<>();
+        extraParameters.put("include.grouping.context", "true");
+        extraParameters.put("query.syntax", "LUCENE");
+
+        Set<Set<String>> expected = new HashSet<>();
+        expected.add(Sets.newHashSet(WiseGuysIngest.sopranoUID, WiseGuysIngest.corleoneUID, WiseGuysIngest.caponeUID));
+        Date startDate = format.parse("20091231");
+        Date endDate = format.parse("20150101");
+
+        String queryString = "UUID:/^[CS].*/ AND #MOST_RECENT_UNIQUE(BOTH_NULL)";
+        runTestQueryWithUniqueness(expected, queryString, startDate, endDate, extraParameters);
+    }
+
     @Test
     public void testMostRecentUniqueness() throws Exception {
         Map<String,String> extraParameters = new HashMap<>();