* Updated with metadata-utils 4.0.5 (index markers and avoid non-inde…

…xed fields for holes)

* Fixed test cases with correct responses and periodic failing test cases
* Updated AncestorQueryLogic to handle federate query planner

pom.xml

@@ -107,7 +107,7 @@
         <version.microservice.common-utils>3.0.0</version.microservice.common-utils>
         <version.microservice.dictionary-api>4.0.0</version.microservice.dictionary-api>
         <version.microservice.mapreduce-query-api>1.0.0</version.microservice.mapreduce-query-api>
-        <version.microservice.metadata-utils>4.0.0</version.microservice.metadata-utils>
+        <version.microservice.metadata-utils>4.0.5</version.microservice.metadata-utils>
         <version.microservice.metrics-reporter>3.0.0</version.microservice.metrics-reporter>
         <version.microservice.query-api>1.0.0</version.microservice.query-api>
         <version.microservice.query-metric-api>4.0.0</version.microservice.query-metric-api>

warehouse/query-core/src/main/java/datawave/query/planner/FederatedQueryPlanner.java

@@ -2,13 +2,12 @@
 
 import java.io.IOException;
 import java.text.SimpleDateFormat;
-import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.Iterator;
-import java.util.List;
+import java.util.LinkedHashSet;
 import java.util.Map;
 import java.util.Set;
 import java.util.SortedSet;
@@ -60,7 +59,8 @@ public class FederatedQueryPlanner extends QueryPlanner implements Cloneable {
     private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd");
     private final Calendar calendar = Calendar.getInstance();
 
-    private final List<String> plans = new ArrayList<>();
+    // we want a unique set of plans, but maintain insertion order (facilitates easier testing)
+    private final Set<String> plans = new LinkedHashSet<>();
     private DefaultQueryPlanner queryPlanner;
     private String plannedScript;
 
@@ -381,15 +381,16 @@ private void updatePlannedScript() {
         if (plans.isEmpty()) {
             this.plannedScript = "";
         } else if (this.plans.size() == 1) {
-            this.plannedScript = this.plans.get(0);
+            this.plannedScript = this.plans.iterator().next();
         } else {
             int lastIndex = plans.size() - 1;
             StringBuilder sb = new StringBuilder();
-            for (int i = 0; i < plans.size(); i++) {
-                sb.append("((plan = ").append((i + 1)).append(") && (").append(plans.get(i)).append("))");
-                if (i != lastIndex) {
+            int i = 0;
+            for (String plan : plans) {
+                if (sb.length() > 0) {
                     sb.append(" || ");
                 }
+                sb.append("((plan = ").append(++i).append(") && (").append(plan).append("))");
             }
             this.plannedScript = sb.toString();
         }

warehouse/query-core/src/main/java/datawave/query/tables/AncestorQueryLogic.java

@@ -6,6 +6,7 @@
 import datawave.query.ancestor.AncestorUidIntersector;
 import datawave.query.index.lookup.AncestorCreateUidsIterator;
 import datawave.query.planner.DefaultQueryPlanner;
+import datawave.query.planner.FederatedQueryPlanner;
 import datawave.query.planner.QueryPlanner;
 
 /**
@@ -33,15 +34,24 @@ public AncestorQueryLogic(AncestorQueryLogic other) {
 
     @Override
     public void setQueryPlanner(QueryPlanner planner) {
-        if (!(planner instanceof DefaultQueryPlanner)) {
-            throw new IllegalArgumentException("Query logic requires DefaultQueryPlanner compatibility");
+        if (!(planner instanceof DefaultQueryPlanner) && !(planner instanceof FederatedQueryPlanner)) {
+            throw new IllegalArgumentException("Query logic requires DefaultQueryPlanner or FederatedQueryPlanner compatibility");
         }
 
         super.setQueryPlanner(planner);
         setRangeStream();
         setIter();
     }
 
+    private DefaultQueryPlanner getDefaultQueryPlanner() {
+        QueryPlanner planner = getQueryPlanner();
+        if (planner instanceof FederatedQueryPlanner) {
+            return ((FederatedQueryPlanner) planner).getQueryPlanner();
+        } else {
+            return (DefaultQueryPlanner) planner;
+        }
+    }
+
     /**
      * Overriding this generates TLD end ranges without generating TLD start ranges. Essentially expanding a hit down the branch so all branch candidates can be
      * potentially hit. This is specifically for the case where the index hits further up the tree than the additional/delayed predicates. Without this no range
@@ -57,7 +67,7 @@ public boolean isTldQuery() {
     }
 
     private void setRangeStream() {
-        ((DefaultQueryPlanner) getQueryPlanner()).setRangeStreamClass(AncestorRangeStream.class.getCanonicalName());
+        getDefaultQueryPlanner().setRangeStreamClass(AncestorRangeStream.class.getCanonicalName());
     }
 
     private void setIter() {

warehouse/query-core/src/test/java/datawave/query/AnyFieldQueryTest.java

@@ -12,7 +12,8 @@
 import java.util.Arrays;
 import java.util.Collections;
 
-import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyValue;
+import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
@@ -107,7 +108,7 @@ public void testEqualMissesRemovedIndexedField() throws Exception {
             runTest(query, anyCity);
 
             // remove the metadata entries
-            Multimap<String,Key> metadata = removeMetadataEntries(JexlASTHelper.getIdentifierNames(JexlASTHelper.parseJexlQuery(anyCity)),
+            Multimap<String,KeyValue> metadata = removeMetadataEntries(JexlASTHelper.getIdentifierNames(JexlASTHelper.parseJexlQuery(anyCity)),
                             ColumnFamilyConstants.COLF_I);
 
             // expect no results
@@ -399,12 +400,14 @@ public void testAndAnd_federatedQueryPlanner() throws Exception {
             String query = Constants.ANY_FIELD + cityPhrase + AND_OP + Constants.ANY_FIELD + statePhrase + AND_OP + Constants.ANY_FIELD + contPhrase;
 
             // Test the plan with all expansions
-            String anyQuery = "((plan = 1) && (false)) || ((plan = 2) && (";
+            String anyQuery = CityField.CONTINENT.name() + contPhrase.toLowerCase() + JEXL_AND_OP;
             if (city.name().equals("london")) {
-                anyQuery += "CONTINENT == 'north america' && ((STATE == 'mississippi' && STATE == 'london') || CITY_STATE == 'london"
-                                + CompositeIngest.DEFAULT_SEPARATOR + "mississippi')))";
-            } else {
-                anyQuery += "CONTINENT == 'north america' && CITY_STATE == '" + city.name() + CompositeIngest.DEFAULT_SEPARATOR + "mississippi'))";
+                anyQuery += "((" + CityField.STATE.name() + statePhrase.toLowerCase() + JEXL_AND_OP + CityField.STATE.name() + cityPhrase + ")" + JEXL_OR_OP;
+            }
+            anyQuery += CityField.CITY.name() + '_' + CityField.STATE.name() + EQ_OP + "'" + city.name() + CompositeIngest.DEFAULT_SEPARATOR
+                            + state.toLowerCase() + "'";
+            if (city.name().equals("london")) {
+                anyQuery += ")";
             }
             String plan = getPlan(query, true, true);
             assertPlanEquals(anyQuery, plan);
@@ -497,7 +500,7 @@ public void testReverseIndexMissesRemovedIndexEntries() throws Exception {
         runTest(query, expect);
 
         // remove the metadata entries
-        Multimap<String,Key> metadata = removeMetadataEntries(JexlASTHelper.getIdentifierNames(JexlASTHelper.parseJexlQuery(expect)),
+        Multimap<String,KeyValue> metadata = removeMetadataEntries(JexlASTHelper.getIdentifierNames(JexlASTHelper.parseJexlQuery(expect)),
                         ColumnFamilyConstants.COLF_RI);
 
         // expect no results
@@ -545,15 +548,15 @@ public void testAndNoMatch_federatedQueryPlanner() throws Exception {
 
         // Test the plan with all expansions
         String plan = getPlan(query, true, true);
-        assertPlanEquals("((plan = 1) && (false)) || ((plan = 2) && (false))", plan);
+        assertPlanEquals("false", plan);
 
         // Test the plan sans value expansion
         plan = getPlan(query, true, false);
-        assertPlanEquals("((plan = 1) && (false)) || ((plan = 2) && (false))", plan);
+        assertPlanEquals("false", plan);
 
         // Test the plan sans field expansion
         plan = getPlan(query, false, true);
-        assertPlanEquals("((plan = 1) && (false)) || ((plan = 2) && (false))", plan);
+        assertPlanEquals("false", plan);
 
         // test running the query
         String expect = first + AND_OP + this.dataManager.convertAnyField(phrase);
@@ -626,7 +629,7 @@ public void testRegexMissesRemovedIndexEntries() throws Exception {
         runTest(query, expect);
 
         // remove the metadata entries
-        Multimap<String,Key> metadata = removeMetadataEntries(JexlASTHelper.getIdentifierNames(JexlASTHelper.parseJexlQuery(expect)),
+        Multimap<String,KeyValue> metadata = removeMetadataEntries(JexlASTHelper.getIdentifierNames(JexlASTHelper.parseJexlQuery(expect)),
                         ColumnFamilyConstants.COLF_I);
 
         // expect no results (or error until #567 is fixed)
@@ -747,12 +750,13 @@ public void testRegexOr_federatedQueryPlanner() throws Exception {
         String query = Constants.ANY_FIELD + roPhrase + OR_OP + Constants.ANY_FIELD + oPhrase;
 
         // Test the plan with all expansions
-        String expect = "((plan = 1) && (false)) || ((plan = 2) && (CITY == 'rome' || STATE == 'lazio' || STATE == 'ohio'))";
+        String expect = CityField.CITY.name() + EQ_OP + "'rome'" + JEXL_OR_OP + CityField.STATE.name() + EQ_OP + "'lazio'" + JEXL_OR_OP + CityField.STATE.name()
+                        + EQ_OP + "'ohio'";
         String plan = getPlan(query, true, true);
         assertPlanEquals(expect, plan);
 
         // Test the plan sans value expansion
-        expect = "((plan = 1) && (false)) || ((plan = 2) && (CITY =~ 'ro.*' || STATE =~ '.*o'))";
+        expect = CityField.CITY.name() + roPhrase + JEXL_OR_OP + CityField.STATE.name() + oPhrase;
         plan = getPlan(query, true, false);
         assertPlanEquals(expect, plan);
 
@@ -871,17 +875,20 @@ public void testRegexAndField_federatedQueryPlanner() throws Exception {
         String query = Constants.ANY_FIELD + roPhrase + AND_OP + CityField.STATE.name() + oPhrase;
 
         // Test the plan with all expansions
-        String expect = "((plan = 1) && (false)) || ((plan = 2) && ((CITY_STATE == 'rome􏿿lazio' || CITY_STATE == 'rome􏿿ohio') && ((_Eval_ = true) && (CITY == 'rome' && STATE =~ '.*o'))))";
+        String expect = "(" + CityField.CITY.name() + '_' + CityField.STATE.name() + EQ_OP + "'rome" + CompositeIngest.DEFAULT_SEPARATOR + "lazio'" + JEXL_OR_OP
+                        + CityField.CITY.name() + '_' + CityField.STATE.name() + EQ_OP + "'rome" + CompositeIngest.DEFAULT_SEPARATOR + "ohio')" + JEXL_AND_OP
+                        + "((_Eval_ = true)" + JEXL_AND_OP + "(" + CityField.CITY.name() + " == 'rome'" + JEXL_AND_OP + CityField.STATE.name() + oPhrase + "))";
         String plan = getPlan(query, true, true);
         assertPlanEquals(expect, plan);
 
         // Test the plan sans value expansion
-        expect = "((plan = 1) && (false)) || ((plan = 2) && (CITY =~ 'ro.*' && STATE =~ '.*o'))";
+        expect = CityField.CITY.name() + roPhrase + JEXL_AND_OP + CityField.STATE.name() + oPhrase;
         plan = getPlan(query, true, false);
         assertPlanEquals(expect, plan);
 
         // Test the plan sans field expansion
-        expect = "((plan = 1) && (false)) || ((plan = 2) && (_ANYFIELD_ == 'rome' && (STATE == 'lazio' || STATE == 'ohio')))";
+        expect = Constants.ANY_FIELD + EQ_OP + "'rome'" + JEXL_AND_OP + "(" + CityField.STATE.name() + EQ_OP + "'lazio'" + JEXL_OR_OP + CityField.STATE.name()
+                        + EQ_OP + "'ohio')";
         plan = getPlan(query, false, true);
         assertPlanEquals(expect, plan);
 
@@ -950,12 +957,12 @@ public void testRegexReverseIndex_federatedQueryPlanner() throws Exception {
         String query = Constants.ANY_FIELD + regPhrase;
 
         // Test the plan with all expansions
-        String expect = "((plan = 1) && (_NOFIELD_ =~ '.*ica')) || ((plan = 2) && (CONTINENT == 'north america'))";
+        String expect = CityField.CONTINENT.name() + EQ_OP + "'north america'";
         String plan = getPlan(query, true, true);
         assertPlanEquals(expect, plan);
 
         // Test the plan sans value expansion
-        expect = "((plan = 1) && (_NOFIELD_ =~ '.*ica')) || ((plan = 2) && (CONTINENT =~ '.*ica'))";
+        expect = CityField.CONTINENT.name() + regPhrase;
         plan = getPlan(query, true, false);
         assertPlanEquals(expect, plan);
 
@@ -1248,22 +1255,20 @@ public void testNegRegexOrDisallowedExpansion_federatedQueryPlanner() throws Exc
                 String query = Constants.ANY_FIELD + cityPhrase + OR_OP + Constants.ANY_FIELD + regPhrase;
 
                 // Test the plan with all expansions
-                String expect = "((plan = 1) && (!(_ANYFIELD_ =~ '.*ica'))) || ((plan = 2) && (";
+                String expect = CityField.CITY.name() + cityPhrase;
                 if (city.name().equals("london")) {
-                    expect += "CITY == 'london' || STATE == 'london' || !(_ANYFIELD_ =~ '.*ica')))";
-                } else {
-                    expect += "CITY == '" + city.name() + "' || !(_ANYFIELD_ =~ '.*ica')))";
+                    expect += JEXL_OR_OP + CityField.STATE.name() + cityPhrase;
                 }
+                expect += JEXL_OR_OP + "!(" + Constants.ANY_FIELD + RE_OP + "'.*ica')";
                 String plan = getPlan(query, true, true);
                 assertPlanEquals(expect, plan);
 
                 // Test the plan sans value expansion
-                expect = "((plan = 1) && (!(_ANYFIELD_ =~ '.*ica'))) || ((plan = 2) && (";
+                expect = CityField.CITY.name() + cityPhrase;
                 if (city.name().equals("london")) {
-                    expect += "CITY == 'london' || STATE == 'london' || !(_ANYFIELD_ =~ '.*ica')))";
-                } else {
-                    expect += "CITY == '" + city.name() + "' || !(_ANYFIELD_ =~ '.*ica')))";
+                    expect += JEXL_OR_OP + CityField.STATE.name() + cityPhrase;
                 }
+                expect += JEXL_OR_OP + "!(" + Constants.ANY_FIELD + RE_OP + "'.*ica')";
                 plan = getPlan(query, true, false);
                 assertPlanEquals(expect, plan);
 
@@ -1370,15 +1375,15 @@ public void testRegexPushdownField_federatedQueryPlanner() throws Exception {
         ((FederatedQueryPlanner) logic.getQueryPlanner()).getQueryPlanner().setTransformRules(Collections.singletonList(rule));
 
         // Test the plan with all expansions
-        String expect = "((plan = 1) && (false)) || ((plan = 2) && (CITY == 'rome' && ((_Eval_ = true) && (COUNTRY =~ '.*y'))))";
+        String expect = "CITY == 'rome' && ((_Eval_ = true) && (COUNTRY =~ '.*y'))";
         String plan = getPlan(query, true, true);
         assertPlanEquals(expect, plan);
 
-        expect = "((plan = 1) && (false)) || ((plan = 2) && (_ANYFIELD_ == 'rome' && ((_Eval_ = true) && (COUNTRY =~ '.*y'))))";
+        expect = "_ANYFIELD_ == 'rome' && ((_Eval_ = true) && (COUNTRY =~ '.*y'))";
         plan = getPlan(query, false, true);
         assertPlanEquals(expect, plan);
 
-        expect = "((plan = 1) && (false)) || ((plan = 2) && (CITY =~ 'ro.*' && ((_Eval_ = true) && (COUNTRY =~ '.*y'))))";
+        expect = "CITY =~ 'ro.*' && ((_Eval_ = true) && (COUNTRY =~ '.*y'))";
         plan = getPlan(query, true, false);
         assertPlanEquals(expect, plan);
 

warehouse/query-core/src/test/java/datawave/query/LuceneQueryTest.java

@@ -81,7 +81,8 @@ public void testSimpleAndEqEvalOnly() throws Exception {
         String country = "italy";
         String query = CityField.CITY.name() + ":\"" + city + "\"" + AND_OP + "#EVALUATION_ONLY('" + CityField.COUNTRY.name() + ":\"" + country + "\"')";
 
-        String expect = "((plan = 1) && (CITY == 'rome' && ((_Eval_ = true) && (COUNTRY == 'italy')))) || ((plan = 2) && (CITY == 'rome' && ((_Eval_ = true) && (COUNTRY == 'italy'))))";
+        String expect = CityField.CITY.name() + EQ_OP + "'" + city + "'" + AND_OP + "((_Eval_ = true) && " + CityField.COUNTRY.name() + EQ_OP + "'" + country
+                        + "')";
         String plan = getPlan(query, true, true);
         assertPlanEquals(expect, plan);
 

warehouse/query-core/src/test/java/datawave/query/RegexQueryTest.java

@@ -201,7 +201,7 @@ public void testAndNot() throws Exception {
     @Test
     public void testAndNotAgain() throws Exception {
         String query = "(NUM == '2' || NUM == '3') && CITY !~ '.*iSs.*'";
-        String expected = "((plan = 1) && ((NUM == '+aE2' || NUM == '+aE3') && !((_Delayed_ = true) && (CITY =~ '.*iss.*')))) || ((plan = 2) && ((NUM == '+aE2' || NUM == '+aE3') && !((_Delayed_ = true) && (CITY =~ '.*iss.*'))))";
+        String expected = "(NUM == '+aE2' || NUM == '+aE3') && !((_Delayed_ = true) && (CITY =~ '.*iss.*'))";
         String plan = getPlan(query, false, false);
         assertEquals(expected, plan);
     }

warehouse/query-core/src/test/java/datawave/query/testframework/AbstractFunctionalQuery.java

@@ -31,6 +31,7 @@
 import org.apache.accumulo.core.client.Scanner;
 import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyValue;
 import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
@@ -705,9 +706,9 @@ protected void assertPlanEquals(String expected, String query) throws ParseExcep
         }
     }
 
-    protected Multimap<String,Key> removeMetadataEntries(Set<String> fields, Text cf)
+    protected Multimap<String,KeyValue> removeMetadataEntries(Set<String> fields, Text cf)
                     throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
-        Multimap<String,Key> metadataEntries = HashMultimap.create();
+        Multimap<String,KeyValue> metadataEntries = HashMultimap.create();
         MultiTableBatchWriter multiTableWriter = client.createMultiTableBatchWriter(new BatchWriterConfig());
         BatchWriter writer = multiTableWriter.getBatchWriter(QueryTestTableHelper.METADATA_TABLE_NAME);
         for (String field : fields) {
@@ -718,8 +719,9 @@ protected Multimap<String,Key> removeMetadataEntries(Set<String> fields, Text cf
             boolean foundEntries = false;
             for (Map.Entry<Key,Value> entry : scanner) {
                 foundEntries = true;
-                metadataEntries.put(field, entry.getKey());
-                mutation.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier(), entry.getKey().getColumnVisibilityParsed());
+                metadataEntries.put(field, new KeyValue(entry.getKey(), entry.getValue()));
+                mutation.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier(), entry.getKey().getColumnVisibilityParsed(),
+                                entry.getKey().getTimestamp() + 1000);
             }
             scanner.close();
             if (foundEntries) {
@@ -731,14 +733,15 @@ protected Multimap<String,Key> removeMetadataEntries(Set<String> fields, Text cf
         return metadataEntries;
     }
 
-    protected void addMetadataEntries(Multimap<String,Key> metadataEntries) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+    protected void addMetadataEntries(Multimap<String,KeyValue> metadataEntries) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
         MultiTableBatchWriter multiTableWriter = client.createMultiTableBatchWriter(new BatchWriterConfig());
         BatchWriter writer = multiTableWriter.getBatchWriter(QueryTestTableHelper.METADATA_TABLE_NAME);
         for (String field : metadataEntries.keySet()) {
             Mutation mutation = new Mutation(new Text(field));
-            for (Key key : metadataEntries.get(field)) {
-                metadataEntries.put(field, key);
-                mutation.put(key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibilityParsed(), new Value());
+            for (KeyValue kv : metadataEntries.get(field)) {
+                Key key = kv.getKey();
+                Value val = kv.getValue();
+                mutation.put(key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibilityParsed(), key.getTimestamp() + 2000, val);
             }
             writer.addMutation(mutation);
         }