Add FederatedQueryPlanner

pom.xml

@@ -91,7 +91,7 @@
         <version.microservice.base-rest-responses>3.0.0</version.microservice.base-rest-responses>
         <version.microservice.common-utils>2.0.0</version.microservice.common-utils>
         <version.microservice.dictionary-api>3.0.0</version.microservice.dictionary-api>
-        <version.microservice.metadata-utils>3.0.0</version.microservice.metadata-utils>
+        <version.microservice.metadata-utils>3.0.2-SNAPSHOT</version.microservice.metadata-utils>
         <version.microservice.metrics-reporter>2.0.0</version.microservice.metrics-reporter>
         <version.microservice.query-metric-api>3.0.0</version.microservice.query-metric-api>
         <version.microservice.type-utils>2.0.1</version.microservice.type-utils>

warehouse/ingest-core/src/main/java/datawave/ingest/metadata/EventMetadata.java

@@ -89,24 +89,27 @@
  */
 public class EventMetadata implements RawRecordMetadata {
 
-    private MetadataWithMostRecentDate compositeFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_CI);
-    private MetadataWithMostRecentDate compositeSeparators = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_CISEP);
-    private MetadataWithMostRecentDate dataTypeFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_T);
-    private MetadataWithMostRecentDate normalizedFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_N);
-
     private static final Logger log = getLogger(EventMetadata.class);
     private final Text metadataTableName;
     private final Text loadDatesTableName;
+
+    private final MetadataWithMostRecentDate compositeFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_CI);
+    private final MetadataWithMostRecentDate compositeSeparators = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_CISEP);
+    private final MetadataWithMostRecentDate dataTypeFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_T);
+    private final MetadataWithMostRecentDate normalizedFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_N);
+
     // stores field name, data type, and most recent event date
     private final MetadataWithMostRecentDate eventFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_E);
-    private final MetadataWithMostRecentDate indexedFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_I);
-    private final MetadataWithMostRecentDate reverseIndexedFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_RI);
     private final MetadataWithMostRecentDate termFrequencyFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_TF);
+
     // stores counts
     private final MetadataCounterGroup frequencyCounts = new MetadataCounterGroup(ColumnFamilyConstants.COLF_F); // by event date
     private final MetadataCounterGroup indexedFieldsLoadDateCounts;
     private final MetadataCounterGroup reverseIndexedFieldsLoadDateCounts;
-    private boolean frequency = false;
+    private final MetadataCounterGroup indexedCounts = new MetadataCounterGroup(ColumnFamilyConstants.COLF_I);
+    private final MetadataCounterGroup reverseIndexedCounts = new MetadataCounterGroup(ColumnFamilyConstants.COLF_RI);
+
+    private boolean writeFrequencyCounts = false;
 
     /**
      * @param shardTableName
@@ -119,32 +122,32 @@ public class EventMetadata implements RawRecordMetadata {
      *            used as part of column family within LOAD_DATES_TABLE_NAME
      * @param loadDatesTableName
      *            the table name for the load dates
-     * @param frequency
+     * @param writeFrequencyCounts
      *            whether to add to the metadata table's counts for field name by event date
      */
     public EventMetadata(@SuppressWarnings("UnusedParameters") Text shardTableName, Text metadataTableName, Text loadDatesTableName, Text shardIndexTableName,
-                    Text shardReverseIndexTableName, boolean frequency) {
+                    Text shardReverseIndexTableName, boolean writeFrequencyCounts) {
         this.metadataTableName = metadataTableName;
         this.loadDatesTableName = loadDatesTableName;
-        this.frequency = frequency;
+        this.writeFrequencyCounts = writeFrequencyCounts;
 
         this.indexedFieldsLoadDateCounts = new MetadataCounterGroup("FIELD_NAME", shardIndexTableName);
         this.reverseIndexedFieldsLoadDateCounts = new MetadataCounterGroup("FIELD_NAME", shardReverseIndexTableName);
     }
 
     @Override
     public void addEvent(IngestHelperInterface helper, RawRecordContainer event, Multimap<String,NormalizedContentInterface> fields, long loadTimeInMillis) {
-        addEvent(helper, event, fields, this.frequency, INCLUDE_LOAD_DATES, loadTimeInMillis);
+        addEvent(helper, event, fields, this.writeFrequencyCounts, INCLUDE_LOAD_DATES, loadTimeInMillis);
     }
 
     @Override
     public void addEvent(IngestHelperInterface helper, RawRecordContainer event, Multimap<String,NormalizedContentInterface> fields) {
-        addEvent(helper, event, fields, this.frequency, INCLUDE_LOAD_DATES, System.currentTimeMillis());
+        addEvent(helper, event, fields, this.writeFrequencyCounts, INCLUDE_LOAD_DATES, System.currentTimeMillis());
     }
 
     @Override
     public void addEventWithoutLoadDates(IngestHelperInterface helper, RawRecordContainer event, Multimap<String,NormalizedContentInterface> fields) {
-        addEvent(helper, event, fields, this.frequency, EXCLUDE_LOAD_DATES, System.currentTimeMillis());
+        addEvent(helper, event, fields, this.writeFrequencyCounts, EXCLUDE_LOAD_DATES, System.currentTimeMillis());
     }
 
     @Override
@@ -241,17 +244,17 @@ protected void addToFrequencyCounts(RawRecordContainer event, String fieldName,
 
     protected void updateForIndexedField(@SuppressWarnings("UnusedParameters") IngestHelperInterface helper, RawRecordContainer event,
                     Multimap<String,NormalizedContentInterface> fields, long countDelta, String loadDate, String tokenDesignator, String fieldName) {
-        update(event, fields.get(fieldName), tokenDesignator, countDelta, loadDate, indexedFieldsInfo, indexedFieldsLoadDateCounts);
+        update(event, fields.get(fieldName), tokenDesignator, countDelta, loadDate, indexedCounts, indexedFieldsLoadDateCounts);
     }
 
     private void updateForCompositeField(@SuppressWarnings("UnusedParameters") IngestHelperInterface helper, RawRecordContainer event,
                     Multimap<String,NormalizedContentInterface> fields, long countDelta, String loadDate, String tokenDesignator, String fieldName) {
-        update(event, fields.get(fieldName), tokenDesignator, countDelta, loadDate, indexedFieldsInfo, indexedFieldsLoadDateCounts);
+        update(event, fields.get(fieldName), tokenDesignator, countDelta, loadDate, indexedCounts, indexedFieldsLoadDateCounts);
     }
 
     protected void updateForReverseIndexedField(@SuppressWarnings("UnusedParameters") IngestHelperInterface helper, RawRecordContainer event,
                     Multimap<String,NormalizedContentInterface> fields, long countDelta, String loadDate, String tokenDesignator, String fieldName) {
-        update(event, fields.get(fieldName), tokenDesignator, countDelta, loadDate, reverseIndexedFieldsInfo, reverseIndexedFieldsLoadDateCounts);
+        update(event, fields.get(fieldName), tokenDesignator, countDelta, loadDate, reverseIndexedCounts, reverseIndexedFieldsLoadDateCounts);
     }
 
     protected void update(List<datawave.data.type.Type<?>> types, RawRecordContainer event, Collection<NormalizedContentInterface> norms,
@@ -300,6 +303,28 @@ protected void update(RawRecordContainer event, Collection<NormalizedContentInte
         }
     }
 
+    protected void update(RawRecordContainer event, Collection<NormalizedContentInterface> norms, String tokenDesignator, long countDelta, String loadDate,
+                    MetadataWithEventDate metadata, MetadataCounterGroup counts) {
+        for (NormalizedContentInterface norm : norms) {
+            String field = norm.getIndexedFieldName() + tokenDesignator;
+            metadata.put(field, event.getDataType().outputName(), event.getDate());
+            if (null != loadDate) {
+                updateLoadDateCounters(counts, event, field, countDelta, loadDate);
+            }
+        }
+    }
+
+    protected void update(RawRecordContainer event, Collection<NormalizedContentInterface> norms, String tokenDesignator, long countDelta, String loadDate,
+                    MetadataCounterGroup metadata, MetadataCounterGroup counts) {
+        for (NormalizedContentInterface norm : norms) {
+            String field = norm.getIndexedFieldName() + tokenDesignator;
+            metadata.addToCount(countDelta, event.getDataType().outputName(), field, DateHelper.format(event.getDate()));
+            if (null != loadDate) {
+                updateLoadDateCounters(counts, event, field, countDelta, loadDate);
+            }
+        }
+    }
+
     protected void updateMetadata(MetadataWithMostRecentDate metadataInfo, @SuppressWarnings("UnusedParameters") IngestHelperInterface helper,
                     RawRecordContainer event, Multimap<String,NormalizedContentInterface> fields, String fieldName) {
         // will ignore the update load counters because it's null
@@ -381,10 +406,9 @@ public Multimap<BulkIngestKey,Value> getBulkMetadata() {
         addIndexedFieldToMetadata(bulkData, eventFieldsInfo);
         addIndexedFieldToMetadata(bulkData, termFrequencyFieldsInfo);
 
-        addIndexedFieldToMetadata(bulkData, indexedFieldsInfo);
-        addIndexedFieldToMetadata(bulkData, reverseIndexedFieldsInfo);
-
-        addFrequenciesToMetadata(bulkData);
+        addCountsToMetadata(bulkData, indexedCounts);
+        addCountsToMetadata(bulkData, reverseIndexedCounts);
+        addCountsToMetadata(bulkData, frequencyCounts);
 
         addIndexedFieldToMetadata(bulkData, dataTypeFieldsInfo);
         addIndexedFieldToMetadata(bulkData, normalizedFieldsInfo);
@@ -400,7 +424,7 @@ public Multimap<BulkIngestKey,Value> getBulkMetadata() {
 
     protected void addToLoadDates(Multimap<BulkIngestKey,Value> results, MetadataCounterGroup countsGroup) {
         if (loadDatesTableName != null) {
-            for (MetadataCounterGroup.CountAndKeyComponents entry : countsGroup.getEntries()) {
+            for (MetadataCounterGroup.Components entry : countsGroup.getEntries()) {
                 Long count = entry.getCount();
                 Key k = new Key(new Text(entry.getRowId()), countsGroup.getColumnFamily(), new Text(entry.getDate() + DELIMITER + entry.getDataType()));
 
@@ -409,13 +433,14 @@ protected void addToLoadDates(Multimap<BulkIngestKey,Value> results, MetadataCou
         }
     }
 
-    protected void addFrequenciesToMetadata(Multimap<BulkIngestKey,Value> results) {
-        if (frequency) {
-            for (MetadataCounterGroup.CountAndKeyComponents entry : frequencyCounts.getEntries()) {
+    protected void addCountsToMetadata(Multimap<BulkIngestKey,Value> results, MetadataCounterGroup frequencies) {
+        // Do not write the counts if these are for "f" rows and writeFrequencyCounts is false.
+        if (!frequencies.getColumnFamily().equals(ColumnFamilyConstants.COLF_F) || writeFrequencyCounts) {
+            for (MetadataCounterGroup.Components entry : frequencies.getEntries()) {
                 Long count = entry.getCount();
-                Key k = new Key(new Text(entry.getRowId()), frequencyCounts.getColumnFamily(), new Text(entry.getDataType() + DELIMITER + entry.getDate()),
+                Key key = new Key(new Text(entry.getRowId()), frequencies.getColumnFamily(), new Text(entry.getDataType() + DELIMITER + entry.getDate()),
                                 DateHelper.parse(entry.getDate()).getTime());
-                addToResults(results, count, k, this.metadataTableName);
+                addToResults(results, count, key, this.metadataTableName);
             }
         }
     }
@@ -426,7 +451,7 @@ protected void addToResults(Multimap<BulkIngestKey,Value> results, Long value, K
     }
 
     protected void addIndexedFieldToMetadata(Multimap<BulkIngestKey,Value> results, MetadataWithMostRecentDate mostRecentDates) {
-        for (MetadataWithMostRecentDate.MostRecentEventDateAndKeyComponents entry : mostRecentDates.entries()) {
+        for (MetadataWithMostRecentDate.Components entry : mostRecentDates.entries()) {
             long mostRecentDate = entry.getMostRecentDate();
             Text fieldName = new Text(entry.getFieldName());
             Text colq = new Text(entry.getDataType());
@@ -445,10 +470,9 @@ public void clear() {
         this.termFrequencyFieldsInfo.clear();
         this.frequencyCounts.clear();
 
-        this.indexedFieldsInfo.clear();
-        this.indexedFieldsLoadDateCounts.clear();
+        this.indexedCounts.clear();
+        this.reverseIndexedCounts.clear();
 
-        this.reverseIndexedFieldsInfo.clear();
         this.reverseIndexedFieldsLoadDateCounts.clear();
     }
 }

warehouse/ingest-core/src/main/java/datawave/ingest/metadata/MetadataCounterGroup.java

@@ -11,7 +11,7 @@
 public class MetadataCounterGroup {
     private static final Logger log = Logger.getLogger(MetadataCounterGroup.class);
     private final Text columnFamily;
-    private HashMap<String,CountAndKeyComponents> counts = new HashMap<>();
+    private HashMap<String,Components> counts = new HashMap<>();
 
     public MetadataCounterGroup(String groupName, Text tableName) {
         this.columnFamily = new Text(groupName + RawRecordMetadata.DELIMITER + tableName);
@@ -28,9 +28,9 @@ private static String createKey(String dataType, String rowid, String date) {
     /* rowId is either the fieldName or the Lac */
     public void addToCount(long countDelta, String dataType, String rowId, String date) {
         String hashMapKey = createKey(dataType, rowId, date);
-        CountAndKeyComponents value = counts.get(hashMapKey);
+        Components value = counts.get(hashMapKey);
         if (null == value) {
-            counts.put(hashMapKey, new CountAndKeyComponents(dataType, rowId, date, countDelta));
+            counts.put(hashMapKey, new Components(dataType, rowId, date, countDelta));
         } else {
             value.incrementCount(countDelta);
         }
@@ -44,17 +44,17 @@ public Text getColumnFamily() {
         return columnFamily;
     }
 
-    public Collection<CountAndKeyComponents> getEntries() {
+    public Collection<Components> getEntries() {
         return counts.values();
     }
 
-    public class CountAndKeyComponents {
+    public static class Components {
         private final String dataType;
         private final String rowId;
         private final String date;
         private long count;
 
-        public CountAndKeyComponents(String dataType, String rowId, String date, long countDelta) {
+        public Components(String dataType, String rowId, String date, long countDelta) {
             this.dataType = dataType;
             this.rowId = rowId;
             this.date = date;
@@ -84,18 +84,18 @@ public int hashCode() {
             return hashCode;
         }
 
+        public long getCount() {
+            return count;
+        }
+
         @Override
         public boolean equals(Object o) {
-            if (!(o instanceof CountAndKeyComponents)) {
+            if (!(o instanceof Components)) {
                 return false;
             }
-            CountAndKeyComponents other = (CountAndKeyComponents) o;
+            Components other = (Components) o;
             return Objects.equal(this.dataType, other.dataType) && Objects.equal(this.rowId, other.rowId) && Objects.equal(this.date, other.date)
                             && count == other.count;
         }
-
-        public long getCount() {
-            return count;
-        }
     }
 }

warehouse/ingest-core/src/main/java/datawave/ingest/metadata/MetadataWithEventDate.java

@@ -0,0 +1,74 @@
+package datawave.ingest.metadata;
+
+import java.util.HashSet;
+import java.util.Objects;
+import java.util.Set;
+
+import org.apache.hadoop.io.Text;
+
+public class MetadataWithEventDate {
+
+    private final Text columnFamily;
+    private final Set<Components> entries = new HashSet<>();
+
+    public MetadataWithEventDate(Text columnFamily) {
+        this.columnFamily = columnFamily;
+    }
+
+    public Text getColumnFamily() {
+        return columnFamily;
+    }
+
+    public void put(String fieldName, String dataTypeName, long date) {
+        entries.add(new Components(fieldName, dataTypeName, date));
+    }
+
+    public void clear() {
+        entries.clear();
+    }
+
+    public Set<Components> getEntries() {
+        return entries;
+    }
+
+    public static class Components {
+        private final String fieldName;
+        private final String dataType;
+        private final long date;
+
+        public Components(String fieldName, String dataType, long date) {
+            this.fieldName = fieldName;
+            this.dataType = dataType;
+            this.date = date;
+        }
+
+        public String getFieldName() {
+            return fieldName;
+        }
+
+        public String getDataType() {
+            return dataType;
+        }
+
+        public long getDate() {
+            return date;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            Components that = (Components) o;
+            return Objects.equals(fieldName, that.fieldName) && Objects.equals(dataType, that.dataType) && Objects.equals(date, that.date);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(fieldName, dataType, date);
+        }
+    }
+}