diff --git a/contrib/datawave-quickstart/docker/Dockerfile b/contrib/datawave-quickstart/docker/Dockerfile index 15bfe844470..24a59eccce5 100644 --- a/contrib/datawave-quickstart/docker/Dockerfile +++ b/contrib/datawave-quickstart/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM centos:centos7 +FROM rockylinux/rockylinux:8 ARG DATAWAVE_COMMIT_ID ARG DATAWAVE_BRANCH_NAME diff --git a/core/query/src/main/java/datawave/core/query/configuration/QueryData.java b/core/query/src/main/java/datawave/core/query/configuration/QueryData.java index 564e3eabea7..a5a4a868a1a 100644 --- a/core/query/src/main/java/datawave/core/query/configuration/QueryData.java +++ b/core/query/src/main/java/datawave/core/query/configuration/QueryData.java @@ -95,35 +95,6 @@ public QueryData(QueryData other) { this.finished = other.finished; } - @Deprecated(since = "6.5.0", forRemoval = true) - public QueryData(String tableName, String query, Collection ranges, List settings) { - setTableName(tableName); - setQuery(query); - setRanges(ranges); - setSettings(settings); - } - - /** - * Weak copy constructor that updates the ranges - * - * @param other - * another QueryData - * @param ranges - * a collection of updated ranges - * @deprecated - */ - @Deprecated(since = "6.5.0", forRemoval = true) - public QueryData(QueryData other, Collection ranges) { - this(other); - setRanges(ranges); - } - - @Deprecated(since = "6.5.0", forRemoval = true) - public QueryData(String tableName, String queryString, List ranges, List settings, Collection columnFamilies) { - this(tableName, queryString, ranges, settings); - this.columnFamilies.addAll(columnFamilies); - } - // builder style methods public QueryData withTableName(String tableName) { diff --git a/core/utils/metadata-utils b/core/utils/metadata-utils index ebca7ceddb2..50670a0ae32 160000 --- a/core/utils/metadata-utils +++ b/core/utils/metadata-utils @@ -1 +1 @@ -Subproject commit ebca7ceddb2de9e5f6ff86ec02db308f0036d897 +Subproject commit 50670a0ae3298021912c84e8796bc2d9b095c99f diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d1842371c68..0aec8892f3e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -47,7 +47,7 @@ services: test: ["CMD-SHELL", "! accumuloStatus | grep DW-WARN > /dev/null"] consul: - image: consul:1.15.4 + image: docker.io/hashicorp/consul:1.15.4 hostname: localhost environment: - 'CONSUL_LOCAL_CONFIG={"log_level": "trace", "datacenter": "demo_dc", "disable_update_check": true, "enable_agent_tls_for_checks": true, "addresses": {"https": "0.0.0.0"}, "ports": {"https": 8501, "grpc_tls": 8503}, "tls": {"defaults": {"key_file": "/etc/pki/testServer.key", "cert_file": "/etc/pki/testServer.crt", "ca_file": "/etc/pki/testCA.pem", "verify_outgoing": true}, "internal_rpc": {"verify_server_hostname": false}}}' @@ -65,7 +65,7 @@ services: - demo rabbitmq: - image: rabbitmq:3.12.4-alpine + image: docker.io/rabbitmq:3.12.4 volumes: - ${RABBITMQ_CONFIG_DIR:-./rabbitmq-config}:/etc/rabbitmq - ./logs:/logs @@ -82,7 +82,7 @@ services: # When auto.create.topics.enable is true, this causes deleted topics to be recreated at random. So, leave it disabled. kafka: - image: bitnami/kafka:3.2.3 + image: docker.io/bitnami/kafka:3.2 ports: - "9094:9094" networks: @@ -104,7 +104,7 @@ services: profiles: - management - full - image: obsidiandynamics/kafdrop + image: docker.io/obsidiandynamics/kafdrop ports: - "8999:9000" networks: @@ -339,7 +339,7 @@ services: mysql: profiles: - cachedresults - image: mysql:8.0.32 + image: docker.io/mysql:8.0.32 environment: - MYSQL_RANDOM_ROOT_PASSWORD=true - MYSQL_DATABASE=cachedresults @@ -584,7 +584,7 @@ services: profiles: - management - full - image: hazelcast/management-center:5.1.2 + image: docker.io/hazelcast/management-center:5.1.2 environment: - |- JAVA_OPTS= diff --git a/microservices/services/query-metric b/microservices/services/query-metric index 18a35f833c5..ffbd425bf9d 160000 --- a/microservices/services/query-metric +++ b/microservices/services/query-metric @@ -1 +1 @@ -Subproject commit 18a35f833c5346d020202f3baa8d6237951853db +Subproject commit ffbd425bf9dbbf4ba4bc775a9c5de62b483d5342 diff --git a/microservices/starters/query-metric b/microservices/starters/query-metric index cac7e346233..03505d85a0b 160000 --- a/microservices/starters/query-metric +++ b/microservices/starters/query-metric @@ -1 +1 @@ -Subproject commit cac7e346233493d8fd3b767f48e2456008d343b9 +Subproject commit 03505d85a0bd497d35864f9f008c0738bc7e909f diff --git a/properties/default.properties b/properties/default.properties index c4ca9644aab..abfd75a7605 100644 --- a/properties/default.properties +++ b/properties/default.properties @@ -425,8 +425,6 @@ hierarchy.field.options= # BaseEventQuery (beq) thresholds beq.baseIteratorPriority=100 -beq.eventPerDayThreshold=40000 -beq.shardsPerDayThreshold=20 # max number of terms BEFORE all expansions (calculated based on how much the initial parser can handle before hitting a stack overflow: between 3500 and 3750) beq.initialMaxTermThreshold=2000 # max number of terms AFTER all expansions (calculated based on how much the initial parser can handle before hitting a stack overflow: between 3500 and 3750) diff --git a/warehouse/query-core/src/main/java/datawave/query/config/LookupUUIDTune.java b/warehouse/query-core/src/main/java/datawave/query/config/LookupUUIDTune.java index 1efa238bb6c..7e5dd61b73a 100644 --- a/warehouse/query-core/src/main/java/datawave/query/config/LookupUUIDTune.java +++ b/warehouse/query-core/src/main/java/datawave/query/config/LookupUUIDTune.java @@ -241,10 +241,12 @@ public String getQueryIteratorClass() { return queryIteratorClass; } + @Deprecated(since = "7.1.0", forRemoval = true) public int getMaxShardsPerDayThreshold() { return maxShardsPerDayThreshold; } + @Deprecated(since = "7.1.0", forRemoval = true) public void setMaxShardsPerDayThreshold(int maxShardsPerDayThreshold) { this.maxShardsPerDayThreshold = maxShardsPerDayThreshold; } diff --git a/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java b/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java index 2dbfdf655b4..82fdb658534 100644 --- a/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java +++ b/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java @@ -1291,18 +1291,22 @@ public void setUnevaluatedFields(Collection unevaluatedFields) { } } + @Deprecated(since = "7.1.0", forRemoval = true) public int getEventPerDayThreshold() { return eventPerDayThreshold; } + @Deprecated(since = "7.1.0", forRemoval = true) public void setEventPerDayThreshold(int eventPerDayThreshold) { this.eventPerDayThreshold = eventPerDayThreshold; } + @Deprecated(since = "7.1.0", forRemoval = true) public int getShardsPerDayThreshold() { return shardsPerDayThreshold; } + @Deprecated(since = "7.1.0", forRemoval = true) public void setShardsPerDayThreshold(int shardsPerDayThreshold) { this.shardsPerDayThreshold = shardsPerDayThreshold; } diff --git a/warehouse/query-core/src/main/java/datawave/query/index/lookup/RangeStream.java b/warehouse/query-core/src/main/java/datawave/query/index/lookup/RangeStream.java index 7da86528ab8..0eb3fe6b144 100644 --- a/warehouse/query-core/src/main/java/datawave/query/index/lookup/RangeStream.java +++ b/warehouse/query-core/src/main/java/datawave/query/index/lookup/RangeStream.java @@ -570,8 +570,7 @@ public ScannerStream visit(ASTEQNode node, Object data) { if (limitScanners) { // Setup the CreateUidsIterator - scannerSession = scanners.newRangeScanner(config.getIndexTableName(), config.getAuthorizations(), config.getQuery(), - config.getShardsPerDayThreshold()); + scannerSession = scanners.newRangeScanner(config.getIndexTableName(), config.getAuthorizations(), config.getQuery()); uidSetting = new IteratorSetting(stackStart++, createUidsIteratorClass); uidSetting.addOption(CreateUidsIterator.COLLAPSE_UIDS, Boolean.toString(collapseUids)); @@ -581,8 +580,7 @@ public ScannerStream visit(ASTEQNode node, Object data) { } else { // Setup so this is a pass-through - scannerSession = scanners.newRangeScanner(config.getIndexTableName(), config.getAuthorizations(), config.getQuery(), - config.getShardsPerDayThreshold()); + scannerSession = scanners.newRangeScanner(config.getIndexTableName(), config.getAuthorizations(), config.getQuery()); uidSetting = new IteratorSetting(stackStart++, createUidsIteratorClass); uidSetting.addOption(CreateUidsIterator.COLLAPSE_UIDS, Boolean.toString(false)); diff --git a/warehouse/query-core/src/main/java/datawave/query/planner/QueryPlan.java b/warehouse/query-core/src/main/java/datawave/query/planner/QueryPlan.java index 176575769b1..2484f4e600d 100644 --- a/warehouse/query-core/src/main/java/datawave/query/planner/QueryPlan.java +++ b/warehouse/query-core/src/main/java/datawave/query/planner/QueryPlan.java @@ -1,17 +1,11 @@ package datawave.query.planner; -import static datawave.query.iterator.QueryOptions.QUERY; -import static datawave.query.iterator.QueryOptions.RANGES; - import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.data.Range; -import org.apache.commons.jexl3.parser.ASTJexlScript; import org.apache.commons.jexl3.parser.JexlNode; import org.apache.commons.jexl3.parser.ParseException; import org.apache.commons.lang.builder.EqualsBuilder; @@ -20,10 +14,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; import datawave.core.common.logging.ThreadConfigurableLogger; -import datawave.core.query.configuration.QueryData; import datawave.query.jexl.JexlASTHelper; import datawave.query.jexl.visitors.JexlStringBuildingVisitor; import datawave.query.util.count.CountMap; @@ -118,105 +110,6 @@ public QueryPlan(QueryPlan other) { this.rebuildHashCode = other.rebuildHashCode; } - /** - * Partial constructor, missing IteratorSetting - * - * @param tableName - * the table name - * @param queryTreeString - * the query string - * @param queryTree - * the query tree - * @param ranges - * the ranges - * @deprecated - */ - @Deprecated(since = "6.9.0", forRemoval = true) - public QueryPlan(String tableName, String queryTreeString, JexlNode queryTree, Iterable ranges) { - this(tableName, queryTreeString, queryTree, ranges, null); - } - - @Deprecated(since = "6.9.0", forRemoval = true) - public QueryPlan(String tableName, String queryTreeString, JexlNode queryTree, Iterable ranges, List settings) { - Preconditions.checkNotNull(queryTree); - this.tableName = tableName; - this.queryTree = queryTree; - this.queryTreeString = queryTreeString; - this.ranges = Lists.newArrayList(ranges); - if (null != settings) { - this.settings = settings; - } - resetHashCode(); - } - - @Deprecated(since = "6.9.0", forRemoval = true) - public QueryPlan(String tableName, JexlNode queryTree, Iterable ranges, Collection columnFamilies) { - Preconditions.checkNotNull(queryTree); - this.tableName = tableName; - this.queryTree = queryTree; - this.ranges = Lists.newArrayList(ranges); - this.columnFamilies = Lists.newArrayList(columnFamilies); - resetHashCode(); - } - - @Deprecated(since = "6.9.0", forRemoval = true) - public QueryPlan(String tableName, JexlNode queryTree, Range range) { - Preconditions.checkNotNull(queryTree); - this.tableName = tableName; - this.queryTree = queryTree; - this.ranges = Lists.newArrayList(range); - resetHashCode(); - } - - @Deprecated(since = "6.9.0", forRemoval = true) - public QueryPlan(QueryData currentQueryData) throws ParseException { - this.tableName = currentQueryData.getTableName(); - this.queryTreeString = currentQueryData.getQuery(); - this.ranges = Lists.newArrayList(currentQueryData.getRanges()); - this.settings.addAll(currentQueryData.getSettings()); - this.columnFamilies.addAll(currentQueryData.getColumnFamilies()); - resetHashCode(); - } - - /** - * @param tableName - * @param queryTree - * @param rangeIter - * @param settings - * @param columnFamilies - */ - @Deprecated(since = "6.9.0", forRemoval = true) - public QueryPlan(String tableName, JexlNode queryTree, Iterable rangeIter, List settings, Collection columnFamilies) { - this.tableName = tableName; - this.queryTree = queryTree; - this.ranges = Lists.newArrayList(rangeIter); - for (IteratorSetting setting : settings) { - IteratorSetting newSetting = new IteratorSetting(setting.getPriority(), setting.getName(), setting.getIteratorClass()); - newSetting.addOptions(setting.getOptions()); - if (newSetting.getOptions().containsKey(QUERY)) { - newSetting.addOption(QUERY, JexlStringBuildingVisitor.buildQuery(queryTree)); - newSetting.addOption(RANGES, this.ranges.stream().map(Range::toString).collect(Collectors.joining(",", "[", "]"))); - } - this.settings.add(newSetting); - - } - if (null != columnFamilies) { - this.columnFamilies.addAll(columnFamilies); - } - resetHashCode(); - } - - /** - * @param tableName - * @param queryTree - * @param rangeIter - * @param settings - */ - @Deprecated(since = "6.9.0", forRemoval = true) - public QueryPlan(String tableName, JexlNode queryTree, Iterable rangeIter, List settings) { - this(tableName, queryTree, rangeIter, settings, null); - } - public QueryPlan(JexlNode queryTree, Collection ranges) { this.queryTree = queryTree; this.ranges = ranges; @@ -340,20 +233,6 @@ public void setQueryTreeString(String queryString) { resetHashCode(); } - @Deprecated(since = "6.9.0", forRemoval = true) - public void setQuery(String queryString, JexlNode queryTree) { - this.queryTree = queryTree; - this.queryTreeString = queryString; - resetHashCode(); - } - - @Deprecated(since = "6.9.0", forRemoval = true) - public void setQuery(String queryString, ASTJexlScript queryTree) { - this.queryTree = queryTree; - this.queryTreeString = queryString; - resetHashCode(); - } - public String getQueryString() { if (null == queryTreeString) { Preconditions.checkNotNull(queryTree); diff --git a/warehouse/query-core/src/main/java/datawave/query/postprocessing/tf/TermOffsetPopulator.java b/warehouse/query-core/src/main/java/datawave/query/postprocessing/tf/TermOffsetPopulator.java index a3c2a83399d..a05714c1a67 100644 --- a/warehouse/query-core/src/main/java/datawave/query/postprocessing/tf/TermOffsetPopulator.java +++ b/warehouse/query-core/src/main/java/datawave/query/postprocessing/tf/TermOffsetPopulator.java @@ -38,7 +38,6 @@ import com.google.protobuf.InvalidProtocolBufferException; import datawave.core.iterators.TermFrequencyIterator; -import datawave.data.type.Type; import datawave.ingest.protobuf.TermWeight; import datawave.ingest.protobuf.TermWeightPosition; import datawave.query.Constants; @@ -242,25 +241,6 @@ public static Multimap getContentFunctions(JexlNode node) { return FunctionReferenceVisitor.functions(node, Collections.singleton(CONTENT_FUNCTION_NAMESPACE)); } - /** - * @deprecated because the term frequency values are already normalized in the query - *

- * Get the list of content function fields to normalized values - * - * @param contentExpansionFields - * set of content expansion fields - * @param dataTypes - * map of datatypes - * @param functions - * map of functions - * @return list of content function fields to normalized values - */ - @Deprecated(since = "5.9.0", forRemoval = true) - public static Multimap getContentFieldValues(Set contentExpansionFields, Multimap>> dataTypes, - Multimap functions) { - return getContentFieldValues(contentExpansionFields, functions); - } - /** * Get the fields and values from the content functions * @@ -308,31 +288,6 @@ public static Multimap getContentFieldValues(Set contentE return contentFieldValues; } - /** - * @deprecated because the datatype normalization is no longer required - *

- * A method to get the list of fields and values for which term frequencies need to be gathered. ASSUMPTION: The query planner (@see - * DefaultQueryPlanner) has: 1) expanded the content functions into the query 2) the values in the query have already been normalized - * appropriately 3) the query has been reduced to those values actually in the index The query is scraped for content functions from which a - * list of zones to normalized values is determined (the contentExpansionFields are used for unfielded content functions). The list of fields to - * values as a subset of the term frequency fields is gathered. The intersection of those two sets are returned. - * - * @param dataTypes - * map of datatypes - * @param contentExpansionFields - * set of content expansion fields - * @param query - * the query script - * @param termFrequencyFields - * set of term frequency fields - * @return list of fields and values for which term frequencies need to be gathered - */ - @Deprecated(since = "5.9.0", forRemoval = true) - public static Multimap getTermFrequencyFieldValues(ASTJexlScript query, Set contentExpansionFields, Set termFrequencyFields, - Multimap>> dataTypes) { - return getTermFrequencyFieldValues(query, contentExpansionFields, termFrequencyFields); - } - public static Multimap getTermFrequencyFieldValues(ASTJexlScript query, Set contentExpansionFields, Set termFrequencyFields) { Multimap functions = TermOffsetPopulator.getContentFunctions(query); @@ -350,24 +305,6 @@ public static Multimap getTermFrequencyFieldValues(ASTJexlScript return HashMultimap.create(); } - /** - * @deprecated in 5.9.0 because the datatypes multimap is not necessary for term frequency aggregation - * @param functions - * a multimap of functions - * @param contentExpansionFields - * the content expansion fields - * @param queryFieldValues - * the query field values - * @param dataTypes - * a multimap of datatypes - * @return term frequency fields and values - */ - @Deprecated(since = "5.9.0", forRemoval = true) - public static Multimap getTermFrequencyFieldValues(Multimap functions, Set contentExpansionFields, - Multimap queryFieldValues, Multimap>> dataTypes) { - return getTermFrequencyFieldValues(functions, contentExpansionFields, queryFieldValues); - } - public static Multimap getTermFrequencyFieldValues(Multimap functions, Set contentExpansionFields, Multimap queryFieldValues) { // get the intersection of the content expansion fields (or term frequency fields) and those that are in the content functions diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/RangeStreamScanner.java b/warehouse/query-core/src/main/java/datawave/query/tables/RangeStreamScanner.java index f6cb3d74f1c..94a332e9772 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/RangeStreamScanner.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/RangeStreamScanner.java @@ -9,7 +9,6 @@ import java.util.Collection; import java.util.Collections; import java.util.Iterator; -import java.util.Map.Entry; import java.util.Queue; import java.util.Set; import java.util.concurrent.Callable; @@ -20,8 +19,6 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import javax.annotation.Nullable; - import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.client.ScannerBase; @@ -33,11 +30,9 @@ import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.PeekingIterator; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; import org.apache.hadoop.io.Text; import org.apache.log4j.Logger; -import com.google.common.base.Function; import com.google.common.base.Throwables; import com.google.common.collect.Iterators; import com.google.common.collect.Queues; @@ -55,28 +50,26 @@ /** * Purpose: Extends Scanner session so that we can modify how we build our subsequent ranges. Breaking this out cleans up the code. May require implementation * specific details if you are using custom iterators, as we are reinitializing a seek - * + *

* Design: Extends Scanner session and only overrides the buildNextRange. - * + *

* The {@link datawave.query.index.lookup.RangeStream} configures the iterator running against the global index. - * + *

* Typically the iterator is a {@link datawave.query.index.lookup.CreateUidsIterator} or variant. - * + *

* The iterator returns a tuple of shard - {@link IndexInfo} object pairs, each shard representing and day or shard range. - * + *

* Results from the iterator are put onto the currentQueue and then flushed into the resultQueue. Under certain circumstances these results may be modified * final to the prior flush into the resultQueue. - * + *

* The RangeStreamScanner supports "seeking" the global index iterator. Because the RangeStreamScanner supports a {@link PeekingIterator} some implementation * details are not immediately obvious. For more information, see {@link #seek(String)}. */ public class RangeStreamScanner extends ScannerSession implements Callable { - private static final int MAX_MEDIAN = 20; private static final Logger log = Logger.getLogger(RangeStreamScanner.class); - private int shardsPerDayThreshold = Integer.MAX_VALUE; // simply compare the strings. no need for a date formatter - protected static final int dateCfLength = 8; + protected static final int DATE_CF_LENGTH = 8; protected boolean seenUnexpectedKey = false; protected ArrayDeque currentQueue; @@ -144,13 +137,13 @@ public RangeStreamScanner setScannerFactory(ScannerFactory factory) { /** * Override this for your specific implementation. - * + *

* In this specific implementation our row key will be the term, the column family will be the field name, and the column family will be the shard,so we * should have the following as our last key - * + *

* bar FOO:20130101_0 - * - * so we should append a null so that we we don't skip shards. similarly, an assumption is made of the key structure within this class. + *

+ * so we should append a null so that we don't skip shards. similarly, an assumption is made of the key structure within this class. * * @param lastKey * the last key @@ -435,9 +428,6 @@ protected int scannerInvariant(final Iterator iter) { PeekingIterator kvIter = new PeekingIterator<>(iter); int retrievalCount = 0; - - Result myEntry; - String currentDay = null; if (null != prevDay) { @@ -452,13 +442,11 @@ protected int scannerInvariant(final Iterator iter) { return 0; } } - // produces stats for us, so we don't have to! - DescriptiveStatistics stats = new DescriptiveStatistics(); writeLock.lock(); try { while (kvIter.hasNext()) { - Result currentKeyValue = kvIter.peek(); + Result currentKeyValue = kvIter.peek(); // become a pass-through if we've seen an unexpected key. if (seenUnexpectedKey) { @@ -475,7 +463,6 @@ protected int scannerInvariant(final Iterator iter) { currentDay = getDay(currentKeyValue.getKey()); currentQueue.add(trimTrailingUnderscore(currentKeyValue)); - lastSeenKey = kvIter.next().getKey(); } else { String nextKeysDay = getDay(currentKeyValue.getKey()); @@ -490,22 +477,7 @@ protected int scannerInvariant(final Iterator iter) { log.trace("adding count of " + info.count()); } - stats.addValue(info.count()); - - if (currentQueue.size() <= shardsPerDayThreshold || stats.getPercentile(50) < MAX_MEDIAN) { - - if (log.isTraceEnabled()) { - log.trace("adding our stats are " + stats.getPercentile(50) + " on " + currentQueue.size()); - } - - currentQueue.add(trimTrailingUnderscore(currentKeyValue)); - - } else { - if (log.isTraceEnabled()) { - log.trace("breaking because our stats are " + stats.getPercentile(50) + " on " + currentQueue.size()); - } - break; - } + currentQueue.add(trimTrailingUnderscore(currentKeyValue)); lastSeenKey = kvIter.next().getKey(); } else { @@ -522,36 +494,8 @@ protected int scannerInvariant(final Iterator iter) { } } - if (currentQueue.size() >= shardsPerDayThreshold && stats.getPercentile(50) > MAX_MEDIAN) { - - Result top = currentQueue.poll(); - - Key topKey = top.getKey(); - if (log.isTraceEnabled()) - log.trace(topKey + " for " + currentDay + " exceeds limit of " + shardsPerDayThreshold + " with " + currentQueue.size()); - Key newKey = new Key(topKey.getRow(), topKey.getColumnFamily(), new Text(currentDay), topKey.getColumnVisibility(), topKey.getTimestamp()); + retrievalCount += dequeue(); - Value newValue = writeInfoToValue(); - - myEntry = new Result(top.getContext(), newKey, newValue); - lastSeenKey = newKey; - - try { - if (!resultQueue.offer(myEntry, 1, TimeUnit.SECONDS)) { - if (log.isTraceEnabled()) { - log.trace("could not add day! converting " + myEntry + " to " + prevDay); - } - prevDay = myEntry; - } - } catch (InterruptedException exception) { - prevDay = myEntry; - } - - currentQueue.clear(); - - } else { - retrievalCount += dequeue(); - } } finally { writeLock.unlock(); } @@ -673,8 +617,8 @@ protected boolean flushNeeded() { protected String getDay(final Key key) { String myDay = null; byte[] cq = key.getColumnQualifierData().getBackingArray(); - if (cq.length >= dateCfLength) { - myDay = new String(cq, 0, dateCfLength); + if (cq.length >= DATE_CF_LENGTH) { + myDay = new String(cq, 0, DATE_CF_LENGTH); if (log.isTraceEnabled()) { log.trace("Day is " + myDay + " for " + key); } @@ -700,11 +644,6 @@ public static String shardFromKey(final Key key) { } } - public RangeStreamScanner setShardsPerDayThreshold(int shardsPerDayThreshold) { - this.shardsPerDayThreshold = shardsPerDayThreshold; - return this; - } - @Override public RangeStreamScanner call() throws Exception { findTop(); @@ -795,14 +734,11 @@ else if (baseScanner instanceof RfileScanner) if (baseScanner instanceof Scanner) ((Scanner) baseScanner).setRange(currentRange); - Iterator iter = Iterators.transform(baseScanner.iterator(), new Function,Result>() { - @Override - public Result apply(@Nullable Entry input) { - if (input == null) { - return null; - } - return new Result(input.getKey(), input.getValue()); + Iterator iter = Iterators.transform(baseScanner.iterator(), input -> { + if (input == null) { + return null; } + return new Result<>(input.getKey(), input.getValue()); }); // do not continue if we've reached the end of the corpus @@ -867,8 +803,8 @@ private boolean isBeyondRange(Key lastSeenKey, Key endKey) { log.trace(cf + " " + endCf); } - if (dateCfLength == cf.length()) { - endCf = endCf.substring(0, dateCfLength); + if (DATE_CF_LENGTH == cf.length()) { + endCf = endCf.substring(0, DATE_CF_LENGTH); if (cf.compareTo(endCf) >= 0) { return true; } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ScannerFactory.java b/warehouse/query-core/src/main/java/datawave/query/tables/ScannerFactory.java index b12970e808a..cf53d01206d 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/ScannerFactory.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/ScannerFactory.java @@ -307,7 +307,7 @@ public RangeStreamScanner newRangeScanner(final String tableName, final Set auths, Query query, int shardsPerDayThreshold) throws Exception { - return newLimitedScanner(RangeStreamScanner.class, tableName, auths, settings).setShardsPerDayThreshold(shardsPerDayThreshold).setScannerFactory(this); + return newLimitedScanner(RangeStreamScanner.class, tableName, auths, settings).setScannerFactory(this); } public boolean close(ScannerBase bs) { diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java index 4ce5f1c1b8e..630baa049b0 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java @@ -1601,18 +1601,22 @@ public void setHitList(boolean hitList) { getConfig().setHitList(hitList); } + @Deprecated(since = "7.1.0", forRemoval = true) public int getEventPerDayThreshold() { return getConfig().getEventPerDayThreshold(); } + @Deprecated(since = "7.1.0", forRemoval = true) public void setEventPerDayThreshold(int eventPerDayThreshold) { getConfig().setEventPerDayThreshold(eventPerDayThreshold); } + @Deprecated(since = "7.1.0", forRemoval = true) public int getShardsPerDayThreshold() { return getConfig().getShardsPerDayThreshold(); } + @Deprecated(since = "7.1.0", forRemoval = true) public void setShardsPerDayThreshold(int shardsPerDayThreshold) { getConfig().setShardsPerDayThreshold(shardsPerDayThreshold); } diff --git a/warehouse/query-core/src/test/java/datawave/query/DataTypeQueryTest.java b/warehouse/query-core/src/test/java/datawave/query/DataTypeQueryTest.java index 579f718240a..51217148fe0 100644 --- a/warehouse/query-core/src/test/java/datawave/query/DataTypeQueryTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/DataTypeQueryTest.java @@ -131,8 +131,6 @@ public void testEventPerDay() throws Exception { String dtFilter = CityEntry.generic.getDataType(); qOptions.put(QueryParameters.DATATYPE_FILTER_SET, dtFilter); - this.logic.setEventPerDayThreshold(1); - String query = CityField.STATE.name() + EQ_OP + "'missouri'"; String expect = "(" + query + ")" + AND_OP + BaseRawData.EVENT_DATATYPE + EQ_OP + "'" + CityEntry.generic.getDataType() + "'"; runTest(query, expect, qOptions); @@ -146,9 +144,6 @@ public void testEventAndShardsPerDay() throws Exception { String dtFilter = CityEntry.generic.getDataType(); qOptions.put(QueryParameters.DATATYPE_FILTER_SET, dtFilter); - this.logic.setEventPerDayThreshold(1); - this.logic.setShardsPerDayThreshold(1); - String query = CityField.STATE.name() + EQ_OP + "'missouri'"; String expect = "(" + query + ")" + AND_OP + BaseRawData.EVENT_DATATYPE + EQ_OP + "'" + CityEntry.generic.getDataType() + "'"; runTest(query, expect, qOptions); diff --git a/warehouse/query-core/src/test/java/datawave/query/MiscQueryTest.java b/warehouse/query-core/src/test/java/datawave/query/MiscQueryTest.java index 5a0ed64db81..a2116196eca 100644 --- a/warehouse/query-core/src/test/java/datawave/query/MiscQueryTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/MiscQueryTest.java @@ -127,8 +127,6 @@ public void testFieldOpField() throws Exception { @Test public void testEventThreshold() throws Exception { log.info("------ testEventThreshold ------"); - // setting event per day does not alter results - this.logic.setEventPerDayThreshold(1); String phrase = RE_OP + "'.*a'"; String query = Constants.ANY_FIELD + phrase; String expect = this.dataManager.convertAnyField(phrase); @@ -138,8 +136,6 @@ public void testEventThreshold() throws Exception { @Test(expected = InvalidQueryException.class) public void testFieldIgnoreParam1() throws Exception { log.info("------ testFieldIgnoreParam1 ------"); - // setting event per day does not alter results - this.logic.setEventPerDayThreshold(1); String phrase = RE_OP + "'.*a'" + "&& FOO == bar2"; String query = Constants.ANY_FIELD + phrase + "&& FOO == bar2"; String expect = this.dataManager.convertAnyField(phrase); @@ -155,8 +151,6 @@ public void testFieldIgnoreParam1() throws Exception { @Test public void testFieldIgnoreParam2() throws Exception { log.info("------ testFieldIgnoreParam2 ------"); - // setting event per day does not alter results - this.logic.setEventPerDayThreshold(1); String phrase = RE_OP + "'.*a'" + "&& FOO == bar2"; String query = Constants.ANY_FIELD + phrase + "&& FOO == bar2"; String expect = this.dataManager.convertAnyField(phrase); @@ -172,8 +166,6 @@ public void testFieldIgnoreParam2() throws Exception { @Test public void testFieldIgnoreParam3() throws Exception { log.info("------ testFieldIgnoreParam3 ------"); - // setting event per day does not alter results - this.logic.setEventPerDayThreshold(1); String phrase = RE_OP + "'.*a' && STATE == 'sta'"; String query = Constants.ANY_FIELD + phrase + "&& STATE == 'sta'"; String expect = this.dataManager.convertAnyField(phrase); @@ -188,8 +180,6 @@ public void testFieldIgnoreParam3() throws Exception { @Test public void testShardThreshold() throws Exception { log.info("------ testShardThreshold ------"); - // setting shards per day does not alter results - - this.logic.setShardsPerDayThreshold(1); String phrase = RE_OP + "'.*a'"; String query = Constants.ANY_FIELD + phrase; String expect = this.dataManager.convertAnyField(phrase); diff --git a/warehouse/query-core/src/test/java/datawave/query/index/lookup/RangeStreamQueryTest.java b/warehouse/query-core/src/test/java/datawave/query/index/lookup/RangeStreamQueryTest.java index 3f25e103f88..6e0a43fbd3e 100644 --- a/warehouse/query-core/src/test/java/datawave/query/index/lookup/RangeStreamQueryTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/index/lookup/RangeStreamQueryTest.java @@ -155,7 +155,6 @@ public void setupTest() throws ParseException { config.setDatatypeFilter(Collections.singleton("datatype")); config.setQueryFieldsDatatypes(fieldToDataType); config.setIndexedFields(fieldToDataType); - config.setShardsPerDayThreshold(2); } @AfterClass diff --git a/warehouse/query-core/src/test/java/datawave/query/index/lookup/RangeStreamTest.java b/warehouse/query-core/src/test/java/datawave/query/index/lookup/RangeStreamTest.java index b2c59176dc9..1e262bf188f 100644 --- a/warehouse/query-core/src/test/java/datawave/query/index/lookup/RangeStreamTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/index/lookup/RangeStreamTest.java @@ -422,7 +422,6 @@ public static void setupAccumulo() throws Exception { public void setupTest() { config = new ShardQueryConfiguration(); config.setClient(client); - config.setShardsPerDayThreshold(20); } @Test @@ -1197,8 +1196,6 @@ public void testDropTwoPredicates() throws Exception { SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd"); config.setBeginDate(sdf.parse("20190314")); config.setEndDate(sdf.parse("20190315")); - config.setShardsPerDayThreshold(3); - config.setDatatypeFilter(Sets.newHashSet("datatype1", "datatype2")); Multimap> dataTypes = HashMultimap.create(); @@ -1212,8 +1209,9 @@ public void testDropTwoPredicates() throws Exception { helper.setIndexedFields(dataTypes.keySet()); helper.addFields(Arrays.asList("FOO", "LAUGH")); + // "20190314_10" never actually hit Set expectedRanges = Sets.newHashSet(); - for (String shard : Arrays.asList("20190314_0", "20190314_1", "20190314_10", "20190314_100", "20190314_9")) { + for (String shard : Arrays.asList("20190314_0", "20190314_1", "20190314_100", "20190314_9")) { expectedRanges.add(makeShardedRange(shard)); } @@ -1523,27 +1521,15 @@ public void testIntersection_ofDayRangesAndShardRange() throws Exception { config.setQueryFieldsDatatypes(dataTypes); config.setIndexedFields(dataTypes); - config.setShardsPerDayThreshold(0); // set to zero to roll up to day ranges MockMetadataHelper helper = new MockMetadataHelper(); helper.setIndexedFields(dataTypes.keySet()); - Range range1 = makeShardedRange("20190310_21"); - // Fun story. It's hard to roll up to a day range when you seek most of the way through the day and don't have all the shards for the day. - Range range2 = makeShardedRange("20190315_51"); - Set expectedRanges = Sets.newHashSet(range1, range2); - RangeStream rangeStream = getRangeStream(helper); rangeStream.setLimitScanners(true); CloseableIterable queryPlans = rangeStream.streamPlans(script); - assertEquals(IndexStream.StreamContext.PRESENT, rangeStream.context()); - for (QueryPlan queryPlan : queryPlans) { - Iterable ranges = queryPlan.getRanges(); - for (Range range : ranges) { - assertTrue("Tried to remove unexpected range " + range.toString() + "\nfrom expected ranges: " + expectedRanges, expectedRanges.remove(range)); - } - } - assertTrue("Expected ranges not found in query plan: " + expectedRanges, expectedRanges.isEmpty()); + assertEquals(IndexStream.StreamContext.ABSENT, rangeStream.context()); + assertFalse(queryPlans.iterator().hasNext()); } private RangeStream getRangeStream(MetadataHelper helper) { diff --git a/warehouse/query-core/src/test/java/datawave/query/index/lookup/RangeStreamTestX.java b/warehouse/query-core/src/test/java/datawave/query/index/lookup/RangeStreamTestX.java index 8062325300d..ef440984a4d 100644 --- a/warehouse/query-core/src/test/java/datawave/query/index/lookup/RangeStreamTestX.java +++ b/warehouse/query-core/src/test/java/datawave/query/index/lookup/RangeStreamTestX.java @@ -1,6 +1,5 @@ package datawave.query.index.lookup; -import static datawave.common.test.utils.query.RangeFactoryForTests.makeDayRange; import static datawave.common.test.utils.query.RangeFactoryForTests.makeShardedRange; import static datawave.common.test.utils.query.RangeFactoryForTests.makeTestRange; import static datawave.util.TableName.SHARD_INDEX; @@ -18,8 +17,6 @@ import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.BatchWriterConfig; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; @@ -51,17 +48,17 @@ /** * Cover some basic tests involving streams of shards for a basic set of query structures. Only tests for correctness of shard intersection, not that the * underlying IndexInfo objects correctly intersect. - * + *

* 6 Basic Types of Query Structures - * + *

* 3 Stream Type Combinations (All shards, shards and days, all days) - * + *

* 2 Types of Unequal Stream Start/Stop (different start/end day) - * + *

* 2 Types of Uneven Stream Start/Stop (same start/end day, different shard) - * + *

* 1 Type of Tick-Tock Shards (alternating shards such that no hits are produced for a day) - * + *

* 1 Type of Missing Shards (missing shards should drop terms from query) */ public class RangeStreamTestX { @@ -73,7 +70,6 @@ public class RangeStreamTestX { // (A && B) || (C && D) // (A || B) && (C || D) - private static InMemoryInstance instance = new InMemoryInstance(RangeStreamTestX.class.toString()); private static AccumuloClient client; private ShardQueryConfiguration config; @@ -374,7 +370,6 @@ private static Value buildValueForDay() { @Before public void setupTest() { config = new ShardQueryConfiguration(); - config.setShardsPerDayThreshold(20); config.setClient(client); } @@ -518,7 +513,7 @@ public void testIntersection_ofShardAndDay() throws Exception { for (int ii = 1; ii <= 5; ii++) { for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); - expectedQueryStrings.add("(B == 'all' && ((_Delayed_ = true) && (A == 'all_day')))"); + expectedQueryStrings.add("(B == 'all' && A == 'all_day')"); } } @@ -536,7 +531,7 @@ public void testIntersection_ofShardAndDay_unequalStart() throws Exception { for (int ii = 2; ii <= 5; ii++) { for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); - expectedQueryStrings.add("(A == 'unequal_start' && ((_Delayed_ = true) && (B == 'all_day')))"); + expectedQueryStrings.add("(A == 'unequal_start' && B == 'all_day')"); } } @@ -554,7 +549,7 @@ public void testIntersection_ofShardAndDay_unequalStop() throws Exception { for (int ii = 1; ii <= 4; ii++) { for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); - expectedQueryStrings.add("(A == 'unequal_stop' && ((_Delayed_ = true) && (B == 'all_day')))"); + expectedQueryStrings.add("(A == 'unequal_stop' && B == 'all_day')"); } } @@ -572,7 +567,7 @@ public void testIntersection_ofShardAndDay_unevenStart() throws Exception { for (int ii = 1; ii <= 5; ii++) { for (int jj = 1; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); - expectedQueryStrings.add("(A == 'uneven_start' && ((_Delayed_ = true) && (B == 'all_day')))"); + expectedQueryStrings.add("(A == 'uneven_start' && B == 'all_day')"); } } @@ -590,7 +585,7 @@ public void testIntersection_ofShardAndDay_unevenStop() throws Exception { for (int ii = 1; ii <= 5; ii++) { for (int jj = 0; jj < 9; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); - expectedQueryStrings.add("(A == 'uneven_stop' && ((_Delayed_ = true) && (B == 'all_day')))"); + expectedQueryStrings.add("(A == 'uneven_stop' && B == 'all_day')"); } } @@ -610,7 +605,7 @@ public void testIntersection_ofShardAndDay_missingShards() throws Exception { continue; for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day')) && B == 'missing_shards'"); + expectedQueryStrings.add("A == 'all_day' && B == 'missing_shards'"); } } @@ -630,7 +625,7 @@ public void testIntersection_ofShardAndDay_tickTockShards() throws Exception { for (int jj = 0; jj < 10; jj++) { if (ii != 3) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); - expectedQueryStrings.add("(A == 'tick_tock' && ((_Delayed_ = true) && (B == 'tick_tock_day')))"); + expectedQueryStrings.add("(A == 'tick_tock' && B == 'tick_tock_day')"); } } } @@ -646,8 +641,10 @@ public void testIntersection_ofDays() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && ((_Delayed_ = true) && (B == 'all_day')))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all_day' && B == 'all_day'"); + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -662,8 +659,10 @@ public void testIntersection_ofDays_unequalStart() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 2; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && ((_Delayed_ = true) && (B == 'unequal_start_day')))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all_day' && B == 'unequal_start_day'"); + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -678,8 +677,10 @@ public void testIntersection_ofDays_unequalStop() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 4; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && ((_Delayed_ = true) && (B == 'unequal_stop_day')))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all_day' && B == 'unequal_stop_day'"); + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -694,8 +695,10 @@ public void testIntersection_ofDays_unevenStart() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && ((_Delayed_ = true) && (B == 'uneven_start_day')))"); + for (int jj = 1; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all_day' && B == 'uneven_start_day'"); + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -710,8 +713,10 @@ public void testIntersection_ofDays_unevenStop() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && ((_Delayed_ = true) && (B == 'uneven_stop_day')))"); + for (int jj = 0; jj < 9; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all_day' && B == 'uneven_stop_day'"); + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -726,10 +731,14 @@ public void testIntersection_ofDays_missingShards() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - if (ii == 3) + if (ii == 3) { continue; - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && ((_Delayed_ = true) && (B == 'missing_shards_day')))"); + } + + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all_day' && B == 'missing_shards_day'"); + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -746,8 +755,10 @@ public void testIntersection_ofDays_tickTockShards() throws Exception { for (int ii = 1; ii <= 5; ii++) { if (ii != 3) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'tick_tock_day')) && ((_Delayed_ = true) && (B == 'tick_tock_day')))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("(A == 'tick_tock_day' && B == 'tick_tock_day')"); + } } } @@ -944,8 +955,10 @@ public void testUnion_ofShardAndDay() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("(A == 'all' || ((_Delayed_ = true) && (B == 'all_day')))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("(A == 'all' || B == 'all_day')"); + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -960,14 +973,14 @@ public void testUnion_ofShardAndDay_unequalStart() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - if (ii == 1) { - for (int jj = 0; jj < 10; jj++) { + for (int jj = 0; jj < 10; jj++) { + if (ii == 1) { expectedRanges.add(makeTestRange("2020010" + ii + "_" + jj, "datatype1\0a.b.c")); expectedQueryStrings.add("A == 'all'"); + } else { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all' || B == 'unequal_start_day'"); } - } else { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("A == 'all' || ((_Delayed_ = true) && (B == 'unequal_start_day'))"); } } @@ -983,14 +996,14 @@ public void testUnion_ofShardAndDay_unequalStop() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - if (ii == 5) { - for (int jj = 0; jj < 10; jj++) { + for (int jj = 0; jj < 10; jj++) { + if (ii == 5) { expectedRanges.add(makeTestRange("2020010" + ii + "_" + jj, "datatype1\0a.b.c")); expectedQueryStrings.add("A == 'all'"); + } else { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all' || B == 'unequal_stop_day'"); } - } else { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("A == 'all' || ((_Delayed_ = true) && (B == 'unequal_stop_day'))"); } } @@ -1006,8 +1019,15 @@ public void testUnion_ofShardAndDay_unevenStart() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("A == 'all' || ((_Delayed_ = true) && (B == 'uneven_start_day'))"); + for (int jj = 0; jj < 10; jj++) { + if (jj == 0) { + expectedRanges.add(makeTestRange("2020010" + ii + "_" + jj, "datatype1\u0000a.b.c")); + expectedQueryStrings.add("A == 'all'"); + } else { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all' || B == 'uneven_start_day'"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -1022,8 +1042,15 @@ public void testUnion_ofShardAndDay_unevenStop() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("A == 'all' || ((_Delayed_ = true) && (B == 'uneven_stop_day'))"); + for (int jj = 0; jj < 10; jj++) { + if (jj == 9) { + expectedRanges.add(makeTestRange("2020010" + ii + "_" + jj, "datatype1\u0000a.b.c")); + expectedQueryStrings.add("A == 'all'"); + } else { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all' || B == 'uneven_stop_day'"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -1038,14 +1065,14 @@ public void testUnion_ofShardAndDay_missingShards() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - if (ii == 3) { - for (int jj = 0; jj < 10; jj++) { + for (int jj = 0; jj < 10; jj++) { + if (ii == 3) { expectedRanges.add(makeTestRange("2020010" + ii + "_" + jj, "datatype1\0a.b.c")); expectedQueryStrings.add("A == 'all'"); + } else { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all' || B == 'missing_shards_day'"); } - } else { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("A == 'all' || ((_Delayed_ = true) && (B == 'missing_shards_day'))"); } } @@ -1060,36 +1087,17 @@ public void testUnion_ofShardAndDay_tickTockShards() throws Exception { // Shards are missing for day 3 List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); - - expectedRanges.add(makeDayRange("20200101")); - expectedRanges.add(makeDayRange("20200102")); - expectedRanges.add(makeTestRange("20200103_0", "datatype1\0a.b.c")); - expectedRanges.add(makeShardedRange("20200103_1")); - expectedRanges.add(makeTestRange("20200103_2", "datatype1\0a.b.c")); - expectedRanges.add(makeTestRange("20200103_3", "datatype1\0a.b.c")); - expectedRanges.add(makeTestRange("20200103_4", "datatype1\0a.b.c")); - expectedRanges.add(makeShardedRange("20200103_5")); - expectedRanges.add(makeTestRange("20200103_6", "datatype1\0a.b.c")); - expectedRanges.add(makeTestRange("20200103_7", "datatype1\0a.b.c")); - expectedRanges.add(makeTestRange("20200103_8", "datatype1\0a.b.c")); - expectedRanges.add(makeShardedRange("20200103_9")); - expectedRanges.add(makeDayRange("20200104")); - expectedRanges.add(makeDayRange("20200105")); - - expectedQueryStrings.add("A == 'all' || ((_Delayed_ = true) && (B == 'tick_tock_day'))"); - expectedQueryStrings.add("A == 'all' || ((_Delayed_ = true) && (B == 'tick_tock_day'))"); - expectedQueryStrings.add("A == 'all'"); - expectedQueryStrings.add("A == 'all' || B == 'tick_tock_day'"); - expectedQueryStrings.add("A == 'all'"); - expectedQueryStrings.add("A == 'all'"); - expectedQueryStrings.add("A == 'all'"); - expectedQueryStrings.add("A == 'all' || B == 'tick_tock_day'"); - expectedQueryStrings.add("A == 'all'"); - expectedQueryStrings.add("A == 'all'"); - expectedQueryStrings.add("A == 'all'"); - expectedQueryStrings.add("A == 'all' || B == 'tick_tock_day'"); - expectedQueryStrings.add("A == 'all' || ((_Delayed_ = true) && (B == 'tick_tock_day'))"); - expectedQueryStrings.add("A == 'all' || ((_Delayed_ = true) && (B == 'tick_tock_day'))"); + for (int ii = 1; ii <= 5; ii++) { + for (int jj = 0; jj < 10; jj++) { + if (ii == 3 && jj % 4 != 1) { + expectedRanges.add(makeTestRange("2020010" + ii + "_" + jj, "datatype1\u0000a.b.c")); + expectedQueryStrings.add("A == 'all'"); + } else { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all' || B == 'tick_tock_day'"); + } + } + } runTest(query, expectedRanges, expectedQueryStrings); } @@ -1102,8 +1110,10 @@ public void testUnion_ofDays() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) || ((_Delayed_ = true) && (B == 'all_day')))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all_day' || B == 'all_day'"); + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -1118,11 +1128,13 @@ public void testUnion_ofDays_unequalStart() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 1) { - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day'))"); - } else { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) || ((_Delayed_ = true) && (B == 'unequal_start_day')))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 1) { + expectedQueryStrings.add("A == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' || B == 'unequal_start_day'"); + } } } @@ -1138,11 +1150,13 @@ public void testUnion_ofDays_unequalStop() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 5) { - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day'))"); - } else { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) || ((_Delayed_ = true) && (B == 'unequal_stop_day')))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 5) { + expectedQueryStrings.add("A == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' || B == 'unequal_stop_day'"); + } } } @@ -1158,8 +1172,14 @@ public void testUnion_ofDays_unevenStart() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) || ((_Delayed_ = true) && (B == 'uneven_start_day')))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (jj == 0) { + expectedQueryStrings.add("A == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' || B == 'uneven_start_day'"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -1174,8 +1194,14 @@ public void testUnion_ofDays_unevenStop() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) || ((_Delayed_ = true) && (B == 'uneven_stop_day')))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (jj == 9) { + expectedQueryStrings.add("A == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' || B == 'uneven_stop_day'"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -1190,11 +1216,13 @@ public void testUnion_ofDays_missingShards() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 3) { - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day'))"); - } else { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) || ((_Delayed_ = true) && (B == 'missing_shards_day')))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 3) { + expectedQueryStrings.add("A == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' || B == 'missing_shards_day'"); + } } } @@ -1211,22 +1239,19 @@ public void testUnion_ofDays_tickTockShards() throws Exception { List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - if (ii != 3) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'tick_tock_day')) || ((_Delayed_ = true) && (B == 'tick_tock_day')))"); - } else { - expectedRanges.add(makeShardedRange("20200103_0")); - expectedRanges.add(makeShardedRange("20200103_1")); - expectedRanges.add(makeShardedRange("20200103_4")); - expectedRanges.add(makeShardedRange("20200103_5")); - expectedRanges.add(makeShardedRange("20200103_8")); - expectedRanges.add(makeShardedRange("20200103_9")); - expectedQueryStrings.add("A == 'tick_tock_day'"); - expectedQueryStrings.add("B == 'tick_tock_day'"); - expectedQueryStrings.add("A == 'tick_tock_day'"); - expectedQueryStrings.add("B == 'tick_tock_day'"); - expectedQueryStrings.add("A == 'tick_tock_day'"); - expectedQueryStrings.add("B == 'tick_tock_day'"); + for (int jj = 0; jj < 10; jj++) { + if (ii != 3) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("(A == 'tick_tock_day' || B == 'tick_tock_day')"); + } else { + if (jj % 4 == 0) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'tick_tock_day'"); + } else if (jj % 4 == 1) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("B == 'tick_tock_day'"); + } + } } } @@ -1384,9 +1409,10 @@ public void testIntersection_withNestedUnion_allDays() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) && (((_Delayed_ = true) && (B == 'all_day')) || ((_Delayed_ = true) && (C == 'all_day'))))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all_day' && (B == 'all_day' || C == 'all_day')"); + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -1400,12 +1426,13 @@ public void testIntersection_withNestedUnion_allDays_unequalStart() throws Excep List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 1) { - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day')) && (((_Delayed_ = true) && (B == 'all_day')))"); - } else { - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) && (((_Delayed_ = true) && (B == 'all_day')) || ((_Delayed_ = true) && (C == 'unequal_start_day'))))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 1) { + expectedQueryStrings.add("A == 'all_day' && B == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' && (B == 'all_day' || C == 'unequal_start_day')"); + } } } @@ -1420,12 +1447,13 @@ public void testIntersection_withNestedUnion_allDays_unequalStop() throws Except List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 5) { - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day')) && (((_Delayed_ = true) && (B == 'all_day')))"); - } else { - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) && (((_Delayed_ = true) && (B == 'all_day')) || ((_Delayed_ = true) && (C == 'unequal_stop_day'))))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 5) { + expectedQueryStrings.add("A == 'all_day' && B == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' && (B == 'all_day' || C == 'unequal_stop_day')"); + } } } @@ -1440,9 +1468,14 @@ public void testIntersection_withNestedUnion_allDays_unevenStart() throws Except List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) && (((_Delayed_ = true) && (B == 'all_day')) || ((_Delayed_ = true) && (C == 'uneven_start_day'))))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (jj == 0) { + expectedQueryStrings.add("A == 'all_day' && B == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' && (B == 'all_day' || C == 'uneven_start_day')"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -1456,9 +1489,14 @@ public void testIntersection_withNestedUnion_allDays_unevenStop() throws Excepti List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) && (((_Delayed_ = true) && (B == 'all_day')) || ((_Delayed_ = true) && (C == 'uneven_stop_day'))))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (jj == 9) { + expectedQueryStrings.add("A == 'all_day' && B == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' && (B == 'all_day' || C == 'uneven_stop_day')"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -1472,12 +1510,13 @@ public void testIntersection_withNestedUnion_allDays_missingShards() throws Exce List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 3) { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && ((_Delayed_ = true) && (B == 'all_day')))"); - } else { - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) && (((_Delayed_ = true) && (B == 'all_day')) || ((_Delayed_ = true) && (C == 'missing_shards_day'))))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 3) { + expectedQueryStrings.add("(A == 'all_day' && B == 'all_day')"); + } else { + expectedQueryStrings.add("A == 'all_day' && (B == 'all_day' || C == 'missing_shards_day')"); + } } } @@ -1492,8 +1531,14 @@ public void testIntersection_withNestedUnion_allDays_tickTockShards() throws Exc List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && (((_Delayed_ = true) && (B == 'all_day')) || C == 'tick_tock'))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 3 && jj % 4 != 2) { + expectedQueryStrings.add("A == 'all_day' && B == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' && (B == 'all_day' || C == 'tick_tock')"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -1509,7 +1554,7 @@ public void testIntersection_dayWithNestedUnionOfShards() throws Exception { for (int ii = 1; ii <= 5; ii++) { for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && (B == 'all' || C == 'all'))"); + expectedQueryStrings.add("(A == 'all_day') && (B == 'all' || C == 'all')"); } } @@ -1527,9 +1572,9 @@ public void testIntersection_dayWithNestedUnionOfShards_unequalStart() throws Ex for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); if (ii == 1) { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && B == 'all')"); + expectedQueryStrings.add("A == 'all_day' && B == 'all'"); } else { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && (B == 'all' || C == 'unequal_start'))"); + expectedQueryStrings.add("A == 'all_day' && (B == 'all' || C == 'unequal_start')"); } } } @@ -1548,9 +1593,9 @@ public void testIntersection_dayWithNestedUnionOfShards_unequalStop() throws Exc for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); if (ii == 5) { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && B == 'all')"); + expectedQueryStrings.add("A == 'all_day' && B == 'all'"); } else { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && (B == 'all' || C == 'unequal_stop'))"); + expectedQueryStrings.add("A == 'all_day' && (B == 'all' || C == 'unequal_stop')"); } } } @@ -1569,9 +1614,9 @@ public void testIntersection_dayWithNestedUnionOfShards_unevenStart() throws Exc for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); if (jj == 0) { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && B == 'all')"); + expectedQueryStrings.add("A == 'all_day' && B == 'all'"); } else { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && (B == 'all' || C == 'uneven_start'))"); + expectedQueryStrings.add("A == 'all_day' && (B == 'all' || C == 'uneven_start')"); } } } @@ -1590,9 +1635,9 @@ public void testIntersection_dayWithNestedUnionOfShards_unevenStop() throws Exce for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); if (jj == 9) { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && B == 'all')"); + expectedQueryStrings.add("A == 'all_day' && B == 'all'"); } else { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && (B == 'all' || C == 'uneven_stop'))"); + expectedQueryStrings.add("A == 'all_day' && (B == 'all' || C == 'uneven_stop')"); } } } @@ -1610,9 +1655,9 @@ public void testIntersection_dayWithNestedUnionOfShards_missingShards() throws E for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); if (ii == 3) { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && B == 'all')"); + expectedQueryStrings.add("A == 'all_day' && B == 'all'"); } else { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && (B == 'all' || C == 'missing_shards'))"); + expectedQueryStrings.add("A == 'all_day' && (B == 'all' || C == 'missing_shards')"); } } } @@ -1631,9 +1676,9 @@ public void testIntersection_dayWithNestedUnionOfShards_tickTockShards() throws for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); if (ii != 3 || jj == 2 || jj == 6) { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && (B == 'all' || C == 'tick_tock'))"); + expectedQueryStrings.add("A == 'all_day' && (B == 'all' || C == 'tick_tock')"); } else { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && B == 'all')"); + expectedQueryStrings.add("A == 'all_day' && B == 'all'"); } } } @@ -1651,7 +1696,7 @@ public void testIntersection_shardWithNestedUnionOfDays() throws Exception { for (int ii = 1; ii <= 5; ii++) { for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); - expectedQueryStrings.add("A == 'all' && (((_Delayed_ = true) && (B == 'all_day')) || ((_Delayed_ = true) && (C == 'all_day')))"); + expectedQueryStrings.add("A == 'all' && (B == 'all_day' || C == 'all_day')"); } } runTest(query, expectedRanges, expectedQueryStrings); @@ -1668,9 +1713,9 @@ public void testIntersection_shardWithNestedUnionOfDays_unequalStart() throws Ex for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); if (ii == 1) { - expectedQueryStrings.add("A == 'all' && (((_Delayed_ = true) && (B == 'all_day')))"); + expectedQueryStrings.add("A == 'all' && B == 'all_day'"); } else { - expectedQueryStrings.add("A == 'all' && (((_Delayed_ = true) && (B == 'all_day')) || ((_Delayed_ = true) && (C == 'unequal_start_day')))"); + expectedQueryStrings.add("A == 'all' && (B == 'all_day' || C == 'unequal_start_day')"); } } } @@ -1688,9 +1733,9 @@ public void testIntersection_shardWithNestedUnionOfDays_unequalStop() throws Exc for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); if (ii == 5) { - expectedQueryStrings.add("A == 'all' && (((_Delayed_ = true) && (B == 'all_day')))"); + expectedQueryStrings.add("A == 'all' && B == 'all_day'"); } else { - expectedQueryStrings.add("A == 'all' && (((_Delayed_ = true) && (B == 'all_day')) || ((_Delayed_ = true) && (C == 'unequal_stop_day')))"); + expectedQueryStrings.add("A == 'all' && (B == 'all_day' || C == 'unequal_stop_day')"); } } } @@ -1707,7 +1752,11 @@ public void testIntersection_shardWithNestedUnionOfDays_unevenStart() throws Exc for (int ii = 1; ii <= 5; ii++) { for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); - expectedQueryStrings.add("A == 'all' && (((_Delayed_ = true) && (B == 'all_day')) || ((_Delayed_ = true) && (C == 'uneven_start_day')))"); + if (jj == 0) { + expectedQueryStrings.add("A == 'all' && B == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all' && (B == 'all_day' || C == 'uneven_start_day')"); + } } } runTest(query, expectedRanges, expectedQueryStrings); @@ -1723,7 +1772,11 @@ public void testIntersection_shardWithNestedUnionOfDays_unevenStop() throws Exce for (int ii = 1; ii <= 5; ii++) { for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); - expectedQueryStrings.add("A == 'all' && (((_Delayed_ = true) && (B == 'all_day')) || ((_Delayed_ = true) && (C == 'uneven_stop_day')))"); + if (jj == 9) { + expectedQueryStrings.add("A == 'all' && B == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all' && (B == 'all_day' || C == 'uneven_stop_day')"); + } } } runTest(query, expectedRanges, expectedQueryStrings); @@ -1740,9 +1793,9 @@ public void testIntersection_shardWithNestedUnionOfDays_missingShards() throws E for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); if (ii == 3) { - expectedQueryStrings.add("A == 'all' && (((_Delayed_ = true) && (B == 'all_day')))"); + expectedQueryStrings.add("A == 'all' && B == 'all_day'"); } else { - expectedQueryStrings.add("A == 'all' && (((_Delayed_ = true) && (B == 'all_day')) || ((_Delayed_ = true) && (C == 'missing_shards_day')))"); + expectedQueryStrings.add("A == 'all' && (B == 'all_day' || C == 'missing_shards_day')"); } } } @@ -1760,11 +1813,10 @@ public void testIntersection_shardWithNestedUnionOfDays_tickTockShards() throws for (int ii = 1; ii <= 5; ii++) { for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); - if (ii == 3) { - // Not enough shards to force rolling the C-term to a day range, thus no delay. - expectedQueryStrings.add("A == 'all' && (((_Delayed_ = true) && (B == 'all_day')) || (C == 'tick_tock_day'))"); + if (ii == 3 && jj % 4 != 2) { + expectedQueryStrings.add("A == 'all' && B == 'all_day'"); } else { - expectedQueryStrings.add("A == 'all' && (((_Delayed_ = true) && (B == 'all_day')) || ((_Delayed_ = true) && (C == 'tick_tock_day')))"); + expectedQueryStrings.add("A == 'all' && (B == 'all_day' || C == 'tick_tock_day')"); } } } @@ -1926,9 +1978,10 @@ public void testUnion_withNestedIntersection_allDays() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) || (((_Delayed_ = true) && (B == 'all_day')) && ((_Delayed_ = true) && (C == 'all_day'))))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all_day' || (B == 'all_day' && C == 'all_day')"); + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -1942,12 +1995,13 @@ public void testUnion_withNestedIntersection_allDays_unequalStart() throws Excep List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 1) { - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day'))"); - } else { - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) || (((_Delayed_ = true) && (B == 'all_day')) && ((_Delayed_ = true) && (C == 'unequal_start_day'))))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 1) { + expectedQueryStrings.add("A == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' || (B == 'all_day' && C == 'unequal_start_day')"); + } } } @@ -1962,12 +2016,13 @@ public void testUnion_withNestedIntersection_allDays_unequalStop() throws Except List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 5) { - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day'))"); - } else { - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) || (((_Delayed_ = true) && (B == 'all_day')) && ((_Delayed_ = true) && (C == 'unequal_stop_day'))))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 5) { + expectedQueryStrings.add("A == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' || (B == 'all_day' && C == 'unequal_stop_day')"); + } } } @@ -1982,9 +2037,14 @@ public void testUnion_withNestedIntersection_allDays_unevenStart() throws Except List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) || (((_Delayed_ = true) && (B == 'all_day')) && ((_Delayed_ = true) && (C == 'uneven_start_day'))))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (jj == 0) { + expectedQueryStrings.add("A == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' || (B == 'all_day' && C == 'uneven_start_day')"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -1998,9 +2058,14 @@ public void testUnion_withNestedIntersection_allDays_unevenStop() throws Excepti List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) || (((_Delayed_ = true) && (B == 'all_day')) && ((_Delayed_ = true) && (C == 'uneven_stop_day'))))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (jj == 9) { + expectedQueryStrings.add("A == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' || (B == 'all_day' && C == 'uneven_stop_day')"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -2014,12 +2079,13 @@ public void testUnion_withNestedIntersection_allDays_missingShards() throws Exce List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 3) { - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day'))"); - } else { - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) || (((_Delayed_ = true) && (B == 'all_day')) && ((_Delayed_ = true) && (C == 'missing_shards_day'))))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 3) { + expectedQueryStrings.add("A == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' || (B == 'all_day' && C == 'missing_shards_day')"); + } } } @@ -2034,12 +2100,13 @@ public void testUnion_withNestedIntersection_allDays_tickTockShards() throws Exc List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 3) { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) || (((_Delayed_ = true) && (B == 'all_day')) && C == 'tick_tock_day'))"); - } else { - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) || (((_Delayed_ = true) && (B == 'all_day')) && ((_Delayed_ = true) && (C == 'tick_tock_day'))))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 3 && jj % 4 != 2) { + expectedQueryStrings.add("A == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' || (B == 'all_day' && C == 'tick_tock_day')"); + } } } @@ -2054,8 +2121,10 @@ public void testUnion_dayWithNestedIntersectionOfShards() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day')) || (B == 'all' && C == 'all')"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all_day' || (B == 'all' && C == 'all')"); + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -2069,11 +2138,13 @@ public void testUnion_dayWithNestedIntersectionOfShards_unequalStart() throws Ex List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 1) { - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day'))"); - } else { - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day')) || (B == 'all' && C == 'unequal_start')"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 1) { + expectedQueryStrings.add("A == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' || (B == 'all' && C == 'unequal_start')"); + } } } @@ -2088,11 +2159,13 @@ public void testUnion_dayWithNestedIntersectionOfShards_unequalStop() throws Exc List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 5) { - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day'))"); - } else { - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day')) || (B == 'all' && C == 'unequal_stop')"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 5) { + expectedQueryStrings.add("A == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' || (B == 'all' && C == 'unequal_stop')"); + } } } @@ -2107,11 +2180,13 @@ public void testUnion_dayWithNestedIntersectionOfShards_unevenStart() throws Exc List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 0) { - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day'))"); - } else { - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day')) || (B == 'all' && C == 'uneven_start')"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (jj == 0) { + expectedQueryStrings.add("A == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' || (B == 'all' && C == 'uneven_start')"); + } } } @@ -2126,11 +2201,13 @@ public void testUnion_dayWithNestedIntersectionOfShards_unevenStop() throws Exce List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 0) { - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day'))"); - } else { - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day')) || (B == 'all' && C == 'uneven_stop')"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (jj == 9) { + expectedQueryStrings.add("A == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' || (B == 'all' && C == 'uneven_stop')"); + } } } runTest(query, expectedRanges, expectedQueryStrings); @@ -2144,11 +2221,13 @@ public void testUnion_dayWithNestedIntersectionOfShards_missingShards() throws E List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 3) { - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day'))"); - } else { - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day')) || (B == 'all' && C == 'missing_shards')"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 3) { + expectedQueryStrings.add("A == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' || (B == 'all' && C == 'missing_shards')"); + } } } @@ -2163,8 +2242,14 @@ public void testUnion_dayWithNestedIntersectionOfShards_tickTockShards() throws List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("((_Delayed_ = true) && (A == 'all_day')) || (B == 'all' && C == 'tick_tock')"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 3 && jj % 4 != 2) { + expectedQueryStrings.add("A == 'all_day'"); + } else { + expectedQueryStrings.add("A == 'all_day' || (B == 'all' && C == 'tick_tock')"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -2178,8 +2263,10 @@ public void testUnion_shardWithNestedIntersectionOfDays() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("A == 'all' || (((_Delayed_ = true) && (B == 'all_day')) && ((_Delayed_ = true) && (C == 'all_day')))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all' || (B == 'all_day' && C == 'all_day')"); + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -2193,11 +2280,11 @@ public void testUnion_shardWithNestedIntersectionOfDays_unequalStart() throws Ex List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - if (ii != 1) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("A == 'all' || (((_Delayed_ = true) && (B == 'all_day')) && ((_Delayed_ = true) && (C == 'unequal_start_day')))"); - } else { - for (int jj = 0; jj < 10; jj++) { + for (int jj = 0; jj < 10; jj++) { + if (ii != 1) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all' || (B == 'all_day' && C == 'unequal_start_day')"); + } else { expectedRanges.add(makeTestRange("2020010" + ii + "_" + jj, "datatype1\0a.b.c")); expectedQueryStrings.add("A == 'all'"); } @@ -2215,11 +2302,12 @@ public void testUnion_shardWithNestedIntersectionOfDays_unequalStop() throws Exc List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - if (ii != 5) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("A == 'all' || (((_Delayed_ = true) && (B == 'all_day')) && ((_Delayed_ = true) && (C == 'unequal_stop_day')))"); - } else { - for (int jj = 0; jj < 10; jj++) { + for (int jj = 0; jj < 10; jj++) { + if (ii != 5) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all' || (B == 'all_day' && C == 'unequal_stop_day')"); + } else { + expectedRanges.add(makeTestRange("2020010" + ii + "_" + jj, "datatype1\0a.b.c")); expectedQueryStrings.add("A == 'all'"); } @@ -2237,8 +2325,15 @@ public void testUnion_shardWithNestedIntersectionOfDays_unevenStart() throws Exc List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("A == 'all' || (((_Delayed_ = true) && (B == 'all_day')) && ((_Delayed_ = true) && (C == 'uneven_start_day')))"); + for (int jj = 0; jj < 10; jj++) { + if (jj == 0) { + expectedRanges.add(makeTestRange("2020010" + ii + "_" + jj, "datatype1\u0000a.b.c")); + expectedQueryStrings.add("A == 'all'"); + } else { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all' || (B == 'all_day' && C == 'uneven_start_day')"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -2252,8 +2347,15 @@ public void testUnion_shardWithNestedIntersectionOfDays_unevenStop() throws Exce List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("A == 'all' || (((_Delayed_ = true) && (B == 'all_day')) && ((_Delayed_ = true) && (C == 'uneven_stop_day')))"); + for (int jj = 0; jj < 10; jj++) { + if (jj == 9) { + expectedRanges.add(makeTestRange("2020010" + ii + "_" + jj, "datatype1\u0000a.b.c")); + expectedQueryStrings.add("A == 'all'"); + } else { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all' || (B == 'all_day' && C == 'uneven_stop_day')"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -2267,14 +2369,14 @@ public void testUnion_shardWithNestedIntersectionOfDays_missingShards() throws E List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - if (ii == 3) { - for (int jj = 0; jj < 10; jj++) { + for (int jj = 0; jj < 10; jj++) { + if (ii == 3) { expectedRanges.add(makeTestRange("2020010" + ii + "_" + jj, "datatype1\0a.b.c")); expectedQueryStrings.add("A == 'all'"); + } else { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all' || (B == 'all_day' && C == 'missing_shards_day')"); } - } else { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("A == 'all' || (((_Delayed_ = true) && (B == 'all_day')) && ((_Delayed_ = true) && (C == 'missing_shards_day')))"); } } @@ -2289,14 +2391,14 @@ public void testUnion_shardWithNestedIntersectionOfDays_tickTockShards() throws List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - if (ii != 3) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("A == 'all' || (((_Delayed_ = true) && (B == 'all_day')) && ((_Delayed_ = true) && (C == 'tick_tock_day')))"); - } else { - for (int jj = 0; jj < 10; jj++) { + for (int jj = 0; jj < 10; jj++) { + if (ii != 3) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("A == 'all' || (B == 'all_day' && C == 'tick_tock_day')"); + } else { if (jj == 2 || jj == 6) { expectedRanges.add(makeShardedRange("20200103_" + jj)); - expectedQueryStrings.add("A == 'all' || (((_Delayed_ = true) && (B == 'all_day')) && (C == 'tick_tock_day'))"); + expectedQueryStrings.add("A == 'all' || (B == 'all_day' && (C == 'tick_tock_day'))"); } else { expectedRanges.add(makeTestRange("2020010" + ii + "_" + jj, "datatype1\0a.b.c")); expectedQueryStrings.add("A == 'all'"); @@ -2459,9 +2561,10 @@ public void testUnion_ofNestedIntersections_allDays() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) && ((_Delayed_ = true) && (B == 'all_day'))) || (((_Delayed_ = true) && (C == 'all_day')) && ((_Delayed_ = true) && (D == 'all_day')))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("(A == 'all_day' && B == 'all_day') || (C == 'all_day' && D == 'all_day')"); + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -2475,12 +2578,13 @@ public void testUnion_ofNestedIntersections_allDays_unequalStart() throws Except List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 1) { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && ((_Delayed_ = true) && (B == 'all_day')))"); - } else { - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) && ((_Delayed_ = true) && (B == 'all_day'))) || (((_Delayed_ = true) && (C == 'all_day')) && D == 'unequal_start')"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 1) { + expectedQueryStrings.add("A == 'all_day' && B == 'all_day'"); + } else { + expectedQueryStrings.add("(A == 'all_day' && B == 'all_day') || (C == 'all_day' && D == 'unequal_start')"); + } } } @@ -2495,12 +2599,13 @@ public void testUnion_ofNestedIntersections_allDays_unequalStop() throws Excepti List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 5) { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && ((_Delayed_ = true) && (B == 'all_day')))"); - } else { - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) && ((_Delayed_ = true) && (B == 'all_day'))) || (((_Delayed_ = true) && (C == 'all_day')) && D == 'unequal_stop')"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 5) { + expectedQueryStrings.add("A == 'all_day' && B == 'all_day'"); + } else { + expectedQueryStrings.add("(A == 'all_day' && B == 'all_day') || (C == 'all_day' && D == 'unequal_stop')"); + } } } @@ -2515,9 +2620,14 @@ public void testUnion_ofNestedIntersections_allDays_unevenStart() throws Excepti List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) && ((_Delayed_ = true) && (B == 'all_day'))) || (((_Delayed_ = true) && (C == 'all_day')) && D == 'uneven_start')"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (jj == 0) { + expectedQueryStrings.add("(A == 'all_day' && B == 'all_day')"); + } else { + expectedQueryStrings.add("(A == 'all_day' && B == 'all_day') || (C == 'all_day' && D == 'uneven_start')"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -2531,9 +2641,14 @@ public void testUnion_ofNestedIntersections_allDays_unevenStop() throws Exceptio List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) && ((_Delayed_ = true) && (B == 'all_day'))) || (((_Delayed_ = true) && (C == 'all_day')) && D == 'uneven_stop')"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (jj == 9) { + expectedQueryStrings.add("(A == 'all_day' && B == 'all_day')"); + } else { + expectedQueryStrings.add("(A == 'all_day' && B == 'all_day') || (C == 'all_day' && D == 'uneven_stop')"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -2547,12 +2662,13 @@ public void testUnion_ofNestedIntersections_allDays_missingShards() throws Excep List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 3) { - expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) && ((_Delayed_ = true) && (B == 'all_day')))"); - } else { - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) && ((_Delayed_ = true) && (B == 'all_day'))) || (((_Delayed_ = true) && (C == 'all_day')) && (((_Delayed_ = true) && (D == 'missing_shards_day'))))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 3) { + expectedQueryStrings.add("A == 'all_day' && B == 'all_day'"); + } else { + expectedQueryStrings.add("(A == 'all_day' && B == 'all_day') || (C == 'all_day' && D == 'missing_shards_day')"); + } } } @@ -2567,9 +2683,14 @@ public void testUnion_ofNestedIntersections_allDays_tickTockShards() throws Exce List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) && ((_Delayed_ = true) && (B == 'all_day'))) || (((_Delayed_ = true) && (C == 'all_day')) && D == 'tick_tock')"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 3 && jj % 4 != 3) { + expectedQueryStrings.add("(A == 'all_day' && B == 'all_day')"); + } else { + expectedQueryStrings.add("(A == 'all_day' && B == 'all_day') || (C == 'all_day' && D == 'tick_tock')"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -2585,8 +2706,7 @@ public void testUnion_ofNestedIntersections_distributedDays() throws Exception { for (int ii = 1; ii <= 5; ii++) { for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); - expectedQueryStrings - .add("(A == 'all' && ((_Delayed_ = true) && (B == 'all_day'))) || (C == 'all' && ((_Delayed_ = true) && (D == 'all_day')))"); + expectedQueryStrings.add("(A == 'all' && B == 'all_day') || (C == 'all' && D == 'all_day')"); } } @@ -2604,10 +2724,9 @@ public void testUnion_ofNestedIntersections_distributedDays_unequalStart() throw for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); if (ii == 1) { - expectedQueryStrings.add("(A == 'all' && ((_Delayed_ = true) && (B == 'all_day')))"); + expectedQueryStrings.add("(A == 'all' && B == 'all_day')"); } else { - expectedQueryStrings.add( - "(A == 'all' && ((_Delayed_ = true) && (B == 'all_day'))) || (C == 'all' && ((_Delayed_ = true) && (D == 'unequal_start_day')))"); + expectedQueryStrings.add("(A == 'all' && B == 'all_day') || (C == 'all' && D == 'unequal_start_day')"); } } } @@ -2626,10 +2745,9 @@ public void testUnion_ofNestedIntersections_distributedDays_unequalStop() throws for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); if (ii == 5) { - expectedQueryStrings.add("(A == 'all' && ((_Delayed_ = true) && (B == 'all_day')))"); + expectedQueryStrings.add("(A == 'all' && B == 'all_day')"); } else { - expectedQueryStrings.add( - "(A == 'all' && ((_Delayed_ = true) && (B == 'all_day'))) || (C == 'all' && ((_Delayed_ = true) && (D == 'unequal_stop_day')))"); + expectedQueryStrings.add("(A == 'all' && B == 'all_day') || (C == 'all' && D == 'unequal_stop_day')"); } } } @@ -2647,11 +2765,10 @@ public void testUnion_ofNestedIntersections_distributedDays_unevenStart() throws for (int ii = 1; ii <= 5; ii++) { for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); - if (ii == 0) { - expectedQueryStrings.add("(A == 'all' && ((_Delayed_ = true) && (B == 'all_day')))"); + if (jj == 0) { + expectedQueryStrings.add("(A == 'all' && B == 'all_day')"); } else { - expectedQueryStrings.add( - "(A == 'all' && ((_Delayed_ = true) && (B == 'all_day'))) || (C == 'all' && ((_Delayed_ = true) && (D == 'uneven_start_day')))"); + expectedQueryStrings.add("(A == 'all' && B == 'all_day') || (C == 'all' && D == 'uneven_start_day')"); } } } @@ -2669,11 +2786,10 @@ public void testUnion_ofNestedIntersections_distributedDays_unevenStop() throws for (int ii = 1; ii <= 5; ii++) { for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); - if (ii == 0) { - expectedQueryStrings.add("(A == 'all' && ((_Delayed_ = true) && (B == 'all_day')))"); + if (jj == 9) { + expectedQueryStrings.add("(A == 'all' && B == 'all_day')"); } else { - expectedQueryStrings.add( - "(A == 'all' && ((_Delayed_ = true) && (B == 'all_day'))) || (C == 'all' && ((_Delayed_ = true) && (D == 'uneven_stop_day')))"); + expectedQueryStrings.add("(A == 'all' && B == 'all_day') || (C == 'all' && D == 'uneven_stop_day')"); } } } @@ -2692,10 +2808,9 @@ public void testUnion_ofNestedIntersections_distributedDays_missingShards() thro for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); if (ii == 3) { - expectedQueryStrings.add("(A == 'all' && ((_Delayed_ = true) && (B == 'all_day')))"); + expectedQueryStrings.add("(A == 'all' && B == 'all_day')"); } else { - expectedQueryStrings.add( - "(A == 'all' && ((_Delayed_ = true) && (B == 'all_day'))) || (C == 'all' && ((_Delayed_ = true) && (D == 'missing_shards_day')))"); + expectedQueryStrings.add("(A == 'all' && B == 'all_day') || (C == 'all' && D == 'missing_shards_day')"); } } } @@ -2714,9 +2829,9 @@ public void testUnion_ofNestedIntersections_distributedDays_tickTockShards() thr for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); if (ii != 3 || jj == 3 || jj == 7) { - expectedQueryStrings.add("(A == 'all' && ((_Delayed_ = true) && (B == 'all_day'))) || (C == 'all' && D == 'tick_tock')"); + expectedQueryStrings.add("(A == 'all' && B == 'all_day') || (C == 'all' && D == 'tick_tock')"); } else { - expectedQueryStrings.add("(A == 'all' && ((_Delayed_ = true) && (B == 'all_day')))"); + expectedQueryStrings.add("(A == 'all' && B == 'all_day')"); } } } @@ -2875,9 +2990,10 @@ public void testIntersection_ofNestedUnions_allDays() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) || ((_Delayed_ = true) && (B == 'all_day'))) && (((_Delayed_ = true) && (C == 'all_day')) || ((_Delayed_ = true) && (D == 'all_day')))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("(A == 'all_day' || B == 'all_day') && (C == 'all_day' || D == 'all_day')"); + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -2891,13 +3007,13 @@ public void testIntersection_ofNestedUnions_allDays_unequalStart() throws Except List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 1) { - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) || ((_Delayed_ = true) && (B == 'all_day'))) && (((_Delayed_ = true) && (C == 'all_day')))"); - } else { - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) || ((_Delayed_ = true) && (B == 'all_day'))) && (((_Delayed_ = true) && (C == 'all_day')) || ((_Delayed_ = true) && (D == 'unequal_start_day')))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 1) { + expectedQueryStrings.add("(A == 'all_day' || B == 'all_day') && (C == 'all_day')"); + } else { + expectedQueryStrings.add("(A == 'all_day' || B == 'all_day') && (C == 'all_day' || D == 'unequal_start_day')"); + } } } @@ -2912,13 +3028,13 @@ public void testIntersection_ofNestedUnions_allDays_unequalStop() throws Excepti List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 5) { - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) || ((_Delayed_ = true) && (B == 'all_day'))) && (((_Delayed_ = true) && (C == 'all_day')))"); - } else { - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) || ((_Delayed_ = true) && (B == 'all_day'))) && (((_Delayed_ = true) && (C == 'all_day')) || ((_Delayed_ = true) && (D == 'unequal_stop_day')))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 5) { + expectedQueryStrings.add("(A == 'all_day' || B == 'all_day') && (C == 'all_day')"); + } else { + expectedQueryStrings.add("(A == 'all_day' || B == 'all_day') && (C == 'all_day' || D == 'unequal_stop_day')"); + } } } @@ -2933,9 +3049,14 @@ public void testIntersection_ofNestedUnions_allDays_unevenStart() throws Excepti List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) || ((_Delayed_ = true) && (B == 'all_day'))) && (((_Delayed_ = true) && (C == 'all_day')) || ((_Delayed_ = true) && (D == 'uneven_start_day')))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (jj == 0) { + expectedQueryStrings.add("(A == 'all_day' || B == 'all_day') && C == 'all_day'"); + } else { + expectedQueryStrings.add("(A == 'all_day' || B == 'all_day') && (C == 'all_day' || D == 'uneven_start_day')"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -2949,14 +3070,14 @@ public void testIntersection_ofNestedUnions_allDays_unevenStop() throws Exceptio List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - // if(ii == 5){ - // expectedQueryStrings.add("(((_Delayed_ = true) && (A == 'all_day')) || ((_Delayed_ = true) && (B == 'all_day'))) && (((_Delayed_ = true) && (C == - // 'all_day')))"); - // } else { - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) || ((_Delayed_ = true) && (B == 'all_day'))) && (((_Delayed_ = true) && (C == 'all_day')) || ((_Delayed_ = true) && (D == 'uneven_stop_day')))"); - // } + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (jj == 9) { + expectedQueryStrings.add("(A == 'all_day' || B == 'all_day') && C == 'all_day'"); + } else { + expectedQueryStrings.add("(A == 'all_day' || B == 'all_day') && (C == 'all_day' || D == 'uneven_stop_day')"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -2970,13 +3091,13 @@ public void testIntersection_ofNestedUnions_allDays_missingShards() throws Excep List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - if (ii == 3) { - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) || ((_Delayed_ = true) && (B == 'all_day'))) && (((_Delayed_ = true) && (C == 'all_day')))"); - } else { - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) || ((_Delayed_ = true) && (B == 'all_day'))) && (((_Delayed_ = true) && (C == 'all_day')) || ((_Delayed_ = true) && (D == 'missing_shards_day')))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 3) { + expectedQueryStrings.add("(A == 'all_day' || B == 'all_day') && (C == 'all_day')"); + } else { + expectedQueryStrings.add("(A == 'all_day' || B == 'all_day') && (C == 'all_day' || D == 'missing_shards_day')"); + } } } @@ -2991,9 +3112,14 @@ public void testIntersection_ofNestedUnions_allDays_tickTockShards() throws Exce List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add( - "(((_Delayed_ = true) && (A == 'all_day')) || ((_Delayed_ = true) && (B == 'all_day'))) && (((_Delayed_ = true) && (C == 'all_day')) || D == 'tick_tock')"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (ii == 3 && jj % 4 != 3) { + expectedQueryStrings.add("(A == 'all_day' || B == 'all_day') && C == 'all_day'"); + } else { + expectedQueryStrings.add("(A == 'all_day' || B == 'all_day') && (C == 'all_day' || D == 'tick_tock')"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -3007,8 +3133,10 @@ public void testIntersection_ofNestedUnions_distributedDays() throws Exception { List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add("((A == 'all') || ((_Delayed_ = true) && (B == 'all_day'))) && ((C == 'all') || (_Delayed_ = true) && (D == 'all_day'))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("(A == 'all' || B == 'all_day') && (C == 'all' || D == 'all_day')"); + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -3022,15 +3150,14 @@ public void testIntersection_ofNestedUnions_distributedDays_unequalStart() throw List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - if (ii == 1) { - for (int jj = 0; jj < 10; jj++) { + for (int jj = 0; jj < 10; jj++) { + if (ii == 1) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("((A == 'all') || B == 'all_day') && ((C == 'all'))"); + } else { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); - expectedQueryStrings.add("((A == 'all') || ((_Delayed_ = true) && (B == 'all_day'))) && ((C == 'all'))"); + expectedQueryStrings.add("(A == 'all' || B == 'all_day') && (C == 'all' || D == 'unequal_start_day')"); } - } else { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add( - "((A == 'all') || ((_Delayed_ = true) && (B == 'all_day'))) && ((C == 'all') || (_Delayed_ = true) && (D == 'unequal_start_day'))"); } } @@ -3045,15 +3172,14 @@ public void testIntersection_ofNestedUnions_distributedDays_unequalStop() throws List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - if (ii == 5) { - for (int jj = 0; jj < 10; jj++) { + for (int jj = 0; jj < 10; jj++) { + if (ii == 5) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + expectedQueryStrings.add("(A == 'all' || B == 'all_day') && (C == 'all')"); + } else { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); - expectedQueryStrings.add("((A == 'all') || ((_Delayed_ = true) && (B == 'all_day'))) && ((C == 'all'))"); + expectedQueryStrings.add("(A == 'all' || B == 'all_day') && (C == 'all' || D == 'unequal_stop_day')"); } - } else { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add( - "((A == 'all') || ((_Delayed_ = true) && (B == 'all_day'))) && ((C == 'all') || (_Delayed_ = true) && (D == 'unequal_stop_day'))"); } } @@ -3068,9 +3194,14 @@ public void testIntersection_ofNestedUnions_distributedDays_unevenStart() throws List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add( - "((A == 'all') || ((_Delayed_ = true) && (B == 'all_day'))) && ((C == 'all') || (_Delayed_ = true) && (D == 'uneven_start_day'))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (jj == 0) { + expectedQueryStrings.add("(A == 'all' || B == 'all_day') && C == 'all'"); + } else { + expectedQueryStrings.add("(A == 'all' || B == 'all_day') && (C == 'all' || D == 'uneven_start_day')"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -3084,9 +3215,14 @@ public void testIntersection_ofNestedUnions_distributedDays_unevenStop() throws List expectedRanges = new ArrayList<>(); List expectedQueryStrings = new ArrayList<>(); for (int ii = 1; ii <= 5; ii++) { - expectedRanges.add(makeDayRange("2020010" + ii)); - expectedQueryStrings.add( - "((A == 'all') || ((_Delayed_ = true) && (B == 'all_day'))) && ((C == 'all') || (_Delayed_ = true) && (D == 'uneven_stop_day'))"); + for (int jj = 0; jj < 10; jj++) { + expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); + if (jj == 9) { + expectedQueryStrings.add("(A == 'all' || B == 'all_day') && C == 'all'"); + } else { + expectedQueryStrings.add("(A == 'all' || B == 'all_day') && (C == 'all' || D == 'uneven_stop_day')"); + } + } } runTest(query, expectedRanges, expectedQueryStrings); @@ -3102,7 +3238,7 @@ public void testIntersection_ofNestedUnions_distributedDays_missingShards() thro for (int ii = 1; ii <= 5; ii++) { for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); - expectedQueryStrings.add("((A == 'all') || ((_Delayed_ = true) && (B == 'all_day'))) && ((C == 'all'))"); + expectedQueryStrings.add("((A == 'all') || B == 'all_day') && ((C == 'all'))"); } } @@ -3120,9 +3256,9 @@ public void testIntersection_ofNestedUnions_distributedDays_tickTockShards() thr for (int jj = 0; jj < 10; jj++) { expectedRanges.add(makeShardedRange("2020010" + ii + "_" + jj)); if (ii != 3 || jj == 3 || jj == 7) { - expectedQueryStrings.add("(A == 'all' || ((_Delayed_ = true) && (B == 'all_day'))) && (C == 'all' || D == 'tick_tock')"); + expectedQueryStrings.add("(A == 'all' || B == 'all_day') && (C == 'all' || D == 'tick_tock')"); } else { - expectedQueryStrings.add("(A == 'all' || ((_Delayed_ = true) && (B == 'all_day'))) && C == 'all'"); + expectedQueryStrings.add("(A == 'all' || B == 'all_day') && C == 'all'"); } } } @@ -3151,20 +3287,20 @@ public void testNestedPruningWithTopLevelIntersection() throws Exception { @Test public void testDelayedNestedPruningWithTopLevelIntersection() throws Exception { - String query = "F1 == '1' && (F3 == '3' || ((_Delayed_ = true) && (F4 == '4')))"; + String query = "F1 == '1' && (F3 == '3' || F4 == '4')"; List expectedRanges = new ArrayList<>(); expectedRanges.add(makeTestRange("20200101_10", "datatype1\0a.b.c")); expectedRanges.add(makeTestRange("20200101_11", "datatype1\0a.b.c")); // F3 skips shard _12, this forces the intersection into a shard range - expectedRanges.add(makeShardedRange("20200101_12")); + expectedRanges.add(makeTestRange("20200101_12", "datatype1\0a.b.c")); expectedRanges.add(makeTestRange("20200101_13", "datatype1\0a.b.c")); List expectedQueries = new ArrayList<>(); - expectedQueries.add("F1 == '1' && (F3 == '3' || ((_Delayed_ = true) && (F4 == '4')))"); - expectedQueries.add("F1 == '1' && (F3 == '3' || ((_Delayed_ = true) && (F4 == '4')))"); - expectedQueries.add("F1 == '1' && ((_Delayed_ = true) && (F4 == '4'))"); // F3 skips shard _12 - expectedQueries.add("F1 == '1' && (F3 == '3' || ((_Delayed_ = true) && (F4 == '4')))"); + expectedQueries.add("F1 == '1' && (F3 == '3' || F4 == '4')"); + expectedQueries.add("F1 == '1' && (F3 == '3' || F4 == '4')"); + expectedQueries.add("F1 == '1' && F4 == '4'"); // F3 skips shard _12 + expectedQueries.add("F1 == '1' && (F3 == '3' || F4 == '4')"); runTest(query, expectedRanges, expectedQueries); } @@ -3190,38 +3326,38 @@ public void testIntersectionOfNestedUnions() throws Exception { @Test public void testIntersectionOfNestedUnionsOnHasDelayedTerm() throws Exception { - String query = "(F1 == '1' || F2 == '2') && (F3 == '3' || ((_Delayed_ = true) && (F4 == '4')))"; + String query = "(F1 == '1' || F2 == '2') && (F3 == '3' || F4 == '4')"; List expectedRanges = new ArrayList<>(); expectedRanges.add(makeTestRange("20200101_10", "datatype1\0a.b.c")); expectedRanges.add(makeTestRange("20200101_11", "datatype1\0a.b.c")); - expectedRanges.add(makeShardedRange("20200101_12")); + expectedRanges.add(makeTestRange("20200101_12", "datatype1\0a.b.c")); expectedRanges.add(makeTestRange("20200101_13", "datatype1\0a.b.c")); List expectedQueries = new ArrayList<>(); - expectedQueries.add("(F1 == '1' || F2 == '2') && (F3 == '3' || ((_Delayed_ = true) && (F4 == '4')))"); - expectedQueries.add("F1 == '1' && (F3 == '3' || ((_Delayed_ = true) && (F4 == '4')))"); // F2 skips shard_11 - expectedQueries.add("(F1 == '1' || F2 == '2') && ((_Delayed_ = true) && (F4 == '4'))"); // F3 skips shard _12 - expectedQueries.add("(F1 == '1' || F2 == '2') && (F3 == '3' || ((_Delayed_ = true) && (F4 == '4')))"); + expectedQueries.add("(F1 == '1' || F2 == '2') && (F3 == '3' || F4 == '4')"); + expectedQueries.add("F1 == '1' && (F3 == '3' || F4 == '4')"); // F2 skips shard_11 + expectedQueries.add("(F1 == '1' || F2 == '2') && F4 == '4'"); // F3 skips shard _12 + expectedQueries.add("(F1 == '1' || F2 == '2') && (F3 == '3' || F4 == '4')"); runTest(query, expectedRanges, expectedQueries); } @Test public void testIntersectionOfNestedUnionsOnHasDelayedTerm_flipped() throws Exception { - String query = "(F3 == '3' || ((_Delayed_ = true) && (F4 == '4'))) && (F1 == '1' || F2 == '2')"; + String query = "(F3 == '3' || F4 == '4') && (F1 == '1' || F2 == '2')"; List expectedRanges = new ArrayList<>(); expectedRanges.add(makeTestRange("20200101_10", "datatype1\0a.b.c")); expectedRanges.add(makeTestRange("20200101_11", "datatype1\0a.b.c")); - expectedRanges.add(makeShardedRange("20200101_12")); + expectedRanges.add(makeTestRange("20200101_12", "datatype1\0a.b.c")); expectedRanges.add(makeTestRange("20200101_13", "datatype1\0a.b.c")); List expectedQueries = new ArrayList<>(); - expectedQueries.add("(F1 == '1' || F2 == '2') && (F3 == '3' || ((_Delayed_ = true) && (F4 == '4')))"); - expectedQueries.add("F1 == '1' && (F3 == '3' || ((_Delayed_ = true) && (F4 == '4')))"); // F2 skips shard_11 - expectedQueries.add("(F1 == '1' || F2 == '2') && ((_Delayed_ = true) && (F4 == '4'))"); // F3 skips shard _12 - expectedQueries.add("(F1 == '1' || F2 == '2') && (F3 == '3' || ((_Delayed_ = true) && (F4 == '4')))"); + expectedQueries.add("(F1 == '1' || F2 == '2') && (F3 == '3' || F4 == '4')"); + expectedQueries.add("F1 == '1' && (F3 == '3' || F4 == '4')"); // F2 skips shard_11 + expectedQueries.add("(F1 == '1' || F2 == '2') && F4 == '4'"); // F3 skips shard _12 + expectedQueries.add("(F1 == '1' || F2 == '2') && (F3 == '3' || F4 == '4')"); runTest(query, expectedRanges, expectedQueries); } @@ -3247,19 +3383,19 @@ public void testOrAndOr() throws Exception { @Test public void testOrAndOrWithDeeplyNestedDelayedTerm() throws Exception { - String query = "F2 == '2' || (F1 == '1' && (F3 == '3' || ((_Delayed_ = true) && (F4 == '4'))))"; + String query = "F2 == '2' || (F1 == '1' && (F3 == '3' || F4 == '4'))"; List expectedRanges = new ArrayList<>(); expectedRanges.add(makeTestRange("20200101_10", "datatype1\0a.b.c")); expectedRanges.add(makeTestRange("20200101_11", "datatype1\0a.b.c")); - expectedRanges.add(makeShardedRange("20200101_12")); + expectedRanges.add(makeTestRange("20200101_12", "datatype1\0a.b.c")); expectedRanges.add(makeTestRange("20200101_13", "datatype1\0a.b.c")); List expectedQueries = new ArrayList<>(); - expectedQueries.add("F2 == '2' || (F1 == '1' && (F3 == '3' || ((_Delayed_ = true) && (F4 == '4'))))"); - expectedQueries.add("(F1 == '1' && (F3 == '3' || ((_Delayed_ = true) && (F4 == '4'))))"); // F2 skips shard _11 - expectedQueries.add("F2 == '2' || (F1 == '1' && ((_Delayed_ = true) && (F4 == '4')))"); // F3 skips shard _12 - expectedQueries.add("F2 == '2' || (F1 == '1' && (F3 == '3' || ((_Delayed_ = true) && (F4 == '4'))))"); + expectedQueries.add("F2 == '2' || (F1 == '1' && (F3 == '3' || F4 == '4'))"); + expectedQueries.add("(F1 == '1' && (F3 == '3' || F4 == '4'))"); // F2 skips shard _11 + expectedQueries.add("F2 == '2' || (F1 == '1' && F4 == '4')"); // F3 skips shard _12 + expectedQueries.add("F2 == '2' || (F1 == '1' && (F3 == '3' || F4 == '4'))"); runTest(query, expectedRanges, expectedQueries); } @@ -3289,7 +3425,6 @@ private void runTest(String query, List expectedRanges, List expe config.setQueryFieldsDatatypes(dataTypes); config.setIndexedFields(dataTypes); - config.setShardsPerDayThreshold(2); MockMetadataHelper helper = new MockMetadataHelper(); helper.setIndexedFields(dataTypes.keySet()); diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/RangeStreamScannerTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/RangeStreamScannerTest.java index a935f250460..bcd023534eb 100644 --- a/warehouse/query-core/src/test/java/datawave/query/tables/RangeStreamScannerTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/tables/RangeStreamScannerTest.java @@ -203,8 +203,7 @@ private RangeStreamScanner buildRangeStreamScanner(String fieldName, String fiel int priority = 50; // Iterator priority - RangeStreamScanner scanSession = scannerFactory.newRangeScanner(config.getIndexTableName(), config.getAuthorizations(), config.getQuery(), - config.getShardsPerDayThreshold()); + RangeStreamScanner scanSession = scannerFactory.newRangeScanner(config.getIndexTableName(), config.getAuthorizations(), config.getQuery()); scanSession.setMaxResults(config.getMaxIndexBatchSize()); scanSession.setExecutor(streamExecutor); @@ -344,7 +343,6 @@ public void testExceedShardsPerDayThresholdAndDocumentsPerShardThreshold() throw // Construct a ScannerStream from RangeStreamScanner, iterator, entry parser. RangeStreamScanner rangeStreamScanner = buildRangeStreamScanner(fieldName, fieldValue); EntryParser entryParser = new EntryParser(eqNode, fieldName, fieldValue, config.getIndexedFields()); - // Iterator> iterator = Iterators.transform(rangeStreamScanner, entryParser); ScannerStream scannerStream = ScannerStream.initialized(rangeStreamScanner, entryParser, eqNode); // Assert the iterator correctly iterates over the iterables without irritating the unit test. @@ -358,8 +356,8 @@ public void testExceedShardsPerDayThresholdAndDocumentsPerShardThreshold() throw documentCount += entry.second().count(); } // A single range with a count of -1 means the shard ranges were collapsed into a day range. - assertEquals(1, shardCount); - assertEquals(-1, documentCount); + assertEquals(15, shardCount); + assertEquals(375, documentCount); assertFalse(scannerStream.hasNext()); } @@ -370,8 +368,7 @@ public void testExceedShardsPerDayThresholdAndDocumentsPerShardThreshold() throw public void testGetDay() throws Exception { // Build RangeStreamScanner ScannerFactory scanners = new ScannerFactory(config); - RangeStreamScanner rangeStreamScanner = scanners.newRangeScanner(config.getIndexTableName(), config.getAuthorizations(), config.getQuery(), - config.getShardsPerDayThreshold()); + RangeStreamScanner rangeStreamScanner = scanners.newRangeScanner(config.getIndexTableName(), config.getAuthorizations(), config.getQuery()); Key key = new Key("row".getBytes(), "cf".getBytes(), "20190314".getBytes()); String expectedDay = "20190314"; diff --git a/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml b/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml index ffa8088b8e0..33c37ac318d 100644 --- a/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml +++ b/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml @@ -91,12 +91,6 @@ - - - - diff --git a/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml b/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml index fe08b21c663..6e810c1985a 100644 --- a/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml +++ b/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml @@ -209,12 +209,6 @@ - - - - diff --git a/web-services/deploy/application/src/main/docker/Dockerfile b/web-services/deploy/application/src/main/docker/Dockerfile index 750db9d81c8..506a6ff45c4 100644 --- a/web-services/deploy/application/src/main/docker/Dockerfile +++ b/web-services/deploy/application/src/main/docker/Dockerfile @@ -1,5 +1,7 @@ FROM azul/zulu-openjdk-centos:11.0.18-11.62.17 +RUN sed -i 's|mirrorlist|#mirrorlist|g;s|#baseurl.*|baseurl=http://archive.kernel.org/centos-vault/centos/$releasever/updates/$basearch/|g' /etc/yum.repos.d/CentOS-Base.repo && \ + sed -i 's/enabled=0/enabled=1/g' /etc/yum.repos.d/CentOS-Vault.repo RUN yum -y install deltarpm centos-release-sc && \ chmod -R 777 /usr/lib/jvm/zulu11/bin/* && \ diff --git a/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml b/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml index 5adce1647d0..ad53f615676 100644 --- a/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml +++ b/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml @@ -249,12 +249,6 @@ - - - - @@ -621,12 +615,6 @@ - - - -