Skip to content

Commit

Permalink
Merge branch 'integration' into feature/separateWebserviceHdfsConfig
Browse files Browse the repository at this point in the history
  • Loading branch information
ivakegg committed Jun 28, 2024
2 parents 35096c7 + 6e516bf commit edabb0a
Show file tree
Hide file tree
Showing 20 changed files with 588 additions and 781 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -95,35 +95,6 @@ public QueryData(QueryData other) {
this.finished = other.finished;
}

@Deprecated(since = "6.5.0", forRemoval = true)
public QueryData(String tableName, String query, Collection<Range> ranges, List<IteratorSetting> settings) {
setTableName(tableName);
setQuery(query);
setRanges(ranges);
setSettings(settings);
}

/**
* Weak copy constructor that updates the ranges
*
* @param other
* another QueryData
* @param ranges
* a collection of updated ranges
* @deprecated
*/
@Deprecated(since = "6.5.0", forRemoval = true)
public QueryData(QueryData other, Collection<Range> ranges) {
this(other);
setRanges(ranges);
}

@Deprecated(since = "6.5.0", forRemoval = true)
public QueryData(String tableName, String queryString, List<Range> ranges, List<IteratorSetting> settings, Collection<String> columnFamilies) {
this(tableName, queryString, ranges, settings);
this.columnFamilies.addAll(columnFamilies);
}

// builder style methods

public QueryData withTableName(String tableName) {
Expand Down
12 changes: 6 additions & 6 deletions docker/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ services:
test: ["CMD-SHELL", "! accumuloStatus | grep DW-WARN > /dev/null"]

consul:
image: consul:1.15.4
image: docker.io/hashicorp/consul:1.15.4
hostname: localhost
environment:
- 'CONSUL_LOCAL_CONFIG={"log_level": "trace", "datacenter": "demo_dc", "disable_update_check": true, "enable_agent_tls_for_checks": true, "addresses": {"https": "0.0.0.0"}, "ports": {"https": 8501, "grpc_tls": 8503}, "tls": {"defaults": {"key_file": "/etc/pki/testServer.key", "cert_file": "/etc/pki/testServer.crt", "ca_file": "/etc/pki/testCA.pem", "verify_outgoing": true}, "internal_rpc": {"verify_server_hostname": false}}}'
Expand All @@ -65,7 +65,7 @@ services:
- demo

rabbitmq:
image: rabbitmq:3.12.4-alpine
image: docker.io/rabbitmq:3.12.4
volumes:
- ${RABBITMQ_CONFIG_DIR:-./rabbitmq-config}:/etc/rabbitmq
- ./logs:/logs
Expand All @@ -82,7 +82,7 @@ services:

# When auto.create.topics.enable is true, this causes deleted topics to be recreated at random. So, leave it disabled.
kafka:
image: bitnami/kafka:3.2.3
image: docker.io/bitnami/kafka:3.2
ports:
- "9094:9094"
networks:
Expand All @@ -104,7 +104,7 @@ services:
profiles:
- management
- full
image: obsidiandynamics/kafdrop
image: docker.io/obsidiandynamics/kafdrop
ports:
- "8999:9000"
networks:
Expand Down Expand Up @@ -339,7 +339,7 @@ services:
mysql:
profiles:
- cachedresults
image: mysql:8.0.32
image: docker.io/mysql:8.0.32
environment:
- MYSQL_RANDOM_ROOT_PASSWORD=true
- MYSQL_DATABASE=cachedresults
Expand Down Expand Up @@ -584,7 +584,7 @@ services:
profiles:
- management
- full
image: hazelcast/management-center:5.1.2
image: docker.io/hazelcast/management-center:5.1.2
environment:
- |-
JAVA_OPTS=
Expand Down
2 changes: 0 additions & 2 deletions properties/default.properties
Original file line number Diff line number Diff line change
Expand Up @@ -425,8 +425,6 @@ hierarchy.field.options=

# BaseEventQuery (beq) thresholds
beq.baseIteratorPriority=100
beq.eventPerDayThreshold=40000
beq.shardsPerDayThreshold=20
# max number of terms BEFORE all expansions (calculated based on how much the initial parser can handle before hitting a stack overflow: between 3500 and 3750)
beq.initialMaxTermThreshold=2000
# max number of terms AFTER all expansions (calculated based on how much the initial parser can handle before hitting a stack overflow: between 3500 and 3750)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -241,10 +241,12 @@ public String getQueryIteratorClass() {
return queryIteratorClass;
}

@Deprecated(since = "7.1.0", forRemoval = true)
public int getMaxShardsPerDayThreshold() {
return maxShardsPerDayThreshold;
}

@Deprecated(since = "7.1.0", forRemoval = true)
public void setMaxShardsPerDayThreshold(int maxShardsPerDayThreshold) {
this.maxShardsPerDayThreshold = maxShardsPerDayThreshold;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1298,18 +1298,22 @@ public void setUnevaluatedFields(Collection<String> unevaluatedFields) {
}
}

@Deprecated(since = "7.1.0", forRemoval = true)
public int getEventPerDayThreshold() {
return eventPerDayThreshold;
}

@Deprecated(since = "7.1.0", forRemoval = true)
public void setEventPerDayThreshold(int eventPerDayThreshold) {
this.eventPerDayThreshold = eventPerDayThreshold;
}

@Deprecated(since = "7.1.0", forRemoval = true)
public int getShardsPerDayThreshold() {
return shardsPerDayThreshold;
}

@Deprecated(since = "7.1.0", forRemoval = true)
public void setShardsPerDayThreshold(int shardsPerDayThreshold) {
this.shardsPerDayThreshold = shardsPerDayThreshold;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -570,8 +570,7 @@ public ScannerStream visit(ASTEQNode node, Object data) {

if (limitScanners) {
// Setup the CreateUidsIterator
scannerSession = scanners.newRangeScanner(config.getIndexTableName(), config.getAuthorizations(), config.getQuery(),
config.getShardsPerDayThreshold());
scannerSession = scanners.newRangeScanner(config.getIndexTableName(), config.getAuthorizations(), config.getQuery());

uidSetting = new IteratorSetting(stackStart++, createUidsIteratorClass);
uidSetting.addOption(CreateUidsIterator.COLLAPSE_UIDS, Boolean.toString(collapseUids));
Expand All @@ -581,8 +580,7 @@ public ScannerStream visit(ASTEQNode node, Object data) {

} else {
// Setup so this is a pass-through
scannerSession = scanners.newRangeScanner(config.getIndexTableName(), config.getAuthorizations(), config.getQuery(),
config.getShardsPerDayThreshold());
scannerSession = scanners.newRangeScanner(config.getIndexTableName(), config.getAuthorizations(), config.getQuery());

uidSetting = new IteratorSetting(stackStart++, createUidsIteratorClass);
uidSetting.addOption(CreateUidsIterator.COLLAPSE_UIDS, Boolean.toString(false));
Expand Down
Original file line number Diff line number Diff line change
@@ -1,17 +1,11 @@
package datawave.query.planner;

import static datawave.query.iterator.QueryOptions.QUERY;
import static datawave.query.iterator.QueryOptions.RANGES;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.data.Range;
import org.apache.commons.jexl3.parser.ASTJexlScript;
import org.apache.commons.jexl3.parser.JexlNode;
import org.apache.commons.jexl3.parser.ParseException;
import org.apache.commons.lang.builder.EqualsBuilder;
Expand All @@ -20,10 +14,8 @@

import com.google.common.base.Preconditions;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;

import datawave.core.common.logging.ThreadConfigurableLogger;
import datawave.core.query.configuration.QueryData;
import datawave.query.jexl.JexlASTHelper;
import datawave.query.jexl.visitors.JexlStringBuildingVisitor;
import datawave.query.util.count.CountMap;
Expand Down Expand Up @@ -118,105 +110,6 @@ public QueryPlan(QueryPlan other) {
this.rebuildHashCode = other.rebuildHashCode;
}

/**
* Partial constructor, missing IteratorSetting
*
* @param tableName
* the table name
* @param queryTreeString
* the query string
* @param queryTree
* the query tree
* @param ranges
* the ranges
* @deprecated
*/
@Deprecated(since = "6.9.0", forRemoval = true)
public QueryPlan(String tableName, String queryTreeString, JexlNode queryTree, Iterable<Range> ranges) {
this(tableName, queryTreeString, queryTree, ranges, null);
}

@Deprecated(since = "6.9.0", forRemoval = true)
public QueryPlan(String tableName, String queryTreeString, JexlNode queryTree, Iterable<Range> ranges, List<IteratorSetting> settings) {
Preconditions.checkNotNull(queryTree);
this.tableName = tableName;
this.queryTree = queryTree;
this.queryTreeString = queryTreeString;
this.ranges = Lists.newArrayList(ranges);
if (null != settings) {
this.settings = settings;
}
resetHashCode();
}

@Deprecated(since = "6.9.0", forRemoval = true)
public QueryPlan(String tableName, JexlNode queryTree, Iterable<Range> ranges, Collection<String> columnFamilies) {
Preconditions.checkNotNull(queryTree);
this.tableName = tableName;
this.queryTree = queryTree;
this.ranges = Lists.newArrayList(ranges);
this.columnFamilies = Lists.newArrayList(columnFamilies);
resetHashCode();
}

@Deprecated(since = "6.9.0", forRemoval = true)
public QueryPlan(String tableName, JexlNode queryTree, Range range) {
Preconditions.checkNotNull(queryTree);
this.tableName = tableName;
this.queryTree = queryTree;
this.ranges = Lists.newArrayList(range);
resetHashCode();
}

@Deprecated(since = "6.9.0", forRemoval = true)
public QueryPlan(QueryData currentQueryData) throws ParseException {
this.tableName = currentQueryData.getTableName();
this.queryTreeString = currentQueryData.getQuery();
this.ranges = Lists.newArrayList(currentQueryData.getRanges());
this.settings.addAll(currentQueryData.getSettings());
this.columnFamilies.addAll(currentQueryData.getColumnFamilies());
resetHashCode();
}

/**
* @param tableName
* @param queryTree
* @param rangeIter
* @param settings
* @param columnFamilies
*/
@Deprecated(since = "6.9.0", forRemoval = true)
public QueryPlan(String tableName, JexlNode queryTree, Iterable<Range> rangeIter, List<IteratorSetting> settings, Collection<String> columnFamilies) {
this.tableName = tableName;
this.queryTree = queryTree;
this.ranges = Lists.newArrayList(rangeIter);
for (IteratorSetting setting : settings) {
IteratorSetting newSetting = new IteratorSetting(setting.getPriority(), setting.getName(), setting.getIteratorClass());
newSetting.addOptions(setting.getOptions());
if (newSetting.getOptions().containsKey(QUERY)) {
newSetting.addOption(QUERY, JexlStringBuildingVisitor.buildQuery(queryTree));
newSetting.addOption(RANGES, this.ranges.stream().map(Range::toString).collect(Collectors.joining(",", "[", "]")));
}
this.settings.add(newSetting);

}
if (null != columnFamilies) {
this.columnFamilies.addAll(columnFamilies);
}
resetHashCode();
}

/**
* @param tableName
* @param queryTree
* @param rangeIter
* @param settings
*/
@Deprecated(since = "6.9.0", forRemoval = true)
public QueryPlan(String tableName, JexlNode queryTree, Iterable<Range> rangeIter, List<IteratorSetting> settings) {
this(tableName, queryTree, rangeIter, settings, null);
}

public QueryPlan(JexlNode queryTree, Collection<Range> ranges) {
this.queryTree = queryTree;
this.ranges = ranges;
Expand Down Expand Up @@ -340,20 +233,6 @@ public void setQueryTreeString(String queryString) {
resetHashCode();
}

@Deprecated(since = "6.9.0", forRemoval = true)
public void setQuery(String queryString, JexlNode queryTree) {
this.queryTree = queryTree;
this.queryTreeString = queryString;
resetHashCode();
}

@Deprecated(since = "6.9.0", forRemoval = true)
public void setQuery(String queryString, ASTJexlScript queryTree) {
this.queryTree = queryTree;
this.queryTreeString = queryString;
resetHashCode();
}

public String getQueryString() {
if (null == queryTreeString) {
Preconditions.checkNotNull(queryTree);
Expand Down
Loading

0 comments on commit edabb0a

Please sign in to comment.