Skip to content

Commit

Permalink
introduce new enum instead of HiveOperation
Browse files Browse the repository at this point in the history
  • Loading branch information
@kasakrisz
kasakrisz committed Aug 23, 2024
1 parent 9ac7895 commit 346f7ab
Show file tree
Hide file tree
Showing 4 changed files with 33 additions and 35 deletions.
7 changes: 4 additions & 3 deletions common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3869,9 +3869,10 @@ public static enum ConfVars {
"Deprecated: use hive.lineage.statement.filter instead."),
HIVE_LINEAGE_STATEMENT_FILTER("hive.lineage.statement.filter", "ALL",
"Whether Hive provides lineage information to hooks for the specified statements only, " +
"the value is a comma-separated list of HiveOperation (ex.: CREATEVIEW,CREATE_MATERIALIZED_VIEW," +
"CREATETABLE,CREATETABLE_AS_SELECT). There are two special values: ALL means lineage information is always " +
"provided, NONE means never."),
"the value is a comma-separated list (ex.: CREATE_MATERIALIZED_VIEW," +
"CREATE_TABLE,CREATE_TABLE_AS_SELECT). Possible values are: CREATE_TABLE, CREATE_TABLE_AS_SELECT, " +
"CREATE_VIEW, CREATE_MATERIALIZED_VIEW, LOAD, QUERY, ALL, NONE." +
" ALL means lineage information is always provided, NONE and empty string means never."),

HIVE_SSL_PROTOCOL_BLACKLIST("hive.ssl.protocol.blacklist", "SSLv2,SSLv3",
"SSL Versions to disable for all Hive Servers"),
Expand Down
52 changes: 24 additions & 28 deletions ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,10 @@
package org.apache.hadoop.hive.ql.optimizer.lineage;

import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Predicate;

import org.apache.commons.lang3.EnumUtils;
Expand Down Expand Up @@ -55,7 +52,6 @@
import org.apache.hadoop.hive.ql.optimizer.lineage.LineageCtx.Index;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -67,59 +63,59 @@
public class Generator extends Transform {

private static final Logger LOG = LoggerFactory.getLogger(Generator.class);
private static final String ALL = "ALL";
private static final String NONE = "NONE";
private static final Map<HiveOperation, Function<ParseContext, Boolean>> filterMap;

private final Predicate<ParseContext> statementFilter;

static {
Map<HiveOperation, Function<ParseContext, Boolean>> map = new EnumMap<>(HiveOperation.class);
map.put(HiveOperation.CREATETABLE, parseContext -> parseContext.getCreateTable() != null);
map.put(HiveOperation.CREATETABLE_AS_SELECT, parseContext -> parseContext.getQueryProperties().isCTAS());
map.put(HiveOperation.CREATEVIEW, parseContext -> parseContext.getQueryProperties().isView());
map.put(HiveOperation.CREATE_MATERIALIZED_VIEW,
parseContext -> parseContext.getQueryProperties().isMaterializedView());
map.put(HiveOperation.LOAD,
parseContext -> !(parseContext.getLoadTableWork() == null || parseContext.getLoadTableWork().isEmpty()));
map.put(HiveOperation.QUERY, parseContext -> parseContext.getQueryProperties().isQuery());
filterMap = Collections.unmodifiableMap(map);
enum LineageInfoFilter {
CREATE_TABLE(parseContext -> parseContext.getCreateTable() != null),
CREATE_TABLE_AS_SELECT(parseContext -> parseContext.getQueryProperties().isCTAS()),
CREATE_VIEW(parseContext -> parseContext.getQueryProperties().isView()),
CREATE_MATERIALIZED_VIEW(parseContext -> parseContext.getQueryProperties().isMaterializedView()),
LOAD(parseContext -> !(parseContext.getLoadTableWork() == null || parseContext.getLoadTableWork().isEmpty())),
QUERY(parseContext -> parseContext.getQueryProperties().isQuery()),
ALL(parseContext -> true),
NONE(parseContext -> false);

final Predicate<ParseContext> predicate;

LineageInfoFilter(Predicate<ParseContext> predicate) {
this.predicate = predicate;
}
}

public static Generator fromConf(HiveConf conf) {
return new Generator(createFilterPredicateFromConf(conf));
}

static Predicate<ParseContext> createFilterPredicateFromConf(Configuration conf) {
Set<HiveOperation> operations = new HashSet<>();
Set<LineageInfoFilter> operations = new HashSet<>();
boolean noneSpecified = false;
for (String valueText : conf.getStringCollection(HiveConf.ConfVars.HIVE_LINEAGE_STATEMENT_FILTER.varname)) {
if (ALL.equalsIgnoreCase(valueText)) {
return parseContext -> true;
LineageInfoFilter enumValue = EnumUtils.getEnumIgnoreCase(LineageInfoFilter.class, valueText);
if (enumValue == null) {
throw new EnumConstantNotPresentException(LineageInfoFilter.class, valueText);
}
if (NONE.equalsIgnoreCase(valueText)) {

if (LineageInfoFilter.NONE == enumValue) {
noneSpecified = true;
continue;
}

HiveOperation enumValue = EnumUtils.getEnumIgnoreCase(HiveOperation.class, valueText);
if (enumValue == null) {
throw new EnumConstantNotPresentException(HiveOperation.class, valueText);
}
operations.add(enumValue);
}

if (noneSpecified) {
if (!operations.isEmpty()) {
throw new IllegalArgumentException("No other value can be specified when None is present!");
throw new IllegalArgumentException(
"No other value can be specified when " + LineageInfoFilter.NONE.name() + " is present!");
}
else {
return parseContext -> false;
}
}

return parseContext ->
operations.stream().anyMatch(hiveOperation -> filterMap.get(hiveOperation).apply(parseContext));
operations.stream().anyMatch(lineageInfoFilter -> lineageInfoFilter.predicate.test(parseContext));
}

public Generator(Predicate<ParseContext> statementFilter) {
Expand Down
7 changes: 4 additions & 3 deletions ql/src/test/org/apache/hadoop/hive/ql/optimizer/lineage/TestGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public class TestGenerator {
public void testCreateFilterPredicateFromConf() {
HiveConf conf = new HiveConf();
conf.set(HiveConf.ConfVars.HIVE_LINEAGE_STATEMENT_FILTER.varname,
"CREATEtABLE_AS_sELECT," + HiveOperation.QUERY.name());
"CREATE_tABLE_AS_sELECT," + HiveOperation.QUERY.name());

Predicate<ParseContext> predicate = Generator.createFilterPredicateFromConf(conf);

Expand Down Expand Up @@ -87,7 +87,8 @@ public void testCreateFilterPredicateFromConfThrowsExceptionWhenInputStringIsInv
() -> Generator.createFilterPredicateFromConf(conf)
);

assertThat(exception.getMessage(), is("org.apache.hadoop.hive.ql.plan.HiveOperation.Invalid"));
assertThat(exception.getMessage(), is(
"org.apache.hadoop.hive.ql.optimizer.lineage.Generator$LineageInfoFilter.Invalid"));
}

@Test
Expand All @@ -100,6 +101,6 @@ public void testCreateFilterPredicateFromConfThrowsExceptionWhenNoneAndAnyOtherC
() -> Generator.createFilterPredicateFromConf(conf)
);

assertThat(exception.getMessage(), is("No other value can be specified when None is present!"));
assertThat(exception.getMessage(), is("No other value can be specified when NONE is present!"));
}
}
2 changes: 1 addition & 1 deletion ql/src/test/queries/clientpositive/lineage6.q
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,6 @@ select table_1_qegkz.id, concat_ws(' ' , table_1_qegkz.first_name, table_2_gkvuw

create view view_fcuyp as (select table_1_qegkz.id, concat_ws(' ' , table_1_qegkz.first_name, table_2_gkvuw.last_name) full_name from table_1_qegkz, table_2_gkvuw where table_1_qegkz.id = table_2_gkvuw.id);

set hive.lineage.statement.filter=Query,CREATEVIEW;
set hive.lineage.statement.filter=Query,CREATE_VIEW;

create view view_fcuyp2 as (select table_1_qegkz.id, concat_ws(' ' , table_1_qegkz.first_name, table_2_gkvuw.last_name) full_name from table_1_qegkz, table_2_gkvuw where table_1_qegkz.id = table_2_gkvuw.id);

0 comments on commit 346f7ab

Please sign in to comment.