diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 89a89310bb89..372c27902d69 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,14 +29,14 @@ env: jobs: macos-jdk8: - name: 'macOS (JDK 8)' + name: 'macOS (JDK 17)' runs-on: macos-latest steps: - uses: actions/checkout@v2 - - name: 'Set up JDK 8' + - name: 'Set up JDK 17' uses: actions/setup-java@v1 with: - java-version: 8 + java-version: 17 - name: 'Build project' run: | mvn clean install -DskipTests -Pitests diff --git a/Jenkinsfile b/Jenkinsfile index 50250d739b3c..eb5d8494793a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -91,11 +91,12 @@ def buildHive(args) { set -x . /etc/profile.d/confs.sh export USER="`whoami`" -export MAVEN_OPTS="-Xmx2g" +export MAVEN_OPTS="-Xmx4G" export -n HIVE_CONF_DIR +sw java 17 && . /etc/profile.d/java.sh cp $SETTINGS .git/settings.xml OPTS=" -s $PWD/.git/settings.xml -B -Dtest.groups= " -OPTS+=" -Pitests,qsplits,dist,errorProne,iceberg" +OPTS+=" -Pitests,qsplits,dist,iceberg" OPTS+=" -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugin.surefire.SurefirePlugin=INFO" OPTS+=" -Dmaven.repo.local=$PWD/.git/m2" git config extra.mavenOpts "$OPTS" @@ -261,7 +262,7 @@ if [ $n != 0 ]; then exit 1 fi ''' - buildHive("-Pspotbugs -pl " + spotbugsProjects.join(",") + " -am test-compile com.github.spotbugs:spotbugs-maven-plugin:4.0.0:check") + buildHive("-Pspotbugs -pl " + spotbugsProjects.join(",") + " -am test-compile com.github.spotbugs:spotbugs-maven-plugin:4.5.0.0:check") } stage('Compile') { buildHive("install -Dtest=noMatches") @@ -290,12 +291,14 @@ fi stage('init-metastore') { withEnv(["dbType=$dbType"]) { sh '''#!/bin/bash -e + sw java 17 && . /etc/profile.d/java.sh set -x echo 127.0.0.1 dev_$dbType | sudo tee -a /etc/hosts . /etc/profile.d/confs.sh sw hive-dev $PWD export DOCKER_NETWORK=host export DBNAME=metastore +export HADOOP_CLIENT_OPTS="--add-opens java.base/java.net=ALL-UNNAMED" reinit_metastore $dbType time docker rm -f dev_$dbType || true ''' @@ -398,6 +401,7 @@ tar -xzf packaging/target/apache-hive-*-nightly-*-src.tar.gz } stage('Generate javadoc') { sh """#!/bin/bash -e + sw java 17 && . /etc/profile.d/java.sh mvn install javadoc:javadoc javadoc:aggregate -DskipTests -pl '!itests/hive-jmh,!itests/util' """ } diff --git a/bin/ext/beeline.sh b/bin/ext/beeline.sh index d825db9139f6..e3a25be43e8e 100644 --- a/bin/ext/beeline.sh +++ b/bin/ext/beeline.sh @@ -29,7 +29,7 @@ beeline () { hadoopClasspath="${HADOOP_CLASSPATH}:" fi export HADOOP_CLASSPATH="${hadoopClasspath}${HIVE_CONF_DIR}:${beelineJarPath}:${superCsvJarPath}:${jlineJarPath}" - export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=beeline-log4j2.properties " + export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=beeline-log4j2.properties --add-opens=java.base/java.net=ALL-UNNAMED " if [ "$EXECUTE_WITH_JAVA" != "true" ] ; then # if CLIUSER is not empty, then pass it as user id / password during beeline redirect diff --git a/common/pom.xml b/common/pom.xml index bbe2fa9cd33f..4d09fe122282 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -34,6 +34,11 @@ hive-classification ${project.version} + + org.apache.avro + avro + ${avro.version} + org.apache.hive hive-shims @@ -69,6 +74,12 @@ org.apache.orc orc-core + + + org.apache.hadoop + hadoop-client-api + + jline @@ -381,6 +392,18 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + + --add-opens + org.apache.hadoop/org.apache.hadoop.fs=ALL-UNNAMED + --add-opens + java.base/java.net=ALL-UNNAMED + + + org.apache.maven.plugins maven-antrun-plugin @@ -439,6 +462,14 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + 17 + 17 + + diff --git a/common/src/java/org/apache/hadoop/hive/ant/GenHiveTemplate.java b/common/src/java/org/apache/hadoop/hive/ant/GenHiveTemplate.java index ff9ad6058358..c751fa5120f7 100644 --- a/common/src/java/org/apache/hadoop/hive/ant/GenHiveTemplate.java +++ b/common/src/java/org/apache/hadoop/hive/ant/GenHiveTemplate.java @@ -73,21 +73,23 @@ private Document generateTemplate() throws Exception { doc.appendChild(doc.createProcessingInstruction( "xml-stylesheet", "type=\"text/xsl\" href=\"configuration.xsl\"")); - doc.appendChild(doc.createComment("\n" + - " Licensed to the Apache Software Foundation (ASF) under one or more\n" + - " contributor license agreements. See the NOTICE file distributed with\n" + - " this work for additional information regarding copyright ownership.\n" + - " The ASF licenses this file to You under the Apache License, Version 2.0\n" + - " (the \"License\"); you may not use this file except in compliance with\n" + - " the License. You may obtain a copy of the License at\n" + - "\n" + - " http://www.apache.org/licenses/LICENSE-2.0\n" + - "\n" + - " Unless required by applicable law or agreed to in writing, software\n" + - " distributed under the License is distributed on an \"AS IS\" BASIS,\n" + - " WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" + - " See the License for the specific language governing permissions and\n" + - " limitations under the License.\n")); + doc.appendChild(doc.createComment(""" + + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + """)); Element root = doc.createElement("configuration"); doc.appendChild(root); diff --git a/common/src/java/org/apache/hadoop/hive/common/CompressionUtils.java b/common/src/java/org/apache/hadoop/hive/common/CompressionUtils.java index c5c050bdcc01..6d84107b9028 100644 --- a/common/src/java/org/apache/hadoop/hive/common/CompressionUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/CompressionUtils.java @@ -169,30 +169,30 @@ public static List unTar(final String inputFileName, final String outputDi // no sub-directories continue; } - LOG.debug(String.format("Attempting to write output directory %s.", - outputFile.getAbsolutePath())); + LOG.debug("Attempting to write output directory %s.".formatted( + outputFile.getAbsolutePath())); if (!outputFile.exists()) { - LOG.debug(String.format("Attempting to create output directory %s.", - outputFile.getAbsolutePath())); + LOG.debug("Attempting to create output directory %s.".formatted( + outputFile.getAbsolutePath())); if (!outputFile.mkdirs()) { - throw new IllegalStateException(String.format("Couldn't create directory %s.", - outputFile.getAbsolutePath())); + throw new IllegalStateException("Couldn't create directory %s.".formatted( + outputFile.getAbsolutePath())); } } } else { final OutputStream outputFileStream; if (flatten) { File flatOutputFile = new File(outputDir, outputFile.getName()); - LOG.debug(String.format("Creating flat output file %s.", flatOutputFile.getAbsolutePath())); + LOG.debug("Creating flat output file %s.".formatted(flatOutputFile.getAbsolutePath())); outputFileStream = new FileOutputStream(flatOutputFile); } else if (!outputFile.getParentFile().exists()) { - LOG.debug(String.format("Attempting to create output directory %s.", - outputFile.getParentFile().getAbsoluteFile())); + LOG.debug("Attempting to create output directory %s.".formatted( + outputFile.getParentFile().getAbsoluteFile())); if (!outputFile.getParentFile().getAbsoluteFile().mkdirs()) { - throw new IllegalStateException(String.format("Couldn't create directory %s.", - outputFile.getParentFile().getAbsolutePath())); + throw new IllegalStateException("Couldn't create directory %s.".formatted( + outputFile.getParentFile().getAbsolutePath())); } - LOG.debug(String.format("Creating output file %s.", outputFile.getAbsolutePath())); + LOG.debug("Creating output file %s.".formatted(outputFile.getAbsolutePath())); outputFileStream = new FileOutputStream(outputFile); } else { outputFileStream = new FileOutputStream(outputFile); diff --git a/common/src/java/org/apache/hadoop/hive/common/CopyOnFirstWriteProperties.java b/common/src/java/org/apache/hadoop/hive/common/CopyOnFirstWriteProperties.java index 41a81cc22540..41e7b883ce3c 100644 --- a/common/src/java/org/apache/hadoop/hive/common/CopyOnFirstWriteProperties.java +++ b/common/src/java/org/apache/hadoop/hive/common/CopyOnFirstWriteProperties.java @@ -361,6 +361,7 @@ public void setInterned(Properties p) { public CopyOnFirstWriteProperties() { } + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public Properties getInterned() { return interned; } diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java index be994461f318..89e3d7376947 100644 --- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java @@ -1007,8 +1007,10 @@ public static boolean rename(FileSystem fs, Path sourcePath, // If destPath directory exists, rename call will move the sourcePath // into destPath without failing. So check it before renaming. if (fs.exists(destPath)) { - throw new IOException("Cannot rename the source path. The destination " - + "path already exists."); + throw new IOException(""" + Cannot rename the source path. The destination \ + path already exists.\ + """); } return fs.rename(sourcePath, destPath); } @@ -1094,8 +1096,10 @@ public static void checkDeletePermission(Path path, Configuration conf, String u if (childStatus.getOwner().equals(user)) { return; } - String msg = String.format("Permission Denied: User %s can't delete %s because sticky bit is" - + " set on the parent dir and user does not own this file or its parent", user, path); + String msg = (""" + Permission Denied: User %s can't delete %s because sticky bit is\ + set on the parent dir and user does not own this file or its parent\ + """).formatted(user, path); throw new IOException(msg); } diff --git a/common/src/java/org/apache/hadoop/hive/common/HeapMemoryMonitor.java b/common/src/java/org/apache/hadoop/hive/common/HeapMemoryMonitor.java index 56ec2fd6a140..bc85e7af138e 100644 --- a/common/src/java/org/apache/hadoop/hive/common/HeapMemoryMonitor.java +++ b/common/src/java/org/apache/hadoop/hive/common/HeapMemoryMonitor.java @@ -100,8 +100,10 @@ private static MemoryPoolMXBean getTenuredGenPool() { if (isUsageThresholdSupported) { return pool; } else { - LOG.error("{} vendor does not support isCollectionUsageThresholdSupported() and isUsageThresholdSupported()" + - " for tenured memory pool '{}'.", vendor, pool.getName()); + LOG.error(""" + {} vendor does not support isCollectionUsageThresholdSupported() and isUsageThresholdSupported()\ + for tenured memory pool '{}'.\ + """, vendor, pool.getName()); } } } diff --git a/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java b/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java index 8d0d6c55e659..d2d867f8fefa 100644 --- a/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/JavaUtils.java @@ -70,8 +70,8 @@ public static boolean closeClassLoadersTo(ClassLoader current, ClassLoader stop) try { closeClassLoader(current); } catch (IOException e) { - String detailedMessage = current instanceof URLClassLoader ? - Arrays.toString(((URLClassLoader) current).getURLs()) : + String detailedMessage = current instanceof URLClassLoader urlcl ? + Arrays.toString(urlcl.getURLs()) : ""; LOG.info("Failed to close class loader " + current + " " + detailedMessage, e); } @@ -90,8 +90,8 @@ private static boolean isValidHierarchy(ClassLoader current, ClassLoader stop) { } public static void closeClassLoader(ClassLoader loader) throws IOException { - if (loader instanceof Closeable) { - ((Closeable) loader).close(); + if (loader instanceof Closeable closeable) { + closeable.close(); } else { LOG.warn("Ignoring attempt to close class loader ({}) -- not instance of UDFClassLoader.", loader == null ? "mull" : loader.getClass().getSimpleName()); diff --git a/common/src/java/org/apache/hadoop/hive/common/LogUtils.java b/common/src/java/org/apache/hadoop/hive/common/LogUtils.java index 806ed9be66cf..c0910858c25c 100644 --- a/common/src/java/org/apache/hadoop/hive/common/LogUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/LogUtils.java @@ -245,16 +245,14 @@ public static void unregisterLoggingContext() { public static String getLogFilePath() { String logFilePath = null; org.apache.logging.log4j.Logger rootLogger = LogManager.getRootLogger(); - if (rootLogger instanceof org.apache.logging.log4j.core.Logger) { - org.apache.logging.log4j.core.Logger coreLogger = - (org.apache.logging.log4j.core.Logger)rootLogger; + if (rootLogger instanceof org.apache.logging.log4j.core.Logger coreLogger) { for (Appender appender : coreLogger.getAppenders().values()) { - if (appender instanceof FileAppender) { - logFilePath = ((FileAppender) appender).getFileName(); - } else if (appender instanceof RollingFileAppender) { - logFilePath = ((RollingFileAppender) appender).getFileName(); - } else if (appender instanceof RollingRandomAccessFileAppender) { - logFilePath = ((RollingRandomAccessFileAppender) appender).getFileName(); + if (appender instanceof FileAppender fileAppender) { + logFilePath = fileAppender.getFileName(); + } else if (appender instanceof RollingFileAppender fileAppender) { + logFilePath = fileAppender.getFileName(); + } else if (appender instanceof RollingRandomAccessFileAppender fileAppender) { + logFilePath = fileAppender.getFileName(); } } } diff --git a/common/src/java/org/apache/hadoop/hive/common/cli/ShellCmdExecutor.java b/common/src/java/org/apache/hadoop/hive/common/cli/ShellCmdExecutor.java index e89327002424..b2ab557d66e0 100644 --- a/common/src/java/org/apache/hadoop/hive/common/cli/ShellCmdExecutor.java +++ b/common/src/java/org/apache/hadoop/hive/common/cli/ShellCmdExecutor.java @@ -22,12 +22,14 @@ import java.io.PrintStream; import org.apache.hive.common.util.StreamPrinter; +import org.apache.hive.common.util.SuppressFBWarnings; public class ShellCmdExecutor { private String cmd; private PrintStream out; private PrintStream err; + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public ShellCmdExecutor(String cmd, PrintStream out, PrintStream err) { this.cmd = cmd; this.out = out; diff --git a/common/src/java/org/apache/hadoop/hive/common/format/datetime/HiveSqlDateTimeFormatter.java b/common/src/java/org/apache/hadoop/hive/common/format/datetime/HiveSqlDateTimeFormatter.java index 00d4796f7bfb..be100464c9dd 100644 --- a/common/src/java/org/apache/hadoop/hive/common/format/datetime/HiveSqlDateTimeFormatter.java +++ b/common/src/java/org/apache/hadoop/hive/common/format/datetime/HiveSqlDateTimeFormatter.java @@ -28,6 +28,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.common.type.Date; import org.apache.hadoop.hive.common.type.Timestamp; +import org.apache.hive.common.util.SuppressFBWarnings; import java.io.Serializable; import java.time.DateTimeException; @@ -520,6 +521,7 @@ public Token(TokenType tokenType, String string) { this(tokenType, null, null, string, string.length(), false); } + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public Token(TokenType tokenType, TemporalField temporalField, TemporalUnit temporalUnit, String string, int length, boolean fillMode) { this.type = tokenType; @@ -847,8 +849,10 @@ private void verifyForParse() { !(temporalFields.contains(ChronoField.MONTH_OF_YEAR) && temporalFields.contains(ChronoField.DAY_OF_MONTH) || temporalFields.contains(ChronoField.DAY_OF_YEAR))) { - throw new IllegalArgumentException("Missing day of year or (month of year + day of month)" - + " tokens."); + throw new IllegalArgumentException(""" + Missing day of year or (month of year + day of month)\ + tokens.\ + """); } if (containsIsoFields && !(temporalFields.contains(IsoFields.WEEK_OF_WEEK_BASED_YEAR) && @@ -856,8 +860,10 @@ private void verifyForParse() { throw new IllegalArgumentException("Missing week of year (iw) or day of week (id) tokens."); } if (roundYearCount > 0 && yearCount > 0) { - throw new IllegalArgumentException("Invalid duplication of format element: Both year and" - + "round year are provided"); + throw new IllegalArgumentException(""" + Invalid duplication of format element: Both year and\ + round year are provided\ + """); } for (TemporalField tokenType : temporalFields) { if (Collections.frequency(temporalFields, tokenType) > 1) { @@ -1281,8 +1287,10 @@ private int parseNumericTemporal(String substring, Token token) { return 0; } if ("0".equals(substring)) { - throw new IllegalArgumentException("Value of hour of day (hh/hh12) in input is 0. " - + "The value should be between 1 and 12."); + throw new IllegalArgumentException(""" + Value of hour of day (hh/hh12) in input is 0. \ + The value should be between 1 and 12.\ + """); } } if (token.temporalField == ChronoField.YEAR diff --git a/common/src/java/org/apache/hadoop/hive/common/io/CachingPrintStream.java b/common/src/java/org/apache/hadoop/hive/common/io/CachingPrintStream.java index 70dab87dffd5..d98b064d6223 100644 --- a/common/src/java/org/apache/hadoop/hive/common/io/CachingPrintStream.java +++ b/common/src/java/org/apache/hadoop/hive/common/io/CachingPrintStream.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.common.io; +import org.apache.hive.common.util.SuppressFBWarnings; + import java.io.FileNotFoundException; import java.io.OutputStream; import java.io.UnsupportedEncodingException; @@ -52,6 +54,7 @@ public void flush() { super.flush(); } + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public List getOutput() { return output; } diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/DagJsonParserUtils.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/DagJsonParserUtils.java index 4590fefc052f..c4a44a03c801 100644 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/DagJsonParserUtils.java +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/DagJsonParserUtils.java @@ -31,6 +31,7 @@ public class DagJsonParserUtils { private static final List operatorNoStats = Arrays.asList(new String[] { "File Output Operator", "Reduce Output Operator" }); + @SuppressFBWarnings(value = "MS_EXPOSE_REP", justification = "intended_to_do") public static List getOperatorNoStats() { return operatorNoStats; } diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Stage.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Stage.java index f2b327392ec9..a28b8538e3e9 100644 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Stage.java +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Stage.java @@ -100,8 +100,7 @@ public void extractVertex(JSONObject object) throws Exception { Object o = edges.get(to); Vertex v = vertices.get(to); // 1 to 1 mapping - if (o instanceof JSONObject) { - JSONObject obj = (JSONObject) o; + if (o instanceof JSONObject obj) { String parent = obj.getString("parent"); Vertex parentVertex = vertices.get(parent); if (parentVertex == null) { @@ -194,8 +193,7 @@ Op extractOp(String opName, JSONObject opObj) throws Exception { Object o = opObj.get(name); if (isPrintable(o) && !o.toString().isEmpty()) { attrs.put(name, o.toString()); - } else if (o instanceof JSONObject) { - JSONObject attrObj = (JSONObject) o; + } else if (o instanceof JSONObject attrObj) { if (attrObj.length() > 0) { if (name.equals("Processor Tree:")) { JSONObject object = new JSONObject(new LinkedHashMap<>()); diff --git a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java index 7771d373b6e7..2cf752320846 100644 --- a/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java +++ b/common/src/java/org/apache/hadoop/hive/common/jsonexplain/Vertex.java @@ -171,13 +171,13 @@ Op extractOp(JSONObject object, Op parent) throws Exception { for (String attrName : JSONObject.getNames(attrObj)) { if (attrName.equals("children")) { Object childrenObj = attrObj.get(attrName); - if (childrenObj instanceof JSONObject) { - if (((JSONObject) childrenObj).length() != 0) { - children.add(extractOp((JSONObject) childrenObj, op)); + if (childrenObj instanceof JSONObject nObject) { + if (nObject.length() != 0) { + children.add(extractOp(nObject, op)); } - } else if (childrenObj instanceof JSONArray) { - if (((JSONArray) childrenObj).length() != 0) { - JSONArray array = ((JSONArray) childrenObj); + } else if (childrenObj instanceof JSONArray nArray) { + if (nArray.length() != 0) { + JSONArray array =nArray; for (int index = 0; index < array.length(); index++) { children.add(extractOp(array.getJSONObject(index), op)); } diff --git a/common/src/java/org/apache/hadoop/hive/common/log/InPlaceUpdate.java b/common/src/java/org/apache/hadoop/hive/common/log/InPlaceUpdate.java index c9023fe7dd1d..bddcbb322135 100644 --- a/common/src/java/org/apache/hadoop/hive/common/log/InPlaceUpdate.java +++ b/common/src/java/org/apache/hadoop/hive/common/log/InPlaceUpdate.java @@ -19,6 +19,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.common.util.SuppressFBWarnings; import org.fusesource.jansi.Ansi; import java.io.PrintStream; @@ -51,6 +52,7 @@ public class InPlaceUpdate { private int lines = 0; private PrintStream out; + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public InPlaceUpdate(PrintStream out) { this.out = out; } @@ -149,14 +151,14 @@ public void render(ProgressMonitor monitor) { // VERTICES STATUS TOTAL COMPLETED RUNNING PENDING FAILED KILLED // ------------------------------------------------------------------------------- reprintLine(SEPARATOR); - reprintLineWithColorAsBold(String.format(HEADER_FORMAT, monitor.headers().toArray()), + reprintLineWithColorAsBold(HEADER_FORMAT.formatted(monitor.headers().toArray()), Ansi.Color.CYAN); reprintLine(SEPARATOR); // Map 1 .......... container SUCCEEDED 7 7 0 0 0 0 List printReady = - monitor.rows().stream().map(row -> String.format(VERTEX_FORMAT, row.toArray())).collect(Collectors.toList()); + monitor.rows().stream().map(row -> VERTEX_FORMAT.formatted(row.toArray())).collect(Collectors.toList()); reprintMultiLine(StringUtils.join(printReady, "\n")); // ------------------------------------------------------------------------------- @@ -165,12 +167,12 @@ public void render(ProgressMonitor monitor) { String progressStr = "" + (int) (monitor.progressedPercentage() * 100) + "%"; float et = (float) (System.currentTimeMillis() - monitor.startTime()) / (float) 1000; String elapsedTime = "ELAPSED TIME: " + secondsFormatter.format(et) + " s"; - String footer = String.format( - FOOTER_FORMAT, - monitor.footerSummary(), - getInPlaceProgressBar(monitor.progressedPercentage()), - progressStr, - elapsedTime); + String footer = + FOOTER_FORMAT.formatted( + monitor.footerSummary(), + getInPlaceProgressBar(monitor.progressedPercentage()), + progressStr, + elapsedTime); reprintLine(SEPARATOR); reprintLineWithColorAsBold(footer, Ansi.Color.RED); diff --git a/common/src/java/org/apache/hadoop/hive/common/log/LogRedirector.java b/common/src/java/org/apache/hadoop/hive/common/log/LogRedirector.java index 3d6909769c3b..469fec398ea7 100644 --- a/common/src/java/org/apache/hadoop/hive/common/log/LogRedirector.java +++ b/common/src/java/org/apache/hadoop/hive/common/log/LogRedirector.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.common.log; +import org.apache.hive.common.util.SuppressFBWarnings; import org.slf4j.Logger; import java.io.BufferedReader; @@ -50,6 +51,7 @@ public LogRedirector(InputStream in, Logger logger, LogSourceCallback callback) this.logger = logger; } + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public LogRedirector(InputStream in, Logger logger, List errLogs, LogSourceCallback callback) { this.in = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); diff --git a/common/src/java/org/apache/hadoop/hive/common/type/Date.java b/common/src/java/org/apache/hadoop/hive/common/type/Date.java index 0f47ba513d47..f4fe28468300 100644 --- a/common/src/java/org/apache/hadoop/hive/common/type/Date.java +++ b/common/src/java/org/apache/hadoop/hive/common/type/Date.java @@ -42,7 +42,8 @@ /** * This is the internal type for Date. The full qualified input format of Date * is "uuuu-MM-dd". For example: "2021-02-11". - * + *
+ * * * * @@ -116,8 +117,8 @@ public int hashCode() { @Override public boolean equals(Object other) { - if (other instanceof Date) { - return compareTo((Date) other) == 0; + if (other instanceof Date date) { + return compareTo(date) == 0; } return false; } diff --git a/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java b/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java index 7d4eaf7c460d..0c88831c2fd5 100644 --- a/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java +++ b/common/src/java/org/apache/hadoop/hive/common/type/Decimal128.java @@ -24,6 +24,7 @@ import java.nio.IntBuffer; import org.apache.hive.common.util.Decimal128FastBuffer; +import org.apache.hive.common.util.SuppressFBWarnings; /** * This code was based on code from Microsoft's PolyBase. @@ -168,7 +169,7 @@ public Decimal128(double val, short scale) { /** * Translates a {@code UnsignedInt128} unscaled value, an {@code int} scale, * and sign flag into a {@code Decimal128} . The value of the - * {@code Decimal128} is (unscaledVal × 10-scale). + * {@code Decimal128} is (unscaledVal × 10 -scale). * * @param unscaledVal * unscaled value of the {@code Decimal128}. @@ -1476,6 +1477,7 @@ public short getScale() { * * @return the unscaled value of this {@code Decimal128}. */ + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public UnsignedInt128 getUnscaledValue() { return unscaledValue; } @@ -1965,6 +1967,7 @@ public Decimal128 fastUpdateFromInternalStorage(byte[] internalStorage, short sc /** * This setter is only for de-serialization, should not be used otherwise. */ + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public void setUnscaledValue(UnsignedInt128 unscaledValue) { this.unscaledValue = unscaledValue; } diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveIntervalYearMonth.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveIntervalYearMonth.java index 7fb4e9f0989d..915958588aef 100644 --- a/common/src/java/org/apache/hadoop/hive/common/type/HiveIntervalYearMonth.java +++ b/common/src/java/org/apache/hadoop/hive/common/type/HiveIntervalYearMonth.java @@ -111,7 +111,7 @@ public int hashCode() { public String toString() { String yearMonthSignStr = totalMonths >= 0 ? "" : "-"; - return String.format("%s%d-%d", + return "%s%d-%d".formatted( yearMonthSignStr, Math.abs(getYears()), Math.abs(getMonths())); } diff --git a/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java b/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java index f4b5c8c584bc..75cc7644227c 100644 --- a/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java +++ b/common/src/java/org/apache/hadoop/hive/common/type/HiveVarchar.java @@ -62,8 +62,8 @@ public boolean equals(Object rhs) { if (rhs == this) { return true; } - if (rhs instanceof HiveVarchar) { - return this.getValue().equals(((HiveVarchar) rhs).getValue()); + if (rhs instanceof HiveVarchar varchar) { + return this.getValue().equals(varchar.getValue()); } return false; } diff --git a/common/src/java/org/apache/hadoop/hive/common/type/SignedInt128.java b/common/src/java/org/apache/hadoop/hive/common/type/SignedInt128.java index 9da8cc91b173..014e22011be2 100644 --- a/common/src/java/org/apache/hadoop/hive/common/type/SignedInt128.java +++ b/common/src/java/org/apache/hadoop/hive/common/type/SignedInt128.java @@ -532,8 +532,7 @@ public boolean isZero() { @Override public boolean equals(Object obj) { - if (obj instanceof SignedInt128) { - SignedInt128 o = (SignedInt128) obj; + if (obj instanceof SignedInt128 o) { return this.negative == o.negative && mag.equals(o.mag); } else { return false; diff --git a/common/src/java/org/apache/hadoop/hive/common/type/Timestamp.java b/common/src/java/org/apache/hadoop/hive/common/type/Timestamp.java index c026f8a8e9e8..238d7bd984d1 100644 --- a/common/src/java/org/apache/hadoop/hive/common/type/Timestamp.java +++ b/common/src/java/org/apache/hadoop/hive/common/type/Timestamp.java @@ -44,7 +44,8 @@ * Timestamp is "uuuu-MM-dd HH:mm:ss[.SSS...]", where the time part is optional. * If time part is absent, a default '00:00:00.0' will be used. * - *
FieldFormat
+ *
+ * * * * @@ -134,8 +135,8 @@ public int hashCode() { @Override public boolean equals(Object other) { - if (other instanceof Timestamp) { - return compareTo((Timestamp) other) == 0; + if (other instanceof Timestamp timestamp) { + return compareTo(timestamp) == 0; } return false; } diff --git a/common/src/java/org/apache/hadoop/hive/common/type/TimestampTZ.java b/common/src/java/org/apache/hadoop/hive/common/type/TimestampTZ.java index 3ff06e0eeadf..97af63b6dda8 100644 --- a/common/src/java/org/apache/hadoop/hive/common/type/TimestampTZ.java +++ b/common/src/java/org/apache/hadoop/hive/common/type/TimestampTZ.java @@ -77,8 +77,8 @@ public int hashCode() { @Override public boolean equals(Object other) { - if (other instanceof TimestampTZ) { - return compareTo((TimestampTZ) other) == 0; + if (other instanceof TimestampTZ tZ) { + return compareTo(tZ) == 0; } return false; } diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index d33d60e7b0f5..efa3c93592c3 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -458,6 +458,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal * Get a set containing configuration parameter names used by LLAP Server instances * @return an unmodifiable set containing llap ConfVars */ + @SuppressFBWarnings(value="MS_EXPOSE_REP", justification = "intended_to_do") public static final Set getLlapDaemonConfVars() { return llapDaemonVarsSet; } @@ -7163,6 +7164,7 @@ public static String generateDeprecationWarning() { private static final Object reverseMapLock = new Object(); private static HashMap reverseMap = null; + @SuppressFBWarnings(value = "MS_EXPOSE_REP", justification = "intended_to_do") public static HashMap getOrCreateReverseMap() { // This should be called rarely enough; for now it's ok to just lock every time. synchronized (reverseMapLock) { diff --git a/common/src/java/org/apache/hadoop/hive/conf/Validator.java b/common/src/java/org/apache/hadoop/hive/conf/Validator.java index 8bc64a63195f..3d729a745e9d 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/Validator.java +++ b/common/src/java/org/apache/hadoop/hive/conf/Validator.java @@ -365,15 +365,15 @@ public String validate(String value) { if (value == null) return null; final Path path = FileSystems.getDefault().getPath(value); if (path == null) { - return String.format("Path '%s' provided could not be located.", value); + return "Path '%s' provided could not be located.".formatted(value); } else { final boolean isDir = Files.isDirectory(path); final boolean isWritable = Files.isWritable(path); if (!isDir) { - return String.format("Path '%s' provided is not a directory.", value); + return "Path '%s' provided is not a directory.".formatted(value); } if (!isWritable) { - return String.format("Path '%s' provided is not writable.", value); + return "Path '%s' provided is not writable.".formatted(value); } return null; } diff --git a/common/src/java/org/apache/hadoop/hive/conf/valcoersion/JavaIOTmpdirVariableCoercion.java b/common/src/java/org/apache/hadoop/hive/conf/valcoersion/JavaIOTmpdirVariableCoercion.java index d2a24f31a08a..3cd27f145e1c 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/valcoersion/JavaIOTmpdirVariableCoercion.java +++ b/common/src/java/org/apache/hadoop/hive/conf/valcoersion/JavaIOTmpdirVariableCoercion.java @@ -50,7 +50,7 @@ private String coerce(String originalValue) { Path absolutePath = FileUtils.makeAbsolute(LOCAL_FILE_SYSTEM, originalPath); return absolutePath.toString(); } catch (IOException exception) { - LOG.warn(String.format("Unable to resolve 'java.io.tmpdir' for absolute path '%s'", originalValue)); + LOG.warn("Unable to resolve 'java.io.tmpdir' for absolute path '%s'".formatted(originalValue)); return originalValue; } } diff --git a/common/src/java/org/apache/hive/common/util/CleanerUtil.java b/common/src/java/org/apache/hive/common/util/CleanerUtil.java index 1fd8cf8b455d..e4a0ca1c65d6 100644 --- a/common/src/java/org/apache/hive/common/util/CleanerUtil.java +++ b/common/src/java/org/apache/hive/common/util/CleanerUtil.java @@ -75,8 +75,8 @@ public static BufferCleaner getCleaner() { static { final Object hack = AccessController.doPrivileged( (PrivilegedAction) CleanerUtil::unmapHackImpl); - if (hack instanceof BufferCleaner) { - CLEANER = (BufferCleaner) hack; + if (hack instanceof BufferCleaner cleaner) { + CLEANER = cleaner; UNMAP_SUPPORTED = true; UNMAP_NOT_SUPPORTED_REASON = null; } else { diff --git a/common/src/java/org/apache/hive/common/util/DateUtils.java b/common/src/java/org/apache/hive/common/util/DateUtils.java index cc023011a6f8..98c4910ab736 100644 --- a/common/src/java/org/apache/hive/common/util/DateUtils.java +++ b/common/src/java/org/apache/hive/common/util/DateUtils.java @@ -52,7 +52,7 @@ public static int parseNumericValueWithRange(String fieldName, if (strVal != null) { result = Integer.parseInt(strVal); if (result < minValue || result > maxValue) { - throw new IllegalArgumentException(String.format("%s value %d outside range [%d, %d]", + throw new IllegalArgumentException("%s value %d outside range [%d, %d]".formatted( fieldName, result, minValue, maxValue)); } } diff --git a/common/src/java/org/apache/hive/common/util/HiveStringUtils.java b/common/src/java/org/apache/hive/common/util/HiveStringUtils.java index ee94e413c726..d703f9f41f79 100644 --- a/common/src/java/org/apache/hive/common/util/HiveStringUtils.java +++ b/common/src/java/org/apache/hive/common/util/HiveStringUtils.java @@ -216,7 +216,7 @@ public static String byteToHexString(byte[] bytes, int start, int end) { } StringBuilder s = new StringBuilder(); for(int i = start; i < end; i++) { - s.append(String.format("%02x", bytes[i])); + s.append("%02x".formatted(bytes[i])); } return s.toString(); } @@ -259,7 +259,7 @@ public static String uriToString(URI[] uris){ /** * @param str * The string array to be parsed into an URI array. - * @return null if str is null, else the URI array + * @return 
null
if str is 
null
, else the URI array * equivalent to str. * @throws IllegalArgumentException * If any string in str violates RFC 2396. diff --git a/common/src/java/org/apache/hive/common/util/ReflectionUtil.java b/common/src/java/org/apache/hive/common/util/ReflectionUtil.java index 5b3da0708224..42fcbaa40f8d 100644 --- a/common/src/java/org/apache/hive/common/util/ReflectionUtil.java +++ b/common/src/java/org/apache/hive/common/util/ReflectionUtil.java @@ -31,8 +31,6 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import static java.lang.String.format; - /** * Same as Hadoop ReflectionUtils, but (1) does not leak classloaders (or shouldn't anyway, we * rely on Guava cache, and could fix it otherwise); (2) does not have a hidden epic lock. @@ -102,8 +100,8 @@ public static T newInstance(Class theClass, Configuration conf) { */ public static void setConf(Object theObject, Configuration conf) { if (conf != null) { - if (theObject instanceof Configurable) { - ((Configurable) theObject).setConf(conf); + if (theObject instanceof Configurable configurable) { + configurable.setConf(conf); } setJobConf(theObject, conf); } @@ -129,13 +127,25 @@ private static void setJobConf(Object theObject, Configuration conf) { * @param value new value * @throws RuntimeException in case the field is not found or cannot be set. */ + @SuppressFBWarnings(value = "REFLF_REFLECTION_MAY_INCREASE_ACCESSIBILITY_OF_FIELD", justification = "intended_to_do") public static void setField(Object object, String field, Object value) { try { Field fieldToChange = object.getClass().getDeclaredField(field); fieldToChange.setAccessible(true); fieldToChange.set(object, value); } catch (NoSuchFieldException | IllegalAccessException e) { - throw new RuntimeException(format("Cannot set field %s in object %s", field, object.getClass())); + throw new RuntimeException("Cannot set field %s in object %s".formatted(field, object.getClass())); + } + } + + @SuppressFBWarnings(value = "RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE", justification = "intended_to_do") + public static void setField(Object object, Field fld, Object value) { + try { + fld.setAccessible(true); + fld.set(object, value); + } catch (IllegalAccessException e) { + String fieldName = null == fld ? "n/a" : fld.getName(); + throw new RuntimeException("Failed to set " + fieldName + " of object", e); } } @@ -157,7 +167,7 @@ public static void setInAllFields(Object object, String field, Object value) { fieldToChange.set(object, value); } catch (NoSuchFieldException | IllegalAccessException e) { - throw new RuntimeException(format("Cannot set field %s in object %s", field, object.getClass())); + throw new RuntimeException("Cannot set field %s in object %s".formatted(field, object.getClass())); } } } diff --git a/common/src/java/org/apache/hive/common/util/RetryUtilities.java b/common/src/java/org/apache/hive/common/util/RetryUtilities.java index 12fe2b7018fd..1eda375abb52 100644 --- a/common/src/java/org/apache/hive/common/util/RetryUtilities.java +++ b/common/src/java/org/apache/hive/common/util/RetryUtilities.java @@ -63,18 +63,18 @@ public static abstract class ExponentiallyDecayingBatchWork public ExponentiallyDecayingBatchWork(int batchSize, int reducingFactor, int maxRetries) { if (batchSize <= 0) { - throw new IllegalArgumentException(String.format( - "Invalid batch size %d provided. Batch size must be greater than 0", batchSize)); + throw new IllegalArgumentException( + "Invalid batch size %d provided. Batch size must be greater than 0".formatted(batchSize)); } this.batchSize = batchSize; if (reducingFactor <= 1) { - throw new IllegalArgumentException(String.format( - "Invalid decaying factor %d provided. Decaying factor must be greater than 1", + throw new IllegalArgumentException( + "Invalid decaying factor %d provided. Decaying factor must be greater than 1".formatted( batchSize)); } if (maxRetries < 0) { - throw new IllegalArgumentException(String.format( - "Invalid number of maximum retries %d provided. It must be a non-negative integer value", + throw new IllegalArgumentException( + "Invalid number of maximum retries %d provided. It must be a non-negative integer value".formatted( maxRetries)); } //if maxRetries is 0 code retries until batch decays to zero @@ -92,12 +92,12 @@ public T run() throws Exception { try { return execute(size); } catch (Exception ex) { - LOG.warn(String.format("Exception thrown while processing using a batch size %d", size), + LOG.warn("Exception thrown while processing using a batch size %d".formatted(size), ex); } finally { attempt++; if (attempt == maxRetries) { - throw new RetryException(String.format("Maximum number of retry attempts %d exhausted", maxRetries)); + throw new RetryException("Maximum number of retry attempts %d exhausted".formatted(maxRetries)); } } } diff --git a/common/src/java/org/apache/hive/common/util/TimestampParser.java b/common/src/java/org/apache/hive/common/util/TimestampParser.java index b028e41fc487..d94de4cea05a 100644 --- a/common/src/java/org/apache/hive/common/util/TimestampParser.java +++ b/common/src/java/org/apache/hive/common/util/TimestampParser.java @@ -54,7 +54,8 @@ * In addition to accepting format patterns, this parser provides support for * three pre-defined formats: * - *
FieldFormat
+ *
+ * * * * diff --git a/common/src/java/org/apache/hive/http/HttpServer.java b/common/src/java/org/apache/hive/http/HttpServer.java index 83555fc5c73e..f89d9228e5ea 100644 --- a/common/src/java/org/apache/hive/http/HttpServer.java +++ b/common/src/java/org/apache/hive/http/HttpServer.java @@ -63,6 +63,7 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.security.http.CrossOriginFilter; +import org.apache.hive.common.util.SuppressFBWarnings; import org.apache.hive.http.security.PamAuthenticator; import org.apache.hive.http.security.PamConstraint; import org.apache.hive.http.security.PamConstraintMapping; @@ -250,6 +251,7 @@ public Builder setUsePAM(boolean usePAM) { return this; } + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public Builder setPAMAuthenticator(PamAuthenticator pamAuthenticator){ this.pamAuthenticator = pamAuthenticator; return this; @@ -440,8 +442,10 @@ static boolean hasAdministratorAccess( if (remoteUser == null) { if (response != null) { response.sendError(HttpServletResponse.SC_UNAUTHORIZED, - "Unauthenticated users are not " + - "authorized to access this page."); + """ + Unauthenticated users are not \ + authorized to access this page.\ + """); } return false; } @@ -751,11 +755,11 @@ String getLogDir(Configuration conf) { LoggerContext context = (LoggerContext)LogManager.getContext(false); for (Logger logger: context.getLoggers()) { for (Appender appender: logger.getAppenders().values()) { - if (appender instanceof AbstractOutputStreamAppender) { + if (appender instanceof AbstractOutputStreamAppender streamAppender) { OutputStreamManager manager = - ((AbstractOutputStreamAppender)appender).getManager(); - if (manager instanceof FileManager) { - String fileName = ((FileManager)manager).getFileName(); + streamAppender.getManager(); + if (manager instanceof FileManager fileManager) { + String fileName = fileManager.getFileName(); if (fileName != null) { return fileName.substring(0, fileName.lastIndexOf('/')); } diff --git a/common/src/java/org/apache/hive/http/JMXJsonServlet.java b/common/src/java/org/apache/hive/http/JMXJsonServlet.java index 7ad8af3df3ff..4f078b5a017e 100644 --- a/common/src/java/org/apache/hive/http/JMXJsonServlet.java +++ b/common/src/java/org/apache/hive/http/JMXJsonServlet.java @@ -383,14 +383,11 @@ private void writeObject(JsonGenerator jg, Object value) throws IOException { writeObject(jg, item); } jg.writeEndArray(); - } else if(value instanceof Number) { - Number n = (Number)value; + } else if(value instanceof Number n) { jg.writeNumber(n.toString()); - } else if(value instanceof Boolean) { - Boolean b = (Boolean)value; + } else if(value instanceof Boolean b) { jg.writeBoolean(b); - } else if(value instanceof CompositeData) { - CompositeData cds = (CompositeData)value; + } else if(value instanceof CompositeData cds) { CompositeType comp = cds.getCompositeType(); Set keys = comp.keySet(); jg.writeStartObject(); @@ -398,8 +395,7 @@ private void writeObject(JsonGenerator jg, Object value) throws IOException { writeAttribute(jg, key, cds.get(key)); } jg.writeEndObject(); - } else if(value instanceof TabularData) { - TabularData tds = (TabularData)value; + } else if(value instanceof TabularData tds) { jg.writeStartArray(); for(Object entry : tds.values()) { writeObject(jg, entry); diff --git a/common/src/java/org/apache/hive/http/security/PamLoginService.java b/common/src/java/org/apache/hive/http/security/PamLoginService.java index 8cf725ad9cfc..a8b354136fc9 100644 --- a/common/src/java/org/apache/hive/http/security/PamLoginService.java +++ b/common/src/java/org/apache/hive/http/security/PamLoginService.java @@ -16,6 +16,7 @@ */ package org.apache.hive.http.security; +import org.apache.hive.common.util.SuppressFBWarnings; import org.eclipse.jetty.security.DefaultIdentityService; import org.eclipse.jetty.security.IdentityService; import org.eclipse.jetty.security.LoginService; @@ -59,11 +60,13 @@ public boolean validate(UserIdentity user) { } @Override + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public IdentityService getIdentityService() { return identityService; } @Override + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public void setIdentityService(IdentityService identityService) { if (isRunning()) throw new IllegalStateException("Running"); diff --git a/common/src/test/org/apache/hadoop/hive/common/TestTezJsonParser.java b/common/src/test/org/apache/hadoop/hive/common/TestTezJsonParser.java index 63ffbc037be1..1b95e1c242d5 100644 --- a/common/src/test/org/apache/hadoop/hive/common/TestTezJsonParser.java +++ b/common/src/test/org/apache/hadoop/hive/common/TestTezJsonParser.java @@ -47,128 +47,130 @@ public void testQuery27() throws Exception { String content = new String(baos.toByteArray(), StandardCharsets.UTF_8); Assert .assertEquals( - "Plan optimized by CBO.\n" - + "\n" - + "Vertex dependency in root stage\n" - + "Map 1 <- Map 5 (BROADCAST_EDGE)\n" - + "Map 6 <- Map 8 (BROADCAST_EDGE)\n" - + "Map 9 <- Map 10 (BROADCAST_EDGE)\n" - + "Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE)\n" - + "Reducer 3 <- Reducer 2 (SIMPLE_EDGE)\n" - + "Reducer 4 <- Reducer 3 (SIMPLE_EDGE)\n" - + "Reducer 7 <- Map 11 (BROADCAST_EDGE), Map 12 (BROADCAST_EDGE), Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE)\n" - + "\n" - + "Stage-0\n" - + " Fetch Operator\n" - + " limit:100\n" - + " Stage-1\n" - + " Reducer 4\n" - + " File Output Operator [FS_55]\n" - + " Limit [LIM_54] (rows=100 width=133)\n" - + " Number of rows:100\n" - + " Select Operator [SEL_53] (rows=2319458 width=133)\n" - + " Output:[\"_col0\",\"_col1\",\"_col2\",\"_col3\",\"_col4\",\"_col5\",\"_col6\"]\n" - + " <-Reducer 3 [SIMPLE_EDGE]\n" - + " SHUFFLE [RS_52]\n" - + " Group By Operator [GBY_50] (rows=2319458 width=133)\n" - + " Output:[\"_col0\",\"_col1\",\"_col2\",\"_col3\",\"_col4\",\"_col5\",\"_col6\"],aggregations:[\"sum(VALUE._col0)\",\"sum(VALUE._col1)\",\"sum(VALUE._col2)\"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3\n" - + " <-Reducer 2 [SIMPLE_EDGE]\n" - + " SHUFFLE [RS_49]\n" - + " PartitionCols:_col0, _col1, _col2, _col3\n" - + " Group By Operator [GBY_48] (rows=4638916 width=133)\n" - + " Output:[\"_col0\",\"_col1\",\"_col2\",\"_col3\",\"_col4\",\"_col5\",\"_col6\"],aggregations:[\"sum(_col18)\",\"sum(_col26)\",\"sum(_col3)\"],keys:_col11, _col12, _col8, _col9\n" - + " Merge Join Operator [MERGEJOIN_99] (rows=4638916 width=133)\n" - + " Conds:RS_44._col1, _col2=RS_45._col17, _col16(Inner),Output:[\"_col3\",\"_col8\",\"_col9\",\"_col11\",\"_col12\",\"_col18\",\"_col26\"]\n" - + " <-Map 1 [SIMPLE_EDGE]\n" - + " SHUFFLE [RS_44]\n" - + " PartitionCols:_col1, _col2\n" - + " Map Join Operator [MAPJOIN_93] (rows=1585702 width=204)\n" - + " Conds:SEL_2._col0=RS_42._col0(Inner),HybridGraceHashJoin:true,Output:[\"_col1\",\"_col2\",\"_col3\"]\n" - + " <-Map 5 [BROADCAST_EDGE]\n" - + " BROADCAST [RS_42]\n" - + " PartitionCols:_col0\n" - + " Select Operator [SEL_5] (rows=4058 width=140)\n" - + " Output:[\"_col0\"]\n" - + " Filter Operator [FIL_86] (rows=4058 width=140)\n" - + " predicate:(d_moy BETWEEN 4 AND 10 and (d_year = 1998) and d_date_sk is not null)\n" - + " TableScan [TS_3] (rows=73049 width=140)\n" - + " default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:[\"d_date_sk\",\"d_year\",\"d_moy\"]\n" - + " <-Select Operator [SEL_2] (rows=1441548 width=204)\n" - + " Output:[\"_col0\",\"_col1\",\"_col2\",\"_col3\"]\n" - + " Filter Operator [FIL_85] (rows=1441548 width=204)\n" - + " predicate:(cs_bill_customer_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null)\n" - + " TableScan [TS_0] (rows=1441548 width=204)\n" - + " default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:[\"cs_sold_date_sk\",\"cs_bill_customer_sk\",\"cs_item_sk\",\"cs_net_profit\"]\n" - + " <-Reducer 7 [SIMPLE_EDGE]\n" - + " SHUFFLE [RS_45]\n" - + " PartitionCols:_col17, _col16\n" - + " Select Operator [SEL_40] (rows=4217197 width=133)\n" - + " Output:[\"_col1\",\"_col2\",\"_col4\",\"_col5\",\"_col11\",\"_col16\",\"_col17\",\"_col19\"]\n" - + " Map Join Operator [MAPJOIN_98] (rows=4217197 width=133)\n" - + " Conds:MAPJOIN_97._col3=RS_38._col0(Inner),HybridGraceHashJoin:true,Output:[\"_col5\",\"_col10\",\"_col11\",\"_col13\",\"_col18\",\"_col19\",\"_col21\",\"_col22\"]\n" - + " <-Map 12 [BROADCAST_EDGE]\n" - + " BROADCAST [RS_38]\n" - + " PartitionCols:_col0\n" - + " Select Operator [SEL_27] (rows=12 width=261)\n" - + " Output:[\"_col0\",\"_col1\",\"_col2\"]\n" - + " Filter Operator [FIL_92] (rows=12 width=261)\n" - + " predicate:s_store_sk is not null\n" - + " TableScan [TS_25] (rows=12 width=261)\n" - + " default@store,store,Tbl:COMPLETE,Col:NONE,Output:[\"s_store_sk\",\"s_store_id\",\"s_store_name\"]\n" - + " <-Map Join Operator [MAPJOIN_97] (rows=3833816 width=133)\n" - + " Conds:MERGEJOIN_96._col1=RS_35._col0(Inner),HybridGraceHashJoin:true,Output:[\"_col3\",\"_col5\",\"_col10\",\"_col11\",\"_col13\",\"_col18\",\"_col19\"]\n" - + " <-Map 11 [BROADCAST_EDGE]\n" - + " BROADCAST [RS_35]\n" - + " PartitionCols:_col0\n" - + " Select Operator [SEL_24] (rows=18000 width=279)\n" - + " Output:[\"_col0\",\"_col1\",\"_col2\"]\n" - + " Filter Operator [FIL_91] (rows=18000 width=279)\n" - + " predicate:i_item_sk is not null\n" - + " TableScan [TS_22] (rows=18000 width=279)\n" - + " default@item,item,Tbl:COMPLETE,Col:NONE,Output:[\"i_item_sk\",\"i_item_id\",\"i_item_desc\"]\n" - + " <-Merge Join Operator [MERGEJOIN_96] (rows=3485288 width=133)\n" - + " Conds:RS_31._col1, _col2, _col4=RS_32._col1, _col2, _col3(Inner),Output:[\"_col1\",\"_col3\",\"_col5\",\"_col10\",\"_col11\",\"_col13\"]\n" - + " <-Map 6 [SIMPLE_EDGE]\n" - + " SHUFFLE [RS_31]\n" - + " PartitionCols:_col1, _col2, _col4\n" - + " Map Join Operator [MAPJOIN_94] (rows=3168444 width=133)\n" - + " Conds:SEL_8._col0=RS_29._col0(Inner),HybridGraceHashJoin:true,Output:[\"_col1\",\"_col2\",\"_col3\",\"_col4\",\"_col5\"]\n" - + " <-Map 8 [BROADCAST_EDGE]\n" - + " BROADCAST [RS_29]\n" - + " PartitionCols:_col0\n" - + " Select Operator [SEL_11] (rows=18262 width=140)\n" - + " Output:[\"_col0\"]\n" - + " Filter Operator [FIL_88] (rows=18262 width=140)\n" - + " predicate:((d_moy = 4) and (d_year = 1998) and d_date_sk is not null)\n" - + " TableScan [TS_9] (rows=73049 width=140)\n" - + " default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:[\"d_date_sk\",\"d_year\",\"d_moy\"]\n" - + " <-Select Operator [SEL_8] (rows=2880404 width=133)\n" - + " Output:[\"_col0\",\"_col1\",\"_col2\",\"_col3\",\"_col4\",\"_col5\"]\n" - + " Filter Operator [FIL_87] (rows=2880404 width=133)\n" - + " predicate:(ss_item_sk is not null and ss_customer_sk is not null and ss_ticket_number is not null and ss_sold_date_sk is not null and ss_store_sk is not null)\n" - + " TableScan [TS_6] (rows=2880404 width=133)\n" - + " default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:[\"ss_sold_date_sk\",\"ss_item_sk\",\"ss_customer_sk\",\"ss_store_sk\",\"ss_ticket_number\",\"ss_net_profit\"]\n" - + " <-Map 9 [SIMPLE_EDGE]\n" - + " SHUFFLE [RS_32]\n" - + " PartitionCols:_col1, _col2, _col3\n" - + " Map Join Operator [MAPJOIN_95] (rows=316265 width=112)\n" - + " Conds:SEL_14._col0=RS_19._col0(Inner),HybridGraceHashJoin:true,Output:[\"_col1\",\"_col2\",\"_col3\",\"_col4\"]\n" - + " <-Map 10 [BROADCAST_EDGE]\n" - + " BROADCAST [RS_19]\n" - + " PartitionCols:_col0\n" - + " Select Operator [SEL_17] (rows=4058 width=140)\n" - + " Output:[\"_col0\"]\n" - + " Filter Operator [FIL_90] (rows=4058 width=140)\n" - + " predicate:(d_moy BETWEEN 4 AND 10 and (d_year = 1998) and d_date_sk is not null)\n" - + " TableScan [TS_15] (rows=73049 width=140)\n" - + " default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:[\"d_date_sk\",\"d_year\",\"d_moy\"]\n" - + " <-Select Operator [SEL_14] (rows=287514 width=112)\n" - + " Output:[\"_col0\",\"_col1\",\"_col2\",\"_col3\",\"_col4\"]\n" - + " Filter Operator [FIL_89] (rows=287514 width=112)\n" - + " predicate:(sr_item_sk is not null and sr_customer_sk is not null and sr_ticket_number is not null and sr_returned_date_sk is not null)\n" - + " TableScan [TS_12] (rows=287514 width=112)\n" - + " default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:[\"sr_returned_date_sk\",\"sr_item_sk\",\"sr_customer_sk\",\"sr_ticket_number\",\"sr_net_loss\"]\n" - + "\n" + "", content); + """ + Plan optimized by CBO. + + Vertex dependency in root stage + Map 1 <- Map 5 (BROADCAST_EDGE) + Map 6 <- Map 8 (BROADCAST_EDGE) + Map 9 <- Map 10 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 7 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Reducer 7 <- Map 11 (BROADCAST_EDGE), Map 12 (BROADCAST_EDGE), Map 6 (SIMPLE_EDGE), Map 9 (SIMPLE_EDGE) + + Stage-0 + Fetch Operator + limit:100 + Stage-1 + Reducer 4 + File Output Operator [FS_55] + Limit [LIM_54] (rows=100 width=133) + Number of rows:100 + Select Operator [SEL_53] (rows=2319458 width=133) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"] + <-Reducer 3 [SIMPLE_EDGE] + SHUFFLE [RS_52] + Group By Operator [GBY_50] (rows=2319458 width=133) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(VALUE._col0)","sum(VALUE._col1)","sum(VALUE._col2)"],keys:KEY._col0, KEY._col1, KEY._col2, KEY._col3 + <-Reducer 2 [SIMPLE_EDGE] + SHUFFLE [RS_49] + PartitionCols:_col0, _col1, _col2, _col3 + Group By Operator [GBY_48] (rows=4638916 width=133) + Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6"],aggregations:["sum(_col18)","sum(_col26)","sum(_col3)"],keys:_col11, _col12, _col8, _col9 + Merge Join Operator [MERGEJOIN_99] (rows=4638916 width=133) + Conds:RS_44._col1, _col2=RS_45._col17, _col16(Inner),Output:["_col3","_col8","_col9","_col11","_col12","_col18","_col26"] + <-Map 1 [SIMPLE_EDGE] + SHUFFLE [RS_44] + PartitionCols:_col1, _col2 + Map Join Operator [MAPJOIN_93] (rows=1585702 width=204) + Conds:SEL_2._col0=RS_42._col0(Inner),HybridGraceHashJoin:true,Output:["_col1","_col2","_col3"] + <-Map 5 [BROADCAST_EDGE] + BROADCAST [RS_42] + PartitionCols:_col0 + Select Operator [SEL_5] (rows=4058 width=140) + Output:["_col0"] + Filter Operator [FIL_86] (rows=4058 width=140) + predicate:(d_moy BETWEEN 4 AND 10 and (d_year = 1998) and d_date_sk is not null) + TableScan [TS_3] (rows=73049 width=140) + default@date_dim,d3,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + <-Select Operator [SEL_2] (rows=1441548 width=204) + Output:["_col0","_col1","_col2","_col3"] + Filter Operator [FIL_85] (rows=1441548 width=204) + predicate:(cs_bill_customer_sk is not null and cs_item_sk is not null and cs_sold_date_sk is not null) + TableScan [TS_0] (rows=1441548 width=204) + default@catalog_sales,catalog_sales,Tbl:COMPLETE,Col:NONE,Output:["cs_sold_date_sk","cs_bill_customer_sk","cs_item_sk","cs_net_profit"] + <-Reducer 7 [SIMPLE_EDGE] + SHUFFLE [RS_45] + PartitionCols:_col17, _col16 + Select Operator [SEL_40] (rows=4217197 width=133) + Output:["_col1","_col2","_col4","_col5","_col11","_col16","_col17","_col19"] + Map Join Operator [MAPJOIN_98] (rows=4217197 width=133) + Conds:MAPJOIN_97._col3=RS_38._col0(Inner),HybridGraceHashJoin:true,Output:["_col5","_col10","_col11","_col13","_col18","_col19","_col21","_col22"] + <-Map 12 [BROADCAST_EDGE] + BROADCAST [RS_38] + PartitionCols:_col0 + Select Operator [SEL_27] (rows=12 width=261) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_92] (rows=12 width=261) + predicate:s_store_sk is not null + TableScan [TS_25] (rows=12 width=261) + default@store,store,Tbl:COMPLETE,Col:NONE,Output:["s_store_sk","s_store_id","s_store_name"] + <-Map Join Operator [MAPJOIN_97] (rows=3833816 width=133) + Conds:MERGEJOIN_96._col1=RS_35._col0(Inner),HybridGraceHashJoin:true,Output:["_col3","_col5","_col10","_col11","_col13","_col18","_col19"] + <-Map 11 [BROADCAST_EDGE] + BROADCAST [RS_35] + PartitionCols:_col0 + Select Operator [SEL_24] (rows=18000 width=279) + Output:["_col0","_col1","_col2"] + Filter Operator [FIL_91] (rows=18000 width=279) + predicate:i_item_sk is not null + TableScan [TS_22] (rows=18000 width=279) + default@item,item,Tbl:COMPLETE,Col:NONE,Output:["i_item_sk","i_item_id","i_item_desc"] + <-Merge Join Operator [MERGEJOIN_96] (rows=3485288 width=133) + Conds:RS_31._col1, _col2, _col4=RS_32._col1, _col2, _col3(Inner),Output:["_col1","_col3","_col5","_col10","_col11","_col13"] + <-Map 6 [SIMPLE_EDGE] + SHUFFLE [RS_31] + PartitionCols:_col1, _col2, _col4 + Map Join Operator [MAPJOIN_94] (rows=3168444 width=133) + Conds:SEL_8._col0=RS_29._col0(Inner),HybridGraceHashJoin:true,Output:["_col1","_col2","_col3","_col4","_col5"] + <-Map 8 [BROADCAST_EDGE] + BROADCAST [RS_29] + PartitionCols:_col0 + Select Operator [SEL_11] (rows=18262 width=140) + Output:["_col0"] + Filter Operator [FIL_88] (rows=18262 width=140) + predicate:((d_moy = 4) and (d_year = 1998) and d_date_sk is not null) + TableScan [TS_9] (rows=73049 width=140) + default@date_dim,d1,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + <-Select Operator [SEL_8] (rows=2880404 width=133) + Output:["_col0","_col1","_col2","_col3","_col4","_col5"] + Filter Operator [FIL_87] (rows=2880404 width=133) + predicate:(ss_item_sk is not null and ss_customer_sk is not null and ss_ticket_number is not null and ss_sold_date_sk is not null and ss_store_sk is not null) + TableScan [TS_6] (rows=2880404 width=133) + default@store_sales,store_sales,Tbl:COMPLETE,Col:NONE,Output:["ss_sold_date_sk","ss_item_sk","ss_customer_sk","ss_store_sk","ss_ticket_number","ss_net_profit"] + <-Map 9 [SIMPLE_EDGE] + SHUFFLE [RS_32] + PartitionCols:_col1, _col2, _col3 + Map Join Operator [MAPJOIN_95] (rows=316265 width=112) + Conds:SEL_14._col0=RS_19._col0(Inner),HybridGraceHashJoin:true,Output:["_col1","_col2","_col3","_col4"] + <-Map 10 [BROADCAST_EDGE] + BROADCAST [RS_19] + PartitionCols:_col0 + Select Operator [SEL_17] (rows=4058 width=140) + Output:["_col0"] + Filter Operator [FIL_90] (rows=4058 width=140) + predicate:(d_moy BETWEEN 4 AND 10 and (d_year = 1998) and d_date_sk is not null) + TableScan [TS_15] (rows=73049 width=140) + default@date_dim,d2,Tbl:COMPLETE,Col:NONE,Output:["d_date_sk","d_year","d_moy"] + <-Select Operator [SEL_14] (rows=287514 width=112) + Output:["_col0","_col1","_col2","_col3","_col4"] + Filter Operator [FIL_89] (rows=287514 width=112) + predicate:(sr_item_sk is not null and sr_customer_sk is not null and sr_ticket_number is not null and sr_returned_date_sk is not null) + TableScan [TS_12] (rows=287514 width=112) + default@store_returns,store_returns,Tbl:COMPLETE,Col:NONE,Output:["sr_returned_date_sk","sr_item_sk","sr_customer_sk","sr_ticket_number","sr_net_loss"] + + """, content); } } diff --git a/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestOp.java b/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestOp.java index 2ec470b37191..5670ffa99b57 100644 --- a/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestOp.java +++ b/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestOp.java @@ -42,11 +42,14 @@ public void setUp() throws Exception { @Test public void testInlineJoinOpJsonHandling() throws Exception { - String jsonString = "{" + - "\"input vertices:\":{\"a\":\"AVERTEX\"}," + "\"condition map:\": [" + - "{\"c1\": \"{\\\"type\\\": \\\"type\\\", \\\"left\\\": \\\"left\\\", " + - "\\\"right\\\": \\\"right\\\"}\"}]," + - "\"keys:\":{\"left\":\"AKEY\", \"right\":\"BKEY\"}}"; + String jsonString = """ + {\ + "input vertices:":{"a":"AVERTEX"},\ + "condition map:": [\ + {"c1": "{\\"type\\": \\"type\\", \\"left\\": \\"left\\", \ + \\"right\\": \\"right\\"}"}],\ + "keys:":{"left":"AKEY", "right":"BKEY"}}\ + """; JSONObject mapJoin = new JSONObject(jsonString); Vertex vertexB = new Vertex("vertex-b", null, null, tezJsonParser); diff --git a/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestStage.java b/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestStage.java index f1660c0724bf..1415ee59fb08 100644 --- a/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestStage.java +++ b/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestStage.java @@ -87,8 +87,10 @@ public void testAddDependencyRoot() throws Exception { @Test public void testExtractVertexNonTez() throws Exception { - String jsonString = "{\"OperatorName\":{\"a\":\"A\",\"b\":\"B\"}," + - "\"attr1\":\"ATTR1\"}"; + String jsonString = """ + {"OperatorName":{"a":"A","b":"B"},\ + "attr1":"ATTR1"}\ + """; JSONObject object = new JSONObject(jsonString); uut.extractVertex(object); @@ -110,9 +112,11 @@ public void testExtractVertexTezNoEdges() throws Exception { @Test public void testExtractVertexTezWithOneEdge() throws Exception { - String jsonString = "{\"Tez\":{\"a\":\"A\"," + - "\"Vertices:\":{\"v1\":{},\"v2\":{}}," + - "\"Edges:\":{\"v2\":{\"parent\":\"v1\",\"type\":\"TYPE\"}}}}"; + String jsonString = """ + {"Tez":{"a":"A",\ + "Vertices:":{"v1":{},"v2":{}},\ + "Edges:":{"v2":{"parent":"v1","type":"TYPE"}}}}\ + """; JSONObject object = new JSONObject(jsonString); uut.extractVertex(object); @@ -130,10 +134,12 @@ public void testExtractVertexTezWithOneEdge() throws Exception { @Test public void testExtractVertexTezWithOneToManyEdge() throws Exception { - String jsonString = "{\"Tez\":{\"a\":\"A\"," + - "\"Vertices:\":{\"v1\":{},\"v2\":{},\"v3\":{}}," + - "\"Edges:\":{\"v1\":[{\"parent\":\"v2\",\"type\":\"TYPE1\"}," + - "{\"parent\":\"v3\",\"type\":\"TYPE2\"}]}}}"; + String jsonString = """ + {"Tez":{"a":"A",\ + "Vertices:":{"v1":{},"v2":{},"v3":{}},\ + "Edges:":{"v1":[{"parent":"v2","type":"TYPE1"},\ + {"parent":"v3","type":"TYPE2"}]}}}\ + """; JSONObject object = new JSONObject(jsonString); uut.extractVertex(object); @@ -177,8 +183,10 @@ public void testExtractOpSimple() throws Exception { @Test public void testExtract() throws Exception { - String jsonString = "{\"b\":{\"b2\":\"B2\",\"b1\":\"B1\"}," + - "\"Processor Tree:\":{\"a1\":{\"t1\":\"T1\"}}}"; + String jsonString = """ + {"b":{"b2":"B2","b1":"B1"},\ + "Processor Tree:":{"a1":{"t1":"T1"}}}\ + """; JSONObject object = new JSONObject(jsonString); Op result = uut.extractOp("op-name", object); diff --git a/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestVertex.java b/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestVertex.java index 38250ddeb34f..d950ebb3d651 100644 --- a/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestVertex.java +++ b/common/src/test/org/apache/hadoop/hive/common/jsonexplain/TestVertex.java @@ -75,8 +75,10 @@ public void testExtractOpNoChildrenOperatorId() throws Exception { @Test public void testExtractOpOneChild() throws Exception { - String jsonString = "{\"opName\":{\"children\":{\"childName\":" + - "{\"OperatorId:\":\"child-operator-id\"}}}}"; + String jsonString = """ + {"opName":{"children":{"childName":\ + {"OperatorId:":"child-operator-id"}}}}\ + """; JSONObject operator = new JSONObject(jsonString); Vertex uut = new Vertex("name", null, null, tezJsonParser); @@ -90,9 +92,11 @@ public void testExtractOpOneChild() throws Exception { @Test public void testExtractOpMultipleChildren() throws Exception { - String jsonString = "{\"opName\":{\"children\":[" + - "{\"childName1\":{\"OperatorId:\":\"child-operator-id1\"}}," + - "{\"childName2\":{\"OperatorId:\":\"child-operator-id2\"}}]}}"; + String jsonString = """ + {"opName":{"children":[\ + {"childName1":{"OperatorId:":"child-operator-id1"}},\ + {"childName2":{"OperatorId:":"child-operator-id2"}}]}}\ + """; JSONObject operator = new JSONObject(jsonString); Vertex uut = new Vertex("name", null, null, tezJsonParser); diff --git a/common/src/test/org/apache/hadoop/hive/common/jsonexplain/tez/TestTezJsonParser.java b/common/src/test/org/apache/hadoop/hive/common/jsonexplain/tez/TestTezJsonParser.java index b1618d6fcfc5..6d6276777a92 100644 --- a/common/src/test/org/apache/hadoop/hive/common/jsonexplain/tez/TestTezJsonParser.java +++ b/common/src/test/org/apache/hadoop/hive/common/jsonexplain/tez/TestTezJsonParser.java @@ -35,8 +35,10 @@ public void setUp() throws Exception { @Test public void testExtractStagesAndPlans() throws Exception { - String jsonString = "{\"STAGE DEPENDENCIES\":{\"s1\":{\"ROOT STAGE\":\"\"}," + - "\"s2\":{\"DEPENDENT STAGES\":\"s1\"}},\"STAGE PLANS\":{}}"; + String jsonString = """ + {"STAGE DEPENDENCIES":{"s1":{"ROOT STAGE":""},\ + "s2":{"DEPENDENT STAGES":"s1"}},"STAGE PLANS":{}}\ + """; JSONObject input = new JSONObject(jsonString); uut.extractStagesAndPlans(input); diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java index 0e8e4c35fc5d..a9b3c089c3f2 100644 --- a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java +++ b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java @@ -229,15 +229,17 @@ public void testAdditionalConfigFiles() throws Exception{ FileUtils.writeStringToFile(fileHiveSite, testHiveSiteString); - String testLdapString = "\n" + - "\n" + - "\n" + - " \n" + - " hive.server2.authentication.ldap.Domain\n" + - " b.com\n" + - "\n" + - "\n" + - ""; + String testLdapString = """ + + + + + hive.server2.authentication.ldap.Domain + b.com + + + \ + """; String newFileName = parFolder+"/ldap-site.xml"; diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConfVarsValidate.java b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConfVarsValidate.java index 42736ddb3d6f..2370ebc7a16f 100644 --- a/common/src/test/org/apache/hadoop/hive/conf/TestHiveConfVarsValidate.java +++ b/common/src/test/org/apache/hadoop/hive/conf/TestHiveConfVarsValidate.java @@ -69,8 +69,10 @@ public static Collection generateParameters() { list.add(new Object[] { HIVE_DATETIME_RESOLVER_STYLE, "smart", null}); list.add(new Object[] { HIVE_DATETIME_RESOLVER_STYLE, "strict", null}); list.add(new Object[] { HIVE_DATETIME_RESOLVER_STYLE, "lenient", null}); - list.add(new Object[] { HIVE_DATETIME_RESOLVER_STYLE, "OTHER", "Invalid value.. expects one of [smart, strict, " + - "lenient]" }); + list.add(new Object[] { HIVE_DATETIME_RESOLVER_STYLE, "OTHER", """ + Invalid value.. expects one of [smart, strict, \ + lenient]\ + """ }); return list; } diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestSystemVariables.java b/common/src/test/org/apache/hadoop/hive/conf/TestSystemVariables.java index 364102040093..c577d9d8f477 100644 --- a/common/src/test/org/apache/hadoop/hive/conf/TestSystemVariables.java +++ b/common/src/test/org/apache/hadoop/hive/conf/TestSystemVariables.java @@ -31,7 +31,7 @@ public class TestSystemVariables { public static final String SYSTEM = "system"; private String makeVarName(String prefix, String value) { - return String.format("${%s:%s}", prefix, value); + return "${%s:%s}".formatted(prefix, value); } @Test diff --git a/contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/TypedBytesOutput.java b/contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/TypedBytesOutput.java index 7f6f75603dfa..bde2a8104688 100644 --- a/contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/TypedBytesOutput.java +++ b/contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/TypedBytesOutput.java @@ -342,7 +342,7 @@ public void writeEndOfRecord() throws IOException { } /** - * Writes a NULL type marker to the output. + * Writes a 
NULL
type marker to the output. * * @throws IOException */ diff --git a/data/conf/hive-site.xml b/data/conf/hive-site.xml index cd716824bb9f..3cbf03984876 100644 --- a/data/conf/hive-site.xml +++ b/data/conf/hive-site.xml @@ -130,6 +130,18 @@ + + yarn.app.mapreduce.am.command-opts + --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.regex=ALL-UNNAMED + + + mapreduce.map.java.opts + --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED + + + mapreduce.reduce.java.opts + --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED + hive.jar.path ${maven.local.repository}/org/apache/hive/hive-exec/${hive.version}/hive-exec-${hive.version}.jar diff --git a/data/conf/iceberg/llap/hive-site.xml b/data/conf/iceberg/llap/hive-site.xml index a4bdb6266b56..fce125eee00a 100644 --- a/data/conf/iceberg/llap/hive-site.xml +++ b/data/conf/iceberg/llap/hive-site.xml @@ -264,7 +264,7 @@ hive.tez.java.opts - -Dlog4j.configurationFile=tez-container-log4j2.properties -Dtez.container.log.level=INFO -Dtez.container.root.logger=CLA + -Dlog4j.configurationFile=tez-container-log4j2.properties -Dtez.container.log.level=INFO -Dtez.container.root.logger=CLA --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED diff --git a/data/conf/iceberg/llap/tez-site.xml b/data/conf/iceberg/llap/tez-site.xml index 754380463589..f763a593c2f7 100644 --- a/data/conf/iceberg/llap/tez-site.xml +++ b/data/conf/iceberg/llap/tez-site.xml @@ -32,4 +32,8 @@ tez.counters.max 1024 + + tez.am.launch.cmd-opts + --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED + diff --git a/data/conf/iceberg/tez/hive-site.xml b/data/conf/iceberg/tez/hive-site.xml index 19984c7ca509..7247517a0613 100644 --- a/data/conf/iceberg/tez/hive-site.xml +++ b/data/conf/iceberg/tez/hive-site.xml @@ -250,7 +250,7 @@ hive.tez.java.opts - -Dlog4j.configurationFile=tez-container-log4j2.properties -Dtez.container.log.level=INFO -Dtez.container.root.logger=CLA + -Dlog4j.configurationFile=tez-container-log4j2.properties -Dtez.container.log.level=INFO -Dtez.container.root.logger=CLA --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util.regex=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED diff --git a/data/conf/iceberg/tez/tez-site.xml b/data/conf/iceberg/tez/tez-site.xml index f0f778452799..05cacd0a6ed8 100644 --- a/data/conf/iceberg/tez/tez-site.xml +++ b/data/conf/iceberg/tez/tez-site.xml @@ -32,4 +32,8 @@ tez.counters.max 1024 + + tez.am.launch.cmd-opts + --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util.regex=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED + diff --git a/data/conf/llap/hive-site.xml b/data/conf/llap/hive-site.xml index 4f5aedca77d2..e62f910b9657 100644 --- a/data/conf/llap/hive-site.xml +++ b/data/conf/llap/hive-site.xml @@ -49,7 +49,7 @@ hive.tez.container.size - 128 + 256 @@ -261,7 +261,7 @@ hive.tez.java.opts - -Dlog4j.configurationFile=tez-container-log4j2.properties -Dtez.container.log.level=INFO -Dtez.container.root.logger=CLA + -Dlog4j.configurationFile=tez-container-log4j2.properties -Dtez.container.log.level=INFO -Dtez.container.root.logger=CLA --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util.regex=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED diff --git a/data/conf/llap/tez-site.xml b/data/conf/llap/tez-site.xml index 143d8a012921..f1798c4cab5d 100644 --- a/data/conf/llap/tez-site.xml +++ b/data/conf/llap/tez-site.xml @@ -24,7 +24,7 @@ tez.am.resource.memory.mb - 128 + 256 tez.runtime.io.sort.mb @@ -38,6 +38,10 @@ tez.runtime.shuffle.fetch.buffer.percent 0.4 + + tez.am.launch.cmd-opts + --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util.regex=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED + diff --git a/data/conf/mr/hive-site.xml b/data/conf/mr/hive-site.xml index 003a661d9b6f..81bf95525887 100644 --- a/data/conf/mr/hive-site.xml +++ b/data/conf/mr/hive-site.xml @@ -151,6 +151,19 @@ Post Execute Hook for Tests + + yarn.app.mapreduce.am.command-opts + --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED + + + mapreduce.map.java.opts + --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.regex=ALL-UNNAMED + + + mapreduce.reduce.java.opts + --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.regex=ALL-UNNAMED + + hive.support.concurrency true diff --git a/data/conf/tez/hive-site.xml b/data/conf/tez/hive-site.xml index 6c969616a720..74c03eb50be5 100644 --- a/data/conf/tez/hive-site.xml +++ b/data/conf/tez/hive-site.xml @@ -40,7 +40,7 @@ hive.tez.container.size - 128 + 256 @@ -251,7 +251,7 @@ hive.tez.java.opts - -Dlog4j.configurationFile=tez-container-log4j2.properties -Dtez.container.log.level=INFO -Dtez.container.root.logger=CLA + -Dlog4j.configurationFile=tez-container-log4j2.properties -Dtez.container.log.level=INFO -Dtez.container.root.logger=CLA --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util.regex=ALL-UNNAMED diff --git a/data/conf/tez/tez-site.xml b/data/conf/tez/tez-site.xml index 104f40367b73..65193aec166b 100644 --- a/data/conf/tez/tez-site.xml +++ b/data/conf/tez/tez-site.xml @@ -36,4 +36,8 @@ tez.am.dag.scheduler.class org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled + + tez.am.launch.cmd-opts + --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util.regex=ALL-UNNAMED + diff --git a/druid-handler/pom.xml b/druid-handler/pom.xml index 0c0e2f96e7ad..3b714170dd13 100644 --- a/druid-handler/pom.xml +++ b/druid-handler/pom.xml @@ -68,6 +68,10 @@ com.fasterxml.jackson.core jackson-databind + + org.apache.hadoop + hadoop-client-api + diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java index 16dfdc2e6265..172caf77a6f2 100644 --- a/druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java +++ b/druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java @@ -19,7 +19,6 @@ import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyObject; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -86,7 +85,7 @@ import org.apache.druid.query.topn.TopNResultValue; import org.junit.rules.ExpectedException; -/** +/**Z * Basic tests for Druid SerDe. The examples are taken from Druid 0.9.1.1 * documentation. */ @@ -664,7 +663,7 @@ private void deserializeQueryResults(DruidSerDe serDe, HttpClient httpClient = mock(HttpClient.class); SettableFuture futureResult = SettableFuture.create(); futureResult.set(new ByteArrayInputStream(resultString)); - when(httpClient.go(anyObject(), any(HttpResponseHandler.class))).thenReturn(futureResult); + when(httpClient.go(any(), any(HttpResponseHandler.class))).thenReturn(futureResult); DruidQueryRecordReader reader = DruidQueryBasedInputFormat.getDruidQueryReader(queryType); final HiveDruidSplit split = new HiveDruidSplit(jsonQuery, new Path("empty"), new String[]{"testing_host"}); @@ -693,7 +692,7 @@ private void deserializeQueryResults(DruidSerDe serDe, // Check mapreduce path futureResult = SettableFuture.create(); futureResult.set(new ByteArrayInputStream(resultString)); - when(httpClient.go(anyObject(), any(HttpResponseHandler.class))).thenReturn(futureResult); + when(httpClient.go(any(), any(HttpResponseHandler.class))).thenReturn(futureResult); reader = DruidQueryBasedInputFormat.getDruidQueryReader(queryType); reader.initialize(split, DruidStorageHandlerUtils.JSON_MAPPER, DruidStorageHandlerUtils.SMILE_MAPPER, httpClient, conf); diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java index 749409a03620..47c216d0b90b 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java @@ -218,7 +218,8 @@ private int callHCatCli(String[] args) throws Exception { argsList.addAll(Arrays.asList(args)); ProcessBuilder builder = new ProcessBuilder().command(argsList.toArray(new String[] {})); builder.environment().put("CLASSPATH", System.getProperty("java.class.path")); - + builder.environment().put("JAVA_TOOL_OPTIONS", "--add-opens=java.base/java.net=ALL-UNNAMED"); + Process p = builder.start(); String line; diff --git a/hcatalog/webhcat/svr/pom.xml b/hcatalog/webhcat/svr/pom.xml index 32d20d9463f9..dc1e5b6e59f5 100644 --- a/hcatalog/webhcat/svr/pom.xml +++ b/hcatalog/webhcat/svr/pom.xml @@ -84,14 +84,14 @@ - com.sun.jersey.contribs - wadl-resourcedoc-doclet - - - com.sun.jersey - jersey-server - - + org.glassfish.jersey.ext + jersey-wadl-doclet + ${jersey-wadl-doclet.version} + + + org.glassfish.jaxb + jaxb-runtime + ${jaxb-runtime.version} org.apache.commons @@ -258,26 +258,21 @@ ${project.build.sourceEncoding} true public - com.sun.jersey.wadl.resourcedoc.ResourceDoclet + org.glassfish.jersey.wadl.doclet.ResourceDoclet - com.sun.jersey.contribs - wadl-resourcedoc-doclet - ${wadl-resourcedoc-doclet.version} + org.glassfish.jersey.ext + jersey-wadl-doclet + ${jersey-wadl-doclet.version} - com.sun.jersey - jersey-server - ${jersey.version} - - - xerces - xercesImpl - ${xerces.version} + org.glassfish.jaxb + jaxb-runtime + ${jaxb-runtime.version} diff --git a/hplsql/pom.xml b/hplsql/pom.xml index c080090b6495..3aea09791666 100644 --- a/hplsql/pom.xml +++ b/hplsql/pom.xml @@ -44,6 +44,12 @@ org.apache.hive hive-exec ${project.version} + + + org.apache.hadoop + hadoop-client-api + + org.antlr diff --git a/iceberg/iceberg-handler/pom.xml b/iceberg/iceberg-handler/pom.xml index 17f0ace1e770..7b566fc22b13 100644 --- a/iceberg/iceberg-handler/pom.xml +++ b/iceberg/iceberg-handler/pom.xml @@ -41,6 +41,12 @@ org.apache.hive hive-exec + + + org.apache.hadoop + hadoop-client-api + + org.apache.hive diff --git a/iceberg/pom.xml b/iceberg/pom.xml index eedbd7e56923..4dd1ba3b32d4 100644 --- a/iceberg/pom.xml +++ b/iceberg/pom.xml @@ -255,7 +255,6 @@ ${validate.skip} - ,\# diff --git a/itests/hive-blobstore/pom.xml b/itests/hive-blobstore/pom.xml index ff4b786a4ba6..74f35f3a0e5f 100644 --- a/itests/hive-blobstore/pom.xml +++ b/itests/hive-blobstore/pom.xml @@ -298,6 +298,7 @@ org.apache.maven.plugins maven-surefire-plugin + ${maven.test.jvm.args} ${test.blobstore.path} diff --git a/itests/hive-minikdc/pom.xml b/itests/hive-minikdc/pom.xml index 827b1e5b2c29..066e67789a76 100644 --- a/itests/hive-minikdc/pom.xml +++ b/itests/hive-minikdc/pom.xml @@ -327,6 +327,7 @@ org.apache.maven.plugins maven-surefire-plugin + ${maven.test.jvm.args} ${exclude.tests} diff --git a/itests/hive-minikdc/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCleanerWithSecureDFS.java b/itests/hive-minikdc/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCleanerWithSecureDFS.java index ecf8472ea994..2667a83d43dc 100644 --- a/itests/hive-minikdc/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCleanerWithSecureDFS.java +++ b/itests/hive-minikdc/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCleanerWithSecureDFS.java @@ -32,8 +32,7 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import org.mockito.internal.util.reflection.FieldSetter; - +import org.apache.hive.common.util.ReflectionUtil; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; @@ -163,7 +162,7 @@ public void testLeakAfterHistoryException() throws Exception { } cleaner.setConf(cleanerConf); cleaner.init(new AtomicBoolean(true)); - FieldSetter.setField(cleaner, MetaStoreCompactorThread.class.getDeclaredField("txnHandler"), txnHandler); + ReflectionUtil.setField(cleaner, MetaStoreCompactorThread.class.getDeclaredField("txnHandler"), txnHandler); Runtime.getRuntime().gc(); long startMem = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); cleaner.run(); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java index afe91493d029..e7b8594af9f1 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java @@ -62,7 +62,7 @@ import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; -import org.mockito.internal.util.reflection.FieldSetter; +import org.apache.hive.common.util.ReflectionUtil; import java.util.ArrayList; import java.util.Arrays; @@ -150,7 +150,7 @@ public void testHeartbeatShutdownOnFailedCompaction() throws Exception { Worker worker = Mockito.spy(new Worker()); worker.setConf(conf); worker.init(new AtomicBoolean(true)); - FieldSetter.setField(worker, RemoteCompactorThread.class.getDeclaredField("msc"), mockedClient); + ReflectionUtil.setField(worker, RemoteCompactorThread.class.getDeclaredField("msc"), mockedClient); worker.run(); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java index 148a7cb32255..c9f0504f7a78 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java @@ -80,7 +80,7 @@ import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; -import org.mockito.internal.util.reflection.FieldSetter; +import org.apache.hive.common.util.ReflectionUtil; import static org.apache.hadoop.hive.ql.TxnCommandsBaseForTests.runWorker; import static org.apache.hadoop.hive.ql.txn.compactor.TestCompactor.execSelectAndDumpData; @@ -731,7 +731,7 @@ public void testMinorCompactionShouldBeRefusedOnTablesWithOriginalFiles() throws Initiator initiator = new Initiator(); initiator.setConf(conf); initiator.init(new AtomicBoolean(true)); - FieldSetter.setField(initiator, MetaStoreCompactorThread.class.getDeclaredField("txnHandler"), mockedHandler); + ReflectionUtil.setField(initiator, MetaStoreCompactorThread.class.getDeclaredField("txnHandler"), mockedHandler); //Run initiator and capture compaction requests initiator.run(); @@ -800,7 +800,7 @@ public void testMinorCompactionShouldBeRefusedOnTablesWithRawData() throws Excep Initiator initiator = new Initiator(); initiator.setConf(conf); initiator.init(new AtomicBoolean(true)); - FieldSetter.setField(initiator, MetaStoreCompactorThread.class.getDeclaredField("txnHandler"), mockedHandler); + ReflectionUtil.setField(initiator, MetaStoreCompactorThread.class.getDeclaredField("txnHandler"), mockedHandler); //Run initiator and capture compaction requests initiator.run(); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java index 6f8110aa9d59..243144e1cbb6 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java @@ -25,6 +25,8 @@ import static org.junit.Assert.fail; import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; import java.lang.reflect.Constructor; import java.lang.reflect.Field; import java.lang.reflect.Modifier; @@ -1234,10 +1236,10 @@ private void setReflectionUtilCache() { constructorCacheField = ReflectionUtil.class.getDeclaredField("CONSTRUCTOR_CACHE"); if (constructorCacheField != null) { constructorCacheField.setAccessible(true); - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(constructorCacheField, constructorCacheField.getModifiers() - & ~Modifier.FINAL); + VarHandle modifiersHandle = MethodHandles.privateLookupIn(Field.class, MethodHandles.lookup()) + .findVarHandle(Field.class, "modifiers", int.class); + int modifiers = constructorCacheField.getModifiers(); + modifiersHandle.set(constructorCacheField, modifiers & ~Modifier.FINAL); tmp = CacheBuilder.newBuilder().expireAfterAccess(5, TimeUnit.SECONDS).concurrencyLevel(64) .weakKeys().weakValues().build(); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/auth/jwt/TestHttpJwtAuthentication.java b/itests/hive-unit/src/test/java/org/apache/hive/service/auth/jwt/TestHttpJwtAuthentication.java index 7aa7adb67e98..2b6f501ecf1e 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/auth/jwt/TestHttpJwtAuthentication.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/auth/jwt/TestHttpJwtAuthentication.java @@ -38,6 +38,8 @@ import org.junit.Test; import java.io.File; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.nio.charset.StandardCharsets; @@ -95,9 +97,10 @@ public static void makeEnvModifiable() throws Exception { private static void removeStaticFinalAndSetValue(Field field, Object value) throws Exception { field.setAccessible(true); - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + VarHandle modifiersHandle = MethodHandles.privateLookupIn(Field.class, MethodHandles.lookup()) + .findVarHandle(Field.class, "modifiers", int.class); + int modifiers = field.getModifiers(); + modifiersHandle.set(field, modifiers & ~Modifier.FINAL); field.set(null, value); } diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftHttpCLIServiceFeatures.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftHttpCLIServiceFeatures.java index f04db3635f31..c60c65563385 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftHttpCLIServiceFeatures.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftHttpCLIServiceFeatures.java @@ -67,7 +67,7 @@ import org.junit.BeforeClass; import org.junit.Test; import org.mockito.ArgumentCaptor; -import org.mockito.Matchers; +import static org.mockito.ArgumentMatchers.anyList; import org.mockito.Mockito; import com.google.common.base.Joiner; @@ -363,8 +363,8 @@ private void verifyForwardedHeaders(ArrayList headerIPs, String cmd) thr .forClass(HiveAuthzContext.class); verify(mockedAuthorizer).checkPrivileges(any(HiveOperationType.class), - Matchers.anyListOf(HivePrivilegeObject.class), - Matchers.anyListOf(HivePrivilegeObject.class), contextCapturer.capture()); + anyList(), + anyList(), contextCapturer.capture()); HiveAuthzContext context = contextCapturer.getValue(); System.err.println("Forwarded IP Addresses " + context.getForwardedAddresses()); diff --git a/itests/qtest-iceberg/pom.xml b/itests/qtest-iceberg/pom.xml index e34a7725097c..ad90b0d088ec 100644 --- a/itests/qtest-iceberg/pom.xml +++ b/itests/qtest-iceberg/pom.xml @@ -485,6 +485,7 @@ org.apache.maven.plugins maven-surefire-plugin + ${maven.test.jvm.args} ${test.conf.dir} ${basedir}/${hive.path.to.root}/conf diff --git a/itests/qtest-kudu/pom.xml b/itests/qtest-kudu/pom.xml index aaf704732992..c5341544570e 100644 --- a/itests/qtest-kudu/pom.xml +++ b/itests/qtest-kudu/pom.xml @@ -104,6 +104,15 @@ junit test + + org.apache.logging.log4j + log4j-1.2-api + + + org.apache.logging.log4j + log4j-core + ${log4j2.version} + @@ -386,6 +395,7 @@ org.apache.maven.plugins maven-surefire-plugin + ${maven.test.jvm.args} ${exclude.tests} diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml index b1724a888c4a..0de5cb34a529 100644 --- a/itests/qtest/pom.xml +++ b/itests/qtest/pom.xml @@ -622,6 +622,7 @@ org.apache.maven.plugins maven-surefire-plugin + ${maven.test.jvm.args} ${test.conf.dir} ${basedir}/${hive.path.to.root}/conf diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java index 3116193576af..fd0bbe2bfc5f 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QOutProcessor.java @@ -124,6 +124,7 @@ public String get() { "at org", "at sun", "at java", + "at jdk", "at junit", "LOCK_QUERYID:", "LOCK_TIME:", diff --git a/jdbc/src/test/org/apache/hive/jdbc/TestHiveBaseResultSet.java b/jdbc/src/test/org/apache/hive/jdbc/TestHiveBaseResultSet.java index bca26f336f30..53441990cd8f 100644 --- a/jdbc/src/test/org/apache/hive/jdbc/TestHiveBaseResultSet.java +++ b/jdbc/src/test/org/apache/hive/jdbc/TestHiveBaseResultSet.java @@ -32,7 +32,7 @@ import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; -import org.mockito.internal.util.reflection.FieldSetter; +import org.apache.hive.common.util.ReflectionUtil; /** * Test suite for {@link HiveBaseResultSet} class. @@ -259,7 +259,7 @@ public void testFindColumnUnqualified() throws Exception { resultSet.normalizedColumnNames = Arrays.asList("one", "two", "three"); Field executorField = HiveBaseResultSet.class.getDeclaredField("columnNameIndexCache"); - FieldSetter.setField(resultSet, executorField, new HashMap<>()); + ReflectionUtil.setField(resultSet, executorField, new HashMap<>()); when(resultSet.getSchema()).thenReturn(schema); when(resultSet.findColumn("one")).thenCallRealMethod(); @@ -290,7 +290,7 @@ public void testFindColumnQualified() throws Exception { resultSet.normalizedColumnNames = Arrays.asList("table.one", "table.two", "table.three"); Field executorField = HiveBaseResultSet.class.getDeclaredField("columnNameIndexCache"); - FieldSetter.setField(resultSet, executorField, new HashMap<>()); + ReflectionUtil.setField(resultSet, executorField, new HashMap<>()); when(resultSet.getSchema()).thenReturn(schema); when(resultSet.findColumn("one")).thenCallRealMethod(); @@ -322,7 +322,7 @@ public void testFindColumnUnknownColumn() throws Exception { resultSet.normalizedColumnNames = Arrays.asList("table.one"); Field executorField = HiveBaseResultSet.class.getDeclaredField("columnNameIndexCache"); - FieldSetter.setField(resultSet, executorField, new HashMap<>()); + ReflectionUtil.setField(resultSet, executorField, new HashMap<>()); when(resultSet.getSchema()).thenReturn(schema); when(resultSet.findColumn("zero")).thenCallRealMethod(); diff --git a/kafka-handler/src/java/org/apache/hadoop/hive/kafka/KafkaSerDe.java b/kafka-handler/src/java/org/apache/hadoop/hive/kafka/KafkaSerDe.java index 718b56fd4686..16d4bc459ede 100644 --- a/kafka-handler/src/java/org/apache/hadoop/hive/kafka/KafkaSerDe.java +++ b/kafka-handler/src/java/org/apache/hadoop/hive/kafka/KafkaSerDe.java @@ -264,7 +264,7 @@ void deserializeKWritable(KafkaWritable kafkaWritable, Object[] rowBoat) throws /** * Returns a view of input object inspector list between: - * 0 inclusive and the specified toIndex, exclusive. + * 
0
inclusive and the specified 
toIndex
, exclusive. */ private static final class SubStructObjectInspector extends StructObjectInspector { diff --git a/kudu-handler/pom.xml b/kudu-handler/pom.xml index c681762fce92..84e2bc0ea194 100644 --- a/kudu-handler/pom.xml +++ b/kudu-handler/pom.xml @@ -41,6 +41,12 @@ hive-exec ${project.version} provided + + + org.apache.hadoop + hadoop-client-api + + @@ -98,6 +104,15 @@ junit-vintage-engine test + + org.apache.logging.log4j + log4j-1.2-api + + + org.apache.logging.log4j + log4j-core + ${log4j2.version} + org.apache.kudu kudu-test-utils @@ -124,6 +139,7 @@ org.apache.maven.plugins maven-surefire-plugin + ${maven.test.jvm.args} ${exclude.tests} diff --git a/llap-common/src/test/org/apache/hadoop/hive/llap/AsyncResponseHandlerTest.java b/llap-common/src/test/org/apache/hadoop/hive/llap/AsyncResponseHandlerTest.java index 75c68d98132a..e639f31efbe9 100644 --- a/llap-common/src/test/org/apache/hadoop/hive/llap/AsyncResponseHandlerTest.java +++ b/llap-common/src/test/org/apache/hadoop/hive/llap/AsyncResponseHandlerTest.java @@ -26,7 +26,7 @@ import java.util.concurrent.TimeUnit; import static java.lang.Thread.sleep; -import static org.mockito.Matchers.any; +import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; diff --git a/llap-server/pom.xml b/llap-server/pom.xml index d90296d6023c..09d72d51cd9c 100644 --- a/llap-server/pom.xml +++ b/llap-server/pom.xml @@ -183,6 +183,12 @@ org.apache.orc orc-core + + + org.apache.hadoop + hadoop-client-api + + org.apache.tez diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/PriorityBlockingDeque.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/PriorityBlockingDeque.java index 63aaed8fdf8d..10c00c508336 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/PriorityBlockingDeque.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/PriorityBlockingDeque.java @@ -83,7 +83,7 @@ public class PriorityBlockingDeque private Comparator comparator; /** - * Creates a PriorityBlockingDeque with a capacity of + * Creates a 
PriorityBlockingDeque
with a capacity of * {@link Integer#MAX_VALUE}. */ public PriorityBlockingDeque() { @@ -91,10 +91,10 @@ public PriorityBlockingDeque() { } /** - * Creates a PriorityBlockingDeque with the given (fixed) capacity. + * Creates a 
PriorityBlockingDeque
with the given (fixed) capacity. * * @param capacity the capacity of this deque - * @throws IllegalArgumentException if capacity is less than 1 + * @throws IllegalArgumentException if 
capacity
is less than 1 */ public PriorityBlockingDeque(int capacity) { this(null, capacity); @@ -546,10 +546,10 @@ public E peek() { * Returns the number of additional elements that this deque can ideally * (in the absence of memory or resource constraints) accept without * blocking. This is always equal to the initial capacity of this deque - * less the current size of this deque. + * less the current 
size
of this deque. *
* Note that you cannot always tell if an attempt to insert - * an element will succeed by inspecting remainingCapacity + * an element will succeed by inspecting 
remainingCapacity
* because it may be the case that another thread is about to * insert or remove an element. */ @@ -637,16 +637,16 @@ public E pop() { /** * Removes the first occurrence of the specified element from this deque. * If the deque does not contain the element, it is unchanged. - * More formally, removes the first element e such that - * o.equals(e) (if such an element exists). - * Returns true if this deque contained the specified element + * More formally, removes the first element 
e
such that + * 
o.equals(e)
(if such an element exists). + * Returns 
true
if this deque contained the specified element * (or equivalently, if this deque changed as a result of the call). *
* This method is equivalent to * {@link #removeFirstOccurrence(Object) removeFirstOccurrence}. * * @param o element to be removed from this deque, if present - * @return true if this deque changed as a result of the call + * @return 
true
if this deque changed as a result of the call */ @Override public boolean remove(Object o) { @@ -669,12 +669,12 @@ public int size() { } /** - * Returns true if this deque contains the specified element. - * More formally, returns true if and only if this deque contains - * at least one element e such that o.equals(e). + * Returns 
true
if this deque contains the specified element. + * More formally, returns 
true
if and only if this deque contains + * at least one element 
e
such that 
o.equals(e)
. * * @param o object to be checked for containment in this deque - * @return true if this deque contains the specified element + * @return 
true
if this deque contains the specified element */ @Override public boolean contains(Object o) { @@ -717,25 +717,23 @@ public Object[] toArray() { * is returned therein. Otherwise, a new array is allocated with the * runtime type of the specified array and the size of this deque. *
- *

If this deque fits in the specified array with room to spare + * If this deque fits in the specified array with room to spare * (i.e., the array has more elements than this deque), the element in * the array immediately following the end of the deque is set to - * null. - *

+ * 
null
. *

Like the {@link #toArray()} method, this method acts as bridge between * array-based and collection-based APIs. Further, this method allows * precise control over the runtime type of the output array, and may, * under certain circumstances, be used to save allocation costs. *

- *

Suppose x is a deque known to contain only strings. + * Suppose 

x
is a deque known to contain only strings. * The following code can be used to dump the deque into a newly - * allocated array of String: - *

+ * allocated array of 
String
: * 
    *     String[] y = x.toArray(new String[0]);
*
- * Note that toArray(new Object[0]) is identical in function to - * toArray(). + * Note that 
toArray(new Object[0])
is identical in function to + * 
toArray()
. * * @param a the array into which the elements of the deque are to * be stored, if it is big enough; otherwise, a new array of the diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemon.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemon.java index 95c71df8f2a6..c8d2ce74a4e6 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemon.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TestLlapDaemon.java @@ -31,7 +31,6 @@ import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.Mock; -import org.mockito.internal.util.reflection.Fields; import org.mockito.internal.util.reflection.InstanceField; import java.io.File; @@ -44,6 +43,8 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import java.lang.reflect.Field; +import java.util.ArrayList; import static java.lang.Integer.parseInt; import static org.junit.Assert.assertEquals; @@ -60,6 +61,18 @@ public class TestLlapDaemon { MetricsUtils.METRICS_PROCESS_NAME }; + private static List allDeclaredFieldsOf(Object testInstance) { + List result = new ArrayList<>(); + for (Class clazz = testInstance.getClass(); + clazz != Object.class; + clazz = clazz.getSuperclass()) { + for (Field field : clazz.getDeclaredFields()) { + result.add(new InstanceField(field, testInstance)); + } + } + return result; + } + public static final String TEST_LOCAL_DIR = new File(System.getProperty("java.io.tmpdir") + File.separator + TestLlapDaemon.class.getCanonicalName() + "-" + System.currentTimeMillis() @@ -179,10 +192,8 @@ public void testUpdateRegistration() throws IOException { } static void trySetMock(Object o, Class clazz, T mock) { - List instanceFields = Fields - .allDeclaredFieldsOf(o) - .filter(instanceField -> !clazz.isAssignableFrom(instanceField.jdkField().getType())) - .instanceFields(); + List instanceFields = allDeclaredFieldsOf(o).stream() + .filter(instanceField -> clazz.isAssignableFrom(instanceField.jdkField().getType())).toList(); if (instanceFields.size() != 1) { throw new RuntimeException("Mocking is only supported, if only one field is assignable from the given class."); } diff --git a/packaging/pom.xml b/packaging/pom.xml index 6a1f6fbe516b..a86dec574a2b 100644 --- a/packaging/pom.xml +++ b/packaging/pom.xml @@ -172,6 +172,7 @@ \Qhttps://www.eclipse.org/org/documents/epl-v10.php\E \Qhttp://www.eclipse.org/org/documents/epl-v10.php\E + \Qhttps://www.eclipse.org/org/documents/edl-v10.php\E https?://(www\.)?opensource\.org/licenses/mit(-license.php)? diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g index 65a35c2e0b7c..f9ca42fe986d 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g @@ -631,3 +631,48 @@ alterDataConnectorSuffixSetUrl -> ^(TOK_ALTERDATACONNECTOR_URL $dcName $newUri) ; +alterScheduledQueryStatement +@init { gParent.pushMsg("alter scheduled query statement", state); } +@after { gParent.popMsg(state); } + : KW_ALTER KW_SCHEDULED KW_QUERY name=identifier + mod=alterScheduledQueryChange + -> ^(TOK_ALTER_SCHEDULED_QUERY + $name + $mod + ) + ; + +alterScheduledQueryChange +@init { gParent.pushMsg("alter scheduled query change", state); } +@after { gParent.popMsg(state); } + : scheduleSpec + | executedAsSpec + | enableSpecification + | definedAsSpec + | KW_EXECUTE -> ^(TOK_EXECUTE) + ; + +alterColumnConstraint[CommonTree fkColName] +@init { gParent.pushMsg("alter column constraint", state); } +@after { gParent.popMsg(state); } + : ( alterForeignKeyConstraint[$fkColName] ) + | ( alterColConstraint ) + ; + +alterForeignKeyConstraint[CommonTree fkColName] +@init { gParent.pushMsg("alter column constraint", state); } +@after { gParent.popMsg(state); } + : (KW_CONSTRAINT constraintName=identifier)? KW_REFERENCES tabName=tableName LPAREN colName=columnName RPAREN constraintOptsAlter? + -> {$constraintName.tree != null}? + ^(TOK_FOREIGN_KEY ^(TOK_CONSTRAINT_NAME $constraintName) ^(TOK_TABCOLNAME {$fkColName}) $tabName ^(TOK_TABCOLNAME $colName) constraintOptsAlter?) + -> ^(TOK_FOREIGN_KEY ^(TOK_TABCOLNAME {$fkColName}) $tabName ^(TOK_TABCOLNAME $colName) constraintOptsAlter?) + ; + +alterColConstraint +@init { gParent.pushMsg("alter column constraint", state); } +@after { gParent.popMsg(state); } + : (KW_CONSTRAINT constraintName=identifier)? columnConstraintType constraintOptsAlter? + -> {$constraintName.tree != null}? + ^({$columnConstraintType.tree} ^(TOK_CONSTRAINT_NAME $constraintName) constraintOptsAlter?) + -> ^({$columnConstraintType.tree} constraintOptsAlter?) + ; diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 745c437d41bb..506ecf5d983a 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -1733,28 +1733,6 @@ dropScheduledQueryStatement ) ; - -alterScheduledQueryStatement -@init { pushMsg("alter scheduled query statement", state); } -@after { popMsg(state); } - : KW_ALTER KW_SCHEDULED KW_QUERY name=identifier - mod=alterScheduledQueryChange - -> ^(TOK_ALTER_SCHEDULED_QUERY - $name - $mod - ) - ; - -alterScheduledQueryChange -@init { pushMsg("alter scheduled query change", state); } -@after { popMsg(state); } - : scheduleSpec - | executedAsSpec - | enableSpecification - | definedAsSpec - | KW_EXECUTE -> ^(TOK_EXECUTE) - ; - scheduleSpec @init { pushMsg("schedule specification", state); } @after { popMsg(state); } @@ -2345,31 +2323,6 @@ colConstraint -> ^({$columnConstraintType.tree} constraintOptsCreate?) ; -alterColumnConstraint[CommonTree fkColName] -@init { pushMsg("alter column constraint", state); } -@after { popMsg(state); } - : ( alterForeignKeyConstraint[$fkColName] ) - | ( alterColConstraint ) - ; - -alterForeignKeyConstraint[CommonTree fkColName] -@init { pushMsg("alter column constraint", state); } -@after { popMsg(state); } - : (KW_CONSTRAINT constraintName=identifier)? KW_REFERENCES tabName=tableName LPAREN colName=columnName RPAREN constraintOptsAlter? - -> {$constraintName.tree != null}? - ^(TOK_FOREIGN_KEY ^(TOK_CONSTRAINT_NAME $constraintName) ^(TOK_TABCOLNAME {$fkColName}) $tabName ^(TOK_TABCOLNAME $colName) constraintOptsAlter?) - -> ^(TOK_FOREIGN_KEY ^(TOK_TABCOLNAME {$fkColName}) $tabName ^(TOK_TABCOLNAME $colName) constraintOptsAlter?) - ; - -alterColConstraint -@init { pushMsg("alter column constraint", state); } -@after { popMsg(state); } - : (KW_CONSTRAINT constraintName=identifier)? columnConstraintType constraintOptsAlter? - -> {$constraintName.tree != null}? - ^({$columnConstraintType.tree} ^(TOK_CONSTRAINT_NAME $constraintName) constraintOptsAlter?) - -> ^({$columnConstraintType.tree} constraintOptsAlter?) - ; - columnConstraintType : KW_NOT KW_NULL -> TOK_NOT_NULL | KW_DEFAULT defaultVal-> ^(TOK_DEFAULT_VALUE defaultVal) diff --git a/pom.xml b/pom.xml index 852fb234b781..839f3ca33f0b 100644 --- a/pom.xml +++ b/pom.xml @@ -65,8 +65,8 @@ 4.1.0-SNAPSHOT 4.1.0 - 1.8 - 1.8 + 17 + 17 false ${settings.localRepository} . @@ -91,7 +91,7 @@ 1.0b3 - -Xmx2048m -DJETTY_AVAILABLE_PROCESSORS=4 + -Xmx2048m -DJETTY_AVAILABLE_PROCESSORS=4 -Dio.netty.tryReflectionSetAccessible=true -Djava.locale.providers=COMPAT,CLDR --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens java.base/java.util.regex=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.sql/java.sql=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.text=ALL-UNNAMED 2.17 3.4.0 2.10 @@ -107,17 +107,17 @@ 3.5.2 4.9.3 - 1.5.7 + 2.0.0-M1 12.0.0 1.12.0 1.11.3 1.78 1.25.0 - 5.2.8 - 5.2.10 + 6.0.3 + 6.0.8 3.2.0-release - 5.2.10 + 6.0.8 1.5.0 1.15 3.2.2 @@ -132,7 +132,7 @@ 2.9.0 1.10.0 10.14.2.0 - 3.1.0 + 3.2.4 0.1.2 0.17.1 2.2.4 @@ -145,6 +145,7 @@ 1.3 2.5.6-hadoop3 0.7.2 + 15.4 3.3.7 4.0.3 @@ -171,7 +172,7 @@ 2.5.0 5.5.0 1.11.9 - 1.12.0 + 1.17.0 0.9.3 0.16.0 @@ -183,8 +184,8 @@ 21.3.0.0 5.9 1.9.4 - 3.4.4 - 4.11.0 + 5.0.0 + 5.0.0 2.0.0-M5 4.1.77.Final 3.10.5.Final @@ -204,7 +205,8 @@ 2.2.0 1.1 1.1.10.4 - 1.4 + 4.0.0-M1 + 4.0.0-M1 2.3 2.12.2 2.3.4 @@ -220,7 +222,7 @@ 2.9.0 3.0.11 1.2.0 - 4.0.3 + 4.5.0 1.1.0.Final 1.0.1 1.12.499 @@ -363,9 +365,14 @@ ${jersey.version} - com.sun.jersey.contribs - wadl-resourcedoc-doclet - ${wadl-resourcedoc-doclet.version} + org.glassfish.jersey.ext + jersey-wadl-doclet + ${jersey-wadl-doclet.version} + + + org.glassfish.jaxb + jaxb-runtime + ${jaxb-runtime.version} com.sun.jersey @@ -407,6 +414,11 @@ commons-math3 ${commons-math3.version} + + org.openjdk.nashorn + nashorn-core + ${nashorn.version} + io.jsonwebtoken jjwt-api @@ -1874,6 +1886,29 @@ jamon-maven-plugin ${jamon.plugin.version} + + org.apache.maven.plugins + maven-javadoc-plugin + + none + false + + + io.dropwizard.metrics + metrics-core + 3.2.4 + + + + + + aggregate + + aggregate + + + + @@ -2011,11 +2046,11 @@ spotbugs - + com.github.spotbugs spotbugs-maven-plugin - 4.0.0 + 4.5.0.0 @@ -2038,7 +2073,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.0.0 + 4.5.0.0 true 2048 diff --git a/ql/pom.xml b/ql/pom.xml index 597b4695d705..248d8c52a8d2 100644 --- a/ql/pom.xml +++ b/ql/pom.xml @@ -32,6 +32,10 @@ + + org.openjdk.nashorn + nashorn-core + org.apache.atlas atlas-client-v2 @@ -198,6 +202,10 @@ org.apache.commons commons-collections4 + + commons-collections + commons-collections + org.apache.commons commons-lang3 @@ -359,6 +367,12 @@ org.apache.orc orc-tools ${orc.version} + + + org.apache.hadoop + hadoop-client-api + + org.apache.ivy @@ -994,6 +1008,21 @@ + + org.apache.maven.plugins + maven-surefire-plugin + + -Xmx2048m --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens java.base/java.util.regex=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.sql/java.sql=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.text=ALL-UNNAMED + + hive-test-cluster-id-env + US/Pacific + en_US.UTF-8 + ${test.conf.dir}:${basedir}/${hive.path.to.root}/conf + ${test.hive.hadoop.classpath} + ${env.PATH}${test.extra.path} + + + org.apache.maven.plugins maven-jar-plugin diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java b/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java index 90bd7339a71d..2786299fdc0f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/RCFile.java @@ -72,7 +72,7 @@ * data of a row split, as the key part of a record, and all the data of a row * split as the value part. When writing, RCFile.Writer first holds records' * value bytes in memory, and determines a row split if the raw bytes size of - * buffered records overflow a given parameterWriter.columnsBufferSize, + * buffered records overflow a given parameter
Writer.columnsBufferSize
, * which can be set like: conf.setInt(COLUMNS_BUFFER_SIZE_CONF_STR, 4 * 1024 * 1024) . *

@@ -99,10 +99,10 @@ * The {@link Reader} is used to read and explain the bytes of RCFile. *

* - *

RCFile Formats

+ *

RCFile Formats

* * - * + * *
    *
  • version - 3 bytes of magic header RCF, followed by 1 byte of * actual version number (e.g. RCF1)
    • diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SemiJoinReductionMerge.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SemiJoinReductionMerge.java index d3764dcc2365..a96662abd7a6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SemiJoinReductionMerge.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SemiJoinReductionMerge.java @@ -90,7 +90,7 @@ *

    * An example of the transformation on three single column semi join reducers is shown below. The plan is simplified for * presentation purposes. - *

    BEFORE:

    + *

    BEFORE:

    * 
      *        / SEL[fname] - GB1 - RS1 - GB2 - RS2  \
  * SOURCE - SEL[lname] - GB1 - RS1 - GB2 - RS2  -> TS[Author] - FIL[in_bloom(fname) ^ in_bloom(lname) ^ in_bloom(age)]
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/CorrelateProjectExtractor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/CorrelateProjectExtractor.java
index a2af59999d08..e0b0b451ef92 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/CorrelateProjectExtractor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/CorrelateProjectExtractor.java
@@ -52,7 +52,7 @@
  * computation over the correlated variables, from the right side of a correlation
  * ({@link org.apache.calcite.rel.core.Correlate}) and places it on the left side.
  *
- * 
    Plan before
    
+ * 
    Plan before
    
  * 
      * LogicalCorrelate(correlation=[$cor0], joinType=[left], requiredColumns=[{7}])
  *   LogicalTableScan(table=[[scott, EMP]])
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/FilterFlattenCorrelatedConditionRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/FilterFlattenCorrelatedConditionRule.java
index ba78c8fd736b..ee93e5cee79e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/FilterFlattenCorrelatedConditionRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/FilterFlattenCorrelatedConditionRule.java
@@ -39,7 +39,7 @@
  * computation in a {@link org.apache.calcite.rel.core.Project} expression. An additional projection
  * may be added on top of the new filter to retain expression equivalence.
  *
- * 
    Sub-plan before
    
+ * 
    Sub-plan before
    
  * 
      * LogicalProject($f0=[true])
  *   LogicalFilter(condition=[=($cor0.DEPTNO, +($7, 30))])
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDAFResolver.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDAFResolver.java
index d82e6e47ac30..bfb2bdb4f5d8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDAFResolver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/AbstractGenericUDAFResolver.java
@@ -22,12 +22,12 @@
 
 /**
  * An abstract class to help facilitate existing implementations of
- * GenericUDAFResolver to migrate towards the newly introduced
+ * 
    GenericUDAFResolver
     to migrate towards the newly introduced
  * interface {@link GenericUDAFResolver2}. This class provides a default
  * implementation of this new API and in turn calls
  * the existing API {@link GenericUDAFResolver#getEvaluator(TypeInfo[])} by
  * ignoring the extra parameter information available via the
- * GenericUDAFParameterInfo interface.
+ * 
    GenericUDAFParameterInfo
     interface.
  *
  */
 @Deprecated
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFParameterInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFParameterInfo.java
index 1afa1ebf007a..9dc0cdd328f2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFParameterInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFParameterInfo.java
@@ -23,24 +23,24 @@
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 
 /**
- * A callback interface used in conjunction with GenericUDAFResolver2
+ * A callback interface used in conjunction with 
    GenericUDAFResolver2
    
  * interface that allows for a more extensible and flexible means of
  * discovering the parameter types provided for UDAF invocation. Apart from
- * allowing the function implementation to discover the TypeInfo of
+ * allowing the function implementation to discover the 
    TypeInfo
     of
  * any types provided in the invocation, this also allows the implementation
- * to determine if the parameters were qualified using DISTINCT. If
+ * to determine if the parameters were qualified using 
    DISTINCT
    . If
  * no parameters were specified explicitly, it allows the function
  * implementation to test if the invocation used the wildcard syntax
- * such as FUNCTION(*).
+ * such as 
    FUNCTION(*)
    .
  * 
    
  * Note: The implementation of function does not have to handle the
- * actual DISTINCT or wildcard implementation. This information is
+ * actual 
    DISTINCT
     or wildcard implementation. This information is
  * provided only to allow the function implementation to accept or reject
- * such invocations. For example - the implementation of COUNT UDAF
- * requires that the DISTINCT qualifier be supplied when more than
+ * such invocations. For example - the implementation of 
    COUNT
     UDAF
+ * requires that the 
    DISTINCT
     qualifier be supplied when more than
  * one parameters are specified in the invocation. The actual filtering of
- * data bound to parameter types for DISTINCT implementation is
- * handled by the framework and not the COUNT UDAF implementation.
+ * data bound to parameter types for 
    DISTINCT
     implementation is
+ * handled by the framework and not the 
    COUNT
     UDAF implementation.
  */
 @InterfaceAudience.Public
 @InterfaceStability.Stable
@@ -60,30 +60,30 @@ public interface GenericUDAFParameterInfo {
   ObjectInspector[] getParameterObjectInspectors();
 
   /**
-   * Returns true if the UDAF invocation was qualified with
-   * DISTINCT keyword. Note that this is provided for informational
+   * Returns 
    true
     if the UDAF invocation was qualified with
+   * 
    DISTINCT
     keyword. Note that this is provided for informational
    * purposes only and the function implementation is not expected to ensure
    * the distinct property for the parameter values. That is handled by the
    * framework.
-   * @return true if the UDAF invocation was qualified with
-   * DISTINCT keyword, false otherwise.
+   * @return 
    true
     if the UDAF invocation was qualified with
+   * 
    DISTINCT
     keyword, 
    false
     otherwise.
    */
   boolean isDistinct();
 
   /**
    * The flag to indicate if the UDAF invocation was from the windowing function
    * call or not.
-   * @return true if the UDAF invocation was from the windowing function
+   * @return 
    true
     if the UDAF invocation was from the windowing function
    * call.
    */
   boolean isWindowing();
   /**
-   * Returns true if the UDAF invocation was done via the wildcard
-   * syntax FUNCTION(*). Note that this is provided for informational
+   * Returns 
    true
     if the UDAF invocation was done via the wildcard
+   * syntax 
    FUNCTION(*)
    . Note that this is provided for informational
    * purposes only and the function implementation is not expected to ensure
    * the wildcard handling of the target relation. That is handled by the
    * framework.
-   * @return true if the UDAF invocation was done with a wildcard
+   * @return 
    true
     if the UDAF invocation was done with a wildcard
    * instead of explicit parameter list.
    */
   boolean isAllColumns();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFResolver2.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFResolver2.java
index dc4eb4e5ddf6..7ea7c0c04654 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFResolver2.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFResolver2.java
@@ -21,21 +21,21 @@
 import org.apache.hadoop.hive.ql.parse.SemanticException;
 
 /**
- * This interface extends the GenericUDAFResolver interface and
+ * This interface extends the 
    GenericUDAFResolver
     interface and
  * provides more flexibility in terms of discovering the parameter types
  * supplied to the UDAF. Implementations that extend this interface will
  * also have access to extra information such as the specification of the
- * DISTINCT qualifier or the invocation with the special wildcard
+ * 
    DISTINCT
     qualifier or the invocation with the special wildcard
  * character.
  * 
    
  * Note: The implementation of function does not have to handle the
- * actual DISTINCT or wildcard implementation. This information is
+ * actual 
    DISTINCT
     or wildcard implementation. This information is
  * provided only to allow the function implementation to accept or reject
- * such invocations. For example - the implementation of COUNT UDAF
- * requires that the DISTINCT qualifier be supplied when more than
+ * such invocations. For example - the implementation of 
    COUNT
     UDAF
+ * requires that the 
    DISTINCT
     qualifier be supplied when more than
  * one parameters are specified in the invocation. The actual filtering of
- * data bound to parameter types for DISTINCT implementation is
- * handled by the framework and not the COUNT UDAF implementation.
+ * data bound to parameter types for 
    DISTINCT
     implementation is
+ * handled by the framework and not the 
    COUNT
     UDAF implementation.
  */
 @Deprecated
 @SuppressWarnings("deprecation")
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/SimpleGenericUDAFParameterInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/SimpleGenericUDAFParameterInfo.java
index b0c75affa969..4d6cacfa5467 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/SimpleGenericUDAFParameterInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/SimpleGenericUDAFParameterInfo.java
@@ -22,7 +22,7 @@
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 
 /**
- * A simple implementation of GenericUDAFParameterInfo.
+ * A simple implementation of 
    GenericUDAFParameterInfo
    .
  *
  */
 public class SimpleGenericUDAFParameterInfo implements GenericUDAFParameterInfo
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFDateFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFDateFormat.java
index 85d6d5184619..269daae5c83d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFDateFormat.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFDateFormat.java
@@ -166,7 +166,7 @@ public void testNullFmt() throws HiveException {
   public void testWrongFmt() throws HiveException {
     GenericUDFDateFormat udf = new GenericUDFDateFormat();
     ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
-    Text fmtText = new Text("B");
+    Text fmtText = new Text("C");                                                                       // 'B' is defined as a pattern in jdk17 to represent period-of-day, changing it to pattern 'C' having no definition in jdk17
     ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory
         .getPrimitiveWritableConstantObjectInspector(TypeInfoFactory.stringTypeInfo, fmtText);
     ObjectInspector[] arguments = { valueOI0, valueOI1 };
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFToUnixTimestamp.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFToUnixTimestamp.java
index 6f427ac0cfbc..2489e2fdc253 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFToUnixTimestamp.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFToUnixTimestamp.java
@@ -167,7 +167,8 @@ public void testStringArg2() throws HiveException {
     runAndVerify(udf2,
         new Text("1400-02-01 00:00:00 ICT"),
         new Text("yyyy-MM-dd HH:mm:ss z"),
-        new LongWritable(TimestampTZUtil.parse("1400-01-31 09:00:22", ZoneId.systemDefault()).getEpochSecond()));
+        new LongWritable(TimestampTZUtil.parse("1400-01-31 09:24:58", ZoneId.systemDefault()).getEpochSecond()));                   // jdk8 & jdk17 use different IANA TZdata versions resulting
+                                                                                                                                       // in different zone rules for conversion of historical ICT time
     runAndVerify(udf2,
         new Text("1400-02-01 00:00:00 UTC"),
         new Text("yyyy-MM-dd HH:mm:ss z"),
diff --git a/ql/src/test/queries/clientpositive/udf_date_format.q b/ql/src/test/queries/clientpositive/udf_date_format.q
index 9e9af09adfed..6e92ace8df84 100644
--- a/ql/src/test/queries/clientpositive/udf_date_format.q
+++ b/ql/src/test/queries/clientpositive/udf_date_format.q
@@ -64,7 +64,7 @@ date_format(cast(null as timestamp), 'HH');
 -- wrong fmt
 select
 date_format('2015-04-08', ''),
-date_format('2015-04-08', 'B');
+date_format('2015-04-08', 'C');
 
 -- with time zone
 set hive.local.time.zone=Asia/Bangkok;
diff --git a/ql/src/test/resources/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFDateFormatEvaluate.csv b/ql/src/test/resources/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFDateFormatEvaluate.csv
index 401e1ef7dec5..afd9dd6d22d3 100644
--- a/ql/src/test/resources/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFDateFormatEvaluate.csv
+++ b/ql/src/test/resources/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFDateFormatEvaluate.csv
@@ -15,7 +15,7 @@ Input datetime;Input pattern;Local timezone;Formatter type;Expected output
 1800-01-01 00:00:00;yyyy-MM-dd HH:mm:ss;Asia/Kolkata;SIMPLE;1799-12-31 23:36:32
 Jul 9 2023;MMM dd yyyy;Etc/GMT;DATETIME;null
 Jul 9 2023;MMM dd yyyy;Etc/GMT;SIMPLE;null
-2023-07-21;DD;Etc/GMT;DATETIME;null
+2023-07-21;DD;Etc/GMT;DATETIME;202
 2023-07-21;DD;Etc/GMT;SIMPLE;202
 2023-07-21;DDD;Etc/GMT;DATETIME;202
 2023-07-21;DDD;Etc/GMT;SIMPLE;202
diff --git a/ql/src/test/resources/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUnixTimeEvaluate.csv b/ql/src/test/resources/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUnixTimeEvaluate.csv
index 8198c33eee6b..3cb7bb7f606e 100644
--- a/ql/src/test/resources/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUnixTimeEvaluate.csv
+++ b/ql/src/test/resources/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUnixTimeEvaluate.csv
@@ -22,7 +22,7 @@ Jul 09 2023;MMM dd yyyy;Etc/GMT;DATETIME;1688860800
 Jul 09 2023;MMM dd yyyy;Etc/GMT;SIMPLE;1688860800
 Jul 21 2023;MMM dd yyyy;Etc/GMT;DATETIME;1689897600
 Jul 21 2023;MMM dd yyyy;Etc/GMT;SIMPLE;1689897600
-Field DayOfYear cannot be printed as the value 202 exceeds the maximum print width of 2;YYYY-MM-DD;Etc/GMT;DATETIME;1689897600
+2023-07-202;YYYY-MM-DD;Etc/GMT;DATETIME;1689897600
 2023-07-202;YYYY-MM-DD;Etc/GMT;SIMPLE;1689897600
 Jul 21 2023 09:13;MMM dd yyyy HH:mm;Etc/GMT;DATETIME;1689930780
 Jul 21 2023 09:13;MMM dd yyyy HH:mm;Etc/GMT;SIMPLE;1689930780
@@ -32,7 +32,7 @@ Jul 21 2023 9:13;MMM dd yyyy H:mm;Etc/GMT;SIMPLE;1689930780
 2023-07-21 09:13;yyyy-MM-dd HH:mm;Etc/GMT;SIMPLE;1689930780
 2023-07-21 9:13;yyyy-MM-dd H:mm;Etc/GMT;DATETIME;1689930780
 2023-07-21 9:13;yyyy-MM-dd H:mm;Etc/GMT;SIMPLE;1689930780
-Field MilliOfDay cannot be printed as the value 33180000 exceeds the maximum print width of 2;yyyy-MM-dd HH:mmAA;Etc/GMT;DATETIME;1689930780
+2023-07-21 09:1333180000;yyyy-MM-dd HH:mmAA;Etc/GMT;DATETIME;1689930780
 Illegal pattern character 'A';yyyy-MM-dd HH:mmAA;Etc/GMT;SIMPLE;1689930780
 Too many pattern letters: a;yyyy-MM-dd HH:mmaa;Etc/GMT;DATETIME;1689930780
 2023-07-21 09:13AM;yyyy-MM-dd HH:mmaa;Etc/GMT;SIMPLE;1689930780
diff --git a/ql/src/test/results/clientpositive/llap/groupby3_map_multi_distinct.q.out b/ql/src/test/results/clientpositive/llap/groupby3_map_multi_distinct.q.out
index e84c42c819b6..de30b7df02c2 100644
--- a/ql/src/test/results/clientpositive/llap/groupby3_map_multi_distinct.q.out
+++ b/ql/src/test/results/clientpositive/llap/groupby3_map_multi_distinct.q.out
@@ -69,12 +69,12 @@ STAGE PLANS:
                       minReductionHashAggr: 0.4
                       mode: hash
                       outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11
-                      Statistics: Num rows: 500 Data size: 263000 Basic stats: COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 307 Data size: 161482 Basic stats: COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
                         key expressions: _col0 (type: double), _col1 (type: string)
                         null sort order: zz
                         sort order: ++
-                        Statistics: Num rows: 500 Data size: 263000 Basic stats: COMPLETE Column stats: COMPLETE
+                        Statistics: Num rows: 307 Data size: 161482 Basic stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col2 (type: double), _col3 (type: double), _col4 (type: bigint), _col7 (type: string), _col8 (type: string), _col9 (type: double)
             Execution mode: llap
             LLAP IO: all inputs
diff --git a/ql/src/test/results/clientpositive/llap/mm_all.q.out b/ql/src/test/results/clientpositive/llap/mm_all.q.out
index 72922cf6da98..6ba0db3ecd67 100644
--- a/ql/src/test/results/clientpositive/llap/mm_all.q.out
+++ b/ql/src/test/results/clientpositive/llap/mm_all.q.out
@@ -1998,7 +1998,7 @@ Table Type:         	MANAGED_TABLE
 Table Parameters:	 	 
 	COLUMN_STATS_ACCURATE	{\"BASIC_STATS\":\"true\"}
 	bucketing_version   	2                   
-	numFiles            	33                  
+	numFiles            	16                  
 	numRows             	500                 
 	rawDataSize         	5312                
 	totalSize           	#Masked#
diff --git a/ql/src/test/results/clientpositive/llap/mm_dp.q.out b/ql/src/test/results/clientpositive/llap/mm_dp.q.out
index 303d751d99c1..643f0ba2f385 100644
--- a/ql/src/test/results/clientpositive/llap/mm_dp.q.out
+++ b/ql/src/test/results/clientpositive/llap/mm_dp.q.out
@@ -2120,23 +2120,23 @@ POSTHOOK: Input: default@dp_mm@val=val_96
 POSTHOOK: Input: default@dp_mm@val=val_97
 POSTHOOK: Input: default@dp_mm@val=val_98
 POSTHOOK: Output: hdfs://### HDFS PATH ###
-0	455	10	val_0
-0	455	16	val_0
-0	455	23	val_0
-0	457	5	val_0
-0	457	5	val_0
-0	457	10	val_0
-0	457	14	val_0
-0	457	16	val_0
-0	457	23	val_0
-0	458	5	val_0
-0	458	5	val_0
-0	458	14	val_0
-0	459	5	val_0
-0	459	5	val_0
-0	459	14	val_0
-2	455	9	val_2
-2	457	9	val_2
+0	455	1	val_0
+0	455	7	val_0
+0	455	13	val_0
+0	457	1	val_0
+0	457	3	val_0
+0	457	7	val_0
+0	457	7	val_0
+0	457	13	val_0
+0	457	19	val_0
+0	458	3	val_0
+0	458	7	val_0
+0	458	19	val_0
+0	459	3	val_0
+0	459	7	val_0
+0	459	19	val_0
+2	455	10	val_2
+2	457	10	val_2
 2	457	10	val_2
 2	458	10	val_2
 2	459	10	val_2
@@ -2145,376 +2145,376 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
 4	457	4	val_4
 4	458	1	val_4
 4	459	1	val_4
-5	455	4	val_5
+5	455	10	val_5
 5	455	10	val_5
 5	455	27	val_5
-5	457	4	val_5
-5	457	6	val_5
 5	457	10	val_5
 5	457	10	val_5
+5	457	16	val_5
+5	457	20	val_5
+5	457	24	val_5
 5	457	27	val_5
-5	457	27	val_5
-5	458	6	val_5
-5	458	10	val_5
-5	458	27	val_5
-5	459	6	val_5
-5	459	10	val_5
-5	459	27	val_5
-8	455	23	val_8
-8	457	20	val_8
-8	457	23	val_8
-8	458	20	val_8
-8	459	20	val_8
-9	455	24	val_9
-9	457	15	val_9
-9	457	24	val_9
-9	458	15	val_9
-9	459	15	val_9
-10	455	13	val_10
-10	457	1	val_10
-10	457	13	val_10
-10	458	1	val_10
-10	459	1	val_10
-11	455	24	val_11
+5	458	16	val_5
+5	458	20	val_5
+5	458	24	val_5
+5	459	16	val_5
+5	459	20	val_5
+5	459	24	val_5
+8	455	9	val_8
+8	457	4	val_8
+8	457	9	val_8
+8	458	4	val_8
+8	459	4	val_8
+9	455	23	val_9
+9	457	6	val_9
+9	457	23	val_9
+9	458	6	val_9
+9	459	6	val_9
+10	455	20	val_10
+10	457	6	val_10
+10	457	20	val_10
+10	458	6	val_10
+10	459	6	val_10
+11	455	10	val_11
 11	457	5	val_11
-11	457	24	val_11
+11	457	10	val_11
 11	458	5	val_11
 11	459	5	val_11
-12	455	9	val_12
+12	455	4	val_12
 12	455	10	val_12
-12	457	9	val_12
+12	457	4	val_12
+12	457	5	val_12
 12	457	10	val_12
-12	457	19	val_12
-12	457	27	val_12
-12	458	19	val_12
-12	458	27	val_12
-12	459	19	val_12
-12	459	27	val_12
-15	455	24	val_15
-15	455	27	val_15
-15	457	7	val_15
-15	457	17	val_15
-15	457	24	val_15
+12	457	10	val_12
+12	458	5	val_12
+12	458	10	val_12
+12	459	5	val_12
+12	459	10	val_12
+15	455	10	val_15
+15	455	28	val_15
+15	457	10	val_15
+15	457	16	val_15
 15	457	27	val_15
-15	458	7	val_15
-15	458	17	val_15
-15	459	7	val_15
-15	459	17	val_15
-17	455	13	val_17
-17	457	4	val_17
-17	457	13	val_17
-17	458	4	val_17
-17	459	4	val_17
+15	457	28	val_15
+15	458	16	val_15
+15	458	27	val_15
+15	459	16	val_15
+15	459	27	val_15
+17	455	9	val_17
+17	457	9	val_17
+17	457	20	val_17
+17	458	20	val_17
+17	459	20	val_17
+18	455	19	val_18
 18	455	24	val_18
-18	455	28	val_18
-18	457	2	val_18
-18	457	13	val_18
+18	457	10	val_18
+18	457	19	val_18
 18	457	24	val_18
 18	457	28	val_18
-18	458	2	val_18
-18	458	13	val_18
-18	459	2	val_18
-18	459	13	val_18
-19	455	10	val_19
-19	457	10	val_19
-19	457	23	val_19
-19	458	23	val_19
-19	459	23	val_19
+18	458	10	val_18
+18	458	28	val_18
+18	459	10	val_18
+18	459	28	val_18
+19	455	9	val_19
+19	457	9	val_19
+19	457	20	val_19
+19	458	20	val_19
+19	459	20	val_19
 20	455	24	val_20
 20	457	24	val_20
 20	457	24	val_20
 20	458	24	val_20
 20	459	24	val_20
+24	455	7	val_24
 24	455	19	val_24
-24	455	28	val_24
-24	457	6	val_24
+24	457	3	val_24
+24	457	7	val_24
+24	457	14	val_24
 24	457	19	val_24
-24	457	28	val_24
-24	457	28	val_24
-24	458	6	val_24
-24	458	28	val_24
-24	459	6	val_24
-24	459	28	val_24
+24	458	3	val_24
+24	458	14	val_24
+24	459	3	val_24
+24	459	14	val_24
+26	455	7	val_26
 26	455	9	val_26
-26	455	19	val_26
+26	457	7	val_26
 26	457	7	val_26
 26	457	9	val_26
-26	457	17	val_26
-26	457	19	val_26
+26	457	27	val_26
 26	458	7	val_26
-26	458	17	val_26
+26	458	27	val_26
 26	459	7	val_26
-26	459	17	val_26
+26	459	27	val_26
 27	455	27	val_27
 27	457	9	val_27
 27	457	27	val_27
 27	458	9	val_27
 27	459	9	val_27
 28	455	13	val_28
+28	457	9	val_28
 28	457	13	val_28
-28	457	20	val_28
-28	458	20	val_28
-28	459	20	val_28
+28	458	9	val_28
+28	459	9	val_28
 30	455	23	val_30
-30	457	3	val_30
+30	457	16	val_30
 30	457	23	val_30
-30	458	3	val_30
-30	459	3	val_30
+30	458	16	val_30
+30	459	16	val_30
 33	455	9	val_33
-33	457	6	val_33
 33	457	9	val_33
-33	458	6	val_33
-33	459	6	val_33
-34	455	9	val_34
-34	457	9	val_34
-34	457	28	val_34
-34	458	28	val_34
-34	459	28	val_34
-35	455	7	val_35
+33	457	11	val_33
+33	458	11	val_33
+33	459	11	val_33
+34	455	19	val_34
+34	457	19	val_34
+34	457	20	val_34
+34	458	20	val_34
+34	459	20	val_34
+35	455	9	val_35
 35	455	23	val_35
-35	455	28	val_35
-35	457	2	val_35
-35	457	7	val_35
+35	455	27	val_35
+35	457	9	val_35
 35	457	14	val_35
-35	457	17	val_35
+35	457	16	val_35
 35	457	23	val_35
-35	457	28	val_35
-35	458	2	val_35
+35	457	27	val_35
+35	457	27	val_35
 35	458	14	val_35
-35	458	17	val_35
-35	459	2	val_35
+35	458	16	val_35
+35	458	27	val_35
 35	459	14	val_35
-35	459	17	val_35
+35	459	16	val_35
+35	459	27	val_35
 37	455	1	val_37
 37	455	19	val_37
 37	457	1	val_37
 37	457	11	val_37
 37	457	19	val_37
-37	457	27	val_37
+37	457	20	val_37
 37	458	11	val_37
-37	458	27	val_37
+37	458	20	val_37
 37	459	11	val_37
-37	459	27	val_37
+37	459	20	val_37
 41	455	13	val_41
-41	457	1	val_41
 41	457	13	val_41
-41	458	1	val_41
-41	459	1	val_41
-42	455	14	val_42
+41	457	28	val_41
+41	458	28	val_41
+41	459	28	val_41
+42	455	19	val_42
 42	455	24	val_42
-42	457	9	val_42
-42	457	11	val_42
-42	457	14	val_42
+42	457	2	val_42
+42	457	19	val_42
 42	457	24	val_42
-42	458	9	val_42
-42	458	11	val_42
-42	459	9	val_42
-42	459	11	val_42
+42	457	24	val_42
+42	458	2	val_42
+42	458	24	val_42
+42	459	2	val_42
+42	459	24	val_42
 43	455	4	val_43
 43	457	4	val_43
 43	457	14	val_43
 43	458	14	val_43
 43	459	14	val_43
-44	455	13	val_44
-44	457	7	val_44
+44	455	9	val_44
+44	457	9	val_44
 44	457	13	val_44
-44	458	7	val_44
-44	459	7	val_44
+44	458	13	val_44
+44	459	13	val_44
 47	455	13	val_47
 47	457	13	val_47
 47	457	23	val_47
 47	458	23	val_47
 47	459	23	val_47
+51	455	10	val_51
 51	455	19	val_51
-51	455	27	val_51
 51	457	1	val_51
-51	457	3	val_51
+51	457	10	val_51
+51	457	10	val_51
 51	457	19	val_51
-51	457	27	val_51
 51	458	1	val_51
-51	458	3	val_51
+51	458	10	val_51
 51	459	1	val_51
-51	459	3	val_51
-53	455	28	val_53
-53	457	17	val_53
-53	457	28	val_53
-53	458	17	val_53
-53	459	17	val_53
+51	459	10	val_51
+53	455	19	val_53
+53	457	19	val_53
+53	457	27	val_53
+53	458	27	val_53
+53	459	27	val_53
 54	455	20	val_54
 54	457	4	val_54
 54	457	20	val_54
 54	458	4	val_54
 54	459	4	val_54
-57	455	1	val_57
+57	455	4	val_57
 57	457	1	val_57
-57	457	15	val_57
-57	458	15	val_57
-57	459	15	val_57
-58	455	4	val_58
-58	455	23	val_58
-58	457	4	val_58
-58	457	7	val_58
-58	457	14	val_58
-58	457	23	val_58
-58	458	7	val_58
-58	458	14	val_58
-58	459	7	val_58
-58	459	14	val_58
+57	457	4	val_57
+57	458	1	val_57
+57	459	1	val_57
+58	455	24	val_58
+58	455	24	val_58
+58	457	2	val_58
+58	457	19	val_58
+58	457	24	val_58
+58	457	24	val_58
+58	458	2	val_58
+58	458	19	val_58
+58	459	2	val_58
+58	459	19	val_58
 64	455	28	val_64
-64	457	5	val_64
 64	457	28	val_64
-64	458	5	val_64
-64	459	5	val_64
-65	455	3	val_65
-65	457	3	val_65
-65	457	15	val_65
-65	458	15	val_65
-65	459	15	val_65
+64	457	28	val_64
+64	458	28	val_64
+64	459	28	val_64
+65	455	7	val_65
+65	457	1	val_65
+65	457	7	val_65
+65	458	1	val_65
+65	459	1	val_65
 66	455	13	val_66
 66	457	10	val_66
 66	457	13	val_66
 66	458	10	val_66
 66	459	10	val_66
-67	455	10	val_67
-67	455	23	val_67
-67	457	10	val_67
-67	457	14	val_67
-67	457	23	val_67
+67	455	13	val_67
+67	455	28	val_67
+67	457	1	val_67
+67	457	2	val_67
+67	457	13	val_67
 67	457	28	val_67
-67	458	14	val_67
-67	458	28	val_67
-67	459	14	val_67
-67	459	28	val_67
+67	458	1	val_67
+67	458	2	val_67
+67	459	1	val_67
+67	459	2	val_67
 69	455	14	val_69
+69	457	6	val_69
 69	457	14	val_69
-69	457	25	val_69
-69	458	25	val_69
-69	459	25	val_69
+69	458	6	val_69
+69	459	6	val_69
+70	455	9	val_70
 70	455	13	val_70
-70	455	16	val_70
 70	455	23	val_70
-70	457	0	val_70
-70	457	1	val_70
-70	457	10	val_70
+70	457	9	val_70
 70	457	13	val_70
-70	457	16	val_70
+70	457	14	val_70
+70	457	20	val_70
 70	457	23	val_70
-70	458	0	val_70
-70	458	1	val_70
-70	458	10	val_70
-70	459	0	val_70
-70	459	1	val_70
-70	459	10	val_70
-72	455	9	val_72
+70	457	27	val_70
+70	458	14	val_70
+70	458	20	val_70
+70	458	27	val_70
+70	459	14	val_70
+70	459	20	val_70
+70	459	27	val_70
 72	455	19	val_72
-72	457	9	val_72
-72	457	13	val_72
+72	455	24	val_72
 72	457	19	val_72
+72	457	24	val_72
+72	457	28	val_72
 72	457	28	val_72
-72	458	13	val_72
 72	458	28	val_72
-72	459	13	val_72
+72	458	28	val_72
+72	459	28	val_72
 72	459	28	val_72
 74	455	24	val_74
-74	457	1	val_74
+74	457	20	val_74
 74	457	24	val_74
-74	458	1	val_74
-74	459	1	val_74
+74	458	20	val_74
+74	459	20	val_74
 76	455	1	val_76
 76	455	10	val_76
 76	457	1	val_76
+76	457	7	val_76
 76	457	10	val_76
-76	457	20	val_76
-76	457	28	val_76
-76	458	20	val_76
-76	458	28	val_76
-76	459	20	val_76
-76	459	28	val_76
-77	455	2	val_77
-77	457	2	val_77
-77	457	2	val_77
-77	458	2	val_77
-77	459	2	val_77
+76	457	23	val_76
+76	458	7	val_76
+76	458	23	val_76
+76	459	7	val_76
+76	459	23	val_76
+77	455	9	val_77
+77	457	9	val_77
+77	457	13	val_77
+77	458	13	val_77
+77	459	13	val_77
 78	455	7	val_78
+78	457	4	val_78
 78	457	7	val_78
-78	457	23	val_78
-78	458	23	val_78
-78	459	23	val_78
-80	455	10	val_80
-80	457	5	val_80
+78	458	4	val_78
+78	459	4	val_78
+80	455	24	val_80
 80	457	10	val_80
-80	458	5	val_80
-80	459	5	val_80
-82	455	14	val_82
-82	457	11	val_82
-82	457	14	val_82
-82	458	11	val_82
-82	459	11	val_82
-83	455	27	val_83
-83	455	27	val_83
-83	457	9	val_83
-83	457	20	val_83
-83	457	27	val_83
-83	457	27	val_83
-83	458	9	val_83
-83	458	20	val_83
-83	459	9	val_83
-83	459	20	val_83
+80	457	24	val_80
+80	458	10	val_80
+80	459	10	val_80
+82	455	27	val_82
+82	457	24	val_82
+82	457	27	val_82
+82	458	24	val_82
+82	459	24	val_82
+83	455	7	val_83
+83	455	13	val_83
+83	457	2	val_83
+83	457	6	val_83
+83	457	7	val_83
+83	457	13	val_83
+83	458	2	val_83
+83	458	6	val_83
+83	459	2	val_83
+83	459	6	val_83
 84	455	10	val_84
-84	455	19	val_84
-84	457	3	val_84
-84	457	9	val_84
+84	455	27	val_84
+84	457	7	val_84
 84	457	10	val_84
-84	457	19	val_84
-84	458	3	val_84
-84	458	9	val_84
-84	459	3	val_84
-84	459	9	val_84
-85	455	10	val_85
+84	457	14	val_84
+84	457	27	val_84
+84	458	7	val_84
+84	458	14	val_84
+84	459	7	val_84
+84	459	14	val_84
+85	455	24	val_85
 85	457	10	val_85
-85	457	27	val_85
-85	458	27	val_85
-85	459	27	val_85
+85	457	24	val_85
+85	458	10	val_85
+85	459	10	val_85
 86	455	10	val_86
 86	457	10	val_86
 86	457	27	val_86
 86	458	27	val_86
 86	459	27	val_86
-87	455	3	val_87
-87	457	3	val_87
-87	457	28	val_87
-87	458	28	val_87
-87	459	28	val_87
+87	455	10	val_87
+87	457	10	val_87
+87	457	23	val_87
+87	458	23	val_87
+87	459	23	val_87
 90	455	1	val_90
-90	455	7	val_90
-90	455	9	val_90
+90	455	1	val_90
+90	455	14	val_90
+90	457	1	val_90
+90	457	1	val_90
 90	457	1	val_90
+90	457	5	val_90
 90	457	6	val_90
-90	457	7	val_90
-90	457	9	val_90
-90	457	15	val_90
-90	457	19	val_90
+90	457	14	val_90
+90	458	1	val_90
+90	458	5	val_90
 90	458	6	val_90
-90	458	15	val_90
-90	458	19	val_90
+90	459	1	val_90
+90	459	5	val_90
 90	459	6	val_90
-90	459	15	val_90
-90	459	19	val_90
 92	455	7	val_92
 92	457	7	val_92
 92	457	19	val_92
 92	458	19	val_92
 92	459	19	val_92
-95	455	1	val_95
+95	455	7	val_95
 95	455	28	val_95
-95	457	1	val_95
-95	457	2	val_95
+95	457	7	val_95
 95	457	16	val_95
 95	457	28	val_95
-95	458	2	val_95
+95	457	28	val_95
 95	458	16	val_95
-95	459	2	val_95
+95	458	28	val_95
 95	459	16	val_95
+95	459	28	val_95
 96	455	24	val_96
 96	457	9	val_96
 96	457	24	val_96
@@ -2522,14 +2522,14 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
 96	459	9	val_96
 97	455	7	val_97
 97	455	14	val_97
-97	457	1	val_97
-97	457	1	val_97
+97	457	4	val_97
 97	457	7	val_97
 97	457	14	val_97
-97	458	1	val_97
-97	458	1	val_97
-97	459	1	val_97
-97	459	1	val_97
+97	457	28	val_97
+97	458	4	val_97
+97	458	28	val_97
+97	459	4	val_97
+97	459	28	val_97
 98	455	9	val_98
 98	455	24	val_98
 98	457	4	val_98
@@ -2540,240 +2540,240 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
 98	458	20	val_98
 98	459	4	val_98
 98	459	20	val_98
-100	455	1	val_100
-100	455	16	val_100
-100	457	1	val_100
+100	455	4	val_100
+100	455	14	val_100
+100	457	4	val_100
+100	457	6	val_100
 100	457	14	val_100
-100	457	16	val_100
-100	457	28	val_100
-100	458	14	val_100
-100	458	17	val_100
-100	458	27	val_100
+100	457	20	val_100
+100	458	6	val_100
+100	458	7	val_100
+100	458	20	val_100
 100	458	28	val_100
-100	459	14	val_100
-100	459	28	val_100
-103	455	2	val_103
+100	459	6	val_100
+100	459	20	val_100
 103	455	20	val_103
-103	457	1	val_103
-103	457	2	val_103
-103	457	11	val_103
+103	455	24	val_103
+103	457	4	val_103
 103	457	20	val_103
-103	458	1	val_103
-103	458	6	val_103
-103	458	11	val_103
-103	458	28	val_103
-103	459	1	val_103
-103	459	11	val_103
-104	455	4	val_104
+103	457	24	val_103
+103	457	24	val_103
+103	458	3	val_103
+103	458	4	val_103
+103	458	24	val_103
+103	458	27	val_103
+103	459	4	val_103
+103	459	24	val_103
 104	455	4	val_104
+104	455	10	val_104
 104	457	4	val_104
-104	457	4	val_104
-104	457	20	val_104
-104	457	25	val_104
-104	458	3	val_104
-104	458	5	val_104
-104	458	20	val_104
-104	458	25	val_104
-104	459	20	val_104
-104	459	25	val_104
-105	455	1	val_105
-105	457	1	val_105
-105	457	11	val_105
-105	458	11	val_105
-105	458	11	val_105
-105	459	11	val_105
+104	457	7	val_104
+104	457	10	val_104
+104	457	28	val_104
+104	458	7	val_104
+104	458	13	val_104
+104	458	16	val_104
+104	458	28	val_104
+104	459	7	val_104
+104	459	28	val_104
+105	455	4	val_105
+105	457	4	val_105
+105	457	10	val_105
+105	458	10	val_105
+105	458	24	val_105
+105	459	10	val_105
 111	455	10	val_111
 111	457	4	val_111
 111	457	10	val_111
 111	458	4	val_111
 111	458	13	val_111
 111	459	4	val_111
-113	455	19	val_113
-113	455	27	val_113
-113	457	1	val_113
-113	457	17	val_113
-113	457	19	val_113
+113	455	7	val_113
+113	455	10	val_113
+113	457	7	val_113
+113	457	10	val_113
+113	457	10	val_113
 113	457	27	val_113
-113	458	1	val_113
-113	458	4	val_113
-113	458	7	val_113
-113	458	17	val_113
-113	459	1	val_113
-113	459	17	val_113
-114	455	14	val_114
-114	457	14	val_114
+113	458	10	val_113
+113	458	10	val_113
+113	458	20	val_113
+113	458	27	val_113
+113	459	10	val_113
+113	459	27	val_113
+114	455	28	val_114
+114	457	20	val_114
 114	457	28	val_114
-114	458	14	val_114
-114	458	28	val_114
-114	459	28	val_114
-116	455	2	val_116
-116	457	2	val_116
-116	457	14	val_116
-116	458	3	val_116
-116	458	14	val_116
-116	459	14	val_116
-118	455	5	val_118
-118	455	24	val_118
-118	457	5	val_118
-118	457	10	val_118
-118	457	24	val_118
+114	458	16	val_114
+114	458	20	val_114
+114	459	20	val_114
+116	455	24	val_116
+116	457	23	val_116
+116	457	24	val_116
+116	458	23	val_116
+116	458	23	val_116
+116	459	23	val_116
+118	455	1	val_118
+118	455	19	val_118
+118	457	1	val_118
+118	457	3	val_118
+118	457	19	val_118
 118	457	24	val_118
-118	458	7	val_118
-118	458	10	val_118
-118	458	11	val_118
+118	458	3	val_118
+118	458	4	val_118
+118	458	20	val_118
 118	458	24	val_118
-118	459	10	val_118
+118	459	3	val_118
 118	459	24	val_118
 119	455	1	val_119
 119	455	9	val_119
-119	455	27	val_119
+119	455	28	val_119
 119	457	1	val_119
 119	457	9	val_119
-119	457	11	val_119
-119	457	13	val_119
-119	457	27	val_119
-119	457	27	val_119
-119	458	6	val_119
+119	457	10	val_119
+119	457	25	val_119
+119	457	28	val_119
+119	457	28	val_119
+119	458	5	val_119
 119	458	7	val_119
 119	458	10	val_119
-119	458	11	val_119
-119	458	13	val_119
+119	458	25	val_119
 119	458	27	val_119
-119	459	11	val_119
-119	459	13	val_119
-119	459	27	val_119
+119	458	28	val_119
+119	459	10	val_119
+119	459	25	val_119
+119	459	28	val_119
 120	455	10	val_120
-120	455	14	val_120
+120	455	27	val_120
 120	457	10	val_120
-120	457	14	val_120
 120	457	23	val_120
-120	457	28	val_120
-120	458	17	val_120
+120	457	23	val_120
+120	457	27	val_120
 120	458	19	val_120
 120	458	23	val_120
-120	458	28	val_120
+120	458	23	val_120
+120	458	23	val_120
+120	459	23	val_120
 120	459	23	val_120
-120	459	28	val_120
 125	455	1	val_125
 125	455	24	val_125
 125	457	1	val_125
 125	457	5	val_125
+125	457	23	val_125
 125	457	24	val_125
-125	457	28	val_125
+125	458	4	val_125
 125	458	5	val_125
 125	458	23	val_125
 125	458	27	val_125
-125	458	28	val_125
 125	459	5	val_125
-125	459	28	val_125
+125	459	23	val_125
 126	455	28	val_126
-126	457	1	val_126
+126	457	20	val_126
 126	457	28	val_126
-126	458	1	val_126
-126	458	3	val_126
-126	459	1	val_126
+126	458	16	val_126
+126	458	20	val_126
+126	459	20	val_126
+128	455	13	val_128
+128	455	19	val_128
 128	455	19	val_128
-128	455	24	val_128
-128	455	28	val_128
 128	457	2	val_128
-128	457	4	val_128
+128	457	5	val_128
+128	457	5	val_128
+128	457	13	val_128
+128	457	19	val_128
 128	457	19	val_128
-128	457	24	val_128
-128	457	27	val_128
-128	457	28	val_128
 128	458	2	val_128
-128	458	4	val_128
+128	458	3	val_128
+128	458	3	val_128
+128	458	5	val_128
+128	458	5	val_128
 128	458	9	val_128
-128	458	14	val_128
-128	458	19	val_128
-128	458	27	val_128
 128	459	2	val_128
-128	459	4	val_128
-128	459	27	val_128
-129	455	9	val_129
+128	459	5	val_128
+128	459	5	val_128
+129	455	4	val_129
 129	455	10	val_129
-129	457	9	val_129
+129	457	4	val_129
 129	457	10	val_129
 129	457	10	val_129
-129	457	14	val_129
+129	457	16	val_129
+129	458	9	val_129
 129	458	10	val_129
 129	458	10	val_129
-129	458	14	val_129
-129	458	25	val_129
+129	458	16	val_129
 129	459	10	val_129
-129	459	14	val_129
+129	459	16	val_129
 131	455	13	val_131
 131	457	13	val_131
 131	457	28	val_131
 131	458	4	val_131
 131	458	28	val_131
 131	459	28	val_131
-133	455	7	val_133
-133	457	7	val_133
-133	457	7	val_133
-133	458	5	val_133
-133	458	7	val_133
-133	459	7	val_133
+133	455	27	val_133
+133	457	13	val_133
+133	457	27	val_133
+133	458	13	val_133
+133	458	13	val_133
+133	459	13	val_133
 134	455	4	val_134
-134	455	7	val_134
+134	455	10	val_134
 134	457	4	val_134
-134	457	7	val_134
-134	457	11	val_134
+134	457	10	val_134
+134	457	10	val_134
 134	457	14	val_134
-134	458	3	val_134
-134	458	11	val_134
-134	458	14	val_134
+134	458	10	val_134
 134	458	14	val_134
-134	459	11	val_134
+134	458	23	val_134
+134	458	27	val_134
+134	459	10	val_134
 134	459	14	val_134
-136	455	24	val_136
-136	457	24	val_136
-136	457	27	val_136
-136	458	20	val_136
-136	458	27	val_136
-136	459	27	val_136
+136	455	10	val_136
+136	457	4	val_136
+136	457	10	val_136
+136	458	4	val_136
+136	458	6	val_136
+136	459	4	val_136
 137	455	10	val_137
-137	455	20	val_137
-137	457	3	val_137
+137	455	10	val_137
+137	457	10	val_137
+137	457	10	val_137
 137	457	10	val_137
-137	457	20	val_137
 137	457	27	val_137
-137	458	3	val_137
-137	458	6	val_137
-137	458	27	val_137
+137	458	9	val_137
+137	458	10	val_137
+137	458	10	val_137
 137	458	27	val_137
-137	459	3	val_137
+137	459	10	val_137
 137	459	27	val_137
 138	455	4	val_138
 138	455	10	val_138
-138	455	10	val_138
-138	455	13	val_138
-138	457	2	val_138
-138	457	2	val_138
+138	455	23	val_138
+138	455	27	val_138
 138	457	4	val_138
 138	457	10	val_138
 138	457	10	val_138
-138	457	13	val_138
-138	457	17	val_138
+138	457	23	val_138
+138	457	24	val_138
+138	457	27	val_138
+138	457	27	val_138
 138	457	27	val_138
-138	458	1	val_138
-138	458	2	val_138
-138	458	2	val_138
-138	458	9	val_138
-138	458	17	val_138
-138	458	20	val_138
+138	458	10	val_138
+138	458	10	val_138
+138	458	14	val_138
+138	458	24	val_138
+138	458	24	val_138
+138	458	24	val_138
 138	458	27	val_138
 138	458	27	val_138
-138	459	2	val_138
-138	459	2	val_138
-138	459	17	val_138
+138	459	10	val_138
+138	459	24	val_138
 138	459	27	val_138
-143	455	20	val_143
-143	457	20	val_143
-143	457	24	val_143
-143	458	17	val_143
-143	458	24	val_143
-143	459	24	val_143
+138	459	27	val_138
+143	455	4	val_143
+143	457	4	val_143
+143	457	25	val_143
+143	458	25	val_143
+143	458	28	val_143
+143	459	25	val_143
 145	455	28	val_145
 145	457	6	val_145
 145	457	28	val_145
@@ -2781,29 +2781,29 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
 145	458	6	val_145
 145	459	6	val_145
 146	455	4	val_146
-146	455	14	val_146
-146	457	0	val_146
+146	455	28	val_146
 146	457	3	val_146
 146	457	4	val_146
-146	457	14	val_146
-146	458	0	val_146
+146	457	6	val_146
+146	457	28	val_146
+146	458	1	val_146
 146	458	3	val_146
+146	458	6	val_146
 146	458	27	val_146
-146	458	28	val_146
-146	459	0	val_146
 146	459	3	val_146
+146	459	6	val_146
 149	455	19	val_149
-149	455	24	val_149
-149	457	9	val_149
-149	457	15	val_149
+149	455	23	val_149
+149	457	11	val_149
+149	457	14	val_149
 149	457	19	val_149
-149	457	24	val_149
-149	458	9	val_149
-149	458	15	val_149
-149	458	15	val_149
-149	458	28	val_149
-149	459	9	val_149
-149	459	15	val_149
+149	457	23	val_149
+149	458	1	val_149
+149	458	11	val_149
+149	458	14	val_149
+149	458	23	val_149
+149	459	11	val_149
+149	459	14	val_149
 150	455	10	val_150
 150	457	10	val_150
 150	457	14	val_150
@@ -2812,28 +2812,28 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
 150	459	14	val_150
 152	455	4	val_152
 152	455	23	val_152
-152	457	2	val_152
 152	457	4	val_152
+152	457	10	val_152
 152	457	20	val_152
 152	457	23	val_152
-152	458	2	val_152
 152	458	3	val_152
-152	458	14	val_152
+152	458	10	val_152
 152	458	20	val_152
-152	459	2	val_152
+152	458	24	val_152
+152	459	10	val_152
 152	459	20	val_152
-153	455	20	val_153
-153	457	15	val_153
-153	457	20	val_153
-153	458	15	val_153
-153	458	17	val_153
-153	459	15	val_153
-155	455	4	val_155
-155	457	3	val_155
-155	457	4	val_155
-155	458	1	val_155
-155	458	3	val_155
-155	459	3	val_155
+153	455	9	val_153
+153	457	1	val_153
+153	457	9	val_153
+153	458	1	val_153
+153	458	23	val_153
+153	459	1	val_153
+155	455	10	val_155
+155	457	10	val_155
+155	457	10	val_155
+155	458	10	val_155
+155	458	27	val_155
+155	459	10	val_155
 156	455	23	val_156
 156	457	20	val_156
 156	457	23	val_156
@@ -2846,42 +2846,42 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
 157	458	7	val_157
 157	458	9	val_157
 157	459	7	val_157
-158	455	20	val_158
-158	457	17	val_158
-158	457	20	val_158
-158	458	11	val_158
-158	458	17	val_158
-158	459	17	val_158
+158	455	23	val_158
+158	457	3	val_158
+158	457	23	val_158
+158	458	3	val_158
+158	458	20	val_158
+158	459	3	val_158
 160	455	10	val_160
 160	457	10	val_160
-160	457	27	val_160
-160	458	27	val_160
-160	458	27	val_160
-160	459	27	val_160
-162	455	9	val_162
-162	457	9	val_162
+160	457	10	val_160
+160	458	10	val_160
+160	458	10	val_160
+160	459	10	val_160
+162	455	10	val_162
 162	457	10	val_162
-162	458	10	val_162
-162	458	10	val_162
-162	459	10	val_162
+162	457	24	val_162
+162	458	24	val_162
+162	458	24	val_162
+162	459	24	val_162
 163	455	28	val_163
-163	457	6	val_163
+163	457	1	val_163
 163	457	28	val_163
-163	458	6	val_163
-163	458	28	val_163
-163	459	6	val_163
+163	458	1	val_163
+163	458	3	val_163
+163	459	1	val_163
 164	455	9	val_164
 164	455	27	val_164
+164	457	1	val_164
+164	457	7	val_164
 164	457	9	val_164
-164	457	10	val_164
-164	457	24	val_164
 164	457	27	val_164
 164	458	1	val_164
 164	458	7	val_164
-164	458	10	val_164
-164	458	24	val_164
-164	459	10	val_164
-164	459	24	val_164
+164	458	9	val_164
+164	458	28	val_164
+164	459	1	val_164
+164	459	7	val_164
 165	455	7	val_165
 165	455	24	val_165
 165	457	7	val_165
@@ -2894,203 +2894,203 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
 165	458	24	val_165
 165	459	7	val_165
 165	459	10	val_165
-166	455	9	val_166
-166	457	9	val_166
-166	457	17	val_166
-166	458	6	val_166
-166	458	17	val_166
-166	459	17	val_166
-167	455	7	val_167
-167	455	10	val_167
+166	455	24	val_166
+166	457	7	val_166
+166	457	24	val_166
+166	458	7	val_166
+166	458	24	val_166
+166	459	7	val_166
+167	455	27	val_167
+167	455	27	val_167
 167	455	27	val_167
 167	457	7	val_167
-167	457	10	val_167
-167	457	10	val_167
-167	457	25	val_167
+167	457	7	val_167
+167	457	24	val_167
+167	457	27	val_167
 167	457	27	val_167
 167	457	27	val_167
-167	458	10	val_167
-167	458	10	val_167
-167	458	25	val_167
-167	458	25	val_167
-167	458	25	val_167
-167	458	27	val_167
-167	459	10	val_167
-167	459	25	val_167
-167	459	27	val_167
-168	455	14	val_168
-168	457	14	val_168
-168	457	15	val_168
-168	458	4	val_168
-168	458	15	val_168
-168	459	15	val_168
-169	455	9	val_169
+167	458	7	val_167
+167	458	7	val_167
+167	458	7	val_167
+167	458	7	val_167
+167	458	24	val_167
+167	458	24	val_167
+167	459	7	val_167
+167	459	7	val_167
+167	459	24	val_167
+168	455	13	val_168
+168	457	7	val_168
+168	457	13	val_168
+168	458	2	val_168
+168	458	7	val_168
+168	459	7	val_168
+169	455	24	val_169
 169	455	24	val_169
 169	455	27	val_169
 169	455	28	val_169
+169	457	1	val_169
+169	457	6	val_169
 169	457	9	val_169
-169	457	14	val_169
-169	457	15	val_169
 169	457	16	val_169
 169	457	24	val_169
-169	457	25	val_169
+169	457	24	val_169
 169	457	27	val_169
 169	457	28	val_169
-169	458	2	val_169
+169	458	1	val_169
+169	458	1	val_169
+169	458	3	val_169
 169	458	6	val_169
+169	458	9	val_169
+169	458	9	val_169
 169	458	14	val_169
-169	458	14	val_169
-169	458	15	val_169
 169	458	16	val_169
-169	458	25	val_169
-169	458	28	val_169
-169	459	14	val_169
-169	459	15	val_169
+169	459	1	val_169
+169	459	6	val_169
+169	459	9	val_169
 169	459	16	val_169
-169	459	25	val_169
 170	455	27	val_170
 170	457	27	val_170
 170	457	27	val_170
 170	458	27	val_170
 170	458	27	val_170
 170	459	27	val_170
-172	455	9	val_172
-172	455	24	val_172
-172	457	2	val_172
-172	457	9	val_172
-172	457	17	val_172
-172	457	24	val_172
-172	458	2	val_172
-172	458	6	val_172
+172	455	10	val_172
+172	455	13	val_172
+172	457	10	val_172
+172	457	10	val_172
+172	457	13	val_172
+172	457	27	val_172
 172	458	10	val_172
-172	458	17	val_172
-172	459	2	val_172
-172	459	17	val_172
+172	458	14	val_172
+172	458	27	val_172
+172	458	27	val_172
+172	459	10	val_172
+172	459	27	val_172
 174	455	10	val_174
-174	455	24	val_174
-174	457	1	val_174
-174	457	7	val_174
+174	455	19	val_174
+174	457	6	val_174
 174	457	10	val_174
-174	457	24	val_174
-174	458	1	val_174
-174	458	1	val_174
-174	458	7	val_174
-174	458	7	val_174
-174	459	1	val_174
-174	459	7	val_174
-175	455	10	val_175
-175	455	23	val_175
-175	457	4	val_175
-175	457	10	val_175
-175	457	20	val_175
-175	457	23	val_175
-175	458	1	val_175
-175	458	4	val_175
-175	458	7	val_175
-175	458	20	val_175
-175	459	4	val_175
-175	459	20	val_175
+174	457	16	val_174
+174	457	19	val_174
+174	458	6	val_174
+174	458	10	val_174
+174	458	16	val_174
+174	458	20	val_174
+174	459	6	val_174
+174	459	16	val_174
+175	455	13	val_175
+175	455	24	val_175
+175	457	13	val_175
+175	457	14	val_175
+175	457	19	val_175
+175	457	24	val_175
+175	458	14	val_175
+175	458	19	val_175
+175	458	19	val_175
+175	458	23	val_175
+175	459	14	val_175
+175	459	19	val_175
 176	455	9	val_176
-176	455	16	val_176
+176	455	9	val_176
+176	457	4	val_176
 176	457	6	val_176
 176	457	9	val_176
-176	457	15	val_176
-176	457	16	val_176
-176	458	3	val_176
+176	457	9	val_176
+176	458	4	val_176
 176	458	6	val_176
 176	458	7	val_176
-176	458	15	val_176
+176	458	19	val_176
+176	459	4	val_176
 176	459	6	val_176
-176	459	15	val_176
-177	455	19	val_177
-177	457	19	val_177
-177	457	25	val_177
-177	458	5	val_177
-177	458	25	val_177
-177	459	25	val_177
-178	455	10	val_178
-178	457	7	val_178
-178	457	10	val_178
-178	458	7	val_178
-178	458	7	val_178
-178	459	7	val_178
-179	455	1	val_179
-179	455	28	val_179
+177	455	10	val_177
+177	457	10	val_177
+177	457	28	val_177
+177	458	28	val_177
+177	458	28	val_177
+177	459	28	val_177
+178	455	9	val_178
+178	457	2	val_178
+178	457	9	val_178
+178	458	2	val_178
+178	458	2	val_178
+178	459	2	val_178
+179	455	10	val_179
+179	455	13	val_179
 179	457	1	val_179
-179	457	3	val_179
-179	457	15	val_179
-179	457	28	val_179
+179	457	10	val_179
+179	457	13	val_179
+179	457	27	val_179
 179	458	1	val_179
-179	458	3	val_179
-179	458	15	val_179
-179	458	17	val_179
-179	459	3	val_179
-179	459	15	val_179
-180	455	24	val_180
-180	457	10	val_180
-180	457	24	val_180
-180	458	7	val_180
+179	458	19	val_179
+179	458	24	val_179
+179	458	27	val_179
+179	459	1	val_179
+179	459	27	val_179
+180	455	19	val_180
+180	457	3	val_180
+180	457	19	val_180
+180	458	3	val_180
 180	458	10	val_180
-180	459	10	val_180
-181	455	7	val_181
-181	457	7	val_181
+180	459	3	val_180
+181	455	23	val_181
+181	457	20	val_181
 181	457	23	val_181
-181	458	19	val_181
-181	458	23	val_181
-181	459	23	val_181
+181	458	3	val_181
+181	458	20	val_181
+181	459	20	val_181
 183	455	9	val_183
-183	457	4	val_183
 183	457	9	val_183
-183	458	4	val_183
+183	457	19	val_183
+183	458	13	val_183
 183	458	19	val_183
-183	459	4	val_183
+183	459	19	val_183
 186	455	10	val_186
 186	457	10	val_186
-186	457	24	val_186
-186	458	24	val_186
-186	458	24	val_186
-186	459	24	val_186
+186	457	27	val_186
+186	458	27	val_186
+186	458	27	val_186
+186	459	27	val_186
 187	455	7	val_187
 187	455	9	val_187
 187	455	14	val_187
 187	457	7	val_187
 187	457	9	val_187
-187	457	9	val_187
 187	457	11	val_187
 187	457	14	val_187
+187	457	14	val_187
 187	457	25	val_187
 187	458	5	val_187
-187	458	9	val_187
 187	458	11	val_187
 187	458	14	val_187
+187	458	24	val_187
 187	458	25	val_187
 187	458	28	val_187
-187	459	9	val_187
 187	459	11	val_187
+187	459	14	val_187
 187	459	25	val_187
-189	455	13	val_189
-189	457	1	val_189
-189	457	13	val_189
-189	458	1	val_189
-189	458	20	val_189
-189	459	1	val_189
-190	455	24	val_190
-190	457	2	val_190
-190	457	24	val_190
-190	458	2	val_190
-190	458	3	val_190
-190	459	2	val_190
-191	455	10	val_191
+189	455	9	val_189
+189	457	4	val_189
+189	457	9	val_189
+189	458	4	val_189
+189	458	4	val_189
+189	459	4	val_189
+190	455	23	val_190
+190	457	23	val_190
+190	457	28	val_190
+190	458	27	val_190
+190	458	28	val_190
+190	459	28	val_190
+191	455	7	val_191
 191	455	27	val_191
+191	457	2	val_191
 191	457	7	val_191
-191	457	10	val_191
 191	457	27	val_191
 191	457	27	val_191
-191	458	7	val_191
-191	458	7	val_191
+191	458	2	val_191
+191	458	10	val_191
 191	458	27	val_191
 191	458	27	val_191
-191	459	7	val_191
+191	459	2	val_191
 191	459	27	val_191
 192	455	24	val_192
 192	457	6	val_192
@@ -3099,685 +3099,685 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
 192	458	6	val_192
 192	459	6	val_192
 193	455	7	val_193
-193	455	19	val_193
-193	455	27	val_193
+193	455	7	val_193
+193	455	7	val_193
 193	457	1	val_193
 193	457	7	val_193
+193	457	7	val_193
+193	457	7	val_193
 193	457	19	val_193
-193	457	20	val_193
 193	457	24	val_193
-193	457	27	val_193
 193	458	1	val_193
 193	458	4	val_193
-193	458	16	val_193
-193	458	20	val_193
+193	458	9	val_193
+193	458	19	val_193
 193	458	24	val_193
-193	458	27	val_193
+193	458	28	val_193
 193	459	1	val_193
-193	459	20	val_193
+193	459	19	val_193
 193	459	24	val_193
 194	455	23	val_194
+194	457	3	val_194
 194	457	23	val_194
-194	457	28	val_194
-194	458	27	val_194
-194	458	28	val_194
-194	459	28	val_194
-195	455	10	val_195
+194	458	3	val_194
+194	458	20	val_194
+194	459	3	val_194
+195	455	24	val_195
 195	455	27	val_195
-195	457	10	val_195
-195	457	11	val_195
+195	457	14	val_195
 195	457	24	val_195
 195	457	27	val_195
-195	458	11	val_195
-195	458	11	val_195
-195	458	24	val_195
-195	458	24	val_195
-195	459	11	val_195
-195	459	24	val_195
+195	457	27	val_195
+195	458	14	val_195
+195	458	14	val_195
+195	458	27	val_195
+195	458	27	val_195
+195	459	14	val_195
+195	459	27	val_195
 196	455	1	val_196
 196	457	1	val_196
 196	457	3	val_196
 196	458	3	val_196
 196	458	20	val_196
 196	459	3	val_196
-197	455	2	val_197
 197	455	14	val_197
-197	457	2	val_197
+197	455	24	val_197
+197	457	1	val_197
 197	457	6	val_197
 197	457	14	val_197
-197	457	28	val_197
-197	458	3	val_197
+197	457	24	val_197
+197	458	1	val_197
 197	458	3	val_197
 197	458	6	val_197
-197	458	28	val_197
+197	458	16	val_197
+197	459	1	val_197
 197	459	6	val_197
-197	459	28	val_197
 199	455	10	val_199
-199	455	24	val_199
-199	455	27	val_199
-199	457	4	val_199
-199	457	5	val_199
+199	455	10	val_199
+199	455	13	val_199
+199	457	7	val_199
+199	457	10	val_199
 199	457	10	val_199
 199	457	10	val_199
-199	457	24	val_199
-199	457	27	val_199
-199	458	4	val_199
-199	458	4	val_199
-199	458	5	val_199
+199	457	13	val_199
+199	457	19	val_199
+199	458	7	val_199
 199	458	10	val_199
 199	458	10	val_199
-199	458	11	val_199
-199	459	4	val_199
-199	459	5	val_199
+199	458	19	val_199
+199	458	19	val_199
+199	458	28	val_199
+199	459	7	val_199
 199	459	10	val_199
-200	456	2	val_200
+199	459	19	val_199
 200	456	10	val_200
-200	457	4	val_200
-200	457	4	val_200
-200	458	1	val_200
-200	458	4	val_200
-200	458	4	val_200
+200	456	27	val_200
+200	457	6	val_200
+200	457	6	val_200
+200	458	6	val_200
+200	458	6	val_200
+200	458	7	val_200
 200	458	20	val_200
-200	459	4	val_200
-200	459	4	val_200
-201	456	24	val_201
-201	457	9	val_201
-201	458	9	val_201
+200	459	6	val_200
+200	459	6	val_200
+201	456	27	val_201
+201	457	24	val_201
 201	458	24	val_201
-201	459	9	val_201
-202	456	4	val_202
+201	458	27	val_201
+201	459	24	val_201
+202	456	2	val_202
 202	457	1	val_202
 202	458	1	val_202
-202	458	28	val_202
+202	458	7	val_202
 202	459	1	val_202
-203	456	6	val_203
 203	456	20	val_203
-203	457	17	val_203
-203	457	24	val_203
-203	458	3	val_203
-203	458	17	val_203
-203	458	24	val_203
-203	458	27	val_203
-203	459	17	val_203
-203	459	24	val_203
-205	456	4	val_205
+203	456	24	val_203
+203	457	2	val_203
+203	457	25	val_203
+203	458	2	val_203
+203	458	10	val_203
+203	458	25	val_203
+203	458	28	val_203
+203	459	2	val_203
+203	459	25	val_203
+205	456	2	val_205
 205	456	9	val_205
 205	457	4	val_205
-205	457	14	val_205
+205	457	25	val_205
 205	458	4	val_205
-205	458	5	val_205
 205	458	13	val_205
-205	458	14	val_205
+205	458	20	val_205
+205	458	25	val_205
 205	459	4	val_205
-205	459	14	val_205
-207	456	13	val_207
+205	459	25	val_205
+207	456	27	val_207
 207	456	28	val_207
 207	457	1	val_207
-207	457	19	val_207
+207	457	5	val_207
 207	458	1	val_207
 207	458	1	val_207
-207	458	19	val_207
-207	458	19	val_207
+207	458	3	val_207
+207	458	5	val_207
 207	459	1	val_207
-207	459	19	val_207
-208	456	4	val_208
-208	456	10	val_208
+207	459	5	val_207
+208	456	2	val_208
 208	456	13	val_208
-208	457	25	val_208
+208	456	19	val_208
+208	457	1	val_208
+208	457	1	val_208
 208	457	27	val_208
-208	457	28	val_208
-208	458	25	val_208
+208	458	1	val_208
+208	458	1	val_208
+208	458	4	val_208
+208	458	7	val_208
 208	458	27	val_208
 208	458	28	val_208
-208	458	28	val_208
-208	458	28	val_208
-208	458	28	val_208
-208	459	25	val_208
+208	459	1	val_208
+208	459	1	val_208
 208	459	27	val_208
-208	459	28	val_208
-209	456	1	val_209
-209	456	3	val_209
+209	456	10	val_209
+209	456	28	val_209
 209	457	3	val_209
-209	457	28	val_209
+209	457	10	val_209
 209	458	3	val_209
-209	458	25	val_209
-209	458	27	val_209
-209	458	28	val_209
+209	458	10	val_209
+209	458	10	val_209
+209	458	20	val_209
 209	459	3	val_209
-209	459	28	val_209
+209	459	10	val_209
 213	456	1	val_213
-213	456	10	val_213
+213	456	27	val_213
+213	457	7	val_213
 213	457	16	val_213
-213	457	17	val_213
-213	458	11	val_213
+213	458	7	val_213
+213	458	9	val_213
 213	458	16	val_213
-213	458	17	val_213
 213	458	20	val_213
+213	459	7	val_213
 213	459	16	val_213
-213	459	17	val_213
-214	456	20	val_214
-214	457	16	val_214
-214	458	2	val_214
-214	458	16	val_214
-214	459	16	val_214
-216	456	2	val_216
+214	456	27	val_214
+214	457	24	val_214
+214	458	24	val_214
+214	458	24	val_214
+214	459	24	val_214
 216	456	10	val_216
-216	457	11	val_216
+216	456	24	val_216
+216	457	10	val_216
 216	457	27	val_216
-216	458	6	val_216
-216	458	11	val_216
-216	458	20	val_216
+216	458	7	val_216
+216	458	10	val_216
+216	458	27	val_216
 216	458	27	val_216
-216	459	11	val_216
+216	459	10	val_216
 216	459	27	val_216
-217	456	9	val_217
 217	456	24	val_217
-217	457	7	val_217
-217	457	16	val_217
-217	458	7	val_217
+217	456	24	val_217
+217	457	9	val_217
+217	457	9	val_217
+217	458	9	val_217
+217	458	9	val_217
 217	458	9	val_217
-217	458	16	val_217
-217	458	27	val_217
-217	459	7	val_217
-217	459	16	val_217
-218	456	4	val_218
-218	457	27	val_218
-218	458	24	val_218
-218	458	27	val_218
-218	459	27	val_218
-219	456	6	val_219
-219	456	28	val_219
-219	457	5	val_219
-219	457	28	val_219
-219	458	5	val_219
-219	458	6	val_219
-219	458	11	val_219
-219	458	28	val_219
-219	459	5	val_219
-219	459	28	val_219
-221	456	5	val_221
+217	458	24	val_217
+217	459	9	val_217
+217	459	9	val_217
+218	456	19	val_218
+218	457	20	val_218
+218	458	1	val_218
+218	458	20	val_218
+218	459	20	val_218
+219	456	7	val_219
+219	456	13	val_219
+219	457	13	val_219
+219	457	23	val_219
+219	458	13	val_219
+219	458	13	val_219
+219	458	19	val_219
+219	458	23	val_219
+219	459	13	val_219
+219	459	23	val_219
+221	456	13	val_221
 221	456	24	val_221
 221	457	13	val_221
-221	457	14	val_221
+221	457	28	val_221
 221	458	10	val_221
-221	458	11	val_221
 221	458	13	val_221
-221	458	14	val_221
+221	458	23	val_221
+221	458	28	val_221
 221	459	13	val_221
-221	459	14	val_221
-222	456	9	val_222
-222	457	20	val_222
-222	458	16	val_222
-222	458	20	val_222
-222	459	20	val_222
-223	456	10	val_223
-223	456	20	val_223
-223	457	15	val_223
-223	457	24	val_223
-223	458	4	val_223
-223	458	15	val_223
-223	458	17	val_223
-223	458	24	val_223
-223	459	15	val_223
-223	459	24	val_223
-224	456	4	val_224
+221	459	28	val_221
+222	456	14	val_222
+222	457	7	val_222
+222	458	2	val_222
+222	458	7	val_222
+222	459	7	val_222
+223	456	7	val_223
+223	456	19	val_223
+223	457	1	val_223
+223	457	28	val_223
+223	458	1	val_223
+223	458	19	val_223
+223	458	28	val_223
+223	458	28	val_223
+223	459	1	val_223
+223	459	28	val_223
+224	456	9	val_224
 224	456	23	val_224
+224	457	4	val_224
 224	457	20	val_224
-224	457	20	val_224
-224	458	20	val_224
+224	458	4	val_224
 224	458	20	val_224
 224	458	24	val_224
 224	458	28	val_224
+224	459	4	val_224
 224	459	20	val_224
-224	459	20	val_224
-226	456	3	val_226
-226	457	11	val_226
-226	458	7	val_226
-226	458	11	val_226
-226	459	11	val_226
-228	456	23	val_228
-228	457	10	val_228
-228	458	10	val_228
-228	458	14	val_228
-228	459	10	val_228
-229	456	25	val_229
-229	456	28	val_229
-229	457	14	val_229
-229	457	14	val_229
+226	456	28	val_226
+226	457	1	val_226
+226	458	1	val_226
+226	458	1	val_226
+226	459	1	val_226
+228	456	4	val_228
+228	457	27	val_228
+228	458	24	val_228
+228	458	27	val_228
+228	459	27	val_228
+229	456	3	val_229
+229	456	10	val_229
+229	457	1	val_229
+229	457	25	val_229
+229	458	1	val_229
 229	458	1	val_229
-229	458	14	val_229
-229	458	14	val_229
-229	458	24	val_229
-229	459	14	val_229
-229	459	14	val_229
-230	456	7	val_230
+229	458	16	val_229
+229	458	25	val_229
+229	459	1	val_229
+229	459	25	val_229
 230	456	9	val_230
+230	456	13	val_230
+230	456	14	val_230
+230	456	14	val_230
 230	456	24	val_230
-230	456	24	val_230
-230	456	27	val_230
-230	457	4	val_230
-230	457	6	val_230
-230	457	11	val_230
+230	457	7	val_230
 230	457	19	val_230
-230	457	24	val_230
-230	458	3	val_230
-230	458	4	val_230
-230	458	6	val_230
+230	457	25	val_230
+230	457	27	val_230
+230	457	28	val_230
+230	458	2	val_230
+230	458	7	val_230
 230	458	10	val_230
-230	458	11	val_230
-230	458	13	val_230
 230	458	16	val_230
 230	458	19	val_230
-230	458	24	val_230
-230	458	24	val_230
-230	459	4	val_230
-230	459	6	val_230
-230	459	11	val_230
+230	458	19	val_230
+230	458	25	val_230
+230	458	27	val_230
+230	458	27	val_230
+230	458	28	val_230
+230	459	7	val_230
 230	459	19	val_230
-230	459	24	val_230
-233	456	10	val_233
-233	456	11	val_233
-233	457	15	val_233
-233	457	28	val_233
+230	459	25	val_230
+230	459	27	val_230
+230	459	28	val_230
+233	456	2	val_233
+233	456	9	val_233
+233	457	6	val_233
+233	457	7	val_233
 233	458	3	val_233
-233	458	3	val_233
-233	458	15	val_233
-233	458	28	val_233
-233	459	15	val_233
-233	459	28	val_233
-235	456	27	val_235
-235	457	28	val_235
-235	458	2	val_235
-235	458	28	val_235
-235	459	28	val_235
-237	456	3	val_237
+233	458	6	val_233
+233	458	7	val_233
+233	458	7	val_233
+233	459	6	val_233
+233	459	7	val_233
+235	456	24	val_235
+235	457	7	val_235
+235	458	7	val_235
+235	458	24	val_235
+235	459	7	val_235
+237	456	10	val_237
 237	456	24	val_237
-237	457	3	val_237
-237	457	24	val_237
-237	458	3	val_237
-237	458	24	val_237
-237	458	24	val_237
-237	458	25	val_237
-237	459	3	val_237
-237	459	24	val_237
+237	457	10	val_237
+237	457	10	val_237
+237	458	10	val_237
+237	458	10	val_237
+237	458	10	val_237
+237	458	20	val_237
+237	459	10	val_237
+237	459	10	val_237
 238	456	10	val_238
-238	456	27	val_238
+238	456	19	val_238
+238	457	10	val_238
 238	457	10	val_238
-238	457	24	val_238
 238	458	10	val_238
 238	458	10	val_238
-238	458	24	val_238
-238	458	24	val_238
+238	458	10	val_238
+238	458	14	val_238
 238	459	10	val_238
-238	459	24	val_238
-239	456	5	val_239
-239	456	27	val_239
-239	457	14	val_239
-239	457	15	val_239
-239	458	4	val_239
-239	458	14	val_239
-239	458	15	val_239
-239	458	25	val_239
-239	459	14	val_239
-239	459	15	val_239
-241	456	27	val_241
-241	457	24	val_241
-241	458	24	val_241
-241	458	24	val_241
-241	459	24	val_241
-242	456	2	val_242
+238	459	10	val_238
+239	456	10	val_239
+239	456	10	val_239
+239	457	9	val_239
+239	457	19	val_239
+239	458	9	val_239
+239	458	19	val_239
+239	458	19	val_239
+239	458	24	val_239
+239	459	9	val_239
+239	459	19	val_239
+241	456	1	val_241
+241	457	10	val_241
+241	458	10	val_241
+241	458	20	val_241
+241	459	10	val_241
+242	456	4	val_242
 242	456	24	val_242
-242	457	1	val_242
-242	457	16	val_242
-242	458	1	val_242
-242	458	3	val_242
-242	458	9	val_242
-242	458	16	val_242
-242	459	1	val_242
-242	459	16	val_242
-244	456	7	val_244
-244	457	13	val_244
-244	458	10	val_244
-244	458	13	val_244
-244	459	13	val_244
-247	456	1	val_247
-247	457	9	val_247
-247	458	9	val_247
-247	458	9	val_247
-247	459	9	val_247
-248	456	7	val_248
-248	457	14	val_248
-248	458	4	val_248
-248	458	14	val_248
-248	459	14	val_248
-249	456	1	val_249
-249	457	24	val_249
-249	458	24	val_249
-249	458	24	val_249
-249	459	24	val_249
-252	456	20	val_252
-252	457	5	val_252
-252	458	3	val_252
-252	458	5	val_252
-252	459	5	val_252
+242	457	4	val_242
+242	457	24	val_242
+242	458	4	val_242
+242	458	4	val_242
+242	458	24	val_242
+242	458	24	val_242
+242	459	4	val_242
+242	459	24	val_242
+244	456	9	val_244
+244	457	3	val_244
+244	458	2	val_244
+244	458	3	val_244
+244	459	3	val_244
+247	456	28	val_247
+247	457	6	val_247
+247	458	6	val_247
+247	458	25	val_247
+247	459	6	val_247
+248	456	9	val_248
+248	457	9	val_248
+248	458	9	val_248
+248	458	9	val_248
+248	459	9	val_248
+249	456	5	val_249
+249	457	6	val_249
+249	458	6	val_249
+249	458	6	val_249
+249	459	6	val_249
+252	456	7	val_252
+252	457	19	val_252
+252	458	10	val_252
+252	458	19	val_252
+252	459	19	val_252
+255	456	9	val_255
 255	456	9	val_255
-255	456	20	val_255
-255	457	7	val_255
+255	457	5	val_255
 255	457	13	val_255
-255	458	7	val_255
+255	458	2	val_255
+255	458	5	val_255
 255	458	7	val_255
 255	458	13	val_255
-255	458	16	val_255
-255	459	7	val_255
+255	459	5	val_255
 255	459	13	val_255
 256	456	1	val_256
-256	456	1	val_256
-256	457	3	val_256
-256	457	24	val_256
+256	456	4	val_256
+256	457	1	val_256
+256	457	20	val_256
 256	458	1	val_256
-256	458	3	val_256
-256	458	11	val_256
-256	458	24	val_256
-256	459	3	val_256
-256	459	24	val_256
-257	456	7	val_257
-257	457	14	val_257
-257	458	6	val_257
+256	458	1	val_256
+256	458	20	val_256
+256	458	20	val_256
+256	459	1	val_256
+256	459	20	val_256
+257	456	23	val_257
+257	457	2	val_257
+257	458	2	val_257
 257	458	14	val_257
-257	459	14	val_257
-258	456	11	val_258
-258	457	14	val_258
-258	458	14	val_258
-258	458	17	val_258
-258	459	14	val_258
-260	456	10	val_260
-260	457	1	val_260
-260	458	1	val_260
-260	458	4	val_260
-260	459	1	val_260
-262	456	4	val_262
-262	457	14	val_262
-262	458	2	val_262
-262	458	14	val_262
-262	459	14	val_262
-263	456	6	val_263
-263	457	14	val_263
-263	458	14	val_263
-263	458	14	val_263
-263	459	14	val_263
+257	459	2	val_257
+258	456	9	val_258
+258	457	1	val_258
+258	458	1	val_258
+258	458	3	val_258
+258	459	1	val_258
+260	456	20	val_260
+260	457	6	val_260
+260	458	6	val_260
+260	458	28	val_260
+260	459	6	val_260
+262	456	27	val_262
+262	457	11	val_262
+262	458	5	val_262
+262	458	11	val_262
+262	459	11	val_262
+263	456	23	val_263
+263	457	24	val_263
+263	458	24	val_263
+263	458	24	val_263
+263	459	24	val_263
+265	456	7	val_265
 265	456	13	val_265
-265	456	24	val_265
-265	457	14	val_265
+265	457	25	val_265
 265	457	28	val_265
-265	458	14	val_265
-265	458	14	val_265
 265	458	19	val_265
+265	458	25	val_265
+265	458	25	val_265
 265	458	28	val_265
-265	459	14	val_265
+265	459	25	val_265
 265	459	28	val_265
-266	456	2	val_266
-266	457	5	val_266
-266	458	5	val_266
-266	458	5	val_266
-266	459	5	val_266
-272	456	1	val_272
-272	456	20	val_272
-272	457	5	val_272
+266	456	13	val_266
+266	457	19	val_266
+266	458	19	val_266
+266	458	19	val_266
+266	459	19	val_266
+272	456	10	val_272
+272	456	27	val_272
 272	457	10	val_272
-272	458	5	val_272
-272	458	9	val_272
+272	457	27	val_272
 272	458	10	val_272
 272	458	10	val_272
-272	459	5	val_272
+272	458	27	val_272
+272	458	27	val_272
 272	459	10	val_272
+272	459	27	val_272
 273	456	4	val_273
-273	456	7	val_273
-273	456	13	val_273
+273	456	9	val_273
+273	456	24	val_273
 273	457	9	val_273
-273	457	14	val_273
-273	457	19	val_273
+273	457	11	val_273
+273	457	13	val_273
 273	458	1	val_273
-273	458	4	val_273
 273	458	9	val_273
+273	458	10	val_273
+273	458	11	val_273
 273	458	13	val_273
-273	458	14	val_273
-273	458	19	val_273
+273	458	16	val_273
 273	459	9	val_273
-273	459	14	val_273
-273	459	19	val_273
-274	456	3	val_274
-274	457	2	val_274
-274	458	2	val_274
-274	458	11	val_274
-274	459	2	val_274
-275	456	10	val_275
-275	457	10	val_275
-275	458	10	val_275
-275	458	10	val_275
-275	459	10	val_275
-277	456	10	val_277
+273	459	11	val_273
+273	459	13	val_273
+274	456	9	val_274
+274	457	10	val_274
+274	458	10	val_274
+274	458	10	val_274
+274	459	10	val_274
+275	456	28	val_275
+275	457	20	val_275
+275	458	14	val_275
+275	458	20	val_275
+275	459	20	val_275
+277	456	1	val_277
 277	456	10	val_277
 277	456	16	val_277
-277	456	27	val_277
+277	456	24	val_277
 277	457	1	val_277
 277	457	20	val_277
-277	457	24	val_277
-277	457	27	val_277
+277	457	20	val_277
+277	457	25	val_277
 277	458	1	val_277
+277	458	2	val_277
 277	458	6	val_277
-277	458	19	val_277
+277	458	20	val_277
+277	458	20	val_277
 277	458	20	val_277
 277	458	23	val_277
-277	458	24	val_277
-277	458	27	val_277
-277	458	27	val_277
+277	458	25	val_277
 277	459	1	val_277
 277	459	20	val_277
-277	459	24	val_277
-277	459	27	val_277
-278	456	3	val_278
+277	459	20	val_277
+277	459	25	val_277
+278	456	7	val_278
 278	456	7	val_278
-278	457	5	val_278
 278	457	19	val_278
-278	458	1	val_278
-278	458	5	val_278
+278	457	19	val_278
 278	458	10	val_278
+278	458	13	val_278
+278	458	19	val_278
 278	458	19	val_278
-278	459	5	val_278
 278	459	19	val_278
+278	459	19	val_278
+280	456	7	val_280
 280	456	9	val_280
-280	456	14	val_280
+280	457	4	val_280
 280	457	24	val_280
-280	457	27	val_280
+280	458	4	val_280
+280	458	11	val_280
 280	458	19	val_280
-280	458	20	val_280
 280	458	24	val_280
-280	458	27	val_280
+280	459	4	val_280
 280	459	24	val_280
-280	459	27	val_280
 281	456	2	val_281
-281	456	10	val_281
+281	456	7	val_281
+281	457	4	val_281
 281	457	4	val_281
-281	457	23	val_281
 281	458	1	val_281
 281	458	4	val_281
 281	458	4	val_281
-281	458	23	val_281
+281	458	19	val_281
+281	459	4	val_281
 281	459	4	val_281
-281	459	23	val_281
-282	456	10	val_282
+282	456	13	val_282
 282	456	24	val_282
 282	457	1	val_282
-282	457	10	val_282
+282	457	20	val_282
+282	458	1	val_282
 282	458	1	val_282
-282	458	10	val_282
-282	458	10	val_282
 282	458	16	val_282
+282	458	20	val_282
 282	459	1	val_282
-282	459	10	val_282
-283	456	9	val_283
-283	457	16	val_283
-283	458	16	val_283
-283	458	27	val_283
-283	459	16	val_283
-284	456	9	val_284
-284	457	13	val_284
-284	458	10	val_284
-284	458	13	val_284
-284	459	13	val_284
-285	456	24	val_285
-285	457	9	val_285
-285	458	9	val_285
-285	458	9	val_285
-285	459	9	val_285
+282	459	20	val_282
+283	456	10	val_283
+283	457	24	val_283
+283	458	10	val_283
+283	458	24	val_283
+283	459	24	val_283
+284	456	14	val_284
+284	457	28	val_284
+284	458	2	val_284
+284	458	28	val_284
+284	459	28	val_284
+285	456	27	val_285
+285	457	24	val_285
+285	458	24	val_285
+285	458	24	val_285
+285	459	24	val_285
 286	456	9	val_286
 286	457	25	val_286
 286	458	25	val_286
 286	458	28	val_286
 286	459	25	val_286
-287	456	25	val_287
-287	457	14	val_287
-287	458	14	val_287
-287	458	14	val_287
-287	459	14	val_287
-288	456	9	val_288
+287	456	19	val_287
+287	457	28	val_287
+287	458	4	val_287
+287	458	28	val_287
+287	459	28	val_287
+288	456	14	val_288
 288	456	23	val_288
-288	457	9	val_288
+288	457	10	val_288
 288	457	28	val_288
-288	458	9	val_288
-288	458	9	val_288
+288	458	10	val_288
+288	458	27	val_288
 288	458	27	val_288
 288	458	28	val_288
-288	459	9	val_288
+288	459	10	val_288
 288	459	28	val_288
-289	456	28	val_289
-289	457	28	val_289
-289	458	17	val_289
-289	458	28	val_289
-289	459	28	val_289
-291	456	14	val_291
-291	457	3	val_291
-291	458	3	val_291
-291	458	10	val_291
-291	459	3	val_291
-292	456	1	val_292
-292	457	15	val_292
-292	458	15	val_292
-292	458	17	val_292
-292	459	15	val_292
-296	456	27	val_296
-296	457	3	val_296
-296	458	3	val_296
-296	458	5	val_296
-296	459	3	val_296
-298	456	4	val_298
-298	456	6	val_298
-298	456	19	val_298
-298	457	0	val_298
-298	457	14	val_298
-298	457	23	val_298
-298	458	0	val_298
-298	458	2	val_298
+289	456	10	val_289
+289	457	23	val_289
+289	458	4	val_289
+289	458	23	val_289
+289	459	23	val_289
+291	456	24	val_291
+291	457	16	val_291
+291	458	16	val_291
+291	458	27	val_291
+291	459	16	val_291
+292	456	10	val_292
+292	457	4	val_292
+292	458	4	val_292
+292	458	13	val_292
+292	459	4	val_292
+296	456	20	val_296
+296	457	1	val_296
+296	458	1	val_296
+296	458	28	val_296
+296	459	1	val_296
+298	456	13	val_298
+298	456	27	val_298
+298	456	27	val_298
+298	457	1	val_298
+298	457	4	val_298
+298	457	4	val_298
+298	458	1	val_298
+298	458	4	val_298
 298	458	4	val_298
-298	458	14	val_298
-298	458	23	val_298
-298	458	27	val_298
-298	459	0	val_298
-298	459	14	val_298
-298	459	23	val_298
-302	456	6	val_302
-302	457	28	val_302
-302	458	2	val_302
-302	458	28	val_302
-302	459	28	val_302
-305	456	1	val_305
-305	457	27	val_305
-305	458	3	val_305
-305	458	27	val_305
-305	459	27	val_305
-306	456	19	val_306
-306	457	9	val_306
-306	458	9	val_306
-306	458	23	val_306
-306	459	9	val_306
-307	456	1	val_307
-307	456	9	val_307
-307	457	3	val_307
-307	457	4	val_307
-307	458	3	val_307
-307	458	4	val_307
-307	458	10	val_307
-307	458	20	val_307
-307	459	3	val_307
-307	459	4	val_307
+298	458	5	val_298
+298	458	19	val_298
+298	458	20	val_298
+298	459	1	val_298
+298	459	4	val_298
+298	459	4	val_298
+302	456	10	val_302
+302	457	6	val_302
+302	458	6	val_302
+302	458	7	val_302
+302	459	6	val_302
+305	456	10	val_305
+305	457	10	val_305
+305	458	10	val_305
+305	458	10	val_305
+305	459	10	val_305
+306	456	14	val_306
+306	457	6	val_306
+306	458	3	val_306
+306	458	6	val_306
+306	459	6	val_306
+307	456	23	val_307
+307	456	28	val_307
+307	457	7	val_307
+307	457	16	val_307
+307	458	7	val_307
+307	458	9	val_307
+307	458	16	val_307
+307	458	27	val_307
+307	459	7	val_307
+307	459	16	val_307
 308	456	4	val_308
 308	457	2	val_308
 308	458	2	val_308
 308	458	9	val_308
 308	459	2	val_308
-309	456	23	val_309
-309	456	27	val_309
-309	457	3	val_309
-309	457	27	val_309
+309	456	7	val_309
+309	456	20	val_309
+309	457	1	val_309
+309	457	10	val_309
 309	458	1	val_309
-309	458	3	val_309
-309	458	3	val_309
-309	458	27	val_309
-309	459	3	val_309
-309	459	27	val_309
-310	456	10	val_310
-310	457	13	val_310
-310	458	13	val_310
-310	458	13	val_310
-310	459	13	val_310
-311	456	1	val_311
-311	456	25	val_311
+309	458	5	val_309
+309	458	10	val_309
+309	458	10	val_309
+309	459	1	val_309
+309	459	10	val_309
+310	456	27	val_310
+310	457	3	val_310
+310	458	3	val_310
+310	458	3	val_310
+310	459	3	val_310
+311	456	20	val_311
+311	456	23	val_311
 311	456	27	val_311
-311	457	0	val_311
-311	457	2	val_311
+311	457	7	val_311
+311	457	9	val_311
 311	457	24	val_311
-311	458	0	val_311
-311	458	2	val_311
-311	458	17	val_311
+311	458	7	val_311
+311	458	9	val_311
+311	458	24	val_311
 311	458	24	val_311
 311	458	27	val_311
-311	458	27	val_311
-311	459	0	val_311
-311	459	2	val_311
+311	458	28	val_311
+311	459	7	val_311
+311	459	9	val_311
 311	459	24	val_311
-315	456	4	val_315
-315	457	14	val_315
-315	458	1	val_315
-315	458	14	val_315
-315	459	14	val_315
-316	456	2	val_316
-316	456	11	val_316
-316	456	23	val_316
-316	457	5	val_316
-316	457	11	val_316
+315	456	19	val_315
+315	457	24	val_315
+315	458	24	val_315
+315	458	28	val_315
+315	459	24	val_315
+316	456	16	val_316
+316	456	27	val_316
+316	456	27	val_316
+316	457	6	val_316
+316	457	20	val_316
 316	457	24	val_316
-316	458	1	val_316
 316	458	5	val_316
-316	458	11	val_316
-316	458	11	val_316
+316	458	6	val_316
+316	458	20	val_316
+316	458	20	val_316
 316	458	24	val_316
-316	458	27	val_316
-316	459	5	val_316
-316	459	11	val_316
+316	458	24	val_316
+316	459	6	val_316
+316	459	20	val_316
 316	459	24	val_316
-317	456	13	val_317
-317	456	20	val_317
-317	457	17	val_317
-317	457	19	val_317
-317	458	17	val_317
-317	458	19	val_317
-317	458	19	val_317
-317	458	25	val_317
-317	459	17	val_317
-317	459	19	val_317
-318	456	7	val_318
+317	456	9	val_317
+317	456	10	val_317
+317	457	27	val_317
+317	457	28	val_317
+317	458	20	val_317
+317	458	27	val_317
+317	458	28	val_317
+317	458	28	val_317
+317	459	27	val_317
+317	459	28	val_317
 318	456	9	val_318
 318	456	14	val_318
+318	456	19	val_318
 318	457	6	val_318
 318	457	11	val_318
 318	457	24	val_318
@@ -3785,581 +3785,581 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
 318	458	6	val_318
 318	458	7	val_318
 318	458	11	val_318
-318	458	14	val_318
 318	458	24	val_318
+318	458	28	val_318
 318	459	6	val_318
 318	459	11	val_318
 318	459	24	val_318
+321	456	9	val_321
 321	456	10	val_321
-321	456	27	val_321
-321	457	1	val_321
-321	457	7	val_321
-321	458	1	val_321
-321	458	3	val_321
+321	457	4	val_321
+321	457	9	val_321
+321	458	4	val_321
 321	458	7	val_321
 321	458	9	val_321
-321	459	1	val_321
-321	459	7	val_321
-322	456	4	val_322
-322	456	5	val_322
-322	457	0	val_322
-322	457	15	val_322
-322	458	0	val_322
-322	458	1	val_322
-322	458	14	val_322
-322	458	15	val_322
-322	459	0	val_322
-322	459	15	val_322
-323	456	4	val_323
+321	458	24	val_321
+321	459	4	val_321
+321	459	9	val_321
+322	456	3	val_322
+322	456	24	val_322
+322	457	6	val_322
+322	457	20	val_322
+322	458	2	val_322
+322	458	6	val_322
+322	458	7	val_322
+322	458	20	val_322
+322	459	6	val_322
+322	459	20	val_322
+323	456	28	val_323
 323	457	14	val_323
 323	458	14	val_323
-323	458	15	val_323
+323	458	14	val_323
 323	459	14	val_323
-325	456	5	val_325
-325	456	10	val_325
-325	457	6	val_325
-325	457	10	val_325
-325	458	6	val_325
-325	458	6	val_325
-325	458	10	val_325
-325	458	10	val_325
-325	459	6	val_325
-325	459	10	val_325
-327	456	1	val_327
+325	456	1	val_325
+325	456	28	val_325
+325	457	1	val_325
+325	457	9	val_325
+325	458	1	val_325
+325	458	1	val_325
+325	458	9	val_325
+325	458	9	val_325
+325	459	1	val_325
+325	459	9	val_325
 327	456	10	val_327
 327	456	27	val_327
-327	457	2	val_327
+327	456	27	val_327
 327	457	6	val_327
 327	457	13	val_327
-327	458	2	val_327
+327	457	27	val_327
 327	458	6	val_327
 327	458	7	val_327
 327	458	13	val_327
-327	458	14	val_327
 327	458	20	val_327
-327	459	2	val_327
+327	458	27	val_327
+327	458	27	val_327
 327	459	6	val_327
 327	459	13	val_327
-331	456	20	val_331
+327	459	27	val_327
+331	456	13	val_331
 331	456	27	val_331
-331	457	3	val_331
-331	457	7	val_331
-331	458	2	val_331
-331	458	3	val_331
-331	458	7	val_331
-331	458	10	val_331
-331	459	3	val_331
-331	459	7	val_331
-332	456	20	val_332
+331	457	9	val_331
+331	457	13	val_331
+331	458	9	val_331
+331	458	13	val_331
+331	458	13	val_331
+331	458	27	val_331
+331	459	9	val_331
+331	459	13	val_331
+332	456	4	val_332
 332	457	14	val_332
+332	458	1	val_332
 332	458	14	val_332
-332	458	17	val_332
 332	459	14	val_332
-333	456	19	val_333
+333	456	2	val_333
 333	456	24	val_333
-333	457	4	val_333
-333	457	7	val_333
-333	458	4	val_333
-333	458	4	val_333
-333	458	7	val_333
-333	458	9	val_333
-333	459	4	val_333
-333	459	7	val_333
-335	456	2	val_335
-335	457	6	val_335
-335	458	6	val_335
-335	458	28	val_335
-335	459	6	val_335
-336	456	6	val_336
-336	457	27	val_336
-336	458	6	val_336
-336	458	27	val_336
-336	459	27	val_336
-338	456	11	val_338
+333	457	5	val_333
+333	457	16	val_333
+333	458	5	val_333
+333	458	5	val_333
+333	458	16	val_333
+333	458	27	val_333
+333	459	5	val_333
+333	459	16	val_333
+335	456	7	val_335
+335	457	23	val_335
+335	458	10	val_335
+335	458	23	val_335
+335	459	23	val_335
+336	456	27	val_336
+336	457	3	val_336
+336	458	3	val_336
+336	458	10	val_336
+336	459	3	val_336
+338	456	4	val_338
 338	457	14	val_338
-338	458	2	val_338
+338	458	1	val_338
 338	458	14	val_338
 338	459	14	val_338
-339	456	1	val_339
-339	457	11	val_339
-339	458	11	val_339
-339	458	17	val_339
-339	459	11	val_339
-341	456	1	val_341
-341	457	24	val_341
-341	458	5	val_341
-341	458	24	val_341
-341	459	24	val_341
-342	456	3	val_342
-342	456	14	val_342
-342	457	6	val_342
-342	457	20	val_342
-342	458	2	val_342
-342	458	6	val_342
-342	458	20	val_342
-342	458	20	val_342
-342	459	6	val_342
-342	459	20	val_342
-344	456	13	val_344
-344	456	16	val_344
-344	457	17	val_344
-344	457	28	val_344
-344	458	6	val_344
-344	458	17	val_344
+339	456	14	val_339
+339	457	16	val_339
+339	458	2	val_339
+339	458	16	val_339
+339	459	16	val_339
+341	456	10	val_341
+341	457	25	val_341
+341	458	7	val_341
+341	458	25	val_341
+341	459	25	val_341
+342	456	4	val_342
+342	456	24	val_342
+342	457	1	val_342
+342	457	23	val_342
+342	458	1	val_342
+342	458	10	val_342
+342	458	23	val_342
+342	458	23	val_342
+342	459	1	val_342
+342	459	23	val_342
+344	456	10	val_344
+344	456	24	val_344
+344	457	9	val_344
+344	457	23	val_344
+344	458	4	val_344
+344	458	9	val_344
+344	458	9	val_344
 344	458	23	val_344
-344	458	28	val_344
-344	459	17	val_344
-344	459	28	val_344
+344	459	9	val_344
+344	459	23	val_344
 345	456	3	val_345
-345	457	14	val_345
-345	458	14	val_345
-345	458	14	val_345
-345	459	14	val_345
-348	456	1	val_348
-348	456	2	val_348
+345	457	4	val_345
+345	458	4	val_345
+345	458	25	val_345
+345	459	4	val_345
+348	456	4	val_348
 348	456	9	val_348
-348	456	14	val_348
-348	456	28	val_348
-348	457	0	val_348
-348	457	7	val_348
-348	457	11	val_348
-348	457	16	val_348
-348	457	28	val_348
-348	458	0	val_348
-348	458	5	val_348
-348	458	5	val_348
+348	456	10	val_348
+348	456	23	val_348
+348	456	24	val_348
+348	457	2	val_348
+348	457	6	val_348
+348	457	9	val_348
+348	457	10	val_348
+348	457	24	val_348
+348	458	2	val_348
+348	458	6	val_348
 348	458	7	val_348
 348	458	7	val_348
 348	458	9	val_348
+348	458	9	val_348
 348	458	10	val_348
-348	458	11	val_348
-348	458	16	val_348
-348	458	28	val_348
-348	459	0	val_348
-348	459	7	val_348
-348	459	11	val_348
-348	459	16	val_348
-348	459	28	val_348
-351	456	9	val_351
-351	457	5	val_351
-351	458	2	val_351
-351	458	5	val_351
-351	459	5	val_351
-353	456	9	val_353
-353	456	23	val_353
+348	458	14	val_348
+348	458	23	val_348
+348	458	24	val_348
+348	459	2	val_348
+348	459	6	val_348
+348	459	9	val_348
+348	459	10	val_348
+348	459	24	val_348
+351	456	14	val_351
+351	457	28	val_351
+351	458	10	val_351
+351	458	28	val_351
+351	459	28	val_351
+353	456	10	val_353
+353	456	16	val_353
+353	457	4	val_353
 353	457	10	val_353
-353	457	20	val_353
+353	458	1	val_353
+353	458	4	val_353
 353	458	10	val_353
-353	458	20	val_353
-353	458	24	val_353
-353	458	27	val_353
+353	458	10	val_353
+353	459	4	val_353
 353	459	10	val_353
-353	459	20	val_353
 356	456	19	val_356
 356	457	10	val_356
 356	458	1	val_356
 356	458	10	val_356
 356	459	10	val_356
-360	456	5	val_360
-360	457	11	val_360
-360	458	6	val_360
-360	458	11	val_360
-360	459	11	val_360
-362	456	27	val_362
-362	457	27	val_362
-362	458	27	val_362
-362	458	27	val_362
-362	459	27	val_362
-364	456	28	val_364
-364	457	14	val_364
-364	458	3	val_364
-364	458	14	val_364
-364	459	14	val_364
+360	456	24	val_360
+360	457	24	val_360
+360	458	24	val_360
+360	458	24	val_360
+360	459	24	val_360
+362	456	10	val_362
+362	457	10	val_362
+362	458	10	val_362
+362	458	10	val_362
+362	459	10	val_362
+364	456	7	val_364
+364	457	28	val_364
+364	458	13	val_364
+364	458	28	val_364
+364	459	28	val_364
 365	456	10	val_365
-365	457	28	val_365
-365	458	28	val_365
-365	458	28	val_365
-365	459	28	val_365
-366	456	3	val_366
-366	457	6	val_366
-366	458	6	val_366
-366	458	20	val_366
-366	459	6	val_366
-367	456	2	val_367
-367	456	28	val_367
-367	457	1	val_367
-367	457	14	val_367
-367	458	1	val_367
-367	458	1	val_367
-367	458	1	val_367
-367	458	14	val_367
-367	459	1	val_367
-367	459	14	val_367
-368	456	20	val_368
-368	457	4	val_368
-368	458	3	val_368
-368	458	4	val_368
-368	459	4	val_368
+365	457	13	val_365
+365	458	13	val_365
+365	458	13	val_365
+365	459	13	val_365
+366	456	13	val_366
+366	457	24	val_366
+366	458	4	val_366
+366	458	24	val_366
+366	459	24	val_366
+367	456	23	val_367
+367	456	27	val_367
+367	457	24	val_367
+367	457	28	val_367
+367	458	24	val_367
+367	458	24	val_367
+367	458	28	val_367
+367	458	28	val_367
+367	459	24	val_367
+367	459	28	val_367
+368	456	9	val_368
+368	457	6	val_368
+368	458	6	val_368
+368	458	16	val_368
+368	459	6	val_368
+369	456	9	val_369
 369	456	10	val_369
-369	456	14	val_369
 369	456	28	val_369
-369	457	3	val_369
-369	457	17	val_369
+369	457	9	val_369
+369	457	27	val_369
 369	457	27	val_369
-369	458	3	val_369
+369	458	9	val_369
+369	458	9	val_369
 369	458	14	val_369
-369	458	17	val_369
-369	458	20	val_369
-369	458	25	val_369
 369	458	27	val_369
-369	459	3	val_369
-369	459	17	val_369
+369	458	27	val_369
+369	458	27	val_369
+369	459	9	val_369
+369	459	27	val_369
 369	459	27	val_369
-373	456	28	val_373
-373	457	27	val_373
-373	458	14	val_373
-373	458	27	val_373
-373	459	27	val_373
+373	456	10	val_373
+373	457	10	val_373
+373	458	10	val_373
+373	458	10	val_373
+373	459	10	val_373
 374	456	9	val_374
 374	457	7	val_374
 374	458	7	val_374
 374	458	16	val_374
 374	459	7	val_374
-375	456	3	val_375
-375	457	25	val_375
-375	458	6	val_375
-375	458	25	val_375
-375	459	25	val_375
-377	456	9	val_377
-377	457	2	val_377
-377	458	2	val_377
-377	458	2	val_377
-377	459	2	val_377
+375	456	13	val_375
+375	457	28	val_375
+375	458	28	val_375
+375	458	28	val_375
+375	459	28	val_375
+377	456	24	val_377
+377	457	9	val_377
+377	458	9	val_377
+377	458	9	val_377
+377	459	9	val_377
 378	456	27	val_378
 378	457	10	val_378
 378	458	7	val_378
 378	458	10	val_378
 378	459	10	val_378
-379	456	25	val_379
-379	457	27	val_379
-379	458	11	val_379
-379	458	27	val_379
-379	459	27	val_379
+379	456	14	val_379
+379	457	3	val_379
+379	458	2	val_379
+379	458	3	val_379
+379	459	3	val_379
+382	456	23	val_382
 382	456	23	val_382
-382	456	28	val_382
 382	457	3	val_382
-382	457	20	val_382
+382	457	9	val_382
 382	458	3	val_382
 382	458	9	val_382
 382	458	9	val_382
-382	458	20	val_382
+382	458	14	val_382
 382	459	3	val_382
-382	459	20	val_382
-384	456	6	val_384
-384	456	19	val_384
-384	456	20	val_384
-384	457	1	val_384
-384	457	14	val_384
-384	457	17	val_384
-384	458	1	val_384
-384	458	4	val_384
-384	458	14	val_384
-384	458	17	val_384
-384	458	17	val_384
-384	458	28	val_384
-384	459	1	val_384
-384	459	14	val_384
-384	459	17	val_384
+382	459	9	val_382
+384	456	10	val_384
+384	456	24	val_384
+384	456	28	val_384
+384	457	9	val_384
+384	457	11	val_384
+384	457	16	val_384
+384	458	7	val_384
+384	458	9	val_384
+384	458	9	val_384
+384	458	10	val_384
+384	458	11	val_384
+384	458	16	val_384
+384	459	9	val_384
+384	459	11	val_384
+384	459	16	val_384
 386	456	1	val_386
 386	457	7	val_386
 386	458	7	val_386
 386	458	10	val_386
 386	459	7	val_386
-389	456	28	val_389
+389	456	10	val_389
 389	457	10	val_389
 389	458	10	val_389
-389	458	24	val_389
+389	458	10	val_389
 389	459	10	val_389
-392	456	9	val_392
-392	457	28	val_392
-392	458	27	val_392
-392	458	28	val_392
-392	459	28	val_392
-393	456	20	val_393
-393	457	15	val_393
-393	458	11	val_393
-393	458	15	val_393
-393	459	15	val_393
-394	456	27	val_394
+392	456	7	val_392
+392	457	7	val_392
+392	458	7	val_392
+392	458	7	val_392
+392	459	7	val_392
+393	456	9	val_393
+393	457	25	val_393
+393	458	25	val_393
+393	458	28	val_393
+393	459	25	val_393
+394	456	1	val_394
 394	457	27	val_394
-394	458	27	val_394
+394	458	9	val_394
 394	458	27	val_394
 394	459	27	val_394
-395	456	1	val_395
-395	456	5	val_395
-395	457	0	val_395
-395	457	11	val_395
-395	458	0	val_395
-395	458	11	val_395
-395	458	11	val_395
-395	458	17	val_395
-395	459	0	val_395
-395	459	11	val_395
-396	456	7	val_396
-396	456	27	val_396
-396	456	28	val_396
-396	457	3	val_396
-396	457	14	val_396
-396	457	24	val_396
-396	458	3	val_396
+395	456	10	val_395
+395	456	20	val_395
+395	457	9	val_395
+395	457	10	val_395
+395	458	9	val_395
+395	458	10	val_395
+395	458	27	val_395
+395	458	28	val_395
+395	459	9	val_395
+395	459	10	val_395
+396	456	3	val_396
+396	456	10	val_396
+396	456	14	val_396
+396	457	11	val_396
+396	457	20	val_396
+396	457	27	val_396
 396	458	6	val_396
-396	458	14	val_396
-396	458	14	val_396
-396	458	24	val_396
+396	458	9	val_396
+396	458	10	val_396
+396	458	11	val_396
+396	458	20	val_396
 396	458	27	val_396
-396	459	3	val_396
-396	459	14	val_396
-396	459	24	val_396
-397	456	1	val_397
-397	456	20	val_397
-397	457	2	val_397
-397	457	16	val_397
-397	458	2	val_397
-397	458	16	val_397
-397	458	16	val_397
-397	458	20	val_397
-397	459	2	val_397
-397	459	16	val_397
+396	459	11	val_396
+396	459	20	val_396
+396	459	27	val_396
+397	456	9	val_397
+397	456	28	val_397
+397	457	7	val_397
+397	457	10	val_397
+397	458	7	val_397
+397	458	7	val_397
+397	458	9	val_397
+397	458	10	val_397
+397	459	7	val_397
+397	459	10	val_397
 399	456	4	val_399
-399	456	23	val_399
+399	456	16	val_399
 399	457	2	val_399
-399	457	24	val_399
+399	457	25	val_399
+399	458	1	val_399
 399	458	2	val_399
 399	458	24	val_399
-399	458	24	val_399
-399	458	24	val_399
+399	458	25	val_399
 399	459	2	val_399
-399	459	24	val_399
-400	456	27	val_400
-400	457	6	val_400
-400	458	6	val_400
-400	458	7	val_400
-400	459	6	val_400
-401	456	10	val_401
+399	459	25	val_399
+400	456	20	val_400
+400	457	11	val_400
+400	458	5	val_400
+400	458	11	val_400
+400	459	11	val_400
+401	456	14	val_401
+401	456	19	val_401
+401	456	19	val_401
 401	456	19	val_401
 401	456	24	val_401
-401	456	25	val_401
-401	456	28	val_401
-401	457	11	val_401
-401	457	15	val_401
-401	457	20	val_401
+401	457	1	val_401
+401	457	7	val_401
+401	457	19	val_401
 401	457	24	val_401
-401	457	24	val_401
-401	458	11	val_401
-401	458	11	val_401
-401	458	15	val_401
-401	458	16	val_401
+401	457	27	val_401
+401	458	1	val_401
+401	458	1	val_401
+401	458	2	val_401
+401	458	7	val_401
+401	458	19	val_401
+401	458	19	val_401
 401	458	20	val_401
 401	458	23	val_401
 401	458	24	val_401
-401	458	24	val_401
-401	458	24	val_401
-401	458	25	val_401
-401	459	11	val_401
-401	459	15	val_401
-401	459	20	val_401
-401	459	24	val_401
+401	458	27	val_401
+401	459	1	val_401
+401	459	7	val_401
+401	459	19	val_401
 401	459	24	val_401
-402	456	5	val_402
-402	457	1	val_402
-402	458	1	val_402
-402	458	1	val_402
-402	459	1	val_402
-403	456	20	val_403
-403	456	28	val_403
-403	456	28	val_403
-403	457	1	val_403
-403	457	11	val_403
-403	457	14	val_403
-403	458	1	val_403
-403	458	5	val_403
-403	458	6	val_403
-403	458	11	val_403
-403	458	11	val_403
-403	458	14	val_403
-403	459	1	val_403
-403	459	11	val_403
-403	459	14	val_403
+401	459	27	val_401
+402	456	24	val_402
+402	457	9	val_402
+402	458	9	val_402
+402	458	14	val_402
+402	459	9	val_402
+403	456	9	val_403
+403	456	19	val_403
+403	456	27	val_403
+403	457	9	val_403
+403	457	9	val_403
+403	457	25	val_403
+403	458	9	val_403
+403	458	9	val_403
+403	458	25	val_403
+403	458	27	val_403
+403	458	28	val_403
+403	458	28	val_403
+403	459	9	val_403
+403	459	9	val_403
+403	459	25	val_403
 404	456	7	val_404
-404	456	13	val_404
+404	456	27	val_404
 404	457	24	val_404
-404	457	24	val_404
-404	458	23	val_404
+404	457	25	val_404
+404	458	5	val_404
 404	458	23	val_404
 404	458	24	val_404
-404	458	24	val_404
-404	459	24	val_404
+404	458	25	val_404
 404	459	24	val_404
-406	456	2	val_406
-406	456	6	val_406
-406	456	20	val_406
+404	459	25	val_404
+406	456	7	val_406
 406	456	24	val_406
+406	456	24	val_406
+406	456	28	val_406
 406	457	5	val_406
-406	457	5	val_406
-406	457	17	val_406
+406	457	10	val_406
+406	457	16	val_406
 406	457	28	val_406
-406	458	4	val_406
-406	458	5	val_406
 406	458	5	val_406
+406	458	9	val_406
 406	458	10	val_406
-406	458	17	val_406
-406	458	17	val_406
-406	458	28	val_406
+406	458	10	val_406
+406	458	10	val_406
+406	458	10	val_406
+406	458	16	val_406
 406	458	28	val_406
 406	459	5	val_406
-406	459	5	val_406
-406	459	17	val_406
+406	459	10	val_406
+406	459	16	val_406
 406	459	28	val_406
-407	456	7	val_407
-407	457	4	val_407
-407	458	4	val_407
-407	458	6	val_407
-407	459	4	val_407
+407	456	23	val_407
+407	457	14	val_407
+407	458	14	val_407
+407	458	14	val_407
+407	459	14	val_407
 409	456	1	val_409
 409	456	20	val_409
 409	456	24	val_409
+409	457	3	val_409
 409	457	4	val_409
 409	457	10	val_409
-409	457	27	val_409
-409	458	1	val_409
+409	458	3	val_409
 409	458	4	val_409
 409	458	9	val_409
 409	458	10	val_409
 409	458	20	val_409
-409	458	27	val_409
+409	458	20	val_409
+409	459	3	val_409
 409	459	4	val_409
 409	459	10	val_409
-409	459	27	val_409
-411	456	20	val_411
-411	457	1	val_411
-411	458	1	val_411
-411	458	16	val_411
-411	459	1	val_411
-413	456	7	val_413
-413	456	16	val_413
-413	457	10	val_413
-413	457	17	val_413
-413	458	4	val_413
-413	458	10	val_413
-413	458	10	val_413
-413	458	17	val_413
-413	459	10	val_413
-413	459	17	val_413
-414	456	10	val_414
-414	456	24	val_414
-414	457	5	val_414
-414	457	6	val_414
-414	458	5	val_414
-414	458	6	val_414
-414	458	10	val_414
-414	458	20	val_414
-414	459	5	val_414
-414	459	6	val_414
-417	456	5	val_417
-417	456	6	val_417
+411	456	10	val_411
+411	457	23	val_411
+411	458	13	val_411
+411	458	23	val_411
+411	459	23	val_411
+413	456	9	val_413
+413	456	24	val_413
+413	457	3	val_413
+413	457	7	val_413
+413	458	2	val_413
+413	458	3	val_413
+413	458	7	val_413
+413	458	7	val_413
+413	459	3	val_413
+413	459	7	val_413
+414	456	1	val_414
+414	456	27	val_414
+414	457	11	val_414
+414	457	28	val_414
+414	458	7	val_414
+414	458	11	val_414
+414	458	27	val_414
+414	458	28	val_414
+414	459	11	val_414
+414	459	28	val_414
+417	456	10	val_417
+417	456	13	val_417
 417	456	24	val_417
-417	457	2	val_417
-417	457	20	val_417
-417	457	27	val_417
-417	458	2	val_417
-417	458	4	val_417
-417	458	17	val_417
-417	458	20	val_417
-417	458	20	val_417
-417	458	27	val_417
-417	459	2	val_417
-417	459	20	val_417
-417	459	27	val_417
+417	457	10	val_417
+417	457	10	val_417
+417	457	13	val_417
+417	458	9	val_417
+417	458	10	val_417
+417	458	10	val_417
+417	458	10	val_417
+417	458	13	val_417
+417	458	13	val_417
+417	459	10	val_417
+417	459	10	val_417
+417	459	13	val_417
 418	456	10	val_418
 418	457	24	val_418
 418	458	24	val_418
 418	458	24	val_418
 418	459	24	val_418
-419	456	24	val_419
-419	457	9	val_419
-419	458	9	val_419
+419	456	4	val_419
+419	457	2	val_419
+419	458	2	val_419
 419	458	9	val_419
-419	459	9	val_419
-421	456	5	val_421
-421	457	6	val_421
-421	458	6	val_421
-421	458	11	val_421
-421	459	6	val_421
-424	456	10	val_424
-424	456	23	val_424
-424	457	10	val_424
-424	457	14	val_424
-424	458	10	val_424
-424	458	10	val_424
-424	458	14	val_424
+419	459	2	val_419
+421	456	4	val_421
+421	457	24	val_421
+421	458	24	val_421
+421	458	24	val_421
+421	459	24	val_421
+424	456	2	val_424
+424	456	4	val_424
+424	457	4	val_424
+424	457	24	val_424
+424	458	1	val_424
+424	458	4	val_424
+424	458	20	val_424
 424	458	24	val_424
-424	459	10	val_424
-424	459	14	val_424
+424	459	4	val_424
+424	459	24	val_424
 427	456	7	val_427
 427	457	9	val_427
 427	458	4	val_427
 427	458	9	val_427
 427	459	9	val_427
-429	456	11	val_429
+429	456	4	val_429
 429	456	14	val_429
 429	457	5	val_429
-429	457	14	val_429
+429	457	28	val_429
 429	458	2	val_429
 429	458	5	val_429
-429	458	14	val_429
-429	458	27	val_429
+429	458	28	val_429
+429	458	28	val_429
 429	459	5	val_429
-429	459	14	val_429
-430	456	7	val_430
-430	456	16	val_430
-430	456	24	val_430
-430	457	2	val_430
-430	457	2	val_430
-430	457	10	val_430
-430	458	1	val_430
-430	458	2	val_430
-430	458	2	val_430
-430	458	4	val_430
+429	459	28	val_429
+430	456	9	val_430
+430	456	9	val_430
+430	456	27	val_430
+430	457	7	val_430
+430	457	13	val_430
+430	457	13	val_430
 430	458	7	val_430
+430	458	7	val_430
+430	458	9	val_430
 430	458	10	val_430
-430	459	2	val_430
-430	459	2	val_430
-430	459	10	val_430
-431	456	3	val_431
-431	456	4	val_431
-431	456	7	val_431
-431	457	1	val_431
-431	457	1	val_431
-431	457	13	val_431
-431	458	1	val_431
+430	458	13	val_430
+430	458	13	val_430
+430	459	7	val_430
+430	459	13	val_430
+430	459	13	val_430
+431	456	9	val_431
+431	456	19	val_431
+431	456	28	val_431
+431	457	10	val_431
+431	457	20	val_431
+431	457	28	val_431
 431	458	1	val_431
+431	458	7	val_431
 431	458	10	val_431
-431	458	13	val_431
 431	458	20	val_431
+431	458	23	val_431
 431	458	28	val_431
-431	459	1	val_431
-431	459	1	val_431
-431	459	13	val_431
-432	456	19	val_432
-432	457	28	val_432
-432	458	23	val_432
-432	458	28	val_432
-432	459	28	val_432
-435	456	7	val_435
-435	457	28	val_435
-435	458	13	val_435
-435	458	28	val_435
-435	459	28	val_435
+431	459	10	val_431
+431	459	20	val_431
+431	459	28	val_431
+432	456	10	val_432
+432	457	20	val_432
+432	458	5	val_432
+432	458	20	val_432
+432	459	20	val_432
+435	456	28	val_435
+435	457	1	val_435
+435	458	1	val_435
+435	458	10	val_435
+435	459	1	val_435
 436	456	10	val_436
 436	457	9	val_436
 436	458	9	val_436
@@ -4370,360 +4370,360 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
 437	458	2	val_437
 437	458	20	val_437
 437	459	20	val_437
-438	456	1	val_438
-438	456	10	val_438
-438	456	27	val_438
-438	457	19	val_438
-438	457	24	val_438
-438	457	25	val_438
-438	458	5	val_438
-438	458	13	val_438
-438	458	19	val_438
-438	458	24	val_438
-438	458	25	val_438
+438	456	16	val_438
+438	456	20	val_438
+438	456	20	val_438
+438	457	6	val_438
+438	457	6	val_438
+438	457	28	val_438
+438	458	6	val_438
+438	458	6	val_438
+438	458	6	val_438
+438	458	16	val_438
 438	458	28	val_438
-438	459	19	val_438
-438	459	24	val_438
-438	459	25	val_438
-439	456	16	val_439
-439	456	28	val_439
-439	457	7	val_439
-439	457	17	val_439
-439	458	7	val_439
-439	458	7	val_439
-439	458	11	val_439
-439	458	17	val_439
-439	459	7	val_439
-439	459	17	val_439
-443	456	6	val_443
+438	458	28	val_438
+438	459	6	val_438
+438	459	6	val_438
+438	459	28	val_438
+439	456	19	val_439
+439	456	27	val_439
+439	457	4	val_439
+439	457	27	val_439
+439	458	4	val_439
+439	458	4	val_439
+439	458	27	val_439
+439	458	27	val_439
+439	459	4	val_439
+439	459	27	val_439
+443	456	23	val_443
 443	457	24	val_443
-443	458	5	val_443
+443	458	24	val_443
 443	458	24	val_443
 443	459	24	val_443
-444	456	25	val_444
-444	457	15	val_444
-444	458	14	val_444
-444	458	15	val_444
-444	459	15	val_444
-446	456	6	val_446
-446	457	0	val_446
-446	458	0	val_446
-446	458	5	val_446
-446	459	0	val_446
-448	456	23	val_448
-448	457	10	val_448
-448	458	10	val_448
-448	458	24	val_448
-448	459	10	val_448
-449	456	19	val_449
-449	457	20	val_449
-449	458	1	val_449
-449	458	20	val_449
-449	459	20	val_449
-452	456	16	val_452
-452	457	6	val_452
-452	458	6	val_452
-452	458	28	val_452
-452	459	6	val_452
-453	456	28	val_453
-453	457	16	val_453
-453	458	16	val_453
-453	458	20	val_453
-453	459	16	val_453
-454	456	1	val_454
+444	456	19	val_444
+444	457	4	val_444
+444	458	4	val_444
+444	458	4	val_444
+444	459	4	val_444
+446	456	23	val_446
+446	457	9	val_446
+446	458	9	val_446
+446	458	24	val_446
+446	459	9	val_446
+448	456	4	val_448
+448	457	27	val_448
+448	458	1	val_448
+448	458	27	val_448
+448	459	27	val_448
+449	456	13	val_449
+449	457	9	val_449
+449	458	9	val_449
+449	458	28	val_449
+449	459	9	val_449
+452	456	23	val_452
+452	457	23	val_452
+452	458	23	val_452
+452	458	23	val_452
+452	459	23	val_452
+453	456	23	val_453
+453	457	2	val_453
+453	458	2	val_453
+453	458	9	val_453
+453	459	2	val_453
+454	456	7	val_454
 454	456	10	val_454
-454	456	16	val_454
-454	457	6	val_454
-454	457	19	val_454
-454	457	24	val_454
-454	458	1	val_454
-454	458	6	val_454
+454	456	20	val_454
+454	457	11	val_454
+454	457	13	val_454
+454	457	23	val_454
+454	458	5	val_454
+454	458	10	val_454
+454	458	11	val_454
 454	458	13	val_454
-454	458	14	val_454
-454	458	19	val_454
-454	458	24	val_454
-454	459	6	val_454
-454	459	19	val_454
-454	459	24	val_454
-455	456	6	val_455
-455	457	17	val_455
-455	458	11	val_455
-455	458	17	val_455
-455	459	17	val_455
+454	458	23	val_454
+454	458	23	val_454
+454	459	11	val_454
+454	459	13	val_454
+454	459	23	val_454
+455	456	9	val_455
+455	457	28	val_455
+455	458	16	val_455
+455	458	28	val_455
+455	459	28	val_455
 457	456	1	val_457
 457	457	7	val_457
 457	458	2	val_457
 457	458	7	val_457
 457	459	7	val_457
-458	456	7	val_458
-458	456	10	val_458
+458	456	9	val_458
+458	456	16	val_458
 458	457	4	val_458
-458	457	10	val_458
+458	457	19	val_458
 458	458	4	val_458
-458	458	10	val_458
-458	458	10	val_458
+458	458	6	val_458
+458	458	13	val_458
 458	458	19	val_458
 458	459	4	val_458
-458	459	10	val_458
-459	456	10	val_459
+458	459	19	val_458
 459	456	20	val_459
+459	456	28	val_459
 459	457	1	val_459
-459	457	10	val_459
+459	457	20	val_459
 459	458	1	val_459
-459	458	10	val_459
-459	458	10	val_459
+459	458	14	val_459
+459	458	20	val_459
 459	458	20	val_459
 459	459	1	val_459
-459	459	10	val_459
-460	456	23	val_460
-460	457	28	val_460
-460	458	28	val_460
-460	458	28	val_460
-460	459	28	val_460
-462	456	11	val_462
-462	456	27	val_462
-462	457	3	val_462
+459	459	20	val_459
+460	456	3	val_460
+460	457	20	val_460
+460	458	20	val_460
+460	458	20	val_460
+460	459	20	val_460
+462	456	9	val_462
+462	456	24	val_462
 462	457	5	val_462
-462	458	3	val_462
+462	457	9	val_462
 462	458	5	val_462
-462	458	14	val_462
+462	458	9	val_462
+462	458	9	val_462
 462	458	16	val_462
-462	459	3	val_462
 462	459	5	val_462
-463	456	16	val_463
-463	456	24	val_463
-463	457	4	val_463
-463	457	20	val_463
-463	458	4	val_463
-463	458	7	val_463
+462	459	9	val_462
+463	456	5	val_463
+463	456	23	val_463
+463	457	6	val_463
+463	457	9	val_463
+463	458	6	val_463
 463	458	9	val_463
-463	458	20	val_463
-463	459	4	val_463
-463	459	20	val_463
-466	456	9	val_466
-466	456	16	val_466
-466	456	19	val_466
-466	457	5	val_466
-466	457	11	val_466
-466	457	23	val_466
-466	458	2	val_466
+463	458	25	val_463
+463	458	28	val_463
+463	459	6	val_463
+463	459	9	val_463
+466	456	7	val_466
+466	456	10	val_466
+466	456	10	val_466
+466	457	1	val_466
+466	457	13	val_466
+466	457	20	val_466
+466	458	1	val_466
 466	458	5	val_466
-466	458	7	val_466
-466	458	11	val_466
-466	458	23	val_466
-466	458	23	val_466
-466	459	5	val_466
-466	459	11	val_466
-466	459	23	val_466
-467	456	13	val_467
-467	457	23	val_467
-467	458	4	val_467
-467	458	23	val_467
-467	459	23	val_467
-468	456	7	val_468
-468	456	7	val_468
+466	458	10	val_466
+466	458	13	val_466
+466	458	19	val_466
+466	458	20	val_466
+466	459	1	val_466
+466	459	13	val_466
+466	459	20	val_466
+467	456	27	val_467
+467	457	7	val_467
+467	458	7	val_467
+467	458	28	val_467
+467	459	7	val_467
+468	456	1	val_468
+468	456	14	val_468
 468	456	19	val_468
-468	456	24	val_468
-468	457	7	val_468
-468	457	10	val_468
+468	456	20	val_468
+468	457	5	val_468
+468	457	6	val_468
 468	457	10	val_468
-468	457	14	val_468
-468	458	7	val_468
-468	458	10	val_468
-468	458	10	val_468
-468	458	10	val_468
+468	457	16	val_468
+468	458	3	val_468
+468	458	5	val_468
+468	458	6	val_468
 468	458	10	val_468
 468	458	14	val_468
-468	458	14	val_468
-468	458	17	val_468
-468	459	7	val_468
-468	459	10	val_468
+468	458	16	val_468
+468	458	16	val_468
+468	458	27	val_468
+468	459	5	val_468
+468	459	6	val_468
 468	459	10	val_468
-468	459	14	val_468
-469	456	1	val_469
+468	459	16	val_468
+469	456	13	val_469
 469	456	13	val_469
 469	456	14	val_469
-469	456	19	val_469
 469	456	20	val_469
+469	456	28	val_469
+469	457	1	val_469
 469	457	1	val_469
 469	457	1	val_469
-469	457	9	val_469
-469	457	24	val_469
+469	457	11	val_469
 469	457	27	val_469
 469	458	1	val_469
 469	458	1	val_469
-469	458	9	val_469
-469	458	14	val_469
+469	458	1	val_469
+469	458	6	val_469
+469	458	11	val_469
 469	458	16	val_469
-469	458	24	val_469
+469	458	23	val_469
 469	458	24	val_469
 469	458	27	val_469
 469	458	28	val_469
-469	458	28	val_469
 469	459	1	val_469
 469	459	1	val_469
-469	459	9	val_469
-469	459	24	val_469
+469	459	1	val_469
+469	459	11	val_469
 469	459	27	val_469
 470	456	27	val_470
 470	457	1	val_470
 470	458	1	val_470
 470	458	1	val_470
 470	459	1	val_470
-472	456	6	val_472
-472	457	14	val_472
+472	456	16	val_472
+472	457	11	val_472
+472	458	5	val_472
 472	458	11	val_472
-472	458	14	val_472
-472	459	14	val_472
-475	456	7	val_475
-475	457	6	val_475
-475	458	6	val_475
-475	458	6	val_475
-475	459	6	val_475
-477	456	14	val_477
-477	457	7	val_477
-477	458	2	val_477
-477	458	7	val_477
-477	459	7	val_477
+472	459	11	val_472
+475	456	1	val_475
+475	457	9	val_475
+475	458	9	val_475
+475	458	9	val_475
+475	459	9	val_475
+477	456	24	val_477
+477	457	5	val_477
+477	458	5	val_477
+477	458	10	val_477
+477	459	5	val_477
 478	456	9	val_478
-478	456	13	val_478
-478	457	4	val_478
-478	457	9	val_478
-478	458	4	val_478
-478	458	9	val_478
-478	458	9	val_478
+478	456	10	val_478
+478	457	10	val_478
+478	457	19	val_478
+478	458	10	val_478
+478	458	10	val_478
+478	458	13	val_478
 478	458	19	val_478
-478	459	4	val_478
-478	459	9	val_478
-479	456	20	val_479
-479	457	17	val_479
-479	458	1	val_479
-479	458	17	val_479
-479	459	17	val_479
-480	456	14	val_480
-480	456	24	val_480
-480	456	27	val_480
-480	457	9	val_480
-480	457	16	val_480
-480	457	27	val_480
-480	458	9	val_480
+478	459	10	val_478
+478	459	19	val_478
+479	456	7	val_479
+479	457	7	val_479
+479	458	7	val_479
+479	458	7	val_479
+479	459	7	val_479
+480	456	1	val_480
+480	456	1	val_480
+480	456	20	val_480
+480	457	2	val_480
+480	457	11	val_480
+480	457	20	val_480
+480	458	2	val_480
 480	458	9	val_480
-480	458	16	val_480
-480	458	27	val_480
-480	458	27	val_480
-480	458	27	val_480
-480	459	9	val_480
-480	459	16	val_480
-480	459	27	val_480
+480	458	11	val_480
+480	458	11	val_480
+480	458	20	val_480
+480	458	20	val_480
+480	459	2	val_480
+480	459	11	val_480
+480	459	20	val_480
 481	456	28	val_481
 481	457	5	val_481
 481	458	5	val_481
 481	458	10	val_481
 481	459	5	val_481
-482	456	9	val_482
-482	457	9	val_482
-482	458	9	val_482
-482	458	9	val_482
-482	459	9	val_482
-483	456	5	val_483
-483	457	3	val_483
-483	458	3	val_483
-483	458	25	val_483
-483	459	3	val_483
+482	456	14	val_482
+482	457	2	val_482
+482	458	2	val_482
+482	458	27	val_482
+482	459	2	val_482
+483	456	4	val_483
+483	457	20	val_483
+483	458	1	val_483
+483	458	20	val_483
+483	459	20	val_483
 484	456	10	val_484
 484	457	23	val_484
 484	458	13	val_484
 484	458	23	val_484
 484	459	23	val_484
-485	456	7	val_485
-485	457	28	val_485
-485	458	1	val_485
-485	458	28	val_485
-485	459	28	val_485
-487	456	10	val_487
-487	457	10	val_487
-487	458	10	val_487
-487	458	10	val_487
-487	459	10	val_487
-489	456	4	val_489
-489	456	9	val_489
+485	456	13	val_485
+485	457	4	val_485
+485	458	4	val_485
+485	458	4	val_485
+485	459	4	val_485
+487	456	28	val_487
+487	457	9	val_487
+487	458	9	val_487
+487	458	14	val_487
+487	459	9	val_487
+489	456	2	val_489
 489	456	10	val_489
-489	456	14	val_489
+489	456	24	val_489
+489	456	27	val_489
+489	457	6	val_489
 489	457	9	val_489
-489	457	9	val_489
-489	457	19	val_489
+489	457	13	val_489
 489	457	27	val_489
-489	458	1	val_489
-489	458	9	val_489
+489	458	6	val_489
 489	458	9	val_489
+489	458	10	val_489
 489	458	13	val_489
-489	458	19	val_489
+489	458	20	val_489
 489	458	24	val_489
 489	458	27	val_489
 489	458	27	val_489
+489	459	6	val_489
 489	459	9	val_489
-489	459	9	val_489
-489	459	19	val_489
+489	459	13	val_489
 489	459	27	val_489
-490	456	3	val_490
-490	457	15	val_490
-490	458	1	val_490
-490	458	15	val_490
-490	459	15	val_490
-491	456	16	val_491
-491	457	1	val_491
+490	456	9	val_490
+490	457	4	val_490
+490	458	4	val_490
+490	458	10	val_490
+490	459	4	val_490
+491	456	2	val_491
+491	457	4	val_491
 491	458	1	val_491
-491	458	6	val_491
-491	459	1	val_491
-492	456	6	val_492
-492	456	13	val_492
-492	457	14	val_492
-492	457	15	val_492
-492	458	14	val_492
-492	458	15	val_492
-492	458	17	val_492
-492	458	23	val_492
-492	459	14	val_492
-492	459	15	val_492
-493	456	4	val_493
-493	457	23	val_493
-493	458	23	val_493
-493	458	23	val_493
-493	459	23	val_493
-494	456	10	val_494
-494	457	13	val_494
-494	458	13	val_494
-494	458	13	val_494
-494	459	13	val_494
+491	458	4	val_491
+491	459	4	val_491
+492	456	10	val_492
+492	456	20	val_492
+492	457	7	val_492
+492	457	24	val_492
+492	458	3	val_492
+492	458	4	val_492
+492	458	7	val_492
+492	458	24	val_492
+492	459	7	val_492
+492	459	24	val_492
+493	456	16	val_493
+493	457	7	val_493
+493	458	7	val_493
+493	458	7	val_493
+493	459	7	val_493
+494	456	20	val_494
+494	457	28	val_494
+494	458	16	val_494
+494	458	28	val_494
+494	459	28	val_494
 495	456	27	val_495
 495	457	25	val_495
 495	458	7	val_495
 495	458	25	val_495
 495	459	25	val_495
-496	456	28	val_496
-496	457	2	val_496
-496	458	2	val_496
-496	458	4	val_496
-496	459	2	val_496
-497	456	28	val_497
-497	457	4	val_497
-497	458	4	val_497
-497	458	20	val_497
-497	459	4	val_497
-498	456	3	val_498
-498	456	10	val_498
+496	456	1	val_496
+496	457	5	val_496
+496	458	5	val_496
+496	458	10	val_496
+496	459	5	val_496
+497	456	1	val_497
+497	457	14	val_497
+497	458	14	val_497
+497	458	24	val_497
+497	459	14	val_497
+498	456	2	val_498
+498	456	28	val_498
 498	456	28	val_498
-498	457	10	val_498
-498	457	14	val_498
+498	457	4	val_498
+498	457	11	val_498
 498	457	28	val_498
-498	458	10	val_498
-498	458	10	val_498
-498	458	14	val_498
-498	458	15	val_498
+498	458	4	val_498
+498	458	11	val_498
+498	458	20	val_498
+498	458	20	val_498
 498	458	20	val_498
 498	458	28	val_498
-498	459	10	val_498
-498	459	14	val_498
+498	459	4	val_498
+498	459	11	val_498
 498	459	28	val_498
 PREHOOK: query: drop table dp_mm
 PREHOOK: type: DROPTABLE
diff --git a/ql/src/test/results/clientpositive/llap/replication_metrics_ingest.q.out b/ql/src/test/results/clientpositive/llap/replication_metrics_ingest.q.out
index ba37d59aaf25..6fd00f4471a5 100644
--- a/ql/src/test/results/clientpositive/llap/replication_metrics_ingest.q.out
+++ b/ql/src/test/results/clientpositive/llap/replication_metrics_ingest.q.out
@@ -92,5 +92,5 @@ POSTHOOK: type: QUERY
 POSTHOOK: Input: sys@replication_metrics
 POSTHOOK: Input: sys@replication_metrics_orig
 #### A masked pattern was here ####
-repl1	1	{"dbName":"src","replicationType":"BOOTSTRAP","replicatedDBSizeInKB":0.0,"stagingDir":"dummyDir","lastReplId":0,"failoverMetadataLoc":null,"failoverEventId":0,"failoverEndPoint":null,"failoverType":null}	H4sIAAAAAAAAAG2PwQ6CMBBE/2XPHOTKTSsmJojEwskQ02gDJKUl2+2J9N8tEohEb7sz83ayI1gS5CwkwCvGUs4hmqRGBuk+gha9DN4tLbLHsboUs/sHQCq7KbqLQOrXOveSsHtubp2qnJXnaz6BT4coNTHjNH3yZEioZfXRCpX7Q5b+EvGWiH0d6hENZqYpBLWQaKdUBCgHxbUYbGsW9MsID9lZ8LV/A7NIwGISAQAA	{"status":"SUCCESS","stages":[{"name":"REPL_DUMP","status":"SUCCESS","startTime":0,"endTime":0,"metrics":[{"name":"FUNCTIONS","currentCount":0,"totalCount":0},{"name":"TABLES","currentCount":1,"totalCount":1}],"errorLogPath":null,"replSnapshotCount":null,"replStats":null}]}	gzip(json-2.0)
-repl2	1	{"dbName":"destination","replicationType":"BOOTSTRAP","replicatedDBSizeInKB":0.00390625,"stagingDir":"dummyDir","lastReplId":0,"failoverMetadataLoc":null,"failoverEventId":0,"failoverEndPoint":null,"failoverType":null}	H4sIAAAAAAAAAG2PwQqDMBBE/yVnD/XqzUYLBbFS9VSkBF1UiImsm5Pk3xu1CtLedmb3zbAzm0iQmVjA8pLzOM+Zt1gtOOs1MyUGcLtnnCXv5BFG2/YPgFT0y+nFY6CaYx6AsK9PWbcy5cX9kS5gbRBBEddG0XpPmoTcpfUOqAivSfxL+GfCt5WrR9SY6DYT1LFAGSk9hjDKXIlx6vSOumgzcARB0KzVTkYgYZP2y7hfpy3EVvYDvpfiNy0BAAA=	{"status":"SUCCESS","stages":[{"name":"REPL_LOAD","status":"SUCCESS","startTime":0,"endTime":0,"metrics":[{"name":"FUNCTIONS","currentCount":0,"totalCount":0},{"name":"TABLES","currentCount":1,"totalCount":1}],"errorLogPath":null,"replSnapshotCount":{"numCreated":0,"numDeleted":0},"replStats":null}]}	gzip(json-2.0)
+repl1	1	{"dbName":"src","replicationType":"BOOTSTRAP","replicatedDBSizeInKB":0.0,"stagingDir":"dummyDir","lastReplId":0,"failoverMetadataLoc":null,"failoverEventId":0,"failoverEndPoint":null,"failoverType":null}	H4sIAAAAAAAA/22PwQ6CMBBE/2XPHOTKTSsmJojEwskQ02gDJKUl2+2J9N8tEohEb7sz83ayI1gS5CwkwCvGUs4hmqRGBuk+gha9DN4tLbLHsboUs/sHQCq7KbqLQOrXOveSsHtubp2qnJXnaz6BT4coNTHjNH3yZEioZfXRCpX7Q5b+EvGWiH0d6hENZqYpBLWQaKdUBCgHxbUYbGsW9MsID9lZ8LV/A7NIwGISAQAA	{"status":"SUCCESS","stages":[{"name":"REPL_DUMP","status":"SUCCESS","startTime":0,"endTime":0,"metrics":[{"name":"FUNCTIONS","currentCount":0,"totalCount":0},{"name":"TABLES","currentCount":1,"totalCount":1}],"errorLogPath":null,"replSnapshotCount":null,"replStats":null}]}	gzip(json-2.0)
+repl2	1	{"dbName":"destination","replicationType":"BOOTSTRAP","replicatedDBSizeInKB":0.00390625,"stagingDir":"dummyDir","lastReplId":0,"failoverMetadataLoc":null,"failoverEventId":0,"failoverEndPoint":null,"failoverType":null}	H4sIAAAAAAAA/22PwQqDMBBE/yVnD/XqzUYLBbFS9VSkBF1UiImsm5Pk3xu1CtLedmb3zbAzm0iQmVjA8pLzOM+Zt1gtOOs1MyUGcLtnnCXv5BFG2/YPgFT0y+nFY6CaYx6AsK9PWbcy5cX9kS5gbRBBEddG0XpPmoTcpfUOqAivSfxL+GfCt5WrR9SY6DYT1LFAGSk9hjDKXIlx6vSOumgzcARB0KzVTkYgYZP2y7hfpy3EVvYDvpfiNy0BAAA=	{"status":"SUCCESS","stages":[{"name":"REPL_LOAD","status":"SUCCESS","startTime":0,"endTime":0,"metrics":[{"name":"FUNCTIONS","currentCount":0,"totalCount":0},{"name":"TABLES","currentCount":1,"totalCount":1}],"errorLogPath":null,"replSnapshotCount":{"numCreated":0,"numDeleted":0},"replStats":null}]}	gzip(json-2.0)
diff --git a/ql/src/test/results/clientpositive/llap/udf5.q.out b/ql/src/test/results/clientpositive/llap/udf5.q.out
index d6ecb2f8de4b..347d64afbf42 100644
--- a/ql/src/test/results/clientpositive/llap/udf5.q.out
+++ b/ql/src/test/results/clientpositive/llap/udf5.q.out
@@ -260,7 +260,7 @@ POSTHOOK: query: select from_unixtime(unix_timestamp('1400-11-08 08:00:00 ICT',
 POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 #### A masked pattern was here ####
-1400-11-08 07:35:24
+1400-11-08 08:00:00
 PREHOOK: query: select from_unixtime(unix_timestamp('1800-11-08 08:00:00 ICT', 'yyyy-MM-dd HH:mm:ss z'))
 PREHOOK: type: QUERY
 PREHOOK: Input: _dummy_database@_dummy_table
@@ -269,7 +269,7 @@ POSTHOOK: query: select from_unixtime(unix_timestamp('1800-11-08 08:00:00 ICT',
 POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 #### A masked pattern was here ####
-1800-11-08 07:35:24
+1800-11-08 08:00:00
 PREHOOK: query: select from_unixtime(unix_timestamp('0000-00-00', 'uuuu-MM-dd'))
 PREHOOK: type: QUERY
 PREHOOK: Input: _dummy_database@_dummy_table
@@ -431,7 +431,7 @@ POSTHOOK: query: SELECT FROM_UNIXTIME(UNIX_TIMESTAMP('1400-01-01 00:00:00 ICT','
 POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 #### A masked pattern was here ####
-1399-12-31 23:35:24
+1400-01-01 00:00:00
 PREHOOK: query: SELECT FROM_UNIXTIME(UNIX_TIMESTAMP('1800-01-01 00:00:00 ICT','yyyy-MM-dd HH:mm:ss z'))
 PREHOOK: type: QUERY
 PREHOOK: Input: _dummy_database@_dummy_table
@@ -440,7 +440,7 @@ POSTHOOK: query: SELECT FROM_UNIXTIME(UNIX_TIMESTAMP('1800-01-01 00:00:00 ICT','
 POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 #### A masked pattern was here ####
-1799-12-31 23:35:24
+1800-01-01 00:00:00
 PREHOOK: query: SELECT FROM_UNIXTIME(UNIX_TIMESTAMP('1900-01-01 00:00:00 ICT','yyyy-MM-dd HH:mm:ss z'))
 PREHOOK: type: QUERY
 PREHOOK: Input: _dummy_database@_dummy_table
@@ -449,7 +449,7 @@ POSTHOOK: query: SELECT FROM_UNIXTIME(UNIX_TIMESTAMP('1900-01-01 00:00:00 ICT','
 POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 #### A masked pattern was here ####
-1899-12-31 23:35:24
+1900-01-01 00:00:00
 PREHOOK: query: SELECT FROM_UNIXTIME(UNIX_TIMESTAMP('2000-01-07 00:00:00 ICT','yyyy-MM-dd HH:mm:ss z'))
 PREHOOK: type: QUERY
 PREHOOK: Input: _dummy_database@_dummy_table
diff --git a/ql/src/test/results/clientpositive/llap/udf_date_format.q.out b/ql/src/test/results/clientpositive/llap/udf_date_format.q.out
index 33db7e82a42b..cd76a545a2c7 100644
--- a/ql/src/test/results/clientpositive/llap/udf_date_format.q.out
+++ b/ql/src/test/results/clientpositive/llap/udf_date_format.q.out
@@ -149,13 +149,13 @@ POSTHOOK: Input: _dummy_database@_dummy_table
 10	30	45	10 AM	08	123	123	NULL
 PREHOOK: query: select
 date_format('2015-04-08', ''),
-date_format('2015-04-08', 'B')
+date_format('2015-04-08', 'C')
 PREHOOK: type: QUERY
 PREHOOK: Input: _dummy_database@_dummy_table
 #### A masked pattern was here ####
 POSTHOOK: query: select
 date_format('2015-04-08', ''),
-date_format('2015-04-08', 'B')
+date_format('2015-04-08', 'C')
 POSTHOOK: type: QUERY
 POSTHOOK: Input: _dummy_database@_dummy_table
 #### A masked pattern was here ####
diff --git a/ql/src/test/results/clientpositive/llap/vectorized_math_funcs.q.out b/ql/src/test/results/clientpositive/llap/vectorized_math_funcs.q.out
index 187b5babf6d2..6b6cfe9db4e6 100644
--- a/ql/src/test/results/clientpositive/llap/vectorized_math_funcs.q.out
+++ b/ql/src/test/results/clientpositive/llap/vectorized_math_funcs.q.out
@@ -409,11 +409,11 @@ POSTHOOK: Input: default@alltypesorc
 15601.0	15601.0	15601	15601	0.37807863784568585	15601.00000000001	9.65509029374725	3.8712010109078907	4.193152436852078	13.929350886124324	NULL	5.584962500721156	NULL	NULL	13.929350886124324	5.584962500721156	13.929350886124324	194.02681610877246	194.02681610877246	124.90396310766124	NULL	1111111111111111111111111111111111010000100101111100000100011000	31353630312E30	FFFFFFFFD097C118	15601.0	48	NULL	-0.14856570831397706	NULL	0.9889025383288114	NULL	1.5707322283397571	893871.4561835973	272.2888166036353	15601.0	-795361000	-15601.0	1.0	-1.0	-0.9740573096878733
 NULL	NULL	NULL	NULL	0.3336458983920575	NULL	NULL	2.0794415416798357	NULL	NULL	NULL	3.0	29.693388204506274	29.58473549442715	NULL	3.0	NULL	NULL	NULL	NULL	29464.580431426475	110011101111110001011111011100	NULL	33BF17DC	NULL	8	1	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	868161500	NULL	NULL	1.0	NULL
 NULL	NULL	NULL	NULL	0.8681331660942196	NULL	NULL	2.0794415416798357	NULL	NULL	NULL	3.0	NULL	29.730832334348488	NULL	3.0	NULL	NULL	NULL	NULL	NULL	1111111111111111111111111111111110010000111111111000101010111000	NULL	FFFFFFFF90FF8AB8	NULL	8	0	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	-1862301000	NULL	NULL	-1.0	NULL
--7196.0	-7196.0	-7196	-7196	0.03951015606275099	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	1111111111111111111111111111111110100000010101110101001001110000	2D373139362E30	FFFFFFFFA0575270	7196.0	59	NULL	-0.9834787875028149	NULL	-0.18102340879563897	NULL	-1.5706573607035177	-412300.4293761404	-125.59389297351194	-7196.0	-1604890000	7196.0	-1.0	-1.0	NULL
--7196.0	-7196.0	-7196	-7196	0.9209252022050654	NULL	NULL	NULL	NULL	NULL	NULL	NULL	30.52255693577237	NULL	NULL	NULL	NULL	NULL	NULL	NULL	39273.76987252433	1011011111011111001100101001000	2D373139362E30	5BEF9948	7196.0	21	NULL	-0.9834787875028149	NULL	-0.18102340879563897	NULL	-1.5706573607035177	-412300.4293761404	-125.59389297351194	-7196.0	1542429000	7196.0	-1.0	1.0	NULL
--7196.0	-7196.0	-7196	-7196	0.4533660450429132	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	1111111111111111111111111111111110100011011110110101000010110100	2D373139362E30	FFFFFFFFA37B50B4	7196.0	14	NULL	-0.9834787875028149	NULL	-0.18102340879563897	NULL	-1.5706573607035177	-412300.4293761404	-125.59389297351194	-7196.0	-1552199500	7196.0	-1.0	-1.0	NULL
--7196.0	-7196.0	-7196	-7196	0.14567136069921982	NULL	NULL	4.07753744390572	NULL	NULL	NULL	5.882643049361842	NULL	NULL	NULL	5.882643049361842	NULL	NULL	NULL	NULL	NULL	1111111111111111111111111111111110111100001011110011111001111100	2D373139362E30	FFFFFFFFBC2F3E7C	7196.0	59	NULL	-0.9834787875028149	NULL	-0.18102340879563897	NULL	-1.5706573607035177	-412300.4293761404	-125.59389297351194	-7196.0	-1137754500	7196.0	-1.0	-1.0	NULL
--7196.0	-7196.0	-7196	-7196	0.5264452612398715	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	1111111111111111111111111111111110010001101110110101111010110100	2D373139362E30	FFFFFFFF91BB5EB4	7196.0	8	NULL	-0.9834787875028149	NULL	-0.18102340879563897	NULL	-1.5706573607035177	-412300.4293761404	-125.59389297351194	-7196.0	-1849991500	7196.0	-1.0	-1.0	NULL
--7196.0	-7196.0	-7196	-7196	0.17837094616515647	NULL	NULL	1.6094379124341003	NULL	NULL	NULL	2.321928094887362	NULL	NULL	NULL	2.321928094887362	NULL	NULL	NULL	NULL	NULL	1111111111111111111111111111111111000011011101110000111100110100	2D373139362E30	FFFFFFFFC3770F34	7196.0	5	NULL	-0.9834787875028149	NULL	-0.18102340879563897	NULL	-1.5706573607035177	-412300.4293761404	-125.59389297351194	-7196.0	-1015607500	7196.0	-1.0	-1.0	NULL
--7196.0	-7196.0	-7196	-7196	0.5456857574763374	NULL	NULL	NULL	NULL	NULL	NULL	NULL	29.62699001935971	NULL	NULL	NULL	NULL	NULL	NULL	NULL	28794.287627930647	110001011010110011101011011000	2D373139362E30	316B3AD8	7196.0	24	NULL	-0.9834787875028149	NULL	-0.18102340879563897	NULL	-1.5706573607035177	-412300.4293761404	-125.59389297351194	-7196.0	829111000	7196.0	-1.0	1.0	NULL
+-7196.0	-7196.0	-7196	-7196	0.03951015606275099	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	1111111111111111111111111111111110100000010101110101001001110000	2D373139362E30	FFFFFFFFA0575270	7196.0	59	NULL	-0.9834787875028149	NULL	-0.18102340879563897	NULL	-1.5706573607035177	-412300.4293761404	-125.59389297351196	-7196.0	-1604890000	7196.0	-1.0	-1.0	NULL
+-7196.0	-7196.0	-7196	-7196	0.9209252022050654	NULL	NULL	NULL	NULL	NULL	NULL	NULL	30.52255693577237	NULL	NULL	NULL	NULL	NULL	NULL	NULL	39273.76987252433	1011011111011111001100101001000	2D373139362E30	5BEF9948	7196.0	21	NULL	-0.9834787875028149	NULL	-0.18102340879563897	NULL	-1.5706573607035177	-412300.4293761404	-125.59389297351196	-7196.0	1542429000	7196.0	-1.0	1.0	NULL
+-7196.0	-7196.0	-7196	-7196	0.4533660450429132	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	1111111111111111111111111111111110100011011110110101000010110100	2D373139362E30	FFFFFFFFA37B50B4	7196.0	14	NULL	-0.9834787875028149	NULL	-0.18102340879563897	NULL	-1.5706573607035177	-412300.4293761404	-125.59389297351196	-7196.0	-1552199500	7196.0	-1.0	-1.0	NULL
+-7196.0	-7196.0	-7196	-7196	0.14567136069921982	NULL	NULL	4.07753744390572	NULL	NULL	NULL	5.882643049361842	NULL	NULL	NULL	5.882643049361842	NULL	NULL	NULL	NULL	NULL	1111111111111111111111111111111110111100001011110011111001111100	2D373139362E30	FFFFFFFFBC2F3E7C	7196.0	59	NULL	-0.9834787875028149	NULL	-0.18102340879563897	NULL	-1.5706573607035177	-412300.4293761404	-125.59389297351196	-7196.0	-1137754500	7196.0	-1.0	-1.0	NULL
+-7196.0	-7196.0	-7196	-7196	0.5264452612398715	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	1111111111111111111111111111111110010001101110110101111010110100	2D373139362E30	FFFFFFFF91BB5EB4	7196.0	8	NULL	-0.9834787875028149	NULL	-0.18102340879563897	NULL	-1.5706573607035177	-412300.4293761404	-125.59389297351196	-7196.0	-1849991500	7196.0	-1.0	-1.0	NULL
+-7196.0	-7196.0	-7196	-7196	0.17837094616515647	NULL	NULL	1.6094379124341003	NULL	NULL	NULL	2.321928094887362	NULL	NULL	NULL	2.321928094887362	NULL	NULL	NULL	NULL	NULL	1111111111111111111111111111111111000011011101110000111100110100	2D373139362E30	FFFFFFFFC3770F34	7196.0	5	NULL	-0.9834787875028149	NULL	-0.18102340879563897	NULL	-1.5706573607035177	-412300.4293761404	-125.59389297351196	-7196.0	-1015607500	7196.0	-1.0	-1.0	NULL
+-7196.0	-7196.0	-7196	-7196	0.5456857574763374	NULL	NULL	NULL	NULL	NULL	NULL	NULL	29.62699001935971	NULL	NULL	NULL	NULL	NULL	NULL	NULL	28794.287627930647	110001011010110011101011011000	2D373139362E30	316B3AD8	7196.0	24	NULL	-0.9834787875028149	NULL	-0.18102340879563897	NULL	-1.5706573607035177	-412300.4293761404	-125.59389297351196	-7196.0	829111000	7196.0	-1.0	1.0	NULL
 NULL	NULL	NULL	NULL	0.282703740641956	NULL	NULL	2.3978952727983707	NULL	NULL	NULL	3.4594316186372978	30.19990821555368	NULL	NULL	3.4594316186372978	NULL	NULL	NULL	NULL	35118.75567271711	1001001100000110001001110011000	NULL	49831398	NULL	11	1	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	1233327000	NULL	NULL	1.0	NULL
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java
index 5cd776a2e3d4..1c37b36bad33 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/ColumnProjectionUtils.java
@@ -182,7 +182,7 @@ public static void appendReadColumns(
 
   /**
    * Returns an array of column ids(start from zero) which is set in the given
-   * parameter conf.
+   * parameter 
    conf
    .
    */
   public static List getReadColumnIDs(Configuration conf) {
     String skips = conf.get(READ_COLUMN_IDS_CONF_STR, READ_COLUMN_IDS_CONF_STR_DEFAULT);
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefArrayWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefArrayWritable.java
index 2f9c3d36d303..c0981433414b 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefArrayWritable.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefArrayWritable.java
@@ -28,13 +28,13 @@
 import org.apache.hadoop.io.WritableFactory;
 
 /**
- * BytesRefArrayWritable holds an array reference to BytesRefWritable,
+ * 
    BytesRefArrayWritable
     holds an array reference to BytesRefWritable,
  * and is able to resize without recreating new array if not necessary.
  * 
    
  *
- * Each BytesRefArrayWritable holds instance has a valid field,
- * which is the desired valid number of BytesRefWritable it holds.
- * resetValid can reset the valid, but it will not care the underlying
+ * Each 
    BytesRefArrayWritable holds
     instance has a valid field,
+ * which is the desired valid number of 
    BytesRefWritable
     it holds.
+ * 
    resetValid
     can reset the valid, but it will not care the underlying
  * BytesRefWritable.
  */
 
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefWritable.java
index dd4007f7b174..ecbfe9fc7d87 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefWritable.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/columnar/BytesRefWritable.java
@@ -28,7 +28,7 @@
 import org.apache.hadoop.io.WritableFactory;
 
 /**
- * BytesRefWritable referenced a section of byte array. It can be used
+ * 
    BytesRefWritable
     referenced a section of byte array. It can be used
  * to avoid unnecessary byte copy.
  */
 public class BytesRefWritable implements Writable, Comparable {
@@ -50,7 +50,7 @@ public BytesRefWritable() {
   }
 
   /**
-   * Create a BytesRefWritable with length bytes.
+   * Create a BytesRefWritable with 
    length
     bytes.
    */
   public BytesRefWritable(int length) {
     assert length > 0;
@@ -70,7 +70,7 @@ public BytesRefWritable(byte[] bytes) {
 
   /**
    * Create a BytesRefWritable referenced to one section of the given bytes. The
-   * section is determined by argument offset and len.
+   * section is determined by argument 
    offset
     and 
    len
    .
    */
   public BytesRefWritable(byte[] data, int offset, int len) {
     bytes = data;
@@ -80,10 +80,10 @@ public BytesRefWritable(byte[] data, int offset, int len) {
 
   /**
    * Create a BytesRefWritable referenced to one section of the given bytes. The
-   * argument lazyDecompressData refers to a LazyDecompressionCallback
-   * object. The arguments offset and len are referred to
-   * uncompressed bytes of lazyDecompressData. Use offset and
-   * len after uncompressing the data.
+   * argument 
    lazyDecompressData
     refers to a LazyDecompressionCallback
+   * object. The arguments 
    offset
     and 
    len
     are referred to
+   * uncompressed bytes of 
    lazyDecompressData
    . Use 
    offset
     and
+   * 
    len
     after uncompressing the data.
    */
   public BytesRefWritable(LazyDecompressionCallback lazyDecompressData,
       int offset, int len) {
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonReader.java b/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonReader.java
index 7ade47b6e5ac..37891d799a97 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonReader.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonReader.java
@@ -73,7 +73,8 @@
  *
  * Support types are:
    
  * 
    
- *
Formatter
+ *
+ * * * * diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index d4d6235d8324..d12dfa4ac7d2 100644 --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@ -293,11 +293,13 @@ public class MiniMrShim implements HadoopShims.MiniMrShim { private final MiniMRCluster mr; private final Configuration conf; + @SuppressFBWarnings(value="EI_EXPOSE_REP2", justification = "intended_to_do") public MiniMrShim() { mr = null; conf = null; } + @SuppressFBWarnings(value="EI_EXPOSE_REP2", justification = "intended_to_do") public MiniMrShim(Configuration conf, int numberOfTaskTrackers, String nameNode, int numDir) throws IOException { this.conf = conf; @@ -353,6 +355,7 @@ public class MiniTezLocalShim extends Hadoop23Shims.MiniMrShim { private final Configuration conf; private final boolean isLlap; + @SuppressFBWarnings(value="EI_EXPOSE_REP2", justification = "intended_to_do") public MiniTezLocalShim(Configuration conf, boolean usingLlap) { this.conf = conf; this.isLlap = usingLlap; @@ -407,6 +410,7 @@ public class MiniTezShim extends Hadoop23Shims.MiniMrShim { private final Configuration conf; private final boolean isLlap; + @SuppressFBWarnings(value="EI_EXPOSE_REP2", justification = "intended_to_do") public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode, boolean usingLlap) throws IOException { mr = new MiniTezCluster("hive", numberOfTaskTrackers); @@ -553,6 +557,7 @@ private static void setKeyProvider(DFSClient dfsClient, KeyProviderCryptoExtensi public static class MiniDFSShim implements HadoopShims.MiniDFSShim { private final MiniDFSCluster cluster; + @SuppressFBWarnings(value="EI_EXPOSE_REP2", justification = "intended_to_do") public MiniDFSShim(MiniDFSCluster cluster) { this.cluster = cluster; } @@ -1005,6 +1010,7 @@ public static class StoragePolicyShim implements HadoopShims.StoragePolicyShim { private final DistributedFileSystem dfs; + @SuppressFBWarnings(value="EI_EXPOSE_REP2", justification = "intended_to_do") public StoragePolicyShim(DistributedFileSystem fs) { this.dfs = fs; } @@ -1334,6 +1340,7 @@ public static class HdfsEncryptionShim implements HadoopShims.HdfsEncryptionShim private final Configuration conf; + @SuppressFBWarnings(value="EI_EXPOSE_REP2", justification = "intended_to_do") public HdfsEncryptionShim(URI uri, Configuration conf) throws IOException { this.conf = conf; this.hdfsAdmin = new HdfsAdmin(uri, conf); diff --git a/shims/0.23/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim23.java b/shims/0.23/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim23.java index 5e40fdb7e5ce..01811eab8736 100644 --- a/shims/0.23/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim23.java +++ b/shims/0.23/src/main/java/org/apache/hadoop/mapred/WebHCatJTShim23.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.mapred; +import org.apache.hadoop.util.SuppressFBWarnings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -51,6 +52,7 @@ public class WebHCatJTShim23 implements WebHCatJTShim { /** * Create a connection to the Job Tracker. */ + @SuppressFBWarnings(value="EI_EXPOSE_REP2", justification = "intended_to_do") public WebHCatJTShim23(final Configuration conf, final UserGroupInformation ugi) throws IOException { try { diff --git a/shims/common/src/main/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java b/shims/common/src/main/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java index 83bb39b49240..7421c461ca44 100644 --- a/shims/common/src/main/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java +++ b/shims/common/src/main/java/org/apache/hadoop/fs/ProxyLocalFileSystem.java @@ -27,6 +27,7 @@ import org.apache.hadoop.util.Shell; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.MD5Hash; +import org.apache.hadoop.util.SuppressFBWarnings; /**************************************************************** * A Proxy for LocalFileSystem @@ -119,6 +120,7 @@ public static class PFileChecksum extends FileChecksum { private MD5Hash md5; private String algorithmName; + @SuppressFBWarnings(value="EI_EXPOSE_REP2", justification = "intended_to_do") public PFileChecksum(MD5Hash md5, String algorithmName) { this.md5 = md5; this.algorithmName = algorithmName; diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java b/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java index adf4d41e5e17..d74c16b5367b 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/io/HdfsUtils.java @@ -39,6 +39,7 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.SuppressFBWarnings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -211,6 +212,7 @@ public HadoopFileStatus(Configuration conf, FileSystem fs, Path file) throws IOE this.aclStatus = aclStatus; } + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public FileStatus getFileStatus() { return fileStatus; } diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java index 9567a3d735f9..af87090a0116 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java @@ -31,6 +31,7 @@ import java.util.Set; import org.apache.commons.lang3.ArrayUtils; +import org.apache.hadoop.util.SuppressFBWarnings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -185,6 +186,7 @@ public float getProgress() throws IOException { * A generic RecordReader that can hand out different recordReaders * for each chunk in the CombineFileSplit. */ + @SuppressFBWarnings(value="EI_EXPOSE_REP2", justification = "intended_to_do") public CombineFileRecordReader(JobConf job, CombineFileSplit split, Reporter reporter, Class> rrClass) diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java index 20b0f60581fe..6f9b45c3a399 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/ShimLoader.java @@ -20,6 +20,7 @@ import java.util.HashMap; import java.util.Map; +import org.apache.hadoop.util.SuppressFBWarnings; import org.apache.hadoop.util.VersionInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,6 +75,7 @@ public abstract class ShimLoader { * Factory method to get an instance of HadoopShims based on the * version of Hadoop on the classpath. */ + @SuppressFBWarnings(value="MS_EXPOSE_REP", justification = "intended_to_do") public static HadoopShims getHadoopShims() { if (hadoopShims == null) { synchronized (ShimLoader.class) { diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/TFilterTransport.java b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/TFilterTransport.java index 29b13fb83ae5..951dafbe03e9 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/TFilterTransport.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/TFilterTransport.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.thrift; +import org.apache.hadoop.util.SuppressFBWarnings; import org.apache.thrift.TConfiguration; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; @@ -29,6 +30,7 @@ public class TFilterTransport extends TTransport { protected final TTransport wrapped; + @SuppressFBWarnings(value="EI_EXPOSE_REP2", justification = "intended_to_do") public TFilterTransport(TTransport wrapped) { this.wrapped = wrapped; } diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/TUGIContainingTransport.java b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/TUGIContainingTransport.java index 0936354580de..d713721e8348 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/TUGIContainingTransport.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/TUGIContainingTransport.java @@ -22,6 +22,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.SuppressFBWarnings; import org.apache.thrift.transport.TSocket; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportFactory; @@ -40,10 +41,12 @@ public TUGIContainingTransport(TTransport wrapped) { super(wrapped); } + @SuppressFBWarnings(value="EI_EXPOSE_REP", justification = "intended_to_do") public UserGroupInformation getClientUGI(){ return ugi; } + @SuppressFBWarnings(value="EI_EXPOSE_REP2", justification = "intended_to_do") public void setClientUGI(UserGroupInformation ugi){ this.ugi = ugi; } diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/client/TUGIAssumingTransport.java b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/client/TUGIAssumingTransport.java index 0c9f81c37aa5..a1c753459b08 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/thrift/client/TUGIAssumingTransport.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/thrift/client/TUGIAssumingTransport.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.thrift.TFilterTransport; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.SuppressFBWarnings; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; @@ -38,6 +39,7 @@ public class TUGIAssumingTransport extends TFilterTransport { protected UserGroupInformation ugi; + @SuppressFBWarnings(value="EI_EXPOSE_REP2", justification = "intended_to_do") public TUGIAssumingTransport(TTransport wrapped, UserGroupInformation ugi) { super(wrapped); this.ugi = ugi; diff --git a/shims/common/src/main/java/org/apache/hadoop/util/SuppressFBWarnings.java b/shims/common/src/main/java/org/apache/hadoop/util/SuppressFBWarnings.java new file mode 100644 index 000000000000..c2d2b8dc0a99 --- /dev/null +++ b/shims/common/src/main/java/org/apache/hadoop/util/SuppressFBWarnings.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.CLASS) +public @interface SuppressFBWarnings { + /** + * The set of FindBugs warnings that are to be suppressed in + * annotated element. The value can be a bug category, kind or pattern. + * + */ + String[] value() default {}; + + /** + * Optional documentation of the reason why the warning is suppressed + */ + String justification() default ""; +} + diff --git a/standalone-metastore/metastore-common/pom.xml b/standalone-metastore/metastore-common/pom.xml index 8ff828018ffd..3d827efb361d 100644 --- a/standalone-metastore/metastore-common/pom.xml +++ b/standalone-metastore/metastore-common/pom.xml @@ -66,6 +66,10 @@ io.netty * + + org.apache.hadoop + hadoop-client-api + @@ -91,14 +95,17 @@ io.dropwizard.metrics metrics-core + 3.2.4 io.dropwizard.metrics metrics-jvm + 3.2.4 io.dropwizard.metrics metrics-json + 3.2.4 javolution @@ -433,13 +440,13 @@ com.github.spotbugs spotbugs-maven-plugin - 4.0.0 + 4.5.0.0 com.github.spotbugs spotbugs - 4.0.3 + ${spotbugs.version} @@ -447,6 +454,7 @@ 2048 -Djava.awt.headless=true -Xmx2048m -Xms512m ${basedir}/${standalone.metastore.path.to.root}/spotbugs/spotbugs-exclude.xml + 60000000 @@ -456,12 +464,13 @@ com.github.spotbugs spotbugs-maven-plugin - 4.0.0 + 4.5.0.0 true 2048 -Djava.awt.headless=true -Xmx2048m -Xms512m ${basedir}/${standalone.metastore.path.to.root}/spotbugs/spotbugs-exclude.xml + 60000000 diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/ZKDeRegisterWatcher.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/ZKDeRegisterWatcher.java index 8ac9c201f862..18ed3a6c6487 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/ZKDeRegisterWatcher.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/ZKDeRegisterWatcher.java @@ -20,13 +20,13 @@ import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.Watcher; - +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; /** * The watcher class which sets the de-register flag when the given znode is deleted. */ public class ZKDeRegisterWatcher implements Watcher { private ZooKeeperHiveHelper zooKeeperHiveHelper; - + @SuppressFBWarnings(value = "EI_EXPOSE_REP2",justification = "intended_to_do") public ZKDeRegisterWatcher(ZooKeeperHiveHelper zooKeeperHiveHelper) { this.zooKeeperHiveHelper = zooKeeperHiveHelper; } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/common/MetricsFactory.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/common/MetricsFactory.java index a4a9f646923b..1924228f8bdc 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/common/MetricsFactory.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/common/MetricsFactory.java @@ -21,7 +21,7 @@ import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import java.lang.reflect.Constructor; - +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; /** * Class that manages a static Metric instance for this process. */ @@ -46,6 +46,7 @@ public synchronized static void init(Configuration conf) throws Exception { /** * Returns static Metrics instance, null if not initialized or closed. */ + @SuppressFBWarnings(value = "MS_EXPOSE_REP", justification="intended_to_do") public static Metrics getInstance() { return metrics; } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java index a69e66398880..1ca04c0d617e 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java @@ -49,6 +49,7 @@ import org.apache.hadoop.hive.common.metrics.common.MetricsScope; import org.apache.hadoop.hive.common.metrics.common.MetricsVariable; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -143,6 +144,7 @@ public void close() { } } + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public CodahaleMetrics(Configuration conf) { this.conf = conf; //Codahale artifacts are lazily-created. diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/metrics2/JmxMetricsReporter.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/metrics2/JmxMetricsReporter.java index 5bd55dc715a9..536dacc587b3 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/metrics2/JmxMetricsReporter.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/metrics2/JmxMetricsReporter.java @@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; /** * A wrapper around Codahale JmxReporter to make it a pluggable/configurable Hive Metrics reporter. @@ -32,6 +33,7 @@ public class JmxMetricsReporter implements CodahaleReporter { private final Configuration conf; private final JmxReporter jmxReporter; + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public JmxMetricsReporter(MetricRegistry registry, Configuration conf) { this.registry = registry; this.conf = conf; diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/metrics2/JsonFileMetricsReporter.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/metrics2/JsonFileMetricsReporter.java index 4038d66b7327..0a07b6e6ef94 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/metrics2/JsonFileMetricsReporter.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/metrics2/JsonFileMetricsReporter.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -93,6 +94,7 @@ public class JsonFileMetricsReporter implements CodahaleReporter, Runnable { // Directory where path resides private final Path metricsDir; + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public JsonFileMetricsReporter(MetricRegistry registry, Configuration conf) { this.metricRegistry = registry; this.jsonWriter = diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/metrics2/Metrics2Reporter.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/metrics2/Metrics2Reporter.java index 347a46417cf4..4c16abee30ab 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/metrics2/Metrics2Reporter.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/metrics/metrics2/Metrics2Reporter.java @@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; @@ -33,6 +34,7 @@ public class Metrics2Reporter implements CodahaleReporter { private final Configuration conf; private final HadoopMetrics2Reporter reporter; + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public Metrics2Reporter(MetricRegistry registry, Configuration conf) { this.conf = conf; String applicationName = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.METRICS_HADOOP2_COMPONENT_NAME); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/util/SuppressFBWarnings.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/util/SuppressFBWarnings.java new file mode 100644 index 000000000000..34de510a57de --- /dev/null +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/util/SuppressFBWarnings.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.util; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.CLASS) +public @interface SuppressFBWarnings { + /** + * The set of FindBugs warnings that are to be suppressed in + * annotated element. The value can be a bug category, kind or pattern. + * + */ + String[] value() default {}; + + /** + * Optional documentation of the reason why the warning is suppressed + */ + String justification() default ""; +} diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/CheckResult.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/CheckResult.java index 9807138c1172..e1464fafd788 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/CheckResult.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/CheckResult.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.hadoop.hive.metastore.api.MetaException; import java.util.Map; @@ -42,6 +43,7 @@ public class CheckResult { /** * @return a list of tables not found on the filesystem. */ + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public Set getTablesNotOnFs() { return tablesNotOnFs; } @@ -50,6 +52,7 @@ public Set getTablesNotOnFs() { * @param tablesNotOnFs * a list of tables not found on the filesystem. */ + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public void setTablesNotOnFs(Set tablesNotOnFs) { this.tablesNotOnFs = tablesNotOnFs; } @@ -57,6 +60,7 @@ public void setTablesNotOnFs(Set tablesNotOnFs) { /** * @return a list of tables not found in the metastore. */ + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public Set getTablesNotInMs() { return tablesNotInMs; } @@ -65,6 +69,7 @@ public Set getTablesNotInMs() { * @param tablesNotInMs * a list of tables not found in the metastore. */ + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public void setTablesNotInMs(Set tablesNotInMs) { this.tablesNotInMs = tablesNotInMs; } @@ -72,6 +77,7 @@ public void setTablesNotInMs(Set tablesNotInMs) { /** * @return a list of partitions not found on the fs */ + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public Set getPartitionsNotOnFs() { return partitionsNotOnFs; } @@ -80,6 +86,7 @@ public Set getPartitionsNotOnFs() { * @param partitionsNotOnFs * a list of partitions not found on the fs */ + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public void setPartitionsNotOnFs(Set partitionsNotOnFs) { this.partitionsNotOnFs = partitionsNotOnFs; } @@ -87,6 +94,7 @@ public void setPartitionsNotOnFs(Set partitionsNotOnFs) { /** * @return a list of partitions not found in the metastore */ + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public Set getPartitionsNotInMs() { return partitionsNotInMs; } @@ -95,22 +103,27 @@ public Set getPartitionsNotInMs() { * @param partitionsNotInMs * a list of partitions not found in the metastore */ + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public void setPartitionsNotInMs(Set partitionsNotInMs) { this.partitionsNotInMs = partitionsNotInMs; } + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public Set getExpiredPartitions() { return expiredPartitions; } + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public void setExpiredPartitions(final Set expiredPartitions) { this.expiredPartitions = expiredPartitions; } + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public Set getCorrectPartitions() { return this.correctPartitions; } + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public void setCorrectPartitions(final Set correctPartitions) { this.correctPartitions = correctPartitions; } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java index 3a4b6f5e9bf6..01de9945087a 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java @@ -204,7 +204,7 @@ public class ColumnType { /** * Given a type string return the type name. For example, passing in the type string - * varchar(256) will return varchar. + * 
varchar(256)
will return 
varchar
. * @param typeString Type string * @return type name, guaranteed to be in lower case */ diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/FileMetadataHandler.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/FileMetadataHandler.java index 73917fc54287..509a4e17a772 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/FileMetadataHandler.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/FileMetadataHandler.java @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -69,6 +70,7 @@ protected MetadataStore getStore() { * @param expressionProxy Expression proxy to access ql stuff. * @param store Storage interface to manipulate the metadata. */ + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public void configure( Configuration conf, PartitionExpressionProxy expressionProxy, MetadataStore store) { this.conf = conf; diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 32beb5831d06..1a66fcbbf908 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -66,6 +66,7 @@ import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.common.ValidTxnWriteIdList; import org.apache.hadoop.hive.common.ValidWriteIdList; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.hadoop.hive.metastore.api.*; import org.apache.hadoop.hive.metastore.api.Package; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; @@ -429,6 +430,7 @@ synchronized private URIResolverHook loadUriResolverHook() throws IllegalStateEx * Swaps the first element of the metastoreUris array with a random element from the * remainder of the array. */ + @SuppressFBWarnings(value ="DMI_RANDOM_USED_ONLY_ONCE", justification = "intended_to_do") private void promoteRandomMetaStoreURI() { if (metastoreUris.length <= 1) { return; @@ -441,11 +443,13 @@ private void promoteRandomMetaStoreURI() { } @VisibleForTesting + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public TTransport getTTransport() { return transport; } @VisibleForTesting + @SuppressFBWarnings(value = "MS_EXPOSE_REP", justification = "intended_to_do") public static AtomicInteger getConnCount() { return connCount; } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java index da6d6342d95f..1f44c0812e28 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/LockComponentBuilder.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.metastore; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.hadoop.hive.metastore.api.DataOperationType; import org.apache.hadoop.hive.metastore.api.LockComponent; import org.apache.hadoop.hive.metastore.api.LockLevel; @@ -66,6 +67,7 @@ public LockComponentBuilder setSharedWrite() { * Set the lock to be shared_read. * @return reference to this builder */ + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public LockComponentBuilder setSharedRead() { component.setType(LockType.SHARED_READ); return this; @@ -120,6 +122,7 @@ public LockComponentBuilder setIsDynamicPartitionWrite(boolean t) { * Get the constructed lock component. * @return lock component. */ + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public LockComponent build() { LockLevel level = LockLevel.DB; if (tableNameSet) level = LockLevel.TABLE; diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java index 7b4aeaeb80e6..acc9391a5d05 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/LockRequestBuilder.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.metastore; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.hadoop.hive.metastore.api.DataOperationType; import org.apache.hadoop.hive.metastore.api.LockComponent; import org.apache.hadoop.hive.metastore.api.LockRequest; @@ -56,6 +57,7 @@ public LockRequestBuilder(String agentInfo) { * Get the constructed LockRequest. * @return lock request */ + @SuppressFBWarnings(value="EI_EXPOSE_REP", justification = "intended_to_do") public LockRequest build() { if (!userSet) { throw new RuntimeException("Cannot build a lock without giving a user"); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java index a0680ca4234b..f2e8955b6eb0 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ReplChangeManager.java @@ -46,6 +46,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hive.common.repl.ReplConst; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Table; @@ -100,6 +101,7 @@ public static class FileInfo { public FileInfo(FileSystem srcFs, Path sourcePath, String subDir) { this(srcFs, sourcePath, null, null, true, subDir); } + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public FileInfo(FileSystem srcFs, Path sourcePath, Path cmPath, String checkSum, boolean useSourcePath, String subDir) { this.srcFs = srcFs; @@ -110,6 +112,7 @@ public FileInfo(FileSystem srcFs, Path sourcePath, Path cmPath, this.subDir = subDir; this.copyDone = false; } + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public FileSystem getSrcFs() { return srcFs; } @@ -475,6 +478,7 @@ public static class CMClearer implements Runnable { private long secRetain; private Configuration conf; + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public CMClearer(long secRetain, Configuration conf) { this.encryptionZones = encryptionZoneToCmrootMapping; this.secRetain = secRetain; diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TableIterable.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TableIterable.java index 1a17fe31c365..e33b5c6e45ea 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TableIterable.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TableIterable.java @@ -22,6 +22,7 @@ import java.util.Iterator; import java.util.List; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.thrift.TException; @@ -95,6 +96,7 @@ public void remove() { * Primary constructor that fetches all tables in a given msc, given a Hive * object,a db name and a table name list. */ + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public TableIterable(IMetaStoreClient msc, String dbname, List tableNames, int batchSize) throws TException { this.msc = msc; @@ -104,6 +106,7 @@ public TableIterable(IMetaStoreClient msc, String dbname, List tableName this.batchSize = batchSize; } + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public TableIterable(IMetaStoreClient msc, String catName, String dbname, List tableNames, int batchSize) throws TException { this.msc = msc; diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java index 2952276020c7..566964c44256 100755 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/Warehouse.java @@ -34,6 +34,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.hadoop.hive.metastore.api.Catalog; import org.apache.hadoop.hive.metastore.api.DatabaseType; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; @@ -86,6 +87,7 @@ public class Warehouse { private boolean storageAuthCheck = false; private ReplChangeManager cm = null; + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public Warehouse(Configuration conf) throws MetaException { this.conf = conf; whRootString = MetastoreConf.getVar(conf, ConfVars.WAREHOUSE); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 713a45aa6f7c..92e551b72add 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -2377,6 +2377,7 @@ public static void setBoolVar(Configuration conf, ConfVars var, boolean val) { * @param var variable to retrieve * @return value, or default value if value not in config file */ + @SuppressFBWarnings(value = "DM_BOXED_PRIMITIVE_FOR_PARSING", justification = "intended_to_do") public static double getDoubleVar(Configuration conf, ConfVars var) { assert var.defaultVal.getClass() == Double.class; String val = conf.get(var.varname); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ldap/Query.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ldap/Query.java index fb4a409ffbb2..6806f9412cca 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ldap/Query.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ldap/Query.java @@ -21,6 +21,8 @@ import java.util.ArrayList; import java.util.List; import javax.naming.directory.SearchControls; + +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.stringtemplate.v4.ST; /** @@ -38,6 +40,7 @@ public final class Query { * @param filter search filter * @param controls search controls */ + @SuppressFBWarnings(value="EI_EXPOSE_REP2", justification = "intended_to_do") public Query(String filter, SearchControls controls) { this.filter = filter; this.controls = controls; @@ -55,6 +58,7 @@ public String getFilter() { * Returns search controls. * @return search controls */ + @SuppressFBWarnings(value="EI_EXPOSE_REP", justification = "intended_to_do") public SearchControls getControls() { return controls; } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ldap/SearchResultHandler.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ldap/SearchResultHandler.java index 88704cec2258..aab39f55d109 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ldap/SearchResultHandler.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ldap/SearchResultHandler.java @@ -24,6 +24,8 @@ import javax.naming.NamingException; import javax.naming.directory.Attribute; import javax.naming.directory.SearchResult; + +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,6 +43,7 @@ public final class SearchResultHandler { * Constructs a search result handler object for the provided search results. * @param searchResults directory service search results */ + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public SearchResultHandler(Collection> searchResults) { this.searchResults = searchResults; } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java index 91d790aa6473..f9f35af7c5d9 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/CompositePartitionSpecProxy.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore.partition.spec; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PartitionSpec; @@ -107,6 +108,7 @@ public static class Iterator implements PartitionIterator { private int index = -1; // Index into partitionSpecs. private PartitionIterator iterator = null; + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public Iterator(CompositePartitionSpecProxy composite) { this.composite = composite; this.partitionSpecProxies = composite.partitionSpecProxies; @@ -245,6 +247,7 @@ public PartitionIterator getPartitionIterator() { } @Override + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public List toPartitionSpec() { return partitionSpecs; } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java index 585b8fd8b6b2..ede542767306 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionListComposingSpecProxy.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore.partition.spec; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PartitionListComposingSpec; @@ -139,6 +140,7 @@ public static class Iterator implements PartitionIterator { List partitionList; int index; + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public Iterator(PartitionListComposingSpecProxy partitionSpecProxy) { this.partitionSpecProxy = partitionSpecProxy; this.partitionList = partitionSpecProxy.partitionSpec.getPartitionList().getPartitions(); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java index 4ea19fa86acd..1f55af92ccdc 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecProxy.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore.partition.spec; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PartitionSpec; @@ -202,8 +203,10 @@ public interface PartitionIterator extends java.util.Iterator { */ public static class SimplePartitionWrapperIterator implements PartitionIterator { private Partition partition; + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public SimplePartitionWrapperIterator(Partition partition) {this.partition = partition;} + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") @Override public Partition getCurrent() { return partition; } @Override public String getCatName() { return partition.getCatName(); } @Override public String getDbName() { return partition.getDbName(); } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java index 5b462066f4f2..78bea3bd0df5 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/partition/spec/PartitionSpecWithSharedSDProxy.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore.partition.spec; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PartitionSpec; @@ -38,6 +39,7 @@ public class PartitionSpecWithSharedSDProxy extends PartitionSpecProxy { private PartitionSpec partitionSpec; + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public PartitionSpecWithSharedSDProxy(PartitionSpec partitionSpec) throws MetaException { assert partitionSpec.isSetSharedSDPartitionSpec(); if (partitionSpec.getSharedSDPartitionSpec().getSd() == null) { diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/HadoopThriftAuthBridge.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/HadoopThriftAuthBridge.java index a5e71d75cf59..b14faa7b5355 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/HadoopThriftAuthBridge.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/HadoopThriftAuthBridge.java @@ -41,6 +41,7 @@ import javax.security.sasl.SaslException; import javax.security.sasl.SaslServer; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -374,6 +375,7 @@ protected Server(String keytabFile, String principalConf, String clientConf) } } + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public void setSecretManager(DelegationTokenSecretManager secretManager) { this.secretManager = secretManager; } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/TFilterTransport.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/TFilterTransport.java index e9670e38dede..94b60285aa5d 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/TFilterTransport.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/TFilterTransport.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore.security; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.thrift.TConfiguration; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; @@ -29,6 +30,7 @@ public class TFilterTransport extends TTransport { protected final TTransport wrapped; + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public TFilterTransport(TTransport wrapped) { this.wrapped = wrapped; } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/TUGIAssumingTransport.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/TUGIAssumingTransport.java index 38a946e2d2f8..e62124ce51ae 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/TUGIAssumingTransport.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/security/TUGIAssumingTransport.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.security.PrivilegedExceptionAction; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.hadoop.security.UserGroupInformation; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; @@ -37,6 +38,7 @@ public class TUGIAssumingTransport extends TFilterTransport { protected UserGroupInformation ugi; + @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do") public TUGIAssumingTransport(TTransport wrapped, UserGroupInformation ugi) { super(wrapped); this.ugi = ugi; diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java index cf8825ec1612..0ba72757a59e 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java @@ -38,6 +38,7 @@ import org.apache.hadoop.fs.Trash; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.SnapshotException; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; @@ -632,6 +633,7 @@ public boolean hasNext() throws IOException { } @Override + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public LocatedFileStatus next() throws IOException { if (!hasNext()) { throw new NoSuchElementException(); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java index fd9995591296..f1ebb79790aa 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/HdfsUtils.java @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.client.HdfsAdmin; +import org.apache.hadoop.hive.common.util.SuppressFBWarnings; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.tools.DistCp; @@ -260,6 +261,7 @@ public HadoopFileStatus(Configuration conf, FileSystem fs, Path file) throws IOE this.aclStatus = aclStatus; } + @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do") public FileStatus getFileStatus() { return fileStatus; } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java index d00c8f050191..f4194c430048 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java @@ -19,6 +19,7 @@ import org.apache.commons.lang3.ClassUtils; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hive.common.util.SuppressFBWarnings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -96,6 +97,7 @@ public static T newInstance(Class theClass, Class[] parameterTypes, * @param the type of the class to be returned * @return an object of the requested type */ + @SuppressFBWarnings(value = "REFLC_REFLECTION_MAY_INCREASE_ACCESSIBILITY_OF_CLASS", justification = "intended_to_do") public static T newInstance(Class theClass) { try { return theClass.newInstance(); diff --git a/standalone-metastore/metastore-server/pom.xml b/standalone-metastore/metastore-server/pom.xml index 8367ad533d5b..efa92b686635 100644 --- a/standalone-metastore/metastore-server/pom.xml +++ b/standalone-metastore/metastore-server/pom.xml @@ -46,6 +46,10 @@ org.apache.orc orc-core + + org.apache.hadoop + hadoop-client-api + org.apache.hadoop hadoop-common @@ -93,14 +97,17 @@ io.dropwizard.metrics metrics-core + 3.2.4 io.dropwizard.metrics metrics-jvm + 3.2.4 io.dropwizard.metrics metrics-json + 3.2.4 javolution @@ -272,18 +279,22 @@ org.datanucleus datanucleus-api-jdo + 6.0.0-release org.datanucleus datanucleus-core + 6.0.0-release org.datanucleus datanucleus-rdbms + 6.0.0-release org.datanucleus javax.jdo + 3.2.0-release org.skyscreamer @@ -439,13 +450,13 @@ com.github.spotbugs spotbugs-maven-plugin - 4.0.0 + 4.5.0.0 com.github.spotbugs spotbugs - 4.0.3 + ${spotbugs.version} @@ -462,7 +473,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.0.0 + 4.5.0.0 true 2048 @@ -550,6 +561,16 @@ exec-maven-plugin ${maven.exec.plugin.version} + + org.apache.maven.plugins + maven-compiler-plugin + + + --add-exports + jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + + + @@ -643,7 +664,7 @@ true false ${test.forkcount} - -Xmx2048m + -Xmx2048m --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/java.util.regex=ALL-UNNAMED --add-opens java.sql/java.sql=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/java.text=ALL-UNNAMED ${project.build.directory} true @@ -719,7 +740,7 @@ org.datanucleus datanucleus-maven-plugin - 5.2.1 + 6.0.0-release JDO false diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlInsertPart.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlInsertPart.java index ba205ebe705b..8e93f9c80422 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlInsertPart.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlInsertPart.java @@ -44,6 +44,7 @@ import org.datanucleus.api.jdo.JDOPersistenceManager; import org.datanucleus.identity.DatastoreId; import org.datanucleus.metadata.AbstractClassMetaData; +import org.datanucleus.metadata.AbstractMemberMetaData; import org.datanucleus.metadata.IdentityType; /** @@ -71,7 +72,7 @@ private Long getDataStoreId(Class modelClass) throws MetaException { ExecutionContext ec = ((JDOPersistenceManager) pm).getExecutionContext(); AbstractClassMetaData cmd = ec.getMetaDataManager().getMetaDataForClass(modelClass, ec.getClassLoaderResolver()); if (cmd.getIdentityType() == IdentityType.DATASTORE) { - return (Long) ec.getStoreManager().getValueGenerationStrategyValue(ec, cmd, -1); + return (Long) ec.getStoreManager().getValueGenerationStrategyValue(ec, cmd, null); } else { throw new MetaException("Identity type is not datastore."); } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdatePart.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdatePart.java index 4059d5df3845..6c9bfb38a08c 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdatePart.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/DirectSqlUpdatePart.java @@ -1095,7 +1095,7 @@ private Long getDataStoreId(Class modelClass) throws MetaException { ExecutionContext ec = ((JDOPersistenceManager) pm).getExecutionContext(); AbstractClassMetaData cmd = ec.getMetaDataManager().getMetaDataForClass(modelClass, ec.getClassLoaderResolver()); if (cmd.getIdentityType() == IdentityType.DATASTORE) { - return (Long) ec.getStoreManager().getValueGenerationStrategyValue(ec, cmd, -1); + return (Long) ec.getStoreManager().getValueGenerationStrategyValue(ec, cmd, null); } else { throw new MetaException("Identity type is not datastore."); } diff --git a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.1.0.derby.sql b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.1.0.derby.sql index ea1f90aff89e..19ddaf607714 100644 --- a/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.1.0.derby.sql +++ b/standalone-metastore/metastore-server/src/main/sql/derby/hive-schema-4.1.0.derby.sql @@ -36,9 +36,9 @@ CREATE TABLE "APP"."DATABASE_PARAMS" ("DB_ID" BIGINT NOT NULL, "PARAM_KEY" VARCH CREATE TABLE "APP"."TBL_COL_PRIVS" ("TBL_COLUMN_GRANT_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(767), "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "TBL_COL_PRIV" VARCHAR(128), "TBL_ID" BIGINT, "AUTHORIZER" VARCHAR(128)); -CREATE TABLE "APP"."SERDE_PARAMS" ("SERDE_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" CLOB); +CREATE TABLE "APP"."SERDE_PARAMS" ("SERDE_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(32672)); -CREATE TABLE "APP"."COLUMNS_V2" ("CD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(4000), "COLUMN_NAME" VARCHAR(767) NOT NULL, "TYPE_NAME" CLOB, "INTEGER_IDX" INTEGER NOT NULL); +CREATE TABLE "APP"."COLUMNS_V2" ("CD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(4000), "COLUMN_NAME" VARCHAR(767) NOT NULL, "TYPE_NAME" VARCHAR(32672), "INTEGER_IDX" INTEGER NOT NULL); CREATE TABLE "APP"."SORT_COLS" ("SD_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(767), "ORDER" INTEGER NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); @@ -62,7 +62,7 @@ CREATE TABLE "APP"."TYPES" ("TYPES_ID" BIGINT NOT NULL, "TYPE_NAME" VARCHAR(128) CREATE TABLE "APP"."GLOBAL_PRIVS" ("USER_GRANT_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "USER_PRIV" VARCHAR(128), "AUTHORIZER" VARCHAR(128)); -CREATE TABLE "APP"."PARTITION_PARAMS" ("PART_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" CLOB); +CREATE TABLE "APP"."PARTITION_PARAMS" ("PART_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(32672)); CREATE TABLE "APP"."PARTITION_EVENTS" ( "PART_NAME_ID" BIGINT NOT NULL, @@ -111,7 +111,7 @@ CREATE TABLE "APP"."TAB_COL_STATS"( "HISTOGRAM" BLOB ); -CREATE TABLE "APP"."TABLE_PARAMS" ("TBL_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" CLOB); +CREATE TABLE "APP"."TABLE_PARAMS" ("TBL_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(32672)); CREATE TABLE "APP"."BUCKETING_COLS" ("SD_ID" BIGINT NOT NULL, "BUCKET_COL_NAME" VARCHAR(256), "INTEGER_IDX" INTEGER NOT NULL); @@ -119,7 +119,7 @@ CREATE TABLE "APP"."TYPE_FIELDS" ("TYPE_NAME" BIGINT NOT NULL, "COMMENT" VARCHAR CREATE TABLE "APP"."NUCLEUS_TABLES" ("CLASS_NAME" VARCHAR(128) NOT NULL, "TABLE_NAME" VARCHAR(128) NOT NULL, "TYPE" VARCHAR(4) NOT NULL, "OWNER" VARCHAR(2) NOT NULL, "VERSION" VARCHAR(20) NOT NULL, "INTERFACE_NAME" VARCHAR(256) DEFAULT NULL); -CREATE TABLE "APP"."SD_PARAMS" ("SD_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" CLOB); +CREATE TABLE "APP"."SD_PARAMS" ("SD_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(32672)); CREATE TABLE "APP"."SKEWED_STRING_LIST" ("STRING_LIST_ID" BIGINT NOT NULL); @@ -171,7 +171,7 @@ CREATE TABLE "APP"."NOTIFICATION_LOG" ( "EVENT_ID" BIGINT NOT NULL, "EVENT_TIME" INTEGER NOT NULL, "EVENT_TYPE" VARCHAR(32) NOT NULL, - "MESSAGE" CLOB, + "MESSAGE" VARCHAR(32672), "TBL_NAME" VARCHAR(256), "MESSAGE_FORMAT" VARCHAR(16) ); @@ -218,7 +218,7 @@ CREATE TABLE "APP"."MV_CREATION_METADATA" ( "CAT_NAME" VARCHAR(256) NOT NULL, "DB_NAME" VARCHAR(128) NOT NULL, "TBL_NAME" VARCHAR(256) NOT NULL, - "TXN_LIST" CLOB, + "TXN_LIST" VARCHAR(32672), "MATERIALIZATION_TIME" BIGINT NOT NULL ); @@ -605,7 +605,7 @@ CREATE TABLE COMPACTION_QUEUE ( CQ_HIGHEST_WRITE_ID bigint, CQ_META_INFO varchar(2048) for bit data, CQ_HADOOP_JOB_ID varchar(32), - CQ_ERROR_MESSAGE clob, + CQ_ERROR_MESSAGE VARCHAR(32672), CQ_NEXT_TXN_ID bigint, CQ_TXN_ID bigint, CQ_COMMIT_TIME bigint, @@ -640,7 +640,7 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_HIGHEST_WRITE_ID bigint, CC_META_INFO varchar(2048) for bit data, CC_HADOOP_JOB_ID varchar(32), - CC_ERROR_MESSAGE clob, + CC_ERROR_MESSAGE VARCHAR(32672), CC_NEXT_TXN_ID bigint, CC_TXN_ID bigint, CC_COMMIT_TIME bigint, @@ -743,7 +743,7 @@ CREATE TABLE "APP"."SCHEMA_VERSION" ( "CD_ID" bigint references "APP"."CDS" ("CD_ID"), "STATE" integer not null, "DESCRIPTION" varchar(4000), - "SCHEMA_TEXT" clob, + "SCHEMA_TEXT" VARCHAR(32672), "FINGERPRINT" varchar(256), "SCHEMA_VERSION_NAME" varchar(256), "SERDE_ID" bigint references "APP"."SERDES" ("SERDE_ID") @@ -774,9 +774,9 @@ CREATE TABLE TXN_WRITE_NOTIFICATION_LOG ( WNL_DATABASE varchar(128) NOT NULL, WNL_TABLE varchar(256) NOT NULL, WNL_PARTITION varchar(767) NOT NULL, - WNL_TABLE_OBJ clob NOT NULL, - WNL_PARTITION_OBJ clob, - WNL_FILES clob, + WNL_TABLE_OBJ VARCHAR(32672) NOT NULL, + WNL_PARTITION_OBJ VARCHAR(32672), + WNL_FILES VARCHAR(32672), WNL_EVENT_TIME integer NOT NULL, PRIMARY KEY (WNL_TXNID, WNL_DATABASE, WNL_TABLE, WNL_PARTITION) ); @@ -835,7 +835,7 @@ CREATE TABLE "APP"."STORED_PROCS" ( "DB_ID" BIGINT NOT NULL, "NAME" VARCHAR(256) NOT NULL, "OWNER_NAME" VARCHAR(128) NOT NULL, - "SOURCE" clob NOT NULL, + "SOURCE" VARCHAR(32672) NOT NULL, PRIMARY KEY ("SP_ID") ); @@ -856,8 +856,8 @@ CREATE TABLE "APP"."PACKAGES" ( "DB_ID" BIGINT NOT NULL, "NAME" VARCHAR(256) NOT NULL, "OWNER_NAME" VARCHAR(128) NOT NULL, - "HEADER" clob NOT NULL, - "BODY" clob NOT NULL, + "HEADER" VARCHAR(32672) NOT NULL, + "BODY" VARCHAR(32672) NOT NULL, PRIMARY KEY ("PKG_ID") ); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTimeout.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTimeout.java index d80f818c0e80..21ca52baacde 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTimeout.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTimeout.java @@ -181,7 +181,7 @@ public void testConnectionTimeout() throws Exception { try(HiveMetaStoreClient c = new HiveMetaStoreClient(newConf)) { Assert.fail("should throw connection timeout exception."); } catch (MetaException e) { - Assert.assertTrue("unexpected Exception", e.getMessage().contains("connect timed out")); + Assert.assertTrue("unexpected Exception", e.getMessage().toLowerCase().contains("connect timed out")); } return null; }); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetastoreWithHttpJwt.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetastoreWithHttpJwt.java index c228e94fd491..ea6faa62114a 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetastoreWithHttpJwt.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetastoreWithHttpJwt.java @@ -28,6 +28,8 @@ import com.nimbusds.jwt.SignedJWT; import java.io.File; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.nio.file.Files; @@ -102,9 +104,10 @@ public static void makeEnvModifiable() throws Exception { private static void removeStaticFinalAndSetValue(Field field, Object value) throws Exception { field.setAccessible(true); - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + VarHandle modifiersHandle = MethodHandles.privateLookupIn(Field.class, MethodHandles.lookup()) + .findVarHandle(Field.class, "modifiers", int.class); + int modifiers = field.getModifiers(); + modifiersHandle.set(field, modifiers & ~Modifier.FINAL); field.set(null, value); } private static int port; diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java index 8632f233e040..4dc60b379ad2 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/utils/TestMetaStoreUtils.java @@ -32,6 +32,7 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; @@ -82,8 +83,9 @@ public void testDateToString() { @Test public void testTimestampToString() { - assertEquals(timestamp, MetaStoreUtils.convertTimestampToString(Timestamp.valueOf(timestamp))); - } + String expectedTimestampString = ZonedDateTime.parse(timestamp, FORMATTER.withZone(ZoneId.systemDefault())).format(FORMATTER); + assertEquals(expectedTimestampString, MetaStoreUtils.convertTimestampToString(Timestamp.valueOf(timestamp))); // In jdk17, Timestamp makes use of daylight savings which + } // needs to be incorporated in expectedString too @Test public void testStringToDate() { diff --git a/standalone-metastore/metastore-tools/metastore-benchmarks/pom.xml b/standalone-metastore/metastore-tools/metastore-benchmarks/pom.xml index 7d62642f9259..07691caf1e25 100644 --- a/standalone-metastore/metastore-tools/metastore-benchmarks/pom.xml +++ b/standalone-metastore/metastore-tools/metastore-benchmarks/pom.xml @@ -188,33 +188,6 @@ - - - - org.apache.maven.plugins - maven-compiler-plugin - - javac-with-errorprone - true - - - - - org.codehaus.plexus - plexus-compiler-javac-errorprone - ${javac.errorprone.version} - - - com.google.errorprone - error_prone_core - ${errorprone.core.version} - - - - - diff --git a/standalone-metastore/metastore-tools/tools-common/pom.xml b/standalone-metastore/metastore-tools/tools-common/pom.xml index f9919757c9c3..24b0573c93ff 100644 --- a/standalone-metastore/metastore-tools/tools-common/pom.xml +++ b/standalone-metastore/metastore-tools/tools-common/pom.xml @@ -68,30 +68,10 @@ org.apache.maven.plugins - maven-compiler-plugin + maven-surefire-plugin - javac-with-errorprone - true + ${maven.test.jvm.args} - - - - org.codehaus.plexus - plexus-compiler-javac-errorprone - ${javac.errorprone.version} - - - com.google.errorprone - error_prone_core - ${errorprone.core.version} - - - - - org.apache.maven.plugins - maven-surefire-plugin diff --git a/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkSuite.java b/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkSuite.java index d32e971eb267..f11551414278 100644 --- a/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkSuite.java +++ b/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/BenchmarkSuite.java @@ -47,7 +47,7 @@ * mean +/- margin * delta is removed from the result set. This helps remove random * outliers. * - *

Example

+ *

Example

* * 
  *   StringBuilder sb = new StringBuilder();
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index c604d5020cfb..497182fb6819 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -37,8 +37,9 @@
     
     UTF-8
     UTF-8
-    1.8
-    1.8
+    17
+    17
+    17
     false
     2.7.10
     ${settings.localRepository}
@@ -65,10 +66,10 @@
     1.1.3
     2.9.0
     1.2.0
-    5.2.8
-    5.2.10
+    6.0.3
+    6.0.8
     3.2.0-release
-    5.2.10
+    6.0.8
     10.14.2.0
     2.5.0
     6.2.1.jre8
@@ -77,7 +78,7 @@
     21.3.0.0
     0.1.2
     
-    3.1.0
+    3.2.4
     22.0
     3.3.6
     4.0.3
@@ -103,7 +104,7 @@
     5.2.0
     3.8.4
     9.1.6
-    4.0.3
+    4.5.0
     2.8.4
     1.7.30
     4.4.13
@@ -515,6 +516,7 @@
           maven-surefire-plugin
           ${maven.surefire.plugin.version}
           
+            ${maven.test.jvm.args}
             false
           
         
@@ -622,11 +624,11 @@
       spotbugs
       
         
-          
+          
           
             com.github.spotbugs
             spotbugs-maven-plugin
-            4.0.0
+            4.5.0.0
             
               
               
@@ -640,6 +642,7 @@
               2048
               -Djava.awt.headless=true -Xmx2048m -Xms512m
               ${basedir}/${standalone.metastore.path.to.root}/spotbugs/spotbugs-exclude.xml
+              60000000
             
           
         
@@ -649,12 +652,13 @@
           
             com.github.spotbugs
             spotbugs-maven-plugin
-            4.0.0
+            4.5.0.0
             
               true
               2048
               -Djava.awt.headless=true -Xmx2048m -Xms512m
               ${basedir}/${standalone.metastore.path.to.root}/spotbugs/spotbugs-exclude.xml
+              60000000
             
           
         
diff --git a/storage-api/pom.xml b/storage-api/pom.xml
index 0c955c425053..768c539b50d8 100644
--- a/storage-api/pom.xml
+++ b/storage-api/pom.xml
@@ -26,8 +26,8 @@
   Hive Storage API
   2008
   
-    1.8
-    1.8
+    17
+    17
     1.1.3
     22.0
     3.3.6
@@ -199,6 +199,7 @@
         maven-surefire-plugin
         ${maven.surefire.plugin.version}
         
+          ${maven.test.jvm.args}
           false
           -Xmx3g
           false
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java b/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
index a2d009bf028b..b5fc3bc228fb 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hive.common;
 
 import org.apache.hadoop.hive.common.io.DiskRangeList;
+import org.apache.hive.common.util.SuppressFBWarnings;
 
 
 /**
@@ -40,6 +41,7 @@ public DiskRangeInfo(int indexBaseOffset) {
     this.totalLength = indexBaseOffset;
   }
 
+  @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do")
   public void addDiskRange(DiskRangeList diskRange) {
     if (tail == null) {
       head = tail = diskRange;
@@ -49,6 +51,7 @@ public void addDiskRange(DiskRangeList diskRange) {
     totalLength += diskRange.getLength();
   }
 
+  @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do")
   public DiskRangeList getDiskRanges() {
     return head;
   }
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/MaterializationSnapshot.java b/storage-api/src/java/org/apache/hadoop/hive/common/MaterializationSnapshot.java
index b1bbba5c2bf1..8102bba68f9d 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/MaterializationSnapshot.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/MaterializationSnapshot.java
@@ -21,6 +21,7 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.hadoop.hive.common.type.SnapshotContext;
+import org.apache.hive.common.util.SuppressFBWarnings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -65,6 +66,7 @@ public MaterializationSnapshot(String validTxnList) {
     this.tableSnapshots = null;
   }
 
+  @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do")
   public MaterializationSnapshot(Map tableSnapshots) {
     this.validTxnList = null;
     this.tableSnapshots = tableSnapshots;
@@ -104,6 +106,7 @@ public String getValidTxnList() {
    * @return {@link Map} of snapshotIds where the key is the fully qualified name of the table and the
    * values is the {@link String} representation of snapshotId or null if all tables are native ACID
    */
+  @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do")
   public Map getTableSnapshots() {
     return tableSnapshots;
   }
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java
index d83ce8b0730e..c2ccb2da9bc2 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hive.common.io;
 
+import org.apache.hive.common.util.SuppressFBWarnings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -246,6 +247,7 @@ public boolean equals(Object other) {
   public static class CreateHelper {
     private DiskRangeList tail = null, head;
 
+    @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do")
     public DiskRangeList getTail() {
       return tail;
     }
@@ -264,6 +266,7 @@ public void addOrMerge(long offset, long end, boolean doMerge, boolean doLogNew)
       }
     }
 
+    @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do")
     public DiskRangeList get() {
       return head;
     }
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/EncodedColumnBatch.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/EncodedColumnBatch.java
index 037c1cec5af0..7e10dd9a2dd3 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/EncodedColumnBatch.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/EncodedColumnBatch.java
@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.apache.hive.common.util.SuppressFBWarnings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -61,10 +62,12 @@ public int decRef() {
       return i;
     }
 
+    @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do")
     public List getCacheBuffers() {
       return cacheBuffers;
     }
 
+    @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do")
     public void setCacheBuffers(List cacheBuffers) {
       this.cacheBuffers = cacheBuffers;
     }
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java
index a06ce39b46c6..6fbbec7da0e6 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/Decimal64ColumnVector.java
@@ -19,6 +19,7 @@
 
 import org.apache.hadoop.hive.common.type.HiveDecimal;
 import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hive.common.util.SuppressFBWarnings;
 
 /**
 
@@ -155,6 +156,7 @@ public void setElement(int outputElementNum, int inputElementNum, ColumnVector i
    * Return a convenience writable object stored by this column vector.
    * @return
    */
+  @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do")
   public HiveDecimalWritable getScratchWritable() {
     return scratchHiveDecWritable;
   }
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java
index 9324bc0c610d..b3821e11d0a9 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/IntervalDayTimeColumnVector.java
@@ -21,6 +21,7 @@
 
 import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
 import org.apache.hadoop.io.Writable;
+import org.apache.hive.common.util.SuppressFBWarnings;
 
 /**
  * This class represents a nullable interval day time column vector capable of handing a
@@ -134,6 +135,7 @@ public void intervalDayTimeUpdate(HiveIntervalDayTime intervalDayTime, int eleme
    * @param elementNum
    * @return
    */
+  @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do")
   public HiveIntervalDayTime asScratchIntervalDayTime(int elementNum) {
     scratchIntervalDayTime.set(totalSeconds[elementNum], nanos[elementNum]);
     return scratchIntervalDayTime;
@@ -143,6 +145,7 @@ public HiveIntervalDayTime asScratchIntervalDayTime(int elementNum) {
    * Return the scratch HiveIntervalDayTime (contents undefined).
    * @return
    */
+  @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do")
   public HiveIntervalDayTime getScratchIntervalDayTime() {
     return scratchIntervalDayTime;
   }
@@ -413,6 +416,7 @@ public void fill(HiveIntervalDayTime intervalDayTime) {
    * Supports keeping a TimestampWritable object without having to import that definition...
    * @return
    */
+  @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do")
   public Writable getScratchWritable() {
     return scratchWritable;
   }
@@ -421,6 +425,7 @@ public Writable getScratchWritable() {
    * Set the convenience writable object stored by this column vector
    * @param scratchWritable
    */
+  @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do")
   public void setScratchWritable(Writable scratchWritable) {
     this.scratchWritable = scratchWritable;
   }
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
index f97156c40381..0c40a9687a6f 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
@@ -478,6 +478,7 @@ public void fill(Timestamp timestamp) {
    * Supports keeping a TimestampWritable object without having to import that definition...
    * @return
    */
+  @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do")
   public Writable getScratchWritable() {
     return scratchWritable;
   }
@@ -486,6 +487,7 @@ public Writable getScratchWritable() {
    * Set the convenience writable object stored by this column vector
    * @param scratchWritable
    */
+  @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do")
   public void setScratchWritable(Writable scratchWritable) {
     this.scratchWritable = scratchWritable;
   }
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/ExpressionTree.java b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/ExpressionTree.java
index 9895c4bfadd6..e0da53a6584e 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/ExpressionTree.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/ExpressionTree.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.hive.ql.io.sarg;
 
+import org.apache.hive.common.util.SuppressFBWarnings;
+
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -160,6 +162,7 @@ public Operator getOperator() {
     return operator;
   }
 
+  @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do")
   public List getChildren() {
     return children;
   }
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
index b7d03fd4c3d6..9cf9853a4853 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
@@ -30,6 +30,7 @@
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.common.NoDynamicValuesException;
 
+import org.apache.hive.common.util.SuppressFBWarnings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -67,6 +68,7 @@ public PredicateLeafImpl(Operator operator,
       this(operator, type, columnName, literal, literalList, null);
     }
 
+    @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do")
     public PredicateLeafImpl(Operator operator,
                              Type type,
                              String columnName,
@@ -111,6 +113,7 @@ public Object getLiteral() {
     }
 
     @Override
+    @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do")
     public List getLiteralList() {
       if (literalList != null && literalList.size() > 0 && literalList.get(0) instanceof LiteralDelegate) {
         List newLiteraList = new ArrayList<>();
@@ -235,6 +238,7 @@ protected void checkLiteralType(Object literal, Type type, Configuration conf) {
   }
 
   @Override
+  @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do")
   public List getLeaves() {
     return leaves;
   }
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 399ee9f56c90..e1ed2e324b1a 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -43,6 +43,12 @@
       org.apache.hive
       hive-exec
       ${project.version}
+      
+        
+          org.apache.hadoop
+          hadoop-client-api
+        
+      
     
     
       org.slf4j
diff --git a/testutils/hadoop b/testutils/hadoop
index 08fa2e87985f..f86d476e890c 100755
--- a/testutils/hadoop
+++ b/testutils/hadoop
@@ -29,7 +29,8 @@ bin=`cd "$bin"; pwd`
 export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4}
 
 JAVA=$JAVA_HOME/bin/java
-JAVA_HEAP_MAX=-Xmx1000m 
+JAVA_HEAP_MAX=-Xmx1000m
+JAVA17_ADDITIONAL_ARGS=" --add-opens java.base/java.net=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED"
 
 # check envvars which might override default args
 if [ "$HADOOP_HEAPSIZE" != "" ]; then
@@ -71,6 +72,9 @@ HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.policy.file=$HADOOP_POLICYFILE"
 # Disable ipv6 as it can cause issues
 HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true"
 
+# Additional arguments for JAVA 17
+HADOOP_OPTS="$HADOOP_OPTS $JAVA17_ADDITIONAL_ARGS"
+
 COMMAND=$1
 # the core commands
 if [ "$COMMAND" = "fs" ] ; then
JSON TypeJava Type