WIP: Dont review

.github/workflows/build.yml

@@ -29,14 +29,14 @@ env:
 
 jobs:
   macos-jdk8:
-    name: 'macOS (JDK 8)'
+    name: 'macOS (JDK 17)'
     runs-on: macos-latest
     steps:
       - uses: actions/checkout@v2
-      - name: 'Set up JDK 8'
+      - name: 'Set up JDK 17'
         uses: actions/setup-java@v1
         with:
-          java-version: 8
+          java-version: 17
       - name: 'Build project'
         run: |
           mvn clean install -DskipTests -Pitests

Jenkinsfile

@@ -84,11 +84,12 @@ def buildHive(args) {
 set -x
 . /etc/profile.d/confs.sh
 export USER="`whoami`"
-export MAVEN_OPTS="-Xmx2g"
+export MAVEN_OPTS="-Xmx4G"
 export -n HIVE_CONF_DIR
+sw java 17 && . /etc/profile.d/java.sh
 cp $SETTINGS .git/settings.xml
 OPTS=" -s $PWD/.git/settings.xml -B -Dtest.groups= "
-OPTS+=" -Pitests,qsplits,dist,errorProne,iceberg"
+OPTS+=" -Pitests,qsplits,dist,iceberg"
 OPTS+=" -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugin.surefire.SurefirePlugin=INFO"
 OPTS+=" -Dmaven.repo.local=$PWD/.git/m2"
 git config extra.mavenOpts "$OPTS"
@@ -254,7 +255,7 @@ if [ $n != 0 ]; then
   exit 1
 fi
 '''
-        buildHive("-Pspotbugs -pl " + spotbugsProjects.join(",") + " -am test-compile com.github.spotbugs:spotbugs-maven-plugin:4.0.0:check")
+         buildHive("-Pspotbugs -pl " + spotbugsProjects.join(",") + " -am test-compile com.github.spotbugs:spotbugs-maven-plugin:4.5.0.0:check")
       }
       stage('Compile') {
         buildHive("install -Dtest=noMatches")
@@ -283,12 +284,14 @@ fi
         stage('init-metastore') {
            withEnv(["dbType=$dbType"]) {
              sh '''#!/bin/bash -e
+             sw java 17 && . /etc/profile.d/java.sh
 set -x
 echo 127.0.0.1 dev_$dbType | sudo tee -a /etc/hosts
 . /etc/profile.d/confs.sh
 sw hive-dev $PWD
 export DOCKER_NETWORK=host
 export DBNAME=metastore
+export HADOOP_CLIENT_OPTS="--add-opens java.base/java.net=ALL-UNNAMED"
 reinit_metastore $dbType
 time docker rm -f dev_$dbType || true
 '''
@@ -393,6 +396,7 @@ tar -xzf packaging/target/apache-hive-*-nightly-*-src.tar.gz
       }
       stage('Generate javadoc') {
           sh """#!/bin/bash -e
+          sw java 17 && . /etc/profile.d/java.sh
 mvn install javadoc:javadoc javadoc:aggregate -DskipTests -pl '!itests/hive-jmh,!itests/util'
 """
       }

bin/ext/beeline.sh

@@ -29,7 +29,7 @@ beeline () {
     hadoopClasspath="${HADOOP_CLASSPATH}:"
   fi
   export HADOOP_CLASSPATH="${hadoopClasspath}${HIVE_CONF_DIR}:${beelineJarPath}:${superCsvJarPath}:${jlineJarPath}"
-  export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=beeline-log4j2.properties "
+  export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=beeline-log4j2.properties --add-opens=java.base/java.net=ALL-UNNAMED "
 
   if [ "$EXECUTE_WITH_JAVA" != "true" ] ; then
     # if CLIUSER is not empty, then pass it as user id / password during beeline redirect

common/pom.xml

@@ -34,6 +34,11 @@
       <artifactId>hive-classification</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.avro</groupId>
+      <artifactId>avro</artifactId>
+      <version>${avro.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.hive</groupId>
       <artifactId>hive-shims</artifactId>
@@ -69,6 +74,12 @@
     <dependency>
       <groupId>org.apache.orc</groupId>
       <artifactId>orc-core</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-client-api</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>jline</groupId>
@@ -373,6 +384,18 @@
       </testResource>
     </testResources>
     <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <compilerArgs>
+            <arg>--add-opens</arg>
+            <arg>org.apache.hadoop/org.apache.hadoop.fs=ALL-UNNAMED</arg>
+            <arg>--add-opens</arg>
+            <arg>java.base/java.net=ALL-UNNAMED</arg>
+          </compilerArgs>
+        </configuration>
+      </plugin>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-antrun-plugin</artifactId>
@@ -431,6 +454,14 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <source>17</source>
+          <target>17</target>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 </project>

common/src/java/org/apache/hadoop/hive/ant/GenHiveTemplate.java

@@ -73,21 +73,23 @@ private Document generateTemplate() throws Exception {
     doc.appendChild(doc.createProcessingInstruction(
         "xml-stylesheet", "type=\"text/xsl\" href=\"configuration.xsl\""));
 
-    doc.appendChild(doc.createComment("\n" +
-        "   Licensed to the Apache Software Foundation (ASF) under one or more\n" +
-        "   contributor license agreements.  See the NOTICE file distributed with\n" +
-        "   this work for additional information regarding copyright ownership.\n" +
-        "   The ASF licenses this file to You under the Apache License, Version 2.0\n" +
-        "   (the \"License\"); you may not use this file except in compliance with\n" +
-        "   the License.  You may obtain a copy of the License at\n" +
-        "\n" +
-        "       http://www.apache.org/licenses/LICENSE-2.0\n" +
-        "\n" +
-        "   Unless required by applicable law or agreed to in writing, software\n" +
-        "   distributed under the License is distributed on an \"AS IS\" BASIS,\n" +
-        "   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" +
-        "   See the License for the specific language governing permissions and\n" +
-        "   limitations under the License.\n"));
+    doc.appendChild(doc.createComment("""
+
+           Licensed to the Apache Software Foundation (ASF) under one or more
+           contributor license agreements.  See the NOTICE file distributed with
+           this work for additional information regarding copyright ownership.
+           The ASF licenses this file to You under the Apache License, Version 2.0
+           (the "License"); you may not use this file except in compliance with
+           the License.  You may obtain a copy of the License at
+
+               http://www.apache.org/licenses/LICENSE-2.0
+
+           Unless required by applicable law or agreed to in writing, software
+           distributed under the License is distributed on an "AS IS" BASIS,
+           WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+           See the License for the specific language governing permissions and
+           limitations under the License.
+        """));
 
     Element root = doc.createElement("configuration");
     doc.appendChild(root);

common/src/java/org/apache/hadoop/hive/common/CompressionUtils.java

@@ -169,30 +169,30 @@ public static List<File> unTar(final String inputFileName, final String outputDi
             // no sub-directories
             continue;
           }
-          LOG.debug(String.format("Attempting to write output directory %s.",
-                  outputFile.getAbsolutePath()));
+          LOG.debug("Attempting to write output directory %s.".formatted(
+              outputFile.getAbsolutePath()));
           if (!outputFile.exists()) {
-            LOG.debug(String.format("Attempting to create output directory %s.",
-                    outputFile.getAbsolutePath()));
+            LOG.debug("Attempting to create output directory %s.".formatted(
+                outputFile.getAbsolutePath()));
             if (!outputFile.mkdirs()) {
-              throw new IllegalStateException(String.format("Couldn't create directory %s.",
-                      outputFile.getAbsolutePath()));
+              throw new IllegalStateException("Couldn't create directory %s.".formatted(
+                  outputFile.getAbsolutePath()));
             }
           }
         } else {
           final OutputStream outputFileStream;
           if (flatten) {
             File flatOutputFile = new File(outputDir, outputFile.getName());
-            LOG.debug(String.format("Creating flat output file %s.", flatOutputFile.getAbsolutePath()));
+            LOG.debug("Creating flat output file %s.".formatted(flatOutputFile.getAbsolutePath()));
             outputFileStream = new FileOutputStream(flatOutputFile);
           } else if (!outputFile.getParentFile().exists()) {
-            LOG.debug(String.format("Attempting to create output directory %s.",
-                    outputFile.getParentFile().getAbsoluteFile()));
+            LOG.debug("Attempting to create output directory %s.".formatted(
+                outputFile.getParentFile().getAbsoluteFile()));
             if (!outputFile.getParentFile().getAbsoluteFile().mkdirs()) {
-              throw new IllegalStateException(String.format("Couldn't create directory %s.",
-                      outputFile.getParentFile().getAbsolutePath()));
+              throw new IllegalStateException("Couldn't create directory %s.".formatted(
+                  outputFile.getParentFile().getAbsolutePath()));
             }
-            LOG.debug(String.format("Creating output file %s.", outputFile.getAbsolutePath()));
+            LOG.debug("Creating output file %s.".formatted(outputFile.getAbsolutePath()));
             outputFileStream = new FileOutputStream(outputFile);
           } else {
             outputFileStream = new FileOutputStream(outputFile);

common/src/java/org/apache/hadoop/hive/common/CopyOnFirstWriteProperties.java

@@ -361,6 +361,7 @@ public void setInterned(Properties p) {
   public CopyOnFirstWriteProperties() {
   }
 
+  @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do")
   public Properties getInterned() {
     return interned;
   }

common/src/java/org/apache/hadoop/hive/common/FileUtils.java

@@ -1007,8 +1007,10 @@ public static boolean rename(FileSystem fs, Path sourcePath,
     // If destPath directory exists, rename call will move the sourcePath
     // into destPath without failing. So check it before renaming.
     if (fs.exists(destPath)) {
-      throw new IOException("Cannot rename the source path. The destination "
-          + "path already exists.");
+      throw new IOException("""
+          Cannot rename the source path. The destination \
+          path already exists.\
+          """);
     }
     return fs.rename(sourcePath, destPath);
   }
@@ -1094,8 +1096,10 @@ public static void checkDeletePermission(Path path, Configuration conf, String u
     if (childStatus.getOwner().equals(user)) {
       return;
     }
-    String msg = String.format("Permission Denied: User %s can't delete %s because sticky bit is"
-        + " set on the parent dir and user does not own this file or its parent", user, path);
+    String msg = ("""
+        Permission Denied: User %s can't delete %s because sticky bit is\
+         set on the parent dir and user does not own this file or its parent\
+        """).formatted(user, path);
     throw new IOException(msg);
 
   }

common/src/java/org/apache/hadoop/hive/common/HeapMemoryMonitor.java

@@ -100,8 +100,10 @@ private static MemoryPoolMXBean getTenuredGenPool() {
         if (isUsageThresholdSupported) {
           return pool;
         } else {
-          LOG.error("{} vendor does not support isCollectionUsageThresholdSupported() and isUsageThresholdSupported()" +
-            " for tenured memory pool '{}'.", vendor, pool.getName());
+          LOG.error("""
+            {} vendor does not support isCollectionUsageThresholdSupported() and isUsageThresholdSupported()\
+             for tenured memory pool '{}'.\
+            """, vendor, pool.getName());
         }
       }
     }

common/src/java/org/apache/hadoop/hive/common/JavaUtils.java

@@ -70,8 +70,8 @@ public static boolean closeClassLoadersTo(ClassLoader current, ClassLoader stop)
       try {
         closeClassLoader(current);
       } catch (IOException e) {
-        String detailedMessage = current instanceof URLClassLoader ?
-            Arrays.toString(((URLClassLoader) current).getURLs()) :
+        String detailedMessage = current instanceof URLClassLoader urlcl ?
+            Arrays.toString(urlcl.getURLs()) :
             "";
         LOG.info("Failed to close class loader " + current + " " + detailedMessage, e);
       }
@@ -90,8 +90,8 @@ private static boolean isValidHierarchy(ClassLoader current, ClassLoader stop) {
   }
 
   public static void closeClassLoader(ClassLoader loader) throws IOException {
-    if (loader instanceof Closeable) {
-      ((Closeable) loader).close();
+    if (loader instanceof Closeable closeable) {
+      closeable.close();
     } else {
       LOG.warn("Ignoring attempt to close class loader ({}) -- not instance of UDFClassLoader.",
           loader == null ? "mull" : loader.getClass().getSimpleName());

common/src/java/org/apache/hadoop/hive/common/LogUtils.java

@@ -245,16 +245,14 @@ public static void unregisterLoggingContext() {
   public static String getLogFilePath() {
     String logFilePath = null;
     org.apache.logging.log4j.Logger rootLogger = LogManager.getRootLogger();
-    if (rootLogger instanceof org.apache.logging.log4j.core.Logger) {
-      org.apache.logging.log4j.core.Logger coreLogger =
-          (org.apache.logging.log4j.core.Logger)rootLogger;
+    if (rootLogger instanceof org.apache.logging.log4j.core.Logger coreLogger) {
       for (Appender appender : coreLogger.getAppenders().values()) {
-        if (appender instanceof FileAppender) {
-          logFilePath = ((FileAppender) appender).getFileName();
-        } else if (appender instanceof RollingFileAppender) {
-          logFilePath = ((RollingFileAppender) appender).getFileName();
-        } else if (appender instanceof RollingRandomAccessFileAppender) {
-          logFilePath = ((RollingRandomAccessFileAppender) appender).getFileName();
+        if (appender instanceof FileAppender fileAppender) {
+          logFilePath = fileAppender.getFileName();
+        } else if (appender instanceof RollingFileAppender fileAppender) {
+          logFilePath = fileAppender.getFileName();
+        } else if (appender instanceof RollingRandomAccessFileAppender fileAppender) {
+          logFilePath = fileAppender.getFileName();
         }
       }
     }

common/src/java/org/apache/hadoop/hive/common/cli/ShellCmdExecutor.java

@@ -22,12 +22,14 @@
 import java.io.PrintStream;
 
 import org.apache.hive.common.util.StreamPrinter;
+import org.apache.hive.common.util.SuppressFBWarnings;
 
 public class ShellCmdExecutor {
   private String cmd;
   private PrintStream out;
   private PrintStream err;
 
+  @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do")
   public ShellCmdExecutor(String cmd, PrintStream out, PrintStream err) {
     this.cmd = cmd;
     this.out = out;

common/src/java/org/apache/hadoop/hive/common/format/datetime/HiveSqlDateTimeFormatter.java

@@ -28,6 +28,7 @@
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.hive.common.type.Date;
 import org.apache.hadoop.hive.common.type.Timestamp;
+import org.apache.hive.common.util.SuppressFBWarnings;
 
 import java.io.Serializable;
 import java.time.DateTimeException;
@@ -520,6 +521,7 @@ public Token(TokenType tokenType, String string) {
       this(tokenType, null, null, string, string.length(), false);
     }
 
+    @SuppressFBWarnings(value = "EI_EXPOSE_REP2", justification = "intended_to_do")
     public Token(TokenType tokenType, TemporalField temporalField, TemporalUnit temporalUnit,
         String string, int length, boolean fillMode) {
       this.type = tokenType;
@@ -847,17 +849,21 @@ private void verifyForParse() {
         !(temporalFields.contains(ChronoField.MONTH_OF_YEAR) &&
             temporalFields.contains(ChronoField.DAY_OF_MONTH) ||
             temporalFields.contains(ChronoField.DAY_OF_YEAR))) {
-      throw new IllegalArgumentException("Missing day of year or (month of year + day of month)"
-          + " tokens.");
+      throw new IllegalArgumentException("""
+          Missing day of year or (month of year + day of month)\
+           tokens.\
+          """);
     }
     if (containsIsoFields &&
         !(temporalFields.contains(IsoFields.WEEK_OF_WEEK_BASED_YEAR) &&
             temporalFields.contains(ChronoField.DAY_OF_WEEK))) {
       throw new IllegalArgumentException("Missing week of year (iw) or day of week (id) tokens.");
     }
     if (roundYearCount > 0 && yearCount > 0) {
-      throw new IllegalArgumentException("Invalid duplication of format element: Both year and"
-          + "round year are provided");
+      throw new IllegalArgumentException("""
+          Invalid duplication of format element: Both year and\
+          round year are provided\
+          """);
     }
     for (TemporalField tokenType : temporalFields) {
       if (Collections.frequency(temporalFields, tokenType) > 1) {
@@ -1281,8 +1287,10 @@ private int parseNumericTemporal(String substring, Token token) {
         return 0;
       }
       if ("0".equals(substring)) {
-        throw new IllegalArgumentException("Value of hour of day (hh/hh12) in input is 0. "
-            + "The value should be between 1 and 12.");
+        throw new IllegalArgumentException("""
+            Value of hour of day (hh/hh12) in input is 0. \
+            The value should be between 1 and 12.\
+            """);
       }
     }
     if (token.temporalField == ChronoField.YEAR

common/src/java/org/apache/hadoop/hive/common/io/CachingPrintStream.java

@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.hive.common.io;
 
+import org.apache.hive.common.util.SuppressFBWarnings;
+
 import java.io.FileNotFoundException;
 import java.io.OutputStream;
 import java.io.UnsupportedEncodingException;
@@ -52,6 +54,7 @@ public void flush() {
     super.flush();
   }
 
+  @SuppressFBWarnings(value = "EI_EXPOSE_REP", justification = "intended_to_do")
   public List<String> getOutput() {
     return output;
   }