ENG-18112: backport java 11 support to v8.4 (#7181)

* ENG-15241: compile voltdb in java 8 and make it runnable in java 11

* pass fileDescriptor object to native

* remove unsupport jvm option

* fix appClassLoader change caused test failure

* fix NumberFormatException change caused test failure

* fix jni GetIntField passing wrong object

* upgrade mockito and powerMock uptodate for java9+

* stricter match for mockito2

* classLoader path for new test

* cleanup some obsolete code

* replace javaClass-1.7 with JAXB package and dependencies

* update felix version for support java 9+

* address the reviews

* update jetty for java11 support

* update Session Management for jetty 9.4

* ENG-18284: add cddl license in thirdyParty for jaxb (#6658)

* ENG-18284: add cddl license in thirdyParty for jaxb

* update more pacakges; also add edl license

* ENG-15874: remove unnessary lookup for cleaner() in Java 11 UnsafeDirectBufferCleaner (#6157)

* ENG-17757: explicitily check cleaner() field for distiguish managed buffers vs unmanged buffers (#6636)

* ENG-15690: fix jetty session invalidation npe (#6047)

* ENG-15690: fix jetty session invalidation npe

* correctly ignore the dynamically encrypted field

* ENG-17757 use designated container for native direct allocated memory (#6639)

* ENG-15830: Suppress Illegal reflective access warning in JAVA 11 (#6133)

* ENG-15863: Suppress Illegal reflective access warning for sqlcmd in JAVA 11 (#6300)

* ENG-15863: Suppress Illegal reflective access warning for sqlcmd

* simplify the regex for match java version

* fix cherry-pick mistakes

* remove obslate jars

Co-authored-by: Zeeman He <[email protected]>

bin/sqlcmd

@@ -131,6 +131,15 @@ for s in "$@"; do
   fi
 done
 
+JAVA_MAJOR_VERSION=$("$JAVA" -version 2>&1 | sed -E -n 's/.* version "([0-9]+).*/\1/p')
+#echo $JAVA_MAJOR_VERSION
+if [[ "$JAVA_MAJOR_VERSION" -ge 11 ]]; then
+    jvmOPTS+=("--add-opens" "java.base/java.lang=ALL-UNNAMED")
+    jvmOPTS+=("--add-opens" "java.base/sun.nio.ch=ALL-UNNAMED")
+    jvmOPTS+=("--add-opens" "java.base/java.net=ALL-UNNAMED")
+    jvmOPTS+=("--add-opens" "java.base/java.nio=ALL-UNNAMED")
+fi
+
 jvmOPTS+=("-Xmx512m" "-Dlog4j.configuration=file://${LOG4J_CONFIG_PATH}")
 jvmOPTS+=("-classpath" "${CLASSPATH}" "org.voltdb.utils.SQLCommand")
 

build-client.xml

@@ -13,7 +13,7 @@
     </sequential>
 </macrodef>
 
-<macrodef name="invoke-javac16">
+<macrodef name="invoke-javac17">
     <attribute name="srcdir"/>
     <attribute name="destdir" default="${build.client.dir}"/>
     <attribute name="excludes" default=""/>
@@ -78,6 +78,7 @@
         <include name='log4j-1.2.16.jar' />
         <include name='commons-logging-1.1.3.jar' />
         <include name='slf4j-api-1.6.2.jar' />
+        <include name='jaxb-api-2.3.0.jar' />
         <include name='netty-all-4.1.43.Final.jar' />
         <include name='netty-tcnative-boringssl-static-2.0.27.Final.jar' />
     </fileset>
@@ -134,6 +135,10 @@
                   org/voltcore/utils/ssl/SSLBufferDecrypter.java
                   org/voltcore/utils/ssl/SSLBufferEncrypter.java
                   org/voltcore/utils/ssl/SSLConfiguration.java
+                  org/voltcore/utils/VoltUnsafe.java
+                  org/voltcore/utils/DirectBufferCleaner.java
+                  org/voltcore/utils/ReflectiveDirectBufferCleaner.java
+                  org/voltcore/utils/UnsafeDirectBufferCleaner.java
 
                   org/voltdb/CLIConfig.java
                   org/voltdb/client/**/*.java
@@ -331,8 +336,8 @@ JAVA COMPILATION
         <fileset dir="${src.gpl.dir}" includes="org/voltdb/*.properties"/>
     </copy>
 
-    <invoke-javac16 srcdir="${build.clientsrc.dir}">
-    </invoke-javac16>
+    <invoke-javac17 srcdir="${build.clientsrc.dir}">
+    </invoke-javac17>
 
 </target>
 

build-importers.xml

@@ -126,7 +126,7 @@
             <exclude name='classes-java1.7.jar' />
         </fileset>
         <fileset dir='${lib.dir}'>
-            <include name='felix.jar'/>
+            <include name='felix-framework-5.6.12.jar'/>
             <include name='commons-lang3-3.8.1.jar'/>
         </fileset>
         <fileset refid="voltpro.classpath"/>

lib/python/voltcli/checkconfig.py

@@ -156,7 +156,7 @@ def test_java_version(output):
     if not java:
         utility.abort('Could not find java in environment, set JAVA_HOME or put java in the path.')
     javaVersion = utility.get_java_version(javaHome=java, verbose=True)
-    if '1.8.' in javaVersion:
+    if '1.8.' in javaVersion or '11.' in javaVersion:
         output['Java'] = ["PASS", javaVersion.strip()]
     elif len(javaVersion) > 0:
         output['Java'] = ["FAIL", "Unsupported " + javaVersion + " Check if Java has been installed properly and JAVA_HOME has been setup correctly."]

lib/python/voltcli/environment.py

@@ -99,7 +99,6 @@
 java_opts.append('-Dsun.net.inetaddr.negative.ttl=3600')
 java_opts.append('-XX:+HeapDumpOnOutOfMemoryError')
 java_opts.append('-XX:HeapDumpPath=/tmp')
-java_opts.append('-XX:+UseParNewGC')
 java_opts.append('-XX:+UseConcMarkSweepGC')
 java_opts.append('-XX:+CMSParallelRemarkEnabled')
 java_opts.append('-XX:+UseTLAB')
@@ -114,10 +113,17 @@
 java_opts.append('-XX:+CMSScavengeBeforeRemark')
 java_opts.append('-XX:+CMSClassUnloadingEnabled')
 
-# skip PermSize in Java 8
-if "1.8" not in java_version:
+# skip PermSize in Java 8 and above
+if "1.7" in java_version:
     java_opts.append('-XX:PermSize=64m')
 
+# Suppress Illegal reflective access warning
+if "11." in java_version:
+    java_opts.extend(['--add-opens', 'java.base/java.lang=ALL-UNNAMED'])
+    java_opts.extend(['--add-opens', 'java.base/sun.nio.ch=ALL-UNNAMED'])
+    java_opts.extend(['--add-opens', 'java.base/java.net=ALL-UNNAMED'])
+    java_opts.extend(['--add-opens', 'java.base/java.nio=ALL-UNNAMED'])
+
 def initialize(standalone_arg, command_name_arg, command_dir_arg, version_arg):
     """
     Set the VOLTDB_LIB and VOLTDB_VOLTDB environment variables based on the

lib/python/voltcli/utility.py

@@ -840,18 +840,12 @@ def get_java_version(javaHome="java", verbose=False):
         version.stdout.close()
         out, err = grep.communicate()
         version.wait()
-        if "1.8" in out:
-            if verbose:
-                return out
-            else:
-                return "1.8"
-        elif "1.7" in out:
-            if verbose:
-                return out
-            else:
-                return "1.7"
-        else:
-            return ""
+        if verbose:
+            return out
+        for version in ('11.0', '1.8', '1.7'):
+            if version in out:
+                return version
+        return ""
     except (OSError):
         return ""
 

src/catgen/in/javasrc/CatalogDiffEngine.java

@@ -267,7 +267,7 @@ public boolean hasSecurityUserChanges() {
         for (Map.Entry<CatalogType, TypeChanges> entry : ccg.groupChanges.entrySet()) {
             Set<String> fields = new HashSet<>();
             fields.addAll(entry.getValue().typeChanges.changedFields);
-            fields.remove(ignoredFields);
+            fields.removeAll(ignoredFields);
             if (fields.isEmpty()) {
                 groupModifications.remove(entry.getKey());
             }

src/ee/voltdbjni.cpp

@@ -67,6 +67,7 @@
 #define  __USE_GNU
 #endif // __USE_GNU
 #include <sched.h>
+#include <cerrno>
 #endif // LINUX
 #ifdef MACOSX
 #include <mach/task.h>
@@ -139,6 +140,7 @@ using namespace voltdb;
  */
 static VoltDBEngine *currentEngine = NULL;
 static JavaVM *currentVM = NULL;
+static jfieldID field_fd;
 
 void signalHandler(int signum, siginfo_t *info, void *context) {
     if (currentVM == NULL || currentEngine == NULL)
@@ -214,6 +216,16 @@ SHAREDLIB_JNIEXPORT jlong JNICALL Java_org_voltdb_jni_ExecutionEngine_nativeCrea
     currentVM = vm;
     if (isSunJVM == JNI_TRUE)
         setupSigHandler();
+    // retrieving the fieldId fd of FileDescriptor for later use
+    jclass class_fdesc = env->FindClass("java/io/FileDescriptor");
+    if (class_fdesc == NULL) {
+        assert(!"Failed to find filed if of FileDescriptor.");
+        throw std::exception();
+        return 0;
+    }
+    // poke the "fd" field with the file descriptor
+    field_fd = env->GetFieldID(class_fdesc, "fd", "I");
+
     JNITopend *topend = NULL;
     VoltDBEngine *engine = NULL;
     try {
@@ -1328,15 +1340,22 @@ SHAREDLIB_JNIEXPORT jlong JNICALL Java_org_voltdb_utils_PosixAdvise_madvise
 #endif
 }
 
+/**
+ * Utility used for access file descriptor number from JAVA FileDescriptor class
+ */
+jint getFdFromFileDescriptor(JNIEnv *env, jobject fdObject) {
+    return env-> GetIntField(fdObject, field_fd);
+}
+
 /*
  * Class:     org_voltdb_utils_PosixAdvise
- * Method:    fadvise
- * Signature: (JJJI)J
+ * Method:    nativeFadvise
+ * Signature: (Ljava/io/FileDescriptor;JJI)J
  */
-SHAREDLIB_JNIEXPORT jlong JNICALL Java_org_voltdb_utils_PosixAdvise_fadvise
-  (JNIEnv *, jclass, jlong fd, jlong offset, jlong length, jint advice) {
+SHAREDLIB_JNIEXPORT jlong JNICALL Java_org_voltdb_utils_PosixAdvise_nativeFadvise
+        (JNIEnv *env, jclass, jobject fdObject, jlong offset, jlong length, jint advice) {
 #ifdef LINUX
-    return posix_fadvise(static_cast<int>(fd), static_cast<off_t>(offset), static_cast<off_t>(length), advice);
+    return posix_fadvise(getFdFromFileDescriptor(env,fdObject), static_cast<off_t>(offset), static_cast<off_t>(length), advice);
 #else
     return 0;
 #endif
@@ -1345,35 +1364,35 @@ SHAREDLIB_JNIEXPORT jlong JNICALL Java_org_voltdb_utils_PosixAdvise_fadvise
 /*
  * Class:     org_voltdb_utils_PosixAdvise
  * Method:    sync_file_range
- * Signature: (JJJI)J
+ * Signature: (Ljava/io/FileDescriptor;JJI)J
  */
 SHAREDLIB_JNIEXPORT jlong JNICALL Java_org_voltdb_utils_PosixAdvise_sync_1file_1range
-  (JNIEnv *, jclass, jlong fd, jlong offset, jlong length, jint advice) {
+        (JNIEnv *env, jclass, jobject fdObject, jlong offset, jlong length, jint advice) {
 #ifdef LINUX
 #ifndef __NR_sync_file_range
 #error VoltDB server requires that your kernel headers define __NR_sync_file_range.
 #endif
-    return syscall(__NR_sync_file_range, static_cast<int>(fd), static_cast<loff_t>(offset), static_cast<loff_t>(length),
+    return syscall(__NR_sync_file_range, getFdFromFileDescriptor(env,fdObject), static_cast<loff_t>(offset), static_cast<loff_t>(length),
                    static_cast<unsigned int>(advice));
 #elif MACOSX
     return -1;
 #else
-    return fdatasync(static_cast<int>(fd));
+    return fdatasync(getFdFromFileDescriptor(env,fdObject));
 #endif
 }
 
 
 /*
  * Class:     org_voltdb_utils_PosixAdvise
  * Method:    fallocate
- * Signature: (JJJ)J
+ * Signature: (Ljava/io/FileDescriptor;JJ)J
  */
 SHAREDLIB_JNIEXPORT jlong JNICALL Java_org_voltdb_utils_PosixAdvise_fallocate
-  (JNIEnv *, jclass, jlong fd, jlong offset, jlong length) {
+        (JNIEnv *env, jclass, jobject fdObject, jlong offset, jlong length) {
 #ifdef MACOSX
     return -1;
 #else
-    return posix_fallocate(static_cast<int>(fd), static_cast<off_t>(offset), static_cast<off_t>(length));
+    return posix_fallocate(getFdFromFileDescriptor(env,fdObject), static_cast<off_t>(offset), static_cast<off_t>(length));
 #endif
 }
 

src/frontend/org/voltcore/utils/DBBPool.java.template

@@ -26,7 +26,7 @@ import java.util.concurrent.atomic.AtomicLong;
 import com.google_voltpatches.common.base.Preconditions;
 import org.cliffc_voltpatches.high_scale_lib.NonBlockingHashMap;
 import org.voltcore.logging.VoltLogger;
-import sun.misc.Cleaner;
+import org.voltcore.utils.VoltUnsafe;
 import sun.nio.ch.DirectBuffer;
 
 #ifndef NO_MEMCHECK
@@ -45,6 +45,7 @@ public final class DBBPool {
 
     private static final VoltLogger TRACE = new VoltLogger("DBBPOOL");
     private static final VoltLogger HOST = new VoltLogger("DBBPOOL");
+
 #ifndef NO_MEMCHECK
 
     static {
@@ -237,6 +238,26 @@ public final class DBBPool {
         }
     }
 
+    /**
+     * Wrapper for native allocated direct memory
+     *
+     */
+    public static final class NDBBWrapperContainer extends BBContainer {
+        private NDBBWrapperContainer(ByteBuffer b) {
+            super( b );
+        }
+
+        @Override
+        public final void discard() {
+            final ByteBuffer buf = checkDoubleFree();
+            if (buf == null || !buf.isDirect()) {
+               return;
+            }
+            final DirectBuffer dbuf = (DirectBuffer) buf;
+            deleteCharArrayMemory(dbuf.address());
+        }
+    }
+
     public static class MBBContainer extends BBContainer {
         private MBBContainer(MappedByteBuffer buf) {
             super(buf);
@@ -544,7 +565,7 @@ public final class DBBPool {
     private static native void nativeDeleteCharArrayMemory(long pointer);
 
     public static BBContainer allocateUnsafeByteBuffer(long size) {
-        final BBContainer retcont = DBBPool.wrapBB(nativeAllocateUnsafeByteBuffer(size));
+        final BBContainer retcont = new NDBBWrapperContainer(nativeAllocateUnsafeByteBuffer(size));
 #ifdef MEMCHECK_FULL
         final long pointer = retcont.address();
         m_allocatedStuff.put(pointer, new Throwable("Thread \"" + Thread.currentThread().getName() + "\" allocated " + Long.toHexString(pointer) + " here at " + System.currentTimeMillis()));
@@ -566,11 +587,12 @@ public final class DBBPool {
      * called deleteCharArrayMemory on the address
      */
     private static void cleanByteBuffer(ByteBuffer buf) {
-        if (buf == null) return;
-        if (!buf.isDirect()) return;
+        if (buf == null || !buf.isDirect()) {
+            return;
+        }
         final DirectBuffer dbuf = (DirectBuffer) buf;
-        final Cleaner cleaner = dbuf.cleaner();
-        if (cleaner != null) cleaner.clean();
-        else deleteCharArrayMemory(dbuf.address());
+        if (!VoltUnsafe.cleanDirectBuffer(buf)) {
+            deleteCharArrayMemory(dbuf.address());
+        }
     }
 }

src/frontend/org/voltcore/utils/DirectBufferCleaner.java

@@ -0,0 +1,31 @@
+/* This file is part of VoltDB.
+ * Copyright (C) 2008-2019 VoltDB Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with VoltDB.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package org.voltcore.utils;
+import java.nio.ByteBuffer;
+
+/**
+ * Cleaner interface for {@code java.nio.ByteBuffer}.
+ */
+public interface DirectBufferCleaner {
+    /**
+     * frees the memory backing the ByteBuffer if possible
+     *
+     * @param buf direct buffer.
+     */
+    public boolean clean(ByteBuffer buf);
+}