Add gRPC server as transport-grpc plugin (opensearch-project#16534)

Introduce auxiliary transport to NetworkPlugin and add gRPC plugin. Auxiliary transports are optional lifecycle components provided by network plugins which run in parallel to the http server/native transport. They are distinct from the existing NetworkPlugin interfaces of 'getTransports' and 'getHttpTransports' as auxiliary transports are optional. Each AuxTransport implements it's own 'aux.transport.type' and 'aux.transport.<type>.ports' setting. Since Security.java initializes previous to Node.java during bootstrap socket binding permissions are granted based on 'aux.transport.<type>.ports' for each enabled 'aux.transport.type', falling back to a default if no ports are specified. Signed-off-by: Finn Carroll <[email protected]> (cherry picked from commit 7a0e8fb)
finnegancarroll · Jan 6, 2025 · 0a7bade · 0a7bade
1 parent cbcc150
commit 0a7bade
Show file tree

Hide file tree

Showing 42 changed files with 2,149 additions and 188 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -0,0 +1,133 @@
+[versions]
+opensearch        = "2.19.0"
+lucene            = "9.12.1"
+
+bundled_jdk_vendor = "adoptium"
+bundled_jdk = "21.0.5+11"
+
+# optional dependencies
+spatial4j         = "0.7"
+jts               = "1.15.0"
+jackson           = "2.18.2"
+jackson_databind  = "2.18.2"
+snakeyaml         = "2.1"
+icu4j             = "75.1"
+supercsv          = "2.4.0"
+log4j             = "2.21.0"
+slf4j             = "1.7.36"
+asm               = "9.7"
+jettison          = "1.5.4"
+woodstox          = "6.4.0"
+kotlin            = "1.7.10"
+antlr4            = "4.13.1"
+guava             = "33.2.1-jre"
+protobuf          = "3.25.5"
+jakarta_annotation = "1.3.5"
+google_http_client = "1.44.1"
+google_auth       = "1.29.0"
+tdigest           = "3.2"
+hdrhistogram      = "2.2.2"
+grpc              = "1.68.0"
+
+# when updating the JNA version, also update the version in buildSrc/build.gradle
+jna               = "5.13.0"
+
+netty             = "4.1.115.Final"
+joda              = "2.12.7"
+roaringbitmap     = "1.3.0"
+
+# project reactor
+reactor_netty     = "1.1.23"
+reactor           = "3.5.20"
+
+# client dependencies
+httpclient        = "4.5.14"
+httpcore          = "4.4.16"
+httpasyncclient   = "4.1.5"
+commonslogging    = "1.2"
+commonscodec      = "1.16.1"
+commonslang       = "3.14.0"
+commonscompress   = "1.26.1"
+commonsio         = "2.16.0"
+# plugin dependencies
+aws               = "2.20.86"
+reactivestreams   = "1.0.4"
+
+# when updating this version, you need to ensure compatibility with:
+#  - plugins/ingest-attachment (transitive dependency, check the upstream POM)
+#  - distribution/tools/plugin-cli
+bouncycastle="1.78"
+# test dependencies
+randomizedrunner  = "2.7.1"
+junit             = "4.13.2"
+hamcrest          = "2.1"
+mockito           = "5.14.2"
+objenesis         = "3.3"
+bytebuddy         = "1.15.10"
+
+# benchmark dependencies
+jmh               = "1.35"
+
+# compression
+zstd              = "1.5.5-5"
+
+jzlib             = "1.1.3"
+
+resteasy          = "6.2.4.Final"
+
+# opentelemetry dependencies
+opentelemetry         = "1.41.0"
+opentelemetrysemconv  = "1.27.0-alpha"
+
+# arrow dependencies
+arrow                 = "17.0.0"
+flatbuffers           = "2.0.0"
+
+[libraries]
+hdrhistogram = { group = "org.hdrhistogram", name = "HdrHistogram", version.ref = "hdrhistogram" }
+jakartaannotation = { group = "jakarta.annotation", name = "jakarta.annotation-api", version.ref = "jakarta_annotation" }
+jodatime = { group = "joda-time", name = "joda-time", version.ref = "joda" }
+jna = { group = "net.java.dev.jna", name = "jna", version.ref = "jna" }
+jtscore = { group = "org.locationtech.jts", name = "jts-core", version.ref = "jts" }
+jzlib = { group = "com.jcraft", name = "jzlib", version.ref = "jzlib" }
+log4japi = { group = "org.apache.logging.log4j", name = "log4j-api", version.ref = "log4j" }
+log4jjul = { group = "org.apache.logging.log4j", name = "log4j-jul", version.ref = "log4j" }
+log4jcore = { group = "org.apache.logging.log4j", name = "log4j-core", version.ref = "log4j" }
+lucene-core = { group = "org.apache.lucene", name = "lucene-core", version.ref = "lucene" }
+lucene-analysis-common = { group = "org.apache.lucene", name = "lucene-analysis-common", version.ref = "lucene" }
+lucene-backward-codecs = { group = "org.apache.lucene", name = "lucene-backward-codecs", version.ref = "lucene" }
+lucene-grouping = { group = "org.apache.lucene", name = "lucene-grouping", version.ref = "lucene" }
+lucene-highlighter = { group = "org.apache.lucene", name = "lucene-highlighter", version.ref = "lucene" }
+lucene-join = { group = "org.apache.lucene", name = "lucene-join", version.ref = "lucene" }
+lucene-memory = { group = "org.apache.lucene", name = "lucene-memory", version.ref = "lucene" }
+lucene-misc = { group = "org.apache.lucene", name = "lucene-misc", version.ref = "lucene" }
+lucene-queries = { group = "org.apache.lucene", name = "lucene-queries", version.ref = "lucene" }
+lucene-queryparser = { group = "org.apache.lucene", name = "lucene-queryparser", version.ref = "lucene" }
+lucene-sandbox = { group = "org.apache.lucene", name = "lucene-sandbox", version.ref = "lucene" }
+lucene-spatial-extras = { group = "org.apache.lucene", name = "lucene-spatial-extras", version.ref = "lucene" }
+lucene-spatial3d = { group = "org.apache.lucene", name = "lucene-spatial3d", version.ref = "lucene" }
+lucene-suggest = { group = "org.apache.lucene", name = "lucene-suggest", version.ref = "lucene" }
+protobuf = { group = "com.google.protobuf", name = "protobuf-java", version.ref = "protobuf" }
+reactivestreams = { group = "io.projectreactor", name = "reactor-core", version.ref = "reactor" }
+reactorcore = { group = "org.reactivestreams", name = "reactive-streams", version.ref = "reactivestreams" }
+roaringbitmap = { group = "org.roaringbitmap", name = "RoaringBitmap", version.ref = "roaringbitmap" }
+spatial4j = { group = "org.locationtech.spatial4j", name = "spatial4j", version.ref = "spatial4j" }
+tdigest = { group = "com.tdunning", name = "t-digest", version.ref = "tdigest" }
+
+[bundles]
+lucene = [
+    "lucene-core",
+    "lucene-analysis-common",
+    "lucene-backward-codecs",
+    "lucene-grouping",
+    "lucene-highlighter",
+    "lucene-join",
+    "lucene-memory",
+    "lucene-misc",
+    "lucene-queries",
+    "lucene-queryparser",
+    "lucene-sandbox",
+    "lucene-spatial-extras",
+    "lucene-spatial3d",
+    "lucene-suggest"
+]
diff --git a/plugins/discovery-gce/licenses/grpc-api-1.68.2.jar.sha1 b/plugins/discovery-gce/licenses/grpc-api-1.68.2.jar.sha1
@@ -0,0 +1 @@
+a257a5dd25dda1c97a99b56d5b9c1e56c12ae554
diff --git a/plugins/repository-gcs/licenses/grpc-api-1.68.2.jar.sha1 b/plugins/repository-gcs/licenses/grpc-api-1.68.2.jar.sha1
@@ -0,0 +1 @@
+a257a5dd25dda1c97a99b56d5b9c1e56c12ae554
diff --git a/plugins/transport-grpc/build.gradle b/plugins/transport-grpc/build.gradle
@@ -0,0 +1,168 @@
+import org.gradle.api.attributes.java.TargetJvmEnvironment
+
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+opensearchplugin {
+  description 'gRPC based transport implementation'
+  classname 'org.opensearch.transport.grpc.GrpcPlugin'
+}
+
+dependencies {
+  compileOnly "com.google.code.findbugs:jsr305:3.0.2"
+  runtimeOnly "com.google.guava:guava:${versions.guava}"
+  implementation "com.google.errorprone:error_prone_annotations:2.24.1"
+  implementation "com.google.guava:failureaccess:1.0.1"
+  implementation "io.grpc:grpc-api:${versions.grpc}"
+  implementation "io.grpc:grpc-core:${versions.grpc}"
+  implementation "io.grpc:grpc-netty-shaded:${versions.grpc}"
+  implementation "io.grpc:grpc-protobuf-lite:${versions.grpc}"
+  implementation "io.grpc:grpc-protobuf:${versions.grpc}"
+  implementation "io.grpc:grpc-services:${versions.grpc}"
+  implementation "io.grpc:grpc-stub:${versions.grpc}"
+  implementation "io.grpc:grpc-util:${versions.grpc}"
+  implementation "io.perfmark:perfmark-api:0.26.0"
+}
+
+tasks.named("dependencyLicenses").configure {
+  mapping from: /grpc-.*/, to: 'grpc'
+}
+
+thirdPartyAudit {
+  ignoreMissingClasses(
+    'com.aayushatharva.brotli4j.Brotli4jLoader',
+    'com.aayushatharva.brotli4j.decoder.DecoderJNI$Status',
+    'com.aayushatharva.brotli4j.decoder.DecoderJNI$Wrapper',
+    'com.aayushatharva.brotli4j.encoder.BrotliEncoderChannel',
+    'com.aayushatharva.brotli4j.encoder.Encoder$Mode',
+    'com.aayushatharva.brotli4j.encoder.Encoder$Parameters',
+    // classes are missing
+
+    // from io.netty.logging.CommonsLoggerFactory (netty)
+    'org.apache.commons.logging.Log',
+    'org.apache.commons.logging.LogFactory',
+
+    // from Log4j (deliberate, Netty will fallback to Log4j 2)
+    'org.apache.log4j.Level',
+    'org.apache.log4j.Logger',
+
+    // from io.netty.handler.ssl.util.BouncyCastleSelfSignedCertGenerator (netty)
+    'org.bouncycastle.cert.X509v3CertificateBuilder',
+    'org.bouncycastle.cert.jcajce.JcaX509CertificateConverter',
+    'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
+    'org.bouncycastle.openssl.PEMEncryptedKeyPair',
+    'org.bouncycastle.openssl.PEMParser',
+    'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
+    'org.bouncycastle.openssl.jcajce.JceOpenSSLPKCS8DecryptorProviderBuilder',
+    'org.bouncycastle.openssl.jcajce.JcePEMDecryptorProviderBuilder',
+    'org.bouncycastle.pkcs.PKCS8EncryptedPrivateKeyInfo',
+
+    // from io.netty.handler.ssl.JettyNpnSslEngine (netty)
+    'org.eclipse.jetty.npn.NextProtoNego$ClientProvider',
+    'org.eclipse.jetty.npn.NextProtoNego$ServerProvider',
+    'org.eclipse.jetty.npn.NextProtoNego',
+
+    // from io.netty.handler.codec.marshalling.ChannelBufferByteInput (netty)
+    'org.jboss.marshalling.ByteInput',
+
+    // from io.netty.handler.codec.marshalling.ChannelBufferByteOutput (netty)
+    'org.jboss.marshalling.ByteOutput',
+
+    // from io.netty.handler.codec.marshalling.CompatibleMarshallingEncoder (netty)
+    'org.jboss.marshalling.Marshaller',
+
+    // from io.netty.handler.codec.marshalling.ContextBoundUnmarshallerProvider (netty)
+    'org.jboss.marshalling.MarshallerFactory',
+    'org.jboss.marshalling.MarshallingConfiguration',
+    'org.jboss.marshalling.Unmarshaller',
+
+    // from io.netty.util.internal.logging.InternalLoggerFactory (netty) - it's optional
+    'org.slf4j.helpers.FormattingTuple',
+    'org.slf4j.helpers.MessageFormatter',
+    'org.slf4j.Logger',
+    'org.slf4j.LoggerFactory',
+    'org.slf4j.spi.LocationAwareLogger',
+
+    'com.google.gson.stream.JsonReader',
+    'com.google.gson.stream.JsonToken',
+    'com.google.protobuf.util.Durations',
+    'com.google.protobuf.util.Timestamps',
+    'com.google.protobuf.nano.CodedOutputByteBufferNano',
+    'com.google.protobuf.nano.MessageNano',
+    'com.google.rpc.Status',
+    'com.google.rpc.Status$Builder',
+    'com.ning.compress.BufferRecycler',
+    'com.ning.compress.lzf.ChunkDecoder',
+    'com.ning.compress.lzf.ChunkEncoder',
+    'com.ning.compress.lzf.LZFChunk',
+    'com.ning.compress.lzf.LZFEncoder',
+    'com.ning.compress.lzf.util.ChunkDecoderFactory',
+    'com.ning.compress.lzf.util.ChunkEncoderFactory',
+    'lzma.sdk.lzma.Encoder',
+    'net.jpountz.lz4.LZ4Compressor',
+    'net.jpountz.lz4.LZ4Factory',
+    'net.jpountz.lz4.LZ4FastDecompressor',
+    'net.jpountz.xxhash.XXHash32',
+    'net.jpountz.xxhash.XXHashFactory',
+    'org.eclipse.jetty.alpn.ALPN$ClientProvider',
+    'org.eclipse.jetty.alpn.ALPN$ServerProvider',
+    'org.eclipse.jetty.alpn.ALPN',
+
+    'org.conscrypt.AllocatedBuffer',
+    'org.conscrypt.BufferAllocator',
+    'org.conscrypt.Conscrypt',
+    'org.conscrypt.HandshakeListener',
+
+    'reactor.blockhound.BlockHound$Builder',
+    'reactor.blockhound.integration.BlockHoundIntegration'
+  )
+
+  ignoreViolations(
+    // uses internal java api: sun.misc.Unsafe
+    'com.google.common.cache.Striped64',
+    'com.google.common.cache.Striped64$1',
+    'com.google.common.cache.Striped64$Cell',
+    'com.google.common.hash.Striped64',
+    'com.google.common.hash.Striped64$1',
+    'com.google.common.hash.Striped64$Cell',
+    'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
+    'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
+    'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
+    'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
+    'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
+    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
+    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
+
+    'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator',
+    'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$1',
+    'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$2',
+    'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$3',
+    'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$4',
+    'io.grpc.netty.shaded.io.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator$5',
+    'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0',
+    'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$1',
+    'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$2',
+    'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$3',
+    'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$4',
+    'io.grpc.netty.shaded.io.netty.util.internal.PlatformDependent0$6',
+    'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef',
+    'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef',
+    'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields',
+    'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueConsumerFields',
+    'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueProducerFields',
+    'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.LinkedQueueNode',
+    'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueConsumerIndexField',
+    'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpmcArrayQueueProducerIndexField',
+    'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueConsumerIndexField',
+    'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerIndexField',
+    'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.queues.MpscArrayQueueProducerLimitField',
+    'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.util.UnsafeAccess',
+    'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.util.UnsafeLongArrayAccess',
+    'io.grpc.netty.shaded.io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess'
+  )
+}
diff --git a/plugins/transport-grpc/licenses/error_prone_annotations-2.24.1.jar.sha1 b/plugins/transport-grpc/licenses/error_prone_annotations-2.24.1.jar.sha1
@@ -0,0 +1 @@
+32b299e45105aa9b0df8279c74dc1edfcf313ff0