Introduce kafka-oidc feature and enable it by default for restate-server builds

To use the Kafka ingress with OIDC authentication, we need to enable the gssapi-vendored
and curl-static features on our rdkafka dependency. To make things properly work with cross
compilation, we need to patch rust-sasl and rdkafka and bump librdkafka to v2.12.1. The latter
allows us to compile with curl-static which saves us the hassle to dynamically link against
libraries from a different target architecture.

Update transitive dependencies to work with cross compilation

Bump rdkafka to use librdkafka 2.12.1

Author: tillrohrmann

.github/workflows/docker.yml

@@ -66,6 +66,11 @@ on:
         required: false
         default: false
         type: boolean
+      features:
+        description: "features to enable in the build"
+        required: false
+        default: ""
+        type: string
       pushToDockerHub:
         description: "push image to DockerHub"
         required: false

Cargo.lock

@@ -260,6 +260,12 @@ zstd = { version = "0.13" }
 [patch.crates-io.restate-workspace-hack]
 path = "workspace-hack"
 
+[patch.crates-io]
+# Cross compilation fixes: Use autotools to detect cross compilation tool chain and disable tests when cross compiling.
+# The fixes are based on rust-sasl v0.1.22.
+# Todo: Upstream changes and get rid of patch
+sasl2-sys = { git = "https://github.com/restatedev/rust-sasl", rev = "6298ec4e1da8cd808be9956a8e32f42a7f10b245" }
+
 [profile.release]
 opt-level = 3
 lto = "thin"

Cargo.toml

@@ -10,6 +10,10 @@ publish = false
 [features]
 default = []
 options_schema = ["dep:schemars"]
+# support for OIDC based authentication via the Kafka consumer, building with rdkafka/curl-static does not work as it
+# fails with "SSL certificate OpenSSL verify result: unable to get local issuer certificate (20) (-1)" when trying to
+# obtain the OIDC token.
+oidc = ["rdkafka/curl-static", "rdkafka/gssapi-vendored"]
 
 [dependencies]
 restate-workspace-hack = { workspace = true }
@@ -32,7 +36,10 @@ opentelemetry = { workspace = true }
 opentelemetry_sdk = { workspace = true }
 parking_lot = { workspace = true }
 # 0.38 was not released yet at the time of writing, so when this happens, remove the pin.
-rdkafka = { version = "0.38", git = "https://github.com/fede1024/rust-rdkafka.git", rev = "47d86d71e340896491b65521594bbf081186201e", features = ["libz-static", "cmake-build", "ssl-vendored"] }
+# The current revision is based on 47d86d71e340896491b65521594bbf081186201e, which was the previously pinned revision.
+# What was added on top of it, was enabling the curl feature of librdkafka if the feature curl-static is enabled and
+# upgrading librdkafka to v2.12.1 which allow us to configure the ca certificate of the statically linked curl library.
+rdkafka = { version = "0.38", git = "https://github.com/restatedev/rust-rdkafka.git", rev = "26064222228405f04963d8c38a6e771a80d23d2a", features = ["libz-static", "cmake-build", "ssl-vendored"] }
 schemars = { workspace = true, optional = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["sync", "rt"] }

crates/ingress-kafka/Cargo.toml

@@ -120,6 +120,8 @@ impl Service {
         task_orchestrator: &mut TaskOrchestrator,
     ) -> anyhow::Result<()> {
         let mut client_config = rdkafka::ClientConfig::new();
+        // enabling probing for the ca certificates if the user does not specify anything else
+        client_config.set("https.ca.location", "probe");
 
         let Source::Kafka { cluster, topic, .. } = subscription.source();
 

crates/ingress-kafka/src/subscription_controller.rs

@@ -22,6 +22,7 @@ options_schema = [
     "restate-admin/options_schema",
     "restate-worker/options_schema"
 ]
+kafka-oidc = ["restate-worker/kafka-oidc"]
 
 [dependencies]
 restate-workspace-hack = { workspace = true }

crates/node/Cargo.toml

@@ -17,6 +17,7 @@ options_schema = [
   "restate-storage-query-datafusion/options_schema",
   "restate-timer/options_schema",
 ]
+kafka-oidc = ["restate-ingress-kafka/oidc"]
 
 [dependencies]
 restate-workspace-hack = { workspace = true }

crates/worker/Cargo.toml

@@ -174,7 +174,6 @@ allow-registry = ["https://github.com/rust-lang/crates.io-index"]
 allow-git = [
     "https://github.com/apache/arrow-rs.git",
     "https://github.com/tikv/raft-rs.git",
-    "https://github.com/fede1024/rust-rdkafka.git",
 ]
 
 [sources.allow-org]

deny.toml

@@ -10,11 +10,11 @@
 
 ARG UPLOAD_DEBUGINFO=false
 
-FROM --platform=$BUILDPLATFORM ghcr.io/restatedev/dev-tools:1.14.5 AS planner
+FROM --platform=$BUILDPLATFORM ghcr.io/restatedev/dev-tools:1.14.7 AS planner
 COPY . .
 RUN just chef-prepare
 
-FROM --platform=$BUILDPLATFORM ghcr.io/restatedev/dev-tools:1.14.5 AS base
+FROM --platform=$BUILDPLATFORM ghcr.io/restatedev/dev-tools:1.14.7 AS base
 COPY --from=planner /restate/recipe.json recipe.json
 COPY justfile justfile
 
@@ -32,6 +32,16 @@ ARG TARGETARCH
 ENV RUSTC_WRAPPER=/usr/bin/sccache
 ENV SCCACHE_DIR=/var/cache/sccache
 
+# todo only enable those env variables when cross compiling
+# Set krb5 cross-compilation env variables (because we cannot run cross compiled tests)
+ENV krb5_cv_attr_constructor_destructor=yes
+ENV ac_cv_func_regcomp=yes
+ENV ac_cv_printf_positional=yes
+
+# todo only enable this env variable when cross compiling
+# Set sasl2-sys cross-compilation env variables (because we cannot run cross compiled tests)
+ENV ac_cv_gssapi_supports_spnego=yes
+
 # Overrides the behaviour of the release profile re including debug symbols, which in our repo is not to include them.
 # Should be set to 'false' or 'true'. See https://doc.rust-lang.org/cargo/reference/environment-variables.html
 ARG CARGO_PROFILE_RELEASE_DEBUG=false

docker/Dockerfile

@@ -8,11 +8,11 @@
 # the Business Source License, use of this software will be governed
 # by the Apache License, Version 2.0.
 
-FROM --platform=$BUILDPLATFORM ghcr.io/restatedev/dev-tools:1.14.5 AS planner
+FROM --platform=$BUILDPLATFORM ghcr.io/restatedev/dev-tools:1.14.7 AS planner
 COPY . .
 RUN just chef-prepare
 
-FROM --platform=$BUILDPLATFORM ghcr.io/restatedev/dev-tools:1.14.5 AS base
+FROM --platform=$BUILDPLATFORM ghcr.io/restatedev/dev-tools:1.14.7 AS base
 COPY --from=planner /restate/recipe.json recipe.json
 COPY justfile justfile
 
@@ -30,6 +30,16 @@ ARG TARGETARCH
 ENV RUSTC_WRAPPER=/usr/bin/sccache
 ENV SCCACHE_DIR=/var/cache/sccache
 
+# todo only enable those env variables when cross compiling
+# Set krb5 cross-compilation env variables (because we cannot run cross compiled tests)
+ENV krb5_cv_attr_constructor_destructor=yes
+ENV ac_cv_func_regcomp=yes
+ENV ac_cv_printf_positional=yes
+
+# todo only enable this env variable when cross compiling
+# Set sasl2-sys cross-compilation env variables (because we cannot run cross compiled tests)
+ENV ac_cv_gssapi_supports_spnego=yes
+
 # Avoids feature unification by building the three binaries individually
 ARG BUILD_INDIVIDUALLY=false
 ARG RESTATE_FEATURES=''