diff --git a/docs/generation/site-local.yml b/docs/generation/site-local.yml index 57012152c6..de852e0a4f 100644 --- a/docs/generation/site-local.yml +++ b/docs/generation/site-local.yml @@ -57,6 +57,9 @@ content: - url: ../../ branches: HEAD start_path: servicetalk-traffic-resilience-http/docs + - url: ../../ + branches: HEAD + start_path: servicetalk-opentelemetry-http/docs asciidoc: attributes: experimental: '' diff --git a/docs/generation/site-remote.yml b/docs/generation/site-remote.yml index 1d1ebfeb47..c03c969af6 100644 --- a/docs/generation/site-remote.yml +++ b/docs/generation/site-remote.yml @@ -72,6 +72,10 @@ content: branches: main tags: [0.42.60] start_path: servicetalk-traffic-resilience-http/docs + - url: https://github.com/apple/servicetalk.git + branches: main + tags: [] + start_path: servicetalk-opentelemetry-http/docs asciidoc: attributes: experimental: '' diff --git a/docs/modules/ROOT/pages/_partials/nav-versioned.adoc b/docs/modules/ROOT/pages/_partials/nav-versioned.adoc index 55cf991bf7..e07972bc42 100644 --- a/docs/modules/ROOT/pages/_partials/nav-versioned.adoc +++ b/docs/modules/ROOT/pages/_partials/nav-versioned.adoc @@ -47,5 +47,11 @@ include::{page-version}@servicetalk-loadbalancer:ROOT:partial$nav-versioned.adoc include::{page-version}@servicetalk-traffic-resilience-http:ROOT:partial$nav-versioned.adoc[] -- + +* Observability ++ +-- +include::{page-version}@servicetalk-opentelemetry-http:ROOT:partial$nav-versioned.adoc[] +-- ++ * xref:{page-version}@servicetalk::javadoc/index.adoc[Javadoc] * xref:{page-version}@servicetalk::contributing.adoc[Contributing] diff --git a/servicetalk-examples/docs/modules/ROOT/pages/_partials/nav-versioned.adoc b/servicetalk-examples/docs/modules/ROOT/pages/_partials/nav-versioned.adoc index f7604e5fe5..77c164df13 100644 --- a/servicetalk-examples/docs/modules/ROOT/pages/_partials/nav-versioned.adoc +++ b/servicetalk-examples/docs/modules/ROOT/pages/_partials/nav-versioned.adoc @@ -14,6 +14,7 @@ ** xref:{page-version}@servicetalk-examples::http/index.adoc#Observer[Observer] ** xref:{page-version}@servicetalk-examples::http/index.adoc#LoadBalancer[LoadBalancer] ** xref:{page-version}@servicetalk-examples::http/index.adoc#OpenTracing[OpenTracing] +** xref:{page-version}@servicetalk-examples::http/index.adoc#OpenTelemetryTracing[OpenTelemetry Tracing] ** xref:{page-version}@servicetalk-examples::http/index.adoc#Redirects[Redirects] ** xref:{page-version}@servicetalk-examples::http/index.adoc#Retries[Retries] ** xref:{page-version}@servicetalk-examples::http/index.adoc#uds[Unix Domain Sockets] diff --git a/servicetalk-examples/docs/modules/ROOT/pages/http/index.adoc b/servicetalk-examples/docs/modules/ROOT/pages/http/index.adoc index 1285ab5566..ab0c9da9df 100644 --- a/servicetalk-examples/docs/modules/ROOT/pages/http/index.adoc +++ b/servicetalk-examples/docs/modules/ROOT/pages/http/index.adoc @@ -264,6 +264,21 @@ Using the following classes: - link:{source-root}/servicetalk-examples/http/opentracing/src/main/java/io/servicetalk/examples/http/opentracing/ZipkinServerSimulator.java[ZipkinServerSimulator] - A server that simulates/mocks a Zipkin server, and logs requests to the console. - link:{source-root}/servicetalk-examples/http/opentracing/src/main/java/io/servicetalk/examples/http/opentracing/BraveTracingServer.java[BraveTracingServer] - A server that uses link:https://github.com/openzipkin-contrib/brave-opentracing[Brave OpenTracing] implementation. +[#OpenTelemetryTracing] +== OpenTelemetry Tracing + +This example demonstrates the following: + +- automatically generate and propagate distributed tracing metadata using OpenTelemetry +- configure OpenTelemetry SDK with logging span exporter for local development +- use of ServiceTalk HTTP filters for OpenTelemetry tracing +- proper xref:{page-version}@servicetalk-concurrent-api::async-context.adoc[async context propagation] for span correlation + +Using the following classes: + +- link:{source-root}/servicetalk-examples/http/opentelemetry-tracing/src/main/java/io/servicetalk/examples/http/opentelemetry/tracing/OpenTelemetryTracingServer.java[OpenTelemetryTracingServer] - A server that demonstrates OpenTelemetry tracing configuration and automatic span generation for HTTP requests. +- link:{source-root}/servicetalk-examples/http/opentelemetry-tracing/src/main/java/io/servicetalk/examples/http/opentelemetry/tracing/OpenTelemetryTracingClient.java[OpenTelemetryTracingClient] - A client that demonstrates OpenTelemetry tracing configuration and automatic span generation for HTTP requests with context propagation. + [#Redirects] == Redirects diff --git a/servicetalk-examples/http/opentelemetry-tracing/build.gradle b/servicetalk-examples/http/opentelemetry-tracing/build.gradle new file mode 100644 index 0000000000..29c9ad0843 --- /dev/null +++ b/servicetalk-examples/http/opentelemetry-tracing/build.gradle @@ -0,0 +1,38 @@ +/* + * Copyright © 2025 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: "java" +apply from: "../../gradle/idea.gradle" + +dependencies { + implementation platform("io.opentelemetry:opentelemetry-bom:$opentelemetryVersion") + + implementation project(":servicetalk-annotations") + implementation project(":servicetalk-http-netty") + implementation project(":servicetalk-http-utils") // For HttpRequestAutoDrainingServiceFilter + implementation project(":servicetalk-opentelemetry-http") // OpenTelemetry client/server filters + + runtimeOnly project(":servicetalk-opentelemetry-asynccontext") // OpenTelemetry async context propagation + + // OpenTelemetry Java SDK + implementation "io.opentelemetry:opentelemetry-api" + implementation "io.opentelemetry:opentelemetry-sdk" + implementation "io.opentelemetry:opentelemetry-exporter-logging" + implementation "io.opentelemetry:opentelemetry-exporter-otlp" + + implementation "org.slf4j:slf4j-api:$slf4jVersion" + runtimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:$log4jVersion" +} diff --git a/servicetalk-examples/http/opentelemetry-tracing/gradle.lockfile b/servicetalk-examples/http/opentelemetry-tracing/gradle.lockfile new file mode 100644 index 0000000000..fdfaec8291 --- /dev/null +++ b/servicetalk-examples/http/opentelemetry-tracing/gradle.lockfile @@ -0,0 +1,56 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +com.google.code.findbugs:jsr305:3.0.2=compileClasspath,runtimeClasspath +com.squareup.okhttp3:okhttp:4.12.0=runtimeClasspath +com.squareup.okio:okio-jvm:3.6.0=runtimeClasspath +com.squareup.okio:okio:3.6.0=runtimeClasspath +io.netty:netty-bom:4.1.127.Final=runtimeClasspath +io.netty:netty-buffer:4.1.127.Final=runtimeClasspath +io.netty:netty-codec-dns:4.1.127.Final=runtimeClasspath +io.netty:netty-codec-http2:4.1.127.Final=runtimeClasspath +io.netty:netty-codec-http:4.1.127.Final=runtimeClasspath +io.netty:netty-codec:4.1.127.Final=runtimeClasspath +io.netty:netty-common:4.1.127.Final=runtimeClasspath +io.netty:netty-handler:4.1.127.Final=runtimeClasspath +io.netty:netty-resolver-dns-classes-macos:4.1.127.Final=runtimeClasspath +io.netty:netty-resolver-dns-native-macos:4.1.127.Final=runtimeClasspath +io.netty:netty-resolver-dns:4.1.127.Final=runtimeClasspath +io.netty:netty-resolver:4.1.127.Final=runtimeClasspath +io.netty:netty-tcnative-boringssl-static:2.0.73.Final=runtimeClasspath +io.netty:netty-tcnative-classes:2.0.73.Final=runtimeClasspath +io.netty:netty-transport-classes-epoll:4.1.127.Final=runtimeClasspath +io.netty:netty-transport-classes-kqueue:4.1.127.Final=runtimeClasspath +io.netty:netty-transport-native-epoll:4.1.127.Final=runtimeClasspath +io.netty:netty-transport-native-kqueue:4.1.127.Final=runtimeClasspath +io.netty:netty-transport-native-unix-common:4.1.127.Final=runtimeClasspath +io.netty:netty-transport:4.1.127.Final=runtimeClasspath +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:2.14.0=runtimeClasspath +io.opentelemetry.instrumentation:opentelemetry-instrumentation-bom:2.14.0=runtimeClasspath +io.opentelemetry.semconv:opentelemetry-semconv:1.30.0=runtimeClasspath +io.opentelemetry:opentelemetry-api-incubator:1.48.0-alpha=runtimeClasspath +io.opentelemetry:opentelemetry-api:1.48.0=compileClasspath,runtimeClasspath +io.opentelemetry:opentelemetry-bom:1.48.0=compileClasspath,runtimeClasspath +io.opentelemetry:opentelemetry-context:1.48.0=compileClasspath,runtimeClasspath +io.opentelemetry:opentelemetry-exporter-common:1.48.0=runtimeClasspath +io.opentelemetry:opentelemetry-exporter-logging:1.48.0=compileClasspath,runtimeClasspath +io.opentelemetry:opentelemetry-exporter-otlp-common:1.48.0=runtimeClasspath +io.opentelemetry:opentelemetry-exporter-otlp:1.48.0=compileClasspath,runtimeClasspath +io.opentelemetry:opentelemetry-exporter-sender-okhttp:1.48.0=runtimeClasspath +io.opentelemetry:opentelemetry-sdk-common:1.48.0=compileClasspath,runtimeClasspath +io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.48.0=runtimeClasspath +io.opentelemetry:opentelemetry-sdk-logs:1.48.0=compileClasspath,runtimeClasspath +io.opentelemetry:opentelemetry-sdk-metrics:1.48.0=compileClasspath,runtimeClasspath +io.opentelemetry:opentelemetry-sdk-trace:1.48.0=compileClasspath,runtimeClasspath +io.opentelemetry:opentelemetry-sdk:1.48.0=compileClasspath,runtimeClasspath +org.apache.logging.log4j:log4j-api:2.23.1=runtimeClasspath +org.apache.logging.log4j:log4j-core:2.23.1=runtimeClasspath +org.apache.logging.log4j:log4j-slf4j-impl:2.23.1=runtimeClasspath +org.jctools:jctools-core:4.0.5=runtimeClasspath +org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10=runtimeClasspath +org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10=runtimeClasspath +org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10=runtimeClasspath +org.jetbrains.kotlin:kotlin-stdlib:1.9.10=runtimeClasspath +org.jetbrains:annotations:13.0=runtimeClasspath +org.slf4j:slf4j-api:1.7.36=compileClasspath,runtimeClasspath +empty=annotationProcessor,jmhCompileClasspath,jmhRuntimeClasspath diff --git a/servicetalk-examples/http/opentelemetry-tracing/src/main/java/io/servicetalk/examples/http/opentelemetry/tracing/OpenTelemetryTracingClient.java b/servicetalk-examples/http/opentelemetry-tracing/src/main/java/io/servicetalk/examples/http/opentelemetry/tracing/OpenTelemetryTracingClient.java new file mode 100644 index 0000000000..16642025e7 --- /dev/null +++ b/servicetalk-examples/http/opentelemetry-tracing/src/main/java/io/servicetalk/examples/http/opentelemetry/tracing/OpenTelemetryTracingClient.java @@ -0,0 +1,82 @@ +/* + * Copyright © 2025 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.examples.http.opentelemetry.tracing; + +import io.servicetalk.http.api.BlockingHttpClient; +import io.servicetalk.http.api.HttpResponse; +import io.servicetalk.http.netty.HttpClients; +import io.servicetalk.opentelemetry.http.OpenTelemetryHttpRequesterFilter; + +import java.nio.charset.StandardCharsets; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.exporter.logging.LoggingSpanExporter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; + +/** + * A client that demonstrates OpenTelemetry distributed tracing with ServiceTalk. + * This example shows how to: + * + */ +public final class OpenTelemetryTracingClient { + public static void main(String[] args) throws Exception { + // Configure OpenTelemetry SDK + final String serviceName = "servicetalk-example-client"; + + final OpenTelemetry openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(SdkTracerProvider.builder() + .addSpanProcessor(BatchSpanProcessor.builder(LoggingSpanExporter.create()).build()) + .build()) + .build(); + + // Set the global OpenTelemetry instance + GlobalOpenTelemetry.set(openTelemetry); + + OpenTelemetryHttpRequesterFilter filter = new OpenTelemetryHttpRequesterFilter.Builder() + .componentName(serviceName) + .build(); + + try (BlockingHttpClient client = HttpClients.forSingleAddress("localhost", 8080) + // IMPORTANT: OpenTelemetry filter should be placed EARLY in the filter chain + // This ensures proper span creation and context propagation for downstream filters + .appendClientFilter(filter) + // Optional: adding the filter a second time as a connection filter will enable 'physical spans' + // where there will be a higher level 'logical' span and a sub-physical span for each + // request sent over the wire. + .appendConnectionFilter(filter) + .buildBlocking()) { + + System.out.println("Making first request..."); + HttpResponse response1 = client.request(client.get("/hello")); + System.out.println("Response 1: " + response1.toString((name, value) -> value)); + System.out.println("Body 1: " + response1.payloadBody().toString(StandardCharsets.UTF_8)); + + System.out.println("\nMaking second request..."); + HttpResponse response2 = client.request(client.get("/world")); + System.out.println("Response 2: " + response2.toString((name, value) -> value)); + System.out.println("Body 2: " + response2.payloadBody().toString(StandardCharsets.UTF_8)); + + } + } +} diff --git a/servicetalk-examples/http/opentelemetry-tracing/src/main/java/io/servicetalk/examples/http/opentelemetry/tracing/OpenTelemetryTracingServer.java b/servicetalk-examples/http/opentelemetry-tracing/src/main/java/io/servicetalk/examples/http/opentelemetry/tracing/OpenTelemetryTracingServer.java new file mode 100644 index 0000000000..9256390787 --- /dev/null +++ b/servicetalk-examples/http/opentelemetry-tracing/src/main/java/io/servicetalk/examples/http/opentelemetry/tracing/OpenTelemetryTracingServer.java @@ -0,0 +1,86 @@ +/* + * Copyright © 2025 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.servicetalk.examples.http.opentelemetry.tracing; + +import io.servicetalk.http.api.HttpExceptionMapperServiceFilter; +import io.servicetalk.http.netty.HttpServers; +import io.servicetalk.http.utils.HttpRequestAutoDrainingServiceFilter; +import io.servicetalk.opentelemetry.http.OpenTelemetryHttpServiceFilter; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.exporter.logging.LoggingSpanExporter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A server that demonstrates OpenTelemetry distributed tracing with ServiceTalk. + * This example shows how to: + * + */ +public final class OpenTelemetryTracingServer { + private static final Logger LOGGER = LoggerFactory.getLogger(OpenTelemetryTracingServer.class); + + public static void main(String[] args) throws Exception { + // Configure OpenTelemetry SDK + final OpenTelemetry openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(SdkTracerProvider.builder() + .addSpanProcessor(BatchSpanProcessor.builder(LoggingSpanExporter.create()).build()) + .build()) + .build(); + + // Set the global OpenTelemetry instance + GlobalOpenTelemetry.set(openTelemetry); + + HttpServers.forPort(8080) + // CRITICAL: OpenTelemetry filter MUST be first for proper context propagation + // Use non-offloading filter to maintain async context across threads + .appendNonOffloadingServiceFilter(new OpenTelemetryHttpServiceFilter.Builder() + .build()) + + // IMPORTANT: Request draining MUST come after OpenTelemetry filter + // This ensures tracing information is captured for auto-drained requests (e.g., GET requests) + // If auto-draining occurs before OpenTelemetry filter processes the request, + // tracing information may be incomplete or incorrect. + .appendNonOffloadingServiceFilter(HttpRequestAutoDrainingServiceFilter.INSTANCE) + // Similarly, exception mapping should also come after the OTEL filter so that the OTEL span properly + // reflects what was sent back to the client. + .appendNonOffloadingServiceFilter(HttpExceptionMapperServiceFilter.INSTANCE) + + + // Other filters can be added after the critical ordering above + // Exception mapping, logging, etc. should come after OpenTelemetry and request draining + .listenBlockingAndAwait((ctx, request, responseFactory) -> { + LOGGER.info("Processing request: {} {}", request.method(), request.requestTarget()); + + // Simulate some processing work + Thread.sleep(50); + + return responseFactory.ok() + .addHeader("content-type", "text/plain") + .payloadBody(ctx.executionContext().bufferAllocator() + .fromAscii("Hello from OpenTelemetry server!")); + }).awaitShutdown(); + } +} diff --git a/servicetalk-examples/http/opentelemetry-tracing/src/main/java/io/servicetalk/examples/http/opentelemetry/tracing/package-info.java b/servicetalk-examples/http/opentelemetry-tracing/src/main/java/io/servicetalk/examples/http/opentelemetry/tracing/package-info.java new file mode 100644 index 0000000000..bba535168c --- /dev/null +++ b/servicetalk-examples/http/opentelemetry-tracing/src/main/java/io/servicetalk/examples/http/opentelemetry/tracing/package-info.java @@ -0,0 +1,22 @@ +/* + * Copyright © 2025 Apple Inc. and the ServiceTalk project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * OpenTelemetry examples showing distributed tracing with ServiceTalk. + */ +@ElementsAreNonnullByDefault +package io.servicetalk.examples.http.opentelemetry.tracing; + +import io.servicetalk.annotations.ElementsAreNonnullByDefault; diff --git a/servicetalk-examples/http/opentelemetry-tracing/src/main/resources/log4j2.xml b/servicetalk-examples/http/opentelemetry-tracing/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..b220fa88cb --- /dev/null +++ b/servicetalk-examples/http/opentelemetry-tracing/src/main/resources/log4j2.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + diff --git a/servicetalk-opentelemetry-http/docs/antora.yml b/servicetalk-opentelemetry-http/docs/antora.yml new file mode 100644 index 0000000000..31689b7352 --- /dev/null +++ b/servicetalk-opentelemetry-http/docs/antora.yml @@ -0,0 +1,21 @@ +# +# Copyright © 2025 Apple Inc. and the ServiceTalk project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: servicetalk-opentelemetry-http +title: OpenTelemetry +version: SNAPSHOT +nav: + - modules/ROOT/nav.adoc diff --git a/servicetalk-opentelemetry-http/docs/modules/ROOT/nav.adoc b/servicetalk-opentelemetry-http/docs/modules/ROOT/nav.adoc new file mode 100644 index 0000000000..5bef1e0060 --- /dev/null +++ b/servicetalk-opentelemetry-http/docs/modules/ROOT/nav.adoc @@ -0,0 +1,5 @@ +ifndef::page-version[] +:page-version: SNAPSHOT +endif::[] + +include::{page-version}@servicetalk-opentelemetry-http:ROOT:partial$nav-versioned.adoc[] diff --git a/servicetalk-opentelemetry-http/docs/modules/ROOT/pages/_partials/nav-versioned.adoc b/servicetalk-opentelemetry-http/docs/modules/ROOT/pages/_partials/nav-versioned.adoc new file mode 100644 index 0000000000..bbf7172a4c --- /dev/null +++ b/servicetalk-opentelemetry-http/docs/modules/ROOT/pages/_partials/nav-versioned.adoc @@ -0,0 +1 @@ +* xref:{page-version}@servicetalk-opentelemetry-http::index.adoc[OpenTelemetry Tracing] diff --git a/servicetalk-opentelemetry-http/docs/modules/ROOT/pages/index.adoc b/servicetalk-opentelemetry-http/docs/modules/ROOT/pages/index.adoc new file mode 100644 index 0000000000..d74ed7c219 --- /dev/null +++ b/servicetalk-opentelemetry-http/docs/modules/ROOT/pages/index.adoc @@ -0,0 +1,224 @@ +// Configure {source-root} values based on how this document is rendered: on GitHub or not +ifdef::env-github[] +:source-root: +endif::[] +ifndef::env-github[] +ifndef::source-root[:source-root: https://github.com/apple/servicetalk/blob/{page-origin-refname}] +endif::[] + += OpenTelemetry Tracing + +ServiceTalk provides comprehensive OpenTelemetry tracing support for HTTP and gRPC clients and servers through a set of filters that automatically instrument requests and responses. + +== Overview + +The `servicetalk-opentelemetry-http` module provides tracing filters that integrate with OpenTelemetry to automatically create spans for HTTP and gRPC operations, propagate trace context, and collect telemetry data. + +IMPORTANT: The `servicetalk-opentelemetry-asynccontext` module must be on your classpath for proper async context propagation. You need to explicitly add both `servicetalk-opentelemetry-http` and `servicetalk-opentelemetry-asynccontext` as dependencies to your project. + +== Key Components + +=== Context Propagation +* link:{source-root}/servicetalk-opentelemetry-asynccontext/src/main/java/io/servicetalk/opentelemetry/asynccontext/OtelCapturedContextProvider.java[OtelCapturedContextProvider] - Service loaded provider that lets ServiceTalk properly propagate the OTEL context and span information. The package just needs to be on the runtime classpath. + +=== Client-Side Tracing +* link:{source-root}/servicetalk-opentelemetry-http/src/main/java/io/servicetalk/opentelemetry/http/OpenTelemetryHttpRequesterFilter.java[OpenTelemetryHttpRequesterFilter] - Client tracing filter for HTTP and gRPC + +=== Server-Side Tracing +* link:{source-root}/servicetalk-opentelemetry-http/src/main/java/io/servicetalk/opentelemetry/http/OpenTelemetryHttpServiceFilter.java[OpenTelemetryHttpServiceFilter] - Server tracing filter for HTTP and gRPC + +=== Configuration +* link:{source-root}/servicetalk-opentelemetry-http/src/main/java/io/servicetalk/opentelemetry/http/OpenTelemetryHttpRequesterFilter.java[OpenTelemetryHttpRequesterFilter.Builder] - Builder for configuring tracing options for requesters. +* link:{source-root}/servicetalk-opentelemetry-http/src/main/java/io/servicetalk/opentelemetry/http/OpenTelemetryHttpServiceFilter.java[OpenTelemetryHttpServiceFilter.Builder] - Builder for configuring tracing options for services. + +== Key Features + +* **Automatic Span Creation**: Creates spans for HTTP and gRPC requests and responses following OpenTelemetry semantic conventions +* **Context Propagation**: Propagates trace context across service boundaries using standard headers +* **Configurable Attributes**: Capture custom request and response headers as span attributes +* **gRPC Support**: Specialized support for gRPC over HTTP/2 tracing +* **Filter Ordering**: Proper integration with ServiceTalk's filter chain for correct context handling + +== Quick Start + +NOTE: For complete dependency setup and OpenTelemetry SDK configuration, see the working examples in the ServiceTalk examples module link:{source-root}/servicetalk-examples/http/opentelemetry-tracing/src/main/java/io/servicetalk/examples/http/opentelemetry/tracing[here]. + +=== Client Configuration + +[source,java] +---- +HttpClient client = HttpClients.forSingleAddress("example.com", 80) + .appendClientFilter(new OpenTelemetryHttpRequesterFilter.Builder() + .componentName("my-client") + .build()) + .build(); +---- + +=== Server Configuration + +[source,java] +---- +HttpServerBuilder.forAddress(localAddress(0)) + // IMPORTANT: Use non-offloading filter for proper context propagation + .appendNonOffloadingServiceFilter(new OpenTelemetryHttpServiceFilter.Builder() + .build()) + // Add request draining and exception mapping AFTER OpenTelemetry filter. + .appendNonOffloadingServiceFilter(HttpRequestAutoDrainingServiceFilter.INSTANCE) + .appendNonOffloadingServiceFilter(HttpExceptionMapperServiceFilter.INSTANCE) + + .listen(service); +---- + +== Advanced Configuration + +=== Capturing Custom Headers + +[source,java] +---- +// Capture specific request and response headers as span attributes +OpenTelemetryHttpRequesterFilter filter = new OpenTelemetryHttpRequesterFilter.Builder() + .componentName("my-client") + .capturedRequestHeaders(Arrays.asList("content-encoding", "content-length")) + .capturedResponseHeaders(Arrays.asList("content-type")) + .build(); +---- + +== Filter Ordering Guidelines + +Proper filter ordering is crucial for OpenTelemetry tracing to work correctly with other ServiceTalk features. + +=== Server Filters + +For servers, use non-offloading filters for proper context propagation and pay special attention to filter ordering: + +[source,java] +---- +HttpServerBuilder.forAddress(localAddress(0)) + // OpenTelemetry filter MUST be first for proper context propagation + .appendNonOffloadingServiceFilter(new OpenTelemetryHttpServiceFilter.Builder() + .build()) + + // Request draining filter MUST come after OpenTelemetry filter + // This ensures tracing information is captured for auto-drained requests + .appendNonOffloadingServiceFilter(HttpRequestAutoDrainingServiceFilter.INSTANCE) + + // Exception mapping should come after tracing to ensure correct status codes + .appendNonOffloadingServiceFilter(HttpExceptionMapperServiceFilter.INSTANCE) + + // Other filters can follow + .listen(service); +---- + +**Important:** The `HttpRequestAutoDrainingServiceFilter` must be placed *after* the OpenTelemetry filter. This is critical for server-side tracing accuracy, particularly for requests like GET where the request body is typically ignored. If auto-draining occurs before the OpenTelemetry filter processes the request, tracing information may be incomplete or incorrect. + +=== Client Filters + +When using multiple filters on clients, place the OpenTelemetry filter appropriately in the chain: + +[source,java] +---- +OpenTelemetryHttpRequesterFilter filter = + new OpenTelemetryHttpRequesterFilter.Builder() + .componentName("my-client") + .build(); +HttpClient client = HttpClients.forSingleAddress("example.com", 80) + // OpenTelemetry filter for span creation and context propagation + .appendClientFilter(filter) + // Optional: adding the same filter as a connection filter will give + // you additional spans for each physical request sent over the wire + .appendConnectionFilter(filter) + // Logging comes after tracing so we can get trace-id's in logs + .appendClientFilter(loggingFilter) + // Retry and other resilience filters also come after tracing + .appendClientFilter(retryFilter) + + .build(); +---- + +=== Filter Ordering Best Practices + +1. **OpenTelemetry filters should be among the first filters** to ensure proper context establishment +2. **Use non-offloading filters** (`appendNonOffloadingServiceFilter`) for OpenTelemetry filters on the server-side to ensure earlier context establishment +3. **Request draining must come after OpenTelemetry** on the server side +4. **Exception mapping should come after OpenTelemetry** to ensure trace status reflects actual response codes +5. **Lifecycle observers should come after OpenTelemetry** to see correct span information + +== Context Propagation + +OpenTelemetry context is automatically propagated through multiple mechanisms to ensure traces are correlated correctly across service boundaries and async operations. + +=== Header Propagation + +OpenTelemetry context is automatically injected into and extracted from headers using the standard OpenTelemetry propagation format: + +* **W3C Trace Context** (`traceparent`, `tracestate` headers) +* **B3 Propagation** (if configured) +* **Custom propagators** (if configured in the OpenTelemetry SDK) + +[source,java] +---- +// Context is automatically propagated via headers +HttpResponse response = client.request(client.get("/api/endpoint")); +// The server will receive trace context via HTTP headers +---- + +=== Async Context Integration + +ServiceTalk's async context system ensures OpenTelemetry context is maintained across: + +* **Thread boundaries** during async operations +* **Publisher/Subscriber chains** in reactive streams +* **Executor transitions** when work is offloaded +* **Filter chains** where context must be preserved + +This integration is provided by the `servicetalk-opentelemetry-asynccontext` module which should be added as a `runtimeOnly` dependency. This module provides the `CapturedContextProvider` class which will be service-loaded by the ServiceTalk framework. + +=== Context Scope Management + +OpenTelemetry spans are automatically activated and deactivated at appropriate points: + +[source,java] +---- +// Client side: span is active during request processing +client.request(client.get("/api")) + .beforeOnSuccess(response -> { + // Current span is still active here + Span currentSpan = Span.current(); + currentSpan.setAttribute(myStringAttributeKey, "attribute value"); + }); + +// Server side: span is active during service method execution +service.handle((ctx, request, responseFactory) -> { + // Current span is active and contains trace context from client + Span currentSpan = Span.current(); + currentSpan.addEvent("Processing request"); + return responseFactory.ok(); +}); +---- + +== gRPC Support + +The tracing filters provide specialized support for gRPC over HTTP/2: + +* Automatic detection of gRPC requests +* gRPC-specific span naming and attributes +* Proper status code mapping + +== Troubleshooting + +=== Common Issues + +**Context Not Propagating** +Ensure `servicetalk-opentelemetry-asynccontext` is on the classpath and the filter is properly ordered. + +**Missing Spans** +Verify OpenTelemetry is properly configured and the global OpenTelemetry instance is set. + +== Examples + +For complete working examples, see the link:{source-root}/servicetalk-examples/http/opentelemetry-tracing/src/main/java/io/servicetalk/examples/http/opentelemetry/tracing[OpenTelemetry tracing examples] in the ServiceTalk examples module. + +== Related Documentation + +* xref:{page-version}@servicetalk-concurrent-api::async-context.adoc[ServiceTalk Asynchronous Context] +* https://opentelemetry.io/docs/instrumentation/java/[OpenTelemetry Java Documentation] diff --git a/settings.gradle b/settings.gradle index 4e540ceb72..10e0c41a71 100755 --- a/settings.gradle +++ b/settings.gradle @@ -66,6 +66,7 @@ include "servicetalk-annotations", "servicetalk-examples:http:http2", "servicetalk-examples:http:jaxrs", "servicetalk-examples:http:metadata", + "servicetalk-examples:http:opentelemetry-tracing", "servicetalk-examples:http:opentracing", "servicetalk-examples:http:observer", "servicetalk-examples:http:retry", @@ -152,6 +153,7 @@ project(":servicetalk-examples:http:timeout").name = "servicetalk-examples-http- project(":servicetalk-examples:http:jaxrs").name = "servicetalk-examples-http-jaxrs" project(":servicetalk-examples:http:loadbalancer").name = "servicetalk-examples-http-loadbalancer" project(":servicetalk-examples:http:metadata").name = "servicetalk-examples-http-metadata" +project(":servicetalk-examples:http:opentelemetry-tracing").name = "servicetalk-examples-http-opentelemetry-tracing" project(":servicetalk-examples:http:opentracing").name = "servicetalk-examples-http-opentracing" project(":servicetalk-examples:http:observer").name = "servicetalk-examples-http-observer" project(":servicetalk-examples:http:retry").name = "servicetalk-examples-http-retry"