diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java index 4f8b091d55..d0dbd1c3da 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java @@ -34,9 +34,9 @@ public class BuiltInMetricsConstant { public static final String METER_NAME = "spanner.googleapis.com/internal/client"; - public static final String GAX_METER_NAME = OpenTelemetryMetricsRecorder.GAX_METER_NAME; - + static final String SPANNER_METER_NAME = "spanner-java"; + static final String GFE_LATENCIES_NAME = "gfe_latencies"; static final String OPERATION_LATENCIES_NAME = "operation_latencies"; static final String ATTEMPT_LATENCIES_NAME = "attempt_latencies"; static final String OPERATION_LATENCY_NAME = "operation_latency"; @@ -114,6 +114,7 @@ static Map getAllViews() { ImmutableMap.Builder views = ImmutableMap.builder(); defineView( views, + BuiltInMetricsConstant.GAX_METER_NAME, BuiltInMetricsConstant.OPERATION_LATENCY_NAME, BuiltInMetricsConstant.OPERATION_LATENCIES_NAME, BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM, @@ -121,6 +122,7 @@ static Map getAllViews() { "ms"); defineView( views, + BuiltInMetricsConstant.GAX_METER_NAME, BuiltInMetricsConstant.ATTEMPT_LATENCY_NAME, BuiltInMetricsConstant.ATTEMPT_LATENCIES_NAME, BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM, @@ -128,6 +130,15 @@ static Map getAllViews() { "ms"); defineView( views, + BuiltInMetricsConstant.SPANNER_METER_NAME, + BuiltInMetricsConstant.GFE_LATENCIES_NAME, + BuiltInMetricsConstant.GFE_LATENCIES_NAME, + BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM, + InstrumentType.HISTOGRAM, + "ms"); + defineView( + views, + BuiltInMetricsConstant.GAX_METER_NAME, BuiltInMetricsConstant.OPERATION_COUNT_NAME, BuiltInMetricsConstant.OPERATION_COUNT_NAME, Aggregation.sum(), @@ -135,6 +146,7 @@ static Map getAllViews() { "1"); defineView( views, + BuiltInMetricsConstant.GAX_METER_NAME, BuiltInMetricsConstant.ATTEMPT_COUNT_NAME, BuiltInMetricsConstant.ATTEMPT_COUNT_NAME, Aggregation.sum(), @@ -145,6 +157,7 @@ static Map getAllViews() { private static void defineView( ImmutableMap.Builder viewMap, + String meterName, String metricName, String metricViewName, Aggregation aggregation, @@ -153,7 +166,7 @@ private static void defineView( InstrumentSelector selector = InstrumentSelector.builder() .setName(BuiltInMetricsConstant.METER_NAME + '/' + metricName) - .setMeterName(BuiltInMetricsConstant.GAX_METER_NAME) + .setMeterName(meterName) .setType(type) .setUnit(unit) .build(); diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java index 9367c45b63..c19f8bfb22 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java @@ -28,6 +28,8 @@ import com.google.cloud.opentelemetry.detection.AttributeKeys; import com.google.cloud.opentelemetry.detection.DetectedPlatform; import com.google.cloud.opentelemetry.detection.GCPPlatformDetector; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.hash.HashFunction; import com.google.common.hash.Hashing; import io.opentelemetry.api.OpenTelemetry; @@ -42,6 +44,7 @@ import java.util.HashMap; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ExecutionException; import java.util.logging.Level; import java.util.logging.Logger; import javax.annotation.Nullable; @@ -57,6 +60,9 @@ final class BuiltInOpenTelemetryMetricsProvider { private OpenTelemetry openTelemetry; + private final Cache> clientAttributesCache = + CacheBuilder.newBuilder().maximumSize(1000).build(); + private BuiltInOpenTelemetryMetricsProvider() {} OpenTelemetry getOrCreateOpenTelemetry(String projectId, @Nullable Credentials credentials) { @@ -78,16 +84,29 @@ OpenTelemetry getOrCreateOpenTelemetry(String projectId, @Nullable Credentials c } } - Map createClientAttributes(String projectId, String client_name) { - Map clientAttributes = new HashMap<>(); - clientAttributes.put(LOCATION_ID_KEY.getKey(), detectClientLocation()); - clientAttributes.put(PROJECT_ID_KEY.getKey(), projectId); - clientAttributes.put(INSTANCE_CONFIG_ID_KEY.getKey(), "unknown"); - clientAttributes.put(CLIENT_NAME_KEY.getKey(), client_name); - String clientUid = getDefaultTaskValue(); - clientAttributes.put(CLIENT_UID_KEY.getKey(), clientUid); - clientAttributes.put(CLIENT_HASH_KEY.getKey(), generateClientHash(clientUid)); - return clientAttributes; + Map createOrGetClientAttributes(String projectId, String client_name) { + try { + String key = projectId + client_name; + return clientAttributesCache.get( + key, + () -> { + Map clientAttributes = new HashMap<>(); + clientAttributes.put(LOCATION_ID_KEY.getKey(), detectClientLocation()); + clientAttributes.put(PROJECT_ID_KEY.getKey(), projectId); + clientAttributes.put(INSTANCE_CONFIG_ID_KEY.getKey(), "unknown"); + clientAttributes.put(CLIENT_NAME_KEY.getKey(), client_name); + String clientUid = getDefaultTaskValue(); + clientAttributes.put(CLIENT_UID_KEY.getKey(), clientUid); + clientAttributes.put(CLIENT_HASH_KEY.getKey(), generateClientHash(clientUid)); + return clientAttributes; + }); + } catch (ExecutionException executionException) { + logger.log( + Level.WARNING, + "Unable to get Client Attributes for client side metrics, will skip exporting client side metrics", + executionException); + return null; + } } /** diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsRecorder.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsRecorder.java new file mode 100644 index 0000000000..d5c72506da --- /dev/null +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsRecorder.java @@ -0,0 +1,88 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.spanner; + +import com.google.api.gax.core.GaxProperties; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.Meter; +import java.util.HashMap; +import java.util.Map; + +/** OpenTelemetry implementation of recording built in metrics. */ +public class BuiltInOpenTelemetryMetricsRecorder { + + private final DoubleHistogram gfeLatencyRecorder; + private final Map attributes = new HashMap<>(); + + /** + * Creates the following instruments for the following metrics: + * + *
    + *
  • GFE Latency: Histogram + *
+ * + * @param openTelemetry OpenTelemetry instance + */ + public BuiltInOpenTelemetryMetricsRecorder( + OpenTelemetry openTelemetry, Map clientAttributes) { + if (openTelemetry != null && clientAttributes != null) { + gfeLatencyRecorder = null; + return; + } + Meter meter = + openTelemetry + .meterBuilder(BuiltInMetricsConstant.SPANNER_METER_NAME) + .setInstrumentationVersion(GaxProperties.getLibraryVersion(getClass())) + .build(); + this.gfeLatencyRecorder = + meter + .histogramBuilder( + BuiltInMetricsConstant.METER_NAME + '/' + BuiltInMetricsConstant.GFE_LATENCIES_NAME) + .setDescription( + "Latency between Google's network receiving an RPC and reading back the first byte of the response") + .setUnit("ms") + .build(); + this.attributes.putAll(clientAttributes); + } + + /** + * Record the latency between Google's network receiving an RPC and reading back the first byte of + * the response. Data is stored in a Histogram. + * + * @param gfeLatency Attempt Latency in ms + * @param attributes Map of the attributes to store + */ + public void recordGFELatency(double gfeLatency, Map attributes) { + if (gfeLatencyRecorder != null) { + this.attributes.putAll(attributes); + gfeLatencyRecorder.record(gfeLatency, toOtelAttributes(this.attributes)); + } + } + + @VisibleForTesting + Attributes toOtelAttributes(Map attributes) { + Preconditions.checkNotNull(attributes, "Attributes map cannot be null"); + AttributesBuilder attributesBuilder = Attributes.builder(); + attributes.forEach(attributesBuilder::put); + return attributesBuilder.build(); + } +} diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java index af54515e7c..3eae3206c4 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java @@ -1680,6 +1680,18 @@ public OpenTelemetry getOpenTelemetry() { } } + /** Returns an instance of OpenTelemetry object for Built-in Client metrics. */ + public OpenTelemetry getBuiltInMetricsOpenTelemetry() { + return this.builtInOpenTelemetryMetricsProvider.getOrCreateOpenTelemetry( + this.getProjectId(), getCredentials()); + } + + /** Returns attributes for an instance of Built-in Client metrics. */ + public Map getBuiltInMetricsClientAttributes() { + return builtInOpenTelemetryMetricsProvider.createOrGetClientAttributes( + this.getProjectId(), "spanner-java/" + GaxProperties.getLibraryVersion(getClass())); + } + @Override public ApiTracerFactory getApiTracerFactory() { return createApiTracerFactory(false, false); @@ -1729,11 +1741,13 @@ private ApiTracerFactory createMetricsApiTracerFactory() { this.builtInOpenTelemetryMetricsProvider.getOrCreateOpenTelemetry( this.getProjectId(), getCredentials()); - return openTelemetry != null + Map clientAttributes = + builtInOpenTelemetryMetricsProvider.createOrGetClientAttributes( + this.getProjectId(), "spanner-java/" + GaxProperties.getLibraryVersion(getClass())); + return openTelemetry != null && clientAttributes != null ? new MetricsTracerFactory( new OpenTelemetryMetricsRecorder(openTelemetry, BuiltInMetricsConstant.METER_NAME), - builtInOpenTelemetryMetricsProvider.createClientAttributes( - this.getProjectId(), "spanner-java/" + GaxProperties.getLibraryVersion(getClass()))) + clientAttributes) : null; } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java index fe23b09798..19c648b579 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java @@ -357,6 +357,8 @@ public GapicSpannerRpc(final SpannerOptions options) { options.getInterceptorProvider(), SpannerInterceptorProvider.createDefault( options.getOpenTelemetry(), + options.getBuiltInMetricsOpenTelemetry(), + options.getBuiltInMetricsClientAttributes(), (() -> directPathEnabledSupplier.get())))) // This sets the trace context headers. .withTraceContext(endToEndTracingEnabled, options.getOpenTelemetry()) diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java index 026f9b4ca9..de5a042ff7 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java @@ -25,6 +25,7 @@ import com.google.api.gax.tracing.ApiTracer; import com.google.cloud.spanner.BuiltInMetricsConstant; +import com.google.cloud.spanner.BuiltInOpenTelemetryMetricsRecorder; import com.google.cloud.spanner.CompositeTracer; import com.google.cloud.spanner.SpannerExceptionFactory; import com.google.cloud.spanner.SpannerRpcMetrics; @@ -94,12 +95,17 @@ class HeaderInterceptor implements ClientInterceptor { private static final Level LEVEL = Level.INFO; private final SpannerRpcMetrics spannerRpcMetrics; + private final BuiltInOpenTelemetryMetricsRecorder builtInOpenTelemetryMetricsRecorder; + private final Supplier directPathEnabledSupplier; HeaderInterceptor( - SpannerRpcMetrics spannerRpcMetrics, Supplier directPathEnabledSupplier) { + SpannerRpcMetrics spannerRpcMetrics, + BuiltInOpenTelemetryMetricsRecorder builtInOpenTelemetryMetricsRecorder, + Supplier directPathEnabledSupplier) { this.spannerRpcMetrics = spannerRpcMetrics; this.directPathEnabledSupplier = directPathEnabledSupplier; + this.builtInOpenTelemetryMetricsRecorder = builtInOpenTelemetryMetricsRecorder; } @Override @@ -118,8 +124,8 @@ public void start(Listener responseListener, Metadata headers) { TagContext tagContext = getTagContext(key, method.getFullMethodName(), databaseName); Attributes attributes = getMetricAttributes(key, method.getFullMethodName(), databaseName); - Map builtInMetricsAttributes = - getBuiltInMetricAttributes(key, databaseName); + Map commonBuiltInMetricAttributes = + getCommonBuiltInMetricAttributes(key, databaseName); super.start( new SimpleForwardingClientCallListener(responseListener) { @Override @@ -127,8 +133,13 @@ public void onHeaders(Metadata metadata) { Boolean isDirectPathUsed = isDirectPathUsed(getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR)); addBuiltInMetricAttributes( - compositeTracer, builtInMetricsAttributes, isDirectPathUsed); - processHeader(metadata, tagContext, attributes, span); + compositeTracer, commonBuiltInMetricAttributes, isDirectPathUsed); + processHeader( + metadata, + tagContext, + attributes, + span, + getBuiltInMetricAttributes(commonBuiltInMetricAttributes, isDirectPathUsed)); super.onHeaders(metadata); } }, @@ -142,7 +153,11 @@ public void onHeaders(Metadata metadata) { } private void processHeader( - Metadata metadata, TagContext tagContext, Attributes attributes, Span span) { + Metadata metadata, + TagContext tagContext, + Attributes attributes, + Span span, + Map builtInMetricsAttributes) { MeasureMap measureMap = STATS_RECORDER.newMeasureMap(); String serverTiming = metadata.get(SERVER_TIMING_HEADER_KEY); if (serverTiming != null && serverTiming.startsWith(SERVER_TIMING_HEADER_PREFIX)) { @@ -154,6 +169,7 @@ private void processHeader( spannerRpcMetrics.recordGfeLatency(latency, attributes); spannerRpcMetrics.recordGfeHeaderMissingCount(0L, attributes); + builtInOpenTelemetryMetricsRecorder.recordGFELatency(latency, builtInMetricsAttributes); if (span != null) { span.setAttribute("gfe_latency", String.valueOf(latency)); @@ -224,8 +240,8 @@ private Attributes getMetricAttributes(String key, String method, DatabaseName d }); } - private Map getBuiltInMetricAttributes(String key, DatabaseName databaseName) - throws ExecutionException { + private Map getCommonBuiltInMetricAttributes( + String key, DatabaseName databaseName) throws ExecutionException { return builtInAttributesCache.get( key, () -> { @@ -240,17 +256,21 @@ private Map getBuiltInMetricAttributes(String key, DatabaseName }); } + private Map getBuiltInMetricAttributes( + Map commonBuiltInMetricsAttributes, Boolean isDirectPathUsed) { + Map builtInMetricAttributes = new HashMap<>(commonBuiltInMetricsAttributes); + builtInMetricAttributes.put( + BuiltInMetricsConstant.DIRECT_PATH_USED_KEY.getKey(), Boolean.toString(isDirectPathUsed)); + return builtInMetricAttributes; + } + private void addBuiltInMetricAttributes( CompositeTracer compositeTracer, - Map builtInMetricsAttributes, + Map commonBuiltInMetricsAttributes, Boolean isDirectPathUsed) { if (compositeTracer != null) { - // Direct Path used attribute - Map attributes = new HashMap<>(builtInMetricsAttributes); - attributes.put( - BuiltInMetricsConstant.DIRECT_PATH_USED_KEY.getKey(), Boolean.toString(isDirectPathUsed)); - - compositeTracer.addAttributes(attributes); + compositeTracer.addAttributes( + getBuiltInMetricAttributes(commonBuiltInMetricsAttributes, isDirectPathUsed)); } } diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/SpannerInterceptorProvider.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/SpannerInterceptorProvider.java index c3c05b8af1..cbb9b62bdd 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/SpannerInterceptorProvider.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/SpannerInterceptorProvider.java @@ -18,6 +18,7 @@ import com.google.api.core.InternalApi; import com.google.api.core.ObsoleteApi; import com.google.api.gax.grpc.GrpcInterceptorProvider; +import com.google.cloud.spanner.BuiltInOpenTelemetryMetricsRecorder; import com.google.cloud.spanner.SpannerRpcMetrics; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; @@ -26,7 +27,9 @@ import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; @@ -44,12 +47,15 @@ private SpannerInterceptorProvider(List clientInterceptors) { @ObsoleteApi("This method always uses Global OpenTelemetry") public static SpannerInterceptorProvider createDefault() { - return createDefault(GlobalOpenTelemetry.get()); + return createDefault(GlobalOpenTelemetry.get(), GlobalOpenTelemetry.get()); } - public static SpannerInterceptorProvider createDefault(OpenTelemetry openTelemetry) { + public static SpannerInterceptorProvider createDefault( + OpenTelemetry openTelemetry, OpenTelemetry builtInMetricsopenTelemetry) { return createDefault( openTelemetry, + builtInMetricsopenTelemetry, + new HashMap<>(), Suppliers.memoize( () -> { return false; @@ -57,13 +63,20 @@ public static SpannerInterceptorProvider createDefault(OpenTelemetry openTelemet } public static SpannerInterceptorProvider createDefault( - OpenTelemetry openTelemetry, Supplier directPathEnabledSupplier) { + OpenTelemetry openTelemetry, + OpenTelemetry builtInMetricsopenTelemetry, + Map builtInMetricsClientAttributes, + Supplier directPathEnabledSupplier) { List defaultInterceptorList = new ArrayList<>(); defaultInterceptorList.add(new SpannerErrorInterceptor()); defaultInterceptorList.add( new LoggingInterceptor(Logger.getLogger(GapicSpannerRpc.class.getName()), Level.FINER)); defaultInterceptorList.add( - new HeaderInterceptor(new SpannerRpcMetrics(openTelemetry), directPathEnabledSupplier)); + new HeaderInterceptor( + new SpannerRpcMetrics(openTelemetry), + new BuiltInOpenTelemetryMetricsRecorder( + builtInMetricsopenTelemetry, builtInMetricsClientAttributes), + directPathEnabledSupplier)); return new SpannerInterceptorProvider(ImmutableList.copyOf(defaultInterceptorList)); } diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java index 9f65402c31..a88c1044f1 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java @@ -88,7 +88,7 @@ public static void setup() { String client_name = "spanner-java/"; openTelemetry = OpenTelemetrySdk.builder().setMeterProvider(meterProvider.build()).build(); - attributes = provider.createClientAttributes("test-project", client_name); + attributes = provider.createOrGetClientAttributes("test-project", client_name); expectedBaseAttributes = Attributes.builder()