From b0fff1c6e45e6c0bd74fc7ce228a9f86da439ce8 Mon Sep 17 00:00:00 2001 From: Gul Jain Date: Thu, 7 Mar 2024 14:15:58 +0530 Subject: [PATCH] Metrics master (#1117) * Instrumentation * Updating comments * Adding global statistics along with GCS Connector specific statistics * testing * Resolving comments * Resolving comments --- .../fs/gcs/GhfsGlobalStorageStatistics.java | 595 ++++++++++++++++++ .../hadoop/fs/gcs/GhfsInstrumentation.java | 42 +- .../cloud/hadoop/fs/gcs/GhfsStatistic.java | 17 +- .../hadoop/fs/gcs/GhfsStorageStatistics.java | 2 +- .../cloud/hadoop/fs/gcs/GhfsStreamStats.java | 85 +++ .../fs/gcs/GoogleHadoopFSInputStream.java | 83 ++- .../hadoop/fs/gcs/GoogleHadoopFileSystem.java | 121 +++- .../fs/gcs/GoogleHadoopOutputStream.java | 64 +- ...gleHadoopFSInputStreamIntegrationTest.java | 57 +- ...eHadoopFileSystemDelegationTokensTest.java | 5 + ...GoogleHadoopFileSystemIntegrationTest.java | 399 +++++++++++- .../gcs/GoogleHadoopFileSystemTestBase.java | 9 + .../fs/gcs/GoogleHadoopOutputStreamTest.java | 17 +- .../fs/gcs/GoogleHadoopStatusMetricsTest.java | 83 --- .../fs/gcs/HadoopFileSystemTestBase.java | 19 +- .../google/cloud/hadoop/fs/gcs/TestUtils.java | 36 +- .../gcsio/GoogleCloudStorageClientImpl.java | 32 +- .../GoogleCloudStorageClientReadChannel.java | 15 + .../GoogleCloudStorageClientWriteChannel.java | 5 + .../GoogleCloudStorageFileSystemImpl.java | 43 +- .../hadoop/gcsio/GoogleCloudStorageImpl.java | 48 +- .../gcsio/GoogleCloudStorageReadChannel.java | 22 + ...java => GoogleCloudStorageStatistics.java} | 36 +- .../gcsio/GoogleCloudStorageWriteChannel.java | 2 + .../cloud/hadoop/util/ApiErrorExtractor.java | 4 + .../util/GcsClientStatisticInterface.java | 22 - .../util/GoogleCloudStorageEventBus.java | 86 +++ .../hadoop/util/RetryHttpInitializer.java | 30 +- .../interceptors/InvocationIdInterceptor.java | 2 + 29 files changed, 1705 insertions(+), 276 deletions(-) create mode 100644 gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsGlobalStorageStatistics.java create mode 100644 gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStreamStats.java delete mode 100644 gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopStatusMetricsTest.java rename gcsio/src/main/java/com/google/cloud/hadoop/gcsio/{GoogleCloudStorageStatusStatistics.java => GoogleCloudStorageStatistics.java} (65%) delete mode 100644 util/src/main/java/com/google/cloud/hadoop/util/GcsClientStatisticInterface.java create mode 100644 util/src/main/java/com/google/cloud/hadoop/util/GoogleCloudStorageEventBus.java diff --git a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsGlobalStorageStatistics.java b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsGlobalStorageStatistics.java new file mode 100644 index 0000000000..d8784dadd3 --- /dev/null +++ b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsGlobalStorageStatistics.java @@ -0,0 +1,595 @@ +/* + * Copyright 2023 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.hadoop.fs.gcs; + +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.DIRECTORIES_CREATED; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.FILES_CREATED; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.FILES_DELETED; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.FILES_DELETE_REJECTED; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_COPY_FROM_LOCAL_FILE; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_EXISTS; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_GET_DELEGATION_TOKEN; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_GET_FILE_CHECKSUM; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_GET_FILE_STATUS; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_GLOB_STATUS; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_LIST_FILES; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_LIST_LOCATED_STATUS; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_LIST_STATUS; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_MKDIRS; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_READ_EXCEPTIONS; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_READ_OPERATIONS; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_READ_SEEK_OPERATIONS; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.EXCEPTION_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_CLIENT_RATE_LIMIT_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_CLIENT_SIDE_ERROR_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_REQUEST_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_SERVER_SIDE_ERROR_COUNT; +import static com.google.cloud.hadoop.gcsio.StatisticTypeEnum.TYPE_DURATION; +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.api.client.googleapis.json.GoogleJsonResponseException; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.HttpResponseException; +import com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics; +import com.google.cloud.hadoop.util.ITraceFactory; +import com.google.cloud.hadoop.util.ITraceOperation; +import com.google.common.base.Stopwatch; +import com.google.common.eventbus.Subscribe; +import com.google.common.flogger.GoogleLogger; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.Nonnull; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.StorageStatistics; +import org.apache.hadoop.fs.statistics.DurationTrackerFactory; +import org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding; +import org.apache.hadoop.util.functional.CallableRaisingIOE; + +/** Storage statistics for GCS */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class GhfsGlobalStorageStatistics extends StorageStatistics { + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + + /** {@value} The key that stores all the registered metrics */ + public static final String NAME = "GhfsStorageStatistics"; + + public static final int LATENCY_LOGGING_THRESHOLD_MS = 150; + + // Instance to be used if it encounters any error while registering to Global Statistics. + // Error can happen for e.g. when different class loaders are used. + // If this instance is used, the metrics will not be reported to metrics sinks. + static final GhfsGlobalStorageStatistics DUMMY_INSTANCE = new GhfsGlobalStorageStatistics(); + + private final Map opsCount = new HashMap<>(); + private final Map minimums = new HashMap<>(); + private final Map maximums = new HashMap<>(); + private final Map means = new HashMap<>(); + + public GhfsGlobalStorageStatistics() { + super(NAME); + + for (GoogleCloudStorageStatistics opType : GoogleCloudStorageStatistics.values()) { + String symbol = opType.getSymbol(); + opsCount.put(symbol, new AtomicLong(0)); + } + + for (GhfsStatistic opType : GhfsStatistic.values()) { + String symbol = opType.getSymbol(); + opsCount.put(symbol, new AtomicLong(0)); + + if (opType.getType() == TYPE_DURATION) { + minimums.put(getMinKey(symbol), null); + maximums.put(getMaxKey(symbol), new AtomicLong(0)); + means.put(getMeanKey(symbol), new MeanStatistic()); + } + } + } + + static B trackDuration( + DurationTrackerFactory factory, + @Nonnull GhfsGlobalStorageStatistics stats, + GhfsStatistic statistic, + Object context, + ITraceFactory traceFactory, + CallableRaisingIOE operation) + throws IOException { + Stopwatch stopwatch = Stopwatch.createStarted(); + + try (ITraceOperation op_fs = + traceFactory.createRootWithLogging(statistic.getSymbol(), context)) { + stats.increment(statistic); + return IOStatisticsBinding.trackDuration(factory, statistic.getSymbol(), operation); + } finally { + stats.updateStats(statistic, stopwatch.elapsed().toMillis(), context); + } + } + + private long increment(GhfsStatistic statistic) { + return incrementCounter(statistic, 1); + } + + private void increment(GoogleCloudStorageStatistics statistic) { + incrementCounter(statistic, 1); + } + + /** + * Increment a specific counter. + * + * @param op operation + * @param count increment value + * @return the new value + */ + long incrementCounter(GhfsStatistic op, long count) { + return opsCount.get(op.getSymbol()).addAndGet(count); + } + + /** + * Increment a specific counter. + * + * @param op operation + * @param count increment value + */ + void incrementCounter(GoogleCloudStorageStatistics op, long count) { + opsCount.get(op.getSymbol()).addAndGet(count); + } + + @Override + public void reset() { + resetMetrics(opsCount); + resetMetrics(maximums); + + for (String ms : means.keySet()) { + means.get(ms).reset(); + } + + for (String ms : minimums.keySet()) { + minimums.put(ms, null); + } + } + + private void resetMetrics(Map metrics) { + for (AtomicLong value : metrics.values()) { + value.set(0); + } + } + + void updateStats(GhfsStatistic statistic, long durationMs, Object context) { + checkArgument( + statistic.getType() == TYPE_DURATION, + String.format("Unexpected instrumentation type %s", statistic)); + updateMinMaxStats(statistic, durationMs, durationMs, context); + + addMeanStatistic(statistic, durationMs, 1); + } + + private void addMeanStatistic(GhfsStatistic statistic, long totalDurationMs, int count) { + String meanKey = getMeanKey(statistic.getSymbol()); + if (means.containsKey(meanKey)) { + means.get(meanKey).addSample(totalDurationMs, count); + } + } + + void updateStats( + GhfsStatistic statistic, + long minLatency, + long maxLatency, + long totalDuration, + int count, + Object context) { + + updateMinMaxStats(statistic, minLatency, maxLatency, context); + addMeanStatistic(statistic, totalDuration, count); + opsCount.get(statistic.getSymbol()).addAndGet(count); + } + + private void updateMinMaxStats( + GhfsStatistic statistic, long minDurationMs, long maxDurationMs, Object context) { + String minKey = getMinKey(statistic.getSymbol()); + + AtomicLong minVal = minimums.get(minKey); + if (minVal == null) { + // There can be race here. It is ok to have the last write win. + minimums.put(minKey, new AtomicLong(minDurationMs)); + } else if (minDurationMs < minVal.get()) { + minVal.set(minDurationMs); + } + + String maxKey = getMaxKey(statistic.getSymbol()); + AtomicLong maxVal = maximums.get(maxKey); + if (maxDurationMs > maxVal.get()) { + if (maxDurationMs > LATENCY_LOGGING_THRESHOLD_MS) { + logger.atInfo().log( + "Detected potential high latency for operation %s. latencyMs=%s; previousMaxLatencyMs=%s; operationCount=%s; context=%s", + statistic, maxDurationMs, maxVal.get(), opsCount.get(statistic.getSymbol()), context); + } + + // There can be race here and can have some data points get missed. This is a corner case. + // Since this function can be called quite frequently, opting for performance over consistency + // here. + maxVal.set(maxDurationMs); + } + } + + /** + * Updating the required gcs specific statistics based on httpresponse. + * + * @param statusCode + */ + private void updateGcsIOSpecificStatistics(int statusCode) { + + if (statusCode >= 400 && statusCode < 500) { + incrementGcsClientSideCounter(); + + if (statusCode == 429) { + incrementRateLimitingCounter(); + } + } + + if (statusCode >= 500 && statusCode < 600) { + incrementGcsServerSideCounter(); + } + } + + /** + * Updating the required gcs specific statistics based on HttpResponseException. + * + * @param responseException contains statusCode based on which metrics are updated + */ + @Subscribe + private void subscriberOnHttpResponseException(@Nonnull HttpResponseException responseException) { + updateGcsIOSpecificStatistics(responseException.getStatusCode()); + } + + /** + * Updating the required gcs specific statistics based on GoogleJsonResponseException. + * + * @param responseException contains statusCode based on which metrics are updated + */ + @Subscribe + private void subscriberOnGoogleJsonResponseException( + @Nonnull GoogleJsonResponseException responseException) { + updateGcsIOSpecificStatistics(responseException.getStatusCode()); + } + + /** + * Updating the required gcs specific statistics based on HttpResponse. + * + * @param response contains statusCode based on which metrics are updated + */ + @Subscribe + private void subscriberOnHttpResponse(@Nonnull HttpResponse response) { + updateGcsIOSpecificStatistics(response.getStatusCode()); + } + + /** + * Updating the GCS_TOTAL_REQUEST_COUNT + * + * @param request + */ + @Subscribe + private void subscriberOnHttpRequest(@Nonnull HttpRequest request) { + incrementGcsTotalRequestCount(); + } + + /** + * Updating the EXCEPTION_COUNT + * + * @param exception + */ + @Subscribe + private void subscriberOnException(IOException exception) { + incrementGcsExceptionCount(); + } + + private void incrementGcsExceptionCount() { + increment(EXCEPTION_COUNT); + } + + private void incrementGcsTotalRequestCount() { + increment(GCS_REQUEST_COUNT); + } + + private void incrementRateLimitingCounter() { + increment(GCS_CLIENT_RATE_LIMIT_COUNT); + } + + private void incrementGcsClientSideCounter() { + increment(GCS_CLIENT_SIDE_ERROR_COUNT); + } + + private void incrementGcsServerSideCounter() { + increment(GCS_SERVER_SIDE_ERROR_COUNT); + } + + void streamReadBytes(int bytesRead) { + incrementCounter(GhfsStatistic.STREAM_READ_BYTES, bytesRead); + } + + /** If more data was requested than was actually returned, this was an incomplete read. */ + void streamReadOperationInComplete(int requested, int actual) { + if (requested > actual) { + increment(GhfsStatistic.STREAM_READ_OPERATIONS_INCOMPLETE); + } + } + + void streamReadOperations() { + incrementCounter(STREAM_READ_OPERATIONS, 1); + } + + void streamReadSeekOperations() { + incrementCounter(STREAM_READ_SEEK_OPERATIONS, 1); + } + + void streamReadSeekBackward(long negativeOffset) { + increment(GhfsStatistic.STREAM_READ_SEEK_BACKWARD_OPERATIONS); + incrementCounter(GhfsStatistic.STREAM_READ_SEEK_BYTES_BACKWARDS, -negativeOffset); + } + + void streamReadSeekForward(long skipped) { + if (skipped > 0) { + incrementCounter(GhfsStatistic.STREAM_READ_SEEK_BYTES_SKIPPED, skipped); + } + + increment(GhfsStatistic.STREAM_READ_SEEK_FORWARD_OPERATIONS); + } + + void streamWriteBytes(int bytesWritten) { + incrementCounter(GhfsStatistic.STREAM_WRITE_BYTES, bytesWritten); + } + + void filesCreated() { + increment(FILES_CREATED); + } + + public void fileDeleted(int count) { + incrementCounter(FILES_DELETED, count); + } + + public void directoryCreated() { + incrementCounter(DIRECTORIES_CREATED, 1); + } + + public void filesDeleteRejected() { + incrementCounter(FILES_DELETE_REJECTED, 1); + } + + public void invocationListStatus() { + incrementCounter(INVOCATION_LIST_STATUS, 1); + } + + public void invocationListFiles() { + incrementCounter(INVOCATION_LIST_FILES, 1); + } + + public void invocationGetFileStatus() { + incrementCounter(INVOCATION_GET_FILE_STATUS, 1); + } + + public void invocationGlobStatus() { + incrementCounter(INVOCATION_GLOB_STATUS, 1); + } + + public void invocationGetDelegationToken() { + incrementCounter(INVOCATION_GET_DELEGATION_TOKEN, 1); + } + + public void invocationCopyFromLocalFile() { + incrementCounter(INVOCATION_COPY_FROM_LOCAL_FILE, 1); + } + + public void invocationGetFileChecksum() { + incrementCounter(INVOCATION_GET_FILE_CHECKSUM, 1); + } + + public void invocationExists() { + incrementCounter(INVOCATION_EXISTS, 1); + } + + public void invocationListLocatedStatus() { + incrementCounter(INVOCATION_LIST_LOCATED_STATUS, 1); + } + + public void streamReadExceptions() { + incrementCounter(STREAM_READ_EXCEPTIONS, 1); + } + + public void invocationMkdirs() { + incrementCounter(INVOCATION_MKDIRS, 1); + } + + private class LongIterator implements Iterator { + private Iterator iterator = getMetricNames(); + + private Iterator getMetricNames() { + ArrayList metrics = new ArrayList<>(); + + metrics.addAll(opsCount.keySet()); + metrics.addAll(minimums.keySet()); + metrics.addAll(maximums.keySet()); + metrics.addAll(means.keySet()); + + return metrics.iterator(); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public LongStatistic next() { + if (!iterator.hasNext()) { + throw new NoSuchElementException(); + } + + final String entry = iterator.next(); + return new LongStatistic(entry, getValue(entry)); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } + + private long getValue(String key) { + if (opsCount.containsKey(key)) { + return opsCount.get(key).longValue(); + } + + if (maximums.containsKey(key)) { + return maximums.get(key).longValue(); + } + + if (minimums.containsKey(key) && minimums.get(key) != null) { + return minimums.get(key).longValue(); + } + + if (means.containsKey(key)) { + return Math.round(means.get(key).getValue()); + } + + return 0L; + } + + @Override + public Iterator getLongStatistics() { + return new LongIterator(); + } + + @Override + public Long getLong(String key) { + return this.getValue(key); + } + + @Override + public boolean isTracked(String key) { + return opsCount.containsKey(key) + || maximums.containsKey(key) + || minimums.containsKey(key) + || means.containsKey(key); + } + + /** + * To get the minimum value which is stored with MINIMUM extension + * + * @param symbol + * @return minimum statistic value + */ + public Long getMin(String symbol) { + AtomicLong minValue = minimums.get(getMinKey(symbol)); + if (minValue == null) { + return 0L; + } + + return minValue.longValue(); + } + + private String getMinKey(String symbol) { + return symbol + "_min"; + } + + private String getMaxKey(String symbol) { + return symbol + "_max"; + } + + private String getMeanKey(String symbol) { + return symbol + "_mean"; + } + + /** + * To get the maximum value which is stored with MAXIMUM extension + * + * @param symbol + * @return maximum statistic value + */ + public Long getMax(String symbol) { + AtomicLong maxValue = maximums.get(getMaxKey(symbol)); + if (maxValue == null) { + return 0L; + } + + return maxValue.longValue(); + } + + /** + * To get the mean value which is stored with MEAN extension + * + * @param key + * @return mean statistic value + */ + public double getMean(String key) { + MeanStatistic val = means.get(getMeanKey(key)); + if (val == null) { + return 0; + } + + return val.getValue(); + } + + /** This class keeps track of mean statistics by keeping track of sum and number of samples. */ + static class MeanStatistic { + private int sample; + + private long sum; + + synchronized void addSample(long val) { + addSample(val, 1); + } + + synchronized void addSample(long val, int count) { + sample += count; + sum += val; + } + + double getValue() { + if (sample == 0) { + return 0; + } + + return sum / Math.max(1, sample); // to protect against the race with reset(). + } + + public void reset() { + sum = 0; + sample = 0; + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (Iterator it = this.getLongStatistics(); it.hasNext(); ) { + LongStatistic statistic = it.next(); + if (sb.length() != 0) { + sb.append(", "); + } + + sb.append(String.format("%s=%s", statistic.getName(), statistic.getValue())); + } + + return String.format("[%s]", sb); + } +} diff --git a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsInstrumentation.java b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsInstrumentation.java index bd0ad89e9a..0a087a1180 100644 --- a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsInstrumentation.java +++ b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsInstrumentation.java @@ -27,14 +27,12 @@ import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_WRITE_CLOSE_OPERATIONS; import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_WRITE_EXCEPTIONS; import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_WRITE_OPERATIONS; -import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatusStatistics.GCS_CLIENT_RATE_LIMIT_COUNT; import static org.apache.hadoop.fs.statistics.IOStatisticsSupport.snapshotIOStatistics; import static org.apache.hadoop.fs.statistics.StoreStatisticNames.SUFFIX_FAILURES; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore; -import com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatusStatistics; +import com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics; import com.google.cloud.hadoop.gcsio.StatisticTypeEnum; -import com.google.cloud.hadoop.util.GcsClientStatisticInterface; import com.google.common.flogger.GoogleLogger; import java.io.Closeable; import java.net.URI; @@ -64,18 +62,14 @@ * Instrumentation of GCS. * *

Counters and metrics are generally addressed in code by their name or {@link GhfsStatistic} - * and {@link GoogleCloudStorageStatusStatistics} key. There may be some Statistics which do - * not have an entry here. To avoid attempts to access such counters failing, the operations to - * increment/query metric values are designed to handle lookup failures. + * key. There may be some Statistics which do not have an entry here. To avoid attempts to + * access such counters failing, the operations to increment/query metric values are designed to + * handle lookup failures. * *

GoogleHadoopFileSystem StorageStatistics are dynamically derived from the IOStatistics. */ public class GhfsInstrumentation - implements Closeable, - MetricsSource, - IOStatisticsSource, - DurationTrackerFactory, - GcsClientStatisticInterface { + implements Closeable, MetricsSource, IOStatisticsSource, DurationTrackerFactory { private static final String METRICS_SOURCE_BASENAME = "GCSMetrics"; @@ -214,7 +208,7 @@ public void incrementCounter(GhfsStatistic op, long count) { * * @param op operation */ - private void incrementCounter(GoogleCloudStorageStatusStatistics op) { + private void incrementCounter(GoogleCloudStorageStatistics op) { String name = op.getSymbol(); incrementMutableCounter(name, 1); @@ -263,7 +257,7 @@ protected final MutableCounterLong counter(GhfsStatistic op) { * @param op statistic to count * @return a new counter */ - private final MutableCounterLong counter(GoogleCloudStorageStatusStatistics op) { + private final MutableCounterLong counter(GoogleCloudStorageStatistics op) { return counter(op.getSymbol(), op.getDescription()); } @@ -353,20 +347,6 @@ private void incrementMutableCounter(String name, long count) { } } - /** - * Counter Metrics updation based on the Http response - * - * @param statusCode of ther Http response - */ - @Override - public void statusMetricsUpdation(int statusCode) { - switch (statusCode) { - case 429: - incrementCounter(GCS_CLIENT_RATE_LIMIT_COUNT); - break; - } - } - /** * A duration tracker which updates a mutable counter with a metric. The metric is updated with * the count on start; after a failure the failures count is incremented by one. @@ -1006,14 +986,6 @@ private IOStatisticsStoreBuilder createStoreBuilder() { } }); - // registering metrics of GoogleCloudStorageStatusStatistics which are all counters - EnumSet.allOf(GoogleCloudStorageStatusStatistics.class) - .forEach( - stat -> { - counter(stat); - storeBuilder.withCounters(stat.getSymbol()); - }); - return storeBuilder; } } diff --git a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStatistic.java b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStatistic.java index 6971b1cb1d..baa8c0d1b2 100644 --- a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStatistic.java +++ b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStatistic.java @@ -81,19 +81,13 @@ public enum GhfsStatistic { "files_delete_rejected", "Total number of files whose delete request was rejected", TYPE_COUNTER), - INVOCATION_COPY_FROM_LOCAL_FILE( - StoreStatisticNames.OP_COPY_FROM_LOCAL_FILE, "Calls of copyFromLocalFile()", TYPE_COUNTER), INVOCATION_CREATE(StoreStatisticNames.OP_CREATE, "Calls of create()", TYPE_DURATION), - INVOCATION_CREATE_NON_RECURSIVE( - StoreStatisticNames.OP_CREATE_NON_RECURSIVE, "Calls of createNonRecursive()", TYPE_DURATION), INVOCATION_DELETE(StoreStatisticNames.OP_DELETE, "Calls of delete()", TYPE_DURATION), INVOCATION_EXISTS(StoreStatisticNames.OP_EXISTS, "Calls of exists()", TYPE_COUNTER), - INVOCATION_GET_DELEGATION_TOKEN( - StoreStatisticNames.OP_GET_DELEGATION_TOKEN, "Calls of getDelegationToken()", TYPE_COUNTER), - INVOCATION_GET_FILE_CHECKSUM( - StoreStatisticNames.OP_GET_FILE_CHECKSUM, "Calls of getFileChecksum()", TYPE_COUNTER), INVOCATION_GET_FILE_STATUS( StoreStatisticNames.OP_GET_FILE_STATUS, "Calls of getFileStatus()", TYPE_COUNTER), + INVOCATION_GET_FILE_CHECKSUM( + StoreStatisticNames.OP_GET_FILE_CHECKSUM, "Calls of getFileChecksum()", TYPE_COUNTER), INVOCATION_GLOB_STATUS(StoreStatisticNames.OP_GLOB_STATUS, "Calls of globStatus()", TYPE_COUNTER), INVOCATION_HFLUSH(StoreStatisticNames.OP_HFLUSH, "Calls of hflush()", TYPE_DURATION), INVOCATION_HSYNC(StoreStatisticNames.OP_HSYNC, "Calls of hsync()", TYPE_DURATION), @@ -102,6 +96,12 @@ public enum GhfsStatistic { INVOCATION_MKDIRS(StoreStatisticNames.OP_MKDIRS, "Calls of mkdirs()", TYPE_COUNTER), INVOCATION_OPEN(StoreStatisticNames.OP_OPEN, "Calls of open()", TYPE_DURATION), INVOCATION_RENAME(StoreStatisticNames.OP_RENAME, "Calls of rename()", TYPE_DURATION), + INVOCATION_COPY_FROM_LOCAL_FILE( + StoreStatisticNames.OP_COPY_FROM_LOCAL_FILE, "Calls of copyFromLocalFile()", TYPE_COUNTER), + INVOCATION_CREATE_NON_RECURSIVE( + StoreStatisticNames.OP_CREATE_NON_RECURSIVE, "Calls of createNonRecursive()", TYPE_DURATION), + INVOCATION_GET_DELEGATION_TOKEN( + StoreStatisticNames.OP_GET_DELEGATION_TOKEN, "Calls of getDelegationToken()", TYPE_COUNTER), INVOCATION_LIST_LOCATED_STATUS( StoreStatisticNames.OP_LIST_LOCATED_STATUS, "Calls of listLocatedStatus()", TYPE_COUNTER), @@ -120,6 +120,7 @@ public enum GhfsStatistic { TYPE_COUNTER), STREAM_READ_OPERATIONS( StreamStatisticNames.STREAM_READ_OPERATIONS, "Calls of read()", TYPE_DURATION), + STREAM_READ_OPERATIONS_INCOMPLETE( StreamStatisticNames.STREAM_READ_OPERATIONS_INCOMPLETE, "Count of incomplete read() operations in an input stream", diff --git a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStorageStatistics.java b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStorageStatistics.java index 91ced4855d..9862f133c7 100644 --- a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStorageStatistics.java +++ b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStorageStatistics.java @@ -26,7 +26,7 @@ public class GhfsStorageStatistics extends StorageStatisticsFromIOStatistics { /** {@value} The key that stores all the registered metrics */ - public static final String NAME = "GhfsStorageStatistics"; + public static final String NAME = "GhfsFileSystemBasedStorageStatistics"; /** Exention for minimum */ private static final String MINIMUM = StoreStatisticNames.SUFFIX_MIN; diff --git a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStreamStats.java b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStreamStats.java new file mode 100644 index 0000000000..745d5b5674 --- /dev/null +++ b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStreamStats.java @@ -0,0 +1,85 @@ +/* + * Copyright 2023 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.hadoop.fs.gcs; + +import javax.annotation.Nonnull; + +class GhfsStreamStats { + private final GhfsGlobalStorageStatistics storageStatistics; + private final GhfsStatistic durationStat; + private final Object context; + private long maxLatencyNs; + private int operationCount; + private long minLatencyNs; + private long totalNs; + + GhfsStreamStats( + @Nonnull GhfsGlobalStorageStatistics storageStatistics, + GhfsStatistic durationStat, + Object context) { + this.storageStatistics = storageStatistics; + this.durationStat = durationStat; + this.context = context; + } + + void close() { + if (operationCount == 0) { + return; + } + + storageStatistics.updateStats( + durationStat, + toMillis(minLatencyNs), + toMillis(maxLatencyNs), + toMillis(totalNs), + operationCount, + context); + this.totalNs = 0; + this.operationCount = 0; + } + + void updateWriteStreamStats(int len, long start) { + updateStats(start); + storageStatistics.streamWriteBytes(len); + } + + void updateReadStreamSeekStats(long start) { + updateStats(start); + } + + void updateReadStreamStats(int len, long start) { + updateStats(start); + storageStatistics.streamReadBytes(len); + } + + private static long toMillis(long nano) { + return nano / 1000_000; + } + + private void updateStats(long start) { + long latency = System.nanoTime() - start; + this.maxLatencyNs = Math.max(latency, this.maxLatencyNs); + if (operationCount == 0) { + this.minLatencyNs = latency; + } else { + this.minLatencyNs = Math.min(latency, this.minLatencyNs); + } + + this.totalNs += latency; + this.operationCount++; + } +} diff --git a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFSInputStream.java b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFSInputStream.java index 02c6486879..485122f512 100644 --- a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFSInputStream.java +++ b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFSInputStream.java @@ -16,7 +16,6 @@ package com.google.cloud.hadoop.fs.gcs; -import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_READ_CLOSE_OPERATIONS; import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_READ_OPERATIONS; import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_READ_SEEK_OPERATIONS; import static com.google.common.base.Preconditions.checkNotNull; @@ -26,17 +25,22 @@ import com.google.cloud.hadoop.gcsio.FileInfo; import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystem; +import com.google.cloud.hadoop.util.GoogleCloudStorageEventBus; +import com.google.cloud.hadoop.util.ITraceFactory; import com.google.common.flogger.GoogleLogger; import java.io.IOException; import java.net.URI; import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; import java.nio.channels.SeekableByteChannel; import javax.annotation.Nonnull; import org.apache.hadoop.fs.FSExceptionMessages; import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.statistics.DurationTrackerFactory; import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.fs.statistics.IOStatisticsSource; +import org.apache.hadoop.util.functional.CallableRaisingIOE; class GoogleHadoopFSInputStream extends FSInputStream implements IOStatisticsSource { @@ -58,9 +62,17 @@ class GoogleHadoopFSInputStream extends FSInputStream implements IOStatisticsSou */ private volatile boolean closed; + private final ITraceFactory traceFactory; + // Statistics tracker provided by the parent GoogleHadoopFileSystem for recording // numbers of bytes read. private final FileSystem.Statistics statistics; + // Statistic tracker of the Input stream + private final GhfsGlobalStorageStatistics storageStatistics; + + private final GhfsStreamStats streamStats; + private final GhfsStreamStats seekStreamStats; + // Statistic tracker of the Input stream private final GhfsInputStreamStatistics streamStatistics; @@ -93,7 +105,16 @@ private GoogleHadoopFSInputStream( this.gcsPath = gcsPath; this.channel = channel; this.statistics = statistics; + this.storageStatistics = ghfs.getGlobalGcsStorageStatistics(); + this.streamStatistics = ghfs.getInstrumentation().newInputStreamStatistics(statistics); + + this.streamStats = + new GhfsStreamStats(storageStatistics, GhfsStatistic.STREAM_READ_OPERATIONS, gcsPath); + this.seekStreamStats = + new GhfsStreamStats(storageStatistics, GhfsStatistic.STREAM_READ_SEEK_OPERATIONS, gcsPath); + + this.traceFactory = ghfs.getTraceFactory(); } @Override @@ -111,10 +132,12 @@ public synchronized int read() throws IOException { @Override public synchronized int read(@Nonnull byte[] buf, int offset, int length) throws IOException { + return trackDuration( streamStatistics, STREAM_READ_OPERATIONS.getSymbol(), () -> { + long startTimeNs = System.nanoTime(); checkNotClosed(); // streamStatistics.readOperationStarted(getPos(), length); checkNotNull(buf, "buf must not be null"); @@ -131,7 +154,10 @@ public synchronized int read(@Nonnull byte[] buf, int offset, int length) throws totalBytesRead += numRead; statistics.incrementBytesRead(numRead); statistics.incrementReadOps(1); + streamStats.updateReadStreamStats(numRead, startTimeNs); } + + storageStatistics.streamReadOperationInComplete(length, Math.max(numRead, 0)); response = numRead; } catch (IOException e) { streamStatistics.readException(); @@ -144,24 +170,31 @@ public synchronized int read(@Nonnull byte[] buf, int offset, int length) throws @Override public synchronized void seek(long pos) throws IOException { + trackDuration( streamStatistics, STREAM_READ_SEEK_OPERATIONS.getSymbol(), () -> { + long startTimeNs = System.nanoTime(); checkNotClosed(); logger.atFiner().log("seek(%d)", pos); long curPos = getPos(); long diff = pos - curPos; if (diff > 0) { streamStatistics.seekForwards(diff); + storageStatistics.streamReadSeekForward(diff); } else { streamStatistics.seekBackwards(diff); + storageStatistics.streamReadSeekBackward(diff); } try { channel.position(pos); } catch (IllegalArgumentException e) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException(e); } + + seekStreamStats.updateReadStreamSeekStats(startTimeNs); return null; }); } @@ -169,17 +202,25 @@ public synchronized void seek(long pos) throws IOException { @Override public synchronized void close() throws IOException { boolean isClosed = closed; - trackDuration( + trackDurationWithTracing( streamStatistics, - STREAM_READ_CLOSE_OPERATIONS.getSymbol(), + storageStatistics, + GhfsStatistic.STREAM_READ_CLOSE_OPERATIONS, + gcsPath, + traceFactory, () -> { if (!closed) { closed = true; - logger.atFiner().log("close(): %s", gcsPath); - if (channel != null) { - logger.atFiner().log( - "Closing '%s' file with %d total bytes read", gcsPath, totalBytesRead); - channel.close(); + try { + logger.atFiner().log("close(): %s", gcsPath); + if (channel != null) { + logger.atFiner().log( + "Closing '%s' file with %d total bytes read", gcsPath, totalBytesRead); + channel.close(); + } + } finally { + streamStats.close(); + seekStreamStats.close(); } } return null; @@ -190,6 +231,23 @@ public synchronized void close() throws IOException { } } + /** + * Tracks the duration of the operation {@code operation}. Also setup operation tracking using + * {@code ThreadTrace}. + */ + private B trackDurationWithTracing( + DurationTrackerFactory factory1, + @Nonnull GhfsGlobalStorageStatistics stats, + GhfsStatistic statistic, + Object context, + ITraceFactory traceFactory, + CallableRaisingIOE operation) + throws IOException { + + return GhfsGlobalStorageStatistics.trackDuration( + factory1, stats, statistic, context, traceFactory, operation); + } + /** * Gets the current position within the file being read. * @@ -217,9 +275,11 @@ public boolean seekToNewSource(long targetPos) { @Override public int available() throws IOException { - logger.atFiner().log("available()"); - checkNotClosed(); - return 0; + if (!channel.isOpen()) { + GoogleCloudStorageEventBus.postOnException(); + throw new ClosedChannelException(); + } + return super.available(); } /** @@ -240,6 +300,7 @@ public IOStatistics getIOStatistics() { */ private void checkNotClosed() throws IOException { if (closed) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException(gcsPath + ": " + FSExceptionMessages.STREAM_IS_CLOSED); } } diff --git a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystem.java b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystem.java index 28323a332f..48496cc8a7 100644 --- a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystem.java +++ b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystem.java @@ -34,7 +34,6 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.flogger.LazyArgs.lazy; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.trackDuration; import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.hadoop.fs.gcs.auth.GcsDelegationTokens; @@ -52,10 +51,10 @@ import com.google.cloud.hadoop.util.AccessTokenProvider; import com.google.cloud.hadoop.util.AccessTokenProvider.AccessTokenType; import com.google.cloud.hadoop.util.ApiErrorExtractor; +import com.google.cloud.hadoop.util.GoogleCloudStorageEventBus; import com.google.cloud.hadoop.util.HadoopCredentialsConfiguration; import com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.AccessTokenProviderCredentials; import com.google.cloud.hadoop.util.ITraceFactory; -import com.google.cloud.hadoop.util.ITraceOperation; import com.google.cloud.hadoop.util.PropertyUtil; import com.google.cloud.hadoop.util.TraceFactory; import com.google.common.annotations.VisibleForTesting; @@ -93,6 +92,7 @@ import java.util.concurrent.ThreadFactory; import java.util.function.Supplier; import java.util.stream.Collectors; +import javax.annotation.Nonnull; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonPathCapabilities; import org.apache.hadoop.fs.ContentSummary; @@ -110,6 +110,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.StorageStatistics; import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.impl.AbstractFSBuilderImpl; import org.apache.hadoop.fs.impl.OpenFileParameters; @@ -199,6 +200,8 @@ public class GoogleHadoopFileSystem extends FileSystem implements IOStatisticsSo /** Instrumentation to track Statistics */ private GhfsInstrumentation instrumentation; /** Storage Statistics Bonded to the instrumentation. */ + private GhfsGlobalStorageStatistics globalStorageStatistics; + private GhfsStorageStatistics storageStatistics; // Thread-pool used for background tasks. private ExecutorService backgroundTasksThreadPool = @@ -218,11 +221,32 @@ public class GoogleHadoopFileSystem extends FileSystem implements IOStatisticsSo private ITraceFactory traceFactory = TraceFactory.get(/* isEnabled */ false); + /** Instrumentation to track Statistics */ + ITraceFactory getTraceFactory() { + return this.traceFactory; + } + /** * Constructs an instance of GoogleHadoopFileSystem; the internal GoogleCloudStorageFileSystem * will be set up with config settings when initialize() is called. */ - public GoogleHadoopFileSystem() {} + public GoogleHadoopFileSystem() { + StorageStatistics globalStats = + GlobalStorageStatistics.INSTANCE.put( + GhfsGlobalStorageStatistics.NAME, () -> new GhfsGlobalStorageStatistics()); + + if (GhfsGlobalStorageStatistics.class.isAssignableFrom(globalStats.getClass())) { + globalStorageStatistics = (GhfsGlobalStorageStatistics) globalStats; + } else { + logger.atWarning().log( + "Encountered an error while registering to GlobalStorageStatistics. Some of the GCS connector metrics will not be reported to metrics sinks. globalStatsClassLoader=<%s>; classLoader=<%s>", + globalStats.getClass().getClassLoader(), + GhfsGlobalStorageStatistics.class.getClassLoader()); + globalStorageStatistics = GhfsGlobalStorageStatistics.DUMMY_INSTANCE; + } + + GoogleCloudStorageEventBus.register(globalStorageStatistics); + } /** * Constructs an instance of GoogleHadoopFileSystem using the provided @@ -230,6 +254,7 @@ public GoogleHadoopFileSystem() {} */ @VisibleForTesting GoogleHadoopFileSystem(GoogleCloudStorageFileSystem gcsfs) { + this(); checkNotNull(gcsfs, "gcsFs must not be null"); initializeGcsFs(gcsfs); } @@ -263,11 +288,13 @@ public void initialize(URI path, Configuration config) throws IOException { initializeWorkingDirectory(config); initializeDelegationTokenSupport(config); instrumentation = new GhfsInstrumentation(initUri); + storageStatistics = (GhfsStorageStatistics) GlobalStorageStatistics.INSTANCE.put( GhfsStorageStatistics.NAME, () -> new GhfsStorageStatistics(instrumentation.getIOStatistics())); + initializeGcsFs(config); this.traceFactory = TraceFactory.get(GCS_TRACE_LOG_ENABLE.get(config, config::getBoolean)); @@ -329,6 +356,7 @@ private synchronized void initializeGcsFs(Configuration config) throws IOExcepti gcsFsInitialized = true; return gcsFs; } catch (IOException e) { + GoogleCloudStorageEventBus.postOnException(); throw new RuntimeException("Failed to create GCS FS", e); } }); @@ -358,10 +386,9 @@ private GoogleCloudStorageFileSystem createGcsFs(Configuration config) throws IO ? new GoogleCloudStorageFileSystemImpl( /* credentials= */ null, accessBoundaries -> accessTokenProvider.getAccessToken(accessBoundaries).getToken(), - gcsFsOptions, - getInstrumentation()) + gcsFsOptions) : new GoogleCloudStorageFileSystemImpl( - credentials, /* downscopedAccessTokenFn= */ null, gcsFsOptions, getInstrumentation()); + credentials, /* downscopedAccessTokenFn= */ null, gcsFsOptions); } private GoogleCredentials getCredentials(Configuration config) throws IOException { @@ -413,6 +440,7 @@ private static FileChecksum getFileChecksum(GcsFileChecksumType type, FileInfo f case MD5: return new GcsFileChecksum(type, fileInfo.getMd5Checksum()); } + GoogleCloudStorageEventBus.postOnException(); throw new IOException("Unrecognized GcsFileChecksumType: " + type); } @@ -424,6 +452,7 @@ protected void checkPath(Path path) { String scheme = uri.getScheme(); if (scheme != null && !scheme.equalsIgnoreCase(getScheme())) { + GoogleCloudStorageEventBus.postOnException(); throw new IllegalArgumentException( String.format( "Wrong scheme: %s, in path: %s, expected scheme: %s", scheme, path, getScheme())); @@ -437,6 +466,7 @@ protected void checkPath(Path path) { return; } + GoogleCloudStorageEventBus.postOnException(); throw new IllegalArgumentException( String.format( "Wrong bucket: %s, in path: %s, expected bucket: %s", bucket, path, rootBucket)); @@ -507,8 +537,10 @@ public String getScheme() { public FSDataInputStream open(Path hadoopPath, int bufferSize) throws IOException { return trackDurationWithTracing( instrumentation, - GhfsStatistic.INVOCATION_OPEN.getSymbol(), + globalStorageStatistics, + GhfsStatistic.INVOCATION_OPEN, hadoopPath, + this.traceFactory, () -> { checkArgument(hadoopPath != null, "hadoopPath must not be null"); checkOpen(); @@ -531,8 +563,10 @@ public FSDataOutputStream create( throws IOException { return trackDurationWithTracing( instrumentation, - GhfsStatistic.INVOCATION_CREATE.getSymbol(), + globalStorageStatistics, + GhfsStatistic.INVOCATION_CREATE, hadoopPath, + traceFactory, () -> { checkArgument(hadoopPath != null, "hadoopPath must not be null"); checkArgument(replication > 0, "replication must be a positive integer: %s", replication); @@ -559,6 +593,7 @@ public FSDataOutputStream create( .build(), statistics), statistics); + globalStorageStatistics.filesCreated(); instrumentation.fileCreated(); return response; }); @@ -576,8 +611,10 @@ public FSDataOutputStream createNonRecursive( throws IOException { return trackDurationWithTracing( instrumentation, - GhfsStatistic.INVOCATION_CREATE_NON_RECURSIVE.getSymbol(), + globalStorageStatistics, + GhfsStatistic.INVOCATION_CREATE_NON_RECURSIVE, hadoopPath, + traceFactory, () -> { // incrementStatistic(GhfsStatistic.INVOCATION_CREATE_NON_RECURSIVE); @@ -585,6 +622,7 @@ public FSDataOutputStream createNonRecursive( URI gcsPath = getGcsPath(checkNotNull(hadoopPath, "hadoopPath must not be null")); URI parentGcsPath = UriPaths.getParentPath(gcsPath); if (!getGcsFs().getFileInfo(parentGcsPath).exists()) { + GoogleCloudStorageEventBus.postOnException(); throw new FileNotFoundException( String.format( "Can not create '%s' file, because parent folder does not exist: %s", @@ -605,8 +643,10 @@ public FSDataOutputStream createNonRecursive( public boolean rename(Path src, Path dst) throws IOException { return trackDurationWithTracing( instrumentation, - GhfsStatistic.INVOCATION_RENAME.getSymbol(), - String.format("%s->%s", src, dst), + globalStorageStatistics, + GhfsStatistic.INVOCATION_RENAME, + String.format("rename(%s -> %s)", src, dst), + this.traceFactory, () -> { checkArgument(src != null, "src must not be null"); checkArgument(dst != null, "dst must not be null"); @@ -623,6 +663,7 @@ public boolean rename(Path src, Path dst) throws IOException { try { renameInternal(src, dst); } catch (IOException e) { + GoogleCloudStorageEventBus.postOnException(); if (ApiErrorExtractor.INSTANCE.requestFailure(e)) { throw e; } @@ -639,21 +680,25 @@ public boolean rename(Path src, Path dst) throws IOException { */ private B trackDurationWithTracing( DurationTrackerFactory factory, - String statistic, + @Nonnull GhfsGlobalStorageStatistics stats, + GhfsStatistic statistic, Object context, + ITraceFactory traceFactory, CallableRaisingIOE operation) throws IOException { - try (ITraceOperation op = traceFactory.createRootWithLogging(statistic, context)) { - return trackDuration(factory, statistic, operation); - } + + return GhfsGlobalStorageStatistics.trackDuration( + factory, stats, statistic, context, traceFactory, operation); } @Override public boolean delete(Path hadoopPath, boolean recursive) throws IOException { return trackDurationWithTracing( instrumentation, - GhfsStatistic.INVOCATION_DELETE.getSymbol(), + globalStorageStatistics, + GhfsStatistic.INVOCATION_DELETE, hadoopPath, + traceFactory, () -> { boolean response; try { @@ -668,6 +713,7 @@ public boolean delete(Path hadoopPath, boolean recursive) throws IOException { } catch (DirectoryNotEmptyException e) { throw e; } catch (IOException e) { + GoogleCloudStorageEventBus.postOnException(); if (ApiErrorExtractor.INSTANCE.requestFailure(e)) { throw e; } @@ -680,8 +726,10 @@ public boolean delete(Path hadoopPath, boolean recursive) throws IOException { "delete(hadoopPath: %s, recursive: %b): true", hadoopPath, recursive); } response = result; + globalStorageStatistics.fileDeleted(1); instrumentation.fileDeleted(1); } catch (IOException e) { + GoogleCloudStorageEventBus.postOnException(); incrementStatistic(GhfsStatistic.FILES_DELETE_REJECTED); throw e; } @@ -710,6 +758,7 @@ public FileStatus[] listStatus(Path hadoopPath) throws IOException { status.add(getGoogleHadoopFileStatus(fileInfo, userName)); } } catch (FileNotFoundException fnfe) { + GoogleCloudStorageEventBus.postOnException(); throw (FileNotFoundException) new FileNotFoundException( String.format( @@ -722,6 +771,7 @@ public FileStatus[] listStatus(Path hadoopPath) throws IOException { @Override public boolean mkdirs(Path hadoopPath, FsPermission permission) throws IOException { incrementStatistic(GhfsStatistic.INVOCATION_MKDIRS); + checkArgument(hadoopPath != null, "hadoopPath must not be null"); checkOpen(); @@ -730,6 +780,7 @@ public boolean mkdirs(Path hadoopPath, FsPermission permission) throws IOExcepti try { getGcsFs().mkdirs(gcsPath); } catch (java.nio.file.FileAlreadyExistsException faee) { + GoogleCloudStorageEventBus.postOnException(); // Need to convert to the Hadoop flavor of FileAlreadyExistsException. throw (FileAlreadyExistsException) new FileAlreadyExistsException( @@ -740,6 +791,7 @@ public boolean mkdirs(Path hadoopPath, FsPermission permission) throws IOExcepti logger.atFiner().log("mkdirs(hadoopPath: %s, permission: %s): true", hadoopPath, permission); boolean response = true; instrumentation.directoryCreated(); + globalStorageStatistics.directoryCreated(); return response; } @@ -753,6 +805,7 @@ public FileStatus getFileStatus(Path hadoopPath) throws IOException { URI gcsPath = getGcsPath(hadoopPath); FileInfo fileInfo = getGcsFs().getFileInfo(gcsPath); if (!fileInfo.exists()) { + GoogleCloudStorageEventBus.postOnException(); throw new FileNotFoundException( String.format( "%s not found: %s", fileInfo.isDirectory() ? "Directory" : "File", hadoopPath)); @@ -815,6 +868,7 @@ public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path[] srcs, Pa public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, Path dst) throws IOException { incrementStatistic(GhfsStatistic.INVOCATION_COPY_FROM_LOCAL_FILE); + logger.atFiner().log( "copyFromLocalFile(delSrc: %b, overwrite: %b, src: %s, dst: %s)", delSrc, overwrite, src, dst); @@ -857,8 +911,10 @@ public RemoteIterator listLocatedStatus(Path f) throws IOExce public byte[] getXAttr(Path path, String name) throws IOException { return trackDurationWithTracing( instrumentation, - GhfsStatistic.INVOCATION_XATTR_GET_NAMED.getSymbol(), - String.format("%s:%s", path, name), + globalStorageStatistics, + GhfsStatistic.INVOCATION_XATTR_GET_NAMED, + path, + traceFactory, () -> { checkNotNull(path, "path should not be null"); checkNotNull(name, "name should not be null"); @@ -880,8 +936,10 @@ public byte[] getXAttr(Path path, String name) throws IOException { public Map getXAttrs(Path path) throws IOException { return trackDurationWithTracing( instrumentation, - GhfsStatistic.INVOCATION_XATTR_GET_MAP.getSymbol(), + globalStorageStatistics, + GhfsStatistic.INVOCATION_XATTR_GET_MAP, path, + traceFactory, () -> { checkNotNull(path, "path should not be null"); @@ -903,8 +961,10 @@ public Map getXAttrs(Path path) throws IOException { public Map getXAttrs(Path path, List names) throws IOException { return trackDurationWithTracing( instrumentation, - GhfsStatistic.INVOCATION_XATTR_GET_NAMED_MAP.getSymbol(), - String.format("%s:%s", path, names == null ? -1 : names.size()), + globalStorageStatistics, + GhfsStatistic.INVOCATION_XATTR_GET_NAMED_MAP, + path, + traceFactory, () -> { checkNotNull(path, "path should not be null"); checkNotNull(names, "names should not be null"); @@ -930,8 +990,10 @@ public Map getXAttrs(Path path, List names) throws IOExc public List listXAttrs(Path path) throws IOException { return trackDurationWithTracing( instrumentation, - GhfsStatistic.INVOCATION_OP_XATTR_LIST.getSymbol(), + globalStorageStatistics, + GhfsStatistic.INVOCATION_OP_XATTR_LIST, path, + traceFactory, () -> { checkNotNull(path, "path should not be null"); @@ -952,6 +1014,7 @@ public List listXAttrs(Path path) throws IOException { */ private void incrementStatistic(GhfsStatistic statistic) { incrementStatistic(statistic, 1); + globalStorageStatistics.incrementCounter(statistic, 1); } /** @@ -967,6 +1030,15 @@ private void incrementStatistic(GhfsStatistic statistic, long count) { instrumentation.incrementCounter(statistic, count); } + /** + * Get the storage statistics of this filesystem. + * + * @return the storage statistics + */ + public GhfsGlobalStorageStatistics getGlobalGcsStorageStatistics() { + return globalStorageStatistics; + } + /** * Get the storage statistics of this filesystem. * @@ -1323,9 +1395,11 @@ private FileStatus[] concurrentGlobInternal(Path fixedPath, PathFilter filter) () -> flatGlobInternal(fixedPath, filter), () -> super.globStatus(fixedPath, filter))); } catch (InterruptedException e) { + GoogleCloudStorageEventBus.postOnException(); Thread.currentThread().interrupt(); throw new IOException(String.format("Concurrent glob execution failed: %s", e), e); } catch (ExecutionException e) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException(String.format("Concurrent glob execution failed: %s", e.getCause()), e); } } @@ -1472,6 +1546,7 @@ private AccessTokenProvider getDelegationAccessTokenProvider(Configuration confi /** Assert that the FileSystem has been initialized and not close()d. */ private void checkOpen() throws IOException { if (isClosed()) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException("GoogleHadoopFileSystem has been closed or not initialized."); } } @@ -1611,12 +1686,14 @@ public void setXAttr(Path path, String name, byte[] value, EnumSet Map attributes = fileInfo.getAttributes(); if (attributes.containsKey(xAttrKey) && !flags.contains(XAttrSetFlag.REPLACE)) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException( String.format( "REPLACE flag must be set to update XAttr (name='%s', value='%s') for '%s'", name, new String(value, UTF_8), path)); } if (!attributes.containsKey(xAttrKey) && !flags.contains(XAttrSetFlag.CREATE)) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException( String.format( "CREATE flag must be set to create XAttr (name='%s', value='%s') for '%s'", diff --git a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopOutputStream.java b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopOutputStream.java index 25adfb705c..fa45c1cfcc 100644 --- a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopOutputStream.java +++ b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopOutputStream.java @@ -28,6 +28,8 @@ import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystemImpl; import com.google.cloud.hadoop.gcsio.GoogleCloudStorageItemInfo; import com.google.cloud.hadoop.gcsio.StorageResourceId; +import com.google.cloud.hadoop.util.GoogleCloudStorageEventBus; +import com.google.cloud.hadoop.util.ITraceFactory; import com.google.common.base.Ascii; import com.google.common.collect.ImmutableList; import com.google.common.flogger.GoogleLogger; @@ -55,14 +57,20 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.StreamCapabilities; import org.apache.hadoop.fs.Syncable; +import org.apache.hadoop.fs.statistics.DurationTrackerFactory; import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.fs.statistics.IOStatisticsSource; +import org.apache.hadoop.util.functional.CallableRaisingIOE; class GoogleHadoopOutputStream extends OutputStream implements IOStatisticsSource, StreamCapabilities, Syncable { private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + private final GhfsGlobalStorageStatistics storageStatistics; + + private final ITraceFactory traceFactory; + // Prefix used for all temporary files created by this stream. public static final String TMP_FILE_PREFIX = "_GHFS_SYNC_TMP_FILE_"; @@ -123,6 +131,8 @@ class GoogleHadoopOutputStream extends OutputStream // Instrumentation to track Statistics private final GhfsInstrumentation instrumentation; + private final GhfsStreamStats streamStats; + /** * Constructs an instance of GoogleHadoopOutputStream object. * @@ -144,8 +154,11 @@ public GoogleHadoopOutputStream( this.ghfs = ghfs; this.dstGcsPath = dstGcsPath; this.statistics = statistics; + this.storageStatistics = ghfs.getGlobalGcsStorageStatistics(); this.streamStatistics = ghfs.getInstrumentation().newOutputStreamStatistics(statistics); + this.streamStats = + new GhfsStreamStats(storageStatistics, GhfsStatistic.STREAM_WRITE_OPERATIONS, dstGcsPath); Duration minSyncInterval = createFileOptions.getMinSyncInterval(); this.instrumentation = ghfs.getInstrumentation(); @@ -179,6 +192,7 @@ public GoogleHadoopOutputStream( tmpGcsPath, tmpIndex == 0 ? createFileOptions : TMP_FILE_CREATE_OPTIONS); this.dstGenerationId = StorageResourceId.UNKNOWN_GENERATION_ID; + this.traceFactory = ghfs.getTraceFactory(); } private static OutputStream createOutputStream( @@ -188,6 +202,7 @@ private static OutputStream createOutputStream( try { channel = gcsfs.create(gcsPath, options); } catch (java.nio.file.FileAlreadyExistsException e) { + GoogleCloudStorageEventBus.postOnException(); throw (FileAlreadyExistsException) new FileAlreadyExistsException(String.format("'%s' already exists", gcsPath)) .initCause(e); @@ -204,26 +219,50 @@ public void write(int b) throws IOException { streamStatistics, GhfsStatistic.STREAM_WRITE_OPERATIONS.getSymbol(), () -> { + long start = System.nanoTime(); throwIfNotOpen(); tmpOut.write(b); streamStatistics.writeBytes(1); + // Using a lightweight implementation to update instrumentation. This method can be called + // quite + // frequently and need to be lightweight. statistics.incrementBytesWritten(1); statistics.incrementWriteOps(1); + streamStats.updateWriteStreamStats(1, start); return null; }); } + /** + * Tracks the duration of the operation {@code operation}. Also setup operation tracking using + * {@code ThreadTrace}. + */ + private B trackDurationWithTracing( + DurationTrackerFactory factory, + @Nonnull GhfsGlobalStorageStatistics stats, + GhfsStatistic statistic, + Object context, + ITraceFactory traceFactory, + CallableRaisingIOE operation) + throws IOException { + + return GhfsGlobalStorageStatistics.trackDuration( + factory, stats, statistic, context, traceFactory, operation); + } + @Override public void write(@Nonnull byte[] b, int offset, int len) throws IOException { trackDuration( streamStatistics, GhfsStatistic.STREAM_WRITE_OPERATIONS.getSymbol(), () -> { + long start = System.nanoTime(); throwIfNotOpen(); tmpOut.write(b, offset, len); - streamStatistics.writeBytes(len); statistics.incrementBytesWritten(len); statistics.incrementWriteOps(1); + streamStats.updateWriteStreamStats(len, start); + streamStatistics.writeBytes(len); return null; }); } @@ -238,9 +277,12 @@ public void write(@Nonnull byte[] b, int offset, int len) throws IOException { */ @Override public void hflush() throws IOException { - trackDuration( + trackDurationWithTracing( streamStatistics, - GhfsStatistic.INVOCATION_HFLUSH.getSymbol(), + storageStatistics, + GhfsStatistic.INVOCATION_HFLUSH, + dstGcsPath, + traceFactory, () -> { logger.atFiner().log("hflush(): %s", dstGcsPath); @@ -262,9 +304,12 @@ public void hflush() throws IOException { @Override public void hsync() throws IOException { - trackDuration( + trackDurationWithTracing( streamStatistics, - GhfsStatistic.INVOCATION_HSYNC.getSymbol(), + storageStatistics, + GhfsStatistic.INVOCATION_HSYNC, + dstGcsPath, + traceFactory, () -> { logger.atFiner().log("hsync(): %s", dstGcsPath); @@ -358,9 +403,12 @@ private URI getNextTmpPath() { @Override public void close() throws IOException { boolean isClosed = tmpOut == null; - trackDuration( + trackDurationWithTracing( streamStatistics, - GhfsStatistic.STREAM_WRITE_CLOSE_OPERATIONS.getSymbol(), + storageStatistics, + GhfsStatistic.STREAM_WRITE_CLOSE_OPERATIONS, + dstGcsPath, + traceFactory, () -> { logger.atFiner().log( "close(): temp tail file: %s final destination: %s", tmpGcsPath, dstGcsPath); @@ -385,6 +433,7 @@ public void close() throws IOException { try { deletion.get(); } catch (ExecutionException | InterruptedException e) { + GoogleCloudStorageEventBus.postOnException(); if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } @@ -398,6 +447,7 @@ public void close() throws IOException { }); if (!isClosed) { + streamStats.close(); streamStatistics.close(); } // TODO: do we need make a `cleanerThreadpool` an object field and close it here? diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFSInputStreamIntegrationTest.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFSInputStreamIntegrationTest.java index 0c6426d508..40b3c6b5da 100644 --- a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFSInputStreamIntegrationTest.java +++ b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFSInputStreamIntegrationTest.java @@ -16,8 +16,14 @@ package com.google.cloud.hadoop.fs.gcs; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_READ_BYTES; import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_READ_CLOSE_OPERATIONS; import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_READ_OPERATIONS; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_READ_OPERATIONS_INCOMPLETE; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_READ_SEEK_BACKWARD_OPERATIONS; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_READ_SEEK_BYTES_BACKWARDS; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_READ_SEEK_BYTES_SKIPPED; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_READ_SEEK_FORWARD_OPERATIONS; import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_READ_SEEK_OPERATIONS; import static com.google.common.truth.Truth.assertThat; import static org.apache.hadoop.fs.statistics.StoreStatisticNames.SUFFIX_FAILURES; @@ -131,7 +137,8 @@ public void testReadAfterClosed() throws Exception { } @Test - public void operation_durationMetric_tests() throws Exception { + public void fs_operation_durationMetric_tests() throws Exception { + URI path = gcsFsIHelper.getUniqueObjectUri(getClass(), "seek_illegalArgument"); GoogleHadoopFileSystem ghfs = @@ -175,6 +182,54 @@ public void operation_durationMetric_tests() throws Exception { ghfs.getInstrumentation().getIOStatistics(), STREAM_READ_OPERATIONS.getSymbol(), 2); } + @Test + public void operation_durationMetric_tests() throws Exception { + URI path = gcsFsIHelper.getUniqueObjectUri(getClass(), "seek_illegalArgument"); + + GoogleHadoopFileSystem ghfs = + GoogleHadoopFileSystemIntegrationHelper.createGhfs( + path, GoogleHadoopFileSystemIntegrationHelper.getTestConfig()); + GhfsGlobalStorageStatistics stats = TestUtils.getStorageStatistics(); + + String testContent = "test content"; + gcsFsIHelper.writeTextFile(path, testContent); + + byte[] value = new byte[2]; + byte[] expected = Arrays.copyOf(testContent.getBytes(StandardCharsets.UTF_8), 2); + + GoogleHadoopFSInputStream in = createGhfsInputStream(ghfs, path); + + assertThat(in.read(value, 0, 1)).isEqualTo(1); + assertThat(in.read(1, value, 1, 1)).isEqualTo(1); + assertThat(value).isEqualTo(expected); + + TestUtils.verifyCounter(stats, STREAM_READ_SEEK_OPERATIONS, 0); + TestUtils.verifyCounter(stats, STREAM_READ_SEEK_BACKWARD_OPERATIONS, 2); + TestUtils.verifyDurationMetric(stats, STREAM_READ_OPERATIONS.getSymbol(), 0); + + in.seek(0); + + TestUtils.verifyDurationMetric(stats, STREAM_READ_SEEK_OPERATIONS.getSymbol(), 0); + TestUtils.verifyDurationMetric(stats, STREAM_READ_SEEK_OPERATIONS.getSymbol(), 0); + TestUtils.verifyDurationMetric(stats, STREAM_READ_OPERATIONS.getSymbol(), 0); + TestUtils.verifyCounter(stats, STREAM_READ_SEEK_BACKWARD_OPERATIONS, 3); + TestUtils.verifyCounter(stats, STREAM_READ_SEEK_BYTES_BACKWARDS, 2); + TestUtils.verifyCounter(stats, STREAM_READ_SEEK_FORWARD_OPERATIONS, 0); + TestUtils.verifyCounter(stats, STREAM_READ_SEEK_BYTES_SKIPPED, 0); + + int expectedSeek = 5; + in.seek(expectedSeek); + in.close(); + + TestUtils.verifyCounter(stats, STREAM_READ_SEEK_FORWARD_OPERATIONS, 1); + TestUtils.verifyCounter(stats, STREAM_READ_SEEK_BYTES_SKIPPED, expectedSeek); + TestUtils.verifyCounter(stats, STREAM_READ_BYTES, 2); + TestUtils.verifyCounter(stats, STREAM_READ_OPERATIONS_INCOMPLETE, 0); + TestUtils.verifyDurationMetric(stats, STREAM_READ_CLOSE_OPERATIONS.getSymbol(), 1); + TestUtils.verifyDurationMetric(stats, STREAM_READ_SEEK_OPERATIONS.getSymbol(), 4); + TestUtils.verifyDurationMetric(stats, STREAM_READ_OPERATIONS.getSymbol(), 2); + } + private static GoogleHadoopFSInputStream createGhfsInputStream( GoogleHadoopFileSystem ghfs, URI path) throws IOException { return GoogleHadoopFSInputStream.create( diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemDelegationTokensTest.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemDelegationTokensTest.java index 5a7868b4c7..8f68de82ee 100644 --- a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemDelegationTokensTest.java +++ b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemDelegationTokensTest.java @@ -31,6 +31,7 @@ import java.net.URI; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.StorageStatistics; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; @@ -132,6 +133,7 @@ public void testTokenAuthValue() throws IOException { @Test public void testDelegationTokenStatistics() throws IOException { GoogleHadoopFileSystem fs = new GoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); fs.initialize(new Path("gs://test/").toUri(), loadConfig()); Token dt = fs.getDelegationToken("current-user"); @@ -140,6 +142,9 @@ public void testDelegationTokenStatistics() throws IOException { .isEqualTo(1); assertThat(fs.getIOStatistics().counters().get(DELEGATION_TOKENS_ISSUED.getSymbol())) .isEqualTo(1); + + TestUtils.verifyCounter( + (GhfsGlobalStorageStatistics) stats, INVOCATION_GET_DELEGATION_TOKEN, 1); fs.close(); } diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationTest.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationTest.java index e792e2eaf4..633985d5f9 100644 --- a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationTest.java +++ b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationTest.java @@ -16,7 +16,12 @@ package com.google.cloud.hadoop.fs.gcs; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.ACTION_HTTP_DELETE_REQUEST; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.ACTION_HTTP_GET_REQUEST; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.ACTION_HTTP_PATCH_REQUEST; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.ACTION_HTTP_PUT_REQUEST; import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.FILES_CREATED; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_COPY_FROM_LOCAL_FILE; import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_CREATE; import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_CREATE_NON_RECURSIVE; import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_DELETE; @@ -34,6 +39,10 @@ import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_XATTR_GET_MAP; import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_XATTR_GET_NAMED; import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.INVOCATION_XATTR_GET_NAMED_MAP; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_READ_BYTES; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_READ_OPERATIONS; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_WRITE_BYTES; +import static com.google.cloud.hadoop.fs.gcs.GhfsStatistic.STREAM_WRITE_OPERATIONS; import static com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemConfiguration.GCS_OUTPUT_STREAM_BUFFER_SIZE; import static com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemConfiguration.GCS_PROJECT_ID; import static com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemTestHelper.createInMemoryGoogleHadoopFileSystem; @@ -79,11 +88,13 @@ import java.util.UUID; import java.util.function.Function; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.StorageStatistics; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.security.UserGroupInformation; @@ -316,30 +327,98 @@ public void create_throwsExceptionWhenBlockSizeIsNotPositiveInteger() throws Exc @Test public void create_IOstatistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); + assertThat(myGhfs.getIOStatistics().counters().get(INVOCATION_CREATE.getSymbol())).isEqualTo(0); assertThat(myGhfs.getIOStatistics().counters().get(FILES_CREATED.getSymbol())).isEqualTo(0); + assertThat(getMetricValue(stats, INVOCATION_CREATE)).isEqualTo(0); + assertThat(getMetricValue(stats, FILES_CREATED)).isEqualTo(0); + try (FSDataOutputStream fout = myGhfs.create(new Path("/file1"))) { fout.writeBytes("Test Content"); } + assertThat(myGhfs.getIOStatistics().counters().get(INVOCATION_CREATE.getSymbol())).isEqualTo(1); assertThat(myGhfs.getIOStatistics().counters().get(FILES_CREATED.getSymbol())).isEqualTo(1); + + assertThat(getMetricValue(stats, INVOCATION_CREATE)).isEqualTo(1); + assertThat(getMetricValue(stats, FILES_CREATED)).isEqualTo(1); + + TestUtils.verifyDurationMetric(myGhfs.getIOStatistics(), INVOCATION_CREATE.getSymbol(), 1); + assertThat(myGhfs.delete(new Path("/file1"))).isTrue(); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_CREATE.getSymbol(), 1); + } + + @Test + public void create_IOstatistics_statistics_check() throws IOException { + GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); + + assertThat(myGhfs.getIOStatistics().counters().get(INVOCATION_CREATE.getSymbol())).isEqualTo(0); + assertThat(myGhfs.getIOStatistics().counters().get(FILES_CREATED.getSymbol())).isEqualTo(0); + + assertThat(getMetricValue(stats, INVOCATION_CREATE)).isEqualTo(0); + assertThat(getMetricValue(stats, FILES_CREATED)).isEqualTo(0); + + try (FSDataOutputStream fout = myGhfs.create(new Path("/file1"))) { + fout.writeBytes("Test Content"); + } + + assertThat(myGhfs.getIOStatistics().counters().get(INVOCATION_CREATE.getSymbol())).isEqualTo(1); + assertThat(myGhfs.getIOStatistics().counters().get(FILES_CREATED.getSymbol())).isEqualTo(1); + + assertThat(getMetricValue(stats, INVOCATION_CREATE)).isEqualTo(1); + assertThat(getMetricValue(stats, FILES_CREATED)).isEqualTo(1); TestUtils.verifyDurationMetric(myGhfs.getIOStatistics(), INVOCATION_CREATE.getSymbol(), 1); + + assertThat(myGhfs.delete(new Path("/file1"))).isTrue(); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_CREATE.getSymbol(), 1); + + myGhfs = createInMemoryGoogleHadoopFileSystem(); + + assertThat(myGhfs.getIOStatistics().counters().get(INVOCATION_CREATE.getSymbol())).isEqualTo(0); + assertThat(myGhfs.getIOStatistics().counters().get(FILES_CREATED.getSymbol())).isEqualTo(0); + + assertThat(getMetricValue(stats, INVOCATION_CREATE)).isEqualTo(1); + assertThat(getMetricValue(stats, FILES_CREATED)).isEqualTo(1); + + try (FSDataOutputStream fout = myGhfs.create(new Path("/file1"))) { + fout.writeBytes("Test Content"); + } + + assertThat(myGhfs.getIOStatistics().counters().get(INVOCATION_CREATE.getSymbol())).isEqualTo(1); + assertThat(myGhfs.getIOStatistics().counters().get(FILES_CREATED.getSymbol())).isEqualTo(1); + TestUtils.verifyDurationMetric(myGhfs.getIOStatistics(), INVOCATION_CREATE.getSymbol(), 1); + + assertThat(getMetricValue(stats, INVOCATION_CREATE)).isEqualTo(2); + assertThat(getMetricValue(stats, FILES_CREATED)).isEqualTo(2); + + assertThat(myGhfs.delete(new Path("/file1"))).isTrue(); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_CREATE.getSymbol(), 2); } @Test public void listLocatedStatus_IOStatistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); try (FSDataOutputStream fout = myGhfs.create(new Path("/file1"))) { fout.writeBytes("Test Content"); fout.close(); myGhfs.listLocatedStatus(new Path("/file1")); + assertThat( myGhfs.getIOStatistics().counters().get(INVOCATION_LIST_LOCATED_STATUS.getSymbol())) .isEqualTo(1); + + TestUtils.verifyCounter( + (GhfsGlobalStorageStatistics) stats, INVOCATION_LIST_LOCATED_STATUS, 1); } assertThat(myGhfs.delete(new Path("/file1"))).isTrue(); } @@ -390,27 +469,154 @@ public void delete_throwsExceptionWhenHadoopPathNull() throws IOException { @Test public void open_IOstatistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); + Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); FSDataOutputStream fout = myGhfs.create(new Path("/directory1/file1")); fout.writeBytes("data"); fout.close(); myGhfs.open(new Path("/directory1/file1")); + assertThat(getMetricValue(stats, INVOCATION_OPEN)).isEqualTo(1); assertThat(myGhfs.getIOStatistics().counters().get(INVOCATION_OPEN.getSymbol())).isEqualTo(1); + assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue(); TestUtils.verifyDurationMetric(myGhfs.getIOStatistics(), INVOCATION_OPEN.getSymbol(), 1); + + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_OPEN.getSymbol(), 1); } @Test public void delete_IOstatistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); + FSDataOutputStream fout = myGhfs.create(new Path("/file1")); fout.writeBytes("data"); fout.close(); myGhfs.delete(new Path("/file1")); + assertThat(getMetricValue(stats, INVOCATION_DELETE)).isEqualTo(1); TestUtils.verifyDurationMetric(myGhfs.getIOStatistics(), INVOCATION_DELETE.getSymbol(), 1); + + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_DELETE.getSymbol(), 1); + } + + @Test + public void statistics_check_read_twice() throws Exception { + GoogleHadoopFileSystem fs1 = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); + + Path testFileToIO = new Path("/test-random.bin"); + createFile(fs1, testFileToIO, 1024, 10 * 1024); + readFile(fs1, testFileToIO, 1024); + fs1.delete(testFileToIO); + + // per FS based statistics + assertThat(fs1.getIOStatistics().counters().get(STREAM_READ_BYTES.getSymbol())) + .isEqualTo(10240); + assertThat(fs1.getIOStatistics().counters().get(STREAM_READ_OPERATIONS.getSymbol())) + .isEqualTo(11); + assertThat(fs1.getIOStatistics().counters().get(GhfsStatistic.INVOCATION_DELETE.getSymbol())) + .isEqualTo(1); + + // Global Statistics + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, STREAM_READ_BYTES.getSymbol(), 10240); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, STREAM_READ_OPERATIONS.getSymbol(), 10); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_DELETE.getSymbol(), 1); + + GoogleHadoopFileSystem fs2 = createInMemoryGoogleHadoopFileSystem(); + createFile(fs2, testFileToIO, 1024, 10 * 1024); + readFile(fs2, testFileToIO, 1024); + fs2.delete(testFileToIO); + + // per FS based statistics + assertThat(fs2.getIOStatistics().counters().get(STREAM_READ_BYTES.getSymbol())) + .isEqualTo(10240); + assertThat(fs2.getIOStatistics().counters().get(STREAM_READ_OPERATIONS.getSymbol())) + .isEqualTo(11); + assertThat(fs1.getIOStatistics().counters().get(GhfsStatistic.INVOCATION_DELETE.getSymbol())) + .isEqualTo(1); + + assertThat(fs1.getIOStatistics().counters().get(STREAM_READ_BYTES.getSymbol())) + .isEqualTo(10240); + assertThat(fs1.getIOStatistics().counters().get(STREAM_READ_OPERATIONS.getSymbol())) + .isEqualTo(11); + assertThat(fs1.getIOStatistics().counters().get(GhfsStatistic.INVOCATION_DELETE.getSymbol())) + .isEqualTo(1); + + // Global Statistics + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, STREAM_READ_BYTES.getSymbol(), 20480); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, STREAM_READ_OPERATIONS.getSymbol(), 20); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_DELETE.getSymbol(), 2); + + fs1.close(); + fs2.close(); + } + + @Test + public void statistics_check_write_twice() throws Exception { + + GoogleHadoopFileSystem fs1 = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); + + Path testFileToIO = new Path("/test-random.bin"); + createFile(fs1, testFileToIO, 1024, 10 * 1024); + fs1.delete(testFileToIO); + + // per FS based statistics + assertThat(fs1.getIOStatistics().counters().get(STREAM_WRITE_BYTES.getSymbol())) + .isEqualTo(10240); + assertThat(fs1.getIOStatistics().counters().get(STREAM_WRITE_OPERATIONS.getSymbol())) + .isEqualTo(10); + assertThat(fs1.getIOStatistics().counters().get(GhfsStatistic.INVOCATION_DELETE.getSymbol())) + .isEqualTo(1); + + // Global Statistics + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, STREAM_WRITE_BYTES.getSymbol(), 10240); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, STREAM_WRITE_OPERATIONS.getSymbol(), 10); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_DELETE.getSymbol(), 1); + + GoogleHadoopFileSystem fs2 = createInMemoryGoogleHadoopFileSystem(); + createFile(fs2, testFileToIO, 1024, 10 * 1024); + fs2.delete(testFileToIO); + + // per FS based statistics + assertThat(fs2.getIOStatistics().counters().get(STREAM_WRITE_BYTES.getSymbol())) + .isEqualTo(10240); + assertThat(fs2.getIOStatistics().counters().get(STREAM_WRITE_OPERATIONS.getSymbol())) + .isEqualTo(10); + assertThat(fs2.getIOStatistics().counters().get(GhfsStatistic.INVOCATION_DELETE.getSymbol())) + .isEqualTo(1); + assertThat(fs1.getIOStatistics().counters().get(STREAM_WRITE_BYTES.getSymbol())) + .isEqualTo(10240); + assertThat(fs1.getIOStatistics().counters().get(STREAM_WRITE_OPERATIONS.getSymbol())) + .isEqualTo(10); + assertThat(fs1.getIOStatistics().counters().get(GhfsStatistic.INVOCATION_DELETE.getSymbol())) + .isEqualTo(1); + + // Global Statistics + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, STREAM_WRITE_BYTES.getSymbol(), 20480); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, STREAM_WRITE_OPERATIONS.getSymbol(), 20); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_DELETE.getSymbol(), 2); + + fs1.close(); + fs2.close(); } @Test @@ -439,8 +645,11 @@ public void mkdirs_throwsExceptionWhenHadoopPathNull() throws IOException { @Test public void mkdirs_IOstatistics() throws IOException { + StorageStatistics stats = TestUtils.getStorageStatistics(); GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); assertThat(myGhfs.getIOStatistics().counters().get(INVOCATION_MKDIRS.getSymbol())).isEqualTo(1); + + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_MKDIRS, 1); } @Test @@ -726,14 +935,21 @@ public void testConfigureBucketsWithRootBucketButNoSystemBucket() throws IOExcep @Test public void create_storage_statistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); - GhfsStorageStatistics StorageStats = new GhfsStorageStatistics(myGhfs.getIOStatistics()); + GhfsStorageStatistics FSStorageStats = new GhfsStorageStatistics(myGhfs.getIOStatistics()); + + StorageStatistics GlobalStorageStats = TestUtils.getStorageStatistics(); Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); FSDataOutputStream fout = myGhfs.create(new Path("/directory1/file1")); fout.writeBytes("Test Content"); fout.close(); - assertThat(StorageStats.isTracked("op_create")).isTrue(); - assertThat(StorageStats.getLong("op_create")).isEqualTo(1); + + assertThat(FSStorageStats.isTracked("op_create")).isTrue(); + assertThat(FSStorageStats.getLong("op_create")).isEqualTo(1); + + assertThat(GlobalStorageStats.isTracked("op_create")).isTrue(); + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) GlobalStorageStats, INVOCATION_CREATE, 1); + assertThat(FSStorageStats.getLong("op_create")).isEqualTo(1); assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue(); } @@ -741,14 +957,21 @@ public void create_storage_statistics() throws IOException { public void listLocatedStatus_storage_statistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); GhfsStorageStatistics StorageStats = new GhfsStorageStatistics(myGhfs.getIOStatistics()); + + StorageStatistics stats = TestUtils.getStorageStatistics(); Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); FSDataOutputStream fout = myGhfs.create(new Path("/directory1/file1")); fout.writeBytes("Test Content"); fout.close(); myGhfs.listLocatedStatus(testRoot); + assertThat(StorageStats.isTracked("op_list_located_status")).isTrue(); assertThat(StorageStats.getLong("op_list_located_status")).isEqualTo(1); + + assertThat(stats.isTracked("op_list_located_status")).isTrue(); + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_LIST_LOCATED_STATUS, 1); + assertThat(StorageStats.getLong("op_list_located_status")).isEqualTo(1); assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue(); } @@ -756,21 +979,30 @@ public void listLocatedStatus_storage_statistics() throws IOException { public void open_storage_statistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); GhfsStorageStatistics StorageStats = new GhfsStorageStatistics(myGhfs.getIOStatistics()); + + StorageStatistics stats = TestUtils.getStorageStatistics(); Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); FSDataOutputStream fout = myGhfs.create(new Path("/directory1/file1")); fout.writeBytes("data"); fout.close(); myGhfs.open(new Path("/directory1/file1")); + assertThat(getMetricValue(stats, INVOCATION_OPEN)).isEqualTo(1); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_OPEN.getSymbol(), 1); + assertThat(StorageStats.isTracked("op_open")).isTrue(); assertThat(StorageStats.getLong("op_open")).isEqualTo(1); + assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue(); } @Test public void copy_from_local_file_storage_statistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); GhfsStorageStatistics StorageStats = new GhfsStorageStatistics(myGhfs.getIOStatistics()); + Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); FSDataOutputStream fout = myGhfs.create(new Path("/directory1/file1")); @@ -784,13 +1016,18 @@ public void copy_from_local_file_storage_statistics() throws IOException { assertThat((StorageStats.isTracked("op_copy_from_local_file"))).isTrue(); assertThat((StorageStats.getLong("op_copy_from_local_file"))).isEqualTo(1); + + TestUtils.verifyCounter( + (GhfsGlobalStorageStatistics) stats, INVOCATION_COPY_FROM_LOCAL_FILE, 1); assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue(); } @Test public void delete_storage_statistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); GhfsStorageStatistics StorageStats = new GhfsStorageStatistics(myGhfs.getIOStatistics()); + Path filePath = new Path("/file1"); FSDataOutputStream fout = myGhfs.create(filePath); fout.writeBytes("Test Content"); @@ -798,49 +1035,67 @@ public void delete_storage_statistics() throws IOException { assertThat(myGhfs.delete(filePath)).isTrue(); assertThat(StorageStats.isTracked("op_delete")).isTrue(); assertThat(StorageStats.getLong("op_delete")).isEqualTo(1); + + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_DELETE, 1); } @Test public void mkdirs_storage_statistics() throws IOException { + StorageStatistics stats = TestUtils.getStorageStatistics(); GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); GhfsStorageStatistics StorageStats = new GhfsStorageStatistics(myGhfs.getIOStatistics()); assertThat(StorageStats.isTracked("op_mkdirs")).isTrue(); assertThat(StorageStats.getLong("op_mkdirs")).isEqualTo(1); + + assertThat(stats.isTracked("op_mkdirs")).isTrue(); + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_MKDIRS, 1); } @Test public void GlobStatus_storage_statistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); GhfsStorageStatistics StorageStats = new GhfsStorageStatistics(myGhfs.getIOStatistics()); + Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); myGhfs.mkdirs(new Path("/directory1/subdirectory1")); myGhfs.create(new Path("/directory1/subdirectory1/file1")).writeBytes("data"); myGhfs.globStatus(new Path("/d*")); + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_GLOB_STATUS, 1); + assertThat(StorageStats.isTracked("op_glob_status")).isTrue(); assertThat(StorageStats.getLong("op_glob_status")).isEqualTo(1); + assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue(); } @Test public void getFileStatus_storage_statistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); GhfsStorageStatistics StorageStats = new GhfsStorageStatistics(myGhfs.getIOStatistics()); + Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); FSDataOutputStream fout = myGhfs.create(new Path("/directory1/file1")); fout.writeBytes("data"); fout.close(); myGhfs.getFileStatus(new Path("/directory1/file1")); + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_GET_FILE_STATUS, 1); + assertThat(StorageStats.isTracked("op_get_file_status")).isTrue(); assertThat(StorageStats.getLong("op_get_file_status")).isEqualTo(1); + assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue(); } @Test public void createNonRecursive_storage_statistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); GhfsStorageStatistics StorageStats = new GhfsStorageStatistics(myGhfs.getIOStatistics()); + Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); Path filePath = new Path("/directory1/file1"); @@ -848,15 +1103,21 @@ public void createNonRecursive_storage_statistics() throws IOException { myGhfs.createNonRecursive(filePath, true, 1, (short) 1, 1, () -> {})) { createNonRecursiveOutputStream.write(1); } + TestUtils.verifyCounter( + (GhfsGlobalStorageStatistics) stats, INVOCATION_CREATE_NON_RECURSIVE, 1); + assertThat(StorageStats.isTracked("op_create_non_recursive")).isTrue(); assertThat(StorageStats.getLong("op_create_non_recursive")).isEqualTo(1); + assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue(); } @Test public void getFileChecksum_storage_statistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); GhfsStorageStatistics StorageStats = new GhfsStorageStatistics(myGhfs.getIOStatistics()); + Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); Path filePath = new Path("/directory1/file1"); @@ -864,37 +1125,50 @@ public void getFileChecksum_storage_statistics() throws IOException { fout.writeBytes("data"); fout.close(); myGhfs.getFileChecksum(filePath); + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_GET_FILE_CHECKSUM, 1); + assertThat(StorageStats.isTracked("op_get_file_checksum")).isTrue(); assertThat(StorageStats.getLong("op_get_file_checksum")).isEqualTo(1); + assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue(); } @Test public void rename_storage_statistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); GhfsStorageStatistics StorageStats = new GhfsStorageStatistics(myGhfs.getIOStatistics()); + Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); Path source = new Path("/directory1/file1"); myGhfs.create(source).writeBytes("data"); Path dest = new Path("/directory1/file2"); myGhfs.rename(source, dest); + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_RENAME, 1); + assertThat(StorageStats.isTracked("op_rename")).isTrue(); assertThat(StorageStats.getLong("op_rename")).isEqualTo(1); + assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue(); } @Test public void exists_storage_statistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); GhfsStorageStatistics StorageStats = new GhfsStorageStatistics(myGhfs.getIOStatistics()); + Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); Path filePath = new Path("/directory1/file1"); myGhfs.create(filePath).writeBytes("data"); myGhfs.exists(filePath); + assertThat(StorageStats.isTracked("op_exists")).isTrue(); assertThat(StorageStats.getLong("op_exists")).isEqualTo(1); + + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_EXISTS, 1); assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue(); } @@ -904,28 +1178,38 @@ public void xattr_storage_statistics() throws IOException { Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); Path filePath = new Path("/directory1/file1"); + StorageStatistics stats = TestUtils.getStorageStatistics(); GhfsStorageStatistics StorageStats = new GhfsStorageStatistics(myGhfs.getIOStatistics()); + myGhfs.create(filePath).writeBytes("data"); myGhfs.getXAttrs(filePath); assertThat(StorageStats.isTracked("op_xattr_get_map")).isTrue(); assertThat(StorageStats.getLong("op_xattr_get_map")).isEqualTo(1); + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_XATTR_GET_MAP, 1); + myGhfs.getXAttr(filePath, "test-xattr_statistics"); assertThat(StorageStats.isTracked("op_xattr_get_named")).isTrue(); assertThat(StorageStats.getLong("op_xattr_get_named")).isEqualTo(1); + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_XATTR_GET_NAMED, 1); + myGhfs.getXAttrs( filePath, ImmutableList.of("test-xattr-statistics", "test-xattr-statistics1", "test-xattr")); - assertThat(StorageStats.isTracked("op_xattr_get_named_map")).isTrue(); assertThat(StorageStats.getLong("op_xattr_get_named_map")).isEqualTo(1); + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_XATTR_GET_NAMED_MAP, 1); + myGhfs.listXAttrs(filePath); + assertThat(StorageStats.isTracked("op_xattr_list")).isTrue(); assertThat(StorageStats.getLong("op_xattr_list")).isEqualTo(1); + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_OP_XATTR_LIST, 1); + assertThat(myGhfs.delete(testRoot, true)).isTrue(); } @@ -1030,13 +1314,18 @@ public void testGlobStatusPathExpansionAndFilter() throws IOException { @Test public void GlobStatus_statistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); + Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); myGhfs.mkdirs(new Path("/directory1/subdirectory1")); myGhfs.create(new Path("/directory1/subdirectory1/file1")).writeBytes("data"); myGhfs.globStatus(new Path("/d*")); + assertThat(myGhfs.getIOStatistics().counters().get(INVOCATION_GLOB_STATUS.getSymbol())) .isEqualTo(1); + + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_GLOB_STATUS, 1); assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue(); } @@ -1099,20 +1388,26 @@ public void testGlobStatus() throws IOException { @Test public void getFileStatus_statistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); + Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); FSDataOutputStream fout = myGhfs.create(new Path("/directory1/file1")); fout.writeBytes("data"); fout.close(); myGhfs.getFileStatus(new Path("/directory1/file1")); + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_GET_FILE_STATUS, 1); assertThat(myGhfs.getIOStatistics().counters().get(INVOCATION_GET_FILE_STATUS.getSymbol())) .isEqualTo(1); + assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue(); } @Test public void createNonRecursive_statistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); + Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); Path filePath = new Path("/directory1/file1"); @@ -1120,17 +1415,26 @@ public void createNonRecursive_statistics() throws IOException { myGhfs.createNonRecursive(filePath, true, 1, (short) 1, 1, () -> {})) { createNonRecursiveOutputStream.write(1); } + assertThat(myGhfs.getIOStatistics().counters().get(INVOCATION_CREATE_NON_RECURSIVE.getSymbol())) .isEqualTo(1); + + TestUtils.verifyCounter( + (GhfsGlobalStorageStatistics) stats, INVOCATION_CREATE_NON_RECURSIVE, 1); assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue(); TestUtils.verifyDurationMetric( myGhfs.getIOStatistics(), INVOCATION_CREATE_NON_RECURSIVE.getSymbol(), 1); + + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_CREATE_NON_RECURSIVE.getSymbol(), 1); } @Test public void getFileChecksum_statistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); + Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); Path filePath = new Path("/directory1/file1"); @@ -1140,6 +1444,7 @@ public void getFileChecksum_statistics() throws IOException { myGhfs.getFileChecksum(filePath); assertThat(myGhfs.getIOStatistics().counters().get(INVOCATION_GET_FILE_CHECKSUM.getSymbol())) .isEqualTo(1); + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_GET_FILE_CHECKSUM, 1); assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue(); } @@ -1288,6 +1593,8 @@ public void rename_throwExceptionWhenSrcNull() throws IOException { @Test public void rename_statistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); + Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); Path source = new Path("/directory1/file1"); @@ -1297,42 +1604,69 @@ public void rename_statistics() throws IOException { assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue(); TestUtils.verifyDurationMetric(myGhfs.getIOStatistics(), INVOCATION_RENAME.getSymbol(), 1); + + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_RENAME.getSymbol(), 1); } @Test public void exists_statistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); + Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); Path filePath = new Path("/directory1/file1"); myGhfs.create(filePath).writeBytes("data"); myGhfs.exists(filePath); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_EXISTS.getSymbol(), 1); + assertThat(myGhfs.getIOStatistics().counters().get(INVOCATION_EXISTS.getSymbol())).isEqualTo(1); + assertThat(myGhfs.delete(testRoot, /* recursive= */ true)).isTrue(); } @Test public void xattr_statistics() throws IOException { GoogleHadoopFileSystem myGhfs = createInMemoryGoogleHadoopFileSystem(); + StorageStatistics stats = TestUtils.getStorageStatistics(); + Path testRoot = new Path("/directory1/"); myGhfs.mkdirs(testRoot); Path filePath = new Path("/directory1/file1"); myGhfs.create(filePath).writeBytes("data"); myGhfs.getXAttrs(filePath); + IOStatistics ioStats = myGhfs.getIOStatistics(); TestUtils.verifyDurationMetric(ioStats, INVOCATION_XATTR_GET_MAP.getSymbol(), 1); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_XATTR_GET_MAP.getSymbol(), 1); + myGhfs.getXAttr(filePath, "test-xattr_statistics"); + TestUtils.verifyDurationMetric(ioStats, INVOCATION_XATTR_GET_NAMED.getSymbol(), 1); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_XATTR_GET_NAMED.getSymbol(), 1); + myGhfs.getXAttrs( filePath, ImmutableList.of("test-xattr-statistics", "test-xattr-statistics1", "test-xattr")); + TestUtils.verifyDurationMetric(ioStats, INVOCATION_XATTR_GET_NAMED_MAP.getSymbol(), 1); TestUtils.verifyDurationMetric(ioStats, INVOCATION_XATTR_GET_MAP.getSymbol(), 2); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_XATTR_GET_NAMED_MAP.getSymbol(), 1); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_XATTR_GET_MAP.getSymbol(), 2); + myGhfs.listXAttrs(filePath); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_OP_XATTR_LIST.getSymbol(), 1); TestUtils.verifyDurationMetric(ioStats, INVOCATION_OP_XATTR_LIST.getSymbol(), 1); assertThat(myGhfs.delete(testRoot, true)).isTrue(); @@ -1340,16 +1674,25 @@ public void xattr_statistics() throws IOException { TestUtils.verifyDurationMetric(ioStats, INVOCATION_XATTR_GET_NAMED.getSymbol(), 1); TestUtils.verifyDurationMetric(ioStats, INVOCATION_XATTR_GET_MAP.getSymbol(), 2); TestUtils.verifyDurationMetric(ioStats, INVOCATION_XATTR_GET_NAMED_MAP.getSymbol(), 1); + + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_XATTR_GET_NAMED.getSymbol(), 1); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_XATTR_GET_MAP.getSymbol(), 2); + TestUtils.verifyDurationMetric( + (GhfsGlobalStorageStatistics) stats, INVOCATION_XATTR_GET_NAMED_MAP.getSymbol(), 1); } @Test @Ignore("Test is failing") public void http_IOStatistics() throws IOException { FSDataOutputStream fout = ghfs.create(new Path("/file1")); + StorageStatistics stats = TestUtils.getStorageStatistics(); fout.writeBytes("Test Content"); fout.close(); // evaluating the iostatistics by extracting the values set for the iostatistics key after each // file operation + assertThat( ((GoogleHadoopFileSystem) ghfs) .getIOStatistics() @@ -1357,7 +1700,10 @@ public void http_IOStatistics() throws IOException { .get(INVOCATION_CREATE.getSymbol())) .isEqualTo(1); + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_CREATE, 1); + // The create and write methods are expected to trigger requests of types GET, PUT and PATCH + assertThat( ((GoogleHadoopFileSystem) ghfs) .getIOStatistics() @@ -1376,6 +1722,13 @@ public void http_IOStatistics() throws IOException { .counters() .get(GhfsStatistic.ACTION_HTTP_PATCH_REQUEST.getSymbol())) .isEqualTo(1); + + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, ACTION_HTTP_GET_REQUEST, 2); + + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, ACTION_HTTP_PUT_REQUEST, 1); + + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, ACTION_HTTP_PATCH_REQUEST, 1); + assertThat(ghfs.delete(new Path("/file1"))).isTrue(); // Delete operation triggers the DELETE type request assertThat( @@ -1384,6 +1737,8 @@ public void http_IOStatistics() throws IOException { .counters() .get(GhfsStatistic.ACTION_HTTP_DELETE_REQUEST.getSymbol())) .isEqualTo(1); + + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, ACTION_HTTP_DELETE_REQUEST, 1); } @Test @@ -1796,12 +2151,18 @@ public void testImpersonationInvalidUserNameIdentifierUsed() throws Exception { @Test public void unauthenticatedAccessToPublicBuckets_fsGsProperties() throws Exception { Configuration config = loadConfig(storageClientType); + StorageStatistics stats = TestUtils.getStorageStatistics(); + config.setEnum("fs.gs.auth.type", AuthenticationType.UNAUTHENTICATED); FileSystem fs = FileSystem.get(new URI(PUBLIC_BUCKET), config); FileStatus[] fileStatuses = fs.listStatus(new Path(PUBLIC_BUCKET)); + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_LIST_FILES, 1); + + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_LIST_STATUS, 1); + assertThat( ((GoogleHadoopFileSystem) fs) .getIOStatistics() @@ -1860,4 +2221,34 @@ public void testThreadTraceEnabledRename() throws Exception { ghfs.rename(testRoot, dest); assertThat(ghfs.exists(dest)).isTrue(); } + + private static Long getMetricValue(StorageStatistics stats, GhfsStatistic invocationCreate) { + return stats.getLong(invocationCreate.getSymbol()); + } + + private void createFile(FileSystem fs, Path testFile, int writeSize, long totalSize) + throws Exception { + byte[] writeBuffer = new byte[writeSize]; + try (FSDataOutputStream output = fs.create(testFile)) { + long fileBytesWrite = 0; + do { + output.write(writeBuffer); + fileBytesWrite += writeSize; + } while (fileBytesWrite < totalSize); + } + } + + private void readFile(FileSystem fs, Path testFile, int readSize) throws Exception { + byte[] readBuffer = new byte[readSize]; + try (FSDataInputStream input = fs.open(testFile)) { + long fileBytesRead = 0; + int bytesRead; + do { + bytesRead = input.read(readBuffer); + if (bytesRead > 0) { + fileBytesRead += bytesRead; + } + } while (bytesRead >= 0); + } + } } diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemTestBase.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemTestBase.java index 45d1641f47..af5ea7770c 100644 --- a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemTestBase.java +++ b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemTestBase.java @@ -37,6 +37,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.StorageStatistics; import org.apache.hadoop.fs.permission.FsPermission; import org.junit.Test; @@ -422,6 +423,7 @@ public void testMakeQualifiedRoot() { @Test public void CopyFromLocalFileIOStatisticsTest() throws IOException { + StorageStatistics stats = TestUtils.getStorageStatistics(); // Temporary file in GHFS. URI tempFileUri = getTempFilePath(); Path tempFilePath = ghfsHelper.castAsHadoopPath(tempFileUri); @@ -443,8 +445,12 @@ public void CopyFromLocalFileIOStatisticsTest() throws IOException { .get(INVOCATION_COPY_FROM_LOCAL_FILE.getSymbol())) .isEqualTo(1); + TestUtils.verifyCounter( + (GhfsGlobalStorageStatistics) stats, INVOCATION_COPY_FROM_LOCAL_FILE, 1); + // Test the IOStatitsics of copyFromLocalFile(delSrc,overwrite,[] srcs,dst) ghfs.copyFromLocalFile(false, true, new Path[] {localTempFilePath}, tempDirPath); + assertThat( ((GoogleHadoopFileSystem) ghfs) .getIOStatistics() @@ -452,6 +458,9 @@ public void CopyFromLocalFileIOStatisticsTest() throws IOException { .get(INVOCATION_COPY_FROM_LOCAL_FILE.getSymbol())) .isEqualTo(2); + TestUtils.verifyCounter( + (GhfsGlobalStorageStatistics) stats, INVOCATION_COPY_FROM_LOCAL_FILE, 2); + if (localTempFile.exists()) { localTempFile.delete(); } diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopOutputStreamTest.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopOutputStreamTest.java index a3644644a4..6666afee1c 100644 --- a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopOutputStreamTest.java +++ b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopOutputStreamTest.java @@ -40,6 +40,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.StorageStatistics; import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; import org.junit.After; import org.junit.Before; @@ -257,13 +258,16 @@ public void write_statistics() throws IOException { } @Test - public void time_statistics() throws IOException { + public void time_statistics() throws Exception { + Path objectPath = new Path(ghfs.getUri().resolve("/dir/object2.txt")); FileSystem.Statistics statistics = new FileSystem.Statistics(ghfs.getScheme()); GoogleHadoopOutputStream fout = new GoogleHadoopOutputStream( ghfs, ghfs.getGcsPath(objectPath), CreateFileOptions.DEFAULT, statistics); + GhfsGlobalStorageStatistics stats = TestUtils.getStorageStatistics(); + byte[] data1 = {0x0f, 0x0e, 0x0e, 0x0d}; byte[] data2 = {0x0b, 0x0d, 0x0e, 0x0e, 0x0f}; @@ -273,12 +277,14 @@ public void time_statistics() throws IOException { fout.hsync(); verifyDurationMetric(fout.getIOStatistics(), INVOCATION_HSYNC.getSymbol(), 1); + verifyDurationMetric(stats, INVOCATION_HSYNC.getSymbol(), 1); fout.write(data1, 0, data1.length); fout.write(data2, 0, data2.length); fout.hflush(); verifyDurationMetric(fout.getIOStatistics(), INVOCATION_HFLUSH.getSymbol(), 1); + verifyDurationMetric(stats, INVOCATION_HFLUSH.getSymbol(), 1); fout.close(); @@ -290,12 +296,18 @@ public void time_statistics() throws IOException { verifyDurationMetric(ghfsStats, STREAM_WRITE_CLOSE_OPERATIONS.getSymbol(), 1); verifyDurationMetric(ghfsStats, INVOCATION_HFLUSH.getSymbol(), 1); verifyDurationMetric(ghfsStats, INVOCATION_HSYNC.getSymbol(), 1); + + verifyDurationMetric(stats, STREAM_WRITE_OPERATIONS.getSymbol(), 4); + verifyDurationMetric(stats, STREAM_WRITE_CLOSE_OPERATIONS.getSymbol(), 1); + verifyDurationMetric(stats, INVOCATION_HFLUSH.getSymbol(), 1); + verifyDurationMetric(stats, INVOCATION_HSYNC.getSymbol(), 1); } @Test public void hsync_statistics() throws IOException { Path objectPath = new Path(ghfs.getUri().resolve("/dir/object2.txt")); FileSystem.Statistics statistics = new FileSystem.Statistics(ghfs.getScheme()); + StorageStatistics stats = TestUtils.getStorageStatistics(); GoogleHadoopOutputStream fout = new GoogleHadoopOutputStream( ghfs, ghfs.getGcsPath(objectPath), CreateFileOptions.DEFAULT, statistics); @@ -306,8 +318,11 @@ public void hsync_statistics() throws IOException { fout.write(data1, 0, data1.length); fout.hsync(); assertThat(fout.getIOStatistics().counters().get(INVOCATION_HFLUSH.getSymbol())).isEqualTo(0); + + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_HFLUSH, 0); fout.write(data2, 0, data2.length); fout.hflush(); + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, INVOCATION_HFLUSH, 1); assertThat(fout.getIOStatistics().counters().get(INVOCATION_HFLUSH.getSymbol())).isEqualTo(1); } diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopStatusMetricsTest.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopStatusMetricsTest.java deleted file mode 100644 index ea4c2c79c2..0000000000 --- a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopStatusMetricsTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.hadoop.fs.gcs; - -import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatusStatistics.GCS_CLIENT_RATE_LIMIT_COUNT; -import static com.google.cloud.hadoop.util.testing.MockHttpTransportHelper.emptyResponse; -import static com.google.cloud.hadoop.util.testing.MockHttpTransportHelper.mockTransport; -import static com.google.common.truth.Truth.assertThat; - -import com.google.api.client.http.GenericUrl; -import com.google.api.client.http.HttpRequest; -import com.google.api.client.http.HttpRequestFactory; -import com.google.api.client.http.HttpResponse; -import com.google.api.client.http.HttpStatusCodes; -import com.google.cloud.hadoop.util.RetryHttpInitializer; -import com.google.cloud.hadoop.util.RetryHttpInitializerOptions; -import com.google.cloud.hadoop.util.interceptors.InvocationIdInterceptor; -import com.google.common.collect.ImmutableMap; -import java.net.URI; -import java.time.Duration; -import org.apache.hadoop.fs.Path; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Unit tests for {@link GoogleCloudStorageStatusStatistics} class. */ -@RunWith(JUnit4.class) -public class GoogleHadoopStatusMetricsTest { - - @Test - public void gcs_client_429_status_metrics_STORAGE_CLIENT() throws Exception { - - URI initUri = new Path("gs://test/").toUri(); - GhfsInstrumentation ghfsInstrumentation = new GhfsInstrumentation(initUri); - - String authHeaderValue = "Bearer: y2.WAKiHahzxGS_a1bd40RjNUF"; - - RetryHttpInitializer retryHttpInitializer = - new RetryHttpInitializer( - null, - RetryHttpInitializerOptions.builder() - .setDefaultUserAgent("foo-user-agent") - .setHttpHeaders(ImmutableMap.of("header-key", "header-value")) - .setMaxRequestRetries(5) - .setConnectTimeout(Duration.ofSeconds(5)) - .setReadTimeout(Duration.ofSeconds(5)) - .build(), - ghfsInstrumentation); - - HttpRequestFactory requestFactory = - mockTransport(emptyResponse(429), emptyResponse(429), emptyResponse(200)) - .createRequestFactory(retryHttpInitializer); - - HttpRequest req = requestFactory.buildGetRequest(new GenericUrl("http://fake-url.com")); - - HttpResponse res = req.execute(); - - assertThat((String) req.getHeaders().get(InvocationIdInterceptor.GOOG_API_CLIENT)) - .contains(InvocationIdInterceptor.GCCL_INVOCATION_ID_PREFIX); - assertThat(res).isNotNull(); - assertThat(res.getStatusCode()).isEqualTo(HttpStatusCodes.STATUS_CODE_OK); - assertThat( - ((GhfsInstrumentation) ghfsInstrumentation) - .getIOStatistics() - .counters() - .get(GCS_CLIENT_RATE_LIMIT_COUNT.getSymbol())) - .isNotEqualTo(0); - } -} diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/HadoopFileSystemTestBase.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/HadoopFileSystemTestBase.java index 3a07cff6b4..43c1199dc5 100644 --- a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/HadoopFileSystemTestBase.java +++ b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/HadoopFileSystemTestBase.java @@ -53,6 +53,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.StorageStatistics; import org.junit.Test; /** @@ -911,6 +912,7 @@ public void testInputStreamSeekIOStatistics() throws IOException { URI path = getTempFilePath(); Path hadoopPath = ghfsHelper.castAsHadoopPath(path); String text = "Hello World!"; + StorageStatistics stats = TestUtils.getStorageStatistics(); int numBytesWritten = ghfsHelper.writeFile(hadoopPath, text, 1, /* overwrite= */ false); @@ -918,6 +920,7 @@ public void testInputStreamSeekIOStatistics() throws IOException { try (FSDataInputStream readStream = ghfs.open(hadoopPath)) { // Check the statistics related to Forward seek operations. readStream.seek(7); + assertThat( readStream .getIOStatistics() @@ -931,6 +934,11 @@ public void testInputStreamSeekIOStatistics() throws IOException { .get(STREAM_READ_SEEK_BYTES_SKIPPED.getSymbol())) .isEqualTo(7); + TestUtils.verifyCounter( + (GhfsGlobalStorageStatistics) stats, STREAM_READ_SEEK_FORWARD_OPERATIONS, 1); + TestUtils.verifyCounter( + (GhfsGlobalStorageStatistics) stats, STREAM_READ_SEEK_BYTES_SKIPPED, 7); + // Check the statistics related to Forward seek operations. readStream.seek(5); assertThat( @@ -946,15 +954,21 @@ public void testInputStreamSeekIOStatistics() throws IOException { .get(STREAM_READ_SEEK_BYTES_BACKWARDS.getSymbol())) .isEqualTo(2); + TestUtils.verifyCounter( + (GhfsGlobalStorageStatistics) stats, STREAM_READ_SEEK_BACKWARD_OPERATIONS, 1); + TestUtils.verifyCounter( + (GhfsGlobalStorageStatistics) stats, STREAM_READ_SEEK_BYTES_BACKWARDS, 2); + // Check the statistics related to seek operations. assertThat( readStream.getIOStatistics().counters().get(STREAM_READ_SEEK_OPERATIONS.getSymbol())) .isEqualTo(2); readStream.close(); - assertThat( readStream.getIOStatistics().counters().get(STREAM_READ_CLOSE_OPERATIONS.getSymbol())) .isEqualTo(1); + + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, STREAM_READ_CLOSE_OPERATIONS, 1); } } @@ -962,13 +976,14 @@ public void testInputStreamSeekIOStatistics() throws IOException { public void testOutputStreamIOStatistics() throws IOException { URI path = getTempFilePath(); Path hadoopPath = ghfsHelper.castAsHadoopPath(path); - + StorageStatistics stats = TestUtils.getStorageStatistics(); // Check the IOstatistics of write operation try (FSDataOutputStream fsos = ghfs.create(hadoopPath)) { fsos.write("Created a file to test statistics".getBytes(UTF_8)); fsos.close(); assertThat(fsos.getIOStatistics().counters().get(STREAM_WRITE_BYTES.getSymbol())) .isEqualTo(33); + TestUtils.verifyCounter((GhfsGlobalStorageStatistics) stats, STREAM_WRITE_BYTES, 33); } assertThat(ghfs.delete(hadoopPath)).isTrue(); diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/TestUtils.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/TestUtils.java index d23b2ed9b9..f27d9765a8 100644 --- a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/TestUtils.java +++ b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/TestUtils.java @@ -1,11 +1,13 @@ +package com.google.cloud.hadoop.fs.gcs; + /* - * Copyright 2022 Google LLC + * Copyright 2023 Google Inc. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -14,14 +16,14 @@ * limitations under the License. */ -package com.google.cloud.hadoop.fs.gcs; - import static com.google.common.truth.Truth.assertThat; +import org.apache.hadoop.fs.GlobalStorageStatistics; import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.fs.statistics.StoreStatisticNames; class TestUtils { + static void verifyDurationMetric(IOStatistics ioStatistics, String symbol, int expected) { assertThat(ioStatistics.counters().get(symbol)).isEqualTo(expected); String minKey = String.format("%s%s", symbol, StoreStatisticNames.SUFFIX_MIN); @@ -36,4 +38,30 @@ static void verifyDurationMetric(IOStatistics ioStatistics, String symbol, int e assertThat(minValue).isLessThan(meanValue + 1); assertThat(meanValue).isLessThan(maxValue + 1); } + + static void verifyDurationMetric( + GhfsGlobalStorageStatistics stats, String statistic, int expected) { + String symbol = statistic; + long minValue = stats.getMin(symbol); + long maxValue = stats.getMax(symbol); + long meanValue = Double.valueOf(stats.getMean(symbol)).longValue(); + + assertThat(stats.getLong(symbol)).isEqualTo(expected); + assertThat(minValue).isLessThan(maxValue + 1); + assertThat(minValue).isLessThan(meanValue + 1); + assertThat(meanValue).isLessThan(maxValue + 1); + } + + static GhfsGlobalStorageStatistics getStorageStatistics() { + GhfsGlobalStorageStatistics stats = + (GhfsGlobalStorageStatistics) + GlobalStorageStatistics.INSTANCE.get(GhfsGlobalStorageStatistics.NAME); + stats.reset(); + return stats; + } + + static void verifyCounter( + GhfsGlobalStorageStatistics stats, GhfsStatistic statName, int expected) { + assertThat(stats.getLong(statName.getSymbol())).isEqualTo(expected); + } } diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientImpl.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientImpl.java index ddad23c1b5..15370992ca 100644 --- a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientImpl.java +++ b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientImpl.java @@ -41,7 +41,7 @@ import com.google.cloud.hadoop.util.AsyncWriteChannelOptions.PartFileCleanupType; import com.google.cloud.hadoop.util.ErrorTypeExtractor; import com.google.cloud.hadoop.util.ErrorTypeExtractor.ErrorType; -import com.google.cloud.hadoop.util.GcsClientStatisticInterface; +import com.google.cloud.hadoop.util.GoogleCloudStorageEventBus; import com.google.cloud.hadoop.util.GrpcErrorTypeExtractor; import com.google.cloud.storage.Blob; import com.google.cloud.storage.BlobId; @@ -159,8 +159,7 @@ private static String encodeMetadataValues(byte[] bytes) { @Nullable HttpRequestInitializer httpRequestInitializer, @Nullable ImmutableList gRPCInterceptors, @Nullable Function, String> downscopedAccessTokenFn, - @Nullable ExecutorService pCUExecutorService, - @Nullable GcsClientStatisticInterface gcsClientStatisticInterface) + @Nullable ExecutorService pCUExecutorService) throws IOException { super( GoogleCloudStorageImpl.builder() @@ -169,7 +168,6 @@ private static String encodeMetadataValues(byte[] bytes) { .setHttpTransport(httpTransport) .setHttpRequestInitializer(httpRequestInitializer) .setDownscopedAccessTokenFn(downscopedAccessTokenFn) - .setGcsClientStatisticInterface(gcsClientStatisticInterface) .build()); this.storageOptions = options; @@ -232,6 +230,7 @@ public void createBucket(String bucketName, CreateBucketOptions options) throws try { storage.create(bucketInfoBuilder.build()); } catch (StorageException e) { + GoogleCloudStorageEventBus.postOnException(); if (errorExtractor.bucketAlreadyExists(e)) { throw (FileAlreadyExistsException) new FileAlreadyExistsException(String.format("Bucket '%s' already exists.", bucketName)) @@ -273,6 +272,7 @@ public void createEmptyObject(StorageResourceId resourceId, CreateObjectOptions logger.atFine().withCause(e).log("Ignored exception while creating empty object"); } else { if (errorExtractor.getErrorType(e) == ErrorType.ALREADY_EXISTS) { + GoogleCloudStorageEventBus.postOnException(); throw (FileAlreadyExistsException) new FileAlreadyExistsException( String.format("Object '%s' already exists.", resourceId)) @@ -362,6 +362,7 @@ public void createEmptyObjects(List resourceIds, CreateObject } if (!innerExceptions.isEmpty()) { + GoogleCloudStorageEventBus.postOnException(); throw GoogleCloudStorageExceptions.createCompositeException(innerExceptions); } } @@ -513,6 +514,7 @@ public void copy(Map sourceToDestinationOb } if (!innerExceptions.isEmpty()) { + GoogleCloudStorageEventBus.postOnException(); throw GoogleCloudStorageExceptions.createCompositeException(innerExceptions); } } @@ -562,6 +564,7 @@ private void copyInternal( } logger.atFiner().log("Successfully copied %s to %s", srcString, dstString); } catch (StorageException e) { + GoogleCloudStorageEventBus.postOnException(); if (errorExtractor.getErrorType(e) == ErrorType.NOT_FOUND) { innerExceptions.add( createFileNotFoundException(srcBucketName, srcObjectName, new IOException(e))); @@ -627,6 +630,7 @@ private List listBucketsInternal() throws IOException { allBuckets.add(bucket); } } catch (StorageException e) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException(e); } return allBuckets; @@ -684,6 +688,7 @@ public void deleteObjects(List fullObjectNames) throws IOExce } if (!innerExceptions.isEmpty()) { + GoogleCloudStorageEventBus.postOnException(); throw GoogleCloudStorageExceptions.createCompositeException(innerExceptions); } } @@ -733,6 +738,7 @@ public void onSuccess(Blob blob) { @Override public void onFailure(Throwable throwable) { + GoogleCloudStorageEventBus.postOnException(); innerExceptions.add( new IOException( String.format("Error deleting %s, stage 1", resourceId), throwable)); @@ -772,6 +778,7 @@ public void onFailure(Throwable throwable) { resourceId, generation, attempt, throwable); queueSingleObjectDelete(resourceId, innerExceptions, batchExecutor, attempt + 1); } else { + GoogleCloudStorageEventBus.postOnException(); innerExceptions.add( new IOException( String.format( @@ -802,6 +809,7 @@ public void deleteBuckets(List bucketNames) throws IOException { innerExceptions.add(createFileNotFoundException(bucketName, null, null)); } } catch (StorageException e) { + GoogleCloudStorageEventBus.postOnException(); innerExceptions.add( new IOException(String.format("Error deleting '%s' bucket", bucketName), e)); } @@ -846,6 +854,7 @@ public List getItemInfos(List res executor.shutdown(); } if (!innerExceptions.isEmpty()) { + GoogleCloudStorageEventBus.postOnException(); throw GoogleCloudStorageExceptions.createCompositeException(innerExceptions); } @@ -891,6 +900,7 @@ public void onSuccess(@Nullable Bucket bucket) { @Override public void onFailure(Throwable throwable) { + GoogleCloudStorageEventBus.postOnException(); innerExceptions.add( new IOException( String.format("Error getting %s bucket", resourceId.getBucketName()), throwable)); @@ -919,6 +929,7 @@ public void onSuccess(@Nullable Blob blob) { @Override public void onFailure(Throwable throwable) { + GoogleCloudStorageEventBus.postOnException(); innerExceptions.add( new IOException(String.format("Error getting %s object", resourceId), throwable)); } @@ -976,6 +987,7 @@ private Bucket getBucket(String bucketName) throws IOException { try { return storage.get(bucketName); } catch (StorageException e) { + GoogleCloudStorageEventBus.postOnException(); if (errorExtractor.getErrorType(e) == ErrorType.NOT_FOUND) { return null; } @@ -1003,6 +1015,7 @@ Blob getBlob(StorageResourceId resourceId) throws IOException { BlobId.of(bucketName, objectName), BlobGetOption.fields(BLOB_FIELDS.toArray(new BlobField[0]))); } catch (StorageException e) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException("Error accessing " + resourceId, e); } return blob; @@ -1111,6 +1124,7 @@ public void onSuccess(Blob blob) { @Override public void onFailure(Throwable throwable) { + GoogleCloudStorageEventBus.postOnException(); innerExceptions.add( new IOException( String.format("Error updating '%s' object", resourceId), throwable)); @@ -1122,6 +1136,7 @@ public void onFailure(Throwable throwable) { } if (!innerExceptions.isEmpty()) { + GoogleCloudStorageEventBus.postOnException(); throw GoogleCloudStorageExceptions.createCompositeException(innerExceptions); } @@ -1180,6 +1195,7 @@ public GoogleCloudStorageItemInfo composeObjects( logger.atFiner().log("composeObjects(%s, %s, %s)", sources, destination, options); for (StorageResourceId inputId : sources) { if (!destination.getBucketName().equals(inputId.getBucketName())) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException( String.format( "Bucket doesn't match for source '%s' and destination '%s'!", @@ -1207,6 +1223,7 @@ public GoogleCloudStorageItemInfo composeObjects( try { composedBlob = storage.compose(request); } catch (StorageException e) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException(e); } GoogleCloudStorageItemInfo compositeInfo = createItemInfoForBlob(destination, composedBlob); @@ -1236,6 +1253,7 @@ private long getWriteGeneration(StorageResourceId resourceId, boolean overwrite) checkState(generation != 0, "Generation should not be 0 for an existing item"); return generation; } + GoogleCloudStorageEventBus.postOnException(); throw new FileAlreadyExistsException(String.format("Object %s already exists.", resourceId)); } @@ -1288,6 +1306,7 @@ private static BlobWriteSessionConfig getSessionConfig( case JOURNALING: if (writeOptions.getTemporaryPaths() == null || writeOptions.getTemporaryPaths().isEmpty()) { + GoogleCloudStorageEventBus.postOnException(); throw new IllegalArgumentException( "Upload using `Journaling` requires the property:fs.gs.write.temporary.dirs to be set."); } @@ -1304,6 +1323,7 @@ private static BlobWriteSessionConfig getSessionConfig( .withExecutorSupplier(getPCUExecutorSupplier(pCUExecutorService)) .withPartNamingStrategy(getPartNamingStrategy(writeOptions.getPartFileNamePrefix())); default: + GoogleCloudStorageEventBus.postOnException(); throw new IllegalArgumentException( String.format("Upload type:%s is not supported.", writeOptions.getUploadType())); } @@ -1318,6 +1338,7 @@ private static PartCleanupStrategy getPartCleanupStrategy(PartFileCleanupType cl case ALWAYS: return PartCleanupStrategy.always(); default: + GoogleCloudStorageEventBus.postOnException(); throw new IllegalArgumentException( String.format("Cleanup type:%s is not handled.", cleanupType)); } @@ -1410,9 +1431,6 @@ public abstract Builder setDownscopedAccessTokenFn( public abstract Builder setGRPCInterceptors( @Nullable ImmutableList gRPCInterceptors); - public abstract Builder setGcsClientStatisticInterface( - @Nullable GcsClientStatisticInterface gcsClientStatisticInterface); - @VisibleForTesting public abstract Builder setClientLibraryStorage(@Nullable Storage clientLibraryStorage); diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientReadChannel.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientReadChannel.java index af8d542b79..b97393dd78 100644 --- a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientReadChannel.java +++ b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientReadChannel.java @@ -28,6 +28,7 @@ import com.google.cloud.ReadChannel; import com.google.cloud.hadoop.gcsio.GoogleCloudStorageReadOptions.Fadvise; import com.google.cloud.hadoop.util.ErrorTypeExtractor; +import com.google.cloud.hadoop.util.GoogleCloudStorageEventBus; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.Storage; import com.google.cloud.storage.Storage.BlobSourceOption; @@ -92,6 +93,7 @@ public GoogleCloudStorageClientReadChannel( protected void initMetadata(@Nullable String encoding, long sizeFromMetadata) throws IOException { gzipEncoded = nullToEmpty(encoding).contains(GZIP_ENCODING); if (gzipEncoded && !readOptions.isGzipEncodingSupportEnabled()) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException( "Cannot read GZIP encoded files - content encoding support is disabled."); } @@ -158,6 +160,7 @@ public long size() throws IOException { @Override public SeekableByteChannel truncate(long size) throws IOException { + GoogleCloudStorageEventBus.postOnException(); throw new UnsupportedOperationException("Cannot mutate read-only channel"); } @@ -173,6 +176,7 @@ public void close() throws IOException { logger.atFiner().log("Closing channel for '%s'", resourceId); contentReadChannel.closeContentChannel(); } catch (Exception e) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException( String.format("Exception occurred while closing channel '%s'", resourceId), e); } finally { @@ -262,6 +266,7 @@ public int readContent(ByteBuffer dst) throws IOException { } if (currentPosition != contentChannelEnd && currentPosition != objectSize) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException( String.format( "Received end of stream result before all requestedBytes were received;" @@ -287,6 +292,7 @@ public int readContent(ByteBuffer dst) throws IOException { contentChannelCurrentPosition, currentPosition); } catch (Exception e) { + GoogleCloudStorageEventBus.postOnException(); int partialBytes = partiallyReadBytes(remainingBeforeRead, dst); totalBytesRead += partialBytes; currentPosition += partialBytes; @@ -380,6 +386,7 @@ private void cacheFooter(ReadableByteChannel readableByteChannel) throws IOExcep footerSize, resourceId); } catch (Exception e) { + GoogleCloudStorageEventBus.postOnException(); footerContent = null; throw e; } @@ -433,6 +440,7 @@ public void closeContentChannel() { try { byteChannel.close(); } catch (Exception e) { + GoogleCloudStorageEventBus.postOnException(); logger.atFine().withCause(e).log( "Got an exception on contentChannel.close() for '%s'; ignoring it.", resourceId); } finally { @@ -479,6 +487,7 @@ private void skipInPlace() { contentChannelCurrentPosition += bytesRead; } } catch (Exception e) { + GoogleCloudStorageEventBus.postOnException(); logger.atInfo().withCause(e).log( "Got an IO exception on contentChannel.read(), a lazy-seek will be pending for '%s'", resourceId); @@ -547,6 +556,7 @@ private ReadableByteChannel getStorageReadChannel(long seek, long limit) throws readChannel.limit(limit); return readChannel; } catch (Exception e) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException( String.format( "Unable to update the boundaries/Range of contentChannel %s", @@ -586,12 +596,14 @@ private static void validate(GoogleCloudStorageItemInfo itemInfo) throws IOExcep checkArgument( resourceId.isStorageObject(), "Can not open a non-file object for read: %s", resourceId); if (!itemInfo.exists()) { + GoogleCloudStorageEventBus.postOnException(); throw new FileNotFoundException(String.format("Item not found: %s", resourceId)); } } private IOException convertError(Exception error) { String msg = String.format("Error reading '%s'", resourceId); + GoogleCloudStorageEventBus.postOnException(); switch (errorExtractor.getErrorType(error)) { case NOT_FOUND: return createFileNotFoundException( @@ -606,6 +618,7 @@ private IOException convertError(Exception error) { /** Validates that the given position is valid for this channel. */ private void validatePosition(long position) throws IOException { if (position < 0) { + GoogleCloudStorageEventBus.postOnException(); throw new EOFException( String.format( "Invalid seek offset: position value (%d) must be >= 0 for '%s'", @@ -613,6 +626,7 @@ private void validatePosition(long position) throws IOException { } if (objectSize >= 0 && position >= objectSize) { + GoogleCloudStorageEventBus.postOnException(); throw new EOFException( String.format( "Invalid seek offset: position value (%d) must be between 0 and %d for '%s'", @@ -623,6 +637,7 @@ private void validatePosition(long position) throws IOException { /** Throws if this channel is not currently open. */ private void throwIfNotOpen() throws IOException { if (!isOpen()) { + GoogleCloudStorageEventBus.postOnException(); throw new ClosedChannelException(); } } diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientWriteChannel.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientWriteChannel.java index ec8b137517..ae6e66d224 100644 --- a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientWriteChannel.java +++ b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientWriteChannel.java @@ -20,6 +20,7 @@ import static com.google.storage.v2.ServiceConstants.Values.MAX_WRITE_CHUNK_BYTES; import com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel; +import com.google.cloud.hadoop.util.GoogleCloudStorageEventBus; import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.BlobWriteSession; @@ -64,6 +65,7 @@ public GoogleCloudStorageClientWriteChannel( try { this.writableByteChannel = blobWriteSession.open(); } catch (StorageException e) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException(e); } } @@ -75,6 +77,7 @@ public void startUpload(InputStream pipeSource) throws IOException { try { uploadOperation = threadPool.submit(new UploadOperation(pipeSource, this.resourceId)); } catch (Exception e) { + GoogleCloudStorageEventBus.postOnException(); throw new RuntimeException(String.format("Failed to start upload for '%s'", resourceId), e); } } @@ -152,6 +155,7 @@ public Boolean call() throws Exception { logger.atFiner().log("Uploaded all chunks for resource %s", resourceId); return true; } catch (Exception e) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException( String.format("Error occurred while uploading resource %s", resourceId), e); } @@ -191,6 +195,7 @@ public void close() throws IOException { // the gcs-object. writableByteChannel.close(); } catch (Exception e) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException(String.format("Upload failed for '%s'", resourceId), e); } finally { writableByteChannel = null; diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemImpl.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemImpl.java index 9ff1429892..a2ca90857a 100644 --- a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemImpl.java +++ b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemImpl.java @@ -32,7 +32,7 @@ import com.google.cloud.hadoop.gcsio.GoogleCloudStorage.ListPage; import com.google.cloud.hadoop.util.AccessBoundary; import com.google.cloud.hadoop.util.CheckedFunction; -import com.google.cloud.hadoop.util.GcsClientStatisticInterface; +import com.google.cloud.hadoop.util.GoogleCloudStorageEventBus; import com.google.cloud.hadoop.util.ITraceOperation; import com.google.cloud.hadoop.util.LazyExecutorService; import com.google.cloud.hadoop.util.ThreadTrace; @@ -123,8 +123,7 @@ public class GoogleCloudStorageFileSystemImpl implements GoogleCloudStorageFileS private static GoogleCloudStorage createCloudStorage( GoogleCloudStorageFileSystemOptions options, Credentials credentials, - Function, String> downscopedAccessTokenFn, - GcsClientStatisticInterface gcsClientStatisticInterface) + Function, String> downscopedAccessTokenFn) throws IOException { checkNotNull(options, "options must not be null"); @@ -134,14 +133,12 @@ private static GoogleCloudStorage createCloudStorage( .setOptions(options.getCloudStorageOptions()) .setCredentials(credentials) .setDownscopedAccessTokenFn(downscopedAccessTokenFn) - .setGcsClientStatisticInterface(gcsClientStatisticInterface) .build(); default: return GoogleCloudStorageImpl.builder() .setOptions(options.getCloudStorageOptions()) .setCredentials(credentials) .setDownscopedAccessTokenFn(downscopedAccessTokenFn) - .setGcsClientStatisticInterface(gcsClientStatisticInterface) .build(); } } @@ -155,13 +152,7 @@ private static GoogleCloudStorage createCloudStorage( */ public GoogleCloudStorageFileSystemImpl( Credentials credentials, GoogleCloudStorageFileSystemOptions options) throws IOException { - this( - createCloudStorage( - options, - credentials, - /* downscopedAccessTokenFn= */ null, /* gcsClientStatisticInterface */ - null), - options); + this(createCloudStorage(options, credentials, /* downscopedAccessTokenFn= */ null), options); logger.atFiner().log("GoogleCloudStorageFileSystem(options: %s)", options); } @@ -172,18 +163,13 @@ public GoogleCloudStorageFileSystemImpl( * @param downscopedAccessTokenFn Function that generates downscoped access token. * @param options Options for how this filesystem should operate and configure its underlying * storage. - * @param gcsClientStatisticInterface for backporting ghfsInstrumentation */ public GoogleCloudStorageFileSystemImpl( Credentials credentials, Function, String> downscopedAccessTokenFn, - GoogleCloudStorageFileSystemOptions options, - GcsClientStatisticInterface gcsClientStatisticInterface) + GoogleCloudStorageFileSystemOptions options) throws IOException { - this( - createCloudStorage( - options, credentials, downscopedAccessTokenFn, gcsClientStatisticInterface), - options); + this(createCloudStorage(options, credentials, downscopedAccessTokenFn), options); logger.atFiner().log("GoogleCloudStorageFileSystem(options: %s)", options); } @@ -254,6 +240,7 @@ public WritableByteChannel create(URI path, CreateFileOptions createOptions) thr StorageResourceId.fromUriPath(path, /* allowEmptyObjectName=*/ true); if (resourceId.isDirectory()) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException( String.format( "Cannot create a file whose name looks like a directory: '%s'", resourceId)); @@ -280,6 +267,7 @@ public WritableByteChannel create(URI path, CreateFileOptions createOptions) thr // Check if a directory with the same name exists. if (getFromFuture(conflictingDirExist)) { + GoogleCloudStorageEventBus.postOnException(); throw new FileAlreadyExistsException("A directory with that name exists: " + path); } } @@ -326,6 +314,7 @@ public void delete(URI path, boolean recursive) throws IOException { FileInfo fileInfo = getFileInfo(path); if (!fileInfo.exists()) { + GoogleCloudStorageEventBus.postOnException(); throw new FileNotFoundException("Item not found: " + path); } @@ -350,6 +339,7 @@ public void delete(URI path, boolean recursive) throws IOException { fileInfo.getPath(), DELETE_RENAME_LIST_OPTIONS, /* pageToken= */ null) .getItems(); if (!itemsToDelete.isEmpty() && !recursive) { + GoogleCloudStorageEventBus.postOnException(); throw new DirectoryNotEmptyException("Cannot delete a non-empty directory."); } } else { @@ -430,6 +420,7 @@ private void mkdirsInternal(StorageResourceId resourceId) throws IOException { try { gcs.createBucket(resourceId.getBucketName()); } catch (FileAlreadyExistsException e) { + GoogleCloudStorageEventBus.postOnException(); // This means that bucket already exist, and we do not need to do anything. logger.atFiner().withCause(e).log( "mkdirs: %s already exists, ignoring creation failure", resourceId); @@ -450,6 +441,7 @@ private void mkdirsInternal(StorageResourceId resourceId) throws IOException { try { gcs.createEmptyObject(resourceId); } catch (FileAlreadyExistsException e) { + GoogleCloudStorageEventBus.postOnException(); // This means that directory object already exist, and we do not need to do anything. logger.atFiner().withCause(e).log( "mkdirs: %s already exists, ignoring creation failure", resourceId); @@ -481,6 +473,7 @@ public void rename(URI src, URI dst) throws IOException { // Throw if the source file does not exist. if (!srcInfo.exists()) { + GoogleCloudStorageEventBus.postOnException(); throw new FileNotFoundException("Item not found: " + src); } @@ -534,16 +527,19 @@ private URI getDstUri(FileInfo srcInfo, FileInfo dstInfo, @Nullable FileInfo dst // Throw if src is a file and dst == GCS_ROOT if (!srcInfo.isDirectory() && dst.equals(GCS_ROOT)) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException("A file cannot be created in root."); } // Throw if the destination is a file that already exists, and it's not a source file. if (dstInfo.exists() && !dstInfo.isDirectory() && (srcInfo.isDirectory() || !dst.equals(src))) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException("Cannot overwrite an existing file: " + dst); } // Rename operation cannot be completed if parent of destination does not exist. if (dstParentInfo != null && !dstParentInfo.exists()) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException( "Cannot rename because path does not exist: " + dstParentInfo.getPath()); } @@ -570,6 +566,7 @@ private URI getDstUri(FileInfo srcInfo, FileInfo dstInfo, @Nullable FileInfo dst // Throw if renaming directory to self - this is forbidden if (src.equals(dst)) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException("Rename dir to self is forbidden"); } @@ -578,6 +575,7 @@ private URI getDstUri(FileInfo srcInfo, FileInfo dstInfo, @Nullable FileInfo dst // because this means that src is a parent directory of dst // and src cannot be "renamed" to its subdirectory if (!dstRelativeToSrc.equals(dst)) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException("Rename to subdir is forbidden"); } @@ -596,6 +594,7 @@ private URI getDstUri(FileInfo srcInfo, FileInfo dstInfo, @Nullable FileInfo dst if (dstInfo.isDirectory()) { if (!dstInfo.exists()) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException("Cannot rename because path does not exist: " + dstInfo.getPath()); } else { dst = dst.resolve(srcItemName); @@ -735,6 +734,7 @@ private void repairImplicitDirectory(Future infoFutu gcs.createEmptyObject(resourceId); logger.atInfo().log("Successfully repaired '%s' directory.", resourceId); } catch (IOException e) { + GoogleCloudStorageEventBus.postOnException(); logger.atWarning().withCause(e).log("Failed to repair '%s' directory", resourceId); } } @@ -743,6 +743,7 @@ static T getFromFuture(Future future) throws IOException { try { return future.get(); } catch (ExecutionException | InterruptedException e) { + GoogleCloudStorageEventBus.postOnException(); if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } @@ -824,6 +825,7 @@ public List listFileInfo(URI path, ListFileOptions listOptions) throws return listedInfo; } } catch (Exception e) { + GoogleCloudStorageEventBus.postOnException(); dirItemInfosFuture.cancel(/* mayInterruptIfRunning= */ true); throw e; } @@ -831,6 +833,7 @@ public List listFileInfo(URI path, ListFileOptions listOptions) throws List dirItemInfos = getFromFuture(dirItemInfosFuture); if (pathId.isStorageObject() && dirItemInfos.isEmpty()) { + GoogleCloudStorageEventBus.postOnException(); throw new FileNotFoundException("Item not found: " + path); } @@ -886,6 +889,7 @@ private GoogleCloudStorageItemInfo getFileInfoInternal( return itemInfo; } } catch (Exception e) { + GoogleCloudStorageEventBus.postOnException(); listDirFuture.cancel(/* mayInterruptIfRunning= */ true); throw e; } @@ -996,6 +1000,7 @@ private void checkNoFilesConflictingWithDirs(StorageResourceId resourceId) throw // we make this check therefore this is a good faith effort and not a guarantee. for (GoogleCloudStorageItemInfo fileInfo : gcs.getItemInfos(fileIds)) { if (fileInfo.exists()) { + GoogleCloudStorageEventBus.postOnException(); throw new FileAlreadyExistsException( "Cannot create directories because of existing file: " + fileInfo.getResourceId()); } diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageImpl.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageImpl.java index 3a0b913bd7..4a1d261312 100644 --- a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageImpl.java +++ b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageImpl.java @@ -57,7 +57,7 @@ import com.google.cloud.hadoop.util.ApiErrorExtractor; import com.google.cloud.hadoop.util.ChainingHttpRequestInitializer; import com.google.cloud.hadoop.util.ClientRequestHelper; -import com.google.cloud.hadoop.util.GcsClientStatisticInterface; +import com.google.cloud.hadoop.util.GoogleCloudStorageEventBus; import com.google.cloud.hadoop.util.HttpTransportFactory; import com.google.cloud.hadoop.util.ITraceOperation; import com.google.cloud.hadoop.util.ResilientOperation; @@ -152,6 +152,7 @@ private static byte[] decodeMetadataValues(String value) { try { return BaseEncoding.base64().decode(value); } catch (IllegalArgumentException iae) { + GoogleCloudStorageEventBus.postOnException(); logger.atSevere().withCause(iae).log( "Failed to parse base64 encoded attribute value %s - %s", value, iae); return null; @@ -205,6 +206,7 @@ public Boolean load(String bucketName) { .executeUnparsed() .disconnect(); } catch (IOException e) { + GoogleCloudStorageEventBus.postOnException(); return errorExtractor.userProjectMissing(e); } return false; @@ -243,7 +245,6 @@ public Boolean load(String bucketName) { private final HttpRequestInitializer httpRequestInitializer; // Backporting GhfsInstrumentation - private final GcsClientStatisticInterface gcsClientStatisticInterface; private final StatisticsTrackingHttpRequestInitializer httpStatistics = new StatisticsTrackingHttpRequestInitializer(); @@ -271,7 +272,6 @@ public Boolean load(String bucketName) { * @param httpTransport transport used for HTTP requests * @param httpRequestInitializer request initializer used to initialize all HTTP requests * @param downscopedAccessTokenFn Function that generates downscoped access token - * @param gcsClientStatisticInterface used for backporting ghfsInstrumentation instances * @throws IOException on IO error */ GoogleCloudStorageImpl( @@ -279,8 +279,7 @@ public Boolean load(String bucketName) { @Nullable Credentials credentials, @Nullable HttpTransport httpTransport, @Nullable HttpRequestInitializer httpRequestInitializer, - @Nullable Function, String> downscopedAccessTokenFn, - @Nullable GcsClientStatisticInterface gcsClientStatisticInterface) + @Nullable Function, String> downscopedAccessTokenFn) throws IOException { logger.atFiner().log("GCS(options: %s)", options); @@ -288,8 +287,6 @@ public Boolean load(String bucketName) { this.storageOptions = options; - this.gcsClientStatisticInterface = gcsClientStatisticInterface; - Credentials finalCredentials; // If credentials is null then use httpRequestInitializer to initialize finalCredentials if (credentials == null && httpRequestInitializer != null) { @@ -308,10 +305,7 @@ public Boolean load(String bucketName) { HttpRequestInitializer finalHttpRequestInitializer; if (httpRequestInitializer == null) { finalHttpRequestInitializer = - new RetryHttpInitializer( - finalCredentials, - options.toRetryHttpInitializerOptions(), - gcsClientStatisticInterface); + new RetryHttpInitializer(finalCredentials, options.toRetryHttpInitializerOptions()); } else { logger.atWarning().log( "ALERT: Overriding httpRequestInitializer - this should not be done in production!"); @@ -481,9 +475,11 @@ public void createBucket(String bucketName, CreateBucketOptions options) throws IOException.class, sleeper); } catch (InterruptedException e) { + GoogleCloudStorageEventBus.postOnException(); Thread.currentThread().interrupt(); throw new IOException("Failed to create bucket", e); } catch (IOException e) { + GoogleCloudStorageEventBus.postOnException(); if (ApiErrorExtractor.INSTANCE.itemAlreadyExists(e)) { throw (FileAlreadyExistsException) new FileAlreadyExistsException(String.format("Bucket '%s' already exists.", bucketName)) @@ -509,6 +505,7 @@ public void createEmptyObject(StorageResourceId resourceId, CreateObjectOptions e.getClass().getSimpleName()); logger.atFine().withCause(e).log("Ignored exception while creating empty object"); } else { + GoogleCloudStorageEventBus.postOnException(); if (ApiErrorExtractor.INSTANCE.itemAlreadyExists(e)) { throw (FileAlreadyExistsException) new FileAlreadyExistsException( @@ -599,11 +596,13 @@ public void createEmptyObjects(List resourceIds, CreateObject try { latch.await(); } catch (InterruptedException ie) { + GoogleCloudStorageEventBus.postOnException(); Thread.currentThread().interrupt(); throw new IOException("Failed to create empty objects", ie); } if (!innerExceptions.isEmpty()) { + GoogleCloudStorageEventBus.postOnException(); throw GoogleCloudStorageExceptions.createCompositeException(innerExceptions); } } @@ -706,11 +705,13 @@ public void deleteBuckets(List bucketNames) throws IOException { ? createFileNotFoundException(bucketName, null, e) : new IOException(String.format("Error deleting '%s' bucket", bucketName), e)); } catch (InterruptedException e) { + GoogleCloudStorageEventBus.postOnException(); Thread.currentThread().interrupt(); throw new IOException("Failed to delete buckets", e); } } if (!innerExceptions.isEmpty()) { + GoogleCloudStorageEventBus.postOnException(); throw GoogleCloudStorageExceptions.createCompositeException(innerExceptions); } } @@ -753,6 +754,7 @@ public void deleteObjects(List fullObjectNames) throws IOExce batchHelper.flush(); if (!innerExceptions.isEmpty()) { + GoogleCloudStorageEventBus.postOnException(); throw GoogleCloudStorageExceptions.createCompositeException(innerExceptions); } } @@ -775,6 +777,7 @@ public void onSuccess(Void obj, HttpHeaders responseHeaders) { public void onFailure(GoogleJsonError jsonError, HttpHeaders responseHeaders) throws IOException { GoogleJsonResponseException cause = createJsonResponseException(jsonError, responseHeaders); + GoogleCloudStorageEventBus.postOnException(); if (errorExtractor.itemNotFound(cause)) { // Ignore item-not-found errors. We do not have to delete what we cannot find. // This error typically shows up when we make a request to delete something and the @@ -844,6 +847,7 @@ public void onSuccess(StorageObject storageObject, HttpHeaders httpHeaders) @Override public void onFailure(GoogleJsonError jsonError, HttpHeaders responseHeaders) { + GoogleCloudStorageEventBus.postOnException(); GoogleJsonResponseException cause = createJsonResponseException(jsonError, responseHeaders); if (errorExtractor.itemNotFound(cause)) { @@ -896,6 +900,7 @@ public static void validateCopyArguments( GoogleCloudStorageItemInfo srcBucketInfo = getGoogleCloudStorageItemInfo(gcsImpl, bucketInfoCache, srcBucketResourceId); if (!srcBucketInfo.exists()) { + GoogleCloudStorageEventBus.postOnException(); throw new FileNotFoundException("Bucket not found: " + srcBucketName); } @@ -903,16 +908,19 @@ public static void validateCopyArguments( GoogleCloudStorageItemInfo dstBucketInfo = getGoogleCloudStorageItemInfo(gcsImpl, bucketInfoCache, dstBucketResourceId); if (!dstBucketInfo.exists()) { + GoogleCloudStorageEventBus.postOnException(); throw new FileNotFoundException("Bucket not found: " + dstBucketName); } if (!gcsImpl.getOptions().isCopyWithRewriteEnabled()) { if (!srcBucketInfo.getLocation().equals(dstBucketInfo.getLocation())) { + GoogleCloudStorageEventBus.postOnException(); throw new UnsupportedOperationException( "This operation is not supported across two different storage locations."); } if (!srcBucketInfo.getStorageClass().equals(dstBucketInfo.getStorageClass())) { + GoogleCloudStorageEventBus.postOnException(); throw new UnsupportedOperationException( "This operation is not supported across two different storage classes."); } @@ -924,6 +932,7 @@ public static void validateCopyArguments( !isNullOrEmpty(destination.getObjectName()), "dstObjectName must not be null or empty"); if (srcBucketName.equals(dstBucketName) && source.getObjectName().equals(destination.getObjectName())) { + GoogleCloudStorageEventBus.postOnException(); throw new IllegalArgumentException( String.format( "Copy destination must be different from source for %s.", @@ -1035,6 +1044,7 @@ public void copy(Map sourceToDestinationOb batchHelper.flush(); if (!innerExceptions.isEmpty()) { + GoogleCloudStorageEventBus.postOnException(); throw GoogleCloudStorageExceptions.createCompositeException(innerExceptions); } } @@ -1101,6 +1111,7 @@ public void onSuccess(RewriteResponse rewriteResponse, HttpHeaders responseHeade @Override public void onFailure(GoogleJsonError e, HttpHeaders responseHeaders) { + GoogleCloudStorageEventBus.postOnException(); onCopyFailure(innerExceptions, e, responseHeaders, srcBucketName, srcObjectName); } }); @@ -1141,6 +1152,7 @@ public void onSuccess(StorageObject copyResponse, HttpHeaders responseHeaders) { @Override public void onFailure(GoogleJsonError jsonError, HttpHeaders responseHeaders) { + GoogleCloudStorageEventBus.postOnException(); onCopyFailure( innerExceptions, jsonError, responseHeaders, srcBucketName, srcObjectName); } @@ -1361,12 +1373,15 @@ private String listStorageObjectsAndPrefixesPage( items = listObject.execute(); op.annotate("resultSize", items == null ? 0 : items.size()); } catch (IOException e) { + GoogleCloudStorageEventBus.postOnException(); String resource = StringPaths.fromComponents(listObject.getBucket(), listObject.getPrefix()); if (errorExtractor.itemNotFound(e)) { logger.atFiner().withCause(e).log( "listStorageObjectsAndPrefixesPage(%s, %s): item not found", resource, listOptions); return null; } + + GoogleCloudStorageEventBus.postOnException(); throw new IOException("Error listing " + resource, e); } @@ -1773,6 +1788,7 @@ public void onFailure(GoogleJsonError jsonError, HttpHeaders responseHeaders) { batchHelper.flush(); if (!innerExceptions.isEmpty()) { + GoogleCloudStorageEventBus.postOnException(); throw GoogleCloudStorageExceptions.createCompositeException(innerExceptions); } @@ -1875,6 +1891,7 @@ public void onFailure(GoogleJsonError jsonError, HttpHeaders responseHeaders) { batchHelper.flush(); if (!innerExceptions.isEmpty()) { + GoogleCloudStorageEventBus.postOnException(); throw GoogleCloudStorageExceptions.createCompositeException(innerExceptions); } @@ -1973,6 +1990,7 @@ private Bucket getBucket(String bucketName) throws IOException { try (ITraceOperation op = TraceOperation.addToExistingTrace("gcs.buckets.get")) { return getBucket.execute(); } catch (IOException e) { + GoogleCloudStorageEventBus.postOnException(); if (errorExtractor.itemNotFound(e)) { logger.atFiner().withCause(e).log("getBucket(%s): not found", bucketName); return null; @@ -2001,6 +2019,7 @@ private long getWriteGeneration(StorageResourceId resourceId, boolean overwrite) checkState(generation != 0, "Generation should not be 0 for an existing item"); return generation; } + GoogleCloudStorageEventBus.postOnException(); throw new FileAlreadyExistsException(String.format("Object %s already exists.", resourceId)); } @@ -2028,9 +2047,11 @@ private StorageObject getObject(StorageResourceId resourceId) throws IOException return getObject.execute(); } catch (IOException e) { if (errorExtractor.itemNotFound(e)) { + GoogleCloudStorageEventBus.postOnException(); logger.atFiner().withCause(e).log("getObject(%s): not found", resourceId); return null; } + GoogleCloudStorageEventBus.postOnException(); throw new IOException("Error accessing " + resourceId, e); } } @@ -2071,6 +2092,7 @@ private boolean canIgnoreExceptionForEmptyObject( try { sleeper.sleep(nextSleep); } catch (InterruptedException e) { + GoogleCloudStorageEventBus.postOnException(); // We caught an InterruptedException, we should set the interrupted bit on this thread. Thread.currentThread().interrupt(); nextSleep = BackOff.STOP; @@ -2119,6 +2141,7 @@ public GoogleCloudStorageItemInfo composeObjects( logger.atFiner().log("composeObjects(%s, %s, %s)", sources, destination, options); for (StorageResourceId inputId : sources) { if (!destination.getBucketName().equals(inputId.getBucketName())) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException( String.format( "Bucket doesn't match for source '%s' and destination '%s'!", @@ -2295,9 +2318,6 @@ public abstract static class Builder { public abstract Builder setHttpTransport(@Nullable HttpTransport httpTransport); - public abstract Builder setGcsClientStatisticInterface( - @Nullable GcsClientStatisticInterface gcsClientStatisticInterface); - @VisibleForTesting public abstract Builder setHttpRequestInitializer( @Nullable HttpRequestInitializer httpRequestInitializer); diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageReadChannel.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageReadChannel.java index da4f149f6e..7a57eee5de 100644 --- a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageReadChannel.java +++ b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageReadChannel.java @@ -35,6 +35,7 @@ import com.google.cloud.hadoop.gcsio.GoogleCloudStorageReadOptions.Fadvise; import com.google.cloud.hadoop.util.ApiErrorExtractor; import com.google.cloud.hadoop.util.ClientRequestHelper; +import com.google.cloud.hadoop.util.GoogleCloudStorageEventBus; import com.google.cloud.hadoop.util.ResilientOperation; import com.google.cloud.hadoop.util.RetryDeterminer; import com.google.common.annotations.VisibleForTesting; @@ -230,10 +231,12 @@ private GoogleCloudStorageItemInfo fetchInitialMetadata() throws IOException { IOException.class, sleeper); } catch (IOException e) { + GoogleCloudStorageEventBus.postOnException(); throw errorExtractor.itemNotFound(e) ? createFileNotFoundException(resourceId, e) : new IOException("Error reading " + resourceId, e); } catch (InterruptedException e) { + GoogleCloudStorageEventBus.postOnException(); Thread.currentThread().interrupt(); throw new IOException("Thread interrupt received.", e); } @@ -323,6 +326,7 @@ public int read(ByteBuffer buffer) throws IOException { // only ignore // this case here. if (currentPosition != contentChannelEnd && currentPosition != size) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException( String.format( "Received end of stream result before all the file data has been received;" @@ -377,6 +381,7 @@ public int read(ByteBuffer buffer) throws IOException { // TODO(user): Refactor any reusable logic for retries into a separate RetryHelper // class. if (retriesAttempted == maxRetries) { + GoogleCloudStorageEventBus.postOnException(); logger.atSevere().log( "Throwing exception after reaching max read retries (%d) for '%s'.", maxRetries, resourceId); @@ -397,6 +402,7 @@ public int read(ByteBuffer buffer) throws IOException { try { boolean backOffSuccessful = BackOffUtils.next(sleeper, readBackOff.get()); if (!backOffSuccessful) { + GoogleCloudStorageEventBus.postOnException(); logger.atSevere().log( "BackOff returned false; maximum total elapsed time exhausted." + " Giving up after %d/%d retries for '%s'", @@ -404,6 +410,7 @@ public int read(ByteBuffer buffer) throws IOException { throw ioe; } } catch (InterruptedException ie) { + GoogleCloudStorageEventBus.postOnException(); Thread.currentThread().interrupt(); logger.atSevere().log( "Interrupted while sleeping before retry. Giving up after %d/%d retries for '%s'", @@ -415,6 +422,7 @@ public int read(ByteBuffer buffer) throws IOException { logger.atInfo().log( "Done sleeping before retry #%d/%d for '%s'", retriesAttempted, maxRetries, resourceId); } catch (RuntimeException r) { + GoogleCloudStorageEventBus.postOnException(); closeContentChannel(); throw r; } @@ -432,6 +440,7 @@ public int read(ByteBuffer buffer) throws IOException { // of the data stream when stream compression is used, so we can only ignore this case // here. if (currentPosition != size) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException( String.format( "Failed to read any data before all the file data has been received;" @@ -480,6 +489,7 @@ protected void closeContentChannel() { try { contentChannel.close(); } catch (Exception e) { + GoogleCloudStorageEventBus.postOnException(); logger.atFine().withCause(e).log( "Got an exception on contentChannel.close() for '%s'; ignoring it.", resourceId); } finally { @@ -594,6 +604,7 @@ private void skipInPlace(long seekDistance) { contentChannelPosition += bytesRead; } } catch (IOException e) { + GoogleCloudStorageEventBus.postOnException(); logger.atInfo().withCause(e).log( "Got an IO exception on contentChannel.read(), a lazy-seek will be pending for '%s'", resourceId); @@ -650,6 +661,7 @@ private void checkEncodingAndAccess() { /** Validates that the given position is valid for this channel. */ protected void validatePosition(long position) throws IOException { if (position < 0) { + GoogleCloudStorageEventBus.postOnException(); throw new EOFException( String.format( "Invalid seek offset: position value (%d) must be >= 0 for '%s'", @@ -657,6 +669,7 @@ protected void validatePosition(long position) throws IOException { } if (size >= 0 && position >= size) { + GoogleCloudStorageEventBus.postOnException(); throw new EOFException( String.format( "Invalid seek offset: position value (%d) must be between 0 and %d for '%s'", @@ -748,6 +761,7 @@ protected void initMetadata(HttpHeaders headers) throws IOException { String generationString = headers.getFirstHeaderStringValue("x-goog-generation"); if (generationString == null) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException(String.format("Failed to retrieve generation for '%s'", resourceId)); } long generation = Long.parseLong(generationString); @@ -775,6 +789,7 @@ protected void initMetadata(@Nullable String encoding, long sizeFromMetadata, lo resourceId); gzipEncoded = nullToEmpty(encoding).contains(GZIP_ENCODING); if (gzipEncoded && !readOptions.isGzipEncodingSupportEnabled()) { + GoogleCloudStorageEventBus.postOnException(); throw new IOException( "Cannot read GZIP encoded files - content encoding support is disabled."); } @@ -825,6 +840,7 @@ private void cacheFooter(HttpResponse response) throws IOException { footerSize, resourceId); } catch (IOException e) { + GoogleCloudStorageEventBus.postOnException(); footerContent = null; throw e; } @@ -932,6 +948,7 @@ protected InputStream openStream(long bytesToRead) throws IOException { // TODO(b/110832992): validate response range header against expected/request range } catch (IOException e) { if (!metadataInitialized && errorExtractor.rangeNotSatisfiable(e) && currentPosition == 0) { + GoogleCloudStorageEventBus.postOnException(); // We don't know the size yet (metadataInitialized == false) and we're seeking to // byte 0, but got 'range not satisfiable'; the object must be empty. logger.atInfo().log( @@ -994,6 +1011,7 @@ protected InputStream openStream(long bytesToRead) throws IOException { } break; } catch (IOException footerException) { + GoogleCloudStorageEventBus.postOnException(); logger.atInfo().withCause(footerException).log( "Failed to prefetch footer (retry #%d/%d) for '%s'", retriesCount + 1, maxRetries, resourceId); @@ -1051,6 +1069,7 @@ protected InputStream openStream(long bytesToRead) throws IOException { return contentStream; } catch (IOException e) { + GoogleCloudStorageEventBus.postOnException(); try { response.disconnect(); } catch (IOException closeException) { @@ -1089,10 +1108,12 @@ private long getContentChannelPositionForFirstRead(long bytesToRead) { */ private HttpResponse handleExecuteMediaException(IOException e) throws IOException { if (errorExtractor.itemNotFound(e)) { + GoogleCloudStorageEventBus.postOnException(); throw createFileNotFoundException(resourceId, e); } String msg = String.format("Error reading '%s' at position %d", resourceId, currentPosition); if (errorExtractor.rangeNotSatisfiable(e)) { + GoogleCloudStorageEventBus.postOnException(); throw (EOFException) new EOFException(msg).initCause(e); } throw new IOException(msg, e); @@ -1144,6 +1165,7 @@ protected Storage.Objects.Get createMetadataRequest() throws IOException { /** Throws if this channel is not currently open. */ private void throwIfNotOpen() throws IOException { if (!isOpen()) { + GoogleCloudStorageEventBus.postOnException(); throw new ClosedChannelException(); } } diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageStatusStatistics.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageStatistics.java similarity index 65% rename from gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageStatusStatistics.java rename to gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageStatistics.java index 7bc6d37ff4..48d74d9fb1 100644 --- a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageStatusStatistics.java +++ b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageStatistics.java @@ -23,18 +23,34 @@ import com.google.common.collect.Maps; import java.util.EnumSet; -/** Statistics which are collected in GCS Client Side. */ -public enum GoogleCloudStorageStatusStatistics { +/** Statistics which are collected in GCS Connector */ +public enum GoogleCloudStorageStatistics { - /** Client-side Status Code statistics */ - GCS_CLIENT_RATE_LIMIT_COUNT("gcs_client_rate_limit_count", "Detects 429 Error", TYPE_COUNTER); + /** GCS connector specific statistics */ + GCS_REQUEST_COUNT( + "gcs_total_request_count", "Counts the total number of gcs requests made", TYPE_COUNTER), - public static final ImmutableSet VALUES = - ImmutableSet.copyOf(EnumSet.allOf(GoogleCloudStorageStatusStatistics.class)); + EXCEPTION_COUNT("exception_count", "Counts the number of exceptions encountered", TYPE_COUNTER), + + GCS_CLIENT_SIDE_ERROR_COUNT( + "gcs_client_side_error_count", + "Counts the occurrence of client side error status code", + TYPE_COUNTER), + + GCS_SERVER_SIDE_ERROR_COUNT( + "gcs_server_side_error_count", + "Counts the occurrence of server side error status code", + TYPE_COUNTER), + + GCS_CLIENT_RATE_LIMIT_COUNT( + "gcs_client_rate_limit_error_count", "Counts the occurence of 429 status code", TYPE_COUNTER); + + public static final ImmutableSet VALUES = + ImmutableSet.copyOf(EnumSet.allOf(GoogleCloudStorageStatistics.class)); /** A map used to support the {@link #fromSymbol(String)} call. */ - private static final ImmutableMap SYMBOL_MAP = - Maps.uniqueIndex(Iterators.forArray(values()), GoogleCloudStorageStatusStatistics::getSymbol); + private static final ImmutableMap SYMBOL_MAP = + Maps.uniqueIndex(Iterators.forArray(values()), GoogleCloudStorageStatistics::getSymbol); /** * Statistic definition. @@ -43,7 +59,7 @@ public enum GoogleCloudStorageStatusStatistics { * @param description description. * @param type type */ - GoogleCloudStorageStatusStatistics(String symbol, String description, StatisticTypeEnum type) { + GoogleCloudStorageStatistics(String symbol, String description, StatisticTypeEnum type) { this.symbol = symbol; this.description = description; this.type = type; @@ -69,7 +85,7 @@ public String getSymbol() { * @param symbol statistic to look up * @return the value or null. */ - public static GoogleCloudStorageStatusStatistics fromSymbol(String symbol) { + public static GoogleCloudStorageStatistics fromSymbol(String symbol) { return SYMBOL_MAP.get(symbol); } diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageWriteChannel.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageWriteChannel.java index b0d3dcfcfd..c511445bb4 100644 --- a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageWriteChannel.java +++ b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageWriteChannel.java @@ -24,6 +24,7 @@ import com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel; import com.google.cloud.hadoop.util.AsyncWriteChannelOptions; import com.google.cloud.hadoop.util.ClientRequestHelper; +import com.google.cloud.hadoop.util.GoogleCloudStorageEventBus; import com.google.cloud.hadoop.util.LoggingMediaHttpUploaderProgressListener; import java.io.IOException; import java.io.InputStream; @@ -173,6 +174,7 @@ public StorageObject call() throws Exception { try (InputStream ignore = pipeSource) { return uploadObject.execute(); } catch (IOException e) { + GoogleCloudStorageEventBus.postOnException(); StorageObject response = createResponseFromException(e); if (response == null) { throw e; diff --git a/util/src/main/java/com/google/cloud/hadoop/util/ApiErrorExtractor.java b/util/src/main/java/com/google/cloud/hadoop/util/ApiErrorExtractor.java index e737fcf2e5..595598d951 100644 --- a/util/src/main/java/com/google/cloud/hadoop/util/ApiErrorExtractor.java +++ b/util/src/main/java/com/google/cloud/hadoop/util/ApiErrorExtractor.java @@ -307,6 +307,8 @@ public static GoogleJsonResponseException getJsonResponseException(Throwable thr Throwable cause = throwable; while (cause != null) { if (cause instanceof GoogleJsonResponseException) { + GoogleCloudStorageEventBus.postOnGoogleJsonResponseException( + (GoogleJsonResponseException) cause); return (GoogleJsonResponseException) cause; } cause = cause.getCause(); @@ -319,6 +321,8 @@ public static HttpResponseException getHttpResponseException(Throwable throwable Throwable cause = throwable; while (cause != null) { if (cause instanceof HttpResponseException) { + + GoogleCloudStorageEventBus.postOnHttpResponseException((HttpResponseException) cause); return (HttpResponseException) cause; } cause = cause.getCause(); diff --git a/util/src/main/java/com/google/cloud/hadoop/util/GcsClientStatisticInterface.java b/util/src/main/java/com/google/cloud/hadoop/util/GcsClientStatisticInterface.java deleted file mode 100644 index c8839851fa..0000000000 --- a/util/src/main/java/com/google/cloud/hadoop/util/GcsClientStatisticInterface.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2023 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.hadoop.util; - -/** Base Interface for Status metrics */ -public interface GcsClientStatisticInterface { - public void statusMetricsUpdation(int statusCode); -} diff --git a/util/src/main/java/com/google/cloud/hadoop/util/GoogleCloudStorageEventBus.java b/util/src/main/java/com/google/cloud/hadoop/util/GoogleCloudStorageEventBus.java new file mode 100644 index 0000000000..760e1e85ce --- /dev/null +++ b/util/src/main/java/com/google/cloud/hadoop/util/GoogleCloudStorageEventBus.java @@ -0,0 +1,86 @@ +/* + * Copyright 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.hadoop.util; + +import com.google.api.client.googleapis.json.GoogleJsonResponseException; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.HttpResponseException; +import com.google.common.eventbus.EventBus; +import java.io.IOException; + +/** Event Bus class */ +public class GoogleCloudStorageEventBus { + + /** Hold the instance of the event bus here */ + private static EventBus eventBus = new EventBus(); + + private static IOException exception = new IOException(); + + /** + * Method to register an obj to event bus + * + * @param obj to register to event bus + */ + public static void register(Object obj) { + eventBus.register(obj); + } + + /** + * Posting GoogleJsonResponseException to invoke corresponding Subscriber method. + * + * @param response contains statusCode based on which metrics are updated in Subscriber method + */ + public static void postOnGoogleJsonResponseException(GoogleJsonResponseException response) { + eventBus.post(response); + } + + /** + * Posting HttpResponseException to invoke corresponding Subscriber method. + * + * @param response contains statusCode based on which metrics are updated in Subscriber method + */ + public static void postOnHttpResponseException(HttpResponseException response) { + eventBus.post(response); + } + + /** + * Posting HttpResponse to invoke corresponding Subscriber method. + * + * @param response contains statusCode based on which metrics are updated in Subscriber method + */ + public static void postOnHttpResponse(HttpResponse response) { + eventBus.post(response); + } + + /** + * Posting HttpRequest to invoke corresponding Subscriber method. + * + * @param request based on which metrics are updated in Subscriber method + */ + public static void postOnHttpRequest(HttpRequest request) { + eventBus.post(request); + } + + /** + * Posting Exception to invoke corresponding Subscriber method. Passing a dummy exception as + * EventBus has @ElementTypesAreNonnullByDefault annotation. + */ + public static void postOnException() { + eventBus.post(exception); + } +} diff --git a/util/src/main/java/com/google/cloud/hadoop/util/RetryHttpInitializer.java b/util/src/main/java/com/google/cloud/hadoop/util/RetryHttpInitializer.java index ecd28723f0..5cacb75579 100644 --- a/util/src/main/java/com/google/cloud/hadoop/util/RetryHttpInitializer.java +++ b/util/src/main/java/com/google/cloud/hadoop/util/RetryHttpInitializer.java @@ -62,25 +62,14 @@ public class RetryHttpInitializer implements HttpRequestInitializer { private final RetryHttpInitializerOptions options; - private final GcsClientStatisticInterface gcsClientStatisticInterface; - /** * @param credentials A credentials which will be used to initialize on HttpRequests and as the * delegate for a {@link UnsuccessfulResponseHandler}. * @param options An options that configure {@link RetryHttpInitializer} instance behaviour. - * @param gcsClientStatisticInterface To backport ghfsInstrumentation instances. */ - public RetryHttpInitializer( - Credentials credentials, - RetryHttpInitializerOptions options, - GcsClientStatisticInterface gcsClientStatisticInterface) { + public RetryHttpInitializer(Credentials credentials, RetryHttpInitializerOptions options) { this.credentials = credentials == null ? null : new HttpCredentialsAdapter(credentials); this.options = options; - this.gcsClientStatisticInterface = gcsClientStatisticInterface; - } - - public RetryHttpInitializer(Credentials credentials, RetryHttpInitializerOptions options) { - this(credentials, options, /* gcsClientStatisticInterface */ null); } @Override @@ -97,8 +86,7 @@ public void initialize(HttpRequest request) throws IOException { // Set the timeout configurations. .setConnectTimeout(toIntExact(options.getConnectTimeout().toMillis())) .setReadTimeout(toIntExact(options.getReadTimeout().toMillis())) - .setUnsuccessfulResponseHandler( - new UnsuccessfulResponseHandler(credentials, gcsClientStatisticInterface)) + .setUnsuccessfulResponseHandler(new UnsuccessfulResponseHandler(credentials)) .setIOExceptionHandler(new IoExceptionHandler()); HttpHeaders headers = request.getHeaders(); @@ -164,17 +152,11 @@ private static class UnsuccessfulResponseHandler implements HttpUnsuccessfulResp private final HttpCredentialsAdapter credentials; private final HttpBackOffUnsuccessfulResponseHandler delegate; - private final GcsClientStatisticInterface gcsClientStatisticInterface; - - public UnsuccessfulResponseHandler( - HttpCredentialsAdapter credentials, - GcsClientStatisticInterface gcsClientStatisticInterface) { + public UnsuccessfulResponseHandler(HttpCredentialsAdapter credentials) { this.credentials = credentials; this.delegate = new HttpBackOffUnsuccessfulResponseHandler(BACKOFF_BUILDER.build()) .setBackOffRequired(BACK_OFF_REQUIRED); - - this.gcsClientStatisticInterface = gcsClientStatisticInterface; } @Override @@ -200,10 +182,8 @@ public boolean handleResponse(HttpRequest request, HttpResponse response, boolea } private void logResponseCode(HttpRequest request, HttpResponse response) { - - if (gcsClientStatisticInterface != null) { - gcsClientStatisticInterface.statusMetricsUpdation(response.getStatusCode()); - } + // Incrementing GCS Static Statistics using status code of response. + GoogleCloudStorageEventBus.postOnHttpResponse(response); if (RESPONSE_CODES_TO_LOG.contains(response.getStatusCode())) { logger diff --git a/util/src/main/java/com/google/cloud/hadoop/util/interceptors/InvocationIdInterceptor.java b/util/src/main/java/com/google/cloud/hadoop/util/interceptors/InvocationIdInterceptor.java index b11efac502..24affed679 100644 --- a/util/src/main/java/com/google/cloud/hadoop/util/interceptors/InvocationIdInterceptor.java +++ b/util/src/main/java/com/google/cloud/hadoop/util/interceptors/InvocationIdInterceptor.java @@ -19,6 +19,7 @@ import com.google.api.client.http.HttpExecuteInterceptor; import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpRequest; +import com.google.cloud.hadoop.util.GoogleCloudStorageEventBus; import com.google.cloud.hadoop.util.ThreadTrace; import com.google.cloud.hadoop.util.TraceOperation; import com.google.common.annotations.VisibleForTesting; @@ -68,6 +69,7 @@ public void intercept(HttpRequest request) throws IOException { } else { newValue = invocationEntry; } + GoogleCloudStorageEventBus.postOnHttpRequest(request); headers.set(GOOG_API_CLIENT, newValue); ThreadTrace tt = TraceOperation.current();