Skip to content

Commit

Permalink
Split out a class for common file dependency key data and operations.
Browse files Browse the repository at this point in the history
These are shared between the serializer and deserializer.

PiperOrigin-RevId: 684427168
Change-Id: I1a5499757462732ef4f394bbaccfba93071b62d4
  • Loading branch information
aoeui authored and copybara-github committed Oct 10, 2024
1 parent efa0303 commit ef8edae
Show file tree
Hide file tree
Showing 3 changed files with 85 additions and 40 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ java_library(
"InvalidationDataReference.java",
],
deps = [
":file_dependency_key_support",
"//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
"//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster",
"//src/main/java/com/google/devtools/build/lib/skyframe:directory_listing_value",
Expand All @@ -77,6 +78,16 @@ java_library(
],
)

java_library(
name = "file_dependency_key_support",
srcs = ["FileDependencyKeySupport.java"],
deps = [
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
"//src/main/protobuf:file_invalidation_data_java_proto",
"//third_party:guava",
],
)

java_library(
name = "options",
srcs = ["RemoteAnalysisCachingOptions.java"],
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
// Copyright 2024 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.skyframe.serialization.analysis;

import static com.google.common.base.Preconditions.checkArgument;
import static java.nio.charset.StandardCharsets.UTF_8;

import com.google.devtools.build.lib.skyframe.serialization.proto.FileInvalidationData;
import com.google.devtools.build.lib.vfs.PathFragment;
import java.math.BigInteger;
import java.util.Arrays;
import java.util.Base64;

/** Constants and methods supporting {@link FileInvalidationData} keys. */
final class FileDependencyKeySupport {
static final int MAX_KEY_LENGTH = 250;
static final long MTSV_SENTINEL = -1;

private static final byte[] EMPTY_BYTES = new byte[0];

/**
* Neither {@link #FILE_KEY_DELIMITER} nor {@link #DIRECTORY_KEY_DELIMITER} are used in Base64,
* making them good delimiters for the Base64-encoded version numbers.
*
* <p>See comment at {@link FileInvalidationData} for more details.
*/
static final byte FILE_KEY_DELIMITER = (byte) ':';

static final byte DIRECTORY_KEY_DELIMITER = (byte) ';';

private static final Base64.Encoder ENCODER = Base64.getEncoder().withoutPadding();

static byte[] encodeMtsv(long mtsv) {
if (mtsv < 0) {
checkArgument(mtsv == MTSV_SENTINEL, mtsv);
return EMPTY_BYTES; // BigInteger.toByteArray is never empty so this is unique.
}
// Uses a BigInteger to trim leading 0 bytes.
return ENCODER.encode(BigInteger.valueOf(mtsv).toByteArray());
}

static String computeCacheKey(PathFragment path, long mtsv, byte delimiter) {
return computeCacheKey(path.getPathString(), mtsv, delimiter);
}

static String computeCacheKey(String path, long mtsv, byte delimiter) {
byte[] encodedMtsv = encodeMtsv(mtsv);
byte[] pathBytes = path.getBytes(UTF_8);

byte[] keyBytes = Arrays.copyOf(encodedMtsv, encodedMtsv.length + 1 + pathBytes.length);
keyBytes[encodedMtsv.length] = delimiter;
System.arraycopy(pathBytes, 0, keyBytes, encodedMtsv.length + 1, pathBytes.length);

return new String(keyBytes, UTF_8);
}

private FileDependencyKeySupport() {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,14 @@
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
import static com.google.devtools.build.lib.actions.FileStateType.SYMLINK;
import static com.google.devtools.build.lib.skyframe.serialization.analysis.FileDependencyKeySupport.DIRECTORY_KEY_DELIMITER;
import static com.google.devtools.build.lib.skyframe.serialization.analysis.FileDependencyKeySupport.FILE_KEY_DELIMITER;
import static com.google.devtools.build.lib.skyframe.serialization.analysis.FileDependencyKeySupport.MAX_KEY_LENGTH;
import static com.google.devtools.build.lib.skyframe.serialization.analysis.FileDependencyKeySupport.MTSV_SENTINEL;
import static com.google.devtools.build.lib.skyframe.serialization.analysis.FileDependencyKeySupport.computeCacheKey;
import static com.google.devtools.build.lib.vfs.RootedPath.toRootedPath;
import static java.nio.charset.StandardCharsets.UTF_8;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.devtools.build.lib.actions.FileStateValue;
Expand All @@ -41,32 +45,14 @@
import com.google.devtools.build.skyframe.InMemoryGraph;
import com.google.devtools.build.skyframe.InMemoryNodeEntry;
import com.google.devtools.build.skyframe.Version;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Consumer;
import javax.annotation.Nullable;

/** Records {@link FileValue.Key} and {@link DirectoryListingValue.Key} invalidation information. */
final class FileDependencySerializer {
@VisibleForTesting static final int MAX_KEY_LENGTH = 250;
private static final long MTSV_SENTINEL = -1;
private static final byte[] EMPTY_BYTES = new byte[0];

/**
* Neither {@link #FILE_KEY_DELIMITER} nor {@link #DIRECTORY_KEY_DELIMITER} are used in Base64,
* making them good delimiters for the Base64-encoded version numbers.
*
* <p>See comment at {@link FileInvalidationData} for more details.
*/
@VisibleForTesting static final byte FILE_KEY_DELIMITER = (byte) ':';

@VisibleForTesting static final byte DIRECTORY_KEY_DELIMITER = (byte) ';';

private static final Base64.Encoder ENCODER = Base64.getEncoder().withoutPadding();

private final VersionNumberExtractor versionExtractor;
private final InMemoryGraph graph;
Expand Down Expand Up @@ -303,27 +289,6 @@ private void processSymlinks(
}
}

@VisibleForTesting
static byte[] encodeMtsv(long mtsv) {
if (mtsv < 0) {
checkArgument(mtsv == MTSV_SENTINEL, mtsv);
return EMPTY_BYTES; // BigInteger.toByteArray is never empty so this is unique.
}
// Uses a BigInteger to trim leading 0 bytes.
return ENCODER.encode(BigInteger.valueOf(mtsv).toByteArray());
}

private String computeCacheKey(PathFragment path, long mtsv, byte delimiter) {
byte[] encodedMtsv = encodeMtsv(mtsv);
byte[] pathBytes = path.getPathString().getBytes(UTF_8);

byte[] keyBytes = Arrays.copyOf(encodedMtsv, encodedMtsv.length + 1 + pathBytes.length);
keyBytes[encodedMtsv.length] = delimiter;
System.arraycopy(pathBytes, 0, keyBytes, encodedMtsv.length + 1, pathBytes.length);

return new String(keyBytes, UTF_8);
}

private static DirentType getDirentType(FileValue value) {
if (!value.exists()) {
return DirentType.NON_EXISTENT;
Expand Down

0 comments on commit ef8edae

Please sign in to comment.