From 79c165648db6066e4b0e4e9982ae58f8b942f44f Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Tue, 14 Oct 2025 18:22:11 +0530 Subject: [PATCH 1/2] [POC] supporting nested directory structure for single shard data Signed-off-by: Varun Bansal --- .../index/store/FsDirectoryFactory.java | 20 +- .../distributed/DefaultFilenameHasher.java | 66 ++ .../store/distributed/DirectoryManager.java | 163 ++++ .../DistributedDirectoryException.java | 67 ++ .../DistributedDirectoryFactory.java | 152 ++++ .../DistributedDirectoryMetrics.java | 321 ++++++++ .../DistributedSegmentDirectory.java | 356 +++++++++ .../store/distributed/FilenameHasher.java | 40 + .../DefaultFilenameHasherTests.java | 117 +++ .../distributed/DirectoryManagerTests.java | 139 ++++ .../DistributedDirectoryExceptionTests.java | 95 +++ .../DistributedDirectoryFactoryTests.java | 225 ++++++ .../DistributedDirectoryMetricsTests.java | 192 +++++ ...ributedSegmentDirectoryBenchmarkTests.java | 185 +++++ ...tributedSegmentDirectoryEdgeCaseTests.java | 0 ...butedSegmentDirectoryIntegrationTests.java | 341 +++++++++ .../DistributedSegmentDirectoryTests.java | 719 ++++++++++++++++++ 17 files changed, 3190 insertions(+), 8 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/store/distributed/DefaultFilenameHasher.java create mode 100644 server/src/main/java/org/opensearch/index/store/distributed/DirectoryManager.java create mode 100644 server/src/main/java/org/opensearch/index/store/distributed/DistributedDirectoryException.java create mode 100644 server/src/main/java/org/opensearch/index/store/distributed/DistributedDirectoryFactory.java create mode 100644 server/src/main/java/org/opensearch/index/store/distributed/DistributedDirectoryMetrics.java create mode 100644 server/src/main/java/org/opensearch/index/store/distributed/DistributedSegmentDirectory.java create mode 100644 server/src/main/java/org/opensearch/index/store/distributed/FilenameHasher.java create mode 100644 server/src/test/java/org/opensearch/index/store/distributed/DefaultFilenameHasherTests.java create mode 100644 server/src/test/java/org/opensearch/index/store/distributed/DirectoryManagerTests.java create mode 100644 server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryExceptionTests.java create mode 100644 server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryFactoryTests.java create mode 100644 server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryMetricsTests.java create mode 100644 server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryBenchmarkTests.java create mode 100644 server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryEdgeCaseTests.java create mode 100644 server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryIntegrationTests.java create mode 100644 server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryTests.java diff --git a/server/src/main/java/org/opensearch/index/store/FsDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/FsDirectoryFactory.java index db1cc9e843e73..ae479b8947f8b 100644 --- a/server/src/main/java/org/opensearch/index/store/FsDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/FsDirectoryFactory.java @@ -49,6 +49,7 @@ import org.opensearch.index.IndexModule; import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.ShardPath; +import org.opensearch.index.store.distributed.DistributedSegmentDirectory; import org.opensearch.plugins.IndexStorePlugin; import java.io.IOException; @@ -96,15 +97,18 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index Set preLoadExtensions = new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING)); switch (type) { case HYBRIDFS: + // return new DistributedSegmentDirectory(null, location) // Use Lucene defaults - final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory); - final Set nioExtensions = new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_HYBRID_NIO_EXTENSIONS)); - if (primaryDirectory instanceof MMapDirectory) { - MMapDirectory mMapDirectory = (MMapDirectory) primaryDirectory; - return new HybridDirectory(lockFactory, setPreload(mMapDirectory, preLoadExtensions), nioExtensions); - } else { - return primaryDirectory; - } + final FSDirectory primaryDirectory = new NIOFSDirectory(location, lockFactory); + return new DistributedSegmentDirectory(primaryDirectory, location); + + // final Set nioExtensions = new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_HYBRID_NIO_EXTENSIONS)); + // if (primaryDirectory instanceof MMapDirectory) { + // MMapDirectory mMapDirectory = (MMapDirectory) primaryDirectory; + // return new HybridDirectory(lockFactory, setPreload(mMapDirectory, preLoadExtensions), nioExtensions); + // } else { + // return primaryDirectory; + // } case MMAPFS: return setPreload(new MMapDirectory(location, lockFactory), preLoadExtensions); // simplefs was removed in Lucene 9; support for enum is maintained for bwc diff --git a/server/src/main/java/org/opensearch/index/store/distributed/DefaultFilenameHasher.java b/server/src/main/java/org/opensearch/index/store/distributed/DefaultFilenameHasher.java new file mode 100644 index 0000000000000..0210cc5dc5d49 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/distributed/DefaultFilenameHasher.java @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import java.util.Set; + +/** + * Default implementation of FilenameHasher that uses consistent hashing + * to distribute segment files across multiple directories while keeping + * critical files like segments_N in the base directory. + * + * @opensearch.internal + */ +public class DefaultFilenameHasher implements FilenameHasher { + + private static final int NUM_DIRECTORIES = 5; + private static final Set EXCLUDED_PREFIXES = Set.of("segments_", "pending_segments_", "write.lock"); + + /** + * Maps a filename to a directory index using consistent hashing. + * Files with excluded prefixes (like segments_N) are always mapped to index 0. + * + * @param filename the segment filename to hash + * @return directory index between 0 and 4 (inclusive) + * @throws IllegalArgumentException if filename is null or empty + */ + @Override + public int getDirectoryIndex(String filename) { + if (filename == null || filename.isEmpty()) { + throw new IllegalArgumentException("Filename cannot be null or empty"); + } + + if (isExcludedFile(filename)) { + return 0; // Base directory for excluded files + } + + // Use consistent hashing with absolute value to ensure positive result + return Math.abs(filename.hashCode()) % NUM_DIRECTORIES; + } + + /** + * Checks if a filename should be excluded from distribution. + * Currently excludes files starting with "segments_" prefix. + * + * @param filename the filename to check + * @return true if file should remain in base directory, false if it can be distributed + */ + @Override + public boolean isExcludedFile(String filename) { + if (filename == null || filename.isEmpty()) { + return true; // Treat invalid filenames as excluded + } + + if (filename.endsWith(".tmp")) { + return true; + } + + return EXCLUDED_PREFIXES.stream().anyMatch(filename::startsWith); + } +} \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/index/store/distributed/DirectoryManager.java b/server/src/main/java/org/opensearch/index/store/distributed/DirectoryManager.java new file mode 100644 index 0000000000000..f723df035123c --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/distributed/DirectoryManager.java @@ -0,0 +1,163 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.FSLockFactory; +import org.apache.lucene.store.NIOFSDirectory; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Manages the creation and access to subdirectories for distributed segment storage. + * Creates up to 5 subdirectories for distributing segment files while keeping + * the base directory (index 0) for critical files like segments_N. + * + * @opensearch.internal + */ +public class DirectoryManager { + + private static final int NUM_DIRECTORIES = 5; + private final Path baseDirectory; + private final Directory[] subdirectories; + + /** + * Creates a new DirectoryManager with the specified base directory. + * + * @param baseDirectory the base Directory instance (used as subdirectory 0) + * @param basePath the base filesystem path for creating subdirectories + * @throws IOException if subdirectory creation fails + */ + public DirectoryManager(Directory baseDirectory, Path basePath) throws IOException { + this.baseDirectory = basePath; + this.subdirectories = createSubdirectories(baseDirectory, basePath); + } + + /** + * Creates the array of subdirectories, with index 0 being the base directory + * and indices 1-4 being newly created subdirectories. + * + * @param base the base Directory instance + * @param basePath the base filesystem path + * @return array of Directory instances + * @throws IOException if subdirectory creation fails + */ + private Directory[] createSubdirectories(Directory base, Path basePath) throws IOException { + Directory[] dirs = new Directory[NUM_DIRECTORIES]; + dirs[0] = base; // Base directory for segments_N and excluded files + + try { + for (int i = 1; i < NUM_DIRECTORIES; i++) { + Path subPath = basePath.resolve("varun_segments_" + i); + + // Create directory if it doesn't exist + if (!Files.exists(subPath)) { + Files.createDirectories(subPath); + } + + // Validate directory is writable + if (!Files.isWritable(subPath)) { + throw new IOException("Subdirectory is not writable: " + subPath); + } + + dirs[i] = new NIOFSDirectory(subPath, FSLockFactory.getDefault()); + } + } catch (IOException e) { + // Clean up any successfully created directories + closeDirectories(dirs); + throw new DistributedDirectoryException( + "Failed to create subdirectories", + -1, + "createSubdirectories", + e + ); + } + + return dirs; + } + + /** + * Gets the Directory instance for the specified index. + * + * @param index the directory index (0-4) + * @return the Directory instance + * @throws IllegalArgumentException if index is out of range + */ + public Directory getDirectory(int index) { + if (index < 0 || index >= NUM_DIRECTORIES) { + throw new IllegalArgumentException("Directory index must be between 0 and " + (NUM_DIRECTORIES - 1) + + ", got: " + index); + } + return subdirectories[index]; + } + + /** + * Gets the number of managed directories. + * + * @return the number of directories (always 5) + */ + public int getNumDirectories() { + return NUM_DIRECTORIES; + } + + /** + * Gets the base filesystem path. + * + * @return the base path + */ + public Path getBasePath() { + return baseDirectory; + } + + /** + * Closes all managed directories except the base directory (index 0). + * The base directory should be closed by the caller since it was provided + * during construction. + * + * @throws IOException if any directory fails to close + */ + public void close() throws IOException { + closeDirectories(subdirectories); + } + + /** + * Helper method to close directories and collect exceptions. + * + * @param dirs array of directories to close + * @throws IOException if any directory fails to close + */ + private void closeDirectories(Directory[] dirs) throws IOException { + IOException exception = null; + + // Close subdirectories (skip index 0 as it's the base directory managed externally) + for (int i = 1; i < dirs.length && dirs[i] != null; i++) { + try { + dirs[i].close(); + } catch (IOException e) { + if (exception == null) { + exception = new DistributedDirectoryException( + "Failed to close subdirectory", + i, + "close", + e + ); + } else { + exception.addSuppressed(e); + } + } + } + + if (exception != null) { + throw exception; + } + } +} \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/index/store/distributed/DistributedDirectoryException.java b/server/src/main/java/org/opensearch/index/store/distributed/DistributedDirectoryException.java new file mode 100644 index 0000000000000..a8d49a2fee2b8 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/distributed/DistributedDirectoryException.java @@ -0,0 +1,67 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import java.io.IOException; + +/** + * Exception thrown when operations on a distributed segment directory fail. + * This exception provides additional context about which directory index + * and operation caused the failure to aid in debugging and monitoring. + * + * @opensearch.internal + */ +public class DistributedDirectoryException extends IOException { + + private final int directoryIndex; + private final String operation; + + /** + * Creates a new DistributedDirectoryException with directory context. + * + * @param message the error message + * @param directoryIndex the index of the directory where the error occurred (0-4) + * @param operation the operation that was being performed when the error occurred + * @param cause the underlying cause of the exception + */ + public DistributedDirectoryException(String message, int directoryIndex, String operation, Throwable cause) { + super(String.format("Directory %d operation '%s' failed: %s", directoryIndex, operation, message), cause); + this.directoryIndex = directoryIndex; + this.operation = operation; + } + + /** + * Creates a new DistributedDirectoryException with directory context. + * + * @param message the error message + * @param directoryIndex the index of the directory where the error occurred (0-4) + * @param operation the operation that was being performed when the error occurred + */ + public DistributedDirectoryException(String message, int directoryIndex, String operation) { + this(message, directoryIndex, operation, null); + } + + /** + * Gets the directory index where the error occurred. + * + * @return the directory index (0-4) + */ + public int getDirectoryIndex() { + return directoryIndex; + } + + /** + * Gets the operation that was being performed when the error occurred. + * + * @return the operation name + */ + public String getOperation() { + return operation; + } +} \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/index/store/distributed/DistributedDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/distributed/DistributedDirectoryFactory.java new file mode 100644 index 0000000000000..8bfa25fa48216 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/distributed/DistributedDirectoryFactory.java @@ -0,0 +1,152 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.store.Directory; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.index.shard.ShardId; + +import java.io.IOException; +import java.nio.file.Path; + +/** + * Factory for creating DistributedSegmentDirectory instances with configuration support. + * Provides integration with OpenSearch settings and can be used to enable/disable + * distributed storage based on configuration. + * + * @opensearch.internal + */ +public class DistributedDirectoryFactory { + + private static final Logger logger = LogManager.getLogger(DistributedDirectoryFactory.class); + + // Configuration keys + public static final String DISTRIBUTED_ENABLED_SETTING = "index.store.distributed.enabled"; + public static final String DISTRIBUTED_SUBDIRECTORIES_SETTING = "index.store.distributed.subdirectories"; + public static final String DISTRIBUTED_HASH_ALGORITHM_SETTING = "index.store.distributed.hash_algorithm"; + + // Default values + public static final boolean DEFAULT_DISTRIBUTED_ENABLED = true; + public static final int DEFAULT_SUBDIRECTORIES = 5; + public static final String DEFAULT_HASH_ALGORITHM = "default"; + + private final Settings settings; + + /** + * Creates a new DistributedDirectoryFactory with the given settings. + * + * @param settings the OpenSearch settings + */ + public DistributedDirectoryFactory(Settings settings) { + this.settings = settings; + } + + /** + * Creates a Directory instance, either distributed or the original delegate + * based on configuration settings. + * + * @param delegate the base Directory instance + * @param basePath the base filesystem path + * @param shardId the shard identifier (for logging) + * @return Directory instance (distributed or original delegate) + * @throws IOException if directory creation fails + */ + public Directory createDirectory(Directory delegate, Path basePath, ShardId shardId) throws IOException { + boolean distributedEnabled = settings.getAsBoolean(DISTRIBUTED_ENABLED_SETTING, DEFAULT_DISTRIBUTED_ENABLED); + + if (!distributedEnabled) { + logger.debug("Distributed storage disabled for shard {}, using delegate directory", shardId); + return delegate; + } + + try { + FilenameHasher hasher = createHasher(); + DistributedSegmentDirectory distributedDirectory = new DistributedSegmentDirectory( + delegate, basePath, hasher + ); + + logger.info("Created distributed segment directory for shard {} at path: {}", shardId, basePath); + return distributedDirectory; + + } catch (IOException e) { + logger.error("Failed to create distributed directory for shard {}, falling back to delegate: {}", + shardId, e.getMessage()); + // Fall back to original directory if distributed creation fails + return delegate; + } + } + + /** + * Creates a Directory instance with default settings (distributed disabled). + * + * @param delegate the base Directory instance + * @param basePath the base filesystem path + * @return Directory instance (usually the original delegate) + * @throws IOException if directory creation fails + */ + public Directory createDirectory(Directory delegate, Path basePath) throws IOException { + return createDirectory(delegate, basePath, null); + } + + /** + * Creates a FilenameHasher based on configuration settings. + * + * @return FilenameHasher instance + */ + private FilenameHasher createHasher() { + String hashAlgorithm = settings.get(DISTRIBUTED_HASH_ALGORITHM_SETTING, DEFAULT_HASH_ALGORITHM); + + switch (hashAlgorithm.toLowerCase()) { + case "default": + return new DefaultFilenameHasher(); + default: + logger.warn("Unknown hash algorithm '{}', using default", hashAlgorithm); + return new DefaultFilenameHasher(); + } + } + + /** + * Checks if distributed storage is enabled in the settings. + * + * @return true if distributed storage is enabled + */ + public boolean isDistributedEnabled() { + return settings.getAsBoolean(DISTRIBUTED_ENABLED_SETTING, DEFAULT_DISTRIBUTED_ENABLED); + } + + /** + * Gets the configured number of subdirectories. + * + * @return number of subdirectories + */ + public int getNumSubdirectories() { + return settings.getAsInt(DISTRIBUTED_SUBDIRECTORIES_SETTING, DEFAULT_SUBDIRECTORIES); + } + + /** + * Gets the configured hash algorithm. + * + * @return hash algorithm name + */ + public String getHashAlgorithm() { + return settings.get(DISTRIBUTED_HASH_ALGORITHM_SETTING, DEFAULT_HASH_ALGORITHM); + } + + /** + * Creates a new factory instance with updated settings. + * + * @param newSettings the new settings + * @return new factory instance + */ + public DistributedDirectoryFactory withSettings(Settings newSettings) { + return new DistributedDirectoryFactory(newSettings); + } +} \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/index/store/distributed/DistributedDirectoryMetrics.java b/server/src/main/java/org/opensearch/index/store/distributed/DistributedDirectoryMetrics.java new file mode 100644 index 0000000000000..d878c2f463a3b --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/distributed/DistributedDirectoryMetrics.java @@ -0,0 +1,321 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; + +/** + * Metrics collection for DistributedSegmentDirectory operations. + * Tracks file operations, distribution patterns, and performance metrics + * across all subdirectories. + * + * @opensearch.internal + */ +public class DistributedDirectoryMetrics { + + private static final Logger logger = LogManager.getLogger(DistributedDirectoryMetrics.class); + private static final int NUM_DIRECTORIES = 5; + + // Operation counters per directory + private final LongAdder[] fileOperationsByDirectory; + private final LongAdder[] openInputOperations; + private final LongAdder[] createOutputOperations; + private final LongAdder[] deleteFileOperations; + private final LongAdder[] fileLengthOperations; + + // Timing metrics per directory (in nanoseconds) + private final LongAdder[] totalOperationTimeByDirectory; + private final AtomicLong[] maxOperationTimeByDirectory; + private final AtomicLong[] minOperationTimeByDirectory; + + // Error counters + private final LongAdder[] errorsByDirectory; + private final LongAdder totalErrors; + + // Distribution metrics + private final LongAdder totalOperations; + private final AtomicLong startTime; + + /** + * Creates a new DistributedDirectoryMetrics instance. + */ + public DistributedDirectoryMetrics() { + this.fileOperationsByDirectory = new LongAdder[NUM_DIRECTORIES]; + this.openInputOperations = new LongAdder[NUM_DIRECTORIES]; + this.createOutputOperations = new LongAdder[NUM_DIRECTORIES]; + this.deleteFileOperations = new LongAdder[NUM_DIRECTORIES]; + this.fileLengthOperations = new LongAdder[NUM_DIRECTORIES]; + + this.totalOperationTimeByDirectory = new LongAdder[NUM_DIRECTORIES]; + this.maxOperationTimeByDirectory = new AtomicLong[NUM_DIRECTORIES]; + this.minOperationTimeByDirectory = new AtomicLong[NUM_DIRECTORIES]; + + this.errorsByDirectory = new LongAdder[NUM_DIRECTORIES]; + this.totalErrors = new LongAdder(); + this.totalOperations = new LongAdder(); + this.startTime = new AtomicLong(System.currentTimeMillis()); + + // Initialize arrays + for (int i = 0; i < NUM_DIRECTORIES; i++) { + this.fileOperationsByDirectory[i] = new LongAdder(); + this.openInputOperations[i] = new LongAdder(); + this.createOutputOperations[i] = new LongAdder(); + this.deleteFileOperations[i] = new LongAdder(); + this.fileLengthOperations[i] = new LongAdder(); + + this.totalOperationTimeByDirectory[i] = new LongAdder(); + this.maxOperationTimeByDirectory[i] = new AtomicLong(0); + this.minOperationTimeByDirectory[i] = new AtomicLong(Long.MAX_VALUE); + + this.errorsByDirectory[i] = new LongAdder(); + } + } + + /** + * Records a file operation with timing information. + * + * @param directoryIndex the directory index (0-4) + * @param operation the operation type + * @param durationNanos the operation duration in nanoseconds + */ + public void recordFileOperation(int directoryIndex, String operation, long durationNanos) { + if (directoryIndex < 0 || directoryIndex >= NUM_DIRECTORIES) { + logger.warn("Invalid directory index: {}", directoryIndex); + return; + } + + // Update counters + fileOperationsByDirectory[directoryIndex].increment(); + totalOperations.increment(); + + // Update operation-specific counters + switch (operation.toLowerCase()) { + case "openinput": + openInputOperations[directoryIndex].increment(); + break; + case "createoutput": + createOutputOperations[directoryIndex].increment(); + break; + case "deletefile": + deleteFileOperations[directoryIndex].increment(); + break; + case "filelength": + fileLengthOperations[directoryIndex].increment(); + break; + } + + // Update timing metrics + totalOperationTimeByDirectory[directoryIndex].add(durationNanos); + + // Update min/max times + long currentMax = maxOperationTimeByDirectory[directoryIndex].get(); + if (durationNanos > currentMax) { + maxOperationTimeByDirectory[directoryIndex].compareAndSet(currentMax, durationNanos); + } + + long currentMin = minOperationTimeByDirectory[directoryIndex].get(); + if (durationNanos < currentMin) { + minOperationTimeByDirectory[directoryIndex].compareAndSet(currentMin, durationNanos); + } + + logger.debug("Recorded {} operation in directory {} took {}ns", operation, directoryIndex, durationNanos); + } + + /** + * Records an error for a specific directory. + * + * @param directoryIndex the directory index (0-4) + * @param operation the operation that failed + */ + public void recordError(int directoryIndex, String operation) { + if (directoryIndex < 0 || directoryIndex >= NUM_DIRECTORIES) { + logger.warn("Invalid directory index for error: {}", directoryIndex); + return; + } + + errorsByDirectory[directoryIndex].increment(); + totalErrors.increment(); + + logger.debug("Recorded error for {} operation in directory {}", operation, directoryIndex); + } + + /** + * Gets the total number of operations for a specific directory. + * + * @param directoryIndex the directory index (0-4) + * @return the operation count + */ + public long getOperationCount(int directoryIndex) { + if (directoryIndex < 0 || directoryIndex >= NUM_DIRECTORIES) { + return 0; + } + return fileOperationsByDirectory[directoryIndex].sum(); + } + + /** + * Gets the total number of operations across all directories. + * + * @return the total operation count + */ + public long getTotalOperations() { + return totalOperations.sum(); + } + + /** + * Gets the error count for a specific directory. + * + * @param directoryIndex the directory index (0-4) + * @return the error count + */ + public long getErrorCount(int directoryIndex) { + if (directoryIndex < 0 || directoryIndex >= NUM_DIRECTORIES) { + return 0; + } + return errorsByDirectory[directoryIndex].sum(); + } + + /** + * Gets the total error count across all directories. + * + * @return the total error count + */ + public long getTotalErrors() { + return totalErrors.sum(); + } + + /** + * Gets the average operation time for a specific directory in nanoseconds. + * + * @param directoryIndex the directory index (0-4) + * @return the average operation time in nanoseconds, or 0 if no operations + */ + public long getAverageOperationTime(int directoryIndex) { + if (directoryIndex < 0 || directoryIndex >= NUM_DIRECTORIES) { + return 0; + } + + long totalTime = totalOperationTimeByDirectory[directoryIndex].sum(); + long operationCount = fileOperationsByDirectory[directoryIndex].sum(); + + return operationCount > 0 ? totalTime / operationCount : 0; + } + + /** + * Gets the maximum operation time for a specific directory in nanoseconds. + * + * @param directoryIndex the directory index (0-4) + * @return the maximum operation time in nanoseconds + */ + public long getMaxOperationTime(int directoryIndex) { + if (directoryIndex < 0 || directoryIndex >= NUM_DIRECTORIES) { + return 0; + } + + long max = maxOperationTimeByDirectory[directoryIndex].get(); + return max == 0 ? 0 : max; + } + + /** + * Gets the minimum operation time for a specific directory in nanoseconds. + * + * @param directoryIndex the directory index (0-4) + * @return the minimum operation time in nanoseconds + */ + public long getMinOperationTime(int directoryIndex) { + if (directoryIndex < 0 || directoryIndex >= NUM_DIRECTORIES) { + return 0; + } + + long min = minOperationTimeByDirectory[directoryIndex].get(); + return min == Long.MAX_VALUE ? 0 : min; + } + + /** + * Gets the distribution of operations across directories as percentages. + * + * @return array of percentages (0-100) for each directory + */ + public double[] getDistributionPercentages() { + double[] percentages = new double[NUM_DIRECTORIES]; + long total = getTotalOperations(); + + if (total == 0) { + return percentages; // All zeros + } + + for (int i = 0; i < NUM_DIRECTORIES; i++) { + percentages[i] = (getOperationCount(i) * 100.0) / total; + } + + return percentages; + } + + /** + * Gets the uptime in milliseconds since metrics collection started. + * + * @return uptime in milliseconds + */ + public long getUptimeMillis() { + return System.currentTimeMillis() - startTime.get(); + } + + /** + * Resets all metrics to zero. + */ + public void reset() { + for (int i = 0; i < NUM_DIRECTORIES; i++) { + fileOperationsByDirectory[i].reset(); + openInputOperations[i].reset(); + createOutputOperations[i].reset(); + deleteFileOperations[i].reset(); + fileLengthOperations[i].reset(); + + totalOperationTimeByDirectory[i].reset(); + maxOperationTimeByDirectory[i].set(0); + minOperationTimeByDirectory[i].set(Long.MAX_VALUE); + + errorsByDirectory[i].reset(); + } + + totalErrors.reset(); + totalOperations.reset(); + startTime.set(System.currentTimeMillis()); + + logger.info("Reset all distributed directory metrics"); + } + + /** + * Returns a summary string of current metrics. + * + * @return metrics summary + */ + public String getSummary() { + StringBuilder sb = new StringBuilder(); + sb.append("DistributedDirectory Metrics Summary:\n"); + sb.append("Total Operations: ").append(getTotalOperations()).append("\n"); + sb.append("Total Errors: ").append(getTotalErrors()).append("\n"); + sb.append("Uptime: ").append(getUptimeMillis()).append("ms\n"); + + double[] distribution = getDistributionPercentages(); + sb.append("Distribution by Directory:\n"); + for (int i = 0; i < NUM_DIRECTORIES; i++) { + sb.append(" Directory ").append(i).append(": ") + .append(getOperationCount(i)).append(" ops (") + .append(String.format("%.1f", distribution[i])).append("%), ") + .append(getErrorCount(i)).append(" errors, ") + .append("avg: ").append(getAverageOperationTime(i) / 1_000_000).append("ms\n"); + } + + return sb.toString(); + } +} \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/index/store/distributed/DistributedSegmentDirectory.java b/server/src/main/java/org/opensearch/index/store/distributed/DistributedSegmentDirectory.java new file mode 100644 index 0000000000000..62c7ffec9e9b1 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/distributed/DistributedSegmentDirectory.java @@ -0,0 +1,356 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FilterDirectory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * A Directory implementation that distributes segment files across multiple + * subdirectories + * based on filename hashing. This helps improve I/O distribution by spreading + * file access + * across multiple storage paths while maintaining full Lucene Directory + * compatibility. + * + * Critical files like segments_N are kept in the base directory to maintain + * compatibility + * with existing Lucene expectations. + * + * @opensearch.internal + */ +public class DistributedSegmentDirectory extends FilterDirectory { + + private static final Logger logger = LogManager.getLogger(DistributedSegmentDirectory.class); + + private final FilenameHasher hasher; + private final DirectoryManager directoryManager; + + /** + * Creates a new DistributedSegmentDirectory with default filename hasher. + * + * @param delegate the base Directory instance + * @param basePath the base filesystem path for creating subdirectories + * @throws IOException if subdirectory creation fails + */ + public DistributedSegmentDirectory(Directory delegate, Path basePath) throws IOException { + this(delegate, basePath, new DefaultFilenameHasher()); + } + + /** + * Creates a new DistributedSegmentDirectory with custom filename hasher. + * + * @param delegate the base Directory instance + * @param basePath the base filesystem path for creating subdirectories + * @param hasher the FilenameHasher implementation to use + * @throws IOException if subdirectory creation fails + */ + public DistributedSegmentDirectory(Directory delegate, Path basePath, FilenameHasher hasher) throws IOException { + super(delegate); + this.hasher = hasher; + this.directoryManager = new DirectoryManager(delegate, basePath); + + logger.info("Created DistributedSegmentDirectory with {} subdirectories at path: {}", + directoryManager.getNumDirectories(), basePath); + } + + /** + * Resolves the appropriate directory for a given filename using the hasher. + * + * @param filename the filename to resolve + * @return the Directory instance that should handle this file + */ + protected Directory resolveDirectory(String filename) { + int directoryIndex = hasher.getDirectoryIndex(filename); + return directoryManager.getDirectory(directoryIndex); + } + + /** + * Gets the directory index for a filename (useful for logging and debugging). + * + * @param filename the filename to check + * @return the directory index (0-4) + */ + protected int getDirectoryIndex(String filename) { + return hasher.getDirectoryIndex(filename); + } + + /** + * Gets the DirectoryManager instance (useful for testing). + * + * @return the DirectoryManager + */ + protected DirectoryManager getDirectoryManager() { + return directoryManager; + } + + /** + * Gets the FilenameHasher instance (useful for testing). + * + * @return the FilenameHasher + */ + protected FilenameHasher getHasher() { + return hasher; + } + + @Override + public void close() throws IOException { + IOException exception = null; + + // Close the directory manager (which closes subdirectories) + try { + directoryManager.close(); + } catch (IOException e) { + exception = e; + } + + // Close the base directory + try { + super.close(); + } catch (IOException e) { + if (exception == null) { + exception = e; + } else { + exception.addSuppressed(e); + } + } + + if (exception != null) { + throw exception; + } + + logger.debug("Closed DistributedSegmentDirectory"); + } + + // Placeholder methods - will be implemented in subsequent tasks + + @Override + public String[] listAll() throws IOException { + Set allFiles = new HashSet<>(); + IOException lastException = null; + + // Collect files from all subdirectories + for (int i = 0; i < directoryManager.getNumDirectories(); i++) { + try { + Directory dir = directoryManager.getDirectory(i); + String[] files = dir.listAll(); + + // Filter out subdirectory names from base directory (index 0) + if (i == 0) { + for (String file : files) { + // Only add files that are not our created subdirectories + if (!file.startsWith("varun_segments_") || !isSubdirectoryName(file)) { + allFiles.add(file); + } + } + } else { + // For other directories, add all files + Collections.addAll(allFiles, files); + } + + logger.info("Listed {} files from directory {} (filtered: {})", + files.length, i, i == 0 ? "yes" : "no"); + } catch (IOException e) { + logger.warn("Failed to list files from directory {}: {}", i, e.getMessage()); + lastException = new DistributedDirectoryException( + "Failed to list files from directory", + i, + "listAll", + e); + // Continue trying other directories + } + } + + // If we couldn't list any directory and have an exception, throw it + if (allFiles.isEmpty() && lastException != null) { + throw lastException; + } + + String[] result = allFiles.toArray(new String[0]); + logger.info("Listed total {} unique files across all directories", result.length); + return result; + } + + /** + * Checks if a filename matches our subdirectory naming pattern. + * Our subdirectories are named "segments_1", "segments_2", etc. + * + * @param filename the filename to check + * @return true if this is one of our subdirectory names + */ + private boolean isSubdirectoryName(String filename) { + if (!filename.startsWith("varun_segments_")) { + return false; + } + + String suffix = filename.substring("varun_segments_".length()); + try { + int dirIndex = Integer.parseInt(suffix); + // Check if this matches our subdirectory naming pattern (1-4) + return dirIndex >= 1 && dirIndex < directoryManager.getNumDirectories(); + } catch (NumberFormatException e) { + // If it's not a number, it's a real segments file, not our subdirectory + return false; + } + } + + @Override + public IndexInput openInput(String name, IOContext context) throws IOException { + int dirIndex = getDirectoryIndex(name); + Directory targetDir = directoryManager.getDirectory(dirIndex); + + logger.info("Opening input for file {} in directory {}", name, dirIndex); + + try { + return targetDir.openInput(name, context); + } catch (IOException e) { + throw new DistributedDirectoryException( + "Failed to open input for file: " + name, + dirIndex, + "openInput", + e); + } + } + + @Override + public IndexOutput createOutput(String name, IOContext context) throws IOException { + int dirIndex = getDirectoryIndex(name); + Directory targetDir = directoryManager.getDirectory(dirIndex); + + logger.info("Creating output for file {} in directory {}", name, dirIndex); + + try { + return targetDir.createOutput(name, context); + } catch (IOException e) { + throw new DistributedDirectoryException( + "Failed to create output for file: " + name, + dirIndex, + "createOutput", + e); + } + } + + @Override + public void deleteFile(String name) throws IOException { + int dirIndex = getDirectoryIndex(name); + Directory targetDir = directoryManager.getDirectory(dirIndex); + + logger.info("Deleting file {} from directory {}", name, dirIndex); + + try { + targetDir.deleteFile(name); + } catch (IOException e) { + throw new DistributedDirectoryException( + "Failed to delete file: " + name, + dirIndex, + "deleteFile", + e); + } + } + + @Override + public long fileLength(String name) throws IOException { + int dirIndex = getDirectoryIndex(name); + Directory targetDir = directoryManager.getDirectory(dirIndex); + + try { + return targetDir.fileLength(name); + } catch (IOException e) { + throw new DistributedDirectoryException( + "Failed to get file length for: " + name, + dirIndex, + "fileLength", + e); + } + } + + @Override + public void sync(Collection names) throws IOException { + // Group files by directory and sync each directory separately + Map> filesByDirectory = names.stream() + .collect(Collectors.groupingBy(this::getDirectoryIndex)); + + List exceptions = new ArrayList<>(); + + for (Map.Entry> entry : filesByDirectory.entrySet()) { + int dirIndex = entry.getKey(); + List files = entry.getValue(); + + try { + Directory dir = directoryManager.getDirectory(dirIndex); + dir.sync(files); + logger.info("Synced {} files in directory {}", files.size(), dirIndex); + } catch (IOException e) { + logger.warn("Failed to sync {} files in directory {}: {}", files.size(), dirIndex, e.getMessage()); + exceptions.add(new DistributedDirectoryException( + "Failed to sync files: " + files, + dirIndex, + "sync", + e)); + } + } + + // If any sync operations failed, throw the first exception with others as + // suppressed + if (!exceptions.isEmpty()) { + IOException primaryException = exceptions.get(0); + for (int i = 1; i < exceptions.size(); i++) { + primaryException.addSuppressed(exceptions.get(i)); + } + throw primaryException; + } + + logger.info("Successfully synced {} files across {} directories", + names.size(), filesByDirectory.size()); + } + + @Override + public void rename(String source, String dest) throws IOException { + int sourceIndex = getDirectoryIndex(source); + int destIndex = getDirectoryIndex(dest); + + if (sourceIndex != destIndex) { + // Cross-directory rename - not supported atomically + throw new DistributedDirectoryException( + "Cross-directory rename not supported: " + source + " (dir " + sourceIndex + + ") -> " + dest + " (dir " + destIndex + ")", + sourceIndex, + "rename"); + } + + Directory targetDir = directoryManager.getDirectory(sourceIndex); + + try { + targetDir.rename(source, dest); + logger.info("Renamed {} to {} in directory {}", source, dest, sourceIndex); + } catch (IOException e) { + throw new DistributedDirectoryException( + "Failed to rename " + source + " to " + dest, + sourceIndex, + "rename", + e); + } + } +} \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/index/store/distributed/FilenameHasher.java b/server/src/main/java/org/opensearch/index/store/distributed/FilenameHasher.java new file mode 100644 index 0000000000000..1b24894416836 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/distributed/FilenameHasher.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +/** + * Interface for mapping filenames to directory indices in a distributed segment directory. + * Implementations should provide consistent hashing to ensure the same filename always + * maps to the same directory index across multiple invocations. + * + * @opensearch.internal + */ +public interface FilenameHasher { + + /** + * Maps a filename to a directory index (0-4 for 5 directories). + * This method must be deterministic - the same filename should always + * return the same directory index. + * + * @param filename the segment filename to hash + * @return directory index between 0 and 4 (inclusive) + * @throws IllegalArgumentException if filename is null or empty + */ + int getDirectoryIndex(String filename); + + /** + * Checks if a filename should be excluded from distribution and kept + * in the base directory (index 0). Typically used for critical files + * like segments_N that must remain in their expected location. + * + * @param filename the filename to check + * @return true if file should remain in base directory, false if it can be distributed + */ + boolean isExcludedFile(String filename); +} \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/DefaultFilenameHasherTests.java b/server/src/test/java/org/opensearch/index/store/distributed/DefaultFilenameHasherTests.java new file mode 100644 index 0000000000000..b4fcfb5d996a8 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/store/distributed/DefaultFilenameHasherTests.java @@ -0,0 +1,117 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.opensearch.test.OpenSearchTestCase; + +import java.util.HashMap; +import java.util.Map; + +public class DefaultFilenameHasherTests extends OpenSearchTestCase { + + private DefaultFilenameHasher hasher; + + @Override + public void setUp() throws Exception { + super.setUp(); + hasher = new DefaultFilenameHasher(); + } + + public void testConsistentHashing() { + String filename = "_0.cfe"; + int firstResult = hasher.getDirectoryIndex(filename); + + // Call multiple times to ensure consistency + for (int i = 0; i < 10; i++) { + assertEquals("Hash should be consistent across multiple calls", firstResult, hasher.getDirectoryIndex(filename)); + } + } + + public void testExcludedFiles() { + // Test segments_N files are excluded + assertTrue("segments_1 should be excluded", hasher.isExcludedFile("segments_1")); + assertTrue("segments_10 should be excluded", hasher.isExcludedFile("segments_10")); + assertEquals("segments_1 should map to directory 0", 0, hasher.getDirectoryIndex("segments_1")); + assertEquals("segments_10 should map to directory 0", 0, hasher.getDirectoryIndex("segments_10")); + + // Test regular files are not excluded + assertFalse("_0.cfe should not be excluded", hasher.isExcludedFile("_0.cfe")); + assertFalse("_1.si should not be excluded", hasher.isExcludedFile("_1.si")); + } + + public void testDirectoryIndexRange() { + String[] testFiles = { + "_0.cfe", "_0.cfs", "_0.si", "_0.fnm", "_0.fdt", + "_1.tim", "_1.tip", "_1.doc", "_1.pos", "_1.pay", + "_2.dvd", "_2.dvm", "_3.fdx", "_4.nvd", "_5.nvm" + }; + + for (String filename : testFiles) { + int index = hasher.getDirectoryIndex(filename); + assertTrue("Directory index should be between 0 and 4, got " + index + " for " + filename, + index >= 0 && index < 5); + } + } + + public void testDistribution() { + String[] testFiles = { + "_0.cfe", "_0.cfs", "_0.si", "_0.fnm", "_0.fdt", + "_1.tim", "_1.tip", "_1.doc", "_1.pos", "_1.pay", + "_2.dvd", "_2.dvm", "_3.fdx", "_4.nvd", "_5.nvm", + "_6.cfe", "_7.cfs", "_8.si", "_9.fnm", "_10.fdt" + }; + + Map distribution = new HashMap<>(); + for (String filename : testFiles) { + int index = hasher.getDirectoryIndex(filename); + distribution.put(index, distribution.getOrDefault(index, 0) + 1); + } + + // Should use multiple directories (not all in one) + assertTrue("Files should be distributed across multiple directories, got: " + distribution, + distribution.size() > 1); + } + + public void testNullAndEmptyFilenames() { + // Test null filename + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> hasher.getDirectoryIndex(null)); + assertEquals("Filename cannot be null or empty", exception.getMessage()); + + // Test empty filename + exception = expectThrows(IllegalArgumentException.class, + () -> hasher.getDirectoryIndex("")); + assertEquals("Filename cannot be null or empty", exception.getMessage()); + + // Test isExcludedFile with null/empty + assertTrue("null filename should be excluded", hasher.isExcludedFile(null)); + assertTrue("empty filename should be excluded", hasher.isExcludedFile("")); + } + + public void testSpecificFileDistribution() { + // Test some specific files to ensure they don't all go to the same directory + String[] files = {"_0.cfe", "_0.cfs", "_0.si", "_0.fnm", "_0.fdt"}; + int[] indices = new int[files.length]; + + for (int i = 0; i < files.length; i++) { + indices[i] = hasher.getDirectoryIndex(files[i]); + } + + // Check that not all files go to the same directory + boolean hasVariation = false; + for (int i = 1; i < indices.length; i++) { + if (indices[i] != indices[0]) { + hasVariation = true; + break; + } + } + + assertTrue("Files should be distributed across different directories", hasVariation); + } +} \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/DirectoryManagerTests.java b/server/src/test/java/org/opensearch/index/store/distributed/DirectoryManagerTests.java new file mode 100644 index 0000000000000..2a78a70d06181 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/store/distributed/DirectoryManagerTests.java @@ -0,0 +1,139 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public class DirectoryManagerTests extends OpenSearchTestCase { + + private Path tempDir; + private Directory baseDirectory; + private DirectoryManager directoryManager; + + @Override + public void setUp() throws Exception { + super.setUp(); + tempDir = createTempDir(); + baseDirectory = FSDirectory.open(tempDir); + } + + @Override + public void tearDown() throws Exception { + if (directoryManager != null) { + directoryManager.close(); + } + if (baseDirectory != null) { + baseDirectory.close(); + } + super.tearDown(); + } + + public void testDirectoryCreation() throws IOException { + directoryManager = new DirectoryManager(baseDirectory, tempDir); + + // Verify all 5 directories are created + assertEquals("Should have 5 directories", 5, directoryManager.getNumDirectories()); + + // Verify base directory is at index 0 + assertSame("Base directory should be at index 0", baseDirectory, directoryManager.getDirectory(0)); + + // Verify subdirectories are created + for (int i = 1; i < 5; i++) { + Directory dir = directoryManager.getDirectory(i); + assertNotNull("Directory " + i + " should not be null", dir); + assertNotSame("Directory " + i + " should not be the base directory", baseDirectory, dir); + } + + // Verify filesystem subdirectories exist + for (int i = 1; i < 5; i++) { + Path subPath = tempDir.resolve("segments_" + i); + assertTrue("Subdirectory should exist: " + subPath, Files.exists(subPath)); + assertTrue("Subdirectory should be a directory: " + subPath, Files.isDirectory(subPath)); + } + } + + public void testGetDirectoryValidation() throws IOException { + directoryManager = new DirectoryManager(baseDirectory, tempDir); + + // Test valid indices + for (int i = 0; i < 5; i++) { + assertNotNull("Directory " + i + " should not be null", directoryManager.getDirectory(i)); + } + + // Test invalid indices + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> directoryManager.getDirectory(-1)); + assertTrue("Exception should mention valid range", exception.getMessage().contains("between 0 and 4")); + + exception = expectThrows(IllegalArgumentException.class, + () -> directoryManager.getDirectory(5)); + assertTrue("Exception should mention valid range", exception.getMessage().contains("between 0 and 4")); + } + + public void testBasePath() throws IOException { + directoryManager = new DirectoryManager(baseDirectory, tempDir); + assertEquals("Base path should match", tempDir, directoryManager.getBasePath()); + } + + public void testClose() throws IOException { + directoryManager = new DirectoryManager(baseDirectory, tempDir); + + // Get references to subdirectories before closing + Directory[] dirs = new Directory[5]; + for (int i = 0; i < 5; i++) { + dirs[i] = directoryManager.getDirectory(i); + } + + // Close the manager + directoryManager.close(); + + // Base directory (index 0) should still be open since it's managed externally + // We can't easily test if subdirectories are closed without accessing internal state + // But we can verify the close operation completed without exception + + // Verify we can still access the base directory + assertNotNull("Base directory should still be accessible", dirs[0]); + } + + public void testSubdirectoryCreationFailure() throws IOException { + // Create a file where we want to create a subdirectory to force failure + Path conflictPath = tempDir.resolve("segments_1"); + Files.createFile(conflictPath); // Create a file, not a directory + + try { + DistributedDirectoryException exception = expectThrows(DistributedDirectoryException.class, + () -> new DirectoryManager(baseDirectory, tempDir)); + assertTrue("Exception should mention subdirectory creation failure", + exception.getMessage().contains("Failed to create subdirectories")); + } finally { + // Clean up the conflicting file + Files.deleteIfExists(conflictPath); + } + } + + public void testExistingSubdirectories() throws IOException { + // Pre-create some subdirectories + Path subDir1 = tempDir.resolve("segments_1"); + Path subDir2 = tempDir.resolve("segments_2"); + Files.createDirectories(subDir1); + Files.createDirectories(subDir2); + + // Should work with existing directories + directoryManager = new DirectoryManager(baseDirectory, tempDir); + + assertNotNull("Should handle existing subdirectory 1", directoryManager.getDirectory(1)); + assertNotNull("Should handle existing subdirectory 2", directoryManager.getDirectory(2)); + } +} \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryExceptionTests.java b/server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryExceptionTests.java new file mode 100644 index 0000000000000..48adf688cce2b --- /dev/null +++ b/server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryExceptionTests.java @@ -0,0 +1,95 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +public class DistributedDirectoryExceptionTests extends OpenSearchTestCase { + + public void testExceptionWithAllParameters() { + String message = "Test error message"; + int directoryIndex = 2; + String operation = "testOperation"; + IOException cause = new IOException("Root cause"); + + DistributedDirectoryException exception = new DistributedDirectoryException( + message, directoryIndex, operation, cause + ); + + assertEquals("Directory index should match", directoryIndex, exception.getDirectoryIndex()); + assertEquals("Operation should match", operation, exception.getOperation()); + assertSame("Cause should match", cause, exception.getCause()); + + String expectedMessage = "Directory 2 operation 'testOperation' failed: Test error message"; + assertEquals("Message should be formatted correctly", expectedMessage, exception.getMessage()); + } + + public void testExceptionWithoutCause() { + String message = "Test error message"; + int directoryIndex = 1; + String operation = "testOperation"; + + DistributedDirectoryException exception = new DistributedDirectoryException( + message, directoryIndex, operation + ); + + assertEquals("Directory index should match", directoryIndex, exception.getDirectoryIndex()); + assertEquals("Operation should match", operation, exception.getOperation()); + assertNull("Cause should be null", exception.getCause()); + + String expectedMessage = "Directory 1 operation 'testOperation' failed: Test error message"; + assertEquals("Message should be formatted correctly", expectedMessage, exception.getMessage()); + } + + public void testExceptionMessageFormatting() { + DistributedDirectoryException exception = new DistributedDirectoryException( + "File not found", 0, "openInput" + ); + + String message = exception.getMessage(); + assertTrue("Message should contain directory index", message.contains("Directory 0")); + assertTrue("Message should contain operation", message.contains("openInput")); + assertTrue("Message should contain original message", message.contains("File not found")); + } + + public void testExceptionInheritance() { + DistributedDirectoryException exception = new DistributedDirectoryException( + "Test", 0, "test" + ); + + assertTrue("Should be instance of IOException", exception instanceof IOException); + } + + public void testExceptionWithLongMessage() { + String longMessage = "This is a very long error message that contains lots of details about what went wrong during the operation"; + DistributedDirectoryException exception = new DistributedDirectoryException( + longMessage, 4, "longOperation" + ); + + assertTrue("Message should contain the long message", exception.getMessage().contains(longMessage)); + assertEquals("Directory index should be preserved", 4, exception.getDirectoryIndex()); + assertEquals("Operation should be preserved", "longOperation", exception.getOperation()); + } + + public void testExceptionWithSpecialCharacters() { + String messageWithSpecialChars = "Error with special chars: !@#$%^&*()"; + String operationWithSpecialChars = "operation:with:colons"; + + DistributedDirectoryException exception = new DistributedDirectoryException( + messageWithSpecialChars, 3, operationWithSpecialChars + ); + + assertTrue("Message should handle special characters", + exception.getMessage().contains(messageWithSpecialChars)); + assertEquals("Operation with special chars should be preserved", + operationWithSpecialChars, exception.getOperation()); + } +} \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryFactoryTests.java b/server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryFactoryTests.java new file mode 100644 index 0000000000000..4cfb5a277046d --- /dev/null +++ b/server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryFactoryTests.java @@ -0,0 +1,225 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.opensearch.common.settings.Settings; +import org.opensearch.core.index.shard.ShardId; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.nio.file.Path; + +public class DistributedDirectoryFactoryTests extends OpenSearchTestCase { + + private Path tempDir; + private Directory baseDirectory; + + @Override + public void setUp() throws Exception { + super.setUp(); + tempDir = createTempDir(); + baseDirectory = FSDirectory.open(tempDir); + } + + @Override + public void tearDown() throws Exception { + if (baseDirectory != null) { + baseDirectory.close(); + } + super.tearDown(); + } + + public void testFactoryWithDefaultSettings() throws IOException { + Settings settings = Settings.EMPTY; + DistributedDirectoryFactory factory = new DistributedDirectoryFactory(settings); + + assertFalse("Distributed should be disabled by default", factory.isDistributedEnabled()); + assertEquals("Should use default subdirectories", 5, factory.getNumSubdirectories()); + assertEquals("Should use default hash algorithm", "default", factory.getHashAlgorithm()); + } + + public void testFactoryWithDistributedDisabled() throws IOException { + Settings settings = Settings.builder() + .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, false) + .build(); + + DistributedDirectoryFactory factory = new DistributedDirectoryFactory(settings); + ShardId shardId = new ShardId("test", "test-uuid", 0); + + Directory result = factory.createDirectory(baseDirectory, tempDir, shardId); + + assertSame("Should return delegate directory when disabled", baseDirectory, result); + assertFalse("Should report as disabled", factory.isDistributedEnabled()); + } + + public void testFactoryWithDistributedEnabled() throws IOException { + Settings settings = Settings.builder() + .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, true) + .build(); + + DistributedDirectoryFactory factory = new DistributedDirectoryFactory(settings); + ShardId shardId = new ShardId("test", "test-uuid", 0); + + Directory result = factory.createDirectory(baseDirectory, tempDir, shardId); + + assertNotSame("Should return distributed directory when enabled", baseDirectory, result); + assertTrue("Should be instance of DistributedSegmentDirectory", + result instanceof DistributedSegmentDirectory); + assertTrue("Should report as enabled", factory.isDistributedEnabled()); + + result.close(); + } + + public void testFactoryWithCustomSettings() throws IOException { + Settings settings = Settings.builder() + .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, true) + .put(DistributedDirectoryFactory.DISTRIBUTED_SUBDIRECTORIES_SETTING, 3) + .put(DistributedDirectoryFactory.DISTRIBUTED_HASH_ALGORITHM_SETTING, "custom") + .build(); + + DistributedDirectoryFactory factory = new DistributedDirectoryFactory(settings); + + assertTrue("Should be enabled", factory.isDistributedEnabled()); + assertEquals("Should use custom subdirectories", 3, factory.getNumSubdirectories()); + assertEquals("Should use custom hash algorithm", "custom", factory.getHashAlgorithm()); + } + + public void testFactoryWithoutShardId() throws IOException { + Settings settings = Settings.builder() + .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, true) + .build(); + + DistributedDirectoryFactory factory = new DistributedDirectoryFactory(settings); + + Directory result = factory.createDirectory(baseDirectory, tempDir); + + assertNotSame("Should return distributed directory", baseDirectory, result); + assertTrue("Should be instance of DistributedSegmentDirectory", + result instanceof DistributedSegmentDirectory); + + result.close(); + } + + public void testFactoryWithInvalidHashAlgorithm() throws IOException { + Settings settings = Settings.builder() + .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, true) + .put(DistributedDirectoryFactory.DISTRIBUTED_HASH_ALGORITHM_SETTING, "invalid") + .build(); + + DistributedDirectoryFactory factory = new DistributedDirectoryFactory(settings); + ShardId shardId = new ShardId("test", "test-uuid", 0); + + // Should still create directory with default hasher + Directory result = factory.createDirectory(baseDirectory, tempDir, shardId); + + assertNotSame("Should return distributed directory", baseDirectory, result); + assertTrue("Should be instance of DistributedSegmentDirectory", + result instanceof DistributedSegmentDirectory); + + result.close(); + } + + public void testFactoryWithSettings() throws IOException { + Settings originalSettings = Settings.builder() + .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, false) + .build(); + + Settings newSettings = Settings.builder() + .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, true) + .build(); + + DistributedDirectoryFactory originalFactory = new DistributedDirectoryFactory(originalSettings); + DistributedDirectoryFactory newFactory = originalFactory.withSettings(newSettings); + + assertFalse("Original factory should be disabled", originalFactory.isDistributedEnabled()); + assertTrue("New factory should be enabled", newFactory.isDistributedEnabled()); + } + + public void testFactoryFallbackOnError() throws IOException { + Settings settings = Settings.builder() + .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, true) + .build(); + + DistributedDirectoryFactory factory = new DistributedDirectoryFactory(settings); + ShardId shardId = new ShardId("test", "test-uuid", 0); + + // Use a path that will cause directory creation to fail + Path invalidPath = tempDir.resolve("nonexistent/invalid/path"); + + Directory result = factory.createDirectory(baseDirectory, invalidPath, shardId); + + // Should fall back to delegate directory on error + assertSame("Should fall back to delegate on error", baseDirectory, result); + } + + public void testFactorySettingsValidation() throws IOException { + // Test with various settings combinations + Settings settings1 = Settings.builder() + .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, "true") + .build(); + + Settings settings2 = Settings.builder() + .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, "false") + .build(); + + DistributedDirectoryFactory factory1 = new DistributedDirectoryFactory(settings1); + DistributedDirectoryFactory factory2 = new DistributedDirectoryFactory(settings2); + + assertTrue("String 'true' should be parsed as enabled", factory1.isDistributedEnabled()); + assertFalse("String 'false' should be parsed as disabled", factory2.isDistributedEnabled()); + } +} + public void testSettingsIntegration() throws IOException { + // Test all configuration options + Settings settings = Settings.builder() + .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, true) + .put(DistributedDirectoryFactory.DISTRIBUTED_SUBDIRECTORIES_SETTING, 3) + .put(DistributedDirectoryFactory.DISTRIBUTED_HASH_ALGORITHM_SETTING, "default") + .build(); + + DistributedDirectoryFactory factory = new DistributedDirectoryFactory(settings); + + assertTrue("Should be enabled", factory.isDistributedEnabled()); + assertEquals("Should use configured subdirectories", 3, factory.getNumSubdirectories()); + assertEquals("Should use configured hash algorithm", "default", factory.getHashAlgorithm()); + } + + public void testSettingsValidation() throws IOException { + // Test with invalid subdirectory count (should still work, just use the value) + Settings settings = Settings.builder() + .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, true) + .put(DistributedDirectoryFactory.DISTRIBUTED_SUBDIRECTORIES_SETTING, 10) + .build(); + + DistributedDirectoryFactory factory = new DistributedDirectoryFactory(settings); + assertEquals("Should accept configured value", 10, factory.getNumSubdirectories()); + } + + public void testDefaultSettingsValues() throws IOException { + DistributedDirectoryFactory factory = new DistributedDirectoryFactory(Settings.EMPTY); + + assertEquals("Default enabled should be false", + DistributedDirectoryFactory.DEFAULT_DISTRIBUTED_ENABLED, factory.isDistributedEnabled()); + assertEquals("Default subdirectories should be 5", + DistributedDirectoryFactory.DEFAULT_SUBDIRECTORIES, factory.getNumSubdirectories()); + assertEquals("Default hash algorithm should be 'default'", + DistributedDirectoryFactory.DEFAULT_HASH_ALGORITHM, factory.getHashAlgorithm()); + } + + public void testSettingsConstants() { + // Verify setting key constants are correct + assertEquals("Enabled setting key", "index.store.distributed.enabled", + DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING); + assertEquals("Subdirectories setting key", "index.store.distributed.subdirectories", + DistributedDirectoryFactory.DISTRIBUTED_SUBDIRECTORIES_SETTING); + assertEquals("Hash algorithm setting key", "index.store.distributed.hash_algorithm", + DistributedDirectoryFactory.DISTRIBUTED_HASH_ALGORITHM_SETTING); + } \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryMetricsTests.java b/server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryMetricsTests.java new file mode 100644 index 0000000000000..094408f0c6565 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryMetricsTests.java @@ -0,0 +1,192 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.opensearch.test.OpenSearchTestCase; + +public class DistributedDirectoryMetricsTests extends OpenSearchTestCase { + + private DistributedDirectoryMetrics metrics; + + @Override + public void setUp() throws Exception { + super.setUp(); + metrics = new DistributedDirectoryMetrics(); + } + + public void testInitialState() { + assertEquals("Total operations should be zero initially", 0, metrics.getTotalOperations()); + assertEquals("Total errors should be zero initially", 0, metrics.getTotalErrors()); + + for (int i = 0; i < 5; i++) { + assertEquals("Operation count should be zero for directory " + i, 0, metrics.getOperationCount(i)); + assertEquals("Error count should be zero for directory " + i, 0, metrics.getErrorCount(i)); + assertEquals("Average time should be zero for directory " + i, 0, metrics.getAverageOperationTime(i)); + } + } + + public void testRecordFileOperation() { + int directoryIndex = 2; + String operation = "openInput"; + long duration = 1_000_000; // 1ms in nanoseconds + + metrics.recordFileOperation(directoryIndex, operation, duration); + + assertEquals("Operation count should be 1", 1, metrics.getOperationCount(directoryIndex)); + assertEquals("Total operations should be 1", 1, metrics.getTotalOperations()); + assertEquals("Average time should match duration", duration, metrics.getAverageOperationTime(directoryIndex)); + assertEquals("Max time should match duration", duration, metrics.getMaxOperationTime(directoryIndex)); + assertEquals("Min time should match duration", duration, metrics.getMinOperationTime(directoryIndex)); + } + + public void testRecordMultipleOperations() { + long[] durations = {1_000_000, 2_000_000, 3_000_000}; // 1ms, 2ms, 3ms + int directoryIndex = 1; + + for (long duration : durations) { + metrics.recordFileOperation(directoryIndex, "test", duration); + } + + assertEquals("Operation count should be 3", 3, metrics.getOperationCount(directoryIndex)); + assertEquals("Total operations should be 3", 3, metrics.getTotalOperations()); + + long expectedAverage = (1_000_000 + 2_000_000 + 3_000_000) / 3; + assertEquals("Average should be calculated correctly", expectedAverage, metrics.getAverageOperationTime(directoryIndex)); + assertEquals("Max should be 3ms", 3_000_000, metrics.getMaxOperationTime(directoryIndex)); + assertEquals("Min should be 1ms", 1_000_000, metrics.getMinOperationTime(directoryIndex)); + } + + public void testRecordError() { + int directoryIndex = 3; + String operation = "deleteFile"; + + metrics.recordError(directoryIndex, operation); + + assertEquals("Error count should be 1", 1, metrics.getErrorCount(directoryIndex)); + assertEquals("Total errors should be 1", 1, metrics.getTotalErrors()); + } + + public void testDistributionAcrossDirectories() { + // Record operations in different directories + metrics.recordFileOperation(0, "test", 1_000_000); + metrics.recordFileOperation(1, "test", 1_000_000); + metrics.recordFileOperation(1, "test", 1_000_000); + metrics.recordFileOperation(2, "test", 1_000_000); + metrics.recordFileOperation(2, "test", 1_000_000); + metrics.recordFileOperation(2, "test", 1_000_000); + + assertEquals("Directory 0 should have 1 operation", 1, metrics.getOperationCount(0)); + assertEquals("Directory 1 should have 2 operations", 2, metrics.getOperationCount(1)); + assertEquals("Directory 2 should have 3 operations", 3, metrics.getOperationCount(2)); + assertEquals("Total should be 6", 6, metrics.getTotalOperations()); + + double[] distribution = metrics.getDistributionPercentages(); + assertEquals("Directory 0 should have ~16.7%", 16.7, distribution[0], 0.1); + assertEquals("Directory 1 should have ~33.3%", 33.3, distribution[1], 0.1); + assertEquals("Directory 2 should have 50%", 50.0, distribution[2], 0.1); + assertEquals("Directory 3 should have 0%", 0.0, distribution[3], 0.1); + assertEquals("Directory 4 should have 0%", 0.0, distribution[4], 0.1); + } + + public void testInvalidDirectoryIndex() { + // Test negative index + metrics.recordFileOperation(-1, "test", 1_000_000); + assertEquals("Should not record operation for negative index", 0, metrics.getTotalOperations()); + + // Test index too large + metrics.recordFileOperation(5, "test", 1_000_000); + assertEquals("Should not record operation for index >= 5", 0, metrics.getTotalOperations()); + + // Test error recording with invalid index + metrics.recordError(-1, "test"); + metrics.recordError(10, "test"); + assertEquals("Should not record errors for invalid indices", 0, metrics.getTotalErrors()); + } + + public void testOperationTypes() { + int directoryIndex = 0; + + metrics.recordFileOperation(directoryIndex, "openInput", 1_000_000); + metrics.recordFileOperation(directoryIndex, "createOutput", 2_000_000); + metrics.recordFileOperation(directoryIndex, "deleteFile", 3_000_000); + metrics.recordFileOperation(directoryIndex, "fileLength", 4_000_000); + + assertEquals("Should record all operation types", 4, metrics.getOperationCount(directoryIndex)); + + // Average should be (1+2+3+4)/4 = 2.5ms + long expectedAverage = (1_000_000 + 2_000_000 + 3_000_000 + 4_000_000) / 4; + assertEquals("Average should be calculated correctly", expectedAverage, metrics.getAverageOperationTime(directoryIndex)); + } + + public void testReset() { + // Record some operations and errors + metrics.recordFileOperation(0, "test", 1_000_000); + metrics.recordFileOperation(1, "test", 2_000_000); + metrics.recordError(0, "test"); + + assertEquals("Should have operations before reset", 2, metrics.getTotalOperations()); + assertEquals("Should have errors before reset", 1, metrics.getTotalErrors()); + + // Reset metrics + metrics.reset(); + + assertEquals("Total operations should be zero after reset", 0, metrics.getTotalOperations()); + assertEquals("Total errors should be zero after reset", 0, metrics.getTotalErrors()); + + for (int i = 0; i < 5; i++) { + assertEquals("Operation count should be zero after reset for directory " + i, + 0, metrics.getOperationCount(i)); + assertEquals("Error count should be zero after reset for directory " + i, + 0, metrics.getErrorCount(i)); + } + } + + public void testUptime() throws InterruptedException { + long startTime = System.currentTimeMillis(); + + // Wait a small amount + Thread.sleep(10); + + long uptime = metrics.getUptimeMillis(); + long actualElapsed = System.currentTimeMillis() - startTime; + + assertTrue("Uptime should be positive", uptime > 0); + assertTrue("Uptime should be reasonable", uptime <= actualElapsed + 10); // Allow some tolerance + } + + public void testSummary() { + metrics.recordFileOperation(0, "openInput", 1_000_000); + metrics.recordFileOperation(1, "createOutput", 2_000_000); + metrics.recordError(0, "test"); + + String summary = metrics.getSummary(); + + assertNotNull("Summary should not be null", summary); + assertTrue("Summary should contain total operations", summary.contains("Total Operations: 2")); + assertTrue("Summary should contain total errors", summary.contains("Total Errors: 1")); + assertTrue("Summary should contain uptime", summary.contains("Uptime:")); + assertTrue("Summary should contain distribution", summary.contains("Distribution by Directory:")); + } + + public void testDistributionPercentagesWithNoOperations() { + double[] distribution = metrics.getDistributionPercentages(); + + assertEquals("Should have 5 percentages", 5, distribution.length); + for (int i = 0; i < 5; i++) { + assertEquals("All percentages should be 0 when no operations", 0.0, distribution[i], 0.001); + } + } + + public void testMinMaxTimesWithNoOperations() { + for (int i = 0; i < 5; i++) { + assertEquals("Max time should be 0 with no operations", 0, metrics.getMaxOperationTime(i)); + assertEquals("Min time should be 0 with no operations", 0, metrics.getMinOperationTime(i)); + } + } +} \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryBenchmarkTests.java b/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryBenchmarkTests.java new file mode 100644 index 0000000000000..7c11194d590f4 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryBenchmarkTests.java @@ -0,0 +1,185 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class DistributedSegmentDirectoryBenchmarkTests extends OpenSearchTestCase { + + private static final int BENCHMARK_DOCS = 1000; + private static final int BENCHMARK_ITERATIONS = 3; + + public void testFileDistributionValidation() throws IOException { + Path tempDir = createTempDir(); + Directory baseDir = FSDirectory.open(tempDir); + DistributedSegmentDirectory distributedDir = new DistributedSegmentDirectory(baseDir, tempDir); + + // Create an index to generate various file types + createTestIndex(distributedDir, 100); + + String[] files = distributedDir.listAll(); + assertTrue("Should have created multiple files", files.length > 5); + + // Analyze file distribution + Map distributionCount = new HashMap<>(); + int segmentsFileCount = 0; + + for (String file : files) { + int dirIndex = distributedDir.getDirectoryIndex(file); + distributionCount.put(dirIndex, distributionCount.getOrDefault(dirIndex, 0) + 1); + + if (file.startsWith("segments_")) { + segmentsFileCount++; + assertEquals("segments_N files should be in directory 0", 0, dirIndex); + } + } + + assertTrue("Should have segments files", segmentsFileCount > 0); + assertTrue("Should use multiple directories", distributionCount.size() > 1); + + // Check distribution is reasonably balanced (no directory should have > 70% of files) + int totalFiles = files.length; + for (Map.Entry entry : distributionCount.entrySet()) { + double percentage = (double) entry.getValue() / totalFiles; + assertTrue("Directory " + entry.getKey() + " has " + (percentage * 100) + "% of files, should be < 70%", + percentage < 0.7); + } + + logger.info("File distribution across directories: {}", distributionCount); + distributedDir.close(); + } + + public void testHashingDistribution() throws IOException { + DefaultFilenameHasher hasher = new DefaultFilenameHasher(); + + // Test with common Lucene file extensions + String[] testFiles = { + "_0.cfe", "_0.cfs", "_0.si", "_0.fnm", "_0.fdt", "_0.fdx", + "_0.tim", "_0.tip", "_0.doc", "_0.pos", "_0.pay", "_0.nvd", "_0.nvm", + "_1.cfe", "_1.cfs", "_1.si", "_1.fnm", "_1.fdt", "_1.fdx", + "_1.tim", "_1.tip", "_1.doc", "_1.pos", "_1.pay", "_1.nvd", "_1.nvm", + "_2.cfe", "_2.cfs", "_2.si", "_2.fnm", "_2.fdt", "_2.fdx" + }; + + Map distribution = new HashMap<>(); + + for (String file : testFiles) { + int dirIndex = hasher.getDirectoryIndex(file); + distribution.put(dirIndex, distribution.getOrDefault(dirIndex, 0) + 1); + } + + logger.info("Hash distribution for {} files: {}", testFiles.length, distribution); + + // Should use multiple directories + assertTrue("Should distribute across multiple directories", distribution.size() > 1); + + // No single directory should have more than 60% of files + for (Map.Entry entry : distribution.entrySet()) { + double percentage = (double) entry.getValue() / testFiles.length; + assertTrue("Directory " + entry.getKey() + " should not have > 60% of files, has " + (percentage * 100) + "%", + percentage <= 0.6); + } + } + + public void testConcurrentAccessValidation() throws IOException, InterruptedException { + Path tempDir = createTempDir(); + Directory baseDir = FSDirectory.open(tempDir); + DistributedSegmentDirectory distributedDir = new DistributedSegmentDirectory(baseDir, tempDir); + + // Create initial index + createTestIndex(distributedDir, 100); + + // Test concurrent read operations + int numThreads = 4; + int operationsPerThread = 25; + + Thread[] threads = new Thread[numThreads]; + final Exception[] exceptions = new Exception[numThreads]; + + for (int i = 0; i < numThreads; i++) { + final int threadId = i; + threads[i] = new Thread(() -> { + try { + for (int j = 0; j < operationsPerThread; j++) { + // Test various operations concurrently + String[] files = distributedDir.listAll(); + assertTrue("Should have files", files.length > 0); + + // Test file length operations + for (String file : files) { + if (!file.startsWith("segments_")) { + long length = distributedDir.fileLength(file); + assertTrue("File length should be positive", length > 0); + } + } + + // Test search operations + try (IndexReader reader = DirectoryReader.open(distributedDir)) { + IndexSearcher searcher = new IndexSearcher(reader); + TermQuery query = new TermQuery(new Term("id", String.valueOf(j % 100))); + TopDocs results = searcher.search(query, 10); + assertTrue("Should find results", results.totalHits.value > 0); + } + } + } catch (Exception e) { + exceptions[threadId] = e; + } + }); + threads[i].start(); + } + + // Wait for all threads to complete + for (Thread thread : threads) { + thread.join(30000); // 30 second timeout + } + + // Check for exceptions + for (int i = 0; i < numThreads; i++) { + if (exceptions[i] != null) { + fail("Thread " + i + " failed with exception: " + exceptions[i].getMessage()); + } + } + + distributedDir.close(); + } + + private void createTestIndex(Directory directory, int numDocs) throws IOException { + IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); + try (IndexWriter writer = new IndexWriter(directory, config)) { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add(new TextField("content", "This is document " + i + " with some content", Field.Store.YES)); + writer.addDocument(doc); + } + writer.commit(); + } + } +} \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryEdgeCaseTests.java b/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryEdgeCaseTests.java new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryIntegrationTests.java b/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryIntegrationTests.java new file mode 100644 index 0000000000000..f1bf8a9e832dd --- /dev/null +++ b/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryIntegrationTests.java @@ -0,0 +1,341 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class DistributedSegmentDirectoryIntegrationTests extends OpenSearchTestCase { + + private Path tempDir; + private Directory baseDirectory; + private DistributedSegmentDirectory distributedDirectory; + + @Override + public void setUp() throws Exception { + super.setUp(); + tempDir = createTempDir(); + baseDirectory = FSDirectory.open(tempDir); + distributedDirectory = new DistributedSegmentDirectory(baseDirectory, tempDir); + } + + @Override + public void tearDown() throws Exception { + if (distributedDirectory != null) { + distributedDirectory.close(); + } + super.tearDown(); + } + + public void testIndexWriterWithDistributedDirectory() throws IOException { + IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); + + try (IndexWriter writer = new IndexWriter(distributedDirectory, config)) { + // Add some documents + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add(new TextField("content", "This is document " + i, Field.Store.YES)); + writer.addDocument(doc); + } + + writer.commit(); + } + + // Verify files were created and distributed + String[] files = distributedDirectory.listAll(); + assertTrue("Should have created segment files", files.length > 0); + + // Check that files are distributed across directories + boolean hasSegmentsFile = false; + boolean hasDistributedFiles = false; + + for (String file : files) { + if (file.startsWith("segments_")) { + hasSegmentsFile = true; + assertEquals("segments_N should be in directory 0", 0, distributedDirectory.getDirectoryIndex(file)); + } else { + hasDistributedFiles = true; + } + } + + assertTrue("Should have segments file", hasSegmentsFile); + assertTrue("Should have other distributed files", hasDistributedFiles); + } + + public void testIndexReaderWithDistributedDirectory() throws IOException { + // First create an index + IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); + + try (IndexWriter writer = new IndexWriter(distributedDirectory, config)) { + for (int i = 0; i < 5; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add(new TextField("content", "Document content " + i, Field.Store.YES)); + writer.addDocument(doc); + } + writer.commit(); + } + + // Now read the index + try (IndexReader reader = DirectoryReader.open(distributedDirectory)) { + assertEquals("Should have 5 documents", 5, reader.numDocs()); + + IndexSearcher searcher = new IndexSearcher(reader); + + // Search for a specific document + TermQuery query = new TermQuery(new Term("id", "2")); + TopDocs results = searcher.search(query, 10); + + assertEquals("Should find one document", 1, results.totalHits.value); + + Document doc = searcher.doc(results.scoreDocs[0].doc); + assertEquals("Should find correct document", "2", doc.get("id")); + } + } + + public void testIndexWriterWithMerging() throws IOException { + IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); + config.setMaxBufferedDocs(2); // Force frequent segment creation + + try (IndexWriter writer = new IndexWriter(distributedDirectory, config)) { + // Add documents to create multiple segments + for (int i = 0; i < 20; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add(new TextField("content", "Content for document " + i, Field.Store.YES)); + writer.addDocument(doc); + + if (i % 5 == 0) { + writer.commit(); // Create multiple commits + } + } + + // Force merge to test segment merging with distributed files + writer.forceMerge(1); + writer.commit(); + } + + // Verify the index is still readable after merging + try (IndexReader reader = DirectoryReader.open(distributedDirectory)) { + assertEquals("Should have all 20 documents after merge", 20, reader.numDocs()); + + // Verify we can search + IndexSearcher searcher = new IndexSearcher(reader); + TermQuery query = new TermQuery(new Term("id", "15")); + TopDocs results = searcher.search(query, 10); + + assertEquals("Should find document after merge", 1, results.totalHits.value); + } + } + + public void testIndexDeletion() throws IOException { + IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); + + try (IndexWriter writer = new IndexWriter(distributedDirectory, config)) { + // Add documents + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add(new TextField("content", "Document " + i, Field.Store.YES)); + writer.addDocument(doc); + } + writer.commit(); + + // Delete some documents + writer.deleteDocuments(new Term("id", "5")); + writer.deleteDocuments(new Term("id", "7")); + writer.commit(); + } + + // Verify deletions + try (IndexReader reader = DirectoryReader.open(distributedDirectory)) { + assertEquals("Should have 8 documents after deletion", 8, reader.numDocs()); + + IndexSearcher searcher = new IndexSearcher(reader); + + // Verify deleted documents are not found + TermQuery query1 = new TermQuery(new Term("id", "5")); + TopDocs results1 = searcher.search(query1, 10); + assertEquals("Deleted document should not be found", 0, results1.totalHits.value); + + TermQuery query2 = new TermQuery(new Term("id", "7")); + TopDocs results2 = searcher.search(query2, 10); + assertEquals("Deleted document should not be found", 0, results2.totalHits.value); + + // Verify non-deleted documents are still found + TermQuery query3 = new TermQuery(new Term("id", "3")); + TopDocs results3 = searcher.search(query3, 10); + assertEquals("Non-deleted document should be found", 1, results3.totalHits.value); + } + } + + public void testConcurrentIndexing() throws IOException, InterruptedException { + IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); + + try (IndexWriter writer = new IndexWriter(distributedDirectory, config)) { + int numThreads = 4; + int docsPerThread = 25; + ExecutorService executor = Executors.newFixedThreadPool(numThreads); + CountDownLatch latch = new CountDownLatch(numThreads); + AtomicInteger errorCount = new AtomicInteger(0); + + // Start concurrent indexing threads + for (int t = 0; t < numThreads; t++) { + final int threadId = t; + executor.submit(() -> { + try { + for (int i = 0; i < docsPerThread; i++) { + Document doc = new Document(); + String docId = threadId + "_" + i; + doc.add(new StringField("id", docId, Field.Store.YES)); + doc.add(new StringField("thread", String.valueOf(threadId), Field.Store.YES)); + doc.add(new TextField("content", "Content from thread " + threadId + " doc " + i, Field.Store.YES)); + + writer.addDocument(doc); + + // Occasionally commit + if (i % 10 == 0) { + writer.commit(); + } + } + } catch (Exception e) { + logger.error("Error in indexing thread " + threadId, e); + errorCount.incrementAndGet(); + } finally { + latch.countDown(); + } + }); + } + + // Wait for all threads to complete + assertTrue("All threads should complete", latch.await(30, TimeUnit.SECONDS)); + assertEquals("No errors should occur during concurrent indexing", 0, errorCount.get()); + + writer.commit(); + executor.shutdown(); + } + + // Verify all documents were indexed correctly + try (IndexReader reader = DirectoryReader.open(distributedDirectory)) { + int expectedDocs = 4 * 25; // 4 threads * 25 docs each + assertEquals("Should have all documents from concurrent indexing", expectedDocs, reader.numDocs()); + + IndexSearcher searcher = new IndexSearcher(reader); + + // Verify documents from each thread + for (int t = 0; t < 4; t++) { + TermQuery query = new TermQuery(new Term("thread", String.valueOf(t))); + TopDocs results = searcher.search(query, 100); + assertEquals("Should have 25 documents from thread " + t, 25, results.totalHits.value); + } + } + } + + public void testIndexOptimization() throws IOException { + IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); + config.setMaxBufferedDocs(5); // Create multiple segments + + try (IndexWriter writer = new IndexWriter(distributedDirectory, config)) { + // Add many documents to create multiple segments + for (int i = 0; i < 50; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add(new TextField("content", "Document content " + i, Field.Store.YES)); + writer.addDocument(doc); + + if (i % 10 == 0) { + writer.commit(); + } + } + + // Get file count before optimization + String[] filesBefore = distributedDirectory.listAll(); + int filesBeforeCount = filesBefore.length; + + // Optimize (force merge to 1 segment) + writer.forceMerge(1); + writer.commit(); + + // Get file count after optimization + String[] filesAfter = distributedDirectory.listAll(); + int filesAfterCount = filesAfter.length; + + // After optimization, we should have fewer files (merged segments) + assertTrue("Should have fewer files after optimization", filesAfterCount <= filesBeforeCount); + } + + // Verify index is still functional after optimization + try (IndexReader reader = DirectoryReader.open(distributedDirectory)) { + assertEquals("Should still have all 50 documents", 50, reader.numDocs()); + assertEquals("Should have only 1 segment after optimization", 1, reader.leaves().size()); + } + } + + public void testLargeDocuments() throws IOException { + IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); + + try (IndexWriter writer = new IndexWriter(distributedDirectory, config)) { + // Create documents with large content + for (int i = 0; i < 5; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + + // Create large content (about 1MB per document) + StringBuilder largeContent = new StringBuilder(); + for (int j = 0; j < 10000; j++) { + largeContent.append("This is a large document with lots of content. Document ID: ").append(i).append(" Line: ").append(j).append(". "); + } + + doc.add(new TextField("content", largeContent.toString(), Field.Store.YES)); + writer.addDocument(doc); + } + + writer.commit(); + } + + // Verify large documents can be read + try (IndexReader reader = DirectoryReader.open(distributedDirectory)) { + assertEquals("Should have 5 large documents", 5, reader.numDocs()); + + IndexSearcher searcher = new IndexSearcher(reader); + TermQuery query = new TermQuery(new Term("id", "2")); + TopDocs results = searcher.search(query, 10); + + assertEquals("Should find the large document", 1, results.totalHits.value); + + Document doc = searcher.doc(results.scoreDocs[0].doc); + String content = doc.get("content"); + assertTrue("Large document content should be preserved", content.length() > 100000); + assertTrue("Content should contain expected text", content.contains("Document ID: 2")); + } + } +} \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryTests.java new file mode 100644 index 0000000000000..99d2d006c680d --- /dev/null +++ b/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryTests.java @@ -0,0 +1,719 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.nio.file.Path; + +public class DistributedSegmentDirectoryTests extends OpenSearchTestCase { + + private Path tempDir; + private Directory baseDirectory; + private DistributedSegmentDirectory distributedDirectory; + + @Override + public void setUp() throws Exception { + super.setUp(); + tempDir = createTempDir(); + baseDirectory = FSDirectory.open(tempDir); + distributedDirectory = new DistributedSegmentDirectory(baseDirectory, tempDir); + } + + @Override + public void tearDown() throws Exception { + if (distributedDirectory != null) { + distributedDirectory.close(); + } + super.tearDown(); + } + + public void testDirectoryCreation() throws IOException { + assertNotNull("Distributed directory should be created", distributedDirectory); + assertEquals("Should have 5 directories", 5, distributedDirectory.getDirectoryManager().getNumDirectories()); + assertNotNull("Hasher should be initialized", distributedDirectory.getHasher()); + } + + public void testDirectoryResolution() throws IOException { + // Test that different files resolve to appropriate directories + String segmentsFile = "segments_1"; + String regularFile = "_0.cfe"; + + Directory segmentsDir = distributedDirectory.resolveDirectory(segmentsFile); + Directory regularDir = distributedDirectory.resolveDirectory(regularFile); + + // segments_N should always go to base directory (index 0) + assertSame("segments_1 should resolve to base directory", + distributedDirectory.getDirectoryManager().getDirectory(0), segmentsDir); + + // Regular files should be distributed + int regularIndex = distributedDirectory.getDirectoryIndex(regularFile); + assertSame("Regular file should resolve to correct directory", + distributedDirectory.getDirectoryManager().getDirectory(regularIndex), regularDir); + } + + public void testOpenInputWithExistingFile() throws IOException { + String filename = "_0.cfe"; + String content = "test content"; + + // First create the file in the appropriate directory + int dirIndex = distributedDirectory.getDirectoryIndex(filename); + Directory targetDir = distributedDirectory.getDirectoryManager().getDirectory(dirIndex); + + try (IndexOutput output = targetDir.createOutput(filename, IOContext.DEFAULT)) { + output.writeString(content); + } + + // Now test opening the file through distributed directory + try (IndexInput input = distributedDirectory.openInput(filename, IOContext.DEFAULT)) { + assertNotNull("Input should not be null", input); + assertEquals("Content should match", content, input.readString()); + } + } + + public void testOpenInputWithNonExistentFile() throws IOException { + String filename = "nonexistent.file"; + + DistributedDirectoryException exception = expectThrows(DistributedDirectoryException.class, + () -> distributedDirectory.openInput(filename, IOContext.DEFAULT)); + + assertTrue("Exception should mention the operation", exception.getMessage().contains("openInput")); + assertTrue("Exception should mention the filename", exception.getMessage().contains(filename)); + assertEquals("Exception should have correct operation", "openInput", exception.getOperation()); + assertTrue("Exception should have valid directory index", + exception.getDirectoryIndex() >= 0 && exception.getDirectoryIndex() < 5); + } + + public void testOpenInputFileDistribution() throws IOException { + String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm", "_3.fdt", "_4.tim"}; + String content = "test content"; + + // Create files in their respective directories + for (String filename : testFiles) { + int dirIndex = distributedDirectory.getDirectoryIndex(filename); + Directory targetDir = distributedDirectory.getDirectoryManager().getDirectory(dirIndex); + + try (IndexOutput output = targetDir.createOutput(filename, IOContext.DEFAULT)) { + output.writeString(content + " for " + filename); + } + } + + // Verify we can read all files through distributed directory + for (String filename : testFiles) { + try (IndexInput input = distributedDirectory.openInput(filename, IOContext.DEFAULT)) { + String readContent = input.readString(); + assertEquals("Content should match for " + filename, content + " for " + filename, readContent); + } + } + } + + public void testSegmentsFileInBaseDirectory() throws IOException { + String segmentsFile = "segments_1"; + String content = "segments content"; + + // Create segments file directly in base directory + try (IndexOutput output = baseDirectory.createOutput(segmentsFile, IOContext.DEFAULT)) { + output.writeString(content); + } + + // Verify we can read it through distributed directory + try (IndexInput input = distributedDirectory.openInput(segmentsFile, IOContext.DEFAULT)) { + assertEquals("Segments file content should match", content, input.readString()); + } + + // Verify it's in the base directory (index 0) + assertEquals("segments_1 should be in directory 0", 0, distributedDirectory.getDirectoryIndex(segmentsFile)); + } + + public void testClose() throws IOException { + // Create some files to ensure directories are in use + String filename = "_0.cfe"; + int dirIndex = distributedDirectory.getDirectoryIndex(filename); + Directory targetDir = distributedDirectory.getDirectoryManager().getDirectory(dirIndex); + + try (IndexOutput output = targetDir.createOutput(filename, IOContext.DEFAULT)) { + output.writeString("test"); + } + + // Close should not throw exception + distributedDirectory.close(); + + // After closing, operations should fail + expectThrows(Exception.class, () -> distributedDirectory.openInput(filename, IOContext.DEFAULT)); + } +} + public void testCreateOutput() throws IOException { + String filename = "_0.cfe"; + String content = "test output content"; + + // Create output through distributed directory + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + assertNotNull("Output should not be null", output); + output.writeString(content); + } + + // Verify file was created in correct directory + int dirIndex = distributedDirectory.getDirectoryIndex(filename); + Directory targetDir = distributedDirectory.getDirectoryManager().getDirectory(dirIndex); + + try (IndexInput input = targetDir.openInput(filename, IOContext.DEFAULT)) { + assertEquals("Content should match", content, input.readString()); + } + } + + public void testCreateOutputSegmentsFile() throws IOException { + String segmentsFile = "segments_1"; + String content = "segments content"; + + // Create segments file through distributed directory + try (IndexOutput output = distributedDirectory.createOutput(segmentsFile, IOContext.DEFAULT)) { + output.writeString(content); + } + + // Verify it was created in base directory (index 0) + assertEquals("segments_1 should be in directory 0", 0, distributedDirectory.getDirectoryIndex(segmentsFile)); + + try (IndexInput input = baseDirectory.openInput(segmentsFile, IOContext.DEFAULT)) { + assertEquals("Segments content should match", content, input.readString()); + } + } + + public void testCreateOutputMultipleFiles() throws IOException { + String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm", "_3.fdt", "_4.tim"}; + + // Create multiple files + for (String filename : testFiles) { + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString("content for " + filename); + } + } + + // Verify all files can be read back + for (String filename : testFiles) { + try (IndexInput input = distributedDirectory.openInput(filename, IOContext.DEFAULT)) { + assertEquals("Content should match for " + filename, + "content for " + filename, input.readString()); + } + } + } + + public void testCreateOutputDistribution() throws IOException { + String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm", "_3.fdt", "_4.tim", "_5.tip", "_6.doc"}; + + // Track which directories are used + boolean[] directoriesUsed = new boolean[5]; + + for (String filename : testFiles) { + int dirIndex = distributedDirectory.getDirectoryIndex(filename); + directoriesUsed[dirIndex] = true; + + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString("test"); + } + } + + // Should use multiple directories + int usedCount = 0; + for (boolean used : directoriesUsed) { + if (used) usedCount++; + } + + assertTrue("Should use multiple directories, used: " + usedCount, usedCount > 1); + } + public void testDeleteFile() throws IOException { + String filename = "_0.cfe"; + String content = "test content"; + + // Create file + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString(content); + } + + // Verify file exists + try (IndexInput input = distributedDirectory.openInput(filename, IOContext.DEFAULT)) { + assertEquals("File should exist", content, input.readString()); + } + + // Delete file + distributedDirectory.deleteFile(filename); + + // Verify file is deleted + expectThrows(Exception.class, () -> distributedDirectory.openInput(filename, IOContext.DEFAULT)); + } + + public void testDeleteSegmentsFile() throws IOException { + String segmentsFile = "segments_1"; + String content = "segments content"; + + // Create segments file + try (IndexOutput output = distributedDirectory.createOutput(segmentsFile, IOContext.DEFAULT)) { + output.writeString(content); + } + + // Delete segments file + distributedDirectory.deleteFile(segmentsFile); + + // Verify it's deleted from base directory + expectThrows(Exception.class, () -> baseDirectory.openInput(segmentsFile, IOContext.DEFAULT)); + } + + public void testDeleteNonExistentFile() throws IOException { + String filename = "nonexistent.file"; + + DistributedDirectoryException exception = expectThrows(DistributedDirectoryException.class, + () -> distributedDirectory.deleteFile(filename)); + + assertTrue("Exception should mention the operation", exception.getMessage().contains("deleteFile")); + assertTrue("Exception should mention the filename", exception.getMessage().contains(filename)); + assertEquals("Exception should have correct operation", "deleteFile", exception.getOperation()); + } + + public void testFileLength() throws IOException { + String filename = "_0.cfe"; + String content = "test content for length"; + + // Create file + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString(content); + } + + // Get file length through distributed directory + long length = distributedDirectory.fileLength(filename); + + // Verify length matches direct access + int dirIndex = distributedDirectory.getDirectoryIndex(filename); + Directory targetDir = distributedDirectory.getDirectoryManager().getDirectory(dirIndex); + long directLength = targetDir.fileLength(filename); + + assertEquals("File length should match", directLength, length); + assertTrue("File length should be positive", length > 0); + } + + public void testFileLengthSegmentsFile() throws IOException { + String segmentsFile = "segments_1"; + String content = "segments content"; + + // Create segments file + try (IndexOutput output = distributedDirectory.createOutput(segmentsFile, IOContext.DEFAULT)) { + output.writeString(content); + } + + // Get length through distributed directory + long length = distributedDirectory.fileLength(segmentsFile); + + // Verify length matches base directory + long baseLength = baseDirectory.fileLength(segmentsFile); + assertEquals("Segments file length should match", baseLength, length); + } + + public void testFileLengthNonExistentFile() throws IOException { + String filename = "nonexistent.file"; + + DistributedDirectoryException exception = expectThrows(DistributedDirectoryException.class, + () -> distributedDirectory.fileLength(filename)); + + assertTrue("Exception should mention the operation", exception.getMessage().contains("fileLength")); + assertTrue("Exception should mention the filename", exception.getMessage().contains(filename)); + assertEquals("Exception should have correct operation", "fileLength", exception.getOperation()); + } + + public void testFileOperationsConsistency() throws IOException { + String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm", "segments_1"}; + + for (String filename : testFiles) { + // Create file + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString("content for " + filename); + } + + // Check length + long length = distributedDirectory.fileLength(filename); + assertTrue("File length should be positive for " + filename, length > 0); + + // Read file + try (IndexInput input = distributedDirectory.openInput(filename, IOContext.DEFAULT)) { + assertEquals("Content should match for " + filename, + "content for " + filename, input.readString()); + } + + // Delete file + distributedDirectory.deleteFile(filename); + + // Verify deletion + expectThrows(Exception.class, () -> distributedDirectory.openInput(filename, IOContext.DEFAULT)); + } + } pub +lic void testListAllEmpty() throws IOException { + String[] files = distributedDirectory.listAll(); + assertNotNull("File list should not be null", files); + assertEquals("Should have no files initially", 0, files.length); + } + + public void testListAllSingleFile() throws IOException { + String filename = "_0.cfe"; + + // Create file + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString("test"); + } + + String[] files = distributedDirectory.listAll(); + assertEquals("Should have one file", 1, files.length); + assertEquals("File should match", filename, files[0]); + } + + public void testListAllMultipleFiles() throws IOException { + String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm", "segments_1", "_3.fdt"}; + + // Create files + for (String filename : testFiles) { + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString("content for " + filename); + } + } + + String[] listedFiles = distributedDirectory.listAll(); + assertEquals("Should have all files", testFiles.length, listedFiles.length); + + // Convert to set for easier comparison + Set expectedFiles = Set.of(testFiles); + Set actualFiles = Set.of(listedFiles); + assertEquals("All files should be listed", expectedFiles, actualFiles); + } + + public void testListAllDistributedFiles() throws IOException { + String[] testFiles = { + "_0.cfe", "_1.si", "_2.fnm", "_3.fdt", "_4.tim", + "_5.tip", "_6.doc", "_7.pos", "segments_1", "segments_2" + }; + + // Create files across different directories + for (String filename : testFiles) { + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString("test"); + } + } + + // Verify files are in different directories + Set usedDirectories = new HashSet<>(); + for (String filename : testFiles) { + usedDirectories.add(distributedDirectory.getDirectoryIndex(filename)); + } + assertTrue("Should use multiple directories", usedDirectories.size() > 1); + + // Verify listAll returns all files + String[] listedFiles = distributedDirectory.listAll(); + Set expectedFiles = Set.of(testFiles); + Set actualFiles = Set.of(listedFiles); + assertEquals("All distributed files should be listed", expectedFiles, actualFiles); + } + + public void testListAllAfterDeletion() throws IOException { + String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm"}; + + // Create files + for (String filename : testFiles) { + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString("test"); + } + } + + // Verify all files are listed + String[] allFiles = distributedDirectory.listAll(); + assertEquals("Should have all files", testFiles.length, allFiles.length); + + // Delete one file + distributedDirectory.deleteFile(testFiles[0]); + + // Verify updated list + String[] remainingFiles = distributedDirectory.listAll(); + assertEquals("Should have one less file", testFiles.length - 1, remainingFiles.length); + + Set remainingSet = Set.of(remainingFiles); + assertFalse("Deleted file should not be listed", remainingSet.contains(testFiles[0])); + assertTrue("Other files should still be listed", remainingSet.contains(testFiles[1])); + assertTrue("Other files should still be listed", remainingSet.contains(testFiles[2])); + } + + public void testListAllNoDuplicates() throws IOException { + String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm", "_3.fdt"}; + + // Create files + for (String filename : testFiles) { + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString("test"); + } + } + + String[] listedFiles = distributedDirectory.listAll(); + + // Check for duplicates + Set uniqueFiles = Set.of(listedFiles); + assertEquals("Should have no duplicates", listedFiles.length, uniqueFiles.size()); + } pu +blic void testSyncSingleFile() throws IOException { + String filename = "_0.cfe"; + + // Create file + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString("test"); + } + + // Sync should not throw exception + distributedDirectory.sync(Collections.singletonList(filename)); + } + + public void testSyncMultipleFilesInSameDirectory() throws IOException { + // Create files that will hash to the same directory + String filename1 = "_0.cfe"; + String filename2 = "_0.cfs"; + + // Create files + try (IndexOutput output = distributedDirectory.createOutput(filename1, IOContext.DEFAULT)) { + output.writeString("test1"); + } + try (IndexOutput output = distributedDirectory.createOutput(filename2, IOContext.DEFAULT)) { + output.writeString("test2"); + } + + // Sync both files + distributedDirectory.sync(List.of(filename1, filename2)); + } + + public void testSyncMultipleFilesAcrossDirectories() throws IOException { + String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm", "segments_1"}; + + // Create files + for (String filename : testFiles) { + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString("content for " + filename); + } + } + + // Sync all files + distributedDirectory.sync(List.of(testFiles)); + } + + public void testSyncEmptyList() throws IOException { + // Sync empty list should not throw exception + distributedDirectory.sync(Collections.emptyList()); + } + + public void testRenameSameDirectory() throws IOException { + String sourceFile = "_0.cfe"; + String destFile = "_0.renamed"; + + // Ensure both files would be in the same directory + int sourceIndex = distributedDirectory.getDirectoryIndex(sourceFile); + int destIndex = distributedDirectory.getDirectoryIndex(destFile); + + // Create source file + try (IndexOutput output = distributedDirectory.createOutput(sourceFile, IOContext.DEFAULT)) { + output.writeString("test content"); + } + + if (sourceIndex == destIndex) { + // Rename should work + distributedDirectory.rename(sourceFile, destFile); + + // Verify rename + expectThrows(Exception.class, () -> distributedDirectory.openInput(sourceFile, IOContext.DEFAULT)); + + try (IndexInput input = distributedDirectory.openInput(destFile, IOContext.DEFAULT)) { + assertEquals("Content should be preserved", "test content", input.readString()); + } + } else { + // If they're in different directories, rename should fail + DistributedDirectoryException exception = expectThrows(DistributedDirectoryException.class, + () -> distributedDirectory.rename(sourceFile, destFile)); + assertTrue("Exception should mention cross-directory rename", + exception.getMessage().contains("Cross-directory rename not supported")); + } + } + + public void testRenameCrossDirectory() throws IOException { + // Find two files that hash to different directories + String sourceFile = null; + String destFile = null; + + String[] candidates = {"_0.cfe", "_1.si", "_2.fnm", "_3.fdt", "_4.tim", "_5.tip"}; + + for (int i = 0; i < candidates.length; i++) { + for (int j = i + 1; j < candidates.length; j++) { + if (distributedDirectory.getDirectoryIndex(candidates[i]) != + distributedDirectory.getDirectoryIndex(candidates[j])) { + sourceFile = candidates[i]; + destFile = candidates[j]; + break; + } + } + if (sourceFile != null) break; + } + + if (sourceFile != null && destFile != null) { + // Create source file + try (IndexOutput output = distributedDirectory.createOutput(sourceFile, IOContext.DEFAULT)) { + output.writeString("test"); + } + + // Cross-directory rename should fail + DistributedDirectoryException exception = expectThrows(DistributedDirectoryException.class, + () -> distributedDirectory.rename(sourceFile, destFile)); + assertTrue("Exception should mention cross-directory rename", + exception.getMessage().contains("Cross-directory rename not supported")); + assertEquals("Exception should have correct operation", "rename", exception.getOperation()); + } + } + + public void testRenameNonExistentFile() throws IOException { + String sourceFile = "_0.cfe"; + String destFile = "_0.renamed"; + + // Ensure both would be in same directory for this test + if (distributedDirectory.getDirectoryIndex(sourceFile) == distributedDirectory.getDirectoryIndex(destFile)) { + DistributedDirectoryException exception = expectThrows(DistributedDirectoryException.class, + () -> distributedDirectory.rename(sourceFile, destFile)); + assertEquals("Exception should have correct operation", "rename", exception.getOperation()); + } + } + public void testCloseWithFiles() throws IOException { + String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm", "segments_1"}; + + // Create files in different directories + for (String filename : testFiles) { + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString("content for " + filename); + } + } + + // Verify files exist + String[] listedFiles = distributedDirectory.listAll(); + assertEquals("Should have all files", testFiles.length, listedFiles.length); + + // Close should not throw exception + distributedDirectory.close(); + + // After closing, operations should fail + expectThrows(Exception.class, () -> distributedDirectory.listAll()); + expectThrows(Exception.class, () -> distributedDirectory.openInput(testFiles[0], IOContext.DEFAULT)); + expectThrows(Exception.class, () -> distributedDirectory.createOutput("newfile", IOContext.DEFAULT)); + } + + public void testCloseEmpty() throws IOException { + // Close empty directory should not throw exception + distributedDirectory.close(); + + // Operations should fail after close + expectThrows(Exception.class, () -> distributedDirectory.listAll()); + } + + public void testCloseIdempotent() throws IOException { + // First close + distributedDirectory.close(); + + // Second close should not throw exception (idempotent) + distributedDirectory.close(); + } + + public void testResourceCleanupOrder() throws IOException { + String filename = "_0.cfe"; + + // Create file + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString("test"); + } + + // Get reference to directory manager before closing + DirectoryManager manager = distributedDirectory.getDirectoryManager(); + assertNotNull("Directory manager should exist", manager); + + // Close should clean up resources properly + distributedDirectory.close(); + + // Verify cleanup by attempting operations + expectThrows(Exception.class, () -> distributedDirectory.openInput(filename, IOContext.DEFAULT)); + } +public void testListAllFiltersSubdirectories() throws IOException { + String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm", "segments_1"}; + + // Create files + for (String filename : testFiles) { + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString("content for " + filename); + } + } + + String[] listedFiles = distributedDirectory.listAll(); + Set listedSet = Set.of(listedFiles); + + // Verify all our test files are listed + for (String testFile : testFiles) { + assertTrue("Test file should be listed: " + testFile, listedSet.contains(testFile)); + } + + // Verify subdirectory names are NOT listed (segments_1, segments_2, etc. directories) + assertFalse("Subdirectory segments_1 should not be listed as a file", + listedSet.contains("segments_1") && isDirectory("segments_1")); + assertFalse("Subdirectory segments_2 should not be listed as a file", + listedSet.contains("segments_2")); + assertFalse("Subdirectory segments_3 should not be listed as a file", + listedSet.contains("segments_3")); + assertFalse("Subdirectory segments_4 should not be listed as a file", + listedSet.contains("segments_4")); + + // But the actual segments_1 file should be listed + assertTrue("Actual segments_1 file should be listed", listedSet.contains("segments_1")); + } + + private boolean isDirectory(String name) { + // Check if this is actually a directory in the base path + return tempDir.resolve(name).toFile().isDirectory(); + } + + public void testListAllWithOnlySubdirectories() throws IOException { + // Don't create any files, just verify subdirectories are not listed + String[] listedFiles = distributedDirectory.listAll(); + + // Should not contain any of our subdirectory names + Set listedSet = Set.of(listedFiles); + for (int i = 1; i < 5; i++) { + String subdirName = "segments_" + i; + if (listedSet.contains(subdirName)) { + // If it's listed, it should be a file, not a directory + assertFalse("If segments_" + i + " is listed, it should be a file not a directory", + tempDir.resolve(subdirName).toFile().isDirectory()); + } + } + } + + public void testListAllDistinguishesFilesFromDirectories() throws IOException { + // Create a real segments_2 file (not just the directory) + String segmentsFile = "segments_2"; + try (IndexOutput output = distributedDirectory.createOutput(segmentsFile, IOContext.DEFAULT)) { + output.writeString("real segments file content"); + } + + String[] listedFiles = distributedDirectory.listAll(); + Set listedSet = Set.of(listedFiles); + + // The real segments_2 file should be listed + assertTrue("Real segments_2 file should be listed", listedSet.contains(segmentsFile)); + + // Verify we can read it back + try (IndexInput input = distributedDirectory.openInput(segmentsFile, IOContext.DEFAULT)) { + assertEquals("Should be able to read the real segments file", + "real segments file content", input.readString()); + } + } \ No newline at end of file From f19ae5a47a8e09abf6324489fe892cd6bcd211d0 Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Sun, 2 Nov 2025 23:42:51 +0530 Subject: [PATCH 2/2] [POC] primary term based directory structure Signed-off-by: Varun Bansal --- .../opensearch/index/shard/IndexShard.java | 30 + .../index/store/FsDirectoryFactory.java | 18 +- .../DistributedSegmentDirectory.java | 537 ++++++++++--- .../store/distributed/FallbackStrategy.java | 144 ++++ .../store/distributed/IndexShardContext.java | 133 ++++ .../PrimaryTermAwareDirectoryWrapper.java | 218 ++++++ .../PrimaryTermDirectoryManager.java | 492 ++++++++++++ .../store/distributed/PrimaryTermRouter.java | 157 ++++ .../PrimaryTermRoutingException.java | 126 +++ .../DefaultFilenameHasherTests.java | 117 --- .../distributed/DirectoryManagerTests.java | 139 ---- .../DistributedDirectoryExceptionTests.java | 95 --- .../DistributedDirectoryFactoryTests.java | 225 ------ .../DistributedDirectoryMetricsTests.java | 192 ----- ...ributedSegmentDirectoryBenchmarkTests.java | 185 ----- ...tributedSegmentDirectoryEdgeCaseTests.java | 0 ...butedSegmentDirectoryIntegrationTests.java | 503 ++++++------ .../DistributedSegmentDirectoryTests.java | 719 ------------------ .../distributed/IndexShardContextTests.java | 126 +++ ...PrimaryTermAwareDirectoryWrapperTests.java | 274 +++++++ .../PrimaryTermDirectoryManagerTests.java | 260 +++++++ .../distributed/PrimaryTermRouterTests.java | 189 +++++ 22 files changed, 2820 insertions(+), 2059 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/store/distributed/FallbackStrategy.java create mode 100644 server/src/main/java/org/opensearch/index/store/distributed/IndexShardContext.java create mode 100644 server/src/main/java/org/opensearch/index/store/distributed/PrimaryTermAwareDirectoryWrapper.java create mode 100644 server/src/main/java/org/opensearch/index/store/distributed/PrimaryTermDirectoryManager.java create mode 100644 server/src/main/java/org/opensearch/index/store/distributed/PrimaryTermRouter.java create mode 100644 server/src/main/java/org/opensearch/index/store/distributed/PrimaryTermRoutingException.java delete mode 100644 server/src/test/java/org/opensearch/index/store/distributed/DefaultFilenameHasherTests.java delete mode 100644 server/src/test/java/org/opensearch/index/store/distributed/DirectoryManagerTests.java delete mode 100644 server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryExceptionTests.java delete mode 100644 server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryFactoryTests.java delete mode 100644 server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryMetricsTests.java delete mode 100644 server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryBenchmarkTests.java delete mode 100644 server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryEdgeCaseTests.java delete mode 100644 server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryTests.java create mode 100644 server/src/test/java/org/opensearch/index/store/distributed/IndexShardContextTests.java create mode 100644 server/src/test/java/org/opensearch/index/store/distributed/PrimaryTermAwareDirectoryWrapperTests.java create mode 100644 server/src/test/java/org/opensearch/index/store/distributed/PrimaryTermDirectoryManagerTests.java create mode 100644 server/src/test/java/org/opensearch/index/store/distributed/PrimaryTermRouterTests.java diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 609a6290d36ce..d7e67f17a0d90 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -184,6 +184,7 @@ import org.opensearch.index.translog.RemoteFsTranslog; import org.opensearch.index.translog.RemoteTranslogStats; import org.opensearch.index.translog.Translog; +import org.opensearch.index.translog.Translog.Durability; import org.opensearch.index.translog.TranslogConfig; import org.opensearch.index.translog.TranslogFactory; import org.opensearch.index.translog.TranslogRecoveryRunner; @@ -569,6 +570,32 @@ private long getInitialGlobalCheckpointForShard(IndexSettings indexSettings) { return UNASSIGNED_SEQ_NO; } + /** + * Initializes primary term routing for the store directory if supported. + * This method should be called after the IndexShard is fully constructed + * to enable primary term-based segment file routing. + */ + private void initializePrimaryTermRouting() { + logger.info("intializing primary term based routing"); + try { + Directory storeDirectory = store.directory(); + logger.info("intializing primary term based routing {}", storeDirectory); + // if (storeDirectory instanceof org.opensearch.index.store.distributed.PrimaryTermAwareDirectoryWrapper) { + org.opensearch.index.store.distributed.PrimaryTermAwareDirectoryWrapper wrapper = + (org.opensearch.index.store.distributed.PrimaryTermAwareDirectoryWrapper) ((FilterDirectory) ((FilterDirectory)storeDirectory).getDelegate()).getDelegate(); + logger.info("intializing primary term based routing"); + + if (!wrapper.isPrimaryTermRoutingEnabled()) { + wrapper.enablePrimaryTermRouting(this); + logger.info("Enabled primary term routing for shard {}", shardId); + } + // } + } catch (Exception e) { + logger.warn("Failed to initialize primary term routing for shard {}", shardId, e); + // Don't fail shard creation if primary term routing setup fails + } + } + public ThreadPool getThreadPool() { return this.threadPool; } @@ -2444,6 +2471,9 @@ public void postRecovery(String reason) throws IndexShardStartedException, Index } recoveryState.setStage(RecoveryState.Stage.DONE); changeState(IndexShardState.POST_RECOVERY, reason); + + // Initialize primary term routing after recovery is complete + initializePrimaryTermRouting(); } } } diff --git a/server/src/main/java/org/opensearch/index/store/FsDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/FsDirectoryFactory.java index ae479b8947f8b..3839712f712fe 100644 --- a/server/src/main/java/org/opensearch/index/store/FsDirectoryFactory.java +++ b/server/src/main/java/org/opensearch/index/store/FsDirectoryFactory.java @@ -50,6 +50,7 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.store.distributed.DistributedSegmentDirectory; +import org.opensearch.index.store.distributed.PrimaryTermAwareDirectoryWrapper; import org.opensearch.plugins.IndexStorePlugin; import java.io.IOException; @@ -97,10 +98,21 @@ protected Directory newFSDirectory(Path location, LockFactory lockFactory, Index Set preLoadExtensions = new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_PRE_LOAD_SETTING)); switch (type) { case HYBRIDFS: - // return new DistributedSegmentDirectory(null, location) - // Use Lucene defaults + // Create primary directory final FSDirectory primaryDirectory = new NIOFSDirectory(location, lockFactory); - return new DistributedSegmentDirectory(primaryDirectory, location); + + return new PrimaryTermAwareDirectoryWrapper(primaryDirectory, location); + // // Check if primary term routing should be enabled + // boolean enablePrimaryTermRouting = indexSettings.getSettings() + // .getAsBoolean("index.store.distributed_segment.enable_primary_term_routing", true); + + // if (enablePrimaryTermRouting) { + // // Use wrapper that can be configured for primary term routing later + // return new PrimaryTermAwareDirectoryWrapper(primaryDirectory, location); + // } else { + // // Use legacy hash-based routing + // return new DistributedSegmentDirectory(primaryDirectory, location); + // } // final Set nioExtensions = new HashSet<>(indexSettings.getValue(IndexModule.INDEX_STORE_HYBRID_NIO_EXTENSIONS)); // if (primaryDirectory instanceof MMapDirectory) { diff --git a/server/src/main/java/org/opensearch/index/store/distributed/DistributedSegmentDirectory.java b/server/src/main/java/org/opensearch/index/store/distributed/DistributedSegmentDirectory.java index 62c7ffec9e9b1..a6ea07ed95cf7 100644 --- a/server/src/main/java/org/opensearch/index/store/distributed/DistributedSegmentDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/distributed/DistributedSegmentDirectory.java @@ -15,12 +15,14 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; +import org.opensearch.index.shard.IndexShard; import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -29,15 +31,12 @@ /** * A Directory implementation that distributes segment files across multiple - * subdirectories - * based on filename hashing. This helps improve I/O distribution by spreading - * file access - * across multiple storage paths while maintaining full Lucene Directory - * compatibility. + * subdirectories based on primary term routing. This helps improve I/O distribution + * by spreading file access across multiple storage paths while maintaining full + * Lucene Directory compatibility and providing better temporal locality. * * Critical files like segments_N are kept in the base directory to maintain - * compatibility - * with existing Lucene expectations. + * compatibility with existing Lucene expectations. * * @opensearch.internal */ @@ -45,11 +44,17 @@ public class DistributedSegmentDirectory extends FilterDirectory { private static final Logger logger = LogManager.getLogger(DistributedSegmentDirectory.class); + // Legacy fields for backward compatibility private final FilenameHasher hasher; private final DirectoryManager directoryManager; + + // New primary term routing fields + private final PrimaryTermRouter router; + private final PrimaryTermDirectoryManager primaryTermDirectoryManager; + private final boolean usePrimaryTermRouting; /** - * Creates a new DistributedSegmentDirectory with default filename hasher. + * Creates a new DistributedSegmentDirectory with default filename hasher (legacy). * * @param delegate the base Directory instance * @param basePath the base filesystem path for creating subdirectories @@ -60,7 +65,7 @@ public DistributedSegmentDirectory(Directory delegate, Path basePath) throws IOE } /** - * Creates a new DistributedSegmentDirectory with custom filename hasher. + * Creates a new DistributedSegmentDirectory with custom filename hasher (legacy). * * @param delegate the base Directory instance * @param basePath the base filesystem path for creating subdirectories @@ -71,59 +76,206 @@ public DistributedSegmentDirectory(Directory delegate, Path basePath, FilenameHa super(delegate); this.hasher = hasher; this.directoryManager = new DirectoryManager(delegate, basePath); + + // Primary term routing not available in legacy constructor + this.router = null; + this.primaryTermDirectoryManager = null; + this.usePrimaryTermRouting = false; - logger.info("Created DistributedSegmentDirectory with {} subdirectories at path: {}", + logger.info("Created DistributedSegmentDirectory with {} subdirectories at path: {} (legacy hash-based routing)", directoryManager.getNumDirectories(), basePath); } /** - * Resolves the appropriate directory for a given filename using the hasher. + * Creates a new DistributedSegmentDirectory with primary term routing. + * + * @param delegate the base Directory instance + * @param basePath the base filesystem path for creating subdirectories + * @param indexShard the IndexShard instance for primary term access + * @throws IOException if subdirectory creation fails + */ + public DistributedSegmentDirectory(Directory delegate, Path basePath, IndexShard indexShard) throws IOException { + super(delegate); + + // Initialize primary term routing components + IndexShardContext shardContext = new IndexShardContext(indexShard); + this.router = new PrimaryTermRouter(shardContext); + this.primaryTermDirectoryManager = new PrimaryTermDirectoryManager(delegate, basePath); + this.usePrimaryTermRouting = true; + + // Legacy fields set to null for primary term routing + this.hasher = null; + this.directoryManager = null; + + logger.info("Created DistributedSegmentDirectory with primary term routing at path: {}", basePath); + } + + /** + * Resolves the appropriate directory for a given filename. + * Uses primary term routing if available, otherwise falls back to hash-based routing. * * @param filename the filename to resolve * @return the Directory instance that should handle this file */ protected Directory resolveDirectory(String filename) { - int directoryIndex = hasher.getDirectoryIndex(filename); - return directoryManager.getDirectory(directoryIndex); + if (usePrimaryTermRouting) { + try { + return router.getDirectoryForFile(filename, primaryTermDirectoryManager); + } catch (IOException e) { + logger.warn("Primary term routing failed for file {}, falling back to base directory", filename, e); + return primaryTermDirectoryManager.getBaseDirectory(); + } + } else { + // Legacy hash-based routing + int directoryIndex = hasher.getDirectoryIndex(filename); + return directoryManager.getDirectory(directoryIndex); + } } /** * Gets the directory index for a filename (useful for logging and debugging). + * For primary term routing, returns the primary term as the "index". * * @param filename the filename to check - * @return the directory index (0-4) + * @return the directory index (0-4) for hash routing, or primary term for primary term routing */ protected int getDirectoryIndex(String filename) { - return hasher.getDirectoryIndex(filename); + if (usePrimaryTermRouting) { + if (router.isExcludedFile(filename)) { + return 0; // Base directory + } + return (int) router.getCurrentPrimaryTerm(); + } else { + return hasher.getDirectoryIndex(filename); + } } /** - * Gets the DirectoryManager instance (useful for testing). + * Gets the current primary term (for primary term routing). * - * @return the DirectoryManager + * @return the current primary term, or -1 if not using primary term routing + */ + public long getCurrentPrimaryTerm() { + if (usePrimaryTermRouting) { + return router.getCurrentPrimaryTerm(); + } + return -1L; + } + + /** + * Checks if this directory is using primary term routing. + * + * @return true if using primary term routing, false if using hash-based routing + */ + public boolean isUsingPrimaryTermRouting() { + return usePrimaryTermRouting; + } + + /** + * Gets the DirectoryManager instance (useful for testing, legacy routing only). + * + * @return the DirectoryManager, or null if using primary term routing */ protected DirectoryManager getDirectoryManager() { return directoryManager; } /** - * Gets the FilenameHasher instance (useful for testing). + * Gets the FilenameHasher instance (useful for testing, legacy routing only). * - * @return the FilenameHasher + * @return the FilenameHasher, or null if using primary term routing */ protected FilenameHasher getHasher() { return hasher; } + /** + * Gets the PrimaryTermRouter instance (useful for testing, primary term routing only). + * + * @return the PrimaryTermRouter, or null if using hash-based routing + */ + protected PrimaryTermRouter getRouter() { + return router; + } + + /** + * Gets the PrimaryTermDirectoryManager instance (useful for testing, primary term routing only). + * + * @return the PrimaryTermDirectoryManager, or null if using hash-based routing + */ + protected PrimaryTermDirectoryManager getPrimaryTermDirectoryManager() { + return primaryTermDirectoryManager; + } + + /** + * Gets detailed routing information for a filename for debugging purposes. + * + * @param filename the filename to analyze + * @return a string describing the routing decision + */ + public String getRoutingInfo(String filename) { + if (usePrimaryTermRouting) { + if (router.isExcludedFile(filename)) { + return String.format("File '%s' excluded from primary term routing, using base directory", filename); + } else { + long primaryTerm = router.getCurrentPrimaryTerm(); + return String.format("File '%s' routed to primary term %d directory", filename, primaryTerm); + } + } else { + int index = hasher.getDirectoryIndex(filename); + return String.format("File '%s' routed to hash-based directory index %d", filename, index); + } + } + + /** + * Gets statistics about the current directory usage. + * + * @return directory usage statistics + */ + public String getDirectoryStats() { + if (usePrimaryTermRouting) { + PrimaryTermDirectoryManager.DirectoryStats stats = primaryTermDirectoryManager.getDirectoryStats(); + return String.format("Primary term routing: %s", stats.toString()); + } else { + return String.format("Hash-based routing: %d directories", directoryManager.getNumDirectories()); + } + } + + /** + * Validates all managed directories for accessibility and health. + * + * @throws IOException if validation fails + */ + public void validateDirectories() throws IOException { + if (usePrimaryTermRouting) { + primaryTermDirectoryManager.validateDirectories(); + } else { + // Legacy validation would go here if DirectoryManager had such a method + logger.debug("Directory validation not implemented for hash-based routing"); + } + } + @Override public void close() throws IOException { IOException exception = null; - // Close the directory manager (which closes subdirectories) - try { - directoryManager.close(); - } catch (IOException e) { - exception = e; + // Close the appropriate directory manager + if (usePrimaryTermRouting) { + try { + if (primaryTermDirectoryManager != null) { + primaryTermDirectoryManager.close(); + } + } catch (IOException e) { + exception = e; + } + } else { + try { + if (directoryManager != null) { + directoryManager.close(); + } + } catch (IOException e) { + exception = e; + } } // Close the base directory @@ -141,7 +293,7 @@ public void close() throws IOException { throw exception; } - logger.debug("Closed DistributedSegmentDirectory"); + logger.debug("Closed DistributedSegmentDirectory (primary term routing: {})", usePrimaryTermRouting); } // Placeholder methods - will be implemented in subsequent tasks @@ -151,35 +303,71 @@ public String[] listAll() throws IOException { Set allFiles = new HashSet<>(); IOException lastException = null; - // Collect files from all subdirectories - for (int i = 0; i < directoryManager.getNumDirectories(); i++) { + if (usePrimaryTermRouting) { + // Primary term routing: collect files from base directory and all primary term directories try { - Directory dir = directoryManager.getDirectory(i); - String[] files = dir.listAll(); - - // Filter out subdirectory names from base directory (index 0) - if (i == 0) { - for (String file : files) { - // Only add files that are not our created subdirectories - if (!file.startsWith("varun_segments_") || !isSubdirectoryName(file)) { - allFiles.add(file); - } + // Add files from base directory + Directory baseDir = primaryTermDirectoryManager.getBaseDirectory(); + String[] baseFiles = baseDir.listAll(); + + for (String file : baseFiles) { + // Filter out primary term subdirectories from base directory listing + if (!file.startsWith(PrimaryTermDirectoryManager.PRIMARY_TERM_DIR_PREFIX)) { + allFiles.add(file); } - } else { - // For other directories, add all files + } + + logger.debug("Listed {} files from base directory (filtered)", baseFiles.length); + } catch (IOException e) { + logger.warn("Failed to list files from base directory: {}", e.getMessage()); + lastException = e; + } + + // Add files from all primary term directories + for (Long primaryTerm : primaryTermDirectoryManager.getAllPrimaryTerms()) { + try { + Directory dir = primaryTermDirectoryManager.getDirectoryForPrimaryTerm(primaryTerm); + String[] files = dir.listAll(); Collections.addAll(allFiles, files); + + logger.debug("Listed {} files from primary term {} directory", files.length, primaryTerm); + } catch (IOException e) { + logger.warn("Failed to list files from primary term {} directory: {}", primaryTerm, e.getMessage()); + lastException = e; + // Continue with other directories } + } + } else { + // Legacy hash-based routing + for (int i = 0; i < directoryManager.getNumDirectories(); i++) { + try { + Directory dir = directoryManager.getDirectory(i); + String[] files = dir.listAll(); + + // Filter out subdirectory names from base directory (index 0) + if (i == 0) { + for (String file : files) { + // Only add files that are not our created subdirectories + if (!file.startsWith("varun_segments_") || !isSubdirectoryName(file)) { + allFiles.add(file); + } + } + } else { + // For other directories, add all files + Collections.addAll(allFiles, files); + } - logger.info("Listed {} files from directory {} (filtered: {})", - files.length, i, i == 0 ? "yes" : "no"); - } catch (IOException e) { - logger.warn("Failed to list files from directory {}: {}", i, e.getMessage()); - lastException = new DistributedDirectoryException( - "Failed to list files from directory", - i, - "listAll", - e); - // Continue trying other directories + logger.debug("Listed {} files from directory {} (filtered: {})", + files.length, i, i == 0 ? "yes" : "no"); + } catch (IOException e) { + logger.warn("Failed to list files from directory {}: {}", i, e.getMessage()); + lastException = new DistributedDirectoryException( + "Failed to list files from directory", + i, + "listAll", + e); + // Continue trying other directories + } } } @@ -189,7 +377,8 @@ public String[] listAll() throws IOException { } String[] result = allFiles.toArray(new String[0]); - logger.info("Listed total {} unique files across all directories", result.length); + logger.info("Listed total {} unique files across all directories (routing: {})", + result.length, usePrimaryTermRouting ? "primary-term" : "hash-based"); return result; } @@ -218,102 +407,178 @@ private boolean isSubdirectoryName(String filename) { @Override public IndexInput openInput(String name, IOContext context) throws IOException { - int dirIndex = getDirectoryIndex(name); - Directory targetDir = directoryManager.getDirectory(dirIndex); - - logger.info("Opening input for file {} in directory {}", name, dirIndex); + Directory targetDir = resolveDirectory(name); + + if (usePrimaryTermRouting) { + long primaryTerm = router.isExcludedFile(name) ? -1 : router.getCurrentPrimaryTerm(); + logger.debug("Opening input for file {} (primary term: {})", name, primaryTerm); + } else { + int dirIndex = getDirectoryIndex(name); + logger.debug("Opening input for file {} in directory {}", name, dirIndex); + } try { return targetDir.openInput(name, context); } catch (IOException e) { - throw new DistributedDirectoryException( - "Failed to open input for file: " + name, - dirIndex, - "openInput", - e); + if (usePrimaryTermRouting) { + throw new PrimaryTermRoutingException( + "Failed to open input for file: " + name, + PrimaryTermRoutingException.ErrorType.FILE_ROUTING_ERROR, + router.getCurrentPrimaryTerm(), + name, + e); + } else { + throw new DistributedDirectoryException( + "Failed to open input for file: " + name, + getDirectoryIndex(name), + "openInput", + e); + } } } @Override public IndexOutput createOutput(String name, IOContext context) throws IOException { - int dirIndex = getDirectoryIndex(name); - Directory targetDir = directoryManager.getDirectory(dirIndex); - - logger.info("Creating output for file {} in directory {}", name, dirIndex); + Directory targetDir = resolveDirectory(name); + + if (usePrimaryTermRouting) { + long primaryTerm = router.isExcludedFile(name) ? -1 : router.getCurrentPrimaryTerm(); + logger.debug("Creating output for file {} (primary term: {})", name, primaryTerm); + } else { + int dirIndex = getDirectoryIndex(name); + logger.debug("Creating output for file {} in directory {}", name, dirIndex); + } try { return targetDir.createOutput(name, context); } catch (IOException e) { - throw new DistributedDirectoryException( - "Failed to create output for file: " + name, - dirIndex, - "createOutput", - e); + if (usePrimaryTermRouting) { + throw new PrimaryTermRoutingException( + "Failed to create output for file: " + name, + PrimaryTermRoutingException.ErrorType.FILE_ROUTING_ERROR, + router.getCurrentPrimaryTerm(), + name, + e); + } else { + throw new DistributedDirectoryException( + "Failed to create output for file: " + name, + getDirectoryIndex(name), + "createOutput", + e); + } } } @Override public void deleteFile(String name) throws IOException { - int dirIndex = getDirectoryIndex(name); - Directory targetDir = directoryManager.getDirectory(dirIndex); - - logger.info("Deleting file {} from directory {}", name, dirIndex); + Directory targetDir = resolveDirectory(name); + + if (usePrimaryTermRouting) { + long primaryTerm = router.isExcludedFile(name) ? -1 : router.getCurrentPrimaryTerm(); + logger.debug("Deleting file {} (primary term: {})", name, primaryTerm); + } else { + int dirIndex = getDirectoryIndex(name); + logger.debug("Deleting file {} from directory {}", name, dirIndex); + } try { targetDir.deleteFile(name); } catch (IOException e) { - throw new DistributedDirectoryException( - "Failed to delete file: " + name, - dirIndex, - "deleteFile", - e); + if (usePrimaryTermRouting) { + throw new PrimaryTermRoutingException( + "Failed to delete file: " + name, + PrimaryTermRoutingException.ErrorType.FILE_ROUTING_ERROR, + router.getCurrentPrimaryTerm(), + name, + e); + } else { + throw new DistributedDirectoryException( + "Failed to delete file: " + name, + getDirectoryIndex(name), + "deleteFile", + e); + } } } @Override public long fileLength(String name) throws IOException { - int dirIndex = getDirectoryIndex(name); - Directory targetDir = directoryManager.getDirectory(dirIndex); + Directory targetDir = resolveDirectory(name); try { return targetDir.fileLength(name); } catch (IOException e) { - throw new DistributedDirectoryException( - "Failed to get file length for: " + name, - dirIndex, - "fileLength", - e); + if (usePrimaryTermRouting) { + throw new PrimaryTermRoutingException( + "Failed to get file length for: " + name, + PrimaryTermRoutingException.ErrorType.FILE_ROUTING_ERROR, + router.getCurrentPrimaryTerm(), + name, + e); + } else { + throw new DistributedDirectoryException( + "Failed to get file length for: " + name, + getDirectoryIndex(name), + "fileLength", + e); + } } } @Override public void sync(Collection names) throws IOException { - // Group files by directory and sync each directory separately - Map> filesByDirectory = names.stream() - .collect(Collectors.groupingBy(this::getDirectoryIndex)); - List exceptions = new ArrayList<>(); - for (Map.Entry> entry : filesByDirectory.entrySet()) { - int dirIndex = entry.getKey(); - List files = entry.getValue(); + if (usePrimaryTermRouting) { + // Group files by directory (base directory or primary term directories) + Map> filesByDirectory = new HashMap<>(); + + for (String name : names) { + Directory targetDir = resolveDirectory(name); + filesByDirectory.computeIfAbsent(targetDir, k -> new ArrayList<>()).add(name); + } - try { - Directory dir = directoryManager.getDirectory(dirIndex); - dir.sync(files); - logger.info("Synced {} files in directory {}", files.size(), dirIndex); - } catch (IOException e) { - logger.warn("Failed to sync {} files in directory {}: {}", files.size(), dirIndex, e.getMessage()); - exceptions.add(new DistributedDirectoryException( - "Failed to sync files: " + files, - dirIndex, - "sync", - e)); + for (Map.Entry> entry : filesByDirectory.entrySet()) { + Directory dir = entry.getKey(); + List files = entry.getValue(); + + try { + dir.sync(files); + logger.debug("Synced {} files in primary term directory", files.size()); + } catch (IOException e) { + logger.warn("Failed to sync {} files in primary term directory: {}", files.size(), e.getMessage()); + exceptions.add(new PrimaryTermRoutingException( + "Failed to sync files: " + files, + PrimaryTermRoutingException.ErrorType.FILE_ROUTING_ERROR, + router.getCurrentPrimaryTerm(), + e)); + } + } + } else { + // Legacy hash-based routing + Map> filesByDirectory = names.stream() + .collect(Collectors.groupingBy(this::getDirectoryIndex)); + + for (Map.Entry> entry : filesByDirectory.entrySet()) { + int dirIndex = entry.getKey(); + List files = entry.getValue(); + + try { + Directory dir = directoryManager.getDirectory(dirIndex); + dir.sync(files); + logger.debug("Synced {} files in directory {}", files.size(), dirIndex); + } catch (IOException e) { + logger.warn("Failed to sync {} files in directory {}: {}", files.size(), dirIndex, e.getMessage()); + exceptions.add(new DistributedDirectoryException( + "Failed to sync files: " + files, + dirIndex, + "sync", + e)); + } } } - // If any sync operations failed, throw the first exception with others as - // suppressed + // If any sync operations failed, throw the first exception with others as suppressed if (!exceptions.isEmpty()) { IOException primaryException = exceptions.get(0); for (int i = 1; i < exceptions.size(); i++) { @@ -322,35 +587,61 @@ public void sync(Collection names) throws IOException { throw primaryException; } - logger.info("Successfully synced {} files across {} directories", - names.size(), filesByDirectory.size()); + logger.info("Successfully synced {} files across directories (routing: {})", + names.size(), usePrimaryTermRouting ? "primary-term" : "hash-based"); } @Override public void rename(String source, String dest) throws IOException { - int sourceIndex = getDirectoryIndex(source); - int destIndex = getDirectoryIndex(dest); + Directory sourceDir = resolveDirectory(source); + Directory destDir = resolveDirectory(dest); - if (sourceIndex != destIndex) { + if (sourceDir != destDir) { // Cross-directory rename - not supported atomically - throw new DistributedDirectoryException( - "Cross-directory rename not supported: " + source + " (dir " + sourceIndex + - ") -> " + dest + " (dir " + destIndex + ")", - sourceIndex, - "rename"); + if (usePrimaryTermRouting) { + long sourcePrimaryTerm = router.isExcludedFile(source) ? -1 : router.getCurrentPrimaryTerm(); + long destPrimaryTerm = router.isExcludedFile(dest) ? -1 : router.getCurrentPrimaryTerm(); + throw new PrimaryTermRoutingException( + "Cross-directory rename not supported: " + source + " (primary term " + sourcePrimaryTerm + + ") -> " + dest + " (primary term " + destPrimaryTerm + ")", + PrimaryTermRoutingException.ErrorType.FILE_ROUTING_ERROR, + sourcePrimaryTerm, + source); + } else { + int sourceIndex = getDirectoryIndex(source); + int destIndex = getDirectoryIndex(dest); + throw new DistributedDirectoryException( + "Cross-directory rename not supported: " + source + " (dir " + sourceIndex + + ") -> " + dest + " (dir " + destIndex + ")", + sourceIndex, + "rename"); + } } - Directory targetDir = directoryManager.getDirectory(sourceIndex); - try { - targetDir.rename(source, dest); - logger.info("Renamed {} to {} in directory {}", source, dest, sourceIndex); + sourceDir.rename(source, dest); + if (usePrimaryTermRouting) { + long primaryTerm = router.isExcludedFile(source) ? -1 : router.getCurrentPrimaryTerm(); + logger.debug("Renamed {} to {} (primary term: {})", source, dest, primaryTerm); + } else { + int dirIndex = getDirectoryIndex(source); + logger.debug("Renamed {} to {} in directory {}", source, dest, dirIndex); + } } catch (IOException e) { - throw new DistributedDirectoryException( - "Failed to rename " + source + " to " + dest, - sourceIndex, - "rename", - e); + if (usePrimaryTermRouting) { + throw new PrimaryTermRoutingException( + "Failed to rename " + source + " to " + dest, + PrimaryTermRoutingException.ErrorType.FILE_ROUTING_ERROR, + router.getCurrentPrimaryTerm(), + source, + e); + } else { + throw new DistributedDirectoryException( + "Failed to rename " + source + " to " + dest, + getDirectoryIndex(source), + "rename", + e); + } } } } \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/index/store/distributed/FallbackStrategy.java b/server/src/main/java/org/opensearch/index/store/distributed/FallbackStrategy.java new file mode 100644 index 0000000000000..da2bb64003557 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/distributed/FallbackStrategy.java @@ -0,0 +1,144 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.store.Directory; + +/** + * Provides fallback strategies for primary term routing failures. + * This class handles graceful degradation scenarios when primary term + * routing encounters errors, ensuring the system continues to function + * by falling back to the base directory. + * + * @opensearch.internal + */ +public class FallbackStrategy { + + private static final Logger logger = LogManager.getLogger(FallbackStrategy.class); + + /** + * Handles primary term routing failures by falling back to the base directory. + * This method is called when primary term access fails or routing encounters errors. + * + * @param filename the filename that failed to route + * @param manager the directory manager to get the base directory from + * @param cause the exception that caused the failure + * @return the base directory as a fallback + */ + public static Directory handlePrimaryTermFailure(String filename, PrimaryTermDirectoryManager manager, Exception cause) { + logger.warn("Primary term routing failed for file '{}', falling back to base directory. Cause: {}", + filename, cause.getMessage()); + + if (logger.isDebugEnabled()) { + logger.debug("Primary term routing failure details for file: " + filename, cause); + } + + return manager.getBaseDirectory(); + } + + /** + * Handles directory creation failures by falling back to the base directory. + * This method is called when creating a new primary term directory fails. + * + * @param primaryTerm the primary term for which directory creation failed + * @param manager the directory manager to get the base directory from + * @param cause the exception that caused the failure + * @return the base directory as a fallback + */ + public static Directory handleDirectoryCreationFailure(long primaryTerm, PrimaryTermDirectoryManager manager, Exception cause) { + logger.error("Failed to create directory for primary term {}, using base directory. Cause: {}", + primaryTerm, cause.getMessage()); + + if (logger.isDebugEnabled()) { + logger.debug("Directory creation failure details for primary term: " + primaryTerm, cause); + } + + return manager.getBaseDirectory(); + } + + /** + * Handles directory validation failures by falling back to the base directory. + * This method is called when directory accessibility or permission checks fail. + * + * @param primaryTerm the primary term for which validation failed + * @param directoryPath the path that failed validation + * @param manager the directory manager to get the base directory from + * @param cause the exception that caused the failure + * @return the base directory as a fallback + */ + public static Directory handleDirectoryValidationFailure(long primaryTerm, String directoryPath, + PrimaryTermDirectoryManager manager, Exception cause) { + logger.warn("Directory validation failed for primary term {} at path '{}', using base directory. Cause: {}", + primaryTerm, directoryPath, cause.getMessage()); + + if (logger.isDebugEnabled()) { + logger.debug("Directory validation failure details for primary term " + primaryTerm + " at path: " + directoryPath, cause); + } + + return manager.getBaseDirectory(); + } + + /** + * Handles IndexShard unavailability by using the default primary term. + * This method is called when IndexShard context is not available for primary term access. + * + * @param operation the operation that was being attempted + * @return the default primary term to use as fallback + */ + public static long handleIndexShardUnavailable(String operation) { + logger.debug("IndexShard unavailable for operation '{}', using default primary term", operation); + return IndexShardContext.DEFAULT_PRIMARY_TERM; + } + + /** + * Creates a PrimaryTermRoutingException with appropriate error context. + * This is a utility method for consistent exception creation with fallback logging. + * + * @param message the error message + * @param errorType the type of error + * @param primaryTerm the primary term associated with the error + * @param filename the filename associated with the error, may be null + * @param cause the underlying cause + * @return a new PrimaryTermRoutingException + */ + public static PrimaryTermRoutingException createRoutingException(String message, + PrimaryTermRoutingException.ErrorType errorType, + long primaryTerm, + String filename, + Throwable cause) { + PrimaryTermRoutingException exception = new PrimaryTermRoutingException(message, errorType, primaryTerm, filename, cause); + + logger.warn("Created routing exception: {}", exception.getMessage()); + if (logger.isDebugEnabled()) { + logger.debug("Routing exception details", exception); + } + + return exception; + } + + /** + * Logs a successful recovery from a fallback scenario. + * This method should be called when the system successfully recovers from a failure. + * + * @param operation the operation that recovered + * @param primaryTerm the primary term involved + * @param filename the filename involved, may be null + */ + public static void logSuccessfulRecovery(String operation, long primaryTerm, String filename) { + if (filename != null) { + logger.info("Successfully recovered from fallback for operation '{}' with primary term {} and file '{}'", + operation, primaryTerm, filename); + } else { + logger.info("Successfully recovered from fallback for operation '{}' with primary term {}", + operation, primaryTerm); + } + } +} \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/index/store/distributed/IndexShardContext.java b/server/src/main/java/org/opensearch/index/store/distributed/IndexShardContext.java new file mode 100644 index 0000000000000..49e2c0d139085 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/distributed/IndexShardContext.java @@ -0,0 +1,133 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.index.shard.IndexShard; + +/** + * Provides safe access to IndexShard for primary term information with caching. + * This class handles cases where IndexShard may be unavailable and provides + * fallback mechanisms for primary term routing. + * + * @opensearch.internal + */ +public class IndexShardContext { + + private static final Logger logger = LogManager.getLogger(IndexShardContext.class); + + /** Default primary term used when IndexShard is unavailable */ + public static final long DEFAULT_PRIMARY_TERM = 0L; + + /** Cache duration for primary term values (1 second) */ + private static final long CACHE_DURATION_MS = 1000L; + + private final IndexShard indexShard; + private volatile long cachedPrimaryTerm = -1L; + private volatile long cacheTimestamp = 0L; + + /** + * Creates a new IndexShardContext with the given IndexShard reference. + * + * @param indexShard the IndexShard instance to access, may be null + */ + public IndexShardContext(IndexShard indexShard) { + this.indexShard = indexShard; + logger.debug("Created IndexShardContext with IndexShard: {}", indexShard != null ? "available" : "null"); + } + + /** + * Gets the current primary term from the IndexShard with caching. + * Returns a cached value if it's recent (within CACHE_DURATION_MS), + * otherwise fetches a fresh value from the IndexShard. + * + * @return the current primary term, or DEFAULT_PRIMARY_TERM if unavailable + */ + public long getPrimaryTerm() { + long currentTime = System.currentTimeMillis(); + + // Use cached value if recent and valid + if (cachedPrimaryTerm != -1L && (currentTime - cacheTimestamp) < CACHE_DURATION_MS) { + logger.trace("Using cached primary term: {}", cachedPrimaryTerm); + return cachedPrimaryTerm; + } + + if (indexShard != null) { + try { + long primaryTerm = indexShard.getOperationPrimaryTerm(); + + // Update cache + cachedPrimaryTerm = primaryTerm; + cacheTimestamp = currentTime; + + logger.debug("Retrieved primary term from IndexShard: {}", primaryTerm); + return primaryTerm; + } catch (Exception e) { + logger.warn("Failed to get primary term from IndexShard, using default", e); + // Don't cache failed attempts + } + } else { + logger.debug("IndexShard is null, using default primary term"); + } + + return DEFAULT_PRIMARY_TERM; + } + + /** + * Checks if the IndexShard is available for primary term access. + * + * @return true if IndexShard is available, false otherwise + */ + public boolean isAvailable() { + return indexShard != null; + } + + /** + * Invalidates the cached primary term, forcing a fresh fetch on next access. + * This can be useful when primary term changes are detected externally. + */ + public void invalidateCache() { + cachedPrimaryTerm = -1L; + cacheTimestamp = 0L; + logger.debug("Invalidated primary term cache"); + } + + /** + * Gets the IndexShard instance (for testing purposes). + * + * @return the IndexShard instance, may be null + */ + protected IndexShard getIndexShard() { + return indexShard; + } + + /** + * Gets the cached primary term value (for testing purposes). + * + * @return the cached primary term, or -1 if not cached + */ + protected long getCachedPrimaryTerm() { + return cachedPrimaryTerm; + } + + /** + * Checks if the cache is valid (for testing purposes). + * + * @return true if cache is valid and recent, false otherwise + */ + protected boolean isCacheValid() { + if (cachedPrimaryTerm == -1L) { + return false; + } + + long currentTime = System.currentTimeMillis(); + return (currentTime - cacheTimestamp) < CACHE_DURATION_MS; + } +} \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/index/store/distributed/PrimaryTermAwareDirectoryWrapper.java b/server/src/main/java/org/opensearch/index/store/distributed/PrimaryTermAwareDirectoryWrapper.java new file mode 100644 index 0000000000000..f83fc5be3d75b --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/distributed/PrimaryTermAwareDirectoryWrapper.java @@ -0,0 +1,218 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FilterDirectory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.opensearch.index.shard.IndexShard; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collection; +import java.util.concurrent.atomic.AtomicReference; + +/** + * A wrapper directory that can be initialized without IndexShard and later + * configured to use primary term routing when the IndexShard becomes available. + * This allows the DirectoryFactory to create the directory before IndexShard + * is created, and then the IndexShard can configure primary term routing later. + * + * @opensearch.internal + */ +public class PrimaryTermAwareDirectoryWrapper extends FilterDirectory { + + private static final Logger logger = LogManager.getLogger(PrimaryTermAwareDirectoryWrapper.class); + + private final Path basePath; + private final AtomicReference distributedDirectory; + private volatile boolean primaryTermRoutingEnabled = false; + + /** + * Creates a new PrimaryTermAwareDirectoryWrapper. + * + * @param delegate the base Directory instance + * @param basePath the base filesystem path for creating subdirectories + */ + public PrimaryTermAwareDirectoryWrapper(Directory delegate, Path basePath) { + super(delegate); + this.basePath = basePath; + this.distributedDirectory = new AtomicReference<>(); + + logger.debug("Created PrimaryTermAwareDirectoryWrapper at path: {}", basePath); + } + + /** + * Enables primary term routing by setting the IndexShard reference. + * This method should be called after the IndexShard is created. + * + * @param indexShard the IndexShard instance for primary term access + * @throws IOException if primary term routing setup fails + */ + public void enablePrimaryTermRouting(IndexShard indexShard) throws IOException { + if (primaryTermRoutingEnabled) { + logger.debug("Primary term routing already enabled"); + return; + } + + try { + DistributedSegmentDirectory newDistributedDirectory = new DistributedSegmentDirectory(in, basePath, indexShard); + distributedDirectory.set(newDistributedDirectory); + primaryTermRoutingEnabled = true; + + logger.info("Enabled primary term routing for directory at path: {}", basePath); + } catch (IOException e) { + logger.error("Failed to enable primary term routing", e); + throw e; + } + } + + /** + * Gets the underlying distributed directory if primary term routing is enabled. + * + * @return the DistributedSegmentDirectory, or null if not enabled + */ + private DistributedSegmentDirectory getDistributedDirectory() { + return distributedDirectory.get(); + } + + /** + * Checks if primary term routing is enabled. + * + * @return true if primary term routing is enabled, false otherwise + */ + public boolean isPrimaryTermRoutingEnabled() { + return primaryTermRoutingEnabled; + } + + @Override + public String[] listAll() throws IOException { + DistributedSegmentDirectory distributed = getDistributedDirectory(); + if (distributed != null) { + return distributed.listAll(); + } + return super.listAll(); + } + + @Override + public IndexInput openInput(String name, IOContext context) throws IOException { + DistributedSegmentDirectory distributed = getDistributedDirectory(); + if (distributed != null) { + return distributed.openInput(name, context); + } + return super.openInput(name, context); + } + + @Override + public IndexOutput createOutput(String name, IOContext context) throws IOException { + DistributedSegmentDirectory distributed = getDistributedDirectory(); + if (distributed != null) { + return distributed.createOutput(name, context); + } + return super.createOutput(name, context); + } + + @Override + public void deleteFile(String name) throws IOException { + DistributedSegmentDirectory distributed = getDistributedDirectory(); + if (distributed != null) { + distributed.deleteFile(name); + } else { + super.deleteFile(name); + } + } + + @Override + public long fileLength(String name) throws IOException { + DistributedSegmentDirectory distributed = getDistributedDirectory(); + if (distributed != null) { + return distributed.fileLength(name); + } + return super.fileLength(name); + } + + @Override + public void sync(Collection names) throws IOException { + DistributedSegmentDirectory distributed = getDistributedDirectory(); + if (distributed != null) { + distributed.sync(names); + } else { + super.sync(names); + } + } + + @Override + public void rename(String source, String dest) throws IOException { + DistributedSegmentDirectory distributed = getDistributedDirectory(); + if (distributed != null) { + distributed.rename(source, dest); + } else { + super.rename(source, dest); + } + } + + @Override + public void close() throws IOException { + IOException exception = null; + + // Close the distributed directory if it exists + DistributedSegmentDirectory distributed = getDistributedDirectory(); + if (distributed != null) { + try { + distributed.close(); + } catch (IOException e) { + exception = e; + } + } + + // Close the base directory + try { + super.close(); + } catch (IOException e) { + if (exception == null) { + exception = e; + } else { + exception.addSuppressed(e); + } + } + + if (exception != null) { + throw exception; + } + + logger.debug("Closed PrimaryTermAwareDirectoryWrapper"); + } + + /** + * Gets routing information for debugging purposes. + * + * @param filename the filename to analyze + * @return routing information string + */ + public String getRoutingInfo(String filename) { + DistributedSegmentDirectory distributed = getDistributedDirectory(); + if (distributed != null) { + return distributed.getRoutingInfo(filename); + } + return "Primary term routing not enabled, using base directory"; + } + + /** + * Gets the base path. + * + * @return the base filesystem path + */ + public Path getBasePath() { + return basePath; + } +} \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/index/store/distributed/PrimaryTermDirectoryManager.java b/server/src/main/java/org/opensearch/index/store/distributed/PrimaryTermDirectoryManager.java new file mode 100644 index 0000000000000..e43c33b0ed33a --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/distributed/PrimaryTermDirectoryManager.java @@ -0,0 +1,492 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Manages the creation and lifecycle of directories based on primary terms. + * This class handles dynamic directory creation, maintains mappings between + * primary terms and Directory instances, and provides directory validation + * and cleanup functionality. + * + * @opensearch.internal + */ +public class PrimaryTermDirectoryManager implements Closeable { + + private static final Logger logger = LogManager.getLogger(PrimaryTermDirectoryManager.class); + + /** Prefix used for primary term directory names */ + public static final String PRIMARY_TERM_DIR_PREFIX = "primary_term_"; + + private final Directory baseDirectory; + private final Path basePath; + private final Map primaryTermDirectories; + private volatile boolean closed = false; + + /** + * Creates a new PrimaryTermDirectoryManager. + * + * @param baseDirectory the base directory for fallback operations + * @param basePath the base filesystem path for creating subdirectories + * @throws IOException if initialization fails + */ + public PrimaryTermDirectoryManager(Directory baseDirectory, Path basePath) throws IOException { + this.baseDirectory = baseDirectory; + this.basePath = basePath; + this.primaryTermDirectories = new ConcurrentHashMap<>(); + + // Ensure base path exists + Files.createDirectories(basePath); + + logger.info("Created PrimaryTermDirectoryManager at path: {}", basePath); + } + + /** + * Gets the directory for a given primary term, creating it if necessary. + * This method uses lazy initialization with thread-safe double-check pattern. + * + * @param primaryTerm the primary term + * @return the Directory instance for this primary term + * @throws IOException if directory creation fails + */ + public Directory getDirectoryForPrimaryTerm(long primaryTerm) throws IOException { + ensureNotClosed(); + + Directory existing = primaryTermDirectories.get(primaryTerm); + if (existing != null) { + return existing; + } + + return createDirectoryForPrimaryTerm(primaryTerm); + } + + /** + * Creates a new directory for the given primary term using thread-safe double-check pattern. + * This method is synchronized to prevent race conditions during directory creation. + * + * @param primaryTerm the primary term + * @return the newly created Directory instance + * @throws IOException if directory creation fails + */ + private synchronized Directory createDirectoryForPrimaryTerm(long primaryTerm) throws IOException { + ensureNotClosed(); + + // Double-check pattern for thread safety + Directory existing = primaryTermDirectories.get(primaryTerm); + if (existing != null) { + return existing; + } + + try { + String dirName = getDirectoryNameForPrimaryTerm(primaryTerm); + Path dirPath = basePath.resolve(dirName); + + // Create directory if it doesn't exist + Files.createDirectories(dirPath); + + // Create and store the Directory instance + Directory newDirectory = FSDirectory.open(dirPath); + primaryTermDirectories.put(primaryTerm, newDirectory); + + logger.info("Created directory for primary term {}: {}", primaryTerm, dirPath); + return newDirectory; + + } catch (IOException e) { + logger.error("Failed to create directory for primary term {}", primaryTerm, e); + throw new PrimaryTermRoutingException( + "Failed to create directory for primary term " + primaryTerm, + PrimaryTermRoutingException.ErrorType.DIRECTORY_CREATION_ERROR, + primaryTerm, + e + ); + } + } + + /** + * Gets the directory name for a given primary term using the standard naming convention. + * + * @param primaryTerm the primary term + * @return the directory name (e.g., "primary_term_1") + */ + public String getDirectoryNameForPrimaryTerm(long primaryTerm) { + return PRIMARY_TERM_DIR_PREFIX + primaryTerm; + } + + /** + * Gets the base directory used for fallback operations. + * + * @return the base Directory instance + */ + public Directory getBaseDirectory() { + return baseDirectory; + } + + /** + * Gets the base filesystem path. + * + * @return the base Path + */ + public Path getBasePath() { + return basePath; + } + + /** + * Lists all primary term directories currently managed. + * + * @return a list of all Directory instances + */ + public List listAllDirectories() { + ensureNotClosed(); + return new ArrayList<>(primaryTermDirectories.values()); + } + + /** + * Gets all primary terms that have directories. + * + * @return a set of all primary terms with directories + */ + public Set getAllPrimaryTerms() { + ensureNotClosed(); + return primaryTermDirectories.keySet(); + } + + /** + * Gets the number of primary term directories currently managed. + * + * @return the number of directories + */ + public int getDirectoryCount() { + return primaryTermDirectories.size(); + } + + /** + * Checks if a directory exists for the given primary term. + * + * @param primaryTerm the primary term to check + * @return true if a directory exists, false otherwise + */ + public boolean hasDirectoryForPrimaryTerm(long primaryTerm) { + return primaryTermDirectories.containsKey(primaryTerm); + } + + /** + * Validates that all managed directories are accessible. + * This method checks each directory for basic accessibility. + * + * @throws IOException if any directory validation fails + */ + public void validateDirectories() throws IOException { + ensureNotClosed(); + + List validationErrors = new ArrayList<>(); + + for (Map.Entry entry : primaryTermDirectories.entrySet()) { + long primaryTerm = entry.getKey(); + Directory directory = entry.getValue(); + + try { + // Basic validation - try to list files + directory.listAll(); + logger.debug("Validated directory for primary term {}", primaryTerm); + } catch (Exception e) { + logger.warn("Validation failed for primary term {} directory", primaryTerm, e); + validationErrors.add(new PrimaryTermRoutingException( + "Directory validation failed for primary term " + primaryTerm, + PrimaryTermRoutingException.ErrorType.DIRECTORY_VALIDATION_ERROR, + primaryTerm, + e + )); + } + } + + if (!validationErrors.isEmpty()) { + IOException combinedException = new IOException("Directory validation failed for " + validationErrors.size() + " directories"); + for (Exception error : validationErrors) { + combinedException.addSuppressed(error); + } + throw combinedException; + } + + logger.info("Successfully validated {} primary term directories", primaryTermDirectories.size()); + } + + /** + * Closes all managed directories and releases resources. + * This method should be called when the manager is no longer needed. + */ + @Override + public void close() throws IOException { + if (closed) { + return; + } + + closed = true; + + List closeExceptions = new ArrayList<>(); + + // Close all primary term directories + for (Map.Entry entry : primaryTermDirectories.entrySet()) { + try { + entry.getValue().close(); + logger.debug("Closed directory for primary term {}", entry.getKey()); + } catch (IOException e) { + logger.warn("Failed to close directory for primary term {}", entry.getKey(), e); + closeExceptions.add(e); + } + } + + primaryTermDirectories.clear(); + + // If there were close exceptions, throw the first one with others as suppressed + if (!closeExceptions.isEmpty()) { + IOException primaryException = closeExceptions.get(0); + for (int i = 1; i < closeExceptions.size(); i++) { + primaryException.addSuppressed(closeExceptions.get(i)); + } + throw primaryException; + } + + logger.info("Closed PrimaryTermDirectoryManager with {} directories", primaryTermDirectories.size()); + } + + /** + * Checks if this manager has been closed. + * + * @return true if closed, false otherwise + */ + public boolean isClosed() { + return closed; + } + + /** + * Ensures that this manager has not been closed. + * + * @throws IllegalStateException if the manager is closed + */ + private void ensureNotClosed() { + if (closed) { + throw new IllegalStateException("PrimaryTermDirectoryManager has been closed"); + } + } + + /** + * Performs comprehensive validation of a specific primary term directory. + * This includes accessibility, permission checks, and basic operations. + * + * @param primaryTerm the primary term to validate + * @throws IOException if validation fails + */ + public void validatePrimaryTermDirectory(long primaryTerm) throws IOException { + ensureNotClosed(); + + Directory directory = primaryTermDirectories.get(primaryTerm); + if (directory == null) { + throw new PrimaryTermRoutingException( + "No directory found for primary term " + primaryTerm, + PrimaryTermRoutingException.ErrorType.DIRECTORY_VALIDATION_ERROR, + primaryTerm, + (Throwable) null + ); + } + + try { + // Test basic operations + String[] files = directory.listAll(); + logger.debug("Primary term {} directory contains {} files", primaryTerm, files.length); + + // Test sync operation + directory.sync(java.util.Arrays.asList(files)); + + // Additional validation could include: + // - Check directory permissions + // - Verify disk space + // - Test write operations + + } catch (Exception e) { + throw new PrimaryTermRoutingException( + "Validation failed for primary term " + primaryTerm + " directory", + PrimaryTermRoutingException.ErrorType.DIRECTORY_VALIDATION_ERROR, + primaryTerm, + e + ); + } + } + + /** + * Cleans up unused directories for primary terms that are no longer active. + * This method can be called periodically to reclaim disk space. + * + * @param activePrimaryTerms set of primary terms that should be kept + * @return the number of directories cleaned up + * @throws IOException if cleanup fails + */ + public int cleanupUnusedDirectories(Set activePrimaryTerms) throws IOException { + ensureNotClosed(); + + List toRemove = new ArrayList<>(); + + for (Long primaryTerm : primaryTermDirectories.keySet()) { + if (!activePrimaryTerms.contains(primaryTerm)) { + toRemove.add(primaryTerm); + } + } + + int cleanedUp = 0; + for (Long primaryTerm : toRemove) { + try { + Directory directory = primaryTermDirectories.remove(primaryTerm); + if (directory != null) { + directory.close(); + cleanedUp++; + logger.info("Cleaned up directory for inactive primary term {}", primaryTerm); + } + } catch (IOException e) { + logger.warn("Failed to cleanup directory for primary term {}", primaryTerm, e); + // Continue with other directories + } + } + + logger.info("Cleaned up {} unused primary term directories", cleanedUp); + return cleanedUp; + } + + /** + * Gets statistics about the managed directories. + * + * @return DirectoryStats containing information about managed directories + */ + public DirectoryStats getDirectoryStats() { + ensureNotClosed(); + + int totalDirectories = primaryTermDirectories.size(); + long minPrimaryTerm = primaryTermDirectories.keySet().stream().mapToLong(Long::longValue).min().orElse(-1L); + long maxPrimaryTerm = primaryTermDirectories.keySet().stream().mapToLong(Long::longValue).max().orElse(-1L); + + return new DirectoryStats(totalDirectories, minPrimaryTerm, maxPrimaryTerm, basePath.toString()); + } + + /** + * Statistics about managed directories. + */ + public static class DirectoryStats { + private final int totalDirectories; + private final long minPrimaryTerm; + private final long maxPrimaryTerm; + private final String basePath; + + public DirectoryStats(int totalDirectories, long minPrimaryTerm, long maxPrimaryTerm, String basePath) { + this.totalDirectories = totalDirectories; + this.minPrimaryTerm = minPrimaryTerm; + this.maxPrimaryTerm = maxPrimaryTerm; + this.basePath = basePath; + } + + public int getTotalDirectories() { + return totalDirectories; + } + + public long getMinPrimaryTerm() { + return minPrimaryTerm; + } + + public long getMaxPrimaryTerm() { + return maxPrimaryTerm; + } + + public String getBasePath() { + return basePath; + } + + @Override + public String toString() { + return String.format("DirectoryStats{totalDirectories=%d, minPrimaryTerm=%d, maxPrimaryTerm=%d, basePath='%s'}", + totalDirectories, minPrimaryTerm, maxPrimaryTerm, basePath); + } + } + + /** + * Checks if the filesystem has sufficient space for new directories. + * This is a basic check that can be extended with more sophisticated logic. + * + * @param requiredSpaceBytes minimum required space in bytes + * @return true if sufficient space is available + */ + public boolean hasAvailableSpace(long requiredSpaceBytes) { + try { + long usableSpace = Files.getFileStore(basePath).getUsableSpace(); + boolean hasSpace = usableSpace >= requiredSpaceBytes; + + if (!hasSpace) { + logger.warn("Insufficient disk space: required={} bytes, available={} bytes", + requiredSpaceBytes, usableSpace); + } + + return hasSpace; + } catch (IOException e) { + logger.warn("Failed to check available disk space", e); + return true; // Assume space is available if check fails + } + } + + /** + * Forces synchronization of all managed directories. + * This ensures all pending writes are flushed to disk. + * + * @throws IOException if sync fails for any directory + */ + public void syncAllDirectories() throws IOException { + ensureNotClosed(); + + List syncErrors = new ArrayList<>(); + + for (Map.Entry entry : primaryTermDirectories.entrySet()) { + try { + String[] files = entry.getValue().listAll(); + entry.getValue().sync(java.util.Arrays.asList(files)); + logger.debug("Synced directory for primary term {}", entry.getKey()); + } catch (IOException e) { + logger.warn("Failed to sync directory for primary term {}", entry.getKey(), e); + syncErrors.add(e); + } + } + + if (!syncErrors.isEmpty()) { + IOException combinedException = new IOException("Failed to sync " + syncErrors.size() + " directories"); + for (IOException error : syncErrors) { + combinedException.addSuppressed(error); + } + throw combinedException; + } + + logger.info("Successfully synced {} primary term directories", primaryTermDirectories.size()); + } + + /** + * Gets the primary term directories map (for testing purposes). + * + * @return the map of primary terms to directories + */ + protected Map getPrimaryTermDirectories() { + return primaryTermDirectories; + } +} \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/index/store/distributed/PrimaryTermRouter.java b/server/src/main/java/org/opensearch/index/store/distributed/PrimaryTermRouter.java new file mode 100644 index 0000000000000..0d32c3ecab16c --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/distributed/PrimaryTermRouter.java @@ -0,0 +1,157 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.store.Directory; + +import java.io.IOException; +import java.util.Set; + +/** + * Routes segment files to appropriate directories based on primary term. + * This class determines which directory should handle a given file by either + * routing to a primary term-specific directory or keeping certain files + * (like segments_N) in the base directory for compatibility. + * + * @opensearch.internal + */ +public class PrimaryTermRouter { + + private static final Logger logger = LogManager.getLogger(PrimaryTermRouter.class); + + /** Files with these prefixes are excluded from primary term routing */ + private static final Set EXCLUDED_PREFIXES = Set.of( + "segments_", + "pending_segments_", + "write.lock"); + + private final IndexShardContext shardContext; + private volatile long lastLoggedPrimaryTerm = -1L; + + /** + * Creates a new PrimaryTermRouter with the given IndexShardContext. + * + * @param shardContext the context for accessing primary term information + */ + public PrimaryTermRouter(IndexShardContext shardContext) { + this.shardContext = shardContext; + logger.debug("Created PrimaryTermRouter with context: {}", + shardContext != null ? "available" : "null"); + } + + /** + * Determines the appropriate directory for a given filename. + * Files are routed based on primary term unless they are excluded + * (segments_N, lock files, temporary files). + * + * @param filename the name of the file to route + * @param directoryManager the manager for accessing directories + * @return the Directory that should handle this file + * @throws IOException if directory access fails + */ + public Directory getDirectoryForFile(String filename, PrimaryTermDirectoryManager directoryManager) + throws IOException { + if (filename == null || filename.isEmpty()) { + throw new IllegalArgumentException("Filename cannot be null or empty"); + } + + if (isExcludedFile(filename)) { + logger.trace("File {} excluded from primary term routing, using base directory", filename); + return directoryManager.getBaseDirectory(); + } + + long primaryTerm = getCurrentPrimaryTerm(); + + // Log primary term changes for debugging + if (primaryTerm != lastLoggedPrimaryTerm) { + logger.info("Primary term changed from {} to {} for file routing", + lastLoggedPrimaryTerm, primaryTerm); + lastLoggedPrimaryTerm = primaryTerm; + } + + try { + Directory directory = directoryManager.getDirectoryForPrimaryTerm(primaryTerm); + logger.trace("Routed file {} to primary term {} directory", filename, primaryTerm); + return directory; + } catch (IOException e) { + logger.warn("Failed to get directory for primary term {}, falling back to base directory for file {}", + primaryTerm, filename, e); + return directoryManager.getBaseDirectory(); + } + } + + /** + * Gets the current primary term from the IndexShardContext. + * Returns DEFAULT_PRIMARY_TERM if the context is unavailable. + * + * @return the current primary term + */ + public long getCurrentPrimaryTerm() { + if (shardContext != null && shardContext.isAvailable()) { + return shardContext.getPrimaryTerm(); + } + + logger.debug("IndexShardContext unavailable, using default primary term"); + return IndexShardContext.DEFAULT_PRIMARY_TERM; + } + + /** + * Checks if a filename should be excluded from primary term routing. + * Excluded files include: + * - Files starting with "segments_", "pending_segments_", "write.lock" + * - Temporary files ending with ".tmp" + * + * @param filename the filename to check + * @return true if the file should be excluded from routing, false otherwise + */ + public boolean isExcludedFile(String filename) { + if (filename == null || filename.isEmpty()) { + return true; // Treat invalid filenames as excluded + } + + // Exclude temporary files + if (filename.endsWith(".tmp")) { + return true; + } + + // Exclude files with specific prefixes + return EXCLUDED_PREFIXES.stream().anyMatch(filename::startsWith); + } + + /** + * Gets the directory name for a given primary term. + * Uses the naming convention "primary_term_X" where X is the primary term. + * + * @param primaryTerm the primary term + * @return the directory name for this primary term + */ + public String getDirectoryNameForPrimaryTerm(long primaryTerm) { + return PrimaryTermDirectoryManager.PRIMARY_TERM_DIR_PREFIX + primaryTerm; + } + + /** + * Gets the IndexShardContext (for testing purposes). + * + * @return the IndexShardContext instance + */ + protected IndexShardContext getShardContext() { + return shardContext; + } + + /** + * Gets the set of excluded prefixes (for testing purposes). + * + * @return the set of excluded file prefixes + */ + protected Set getExcludedPrefixes() { + return EXCLUDED_PREFIXES; + } +} \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/index/store/distributed/PrimaryTermRoutingException.java b/server/src/main/java/org/opensearch/index/store/distributed/PrimaryTermRoutingException.java new file mode 100644 index 0000000000000..14319c5bbe620 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/distributed/PrimaryTermRoutingException.java @@ -0,0 +1,126 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import java.io.IOException; + +/** + * Exception thrown when primary term-based routing operations fail. + * Provides detailed context about the failure including error type, + * primary term, and filename for debugging purposes. + * + * @opensearch.internal + */ +public class PrimaryTermRoutingException extends IOException { + + private final ErrorType errorType; + private final long primaryTerm; + private final String filename; + + /** + * Types of errors that can occur during primary term routing. + */ + public enum ErrorType { + /** Error accessing primary term from IndexShard */ + PRIMARY_TERM_ACCESS_ERROR, + + /** Error creating or accessing primary term directory */ + DIRECTORY_CREATION_ERROR, + + /** Error routing file to appropriate directory */ + FILE_ROUTING_ERROR, + + /** Error validating directory accessibility */ + DIRECTORY_VALIDATION_ERROR + } + + /** + * Creates a new PrimaryTermRoutingException. + * + * @param message the error message + * @param errorType the type of error that occurred + * @param primaryTerm the primary term associated with the error + * @param filename the filename associated with the error, may be null + * @param cause the underlying cause, may be null + */ + public PrimaryTermRoutingException(String message, ErrorType errorType, long primaryTerm, String filename, Throwable cause) { + super(buildDetailedMessage(message, errorType, primaryTerm, filename), cause); + this.errorType = errorType; + this.primaryTerm = primaryTerm; + this.filename = filename; + } + + /** + * Creates a new PrimaryTermRoutingException without a filename. + * + * @param message the error message + * @param errorType the type of error that occurred + * @param primaryTerm the primary term associated with the error + * @param cause the underlying cause, may be null + */ + public PrimaryTermRoutingException(String message, ErrorType errorType, long primaryTerm, Throwable cause) { + this(message, errorType, primaryTerm, null, cause); + } + + /** + * Creates a new PrimaryTermRoutingException without a cause. + * + * @param message the error message + * @param errorType the type of error that occurred + * @param primaryTerm the primary term associated with the error + * @param filename the filename associated with the error, may be null + */ + public PrimaryTermRoutingException(String message, ErrorType errorType, long primaryTerm, String filename) { + this(message, errorType, primaryTerm, filename, null); + } + + /** + * Gets the error type. + * + * @return the ErrorType + */ + public ErrorType getErrorType() { + return errorType; + } + + /** + * Gets the primary term associated with the error. + * + * @return the primary term + */ + public long getPrimaryTerm() { + return primaryTerm; + } + + /** + * Gets the filename associated with the error. + * + * @return the filename, may be null + */ + public String getFilename() { + return filename; + } + + /** + * Builds a detailed error message including context information. + */ + private static String buildDetailedMessage(String message, ErrorType errorType, long primaryTerm, String filename) { + StringBuilder sb = new StringBuilder(); + sb.append("[").append(errorType).append("] "); + sb.append(message); + sb.append(" (primaryTerm=").append(primaryTerm); + + if (filename != null) { + sb.append(", filename=").append(filename); + } + + sb.append(")"); + return sb.toString(); + } +} \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/DefaultFilenameHasherTests.java b/server/src/test/java/org/opensearch/index/store/distributed/DefaultFilenameHasherTests.java deleted file mode 100644 index b4fcfb5d996a8..0000000000000 --- a/server/src/test/java/org/opensearch/index/store/distributed/DefaultFilenameHasherTests.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.store.distributed; - -import org.opensearch.test.OpenSearchTestCase; - -import java.util.HashMap; -import java.util.Map; - -public class DefaultFilenameHasherTests extends OpenSearchTestCase { - - private DefaultFilenameHasher hasher; - - @Override - public void setUp() throws Exception { - super.setUp(); - hasher = new DefaultFilenameHasher(); - } - - public void testConsistentHashing() { - String filename = "_0.cfe"; - int firstResult = hasher.getDirectoryIndex(filename); - - // Call multiple times to ensure consistency - for (int i = 0; i < 10; i++) { - assertEquals("Hash should be consistent across multiple calls", firstResult, hasher.getDirectoryIndex(filename)); - } - } - - public void testExcludedFiles() { - // Test segments_N files are excluded - assertTrue("segments_1 should be excluded", hasher.isExcludedFile("segments_1")); - assertTrue("segments_10 should be excluded", hasher.isExcludedFile("segments_10")); - assertEquals("segments_1 should map to directory 0", 0, hasher.getDirectoryIndex("segments_1")); - assertEquals("segments_10 should map to directory 0", 0, hasher.getDirectoryIndex("segments_10")); - - // Test regular files are not excluded - assertFalse("_0.cfe should not be excluded", hasher.isExcludedFile("_0.cfe")); - assertFalse("_1.si should not be excluded", hasher.isExcludedFile("_1.si")); - } - - public void testDirectoryIndexRange() { - String[] testFiles = { - "_0.cfe", "_0.cfs", "_0.si", "_0.fnm", "_0.fdt", - "_1.tim", "_1.tip", "_1.doc", "_1.pos", "_1.pay", - "_2.dvd", "_2.dvm", "_3.fdx", "_4.nvd", "_5.nvm" - }; - - for (String filename : testFiles) { - int index = hasher.getDirectoryIndex(filename); - assertTrue("Directory index should be between 0 and 4, got " + index + " for " + filename, - index >= 0 && index < 5); - } - } - - public void testDistribution() { - String[] testFiles = { - "_0.cfe", "_0.cfs", "_0.si", "_0.fnm", "_0.fdt", - "_1.tim", "_1.tip", "_1.doc", "_1.pos", "_1.pay", - "_2.dvd", "_2.dvm", "_3.fdx", "_4.nvd", "_5.nvm", - "_6.cfe", "_7.cfs", "_8.si", "_9.fnm", "_10.fdt" - }; - - Map distribution = new HashMap<>(); - for (String filename : testFiles) { - int index = hasher.getDirectoryIndex(filename); - distribution.put(index, distribution.getOrDefault(index, 0) + 1); - } - - // Should use multiple directories (not all in one) - assertTrue("Files should be distributed across multiple directories, got: " + distribution, - distribution.size() > 1); - } - - public void testNullAndEmptyFilenames() { - // Test null filename - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, - () -> hasher.getDirectoryIndex(null)); - assertEquals("Filename cannot be null or empty", exception.getMessage()); - - // Test empty filename - exception = expectThrows(IllegalArgumentException.class, - () -> hasher.getDirectoryIndex("")); - assertEquals("Filename cannot be null or empty", exception.getMessage()); - - // Test isExcludedFile with null/empty - assertTrue("null filename should be excluded", hasher.isExcludedFile(null)); - assertTrue("empty filename should be excluded", hasher.isExcludedFile("")); - } - - public void testSpecificFileDistribution() { - // Test some specific files to ensure they don't all go to the same directory - String[] files = {"_0.cfe", "_0.cfs", "_0.si", "_0.fnm", "_0.fdt"}; - int[] indices = new int[files.length]; - - for (int i = 0; i < files.length; i++) { - indices[i] = hasher.getDirectoryIndex(files[i]); - } - - // Check that not all files go to the same directory - boolean hasVariation = false; - for (int i = 1; i < indices.length; i++) { - if (indices[i] != indices[0]) { - hasVariation = true; - break; - } - } - - assertTrue("Files should be distributed across different directories", hasVariation); - } -} \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/DirectoryManagerTests.java b/server/src/test/java/org/opensearch/index/store/distributed/DirectoryManagerTests.java deleted file mode 100644 index 2a78a70d06181..0000000000000 --- a/server/src/test/java/org/opensearch/index/store/distributed/DirectoryManagerTests.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.store.distributed; - -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.opensearch.test.OpenSearchTestCase; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; - -public class DirectoryManagerTests extends OpenSearchTestCase { - - private Path tempDir; - private Directory baseDirectory; - private DirectoryManager directoryManager; - - @Override - public void setUp() throws Exception { - super.setUp(); - tempDir = createTempDir(); - baseDirectory = FSDirectory.open(tempDir); - } - - @Override - public void tearDown() throws Exception { - if (directoryManager != null) { - directoryManager.close(); - } - if (baseDirectory != null) { - baseDirectory.close(); - } - super.tearDown(); - } - - public void testDirectoryCreation() throws IOException { - directoryManager = new DirectoryManager(baseDirectory, tempDir); - - // Verify all 5 directories are created - assertEquals("Should have 5 directories", 5, directoryManager.getNumDirectories()); - - // Verify base directory is at index 0 - assertSame("Base directory should be at index 0", baseDirectory, directoryManager.getDirectory(0)); - - // Verify subdirectories are created - for (int i = 1; i < 5; i++) { - Directory dir = directoryManager.getDirectory(i); - assertNotNull("Directory " + i + " should not be null", dir); - assertNotSame("Directory " + i + " should not be the base directory", baseDirectory, dir); - } - - // Verify filesystem subdirectories exist - for (int i = 1; i < 5; i++) { - Path subPath = tempDir.resolve("segments_" + i); - assertTrue("Subdirectory should exist: " + subPath, Files.exists(subPath)); - assertTrue("Subdirectory should be a directory: " + subPath, Files.isDirectory(subPath)); - } - } - - public void testGetDirectoryValidation() throws IOException { - directoryManager = new DirectoryManager(baseDirectory, tempDir); - - // Test valid indices - for (int i = 0; i < 5; i++) { - assertNotNull("Directory " + i + " should not be null", directoryManager.getDirectory(i)); - } - - // Test invalid indices - IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, - () -> directoryManager.getDirectory(-1)); - assertTrue("Exception should mention valid range", exception.getMessage().contains("between 0 and 4")); - - exception = expectThrows(IllegalArgumentException.class, - () -> directoryManager.getDirectory(5)); - assertTrue("Exception should mention valid range", exception.getMessage().contains("between 0 and 4")); - } - - public void testBasePath() throws IOException { - directoryManager = new DirectoryManager(baseDirectory, tempDir); - assertEquals("Base path should match", tempDir, directoryManager.getBasePath()); - } - - public void testClose() throws IOException { - directoryManager = new DirectoryManager(baseDirectory, tempDir); - - // Get references to subdirectories before closing - Directory[] dirs = new Directory[5]; - for (int i = 0; i < 5; i++) { - dirs[i] = directoryManager.getDirectory(i); - } - - // Close the manager - directoryManager.close(); - - // Base directory (index 0) should still be open since it's managed externally - // We can't easily test if subdirectories are closed without accessing internal state - // But we can verify the close operation completed without exception - - // Verify we can still access the base directory - assertNotNull("Base directory should still be accessible", dirs[0]); - } - - public void testSubdirectoryCreationFailure() throws IOException { - // Create a file where we want to create a subdirectory to force failure - Path conflictPath = tempDir.resolve("segments_1"); - Files.createFile(conflictPath); // Create a file, not a directory - - try { - DistributedDirectoryException exception = expectThrows(DistributedDirectoryException.class, - () -> new DirectoryManager(baseDirectory, tempDir)); - assertTrue("Exception should mention subdirectory creation failure", - exception.getMessage().contains("Failed to create subdirectories")); - } finally { - // Clean up the conflicting file - Files.deleteIfExists(conflictPath); - } - } - - public void testExistingSubdirectories() throws IOException { - // Pre-create some subdirectories - Path subDir1 = tempDir.resolve("segments_1"); - Path subDir2 = tempDir.resolve("segments_2"); - Files.createDirectories(subDir1); - Files.createDirectories(subDir2); - - // Should work with existing directories - directoryManager = new DirectoryManager(baseDirectory, tempDir); - - assertNotNull("Should handle existing subdirectory 1", directoryManager.getDirectory(1)); - assertNotNull("Should handle existing subdirectory 2", directoryManager.getDirectory(2)); - } -} \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryExceptionTests.java b/server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryExceptionTests.java deleted file mode 100644 index 48adf688cce2b..0000000000000 --- a/server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryExceptionTests.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.store.distributed; - -import org.opensearch.test.OpenSearchTestCase; - -import java.io.IOException; - -public class DistributedDirectoryExceptionTests extends OpenSearchTestCase { - - public void testExceptionWithAllParameters() { - String message = "Test error message"; - int directoryIndex = 2; - String operation = "testOperation"; - IOException cause = new IOException("Root cause"); - - DistributedDirectoryException exception = new DistributedDirectoryException( - message, directoryIndex, operation, cause - ); - - assertEquals("Directory index should match", directoryIndex, exception.getDirectoryIndex()); - assertEquals("Operation should match", operation, exception.getOperation()); - assertSame("Cause should match", cause, exception.getCause()); - - String expectedMessage = "Directory 2 operation 'testOperation' failed: Test error message"; - assertEquals("Message should be formatted correctly", expectedMessage, exception.getMessage()); - } - - public void testExceptionWithoutCause() { - String message = "Test error message"; - int directoryIndex = 1; - String operation = "testOperation"; - - DistributedDirectoryException exception = new DistributedDirectoryException( - message, directoryIndex, operation - ); - - assertEquals("Directory index should match", directoryIndex, exception.getDirectoryIndex()); - assertEquals("Operation should match", operation, exception.getOperation()); - assertNull("Cause should be null", exception.getCause()); - - String expectedMessage = "Directory 1 operation 'testOperation' failed: Test error message"; - assertEquals("Message should be formatted correctly", expectedMessage, exception.getMessage()); - } - - public void testExceptionMessageFormatting() { - DistributedDirectoryException exception = new DistributedDirectoryException( - "File not found", 0, "openInput" - ); - - String message = exception.getMessage(); - assertTrue("Message should contain directory index", message.contains("Directory 0")); - assertTrue("Message should contain operation", message.contains("openInput")); - assertTrue("Message should contain original message", message.contains("File not found")); - } - - public void testExceptionInheritance() { - DistributedDirectoryException exception = new DistributedDirectoryException( - "Test", 0, "test" - ); - - assertTrue("Should be instance of IOException", exception instanceof IOException); - } - - public void testExceptionWithLongMessage() { - String longMessage = "This is a very long error message that contains lots of details about what went wrong during the operation"; - DistributedDirectoryException exception = new DistributedDirectoryException( - longMessage, 4, "longOperation" - ); - - assertTrue("Message should contain the long message", exception.getMessage().contains(longMessage)); - assertEquals("Directory index should be preserved", 4, exception.getDirectoryIndex()); - assertEquals("Operation should be preserved", "longOperation", exception.getOperation()); - } - - public void testExceptionWithSpecialCharacters() { - String messageWithSpecialChars = "Error with special chars: !@#$%^&*()"; - String operationWithSpecialChars = "operation:with:colons"; - - DistributedDirectoryException exception = new DistributedDirectoryException( - messageWithSpecialChars, 3, operationWithSpecialChars - ); - - assertTrue("Message should handle special characters", - exception.getMessage().contains(messageWithSpecialChars)); - assertEquals("Operation with special chars should be preserved", - operationWithSpecialChars, exception.getOperation()); - } -} \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryFactoryTests.java b/server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryFactoryTests.java deleted file mode 100644 index 4cfb5a277046d..0000000000000 --- a/server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryFactoryTests.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.store.distributed; - -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.opensearch.common.settings.Settings; -import org.opensearch.core.index.shard.ShardId; -import org.opensearch.test.OpenSearchTestCase; - -import java.io.IOException; -import java.nio.file.Path; - -public class DistributedDirectoryFactoryTests extends OpenSearchTestCase { - - private Path tempDir; - private Directory baseDirectory; - - @Override - public void setUp() throws Exception { - super.setUp(); - tempDir = createTempDir(); - baseDirectory = FSDirectory.open(tempDir); - } - - @Override - public void tearDown() throws Exception { - if (baseDirectory != null) { - baseDirectory.close(); - } - super.tearDown(); - } - - public void testFactoryWithDefaultSettings() throws IOException { - Settings settings = Settings.EMPTY; - DistributedDirectoryFactory factory = new DistributedDirectoryFactory(settings); - - assertFalse("Distributed should be disabled by default", factory.isDistributedEnabled()); - assertEquals("Should use default subdirectories", 5, factory.getNumSubdirectories()); - assertEquals("Should use default hash algorithm", "default", factory.getHashAlgorithm()); - } - - public void testFactoryWithDistributedDisabled() throws IOException { - Settings settings = Settings.builder() - .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, false) - .build(); - - DistributedDirectoryFactory factory = new DistributedDirectoryFactory(settings); - ShardId shardId = new ShardId("test", "test-uuid", 0); - - Directory result = factory.createDirectory(baseDirectory, tempDir, shardId); - - assertSame("Should return delegate directory when disabled", baseDirectory, result); - assertFalse("Should report as disabled", factory.isDistributedEnabled()); - } - - public void testFactoryWithDistributedEnabled() throws IOException { - Settings settings = Settings.builder() - .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, true) - .build(); - - DistributedDirectoryFactory factory = new DistributedDirectoryFactory(settings); - ShardId shardId = new ShardId("test", "test-uuid", 0); - - Directory result = factory.createDirectory(baseDirectory, tempDir, shardId); - - assertNotSame("Should return distributed directory when enabled", baseDirectory, result); - assertTrue("Should be instance of DistributedSegmentDirectory", - result instanceof DistributedSegmentDirectory); - assertTrue("Should report as enabled", factory.isDistributedEnabled()); - - result.close(); - } - - public void testFactoryWithCustomSettings() throws IOException { - Settings settings = Settings.builder() - .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, true) - .put(DistributedDirectoryFactory.DISTRIBUTED_SUBDIRECTORIES_SETTING, 3) - .put(DistributedDirectoryFactory.DISTRIBUTED_HASH_ALGORITHM_SETTING, "custom") - .build(); - - DistributedDirectoryFactory factory = new DistributedDirectoryFactory(settings); - - assertTrue("Should be enabled", factory.isDistributedEnabled()); - assertEquals("Should use custom subdirectories", 3, factory.getNumSubdirectories()); - assertEquals("Should use custom hash algorithm", "custom", factory.getHashAlgorithm()); - } - - public void testFactoryWithoutShardId() throws IOException { - Settings settings = Settings.builder() - .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, true) - .build(); - - DistributedDirectoryFactory factory = new DistributedDirectoryFactory(settings); - - Directory result = factory.createDirectory(baseDirectory, tempDir); - - assertNotSame("Should return distributed directory", baseDirectory, result); - assertTrue("Should be instance of DistributedSegmentDirectory", - result instanceof DistributedSegmentDirectory); - - result.close(); - } - - public void testFactoryWithInvalidHashAlgorithm() throws IOException { - Settings settings = Settings.builder() - .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, true) - .put(DistributedDirectoryFactory.DISTRIBUTED_HASH_ALGORITHM_SETTING, "invalid") - .build(); - - DistributedDirectoryFactory factory = new DistributedDirectoryFactory(settings); - ShardId shardId = new ShardId("test", "test-uuid", 0); - - // Should still create directory with default hasher - Directory result = factory.createDirectory(baseDirectory, tempDir, shardId); - - assertNotSame("Should return distributed directory", baseDirectory, result); - assertTrue("Should be instance of DistributedSegmentDirectory", - result instanceof DistributedSegmentDirectory); - - result.close(); - } - - public void testFactoryWithSettings() throws IOException { - Settings originalSettings = Settings.builder() - .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, false) - .build(); - - Settings newSettings = Settings.builder() - .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, true) - .build(); - - DistributedDirectoryFactory originalFactory = new DistributedDirectoryFactory(originalSettings); - DistributedDirectoryFactory newFactory = originalFactory.withSettings(newSettings); - - assertFalse("Original factory should be disabled", originalFactory.isDistributedEnabled()); - assertTrue("New factory should be enabled", newFactory.isDistributedEnabled()); - } - - public void testFactoryFallbackOnError() throws IOException { - Settings settings = Settings.builder() - .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, true) - .build(); - - DistributedDirectoryFactory factory = new DistributedDirectoryFactory(settings); - ShardId shardId = new ShardId("test", "test-uuid", 0); - - // Use a path that will cause directory creation to fail - Path invalidPath = tempDir.resolve("nonexistent/invalid/path"); - - Directory result = factory.createDirectory(baseDirectory, invalidPath, shardId); - - // Should fall back to delegate directory on error - assertSame("Should fall back to delegate on error", baseDirectory, result); - } - - public void testFactorySettingsValidation() throws IOException { - // Test with various settings combinations - Settings settings1 = Settings.builder() - .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, "true") - .build(); - - Settings settings2 = Settings.builder() - .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, "false") - .build(); - - DistributedDirectoryFactory factory1 = new DistributedDirectoryFactory(settings1); - DistributedDirectoryFactory factory2 = new DistributedDirectoryFactory(settings2); - - assertTrue("String 'true' should be parsed as enabled", factory1.isDistributedEnabled()); - assertFalse("String 'false' should be parsed as disabled", factory2.isDistributedEnabled()); - } -} - public void testSettingsIntegration() throws IOException { - // Test all configuration options - Settings settings = Settings.builder() - .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, true) - .put(DistributedDirectoryFactory.DISTRIBUTED_SUBDIRECTORIES_SETTING, 3) - .put(DistributedDirectoryFactory.DISTRIBUTED_HASH_ALGORITHM_SETTING, "default") - .build(); - - DistributedDirectoryFactory factory = new DistributedDirectoryFactory(settings); - - assertTrue("Should be enabled", factory.isDistributedEnabled()); - assertEquals("Should use configured subdirectories", 3, factory.getNumSubdirectories()); - assertEquals("Should use configured hash algorithm", "default", factory.getHashAlgorithm()); - } - - public void testSettingsValidation() throws IOException { - // Test with invalid subdirectory count (should still work, just use the value) - Settings settings = Settings.builder() - .put(DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING, true) - .put(DistributedDirectoryFactory.DISTRIBUTED_SUBDIRECTORIES_SETTING, 10) - .build(); - - DistributedDirectoryFactory factory = new DistributedDirectoryFactory(settings); - assertEquals("Should accept configured value", 10, factory.getNumSubdirectories()); - } - - public void testDefaultSettingsValues() throws IOException { - DistributedDirectoryFactory factory = new DistributedDirectoryFactory(Settings.EMPTY); - - assertEquals("Default enabled should be false", - DistributedDirectoryFactory.DEFAULT_DISTRIBUTED_ENABLED, factory.isDistributedEnabled()); - assertEquals("Default subdirectories should be 5", - DistributedDirectoryFactory.DEFAULT_SUBDIRECTORIES, factory.getNumSubdirectories()); - assertEquals("Default hash algorithm should be 'default'", - DistributedDirectoryFactory.DEFAULT_HASH_ALGORITHM, factory.getHashAlgorithm()); - } - - public void testSettingsConstants() { - // Verify setting key constants are correct - assertEquals("Enabled setting key", "index.store.distributed.enabled", - DistributedDirectoryFactory.DISTRIBUTED_ENABLED_SETTING); - assertEquals("Subdirectories setting key", "index.store.distributed.subdirectories", - DistributedDirectoryFactory.DISTRIBUTED_SUBDIRECTORIES_SETTING); - assertEquals("Hash algorithm setting key", "index.store.distributed.hash_algorithm", - DistributedDirectoryFactory.DISTRIBUTED_HASH_ALGORITHM_SETTING); - } \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryMetricsTests.java b/server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryMetricsTests.java deleted file mode 100644 index 094408f0c6565..0000000000000 --- a/server/src/test/java/org/opensearch/index/store/distributed/DistributedDirectoryMetricsTests.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.store.distributed; - -import org.opensearch.test.OpenSearchTestCase; - -public class DistributedDirectoryMetricsTests extends OpenSearchTestCase { - - private DistributedDirectoryMetrics metrics; - - @Override - public void setUp() throws Exception { - super.setUp(); - metrics = new DistributedDirectoryMetrics(); - } - - public void testInitialState() { - assertEquals("Total operations should be zero initially", 0, metrics.getTotalOperations()); - assertEquals("Total errors should be zero initially", 0, metrics.getTotalErrors()); - - for (int i = 0; i < 5; i++) { - assertEquals("Operation count should be zero for directory " + i, 0, metrics.getOperationCount(i)); - assertEquals("Error count should be zero for directory " + i, 0, metrics.getErrorCount(i)); - assertEquals("Average time should be zero for directory " + i, 0, metrics.getAverageOperationTime(i)); - } - } - - public void testRecordFileOperation() { - int directoryIndex = 2; - String operation = "openInput"; - long duration = 1_000_000; // 1ms in nanoseconds - - metrics.recordFileOperation(directoryIndex, operation, duration); - - assertEquals("Operation count should be 1", 1, metrics.getOperationCount(directoryIndex)); - assertEquals("Total operations should be 1", 1, metrics.getTotalOperations()); - assertEquals("Average time should match duration", duration, metrics.getAverageOperationTime(directoryIndex)); - assertEquals("Max time should match duration", duration, metrics.getMaxOperationTime(directoryIndex)); - assertEquals("Min time should match duration", duration, metrics.getMinOperationTime(directoryIndex)); - } - - public void testRecordMultipleOperations() { - long[] durations = {1_000_000, 2_000_000, 3_000_000}; // 1ms, 2ms, 3ms - int directoryIndex = 1; - - for (long duration : durations) { - metrics.recordFileOperation(directoryIndex, "test", duration); - } - - assertEquals("Operation count should be 3", 3, metrics.getOperationCount(directoryIndex)); - assertEquals("Total operations should be 3", 3, metrics.getTotalOperations()); - - long expectedAverage = (1_000_000 + 2_000_000 + 3_000_000) / 3; - assertEquals("Average should be calculated correctly", expectedAverage, metrics.getAverageOperationTime(directoryIndex)); - assertEquals("Max should be 3ms", 3_000_000, metrics.getMaxOperationTime(directoryIndex)); - assertEquals("Min should be 1ms", 1_000_000, metrics.getMinOperationTime(directoryIndex)); - } - - public void testRecordError() { - int directoryIndex = 3; - String operation = "deleteFile"; - - metrics.recordError(directoryIndex, operation); - - assertEquals("Error count should be 1", 1, metrics.getErrorCount(directoryIndex)); - assertEquals("Total errors should be 1", 1, metrics.getTotalErrors()); - } - - public void testDistributionAcrossDirectories() { - // Record operations in different directories - metrics.recordFileOperation(0, "test", 1_000_000); - metrics.recordFileOperation(1, "test", 1_000_000); - metrics.recordFileOperation(1, "test", 1_000_000); - metrics.recordFileOperation(2, "test", 1_000_000); - metrics.recordFileOperation(2, "test", 1_000_000); - metrics.recordFileOperation(2, "test", 1_000_000); - - assertEquals("Directory 0 should have 1 operation", 1, metrics.getOperationCount(0)); - assertEquals("Directory 1 should have 2 operations", 2, metrics.getOperationCount(1)); - assertEquals("Directory 2 should have 3 operations", 3, metrics.getOperationCount(2)); - assertEquals("Total should be 6", 6, metrics.getTotalOperations()); - - double[] distribution = metrics.getDistributionPercentages(); - assertEquals("Directory 0 should have ~16.7%", 16.7, distribution[0], 0.1); - assertEquals("Directory 1 should have ~33.3%", 33.3, distribution[1], 0.1); - assertEquals("Directory 2 should have 50%", 50.0, distribution[2], 0.1); - assertEquals("Directory 3 should have 0%", 0.0, distribution[3], 0.1); - assertEquals("Directory 4 should have 0%", 0.0, distribution[4], 0.1); - } - - public void testInvalidDirectoryIndex() { - // Test negative index - metrics.recordFileOperation(-1, "test", 1_000_000); - assertEquals("Should not record operation for negative index", 0, metrics.getTotalOperations()); - - // Test index too large - metrics.recordFileOperation(5, "test", 1_000_000); - assertEquals("Should not record operation for index >= 5", 0, metrics.getTotalOperations()); - - // Test error recording with invalid index - metrics.recordError(-1, "test"); - metrics.recordError(10, "test"); - assertEquals("Should not record errors for invalid indices", 0, metrics.getTotalErrors()); - } - - public void testOperationTypes() { - int directoryIndex = 0; - - metrics.recordFileOperation(directoryIndex, "openInput", 1_000_000); - metrics.recordFileOperation(directoryIndex, "createOutput", 2_000_000); - metrics.recordFileOperation(directoryIndex, "deleteFile", 3_000_000); - metrics.recordFileOperation(directoryIndex, "fileLength", 4_000_000); - - assertEquals("Should record all operation types", 4, metrics.getOperationCount(directoryIndex)); - - // Average should be (1+2+3+4)/4 = 2.5ms - long expectedAverage = (1_000_000 + 2_000_000 + 3_000_000 + 4_000_000) / 4; - assertEquals("Average should be calculated correctly", expectedAverage, metrics.getAverageOperationTime(directoryIndex)); - } - - public void testReset() { - // Record some operations and errors - metrics.recordFileOperation(0, "test", 1_000_000); - metrics.recordFileOperation(1, "test", 2_000_000); - metrics.recordError(0, "test"); - - assertEquals("Should have operations before reset", 2, metrics.getTotalOperations()); - assertEquals("Should have errors before reset", 1, metrics.getTotalErrors()); - - // Reset metrics - metrics.reset(); - - assertEquals("Total operations should be zero after reset", 0, metrics.getTotalOperations()); - assertEquals("Total errors should be zero after reset", 0, metrics.getTotalErrors()); - - for (int i = 0; i < 5; i++) { - assertEquals("Operation count should be zero after reset for directory " + i, - 0, metrics.getOperationCount(i)); - assertEquals("Error count should be zero after reset for directory " + i, - 0, metrics.getErrorCount(i)); - } - } - - public void testUptime() throws InterruptedException { - long startTime = System.currentTimeMillis(); - - // Wait a small amount - Thread.sleep(10); - - long uptime = metrics.getUptimeMillis(); - long actualElapsed = System.currentTimeMillis() - startTime; - - assertTrue("Uptime should be positive", uptime > 0); - assertTrue("Uptime should be reasonable", uptime <= actualElapsed + 10); // Allow some tolerance - } - - public void testSummary() { - metrics.recordFileOperation(0, "openInput", 1_000_000); - metrics.recordFileOperation(1, "createOutput", 2_000_000); - metrics.recordError(0, "test"); - - String summary = metrics.getSummary(); - - assertNotNull("Summary should not be null", summary); - assertTrue("Summary should contain total operations", summary.contains("Total Operations: 2")); - assertTrue("Summary should contain total errors", summary.contains("Total Errors: 1")); - assertTrue("Summary should contain uptime", summary.contains("Uptime:")); - assertTrue("Summary should contain distribution", summary.contains("Distribution by Directory:")); - } - - public void testDistributionPercentagesWithNoOperations() { - double[] distribution = metrics.getDistributionPercentages(); - - assertEquals("Should have 5 percentages", 5, distribution.length); - for (int i = 0; i < 5; i++) { - assertEquals("All percentages should be 0 when no operations", 0.0, distribution[i], 0.001); - } - } - - public void testMinMaxTimesWithNoOperations() { - for (int i = 0; i < 5; i++) { - assertEquals("Max time should be 0 with no operations", 0, metrics.getMaxOperationTime(i)); - assertEquals("Min time should be 0 with no operations", 0, metrics.getMinOperationTime(i)); - } - } -} \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryBenchmarkTests.java b/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryBenchmarkTests.java deleted file mode 100644 index 7c11194d590f4..0000000000000 --- a/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryBenchmarkTests.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.store.distributed; - -import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.opensearch.test.OpenSearchTestCase; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -public class DistributedSegmentDirectoryBenchmarkTests extends OpenSearchTestCase { - - private static final int BENCHMARK_DOCS = 1000; - private static final int BENCHMARK_ITERATIONS = 3; - - public void testFileDistributionValidation() throws IOException { - Path tempDir = createTempDir(); - Directory baseDir = FSDirectory.open(tempDir); - DistributedSegmentDirectory distributedDir = new DistributedSegmentDirectory(baseDir, tempDir); - - // Create an index to generate various file types - createTestIndex(distributedDir, 100); - - String[] files = distributedDir.listAll(); - assertTrue("Should have created multiple files", files.length > 5); - - // Analyze file distribution - Map distributionCount = new HashMap<>(); - int segmentsFileCount = 0; - - for (String file : files) { - int dirIndex = distributedDir.getDirectoryIndex(file); - distributionCount.put(dirIndex, distributionCount.getOrDefault(dirIndex, 0) + 1); - - if (file.startsWith("segments_")) { - segmentsFileCount++; - assertEquals("segments_N files should be in directory 0", 0, dirIndex); - } - } - - assertTrue("Should have segments files", segmentsFileCount > 0); - assertTrue("Should use multiple directories", distributionCount.size() > 1); - - // Check distribution is reasonably balanced (no directory should have > 70% of files) - int totalFiles = files.length; - for (Map.Entry entry : distributionCount.entrySet()) { - double percentage = (double) entry.getValue() / totalFiles; - assertTrue("Directory " + entry.getKey() + " has " + (percentage * 100) + "% of files, should be < 70%", - percentage < 0.7); - } - - logger.info("File distribution across directories: {}", distributionCount); - distributedDir.close(); - } - - public void testHashingDistribution() throws IOException { - DefaultFilenameHasher hasher = new DefaultFilenameHasher(); - - // Test with common Lucene file extensions - String[] testFiles = { - "_0.cfe", "_0.cfs", "_0.si", "_0.fnm", "_0.fdt", "_0.fdx", - "_0.tim", "_0.tip", "_0.doc", "_0.pos", "_0.pay", "_0.nvd", "_0.nvm", - "_1.cfe", "_1.cfs", "_1.si", "_1.fnm", "_1.fdt", "_1.fdx", - "_1.tim", "_1.tip", "_1.doc", "_1.pos", "_1.pay", "_1.nvd", "_1.nvm", - "_2.cfe", "_2.cfs", "_2.si", "_2.fnm", "_2.fdt", "_2.fdx" - }; - - Map distribution = new HashMap<>(); - - for (String file : testFiles) { - int dirIndex = hasher.getDirectoryIndex(file); - distribution.put(dirIndex, distribution.getOrDefault(dirIndex, 0) + 1); - } - - logger.info("Hash distribution for {} files: {}", testFiles.length, distribution); - - // Should use multiple directories - assertTrue("Should distribute across multiple directories", distribution.size() > 1); - - // No single directory should have more than 60% of files - for (Map.Entry entry : distribution.entrySet()) { - double percentage = (double) entry.getValue() / testFiles.length; - assertTrue("Directory " + entry.getKey() + " should not have > 60% of files, has " + (percentage * 100) + "%", - percentage <= 0.6); - } - } - - public void testConcurrentAccessValidation() throws IOException, InterruptedException { - Path tempDir = createTempDir(); - Directory baseDir = FSDirectory.open(tempDir); - DistributedSegmentDirectory distributedDir = new DistributedSegmentDirectory(baseDir, tempDir); - - // Create initial index - createTestIndex(distributedDir, 100); - - // Test concurrent read operations - int numThreads = 4; - int operationsPerThread = 25; - - Thread[] threads = new Thread[numThreads]; - final Exception[] exceptions = new Exception[numThreads]; - - for (int i = 0; i < numThreads; i++) { - final int threadId = i; - threads[i] = new Thread(() -> { - try { - for (int j = 0; j < operationsPerThread; j++) { - // Test various operations concurrently - String[] files = distributedDir.listAll(); - assertTrue("Should have files", files.length > 0); - - // Test file length operations - for (String file : files) { - if (!file.startsWith("segments_")) { - long length = distributedDir.fileLength(file); - assertTrue("File length should be positive", length > 0); - } - } - - // Test search operations - try (IndexReader reader = DirectoryReader.open(distributedDir)) { - IndexSearcher searcher = new IndexSearcher(reader); - TermQuery query = new TermQuery(new Term("id", String.valueOf(j % 100))); - TopDocs results = searcher.search(query, 10); - assertTrue("Should find results", results.totalHits.value > 0); - } - } - } catch (Exception e) { - exceptions[threadId] = e; - } - }); - threads[i].start(); - } - - // Wait for all threads to complete - for (Thread thread : threads) { - thread.join(30000); // 30 second timeout - } - - // Check for exceptions - for (int i = 0; i < numThreads; i++) { - if (exceptions[i] != null) { - fail("Thread " + i + " failed with exception: " + exceptions[i].getMessage()); - } - } - - distributedDir.close(); - } - - private void createTestIndex(Directory directory, int numDocs) throws IOException { - IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); - try (IndexWriter writer = new IndexWriter(directory, config)) { - for (int i = 0; i < numDocs; i++) { - Document doc = new Document(); - doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); - doc.add(new TextField("content", "This is document " + i + " with some content", Field.Store.YES)); - writer.addDocument(doc); - } - writer.commit(); - } - } -} \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryEdgeCaseTests.java b/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryEdgeCaseTests.java deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryIntegrationTests.java b/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryIntegrationTests.java index f1bf8a9e832dd..d96253250af1c 100644 --- a/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryIntegrationTests.java +++ b/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryIntegrationTests.java @@ -8,46 +8,47 @@ package org.opensearch.index.store.distributed; -import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.opensearch.index.shard.IndexShard; import org.opensearch.test.OpenSearchTestCase; +import org.junit.After; +import org.junit.Before; import java.io.IOException; import java.nio.file.Path; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class DistributedSegmentDirectoryIntegrationTests extends OpenSearchTestCase { private Path tempDir; private Directory baseDirectory; + private IndexShard mockIndexShard; private DistributedSegmentDirectory distributedDirectory; - @Override + @Before public void setUp() throws Exception { super.setUp(); tempDir = createTempDir(); baseDirectory = FSDirectory.open(tempDir); - distributedDirectory = new DistributedSegmentDirectory(baseDirectory, tempDir); + mockIndexShard = mock(IndexShard.class); + + // Mock IndexShard to return a primary term + when(mockIndexShard.getOperationPrimaryTerm()).thenReturn(1L); + + distributedDirectory = new DistributedSegmentDirectory(baseDirectory, tempDir, mockIndexShard); } - @Override + @After public void tearDown() throws Exception { if (distributedDirectory != null) { distributedDirectory.close(); @@ -55,287 +56,267 @@ public void tearDown() throws Exception { super.tearDown(); } - public void testIndexWriterWithDistributedDirectory() throws IOException { - IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); - - try (IndexWriter writer = new IndexWriter(distributedDirectory, config)) { - // Add some documents - for (int i = 0; i < 10; i++) { - Document doc = new Document(); - doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); - doc.add(new TextField("content", "This is document " + i, Field.Store.YES)); - writer.addDocument(doc); + public void testFileCreationAndReading() throws IOException { + String filename = "_0.si"; + String content = "test segment info content"; + + // Create a file + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString(content); + } + + // Verify file exists and can be read + assertTrue(Arrays.asList(distributedDirectory.listAll()).contains(filename)); + + try (IndexInput input = distributedDirectory.openInput(filename, IOContext.DEFAULT)) { + assertEquals(content, input.readString()); + } + + // Verify file length + assertEquals(content.getBytes().length + 4, distributedDirectory.fileLength(filename)); // +4 for string length prefix + } + + public void testSegmentsFileInBaseDirectory() throws IOException { + String segmentsFile = "segments_1"; + String content = "segments file content"; + + // Create segments file + try (IndexOutput output = distributedDirectory.createOutput(segmentsFile, IOContext.DEFAULT)) { + output.writeString(content); + } + + // Verify it's in the base directory (not routed to primary term directory) + assertTrue(Arrays.asList(distributedDirectory.listAll()).contains(segmentsFile)); + + // Verify routing info shows it's excluded + String routingInfo = distributedDirectory.getRoutingInfo(segmentsFile); + assertTrue(routingInfo.contains("excluded")); + } + + public void testMultipleFilesWithDifferentPrimaryTerms() throws IOException { + String[] filenames = {"_0.si", "_1.cfs", "_2.cfe"}; + String[] contents = {"content1", "content2", "content3"}; + + // Create files with primary term 1 + for (int i = 0; i < filenames.length; i++) { + try (IndexOutput output = distributedDirectory.createOutput(filenames[i], IOContext.DEFAULT)) { + output.writeString(contents[i]); } - - writer.commit(); } - // Verify files were created and distributed - String[] files = distributedDirectory.listAll(); - assertTrue("Should have created segment files", files.length > 0); + // Change primary term + when(mockIndexShard.getOperationPrimaryTerm()).thenReturn(2L); + + String newFilename = "_3.doc"; + String newContent = "content for primary term 2"; + + // Create file with primary term 2 + try (IndexOutput output = distributedDirectory.createOutput(newFilename, IOContext.DEFAULT)) { + output.writeString(newContent); + } + + // Verify all files are listed + String[] allFiles = distributedDirectory.listAll(); + Set fileSet = new HashSet<>(Arrays.asList(allFiles)); - // Check that files are distributed across directories - boolean hasSegmentsFile = false; - boolean hasDistributedFiles = false; + for (String filename : filenames) { + assertTrue("File " + filename + " should be listed", fileSet.contains(filename)); + } + assertTrue("New file should be listed", fileSet.contains(newFilename)); - for (String file : files) { - if (file.startsWith("segments_")) { - hasSegmentsFile = true; - assertEquals("segments_N should be in directory 0", 0, distributedDirectory.getDirectoryIndex(file)); - } else { - hasDistributedFiles = true; + // Verify files can be read correctly + for (int i = 0; i < filenames.length; i++) { + try (IndexInput input = distributedDirectory.openInput(filenames[i], IOContext.DEFAULT)) { + assertEquals(contents[i], input.readString()); } } - assertTrue("Should have segments file", hasSegmentsFile); - assertTrue("Should have other distributed files", hasDistributedFiles); + try (IndexInput input = distributedDirectory.openInput(newFilename, IOContext.DEFAULT)) { + assertEquals(newContent, input.readString()); + } + } + + public void testFileDeletion() throws IOException { + String filename = "_0.si"; + String content = "test content"; + + // Create file + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString(content); + } + + // Verify file exists + assertTrue(Arrays.asList(distributedDirectory.listAll()).contains(filename)); + + // Delete file + distributedDirectory.deleteFile(filename); + + // Verify file is deleted + assertFalse(Arrays.asList(distributedDirectory.listAll()).contains(filename)); } - public void testIndexReaderWithDistributedDirectory() throws IOException { - // First create an index - IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); - - try (IndexWriter writer = new IndexWriter(distributedDirectory, config)) { - for (int i = 0; i < 5; i++) { - Document doc = new Document(); - doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); - doc.add(new TextField("content", "Document content " + i, Field.Store.YES)); - writer.addDocument(doc); + public void testSyncOperation() throws IOException { + String[] filenames = {"_0.si", "_1.cfs", "segments_1"}; + String content = "sync test content"; + + // Create multiple files + for (String filename : filenames) { + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString(content); } - writer.commit(); } - // Now read the index - try (IndexReader reader = DirectoryReader.open(distributedDirectory)) { - assertEquals("Should have 5 documents", 5, reader.numDocs()); - - IndexSearcher searcher = new IndexSearcher(reader); - - // Search for a specific document - TermQuery query = new TermQuery(new Term("id", "2")); - TopDocs results = searcher.search(query, 10); - - assertEquals("Should find one document", 1, results.totalHits.value); - - Document doc = searcher.doc(results.scoreDocs[0].doc); - assertEquals("Should find correct document", "2", doc.get("id")); + // Sync all files - should not throw exception + Collection filesToSync = Arrays.asList(filenames); + try { + distributedDirectory.sync(filesToSync); + } catch (IOException e) { + fail("sync should not throw exception: " + e.getMessage()); } } - public void testIndexWriterWithMerging() throws IOException { - IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); - config.setMaxBufferedDocs(2); // Force frequent segment creation - - try (IndexWriter writer = new IndexWriter(distributedDirectory, config)) { - // Add documents to create multiple segments - for (int i = 0; i < 20; i++) { - Document doc = new Document(); - doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); - doc.add(new TextField("content", "Content for document " + i, Field.Store.YES)); - writer.addDocument(doc); - - if (i % 5 == 0) { - writer.commit(); // Create multiple commits - } - } - - // Force merge to test segment merging with distributed files - writer.forceMerge(1); - writer.commit(); + public void testRenameOperation() throws IOException { + String sourceFilename = "_0.si"; + String destFilename = "_0_renamed.si"; + String content = "rename test content"; + + // Create source file + try (IndexOutput output = distributedDirectory.createOutput(sourceFilename, IOContext.DEFAULT)) { + output.writeString(content); } - // Verify the index is still readable after merging - try (IndexReader reader = DirectoryReader.open(distributedDirectory)) { - assertEquals("Should have all 20 documents after merge", 20, reader.numDocs()); - - // Verify we can search - IndexSearcher searcher = new IndexSearcher(reader); - TermQuery query = new TermQuery(new Term("id", "15")); - TopDocs results = searcher.search(query, 10); - - assertEquals("Should find document after merge", 1, results.totalHits.value); + // Rename file + distributedDirectory.rename(sourceFilename, destFilename); + + // Verify source file is gone and dest file exists + String[] allFiles = distributedDirectory.listAll(); + Set fileSet = new HashSet<>(Arrays.asList(allFiles)); + + assertFalse("Source file should be gone", fileSet.contains(sourceFilename)); + assertTrue("Dest file should exist", fileSet.contains(destFilename)); + + // Verify content is preserved + try (IndexInput input = distributedDirectory.openInput(destFilename, IOContext.DEFAULT)) { + assertEquals(content, input.readString()); } } - public void testIndexDeletion() throws IOException { - IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); - - try (IndexWriter writer = new IndexWriter(distributedDirectory, config)) { - // Add documents - for (int i = 0; i < 10; i++) { - Document doc = new Document(); - doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); - doc.add(new TextField("content", "Document " + i, Field.Store.YES)); - writer.addDocument(doc); - } - writer.commit(); - - // Delete some documents - writer.deleteDocuments(new Term("id", "5")); - writer.deleteDocuments(new Term("id", "7")); - writer.commit(); + public void testCrossDirectoryRenameFailure() throws IOException { + String sourceFilename = "_0.si"; // Regular file (goes to primary term directory) + String destFilename = "segments_1"; // Excluded file (goes to base directory) + String content = "cross directory test"; + + // Create source file + try (IndexOutput output = distributedDirectory.createOutput(sourceFilename, IOContext.DEFAULT)) { + output.writeString(content); } - // Verify deletions - try (IndexReader reader = DirectoryReader.open(distributedDirectory)) { - assertEquals("Should have 8 documents after deletion", 8, reader.numDocs()); - - IndexSearcher searcher = new IndexSearcher(reader); - - // Verify deleted documents are not found - TermQuery query1 = new TermQuery(new Term("id", "5")); - TopDocs results1 = searcher.search(query1, 10); - assertEquals("Deleted document should not be found", 0, results1.totalHits.value); - - TermQuery query2 = new TermQuery(new Term("id", "7")); - TopDocs results2 = searcher.search(query2, 10); - assertEquals("Deleted document should not be found", 0, results2.totalHits.value); - - // Verify non-deleted documents are still found - TermQuery query3 = new TermQuery(new Term("id", "3")); - TopDocs results3 = searcher.search(query3, 10); - assertEquals("Non-deleted document should be found", 1, results3.totalHits.value); + // Attempt cross-directory rename should fail + PrimaryTermRoutingException exception = expectThrows( + PrimaryTermRoutingException.class, + () -> distributedDirectory.rename(sourceFilename, destFilename) + ); + + assertEquals(PrimaryTermRoutingException.ErrorType.FILE_ROUTING_ERROR, exception.getErrorType()); + assertTrue(exception.getMessage().contains("Cross-directory rename not supported")); + } + + public void testDirectoryStats() throws IOException { + // Initially should show primary term routing + String stats = distributedDirectory.getDirectoryStats(); + assertTrue(stats.contains("Primary term routing")); + + // Create some files to populate directories + try (IndexOutput output = distributedDirectory.createOutput("_0.si", IOContext.DEFAULT)) { + output.writeString("test"); } + + // Stats should still be available + stats = distributedDirectory.getDirectoryStats(); + assertNotNull(stats); } - public void testConcurrentIndexing() throws IOException, InterruptedException { - IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); - - try (IndexWriter writer = new IndexWriter(distributedDirectory, config)) { - int numThreads = 4; - int docsPerThread = 25; - ExecutorService executor = Executors.newFixedThreadPool(numThreads); - CountDownLatch latch = new CountDownLatch(numThreads); - AtomicInteger errorCount = new AtomicInteger(0); - - // Start concurrent indexing threads - for (int t = 0; t < numThreads; t++) { - final int threadId = t; - executor.submit(() -> { - try { - for (int i = 0; i < docsPerThread; i++) { - Document doc = new Document(); - String docId = threadId + "_" + i; - doc.add(new StringField("id", docId, Field.Store.YES)); - doc.add(new StringField("thread", String.valueOf(threadId), Field.Store.YES)); - doc.add(new TextField("content", "Content from thread " + threadId + " doc " + i, Field.Store.YES)); - - writer.addDocument(doc); - - // Occasionally commit - if (i % 10 == 0) { - writer.commit(); - } - } - } catch (Exception e) { - logger.error("Error in indexing thread " + threadId, e); - errorCount.incrementAndGet(); - } finally { - latch.countDown(); - } - }); - } - - // Wait for all threads to complete - assertTrue("All threads should complete", latch.await(30, TimeUnit.SECONDS)); - assertEquals("No errors should occur during concurrent indexing", 0, errorCount.get()); - - writer.commit(); - executor.shutdown(); + public void testRoutingInfo() { + // Test excluded file + String segmentsFile = "segments_1"; + String routingInfo = distributedDirectory.getRoutingInfo(segmentsFile); + assertTrue(routingInfo.contains("excluded")); + assertTrue(routingInfo.contains("base directory")); + + // Test regular file + String regularFile = "_0.si"; + routingInfo = distributedDirectory.getRoutingInfo(regularFile); + assertTrue(routingInfo.contains("primary term")); + } + + public void testValidateDirectories() throws IOException { + // Create some files to ensure directories exist + try (IndexOutput output = distributedDirectory.createOutput("_0.si", IOContext.DEFAULT)) { + output.writeString("test"); } - // Verify all documents were indexed correctly - try (IndexReader reader = DirectoryReader.open(distributedDirectory)) { - int expectedDocs = 4 * 25; // 4 threads * 25 docs each - assertEquals("Should have all documents from concurrent indexing", expectedDocs, reader.numDocs()); - - IndexSearcher searcher = new IndexSearcher(reader); - - // Verify documents from each thread - for (int t = 0; t < 4; t++) { - TermQuery query = new TermQuery(new Term("thread", String.valueOf(t))); - TopDocs results = searcher.search(query, 100); - assertEquals("Should have 25 documents from thread " + t, 25, results.totalHits.value); - } + // Validation should not throw exception + try { + distributedDirectory.validateDirectories(); + } catch (IOException e) { + fail("validateDirectories should not throw exception: " + e.getMessage()); } } - public void testIndexOptimization() throws IOException { - IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); - config.setMaxBufferedDocs(5); // Create multiple segments - - try (IndexWriter writer = new IndexWriter(distributedDirectory, config)) { - // Add many documents to create multiple segments - for (int i = 0; i < 50; i++) { - Document doc = new Document(); - doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); - doc.add(new TextField("content", "Document content " + i, Field.Store.YES)); - writer.addDocument(doc); - - if (i % 10 == 0) { - writer.commit(); - } - } - - // Get file count before optimization - String[] filesBefore = distributedDirectory.listAll(); - int filesBeforeCount = filesBefore.length; + public void testCurrentPrimaryTerm() { + // Should return the mocked primary term + assertEquals(1L, distributedDirectory.getCurrentPrimaryTerm()); + + // Change mock and verify + when(mockIndexShard.getOperationPrimaryTerm()).thenReturn(5L); + assertEquals(5L, distributedDirectory.getCurrentPrimaryTerm()); + } + + public void testIsUsingPrimaryTermRouting() { + assertTrue(distributedDirectory.isUsingPrimaryTermRouting()); + } + + public void testLegacyHashBasedRouting() throws IOException { + // Create a legacy distributed directory (without IndexShard) + DistributedSegmentDirectory legacyDirectory = new DistributedSegmentDirectory(baseDirectory, tempDir); + + try { + assertFalse(legacyDirectory.isUsingPrimaryTermRouting()); + assertEquals(-1L, legacyDirectory.getCurrentPrimaryTerm()); - // Optimize (force merge to 1 segment) - writer.forceMerge(1); - writer.commit(); + // Should still work for basic operations + try (IndexOutput output = legacyDirectory.createOutput("_0.si", IOContext.DEFAULT)) { + output.writeString("legacy test"); + } - // Get file count after optimization - String[] filesAfter = distributedDirectory.listAll(); - int filesAfterCount = filesAfter.length; + assertTrue(Arrays.asList(legacyDirectory.listAll()).contains("_0.si")); - // After optimization, we should have fewer files (merged segments) - assertTrue("Should have fewer files after optimization", filesAfterCount <= filesBeforeCount); - } - - // Verify index is still functional after optimization - try (IndexReader reader = DirectoryReader.open(distributedDirectory)) { - assertEquals("Should still have all 50 documents", 50, reader.numDocs()); - assertEquals("Should have only 1 segment after optimization", 1, reader.leaves().size()); + } finally { + legacyDirectory.close(); } } - public void testLargeDocuments() throws IOException { - IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer()); - - try (IndexWriter writer = new IndexWriter(distributedDirectory, config)) { - // Create documents with large content - for (int i = 0; i < 5; i++) { - Document doc = new Document(); - doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); - - // Create large content (about 1MB per document) - StringBuilder largeContent = new StringBuilder(); - for (int j = 0; j < 10000; j++) { - largeContent.append("This is a large document with lots of content. Document ID: ").append(i).append(" Line: ").append(j).append(". "); - } - - doc.add(new TextField("content", largeContent.toString(), Field.Store.YES)); - writer.addDocument(doc); + public void testConcurrentFileOperations() throws IOException { + String[] filenames = {"_0.si", "_1.cfs", "_2.cfe", "_3.doc"}; + String content = "concurrent test content"; + + // Create multiple files concurrently (simulated) + for (String filename : filenames) { + try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { + output.writeString(content + "_" + filename); } - - writer.commit(); } - // Verify large documents can be read - try (IndexReader reader = DirectoryReader.open(distributedDirectory)) { - assertEquals("Should have 5 large documents", 5, reader.numDocs()); - - IndexSearcher searcher = new IndexSearcher(reader); - TermQuery query = new TermQuery(new Term("id", "2")); - TopDocs results = searcher.search(query, 10); - - assertEquals("Should find the large document", 1, results.totalHits.value); + // Verify all files exist and have correct content + String[] allFiles = distributedDirectory.listAll(); + Set fileSet = new HashSet<>(Arrays.asList(allFiles)); + + for (String filename : filenames) { + assertTrue("File " + filename + " should exist", fileSet.contains(filename)); - Document doc = searcher.doc(results.scoreDocs[0].doc); - String content = doc.get("content"); - assertTrue("Large document content should be preserved", content.length() > 100000); - assertTrue("Content should contain expected text", content.contains("Document ID: 2")); + try (IndexInput input = distributedDirectory.openInput(filename, IOContext.DEFAULT)) { + assertEquals(content + "_" + filename, input.readString()); + } } } } \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryTests.java deleted file mode 100644 index 99d2d006c680d..0000000000000 --- a/server/src/test/java/org/opensearch/index/store/distributed/DistributedSegmentDirectoryTests.java +++ /dev/null @@ -1,719 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.store.distributed; - -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.opensearch.test.OpenSearchTestCase; - -import java.io.IOException; -import java.nio.file.Path; - -public class DistributedSegmentDirectoryTests extends OpenSearchTestCase { - - private Path tempDir; - private Directory baseDirectory; - private DistributedSegmentDirectory distributedDirectory; - - @Override - public void setUp() throws Exception { - super.setUp(); - tempDir = createTempDir(); - baseDirectory = FSDirectory.open(tempDir); - distributedDirectory = new DistributedSegmentDirectory(baseDirectory, tempDir); - } - - @Override - public void tearDown() throws Exception { - if (distributedDirectory != null) { - distributedDirectory.close(); - } - super.tearDown(); - } - - public void testDirectoryCreation() throws IOException { - assertNotNull("Distributed directory should be created", distributedDirectory); - assertEquals("Should have 5 directories", 5, distributedDirectory.getDirectoryManager().getNumDirectories()); - assertNotNull("Hasher should be initialized", distributedDirectory.getHasher()); - } - - public void testDirectoryResolution() throws IOException { - // Test that different files resolve to appropriate directories - String segmentsFile = "segments_1"; - String regularFile = "_0.cfe"; - - Directory segmentsDir = distributedDirectory.resolveDirectory(segmentsFile); - Directory regularDir = distributedDirectory.resolveDirectory(regularFile); - - // segments_N should always go to base directory (index 0) - assertSame("segments_1 should resolve to base directory", - distributedDirectory.getDirectoryManager().getDirectory(0), segmentsDir); - - // Regular files should be distributed - int regularIndex = distributedDirectory.getDirectoryIndex(regularFile); - assertSame("Regular file should resolve to correct directory", - distributedDirectory.getDirectoryManager().getDirectory(regularIndex), regularDir); - } - - public void testOpenInputWithExistingFile() throws IOException { - String filename = "_0.cfe"; - String content = "test content"; - - // First create the file in the appropriate directory - int dirIndex = distributedDirectory.getDirectoryIndex(filename); - Directory targetDir = distributedDirectory.getDirectoryManager().getDirectory(dirIndex); - - try (IndexOutput output = targetDir.createOutput(filename, IOContext.DEFAULT)) { - output.writeString(content); - } - - // Now test opening the file through distributed directory - try (IndexInput input = distributedDirectory.openInput(filename, IOContext.DEFAULT)) { - assertNotNull("Input should not be null", input); - assertEquals("Content should match", content, input.readString()); - } - } - - public void testOpenInputWithNonExistentFile() throws IOException { - String filename = "nonexistent.file"; - - DistributedDirectoryException exception = expectThrows(DistributedDirectoryException.class, - () -> distributedDirectory.openInput(filename, IOContext.DEFAULT)); - - assertTrue("Exception should mention the operation", exception.getMessage().contains("openInput")); - assertTrue("Exception should mention the filename", exception.getMessage().contains(filename)); - assertEquals("Exception should have correct operation", "openInput", exception.getOperation()); - assertTrue("Exception should have valid directory index", - exception.getDirectoryIndex() >= 0 && exception.getDirectoryIndex() < 5); - } - - public void testOpenInputFileDistribution() throws IOException { - String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm", "_3.fdt", "_4.tim"}; - String content = "test content"; - - // Create files in their respective directories - for (String filename : testFiles) { - int dirIndex = distributedDirectory.getDirectoryIndex(filename); - Directory targetDir = distributedDirectory.getDirectoryManager().getDirectory(dirIndex); - - try (IndexOutput output = targetDir.createOutput(filename, IOContext.DEFAULT)) { - output.writeString(content + " for " + filename); - } - } - - // Verify we can read all files through distributed directory - for (String filename : testFiles) { - try (IndexInput input = distributedDirectory.openInput(filename, IOContext.DEFAULT)) { - String readContent = input.readString(); - assertEquals("Content should match for " + filename, content + " for " + filename, readContent); - } - } - } - - public void testSegmentsFileInBaseDirectory() throws IOException { - String segmentsFile = "segments_1"; - String content = "segments content"; - - // Create segments file directly in base directory - try (IndexOutput output = baseDirectory.createOutput(segmentsFile, IOContext.DEFAULT)) { - output.writeString(content); - } - - // Verify we can read it through distributed directory - try (IndexInput input = distributedDirectory.openInput(segmentsFile, IOContext.DEFAULT)) { - assertEquals("Segments file content should match", content, input.readString()); - } - - // Verify it's in the base directory (index 0) - assertEquals("segments_1 should be in directory 0", 0, distributedDirectory.getDirectoryIndex(segmentsFile)); - } - - public void testClose() throws IOException { - // Create some files to ensure directories are in use - String filename = "_0.cfe"; - int dirIndex = distributedDirectory.getDirectoryIndex(filename); - Directory targetDir = distributedDirectory.getDirectoryManager().getDirectory(dirIndex); - - try (IndexOutput output = targetDir.createOutput(filename, IOContext.DEFAULT)) { - output.writeString("test"); - } - - // Close should not throw exception - distributedDirectory.close(); - - // After closing, operations should fail - expectThrows(Exception.class, () -> distributedDirectory.openInput(filename, IOContext.DEFAULT)); - } -} - public void testCreateOutput() throws IOException { - String filename = "_0.cfe"; - String content = "test output content"; - - // Create output through distributed directory - try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { - assertNotNull("Output should not be null", output); - output.writeString(content); - } - - // Verify file was created in correct directory - int dirIndex = distributedDirectory.getDirectoryIndex(filename); - Directory targetDir = distributedDirectory.getDirectoryManager().getDirectory(dirIndex); - - try (IndexInput input = targetDir.openInput(filename, IOContext.DEFAULT)) { - assertEquals("Content should match", content, input.readString()); - } - } - - public void testCreateOutputSegmentsFile() throws IOException { - String segmentsFile = "segments_1"; - String content = "segments content"; - - // Create segments file through distributed directory - try (IndexOutput output = distributedDirectory.createOutput(segmentsFile, IOContext.DEFAULT)) { - output.writeString(content); - } - - // Verify it was created in base directory (index 0) - assertEquals("segments_1 should be in directory 0", 0, distributedDirectory.getDirectoryIndex(segmentsFile)); - - try (IndexInput input = baseDirectory.openInput(segmentsFile, IOContext.DEFAULT)) { - assertEquals("Segments content should match", content, input.readString()); - } - } - - public void testCreateOutputMultipleFiles() throws IOException { - String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm", "_3.fdt", "_4.tim"}; - - // Create multiple files - for (String filename : testFiles) { - try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { - output.writeString("content for " + filename); - } - } - - // Verify all files can be read back - for (String filename : testFiles) { - try (IndexInput input = distributedDirectory.openInput(filename, IOContext.DEFAULT)) { - assertEquals("Content should match for " + filename, - "content for " + filename, input.readString()); - } - } - } - - public void testCreateOutputDistribution() throws IOException { - String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm", "_3.fdt", "_4.tim", "_5.tip", "_6.doc"}; - - // Track which directories are used - boolean[] directoriesUsed = new boolean[5]; - - for (String filename : testFiles) { - int dirIndex = distributedDirectory.getDirectoryIndex(filename); - directoriesUsed[dirIndex] = true; - - try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { - output.writeString("test"); - } - } - - // Should use multiple directories - int usedCount = 0; - for (boolean used : directoriesUsed) { - if (used) usedCount++; - } - - assertTrue("Should use multiple directories, used: " + usedCount, usedCount > 1); - } - public void testDeleteFile() throws IOException { - String filename = "_0.cfe"; - String content = "test content"; - - // Create file - try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { - output.writeString(content); - } - - // Verify file exists - try (IndexInput input = distributedDirectory.openInput(filename, IOContext.DEFAULT)) { - assertEquals("File should exist", content, input.readString()); - } - - // Delete file - distributedDirectory.deleteFile(filename); - - // Verify file is deleted - expectThrows(Exception.class, () -> distributedDirectory.openInput(filename, IOContext.DEFAULT)); - } - - public void testDeleteSegmentsFile() throws IOException { - String segmentsFile = "segments_1"; - String content = "segments content"; - - // Create segments file - try (IndexOutput output = distributedDirectory.createOutput(segmentsFile, IOContext.DEFAULT)) { - output.writeString(content); - } - - // Delete segments file - distributedDirectory.deleteFile(segmentsFile); - - // Verify it's deleted from base directory - expectThrows(Exception.class, () -> baseDirectory.openInput(segmentsFile, IOContext.DEFAULT)); - } - - public void testDeleteNonExistentFile() throws IOException { - String filename = "nonexistent.file"; - - DistributedDirectoryException exception = expectThrows(DistributedDirectoryException.class, - () -> distributedDirectory.deleteFile(filename)); - - assertTrue("Exception should mention the operation", exception.getMessage().contains("deleteFile")); - assertTrue("Exception should mention the filename", exception.getMessage().contains(filename)); - assertEquals("Exception should have correct operation", "deleteFile", exception.getOperation()); - } - - public void testFileLength() throws IOException { - String filename = "_0.cfe"; - String content = "test content for length"; - - // Create file - try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { - output.writeString(content); - } - - // Get file length through distributed directory - long length = distributedDirectory.fileLength(filename); - - // Verify length matches direct access - int dirIndex = distributedDirectory.getDirectoryIndex(filename); - Directory targetDir = distributedDirectory.getDirectoryManager().getDirectory(dirIndex); - long directLength = targetDir.fileLength(filename); - - assertEquals("File length should match", directLength, length); - assertTrue("File length should be positive", length > 0); - } - - public void testFileLengthSegmentsFile() throws IOException { - String segmentsFile = "segments_1"; - String content = "segments content"; - - // Create segments file - try (IndexOutput output = distributedDirectory.createOutput(segmentsFile, IOContext.DEFAULT)) { - output.writeString(content); - } - - // Get length through distributed directory - long length = distributedDirectory.fileLength(segmentsFile); - - // Verify length matches base directory - long baseLength = baseDirectory.fileLength(segmentsFile); - assertEquals("Segments file length should match", baseLength, length); - } - - public void testFileLengthNonExistentFile() throws IOException { - String filename = "nonexistent.file"; - - DistributedDirectoryException exception = expectThrows(DistributedDirectoryException.class, - () -> distributedDirectory.fileLength(filename)); - - assertTrue("Exception should mention the operation", exception.getMessage().contains("fileLength")); - assertTrue("Exception should mention the filename", exception.getMessage().contains(filename)); - assertEquals("Exception should have correct operation", "fileLength", exception.getOperation()); - } - - public void testFileOperationsConsistency() throws IOException { - String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm", "segments_1"}; - - for (String filename : testFiles) { - // Create file - try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { - output.writeString("content for " + filename); - } - - // Check length - long length = distributedDirectory.fileLength(filename); - assertTrue("File length should be positive for " + filename, length > 0); - - // Read file - try (IndexInput input = distributedDirectory.openInput(filename, IOContext.DEFAULT)) { - assertEquals("Content should match for " + filename, - "content for " + filename, input.readString()); - } - - // Delete file - distributedDirectory.deleteFile(filename); - - // Verify deletion - expectThrows(Exception.class, () -> distributedDirectory.openInput(filename, IOContext.DEFAULT)); - } - } pub -lic void testListAllEmpty() throws IOException { - String[] files = distributedDirectory.listAll(); - assertNotNull("File list should not be null", files); - assertEquals("Should have no files initially", 0, files.length); - } - - public void testListAllSingleFile() throws IOException { - String filename = "_0.cfe"; - - // Create file - try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { - output.writeString("test"); - } - - String[] files = distributedDirectory.listAll(); - assertEquals("Should have one file", 1, files.length); - assertEquals("File should match", filename, files[0]); - } - - public void testListAllMultipleFiles() throws IOException { - String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm", "segments_1", "_3.fdt"}; - - // Create files - for (String filename : testFiles) { - try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { - output.writeString("content for " + filename); - } - } - - String[] listedFiles = distributedDirectory.listAll(); - assertEquals("Should have all files", testFiles.length, listedFiles.length); - - // Convert to set for easier comparison - Set expectedFiles = Set.of(testFiles); - Set actualFiles = Set.of(listedFiles); - assertEquals("All files should be listed", expectedFiles, actualFiles); - } - - public void testListAllDistributedFiles() throws IOException { - String[] testFiles = { - "_0.cfe", "_1.si", "_2.fnm", "_3.fdt", "_4.tim", - "_5.tip", "_6.doc", "_7.pos", "segments_1", "segments_2" - }; - - // Create files across different directories - for (String filename : testFiles) { - try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { - output.writeString("test"); - } - } - - // Verify files are in different directories - Set usedDirectories = new HashSet<>(); - for (String filename : testFiles) { - usedDirectories.add(distributedDirectory.getDirectoryIndex(filename)); - } - assertTrue("Should use multiple directories", usedDirectories.size() > 1); - - // Verify listAll returns all files - String[] listedFiles = distributedDirectory.listAll(); - Set expectedFiles = Set.of(testFiles); - Set actualFiles = Set.of(listedFiles); - assertEquals("All distributed files should be listed", expectedFiles, actualFiles); - } - - public void testListAllAfterDeletion() throws IOException { - String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm"}; - - // Create files - for (String filename : testFiles) { - try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { - output.writeString("test"); - } - } - - // Verify all files are listed - String[] allFiles = distributedDirectory.listAll(); - assertEquals("Should have all files", testFiles.length, allFiles.length); - - // Delete one file - distributedDirectory.deleteFile(testFiles[0]); - - // Verify updated list - String[] remainingFiles = distributedDirectory.listAll(); - assertEquals("Should have one less file", testFiles.length - 1, remainingFiles.length); - - Set remainingSet = Set.of(remainingFiles); - assertFalse("Deleted file should not be listed", remainingSet.contains(testFiles[0])); - assertTrue("Other files should still be listed", remainingSet.contains(testFiles[1])); - assertTrue("Other files should still be listed", remainingSet.contains(testFiles[2])); - } - - public void testListAllNoDuplicates() throws IOException { - String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm", "_3.fdt"}; - - // Create files - for (String filename : testFiles) { - try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { - output.writeString("test"); - } - } - - String[] listedFiles = distributedDirectory.listAll(); - - // Check for duplicates - Set uniqueFiles = Set.of(listedFiles); - assertEquals("Should have no duplicates", listedFiles.length, uniqueFiles.size()); - } pu -blic void testSyncSingleFile() throws IOException { - String filename = "_0.cfe"; - - // Create file - try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { - output.writeString("test"); - } - - // Sync should not throw exception - distributedDirectory.sync(Collections.singletonList(filename)); - } - - public void testSyncMultipleFilesInSameDirectory() throws IOException { - // Create files that will hash to the same directory - String filename1 = "_0.cfe"; - String filename2 = "_0.cfs"; - - // Create files - try (IndexOutput output = distributedDirectory.createOutput(filename1, IOContext.DEFAULT)) { - output.writeString("test1"); - } - try (IndexOutput output = distributedDirectory.createOutput(filename2, IOContext.DEFAULT)) { - output.writeString("test2"); - } - - // Sync both files - distributedDirectory.sync(List.of(filename1, filename2)); - } - - public void testSyncMultipleFilesAcrossDirectories() throws IOException { - String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm", "segments_1"}; - - // Create files - for (String filename : testFiles) { - try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { - output.writeString("content for " + filename); - } - } - - // Sync all files - distributedDirectory.sync(List.of(testFiles)); - } - - public void testSyncEmptyList() throws IOException { - // Sync empty list should not throw exception - distributedDirectory.sync(Collections.emptyList()); - } - - public void testRenameSameDirectory() throws IOException { - String sourceFile = "_0.cfe"; - String destFile = "_0.renamed"; - - // Ensure both files would be in the same directory - int sourceIndex = distributedDirectory.getDirectoryIndex(sourceFile); - int destIndex = distributedDirectory.getDirectoryIndex(destFile); - - // Create source file - try (IndexOutput output = distributedDirectory.createOutput(sourceFile, IOContext.DEFAULT)) { - output.writeString("test content"); - } - - if (sourceIndex == destIndex) { - // Rename should work - distributedDirectory.rename(sourceFile, destFile); - - // Verify rename - expectThrows(Exception.class, () -> distributedDirectory.openInput(sourceFile, IOContext.DEFAULT)); - - try (IndexInput input = distributedDirectory.openInput(destFile, IOContext.DEFAULT)) { - assertEquals("Content should be preserved", "test content", input.readString()); - } - } else { - // If they're in different directories, rename should fail - DistributedDirectoryException exception = expectThrows(DistributedDirectoryException.class, - () -> distributedDirectory.rename(sourceFile, destFile)); - assertTrue("Exception should mention cross-directory rename", - exception.getMessage().contains("Cross-directory rename not supported")); - } - } - - public void testRenameCrossDirectory() throws IOException { - // Find two files that hash to different directories - String sourceFile = null; - String destFile = null; - - String[] candidates = {"_0.cfe", "_1.si", "_2.fnm", "_3.fdt", "_4.tim", "_5.tip"}; - - for (int i = 0; i < candidates.length; i++) { - for (int j = i + 1; j < candidates.length; j++) { - if (distributedDirectory.getDirectoryIndex(candidates[i]) != - distributedDirectory.getDirectoryIndex(candidates[j])) { - sourceFile = candidates[i]; - destFile = candidates[j]; - break; - } - } - if (sourceFile != null) break; - } - - if (sourceFile != null && destFile != null) { - // Create source file - try (IndexOutput output = distributedDirectory.createOutput(sourceFile, IOContext.DEFAULT)) { - output.writeString("test"); - } - - // Cross-directory rename should fail - DistributedDirectoryException exception = expectThrows(DistributedDirectoryException.class, - () -> distributedDirectory.rename(sourceFile, destFile)); - assertTrue("Exception should mention cross-directory rename", - exception.getMessage().contains("Cross-directory rename not supported")); - assertEquals("Exception should have correct operation", "rename", exception.getOperation()); - } - } - - public void testRenameNonExistentFile() throws IOException { - String sourceFile = "_0.cfe"; - String destFile = "_0.renamed"; - - // Ensure both would be in same directory for this test - if (distributedDirectory.getDirectoryIndex(sourceFile) == distributedDirectory.getDirectoryIndex(destFile)) { - DistributedDirectoryException exception = expectThrows(DistributedDirectoryException.class, - () -> distributedDirectory.rename(sourceFile, destFile)); - assertEquals("Exception should have correct operation", "rename", exception.getOperation()); - } - } - public void testCloseWithFiles() throws IOException { - String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm", "segments_1"}; - - // Create files in different directories - for (String filename : testFiles) { - try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { - output.writeString("content for " + filename); - } - } - - // Verify files exist - String[] listedFiles = distributedDirectory.listAll(); - assertEquals("Should have all files", testFiles.length, listedFiles.length); - - // Close should not throw exception - distributedDirectory.close(); - - // After closing, operations should fail - expectThrows(Exception.class, () -> distributedDirectory.listAll()); - expectThrows(Exception.class, () -> distributedDirectory.openInput(testFiles[0], IOContext.DEFAULT)); - expectThrows(Exception.class, () -> distributedDirectory.createOutput("newfile", IOContext.DEFAULT)); - } - - public void testCloseEmpty() throws IOException { - // Close empty directory should not throw exception - distributedDirectory.close(); - - // Operations should fail after close - expectThrows(Exception.class, () -> distributedDirectory.listAll()); - } - - public void testCloseIdempotent() throws IOException { - // First close - distributedDirectory.close(); - - // Second close should not throw exception (idempotent) - distributedDirectory.close(); - } - - public void testResourceCleanupOrder() throws IOException { - String filename = "_0.cfe"; - - // Create file - try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { - output.writeString("test"); - } - - // Get reference to directory manager before closing - DirectoryManager manager = distributedDirectory.getDirectoryManager(); - assertNotNull("Directory manager should exist", manager); - - // Close should clean up resources properly - distributedDirectory.close(); - - // Verify cleanup by attempting operations - expectThrows(Exception.class, () -> distributedDirectory.openInput(filename, IOContext.DEFAULT)); - } -public void testListAllFiltersSubdirectories() throws IOException { - String[] testFiles = {"_0.cfe", "_1.si", "_2.fnm", "segments_1"}; - - // Create files - for (String filename : testFiles) { - try (IndexOutput output = distributedDirectory.createOutput(filename, IOContext.DEFAULT)) { - output.writeString("content for " + filename); - } - } - - String[] listedFiles = distributedDirectory.listAll(); - Set listedSet = Set.of(listedFiles); - - // Verify all our test files are listed - for (String testFile : testFiles) { - assertTrue("Test file should be listed: " + testFile, listedSet.contains(testFile)); - } - - // Verify subdirectory names are NOT listed (segments_1, segments_2, etc. directories) - assertFalse("Subdirectory segments_1 should not be listed as a file", - listedSet.contains("segments_1") && isDirectory("segments_1")); - assertFalse("Subdirectory segments_2 should not be listed as a file", - listedSet.contains("segments_2")); - assertFalse("Subdirectory segments_3 should not be listed as a file", - listedSet.contains("segments_3")); - assertFalse("Subdirectory segments_4 should not be listed as a file", - listedSet.contains("segments_4")); - - // But the actual segments_1 file should be listed - assertTrue("Actual segments_1 file should be listed", listedSet.contains("segments_1")); - } - - private boolean isDirectory(String name) { - // Check if this is actually a directory in the base path - return tempDir.resolve(name).toFile().isDirectory(); - } - - public void testListAllWithOnlySubdirectories() throws IOException { - // Don't create any files, just verify subdirectories are not listed - String[] listedFiles = distributedDirectory.listAll(); - - // Should not contain any of our subdirectory names - Set listedSet = Set.of(listedFiles); - for (int i = 1; i < 5; i++) { - String subdirName = "segments_" + i; - if (listedSet.contains(subdirName)) { - // If it's listed, it should be a file, not a directory - assertFalse("If segments_" + i + " is listed, it should be a file not a directory", - tempDir.resolve(subdirName).toFile().isDirectory()); - } - } - } - - public void testListAllDistinguishesFilesFromDirectories() throws IOException { - // Create a real segments_2 file (not just the directory) - String segmentsFile = "segments_2"; - try (IndexOutput output = distributedDirectory.createOutput(segmentsFile, IOContext.DEFAULT)) { - output.writeString("real segments file content"); - } - - String[] listedFiles = distributedDirectory.listAll(); - Set listedSet = Set.of(listedFiles); - - // The real segments_2 file should be listed - assertTrue("Real segments_2 file should be listed", listedSet.contains(segmentsFile)); - - // Verify we can read it back - try (IndexInput input = distributedDirectory.openInput(segmentsFile, IOContext.DEFAULT)) { - assertEquals("Should be able to read the real segments file", - "real segments file content", input.readString()); - } - } \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/IndexShardContextTests.java b/server/src/test/java/org/opensearch/index/store/distributed/IndexShardContextTests.java new file mode 100644 index 0000000000000..7919c9349530b --- /dev/null +++ b/server/src/test/java/org/opensearch/index/store/distributed/IndexShardContextTests.java @@ -0,0 +1,126 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.opensearch.index.shard.IndexShard; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + + +public class IndexShardContextTests extends OpenSearchTestCase { + + private IndexShard mockIndexShard; + private IndexShardContext context; + + @Before + public void setUp() throws Exception { + super.setUp(); + mockIndexShard = mock(IndexShard.class); + } + + public void testGetPrimaryTermWithValidIndexShard() { + long expectedPrimaryTerm = 5L; + when(mockIndexShard.getOperationPrimaryTerm()).thenReturn(expectedPrimaryTerm); + + context = new IndexShardContext(mockIndexShard); + + assertEquals(expectedPrimaryTerm, context.getPrimaryTerm()); + assertTrue(context.isAvailable()); + } + + public void testGetPrimaryTermWithNullIndexShard() { + context = new IndexShardContext(null); + + assertEquals(IndexShardContext.DEFAULT_PRIMARY_TERM, context.getPrimaryTerm()); + assertFalse(context.isAvailable()); + } + + public void testGetPrimaryTermWithExceptionFromIndexShard() { + when(mockIndexShard.getOperationPrimaryTerm()).thenThrow(new RuntimeException("Test exception")); + + context = new IndexShardContext(mockIndexShard); + + assertEquals(IndexShardContext.DEFAULT_PRIMARY_TERM, context.getPrimaryTerm()); + assertTrue(context.isAvailable()); // IndexShard is available, but throws exception + } + + public void testPrimaryTermCaching() { + long primaryTerm1 = 3L; + long primaryTerm2 = 4L; + + when(mockIndexShard.getOperationPrimaryTerm()).thenReturn(primaryTerm1); + + context = new IndexShardContext(mockIndexShard); + + // First call should fetch from IndexShard + assertEquals(primaryTerm1, context.getPrimaryTerm()); + + // Second call within cache duration should return cached value + assertEquals(primaryTerm1, context.getPrimaryTerm()); + + // Change the mock to return different value + when(mockIndexShard.getOperationPrimaryTerm()).thenReturn(primaryTerm2); + + // Should still return cached value + assertEquals(primaryTerm1, context.getPrimaryTerm()); + + // Invalidate cache and verify new value is fetched + context.invalidateCache(); + assertEquals(primaryTerm2, context.getPrimaryTerm()); + } + + public void testCacheExpiration() throws InterruptedException { + long primaryTerm1 = 7L; + long primaryTerm2 = 8L; + + when(mockIndexShard.getOperationPrimaryTerm()).thenReturn(primaryTerm1); + + context = new IndexShardContext(mockIndexShard); + + // First call + assertEquals(primaryTerm1, context.getPrimaryTerm()); + assertTrue(context.isCacheValid()); + + // Change mock return value + when(mockIndexShard.getOperationPrimaryTerm()).thenReturn(primaryTerm2); + + // Wait for cache to expire (cache duration is 1 second) + Thread.sleep(1100); + + // Should fetch new value after cache expiration + assertEquals(primaryTerm2, context.getPrimaryTerm()); + } + + public void testInvalidateCache() { + long primaryTerm = 10L; + when(mockIndexShard.getOperationPrimaryTerm()).thenReturn(primaryTerm); + + context = new IndexShardContext(mockIndexShard); + + // Populate cache + assertEquals(primaryTerm, context.getPrimaryTerm()); + assertTrue(context.isCacheValid()); + + // Invalidate cache + context.invalidateCache(); + assertFalse(context.isCacheValid()); + assertEquals(-1L, context.getCachedPrimaryTerm()); + } + + public void testGetIndexShard() { + context = new IndexShardContext(mockIndexShard); + assertEquals(mockIndexShard, context.getIndexShard()); + + context = new IndexShardContext(null); + assertNull(context.getIndexShard()); + } +} \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/PrimaryTermAwareDirectoryWrapperTests.java b/server/src/test/java/org/opensearch/index/store/distributed/PrimaryTermAwareDirectoryWrapperTests.java new file mode 100644 index 0000000000000..89b6536dc3089 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/store/distributed/PrimaryTermAwareDirectoryWrapperTests.java @@ -0,0 +1,274 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; +import org.opensearch.index.shard.IndexShard; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Arrays; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class PrimaryTermAwareDirectoryWrapperTests extends OpenSearchTestCase { + + private Path tempDir; + private Directory baseDirectory; + private IndexShard mockIndexShard; + private PrimaryTermAwareDirectoryWrapper wrapper; + + @Before + public void setUp() throws Exception { + super.setUp(); + tempDir = createTempDir(); + baseDirectory = FSDirectory.open(tempDir); + mockIndexShard = mock(IndexShard.class); + + wrapper = new PrimaryTermAwareDirectoryWrapper(baseDirectory, tempDir); + } + + @After + public void tearDown() throws Exception { + if (wrapper != null) { + wrapper.close(); + } + super.tearDown(); + } + + public void testInitialState() { + assertFalse(wrapper.isPrimaryTermRoutingEnabled()); + assertEquals(tempDir, wrapper.getBasePath()); + + String routingInfo = wrapper.getRoutingInfo("test.txt"); + assertTrue(routingInfo.contains("Primary term routing not enabled")); + } + + public void testEnablePrimaryTermRouting() throws IOException { + when(mockIndexShard.getOperationPrimaryTerm()).thenReturn(1L); + + assertFalse(wrapper.isPrimaryTermRoutingEnabled()); + + wrapper.enablePrimaryTermRouting(mockIndexShard); + + assertTrue(wrapper.isPrimaryTermRoutingEnabled()); + } + + public void testEnablePrimaryTermRoutingTwice() throws IOException { + when(mockIndexShard.getOperationPrimaryTerm()).thenReturn(1L); + + wrapper.enablePrimaryTermRouting(mockIndexShard); + assertTrue(wrapper.isPrimaryTermRoutingEnabled()); + + // Enabling again should not cause issues + wrapper.enablePrimaryTermRouting(mockIndexShard); + assertTrue(wrapper.isPrimaryTermRoutingEnabled()); + } + + public void testFileOperationsBeforePrimaryTermRouting() throws IOException { + String filename = "test.txt"; + String content = "test content"; + + // Operations should work with base directory before primary term routing is enabled + try (IndexOutput output = wrapper.createOutput(filename, IOContext.DEFAULT)) { + output.writeString(content); + } + + assertTrue(Arrays.asList(wrapper.listAll()).contains(filename)); + assertEquals(content.getBytes().length + 4, wrapper.fileLength(filename)); // +4 for string length prefix + } + + public void testFileOperationsAfterPrimaryTermRouting() throws IOException { + when(mockIndexShard.getOperationPrimaryTerm()).thenReturn(2L); + + // Enable primary term routing + wrapper.enablePrimaryTermRouting(mockIndexShard); + + String filename = "_0.si"; + String content = "primary term content"; + + // Operations should work with primary term routing + try (IndexOutput output = wrapper.createOutput(filename, IOContext.DEFAULT)) { + output.writeString(content); + } + + assertTrue(Arrays.asList(wrapper.listAll()).contains(filename)); + + String routingInfo = wrapper.getRoutingInfo(filename); + assertTrue(routingInfo.contains("primary term")); + } + + public void testSegmentsFileRouting() throws IOException { + when(mockIndexShard.getOperationPrimaryTerm()).thenReturn(3L); + + wrapper.enablePrimaryTermRouting(mockIndexShard); + + String segmentsFile = "segments_1"; + String content = "segments content"; + + try (IndexOutput output = wrapper.createOutput(segmentsFile, IOContext.DEFAULT)) { + output.writeString(content); + } + + String routingInfo = wrapper.getRoutingInfo(segmentsFile); + assertTrue(routingInfo.contains("excluded")); + assertTrue(routingInfo.contains("base directory")); + } + + public void testFileDeletion() throws IOException { + when(mockIndexShard.getOperationPrimaryTerm()).thenReturn(1L); + + String filename = "_0.cfs"; + String content = "deletion test"; + + // Create file before enabling primary term routing + try (IndexOutput output = wrapper.createOutput(filename, IOContext.DEFAULT)) { + output.writeString(content); + } + + assertTrue(Arrays.asList(wrapper.listAll()).contains(filename)); + + // Enable primary term routing + wrapper.enablePrimaryTermRouting(mockIndexShard); + + // Delete should work + wrapper.deleteFile(filename); + assertFalse(Arrays.asList(wrapper.listAll()).contains(filename)); + } + + public void testSyncOperation() throws IOException { + when(mockIndexShard.getOperationPrimaryTerm()).thenReturn(1L); + + String[] filenames = {"_0.si", "_1.cfs"}; + + // Create files + for (String filename : filenames) { + try (IndexOutput output = wrapper.createOutput(filename, IOContext.DEFAULT)) { + output.writeString("sync test"); + } + } + + // Enable primary term routing + wrapper.enablePrimaryTermRouting(mockIndexShard); + + // Sync should work + try { + wrapper.sync(Arrays.asList(filenames)); + } catch (IOException e) { + fail("sync should not throw exception: " + e.getMessage()); + } + } + + public void testRenameOperation() throws IOException { + when(mockIndexShard.getOperationPrimaryTerm()).thenReturn(1L); + + String sourceFile = "_0.si"; + String destFile = "_0_renamed.si"; + String content = "rename test"; + + // Create source file + try (IndexOutput output = wrapper.createOutput(sourceFile, IOContext.DEFAULT)) { + output.writeString(content); + } + + // Enable primary term routing + wrapper.enablePrimaryTermRouting(mockIndexShard); + + // Rename should work + wrapper.rename(sourceFile, destFile); + + assertFalse(Arrays.asList(wrapper.listAll()).contains(sourceFile)); + assertTrue(Arrays.asList(wrapper.listAll()).contains(destFile)); + } + + public void testMixedOperationsBeforeAndAfterEnabling() throws IOException { + when(mockIndexShard.getOperationPrimaryTerm()).thenReturn(1L); + + String file1 = "before.txt"; + String file2 = "_0.si"; + String content1 = "before enabling"; + String content2 = "after enabling"; + + // Create file before enabling + try (IndexOutput output = wrapper.createOutput(file1, IOContext.DEFAULT)) { + output.writeString(content1); + } + + // Enable primary term routing + wrapper.enablePrimaryTermRouting(mockIndexShard); + + // Create file after enabling + try (IndexOutput output = wrapper.createOutput(file2, IOContext.DEFAULT)) { + output.writeString(content2); + } + + // Both files should be accessible + String[] allFiles = wrapper.listAll(); + assertTrue(Arrays.asList(allFiles).contains(file1)); + assertTrue(Arrays.asList(allFiles).contains(file2)); + + // Verify routing info + String routingInfo2 = wrapper.getRoutingInfo(file2); + + // Should show primary term routing (since it's enabled) + assertTrue(routingInfo2.contains("primary term")); + } + + public void testCloseWithoutEnabling() throws IOException { + String filename = "test.txt"; + + try (IndexOutput output = wrapper.createOutput(filename, IOContext.DEFAULT)) { + output.writeString("test"); + } + + // Should close without issues + try { + wrapper.close(); + } catch (IOException e) { + fail("close should not throw exception: " + e.getMessage()); + } + } + + public void testCloseAfterEnabling() throws IOException { + when(mockIndexShard.getOperationPrimaryTerm()).thenReturn(1L); + + wrapper.enablePrimaryTermRouting(mockIndexShard); + + String filename = "_0.si"; + try (IndexOutput output = wrapper.createOutput(filename, IOContext.DEFAULT)) { + output.writeString("test"); + } + + // Should close without issues + try { + wrapper.close(); + } catch (IOException e) { + fail("close should not throw exception: " + e.getMessage()); + } + } + + public void testEnablePrimaryTermRoutingWithException() throws IOException { + // Mock IndexShard to throw exception + when(mockIndexShard.getOperationPrimaryTerm()).thenThrow(new RuntimeException("Mock exception")); + + // Should handle exception gracefully and not enable routing + expectThrows(IOException.class, () -> { + wrapper.enablePrimaryTermRouting(mockIndexShard); + }); + + assertFalse(wrapper.isPrimaryTermRoutingEnabled()); + } +} \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/PrimaryTermDirectoryManagerTests.java b/server/src/test/java/org/opensearch/index/store/distributed/PrimaryTermDirectoryManagerTests.java new file mode 100644 index 0000000000000..ffe38b3dd6a07 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/store/distributed/PrimaryTermDirectoryManagerTests.java @@ -0,0 +1,260 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Set; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class PrimaryTermDirectoryManagerTests extends OpenSearchTestCase { + + private Path tempDir; + private Directory mockBaseDirectory; + private PrimaryTermDirectoryManager manager; + + @Before + public void setUp() throws Exception { + super.setUp(); + tempDir = createTempDir(); + mockBaseDirectory = mock(Directory.class); + manager = new PrimaryTermDirectoryManager(mockBaseDirectory, tempDir); + } + + @After + public void tearDown() throws Exception { + if (manager != null && !manager.isClosed()) { + manager.close(); + } + super.tearDown(); + } + + public void testGetBaseDirectory() { + assertEquals(mockBaseDirectory, manager.getBaseDirectory()); + } + + public void testGetBasePath() { + assertEquals(tempDir, manager.getBasePath()); + } + + public void testGetDirectoryNameForPrimaryTerm() { + assertEquals("primary_term_0", manager.getDirectoryNameForPrimaryTerm(0L)); + assertEquals("primary_term_5", manager.getDirectoryNameForPrimaryTerm(5L)); + assertEquals("primary_term_100", manager.getDirectoryNameForPrimaryTerm(100L)); + } + + public void testCreateDirectoryForPrimaryTerm() throws IOException { + long primaryTerm = 1L; + + assertFalse(manager.hasDirectoryForPrimaryTerm(primaryTerm)); + assertEquals(0, manager.getDirectoryCount()); + + Directory directory = manager.getDirectoryForPrimaryTerm(primaryTerm); + + assertNotNull(directory); + assertTrue(manager.hasDirectoryForPrimaryTerm(primaryTerm)); + assertEquals(1, manager.getDirectoryCount()); + + // Verify directory was created on filesystem + Path expectedPath = tempDir.resolve("primary_term_1"); + assertTrue(Files.exists(expectedPath)); + assertTrue(Files.isDirectory(expectedPath)); + } + + public void testGetDirectoryForPrimaryTermReturnsExisting() throws IOException { + long primaryTerm = 2L; + + Directory directory1 = manager.getDirectoryForPrimaryTerm(primaryTerm); + Directory directory2 = manager.getDirectoryForPrimaryTerm(primaryTerm); + + assertSame("Should return same directory instance", directory1, directory2); + assertEquals(1, manager.getDirectoryCount()); + } + + public void testMultiplePrimaryTermDirectories() throws IOException { + long[] primaryTerms = {1L, 3L, 5L}; + + for (long primaryTerm : primaryTerms) { + Directory directory = manager.getDirectoryForPrimaryTerm(primaryTerm); + assertNotNull(directory); + assertTrue(manager.hasDirectoryForPrimaryTerm(primaryTerm)); + } + + assertEquals(primaryTerms.length, manager.getDirectoryCount()); + + Set allPrimaryTerms = manager.getAllPrimaryTerms(); + assertEquals(primaryTerms.length, allPrimaryTerms.size()); + for (long primaryTerm : primaryTerms) { + assertTrue(allPrimaryTerms.contains(primaryTerm)); + } + } + + public void testListAllDirectories() throws IOException { + long[] primaryTerms = {1L, 2L, 3L}; + + for (long primaryTerm : primaryTerms) { + manager.getDirectoryForPrimaryTerm(primaryTerm); + } + + var directories = manager.listAllDirectories(); + assertEquals(primaryTerms.length, directories.size()); + + // All directories should be different instances + for (int i = 0; i < directories.size(); i++) { + for (int j = i + 1; j < directories.size(); j++) { + assertNotSame(directories.get(i), directories.get(j)); + } + } + } + + public void testValidateDirectories() throws IOException { + // Create some directories + manager.getDirectoryForPrimaryTerm(1L); + manager.getDirectoryForPrimaryTerm(2L); + + // Should not throw exception for valid directories + try { + manager.validateDirectories(); + } catch (IOException e) { + fail("validateDirectories should not throw exception for valid directories: " + e.getMessage()); + } + } + + public void testValidatePrimaryTermDirectory() throws IOException { + long primaryTerm = 1L; + manager.getDirectoryForPrimaryTerm(primaryTerm); + + // Should not throw exception for valid directory + try { + manager.validatePrimaryTermDirectory(primaryTerm); + } catch (IOException e) { + fail("validatePrimaryTermDirectory should not throw exception for valid directory: " + e.getMessage()); + } + } + + public void testValidatePrimaryTermDirectoryNotFound() { + long primaryTerm = 999L; + + PrimaryTermRoutingException exception = expectThrows( + PrimaryTermRoutingException.class, + () -> manager.validatePrimaryTermDirectory(primaryTerm) + ); + + assertEquals(PrimaryTermRoutingException.ErrorType.DIRECTORY_VALIDATION_ERROR, exception.getErrorType()); + assertEquals(primaryTerm, exception.getPrimaryTerm()); + } + + public void testGetDirectoryStats() throws IOException { + // Initially no directories + var stats = manager.getDirectoryStats(); + assertEquals(0, stats.getTotalDirectories()); + assertEquals(-1L, stats.getMinPrimaryTerm()); + assertEquals(-1L, stats.getMaxPrimaryTerm()); + assertEquals(tempDir.toString(), stats.getBasePath()); + + // Add some directories + manager.getDirectoryForPrimaryTerm(3L); + manager.getDirectoryForPrimaryTerm(1L); + manager.getDirectoryForPrimaryTerm(7L); + + stats = manager.getDirectoryStats(); + assertEquals(3, stats.getTotalDirectories()); + assertEquals(1L, stats.getMinPrimaryTerm()); + assertEquals(7L, stats.getMaxPrimaryTerm()); + } + + public void testCleanupUnusedDirectories() throws IOException { + // Create directories for primary terms 1, 2, 3 + manager.getDirectoryForPrimaryTerm(1L); + manager.getDirectoryForPrimaryTerm(2L); + manager.getDirectoryForPrimaryTerm(3L); + + assertEquals(3, manager.getDirectoryCount()); + + // Keep only primary terms 1 and 3 + Set activePrimaryTerms = Set.of(1L, 3L); + int cleanedUp = manager.cleanupUnusedDirectories(activePrimaryTerms); + + assertEquals(1, cleanedUp); // Should have cleaned up primary term 2 + assertEquals(2, manager.getDirectoryCount()); + assertTrue(manager.hasDirectoryForPrimaryTerm(1L)); + assertFalse(manager.hasDirectoryForPrimaryTerm(2L)); + assertTrue(manager.hasDirectoryForPrimaryTerm(3L)); + } + + public void testHasAvailableSpace() { + // Should return true for reasonable space requirements + assertTrue(manager.hasAvailableSpace(1024L)); // 1KB + assertTrue(manager.hasAvailableSpace(1024L * 1024L)); // 1MB + + // Should return false for unreasonably large space requirements + assertFalse(manager.hasAvailableSpace(Long.MAX_VALUE)); + } + + public void testSyncAllDirectories() throws IOException { + // Create some directories + manager.getDirectoryForPrimaryTerm(1L); + manager.getDirectoryForPrimaryTerm(2L); + + // Should not throw exception + try { + manager.syncAllDirectories(); + } catch (IOException e) { + fail("syncAllDirectories should not throw exception: " + e.getMessage()); + } + } + + public void testClose() throws IOException { + // Create some directories + manager.getDirectoryForPrimaryTerm(1L); + manager.getDirectoryForPrimaryTerm(2L); + + assertFalse(manager.isClosed()); + assertEquals(2, manager.getDirectoryCount()); + + manager.close(); + + assertTrue(manager.isClosed()); + assertEquals(0, manager.getDirectoryCount()); + } + + public void testOperationsAfterClose() throws IOException { + manager.close(); + + IllegalStateException exception = expectThrows( + IllegalStateException.class, + () -> manager.getDirectoryForPrimaryTerm(1L) + ); + + assertTrue(exception.getMessage().contains("has been closed")); + } + + public void testConcurrentDirectoryCreation() throws IOException { + long primaryTerm = 1L; + + // Simulate concurrent access by calling getDirectoryForPrimaryTerm multiple times + // This tests the double-check locking pattern + Directory dir1 = manager.getDirectoryForPrimaryTerm(primaryTerm); + Directory dir2 = manager.getDirectoryForPrimaryTerm(primaryTerm); + Directory dir3 = manager.getDirectoryForPrimaryTerm(primaryTerm); + + assertSame(dir1, dir2); + assertSame(dir2, dir3); + assertEquals(1, manager.getDirectoryCount()); + } +} \ No newline at end of file diff --git a/server/src/test/java/org/opensearch/index/store/distributed/PrimaryTermRouterTests.java b/server/src/test/java/org/opensearch/index/store/distributed/PrimaryTermRouterTests.java new file mode 100644 index 0000000000000..06d442b1533d1 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/store/distributed/PrimaryTermRouterTests.java @@ -0,0 +1,189 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store.distributed; + +import org.apache.lucene.store.Directory; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import java.io.IOException; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.verify; + +public class PrimaryTermRouterTests extends OpenSearchTestCase { + + private IndexShardContext mockShardContext; + private PrimaryTermDirectoryManager mockDirectoryManager; + private Directory mockBaseDirectory; + private Directory mockPrimaryTermDirectory; + private PrimaryTermRouter router; + + @Before + public void setUp() throws Exception { + super.setUp(); + mockShardContext = mock(IndexShardContext.class); + mockDirectoryManager = mock(PrimaryTermDirectoryManager.class); + mockBaseDirectory = mock(Directory.class); + mockPrimaryTermDirectory = mock(Directory.class); + + when(mockDirectoryManager.getBaseDirectory()).thenReturn(mockBaseDirectory); + + router = new PrimaryTermRouter(mockShardContext); + } + + public void testGetDirectoryForExcludedFile() throws IOException { + String[] excludedFiles = { + "segments_1", + "pending_segments_1", + "write.lock", + "test.tmp" + }; + + for (String filename : excludedFiles) { + Directory result = router.getDirectoryForFile(filename, mockDirectoryManager); + assertEquals("File " + filename + " should use base directory", mockBaseDirectory, result); + } + } + + public void testGetDirectoryForRegularFile() throws IOException { + String filename = "_0.si"; + long primaryTerm = 5L; + + when(mockShardContext.isAvailable()).thenReturn(true); + when(mockShardContext.getPrimaryTerm()).thenReturn(primaryTerm); + when(mockDirectoryManager.getDirectoryForPrimaryTerm(primaryTerm)).thenReturn(mockPrimaryTermDirectory); + + Directory result = router.getDirectoryForFile(filename, mockDirectoryManager); + + assertEquals(mockPrimaryTermDirectory, result); + verify(mockDirectoryManager).getDirectoryForPrimaryTerm(primaryTerm); + } + + public void testGetDirectoryForFileWithUnavailableContext() throws IOException { + String filename = "_0.cfs"; + + when(mockShardContext.isAvailable()).thenReturn(false); + when(mockDirectoryManager.getDirectoryForPrimaryTerm(IndexShardContext.DEFAULT_PRIMARY_TERM)) + .thenReturn(mockBaseDirectory); + + Directory result = router.getDirectoryForFile(filename, mockDirectoryManager); + + assertEquals(mockBaseDirectory, result); + verify(mockDirectoryManager).getDirectoryForPrimaryTerm(IndexShardContext.DEFAULT_PRIMARY_TERM); + } + + public void testGetDirectoryForFileWithNullContext() throws IOException { + router = new PrimaryTermRouter(null); + String filename = "_0.cfe"; + + when(mockDirectoryManager.getDirectoryForPrimaryTerm(IndexShardContext.DEFAULT_PRIMARY_TERM)) + .thenReturn(mockBaseDirectory); + + Directory result = router.getDirectoryForFile(filename, mockDirectoryManager); + + assertEquals(mockBaseDirectory, result); + } + + public void testGetDirectoryForFileWithDirectoryManagerException() throws IOException { + String filename = "_1.si"; + long primaryTerm = 3L; + + when(mockShardContext.isAvailable()).thenReturn(true); + when(mockShardContext.getPrimaryTerm()).thenReturn(primaryTerm); + when(mockDirectoryManager.getDirectoryForPrimaryTerm(primaryTerm)) + .thenThrow(new IOException("Directory creation failed")); + + Directory result = router.getDirectoryForFile(filename, mockDirectoryManager); + + // Should fall back to base directory on exception + assertEquals(mockBaseDirectory, result); + } + + public void testIsExcludedFile() { + // Test excluded prefixes + assertTrue(router.isExcludedFile("segments_1")); + assertTrue(router.isExcludedFile("pending_segments_2")); + assertTrue(router.isExcludedFile("write.lock")); + + // Test temporary files + assertTrue(router.isExcludedFile("test.tmp")); + assertTrue(router.isExcludedFile("_0.si.tmp")); + + // Test null and empty + assertTrue(router.isExcludedFile(null)); + assertTrue(router.isExcludedFile("")); + + // Test regular files + assertFalse(router.isExcludedFile("_0.si")); + assertFalse(router.isExcludedFile("_0.cfs")); + assertFalse(router.isExcludedFile("_0.cfe")); + assertFalse(router.isExcludedFile("_1.doc")); + } + + public void testGetCurrentPrimaryTerm() { + long expectedPrimaryTerm = 7L; + + // Test with available context + when(mockShardContext.isAvailable()).thenReturn(true); + when(mockShardContext.getPrimaryTerm()).thenReturn(expectedPrimaryTerm); + + assertEquals(expectedPrimaryTerm, router.getCurrentPrimaryTerm()); + + // Test with unavailable context + when(mockShardContext.isAvailable()).thenReturn(false); + + assertEquals(IndexShardContext.DEFAULT_PRIMARY_TERM, router.getCurrentPrimaryTerm()); + } + + public void testGetCurrentPrimaryTermWithNullContext() { + router = new PrimaryTermRouter(null); + + assertEquals(IndexShardContext.DEFAULT_PRIMARY_TERM, router.getCurrentPrimaryTerm()); + } + + public void testGetDirectoryNameForPrimaryTerm() { + assertEquals("primary_term_0", router.getDirectoryNameForPrimaryTerm(0L)); + assertEquals("primary_term_5", router.getDirectoryNameForPrimaryTerm(5L)); + assertEquals("primary_term_100", router.getDirectoryNameForPrimaryTerm(100L)); + } + + public void testGetDirectoryForFileWithNullFilename() { + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { + router.getDirectoryForFile(null, mockDirectoryManager); + }); + + assertTrue(exception.getMessage().contains("Filename cannot be null or empty")); + } + + public void testGetDirectoryForFileWithEmptyFilename() { + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { + router.getDirectoryForFile("", mockDirectoryManager); + }); + + assertTrue(exception.getMessage().contains("Filename cannot be null or empty")); + } + + public void testGetExcludedPrefixes() { + var excludedPrefixes = router.getExcludedPrefixes(); + + assertTrue(excludedPrefixes.contains("segments_")); + assertTrue(excludedPrefixes.contains("pending_segments_")); + assertTrue(excludedPrefixes.contains("write.lock")); + assertEquals(3, excludedPrefixes.size()); + } + + public void testGetShardContext() { + assertEquals(mockShardContext, router.getShardContext()); + + router = new PrimaryTermRouter(null); + assertNull(router.getShardContext()); + } +} \ No newline at end of file