diff --git a/docs/changelog/128139.yaml b/docs/changelog/128139.yaml new file mode 100644 index 0000000000000..b21e2c744f4d9 --- /dev/null +++ b/docs/changelog/128139.yaml @@ -0,0 +1,5 @@ +pr: 128139 +summary: Skip indexing points for `seq_no` in tsdb and logsdb +area: Mapping +type: enhancement +issues: [] diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java index 905f0b350798e..7b833e92f6b2c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/diskusage/IndexDiskUsageAnalyzerIT.java @@ -36,6 +36,7 @@ import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.Random; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; @@ -61,6 +62,13 @@ protected Collection> nodePlugins() { return plugins; } + @Override + protected Settings.Builder setRandomIndexSettings(Random random, Settings.Builder builder) { + var b = super.setRandomIndexSettings(random, builder); + b.remove(IndexSettings.SEQ_NO_INDEX_OPTIONS_SETTING.getKey()); + return b; + } + private static final Set failOnFlushShards = ConcurrentCollections.newConcurrentSet(); public static class EngineTestPlugin extends Plugin implements EnginePlugin { diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index a4b239c10ba6a..0ff64f14dc17c 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -224,6 +224,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexMetadata.INDEX_ROUTING_PATH, IndexSettings.TIME_SERIES_START_TIME, IndexSettings.TIME_SERIES_END_TIME, + IndexSettings.SEQ_NO_INDEX_OPTIONS_SETTING, // Legacy index settings we must keep around for BWC from 7.x EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS, diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 7a3b2ad938d1b..4b89ab2e60021 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -28,6 +28,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.index.translog.Translog; @@ -829,6 +830,23 @@ private static String getIgnoreAboveDefaultValue(final Settings settings) { } } + public static final Setting SEQ_NO_INDEX_OPTIONS_SETTING = Setting.enumSetting( + SeqNoFieldMapper.SeqNoIndexOptions.class, + settings -> { + final IndexMode indexMode = IndexSettings.MODE.get(settings); + if ((indexMode == IndexMode.LOGSDB || indexMode == IndexMode.TIME_SERIES) + && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.SEQ_NO_WITHOUT_POINTS)) { + return SeqNoFieldMapper.SeqNoIndexOptions.DOC_VALUES_ONLY.toString(); + } else { + return SeqNoFieldMapper.SeqNoIndexOptions.POINTS_AND_DOC_VALUES.toString(); + } + }, + "index.seq_no.index_options", + value -> {}, + Property.IndexScope, + Property.Final + ); + private final Index index; private final IndexVersion version; private final Logger logger; @@ -933,6 +951,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private volatile int maxRegexLength; private final IndexRouting indexRouting; + private final SeqNoFieldMapper.SeqNoIndexOptions seqNoIndexOptions; /** * The default mode for storing source, for all mappers not overriding this setting. @@ -1099,6 +1118,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti recoverySourceSyntheticEnabled = DiscoveryNode.isStateless(nodeSettings) == false && scopedSettings.get(RECOVERY_USE_SYNTHETIC_SOURCE_SETTING); useDocValuesSkipper = DOC_VALUES_SKIPPER && scopedSettings.get(USE_DOC_VALUES_SKIPPER); + seqNoIndexOptions = scopedSettings.get(SEQ_NO_INDEX_OPTIONS_SETTING); if (recoverySourceSyntheticEnabled) { if (DiscoveryNode.isStateless(settings)) { throw new IllegalArgumentException("synthetic recovery source is only allowed in stateful"); @@ -1837,4 +1857,8 @@ public DenseVectorFieldMapper.FilterHeuristic getHnswFilterHeuristic() { private void setHnswFilterHeuristic(DenseVectorFieldMapper.FilterHeuristic heuristic) { this.hnswFilterHeuristic = heuristic; } + + public SeqNoFieldMapper.SeqNoIndexOptions seqNoIndexOptions() { + return seqNoIndexOptions; + } } diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 74eb6721dda81..24129a50ac344 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -168,6 +168,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion DEFAULT_OVERSAMPLE_VALUE_FOR_BBQ = def(9_024_0_00, Version.LUCENE_10_2_1); public static final IndexVersion SEMANTIC_TEXT_DEFAULTS_TO_BBQ = def(9_025_0_00, Version.LUCENE_10_2_1); public static final IndexVersion DEFAULT_TO_ACORN_HNSW_FILTER_HEURISTIC = def(9_026_0_00, Version.LUCENE_10_2_1); + public static final IndexVersion SEQ_NO_WITHOUT_POINTS = def(9_027_0_00, Version.LUCENE_10_2_1); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index bd1c5c26bb450..f48318e519e74 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -10,7 +10,6 @@ package org.elasticsearch.index.engine; import org.apache.logging.log4j.Logger; -import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexCommit; @@ -1811,7 +1810,10 @@ private DeletionStrategy planDeletionAsPrimary(Delete delete) throws IOException private DeleteResult deleteInLucene(Delete delete, DeletionStrategy plan) throws IOException { assert assertMaxSeqNoOfUpdatesIsAdvanced(delete.uid(), delete.seqNo(), false, false); try { - final ParsedDocument tombstone = ParsedDocument.deleteTombstone(delete.id()); + final ParsedDocument tombstone = ParsedDocument.deleteTombstone( + engineConfig.getIndexSettings().seqNoIndexOptions(), + delete.id() + ); assert tombstone.docs().size() == 1 : "Tombstone doc should have single doc [" + tombstone + "]"; tombstone.updateSeqID(delete.seqNo(), delete.primaryTerm()); tombstone.version().setLongValue(plan.versionOfDeletion); @@ -1970,7 +1972,10 @@ private NoOpResult innerNoOp(final NoOp noOp) throws IOException { markSeqNoAsSeen(noOp.seqNo()); if (hasBeenProcessedBefore(noOp) == false) { try { - final ParsedDocument tombstone = ParsedDocument.noopTombstone(noOp.reason()); + final ParsedDocument tombstone = ParsedDocument.noopTombstone( + engineConfig.getIndexSettings().seqNoIndexOptions(), + noOp.reason() + ); tombstone.updateSeqID(noOp.seqNo(), noOp.primaryTerm()); // A noop tombstone does not require a _version but it's added to have a fully dense docvalues for the version // field. 1L is selected to optimize the compression because it might probably be the most common value in @@ -2753,10 +2758,10 @@ private IndexWriterConfig getIndexWriterConfig() { ? SourceFieldMapper.RECOVERY_SOURCE_SIZE_NAME : SourceFieldMapper.RECOVERY_SOURCE_NAME, engineConfig.getIndexSettings().getMode() == IndexMode.TIME_SERIES, - softDeletesPolicy::getRetentionQuery, + () -> softDeletesPolicy.getRetentionQuery(engineConfig.getIndexSettings().seqNoIndexOptions()), new SoftDeletesRetentionMergePolicy( Lucene.SOFT_DELETES_FIELD, - softDeletesPolicy::getRetentionQuery, + () -> softDeletesPolicy.getRetentionQuery(engineConfig.getIndexSettings().seqNoIndexOptions()), new PrunePostingsMergePolicy(mergePolicy, IdFieldMapper.NAME) ) ); @@ -3215,12 +3220,7 @@ public int countChanges(String source, long fromSeqNo, long toSeqNo) throws IOEx ensureOpen(); refreshIfNeeded(source, toSeqNo); try (Searcher searcher = acquireSearcher(source, SearcherScope.INTERNAL)) { - return LuceneChangesSnapshot.countOperations( - searcher, - fromSeqNo, - toSeqNo, - config().getIndexSettings().getIndexVersionCreated() - ); + return LuceneChangesSnapshot.countOperations(searcher, engineConfig.getIndexSettings(), fromSeqNo, toSeqNo); } catch (Exception e) { try { maybeFailEngine("count changes", e); @@ -3448,7 +3448,11 @@ private void restoreVersionMapAndCheckpointTracker(DirectoryReader directoryRead final IndexSearcher searcher = new IndexSearcher(directoryReader); searcher.setQueryCache(null); final Query query = new BooleanQuery.Builder().add( - LongPoint.newRangeQuery(SeqNoFieldMapper.NAME, getPersistedLocalCheckpoint() + 1, Long.MAX_VALUE), + SeqNoFieldMapper.rangeQueryForSeqNo( + engineConfig.getIndexSettings().seqNoIndexOptions(), + getPersistedLocalCheckpoint() + 1, + Long.MAX_VALUE + ), BooleanClause.Occur.MUST ) .add(Queries.newNonNestedFilter(indexVersionCreated), BooleanClause.Occur.MUST) // exclude non-root nested documents diff --git a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java index 30c6c639b9cf7..5c50bbeac796b 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java +++ b/server/src/main/java/org/elasticsearch/index/engine/LuceneChangesSnapshot.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader; import org.elasticsearch.core.Assertions; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fieldvisitor.FieldsVisitor; import org.elasticsearch.index.mapper.MapperService; @@ -187,12 +188,12 @@ private static boolean hasSequentialAccess(ScoreDoc[] scoreDocs) { return true; } - static int countOperations(Engine.Searcher engineSearcher, long fromSeqNo, long toSeqNo, IndexVersion indexVersionCreated) + static int countOperations(Engine.Searcher engineSearcher, IndexSettings indexSettings, long fromSeqNo, long toSeqNo) throws IOException { if (fromSeqNo < 0 || toSeqNo < 0 || fromSeqNo > toSeqNo) { throw new IllegalArgumentException("Invalid range; from_seqno [" + fromSeqNo + "], to_seqno [" + toSeqNo + "]"); } - return newIndexSearcher(engineSearcher).count(rangeQuery(fromSeqNo, toSeqNo, indexVersionCreated)); + return newIndexSearcher(engineSearcher).count(rangeQuery(indexSettings, fromSeqNo, toSeqNo)); } private Translog.Operation readDocAsOp(int docIndex) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/index/engine/SearchBasedChangesSnapshot.java b/server/src/main/java/org/elasticsearch/index/engine/SearchBasedChangesSnapshot.java index 17f31bcb7b847..ac97ff5812300 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/SearchBasedChangesSnapshot.java +++ b/server/src/main/java/org/elasticsearch/index/engine/SearchBasedChangesSnapshot.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.engine; -import org.apache.lucene.document.LongPoint; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; @@ -26,6 +25,7 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; import org.elasticsearch.index.mapper.MapperService; @@ -47,7 +47,7 @@ public abstract class SearchBasedChangesSnapshot implements Translog.Snapshot, Closeable { public static final int DEFAULT_BATCH_SIZE = 1024; - private final IndexVersion indexVersionCreated; + private final IndexSettings indexSettings; private final IndexSearcher indexSearcher; private final ValueFetcher sourceMetadataFetcher; private final Closeable onClose; @@ -97,7 +97,7 @@ protected SearchBasedChangesSnapshot( } }; - this.indexVersionCreated = indexVersionCreated; + this.indexSettings = mapperService.getIndexSettings(); this.fromSeqNo = fromSeqNo; this.toSeqNo = toSeqNo; this.lastSeenSeqNo = fromSeqNo - 1; @@ -109,7 +109,7 @@ protected SearchBasedChangesSnapshot( this.searchBatchSize = (int) Math.min(requestingSize, searchBatchSize); this.accessStats = accessStats; - this.totalHits = accessStats ? indexSearcher.count(rangeQuery(fromSeqNo, toSeqNo, indexVersionCreated)) : -1; + this.totalHits = accessStats ? indexSearcher.count(rangeQuery(indexSettings, fromSeqNo, toSeqNo)) : -1; this.sourceMetadataFetcher = createSourceMetadataValueFetcher(mapperService, indexSearcher); } @@ -183,7 +183,7 @@ public void close() throws IOException { * @return TopDocs instance containing the documents in the current batch. */ protected TopDocs nextTopDocs() throws IOException { - Query rangeQuery = rangeQuery(Math.max(fromSeqNo, lastSeenSeqNo), toSeqNo, indexVersionCreated); + Query rangeQuery = rangeQuery(indexSettings, Math.max(fromSeqNo, lastSeenSeqNo), toSeqNo); SortField sortBySeqNo = new SortField(SeqNoFieldMapper.NAME, SortField.Type.LONG); TopFieldCollectorManager collectorManager = new TopFieldCollectorManager(new Sort(sortBySeqNo), searchBatchSize, afterDoc, 0); @@ -241,9 +241,10 @@ static IndexSearcher newIndexSearcher(Engine.Searcher engineSearcher) throws IOE return new IndexSearcher(Lucene.wrapAllDocsLive(engineSearcher.getDirectoryReader())); } - static Query rangeQuery(long fromSeqNo, long toSeqNo, IndexVersion indexVersionCreated) { - return new BooleanQuery.Builder().add(LongPoint.newRangeQuery(SeqNoFieldMapper.NAME, fromSeqNo, toSeqNo), BooleanClause.Occur.MUST) - .add(Queries.newNonNestedFilter(indexVersionCreated), BooleanClause.Occur.MUST) + static Query rangeQuery(IndexSettings indexSettings, long fromSeqNo, long toSeqNo) { + Query seqNoQuery = SeqNoFieldMapper.rangeQueryForSeqNo(indexSettings.seqNoIndexOptions(), fromSeqNo, toSeqNo); + return new BooleanQuery.Builder().add(seqNoQuery, BooleanClause.Occur.MUST) + .add(Queries.newNonNestedFilter(indexSettings.getIndexVersionCreated()), BooleanClause.Occur.MUST) .build(); } diff --git a/server/src/main/java/org/elasticsearch/index/engine/SoftDeletesPolicy.java b/server/src/main/java/org/elasticsearch/index/engine/SoftDeletesPolicy.java index b6c75f5b14f8d..5689a6a5b8532 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/SoftDeletesPolicy.java +++ b/server/src/main/java/org/elasticsearch/index/engine/SoftDeletesPolicy.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.engine; -import org.apache.lucene.document.LongPoint; import org.apache.lucene.search.Query; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; @@ -147,8 +146,8 @@ synchronized long getMinRetainedSeqNo() { * Returns a soft-deletes retention query that will be used in {@link org.apache.lucene.index.SoftDeletesRetentionMergePolicy} * Documents including tombstones are soft-deleted and matched this query will be retained and won't cleaned up by merges. */ - Query getRetentionQuery() { - return LongPoint.newRangeQuery(SeqNoFieldMapper.NAME, getMinRetainedSeqNo(), Long.MAX_VALUE); + Query getRetentionQuery(SeqNoFieldMapper.SeqNoIndexOptions seqNoIndexOptions) { + return SeqNoFieldMapper.rangeQueryForSeqNo(seqNoIndexOptions, getMinRetainedSeqNo(), Long.MAX_VALUE); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index e20c592e46aab..b77c0426c23d4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -264,7 +264,7 @@ protected DocumentParserContext( new HashMap<>(), null, null, - SeqNoFieldMapper.SequenceIDFields.emptySeqID(), + SeqNoFieldMapper.SequenceIDFields.emptySeqID(mappingParserContext.getIndexSettings().seqNoIndexOptions()), RoutingFields.fromIndexSettings(mappingParserContext.getIndexSettings()), parent, dynamic, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java b/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java index f2ddf38fe4357..72fd812d982d8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java @@ -45,9 +45,9 @@ public class ParsedDocument { * Create a no-op tombstone document * @param reason the reason for the no-op */ - public static ParsedDocument noopTombstone(String reason) { + public static ParsedDocument noopTombstone(SeqNoFieldMapper.SeqNoIndexOptions seqNoIndexOptions, String reason) { LuceneDocument document = new LuceneDocument(); - SeqNoFieldMapper.SequenceIDFields seqIdFields = SeqNoFieldMapper.SequenceIDFields.tombstone(); + var seqIdFields = SeqNoFieldMapper.SequenceIDFields.tombstone(seqNoIndexOptions); seqIdFields.addFields(document); Field versionField = VersionFieldMapper.versionField(); document.add(versionField); @@ -72,9 +72,9 @@ public static ParsedDocument noopTombstone(String reason) { * The returned document consists only _uid, _seqno, _term and _version fields; other metadata fields are excluded. * @param id the id of the deleted document */ - public static ParsedDocument deleteTombstone(String id) { + public static ParsedDocument deleteTombstone(SeqNoFieldMapper.SeqNoIndexOptions seqNoIndexOptions, String id) { LuceneDocument document = new LuceneDocument(); - SeqNoFieldMapper.SequenceIDFields seqIdFields = SeqNoFieldMapper.SequenceIDFields.tombstone(); + SeqNoFieldMapper.SequenceIDFields seqIdFields = SeqNoFieldMapper.SequenceIDFields.tombstone(seqNoIndexOptions); seqIdFields.addFields(document); Field versionField = VersionFieldMapper.versionField(); document.add(versionField); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index 66ee42dfc56f9..8dad7a14c4336 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -84,12 +84,16 @@ public static class SequenceIDFields { private static final Field TOMBSTONE_FIELD = new NumericDocValuesField(TOMBSTONE_NAME, 1); - private final Field seqNo = new SingleValueLongField(NAME); - private final Field primaryTerm = new NumericDocValuesField(PRIMARY_TERM_NAME, 0); + private final Field seqNo; + private final Field primaryTerm = new NumericDocValuesField(PRIMARY_TERM_NAME, SequenceNumbers.UNASSIGNED_PRIMARY_TERM); private final boolean isTombstone; - private SequenceIDFields(boolean isTombstone) { + private SequenceIDFields(SeqNoIndexOptions seqNoIndexOptions, boolean isTombstone) { this.isTombstone = isTombstone; + this.seqNo = switch (seqNoIndexOptions) { + case POINTS_AND_DOC_VALUES -> new SingleValueLongField(NAME); + case DOC_VALUES_ONLY -> NumericDocValuesField.indexedField(NAME, SequenceNumbers.UNASSIGNED_SEQ_NO); + }; } public void addFields(LuceneDocument document) { @@ -113,12 +117,12 @@ public void set(long seqNo, long primaryTerm) { * Build and empty sequence ID who's values can be assigned later by * calling {@link #set}. */ - public static SequenceIDFields emptySeqID() { - return new SequenceIDFields(false); + public static SequenceIDFields emptySeqID(SeqNoIndexOptions seqNoIndexOptions) { + return new SequenceIDFields(seqNoIndexOptions, false); } - public static SequenceIDFields tombstone() { - return new SequenceIDFields(true); + public static SequenceIDFields tombstone(SeqNoIndexOptions seqNoIndexOptions) { + return new SequenceIDFields(seqNoIndexOptions, true); } } @@ -127,16 +131,20 @@ public static SequenceIDFields tombstone() { public static final String PRIMARY_TERM_NAME = "_primary_term"; public static final String TOMBSTONE_NAME = "_tombstone"; - public static final SeqNoFieldMapper INSTANCE = new SeqNoFieldMapper(); + public static final SeqNoFieldMapper WITH_POINT = new SeqNoFieldMapper(true); + public static final SeqNoFieldMapper NO_POINT = new SeqNoFieldMapper(false); - public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE); + public static final TypeParser PARSER = new FixedTypeParser(c -> switch (c.getIndexSettings().seqNoIndexOptions()) { + case POINTS_AND_DOC_VALUES -> WITH_POINT; + case DOC_VALUES_ONLY -> NO_POINT; + }); static final class SeqNoFieldType extends SimpleMappedFieldType { + private static final SeqNoFieldType WITH_POINT = new SeqNoFieldType(true); + private static final SeqNoFieldType NO_POINT = new SeqNoFieldType(false); - private static final SeqNoFieldType INSTANCE = new SeqNoFieldType(); - - private SeqNoFieldType() { - super(NAME, true, false, true, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, Collections.emptyMap()); + private SeqNoFieldType(boolean indexed) { + super(NAME, indexed, false, true, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, Collections.emptyMap()); } @Override @@ -174,13 +182,21 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) @Override public Query termQuery(Object value, @Nullable SearchExecutionContext context) { long v = parse(value); - return LongPoint.newExactQuery(name(), v); + if (isIndexed()) { + return LongPoint.newExactQuery(name(), v); + } else { + return NumericDocValuesField.newSlowExactQuery(name(), v); + } } @Override public Query termsQuery(Collection values, @Nullable SearchExecutionContext context) { long[] v = values.stream().mapToLong(SeqNoFieldType::parse).toArray(); - return LongPoint.newSetQuery(name(), v); + if (isIndexed()) { + return LongPoint.newSetQuery(name(), v); + } else { + return NumericDocValuesField.newSlowSetQuery(name(), v); + } } @Override @@ -211,7 +227,7 @@ public Query rangeQuery( --u; } } - return LongPoint.newRangeQuery(name(), l, u); + return rangeQueryForSeqNo(isIndexed(), l, u); } @Override @@ -219,10 +235,15 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG, SeqNoDocValuesField::new, isIndexed()); } + + @Override + public boolean isSearchable() { + return isIndexed() || hasDocValues(); + } } - private SeqNoFieldMapper() { - super(SeqNoFieldType.INSTANCE); + private SeqNoFieldMapper(boolean indexedPoints) { + super(indexedPoints ? SeqNoFieldType.WITH_POINT : SeqNoFieldType.NO_POINT); } @Override @@ -245,4 +266,28 @@ public void postParse(DocumentParserContext context) { protected String contentType() { return CONTENT_TYPE; } + + private static Query rangeQueryForSeqNo(boolean withPoints, long lowerValue, long upperValue) { + if (withPoints) { + // TODO: Use IndexOrDocValuesQuery + return LongPoint.newRangeQuery(SeqNoFieldMapper.NAME, lowerValue, upperValue); + } else { + return NumericDocValuesField.newSlowRangeQuery(SeqNoFieldMapper.NAME, lowerValue, upperValue); + } + } + + /** + * Create a range query that matches all documents whose seq_no is between {@code lowerValue} and {@code upperValue} included. + */ + public static Query rangeQueryForSeqNo(SeqNoIndexOptions seqNoIndexOptions, long lowerValue, long upperValue) { + return switch (seqNoIndexOptions) { + case POINTS_AND_DOC_VALUES -> rangeQueryForSeqNo(true, lowerValue, upperValue); + case DOC_VALUES_ONLY -> rangeQueryForSeqNo(false, lowerValue, upperValue); + }; + } + + public enum SeqNoIndexOptions { + DOC_VALUES_ONLY, + POINTS_AND_DOC_VALUES, + } } diff --git a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java index 6a71179bb7b48..e6c0742cb9979 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionLookupTests.java @@ -158,7 +158,7 @@ public void testLoadTimestampRange() throws Exception { public void testLoadTimestampRangeWithDeleteTombstone() throws Exception { Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER).setMergePolicy(NoMergePolicy.INSTANCE)); - writer.addDocument(ParsedDocument.deleteTombstone("_id").docs().get(0)); + writer.addDocument(ParsedDocument.deleteTombstone(randomFrom(SeqNoFieldMapper.SeqNoIndexOptions.values()), "_id").docs().get(0)); DirectoryReader reader = DirectoryReader.open(writer); LeafReaderContext segment = reader.leaves().get(0); PerThreadIDVersionAndSeqNoLookup lookup = new PerThreadIDVersionAndSeqNoLookup(segment.reader(), true); diff --git a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java index ce8b0e457b762..dc2182eb6331e 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java @@ -210,7 +210,7 @@ public void testSlowLogMessageHasJsonFields() throws IOException { BytesReference source = BytesReference.bytes(JsonXContent.contentBuilder().startObject().field("foo", "bar").endObject()); ParsedDocument pd = new ParsedDocument( new NumericDocValuesField("version", 1), - SeqNoFieldMapper.SequenceIDFields.emptySeqID(), + SeqNoFieldMapper.SequenceIDFields.emptySeqID(randomFrom(SeqNoFieldMapper.SeqNoIndexOptions.values())), "id", "routingValue", null, @@ -239,7 +239,7 @@ public void testSlowLogMessageHasAdditionalFields() throws IOException { BytesReference source = BytesReference.bytes(JsonXContent.contentBuilder().startObject().field("foo", "bar").endObject()); ParsedDocument pd = new ParsedDocument( new NumericDocValuesField("version", 1), - SeqNoFieldMapper.SequenceIDFields.emptySeqID(), + SeqNoFieldMapper.SequenceIDFields.emptySeqID(randomFrom(SeqNoFieldMapper.SeqNoIndexOptions.values())), "id", "routingValue", null, @@ -269,7 +269,7 @@ public void testEmptyRoutingField() throws IOException { BytesReference source = BytesReference.bytes(JsonXContent.contentBuilder().startObject().field("foo", "bar").endObject()); ParsedDocument pd = new ParsedDocument( new NumericDocValuesField("version", 1), - SeqNoFieldMapper.SequenceIDFields.emptySeqID(), + SeqNoFieldMapper.SequenceIDFields.emptySeqID(randomFrom(SeqNoFieldMapper.SeqNoIndexOptions.values())), "id", null, null, @@ -288,7 +288,7 @@ public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException { BytesReference source = BytesReference.bytes(JsonXContent.contentBuilder().startObject().field("foo", "bar").endObject()); ParsedDocument pd = new ParsedDocument( new NumericDocValuesField("version", 1), - SeqNoFieldMapper.SequenceIDFields.emptySeqID(), + SeqNoFieldMapper.SequenceIDFields.emptySeqID(randomFrom(SeqNoFieldMapper.SeqNoIndexOptions.values())), "id", null, null, @@ -320,7 +320,7 @@ public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException { source = new BytesArray("invalid"); ParsedDocument doc = new ParsedDocument( new NumericDocValuesField("version", 1), - SeqNoFieldMapper.SequenceIDFields.emptySeqID(), + SeqNoFieldMapper.SequenceIDFields.emptySeqID(randomFrom(SeqNoFieldMapper.SeqNoIndexOptions.values())), "id", null, null, diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index bc63ef763ec57..2f2d7fa142976 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -47,6 +47,7 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.TieredMergePolicy; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; @@ -2907,14 +2908,33 @@ public void testConcurrentWritesAndCommits() throws Exception { } } - private static Long getHighestSeqNo(final IndexReader reader) throws IOException { - final String fieldName = SeqNoFieldMapper.NAME; - long size = PointValues.size(reader, fieldName); - if (size == 0) { - return null; + private Long getHighestSeqNo(final IndexReader reader) throws IOException { + boolean usePoints = switch (defaultSettings.seqNoIndexOptions()) { + case POINTS_AND_DOC_VALUES -> randomBoolean(); + case DOC_VALUES_ONLY -> false; + }; + if (usePoints) { + final String fieldName = SeqNoFieldMapper.NAME; + long size = PointValues.size(reader, fieldName); + if (size == 0) { + return null; + } + byte[] max = PointValues.getMaxPackedValue(reader, fieldName); + return LongPoint.decodeDimension(max, 0); + } else { + Long value = null; + for (LeafReaderContext leaf : reader.leaves()) { + NumericDocValues dv = leaf.reader().getNumericDocValues(SeqNoFieldMapper.NAME); + while (dv.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { + if (value == null) { + value = dv.longValue(); + } else { + value = Math.max(value, dv.longValue()); + } + } + } + return value; } - byte[] max = PointValues.getMaxPackedValue(reader, fieldName); - return LongPoint.decodeDimension(max, 0); } private static FixedBitSet getSeqNosSet(final IndexReader reader, final long highestSeqNo) throws IOException { @@ -5498,7 +5518,7 @@ public void testSeqNoGenerator() throws IOException { final String id = "id"; final Field uidField = new StringField("_id", id, Field.Store.YES); final Field versionField = new NumericDocValuesField("_version", 0); - final SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); + final var seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID(seqNoIndexOptions); final LuceneDocument document = new LuceneDocument(); document.add(uidField); document.add(versionField); diff --git a/server/src/test/java/org/elasticsearch/index/engine/SoftDeletesPolicyTests.java b/server/src/test/java/org/elasticsearch/index/engine/SoftDeletesPolicyTests.java index 274192c7d6c72..fac6a14df8cee 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/SoftDeletesPolicyTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/SoftDeletesPolicyTests.java @@ -13,6 +13,7 @@ import org.apache.lucene.search.PointRangeQuery; import org.apache.lucene.search.Query; import org.elasticsearch.core.Releasable; +import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.seqno.RetentionLease; import org.elasticsearch.index.seqno.RetentionLeases; import org.elasticsearch.test.ESTestCase; @@ -74,7 +75,7 @@ public void testSoftDeletesRetentionLock() { releasingLocks.forEach(Releasable::close); // getting the query has side effects, updating the internal state of the policy - final Query query = policy.getRetentionQuery(); + final Query query = policy.getRetentionQuery(SeqNoFieldMapper.SeqNoIndexOptions.POINTS_AND_DOC_VALUES); assertThat(query, instanceOf(PointRangeQuery.class)); final PointRangeQuery retentionQuery = (PointRangeQuery) query; diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 48257bbe3fa21..1030e3d09b9a4 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -4482,7 +4482,7 @@ public void testOnCloseStats() throws IOException { public void testSupplyTombstoneDoc() throws Exception { IndexShard shard = newStartedShard(); String id = randomRealisticUnicodeOfLengthBetween(1, 10); - ParsedDocument deleteTombstone = ParsedDocument.deleteTombstone(id); + ParsedDocument deleteTombstone = ParsedDocument.deleteTombstone(shard.indexSettings.seqNoIndexOptions(), id); assertThat(deleteTombstone.docs(), hasSize(1)); LuceneDocument deleteDoc = deleteTombstone.docs().get(0); assertThat( @@ -4499,7 +4499,7 @@ public void testSupplyTombstoneDoc() throws Exception { assertThat(deleteDoc.getField(SeqNoFieldMapper.TOMBSTONE_NAME).numericValue().longValue(), equalTo(1L)); final String reason = randomUnicodeOfLength(200); - ParsedDocument noopTombstone = ParsedDocument.noopTombstone(reason); + ParsedDocument noopTombstone = ParsedDocument.noopTombstone(shard.indexSettings.seqNoIndexOptions(), reason); assertThat(noopTombstone.docs(), hasSize(1)); LuceneDocument noopDoc = noopTombstone.docs().get(0); assertThat( diff --git a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java index 6367a33318abc..699b5e93d79f9 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java @@ -562,7 +562,7 @@ private Engine.IndexResult index(String id, String testFieldValue) throws IOExce document.add(new TextField("test", testFieldValue, Field.Store.YES)); Field idField = new StringField(IdFieldMapper.NAME, uid, Field.Store.YES); Field versionField = new NumericDocValuesField("_version", Versions.MATCH_ANY); - SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); + var seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID(engine.config().getIndexSettings().seqNoIndexOptions()); document.add(idField); document.add(versionField); seqID.addFields(document); diff --git a/server/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java b/server/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java index aa89f31757ef4..811d313bd5447 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java @@ -229,7 +229,7 @@ private Iterable topLevel(IndexRouting indexRouting, int id, @Nu if (routing != null) { topLevel.add(new StringField(RoutingFieldMapper.NAME, routing, Field.Store.YES)); } - SeqNoFieldMapper.SequenceIDFields.emptySeqID().addFields(topLevel); + SeqNoFieldMapper.SequenceIDFields.emptySeqID(SeqNoFieldMapper.SeqNoIndexOptions.POINTS_AND_DOC_VALUES).addFields(topLevel); return topLevel; } diff --git a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index 69cf9d856ed4f..74e68c39e78fa 100644 --- a/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -3423,7 +3423,7 @@ public static Translog.Location randomTranslogLocation() { public void testTranslogOpSerialization() throws Exception { BytesReference B_1 = new BytesArray(new byte[] { 1 }); - SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); + var seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID(randomFrom(SeqNoFieldMapper.SeqNoIndexOptions.values())); long randomSeqNum = randomNonNegativeLong(); long randomPrimaryTerm = randomBoolean() ? 0 : randomNonNegativeLong(); seqID.set(randomSeqNum, randomPrimaryTerm); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index 28a032e7281e6..154dadccf695c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -780,7 +780,9 @@ public void testUsingTestCase() throws Exception { public void testSubAggregationOfNested() throws Exception { final String nestedPath = "sellers"; objectMappers.add(nestedObject(nestedPath)); - SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); + SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID( + SeqNoFieldMapper.SeqNoIndexOptions.POINTS_AND_DOC_VALUES + ); final String leafNameField = "name"; final String rootNameField = "name"; TermsValuesSourceBuilder terms = new TermsValuesSourceBuilder("keyword").field(nestedPath + "." + leafNameField); @@ -834,7 +836,7 @@ public void testSubAggregationOfNested() throws Exception { public void testSubAggregationOfNestedAggregateAfter() throws Exception { final String nestedPath = "sellers"; objectMappers.add(nestedObject(nestedPath)); - SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); + var sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(SeqNoFieldMapper.SeqNoIndexOptions.POINTS_AND_DOC_VALUES); final String leafNameField = "name"; final String rootNameField = "name"; TermsValuesSourceBuilder terms = new TermsValuesSourceBuilder("keyword").field(nestedPath + "." + leafNameField); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java index bac0ed0c242ae..b24bd9617c38f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/NestedAggregatorTests.java @@ -117,7 +117,9 @@ public class NestedAggregatorTests extends AggregatorTestCase { private static final String SUM_AGG_NAME = "sumAgg"; private static final String INVERSE_SCRIPT = "inverse"; - private static final SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); + private static final SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID( + SeqNoFieldMapper.SeqNoIndexOptions.POINTS_AND_DOC_VALUES + ); /** * Nested aggregations need the {@linkplain DirectoryReader} wrapped. @@ -308,7 +310,9 @@ public void testOrphanedDocs() throws IOException { public void testResetRootDocId() throws Exception { IndexWriterConfig iwc = new IndexWriterConfig(null).setMergePolicy(new LogDocMergePolicy()); iwc.setMergePolicy(NoMergePolicy.INSTANCE); - SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); + SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID( + randomFrom(SeqNoFieldMapper.SeqNoIndexOptions.values()) + ); try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = new RandomIndexWriter(random(), directory, iwc)) { List> documents = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java index 6490898c0ea2a..6eb88dbcf24cb 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/nested/ReverseNestedAggregatorTests.java @@ -93,6 +93,7 @@ public void testMaxFromParentDocs() throws IOException { int expectedParentDocs = 0; int expectedNestedDocs = 0; double expectedMaxValue = Double.NEGATIVE_INFINITY; + var seqNoIndexOptions = randomFrom(SeqNoFieldMapper.SeqNoIndexOptions.values()); try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = newRandomIndexWriterWithLogDocMergePolicy(directory)) { for (int i = 0; i < numParentDocs; i++) { @@ -110,7 +111,7 @@ public void testMaxFromParentDocs() throws IOException { document.add(new StringField(NestedPathFieldMapper.NAME, "test", Field.Store.NO)); long value = randomNonNegativeLong() % 10000; document.add(new SortedNumericDocValuesField(VALUE_FIELD_NAME, value)); - SeqNoFieldMapper.SequenceIDFields.emptySeqID().addFields(document); + SeqNoFieldMapper.SequenceIDFields.emptySeqID(seqNoIndexOptions).addFields(document); if (numNestedDocs > 0) { expectedMaxValue = Math.max(expectedMaxValue, value); expectedParentDocs++; @@ -150,7 +151,7 @@ public void testFieldAlias() throws IOException { int expectedParentDocs = 0; MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(VALUE_FIELD_NAME, NumberFieldMapper.NumberType.LONG); - + SeqNoFieldMapper.SeqNoIndexOptions seqNoIndexOptions = randomFrom(SeqNoFieldMapper.SeqNoIndexOptions.values()); try (Directory directory = newDirectory()) { try (RandomIndexWriter iw = newRandomIndexWriterWithLogDocMergePolicy(directory)) { for (int i = 0; i < numParentDocs; i++) { @@ -172,7 +173,7 @@ public void testFieldAlias() throws IOException { long value = randomNonNegativeLong() % 10000; document.add(new SortedNumericDocValuesField(VALUE_FIELD_NAME, value)); - SeqNoFieldMapper.SequenceIDFields.emptySeqID().addFields(document); + SeqNoFieldMapper.SequenceIDFields.emptySeqID(seqNoIndexOptions).addFields(document); documents.add(document); iw.addDocuments(documents); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java index 48aabb61371e9..948bbaedb5a23 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/RareTermsAggregatorTests.java @@ -425,7 +425,9 @@ public void testWithNestedScoringAggregations() throws IOException { } } - private final SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); + private final SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID( + SeqNoFieldMapper.SeqNoIndexOptions.POINTS_AND_DOC_VALUES + ); private List> generateDocsWithNested(String id, int value, int[] nestedValues) { List> documents = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index 16509daca79f4..ece24f13be4f3 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -2253,7 +2253,9 @@ public void testFewBuckets() throws IOException { }, keywordFt); } - private final SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); + private final SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID( + SeqNoFieldMapper.SeqNoIndexOptions.POINTS_AND_DOC_VALUES + ); private List> generateDocsWithNested(String id, int value, int[] nestedValues) { List> documents = new ArrayList<>(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index caeabb6abddae..9bb6696b1ee6d 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -173,6 +173,7 @@ public abstract class EngineTestCase extends ESTestCase { protected Path replicaTranslogDir; // A default primary term is used by engine instances created in this test. protected final PrimaryTermSupplier primaryTerm = new PrimaryTermSupplier(1L); + protected static SeqNoFieldMapper.SeqNoIndexOptions seqNoIndexOptions = SeqNoFieldMapper.SeqNoIndexOptions.POINTS_AND_DOC_VALUES; protected static void assertVisibleCount(Engine engine, int numDocs) throws IOException { assertVisibleCount(engine, numDocs, true); @@ -200,6 +201,7 @@ protected Settings indexSettings() { ) .put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), between(0, 1000)) .put(ThreadPoolMergeScheduler.USE_THREAD_POOL_MERGE_SCHEDULER_SETTING.getKey(), randomBoolean()) + .put(IndexSettings.SEQ_NO_INDEX_OPTIONS_SETTING.getKey(), seqNoIndexOptions) .build(); } @@ -451,7 +453,7 @@ protected static ParsedDocument testParsedDocument( ) { Field idField = new StringField("_id", Uid.encodeId(id), Field.Store.YES); Field versionField = new NumericDocValuesField("_version", 0); - SeqNoFieldMapper.SequenceIDFields seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); + var seqID = SeqNoFieldMapper.SequenceIDFields.emptySeqID(seqNoIndexOptions); document.add(idField); document.add(versionField); seqID.addFields(document); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 765162fe9dc5e..509475531f3ef 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -136,6 +136,7 @@ import org.elasticsearch.index.engine.Segment; import org.elasticsearch.index.engine.ThreadPoolMergeScheduler; import org.elasticsearch.index.mapper.MockFieldFilterPlugin; +import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.IndicesQueryCache; import org.elasticsearch.indices.IndicesRequestCache; @@ -474,6 +475,9 @@ protected Settings.Builder setRandomIndexSettings(Random random, Settings.Builde if (randomBoolean()) { builder.put(IndexSettings.BLOOM_FILTER_ID_FIELD_ENABLED_SETTING.getKey(), randomBoolean()); } + if (randomBoolean()) { + builder.put(IndexSettings.SEQ_NO_INDEX_OPTIONS_SETTING.getKey(), randomFrom(SeqNoFieldMapper.SeqNoIndexOptions.values())); + } return builder; } diff --git a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/SpatialDiskUsageIT.java b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/SpatialDiskUsageIT.java index a3dd8cdf5bcbb..ee63373983be5 100644 --- a/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/SpatialDiskUsageIT.java +++ b/x-pack/plugin/spatial/src/internalClusterTest/java/org/elasticsearch/xpack/spatial/SpatialDiskUsageIT.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xcontent.ToXContent; @@ -30,6 +31,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Map; +import java.util.Random; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -41,6 +43,13 @@ protected Collection> nodePlugins() { return Collections.singleton(LocalStateSpatialPlugin.class); } + @Override + protected Settings.Builder setRandomIndexSettings(Random random, Settings.Builder builder) { + var b = super.setRandomIndexSettings(random, builder); + b.remove(IndexSettings.SEQ_NO_INDEX_OPTIONS_SETTING.getKey()); + return b; + } + public void testGeoShape() throws Exception { doTestSpatialField(GeoShapeWithDocValuesFieldMapper.CONTENT_TYPE); }