From d25ae8e000633869ffe233863c41859b38a2ce80 Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Sat, 23 Aug 2025 14:18:21 +0100 Subject: [PATCH 1/9] add lucene version to ddoc and index definition --- .../couchdb/nouveau/NouveauApplication.java | 6 +-- .../couchdb/nouveau/api/IndexDefinition.java | 22 +++++++++- .../couchdb/nouveau/core/IndexManager.java | 11 ++--- .../nouveau/health/IndexHealthCheck.java | 4 +- .../LuceneAnalyzerFactory.java} | 6 +-- .../LuceneIndex.java} | 18 ++++---- .../LuceneIndexSchema.java} | 14 +++---- .../LuceneModule.java} | 8 ++-- .../NouveauQueryParser.java | 2 +- .../ParallelSearcherFactory.java | 2 +- .../QueryDeserializer.java | 2 +- .../{lucene9 => lucene}/QuerySerializer.java | 2 +- .../SimpleAsciiFoldingAnalyzer.java | 2 +- .../nouveau/resources/AnalyzeResource.java | 4 +- .../nouveau/api/IndexDefinitionTest.java | 42 +++++++++++++++++++ .../nouveau/core/IndexManagerTest.java | 2 +- .../LuceneAnalyzerFactoryTest.java} | 19 +++++---- .../LuceneIndexTest.java} | 8 ++-- .../NouveauQueryParserTest.java | 2 +- .../QuerySerializationTest.java | 4 +- src/nouveau/include/nouveau.hrl | 3 ++ src/nouveau/src/nouveau_index_updater.erl | 1 + src/nouveau/src/nouveau_util.erl | 16 +++++-- 23 files changed, 139 insertions(+), 61 deletions(-) rename nouveau/src/main/java/org/apache/couchdb/nouveau/{lucene9/Lucene9AnalyzerFactory.java => lucene/LuceneAnalyzerFactory.java} (97%) rename nouveau/src/main/java/org/apache/couchdb/nouveau/{lucene9/Lucene9Index.java => lucene/LuceneIndex.java} (98%) rename nouveau/src/main/java/org/apache/couchdb/nouveau/{lucene9/Lucene9IndexSchema.java => lucene/LuceneIndexSchema.java} (91%) rename nouveau/src/main/java/org/apache/couchdb/nouveau/{lucene9/Lucene9Module.java => lucene/LuceneModule.java} (82%) rename nouveau/src/main/java/org/apache/couchdb/nouveau/{lucene9 => lucene}/NouveauQueryParser.java (98%) rename nouveau/src/main/java/org/apache/couchdb/nouveau/{lucene9 => lucene}/ParallelSearcherFactory.java (96%) rename nouveau/src/main/java/org/apache/couchdb/nouveau/{lucene9 => lucene}/QueryDeserializer.java (99%) rename nouveau/src/main/java/org/apache/couchdb/nouveau/{lucene9 => lucene}/QuerySerializer.java (99%) rename nouveau/src/main/java/org/apache/couchdb/nouveau/{lucene9 => lucene}/SimpleAsciiFoldingAnalyzer.java (96%) create mode 100644 nouveau/src/test/java/org/apache/couchdb/nouveau/api/IndexDefinitionTest.java rename nouveau/src/test/java/org/apache/couchdb/nouveau/{lucene9/Lucene9AnalyzerFactoryTest.java => lucene/LuceneAnalyzerFactoryTest.java} (92%) rename nouveau/src/test/java/org/apache/couchdb/nouveau/{lucene9/Lucene9IndexTest.java => lucene/LuceneIndexTest.java} (97%) rename nouveau/src/test/java/org/apache/couchdb/nouveau/{lucene9 => lucene}/NouveauQueryParserTest.java (98%) rename nouveau/src/test/java/org/apache/couchdb/nouveau/{lucene9 => lucene}/QuerySerializationTest.java (97%) diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java index 7179eadc06a..a6c20290360 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java @@ -24,8 +24,8 @@ import org.apache.couchdb.nouveau.core.UserAgentFilter; import org.apache.couchdb.nouveau.health.AnalyzeHealthCheck; import org.apache.couchdb.nouveau.health.IndexHealthCheck; -import org.apache.couchdb.nouveau.lucene9.Lucene9Module; -import org.apache.couchdb.nouveau.lucene9.ParallelSearcherFactory; +import org.apache.couchdb.nouveau.lucene.LuceneModule; +import org.apache.couchdb.nouveau.lucene.ParallelSearcherFactory; import org.apache.couchdb.nouveau.resources.AnalyzeResource; import org.apache.couchdb.nouveau.resources.IndexResource; import org.apache.couchdb.nouveau.tasks.CloseAllIndexesTask; @@ -65,7 +65,7 @@ public void run(NouveauApplicationConfiguration configuration, Environment envir environment.lifecycle().manage(indexManager); // Serialization classes - environment.getObjectMapper().registerModule(new Lucene9Module()); + environment.getObjectMapper().registerModule(new LuceneModule()); // AnalyzeResource final AnalyzeResource analyzeResource = new AnalyzeResource(); diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java index 3d79fca654e..21811e66fdb 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java @@ -22,6 +22,10 @@ @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) public class IndexDefinition { + public static final int LEGACY_LUCENE_VERSION = 9; + + private int luceneVersion = LEGACY_LUCENE_VERSION; // Legacy version if not set. + @NotEmpty private String defaultAnalyzer; @@ -31,11 +35,22 @@ public IndexDefinition() { // Jackson deserialization } - public IndexDefinition(final String defaultAnalyzer, final Map fieldAnalyzers) { + public IndexDefinition( + final int luceneVersion, final String defaultAnalyzer, final Map fieldAnalyzers) { + this.luceneVersion = luceneVersion; this.defaultAnalyzer = defaultAnalyzer; this.fieldAnalyzers = fieldAnalyzers; } + @JsonProperty + public int getLuceneVersion() { + return luceneVersion; + } + + public void setLuceneVersion(int luceneVersion) { + this.luceneVersion = luceneVersion; + } + @JsonProperty public String getDefaultAnalyzer() { return defaultAnalyzer; @@ -62,6 +77,7 @@ public boolean hasFieldAnalyzers() { public int hashCode() { final int prime = 31; int result = 1; + result = prime * result + luceneVersion; result = prime * result + ((defaultAnalyzer == null) ? 0 : defaultAnalyzer.hashCode()); result = prime * result + ((fieldAnalyzers == null) ? 0 : fieldAnalyzers.hashCode()); return result; @@ -73,6 +89,7 @@ public boolean equals(Object obj) { if (obj == null) return false; if (getClass() != obj.getClass()) return false; IndexDefinition other = (IndexDefinition) obj; + if (luceneVersion != other.luceneVersion) return false; if (defaultAnalyzer == null) { if (other.defaultAnalyzer != null) return false; } else if (!defaultAnalyzer.equals(other.defaultAnalyzer)) return false; @@ -84,6 +101,7 @@ public boolean equals(Object obj) { @Override public String toString() { - return "IndexDefinition [defaultAnalyzer=" + defaultAnalyzer + ", fieldAnalyzers=" + fieldAnalyzers + "]"; + return "IndexDefinition [luceneVersion=" + luceneVersion + ", defaultAnalyzer=" + defaultAnalyzer + + ", fieldAnalyzers=" + fieldAnalyzers + "]"; } } diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java index c4f59f7a15f..1a5caf986e9 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java @@ -34,8 +34,8 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.couchdb.nouveau.api.IndexDefinition; -import org.apache.couchdb.nouveau.lucene9.Lucene9AnalyzerFactory; -import org.apache.couchdb.nouveau.lucene9.Lucene9Index; +import org.apache.couchdb.nouveau.lucene.LuceneAnalyzerFactory; +import org.apache.couchdb.nouveau.lucene.LuceneIndex; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -392,15 +392,16 @@ private Index load(final String name) throws IOException { LOGGER.info("opening {}", name); final Path path = indexPath(name); final IndexDefinition indexDefinition = loadIndexDefinition(name); - final Analyzer analyzer = Lucene9AnalyzerFactory.fromDefinition(indexDefinition); - final Directory dir = new DirectIODirectory(FSDirectory.open(path.resolve("9"))); + final Analyzer analyzer = LuceneAnalyzerFactory.fromDefinition(indexDefinition); + final Directory dir = new DirectIODirectory( + FSDirectory.open(path.resolve(Integer.toString(indexDefinition.getLuceneVersion())))); final IndexWriterConfig config = new IndexWriterConfig(analyzer); config.setUseCompoundFile(false); final IndexWriter writer = new IndexWriter(dir, config); final long updateSeq = getSeq(writer, "update_seq"); final long purgeSeq = getSeq(writer, "purge_seq"); final SearcherManager searcherManager = new SearcherManager(writer, searcherFactory); - return new Lucene9Index(analyzer, writer, updateSeq, purgeSeq, searcherManager); + return new LuceneIndex(analyzer, writer, updateSeq, purgeSeq, searcherManager); } private long getSeq(final IndexWriter writer, final String key) throws IOException { diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java index 0ee8fefa2e7..e34bebef517 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java @@ -32,14 +32,14 @@ public IndexHealthCheck(final IndexResource indexResource) { @Override protected Result check() throws Exception { - final String name = "___test9"; + final String name = "___test"; try { indexResource.deletePath(name, null); } catch (IOException e) { // Ignored, index might not exist yet. } - indexResource.createIndex(name, new IndexDefinition("standard", null)); + indexResource.createIndex(name, new IndexDefinition(IndexDefinition.LEGACY_LUCENE_VERSION, "standard", null)); try { final DocumentUpdateRequest documentUpdateRequest = new DocumentUpdateRequest(0, 1, null, Collections.emptyList()); diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactory.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactory.java similarity index 97% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactory.java rename to nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactory.java index 2bd47ed9740..b95230cec7b 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactory.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactory.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Response.Status; @@ -60,9 +60,9 @@ import org.apache.lucene.analysis.th.ThaiAnalyzer; import org.apache.lucene.analysis.tr.TurkishAnalyzer; -public final class Lucene9AnalyzerFactory { +public final class LuceneAnalyzerFactory { - private Lucene9AnalyzerFactory() {} + private LuceneAnalyzerFactory() {} public static Analyzer fromDefinition(final IndexDefinition indexDefinition) { final Analyzer defaultAnalyzer = newAnalyzer(indexDefinition.getDefaultAnalyzer()); diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndex.java similarity index 98% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java rename to nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndex.java index dad3040f328..9dda62f8906 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndex.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Response.Status; @@ -97,7 +97,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -public class Lucene9Index extends Index { +public class LuceneIndex extends Index { private static final Sort DEFAULT_SORT = new Sort(SortField.FIELD_SCORE, new SortField("_id", SortField.Type.STRING)); @@ -106,9 +106,9 @@ public class Lucene9Index extends Index { private final Analyzer analyzer; private final IndexWriter writer; private final SearcherManager searcherManager; - private final Lucene9IndexSchema schema; + private final LuceneIndexSchema schema; - public Lucene9Index( + public LuceneIndex( final Analyzer analyzer, final IndexWriter writer, final long updateSeq, @@ -537,22 +537,22 @@ private Query parse(final SearchRequest request) { return result; } - private Lucene9IndexSchema initSchema(IndexWriter writer) { + private LuceneIndexSchema initSchema(IndexWriter writer) { var commitData = writer.getLiveCommitData(); if (commitData == null) { - return Lucene9IndexSchema.emptySchema(); + return LuceneIndexSchema.emptySchema(); } for (var entry : commitData) { if (entry.getKey().equals("_schema")) { - return Lucene9IndexSchema.fromString(entry.getValue()); + return LuceneIndexSchema.fromString(entry.getValue()); } } - return Lucene9IndexSchema.emptySchema(); + return LuceneIndexSchema.emptySchema(); } @Override public String toString() { - return "Lucene9Index [analyzer=" + analyzer + ", writer=" + writer + ", searcherManager=" + searcherManager + return "LuceneIndex [analyzer=" + analyzer + ", writer=" + writer + ", searcherManager=" + searcherManager + "]"; } diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexSchema.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndexSchema.java similarity index 91% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexSchema.java rename to nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndexSchema.java index 92cc5fc2e8f..ef335bf5a20 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexSchema.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndexSchema.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Response.Status; @@ -32,7 +32,7 @@ import org.apache.couchdb.nouveau.api.TextField; import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig; -final class Lucene9IndexSchema { +final class LuceneIndexSchema { public enum Type { STRING, @@ -56,23 +56,23 @@ private static Type fromField(final Field field) { private final ConcurrentMap map; - private Lucene9IndexSchema(Map map) { + private LuceneIndexSchema(Map map) { this.map = new ConcurrentHashMap<>(map); this.map.put("_id", Type.STRING); } - public static Lucene9IndexSchema emptySchema() { - return new Lucene9IndexSchema(new HashMap()); + public static LuceneIndexSchema emptySchema() { + return new LuceneIndexSchema(new HashMap()); } - public static Lucene9IndexSchema fromString(final String schemaStr) { + public static LuceneIndexSchema fromString(final String schemaStr) { Objects.requireNonNull(schemaStr); if (schemaStr.isEmpty()) { return emptySchema(); } var map = Arrays.stream(schemaStr.split(",")) .collect(Collectors.toMap(i -> i.split(":")[0], i -> Type.valueOf(i.split(":")[1]))); - return new Lucene9IndexSchema(map); + return new LuceneIndexSchema(map); } public void update(final Collection fields) { diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Module.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneModule.java similarity index 82% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Module.java rename to nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneModule.java index 06102de84cb..35be8efe27f 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Module.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneModule.java @@ -11,16 +11,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import com.fasterxml.jackson.core.Version; import com.fasterxml.jackson.databind.module.SimpleModule; import org.apache.lucene.search.Query; -public class Lucene9Module extends SimpleModule { +public class LuceneModule extends SimpleModule { - public Lucene9Module() { - super("lucene9", Version.unknownVersion()); + public LuceneModule() { + super("lucene", Version.unknownVersion()); // Query addSerializer(Query.class, new QuerySerializer()); diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParser.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParser.java similarity index 98% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParser.java rename to nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParser.java index fc7acf5091d..d37ab3d37c5 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParser.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParser.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import java.util.Map; import org.apache.lucene.analysis.Analyzer; diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/ParallelSearcherFactory.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/ParallelSearcherFactory.java similarity index 96% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/ParallelSearcherFactory.java rename to nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/ParallelSearcherFactory.java index 4553fa76b2f..bd9050af86f 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/ParallelSearcherFactory.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/ParallelSearcherFactory.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import java.io.IOException; import java.util.concurrent.Executor; diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QueryDeserializer.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QueryDeserializer.java similarity index 99% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QueryDeserializer.java rename to nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QueryDeserializer.java index b0620e661dc..04a771cf52f 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QueryDeserializer.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QueryDeserializer.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QuerySerializer.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QuerySerializer.java similarity index 99% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QuerySerializer.java rename to nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QuerySerializer.java index 5e1d5087193..2c88383a139 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QuerySerializer.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QuerySerializer.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import com.fasterxml.jackson.core.JsonGenerationException; import com.fasterxml.jackson.core.JsonGenerator; diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/SimpleAsciiFoldingAnalyzer.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/SimpleAsciiFoldingAnalyzer.java similarity index 96% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/SimpleAsciiFoldingAnalyzer.java rename to nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/SimpleAsciiFoldingAnalyzer.java index 6b4c8c64222..464b49d6370 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/SimpleAsciiFoldingAnalyzer.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/SimpleAsciiFoldingAnalyzer.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java index 2ae8b78f10a..d5249e20585 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java @@ -30,7 +30,7 @@ import java.util.List; import org.apache.couchdb.nouveau.api.AnalyzeRequest; import org.apache.couchdb.nouveau.api.AnalyzeResponse; -import org.apache.couchdb.nouveau.lucene9.Lucene9AnalyzerFactory; +import org.apache.couchdb.nouveau.lucene.LuceneAnalyzerFactory; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; @@ -47,7 +47,7 @@ public final class AnalyzeResource { public AnalyzeResponse analyzeText(@NotNull @Valid AnalyzeRequest request) throws IOException { try { final List tokens = - tokenize(Lucene9AnalyzerFactory.newAnalyzer(request.getAnalyzer()), request.getText()); + tokenize(LuceneAnalyzerFactory.newAnalyzer(request.getAnalyzer()), request.getText()); return new AnalyzeResponse(tokens); } catch (IllegalArgumentException e) { throw new WebApplicationException(request.getAnalyzer() + " not a valid analyzer", Status.BAD_REQUEST); diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/api/IndexDefinitionTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/api/IndexDefinitionTest.java new file mode 100644 index 00000000000..2e1e7ee0353 --- /dev/null +++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/api/IndexDefinitionTest.java @@ -0,0 +1,42 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.couchdb.nouveau.api; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class IndexDefinitionTest { + + private static ObjectMapper mapper; + + @BeforeAll + public static void setup() { + mapper = new ObjectMapper(); + } + + @Test + public void legacyLuceneVersionIfMissing() throws Exception { + var indexDefinition = mapper.readValue("{}", IndexDefinition.class); + assertThat(indexDefinition.getLuceneVersion()).isEqualTo(IndexDefinition.LEGACY_LUCENE_VERSION); + } + + @Test + public void luceneVersionIsDeserializedIfPresent() throws Exception { + var indexDefinition = mapper.readValue("{\"lucene_version\":10}", IndexDefinition.class); + assertThat(indexDefinition.getLuceneVersion()).isEqualTo(10); + } +} diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java index bb8f0b6470e..447087c7bad 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java +++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java @@ -26,7 +26,7 @@ import java.util.concurrent.TimeUnit; import org.apache.couchdb.nouveau.api.IndexDefinition; import org.apache.couchdb.nouveau.api.SearchRequest; -import org.apache.couchdb.nouveau.lucene9.ParallelSearcherFactory; +import org.apache.couchdb.nouveau.lucene.ParallelSearcherFactory; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactoryTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactoryTest.java similarity index 92% rename from nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactoryTest.java rename to nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactoryTest.java index eb8e30e0884..2fa3c26936b 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactoryTest.java +++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactoryTest.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -62,7 +62,7 @@ import org.apache.lucene.analysis.tr.TurkishAnalyzer; import org.junit.jupiter.api.Test; -public class Lucene9AnalyzerFactoryTest { +public class LuceneAnalyzerFactoryTest { @Test public void testkeyword() throws Exception { @@ -256,9 +256,11 @@ public void testturkish() throws Exception { @Test public void testFieldAnalyzers() throws Exception { - final IndexDefinition indexDefinition = - new IndexDefinition("standard", Map.of("english", "english", "thai", "thai", "email", "email")); - final Analyzer analyzer = Lucene9AnalyzerFactory.fromDefinition(indexDefinition); + final IndexDefinition indexDefinition = new IndexDefinition( + IndexDefinition.LEGACY_LUCENE_VERSION, + "standard", + Map.of("english", "english", "thai", "thai", "email", "email")); + final Analyzer analyzer = LuceneAnalyzerFactory.fromDefinition(indexDefinition); assertThat(analyzer).isInstanceOf(PerFieldAnalyzerWrapper.class); final Method m = PerFieldAnalyzerWrapper.class.getDeclaredMethod("getWrappedAnalyzer", String.class); m.setAccessible(true); @@ -270,12 +272,13 @@ public void testFieldAnalyzers() throws Exception { @Test public void testUnknownAnalyzer() throws Exception { - assertThrows(WebApplicationException.class, () -> Lucene9AnalyzerFactory.newAnalyzer("foo")); + assertThrows(WebApplicationException.class, () -> LuceneAnalyzerFactory.newAnalyzer("foo")); } private void assertAnalyzer(final String name, final Class clazz) throws Exception { - assertThat(Lucene9AnalyzerFactory.newAnalyzer(name)).isInstanceOf(clazz); - assertThat(Lucene9AnalyzerFactory.fromDefinition(new IndexDefinition(name, null))) + assertThat(LuceneAnalyzerFactory.newAnalyzer(name)).isInstanceOf(clazz); + assertThat(LuceneAnalyzerFactory.fromDefinition( + new IndexDefinition(IndexDefinition.LEGACY_LUCENE_VERSION, name, null))) .isInstanceOf(clazz); } } diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneIndexTest.java similarity index 97% rename from nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexTest.java rename to nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneIndexTest.java index 428e3eb6ae0..9206b83d9f6 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexTest.java +++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneIndexTest.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -45,18 +45,18 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -public class Lucene9IndexTest { +public class LuceneIndexTest { protected final Index setup(final Path path) throws IOException { final IndexDefinition indexDefinition = new IndexDefinition(); indexDefinition.setDefaultAnalyzer("standard"); - final Analyzer analyzer = Lucene9AnalyzerFactory.fromDefinition(indexDefinition); + final Analyzer analyzer = LuceneAnalyzerFactory.fromDefinition(indexDefinition); final Directory dir = new DirectIODirectory(FSDirectory.open(path)); final IndexWriterConfig config = new IndexWriterConfig(analyzer); config.setUseCompoundFile(false); final IndexWriter writer = new IndexWriter(dir, config); final SearcherManager searcherManager = new SearcherManager(writer, null); - return new Lucene9Index(analyzer, writer, 0L, 0L, searcherManager); + return new LuceneIndex(analyzer, writer, 0L, 0L, searcherManager); } protected final void cleanup(final Index index) throws IOException { diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParserTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParserTest.java similarity index 98% rename from nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParserTest.java rename to nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParserTest.java index 0f4689b2137..ab7c3dfdd68 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParserTest.java +++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParserTest.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import static org.assertj.core.api.Assertions.assertThat; diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/QuerySerializationTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/QuerySerializationTest.java similarity index 97% rename from nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/QuerySerializationTest.java rename to nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/QuerySerializationTest.java index 555495a7a33..9d20907c948 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/QuerySerializationTest.java +++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/QuerySerializationTest.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -39,7 +39,7 @@ public class QuerySerializationTest { @BeforeAll public static void setup() { mapper = new ObjectMapper(); - mapper.registerModule(new Lucene9Module()); + mapper.registerModule(new LuceneModule()); } @Test diff --git a/src/nouveau/include/nouveau.hrl b/src/nouveau/include/nouveau.hrl index e50cd45d3a9..16f46bdd9b9 100644 --- a/src/nouveau/include/nouveau.hrl +++ b/src/nouveau/include/nouveau.hrl @@ -11,9 +11,12 @@ %% See the License for the specific language governing permissions and %% limitations under the License. +-define(LEGACY_LUCENE_VERSION, 9). + -record(index, { dbname, ddoc_id, + lucene_version, default_analyzer, field_analyzers, def, diff --git a/src/nouveau/src/nouveau_index_updater.erl b/src/nouveau/src/nouveau_index_updater.erl index 3952a893f24..4bfea753a15 100644 --- a/src/nouveau/src/nouveau_index_updater.erl +++ b/src/nouveau/src/nouveau_index_updater.erl @@ -204,6 +204,7 @@ get_db_info(#index{} = Index) -> index_definition(#index{} = Index) -> #{ + <<"lucene_version">> => Index#index.lucene_version, <<"default_analyzer">> => Index#index.default_analyzer, <<"field_analyzers">> => Index#index.field_analyzers }. diff --git a/src/nouveau/src/nouveau_util.erl b/src/nouveau/src/nouveau_util.erl index 0dfcb1e1e0b..dbc30927a33 100644 --- a/src/nouveau/src/nouveau_util.erl +++ b/src/nouveau/src/nouveau_util.erl @@ -67,23 +67,33 @@ design_doc_to_index(DbName, #doc{id = Id, body = {Fields}}, IndexName) -> false -> {error, {not_found, <>}}; {IndexName, {Index}} -> + LuceneVersion = couch_util:get_value( + <<"lucene_version">>, Index, ?LEGACY_LUCENE_VERSION + ), DefaultAnalyzer = couch_util:get_value(<<"default_analyzer">>, Index, <<"standard">>), FieldAnalyzers = couch_util:get_value(<<"field_analyzers">>, Index, #{}), case couch_util:get_value(<<"index">>, Index) of undefined -> {error, InvalidDDocError}; Def -> - Sig = - couch_util:to_hex_bin( + SigTerm = + case LuceneVersion of + ?LEGACY_LUCENE_VERSION -> + {DefaultAnalyzer, FieldAnalyzers, Def}; + _ -> + {LuceneVersion, DefaultAnalyzer, FieldAnalyzers, Def} + end, + Sig = couch_util:to_hex_bin( crypto:hash( sha256, ?term_to_bin( - {DefaultAnalyzer, FieldAnalyzers, Def} + SigTerm ) ) ), {ok, #index{ dbname = DbName, + lucene_version = LuceneVersion, default_analyzer = DefaultAnalyzer, field_analyzers = FieldAnalyzers, ddoc_id = Id, From bfc1a71613fd6d61fc8c7c8fe456f1de2781188d Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Thu, 2 Oct 2025 09:25:54 +0100 Subject: [PATCH 2/9] upgrade to Lucene 10 --- nouveau/build.gradle | 2 +- .../java/org/apache/couchdb/nouveau/lucene/LuceneIndex.java | 4 ++-- .../org/apache/couchdb/nouveau/lucene/QuerySerializer.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nouveau/build.gradle b/nouveau/build.gradle index 356295ee163..fd159525a36 100644 --- a/nouveau/build.gradle +++ b/nouveau/build.gradle @@ -23,7 +23,7 @@ dependencies { implementation 'io.dropwizard.metrics:metrics-jersey2' testImplementation 'io.dropwizard:dropwizard-testing' - def luceneVersion = '9.12.1' + def luceneVersion = '10.3.0' implementation group: 'org.apache.lucene', name: 'lucene-core', version: luceneVersion implementation group: 'org.apache.lucene', name: 'lucene-queryparser', version: luceneVersion implementation group: 'org.apache.lucene', name: 'lucene-analysis-common', version: luceneVersion diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndex.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndex.java index 9dda62f8906..e62ffdb5a96 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndex.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndex.java @@ -290,8 +290,8 @@ private void collectHits(final IndexSearcher searcher, final TopDocs topDocs, fi hits.add(new SearchHit(doc.get("_id"), after, fields)); } - searchResults.setTotalHits(topDocs.totalHits.value); - searchResults.setTotalHitsRelation(topDocs.totalHits.relation); + searchResults.setTotalHits(topDocs.totalHits.value()); + searchResults.setTotalHitsRelation(topDocs.totalHits.relation()); searchResults.setHits(hits); } diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QuerySerializer.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QuerySerializer.java index 2c88383a139..7e42ee6dbdd 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QuerySerializer.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QuerySerializer.java @@ -65,8 +65,8 @@ public void serialize(final Query query, final JsonGenerator gen, final Serializ for (final BooleanClause clause : booleanQuery.clauses()) { gen.writeStartObject(); gen.writeFieldName("query"); - serialize(clause.getQuery(), gen, provider); - gen.writeStringField("occur", clause.getOccur().name().toLowerCase()); + serialize(clause.query(), gen, provider); + gen.writeStringField("occur", clause.occur().name().toLowerCase()); gen.writeEndObject(); } gen.writeEndArray(); From 33f4c4e1b3b6c4c33a9508f46820a79048baf186 Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Thu, 2 Oct 2025 09:27:03 +0100 Subject: [PATCH 3/9] support legacy codec but prevent new legacy indexes --- .../couchdb/nouveau/api/IndexDefinition.java | 5 ++++ .../couchdb/nouveau/core/IndexManager.java | 8 +++++-- .../nouveau/health/IndexHealthCheck.java | 2 +- .../nouveau/core/IndexManagerTest.java | 24 +++++++++---------- .../lucene/LuceneAnalyzerFactoryTest.java | 4 ++-- .../nouveau/lucene/LuceneIndexTest.java | 4 ++-- 6 files changed, 28 insertions(+), 19 deletions(-) diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java index 21811e66fdb..de4f2d35be4 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java @@ -16,6 +16,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.PropertyNamingStrategies; import com.fasterxml.jackson.databind.annotation.JsonNaming; +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotEmpty; import java.util.Map; @@ -23,7 +25,10 @@ public class IndexDefinition { public static final int LEGACY_LUCENE_VERSION = 9; + public static final int LATEST_LUCENE_VERSION = 10; + @Min(LEGACY_LUCENE_VERSION) + @Max(LATEST_LUCENE_VERSION) private int luceneVersion = LEGACY_LUCENE_VERSION; // Legacy version if not set. @NotEmpty diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java index 1a5caf986e9..370703946fe 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java @@ -39,6 +39,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.misc.store.DirectIODirectory; import org.apache.lucene.search.SearcherFactory; import org.apache.lucene.search.SearcherManager; @@ -393,9 +394,12 @@ private Index load(final String name) throws IOException { final Path path = indexPath(name); final IndexDefinition indexDefinition = loadIndexDefinition(name); final Analyzer analyzer = LuceneAnalyzerFactory.fromDefinition(indexDefinition); - final Directory dir = new DirectIODirectory( - FSDirectory.open(path.resolve(Integer.toString(indexDefinition.getLuceneVersion())))); + final int luceneVersion = indexDefinition.getLuceneVersion(); + final Directory dir = new DirectIODirectory(FSDirectory.open(path.resolve(Integer.toString(luceneVersion)))); final IndexWriterConfig config = new IndexWriterConfig(analyzer); + if (luceneVersion != IndexDefinition.LATEST_LUCENE_VERSION) { + config.setOpenMode(OpenMode.APPEND); + } config.setUseCompoundFile(false); final IndexWriter writer = new IndexWriter(dir, config); final long updateSeq = getSeq(writer, "update_seq"); diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java index e34bebef517..7e5facb2e26 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java @@ -39,7 +39,7 @@ protected Result check() throws Exception { // Ignored, index might not exist yet. } - indexResource.createIndex(name, new IndexDefinition(IndexDefinition.LEGACY_LUCENE_VERSION, "standard", null)); + indexResource.createIndex(name, new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null)); try { final DocumentUpdateRequest documentUpdateRequest = new DocumentUpdateRequest(0, 1, null, Collections.emptyList()); diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java index 447087c7bad..7a122e67c85 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java +++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java @@ -61,8 +61,8 @@ public void cleanup() throws Exception { @Test public void managerReturnsUsableIndex() throws Exception { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); manager.create("foo", indexDefinition); var searchRequest = new SearchRequest(); searchRequest.setQuery("*:*"); @@ -72,8 +72,8 @@ public void managerReturnsUsableIndex() throws Exception { @Test public void managerReopensAClosedIndex() throws Exception { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); manager.create("bar", indexDefinition); @@ -90,8 +90,8 @@ public void managerReopensAClosedIndex() throws Exception { @Test public void deleteAllRemovesIndexByName() throws Exception { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); assertThat(countIndexes()).isEqualTo(0); manager.create("bar", indexDefinition); @@ -102,8 +102,8 @@ public void deleteAllRemovesIndexByName() throws Exception { @Test public void deleteAllRemovesIndexByPath() throws Exception { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); assertThat(countIndexes()).isEqualTo(0); manager.create("foo/bar", indexDefinition); @@ -114,8 +114,8 @@ public void deleteAllRemovesIndexByPath() throws Exception { @Test public void deleteAllRemovesIndexByGlob() throws Exception { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); assertThat(countIndexes()).isEqualTo(0); manager.create("foo/bar", indexDefinition); @@ -126,8 +126,8 @@ public void deleteAllRemovesIndexByGlob() throws Exception { @Test public void deleteAllRemovesIndexByGlobExceptExclusions() throws Exception { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); assertThat(countIndexes()).isEqualTo(0); manager.create("foo/bar", indexDefinition); diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactoryTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactoryTest.java index 2fa3c26936b..383cf34dc8a 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactoryTest.java +++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactoryTest.java @@ -257,7 +257,7 @@ public void testturkish() throws Exception { @Test public void testFieldAnalyzers() throws Exception { final IndexDefinition indexDefinition = new IndexDefinition( - IndexDefinition.LEGACY_LUCENE_VERSION, + IndexDefinition.LATEST_LUCENE_VERSION, "standard", Map.of("english", "english", "thai", "thai", "email", "email")); final Analyzer analyzer = LuceneAnalyzerFactory.fromDefinition(indexDefinition); @@ -278,7 +278,7 @@ public void testUnknownAnalyzer() throws Exception { private void assertAnalyzer(final String name, final Class clazz) throws Exception { assertThat(LuceneAnalyzerFactory.newAnalyzer(name)).isInstanceOf(clazz); assertThat(LuceneAnalyzerFactory.fromDefinition( - new IndexDefinition(IndexDefinition.LEGACY_LUCENE_VERSION, name, null))) + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, name, null))) .isInstanceOf(clazz); } } diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneIndexTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneIndexTest.java index 9206b83d9f6..f87af2fe0f7 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneIndexTest.java +++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneIndexTest.java @@ -48,8 +48,8 @@ public class LuceneIndexTest { protected final Index setup(final Path path) throws IOException { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); final Analyzer analyzer = LuceneAnalyzerFactory.fromDefinition(indexDefinition); final Directory dir = new DirectIODirectory(FSDirectory.open(path)); final IndexWriterConfig config = new IndexWriterConfig(analyzer); From 535079cf92ffe90f33134046c60d260afa4b35a0 Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Fri, 26 Sep 2025 09:44:43 +0100 Subject: [PATCH 4/9] nouveau welcome message for version negotiation --- .../couchdb/nouveau/NouveauApplication.java | 5 +++ .../couchdb/nouveau/api/WelcomeResponse.java | 34 +++++++++++++++++++ .../couchdb/nouveau/core/IndexManager.java | 1 - .../nouveau/resources/WelcomeResource.java | 30 ++++++++++++++++ 4 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 nouveau/src/main/java/org/apache/couchdb/nouveau/api/WelcomeResponse.java create mode 100644 nouveau/src/main/java/org/apache/couchdb/nouveau/resources/WelcomeResource.java diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java index a6c20290360..c2230d1eb9c 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java @@ -28,6 +28,7 @@ import org.apache.couchdb.nouveau.lucene.ParallelSearcherFactory; import org.apache.couchdb.nouveau.resources.AnalyzeResource; import org.apache.couchdb.nouveau.resources.IndexResource; +import org.apache.couchdb.nouveau.resources.WelcomeResource; import org.apache.couchdb.nouveau.tasks.CloseAllIndexesTask; public class NouveauApplication extends Application { @@ -67,6 +68,10 @@ public void run(NouveauApplicationConfiguration configuration, Environment envir // Serialization classes environment.getObjectMapper().registerModule(new LuceneModule()); + // WelcomeResource + final WelcomeResource welcomeResource = new WelcomeResource(); + environment.jersey().register(welcomeResource); + // AnalyzeResource final AnalyzeResource analyzeResource = new AnalyzeResource(); environment.jersey().register(analyzeResource); diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/WelcomeResponse.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/api/WelcomeResponse.java new file mode 100644 index 00000000000..6043da4c7ce --- /dev/null +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/api/WelcomeResponse.java @@ -0,0 +1,34 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.couchdb.nouveau.api; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.databind.annotation.JsonNaming; + +@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) +public final class WelcomeResponse { + + public static final WelcomeResponse INSTANCE = new WelcomeResponse(); + + private final int[] supportedLuceneVersions = + new int[] {IndexDefinition.LEGACY_LUCENE_VERSION, IndexDefinition.LATEST_LUCENE_VERSION}; + + private WelcomeResponse() {} + + @JsonProperty + public int[] getSupportedLuceneVersions() { + return supportedLuceneVersions; + } +} diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java index 370703946fe..4067716eed9 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java @@ -219,7 +219,6 @@ public void create(final String name, IndexDefinition indexDefinition) throws IO assertSame(indexDefinition, loadIndexDefinition(name)); return; } - final Lock lock = this.createLock.writeLock(name); lock.lock(); try { diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/WelcomeResource.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/WelcomeResource.java new file mode 100644 index 00000000000..0992623fc02 --- /dev/null +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/WelcomeResource.java @@ -0,0 +1,30 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.couchdb.nouveau.resources; + +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import org.apache.couchdb.nouveau.api.WelcomeResponse; + +@Path("/") +@Produces(MediaType.APPLICATION_JSON) +public final class WelcomeResource { + + @GET + public WelcomeResponse welcome() { + return WelcomeResponse.INSTANCE; + } +} From 88a5983ff84d9825d0a25ebca916cdb3b8a207a2 Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Sat, 23 Aug 2025 16:18:31 +0100 Subject: [PATCH 5/9] insert lucene version in new design documents if missing --- src/nouveau/include/nouveau.hrl | 1 + src/nouveau/src/nouveau_plugin_couch_db.erl | 36 +++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/src/nouveau/include/nouveau.hrl b/src/nouveau/include/nouveau.hrl index 16f46bdd9b9..efdc6eb7e62 100644 --- a/src/nouveau/include/nouveau.hrl +++ b/src/nouveau/include/nouveau.hrl @@ -12,6 +12,7 @@ %% limitations under the License. -define(LEGACY_LUCENE_VERSION, 9). +-define(TARGET_LUCENE_VERSION, 10). -record(index, { dbname, diff --git a/src/nouveau/src/nouveau_plugin_couch_db.erl b/src/nouveau/src/nouveau_plugin_couch_db.erl index dcd3ae1f156..d10d7adfc4d 100644 --- a/src/nouveau/src/nouveau_plugin_couch_db.erl +++ b/src/nouveau/src/nouveau_plugin_couch_db.erl @@ -13,10 +13,46 @@ -module(nouveau_plugin_couch_db). -export([ + before_doc_update/3, is_valid_purge_client/2, on_compact/2 ]). +-include("nouveau.hrl"). +-include_lib("couch/include/couch_db.hrl"). + +%% New index definitions get an explicit lucene version property, if missing. +before_doc_update( + #doc{id = <>, revs = {0, []}} = Doc, + Db, + ?INTERACTIVE_EDIT = UpdateType +) -> + #doc{body = {Fields}} = Doc, + case couch_util:get_value(<<"nouveau">>, Fields) of + {Indexes} when is_list(Indexes) -> + [add_versions_to_doc(Doc), Db, UpdateType]; + _ -> + [Doc, Db, UpdateType] + end; +before_doc_update(Doc, Db, UpdateType) -> + [Doc, Db, UpdateType]. + +add_versions_to_doc(#doc{} = Doc) -> + #doc{body = {Fields0}} = Doc, + {Indexes0} = couch_util:get_value(<<"nouveau">>, Fields0), + Indexes1 = lists:map(fun add_version_to_index/1, Indexes0), + Fields1 = couch_util:set_value(<<"nouveau">>, Fields0, {Indexes1}), + Doc#doc{body = {Fields1}}. + +add_version_to_index({IndexName, {Index}}) -> + case couch_util:get_value(<<"lucene_version">>, Index) of + undefined -> + {IndexName, + {couch_util:set_value(<<"lucene_version">>, Index, ?TARGET_LUCENE_VERSION)}}; + _ -> + {IndexName, {Index}} + end. + is_valid_purge_client(DbName, Props) -> nouveau_util:verify_index_exists(DbName, Props). From 1943b14443273b5b54dbc286a6521ec622c8df86 Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Sat, 23 Aug 2025 19:45:17 +0100 Subject: [PATCH 6/9] add scanner to upgrade nouveau indexes --- rel/overlay/etc/default.ini | 7 + src/nouveau/src/nouveau_api.erl | 13 ++ src/nouveau/src/nouveau_fabric_search.erl | 6 +- src/nouveau/src/nouveau_index_upgrader.erl | 165 ++++++++++++++++++ .../eunit/nouveau_index_upgrader_tests.erl | 131 ++++++++++++++ 5 files changed, 319 insertions(+), 3 deletions(-) create mode 100644 src/nouveau/src/nouveau_index_upgrader.erl create mode 100644 src/nouveau/test/eunit/nouveau_index_upgrader_tests.erl diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index db4d37f3e17..f1537575736 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -1080,6 +1080,7 @@ url = {{nouveau_url}} ;couch_scanner_plugin_find = false ;couch_scanner_plugin_conflict_finder = false ;couch_quickjs_scanner_plugin = false +;nouveau_index_upgrader = false ; The following [$plugin*] settings apply to all plugins @@ -1186,6 +1187,12 @@ url = {{nouveau_url}} ; Scanner settings to skip dbs and docs would also work: ;[couch_quickjs_scanner_plugin.skip_{dbs,ddoc,docs}] +[nouveau_index_upgrader] +; Common scanner scheduling settings +;after = restart +;repeat = restart + + [chttpd_auth_lockout] ; CouchDB can temporarily lock out IP addresses that repeatedly fail authentication ; mode can be set to one of three recognised values; diff --git a/src/nouveau/src/nouveau_api.erl b/src/nouveau/src/nouveau_api.erl index cfc88af4f7e..f1bd1d9f87c 100644 --- a/src/nouveau/src/nouveau_api.erl +++ b/src/nouveau/src/nouveau_api.erl @@ -29,6 +29,7 @@ search/2, set_purge_seq/3, set_update_seq/3, + supported_lucene_versions/0, jaxrs_error/2 ]). @@ -214,6 +215,18 @@ set_seq(#index{} = Index, ReqBody) -> send_error(Reason) end. +supported_lucene_versions() -> + Resp = send_if_enabled(<<"/">>, [], <<"GET">>), + case Resp of + {ok, 200, _, RespBody} -> + Json = jiffy:decode(RespBody, [return_maps]), + {ok, maps:get(<<"supported_lucene_versions">>, Json, [])}; + {ok, StatusCode, _, RespBody} -> + {error, jaxrs_error(StatusCode, RespBody)}; + {error, Reason} -> + send_error(Reason) + end. + %% private functions index_path(Path) when is_binary(Path) -> diff --git a/src/nouveau/src/nouveau_fabric_search.erl b/src/nouveau/src/nouveau_fabric_search.erl index 3ec9d96fd6e..b082f6ff973 100644 --- a/src/nouveau/src/nouveau_fabric_search.erl +++ b/src/nouveau/src/nouveau_fabric_search.erl @@ -15,7 +15,7 @@ -module(nouveau_fabric_search). --export([go/4]). +-export([go/3, go/4]). -include_lib("mem3/include/mem3.hrl"). -include_lib("couch/include/couch_db.hrl"). @@ -38,12 +38,12 @@ go(DbName, GroupId, IndexName, QueryArgs0) when is_binary(GroupId) -> go(DbName, #doc{} = DDoc, IndexName, QueryArgs0) -> case nouveau_util:design_doc_to_index(DbName, DDoc, IndexName) of {ok, Index} -> - go(DbName, DDoc, IndexName, QueryArgs0, Index); + go(DbName, QueryArgs0, Index); {error, Reason} -> {error, Reason} end. -go(DbName, #doc{} = _DDoc, _IndexName, QueryArgs0, Index) -> +go(DbName, QueryArgs0, Index) -> Shards = get_shards(DbName, QueryArgs0), {PackedBookmark, #{limit := Limit, sort := Sort} = QueryArgs1} = maps:take(bookmark, QueryArgs0), diff --git a/src/nouveau/src/nouveau_index_upgrader.erl b/src/nouveau/src/nouveau_index_upgrader.erl new file mode 100644 index 00000000000..9f757cf5577 --- /dev/null +++ b/src/nouveau/src/nouveau_index_upgrader.erl @@ -0,0 +1,165 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(nouveau_index_upgrader). +-behaviour(couch_scanner_plugin). + +-export([ + start/2, + resume/2, + complete/1, + checkpoint/1, + db/2, + ddoc/3 +]). + +-include("nouveau.hrl"). +-include_lib("couch_scanner/include/couch_scanner_plugin.hrl"). + +start(ScanId, #{}) -> + St = init_config(ScanId), + case should_run(St) of + true -> + ?INFO("Starting.", [], St), + {ok, St}; + false -> + ?INFO("Not starting.", [], St), + skip + end. + +resume(ScanId, #{}) -> + St = init_config(ScanId), + case should_run(St) of + true -> + ?INFO("Resuming.", [], St), + {ok, St}; + false -> + ?INFO("Not resuming.", [], St), + skip + end. + +complete(St) -> + ?INFO("Completed", [], St), + {ok, #{}}. + +checkpoint(_St) -> + {ok, #{}}. + +db(St, _DbName) -> + {ok, St}. + +ddoc(St, _DbName, #doc{id = <<"_design/_", _/binary>>}) -> + {ok, St}; +ddoc(St, DbName, #doc{} = DDoc0) -> + case update_ddoc_versions(DDoc0) of + DDoc0 -> + ok; + DDoc1 -> + Indexes = nouveau_util:design_doc_to_indexes(DbName, DDoc1), + case upgrade_indexes(DbName, Indexes) of + true -> + save_ddoc(DbName, DDoc1); + false -> + ok + end + end, + {ok, St}. + +upgrade_indexes(_DbName, []) -> + true; +upgrade_indexes(DbName, [Index | Rest]) -> + case upgrade_index(DbName, Index) of + true -> + upgrade_indexes(DbName, Rest); + false -> + false + end. + +upgrade_index(DbName, #index{} = Index) -> + ?INFO("Upgrading ~s/~s/~s to version ~B", [ + DbName, + Index#index.ddoc_id, + Index#index.name, + ?TARGET_LUCENE_VERSION + ]), + case + nouveau_fabric_search:go( + DbName, + #{query => <<"*:*">>, bookmark => null, sort => null, limit => 1}, + Index#index{lucene_version = ?TARGET_LUCENE_VERSION} + ) + of + {ok, _SearchResults} -> + true; + {error, _Reason} -> + false + end. + +update_ddoc_versions(#doc{} = Doc) -> + #doc{body = {Fields0}} = Doc, + {Indexes0} = couch_util:get_value(<<"nouveau">>, Fields0), + Indexes1 = lists:map(fun update_version/1, Indexes0), + Fields1 = couch_util:set_value(<<"nouveau">>, Fields0, {Indexes1}), + Doc#doc{body = {Fields1}}. + +save_ddoc(DbName, #doc{} = DDoc) -> + {Pid, Ref} = spawn_monitor(fun() -> + case fabric:update_doc(DbName, DDoc, [?ADMIN_CTX]) of + {ok, _} -> + exit(ok); + Else -> + exit(Else) + end + end), + receive + {'DOWN', Ref, process, Pid, ok} -> + ?INFO( + "Updated ~s/~s indexes to version ~B", [DbName, DDoc#doc.id, ?TARGET_LUCENE_VERSION] + ); + {'DOWN', Ref, process, Pid, Else} -> + ?INFO("Failed to update ~s/~s for reason ~p", [DbName, DDoc#doc.id, Else]) + end. + +update_version({IndexName, {Index}}) -> + {IndexName, {couch_util:set_value(<<"lucene_version">>, Index, ?TARGET_LUCENE_VERSION)}}. + +init_config(ScanId) -> + #{sid => ScanId}. + +should_run(St) -> + couch_scanner_util:on_first_node() andalso upgrade_supported(St). + +upgrade_supported(St) -> + case nouveau_api:supported_lucene_versions() of + {ok, Versions} -> + case lists:member(?TARGET_LUCENE_VERSION, Versions) of + true -> + ?INFO( + "Nouveau server supports upgrades to Lucene ~B", + [?TARGET_LUCENE_VERSION], + St + ), + true; + false -> + ?WARN( + "Nouveau server does not support upgrades to Lucene ~B", + [?TARGET_LUCENE_VERSION], + St + ), + false + end; + {error, Reason} -> + ?ERR( + "Nouveau server upgrade check failed for reason ~p", [Reason], St + ), + false + end. diff --git a/src/nouveau/test/eunit/nouveau_index_upgrader_tests.erl b/src/nouveau/test/eunit/nouveau_index_upgrader_tests.erl new file mode 100644 index 00000000000..d37a1d0071b --- /dev/null +++ b/src/nouveau/test/eunit/nouveau_index_upgrader_tests.erl @@ -0,0 +1,131 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(nouveau_index_upgrader_tests). + +-include_lib("couch/include/couch_eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). +-include_lib("nouveau/include/nouveau.hrl"). + +-define(PLUGIN, nouveau_index_upgrader). + +nouveau_index_upgrader_test_() -> + { + foreach, + fun setup/0, + fun teardown/1, + [ + ?TDEF_FE(t_upgrade_legacy_index, 10), + ?TDEF_FE(t_dont_upgrade_latest_index, 10) + ] + }. + +setup() -> + {module, _} = code:ensure_loaded(?PLUGIN), + meck:new(?PLUGIN, [passthrough]), + meck:new(couch_scanner_server, [passthrough]), + meck:new(couch_scanner_util, [passthrough]), + meck:new(nouveau_api, [passthrough]), + meck:expect(nouveau_api, supported_lucene_versions, fun() -> + {ok, [?LEGACY_LUCENE_VERSION, ?TARGET_LUCENE_VERSION]} + end), + Ctx = test_util:start_couch([fabric, couch_scanner]), + DbName = ?tempdb(), + ok = fabric:create_db(DbName, [{q, "2"}, {n, "1"}]), + config:set(atom_to_list(?PLUGIN), "max_batch_items", "1", false), + reset_stats(), + {Ctx, DbName}. + +teardown({Ctx, DbName}) -> + config_delete_section("couch_scanner"), + config_delete_section("couch_scanner_plugins"), + config_delete_section(atom_to_list(?PLUGIN)), + couch_scanner:reset_checkpoints(), + couch_scanner:resume(), + fabric:delete_db(DbName), + test_util:stop_couch(Ctx), + meck:unload(). + +t_upgrade_legacy_index({_, DbName}) -> + DDocId = <<"_design/foo">>, + IndexName = <<"bar">>, + ok = add_ddoc(DbName, DDocId, IndexName, ?LEGACY_LUCENE_VERSION), + meck:reset(couch_scanner_server), + meck:reset(?PLUGIN), + meck:new(nouveau_fabric_search, [passthrough]), + meck:expect(nouveau_fabric_search, go, fun(_DbName, _Args, _Index) -> {ok, []} end), + config:set("couch_scanner_plugins", atom_to_list(?PLUGIN), "true", false), + wait_exit(10000), + ?assertEqual(1, num_calls(start, 2)), + ?assertEqual(1, num_calls(complete, 1)), + ?assertEqual(?TARGET_LUCENE_VERSION, get_lucene_version(DbName, DDocId, IndexName)), + ok. + +t_dont_upgrade_latest_index({_, DbName}) -> + DDocId = <<"_design/foo">>, + IndexName = <<"bar">>, + ok = add_ddoc(DbName, DDocId, IndexName, ?TARGET_LUCENE_VERSION), + meck:reset(couch_scanner_server), + meck:reset(?PLUGIN), + config:set("couch_scanner_plugins", atom_to_list(?PLUGIN), "true", false), + wait_exit(10000), + ?assertEqual(1, num_calls(start, 2)), + ?assertEqual(1, num_calls(complete, 1)), + ?assertEqual(?TARGET_LUCENE_VERSION, get_lucene_version(DbName, DDocId, IndexName)), + ok. + +reset_stats() -> + Counters = [ + [couchdb, query_server, process_error_exits], + [couchdb, query_server, process_errors], + [couchdb, query_server, process_exits] + ], + [reset_counter(C) || C <- Counters]. + +reset_counter(Counter) -> + case couch_stats:sample(Counter) of + 0 -> + ok; + N when is_integer(N), N > 0 -> + couch_stats:decrement_counter(Counter, N) + end. + +config_delete_section(Section) -> + [config:delete(K, V, false) || {K, V} <- config:get(Section)]. + +add_ddoc(DbName, DDocId, IndexName, LuceneVersion) -> + {ok, _} = fabric:update_doc(DbName, mkddoc(DDocId, IndexName, LuceneVersion), [?ADMIN_CTX]), + ok. + +get_lucene_version(DbName, DDocId, IndexName) -> + {ok, #doc{body = {Props}}} = fabric:open_doc(DbName, DDocId, [?ADMIN_CTX]), + {Indexes} = couch_util:get_value(<<"nouveau">>, Props), + {Index} = couch_util:get_value(IndexName, Indexes), + couch_util:get_value(<<"lucene_version">>, Index). + +mkddoc(DocId, IndexName, LuceneVersion) -> + Body = #{ + <<"_id">> => DocId, + <<"nouveau">> => #{ + IndexName => #{ + <<"lucene_version">> => LuceneVersion, + <<"index">> => <<"function(doc){}">> + } + } + }, + jiffy:decode(jiffy:encode(Body)). + +num_calls(Fun, Args) -> + meck:num_calls(?PLUGIN, Fun, Args). + +wait_exit(MSec) -> + meck:wait(couch_scanner_server, handle_info, [{'EXIT', '_', '_'}, '_'], MSec). From 191763f9eb217c80cc43ed1b8171b24d546a52b7 Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Tue, 26 Aug 2025 13:06:25 +0100 Subject: [PATCH 7/9] fix formatting error in previous PR --- src/nouveau/src/nouveau_util.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/nouveau/src/nouveau_util.erl b/src/nouveau/src/nouveau_util.erl index dbc30927a33..892ad48271f 100644 --- a/src/nouveau/src/nouveau_util.erl +++ b/src/nouveau/src/nouveau_util.erl @@ -84,13 +84,13 @@ design_doc_to_index(DbName, #doc{id = Id, body = {Fields}}, IndexName) -> {LuceneVersion, DefaultAnalyzer, FieldAnalyzers, Def} end, Sig = couch_util:to_hex_bin( - crypto:hash( - sha256, - ?term_to_bin( - SigTerm - ) + crypto:hash( + sha256, + ?term_to_bin( + SigTerm ) - ), + ) + ), {ok, #index{ dbname = DbName, lucene_version = LuceneVersion, From 2a7aff155757c200613f852119ef1bb726b77095 Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Wed, 3 Sep 2025 13:33:14 +0100 Subject: [PATCH 8/9] document nouveau handling of lucene 9 & 10 --- nouveau/README.md | 6 ++--- src/docs/src/api/database/cleanup.rst | 2 ++ src/docs/src/ddocs/nouveau.rst | 33 +++++++++++++++++++++++++++ src/docs/src/install/nouveau.rst | 2 +- 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/nouveau/README.md b/nouveau/README.md index 2ebacd9612f..97825c82035 100644 --- a/nouveau/README.md +++ b/nouveau/README.md @@ -3,8 +3,8 @@ Nouveau is a modern replacement for dreyfus/clouseau and is built on; 1) the Dropwizard framework (https://dropwizard.io) -2) Java 11+ -3) Lucene 9 +2) Java 21+ +3) Lucene 10 Nouveau transforms Apache CouchDB databases into Apache Lucene indexes at the shard level and then merges the results together. @@ -23,7 +23,7 @@ This work is currently EXPERIMENTAL and may change in ways that invalidate any e * integration with resharding * update=false * `_nouveau_info` -* `_search_cleanup` +* `_nouveau_cleanup` * /openapi.{json.yaml} ## What doesn't work yet? diff --git a/src/docs/src/api/database/cleanup.rst b/src/docs/src/api/database/cleanup.rst index 6193888e90a..d93eb301a13 100644 --- a/src/docs/src/api/database/cleanup.rst +++ b/src/docs/src/api/database/cleanup.rst @@ -58,6 +58,8 @@ "ok": true } +.. _api/db/nouveau_cleanup: + ========================== ``/{db}/_nouveau_cleanup`` ========================== diff --git a/src/docs/src/ddocs/nouveau.rst b/src/docs/src/ddocs/nouveau.rst index d091ca2e8f7..d7626cfc5df 100644 --- a/src/docs/src/ddocs/nouveau.rst +++ b/src/docs/src/ddocs/nouveau.rst @@ -63,6 +63,39 @@ results from deeper in the result set. A nouveau index will inherit the partitioning type from the ``options.partitioned`` field of the design document that contains it. +.. _ddoc/nouveau/lucene_upgrade: + +Lucene Version Upgrade +====================== + +Nouveau has been upgraded to use Lucene 10, earlier releases used Lucene 9. + +Nouveau can query and update indexes created by Lucene 9 but will not create new +ones. The index definition can optionally define a ``lucene_version`` field +(which must be either 9 or 10 expressed as an integer). If not specified +when defining a new index the current version (10) will be automatically +added to the definition. + +As Lucene only supports indexes up to one major release behind the current, it +is important to rebuild all indexes to the current release. As Lucene major +releases are infrequent, and Nouveau supports 9 and 10 versions simultaneously +it is only necessary to rebuild version 9 indexes before Nouveau upgrades to +Lucene 11 (when it exists). A ``couch_scanner`` plugin is available to +automate this process, and can be enabled as follows; + +.. code-block:: ini + + [couch_scanner_plugins] + nouveau_index_upgrader = true + +The plugin will scan all design documents for index definitions either with no +``lucene_version`` field or one equal to a previous version (lower than +10). The new index will be built by the plugin and, on successful +completion, will update the ``lucene_version`` field in the index +definition. Search requests against that index will seamlessly switch from the +old index to the new one. Invoking the :ref:`_nouveau_cleanup ` +will delete the old indexes. + .. _ddoc/nouveau/field_types: Field Types diff --git a/src/docs/src/install/nouveau.rst b/src/docs/src/install/nouveau.rst index 0dc031914f3..152a2fa2f23 100644 --- a/src/docs/src/install/nouveau.rst +++ b/src/docs/src/install/nouveau.rst @@ -25,7 +25,7 @@ service that embeds `Apache Lucene `_. Typically, thi service is installed on the same host as CouchDB and communicates with it over the loopback network. -Nouveau server is runtime-compatible with Java 11 or higher. +Nouveau server is runtime-compatible with Java 21 or higher. Enable Nouveau ============== From e8b21769666fc97fcc7df33d76fe13417e933b32 Mon Sep 17 00:00:00 2001 From: Robert Newson Date: Tue, 9 Sep 2025 15:23:15 +0100 Subject: [PATCH 9/9] Move up to match Lucene source compatibility --- nouveau/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nouveau/build.gradle b/nouveau/build.gradle index fd159525a36..23db6f77e35 100644 --- a/nouveau/build.gradle +++ b/nouveau/build.gradle @@ -46,7 +46,7 @@ group = 'org.apache.couchdb' version = '1.0-SNAPSHOT' java { - sourceCompatibility = "11" + sourceCompatibility = "21" } jar {