diff --git a/nouveau/README.md b/nouveau/README.md index 2ebacd9612f..97825c82035 100644 --- a/nouveau/README.md +++ b/nouveau/README.md @@ -3,8 +3,8 @@ Nouveau is a modern replacement for dreyfus/clouseau and is built on; 1) the Dropwizard framework (https://dropwizard.io) -2) Java 11+ -3) Lucene 9 +2) Java 21+ +3) Lucene 10 Nouveau transforms Apache CouchDB databases into Apache Lucene indexes at the shard level and then merges the results together. @@ -23,7 +23,7 @@ This work is currently EXPERIMENTAL and may change in ways that invalidate any e * integration with resharding * update=false * `_nouveau_info` -* `_search_cleanup` +* `_nouveau_cleanup` * /openapi.{json.yaml} ## What doesn't work yet? diff --git a/nouveau/build.gradle b/nouveau/build.gradle index 356295ee163..23db6f77e35 100644 --- a/nouveau/build.gradle +++ b/nouveau/build.gradle @@ -23,7 +23,7 @@ dependencies { implementation 'io.dropwizard.metrics:metrics-jersey2' testImplementation 'io.dropwizard:dropwizard-testing' - def luceneVersion = '9.12.1' + def luceneVersion = '10.3.0' implementation group: 'org.apache.lucene', name: 'lucene-core', version: luceneVersion implementation group: 'org.apache.lucene', name: 'lucene-queryparser', version: luceneVersion implementation group: 'org.apache.lucene', name: 'lucene-analysis-common', version: luceneVersion @@ -46,7 +46,7 @@ group = 'org.apache.couchdb' version = '1.0-SNAPSHOT' java { - sourceCompatibility = "11" + sourceCompatibility = "21" } jar { diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java index 7179eadc06a..c2230d1eb9c 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/NouveauApplication.java @@ -24,10 +24,11 @@ import org.apache.couchdb.nouveau.core.UserAgentFilter; import org.apache.couchdb.nouveau.health.AnalyzeHealthCheck; import org.apache.couchdb.nouveau.health.IndexHealthCheck; -import org.apache.couchdb.nouveau.lucene9.Lucene9Module; -import org.apache.couchdb.nouveau.lucene9.ParallelSearcherFactory; +import org.apache.couchdb.nouveau.lucene.LuceneModule; +import org.apache.couchdb.nouveau.lucene.ParallelSearcherFactory; import org.apache.couchdb.nouveau.resources.AnalyzeResource; import org.apache.couchdb.nouveau.resources.IndexResource; +import org.apache.couchdb.nouveau.resources.WelcomeResource; import org.apache.couchdb.nouveau.tasks.CloseAllIndexesTask; public class NouveauApplication extends Application { @@ -65,7 +66,11 @@ public void run(NouveauApplicationConfiguration configuration, Environment envir environment.lifecycle().manage(indexManager); // Serialization classes - environment.getObjectMapper().registerModule(new Lucene9Module()); + environment.getObjectMapper().registerModule(new LuceneModule()); + + // WelcomeResource + final WelcomeResource welcomeResource = new WelcomeResource(); + environment.jersey().register(welcomeResource); // AnalyzeResource final AnalyzeResource analyzeResource = new AnalyzeResource(); diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java index 3d79fca654e..de4f2d35be4 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/api/IndexDefinition.java @@ -16,12 +16,21 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.PropertyNamingStrategies; import com.fasterxml.jackson.databind.annotation.JsonNaming; +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotEmpty; import java.util.Map; @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) public class IndexDefinition { + public static final int LEGACY_LUCENE_VERSION = 9; + public static final int LATEST_LUCENE_VERSION = 10; + + @Min(LEGACY_LUCENE_VERSION) + @Max(LATEST_LUCENE_VERSION) + private int luceneVersion = LEGACY_LUCENE_VERSION; // Legacy version if not set. + @NotEmpty private String defaultAnalyzer; @@ -31,11 +40,22 @@ public IndexDefinition() { // Jackson deserialization } - public IndexDefinition(final String defaultAnalyzer, final Map fieldAnalyzers) { + public IndexDefinition( + final int luceneVersion, final String defaultAnalyzer, final Map fieldAnalyzers) { + this.luceneVersion = luceneVersion; this.defaultAnalyzer = defaultAnalyzer; this.fieldAnalyzers = fieldAnalyzers; } + @JsonProperty + public int getLuceneVersion() { + return luceneVersion; + } + + public void setLuceneVersion(int luceneVersion) { + this.luceneVersion = luceneVersion; + } + @JsonProperty public String getDefaultAnalyzer() { return defaultAnalyzer; @@ -62,6 +82,7 @@ public boolean hasFieldAnalyzers() { public int hashCode() { final int prime = 31; int result = 1; + result = prime * result + luceneVersion; result = prime * result + ((defaultAnalyzer == null) ? 0 : defaultAnalyzer.hashCode()); result = prime * result + ((fieldAnalyzers == null) ? 0 : fieldAnalyzers.hashCode()); return result; @@ -73,6 +94,7 @@ public boolean equals(Object obj) { if (obj == null) return false; if (getClass() != obj.getClass()) return false; IndexDefinition other = (IndexDefinition) obj; + if (luceneVersion != other.luceneVersion) return false; if (defaultAnalyzer == null) { if (other.defaultAnalyzer != null) return false; } else if (!defaultAnalyzer.equals(other.defaultAnalyzer)) return false; @@ -84,6 +106,7 @@ public boolean equals(Object obj) { @Override public String toString() { - return "IndexDefinition [defaultAnalyzer=" + defaultAnalyzer + ", fieldAnalyzers=" + fieldAnalyzers + "]"; + return "IndexDefinition [luceneVersion=" + luceneVersion + ", defaultAnalyzer=" + defaultAnalyzer + + ", fieldAnalyzers=" + fieldAnalyzers + "]"; } } diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/api/WelcomeResponse.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/api/WelcomeResponse.java new file mode 100644 index 00000000000..6043da4c7ce --- /dev/null +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/api/WelcomeResponse.java @@ -0,0 +1,34 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.couchdb.nouveau.api; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.databind.annotation.JsonNaming; + +@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) +public final class WelcomeResponse { + + public static final WelcomeResponse INSTANCE = new WelcomeResponse(); + + private final int[] supportedLuceneVersions = + new int[] {IndexDefinition.LEGACY_LUCENE_VERSION, IndexDefinition.LATEST_LUCENE_VERSION}; + + private WelcomeResponse() {} + + @JsonProperty + public int[] getSupportedLuceneVersions() { + return supportedLuceneVersions; + } +} diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java index c4f59f7a15f..4067716eed9 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/core/IndexManager.java @@ -34,11 +34,12 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.couchdb.nouveau.api.IndexDefinition; -import org.apache.couchdb.nouveau.lucene9.Lucene9AnalyzerFactory; -import org.apache.couchdb.nouveau.lucene9.Lucene9Index; +import org.apache.couchdb.nouveau.lucene.LuceneAnalyzerFactory; +import org.apache.couchdb.nouveau.lucene.LuceneIndex; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.misc.store.DirectIODirectory; import org.apache.lucene.search.SearcherFactory; import org.apache.lucene.search.SearcherManager; @@ -218,7 +219,6 @@ public void create(final String name, IndexDefinition indexDefinition) throws IO assertSame(indexDefinition, loadIndexDefinition(name)); return; } - final Lock lock = this.createLock.writeLock(name); lock.lock(); try { @@ -392,15 +392,19 @@ private Index load(final String name) throws IOException { LOGGER.info("opening {}", name); final Path path = indexPath(name); final IndexDefinition indexDefinition = loadIndexDefinition(name); - final Analyzer analyzer = Lucene9AnalyzerFactory.fromDefinition(indexDefinition); - final Directory dir = new DirectIODirectory(FSDirectory.open(path.resolve("9"))); + final Analyzer analyzer = LuceneAnalyzerFactory.fromDefinition(indexDefinition); + final int luceneVersion = indexDefinition.getLuceneVersion(); + final Directory dir = new DirectIODirectory(FSDirectory.open(path.resolve(Integer.toString(luceneVersion)))); final IndexWriterConfig config = new IndexWriterConfig(analyzer); + if (luceneVersion != IndexDefinition.LATEST_LUCENE_VERSION) { + config.setOpenMode(OpenMode.APPEND); + } config.setUseCompoundFile(false); final IndexWriter writer = new IndexWriter(dir, config); final long updateSeq = getSeq(writer, "update_seq"); final long purgeSeq = getSeq(writer, "purge_seq"); final SearcherManager searcherManager = new SearcherManager(writer, searcherFactory); - return new Lucene9Index(analyzer, writer, updateSeq, purgeSeq, searcherManager); + return new LuceneIndex(analyzer, writer, updateSeq, purgeSeq, searcherManager); } private long getSeq(final IndexWriter writer, final String key) throws IOException { diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java index 0ee8fefa2e7..7e5facb2e26 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/health/IndexHealthCheck.java @@ -32,14 +32,14 @@ public IndexHealthCheck(final IndexResource indexResource) { @Override protected Result check() throws Exception { - final String name = "___test9"; + final String name = "___test"; try { indexResource.deletePath(name, null); } catch (IOException e) { // Ignored, index might not exist yet. } - indexResource.createIndex(name, new IndexDefinition("standard", null)); + indexResource.createIndex(name, new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null)); try { final DocumentUpdateRequest documentUpdateRequest = new DocumentUpdateRequest(0, 1, null, Collections.emptyList()); diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactory.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactory.java similarity index 97% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactory.java rename to nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactory.java index 2bd47ed9740..b95230cec7b 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactory.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactory.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Response.Status; @@ -60,9 +60,9 @@ import org.apache.lucene.analysis.th.ThaiAnalyzer; import org.apache.lucene.analysis.tr.TurkishAnalyzer; -public final class Lucene9AnalyzerFactory { +public final class LuceneAnalyzerFactory { - private Lucene9AnalyzerFactory() {} + private LuceneAnalyzerFactory() {} public static Analyzer fromDefinition(final IndexDefinition indexDefinition) { final Analyzer defaultAnalyzer = newAnalyzer(indexDefinition.getDefaultAnalyzer()); diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndex.java similarity index 97% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java rename to nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndex.java index dad3040f328..e62ffdb5a96 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Index.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndex.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Response.Status; @@ -97,7 +97,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -public class Lucene9Index extends Index { +public class LuceneIndex extends Index { private static final Sort DEFAULT_SORT = new Sort(SortField.FIELD_SCORE, new SortField("_id", SortField.Type.STRING)); @@ -106,9 +106,9 @@ public class Lucene9Index extends Index { private final Analyzer analyzer; private final IndexWriter writer; private final SearcherManager searcherManager; - private final Lucene9IndexSchema schema; + private final LuceneIndexSchema schema; - public Lucene9Index( + public LuceneIndex( final Analyzer analyzer, final IndexWriter writer, final long updateSeq, @@ -290,8 +290,8 @@ private void collectHits(final IndexSearcher searcher, final TopDocs topDocs, fi hits.add(new SearchHit(doc.get("_id"), after, fields)); } - searchResults.setTotalHits(topDocs.totalHits.value); - searchResults.setTotalHitsRelation(topDocs.totalHits.relation); + searchResults.setTotalHits(topDocs.totalHits.value()); + searchResults.setTotalHitsRelation(topDocs.totalHits.relation()); searchResults.setHits(hits); } @@ -537,22 +537,22 @@ private Query parse(final SearchRequest request) { return result; } - private Lucene9IndexSchema initSchema(IndexWriter writer) { + private LuceneIndexSchema initSchema(IndexWriter writer) { var commitData = writer.getLiveCommitData(); if (commitData == null) { - return Lucene9IndexSchema.emptySchema(); + return LuceneIndexSchema.emptySchema(); } for (var entry : commitData) { if (entry.getKey().equals("_schema")) { - return Lucene9IndexSchema.fromString(entry.getValue()); + return LuceneIndexSchema.fromString(entry.getValue()); } } - return Lucene9IndexSchema.emptySchema(); + return LuceneIndexSchema.emptySchema(); } @Override public String toString() { - return "Lucene9Index [analyzer=" + analyzer + ", writer=" + writer + ", searcherManager=" + searcherManager + return "LuceneIndex [analyzer=" + analyzer + ", writer=" + writer + ", searcherManager=" + searcherManager + "]"; } diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexSchema.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndexSchema.java similarity index 91% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexSchema.java rename to nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndexSchema.java index 92cc5fc2e8f..ef335bf5a20 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexSchema.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneIndexSchema.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Response.Status; @@ -32,7 +32,7 @@ import org.apache.couchdb.nouveau.api.TextField; import org.apache.lucene.queryparser.flexible.standard.config.PointsConfig; -final class Lucene9IndexSchema { +final class LuceneIndexSchema { public enum Type { STRING, @@ -56,23 +56,23 @@ private static Type fromField(final Field field) { private final ConcurrentMap map; - private Lucene9IndexSchema(Map map) { + private LuceneIndexSchema(Map map) { this.map = new ConcurrentHashMap<>(map); this.map.put("_id", Type.STRING); } - public static Lucene9IndexSchema emptySchema() { - return new Lucene9IndexSchema(new HashMap()); + public static LuceneIndexSchema emptySchema() { + return new LuceneIndexSchema(new HashMap()); } - public static Lucene9IndexSchema fromString(final String schemaStr) { + public static LuceneIndexSchema fromString(final String schemaStr) { Objects.requireNonNull(schemaStr); if (schemaStr.isEmpty()) { return emptySchema(); } var map = Arrays.stream(schemaStr.split(",")) .collect(Collectors.toMap(i -> i.split(":")[0], i -> Type.valueOf(i.split(":")[1]))); - return new Lucene9IndexSchema(map); + return new LuceneIndexSchema(map); } public void update(final Collection fields) { diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Module.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneModule.java similarity index 82% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Module.java rename to nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneModule.java index 06102de84cb..35be8efe27f 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/Lucene9Module.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/LuceneModule.java @@ -11,16 +11,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import com.fasterxml.jackson.core.Version; import com.fasterxml.jackson.databind.module.SimpleModule; import org.apache.lucene.search.Query; -public class Lucene9Module extends SimpleModule { +public class LuceneModule extends SimpleModule { - public Lucene9Module() { - super("lucene9", Version.unknownVersion()); + public LuceneModule() { + super("lucene", Version.unknownVersion()); // Query addSerializer(Query.class, new QuerySerializer()); diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParser.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParser.java similarity index 98% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParser.java rename to nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParser.java index fc7acf5091d..d37ab3d37c5 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParser.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParser.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import java.util.Map; import org.apache.lucene.analysis.Analyzer; diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/ParallelSearcherFactory.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/ParallelSearcherFactory.java similarity index 96% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/ParallelSearcherFactory.java rename to nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/ParallelSearcherFactory.java index 4553fa76b2f..bd9050af86f 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/ParallelSearcherFactory.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/ParallelSearcherFactory.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import java.io.IOException; import java.util.concurrent.Executor; diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QueryDeserializer.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QueryDeserializer.java similarity index 99% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QueryDeserializer.java rename to nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QueryDeserializer.java index b0620e661dc..04a771cf52f 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QueryDeserializer.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QueryDeserializer.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QuerySerializer.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QuerySerializer.java similarity index 97% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QuerySerializer.java rename to nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QuerySerializer.java index 5e1d5087193..7e42ee6dbdd 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/QuerySerializer.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/QuerySerializer.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import com.fasterxml.jackson.core.JsonGenerationException; import com.fasterxml.jackson.core.JsonGenerator; @@ -65,8 +65,8 @@ public void serialize(final Query query, final JsonGenerator gen, final Serializ for (final BooleanClause clause : booleanQuery.clauses()) { gen.writeStartObject(); gen.writeFieldName("query"); - serialize(clause.getQuery(), gen, provider); - gen.writeStringField("occur", clause.getOccur().name().toLowerCase()); + serialize(clause.query(), gen, provider); + gen.writeStringField("occur", clause.occur().name().toLowerCase()); gen.writeEndObject(); } gen.writeEndArray(); diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/SimpleAsciiFoldingAnalyzer.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/SimpleAsciiFoldingAnalyzer.java similarity index 96% rename from nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/SimpleAsciiFoldingAnalyzer.java rename to nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/SimpleAsciiFoldingAnalyzer.java index 6b4c8c64222..464b49d6370 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene9/SimpleAsciiFoldingAnalyzer.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/lucene/SimpleAsciiFoldingAnalyzer.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java index 2ae8b78f10a..d5249e20585 100644 --- a/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/AnalyzeResource.java @@ -30,7 +30,7 @@ import java.util.List; import org.apache.couchdb.nouveau.api.AnalyzeRequest; import org.apache.couchdb.nouveau.api.AnalyzeResponse; -import org.apache.couchdb.nouveau.lucene9.Lucene9AnalyzerFactory; +import org.apache.couchdb.nouveau.lucene.LuceneAnalyzerFactory; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; @@ -47,7 +47,7 @@ public final class AnalyzeResource { public AnalyzeResponse analyzeText(@NotNull @Valid AnalyzeRequest request) throws IOException { try { final List tokens = - tokenize(Lucene9AnalyzerFactory.newAnalyzer(request.getAnalyzer()), request.getText()); + tokenize(LuceneAnalyzerFactory.newAnalyzer(request.getAnalyzer()), request.getText()); return new AnalyzeResponse(tokens); } catch (IllegalArgumentException e) { throw new WebApplicationException(request.getAnalyzer() + " not a valid analyzer", Status.BAD_REQUEST); diff --git a/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/WelcomeResource.java b/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/WelcomeResource.java new file mode 100644 index 00000000000..0992623fc02 --- /dev/null +++ b/nouveau/src/main/java/org/apache/couchdb/nouveau/resources/WelcomeResource.java @@ -0,0 +1,30 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.couchdb.nouveau.resources; + +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import org.apache.couchdb.nouveau.api.WelcomeResponse; + +@Path("/") +@Produces(MediaType.APPLICATION_JSON) +public final class WelcomeResource { + + @GET + public WelcomeResponse welcome() { + return WelcomeResponse.INSTANCE; + } +} diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/api/IndexDefinitionTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/api/IndexDefinitionTest.java new file mode 100644 index 00000000000..2e1e7ee0353 --- /dev/null +++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/api/IndexDefinitionTest.java @@ -0,0 +1,42 @@ +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.couchdb.nouveau.api; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class IndexDefinitionTest { + + private static ObjectMapper mapper; + + @BeforeAll + public static void setup() { + mapper = new ObjectMapper(); + } + + @Test + public void legacyLuceneVersionIfMissing() throws Exception { + var indexDefinition = mapper.readValue("{}", IndexDefinition.class); + assertThat(indexDefinition.getLuceneVersion()).isEqualTo(IndexDefinition.LEGACY_LUCENE_VERSION); + } + + @Test + public void luceneVersionIsDeserializedIfPresent() throws Exception { + var indexDefinition = mapper.readValue("{\"lucene_version\":10}", IndexDefinition.class); + assertThat(indexDefinition.getLuceneVersion()).isEqualTo(10); + } +} diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java index bb8f0b6470e..7a122e67c85 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java +++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/core/IndexManagerTest.java @@ -26,7 +26,7 @@ import java.util.concurrent.TimeUnit; import org.apache.couchdb.nouveau.api.IndexDefinition; import org.apache.couchdb.nouveau.api.SearchRequest; -import org.apache.couchdb.nouveau.lucene9.ParallelSearcherFactory; +import org.apache.couchdb.nouveau.lucene.ParallelSearcherFactory; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -61,8 +61,8 @@ public void cleanup() throws Exception { @Test public void managerReturnsUsableIndex() throws Exception { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); manager.create("foo", indexDefinition); var searchRequest = new SearchRequest(); searchRequest.setQuery("*:*"); @@ -72,8 +72,8 @@ public void managerReturnsUsableIndex() throws Exception { @Test public void managerReopensAClosedIndex() throws Exception { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); manager.create("bar", indexDefinition); @@ -90,8 +90,8 @@ public void managerReopensAClosedIndex() throws Exception { @Test public void deleteAllRemovesIndexByName() throws Exception { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); assertThat(countIndexes()).isEqualTo(0); manager.create("bar", indexDefinition); @@ -102,8 +102,8 @@ public void deleteAllRemovesIndexByName() throws Exception { @Test public void deleteAllRemovesIndexByPath() throws Exception { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); assertThat(countIndexes()).isEqualTo(0); manager.create("foo/bar", indexDefinition); @@ -114,8 +114,8 @@ public void deleteAllRemovesIndexByPath() throws Exception { @Test public void deleteAllRemovesIndexByGlob() throws Exception { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); assertThat(countIndexes()).isEqualTo(0); manager.create("foo/bar", indexDefinition); @@ -126,8 +126,8 @@ public void deleteAllRemovesIndexByGlob() throws Exception { @Test public void deleteAllRemovesIndexByGlobExceptExclusions() throws Exception { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); assertThat(countIndexes()).isEqualTo(0); manager.create("foo/bar", indexDefinition); diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactoryTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactoryTest.java similarity index 92% rename from nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactoryTest.java rename to nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactoryTest.java index eb8e30e0884..383cf34dc8a 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9AnalyzerFactoryTest.java +++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneAnalyzerFactoryTest.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -62,7 +62,7 @@ import org.apache.lucene.analysis.tr.TurkishAnalyzer; import org.junit.jupiter.api.Test; -public class Lucene9AnalyzerFactoryTest { +public class LuceneAnalyzerFactoryTest { @Test public void testkeyword() throws Exception { @@ -256,9 +256,11 @@ public void testturkish() throws Exception { @Test public void testFieldAnalyzers() throws Exception { - final IndexDefinition indexDefinition = - new IndexDefinition("standard", Map.of("english", "english", "thai", "thai", "email", "email")); - final Analyzer analyzer = Lucene9AnalyzerFactory.fromDefinition(indexDefinition); + final IndexDefinition indexDefinition = new IndexDefinition( + IndexDefinition.LATEST_LUCENE_VERSION, + "standard", + Map.of("english", "english", "thai", "thai", "email", "email")); + final Analyzer analyzer = LuceneAnalyzerFactory.fromDefinition(indexDefinition); assertThat(analyzer).isInstanceOf(PerFieldAnalyzerWrapper.class); final Method m = PerFieldAnalyzerWrapper.class.getDeclaredMethod("getWrappedAnalyzer", String.class); m.setAccessible(true); @@ -270,12 +272,13 @@ public void testFieldAnalyzers() throws Exception { @Test public void testUnknownAnalyzer() throws Exception { - assertThrows(WebApplicationException.class, () -> Lucene9AnalyzerFactory.newAnalyzer("foo")); + assertThrows(WebApplicationException.class, () -> LuceneAnalyzerFactory.newAnalyzer("foo")); } private void assertAnalyzer(final String name, final Class clazz) throws Exception { - assertThat(Lucene9AnalyzerFactory.newAnalyzer(name)).isInstanceOf(clazz); - assertThat(Lucene9AnalyzerFactory.fromDefinition(new IndexDefinition(name, null))) + assertThat(LuceneAnalyzerFactory.newAnalyzer(name)).isInstanceOf(clazz); + assertThat(LuceneAnalyzerFactory.fromDefinition( + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, name, null))) .isInstanceOf(clazz); } } diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneIndexTest.java similarity index 96% rename from nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexTest.java rename to nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneIndexTest.java index 428e3eb6ae0..f87af2fe0f7 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/Lucene9IndexTest.java +++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/LuceneIndexTest.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -45,18 +45,18 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -public class Lucene9IndexTest { +public class LuceneIndexTest { protected final Index setup(final Path path) throws IOException { - final IndexDefinition indexDefinition = new IndexDefinition(); - indexDefinition.setDefaultAnalyzer("standard"); - final Analyzer analyzer = Lucene9AnalyzerFactory.fromDefinition(indexDefinition); + final IndexDefinition indexDefinition = + new IndexDefinition(IndexDefinition.LATEST_LUCENE_VERSION, "standard", null); + final Analyzer analyzer = LuceneAnalyzerFactory.fromDefinition(indexDefinition); final Directory dir = new DirectIODirectory(FSDirectory.open(path)); final IndexWriterConfig config = new IndexWriterConfig(analyzer); config.setUseCompoundFile(false); final IndexWriter writer = new IndexWriter(dir, config); final SearcherManager searcherManager = new SearcherManager(writer, null); - return new Lucene9Index(analyzer, writer, 0L, 0L, searcherManager); + return new LuceneIndex(analyzer, writer, 0L, 0L, searcherManager); } protected final void cleanup(final Index index) throws IOException { diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParserTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParserTest.java similarity index 98% rename from nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParserTest.java rename to nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParserTest.java index 0f4689b2137..ab7c3dfdd68 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/NouveauQueryParserTest.java +++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/NouveauQueryParserTest.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import static org.assertj.core.api.Assertions.assertThat; diff --git a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/QuerySerializationTest.java b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/QuerySerializationTest.java similarity index 97% rename from nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/QuerySerializationTest.java rename to nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/QuerySerializationTest.java index 555495a7a33..9d20907c948 100644 --- a/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene9/QuerySerializationTest.java +++ b/nouveau/src/test/java/org/apache/couchdb/nouveau/lucene/QuerySerializationTest.java @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.apache.couchdb.nouveau.lucene9; +package org.apache.couchdb.nouveau.lucene; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -39,7 +39,7 @@ public class QuerySerializationTest { @BeforeAll public static void setup() { mapper = new ObjectMapper(); - mapper.registerModule(new Lucene9Module()); + mapper.registerModule(new LuceneModule()); } @Test diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index db4d37f3e17..f1537575736 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -1080,6 +1080,7 @@ url = {{nouveau_url}} ;couch_scanner_plugin_find = false ;couch_scanner_plugin_conflict_finder = false ;couch_quickjs_scanner_plugin = false +;nouveau_index_upgrader = false ; The following [$plugin*] settings apply to all plugins @@ -1186,6 +1187,12 @@ url = {{nouveau_url}} ; Scanner settings to skip dbs and docs would also work: ;[couch_quickjs_scanner_plugin.skip_{dbs,ddoc,docs}] +[nouveau_index_upgrader] +; Common scanner scheduling settings +;after = restart +;repeat = restart + + [chttpd_auth_lockout] ; CouchDB can temporarily lock out IP addresses that repeatedly fail authentication ; mode can be set to one of three recognised values; diff --git a/src/docs/src/api/database/cleanup.rst b/src/docs/src/api/database/cleanup.rst index 6193888e90a..d93eb301a13 100644 --- a/src/docs/src/api/database/cleanup.rst +++ b/src/docs/src/api/database/cleanup.rst @@ -58,6 +58,8 @@ "ok": true } +.. _api/db/nouveau_cleanup: + ========================== ``/{db}/_nouveau_cleanup`` ========================== diff --git a/src/docs/src/ddocs/nouveau.rst b/src/docs/src/ddocs/nouveau.rst index d091ca2e8f7..d7626cfc5df 100644 --- a/src/docs/src/ddocs/nouveau.rst +++ b/src/docs/src/ddocs/nouveau.rst @@ -63,6 +63,39 @@ results from deeper in the result set. A nouveau index will inherit the partitioning type from the ``options.partitioned`` field of the design document that contains it. +.. _ddoc/nouveau/lucene_upgrade: + +Lucene Version Upgrade +====================== + +Nouveau has been upgraded to use Lucene 10, earlier releases used Lucene 9. + +Nouveau can query and update indexes created by Lucene 9 but will not create new +ones. The index definition can optionally define a ``lucene_version`` field +(which must be either 9 or 10 expressed as an integer). If not specified +when defining a new index the current version (10) will be automatically +added to the definition. + +As Lucene only supports indexes up to one major release behind the current, it +is important to rebuild all indexes to the current release. As Lucene major +releases are infrequent, and Nouveau supports 9 and 10 versions simultaneously +it is only necessary to rebuild version 9 indexes before Nouveau upgrades to +Lucene 11 (when it exists). A ``couch_scanner`` plugin is available to +automate this process, and can be enabled as follows; + +.. code-block:: ini + + [couch_scanner_plugins] + nouveau_index_upgrader = true + +The plugin will scan all design documents for index definitions either with no +``lucene_version`` field or one equal to a previous version (lower than +10). The new index will be built by the plugin and, on successful +completion, will update the ``lucene_version`` field in the index +definition. Search requests against that index will seamlessly switch from the +old index to the new one. Invoking the :ref:`_nouveau_cleanup ` +will delete the old indexes. + .. _ddoc/nouveau/field_types: Field Types diff --git a/src/docs/src/install/nouveau.rst b/src/docs/src/install/nouveau.rst index 0dc031914f3..152a2fa2f23 100644 --- a/src/docs/src/install/nouveau.rst +++ b/src/docs/src/install/nouveau.rst @@ -25,7 +25,7 @@ service that embeds `Apache Lucene `_. Typically, thi service is installed on the same host as CouchDB and communicates with it over the loopback network. -Nouveau server is runtime-compatible with Java 11 or higher. +Nouveau server is runtime-compatible with Java 21 or higher. Enable Nouveau ============== diff --git a/src/nouveau/include/nouveau.hrl b/src/nouveau/include/nouveau.hrl index e50cd45d3a9..efdc6eb7e62 100644 --- a/src/nouveau/include/nouveau.hrl +++ b/src/nouveau/include/nouveau.hrl @@ -11,9 +11,13 @@ %% See the License for the specific language governing permissions and %% limitations under the License. +-define(LEGACY_LUCENE_VERSION, 9). +-define(TARGET_LUCENE_VERSION, 10). + -record(index, { dbname, ddoc_id, + lucene_version, default_analyzer, field_analyzers, def, diff --git a/src/nouveau/src/nouveau_api.erl b/src/nouveau/src/nouveau_api.erl index cfc88af4f7e..f1bd1d9f87c 100644 --- a/src/nouveau/src/nouveau_api.erl +++ b/src/nouveau/src/nouveau_api.erl @@ -29,6 +29,7 @@ search/2, set_purge_seq/3, set_update_seq/3, + supported_lucene_versions/0, jaxrs_error/2 ]). @@ -214,6 +215,18 @@ set_seq(#index{} = Index, ReqBody) -> send_error(Reason) end. +supported_lucene_versions() -> + Resp = send_if_enabled(<<"/">>, [], <<"GET">>), + case Resp of + {ok, 200, _, RespBody} -> + Json = jiffy:decode(RespBody, [return_maps]), + {ok, maps:get(<<"supported_lucene_versions">>, Json, [])}; + {ok, StatusCode, _, RespBody} -> + {error, jaxrs_error(StatusCode, RespBody)}; + {error, Reason} -> + send_error(Reason) + end. + %% private functions index_path(Path) when is_binary(Path) -> diff --git a/src/nouveau/src/nouveau_fabric_search.erl b/src/nouveau/src/nouveau_fabric_search.erl index 3ec9d96fd6e..b082f6ff973 100644 --- a/src/nouveau/src/nouveau_fabric_search.erl +++ b/src/nouveau/src/nouveau_fabric_search.erl @@ -15,7 +15,7 @@ -module(nouveau_fabric_search). --export([go/4]). +-export([go/3, go/4]). -include_lib("mem3/include/mem3.hrl"). -include_lib("couch/include/couch_db.hrl"). @@ -38,12 +38,12 @@ go(DbName, GroupId, IndexName, QueryArgs0) when is_binary(GroupId) -> go(DbName, #doc{} = DDoc, IndexName, QueryArgs0) -> case nouveau_util:design_doc_to_index(DbName, DDoc, IndexName) of {ok, Index} -> - go(DbName, DDoc, IndexName, QueryArgs0, Index); + go(DbName, QueryArgs0, Index); {error, Reason} -> {error, Reason} end. -go(DbName, #doc{} = _DDoc, _IndexName, QueryArgs0, Index) -> +go(DbName, QueryArgs0, Index) -> Shards = get_shards(DbName, QueryArgs0), {PackedBookmark, #{limit := Limit, sort := Sort} = QueryArgs1} = maps:take(bookmark, QueryArgs0), diff --git a/src/nouveau/src/nouveau_index_updater.erl b/src/nouveau/src/nouveau_index_updater.erl index 3952a893f24..4bfea753a15 100644 --- a/src/nouveau/src/nouveau_index_updater.erl +++ b/src/nouveau/src/nouveau_index_updater.erl @@ -204,6 +204,7 @@ get_db_info(#index{} = Index) -> index_definition(#index{} = Index) -> #{ + <<"lucene_version">> => Index#index.lucene_version, <<"default_analyzer">> => Index#index.default_analyzer, <<"field_analyzers">> => Index#index.field_analyzers }. diff --git a/src/nouveau/src/nouveau_index_upgrader.erl b/src/nouveau/src/nouveau_index_upgrader.erl new file mode 100644 index 00000000000..9f757cf5577 --- /dev/null +++ b/src/nouveau/src/nouveau_index_upgrader.erl @@ -0,0 +1,165 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(nouveau_index_upgrader). +-behaviour(couch_scanner_plugin). + +-export([ + start/2, + resume/2, + complete/1, + checkpoint/1, + db/2, + ddoc/3 +]). + +-include("nouveau.hrl"). +-include_lib("couch_scanner/include/couch_scanner_plugin.hrl"). + +start(ScanId, #{}) -> + St = init_config(ScanId), + case should_run(St) of + true -> + ?INFO("Starting.", [], St), + {ok, St}; + false -> + ?INFO("Not starting.", [], St), + skip + end. + +resume(ScanId, #{}) -> + St = init_config(ScanId), + case should_run(St) of + true -> + ?INFO("Resuming.", [], St), + {ok, St}; + false -> + ?INFO("Not resuming.", [], St), + skip + end. + +complete(St) -> + ?INFO("Completed", [], St), + {ok, #{}}. + +checkpoint(_St) -> + {ok, #{}}. + +db(St, _DbName) -> + {ok, St}. + +ddoc(St, _DbName, #doc{id = <<"_design/_", _/binary>>}) -> + {ok, St}; +ddoc(St, DbName, #doc{} = DDoc0) -> + case update_ddoc_versions(DDoc0) of + DDoc0 -> + ok; + DDoc1 -> + Indexes = nouveau_util:design_doc_to_indexes(DbName, DDoc1), + case upgrade_indexes(DbName, Indexes) of + true -> + save_ddoc(DbName, DDoc1); + false -> + ok + end + end, + {ok, St}. + +upgrade_indexes(_DbName, []) -> + true; +upgrade_indexes(DbName, [Index | Rest]) -> + case upgrade_index(DbName, Index) of + true -> + upgrade_indexes(DbName, Rest); + false -> + false + end. + +upgrade_index(DbName, #index{} = Index) -> + ?INFO("Upgrading ~s/~s/~s to version ~B", [ + DbName, + Index#index.ddoc_id, + Index#index.name, + ?TARGET_LUCENE_VERSION + ]), + case + nouveau_fabric_search:go( + DbName, + #{query => <<"*:*">>, bookmark => null, sort => null, limit => 1}, + Index#index{lucene_version = ?TARGET_LUCENE_VERSION} + ) + of + {ok, _SearchResults} -> + true; + {error, _Reason} -> + false + end. + +update_ddoc_versions(#doc{} = Doc) -> + #doc{body = {Fields0}} = Doc, + {Indexes0} = couch_util:get_value(<<"nouveau">>, Fields0), + Indexes1 = lists:map(fun update_version/1, Indexes0), + Fields1 = couch_util:set_value(<<"nouveau">>, Fields0, {Indexes1}), + Doc#doc{body = {Fields1}}. + +save_ddoc(DbName, #doc{} = DDoc) -> + {Pid, Ref} = spawn_monitor(fun() -> + case fabric:update_doc(DbName, DDoc, [?ADMIN_CTX]) of + {ok, _} -> + exit(ok); + Else -> + exit(Else) + end + end), + receive + {'DOWN', Ref, process, Pid, ok} -> + ?INFO( + "Updated ~s/~s indexes to version ~B", [DbName, DDoc#doc.id, ?TARGET_LUCENE_VERSION] + ); + {'DOWN', Ref, process, Pid, Else} -> + ?INFO("Failed to update ~s/~s for reason ~p", [DbName, DDoc#doc.id, Else]) + end. + +update_version({IndexName, {Index}}) -> + {IndexName, {couch_util:set_value(<<"lucene_version">>, Index, ?TARGET_LUCENE_VERSION)}}. + +init_config(ScanId) -> + #{sid => ScanId}. + +should_run(St) -> + couch_scanner_util:on_first_node() andalso upgrade_supported(St). + +upgrade_supported(St) -> + case nouveau_api:supported_lucene_versions() of + {ok, Versions} -> + case lists:member(?TARGET_LUCENE_VERSION, Versions) of + true -> + ?INFO( + "Nouveau server supports upgrades to Lucene ~B", + [?TARGET_LUCENE_VERSION], + St + ), + true; + false -> + ?WARN( + "Nouveau server does not support upgrades to Lucene ~B", + [?TARGET_LUCENE_VERSION], + St + ), + false + end; + {error, Reason} -> + ?ERR( + "Nouveau server upgrade check failed for reason ~p", [Reason], St + ), + false + end. diff --git a/src/nouveau/src/nouveau_plugin_couch_db.erl b/src/nouveau/src/nouveau_plugin_couch_db.erl index dcd3ae1f156..d10d7adfc4d 100644 --- a/src/nouveau/src/nouveau_plugin_couch_db.erl +++ b/src/nouveau/src/nouveau_plugin_couch_db.erl @@ -13,10 +13,46 @@ -module(nouveau_plugin_couch_db). -export([ + before_doc_update/3, is_valid_purge_client/2, on_compact/2 ]). +-include("nouveau.hrl"). +-include_lib("couch/include/couch_db.hrl"). + +%% New index definitions get an explicit lucene version property, if missing. +before_doc_update( + #doc{id = <>, revs = {0, []}} = Doc, + Db, + ?INTERACTIVE_EDIT = UpdateType +) -> + #doc{body = {Fields}} = Doc, + case couch_util:get_value(<<"nouveau">>, Fields) of + {Indexes} when is_list(Indexes) -> + [add_versions_to_doc(Doc), Db, UpdateType]; + _ -> + [Doc, Db, UpdateType] + end; +before_doc_update(Doc, Db, UpdateType) -> + [Doc, Db, UpdateType]. + +add_versions_to_doc(#doc{} = Doc) -> + #doc{body = {Fields0}} = Doc, + {Indexes0} = couch_util:get_value(<<"nouveau">>, Fields0), + Indexes1 = lists:map(fun add_version_to_index/1, Indexes0), + Fields1 = couch_util:set_value(<<"nouveau">>, Fields0, {Indexes1}), + Doc#doc{body = {Fields1}}. + +add_version_to_index({IndexName, {Index}}) -> + case couch_util:get_value(<<"lucene_version">>, Index) of + undefined -> + {IndexName, + {couch_util:set_value(<<"lucene_version">>, Index, ?TARGET_LUCENE_VERSION)}}; + _ -> + {IndexName, {Index}} + end. + is_valid_purge_client(DbName, Props) -> nouveau_util:verify_index_exists(DbName, Props). diff --git a/src/nouveau/src/nouveau_util.erl b/src/nouveau/src/nouveau_util.erl index 0dfcb1e1e0b..892ad48271f 100644 --- a/src/nouveau/src/nouveau_util.erl +++ b/src/nouveau/src/nouveau_util.erl @@ -67,23 +67,33 @@ design_doc_to_index(DbName, #doc{id = Id, body = {Fields}}, IndexName) -> false -> {error, {not_found, <>}}; {IndexName, {Index}} -> + LuceneVersion = couch_util:get_value( + <<"lucene_version">>, Index, ?LEGACY_LUCENE_VERSION + ), DefaultAnalyzer = couch_util:get_value(<<"default_analyzer">>, Index, <<"standard">>), FieldAnalyzers = couch_util:get_value(<<"field_analyzers">>, Index, #{}), case couch_util:get_value(<<"index">>, Index) of undefined -> {error, InvalidDDocError}; Def -> - Sig = - couch_util:to_hex_bin( - crypto:hash( - sha256, - ?term_to_bin( - {DefaultAnalyzer, FieldAnalyzers, Def} - ) + SigTerm = + case LuceneVersion of + ?LEGACY_LUCENE_VERSION -> + {DefaultAnalyzer, FieldAnalyzers, Def}; + _ -> + {LuceneVersion, DefaultAnalyzer, FieldAnalyzers, Def} + end, + Sig = couch_util:to_hex_bin( + crypto:hash( + sha256, + ?term_to_bin( + SigTerm ) - ), + ) + ), {ok, #index{ dbname = DbName, + lucene_version = LuceneVersion, default_analyzer = DefaultAnalyzer, field_analyzers = FieldAnalyzers, ddoc_id = Id, diff --git a/src/nouveau/test/eunit/nouveau_index_upgrader_tests.erl b/src/nouveau/test/eunit/nouveau_index_upgrader_tests.erl new file mode 100644 index 00000000000..d37a1d0071b --- /dev/null +++ b/src/nouveau/test/eunit/nouveau_index_upgrader_tests.erl @@ -0,0 +1,131 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(nouveau_index_upgrader_tests). + +-include_lib("couch/include/couch_eunit.hrl"). +-include_lib("couch/include/couch_db.hrl"). +-include_lib("nouveau/include/nouveau.hrl"). + +-define(PLUGIN, nouveau_index_upgrader). + +nouveau_index_upgrader_test_() -> + { + foreach, + fun setup/0, + fun teardown/1, + [ + ?TDEF_FE(t_upgrade_legacy_index, 10), + ?TDEF_FE(t_dont_upgrade_latest_index, 10) + ] + }. + +setup() -> + {module, _} = code:ensure_loaded(?PLUGIN), + meck:new(?PLUGIN, [passthrough]), + meck:new(couch_scanner_server, [passthrough]), + meck:new(couch_scanner_util, [passthrough]), + meck:new(nouveau_api, [passthrough]), + meck:expect(nouveau_api, supported_lucene_versions, fun() -> + {ok, [?LEGACY_LUCENE_VERSION, ?TARGET_LUCENE_VERSION]} + end), + Ctx = test_util:start_couch([fabric, couch_scanner]), + DbName = ?tempdb(), + ok = fabric:create_db(DbName, [{q, "2"}, {n, "1"}]), + config:set(atom_to_list(?PLUGIN), "max_batch_items", "1", false), + reset_stats(), + {Ctx, DbName}. + +teardown({Ctx, DbName}) -> + config_delete_section("couch_scanner"), + config_delete_section("couch_scanner_plugins"), + config_delete_section(atom_to_list(?PLUGIN)), + couch_scanner:reset_checkpoints(), + couch_scanner:resume(), + fabric:delete_db(DbName), + test_util:stop_couch(Ctx), + meck:unload(). + +t_upgrade_legacy_index({_, DbName}) -> + DDocId = <<"_design/foo">>, + IndexName = <<"bar">>, + ok = add_ddoc(DbName, DDocId, IndexName, ?LEGACY_LUCENE_VERSION), + meck:reset(couch_scanner_server), + meck:reset(?PLUGIN), + meck:new(nouveau_fabric_search, [passthrough]), + meck:expect(nouveau_fabric_search, go, fun(_DbName, _Args, _Index) -> {ok, []} end), + config:set("couch_scanner_plugins", atom_to_list(?PLUGIN), "true", false), + wait_exit(10000), + ?assertEqual(1, num_calls(start, 2)), + ?assertEqual(1, num_calls(complete, 1)), + ?assertEqual(?TARGET_LUCENE_VERSION, get_lucene_version(DbName, DDocId, IndexName)), + ok. + +t_dont_upgrade_latest_index({_, DbName}) -> + DDocId = <<"_design/foo">>, + IndexName = <<"bar">>, + ok = add_ddoc(DbName, DDocId, IndexName, ?TARGET_LUCENE_VERSION), + meck:reset(couch_scanner_server), + meck:reset(?PLUGIN), + config:set("couch_scanner_plugins", atom_to_list(?PLUGIN), "true", false), + wait_exit(10000), + ?assertEqual(1, num_calls(start, 2)), + ?assertEqual(1, num_calls(complete, 1)), + ?assertEqual(?TARGET_LUCENE_VERSION, get_lucene_version(DbName, DDocId, IndexName)), + ok. + +reset_stats() -> + Counters = [ + [couchdb, query_server, process_error_exits], + [couchdb, query_server, process_errors], + [couchdb, query_server, process_exits] + ], + [reset_counter(C) || C <- Counters]. + +reset_counter(Counter) -> + case couch_stats:sample(Counter) of + 0 -> + ok; + N when is_integer(N), N > 0 -> + couch_stats:decrement_counter(Counter, N) + end. + +config_delete_section(Section) -> + [config:delete(K, V, false) || {K, V} <- config:get(Section)]. + +add_ddoc(DbName, DDocId, IndexName, LuceneVersion) -> + {ok, _} = fabric:update_doc(DbName, mkddoc(DDocId, IndexName, LuceneVersion), [?ADMIN_CTX]), + ok. + +get_lucene_version(DbName, DDocId, IndexName) -> + {ok, #doc{body = {Props}}} = fabric:open_doc(DbName, DDocId, [?ADMIN_CTX]), + {Indexes} = couch_util:get_value(<<"nouveau">>, Props), + {Index} = couch_util:get_value(IndexName, Indexes), + couch_util:get_value(<<"lucene_version">>, Index). + +mkddoc(DocId, IndexName, LuceneVersion) -> + Body = #{ + <<"_id">> => DocId, + <<"nouveau">> => #{ + IndexName => #{ + <<"lucene_version">> => LuceneVersion, + <<"index">> => <<"function(doc){}">> + } + } + }, + jiffy:decode(jiffy:encode(Body)). + +num_calls(Fun, Args) -> + meck:num_calls(?PLUGIN, Fun, Args). + +wait_exit(MSec) -> + meck:wait(couch_scanner_server, handle_info, [{'EXIT', '_', '_'}, '_'], MSec).