Skip to content

changes for elasticsearch 7.9.x #42

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: 7.8
Choose a base branch
from
Open

changes for elasticsearch 7.9.x #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3c0be04
changes for elasticsearch 7.9.x
Aug 21, 2020
d240954
elasticsearch 7.9.3
Nov 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions gradle.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
group = org.xbib.elasticsearch
name = elasticsearch-plugin-bundle
version = 7.8.1.0
version = 7.9.3.0

elasticsearch.version = 7.8.1
lucene.version = 8.5.1
elasticsearch.version = 7.9.3
lucene.version = 8.6.2

icu4j.version = 67.1
standardnumber.version = 1.0.1
Expand Down
3 changes: 2 additions & 1 deletion ...main/java/org/xbib/elasticsearch/plugin/bundle/common/decompound/fst/FstDecompounder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ public class FstDecompounder {

public FstDecompounder(InputStream inputStream, List<String> glue) throws IOException {
try {
this.surfaceForms = new FST<>(new InputStreamDataInput(inputStream), NoOutputs.getSingleton());
InputStreamDataInput input = new InputStreamDataInput(inputStream);
this.surfaceForms = new FST<>(input, input, NoOutputs.getSingleton());
// set up glue morphemes
this.glueMorphemes = createGlueMorphemes(glue != null && glue.size() > 0 ? glue :morphemes);
} finally {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/xbib/elasticsearch/plugin/bundle/common/fst/FstCompiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public void compile(InputStream inputStream, OutputStream outputStream) throws I
}
final FST<Object> fst = builder.finish();
try (OutputStreamDataOutput out = new OutputStreamDataOutput(outputStream)) {
fst.save(out);
fst.save(out, out);
}
}
}
132 changes: 67 additions & 65 deletions ...ava/org/xbib/elasticsearch/plugin/bundle/index/mapper/icu/IcuCollationKeyFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@
import com.ibm.icu.text.Collator;
import com.ibm.icu.text.RawCollationKey;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.DocValuesFieldExistsQuery;
import org.apache.lucene.search.MultiTermQuery;
Expand All @@ -20,28 +20,30 @@
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.plain.DocValuesIndexFieldData;
import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
import org.elasticsearch.index.mapper.KeywordFieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.StringFieldType;
import org.elasticsearch.index.mapper.TextSearchInfo;
import org.elasticsearch.index.mapper.TypeParsers;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
import org.xbib.elasticsearch.plugin.bundle.index.analysis.icu.IcuCollationKeyAnalyzerProvider;
import org.xbib.elasticsearch.plugin.bundle.index.analysis.icu.IndexableBinaryStringTools;

import java.io.IOException;
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.BiFunction;
import java.util.function.LongSupplier;

/**
Expand All @@ -51,7 +53,7 @@ public class IcuCollationKeyFieldMapper extends FieldMapper {

public static final String CONTENT_TYPE = "icu_collation_key";

public static final MappedFieldType FIELD_TYPE = new CollationFieldType();
public static final FieldType FIELD_TYPE = new FieldType();

public static class Defaults {

Expand All @@ -65,39 +67,38 @@ public static class Defaults {

public static final class CollationFieldType extends StringFieldType {

private Collator collator = null;
private final Collator collator;

public CollationFieldType() {
public CollationFieldType(String name, boolean isSearchable, boolean hasDocValues, Collator collator, Map<String, String> meta) {
super(name, true, true, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
setSearchAnalyzer(Lucene.KEYWORD_ANALYZER);
this.collator = collator;
}

protected CollationFieldType(CollationFieldType ref) {
super(ref);
this.collator = ref.collator;
}

public CollationFieldType clone() {
return new CollationFieldType(this);
public CollationFieldType(String name, Collator collator) {
this(name, true, true, collator, Collections.emptyMap());
}

@Override
public boolean equals(Object o) {
return super.equals(o) && Objects.equals(collator, ((CollationFieldType) o).collator);
public int hashCode() {
return 31 * super.hashCode() + Objects.hashCode(collator);
}

@Override
public void checkCompatibility(MappedFieldType otherFT, List<String> conflicts) {
super.checkCompatibility(otherFT,conflicts);
CollationFieldType other = (CollationFieldType) otherFT;
if (!Objects.equals(collator, other.collator)) {
conflicts.add("mapper [" + name() + "] has different [collator]");
public boolean equals(Object o)
{
if (this == o)
{
return true;
}
if (o == null || getClass() != o.getClass())
{
return false;
}
}

@Override
public int hashCode() {
return 31 * super.hashCode() + Objects.hashCode(collator);
CollationFieldType that = (CollationFieldType) o;

return collator != null ? collator.equals(that.collator) : that.collator == null;
}

@Override
Expand All @@ -109,11 +110,6 @@ public Collator collator() {
return collator;
}

public void setCollator(Collator collator) {
checkIfFrozen();
this.collator = collator.isFrozen() ? collator : collator.freeze();
}

@Override
public Query existsQuery(QueryShardContext context) {
if (hasDocValues()) {
Expand All @@ -126,7 +122,7 @@ public Query existsQuery(QueryShardContext context) {
@Override
public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) {
failIfNoDocValues();
return new DocValuesIndexFieldData.Builder();
return new SortedSetOrdinalsIndexFieldData.Builder(CoreValuesSourceType.BYTES);
}

@Override
Expand Down Expand Up @@ -216,20 +212,16 @@ public BytesRef parseBytesRef(String value) {
};
}

public static class Builder extends FieldMapper.Builder<Builder, IcuCollationKeyFieldMapper> {
public static class Builder extends FieldMapper.Builder<Builder> {
private Settings.Builder settingsBuilder;
protected String nullValue;

public Builder(String name) {
super(name, FIELD_TYPE, FIELD_TYPE);
super(name, FIELD_TYPE);
builder = this;
this.settingsBuilder = Settings.builder();
}

@Override
public CollationFieldType fieldType() {
return (CollationFieldType) super.fieldType();
}

@Override
public Builder indexOptions(IndexOptions indexOptions) {
if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) > 0) {
Expand Down Expand Up @@ -329,23 +321,27 @@ public Builder numeric(final boolean numeric) {
return this;
}

public Builder nullValue(String nullValue) {
this.nullValue = nullValue;
return this;
}

public Collator buildCollator() {
return IcuCollationKeyAnalyzerProvider.createCollator(settingsBuilder.build());
}

@Override
public IcuCollationKeyFieldMapper build(BuilderContext context) {
final Collator collator = buildCollator();
fieldType().setCollator(collator);
setupFieldType(context);
return new IcuCollationKeyFieldMapper(name, fieldType, defaultFieldType, context.indexSettings(),
multiFieldsBuilder.build(this, context), copyTo, settingsBuilder.build(), collator);
CollationFieldType ft = new CollationFieldType(buildFullName(context), indexed, hasDocValues, collator, meta);
return new IcuCollationKeyFieldMapper(name, fieldType, ft,
multiFieldsBuilder.build(this, context), copyTo, settingsBuilder.build(), collator, nullValue);
}
}

public static class TypeParser implements Mapper.TypeParser {
@Override
public Mapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext)
public Mapper.Builder<?> parse(String name, Map<String, Object> node, ParserContext parserContext)
throws MapperParsingException {
Builder builder = new Builder(name);
TypeParsers.parseField(builder, name, node, parserContext);
Expand Down Expand Up @@ -415,16 +411,16 @@ public static class TypeParser implements Mapper.TypeParser {

private final Settings collatorSettings;
private final Collator collator;
private final BiFunction<String, BytesRef, Field> getDVField;
private final String nullValue;

protected IcuCollationKeyFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
Settings indexSettings, MultiFields multiFields,
CopyTo copyTo, Settings collatorSettings, Collator collator) {
super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
protected IcuCollationKeyFieldMapper(String simpleName, FieldType fieldType, MappedFieldType defaultFieldType,
MultiFields multiFields,
CopyTo copyTo, Settings collatorSettings, Collator collator, String nullValue) {
super(simpleName, fieldType, defaultFieldType, multiFields, copyTo);
assert collator.isFrozen();
this.collatorSettings = collatorSettings;
this.collator = collator;
this.getDVField = SortedSetDocValuesField::new;
this.nullValue = nullValue;
}

@Override
Expand All @@ -438,10 +434,11 @@ protected String contentType() {
}

@Override
protected void doMerge(Mapper mergeWith) {
super.doMerge(mergeWith);
List<String> conflicts = new ArrayList<>();
IcuCollationKeyFieldMapper icuMergeWith = (IcuCollationKeyFieldMapper) mergeWith;
protected void mergeOptions(FieldMapper other, List<String> conflicts) {
IcuCollationKeyFieldMapper icuMergeWith = (IcuCollationKeyFieldMapper) other;
if (!Objects.equals(collator, icuMergeWith.collator)) {
conflicts.add("mapper [" + name() + "] has different [collator]");
}
if (!Objects.equals(collatorSettings.get("rules"), icuMergeWith.collatorSettings.get("rules"))) {
conflicts.add("Cannot update rules setting for [" + CONTENT_TYPE + "]");
}
Expand Down Expand Up @@ -472,17 +469,20 @@ protected void doMerge(Mapper mergeWith) {
if (collatorSettings.getAsBoolean("numeric", true) != icuMergeWith.collatorSettings.getAsBoolean("numeric", true)) {
conflicts.add("Cannot update numeric setting for [" + CONTENT_TYPE + "]");
}
if (!conflicts.isEmpty()) {
throw new IllegalArgumentException("Can't merge because of conflicts: " + conflicts);
}
}


@Override
protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
super.doXContentBody(builder, includeDefaults, params);
if (includeDefaults || fieldType().nullValue() != null) {
builder.field("null_value", fieldType().nullValue());
if (fieldType.indexOptions() != IndexOptions.NONE && (includeDefaults || fieldType.indexOptions() != IndexOptions.DOCS)) {
builder.field("index_options", indexOptionToString(fieldType.indexOptions()));
}
if (nullValue != null) {
builder.field("null_value", nullValue);
}
if (includeDefaults || fieldType.omitNorms() != KeywordFieldMapper.Defaults.FIELD_TYPE.omitNorms()) {
builder.field("norms", fieldType.omitNorms() == false);
}
if (includeDefaults) {
builder.field("rules", collatorSettings.get("rules"));
Expand Down Expand Up @@ -517,14 +517,14 @@ protected void doXContentBody(XContentBuilder builder, boolean includeDefaults,
}

@Override
protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
protected void parseCreateField(ParseContext context) throws IOException {
final String value;
if (context.externalValueSet()) {
value = context.externalValue().toString();
} else {
XContentParser parser = context.parser();
if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
value = fieldType().nullValueAsString();
value = nullValue;
} else {
value = parser.textOrNull();
}
Expand All @@ -534,12 +534,14 @@ protected void parseCreateField(ParseContext context, List<IndexableField> field
}
RawCollationKey key = collator.getRawCollationKey(value, null);
final BytesRef binaryValue = new BytesRef(key.bytes, 0, key.size);
if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
Field field = new Field(fieldType().name(), binaryValue, fieldType());
fields.add(field);
if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
Field field = new Field(mappedFieldType.name(), binaryValue, fieldType);
context.doc().add(field);
}
if (fieldType().hasDocValues()) {
fields.add(getDVField.apply(fieldType().name(), binaryValue));
context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue));
} else if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
createFieldNamesField(context);
}
}
}
Loading