diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index a68a389e6021..3cbbf2135ae7 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -138,6 +138,9 @@ public class RankProfile implements Cloneable { private Set filterFields = new HashSet<>(); + // Field-level `rank my_field { filter-threshold: ... }` that overrides the profile-level `filter-threshold` (if any) + private Map explicitFieldRankFilterThresholds = new LinkedHashMap<>(); + private final RankProfileRegistry rankProfileRegistry; private final TypeSettings attributeTypes = new TypeSettings(); @@ -1012,6 +1015,14 @@ public Set allFilterFields() { return combined; } + public void setExplicitFieldRankFilterThresholds(Map fieldFilterThresholds) { + explicitFieldRankFilterThresholds = new LinkedHashMap<>(fieldFilterThresholds); + } + + public Map explicitFieldRankFilterThresholds() { + return explicitFieldRankFilterThresholds; + } + private ExpressionFunction parseRankingExpression(String name, List arguments, String expression) throws ParseException { if (expression.trim().isEmpty()) throw new ParseException("Empty expression"); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java index 0848b1dd9739..34831dcfeceb 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java @@ -184,6 +184,7 @@ private static class Deriver { private final Map attributeTypes; private final Map inputs; private final Set filterFields = new java.util.LinkedHashSet<>(); + private Map explicitFieldRankFilterThresholds = new LinkedHashMap<>(); private final String rankprofileName; private RankingExpression firstPhaseRanking; @@ -271,6 +272,7 @@ private void deriveFeatureDeclarations(Collection features, private void deriveFilterFields(RankProfile rp) { filterFields.addAll(rp.allFilterFields()); + explicitFieldRankFilterThresholds.putAll(rp.explicitFieldRankFilterThresholds()); } private void derivePropertiesAndFeaturesFromFunctions(Map functions, @@ -498,6 +500,9 @@ else if (RankingExpression.propertyName(RankProfile.GLOBAL_PHASE).equals(propert if (filterThreshold.isPresent()) { properties.add(new Pair<>("vespa.matching.filter_threshold", String.valueOf(filterThreshold.getAsDouble()))); } + for (var fieldAndThreshold : explicitFieldRankFilterThresholds.entrySet()) { + properties.add(new Pair<>("vespa.matching.filter_threshold.%s".formatted(fieldAndThreshold.getKey()), String.valueOf(fieldAndThreshold.getValue()))); + } if (matchPhaseSettings != null) { properties.add(new Pair<>("vespa.matchphase.degradation.attribute", matchPhaseSettings.getAttribute())); properties.add(new Pair<>("vespa.matchphase.degradation.ascendingorder", matchPhaseSettings.getAscending() + "")); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java index 95e957fab84d..59753adb78dc 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java @@ -54,6 +54,7 @@ public class ParsedRankProfile extends ParsedBlock { private final List mutateOperations = new ArrayList<>(); private final List inherited = new ArrayList<>(); private final Map fieldsRankFilter = new LinkedHashMap<>(); + private final Map fieldsRankFilterThreshold = new LinkedHashMap<>(); private final Map fieldsRankWeight = new LinkedHashMap<>(); private final Map functions = new LinkedHashMap<>(); private final Map fieldsRankType = new LinkedHashMap<>(); @@ -94,6 +95,7 @@ public ParsedRankProfile(String name) { Optional getGlobalPhaseExpression() { return Optional.ofNullable(this.globalPhaseExpression); } Map getFieldsWithRankFilter() { return Collections.unmodifiableMap(fieldsRankFilter); } + Map getFieldsWithRankFilterThreshold() { return Collections.unmodifiableMap(fieldsRankFilterThreshold); } Map getFieldsWithRankWeight() { return Collections.unmodifiableMap(fieldsRankWeight); } Map getFieldsWithRankType() { return Collections.unmodifiableMap(fieldsRankType); } Map> getRankProperties() { return Collections.unmodifiableMap(rankProperties); } @@ -140,6 +142,12 @@ public void addFieldRankFilter(String field, boolean filter) { fieldsRankFilter.put(field, filter); } + public void addFieldRankFilterThreshold(String field, double filterThreshold) { + verifyThat(!fieldsRankFilterThreshold.containsKey(field), "already has rank filter-threshold for field", field); + verifyThat(filterThreshold >= 0.0 && filterThreshold <= 1.0, "must be a value in range [0, 1]", field); + fieldsRankFilterThreshold.put(field, filterThreshold); + } + public void addFieldRankType(String field, String type) { verifyThat(! fieldsRankType.containsKey(field), "already has rank type for field", field); fieldsRankType.put(field, type); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankingConverter.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankingConverter.java index 12fe98ee9d87..1fb43b8fe04d 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankingConverter.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankingConverter.java @@ -94,6 +94,8 @@ private void populateFrom(ParsedRankProfile parsed, RankProfile profile) { parsed.getFieldsWithRankFilter().forEach ((fieldName, isFilter) -> profile.addRankSetting(fieldName, RankProfile.RankSetting.Type.PREFERBITVECTOR, isFilter)); + profile.setExplicitFieldRankFilterThresholds(parsed.getFieldsWithRankFilterThreshold()); + parsed.getFieldsWithRankWeight().forEach ((fieldName, weight) -> profile.addRankSetting(fieldName, RankProfile.RankSetting.Type.WEIGHT, weight)); diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index e595d1d15731..63ad44985ebb 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -2428,10 +2428,15 @@ void fieldRankType(ParsedRankProfile profile) : void fieldRankFilter(ParsedRankProfile profile) : { String name; + double filterThreshold; } { - name = identifier() - { profile.addFieldRankFilter(name, true); } + name = identifier() + ( ( ) { profile.addFieldRankFilter(name, true); } + | ( lbrace() filterThreshold = floatValue() + { profile.addFieldRankFilterThreshold(name, filterThreshold); } + ( )* ) + ) } /** diff --git a/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java index 6d41166043c1..966363cf76fa 100644 --- a/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java @@ -582,6 +582,49 @@ private void verifyFilterThreshold(Double threshold) throws ParseException { threshold, "vespa.matching.filter_threshold"); } + private static OptionalDouble optionalDoubleOfNullable(Double maybeDouble) { + // No ofNullable in OptionalDouble, probably due to auto boxing magics + return maybeDouble != null ? OptionalDouble.of(maybeDouble) : OptionalDouble.empty(); + } + + @Test + void field_specific_filter_threshold_is_configurable() throws ParseException { + var rps = """ + search test { + document test { + field f1 type string { + indexing: index + } + field f2 type string { + indexing: index + } + field f3 type string { + indexing: index + } + } + rank-profile my_profile { + rank f1 { + filter-threshold: 0.08 + } + rank f2 { + filter-threshold: 0.11 + } + } + } + """; + var rp = createRankProfile(rps); + + verifyRankProfileSetting(rp.getFirst(), rp.getSecond(), + (myRp) -> optionalDoubleOfNullable(myRp.explicitFieldRankFilterThresholds().get("f1")), + 0.08, "vespa.matching.filter_threshold.f1"); + verifyRankProfileSetting(rp.getFirst(), rp.getSecond(), + (myRp) -> optionalDoubleOfNullable(myRp.explicitFieldRankFilterThresholds().get("f2")), + 0.11, "vespa.matching.filter_threshold.f2"); + verifyRankProfileSetting(rp.getFirst(), rp.getSecond(), + (myRp) -> optionalDoubleOfNullable(myRp.explicitFieldRankFilterThresholds().get("f3")), + null, "vespa.matching.filter_threshold.f3"); + } + private void verifyRankProfileSetting(RankProfile rankProfile, RawRankProfile rawRankProfile, Function func, Double expValue, String expPropertyName) { if (expValue != null) { diff --git a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java index a8f939ecab87..8dc30c4e1998 100644 --- a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java @@ -197,6 +197,31 @@ void filter_threshold_can_be_parsed() throws Exception { assertEquals(0.05, target.get()); } + @Test + void field_rank_specific_filter_threshold_can_be_parsed() throws Exception { + String input = """ + schema foo { + rank-profile rp { + rank bar { + filter-threshold: 0.05 + } + rank zoid { + filter-threshold: 0.07 + } + rank baz: filter + } + }"""; + var schema = parseString(input); + var rp = schema.getRankProfiles().get(0); + var thresholds = rp.getFieldsWithRankFilterThreshold(); + assertEquals(2, thresholds.size()); + assertEquals(0.05, thresholds.getOrDefault("bar", 0.0), 0.000001); + assertEquals(0.07, thresholds.getOrDefault("zoid", 0.0), 0.000001); + // Old-school binary rank filter still supported as expected + assertEquals(1, rp.getFieldsWithRankFilter().size()); + assertTrue(rp.getFieldsWithRankFilter().get("baz")); + } + @Test void maxOccurrencesCanBeParsed() throws Exception { String input = joinLines diff --git a/integration/schema-language-server/language-server/src/main/ccc/SchemaParser.ccc b/integration/schema-language-server/language-server/src/main/ccc/SchemaParser.ccc index a28185339c91..ff513b4120a5 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/SchemaParser.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/SchemaParser.ccc @@ -2755,10 +2755,15 @@ void fieldRankType(ParsedRankProfile profile) : void fieldRankFilter(ParsedRankProfile profile) : { String name; + double filterThreshold; } - name = identifierStr() - { profile.addFieldRankFilter(name, true); } + name = identifierStr() + ( ( ) { profile.addFieldRankFilter(name, true); } + | ( openLbrace() filterThreshold = floatValue() + { profile.addFieldRankFilterThreshold(name, filterThreshold); } + ( )* ) + ) ; /**