propagate Dictionary settings to attributes for struct fields

vespa-engine · Jan 22, 2025 · c2b6416 · c2b6416
1 parent 595ab13
commit c2b6416
Show file tree

Hide file tree

Showing 6 changed files with 114 additions and 2 deletions.
diff --git a/config-model/src/main/java/com/yahoo/schema/Schema.java b/config-model/src/main/java/com/yahoo/schema/Schema.java
@@ -343,6 +343,19 @@ public List<SDField> allConcreteFields() {
         return allFields;
     }
 
+    /**
+     * as allConcreteFields, but includes sub-fields for arrays of structs etc.
+     */
+    public List<SDField> allConcreteFieldsWithSubFields() {
+        var allFields = allConcreteFields();
+        List<SDField> subFields = new ArrayList<>();
+        for (SDField f : allFields) {
+            subFields.addAll(f.getStructFields());
+        }
+        allFields.addAll(subFields);
+        return allFields;
+    }
+
     /**
      * Returns the content of a ranking expression file
      */

diff --git a/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java
@@ -26,7 +26,7 @@ public DictionaryProcessor(Schema schema, DeployLogger deployLogger, RankProfile
 
     @Override
     public void process(boolean validate, boolean documentsOnly) {
-        for (SDField field : schema.allConcreteFields()) {
+        for (SDField field : schema.allConcreteFieldsWithSubFields()) {
             Attribute attribute = field.getAttribute();
             if (attribute == null) continue;
             attribute.setCase(field.getMatching().getCase());

diff --git a/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg b/config-model/src/test/derived/array_of_struct_attribute/attributes.cfg
@@ -1,3 +1,61 @@
+attribute[].name "kv_array.mykey"
+attribute[].datatype STRING
+attribute[].collectiontype ARRAY
+attribute[].dictionary.type BTREE
+attribute[].dictionary.match CASED
+attribute[].match CASED
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch true
+attribute[].paged false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported false
+attribute[].maxuncommittedmemory 77777
+attribute[].distancemetric EUCLIDEAN
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].index.hnsw.multithreadedindexing true
+attribute[].name "kv_array.myval"
+attribute[].datatype STRING
+attribute[].collectiontype ARRAY
+attribute[].dictionary.type BTREE
+attribute[].dictionary.match CASED
+attribute[].match CASED
+attribute[].removeifzero false
+attribute[].createifnonexistent false
+attribute[].fastsearch false
+attribute[].paged false
+attribute[].ismutable false
+attribute[].sortascending true
+attribute[].sortfunction UCA
+attribute[].sortstrength PRIMARY
+attribute[].sortlocale ""
+attribute[].enableonlybitvector false
+attribute[].fastaccess false
+attribute[].arity 8
+attribute[].lowerbound -9223372036854775808
+attribute[].upperbound 9223372036854775807
+attribute[].densepostinglistthreshold 0.4
+attribute[].tensortype ""
+attribute[].imported false
+attribute[].maxuncommittedmemory 77777
+attribute[].distancemetric EUCLIDEAN
+attribute[].index.hnsw.enabled false
+attribute[].index.hnsw.maxlinkspernode 16
+attribute[].index.hnsw.neighborstoexploreatinsert 200
+attribute[].index.hnsw.multithreadedindexing true
 attribute[].name "elem_array.name"
 attribute[].datatype STRING
 attribute[].collectiontype ARRAY

diff --git a/config-model/src/test/derived/array_of_struct_attribute/index-info.cfg b/config-model/src/test/derived/array_of_struct_attribute/index-info.cfg
@@ -3,6 +3,32 @@ indexinfo[].command[].indexname "sddocname"
 indexinfo[].command[].command "index"
 indexinfo[].command[].indexname "sddocname"
 indexinfo[].command[].command "word"
+indexinfo[].command[].indexname "kv_array.mykey"
+indexinfo[].command[].command "multivalue"
+indexinfo[].command[].indexname "kv_array.mykey"
+indexinfo[].command[].command "attribute"
+indexinfo[].command[].indexname "kv_array.mykey"
+indexinfo[].command[].command "fast-search"
+indexinfo[].command[].indexname "kv_array.mykey"
+indexinfo[].command[].command "string"
+indexinfo[].command[].indexname "kv_array.mykey"
+indexinfo[].command[].command "type string"
+indexinfo[].command[].indexname "kv_array.mykey"
+indexinfo[].command[].command "word"
+indexinfo[].command[].indexname "kv_array.myval"
+indexinfo[].command[].command "multivalue"
+indexinfo[].command[].indexname "kv_array.myval"
+indexinfo[].command[].command "attribute"
+indexinfo[].command[].indexname "kv_array.myval"
+indexinfo[].command[].command "string"
+indexinfo[].command[].indexname "kv_array.myval"
+indexinfo[].command[].command "type string"
+indexinfo[].command[].indexname "kv_array.myval"
+indexinfo[].command[].command "word"
+indexinfo[].command[].indexname "kv_array"
+indexinfo[].command[].command "multivalue"
+indexinfo[].command[].indexname "kv_array"
+indexinfo[].command[].command "type Array<mykv>"
 indexinfo[].command[].indexname "elem_array.name"
 indexinfo[].command[].command "lowercase"
 indexinfo[].command[].indexname "elem_array.name"

diff --git a/config-model/src/test/derived/array_of_struct_attribute/test.sd b/config-model/src/test/derived/array_of_struct_attribute/test.sd
@@ -15,6 +15,21 @@ schema test {
         indexing: attribute
       }
     }
+    struct mykv {
+      field mykey type string {}
+      field myval type string {}
+    }
+    field kv_array type array<mykv> {
+      struct-field mykey {
+          indexing: attribute
+          match: cased
+          attribute: fast-search
+      }
+      struct-field myval {
+          indexing: attribute
+          match: cased
+      }
+    }
   }
   document-summary rename {
     summary new_elem_array { source: elem_array }

diff --git a/config-model/src/test/java/com/yahoo/schema/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/AttributeListTestCase.java
@@ -77,7 +77,7 @@ void fields_in_array_of_struct_are_derived_into_array_attributes() throws IOExce
 
         assertAttribute("elem_array.name", Attribute.Type.STRING, Attribute.CollectionType.ARRAY, true, attributes.next());
         assertAttribute("elem_array.weight", Attribute.Type.INTEGER, Attribute.CollectionType.ARRAY, false, attributes.next());
-        assertFalse(attributes.hasNext());
+        // more stuff follows, tested elsewhere
     }
 
     @Test