Glebashnik/feed field generator

config-model/src/main/java/com/yahoo/schema/document/SDField.java

@@ -13,6 +13,7 @@
 import com.yahoo.documentmodel.TemporaryUnknownType;
 import com.yahoo.language.Linguistics;
 import com.yahoo.language.process.Embedder;
+import com.yahoo.language.process.TextGenerator;
 import com.yahoo.language.simple.SimpleLinguistics;
 import com.yahoo.schema.Index;
 import com.yahoo.schema.Schema;
@@ -399,12 +400,13 @@ public boolean hasSingleAttribute() {
 
     /** Parse an indexing expression which will use the simple linguistics implementation suitable for testing */
     public void parseIndexingScript(String schemaName, String script) {
-        parseIndexingScript(schemaName, script, new SimpleLinguistics(), Embedder.throwsOnUse.asMap());
+        parseIndexingScript(schemaName, script, new SimpleLinguistics(), Embedder.throwsOnUse.asMap(), TextGenerator.throwsOnUse.asMap());
     }
 
-    public void parseIndexingScript(String schemaName, String script, Linguistics linguistics, Map<String, Embedder> embedders) {
+    public void parseIndexingScript(String schemaName, String script, Linguistics linguistics, 
+                                    Map<String, Embedder> embedders, Map<String, TextGenerator> generators) {
         try {
-            ScriptParserContext config = new ScriptParserContext(linguistics, embedders);
+            ScriptParserContext config = new ScriptParserContext(linguistics, embedders, generators);
             config.setInputStream(new IndexingInput(script));
             setIndexingScript(schemaName, ScriptExpression.newInstance(config));
         } catch (ParseException e) {

config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java

@@ -3,6 +3,7 @@
 
 import com.yahoo.language.Linguistics;
 import com.yahoo.language.process.Embedder;
+import com.yahoo.language.process.TextGenerator;
 import com.yahoo.language.simple.SimpleLinguistics;
 import com.yahoo.schema.document.SDField;
 import com.yahoo.schema.parser.ParseException;
@@ -34,13 +35,14 @@ public void apply(String schemaName, SDField field) {
 
     /** Creates an indexing operation which will use the simple linguistics implementation suitable for testing */
     public static IndexingOperation fromStream(SimpleCharStream input, boolean multiLine) throws ParseException {
-        return fromStream(input, multiLine, new SimpleLinguistics(), Embedder.throwsOnUse.asMap());
+        return fromStream(input, multiLine, new SimpleLinguistics(), Embedder.throwsOnUse.asMap(), 
+                TextGenerator.throwsOnUse.asMap());
     }
 
-    public static IndexingOperation fromStream(SimpleCharStream input, boolean multiLine,
-                                               Linguistics linguistics, Map<String, Embedder> embedders)
-            throws ParseException {
-        ScriptParserContext config = new ScriptParserContext(linguistics, embedders);
+    public static IndexingOperation fromStream(
+            SimpleCharStream input, boolean multiLine, Linguistics linguistics, Map<String, Embedder> embedders, 
+            Map<String, TextGenerator> generators) throws ParseException {
+        ScriptParserContext config = new ScriptParserContext(linguistics, embedders, generators);
         config.setAnnotatorConfig(new AnnotatorConfig());
         config.setInputStream(input);
         ScriptExpression exp;

config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java

@@ -127,6 +127,7 @@ public ApplicationContainerCluster(TreeConfigProducer<?> parent, String configSu
 
         addSimpleComponent("com.yahoo.language.provider.DefaultLinguisticsProvider");
         addSimpleComponent("com.yahoo.language.provider.DefaultEmbedderProvider");
+        addSimpleComponent("com.yahoo.language.provider.DefaultGeneratorProvider");
         addSimpleComponent("com.yahoo.container.jdisc.SecretStoreProvider");
         addSimpleComponent("com.yahoo.container.jdisc.CertificateStoreProvider");
         addSimpleComponent("com.yahoo.container.jdisc.AthenzIdentityProviderProvider");

config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java

@@ -159,7 +159,8 @@ public static boolean isModelIntegrationClass(String className) {
     // This is a hack to allow users to declare components from the model-integration bundle without naming the bundle.
     private static final Set<String> modelIntegrationComponents = Set.of(
             ai.vespa.llm.clients.OpenAI.class.getName(),
-            ai.vespa.llm.clients.LocalLLM.class.getName()
+            ai.vespa.llm.clients.LocalLLM.class.getName(),
+            ai.vespa.llm.generation.LanguageModelTextGenerator.class.getName()
     );
 
 }

config-model/src/main/javacc/SchemaParser.jj

@@ -17,6 +17,7 @@ import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.model.api.ModelContext;
 import com.yahoo.language.Linguistics;
 import com.yahoo.language.process.Embedder;
+import com.yahoo.language.process.TextGenerator;
 import com.yahoo.language.simple.SimpleLinguistics;
 import com.yahoo.search.query.ranking.Diversity;
 import com.yahoo.schema.DistributableResource;
@@ -31,14 +32,11 @@ import com.yahoo.schema.document.MatchAlgorithm;
 import com.yahoo.schema.document.HnswIndexParams;
 import com.yahoo.schema.document.Sorting;
 import com.yahoo.schema.document.Stemming;
-import com.yahoo.schema.document.SDField;
 import com.yahoo.schema.FeatureNames;
 import com.yahoo.schema.fieldoperation.IndexingOperation;
 import com.yahoo.search.schema.RankProfile.InputType;
 import com.yahoo.searchlib.rankingexpression.FeatureList;
 import com.yahoo.searchlib.rankingexpression.Reference;
-import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue;
-import com.yahoo.searchlib.rankingexpression.evaluation.Value;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.IndexedTensor;
 import com.yahoo.tensor.MixedTensor;
@@ -49,7 +47,6 @@ import java.util.Optional;
 import java.util.Map;
 import java.util.List;
 import java.util.ArrayList;
-import java.util.LinkedHashMap;
 import java.util.logging.Level;
 
 /**
@@ -82,7 +79,7 @@ public class SchemaParser {
      */
     @SuppressWarnings("deprecation")
     private IndexingOperation newIndexingOperation(boolean multiline) throws ParseException {
-        return newIndexingOperation(multiline, new SimpleLinguistics(), Embedder.throwsOnUse.asMap());
+        return newIndexingOperation(multiline, new SimpleLinguistics(), Embedder.throwsOnUse.asMap(), TextGenerator.throwsOnUse.asMap());
     }
 
     /**
@@ -91,13 +88,15 @@ public class SchemaParser {
      * @param multiline Whether or not to allow multi-line expressions.
      * @param linguistics What to use for tokenizing.
      */
-    private IndexingOperation newIndexingOperation(boolean multiline, Linguistics linguistics, Map<String, Embedder> embedders) throws ParseException {
+    private IndexingOperation newIndexingOperation(
+            boolean multiline, Linguistics linguistics, Map<String, Embedder> embedders, 
+            Map<String, TextGenerator> generators) throws ParseException {
         SimpleCharStream input = (SimpleCharStream)token_source.input_stream;
         if (token.next != null) {
             input.backup(token.next.image.length());
         }
         try {
-            return IndexingOperation.fromStream(input, multiline, linguistics, embedders);
+            return IndexingOperation.fromStream(input, multiline, linguistics, embedders, generators);
         } finally {
             token.next = null;
             jj_ntk = -1;

container-core/src/main/java/com/yahoo/language/provider/DefaultGeneratorProvider.java

@@ -0,0 +1,26 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.language.provider;
+
+import com.yahoo.component.annotation.Inject;
+import com.yahoo.container.di.componentgraph.Provider;
+import com.yahoo.language.process.TextGenerator;
+
+/**
+ * Provides the default generator implementation if no generator component has been explicitly configured
+ * (dependency injection will fall back to providers if no components of the requested type is found).
+ *
+ * @author lesters
+ */
+@SuppressWarnings("unused") // Injected
+public class DefaultGeneratorProvider implements Provider<TextGenerator> {
+
+    @Inject
+    public DefaultGeneratorProvider() { }
+
+    @Override
+    public TextGenerator get() { return TextGenerator.throwsOnUse; }
+
+    @Override
+    public void deconstruct() {}
+
+}

docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java

@@ -22,7 +22,9 @@
 import com.yahoo.io.GrowableByteBuffer;
 import com.yahoo.language.Linguistics;
 import com.yahoo.language.process.Embedder;
+import com.yahoo.language.process.TextGenerator;
 import com.yahoo.language.provider.DefaultEmbedderProvider;
+import com.yahoo.language.provider.DefaultGeneratorProvider;
 import com.yahoo.vespa.configdefinition.IlscriptsConfig;
 import com.yahoo.vespa.indexinglanguage.AdapterFactory;
 import com.yahoo.vespa.indexinglanguage.SimpleAdapterFactory;
@@ -58,8 +60,18 @@ public Expression selectExpression(DocumentType documentType, String fieldName)
     public IndexingProcessor(DocumentTypeManager documentTypeManager,
                              IlscriptsConfig ilscriptsConfig,
                              Linguistics linguistics,
-                             ComponentRegistry<Embedder> embedders) {
-        this(documentTypeManager, new ScriptManager(documentTypeManager, ilscriptsConfig, linguistics, toMap(embedders)));
+                             ComponentRegistry<Embedder> embedders,
+                             ComponentRegistry<TextGenerator> generators) {
+        this(
+                documentTypeManager, 
+                new ScriptManager(
+                        documentTypeManager, 
+                        ilscriptsConfig, 
+                        linguistics, 
+                        toMap(embedders, DefaultEmbedderProvider.class), 
+                        toMap(generators, DefaultGeneratorProvider.class)
+                )
+        );
     }
 
     public IndexingProcessor(DocumentTypeManager documentTypeManager,
@@ -137,11 +149,11 @@ private void processRemove(DocumentRemove input, List<DocumentOperation> out) {
         out.add(input);
     }
 
-    private static Map<String, Embedder> toMap(ComponentRegistry<Embedder> embedders) {
-        var map = embedders.allComponentsById().entrySet().stream()
-                    .collect(Collectors.toMap(e -> e.getKey().stringValue(), Map.Entry::getValue));
+    private static <T> Map<String, T> toMap(ComponentRegistry<T> registry, Class<?> defaultProviderClass) {
+        var map = registry.allComponentsById().entrySet().stream()
+                .collect(Collectors.toMap(e -> e.getKey().stringValue(), Map.Entry::getValue));
         if (map.size() > 1) {
-            map.remove(DefaultEmbedderProvider.class.getName());
+            map.remove(defaultProviderClass.getName());
             // Ideally, this should be handled by dependency injection, however for now this workaround is necessary.
         }
         return map;

docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java

@@ -6,6 +6,7 @@
 import com.yahoo.language.Linguistics;
 
 import com.yahoo.language.process.Embedder;
+import com.yahoo.language.process.TextGenerator;
 import com.yahoo.vespa.configdefinition.IlscriptsConfig;
 import com.yahoo.vespa.indexinglanguage.ScriptParserContext;
 import com.yahoo.vespa.indexinglanguage.expressions.InputExpression;
@@ -31,9 +32,9 @@ public class ScriptManager {
     private final DocumentTypeManager documentTypeManager;
 
     public ScriptManager(DocumentTypeManager documentTypeManager, IlscriptsConfig config, Linguistics linguistics,
-                         Map<String, Embedder> embedders) {
+                         Map<String, Embedder> embedders, Map<String, TextGenerator> generators) {
         this.documentTypeManager = documentTypeManager;
-        documentFieldScripts = createScriptsMap(documentTypeManager, config, linguistics, embedders);
+        documentFieldScripts = createScriptsMap(documentTypeManager, config, linguistics, embedders, generators);
     }
 
     private Map<String, DocumentScript> getScripts(DocumentType inputType) {
@@ -70,9 +71,10 @@ public DocumentScript getScript(DocumentType inputType, String inputFieldName) {
     private static Map<String, Map<String, DocumentScript>>  createScriptsMap(DocumentTypeManager documentTypes,
                                                                               IlscriptsConfig config,
                                                                               Linguistics linguistics,
-                                                                              Map<String, Embedder> embedders) {
+                                                                              Map<String, Embedder> embedders,
+                                                                              Map<String, TextGenerator> generators) {
         Map<String, Map<String, DocumentScript>> documentFieldScripts = new HashMap<>(config.ilscript().size());
-        ScriptParserContext parserContext = new ScriptParserContext(linguistics, embedders);
+        ScriptParserContext parserContext = new ScriptParserContext(linguistics, embedders, generators);
         parserContext.getAnnotatorConfig().setMaxTermOccurrences(config.maxtermoccurrences());
         parserContext.getAnnotatorConfig().setMaxTokenizeLength(config.fieldmatchmaxlength());
 

docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTestCase.java

@@ -15,6 +15,7 @@
 import com.yahoo.document.update.AssignValueUpdate;
 import com.yahoo.document.update.FieldUpdate;
 import com.yahoo.language.process.Embedder;
+import com.yahoo.language.process.TextGenerator;
 import com.yahoo.tensor.Tensor;
 import com.yahoo.tensor.Tensors;
 import com.yahoo.tensor.TensorType;
@@ -251,7 +252,9 @@ public void testEmbedBinarizeAndPack() {
         config.ilscript(new IlscriptsConfig.Ilscript.Builder().doctype("test")
                                                               .content("input myText | embed | binarize | pack_bits | attribute embedding")
                                                               .docfield("myText"));
-        var scripts = new ScriptManager(documentTypes, new IlscriptsConfig(config), null, Map.of("test", new TestEmbedder()));
+        var scripts = new ScriptManager(documentTypes, new IlscriptsConfig(config), null, 
+                Map.of("test", new TestEmbedder()), TextGenerator.throwsOnUse.asMap());
+
         assertNotNull(scripts.getScript(documentTypes.getDocumentType("test")));
 
         var tester = new IndexingProcessorTester(documentTypes, scripts);

docprocs/src/test/java/com/yahoo/docprocs/indexing/IndexingProcessorTester.java

@@ -71,6 +71,7 @@ private static IndexingProcessor newProcessor(String configId) {
         return new IndexingProcessor(new DocumentTypeManager(ConfigGetter.getConfig(DocumentmanagerConfig.class, configId)),
                                      ConfigGetter.getConfig(IlscriptsConfig.class, configId),
                                      new SimpleLinguistics(),
+                                     new ComponentRegistry<>(),
                                      new ComponentRegistry<>());
     }
 

docprocs/src/test/java/com/yahoo/docprocs/indexing/ScriptManagerTestCase.java

@@ -4,6 +4,7 @@
 import com.yahoo.document.DocumentType;
 import com.yahoo.document.DocumentTypeManager;
 import com.yahoo.language.process.Embedder;
+import com.yahoo.language.process.TextGenerator;
 import com.yahoo.vespa.configdefinition.IlscriptsConfig;
 import org.junit.Test;
 
@@ -27,7 +28,7 @@ public void requireThatScriptsAreAppliedToSubType() {
         IlscriptsConfig.Builder config = new IlscriptsConfig.Builder();
         config.ilscript(new IlscriptsConfig.Ilscript.Builder().doctype("newssummary")
                                                               .content("input title | index title"));
-        ScriptManager scriptMgr = new ScriptManager(typeMgr, new IlscriptsConfig(config), null, Embedder.throwsOnUse.asMap());
+        ScriptManager scriptMgr = new ScriptManager(typeMgr, new IlscriptsConfig(config), null, Embedder.throwsOnUse.asMap(), TextGenerator.throwsOnUse.asMap());
         assertNotNull(scriptMgr.getScript(typeMgr.getDocumentType("newsarticle")));
         assertNull(scriptMgr.getScript(new DocumentType("unknown")));
     }
@@ -41,22 +42,22 @@ public void requireThatScriptsAreAppliedToSuperType() {
         IlscriptsConfig.Builder config = new IlscriptsConfig.Builder();
         config.ilscript(new IlscriptsConfig.Ilscript.Builder().doctype("newsarticle")
                                                               .content("input title | index title"));
-        ScriptManager scriptMgr = new ScriptManager(typeMgr, new IlscriptsConfig(config), null, Embedder.throwsOnUse.asMap());
+        ScriptManager scriptMgr = new ScriptManager(typeMgr, new IlscriptsConfig(config), null, Embedder.throwsOnUse.asMap(), TextGenerator.throwsOnUse.asMap());
         assertNotNull(scriptMgr.getScript(typeMgr.getDocumentType("newssummary")));
         assertNull(scriptMgr.getScript(new DocumentType("unknown")));
     }
 
     @Test
     public void requireThatEmptyConfigurationDoesNotThrow() {
         var typeMgr = DocumentTypeManager.fromFile("src/test/cfg/documentmanager_inherit.cfg");
-        ScriptManager scriptMgr = new ScriptManager(typeMgr, new IlscriptsConfig(new IlscriptsConfig.Builder()), null, Embedder.throwsOnUse.asMap());
+        ScriptManager scriptMgr = new ScriptManager(typeMgr, new IlscriptsConfig(new IlscriptsConfig.Builder()), null, Embedder.throwsOnUse.asMap(), TextGenerator.throwsOnUse.asMap());
         assertNull(scriptMgr.getScript(new DocumentType("unknown")));
     }
 
     @Test
     public void requireThatUnknownDocumentTypeReturnsNull() {
         var typeMgr = DocumentTypeManager.fromFile("src/test/cfg/documentmanager_inherit.cfg");
-        ScriptManager scriptMgr = new ScriptManager(typeMgr, new IlscriptsConfig(new IlscriptsConfig.Builder()), null, Embedder.throwsOnUse.asMap());
+        ScriptManager scriptMgr = new ScriptManager(typeMgr, new IlscriptsConfig(new IlscriptsConfig.Builder()), null, Embedder.throwsOnUse.asMap(), TextGenerator.throwsOnUse.asMap());
         for (Iterator<DocumentType> it = typeMgr.documentTypeIterator(); it.hasNext(); ) {
             assertNull(scriptMgr.getScript(it.next()));
         }

indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ScriptParser.java

@@ -47,6 +47,8 @@ private static <T extends Expression> T parse(ScriptParserContext context, Parse
         parser.setDefaultFieldName(context.getDefaultFieldName());
         parser.setLinguistics(context.getLinguistcs());
         parser.setEmbedders(context.getEmbedders());
+        parser.setGenerators(context.getGenerators());
+
         try {
             return method.call(parser);
         } catch (ParseException e) {

indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ScriptParserContext.java

@@ -3,6 +3,7 @@
 
 import com.yahoo.language.Linguistics;
 import com.yahoo.language.process.Embedder;
+import com.yahoo.language.process.TextGenerator;
 import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig;
 import com.yahoo.vespa.indexinglanguage.parser.CharStream;
 
@@ -17,12 +18,14 @@ public class ScriptParserContext {
     private AnnotatorConfig annotatorConfig = new AnnotatorConfig();
     private Linguistics linguistics;
     private final Map<String, Embedder> embedders;
+    private final Map<String, TextGenerator> generators;
     private String defaultFieldName = null;
     private CharStream inputStream = null;
 
-    public ScriptParserContext(Linguistics linguistics, Map<String, Embedder> embedders) {
+    public ScriptParserContext(Linguistics linguistics, Map<String, Embedder> embedders, Map<String, TextGenerator> generators) {
         this.linguistics = linguistics;
         this.embedders = embedders;
+        this.generators = generators;
     }
 
     public AnnotatorConfig getAnnotatorConfig() {
@@ -46,6 +49,9 @@ public ScriptParserContext setLinguistics(Linguistics linguistics) {
     public Map<String, Embedder> getEmbedders() {
         return Collections.unmodifiableMap(embedders);
     }
+
+    public Map<String, TextGenerator> getGenerators() { return Collections.unmodifiableMap(generators);
+    }
 
     public String getDefaultFieldName() {
         return defaultFieldName;