Introduced IpromptStore

Author: alkampfergit

src/KernelMemory.Extensions.FunctionalTests/Helper/OpenaiRagQueryExecutorTests.cs

@@ -0,0 +1,93 @@
+using KernelMemory.Extensions.QueryPipeline;
+using Microsoft.Extensions.Logging;
+using Microsoft.SemanticKernel;
+using Moq;
+using Fasterflect;
+
+namespace KernelMemory.Extensions.FunctionalTests.Helper;
+
+public class OpenaiRagQueryExecutorTests
+{
+    private Kernel _kernel;
+    private OpenaiRagQueryExecutor _sut;
+    private Mock<IPromptStore> _mockPromptStore;
+    private Mock<ILogger<StandardRagQueryExecutor>> _mockLogger;
+
+    public OpenaiRagQueryExecutorTests()
+    {
+        _kernel = new Kernel();
+        _mockPromptStore = new Mock<IPromptStore>();
+        _mockLogger = new Mock<ILogger<StandardRagQueryExecutor>>();
+        _sut = new OpenaiRagQueryExecutor(_kernel, new OpenAIRagQueryExecutorConfiguration(), _mockLogger.Object, _mockPromptStore.Object);
+    }
+
+    [Fact]
+    public async Task GetPromptAsync_ShouldReturnPromptFromMock()
+    {
+        // Arrange
+        var expectedPrompt = "Test Prompt";
+        _mockPromptStore.Setup(store => store.GetPromptAsync(It.IsAny<string>())).ReturnsAsync(expectedPrompt);
+
+        // Act
+        var task = (Task<string>)_sut.CallMethod("GetPromptAsync");
+        var actualPrompt = await task;
+
+        // Assert
+        Assert.Equal(expectedPrompt, actualPrompt);
+    }
+
+    [Fact]
+    public async Task GetPromptAsync_Should_validate_log_called()
+    {
+        // Arrange
+        var invalidPrompt = "Invalid Prompt";
+        _mockPromptStore.Setup(store => store.GetPromptAsync(It.IsAny<string>())).ReturnsAsync(invalidPrompt);
+
+        // Act
+        var task = (Task<string>)_sut.CallMethod("GetPromptAsync");
+        var actualPrompt = await task;
+
+        // Assert
+        // template is not valid, we expect two log error
+        // void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter);
+        _mockLogger.Verify(
+            logger => logger.Log(
+                LogLevel.Error,
+                It.IsAny<EventId>(),
+                It.Is<It.IsAnyType>((value, type) => value.ToString()!.Contains("{{$question}}")),
+                It.IsAny<Exception>(),
+                It.IsAny<Func<It.IsAnyType, Exception?, string>>()),
+            Times.Once);
+
+        _mockLogger.Verify(
+            logger => logger.Log(
+                LogLevel.Error,
+                It.IsAny<EventId>(),
+                It.Is<It.IsAnyType>((value, type) => value.ToString()!.Contains("{{$documents}}")),
+                It.IsAny<Exception>(),
+                It.IsAny<Func<It.IsAnyType, Exception?, string>>()),
+            Times.Once);
+
+         // verify that store method of Ipromptstore is not called
+        _mockPromptStore.Verify(
+            store => store.SetPromptAsync(It.IsAny<string>(), It.IsAny<string>()),
+            Times.Never);
+    }
+
+    [Fact]
+    public async Task If_prompt_not_saved_reload()
+    {
+        // Arrange
+        _mockPromptStore.Setup(store => store.GetPromptAsync(It.IsAny<string>())).ReturnsAsync((String?) null); 
+
+        // Act
+        var task = (Task<string>)_sut.CallMethod("GetPromptAsync");
+        var actualPrompt = await task;
+
+        // Assert
+        // verify that store method of Ipromptstore is called
+        _mockPromptStore.Verify(
+            store => store.SetPromptAsync("OpenaiRagQueryExecutor", It.IsAny<string>()),
+            Times.Once);
+    }
+}

src/KernelMemory.Extensions/IPromptStore.cs

@@ -0,0 +1,59 @@
+using System.Threading.Tasks;
+
+namespace KernelMemory.Extensions;
+
+/// <summary>
+/// <para>
+/// To let the user to change prompt used in the various part of extensions
+/// we introduce an interface that is capable of providing user prompt given a key.
+/// The user can implement its own implementation of this interface and pass it to the extension.
+/// </para>
+/// <para>
+/// We do not use propmtp provider from kernel memory because we need to have a way to set
+/// the prompt into the provider for a better experience of the user
+/// </para>
+/// </summary>
+public interface IPromptStore
+{
+    /// <summary>
+    /// Get the prompt for the given key.
+    /// </summary>
+    /// <param name="key">The key for which the prompt is requested.</param>
+    /// <returns>The prompt for the given key or null if the prompt is not present. If null is returned the 
+    /// various components will use some default prompts.</returns>
+    Task<string?> GetPromptAsync(string key);
+
+    /// <summary>
+    /// Allow setting prompt value.
+    /// </summary>
+    /// <param name="key"></param>
+    /// <param name="prompt"></param>
+    Task SetPromptAsync(string key, string prompt);
+}
+
+public class NullPromptStore : IPromptStore
+{
+    public static NullPromptStore Instance { get; } = new NullPromptStore();
+
+    /// <summary>
+    /// Get the prompt for the given key.
+    /// </summary>
+    /// <param name="key">The key for which the prompt is requested.</param>
+    /// <returns>An empty prompt.</returns>
+    public Task<string> GetPromptAsync(string key)
+    {
+        return Task.FromResult(string.Empty);
+    }
+
+    /// <summary>
+    /// Allow setting prompt value.
+    /// </summary>
+    /// <param name="key">The key for which the prompt is set.</param>
+    /// <param name="prompt">The prompt value to set.</param>
+    public Task SetPromptAsync(string key, string prompt)
+    {
+        // No operation as this is a null implementation.
+        return Task.CompletedTask;
+    }
+}
+

src/KernelMemory.Extensions/QueryPipeline/OpenaiRagQueryExecutor.cs

@@ -24,7 +24,7 @@ public class OpenAIRagQueryExecutorConfiguration
     /// the default <see cref="ModelId"/> automatically
     /// we will use a standard gpt3.5 model
     /// </summary>
-    public string ModelName { get; set; } = "gpt35";
+    public string ModelName { get; set; } = "gpt-35";
 
     /// <summary>
     /// This is the modelId configured in Semantic Kernel 
@@ -63,16 +63,26 @@ public class OpenaiRagQueryExecutor : BasicQueryHandler
     private readonly OpenAIRagQueryExecutorConfiguration _config;
     private readonly Tokenizer _tokenizer;
     private readonly ILogger<StandardRagQueryExecutor> _log;
+    private readonly IPromptStore _promptStore;
+
+    private const string DefaultPrompt = @"You are an AI assistant that helps users answer questions given a specific context. You will be given a context and asked a question based on that context. Your answer should be as precise as possible and should only come from the context.
+Please add all documents used as citations.
+Question: {{$question}}
+
+Documents:
+{{$documents}}";
 
     public OpenaiRagQueryExecutor(
         Kernel kernel,
         OpenAIRagQueryExecutorConfiguration? config = null,
-        ILogger<StandardRagQueryExecutor>? log = null)
+        ILogger<StandardRagQueryExecutor>? log = null,
+        IPromptStore? promptStore = null)
     {
         _kernel = kernel;
         _config = config ?? new OpenAIRagQueryExecutorConfiguration();
         _tokenizer = TiktokenTokenizer.CreateForModel(_config.ModelName);
         _log = log ?? DefaultLogger<StandardRagQueryExecutor>.Instance;
+        _promptStore = promptStore ?? NullPromptStore.Instance;
     }
 
     protected override async Task OnHandleAsync(
@@ -170,9 +180,9 @@ protected override async Task OnHandleAsync(
 
     private class GptAnswer
     {
-        public string Answer { get; set; }
+        public string Answer { get; set; } = null!;
 
-        public HashSet<int> Documents { get; set; }
+        public HashSet<int> Documents { get; set; } = null!;
     }
 
     /// <summary>
@@ -194,22 +204,19 @@ private class GptAnswer
             [Description("Answer of the question")] string answer,
             [Description("Documents used to formulate the answer")] int[] documents
         ) =>
-        {
-        }, "return_result");
+            {
+            }, "return_result");
         var plugin = KernelPluginFactory.CreateFromFunctions("MyPlugin", [function]);
         var openAIFunction = plugin.GetFunctionsMetadata().First().ToOpenAIFunction();
 
+        string prompt = await GetPromptAsync();
+
         // Create a template for chat with settings
         var chat = _kernel.CreateFunctionFromPrompt(new PromptTemplateConfig()
         {
             Name = "Rag",
             Description = "Answer user question with documents.",
-            Template = @"You are an AI assistant that helps users answer questions given a specific context. You will be given a context and asked a question based on that context. Your answer should be as precise as possible and should only come from the context.
-Please add all documents used as citations.
-Question: {{$question}}
-
-Documents:
-{{$documents}}",
+            Template = prompt,
             TemplateFormat = "semantic-kernel",
             InputVariables =
             [
@@ -252,4 +259,27 @@ Please add all documents used as citations.
 
         return null;
     }
+
+    private async Task<string> GetPromptAsync()
+    {
+        var prompt = await _promptStore.GetPromptAsync(nameof(OpenaiRagQueryExecutor));
+        if (prompt == null)
+        {
+            //Set the default prompt into the storage so the user can change.
+            await _promptStore.SetPromptAsync(nameof(OpenaiRagQueryExecutor), DefaultPrompt);
+            prompt = DefaultPrompt;
+        }
+
+        if (!prompt.Contains("{{$question}}"))
+        {
+            _log.LogError("The prompt does not contain {{$question}} placeholder, the prompt will not work correctly");
+        }
+
+        if (!prompt.Contains("{{$documents}}"))
+        {
+            _log.LogError("The prompt does not contain {{$documents}} placeholder, the prompt will not work correctly");
+        }
+
+        return prompt;
+    }
 }

src/NullPromptStore.cs

@@ -0,0 +1,31 @@
+using System.Threading.Tasks;
+
+namespace KernelMemory.Extensions
+{
+    /// <summary>
+    /// A null implementation of the IPromptStore interface that returns empty prompts.
+    /// </summary>
+    public class NullPromptStore : IPromptStore
+    {
+        /// <summary>
+        /// Get the prompt for the given key.
+        /// </summary>
+        /// <param name="key">The key for which the prompt is requested.</param>
+        /// <returns>An empty prompt.</returns>
+        public Task<string> GetPromptAsync(string key)
+        {
+            return Task.FromResult(string.Empty);
+        }
+
+        /// <summary>
+        /// Allow setting prompt value.
+        /// </summary>
+        /// <param name="key">The key for which the prompt is set.</param>
+        /// <param name="prompt">The prompt value to set.</param>
+        public Task SetPromptAsync(string key, string prompt)
+        {
+            // No operation as this is a null implementation.
+            return Task.CompletedTask;
+        }
+    }
+}