New tokenizer for gpt4o (#714)

* Add new tokenizer for GPT-4o and 4o mini
* Fix service bootstrap warnings caused by unselected tokenizer. Default
to GPT4 tokenizer.

Author: dluc

extensions/OpenAI/OpenAI/Tokenizers/GPT2Tokenizer.cs

@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using Microsoft.ML.Tokenizers;
 
@@ -11,7 +10,6 @@ namespace Microsoft.KernelMemory.AI.OpenAI;
 /// <summary>
 /// TikToken GPT2 tokenizer (gpt2.tiktoken)
 /// </summary>
-[Experimental("KMEXP01")]
 public sealed class GPT2Tokenizer : ITextTokenizer
 {
     private static readonly Tokenizer s_tokenizer = Tokenizer.CreateTiktokenForModel("gpt2");

extensions/OpenAI/OpenAI/Tokenizers/GPT3Tokenizer.cs

@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using Microsoft.ML.Tokenizers;
 
@@ -11,7 +10,6 @@ namespace Microsoft.KernelMemory.AI.OpenAI;
 /// <summary>
 /// TikToken GPT3 tokenizer (p50k_base.tiktoken)
 /// </summary>
-[Experimental("KMEXP01")]
 public sealed class GPT3Tokenizer : ITextTokenizer
 {
     private static readonly Tokenizer s_tokenizer = Tokenizer.CreateTiktokenForModel("text-davinci-003");

extensions/OpenAI/OpenAI/Tokenizers/GPT4Tokenizer.cs

@@ -1,20 +1,19 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using Microsoft.ML.Tokenizers;
 
 // ReSharper disable once CheckNamespace
 namespace Microsoft.KernelMemory.AI.OpenAI;
 
 /// <summary>
-/// GPT 3.5 and GPT 4+ tokenizer (cl100k_base.tiktoken + special tokens)
+/// GPT 3.5 and GPT 4 tokenizer (cl100k_base.tiktoken + special tokens)
 /// </summary>
-[Experimental("KMEXP01")]
 public sealed class GPT4Tokenizer : ITextTokenizer
 {
-    private static readonly Tokenizer s_tokenizer = Tokenizer.CreateTiktokenForModel("gpt-4", new Dictionary<string, int> { { "<|im_start|>", 100264 }, { "<|im_end|>", 100265 } });
+    private static readonly Tokenizer s_tokenizer = Tokenizer.CreateTiktokenForModel("gpt-4",
+        new Dictionary<string, int> { { "<|im_start|>", 100264 }, { "<|im_end|>", 100265 } });
 
     /// <inheritdoc />
     public int CountTokens(string text)

extensions/OpenAI/OpenAI/Tokenizers/GPT4oTokenizer.cs

@@ -0,0 +1,30 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ML.Tokenizers;
+
+// ReSharper disable once CheckNamespace
+namespace Microsoft.KernelMemory.AI.OpenAI;
+
+/// <summary>
+/// GPT 4o / 4o mini tokenizer (cl200k_base.tiktoken + special tokens)
+/// </summary>
+// ReSharper disable once InconsistentNaming
+public sealed class GPT4oTokenizer : ITextTokenizer
+{
+    private static readonly Tokenizer s_tokenizer = Tokenizer.CreateTiktokenForModel("gpt-4o",
+        new Dictionary<string, int> { { "<|im_start|>", 100264 }, { "<|im_end|>", 100265 } });
+
+    /// <inheritdoc />
+    public int CountTokens(string text)
+    {
+        return s_tokenizer.CountTokens(text);
+    }
+
+    /// <inheritdoc />
+    public IReadOnlyList<string> GetTokens(string text)
+    {
+        return s_tokenizer.Encode(text, out string? _).Select(t => t.Value).ToList();
+    }
+}

service/Service/ServiceConfiguration.cs

@@ -6,6 +6,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.KernelMemory.AI;
 using Microsoft.KernelMemory.AI.Anthropic;
+using Microsoft.KernelMemory.AI.OpenAI;
 using Microsoft.KernelMemory.DocumentStorage.DevTools;
 using Microsoft.KernelMemory.MemoryDb.SQLServer;
 using Microsoft.KernelMemory.MemoryStorage;
@@ -212,15 +213,19 @@ private void ConfigureIngestionEmbeddingGenerators(IKernelMemoryBuilder builder)
                 case string y when y.Equals("AzureOpenAIEmbedding", StringComparison.OrdinalIgnoreCase):
                 {
                     var instance = this.GetServiceInstance<ITextEmbeddingGenerator>(builder,
-                        s => s.AddAzureOpenAIEmbeddingGeneration(this.GetServiceConfig<AzureOpenAIConfig>("AzureOpenAIEmbedding")));
+                        s => s.AddAzureOpenAIEmbeddingGeneration(
+                            config: this.GetServiceConfig<AzureOpenAIConfig>("AzureOpenAIEmbedding"),
+                            textTokenizer: new GPT4Tokenizer()));
                     builder.AddIngestionEmbeddingGenerator(instance);
                     break;
                 }
 
                 case string x when x.Equals("OpenAI", StringComparison.OrdinalIgnoreCase):
                 {
                     var instance = this.GetServiceInstance<ITextEmbeddingGenerator>(builder,
-                        s => s.AddOpenAITextEmbeddingGeneration(this.GetServiceConfig<OpenAIConfig>("OpenAI")));
+                        s => s.AddOpenAITextEmbeddingGeneration(
+                            config: this.GetServiceConfig<OpenAIConfig>("OpenAI"),
+                            textTokenizer: new GPT4Tokenizer()));
                     builder.AddIngestionEmbeddingGenerator(instance);
                     break;
                 }
@@ -345,11 +350,15 @@ private void ConfigureRetrievalEmbeddingGenerator(IKernelMemoryBuilder builder)
         {
             case string x when x.Equals("AzureOpenAI", StringComparison.OrdinalIgnoreCase):
             case string y when y.Equals("AzureOpenAIEmbedding", StringComparison.OrdinalIgnoreCase):
-                builder.Services.AddAzureOpenAIEmbeddingGeneration(this.GetServiceConfig<AzureOpenAIConfig>("AzureOpenAIEmbedding"));
+                builder.Services.AddAzureOpenAIEmbeddingGeneration(
+                    config: this.GetServiceConfig<AzureOpenAIConfig>("AzureOpenAIEmbedding"),
+                    textTokenizer: new GPT4Tokenizer());
                 break;
 
             case string x when x.Equals("OpenAI", StringComparison.OrdinalIgnoreCase):
-                builder.Services.AddOpenAITextEmbeddingGeneration(this.GetServiceConfig<OpenAIConfig>("OpenAI"));
+                builder.Services.AddOpenAITextEmbeddingGeneration(
+                    config: this.GetServiceConfig<OpenAIConfig>("OpenAI"),
+                    textTokenizer: new GPT4Tokenizer());
                 break;
 
             default:
@@ -412,11 +421,15 @@ private void ConfigureTextGenerator(IKernelMemoryBuilder builder)
         {
             case string x when x.Equals("AzureOpenAI", StringComparison.OrdinalIgnoreCase):
             case string y when y.Equals("AzureOpenAIText", StringComparison.OrdinalIgnoreCase):
-                builder.Services.AddAzureOpenAITextGeneration(this.GetServiceConfig<AzureOpenAIConfig>("AzureOpenAIText"));
+                builder.Services.AddAzureOpenAITextGeneration(
+                    config: this.GetServiceConfig<AzureOpenAIConfig>("AzureOpenAIText"),
+                    textTokenizer: new GPT4Tokenizer());
                 break;
 
             case string x when x.Equals("OpenAI", StringComparison.OrdinalIgnoreCase):
-                builder.Services.AddOpenAITextGeneration(this.GetServiceConfig<OpenAIConfig>("OpenAI"));
+                builder.Services.AddOpenAITextGeneration(
+                    config: this.GetServiceConfig<OpenAIConfig>("OpenAI"),
+                    textTokenizer: new GPT4Tokenizer());
                 break;
 
             case string x when x.Equals("Anthropic", StringComparison.OrdinalIgnoreCase):