Skip to content

Commit

Permalink
Update with SK 1.0.0-beta3 & simplify plugin API
Browse files Browse the repository at this point in the history
  • Loading branch information
Swimburger committed Oct 24, 2023
1 parent 6e7a2cd commit 6a8a5c0
Show file tree
Hide file tree
Showing 6 changed files with 66 additions and 115 deletions.
46 changes: 16 additions & 30 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ Next, register the `TranscriptPlugin` into your kernel:
```csharp
using AssemblyAI.SemanticKernel;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Orchestration;

// Build your kernel
var kernel = new KernelBuilder().Build();
Expand All @@ -36,7 +35,7 @@ var kernel = new KernelBuilder().Build();
string apiKey = Environment.GetEnvironmentVariable("ASSEMBLYAI_API_KEY")
?? throw new Exception("ASSEMBLYAI_API_KEY env variable not configured.");

var transcriptPlugin = kernel.ImportSkill(
kernel.ImportFunctions(
new TranscriptPlugin(apiKey: apiKey),
TranscriptPlugin.PluginName
);
Expand All @@ -46,47 +45,37 @@ var transcriptPlugin = kernel.ImportSkill(

Get the `Transcribe` function from the transcript plugin and invoke it with the context variables.
```csharp
var function = kernel.Skills
var function = kernel.Functions
.GetFunction(TranscriptPlugin.PluginName, TranscriptPlugin.TranscribeFunctionName);
var context = kernel.CreateNewContext();
context.Variables["audioUrl"] = "https://storage.googleapis.com/aai-docs-samples/espn.m4a";
await function.InvokeAsync(context);
Console.WriteLine(context.Result);
context.Variables["INPUT"] = "https://storage.googleapis.com/aai-docs-samples/espn.m4a";
var result = await function.InvokeAsync(context);
Console.WriteLine(result.GetValue<string());
```

The `context.Result` property contains the transcript text if successful.
You can get the transcript using `result.GetValue<string>()`.

You can also upload local audio and video file. To do this:
- Set the `TranscriptPlugin.AllowFileSystemAccess` property to `true`
- Configure the path of the file to upload as the `filePath` parameter
- Set the `TranscriptPlugin.AllowFileSystemAccess` property to `true`.
- Configure the `INPUT` variable with a local file path.

```csharp
var transcriptPlugin = kernel.ImportSkill(
kernel.ImportFunctions(
new TranscriptPlugin(apiKey: apiKey)
{
AllowFileSystemAccess = true
},
TranscriptPlugin.PluginName
);
var function = kernel.Skills
var function = kernel.Functions
.GetFunction(TranscriptPlugin.PluginName, TranscriptPlugin.TranscribeFunctionName);
var context = kernel.CreateNewContext();
context.Variables["filePath"] = "./espn.m4a";
await function.InvokeAsync(context);
Console.WriteLine(context.Result);
context.Variables["INPUT"] = "./espn.m4a";
var result = await function.InvokeAsync(context);
Console.WriteLine(result.GetValue<string>());
```

If `filePath` and `audioUrl` are specified, the `filePath` will be used to upload the file and `audioUrl` will be overridden.

Lastly, you can also use the `INPUT` variable, so you can transcribe a file like this.

```csharp
var function = kernel.Skills
.GetFunction(TranscriptPlugin.PluginName, TranscriptPlugin.TranscribeFunctionName);
var context = await function.InvokeAsync("./espn.m4a");
```

Or from within a semantic function like this.
You can also invoke the function from within a semantic function like this.

```csharp
var prompt = """
Expand All @@ -97,13 +86,10 @@ var prompt = """
""";
var context = kernel.CreateNewContext();
var function = kernel.CreateSemanticFunction(prompt);
await function.InvokeAsync(context);
Console.WriteLine(context.Result);
var result = await function.InvokeAsync(context);
Console.WriteLine(result.GetValue<string>());
```

If the `INPUT` variable is a URL, it'll be used as the `audioUrl`, otherwise, it'll be used as the `filePath`.
If either `audioUrl` or `filePath` are configured, `INPUT` is ignored.

All the code above explicitly invokes the transcript plugin, but it can also be invoked as part of a plan.
Check out [the Sample project](./src/Sample/Program.cs#L96)) which uses a plan to transcribe an audio file in addition to explicit invocation.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.SemanticKernel">
<Version>0.24.230918.1-preview</Version>
<Version>1.0.0-beta3</Version>
</PackageReference>
<PackageReference Include="System.Net.Http.Json">
<Version>7.0.1</Version>
Expand Down
73 changes: 25 additions & 48 deletions src/AssemblyAI.SemanticKernel/TranscriptPlugin.cs
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;
using Microsoft.SemanticKernel.Orchestration;
using Microsoft.SemanticKernel.SkillDefinition;
using Microsoft.SemanticKernel;

namespace AssemblyAI.SemanticKernel
{
Expand All @@ -26,19 +25,21 @@ public TranscriptPlugin(string apiKey)
public const string TranscribeFunctionName = nameof(Transcribe);

[SKFunction, Description("Transcribe an audio or video file to text.")]
[SKParameter("filePath", @"The path of the audio or video file.
If filePath is configured, the file will be uploaded to AssemblyAI, and then used as the audioUrl to transcribe.
Optional if audioUrl is configured. The uploaded file will override the audioUrl parameter.")]
[SKParameter("audioUrl", @"The public URL of the audio or video file to transcribe.
Optional if filePath is configured.")]
public async Task<string> Transcribe(SKContext context)
public async Task<string> Transcribe(
[Description("The public URL or the local path of the audio or video file to transcribe.")]
string input
)
{
SetPathAndUrl(context, out var filePath, out var audioUrl);
if (string.IsNullOrEmpty(input))
{
throw new Exception("The INPUT parameter is required.");
}

using (var httpClient = new HttpClient())
{
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(_apiKey);

if (filePath != null)
string audioUrl;
if (TryGetPath(input, out var filePath))
{
if (AllowFileSystemAccess == false)
{
Expand All @@ -49,57 +50,33 @@ public async Task<string> Transcribe(SKContext context)

audioUrl = await UploadFileAsync(filePath, httpClient);
}
else
{
audioUrl = input;
}

var transcript = await CreateTranscriptAsync(audioUrl, httpClient);
transcript = await WaitForTranscriptToProcess(transcript, httpClient);
return transcript.Text ?? throw new Exception("Transcript text is null. This should not happen.");
}
}

private static void SetPathAndUrl(SKContext context, out string filePath, out string audioUrl)
private static bool TryGetPath(string input, out string filePath)
{
filePath = null;
audioUrl = null;
if (context.Variables.TryGetValue("filePath", out filePath))
{
return;
}

if (context.Variables.TryGetValue("audioUrl", out audioUrl))
{
var uri = new Uri(audioUrl);
if (uri.IsFile)
{
filePath = uri.LocalPath;
audioUrl = null;
}
else
{
return;
}
}

context.Variables.TryGetValue("INPUT", out var input);
if (input == null)
{
throw new Exception("You must pass in INPUT, filePath, or audioUrl parameter.");
}

if (Uri.TryCreate(input, UriKind.Absolute, out var inputUrl))
{
if (inputUrl.IsFile)
{
filePath = inputUrl.LocalPath;
return true;
}
else
{
audioUrl = input;
}
}
else
{
filePath = input;

filePath = null;
return false;
}

filePath = input;
return true;
}

private static async Task<string> UploadFileAsync(string path, HttpClient httpClient)
Expand Down Expand Up @@ -128,7 +105,7 @@ private static async Task<Transcript> CreateTranscriptAsync(string audioUrl, Htt
using (var response = await httpClient.PostAsync("https://api.assemblyai.com/v2/transcript", content))
{
response.EnsureSuccessStatusCode();
var transcript = (await response.Content.ReadFromJsonAsync<Transcript>());
var transcript = await response.Content.ReadFromJsonAsync<Transcript>();
if (transcript.Status == "error") throw new Exception(transcript.Error);
return transcript;
}
Expand Down
22 changes: 8 additions & 14 deletions src/Sample/FindFilePlugin.cs
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
using System.ComponentModel;
using System.Text.RegularExpressions;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Orchestration;
using Microsoft.SemanticKernel.SkillDefinition;

namespace AssemblyAI.SemanticKernel.Sample;

public class FindFilePlugin
{
public const string PluginName = "FindFilePlugin";
public const string PluginName = nameof(FindFilePlugin);
private readonly IKernel _kernel;


public FindFilePlugin(IKernel kernel)
{
_kernel = kernel;
Expand All @@ -22,27 +21,22 @@ public FindFilePlugin(IKernel kernel)
$"on operating platform {Environment.OSVersion.Platform.ToString()} " +
$"with user profile path '{Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)}' is ";
var context = await _kernel.InvokeSemanticFunctionAsync(
promptTemplate: prompt,
temperature: 0
template: prompt
);
var matches = Regex.Matches(
context.Result,
context.GetValue<string>()!,
@"([a-zA-Z]?\:?[\/\\][\S-[""'\. ]]*[\/\\][\S-[""'\. ]]*)",
RegexOptions.IgnoreCase
);
return matches.LastOrDefault()?.Value ?? null;
}


public const string LocateFileFunctionName = nameof(LocateFile);

[SKFunction, Description("Find files in common folders.")]
[SKParameter("fileName", "The name of the file")]
[SKParameter("commonFolderName", "The name of the common folder")]
public async Task<string> LocateFile(SKContext context)
public async Task<string> LocateFile(
[Description("The name of the file")] string fileName,
[Description("The name of the common folder")]
string? commonFolderName)
{
var fileName = context.Variables["fileName"];
var commonFolderName = context.Variables["commonFolderName"];
var commonFolderPath = commonFolderName?.ToLower() switch
{
null => Environment.CurrentDirectory,
Expand Down
36 changes: 15 additions & 21 deletions src/Sample/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.Orchestration;
using Microsoft.SemanticKernel.Planning;
using Microsoft.SemanticKernel.Planners;

namespace AssemblyAI.SemanticKernel.Sample;

Expand All @@ -16,8 +15,8 @@ public static async Task Main(string[] args)
var kernel = BuildKernel(config);

await TranscribeFileUsingPluginDirectly(kernel);
//await TranscribeFileUsingPluginFromSemanticFunction(kernel);
//await TranscribeFileUsingPlan(kernel);
await TranscribeFileUsingPluginFromSemanticFunction(kernel);
await TranscribeFileUsingPlan(kernel);
}

private static IKernel BuildKernel(IConfiguration config)
Expand All @@ -33,16 +32,16 @@ private static IKernel BuildKernel(IConfiguration config)

var apiKey = config["AssemblyAI:ApiKey"] ?? throw new Exception("AssemblyAI:ApiKey configuration is required.");

kernel.ImportSkill(
kernel.ImportFunctions(
new TranscriptPlugin(apiKey: apiKey)
{
AllowFileSystemAccess = true
},
TranscriptPlugin.PluginName
);

kernel.ImportSkill(
new FindFilePlugin(kernel: kernel),
kernel.ImportFunctions(
new FindFilePlugin(kernel),
FindFilePlugin.PluginName
);
return kernel;
Expand All @@ -61,17 +60,13 @@ private static IConfigurationRoot BuildConfig(string[] args)
private static async Task TranscribeFileUsingPluginDirectly(IKernel kernel)
{
Console.WriteLine("Transcribing file using plugin directly");
var variables = new ContextVariables
{
["audioUrl"] = "https://storage.googleapis.com/aai-docs-samples/espn.m4a",
// ["filePath"] = "./espn.m4a" // you can also use `filePath` which will upload the file and override `audioUrl`
};

var result = await kernel.Skills
var context = kernel.CreateNewContext();
context.Variables["INPUT"] = "https://storage.googleapis.com/aai-docs-samples/espn.m4a";
var result = await kernel.Functions
.GetFunction(TranscriptPlugin.PluginName, TranscriptPlugin.TranscribeFunctionName)
.InvokeAsync(variables);
.InvokeAsync(context);

Console.WriteLine(result.Result);
Console.WriteLine(result.GetValue<string>());
Console.WriteLine();
}

Expand All @@ -88,23 +83,22 @@ private static async Task TranscribeFileUsingPluginFromSemanticFunction(IKernel
""";
var context = kernel.CreateNewContext();
var function = kernel.CreateSemanticFunction(prompt);
await function.InvokeAsync(context);
Console.WriteLine(context.Result);
var result = await function.InvokeAsync(context);
Console.WriteLine(result.GetValue<string>());
Console.WriteLine();
}

private static async Task TranscribeFileUsingPlan(IKernel kernel)
{
Console.WriteLine("Transcribing file from a plan");
var planner = new SequentialPlanner(kernel);

const string prompt = "Transcribe the espn.m4a in my downloads folder.";
const string prompt = "Find the espn.m4a in my downloads folder and transcribe it.";
var plan = await planner.CreatePlanAsync(prompt);

Console.WriteLine("Plan:\n");
Console.WriteLine(JsonSerializer.Serialize(plan, new JsonSerializerOptions { WriteIndented = true }));

var transcript = (await kernel.RunAsync(plan)).Result;
var transcript = (await kernel.RunAsync(plan)).GetValue<string>();
Console.WriteLine(transcript);
Console.WriteLine();
}
Expand Down
2 changes: 1 addition & 1 deletion src/Sample/Sample.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
<Version>7.0.0</Version>
</PackageReference>
<PackageReference Include="Microsoft.SemanticKernel">
<Version>0.24.230918.1-preview</Version>
<Version>1.0.0-beta3</Version>
</PackageReference>
</ItemGroup>
</Project>

0 comments on commit 6a8a5c0

Please sign in to comment.