From 51d0ec0ccd28db7f5cfbc84a677151ca6f2eea3b Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 9 Jul 2024 16:13:55 +1000 Subject: [PATCH 1/9] Fix casing edge cases with _ at the end of an argument or doubles --- .../ExamplePythonDependency.csproj | 2 +- ExamplePythonDependency/mistral_demo.py | 18 ------ ExamplePythonDependency/phi3_demo.py | 34 ++++++++++ ExamplePythonDependency/requirements.txt | 7 ++- .../{SignatureTests.cs => BasicSmokeTests.cs} | 0 .../CaseHelperTests.cs | 25 ++++++++ .../IntegrationTests.cs | 63 +++++++++++++++++++ PythonSourceGenerator/CaseHelper.cs | 2 +- QuickConsoleTest/Program.cs | 5 ++ 9 files changed, 135 insertions(+), 21 deletions(-) delete mode 100644 ExamplePythonDependency/mistral_demo.py create mode 100644 ExamplePythonDependency/phi3_demo.py rename PythonSourceGenerator.Tests/{SignatureTests.cs => BasicSmokeTests.cs} (100%) create mode 100644 PythonSourceGenerator.Tests/CaseHelperTests.cs create mode 100644 PythonSourceGenerator.Tests/IntegrationTests.cs diff --git a/ExamplePythonDependency/ExamplePythonDependency.csproj b/ExamplePythonDependency/ExamplePythonDependency.csproj index 3badf904..89a45756 100644 --- a/ExamplePythonDependency/ExamplePythonDependency.csproj +++ b/ExamplePythonDependency/ExamplePythonDependency.csproj @@ -22,7 +22,7 @@ Always - + Always diff --git a/ExamplePythonDependency/mistral_demo.py b/ExamplePythonDependency/mistral_demo.py deleted file mode 100644 index b822923a..00000000 --- a/ExamplePythonDependency/mistral_demo.py +++ /dev/null @@ -1,18 +0,0 @@ -from mistral_inference.model import Transformer -from mistral_inference.generate import generate - -from mistral_common.tokens.tokenizers.mistral import MistralTokenizer -from mistral_common.protocol.instruct.messages import UserMessage -from mistral_common.protocol.instruct.request import ChatCompletionRequest - - -def invoke_mistral_inference(messages: list[str], lang: str = "en-US", temperature: float=0.0) -> str: - tokenizer = MistralTokenizer.from_file(f"{mistral_models_path}/tokenizer.model.v3") - model = Transformer.from_folder(mistral_models_path) - - completion_request = ChatCompletionRequest(messages=[UserMessage(content=message) for message in messages]) - - tokens = tokenizer.encode_chat_completion(completion_request).tokens - - out_tokens, _ = generate([tokens], model, max_tokens=64, temperature=temperature, eos_id=tokenizer.instruct_tokenizer.tokenizer.eos_id) - return tokenizer.instruct_tokenizer.tokenizer.decode(out_tokens[0]) diff --git a/ExamplePythonDependency/phi3_demo.py b/ExamplePythonDependency/phi3_demo.py new file mode 100644 index 00000000..2527b60f --- /dev/null +++ b/ExamplePythonDependency/phi3_demo.py @@ -0,0 +1,34 @@ +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline + +def phi3_inference_demo(user_message: str, system_message: str = "You are a helpful AI assistant.", temperature: float = 0.0) -> str: + torch.random.manual_seed(0) + model = AutoModelForCausalLM.from_pretrained( + "microsoft/Phi-3-mini-4k-instruct", + device_map="cuda", + torch_dtype="auto", + trust_remote_code=True, + ) + + tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-4k-instruct") + + messages = [ + {"role": "system", "content": system_message}, + {"role": "user", "content": user_message}, + ] + + pipe = pipeline( + "text-generation", + model=model, + tokenizer=tokenizer, + ) + + generation_args = { + "max_new_tokens": 500, + "return_full_text": False, + "temperature": temperature, + "do_sample": False, + } + + output = pipe(messages, **generation_args) + return output[0]['generated_text'] \ No newline at end of file diff --git a/ExamplePythonDependency/requirements.txt b/ExamplePythonDependency/requirements.txt index 9edce704..2102c166 100644 --- a/ExamplePythonDependency/requirements.txt +++ b/ExamplePythonDependency/requirements.txt @@ -1,2 +1,7 @@ numpy -scikit-learn \ No newline at end of file +scikit-learn +wheel +flash_attn==2.5.8 +torch==2.3.1 +accelerate==0.31.0 +transformers==4.41.2 \ No newline at end of file diff --git a/PythonSourceGenerator.Tests/SignatureTests.cs b/PythonSourceGenerator.Tests/BasicSmokeTests.cs similarity index 100% rename from PythonSourceGenerator.Tests/SignatureTests.cs rename to PythonSourceGenerator.Tests/BasicSmokeTests.cs diff --git a/PythonSourceGenerator.Tests/CaseHelperTests.cs b/PythonSourceGenerator.Tests/CaseHelperTests.cs new file mode 100644 index 00000000..cff71dbf --- /dev/null +++ b/PythonSourceGenerator.Tests/CaseHelperTests.cs @@ -0,0 +1,25 @@ +namespace PythonSourceGenerator.Tests; + +public class CaseHelperTests +{ + [Fact] + public void VerifyToPascalCase() + { + Assert.Equal("Hello", CaseHelper.ToPascalCase("hello")); + Assert.Equal("HelloWorld", CaseHelper.ToPascalCase("hello_world")); + Assert.Equal("Hello_", CaseHelper.ToPascalCase("hello_")); + Assert.Equal("Hello_World", CaseHelper.ToPascalCase("hello__world")); + Assert.Equal("_Hello_World", CaseHelper.ToPascalCase("_hello__world")); + } + + [Fact] + public void VerifyToLowerPascalCase() + { + Assert.Equal("hello", CaseHelper.ToLowerPascalCase("hello")); + Assert.Equal("helloWorld", CaseHelper.ToLowerPascalCase("hello_world")); + Assert.Equal("hello_", CaseHelper.ToLowerPascalCase("hello_")); + Assert.Equal("hello_World", CaseHelper.ToLowerPascalCase("hello__world")); + // TODO: This instance could arguably be _hello_World although the name is already weird + Assert.Equal("_Hello_World", CaseHelper.ToLowerPascalCase("_hello__world")); + } +} diff --git a/PythonSourceGenerator.Tests/IntegrationTests.cs b/PythonSourceGenerator.Tests/IntegrationTests.cs new file mode 100644 index 00000000..410a4bcd --- /dev/null +++ b/PythonSourceGenerator.Tests/IntegrationTests.cs @@ -0,0 +1,63 @@ +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis; +using Python.Runtime; +using PythonSourceGenerator.Parser; +using PythonSourceGenerator.Reflection; +using System.Reflection; + +namespace PythonSourceGenerator.Tests; + +public class IntegrationTests : IClassFixture +{ + TestEnvironment testEnv; + + public IntegrationTests(TestEnvironment testEnv) + { + this.testEnv = testEnv; + } + + private bool Compile(string code, string assemblyName) + { + var tempName = string.Format("{0}_{1:N}", "test", Guid.NewGuid().ToString("N")); + File.WriteAllText(Path.Combine(testEnv.TempDir, $"{tempName}.py"), code); + + // create a Python scope + PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors); + + var module = ModuleReflection.MethodsFromFunctionDefinitions(functions, "test"); + var csharp = module.Select(m => m.Syntax).Compile(); + + // Check that the sample C# code compiles + string compiledCode = PythonStaticGenerator.FormatClassFromMethods("Python.Generated.Tests", "TestClass", module); + var tree = CSharpSyntaxTree.ParseText(compiledCode); + var compilation = CSharpCompilation.Create(assemblyName, options: new CSharpCompilationOptions(OutputKind.DynamicallyLinkedLibrary)) + .AddReferences(MetadataReference.CreateFromFile(typeof(object).Assembly.Location)) + .AddReferences(MetadataReference.CreateFromFile(typeof(IEnumerable<>).Assembly.Location)) + .AddReferences(MetadataReference.CreateFromFile(typeof(Enumerable).Assembly.Location)) + .AddReferences(MetadataReference.CreateFromFile(typeof(IReadOnlyDictionary<,>).Assembly.Location)) + .AddReferences(MetadataReference.CreateFromFile(typeof(PythonEnvironments.PythonEnvironment).Assembly.Location)) + .AddReferences(MetadataReference.CreateFromFile(typeof(Py).Assembly.Location)) + .AddReferences(MetadataReference.CreateFromFile(AppDomain.CurrentDomain.GetAssemblies().Single(a => a.GetName().Name == "netstandard").Location)) // TODO: Ensure 2.0 + .AddReferences(MetadataReference.CreateFromFile(AppDomain.CurrentDomain.GetAssemblies().Single(a => a.GetName().Name == "System.Runtime").Location)) + .AddReferences(MetadataReference.CreateFromFile(AppDomain.CurrentDomain.GetAssemblies().Single(a => a.GetName().Name == "System.Collections").Location)) + .AddReferences(MetadataReference.CreateFromFile(AppDomain.CurrentDomain.GetAssemblies().Single(a => a.GetName().Name == "System.Linq.Expressions").Location)) + + .AddSyntaxTrees(tree); + var path = testEnv.TempDir + $"/{assemblyName}.dll"; + var result = compilation.Emit(path); + Assert.True(result.Success, compiledCode + "\n" + string.Join("\n", result.Diagnostics)); + // Delete assembly + File.Delete(path); + return result.Success; + } + + [Fact] + public void TestBasicString() + { + var code = """ +def foo(in_: str) -> str: + return in_.upper() +"""; + Assert.True(Compile(code, "stringFoo")); + } +} diff --git a/PythonSourceGenerator/CaseHelper.cs b/PythonSourceGenerator/CaseHelper.cs index 842a7025..e8e0ee85 100644 --- a/PythonSourceGenerator/CaseHelper.cs +++ b/PythonSourceGenerator/CaseHelper.cs @@ -6,7 +6,7 @@ public static class CaseHelper { public static string ToPascalCase(this string snakeCase) { - return string.Join("", snakeCase.Split('_').Select(s => char.ToUpperInvariant(s[0]) + s.Substring(1))); + return string.Join("", snakeCase.Split('_').Select(s => s.Length > 1 ? char.ToUpperInvariant(s[0]) + s.Substring(1): "_")); } public static string ToLowerPascalCase(this string snakeCase) diff --git a/QuickConsoleTest/Program.cs b/QuickConsoleTest/Program.cs index f9774b98..9945bebd 100644 --- a/QuickConsoleTest/Program.cs +++ b/QuickConsoleTest/Program.cs @@ -35,3 +35,8 @@ var centroids = JsonSerializer.Serialize(interiaResult.Item1); var interia = interiaResult.Item2; Console.WriteLine($"KMeans interia for 4 clusters is {centroids}, interia is {interia}"); + + +var phi3demo = env.Phi3Demo(); +// var result = phi3demo.Phi3InferenceDemo("What kind of food is Brie?"); +// Console.WriteLine(result); \ No newline at end of file From b94084fe90de5ab393cad8359811c4023b4775da Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 9 Jul 2024 16:23:08 +1000 Subject: [PATCH 2/9] Make a comment about mangled keywords --- PythonSourceGenerator.Tests/IntegrationTests.cs | 1 - PythonSourceGenerator/Reflection/ArgumentReflection.cs | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/PythonSourceGenerator.Tests/IntegrationTests.cs b/PythonSourceGenerator.Tests/IntegrationTests.cs index 410a4bcd..df74b693 100644 --- a/PythonSourceGenerator.Tests/IntegrationTests.cs +++ b/PythonSourceGenerator.Tests/IntegrationTests.cs @@ -3,7 +3,6 @@ using Python.Runtime; using PythonSourceGenerator.Parser; using PythonSourceGenerator.Reflection; -using System.Reflection; namespace PythonSourceGenerator.Tests; diff --git a/PythonSourceGenerator/Reflection/ArgumentReflection.cs b/PythonSourceGenerator/Reflection/ArgumentReflection.cs index 47813740..46ee64df 100644 --- a/PythonSourceGenerator/Reflection/ArgumentReflection.cs +++ b/PythonSourceGenerator/Reflection/ArgumentReflection.cs @@ -13,6 +13,7 @@ public static ParameterSyntax ArgumentSyntax(PythonFunctionParameter parameter) if (parameter.DefaultValue == null) { + // TODO : Mangle reserved keyword identifiers, e.g. "new" return SyntaxFactory .Parameter(SyntaxFactory.Identifier(parameter.Name.ToLowerPascalCase())) .WithType(reflectedType); @@ -42,6 +43,7 @@ public static ParameterSyntax ArgumentSyntax(PythonFunctionParameter parameter) // TODO : Handle other types? literalExpressionSyntax = SyntaxFactory.LiteralExpression( SyntaxKind.NullLiteralExpression); + // TODO : Mangle reserved keyword identifiers, e.g. "new" return SyntaxFactory .Parameter(SyntaxFactory.Identifier(parameter.Name.ToLowerPascalCase())) .WithType(reflectedType) From 823b4e1ebb8209af6a51c3bf595018f273679fab Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 9 Jul 2024 16:45:24 +1000 Subject: [PATCH 3/9] Handle *args as nullable tupleand **kwargs as nullable dict --- .../BasicSmokeTests.cs | 4 +++- .../Parser/PythonSignatureParser.cs | 7 +++++-- .../Parser/PythonSignatureTokenizer.cs | 2 +- .../Parser/Types/PythonFunctionParameter.cs | 4 ++++ .../Reflection/ArgumentReflection.cs | 21 ++++++++++++++++++- 5 files changed, 33 insertions(+), 5 deletions(-) diff --git a/PythonSourceGenerator.Tests/BasicSmokeTests.cs b/PythonSourceGenerator.Tests/BasicSmokeTests.cs index 002af014..837a5ff1 100644 --- a/PythonSourceGenerator.Tests/BasicSmokeTests.cs +++ b/PythonSourceGenerator.Tests/BasicSmokeTests.cs @@ -26,6 +26,8 @@ public BasicSmokeTest(TestEnvironment testEnv) [InlineData("def hello_world(numbers: list[float]) -> list[int]:\n ...\n", "IEnumerable HelloWorld(IEnumerable numbers)")] [InlineData("def hello_world(a: bool, b: str, c: list[tuple[int, float]]) -> bool: \n ...\n", "bool HelloWorld(bool a, string b, IEnumerable> c)")] [InlineData("def hello_world(a: bool = True, b: str = None) -> bool: \n ...\n", "bool HelloWorld(bool a = true, string b = null)")] + [InlineData("def hello_world(a: str, *args) -> None: \n ...\n", "void HelloWorld(string a, Tuple args = null)")] + [InlineData("def hello_world(a: str, *args, **kwargs) -> None: \n ...\n", "void HelloWorld(string a, Tuple args = null, IReadOnlyDictionary kwargs = null)")] public void TestGeneratedSignature(string code, string expected) { @@ -34,7 +36,7 @@ public void TestGeneratedSignature(string code, string expected) // create a Python scope PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors); - + Assert.Empty(errors); var module = ModuleReflection.MethodsFromFunctionDefinitions(functions, "test"); var csharp = module.Select(m => m.Syntax).Compile(); Assert.Contains(expected, csharp); diff --git a/PythonSourceGenerator/Parser/PythonSignatureParser.cs b/PythonSourceGenerator/Parser/PythonSignatureParser.cs index 8359aa24..131f0679 100644 --- a/PythonSourceGenerator/Parser/PythonSignatureParser.cs +++ b/PythonSourceGenerator/Parser/PythonSignatureParser.cs @@ -144,13 +144,16 @@ from openBracket in Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.Ope .Named("Constant"); public static TokenListParser PythonParameterTokenizer { get; } = - (from name in Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.Identifier) + ( + from star in Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.Asterisk).Optional() + from doubleStar in Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.DoubleAsterisk).Optional() + from name in Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.Identifier) from colon in Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.Colon).Optional() from type in PythonTypeDefinitionTokenizer.OptionalOrDefault() from defaultValue in Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.Equal).Optional().Then( _ => ConstantValueTokenizer.OptionalOrDefault() ) - select new PythonFunctionParameter { Name = name.ToStringValue(), Type = type, DefaultValue = defaultValue }) + select new PythonFunctionParameter { Name = name.ToStringValue(), Type = type, DefaultValue = defaultValue, IsStar = star != null, IsDoubleStar = doubleStar != null }) .Named("Parameter"); public static TokenListParser PythonParameterListTokenizer { get; } = diff --git a/PythonSourceGenerator/Parser/PythonSignatureTokenizer.cs b/PythonSourceGenerator/Parser/PythonSignatureTokenizer.cs index fe2e2be7..c9c565c7 100644 --- a/PythonSourceGenerator/Parser/PythonSignatureTokenizer.cs +++ b/PythonSourceGenerator/Parser/PythonSignatureTokenizer.cs @@ -14,8 +14,8 @@ public static class PythonSignatureTokenizer .Match(Character.EqualTo(']'), PythonSignatureTokens.PythonSignatureToken.CloseBracket) .Match(Character.EqualTo(':'), PythonSignatureTokens.PythonSignatureToken.Colon) .Match(Character.EqualTo(','), PythonSignatureTokens.PythonSignatureToken.Comma) - .Match(Character.EqualTo('*'), PythonSignatureTokens.PythonSignatureToken.Asterisk) .Match(Character.EqualTo('*').IgnoreThen(Character.EqualTo('*')), PythonSignatureTokens.PythonSignatureToken.DoubleAsterisk) + .Match(Character.EqualTo('*'), PythonSignatureTokens.PythonSignatureToken.Asterisk) .Match(Character.EqualTo('-').IgnoreThen(Character.EqualTo('>')), PythonSignatureTokens.PythonSignatureToken.Arrow) .Match(Character.EqualTo('/'), PythonSignatureTokens.PythonSignatureToken.Slash) .Match(Character.EqualTo('='), PythonSignatureTokens.PythonSignatureToken.Equal) diff --git a/PythonSourceGenerator/Parser/Types/PythonFunctionParameter.cs b/PythonSourceGenerator/Parser/Types/PythonFunctionParameter.cs index 06f10abc..4805d91f 100644 --- a/PythonSourceGenerator/Parser/Types/PythonFunctionParameter.cs +++ b/PythonSourceGenerator/Parser/Types/PythonFunctionParameter.cs @@ -16,6 +16,10 @@ public PythonTypeSpec Type { public PythonConstant? DefaultValue { get; set; } + public bool IsStar { get; set; } + + public bool IsDoubleStar { get; set; } + public bool HasTypeAnnotation() { return _type != null; diff --git a/PythonSourceGenerator/Reflection/ArgumentReflection.cs b/PythonSourceGenerator/Reflection/ArgumentReflection.cs index 46ee64df..9fc1b294 100644 --- a/PythonSourceGenerator/Reflection/ArgumentReflection.cs +++ b/PythonSourceGenerator/Reflection/ArgumentReflection.cs @@ -7,9 +7,28 @@ namespace PythonSourceGenerator.Reflection; public class ArgumentReflection { + private static readonly PythonTypeSpec TupleAny = new() { Name = "tuple", Arguments = [new PythonTypeSpec { Name = "Any" }] }; + private static readonly PythonTypeSpec DictStrAny = new() { Name = "dict", Arguments = [new PythonTypeSpec { Name = "str" }, new PythonTypeSpec { Name = "Any" }] }; + public static ParameterSyntax ArgumentSyntax(PythonFunctionParameter parameter) { - var reflectedType = TypeReflection.AsPredefinedType(parameter.Type); + TypeSyntax reflectedType; + // Treat *args as tuple and **kwargs as dict + if (parameter.IsStar) + { + reflectedType = TypeReflection.AsPredefinedType(TupleAny); + parameter.DefaultValue = new PythonConstant { IsNone = true }; + } + else if (parameter.IsDoubleStar) + { + reflectedType = TypeReflection.AsPredefinedType(DictStrAny); + parameter.DefaultValue = new PythonConstant { IsNone = true }; + } + else + { + reflectedType = TypeReflection.AsPredefinedType(parameter.Type); + + } if (parameter.DefaultValue == null) { From 00bbf35c49eda7a8a9b04a3c696ba55b39a20c22 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 18 Jul 2024 14:54:40 +1000 Subject: [PATCH 4/9] Cleanup repo --- AutoUpdateAssemblyName.txt | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 AutoUpdateAssemblyName.txt diff --git a/AutoUpdateAssemblyName.txt b/AutoUpdateAssemblyName.txt deleted file mode 100644 index 128b2817..00000000 --- a/AutoUpdateAssemblyName.txt +++ /dev/null @@ -1,2 +0,0 @@ -ExamplePythonDependency -QuickConsoleTest \ No newline at end of file From 3594eef21f04e1119621e53504b85c42796061e7 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 18 Jul 2024 15:20:19 +1000 Subject: [PATCH 5/9] Test issue #1 --- PythonSourceGenerator.Tests/TokenizerTests.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/PythonSourceGenerator.Tests/TokenizerTests.cs b/PythonSourceGenerator.Tests/TokenizerTests.cs index 4c3ad260..080734f6 100644 --- a/PythonSourceGenerator.Tests/TokenizerTests.cs +++ b/PythonSourceGenerator.Tests/TokenizerTests.cs @@ -355,4 +355,22 @@ def bar(a: int, Assert.Equal("str", functions[0].Parameters[1].Type.Name); Assert.Equal("None", functions[0].ReturnType.Name); } + + [Fact] + public void ParseFunctionNoBlankLineAtEnd() + { + var code = @"""def bar(a: int, + b: str) -> None: + pass"""; + _ = PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors); + + Assert.NotNull(functions); + Assert.Single(functions); + Assert.Equal("bar", functions[0].Name); + Assert.Equal("a", functions[0].Parameters[0].Name); + Assert.Equal("int", functions[0].Parameters[0].Type.Name); + Assert.Equal("b", functions[0].Parameters[1].Name); + Assert.Equal("str", functions[0].Parameters[1].Type.Name); + Assert.Equal("None", functions[0].ReturnType.Name); + } } From 65f4c0e2a7b835345dfed1b5d8d9b7512e6a0e0c Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 19 Jul 2024 10:28:37 +1000 Subject: [PATCH 6/9] Add test cases for function signatures --- PythonSourceGenerator.Tests/TokenizerTests.cs | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/PythonSourceGenerator.Tests/TokenizerTests.cs b/PythonSourceGenerator.Tests/TokenizerTests.cs index 080734f6..a7d78a6b 100644 --- a/PythonSourceGenerator.Tests/TokenizerTests.cs +++ b/PythonSourceGenerator.Tests/TokenizerTests.cs @@ -332,7 +332,7 @@ def baz(c: float, d: bool) -> None: [Fact] public void ParseMultiLineFunctionDefinition() { - var code = @""" + var code = @" import foo def bar(a: int, @@ -343,7 +343,7 @@ def bar(a: int, if __name__ == '__main__': xyz = 1 - """; + "; _ = PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors); Assert.NotNull(functions); @@ -356,12 +356,25 @@ def bar(a: int, Assert.Equal("None", functions[0].ReturnType.Name); } + [Fact] + public void ParseFunctionTrailingSpaceAfterColon() + { + var code = @"def bar(a: int, + b: str) -> None: + pass"; // There is a trailing space after None: + _ = PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors); + + Assert.NotNull(functions); + Assert.Single(functions); + Assert.Equal("bar", functions[0].Name); + } + [Fact] public void ParseFunctionNoBlankLineAtEnd() { - var code = @"""def bar(a: int, + var code = @"def bar(a: int, b: str) -> None: - pass"""; + pass"; _ = PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors); Assert.NotNull(functions); From 656ee61bc64e0c4f8bd4a750aa3a838c4069207d Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 19 Jul 2024 10:34:58 +1000 Subject: [PATCH 7/9] Update all the todo items we want to test --- PythonEnvironments/PythonEnvironment.cs | 6 +++--- PythonSourceGenerator.Tests/BasicSmokeTests.cs | 2 +- PythonSourceGenerator.Tests/CaseHelperTests.cs | 2 +- PythonSourceGenerator.Tests/IntegrationTests.cs | 2 +- PythonSourceGenerator/Parser/PythonSignatureParser.cs | 7 +++---- PythonSourceGenerator/Parser/PythonSignatureTokenizer.cs | 2 +- PythonSourceGenerator/PythonStaticGenerator.cs | 2 +- PythonSourceGenerator/Reflection/TypeReflection.cs | 2 +- 8 files changed, 12 insertions(+), 13 deletions(-) diff --git a/PythonEnvironments/PythonEnvironment.cs b/PythonEnvironments/PythonEnvironment.cs index ed3f1597..fbadf5b8 100644 --- a/PythonEnvironments/PythonEnvironment.cs +++ b/PythonEnvironments/PythonEnvironment.cs @@ -49,8 +49,8 @@ private static string TryLocatePython(string version) var versionPath = MapVersion(version); var windowsStorePath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "Programs", "Python", "Python" + versionPath); var officialInstallerPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles), "Python", MapVersion(version, ".")); - // TODO: Locate from PATH - // TODO: Add standard paths for Linux and MacOS + // TODO: (track) Locate from PATH + // TODO: (track) Add standard paths for Linux and MacOS if (Directory.Exists(windowsStorePath)) { return windowsStorePath; @@ -115,7 +115,7 @@ public PythonEnvironmentInternal(string pythonLocation, string versionPath, stri } else { - // TODO: C extension path for linux/macos + // TODO: (track) C extension path for linux/macos } if (extraPath.Length > 0) diff --git a/PythonSourceGenerator.Tests/BasicSmokeTests.cs b/PythonSourceGenerator.Tests/BasicSmokeTests.cs index 837a5ff1..a582fdf5 100644 --- a/PythonSourceGenerator.Tests/BasicSmokeTests.cs +++ b/PythonSourceGenerator.Tests/BasicSmokeTests.cs @@ -51,7 +51,7 @@ public void TestGeneratedSignature(string code, string expected) .AddReferences(MetadataReference.CreateFromFile(typeof(IReadOnlyDictionary<,>).Assembly.Location)) .AddReferences(MetadataReference.CreateFromFile(typeof(PythonEnvironments.PythonEnvironment).Assembly.Location)) .AddReferences(MetadataReference.CreateFromFile(typeof(Py).Assembly.Location)) - .AddReferences(MetadataReference.CreateFromFile(AppDomain.CurrentDomain.GetAssemblies().Single(a => a.GetName().Name == "netstandard").Location)) // TODO: Ensure 2.0 + .AddReferences(MetadataReference.CreateFromFile(AppDomain.CurrentDomain.GetAssemblies().Single(a => a.GetName().Name == "netstandard").Location)) // TODO: (track) Ensure 2.0 .AddReferences(MetadataReference.CreateFromFile(AppDomain.CurrentDomain.GetAssemblies().Single(a => a.GetName().Name == "System.Runtime").Location)) .AddReferences(MetadataReference.CreateFromFile(AppDomain.CurrentDomain.GetAssemblies().Single(a => a.GetName().Name == "System.Collections").Location)) .AddReferences(MetadataReference.CreateFromFile(AppDomain.CurrentDomain.GetAssemblies().Single(a => a.GetName().Name == "System.Linq.Expressions").Location)) diff --git a/PythonSourceGenerator.Tests/CaseHelperTests.cs b/PythonSourceGenerator.Tests/CaseHelperTests.cs index cff71dbf..0f4af566 100644 --- a/PythonSourceGenerator.Tests/CaseHelperTests.cs +++ b/PythonSourceGenerator.Tests/CaseHelperTests.cs @@ -19,7 +19,7 @@ public void VerifyToLowerPascalCase() Assert.Equal("helloWorld", CaseHelper.ToLowerPascalCase("hello_world")); Assert.Equal("hello_", CaseHelper.ToLowerPascalCase("hello_")); Assert.Equal("hello_World", CaseHelper.ToLowerPascalCase("hello__world")); - // TODO: This instance could arguably be _hello_World although the name is already weird + // TODO: (track) This instance could arguably be _hello_World although the name is already weird Assert.Equal("_Hello_World", CaseHelper.ToLowerPascalCase("_hello__world")); } } diff --git a/PythonSourceGenerator.Tests/IntegrationTests.cs b/PythonSourceGenerator.Tests/IntegrationTests.cs index df74b693..de14b1b9 100644 --- a/PythonSourceGenerator.Tests/IntegrationTests.cs +++ b/PythonSourceGenerator.Tests/IntegrationTests.cs @@ -36,7 +36,7 @@ private bool Compile(string code, string assemblyName) .AddReferences(MetadataReference.CreateFromFile(typeof(IReadOnlyDictionary<,>).Assembly.Location)) .AddReferences(MetadataReference.CreateFromFile(typeof(PythonEnvironments.PythonEnvironment).Assembly.Location)) .AddReferences(MetadataReference.CreateFromFile(typeof(Py).Assembly.Location)) - .AddReferences(MetadataReference.CreateFromFile(AppDomain.CurrentDomain.GetAssemblies().Single(a => a.GetName().Name == "netstandard").Location)) // TODO: Ensure 2.0 + .AddReferences(MetadataReference.CreateFromFile(AppDomain.CurrentDomain.GetAssemblies().Single(a => a.GetName().Name == "netstandard").Location)) // TODO: (track) Ensure 2.0 .AddReferences(MetadataReference.CreateFromFile(AppDomain.CurrentDomain.GetAssemblies().Single(a => a.GetName().Name == "System.Runtime").Location)) .AddReferences(MetadataReference.CreateFromFile(AppDomain.CurrentDomain.GetAssemblies().Single(a => a.GetName().Name == "System.Collections").Location)) .AddReferences(MetadataReference.CreateFromFile(AppDomain.CurrentDomain.GetAssemblies().Single(a => a.GetName().Name == "System.Linq.Expressions").Location)) diff --git a/PythonSourceGenerator/Parser/PythonSignatureParser.cs b/PythonSourceGenerator/Parser/PythonSignatureParser.cs index 131f0679..8d27430c 100644 --- a/PythonSourceGenerator/Parser/PythonSignatureParser.cs +++ b/PythonSourceGenerator/Parser/PythonSignatureParser.cs @@ -41,7 +41,6 @@ from close in Character.EqualTo('\'') static class ConstantParsers { - // TODO: See if we can combine these parsers public static TextParser DoubleQuotedString { get; } = from open in Character.EqualTo('"') from chars in Character.ExceptIn('"', '\\') @@ -71,7 +70,7 @@ from sign in Character.EqualTo('-').Value(-1).OptionalOrDefault(1) from whole in Numerics.Natural.Select(n => int.Parse(n.ToStringValue())) select whole * sign ; - // TODO: This a copy from the JSON spec and probably doesn't reflect Python's other numeric literals like Hex and Real + // TODO: (track) This a copy from the JSON spec and probably doesn't reflect Python's other numeric literals like Hex and Real public static TextParser Decimal { get; } = from sign in Character.EqualTo('-').Value(-1.0).OptionalOrDefault(1.0) from whole in Numerics.Natural.Select(n => double.Parse(n.ToStringValue())) @@ -200,7 +199,7 @@ public static bool TryParseFunctionDefinitions(string source, out PythonFunction // If this is a function definition on one line.. if (result.Value.Last().Kind == PythonSignatureTokens.PythonSignatureToken.Colon) { - // TODO: Is an empty string the right joining character? + // TODO: (track) Is an empty string the right joining character? functionLines.Add(string.Join("", currentBuffer)); currentBuffer = []; unfinishedFunctionSpec = false; @@ -213,7 +212,7 @@ public static bool TryParseFunctionDefinitions(string source, out PythonFunction } foreach (var line in functionLines) { - // TODO: This means we end up tokenizing the lines twice (one individually and again merged). Optimize. + // TODO: (track) This means we end up tokenizing the lines twice (one individually and again merged). Optimize. var result = PythonSignatureTokenizer.Instance.TryTokenize(line); if (!result.HasValue) { diff --git a/PythonSourceGenerator/Parser/PythonSignatureTokenizer.cs b/PythonSourceGenerator/Parser/PythonSignatureTokenizer.cs index c9c565c7..cabb7c78 100644 --- a/PythonSourceGenerator/Parser/PythonSignatureTokenizer.cs +++ b/PythonSourceGenerator/Parser/PythonSignatureTokenizer.cs @@ -25,7 +25,7 @@ public static class PythonSignatureTokenizer .Match(Span.EqualTo("None"), PythonSignatureTokens.PythonSignatureToken.None, requireDelimiters: true) .Match(Span.EqualTo("True"), PythonSignatureTokens.PythonSignatureToken.True, requireDelimiters: true) .Match(Span.EqualTo("False"), PythonSignatureTokens.PythonSignatureToken.False, requireDelimiters: true) - .Match(Identifier.CStyle, PythonSignatureTokens.PythonSignatureToken.Identifier, requireDelimiters: true) // TODO: Does this require delimiters? + .Match(Identifier.CStyle, PythonSignatureTokens.PythonSignatureToken.Identifier, requireDelimiters: true) // TODO: (track) Does this require delimiters? .Match(PythonSignatureParser.IntegerConstantToken, PythonSignatureTokens.PythonSignatureToken.Integer, requireDelimiters: true) .Match(PythonSignatureParser.DecimalConstantToken, PythonSignatureTokens.PythonSignatureToken.Decimal, requireDelimiters: true) .Match(PythonSignatureParser.DoubleQuotedStringConstantToken, PythonSignatureTokens.PythonSignatureToken.DoubleQuotedString) diff --git a/PythonSourceGenerator/PythonStaticGenerator.cs b/PythonSourceGenerator/PythonStaticGenerator.cs index 71feb8c7..7862d8ec 100644 --- a/PythonSourceGenerator/PythonStaticGenerator.cs +++ b/PythonSourceGenerator/PythonStaticGenerator.cs @@ -40,7 +40,7 @@ public void Initialize(IncrementalGeneratorInitializationContext context) foreach (var error in errors) { - // TODO: Match source/target + // TODO: (track) Match source/target sourceContext.ReportDiagnostic(Diagnostic.Create(new DiagnosticDescriptor("PSG004", "PythonStaticGenerator", $"{file.Path} : {error}", "PythonStaticGenerator", DiagnosticSeverity.Error, true), Location.None)); } diff --git a/PythonSourceGenerator/Reflection/TypeReflection.cs b/PythonSourceGenerator/Reflection/TypeReflection.cs index 9afa6ac9..95275ff3 100644 --- a/PythonSourceGenerator/Reflection/TypeReflection.cs +++ b/PythonSourceGenerator/Reflection/TypeReflection.cs @@ -43,7 +43,7 @@ private static TypeSyntax CreateDictionaryType(PythonTypeSpec keyType, PythonTyp private static TypeSyntax CreateTupleType(PythonTypeSpec[] tupleTypes) { var tupleTypeSyntax = new TypeSyntax[tupleTypes.Length]; - if (tupleTypes.Length > 8) // TODO: Implement up to 21 + if (tupleTypes.Length > 8) // TODO: (track) Implement up to 21 { throw new NotSupportedException("Maximum tuple items is 8"); } From acefb1d24a080b325c031b7e70433206f89eebec Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 19 Jul 2024 11:57:17 +1000 Subject: [PATCH 8/9] Map parser and tokenizer errors to the source files. --- .../IntegrationTests.cs | 2 +- PythonSourceGenerator.Tests/TokenizerTests.cs | 23 ++++++++++--- PythonSourceGenerator/GeneratorError.cs | 27 +++++++++++++++ .../Parser/PythonSignatureParser.cs | 34 ++++++++++++------- .../PythonStaticGenerator.cs | 8 +++-- 5 files changed, 74 insertions(+), 20 deletions(-) create mode 100644 PythonSourceGenerator/GeneratorError.cs diff --git a/PythonSourceGenerator.Tests/IntegrationTests.cs b/PythonSourceGenerator.Tests/IntegrationTests.cs index de14b1b9..77f13235 100644 --- a/PythonSourceGenerator.Tests/IntegrationTests.cs +++ b/PythonSourceGenerator.Tests/IntegrationTests.cs @@ -22,7 +22,7 @@ private bool Compile(string code, string assemblyName) // create a Python scope PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors); - + Assert.Empty(errors); var module = ModuleReflection.MethodsFromFunctionDefinitions(functions, "test"); var csharp = module.Select(m => m.Syntax).Compile(); diff --git a/PythonSourceGenerator.Tests/TokenizerTests.cs b/PythonSourceGenerator.Tests/TokenizerTests.cs index a7d78a6b..3a859bad 100644 --- a/PythonSourceGenerator.Tests/TokenizerTests.cs +++ b/PythonSourceGenerator.Tests/TokenizerTests.cs @@ -311,7 +311,7 @@ def baz(c: float, d: bool) -> None: xyz = 1 """; _ = PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors); - + Assert.Empty(errors); Assert.NotNull(functions); Assert.Equal(2, functions.Length); Assert.Equal("bar", functions[0].Name); @@ -345,7 +345,7 @@ def bar(a: int, xyz = 1 "; _ = PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors); - + Assert.Empty(errors); Assert.NotNull(functions); Assert.Single(functions); Assert.Equal("bar", functions[0].Name); @@ -363,7 +363,7 @@ public void ParseFunctionTrailingSpaceAfterColon() b: str) -> None: pass"; // There is a trailing space after None: _ = PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors); - + Assert.Empty(errors); Assert.NotNull(functions); Assert.Single(functions); Assert.Equal("bar", functions[0].Name); @@ -376,7 +376,7 @@ public void ParseFunctionNoBlankLineAtEnd() b: str) -> None: pass"; _ = PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors); - + Assert.Empty(errors); Assert.NotNull(functions); Assert.Single(functions); Assert.Equal("bar", functions[0].Name); @@ -386,4 +386,19 @@ public void ParseFunctionNoBlankLineAtEnd() Assert.Equal("str", functions[0].Parameters[1].Type.Name); Assert.Equal("None", functions[0].ReturnType.Name); } + + [Fact] + public void VerifyErrors() + { + var code = @" + + + +def bar(a: int, b:= str) -> None: + pass"; + _ = PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors); + Assert.NotEmpty(errors); + Assert.Equal(4, errors[0].StartLine); + Assert.Equal(4, errors[0].EndLine); + } } diff --git a/PythonSourceGenerator/GeneratorError.cs b/PythonSourceGenerator/GeneratorError.cs new file mode 100644 index 00000000..f164e4a6 --- /dev/null +++ b/PythonSourceGenerator/GeneratorError.cs @@ -0,0 +1,27 @@ +namespace PythonSourceGenerator; + +public class GeneratorError +{ + + public string Message { get; } + + public int StartLine { get; } + + public int StartColumn { get; } + + public int EndLine { get; } + + public int EndColumn { get; } + + public string Code { get; } + + public GeneratorError(int startLine, int endLine, int startColumn, int endColumn, string message) + { + Message = message; + StartLine = startLine; + StartColumn = startColumn; + EndLine = endLine; + EndColumn = endColumn; + Code = "hello"; + } +} diff --git a/PythonSourceGenerator/Parser/PythonSignatureParser.cs b/PythonSourceGenerator/Parser/PythonSignatureParser.cs index 8d27430c..092587f5 100644 --- a/PythonSourceGenerator/Parser/PythonSignatureParser.cs +++ b/PythonSourceGenerator/Parser/PythonSignatureParser.cs @@ -171,27 +171,36 @@ from colon in Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.Colon) select new PythonFunctionDefinition { Name = name.ToStringValue(), Parameters = parameters, ReturnType = arrow }) .Named("Function Definition"); - public static bool TryParseFunctionDefinitions(string source, out PythonFunctionDefinition[]? pythonSignatures, out string[] errors) + public static bool TryParseFunctionDefinitions(string source, out PythonFunctionDefinition[]? pythonSignatures, out GeneratorError[] errors) { List functionDefinitions = []; // Go line by line var lines = source.Split(["\r\n", "\n"], StringSplitOptions.None); - var currentErrors = new List(); - List functionLines = []; + var currentErrors = new List(); + List<(int startLine, int endLine, string code)> functionLines = []; List currentBuffer = []; + int currentBufferStartLine = -1; bool unfinishedFunctionSpec = false; - foreach (var line in lines) + for (int i = 0; i < lines.Length; i++) { - if (IsFunctionSignature(line) || unfinishedFunctionSpec) + if (IsFunctionSignature(lines[i]) || unfinishedFunctionSpec) { - currentBuffer.Add(line); + currentBuffer.Add(lines[i]); + if (currentBufferStartLine == -1) + { + currentBufferStartLine = i; + } // Parse the function signature - var result = PythonSignatureTokenizer.Instance.TryTokenize(line); + var result = PythonSignatureTokenizer.Instance.TryTokenize(lines[i]); if (!result.HasValue) { - currentErrors.Add(result.ToString()); + // TODO: Work out end column and add to the other places in this function where it's raised + currentErrors.Add(new GeneratorError(i, i, result.ErrorPosition.Column, result.ErrorPosition.Column, result.FormatErrorMessageFragment())); + + // Reset buffer currentBuffer = []; + currentBufferStartLine = -1; unfinishedFunctionSpec = false; continue; } @@ -200,8 +209,9 @@ public static bool TryParseFunctionDefinitions(string source, out PythonFunction if (result.Value.Last().Kind == PythonSignatureTokens.PythonSignatureToken.Colon) { // TODO: (track) Is an empty string the right joining character? - functionLines.Add(string.Join("", currentBuffer)); + functionLines.Add((currentBufferStartLine, i, string.Join("", currentBuffer))); currentBuffer = []; + currentBufferStartLine = -1; unfinishedFunctionSpec = false; continue; } else @@ -213,10 +223,10 @@ public static bool TryParseFunctionDefinitions(string source, out PythonFunction foreach (var line in functionLines) { // TODO: (track) This means we end up tokenizing the lines twice (one individually and again merged). Optimize. - var result = PythonSignatureTokenizer.Instance.TryTokenize(line); + var result = PythonSignatureTokenizer.Instance.TryTokenize(line.code); if (!result.HasValue) { - currentErrors.Add(result.ToString()); + currentErrors.Add(new GeneratorError(line.startLine, line.endLine, result.ErrorPosition.Column, result.ErrorPosition.Column, result.FormatErrorMessageFragment())); continue; } var functionDefinition = PythonFunctionDefinitionTokenizer.TryParse(result.Value); @@ -227,7 +237,7 @@ public static bool TryParseFunctionDefinitions(string source, out PythonFunction else { // Error parsing the function definition - currentErrors.Add(functionDefinition.ToString()); + currentErrors.Add(new GeneratorError(line.startLine, line.endLine, functionDefinition.ErrorPosition.Column, functionDefinition.ErrorPosition.Column + 1, functionDefinition.FormatErrorMessageFragment())); } } diff --git a/PythonSourceGenerator/PythonStaticGenerator.cs b/PythonSourceGenerator/PythonStaticGenerator.cs index 7862d8ec..0b2d046c 100644 --- a/PythonSourceGenerator/PythonStaticGenerator.cs +++ b/PythonSourceGenerator/PythonStaticGenerator.cs @@ -1,5 +1,6 @@ using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.Text; using PythonSourceGenerator.Parser; using PythonSourceGenerator.Reflection; using System; @@ -24,7 +25,7 @@ public void Initialize(IncrementalGeneratorInitializationContext context) foreach (var file in inputFiles) { // Add environment path - var @namespace = "Python.Generated"; // TODO : Infer from project + var @namespace = "Python.Generated"; // TODO: (track) Infer namespace from project var fileName = Path.GetFileNameWithoutExtension(file.Path); @@ -40,8 +41,9 @@ public void Initialize(IncrementalGeneratorInitializationContext context) foreach (var error in errors) { - // TODO: (track) Match source/target - sourceContext.ReportDiagnostic(Diagnostic.Create(new DiagnosticDescriptor("PSG004", "PythonStaticGenerator", $"{file.Path} : {error}", "PythonStaticGenerator", DiagnosticSeverity.Error, true), Location.None)); + // Update text span + Location errorLocation = Location.Create(file.Path, TextSpan.FromBounds(0, 1), new LinePositionSpan(new LinePosition(error.StartLine, error.StartColumn), new LinePosition(error.EndLine, error.EndColumn))); + sourceContext.ReportDiagnostic(Diagnostic.Create(new DiagnosticDescriptor("PSG004", "PythonStaticGenerator", error.Message, "PythonStaticGenerator", DiagnosticSeverity.Error, true), errorLocation)); } if (result) { From bda686397ebef65803aaa95c35ba60519b90d42c Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 19 Jul 2024 12:28:10 +1000 Subject: [PATCH 9/9] Ignore code comments and add a test that they can be tokenized when trailing a function definition --- PythonSourceGenerator.Tests/TokenizerTests.cs | 12 ++++++++++++ .../Parser/PythonSignatureTokenizer.cs | 1 + 2 files changed, 13 insertions(+) diff --git a/PythonSourceGenerator.Tests/TokenizerTests.cs b/PythonSourceGenerator.Tests/TokenizerTests.cs index 3a859bad..559bf275 100644 --- a/PythonSourceGenerator.Tests/TokenizerTests.cs +++ b/PythonSourceGenerator.Tests/TokenizerTests.cs @@ -356,6 +356,18 @@ def bar(a: int, Assert.Equal("None", functions[0].ReturnType.Name); } + [Fact] + public void ParseFunctionWithTrailingComment() + { + var code = @"def bar(a: int, b: str) -> None: # this is a comment + pass"; + _ = PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors); + Assert.Empty(errors); + Assert.NotNull(functions); + Assert.Single(functions); + Assert.Equal("bar", functions[0].Name); + } + [Fact] public void ParseFunctionTrailingSpaceAfterColon() { diff --git a/PythonSourceGenerator/Parser/PythonSignatureTokenizer.cs b/PythonSourceGenerator/Parser/PythonSignatureTokenizer.cs index cabb7c78..10fae00c 100644 --- a/PythonSourceGenerator/Parser/PythonSignatureTokenizer.cs +++ b/PythonSourceGenerator/Parser/PythonSignatureTokenizer.cs @@ -8,6 +8,7 @@ public static class PythonSignatureTokenizer public static Tokenizer Instance { get; } = new TokenizerBuilder() .Ignore(Span.WhiteSpace) + .Ignore(Comment.ShellStyle) .Match(Character.EqualTo('('), PythonSignatureTokens.PythonSignatureToken.OpenParenthesis) .Match(Character.EqualTo(')'), PythonSignatureTokens.PythonSignatureToken.CloseParenthesis) .Match(Character.EqualTo('['), PythonSignatureTokens.PythonSignatureToken.OpenBracket)