Merge pull request #6 from tonybaloney/none_default

Support None and boolean default values, treat None True and False as special tokens
tonybaloney · Jul 8, 2024 · 6ccdd9b · 6ccdd9b
2 parents b15bece + ce51f62
commit 6ccdd9b
Show file tree

Hide file tree

Showing 9 changed files with 95 additions and 10 deletions.
diff --git a/ExamplePythonDependency/ExamplePythonDependency.csproj b/ExamplePythonDependency/ExamplePythonDependency.csproj
@@ -8,7 +8,6 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <None Remove="HelloWorldOld.cs.txt" />
         <None Remove="quick_demo.py" />
         <None Remove="type_demos.py" />
     </ItemGroup>
@@ -23,9 +22,9 @@
         <AdditionalFiles Include="type_demos.py">
           <CopyToOutputDirectory>Always</CopyToOutputDirectory>
         </AdditionalFiles>
-        <!--<AdditionalFiles Include="mistral_demo.py">
+        <AdditionalFiles Include="mistral_demo.py">
             <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-        </AdditionalFiles>-->
+        </AdditionalFiles>
         <AdditionalFiles Include="kmeans_example.py">
             <CopyToOutputDirectory>Always</CopyToOutputDirectory>
         </AdditionalFiles>

diff --git a/ExamplePythonDependency/mistral_demo.py b/ExamplePythonDependency/mistral_demo.py
@@ -6,7 +6,7 @@
 from mistral_common.protocol.instruct.request import ChatCompletionRequest
 
 
-def invoke_mistral_inference(messages: list[str], lang: str = "en-US", temperature=0.0) -> str:
+def invoke_mistral_inference(messages: list[str], lang: str = "en-US", temperature: float=0.0) -> str:
 	tokenizer = MistralTokenizer.from_file(f"{mistral_models_path}/tokenizer.model.v3")
 	model = Transformer.from_folder(mistral_models_path)
 

diff --git a/PythonSourceGenerator.Tests/SignatureTests.cs b/PythonSourceGenerator.Tests/SignatureTests.cs
@@ -25,6 +25,7 @@ public BasicSmokeTest(TestEnvironment testEnv)
         [InlineData("def hello_world(name: str, age: int) -> str:\n    ...\n", "string HelloWorld(string name, long age)")]
         [InlineData("def hello_world(numbers: list[float]) -> list[int]:\n    ...\n", "IEnumerable<long> HelloWorld(IEnumerable<double> numbers)")]
         [InlineData("def hello_world(a: bool, b: str, c: list[tuple[int, float]]) -> bool: \n ...\n", "bool HelloWorld(bool a, string b, IEnumerable<Tuple<long, double>> c)")]
+        [InlineData("def hello_world(a: bool = True, b: str = None) -> bool: \n ...\n", "bool HelloWorld(bool a = true, string b = null)")]
         public void TestGeneratedSignature(string code, string expected)
         {
 

diff --git a/PythonSourceGenerator.Tests/TokenizerTests.cs b/PythonSourceGenerator.Tests/TokenizerTests.cs
@@ -24,7 +24,7 @@ public void Tokenize()
             PythonSignatureTokens.PythonSignatureToken.Identifier,
             PythonSignatureTokens.PythonSignatureToken.CloseParenthesis,
             PythonSignatureTokens.PythonSignatureToken.Arrow,
-            PythonSignatureTokens.PythonSignatureToken.Identifier,
+            PythonSignatureTokens.PythonSignatureToken.None,
             PythonSignatureTokens.PythonSignatureToken.Colon,
         ], tokens.Select(t => t.Kind));
     }
@@ -39,6 +39,9 @@ public void Tokenize()
     [InlineData("abc123", PythonSignatureTokens.PythonSignatureToken.Identifier)]
     [InlineData("'hello'", PythonSignatureTokens.PythonSignatureToken.SingleQuotedString)]
     [InlineData("\"hello\"", PythonSignatureTokens.PythonSignatureToken.DoubleQuotedString)]
+    [InlineData("True", PythonSignatureTokens.PythonSignatureToken.True)]
+    [InlineData("False", PythonSignatureTokens.PythonSignatureToken.False)]
+    [InlineData("None", PythonSignatureTokens.PythonSignatureToken.None)]
     public void AssertTokenKinds(string code, PythonSignatureTokens.PythonSignatureToken expectedToken)
     {
         var tokens = PythonSignatureTokenizer.Instance.Tokenize(code);
@@ -65,7 +68,7 @@ public void TokenizeWithDefaultValue()
             PythonSignatureTokens.PythonSignatureToken.SingleQuotedString,
             PythonSignatureTokens.PythonSignatureToken.CloseParenthesis,
             PythonSignatureTokens.PythonSignatureToken.Arrow,
-            PythonSignatureTokens.PythonSignatureToken.Identifier,
+            PythonSignatureTokens.PythonSignatureToken.None,
             PythonSignatureTokens.PythonSignatureToken.Colon,
         ], tokens.Select(t => t.Kind));
     }
@@ -149,6 +152,61 @@ public void ParseFunctionParameterDefaultDoubleQuotedString()
         Assert.False(result.Value.HasTypeAnnotation());
     }
 
+    [Fact]
+    public void ParseFunctionParameterDefaultDouble()
+    {
+        var tokens = PythonSignatureTokenizer.Instance.Tokenize($"a: float = -1.1");
+        var result = PythonSignatureParser.PythonParameterTokenizer.TryParse(tokens);
+        Assert.True(result.HasValue);
+        Assert.Equal("a", result.Value.Name);
+        Assert.Equal("-1.1", result.Value.DefaultValue?.ToString());
+        Assert.True(result.Value.HasTypeAnnotation());
+    }
+
+    [Fact]
+    public void ParseFunctionParameterDefaultInt()
+    {
+        var tokens = PythonSignatureTokenizer.Instance.Tokenize($"a: int = 1234");
+        var result = PythonSignatureParser.PythonParameterTokenizer.TryParse(tokens);
+        Assert.True(result.HasValue);
+        Assert.Equal("a", result.Value.Name);
+        Assert.Equal("1234", result.Value.DefaultValue?.ToString());
+        Assert.True(result.Value.HasTypeAnnotation());
+    }
+
+    [Fact]
+    public void ParseFunctionParameterDefaultBoolTrue()
+    {
+        var tokens = PythonSignatureTokenizer.Instance.Tokenize($"a: bool = True");
+        var result = PythonSignatureParser.PythonParameterTokenizer.TryParse(tokens);
+        Assert.True(result.HasValue);
+        Assert.Equal("a", result.Value.Name);
+        Assert.Equal("True", result.Value.DefaultValue?.ToString());
+        Assert.True(result.Value.HasTypeAnnotation());
+    }
+
+    [Fact]
+    public void ParseFunctionParameterDefaultBoolFalse()
+    {
+        var tokens = PythonSignatureTokenizer.Instance.Tokenize($"a: bool = False");
+        var result = PythonSignatureParser.PythonParameterTokenizer.TryParse(tokens);
+        Assert.True(result.HasValue);
+        Assert.Equal("a", result.Value.Name);
+        Assert.Equal("False", result.Value.DefaultValue?.ToString());
+        Assert.True(result.Value.HasTypeAnnotation());
+    }
+
+    [Fact]
+    public void ParseFunctionParameterDefaultNone()
+    {
+        var tokens = PythonSignatureTokenizer.Instance.Tokenize($"a: bool = None");
+        var result = PythonSignatureParser.PythonParameterTokenizer.TryParse(tokens);
+        Assert.True(result.HasValue);
+        Assert.Equal("a", result.Value.Name);
+        Assert.Equal("None", result.Value.DefaultValue?.ToString());
+        Assert.True(result.Value.HasTypeAnnotation());
+    }
+
     [Fact]
     public void ParseFunctionParameterListSingleGeneric()
     {

diff --git a/PythonSourceGenerator/Parser/PythonSignatureParser.cs b/PythonSourceGenerator/Parser/PythonSignatureParser.cs
@@ -89,8 +89,8 @@ from exp in Character.EqualToIgnoreCase('e')
     }
 
     public static TokenListParser<PythonSignatureTokens.PythonSignatureToken, PythonTypeSpec> PythonTypeDefinitionTokenizer { get; } =
-        (from name in Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.Identifier)
-        from openBracket in Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.OpenBracket)
+        (from name in Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.Identifier).Or(Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.None))
+         from openBracket in Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.OpenBracket)
             .Then(_ => PythonTypeDefinitionTokenizer.ManyDelimitedBy(
                 Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.Comma),
                 end: Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.CloseBracket)
@@ -123,13 +123,24 @@ from openBracket in Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.Ope
         .Select(d => new PythonConstant { IsInteger = true, IntegerValue = d })
         .Named("Integer Constant");
 
+    public static TokenListParser<PythonSignatureTokens.PythonSignatureToken, PythonConstant> BoolConstantTokenizer { get; } =
+    Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.True).Or(Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.False))
+    .Select(d => new PythonConstant { IsBool = true, BoolValue = d.Kind == PythonSignatureTokens.PythonSignatureToken.True })
+    .Named("Bool Constant");
+
+    public static TokenListParser<PythonSignatureTokens.PythonSignatureToken, PythonConstant> NoneConstantTokenizer { get; } =
+    Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.None)
+    .Select(d => new PythonConstant { IsNone = true })
+    .Named("None Constant");
+
     // Any constant value
     public static TokenListParser<PythonSignatureTokens.PythonSignatureToken, PythonConstant> ConstantValueTokenizer { get; } =
         DecimalConstantTokenizer.AsNullable()
         .Or(IntegerConstantTokenizer.AsNullable())
+        .Or(BoolConstantTokenizer.AsNullable())
+        .Or(NoneConstantTokenizer.AsNullable())
         .Or(DoubleQuotedStringConstantTokenizer.AsNullable())
         .Or(SingleQuotedStringConstantTokenizer.AsNullable())
-        // TODO: Add None token
         .Named("Constant");
 
     public static TokenListParser<PythonSignatureTokens.PythonSignatureToken, PythonFunctionParameter> PythonParameterTokenizer { get; } = 

diff --git a/PythonSourceGenerator/Parser/PythonSignatureTokenizer.cs b/PythonSourceGenerator/Parser/PythonSignatureTokenizer.cs
@@ -22,11 +22,13 @@ public static class PythonSignatureTokenizer
         .Match(Span.EqualTo("def"), PythonSignatureTokens.PythonSignatureToken.Def, requireDelimiters: true)
         .Match(Span.EqualTo("async"), PythonSignatureTokens.PythonSignatureToken.Async, requireDelimiters: true)
         .Match(Span.EqualTo("..."), PythonSignatureTokens.PythonSignatureToken.Ellipsis)
+        .Match(Span.EqualTo("None"), PythonSignatureTokens.PythonSignatureToken.None, requireDelimiters: true)
+        .Match(Span.EqualTo("True"), PythonSignatureTokens.PythonSignatureToken.True, requireDelimiters: true)
+        .Match(Span.EqualTo("False"), PythonSignatureTokens.PythonSignatureToken.False, requireDelimiters: true)
         .Match(Identifier.CStyle, PythonSignatureTokens.PythonSignatureToken.Identifier, requireDelimiters: true) // TODO: Does this require delimiters?
         .Match(PythonSignatureParser.IntegerConstantToken, PythonSignatureTokens.PythonSignatureToken.Integer, requireDelimiters: true)
         .Match(PythonSignatureParser.DecimalConstantToken, PythonSignatureTokens.PythonSignatureToken.Decimal, requireDelimiters: true)
         .Match(PythonSignatureParser.DoubleQuotedStringConstantToken, PythonSignatureTokens.PythonSignatureToken.DoubleQuotedString)
         .Match(PythonSignatureParser.SingleQuotedStringConstantToken, PythonSignatureTokens.PythonSignatureToken.SingleQuotedString)
-        // TODO: Treat None as a special token
         .Build();
 }
diff --git a/PythonSourceGenerator/Parser/PythonSignatureTokens.cs b/PythonSourceGenerator/Parser/PythonSignatureTokens.cs
@@ -50,6 +50,9 @@ public enum PythonSignatureToken
         Decimal,
         DoubleQuotedString,
         SingleQuotedString,
+        True,
+        False,
+        None,
 
         [Token(Example = "...")]
         Ellipsis

diff --git a/PythonSourceGenerator/Parser/Types/PythonConstant.cs b/PythonSourceGenerator/Parser/Types/PythonConstant.cs
@@ -13,6 +13,9 @@ public class PythonConstant
     public bool IsFloat { get; set; }
     public double FloatValue { get; set; }
 
+    public bool IsBool { get; set; }
+    public bool BoolValue { get; set; }
+
     public override string ToString()
     {
         if (IsInteger)
@@ -31,6 +34,10 @@ public override string ToString()
         {
             return "None";
         }
+        if (IsBool)
+        {
+            return BoolValue.ToString();
+        }
         return "unknown";
     }
 }
diff --git a/PythonSourceGenerator/Reflection/ArgumentReflection.cs b/PythonSourceGenerator/Reflection/ArgumentReflection.cs
@@ -31,6 +31,10 @@ public static ParameterSyntax ArgumentSyntax(PythonFunctionParameter parameter)
                 literalExpressionSyntax = SyntaxFactory.LiteralExpression(
                                                             SyntaxKind.NumericLiteralExpression,
                                                             SyntaxFactory.Literal(parameter.DefaultValue.FloatValue));
+            else if (parameter.DefaultValue.IsBool && parameter.DefaultValue.BoolValue == true)
+                literalExpressionSyntax = SyntaxFactory.LiteralExpression(SyntaxKind.TrueLiteralExpression);
+            else if (parameter.DefaultValue.IsBool && parameter.DefaultValue.BoolValue == false)
+                literalExpressionSyntax = SyntaxFactory.LiteralExpression(SyntaxKind.FalseLiteralExpression);
             else if (parameter.DefaultValue.IsNone)
                 literalExpressionSyntax = SyntaxFactory.LiteralExpression(
                                                             SyntaxKind.NullLiteralExpression);