Map parser and tokenizer errors to the source files.

tonybaloney · Jul 19, 2024 · acefb1d · acefb1d
1 parent 656ee61
commit acefb1d
Show file tree

Hide file tree

Showing 5 changed files with 74 additions and 20 deletions.
diff --git a/PythonSourceGenerator.Tests/IntegrationTests.cs b/PythonSourceGenerator.Tests/IntegrationTests.cs
@@ -22,7 +22,7 @@ private bool Compile(string code, string assemblyName)
 
         // create a Python scope
         PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors);
-
+        Assert.Empty(errors);
         var module = ModuleReflection.MethodsFromFunctionDefinitions(functions, "test");
         var csharp = module.Select(m => m.Syntax).Compile();
 

diff --git a/PythonSourceGenerator.Tests/TokenizerTests.cs b/PythonSourceGenerator.Tests/TokenizerTests.cs
@@ -311,7 +311,7 @@ def baz(c: float, d: bool) -> None:
   xyz  = 1
         """;
         _ = PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors);
-
+        Assert.Empty(errors);
         Assert.NotNull(functions);
         Assert.Equal(2, functions.Length);
         Assert.Equal("bar", functions[0].Name);
@@ -345,7 +345,7 @@ def bar(a: int,
   xyz  = 1
         ";
         _ = PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors);
-
+        Assert.Empty(errors);
         Assert.NotNull(functions);
         Assert.Single(functions);
         Assert.Equal("bar", functions[0].Name);
@@ -363,7 +363,7 @@ public void ParseFunctionTrailingSpaceAfterColon()
         b: str) -> None:   
     pass"; // There is a trailing space after None:
         _ = PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors);
-
+        Assert.Empty(errors);
         Assert.NotNull(functions);
         Assert.Single(functions);
         Assert.Equal("bar", functions[0].Name);
@@ -376,7 +376,7 @@ public void ParseFunctionNoBlankLineAtEnd()
         b: str) -> None:
     pass";
         _ = PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors);
-
+        Assert.Empty(errors);
         Assert.NotNull(functions);
         Assert.Single(functions);
         Assert.Equal("bar", functions[0].Name);
@@ -386,4 +386,19 @@ public void ParseFunctionNoBlankLineAtEnd()
         Assert.Equal("str", functions[0].Parameters[1].Type.Name);
         Assert.Equal("None", functions[0].ReturnType.Name);
     }
+
+    [Fact]
+    public void VerifyErrors()
+    {
+        var code = @"
+
+
+
+def bar(a: int, b:= str) -> None:
+    pass";
+        _ = PythonSignatureParser.TryParseFunctionDefinitions(code, out var functions, out var errors);
+        Assert.NotEmpty(errors);
+        Assert.Equal(4, errors[0].StartLine);
+        Assert.Equal(4, errors[0].EndLine);
+    }
 }
diff --git a/PythonSourceGenerator/GeneratorError.cs b/PythonSourceGenerator/GeneratorError.cs
@@ -0,0 +1,27 @@
+namespace PythonSourceGenerator;
+
+public class GeneratorError
+{
+
+    public string Message { get; }
+
+    public int StartLine { get; }
+
+    public int StartColumn { get; }
+
+    public int EndLine { get; }
+
+    public int EndColumn { get; }
+
+    public string Code { get; }
+
+    public GeneratorError(int startLine, int endLine, int startColumn, int endColumn, string message)
+    {
+        Message = message;
+        StartLine = startLine;
+        StartColumn = startColumn;
+        EndLine = endLine;
+        EndColumn = endColumn;
+        Code = "hello";
+    }
+}
diff --git a/PythonSourceGenerator/Parser/PythonSignatureParser.cs b/PythonSourceGenerator/Parser/PythonSignatureParser.cs
@@ -171,27 +171,36 @@ from colon in Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.Colon)
         select new PythonFunctionDefinition { Name = name.ToStringValue(), Parameters = parameters, ReturnType = arrow })
         .Named("Function Definition");
 
-    public static bool TryParseFunctionDefinitions(string source, out PythonFunctionDefinition[]? pythonSignatures, out string[] errors)
+    public static bool TryParseFunctionDefinitions(string source, out PythonFunctionDefinition[]? pythonSignatures, out GeneratorError[] errors)
     {
         List<PythonFunctionDefinition> functionDefinitions = [];
 
         // Go line by line
         var lines = source.Split(["\r\n", "\n"], StringSplitOptions.None);
-        var currentErrors = new List<string>();
-        List<string> functionLines = [];
+        var currentErrors = new List<GeneratorError>();
+        List<(int startLine, int endLine, string code)> functionLines = [];
         List<string> currentBuffer = [];
+        int currentBufferStartLine = -1;
         bool unfinishedFunctionSpec = false;
-        foreach (var line in lines)
+        for (int i = 0; i < lines.Length; i++)
         {
-            if (IsFunctionSignature(line) || unfinishedFunctionSpec)
+            if (IsFunctionSignature(lines[i]) || unfinishedFunctionSpec)
             {
-                currentBuffer.Add(line);
+                currentBuffer.Add(lines[i]);
+                if (currentBufferStartLine == -1)
+                {
+                    currentBufferStartLine = i;
+                }
                 // Parse the function signature
-                var result = PythonSignatureTokenizer.Instance.TryTokenize(line);
+                var result = PythonSignatureTokenizer.Instance.TryTokenize(lines[i]);
                 if (!result.HasValue)
                 {
-                    currentErrors.Add(result.ToString());
+                    // TODO: Work out end column and add to the other places in this function where it's raised
+                    currentErrors.Add(new GeneratorError(i, i, result.ErrorPosition.Column, result.ErrorPosition.Column, result.FormatErrorMessageFragment()));
+
+                    // Reset buffer
                     currentBuffer = [];
+                    currentBufferStartLine = -1;
                     unfinishedFunctionSpec = false;
                     continue;
                 }
@@ -200,8 +209,9 @@ public static bool TryParseFunctionDefinitions(string source, out PythonFunction
                 if (result.Value.Last().Kind == PythonSignatureTokens.PythonSignatureToken.Colon)
                 {
                     // TODO: (track) Is an empty string the right joining character?
-                    functionLines.Add(string.Join("", currentBuffer));
+                    functionLines.Add((currentBufferStartLine, i, string.Join("", currentBuffer)));
                     currentBuffer = [];
+                    currentBufferStartLine = -1;
                     unfinishedFunctionSpec = false;
                     continue;
                 } else
@@ -213,10 +223,10 @@ public static bool TryParseFunctionDefinitions(string source, out PythonFunction
         foreach (var line in functionLines)
         {
             // TODO: (track) This means we end up tokenizing the lines twice (one individually and again merged). Optimize.
-            var result = PythonSignatureTokenizer.Instance.TryTokenize(line);
+            var result = PythonSignatureTokenizer.Instance.TryTokenize(line.code);
             if (!result.HasValue)
             {
-                currentErrors.Add(result.ToString());
+                currentErrors.Add(new GeneratorError(line.startLine, line.endLine, result.ErrorPosition.Column, result.ErrorPosition.Column, result.FormatErrorMessageFragment()));
                 continue;
             }
             var functionDefinition = PythonFunctionDefinitionTokenizer.TryParse(result.Value);
@@ -227,7 +237,7 @@ public static bool TryParseFunctionDefinitions(string source, out PythonFunction
             else
             {
                 // Error parsing the function definition
-                currentErrors.Add(functionDefinition.ToString());
+                currentErrors.Add(new GeneratorError(line.startLine, line.endLine, functionDefinition.ErrorPosition.Column, functionDefinition.ErrorPosition.Column + 1, functionDefinition.FormatErrorMessageFragment()));
             }
         }
 

diff --git a/PythonSourceGenerator/PythonStaticGenerator.cs b/PythonSourceGenerator/PythonStaticGenerator.cs
@@ -1,5 +1,6 @@
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp.Syntax;
+using Microsoft.CodeAnalysis.Text;
 using PythonSourceGenerator.Parser;
 using PythonSourceGenerator.Reflection;
 using System;
@@ -24,7 +25,7 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
             foreach (var file in inputFiles)
             {
                 // Add environment path
-                var @namespace = "Python.Generated"; // TODO : Infer from project
+                var @namespace = "Python.Generated"; // TODO: (track) Infer namespace from project
 
                 var fileName = Path.GetFileNameWithoutExtension(file.Path);
 
@@ -40,8 +41,9 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
 
                 foreach (var error in errors)
                 {
-                    // TODO: (track) Match source/target
-                    sourceContext.ReportDiagnostic(Diagnostic.Create(new DiagnosticDescriptor("PSG004", "PythonStaticGenerator", $"{file.Path} : {error}", "PythonStaticGenerator", DiagnosticSeverity.Error, true), Location.None));
+                    // Update text span
+                    Location errorLocation = Location.Create(file.Path, TextSpan.FromBounds(0, 1), new LinePositionSpan(new LinePosition(error.StartLine, error.StartColumn), new LinePosition(error.EndLine, error.EndColumn)));
+                    sourceContext.ReportDiagnostic(Diagnostic.Create(new DiagnosticDescriptor("PSG004", "PythonStaticGenerator", error.Message, "PythonStaticGenerator", DiagnosticSeverity.Error, true), errorLocation));
                 }
 
                 if (result) {