Skip to content

Commit 9a833e7

Browse files
aaronpowellaaronpowell
authored
Merge pull request #50 from tonybaloney/underscore_literals
Underscore literals
2 parents 8400ea7 + 2fc3ad9 commit 9a833e7

File tree

12 files changed

+76
-58
lines changed

12 files changed

+76
-58
lines changed

‎Integration.Tests/BasicTest.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ public void TestBasic()
2424
}
2525

2626
[Fact]
27-
public void TestFalseReturnTypes() {
27+
public void TestFalseReturnTypes()
28+
{
2829
var falseReturns = testEnv.Env.TestFalseReturns();
2930

3031
// TODO: Standardise the exception that gets raised when the response type is invalid.

‎PythonEnvironments/CultureExtensions.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
using System.Globalization;
2-
using Python.Runtime;
1+
using Python.Runtime;
2+
using System.Globalization;
33

44
namespace PythonEnvironments
55
{

‎PythonSourceGenerator.Tests/TestEnvironment.cs‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,4 @@
1-

2-
using PythonSourceGenerator;
3-
4-
namespace PythonSourceGenerator.Tests
1+
namespace PythonSourceGenerator.Tests
52
{
63
public class TestEnvironment : IDisposable
74
{

‎PythonSourceGenerator.Tests/TokenizerTests.cs‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -432,18 +432,53 @@ def bar(a: int, b:= str) -> None:
432432
[InlineData("255", 255)]
433433
[InlineData("0b10", 0b10)]
434434
[InlineData("0b10101101", 0b10101101)]
435+
[InlineData("0b1010_1101", 0b10101101)]
435436
// [InlineData("0o123", 0o123)] Octal literals are not supported in C#
436437
[InlineData("1234567", 1234567)]
437438
[InlineData("-1234567", -1234567)]
438439
[InlineData("0xdeadbeef", 0xdeadbeef)]
440+
[InlineData("0xdead_beef", 0xdeadbeef)]
441+
// See https://github.com/python/cpython/blob/main/Lib/test/test_grammar.py#L25
442+
[InlineData("1_000_000", 1_000_000)]
443+
[InlineData("4_2", 42)]
444+
[InlineData("0_0", 0)]
439445
public void TestIntegerTokenization(string code, long expectedValue)
440446
{
441447
var tokens = PythonSignatureTokenizer.Instance.Tokenize(code);
448+
Assert.Single(tokens);
442449
var result = PythonSignatureParser.ConstantValueTokenizer.TryParse(tokens);
443450

444451
Assert.True(result.HasValue);
445452
Assert.NotNull(result.Value);
446453
Assert.True(result.Value?.IsInteger);
447454
Assert.Equal(expectedValue, result.Value?.IntegerValue);
448455
}
456+
457+
[Theory]
458+
/* See https://github.com/python/cpython/blob/main/Lib/test/test_tokenize.py#L2063-L2126 */
459+
[InlineData("1.0", 1.0)]
460+
[InlineData("-1.0", -1.0)]
461+
[InlineData("-1_000.0", -1000.0)]
462+
[InlineData("3.14159", 3.14159)]
463+
[InlineData("314159.", 314159.0)]
464+
// [InlineData(".314159", .314159)] // TODO: (track) Support no leading 0
465+
// [InlineData(".1_4", 0.14)]
466+
// [InlineData(".1_4e1", 0.14e1)]
467+
[InlineData("3E123", 3E123)]
468+
[InlineData("3e-1230", 3e-1230)]
469+
[InlineData("3.14e159", 3.14e159)]
470+
[InlineData("1_000_000.4e5", 1_000_000.4e5)]
471+
[InlineData("1e1_0", 1e1_0)]
472+
public void TestDoubleTokenization(string code, double expectedValue)
473+
{
474+
var tokens = PythonSignatureTokenizer.Instance.Tokenize(code);
475+
Assert.Single(tokens);
476+
Assert.Equal(PythonSignatureTokens.PythonSignatureToken.Decimal, tokens.First().Kind);
477+
var result = PythonSignatureParser.DecimalConstantTokenizer.TryParse(tokens);
478+
479+
Assert.True(result.HasValue);
480+
Assert.NotNull(result.Value);
481+
Assert.Equal(expectedValue, result.Value?.FloatValue);
482+
}
449483
}
484+

‎PythonSourceGenerator/CaseHelper.cs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,10 @@
1-
using System.Linq;
2-
3-
namespace PythonSourceGenerator
1+
namespace PythonSourceGenerator
42
{
53
public static class CaseHelper
64
{
75
public static string ToPascalCase(this string snakeCase)
86
{
9-
return string.Join("", snakeCase.Split('_').Select(s => s.Length > 1 ? char.ToUpperInvariant(s[0]) + s.Substring(1): "_"));
7+
return string.Join("", snakeCase.Split('_').Select(s => s.Length > 1 ? char.ToUpperInvariant(s[0]) + s.Substring(1) : "_"));
108
}
119

1210
public static string ToLowerPascalCase(this string snakeCase)

‎PythonSourceGenerator/Keywords.cs‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
using System.Linq;
2-
3-
namespace PythonSourceGenerator;
1+
namespace PythonSourceGenerator;
42
internal static class Keywords
53
{
64
static readonly string[] cSharpKeywords = [
@@ -20,13 +18,13 @@ internal static class Keywords
2018
"decimal",
2119
"default",
2220
"delegate",
23-
"do",
21+
"do",
2422
"double",
2523
"else",
2624
"enum",
2725
"event",
2826
"explicit",
29-
"extern",
27+
"extern",
3028
"false",
3129
"finally",
3230
"fixed",

‎PythonSourceGenerator/Parser/PythonSignatureParser.Constants.cs‎

Lines changed: 19 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -2,30 +2,34 @@
22
using Superpower;
33
using Superpower.Model;
44
using Superpower.Parsers;
5+
using System.Globalization;
56

67
namespace PythonSourceGenerator.Parser;
78
public static partial class PythonSignatureParser
89
{
10+
public static TextParser<char> UnderScoreOrDigit { get; } =
11+
Character.Matching(char.IsDigit, "digit").Or(Character.EqualTo('_'));
12+
913
public static TextParser<Unit> IntegerConstantToken { get; } =
1014
from sign in Character.EqualTo('-').OptionalOrDefault()
11-
from digits in Character.Digit.AtLeastOnce()
15+
from firstdigit in Character.Digit
16+
from digits in UnderScoreOrDigit.Many().OptionalOrDefault([])
1217
select Unit.Value;
1318

1419
public static TextParser<Unit> DecimalConstantToken { get; } =
15-
from sign in Character.EqualTo('-').OptionalOrDefault()
16-
from digits in Character.Digit.Many().OptionalOrDefault(['0'])
17-
from decimal_ in Character.EqualTo('.')
18-
from rest in Character.Digit.Many()
19-
select Unit.Value;
20+
from sign in Character.EqualTo('-').OptionalOrDefault()
21+
from first in Character.Digit
22+
from rest in UnderScoreOrDigit.Or(Character.In('.', 'e', 'E', '+', '-')).IgnoreMany()
23+
select Unit.Value;
2024

2125
public static TextParser<Unit> HexidecimalConstantToken { get; } =
2226
from prefix in Span.EqualTo("0x")
23-
from digits in Character.HexDigit.AtLeastOnce()
27+
from digits in Character.EqualTo('_').Or(Character.HexDigit).AtLeastOnce()
2428
select Unit.Value;
2529

2630
public static TextParser<Unit> BinaryConstantToken { get; } =
2731
from prefix in Span.EqualTo("0b")
28-
from digits in Character.EqualTo('0').Or(Character.EqualTo('1')).AtLeastOnce()
32+
from digits in Character.In('0', '1', '_').AtLeastOnce()
2933
select Unit.Value;
3034

3135
public static TextParser<Unit> DoubleQuotedStringConstantToken { get; } =
@@ -54,31 +58,28 @@ from close in Character.EqualTo('\'')
5458

5559
public static TokenListParser<PythonSignatureTokens.PythonSignatureToken, PythonConstant> DecimalConstantTokenizer { get; } =
5660
Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.Decimal)
57-
.Apply(Numerics.DecimalDouble)
58-
.Select(d => new PythonConstant(d))
61+
.Select(token => new PythonConstant(double.Parse(token.ToStringValue().Replace("_", ""), NumberStyles.Float, CultureInfo.InvariantCulture)))
5962
.Named("Decimal Constant");
6063

6164
public static TokenListParser<PythonSignatureTokens.PythonSignatureToken, PythonConstant> IntegerConstantTokenizer { get; } =
6265
Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.Integer)
63-
.Apply(Numerics.IntegerInt64)
64-
.Select(d => new PythonConstant(d))
66+
.Select(d => new PythonConstant(long.Parse(d.ToStringValue().Replace("_", ""), NumberStyles.Integer)))
6567
.Named("Integer Constant");
6668

6769
public static TokenListParser<PythonSignatureTokens.PythonSignatureToken, PythonConstant> HexidecimalIntegerConstantTokenizer { get; } =
6870
Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.HexidecimalInteger)
69-
.Apply(ConstantParsers.HexidecimalConstantParser)
70-
.Select(d => new PythonConstant { Type = PythonConstant.ConstantType.HexidecimalInteger, IntegerValue = (long)d })
71+
.Select(d => new PythonConstant { Type = PythonConstant.ConstantType.HexidecimalInteger, IntegerValue = long.Parse(d.ToStringValue().Substring(2).Replace("_", ""), NumberStyles.HexNumber) })
7172
.Named("Hexidecimal Integer Constant");
72-
73+
7374
public static TokenListParser<PythonSignatureTokens.PythonSignatureToken, PythonConstant> BinaryIntegerConstantTokenizer { get; } =
7475
Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.BinaryInteger)
75-
.Apply(ConstantParsers.BinaryConstantParser)
76-
.Select(d => new PythonConstant { Type = PythonConstant.ConstantType.BinaryInteger, IntegerValue = (long)d })
76+
// TODO: Consider Binary Format specifier introduced in .NET 8 https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings#binary-format-specifier-b
77+
.Select(d => new PythonConstant { Type = PythonConstant.ConstantType.BinaryInteger, IntegerValue = (long)Convert.ToUInt64(d.ToStringValue().Substring(2).Replace("_", ""), 2) })
7778
.Named("Binary Integer Constant");
7879

7980
public static TokenListParser<PythonSignatureTokens.PythonSignatureToken, PythonConstant> BoolConstantTokenizer { get; } =
8081
Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.True).Or(Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.False))
81-
.Select(d => new PythonConstant(d.Kind == PythonSignatureTokens.PythonSignatureToken.True ))
82+
.Select(d => new PythonConstant(d.Kind == PythonSignatureTokens.PythonSignatureToken.True))
8283
.Named("Bool Constant");
8384

8485
public static TokenListParser<PythonSignatureTokens.PythonSignatureToken, PythonConstant> NoneConstantTokenizer { get; } =
@@ -123,15 +124,5 @@ from chars in Character.ExceptIn('\'', '\\')
123124
.Many()
124125
from close in Character.EqualTo('\'')
125126
select new string(chars);
126-
127-
public static TextParser<UInt64> HexidecimalConstantParser { get; } =
128-
from prefix in Span.EqualTo("0x")
129-
from digits in Character.HexDigit.AtLeastOnce()
130-
select UInt64.Parse(new string(digits), System.Globalization.NumberStyles.HexNumber);
131-
132-
public static TextParser<UInt64> BinaryConstantParser { get; } =
133-
from prefix in Span.EqualTo("0b")
134-
from digits in Character.EqualTo('0').Or(Character.EqualTo('1')).AtLeastOnce()
135-
select Convert.ToUInt64(new string(digits), 2);
136127
}
137128
}

‎PythonSourceGenerator/Parser/PythonSignatureParser.Function.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ public static bool TryParseFunctionDefinitions(SourceText source, out PythonFunc
3131
// Go line by line
3232
TextLineCollection lines = source.Lines;
3333
List<GeneratorError> currentErrors = [];
34-
List<(IEnumerable<TextLine> lines, TokenList<PythonSignatureTokens.PythonSignatureToken> tokens )> functionLines = [];
34+
List<(IEnumerable<TextLine> lines, TokenList<PythonSignatureTokens.PythonSignatureToken> tokens)> functionLines = [];
3535
List<(TextLine line, TokenList<PythonSignatureTokens.PythonSignatureToken> tokens)> currentBuffer = [];
3636
bool unfinishedFunctionSpec = false;
3737
foreach (TextLine line in lines)
@@ -81,7 +81,7 @@ public static bool TryParseFunctionDefinitions(SourceText source, out PythonFunc
8181
else
8282
{
8383
// Error parsing the function definition
84-
currentErrors.Add(new GeneratorError(currentLines.First().LineNumber, currentLines.First().LineNumber + functionDefinition.ErrorPosition.Line -1, functionDefinition.ErrorPosition.Column, functionDefinition.ErrorPosition.Column + 1, functionDefinition.FormatErrorMessageFragment()));
84+
currentErrors.Add(new GeneratorError(currentLines.First().LineNumber, currentLines.First().LineNumber + functionDefinition.ErrorPosition.Line - 1, functionDefinition.ErrorPosition.Column, functionDefinition.ErrorPosition.Column + 1, functionDefinition.FormatErrorMessageFragment()));
8585
}
8686
}
8787

‎PythonSourceGenerator/Parser/PythonSignatureParser.Parameters.cs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,12 @@ public static partial class PythonSignatureParser
77
{
88
public static TokenListParser<PythonSignatureTokens.PythonSignatureToken, PythonFunctionParameter> PythonParameterTokenizer { get; } =
99
(from arg in PythonArgTokenizer
10-
from colon in Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.Colon).Optional()
11-
from type in PythonTypeDefinitionTokenizer.OptionalOrDefault()
12-
from defaultValue in Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.Equal).Optional().Then(
13-
_ => ConstantValueTokenizer.OptionalOrDefault()
14-
)
15-
select new PythonFunctionParameter(arg.Name, type, defaultValue, arg.ParameterType))
10+
from colon in Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.Colon).Optional()
11+
from type in PythonTypeDefinitionTokenizer.OptionalOrDefault()
12+
from defaultValue in Token.EqualTo(PythonSignatureTokens.PythonSignatureToken.Equal).Optional().Then(
13+
_ => ConstantValueTokenizer.OptionalOrDefault()
14+
)
15+
select new PythonFunctionParameter(arg.Name, type, defaultValue, arg.ParameterType))
1616
.Named("Parameter");
1717

1818
public static TokenListParser<PythonSignatureTokens.PythonSignatureToken, PythonFunctionParameter[]> PythonParameterListTokenizer { get; } =

‎PythonSourceGenerator/PythonStaticGenerator.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,8 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
4545
sourceContext.ReportDiagnostic(Diagnostic.Create(new DiagnosticDescriptor("PSG004", "PythonStaticGenerator", error.Message, "PythonStaticGenerator", DiagnosticSeverity.Error, true), errorLocation));
4646
}
4747

48-
if (result) {
48+
if (result)
49+
{
4950
methods = ModuleReflection.MethodsFromFunctionDefinitions(functions, fileName);
5051
string source = FormatClassFromMethods(@namespace, pascalFileName, methods);
5152
sourceContext.AddSource($"{pascalFileName}.py.cs", source);

0 commit comments

Comments
 (0)