Merge pull request #1 from tonybaloney/integration_testing

Integration testing

ExamplePythonDependency/example_model.py

@@ -0,0 +1,23 @@
+import re
+import torch
+from transformers import pipeline
+
+
+def invoke_model(input_string: str) -> str:
+    pipe = pipeline("text-generation", model="AI-MO/NuminaMath-7B-TIR", torch_dtype=torch.bfloat16, device_map="auto")
+
+    messages = [
+        {"role": "user", "content": input_string},
+    ]
+    prompt = pipe.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+
+    gen_config = {
+        "max_new_tokens": 1024,
+        "do_sample": False,
+        "stop_strings": ["```output"], # Generate until Python code block is complete
+        "tokenizer": pipe.tokenizer,
+    }
+
+    outputs = pipe(prompt, **gen_config)
+    text = outputs[0]["generated_text"]
+    return text

Integration.Tests/BasicTest.cs

@@ -0,0 +1,51 @@
+using Python.Generated;
+using Python.Runtime;
+using System.Reflection;
+
+namespace Integration.Tests;
+
+public class BasicTest(TestEnvironment testEnv) : IClassFixture<TestEnvironment>
+{
+    TestEnvironment testEnv = testEnv;
+
+    [Fact]
+    public void TestBasic()
+    {
+
+        var testModule = testEnv.Env.TestBasic();
+
+        Assert.Equal(4.3, testModule.TestIntFloat(4, 0.3));
+        Assert.Equal(4.3, testModule.TestFloatInt(0.3, 4));
+        Assert.Equal(4.3, testModule.TestFloatFloat(0.3, 4.0));
+        Assert.Equal(6, testModule.TestIntInt(4, 2));
+        Assert.Equal([1, 2, 3], testModule.TestListOfInts([1, 2, 3]));
+        Assert.Equal("hello world", testModule.TestTwoStrings("hello ", "world"));
+        Assert.Equal(["hello", "world", "this", "is", "a", "test"], testModule.TestTwoListsOfStrings(["hello", "world"], ["this", "is", "a", "test"]));
+
+
+    }
+
+    [Fact]
+    public void TestFalseReturnTypes() { 
+        var falseReturns = testEnv.Env.TestFalseReturns();
+
+        // TODO: Standardise the exception that gets raised when the response type is invalid.
+        Assert.Throws<InvalidCastException>(() => falseReturns.TestStrActuallyReturnsInt());
+        Assert.Throws<InvalidCastException>(() => falseReturns.TestStrActuallyReturnsFloat());
+        Assert.Throws<InvalidCastException>(() => falseReturns.TestStrActuallyReturnsList());
+        Assert.Throws<InvalidCastException>(() => falseReturns.TestStrActuallyReturnsTuple());
+        Assert.Throws<TargetInvocationException>(() => falseReturns.TestTupleActuallyReturnsInt());
+        Assert.Throws<TargetInvocationException>(() => falseReturns.TestTupleActuallyReturnsFloat());
+        Assert.Throws<TargetInvocationException>(() => falseReturns.TestTupleActuallyReturnsList());
+    }
+
+    [Fact]
+    public void TestDefaults()
+    {
+        var testDefaults = testEnv.Env.TestDefaults();
+
+        Assert.Equal("hello", testDefaults.TestDefaultStrArg());
+        Assert.Equal(1337, testDefaults.TestDefaultIntArg());
+        Assert.Equal(-1, testDefaults.TestDefaultFloatArg());
+    }
+}

Integration.Tests/Integration.Tests.csproj

@@ -0,0 +1,45 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+
+    <IsPackable>false</IsPackable>
+    <IsTestProject>true</IsTestProject>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <None Remove="python\test_basic.py" />
+    <None Remove="python\test_defaults.py" />
+    <None Remove="python\test_false_returns.py" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <AdditionalFiles Include="python\test_basic.py">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </AdditionalFiles>
+    <AdditionalFiles Include="python\test_defaults.py">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </AdditionalFiles>
+    <AdditionalFiles Include="python\test_false_returns.py">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </AdditionalFiles>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="coverlet.collector" Version="6.0.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
+    <PackageReference Include="xunit" Version="2.5.3" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Using Include="Xunit" />
+  </ItemGroup>
+
+	<ItemGroup>
+		<ProjectReference Include="..\PythonEnvironments\PythonEnvironments.csproj" />
+		<ProjectReference Include="..\PythonSourceGenerator\PythonSourceGenerator.csproj" ReferenceOutputAssembly="false" OutputItemType="Analyzer" />
+	</ItemGroup>
+</Project>

Integration.Tests/TestEnvironment.cs

@@ -0,0 +1,24 @@
+using PythonEnvironments;
+
+namespace Integration.Tests;
+public class TestEnvironment : IDisposable
+{
+    IPythonEnvironment env;
+
+    public TestEnvironment()
+    {
+        var userProfile = Environment.GetEnvironmentVariable("USERPROFILE");
+        var builder = new PythonEnvironment(
+            Environment.GetEnvironmentVariable("USERPROFILE") + "\\.nuget\\packages\\python\\3.12.4\\tools",
+            "3.12.4");
+        env = builder.Build(Path.Join(Environment.CurrentDirectory, "python"));
+
+    }
+
+    public void Dispose()
+    {
+        env.Dispose();
+    }
+
+    public IPythonEnvironment Env { get { return env; } }
+}

Integration.Tests/python/test_basic.py

@@ -0,0 +1,23 @@
+def test_int_float(a: int, b: float) -> float:
+    return a + b
+
+def test_int_int(a: int, b: int) -> int:
+    return a + b
+
+def test_float_float(a: float, b: float) -> float:
+    return a + b
+
+def test_float_int(a: float, b: int) -> float:
+    return a + b
+
+def test_list_of_ints(a: list[int]) -> list[int]:
+    return a
+
+def test_two_strings(a: str, b: str) -> str:
+    return a + b
+
+def test_two_lists_of_strings(a: list[str], b: list[str]) -> list[str]:
+    return a + b
+
+def test_two_dicts(a: dict[str, int], b: dict[str, int]) -> dict[str, int]:
+    return {**a, **b}

Integration.Tests/python/test_defaults.py

@@ -0,0 +1,8 @@
+def test_default_str_arg(a: str = "hello") -> str:
+    return a
+
+def test_default_int_arg(a: int = 1337) -> int:
+    return a
+
+def test_default_float_arg(a: float = -1) -> float:
+    return a

Integration.Tests/python/test_false_returns.py

@@ -0,0 +1,21 @@
+def test_str_actually_returns_int() -> str:
+    return 1
+
+def test_str_actually_returns_float() -> str:
+    return 1.0
+
+def test_str_actually_returns_list() -> str:
+    return [1, 2, 3]
+
+def test_str_actually_returns_tuple() -> str:
+    return (1, 2, 3)
+
+def test_tuple_actually_returns_int() -> tuple[int, int]:
+    return 1
+
+def test_tuple_actually_returns_float() -> tuple[float, float]:
+    return 1.0
+
+def test_tuple_actually_returns_list() -> tuple[list[int], list[int]]:
+    return [1, 2, 3]
+

PythonCodeGen.sln

@@ -19,6 +19,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PythonEnvironments", "Pytho
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PythonSourceGenerator.Tests", "PythonSourceGenerator.Tests\PythonSourceGenerator.Tests.csproj", "{E01D5C27-CB38-4A0D-B297-33E1E10C1A82}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Integration.Tests", "Integration.Tests\Integration.Tests.csproj", "{F179777C-913A-46FE-BE2F-15A59EAB7E88}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -45,6 +47,10 @@ Global
 		{E01D5C27-CB38-4A0D-B297-33E1E10C1A82}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{E01D5C27-CB38-4A0D-B297-33E1E10C1A82}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{E01D5C27-CB38-4A0D-B297-33E1E10C1A82}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F179777C-913A-46FE-BE2F-15A59EAB7E88}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F179777C-913A-46FE-BE2F-15A59EAB7E88}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F179777C-913A-46FE-BE2F-15A59EAB7E88}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F179777C-913A-46FE-BE2F-15A59EAB7E88}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

PythonSourceGenerator.Tests/PythonSourceGenerator.Tests.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>