Merge pull request #178 from tonybaloney/lazy_dictionary

Make the returned dictionary and list types lazy

docs/examples.md

@@ -39,7 +39,7 @@ Because the input and output types are more complex, we've used a list of tuple
 This demo generates the following C# method signature:
 
 ```csharp
-public (IEnumerable<IEnumerable<double>>, double) CalculateKmeansInertia(IEnumerable<(long, long)> data, long nClusters);
+public (IReadOnlyList<IReadOnlyList<double>>, double) CalculateKmeansInertia(IReadOnlyList<(long, long)> data, long nClusters);
 ```
 
 ## Phi-3 inference demo

docs/reference.md

@@ -11,10 +11,10 @@ CSnakes supports the following typed scenarios:
 | `str`                  | `string`          |
 | `bytes`                | `byte[]`          |
 | `bool`                 | `bool`            |
-| `list[T]`              | `IEnumerable<T>`  |
+| `list[T]`              | `IReadOnlyList<T>`  |
 | `dict[K, V]`           | `IReadOnlyDictionary<K, V>` |
 | `tuple[T1, T2, ...]`   | `(T1, T2, ...)`   |
-| `typing.Sequence[T]`   | `IEnumerable<T>`  |
+| `typing.Sequence[T]`   | `IReadOnlyList<T>`  |
 | `typing.Dict[K, V]`    | `IReadOnlyDictionary<K, V>` |
 | `typing.Mapping[K, V]` | `IReadOnlyDictionary<K, V>` |
 | `typing.Tuple[T1, T2, ...]` | `(T1, T2, ...)` |

samples/Aspire/CSnakesAspire.ApiService/CSnakesAspire.ApiService.csproj

@@ -15,7 +15,7 @@
     </ItemGroup>
 
     <ItemGroup>
-        <PackageReference Include="Aspire.Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.1.0" />
+        <PackageReference Include="Aspire.Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.2.0" />
         <PackageReference Include="CSnakes" Version="1.*" />
         <PackageReference Include="CSnakes.Runtime" Version="1.*" />
     </ItemGroup>

samples/Aspire/CSnakesAspire.AppHost/CSnakesAspire.AppHost.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
@@ -15,8 +15,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Aspire.Hosting.AppHost" Version="8.1.0" />
-    <PackageReference Include="Aspire.Hosting.PostgreSQL" Version="8.1.0" />
+    <PackageReference Include="Aspire.Hosting.AppHost" Version="8.2.0" />
+    <PackageReference Include="Aspire.Hosting.PostgreSQL" Version="8.2.0" />
   </ItemGroup>
 
 </Project>

samples/Aspire/CSnakesAspire.ServiceDefaults/CSnakesAspire.ServiceDefaults.csproj

@@ -11,7 +11,7 @@
     <FrameworkReference Include="Microsoft.AspNetCore.App" />
 
     <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="8.8.0" />
-    <PackageReference Include="Microsoft.Extensions.ServiceDiscovery" Version="8.1.0" />
+    <PackageReference Include="Microsoft.Extensions.ServiceDiscovery" Version="8.2.0" />
     <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.9.0" />
     <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.9.0" />
     <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.9.0" />

src/CSnakes.Runtime.Tests/Converter/ListConverterTest.cs

@@ -2,11 +2,7 @@
 
 public class ListConverterTest : ConverterTestBase
 {
-    [Fact]
-    public void IEnumerableConverter() =>
-        RunTest<IReadOnlyCollection<string>>(["Hell0", "W0rld"]);
-
     [Fact]
     public void ListConverter() =>
-        RunTest<List<long>>([123456, 123562]);
+        RunTest<IReadOnlyList<long>>([123456, 123562]);
 }

src/CSnakes.Runtime.Tests/Python/PyDictionaryTests.cs

@@ -0,0 +1,102 @@
+using CSnakes.Runtime.Python;
+
+namespace CSnakes.Runtime.Tests.Python;
+public class PyDictionaryTests : RuntimeTestBase
+{
+    [Fact]
+    public void TestIndex()
+    {
+        var testDict = new Dictionary<string, string>()
+        {
+            ["Hello"] = "World?",
+            ["Foo"] = "Bar"
+        };
+        var pyObject = PyObject.From(testDict);
+        var pyDict = pyObject.As<IReadOnlyDictionary<string, string>>();
+        Assert.NotNull(pyDict);
+        Assert.Equal("World?", pyDict["Hello"]);
+        Assert.Equal("Bar", pyDict["Foo"]);
+        // Try twice 
+        Assert.Equal("World?", pyDict["Hello"]);
+    }
+
+    [Fact]
+    public void TestKeys()
+    {
+        var testDict = new Dictionary<string, string>()
+        {
+            ["Hello"] = "World?",
+            ["Foo"] = "Bar"
+        };
+        var pyObject = PyObject.From(testDict);
+        var pyDict = pyObject.As<IReadOnlyDictionary<string, string>>();
+        Assert.NotNull(pyDict);
+        Assert.Equal(2, pyDict.Count);
+        Assert.Contains("Hello", pyDict.Keys);
+        Assert.Contains("Foo", pyDict.Keys);
+    }
+
+    [Fact]
+    public void TestValues()
+    {
+        var testDict = new Dictionary<string, string>()
+        {
+            ["Hello"] = "World?",
+            ["Foo"] = "Bar"
+        };
+        var pyObject = PyObject.From(testDict);
+        var pyDict = pyObject.As<IReadOnlyDictionary<string, string>>();
+        Assert.NotNull(pyDict);
+        Assert.Equal(2, pyDict.Count);
+        Assert.Contains("World?", pyDict.Values);
+        Assert.Contains("Bar", pyDict.Values);
+    }
+
+    [Fact]
+    public void TestCount()
+    {
+        var testDict = new Dictionary<string, string>()
+        {
+            ["Hello"] = "World?",
+            ["Foo"] = "Bar"
+        };
+        var pyObject = PyObject.From(testDict);
+        var pyDict = pyObject.As<IReadOnlyDictionary<string, string>>();
+        Assert.NotNull(pyDict);
+        Assert.Equal(2, pyDict.Count);
+    }
+
+    [Fact]
+    public void TestContainsKey()
+    {
+        var testDict = new Dictionary<string, string>()
+        {
+            ["Hello"] = "World?",
+            ["Foo"] = "Bar"
+        };
+        var pyObject = PyObject.From(testDict);
+        var pyDict = pyObject.As<IReadOnlyDictionary<string, string>>();
+        Assert.NotNull(pyDict);
+        Assert.True(pyDict.ContainsKey("Hello"));
+        Assert.True(pyDict.ContainsKey("Foo"));
+        Assert.False(pyDict.ContainsKey("Bar"));
+    }
+
+    [Fact]
+    public void TestTryGetValue()
+    {
+        var testDict = new Dictionary<string, string>()
+        {
+            ["Hello"] = "World?",
+            ["Foo"] = "Bar"
+        };
+        var pyObject = PyObject.From(testDict);
+        var pyDict = pyObject.As<IReadOnlyDictionary<string, string>>();
+        Assert.NotNull(pyDict);
+        Assert.True(pyDict.TryGetValue("Hello", out var value));
+        Assert.Equal("World?", value);
+        Assert.True(pyDict.TryGetValue("Foo", out value));
+        Assert.Equal("Bar", value);
+        Assert.False(pyDict.TryGetValue("Bar", out _));
+    }
+}

src/CSnakes.Runtime.Tests/Python/PyObjectTests.cs

@@ -1,5 +1,4 @@
 using CSnakes.Runtime.Python;
-using System.Collections;
 
 namespace CSnakes.Runtime.Tests.Python;
 public class PyObjectTests : RuntimeTestBase
@@ -159,22 +158,4 @@ public void TestObjectNotStrictInequality(object? o1, object? o2, bool expectedL
         Assert.Equal(expectedLT, obj1 <= obj2);
         Assert.Equal(expectedGT, obj1 >= obj2);
     }
-
-    [Fact]
-    public void TestAsCollection()
-    {
-        using PyObject o = PyObject.From<IEnumerable<string>>(new[] { "Hello", "World" })!;
-        var collection = o.AsCollection<IReadOnlyCollection<string>, string>();
-        Assert.NotNull(collection);
-        Assert.Equal(2, collection!.Count());
-    }
-
-    [Fact]
-    public void TestAsDictionary()
-    {
-        using PyObject o = PyObject.From<IDictionary<string, string>>(new Dictionary<string, string> { { "Hello", "World" } })!;
-        var dictionary = o.AsDictionary<IReadOnlyDictionary<string, string>, string, string>();
-        Assert.NotNull(dictionary);
-        Assert.Equal("World", dictionary!["Hello"]);
-    }
 }

src/CSnakes.Runtime/CPython/Dict.cs

@@ -61,6 +61,22 @@ internal static nint PyDict_GetItem(PyObject dict, PyObject key)
         return result;
     }
 
+    /// <summary>
+    /// Does the dictionary contain the key? Raises exception on failure
+    /// </summary>
+    /// <param name="dict"></param>
+    /// <param name="key"></param>
+    /// <returns></returns>
+    internal static bool PyDict_Contains(PyObject dict, PyObject key)
+    {
+        int result = PyDict_Contains_(dict, key);
+        if(result == -1)
+        {
+            throw PyObject.ThrowPythonExceptionAsClrException();
+        }
+        return result == 1;
+    }
+
     /// <summary>
     /// Return the object from dictionary p which has a key `key`. 
     /// Return NULL if the key key is not present, but without setting an exception.
@@ -93,4 +109,13 @@ internal static nint PyDict_GetItem(PyObject dict, PyObject key)
     /// <returns>New reference to the items().</returns>
     [LibraryImport(PythonLibraryName)]
     internal static partial nint PyDict_Items(PyObject dict);
+
+    [LibraryImport(PythonLibraryName, EntryPoint = "PyDict_Contains")]
+    private static partial int PyDict_Contains_(PyObject dict, PyObject key);
+
+    [LibraryImport(PythonLibraryName)]
+    internal static partial nint PyDict_Keys(PyObject dict);
+
+    [LibraryImport(PythonLibraryName)]
+    internal static partial nint PyDict_Values(PyObject dict);
 }

src/CSnakes.Runtime/CPython/Iter.cs

@@ -0,0 +1,18 @@
+using CSnakes.Runtime.Python;
+using System.Runtime.InteropServices;
+
+namespace CSnakes.Runtime.CPython;
+internal unsafe partial class CPythonAPI
+{
+    /// <summary>
+    /// Return the next value from the iterator o. 
+    /// The object must be an iterator according to PyIter_Check() 
+    /// (it is up to the caller to check this).
+    /// If there are no remaining values, returns NULL with no exception set.
+    /// If an error occurs while retrieving the item, returns NULL and passes along the exception.
+    /// </summary>
+    /// <param name="iter"></param>
+    /// <returns>New refernce to the next item</returns>
+    [LibraryImport(PythonLibraryName)]
+    internal static partial nint PyIter_Next(PyObject iter);
+}

src/CSnakes.Runtime/CPython/Mapping.cs

@@ -54,4 +54,23 @@ internal static bool PyMapping_SetItem(PyObject dict, PyObject key, PyObject val
     /// <returns>New reference to the items().</returns>
     [LibraryImport(PythonLibraryName)]
     internal static partial nint PyMapping_Items(PyObject dict);
+
+    [LibraryImport(PythonLibraryName)]
+    internal static partial nint PyMapping_Keys(PyObject dict);
+
+    [LibraryImport(PythonLibraryName)]
+    internal static partial nint PyMapping_Values(PyObject dict);
+
+    [LibraryImport(PythonLibraryName)]
+    internal static partial nint PyMapping_Size(PyObject dict);
+
+    /// <summary>
+    /// Return 1 if the mapping object has the key key and 0 otherwise. 
+    /// This is equivalent to the Python expression key in o. This function always succeeds.
+    /// </summary>
+    /// <param name="dict"></param>
+    /// <param name="key"></param>
+    /// <returns></returns>
+    [LibraryImport(PythonLibraryName)]
+    internal static partial int PyMapping_HasKey(PyObject dict, PyObject key);
 }

src/CSnakes.Runtime/PyObjectTypeConverter.Dictionary.cs

@@ -1,92 +1,23 @@
-using CSnakes.Runtime.CPython;
 using CSnakes.Runtime.Python;
 using System.Collections;
-using System.Collections.ObjectModel;
 
 namespace CSnakes.Runtime;
 internal partial class PyObjectTypeConverter
 {
     private static object ConvertToDictionary(PyObject pyObject, Type destinationType, bool useMappingProtocol = false)
     {
-        using PyObject items = useMappingProtocol ? 
-            PyObject.Create(CPythonAPI.PyMapping_Items(pyObject)) : 
-            PyObject.Create(CPythonAPI.PyDict_Items(pyObject));
-
         Type keyType = destinationType.GetGenericArguments()[0];
         Type valueType = destinationType.GetGenericArguments()[1];
 
         if (!knownDynamicTypes.TryGetValue(destinationType, out DynamicTypeInfo? typeInfo))
         {
-            Type dictType = typeof(Dictionary<,>).MakeGenericType(keyType, valueType);
-            Type returnType = typeof(ReadOnlyDictionary<,>).MakeGenericType(keyType, valueType);
+            Type dictType = typeof(PyDictionary<,>).MakeGenericType(keyType, valueType);
 
-            typeInfo = new(returnType.GetConstructor([dictType])!, dictType.GetConstructor([typeof(int)])!);
+            typeInfo = new(dictType.GetConstructor([typeof(PyObject)])!);
             knownDynamicTypes[destinationType] = typeInfo;
         }
 
-        nint itemsLength = CPythonAPI.PyList_Size(items);
-        IDictionary dict = (IDictionary)typeInfo.TransientTypeConstructor!.Invoke([(int)itemsLength]);
-
-        for (nint i = 0; i < itemsLength; i++)
-        {
-            // TODO: We make 3 heap allocations per item here.
-            // 1. The item, which could be inlined as it's only used to call PyTuple_GetItem.
-            // 2. The key, which we need to recursively convert -- although if this is a string, which it mostly is, then we _could_ avoid this.
-            // 3. The value, which we need to recursively convert.
-            nint kvpTuple = CPythonAPI.PyList_GetItem(items, i);
-
-            // Optimize keys as string because this is the most common case.
-            if (keyType == typeof(string))
-            {
-                nint itemKey = CPythonAPI.PyTuple_GetItemWithNewRefRaw(kvpTuple, 0);
-                using PyObject value = PyObject.Create(CPythonAPI.PyTuple_GetItemWithNewRefRaw(kvpTuple, 1));
-
-                string? keyAsString = CPythonAPI.PyUnicode_AsUTF8Raw(itemKey);
-                if (keyAsString is null)
-                {
-                    CPythonAPI.Py_DecRefRaw(itemKey);
-                    throw PyObject.ThrowPythonExceptionAsClrException();
-                }
-                object? convertedValue = value.As(valueType);
-
-                dict.Add(keyAsString!, convertedValue);
-                CPythonAPI.Py_DecRefRaw(itemKey);
-            } else
-            {
-                using PyObject key = PyObject.Create(CPythonAPI.PyTuple_GetItemWithNewRefRaw(kvpTuple, 0));
-                using PyObject value = PyObject.Create(CPythonAPI.PyTuple_GetItemWithNewRefRaw(kvpTuple, 1));
-
-                object? convertedKey = key.As(keyType);
-                object? convertedValue = value.As(valueType);
-
-                dict.Add(convertedKey!, convertedValue);
-            }
-            CPythonAPI.Py_DecRefRaw(kvpTuple);
-        }
-
-        return typeInfo.ReturnTypeConstructor.Invoke([dict]);
-    }
-
-    internal static IReadOnlyDictionary<TKey, TValue> ConvertToDictionary<TKey, TValue>(PyObject pyObject) where TKey : notnull
-    {
-        using PyObject items = PyObject.Create(CPythonAPI.PyMapping_Items(pyObject));
-
-        var dict = new Dictionary<TKey, TValue>();
-        nint itemsLength = CPythonAPI.PyList_Size(items);
-        for (nint i = 0; i < itemsLength; i++)
-        {
-            nint kvpTuple = CPythonAPI.PyList_GetItem(items, i);
-            using PyObject key = PyObject.Create(CPythonAPI.PyTuple_GetItemWithNewRefRaw(kvpTuple, 0));
-            using PyObject value = PyObject.Create(CPythonAPI.PyTuple_GetItemWithNewRefRaw(kvpTuple, 1));
-
-            TKey convertedKey = key.As<TKey>();
-            TValue convertedValue = value.As<TValue>();
-
-            dict.Add(convertedKey, convertedValue);
-            CPythonAPI.Py_DecRefRaw(kvpTuple);
-        }
-
-        return dict;
+        return typeInfo.ReturnTypeConstructor.Invoke([pyObject.Clone()]);
     }
 
     internal static PyObject ConvertFromDictionary(IDictionary dictionary)

src/CSnakes.Runtime/PyObjectTypeConverter.KnownTypes.cs

@@ -4,7 +4,7 @@
 namespace CSnakes.Runtime;
 internal partial class PyObjectTypeConverter
 {
-    static readonly Type listType = typeof(IReadOnlyCollection<>);
+    static readonly Type listType = typeof(IReadOnlyList<>);
     static readonly Type collectionType = typeof(IEnumerable);
     static readonly Type dictionaryType = typeof(IReadOnlyDictionary<,>);
     static readonly Type pyObjectType = typeof(PyObject);