From be61eff905aaf73e0ac1320d124169686cc6441f Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 29 Aug 2024 19:51:41 +1000 Subject: [PATCH 01/15] Implement a custom dictionary class that only fetches PyObject values when it needs to. Also implement an enumerable type as we need it for the dict. --- .../Python/PyObjectTests.cs | 18 --- src/CSnakes.Runtime/CPython/Dict.cs | 25 ++++ src/CSnakes.Runtime/CPython/Iter.cs | 19 +++ .../PyObjectTypeConverter.Dictionary.cs | 73 +----------- .../Python/GeneratorIterator.cs | 2 +- src/CSnakes.Runtime/Python/PyDictionary.cs | 108 ++++++++++++++++++ src/CSnakes.Runtime/Python/PyEnumerable.cs | 58 ++++++++++ src/CSnakes.Runtime/Python/PyObject.cs | 20 ---- src/Integration.Tests/IntegrationTestBase.cs | 3 +- 9 files changed, 218 insertions(+), 108 deletions(-) create mode 100644 src/CSnakes.Runtime/CPython/Iter.cs create mode 100644 src/CSnakes.Runtime/Python/PyDictionary.cs create mode 100644 src/CSnakes.Runtime/Python/PyEnumerable.cs diff --git a/src/CSnakes.Runtime.Tests/Python/PyObjectTests.cs b/src/CSnakes.Runtime.Tests/Python/PyObjectTests.cs index bfcf96de..7d58b41d 100644 --- a/src/CSnakes.Runtime.Tests/Python/PyObjectTests.cs +++ b/src/CSnakes.Runtime.Tests/Python/PyObjectTests.cs @@ -159,22 +159,4 @@ public void TestObjectNotStrictInequality(object? o1, object? o2, bool expectedL Assert.Equal(expectedLT, obj1 <= obj2); Assert.Equal(expectedGT, obj1 >= obj2); } - - [Fact] - public void TestAsCollection() - { - using PyObject o = PyObject.From>(new[] { "Hello", "World" })!; - var collection = o.AsCollection, string>(); - Assert.NotNull(collection); - Assert.Equal(2, collection!.Count()); - } - - [Fact] - public void TestAsDictionary() - { - using PyObject o = PyObject.From>(new Dictionary { { "Hello", "World" } })!; - var dictionary = o.AsDictionary, string, string>(); - Assert.NotNull(dictionary); - Assert.Equal("World", dictionary!["Hello"]); - } } \ No newline at end of file diff --git a/src/CSnakes.Runtime/CPython/Dict.cs b/src/CSnakes.Runtime/CPython/Dict.cs index 29e048da..b07d43e6 100644 --- a/src/CSnakes.Runtime/CPython/Dict.cs +++ b/src/CSnakes.Runtime/CPython/Dict.cs @@ -61,6 +61,22 @@ internal static nint PyDict_GetItem(PyObject dict, PyObject key) return result; } + /// + /// Does the dictionary contain the key? Raises exception on failure + /// + /// + /// + /// + internal static bool PyDict_Contains(PyObject dict, PyObject key) + { + int result = PyDict_Contains_(dict, key); + if(result == -1) + { + throw PyObject.ThrowPythonExceptionAsClrException(); + } + return result == 1; + } + /// /// Return the object from dictionary p which has a key `key`. /// Return NULL if the key key is not present, but without setting an exception. @@ -93,4 +109,13 @@ internal static nint PyDict_GetItem(PyObject dict, PyObject key) /// New reference to the items(). [LibraryImport(PythonLibraryName)] internal static partial nint PyDict_Items(PyObject dict); + + [LibraryImport(PythonLibraryName, EntryPoint = "PyDict_Contains")] + private static partial int PyDict_Contains_(PyObject dict, PyObject key); + + [LibraryImport(PythonLibraryName)] + internal static partial nint PyDict_Keys(PyObject dict); + + [LibraryImport(PythonLibraryName)] + internal static partial nint PyDict_Values(PyObject dict); } diff --git a/src/CSnakes.Runtime/CPython/Iter.cs b/src/CSnakes.Runtime/CPython/Iter.cs new file mode 100644 index 00000000..340853b1 --- /dev/null +++ b/src/CSnakes.Runtime/CPython/Iter.cs @@ -0,0 +1,19 @@ +using CSnakes.Runtime.Python; +using System.Runtime.InteropServices; + + +namespace CSnakes.Runtime.CPython; +internal unsafe partial class CPythonAPI +{ + /// + /// Return the next value from the iterator o. + /// The object must be an iterator according to PyIter_Check() + /// (it is up to the caller to check this). + /// If there are no remaining values, returns NULL with no exception set. + /// If an error occurs while retrieving the item, returns NULL and passes along the exception. + /// + /// + /// New refernce to the next item + [LibraryImport(PythonLibraryName)] + internal static partial nint PyIter_Next(PyObject iter); +} diff --git a/src/CSnakes.Runtime/PyObjectTypeConverter.Dictionary.cs b/src/CSnakes.Runtime/PyObjectTypeConverter.Dictionary.cs index c8f1c0ef..733a6f94 100644 --- a/src/CSnakes.Runtime/PyObjectTypeConverter.Dictionary.cs +++ b/src/CSnakes.Runtime/PyObjectTypeConverter.Dictionary.cs @@ -1,92 +1,29 @@ using CSnakes.Runtime.CPython; using CSnakes.Runtime.Python; using System.Collections; -using System.Collections.ObjectModel; namespace CSnakes.Runtime; internal partial class PyObjectTypeConverter { private static object ConvertToDictionary(PyObject pyObject, Type destinationType, bool useMappingProtocol = false) { - using PyObject items = useMappingProtocol ? - PyObject.Create(CPythonAPI.PyMapping_Items(pyObject)) : - PyObject.Create(CPythonAPI.PyDict_Items(pyObject)); - Type keyType = destinationType.GetGenericArguments()[0]; Type valueType = destinationType.GetGenericArguments()[1]; if (!knownDynamicTypes.TryGetValue(destinationType, out DynamicTypeInfo? typeInfo)) { - Type dictType = typeof(Dictionary<,>).MakeGenericType(keyType, valueType); - Type returnType = typeof(ReadOnlyDictionary<,>).MakeGenericType(keyType, valueType); + Type dictType = typeof(PyDictionary<,>).MakeGenericType(keyType, valueType); - typeInfo = new(returnType.GetConstructor([dictType])!, dictType.GetConstructor([typeof(int)])!); + typeInfo = new(dictType.GetConstructor([typeof(PyObject)])!); knownDynamicTypes[destinationType] = typeInfo; } - nint itemsLength = CPythonAPI.PyList_Size(items); - IDictionary dict = (IDictionary)typeInfo.TransientTypeConstructor!.Invoke([(int)itemsLength]); - - for (nint i = 0; i < itemsLength; i++) - { - // TODO: We make 3 heap allocations per item here. - // 1. The item, which could be inlined as it's only used to call PyTuple_GetItem. - // 2. The key, which we need to recursively convert -- although if this is a string, which it mostly is, then we _could_ avoid this. - // 3. The value, which we need to recursively convert. - nint kvpTuple = CPythonAPI.PyList_GetItem(items, i); - - // Optimize keys as string because this is the most common case. - if (keyType == typeof(string)) - { - nint itemKey = CPythonAPI.PyTuple_GetItemWithNewRefRaw(kvpTuple, 0); - using PyObject value = PyObject.Create(CPythonAPI.PyTuple_GetItemWithNewRefRaw(kvpTuple, 1)); - - string? keyAsString = CPythonAPI.PyUnicode_AsUTF8Raw(itemKey); - if (keyAsString is null) - { - CPythonAPI.Py_DecRefRaw(itemKey); - throw PyObject.ThrowPythonExceptionAsClrException(); - } - object? convertedValue = value.As(valueType); - - dict.Add(keyAsString!, convertedValue); - CPythonAPI.Py_DecRefRaw(itemKey); - } else - { - using PyObject key = PyObject.Create(CPythonAPI.PyTuple_GetItemWithNewRefRaw(kvpTuple, 0)); - using PyObject value = PyObject.Create(CPythonAPI.PyTuple_GetItemWithNewRefRaw(kvpTuple, 1)); - - object? convertedKey = key.As(keyType); - object? convertedValue = value.As(valueType); - - dict.Add(convertedKey!, convertedValue); - } - CPythonAPI.Py_DecRefRaw(kvpTuple); - } - - return typeInfo.ReturnTypeConstructor.Invoke([dict]); + return typeInfo.ReturnTypeConstructor.Invoke([pyObject.Clone()]); } internal static IReadOnlyDictionary ConvertToDictionary(PyObject pyObject) where TKey : notnull - { - using PyObject items = PyObject.Create(CPythonAPI.PyMapping_Items(pyObject)); - - var dict = new Dictionary(); - nint itemsLength = CPythonAPI.PyList_Size(items); - for (nint i = 0; i < itemsLength; i++) - { - nint kvpTuple = CPythonAPI.PyList_GetItem(items, i); - using PyObject key = PyObject.Create(CPythonAPI.PyTuple_GetItemWithNewRefRaw(kvpTuple, 0)); - using PyObject value = PyObject.Create(CPythonAPI.PyTuple_GetItemWithNewRefRaw(kvpTuple, 1)); - - TKey convertedKey = key.As(); - TValue convertedValue = value.As(); - - dict.Add(convertedKey, convertedValue); - CPythonAPI.Py_DecRefRaw(kvpTuple); - } - - return dict; + { + return new PyDictionary(pyObject); } internal static PyObject ConvertFromDictionary(IDictionary dictionary) diff --git a/src/CSnakes.Runtime/Python/GeneratorIterator.cs b/src/CSnakes.Runtime/Python/GeneratorIterator.cs index 59cab376..da222bec 100644 --- a/src/CSnakes.Runtime/Python/GeneratorIterator.cs +++ b/src/CSnakes.Runtime/Python/GeneratorIterator.cs @@ -49,7 +49,7 @@ public bool MoveNext() } } - public void Reset() => throw new NotImplementedException(); + public void Reset() => throw new NotSupportedException(); public TYield Send(TSend value) diff --git a/src/CSnakes.Runtime/Python/PyDictionary.cs b/src/CSnakes.Runtime/Python/PyDictionary.cs new file mode 100644 index 00000000..318fb121 --- /dev/null +++ b/src/CSnakes.Runtime/Python/PyDictionary.cs @@ -0,0 +1,108 @@ +using CSnakes.Runtime.CPython; +using System.Collections; +using System.Diagnostics.CodeAnalysis; + + +namespace CSnakes.Runtime.Python +{ + public class PyDictionary : IReadOnlyDictionary + { + private readonly Dictionary _dictionary; + private readonly PyObject _dictionaryObject; + public PyDictionary(PyObject dictionary) { + _dictionaryObject = dictionary; + _dictionary = []; + } + + public TValue this[TKey key] { + get + { + if (_dictionary.ContainsKey(key)) + { + return _dictionary[key]; + } else + { + using (GIL.Acquire()) + { + using PyObject keyPyObject = PyObject.From(key); + using PyObject value = PyObject.Create(CPythonAPI.PyDict_GetItem(_dictionaryObject, keyPyObject)); + var managedValue = value.As(); + + _dictionary[key] = managedValue; + return managedValue; + } + } + } + } + + public IEnumerable Keys { + get { + using (GIL.Acquire()) { + return new PyEnumerable(PyObject.Create(CPythonAPI.PyDict_Keys(_dictionaryObject))); + } + } + } + + public IEnumerable Values + { + get + { + using (GIL.Acquire()) + { + return new PyEnumerable(PyObject.Create(CPythonAPI.PyDict_Values(_dictionaryObject))); + } + } + } + + public int Count + { + get + { + using (GIL.Acquire()) + { + return (int)CPythonAPI.PyDict_Size(_dictionaryObject); + } + } + } + + public bool ContainsKey(TKey key) + { + if (_dictionary.ContainsKey(key)) + { + return true; + } + else + { + using (GIL.Acquire()) + { + using PyObject keyPyObject = PyObject.From(key); + return CPythonAPI.PyDict_Contains(_dictionaryObject, keyPyObject); + } + } + } + + public IEnumerator> GetEnumerator() + { + throw new NotImplementedException(); + } + + public bool TryGetValue(TKey key, [MaybeNullWhen(false)] out TValue value) + { + if (ContainsKey(key)) + { + value = this[key]; + return true; + } + else + { + value = default; + return false; + } + } + + IEnumerator IEnumerable.GetEnumerator() + { + throw new NotImplementedException(); + } + } +} diff --git a/src/CSnakes.Runtime/Python/PyEnumerable.cs b/src/CSnakes.Runtime/Python/PyEnumerable.cs new file mode 100644 index 00000000..a1861e5e --- /dev/null +++ b/src/CSnakes.Runtime/Python/PyEnumerable.cs @@ -0,0 +1,58 @@ +using CSnakes.Runtime.CPython; +using System.Collections; + + +namespace CSnakes.Runtime.Python; +internal class PyEnumerable : IEnumerable, IEnumerator +{ + private readonly PyObject _pyIterator; + private TValue current = default!; + + internal PyEnumerable(PyObject pyObject) + { + using (GIL.Acquire()) + { + _pyIterator = pyObject.GetIter(); + } + } + + public TValue Current => current; + + object IEnumerator.Current => current!; + + public void Dispose() + { + _pyIterator.Dispose(); + } + + public IEnumerator GetEnumerator() => this; + + public bool MoveNext() + { + using (GIL.Acquire()) + { + nint result = CPythonAPI.PyIter_Next(_pyIterator); + if (result == IntPtr.Zero && CPythonAPI.PyErr_Occurred()) + { + throw PyObject.ThrowPythonExceptionAsClrException(); + } + else if (result == IntPtr.Zero) + { + return false; + } + else + { + using PyObject pyObject = PyObject.Create(result); + current = pyObject.As(); + return true; + } + } + } + + public void Reset() + { + throw new NotSupportedException("Python iterators cannot be reset"); + } + + IEnumerator IEnumerable.GetEnumerator() => this; +} diff --git a/src/CSnakes.Runtime/Python/PyObject.cs b/src/CSnakes.Runtime/Python/PyObject.cs index c631651a..313069a2 100644 --- a/src/CSnakes.Runtime/Python/PyObject.cs +++ b/src/CSnakes.Runtime/Python/PyObject.cs @@ -460,26 +460,6 @@ var t when t.IsAssignableTo(typeof(IGeneratorIterator)) => PyObjectTypeConverter } } - public IReadOnlyCollection AsCollection() where TCollection : IReadOnlyCollection - { - using (GIL.Acquire()) - { - return PyObjectTypeConverter.ConvertToCollection(this); - } - } - public IReadOnlyDictionary AsDictionary() where TDict : IReadOnlyDictionary where TKey : notnull - { - using (GIL.Acquire()) - { - return PyObjectTypeConverter.ConvertToDictionary(this); - } - } - - public IReadOnlyCollection As() where TCollection : IReadOnlyCollection => - PyObjectTypeConverter.ConvertToCollection(this); - public IReadOnlyDictionary As() where TDict : IReadOnlyDictionary where TKey : notnull => - PyObjectTypeConverter.ConvertToDictionary(this); - public static PyObject From(T value) { using (GIL.Acquire()) diff --git a/src/Integration.Tests/IntegrationTestBase.cs b/src/Integration.Tests/IntegrationTestBase.cs index d0a5dc0c..4408b2f9 100644 --- a/src/Integration.Tests/IntegrationTestBase.cs +++ b/src/Integration.Tests/IntegrationTestBase.cs @@ -22,7 +22,8 @@ public IntegrationTestBase() var pb = services.WithPython(); pb.WithHome(Path.Join(Environment.CurrentDirectory, "python")); - pb.FromNuGet(pythonVersionWindows) + pb.FromSource(@"C:\Users\anthonyshaw\source\repos\cpython", "3.12") + .FromNuGet(pythonVersionWindows) .FromMacOSInstallerLocator(pythonVersionMacOS) .FromEnvironmentVariable("Python3_ROOT_DIR", pythonVersionLinux) .WithVirtualEnvironment(venvPath) From ff3dff8a84fa82b28f8f1037ff0ff82b90a0bdb1 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 30 Aug 2024 07:43:58 +1000 Subject: [PATCH 02/15] Implement a keyvaluepair converter for tuples and complete the rest of the dictionary type --- .../PyObjectTypeConverter.Tuple.cs | 17 ++++++ src/CSnakes.Runtime/Python/PyDictionary.cs | 11 ++-- src/CSnakes.Runtime/Python/PyEnumerable.cs | 2 +- .../Python/PyKeyValuePairEnumerable.cs | 58 +++++++++++++++++++ src/CSnakes.Runtime/Python/PyObject.cs | 13 +++++ 5 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 src/CSnakes.Runtime/Python/PyKeyValuePairEnumerable.cs diff --git a/src/CSnakes.Runtime/PyObjectTypeConverter.Tuple.cs b/src/CSnakes.Runtime/PyObjectTypeConverter.Tuple.cs index a9b9a898..f2d10028 100644 --- a/src/CSnakes.Runtime/PyObjectTypeConverter.Tuple.cs +++ b/src/CSnakes.Runtime/PyObjectTypeConverter.Tuple.cs @@ -84,4 +84,21 @@ internal static ITuple ConvertToTuple(PyObject pyObj, Type destinationType) return (ITuple)typeInfo.ReturnTypeConstructor.Invoke([.. clrValues]); } + + internal static KeyValuePair ConvertToKeyValuePair(PyObject pyObj) + { + if (!CPythonAPI.IsPyTuple(pyObj)) + { + throw new InvalidCastException($"Cannot convert {pyObj.GetPythonType()} to a keyvaluepair."); + } + if (CPythonAPI.PyTuple_Size(pyObj) != 2) + { + throw new InvalidDataException("Tuple must have exactly 2 elements to be converted to a KeyValuePair."); + } + + using PyObject key = PyObject.Create(CPythonAPI.PyTuple_GetItemWithNewRef(pyObj, 0)); + using PyObject value = PyObject.Create(CPythonAPI.PyTuple_GetItemWithNewRef(pyObj, 1)); + + return new KeyValuePair(key.As(), value.As()); + } } \ No newline at end of file diff --git a/src/CSnakes.Runtime/Python/PyDictionary.cs b/src/CSnakes.Runtime/Python/PyDictionary.cs index 318fb121..e6270ac4 100644 --- a/src/CSnakes.Runtime/Python/PyDictionary.cs +++ b/src/CSnakes.Runtime/Python/PyDictionary.cs @@ -83,7 +83,11 @@ public bool ContainsKey(TKey key) public IEnumerator> GetEnumerator() { - throw new NotImplementedException(); + using (GIL.Acquire()) + { + using var items = PyObject.Create(CPythonAPI.PyDict_Items(_dictionaryObject)); + return new PyKeyValuePairEnumerable(items).GetEnumerator(); + } } public bool TryGetValue(TKey key, [MaybeNullWhen(false)] out TValue value) @@ -100,9 +104,6 @@ public bool TryGetValue(TKey key, [MaybeNullWhen(false)] out TValue value) } } - IEnumerator IEnumerable.GetEnumerator() - { - throw new NotImplementedException(); - } + IEnumerator IEnumerable.GetEnumerator() => this.GetEnumerator(); } } diff --git a/src/CSnakes.Runtime/Python/PyEnumerable.cs b/src/CSnakes.Runtime/Python/PyEnumerable.cs index a1861e5e..f4f23ac5 100644 --- a/src/CSnakes.Runtime/Python/PyEnumerable.cs +++ b/src/CSnakes.Runtime/Python/PyEnumerable.cs @@ -1,8 +1,8 @@ using CSnakes.Runtime.CPython; using System.Collections; - namespace CSnakes.Runtime.Python; + internal class PyEnumerable : IEnumerable, IEnumerator { private readonly PyObject _pyIterator; diff --git a/src/CSnakes.Runtime/Python/PyKeyValuePairEnumerable.cs b/src/CSnakes.Runtime/Python/PyKeyValuePairEnumerable.cs new file mode 100644 index 00000000..8ecd7e11 --- /dev/null +++ b/src/CSnakes.Runtime/Python/PyKeyValuePairEnumerable.cs @@ -0,0 +1,58 @@ +using CSnakes.Runtime.CPython; +using System.Collections; + +namespace CSnakes.Runtime.Python; + +internal class PyKeyValuePairEnumerable : IEnumerable>, IEnumerator> +{ + private readonly PyObject _pyIterator; + private KeyValuePair current = default!; + + internal PyKeyValuePairEnumerable(PyObject pyObject) + { + using (GIL.Acquire()) + { + _pyIterator = pyObject.GetIter(); + } + } + + public KeyValuePair Current => current; + + object IEnumerator.Current => current!; + + public void Dispose() + { + _pyIterator.Dispose(); + } + + public IEnumerator> GetEnumerator() => this; + + public bool MoveNext() + { + using (GIL.Acquire()) + { + nint result = CPythonAPI.PyIter_Next(_pyIterator); + if (result == IntPtr.Zero && CPythonAPI.PyErr_Occurred()) + { + throw PyObject.ThrowPythonExceptionAsClrException(); + } + else if (result == IntPtr.Zero) + { + return false; + } + else + { + using PyObject pyObject = PyObject.Create(result); + current = pyObject.As(); + return true; + } + } + } + + public void Reset() + { + throw new NotSupportedException("Python iterators cannot be reset"); + } + + IEnumerator IEnumerable.GetEnumerator() => this; +} diff --git a/src/CSnakes.Runtime/Python/PyObject.cs b/src/CSnakes.Runtime/Python/PyObject.cs index 313069a2..b62f4b18 100644 --- a/src/CSnakes.Runtime/Python/PyObject.cs +++ b/src/CSnakes.Runtime/Python/PyObject.cs @@ -439,6 +439,19 @@ public override string ToString() public T As() => (T)As(typeof(T)); + /// + /// Unpack a tuple of 2 elements into a KeyValuePair + /// + /// The type of the key + /// The type of the value + /// + public KeyValuePair As() { + using (GIL.Acquire()) + { + return PyObjectTypeConverter.ConvertToKeyValuePair(this); + } + } + internal object As(Type type) { using (GIL.Acquire()) From 51020171c188ef94b66fbb6477f6a49488aaa70d Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 30 Aug 2024 07:55:53 +1000 Subject: [PATCH 03/15] Use mapping protocols. Add tests --- .../Python/PyDictionaryTests.cs | 100 ++++++++++++++++++ src/CSnakes.Runtime/CPython/Mapping.cs | 19 ++++ src/CSnakes.Runtime/Python/PyDictionary.cs | 12 +-- 3 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 src/CSnakes.Runtime.Tests/Python/PyDictionaryTests.cs diff --git a/src/CSnakes.Runtime.Tests/Python/PyDictionaryTests.cs b/src/CSnakes.Runtime.Tests/Python/PyDictionaryTests.cs new file mode 100644 index 00000000..b8dbb74d --- /dev/null +++ b/src/CSnakes.Runtime.Tests/Python/PyDictionaryTests.cs @@ -0,0 +1,100 @@ +using CSnakes.Runtime.Python; + +namespace CSnakes.Runtime.Tests.Python; +public class PyDictionaryTests : RuntimeTestBase +{ + [Fact] + public void TestIndex() + { + var testDict = new Dictionary() + { + ["Hello"] = "World?", + ["Foo"] = "Bar" + }; + var pyObject = PyObject.From(testDict); + var pyDict = new PyDictionary(pyObject); + Assert.NotNull(pyDict); + Assert.Equal("World?", pyDict["Hello"]); + Assert.Equal("Bar", pyDict["Foo"]); + // Try twice + Assert.Equal("World?", pyDict["Hello"]); + } + + [Fact] + public void TestKeys() + { + var testDict = new Dictionary() + { + ["Hello"] = "World?", + ["Foo"] = "Bar" + }; + var pyObject = PyObject.From(testDict); + var pyDict = new PyDictionary(pyObject); + Assert.NotNull(pyDict); + Assert.Equal(2, pyDict.Count); + Assert.Contains("Hello", pyDict.Keys); + Assert.Contains("Foo", pyDict.Keys); + } + + [Fact] + public void TestValues() { + var testDict = new Dictionary() + { + ["Hello"] = "World?", + ["Foo"] = "Bar" + }; + var pyObject = PyObject.From(testDict); + var pyDict = new PyDictionary(pyObject); + Assert.NotNull(pyDict); + Assert.Equal(2, pyDict.Count); + Assert.Contains("World?", pyDict.Values); + Assert.Contains("Bar", pyDict.Values); + } + + [Fact] + public void TestCount() + { + var testDict = new Dictionary() + { + ["Hello"] = "World?", + ["Foo"] = "Bar" + }; + var pyObject = PyObject.From(testDict); + var pyDict = new PyDictionary(pyObject); + Assert.NotNull(pyDict); + Assert.Equal(2, pyDict.Count); + } + + [Fact] + public void TestContainsKey() { + var testDict = new Dictionary() + { + ["Hello"] = "World?", + ["Foo"] = "Bar" + }; + var pyObject = PyObject.From(testDict); + var pyDict = new PyDictionary(pyObject); + Assert.NotNull(pyDict); + Assert.True(pyDict.ContainsKey("Hello")); + Assert.True(pyDict.ContainsKey("Foo")); + Assert.False(pyDict.ContainsKey("Bar")); + } + + [Fact] + public void TestTryGetValue() + { + var testDict = new Dictionary() + { + ["Hello"] = "World?", + ["Foo"] = "Bar" + }; + var pyObject = PyObject.From(testDict); + var pyDict = new PyDictionary(pyObject); + Assert.NotNull(pyDict); + Assert.True(pyDict.TryGetValue("Hello", out var value)); + Assert.Equal("World?", value); + Assert.True(pyDict.TryGetValue("Foo", out value)); + Assert.Equal("Bar", value); + Assert.False(pyDict.TryGetValue("Bar", out value)); + } +} \ No newline at end of file diff --git a/src/CSnakes.Runtime/CPython/Mapping.cs b/src/CSnakes.Runtime/CPython/Mapping.cs index f7382b7f..493ae04f 100644 --- a/src/CSnakes.Runtime/CPython/Mapping.cs +++ b/src/CSnakes.Runtime/CPython/Mapping.cs @@ -54,4 +54,23 @@ internal static bool PyMapping_SetItem(PyObject dict, PyObject key, PyObject val /// New reference to the items(). [LibraryImport(PythonLibraryName)] internal static partial nint PyMapping_Items(PyObject dict); + + [LibraryImport(PythonLibraryName)] + internal static partial nint PyMapping_Keys(PyObject dict); + + [LibraryImport(PythonLibraryName)] + internal static partial nint PyMapping_Values(PyObject dict); + + [LibraryImport(PythonLibraryName)] + internal static partial nint PyMapping_Size(PyObject dict); + + /// + /// Return 1 if the mapping object has the key key and 0 otherwise. + /// This is equivalent to the Python expression key in o. This function always succeeds. + /// + /// + /// + /// + [LibraryImport(PythonLibraryName)] + internal static partial int PyMapping_HasKey(PyObject dict, PyObject key); } diff --git a/src/CSnakes.Runtime/Python/PyDictionary.cs b/src/CSnakes.Runtime/Python/PyDictionary.cs index e6270ac4..e961ded2 100644 --- a/src/CSnakes.Runtime/Python/PyDictionary.cs +++ b/src/CSnakes.Runtime/Python/PyDictionary.cs @@ -25,7 +25,7 @@ public TValue this[TKey key] { using (GIL.Acquire()) { using PyObject keyPyObject = PyObject.From(key); - using PyObject value = PyObject.Create(CPythonAPI.PyDict_GetItem(_dictionaryObject, keyPyObject)); + using PyObject value = PyObject.Create(CPythonAPI.PyMapping_GetItem(_dictionaryObject, keyPyObject)); var managedValue = value.As(); _dictionary[key] = managedValue; @@ -38,7 +38,7 @@ public TValue this[TKey key] { public IEnumerable Keys { get { using (GIL.Acquire()) { - return new PyEnumerable(PyObject.Create(CPythonAPI.PyDict_Keys(_dictionaryObject))); + return new PyEnumerable(PyObject.Create(CPythonAPI.PyMapping_Keys(_dictionaryObject))); } } } @@ -49,7 +49,7 @@ public IEnumerable Values { using (GIL.Acquire()) { - return new PyEnumerable(PyObject.Create(CPythonAPI.PyDict_Values(_dictionaryObject))); + return new PyEnumerable(PyObject.Create(CPythonAPI.PyMapping_Values(_dictionaryObject))); } } } @@ -60,7 +60,7 @@ public int Count { using (GIL.Acquire()) { - return (int)CPythonAPI.PyDict_Size(_dictionaryObject); + return (int)CPythonAPI.PyMapping_Size(_dictionaryObject); } } } @@ -76,7 +76,7 @@ public bool ContainsKey(TKey key) using (GIL.Acquire()) { using PyObject keyPyObject = PyObject.From(key); - return CPythonAPI.PyDict_Contains(_dictionaryObject, keyPyObject); + return CPythonAPI.PyMapping_HasKey(_dictionaryObject, keyPyObject) == 1; } } } @@ -85,7 +85,7 @@ public IEnumerator> GetEnumerator() { using (GIL.Acquire()) { - using var items = PyObject.Create(CPythonAPI.PyDict_Items(_dictionaryObject)); + using var items = PyObject.Create(CPythonAPI.PyMapping_Items(_dictionaryObject)); return new PyKeyValuePairEnumerable(items).GetEnumerator(); } } From c0609a330e99f2d20a95d394beac75c538fc1def Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 30 Aug 2024 08:07:40 +1000 Subject: [PATCH 04/15] Stop hardcoding! --- src/Integration.Tests/IntegrationTestBase.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Integration.Tests/IntegrationTestBase.cs b/src/Integration.Tests/IntegrationTestBase.cs index 4408b2f9..d0a5dc0c 100644 --- a/src/Integration.Tests/IntegrationTestBase.cs +++ b/src/Integration.Tests/IntegrationTestBase.cs @@ -22,8 +22,7 @@ public IntegrationTestBase() var pb = services.WithPython(); pb.WithHome(Path.Join(Environment.CurrentDirectory, "python")); - pb.FromSource(@"C:\Users\anthonyshaw\source\repos\cpython", "3.12") - .FromNuGet(pythonVersionWindows) + pb.FromNuGet(pythonVersionWindows) .FromMacOSInstallerLocator(pythonVersionMacOS) .FromEnvironmentVariable("Python3_ROOT_DIR", pythonVersionLinux) .WithVirtualEnvironment(venvPath) From dc0ca257c27abfc0377cefa79443796143eef489 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 30 Aug 2024 08:35:17 +1000 Subject: [PATCH 05/15] Setup collection --- src/CSnakes.Runtime/Python/PyCollection.cs | 25 ++++++++++++++++++++++ src/CSnakes.Runtime/Python/PyDictionary.cs | 3 ++- 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 src/CSnakes.Runtime/Python/PyCollection.cs diff --git a/src/CSnakes.Runtime/Python/PyCollection.cs b/src/CSnakes.Runtime/Python/PyCollection.cs new file mode 100644 index 00000000..a9ddf782 --- /dev/null +++ b/src/CSnakes.Runtime/Python/PyCollection.cs @@ -0,0 +1,25 @@ +using System.Collections; + +namespace CSnakes.Runtime.Python; + +public class PyCollection : IReadOnlyCollection, IReadOnlyList +{ + public PyCollection(PyObject listObject) + { + throw new NotImplementedException(); + } + + public TItem this[int index] => throw new NotImplementedException(); + + public int Count => throw new NotImplementedException(); + + public IEnumerator GetEnumerator() + { + throw new NotImplementedException(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + throw new NotImplementedException(); + } +} diff --git a/src/CSnakes.Runtime/Python/PyDictionary.cs b/src/CSnakes.Runtime/Python/PyDictionary.cs index e961ded2..6264608a 100644 --- a/src/CSnakes.Runtime/Python/PyDictionary.cs +++ b/src/CSnakes.Runtime/Python/PyDictionary.cs @@ -9,7 +9,8 @@ public class PyDictionary : IReadOnlyDictionary { private readonly Dictionary _dictionary; private readonly PyObject _dictionaryObject; - public PyDictionary(PyObject dictionary) { + + internal PyDictionary(PyObject dictionary) { _dictionaryObject = dictionary; _dictionary = []; } From e872a9430c617f6ffee3807fb92cf791f696e0c3 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 30 Aug 2024 08:35:34 +1000 Subject: [PATCH 06/15] Make the constructor internal --- src/CSnakes.Runtime/Python/PyDictionary.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSnakes.Runtime/Python/PyDictionary.cs b/src/CSnakes.Runtime/Python/PyDictionary.cs index e961ded2..99734114 100644 --- a/src/CSnakes.Runtime/Python/PyDictionary.cs +++ b/src/CSnakes.Runtime/Python/PyDictionary.cs @@ -9,7 +9,7 @@ public class PyDictionary : IReadOnlyDictionary { private readonly Dictionary _dictionary; private readonly PyObject _dictionaryObject; - public PyDictionary(PyObject dictionary) { + internal PyDictionary(PyObject dictionary) { _dictionaryObject = dictionary; _dictionary = []; } From 492e5693d518f36994bf38e9a07b5f13cf42bde5 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 30 Aug 2024 08:38:09 +1000 Subject: [PATCH 07/15] Update Aspire workloads --- .../CSnakesAspire.ApiService.csproj | 2 +- .../CSnakesAspire.AppHost/CSnakesAspire.AppHost.csproj | 6 +++--- .../CSnakesAspire.ServiceDefaults.csproj | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/samples/Aspire/CSnakesAspire.ApiService/CSnakesAspire.ApiService.csproj b/samples/Aspire/CSnakesAspire.ApiService/CSnakesAspire.ApiService.csproj index ed7f7b3d..fcbf5acb 100644 --- a/samples/Aspire/CSnakesAspire.ApiService/CSnakesAspire.ApiService.csproj +++ b/samples/Aspire/CSnakesAspire.ApiService/CSnakesAspire.ApiService.csproj @@ -15,7 +15,7 @@ - + diff --git a/samples/Aspire/CSnakesAspire.AppHost/CSnakesAspire.AppHost.csproj b/samples/Aspire/CSnakesAspire.AppHost/CSnakesAspire.AppHost.csproj index 88285624..7953197a 100644 --- a/samples/Aspire/CSnakesAspire.AppHost/CSnakesAspire.AppHost.csproj +++ b/samples/Aspire/CSnakesAspire.AppHost/CSnakesAspire.AppHost.csproj @@ -1,4 +1,4 @@ - + Exe @@ -15,8 +15,8 @@ - - + + diff --git a/samples/Aspire/CSnakesAspire.ServiceDefaults/CSnakesAspire.ServiceDefaults.csproj b/samples/Aspire/CSnakesAspire.ServiceDefaults/CSnakesAspire.ServiceDefaults.csproj index 16712f26..2609cf58 100644 --- a/samples/Aspire/CSnakesAspire.ServiceDefaults/CSnakesAspire.ServiceDefaults.csproj +++ b/samples/Aspire/CSnakesAspire.ServiceDefaults/CSnakesAspire.ServiceDefaults.csproj @@ -11,7 +11,7 @@ - + From 6b9d76c7cc153b364359762af09873ee64ec76a6 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 30 Aug 2024 08:50:01 +1000 Subject: [PATCH 08/15] Use the convertor for tests. Make the dictionary class internal --- .../Python/PyDictionaryTests.cs | 12 ++++++------ src/CSnakes.Runtime/Python/PyDictionary.cs | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/CSnakes.Runtime.Tests/Python/PyDictionaryTests.cs b/src/CSnakes.Runtime.Tests/Python/PyDictionaryTests.cs index b8dbb74d..bcc39fe3 100644 --- a/src/CSnakes.Runtime.Tests/Python/PyDictionaryTests.cs +++ b/src/CSnakes.Runtime.Tests/Python/PyDictionaryTests.cs @@ -12,7 +12,7 @@ public void TestIndex() ["Foo"] = "Bar" }; var pyObject = PyObject.From(testDict); - var pyDict = new PyDictionary(pyObject); + var pyDict = pyObject.As>(); Assert.NotNull(pyDict); Assert.Equal("World?", pyDict["Hello"]); Assert.Equal("Bar", pyDict["Foo"]); @@ -29,7 +29,7 @@ public void TestKeys() ["Foo"] = "Bar" }; var pyObject = PyObject.From(testDict); - var pyDict = new PyDictionary(pyObject); + var pyDict = pyObject.As>(); Assert.NotNull(pyDict); Assert.Equal(2, pyDict.Count); Assert.Contains("Hello", pyDict.Keys); @@ -44,7 +44,7 @@ public void TestValues() { ["Foo"] = "Bar" }; var pyObject = PyObject.From(testDict); - var pyDict = new PyDictionary(pyObject); + var pyDict = pyObject.As>(); Assert.NotNull(pyDict); Assert.Equal(2, pyDict.Count); Assert.Contains("World?", pyDict.Values); @@ -60,7 +60,7 @@ public void TestCount() ["Foo"] = "Bar" }; var pyObject = PyObject.From(testDict); - var pyDict = new PyDictionary(pyObject); + var pyDict = pyObject.As>(); Assert.NotNull(pyDict); Assert.Equal(2, pyDict.Count); } @@ -73,7 +73,7 @@ public void TestContainsKey() { ["Foo"] = "Bar" }; var pyObject = PyObject.From(testDict); - var pyDict = new PyDictionary(pyObject); + var pyDict = pyObject.As>(); Assert.NotNull(pyDict); Assert.True(pyDict.ContainsKey("Hello")); Assert.True(pyDict.ContainsKey("Foo")); @@ -89,7 +89,7 @@ public void TestTryGetValue() ["Foo"] = "Bar" }; var pyObject = PyObject.From(testDict); - var pyDict = new PyDictionary(pyObject); + var pyDict = pyObject.As>(); Assert.NotNull(pyDict); Assert.True(pyDict.TryGetValue("Hello", out var value)); Assert.Equal("World?", value); diff --git a/src/CSnakes.Runtime/Python/PyDictionary.cs b/src/CSnakes.Runtime/Python/PyDictionary.cs index 99734114..05567db9 100644 --- a/src/CSnakes.Runtime/Python/PyDictionary.cs +++ b/src/CSnakes.Runtime/Python/PyDictionary.cs @@ -5,11 +5,11 @@ namespace CSnakes.Runtime.Python { - public class PyDictionary : IReadOnlyDictionary + internal class PyDictionary : IReadOnlyDictionary { private readonly Dictionary _dictionary; private readonly PyObject _dictionaryObject; - internal PyDictionary(PyObject dictionary) { + public PyDictionary(PyObject dictionary) { _dictionaryObject = dictionary; _dictionary = []; } From bc53012a0c13b0c61edcae146c22adb7896b37c0 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 30 Aug 2024 09:00:28 +1000 Subject: [PATCH 09/15] Use IReadOnlyList --- .../Converter/ListConverterTest.cs | 2 +- .../PyObjectTypeConverter.KnownTypes.cs | 2 +- .../PyObjectTypeConverter.List.cs | 13 ---------- src/CSnakes.Runtime/Python/PyCollection.cs | 2 +- src/CSnakes.Runtime/PythonRuntimeException.cs | 2 +- src/CSnakes.Tests/GeneratedSignatureTests.cs | 6 ++--- src/CSnakes.Tests/TypeReflectionTests.cs | 24 +++++++++---------- src/CSnakes/Reflection/TypeReflection.cs | 2 +- src/Integration.Tests/DictsTests.cs | 2 +- 9 files changed, 21 insertions(+), 34 deletions(-) diff --git a/src/CSnakes.Runtime.Tests/Converter/ListConverterTest.cs b/src/CSnakes.Runtime.Tests/Converter/ListConverterTest.cs index 1ffb3709..1c0d2829 100644 --- a/src/CSnakes.Runtime.Tests/Converter/ListConverterTest.cs +++ b/src/CSnakes.Runtime.Tests/Converter/ListConverterTest.cs @@ -4,7 +4,7 @@ public class ListConverterTest : ConverterTestBase { [Fact] public void IEnumerableConverter() => - RunTest>(["Hell0", "W0rld"]); + RunTest>(["Hell0", "W0rld"]); [Fact] public void ListConverter() => diff --git a/src/CSnakes.Runtime/PyObjectTypeConverter.KnownTypes.cs b/src/CSnakes.Runtime/PyObjectTypeConverter.KnownTypes.cs index 01e3c132..e64f94b7 100644 --- a/src/CSnakes.Runtime/PyObjectTypeConverter.KnownTypes.cs +++ b/src/CSnakes.Runtime/PyObjectTypeConverter.KnownTypes.cs @@ -4,7 +4,7 @@ namespace CSnakes.Runtime; internal partial class PyObjectTypeConverter { - static readonly Type listType = typeof(IReadOnlyCollection<>); + static readonly Type listType = typeof(IReadOnlyList<>); static readonly Type collectionType = typeof(IEnumerable); static readonly Type dictionaryType = typeof(IReadOnlyDictionary<,>); static readonly Type pyObjectType = typeof(PyObject); diff --git a/src/CSnakes.Runtime/PyObjectTypeConverter.List.cs b/src/CSnakes.Runtime/PyObjectTypeConverter.List.cs index 465ef7e4..7f9a5ab0 100644 --- a/src/CSnakes.Runtime/PyObjectTypeConverter.List.cs +++ b/src/CSnakes.Runtime/PyObjectTypeConverter.List.cs @@ -52,19 +52,6 @@ private static ICollection ConvertToListFromSequence(PyObject pyObject, Type des return list; } - internal static IReadOnlyCollection ConvertToCollection(PyObject pyObject) - { - nint listSize = CPythonAPI.PySequence_Size(pyObject); - var list = new List((int)listSize); - for (var i = 0; i < listSize; i++) - { - using PyObject item = PyObject.Create(CPythonAPI.PySequence_GetItem(pyObject, i)); - list.Add(item.As()); - } - - return list; - } - internal static PyObject ConvertFromList(ICollection e) { List pyObjects = new(e.Count); diff --git a/src/CSnakes.Runtime/Python/PyCollection.cs b/src/CSnakes.Runtime/Python/PyCollection.cs index a9ddf782..ce38ef7b 100644 --- a/src/CSnakes.Runtime/Python/PyCollection.cs +++ b/src/CSnakes.Runtime/Python/PyCollection.cs @@ -2,7 +2,7 @@ namespace CSnakes.Runtime.Python; -public class PyCollection : IReadOnlyCollection, IReadOnlyList +internal class PyCollection : IReadOnlyList { public PyCollection(PyObject listObject) { diff --git a/src/CSnakes.Runtime/PythonRuntimeException.cs b/src/CSnakes.Runtime/PythonRuntimeException.cs index 5ec437a4..6f4e8c17 100644 --- a/src/CSnakes.Runtime/PythonRuntimeException.cs +++ b/src/CSnakes.Runtime/PythonRuntimeException.cs @@ -39,7 +39,7 @@ private static string[] FormatPythonStackTrace(PyObject pythonStackTrace) using var formatTbFunction = tracebackModule.GetAttr("format_tb"); using var formattedStackTrace = formatTbFunction.Call(pythonStackTrace); - return [.. formattedStackTrace.As>()]; + return [.. formattedStackTrace.As>()]; } } diff --git a/src/CSnakes.Tests/GeneratedSignatureTests.cs b/src/CSnakes.Tests/GeneratedSignatureTests.cs index 1190e027..fd0bcf33 100644 --- a/src/CSnakes.Tests/GeneratedSignatureTests.cs +++ b/src/CSnakes.Tests/GeneratedSignatureTests.cs @@ -20,10 +20,10 @@ public class GeneratedSignatureTests(TestEnvironment testEnv) : IClassFixture float:\n ...\n", "double HelloWorld(string name)")] [InlineData("def hello_world(name: str) -> int:\n ...\n", "long HelloWorld(string name)")] [InlineData("def hello_world(name: str, age: int) -> str:\n ...\n", "string HelloWorld(string name, long age)")] - [InlineData("def hello_world(numbers: list[float]) -> list[int]:\n ...\n", "IReadOnlyCollection HelloWorld(IReadOnlyCollection numbers)")] - [InlineData("def hello_world(numbers: List[float]) -> List[int]:\n ...\n", "IReadOnlyCollection HelloWorld(IReadOnlyCollection numbers)")] + [InlineData("def hello_world(numbers: list[float]) -> list[int]:\n ...\n", "IReadOnlyList HelloWorld(IReadOnlyList numbers)")] + [InlineData("def hello_world(numbers: List[float]) -> List[int]:\n ...\n", "IReadOnlyList HelloWorld(IReadOnlyList numbers)")] [InlineData("def hello_world(value: tuple[int]) -> None:\n ...\n", "void HelloWorld(ValueTuple value)")] - [InlineData("def hello_world(a: bool, b: str, c: list[tuple[int, float]]) -> bool: \n ...\n", "bool HelloWorld(bool a, string b, IReadOnlyCollection<(long, double)> c)")] + [InlineData("def hello_world(a: bool, b: str, c: list[tuple[int, float]]) -> bool: \n ...\n", "bool HelloWorld(bool a, string b, IReadOnlyList<(long, double)> c)")] [InlineData("def hello_world(a: bool = True, b: str = None) -> bool: \n ...\n", "bool HelloWorld(bool a = true, string? b = null)")] [InlineData("def hello_world(a: bytes, b: bool = False, c: float = 0.1) -> None: \n ...\n", "void HelloWorld(byte[] a, bool b = false, double c = 0.1)")] [InlineData("def hello_world(a: str = 'default') -> None: \n ...\n", "void HelloWorld(string a = \"default\")")] diff --git a/src/CSnakes.Tests/TypeReflectionTests.cs b/src/CSnakes.Tests/TypeReflectionTests.cs index 57e41068..08919054 100644 --- a/src/CSnakes.Tests/TypeReflectionTests.cs +++ b/src/CSnakes.Tests/TypeReflectionTests.cs @@ -11,34 +11,34 @@ public class TypeReflectionTests [InlineData("str", "string")] [InlineData("float", "double")] [InlineData("bool", "bool")] - [InlineData("list[int]", "IReadOnlyCollection")] - [InlineData("list[str]", "IReadOnlyCollection")] - [InlineData("list[float]", "IReadOnlyCollection")] - [InlineData("list[bool]", "IReadOnlyCollection")] - [InlineData("list[object]", "IReadOnlyCollection")] + [InlineData("list[int]", "IReadOnlyList")] + [InlineData("list[str]", "IReadOnlyList")] + [InlineData("list[float]", "IReadOnlyList")] + [InlineData("list[bool]", "IReadOnlyList")] + [InlineData("list[object]", "IReadOnlyList")] [InlineData("tuple[int, int]", "(long,long)")] [InlineData("tuple[str, str]", "(string,string)")] [InlineData("tuple[float, float]", "(double,double)")] [InlineData("tuple[bool, bool]", "(bool,bool)")] [InlineData("tuple[str, Any]", "(string,PyObject)")] - [InlineData("tuple[str, list[int]]", "(string,IReadOnlyCollection)")] + [InlineData("tuple[str, list[int]]", "(string,IReadOnlyList)")] [InlineData("dict[str, int]", "IReadOnlyDictionary")] [InlineData("tuple[int, int, tuple[int, int]]", "(long,long,(long,long))")] public void AsPredefinedType(string pythonType, string expectedType) => ParsingTestInternal(pythonType, expectedType); [Theory] - [InlineData("List[int]", "IReadOnlyCollection")] - [InlineData("List[str]", "IReadOnlyCollection")] - [InlineData("List[float]", "IReadOnlyCollection")] - [InlineData("List[bool]", "IReadOnlyCollection")] - [InlineData("List[object]", "IReadOnlyCollection")] + [InlineData("List[int]", "IReadOnlyList")] + [InlineData("List[str]", "IReadOnlyList")] + [InlineData("List[float]", "IReadOnlyList")] + [InlineData("List[bool]", "IReadOnlyList")] + [InlineData("List[object]", "IReadOnlyList")] [InlineData("Tuple[int, int]", "(long,long)")] [InlineData("Tuple[str, str]", "(string,string)")] [InlineData("Tuple[float, float]", "(double,double)")] [InlineData("Tuple[bool, bool]", "(bool,bool)")] [InlineData("Tuple[str, Any]", "(string,PyObject)")] - [InlineData("Tuple[str, list[int]]", "(string,IReadOnlyCollection)")] + [InlineData("Tuple[str, list[int]]", "(string,IReadOnlyList)")] [InlineData("Dict[str, int]", "IReadOnlyDictionary")] [InlineData("Tuple[int, int, Tuple[int, int]]", "(long,long,(long,long))")] public void AsPredefinedTypeOldTypeNames(string pythonType, string expectedType) => diff --git a/src/CSnakes/Reflection/TypeReflection.cs b/src/CSnakes/Reflection/TypeReflection.cs index 36b6610f..a21fdcd7 100644 --- a/src/CSnakes/Reflection/TypeReflection.cs +++ b/src/CSnakes/Reflection/TypeReflection.cs @@ -70,7 +70,7 @@ private static TypeSyntax CreateTupleType(PythonTypeSpec[] tupleTypes) SyntaxFactory.Token(SyntaxKind.CloseParenToken)); } - private static TypeSyntax CreateListType(PythonTypeSpec genericOf) => CreateGenericType("IReadOnlyCollection", [AsPredefinedType(genericOf)]); + private static TypeSyntax CreateListType(PythonTypeSpec genericOf) => CreateGenericType("IReadOnlyList", [AsPredefinedType(genericOf)]); internal static TypeSyntax CreateGenericType(string typeName, IEnumerable genericArguments) => SyntaxFactory.GenericName( diff --git a/src/Integration.Tests/DictsTests.cs b/src/Integration.Tests/DictsTests.cs index e0072656..f7d010a1 100644 --- a/src/Integration.Tests/DictsTests.cs +++ b/src/Integration.Tests/DictsTests.cs @@ -17,7 +17,7 @@ public void TestDicts_TestDictStrListInt() { var testDicts = Env.TestDicts(); - IReadOnlyDictionary> testListDict = new Dictionary> { { "hello", new List { 1, 2, 3 } }, { "world2", new List { 4, 5, 6 } } }; + IReadOnlyDictionary> testListDict = new Dictionary> { { "hello", new List { 1, 2, 3 } }, { "world2", new List { 4, 5, 6 } } }; var result = testDicts.TestDictStrListInt(testListDict); Assert.Equal(new List { 1, 2, 3 }, result["hello"]); Assert.Equal(new List { 4, 5, 6 }, result["world2"]); From 1ac6c65199b171dcb72f552a997febf3fac09e21 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 30 Aug 2024 09:16:06 +1000 Subject: [PATCH 10/15] Implements list iterator and get at index --- .../Converter/ListConverterTest.cs | 6 +-- .../PyObjectTypeConverter.Dictionary.cs | 6 --- .../PyObjectTypeConverter.List.cs | 41 ++-------------- src/CSnakes.Runtime/PyObjectTypeConverter.cs | 2 +- src/CSnakes.Runtime/Python/PyCollection.cs | 25 ---------- src/CSnakes.Runtime/Python/PyDictionary.cs | 25 +++++++--- src/CSnakes.Runtime/Python/PyEnumerable.cs | 2 +- src/CSnakes.Runtime/Python/PyList.cs | 49 +++++++++++++++++++ 8 files changed, 74 insertions(+), 82 deletions(-) delete mode 100644 src/CSnakes.Runtime/Python/PyCollection.cs create mode 100644 src/CSnakes.Runtime/Python/PyList.cs diff --git a/src/CSnakes.Runtime.Tests/Converter/ListConverterTest.cs b/src/CSnakes.Runtime.Tests/Converter/ListConverterTest.cs index 1c0d2829..1b08418c 100644 --- a/src/CSnakes.Runtime.Tests/Converter/ListConverterTest.cs +++ b/src/CSnakes.Runtime.Tests/Converter/ListConverterTest.cs @@ -2,11 +2,7 @@ public class ListConverterTest : ConverterTestBase { - [Fact] - public void IEnumerableConverter() => - RunTest>(["Hell0", "W0rld"]); - [Fact] public void ListConverter() => - RunTest>([123456, 123562]); + RunTest>([123456, 123562]); } diff --git a/src/CSnakes.Runtime/PyObjectTypeConverter.Dictionary.cs b/src/CSnakes.Runtime/PyObjectTypeConverter.Dictionary.cs index 733a6f94..4bf18d8a 100644 --- a/src/CSnakes.Runtime/PyObjectTypeConverter.Dictionary.cs +++ b/src/CSnakes.Runtime/PyObjectTypeConverter.Dictionary.cs @@ -1,4 +1,3 @@ -using CSnakes.Runtime.CPython; using CSnakes.Runtime.Python; using System.Collections; @@ -21,11 +20,6 @@ private static object ConvertToDictionary(PyObject pyObject, Type destinationTyp return typeInfo.ReturnTypeConstructor.Invoke([pyObject.Clone()]); } - internal static IReadOnlyDictionary ConvertToDictionary(PyObject pyObject) where TKey : notnull - { - return new PyDictionary(pyObject); - } - internal static PyObject ConvertFromDictionary(IDictionary dictionary) { int len = dictionary.Keys.Count; diff --git a/src/CSnakes.Runtime/PyObjectTypeConverter.List.cs b/src/CSnakes.Runtime/PyObjectTypeConverter.List.cs index 7f9a5ab0..89e0a275 100644 --- a/src/CSnakes.Runtime/PyObjectTypeConverter.List.cs +++ b/src/CSnakes.Runtime/PyObjectTypeConverter.List.cs @@ -1,4 +1,3 @@ -using CSnakes.Runtime.CPython; using CSnakes.Runtime.Python; using System.Collections; using System.Runtime.InteropServices; @@ -6,50 +5,18 @@ namespace CSnakes.Runtime; internal partial class PyObjectTypeConverter { - private static ICollection ConvertToList(PyObject pyObject, Type destinationType) + private static object ConvertToList(PyObject pyObject, Type destinationType) { Type genericArgument = destinationType.GetGenericArguments()[0]; - nint listSize = CPythonAPI.PySequence_Size(pyObject); if (!knownDynamicTypes.TryGetValue(destinationType, out DynamicTypeInfo? typeInfo)) { - Type listType = typeof(List<>).MakeGenericType(genericArgument); - typeInfo = new(listType.GetConstructor([typeof(int)])!); + Type listType = typeof(PyList<>).MakeGenericType(genericArgument); + typeInfo = new(listType.GetConstructor([typeof(PyObject)])!); knownDynamicTypes[destinationType] = typeInfo; } - IList list = (IList)typeInfo.ReturnTypeConstructor.Invoke([(int)listSize]); - - for (var i = 0; i < listSize; i++) - { - using PyObject item = PyObject.Create(CPythonAPI.PyList_GetItem(pyObject, i)); - list.Add(item.As(genericArgument)); - } - - return list; - } - - private static ICollection ConvertToListFromSequence(PyObject pyObject, Type destinationType) - { - Type genericArgument = destinationType.GetGenericArguments()[0]; - - nint listSize = CPythonAPI.PySequence_Size(pyObject); - if (!knownDynamicTypes.TryGetValue(destinationType, out DynamicTypeInfo? typeInfo)) - { - Type listType = typeof(List<>).MakeGenericType(genericArgument); - typeInfo = new(listType.GetConstructor([typeof(int)])!); - knownDynamicTypes[destinationType] = typeInfo; - } - - IList list = (IList)typeInfo.ReturnTypeConstructor.Invoke([(int)listSize]); - - for (var i = 0; i < listSize; i++) - { - using PyObject item = PyObject.Create(CPythonAPI.PySequence_GetItem(pyObject, i)); - list.Add(item.As(genericArgument)); - } - - return list; + return typeInfo.ReturnTypeConstructor.Invoke([pyObject.Clone()]); } internal static PyObject ConvertFromList(ICollection e) diff --git a/src/CSnakes.Runtime/PyObjectTypeConverter.cs b/src/CSnakes.Runtime/PyObjectTypeConverter.cs index 3ec49e07..e791d14c 100644 --- a/src/CSnakes.Runtime/PyObjectTypeConverter.cs +++ b/src/CSnakes.Runtime/PyObjectTypeConverter.cs @@ -31,7 +31,7 @@ public static object PyObjectToManagedType(PyObject pyObject, Type destinationTy if (CPythonAPI.IsPySequence(pyObject) && IsAssignableToGenericType(destinationType, listType)) { - return ConvertToListFromSequence(pyObject, destinationType); + return ConvertToList(pyObject, destinationType); } throw new InvalidCastException($"Attempting to cast {destinationType} from {pyObject.GetPythonType()}"); diff --git a/src/CSnakes.Runtime/Python/PyCollection.cs b/src/CSnakes.Runtime/Python/PyCollection.cs deleted file mode 100644 index ce38ef7b..00000000 --- a/src/CSnakes.Runtime/Python/PyCollection.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System.Collections; - -namespace CSnakes.Runtime.Python; - -internal class PyCollection : IReadOnlyList -{ - public PyCollection(PyObject listObject) - { - throw new NotImplementedException(); - } - - public TItem this[int index] => throw new NotImplementedException(); - - public int Count => throw new NotImplementedException(); - - public IEnumerator GetEnumerator() - { - throw new NotImplementedException(); - } - - IEnumerator IEnumerable.GetEnumerator() - { - throw new NotImplementedException(); - } -} diff --git a/src/CSnakes.Runtime/Python/PyDictionary.cs b/src/CSnakes.Runtime/Python/PyDictionary.cs index 78aabe73..fbc8e3ec 100644 --- a/src/CSnakes.Runtime/Python/PyDictionary.cs +++ b/src/CSnakes.Runtime/Python/PyDictionary.cs @@ -5,23 +5,26 @@ namespace CSnakes.Runtime.Python { - internal class PyDictionary : IReadOnlyDictionary + internal class PyDictionary : IReadOnlyDictionary, IDisposable { private readonly Dictionary _dictionary; private readonly PyObject _dictionaryObject; - internal PyDictionary(PyObject dictionary) { + public PyDictionary(PyObject dictionary) + { _dictionaryObject = dictionary; _dictionary = []; } - public TValue this[TKey key] { + public TValue this[TKey key] + { get { if (_dictionary.ContainsKey(key)) { return _dictionary[key]; - } else + } + else { using (GIL.Acquire()) { @@ -36,9 +39,12 @@ public TValue this[TKey key] { } } - public IEnumerable Keys { - get { - using (GIL.Acquire()) { + public IEnumerable Keys + { + get + { + using (GIL.Acquire()) + { return new PyEnumerable(PyObject.Create(CPythonAPI.PyMapping_Keys(_dictionaryObject))); } } @@ -82,6 +88,11 @@ public bool ContainsKey(TKey key) } } + public void Dispose() + { + _dictionaryObject.Dispose(); + } + public IEnumerator> GetEnumerator() { using (GIL.Acquire()) diff --git a/src/CSnakes.Runtime/Python/PyEnumerable.cs b/src/CSnakes.Runtime/Python/PyEnumerable.cs index f4f23ac5..f9fb1257 100644 --- a/src/CSnakes.Runtime/Python/PyEnumerable.cs +++ b/src/CSnakes.Runtime/Python/PyEnumerable.cs @@ -3,7 +3,7 @@ namespace CSnakes.Runtime.Python; -internal class PyEnumerable : IEnumerable, IEnumerator +internal class PyEnumerable : IEnumerable, IEnumerator, IDisposable { private readonly PyObject _pyIterator; private TValue current = default!; diff --git a/src/CSnakes.Runtime/Python/PyList.cs b/src/CSnakes.Runtime/Python/PyList.cs new file mode 100644 index 00000000..f764a198 --- /dev/null +++ b/src/CSnakes.Runtime/Python/PyList.cs @@ -0,0 +1,49 @@ +using CSnakes.Runtime.CPython; +using System.Collections; + +namespace CSnakes.Runtime.Python; + +internal class PyList : IReadOnlyList, IDisposable +{ + private readonly PyObject _listObject; + + public PyList(PyObject listObject) => _listObject = listObject; + + public TItem this[int index] + { + get + { + using (GIL.Acquire()) + { + using PyObject value = PyObject.Create(CPythonAPI.PySequence_GetItem(_listObject, index)); + return value.As(); + } + } + } + + public int Count + { + get + { + using (GIL.Acquire()) + { + return (int)CPythonAPI.PySequence_Size(_listObject); + } + } + } + + public void Dispose() + { + _listObject.Dispose(); + } + + public IEnumerator GetEnumerator() + { + using (GIL.Acquire()) + { + return new PyEnumerable(_listObject); + } + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); +} From 2d21d1513828042402d8d0a0489ee871f72fd11e Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 30 Aug 2024 09:21:04 +1000 Subject: [PATCH 11/15] Add an index cache --- src/CSnakes.Runtime/Python/PyList.cs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/CSnakes.Runtime/Python/PyList.cs b/src/CSnakes.Runtime/Python/PyList.cs index f764a198..395142bb 100644 --- a/src/CSnakes.Runtime/Python/PyList.cs +++ b/src/CSnakes.Runtime/Python/PyList.cs @@ -7,16 +7,26 @@ internal class PyList : IReadOnlyList, IDisposable { private readonly PyObject _listObject; + // If someone fetches the same index multiple times, we cache the result to avoid multiple round trips to Python + private readonly Dictionary _convertedItems = new(); + public PyList(PyObject listObject) => _listObject = listObject; public TItem this[int index] { get { + if (_convertedItems.TryGetValue(index, out TItem cachedValue)) + { + return cachedValue; + } + using (GIL.Acquire()) { using PyObject value = PyObject.Create(CPythonAPI.PySequence_GetItem(_listObject, index)); - return value.As(); + TItem result = value.As(); + _convertedItems[index] = result; + return result; } } } From 51bb6cffde39e6023e13dcd2f46d1b2d77733ce0 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 30 Aug 2024 09:21:42 +1000 Subject: [PATCH 12/15] Add a todo about caching the enumerator --- src/CSnakes.Runtime/Python/PyList.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CSnakes.Runtime/Python/PyList.cs b/src/CSnakes.Runtime/Python/PyList.cs index 395142bb..94e0f3f2 100644 --- a/src/CSnakes.Runtime/Python/PyList.cs +++ b/src/CSnakes.Runtime/Python/PyList.cs @@ -49,6 +49,7 @@ public void Dispose() public IEnumerator GetEnumerator() { + // TODO: If someone fetches the same index multiple times, we cache the result to avoid multiple round trips to Python using (GIL.Acquire()) { return new PyEnumerable(_listObject); From b4997f69c4c8f11e71877e71662a9bd368680922 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 30 Aug 2024 09:24:13 +1000 Subject: [PATCH 13/15] Update docs for API changes --- docs/examples.md | 2 +- docs/reference.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index 4ea97e01..3031c0d3 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -39,7 +39,7 @@ Because the input and output types are more complex, we've used a list of tuple This demo generates the following C# method signature: ```csharp -public (IEnumerable>, double) CalculateKmeansInertia(IEnumerable<(long, long)> data, long nClusters); +public (IReadOnlyList>, double) CalculateKmeansInertia(IReadOnlyList<(long, long)> data, long nClusters); ``` ## Phi-3 inference demo diff --git a/docs/reference.md b/docs/reference.md index 99c5ef94..4f8c998e 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -11,10 +11,10 @@ CSnakes supports the following typed scenarios: | `str` | `string` | | `bytes` | `byte[]` | | `bool` | `bool` | -| `list[T]` | `IEnumerable` | +| `list[T]` | `IReadOnlyList` | | `dict[K, V]` | `IReadOnlyDictionary` | | `tuple[T1, T2, ...]` | `(T1, T2, ...)` | -| `typing.Sequence[T]` | `IEnumerable` | +| `typing.Sequence[T]` | `IReadOnlyList` | | `typing.Dict[K, V]` | `IReadOnlyDictionary` | | `typing.Mapping[K, V]` | `IReadOnlyDictionary` | | `typing.Tuple[T1, T2, ...]` | `(T1, T2, ...)` | From 41061c632e5a52102d07bf2205a774590e656132 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 30 Aug 2024 10:25:37 +1000 Subject: [PATCH 14/15] Change the API for iterators. --- src/CSnakes.Runtime/Python/PyObject.cs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/CSnakes.Runtime/Python/PyObject.cs b/src/CSnakes.Runtime/Python/PyObject.cs index b62f4b18..7dec158f 100644 --- a/src/CSnakes.Runtime/Python/PyObject.cs +++ b/src/CSnakes.Runtime/Python/PyObject.cs @@ -149,11 +149,8 @@ public virtual bool HasAttr(string name) } } - /// - /// Get the iterator for the object. This is equivalent to iter(obj) in Python. - /// - /// The iterator object (new ref) - public virtual PyObject GetIter() + + internal virtual PyObject GetIter() { RaiseOnPythonNotInitialized(); using (GIL.Acquire()) @@ -162,6 +159,19 @@ public virtual PyObject GetIter() } } + /// + /// Calls iter() on the object and returns an IEnumerable that yields values of type T. + /// + /// The type for each item in the iterator + /// + public IEnumerable AsEnumerable() + { + using (GIL.Acquire()) + { + return new PyEnumerable(this); + } + } + /// /// Get the results of the repr() function on the object. /// From e320007e09b2cb4003b727c100bbee8d0d24243f Mon Sep 17 00:00:00 2001 From: Aaron Powell Date: Fri, 30 Aug 2024 11:14:27 +1000 Subject: [PATCH 15/15] Fixing some formatting and tightening some of the code --- .../Python/PyDictionaryTests.cs | 200 ++++++++-------- .../Python/PyObjectTests.cs | 1 - src/CSnakes.Runtime/CPython/Iter.cs | 37 ++- .../Python/GeneratorIterator.cs | 3 +- src/CSnakes.Runtime/Python/PyDictionary.cs | 225 ++++++++---------- src/CSnakes.Runtime/Python/PyEnumerable.cs | 109 ++++----- .../Python/PyKeyValuePairEnumerable.cs | 109 ++++----- src/CSnakes.Runtime/Python/PyList.cs | 117 +++++---- src/CSnakes.Runtime/Python/PyObject.cs | 8 +- 9 files changed, 388 insertions(+), 421 deletions(-) diff --git a/src/CSnakes.Runtime.Tests/Python/PyDictionaryTests.cs b/src/CSnakes.Runtime.Tests/Python/PyDictionaryTests.cs index bcc39fe3..8d1c5e71 100644 --- a/src/CSnakes.Runtime.Tests/Python/PyDictionaryTests.cs +++ b/src/CSnakes.Runtime.Tests/Python/PyDictionaryTests.cs @@ -1,100 +1,102 @@ -using CSnakes.Runtime.Python; - -namespace CSnakes.Runtime.Tests.Python; -public class PyDictionaryTests : RuntimeTestBase -{ - [Fact] - public void TestIndex() - { - var testDict = new Dictionary() - { - ["Hello"] = "World?", - ["Foo"] = "Bar" - }; - var pyObject = PyObject.From(testDict); - var pyDict = pyObject.As>(); - Assert.NotNull(pyDict); - Assert.Equal("World?", pyDict["Hello"]); - Assert.Equal("Bar", pyDict["Foo"]); - // Try twice - Assert.Equal("World?", pyDict["Hello"]); - } - - [Fact] - public void TestKeys() - { - var testDict = new Dictionary() - { - ["Hello"] = "World?", - ["Foo"] = "Bar" - }; - var pyObject = PyObject.From(testDict); - var pyDict = pyObject.As>(); - Assert.NotNull(pyDict); - Assert.Equal(2, pyDict.Count); - Assert.Contains("Hello", pyDict.Keys); - Assert.Contains("Foo", pyDict.Keys); - } - - [Fact] - public void TestValues() { - var testDict = new Dictionary() - { - ["Hello"] = "World?", - ["Foo"] = "Bar" - }; - var pyObject = PyObject.From(testDict); - var pyDict = pyObject.As>(); - Assert.NotNull(pyDict); - Assert.Equal(2, pyDict.Count); - Assert.Contains("World?", pyDict.Values); - Assert.Contains("Bar", pyDict.Values); - } - - [Fact] - public void TestCount() - { - var testDict = new Dictionary() - { - ["Hello"] = "World?", - ["Foo"] = "Bar" - }; - var pyObject = PyObject.From(testDict); - var pyDict = pyObject.As>(); - Assert.NotNull(pyDict); - Assert.Equal(2, pyDict.Count); - } - - [Fact] - public void TestContainsKey() { - var testDict = new Dictionary() - { - ["Hello"] = "World?", - ["Foo"] = "Bar" - }; - var pyObject = PyObject.From(testDict); - var pyDict = pyObject.As>(); - Assert.NotNull(pyDict); - Assert.True(pyDict.ContainsKey("Hello")); - Assert.True(pyDict.ContainsKey("Foo")); - Assert.False(pyDict.ContainsKey("Bar")); - } - - [Fact] - public void TestTryGetValue() - { - var testDict = new Dictionary() - { - ["Hello"] = "World?", - ["Foo"] = "Bar" - }; - var pyObject = PyObject.From(testDict); - var pyDict = pyObject.As>(); - Assert.NotNull(pyDict); - Assert.True(pyDict.TryGetValue("Hello", out var value)); - Assert.Equal("World?", value); - Assert.True(pyDict.TryGetValue("Foo", out value)); - Assert.Equal("Bar", value); - Assert.False(pyDict.TryGetValue("Bar", out value)); - } +using CSnakes.Runtime.Python; + +namespace CSnakes.Runtime.Tests.Python; +public class PyDictionaryTests : RuntimeTestBase +{ + [Fact] + public void TestIndex() + { + var testDict = new Dictionary() + { + ["Hello"] = "World?", + ["Foo"] = "Bar" + }; + var pyObject = PyObject.From(testDict); + var pyDict = pyObject.As>(); + Assert.NotNull(pyDict); + Assert.Equal("World?", pyDict["Hello"]); + Assert.Equal("Bar", pyDict["Foo"]); + // Try twice + Assert.Equal("World?", pyDict["Hello"]); + } + + [Fact] + public void TestKeys() + { + var testDict = new Dictionary() + { + ["Hello"] = "World?", + ["Foo"] = "Bar" + }; + var pyObject = PyObject.From(testDict); + var pyDict = pyObject.As>(); + Assert.NotNull(pyDict); + Assert.Equal(2, pyDict.Count); + Assert.Contains("Hello", pyDict.Keys); + Assert.Contains("Foo", pyDict.Keys); + } + + [Fact] + public void TestValues() + { + var testDict = new Dictionary() + { + ["Hello"] = "World?", + ["Foo"] = "Bar" + }; + var pyObject = PyObject.From(testDict); + var pyDict = pyObject.As>(); + Assert.NotNull(pyDict); + Assert.Equal(2, pyDict.Count); + Assert.Contains("World?", pyDict.Values); + Assert.Contains("Bar", pyDict.Values); + } + + [Fact] + public void TestCount() + { + var testDict = new Dictionary() + { + ["Hello"] = "World?", + ["Foo"] = "Bar" + }; + var pyObject = PyObject.From(testDict); + var pyDict = pyObject.As>(); + Assert.NotNull(pyDict); + Assert.Equal(2, pyDict.Count); + } + + [Fact] + public void TestContainsKey() + { + var testDict = new Dictionary() + { + ["Hello"] = "World?", + ["Foo"] = "Bar" + }; + var pyObject = PyObject.From(testDict); + var pyDict = pyObject.As>(); + Assert.NotNull(pyDict); + Assert.True(pyDict.ContainsKey("Hello")); + Assert.True(pyDict.ContainsKey("Foo")); + Assert.False(pyDict.ContainsKey("Bar")); + } + + [Fact] + public void TestTryGetValue() + { + var testDict = new Dictionary() + { + ["Hello"] = "World?", + ["Foo"] = "Bar" + }; + var pyObject = PyObject.From(testDict); + var pyDict = pyObject.As>(); + Assert.NotNull(pyDict); + Assert.True(pyDict.TryGetValue("Hello", out var value)); + Assert.Equal("World?", value); + Assert.True(pyDict.TryGetValue("Foo", out value)); + Assert.Equal("Bar", value); + Assert.False(pyDict.TryGetValue("Bar", out _)); + } } \ No newline at end of file diff --git a/src/CSnakes.Runtime.Tests/Python/PyObjectTests.cs b/src/CSnakes.Runtime.Tests/Python/PyObjectTests.cs index 7d58b41d..5e42a58f 100644 --- a/src/CSnakes.Runtime.Tests/Python/PyObjectTests.cs +++ b/src/CSnakes.Runtime.Tests/Python/PyObjectTests.cs @@ -1,5 +1,4 @@ using CSnakes.Runtime.Python; -using System.Collections; namespace CSnakes.Runtime.Tests.Python; public class PyObjectTests : RuntimeTestBase diff --git a/src/CSnakes.Runtime/CPython/Iter.cs b/src/CSnakes.Runtime/CPython/Iter.cs index 340853b1..47db503a 100644 --- a/src/CSnakes.Runtime/CPython/Iter.cs +++ b/src/CSnakes.Runtime/CPython/Iter.cs @@ -1,19 +1,18 @@ -using CSnakes.Runtime.Python; -using System.Runtime.InteropServices; - - -namespace CSnakes.Runtime.CPython; -internal unsafe partial class CPythonAPI -{ - /// - /// Return the next value from the iterator o. - /// The object must be an iterator according to PyIter_Check() - /// (it is up to the caller to check this). - /// If there are no remaining values, returns NULL with no exception set. - /// If an error occurs while retrieving the item, returns NULL and passes along the exception. - /// - /// - /// New refernce to the next item - [LibraryImport(PythonLibraryName)] - internal static partial nint PyIter_Next(PyObject iter); -} +using CSnakes.Runtime.Python; +using System.Runtime.InteropServices; + +namespace CSnakes.Runtime.CPython; +internal unsafe partial class CPythonAPI +{ + /// + /// Return the next value from the iterator o. + /// The object must be an iterator according to PyIter_Check() + /// (it is up to the caller to check this). + /// If there are no remaining values, returns NULL with no exception set. + /// If an error occurs while retrieving the item, returns NULL and passes along the exception. + /// + /// + /// New refernce to the next item + [LibraryImport(PythonLibraryName)] + internal static partial nint PyIter_Next(PyObject iter); +} diff --git a/src/CSnakes.Runtime/Python/GeneratorIterator.cs b/src/CSnakes.Runtime/Python/GeneratorIterator.cs index da222bec..65c4edd7 100644 --- a/src/CSnakes.Runtime/Python/GeneratorIterator.cs +++ b/src/CSnakes.Runtime/Python/GeneratorIterator.cs @@ -51,12 +51,11 @@ public bool MoveNext() public void Reset() => throw new NotSupportedException(); - public TYield Send(TSend value) { try { - using PyObject sendValue = PyObject.From(value) !; + using PyObject sendValue = PyObject.From(value); using PyObject result = sendPyFunction.Call(sendValue); current = result.As(); return current; diff --git a/src/CSnakes.Runtime/Python/PyDictionary.cs b/src/CSnakes.Runtime/Python/PyDictionary.cs index fbc8e3ec..56b2f3ca 100644 --- a/src/CSnakes.Runtime/Python/PyDictionary.cs +++ b/src/CSnakes.Runtime/Python/PyDictionary.cs @@ -1,121 +1,104 @@ -using CSnakes.Runtime.CPython; -using System.Collections; -using System.Diagnostics.CodeAnalysis; - - -namespace CSnakes.Runtime.Python -{ - internal class PyDictionary : IReadOnlyDictionary, IDisposable - { - private readonly Dictionary _dictionary; - private readonly PyObject _dictionaryObject; - - public PyDictionary(PyObject dictionary) - { - _dictionaryObject = dictionary; - _dictionary = []; - } - - public TValue this[TKey key] - { - get - { - if (_dictionary.ContainsKey(key)) - { - return _dictionary[key]; - } - else - { - using (GIL.Acquire()) - { - using PyObject keyPyObject = PyObject.From(key); - using PyObject value = PyObject.Create(CPythonAPI.PyMapping_GetItem(_dictionaryObject, keyPyObject)); - var managedValue = value.As(); - - _dictionary[key] = managedValue; - return managedValue; - } - } - } - } - - public IEnumerable Keys - { - get - { - using (GIL.Acquire()) - { - return new PyEnumerable(PyObject.Create(CPythonAPI.PyMapping_Keys(_dictionaryObject))); - } - } - } - - public IEnumerable Values - { - get - { - using (GIL.Acquire()) - { - return new PyEnumerable(PyObject.Create(CPythonAPI.PyMapping_Values(_dictionaryObject))); - } - } - } - - public int Count - { - get - { - using (GIL.Acquire()) - { - return (int)CPythonAPI.PyMapping_Size(_dictionaryObject); - } - } - } - - public bool ContainsKey(TKey key) - { - if (_dictionary.ContainsKey(key)) - { - return true; - } - else - { - using (GIL.Acquire()) - { - using PyObject keyPyObject = PyObject.From(key); - return CPythonAPI.PyMapping_HasKey(_dictionaryObject, keyPyObject) == 1; - } - } - } - - public void Dispose() - { - _dictionaryObject.Dispose(); - } - - public IEnumerator> GetEnumerator() - { - using (GIL.Acquire()) - { - using var items = PyObject.Create(CPythonAPI.PyMapping_Items(_dictionaryObject)); - return new PyKeyValuePairEnumerable(items).GetEnumerator(); - } - } - - public bool TryGetValue(TKey key, [MaybeNullWhen(false)] out TValue value) - { - if (ContainsKey(key)) - { - value = this[key]; - return true; - } - else - { - value = default; - return false; - } - } - - IEnumerator IEnumerable.GetEnumerator() => this.GetEnumerator(); - } -} +using CSnakes.Runtime.CPython; +using System.Collections; +using System.Diagnostics.CodeAnalysis; + +namespace CSnakes.Runtime.Python; + +internal class PyDictionary(PyObject dictionary) : IReadOnlyDictionary, IDisposable + where TKey : notnull +{ + private readonly Dictionary _dictionary = []; + private readonly PyObject _dictionaryObject = dictionary; + + public TValue this[TKey key] + { + get + { + if (_dictionary.TryGetValue(key, out TValue? value)) + { + return value; + } + using (GIL.Acquire()) + { + using PyObject keyPyObject = PyObject.From(key); + using PyObject pyObjValue = PyObject.Create(CPythonAPI.PyMapping_GetItem(_dictionaryObject, keyPyObject)); + TValue managedValue = pyObjValue.As(); + + _dictionary[key] = managedValue; + return managedValue; + } + } + } + + public IEnumerable Keys + { + get + { + using (GIL.Acquire()) + { + return new PyEnumerable(PyObject.Create(CPythonAPI.PyMapping_Keys(_dictionaryObject))); + } + } + } + + public IEnumerable Values + { + get + { + using (GIL.Acquire()) + { + return new PyEnumerable(PyObject.Create(CPythonAPI.PyMapping_Values(_dictionaryObject))); + } + } + } + + public int Count + { + get + { + using (GIL.Acquire()) + { + return (int)CPythonAPI.PyMapping_Size(_dictionaryObject); + } + } + } + + public bool ContainsKey(TKey key) + { + if (_dictionary.ContainsKey(key)) + { + return true; + } + + using (GIL.Acquire()) + { + using PyObject keyPyObject = PyObject.From(key); + return CPythonAPI.PyMapping_HasKey(_dictionaryObject, keyPyObject) == 1; + } + } + + public void Dispose() => _dictionaryObject.Dispose(); + + public IEnumerator> GetEnumerator() + { + using (GIL.Acquire()) + { + using var items = PyObject.Create(CPythonAPI.PyMapping_Items(_dictionaryObject)); + return new PyKeyValuePairEnumerable(items).GetEnumerator(); + } + } + + public bool TryGetValue(TKey key, [MaybeNullWhen(false)] out TValue value) + { + if (ContainsKey(key)) + { + value = this[key]; + return true; + } + + value = default; + return false; + } + + IEnumerator IEnumerable.GetEnumerator() => this.GetEnumerator(); +} diff --git a/src/CSnakes.Runtime/Python/PyEnumerable.cs b/src/CSnakes.Runtime/Python/PyEnumerable.cs index f9fb1257..d1d922b3 100644 --- a/src/CSnakes.Runtime/Python/PyEnumerable.cs +++ b/src/CSnakes.Runtime/Python/PyEnumerable.cs @@ -1,58 +1,51 @@ -using CSnakes.Runtime.CPython; -using System.Collections; - -namespace CSnakes.Runtime.Python; - -internal class PyEnumerable : IEnumerable, IEnumerator, IDisposable -{ - private readonly PyObject _pyIterator; - private TValue current = default!; - - internal PyEnumerable(PyObject pyObject) - { - using (GIL.Acquire()) - { - _pyIterator = pyObject.GetIter(); - } - } - - public TValue Current => current; - - object IEnumerator.Current => current!; - - public void Dispose() - { - _pyIterator.Dispose(); - } - - public IEnumerator GetEnumerator() => this; - - public bool MoveNext() - { - using (GIL.Acquire()) - { - nint result = CPythonAPI.PyIter_Next(_pyIterator); - if (result == IntPtr.Zero && CPythonAPI.PyErr_Occurred()) - { - throw PyObject.ThrowPythonExceptionAsClrException(); - } - else if (result == IntPtr.Zero) - { - return false; - } - else - { - using PyObject pyObject = PyObject.Create(result); - current = pyObject.As(); - return true; - } - } - } - - public void Reset() - { - throw new NotSupportedException("Python iterators cannot be reset"); - } - - IEnumerator IEnumerable.GetEnumerator() => this; -} +using CSnakes.Runtime.CPython; +using System.Collections; + +namespace CSnakes.Runtime.Python; + +internal class PyEnumerable : IEnumerable, IEnumerator, IDisposable +{ + private readonly PyObject _pyIterator; + private TValue current = default!; + + internal PyEnumerable(PyObject pyObject) + { + using (GIL.Acquire()) + { + _pyIterator = pyObject.GetIter(); + } + } + + public TValue Current => current; + + object IEnumerator.Current => current!; + + public void Dispose() => _pyIterator.Dispose(); + + public IEnumerator GetEnumerator() => this; + + public bool MoveNext() + { + using (GIL.Acquire()) + { + nint result = CPythonAPI.PyIter_Next(_pyIterator); + if (result == IntPtr.Zero && CPythonAPI.PyErr_Occurred()) + { + throw PyObject.ThrowPythonExceptionAsClrException(); + } + + if (result == IntPtr.Zero) + { + return false; + } + + using PyObject pyObject = PyObject.Create(result); + current = pyObject.As(); + return true; + } + } + + public void Reset() => throw new NotSupportedException("Python iterators cannot be reset"); + + IEnumerator IEnumerable.GetEnumerator() => this; +} diff --git a/src/CSnakes.Runtime/Python/PyKeyValuePairEnumerable.cs b/src/CSnakes.Runtime/Python/PyKeyValuePairEnumerable.cs index 8ecd7e11..40a641d3 100644 --- a/src/CSnakes.Runtime/Python/PyKeyValuePairEnumerable.cs +++ b/src/CSnakes.Runtime/Python/PyKeyValuePairEnumerable.cs @@ -1,58 +1,51 @@ -using CSnakes.Runtime.CPython; -using System.Collections; - -namespace CSnakes.Runtime.Python; - -internal class PyKeyValuePairEnumerable : IEnumerable>, IEnumerator> -{ - private readonly PyObject _pyIterator; - private KeyValuePair current = default!; - - internal PyKeyValuePairEnumerable(PyObject pyObject) - { - using (GIL.Acquire()) - { - _pyIterator = pyObject.GetIter(); - } - } - - public KeyValuePair Current => current; - - object IEnumerator.Current => current!; - - public void Dispose() - { - _pyIterator.Dispose(); - } - - public IEnumerator> GetEnumerator() => this; - - public bool MoveNext() - { - using (GIL.Acquire()) - { - nint result = CPythonAPI.PyIter_Next(_pyIterator); - if (result == IntPtr.Zero && CPythonAPI.PyErr_Occurred()) - { - throw PyObject.ThrowPythonExceptionAsClrException(); - } - else if (result == IntPtr.Zero) - { - return false; - } - else - { - using PyObject pyObject = PyObject.Create(result); - current = pyObject.As(); - return true; - } - } - } - - public void Reset() - { - throw new NotSupportedException("Python iterators cannot be reset"); - } - - IEnumerator IEnumerable.GetEnumerator() => this; -} +using CSnakes.Runtime.CPython; +using System.Collections; + +namespace CSnakes.Runtime.Python; + +internal class PyKeyValuePairEnumerable : IEnumerable>, IEnumerator> +{ + private readonly PyObject _pyIterator; + private KeyValuePair current = default!; + + internal PyKeyValuePairEnumerable(PyObject pyObject) + { + using (GIL.Acquire()) + { + _pyIterator = pyObject.GetIter(); + } + } + + public KeyValuePair Current => current; + + object IEnumerator.Current => current!; + + public void Dispose() => _pyIterator.Dispose(); + + public IEnumerator> GetEnumerator() => this; + + public bool MoveNext() + { + using (GIL.Acquire()) + { + nint result = CPythonAPI.PyIter_Next(_pyIterator); + if (result == IntPtr.Zero && CPythonAPI.PyErr_Occurred()) + { + throw PyObject.ThrowPythonExceptionAsClrException(); + } + + if (result == IntPtr.Zero) + { + return false; + } + + using PyObject pyObject = PyObject.Create(result); + current = pyObject.As(); + return true; + } + } + + public void Reset() => throw new NotSupportedException("Python iterators cannot be reset"); + + IEnumerator IEnumerable.GetEnumerator() => this; +} diff --git a/src/CSnakes.Runtime/Python/PyList.cs b/src/CSnakes.Runtime/Python/PyList.cs index 94e0f3f2..2d55d762 100644 --- a/src/CSnakes.Runtime/Python/PyList.cs +++ b/src/CSnakes.Runtime/Python/PyList.cs @@ -1,60 +1,57 @@ -using CSnakes.Runtime.CPython; -using System.Collections; - -namespace CSnakes.Runtime.Python; - -internal class PyList : IReadOnlyList, IDisposable -{ - private readonly PyObject _listObject; - - // If someone fetches the same index multiple times, we cache the result to avoid multiple round trips to Python - private readonly Dictionary _convertedItems = new(); - - public PyList(PyObject listObject) => _listObject = listObject; - - public TItem this[int index] - { - get - { - if (_convertedItems.TryGetValue(index, out TItem cachedValue)) - { - return cachedValue; - } - - using (GIL.Acquire()) - { - using PyObject value = PyObject.Create(CPythonAPI.PySequence_GetItem(_listObject, index)); - TItem result = value.As(); - _convertedItems[index] = result; - return result; - } - } - } - - public int Count - { - get - { - using (GIL.Acquire()) - { - return (int)CPythonAPI.PySequence_Size(_listObject); - } - } - } - - public void Dispose() - { - _listObject.Dispose(); - } - - public IEnumerator GetEnumerator() - { - // TODO: If someone fetches the same index multiple times, we cache the result to avoid multiple round trips to Python - using (GIL.Acquire()) - { - return new PyEnumerable(_listObject); - } - } - - IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); -} +using CSnakes.Runtime.CPython; +using System.Collections; + +namespace CSnakes.Runtime.Python; + +internal class PyList : IReadOnlyList, IDisposable +{ + private readonly PyObject _listObject; + + // If someone fetches the same index multiple times, we cache the result to avoid multiple round trips to Python + private readonly Dictionary _convertedItems = new(); + + public PyList(PyObject listObject) => _listObject = listObject; + + public TItem this[int index] + { + get + { + if (_convertedItems.TryGetValue(index, out TItem? cachedValue)) + { + return cachedValue; + } + + using (GIL.Acquire()) + { + using PyObject value = PyObject.Create(CPythonAPI.PySequence_GetItem(_listObject, index)); + TItem result = value.As(); + _convertedItems[index] = result; + return result; + } + } + } + + public int Count + { + get + { + using (GIL.Acquire()) + { + return (int)CPythonAPI.PySequence_Size(_listObject); + } + } + } + + public void Dispose() => _listObject.Dispose(); + + public IEnumerator GetEnumerator() + { + // TODO: If someone fetches the same index multiple times, we cache the result to avoid multiple round trips to Python + using (GIL.Acquire()) + { + return new PyEnumerable(_listObject); + } + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); +} diff --git a/src/CSnakes.Runtime/Python/PyObject.cs b/src/CSnakes.Runtime/Python/PyObject.cs index 7dec158f..d5d4ab45 100644 --- a/src/CSnakes.Runtime/Python/PyObject.cs +++ b/src/CSnakes.Runtime/Python/PyObject.cs @@ -48,7 +48,8 @@ protected override bool ReleaseHandle() { CPythonAPI.Py_DecRefRaw(handle); } - } else + } + else { // Probably in the GC finalizer thread, instead of causing GIL contention, put this on a queue to be processed later. GIL.QueueForDisposal(handle); @@ -149,7 +150,7 @@ public virtual bool HasAttr(string name) } } - + internal virtual PyObject GetIter() { RaiseOnPythonNotInitialized(); @@ -455,7 +456,8 @@ public override string ToString() /// The type of the key /// The type of the value /// - public KeyValuePair As() { + public KeyValuePair As() + { using (GIL.Acquire()) { return PyObjectTypeConverter.ConvertToKeyValuePair(this);