Solve bug in c loader, improved python port add reference and dereference APIs in reflect.

Author: viferga

source/loaders/c_loader/source/c_loader_impl.cpp

@@ -826,9 +826,9 @@ int c_loader_impl_initialize_types(loader_impl impl)
 		{ TYPE_FLOAT, "float" },
 		{ TYPE_DOUBLE, "double" },
 
-		{ TYPE_INVALID, "void" }
+		{ TYPE_NULL, "void" }
 
-		/* TODO: Do more types (and the unsigned versions too?) */
+		/* TODO: Do more types */
 	};
 
 	size_t size = sizeof(type_id_name_pair) / sizeof(type_id_name_pair[0]);
@@ -956,7 +956,7 @@ static type_id c_loader_impl_clang_type(loader_impl impl, CXCursor cursor, CXTyp
 			return TYPE_INT;
 
 		case CXType_Void:
-			return TYPE_INVALID;
+			return TYPE_NULL;
 
 		case CXType_Enum: {
 			CXCursor referenced = clang_isReference(cursor.kind) ? clang_getCursorReferenced(cursor) : cursor;

source/loaders/py_loader/include/py_loader/py_loader_impl.h

@@ -60,6 +60,8 @@ PY_LOADER_NO_EXPORT PyObject *py_loader_impl_value_to_capi(loader_impl impl, typ
 
 PY_LOADER_NO_EXPORT int py_loader_impl_finalizer_object(loader_impl impl, PyObject *obj, value v);
 
+PY_LOADER_NO_EXPORT PyObject *py_loader_impl_capsule_new_null(void);
+
 #ifdef __cplusplus
 }
 #endif

source/loaders/py_loader/source/py_loader_impl.c

@@ -221,6 +221,9 @@ static char *py_loader_impl_main_module = NULL;
 /* Holds reference to the original PyCFunction.tp_dealloc method */
 static void (*py_loader_impl_pycfunction_dealloc)(PyObject *) = NULL;
 
+/* Implements PyCapsules with null value internally */
+static const char py_loader_capsule_null_id[] = "__metacall_capsule_null__";
+
 PyObject *py_loader_impl_finalizer_object_impl(PyObject *self, PyObject *Py_UNUSED(args))
 {
 	value v = PyCapsule_GetPointer(self, NULL);
@@ -273,6 +276,14 @@ int py_loader_impl_finalizer_object(loader_impl impl, PyObject *obj, value v)
 	return 0;
 }
 
+PyObject *py_loader_impl_capsule_new_null(void)
+{
+	/* We want to create a new capsule with contents set to NULL, but PyCapsule
+	* does not allow that, instead we are going to identify our NULL capsule with
+	* this configuration (setting the capsule to Py_None) */
+	return PyCapsule_New(Py_None, py_loader_capsule_null_id, NULL);
+}
+
 void py_loader_impl_value_invoke_state_finalize(value v, void *data)
 {
 	PyObject *capsule = (PyObject *)data;
@@ -1142,17 +1153,17 @@ value py_loader_impl_capi_to_value(loader_impl impl, PyObject *obj, type_id id)
 	}
 	else if (id == TYPE_PTR)
 	{
-		void *ptr = NULL;
-
-#if PY_MAJOR_VERSION == 2
+		const char *name = PyCapsule_GetName(obj);
+		void *ptr = PyCapsule_GetPointer(obj, name);
 
-		/* TODO */
-
-#elif PY_MAJOR_VERSION == 3
-		ptr = PyCapsule_GetPointer(obj, NULL);
-
-		v = value_create_ptr(ptr);
-#endif
+		if (ptr == Py_None && name == py_loader_capsule_null_id)
+		{
+			v = value_create_ptr(NULL);
+		}
+		else
+		{
+			v = value_create_ptr(ptr);
+		}
 	}
 	else if (id == TYPE_FUNCTION)
 	{
@@ -1440,13 +1451,12 @@ PyObject *py_loader_impl_value_to_capi(loader_impl impl, type_id id, value v)
 	{
 		void *ptr = value_to_ptr(v);
 
-#if PY_MAJOR_VERSION == 2
-
-		/* TODO */
+		if (ptr == NULL)
+		{
+			return py_loader_impl_capsule_new_null();
+		}
 
-#elif PY_MAJOR_VERSION == 3
 		return PyCapsule_New(ptr, NULL, NULL);
-#endif
 	}
 	else if (id == TYPE_FUTURE)
 	{

source/loaders/py_loader/source/py_loader_port.c

@@ -493,6 +493,7 @@ static PyObject *py_loader_port_invoke(PyObject *self, PyObject *var_args)
 	/* Obtain Python loader implementation */
 	impl = loader_get_impl(py_loader_tag);
 
+	/* TODO: Remove this check when we implement this: https://github.com/metacall/core/issues/231 */
 	if (impl == NULL)
 	{
 		PyErr_SetString(PyExc_ValueError, "Invalid Python loader instance, MetaCall Port must be used from MetaCall CLI");
@@ -622,6 +623,7 @@ static PyObject *py_loader_port_await(PyObject *self, PyObject *var_args)
 	/* Obtain Python loader implementation */
 	impl = loader_get_impl(py_loader_tag);
 
+	/* TODO: Remove this check when we implement this: https://github.com/metacall/core/issues/231 */
 	if (impl == NULL)
 	{
 		PyErr_SetString(PyExc_ValueError, "Invalid Python loader instance, MetaCall Port must be used from MetaCall CLI");
@@ -778,6 +780,184 @@ static PyObject *py_loader_port_inspect(PyObject *self, PyObject *args)
 	return result;
 }
 
+static PyObject *py_loader_port_value_create_ptr(PyObject *self, PyObject *args)
+{
+	static const char format[] = "O:metacall_value_create_ptr";
+	PyObject *pointer;
+
+	(void)self;
+
+	/* Parse arguments */
+	if (!PyArg_ParseTuple(args, (char *)format, &pointer))
+	{
+		PyErr_SetString(PyExc_TypeError, "Invalid number of arguments, use it like: metacall_value_create_ptr(None); or metacall_value_create_ptr(previous_allocated_ptr);");
+		return py_loader_port_none();
+	}
+
+	if (!PyCapsule_CheckExact(pointer) && pointer != Py_None)
+	{
+		PyErr_SetString(PyExc_TypeError, "Invalid parameter type in first argument must be None or a PyCapsule (i.e a previously allocated pointer)");
+		return py_loader_port_none();
+	}
+
+	if (pointer == Py_None)
+	{
+		return py_loader_impl_capsule_new_null();
+	}
+	else
+	{
+		/* Get capsule pointer */
+		const char *name = PyCapsule_GetName(pointer);
+		void *pointer_addr = PyCapsule_GetPointer(pointer, name);
+
+		/* Return a copy of the capsule */
+		return PyCapsule_New(pointer_addr, name, NULL);
+	}
+}
+
+static const char py_loader_capsule_reference_id[] = "__metacall_capsule_reference__";
+
+static void py_loader_port_value_reference_destroy(PyObject *capsule)
+{
+	void *ref = PyCapsule_GetPointer(capsule, py_loader_capsule_reference_id);
+	void *v = PyCapsule_GetContext(capsule);
+
+	metacall_value_destroy(ref);
+	metacall_value_destroy(v);
+}
+
+static PyObject *py_loader_port_value_reference(PyObject *self, PyObject *args)
+{
+	static const char format[] = "O:metacall_value_reference";
+	PyObject *obj;
+	loader_impl impl;
+	void *v, *ref;
+	PyObject *capsule;
+
+	(void)self;
+
+	/* Parse arguments */
+	if (!PyArg_ParseTuple(args, (char *)format, &obj))
+	{
+		PyErr_SetString(PyExc_TypeError, "Invalid number of arguments, use it like: metacall_value_reference(obj);");
+		goto error_none;
+	}
+
+	/* Obtain Python loader implementation */
+	impl = loader_get_impl(py_loader_tag);
+
+	/* TODO: When using the port outside MetaCall this is going to segfault for functions and similar
+	* structures that require py loader internal structure to be initialized. For those cases, we
+	* must implement this: https://github.com/metacall/core/issues/231
+	*/
+	v = py_loader_impl_capi_to_value(impl, obj, py_loader_impl_capi_to_value_type(impl, obj));
+
+	if (v == NULL)
+	{
+		PyErr_SetString(PyExc_ValueError, "Failed to convert the Python object to MetaCall value.");
+		goto error_none;
+	}
+
+	ref = metacall_value_reference(v);
+
+	if (ref == NULL)
+	{
+		PyErr_SetString(PyExc_ValueError, "Failed to create the reference from MetaCall value.");
+		goto error_value;
+	}
+
+	capsule = PyCapsule_New(ref, py_loader_capsule_reference_id, &py_loader_port_value_reference_destroy);
+
+	if (capsule == NULL)
+	{
+		goto error_ref;
+	}
+
+	if (PyCapsule_SetContext(capsule, v) != 0)
+	{
+		goto error_ref;
+	}
+
+	return capsule;
+
+error_ref:
+	metacall_value_destroy(ref);
+error_value:
+	metacall_value_destroy(v);
+error_none:
+	return py_loader_port_none();
+}
+
+static PyObject *py_loader_port_value_dereference(PyObject *self, PyObject *args)
+{
+	static const char format[] = "O:metacall_value_dereference";
+	PyObject *capsule;
+	const char *name = NULL;
+	void *ref, *v;
+	loader_impl impl;
+	PyObject *result;
+
+	(void)self;
+
+	/* Parse arguments */
+	if (!PyArg_ParseTuple(args, (char *)format, &capsule))
+	{
+		PyErr_SetString(PyExc_TypeError, "Invalid number of arguments, use it like: metacall_value_dereference(ptr);");
+		return py_loader_port_none();
+	}
+
+	/* Check if it is a valid reference */
+	if (!PyCapsule_CheckExact(capsule))
+	{
+		PyErr_SetString(PyExc_TypeError, "Invalid parameter type in first argument must be a PyCapsule (i.e a previously allocated pointer)");
+		return py_loader_port_none();
+	}
+
+	/* Check if it is a valid MetaCall reference */
+	name = PyCapsule_GetName(capsule);
+
+	if (name != py_loader_capsule_reference_id)
+	{
+		PyErr_SetString(PyExc_TypeError, "Invalid reference, argument must be a PyCapsule from MetaCall");
+		return py_loader_port_none();
+	}
+
+	/* Get the reference */
+	ref = PyCapsule_GetPointer(capsule, name);
+
+	if (ref == NULL)
+	{
+		return py_loader_port_none();
+	}
+
+	/* Get the value */
+	v = metacall_value_dereference(ref);
+
+	/* Validate the result */
+	if (v != PyCapsule_GetContext(capsule))
+	{
+		PyErr_SetString(PyExc_TypeError, "Invalid reference, the PyCapsule context does not match the dereferenced value");
+		return py_loader_port_none();
+	}
+
+	/* Obtain Python loader implementation */
+	impl = loader_get_impl(py_loader_tag);
+
+	/* TODO: When using the port outside MetaCall this is going to segfault for functions and similar
+	* structures that require py loader internal structure to be initialized. For those cases, we
+	* must implement this: https://github.com/metacall/core/issues/231
+	*/
+	result = py_loader_impl_value_to_capi(impl, value_type_id(v), v);
+
+	if (result == NULL)
+	{
+		PyErr_SetString(PyExc_ValueError, "Failed to convert the MetaCall value to Python object.");
+		return py_loader_port_none();
+	}
+
+	return result;
+}
+
 static PyMethodDef metacall_methods[] = {
 	{ "metacall_load_from_file", py_loader_port_load_from_file, METH_VARARGS,
 		"Loads a script from file." },
@@ -793,6 +973,12 @@ static PyMethodDef metacall_methods[] = {
 		"Get information about all loaded objects." },
 	{ "metacall", py_loader_port_invoke, METH_VARARGS,
 		"Call a function anonymously." },
+	{ "metacall_value_create_ptr", py_loader_port_value_create_ptr, METH_VARARGS,
+		"Create a new value of type Pointer." },
+	{ "metacall_value_reference", py_loader_port_value_reference, METH_VARARGS,
+		"Create a new value of type Pointer." },
+	{ "metacall_value_dereference", py_loader_port_value_dereference, METH_VARARGS,
+		"Get the data which a value of type Pointer is pointing to." },
 	{ NULL, NULL, 0, NULL }
 };
 

source/metacall/include/metacall/metacall_value.h

@@ -394,6 +394,48 @@ METACALL_API const char *metacall_value_type_name(void *v);
 */
 METACALL_API void *metacall_value_copy(void *v);
 
+/**
+*  @brief
+*    Creates a new pointer value, with a reference to the
+*    data contained inside the value @v. For example:
+*
+*    void *v = metacall_value_create_int(20);
+*    void *ptr = metacall_value_reference(v);
+*
+*    In this case, void *ptr is a value equivalent to int*,
+*    and it points directly to the integer contained in void *v.
+*    Note that if we destroy the value @v, the reference will
+*    point to already freed memory, causing use-after-free when used.
+*
+*  @param[in] v
+*    Reference to the value to be referenced
+*
+*  @return
+*    A new value of type pointer, pointing to the @v data
+*/
+METACALL_API void *metacall_value_reference(void *v);
+
+/**
+*  @brief
+*    If you pass a reference previously created (i.e a value of
+*    type pointer, pointing to another value), it returns the
+*    original value. It does not modify the memory of the values
+*    neither allocates anything. If the value @v is pointing to
+*    has been deleted, it will cause an use-after-free. For example:
+*
+*    void *v = metacall_value_create_int(20);
+*    void *ptr = metacall_value_reference(v);
+*    void *w = metacall_value_dereference(ptr);
+*    assert(v == w); // Both are the same value
+*
+*  @param[in] v
+*    Reference to the value to be dereferenced
+*
+*  @return
+*    The value containing the data which ptr is pointing to
+*/
+METACALL_API void *metacall_value_dereference(void *v);
+
 /**
 *  @brief
 *    Copies the ownership from @src to @dst, including the finalizer,

source/metacall/source/metacall_value.c

@@ -223,6 +223,16 @@ void *metacall_value_copy(void *v)
 	return value_type_copy(v);
 }
 
+void *metacall_value_reference(void *v)
+{
+	return value_type_reference(v);
+}
+
+void *metacall_value_dereference(void *v)
+{
+	return value_type_dereference(v);
+}
+
 void metacall_value_move(void *src, void *dst)
 {
 	value_move(src, dst);

source/ports/py_port/metacall/__init__.py

@@ -17,4 +17,4 @@
 #	See the License for the specific language governing permissions and
 #	limitations under the License.
 
-from metacall.api import metacall, metacall_load_from_file, metacall_load_from_memory, metacall_load_from_package, metacall_inspect
+from metacall.api import metacall, metacall_load_from_file, metacall_load_from_memory, metacall_load_from_package, metacall_inspect, metacall_value_create_ptr, metacall_value_reference, metacall_value_dereference