feat: iterate over more than one prefix (#109)

.gitignore

@@ -12,3 +12,4 @@ dist
 .settings
 .cproject
 .autotools
+.*.swp

src/ijson/__init__.py

@@ -49,11 +49,13 @@ def _default_backend():
 parse_coro = backend.parse_coro
 items = backend.items
 items_coro = backend.items_coro
+prefixed_items = backend.prefixed_items
+prefixed_items_coro = backend.prefixed_items_coro
 kvitems = backend.kvitems
 kvitems_coro = backend.kvitems_coro
 basic_parse_async = backend.basic_parse_async
 parse_async = backend.parse_async
 items_async = backend.items_async
 kvitems_async = backend.kvitems_async
 backend_name = backend.backend_name
-backend = backend.backend
+backend = backend.backend

src/ijson/backends/ext/_yajl2/items_basecoro.c

@@ -19,6 +19,7 @@ static int items_basecoro_init(ItemsBasecoro *self, PyObject *args, PyObject *kw
 	self->target_send = NULL;
 	self->prefix = NULL;
 	self->object_depth = 0;
+	self->pending_builder_reset = 0;
 	M1_N(self->module_state = get_state_from_imported_module());
 	builder_create(&self->builder);
 
@@ -42,35 +43,13 @@ static void items_basecoro_dealloc(ItemsBasecoro *self)
 PyObject* items_basecoro_send_impl(PyObject *self, PyObject *path, PyObject *event, PyObject *value)
 {
 	ItemsBasecoro *coro = (ItemsBasecoro *)self;
-	enames_t enames = coro->module_state->enames;
-
-	if (builder_isactive(&coro->builder)) {
-		coro->object_depth += (event == enames.start_map_ename || event == enames.start_array_ename);
-		coro->object_depth -= (event == enames.end_map_ename || event == enames.end_array_ename);
-		if (coro->object_depth > 0) {
-			N_M1( builder_event(&coro->builder, enames, event, value) );
-		}
-		else {
-			PyObject *retval = builder_value(&coro->builder);
-			CORO_SEND(coro->target_send, retval);
-			Py_DECREF(retval);
-			N_M1(builder_reset(&coro->builder));
-		}
-	}
-	else {
-		int cmp = PyObject_RichCompareBool(path, coro->prefix, Py_EQ);
-		N_M1(cmp);
-		if (cmp) {
-			if (event == enames.start_map_ename || event == enames.start_array_ename) {
-				coro->object_depth = 1;
-				N_M1(builder_event(&coro->builder, enames, event, value));
-			}
-			else {
-				CORO_SEND(coro->target_send, value);
-			}
-		}
-	}
 
+	PyObject *retval;
+	N_N(retval = items_common_send_top(self, path, event, value));
+	if (retval == Py_None)
+		Py_RETURN_NONE;
+	CORO_SEND(coro->target_send, retval);
+	items_common_send_bottom(self, retval);
 	Py_RETURN_NONE;
 }
 

src/ijson/backends/ext/_yajl2/items_basecoro.h

@@ -11,20 +11,17 @@
 #ifndef ITEMS_BASECORO_H
 #define ITEMS_BASECORO_H
 
-#include "builder.h"
-#include "module_state.h"
+#define SPECIFIC_ITEMS_CMP(prefix, path) \
+	PyObject_RichCompareBool((path), (prefix), Py_EQ)
+
+#include "items_common.h"
+
+#undef SPECIFIC_ITEMS_CMP
 
 /**
  * items_basecoro coroutine object structure
  */
-typedef struct {
-    PyObject_HEAD
-    builder_t builder;
-    PyObject *target_send;
-    PyObject *prefix;
-    int object_depth;
-    yajl2_state *module_state;
-} ItemsBasecoro;
+typedef ItemsCommonBasecoro ItemsBasecoro;
 
 /**
  * items_basecoro coroutine object type
@@ -47,4 +44,4 @@ extern PyTypeObject ItemsBasecoro_Type;
  */
 PyObject* items_basecoro_send_impl(PyObject *self, PyObject *path, PyObject *event, PyObject *value);
 
-#endif // ITEMS_BASECORO_H
+#endif // ITEMS_BASECORO_H

src/ijson/backends/ext/_yajl2/items_common.h

@@ -0,0 +1,68 @@
+#ifndef ITEMS_COMMON_H
+#define ITEMS_COMMON_H
+
+#include "builder.h"
+#include "module_state.h"
+
+/**
+ * common items/prefixed_items_basecoro coroutine object structure
+ */
+typedef struct {
+	PyObject_HEAD
+	builder_t builder;
+	int pending_builder_reset;
+	PyObject *target_send;
+	PyObject *prefix;
+	int object_depth;
+	yajl2_state *module_state;
+} ItemsCommonBasecoro;
+
+static inline
+PyObject* items_common_send_top(PyObject *self, PyObject *path, PyObject *event, PyObject *value)
+{
+	ItemsCommonBasecoro *coro = (ItemsCommonBasecoro *)self;
+	enames_t enames = coro->module_state->enames;
+
+	if (builder_isactive(&coro->builder)) {
+		coro->object_depth += (event == enames.start_map_ename || event == enames.start_array_ename);
+		coro->object_depth -= (event == enames.end_map_ename || event == enames.end_array_ename);
+		if (coro->object_depth > 0) {
+			N_M1( builder_event(&coro->builder, enames, event, value) );
+		}
+		else {
+			PyObject *retval = builder_value(&coro->builder);
+			coro->pending_builder_reset = 1;
+			return retval;
+		}
+	}
+	else {
+		int cmp = SPECIFIC_ITEMS_CMP(coro->prefix, path);
+		N_M1(cmp);
+		if (cmp) {
+			if (event == enames.start_map_ename || event == enames.start_array_ename) {
+				coro->object_depth = 1;
+				N_M1(builder_event(&coro->builder, enames, event, value));
+			}
+			else {
+				Py_INCREF(value);
+				return value;
+			}
+		}
+	}
+
+	Py_RETURN_NONE;
+}
+
+static inline
+void items_common_send_bottom(PyObject *self, PyObject *retval)
+{
+	ItemsCommonBasecoro *coro = (ItemsCommonBasecoro *)self;
+
+	Py_DECREF(retval);
+	if (coro->pending_builder_reset) {
+		builder_reset(&coro->builder);
+		coro->pending_builder_reset = 0;
+	}
+}
+
+#endif // ITEMS_COMMON_H

src/ijson/backends/ext/_yajl2/module.c

@@ -20,6 +20,9 @@
 #include "items.h"
 #include "items_async.h"
 #include "items_basecoro.h"
+#include "prefixed_items.h"
+#include "prefixed_items_async.h"
+#include "prefixed_items_basecoro.h"
 #include "kvitems.h"
 #include "kvitems_async.h"
 #include "kvitems_basecoro.h"
@@ -114,11 +117,14 @@ static int _yajl2_mod_exec(PyObject *m)
 	ADD_TYPE("kvitems", KVItemsGen_Type);
 	ADD_TYPE("items_basecoro", ItemsBasecoro_Type);
 	ADD_TYPE("items", ItemsGen_Type);
+	ADD_TYPE("prefixed_items_basecoro", PrefixedItemsBasecoro_Type);
+	ADD_TYPE("prefixed_items", PrefixedItemsGen_Type);
 	ADD_TYPE("_async_reading_iterator", AsyncReadingGeneratorType);
 	ADD_TYPE("basic_parse_async", BasicParseAsync_Type);
 	ADD_TYPE("parse_async", ParseAsync_Type);
 	ADD_TYPE("kvitems_async", KVItemsAsync_Type);
 	ADD_TYPE("items_async", ItemsAsync_Type);
+	ADD_TYPE("prefixed_items_async", PrefixedItemsAsync_Type);
 
 	yajl2_state *state;
 	M1_N(state = get_state(m));
@@ -153,4 +159,4 @@ static int _yajl2_mod_exec(PyObject *m)
 	M1_N(state->Decimal);
 
 	return 0;
-}
+}

src/ijson/backends/ext/_yajl2/parse_basecoro.c

@@ -10,6 +10,7 @@
 
 #include "common.h"
 #include "items_basecoro.h"
+#include "prefixed_items_basecoro.h"
 #include "kvitems_basecoro.h"
 #include "parse_basecoro.h"
 
@@ -116,6 +117,9 @@ PyObject* parse_basecoro_send_impl(PyObject *self, PyObject *event, PyObject *va
 	else if (ItemsBasecoro_Check(gen->target_send)) {
 		items_basecoro_send_impl(gen->target_send, prefix, event, value);
 	}
+	else if (PrefixedItemsBasecoro_Check(gen->target_send)) {
+		prefixed_items_basecoro_send_impl(gen->target_send, prefix, event, value);
+	}
 	else {
 		PyObject *res = PyTuple_Pack(3, prefix, event, value);
 		CORO_SEND(gen->target_send, res);

src/ijson/backends/ext/_yajl2/prefixed_items.c

@@ -0,0 +1,61 @@
+/*
+ * items generator implementation for ijson's C backend
+ *
+ * Contributed by Rodrigo Tobar <[email protected]>
+ *
+ * ICRAR - International Centre for Radio Astronomy Research
+ * (c) UWA - The University of Western Australia, 2020
+ * Copyright by UWA (in the framework of the ICRAR)
+ */
+
+#include "common.h"
+#include "prefixed_items.h"
+#include "prefixed_items_basecoro.h"
+#include "parse_basecoro.h"
+#include "basic_parse_basecoro.h"
+
+/*
+ * __init__, destructor, __iter__ and __next__
+ */
+static int prefixed_itemsgen_init(PrefixedItemsGen *self, PyObject *args, PyObject *kwargs)
+{
+	PyObject *reading_args = PySequence_GetSlice(args, 0, 2);
+	PyObject *items_args = PySequence_GetSlice(args, 2, 4);
+	pipeline_node coro_pipeline[] = {
+		{&PrefixedItemsBasecoro_Type, items_args, NULL},
+		{&ParseBasecoro_Type, NULL, NULL},
+		{&BasicParseBasecoro_Type, NULL, kwargs},
+		{NULL}
+	};
+	int res = reading_generator_init(&self->reading_gen, reading_args, coro_pipeline);
+	Py_DECREF(items_args);
+	Py_DECREF(reading_args);
+	return res;
+}
+
+static void prefixed_itemsgen_dealloc(PrefixedItemsGen *self)
+{
+	reading_generator_dealloc(&self->reading_gen);
+	Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+static PyObject* prefixed_itemsgen_iternext(PyObject *self)
+{
+	PrefixedItemsGen *gen = (PrefixedItemsGen *)self;
+	return reading_generator_next(&gen->reading_gen);
+}
+
+/*
+ * items generator object type
+ */
+PyTypeObject PrefixedItemsGen_Type = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_basicsize = sizeof(PrefixedItemsGen),
+	.tp_name = "_yajl2.prefixed_items",
+	.tp_doc = "Generates items",
+	.tp_init = (initproc)prefixed_itemsgen_init,
+	.tp_dealloc = (destructor)prefixed_itemsgen_dealloc,
+	.tp_flags = Py_TPFLAGS_DEFAULT,
+	.tp_iter = ijson_return_self,
+	.tp_iternext = prefixed_itemsgen_iternext
+};

src/ijson/backends/ext/_yajl2/prefixed_items.h

@@ -0,0 +1,30 @@
+/*
+ * items generator for ijson's C backend
+ *
+ * Contributed by Rodrigo Tobar <[email protected]>
+ *
+ * ICRAR - International Centre for Radio Astronomy Research
+ * (c) UWA - The University of Western Australia, 2020
+ * Copyright by UWA (in the framework of the ICRAR)
+ */
+
+#ifndef PREFIXED_ITEMS_H
+#define PREFIXED_ITEMS_H
+
+#include "reading_generator.h"
+
+/**
+ * items generator object structure
+ */
+typedef struct {
+	PyObject_HEAD
+	reading_generator_t reading_gen;
+} PrefixedItemsGen;
+
+
+/**
+ * items generator object type
+ */
+extern PyTypeObject PrefixedItemsGen_Type;
+
+#endif /* PREFIXED_ITEMS_H */

src/ijson/backends/ext/_yajl2/prefixed_items_async.c

@@ -0,0 +1,74 @@
+/*
+ * items_async asynchronous iterable implementation for ijson's C backend
+ *
+ * Contributed by Rodrigo Tobar <[email protected]>
+ *
+ * ICRAR - International Centre for Radio Astronomy Research
+ * (c) UWA - The University of Western Australia, 2020
+ * Copyright by UWA (in the framework of the ICRAR)
+ */
+
+
+#include "async_reading_generator.h"
+#include "basic_parse_basecoro.h"
+#include "parse_basecoro.h"
+#include "prefixed_items_basecoro.h"
+#include "common.h"
+#include "coro_utils.h"
+
+/**
+ * items_async asynchronous iterable object structure
+ */
+typedef struct {
+	PyObject_HEAD
+	async_reading_generator *reading_generator;
+} PrefixedItemsAsync;
+
+/*
+ * __init__, destructor and __anext__
+ */
+static int prefixed_itemsasync_init(PrefixedItemsAsync *self, PyObject *args, PyObject *kwargs)
+{
+	PyObject *reading_args = PySequence_GetSlice(args, 0, 2);
+	PyObject *items_args = PySequence_GetSlice(args, 2, 4);
+	pipeline_node coro_pipeline[] = {
+		{&PrefixedItemsBasecoro_Type, items_args, NULL},
+		{&ParseBasecoro_Type, NULL, NULL},
+		{&BasicParseBasecoro_Type, NULL, kwargs},
+		{NULL}
+	};
+	M1_N(self->reading_generator = (async_reading_generator *)PyObject_CallObject((PyObject *)&AsyncReadingGeneratorType, reading_args));
+	int ret = async_reading_generator_add_coro(self->reading_generator, coro_pipeline);
+	Py_DECREF(items_args);
+	Py_DECREF(reading_args);
+	return ret;
+}
+
+static void prefixed_itemsasync_dealloc(PrefixedItemsAsync *self) {
+	Py_XDECREF(self->reading_generator);
+	Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+static PyObject *prefixed_itemsasync_anext(PyObject *self)
+{
+	PrefixedItemsAsync *gen = (PrefixedItemsAsync *)self;
+	Py_INCREF(gen->reading_generator);
+	return (PyObject *)gen->reading_generator;
+}
+
+static PyAsyncMethods prefixed_itemsasync_methods = {
+	.am_await = ijson_return_self,
+	.am_aiter = ijson_return_self,
+	.am_anext = prefixed_itemsasync_anext
+};
+
+PyTypeObject PrefixedItemsAsync_Type = {
+	PyVarObject_HEAD_INIT(NULL, 0)
+	.tp_basicsize = sizeof(PrefixedItemsAsync),
+	.tp_name = "_yajl2._prefixed_items_async",
+	.tp_doc = "Asynchronous iterable yielding fully-built items",
+	.tp_init = (initproc)prefixed_itemsasync_init,
+	.tp_dealloc = (destructor)prefixed_itemsasync_dealloc,
+	.tp_flags = Py_TPFLAGS_DEFAULT,
+	.tp_as_async = &prefixed_itemsasync_methods
+};