diff --git a/.gitignore b/.gitignore index d35b1d2..4c8940d 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ dist .settings .cproject .autotools +.*.swp diff --git a/src/ijson/__init__.py b/src/ijson/__init__.py index 9f7c679..4b98686 100644 --- a/src/ijson/__init__.py +++ b/src/ijson/__init__.py @@ -49,6 +49,8 @@ def _default_backend(): parse_coro = backend.parse_coro items = backend.items items_coro = backend.items_coro +prefixed_items = backend.prefixed_items +prefixed_items_coro = backend.prefixed_items_coro kvitems = backend.kvitems kvitems_coro = backend.kvitems_coro basic_parse_async = backend.basic_parse_async @@ -56,4 +58,4 @@ def _default_backend(): items_async = backend.items_async kvitems_async = backend.kvitems_async backend_name = backend.backend_name -backend = backend.backend \ No newline at end of file +backend = backend.backend diff --git a/src/ijson/backends/ext/_yajl2/items_basecoro.c b/src/ijson/backends/ext/_yajl2/items_basecoro.c index 422d792..87fc631 100644 --- a/src/ijson/backends/ext/_yajl2/items_basecoro.c +++ b/src/ijson/backends/ext/_yajl2/items_basecoro.c @@ -19,6 +19,7 @@ static int items_basecoro_init(ItemsBasecoro *self, PyObject *args, PyObject *kw self->target_send = NULL; self->prefix = NULL; self->object_depth = 0; + self->pending_builder_reset = 0; M1_N(self->module_state = get_state_from_imported_module()); builder_create(&self->builder); @@ -42,35 +43,13 @@ static void items_basecoro_dealloc(ItemsBasecoro *self) PyObject* items_basecoro_send_impl(PyObject *self, PyObject *path, PyObject *event, PyObject *value) { ItemsBasecoro *coro = (ItemsBasecoro *)self; - enames_t enames = coro->module_state->enames; - - if (builder_isactive(&coro->builder)) { - coro->object_depth += (event == enames.start_map_ename || event == enames.start_array_ename); - coro->object_depth -= (event == enames.end_map_ename || event == enames.end_array_ename); - if (coro->object_depth > 0) { - N_M1( builder_event(&coro->builder, enames, event, value) ); - } - else { - PyObject *retval = builder_value(&coro->builder); - CORO_SEND(coro->target_send, retval); - Py_DECREF(retval); - N_M1(builder_reset(&coro->builder)); - } - } - else { - int cmp = PyObject_RichCompareBool(path, coro->prefix, Py_EQ); - N_M1(cmp); - if (cmp) { - if (event == enames.start_map_ename || event == enames.start_array_ename) { - coro->object_depth = 1; - N_M1(builder_event(&coro->builder, enames, event, value)); - } - else { - CORO_SEND(coro->target_send, value); - } - } - } + PyObject *retval; + N_N(retval = items_common_send_top(self, path, event, value)); + if (retval == Py_None) + Py_RETURN_NONE; + CORO_SEND(coro->target_send, retval); + items_common_send_bottom(self, retval); Py_RETURN_NONE; } diff --git a/src/ijson/backends/ext/_yajl2/items_basecoro.h b/src/ijson/backends/ext/_yajl2/items_basecoro.h index 6f3c4f9..a089732 100644 --- a/src/ijson/backends/ext/_yajl2/items_basecoro.h +++ b/src/ijson/backends/ext/_yajl2/items_basecoro.h @@ -11,20 +11,17 @@ #ifndef ITEMS_BASECORO_H #define ITEMS_BASECORO_H -#include "builder.h" -#include "module_state.h" +#define SPECIFIC_ITEMS_CMP(prefix, path) \ + PyObject_RichCompareBool((path), (prefix), Py_EQ) + +#include "items_common.h" + +#undef SPECIFIC_ITEMS_CMP /** * items_basecoro coroutine object structure */ -typedef struct { - PyObject_HEAD - builder_t builder; - PyObject *target_send; - PyObject *prefix; - int object_depth; - yajl2_state *module_state; -} ItemsBasecoro; +typedef ItemsCommonBasecoro ItemsBasecoro; /** * items_basecoro coroutine object type @@ -47,4 +44,4 @@ extern PyTypeObject ItemsBasecoro_Type; */ PyObject* items_basecoro_send_impl(PyObject *self, PyObject *path, PyObject *event, PyObject *value); -#endif // ITEMS_BASECORO_H \ No newline at end of file +#endif // ITEMS_BASECORO_H diff --git a/src/ijson/backends/ext/_yajl2/items_common.h b/src/ijson/backends/ext/_yajl2/items_common.h new file mode 100644 index 0000000..88b418f --- /dev/null +++ b/src/ijson/backends/ext/_yajl2/items_common.h @@ -0,0 +1,68 @@ +#ifndef ITEMS_COMMON_H +#define ITEMS_COMMON_H + +#include "builder.h" +#include "module_state.h" + +/** + * common items/prefixed_items_basecoro coroutine object structure + */ +typedef struct { + PyObject_HEAD + builder_t builder; + int pending_builder_reset; + PyObject *target_send; + PyObject *prefix; + int object_depth; + yajl2_state *module_state; +} ItemsCommonBasecoro; + +static inline +PyObject* items_common_send_top(PyObject *self, PyObject *path, PyObject *event, PyObject *value) +{ + ItemsCommonBasecoro *coro = (ItemsCommonBasecoro *)self; + enames_t enames = coro->module_state->enames; + + if (builder_isactive(&coro->builder)) { + coro->object_depth += (event == enames.start_map_ename || event == enames.start_array_ename); + coro->object_depth -= (event == enames.end_map_ename || event == enames.end_array_ename); + if (coro->object_depth > 0) { + N_M1( builder_event(&coro->builder, enames, event, value) ); + } + else { + PyObject *retval = builder_value(&coro->builder); + coro->pending_builder_reset = 1; + return retval; + } + } + else { + int cmp = SPECIFIC_ITEMS_CMP(coro->prefix, path); + N_M1(cmp); + if (cmp) { + if (event == enames.start_map_ename || event == enames.start_array_ename) { + coro->object_depth = 1; + N_M1(builder_event(&coro->builder, enames, event, value)); + } + else { + Py_INCREF(value); + return value; + } + } + } + + Py_RETURN_NONE; +} + +static inline +void items_common_send_bottom(PyObject *self, PyObject *retval) +{ + ItemsCommonBasecoro *coro = (ItemsCommonBasecoro *)self; + + Py_DECREF(retval); + if (coro->pending_builder_reset) { + builder_reset(&coro->builder); + coro->pending_builder_reset = 0; + } +} + +#endif // ITEMS_COMMON_H diff --git a/src/ijson/backends/ext/_yajl2/module.c b/src/ijson/backends/ext/_yajl2/module.c index eaad854..032805c 100644 --- a/src/ijson/backends/ext/_yajl2/module.c +++ b/src/ijson/backends/ext/_yajl2/module.c @@ -20,6 +20,9 @@ #include "items.h" #include "items_async.h" #include "items_basecoro.h" +#include "prefixed_items.h" +#include "prefixed_items_async.h" +#include "prefixed_items_basecoro.h" #include "kvitems.h" #include "kvitems_async.h" #include "kvitems_basecoro.h" @@ -114,11 +117,14 @@ static int _yajl2_mod_exec(PyObject *m) ADD_TYPE("kvitems", KVItemsGen_Type); ADD_TYPE("items_basecoro", ItemsBasecoro_Type); ADD_TYPE("items", ItemsGen_Type); + ADD_TYPE("prefixed_items_basecoro", PrefixedItemsBasecoro_Type); + ADD_TYPE("prefixed_items", PrefixedItemsGen_Type); ADD_TYPE("_async_reading_iterator", AsyncReadingGeneratorType); ADD_TYPE("basic_parse_async", BasicParseAsync_Type); ADD_TYPE("parse_async", ParseAsync_Type); ADD_TYPE("kvitems_async", KVItemsAsync_Type); ADD_TYPE("items_async", ItemsAsync_Type); + ADD_TYPE("prefixed_items_async", PrefixedItemsAsync_Type); yajl2_state *state; M1_N(state = get_state(m)); @@ -153,4 +159,4 @@ static int _yajl2_mod_exec(PyObject *m) M1_N(state->Decimal); return 0; -} \ No newline at end of file +} diff --git a/src/ijson/backends/ext/_yajl2/parse_basecoro.c b/src/ijson/backends/ext/_yajl2/parse_basecoro.c index 6b9ec79..97c1bb8 100644 --- a/src/ijson/backends/ext/_yajl2/parse_basecoro.c +++ b/src/ijson/backends/ext/_yajl2/parse_basecoro.c @@ -10,6 +10,7 @@ #include "common.h" #include "items_basecoro.h" +#include "prefixed_items_basecoro.h" #include "kvitems_basecoro.h" #include "parse_basecoro.h" @@ -116,6 +117,9 @@ PyObject* parse_basecoro_send_impl(PyObject *self, PyObject *event, PyObject *va else if (ItemsBasecoro_Check(gen->target_send)) { items_basecoro_send_impl(gen->target_send, prefix, event, value); } + else if (PrefixedItemsBasecoro_Check(gen->target_send)) { + prefixed_items_basecoro_send_impl(gen->target_send, prefix, event, value); + } else { PyObject *res = PyTuple_Pack(3, prefix, event, value); CORO_SEND(gen->target_send, res); diff --git a/src/ijson/backends/ext/_yajl2/prefixed_items.c b/src/ijson/backends/ext/_yajl2/prefixed_items.c new file mode 100644 index 0000000..a8ba8c0 --- /dev/null +++ b/src/ijson/backends/ext/_yajl2/prefixed_items.c @@ -0,0 +1,61 @@ +/* + * items generator implementation for ijson's C backend + * + * Contributed by Rodrigo Tobar + * + * ICRAR - International Centre for Radio Astronomy Research + * (c) UWA - The University of Western Australia, 2020 + * Copyright by UWA (in the framework of the ICRAR) + */ + +#include "common.h" +#include "prefixed_items.h" +#include "prefixed_items_basecoro.h" +#include "parse_basecoro.h" +#include "basic_parse_basecoro.h" + +/* + * __init__, destructor, __iter__ and __next__ + */ +static int prefixed_itemsgen_init(PrefixedItemsGen *self, PyObject *args, PyObject *kwargs) +{ + PyObject *reading_args = PySequence_GetSlice(args, 0, 2); + PyObject *items_args = PySequence_GetSlice(args, 2, 4); + pipeline_node coro_pipeline[] = { + {&PrefixedItemsBasecoro_Type, items_args, NULL}, + {&ParseBasecoro_Type, NULL, NULL}, + {&BasicParseBasecoro_Type, NULL, kwargs}, + {NULL} + }; + int res = reading_generator_init(&self->reading_gen, reading_args, coro_pipeline); + Py_DECREF(items_args); + Py_DECREF(reading_args); + return res; +} + +static void prefixed_itemsgen_dealloc(PrefixedItemsGen *self) +{ + reading_generator_dealloc(&self->reading_gen); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static PyObject* prefixed_itemsgen_iternext(PyObject *self) +{ + PrefixedItemsGen *gen = (PrefixedItemsGen *)self; + return reading_generator_next(&gen->reading_gen); +} + +/* + * items generator object type + */ +PyTypeObject PrefixedItemsGen_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_basicsize = sizeof(PrefixedItemsGen), + .tp_name = "_yajl2.prefixed_items", + .tp_doc = "Generates items", + .tp_init = (initproc)prefixed_itemsgen_init, + .tp_dealloc = (destructor)prefixed_itemsgen_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_iter = ijson_return_self, + .tp_iternext = prefixed_itemsgen_iternext +}; diff --git a/src/ijson/backends/ext/_yajl2/prefixed_items.h b/src/ijson/backends/ext/_yajl2/prefixed_items.h new file mode 100644 index 0000000..2cd4faa --- /dev/null +++ b/src/ijson/backends/ext/_yajl2/prefixed_items.h @@ -0,0 +1,30 @@ +/* + * items generator for ijson's C backend + * + * Contributed by Rodrigo Tobar + * + * ICRAR - International Centre for Radio Astronomy Research + * (c) UWA - The University of Western Australia, 2020 + * Copyright by UWA (in the framework of the ICRAR) + */ + +#ifndef PREFIXED_ITEMS_H +#define PREFIXED_ITEMS_H + +#include "reading_generator.h" + +/** + * items generator object structure + */ +typedef struct { + PyObject_HEAD + reading_generator_t reading_gen; +} PrefixedItemsGen; + + +/** + * items generator object type + */ +extern PyTypeObject PrefixedItemsGen_Type; + +#endif /* PREFIXED_ITEMS_H */ diff --git a/src/ijson/backends/ext/_yajl2/prefixed_items_async.c b/src/ijson/backends/ext/_yajl2/prefixed_items_async.c new file mode 100644 index 0000000..e95cc77 --- /dev/null +++ b/src/ijson/backends/ext/_yajl2/prefixed_items_async.c @@ -0,0 +1,74 @@ +/* + * items_async asynchronous iterable implementation for ijson's C backend + * + * Contributed by Rodrigo Tobar + * + * ICRAR - International Centre for Radio Astronomy Research + * (c) UWA - The University of Western Australia, 2020 + * Copyright by UWA (in the framework of the ICRAR) + */ + + +#include "async_reading_generator.h" +#include "basic_parse_basecoro.h" +#include "parse_basecoro.h" +#include "prefixed_items_basecoro.h" +#include "common.h" +#include "coro_utils.h" + +/** + * items_async asynchronous iterable object structure + */ +typedef struct { + PyObject_HEAD + async_reading_generator *reading_generator; +} PrefixedItemsAsync; + +/* + * __init__, destructor and __anext__ + */ +static int prefixed_itemsasync_init(PrefixedItemsAsync *self, PyObject *args, PyObject *kwargs) +{ + PyObject *reading_args = PySequence_GetSlice(args, 0, 2); + PyObject *items_args = PySequence_GetSlice(args, 2, 4); + pipeline_node coro_pipeline[] = { + {&PrefixedItemsBasecoro_Type, items_args, NULL}, + {&ParseBasecoro_Type, NULL, NULL}, + {&BasicParseBasecoro_Type, NULL, kwargs}, + {NULL} + }; + M1_N(self->reading_generator = (async_reading_generator *)PyObject_CallObject((PyObject *)&AsyncReadingGeneratorType, reading_args)); + int ret = async_reading_generator_add_coro(self->reading_generator, coro_pipeline); + Py_DECREF(items_args); + Py_DECREF(reading_args); + return ret; +} + +static void prefixed_itemsasync_dealloc(PrefixedItemsAsync *self) { + Py_XDECREF(self->reading_generator); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static PyObject *prefixed_itemsasync_anext(PyObject *self) +{ + PrefixedItemsAsync *gen = (PrefixedItemsAsync *)self; + Py_INCREF(gen->reading_generator); + return (PyObject *)gen->reading_generator; +} + +static PyAsyncMethods prefixed_itemsasync_methods = { + .am_await = ijson_return_self, + .am_aiter = ijson_return_self, + .am_anext = prefixed_itemsasync_anext +}; + +PyTypeObject PrefixedItemsAsync_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_basicsize = sizeof(PrefixedItemsAsync), + .tp_name = "_yajl2._prefixed_items_async", + .tp_doc = "Asynchronous iterable yielding fully-built items", + .tp_init = (initproc)prefixed_itemsasync_init, + .tp_dealloc = (destructor)prefixed_itemsasync_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_as_async = &prefixed_itemsasync_methods +}; diff --git a/src/ijson/backends/ext/_yajl2/prefixed_items_async.h b/src/ijson/backends/ext/_yajl2/prefixed_items_async.h new file mode 100644 index 0000000..3241652 --- /dev/null +++ b/src/ijson/backends/ext/_yajl2/prefixed_items_async.h @@ -0,0 +1,22 @@ +/* + * items_async asynchronous iterable for ijson's C backend + * + * Contributed by Rodrigo Tobar + * + * ICRAR - International Centre for Radio Astronomy Research + * (c) UWA - The University of Western Australia, 2020 + * Copyright by UWA (in the framework of the ICRAR) + */ + +#ifndef PREFIXED_ITEMS_ASYNC_H +#define PREFIXED_ITEMS_ASYNC_H + +#define PY_SSIZE_T_CLEAN +#include + +/** + * items_async asynchronous iterable object type + */ +extern PyTypeObject PrefixedItemsAsync_Type; + +#endif // PREFIXED_ITEMS_ASYNC_H diff --git a/src/ijson/backends/ext/_yajl2/prefixed_items_basecoro.c b/src/ijson/backends/ext/_yajl2/prefixed_items_basecoro.c new file mode 100644 index 0000000..ad550d4 --- /dev/null +++ b/src/ijson/backends/ext/_yajl2/prefixed_items_basecoro.c @@ -0,0 +1,92 @@ +/* + * prefixed_items_basecoro coroutine implementation for ijson's C backend + * + * Contributed by Rodrigo Tobar + * + * ICRAR - International Centre for Radio Astronomy Research + * (c) UWA - The University of Western Australia, 2020 + * Copyright by UWA (in the framework of the ICRAR) + */ + +#include "common.h" +#include "prefixed_items_basecoro.h" + +/* + * __init__, destructor, __iter__ and __next__ + */ +static int prefixed_items_basecoro_init(PrefixedItemsBasecoro *self, PyObject *args, PyObject *kwargs) +{ + self->target_send = NULL; + self->prefix = NULL; + self->object_depth = 0; + self->pending_builder_reset = 0; + M1_N(self->module_state = get_state_from_imported_module()); + builder_create(&self->builder); + + PyObject *map_type; + PyObject *prefix; + M1_Z(PyArg_ParseTuple(args, "OOO", &(self->target_send), &prefix, &map_type)); + Py_INCREF(self->target_send); + if (PyUnicode_Check(prefix)) { + M1_N(self->prefix = PySet_New(NULL)); + M1_M1(PySet_Add(self->prefix, prefix)); + } + else { + M1_N(self->prefix = PySet_New(prefix)); + } + M1_M1(builder_init(&self->builder, map_type)); + + return 0; +} + +static void prefixed_items_basecoro_dealloc(PrefixedItemsBasecoro *self) +{ + Py_XDECREF(self->prefix); + Py_XDECREF(self->target_send); + builder_destroy(&self->builder); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +PyObject* prefixed_items_basecoro_send_impl(PyObject *self, PyObject *path, PyObject *event, PyObject *value) +{ + PrefixedItemsBasecoro *coro = (PrefixedItemsBasecoro *)self; + + PyObject *retval; + N_N(retval = items_common_send_top(self, path, event, value)); + if (retval == Py_None) + Py_RETURN_NONE; + PyObject *tuple = PyTuple_Pack(2, path, retval); + CORO_SEND(coro->target_send, tuple); + Py_DECREF(tuple); + items_common_send_bottom(self, retval); + Py_RETURN_NONE; +} + +static PyObject* prefixed_items_basecoro_send(PyObject *self, PyObject *tuple) +{ + PyObject *path = PyTuple_GetItem(tuple, 0); + PyObject *event = PyTuple_GetItem(tuple, 1); + PyObject *value = PyTuple_GetItem(tuple, 2); + return prefixed_items_basecoro_send_impl(self, path, event, value); +} + +static PyMethodDef prefixed_items_basecoro_methods[] = { + {"send", prefixed_items_basecoro_send, METH_O, "coroutine's send method"}, + {NULL, NULL, 0, NULL} +}; + +/* + * items generator object type + */ +PyTypeObject PrefixedItemsBasecoro_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_basicsize = sizeof(PrefixedItemsBasecoro), + .tp_name = "_yajl2.prefixed_items_basecoro", + .tp_doc = "Coroutine dispatching fully-built objects for the given prefix", + .tp_init = (initproc)prefixed_items_basecoro_init, + .tp_dealloc = (destructor)prefixed_items_basecoro_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_iter = ijson_return_self, + .tp_iternext = ijson_return_none, + .tp_methods = prefixed_items_basecoro_methods +}; diff --git a/src/ijson/backends/ext/_yajl2/prefixed_items_basecoro.h b/src/ijson/backends/ext/_yajl2/prefixed_items_basecoro.h new file mode 100644 index 0000000..fd1d579 --- /dev/null +++ b/src/ijson/backends/ext/_yajl2/prefixed_items_basecoro.h @@ -0,0 +1,47 @@ +/* + * prefixed_items_basecoro coroutine for ijson's C backend + * + * Contributed by Rodrigo Tobar + * + * ICRAR - International Centre for Radio Astronomy Research + * (c) UWA - The University of Western Australia, 2020 + * Copyright by UWA (in the framework of the ICRAR) + */ + +#ifndef PREFIXED_ITEMS_BASECORO_H +#define PREFIXED_ITEMS_BASECORO_H + +#define SPECIFIC_ITEMS_CMP(prefix, path) \ + PySet_Contains((prefix), (path)) + +#include "items_common.h" + +#undef SPECIFIC_ITEMS_CMP + +/** + * prefixed_items_basecoro coroutine object structure + */ +typedef ItemsCommonBasecoro PrefixedItemsBasecoro; + +/** + * prefixed_items_basecoro coroutine object type + */ +extern PyTypeObject PrefixedItemsBasecoro_Type; + +/** + * Utility function to check if an object is an prefixed_items_basecoro coroutine or not + */ +#define PrefixedItemsBasecoro_Check(o) (Py_TYPE(o) == &PrefixedItemsBasecoro_Type) + +/** + * The implementation of the prefixed_items_basecoro.send() method accepting an unpacked + * event + * @param self An prefixed_items_basecoro coroutine + * @param path The path of this event + * @param event The event name + * @param value The value of this event + * @return None, or NULL in case of an error + */ +PyObject* prefixed_items_basecoro_send_impl(PyObject *self, PyObject *path, PyObject *event, PyObject *value); + +#endif // PREFIXED_ITEMS_BASECORO_H diff --git a/src/ijson/backends/yajl2_c.py b/src/ijson/backends/yajl2_c.py index 6511870..cc15669 100644 --- a/src/ijson/backends/yajl2_c.py +++ b/src/ijson/backends/yajl2_c.py @@ -67,4 +67,17 @@ def items_async(file, prefix, map_type=None, **kwargs): buf_size = _get_buf_size(kwargs) return _yajl2.items_async(file, buf_size, prefix, map_type, **kwargs) +@utils.coroutine +def prefixed_items_basecoro(target, prefix, map_type=None, **kwargs): + return _yajl2.prefixed_items_basecoro(target.send, prefix, map_type, **kwargs) + +def prefixed_items_gen(file, prefix, map_type=None, **kwargs): + f = compat.bytes_reader(file) + buf_size = _get_buf_size(kwargs) + return _yajl2.prefixed_items(f, buf_size, prefix, map_type, **kwargs) + +def prefixed_items_async(file, prefix, map_type=None, **kwargs): + buf_size = _get_buf_size(kwargs) + return _yajl2.prefixed_items_async(file, buf_size, prefix, map_type, **kwargs) + common.enrich_backend(globals()) diff --git a/src/ijson/common.py b/src/ijson/common.py index 99b8bc4..ee95f8e 100644 --- a/src/ijson/common.py +++ b/src/ijson/common.py @@ -136,16 +136,12 @@ def setter(value): else: self.containers[-1](value) - @utils.coroutine -def items_basecoro(target, prefix, map_type=None): - ''' - An couroutine dispatching native Python objects constructed from the events - under a given prefix. - ''' +def prefixed_items_basecoro(target, prefix, map_type=None): + prefix = set([prefix]) if isinstance(prefix, str) else set(prefix) while True: current, event, value = (yield) - if current == prefix: + if current in prefix: if event in ('start_map', 'start_array'): object_depth = 1 builder = ObjectBuilder(map_type=map_type) @@ -157,10 +153,22 @@ def items_basecoro(target, prefix, map_type=None): elif event in ('end_map', 'end_array'): object_depth -= 1 del builder.containers[:] - target.send(builder.value) + target.send((prefix, builder.value)) else: - target.send(value) + target.send((prefix, value)) + +def items_basecoro(target, prefix, map_type=None): + ''' + An couroutine dispatching native Python objects constructed from the events + under a given prefix. + ''' + @utils.coroutine + def strip_prefix(): + while True: + prefix, value = (yield) + target.send(value) + return prefixed_items_basecoro(strip_prefix(), prefix, map_type) @utils.coroutine def kvitems_basecoro(target, prefix, map_type=None): @@ -244,6 +252,14 @@ def _items_pipeline(backend, prefix, map_type, config): ) +def _prefixed_items_pipeline(backend, prefix, map_type, config): + return ( + (backend['prefixed_items_basecoro'], (prefix,), {'map_type': map_type}), + (backend['parse_basecoro'], [], {}), + (backend['basic_parse_basecoro'], [], config) + ) + + def _kvitems_pipeline(backend, prefix, map_type, config): return ( (backend['kvitems_basecoro'], (prefix,), {'map_type': map_type}), @@ -278,6 +294,13 @@ def items_coro(target, prefix, map_type=None, **config): ) return items_coro +def _make_prefixed_items_coro(backend): + def prefixed_items_coro(target, prefix, map_type=None, **config): + return utils.chain( + target, + *_prefixed_items_pipeline(backend, prefix, map_type, config) + ) + return prefixed_items_coro def _make_kvitems_coro(backend): def kvitems_coro(target, prefix, map_type=None, **config): @@ -349,6 +372,15 @@ def items_gen(file_obj, prefix, map_type=None, buf_size=64*1024, **config): return items_gen +def _make_prefixed_items_gen(backend): + def prefixed_items_gen(file_obj, prefix, map_type=None, buf_size=64*1024, **config): + return utils.coros2gen( + file_source(file_obj, buf_size=buf_size), + *_prefixed_items_pipeline(backend, prefix, map_type, config) + ) + return prefixed_items_gen + + def _make_kvitems_gen(backend): def kvitems_gen(file_obj, prefix, map_type=None, buf_size=64*1024, **config): return utils.coros2gen( @@ -411,6 +443,25 @@ def items(source, prefix, map_type=None, buf_size=64*1024, **config): return items +def _make_prefixed_items(backend): + def prefixed_items(source, prefix, map_type=None, buf_size=64*1024, **config): + source = _get_source(source) + if is_async_file(source): + return backend['prefixed_items_async']( + source, prefix, map_type=map_type, buf_size=buf_size, **config + ) + elif is_file(source): + return backend['prefixed_items_gen']( + source, prefix, map_type=map_type, buf_size=buf_size, **config + ) + elif is_iterable(source): + return utils.coros2gen(source, + (backend['prefixed_items_basecoro'], (prefix,), {'map_type': map_type}) + ) + raise ValueError("Unknown source type: %r" % type(source)) + return prefixed_items + + def _make_kvitems(backend): def kvitems(source, prefix, map_type=None, buf_size=64*1024, **config): source = _get_source(source) @@ -462,6 +513,14 @@ def items(events, prefix, map_type=None): ) +def prefixed_items(events, prefix, map_type=None): + """Like ijson.prefixed_items, but takes events generated via ijson.parse instead of + a file""" + warnings.warn(_common_functions_warn, DeprecationWarning) + return utils.coros2gen(events, + (prefixed_items_basecoro, (prefix,), {'map_type': map_type}) + ) + class BackendCapabilities: ''' Capabilities supported by a backend. @@ -495,7 +554,7 @@ def enrich_backend(backend, **capabilities_overrides): backend['capabilities'] = capabilities backend['backend'] = backend['__name__'].split('.')[-1] backend['backend_name'] = backend['backend'] - for name in ('basic_parse', 'parse', 'items', 'kvitems'): + for name in ('basic_parse', 'parse', 'items', 'prefixed_items', 'kvitems'): basecoro_name = name + '_basecoro' if basecoro_name not in backend: backend[basecoro_name] = globals()[basecoro_name] @@ -512,4 +571,4 @@ def enrich_backend(backend, **capabilities_overrides): factory = getattr(utils35, '_make_' + async_name) backend[async_name] = factory(backend) factory = globals()['_make_' + name] - backend[name] = factory(backend) \ No newline at end of file + backend[name] = factory(backend) diff --git a/src/ijson/utils35.py b/src/ijson/utils35.py index dd56715..654e3ac 100644 --- a/src/ijson/utils35.py +++ b/src/ijson/utils35.py @@ -84,9 +84,16 @@ def items_async(f, prefix, map_type=None, buf_size=64*1024, **config): ) return items_async +def _make_prefixed_items_async(backend): + def prefixed_items_async(f, prefix, map_type=None, buf_size=64*1024, **config): + return async_iterable(f, buf_size, + *common._prefixed_items_pipeline(backend, prefix, map_type, config) + ) + return prefixed_items_async + def _make_kvitems_async(backend): def kvitems_async(f, prefix, map_type=None, buf_size=64*1024, **config): return async_iterable(f, buf_size, *common._kvitems_pipeline(backend, prefix, map_type, config) ) - return kvitems_async \ No newline at end of file + return kvitems_async