diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 963e65416..1cca74b96 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,6 +12,9 @@ jobs: steps: - uses: actions/checkout@v4 + with: + submodules: 'recursive' + token: ${{ secrets.SUBMODULE_PAT }} - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/coveralls.yml b/.github/workflows/coveralls.yml index a65d2a41c..7da631517 100644 --- a/.github/workflows/coveralls.yml +++ b/.github/workflows/coveralls.yml @@ -12,6 +12,9 @@ jobs: steps: - uses: actions/checkout@v4 + with: + submodules: 'recursive' + token: ${{ secrets.SUBMODULE_PAT }} - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 48b1c8570..823f7afeb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,6 +17,9 @@ jobs: steps: - uses: actions/checkout@v4 + with: + submodules: 'recursive' + token: ${{ secrets.SUBMODULE_PAT }} - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index bc48423b5..f410bb531 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -13,6 +13,9 @@ jobs: steps: - uses: actions/checkout@v4 + with: + submodules: 'recursive' + token: ${{ secrets.SUBMODULE_PAT }} # Used to host cibuildwheel - uses: actions/setup-python@v5 diff --git a/.gitignore b/.gitignore index 598f821d1..034e9720e 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,7 @@ dist/ downloads/ eggs/ .eggs/ -lib/ +/lib/ lib64/ parts/ sdist/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..aacea2395 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "ext/libdeflate"] + path = ext/libdeflate + url = https://github.com/ebiggers/libdeflate diff --git a/CMakeLists.txt b/CMakeLists.txt index d6fa4bd85..2d0e31992 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,24 +1,26 @@ -cmake_minimum_required(VERSION 3.15...3.26) - -project( - ${SKBUILD_PROJECT_NAME} - LANGUAGES C - VERSION ${SKBUILD_PROJECT_VERSION}) +cmake_minimum_required(VERSION 3.26...3.29) +project(${SKBUILD_PROJECT_NAME} LANGUAGES C VERSION ${SKBUILD_PROJECT_VERSION}) find_package( Python + REQUIRED COMPONENTS Interpreter Development.Module ${SKBUILD_SABI_COMPONENT} NumPy - REQUIRED) + ) + +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(LIBDEFLATE_COMPRESSION_SUPPORT OFF) +set(LIBDEFLATE_GZIP_SUPPORT OFF) +set(LIBDEFLATE_BUILD_GZIP OFF) + + # Add submodule libdeflate +add_subdirectory(ext/libdeflate EXCLUDE_FROM_ALL) -python_add_library(cutils - MODULE - src/asammdf/blocks/cutils.c - WITH_SOABI USE_SABI 3.9) +python_add_library(cutils MODULE WITH_SOABI USE_SABI 3.9 src/asammdf/blocks/cutils.c) -target_link_libraries(cutils PRIVATE Python::NumPy) +target_link_libraries(cutils PRIVATE Python::NumPy libdeflate::libdeflate_static) install(TARGETS cutils DESTINATION "asammdf/blocks") diff --git a/LICENSE b/LICENSE index a8f6ca6dc..1e7c9849f 100644 --- a/LICENSE +++ b/LICENSE @@ -163,3 +163,34 @@ whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library. + +-------------------------------------------------------------------------------- + +3rdparty dependency libdeflate is statically linked. +libdeflate has the following license: + + +Copyright 2016 Eric Biggers +Copyright 2024 Google LLC + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation files +(the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, +and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- \ No newline at end of file diff --git a/ext/libdeflate b/ext/libdeflate new file mode 160000 index 000000000..78051988f --- /dev/null +++ b/ext/libdeflate @@ -0,0 +1 @@ +Subproject commit 78051988f96dc8d8916310d8b24021f01bd9e102 diff --git a/pyproject.toml b/pyproject.toml index c6b9c0a14..73ab06339 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,7 +103,7 @@ skip = ["pp*", "*_ppc64le", "*-musllinux*", "*_s390x"] [tool.ruff] target-version = "py39" -extend-exclude = ["./src/asammdf/gui/ui"] +extend-exclude = ["./src/asammdf/gui/ui", "./ext"] force-exclude = true [tool.ruff.lint] diff --git a/src/asammdf/__init__.py b/src/asammdf/__init__.py index 307bcb2a4..9f627270f 100644 --- a/src/asammdf/__init__.py +++ b/src/asammdf/__init__.py @@ -15,7 +15,7 @@ from .blocks.utils import load_channel_names_from_file from .gui import plot from .mdf import MDF, SUPPORTED_VERSIONS -from .signal import Signal +from .signal import InvalidationArray, Signal from .version import __version__ try: @@ -28,6 +28,7 @@ __all__ = [ "MDF", "SUPPORTED_VERSIONS", + "InvalidationArray", "Signal", "Source", "__cextension__", diff --git a/src/asammdf/blocks/bus_logging_utils.py b/src/asammdf/blocks/bus_logging_utils.py index 7cbb311b9..fdecca615 100644 --- a/src/asammdf/blocks/bus_logging_utils.py +++ b/src/asammdf/blocks/bus_logging_utils.py @@ -396,10 +396,16 @@ def extract_mux( sig_name = sig.name try: + scale_ranges = getattr(sig, "scale_ranges", None) + if scale_ranges: + unit = scale_ranges[0]["unit"] or "" + else: + unit = sig.unit or "" + signals[sig_name] = { "name": sig_name, "comment": sig.comment or "", - "unit": sig.unit or "", + "unit": unit, "samples": samples if raw else apply_conversion(samples, sig, ignore_value2text_conversion), "conversion": get_conversion(sig) if raw else None, "t": t_, @@ -441,8 +447,24 @@ def get_conversion(signal: Signal) -> v4b.ChannelConversion | None: a, b = float(signal.factor), float(signal.offset) - if signal.values: - conv = {} + conv = {} + + scale_ranges = getattr(signal, "scale_ranges", None) + if scale_ranges: + for i, scale_info in enumerate(scale_ranges): + conv[f"upper_{i}"] = scale_info["max"] + conv[f"lower_{i}"] = scale_info["min"] + conv[f"text_{i}"] = from_dict({"a": scale_info["factor"], "b": scale_info["offset"]}) + + for i, (val, text) in enumerate(signal.values.items(), len(scale_ranges)): + conv[f"upper_{i}"] = val + conv[f"lower_{i}"] = val + conv[f"text_{i}"] = text + + conv["default"] = from_dict({"a": a, "b": b}) + + elif signal.values: + for i, (val, text) in enumerate(signal.values.items()): conv[f"upper_{i}"] = val conv[f"lower_{i}"] = val diff --git a/src/asammdf/blocks/cutils.c b/src/asammdf/blocks/cutils.c index 13f1e03e1..f86688e36 100644 --- a/src/asammdf/blocks/cutils.c +++ b/src/asammdf/blocks/cutils.c @@ -1,14 +1,33 @@ #define NPY_NO_DEPRECATED_API NPY_1_22_API_VERSION -//#define Py_LIMITED_API 0x030900f0 - #define PY_SSIZE_T_CLEAN 1 #include +#define _FILE_OFFSET_BITS 64 #include "numpy/arrayobject.h" #include "numpy/ndarrayobject.h" #include #include #include #include +#include + +#if defined(_WIN32) +#include +#include +#define FSEEK64(file, address, whence) _fseeki64((file), (address), (whence)) +#define FTELL64(file) _ftelli64(file) +#else +#include +#include +#include +#include +#include +#define Sleep(x) usleep((int)(1000 * (x))) +#define FSEEK64(file, address, whence) fseeko((file), (address), (whence)) +#define FTELL64(file) ftello(file) +#endif + +#define MAX(a,b) ((a) > (b) ? (a) : (b)) +#define MIN(a,b) ((a) < (b) ? (a) : (b)) #define PY_PRINTF(o) \ PyObject_Print(o, stdout, 0); \ @@ -18,1524 +37,2669 @@ char err_string[1024]; struct rec_info { - uint32_t id; - uint32_t size; - PyObject *mlist; + uint32_t id; + uint32_t size; + PyObject *mlist; }; struct node { - struct node *next; - struct rec_info info; + struct node *next; + struct rec_info info; }; + static PyObject *sort_data_block(PyObject *self, PyObject *args) { - uint64_t id_size = 0, position = 0, size; - uint32_t rec_size, length, rec_id; - PyObject *signal_data, *partial_records, *record_size, *optional, *mlist; - PyObject *bts, *key, *value, *rem = NULL; - unsigned char *buf, *end, *orig; - struct node *head = NULL, *last = NULL, *item; - - if (!PyArg_ParseTuple(args, "OOOK|O", &signal_data, &partial_records, &record_size, &id_size, &optional)) + uint64_t id_size = 0, position = 0, size; + uint32_t rec_size, length, rec_id; + PyObject *signal_data, *partial_records, *record_size, *optional, *mlist; + PyObject *bts, *key, *value, *rem = NULL; + unsigned char *buf, *end, *orig; + struct node *head = NULL, *last = NULL, *item; + + if (!PyArg_ParseTuple(args, "OOOK|O", &signal_data, &partial_records, &record_size, &id_size, &optional)) + { + return 0; + } + else + { + Py_ssize_t pos = 0; + position = 0; + + while (PyDict_Next(record_size, &pos, &key, &value)) { - return 0; + item = malloc(sizeof(struct node)); + item->info.id = PyLong_AsUnsignedLong(key); + item->info.size = PyLong_AsUnsignedLong(value); + item->info.mlist = PyDict_GetItem(partial_records, key); + item->next = NULL; + if (last) + last->next = item; + if (!head) + head = item; + last = item; } - else + + buf = (unsigned char *)PyBytes_AsString(signal_data); + orig = buf; + size = (uint64_t)PyBytes_Size(signal_data); + end = buf + size; + + while ((buf + id_size) < end) { - Py_ssize_t pos = 0; - position = 0; - while (PyDict_Next(record_size, &pos, &key, &value)) - { - item = malloc(sizeof(struct node)); - item->info.id = PyLong_AsUnsignedLong(key); - item->info.size = PyLong_AsUnsignedLong(value); - item->info.mlist = PyDict_GetItem(partial_records, key); - item->next = NULL; - if (last) - last->next = item; - if (!head) - head = item; - last = item; - } + rec_id = 0; + for (unsigned char i = 0; i < id_size; i++, buf++) + { + rec_id += (*buf) << (i << 3); + } - buf = (unsigned char *)PyBytes_AsString(signal_data); - orig = buf; - size = (uint64_t)PyBytes_Size(signal_data); - end = buf + size; + key = PyLong_FromUnsignedLong(rec_id); + value = PyDict_GetItem(record_size, key); - while ((buf + id_size) < end) - { + if (!value) + { + rem = PyBytes_FromStringAndSize(NULL, 0); + Py_XDECREF(key); + return rem; + } + else + { + rec_size = PyLong_AsUnsignedLong(value); + } + + mlist = PyDict_GetItem(partial_records, key); + + if (!mlist) + { + rem = PyBytes_FromStringAndSize(NULL, 0); + Py_XDECREF(key); + return rem; + } - rec_id = 0; - for (unsigned char i = 0; i < id_size; i++, buf++) - { - rec_id += (*buf) << (i << 3); - } - - key = PyLong_FromUnsignedLong(rec_id); - value = PyDict_GetItem(record_size, key); - - if (!value) - { - rem = PyBytes_FromStringAndSize(NULL, 0); - Py_XDECREF(key); - return rem; - } - else - { - rec_size = PyLong_AsUnsignedLong(value); - } - - mlist = PyDict_GetItem(partial_records, key); - - if (!mlist) - { - rem = PyBytes_FromStringAndSize(NULL, 0); - Py_XDECREF(key); - return rem; - } - - Py_XDECREF(key); - - if (rec_size) - { - if (rec_size + position + id_size > size) - { - break; - } - bts = PyBytes_FromStringAndSize((const char *)buf, (Py_ssize_t)rec_size); - PyList_Append( - mlist, - bts); - Py_XDECREF(bts); - - buf += rec_size; - } - else - { - if (4 + position + id_size > size) - { - break; - } - rec_size = (buf[3] << 24) + (buf[2] << 16) + (buf[1] << 8) + buf[0]; - length = rec_size + 4; - if (position + length + id_size > size) - { - break; - } - bts = PyBytes_FromStringAndSize((const char *)buf, (Py_ssize_t)length); - PyList_Append(mlist, bts); - Py_XDECREF(bts); - buf += length; - } - - position = (uint64_t)(buf - orig); - } + Py_XDECREF(key); - while (head != NULL) + if (rec_size) + { + if (rec_size + position + id_size > size) { - item = head; - item->info.mlist = NULL; - - head = head->next; - item->next = NULL; - free(item); + break; } + bts = PyBytes_FromStringAndSize((const char *)buf, (Py_ssize_t)rec_size); + PyList_Append( + mlist, + bts); + Py_XDECREF(bts); + + buf += rec_size; + } + else + { + if (4 + position + id_size > size) + { + break; + } + rec_size = (buf[3] << 24) + (buf[2] << 16) + (buf[1] << 8) + buf[0]; + length = rec_size + 4; + if (position + length + id_size > size) + { + break; + } + bts = PyBytes_FromStringAndSize((const char *)buf, (Py_ssize_t)length); + PyList_Append(mlist, bts); + Py_XDECREF(bts); + buf += length; + } - head = NULL; - last = NULL; - item = NULL; - mlist = NULL; - - rem = PyBytes_FromStringAndSize((const char *)(orig + position), (Py_ssize_t)(size - position)); + position = (uint64_t)(buf - orig); + } - buf = NULL; - orig = NULL; - end = NULL; + while (head != NULL) + { + item = head; + item->info.mlist = NULL; - return rem; + head = head->next; + item->next = NULL; + free(item); } + + head = NULL; + last = NULL; + item = NULL; + mlist = NULL; + + rem = PyBytes_FromStringAndSize((const char *)(orig + position), (Py_ssize_t)(size - position)); + + buf = NULL; + orig = NULL; + end = NULL; + + return rem; + } } static Py_ssize_t calc_size(char *buf) { - return (unsigned char)buf[3] << 24 | - (unsigned char)buf[2] << 16 | - (unsigned char)buf[1] << 8 | - (unsigned char)buf[0]; + return (unsigned char)buf[3] << 24 | + (unsigned char)buf[2] << 16 | + (unsigned char)buf[1] << 8 | + (unsigned char)buf[0]; } static PyObject *extract(PyObject *self, PyObject *args) { - Py_ssize_t i = 0, count, max = 0, list_count; - int64_t offset; - Py_ssize_t pos = 0, size = 0; - PyObject *signal_data, *is_byte_array, *offsets, *offsets_list = NULL; - char *buf; - PyArrayObject *vals; - PyArray_Descr *descr; - unsigned char *addr2; - - if (!PyArg_ParseTuple(args, "OOO", &signal_data, &is_byte_array, &offsets)) + Py_ssize_t i = 0, count, max = 0, list_count; + int64_t offset; + Py_ssize_t pos = 0, size = 0; + PyObject *signal_data, *is_byte_array, *offsets, *offsets_list = NULL; + char *buf; + PyArrayObject *vals; + PyArray_Descr *descr; + unsigned char *addr2; + + if (!PyArg_ParseTuple(args, "OOO", &signal_data, &is_byte_array, &offsets)) + { + return 0; + } + else + { + Py_ssize_t max_size = 0; + Py_ssize_t retval = PyBytes_AsStringAndSize(signal_data, &buf, &max_size); + + if (retval == -1) + { + printf("PyBytes_AsStringAndSize error\n"); + return NULL; + } + + count = 0; + pos = 0; + + if (offsets == Py_None) { - return 0; + while ((pos + 4) <= max_size) + { + size = calc_size(&buf[pos]); + + if ((pos + 4 + size) > max_size) + break; + + if (max < size) + max = size; + pos += 4 + size; + count++; + } } else { - Py_ssize_t max_size = 0; - Py_ssize_t retval = PyBytes_AsStringAndSize(signal_data, &buf, &max_size); + offsets_list = PyObject_CallMethod(offsets, "tolist", NULL); + list_count = (Py_ssize_t)PyList_Size(offsets_list); + for (i = 0; i < list_count; i++) + { + offset = (int64_t)PyLong_AsLongLong(PyList_GetItem(offsets_list, i)); + if ((offset + 4) > max_size) + break; + size = calc_size(&buf[offset]); + if ((offset + 4 + size) > max_size) + break; + if (max < size) + max = size; + count++; + } + } - if (retval == -1) - { - printf("PyBytes_AsStringAndSize error\n"); - return NULL; - } + if (PyObject_IsTrue(is_byte_array)) + { - count = 0; - pos = 0; + npy_intp dims[2]; + dims[0] = count; + dims[1] = max; - if (offsets == Py_None) - { - while ((pos + 4) <= max_size) - { - size = calc_size(&buf[pos]); - - if ((pos + 4 + size) > max_size) - break; - - if (max < size) - max = size; - pos += 4 + size; - count++; - } - } - else - { - offsets_list = PyObject_CallMethod(offsets, "tolist", NULL); - list_count = (Py_ssize_t)PyList_Size(offsets_list); - for (i = 0; i < list_count; i++) - { - offset = (int64_t)PyLong_AsLongLong(PyList_GetItem(offsets_list, i)); - if ((offset + 4) > max_size) - break; - size = calc_size(&buf[offset]); - if ((offset + 4 + size) > max_size) - break; - if (max < size) - max = size; - count++; - } - } + vals = (PyArrayObject *)PyArray_ZEROS(2, dims, NPY_UBYTE, 0); - if (PyObject_IsTrue(is_byte_array)) + if (offsets == Py_None) + { + pos = 0; + for (i = 0; i < count; i++) { - - npy_intp dims[2]; - dims[0] = count; - dims[1] = max; - - vals = (PyArrayObject *)PyArray_ZEROS(2, dims, NPY_UBYTE, 0); - - if (offsets == Py_None) - { - pos = 0; - for (i = 0; i < count; i++) - { - addr2 = (unsigned char *)PyArray_GETPTR2(vals, i, 0); - size = calc_size(&buf[pos]); - pos += 4; - memcpy(addr2, &buf[pos], size); - pos += size; - } - } - else - { - for (i = 0; i < count; i++) - { - addr2 = (unsigned char *)PyArray_GETPTR2(vals, i, 0); - offset = (int64_t)PyLong_AsLongLong(PyList_GetItem(offsets_list, i)); - size = calc_size(&buf[offset]); - memcpy(addr2, &buf[offset + 4], size); - } - } + addr2 = (unsigned char *)PyArray_GETPTR2(vals, i, 0); + size = calc_size(&buf[pos]); + pos += 4; + memcpy(addr2, &buf[pos], size); + pos += size; } - else + } + else + { + for (i = 0; i < count; i++) { - npy_intp dims[1]; - dims[0] = count; + addr2 = (unsigned char *)PyArray_GETPTR2(vals, i, 0); + offset = (int64_t)PyLong_AsLongLong(PyList_GetItem(offsets_list, i)); + size = calc_size(&buf[offset]); + memcpy(addr2, &buf[offset + 4], size); + } + } + } + else + { + npy_intp dims[1]; + dims[0] = count; - descr = PyArray_DescrFromType(NPY_STRING); - descr = PyArray_DescrNew(descr); + descr = PyArray_DescrFromType(NPY_STRING); + descr = PyArray_DescrNew(descr); #if NPY_ABI_VERSION < 0x02000000 - descr->elsize = (int)max; + descr->elsize = (int)max; #else - PyDataType_SET_ELSIZE(descr, max); + PyDataType_SET_ELSIZE(descr, max); #endif - vals = (PyArrayObject *)PyArray_Zeros(1, dims, descr, 0); - - if (offsets == Py_None) - { - - pos = 0; - for (i = 0; i < count; i++) - { - addr2 = (unsigned char *)PyArray_GETPTR1(vals, i); - size = calc_size(&buf[pos]); - pos += 4; - memcpy(addr2, &buf[pos], size); - pos += size; - } - } - else - { - for (i = 0; i < count; i++) - { - addr2 = (unsigned char *)PyArray_GETPTR1(vals, i); - offset = (int64_t)PyLong_AsLongLong(PyList_GetItem(offsets_list, i)); - size = calc_size(&buf[offset]); - memcpy(addr2, &buf[offset + 4], size); - } - Py_XDECREF(offsets_list); - } + vals = (PyArrayObject *)PyArray_Zeros(1, dims, descr, 0); + + if (offsets == Py_None) + { + + pos = 0; + for (i = 0; i < count; i++) + { + addr2 = (unsigned char *)PyArray_GETPTR1(vals, i); + size = calc_size(&buf[pos]); + pos += 4; + memcpy(addr2, &buf[pos], size); + pos += size; + } + } + else + { + for (i = 0; i < count; i++) + { + addr2 = (unsigned char *)PyArray_GETPTR1(vals, i); + offset = (int64_t)PyLong_AsLongLong(PyList_GetItem(offsets_list, i)); + size = calc_size(&buf[offset]); + memcpy(addr2, &buf[offset + 4], size); } + Py_XDECREF(offsets_list); + } } + } - return (PyObject *)vals; + return (PyObject *)vals; } static PyObject *lengths(PyObject *self, PyObject *args) { - Py_ssize_t i = 0; - Py_ssize_t count; - PyObject *lst, *values, *item; + Py_ssize_t i = 0; + Py_ssize_t count; + PyObject *lst, *values, *item; - if (!PyArg_ParseTuple(args, "O", &lst)) - { - return 0; - } - else - { + if (!PyArg_ParseTuple(args, "O", &lst)) + { + return 0; + } + else + { - count = PyList_Size(lst); + count = PyList_Size(lst); - values = PyTuple_New(count); + values = PyTuple_New(count); - for (i = 0; i < (Py_ssize_t)count; i++) - { - item = PyList_GetItem(lst, i); - PyTuple_SetItem(values, i, PyLong_FromSsize_t(PyBytes_Size(item))); - } + for (i = 0; i < (Py_ssize_t)count; i++) + { + item = PyList_GetItem(lst, i); + PyTuple_SetItem(values, i, PyLong_FromSsize_t(PyBytes_Size(item))); } + } - return values; + return values; } static PyObject *get_vlsd_offsets(PyObject *self, PyObject *args) { - Py_ssize_t i = 0; - Py_ssize_t count; - PyObject *lst, *item, *result; - npy_intp dim[1]; - PyArrayObject *values; + Py_ssize_t i = 0; + Py_ssize_t count; + PyObject *lst, *item, *result; + npy_intp dim[1]; + PyArrayObject *values; - uint64_t current_size = 0; + uint64_t current_size = 0; - void *h_result; + void *h_result; - if (!PyArg_ParseTuple(args, "O", &lst)) - { - return 0; - } - else - { + if (!PyArg_ParseTuple(args, "O", &lst)) + { + return 0; + } + else + { - count = PyList_Size(lst); - dim[0] = (Py_ssize_t)count; - values = (PyArrayObject *)PyArray_SimpleNew(1, dim, NPY_ULONGLONG); + count = PyList_Size(lst); + dim[0] = (Py_ssize_t)count; + values = (PyArrayObject *)PyArray_SimpleNew(1, dim, NPY_ULONGLONG); - for (i = 0; i < (Py_ssize_t)count; i++) - { - h_result = PyArray_GETPTR1(values, i); - item = PyList_GetItem(lst, i); - *((uint64_t *)h_result) = current_size; - current_size += (uint64_t)PyBytes_Size(item); - } + for (i = 0; i < (Py_ssize_t)count; i++) + { + h_result = PyArray_GETPTR1(values, i); + item = PyList_GetItem(lst, i); + *((uint64_t *)h_result) = current_size; + current_size += (uint64_t)PyBytes_Size(item); } + } - result = PyTuple_Pack(2, values, PyLong_FromUnsignedLongLong(current_size)); + result = PyTuple_Pack(2, values, PyLong_FromUnsignedLongLong(current_size)); - return result; + return result; } static PyObject *get_vlsd_max_sample_size(PyObject *self, PyObject *args) { - Py_ssize_t i = 0; - Py_ssize_t count = 0; - PyObject *data, *offsets; - uint64_t max_size = 0; - uint32_t vlsd_size = 0; - char *inptr = NULL, *data_end = NULL, *current_position = NULL; - - uint64_t current_size = 0, *offsets_array; - - if (!PyArg_ParseTuple(args, "OOn", &data, &offsets, &count)) - { - return 0; - } - else + Py_ssize_t i = 0; + Py_ssize_t count = 0; + PyObject *data, *offsets; + uint64_t max_size = 0; + uint32_t vlsd_size = 0; + char *inptr = NULL, *data_end = NULL, *current_position = NULL; + + uint64_t current_size = 0, *offsets_array; + + if (!PyArg_ParseTuple(args, "OOn", &data, &offsets, &count)) + { + return 0; + } + else + { + offsets_array = (uint64_t *)PyArray_GETPTR1((PyArrayObject *)offsets, 0); + inptr = PyBytes_AsString(data); + data_end = inptr + PyBytes_Size(data); + + for (i = 0; i < count; i++, offsets_array++) { - offsets_array = (uint64_t *)PyArray_GETPTR1((PyArrayObject *)offsets, 0); - inptr = PyBytes_AsString(data); - data_end = inptr + PyBytes_Size(data); - - for (i = 0; i < count; i++, offsets_array++) - { - current_position = inptr + *offsets_array; - if (current_position >= data_end) - { - return PyLong_FromUnsignedLongLong(max_size); - } - memcpy(&vlsd_size, inptr + *offsets_array, 4); - if (vlsd_size > max_size) - { - max_size = vlsd_size; - } - } + current_position = inptr + *offsets_array; + if (current_position >= data_end) + { + return PyLong_FromUnsignedLongLong(max_size); + } + memcpy(&vlsd_size, inptr + *offsets_array, 4); + if (vlsd_size > max_size) + { + max_size = vlsd_size; + } } + } - return PyLong_FromUnsignedLongLong(max_size); + return PyLong_FromUnsignedLongLong(max_size); } void positions_char(PyObject *samples, PyObject *timestamps, PyObject *plot_samples, PyObject *plot_timestamps, PyObject *result, int32_t step, int32_t count, int32_t last) { - char min, max, *indata; - int32_t *outdata; - Py_ssize_t pos_min = 0, pos_max = 0; + char min, max, *indata; + int32_t *outdata; + Py_ssize_t pos_min = 0, pos_max = 0; - indata = (char *)PyArray_GETPTR1((PyArrayObject *)samples, 0); - outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); + indata = (char *)PyArray_GETPTR1((PyArrayObject *)samples, 0); + outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); - char *ps; - double tmin, tmax, *ts, *pt; + char *ps; + double tmin, tmax, *ts, *pt; - ps = (char *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); - pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); - ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); + ps = (char *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); + pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); + ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); - Py_ssize_t current_pos = 0, stop_index = count - 1; - for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) - { - - pos_min = current_pos; - pos_max = current_pos; - min = max = *indata; - indata++; - current_pos++; + Py_ssize_t current_pos = 0, stop_index = count - 1; + for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) + { - tmin = tmax = *ts; - ts++; + pos_min = current_pos; + pos_max = current_pos; + min = max = *indata; + indata++; + current_pos++; - if ((i != stop_index) || (0 != last)) - { + tmin = tmax = *ts; + ts++; - for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) - { - if (*indata < min) - { - min = *indata; - pos_min = current_pos; - tmin = *ts; - } - else if (*indata > max) - { - max = *indata; - pos_max = current_pos; - tmax = *ts; - } - - current_pos++; - - if ((i == stop_index) && (j == last)) - break; - } - } + if ((i != stop_index) || (0 != last)) + { - if (pos_min < pos_max) + for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) + { + if (*indata < min) { - *outdata++ = pos_min; - *outdata++ = pos_max; - - *ps++ = min; - *pt++ = tmin; - *ps++ = max; - *pt++ = tmax; + min = *indata; + pos_min = current_pos; + tmin = *ts; } - else + else if (*indata > max) { - *outdata++ = pos_max; - *outdata++ = pos_min; - - *ps++ = max; - *pt++ = tmax; - *ps++ = min; - *pt++ = tmin; + max = *indata; + pos_max = current_pos; + tmax = *ts; } + + current_pos++; + + if ((i == stop_index) && (j == last)) + break; + } + } + + if (pos_min < pos_max) + { + *outdata++ = pos_min; + *outdata++ = pos_max; + + *ps++ = min; + *pt++ = tmin; + *ps++ = max; + *pt++ = tmax; + } + else + { + *outdata++ = pos_max; + *outdata++ = pos_min; + + *ps++ = max; + *pt++ = tmax; + *ps++ = min; + *pt++ = tmin; } + } } void positions_short(PyObject *samples, PyObject *timestamps, PyObject *plot_samples, PyObject *plot_timestamps, PyObject *result, int32_t step, int32_t count, int32_t last) { - short min, max, *indata; - int32_t *outdata; - Py_ssize_t pos_min = 0, pos_max = 0; - - indata = (short *)PyArray_GETPTR1((PyArrayObject *)samples, 0); - outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); + short min, max, *indata; + int32_t *outdata; + Py_ssize_t pos_min = 0, pos_max = 0; - short *ps; - double tmin, tmax, *ts, *pt; + indata = (short *)PyArray_GETPTR1((PyArrayObject *)samples, 0); + outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); - ps = (short *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); - pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); - ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); + short *ps; + double tmin, tmax, *ts, *pt; - Py_ssize_t current_pos = 0, stop_index = count - 1; - for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) - { + ps = (short *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); + pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); + ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); - pos_min = current_pos; - pos_max = current_pos; - min = max = *indata; - indata++; - current_pos++; + Py_ssize_t current_pos = 0, stop_index = count - 1; + for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) + { - tmin = tmax = *ts; - ts++; + pos_min = current_pos; + pos_max = current_pos; + min = max = *indata; + indata++; + current_pos++; - if ((i != stop_index) || (0 != last)) - { + tmin = tmax = *ts; + ts++; - for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) - { - if (*indata < min) - { - min = *indata; - pos_min = current_pos; - tmin = *ts; - } - else if (*indata > max) - { - max = *indata; - pos_max = current_pos; - tmax = *ts; - } - - current_pos++; - - if ((i == stop_index) && (j == last)) - break; - } - } + if ((i != stop_index) || (0 != last)) + { - if (pos_min < pos_max) + for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) + { + if (*indata < min) { - *outdata++ = pos_min; - *outdata++ = pos_max; - - *ps++ = min; - *pt++ = tmin; - *ps++ = max; - *pt++ = tmax; + min = *indata; + pos_min = current_pos; + tmin = *ts; } - else + else if (*indata > max) { - *outdata++ = pos_max; - *outdata++ = pos_min; - - *ps++ = max; - *pt++ = tmax; - *ps++ = min; - *pt++ = tmin; + max = *indata; + pos_max = current_pos; + tmax = *ts; } + + current_pos++; + + if ((i == stop_index) && (j == last)) + break; + } + } + + if (pos_min < pos_max) + { + *outdata++ = pos_min; + *outdata++ = pos_max; + + *ps++ = min; + *pt++ = tmin; + *ps++ = max; + *pt++ = tmax; + } + else + { + *outdata++ = pos_max; + *outdata++ = pos_min; + + *ps++ = max; + *pt++ = tmax; + *ps++ = min; + *pt++ = tmin; } + } } void positions_long(PyObject *samples, PyObject *timestamps, PyObject *plot_samples, PyObject *plot_timestamps, PyObject *result, int32_t step, int32_t count, int32_t last) { - int32_t min, max, *indata; - int32_t *outdata; - Py_ssize_t pos_min = 0, pos_max = 0; + int32_t min, max, *indata; + int32_t *outdata; + Py_ssize_t pos_min = 0, pos_max = 0; - indata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)samples, 0); - outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); + indata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)samples, 0); + outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); - long *ps; - double tmin, tmax, *ts, *pt; + long *ps; + double tmin, tmax, *ts, *pt; - ps = (int32_t *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); - pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); - ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); - - Py_ssize_t current_pos = 0, stop_index = count - 1; - for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) - { + ps = (int32_t *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); + pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); + ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); - pos_min = current_pos; - pos_max = current_pos; - min = max = *indata; - indata++; - current_pos++; + Py_ssize_t current_pos = 0, stop_index = count - 1; + for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) + { - tmin = tmax = *ts; - ts++; + pos_min = current_pos; + pos_max = current_pos; + min = max = *indata; + indata++; + current_pos++; - if ((i != stop_index) || (0 != last)) - { + tmin = tmax = *ts; + ts++; - for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) - { - if (*indata < min) - { - min = *indata; - pos_min = current_pos; - tmin = *ts; - } - else if (*indata > max) - { - max = *indata; - pos_max = current_pos; - tmax = *ts; - } - - current_pos++; - - if ((i == stop_index) && (j == last)) - break; - } - } + if ((i != stop_index) || (0 != last)) + { - if (pos_min < pos_max) + for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) + { + if (*indata < min) { - *outdata++ = pos_min; - *outdata++ = pos_max; - - *ps++ = min; - *pt++ = tmin; - *ps++ = max; - *pt++ = tmax; + min = *indata; + pos_min = current_pos; + tmin = *ts; } - else + else if (*indata > max) { - *outdata++ = pos_max; - *outdata++ = pos_min; - - *ps++ = max; - *pt++ = tmax; - *ps++ = min; - *pt++ = tmin; + max = *indata; + pos_max = current_pos; + tmax = *ts; } + + current_pos++; + + if ((i == stop_index) && (j == last)) + break; + } + } + + if (pos_min < pos_max) + { + *outdata++ = pos_min; + *outdata++ = pos_max; + + *ps++ = min; + *pt++ = tmin; + *ps++ = max; + *pt++ = tmax; + } + else + { + *outdata++ = pos_max; + *outdata++ = pos_min; + + *ps++ = max; + *pt++ = tmax; + *ps++ = min; + *pt++ = tmin; } + } } void positions_long_long(PyObject *samples, PyObject *timestamps, PyObject *plot_samples, PyObject *plot_timestamps, PyObject *result, int32_t step, int32_t count, int32_t last) { - int64_t min, max, *indata; - int32_t *outdata; - Py_ssize_t pos_min = 0, pos_max = 0; - - indata = (int64_t *)PyArray_GETPTR1((PyArrayObject *)samples, 0); - outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); + int64_t min, max, *indata; + int32_t *outdata; + Py_ssize_t pos_min = 0, pos_max = 0; - int64_t *ps; - double tmin, tmax, *ts, *pt; + indata = (int64_t *)PyArray_GETPTR1((PyArrayObject *)samples, 0); + outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); - ps = (int64_t *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); - pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); - ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); + int64_t *ps; + double tmin, tmax, *ts, *pt; - Py_ssize_t current_pos = 0, stop_index = count - 1; - for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) - { + ps = (int64_t *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); + pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); + ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); - pos_min = current_pos; - pos_max = current_pos; - min = max = *indata; - indata++; - current_pos++; + Py_ssize_t current_pos = 0, stop_index = count - 1; + for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) + { - tmin = tmax = *ts; - ts++; + pos_min = current_pos; + pos_max = current_pos; + min = max = *indata; + indata++; + current_pos++; - if ((i != stop_index) || (0 != last)) - { + tmin = tmax = *ts; + ts++; - for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) - { - if (*indata < min) - { - min = *indata; - pos_min = current_pos; - tmin = *ts; - } - else if (*indata > max) - { - max = *indata; - pos_max = current_pos; - tmax = *ts; - } - - current_pos++; - - if ((i == stop_index) && (j == last)) - break; - } - } + if ((i != stop_index) || (0 != last)) + { - if (pos_min < pos_max) + for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) + { + if (*indata < min) { - *outdata++ = pos_min; - *outdata++ = pos_max; - - *ps++ = min; - *pt++ = tmin; - *ps++ = max; - *pt++ = tmax; + min = *indata; + pos_min = current_pos; + tmin = *ts; } - else + else if (*indata > max) { - *outdata++ = pos_max; - *outdata++ = pos_min; - - *ps++ = max; - *pt++ = tmax; - *ps++ = min; - *pt++ = tmin; + max = *indata; + pos_max = current_pos; + tmax = *ts; } + + current_pos++; + + if ((i == stop_index) && (j == last)) + break; + } } + + if (pos_min < pos_max) + { + *outdata++ = pos_min; + *outdata++ = pos_max; + + *ps++ = min; + *pt++ = tmin; + *ps++ = max; + *pt++ = tmax; + } + else + { + *outdata++ = pos_max; + *outdata++ = pos_min; + + *ps++ = max; + *pt++ = tmax; + *ps++ = min; + *pt++ = tmin; + } + } } void positions_unsigned_char(PyObject *samples, PyObject *timestamps, PyObject *plot_samples, PyObject *plot_timestamps, PyObject *result, int32_t step, int32_t count, int32_t last) { - unsigned char min, max, *indata; - int32_t *outdata; - Py_ssize_t pos_min = 0, pos_max = 0; + unsigned char min, max, *indata; + int32_t *outdata; + Py_ssize_t pos_min = 0, pos_max = 0; - indata = (unsigned char *)PyArray_GETPTR1((PyArrayObject *)samples, 0); - outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); + indata = (unsigned char *)PyArray_GETPTR1((PyArrayObject *)samples, 0); + outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); - unsigned char *ps; - double tmin, tmax, *ts, *pt; + unsigned char *ps; + double tmin, tmax, *ts, *pt; - ps = (unsigned char *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); - pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); - ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); + ps = (unsigned char *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); + pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); + ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); - Py_ssize_t current_pos = 0, stop_index = count - 1; - for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) - { - - pos_min = current_pos; - pos_max = current_pos; - min = max = *indata; - indata++; - current_pos++; + Py_ssize_t current_pos = 0, stop_index = count - 1; + for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) + { - tmin = tmax = *ts; - ts++; + pos_min = current_pos; + pos_max = current_pos; + min = max = *indata; + indata++; + current_pos++; - if ((i != stop_index) || (0 != last)) - { + tmin = tmax = *ts; + ts++; - for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) - { - if (*indata < min) - { - min = *indata; - pos_min = current_pos; - tmin = *ts; - } - else if (*indata > max) - { - max = *indata; - pos_max = current_pos; - tmax = *ts; - } - - current_pos++; - - if ((i == stop_index) && (j == last)) - break; - } - } + if ((i != stop_index) || (0 != last)) + { - if (pos_min < pos_max) + for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) + { + if (*indata < min) { - *outdata++ = pos_min; - *outdata++ = pos_max; - - *ps++ = min; - *pt++ = tmin; - *ps++ = max; - *pt++ = tmax; + min = *indata; + pos_min = current_pos; + tmin = *ts; } - else + else if (*indata > max) { - *outdata++ = pos_max; - *outdata++ = pos_min; - - *ps++ = max; - *pt++ = tmax; - *ps++ = min; - *pt++ = tmin; + max = *indata; + pos_max = current_pos; + tmax = *ts; } + + current_pos++; + + if ((i == stop_index) && (j == last)) + break; + } + } + + if (pos_min < pos_max) + { + *outdata++ = pos_min; + *outdata++ = pos_max; + + *ps++ = min; + *pt++ = tmin; + *ps++ = max; + *pt++ = tmax; + } + else + { + *outdata++ = pos_max; + *outdata++ = pos_min; + + *ps++ = max; + *pt++ = tmax; + *ps++ = min; + *pt++ = tmin; } + } } void positions_unsigned_short(PyObject *samples, PyObject *timestamps, PyObject *plot_samples, PyObject *plot_timestamps, PyObject *result, int32_t step, int32_t count, int32_t last) { - unsigned short min, max, *indata; - int32_t *outdata; - Py_ssize_t pos_min = 0, pos_max = 0; - - indata = (unsigned short *)PyArray_GETPTR1((PyArrayObject *)samples, 0); - outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); + unsigned short min, max, *indata; + int32_t *outdata; + Py_ssize_t pos_min = 0, pos_max = 0; - unsigned short *ps; - double tmin, tmax, *ts, *pt; + indata = (unsigned short *)PyArray_GETPTR1((PyArrayObject *)samples, 0); + outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); - ps = (unsigned short *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); - pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); - ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); + unsigned short *ps; + double tmin, tmax, *ts, *pt; - Py_ssize_t current_pos = 0, stop_index = count - 1; - for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) - { + ps = (unsigned short *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); + pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); + ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); - pos_min = current_pos; - pos_max = current_pos; - min = max = *indata; - indata++; - current_pos++; + Py_ssize_t current_pos = 0, stop_index = count - 1; + for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) + { - tmin = tmax = *ts; - ts++; + pos_min = current_pos; + pos_max = current_pos; + min = max = *indata; + indata++; + current_pos++; - if ((i != stop_index) || (0 != last)) - { + tmin = tmax = *ts; + ts++; - for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) - { - if (*indata < min) - { - min = *indata; - pos_min = current_pos; - tmin = *ts; - } - else if (*indata > max) - { - max = *indata; - pos_max = current_pos; - tmax = *ts; - } - - current_pos++; - - if ((i == stop_index) && (j == last)) - break; - } - } + if ((i != stop_index) || (0 != last)) + { - if (pos_min < pos_max) + for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) + { + if (*indata < min) { - *outdata++ = pos_min; - *outdata++ = pos_max; - - *ps++ = min; - *pt++ = tmin; - *ps++ = max; - *pt++ = tmax; + min = *indata; + pos_min = current_pos; + tmin = *ts; } - else + else if (*indata > max) { - *outdata++ = pos_max; - *outdata++ = pos_min; - - *ps++ = max; - *pt++ = tmax; - *ps++ = min; - *pt++ = tmin; + max = *indata; + pos_max = current_pos; + tmax = *ts; } + + current_pos++; + + if ((i == stop_index) && (j == last)) + break; + } + } + + if (pos_min < pos_max) + { + *outdata++ = pos_min; + *outdata++ = pos_max; + + *ps++ = min; + *pt++ = tmin; + *ps++ = max; + *pt++ = tmax; + } + else + { + *outdata++ = pos_max; + *outdata++ = pos_min; + + *ps++ = max; + *pt++ = tmax; + *ps++ = min; + *pt++ = tmin; } + } } void positions_unsigned_long(PyObject *samples, PyObject *timestamps, PyObject *plot_samples, PyObject *plot_timestamps, PyObject *result, int32_t step, int32_t count, int32_t last) { - uint32_t min, max, *indata; - int32_t *outdata; - Py_ssize_t pos_min = 0, pos_max = 0; + uint32_t min, max, *indata; + int32_t *outdata; + Py_ssize_t pos_min = 0, pos_max = 0; - indata = (uint32_t *)PyArray_GETPTR1((PyArrayObject *)samples, 0); - outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); + indata = (uint32_t *)PyArray_GETPTR1((PyArrayObject *)samples, 0); + outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); - uint32_t *ps; - double tmin, tmax, *ts, *pt; + uint32_t *ps; + double tmin, tmax, *ts, *pt; - ps = (uint32_t *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); - pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); - ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); - - Py_ssize_t current_pos = 0, stop_index = count - 1; - for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) - { + ps = (uint32_t *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); + pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); + ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); - pos_min = current_pos; - pos_max = current_pos; - min = max = *indata; - indata++; - current_pos++; + Py_ssize_t current_pos = 0, stop_index = count - 1; + for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) + { - tmin = tmax = *ts; - ts++; + pos_min = current_pos; + pos_max = current_pos; + min = max = *indata; + indata++; + current_pos++; - if ((i != stop_index) || (0 != last)) - { + tmin = tmax = *ts; + ts++; - for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) - { - if (*indata < min) - { - min = *indata; - pos_min = current_pos; - tmin = *ts; - } - else if (*indata > max) - { - max = *indata; - pos_max = current_pos; - tmax = *ts; - } - - current_pos++; - - if ((i == stop_index) && (j == last)) - break; - } - } + if ((i != stop_index) || (0 != last)) + { - if (pos_min < pos_max) + for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) + { + if (*indata < min) { - *outdata++ = pos_min; - *outdata++ = pos_max; - - *ps++ = min; - *pt++ = tmin; - *ps++ = max; - *pt++ = tmax; + min = *indata; + pos_min = current_pos; + tmin = *ts; } - else + else if (*indata > max) { - *outdata++ = pos_max; - *outdata++ = pos_min; - - *ps++ = max; - *pt++ = tmax; - *ps++ = min; - *pt++ = tmin; + max = *indata; + pos_max = current_pos; + tmax = *ts; } + + current_pos++; + + if ((i == stop_index) && (j == last)) + break; + } + } + + if (pos_min < pos_max) + { + *outdata++ = pos_min; + *outdata++ = pos_max; + + *ps++ = min; + *pt++ = tmin; + *ps++ = max; + *pt++ = tmax; + } + else + { + *outdata++ = pos_max; + *outdata++ = pos_min; + + *ps++ = max; + *pt++ = tmax; + *ps++ = min; + *pt++ = tmin; } + } } void positions_unsigned_long_long(PyObject *samples, PyObject *timestamps, PyObject *plot_samples, PyObject *plot_timestamps, PyObject *result, int32_t step, int32_t count, int32_t last) { - uint64_t min, max, *indata; - int32_t *outdata; - Py_ssize_t pos_min = 0, pos_max = 0; - - indata = (uint64_t *)PyArray_GETPTR1((PyArrayObject *)samples, 0); - outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); + uint64_t min, max, *indata; + int32_t *outdata; + Py_ssize_t pos_min = 0, pos_max = 0; - uint64_t *ps; - double tmin, tmax, *ts, *pt; + indata = (uint64_t *)PyArray_GETPTR1((PyArrayObject *)samples, 0); + outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); - ps = (uint64_t *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); - pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); - ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); + uint64_t *ps; + double tmin, tmax, *ts, *pt; - Py_ssize_t current_pos = 0, stop_index = count - 1; - for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) - { + ps = (uint64_t *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); + pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); + ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); - pos_min = current_pos; - pos_max = current_pos; - min = max = *indata; - indata++; - current_pos++; + Py_ssize_t current_pos = 0, stop_index = count - 1; + for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) + { - tmin = tmax = *ts; - ts++; + pos_min = current_pos; + pos_max = current_pos; + min = max = *indata; + indata++; + current_pos++; - if ((i != stop_index) || (0 != last)) - { + tmin = tmax = *ts; + ts++; - for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) - { - if (*indata < min) - { - min = *indata; - pos_min = current_pos; - tmin = *ts; - } - else if (*indata > max) - { - max = *indata; - pos_max = current_pos; - tmax = *ts; - } - - current_pos++; - - if ((i == stop_index) && (j == last)) - break; - } - } + if ((i != stop_index) || (0 != last)) + { - if (pos_min < pos_max) + for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) + { + if (*indata < min) { - *outdata++ = pos_min; - *outdata++ = pos_max; - - *ps++ = min; - *pt++ = tmin; - *ps++ = max; - *pt++ = tmax; + min = *indata; + pos_min = current_pos; + tmin = *ts; } - else + else if (*indata > max) { - *outdata++ = pos_max; - *outdata++ = pos_min; - - *ps++ = max; - *pt++ = tmax; - *ps++ = min; - *pt++ = tmin; + max = *indata; + pos_max = current_pos; + tmax = *ts; } + + current_pos++; + + if ((i == stop_index) && (j == last)) + break; + } + } + + if (pos_min < pos_max) + { + *outdata++ = pos_min; + *outdata++ = pos_max; + + *ps++ = min; + *pt++ = tmin; + *ps++ = max; + *pt++ = tmax; } + else + { + *outdata++ = pos_max; + *outdata++ = pos_min; + + *ps++ = max; + *pt++ = tmax; + *ps++ = min; + *pt++ = tmin; + } + } } void positions_float(PyObject *samples, PyObject *timestamps, PyObject *plot_samples, PyObject *plot_timestamps, PyObject *result, int32_t step, int32_t count, int32_t last) { - float min, max, *indata = NULL; - int32_t *outdata = NULL; - Py_ssize_t pos_min = 0, pos_max = 0; + float min, max, *indata = NULL; + int32_t *outdata = NULL; + Py_ssize_t pos_min = 0, pos_max = 0; - indata = (float *)PyArray_GETPTR1((PyArrayObject *)samples, 0); - outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); + indata = (float *)PyArray_GETPTR1((PyArrayObject *)samples, 0); + outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); - float *ps; - double tmin, tmax, *ts, *pt; + float *ps; + double tmin, tmax, *ts, *pt; - ps = (float *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); - pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); - ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); + ps = (float *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); + pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); + ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); - Py_ssize_t current_pos = 0, stop_index = count - 1; - for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) - { - - pos_min = current_pos; - pos_max = current_pos; - min = max = *indata; - indata++; - current_pos++; + Py_ssize_t current_pos = 0, stop_index = count - 1; + for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) + { - tmin = tmax = *ts; - ts++; + pos_min = current_pos; + pos_max = current_pos; + min = max = *indata; + indata++; + current_pos++; - if ((i != stop_index) || (0 != last)) - { + tmin = tmax = *ts; + ts++; - for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) - { - if (*indata < min) - { - min = *indata; - pos_min = current_pos; - tmin = *ts; - } - else if (*indata > max) - { - max = *indata; - pos_max = current_pos; - tmax = *ts; - } - - current_pos++; - - if ((i == stop_index) && (j == last)) - break; - } - } + if ((i != stop_index) || (0 != last)) + { - if (pos_min < pos_max) + for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) + { + if (*indata < min) { - *outdata++ = pos_min; - *outdata++ = pos_max; - - *ps++ = min; - *pt++ = tmin; - *ps++ = max; - *pt++ = tmax; + min = *indata; + pos_min = current_pos; + tmin = *ts; } - else + else if (*indata > max) { - *outdata++ = pos_max; - *outdata++ = pos_min; - - *ps++ = max; - *pt++ = tmax; - *ps++ = min; - *pt++ = tmin; + max = *indata; + pos_max = current_pos; + tmax = *ts; } + + current_pos++; + + if ((i == stop_index) && (j == last)) + break; + } + } + + if (pos_min < pos_max) + { + *outdata++ = pos_min; + *outdata++ = pos_max; + + *ps++ = min; + *pt++ = tmin; + *ps++ = max; + *pt++ = tmax; + } + else + { + *outdata++ = pos_max; + *outdata++ = pos_min; + + *ps++ = max; + *pt++ = tmax; + *ps++ = min; + *pt++ = tmin; } + } } void positions_double(PyObject *samples, PyObject *timestamps, PyObject *plot_samples, PyObject *plot_timestamps, PyObject *result, int32_t step, int32_t count, int32_t last) { - double min, max, *indata = NULL; - int32_t *outdata = NULL; - Py_ssize_t pos_min = 0, pos_max = 0; - - indata = (double *)PyArray_GETPTR1((PyArrayObject *)samples, 0); - outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); + double min, max, *indata = NULL; + int32_t *outdata = NULL; + Py_ssize_t pos_min = 0, pos_max = 0; - double *ps = NULL; - double tmin, tmax, *ts = NULL, *pt = NULL; + indata = (double *)PyArray_GETPTR1((PyArrayObject *)samples, 0); + outdata = (int32_t *)PyArray_GETPTR1((PyArrayObject *)result, 0); - ps = (double *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); - pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); - ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); + double *ps = NULL; + double tmin, tmax, *ts = NULL, *pt = NULL; - Py_ssize_t current_pos = 0, stop_index = count - 1; - for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) - { + ps = (double *)PyArray_GETPTR1((PyArrayObject *)plot_samples, 0); + pt = (double *)PyArray_GETPTR1((PyArrayObject *)plot_timestamps, 0); + ts = (double *)PyArray_GETPTR1((PyArrayObject *)timestamps, 0); - pos_min = current_pos; - pos_max = current_pos; - min = max = *indata; - indata++; - current_pos++; + Py_ssize_t current_pos = 0, stop_index = count - 1; + for (Py_ssize_t i = 0; i < (Py_ssize_t)count; i++) + { - tmin = tmax = *ts; - ts++; + pos_min = current_pos; + pos_max = current_pos; + min = max = *indata; + indata++; + current_pos++; - if ((i != stop_index) || (0 != last)) - { + tmin = tmax = *ts; + ts++; - for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) - { - if (*indata < min) - { - min = *indata; - pos_min = current_pos; - tmin = *ts; - } - else if (*indata > max) - { - max = *indata; - pos_max = current_pos; - tmax = *ts; - } - - current_pos++; - - if ((i == stop_index) && (j == last)) - break; - } - } + if ((i != stop_index) || (0 != last)) + { - if (pos_min < pos_max) + for (Py_ssize_t j = 1; j < step; j++, indata++, ts++) + { + if (*indata < min) { - *outdata++ = pos_min; - *outdata++ = pos_max; - - *ps++ = min; - *pt++ = tmin; - *ps++ = max; - *pt++ = tmax; + min = *indata; + pos_min = current_pos; + tmin = *ts; } - else + else if (*indata > max) { - *outdata++ = pos_max; - *outdata++ = pos_min; - - *ps++ = max; - *pt++ = tmax; - *ps++ = min; - *pt++ = tmin; + max = *indata; + pos_max = current_pos; + tmax = *ts; } - } + + current_pos++; + + if ((i == stop_index) && (j == last)) + break; + } + } + + if (pos_min < pos_max) + { + *outdata++ = pos_min; + *outdata++ = pos_max; + + *ps++ = min; + *pt++ = tmin; + *ps++ = max; + *pt++ = tmax; + } + else + { + *outdata++ = pos_max; + *outdata++ = pos_min; + + *ps++ = max; + *pt++ = tmax; + *ps++ = min; + *pt++ = tmin; + } + } } static PyObject *positions(PyObject *self, PyObject *args) { - int32_t count, step, last; - unsigned char itemsize; - char *kind; - Py_ssize_t _size; + int32_t count, step, last; + unsigned char itemsize; + char *kind; + Py_ssize_t _size; + + PyObject *samples, *timestamps, *result, *step_obj, *count_obj, *last_obj, *plot_samples, *plot_timestamps; + + if (!PyArg_ParseTuple(args, "OOOOOOOOs#B", + &samples, ×tamps, &plot_samples, &plot_timestamps, &result, &step_obj, &count_obj, &last_obj, &kind, &_size, &itemsize)) + { + return NULL; + } + else + { + count = PyLong_AsLong(count_obj); + step = PyLong_AsLong(step_obj); + last = PyLong_AsLong(last_obj) - 1; + + if (kind[0] == 'u') + { + if (itemsize == 1) + positions_unsigned_char(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); + else if (itemsize == 2) + positions_unsigned_short(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); + else if (itemsize == 4) + positions_unsigned_long(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); + else + positions_unsigned_long_long(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); + } + else if (kind[0] == 'i') + { + if (itemsize == 1) + positions_char(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); + else if (itemsize == 2) + positions_short(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); + else if (itemsize == 4) + positions_long(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); + else + positions_long_long(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); + } + else if (kind[0] == 'f') + { + if (itemsize == 4) + positions_float(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); + else + positions_double(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); + } - PyObject *samples, *timestamps, *result, *step_obj, *count_obj, *last_obj, *plot_samples, *plot_timestamps; + Py_INCREF(Py_None); + return Py_None; + } +} - if (!PyArg_ParseTuple(args, "OOOOOOOOs#B", - &samples, ×tamps, &plot_samples, &plot_timestamps, &result, &step_obj, &count_obj, &last_obj, &kind, &_size, &itemsize)) +static PyObject *get_channel_raw_bytes(PyObject *self, PyObject *args) +{ + Py_ssize_t count, size, actual_byte_count, delta; + PyObject *data_block, *out; + + Py_ssize_t record_size, byte_offset, byte_count; + + char *inptr, *outptr; + + if (!PyArg_ParseTuple(args, "Onnn", &data_block, &record_size, &byte_offset, &byte_count)) + { + return 0; + } + else + { + if (PyBytes_Check(data_block)) { + size = PyBytes_Size(data_block); + inptr = PyBytes_AsString(data_block); + } + else { + size = PyByteArray_Size(data_block); + inptr = PyByteArray_AsString(data_block); + } + + if (!record_size) { - return NULL; + out = PyByteArray_FromStringAndSize(NULL, 0); } - else + else if (record_size < byte_offset + byte_count) { - count = PyLong_AsLong(count_obj); - step = PyLong_AsLong(step_obj); - last = PyLong_AsLong(last_obj) - 1; + delta = byte_offset + byte_count - record_size; + actual_byte_count = record_size - byte_offset; - if (kind[0] == 'u') - { - if (itemsize == 1) - positions_unsigned_char(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); - else if (itemsize == 2) - positions_unsigned_short(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); - else if (itemsize == 4) - positions_unsigned_long(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); - else - positions_unsigned_long_long(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); - } - else if (kind[0] == 'i') - { - if (itemsize == 1) - positions_char(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); - else if (itemsize == 2) - positions_short(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); - else if (itemsize == 4) - positions_long(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); - else - positions_long_long(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); - } - else if (kind[0] == 'f') + count = size / record_size; + + out = PyByteArray_FromStringAndSize(NULL, count * byte_count); + outptr = PyByteArray_AsString(out); + + inptr += byte_offset; + + for (Py_ssize_t i = 0; i < count; i++) + { + for (Py_ssize_t j = 0; j < actual_byte_count; j++) + *outptr++ = *inptr++; + + inptr += record_size - actual_byte_count; + for (Py_ssize_t j = 0; j < delta; j++) { - if (itemsize == 4) - positions_float(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); - else - positions_double(samples, timestamps, plot_samples, plot_timestamps, result, step, count, last); + *outptr++ = '\0'; } + } + } + else + { + count = size / record_size; + + out = PyByteArray_FromStringAndSize(NULL, count * byte_count); + outptr = PyByteArray_AsString(out); + + inptr += byte_offset; - Py_INCREF(Py_None); - return Py_None; + delta = record_size - byte_count; + + for (Py_ssize_t i = 0; i < count; i++) + { + for (Py_ssize_t j = 0; j < byte_count; j++) + *outptr++ = *inptr++; + inptr += delta; + } } + + data_block = NULL; + + return out; + } } -static PyObject *get_channel_raw_bytes(PyObject *self, PyObject *args) +static PyObject *get_invalidation_bits_array(PyObject *self, PyObject *args) { - Py_ssize_t count, size, actual_byte_count, delta; - PyObject *data_block, *out; + Py_ssize_t count, size, actual_byte_count, delta, invalidation_pos, invalidation_size; + PyObject *data_block, *out; + + Py_ssize_t record_size, byte_offset, byte_count; + + uint8_t mask, *inptr, *outptr; + + if (!PyArg_ParseTuple(args, "Onn", &data_block, &invalidation_size, &invalidation_pos)) + { + return 0; + } + else + { + if (PyBytes_Check(data_block)) { + size = PyBytes_Size(data_block); + inptr = (uint8_t *)PyBytes_AsString(data_block); + } + else { + size = PyByteArray_Size(data_block); + inptr = (uint8_t *)PyByteArray_AsString(data_block); + } - Py_ssize_t record_size, byte_offset, byte_count; + count = size / invalidation_size; + byte_offset = invalidation_pos / 8; + mask = (uint8_t ) (1 << (invalidation_pos % 8)); - char *inptr, *outptr; + inptr += byte_offset; - if (!PyArg_ParseTuple(args, "Onnn", &data_block, &record_size, &byte_offset, &byte_count)) - { - return 0; + npy_intp dims[1]; + dims[0] = count; + out = (PyArrayObject *)PyArray_EMPTY(1, dims, NPY_BOOL, 0); + outptr = (uint8_t *)PyArray_GETPTR1(out, 0); + + for (int i=0; icount; + thread_idx = indata->idx; + thread_count = indata->thread_count; + data = indata->data; + for (Py_ssize_t i = 0; irecord_size < data->byte_offset + data->byte_count) { - if (PyBytes_Check(data_block)) { - size = PyBytes_Size(data_block); - inptr = PyBytes_AsString(data_block); - } - else { - size = PyByteArray_Size(data_block); - inptr = PyByteArray_AsString(data_block); - } - - if (!record_size) + inptr = data->inptr; + delta = data->byte_offset + data->byte_count - data->record_size; + actual_byte_count = data->record_size - data->byte_offset; + + count = data->cycles; + + outptr = data->outptr; + inptr += data->byte_offset; + + for (Py_ssize_t i = 0; i < count; i++) + { + for (Py_ssize_t j = 0; j < actual_byte_count; j++) + *outptr++ = *inptr++; + + inptr += data->record_size - actual_byte_count; + for (Py_ssize_t j = 0; j < delta; j++) { - out = PyByteArray_FromStringAndSize(NULL, 0); + *outptr++ = 0; } - else if (record_size < byte_offset + byte_count) - { - delta = byte_offset + byte_count - record_size; - actual_byte_count = record_size - byte_offset; + } + } + else + { + inptr = data->inptr; + count = data->cycles; + outptr = data->outptr; + inptr += data->byte_offset; + + delta = data->record_size - data->byte_count; + + for (Py_ssize_t i = 0; i < count; i++) + { + for (Py_ssize_t j = 0; j < data->byte_count; j++) + *outptr++ = *inptr++; + inptr += delta; + } + } + + for (Py_ssize_t i = 0; icount; + thread_idx = indata->idx; + data = indata->data; + thread_count= indata->thread_count; + for (Py_ssize_t i = 0; irecord_size; + step = record_size - data->byte_count; + cycles = data->cycles; + byte_count = data->byte_count; + inptr = data->inptr; + + if (!record_size) continue; + + outptr = data->outptr + data->byte_offset; + + for (Py_ssize_t i=0; i size) size = current_size; + } + + return PyLong_FromSsize_t(size); + } +} + + +void transpose(uint8_t * restrict dst, uint8_t * restrict src, uint64_t p, uint64_t n, size_t block) { + for (size_t i = 0; i < n; i += block) { + for(size_t j = 0; j < p; ++j) { + for(size_t b = 0; b < block && i + b < n; ++b) { + dst[j*n + i + b] = src[(i + b)*p + j]; + } + } + } +} + + + +typedef struct InfoBlock { + int64_t address; + int64_t original_size; + int64_t compressed_size; + int64_t block_limit; + int64_t record_offset; + Py_ssize_t param; + Py_ssize_t block_type; + Py_ssize_t idx; + Py_ssize_t count; + +} InfoBlock, *PtrInfoBlock; + + +typedef struct SignalInfo { + int64_t byte_offset; + int64_t byte_count; + int32_t invalidation_bit_position; + uint8_t *data; + PyObject *obj; +} SignalInfo, *PtrSignalInfo; + + +typedef struct ProcessesingBlock { + uint8_t stop; + uint8_t * inptr; + PtrInfoBlock block_info; + struct SignalInfo *signals; + Py_ssize_t signal_count; + Py_ssize_t record_size; + Py_ssize_t idx; +#if defined(_WIN32) + HANDLE bytes_ready; + HANDLE block_ready; +#else + pthread_cond_t bytes_ready; + pthread_cond_t block_ready; + pthread_mutex_t bytes_ready_lock; + pthread_mutex_t block_ready_lock; +#endif +} ProcessesingBlock, *PtrProcessesingBlock; + + +void * get_channel_raw_bytes_complete_C(void *lpParam ) +{ + Py_ssize_t count, byte_count, byte_offset, delta, thread_count, param, block_type; + int64_t original_size, compressed_size, record_offset, block_limit, cycles, current_uncompressed_size=0, current_out_size=0, max_cycles=0; + PtrProcessesingBlock thread_info; + thread_info = (PtrProcessesingBlock) lpParam; + PtrInfoBlock block_info; + + Py_ssize_t signal_count, thread_idx, record_size, in_size, cols, lines; + + record_size = thread_info->record_size; + + int result; + clock_t start, end; + double t1=0, t2=0, t3=0, t4=0, t5=0, t6=0, t7=0; + + uint8_t *outptr, *inptr, *write; + uint8_t *pUncomp=NULL, *pUncompTr=NULL, *read, *data_ptr; + + inptr = thread_info->inptr; + + while (1) { +#if defined(_WIN32) + start = clock(); + WaitForSingleObject(thread_info->block_ready, INFINITE); + ResetEvent(thread_info->block_ready); + end = clock(); + t3 += end - start; + if (thread_info->stop) break; + + /* printf("Thr %d processing\n", thread_info->idx); + printf("Block type=%d\n", thread_info->block_info->block_type); + printf("Block limit=%d\n", thread_info->block_info->block_limit); + printf("Block original_size%d\n", thread_info->block_info->original_size); + printf("Block compressed_size=%d\n", thread_info->block_info->compressed_size); + printf("Block param=%d\n", thread_info->block_info->param); + printf("Block record_offset=%d\n", thread_info->block_info->record_offset); */ + +#else + pthread_mutex_lock(&thread_info->block_ready_lock); + pthread_cond_wait(&thread_info->block_ready, &thread_info->block_ready_lock); + pthread_mutex_unlock(&thread_info->block_ready_lock); +#endif + + original_size = thread_info->block_info->original_size; + compressed_size = thread_info->block_info->compressed_size; + param = thread_info->block_info->param; + block_type = thread_info->block_info->block_type; + record_offset = thread_info->block_info->record_offset; + + count = original_size / record_size; + + if (thread_info->block_info->block_limit >= 0) { + cycles = thread_info->block_info->block_limit / record_size ; + } + else { + cycles = count; + } + + if (block_type == 0) { + data_ptr = inptr; + + } + else { + + // decompress + if (original_size > current_uncompressed_size) { + //printf("\tThr %d new ptr\n", thread_info->idx); + if (pUncomp) free(pUncomp); + pUncomp = (uint8_t *) malloc(original_size); + //if (!pUncomp) printf("\tThr %d pUncomp error\n", thread_info->idx); + current_uncompressed_size=original_size; + } + //printf("\tThr %d start decompress %p %d\n", thread_info->idx, inptr, compressed_size); + struct libdeflate_decompressor *decompressor = libdeflate_alloc_decompressor(); + libdeflate_zlib_decompress(decompressor, + inptr, compressed_size, + pUncomp, original_size, + NULL); + libdeflate_free_decompressor(decompressor); + + //printf("\tThr %d decmpressed\n", thread_info->idx); + + // reverse transposition + if (block_type == 2) { + cols = param; + lines = original_size / cols; + + if (current_out_size < original_size) { + //printf("\tThr %d new trtrtrptr\n", thread_info->idx); + if (pUncompTr) free(pUncompTr); + pUncompTr = (uint8_t *) malloc(original_size); + //if (!pUncompTr) printf("\tThr %d pUncompTr error\n", thread_info->idx); + current_out_size = original_size; + } + + start = clock(); + read = pUncomp; + for (int j = 0; j < (Py_ssize_t)cols; j++) { - current = *idx_array; - if (current) - { - if (current != previous) - { - *(out_array - 1) = 1; - } - *out_array = 1; - } - else - { - if (current != previous && i) - { - *(out_array - 1) = 0; - } - } - previous = current; + write = pUncompTr + j; + for (int i = 0; i < (Py_ssize_t)lines; i++) + { + *write = *read++; + write += cols; + } } + end = clock(); + t7 += end - start; + + + data_ptr = pUncompTr; + + //printf("\tThr %d transposed\n", thread_info->idx); + + } + else { + data_ptr = pUncomp; + } + } + + //printf("\tThr %d %d %d\n", thread_info->idx, cycles, max_cycles); + + for (int i =0; isignal_count; i++) { + byte_offset = thread_info->signals[i].byte_offset; + byte_count = thread_info->signals[i].byte_count; + + read = data_ptr + byte_offset; + write = thread_info->signals[i].data + record_offset * byte_count; + + for (Py_ssize_t j = 0; j < cycles; j++) + { + memcpy(write, read, byte_count); + write += byte_count; + read += record_size; + } + } - return (PyObject *)result; + //printf("\tThr %d set event\n", thread_info->idx); + +#if defined(_WIN32) + SetEvent(thread_info->bytes_ready); +#else + pthread_mutex_lock(&thread_info->bytes_ready_lock); + pthread_cond_signal(&thread_info->bytes_ready); + pthread_mutex_unlock(&thread_info->bytes_ready_lock); +#endif + + } + + if (pUncomp) free(pUncomp); + if (pUncompTr) free(pUncompTr); + //printf("t1=%lf t2=%lf t3=%lf t4=%lf t5=%lf t6=%lf t7=%lf\n", t1, t2, t3, t4, t5, t6, t7); + return 0; } -static PyObject *reverse_transposition(PyObject *self, PyObject *args) + +static PyObject *get_channel_raw_bytes_complete(PyObject *self, PyObject *args) { - Py_ssize_t i = 0, j = 0; - Py_ssize_t count, lines = 0, cols = 0; - PyObject *data, *values; - char *read, *write_original, *write; + Py_ssize_t info_count, signal_count, signal_and_invalidation_count, thread_count=11; + PyObject *data_blocks_info, *signals, *out = NULL, *item, *ref, *obj, *group_index, *InvalidationArray; + + char *outptr, *file_name; + char *read_pos = NULL, *write_pos = NULL; + Py_ssize_t position = 0, record_size = 0, + cycles, step = 0, invalidation_bytes; + Py_ssize_t isize = 0, offset = 0; + int is_list; + int64_t byte_offset, byte_count, new_cycles, max_uncompressed=0, max_compressed=0, record_offset=0; + int32_t invalidation_bit_position; + + PtrInfoBlock block_info; + InfoBlock info_block; + PtrProcessesingBlock thread_info; + PtrProcessesingBlock thread; + + FILE *fptr; + uint8_t *buffer; + int result; + clock_t start, end; + double tt=0; + + if (!PyArg_ParseTuple(args, "OOsnnnO|n", + &data_blocks_info, &signals, &file_name, &cycles, &record_size, &invalidation_bytes, &group_index, + &thread_count)) + { + return NULL; + } + else + { + + ref = PyImport_ImportModule("asammdf"); + InvalidationArray = PyObject_GetAttrString(ref, "InvalidationArray"); + Py_XDECREF(ref); + //fptr = fopen(file_name,"rb"); + +#if defined(_WIN32) + TCHAR *lpFileName = TEXT(file_name); + HANDLE hFile; + HANDLE hMap; + LPVOID lpBasePtr; + LARGE_INTEGER liFileSize; + + hFile = CreateFile(lpFileName, + GENERIC_READ, // dwDesiredAccess + FILE_SHARE_READ, // dwShareMode + NULL, // lpSecurityAttributes + OPEN_EXISTING, // dwCreationDisposition + FILE_FLAG_RANDOM_ACCESS, // dwFlagsAndAttributes + 0); // hTemplateFile + if (hFile == INVALID_HANDLE_VALUE) { + fprintf(stderr, "CreateFile failed with error %d\n", GetLastError()); + return 1; + } - if (!PyArg_ParseTuple(args, "Onn", &data, &lines, &cols)) - { + if (!GetFileSizeEx(hFile, &liFileSize)) { + fprintf(stderr, "GetFileSize failed with error %d\n", GetLastError()); + CloseHandle(hFile); + return 1; + } + + if (liFileSize.QuadPart == 0) { + fprintf(stderr, "File is empty\n"); + CloseHandle(hFile); + return 1; + } + + hMap = CreateFileMapping( + hFile, + NULL, // Mapping attributes + PAGE_READONLY, // Protection flags + 0, // MaximumSizeHigh + 0, // MaximumSizeLow + NULL); // Name + if (hMap == 0) { + fprintf(stderr, "CreateFileMapping failed with error %d\n", GetLastError()); + CloseHandle(hFile); + return 1; + } + + lpBasePtr = MapViewOfFile( + hMap, + FILE_MAP_READ, // dwDesiredAccess + 0, // dwFileOffsetHigh + 0, // dwFileOffsetLow + 0); // dwNumberOfBytesToMap + if (lpBasePtr == NULL) { + fprintf(stderr, "MapViewOfFile failed with error %d\n", GetLastError()); + CloseHandle(hMap); + CloseHandle(hFile); + return 1; + } + + HANDLE *hThreads, *block_ready, *bytes_ready; + DWORD *dwThreadIdArray; + hThreads = (HANDLE *) malloc(sizeof(HANDLE) * thread_count); + dwThreadIdArray = (DWORD *) malloc(sizeof(DWORD) * thread_count); + block_ready = (HANDLE *) malloc(sizeof(HANDLE) * thread_count); + bytes_ready = (HANDLE *) malloc(sizeof(HANDLE) * thread_count); +#else + int fdin = open(file_name, O_RDONLY); + struct stat statbuf; + uint8_t * lpBasePtr; + + fstat (fdin, &statbuf); + + lpBasePtr = mmap (0, statbuf.st_size, PROT_READ, MAP_SHARED, fdin, 0); + + pthread_t *dwThreadIdArray = (pthread_t *) malloc(sizeof(pthread_t) * thread_count); + + pthread_mutex_t *bytes_ready_locks, *block_ready_locks; // Declare mutex + pthread_cond_t *block_ready, *bytes_ready; + + block_ready = (pthread_cond_t *) malloc(sizeof(pthread_cond_t) * thread_count); + bytes_ready = (pthread_cond_t *) malloc(sizeof(pthread_cond_t) * thread_count); + bytes_ready_locks = (pthread_mutex_t *) malloc(sizeof(pthread_mutex_t) * thread_count); + block_ready_locks = (pthread_mutex_t *) malloc(sizeof(pthread_mutex_t) * thread_count); +#endif + + PtrSignalInfo signal_info; + + is_list = PyList_Check(signals); + if (is_list) { + signal_count = PyList_Size(signals); + } + else { + signal_count = PyTuple_Size(signals); + } + + if (invalidation_bytes) { + signal_and_invalidation_count = signal_count +1; + signal_info = (PtrSignalInfo) malloc(sizeof(SignalInfo) * (signal_count + 1)); + if (!signal_info) { + PyErr_SetString(PyExc_ValueError, "Memmory allocation error for signal_info\n\0"); return NULL; + } } - else + else { + signal_and_invalidation_count = signal_count; + signal_info = (PtrSignalInfo) malloc(sizeof(SignalInfo) * signal_count); + if (!signal_info) { + PyErr_SetString(PyExc_ValueError, "Memmory allocation error for signal_info\n\0"); + return NULL; + } + } + for (int i=0; i= thread_count) { +#if defined(_WIN32) + start = clock(); + WaitForSingleObject(bytes_ready[position], INFINITE); + end = clock(); + tt += end - start; + ResetEvent(bytes_ready[position]); +#else + pthread_mutex_lock(&bytes_ready_locks[position]); + pthread_cond_wait(&bytes_ready[position], &bytes_ready_locks[position]); + pthread_mutex_unlock(&bytes_ready_locks[position]); +#endif + } + + thread->block_info = &block_info[i]; + thread->block_info->record_offset = record_offset; + record_offset += block_info[i].original_size / record_size; + memcpy(thread->inptr, ((uint8_t*)lpBasePtr) + block_info[i].address, block_info[i].compressed_size); - return values; + //FSEEK64(fptr, block_info[i].address, 0); + //result = fread(thread->inptr, 1, block_info[i].compressed_size, fptr); + +#if defined(_WIN32) + SetEvent(block_ready[position]); +#else + pthread_mutex_lock(&block_ready_locks[position]); + pthread_cond_signal(&block_ready[position]); + pthread_mutex_unlock(&block_ready_locks[position]); +#endif + + position++; + if (position == thread_count) position = 0; + + } + + //printf("TT=%lf\n", tt); + + for (int i=0; istop = 1; + +#if defined(_WIN32) + SetEvent(block_ready[position]); +#else + pthread_mutex_lock(&block_ready_locks[position]); + pthread_cond_signal(&block_ready[position]); + pthread_mutex_unlock(&block_ready_locks[position]); +#endif + + position++; + if (position == thread_count) position = 0; + } + +#if defined(_WIN32) + UnmapViewOfFile(lpBasePtr); + CloseHandle(hMap); + CloseHandle(hFile); + + WaitForMultipleObjects(thread_count, hThreads, true, INFINITE); + for (int i=0; i< thread_count; i++) { + CloseHandle(hThreads[i]); + CloseHandle(block_ready[i]); + CloseHandle(bytes_ready[i]); + } +#else + munmap(lpBasePtr, statbuf.st_size); + close(fdin); + for (int i=0; i< thread_count; i++) { + pthread_join(dwThreadIdArray[i], NULL); + } +#endif + + for (int i=0; i size) size = current_size; + + PyObject **cache = malloc(sizeof(PyObject *) * invalidation_bytes * 8); + if (!cache) { + PyErr_SetString(PyExc_ValueError, "Failed to allocate memory for cache\n\0"); + return NULL; + } + for (int i=0; i< invalidation_bytes * 8; i++) cache[i] = NULL; + + for (int i=0; i None: inval_total_size=inval_total_size, record_size=record_size, ) - data_blocks = [] + data_blocks = list(data_blocks_info) # load the info blocks directly here uses_ld = self._uses_ld( address=address, stream=stream, @@ -826,7 +830,7 @@ def _read_channels( stream: ReadableBufferType, dg_cntr: int, ch_cntr: int, - channel_composition: bool = False, + parent_channel: Channel | None = None, mapped: bool = False, ) -> tuple[int, list[tuple[int, int]] | None, dtype | None]: filter_channels = self.use_load_filter @@ -838,7 +842,7 @@ def _read_channels( unique_names = UniqueDB() - if channel_composition: + if parent_channel: composition = [] composition_channels = [] @@ -903,7 +907,7 @@ def _read_channels( display_names = {_name.split(path_separator, 1)[0]: val for _name, val in display_names.items()} if ( - channel_composition + parent_channel or channel_type in v4c.MASTER_TYPES or name in self.load_filter or (use_display_names and any(dsp_name in self.load_filter for dsp_name in display_names)) @@ -946,7 +950,7 @@ def _read_channels( stream, dg_cntr, ch_cntr, - False, + None, mapped=mapped, ) @@ -987,7 +991,7 @@ def _read_channels( self._ch_map[ch_addr] = entry channels.append(channel) - if channel_composition: + if parent_channel: composition.append(entry) composition_channels.append(channel) @@ -1036,7 +1040,7 @@ def _read_channels( stream, dg_cntr, ch_cntr, - True, + channel, mapped=mapped, ) dependencies[index] = ret_composition @@ -1080,10 +1084,12 @@ def _read_channels( stream, dg_cntr, ch_cntr, - True, + channel, mapped=mapped, ) + channel.dtype_fmt = ret_composition_dtype + ca_cnt = len(dependencies[index]) if ret_composition: dependencies[index].extend(ret_composition) @@ -1205,11 +1211,20 @@ def _get_name_with_indices(ch_name: str, ch_parent_name: str, indices: list[int] # go to next channel of the current channel group ch_addr = channel.next_ch_addr - if channel_composition: - composition_channels.sort() - composition_dtype = dtype( - [(unique_names.get_unique_name(channel.name), channel.dtype_fmt) for channel in composition_channels] - ) + if parent_channel: + composition_channels.sort(key=lambda x: x.byte_offset) + padding = 0 + dtype_fields = [] + offset = parent_channel.byte_offset + + for comp_channel in composition_channels: + if (delta := (comp_channel.byte_offset - offset)) > 0: + dtype_fields.append((f"__padding_{padding}__", f"V{delta}")) + padding += 1 + dtype_fields.append((unique_names.get_unique_name(comp_channel.name), comp_channel.dtype_fmt)) + offset = comp_channel.byte_offset + comp_channel.dtype_fmt.itemsize + + composition_dtype = dtype(dtype_fields) else: composition = None @@ -1381,6 +1396,10 @@ def _load_data( ) -> Iterator[tuple[bytes, int, int, bytes | None]]: """get group's data block bytes""" + from time import perf_counter + + cc = 0 + offset = 0 invalidation_offset = 0 has_yielded = False @@ -1413,12 +1432,15 @@ def _load_data( if record_count is not None: invalidation_record_count = record_count * invalidation_size record_count *= samples_size + max_size = record_count + invalidation_record_count + else: + max_size = (invalidation_size + samples_size) * channel_group.cycles_nr if not samples_size: if rm: - yield b"", offset, _count, b"" + yield Fragment(b"", offset, _count, b"") else: - yield b"", offset, _count, None + yield Fragment(b"", offset, _count, None) else: if group.read_split_count: split_size = group.read_split_count * samples_size @@ -1447,6 +1469,9 @@ def _load_data( invalidation_split_size = invalidation_size split_size = int(split_size) + if split_size > max_size: + invalidation_split_size = (max_size // samples_size) * invalidation_size + split_size = max_size buffer = bytearray(split_size) buffer_view = memoryview(buffer) @@ -1461,6 +1486,10 @@ def _load_data( cur_invalidation_size = 0 invalidation_data = [] + tt = perf_counter() + ss = 0 + cc = 0 + while True: try: info = next(blocks) @@ -1524,6 +1553,9 @@ def _load_data( seek(address) new_data = read(compressed_size) + cc += 1 + ss += original_size + if block_type == v4c.DZ_BLOCK_DEFLATE: new_data = decompress(new_data, bufsize=original_size) elif block_type == v4c.DZ_BLOCK_TRANSPOSED: @@ -1608,14 +1640,17 @@ def _load_data( if rm and invalidation_size: __data = buffer[:record_count] _count = len(__data) // samples_size - yield __data, offset // samples_size, _count, invalidation_data_[ - :invalidation_record_count - ] + yield Fragment( + __data, + offset // samples_size, + _count, + invalidation_data_[:invalidation_record_count], + ) invalidation_record_count -= len(invalidation_data_) else: __data = buffer[:record_count] _count = len(__data) // samples_size - yield __data, offset // samples_size, _count, None + yield Fragment(__data, offset // samples_size, _count, None) has_yielded = True record_count -= split_size if record_count <= 0: @@ -1624,10 +1659,10 @@ def _load_data( else: if rm and invalidation_size: _count = split_size // samples_size - yield buffer, offset // samples_size, _count, invalidation_data_ + yield Fragment(buffer, offset // samples_size, _count, invalidation_data_) else: _count = split_size // samples_size - yield buffer, offset // samples_size, _count, None + yield Fragment(buffer, offset // samples_size, _count, None) has_yielded = True else: @@ -1639,14 +1674,17 @@ def _load_data( if record_count is not None: if rm and invalidation_size: - yield buffer[:record_count], offset // samples_size, _count, invalidation_data_[ - :invalidation_record_count - ] + yield Fragment( + buffer[:record_count], + offset // samples_size, + _count, + invalidation_data_[:invalidation_record_count], + ) invalidation_record_count -= len(invalidation_data_) else: __data = buffer[:record_count] _count = len(__data) // samples_size - yield __data, offset // samples_size, _count, None + yield Fragment(__data, offset // samples_size, _count, None) has_yielded = True record_count -= split_size if record_count <= 0: @@ -1655,10 +1693,10 @@ def _load_data( else: if rm and invalidation_size: _count = split_size // samples_size - yield buffer, offset // samples_size, _count, invalidation_data_ + yield Fragment(buffer, offset // samples_size, _count, invalidation_data_) else: _count = split_size // samples_size - yield buffer, offset // samples_size, _count, None + yield Fragment(buffer, offset // samples_size, _count, None) has_yielded = True offset += split_size @@ -1685,6 +1723,12 @@ def _load_data( invalidation_data.append(new_invalidation_data) cur_invalidation_size += inv_size + if (vv := (perf_counter() - tt)) > 10: + print(f"{ss / 1024/1024 / vv:.6f} MB/s {cc=} {vv=}") + cc = 0 + ss = 0 + tt = perf_counter() + if cur_size: data_ = buffer[:cur_size] if rm and invalidation_size: @@ -1693,28 +1737,30 @@ def _load_data( if rm and invalidation_size: __data = data_[:record_count] _count = len(__data) // samples_size - yield __data, offset // samples_size, _count, invalidation_data_[:invalidation_record_count] + yield Fragment( + __data, offset // samples_size, _count, invalidation_data_[:invalidation_record_count] + ) invalidation_record_count -= len(invalidation_data_) else: __data = data_[:record_count] _count = len(__data) // samples_size - yield __data, offset // samples_size, _count, None + yield Fragment(__data, offset // samples_size, _count, None) has_yielded = True record_count -= len(data_) else: if rm and invalidation_size: _count = len(data_) // samples_size - yield data_, offset // samples_size, _count, invalidation_data_ + yield Fragment(data_, offset // samples_size, _count, invalidation_data_) else: _count = len(data_) // samples_size - yield data_, offset // samples_size, _count, None + yield Fragment(data_, offset // samples_size, _count, None) has_yielded = True if not has_yielded: if rm and invalidation_size: - yield b"", 0, 0, b"" + yield Fragment(b"", 0, 0, b"") else: - yield b"", 0, 0, None + yield Fragment(b"", 0, 0, None) def _prepare_record(self, group: Group) -> list: """compute record @@ -1746,7 +1792,7 @@ def _prepare_record(self, group: Group) -> list: ch_type = new_ch.channel_type dependency_list = group.channel_dependencies[idx] - if ch_type not in v4c.VIRTUAL_TYPES and not dependency_list: + if ch_type not in v4c.VIRTUAL_TYPES: # adjust size to 1, 2, 4 or 8 bytes size = bit_offset + bit_count @@ -1769,7 +1815,11 @@ def _prepare_record(self, group: Group) -> list: if not new_ch.dtype_fmt: new_ch.dtype_fmt = dtype(get_fmt_v4(data_type, size, ch_type)) - if bit_offset or (new_ch.dtype_fmt.kind in "ui" and size < 64 and size not in (8, 16, 32)): + if ( + bit_offset + or dependency_list + or (new_ch.dtype_fmt.kind in "ui" and size < 64 and size not in (8, 16, 32)) + ): new_ch.standard_C_size = False record.append( @@ -1850,12 +1900,14 @@ def _get_data_blocks_info( ) -> Iterator[DataBlockInfo]: mapped = mapped or not is_file_like(stream) - if record_size > 32 * 1024 * 1024: + if record_size > 4 * 1024 * 1024: READ_CHUNK_SIZE = record_size elif record_size: - READ_CHUNK_SIZE = 32 * 1024 * 1024 // record_size * record_size + READ_CHUNK_SIZE = 4 * 1024 * 1024 // record_size * record_size else: - READ_CHUNK_SIZE = 32 * 1024 * 1024 + READ_CHUNK_SIZE = 4 * 1024 * 1024 + + READ_CHUNK_SIZE = min(READ_CHUNK_SIZE, total_size) if mapped: if address: @@ -1865,6 +1917,8 @@ def _get_data_blocks_info( if id_string == block_type: size = block_len - 24 if size: + size = min(size, total_size) + address = address + COMMON_SIZE # split the DTBLOCK into chucks of up to 32MB @@ -1932,14 +1986,16 @@ def _get_data_blocks_info( while address: dl = DataList(address=address, stream=stream, mapped=mapped) for i in range(dl.data_block_nr): - addr = dl[f"data_block_addr{i}"] + addr = getattr(dl, f"data_block_addr{i}") id_string, block_len = COMMON_SHORT_uf(stream, addr) # can be a DataBlock if id_string == block_type: size = block_len - 24 if size: - addr = addr + COMMON_SIZE + size = min(size, total_size) + + addr += COMMON_SIZE # split the DTBLOCK into chucks of up to 32MB while True: @@ -1975,7 +2031,7 @@ def _get_data_blocks_info( break # or a DataZippedBlock - elif id_string == b"##DZ": + else: ( zip_type, param, @@ -2011,7 +2067,7 @@ def _get_data_blocks_info( ld = ListData(address=address, stream=stream, mapped=mapped) has_invalidation = ld.flags & v4c.FLAG_LD_INVALIDATION_PRESENT for i in range(ld.data_block_nr): - addr = ld[f"data_block_addr_{i}"] + addr = getattr(ld, f"data_block_addr_{i}") id_string, block_len = COMMON_SHORT_uf(stream, addr) # can be a DataBlock @@ -2148,6 +2204,7 @@ def _get_data_blocks_info( if id_string == block_type: size = block_len - 24 if size: + size = min(size, total_size) address = address + COMMON_SIZE # split the DTBLOCK into chucks of up to 32MB @@ -2216,7 +2273,7 @@ def _get_data_blocks_info( while address: dl = DataList(address=address, stream=stream) for i in range(dl.data_block_nr): - addr = dl[f"data_block_addr{i}"] + addr = getattr(dl, f"data_block_addr{i}") stream.seek(addr) id_string, block_len = COMMON_SHORT_u(stream.read(COMMON_SHORT_SIZE)) @@ -2299,7 +2356,7 @@ def _get_data_blocks_info( ld = ListData(address=address, stream=stream) has_invalidation = ld.flags & v4c.FLAG_LD_INVALIDATION_PRESENT for i in range(ld.data_block_nr): - addr = ld[f"data_block_addr{i}"] + addr = getattr(ld, f"data_block_addr{i}") stream.seek(addr) id_string, block_len = COMMON_SHORT_u(stream.read(COMMON_SHORT_SIZE)) @@ -2579,7 +2636,7 @@ def _filter_occurrences( def get_invalidation_bits( self, group_index: int, - channel: Channel, + pos_invalidation_bit: int, fragment: tuple[bytes, int, int, ReadableBufferType | None], ) -> NDArray[bool_]: """get invalidation indexes for the channel @@ -2588,8 +2645,8 @@ def get_invalidation_bits( ---------- group_index : int group index - channel : Channel - channel object + pos_invalidation_bit : int + channel invalidation bit position fragment : (bytes, int) (fragment bytes, fragment offset) @@ -2602,36 +2659,37 @@ def get_invalidation_bits( """ group = self.groups[group_index] - data_bytes, offset, _count, invalidation_bytes = fragment - try: - invalidation = self._invalidation_cache[(group_index, offset, _count)] - except KeyError: - size = group.channel_group.invalidation_bytes_nr - - if invalidation_bytes is None: - record = group.record - if record is None: - self._prepare_record(group) - - invalidation_bytes = get_channel_raw_bytes( - data_bytes, - group.channel_group.samples_byte_nr + group.channel_group.invalidation_bytes_nr, - group.channel_group.samples_byte_nr, - size, - ) - - invalidation = frombuffer(invalidation_bytes, dtype=f"({size},)u1") - self._invalidation_cache[(group_index, offset, _count)] = invalidation - - ch_invalidation_pos = channel.pos_invalidation_bit - pos_byte, pos_offset = divmod(ch_invalidation_pos, 8) + data_bytes, offset, _count, invalidation_bytes = ( + fragment.data, + fragment.record_offset, + fragment.record_count, + fragment.invalidation_data, + ) - mask = 1 << pos_offset + if invalidation_bytes is None: + invalidation_bytes_nr = group.channel_group.invalidation_bytes_nr + samples_byte_nr = group.channel_group.samples_byte_nr + record = group.record + if record is None: + self._prepare_record(group) + + invalidation_bytes = get_channel_raw_bytes( + data_bytes, + samples_byte_nr + invalidation_bytes_nr, + samples_byte_nr, + invalidation_bytes_nr, + ) - invalidation_bits = invalidation[:, pos_byte] & mask - invalidation_bits = invalidation_bits.view(bool) + key = (group_index, offset, _count, pos_invalidation_bit) + if key not in self._invalidation_cache: + self._invalidation_cache[key] = InvalidationArray( + get_invalidation_bits_array( + invalidation_bytes, group.channel_group.invalidation_bytes_nr, pos_invalidation_bit + ), + (group_index, pos_invalidation_bit), + ) - return InvalidationArray(invalidation_bits, (group_index, ch_invalidation_pos)) + return self._invalidation_cache[key] def append( self, @@ -2860,7 +2918,7 @@ def append( ch_cntr += 1 - gp_sig_types.append(v4c.SIGNAL_TYPE_VIRTUAL) + gp_sig_types.append((v4c.SIGNAL_TYPE_VIRTUAL, 0)) else: t_type, t_size = fmt_to_datatype_v4(t.dtype, t.shape) @@ -2905,7 +2963,7 @@ def append( offset += t_size // 8 ch_cntr += 1 - gp_sig_types.append(v4c.SIGNAL_TYPE_SCALAR) + gp_sig_types.append((v4c.SIGNAL_TYPE_SCALAR, t_size // 8)) for signal in signals: sig = signal @@ -2931,8 +2989,6 @@ def append( else: sig_type = v4c.SIGNAL_TYPE_ARRAY - gp_sig_types.append(sig_type) - # first add the signals in the simple signal list if sig_type == v4c.SIGNAL_TYPE_SCALAR: # compute additional byte offset for large records size @@ -3031,6 +3087,7 @@ def append( 0, ) ) + gp_sig_types.append((sig_type, 8)) offset += byte_size @@ -3052,6 +3109,8 @@ def append( byte_size = s_size // 8 or 1 data_block_addr = 0 + gp_sig_types.append((sig_type, byte_size)) + if sig_dtype.kind == "u" and signal.bit_count <= 4: s_size = signal.bit_count @@ -3205,6 +3264,7 @@ def append( ch.source = new_source gp_channels.append(ch) + gp_sig_types.append((sig_type, 0)) gp_sdata.append(None) entry = (dg_cntr, ch_cntr) @@ -3236,6 +3296,7 @@ def append( byte_size = 6 s_type = v4c.DATA_TYPE_CANOPEN_TIME s_dtype = dtype("V6") + gp_sig_types.append((sig_type, 6)) else: record.append( @@ -3263,6 +3324,7 @@ def append( byte_size = 7 s_type = v4c.DATA_TYPE_CANOPEN_DATE s_dtype = dtype("V7") + gp_sig_types.append((sig_type, 7)) s_size = byte_size * 8 @@ -3339,6 +3401,7 @@ def append( inval_bits, ) fields.extend(new_fields) + gp_sig_types.append((sig_type, signal.samples.dtype.itemsize)) elif sig_type == v4c.SIGNAL_TYPE_ARRAY: # here we have channel arrays or mdf v3 channel dependencies @@ -3457,6 +3520,7 @@ def append( if not samples.flags["C_CONTIGUOUS"]: samples = np.ascontiguousarray(samples) fields.append((samples, size)) + gp_sig_types.append((sig_type, size)) gp_sdata.append(None) entry = (dg_cntr, ch_cntr) @@ -3638,6 +3702,7 @@ def append( # compute additional byte offset for large records size byte_size = 8 + gp_sig_types.append((sig_type, 8)) kwargs = { "channel_type": v4c.CHANNEL_TYPE_VLSD, "bit_count": 64, @@ -3757,22 +3822,24 @@ def append( gp.sorted = True - samples = data_block_from_arrays(fields, cycles_nr) + samples = data_block_from_arrays(fields, cycles_nr, THREAD_COUNT) size = len(samples) samples = memoryview(samples) del fields + record_size = offset + invalidation_bytes_nr + if size: if self.version < "4.20": - block_size = self._write_fragment_size or 20 * 1024 * 1024 + block_size = 32 * 1024 * 1024 // record_size * record_size count = ceil(size / block_size) for i in range(count): data_ = samples[i * block_size : (i + 1) * block_size] raw_size = len(data_) - data_ = lz_compress(data_) + data_ = lz_compress(data_, store_size=True) size = len(data_) data_address = self._tempfile.tell() @@ -3789,13 +3856,12 @@ def append( ) else: - data_address = self._tempfile.tell() gp.uses_ld = True data_address = tell() data = samples raw_size = len(data) - data = lz_compress(data) + data = lz_compress(data, store_size=True) size = len(data) self._tempfile.write(data) @@ -3814,7 +3880,7 @@ def append( addr = tell() data = inval_bits.tobytes() raw_size = len(data) - data = lz_compress(data) + data = lz_compress(data, store_size=True) size = len(data) self._tempfile.write(data) @@ -4613,7 +4679,7 @@ def _append_column_oriented( data = samples.tobytes() raw_size = len(data) - data = lz_compress(data) + data = lz_compress(data, store_size=True) size = len(data) write(data) @@ -4632,7 +4698,7 @@ def _append_column_oriented( addr = tell() data = invalidation_bits.tobytes() raw_size = len(data) - data = lz_compress(data) + data = lz_compress(data, store_size=True) size = len(data) write(data) @@ -5855,25 +5921,23 @@ def extend(self, index: int, signals: list[tuple[NDArray[Any], NDArray[Any] | No added_cycles = len(signals[0][0]) invalidation_bytes_nr = gp.channel_group.invalidation_bytes_nr - for i, ((signal, invalidation_bits), sig_type) in enumerate(zip(signals, gp.signal_types)): - if invalidation_bits is not None and not isinstance(invalidation_bits, InvalidationArray): - invalidation_bits = InvalidationArray(invalidation_bits) + for i, ((signal, invalidation_bits), (sig_type, sig_size)) in enumerate(zip(signals, gp.signal_types)): + if invalidation_bytes_nr: + if invalidation_bits is not None: + if not isinstance(invalidation_bits, InvalidationArray): + invalidation_bits = InvalidationArray(invalidation_bits) + if (origin := invalidation_bits.origin) == InvalidationArray.ORIGIN_UNKNOWN: + inval_bits[origin].append(invalidation_bits) + else: + inval_bits[origin] = invalidation_bits # first add the signals in the simple signal list if sig_type == v4c.SIGNAL_TYPE_SCALAR: - s_type, s_size = fmt_to_datatype_v4(signal.dtype, signal.shape) - byte_size = s_size // 8 or 1 if not signal.flags["C_CONTIGUOUS"]: signal = np.ascontiguousarray(signal) - fields.append((signal, byte_size)) - - if invalidation_bytes_nr and invalidation_bits is not None: - if (origin := invalidation_bits.origin) == InvalidationArray.ORIGIN_UNKNOWN: - inval_bits[origin].append(invalidation_bits) - else: - inval_bits[origin] = invalidation_bits + fields.append((signal, sig_size)) elif sig_type == v4c.SIGNAL_TYPE_CANOPEN: names = signal.dtype.names @@ -5882,7 +5946,7 @@ def extend(self, index: int, signals: list[tuple[NDArray[Any], NDArray[Any] | No if not signal.flags["C_CONTIGUOUS"]: signal = np.ascontiguousarray(signal) - fields.append((signal, 6)) + fields.append((signal, sig_size)) else: vals = [] @@ -5890,47 +5954,24 @@ def extend(self, index: int, signals: list[tuple[NDArray[Any], NDArray[Any] | No vals.append(signal[field]) vals = np.rec.fromarrays(vals).tobytes() - fields.append((vals, 7)) - - if invalidation_bytes_nr and invalidation_bits is not None: - if (origin := invalidation_bits.origin) == InvalidationArray.ORIGIN_UNKNOWN: - inval_bits[origin].append(invalidation_bits) - else: - inval_bits[origin] = invalidation_bits + fields.append((vals, sig_size)) elif sig_type == v4c.SIGNAL_TYPE_STRUCTURE_COMPOSITION: - if invalidation_bytes_nr and invalidation_bits is not None: - if (origin := invalidation_bits.origin) == InvalidationArray.ORIGIN_UNKNOWN: - inval_bits[origin].append(invalidation_bits) - else: - inval_bits[origin] = invalidation_bits if not signal.flags["C_CONTIGUOUS"]: signal = np.ascontiguousarray(signal) - fields.append((signal, signal.dtype.itemsize)) + fields.append((signal, sig_size)) elif sig_type == v4c.SIGNAL_TYPE_ARRAY: names = signal.dtype.names samples = signal[names[0]] - shape = samples.shape[1:] - s_type, s_size = fmt_to_datatype_v4(samples.dtype, samples.shape, True) - size = s_size // 8 - for dim in shape: - size *= dim - if not samples.flags["C_CONTIGUOUS"]: samples = np.ascontiguousarray(samples) - fields.append((samples, size)) - - if invalidation_bytes_nr and invalidation_bits is not None: - if (origin := invalidation_bits.origin) == InvalidationArray.ORIGIN_UNKNOWN: - inval_bits[origin].append(invalidation_bits) - else: - inval_bits[origin] = invalidation_bits + fields.append((samples, sig_size)) for name in names[1:]: samples = signal[name] @@ -5945,12 +5986,6 @@ def extend(self, index: int, signals: list[tuple[NDArray[Any], NDArray[Any] | No fields.append((samples, size)) - if invalidation_bytes_nr and invalidation_bits is not None: - if (origin := invalidation_bits.origin) == InvalidationArray.ORIGIN_UNKNOWN: - inval_bits[origin].append(invalidation_bits) - else: - inval_bits[origin] = invalidation_bits - else: if self.compact_vlsd: cur_offset = sum(blk.original_size for blk in gp.get_signal_data_blocks(i)) @@ -6024,75 +6059,78 @@ def extend(self, index: int, signals: list[tuple[NDArray[Any], NDArray[Any] | No offsets = np.ascontiguousarray(offsets) fields.append((offsets, 8)) - if invalidation_bytes_nr and invalidation_bits is not None: - if (origin := invalidation_bits.origin) == InvalidationArray.ORIGIN_UNKNOWN: - inval_bits[origin].append(invalidation_bits) - else: - inval_bits[origin] = invalidation_bits - if invalidation_bytes_nr: unknown_origin = inval_bits.pop(InvalidationArray.ORIGIN_UNKNOWN) - _pos_map = {key: idx for idx, key in enumerate(inval_bits)} - - _unknown_pos_map = deque(list(range(len(inval_bits), len(inval_bits) + len(unknown_origin)))) - inval_bits = list(inval_bits.values()) + unknown_origin cycles_nr = len(inval_bits[0]) - invalidation_bytes_nr = len(inval_bits) + new_invalidation_bytes_nr = len(inval_bits) - for _ in range(8 - invalidation_bytes_nr % 8): + for _ in range(8 - new_invalidation_bytes_nr % 8): inval_bits.append(zeros(cycles_nr, dtype=bool)) inval_bits.reverse() - invalidation_bytes_nr = len(inval_bits) // 8 + new_invalidation_bytes_nr = len(inval_bits) // 8 - gp.channel_group.invalidation_bytes_nr = invalidation_bytes_nr + if new_invalidation_bytes_nr != invalidation_bytes_nr: + raise MdfException( + "The invalidation bytes number in the extend methods differs from the one from the append" + ) inval_bits = np.fliplr(np.packbits(array(inval_bits).T).reshape((cycles_nr, invalidation_bytes_nr))).ravel() if self.version < "4.20": fields.append((inval_bits, invalidation_bytes_nr)) - samples = data_block_from_arrays(fields, added_cycles) + samples = data_block_from_arrays(fields, added_cycles, THREAD_COUNT) size = len(samples) + samples = memoryview(samples) del fields stream.seek(0, 2) addr = stream.tell() + record_size = gp.channel_group.samples_byte_nr + gp.channel_group.invalidation_bytes_nr + if size: if self.version < "4.20": - data = samples - raw_size = size - data = lz_compress(data) - size = len(data) - stream.write(data) - gp.data_blocks.append( - DataBlockInfo( - address=addr, - block_type=v4c.DZ_BLOCK_LZ, - original_size=raw_size, - compressed_size=size, - param=0, + block_size = 32 * 1024 * 1024 // record_size * record_size + + count = ceil(size / block_size) + + for i in range(count): + data_ = samples[i * block_size : (i + 1) * block_size] + raw_size = len(data_) + data_ = lz_compress(data_, store_size=True) + + size = len(data_) + data_address = self._tempfile.tell() + self._tempfile.write(data_) + + gp.data_blocks.append( + DataBlockInfo( + address=data_address, + block_type=v4c.DZ_BLOCK_LZ, + original_size=raw_size, + compressed_size=size, + param=0, + ) ) - ) gp.channel_group.cycles_nr += added_cycles self.virtual_groups[index].cycles_nr += added_cycles else: - data = samples raw_size = size - data = lz_compress(data) + data = lz_compress(samples, store_size=True) size = len(data) stream.write(data) gp.data_blocks.append( DataBlockInfo( address=addr, - block_type=v4c.DT_BLOCK_LZ, + block_type=v4c.DZ_BLOCK_LZ, original_size=raw_size, compressed_size=size, param=0, @@ -6107,17 +6145,17 @@ def extend(self, index: int, signals: list[tuple[NDArray[Any], NDArray[Any] | No data = inval_bits.tobytes() raw_size = len(data) - data = lz_compress(data) + data = lz_compress(data, store_size=True) size = len(data) stream.write(data) gp.data_blocks[-1].invalidation_block( InvalidationBlockInfo( address=addr, - block_type=v4c.DT_BLOCK_LZ, + block_type=v4c.DZ_BLOCK_LZ, original_size=raw_size, compressed_size=size, - param=None, + param=0, ) ) @@ -6229,7 +6267,7 @@ def _extend_column_oriented(self, index: int, signals: list[tuple[NDArray[Any], if added_cycles: data = samples.tobytes() raw_size = len(data) - data = lz_compress(data) + data = lz_compress(data, store_size=True) size = len(data) write(data) @@ -6250,7 +6288,7 @@ def _extend_column_oriented(self, index: int, signals: list[tuple[NDArray[Any], addr = tell() data = invalidation_bits.tobytes() raw_size = len(data) - data = lz_compress(data) + data = lz_compress(data, store_size=True) size = len(data) write(data) @@ -6842,19 +6880,28 @@ def get( ) else: - vals, timestamps, invalidation_bits, encoding = self._get_scalar( - channel=channel, - group=grp, - group_index=gp_nr, - channel_index=ch_nr, - dependency_list=dependency_list, - raster=raster, - data=data, - ignore_invalidation_bits=ignore_invalidation_bits, - record_offset=record_offset, - record_count=record_count, - master_is_required=master_is_required, - ) + if ( + data + and (fast_path := channel.fast_path) is not None + and not master_is_required + and ignore_invalidation_bits + and not raster + ): + vals, timestamps, invalidation_bits, encoding = self._fast_scalar_path(*fast_path, data) + else: + vals, timestamps, invalidation_bits, encoding = self._get_scalar( + channel=channel, + group=grp, + group_index=gp_nr, + channel_index=ch_nr, + dependency_list=dependency_list, + raster=raster, + data=data, + ignore_invalidation_bits=ignore_invalidation_bits, + record_offset=record_offset, + record_count=record_count, + master_is_required=master_is_required, + ) if all_invalid: invalidation_bits = np.ones(len(vals), dtype=bool) @@ -6934,6 +6981,9 @@ def get( debug_channel(self, grp, channel, dependency_list) raise + if data is None: + self._invalidation_cache.clear() + return res def _get_structure( @@ -6985,7 +7035,7 @@ def _get_structure( count = 0 for fragment in data: - bts = fragment[0] + bts = fragment.data buffer = get_channel_raw_bytes(bts, record_size, byte_offset, _dtype.itemsize) @@ -6994,7 +7044,7 @@ def _get_structure( if master_is_required: timestamps.append(self.get_master(gp_nr, fragment, one_piece=True)) if channel_invalidation_present: - invalidation_bits.append(self.get_invalidation_bits(gp_nr, channel, fragment)) + invalidation_bits.append(self.get_invalidation_bits(gp_nr, channel.pos_invalidation_bit, fragment)) count += 1 else: @@ -7023,7 +7073,7 @@ def _get_structure( if master_is_required: timestamps.append(self.get_master(gp_nr, fragment, one_piece=True)) if channel_invalidation_present: - invalidation_bits.append(self.get_invalidation_bits(gp_nr, channel, fragment)) + invalidation_bits.append(self.get_invalidation_bits(gp_nr, channel.pos_invalidation_bit, fragment)) count += 1 @@ -7204,7 +7254,12 @@ def _get_array( arrays = [] types = [] - data_bytes, offset, _count, invalidation_bytes = fragment + data_bytes, offset, _count, invalidation_bytes = ( + fragment.data, + fragment.record_offset, + fragment.record_count, + fragment.invalidation_data, + ) cycles = len(data_bytes) // samples_size @@ -7388,7 +7443,7 @@ def _get_array( if master_is_required: timestamps.append(self.get_master(gp_nr, fragment, one_piece=True)) if channel_invalidation_present: - invalidation_bits.append(self.get_invalidation_bits(gp_nr, channel, fragment)) + invalidation_bits.append(self.get_invalidation_bits(gp_nr, channel.pos_invalidation_bit, fragment)) channel_values.append(vals) count += 1 @@ -7445,6 +7500,38 @@ def _get_array( return vals, timestamps, invalidation_bits, None + def _fast_scalar_path( + self, + gp_nr, + record_size, + byte_offset, + byte_size, + pos_invalidation_bit, + # data_type, + # channel_type, + # bit_count, + dtype, + fragment, + ): + if fragment.is_record: + buffer = get_channel_raw_bytes( + fragment.data, + record_size, + byte_offset, + byte_size, + ) + else: + buffer = fragment.data + + vals = frombuffer(buffer, dtype=dtype) + + if pos_invalidation_bit >= 0: + invalidation_bits = self.get_invalidation_bits(gp_nr, pos_invalidation_bit, fragment) + else: + invalidation_bits = None + + return vals, None, invalidation_bits, None + def _get_scalar( self, channel: Channel, @@ -7460,10 +7547,8 @@ def _get_scalar( master_is_required: bool, skip_vlsd: bool = False, ) -> tuple[NDArray[Any], NDArray[Any] | None, NDArray[Any] | None, str | None]: - grp = group - gp_nr = group_index - ch_nr = channel_index + grp = group # get group data if data is None: data = self._load_data(grp, record_offset=record_offset, record_count=record_count) @@ -7471,6 +7556,9 @@ def _get_scalar( else: one_piece = True + gp_nr = group_index + ch_nr = channel_index + channel_invalidation_present = channel.flags & (v4c.FLAG_CN_ALL_INVALID | v4c.FLAG_CN_INVALIDATION_PRESENT) data_type = channel.data_type @@ -7504,7 +7592,12 @@ def _get_scalar( data = (data,) for fragment in data: - data_bytes, offset, _count, invalidation_bytes = fragment + data_bytes, offset, _count, invalidation_bytes = ( + fragment.data, + fragment.record_offset, + fragment.record_count, + fragment.invalidation_data, + ) offset = offset // record_size vals = arange(len(data_bytes) // record_size, dtype=ch_dtype) @@ -7521,7 +7614,7 @@ def _get_scalar( ) ) if channel_invalidation_present: - invalidation_bits.append(self.get_invalidation_bits(gp_nr, channel, fragment)) + invalidation_bits.append(self.get_invalidation_bits(gp_nr, channel.pos_invalidation_bit, fragment)) channel_values.append(vals) count += 1 @@ -7588,7 +7681,7 @@ def _get_scalar( if one_piece: fragment = data - data_bytes = fragment[0] + data_bytes, rec_offset, rec_count = fragment.data, fragment.record_offset, fragment.record_count info = grp.record[ch_nr] @@ -7597,12 +7690,15 @@ def _get_scalar( if ch_nr == 0 and len(grp.channels) == 1 and channel.dtype_fmt.itemsize == record_size: buffer = bytearray(data_bytes) else: - buffer = get_channel_raw_bytes( - data_bytes, - record_size + channel_group.invalidation_bytes_nr, - byte_offset, - byte_size, - ) + if fragment.is_record: + buffer = get_channel_raw_bytes( + data_bytes, + record_size + channel_group.invalidation_bytes_nr, + byte_offset, + byte_size, + ) + else: + buffer = data_bytes vals = frombuffer(buffer, dtype=dtype_) @@ -7634,6 +7730,19 @@ def _get_scalar( if dtype(view) != vals.dtype: vals = vals.view(view) + elif channel_type == v4c.CHANNEL_TYPE_VALUE and channel.fast_path is None: + channel.fast_path = ( + gp_nr, + record_size + channel_group.invalidation_bytes_nr, + byte_offset, + byte_size, + channel.pos_invalidation_bit if channel_invalidation_present else -1, + # data_type, + # channel_type, + # bit_count, + dtype_, + ) + else: vals = self._get_not_byte_aligned_data(data_bytes, grp, ch_nr) @@ -7646,7 +7755,7 @@ def _get_scalar( timestamps = None if channel_invalidation_present: - invalidation_bits = self.get_invalidation_bits(gp_nr, channel, fragment) + invalidation_bits = self.get_invalidation_bits(gp_nr, channel.pos_invalidation_bit, fragment) if not ignore_invalidation_bits: vals = vals[nonzero(~invalidation_bits)[0]] @@ -7664,7 +7773,12 @@ def _get_scalar( if info is None: for count, fragment in enumerate(data, 1): - data_bytes, offset, _count, invalidation_bytes = fragment + data_bytes, offset, _count, invalidation_bytes = ( + fragment.data, + fragment.record_offset, + fragment.record_count, + fragment.invalidation_data, + ) vals = self._get_not_byte_aligned_data(data_bytes, grp, ch_nr) @@ -7674,7 +7788,9 @@ def _get_scalar( if master_is_required: timestamps.append(self.get_master(gp_nr, fragment, one_piece=True)) if channel_invalidation_present: - invalidation_bits.append(self.get_invalidation_bits(gp_nr, channel, fragment)) + invalidation_bits.append( + self.get_invalidation_bits(gp_nr, channel.pos_invalidation_bit, fragment) + ) channel_values.append(vals) vals = concatenate(channel_values) @@ -7685,7 +7801,7 @@ def _get_scalar( count = 0 for count, fragment in enumerate(data, 1): - data_bytes = fragment[0] + data_bytes = fragment.data if ch_nr == 0 and len(grp.channels) == 1 and channel.dtype_fmt.itemsize == record_size: buffer.append(bytearray(data_bytes)) @@ -7702,7 +7818,9 @@ def _get_scalar( if master_is_required: timestamps.append(self.get_master(gp_nr, fragment, one_piece=True)) if channel_invalidation_present: - invalidation_bits.append(self.get_invalidation_bits(gp_nr, channel, fragment)) + invalidation_bits.append( + self.get_invalidation_bits(gp_nr, channel.pos_invalidation_bit, fragment) + ) if count > 1: buffer = bytearray().join(buffer) @@ -8251,6 +8369,10 @@ def _yield_selected_signals( virtual_channel_group = self.virtual_groups[index] record_size = virtual_channel_group.record_size + from time import perf_counter + + tt = perf_counter() + if groups is None: groups = self.included_channels(index, skip_master=skip_master)[index] @@ -8282,13 +8404,44 @@ def _yield_selected_signals( self._set_temporary_master(None) idx = 0 + group_info = {} + for group_index, channels in groups.items(): + grp = self.groups[group_index] + if not grp.single_channel_dtype: + self._prepare_record(grp) + + group_info[group_index] = ch_info = [] + dependency_list = grp.channel_dependencies + info = grp.record + + for channel_index in channels: + channel = grp.channels[channel_index] + + if ( + channel.byte_offset + (channel.bit_offset + channel.bit_count) / 8 + > grp.channel_group.samples_byte_nr + ): + ch_info.append([0, 0]) + elif dependency_list[channel_index]: + ch_info.append([0, 0]) + else: + if info[channel_index] is not None: + _, byte_size, byte_offset, _ = info[channel_index] + ch_info.append([byte_offset, byte_size]) + else: + ch_info.append([0, 0]) + while True: try: fragments = [next(stream) for stream in data_streams] except: break + # + # if perf_counter() - tt > 120: + # x = 1 / 0 - _master = self.get_master(index, data=fragments[master_index]) + # prepare the master + _master = self.get_master(index, data=fragments[master_index], one_piece=True) self._set_temporary_master(_master) if idx == 0: @@ -8296,38 +8449,103 @@ def _yield_selected_signals( else: signals = [(_master, None)] - vlsd_max_sizes = [] - for fragment, (group_index, channels) in zip(fragments, groups.items()): grp = self.groups[group_index] if not grp.single_channel_dtype: self._prepare_record(grp) - if idx == 0: - for channel_index in channels: - signal = self.get( - group=group_index, - index=channel_index, - data=fragment, - raw=True, - ignore_invalidation_bits=True, - samples_only=False, - ) + self._invalidation_cache.clear() - signals.append(signal) + if 1 and len(channels) >= 100: + # prepare the invalidation bytes for this group and fragment + invalidation_bytes = get_channel_raw_bytes( + fragment.data, + grp.channel_group.samples_byte_nr + grp.channel_group.invalidation_bytes_nr, + grp.channel_group.samples_byte_nr, + grp.channel_group.invalidation_bytes_nr, + ) + channels_raw_data = get_channel_raw_bytes_parallel( + fragment.data, + grp.channel_group.samples_byte_nr + grp.channel_group.invalidation_bytes_nr, + group_info[group_index], + THREAD_COUNT, + ) + + if idx == 0: + for channel_index, raw_data in zip(channels, channels_raw_data): + signal = self.get( + group=group_index, + index=channel_index, + data=( + Fragment( + data=raw_data, + record_offset=fragment.record_offset, + record_count=fragment.record_count, + invalidation_data=invalidation_bytes, + is_record=False, + ) + if raw_data + else fragment + ), + raw=True, + ignore_invalidation_bits=True, + samples_only=False, + skip_channel_validation=True, + ) + + signals.append(signal) + + else: + for channel_index, raw_data in zip(channels, channels_raw_data): + signal, invalidation_bits = self.get( + group=group_index, + index=channel_index, + data=( + Fragment( + data=raw_data, + record_offset=fragment.record_offset, + record_count=fragment.record_count, + invalidation_data=invalidation_bytes, + is_record=False, + ) + if raw_data + else fragment + ), + raw=True, + ignore_invalidation_bits=True, + samples_only=True, + skip_channel_validation=True, + ) + + signals.append((signal, invalidation_bits)) else: - for channel_index in channels: - signal, invalidation_bits = self.get( - group=group_index, - index=channel_index, - data=fragment, - raw=True, - ignore_invalidation_bits=True, - samples_only=True, - ) - signals.append((signal, invalidation_bits)) + if idx == 0: + for channel_index in channels: + signal = self.get( + group=group_index, + index=channel_index, + data=fragment, + raw=True, + ignore_invalidation_bits=True, + samples_only=False, + ) + + signals.append(signal) + + else: + for channel_index in channels: + signal, invalidation_bits = self.get( + group=group_index, + index=channel_index, + data=fragment, + raw=True, + ignore_invalidation_bits=True, + samples_only=True, + ) + + signals.append((signal, invalidation_bits)) if version < "4.00": if idx == 0: @@ -8375,12 +8593,12 @@ def _yield_selected_signals( yield signals grp.read_split_count = 0 + self._invalidation_cache.clear() def get_master( self, index: int, data: bytes | None = None, - raster: RasterType | None = None, record_offset: int = 0, record_count: int | None = None, one_piece: bool = False, @@ -8393,10 +8611,6 @@ def get_master( group index data : (bytes, int, int, bytes|None) (data block raw bytes, fragment offset, count, invalidation bytes); default None - raster : float - raster to be used for interpolation; default None - - .. deprecated:: 5.13.0 record_offset : int if *data=None* use this to select the record offset from which the @@ -8412,10 +8626,6 @@ def get_master( """ - if raster is not None: - raise PendingDeprecationWarning( - "the argument raster is deprecated since version 5.13.0 " "and will be removed in a future release" - ) if self._master is not None: return self._master @@ -8441,7 +8651,12 @@ def get_master( fragment = data if fragment: - data_bytes, offset, _count, invalidation_bytes = fragment + data_bytes, offset, _count, invalidation_bytes = ( + fragment.data, + fragment.record_offset, + fragment.record_count, + fragment.invalidation_data, + ) cycles_nr = len(data_bytes) // record_size if record_size else 0 else: offset = 0 @@ -8490,7 +8705,7 @@ def get_master( else: data = (fragment,) - buffer = bytearray().join([fragment[0] for fragment in data]) + buffer = bytearray().join([fragment.data for fragment in data]) t = frombuffer(buffer, dtype=time_ch.dtype_fmt) @@ -8498,7 +8713,7 @@ def get_master( dtype_, byte_size, byte_offset, bit_offset = group.record[time_ch_nr] if one_piece: - data_bytes = data[0] + data_bytes = data.data buffer = get_channel_raw_bytes( data_bytes, @@ -8523,7 +8738,7 @@ def get_master( buffer = bytearray().join( [ get_channel_raw_bytes( - fragment[0], + fragment.data, record_size, byte_offset, byte_size, @@ -8575,17 +8790,7 @@ def get_master( if t.dtype != float64: t = t.astype(float64) - if raster and t.size: - timestamps = t - if len(t) > 1: - num = float(float32((timestamps[-1] - timestamps[0]) / raster)) - if int(num) == num: - timestamps = linspace(t[0], t[-1], int(num)) - else: - timestamps = arange(t[0], t[-1], raster) - else: - timestamps = t - return timestamps + return t def get_bus_signal( self, @@ -9302,7 +9507,8 @@ def save( data = self._load_data(gp) if chunks == 1: - data_, _1, _2, inval_ = next(data) + fragment = next(data) + data_, inval_ = fragment.data, fragment.invalidation_data if self.version >= "4.20" and gp.uses_ld: if compression: if gp.channel_group.samples_byte_nr > 1: @@ -9405,7 +9611,8 @@ def save( dv_addr = [] di_addr = [] block_size = 0 - for i, (data_, _1, _2, inval_) in enumerate(data): + for i, fragment in enumerate(data): + data_, inval_ = fragment.data, fragment.invalidation_data if i == 0: block_size = len(data_) if compression: @@ -9493,9 +9700,8 @@ def save( } dl_block = DataList(**kwargs) - for i, data__ in enumerate(data): - - data_ = data__[0] + for i, fragment in enumerate(data): + data_ = fragment.data if compression and self.version >= "4.10": if compression == 1: @@ -10416,7 +10622,7 @@ def _sort( original_size = len(new_data) if original_size: if compress: - new_data = lz_compress(new_data) + new_data = lz_compress(new_data, store_size=True) compressed_size = len(new_data) write(new_data) @@ -10498,7 +10704,7 @@ def _sort( original_size = len(new_data) if original_size: if compress: - new_data = lz_compress(new_data) + new_data = lz_compress(new_data, store_size=True) compressed_size = len(new_data) write(new_data) @@ -10649,7 +10855,7 @@ def _process_can_logging(self, group_index: int, grp: Group) -> None: for fragment in data: self._set_temporary_master(None) - self._set_temporary_master(self.get_master(group_index, data=fragment)) + self._set_temporary_master(self.get_master(group_index, data=fragment, one_piece=True)) bus_ids = self.get( "CAN_DataFrame.BusChannel", @@ -10689,7 +10895,7 @@ def _process_can_logging(self, group_index: int, grp: Group) -> None: for fragment in data: self._set_temporary_master(None) - self._set_temporary_master(self.get_master(group_index, data=fragment)) + self._set_temporary_master(self.get_master(group_index, data=fragment, one_piece=True)) bus_ids = self.get( "CAN_DataFrame.BusChannel", @@ -10745,7 +10951,7 @@ def _process_can_logging(self, group_index: int, grp: Group) -> None: for fragment in data: self._set_temporary_master(None) - self._set_temporary_master(self.get_master(group_index, data=fragment)) + self._set_temporary_master(self.get_master(group_index, data=fragment, one_piece=True)) data_bytes = self.get( "CAN_DataFrame.DataBytes", @@ -10905,7 +11111,7 @@ def _process_lin_logging(self, group_index: int, grp: Group) -> None: for fragment in data: self._set_temporary_master(None) - self._set_temporary_master(self.get_master(group_index, data=fragment)) + self._set_temporary_master(self.get_master(group_index, data=fragment, one_piece=True)) msg_ids = ( self.get( @@ -10938,7 +11144,7 @@ def _process_lin_logging(self, group_index: int, grp: Group) -> None: for fragment in data: self._set_temporary_master(None) - self._set_temporary_master(self.get_master(group_index, data=fragment)) + self._set_temporary_master(self.get_master(group_index, data=fragment, one_piece=True)) msg_ids = self.get("LIN_Frame.ID", group=group_index, data=fragment).astype(".+)") @@ -537,7 +539,7 @@ def get_fmt_v4(data_type: int, size: int, channel_type: int = v4c.CHANNEL_TYPE_V """ if data_type in v4c.NON_SCALAR_TYPES: - size = size // 8 + size = size // 8 or 1 if data_type in ( v4c.DATA_TYPE_BYTEARRAY, @@ -1304,6 +1306,10 @@ def get_signal_data_blocks(self, index: int) -> Iterator[SignalDataBlockInfo]: except StopIteration: break + def load_all_data_blocks(self): + for _ in self.get_data_blocks(): + continue + class VirtualChannelGroup: """starting with MDF v4.20 it is possible to use remote masters and column @@ -1508,6 +1514,30 @@ def __repr__(self) -> str: ) +class Fragment: + def __init__( + self, + data, + record_offset=-1, + record_count=-1, + invalidation_data=None, + is_record=True, + ) -> None: + self.data = data + self.record_count = record_count + self.record_offset = record_offset + self.invalidation_data = invalidation_data + self.is_record = is_record + + def __repr__(self) -> str: + return ( + f"FragmentInfo({len(self.data)} bytes, " + f"record_offset={self.record_offset}, " + f"record_count={self.record_count}, " + f"is_record={self.is_record})" + ) + + class InvalidationBlockInfo(DataBlockInfo): __slots__ = ("all_valid",) @@ -2413,3 +2443,36 @@ def timed(*args, **kwargs): return ret return timed + + +class Timer: + + def __init__(self, name=""): + self.name = name or str(id(self)) + self.count = 0 + self.total_time = 0.0 + + def __enter__(self): + now = perf_counter() + self.start = now + return self + + def __exit__(self, type, value, traceback): + now = perf_counter() + self.total_time += now - self.start + self.count += 1 + + def display(self): + if self.count: + for factor, r, unit in ((1e3, 3, "ms"), (1e6, 6, "us"), (1e9, 9, "ns")): + tpi = round(self.total_time / self.count, r) + if tpi: + break + print( + f"""TIMER {self.name}: +\t* {self.count} iterations in {self.total_time * 1000:.3f}ms +\t* {self.count / self.total_time:.3f} iter/s +\t* {self.total_time / self.count * factor:.3f} {unit}/iter""" + ) + else: + print(f"TIMER {self.name}:\n\t* inactive") diff --git a/src/asammdf/blocks/v4_blocks.py b/src/asammdf/blocks/v4_blocks.py index 8eef85d9c..916798ece 100644 --- a/src/asammdf/blocks/v4_blocks.py +++ b/src/asammdf/blocks/v4_blocks.py @@ -349,10 +349,10 @@ def to_blocks(self, address: int, blocks: list[Any], defined_texts: dict[str, in return address def __getitem__(self, item: str) -> Any: - return self.__getattribute__(item) + return getattr(self, item) def __setitem__(self, item: str, value: Any) -> None: - self.__setattr__(item, value) + setattr(self, item, value) def __bytes__(self) -> bytes: fmt = f"{v4c.FMT_AT_COMMON}{self.embedded_size}s" @@ -476,6 +476,7 @@ class Channel: "default_X_dg_addr", "display_names", "dtype_fmt", + "fast_path", "flags", "id", "links_nr", @@ -1020,12 +1021,13 @@ def __init__(self, **kwargs) -> None: del self.display_names[self.name] self.standard_C_size = True + self.fast_path = None def __getitem__(self, item: str) -> Any: - return self.__getattribute__(item) + return getattr(self, item) def __setitem__(self, item: str, value: Any) -> None: - self.__setattr__(item, value) + setattr(self, item, value) def to_blocks( self, @@ -1667,10 +1669,10 @@ def __init__(self, **kwargs) -> None: self[f"dim_size_{i}"] = kwargs[f"dim_size_{i}"] def __getitem__(self, item: str) -> Any: - return self.__getattribute__(item) + return getattr(self, item) def __setitem__(self, item: str, value: Any) -> None: - self.__setattr__(item, value) + setattr(self, item, value) def __str__(self) -> str: return f"" @@ -2011,10 +2013,10 @@ def __init__(self, **kwargs) -> None: self.links_nr = 6 def __getitem__(self, item: str) -> Any: - return self.__getattribute__(item) + return getattr(self, item) def __setitem__(self, item: str, value: Any) -> None: - self.__setattr__(item, value) + setattr(self, item, value) def to_blocks( self, @@ -4103,10 +4105,10 @@ def metadata(self, indent: str = "") -> str: return "\n".join(metadata) def __getitem__(self, item: str) -> Any: - return self.__getattribute__(item) + return getattr(self, item) def __setitem__(self, item: str, value: Any) -> None: - self.__setattr__(item, value) + setattr(self, item, value) def __contains__(self, item: str) -> bool: return hasattr(self, item) @@ -4420,10 +4422,10 @@ def __init__(self, **kwargs) -> None: self.data = kwargs["data"] def __getitem__(self, item: str) -> Any: - return self.__getattribute__(item) + return getattr(self, item) def __setitem__(self, item: str, value: Any) -> None: - self.__setattr__(item, value) + setattr(self, item, value) def __bytes__(self) -> bytes: return v4c.COMMON_p(self.id, self.reserved0, self.block_len, self.links_nr) + self.data @@ -4595,10 +4597,10 @@ def __getattribute__(self, item: str) -> Any: return value def __setitem__(self, item: str, value: Any) -> None: - self.__setattr__(item, value) + setattr(self, item, value) def __getitem__(self, item: str) -> Any: - return self.__getattribute__(item) + return getattr(self, item) def __str__(self) -> str: return f"""""" @@ -4767,10 +4769,10 @@ def to_blocks(self, address: int, blocks: list[Any], defined_texts: dict[str, in return address def __getitem__(self, item: str) -> Any: - return self.__getattribute__(item) + return getattr(self, item) def __setitem__(self, item: str, value: Any) -> None: - self.__setattr__(item, value) + setattr(self, item, value) def __bytes__(self) -> bytes: result = v4c.DATA_GROUP_p( @@ -4859,11 +4861,11 @@ def __init__(self, **kwargs) -> None: self.next_dl_addr = links[0] for i, addr in enumerate(links[1:]): - self[f"data_block_addr{i}"] = addr + setattr(self, f"data_block_addr{i}", addr) stream.seek(address + self.links_nr * 8) - self.flags = stream.read(1)[0] + self.flags = stream.read_byte() if self.flags & v4c.FLAG_DL_EQUAL_LENGHT: (self.reserved1, self.data_block_nr, self.data_block_len) = unpack("<3sIQ", stream.read(15)) else: @@ -4873,7 +4875,7 @@ def __init__(self, **kwargs) -> None: stream.read((self.links_nr - 1) * 8), ) for i, offset in enumerate(offsets): - self[f"offset_{i}"] = offset + setattr(self, f"offset_{i}", offset) else: stream.seek(address) @@ -4926,10 +4928,10 @@ def __init__(self, **kwargs) -> None: self[f"offset_{i}"] = kwargs[f"offset_{i}"] def __getitem__(self, item: str) -> Any: - return self.__getattribute__(item) + return getattr(self, item) def __setitem__(self, item: str, value: Any) -> None: - self.__setattr__(item, value) + setattr(self, item, value) def __bytes__(self) -> bytes: keys = ("id", "reserved0", "block_len", "links_nr", "next_dl_addr") @@ -5182,10 +5184,10 @@ def __bytes__(self) -> bytes: return result def __getitem__(self, item: str) -> Any: - return self.__getattribute__(item) + return getattr(self, item) def __setitem__(self, item: str, value: Any) -> None: - self.__setattr__(item, value) + setattr(self, item, value) def __str__(self) -> str: return f"EventBlock (name: {self.name}, comment: {self.comment}, address: {hex(self.address)}, scopes: {self.scopes}, fields: {super().__str__()})" @@ -5303,10 +5305,10 @@ def __init__(self, **kwargs) -> None: self.unfinalized_custom_flags = 0 def __getitem__(self, item: str) -> Any: - return self.__getattribute__(item) + return getattr(self, item) def __setitem__(self, item: str, value: Any) -> None: - self.__setattr__(item, value) + setattr(self, item, value) def __bytes__(self) -> bytes: result = pack( @@ -5430,10 +5432,10 @@ def to_blocks(self, address: int, blocks: list[Any], defined_texts: dict[str, in return address def __getitem__(self, item: str) -> Any: - return self.__getattribute__(item) + return getattr(self, item) def __setitem__(self, item: str, value: Any) -> None: - self.__setattr__(item, value) + setattr(self, item, value) def __bytes__(self) -> bytes: result = pack(v4c.FMT_FILE_HISTORY, *[self[key] for key in v4c.KEYS_FILE_HISTORY]) @@ -5727,10 +5729,10 @@ def subject(self, value): self._common_properties["subject"] = value def __getitem__(self, item: str) -> Any: - return self.__getattribute__(item) + return getattr(self, item) def __setitem__(self, item: str, value: Any) -> None: - self.__setattr__(item, value) + setattr(self, item, value) @property def start_time(self) -> datetime: @@ -5914,10 +5916,10 @@ def __init__(self, **kwargs) -> None: self.reserved1 = b"\x00" * 5 def __getitem__(self, item: str) -> Any: - return self.__getattribute__(item) + return getattr(self, item) def __setitem__(self, item: str, value: Any) -> None: - self.__setattr__(item, value) + setattr(self, item, value) def __bytes__(self) -> bytes: result = pack(v4c.FMT_HL_BLOCK, *[self[key] for key in v4c.KEYS_HL_BLOCK]) @@ -6128,10 +6130,10 @@ def __init__(self, **kwargs) -> None: self.block_len = 24 + self.links_nr * 8 + 16 def __getitem__(self, item: str) -> Any: - return self.__getattribute__(item) + return getattr(self, item) def __setitem__(self, item: str, value: Any) -> None: - self.__setattr__(item, value) + setattr(self, item, value) def __bytes__(self) -> bytes: fmt = "<4sI3Q" @@ -6296,10 +6298,10 @@ def __init__(self, **kwargs) -> None: self.reserved1 = b"\x00" * 5 def __getitem__(self, item: str) -> Any: - return self.__getattribute__(item) + return getattr(self, item) def __setitem__(self, item: str, value: Any) -> None: - self.__setattr__(item, value) + setattr(self, item, value) def __contains__(self, item: str) -> bool: return hasattr(self, item) @@ -6566,10 +6568,10 @@ def __init__(self, **kwargs) -> None: self.block_len = size + 32 - size % 8 def __getitem__(self, item: str) -> Any: - return self.__getattribute__(item) + return getattr(self, item) def __setitem__(self, item: str, value: Any) -> None: - self.__setattr__(item, value) + setattr(self, item, value) def __bytes__(self) -> bytes: return pack( diff --git a/src/asammdf/gui/widgets/channel_group_info.py b/src/asammdf/gui/widgets/channel_group_info.py index 90961ff8a..547865c37 100644 --- a/src/asammdf/gui/widgets/channel_group_info.py +++ b/src/asammdf/gui/widgets/channel_group_info.py @@ -79,7 +79,8 @@ def _display(self, position): record_count = record_end - record_offset data = b"".join( - e[0] for e in self.mdf._load_data(self.group, record_offset=record_offset, record_count=record_count) + fragment.data + for fragment in self.mdf._load_data(self.group, record_offset=record_offset, record_count=record_count) ) data = pd.Series(list(np.frombuffer(data, dtype=f"({self.record_size},)u1"))) diff --git a/src/asammdf/gui/widgets/file.py b/src/asammdf/gui/widgets/file.py index 283649932..3cb0db7c0 100644 --- a/src/asammdf/gui/widgets/file.py +++ b/src/asammdf/gui/widgets/file.py @@ -1051,7 +1051,7 @@ def to_config(self): config["windows"] = windows config["active_window"] = current_window.windowTitle() if current_window else "" config["functions"] = self.functions - config["global_variables"] = self.global_variables + config["global_variables"] = "\n".join([line for line in self.global_variables.splitlines() if line]) return config @@ -1268,6 +1268,7 @@ def load_channel_list(self, event=None, file_name=None, manually=False, show_pro self.functions.update(info.get("functions", {})) self.global_variables = f'{self.global_variables}\n{info.get("global_variables", "")}' + self.global_variables = "\n".join([line for line in self.global_variables.splitlines() if line]) if channels: iterator = QtWidgets.QTreeWidgetItemIterator(self.channels_tree) diff --git a/src/asammdf/mdf.py b/src/asammdf/mdf.py index e6b0fd39e..1ce75b35f 100644 --- a/src/asammdf/mdf.py +++ b/src/asammdf/mdf.py @@ -37,9 +37,11 @@ from .blocks import v2_v3_constants as v23c from .blocks import v4_constants as v4c from .blocks.conversion_utils import from_dict +from .blocks.cutils import get_channel_raw_bytes_complete from .blocks.options import FloatInterpolation, IntegerInterpolation from .blocks.source_utils import Source from .blocks.utils import ( + as_non_byte_sized_signed_int, components, csv_bytearray2hex, csv_int2hex, @@ -56,6 +58,7 @@ randomized_string, SUPPORTED_VERSIONS, TERMINATED, + THREAD_COUNT, UINT16_u, UINT64_u, UniqueDB, @@ -200,6 +203,11 @@ class MDF: .. versionadded:: 7.0.0 + process_bus_logging (\*\*kwargs) : bool + controls if the bus processing of MDF v4 files is done when the file is loaded. Default True + + .. versionadded:: 8.0.0 + Examples -------- >>> mdf = MDF(version='3.30') # new MDF object with version 3.30 @@ -2242,6 +2250,11 @@ def concatenate( use_display_names (False) : bool + process_bus_logging (True) : bool + controls if the bus processing of MDF v4 files is done when the file is loaded. Default True + + .. versionadded:: 8.1.0 + Examples -------- >>> conc = MDF.concatenate( @@ -2643,7 +2656,8 @@ def concatenate( first_mdf.close() try: - merged._process_bus_logging() + if kwargs.get("process_bus_logging", True): + merged._process_bus_logging() except: pass @@ -2677,6 +2691,11 @@ def stack( use_display_names (False) : bool + process_bus_logging (True) : bool + controls if the bus processing of MDF v4 files is done when the file is loaded. Default True + + .. versionadded:: 8.1.0 + Examples -------- >>> stacked = MDF.stack( @@ -2831,7 +2850,8 @@ def stack( return TERMINATED try: - stacked._process_bus_logging() + if kwargs.get("process_bus_logging", True): + stacked._process_bus_logging() except: pass @@ -3324,6 +3344,308 @@ def select( """ + def validate_blocks(blocks, record_size): + for block in blocks: + if block.original_size % record_size: + return False + + return True + + if ( + self.version < "4.00" + or not self._mapped_file + or record_offset + or record_count is not None + or True # disable for now + ): + return self._select_fallback( + channels, record_offset, raw, copy_master, ignore_value2text_conversions, record_count, validate + ) + + if isinstance(raw, dict): + if "__default__" not in raw: + raise MdfException("The raw argument given as dict must contain the __default__ key") + + __default__ = raw["__default__"] + raw_dict = True + else: + raw_dict = False + + virtual_groups = self.included_channels(channels=channels, minimal=False, skip_master=False) + for virtual_group, groups in virtual_groups.items(): + if len(self._mdf.virtual_groups[virtual_group].groups) > 1: + return self._select_fallback( + channels, record_offset, raw, copy_master, ignore_value2text_conversions, record_count, validate + ) + + output_signals = {} + + for virtual_group, groups in virtual_groups.items(): + group_index = virtual_group + grp = self._mdf.groups[group_index] + grp.load_all_data_blocks() + blocks = grp.data_blocks + record_size = grp.channel_group.samples_byte_nr + grp.channel_group.invalidation_bytes_nr + cycles_nr = grp.channel_group.cycles_nr + channel_indexes = groups[group_index] + + pairs = [(group_index, ch_index) for ch_index in channel_indexes] + + master_index = self.masters_db.get(group_index, None) + if master_index is None or grp.record[master_index] is None: + return self._select_fallback( + channels, record_offset, raw, copy_master, ignore_value2text_conversions, record_count, validate + ) + + channel = grp.channels[master_index] + master_dtype, byte_size, byte_offset, _ = grp.record[master_index] + signals = [(byte_offset, byte_size, channel.pos_invalidation_bit)] + + for ch_index in channel_indexes: + channel = grp.channels[ch_index] + + if (info := grp.record[ch_index]) is None: + print("NASOl") + return self._select_fallback( + channels, record_offset, raw, copy_master, ignore_value2text_conversions, record_count, validate + ) + else: + _, byte_size, byte_offset, _ = info + signals.append((byte_offset, byte_size, channel.pos_invalidation_bit)) + + raw_and_invalidation = get_channel_raw_bytes_complete( + blocks, + signals, + self._mapped_file.name, + cycles_nr, + record_size, + grp.channel_group.invalidation_bytes_nr, + THREAD_COUNT, + ) + master_bytes, _ = raw_and_invalidation[0] + raw_and_invalidation = raw_and_invalidation[1:] + + # prepare the master + master = np.frombuffer(master_bytes, dtype=master_dtype) + + for pair, (raw_data, invalidation_bits) in zip(pairs, raw_and_invalidation): + ch_index = pair[-1] + channel = grp.channels[ch_index] + channel_dtype, byte_size, byte_offset, bit_offset = grp.record[ch_index] + vals = np.frombuffer(raw_data, dtype=channel_dtype) + + data_type = channel.data_type + + if not channel.standard_C_size: + size = byte_size + + if channel_dtype.byteorder == "=" and data_type in ( + v4c.DATA_TYPE_SIGNED_MOTOROLA, + v4c.DATA_TYPE_UNSIGNED_MOTOROLA, + ): + view = np.dtype(f">u{vals.itemsize}") + else: + view = np.dtype(f"{channel_dtype.byteorder}u{vals.itemsize}") + + if view != vals.dtype: + vals = vals.view(view) + + if bit_offset: + vals >>= bit_offset + + if channel.bit_count != size * 8: + if data_type in v4c.SIGNED_INT: + vals = as_non_byte_sized_signed_int(vals, channel.bit_count) + else: + mask = (1 << channel.bit_count) - 1 + vals &= mask + elif data_type in v4c.SIGNED_INT: + view = f"{channel_dtype.byteorder}i{vals.itemsize}" + if np.dtype(view) != vals.dtype: + vals = vals.view(view) + + conversion = channel.conversion + unit = (conversion and conversion.unit) or channel.unit + + source = channel.source + + if source: + source = Source.from_source(source) + else: + cg_source = grp.channel_group.acq_source + if cg_source: + source = Source.from_source(cg_source) + else: + source = None + + master_metadata = self._master_channel_metadata.get(group_index, None) + + output_signals[pair] = Signal( + samples=vals, + timestamps=master, + unit=unit, + name=channel.name, + comment=channel.comment, + conversion=conversion, + raw=True, + master_metadata=master_metadata, + attachment=None, + source=source, + display_names=channel.display_names, + bit_count=channel.bit_count, + flags=Signal.Flags.no_flags, + invalidation_bits=invalidation_bits, + encoding=None, + group_index=group_index, + channel_index=ch_index, + ) + + indexes = [] + + for item in channels: + if not isinstance(item, (list, tuple)): + item = [item] + indexes.append(self._validate_channel_selection(*item)) + + signals = [output_signals[pair] for pair in indexes] + + if copy_master: + for signal in signals: + signal.timestamps = signal.timestamps.copy() + + for signal in signals: + if (raw_dict and not raw.get(signal.name, __default__)) or (not raw_dict and not raw): + conversion = signal.conversion + if conversion: + samples = conversion.convert( + signal.samples, ignore_value2text_conversions=ignore_value2text_conversions + ) + signal.samples = samples + + signal.raw = False + signal.conversion = None + if signal.samples.dtype.kind == "S": + signal.encoding = "utf-8" if self.version >= "4.00" else "latin-1" + + if validate: + signals = [sig.validate(copy=False) for sig in signals] + + for signal, channel in zip(signals, channels): + if isinstance(channel, str): + signal.name = channel + else: + name = channel[0] + if name is not None: + signal.name = name + + unique = set() + for i, signal in enumerate(signals): + obj_id = id(signal) + if id(signal) in unique: + signals[i] = signal.copy() + unique.add(obj_id) + + return signals + + def _select_fallback( + self, + channels: ChannelsType, + record_offset: int = 0, + raw: bool | dict[str, bool] = False, + copy_master: bool = True, + ignore_value2text_conversions: bool = False, + record_count: int | None = None, + validate: bool = False, + ) -> list[Signal]: + """retrieve the channels listed in *channels* argument as *Signal* + objects + + .. note:: the *dataframe* argument was removed in version 5.8.0 + use the ``to_dataframe`` method instead + + Parameters + ---------- + channels : list + list of items to be filtered; each item can be : + + * a channel name string + * (channel name, group index, channel index) list or tuple + * (channel name, group index) list or tuple + * (None, group index, channel index) list or tuple + + record_offset : int + record number offset; optimization to get the last part of signal samples + raw : bool | dict[str, bool] + get raw channel samples; default *False* + + .. versionchanged:: 8.0.0 + + provide individual raw mode based on a dict. If the parameters is given + as dict then it must contain the key ``__default__`` with the default raw value. The dict keys + are the channel names and the values are the boolean raw values for each channel. + + copy_master : bool + option to get a new timestamps array for each selected Signal or to + use a shared array for channels of the same channel group; default *True* + ignore_value2text_conversions (False) : bool + valid only for the channels that have value to text conversions and + if *raw=False*. If this is True then the raw numeric values will be + used, and the conversion will not be applied. + + .. versionchanged:: 5.8.0 + + validate (False) : bool + consider the invalidation bits + + .. versionadded:: 5.16.0 + + Returns + ------- + signals : list + list of *Signal* objects based on the input channel list + + Examples + -------- + >>> from asammdf import MDF, Signal + >>> import numpy as np + >>> t = np.arange(5) + >>> s = np.ones(5) + >>> mdf = MDF() + >>> for i in range(4): + ... sigs = [Signal(s*(i*10+j), t, name='SIG') for j in range(1,4)] + ... mdf.append(sigs) + ... + >>> # select SIG group 0 default index 1 default, SIG group 3 index 1, SIG group 2 index 1 default and channel index 2 from group 1 + ... + >>> mdf.select(['SIG', ('SIG', 3, 1), ['SIG', 2], (None, 1, 2)]) + [ + , + , + , + ] + + """ + if isinstance(raw, dict): if "__default__" not in raw: raise MdfException("The raw argument given as dict must contain the __default__ key") @@ -4917,7 +5239,7 @@ def _extract_can_logging( for fragment in data: self._set_temporary_master(None) - self._set_temporary_master(self.get_master(i, data=fragment)) + self._set_temporary_master(self.get_master(i, data=fragment, one_piece=True)) bus_ids = self.get( "CAN_DataFrame.BusChannel", @@ -5254,7 +5576,7 @@ def _extract_lin_logging( for fragment in data: self._set_temporary_master(None) - self._set_temporary_master(self.get_master(i, data=fragment)) + self._set_temporary_master(self.get_master(i, data=fragment, one_piece=True)) msg_ids = self.get("LIN_Frame.ID", group=i, data=fragment).astype(" Signal: if self.invalidation_bits is None and other.invalidation_bits is None: invalidation_bits = None elif self.invalidation_bits is None and other.invalidation_bits is not None: - invalidation_bits = np.concatenate((np.zeros(len(self), dtype=bool), other.invalidation_bits)) + invalidation_bits = InvalidationArray( + np.concatenate((np.zeros(len(self), dtype=bool), other.invalidation_bits)), + other.invalidation_bits.origin, + ) elif self.invalidation_bits is not None and other.invalidation_bits is None: - invalidation_bits = np.concatenate((self.invalidation_bits, np.zeros(len(other), dtype=bool))) + invalidation_bits = InvalidationArray( + np.concatenate((self.invalidation_bits, np.zeros(len(other), dtype=bool))), + self.invalidation_bits.origin, + ) else: - invalidation_bits = np.append(self.invalidation_bits, other.invalidation_bits) + invalidation_bits = InvalidationArray( + np.append(self.invalidation_bits, other.invalidation_bits), self.invalidation_bits.origin + ) result = Signal( np.append(self.samples, other.samples, axis=0), diff --git a/src/asammdf/version.py b/src/asammdf/version.py index 7f42128f4..4520a7464 100644 --- a/src/asammdf/version.py +++ b/src/asammdf/version.py @@ -1,3 +1,3 @@ """ asammdf version module """ -__version__ = "8.1.0.dev3" +__version__ = "8.1.0.dev4"