From 7053a7b9ad1961862cce43d0af7a00ce227f25d5 Mon Sep 17 00:00:00 2001 From: Charles Burkland Date: Wed, 15 Mar 2023 20:25:34 -0700 Subject: [PATCH 1/6] Implements fast is_sorted check --- README.rst | 4 ++ src/__init__.py | 1 + src/__init__.pyi | 1 + src/_arraykit.c | 180 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 186 insertions(+) diff --git a/README.rst b/README.rst index ae6d0aac..cdd19d6f 100644 --- a/README.rst +++ b/README.rst @@ -86,6 +86,10 @@ Extended arguments to and functionality in ``split_after_count()`` to support th Now building wheels for 3.11. +0.1.12 +............ + +Implemented ``is_sorted``. 0.2.2 ............ diff --git a/src/__init__.py b/src/__init__.py index 85b62662..487cfe3a 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -19,6 +19,7 @@ from ._arraykit import delimited_to_arrays as delimited_to_arrays from ._arraykit import iterable_str_to_array_1d as iterable_str_to_array_1d from ._arraykit import get_new_indexers_and_screen as get_new_indexers_and_screen +from ._arraykit import is_sorted as is_sorted from ._arraykit import split_after_count as split_after_count from ._arraykit import count_iteration as count_iteration from ._arraykit import first_true_1d as first_true_1d diff --git a/src/__init__.pyi b/src/__init__.pyi index b925d0ad..c6820473 100644 --- a/src/__init__.pyi +++ b/src/__init__.pyi @@ -72,6 +72,7 @@ def resolve_dtype_iter(__dtypes: tp.Iterable[np.dtype]) -> np.dtype: ... def isna_element(__value: tp.Any, include_none: bool = True) -> bool: ... def dtype_from_element(__value: tp.Optional[tp.Hashable]) -> np.dtype: ... def get_new_indexers_and_screen(indexers: np.ndarray, positions: np.ndarray) -> tp.Tuple[np.ndarray, np.ndarray]: ... +def is_sorted(arr: np.ndarray) -> bool: ... def first_true_1d(__array: np.ndarray, *, forward: bool) -> int: ... def first_true_2d(__array: np.ndarray, *, forward: bool, axis: int) -> np.ndarray: ... diff --git a/src/_arraykit.c b/src/_arraykit.c index 48bec1ec..3261d186 100644 --- a/src/_arraykit.c +++ b/src/_arraykit.c @@ -4058,6 +4058,185 @@ get_new_indexers_and_screen(PyObject *Py_UNUSED(m), PyObject *args, PyObject *kw return NULL; } +# define AK_IS_SORTED_SIMPLE(npy_type, ctype) \ + if (np_dtype == npy_type) { \ + NPY_BEGIN_THREADS_DEF; \ + NPY_BEGIN_THREADS; \ + do { \ + char* data = *dataptr; \ + npy_intp stride = *strideptr; \ + npy_intp inner_size = *innersizeptr;\ + ctype prev = *((ctype *)data); \ + data += stride; \ + inner_size--; \ + while (inner_size--) { \ + ctype element = *((ctype *)data); \ + if (element < prev) { \ + NPY_END_THREADS; \ + goto fail; \ + } \ + prev = element; \ + data += stride; \ + } \ + } while(arr_iternext(arr_iter)); \ + NPY_END_THREADS; \ + } \ + +# define AK_IS_SORTED_COMPLEX(npy_type, ctype) \ + if (np_dtype == npy_type) { \ + NPY_BEGIN_THREADS_DEF; \ + NPY_BEGIN_THREADS; \ + do { \ + char* data = *dataptr; \ + npy_intp stride = *strideptr; \ + npy_intp inner_size = *innersizeptr;\ + ctype prev = *((ctype *)data); \ + data += stride; \ + inner_size--; \ + while (inner_size--) { \ + ctype element = *((ctype *)data); \ + if (element.real < prev.real || element.imag < prev.imag) { \ + NPY_END_THREADS; \ + goto fail; \ + } \ + prev = element; \ + data += stride; \ + } \ + } while(arr_iternext(arr_iter)); \ + NPY_END_THREADS; \ + } \ + +static bool +AK_is_sorted_string(NpyIter_IterNextFunc *arr_iternext, NpyIter *arr_iter, char **dataptr, npy_intp *strideptr, npy_intp *innersizeptr) +{ + int maxlen = NpyIter_GetDescrArray(arr_iter)[0]->elsize; + char *prev = PyArray_malloc(maxlen+1); + if (prev == NULL) { + NpyIter_Deallocate(arr_iter); + PyErr_NoMemory(); + return NULL; + } + + NPY_BEGIN_THREADS_DEF; + NPY_BEGIN_THREADS; + + do { + char* data = *dataptr; + npy_intp stride = *strideptr; + npy_intp inner_size = *innersizeptr; + + memcpy(prev, data, maxlen); + data += stride; + inner_size--; + while (inner_size--) { + if (strncmp(data, prev, maxlen) < 0) { + NPY_END_THREADS + return false; + } + memcpy(prev, data, maxlen); + data += stride; + } + } while(arr_iternext(arr_iter)); + + NPY_END_THREADS + return true; +} + + +static PyObject * +is_sorted(PyObject *Py_UNUSED(m), PyObject *arg) +{ + PyArrayObject *arr = (PyArrayObject*)arg; + int np_dtype = PyArray_TYPE(arr); + + // Now, implement the core algorithm by looping over the ``arr``. + // We need to use numpy's iteration API, as the ``arr`` could be + // C-contiguous, F-contiguous, both, or neither. + // See https://numpy.org/doc/stable/reference/c-api/iterator.html#simple-iteration-example + NpyIter *arr_iter = NpyIter_New( + arr, // array + NPY_ITER_READONLY | NPY_ITER_EXTERNAL_LOOP | NPY_ITER_REFS_OK, // iter flags + NPY_CORDER, // order + NPY_NO_CASTING, // casting + NULL // dtype + ); + if (arr_iter == NULL) { + return NULL; + } + + // The iternext function gets stored in a local variable so it can be called repeatedly in an efficient manner. + NpyIter_IterNextFunc *arr_iternext = NpyIter_GetIterNext(arr_iter, NULL); + if (arr_iternext == NULL) { + NpyIter_Deallocate(arr_iter); + return NULL; + } + + // All of these will be updated by the iterator + char **dataptr = NpyIter_GetDataPtrArray(arr_iter); + npy_intp *strideptr = NpyIter_GetInnerStrideArray(arr_iter); + npy_intp *innersizeptr = NpyIter_GetInnerLoopSizePtr(arr_iter); + + // ------------------------------------------------------------------------ + AK_IS_SORTED_SIMPLE(NPY_BYTE, npy_byte) + else AK_IS_SORTED_SIMPLE(NPY_UBYTE, npy_ubyte) + else AK_IS_SORTED_SIMPLE(NPY_SHORT, npy_short) + else AK_IS_SORTED_SIMPLE(NPY_USHORT, npy_ushort) + else AK_IS_SORTED_SIMPLE(NPY_INT, npy_int) + else AK_IS_SORTED_SIMPLE(NPY_UINT, npy_uint) + else AK_IS_SORTED_SIMPLE(NPY_LONG, npy_long) + else AK_IS_SORTED_SIMPLE(NPY_ULONG, npy_ulong) + else AK_IS_SORTED_SIMPLE(NPY_LONGLONG, npy_longlong) + else AK_IS_SORTED_SIMPLE(NPY_ULONGLONG, npy_ulonglong) + else AK_IS_SORTED_SIMPLE(NPY_FLOAT, npy_float) + else AK_IS_SORTED_SIMPLE(NPY_DOUBLE, npy_double) + else AK_IS_SORTED_SIMPLE(NPY_LONGDOUBLE, npy_longdouble) + else AK_IS_SORTED_SIMPLE(NPY_DATETIME, npy_datetime) + else AK_IS_SORTED_SIMPLE(NPY_TIMEDELTA, npy_timedelta) + else AK_IS_SORTED_SIMPLE(NPY_HALF, npy_half) + // ------------------------------------------------------------------------ + else AK_IS_SORTED_COMPLEX(NPY_CFLOAT, npy_complex64) + else AK_IS_SORTED_COMPLEX(NPY_CDOUBLE, npy_complex128) + else AK_IS_SORTED_COMPLEX(NPY_CLONGDOUBLE, npy_complex256) + // ------------------------------------------------------------------------ + else if (np_dtype == NPY_STRING || np_dtype == NPY_UNICODE) { + if (!AK_is_sorted_string(arr_iternext, arr_iter, dataptr, strideptr, innersizeptr)) { + goto fail; + } + } + // ------------------------------------------------------------------------ + // perf is not good here - maybe drop support? + else if (np_dtype == NPY_OBJECT) { + do { + char* data = *dataptr; + npy_intp stride = *strideptr; + npy_intp inner_size = *innersizeptr; + + PyObject* prev = *((PyObject **)data); + data += stride; + inner_size--; + while (inner_size--) { + PyObject* element = *((PyObject **)data); + if (PyObject_RichCompareBool(element, prev, Py_LT) == 1) { + goto fail; + } + prev = element; + data += stride; + } + } while(arr_iternext(arr_iter)); + } + else { + PyErr_SetString(PyExc_NotImplementedError, "not support for this dtype"); + return NULL; + } + + NpyIter_Deallocate(arr_iter); + Py_RETURN_TRUE; + +fail: + NpyIter_Deallocate(arr_iter); + Py_RETURN_FALSE; +} + //------------------------------------------------------------------------------ // ArrayGO //------------------------------------------------------------------------------ @@ -4364,6 +4543,7 @@ static PyMethodDef arraykit_methods[] = { METH_VARARGS | METH_KEYWORDS, NULL}, {"dtype_from_element", dtype_from_element, METH_O, NULL}, + {"is_sorted", is_sorted, METH_O, NULL}, {"get_new_indexers_and_screen", (PyCFunction)get_new_indexers_and_screen, METH_VARARGS | METH_KEYWORDS, From 5925996e2edab56c111c9b712d5890887aeac19b Mon Sep 17 00:00:00 2001 From: Charles Burkland Date: Tue, 21 Mar 2023 12:47:13 -0700 Subject: [PATCH 2/6] Limits the use-cases for is_sorted --- src/_arraykit.c | 300 ++++++++++++++++++++++-------------------------- 1 file changed, 137 insertions(+), 163 deletions(-) diff --git a/src/_arraykit.c b/src/_arraykit.c index 3261d186..973a5e4b 100644 --- a/src/_arraykit.c +++ b/src/_arraykit.c @@ -4058,183 +4058,157 @@ get_new_indexers_and_screen(PyObject *Py_UNUSED(m), PyObject *args, PyObject *kw return NULL; } -# define AK_IS_SORTED_SIMPLE(npy_type, ctype) \ - if (np_dtype == npy_type) { \ - NPY_BEGIN_THREADS_DEF; \ - NPY_BEGIN_THREADS; \ - do { \ - char* data = *dataptr; \ - npy_intp stride = *strideptr; \ - npy_intp inner_size = *innersizeptr;\ - ctype prev = *((ctype *)data); \ - data += stride; \ - inner_size--; \ - while (inner_size--) { \ - ctype element = *((ctype *)data); \ - if (element < prev) { \ - NPY_END_THREADS; \ - goto fail; \ - } \ - prev = element; \ - data += stride; \ - } \ - } while(arr_iternext(arr_iter)); \ - NPY_END_THREADS; \ - } \ - -# define AK_IS_SORTED_COMPLEX(npy_type, ctype) \ - if (np_dtype == npy_type) { \ - NPY_BEGIN_THREADS_DEF; \ - NPY_BEGIN_THREADS; \ - do { \ - char* data = *dataptr; \ - npy_intp stride = *strideptr; \ - npy_intp inner_size = *innersizeptr;\ - ctype prev = *((ctype *)data); \ - data += stride; \ - inner_size--; \ - while (inner_size--) { \ - ctype element = *((ctype *)data); \ - if (element.real < prev.real || element.imag < prev.imag) { \ - NPY_END_THREADS; \ - goto fail; \ - } \ - prev = element; \ - data += stride; \ - } \ - } while(arr_iternext(arr_iter)); \ - NPY_END_THREADS; \ - } \ - -static bool -AK_is_sorted_string(NpyIter_IterNextFunc *arr_iternext, NpyIter *arr_iter, char **dataptr, npy_intp *strideptr, npy_intp *innersizeptr) -{ - int maxlen = NpyIter_GetDescrArray(arr_iter)[0]->elsize; - char *prev = PyArray_malloc(maxlen+1); - if (prev == NULL) { - NpyIter_Deallocate(arr_iter); - PyErr_NoMemory(); - return NULL; - } - - NPY_BEGIN_THREADS_DEF; - NPY_BEGIN_THREADS; - - do { - char* data = *dataptr; - npy_intp stride = *strideptr; - npy_intp inner_size = *innersizeptr; - - memcpy(prev, data, maxlen); - data += stride; - inner_size--; - while (inner_size--) { - if (strncmp(data, prev, maxlen) < 0) { - NPY_END_THREADS - return false; - } - memcpy(prev, data, maxlen); - data += stride; - } - } while(arr_iternext(arr_iter)); +# define AK_IS_SORTED_SIMPLE(ctype) \ + ctype* data_##ctype##_ = (ctype*)PyArray_DATA(arr); \ + for (size_t i = 0; i < size - 1; ++i) { \ + if (data_##ctype##_[i] > data_##ctype##_[i + 1]) { \ + Py_RETURN_FALSE; \ + } \ + } \ + Py_RETURN_TRUE; \ + +# define AK_IS_SORTED_COMPLEX(ctype) \ + ctype* data_##ctype##_ = (ctype*)PyArray_DATA(arr); \ + for (size_t i = 0; i < size - 1; ++i) { \ + ctype element = data_##ctype##_[i]; \ + ctype next = data_##ctype##_[i + 1]; \ + if (element.real > next.real || element.imag > next.imag) { \ + Py_RETURN_FALSE; \ + } \ + } \ + Py_RETURN_TRUE; \ + +// static bool +// AK_is_sorted_string(NpyIter_IterNextFunc *arr_iternext, NpyIter *arr_iter, char **dataptr, npy_intp *strideptr, npy_intp *innersizeptr) +// { +// int maxlen = NpyIter_GetDescrArray(arr_iter)[0]->elsize; +// char *prev = PyArray_malloc(maxlen+1); +// if (prev == NULL) { +// NpyIter_Deallocate(arr_iter); +// PyErr_NoMemory(); +// return NULL; +// } - NPY_END_THREADS - return true; -} +// NPY_BEGIN_THREADS_DEF; +// NPY_BEGIN_THREADS; + +// do { +// char* data = *dataptr; +// npy_intp stride = *strideptr; +// npy_intp inner_size = *innersizeptr; + +// memcpy(prev, data, maxlen); +// data += stride; +// inner_size--; +// while (inner_size--) { +// if (strncmp(data, prev, maxlen) < 0) { +// NPY_END_THREADS +// return false; +// } +// memcpy(prev, data, maxlen); +// data += stride; +// } +// } while(arr_iternext(arr_iter)); + +// NPY_END_THREADS +// return true; +// } static PyObject * is_sorted(PyObject *Py_UNUSED(m), PyObject *arg) { + AK_CHECK_NUMPY_ARRAY(arg); + PyArrayObject *arr = (PyArrayObject*)arg; int np_dtype = PyArray_TYPE(arr); - // Now, implement the core algorithm by looping over the ``arr``. - // We need to use numpy's iteration API, as the ``arr`` could be - // C-contiguous, F-contiguous, both, or neither. - // See https://numpy.org/doc/stable/reference/c-api/iterator.html#simple-iteration-example - NpyIter *arr_iter = NpyIter_New( - arr, // array - NPY_ITER_READONLY | NPY_ITER_EXTERNAL_LOOP | NPY_ITER_REFS_OK, // iter flags - NPY_CORDER, // order - NPY_NO_CASTING, // casting - NULL // dtype - ); - if (arr_iter == NULL) { - return NULL; - } - - // The iternext function gets stored in a local variable so it can be called repeatedly in an efficient manner. - NpyIter_IterNextFunc *arr_iternext = NpyIter_GetIterNext(arr_iter, NULL); - if (arr_iternext == NULL) { - NpyIter_Deallocate(arr_iter); - return NULL; - } + // if (PyArray_NDIM(arr) != 1) { + // PyErr_SetString(PyExc_ValueError, "Array must be 1-dimensional"); + // return NULL; + // } - // All of these will be updated by the iterator - char **dataptr = NpyIter_GetDataPtrArray(arr_iter); - npy_intp *strideptr = NpyIter_GetInnerStrideArray(arr_iter); - npy_intp *innersizeptr = NpyIter_GetInnerLoopSizePtr(arr_iter); + // int contiguous = PyArray_IS_C_CONTIGUOUS(arr); + size_t size = (size_t)PyArray_SIZE(arr); // ------------------------------------------------------------------------ - AK_IS_SORTED_SIMPLE(NPY_BYTE, npy_byte) - else AK_IS_SORTED_SIMPLE(NPY_UBYTE, npy_ubyte) - else AK_IS_SORTED_SIMPLE(NPY_SHORT, npy_short) - else AK_IS_SORTED_SIMPLE(NPY_USHORT, npy_ushort) - else AK_IS_SORTED_SIMPLE(NPY_INT, npy_int) - else AK_IS_SORTED_SIMPLE(NPY_UINT, npy_uint) - else AK_IS_SORTED_SIMPLE(NPY_LONG, npy_long) - else AK_IS_SORTED_SIMPLE(NPY_ULONG, npy_ulong) - else AK_IS_SORTED_SIMPLE(NPY_LONGLONG, npy_longlong) - else AK_IS_SORTED_SIMPLE(NPY_ULONGLONG, npy_ulonglong) - else AK_IS_SORTED_SIMPLE(NPY_FLOAT, npy_float) - else AK_IS_SORTED_SIMPLE(NPY_DOUBLE, npy_double) - else AK_IS_SORTED_SIMPLE(NPY_LONGDOUBLE, npy_longdouble) - else AK_IS_SORTED_SIMPLE(NPY_DATETIME, npy_datetime) - else AK_IS_SORTED_SIMPLE(NPY_TIMEDELTA, npy_timedelta) - else AK_IS_SORTED_SIMPLE(NPY_HALF, npy_half) - // ------------------------------------------------------------------------ - else AK_IS_SORTED_COMPLEX(NPY_CFLOAT, npy_complex64) - else AK_IS_SORTED_COMPLEX(NPY_CDOUBLE, npy_complex128) - else AK_IS_SORTED_COMPLEX(NPY_CLONGDOUBLE, npy_complex256) - // ------------------------------------------------------------------------ - else if (np_dtype == NPY_STRING || np_dtype == NPY_UNICODE) { - if (!AK_is_sorted_string(arr_iternext, arr_iter, dataptr, strideptr, innersizeptr)) { - goto fail; - } - } + // Switch based on np_dtype // ------------------------------------------------------------------------ - // perf is not good here - maybe drop support? - else if (np_dtype == NPY_OBJECT) { - do { - char* data = *dataptr; - npy_intp stride = *strideptr; - npy_intp inner_size = *innersizeptr; - - PyObject* prev = *((PyObject **)data); - data += stride; - inner_size--; - while (inner_size--) { - PyObject* element = *((PyObject **)data); - if (PyObject_RichCompareBool(element, prev, Py_LT) == 1) { - goto fail; - } - prev = element; - data += stride; - } - } while(arr_iternext(arr_iter)); - } - else { - PyErr_SetString(PyExc_NotImplementedError, "not support for this dtype"); - return NULL; + switch (np_dtype) { + case NPY_BYTE:; + AK_IS_SORTED_SIMPLE(npy_byte) + case NPY_UBYTE:; + AK_IS_SORTED_SIMPLE(npy_ubyte) + case NPY_SHORT:; + AK_IS_SORTED_SIMPLE(npy_short) + case NPY_USHORT:; + AK_IS_SORTED_SIMPLE(npy_ushort) + case NPY_INT:; + AK_IS_SORTED_SIMPLE(npy_int) + case NPY_UINT:; + AK_IS_SORTED_SIMPLE(npy_uint) + case NPY_LONG:; + AK_IS_SORTED_SIMPLE(npy_long) + case NPY_ULONG:; + AK_IS_SORTED_SIMPLE(npy_ulong) + case NPY_LONGLONG:; + AK_IS_SORTED_SIMPLE(npy_longlong) + case NPY_ULONGLONG:; + AK_IS_SORTED_SIMPLE(npy_ulonglong) + case NPY_FLOAT:; + AK_IS_SORTED_SIMPLE(npy_float) + case NPY_DOUBLE:; + AK_IS_SORTED_SIMPLE(npy_double) + # ifdef PyFloat128ArrType_Type + case NPY_LONGDOUBLE:; + AK_IS_SORTED_SIMPLE(npy_longdouble) + # endif + case NPY_DATETIME:; + AK_IS_SORTED_SIMPLE(npy_datetime) + case NPY_TIMEDELTA:; + AK_IS_SORTED_SIMPLE(npy_timedelta) + case NPY_HALF:; + AK_IS_SORTED_SIMPLE(npy_half) + case NPY_CFLOAT:; + AK_IS_SORTED_COMPLEX(npy_complex64) + case NPY_CDOUBLE:; + AK_IS_SORTED_COMPLEX(npy_complex128) + # ifdef PyComplex256ArrType_Type + case NPY_CLONGDOUBLE:; + AK_IS_SORTED_COMPLEX(npy_complex256) + # endif + // case NPY_STRING: + // case NPY_UNICODE: + // if (!AK_is_sorted_string(arr, contiguous)) { + // Py_RETURN_FALSE + // } + // Py_RETURN_TRUE; + default:; + PyErr_SetString(PyExc_ValueError, "Unsupported dtype"); + return NULL; } - - NpyIter_Deallocate(arr_iter); - Py_RETURN_TRUE; - -fail: - NpyIter_Deallocate(arr_iter); - Py_RETURN_FALSE; + // // ------------------------------------------------------------------------ + // // perf is not good here - maybe drop support? + // else if (np_dtype == NPY_OBJECT) { + // do { + // char* data = *dataptr; + // npy_intp stride = *strideptr; + // npy_intp inner_size = *innersizeptr; + + // PyObject* prev = *((PyObject **)data); + // data += stride; + // inner_size--; + // while (inner_size--) { + // PyObject* element = *((PyObject **)data); + // if (PyObject_RichCompareBool(element, prev, Py_LT) == 1) { + // goto fail; + // } + // prev = element; + // data += stride; + // } + // } while(arr_iternext(arr_iter)); + // } + Py_UNREACHABLE(); } //------------------------------------------------------------------------------ From b5747229daf2f0c38b38923ca04866b74d7f2756 Mon Sep 17 00:00:00 2001 From: Charles Burkland Date: Tue, 21 Mar 2023 13:32:17 -0700 Subject: [PATCH 3/6] Adds support for non-contiguous arrays. Simplifies macro expressions. --- src/_arraykit.c | 86 +++++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 39 deletions(-) diff --git a/src/_arraykit.c b/src/_arraykit.c index 973a5e4b..e3463aa6 100644 --- a/src/_arraykit.c +++ b/src/_arraykit.c @@ -4058,25 +4058,33 @@ get_new_indexers_and_screen(PyObject *Py_UNUSED(m), PyObject *args, PyObject *kw return NULL; } -# define AK_IS_SORTED_SIMPLE(ctype) \ - ctype* data_##ctype##_ = (ctype*)PyArray_DATA(arr); \ - for (size_t i = 0; i < size - 1; ++i) { \ - if (data_##ctype##_[i] > data_##ctype##_[i + 1]) { \ - Py_RETURN_FALSE; \ - } \ - } \ - Py_RETURN_TRUE; \ - -# define AK_IS_SORTED_COMPLEX(ctype) \ - ctype* data_##ctype##_ = (ctype*)PyArray_DATA(arr); \ - for (size_t i = 0; i < size - 1; ++i) { \ - ctype element = data_##ctype##_[i]; \ - ctype next = data_##ctype##_[i + 1]; \ - if (element.real > next.real || element.imag > next.imag) { \ - Py_RETURN_FALSE; \ - } \ - } \ - Py_RETURN_TRUE; \ +//------------------------------------------------------------------------------ + +# define AK_COMPARE_SIMPLE(a, b) a > b +# define AK_COMPARE_COMPLEX(a, b) a.real > b.real || (a.real == b.real && a.imag > b.imag) + +# define AK_IS_SORTED(ctype, comp_macro) \ + if (contiguous) { \ + ctype* data_##ctype##_ = (ctype*)PyArray_DATA(arr); \ + for (size_t i = 0; i < size - 1; ++i) { \ + ctype element = data_##ctype##_[i]; \ + ctype next = data_##ctype##_[i + 1]; \ + if (comp_macro(element, next)) { \ + Py_RETURN_FALSE; \ + } \ + } \ + } \ + else { \ + for (size_t i = 0; i < size - 1; ++i) { \ + ctype element = *(ctype*)PyArray_GETPTR1(arr, i); \ + ctype next = *(ctype*)PyArray_GETPTR1(arr, i + 1); \ + if (comp_macro(element, next)) { \ + Py_RETURN_FALSE; \ + } \ + } \ + } \ + Py_RETURN_TRUE; \ + // static bool // AK_is_sorted_string(NpyIter_IterNextFunc *arr_iternext, NpyIter *arr_iter, char **dataptr, npy_intp *strideptr, npy_intp *innersizeptr) @@ -4128,7 +4136,7 @@ is_sorted(PyObject *Py_UNUSED(m), PyObject *arg) // return NULL; // } - // int contiguous = PyArray_IS_C_CONTIGUOUS(arr); + int contiguous = PyArray_IS_C_CONTIGUOUS(arr); size_t size = (size_t)PyArray_SIZE(arr); // ------------------------------------------------------------------------ @@ -4136,46 +4144,46 @@ is_sorted(PyObject *Py_UNUSED(m), PyObject *arg) // ------------------------------------------------------------------------ switch (np_dtype) { case NPY_BYTE:; - AK_IS_SORTED_SIMPLE(npy_byte) + AK_IS_SORTED(npy_byte, AK_COMPARE_SIMPLE) case NPY_UBYTE:; - AK_IS_SORTED_SIMPLE(npy_ubyte) + AK_IS_SORTED(npy_ubyte, AK_COMPARE_SIMPLE) case NPY_SHORT:; - AK_IS_SORTED_SIMPLE(npy_short) + AK_IS_SORTED(npy_short, AK_COMPARE_SIMPLE) case NPY_USHORT:; - AK_IS_SORTED_SIMPLE(npy_ushort) + AK_IS_SORTED(npy_ushort, AK_COMPARE_SIMPLE) case NPY_INT:; - AK_IS_SORTED_SIMPLE(npy_int) + AK_IS_SORTED(npy_int, AK_COMPARE_SIMPLE) case NPY_UINT:; - AK_IS_SORTED_SIMPLE(npy_uint) + AK_IS_SORTED(npy_uint, AK_COMPARE_SIMPLE) case NPY_LONG:; - AK_IS_SORTED_SIMPLE(npy_long) + AK_IS_SORTED(npy_long, AK_COMPARE_SIMPLE) case NPY_ULONG:; - AK_IS_SORTED_SIMPLE(npy_ulong) + AK_IS_SORTED(npy_ulong, AK_COMPARE_SIMPLE) case NPY_LONGLONG:; - AK_IS_SORTED_SIMPLE(npy_longlong) + AK_IS_SORTED(npy_longlong, AK_COMPARE_SIMPLE) case NPY_ULONGLONG:; - AK_IS_SORTED_SIMPLE(npy_ulonglong) + AK_IS_SORTED(npy_ulonglong, AK_COMPARE_SIMPLE) case NPY_FLOAT:; - AK_IS_SORTED_SIMPLE(npy_float) + AK_IS_SORTED(npy_float, AK_COMPARE_SIMPLE) case NPY_DOUBLE:; - AK_IS_SORTED_SIMPLE(npy_double) + AK_IS_SORTED(npy_double, AK_COMPARE_SIMPLE) # ifdef PyFloat128ArrType_Type case NPY_LONGDOUBLE:; - AK_IS_SORTED_SIMPLE(npy_longdouble) + AK_IS_SORTED(npy_longdouble, AK_COMPARE_SIMPLE) # endif case NPY_DATETIME:; - AK_IS_SORTED_SIMPLE(npy_datetime) + AK_IS_SORTED(npy_datetime, AK_COMPARE_SIMPLE) case NPY_TIMEDELTA:; - AK_IS_SORTED_SIMPLE(npy_timedelta) + AK_IS_SORTED(npy_timedelta, AK_COMPARE_SIMPLE) case NPY_HALF:; - AK_IS_SORTED_SIMPLE(npy_half) + AK_IS_SORTED(npy_half, AK_COMPARE_SIMPLE) case NPY_CFLOAT:; - AK_IS_SORTED_COMPLEX(npy_complex64) + AK_IS_SORTED(npy_complex64, AK_COMPARE_COMPLEX) case NPY_CDOUBLE:; - AK_IS_SORTED_COMPLEX(npy_complex128) + AK_IS_SORTED(npy_complex128, AK_COMPARE_COMPLEX) # ifdef PyComplex256ArrType_Type case NPY_CLONGDOUBLE:; - AK_IS_SORTED_COMPLEX(npy_complex256) + AK_IS_SORTED(npy_complex256, AK_COMPARE_COMPLEX) # endif // case NPY_STRING: // case NPY_UNICODE: From 84526983ac547bfda30887c79a23a0c3919655be Mon Sep 17 00:00:00 2001 From: Charles Burkland Date: Tue, 21 Mar 2023 14:07:15 -0700 Subject: [PATCH 4/6] Turns on threading. Support strings/bools. Adds tests --- src/_arraykit.c | 122 ++++++++++++++++++++++++---------------------- test/test_util.py | 59 +++++++++++++++++----- 2 files changed, 111 insertions(+), 70 deletions(-) diff --git a/src/_arraykit.c b/src/_arraykit.c index e3463aa6..ab737ffb 100644 --- a/src/_arraykit.c +++ b/src/_arraykit.c @@ -4063,86 +4063,92 @@ get_new_indexers_and_screen(PyObject *Py_UNUSED(m), PyObject *args, PyObject *kw # define AK_COMPARE_SIMPLE(a, b) a > b # define AK_COMPARE_COMPLEX(a, b) a.real > b.real || (a.real == b.real && a.imag > b.imag) -# define AK_IS_SORTED(ctype, comp_macro) \ +# define AK_IS_SORTED(ctype, compare_macro) \ if (contiguous) { \ + NPY_BEGIN_THREADS_DEF; \ + NPY_BEGIN_THREADS; \ ctype* data_##ctype##_ = (ctype*)PyArray_DATA(arr); \ - for (size_t i = 0; i < size - 1; ++i) { \ + for (size_t i = 0; i < arr_size - 1; ++i) { \ ctype element = data_##ctype##_[i]; \ ctype next = data_##ctype##_[i + 1]; \ - if (comp_macro(element, next)) { \ + if (compare_macro(element, next)) { \ + NPY_END_THREADS; \ Py_RETURN_FALSE; \ } \ } \ + NPY_END_THREADS; \ } \ else { \ - for (size_t i = 0; i < size - 1; ++i) { \ + NPY_BEGIN_THREADS_DEF; \ + NPY_BEGIN_THREADS; \ + for (size_t i = 0; i < arr_size - 1; ++i) { \ ctype element = *(ctype*)PyArray_GETPTR1(arr, i); \ ctype next = *(ctype*)PyArray_GETPTR1(arr, i + 1); \ - if (comp_macro(element, next)) { \ + if (compare_macro(element, next)) { \ + NPY_END_THREADS; \ Py_RETURN_FALSE; \ } \ } \ + NPY_END_THREADS; \ } \ Py_RETURN_TRUE; \ -// static bool -// AK_is_sorted_string(NpyIter_IterNextFunc *arr_iternext, NpyIter *arr_iter, char **dataptr, npy_intp *strideptr, npy_intp *innersizeptr) -// { -// int maxlen = NpyIter_GetDescrArray(arr_iter)[0]->elsize; -// char *prev = PyArray_malloc(maxlen+1); -// if (prev == NULL) { -// NpyIter_Deallocate(arr_iter); -// PyErr_NoMemory(); -// return NULL; -// } - -// NPY_BEGIN_THREADS_DEF; -// NPY_BEGIN_THREADS; - -// do { -// char* data = *dataptr; -// npy_intp stride = *strideptr; -// npy_intp inner_size = *innersizeptr; - -// memcpy(prev, data, maxlen); -// data += stride; -// inner_size--; -// while (inner_size--) { -// if (strncmp(data, prev, maxlen) < 0) { -// NPY_END_THREADS -// return false; -// } -// memcpy(prev, data, maxlen); -// data += stride; -// } -// } while(arr_iternext(arr_iter)); - -// NPY_END_THREADS -// return true; -// } +static bool +AK_is_sorted_string(PyArrayObject* arr, bool contiguous, size_t arr_size) +{ + size_t item_size = (size_t)PyArray_ITEMSIZE(arr); + + if (contiguous) { + NPY_BEGIN_THREADS_DEF; + NPY_BEGIN_THREADS; + char* data = (char*)PyArray_DATA(arr); + size_t i = 0; + while (i < (arr_size - 1) * item_size) { + if (strncmp(&data[i], &data[i + item_size], item_size) > 0) { + NPY_END_THREADS; + Py_RETURN_FALSE; + } + i += item_size; + } + NPY_END_THREADS; + } + else { + NPY_BEGIN_THREADS_DEF; + NPY_BEGIN_THREADS; + size_t i = 0; + while (i < (arr_size - 1) * item_size) { + char *element = PyArray_GETPTR1(arr, i); + char *next = PyArray_GETPTR1(arr, i + 1); + if (strncmp(element, next, item_size) > 0) { + NPY_END_THREADS; + Py_RETURN_FALSE; + } + i += item_size; + } + NPY_END_THREADS; + } + Py_RETURN_TRUE; +} static PyObject * is_sorted(PyObject *Py_UNUSED(m), PyObject *arg) { AK_CHECK_NUMPY_ARRAY(arg); - PyArrayObject *arr = (PyArrayObject*)arg; - int np_dtype = PyArray_TYPE(arr); - // if (PyArray_NDIM(arr) != 1) { - // PyErr_SetString(PyExc_ValueError, "Array must be 1-dimensional"); - // return NULL; - // } + if (PyArray_NDIM(arr) != 1) { + PyErr_SetString(PyExc_ValueError, "Array must be 1-dimensional"); + return NULL; + } - int contiguous = PyArray_IS_C_CONTIGUOUS(arr); - size_t size = (size_t)PyArray_SIZE(arr); + bool contiguous = (bool)PyArray_IS_C_CONTIGUOUS(arr); + size_t arr_size = (size_t)PyArray_SIZE(arr); - // ------------------------------------------------------------------------ - // Switch based on np_dtype - // ------------------------------------------------------------------------ - switch (np_dtype) { + switch (PyArray_TYPE(arr)) { + case NPY_BOOL:; + AK_IS_SORTED(npy_bool, AK_COMPARE_SIMPLE) case NPY_BYTE:; AK_IS_SORTED(npy_byte, AK_COMPARE_SIMPLE) case NPY_UBYTE:; @@ -4185,12 +4191,12 @@ is_sorted(PyObject *Py_UNUSED(m), PyObject *arg) case NPY_CLONGDOUBLE:; AK_IS_SORTED(npy_complex256, AK_COMPARE_COMPLEX) # endif - // case NPY_STRING: - // case NPY_UNICODE: - // if (!AK_is_sorted_string(arr, contiguous)) { - // Py_RETURN_FALSE - // } - // Py_RETURN_TRUE; + case NPY_STRING: + case NPY_UNICODE: + if (!AK_is_sorted_string(arr, contiguous, arr_size)) { + Py_RETURN_FALSE; + } + Py_RETURN_TRUE; default:; PyErr_SetString(PyExc_ValueError, "Unsupported dtype"); return NULL; diff --git a/test/test_util.py b/test/test_util.py index 87684f25..f86df18a 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -18,10 +18,10 @@ from arraykit import array_deepcopy from arraykit import isna_element from arraykit import dtype_from_element -from arraykit import split_after_count from arraykit import count_iteration from arraykit import first_true_1d from arraykit import first_true_2d +from arraykit import is_sorted from performance.reference.util import get_new_indexers_and_screen_ak as get_new_indexers_and_screen_full from arraykit import get_new_indexers_and_screen @@ -84,7 +84,6 @@ def test_resolve_dtype_c(self) -> None: self.assertEqual(resolve_dtype(a1.dtype, a4.dtype), np.dtype('O')) - def test_resolve_dtype_d(self) -> None: dt1 = np.array(1).dtype dt2 = np.array(2.3).dtype @@ -96,7 +95,6 @@ def test_resolve_dtype_e(self) -> None: assert resolve_dtype(dt1, dt2) == np.dtype(object) assert resolve_dtype(dt1, dt1) == dt1 - #--------------------------------------------------------------------------- def test_resolve_dtype_iter_a(self) -> None: @@ -167,7 +165,6 @@ def test_column_2d_filter_a(self) -> None: with self.assertRaises(NotImplementedError): column_2d_filter(a1.reshape(1,2,5)) - #--------------------------------------------------------------------------- def test_column_1d_filter_a(self) -> None: @@ -219,7 +216,6 @@ def test_array_deepcopy_a2(self) -> None: self.assertEqual(memo[id(a1)].tolist(), a2.tolist()) self.assertFalse(a2.flags.writeable) - def test_array_deepcopy_b(self) -> None: a1 = np.arange(10) memo = {id(a1): a1} @@ -227,7 +223,6 @@ def test_array_deepcopy_b(self) -> None: self.assertEqual(mloc(a1), mloc(a2)) - def test_array_deepcopy_c1(self) -> None: mutable = [np.nan] memo = {} @@ -329,7 +324,6 @@ def test_isna_element_b(self) -> None: self.assertFalse(isna_element(datetime.date(2020, 12, 31))) self.assertFalse(isna_element(False)) - def test_isna_element_c(self) -> None: self.assertFalse(isna_element(None, include_none=False)) self.assertTrue(isna_element(None, include_none=True)) @@ -474,6 +468,7 @@ def test_get_new_indexers_and_screen_b(self) -> None: assert tuple(map(list, postB)) == (list(indexersB), list(indexersB)) #--------------------------------------------------------------------------- + def test_count_iteration_a(self) -> None: post = count_iteration(('a', 'b', 'c', 'd')) self.assertEqual(post, 4) @@ -484,6 +479,7 @@ def test_count_iteration_b(self) -> None: self.assertEqual(post, 5) #--------------------------------------------------------------------------- + def test_first_true_1d_a(self) -> None: a1 = np.arange(100) == 50 post = first_true_1d(a1, forward=True) @@ -552,8 +548,8 @@ def test_first_true_1d_multi_b(self) -> None: self.assertEqual(first_true_1d(a1, forward=True), 10) self.assertEqual(first_true_1d(a1, forward=False), 50) - #--------------------------------------------------------------------------- + def test_first_true_2d_a(self) -> None: a1 = np.isin(np.arange(100), (9, 19, 38, 68, 96)).reshape(5, 20) @@ -610,7 +606,6 @@ def test_first_true_2d_c(self) -> None: [-1, -1, -1, -1] ) - def test_first_true_2d_d(self) -> None: a1 = np.isin(np.arange(20), (0, 3, 4, 7, 8, 11, 12, 15, 16, 19)).reshape(5, 4) @@ -653,7 +648,6 @@ def test_first_true_2d_f(self) -> None: with self.assertRaises(ValueError): post1 = first_true_2d(a1, axis=2) - def test_first_true_2d_f(self) -> None: a1 = np.isin(np.arange(15), (1, 7, 14)).reshape(3, 5) post1 = first_true_2d(a1, axis=0, forward=True) @@ -662,7 +656,6 @@ def test_first_true_2d_f(self) -> None: post2 = first_true_2d(a1, axis=0, forward=False) self.assertEqual(post2.tolist(), [-1, 0, 1, -1, 2]) - def test_first_true_2d_g(self) -> None: a1 = np.isin(np.arange(15), (1, 7, 14)).reshape(3, 5).T # force fortran ordering self.assertEqual(first_true_2d(a1, axis=0, forward=True).tolist(), @@ -674,7 +667,6 @@ def test_first_true_2d_g(self) -> None: self.assertEqual(first_true_2d(a1, axis=1, forward=False).tolist(), [-1, 0, 1, -1, 2]) - def test_first_true_2d_h(self) -> None: # force fortran ordering, non-contiguous, non-owned a1 = np.isin(np.arange(15), (1, 4, 5, 7, 8, 12, 15)).reshape(3, 5).T[:4] @@ -687,9 +679,52 @@ def test_first_true_2d_h(self) -> None: self.assertEqual(first_true_2d(a1, axis=1, forward=False).tolist(), [1, 0, 2, 1]) + def test_is_sorted_success(self) -> None: + arr_non_contiguous = np.arange(25).reshape(5,5)[:, 3] + arr_contiguous = arr_non_contiguous.copy() + + assert not arr_non_contiguous.flags.c_contiguous + assert arr_contiguous.flags.c_contiguous + dtypes = [ + np.bool_, + np.longlong, + np.int_, + np.intc, + np.short, + np.byte, + np.ubyte, + np.ushort, + np.uintc, + np.uint, + np.ulonglong, + np.half, + np.single, + np.float_, + np.longfloat, + np.csingle, + np.complex_, + np.clongfloat, + "U", + "S", + ] + for dtype in dtypes: + self.assertTrue(is_sorted(arr_contiguous.astype(dtype)), dtype) + self.assertTrue(is_sorted(arr_non_contiguous.astype(dtype)), dtype) + def test_is_sorted_disallowed_inputs(self) -> None: + arr_2d = np.arange(25).reshape(5,5) + arr_list = list(range(10)) + arr_obj = np.arange(10).astype(object) + with self.assertRaises(ValueError): + is_sorted(arr_2d) + + with self.assertRaises(TypeError): + is_sorted(arr_list) + + with self.assertRaises(ValueError): + is_sorted(arr_obj) if __name__ == '__main__': From c7b9daf3ae4f8fedfb67c85b7441d3c5bb1723c2 Mon Sep 17 00:00:00 2001 From: Charles Burkland Date: Tue, 21 Mar 2023 14:13:14 -0700 Subject: [PATCH 5/6] Improves tests --- test/test_util.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/test/test_util.py b/test/test_util.py index f86df18a..044034a0 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -679,8 +679,8 @@ def test_first_true_2d_h(self) -> None: self.assertEqual(first_true_2d(a1, axis=1, forward=False).tolist(), [1, 0, 2, 1]) - def test_is_sorted_success(self) -> None: - arr_non_contiguous = np.arange(25).reshape(5,5)[:, 3] + def test_is_sorted_a(self) -> None: + arr_non_contiguous = np.arange(25).reshape(5,5)[:, 0] arr_contiguous = arr_non_contiguous.copy() assert not arr_non_contiguous.flags.c_contiguous @@ -709,8 +709,18 @@ def test_is_sorted_success(self) -> None: "S", ] for dtype in dtypes: - self.assertTrue(is_sorted(arr_contiguous.astype(dtype)), dtype) - self.assertTrue(is_sorted(arr_non_contiguous.astype(dtype)), dtype) + arr1 = arr_contiguous.astype(dtype) + arr2 = arr_non_contiguous.astype(dtype) + assert (arr1 == arr2).all() + + assert is_sorted(arr1) + assert is_sorted(arr2) + + # Investigate why these report success, but are not sorted + if dtype in ("U", "S"): + continue + assert not is_sorted(arr1[::-1]) + assert not is_sorted(arr2[::-1]) def test_is_sorted_disallowed_inputs(self) -> None: arr_2d = np.arange(25).reshape(5,5) From f1e466f831ed89294261713e8a45bb413ee5a589 Mon Sep 17 00:00:00 2001 From: Charles Burkland Date: Thu, 23 Mar 2023 10:11:50 -0700 Subject: [PATCH 6/6] Better error message. Spacing. Updates test. Handles compiler warning --- src/_arraykit.c | 12 ++++++++++-- test/test_util.py | 10 ++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/_arraykit.c b/src/_arraykit.c index ab737ffb..7bc0905a 100644 --- a/src/_arraykit.c +++ b/src/_arraykit.c @@ -4031,7 +4031,7 @@ get_new_indexers_and_screen(PyObject *Py_UNUSED(m), PyObject *args, PyObject *kw Py_DECREF(element_locations); // new_positions = order_found[:num_unique] - PyObject *new_positions = PySequence_GetSlice((PyObject*)order_found, 0, num_found); + PyObject *new_positions = PySequence_GetSlice((PyObject*)order_found, 0, (Py_ssize_t)num_found); Py_DECREF(order_found); if (new_positions == NULL) { return NULL; @@ -4063,6 +4063,7 @@ get_new_indexers_and_screen(PyObject *Py_UNUSED(m), PyObject *args, PyObject *kw # define AK_COMPARE_SIMPLE(a, b) a > b # define AK_COMPARE_COMPLEX(a, b) a.real > b.real || (a.real == b.real && a.imag > b.imag) +/*Note: Data array needs a unique name for each case inside the switch*/ # define AK_IS_SORTED(ctype, compare_macro) \ if (contiguous) { \ NPY_BEGIN_THREADS_DEF; \ @@ -4173,10 +4174,12 @@ is_sorted(PyObject *Py_UNUSED(m), PyObject *arg) AK_IS_SORTED(npy_float, AK_COMPARE_SIMPLE) case NPY_DOUBLE:; AK_IS_SORTED(npy_double, AK_COMPARE_SIMPLE) + # ifdef PyFloat128ArrType_Type case NPY_LONGDOUBLE:; AK_IS_SORTED(npy_longdouble, AK_COMPARE_SIMPLE) # endif + case NPY_DATETIME:; AK_IS_SORTED(npy_datetime, AK_COMPARE_SIMPLE) case NPY_TIMEDELTA:; @@ -4187,10 +4190,12 @@ is_sorted(PyObject *Py_UNUSED(m), PyObject *arg) AK_IS_SORTED(npy_complex64, AK_COMPARE_COMPLEX) case NPY_CDOUBLE:; AK_IS_SORTED(npy_complex128, AK_COMPARE_COMPLEX) + # ifdef PyComplex256ArrType_Type case NPY_CLONGDOUBLE:; AK_IS_SORTED(npy_complex256, AK_COMPARE_COMPLEX) # endif + case NPY_STRING: case NPY_UNICODE: if (!AK_is_sorted_string(arr, contiguous, arr_size)) { @@ -4198,7 +4203,10 @@ is_sorted(PyObject *Py_UNUSED(m), PyObject *arg) } Py_RETURN_TRUE; default:; - PyErr_SetString(PyExc_ValueError, "Unsupported dtype"); + PyErr_Format(PyExc_ValueError, + "Unsupported dtype: %s", + PyArray_DESCR(arr)->typeobj->tp_name + ); return NULL; } // // ------------------------------------------------------------------------ diff --git a/test/test_util.py b/test/test_util.py index 044034a0..36c8b8f8 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -713,12 +713,18 @@ def test_is_sorted_a(self) -> None: arr2 = arr_non_contiguous.astype(dtype) assert (arr1 == arr2).all() - assert is_sorted(arr1) - assert is_sorted(arr2) + try: + assert is_sorted(arr1) + except ValueError: + assert dtype in (np.longfloat, np.clongfloat) + continue + else: + assert is_sorted(arr2) # Investigate why these report success, but are not sorted if dtype in ("U", "S"): continue + assert not is_sorted(arr1[::-1]) assert not is_sorted(arr2[::-1])