From 1b74978896f499e3516622a950df5abdd02e5612 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 16 Jun 2022 15:22:47 -0600 Subject: [PATCH 001/218] Do not require skipped axes in iter_indices to be broadcast compatible I still need to update the tests to test for this. --- ndindex/ndindex.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/ndindex/ndindex.py b/ndindex/ndindex.py index f9120d19..74b9863e 100644 --- a/ndindex/ndindex.py +++ b/ndindex/ndindex.py @@ -638,9 +638,9 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): Iterate indices for every element of an arrays of shape `shapes`. Each shape in `shapes` should be a shape tuple, which are broadcast - compatible. Each iteration step will produce a tuple of indices, one for - each shape, which would correspond to the same elements if the arrays of - the given shapes were first broadcast together. + compatible along the non-skipped axes. Each iteration step will produce a + tuple of indices, one for each shape, which would correspond to the same + elements if the arrays of the given shapes were first broadcast together. This is a generalization of the NumPy :py:class:`np.ndindex() ` function (which otherwise has no relation). @@ -661,7 +661,8 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): shape of `shapes`. For example, `iter_indices((3,), (1, 2, 3), skip_axes=(0,))` will skip the first axis, and only applies to the second shape, since the first shape corresponds to axis `2` of the final - broadcasted shape `(1, 2, 3)` + broadcasted shape `(1, 2, 3)`. Note that the skipped axes do not + themselves need to be broadcast compatible. For example, suppose `a` is an array with shape `(3, 2, 4, 4)`, which we wish to think of as a `(3, 2)` stack of 4 x 4 matrices. We can generate an @@ -744,8 +745,10 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): _skip_axes.append(a) _shapes = [(1,)*(ndim - len(shape)) + shape for shape in shapes] + _shapes = [tuple(1 if i in _skip_axes else shape[i] for i in range(ndim)) + for shape in _shapes] iters = [[] for i in range(len(shapes))] - broadcasted_shape = broadcast_shapes(*shapes) + broadcasted_shape = broadcast_shapes(*_shapes) for i in range(-1, -ndim-1, -1): for it, shape, _shape in zip(iters, shapes, _shapes): From 016c487697ec15a26131acb470e344f09a6b4e5f Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 13 Oct 2022 18:19:14 -0600 Subject: [PATCH 002/218] Add isvalid() method to test if an index is valid for a given shape This still needs tests. --- ndindex/ndindex.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/ndindex/ndindex.py b/ndindex/ndindex.py index 74b9863e..c3065412 100644 --- a/ndindex/ndindex.py +++ b/ndindex/ndindex.py @@ -297,6 +297,27 @@ def reduce(self, shape=None): # XXX: Should the default be raise NotImplementedError or return self? raise NotImplementedError + def isvalid(self, shape): + """ + Check whether a given index is valid on an array of a given shape. + + Returns True if an array of shape `shape` can be indexed by `self` and + False if it would raise `IndexError`. + + >>> from ndindex import ndindex + >>> ndindex(3).isvalid((4,)) + True + >>> ndindex(3).isvalid((2,)) + False + + """ + # TODO: More direct, efficient implementation + try: + self.reduce(shape) + return True + except IndexError: + return False + def expand(self, shape): r""" Expand a Tuple index on an array of shape `shape` From 9d40d652cb04cdbaef8ba4ae7337108396e38190 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 13 Oct 2022 18:20:17 -0600 Subject: [PATCH 003/218] Start updating test_iter_indices for non-broadcastable skip_axes --- ndindex/tests/helpers.py | 36 +++++++++++++++++- ndindex/tests/test_ndindex.py | 69 ++++++++++++++++++++++------------- 2 files changed, 79 insertions(+), 26 deletions(-) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index a14ddd19..a694db40 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -114,7 +114,7 @@ def _mutually_broadcastable_shapes(draw): mutually_broadcastable_shapes = shared(_mutually_broadcastable_shapes()) @composite -def skip_axes(draw): +def _skip_axes(draw): shapes, result_shape = draw(mutually_broadcastable_shapes) n = len(result_shape) axes = draw(one_of(none(), @@ -126,6 +126,40 @@ def skip_axes(draw): return axes[0] return axes +skip_axes = shared(_skip_axes()) + +@composite +def mutually_broadcastable_shapes_with_skipped_axes(draw): + """ + mutually_broadcastable_shapes except skip_axes() axes might not be + broadcastable + + The result_shape will be None in the position of skip_axes. + """ + skip_axes_ = draw(skip_axes) + shapes, result_shape = draw(mutually_broadcastable_shapes) + if skip_axes_ is None: + return shapes, result_shape + if isinstance(skip_axes_, int): + skip_axes_ = (skip_axes_,) + + _shapes = [] + for shape in shapes: + _shape = list(shape) + for i in skip_axes_: + if draw(booleans()): + _shape[i] = draw(integers(0)) + + _shapes.append(tuple(_shape)) + + _result_shape = list(result_shape) + for i in skip_axes_: + _result_shape[i] = None + _result_shape = tuple(_result_shape) + + return BroadcastableShapes(_shapes, _result_shape) + + # We need to make sure shapes for boolean arrays are generated in a way that # makes them related to the test array shape. Otherwise, it will be very # difficult for the boolean array index to match along the test array, which diff --git a/ndindex/tests/test_ndindex.py b/ndindex/tests/test_ndindex.py index 0f90b8e1..bf54dc4c 100644 --- a/ndindex/tests/test_ndindex.py +++ b/ndindex/tests/test_ndindex.py @@ -14,7 +14,8 @@ from ..integerarray import IntegerArray from ..tuple import Tuple from .helpers import (ndindices, check_same, assert_equal, prod, - mutually_broadcastable_shapes, skip_axes) + mutually_broadcastable_shapes_with_skipped_axes, + skip_axes) @given(ndindices) def test_eq(idx): @@ -155,37 +156,52 @@ def test_asshape(): @example([((1, 1), (1, 1)), (1, 1)], (0, 0)) @example([((), (0,)), (0,)], (0,)) @example([((1, 2), (2, 1)), (2, 2)], 1) -@given(mutually_broadcastable_shapes, skip_axes()) -def test_iter_indices(broadcastable_shapes, skip_axes): +@given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes) +def test_iter_indices(broadcastable_shapes, _skip_axes): + # broadcasted_shape will contain None on the skip_axes, as those axes + # might not be broadcast compatible shapes, broadcasted_shape = broadcastable_shapes - if skip_axes is None: - res = iter_indices(*shapes) - broadcasted_res = iter_indices(np.broadcast_shapes(*shapes)) - skip_axes = () - else: - res = iter_indices(*shapes, skip_axes=skip_axes) - broadcasted_res = iter_indices(np.broadcast_shapes(*shapes), - skip_axes=skip_axes) - - if isinstance(skip_axes, int): - skip_axes = (skip_axes,) - - sizes = [prod(shape) for shape in shapes] + # 1. Normalize inputs + skip_axes = (_skip_axes,) if isinstance(_skip_axes, int) else _skip_axes ndim = len(broadcasted_shape) - arrays = [np.arange(size).reshape(shape) for size, shape in zip(sizes, shapes)] - broadcasted_arrays = np.broadcast_arrays(*arrays) # Use negative indices to index the skip axes since only shapes that have # the skip axis will include a slice. normalized_skip_axes = sorted(ndindex(i).reduce(ndim).args[0] - ndim for i in skip_axes) - skip_shapes = [tuple(shape[i] for i in normalized_skip_axes if -i <= len(shape)) for shape in shapes] + canonical_shapes = [list(s) for s in shapes] + for i in normalized_skip_axes: + for s in canonical_shapes: + if ndindex(i).isvalid(s): + s[i] = 1 + skip_shapes = [tuple(shape[i] for i in normalized_skip_axes if ndindex(i).isvalid(shape)) for shape in canonical_shapes] broadcasted_skip_shape = tuple(broadcasted_shape[i] for i in normalized_skip_axes) broadcasted_non_skip_shape = tuple(broadcasted_shape[i] for i in range(-1, -ndim-1, -1) if i not in normalized_skip_axes) nitems = prod(broadcasted_non_skip_shape) - broadcasted_nitems = prod(broadcasted_shape) + broadcasted_nitems = prod([i for i in broadcasted_shape if i is not None]) + if skip_axes is None: + res = iter_indices(*shapes) + broadcasted_res = iter_indices(np.broadcast_shapes(*shapes)) + skip_axes = () + else: + # Skipped axes may not be broadcast compatible. Since the index for a + # skipped axis should always be a slice(None), the result should be + # the same if the skipped axes are all replaced with 1. + res = iter_indices(*shapes, skip_axes=_skip_axes) + broadcasted_res = iter_indices(np.broadcast_shapes(*canonical_shapes), + skip_axes=_skip_axes) + + sizes = [prod(shape) for shape in shapes] + arrays = [np.arange(size).reshape(shape) for size, shape in zip(sizes, shapes)] + canonical_sizes = [prod(shape) for shape in canonical_shapes] + canonical_arrays = [np.arange(size).reshape(shape) for size, shape in zip(canonical_sizes, canonical_shapes)] + broadcasted_arrays = np.broadcast_arrays(*canonical_arrays) + + # 2. Check that iter_indices is the same whether or not the shapes are + # broadcasted together first. Also Check that every iterated index is the + # expected type and there are as many as expected. vals = [] n = -1 try: @@ -201,10 +217,11 @@ def test_iter_indices(broadcastable_shapes, skip_axes): assert isinstance(idx.args[i], Integer) aidxes = tuple([a[idx.raw] for a, idx in zip(arrays, idxes)]) + canonical_aidxes = tuple([a[idx.raw] for a, idx in zip(canonical_arrays, idxes)]) a_broadcasted_idxs = [a[idx.raw] for a, idx in zip(broadcasted_arrays, bidxes)] - for aidx, abidx, skip_shape in zip(aidxes, a_broadcasted_idxs, skip_shapes): + for aidx, abidx, skip_shape in zip(canonical_aidxes, a_broadcasted_idxs, skip_shapes): if skip_shape == broadcasted_skip_shape: assert_equal(aidx, abidx) assert aidx.shape == skip_shape @@ -212,7 +229,7 @@ def test_iter_indices(broadcastable_shapes, skip_axes): if skip_axes: # If there are skipped axes, recursively call iter_indices to # get each individual element of the resulting subarrays. - for subidxes in iter_indices(*[x.shape for x in aidxes]): + for subidxes in iter_indices(*[x.shape for x in canonical_aidxes]): items = [x[i.raw] for x, i in zip(aidxes, subidxes)] # An empty array means the iteration would be skipped. if any(a.size == 0 for a in items): @@ -227,15 +244,17 @@ def test_iter_indices(broadcastable_shapes, skip_axes): return raise # pragma: no cover - assert len(set(vals)) == len(vals) == broadcasted_nitems + assert len(set(vals)) == len(vals) == nitems + + # 3. Check that every element of the (broadcasted) arrays is represented + # by an iterated index. # The indices should correspond to the values that would be matched up # if the arrays were broadcasted together. if not arrays: assert vals == [()] else: - correct_vals = [tuple(i) for i in np.stack(broadcasted_arrays, axis=-1) - .reshape((broadcasted_nitems, len(arrays)))] + correct_vals = [tuple(i) for i in np.stack(broadcasted_arrays, axis=-1).reshape((nitems, len(arrays)))] # Also test that the indices are produced in a lexicographic order # (even though this isn't strictly guaranteed by the iter_indices # docstring) in the case when there are no skip axes. The order when From 66f2d3c9526d1ac87b91a4265e4d31da0d8a70bc Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 16:56:45 -0600 Subject: [PATCH 004/218] Fix the test for handling of invalid ragged arrays --- ndindex/array.py | 7 +++++-- ndindex/tests/test_ndindex.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ndindex/array.py b/ndindex/array.py index 618e5ff4..2e2d3b91 100644 --- a/ndindex/array.py +++ b/ndindex/array.py @@ -19,7 +19,7 @@ class ArrayIndex(NDIndex): def _typecheck(self, idx, shape=None, _copy=True): try: - from numpy import ndarray, asarray, integer, bool_, empty, intp + from numpy import ndarray, asarray, integer, bool_, empty, intp, VisibleDeprecationWarning except ImportError: # pragma: no cover raise ImportError("NumPy must be installed to create array indices") @@ -37,7 +37,10 @@ def _typecheck(self, idx, shape=None, _copy=True): if isinstance(idx, (list, ndarray, bool, integer, int, bool_)): # Ignore deprecation warnings for things like [1, []]. These will be # filtered out anyway since they produce object arrays. - with warnings.catch_warnings(record=True): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', + category=VisibleDeprecationWarning, + message='Creating an ndarray from ragged nested sequences') a = asarray(idx) if a is idx and _copy: a = a.copy() diff --git a/ndindex/tests/test_ndindex.py b/ndindex/tests/test_ndindex.py index 0f90b8e1..ce42eeb3 100644 --- a/ndindex/tests/test_ndindex.py +++ b/ndindex/tests/test_ndindex.py @@ -1,11 +1,12 @@ import inspect +import warnings import numpy as np from hypothesis import given, example, settings from hypothesis.strategies import integers -from pytest import raises, warns +from pytest import raises from ..ndindex import ndindex, asshape, iter_indices, ncycles, BroadcastError, AxisError from ..booleanarray import BooleanArray @@ -103,7 +104,9 @@ def test_ndindex_invalid(): # This index is allowed by NumPy, but gives a deprecation warnings. We are # not going to allow indices that give deprecation warnings in ndindex. - with warns(None) as r: # Make sure no warnings are emitted from ndindex() + with warnings.catch_warnings(record=True) as r: + # Make sure no warnings are emitted from ndindex() + warnings.simplefilter("error") raises(IndexError, lambda: ndindex([1, []])) assert not r From 07157430c164c88b32646e35061a9b54b8ac23de Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 17:25:29 -0600 Subject: [PATCH 005/218] Add an @example for coverage --- ndindex/tests/test_ndindex.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ndindex/tests/test_ndindex.py b/ndindex/tests/test_ndindex.py index ce42eeb3..a58b0260 100644 --- a/ndindex/tests/test_ndindex.py +++ b/ndindex/tests/test_ndindex.py @@ -17,6 +17,7 @@ from .helpers import (ndindices, check_same, assert_equal, prod, mutually_broadcastable_shapes, skip_axes) +@example([1, 2]) @given(ndindices) def test_eq(idx): index = ndindex(idx) From 1bcd427382c61311c597a3e6a32c931cdeb54ad1 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 17:25:35 -0600 Subject: [PATCH 006/218] Ignore coverage for some test code that only triggers in NumPy < 1.23 --- ndindex/tests/helpers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index a14ddd19..d8d452fd 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -215,7 +215,10 @@ def assert_equal(x, y): try: a_raw = raw_func(a, idx) except Warning as w: - if ("Using a non-tuple sequence for multidimensional indexing is deprecated" in w.args[0]): + # In NumPy < 1.23, this is a FutureWarning. In 1.23 the + # deprecation was removed and lists are always interpreted as + # array indices. + if ("Using a non-tuple sequence for multidimensional indexing is deprecated" in w.args[0]): # pragma: no cover idx = array(idx) a_raw = raw_func(a, idx) elif "Out of bound index found. This was previously ignored when the indexing result contained no elements. In the future the index error will be raised. This error occurs either due to an empty slice, or if an array has zero elements even before indexing." in w.args[0]: From 335046b1719649846a0356a09dca4a8670083cb6 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 17:47:13 -0600 Subject: [PATCH 007/218] Fix docs build warnings --- docs/changelog.md | 2 +- docs/slices.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 6537ab43..34d7af2f 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -202,7 +202,7 @@ run the ndindex test suite due to the way ndindex tests itself against NumPy. the [type confusion](type-confusion-tuples) between `Tuple((1, 2))` and `Tuple(1, 2)` (only the latter form is correct). -- Document the [`.args`](args) attribute. +- Document the [`.args`](ndindex.ndindex.ImmutableObject.args) attribute. - New internal function {func}`~.operator_index`, which acts like `operator.index()` except it disallows boolean types. A consequence of this diff --git a/docs/slices.md b/docs/slices.md index b9a1573c..2aadd5c4 100644 --- a/docs/slices.md +++ b/docs/slices.md @@ -286,7 +286,7 @@ One consequence of this is that, unlike integer indices, **slices will never raise `IndexError`, even if the slice is empty**. Therefore you cannot rely on runtime errors to alert you to coding mistakes relating to slice bounds that are too large. A slice cannot be "out of bounds." See the section on -[clipping](#clipping) below. +[clipping](clipping) below. (0-based)= ### 0-based From d9fcd1a5a8086a030cd689486cfef7e1e4b0aea1 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 17:48:38 -0600 Subject: [PATCH 008/218] Remove redundant headers from the API docs Sphinx now includes the API docs in the toctree, so this is no longer necessary. --- docs/api.rst | 60 ---------------------------------------------------- 1 file changed, 60 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 57013e83..4676410c 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -10,9 +10,6 @@ with indices. ndindex ======= -ndindex -------- - .. autofunction:: ndindex.ndindex Index Types @@ -20,8 +17,6 @@ Index Types The following classes represent different types of indices. -Integer -------- .. autoclass:: ndindex.Integer :members: @@ -29,35 +24,19 @@ Integer .. _slice-api: -Slice ------ - .. autoclass:: ndindex.Slice :members: :special-members: -ellipsis --------- - .. autoclass:: ndindex.ellipsis :members: - -Newaxis -------- - .. autoclass:: ndindex.Newaxis :members: -Tuple ------ - .. autoclass:: ndindex.Tuple :members: -IntegerArray ------------- - .. autoclass:: ndindex.IntegerArray :members: :inherited-members: @@ -66,9 +45,6 @@ IntegerArray .. autoattribute:: dtype :annotation: -BooleanArray ------------- - .. autoclass:: ndindex.BooleanArray :members: :inherited-members: @@ -83,19 +59,10 @@ Index Helpers The functions here are helpers for working with indices that aren't methods of the index objects. -iter_indices ------------- - .. autofunction:: ndindex.iter_indices -BroadcastError --------------- - .. autoexception:: ndindex.BroadcastError -AxisError ---------- - .. autoexception:: ndindex.AxisError Chunking @@ -103,9 +70,6 @@ Chunking ndindex contains objects to represent chunking an array. -ChunkSize ---------- - .. autoclass:: ndindex.ChunkSize :members: @@ -115,21 +79,12 @@ Internal API These classes are only intended for internal use in ndindex. They shouldn't relied on as they may be removed or changed. -ImmutableObject ---------------- - .. autoclass:: ndindex.ndindex.ImmutableObject :members: -NDIndex -------- - .. autoclass:: ndindex.ndindex.NDIndex :members: -ArrayIndex ----------- - .. autoclass:: ndindex.array.ArrayIndex :members: :exclude-members: dtype @@ -137,27 +92,12 @@ ArrayIndex .. autoattribute:: dtype :annotation: Subclasses should redefine this -default -------- - .. autoclass:: ndindex.slice.default -asshape -------- - .. autofunction:: ndindex.ndindex.asshape -operator_index --------------- - .. autofunction:: ndindex.ndindex.operator_index -ncycles -------- - .. autofunction:: ndindex.ndindex.ncycles -broadcast_shapes ----------------- - .. autofunction:: ndindex.ndindex.broadcast_shapes From b7694a87bd9890e9233d2896930d2ea71a38ece8 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 18:06:33 -0600 Subject: [PATCH 009/218] Fix the symbolic link to the logo in _static --- docs/logo/ndindex_logo_white_bg.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/logo/ndindex_logo_white_bg.svg b/docs/logo/ndindex_logo_white_bg.svg index 3928df16..7a9255c0 120000 --- a/docs/logo/ndindex_logo_white_bg.svg +++ b/docs/logo/ndindex_logo_white_bg.svg @@ -1 +1 @@ -_static/ndindex_logo_white_bg.svg \ No newline at end of file +../_static/ndindex_logo_white_bg.svg \ No newline at end of file From 7e83052b6112efc794446256bdfde92d47091a39 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 19:15:58 -0600 Subject: [PATCH 010/218] Switch the docs theme to Furo --- .github/workflows/docs.yml | 2 +- docs/_pygments/styles.py | 102 ++++++++++++++++++++++++ docs/_static/custom.css | 156 ++++++++++++++++--------------------- docs/conf.py | 156 ++++++++++++++++++++++++++++--------- 4 files changed, 291 insertions(+), 125 deletions(-) create mode 100644 docs/_pygments/styles.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8d05cc2a..93617ab2 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,7 +22,7 @@ jobs: conda config --add channels conda-forge conda update -q conda conda info -a - conda create -n test-environment python=${{ matrix.python-version }} pyflakes pytest pytest-doctestplus numpy hypothesis doctr sphinx myst-parser sphinx_rtd_theme pytest-cov pytest-flakes + conda create -n test-environment python=${{ matrix.python-version }} pyflakes pytest pytest-doctestplus numpy hypothesis doctr sphinx myst-parser sphinx_rtd_theme pytest-cov pytest-flakes furo sphinx-copybutton conda init - name: Build Docs diff --git a/docs/_pygments/styles.py b/docs/_pygments/styles.py new file mode 100644 index 00000000..1269bf5e --- /dev/null +++ b/docs/_pygments/styles.py @@ -0,0 +1,102 @@ +""" +Pygments styles used for syntax highlighting. + +These are based on the Sphinx style (see +https://github.com/sphinx-doc/sphinx/blob/master/sphinx/pygments_styles.py) +for light mode and the Friendly style for dark mode. + +The styles here have been adjusted so that they are WCAG AA compatible. The +tool at https://github.com/mpchadwick/pygments-high-contrast-stylesheets was +used to identify colors that should be adjusted. + +""" +from pygments.style import Style +from pygments.styles.friendly import FriendlyStyle +from pygments.styles.native import NativeStyle +from pygments.token import Comment, Generic, Literal, Name, Number, Text + +class SphinxHighContrastStyle(Style): + """ + Like Sphinx (which is like friendly, but a bit darker to enhance contrast + on the green background) but with higher contrast colors. + + """ + + @property + def _pre_style(self): + # This is used instead of the default 125% so that multiline Unicode + # pprint output looks good + return 'line-height: 120%;' + + background_color = '#eeffcc' + default_style = '' + + styles = FriendlyStyle.styles + styles.update({ + # These are part of the Sphinx modification to "friendly" + Generic.Output: '#333', + Number: '#208050', + + # These are adjusted from "friendly" (Comment is adjusted from + # "sphinx") to have better color contrast against the background. + Comment: 'italic #3c7a88', + Comment.Hashbang: 'italic #3c7a88', + Comment.Multiline: 'italic #3c7a88', + Comment.PreprocFile: 'italic #3c7a88', + Comment.Single: 'italic #3c7a88', + Comment.Special: '#3a7784 bg:#fff0f0', + Generic.Error: '#e60000', + Generic.Inserted: '#008200', + Generic.Prompt: 'bold #b75709', + Name.Class: 'bold #0e7ba6', + Name.Constant: '#2b79a1', + Name.Entity: 'bold #c54629', + Name.Namespace: 'bold #0e7ba6', + Name.Variable: '#ab40cd', + Text.Whitespace: '#707070', + Literal.String.Interpol: 'italic #3973b7', + Literal.String.Other: '#b75709', + Name.Variable.Class: '#ab40cd', + Name.Variable.Global: '#ab40cd', + Name.Variable.Instance: '#ab40cd', + Name.Variable.Magic: '#ab40cd', + }) + + + +class NativeHighContrastStyle(NativeStyle): + """ + Like native, but with higher contrast colors. + """ + @property + def _pre_style(self): + # This is used instead of the default 125% so that multiline Unicode + # pprint output looks good + return 'line-height: 120%;' + + styles = NativeStyle.styles + + # These are adjusted to have better color contrast against the background + styles.update({ + Comment.Preproc: 'bold #e15a5a', + Comment.Special: 'bold #f75050 bg:#520000', + Generic.Deleted: '#e75959', + Generic.Error: '#e75959', + Generic.Traceback: '#e75959', + Literal.Number: '#438dc4', + Name.Builtin: '#2594a1', + # We also remove the underline here from the original style + Name.Class: '#548bd3', + Name.Function: '#548bd3', + # We also remove the underline here from the original style + Name.Namespace: '#548bd3', + Text.Whitespace: '#878787', + Literal.Number.Bin: '#438dc4', + Literal.Number.Float: '#438dc4', + Literal.Number.Hex: '#438dc4', + Literal.Number.Integer: '#438dc4', + Literal.Number.Oct: '#438dc4', + Name.Builtin.Pseudo: '#2594a1', + Name.Function.Magic: '#548bd3', + Literal.Number.Integer.Long: '#438dc4', + }) diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 3b3ff689..239daae2 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -1,110 +1,92 @@ -nav#rellinks { - float: left; - width: 100%; +/* Make the title text in the sidebar bold */ +.sidebar-brand-text { + font-weight: bold; } -.related.prev { - float: left; - text-align: left; - width: 50%; +:root { + --color-brand-light-blue: #8DBEFE; + --color-brand-green: #1EB881; + --color-brand-medium-blue: #1041F3; + --color-brand-dark-blue: #0D2B9C; + + --color-brand-bg: white; + + --color-sidebar-current: white; + --color-sidebar-background-current: var(--color-brand-dark-blue); + } -.related.next { - float: right; - text-align: right; - width: 50% +@media (prefers-color-scheme: dark) { + :root { + --color-brand-bg: #05002A; + } +} +[data-theme='dark'] { + --color-brand-bg: #05002A; } -nav#rellinks li+li:before { - content: ""; +/* Make top-level items in the sidebar bold */ +.sidebar-tree .toctree-l1>.reference, .sidebar-tree .toctree-l1>label .icon { + font-weight: bold !important; } -div.sphinxsidebar { - max-height: 95%; - overflow-y: auto; +/* Indicate the current page using a background color rather than bold text */ +.sidebar-tree .current-page>.reference { + font-weight: normal; + background-color: var(--color-sidebar-background-current); + color: var(--color-sidebar-current); +} +.sidebar-tree .reference:hover { + color: white; } -div.sphinxsidebar p.logo { - display: inline; +/* The "hide search matches" text after doing a search. Defaults to the same + color as the icon which is illegible on the colored background. */ +.highlight-link a { + color: white !important; } -div.sphinxsidebar hr { - width: 100%; +.admonition.warning>.admonition-title { + color: white; } -/* Disable white background on some links, since our background is not white. */ -tt, code { - background-color: inherit; +/* Disable underlines on links except on hover */ +a { + text-decoration: none; +} +a:hover { + text-decoration: underline; +} +/* Keep the underline in the announcement header */ +.announcement-content a { + text-decoration: underline; } -div.admonition tt.xref, div.admonition code.xref, div.admonition a tt, tt.xref, code.xref, a tt { - background-color: inherit; - border-bottom: 0; +/* Remove the background from code in titles and the sidebar */ +code.literal { + background: inherit; } -/* Pure CSS "Fork me on GitHub" ribbon. - * From - * https://github.com/ssokolow/quicktile/commit/1ae5388ac0f2a2bfa494045644b0ba19eb042329 - * See https://github.com/bitprophet/alabaster/issues/166 - */ +/* Make "Warning" white */ +.admonition.warning>.admonition-title { + color: white; +} -#forkongithub { - position: absolute; - display: block; - top: 0; - right: 0; - width: 200px; - overflow: hidden; - height: 200px; - z-index: 9999; -} -#forkongithub a { - background: #000; - color: #fff; - text-decoration: none; - font-family: arial, sans-serif; - text-align: center; - font-weight: bold; - padding: 5px 40px; - font-size: 10pt; - line-height: 15pt; - width: 200px; - position: absolute; - top: 40px; - right: -82px; - transform: rotate(45deg); - -webkit-transform: rotate(45deg); - -ms-transform: rotate(45deg); - -moz-transform: rotate(45deg); - -o-transform: rotate(45deg); - box-shadow: 2px 2px 5px rgba(0, 0, 0, 0.2); -} -#forkongithub a::before, -#forkongithub a::after { - content: ""; - width: 100%; - display: block; - position: absolute; - top: 1px; - left: 0; - height: 1px; - background: #777; -} -#forkongithub a::after { - bottom: 1px; - top: auto; -} -@media screen and (max-width: 979px) { - #forkongithub a { - display: none; - } +/* Makes the text look better on Mac retina displays (the Furo CSS disables*/ +/* subpixel antialiasing). */ +body { + -webkit-font-smoothing: auto; + -moz-osx-font-smoothing: auto; } -@media print { - #forkongithub a { - display: none; - } + +/* Disable upcasing of headers 4+ (they are still distinguishable by*/ +/* font-weight and size) */ +h4, h5, h6 { + text-transform: inherit; } -strong:hover > a.headerlink { - visibility: visible; +/* Disable the fancy scrolling behavior when jumping to headers (this is too + slow for long pages) */ +html { + scroll-behavior: auto; } diff --git a/docs/conf.py b/docs/conf.py index f194e553..e1aa5464 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -32,6 +32,7 @@ 'sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.intersphinx', + 'sphinx_copybutton', ] intersphinx_mapping = { @@ -70,7 +71,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = 'furo' # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -80,45 +81,126 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] -html_theme_options = { - 'fixed_sidebar': True, - 'github_user': 'Quansight-Labs', - 'github_repo': 'ndindex', - 'github_banner': False, - 'logo': 'ndindex_logo_white_bg.svg', - 'logo_name': False, - # 'show_related': True, - # Needs a release with https://github.com/bitprophet/alabaster/pull/101 first - 'show_relbars': True, - - # Colors - - 'base_bg': '#EEEEEE', - 'narrow_sidebar_bg': '#DDDDDD', - # Sidebar text - 'gray_1': '#000000', - 'narrow_sidebar_link': '#333333', - # Doctest background - 'gray_2': '#F0F8FF', - - # Remove gray background from inline code - 'code_bg': '#EEEEEE', - - # Originally 940px - 'page_width': '1000px', - - # Fonts - 'font_family': "Palatino, 'goudy old style', 'minion pro', 'bell mt', Georgia, 'Hiragino Mincho Pro', serif", - 'font_size': '18px', - 'code_font_family': "'Menlo', 'DejaVu Sans Mono', 'Consolas', 'Bitstream Vera Sans Mono', monospace", - 'code_font_size': '0.85em', - } +# ndindex brand colors, from logo/ndindex_final_2.pdf. + +light_blue = "#8DBEFE" +green = "#1EB881" +medium_blue = "#1041F3" +dark_blue = "#0D2B9C" +dark_bg = "#05002A" +white = "white" + +theme_colors_common = { + "color-sidebar-background-border": "var(--color-background-primary)", + "color-sidebar-brand-text": "var(--color-sidebar-link-text--top-level)", + + "color-admonition-title-background--seealso": "#CCCCCC", + "color-admonition-title--seealso": "black", + "color-admonition-title-background--note": "#CCCCCC", + "color-admonition-title--note": "black", + "color-admonition-title-background--warning": "var(--color-problematic)", + "color-admonition-title--warning": "white", + "admonition-font-size": "var(--font-size--normal)", + "admonition-title-font-size": "var(--font-size--normal)", + + "color-link-underline--hover": "var(--color-link)", + + "color-api-keyword": "#000000bd", + "color-api-name": "var(--color-brand-content)", + "color-api-pre-name": "var(--color-brand-content)", + "api-font-size": "var(--font-size--normal)", -html_sidebars = { - '**': ['globaltocindex.html', 'searchbox.html'], -} + } +html_theme_options = { + "light_css_variables": { + **theme_colors_common, + "color-brand-primary": dark_blue, + "color-brand-content": dark_blue, + + "color-sidebar-background": light_blue, + "color-sidebar-item-background--hover": medium_blue, + "color-sidebar-item-expander-background--hover": medium_blue, + + }, + # Don't use any custom colors for dark mode. See https://github.com/pradyunsg/furo/blob/main/src/furo/assets/styles/variables/_colors.scss + "dark_css_variables": { + **theme_colors_common, + "color-brand-primary": light_blue, + "color-brand-content": light_blue, + + "color-api-keyword": "#FFFFFFbd", + "color-api-overall": "#FFFFFF90", + "color-api-paren": "#FFFFFF90", + + "color-sidebar-background": dark_bg, + "color-sidebar-item-background--hover": dark_blue, + "color-sidebar-item-expander-background--hover": dark_blue, + + "color-highlight-on-target": dark_blue, + + "color-admonition-title-background--seealso": "#555555", + "color-admonition-title-background--note": "#555555", + "color-problematic": "#B30000", + }, + # See https://pradyunsg.me/furo/customisation/footer/ + "footer_icons": [ + { + "name": "GitHub", + "url": "https://github.com/Quansight-Labs/ndindex", + "html": """ + + + + """, + "class": "", + }, + ], +} +# html_theme_options = { +# 'fixed_sidebar': True, +# 'github_user': 'Quansight-Labs', +# 'github_repo': 'ndindex', +# 'github_banner': False, +# 'logo': 'ndindex_logo_white_bg.svg', +# 'logo_name': False, +# # 'show_related': True, +# # Needs a release with https://github.com/bitprophet/alabaster/pull/101 first +# 'show_relbars': True, +# +# # Colors +# +# 'base_bg': '#EEEEEE', +# 'narrow_sidebar_bg': '#DDDDDD', +# # Sidebar text +# 'gray_1': '#000000', +# 'narrow_sidebar_link': '#333333', +# # Doctest background +# 'gray_2': '#F0F8FF', +# +# # Remove gray background from inline code +# 'code_bg': '#EEEEEE', +# +# # Originally 940px +# 'page_width': '1000px', +# +# # Fonts +# 'font_family': "Palatino, 'goudy old style', 'minion pro', 'bell mt', Georgia, 'Hiragino Mincho Pro', serif", +# 'font_size': '18px', +# 'code_font_family': "'Menlo', 'DejaVu Sans Mono', 'Consolas', 'Bitstream Vera Sans Mono', monospace", +# 'code_font_size': '0.85em', +# } + +# custom.css contains changes that aren't possible with the above because they +# aren't specified in the Furo theme as CSS variables +html_css_files = ['custom.css'] + +sys.path.append(os.path.abspath("./_pygments")) +pygments_style = 'styles.SphinxHighContrastStyle' +pygments_dark_style = 'styles.NativeHighContrastStyle' + +html_logo = '_static/ndindex_logo_white_bg.svg' html_favicon = "logo/favicon.ico" mathjax3_config = { From b668c3b43cd1dafdc7afc4ddabb6d9b2f9447afe Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 19:17:08 -0600 Subject: [PATCH 011/218] Add docs deployment to GitHub Actions --- .github/workflows/docs.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 93617ab2..ad69abc4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -51,3 +51,10 @@ jobs: conda activate test-environment cd docs make html + + - name: Deploy + uses: JamesIves/github-pages-deploy-action@v4 + if: ${{ github.ref == 'refs/heads/master' }} + with: + folder: docs/_build/html + ssh-key: ${{ secrets.DEPLOY_KEY }} From eb2ba8e059468e8706f1c13dde862417601a5c92 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 19:17:42 -0600 Subject: [PATCH 012/218] Disable gh-pages deploy from the release script --- rever.xsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rever.xsh b/rever.xsh index b443608e..b9b21282 100644 --- a/rever.xsh +++ b/rever.xsh @@ -47,7 +47,7 @@ $ACTIVITIES = [ 'pypi', # Sends the package to pypi 'push_tag', # Pushes the tag up to the $TAG_REMOTE 'ghrelease', # Creates a Github release entry for the new tag - 'ghpages', # Update GitHub Pages + # 'ghpages', # Update GitHub Pages ] $PUSH_TAG_REMOTE = 'git@github.com:Quansight-Labs/ndindex.git' # Repo to push tags to From 598b67bb56f6f1d30fe186c833d090bfcd1558c5 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 19:19:41 -0600 Subject: [PATCH 013/218] Add Python 3.11 to the GitHub Actions test matrix This might not actually work yet if the conda packages aren't all built for 3.11. --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6e548941..e7d473cd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,7 +5,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] fail-fast: false steps: - uses: actions/checkout@v2 From 08a0c3469c38dd1aa7c8e65252407a65790cdda1 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 19:21:24 -0600 Subject: [PATCH 014/218] Clean up the dependencies that are installed in GitHub Actions --- .github/workflows/docs.yml | 2 +- .github/workflows/tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ad69abc4..3e35e288 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,7 +22,7 @@ jobs: conda config --add channels conda-forge conda update -q conda conda info -a - conda create -n test-environment python=${{ matrix.python-version }} pyflakes pytest pytest-doctestplus numpy hypothesis doctr sphinx myst-parser sphinx_rtd_theme pytest-cov pytest-flakes furo sphinx-copybutton + conda create -n test-environment python=${{ matrix.python-version }} sphinx myst-parser furo sphinx-copybutton conda init - name: Build Docs diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6e548941..5619098b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,7 +22,7 @@ jobs: conda config --add channels conda-forge conda update -q conda conda info -a - conda create -n test-environment python=${{ matrix.python-version }} pyflakes pytest pytest-doctestplus numpy hypothesis doctr sphinx myst-parser sphinx_rtd_theme pytest-cov pytest-flakes packaging + conda create -n test-environment python=${{ matrix.python-version }} pyflakes pytest pytest-doctestplus numpy hypothesis pytest-cov pytest-flakes packaging conda init - name: Run Tests From 3a2b95f39c2680ebb3bcb967c1829d18742a5d1f Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 19:36:19 -0600 Subject: [PATCH 015/218] Updates to some comments --- docs/_static/custom.css | 2 +- docs/conf.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 239daae2..96203e0a 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -41,7 +41,7 @@ } /* The "hide search matches" text after doing a search. Defaults to the same - color as the icon which is illegible on the colored background. */ + color as the search icon which is illegible on the colored background. */ .highlight-link a { color: white !important; } diff --git a/docs/conf.py b/docs/conf.py index e1aa5464..9bc7199d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -123,7 +123,6 @@ "color-sidebar-item-expander-background--hover": medium_blue, }, - # Don't use any custom colors for dark mode. See https://github.com/pradyunsg/furo/blob/main/src/furo/assets/styles/variables/_colors.scss "dark_css_variables": { **theme_colors_common, "color-brand-primary": light_blue, From f077540a769108039d25c907bd9f00acbaaa3e1d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 19:43:40 -0600 Subject: [PATCH 016/218] Add a docs preview build to the CI --- .circleci/config.yml | 42 ++++++++++++++++++++++++++++++ .github/workflows/docs-preview.yml | 20 ++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 .circleci/config.yml create mode 100644 .github/workflows/docs-preview.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..ce9e5369 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,42 @@ +# This builds a preview of the docs which can be seen on pull requests. It +# also uses the .github/workflows/docs-preview.yml GitHub Actions workflow. + +# This is separate from the GitHub Actions build that builds the docs, which +# also deploys the docs +version: 2 + +# Aliases to reuse +_defaults: &defaults + docker: + # CircleCI maintains a library of pre-built images + # documented at https://circleci.com/docs/2.0/circleci-images/ + - image: cimg/python:3.10.2 + working_directory: ~/repo + +jobs: + Build Docs Preview: + <<: *defaults + steps: + - checkout + - attach_workspace: + at: ~/ + - run: + name: Install dependencies + no_output_timeout: 25m + command: | + cd doc + pip install sphinx myst-parser furo sphinx-copybutton + - run: + name: Build docs + no_output_timeout: 25m + command: | + cd doc + make html + - store_artifacts: + path: doc/_build/html + +workflows: + version: 2 + default: + jobs: + - Build Docs Preview diff --git a/.github/workflows/docs-preview.yml b/.github/workflows/docs-preview.yml new file mode 100644 index 00000000..15b5c004 --- /dev/null +++ b/.github/workflows/docs-preview.yml @@ -0,0 +1,20 @@ +name: Docs Preview +on: [status] +jobs: + circleci_artifacts_redirector_job: + if: "${{ github.event.context == 'ci/circleci: Build Docs Preview' }}" + runs-on: ubuntu-latest + name: Run CircleCI artifacts redirector + steps: + - name: GitHub Action step + id: step1 + uses: larsoner/circleci-artifacts-redirector-action@master + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + artifact-path: 0/doc/_build/html/index.html + circleci-jobs: Build Docs Preview + job-title: Click here to see a preview of the documentation. + - name: Check the URL + if: github.event.status != 'pending' + run: | + curl --fail ${{ steps.step1.outputs.url }} | grep $GITHUB_SHA From 6886146d4c975e33f68556188bab12811fae327a Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 19:17:08 -0600 Subject: [PATCH 017/218] Add docs deployment to GitHub Actions --- .github/workflows/docs.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8d05cc2a..c58b5d35 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -51,3 +51,10 @@ jobs: conda activate test-environment cd docs make html + + - name: Deploy + uses: JamesIves/github-pages-deploy-action@v4 + if: ${{ github.ref == 'refs/heads/master' }} + with: + folder: docs/_build/html + ssh-key: ${{ secrets.DEPLOY_KEY }} From 030ca9d2a566cdf0ff75e3fe17b34a273c69cd2a Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 19:17:42 -0600 Subject: [PATCH 018/218] Disable gh-pages deploy from the release script --- rever.xsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rever.xsh b/rever.xsh index b443608e..b9b21282 100644 --- a/rever.xsh +++ b/rever.xsh @@ -47,7 +47,7 @@ $ACTIVITIES = [ 'pypi', # Sends the package to pypi 'push_tag', # Pushes the tag up to the $TAG_REMOTE 'ghrelease', # Creates a Github release entry for the new tag - 'ghpages', # Update GitHub Pages + # 'ghpages', # Update GitHub Pages ] $PUSH_TAG_REMOTE = 'git@github.com:Quansight-Labs/ndindex.git' # Repo to push tags to From b6fef56961202324392d208ae0dacb1549379fd7 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 19:49:55 -0600 Subject: [PATCH 019/218] Add note about setting up a deploy key --- .github/workflows/docs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c58b5d35..492ca3a3 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -52,6 +52,9 @@ jobs: cd docs make html + # Note, the gh-pages deployment requires setting up a SSH deploy key. + # See + # https://github.com/JamesIves/github-pages-deploy-action/tree/dev#using-an-ssh-deploy-key- - name: Deploy uses: JamesIves/github-pages-deploy-action@v4 if: ${{ github.ref == 'refs/heads/master' }} From 8df73a98612018c75f3f27216257ef2af9db213a Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 19:54:21 -0600 Subject: [PATCH 020/218] Fix docs build script on Circle --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ce9e5369..fb757b50 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,13 +24,13 @@ jobs: name: Install dependencies no_output_timeout: 25m command: | - cd doc + cd docs pip install sphinx myst-parser furo sphinx-copybutton - run: name: Build docs no_output_timeout: 25m command: | - cd doc + cd docs make html - store_artifacts: path: doc/_build/html From 51edad94f0f0447d5a3e3caa1d389548587ea107 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 19:56:19 -0600 Subject: [PATCH 021/218] Trigger CI From 929225dad9bfb73128d468530bfb158be5afcd76 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 19:59:21 -0600 Subject: [PATCH 022/218] Fix Circle docs preview artifact path --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fb757b50..c4d57436 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,7 +33,7 @@ jobs: cd docs make html - store_artifacts: - path: doc/_build/html + path: docs/_build/html workflows: version: 2 From 20e28d58545fbc9d7184f29f7099ca80f4172df1 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 20:01:22 -0600 Subject: [PATCH 023/218] Fix the docs preview path in the docs-preview GitHub Actions workflow --- .github/workflows/docs-preview.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-preview.yml b/.github/workflows/docs-preview.yml index 15b5c004..fce6fc2a 100644 --- a/.github/workflows/docs-preview.yml +++ b/.github/workflows/docs-preview.yml @@ -11,7 +11,7 @@ jobs: uses: larsoner/circleci-artifacts-redirector-action@master with: repo-token: ${{ secrets.GITHUB_TOKEN }} - artifact-path: 0/doc/_build/html/index.html + artifact-path: 0/docs/_build/html/index.html circleci-jobs: Build Docs Preview job-title: Click here to see a preview of the documentation. - name: Check the URL From a066389a034e57d2e5539e664edb9bb693d28dbc Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 20:05:35 -0600 Subject: [PATCH 024/218] Remove the underline from the title in the sidebar on hover --- docs/_static/custom.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 96203e0a..d09f793d 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -2,6 +2,10 @@ .sidebar-brand-text { font-weight: bold; } +/* Remove the underline from the title text on hover */ +.sidebar-brand:hover { + text-decoration: none !important; +} :root { --color-brand-light-blue: #8DBEFE; From cbc24002a43b03fcdf3fa9296b283e3fdbe6f27a Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 20:14:50 -0600 Subject: [PATCH 025/218] Make the current page background color different from hover in dark mode --- docs/_static/custom.css | 2 ++ docs/conf.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/_static/custom.css b/docs/_static/custom.css index d09f793d..29f33565 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -22,10 +22,12 @@ @media (prefers-color-scheme: dark) { :root { + --color-sidebar-background-current: var(--color-brand-dark-blue); --color-brand-bg: #05002A; } } [data-theme='dark'] { + --color-sidebar-background-current: var(--color-brand-dark-blue); --color-brand-bg: #05002A; } diff --git a/docs/conf.py b/docs/conf.py index 9bc7199d..f6d7b80d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -133,8 +133,8 @@ "color-api-paren": "#FFFFFF90", "color-sidebar-background": dark_bg, - "color-sidebar-item-background--hover": dark_blue, - "color-sidebar-item-expander-background--hover": dark_blue, + "color-sidebar-item-background--hover": medium_blue, + "color-sidebar-item-expander-background--hover": medium_blue, "color-highlight-on-target": dark_blue, From 44ca50c390cb12c448a8717f72404c0490083b8f Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 20:19:37 -0600 Subject: [PATCH 026/218] Add a header to the docs in the prevew builds --- docs/conf.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index f6d7b80d..aca42cc3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -214,3 +214,14 @@ # Lets us use single backticks for code default_role = 'code' + +# Add a header for PR preview builds. See the Circle CI configuration. +if os.environ.get("CIRCLECI") == "true": + PR_NUMBER = os.environ.get('CIRCLE_PR_NUMBER') + SHA1 = os.environ.get('CIRCLE_SHA1') + html_theme_options['announcement'] = f"""This is a preview build from +ndindex pull request +#{PR_NUMBER}. It was built against {SHA1[:7]}. +If you aren't looking for a PR preview, go to the main ndindex documentation. """ From 3bdba1ad87cd62b44d35887f6d5e4cb8d67f0cea Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 20:32:34 -0600 Subject: [PATCH 027/218] Fix MathJax in the docs --- docs/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index f194e553..57449fd1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -121,6 +121,8 @@ html_favicon = "logo/favicon.ico" +myst_enable_extensions = ["dollarmath"] + mathjax3_config = { 'TeX': { 'equationNumbers': { From 82a51d8c9be673a0334596b1f5b6a41a8532a10e Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 20:40:16 -0600 Subject: [PATCH 028/218] Remove the GITHUB_SHA grep from the docs preview URL check It is failing for some reason but the URLs seem to be fine. --- .github/workflows/docs-preview.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-preview.yml b/.github/workflows/docs-preview.yml index fce6fc2a..b80ea9dc 100644 --- a/.github/workflows/docs-preview.yml +++ b/.github/workflows/docs-preview.yml @@ -17,4 +17,4 @@ jobs: - name: Check the URL if: github.event.status != 'pending' run: | - curl --fail ${{ steps.step1.outputs.url }} | grep $GITHUB_SHA + curl --fail ${{ steps.step1.outputs.url }} From 858d588ba0a09b4e75e59ee16415d2251248bc47 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 20:45:04 -0600 Subject: [PATCH 029/218] Enable the linkify docs extesion --- .circleci/config.yml | 2 +- .github/workflows/docs.yml | 2 +- docs/conf.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c4d57436..a80beb29 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ jobs: no_output_timeout: 25m command: | cd docs - pip install sphinx myst-parser furo sphinx-copybutton + pip install sphinx myst-parser furo sphinx-copybutton linkify-it-py - run: name: Build docs no_output_timeout: 25m diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index dc96b412..65d6f849 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,7 +22,7 @@ jobs: conda config --add channels conda-forge conda update -q conda conda info -a - conda create -n test-environment python=${{ matrix.python-version }} sphinx myst-parser furo sphinx-copybutton + conda create -n test-environment python=${{ matrix.python-version }} sphinx myst-parser furo sphinx-copybutton linkify-it-py conda init - name: Build Docs diff --git a/docs/conf.py b/docs/conf.py index 495b7664..e3bdc2e4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -202,7 +202,7 @@ html_logo = '_static/ndindex_logo_white_bg.svg' html_favicon = "logo/favicon.ico" -myst_enable_extensions = ["dollarmath"] +myst_enable_extensions = ["dollarmath", "linkify"] mathjax3_config = { 'TeX': { From d1e8ae27c9e1f1c28f98f5dde8d984137d5cab48 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 20:46:53 -0600 Subject: [PATCH 030/218] Move the docs dependencies into a requirements.txt file --- .circleci/config.yml | 2 +- .github/workflows/docs.yml | 2 +- docs/requirements.txt | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 docs/requirements.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index a80beb29..c2553a0d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ jobs: no_output_timeout: 25m command: | cd docs - pip install sphinx myst-parser furo sphinx-copybutton linkify-it-py + pip install -r docs/requirements.txt - run: name: Build docs no_output_timeout: 25m diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 65d6f849..c16fa6ee 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -22,7 +22,7 @@ jobs: conda config --add channels conda-forge conda update -q conda conda info -a - conda create -n test-environment python=${{ matrix.python-version }} sphinx myst-parser furo sphinx-copybutton linkify-it-py + conda create -n test-environment python=${{ matrix.python-version }} --file docs/requirements.txt conda init - name: Build Docs diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..125d74ef --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,5 @@ +furo +linkify-it-py +myst-parser +sphinx +sphinx-copybutton From 6b5c87681b4ba364be30cd1b0897237d1047ad6e Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 20:51:01 -0600 Subject: [PATCH 031/218] Fix Circle CI build --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c2553a0d..742726b6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,7 +25,7 @@ jobs: no_output_timeout: 25m command: | cd docs - pip install -r docs/requirements.txt + pip install -r requirements.txt - run: name: Build docs no_output_timeout: 25m From b51bf0d7472ea803b2710ae43c9af270edbf26af Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Oct 2022 20:54:03 -0600 Subject: [PATCH 032/218] Docs preview test From 3388854405a8a7313d46d916b050b259064cb803 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 28 Oct 2022 02:30:01 -0600 Subject: [PATCH 033/218] Revert "Remove the GITHUB_SHA grep from the docs preview URL check" This reverts commit 82a51d8c9be673a0334596b1f5b6a41a8532a10e. --- .github/workflows/docs-preview.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs-preview.yml b/.github/workflows/docs-preview.yml index b80ea9dc..fce6fc2a 100644 --- a/.github/workflows/docs-preview.yml +++ b/.github/workflows/docs-preview.yml @@ -17,4 +17,4 @@ jobs: - name: Check the URL if: github.event.status != 'pending' run: | - curl --fail ${{ steps.step1.outputs.url }} + curl --fail ${{ steps.step1.outputs.url }} | grep $GITHUB_SHA From dc8f371f8b63f33f92232acc23ab8bdfb805c27d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 28 Oct 2022 02:40:23 -0600 Subject: [PATCH 034/218] Delete the old theme options from conf.py --- docs/conf.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index e3bdc2e4..59c86489 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -157,40 +157,6 @@ ], } -# html_theme_options = { -# 'fixed_sidebar': True, -# 'github_user': 'Quansight-Labs', -# 'github_repo': 'ndindex', -# 'github_banner': False, -# 'logo': 'ndindex_logo_white_bg.svg', -# 'logo_name': False, -# # 'show_related': True, -# # Needs a release with https://github.com/bitprophet/alabaster/pull/101 first -# 'show_relbars': True, -# -# # Colors -# -# 'base_bg': '#EEEEEE', -# 'narrow_sidebar_bg': '#DDDDDD', -# # Sidebar text -# 'gray_1': '#000000', -# 'narrow_sidebar_link': '#333333', -# # Doctest background -# 'gray_2': '#F0F8FF', -# -# # Remove gray background from inline code -# 'code_bg': '#EEEEEE', -# -# # Originally 940px -# 'page_width': '1000px', -# -# # Fonts -# 'font_family': "Palatino, 'goudy old style', 'minion pro', 'bell mt', Georgia, 'Hiragino Mincho Pro', serif", -# 'font_size': '18px', -# 'code_font_family': "'Menlo', 'DejaVu Sans Mono', 'Consolas', 'Bitstream Vera Sans Mono', monospace", -# 'code_font_size': '0.85em', -# } - # custom.css contains changes that aren't possible with the above because they # aren't specified in the Furo theme as CSS variables html_css_files = ['custom.css'] From c809da3ed755b8caa851c09e727bd594b7f8f7b2 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 28 Oct 2022 02:43:39 -0600 Subject: [PATCH 035/218] Use separate light and dark background logos --- docs/_static/ndindex_logo_dark_bg.svg | 91 ++++++++++++++++++++++++++ docs/conf.py | 3 +- docs/logo/ndindex_logo_dark_bg.svg | 92 +-------------------------- 3 files changed, 94 insertions(+), 92 deletions(-) create mode 100644 docs/_static/ndindex_logo_dark_bg.svg mode change 100644 => 120000 docs/logo/ndindex_logo_dark_bg.svg diff --git a/docs/_static/ndindex_logo_dark_bg.svg b/docs/_static/ndindex_logo_dark_bg.svg new file mode 100644 index 00000000..46dcd6ab --- /dev/null +++ b/docs/_static/ndindex_logo_dark_bg.svg @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/conf.py b/docs/conf.py index 59c86489..dedb0192 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -113,6 +113,8 @@ } html_theme_options = { + 'light_logo': 'ndindex_logo_white_bg.svg', + 'dark_logo': 'ndindex_logo_dark_bg.svg', "light_css_variables": { **theme_colors_common, "color-brand-primary": dark_blue, @@ -165,7 +167,6 @@ pygments_style = 'styles.SphinxHighContrastStyle' pygments_dark_style = 'styles.NativeHighContrastStyle' -html_logo = '_static/ndindex_logo_white_bg.svg' html_favicon = "logo/favicon.ico" myst_enable_extensions = ["dollarmath", "linkify"] diff --git a/docs/logo/ndindex_logo_dark_bg.svg b/docs/logo/ndindex_logo_dark_bg.svg deleted file mode 100644 index 46dcd6ab..00000000 --- a/docs/logo/ndindex_logo_dark_bg.svg +++ /dev/null @@ -1,91 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/logo/ndindex_logo_dark_bg.svg b/docs/logo/ndindex_logo_dark_bg.svg new file mode 120000 index 00000000..a6f33953 --- /dev/null +++ b/docs/logo/ndindex_logo_dark_bg.svg @@ -0,0 +1 @@ +../_static/ndindex_logo_dark_bg.svg \ No newline at end of file From 1606c6c137b25e0459ea51dad279641fb412f3ef Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 28 Oct 2022 02:56:56 -0600 Subject: [PATCH 036/218] Make the docs sidebar background gray instead of blue in light mode --- docs/_static/custom.css | 14 ++++++++------ docs/conf.py | 7 ++++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 29f33565..2ba5451c 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -12,23 +12,25 @@ --color-brand-green: #1EB881; --color-brand-medium-blue: #1041F3; --color-brand-dark-blue: #0D2B9C; - + --color-brand-dark-bg: #05002A; --color-brand-bg: white; --color-sidebar-current: white; - --color-sidebar-background-current: var(--color-brand-dark-blue); - + --color-sidebar-background-current: var(--color-brand-medium-blue); + --color-sidebar--hover: var(--color-brand-dark-blue); } @media (prefers-color-scheme: dark) { :root { --color-sidebar-background-current: var(--color-brand-dark-blue); - --color-brand-bg: #05002A; + --color-brand-bg: var(--color-brand-dark-bg); + --color-sidebar--hover: white; } } [data-theme='dark'] { --color-sidebar-background-current: var(--color-brand-dark-blue); - --color-brand-bg: #05002A; + --color-brand-bg: var(--color-brand-dark-bg); + --color-sidebar--hover: white; } /* Make top-level items in the sidebar bold */ @@ -43,7 +45,7 @@ color: var(--color-sidebar-current); } .sidebar-tree .reference:hover { - color: white; + color: var(--color-sidebar--hover); } /* The "hide search matches" text after doing a search. Defaults to the same diff --git a/docs/conf.py b/docs/conf.py index dedb0192..ed6f0782 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -89,6 +89,7 @@ dark_blue = "#0D2B9C" dark_bg = "#05002A" white = "white" +gray = "#EEEEEE" theme_colors_common = { "color-sidebar-background-border": "var(--color-background-primary)", @@ -120,9 +121,9 @@ "color-brand-primary": dark_blue, "color-brand-content": dark_blue, - "color-sidebar-background": light_blue, - "color-sidebar-item-background--hover": medium_blue, - "color-sidebar-item-expander-background--hover": medium_blue, + "color-sidebar-background": gray, + "color-sidebar-item-background--hover": light_blue, + "color-sidebar-item-expander-background--hover": light_blue, }, "dark_css_variables": { From 65220b39880390c64282efd102fb0de8b1fb3f99 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 28 Oct 2022 14:23:04 -0600 Subject: [PATCH 037/218] Fix the contrast ratio of the reds and blues in the slices document --- docs/slices.md | 546 ++++++++++++++++++++++++------------------------- 1 file changed, 273 insertions(+), 273 deletions(-) diff --git a/docs/slices.md b/docs/slices.md index 2aadd5c4..53f55114 100644 --- a/docs/slices.md +++ b/docs/slices.md @@ -135,14 +135,14 @@ $$ \begin{aligned} \begin{array}{r c c c c c c c} a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{red}{\text{index}} - & \color{red}{0\phantom{,}} - & \color{red}{1\phantom{,}} - & \color{red}{2\phantom{,}} - & \color{blue}3{\phantom{,}} - & \color{red}{4\phantom{,}} - & \color{red}{5\phantom{,}} - & \color{red}{6\phantom{,}}\\ +\color{#EE0000}{\text{index}} + & \color{#EE0000}{0\phantom{,}} + & \color{#EE0000}{1\phantom{,}} + & \color{#EE0000}{2\phantom{,}} + & \color{#5E5EFF}3{\phantom{,}} + & \color{#EE0000}{4\phantom{,}} + & \color{#EE0000}{5\phantom{,}} + & \color{#EE0000}{6\phantom{,}}\\ \end{array} \end{aligned} $$ @@ -170,14 +170,14 @@ $$ \begin{aligned} \begin{array}{r c c c c c c c} a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{red}{\text{index}} - & \color{red}{-7\phantom{,}} - & \color{red}{-6\phantom{,}} - & \color{red}{-5\phantom{,}} - & \color{red}{-4\phantom{,}} - & \color{blue}{-3\phantom{,}} - & \color{red}{-2\phantom{,}} - & \color{red}{-1\phantom{,}}\\ +\color{#EE0000}{\text{index}} + & \color{#EE0000}{-7\phantom{,}} + & \color{#EE0000}{-6\phantom{,}} + & \color{#EE0000}{-5\phantom{,}} + & \color{#EE0000}{-4\phantom{,}} + & \color{#5E5EFF}{-3\phantom{,}} + & \color{#EE0000}{-2\phantom{,}} + & \color{#EE0000}{-1\phantom{,}}\\ \end{array} \end{aligned} $$ @@ -304,14 +304,14 @@ $$ \begin{aligned} \begin{array}{r c c c c c c c} a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{red}{\text{index}} - & \color{red}{0\phantom{,}} - & \color{red}{1\phantom{,}} - & \color{red}{2\phantom{,}} - & \color{blue}{3\phantom{,}} - & \color{blue}{4\phantom{,}} - & \color{red}{5\phantom{,}} - & \color{red}{6\phantom{,}}\\ +\color{#EE0000}{\text{index}} + & \color{#EE0000}{0\phantom{,}} + & \color{#EE0000}{1\phantom{,}} + & \color{#EE0000}{2\phantom{,}} + & \color{#5E5EFF}{3\phantom{,}} + & \color{#5E5EFF}{4\phantom{,}} + & \color{#EE0000}{5\phantom{,}} + & \color{#EE0000}{6\phantom{,}}\\ \end{array} \end{aligned} $$ @@ -346,14 +346,14 @@ $$ \begin{aligned} \begin{array}{r c c c c c c c} a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{red}{\text{index}} - & \color{red}{0\phantom{,}} - & \color{red}{1\phantom{,}} - & \color{red}{2\phantom{,}} - & \color{blue}{\enclose{circle}{3}} - & \color{blue}{\enclose{circle}{4}} - & \color{red}{\enclose{circle}{5}} - & \color{red}{6\phantom{,}}\\ +\color{#EE0000}{\text{index}} + & \color{#EE0000}{0\phantom{,}} + & \color{#EE0000}{1\phantom{,}} + & \color{#EE0000}{2\phantom{,}} + & \color{#5E5EFF}{\enclose{circle}{3}} + & \color{#5E5EFF}{\enclose{circle}{4}} + & \color{#EE0000}{\enclose{circle}{5}} + & \color{#EE0000}{6\phantom{,}}\\ \end{array} \end{aligned} $$ @@ -450,20 +450,20 @@ $[3, 5)$ but in reverse order.
a[5:3:-1] "==" ['e', 'd'] -
(WRONG)
+
(WRONG)
$$ \begin{aligned} \begin{array}{r c c c c c c c} a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{red}{\text{index}} - & \color{red}{0\phantom{,}} - & \color{red}{1\phantom{,}} - & \color{red}{2\phantom{,}} - & \color{blue}{3\phantom{,}} - & \color{blue}{4\phantom{,}} - & \color{red}{5\phantom{,}} - & \color{red}{6\phantom{,}}\\ -\color{red}{\text{WRONG}}& +\color{#EE0000}{\text{index}} + & \color{#EE0000}{0\phantom{,}} + & \color{#EE0000}{1\phantom{,}} + & \color{#EE0000}{2\phantom{,}} + & \color{#5E5EFF}{3\phantom{,}} + & \color{#5E5EFF}{4\phantom{,}} + & \color{#EE0000}{5\phantom{,}} + & \color{#EE0000}{6\phantom{,}}\\ +\color{#EE0000}{\text{WRONG}}& & & & [\phantom{3,} @@ -500,14 +500,14 @@ $$ \begin{aligned} \begin{array}{r r r r r r r r} a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{red}{\text{index}} - & \color{red}{0\phantom{\textsf{'},}} - & \color{red}{1\phantom{\textsf{'},}} - & \color{red}{2\phantom{\textsf{'},}} - & \color{red}{\enclose{circle}{3}\phantom{,}} - & \leftarrow\color{blue}{\enclose{circle}{4}\phantom{,}} - & \leftarrow\color{blue}{\enclose{circle}{5}\phantom{,}} - & \color{red}{6\phantom{\textsf{'},}}\\ +\color{#EE0000}{\text{index}} + & \color{#EE0000}{0\phantom{\textsf{'},}} + & \color{#EE0000}{1\phantom{\textsf{'},}} + & \color{#EE0000}{2\phantom{\textsf{'},}} + & \color{#EE0000}{\enclose{circle}{3}\phantom{,}} + & \leftarrow\color{#5E5EFF}{\enclose{circle}{4}\phantom{,}} + & \leftarrow\color{#5E5EFF}{\enclose{circle}{5}\phantom{,}} + & \color{#EE0000}{6\phantom{\textsf{'},}}\\ \end{array} \end{aligned} $$ @@ -559,38 +559,38 @@ $$ \begin{array}{r r r r r r r r r r r r r r r r r r} a = & [&\phantom{|}&\mathtt{\textsf{'}a\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}b\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}c\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}d\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}e\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}f\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}g\textsf{'}}&\phantom{|}&]&\\ & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{blue}{|} + & \color{#5E5EFF}{|} & - & \color{blue}{|} + & \color{#5E5EFF}{|} & - & \color{blue}{|} + & \color{#5E5EFF}{|} & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{red}{|}\\ -\color{red}{\text{index}} + & \color{#EE0000}{|}\\ +\color{#EE0000}{\text{index}} & - & \color{red}{0} + & \color{#EE0000}{0} & - & \color{red}{1} + & \color{#EE0000}{1} & - & \color{red}{2} + & \color{#EE0000}{2} & - & \color{blue}{3} + & \color{#5E5EFF}{3} & - & \color{blue}{4} + & \color{#5E5EFF}{4} & - & \color{blue}{5} + & \color{#5E5EFF}{5} & - & \color{red}{6} + & \color{#EE0000}{6} & - & \color{red}{7}\\ + & \color{#EE0000}{7}\\ \end{array}\\ \end{aligned} $$ @@ -615,7 +615,7 @@ reasons why this way of thinking creates more confusion than it removes.
a[5:3:-1] "==" ['e', 'd'] -
(WRONG)
+
(WRONG)
$$ \require{enclose} \begin{aligned} @@ -623,40 +623,40 @@ reasons why this way of thinking creates more confusion than it removes. \begin{array}{r r r r r r r r r r r r r r r r r r} a = & [&\phantom{|}&\mathtt{\textsf{'}a\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}b\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}c\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}d\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}e\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}f\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}g\textsf{'}}&\phantom{|}&]&\\ & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{blue}{|} + & \color{#5E5EFF}{|} & - & \color{blue}{|} + & \color{#5E5EFF}{|} & - & \color{blue}{|} + & \color{#5E5EFF}{|} & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{red}{|}\\ - \color{red}{\text{index}} + & \color{#EE0000}{|}\\ + \color{#EE0000}{\text{index}} & - & \color{red}{0} + & \color{#EE0000}{0} & - & \color{red}{1} + & \color{#EE0000}{1} & - & \color{red}{2} + & \color{#EE0000}{2} & - & \color{blue}{3} + & \color{#5E5EFF}{3} & - & \color{blue}{4} + & \color{#5E5EFF}{4} & - & \color{blue}{5} + & \color{#5E5EFF}{5} & - & \color{red}{6} + & \color{#EE0000}{6} & - & \color{red}{7}\\ + & \color{#EE0000}{7}\\ \end{array}\\ - \small{\color{red}{\textbf{THIS IS WRONG!}}} + \small{\color{#EE0000}{\textbf{THIS IS WRONG!}}} \end{array} \end{aligned} $$ @@ -690,31 +690,31 @@ reasons why this way of thinking creates more confusion than it removes. \begin{aligned} \begin{array}{r r c c c c c c c c c c c c l} a = & [\mathtt{\textsf{'}a\textsf{'}}, && \mathtt{\textsf{'}b\textsf{'}}, && \mathtt{\textsf{'}c\textsf{'}}, && \mathtt{\textsf{'}d\textsf{'}}, && \mathtt{\textsf{'}e\textsf{'}}, && \mathtt{\textsf{'}f\textsf{'}}, && \mathtt{\textsf{'}g\textsf{'}}]\\ - \color{red}{\text{index}} - & \color{red}{0\phantom{,}} + \color{#EE0000}{\text{index}} + & \color{#EE0000}{0\phantom{,}} & - & \color{red}{1\phantom{,}} + & \color{#EE0000}{1\phantom{,}} & - & \color{red}{2\phantom{,}} + & \color{#EE0000}{2\phantom{,}} & - & \color{red}{\enclose{circle}{3}} + & \color{#EE0000}{\enclose{circle}{3}} & - & \color{blue}{\enclose{circle}{4}} + & \color{#5E5EFF}{\enclose{circle}{4}} & - & \color{blue}{\enclose{circle}{5}} + & \color{#5E5EFF}{\enclose{circle}{5}} & - & \color{red}{6\phantom{,}}\\ + & \color{#EE0000}{6\phantom{,}}\\ & & \phantom{\leftarrow} & & \phantom{\leftarrow} & & \phantom{\leftarrow} - & \color{red}{-1} + & \color{#EE0000}{-1} & \leftarrow - & \color{blue}{-1} + & \color{#5E5EFF}{-1} & \leftarrow - & \color{blue}{\text{start}} + & \color{#5E5EFF}{\text{start}} \end{array} \end{aligned} $$ @@ -733,38 +733,38 @@ reasons why this way of thinking creates more confusion than it removes. \begin{array}{r r r r r r r r r r r r r r r r r r} a = & [&\phantom{|}&\mathtt{\textsf{'}a\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}b\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}c\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}d\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}e\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}f\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}g\textsf{'}}&\phantom{|}&]&\\ & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{blue}{|} + & \color{#5E5EFF}{|} & - & \color{blue}{|} + & \color{#5E5EFF}{|} & - & \color{blue}{|} + & \color{#5E5EFF}{|} & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{red}{|}\\ - \color{red}{\text{index}} + & \color{#EE0000}{|}\\ + \color{#EE0000}{\text{index}} & - & \color{red}{-7} + & \color{#EE0000}{-7} & - & \color{red}{-6} + & \color{#EE0000}{-6} & - & \color{red}{-5} + & \color{#EE0000}{-5} & - & \color{blue}{-4} + & \color{#5E5EFF}{-4} & - & \color{blue}{-3} + & \color{#5E5EFF}{-3} & - & \color{blue}{-2} + & \color{#5E5EFF}{-2} & - & \color{red}{-1} + & \color{#EE0000}{-1} & - & \color{red}{0}\\ + & \color{#EE0000}{0}\\ \end{array}\\ \small{\text{(not a great way of thinking about negative indices)}} \end{array} @@ -785,7 +785,7 @@ reasons why this way of thinking creates more confusion than it removes.
a[-4:-2] "==" ['e', 'f'] -
(WRONG)
+
(WRONG)
$$ \require{enclose} \begin{aligned} @@ -793,40 +793,40 @@ reasons why this way of thinking creates more confusion than it removes. \begin{array}{r r r r r r r r r r r r r r r r r r} a = & [&\phantom{|}&\mathtt{\textsf{'}a\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}b\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}c\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}d\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}e\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}f\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}g\textsf{'}}&\phantom{|}&]&\\ & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{blue}{|} + & \color{#5E5EFF}{|} & - & \color{blue}{|} + & \color{#5E5EFF}{|} & - & \color{blue}{|} + & \color{#5E5EFF}{|} & - & \color{red}{|}\\ - \color{red}{\text{index}} + & \color{#EE0000}{|}\\ + \color{#EE0000}{\text{index}} & - & \color{red}{-8} + & \color{#EE0000}{-8} & - & \color{red}{-7} + & \color{#EE0000}{-7} & - & \color{red}{-6} + & \color{#EE0000}{-6} & - & \color{red}{-5} + & \color{#EE0000}{-5} & - & \color{blue}{-4} + & \color{#5E5EFF}{-4} & - & \color{blue}{-3} + & \color{#5E5EFF}{-3} & - & \color{blue}{-2} + & \color{#5E5EFF}{-2} & - & \color{red}{-1}\\ + & \color{#EE0000}{-1}\\ \end{array}\\ - \small{\color{red}{\textbf{THIS IS WRONG!}}} + \small{\color{#EE0000}{\textbf{THIS IS WRONG!}}} \end{array} \end{aligned} $$ @@ -847,7 +847,7 @@ reasons why this way of thinking creates more confusion than it removes.
a[-2:-4:-1] == ['f', 'e'] -
NOW RIGHT!
+
NOW RIGHT!
$$ \require{enclose} \begin{aligned} @@ -855,38 +855,38 @@ reasons why this way of thinking creates more confusion than it removes. \begin{array}{r r r r r r r r r r r r r r r r r r} a = & [&\phantom{|}&\mathtt{\textsf{'}a\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}b\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}c\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}d\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}e\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}f\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}g\textsf{'}}&\phantom{|}&]&\\ & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{blue}{|} + & \color{#5E5EFF}{|} & - & \color{blue}{|} + & \color{#5E5EFF}{|} & - & \color{blue}{|} + & \color{#5E5EFF}{|} & - & \color{red}{|}\\ - \color{red}{\text{index}} + & \color{#EE0000}{|}\\ + \color{#EE0000}{\text{index}} & - & \color{red}{-8} + & \color{#EE0000}{-8} & - & \color{red}{-7} + & \color{#EE0000}{-7} & - & \color{red}{-6} + & \color{#EE0000}{-6} & - & \color{red}{-5} + & \color{#EE0000}{-5} & - & \color{blue}{-4} + & \color{#5E5EFF}{-4} & - & \color{blue}{-3} + & \color{#5E5EFF}{-3} & - & \color{blue}{-2} + & \color{#5E5EFF}{-2} & - & \color{red}{-1}\\ + & \color{#EE0000}{-1}\\ \end{array}\\ \small{\text{(not a great way of thinking about negative indices)}} \end{array} @@ -896,7 +896,7 @@ reasons why this way of thinking creates more confusion than it removes.
a[-2:-4:-1] "==" ['e', 'd'] -
(WRONG)
+
(WRONG)
$$ \require{enclose} \begin{aligned} @@ -904,40 +904,40 @@ reasons why this way of thinking creates more confusion than it removes. \begin{array}{r r r r r r r r r r r r r r r r r r} a = & [&\phantom{|}&\mathtt{\textsf{'}a\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}b\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}c\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}d\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}e\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}f\textsf{'}}, &\phantom{|}& \mathtt{\textsf{'}g\textsf{'}}&\phantom{|}&]&\\ & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{blue}{|} + & \color{#5E5EFF}{|} & - & \color{blue}{|} + & \color{#5E5EFF}{|} & - & \color{blue}{|} + & \color{#5E5EFF}{|} & - & \color{red}{|} + & \color{#EE0000}{|} & - & \color{red}{|}\\ - \color{red}{\text{index}} + & \color{#EE0000}{|}\\ + \color{#EE0000}{\text{index}} & - & \color{red}{-7} + & \color{#EE0000}{-7} & - & \color{red}{-6} + & \color{#EE0000}{-6} & - & \color{red}{-5} + & \color{#EE0000}{-5} & - & \color{blue}{-4} + & \color{#5E5EFF}{-4} & - & \color{blue}{-3} + & \color{#5E5EFF}{-3} & - & \color{blue}{-2} + & \color{#5E5EFF}{-2} & - & \color{red}{-1} + & \color{#EE0000}{-1} & - & \color{red}{0}\\ + & \color{#EE0000}{0}\\ \end{array}\\ - \small{\color{red}{\textbf{THIS IS WRONG!}}} + \small{\color{#EE0000}{\textbf{THIS IS WRONG!}}} \end{array} \end{aligned} $$ @@ -1023,22 +1023,22 @@ $$ \begin{aligned} \begin{array}{r c c c c c c c} a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{red}{\text{nonnegative index}} - & \color{red}{0\phantom{,}} - & \color{red}{1\phantom{,}} - & \color{red}{\enclose{circle}{2}\phantom{,}} - & \color{blue}{3\phantom{,}} - & \color{blue}{4\phantom{,}} - & \color{red}{\enclose{circle}{5}\phantom{,}} - & \color{red}{6\phantom{,}}\\ -\color{red}{\text{negative index}} - & \color{red}{-7\phantom{,}} - & \color{red}{-6\phantom{,}} - & \color{red}{\enclose{circle}{-5}\phantom{,}} - & \color{blue}{-4\phantom{,}} - & \color{blue}{-3\phantom{,}} - & \color{red}{\enclose{circle}{-2}\phantom{,}} - & \color{red}{-1\phantom{,}}\\ +\color{#EE0000}{\text{nonnegative index}} + & \color{#EE0000}{0\phantom{,}} + & \color{#EE0000}{1\phantom{,}} + & \color{#EE0000}{\enclose{circle}{2}\phantom{,}} + & \color{#5E5EFF}{3\phantom{,}} + & \color{#5E5EFF}{4\phantom{,}} + & \color{#EE0000}{\enclose{circle}{5}\phantom{,}} + & \color{#EE0000}{6\phantom{,}}\\ +\color{#EE0000}{\text{negative index}} + & \color{#EE0000}{-7\phantom{,}} + & \color{#EE0000}{-6\phantom{,}} + & \color{#EE0000}{\enclose{circle}{-5}\phantom{,}} + & \color{#5E5EFF}{-4\phantom{,}} + & \color{#5E5EFF}{-3\phantom{,}} + & \color{#EE0000}{\enclose{circle}{-2}\phantom{,}} + & \color{#EE0000}{-1\phantom{,}}\\ \end{array} \end{aligned} $$ @@ -1218,21 +1218,21 @@ $$ \begin{aligned} \begin{array}{r c c c c c c l} a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{red}{\text{index}} - & \color{blue}{\enclose{circle}{0}} - & \color{red}{1\phantom{,}} - & \color{red}{2\phantom{,}} - & \color{blue}{\enclose{circle}{3}} - & \color{red}{4\phantom{,}} - & \color{red}{5\phantom{,}} - & \color{red}{\enclose{circle}{6}}\\ - & \color{blue}{\text{start}} +\color{#EE0000}{\text{index}} + & \color{#5E5EFF}{\enclose{circle}{0}} + & \color{#EE0000}{1\phantom{,}} + & \color{#EE0000}{2\phantom{,}} + & \color{#5E5EFF}{\enclose{circle}{3}} + & \color{#EE0000}{4\phantom{,}} + & \color{#EE0000}{5\phantom{,}} + & \color{#EE0000}{\enclose{circle}{6}}\\ + & \color{#5E5EFF}{\text{start}} & & \rightarrow - & \color{blue}{+3} + & \color{#5E5EFF}{+3} & & \rightarrow - & \color{red}{+3\ (\geq \text{stop})} + & \color{#EE0000}{+3\ (\geq \text{stop})} \end{array} \end{aligned} $$ @@ -1265,23 +1265,23 @@ $$ \begin{aligned} \begin{array}{r c c c c c c c l} a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{red}{\text{index}} - & \color{red}{0\phantom{,}} - & \color{blue}{\enclose{circle}{1}} - & \color{red}{2\phantom{,}} - & \color{red}{3\phantom{,}} - & \color{blue}{\enclose{circle}{4}} - & \color{red}{5\phantom{,}} - & \color{red}{\underline{6}\phantom{,}} - & \color{red}{\enclose{circle}{\phantom{7}}}\\ +\color{#EE0000}{\text{index}} + & \color{#EE0000}{0\phantom{,}} + & \color{#5E5EFF}{\enclose{circle}{1}} + & \color{#EE0000}{2\phantom{,}} + & \color{#EE0000}{3\phantom{,}} + & \color{#5E5EFF}{\enclose{circle}{4}} + & \color{#EE0000}{5\phantom{,}} + & \color{#EE0000}{\underline{6}\phantom{,}} + & \color{#EE0000}{\enclose{circle}{\phantom{7}}}\\ & - & \color{blue}{\text{start}} + & \color{#5E5EFF}{\text{start}} & & \rightarrow - & \color{blue}{+3} + & \color{#5E5EFF}{+3} & & \rightarrow - & \color{red}{+3\ (\geq \text{stop})} + & \color{#EE0000}{+3\ (\geq \text{stop})} \end{array} \end{aligned} $$ @@ -1415,22 +1415,22 @@ $$ \begin{aligned} \begin{array}{r r c c c c c c l} a = & [\mathtt{\textsf{'}a\textsf{'}}, & \mathtt{\textsf{'}b\textsf{'}}, & \mathtt{\textsf{'}c\textsf{'}}, & \mathtt{\textsf{'}d\textsf{'}}, & \mathtt{\textsf{'}e\textsf{'}}, & \mathtt{\textsf{'}f\textsf{'}}, & \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{red}{\text{index}} - & \color{red}{\enclose{circle}{0}\phantom{,}} - & \color{red}{1\phantom{,}} - & \color{red}{2\phantom{,}} - & \color{blue}{\enclose{circle}{3}} - & \color{red}{4\phantom{,}} - & \color{red}{5\phantom{,}} - & \color{blue}{\enclose{circle}{6}}\\ - & \color{red}{-3}\phantom{\mathtt{\textsf{'},}} +\color{#EE0000}{\text{index}} + & \color{#EE0000}{\enclose{circle}{0}\phantom{,}} + & \color{#EE0000}{1\phantom{,}} + & \color{#EE0000}{2\phantom{,}} + & \color{#5E5EFF}{\enclose{circle}{3}} + & \color{#EE0000}{4\phantom{,}} + & \color{#EE0000}{5\phantom{,}} + & \color{#5E5EFF}{\enclose{circle}{6}}\\ + & \color{#EE0000}{-3}\phantom{\mathtt{\textsf{'},}} & \leftarrow & - & \color{blue}{-3}\phantom{\mathtt{\textsf{'},}} + & \color{#5E5EFF}{-3}\phantom{\mathtt{\textsf{'},}} & \leftarrow & - & \color{blue}{\text{start}}\\ - & \color{red}{(\leq \text{stop})} + & \color{#5E5EFF}{\text{start}}\\ + & \color{#EE0000}{(\leq \text{stop})} \end{array} \end{aligned} $$ @@ -1493,28 +1493,28 @@ $$ \begin{aligned} \begin{array}{r r c c c c c c c c c c c c l} a = & [\mathtt{\textsf{'}a\textsf{'}}, && \mathtt{\textsf{'}b\textsf{'}}, && \mathtt{\textsf{'}c\textsf{'}}, && \mathtt{\textsf{'}d\textsf{'}}, && \mathtt{\textsf{'}e\textsf{'}}, && \mathtt{\textsf{'}f\textsf{'}}, && \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{red}{\text{index}} - & \color{blue}{\enclose{circle}{0}} +\color{#EE0000}{\text{index}} + & \color{#5E5EFF}{\enclose{circle}{0}} & - & \color{blue}{\enclose{circle}{1}} + & \color{#5E5EFF}{\enclose{circle}{1}} & - & \color{blue}{\enclose{circle}{2}} + & \color{#5E5EFF}{\enclose{circle}{2}} & - & \color{red}{\enclose{circle}{3}} + & \color{#EE0000}{\enclose{circle}{3}} & - & \color{red}{4\phantom{,}} + & \color{#EE0000}{4\phantom{,}} & - & \color{red}{5\phantom{,}} + & \color{#EE0000}{5\phantom{,}} & - & \color{red}{6\phantom{,}}\\ - \color{blue}{\text{start}} - & \color{blue}{\text{(beginning)}} + & \color{#EE0000}{6\phantom{,}}\\ + \color{#5E5EFF}{\text{start}} + & \color{#5E5EFF}{\text{(beginning)}} & \rightarrow - & \color{blue}{+1} + & \color{#5E5EFF}{+1} & \rightarrow - & \color{blue}{+1} + & \color{#5E5EFF}{+1} & \rightarrow - & \color{red}{\text{stop}} + & \color{#EE0000}{\text{stop}} & & \phantom{\rightarrow} & @@ -1533,34 +1533,34 @@ $$ \begin{aligned} \begin{array}{r r c c c c c c c c c c c c l} a = & [\mathtt{\textsf{'}a\textsf{'}}, && \mathtt{\textsf{'}b\textsf{'}}, && \mathtt{\textsf{'}c\textsf{'}}, && \mathtt{\textsf{'}d\textsf{'}}, && \mathtt{\textsf{'}e\textsf{'}}, && \mathtt{\textsf{'}f\textsf{'}}, && \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{red}{\text{index}} - & \color{red}{0\phantom{,}} +\color{#EE0000}{\text{index}} + & \color{#EE0000}{0\phantom{,}} & - & \color{red}{1\phantom{,}} + & \color{#EE0000}{1\phantom{,}} & - & \color{red}{2\phantom{,}} + & \color{#EE0000}{2\phantom{,}} & - & \color{blue}{\enclose{circle}{3}} + & \color{#5E5EFF}{\enclose{circle}{3}} & - & \color{blue}{\enclose{circle}{4}} + & \color{#5E5EFF}{\enclose{circle}{4}} & - & \color{blue}{\enclose{circle}{5}} + & \color{#5E5EFF}{\enclose{circle}{5}} & - & \color{blue}{\enclose{circle}{6}\phantom{,}}\\ + & \color{#5E5EFF}{\enclose{circle}{6}\phantom{,}}\\ & & \phantom{\rightarrow} & & \phantom{\rightarrow} & & \phantom{\rightarrow} - & \color{blue}{\text{start}} + & \color{#5E5EFF}{\text{start}} & \rightarrow - & \color{blue}{+1} + & \color{#5E5EFF}{+1} & \rightarrow - & \color{blue}{+1} + & \color{#5E5EFF}{+1} & \rightarrow - & \color{blue}{\text{stop}} - & \color{blue}{\text{(end)}} + & \color{#5E5EFF}{\text{stop}} + & \color{#5E5EFF}{\text{(end)}} \end{array} \end{aligned} $$ @@ -1573,34 +1573,34 @@ $$ \begin{aligned} \begin{array}{r r c c c c c c c c c c c c l} a = & [\mathtt{\textsf{'}a\textsf{'}}, && \mathtt{\textsf{'}b\textsf{'}}, && \mathtt{\textsf{'}c\textsf{'}}, && \mathtt{\textsf{'}d\textsf{'}}, && \mathtt{\textsf{'}e\textsf{'}}, && \mathtt{\textsf{'}f\textsf{'}}, && \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{red}{\text{index}} - & \color{red}{0\phantom{,}} +\color{#EE0000}{\text{index}} + & \color{#EE0000}{0\phantom{,}} & - & \color{red}{1\phantom{,}} + & \color{#EE0000}{1\phantom{,}} & - & \color{red}{2\phantom{,}} + & \color{#EE0000}{2\phantom{,}} & - & \color{red}{\enclose{circle}{3}} + & \color{#EE0000}{\enclose{circle}{3}} & - & \color{blue}{\enclose{circle}{4}} + & \color{#5E5EFF}{\enclose{circle}{4}} & - & \color{blue}{\enclose{circle}{5}} + & \color{#5E5EFF}{\enclose{circle}{5}} & - & \color{blue}{\enclose{circle}{6}\phantom{,}}\\ + & \color{#5E5EFF}{\enclose{circle}{6}\phantom{,}}\\ & & \phantom{\leftarrow} & & \phantom{\leftarrow} & & \phantom{\leftarrow} - & \color{red}{\text{stop}} + & \color{#EE0000}{\text{stop}} & \leftarrow - & \color{blue}{-1} + & \color{#5E5EFF}{-1} & \leftarrow - & \color{blue}{-1} + & \color{#5E5EFF}{-1} & \leftarrow - & \color{blue}{\text{start}} - & \color{blue}{\text{(end)}} + & \color{#5E5EFF}{\text{start}} + & \color{#5E5EFF}{\text{(end)}} \end{array} \end{aligned} $$ @@ -1613,28 +1613,28 @@ $$ \begin{aligned} \begin{array}{r r c c c c c c c c c c c c l} a = & [\mathtt{\textsf{'}a\textsf{'}}, && \mathtt{\textsf{'}b\textsf{'}}, && \mathtt{\textsf{'}c\textsf{'}}, && \mathtt{\textsf{'}d\textsf{'}}, && \mathtt{\textsf{'}e\textsf{'}}, && \mathtt{\textsf{'}f\textsf{'}}, && \mathtt{\textsf{'}g\textsf{'}}]\\ -\color{red}{\text{index}} - & \color{blue}{\enclose{circle}{0}} +\color{#EE0000}{\text{index}} + & \color{#5E5EFF}{\enclose{circle}{0}} & - & \color{blue}{\enclose{circle}{1}} + & \color{#5E5EFF}{\enclose{circle}{1}} & - & \color{blue}{\enclose{circle}{2}} + & \color{#5E5EFF}{\enclose{circle}{2}} & - & \color{blue}{\enclose{circle}{3}} + & \color{#5E5EFF}{\enclose{circle}{3}} & - & \color{red}{4\phantom{,}} + & \color{#EE0000}{4\phantom{,}} & - & \color{red}{5\phantom{,}} + & \color{#EE0000}{5\phantom{,}} & - & \color{red}{6\phantom{,}}\\ - \color{blue}{\text{stop}} - & \color{blue}{\text{(beginning)}} + & \color{#EE0000}{6\phantom{,}}\\ + \color{#5E5EFF}{\text{stop}} + & \color{#5E5EFF}{\text{(beginning)}} & \leftarrow - & \color{blue}{-1} + & \color{#5E5EFF}{-1} & \leftarrow - & \color{blue}{-1} + & \color{#5E5EFF}{-1} & \leftarrow - & \color{blue}{\text{start}} + & \color{#5E5EFF}{\text{start}} & & \phantom{\leftarrow} & From a2f53ec33efd791fe68503dc0cdd27677427da65 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 28 Oct 2022 14:34:12 -0600 Subject: [PATCH 038/218] Set the dark mode background to pure black Otherwise it's hard to find a red text color that works in both schemes for the slices document (and I don't know how to use different colors for each in MathJax). --- docs/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index ed6f0782..e341905f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -135,6 +135,8 @@ "color-api-overall": "#FFFFFF90", "color-api-paren": "#FFFFFF90", + "color-background-primary": "black", + "color-sidebar-background": dark_bg, "color-sidebar-item-background--hover": medium_blue, "color-sidebar-item-expander-background--hover": medium_blue, From f8b5e7b10830e42c9195d7c4cdc4783e00de831c Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 28 Oct 2022 14:59:10 -0600 Subject: [PATCH 039/218] Only define the brand colors once, in custom.css --- docs/conf.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index e341905f..6a385dfc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -81,14 +81,14 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] -# ndindex brand colors, from logo/ndindex_final_2.pdf. - -light_blue = "#8DBEFE" -green = "#1EB881" -medium_blue = "#1041F3" -dark_blue = "#0D2B9C" -dark_bg = "#05002A" +# These are defined in _static/custom.css +light_blue = "var(--color-brand-light-blue)" +green = "var(--color-brand-green)" +medium_blue = "var(--color-brand-medium-blue)" +dark_blue = "var(--color-brand-dark-blue)" +dark_bg = "var(--color-brand-dark-bg)" white = "white" +black = "black" gray = "#EEEEEE" theme_colors_common = { @@ -96,11 +96,11 @@ "color-sidebar-brand-text": "var(--color-sidebar-link-text--top-level)", "color-admonition-title-background--seealso": "#CCCCCC", - "color-admonition-title--seealso": "black", + "color-admonition-title--seealso": black, "color-admonition-title-background--note": "#CCCCCC", - "color-admonition-title--note": "black", + "color-admonition-title--note": black, "color-admonition-title-background--warning": "var(--color-problematic)", - "color-admonition-title--warning": "white", + "color-admonition-title--warning": white, "admonition-font-size": "var(--font-size--normal)", "admonition-title-font-size": "var(--font-size--normal)", @@ -135,7 +135,7 @@ "color-api-overall": "#FFFFFF90", "color-api-paren": "#FFFFFF90", - "color-background-primary": "black", + "color-background-primary": black, "color-sidebar-background": dark_bg, "color-sidebar-item-background--hover": medium_blue, From eff269f2777a6b9886003c9611aabf13ed0e4679 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 28 Oct 2022 15:10:10 -0600 Subject: [PATCH 040/218] Fix the ndindex brand color definitions I was using "Generic RBG" in Digital Color Meter to extract the colors from the logo PDF, but the colors it selects don't seem to match when used in the CSS. The tool https://pickcoloronline.com/ seems to get the correct colors. --- docs/_static/custom.css | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 2ba5451c..2c6cbb8c 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -8,11 +8,12 @@ } :root { - --color-brand-light-blue: #8DBEFE; - --color-brand-green: #1EB881; - --color-brand-medium-blue: #1041F3; - --color-brand-dark-blue: #0D2B9C; - --color-brand-dark-bg: #05002A; + /* ndindex brand colors, from logo/ndindex_final_2.pdf. */ + --color-brand-light-blue: #9ECBFF; + --color-brand-green: #15C293; + --color-brand-medium-blue: #115DF6; + --color-brand-dark-blue: #0D41AC; + --color-brand-dark-bg: #050038; --color-brand-bg: white; --color-sidebar-current: white; From f1999e2bdd08e5123ca5199fe658baf979f56f16 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 28 Oct 2022 16:56:14 -0600 Subject: [PATCH 041/218] Fix some markup formatting --- ndindex/slice.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ndindex/slice.py b/ndindex/slice.py index 00497d89..f8cb40f7 100644 --- a/ndindex/slice.py +++ b/ndindex/slice.py @@ -28,8 +28,8 @@ class Slice(NDIndex): `Slice.args` always has three arguments, and does not make any distinction between, for instance, `Slice(x, y)` and `Slice(x, y, None)`. This is - because Python itself does not make the distinction between x:y and x:y: - syntactically. + because Python itself does not make the distinction between `x:y` and + `x:y:` syntactically. See :ref:`slices-docs` for a description of the semantic meaning of slices on arrays. From 7b067223b4a30d4fb3501303e00b93146eadae83 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 28 Oct 2022 18:36:40 -0600 Subject: [PATCH 042/218] Remove custom section highlight color It makes the API names unreadable. --- docs/conf.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 6a385dfc..899c00c1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -141,8 +141,6 @@ "color-sidebar-item-background--hover": medium_blue, "color-sidebar-item-expander-background--hover": medium_blue, - "color-highlight-on-target": dark_blue, - "color-admonition-title-background--seealso": "#555555", "color-admonition-title-background--note": "#555555", "color-problematic": "#B30000", From ebe9501fba8d5101832efa41177b856cd329c32d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 28 Oct 2022 18:47:38 -0600 Subject: [PATCH 043/218] Use a bigger font size for the code examples --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 899c00c1..bf36f79f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -111,6 +111,7 @@ "color-api-pre-name": "var(--color-brand-content)", "api-font-size": "var(--font-size--normal)", + "code-font-size": "var(--font-size--small)", } html_theme_options = { From f0a18598d826863d003cf3a65cd3a49602218dbe Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 28 Oct 2022 19:08:24 -0600 Subject: [PATCH 044/218] Put the main slices rules in block quotes --- docs/slices.md | 67 +++++++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/docs/slices.md b/docs/slices.md index 53f55114..6cf6860a 100644 --- a/docs/slices.md +++ b/docs/slices.md @@ -253,14 +253,15 @@ to demystify them through simple [rules](rules). (subarray)= ### Subarray -**A slice always produces a subarray (or sub-list, sub-tuple, sub-string, +> **A slice always produces a subarray (or sub-list, sub-tuple, sub-string, etc.). For NumPy arrays, this means that a slice will always *preserve* the -dimension that is sliced.** This is true even if the slice chooses only a -single element, or even if it chooses no elements. This is also true for -lists, tuples, and strings, in the sense that a slice on a list, tuple, or -string will always produce a list, tuple, or string. This behavior is -different from integer indices, which always remove the dimension that they -index. +dimension that is sliced.** + +This is true even if the slice chooses only a single element, or even if it +chooses no elements. This is also true for lists, tuples, and strings, in the +sense that a slice on a list, tuple, or string will always produce a list, +tuple, or string. This behavior is different from integer indices, which +always remove the dimension that they index. For example @@ -409,8 +410,8 @@ advantages: #### Wrong Ways of Thinking about Half-open Semantics -**The proper rule to remember for half-open semantics is "the `stop` is not -included".** +> **The proper rule to remember for half-open semantics is "the `stop` is not + included".** There are several alternative ways that one might think of slice semantics, but they are all wrong in subtle ways. To be sure, for each of these, one @@ -491,7 +492,7 @@ Actually, what we really get is ``` This is because the slice `5:3:-1` starts at index `5` and steps backwards to -index `3`, but not including `3`. +index `3`, but not including `3` (see [](negative-steps) below).
a[5:3:-1] == ['f', 'e'] @@ -1006,12 +1007,16 @@ here. It isn't worth it. ### Negative Indices Negative indices in slices work the same way they do with [integer -indices](integer-indices). **For `a[start:stop:step]`, negative `start` or -`stop` use −1-based indexing from the end of `a`.** However, negative `start` -or `stop` does *not* change the order of the slicing---only the `step` does -that. The other [rules](rules) of slicing do not change when the `start` or -`stop` is negative. [The `stop` is still not included](half-open), values less -than `-len(a)` still [clip](clipping), and so on. +indices](integer-indices). + +> **For `a[start:stop:step]`, negative `start` or `stop` use −1-based indexing + from the end of `a`.** + +However, negative `start` or `stop` does *not* change the order of the +slicing---only the [`step`](steps) does that. The other [rules](rules) of +slicing do not change when the `start` or `stop` is negative. [The `stop` is +still not included](half-open), values less than `-len(a)` still +[clip](clipping), and so on. Note that positive and negative indices can be mixed. The following slices of `a` all produce `['d', 'e']`: @@ -1141,7 +1146,8 @@ example, instead of using `mid - n//2`, we could use `max(mid - n//2, 0)`. Slices can never give an out-of-bounds `IndexError`. This is different from [integer indices](integer-indices) which require the index to be in bounds. -**If `start` or `stop` index before the beginning or after the end of the + +> **If `start` or `stop` index before the beginning or after the end of the `a`, they will clip to the bounds of `a`**: ```py @@ -1198,9 +1204,9 @@ Thus far, we have only considered slices with the default step size of 1. When the step is greater than 1, the slice picks every `step` element contained in the bounds of `start` and `stop`. -**The proper way to think about `step` is that the slice starts at `start` and -successively adds `step` until it reaches an index that is at or past the -`stop`, and then stops without including that index.** +> **The proper way to think about `step` is that the slice starts at `start` + and successively adds `step` until it reaches an index that is at or past + the `stop`, and then stops without including that index.** The important thing to remember about the `step` is that it being non-1 does not change the fundamental [rules](rules) of slices that we have learned so @@ -1334,9 +1340,9 @@ slices will necessarily have many piecewise conditions. Recall what we said [above](steps): -**The proper way to think about `step` is that the slice starts at `start` and -successively adds `step` until it reaches an index that is at or past the -`stop`, and then stops without including that index.** +> **The proper way to think about `step` is that the slice starts at `start` + and successively adds `step` until it reaches an index that is at or past + the `stop`, and then stops without including that index.** The key thing to remember with negative `step` is that this rule still applies. That is, the index starts at `start` then adds the `step` (which @@ -1468,9 +1474,11 @@ trying to remember some rule based on where a colon is. But the colons in a slice are not indicators, they are separators. As to the semantic meaning of omitted entries, the easiest one is the `step`. -**If the `step` is omitted, it always defaults to `1`.** If the `step` is -omitted the second colon before the `step` can also be omitted. That is to -say, the following are completely equivalent: + +> **If the `step` is omitted, it always defaults to `1`.** + +If the `step` is omitted the second colon before the `step` can also be +omitted. That is to say, the following are completely equivalent: ```py a[i:j:1] @@ -1480,8 +1488,11 @@ a[i:j] -**For the `start` and `stop`, the rule is that being omitted extends the slice -all the way to the beginning or end of `a` in the direction being sliced.** If +> **For the `start` and `stop`, the rule is that being omitted extends the + slice all the way to the beginning or end of `a` in the direction being + sliced.** + +If the `step` is positive, this means `start` extends to the beginning of `a` and `stop` extends to the end. If `step` is negative, it is reversed: `start` extends to the end of `a` and `stop` extends to the beginning. From 1186a3892e75e1da63c22dae415842ca09b25187 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 28 Oct 2022 19:11:08 -0600 Subject: [PATCH 045/218] Fix a typo in a formula --- docs/slices.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/slices.md b/docs/slices.md index 6cf6860a..2e0b0bab 100644 --- a/docs/slices.md +++ b/docs/slices.md @@ -206,7 +206,7 @@ allows one to specify parts of a list that would otherwise need to be specified in terms of the size of the list. If an integer index is greater than or equal to the size of the list, or less -than negative the size of the list (`i >= len(a)` or `i < len(a)`), then it +than negative the size of the list (`i >= len(a)` or `i < -len(a)`), then it is out of bounds and will raise an `IndexError`. ```py From e9f7bd1172a38f3b7efe140f2435bb0d06949f86 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 28 Oct 2022 19:14:18 -0600 Subject: [PATCH 046/218] Small sentence fix --- docs/slices.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/slices.md b/docs/slices.md index 2e0b0bab..3cb8832d 100644 --- a/docs/slices.md +++ b/docs/slices.md @@ -62,9 +62,9 @@ We will use these names throughout this guide. It is worth noting that the `x:y:z` syntax is not valid outside of square brackets. However, slice objects can be created manually using the `slice()` -builtin (`a[x:y:z]` is the same as `a[slice(x, y, z)]`). You can also use the -[`ndindex.Slice()`](slice-api) object if you want to perform more advanced -operations. +builtin (`a[x:y:z]` is the same as `a[slice(x, y, z)]`). If you want to +perform more advanced operations like arithmetic on slices, consider using +the [`ndindex.Slice()`](slice-api) object. (rules)= ## Rules From 93684a8f350f61e2cbeebf94a4a197d76281d877 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 1 Nov 2022 15:43:26 -0600 Subject: [PATCH 047/218] Add dark mode versions of the acknowledgment logos --- docs/_static/custom.css | 27 +++++++++++++++++++++++++++ docs/index.md | 8 ++++---- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 2c6cbb8c..089432d5 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -34,6 +34,33 @@ --color-sidebar--hover: white; } +/* The furo theme uses only-light and only-dark for light/dark-mode only + images, but they use display:block, so define + only-light-inline/only-dark-inline to use display:inline. */ + +.only-light-inline { + display: inline !important; +} +html body .only-dark-inline { + display: none !important; +} +@media not print { + html body[data-theme=dark] .only-light-inline { + display: none !important; + } + body[data-theme=dark] .only-dark-inline { + display: inline !important; + } + @media (prefers-color-scheme: dark) { + html body:not([data-theme=light]) .only-light-inline { + display: none !important; + } + body:not([data-theme=light]) .only-dark-inline { + display: inline !important; + } + } +} + /* Make top-level items in the sidebar bold */ .sidebar-tree .toctree-l1>.reference, .sidebar-tree .toctree-l1>label .icon { font-weight: bold !important; diff --git a/docs/index.md b/docs/index.md index 828a8c43..cde0bee8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -319,10 +319,10 @@ Quansight on numerous open source projects, including Numba, Dask and Project Jupyter.
-https://labs.quansight.org/ -https://www.deshaw.com +https://labs.quansight.org/images/QuansightLabs_logo_V2.png +https://labs.quansight.org/images/QuansightLabs_logo_V2_white.png +https://www.deshaw.com/ +https://www.deshaw.com/
## Table of Contents From 41d090598096decdd77f3fe6d197b652b0c0952d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 4 Nov 2022 14:26:47 -0600 Subject: [PATCH 048/218] Don't generate large shapes in mutually_broadcastable_shapes_with_skipped_axes --- ndindex/tests/helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index a694db40..251bc6cd 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -157,6 +157,7 @@ def mutually_broadcastable_shapes_with_skipped_axes(draw): _result_shape[i] = None _result_shape = tuple(_result_shape) + assume(prod([i for i in _result_shape if i]) < MAX_ARRAY_SIZE) return BroadcastableShapes(_shapes, _result_shape) From dcc422bbefd54351f66c16ce0e2b4071db31f04a Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 4 Nov 2022 14:38:17 -0600 Subject: [PATCH 049/218] More correctly limit the sizes of shapes from mutually_broadcastable_shapes_with_skipped_axes --- ndindex/tests/helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index 251bc6cd..c7d1641c 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -157,7 +157,8 @@ def mutually_broadcastable_shapes_with_skipped_axes(draw): _result_shape[i] = None _result_shape = tuple(_result_shape) - assume(prod([i for i in _result_shape if i]) < MAX_ARRAY_SIZE) + for shape in _shapes: + assume(prod([i for i in shape if i]) < SHORT_MAX_ARRAY_SIZE) return BroadcastableShapes(_shapes, _result_shape) From 185970e3931b19d32f427e52bbf7920fb6cede7b Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 4 Nov 2022 14:38:37 -0600 Subject: [PATCH 050/218] Fix an IndexError in mutually_broadcastable_shapes_with_skipped_axes --- ndindex/tests/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index c7d1641c..a4d0d506 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -147,7 +147,7 @@ def mutually_broadcastable_shapes_with_skipped_axes(draw): for shape in shapes: _shape = list(shape) for i in skip_axes_: - if draw(booleans()): + if ndindex(i).isvalid(len(shape)) and draw(booleans()): _shape[i] = draw(integers(0)) _shapes.append(tuple(_shape)) From 2c9284446732bffc5a7c9b1150299f5ed0f23a61 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sat, 5 Nov 2022 00:45:01 -0600 Subject: [PATCH 051/218] More cleanups to test_iter_indices The test passes now but I still need to confirm that it is actually testing the correct things and isn't missing any cases. --- ndindex/tests/test_ndindex.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/ndindex/tests/test_ndindex.py b/ndindex/tests/test_ndindex.py index bf54dc4c..885b3f4b 100644 --- a/ndindex/tests/test_ndindex.py +++ b/ndindex/tests/test_ndindex.py @@ -153,9 +153,9 @@ def test_asshape(): raises(TypeError, lambda: asshape(Tuple(1, 2))) raises(TypeError, lambda: asshape((True,))) -@example([((1, 1), (1, 1)), (1, 1)], (0, 0)) -@example([((), (0,)), (0,)], (0,)) -@example([((1, 2), (2, 1)), (2, 2)], 1) +@example([((1, 1), (1, 1)), (None, 1)], (0, 0)) +@example([((), (0,)), (None,)], (0,)) +@example([((1, 2), (2, 1)), (2, None)], 1) @given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes) def test_iter_indices(broadcastable_shapes, _skip_axes): # broadcasted_shape will contain None on the skip_axes, as those axes @@ -163,28 +163,32 @@ def test_iter_indices(broadcastable_shapes, _skip_axes): shapes, broadcasted_shape = broadcastable_shapes # 1. Normalize inputs - skip_axes = (_skip_axes,) if isinstance(_skip_axes, int) else _skip_axes + skip_axes = (_skip_axes,) if isinstance(_skip_axes, int) else () if _skip_axes is None else _skip_axes ndim = len(broadcasted_shape) + # Double check the mutually_broadcastable_shapes_with_skipped_axes + # strategy + for i in skip_axes: + assert broadcasted_shape[i] is None + # Use negative indices to index the skip axes since only shapes that have # the skip axis will include a slice. normalized_skip_axes = sorted(ndindex(i).reduce(ndim).args[0] - ndim for i in skip_axes) canonical_shapes = [list(s) for s in shapes] for i in normalized_skip_axes: for s in canonical_shapes: - if ndindex(i).isvalid(s): + if ndindex(i).isvalid(len(s)): s[i] = 1 - skip_shapes = [tuple(shape[i] for i in normalized_skip_axes if ndindex(i).isvalid(shape)) for shape in canonical_shapes] + skip_shapes = [tuple(shape[i] for i in normalized_skip_axes if ndindex(i).isvalid(len(shape))) for shape in canonical_shapes] broadcasted_skip_shape = tuple(broadcasted_shape[i] for i in normalized_skip_axes) broadcasted_non_skip_shape = tuple(broadcasted_shape[i] for i in range(-1, -ndim-1, -1) if i not in normalized_skip_axes) nitems = prod(broadcasted_non_skip_shape) broadcasted_nitems = prod([i for i in broadcasted_shape if i is not None]) - if skip_axes is None: + if _skip_axes is None: res = iter_indices(*shapes) broadcasted_res = iter_indices(np.broadcast_shapes(*shapes)) - skip_axes = () else: # Skipped axes may not be broadcast compatible. Since the index for a # skipped axis should always be a slice(None), the result should be @@ -200,7 +204,7 @@ def test_iter_indices(broadcastable_shapes, _skip_axes): broadcasted_arrays = np.broadcast_arrays(*canonical_arrays) # 2. Check that iter_indices is the same whether or not the shapes are - # broadcasted together first. Also Check that every iterated index is the + # broadcasted together first. Also check that every iterated index is the # expected type and there are as many as expected. vals = [] n = -1 @@ -230,7 +234,7 @@ def test_iter_indices(broadcastable_shapes, _skip_axes): # If there are skipped axes, recursively call iter_indices to # get each individual element of the resulting subarrays. for subidxes in iter_indices(*[x.shape for x in canonical_aidxes]): - items = [x[i.raw] for x, i in zip(aidxes, subidxes)] + items = [x[i.raw] for x, i in zip(canonical_aidxes, subidxes)] # An empty array means the iteration would be skipped. if any(a.size == 0 for a in items): continue From d5ae7eb2f33c2f6702cc7a41b478c29af5725258 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sat, 5 Nov 2022 00:50:44 -0600 Subject: [PATCH 052/218] Add tests for isvalid --- ndindex/tests/test_isvalid.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 ndindex/tests/test_isvalid.py diff --git a/ndindex/tests/test_isvalid.py b/ndindex/tests/test_isvalid.py new file mode 100644 index 00000000..974315a2 --- /dev/null +++ b/ndindex/tests/test_isvalid.py @@ -0,0 +1,24 @@ +from pytest import raises + +from hypothesis import given + +from numpy import arange, prod + +from ..ndindex import ndindex +from .helpers import ndindices, shapes + +@given(ndindices, shapes) +def test_isvalid_hypothesis(idx, shape): + index = ndindex(idx) + + if isinstance(shape, int): + a = arange(shape) + else: + a = arange(prod(shape)).reshape(shape) + + valid = index.isvalid(shape) + + if valid: + a[idx] # doesn't raise + else: + raises(IndexError, lambda: a[idx]) From 9cac3b0f805ec3cffd81276dc0ea3d82913f56f9 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sat, 5 Nov 2022 01:04:32 -0600 Subject: [PATCH 053/218] Update documentation for isvalid --- ndindex/ndindex.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ndindex/ndindex.py b/ndindex/ndindex.py index c3065412..ed7093c0 100644 --- a/ndindex/ndindex.py +++ b/ndindex/ndindex.py @@ -301,8 +301,8 @@ def isvalid(self, shape): """ Check whether a given index is valid on an array of a given shape. - Returns True if an array of shape `shape` can be indexed by `self` and - False if it would raise `IndexError`. + Returns `True` if an array of shape `shape` can be indexed by `self` + and `False` if it would raise `IndexError`. >>> from ndindex import ndindex >>> ndindex(3).isvalid((4,)) @@ -310,6 +310,12 @@ def isvalid(self, shape): >>> ndindex(3).isvalid((2,)) False + Note that :class:`~.Slice`, :class:`~.ellipsis`, and + :class:`~.Newaxis` indices are always valid regardless of the `shape`. + + >>> ndindex(slice(0, 10)).isvalid((3,)) + True + """ # TODO: More direct, efficient implementation try: From 597e59943b23a723ad877d1de195ca90e9370bf6 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 8 Nov 2022 02:02:20 -0700 Subject: [PATCH 054/218] Split out the logic for isvalid to individual classes Tuple.isvalid() still just reuses newshape() because it would effectively just need to reimplement expand() to be specialized. --- ndindex/array.py | 10 ++++++++++ ndindex/booleanarray.py | 21 +++++++++++---------- ndindex/ellipsis.py | 3 +++ ndindex/integer.py | 20 +++++++++++++++----- ndindex/integerarray.py | 15 +++++++++------ ndindex/ndindex.py | 22 ++++++++++++++++------ ndindex/newaxis.py | 3 +++ ndindex/slice.py | 6 ++++++ 8 files changed, 73 insertions(+), 27 deletions(-) diff --git a/ndindex/array.py b/ndindex/array.py index 618e5ff4..7d8692d2 100644 --- a/ndindex/array.py +++ b/ndindex/array.py @@ -159,3 +159,13 @@ def __str__(self): def __hash__(self): return hash(self.array.tobytes()) + + def isvalid(self, shape, _axis=0): + try: + # The logic is in _raise_indexerror because the error message uses + # the additional information that is computed when checking if the + # array is valid. + self._raise_indexerror(shape, _axis) + except IndexError: + return False + return True diff --git a/ndindex/booleanarray.py b/ndindex/booleanarray.py index 262b2b5b..da96b3aa 100644 --- a/ndindex/booleanarray.py +++ b/ndindex/booleanarray.py @@ -102,6 +102,15 @@ def count_nonzero(self): from numpy import count_nonzero return count_nonzero(self.array) + def _raise_indexerror(self, shape, axis=0): + if len(shape) < self.ndim + axis: + raise IndexError(f"too many indices for array: array is {len(shape)}-dimensional, but {self.ndim + axis} were indexed") + + for i in range(axis, axis+self.ndim): + if self.shape[i-axis] != 0 and shape[i] != self.shape[i-axis]: + + raise IndexError(f"boolean index did not match indexed array along dimension {i}; dimension is {shape[i]} but corresponding boolean dimension is {self.shape[i-axis]}") + def reduce(self, shape=None, axis=0): """ Reduce a `BooleanArray` index on an array of shape `shape`. @@ -137,22 +146,14 @@ def reduce(self, shape=None, axis=0): shape = asshape(shape) - if len(shape) < self.ndim + axis: - raise IndexError(f"too many indices for array: array is {len(shape)}-dimensional, but {self.ndim + axis} were indexed") - - for i in range(axis, axis+self.ndim): - if self.shape[i-axis] != 0 and shape[i] != self.shape[i-axis]: - - raise IndexError(f"boolean index did not match indexed array along dimension {i}; dimension is {shape[i]} but corresponding boolean dimension is {self.shape[i-axis]}") - + self._raise_indexerror(shape, axis) return self def newshape(self, shape): # The docstring for this method is on the NDIndex base class shape = asshape(shape) - # reduce will raise IndexError if it should be raised - self.reduce(shape) + self._raise_indexerror(shape) return (self.count_nonzero,) + shape[self.ndim:] def isempty(self, shape=None): diff --git a/ndindex/ellipsis.py b/ndindex/ellipsis.py index 6caa1cbc..05d4787f 100644 --- a/ndindex/ellipsis.py +++ b/ndindex/ellipsis.py @@ -77,6 +77,9 @@ def reduce(self, shape=None): def raw(self): return ... + def isvalid(self, shape): + return True + def newshape(self, shape): # The docstring for this method is on the NDIndex base class shape = asshape(shape) diff --git a/ndindex/integer.py b/ndindex/integer.py index 84ec7a87..4af327a4 100644 --- a/ndindex/integer.py +++ b/ndindex/integer.py @@ -48,6 +48,18 @@ def __len__(self): """ return 1 + def isvalid(self, shape, _axis=0): + # The docstring for this method is on the NDIndex base class + if not shape: + return False + size = shape[_axis] + return -size <= self.raw < size + + def _raise_indexerror(self, shape, axis=0): + if not self.isvalid(shape, axis): + size = shape[axis] + raise IndexError(f"index {self.raw} is out of bounds for axis {axis} with size {size}") + def reduce(self, shape=None, axis=0): """ Reduce an Integer index on an array of shape `shape`. @@ -80,11 +92,10 @@ def reduce(self, shape=None, axis=0): return self shape = asshape(shape, axis=axis) - size = shape[axis] - if self.raw >= size or -size > self.raw < 0: - raise IndexError(f"index {self.raw} is out of bounds for axis {axis} with size {size}") + self._raise_indexerror(shape, axis) if self.raw < 0: + size = shape[axis] return self.__class__(size + self.raw) return self @@ -93,8 +104,7 @@ def newshape(self, shape): # The docstring for this method is on the NDIndex base class shape = asshape(shape) - # reduce will raise IndexError if it should be raised - self.reduce(shape) + self._raise_indexerror(shape) return shape[1:] def as_subindex(self, index): diff --git a/ndindex/integerarray.py b/ndindex/integerarray.py index 13938ed3..98d679fc 100644 --- a/ndindex/integerarray.py +++ b/ndindex/integerarray.py @@ -51,6 +51,12 @@ def dtype(self): from numpy import intp return intp + def _raise_indexerror(self, shape, axis=0): + size = shape[axis] + out_of_bounds = (self.array >= size) | ((-size > self.array) & (self.array < 0)) + if out_of_bounds.any(): + raise IndexError(f"index {self.array[out_of_bounds].flat[0]} is out of bounds for axis {axis} with size {size}") + def reduce(self, shape=None, axis=0): """ Reduce an `IntegerArray` index on an array of shape `shape`. @@ -89,12 +95,10 @@ def reduce(self, shape=None, axis=0): shape = asshape(shape, axis=axis) + self._raise_indexerror(shape, axis) + size = shape[axis] new_array = self.array.copy() - out_of_bounds = (new_array >= size) | ((-size > new_array) & (new_array < 0)) - if out_of_bounds.any(): - raise IndexError(f"index {new_array[out_of_bounds].flat[0]} is out of bounds for axis {axis} with size {size}") - new_array[new_array < 0] += size return IntegerArray(new_array) @@ -102,8 +106,7 @@ def newshape(self, shape): # The docstring for this method is on the NDIndex base class shape = asshape(shape) - # reduce will raise IndexError if it should be raised - self.reduce(shape) + self._raise_indexerror(shape) return self.shape + shape[1:] def isempty(self, shape=None): diff --git a/ndindex/ndindex.py b/ndindex/ndindex.py index ed7093c0..ae019f9c 100644 --- a/ndindex/ndindex.py +++ b/ndindex/ndindex.py @@ -310,11 +310,17 @@ def isvalid(self, shape): >>> ndindex(3).isvalid((2,)) False - Note that :class:`~.Slice`, :class:`~.ellipsis`, and - :class:`~.Newaxis` indices are always valid regardless of the `shape`. + Some indices can never be valid and will raise a `TypeError` if you + attempt to construct them. - >>> ndindex(slice(0, 10)).isvalid((3,)) - True + >>> ndindex((..., 0, ...)) + Traceback (most recent call last): + ... + IndexError: an index can only have a single ellipsis ('...') + + See Also + ======== + .NDIndex.newshape """ # TODO: More direct, efficient implementation @@ -404,8 +410,8 @@ def newshape(self, shape): `shape` should be a tuple of ints, or an int, which is equivalent to a 1-D shape. - Raises `IndexError` if `self` would be out of shape for an array of - shape `shape`. + Raises `IndexError` if `self` would be invalid for an array of shape + `shape`. >>> from ndindex import Slice, Integer, Tuple >>> shape = (6, 7, 8) @@ -420,6 +426,10 @@ def newshape(self, shape): >>> Tuple(0, ..., Slice(1, 3)).newshape(shape) (7, 2) + See Also + ======== + .NDIndex.isvalid + """ raise NotImplementedError diff --git a/ndindex/newaxis.py b/ndindex/newaxis.py index 28df499b..20828e25 100644 --- a/ndindex/newaxis.py +++ b/ndindex/newaxis.py @@ -69,6 +69,9 @@ def reduce(self, shape=None, axis=0): shape = asshape(shape) return self + def isvalid(self, shape): + return True + def newshape(self, shape): # The docstring for this method is on the NDIndex base class shape = asshape(shape) diff --git a/ndindex/slice.py b/ndindex/slice.py index 00497d89..bac1d662 100644 --- a/ndindex/slice.py +++ b/ndindex/slice.py @@ -470,6 +470,12 @@ def reduce(self, shape=None, axis=0): stop = start % -step - 1 return self.__class__(start, stop, step) + def isvalid(self, shape): + # The docstring for this method is on the NDIndex base class + + # All slices are valid as long as there is at least one dimension + return bool(shape) + def newshape(self, shape): # The docstring for this method is on the NDIndex base class shape = asshape(shape) From cd7c78ad92dbe3584f512345dd9a6128847c11af Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 8 Nov 2022 02:26:30 -0700 Subject: [PATCH 055/218] Remove the defunct _templates directory in the docs --- docs/_templates/globaltocindex.html | 25 ------------------------- docs/_templates/layout.html | 11 ----------- docs/conf.py | 3 --- 3 files changed, 39 deletions(-) delete mode 100644 docs/_templates/globaltocindex.html delete mode 100644 docs/_templates/layout.html diff --git a/docs/_templates/globaltocindex.html b/docs/_templates/globaltocindex.html deleted file mode 100644 index c1657573..00000000 --- a/docs/_templates/globaltocindex.html +++ /dev/null @@ -1,25 +0,0 @@ -{# - basic/globaltoc.html - ~~~~~~~~~~~~~~~~~~~~ - - Sphinx sidebar template: global table of contents. - - :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS. - :license: BSD, see LICENSE for details. - - Modified to work properly with the index page -#} -{% if theme_logo %} -

{{ project }}

- {% endif %} - -

-{% else %} -

{{ project }}

-{% endif %} - -{{ toctree(maxdepth=-1, collapse=True) }} diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html deleted file mode 100644 index 2b047659..00000000 --- a/docs/_templates/layout.html +++ /dev/null @@ -1,11 +0,0 @@ -{% extends "!layout.html" %} - - {%- block extrahead %} - {{ super() }} - - {% endblock %} - - {%- block footer %} - {{super()}} - Fork me on GitHub - {%- endblock %} diff --git a/docs/conf.py b/docs/conf.py index bf36f79f..c520ebc1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -73,9 +73,6 @@ # html_theme = 'furo' -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". From d318ba23071b28c599a340e200e0bdb25c9b1f41 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 8 Nov 2022 02:26:58 -0700 Subject: [PATCH 056/218] Use titlesonly in the table of contents --- docs/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/index.md b/docs/index.md index cde0bee8..26ec156a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -328,6 +328,8 @@ Jupyter. ## Table of Contents ```{toctree} +:titlesonly: + api.md slices.md type-confusion.md From c1a9d6560c27ac741fabd7bb19241442d236f9c9 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 8 Nov 2022 17:20:24 -0700 Subject: [PATCH 057/218] Fix bug in Integer.isvalid() --- ndindex/integer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ndindex/integer.py b/ndindex/integer.py index 4af327a4..0dc490b3 100644 --- a/ndindex/integer.py +++ b/ndindex/integer.py @@ -50,6 +50,7 @@ def __len__(self): def isvalid(self, shape, _axis=0): # The docstring for this method is on the NDIndex base class + shape = asshape(shape) if not shape: return False size = shape[_axis] From 0e841013fe089fa4b28ac1b5ca9eebf4ed760aff Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 8 Nov 2022 17:22:18 -0700 Subject: [PATCH 058/218] Remove unused variable --- ndindex/tests/test_ndindex.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ndindex/tests/test_ndindex.py b/ndindex/tests/test_ndindex.py index 07df540a..6adfd48a 100644 --- a/ndindex/tests/test_ndindex.py +++ b/ndindex/tests/test_ndindex.py @@ -188,7 +188,6 @@ def test_iter_indices(broadcastable_shapes, _skip_axes): broadcasted_non_skip_shape = tuple(broadcasted_shape[i] for i in range(-1, -ndim-1, -1) if i not in normalized_skip_axes) nitems = prod(broadcasted_non_skip_shape) - broadcasted_nitems = prod([i for i in broadcasted_shape if i is not None]) if _skip_axes is None: res = iter_indices(*shapes) From bc39a1dac325deda25981dd821387d9b4217d36d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 8 Nov 2022 18:25:46 -0700 Subject: [PATCH 059/218] Fix logic in mutually_broadcastable_shapes_with_skipped_axes --- ndindex/tests/helpers.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index a9e84248..1a1c6d4f 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -138,6 +138,7 @@ def mutually_broadcastable_shapes_with_skipped_axes(draw): """ skip_axes_ = draw(skip_axes) shapes, result_shape = draw(mutually_broadcastable_shapes) + ndim = len(result_shape) if skip_axes_ is None: return shapes, result_shape if isinstance(skip_axes_, int): @@ -147,8 +148,13 @@ def mutually_broadcastable_shapes_with_skipped_axes(draw): for shape in shapes: _shape = list(shape) for i in skip_axes_: - if ndindex(i).isvalid(len(shape)) and draw(booleans()): - _shape[i] = draw(integers(0)) + # skip axes index the broadcasted shape, so are only valid on the + # individual shapes in negative form. TODO: Add this as a keyword + # to reduce(). + neg_i = ndindex(i).reduce(ndim).raw - ndim + assert ndindex(i).reduce(ndim) == ndindex(neg_i).reduce(ndim) + if ndindex(neg_i).isvalid(len(shape)) and draw(booleans()): + _shape[neg_i] = draw(integers(0)) _shapes.append(tuple(_shape)) From 6c9bacb775fcfb091f80522b2e1599dda1106d4e Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 8 Nov 2022 18:26:37 -0700 Subject: [PATCH 060/218] Remove unused lines test_iter_indices --- ndindex/tests/test_ndindex.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ndindex/tests/test_ndindex.py b/ndindex/tests/test_ndindex.py index 6adfd48a..a464ca92 100644 --- a/ndindex/tests/test_ndindex.py +++ b/ndindex/tests/test_ndindex.py @@ -238,9 +238,6 @@ def test_iter_indices(broadcastable_shapes, _skip_axes): # get each individual element of the resulting subarrays. for subidxes in iter_indices(*[x.shape for x in canonical_aidxes]): items = [x[i.raw] for x, i in zip(canonical_aidxes, subidxes)] - # An empty array means the iteration would be skipped. - if any(a.size == 0 for a in items): - continue vals.append(tuple(items)) else: vals.append(aidxes) From defd6f2cfa2155ccfa5a72f390fe1e551df17b16 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 8 Nov 2022 18:38:37 -0700 Subject: [PATCH 061/218] Handle NumPy deprecation warning in test_isvalid --- ndindex/tests/test_isvalid.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ndindex/tests/test_isvalid.py b/ndindex/tests/test_isvalid.py index 974315a2..4edadf2b 100644 --- a/ndindex/tests/test_isvalid.py +++ b/ndindex/tests/test_isvalid.py @@ -1,4 +1,4 @@ -from pytest import raises +from pytest import raises, fail from hypothesis import given @@ -21,4 +21,11 @@ def test_isvalid_hypothesis(idx, shape): if valid: a[idx] # doesn't raise else: - raises(IndexError, lambda: a[idx]) + with raises(IndexError): + try: + a[idx] + except Warning as w: + if "Out of bound index found. This was previously ignored when the indexing result contained no elements. In the future the index error will be raised. This error occurs either due to an empty slice, or if an array has zero elements even before indexing." in w.args[0]: + raise IndexError + else: # pragma: no cover + fail(f"Unexpected warning raised: {w}") From 0ffacf6e4d4fc7a3b9624affd17737b5bbf3591f Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 8 Nov 2022 18:38:54 -0700 Subject: [PATCH 062/218] Fix isvalid with integers as shapes --- ndindex/array.py | 1 + ndindex/tests/test_isvalid.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ndindex/array.py b/ndindex/array.py index b31256ba..390cc220 100644 --- a/ndindex/array.py +++ b/ndindex/array.py @@ -164,6 +164,7 @@ def __hash__(self): return hash(self.array.tobytes()) def isvalid(self, shape, _axis=0): + shape = asshape(shape) try: # The logic is in _raise_indexerror because the error message uses # the additional information that is computed when checking if the diff --git a/ndindex/tests/test_isvalid.py b/ndindex/tests/test_isvalid.py index 4edadf2b..a7ee9fa9 100644 --- a/ndindex/tests/test_isvalid.py +++ b/ndindex/tests/test_isvalid.py @@ -1,13 +1,14 @@ from pytest import raises, fail from hypothesis import given +from hypothesis.strategies import one_of, integers from numpy import arange, prod from ..ndindex import ndindex -from .helpers import ndindices, shapes +from .helpers import ndindices, shapes, MAX_ARRAY_SIZE -@given(ndindices, shapes) +@given(ndindices, one_of(shapes, integers(0, MAX_ARRAY_SIZE))) def test_isvalid_hypothesis(idx, shape): index = ndindex(idx) From 08bbeaad7bb6e7df0427c3c802cbe9c7c83d1a78 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 8 Nov 2022 18:42:51 -0700 Subject: [PATCH 063/218] Check for valid shapes in ellipsis and Newaxis isvalid --- ndindex/ellipsis.py | 1 + ndindex/newaxis.py | 1 + 2 files changed, 2 insertions(+) diff --git a/ndindex/ellipsis.py b/ndindex/ellipsis.py index 05d4787f..9d6ce64a 100644 --- a/ndindex/ellipsis.py +++ b/ndindex/ellipsis.py @@ -78,6 +78,7 @@ def raw(self): return ... def isvalid(self, shape): + shape = asshape(shape) return True def newshape(self, shape): diff --git a/ndindex/newaxis.py b/ndindex/newaxis.py index 20828e25..d86f57b9 100644 --- a/ndindex/newaxis.py +++ b/ndindex/newaxis.py @@ -70,6 +70,7 @@ def reduce(self, shape=None, axis=0): return self def isvalid(self, shape): + shape = asshape(shape) return True def newshape(self, shape): From 98d9d023240504ff1b80cff8ec08165ee59acf26 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 8 Nov 2022 18:44:28 -0700 Subject: [PATCH 064/218] Remove a fixed TODO comment --- ndindex/ndindex.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ndindex/ndindex.py b/ndindex/ndindex.py index ae019f9c..b42dc159 100644 --- a/ndindex/ndindex.py +++ b/ndindex/ndindex.py @@ -323,7 +323,10 @@ def isvalid(self, shape): .NDIndex.newshape """ - # TODO: More direct, efficient implementation + # Every class except for Tuple has a more direct efficient + # implementation. The logic for checking if a Tuple index is valid is + # basically the same as the logic in reduce/expand, so there's no + # point in duplicating it. try: self.reduce(shape) return True From a11147dd30203630bdfe8d95a8fa3b570b874367 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 8 Nov 2022 18:44:42 -0700 Subject: [PATCH 065/218] Small code cleanup --- ndindex/ndindex.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ndindex/ndindex.py b/ndindex/ndindex.py index b42dc159..0ea1fa9b 100644 --- a/ndindex/ndindex.py +++ b/ndindex/ndindex.py @@ -647,12 +647,12 @@ def _broadcast_shapes(shape1, shape2): i = N - 1 while i >= 0: n1 = N1 - N + i - if N1 - N + i >= 0: + if n1 >= 0: d1 = shape1[n1] else: d1 = 1 n2 = N2 - N + i - if N2 - N + i >= 0: + if n2 >= 0: d2 = shape2[n2] else: d2 = 1 From bce2254d244bc02ab20cb1fc0bc72b89cbcaf0e9 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 9 Nov 2022 01:18:35 -0700 Subject: [PATCH 066/218] Add a test for broadcast_shapes() --- ndindex/tests/test_ndindex.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/ndindex/tests/test_ndindex.py b/ndindex/tests/test_ndindex.py index a464ca92..0345c4d1 100644 --- a/ndindex/tests/test_ndindex.py +++ b/ndindex/tests/test_ndindex.py @@ -4,11 +4,13 @@ import numpy as np from hypothesis import given, example, settings -from hypothesis.strategies import integers +from hypothesis.strategies import (one_of, integers, tuples as + hypothesis_tuples, just) from pytest import raises -from ..ndindex import ndindex, asshape, iter_indices, ncycles, BroadcastError, AxisError +from ..ndindex import (ndindex, asshape, iter_indices, ncycles, + BroadcastError, AxisError, broadcast_shapes) from ..booleanarray import BooleanArray from ..integer import Integer from ..ellipsis import ellipsis @@ -16,7 +18,8 @@ from ..tuple import Tuple from .helpers import (ndindices, check_same, assert_equal, prod, mutually_broadcastable_shapes_with_skipped_axes, - skip_axes) + skip_axes, mutually_broadcastable_shapes, tuples, + shapes) @example([1, 2]) @given(ndindices) @@ -331,3 +334,22 @@ def test_ncycles(i, n, m): assert isinstance(M, ncycles) assert M.iterable == range(i) assert M.n == n*m + +@given(one_of(mutually_broadcastable_shapes, + hypothesis_tuples(tuples(shapes), just(None)))) +def test_broadcast_shapes(broadcastable_shapes): + shapes, broadcasted_shape = broadcastable_shapes + if broadcasted_shape is not None: + assert broadcast_shapes(*shapes) == broadcasted_shape + + arrays = [np.empty(shape) for shape in shapes] + broadcastable = True + try: + broadcasted_shape = np.broadcast(*arrays).shape + except ValueError: + broadcastable = False + + if broadcastable: + assert broadcast_shapes(*shapes) == broadcasted_shape + else: + raises(BroadcastError, lambda: broadcast_shapes(*shapes)) From 52e889b356582d6a384f77b63cd87161a3e14638 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 9 Nov 2022 16:53:45 -0700 Subject: [PATCH 067/218] Add an @example for coverage --- ndindex/tests/test_isvalid.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ndindex/tests/test_isvalid.py b/ndindex/tests/test_isvalid.py index a7ee9fa9..14338923 100644 --- a/ndindex/tests/test_isvalid.py +++ b/ndindex/tests/test_isvalid.py @@ -1,6 +1,6 @@ from pytest import raises, fail -from hypothesis import given +from hypothesis import given, example from hypothesis.strategies import one_of, integers from numpy import arange, prod @@ -8,6 +8,7 @@ from ..ndindex import ndindex from .helpers import ndindices, shapes, MAX_ARRAY_SIZE +@example([[1]], (0, 0, 1)) @given(ndindices, one_of(shapes, integers(0, MAX_ARRAY_SIZE))) def test_isvalid_hypothesis(idx, shape): index = ndindex(idx) From 98e3dde19241b01303f6014535892ee920d1bebb Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 9 Nov 2022 16:57:01 -0700 Subject: [PATCH 068/218] Fix the Slice.isvalid implementation --- ndindex/slice.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ndindex/slice.py b/ndindex/slice.py index 2a91c222..51b757f0 100644 --- a/ndindex/slice.py +++ b/ndindex/slice.py @@ -472,6 +472,7 @@ def reduce(self, shape=None, axis=0): def isvalid(self, shape): # The docstring for this method is on the NDIndex base class + shape = asshape(shape) # All slices are valid as long as there is at least one dimension return bool(shape) From 9209f463cf5070b67584debed75370c0a9b86c98 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 9 Nov 2022 16:58:16 -0700 Subject: [PATCH 069/218] Add an @example for coverage --- ndindex/tests/test_isvalid.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ndindex/tests/test_isvalid.py b/ndindex/tests/test_isvalid.py index 14338923..a817ff9f 100644 --- a/ndindex/tests/test_isvalid.py +++ b/ndindex/tests/test_isvalid.py @@ -8,6 +8,7 @@ from ..ndindex import ndindex from .helpers import ndindices, shapes, MAX_ARRAY_SIZE +@example((0,), ()) @example([[1]], (0, 0, 1)) @given(ndindices, one_of(shapes, integers(0, MAX_ARRAY_SIZE))) def test_isvalid_hypothesis(idx, shape): From aa7a8d36d5977fe7b1d2970ff780e46b2f4262f5 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 9 Nov 2022 17:31:58 -0700 Subject: [PATCH 070/218] Show the NumPy version in the tests header --- conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conftest.py b/conftest.py index 5b2966a3..295cda47 100644 --- a/conftest.py +++ b/conftest.py @@ -13,6 +13,10 @@ if LooseVersion(numpy.__version__) < LooseVersion('1.20'): raise RuntimeError("NumPy 1.20 (development version) or greater is required to run the ndindex tests") +# Show the NumPy version in the pytest header +def pytest_report_header(config): + return f"project deps: numpy-{numpy.__version__}" + # Add a --hypothesis-max-examples flag to pytest. See # https://github.com/HypothesisWorks/hypothesis/issues/2434#issuecomment-630309150 From a005eba339ddd4d89fe5df6a5e59c64246be84d6 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 9 Nov 2022 17:37:33 -0700 Subject: [PATCH 071/218] Fix the handling of an old NumPy deprecation warning in the tests --- ndindex/tests/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index 1a1c6d4f..6e4a17f5 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -261,7 +261,7 @@ def assert_equal(x, y): # deprecation was removed and lists are always interpreted as # array indices. if ("Using a non-tuple sequence for multidimensional indexing is deprecated" in w.args[0]): # pragma: no cover - idx = array(idx) + idx = array(idx, dtype=intp) a_raw = raw_func(a, idx) elif "Out of bound index found. This was previously ignored when the indexing result contained no elements. In the future the index error will be raised. This error occurs either due to an empty slice, or if an array has zero elements even before indexing." in w.args[0]: same_exception = False From f1e41de38a87d5ae6607cd0b82a40160228bebef Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 9 Nov 2022 17:38:29 -0700 Subject: [PATCH 072/218] Use check_same in test_isvalid --- ndindex/tests/test_isvalid.py | 41 +++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/ndindex/tests/test_isvalid.py b/ndindex/tests/test_isvalid.py index a817ff9f..bb312eab 100644 --- a/ndindex/tests/test_isvalid.py +++ b/ndindex/tests/test_isvalid.py @@ -1,34 +1,37 @@ -from pytest import raises, fail - from hypothesis import given, example from hypothesis.strategies import one_of, integers -from numpy import arange, prod +from numpy import arange -from ..ndindex import ndindex -from .helpers import ndindices, shapes, MAX_ARRAY_SIZE +from .helpers import ndindices, shapes, MAX_ARRAY_SIZE, check_same, prod @example((0,), ()) @example([[1]], (0, 0, 1)) @given(ndindices, one_of(shapes, integers(0, MAX_ARRAY_SIZE))) def test_isvalid_hypothesis(idx, shape): - index = ndindex(idx) - if isinstance(shape, int): a = arange(shape) else: a = arange(prod(shape)).reshape(shape) - valid = index.isvalid(shape) + def raw_func(a, idx): + try: + a[idx] + return True + except Warning as w: + # check_same unconditionally turns this warning into raise + # IndexError, so we have to handle it separately here. + if "Out of bound index found. This was previously ignored when the indexing result contained no elements. In the future the index error will be raised. This error occurs either due to an empty slice, or if an array has zero elements even before indexing." in w.args[0]: + return False + raise + except IndexError: + return False - if valid: - a[idx] # doesn't raise - else: - with raises(IndexError): - try: - a[idx] - except Warning as w: - if "Out of bound index found. This was previously ignored when the indexing result contained no elements. In the future the index error will be raised. This error occurs either due to an empty slice, or if an array has zero elements even before indexing." in w.args[0]: - raise IndexError - else: # pragma: no cover - fail(f"Unexpected warning raised: {w}") + def ndindex_func(a, index): + return index.isvalid(a.shape) + + def assert_equal(x, y): + assert x == y + + check_same(a, idx, raw_func=raw_func, ndindex_func=ndindex_func, + assert_equal=assert_equal) From a9e39b3fc0d4f8043bc65bb11ded7502367f0ddf Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 9 Nov 2022 17:46:02 -0700 Subject: [PATCH 073/218] Use keyword-only arguments in check_same --- ndindex/tests/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index 6e4a17f5..8cc4dd6e 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -223,7 +223,7 @@ def assert_equal(actual, desired, err_msg='', verbose=True): assert actual.shape == desired.shape, err_msg or f"{actual.shape} != {desired.shape}" assert actual.dtype == desired.dtype, err_msg or f"{actual.dtype} != {desired.dtype}" -def check_same(a, idx, raw_func=lambda a, idx: a[idx], +def check_same(a, idx, *, raw_func=lambda a, idx: a[idx], ndindex_func=lambda a, index: a[index.raw], same_exception=True, assert_equal=assert_equal): """ From 82ff107460ebfe69b3c7dd30b97e4717f768ee8e Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 9 Nov 2022 17:46:18 -0700 Subject: [PATCH 074/218] Add an @example for coverage --- ndindex/tests/test_isvalid.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ndindex/tests/test_isvalid.py b/ndindex/tests/test_isvalid.py index bb312eab..aa1bbdc9 100644 --- a/ndindex/tests/test_isvalid.py +++ b/ndindex/tests/test_isvalid.py @@ -7,6 +7,7 @@ @example((0,), ()) @example([[1]], (0, 0, 1)) +@example(None, ()) @given(ndindices, one_of(shapes, integers(0, MAX_ARRAY_SIZE))) def test_isvalid_hypothesis(idx, shape): if isinstance(shape, int): From 51ba6cb431621a19759fab2af82a2269b300e706 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 9 Nov 2022 17:51:47 -0700 Subject: [PATCH 075/218] Add missing pragma: no cover --- ndindex/tests/test_isvalid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndindex/tests/test_isvalid.py b/ndindex/tests/test_isvalid.py index aa1bbdc9..5bad0ac2 100644 --- a/ndindex/tests/test_isvalid.py +++ b/ndindex/tests/test_isvalid.py @@ -24,7 +24,7 @@ def raw_func(a, idx): # IndexError, so we have to handle it separately here. if "Out of bound index found. This was previously ignored when the indexing result contained no elements. In the future the index error will be raised. This error occurs either due to an empty slice, or if an array has zero elements even before indexing." in w.args[0]: return False - raise + raise # pragma: no cover except IndexError: return False From 83ffdb18a1b2a04c8bfffc9024ec617b0bb8a7cc Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 11 Jan 2023 15:03:20 -0700 Subject: [PATCH 076/218] Fix a test failure with newer versions of NumPy --- ndindex/tests/test_ndindex.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ndindex/tests/test_ndindex.py b/ndindex/tests/test_ndindex.py index a58b0260..228e4e07 100644 --- a/ndindex/tests/test_ndindex.py +++ b/ndindex/tests/test_ndindex.py @@ -103,12 +103,13 @@ def test_ndindex_invalid(): np.array([])]: check_same(a, idx) - # This index is allowed by NumPy, but gives a deprecation warnings. We are - # not going to allow indices that give deprecation warnings in ndindex. + # Older versions of NumPy gives a deprecation warning for this index. We + # are not going to allow indices that give deprecation warnings in + # ndindex. with warnings.catch_warnings(record=True) as r: # Make sure no warnings are emitted from ndindex() warnings.simplefilter("error") - raises(IndexError, lambda: ndindex([1, []])) + raises((IndexError, ValueError), lambda: ndindex([1, []])) assert not r def test_ndindex_ellipsis(): From 45f1ee42e4985d1480541ab5dee7ff1af75b9ba1 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 31 Jan 2023 10:21:32 -0700 Subject: [PATCH 077/218] Don't delete the benchmarks from gh-pages on deploy --- .github/workflows/docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c16fa6ee..73ef4193 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -61,3 +61,4 @@ jobs: with: folder: docs/_build/html ssh-key: ${{ secrets.DEPLOY_KEY }} + clean-exclude: benchmarks/ From 953ade1b06266040fa3d13afe58a25e395905d4a Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 30 Mar 2023 13:55:44 -0600 Subject: [PATCH 078/218] Be more robust with testing against deprecated numpy functionality Previously we only relied on filterwarnings=error in pytest.ini to turn deprecation warnings into errors. We now also do this explicitly in the check_same() function. Also add pytest.ini and conftest.py to the MANIFEST.in so that they are included in the sdist. We still use filterwarnings=error in pytest.ini because we want to make sure we aren't using any other deprecated behavior anywhere else. Fixes #150. --- MANIFEST.in | 2 ++ ndindex/tests/helpers.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index bccc50cb..70fb470e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,5 @@ include versioneer.py include ndindex/_version.py include LICENSE +include pytest.ini +include conftest.py diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index d8d452fd..6f8409de 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -1,7 +1,8 @@ import sys from itertools import chain -from functools import reduce +from functools import reduce, wraps from operator import mul +import warnings from numpy import intp, bool_, array, broadcast_shapes import numpy.testing @@ -181,6 +182,15 @@ def assert_equal(actual, desired, err_msg='', verbose=True): assert actual.shape == desired.shape, err_msg or f"{actual.shape} != {desired.shape}" assert actual.dtype == desired.dtype, err_msg or f"{actual.dtype} != {desired.dtype}" +def warnings_are_errors(f): + @wraps(f) + def inner(*args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("error") + return f(*args, **kwargs) + return inner + +@warnings_are_errors def check_same(a, idx, raw_func=lambda a, idx: a[idx], ndindex_func=lambda a, index: a[index.raw], same_exception=True, assert_equal=assert_equal): From 77d6651eb83f8d0b1be438dd601993b308c663c5 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 30 Mar 2023 14:00:43 -0600 Subject: [PATCH 079/218] Make sure specific tests that catch DeprecationWarning use @warnings_are_errors --- ndindex/tests/test_as_subindex.py | 3 ++- ndindex/tests/test_broadcast_arrays.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ndindex/tests/test_as_subindex.py b/ndindex/tests/test_as_subindex.py index a664e0bb..f6a466a4 100644 --- a/ndindex/tests/test_as_subindex.py +++ b/ndindex/tests/test_as_subindex.py @@ -8,7 +8,7 @@ from ..ndindex import ndindex from ..integerarray import IntegerArray from ..tuple import Tuple -from .helpers import ndindices, short_shapes, assert_equal +from .helpers import ndindices, short_shapes, assert_equal, warnings_are_errors @example((slice(0, 8), slice(0, 9), slice(0, 10)), ([2, 5, 6, 7], slice(1, 9, 1), slice(5, 10, 1)), @@ -50,6 +50,7 @@ @example(0, (slice(None, 0, None), Ellipsis), 1) @example(0, (slice(1, 2),), 1) @given(ndindices, ndindices, one_of(integers(0, 100), short_shapes)) +@warnings_are_errors def test_as_subindex_hypothesis(idx1, idx2, shape): if isinstance(shape, int): a = arange(shape) diff --git a/ndindex/tests/test_broadcast_arrays.py b/ndindex/tests/test_broadcast_arrays.py index 64858fda..75780a93 100644 --- a/ndindex/tests/test_broadcast_arrays.py +++ b/ndindex/tests/test_broadcast_arrays.py @@ -9,7 +9,7 @@ from ..integerarray import IntegerArray from ..integer import Integer from ..tuple import Tuple -from .helpers import ndindices, check_same, short_shapes +from .helpers import ndindices, check_same, short_shapes, warnings_are_errors @example((..., False, False), 1) @example((True, False), 1) @@ -21,6 +21,7 @@ @example(False, 1) @example([[True, False], [False, False]], (2, 2, 3)) @given(ndindices, one_of(short_shapes, integers(0, 10))) +@warnings_are_errors def test_broadcast_arrays_hypothesis(idx, shape): if isinstance(shape, int): a = arange(shape) From 37c8fd7c5583fadf112cff645028eed5e777603b Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 30 Mar 2023 14:03:10 -0600 Subject: [PATCH 080/218] Add the numpy version to the tests header --- conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conftest.py b/conftest.py index 5b2966a3..0c89290e 100644 --- a/conftest.py +++ b/conftest.py @@ -13,6 +13,9 @@ if LooseVersion(numpy.__version__) < LooseVersion('1.20'): raise RuntimeError("NumPy 1.20 (development version) or greater is required to run the ndindex tests") +def pytest_report_header(config): + return f"project deps: numpy-{numpy.__version__}" + # Add a --hypothesis-max-examples flag to pytest. See # https://github.com/HypothesisWorks/hypothesis/issues/2434#issuecomment-630309150 From 7c130d9f78755d7deaf83dee5663152614dfcccc Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 30 Mar 2023 14:06:26 -0600 Subject: [PATCH 081/218] Include run_doctests.py in the sdist --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 70fb470e..caaeea04 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,4 @@ include ndindex/_version.py include LICENSE include pytest.ini include conftest.py +include run_doctests.py From 53d24d76879c9209e06012d9f63ec0ea2fc803f4 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 6 Apr 2023 02:28:12 -0600 Subject: [PATCH 082/218] Change skip_axes in iter_indices() to apply to the shapes before broadcasting This is the more standard behavior, e.g., for np.cross (which removes the size-3 axis dimension from the arrays before broadcasting them). This only really affects skip axes that are >= 0. It no longer requires >= 0 skip axes to be distinct from < 0 ones, since they might be unequal for different dimension shapes. This commit does not yet update test_iter_indices(), and may contain bugs. --- ndindex/ndindex.py | 106 ++++++++++++++++++++++++---------- ndindex/tests/test_ndindex.py | 38 +++++++++++- 2 files changed, 112 insertions(+), 32 deletions(-) diff --git a/ndindex/ndindex.py b/ndindex/ndindex.py index 0ea1fa9b..4f42acc5 100644 --- a/ndindex/ndindex.py +++ b/ndindex/ndindex.py @@ -4,6 +4,7 @@ import numbers import operator import functools +from collections import defaultdict newaxis = None @@ -696,20 +697,29 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): `skip_axes` should be a tuple of axes to skip. It can use negative integers, e.g., `skip_axes=(-1,)` will skip the last axis. The order of - the axes in `skip_axes` does not matter, but it should not contain - duplicate axes. The axes in `skip_axes` refer to the final broadcasted - shape of `shapes`. For example, `iter_indices((3,), (1, 2, 3), - skip_axes=(0,))` will skip the first axis, and only applies to the second - shape, since the first shape corresponds to axis `2` of the final - broadcasted shape `(1, 2, 3)`. Note that the skipped axes do not - themselves need to be broadcast compatible. + the axes in `skip_axes` does not matter. The axes in `skip_axes` refer to + the shapes *before* broadcasting (if you want to refer to the axes after + broadcasting, either broadcast the shapes and arrays first, or refer to + the axes using negative integers). For example, `iter_indices((10, 2), + (20, 1, 2), skip_axes=(0,))` will skip the size `10` axis of `(10, 2)` and + the size `20` axis of `(20, 1, 2)`. The result is two sets of indices, one + for each element of the non-skipped dimensions: + + >>> from ndindex import iter_indices + >>> for idx1, idx2 in iter_indices((10, 2), (20, 1, 2), skip_axes=(0,)): + ... print(idx1, idx2) + Tuple(slice(None, None, None), 0) Tuple(slice(None, None, None), 0, 0) + Tuple(slice(None, None, None), 1) Tuple(slice(None, None, None), 0, 1) + + The skipped axes do not themselves need to be broadcast compatible, but + the shapes with all the skipped axes removed should be broadcast + compatible. For example, suppose `a` is an array with shape `(3, 2, 4, 4)`, which we wish to think of as a `(3, 2)` stack of 4 x 4 matrices. We can generate an iterator for each matrix in the "stack" with `iter_indices((3, 2, 4, 4), skip_axes=(-1, -2))`: - >>> from ndindex import iter_indices >>> for idx in iter_indices((3, 2, 4, 4), skip_axes=(-1, -2)): ... print(idx) (Tuple(0, 0, slice(None, None, None), slice(None, None, None)),) @@ -719,9 +729,11 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): (Tuple(2, 0, slice(None, None, None), slice(None, None, None)),) (Tuple(2, 1, slice(None, None, None), slice(None, None, None)),) - Note that the iterates of `iter_indices` are always a tuple, even if only - a single shape is provided (one could instead use `for idx, in - iter_indices(...)` above). + .. note:: + + The iterates of `iter_indices` are always a tuple, even if only a + single shape is provided (one could instead use `for idx, in + iter_indices(...)` above). As another example, say `a` is shape `(1, 3)` and `b` is shape `(2, 1)`, and we want to generate indices for every value of the broadcasted @@ -771,37 +783,50 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): shapes = [asshape(shape) for shape in shapes] ndim = len(max(shapes, key=len)) + min_ndim = len(min(shapes, key=len)) if isinstance(skip_axes, int): skip_axes = (skip_axes,) - _skip_axes = [] - for a in skip_axes: - try: - a = ndindex(a).reduce(ndim).args[0] - except IndexError: - raise AxisError(a, ndim) - if a in _skip_axes: - raise ValueError("skip_axes should not contain duplicate axes") - _skip_axes.append(a) - - _shapes = [(1,)*(ndim - len(shape)) + shape for shape in shapes] - _shapes = [tuple(1 if i in _skip_axes else shape[i] for i in range(ndim)) - for shape in _shapes] + + n = len(skip_axes) + + if len(set(skip_axes)) != n: + raise ValueError("skip_axes should not contain duplicate axes") + + _skip_axes = defaultdict(list) + for shape in shapes: + for a in skip_axes: + try: + a = ndindex(a).reduce(len(shape)).raw + except IndexError: + raise AxisError(a, min_ndim) + _skip_axes[shape].append(a) + _skip_axes[shape].sort() + + _shapes = [remove_indices(shape, skip_axes) for shape in shapes] + # _shapes = [(1,)*(ndim - len(shape)) + shape for shape in shapes] + # _shapes = [tuple(1 if i in _skip_axes else shape[i] for i in range(ndim)) + # for shape in _shapes] iters = [[] for i in range(len(shapes))] broadcasted_shape = broadcast_shapes(*_shapes) + _broadcasted_shape = unremove_indices(broadcasted_shape, skip_axes, ndim) for i in range(-1, -ndim-1, -1): for it, shape, _shape in zip(iters, shapes, _shapes): if -i > len(shape): + # for every dimension prepended by broadcasting, repeat the + # indices that many times for j in range(len(it)): - if broadcasted_shape[i] not in [0, 1]: - it[j] = ncycles(it[j], broadcasted_shape[i]) + if broadcasted_shape[i+n] not in [0, 1]: + it[j] = ncycles(it[j], broadcasted_shape[i+n]) break - elif ndim + i in _skip_axes: + elif len(shape) + i in _skip_axes[shape]: it.insert(0, [slice(None)]) else: - if broadcasted_shape[i] != 1 and shape[i] == 1: - it.insert(0, ncycles(range(shape[i]), broadcasted_shape[i])) + if _broadcasted_shape[i] is None: + pass + elif _broadcasted_shape[i] != 1 and shape[i] == 1: + it.insert(0, ncycles(range(shape[i]), _broadcasted_shape[i])) else: it.insert(0, range(shape[i])) @@ -813,6 +838,29 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): iters], fillvalue=()): yield tuple(ndindex(idx) for idx in idxes) +def remove_indices(x, idxes): + """ + Return `x` with the indices `idxes` removed. + """ + dim = len(x) + _idxes = sorted({i if i >= 0 else i + dim for i in idxes}) + _idxes = [i - a for i, a in zip(_idxes, range(len(_idxes)))] + _x = list(x) + for i in _idxes: + _x.pop(i) + return tuple(_x) + +def unremove_indices(x, idxes, n, val=None): + """ + Insert `val` in `x` so that it appears at `idxes`, assuming the original + list had size `n` (reverse of `remove_indices`) + """ + x = list(x) + _idxes = sorted({i if i >= 0 else i + n for i in idxes}) + for i in _idxes: + x.insert(i, val) + return tuple(x) + class ncycles: """ Iterate `iterable` repeated `n` times. diff --git a/ndindex/tests/test_ndindex.py b/ndindex/tests/test_ndindex.py index 0345c4d1..6cd46da9 100644 --- a/ndindex/tests/test_ndindex.py +++ b/ndindex/tests/test_ndindex.py @@ -3,14 +3,15 @@ import numpy as np -from hypothesis import given, example, settings +from hypothesis import assume, given, example, settings from hypothesis.strategies import (one_of, integers, tuples as - hypothesis_tuples, just) + hypothesis_tuples, just, lists, shared) from pytest import raises from ..ndindex import (ndindex, asshape, iter_indices, ncycles, - BroadcastError, AxisError, broadcast_shapes) + BroadcastError, AxisError, broadcast_shapes, + remove_indices, unremove_indices) from ..booleanarray import BooleanArray from ..integer import Integer from ..ellipsis import ellipsis @@ -353,3 +354,34 @@ def test_broadcast_shapes(broadcastable_shapes): assert broadcast_shapes(*shapes) == broadcasted_shape else: raises(BroadcastError, lambda: broadcast_shapes(*shapes)) + +remove_indices_n = shared(integers(0, 100)) + +@given(remove_indices_n, + remove_indices_n.flatmap(lambda n: lists(integers(-n, n), unique=True))) +def test_remove_indices(n, idxes): + if idxes: + assume(max(idxes) < n) + assume(min(idxes) >= -n) + a = tuple(range(n)) + b = remove_indices(a, idxes) + + A = list(a) + for i in idxes: + A[i] = None + + assert set(A) - set(b) == ({None} if idxes else set()) + assert set(b) - set(A) == set() + + # Check the order is correct + j = 0 + for i in range(n): + val = A[i] + if val == None: + assert val not in b + else: + assert b[j] == val + j += 1 + + # Test that unremove_indices is the inverse + assert unremove_indices(b, idxes, n) == tuple(A) From af89e6d62158b541e4ec24378179666f6e4b1c5b Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 7 Apr 2023 17:32:18 -0600 Subject: [PATCH 083/218] Move iter_indices and related code to a separate file --- ndindex/__init__.py | 8 +- ndindex/iterators.py | 304 ++++++++++++++++++++++++++++++++ ndindex/ndindex.py | 303 ------------------------------- ndindex/tests/test_iterators.py | 243 +++++++++++++++++++++++++ ndindex/tests/test_ndindex.py | 239 +------------------------ 5 files changed, 557 insertions(+), 540 deletions(-) create mode 100644 ndindex/iterators.py create mode 100644 ndindex/tests/test_iterators.py diff --git a/ndindex/__init__.py b/ndindex/__init__.py index 8a5e60cf..12dc4886 100644 --- a/ndindex/__init__.py +++ b/ndindex/__init__.py @@ -1,8 +1,12 @@ __all__ = [] -from .ndindex import ndindex, iter_indices, AxisError, BroadcastError +from .ndindex import ndindex -__all__ += ['ndindex', 'iter_indices', 'AxisError', 'BroadcastError'] +__all__ += ['ndindex'] + +from .iterators import iter_indices, AxisError, BroadcastError + +__all__ += ['iter_indices', 'AxisError', 'BroadcastError'] from .slice import Slice diff --git a/ndindex/iterators.py b/ndindex/iterators.py new file mode 100644 index 00000000..c7feca03 --- /dev/null +++ b/ndindex/iterators.py @@ -0,0 +1,304 @@ +import itertools +import functools +from collections import defaultdict + +from .ndindex import asshape, ndindex + +# TODO: Use this in other places in the code that check broadcast compatibility. +class BroadcastError(ValueError): + """ + Exception raised by :func:`iter_indices()` when the input shapes are not + broadcast compatible. + + This is used instead of the NumPy exception of the same name so that + `iter_indices` does not need to depend on NumPy. + """ + +class AxisError(ValueError, IndexError): + """ + Exception raised by :func:`iter_indices()` when the `skip_axes` argument + is out of bounds. + + This is used instead of the NumPy exception of the same name so that + `iter_indices` does not need to depend on NumPy. + + """ + __slots__ = ("axis", "ndim") + + def __init__(self, axis, ndim): + self.axis = axis + self.ndim = ndim + + def __str__(self): + return f"axis {self.axis} is out of bounds for array of dimension {self.ndim}" + +def broadcast_shapes(*shapes): + """ + Broadcast the input shapes `shapes` to a single shape. + + This is the same as :py:func:`np.broadcast_shapes() + `. It is included as a separate helper function + because `np.broadcast_shapes()` is on available in NumPy 1.20 or newer, and + so that ndindex functions that use this function can do without requiring + NumPy to be installed. + + """ + + def _broadcast_shapes(shape1, shape2): + """Broadcasts `shape1` and `shape2`""" + N1 = len(shape1) + N2 = len(shape2) + N = max(N1, N2) + shape = [None for _ in range(N)] + i = N - 1 + while i >= 0: + n1 = N1 - N + i + if n1 >= 0: + d1 = shape1[n1] + else: + d1 = 1 + n2 = N2 - N + i + if n2 >= 0: + d2 = shape2[n2] + else: + d2 = 1 + + if d1 == 1: + shape[i] = d2 + elif d2 == 1: + shape[i] = d1 + elif d1 == d2: + shape[i] = d1 + else: + # TODO: Build an error message that matches NumPy + raise BroadcastError("shape mismatch: objects cannot be broadcast to a single shape.") + + i = i - 1 + + return tuple(shape) + + return functools.reduce(_broadcast_shapes, shapes, ()) + +def iter_indices(*shapes, skip_axes=(), _debug=False): + """ + Iterate indices for every element of an arrays of shape `shapes`. + + Each shape in `shapes` should be a shape tuple, which are broadcast + compatible along the non-skipped axes. Each iteration step will produce a + tuple of indices, one for each shape, which would correspond to the same + elements if the arrays of the given shapes were first broadcast together. + + This is a generalization of the NumPy :py:class:`np.ndindex() + ` function (which otherwise has no relation). + `np.ndindex()` only iterates indices for a single shape, whereas + `iter_indices()` supports generating indices for multiple broadcast + compatible shapes at once. This is equivalent to first broadcasting the + arrays then generating indices for the single broadcasted shape. + + Additionally, this function supports the ability to skip axes of the + shapes using `skip_axes`. These axes will be fully sliced in each index. + The remaining axes will be indexed one element at a time with integer + indices. + + `skip_axes` should be a tuple of axes to skip. It can use negative + integers, e.g., `skip_axes=(-1,)` will skip the last axis. The order of + the axes in `skip_axes` does not matter. The axes in `skip_axes` refer to + the shapes *before* broadcasting (if you want to refer to the axes after + broadcasting, either broadcast the shapes and arrays first, or refer to + the axes using negative integers). For example, `iter_indices((10, 2), + (20, 1, 2), skip_axes=(0,))` will skip the size `10` axis of `(10, 2)` and + the size `20` axis of `(20, 1, 2)`. The result is two sets of indices, one + for each element of the non-skipped dimensions: + + >>> from ndindex import iter_indices + >>> for idx1, idx2 in iter_indices((10, 2), (20, 1, 2), skip_axes=(0,)): + ... print(idx1, idx2) + Tuple(slice(None, None, None), 0) Tuple(slice(None, None, None), 0, 0) + Tuple(slice(None, None, None), 1) Tuple(slice(None, None, None), 0, 1) + + The skipped axes do not themselves need to be broadcast compatible, but + the shapes with all the skipped axes removed should be broadcast + compatible. + + For example, suppose `a` is an array with shape `(3, 2, 4, 4)`, which we + wish to think of as a `(3, 2)` stack of 4 x 4 matrices. We can generate an + iterator for each matrix in the "stack" with `iter_indices((3, 2, 4, 4), + skip_axes=(-1, -2))`: + + >>> for idx in iter_indices((3, 2, 4, 4), skip_axes=(-1, -2)): + ... print(idx) + (Tuple(0, 0, slice(None, None, None), slice(None, None, None)),) + (Tuple(0, 1, slice(None, None, None), slice(None, None, None)),) + (Tuple(1, 0, slice(None, None, None), slice(None, None, None)),) + (Tuple(1, 1, slice(None, None, None), slice(None, None, None)),) + (Tuple(2, 0, slice(None, None, None), slice(None, None, None)),) + (Tuple(2, 1, slice(None, None, None), slice(None, None, None)),) + + .. note:: + + The iterates of `iter_indices` are always a tuple, even if only a + single shape is provided (one could instead use `for idx, in + iter_indices(...)` above). + + As another example, say `a` is shape `(1, 3)` and `b` is shape `(2, 1)`, + and we want to generate indices for every value of the broadcasted + operation `a + b`. We can do this by using `a[idx1.raw] + b[idx2.raw]` for every + `idx1` and `idx2` as below: + + >>> import numpy as np + >>> a = np.arange(3).reshape((1, 3)) + >>> b = np.arange(100, 111, 10).reshape((2, 1)) + >>> a + array([[0, 1, 2]]) + >>> b + array([[100], + [110]]) + >>> for idx1, idx2 in iter_indices((1, 3), (2, 1)): # doctest: +SKIP37 + ... print(f"{idx1 = }; {idx2 = }; {(a[idx1.raw], b[idx2.raw]) = }") + idx1 = Tuple(0, 0); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (0, 100) + idx1 = Tuple(0, 1); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (1, 100) + idx1 = Tuple(0, 2); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (2, 100) + idx1 = Tuple(0, 0); idx2 = Tuple(1, 0); (a[idx1.raw], b[idx2.raw]) = (0, 110) + idx1 = Tuple(0, 1); idx2 = Tuple(1, 0); (a[idx1.raw], b[idx2.raw]) = (1, 110) + idx1 = Tuple(0, 2); idx2 = Tuple(1, 0); (a[idx1.raw], b[idx2.raw]) = (2, 110) + >>> a + b + array([[100, 101, 102], + [110, 111, 112]]) + + To include an index into the final broadcasted array, you can simply + include the final broadcasted shape as one of the shapes (the NumPy + function :func:`np.broadcast_shapes() ` is + useful here). + + >>> np.broadcast_shapes((1, 3), (2, 1)) + (2, 3) + >>> for idx1, idx2, broadcasted_idx in iter_indices((1, 3), (2, 1), (2, 3)): + ... print(broadcasted_idx) + Tuple(0, 0) + Tuple(0, 1) + Tuple(0, 2) + Tuple(1, 0) + Tuple(1, 1) + Tuple(1, 2) + + """ + if not shapes: + yield () + return + + shapes = [asshape(shape) for shape in shapes] + ndim = len(max(shapes, key=len)) + min_ndim = len(min(shapes, key=len)) + + if isinstance(skip_axes, int): + skip_axes = (skip_axes,) + + n = len(skip_axes) + + if len(set(skip_axes)) != n: + raise ValueError("skip_axes should not contain duplicate axes") + + _skip_axes = defaultdict(list) + for shape in shapes: + for a in skip_axes: + try: + a = ndindex(a).reduce(len(shape)).raw + except IndexError: + raise AxisError(a, min_ndim) + _skip_axes[shape].append(a) + _skip_axes[shape].sort() + + _shapes = [remove_indices(shape, skip_axes) for shape in shapes] + # _shapes = [(1,)*(ndim - len(shape)) + shape for shape in shapes] + # _shapes = [tuple(1 if i in _skip_axes else shape[i] for i in range(ndim)) + # for shape in _shapes] + iters = [[] for i in range(len(shapes))] + broadcasted_shape = broadcast_shapes(*_shapes) + _broadcasted_shape = unremove_indices(broadcasted_shape, skip_axes, ndim) + + for i in range(-1, -ndim-1, -1): + for it, shape, _shape in zip(iters, shapes, _shapes): + if -i > len(shape): + # for every dimension prepended by broadcasting, repeat the + # indices that many times + for j in range(len(it)): + if broadcasted_shape[i+n] not in [0, 1]: + it[j] = ncycles(it[j], broadcasted_shape[i+n]) + break + elif len(shape) + i in _skip_axes[shape]: + it.insert(0, [slice(None)]) + else: + if _broadcasted_shape[i] is None: + pass + elif _broadcasted_shape[i] != 1 and shape[i] == 1: + it.insert(0, ncycles(range(shape[i]), _broadcasted_shape[i])) + else: + it.insert(0, range(shape[i])) + + if _debug: # pragma: no cover + print(iters) + # Use this instead when we drop Python 3.7 support + # print(f"{iters = }") + for idxes in itertools.zip_longest(*[itertools.product(*i) for i in + iters], fillvalue=()): + yield tuple(ndindex(idx) for idx in idxes) + +def remove_indices(x, idxes): + """ + Return `x` with the indices `idxes` removed. + """ + dim = len(x) + _idxes = sorted({i if i >= 0 else i + dim for i in idxes}) + _idxes = [i - a for i, a in zip(_idxes, range(len(_idxes)))] + _x = list(x) + for i in _idxes: + _x.pop(i) + return tuple(_x) + +def unremove_indices(x, idxes, n, val=None): + """ + Insert `val` in `x` so that it appears at `idxes`, assuming the original + list had size `n` (reverse of `remove_indices`) + """ + x = list(x) + _idxes = sorted({i if i >= 0 else i + n for i in idxes}) + for i in _idxes: + x.insert(i, val) + return tuple(x) + +class ncycles: + """ + Iterate `iterable` repeated `n` times. + + This is based on a recipe from the `Python itertools docs + `_, + but improved to give a repr, and to denest when it can. This makes + debugging :func:`~.iter_indices` easier. + + >>> from ndindex.iterators import ncycles + >>> ncycles(range(3), 2) + ncycles(range(0, 3), 2) + >>> list(_) + [0, 1, 2, 0, 1, 2] + >>> ncycles(ncycles(range(3), 3), 2) + ncycles(range(0, 3), 6) + + """ + def __new__(cls, iterable, n): + if n == 1: + return iterable + return object.__new__(cls) + + def __init__(self, iterable, n): + if isinstance(iterable, ncycles): + self.iterable = iterable.iterable + self.n = iterable.n*n + else: + self.iterable = iterable + self.n = n + + def __repr__(self): + return f"ncycles({self.iterable!r}, {self.n!r})" + + def __iter__(self): + return itertools.chain.from_iterable(itertools.repeat(tuple(self.iterable), self.n)) diff --git a/ndindex/ndindex.py b/ndindex/ndindex.py index 4f42acc5..2ef18e29 100644 --- a/ndindex/ndindex.py +++ b/ndindex/ndindex.py @@ -1,10 +1,7 @@ import sys import inspect -import itertools import numbers import operator -import functools -from collections import defaultdict newaxis = None @@ -598,306 +595,6 @@ def broadcast_arrays(self): """ return self - -# TODO: Use this in other places in the code that check broadcast compatibility. -class BroadcastError(ValueError): - """ - Exception raised by :func:`iter_indices()` when the input shapes are not - broadcast compatible. - - This is used instead of the NumPy exception of the same name so that - `iter_indices` does not need to depend on NumPy. - """ - -class AxisError(ValueError, IndexError): - """ - Exception raised by :func:`iter_indices()` when the `skip_axes` argument - is out of bounds. - - This is used instead of the NumPy exception of the same name so that - `iter_indices` does not need to depend on NumPy. - - """ - __slots__ = ("axis", "ndim") - - def __init__(self, axis, ndim): - self.axis = axis - self.ndim = ndim - - def __str__(self): - return f"axis {self.axis} is out of bounds for array of dimension {self.ndim}" - -def broadcast_shapes(*shapes): - """ - Broadcast the input shapes `shapes` to a single shape. - - This is the same as :py:func:`np.broadcast_shapes() - `. It is included as a separate helper function - because `np.broadcast_shapes()` is on available in NumPy 1.20 or newer, and - so that ndindex functions that use this function can do without requiring - NumPy to be installed. - - """ - - def _broadcast_shapes(shape1, shape2): - """Broadcasts `shape1` and `shape2`""" - N1 = len(shape1) - N2 = len(shape2) - N = max(N1, N2) - shape = [None for _ in range(N)] - i = N - 1 - while i >= 0: - n1 = N1 - N + i - if n1 >= 0: - d1 = shape1[n1] - else: - d1 = 1 - n2 = N2 - N + i - if n2 >= 0: - d2 = shape2[n2] - else: - d2 = 1 - - if d1 == 1: - shape[i] = d2 - elif d2 == 1: - shape[i] = d1 - elif d1 == d2: - shape[i] = d1 - else: - # TODO: Build an error message that matches NumPy - raise BroadcastError("shape mismatch: objects cannot be broadcast to a single shape.") - - i = i - 1 - - return tuple(shape) - - return functools.reduce(_broadcast_shapes, shapes, ()) - -def iter_indices(*shapes, skip_axes=(), _debug=False): - """ - Iterate indices for every element of an arrays of shape `shapes`. - - Each shape in `shapes` should be a shape tuple, which are broadcast - compatible along the non-skipped axes. Each iteration step will produce a - tuple of indices, one for each shape, which would correspond to the same - elements if the arrays of the given shapes were first broadcast together. - - This is a generalization of the NumPy :py:class:`np.ndindex() - ` function (which otherwise has no relation). - `np.ndindex()` only iterates indices for a single shape, whereas - `iter_indices()` supports generating indices for multiple broadcast - compatible shapes at once. This is equivalent to first broadcasting the - arrays then generating indices for the single broadcasted shape. - - Additionally, this function supports the ability to skip axes of the - shapes using `skip_axes`. These axes will be fully sliced in each index. - The remaining axes will be indexed one element at a time with integer - indices. - - `skip_axes` should be a tuple of axes to skip. It can use negative - integers, e.g., `skip_axes=(-1,)` will skip the last axis. The order of - the axes in `skip_axes` does not matter. The axes in `skip_axes` refer to - the shapes *before* broadcasting (if you want to refer to the axes after - broadcasting, either broadcast the shapes and arrays first, or refer to - the axes using negative integers). For example, `iter_indices((10, 2), - (20, 1, 2), skip_axes=(0,))` will skip the size `10` axis of `(10, 2)` and - the size `20` axis of `(20, 1, 2)`. The result is two sets of indices, one - for each element of the non-skipped dimensions: - - >>> from ndindex import iter_indices - >>> for idx1, idx2 in iter_indices((10, 2), (20, 1, 2), skip_axes=(0,)): - ... print(idx1, idx2) - Tuple(slice(None, None, None), 0) Tuple(slice(None, None, None), 0, 0) - Tuple(slice(None, None, None), 1) Tuple(slice(None, None, None), 0, 1) - - The skipped axes do not themselves need to be broadcast compatible, but - the shapes with all the skipped axes removed should be broadcast - compatible. - - For example, suppose `a` is an array with shape `(3, 2, 4, 4)`, which we - wish to think of as a `(3, 2)` stack of 4 x 4 matrices. We can generate an - iterator for each matrix in the "stack" with `iter_indices((3, 2, 4, 4), - skip_axes=(-1, -2))`: - - >>> for idx in iter_indices((3, 2, 4, 4), skip_axes=(-1, -2)): - ... print(idx) - (Tuple(0, 0, slice(None, None, None), slice(None, None, None)),) - (Tuple(0, 1, slice(None, None, None), slice(None, None, None)),) - (Tuple(1, 0, slice(None, None, None), slice(None, None, None)),) - (Tuple(1, 1, slice(None, None, None), slice(None, None, None)),) - (Tuple(2, 0, slice(None, None, None), slice(None, None, None)),) - (Tuple(2, 1, slice(None, None, None), slice(None, None, None)),) - - .. note:: - - The iterates of `iter_indices` are always a tuple, even if only a - single shape is provided (one could instead use `for idx, in - iter_indices(...)` above). - - As another example, say `a` is shape `(1, 3)` and `b` is shape `(2, 1)`, - and we want to generate indices for every value of the broadcasted - operation `a + b`. We can do this by using `a[idx1.raw] + b[idx2.raw]` for every - `idx1` and `idx2` as below: - - >>> import numpy as np - >>> a = np.arange(3).reshape((1, 3)) - >>> b = np.arange(100, 111, 10).reshape((2, 1)) - >>> a - array([[0, 1, 2]]) - >>> b - array([[100], - [110]]) - >>> for idx1, idx2 in iter_indices((1, 3), (2, 1)): # doctest: +SKIP37 - ... print(f"{idx1 = }; {idx2 = }; {(a[idx1.raw], b[idx2.raw]) = }") - idx1 = Tuple(0, 0); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (0, 100) - idx1 = Tuple(0, 1); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (1, 100) - idx1 = Tuple(0, 2); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (2, 100) - idx1 = Tuple(0, 0); idx2 = Tuple(1, 0); (a[idx1.raw], b[idx2.raw]) = (0, 110) - idx1 = Tuple(0, 1); idx2 = Tuple(1, 0); (a[idx1.raw], b[idx2.raw]) = (1, 110) - idx1 = Tuple(0, 2); idx2 = Tuple(1, 0); (a[idx1.raw], b[idx2.raw]) = (2, 110) - >>> a + b - array([[100, 101, 102], - [110, 111, 112]]) - - To include an index into the final broadcasted array, you can simply - include the final broadcasted shape as one of the shapes (the NumPy - function :func:`np.broadcast_shapes() ` is - useful here). - - >>> np.broadcast_shapes((1, 3), (2, 1)) - (2, 3) - >>> for idx1, idx2, broadcasted_idx in iter_indices((1, 3), (2, 1), (2, 3)): - ... print(broadcasted_idx) - Tuple(0, 0) - Tuple(0, 1) - Tuple(0, 2) - Tuple(1, 0) - Tuple(1, 1) - Tuple(1, 2) - - """ - if not shapes: - yield () - return - - shapes = [asshape(shape) for shape in shapes] - ndim = len(max(shapes, key=len)) - min_ndim = len(min(shapes, key=len)) - - if isinstance(skip_axes, int): - skip_axes = (skip_axes,) - - n = len(skip_axes) - - if len(set(skip_axes)) != n: - raise ValueError("skip_axes should not contain duplicate axes") - - _skip_axes = defaultdict(list) - for shape in shapes: - for a in skip_axes: - try: - a = ndindex(a).reduce(len(shape)).raw - except IndexError: - raise AxisError(a, min_ndim) - _skip_axes[shape].append(a) - _skip_axes[shape].sort() - - _shapes = [remove_indices(shape, skip_axes) for shape in shapes] - # _shapes = [(1,)*(ndim - len(shape)) + shape for shape in shapes] - # _shapes = [tuple(1 if i in _skip_axes else shape[i] for i in range(ndim)) - # for shape in _shapes] - iters = [[] for i in range(len(shapes))] - broadcasted_shape = broadcast_shapes(*_shapes) - _broadcasted_shape = unremove_indices(broadcasted_shape, skip_axes, ndim) - - for i in range(-1, -ndim-1, -1): - for it, shape, _shape in zip(iters, shapes, _shapes): - if -i > len(shape): - # for every dimension prepended by broadcasting, repeat the - # indices that many times - for j in range(len(it)): - if broadcasted_shape[i+n] not in [0, 1]: - it[j] = ncycles(it[j], broadcasted_shape[i+n]) - break - elif len(shape) + i in _skip_axes[shape]: - it.insert(0, [slice(None)]) - else: - if _broadcasted_shape[i] is None: - pass - elif _broadcasted_shape[i] != 1 and shape[i] == 1: - it.insert(0, ncycles(range(shape[i]), _broadcasted_shape[i])) - else: - it.insert(0, range(shape[i])) - - if _debug: # pragma: no cover - print(iters) - # Use this instead when we drop Python 3.7 support - # print(f"{iters = }") - for idxes in itertools.zip_longest(*[itertools.product(*i) for i in - iters], fillvalue=()): - yield tuple(ndindex(idx) for idx in idxes) - -def remove_indices(x, idxes): - """ - Return `x` with the indices `idxes` removed. - """ - dim = len(x) - _idxes = sorted({i if i >= 0 else i + dim for i in idxes}) - _idxes = [i - a for i, a in zip(_idxes, range(len(_idxes)))] - _x = list(x) - for i in _idxes: - _x.pop(i) - return tuple(_x) - -def unremove_indices(x, idxes, n, val=None): - """ - Insert `val` in `x` so that it appears at `idxes`, assuming the original - list had size `n` (reverse of `remove_indices`) - """ - x = list(x) - _idxes = sorted({i if i >= 0 else i + n for i in idxes}) - for i in _idxes: - x.insert(i, val) - return tuple(x) - -class ncycles: - """ - Iterate `iterable` repeated `n` times. - - This is based on a recipe from the `Python itertools docs - `_, - but improved to give a repr, and to denest when it can. This makes - debugging :func:`~.iter_indices` easier. - - >>> from ndindex.ndindex import ncycles - >>> ncycles(range(3), 2) - ncycles(range(0, 3), 2) - >>> list(_) - [0, 1, 2, 0, 1, 2] - >>> ncycles(ncycles(range(3), 3), 2) - ncycles(range(0, 3), 6) - - """ - def __new__(cls, iterable, n): - if n == 1: - return iterable - return object.__new__(cls) - - def __init__(self, iterable, n): - if isinstance(iterable, ncycles): - self.iterable = iterable.iterable - self.n = iterable.n*n - else: - self.iterable = iterable - self.n = n - - def __repr__(self): - return f"ncycles({self.iterable!r}, {self.n!r})" - - def __iter__(self): - return itertools.chain.from_iterable(itertools.repeat(tuple(self.iterable), self.n)) - def asshape(shape, axis=None): """ Cast `shape` as a valid NumPy shape. diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py new file mode 100644 index 00000000..8697f12a --- /dev/null +++ b/ndindex/tests/test_iterators.py @@ -0,0 +1,243 @@ +import numpy as np + +from hypothesis import assume, given, example +from hypothesis.strategies import (one_of, integers, tuples as + hypothesis_tuples, just, lists, shared) + +from pytest import raises + +from ..ndindex import ndindex +from ..iterators import (iter_indices, ncycles, BroadcastError, + AxisError, broadcast_shapes, remove_indices, + unremove_indices) +from ..integer import Integer +from ..tuple import Tuple +from .helpers import (assert_equal, prod, + mutually_broadcastable_shapes_with_skipped_axes, + skip_axes, mutually_broadcastable_shapes, tuples, + shapes) + +@example([((1, 1), (1, 1)), (None, 1)], (0, 0)) +@example([((), (0,)), (None,)], (0,)) +@example([((1, 2), (2, 1)), (2, None)], 1) +@given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes) +def test_iter_indices(broadcastable_shapes, _skip_axes): + # broadcasted_shape will contain None on the skip_axes, as those axes + # might not be broadcast compatible + shapes, broadcasted_shape = broadcastable_shapes + + # 1. Normalize inputs + skip_axes = (_skip_axes,) if isinstance(_skip_axes, int) else () if _skip_axes is None else _skip_axes + ndim = len(broadcasted_shape) + + # Double check the mutually_broadcastable_shapes_with_skipped_axes + # strategy + for i in skip_axes: + assert broadcasted_shape[i] is None + + # Use negative indices to index the skip axes since only shapes that have + # the skip axis will include a slice. + normalized_skip_axes = sorted(ndindex(i).reduce(ndim).args[0] - ndim for i in skip_axes) + canonical_shapes = [list(s) for s in shapes] + for i in normalized_skip_axes: + for s in canonical_shapes: + if ndindex(i).isvalid(len(s)): + s[i] = 1 + skip_shapes = [tuple(shape[i] for i in normalized_skip_axes if ndindex(i).isvalid(len(shape))) for shape in canonical_shapes] + broadcasted_skip_shape = tuple(broadcasted_shape[i] for i in normalized_skip_axes) + + broadcasted_non_skip_shape = tuple(broadcasted_shape[i] for i in range(-1, -ndim-1, -1) if i not in normalized_skip_axes) + nitems = prod(broadcasted_non_skip_shape) + + if _skip_axes is None: + res = iter_indices(*shapes) + broadcasted_res = iter_indices(np.broadcast_shapes(*shapes)) + else: + # Skipped axes may not be broadcast compatible. Since the index for a + # skipped axis should always be a slice(None), the result should be + # the same if the skipped axes are all replaced with 1. + res = iter_indices(*shapes, skip_axes=_skip_axes) + broadcasted_res = iter_indices(np.broadcast_shapes(*canonical_shapes), + skip_axes=_skip_axes) + + sizes = [prod(shape) for shape in shapes] + arrays = [np.arange(size).reshape(shape) for size, shape in zip(sizes, shapes)] + canonical_sizes = [prod(shape) for shape in canonical_shapes] + canonical_arrays = [np.arange(size).reshape(shape) for size, shape in zip(canonical_sizes, canonical_shapes)] + broadcasted_arrays = np.broadcast_arrays(*canonical_arrays) + + # 2. Check that iter_indices is the same whether or not the shapes are + # broadcasted together first. Also check that every iterated index is the + # expected type and there are as many as expected. + vals = [] + n = -1 + try: + for n, (idxes, bidxes) in enumerate(zip(res, broadcasted_res)): + assert len(idxes) == len(shapes) + for idx, shape in zip(idxes, shapes): + assert isinstance(idx, Tuple) + assert len(idx.args) == len(shape) + for i in range(-1, -len(idx.args)-1, -1): + if i in normalized_skip_axes and len(idx.args) >= -i: + assert idx.args[i] == slice(None) + else: + assert isinstance(idx.args[i], Integer) + + aidxes = tuple([a[idx.raw] for a, idx in zip(arrays, idxes)]) + canonical_aidxes = tuple([a[idx.raw] for a, idx in zip(canonical_arrays, idxes)]) + a_broadcasted_idxs = [a[idx.raw] for a, idx in + zip(broadcasted_arrays, bidxes)] + + for aidx, abidx, skip_shape in zip(canonical_aidxes, a_broadcasted_idxs, skip_shapes): + if skip_shape == broadcasted_skip_shape: + assert_equal(aidx, abidx) + assert aidx.shape == skip_shape + + if skip_axes: + # If there are skipped axes, recursively call iter_indices to + # get each individual element of the resulting subarrays. + for subidxes in iter_indices(*[x.shape for x in canonical_aidxes]): + items = [x[i.raw] for x, i in zip(canonical_aidxes, subidxes)] + vals.append(tuple(items)) + else: + vals.append(aidxes) + except ValueError as e: + if "duplicate axes" in str(e): + # There should be actual duplicate axes + assert len({broadcasted_shape[i] for i in skip_axes}) < len(skip_axes) + return + raise # pragma: no cover + + assert len(set(vals)) == len(vals) == nitems + + # 3. Check that every element of the (broadcasted) arrays is represented + # by an iterated index. + + # The indices should correspond to the values that would be matched up + # if the arrays were broadcasted together. + if not arrays: + assert vals == [()] + else: + correct_vals = [tuple(i) for i in np.stack(broadcasted_arrays, axis=-1).reshape((nitems, len(arrays)))] + # Also test that the indices are produced in a lexicographic order + # (even though this isn't strictly guaranteed by the iter_indices + # docstring) in the case when there are no skip axes. The order when + # there are skip axes is more complicated because the skipped axes are + # iterated together. + if not skip_axes: + assert vals == correct_vals + else: + assert set(vals) == set(correct_vals) + + assert n == nitems - 1 + +def test_iter_indices_errors(): + try: + list(iter_indices((10,), skip_axes=(2,))) + except AxisError as e: + msg1 = str(e) + else: + raise RuntimeError("iter_indices did not raise AxisError") # pragma: no cover + + # Check that the message is the same one used by NumPy + try: + np.sum(np.arange(10), axis=2) + except np.AxisError as e: + msg2 = str(e) + else: + raise RuntimeError("np.sum() did not raise AxisError") # pragma: no cover + + assert msg1 == msg2 + + try: + list(iter_indices((2, 3), (3, 2))) + except BroadcastError as e: + msg1 = str(e) + else: + raise RuntimeError("iter_indices did not raise BroadcastError") # pragma: no cover + + # TODO: Check that the message is the same one used by NumPy + # try: + # np.broadcast_shapes((2, 3), (3, 2)) + # except np.Error as e: + # msg2 = str(e) + # else: + # raise RuntimeError("np.broadcast_shapes() did not raise AxisError") # pragma: no cover + # + # assert msg1 == msg2 + +@example(1, 1, 1) +@given(integers(0, 100), integers(0, 100), integers(0, 100)) +def test_ncycles(i, n, m): + N = ncycles(range(i), n) + if n == 1: + assert N == range(i) + else: + assert isinstance(N, ncycles) + assert N.iterable == range(i) + assert N.n == n + assert f"range(0, {i})" in repr(N) + assert str(n) in repr(N) + + L = list(N) + assert len(L) == i*n + for j in range(i*n): + assert L[j] == j % i + + M = ncycles(N, m) + if n*m == 1: + assert M == range(i) + else: + assert isinstance(M, ncycles) + assert M.iterable == range(i) + assert M.n == n*m + +@given(one_of(mutually_broadcastable_shapes, + hypothesis_tuples(tuples(shapes), just(None)))) +def test_broadcast_shapes(broadcastable_shapes): + shapes, broadcasted_shape = broadcastable_shapes + if broadcasted_shape is not None: + assert broadcast_shapes(*shapes) == broadcasted_shape + + arrays = [np.empty(shape) for shape in shapes] + broadcastable = True + try: + broadcasted_shape = np.broadcast(*arrays).shape + except ValueError: + broadcastable = False + + if broadcastable: + assert broadcast_shapes(*shapes) == broadcasted_shape + else: + raises(BroadcastError, lambda: broadcast_shapes(*shapes)) + +remove_indices_n = shared(integers(0, 100)) + +@given(remove_indices_n, + remove_indices_n.flatmap(lambda n: lists(integers(-n, n), unique=True))) +def test_remove_indices(n, idxes): + if idxes: + assume(max(idxes) < n) + assume(min(idxes) >= -n) + a = tuple(range(n)) + b = remove_indices(a, idxes) + + A = list(a) + for i in idxes: + A[i] = None + + assert set(A) - set(b) == ({None} if idxes else set()) + assert set(b) - set(A) == set() + + # Check the order is correct + j = 0 + for i in range(n): + val = A[i] + if val == None: + assert val not in b + else: + assert b[j] == val + j += 1 + + # Test that unremove_indices is the inverse + assert unremove_indices(b, idxes, n) == tuple(A) diff --git a/ndindex/tests/test_ndindex.py b/ndindex/tests/test_ndindex.py index 6cd46da9..bb2672f7 100644 --- a/ndindex/tests/test_ndindex.py +++ b/ndindex/tests/test_ndindex.py @@ -3,24 +3,18 @@ import numpy as np -from hypothesis import assume, given, example, settings -from hypothesis.strategies import (one_of, integers, tuples as - hypothesis_tuples, just, lists, shared) +from hypothesis import given, example, settings from pytest import raises -from ..ndindex import (ndindex, asshape, iter_indices, ncycles, - BroadcastError, AxisError, broadcast_shapes, - remove_indices, unremove_indices) +from ..ndindex import ndindex, asshape from ..booleanarray import BooleanArray from ..integer import Integer from ..ellipsis import ellipsis from ..integerarray import IntegerArray from ..tuple import Tuple -from .helpers import (ndindices, check_same, assert_equal, prod, - mutually_broadcastable_shapes_with_skipped_axes, - skip_axes, mutually_broadcastable_shapes, tuples, - shapes) +from .helpers import ndindices, check_same, assert_equal + @example([1, 2]) @given(ndindices) @@ -160,228 +154,3 @@ def test_asshape(): raises(TypeError, lambda: asshape(Integer(1))) raises(TypeError, lambda: asshape(Tuple(1, 2))) raises(TypeError, lambda: asshape((True,))) - -@example([((1, 1), (1, 1)), (None, 1)], (0, 0)) -@example([((), (0,)), (None,)], (0,)) -@example([((1, 2), (2, 1)), (2, None)], 1) -@given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes) -def test_iter_indices(broadcastable_shapes, _skip_axes): - # broadcasted_shape will contain None on the skip_axes, as those axes - # might not be broadcast compatible - shapes, broadcasted_shape = broadcastable_shapes - - # 1. Normalize inputs - skip_axes = (_skip_axes,) if isinstance(_skip_axes, int) else () if _skip_axes is None else _skip_axes - ndim = len(broadcasted_shape) - - # Double check the mutually_broadcastable_shapes_with_skipped_axes - # strategy - for i in skip_axes: - assert broadcasted_shape[i] is None - - # Use negative indices to index the skip axes since only shapes that have - # the skip axis will include a slice. - normalized_skip_axes = sorted(ndindex(i).reduce(ndim).args[0] - ndim for i in skip_axes) - canonical_shapes = [list(s) for s in shapes] - for i in normalized_skip_axes: - for s in canonical_shapes: - if ndindex(i).isvalid(len(s)): - s[i] = 1 - skip_shapes = [tuple(shape[i] for i in normalized_skip_axes if ndindex(i).isvalid(len(shape))) for shape in canonical_shapes] - broadcasted_skip_shape = tuple(broadcasted_shape[i] for i in normalized_skip_axes) - - broadcasted_non_skip_shape = tuple(broadcasted_shape[i] for i in range(-1, -ndim-1, -1) if i not in normalized_skip_axes) - nitems = prod(broadcasted_non_skip_shape) - - if _skip_axes is None: - res = iter_indices(*shapes) - broadcasted_res = iter_indices(np.broadcast_shapes(*shapes)) - else: - # Skipped axes may not be broadcast compatible. Since the index for a - # skipped axis should always be a slice(None), the result should be - # the same if the skipped axes are all replaced with 1. - res = iter_indices(*shapes, skip_axes=_skip_axes) - broadcasted_res = iter_indices(np.broadcast_shapes(*canonical_shapes), - skip_axes=_skip_axes) - - sizes = [prod(shape) for shape in shapes] - arrays = [np.arange(size).reshape(shape) for size, shape in zip(sizes, shapes)] - canonical_sizes = [prod(shape) for shape in canonical_shapes] - canonical_arrays = [np.arange(size).reshape(shape) for size, shape in zip(canonical_sizes, canonical_shapes)] - broadcasted_arrays = np.broadcast_arrays(*canonical_arrays) - - # 2. Check that iter_indices is the same whether or not the shapes are - # broadcasted together first. Also check that every iterated index is the - # expected type and there are as many as expected. - vals = [] - n = -1 - try: - for n, (idxes, bidxes) in enumerate(zip(res, broadcasted_res)): - assert len(idxes) == len(shapes) - for idx, shape in zip(idxes, shapes): - assert isinstance(idx, Tuple) - assert len(idx.args) == len(shape) - for i in range(-1, -len(idx.args)-1, -1): - if i in normalized_skip_axes and len(idx.args) >= -i: - assert idx.args[i] == slice(None) - else: - assert isinstance(idx.args[i], Integer) - - aidxes = tuple([a[idx.raw] for a, idx in zip(arrays, idxes)]) - canonical_aidxes = tuple([a[idx.raw] for a, idx in zip(canonical_arrays, idxes)]) - a_broadcasted_idxs = [a[idx.raw] for a, idx in - zip(broadcasted_arrays, bidxes)] - - for aidx, abidx, skip_shape in zip(canonical_aidxes, a_broadcasted_idxs, skip_shapes): - if skip_shape == broadcasted_skip_shape: - assert_equal(aidx, abidx) - assert aidx.shape == skip_shape - - if skip_axes: - # If there are skipped axes, recursively call iter_indices to - # get each individual element of the resulting subarrays. - for subidxes in iter_indices(*[x.shape for x in canonical_aidxes]): - items = [x[i.raw] for x, i in zip(canonical_aidxes, subidxes)] - vals.append(tuple(items)) - else: - vals.append(aidxes) - except ValueError as e: - if "duplicate axes" in str(e): - # There should be actual duplicate axes - assert len({broadcasted_shape[i] for i in skip_axes}) < len(skip_axes) - return - raise # pragma: no cover - - assert len(set(vals)) == len(vals) == nitems - - # 3. Check that every element of the (broadcasted) arrays is represented - # by an iterated index. - - # The indices should correspond to the values that would be matched up - # if the arrays were broadcasted together. - if not arrays: - assert vals == [()] - else: - correct_vals = [tuple(i) for i in np.stack(broadcasted_arrays, axis=-1).reshape((nitems, len(arrays)))] - # Also test that the indices are produced in a lexicographic order - # (even though this isn't strictly guaranteed by the iter_indices - # docstring) in the case when there are no skip axes. The order when - # there are skip axes is more complicated because the skipped axes are - # iterated together. - if not skip_axes: - assert vals == correct_vals - else: - assert set(vals) == set(correct_vals) - - assert n == nitems - 1 - -def test_iter_indices_errors(): - try: - list(iter_indices((10,), skip_axes=(2,))) - except AxisError as e: - msg1 = str(e) - else: - raise RuntimeError("iter_indices did not raise AxisError") # pragma: no cover - - # Check that the message is the same one used by NumPy - try: - np.sum(np.arange(10), axis=2) - except np.AxisError as e: - msg2 = str(e) - else: - raise RuntimeError("np.sum() did not raise AxisError") # pragma: no cover - - assert msg1 == msg2 - - try: - list(iter_indices((2, 3), (3, 2))) - except BroadcastError as e: - msg1 = str(e) - else: - raise RuntimeError("iter_indices did not raise BroadcastError") # pragma: no cover - - # TODO: Check that the message is the same one used by NumPy - # try: - # np.broadcast_shapes((2, 3), (3, 2)) - # except np.Error as e: - # msg2 = str(e) - # else: - # raise RuntimeError("np.broadcast_shapes() did not raise AxisError") # pragma: no cover - # - # assert msg1 == msg2 - -@example(1, 1, 1) -@given(integers(0, 100), integers(0, 100), integers(0, 100)) -def test_ncycles(i, n, m): - N = ncycles(range(i), n) - if n == 1: - assert N == range(i) - else: - assert isinstance(N, ncycles) - assert N.iterable == range(i) - assert N.n == n - assert f"range(0, {i})" in repr(N) - assert str(n) in repr(N) - - L = list(N) - assert len(L) == i*n - for j in range(i*n): - assert L[j] == j % i - - M = ncycles(N, m) - if n*m == 1: - assert M == range(i) - else: - assert isinstance(M, ncycles) - assert M.iterable == range(i) - assert M.n == n*m - -@given(one_of(mutually_broadcastable_shapes, - hypothesis_tuples(tuples(shapes), just(None)))) -def test_broadcast_shapes(broadcastable_shapes): - shapes, broadcasted_shape = broadcastable_shapes - if broadcasted_shape is not None: - assert broadcast_shapes(*shapes) == broadcasted_shape - - arrays = [np.empty(shape) for shape in shapes] - broadcastable = True - try: - broadcasted_shape = np.broadcast(*arrays).shape - except ValueError: - broadcastable = False - - if broadcastable: - assert broadcast_shapes(*shapes) == broadcasted_shape - else: - raises(BroadcastError, lambda: broadcast_shapes(*shapes)) - -remove_indices_n = shared(integers(0, 100)) - -@given(remove_indices_n, - remove_indices_n.flatmap(lambda n: lists(integers(-n, n), unique=True))) -def test_remove_indices(n, idxes): - if idxes: - assume(max(idxes) < n) - assume(min(idxes) >= -n) - a = tuple(range(n)) - b = remove_indices(a, idxes) - - A = list(a) - for i in idxes: - A[i] = None - - assert set(A) - set(b) == ({None} if idxes else set()) - assert set(b) - set(A) == set() - - # Check the order is correct - j = 0 - for i in range(n): - val = A[i] - if val == None: - assert val not in b - else: - assert b[j] == val - j += 1 - - # Test that unremove_indices is the inverse - assert unremove_indices(b, idxes, n) == tuple(A) From 15b1bfd62508b5581e470fd79fccf18a58a589a3 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 7 Apr 2023 18:02:45 -0600 Subject: [PATCH 084/218] Fix test failure with newer numpy versions --- ndindex/tests/test_ndindex.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ndindex/tests/test_ndindex.py b/ndindex/tests/test_ndindex.py index bb2672f7..706dd43a 100644 --- a/ndindex/tests/test_ndindex.py +++ b/ndindex/tests/test_ndindex.py @@ -107,7 +107,9 @@ def test_ndindex_invalid(): with warnings.catch_warnings(record=True) as r: # Make sure no warnings are emitted from ndindex() warnings.simplefilter("error") - raises(IndexError, lambda: ndindex([1, []])) + # Newer numpy versions raise ValueError with this index (although + # perhaps they shouldn't) + raises((IndexError, ValueError), lambda: ndindex([1, []])) assert not r def test_ndindex_ellipsis(): From 602a261af09c5f3470e38c33a45c1142596f29c2 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 10 Apr 2023 16:53:02 -0600 Subject: [PATCH 085/218] Some in-progress fixes to the iter_indices hypothesis strategies The strategies as written here still do not work properly. Also, rename the skip_axes strategy to skip_axes_st to avoid naming confusion. --- ndindex/tests/helpers.py | 58 ++++++++++++++++++--------------- ndindex/tests/test_iterators.py | 16 +++++++-- 2 files changed, 46 insertions(+), 28 deletions(-) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index 8cc4dd6e..e03d7cc3 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -16,6 +16,7 @@ mbs, BroadcastableShapes) from ..ndindex import ndindex +from ..iterators import remove_indices, unremove_indices # Hypothesis strategies for generating indices. Note that some of these # strategies are nominally already defined in hypothesis, but we redefine them @@ -114,11 +115,11 @@ def _mutually_broadcastable_shapes(draw): mutually_broadcastable_shapes = shared(_mutually_broadcastable_shapes()) @composite -def _skip_axes(draw): +def _skip_axes_st(draw): shapes, result_shape = draw(mutually_broadcastable_shapes) - n = len(result_shape) + N = len(min(shapes, key=len, default=())) axes = draw(one_of(none(), - lists(integers(-n, max(0, n-1)), max_size=n))) + lists(integers(-N, max(0, N-1)), unique=True))) if isinstance(axes, list): axes = tuple(axes) # Sometimes return an integer @@ -126,7 +127,7 @@ def _skip_axes(draw): return axes[0] return axes -skip_axes = shared(_skip_axes()) +skip_axes_st = shared(_skip_axes_st()) @composite def mutually_broadcastable_shapes_with_skipped_axes(draw): @@ -136,36 +137,41 @@ def mutually_broadcastable_shapes_with_skipped_axes(draw): The result_shape will be None in the position of skip_axes. """ - skip_axes_ = draw(skip_axes) + skip_axes_ = draw(skip_axes_st) shapes, result_shape = draw(mutually_broadcastable_shapes) - ndim = len(result_shape) if skip_axes_ is None: return shapes, result_shape if isinstance(skip_axes_, int): skip_axes_ = (skip_axes_,) - _shapes = [] + # Randomize the shape values in the skipped axes + shapes_ = [] for shape in shapes: - _shape = list(shape) - for i in skip_axes_: - # skip axes index the broadcasted shape, so are only valid on the - # individual shapes in negative form. TODO: Add this as a keyword - # to reduce(). - neg_i = ndindex(i).reduce(ndim).raw - ndim - assert ndindex(i).reduce(ndim) == ndindex(neg_i).reduce(ndim) - if ndindex(neg_i).isvalid(len(shape)) and draw(booleans()): - _shape[neg_i] = draw(integers(0)) - - _shapes.append(tuple(_shape)) - - _result_shape = list(result_shape) - for i in skip_axes_: - _result_shape[i] = None - _result_shape = tuple(_result_shape) - - for shape in _shapes: + n = len(shape) + len(skip_axes_) + _shape = unremove_indices(shape, skip_axes_, n) + # sanity check + assert remove_indices(_shape, skip_axes_) == shape, (_shape, skip_axes_, shape) + # Allow negative skip axes to sometimes be duplicated by positive + # ones (e.g., _shape = [0, None, 1], skip_axes = [1, -2] + for i in range(n): + _shape2 = unremove_indices(shape, skip_axes_, n-i) + if remove_indices(_shape2, skip_axes_) == shape and draw(booleans()): + _shape = _shape2 + + # Replace None values with random values + for j in range(len(shape)): + if shape[j] is None: + shape[j] = draw(integers(0)) + shapes_.append(tuple(_shape)) + # sanity check + assert remove_indices(_shape, skip_axes_) == tuple(shape), (_shape, skip_axes_, shape) + + result_shape_ = unremove_indices(result_shape, skip_axes_, len(result_shape) + len(skip_axes_)) + assert remove_indices(result_shape_, skip_axes_) == result_shape + + for shape in shapes_: assume(prod([i for i in shape if i]) < SHORT_MAX_ARRAY_SIZE) - return BroadcastableShapes(_shapes, _result_shape) + return BroadcastableShapes(shapes_, result_shape_) # We need to make sure shapes for boolean arrays are generated in a way that diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index 8697f12a..7ecca80f 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -14,13 +14,13 @@ from ..tuple import Tuple from .helpers import (assert_equal, prod, mutually_broadcastable_shapes_with_skipped_axes, - skip_axes, mutually_broadcastable_shapes, tuples, + skip_axes_st, mutually_broadcastable_shapes, tuples, shapes) @example([((1, 1), (1, 1)), (None, 1)], (0, 0)) @example([((), (0,)), (None,)], (0,)) @example([((1, 2), (2, 1)), (2, None)], 1) -@given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes) +@given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes_st) def test_iter_indices(broadcastable_shapes, _skip_axes): # broadcasted_shape will contain None on the skip_axes, as those axes # might not be broadcast compatible @@ -241,3 +241,15 @@ def test_remove_indices(n, idxes): # Test that unremove_indices is the inverse assert unremove_indices(b, idxes, n) == tuple(A) + +# Meta-test for the hypothesis strategy +@given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes_st) +def test_mutually_broadcastable_shapes_with_skipped_axes(broadcastable_shapes, + skip_axes): + shapes, broadcasted_shape = broadcastable_shapes + _skip_axes = (skip_axes,) if isinstance(skip_axes, int) else () if skip_axes is None else skip_axes + + _shapes = [remove_indices(shape, _skip_axes) for shape in shapes] + _broadcasted_shapes = remove_indices(broadcasted_shape, _skip_axes) + + assert broadcast_shapes(*_shapes) == _broadcasted_shapes From 40aac89a02d2b9ca0d8773b008e923f22345ad75 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 11 Apr 2023 01:50:06 -0600 Subject: [PATCH 086/218] Disallow skip_axes that mix positive and negative integers (for now) Given the updated logic where skip_axes apply before broadcasting, this is too difficult to handle correctly, due to the inherent ambiguity where a positive and negative index could refer to the same value. It's especially difficult to do test generation in this case, and I can't figure out how to do it. Given that this isn't really something that I need support for, I'm going to leave it unimplemented for now. --- ndindex/iterators.py | 69 +++++++++++++++++++++++++-------- ndindex/tests/helpers.py | 29 +++++++------- ndindex/tests/test_iterators.py | 6 ++- 3 files changed, 71 insertions(+), 33 deletions(-) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index c7feca03..ad27d809 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -101,14 +101,16 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): indices. `skip_axes` should be a tuple of axes to skip. It can use negative - integers, e.g., `skip_axes=(-1,)` will skip the last axis. The order of - the axes in `skip_axes` does not matter. The axes in `skip_axes` refer to - the shapes *before* broadcasting (if you want to refer to the axes after - broadcasting, either broadcast the shapes and arrays first, or refer to - the axes using negative integers). For example, `iter_indices((10, 2), - (20, 1, 2), skip_axes=(0,))` will skip the size `10` axis of `(10, 2)` and - the size `20` axis of `(20, 1, 2)`. The result is two sets of indices, one - for each element of the non-skipped dimensions: + integers, e.g., `skip_axes=(-1,)` will skip the last axis (but note that + mixing negative and nonnegative skip axes is currently not supported). The + order of the axes in `skip_axes` does not matter. The axes in `skip_axes` + refer to the shapes *before* broadcasting (if you want to refer to the + axes after broadcasting, either broadcast the shapes and arrays first, or + refer to the axes using negative integers). For example, + `iter_indices((10, 2), (20, 1, 2), skip_axes=(0,))` will skip the size + `10` axis of `(10, 2)` and the size `20` axis of `(20, 1, 2)`. The result + is two sets of indices, one for each element of the non-skipped + dimensions: >>> from ndindex import iter_indices >>> for idx1, idx2 in iter_indices((10, 2), (20, 1, 2), skip_axes=(0,)): @@ -182,7 +184,28 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): Tuple(1, 2) """ + if any(i >= 0 for i in skip_axes) and any(i < 0 for i in skip_axes): + # Mixing positive and negative skip_axes is too difficult to deal with + # (see the comment in unremove_indices). It's a bit of an unusual + # thing to support, at least in the general case, because a positive + # and negative index can index the same element, but only for shapes + # that are a specific size. So while, in principle something like + # iter_indices((2, 10, 20, 4), (2, 30, 4), skip_axes=(1, -2)) could + # make sense, it's a bit odd to do so. Of course, there's no reason we + # couldn't support cases like that, but they complicate the + # implementation and, especially, complicate the test generation in + # the hypothesis strategies. Given that I'm not completely sure how to + # implement it correctly, and I don't actually need support for it, + # I'm leaving it as not implemented for now. + raise NotImplementedError("Mixing both negative and nonnegative idxes is not yet supported") + + n = len(skip_axes) + if len(set(skip_axes)) != n: + raise ValueError("skip_axes should not contain duplicate axes") + if not shapes: + if skip_axes: + raise AxisError(skip_axes[0], 0) yield () return @@ -193,10 +216,6 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): if isinstance(skip_axes, int): skip_axes = (skip_axes,) - n = len(skip_axes) - - if len(set(skip_axes)) != n: - raise ValueError("skip_axes should not contain duplicate axes") _skip_axes = defaultdict(list) for shape in shapes: @@ -214,7 +233,7 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): # for shape in _shapes] iters = [[] for i in range(len(shapes))] broadcasted_shape = broadcast_shapes(*_shapes) - _broadcasted_shape = unremove_indices(broadcasted_shape, skip_axes, ndim) + _broadcasted_shape = unremove_indices(broadcasted_shape, skip_axes) for i in range(-1, -ndim-1, -1): for it, shape, _shape in zip(iters, shapes, _shapes): @@ -255,12 +274,30 @@ def remove_indices(x, idxes): _x.pop(i) return tuple(_x) -def unremove_indices(x, idxes, n, val=None): +def unremove_indices(x, idxes, *, val=None): """ - Insert `val` in `x` so that it appears at `idxes`, assuming the original - list had size `n` (reverse of `remove_indices`) + Insert `val` in `x` so that it appears at `idxes`. + + Note that idxes must be either all negative or all nonnegative """ + if any(i >= 0 for i in idxes) and any(i < 0 for i in idxes): + # A mix of positive and negative indices provides a fundamental + # problem. Sometimes, the result is not unique: for example, x = [0]; + # idxes = [1, -1] could be satisfied by both [0, None] or [0, None, + # None], depending on whether each index refers to a separate None or + # not (note that both cases are supported by remove_indices(), because + # there it is unambiguous). But even worse, in some cases, there may + # be no way to satisfy the given requirement. For example, given x = + # [0, 1, 2, 3]; idxes = [3, -3], there is no way to insert None into x + # so that remove_indices(res, idxes) == x. To see this, simply observe + # that there is no size list x such that remove_indices(x, [3, -3]) + # returns a tuple of size 4: + # + # >>> [len(remove_indices(list(range(n)), [3, -3])) for n in range(4, 10)] + # [2, 3, 5, 5, 6, 7] + raise NotImplementedError("Mixing both negative and nonnegative idxes is not yet supported") x = list(x) + n = len(idxes) + len(x) _idxes = sorted({i if i >= 0 else i + n for i in idxes}) for i in _idxes: x.insert(i, val) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index e03d7cc3..925110c1 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -117,13 +117,20 @@ def _mutually_broadcastable_shapes(draw): @composite def _skip_axes_st(draw): shapes, result_shape = draw(mutually_broadcastable_shapes) - N = len(min(shapes, key=len, default=())) - axes = draw(one_of(none(), - lists(integers(-N, max(0, N-1)), unique=True))) + if result_shape == (): + return () + negative = draw(booleans(), label='skip_axes < 0') + N = len(min(shapes, key=len)) + if N == 0: + return () + if negative: + axes = draw(one_of(none(), lists(integers(-N, -1), unique=True))) + else: + axes = draw(one_of(none(), lists(integers(0, N-1), unique=True))) if isinstance(axes, list): axes = tuple(axes) # Sometimes return an integer - if len(axes) == 1 and draw(booleans()): # pragma: no cover + if len(axes) == 1 and draw(booleans(), label='skip_axes integer'): # pragma: no cover return axes[0] return axes @@ -147,26 +154,18 @@ def mutually_broadcastable_shapes_with_skipped_axes(draw): # Randomize the shape values in the skipped axes shapes_ = [] for shape in shapes: - n = len(shape) + len(skip_axes_) - _shape = unremove_indices(shape, skip_axes_, n) + _shape = unremove_indices(shape, skip_axes_) # sanity check assert remove_indices(_shape, skip_axes_) == shape, (_shape, skip_axes_, shape) - # Allow negative skip axes to sometimes be duplicated by positive - # ones (e.g., _shape = [0, None, 1], skip_axes = [1, -2] - for i in range(n): - _shape2 = unremove_indices(shape, skip_axes_, n-i) - if remove_indices(_shape2, skip_axes_) == shape and draw(booleans()): - _shape = _shape2 # Replace None values with random values for j in range(len(shape)): if shape[j] is None: shape[j] = draw(integers(0)) shapes_.append(tuple(_shape)) - # sanity check - assert remove_indices(_shape, skip_axes_) == tuple(shape), (_shape, skip_axes_, shape) - result_shape_ = unremove_indices(result_shape, skip_axes_, len(result_shape) + len(skip_axes_)) + result_shape_ = unremove_indices(result_shape, skip_axes_) + # sanity check assert remove_indices(result_shape_, skip_axes_) == result_shape for shape in shapes_: diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index 7ecca80f..b60e634d 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -240,8 +240,10 @@ def test_remove_indices(n, idxes): j += 1 # Test that unremove_indices is the inverse - assert unremove_indices(b, idxes, n) == tuple(A) - + if all(i >= 0 for i in idxes) or all(i < 0 for i in idxes): + assert unremove_indices(b, idxes) == tuple(A) + else: + raises(NotImplementedError, lambda: unremove_indices(b, idxes)) # Meta-test for the hypothesis strategy @given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes_st) def test_mutually_broadcastable_shapes_with_skipped_axes(broadcastable_shapes, From e40439cddff163f5416466c347f6eeae309d36dc Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 11 Apr 2023 23:47:18 -0600 Subject: [PATCH 087/218] Add a test for iter_indices NotImplementedError --- ndindex/tests/test_iterators.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index b60e634d..f7eae6c7 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -156,6 +156,7 @@ def test_iter_indices_errors(): else: raise RuntimeError("iter_indices did not raise BroadcastError") # pragma: no cover + raises(NotImplementedError, lambda: list(iter_indices((1, 2), skip_axes=(0, -1)))) # TODO: Check that the message is the same one used by NumPy # try: # np.broadcast_shapes((2, 3), (3, 2)) @@ -244,6 +245,7 @@ def test_remove_indices(n, idxes): assert unremove_indices(b, idxes) == tuple(A) else: raises(NotImplementedError, lambda: unremove_indices(b, idxes)) + # Meta-test for the hypothesis strategy @given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes_st) def test_mutually_broadcastable_shapes_with_skipped_axes(broadcastable_shapes, From 0036c399740d072b57d5de6f199bf84b290328cc Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 12 Apr 2023 02:55:20 -0600 Subject: [PATCH 088/218] Allow skip_axes to be an int in iter_indices --- ndindex/iterators.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index ad27d809..26b3e0a9 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -184,6 +184,9 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): Tuple(1, 2) """ + if isinstance(skip_axes, int): + skip_axes = (skip_axes,) + if any(i >= 0 for i in skip_axes) and any(i < 0 for i in skip_axes): # Mixing positive and negative skip_axes is too difficult to deal with # (see the comment in unremove_indices). It's a bit of an unusual From a7cfb25781c36c4352a00d0e7831d0e2d24b000e Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 12 Apr 2023 02:55:35 -0600 Subject: [PATCH 089/218] Fix an error message --- ndindex/iterators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index 26b3e0a9..c3444c86 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -200,7 +200,7 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): # the hypothesis strategies. Given that I'm not completely sure how to # implement it correctly, and I don't actually need support for it, # I'm leaving it as not implemented for now. - raise NotImplementedError("Mixing both negative and nonnegative idxes is not yet supported") + raise NotImplementedError("Mixing both negative and nonnegative skip_axes is not yet supported") n = len(skip_axes) if len(set(skip_axes)) != n: From 9c496f220979df981a3674e67389125af91d622d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 12 Apr 2023 02:55:43 -0600 Subject: [PATCH 090/218] Allow idxes to be an int in remove_indices --- ndindex/iterators.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index c3444c86..656c63a6 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -269,6 +269,8 @@ def remove_indices(x, idxes): """ Return `x` with the indices `idxes` removed. """ + if isinstance(idxes, int): + idxes = (idxes,) dim = len(x) _idxes = sorted({i if i >= 0 else i + dim for i in idxes}) _idxes = [i - a for i, a in zip(_idxes, range(len(_idxes)))] From 5c11b6665e33069e947c1f09b0f438db17fc9903 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 12 Apr 2023 02:56:30 -0600 Subject: [PATCH 091/218] Don't let skip_axes be None in the hypothesis generator --- ndindex/tests/helpers.py | 13 ++++++------- ndindex/tests/test_iterators.py | 14 ++++++++++---- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index 925110c1..e10b91a2 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -124,14 +124,13 @@ def _skip_axes_st(draw): if N == 0: return () if negative: - axes = draw(one_of(none(), lists(integers(-N, -1), unique=True))) + axes = draw(one_of(lists(integers(-N, -1), unique=True))) else: - axes = draw(one_of(none(), lists(integers(0, N-1), unique=True))) - if isinstance(axes, list): - axes = tuple(axes) - # Sometimes return an integer - if len(axes) == 1 and draw(booleans(), label='skip_axes integer'): # pragma: no cover - return axes[0] + axes = draw(one_of(lists(integers(0, N-1), unique=True))) + axes = tuple(axes) + # Sometimes return an integer + if len(axes) == 1 and draw(booleans(), label='skip_axes integer'): # pragma: no cover + return axes[0] return axes skip_axes_st = shared(_skip_axes_st()) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index f7eae6c7..dca42978 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -251,9 +251,15 @@ def test_remove_indices(n, idxes): def test_mutually_broadcastable_shapes_with_skipped_axes(broadcastable_shapes, skip_axes): shapes, broadcasted_shape = broadcastable_shapes - _skip_axes = (skip_axes,) if isinstance(skip_axes, int) else () if skip_axes is None else skip_axes + _skip_axes = (skip_axes,) if isinstance(skip_axes, int) else () - _shapes = [remove_indices(shape, _skip_axes) for shape in shapes] - _broadcasted_shapes = remove_indices(broadcasted_shape, _skip_axes) + for shape in shapes: + assert None not in shape + for i in _skip_axes: + assert broadcasted_shape[i] is None + + _shapes = [remove_indices(shape, skip_axes) for shape in shapes] + _broadcasted_shape = remove_indices(broadcasted_shape, skip_axes) - assert broadcast_shapes(*_shapes) == _broadcasted_shapes + assert None not in _broadcasted_shape + assert broadcast_shapes(*_shapes) == _broadcasted_shape From 780af0b1848109dd878c06ea2a7d65ca2c904dde Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 12 Apr 2023 02:56:57 -0600 Subject: [PATCH 092/218] Add a normal test for duplicate skip axes in iter_indices --- ndindex/tests/test_iterators.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index dca42978..80f85782 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -156,7 +156,6 @@ def test_iter_indices_errors(): else: raise RuntimeError("iter_indices did not raise BroadcastError") # pragma: no cover - raises(NotImplementedError, lambda: list(iter_indices((1, 2), skip_axes=(0, -1)))) # TODO: Check that the message is the same one used by NumPy # try: # np.broadcast_shapes((2, 3), (3, 2)) @@ -167,6 +166,11 @@ def test_iter_indices_errors(): # # assert msg1 == msg2 + raises(NotImplementedError, lambda: list(iter_indices((1, 2), skip_axes=(0, -1)))) + + with raises(ValueError, match=r"duplicate axes"): + list(iter_indices((1, 2), skip_axes=(0, 1, 0))) + @example(1, 1, 1) @given(integers(0, 100), integers(0, 100), integers(0, 100)) def test_ncycles(i, n, m): From dae15a6f83b1645afbf4d7c2224737c00398d490 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 12 Apr 2023 02:57:21 -0600 Subject: [PATCH 093/218] Fix the mutually_broadcastable_shapes_with_skipped_axes strategy --- ndindex/tests/helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index e10b91a2..b7d210f2 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -153,14 +153,14 @@ def mutually_broadcastable_shapes_with_skipped_axes(draw): # Randomize the shape values in the skipped axes shapes_ = [] for shape in shapes: - _shape = unremove_indices(shape, skip_axes_) + _shape = list(unremove_indices(shape, skip_axes_)) # sanity check assert remove_indices(_shape, skip_axes_) == shape, (_shape, skip_axes_, shape) # Replace None values with random values - for j in range(len(shape)): - if shape[j] is None: - shape[j] = draw(integers(0)) + for j in range(len(_shape)): + if _shape[j] is None: + _shape[j] = draw(integers(0)) shapes_.append(tuple(_shape)) result_shape_ = unremove_indices(result_shape, skip_axes_) From 4980b2c9b2a87e3a25420caa6b9810f3abde39f0 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 12 Apr 2023 02:57:44 -0600 Subject: [PATCH 094/218] Work in progress making broadcast_shapes accept skip_axes --- ndindex/iterators.py | 29 +++++++++++++++++++++++++---- ndindex/tests/test_iterators.py | 5 +++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index 656c63a6..e33e635b 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -32,7 +32,7 @@ def __init__(self, axis, ndim): def __str__(self): return f"axis {self.axis} is out of bounds for array of dimension {self.ndim}" -def broadcast_shapes(*shapes): +def broadcast_shapes(*shapes, skip_axes=()): """ Broadcast the input shapes `shapes` to a single shape. @@ -43,27 +43,41 @@ def broadcast_shapes(*shapes): NumPy to be installed. """ + if isinstance(skip_axes, int): + skip_axes = (skip_axes,) + + if any(i >= 0 for i in skip_axes) and any(i < 0 for i in skip_axes): + # See the comments in remove_indices and iter_indices + raise NotImplementedError("Mixing both negative and nonnegative skip_axes is not yet supported") def _broadcast_shapes(shape1, shape2): """Broadcasts `shape1` and `shape2`""" N1 = len(shape1) N2 = len(shape2) + skip_axes1 = [ndindex(i).reduce(N1).raw - N1 for i in skip_axes] + skip_axes2 = [ndindex(i).reduce(N2).raw - N2 for i in skip_axes] N = max(N1, N2) shape = [None for _ in range(N)] i = N - 1 while i >= 0: n1 = N1 - N + i - if n1 >= 0: + if i in skip_axes1: + d1 = None + elif n1 >= 0: d1 = shape1[n1] else: d1 = 1 n2 = N2 - N + i - if n2 >= 0: + if i in skip_axes2: + d2 = None + elif n2 >= 0: d2 = shape2[n2] else: d2 = 1 - if d1 == 1: + if d1 == None or d2 == None: + shape[i] = None + elif d1 == 1: shape[i] = d2 elif d2 == 1: shape[i] = d1 @@ -77,6 +91,13 @@ def _broadcast_shapes(shape1, shape2): return tuple(shape) + if len(shapes) == 1: + shape = shapes[0] + N = len(shape) + # Check that skip_axes are valid + [ndindex(i).reduce(N).raw - N for i in skip_axes] + return shape + return functools.reduce(_broadcast_shapes, shapes, ()) def iter_indices(*shapes, skip_axes=(), _debug=False): diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index 80f85782..822da64d 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -216,6 +216,11 @@ def test_broadcast_shapes(broadcastable_shapes): else: raises(BroadcastError, lambda: broadcast_shapes(*shapes)) +@given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes_st) +def test_broadcast_shapes_skip_axes(broadcastable_shapes, skip_axes): + shapes, broadcasted_shape = broadcastable_shapes + assert broadcast_shapes(*shapes, skip_axes=skip_axes) == broadcasted_shape + remove_indices_n = shared(integers(0, 100)) @given(remove_indices_n, From f88a5f53c0f21bf4453c16f1f8375fe7e38d1b25 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 12 Apr 2023 02:57:56 -0600 Subject: [PATCH 095/218] Work in progress updating test_iter_indices --- ndindex/tests/test_iterators.py | 117 +++++++++++++++++--------------- 1 file changed, 61 insertions(+), 56 deletions(-) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index 822da64d..87201008 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -17,17 +17,17 @@ skip_axes_st, mutually_broadcastable_shapes, tuples, shapes) -@example([((1, 1), (1, 1)), (None, 1)], (0, 0)) -@example([((), (0,)), (None,)], (0,)) +@example([((1, 1), (1, 1)), (None, 1)], (0,)) +@example([((0,), (0,)), (None,)], (0,)) @example([((1, 2), (2, 1)), (2, None)], 1) @given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes_st) -def test_iter_indices(broadcastable_shapes, _skip_axes): +def test_iter_indices(broadcastable_shapes, skip_axes): # broadcasted_shape will contain None on the skip_axes, as those axes # might not be broadcast compatible shapes, broadcasted_shape = broadcastable_shapes # 1. Normalize inputs - skip_axes = (_skip_axes,) if isinstance(_skip_axes, int) else () if _skip_axes is None else _skip_axes + _skip_axes = (skip_axes,) if isinstance(skip_axes, int) else () ndim = len(broadcasted_shape) # Double check the mutually_broadcastable_shapes_with_skipped_axes @@ -35,79 +35,84 @@ def test_iter_indices(broadcastable_shapes, _skip_axes): for i in skip_axes: assert broadcasted_shape[i] is None - # Use negative indices to index the skip axes since only shapes that have - # the skip axis will include a slice. - normalized_skip_axes = sorted(ndindex(i).reduce(ndim).args[0] - ndim for i in skip_axes) + + # Skipped axes may not be broadcast compatible. Since the index for a + # skipped axis should always be a slice(None), the result should be the + # same if the skipped axes are all moved to the end of the shape. canonical_shapes = [list(s) for s in shapes] - for i in normalized_skip_axes: + for i in skip_axes: for s in canonical_shapes: - if ndindex(i).isvalid(len(s)): - s[i] = 1 - skip_shapes = [tuple(shape[i] for i in normalized_skip_axes if ndindex(i).isvalid(len(shape))) for shape in canonical_shapes] - broadcasted_skip_shape = tuple(broadcasted_shape[i] for i in normalized_skip_axes) + x = s.pop(i) + s.append(x) + + skip_shapes = [tuple(shape[i] for i in _skip_axes) for shape in shapes] + non_skip_shapes = [remove_indices(shape, skip_axes) for shape in shapes] + broadcasted_non_skip_shape = remove_indices(broadcasted_shape, skip_axes) + assert None not in broadcasted_non_skip_shape + assert broadcast_shapes(*non_skip_shapes) == broadcasted_non_skip_shape - broadcasted_non_skip_shape = tuple(broadcasted_shape[i] for i in range(-1, -ndim-1, -1) if i not in normalized_skip_axes) nitems = prod(broadcasted_non_skip_shape) if _skip_axes is None: res = iter_indices(*shapes) broadcasted_res = iter_indices(np.broadcast_shapes(*shapes)) else: - # Skipped axes may not be broadcast compatible. Since the index for a - # skipped axis should always be a slice(None), the result should be - # the same if the skipped axes are all replaced with 1. - res = iter_indices(*shapes, skip_axes=_skip_axes) + res = iter_indices(*shapes, skip_axes=skip_axes) broadcasted_res = iter_indices(np.broadcast_shapes(*canonical_shapes), - skip_axes=_skip_axes) + skip_axes=skip_axes) sizes = [prod(shape) for shape in shapes] arrays = [np.arange(size).reshape(shape) for size, shape in zip(sizes, shapes)] canonical_sizes = [prod(shape) for shape in canonical_shapes] canonical_arrays = [np.arange(size).reshape(shape) for size, shape in zip(canonical_sizes, canonical_shapes)] - broadcasted_arrays = np.broadcast_arrays(*canonical_arrays) # 2. Check that iter_indices is the same whether or not the shapes are # broadcasted together first. Also check that every iterated index is the # expected type and there are as many as expected. vals = [] n = -1 - try: - for n, (idxes, bidxes) in enumerate(zip(res, broadcasted_res)): - assert len(idxes) == len(shapes) - for idx, shape in zip(idxes, shapes): - assert isinstance(idx, Tuple) - assert len(idx.args) == len(shape) - for i in range(-1, -len(idx.args)-1, -1): - if i in normalized_skip_axes and len(idx.args) >= -i: - assert idx.args[i] == slice(None) - else: - assert isinstance(idx.args[i], Integer) - - aidxes = tuple([a[idx.raw] for a, idx in zip(arrays, idxes)]) - canonical_aidxes = tuple([a[idx.raw] for a, idx in zip(canonical_arrays, idxes)]) - a_broadcasted_idxs = [a[idx.raw] for a, idx in - zip(broadcasted_arrays, bidxes)] - - for aidx, abidx, skip_shape in zip(canonical_aidxes, a_broadcasted_idxs, skip_shapes): - if skip_shape == broadcasted_skip_shape: - assert_equal(aidx, abidx) - assert aidx.shape == skip_shape - - if skip_axes: - # If there are skipped axes, recursively call iter_indices to - # get each individual element of the resulting subarrays. - for subidxes in iter_indices(*[x.shape for x in canonical_aidxes]): - items = [x[i.raw] for x, i in zip(canonical_aidxes, subidxes)] - vals.append(tuple(items)) - else: - vals.append(aidxes) - except ValueError as e: - if "duplicate axes" in str(e): - # There should be actual duplicate axes - assert len({broadcasted_shape[i] for i in skip_axes}) < len(skip_axes) - return - raise # pragma: no cover + def _move_slices_to_end(idx): + assert isinstance(idx, Tuple) + idx2 = list(idx.args) + for i in range(len(idx2)): + if idx.args[i] == slice(None): + idx2.pop(i) + idx2.append(slice(None)) + return Tuple(*idx2) + + for n, (idxes, bidxes) in enumerate(zip(res, broadcasted_res)): + assert len(idxes) == len(shapes) + assert len(bidxes) == 1 + for idx, shape in zip(idxes, shapes): + assert isinstance(idx, Tuple) + assert len(idx.args) == len(shape) + + normalized_skip_axes = sorted(ndindex(i).reduce(len(shape)).raw for i in _skip_axes) + for i in range(len(idx.args)): + if i in normalized_skip_axes: + assert idx.args[i] == slice(None) + else: + assert isinstance(idx.args[i], Integer) + + canonical_idxes = [_move_slices_to_end(idx) for idx in idxes] + a_indexed = tuple([a[idx.raw] for a, idx in zip(arrays, idxes)]) + canonical_a_indexed = tuple([a[idx.raw] for a, idx in + zip(canonical_arrays, canonical_idxes)]) + + for a_indexed, skip_shape in zip(canonical_a_indexed, skip_shapes): + assert a_indexed.shape == skip_shape + + # if skip_axes: + # # If there are skipped axes, recursively call iter_indices to + # # get each individual element of the resulting subarrays. + # for subidxes in iter_indices(*[x.shape for x in canonical_a_indexed]): + # items = [x[i.raw] for x, i in zip(canonical_a_indexed, subidxes)] + # vals.append(tuple(items)) + # else: + # vals.append(a_indexed) + + return assert len(set(vals)) == len(vals) == nitems # 3. Check that every element of the (broadcasted) arrays is represented @@ -124,7 +129,7 @@ def test_iter_indices(broadcastable_shapes, _skip_axes): # docstring) in the case when there are no skip axes. The order when # there are skip axes is more complicated because the skipped axes are # iterated together. - if not skip_axes: + if not _skip_axes: assert vals == correct_vals else: assert set(vals) == set(correct_vals) From 52380579d7abc8f4383badfab72ee911ad4e0bf3 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 12 Apr 2023 17:19:53 -0600 Subject: [PATCH 096/218] Add skip_axes support to broadcast_shapes and make it a public function --- ndindex/__init__.py | 4 +- ndindex/iterators.py | 121 ++++++++++++++++++-------------- ndindex/tests/test_iterators.py | 68 ++++++++++++++++++ 3 files changed, 139 insertions(+), 54 deletions(-) diff --git a/ndindex/__init__.py b/ndindex/__init__.py index 12dc4886..d39ac3c4 100644 --- a/ndindex/__init__.py +++ b/ndindex/__init__.py @@ -4,9 +4,9 @@ __all__ += ['ndindex'] -from .iterators import iter_indices, AxisError, BroadcastError +from .iterators import broadcast_shapes, iter_indices, AxisError, BroadcastError -__all__ += ['iter_indices', 'AxisError', 'BroadcastError'] +__all__ += ['broadcast_shapes', 'iter_indices', 'AxisError', 'BroadcastError'] from .slice import Slice diff --git a/ndindex/iterators.py b/ndindex/iterators.py index e33e635b..480f4ac2 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -5,6 +5,7 @@ from .ndindex import asshape, ndindex # TODO: Use this in other places in the code that check broadcast compatibility. + class BroadcastError(ValueError): """ Exception raised by :func:`iter_indices()` when the input shapes are not @@ -13,6 +14,17 @@ class BroadcastError(ValueError): This is used instead of the NumPy exception of the same name so that `iter_indices` does not need to depend on NumPy. """ + __slots__ = ("arg1", "shape1", "arg2", "shape2") + + def __init__(self, arg1, shape1, arg2, shape2): + self.arg1 = arg1 + self.shape1 = shape1 + self.arg2 = arg2 + self.shape2 = shape2 + + def __str__(self): + arg1, shape1, arg2, shape2 = self.args + return f"shape mismatch: objects cannot be broadcast to a single shape. Mismatch is between arg {arg1} with shape {shape1} and arg {arg2} with shape {shape2}." class AxisError(ValueError, IndexError): """ @@ -37,10 +49,28 @@ def broadcast_shapes(*shapes, skip_axes=()): Broadcast the input shapes `shapes` to a single shape. This is the same as :py:func:`np.broadcast_shapes() - `. It is included as a separate helper function - because `np.broadcast_shapes()` is on available in NumPy 1.20 or newer, and - so that ndindex functions that use this function can do without requiring - NumPy to be installed. + `, except is also supports skipping axes in the + shape with `skip_axes`. + + If the shapes are not broadcast compatible (excluding `skip_axes`), + `BroadcastError` is raised. + + >>> from ndindex import broadcast_shapes + >>> broadcast_shapes((2, 3), (3,), (4, 2, 1)) + (4, 2, 3) + >>> broadcast_shapes((2, 3), (5,), (4, 2, 1)) + Traceback (most recent call last): + ... + ndindex.iterators.BroadcastError: shape mismatch: objects cannot be broadcast to a single shape. Mismatch is between arg 0 with shape (2, 3) and arg 1 with shape (5,). + + Axes in `skip_axes` apply to each shape *before* being broadcasted. Each + shape will be broadcasted together with these axes removed. The dimensions + in skip_axes do not need to be equal or broadcast compatible with one + another. The final broadcasted shape will have `None` in each `skip_axes` + location, and the broadcasted remaining `shapes` axes elsewhere. + + >>> broadcast_shapes((10, 3, 2), (20, 2), skip_axes=(0,)) + (None, 3, 2) """ if isinstance(skip_axes, int): @@ -50,55 +80,42 @@ def broadcast_shapes(*shapes, skip_axes=()): # See the comments in remove_indices and iter_indices raise NotImplementedError("Mixing both negative and nonnegative skip_axes is not yet supported") - def _broadcast_shapes(shape1, shape2): - """Broadcasts `shape1` and `shape2`""" - N1 = len(shape1) - N2 = len(shape2) - skip_axes1 = [ndindex(i).reduce(N1).raw - N1 for i in skip_axes] - skip_axes2 = [ndindex(i).reduce(N2).raw - N2 for i in skip_axes] - N = max(N1, N2) - shape = [None for _ in range(N)] - i = N - 1 - while i >= 0: - n1 = N1 - N + i - if i in skip_axes1: - d1 = None - elif n1 >= 0: - d1 = shape1[n1] - else: - d1 = 1 - n2 = N2 - N + i - if i in skip_axes2: - d2 = None - elif n2 >= 0: - d2 = shape2[n2] - else: - d2 = 1 - - if d1 == None or d2 == None: - shape[i] = None - elif d1 == 1: - shape[i] = d2 - elif d2 == 1: - shape[i] = d1 - elif d1 == d2: - shape[i] = d1 - else: - # TODO: Build an error message that matches NumPy - raise BroadcastError("shape mismatch: objects cannot be broadcast to a single shape.") - - i = i - 1 - - return tuple(shape) - - if len(shapes) == 1: - shape = shapes[0] - N = len(shape) - # Check that skip_axes are valid - [ndindex(i).reduce(N).raw - N for i in skip_axes] - return shape + if not shapes: + if skip_axes: + # Raise IndexError + ndindex(skip_axes[0]).reduce(0) + return () + + dims = [len(shape) for shape in shapes] + shape_skip_axes = [[ndindex(i).reduce(n).raw - n for i in skip_axes] for n in dims] + N = max(dims) + broadcasted_skip_axes = [ndindex(i).reduce(N).raw - N for i in skip_axes] + + broadcasted_shape = [None]*N + + for i in range(-1, -N-1, -1): + broadcasted_side = 1 + arg = None + for j in range(len(shapes)): + shape = shapes[j] + if dims[j] < -i: + continue + shape_side = shape[i] + if i in shape_skip_axes[j]: + continue + elif shape_side == 1: + continue + elif broadcasted_side == 1: + broadcasted_side = shape_side + arg = j + elif shape_side != broadcasted_side: + raise BroadcastError(arg, shapes[arg], j, shapes[j]) + if i in broadcasted_skip_axes: + broadcasted_shape[i] = None + else: + broadcasted_shape[i] = broadcasted_side - return functools.reduce(_broadcast_shapes, shapes, ()) + return tuple(broadcasted_shape) def iter_indices(*shapes, skip_axes=(), _debug=False): """ diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index 87201008..f35e38ce 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -221,11 +221,79 @@ def test_broadcast_shapes(broadcastable_shapes): else: raises(BroadcastError, lambda: broadcast_shapes(*shapes)) + +@given(lists(shapes, max_size=32)) +def test_broadcast_shapes_errors(shapes): + error = True + try: + broadcast_shapes(*shapes) + except BroadcastError as exc: + e = exc + else: + error = False + + # The ndindex and numpy errors won't match in general, because + # ndindex.broadcast_shapes gives an error with the first two shapes that + # aren't broadcast compatible, but numpy doesn't always, due to different + # implementation algorithms (e.g., the message from + # np.broadcast_shapes((0,), (0, 2), (2, 0)) mentions the last two shapes + # whereas ndindex.broadcast_shapes mentions the first two). + + # Instead, just confirm that the error message is correct as stated, and + # check against the numpy error message when just broadcasting the two + # reportedly bad shapes. + + if not error: + try: + np.broadcast_shapes(*shapes) + except: + raise RuntimeError("ndindex.broadcast_shapes raised but np.broadcast_shapes did not") + return + + assert shapes[e.arg1] == e.shape1 + assert shapes[e.arg2] == e.shape2 + + try: + np.broadcast_shapes(e.shape1, e.shape2) + except ValueError as np_exc: + # Check that they do in fact not broadcast, and the error messages are + # the same modulo the different arg positions. + assert str(BroadcastError(0, e.shape1, 1, e.shape2)) == str(np_exc) + else: + raise RuntimeError("ndindex.broadcast_shapes raised but np.broadcast_shapes did not") + @given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes_st) def test_broadcast_shapes_skip_axes(broadcastable_shapes, skip_axes): shapes, broadcasted_shape = broadcastable_shapes assert broadcast_shapes(*shapes, skip_axes=skip_axes) == broadcasted_shape +@given(mutually_broadcastable_shapes, lists(integers(-20, 20), max_size=20)) +def test_broadcast_shapes_skip_axes_errors(broadcastable_shapes, skip_axes): + shapes, broadcasted_shape = broadcastable_shapes + if any(i < 0 for i in skip_axes) and any(i >= 0 for i in skip_axes): + raises(NotImplementedError, lambda: broadcast_shapes(*shapes, skip_axes=skip_axes)) + return + + try: + if not shapes and skip_axes: + raise IndexError + for shape in shapes: + for i in skip_axes: + shape[i] + except IndexError: + error = True + else: + error = False + + try: + broadcast_shapes(*shapes, skip_axes=skip_axes) + except IndexError: + if not error: + raise RuntimeError("broadcast_shapes raised but should not have") + else: + if error: + raise RuntimeError("broadcast_shapes did not raise but should have") + remove_indices_n = shared(integers(0, 100)) @given(remove_indices_n, From 1edcd0fd4c9de61869e9a2340c49abe99ffb3667 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 13 Apr 2023 01:28:48 -0600 Subject: [PATCH 097/218] Remove an unused import --- ndindex/iterators.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index 480f4ac2..bf1ae3e5 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -1,5 +1,4 @@ import itertools -import functools from collections import defaultdict from .ndindex import asshape, ndindex From fd0d4740ab2011ebeef1c33ea965d9e7e08c320c Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 13 Apr 2023 01:28:53 -0600 Subject: [PATCH 098/218] Make iter_indices() always iterate nothing if a shape has a 0 The docstring says that it is equivalent to broadcasting first and iterating the broadcasted shape. Previously this was not the case if the broadcasted shape was size 0. --- ndindex/iterators.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index bf1ae3e5..d86441a5 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -289,6 +289,8 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): else: if _broadcasted_shape[i] is None: pass + elif _broadcasted_shape[i] == 0: + return elif _broadcasted_shape[i] != 1 and shape[i] == 1: it.insert(0, ncycles(range(shape[i]), _broadcasted_shape[i])) else: From 0a51c24d6b315b332f9f90f88b7a8a151ebd921b Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 13 Apr 2023 01:29:50 -0600 Subject: [PATCH 099/218] Several fixes in the test_iter_indices logic --- ndindex/tests/test_iterators.py | 36 ++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index f35e38ce..af390770 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -27,23 +27,28 @@ def test_iter_indices(broadcastable_shapes, skip_axes): shapes, broadcasted_shape = broadcastable_shapes # 1. Normalize inputs - _skip_axes = (skip_axes,) if isinstance(skip_axes, int) else () + _skip_axes = (skip_axes,) if isinstance(skip_axes, int) else skip_axes ndim = len(broadcasted_shape) # Double check the mutually_broadcastable_shapes_with_skipped_axes # strategy - for i in skip_axes: + for i in _skip_axes: assert broadcasted_shape[i] is None - # Skipped axes may not be broadcast compatible. Since the index for a # skipped axis should always be a slice(None), the result should be the # same if the skipped axes are all moved to the end of the shape. - canonical_shapes = [list(s) for s in shapes] - for i in skip_axes: - for s in canonical_shapes: - x = s.pop(i) - s.append(x) + canonical_shapes = [] + for s in shapes: + c = remove_indices(s, _skip_axes) + c = c + tuple(s[i] for i in _skip_axes) + canonical_shapes.append(c) + canonical_skip_axes = list(range(-1, -len(_skip_axes) - 1, -1)) + broadcasted_canonical_shape = list(broadcast_shapes(*canonical_shapes, + skip_axes=canonical_skip_axes)) + for i in range(len(broadcasted_canonical_shape)): + if broadcasted_canonical_shape[i] is None: + broadcasted_canonical_shape[i] = 1 skip_shapes = [tuple(shape[i] for i in _skip_axes) for shape in shapes] non_skip_shapes = [remove_indices(shape, skip_axes) for shape in shapes] @@ -53,13 +58,13 @@ def test_iter_indices(broadcastable_shapes, skip_axes): nitems = prod(broadcasted_non_skip_shape) - if _skip_axes is None: + if _skip_axes == (): res = iter_indices(*shapes) - broadcasted_res = iter_indices(np.broadcast_shapes(*shapes)) + broadcasted_res = iter_indices(broadcast_shapes(*shapes)) else: res = iter_indices(*shapes, skip_axes=skip_axes) - broadcasted_res = iter_indices(np.broadcast_shapes(*canonical_shapes), - skip_axes=skip_axes) + broadcasted_res = iter_indices(broadcasted_canonical_shape, + skip_axes=canonical_skip_axes) sizes = [prod(shape) for shape in shapes] arrays = [np.arange(size).reshape(shape) for size, shape in zip(sizes, shapes)] @@ -75,10 +80,9 @@ def test_iter_indices(broadcastable_shapes, skip_axes): def _move_slices_to_end(idx): assert isinstance(idx, Tuple) idx2 = list(idx.args) - for i in range(len(idx2)): - if idx.args[i] == slice(None): - idx2.pop(i) - idx2.append(slice(None)) + slices = [i for i in range(len(idx2)) if idx2[i] == slice(None)] + idx2 = remove_indices(idx2, slices) + idx2 = idx2 + (slice(None),)*len(slices) return Tuple(*idx2) for n, (idxes, bidxes) in enumerate(zip(res, broadcasted_res)): From 225147c94c091a90800fa040e52fa48afba59616 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 13 Apr 2023 01:30:03 -0600 Subject: [PATCH 100/218] Test that the broadcasted shape iter_indices produces the same number of indices --- ndindex/tests/test_iterators.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index af390770..2c6b70e0 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -116,6 +116,10 @@ def _move_slices_to_end(idx): # else: # vals.append(a_indexed) + # assert both iterators have the same length + raises(StopIteration, lambda: next(res)) + raises(StopIteration, lambda: next(broadcasted_res)) + return assert len(set(vals)) == len(vals) == nitems From 8aa1bb341d286963cfdcf6766cadfbc746f206a5 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 13 Apr 2023 01:30:27 -0600 Subject: [PATCH 101/218] Move the number of iterates test in test_iter_indices up --- ndindex/tests/test_iterators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index 2c6b70e0..454d5958 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -120,6 +120,7 @@ def _move_slices_to_end(idx): raises(StopIteration, lambda: next(res)) raises(StopIteration, lambda: next(broadcasted_res)) + assert n == nitems - 1 return assert len(set(vals)) == len(vals) == nitems @@ -142,7 +143,6 @@ def _move_slices_to_end(idx): else: assert set(vals) == set(correct_vals) - assert n == nitems - 1 def test_iter_indices_errors(): try: From 9842404db598bb65a791bb38a707a2c0fb28a808 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 13 Apr 2023 13:05:25 -0600 Subject: [PATCH 102/218] Some code cleanups in iter_indices() --- ndindex/iterators.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index d86441a5..ffa2cb77 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -267,32 +267,28 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): _skip_axes[shape].append(a) _skip_axes[shape].sort() - _shapes = [remove_indices(shape, skip_axes) for shape in shapes] - # _shapes = [(1,)*(ndim - len(shape)) + shape for shape in shapes] - # _shapes = [tuple(1 if i in _skip_axes else shape[i] for i in range(ndim)) - # for shape in _shapes] iters = [[] for i in range(len(shapes))] - broadcasted_shape = broadcast_shapes(*_shapes) - _broadcasted_shape = unremove_indices(broadcasted_shape, skip_axes) + broadcasted_shape = broadcast_shapes(*shapes, skip_axes=skip_axes) + non_skip_broadcasted_shape = remove_indices(broadcasted_shape, skip_axes) for i in range(-1, -ndim-1, -1): - for it, shape, _shape in zip(iters, shapes, _shapes): + for it, shape in zip(iters, shapes): if -i > len(shape): # for every dimension prepended by broadcasting, repeat the # indices that many times for j in range(len(it)): - if broadcasted_shape[i+n] not in [0, 1]: - it[j] = ncycles(it[j], broadcasted_shape[i+n]) + if non_skip_broadcasted_shape[i+n] not in [0, 1]: + it[j] = ncycles(it[j], non_skip_broadcasted_shape[i+n]) break elif len(shape) + i in _skip_axes[shape]: it.insert(0, [slice(None)]) else: - if _broadcasted_shape[i] is None: + if broadcasted_shape[i] is None: pass - elif _broadcasted_shape[i] == 0: + elif broadcasted_shape[i] == 0: return - elif _broadcasted_shape[i] != 1 and shape[i] == 1: - it.insert(0, ncycles(range(shape[i]), _broadcasted_shape[i])) + elif broadcasted_shape[i] != 1 and shape[i] == 1: + it.insert(0, ncycles(range(shape[i]), broadcasted_shape[i])) else: it.insert(0, range(shape[i])) From a43747468a1e0e686228260c825df0fe60755921 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 13 Apr 2023 13:42:08 -0600 Subject: [PATCH 103/218] Fix test_mutually_broadcastable_shapes_with_skipped_axes to actually test something --- ndindex/tests/test_iterators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index 454d5958..1304cda4 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -341,7 +341,7 @@ def test_remove_indices(n, idxes): def test_mutually_broadcastable_shapes_with_skipped_axes(broadcastable_shapes, skip_axes): shapes, broadcasted_shape = broadcastable_shapes - _skip_axes = (skip_axes,) if isinstance(skip_axes, int) else () + _skip_axes = (skip_axes,) if isinstance(skip_axes, int) else skip_axes for shape in shapes: assert None not in shape From 70e1b9d13a5b8ab568b0a54af71f34fa57d73920 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 13 Apr 2023 16:34:43 -0600 Subject: [PATCH 104/218] Add a helper function associated_axis() in iterators.py --- ndindex/iterators.py | 25 +++++++++++++++++++++++++ ndindex/tests/test_iterators.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index ffa2cb77..fc0da740 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -300,6 +300,31 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): iters], fillvalue=()): yield tuple(ndindex(idx) for idx in idxes) +def associated_axis(shape, broadcasted_shape, i, skip_axes): + """ + Return the associated index into broadcast_shape corresponding to + shape[i] given skip_axes. + + """ + n = len(shape) + N = len(broadcasted_shape) + skip_axes = sorted(skip_axes) + if i >= 0: + raise NotImplementedError + if not skip_axes: + return i + if skip_axes[0] < 0: + return i + elif skip_axes[0] >= 0: + k = m = 0 + for j in range(len(skip_axes)): + s = skip_axes[j] + if s <= i + n: + k = s + if s <= i + N: + m = s + return i + n + m - k + def remove_indices(x, idxes): """ Return `x` with the indices `idxes` removed. diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index 1304cda4..0f4372cc 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -9,7 +9,7 @@ from ..ndindex import ndindex from ..iterators import (iter_indices, ncycles, BroadcastError, AxisError, broadcast_shapes, remove_indices, - unremove_indices) + unremove_indices, associated_axis) from ..integer import Integer from ..tuple import Tuple from .helpers import (assert_equal, prod, @@ -353,3 +353,30 @@ def test_mutually_broadcastable_shapes_with_skipped_axes(broadcastable_shapes, assert None not in _broadcasted_shape assert broadcast_shapes(*_shapes) == _broadcasted_shape + +associated_axis_broadcastable_shapes = \ + mutually_broadcastable_shapes_with_skipped_axes()\ + .filter(lambda i: i[0])\ + .filter(lambda i: i[0][0]) + +@given(associated_axis_broadcastable_shapes, + associated_axis_broadcastable_shapes.flatmap(lambda i: integers(-len(i[0][0]), -1)), + skip_axes_st) +def test_associated_axis(broadcastable_shapes, i, skip_axes): + _skip_axes = (skip_axes,) if isinstance(skip_axes, int) else skip_axes + + shapes, broadcasted_shape = broadcastable_shapes + ndim = len(broadcasted_shape) + + normalized_skip_axes = [ndindex(i).reduce(ndim) for i in _skip_axes] + + shape = shapes[0] + val = shape[i] + assume(val != 1) + + idx = associated_axis(shape, broadcasted_shape, i, _skip_axes) + bval = broadcasted_shape[idx] + if bval is None: + assert ndindex(idx).reduce(ndim) in normalized_skip_axes + else: + assert bval == val From 6499e3a1f1763d11e22562670308138075592cc2 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 14 Apr 2023 10:59:58 -0600 Subject: [PATCH 105/218] Fix associated_axis helper function --- ndindex/iterators.py | 21 +++++++++++++-------- ndindex/tests/test_iterators.py | 6 +++--- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index fc0da740..1740ac5b 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -308,22 +308,27 @@ def associated_axis(shape, broadcasted_shape, i, skip_axes): """ n = len(shape) N = len(broadcasted_shape) - skip_axes = sorted(skip_axes) + skip_axes = sorted(skip_axes, reverse=True) if i >= 0: raise NotImplementedError if not skip_axes: return i + # We assume skip_axes are either all negative or all nonnegative if skip_axes[0] < 0: return i elif skip_axes[0] >= 0: + posi = ndindex(i).reduce(n).raw + if posi in skip_axes: + return posi k = m = 0 - for j in range(len(skip_axes)): - s = skip_axes[j] - if s <= i + n: - k = s - if s <= i + N: - m = s - return i + n + m - k + for s in skip_axes: + s_s = s - n + b_s = s - N + if s_s > i: + k += 1 + if b_s > i + k: + m += 1 + return i + k - m def remove_indices(x, idxes): """ diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index 0f4372cc..ffb7b05d 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -355,9 +355,9 @@ def test_mutually_broadcastable_shapes_with_skipped_axes(broadcastable_shapes, assert broadcast_shapes(*_shapes) == _broadcasted_shape associated_axis_broadcastable_shapes = \ - mutually_broadcastable_shapes_with_skipped_axes()\ - .filter(lambda i: i[0])\ - .filter(lambda i: i[0][0]) + shared(mutually_broadcastable_shapes_with_skipped_axes()\ + .filter(lambda i: i[0])\ + .filter(lambda i: i[0][0])) @given(associated_axis_broadcastable_shapes, associated_axis_broadcastable_shapes.flatmap(lambda i: integers(-len(i[0][0]), -1)), From 22f4bb27208ae6076a3de43bc584deae432fdec5 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 17 Apr 2023 03:29:58 -0600 Subject: [PATCH 106/218] Fix associated_axis and the corresponding test --- ndindex/iterators.py | 2 +- ndindex/tests/test_iterators.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index 1740ac5b..b79062d4 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -326,7 +326,7 @@ def associated_axis(shape, broadcasted_shape, i, skip_axes): b_s = s - N if s_s > i: k += 1 - if b_s > i + k: + if b_s > i: m += 1 return i + k - m diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index ffb7b05d..5e44102a 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -371,12 +371,17 @@ def test_associated_axis(broadcastable_shapes, i, skip_axes): normalized_skip_axes = [ndindex(i).reduce(ndim) for i in _skip_axes] shape = shapes[0] + n = len(shape) val = shape[i] assume(val != 1) idx = associated_axis(shape, broadcasted_shape, i, _skip_axes) bval = broadcasted_shape[idx] if bval is None: - assert ndindex(idx).reduce(ndim) in normalized_skip_axes + if _skip_axes[0] >= 0: + assert ndindex(i).reduce(n) == ndindex(idx).reduce(ndim) in normalized_skip_axes + else: + assert ndindex(i).reduce(n).raw - n == \ + ndindex(idx).reduce(ndim).raw - ndim in _skip_axes else: assert bval == val From 679bf132ab06748e02293690bcd5bac6a2d8a059 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 17 Apr 2023 03:30:27 -0600 Subject: [PATCH 107/218] Rewrite broadcast_shapes() and iter_indices() to use associated_axis() This fixes incorrect behavior in both. --- ndindex/iterators.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index b79062d4..984eb1b9 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -88,17 +88,18 @@ def broadcast_shapes(*shapes, skip_axes=()): dims = [len(shape) for shape in shapes] shape_skip_axes = [[ndindex(i).reduce(n).raw - n for i in skip_axes] for n in dims] N = max(dims) - broadcasted_skip_axes = [ndindex(i).reduce(N).raw - N for i in skip_axes] + broadcasted_skip_axes = [ndindex(i).reduce(N) for i in skip_axes] - broadcasted_shape = [None]*N + broadcasted_shape = [None if i in broadcasted_skip_axes else 1 for i in range(N)] for i in range(-1, -N-1, -1): - broadcasted_side = 1 arg = None for j in range(len(shapes)): - shape = shapes[j] if dims[j] < -i: continue + shape = shapes[j] + idx = associated_axis(shape, broadcasted_shape, i, skip_axes) + broadcasted_side = broadcasted_shape[idx] shape_side = shape[i] if i in shape_skip_axes[j]: continue @@ -109,10 +110,7 @@ def broadcast_shapes(*shapes, skip_axes=()): arg = j elif shape_side != broadcasted_side: raise BroadcastError(arg, shapes[arg], j, shapes[j]) - if i in broadcasted_skip_axes: - broadcasted_shape[i] = None - else: - broadcasted_shape[i] = broadcasted_side + broadcasted_shape[idx] = broadcasted_side return tuple(broadcasted_shape) @@ -283,12 +281,14 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): elif len(shape) + i in _skip_axes[shape]: it.insert(0, [slice(None)]) else: - if broadcasted_shape[i] is None: + idx = associated_axis(shape, broadcasted_shape, i, skip_axes) + val = broadcasted_shape[idx] + if val is None: pass - elif broadcasted_shape[i] == 0: + elif val == 0: return - elif broadcasted_shape[i] != 1 and shape[i] == 1: - it.insert(0, ncycles(range(shape[i]), broadcasted_shape[i])) + elif val != 1 and shape[i] == 1: + it.insert(0, ncycles(range(shape[i]), val)) else: it.insert(0, range(shape[i])) From 6bb91241863bc3fc731a41da83dfd68ed1b2932d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 17 Apr 2023 03:31:49 -0600 Subject: [PATCH 108/218] Some more work on fixing test_iter_indices --- ndindex/tests/test_iterators.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index 5e44102a..0bf66e23 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -41,7 +41,7 @@ def test_iter_indices(broadcastable_shapes, skip_axes): canonical_shapes = [] for s in shapes: c = remove_indices(s, _skip_axes) - c = c + tuple(s[i] for i in _skip_axes) + c = c + tuple(1 for i in _skip_axes) canonical_shapes.append(c) canonical_skip_axes = list(range(-1, -len(_skip_axes) - 1, -1)) broadcasted_canonical_shape = list(broadcast_shapes(*canonical_shapes, @@ -50,7 +50,7 @@ def test_iter_indices(broadcastable_shapes, skip_axes): if broadcasted_canonical_shape[i] is None: broadcasted_canonical_shape[i] = 1 - skip_shapes = [tuple(shape[i] for i in _skip_axes) for shape in shapes] + skip_shapes = [tuple(1 for i in _skip_axes) for shape in shapes] non_skip_shapes = [remove_indices(shape, skip_axes) for shape in shapes] broadcasted_non_skip_shape = remove_indices(broadcasted_shape, skip_axes) assert None not in broadcasted_non_skip_shape @@ -107,22 +107,22 @@ def _move_slices_to_end(idx): for a_indexed, skip_shape in zip(canonical_a_indexed, skip_shapes): assert a_indexed.shape == skip_shape - # if skip_axes: - # # If there are skipped axes, recursively call iter_indices to - # # get each individual element of the resulting subarrays. - # for subidxes in iter_indices(*[x.shape for x in canonical_a_indexed]): - # items = [x[i.raw] for x, i in zip(canonical_a_indexed, subidxes)] - # vals.append(tuple(items)) - # else: - # vals.append(a_indexed) + if skip_axes: + # If there are skipped axes, recursively call iter_indices to + # get each individual element of the resulting subarrays. + for subidxes in iter_indices(*[x.shape for x in canonical_a_indexed]): + items = [x[i.raw] for x, i in zip(canonical_a_indexed, subidxes)] + vals.append(tuple(items)) + else: + vals.append(a_indexed) # assert both iterators have the same length raises(StopIteration, lambda: next(res)) raises(StopIteration, lambda: next(broadcasted_res)) assert n == nitems - 1 + # assert len(set(vals)) == len(vals) == nitems return - assert len(set(vals)) == len(vals) == nitems # 3. Check that every element of the (broadcasted) arrays is represented # by an iterated index. From 4370acefe0217da1ec778c05b40b2ad2ca535692 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 17 Apr 2023 16:35:15 -0600 Subject: [PATCH 109/218] Some fixes to test_iter_indices --- ndindex/tests/test_iterators.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index 0bf66e23..c98a29ef 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -104,10 +104,10 @@ def _move_slices_to_end(idx): canonical_a_indexed = tuple([a[idx.raw] for a, idx in zip(canonical_arrays, canonical_idxes)]) - for a_indexed, skip_shape in zip(canonical_a_indexed, skip_shapes): - assert a_indexed.shape == skip_shape + for c_indexed, skip_shape in zip(canonical_a_indexed, skip_shapes): + assert c_indexed.shape == skip_shape - if skip_axes: + if _skip_axes: # If there are skipped axes, recursively call iter_indices to # get each individual element of the resulting subarrays. for subidxes in iter_indices(*[x.shape for x in canonical_a_indexed]): @@ -121,7 +121,7 @@ def _move_slices_to_end(idx): raises(StopIteration, lambda: next(broadcasted_res)) assert n == nitems - 1 - # assert len(set(vals)) == len(vals) == nitems + assert len(set(vals)) == len(vals) == nitems return # 3. Check that every element of the (broadcasted) arrays is represented From cb4b0a036e1de607e63578d130d17706a0d94f03 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 17 Apr 2023 16:56:28 -0600 Subject: [PATCH 110/218] Fix pyflakes errors --- ndindex/tests/test_iterators.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index c98a29ef..1e7ceda9 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -12,8 +12,7 @@ unremove_indices, associated_axis) from ..integer import Integer from ..tuple import Tuple -from .helpers import (assert_equal, prod, - mutually_broadcastable_shapes_with_skipped_axes, +from .helpers import (prod, mutually_broadcastable_shapes_with_skipped_axes, skip_axes_st, mutually_broadcastable_shapes, tuples, shapes) @@ -28,7 +27,6 @@ def test_iter_indices(broadcastable_shapes, skip_axes): # 1. Normalize inputs _skip_axes = (skip_axes,) if isinstance(skip_axes, int) else skip_axes - ndim = len(broadcasted_shape) # Double check the mutually_broadcastable_shapes_with_skipped_axes # strategy From 8535f6c0bdb5e33d110f87d0ccfac822b9ac594c Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 17 Apr 2023 16:56:34 -0600 Subject: [PATCH 111/218] Fix/enable the rest of test_iter_indices --- ndindex/tests/test_iterators.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index 1e7ceda9..62cea3b3 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -118,9 +118,9 @@ def _move_slices_to_end(idx): raises(StopIteration, lambda: next(res)) raises(StopIteration, lambda: next(broadcasted_res)) + # Check that the correct number of items are iterated assert n == nitems - 1 assert len(set(vals)) == len(vals) == nitems - return # 3. Check that every element of the (broadcasted) arrays is represented # by an iterated index. @@ -130,7 +130,7 @@ def _move_slices_to_end(idx): if not arrays: assert vals == [()] else: - correct_vals = [tuple(i) for i in np.stack(broadcasted_arrays, axis=-1).reshape((nitems, len(arrays)))] + correct_vals = [tuple(i) for i in np.stack(np.broadcast_arrays(*canonical_arrays), axis=-1).reshape((nitems, len(arrays)))] # Also test that the indices are produced in a lexicographic order # (even though this isn't strictly guaranteed by the iter_indices # docstring) in the case when there are no skip axes. The order when @@ -141,7 +141,6 @@ def _move_slices_to_end(idx): else: assert set(vals) == set(correct_vals) - def test_iter_indices_errors(): try: list(iter_indices((10,), skip_axes=(2,))) From eef64caebc9fee0a463930014c36640bdcb4157e Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 17 Apr 2023 17:03:11 -0600 Subject: [PATCH 112/218] Uncomment a test in test_iter_indices_errors --- ndindex/tests/test_iterators.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index 62cea3b3..068248cd 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -166,15 +166,14 @@ def test_iter_indices_errors(): else: raise RuntimeError("iter_indices did not raise BroadcastError") # pragma: no cover - # TODO: Check that the message is the same one used by NumPy - # try: - # np.broadcast_shapes((2, 3), (3, 2)) - # except np.Error as e: - # msg2 = str(e) - # else: - # raise RuntimeError("np.broadcast_shapes() did not raise AxisError") # pragma: no cover - # - # assert msg1 == msg2 + try: + np.broadcast_shapes((2, 3), (3, 2)) + except ValueError as e: + msg2 = str(e) + else: + raise RuntimeError("np.broadcast_shapes() did not raise ValueError") # pragma: no cover + + assert msg1 == msg2 raises(NotImplementedError, lambda: list(iter_indices((1, 2), skip_axes=(0, -1)))) From c5f20ea87960bfb856b5267c412d7d228d2c8079 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 17 Apr 2023 17:15:36 -0600 Subject: [PATCH 113/218] Use ndindex.broadcast_shapes() in ndindex library code --- ndindex/tuple.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/ndindex/tuple.py b/ndindex/tuple.py index 46afe00c..de1f5d53 100644 --- a/ndindex/tuple.py +++ b/ndindex/tuple.py @@ -2,6 +2,7 @@ from .ndindex import NDIndex, ndindex, asshape from .subindex_helpers import subindex_slice +from .iterators import broadcast_shapes, BroadcastError class Tuple(NDIndex): """ @@ -92,15 +93,12 @@ def _typecheck(self, *args): if has_boolean_scalar: raise NotImplementedError("Tuples mixing boolean scalars (True or False) with arrays are not yet supported.") - from numpy import broadcast try: - broadcast(*[i for i in arrays]) - except ValueError as e: - assert str(e).startswith("shape mismatch: objects cannot be broadcast to a single shape"), e.args - # TODO: Newer versions of NumPy include where the mismatch is - # in the error message in a more informative way than this - # (but we can't use it directly because it talks about the - # "arg"s to broadcast()). + broadcast_shapes(*[i.shape for i in arrays]) + except BroadcastError: + # This matches the NumPy error message. The BroadcastError has + # a better error message, but it will be shown in the chained + # traceback. raise IndexError("shape mismatch: indexing arrays could not be broadcast together with shapes %s" % ' '.join([str(i.shape) for i in arrays])) return tuple(newargs) @@ -292,9 +290,9 @@ def reduce(self, shape=None): # TODO: Avoid explicitly calling nonzero arrays.extend(i.raw.nonzero()) if arrays: - from numpy import broadcast, broadcast_to + from numpy import broadcast_to - broadcast_shape = broadcast(*arrays).shape + broadcast_shape = broadcast_shapes(*[a.shape for a in arrays]) else: broadcast_shape = () @@ -428,9 +426,9 @@ def broadcast_arrays(self): if not arrays: return self - from numpy import array, broadcast, broadcast_to, intp + from numpy import array, broadcast_to, intp - broadcast_shape = broadcast(*arrays).shape + broadcast_shape = broadcast_shapes(*[a.shape for a in arrays]) newargs = [] for s in args: @@ -487,9 +485,9 @@ def expand(self, shape): arrays.extend(i.raw.nonzero()) if arrays: - from numpy import broadcast, broadcast_to, array, intp + from numpy import broadcast_to, array, intp - broadcast_shape = broadcast(*arrays).shape + broadcast_shape = broadcast_shapes(*[a.shape for a in arrays]) # If the broadcast shape is empty, out of bounds indices in # non-empty arrays are ignored, e.g., ([], [10]) would broadcast to # ([], []), so the bounds for 10 are not checked. Thus, we must do From 4960707680543a66025e8b97f2af9d7f603d1842 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 17 Apr 2023 17:28:38 -0600 Subject: [PATCH 114/218] Fix test failure in test_iter_indices --- ndindex/tests/test_iterators.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index 068248cd..bb020717 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -24,6 +24,9 @@ def test_iter_indices(broadcastable_shapes, skip_axes): # broadcasted_shape will contain None on the skip_axes, as those axes # might not be broadcast compatible shapes, broadcasted_shape = broadcastable_shapes + # We need no more than 31 dimensions so that the np.stack call below + # doesn't fail. + assume(len(broadcasted_shape) < 32) # 1. Normalize inputs _skip_axes = (skip_axes,) if isinstance(skip_axes, int) else skip_axes From fe249e6089be2b1d43762c58f7a7ba317ec7985d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 18 Apr 2023 19:02:21 -0600 Subject: [PATCH 115/218] Clean up test_associated_axis() - Add some examples that seem to be hard to come across from the hypothesis generation. - Loop over all shapes and indices in the test. - Remove all filtering of inputs. Note that the current associated_axis() implementation has a bug. --- ndindex/tests/test_iterators.py | 45 +++++++++++++++------------------ 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index bb020717..c89f7ca8 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -353,15 +353,12 @@ def test_mutually_broadcastable_shapes_with_skipped_axes(broadcastable_shapes, assert None not in _broadcasted_shape assert broadcast_shapes(*_shapes) == _broadcasted_shape -associated_axis_broadcastable_shapes = \ - shared(mutually_broadcastable_shapes_with_skipped_axes()\ - .filter(lambda i: i[0])\ - .filter(lambda i: i[0][0])) - -@given(associated_axis_broadcastable_shapes, - associated_axis_broadcastable_shapes.flatmap(lambda i: integers(-len(i[0][0]), -1)), - skip_axes_st) -def test_associated_axis(broadcastable_shapes, i, skip_axes): +@example([[(0, 10, 2, 3, 10, 4), (1, 10, 1, 0, 10, 2, 3, 4)], + (1, None, 1, 0, None, 2, 3, 4)], (1, 4)) +@example([[(2, 0, 3, 4)], (2, None, 3, 4)], (1,)) +@example([[(0, 0, 0, 0, 0), (0, 0, 0, 0, 0, 0)], (0, None, None, 0, 0, 0)], (1, 2)) +@given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes_st) +def test_associated_axis(broadcastable_shapes, skip_axes): _skip_axes = (skip_axes,) if isinstance(skip_axes, int) else skip_axes shapes, broadcasted_shape = broadcastable_shapes @@ -369,18 +366,18 @@ def test_associated_axis(broadcastable_shapes, i, skip_axes): normalized_skip_axes = [ndindex(i).reduce(ndim) for i in _skip_axes] - shape = shapes[0] - n = len(shape) - val = shape[i] - assume(val != 1) - - idx = associated_axis(shape, broadcasted_shape, i, _skip_axes) - bval = broadcasted_shape[idx] - if bval is None: - if _skip_axes[0] >= 0: - assert ndindex(i).reduce(n) == ndindex(idx).reduce(ndim) in normalized_skip_axes - else: - assert ndindex(i).reduce(n).raw - n == \ - ndindex(idx).reduce(ndim).raw - ndim in _skip_axes - else: - assert bval == val + for shape in shapes: + n = len(shape) + for i in range(-len(shape), 0): + val = shape[i] + + idx = associated_axis(shape, broadcasted_shape, i, _skip_axes) + bval = broadcasted_shape[idx] + if bval is None: + if _skip_axes[0] >= 0: + assert ndindex(i).reduce(n) == ndindex(idx).reduce(ndim) in normalized_skip_axes + else: + assert ndindex(i).reduce(n).raw - n == \ + ndindex(idx).reduce(ndim).raw - ndim in _skip_axes + else: + assert val == 1 or bval == val From 65d2331c93ebb8331ed4597928163c2ba8927102 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 18 Apr 2023 19:03:45 -0600 Subject: [PATCH 116/218] More correct associated_axis implementation Hopefully fully correct now. At least the implementation actually makes sense now so it definitely feels like it should be correct, which wasn't true before. It is less efficient so I'm still hoping I can simplify it down to something with fewer inner loops. --- ndindex/iterators.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index 984eb1b9..0d9e7007 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -317,18 +317,19 @@ def associated_axis(shape, broadcasted_shape, i, skip_axes): if skip_axes[0] < 0: return i elif skip_axes[0] >= 0: - posi = ndindex(i).reduce(n).raw - if posi in skip_axes: - return posi - k = m = 0 + invmapping = [None]*N for s in skip_axes: - s_s = s - n - b_s = s - N - if s_s > i: - k += 1 - if b_s > i: - m += 1 - return i + k - m + invmapping[s] = s + + for j in range(-1, i-1, -1): + if j + n in skip_axes: + k = j + n #- N + continue + for k in range(-1, -N-1, -1): + if invmapping[k] is None: + invmapping[k] = j + break + return k def remove_indices(x, idxes): """ From cd4a7508fa9fc167c2eba94fb8f412b4dfbce845 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 15:19:58 -0600 Subject: [PATCH 117/218] Fix error message construction for broadcast_shapes with skip axes --- ndindex/iterators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index 0d9e7007..ebd08a31 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -92,8 +92,8 @@ def broadcast_shapes(*shapes, skip_axes=()): broadcasted_shape = [None if i in broadcasted_skip_axes else 1 for i in range(N)] + arg = None for i in range(-1, -N-1, -1): - arg = None for j in range(len(shapes)): if dims[j] < -i: continue From 3a894efb7870f72d589d825c05032241da4f56bc Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 15:20:25 -0600 Subject: [PATCH 118/218] Fix test_broadcast_shapes_skip_axes_errors() to handle BroadcastError --- ndindex/tests/test_iterators.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index c89f7ca8..81a4e592 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -281,6 +281,8 @@ def test_broadcast_shapes_skip_axes_errors(broadcastable_shapes, skip_axes): raises(NotImplementedError, lambda: broadcast_shapes(*shapes, skip_axes=skip_axes)) return + # Test that broadcast_shapes raises IndexError when skip_axes are out of + # bounds try: if not shapes and skip_axes: raise IndexError @@ -288,18 +290,32 @@ def test_broadcast_shapes_skip_axes_errors(broadcastable_shapes, skip_axes): for i in skip_axes: shape[i] except IndexError: - error = True + indexerror = True else: - error = False + indexerror = False + broadcasterror = False try: broadcast_shapes(*shapes, skip_axes=skip_axes) except IndexError: - if not error: - raise RuntimeError("broadcast_shapes raised but should not have") + if not indexerror: + raise RuntimeError("broadcast_shapes raised IndexError but should not have") + except BroadcastError: + broadcasterror = True else: - if error: - raise RuntimeError("broadcast_shapes did not raise but should have") + if indexerror: + raise RuntimeError("broadcast_shapes did not raise IndexError but should have") + + if not indexerror: + broadcastable = [remove_indices(shape, skip_axes) for shape in shapes] + try: + np.broadcast_shapes(*broadcastable) + except ValueError: + if not broadcasterror: + raise RuntimeError("broadcast_shapes did not raise BroadcastError but should have") + else: + if broadcasterror: + raise RuntimeError("broadcast_shapes raised BroadcastError but should not have") remove_indices_n = shared(integers(0, 100)) From 1686269978eb32f6516c9be93e2677d3be8f431a Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 15:27:32 -0600 Subject: [PATCH 119/218] Remove todo that is completed --- ndindex/iterators.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index ebd08a31..90e11f7d 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -3,8 +3,6 @@ from .ndindex import asshape, ndindex -# TODO: Use this in other places in the code that check broadcast compatibility. - class BroadcastError(ValueError): """ Exception raised by :func:`iter_indices()` when the input shapes are not From 269ebb4efd2fed54d946a7f9c31433d048e59e58 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 18:58:09 -0600 Subject: [PATCH 120/218] Add allowint flag to asshape(), and handle some corner cases better --- ndindex/ndindex.py | 21 ++++++++++++++------- ndindex/tests/test_ndindex.py | 8 ++++++++ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/ndindex/ndindex.py b/ndindex/ndindex.py index 2ef18e29..77cd7e8a 100644 --- a/ndindex/ndindex.py +++ b/ndindex/ndindex.py @@ -2,6 +2,7 @@ import inspect import numbers import operator +from collections.abc import Sequence newaxis = None @@ -595,7 +596,7 @@ def broadcast_arrays(self): """ return self -def asshape(shape, axis=None): +def asshape(shape, axis=None, allowint=True): """ Cast `shape` as a valid NumPy shape. @@ -623,21 +624,27 @@ def asshape(shape, axis=None): "did you mean to use the built-in tuple type?") if isinstance(shape, numbers.Number): - shape = (operator_index(shape),) + if allowint: + shape = (operator_index(shape),) + else: + raise TypeError(f"expected sequence of integers, not {type(shape).__name__}") - try: - l = len(shape) - except TypeError: - raise TypeError("expected sequence object with len >= 0 or a single integer") + if not isinstance(shape, Sequence) or isinstance(shape, str): + raise TypeError("expected sequence of integers" + allowint*" or a single integer" + ", not " + type(shape).__name__) + l = len(shape) newshape = [] # numpy uses __getitem__ rather than __iter__ to index into shape, so we # match that for i in range(l): # Raise TypeError if invalid + val = shape[i] + if val is None: + raise ValueError("unknonwn (None) dimensions are not supported") + newshape.append(operator_index(shape[i])) - if shape[i] < 0: + if val < 0: raise ValueError("unknown (negative) dimensions are not supported") if axis is not None: diff --git a/ndindex/tests/test_ndindex.py b/ndindex/tests/test_ndindex.py index 706dd43a..061bcee7 100644 --- a/ndindex/tests/test_ndindex.py +++ b/ndindex/tests/test_ndindex.py @@ -144,6 +144,8 @@ def test_asshape(): assert type(asshape(np.int64(2))[0]) == int assert asshape((1, 2)) == (1, 2) assert asshape([1, 2]) == (1, 2) + assert asshape((1, 2), allowint=False) == (1, 2) + assert asshape([1, 2], allowint=False) == (1, 2) assert asshape((np.int64(1), np.int64(2))) == (1, 2) assert type(asshape((np.int64(1), np.int64(2)))[0]) == int assert type(asshape((np.int64(1), np.int64(2)))[1]) == int @@ -152,7 +154,13 @@ def test_asshape(): raises(TypeError, lambda: asshape((1.0,))) raises(ValueError, lambda: asshape(-1)) raises(ValueError, lambda: asshape((1, -1))) + raises(ValueError, lambda: asshape((1, None))) raises(TypeError, lambda: asshape(...)) raises(TypeError, lambda: asshape(Integer(1))) raises(TypeError, lambda: asshape(Tuple(1, 2))) raises(TypeError, lambda: asshape((True,))) + raises(TypeError, lambda: asshape({1, 2})) + raises(TypeError, lambda: asshape({1: 2})) + raises(TypeError, lambda: asshape('1')) + raises(TypeError, lambda: asshape(1, allowint=False)) + raises(TypeError, lambda: asshape(np.int64(1), allowint=False)) From 4a8f1d4c41356c9731e363c2a82132a0d4e42c64 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 19:02:52 -0600 Subject: [PATCH 121/218] Rename allowint to allow_int and add allow_negative in asshape() --- ndindex/ndindex.py | 15 ++++++++------- ndindex/tests/test_ndindex.py | 13 +++++++++---- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/ndindex/ndindex.py b/ndindex/ndindex.py index 77cd7e8a..db3c0943 100644 --- a/ndindex/ndindex.py +++ b/ndindex/ndindex.py @@ -596,17 +596,18 @@ def broadcast_arrays(self): """ return self -def asshape(shape, axis=None, allowint=True): +def asshape(shape, axis=None, *, allow_int=True, allow_negative=False): """ Cast `shape` as a valid NumPy shape. - The input can be an integer `n`, which is equivalent to `(n,)`, or a tuple - of integers. + The input can be an integer `n` (if `allow_int=True`), which is equivalent + to `(n,)`, or a tuple of integers. If the `axis` argument is provided, an `IndexError` is raised if it is out of bounds for the shape. - The resulting shape is always a tuple of nonnegative integers. + The resulting shape is always a tuple of nonnegative integers. If + `allow_negative=True`, negative integers are also allowed. All ndindex functions that take a shape input should use:: @@ -624,13 +625,13 @@ def asshape(shape, axis=None, allowint=True): "did you mean to use the built-in tuple type?") if isinstance(shape, numbers.Number): - if allowint: + if allow_int: shape = (operator_index(shape),) else: raise TypeError(f"expected sequence of integers, not {type(shape).__name__}") if not isinstance(shape, Sequence) or isinstance(shape, str): - raise TypeError("expected sequence of integers" + allowint*" or a single integer" + ", not " + type(shape).__name__) + raise TypeError("expected sequence of integers" + allow_int*" or a single integer" + ", not " + type(shape).__name__) l = len(shape) newshape = [] @@ -644,7 +645,7 @@ def asshape(shape, axis=None, allowint=True): newshape.append(operator_index(shape[i])) - if val < 0: + if not allow_negative and val < 0: raise ValueError("unknown (negative) dimensions are not supported") if axis is not None: diff --git a/ndindex/tests/test_ndindex.py b/ndindex/tests/test_ndindex.py index 061bcee7..031f59a9 100644 --- a/ndindex/tests/test_ndindex.py +++ b/ndindex/tests/test_ndindex.py @@ -144,11 +144,14 @@ def test_asshape(): assert type(asshape(np.int64(2))[0]) == int assert asshape((1, 2)) == (1, 2) assert asshape([1, 2]) == (1, 2) - assert asshape((1, 2), allowint=False) == (1, 2) - assert asshape([1, 2], allowint=False) == (1, 2) + assert asshape((1, 2), allow_int=False) == (1, 2) + assert asshape([1, 2], allow_int=False) == (1, 2) assert asshape((np.int64(1), np.int64(2))) == (1, 2) assert type(asshape((np.int64(1), np.int64(2)))[0]) == int assert type(asshape((np.int64(1), np.int64(2)))[1]) == int + assert asshape((-1, -2), allow_negative=True) == (-1, -2) + assert asshape(-2, allow_negative=True) == (-2,) + raises(TypeError, lambda: asshape(1.0)) raises(TypeError, lambda: asshape((1.0,))) @@ -162,5 +165,7 @@ def test_asshape(): raises(TypeError, lambda: asshape({1, 2})) raises(TypeError, lambda: asshape({1: 2})) raises(TypeError, lambda: asshape('1')) - raises(TypeError, lambda: asshape(1, allowint=False)) - raises(TypeError, lambda: asshape(np.int64(1), allowint=False)) + raises(TypeError, lambda: asshape(1, allow_int=False)) + raises(TypeError, lambda: asshape(-1, allow_int=False)) + raises(TypeError, lambda: asshape(-1, allow_negative=True, allow_int=False)) + raises(TypeError, lambda: asshape(np.int64(1), allow_int=False)) From d1d699c14c79425a733d73eb7caded07f679d11c Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 19:08:04 -0600 Subject: [PATCH 122/218] Use asshape() for type checking in broadcast_shapes() and iter_indices() --- ndindex/iterators.py | 8 ++++---- ndindex/tests/test_iterators.py | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index 90e11f7d..43d6a713 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -70,8 +70,8 @@ def broadcast_shapes(*shapes, skip_axes=()): (None, 3, 2) """ - if isinstance(skip_axes, int): - skip_axes = (skip_axes,) + skip_axes = asshape(skip_axes, allow_negative=True) + shapes = [asshape(shape, allow_int=False) for shape in shapes] if any(i >= 0 for i in skip_axes) and any(i < 0 for i in skip_axes): # See the comments in remove_indices and iter_indices @@ -217,8 +217,8 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): Tuple(1, 2) """ - if isinstance(skip_axes, int): - skip_axes = (skip_axes,) + skip_axes = asshape(skip_axes, allow_negative=True) + shapes = [asshape(shape, allow_int=False) for shape in shapes] if any(i >= 0 for i in skip_axes) and any(i < 0 for i in skip_axes): # Mixing positive and negative skip_axes is too difficult to deal with diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index 81a4e592..541d3d43 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -183,6 +183,10 @@ def test_iter_indices_errors(): with raises(ValueError, match=r"duplicate axes"): list(iter_indices((1, 2), skip_axes=(0, 1, 0))) + raises(TypeError, lambda: list(iter_indices(1, 2))) + raises(TypeError, lambda: list(iter_indices(1, 2, (2, 2)))) + raises(TypeError, lambda: list(iter_indices([(1, 2), (2, 2)]))) + @example(1, 1, 1) @given(integers(0, 100), integers(0, 100), integers(0, 100)) def test_ncycles(i, n, m): @@ -269,6 +273,10 @@ def test_broadcast_shapes_errors(shapes): else: raise RuntimeError("ndindex.broadcast_shapes raised but np.broadcast_shapes did not") + raises(TypeError, lambda: broadcast_shapes(1, 2)) + raises(TypeError, lambda: broadcast_shapes(1, 2, (2, 2))) + raises(TypeError, lambda: broadcast_shapes([(1, 2), (2, 2)])) + @given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes_st) def test_broadcast_shapes_skip_axes(broadcastable_shapes, skip_axes): shapes, broadcasted_shape = broadcastable_shapes From 61066afbc43066bc43d4a8bd740dc0ce99bca822 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 19:08:28 -0600 Subject: [PATCH 123/218] Note that some functions are internal only --- ndindex/iterators.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index 43d6a713..946cfe57 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -303,6 +303,8 @@ def associated_axis(shape, broadcasted_shape, i, skip_axes): Return the associated index into broadcast_shape corresponding to shape[i] given skip_axes. + This function makes implicit assumptions about its input and is only + designed for internal use. """ n = len(shape) N = len(broadcasted_shape) @@ -332,6 +334,8 @@ def associated_axis(shape, broadcasted_shape, i, skip_axes): def remove_indices(x, idxes): """ Return `x` with the indices `idxes` removed. + + This function is only intended for internal usage. """ if isinstance(idxes, int): idxes = (idxes,) @@ -347,7 +351,9 @@ def unremove_indices(x, idxes, *, val=None): """ Insert `val` in `x` so that it appears at `idxes`. - Note that idxes must be either all negative or all nonnegative + Note that idxes must be either all negative or all nonnegative. + + This function is only intended for internal usage. """ if any(i >= 0 for i in idxes) and any(i < 0 for i in idxes): # A mix of positive and negative indices provides a fundamental @@ -381,6 +387,8 @@ class ncycles: but improved to give a repr, and to denest when it can. This makes debugging :func:`~.iter_indices` easier. + This is only intended for internal usage. + >>> from ndindex.iterators import ncycles >>> ncycles(range(3), 2) ncycles(range(0, 3), 2) From 835dba887682c32caf0b94b14f5343fd6bf09deb Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 19:14:54 -0600 Subject: [PATCH 124/218] Fix the docs build --- docs/api.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 4676410c..0126e030 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -6,7 +6,6 @@ The ndindex API consists of classes representing the different types of index objects (integers, slices, etc.), as well as some helper functions for dealing with indices. - ndindex ======= @@ -61,6 +60,8 @@ the index objects. .. autofunction:: ndindex.iter_indices +.. autofunction:: ndindex.broadcast_shapes + .. autoexception:: ndindex.BroadcastError .. autoexception:: ndindex.AxisError @@ -98,6 +99,4 @@ relied on as they may be removed or changed. .. autofunction:: ndindex.ndindex.operator_index -.. autofunction:: ndindex.ndindex.ncycles - -.. autofunction:: ndindex.ndindex.broadcast_shapes +.. autofunction:: ndindex.iterators.ncycles From 6ffdebb685c4007f6037abccda60e5e9761aef85 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 19:22:38 -0600 Subject: [PATCH 125/218] Remove some dead code from the test helpers --- ndindex/tests/helpers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index b7d210f2..c4fd299c 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -145,8 +145,6 @@ def mutually_broadcastable_shapes_with_skipped_axes(draw): """ skip_axes_ = draw(skip_axes_st) shapes, result_shape = draw(mutually_broadcastable_shapes) - if skip_axes_ is None: - return shapes, result_shape if isinstance(skip_axes_, int): skip_axes_ = (skip_axes_,) From 751bfa1d04cef29e0ce7664d764e9583afac80ff Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 19:27:37 -0600 Subject: [PATCH 126/218] Add some # pragma: no covers --- ndindex/tests/test_iterators.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index 541d3d43..b44bf75b 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -257,7 +257,7 @@ def test_broadcast_shapes_errors(shapes): if not error: try: np.broadcast_shapes(*shapes) - except: + except: # pragma: no cover raise RuntimeError("ndindex.broadcast_shapes raised but np.broadcast_shapes did not") return @@ -270,7 +270,7 @@ def test_broadcast_shapes_errors(shapes): # Check that they do in fact not broadcast, and the error messages are # the same modulo the different arg positions. assert str(BroadcastError(0, e.shape1, 1, e.shape2)) == str(np_exc) - else: + else: # pragma: no cover raise RuntimeError("ndindex.broadcast_shapes raised but np.broadcast_shapes did not") raises(TypeError, lambda: broadcast_shapes(1, 2)) @@ -306,12 +306,12 @@ def test_broadcast_shapes_skip_axes_errors(broadcastable_shapes, skip_axes): try: broadcast_shapes(*shapes, skip_axes=skip_axes) except IndexError: - if not indexerror: + if not indexerror: # pragma: no cover raise RuntimeError("broadcast_shapes raised IndexError but should not have") except BroadcastError: broadcasterror = True else: - if indexerror: + if indexerror: # pragma: no cover raise RuntimeError("broadcast_shapes did not raise IndexError but should have") if not indexerror: @@ -319,10 +319,10 @@ def test_broadcast_shapes_skip_axes_errors(broadcastable_shapes, skip_axes): try: np.broadcast_shapes(*broadcastable) except ValueError: - if not broadcasterror: + if not broadcasterror: # pragma: no cover raise RuntimeError("broadcast_shapes did not raise BroadcastError but should have") else: - if broadcasterror: + if broadcasterror: # pragma: no cover raise RuntimeError("broadcast_shapes raised BroadcastError but should not have") remove_indices_n = shared(integers(0, 100)) From 7ff2a4ef1d8b0c58722abf90a68f45c2731b7abb Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 19:32:25 -0600 Subject: [PATCH 127/218] Revert "Fix test_broadcast_shapes_skip_axes_errors() to handle BroadcastError" This reverts commit 3a894efb7870f72d589d825c05032241da4f56bc. --- ndindex/tests/test_iterators.py | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index b44bf75b..7b8bff3c 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -289,8 +289,6 @@ def test_broadcast_shapes_skip_axes_errors(broadcastable_shapes, skip_axes): raises(NotImplementedError, lambda: broadcast_shapes(*shapes, skip_axes=skip_axes)) return - # Test that broadcast_shapes raises IndexError when skip_axes are out of - # bounds try: if not shapes and skip_axes: raise IndexError @@ -298,32 +296,18 @@ def test_broadcast_shapes_skip_axes_errors(broadcastable_shapes, skip_axes): for i in skip_axes: shape[i] except IndexError: - indexerror = True + error = True else: - indexerror = False + error = False - broadcasterror = False try: broadcast_shapes(*shapes, skip_axes=skip_axes) except IndexError: - if not indexerror: # pragma: no cover - raise RuntimeError("broadcast_shapes raised IndexError but should not have") - except BroadcastError: - broadcasterror = True + if not error: # pragma: no cover + raise RuntimeError("broadcast_shapes raised but should not have") else: - if indexerror: # pragma: no cover - raise RuntimeError("broadcast_shapes did not raise IndexError but should have") - - if not indexerror: - broadcastable = [remove_indices(shape, skip_axes) for shape in shapes] - try: - np.broadcast_shapes(*broadcastable) - except ValueError: - if not broadcasterror: # pragma: no cover - raise RuntimeError("broadcast_shapes did not raise BroadcastError but should have") - else: - if broadcasterror: # pragma: no cover - raise RuntimeError("broadcast_shapes raised BroadcastError but should not have") + if error: # pragma: no cover + raise RuntimeError("broadcast_shapes did not raise but should have") remove_indices_n = shared(integers(0, 100)) From 50a42335627e61bf3de0880fae7befe832a0d9ea Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 19:34:26 -0600 Subject: [PATCH 128/218] Add some @examples for coverage --- ndindex/tests/test_iterators.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index 7b8bff3c..dd96256b 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -282,6 +282,8 @@ def test_broadcast_shapes_skip_axes(broadcastable_shapes, skip_axes): shapes, broadcasted_shape = broadcastable_shapes assert broadcast_shapes(*shapes, skip_axes=skip_axes) == broadcasted_shape +@example([[(0, 1)], (0, 1)], (2,)) +@example([[(0, 1)], (0, 1)], (0, -1)) @given(mutually_broadcastable_shapes, lists(integers(-20, 20), max_size=20)) def test_broadcast_shapes_skip_axes_errors(broadcastable_shapes, skip_axes): shapes, broadcasted_shape = broadcastable_shapes From c6a36aa048e41b9ff634ace8e4c51fbf76f4574e Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 19:38:42 -0600 Subject: [PATCH 129/218] Handle BroadcastErrors in test_broadcast_shapes_skip_axes_errors() (correctly this time) --- ndindex/tests/test_iterators.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index dd96256b..015a38ec 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -307,9 +307,12 @@ def test_broadcast_shapes_skip_axes_errors(broadcastable_shapes, skip_axes): except IndexError: if not error: # pragma: no cover raise RuntimeError("broadcast_shapes raised but should not have") - else: - if error: # pragma: no cover - raise RuntimeError("broadcast_shapes did not raise but should have") + return + except BroadcastError: + pass + + if error: # pragma: no cover + raise RuntimeError("broadcast_shapes did not raise but should have") remove_indices_n = shared(integers(0, 100)) From e1bf8731a5155daa2e110ed167e0f60d466eaeab Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 19:43:57 -0600 Subject: [PATCH 130/218] Remove some dead code --- ndindex/iterators.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index 946cfe57..5119484a 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -249,10 +249,6 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): ndim = len(max(shapes, key=len)) min_ndim = len(min(shapes, key=len)) - if isinstance(skip_axes, int): - skip_axes = (skip_axes,) - - _skip_axes = defaultdict(list) for shape in shapes: for a in skip_axes: From 14f28812dda10a5cddea950207bfa5d98184d372 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 19:44:05 -0600 Subject: [PATCH 131/218] Add a test for a given error case --- ndindex/tests/test_iterators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index 015a38ec..be38f242 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -183,6 +183,7 @@ def test_iter_indices_errors(): with raises(ValueError, match=r"duplicate axes"): list(iter_indices((1, 2), skip_axes=(0, 1, 0))) + raises(AxisError, lambda: list(iter_indices(skip_axes=(0,)))) raises(TypeError, lambda: list(iter_indices(1, 2))) raises(TypeError, lambda: list(iter_indices(1, 2, (2, 2)))) raises(TypeError, lambda: list(iter_indices([(1, 2), (2, 2)]))) From f27fa0d3b385ddddff5b582ef6294d387c880ca7 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 19:44:16 -0600 Subject: [PATCH 132/218] Add an @example for coverage --- ndindex/tests/test_iterators.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_iterators.py index be38f242..040d016a 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_iterators.py @@ -285,6 +285,7 @@ def test_broadcast_shapes_skip_axes(broadcastable_shapes, skip_axes): @example([[(0, 1)], (0, 1)], (2,)) @example([[(0, 1)], (0, 1)], (0, -1)) +@example([[(0, 1, 0, 0, 0), (2, 0, 0, 0)], (0, 2, 0, 0, 0)], [1]) @given(mutually_broadcastable_shapes, lists(integers(-20, 20), max_size=20)) def test_broadcast_shapes_skip_axes_errors(broadcastable_shapes, skip_axes): shapes, broadcasted_shape = broadcastable_shapes @@ -310,6 +311,8 @@ def test_broadcast_shapes_skip_axes_errors(broadcastable_shapes, skip_axes): raise RuntimeError("broadcast_shapes raised but should not have") return except BroadcastError: + # Broadcastable shapes can become unbroadcastable after skipping axes + # (see the @example above). pass if error: # pragma: no cover From 722516ad9c08bf5c4188aeea5ccf0efc04077ef7 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 19:47:58 -0600 Subject: [PATCH 133/218] Replace if check that shouldn't ever happen with an assertion --- ndindex/iterators.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ndindex/iterators.py b/ndindex/iterators.py index 5119484a..64412626 100644 --- a/ndindex/iterators.py +++ b/ndindex/iterators.py @@ -277,9 +277,8 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): else: idx = associated_axis(shape, broadcasted_shape, i, skip_axes) val = broadcasted_shape[idx] - if val is None: - pass - elif val == 0: + assert val is not None + if val == 0: return elif val != 1 and shape[i] == 1: it.insert(0, ncycles(range(shape[i]), val)) From da6e9dfaf1da51e74f1a2bc9abc5fcbb3f2df556 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 23:39:15 -0600 Subject: [PATCH 134/218] Rename the iterators submodule to shapetools The name doesn't really matter because people should only be using the top-level namespace for public API functions. But shapetools makes more sense given the functions that are actually there. --- docs/api.rst | 2 +- ndindex/__init__.py | 2 +- ndindex/{iterators.py => shapetools.py} | 4 ++-- ndindex/tests/helpers.py | 2 +- ndindex/tests/{test_iterators.py => test_shapetools.py} | 2 +- ndindex/tuple.py | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) rename ndindex/{iterators.py => shapetools.py} (98%) rename ndindex/tests/{test_iterators.py => test_shapetools.py} (99%) diff --git a/docs/api.rst b/docs/api.rst index 0126e030..6648c3b0 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -99,4 +99,4 @@ relied on as they may be removed or changed. .. autofunction:: ndindex.ndindex.operator_index -.. autofunction:: ndindex.iterators.ncycles +.. autofunction:: ndindex.shapetools.ncycles diff --git a/ndindex/__init__.py b/ndindex/__init__.py index d39ac3c4..debe4015 100644 --- a/ndindex/__init__.py +++ b/ndindex/__init__.py @@ -4,7 +4,7 @@ __all__ += ['ndindex'] -from .iterators import broadcast_shapes, iter_indices, AxisError, BroadcastError +from .shapetools import broadcast_shapes, iter_indices, AxisError, BroadcastError __all__ += ['broadcast_shapes', 'iter_indices', 'AxisError', 'BroadcastError'] diff --git a/ndindex/iterators.py b/ndindex/shapetools.py similarity index 98% rename from ndindex/iterators.py rename to ndindex/shapetools.py index 64412626..d0d413c9 100644 --- a/ndindex/iterators.py +++ b/ndindex/shapetools.py @@ -58,7 +58,7 @@ def broadcast_shapes(*shapes, skip_axes=()): >>> broadcast_shapes((2, 3), (5,), (4, 2, 1)) Traceback (most recent call last): ... - ndindex.iterators.BroadcastError: shape mismatch: objects cannot be broadcast to a single shape. Mismatch is between arg 0 with shape (2, 3) and arg 1 with shape (5,). + ndindex.shapetools.BroadcastError: shape mismatch: objects cannot be broadcast to a single shape. Mismatch is between arg 0 with shape (2, 3) and arg 1 with shape (5,). Axes in `skip_axes` apply to each shape *before* being broadcasted. Each shape will be broadcasted together with these axes removed. The dimensions @@ -384,7 +384,7 @@ class ncycles: This is only intended for internal usage. - >>> from ndindex.iterators import ncycles + >>> from ndindex.shapetools import ncycles >>> ncycles(range(3), 2) ncycles(range(0, 3), 2) >>> list(_) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index c4fd299c..17e2a8eb 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -16,7 +16,7 @@ mbs, BroadcastableShapes) from ..ndindex import ndindex -from ..iterators import remove_indices, unremove_indices +from ..shapetools import remove_indices, unremove_indices # Hypothesis strategies for generating indices. Note that some of these # strategies are nominally already defined in hypothesis, but we redefine them diff --git a/ndindex/tests/test_iterators.py b/ndindex/tests/test_shapetools.py similarity index 99% rename from ndindex/tests/test_iterators.py rename to ndindex/tests/test_shapetools.py index 040d016a..4029431c 100644 --- a/ndindex/tests/test_iterators.py +++ b/ndindex/tests/test_shapetools.py @@ -7,7 +7,7 @@ from pytest import raises from ..ndindex import ndindex -from ..iterators import (iter_indices, ncycles, BroadcastError, +from ..shapetools import (iter_indices, ncycles, BroadcastError, AxisError, broadcast_shapes, remove_indices, unremove_indices, associated_axis) from ..integer import Integer diff --git a/ndindex/tuple.py b/ndindex/tuple.py index de1f5d53..7158a697 100644 --- a/ndindex/tuple.py +++ b/ndindex/tuple.py @@ -2,7 +2,7 @@ from .ndindex import NDIndex, ndindex, asshape from .subindex_helpers import subindex_slice -from .iterators import broadcast_shapes, BroadcastError +from .shapetools import broadcast_shapes, BroadcastError class Tuple(NDIndex): """ From fdfc839c8b048f234a1bc838ef88e23c80c8dc54 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 23:45:53 -0600 Subject: [PATCH 135/218] Move asshape() to shapetools.py --- docs/api.rst | 4 +- ndindex/array.py | 3 +- ndindex/booleanarray.py | 2 +- ndindex/chunking.py | 3 +- ndindex/ellipsis.py | 3 +- ndindex/integer.py | 3 +- ndindex/integerarray.py | 2 +- ndindex/ndindex.py | 60 ----------------------------- ndindex/newaxis.py | 3 +- ndindex/shapetools.py | 65 +++++++++++++++++++++++++++++++- ndindex/slice.py | 3 +- ndindex/tests/test_ndindex.py | 35 +---------------- ndindex/tests/test_shapetools.py | 38 +++++++++++++++++-- ndindex/tests/test_slice.py | 2 +- ndindex/tuple.py | 4 +- 15 files changed, 119 insertions(+), 111 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 6648c3b0..2cd2e1ae 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -95,8 +95,8 @@ relied on as they may be removed or changed. .. autoclass:: ndindex.slice.default -.. autofunction:: ndindex.ndindex.asshape - .. autofunction:: ndindex.ndindex.operator_index +.. autofunction:: ndindex.shapetools.asshape + .. autofunction:: ndindex.shapetools.ncycles diff --git a/ndindex/array.py b/ndindex/array.py index 390cc220..179b7433 100644 --- a/ndindex/array.py +++ b/ndindex/array.py @@ -1,6 +1,7 @@ import warnings -from .ndindex import NDIndex, asshape +from .ndindex import NDIndex +from .shapetools import asshape class ArrayIndex(NDIndex): """ diff --git a/ndindex/booleanarray.py b/ndindex/booleanarray.py index da96b3aa..325f209f 100644 --- a/ndindex/booleanarray.py +++ b/ndindex/booleanarray.py @@ -1,5 +1,5 @@ from .array import ArrayIndex -from .ndindex import asshape +from .shapetools import asshape class BooleanArray(ArrayIndex): """ diff --git a/ndindex/chunking.py b/ndindex/chunking.py index 60f83d77..5e3e9517 100644 --- a/ndindex/chunking.py +++ b/ndindex/chunking.py @@ -1,12 +1,13 @@ from collections.abc import Sequence from itertools import product -from .ndindex import ImmutableObject, operator_index, asshape, ndindex +from .ndindex import ImmutableObject, operator_index, ndindex from .tuple import Tuple from .slice import Slice from .integer import Integer from .integerarray import IntegerArray from .newaxis import Newaxis +from .shapetools import asshape from .subindex_helpers import ceiling from ._crt import prod diff --git a/ndindex/ellipsis.py b/ndindex/ellipsis.py index 9d6ce64a..07755ed1 100644 --- a/ndindex/ellipsis.py +++ b/ndindex/ellipsis.py @@ -1,5 +1,6 @@ -from .ndindex import NDIndex, asshape +from .ndindex import NDIndex from .tuple import Tuple +from .shapetools import asshape class ellipsis(NDIndex): """ diff --git a/ndindex/integer.py b/ndindex/integer.py index 0dc490b3..9e454572 100644 --- a/ndindex/integer.py +++ b/ndindex/integer.py @@ -1,4 +1,5 @@ -from .ndindex import NDIndex, asshape, operator_index +from .ndindex import NDIndex, operator_index +from .shapetools import asshape class Integer(NDIndex): """ diff --git a/ndindex/integerarray.py b/ndindex/integerarray.py index 98d679fc..df2d87af 100644 --- a/ndindex/integerarray.py +++ b/ndindex/integerarray.py @@ -1,5 +1,5 @@ from .array import ArrayIndex -from .ndindex import asshape +from .shapetools import asshape from .subindex_helpers import subindex_slice class IntegerArray(ArrayIndex): diff --git a/ndindex/ndindex.py b/ndindex/ndindex.py index db3c0943..6075a031 100644 --- a/ndindex/ndindex.py +++ b/ndindex/ndindex.py @@ -1,8 +1,6 @@ import sys import inspect -import numbers import operator -from collections.abc import Sequence newaxis = None @@ -596,64 +594,6 @@ def broadcast_arrays(self): """ return self -def asshape(shape, axis=None, *, allow_int=True, allow_negative=False): - """ - Cast `shape` as a valid NumPy shape. - - The input can be an integer `n` (if `allow_int=True`), which is equivalent - to `(n,)`, or a tuple of integers. - - If the `axis` argument is provided, an `IndexError` is raised if it is out - of bounds for the shape. - - The resulting shape is always a tuple of nonnegative integers. If - `allow_negative=True`, negative integers are also allowed. - - All ndindex functions that take a shape input should use:: - - shape = asshape(shape) - - or:: - - shape = asshape(shape, axis=axis) - - """ - from .integer import Integer - from .tuple import Tuple - if isinstance(shape, (Tuple, Integer)): - raise TypeError("ndindex types are not meant to be used as a shape - " - "did you mean to use the built-in tuple type?") - - if isinstance(shape, numbers.Number): - if allow_int: - shape = (operator_index(shape),) - else: - raise TypeError(f"expected sequence of integers, not {type(shape).__name__}") - - if not isinstance(shape, Sequence) or isinstance(shape, str): - raise TypeError("expected sequence of integers" + allow_int*" or a single integer" + ", not " + type(shape).__name__) - l = len(shape) - - newshape = [] - # numpy uses __getitem__ rather than __iter__ to index into shape, so we - # match that - for i in range(l): - # Raise TypeError if invalid - val = shape[i] - if val is None: - raise ValueError("unknonwn (None) dimensions are not supported") - - newshape.append(operator_index(shape[i])) - - if not allow_negative and val < 0: - raise ValueError("unknown (negative) dimensions are not supported") - - if axis is not None: - if len(newshape) <= axis: - raise IndexError(f"too many indices for array: array is {len(shape)}-dimensional, but {axis + 1} were indexed") - - return tuple(newshape) - def operator_index(idx): """ Convert `idx` into an integer index using `__index__()` or raise diff --git a/ndindex/newaxis.py b/ndindex/newaxis.py index d86f57b9..18b2300a 100644 --- a/ndindex/newaxis.py +++ b/ndindex/newaxis.py @@ -1,4 +1,5 @@ -from .ndindex import NDIndex, asshape +from .ndindex import NDIndex +from .shapetools import asshape class Newaxis(NDIndex): """ diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index d0d413c9..61f42f45 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -1,7 +1,9 @@ +import numbers import itertools from collections import defaultdict +from collections.abc import Sequence -from .ndindex import asshape, ndindex +from .ndindex import ndindex, operator_index class BroadcastError(ValueError): """ @@ -293,6 +295,67 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): iters], fillvalue=()): yield tuple(ndindex(idx) for idx in idxes) +#### Internal helpers + + +def asshape(shape, axis=None, *, allow_int=True, allow_negative=False): + """ + Cast `shape` as a valid NumPy shape. + + The input can be an integer `n` (if `allow_int=True`), which is equivalent + to `(n,)`, or a tuple of integers. + + If the `axis` argument is provided, an `IndexError` is raised if it is out + of bounds for the shape. + + The resulting shape is always a tuple of nonnegative integers. If + `allow_negative=True`, negative integers are also allowed. + + All ndindex functions that take a shape input should use:: + + shape = asshape(shape) + + or:: + + shape = asshape(shape, axis=axis) + + """ + from .integer import Integer + from .tuple import Tuple + if isinstance(shape, (Tuple, Integer)): + raise TypeError("ndindex types are not meant to be used as a shape - " + "did you mean to use the built-in tuple type?") + + if isinstance(shape, numbers.Number): + if allow_int: + shape = (operator_index(shape),) + else: + raise TypeError(f"expected sequence of integers, not {type(shape).__name__}") + + if not isinstance(shape, Sequence) or isinstance(shape, str): + raise TypeError("expected sequence of integers" + allow_int*" or a single integer" + ", not " + type(shape).__name__) + l = len(shape) + + newshape = [] + # numpy uses __getitem__ rather than __iter__ to index into shape, so we + # match that + for i in range(l): + # Raise TypeError if invalid + val = shape[i] + if val is None: + raise ValueError("unknonwn (None) dimensions are not supported") + + newshape.append(operator_index(shape[i])) + + if not allow_negative and val < 0: + raise ValueError("unknown (negative) dimensions are not supported") + + if axis is not None: + if len(newshape) <= axis: + raise IndexError(f"too many indices for array: array is {len(shape)}-dimensional, but {axis + 1} were indexed") + + return tuple(newshape) + def associated_axis(shape, broadcasted_shape, i, skip_axes): """ Return the associated index into broadcast_shape corresponding to diff --git a/ndindex/slice.py b/ndindex/slice.py index 51b757f0..70405a28 100644 --- a/ndindex/slice.py +++ b/ndindex/slice.py @@ -1,5 +1,6 @@ -from .ndindex import NDIndex, asshape, operator_index +from .ndindex import NDIndex, operator_index from .subindex_helpers import subindex_slice +from .shapetools import asshape class default: """ diff --git a/ndindex/tests/test_ndindex.py b/ndindex/tests/test_ndindex.py index bdfe45a5..8b537a99 100644 --- a/ndindex/tests/test_ndindex.py +++ b/ndindex/tests/test_ndindex.py @@ -7,12 +7,11 @@ from pytest import raises -from ..ndindex import ndindex, asshape +from ..ndindex import ndindex from ..booleanarray import BooleanArray from ..integer import Integer from ..ellipsis import ellipsis from ..integerarray import IntegerArray -from ..tuple import Tuple from .helpers import ndindices, check_same, assert_equal @@ -138,35 +137,3 @@ def test_repr_str(idx): # Str may not be re-creatable. Just test that it doesn't give an exception. str(index) - -def test_asshape(): - assert asshape(1) == (1,) - assert asshape(np.int64(2)) == (2,) - assert type(asshape(np.int64(2))[0]) == int - assert asshape((1, 2)) == (1, 2) - assert asshape([1, 2]) == (1, 2) - assert asshape((1, 2), allow_int=False) == (1, 2) - assert asshape([1, 2], allow_int=False) == (1, 2) - assert asshape((np.int64(1), np.int64(2))) == (1, 2) - assert type(asshape((np.int64(1), np.int64(2)))[0]) == int - assert type(asshape((np.int64(1), np.int64(2)))[1]) == int - assert asshape((-1, -2), allow_negative=True) == (-1, -2) - assert asshape(-2, allow_negative=True) == (-2,) - - - raises(TypeError, lambda: asshape(1.0)) - raises(TypeError, lambda: asshape((1.0,))) - raises(ValueError, lambda: asshape(-1)) - raises(ValueError, lambda: asshape((1, -1))) - raises(ValueError, lambda: asshape((1, None))) - raises(TypeError, lambda: asshape(...)) - raises(TypeError, lambda: asshape(Integer(1))) - raises(TypeError, lambda: asshape(Tuple(1, 2))) - raises(TypeError, lambda: asshape((True,))) - raises(TypeError, lambda: asshape({1, 2})) - raises(TypeError, lambda: asshape({1: 2})) - raises(TypeError, lambda: asshape('1')) - raises(TypeError, lambda: asshape(1, allow_int=False)) - raises(TypeError, lambda: asshape(-1, allow_int=False)) - raises(TypeError, lambda: asshape(-1, allow_negative=True, allow_int=False)) - raises(TypeError, lambda: asshape(np.int64(1), allow_int=False)) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 4029431c..f3c0a743 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -7,9 +7,9 @@ from pytest import raises from ..ndindex import ndindex -from ..shapetools import (iter_indices, ncycles, BroadcastError, - AxisError, broadcast_shapes, remove_indices, - unremove_indices, associated_axis) +from ..shapetools import (asshape, iter_indices, ncycles, BroadcastError, + AxisError, broadcast_shapes, remove_indices, + unremove_indices, associated_axis) from ..integer import Integer from ..tuple import Tuple from .helpers import (prod, mutually_broadcastable_shapes_with_skipped_axes, @@ -398,3 +398,35 @@ def test_associated_axis(broadcastable_shapes, skip_axes): ndindex(idx).reduce(ndim).raw - ndim in _skip_axes else: assert val == 1 or bval == val + +def test_asshape(): + assert asshape(1) == (1,) + assert asshape(np.int64(2)) == (2,) + assert type(asshape(np.int64(2))[0]) == int + assert asshape((1, 2)) == (1, 2) + assert asshape([1, 2]) == (1, 2) + assert asshape((1, 2), allow_int=False) == (1, 2) + assert asshape([1, 2], allow_int=False) == (1, 2) + assert asshape((np.int64(1), np.int64(2))) == (1, 2) + assert type(asshape((np.int64(1), np.int64(2)))[0]) == int + assert type(asshape((np.int64(1), np.int64(2)))[1]) == int + assert asshape((-1, -2), allow_negative=True) == (-1, -2) + assert asshape(-2, allow_negative=True) == (-2,) + + + raises(TypeError, lambda: asshape(1.0)) + raises(TypeError, lambda: asshape((1.0,))) + raises(ValueError, lambda: asshape(-1)) + raises(ValueError, lambda: asshape((1, -1))) + raises(ValueError, lambda: asshape((1, None))) + raises(TypeError, lambda: asshape(...)) + raises(TypeError, lambda: asshape(Integer(1))) + raises(TypeError, lambda: asshape(Tuple(1, 2))) + raises(TypeError, lambda: asshape((True,))) + raises(TypeError, lambda: asshape({1, 2})) + raises(TypeError, lambda: asshape({1: 2})) + raises(TypeError, lambda: asshape('1')) + raises(TypeError, lambda: asshape(1, allow_int=False)) + raises(TypeError, lambda: asshape(-1, allow_int=False)) + raises(TypeError, lambda: asshape(-1, allow_negative=True, allow_int=False)) + raises(TypeError, lambda: asshape(np.int64(1), allow_int=False)) diff --git a/ndindex/tests/test_slice.py b/ndindex/tests/test_slice.py index 6135cc9b..7d37427c 100644 --- a/ndindex/tests/test_slice.py +++ b/ndindex/tests/test_slice.py @@ -8,7 +8,7 @@ from ..slice import Slice from ..integer import Integer from ..ellipsis import ellipsis -from ..ndindex import asshape +from ..shapetools import asshape from .helpers import check_same, slices, prod, shapes, iterslice, assert_equal def test_slice_args(): diff --git a/ndindex/tuple.py b/ndindex/tuple.py index 7158a697..ea6229dc 100644 --- a/ndindex/tuple.py +++ b/ndindex/tuple.py @@ -1,8 +1,8 @@ import sys -from .ndindex import NDIndex, ndindex, asshape +from .ndindex import NDIndex, ndindex from .subindex_helpers import subindex_slice -from .shapetools import broadcast_shapes, BroadcastError +from .shapetools import asshape, broadcast_shapes, BroadcastError class Tuple(NDIndex): """ From 84fe6b9e2fb28e0bc3c4f64629cec1578ce4f11a Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 23:46:37 -0600 Subject: [PATCH 136/218] Remove duplicate prod() helper function --- ndindex/tests/helpers.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index 17e2a8eb..5d71ac2f 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -1,7 +1,5 @@ import sys from itertools import chain -from functools import reduce -from operator import mul from numpy import intp, bool_, array, broadcast_shapes import numpy.testing @@ -17,6 +15,7 @@ from ..ndindex import ndindex from ..shapetools import remove_indices, unremove_indices +from .._crt import prod # Hypothesis strategies for generating indices. Note that some of these # strategies are nominally already defined in hypothesis, but we redefine them @@ -24,10 +23,6 @@ # hypothesis's slices strategy does not generate slices with negative indices. # Similarly, hypothesis.extra.numpy.basic_indices only generates tuples. -# np.prod has overflow and math.prod is Python 3.8+ only -def prod(seq): - return reduce(mul, seq, 1) - nonnegative_ints = integers(0, 10) negative_ints = integers(-10, -1) ints = lambda: one_of(nonnegative_ints, negative_ints) From df2cfbb6899d97859c98d9d064b355bdb4686192 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 23:49:12 -0600 Subject: [PATCH 137/218] Add internal shapetools functions to the internal API reference --- docs/api.rst | 6 ++++++ ndindex/shapetools.py | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 2cd2e1ae..9ff9e894 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -100,3 +100,9 @@ relied on as they may be removed or changed. .. autofunction:: ndindex.shapetools.asshape .. autofunction:: ndindex.shapetools.ncycles + +.. autofunction:: ndindex.shapetools.associated_axis + +.. autofunction:: ndindex.shapetools.remove_indices + +.. autofunction:: ndindex.shapetools.unremove_indices diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index 61f42f45..e9806a32 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -358,11 +358,12 @@ def asshape(shape, axis=None, *, allow_int=True, allow_negative=False): def associated_axis(shape, broadcasted_shape, i, skip_axes): """ - Return the associated index into broadcast_shape corresponding to - shape[i] given skip_axes. + Return the associated index into `broadcast_shape` corresponding to + `shape[i]` given `skip_axes`. This function makes implicit assumptions about its input and is only designed for internal use. + """ n = len(shape) N = len(broadcasted_shape) From 9bc9e7b52fefd9870c24b155b2c14414a7d24bf6 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 23:53:28 -0600 Subject: [PATCH 138/218] Fix tests with older NumPy (which gets tested on the Python 3.7 CI) --- ndindex/tests/test_shapetools.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index f3c0a743..d050065e 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -270,7 +270,9 @@ def test_broadcast_shapes_errors(shapes): except ValueError as np_exc: # Check that they do in fact not broadcast, and the error messages are # the same modulo the different arg positions. - assert str(BroadcastError(0, e.shape1, 1, e.shape2)) == str(np_exc) + if 'Mismatch' in str(np_exc): + # Older versions of NumPy do not have the more helpful error message + assert str(BroadcastError(0, e.shape1, 1, e.shape2)) == str(np_exc) else: # pragma: no cover raise RuntimeError("ndindex.broadcast_shapes raised but np.broadcast_shapes did not") From ac6bfae4ad0418e725470f4c70ffe2cffdac763e Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 23:58:10 -0600 Subject: [PATCH 139/218] Add an @example for coverage --- ndindex/tests/test_shapetools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index d050065e..459fd686 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -372,6 +372,7 @@ def test_mutually_broadcastable_shapes_with_skipped_axes(broadcastable_shapes, assert None not in _broadcasted_shape assert broadcast_shapes(*_shapes) == _broadcasted_shape +@example([[(2, 10, 3, 4), (10, 3, 4)], (2, None, 3, 4)], (-3,)) @example([[(0, 10, 2, 3, 10, 4), (1, 10, 1, 0, 10, 2, 3, 4)], (1, None, 1, 0, None, 2, 3, 4)], (1, 4)) @example([[(2, 0, 3, 4)], (2, None, 3, 4)], (1,)) From 73028832a85a0b9128661fe3a0e9ee2ca3517be0 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 19 Apr 2023 23:59:48 -0600 Subject: [PATCH 140/218] Completely skip coverage for test_mutually_broadcastable_shapes_with_skipped_axes It is a meta-test of the hypothesis strategy so it wouldn't make sense to add explicit @examples, since the whole point of the test is to make sure that the inputs actually generated by the strategy actually work correctly. --- ndindex/tests/test_shapetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 459fd686..6b4835ef 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -357,7 +357,7 @@ def test_remove_indices(n, idxes): # Meta-test for the hypothesis strategy @given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes_st) def test_mutually_broadcastable_shapes_with_skipped_axes(broadcastable_shapes, - skip_axes): + skip_axes): # pragma: no cover shapes, broadcasted_shape = broadcastable_shapes _skip_axes = (skip_axes,) if isinstance(skip_axes, int) else skip_axes From b335bcefd7d2410a6e9fec0818ec7cc1146d75ad Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 20 Apr 2023 01:17:32 -0600 Subject: [PATCH 141/218] Add a practical test of iter_indices against np.cross --- ndindex/tests/helpers.py | 44 +++++++++++++++++-------- ndindex/tests/test_shapetools.py | 56 ++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 15 deletions(-) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index 5d71ac2f..df8044ed 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -58,7 +58,7 @@ def tuples(elements, *, min_size=0, max_size=None, unique_by=None, unique=False) @composite -def _mutually_broadcastable_shapes(draw): +def _mutually_broadcastable_shapes(draw, min_shapes=0, max_shapes=32, min_side=0): # mutually_broadcastable_shapes() with the default inputs doesn't generate # very interesting examples (see # https://github.com/HypothesisWorks/hypothesis/issues/3170). It's very @@ -77,19 +77,19 @@ def _mutually_broadcastable_shapes(draw): input_shapes, result_shape = draw( mbs( - num_shapes=32, + num_shapes=max_shapes, base_shape=base_shape, - min_side=0, + min_side=min_side, )) # The hypothesis mutually_broadcastable_shapes doesn't allow num_shapes to # be a strategy. It's tempting to do something like num_shapes = - # draw(integers(1, 32)), but this shrinks poorly. See + # draw(integers(min_shapes, max_shapes)), but this shrinks poorly. See # https://github.com/HypothesisWorks/hypothesis/issues/3151. So instead of - # using a strategy to draw the number of shapes, we just generate 32 + # using a strategy to draw the number of shapes, we just generate max_shapes # shapes and pick a subset of them. - final_input_shapes = draw(lists(sampled_from(input_shapes), min_size=0, max_size=32, - unique_by=id,)) + final_input_shapes = draw(lists(sampled_from(input_shapes), + min_size=min_shapes, max_size=max_shapes)) # Note: result_shape is input_shapes broadcasted with base_shape, but @@ -110,28 +110,39 @@ def _mutually_broadcastable_shapes(draw): mutually_broadcastable_shapes = shared(_mutually_broadcastable_shapes()) @composite -def _skip_axes_st(draw): +def _skip_axes_st(draw, + mutually_broadcastable_shapes=mutually_broadcastable_shapes, + num_skip_axes=None): shapes, result_shape = draw(mutually_broadcastable_shapes) if result_shape == (): + assume(num_skip_axes is None) return () negative = draw(booleans(), label='skip_axes < 0') N = len(min(shapes, key=len)) if N == 0: + assume(num_skip_axes is None) return () + if num_skip_axes is not None: + min_size = max_size = num_skip_axes + assume(len(s) >= num_skip_axes for s in shapes) + else: + min_size = 0 + max_size = None if negative: - axes = draw(one_of(lists(integers(-N, -1), unique=True))) + axes = draw(lists(integers(-N, -1), min_size=min_size, max_size=max_size, unique=True)) else: - axes = draw(one_of(lists(integers(0, N-1), unique=True))) + axes = draw(lists(integers(0, N-1), min_size=min_size, max_size=max_size, unique=True)) axes = tuple(axes) # Sometimes return an integer - if len(axes) == 1 and draw(booleans(), label='skip_axes integer'): # pragma: no cover + if num_skip_axes is None and len(axes) == 1 and draw(booleans(), label='skip_axes integer'): # pragma: no cover return axes[0] return axes skip_axes_st = shared(_skip_axes_st()) @composite -def mutually_broadcastable_shapes_with_skipped_axes(draw): +def mutually_broadcastable_shapes_with_skipped_axes(draw, skip_axes_st=skip_axes_st, mutually_broadcastable_shapes=mutually_broadcastable_shapes, +skip_axes_values=integers(0)): """ mutually_broadcastable_shapes except skip_axes() axes might not be broadcastable @@ -153,7 +164,7 @@ def mutually_broadcastable_shapes_with_skipped_axes(draw): # Replace None values with random values for j in range(len(_shape)): if _shape[j] is None: - _shape[j] = draw(integers(0)) + _shape[j] = draw(skip_axes_values) shapes_.append(tuple(_shape)) result_shape_ = unremove_indices(result_shape, skip_axes_) @@ -164,6 +175,13 @@ def mutually_broadcastable_shapes_with_skipped_axes(draw): assume(prod([i for i in shape if i]) < SHORT_MAX_ARRAY_SIZE) return BroadcastableShapes(shapes_, result_shape_) +two_mutually_broadcastable_shapes = shared(_mutually_broadcastable_shapes( + min_shapes=2, + max_shapes=2, + min_side=1)) +one_skip_axes = shared(_skip_axes_st( + mutually_broadcastable_shapes=two_mutually_broadcastable_shapes, + num_skip_axes=1)) # We need to make sure shapes for boolean arrays are generated in a way that # makes them related to the test array shape. Otherwise, it will be very diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 6b4835ef..32849c97 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -2,7 +2,9 @@ from hypothesis import assume, given, example from hypothesis.strategies import (one_of, integers, tuples as - hypothesis_tuples, just, lists, shared) + hypothesis_tuples, just, lists, shared, + composite, nothing) +from hypothesis.extra.numpy import arrays from pytest import raises @@ -14,7 +16,8 @@ from ..tuple import Tuple from .helpers import (prod, mutually_broadcastable_shapes_with_skipped_axes, skip_axes_st, mutually_broadcastable_shapes, tuples, - shapes) + shapes, two_mutually_broadcastable_shapes, + one_skip_axes, assert_equal) @example([((1, 1), (1, 1)), (None, 1)], (0,)) @example([((0,), (0,)), (None,)], (0,)) @@ -144,6 +147,55 @@ def _move_slices_to_end(idx): else: assert set(vals) == set(correct_vals) +cross_shapes = mutually_broadcastable_shapes_with_skipped_axes( + mutually_broadcastable_shapes=two_mutually_broadcastable_shapes, + skip_axes_st=one_skip_axes, + skip_axes_values=integers(3, 3)) + +@composite +def cross_arrays(draw): + broadcastable_shapes = draw(cross_shapes) + shapes, broadcasted_shape = broadcastable_shapes + + # Sanity check + assert len(shapes) == 2 + # We need to generate fairly random arrays. Otherwise, if they are too + # similar to each other, like two arange arrays would be, the cross + # product will be 0. We also disable the fill feature in arrays() for the + # same reason, as it would otherwise generate too many vectors that are + # colinear. + a = draw(arrays(dtype=int, shape=shapes[0], elements=integers(-100, 100), fill=nothing())) + b = draw(arrays(dtype=int, shape=shapes[1], elements=integers(-100, 100), fill=nothing())) + + return a, b + +@given(cross_arrays(), cross_shapes, one_skip_axes) +def test_iter_indices_cross(cross_arrays, broadcastable_shapes, skip_axes): + # Test iter_indices behavior against np.cross, which effectively skips the + # crossed axis. Note that we don't test against cross products of size 2 + # because a 2 x 2 cross product just returns the z-axis (i.e., it doesn't + # actually skip an axis in the result shape), and also that behavior is + # going to be removed in NumPy 2.0. + a, b = cross_arrays + shapes, broadcasted_shape = broadcastable_shapes + skip_axis = skip_axes[0] + + broadcasted_shape = list(broadcasted_shape) + # Remove None from the shape for iter_indices + broadcasted_shape[skip_axis] = 3 + broadcasted_shape = tuple(broadcasted_shape) + + res = np.cross(a, b, axisa=skip_axis, axisb=skip_axis, axisc=skip_axis) + assert res.shape == broadcasted_shape + + for idx1, idx2, idx3 in iter_indices(*shapes, broadcasted_shape, skip_axes=skip_axes): + assert a[idx1.raw].shape == (3,) + assert b[idx2.raw].shape == (3,) + assert_equal(np.cross( + a[idx1.raw], + b[idx2.raw]), + res[idx3.raw]) + def test_iter_indices_errors(): try: list(iter_indices((10,), skip_axes=(2,))) From 327daef8e443fc63af17b81cbd1807919a101887 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 20 Apr 2023 14:02:19 -0600 Subject: [PATCH 142/218] Move some definitions to a more logical place in the helpers.py file --- ndindex/tests/helpers.py | 89 ++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 45 deletions(-) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index df8044ed..32463a76 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -51,12 +51,55 @@ def tuples(elements, *, min_size=0, max_size=None, unique_by=None, unique=False) # See https://github.com/numpy/numpy/issues/15753 lambda shape: prod([i for i in shape if i]) < SHORT_MAX_ARRAY_SIZE) +# short_shapes should be used in place of shapes in any test function that +# uses ndindices, boolean_arrays, or tuples +short_shapes = shared(_short_shapes) + +_integer_arrays = arrays(intp, short_shapes) +integer_scalars = arrays(intp, ()).map(lambda x: x[()]) +integer_arrays = one_of(integer_scalars, _integer_arrays.flatmap(lambda x: one_of(just(x), just(x.tolist())))) + +# We need to make sure shapes for boolean arrays are generated in a way that +# makes them related to the test array shape. Otherwise, it will be very +# difficult for the boolean array index to match along the test array, which +# means we won't test any behavior other than IndexError. + +@composite +def subsequences(draw, sequence): + seq = draw(sequence) + start = draw(integers(0, max(0, len(seq)-1))) + stop = draw(integers(start, len(seq))) + return seq[start:stop] + +_boolean_arrays = arrays(bool_, one_of(subsequences(short_shapes), short_shapes)) +boolean_scalars = arrays(bool_, ()).map(lambda x: x[()]) +boolean_arrays = one_of(boolean_scalars, _boolean_arrays.flatmap(lambda x: one_of(just(x), just(x.tolist())))) + +def _doesnt_raise(idx): + try: + ndindex(idx) + except (IndexError, ValueError, NotImplementedError): + return False + return True + +Tuples = tuples(one_of(ellipses(), ints(), slices(), newaxes(), + integer_arrays, boolean_arrays)).filter(_doesnt_raise) + +ndindices = one_of( + ints(), + slices(), + ellipses(), + newaxes(), + Tuples, + integer_arrays, + boolean_arrays, +).filter(_doesnt_raise) + # Note: We could use something like this: # mutually_broadcastable_shapes = shared(integers(1, 32).flatmap(lambda i: mbs(num_shapes=i).filter( # lambda broadcastable_shapes: prod([i for i in broadcastable_shapes.result_shape if i]) < MAX_ARRAY_SIZE))) - @composite def _mutually_broadcastable_shapes(draw, min_shapes=0, max_shapes=32, min_side=0): # mutually_broadcastable_shapes() with the default inputs doesn't generate @@ -183,50 +226,6 @@ def mutually_broadcastable_shapes_with_skipped_axes(draw, skip_axes_st=skip_axes mutually_broadcastable_shapes=two_mutually_broadcastable_shapes, num_skip_axes=1)) -# We need to make sure shapes for boolean arrays are generated in a way that -# makes them related to the test array shape. Otherwise, it will be very -# difficult for the boolean array index to match along the test array, which -# means we won't test any behavior other than IndexError. - -# short_shapes should be used in place of shapes in any test function that -# uses ndindices, boolean_arrays, or tuples -short_shapes = shared(_short_shapes) - -_integer_arrays = arrays(intp, short_shapes) -integer_scalars = arrays(intp, ()).map(lambda x: x[()]) -integer_arrays = one_of(integer_scalars, _integer_arrays.flatmap(lambda x: one_of(just(x), just(x.tolist())))) - -@composite -def subsequences(draw, sequence): - seq = draw(sequence) - start = draw(integers(0, max(0, len(seq)-1))) - stop = draw(integers(start, len(seq))) - return seq[start:stop] - -_boolean_arrays = arrays(bool_, one_of(subsequences(short_shapes), short_shapes)) -boolean_scalars = arrays(bool_, ()).map(lambda x: x[()]) -boolean_arrays = one_of(boolean_scalars, _boolean_arrays.flatmap(lambda x: one_of(just(x), just(x.tolist())))) - -def _doesnt_raise(idx): - try: - ndindex(idx) - except (IndexError, ValueError, NotImplementedError): - return False - return True - -Tuples = tuples(one_of(ellipses(), ints(), slices(), newaxes(), - integer_arrays, boolean_arrays)).filter(_doesnt_raise) - -ndindices = one_of( - ints(), - slices(), - ellipses(), - newaxes(), - Tuples, - integer_arrays, - boolean_arrays, -).filter(_doesnt_raise) - def assert_equal(actual, desired, err_msg='', verbose=True): """ Same as numpy.testing.assert_equal except it also requires the shapes and From 8dfc1636a0838ca97e0d97c4bb0d73e666f6d4bc Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 20 Apr 2023 14:05:32 -0600 Subject: [PATCH 143/218] Add test_iter_indices_matmul() --- ndindex/tests/helpers.py | 28 +++++++----- ndindex/tests/test_shapetools.py | 74 +++++++++++++++++++++++++++++--- 2 files changed, 87 insertions(+), 15 deletions(-) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index 32463a76..a65bf3e6 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -46,14 +46,14 @@ def tuples(elements, *, min_size=0, max_size=None, unique_by=None, unique=False) # See https://github.com/numpy/numpy/issues/15753 lambda shape: prod([i for i in shape if i]) < MAX_ARRAY_SIZE) -_short_shapes = tuples(integers(0, 10)).filter( +_short_shapes = lambda n: tuples(integers(0, 10), min_size=n).filter( # numpy gives errors with empty arrays with large shapes. # See https://github.com/numpy/numpy/issues/15753 lambda shape: prod([i for i in shape if i]) < SHORT_MAX_ARRAY_SIZE) # short_shapes should be used in place of shapes in any test function that # uses ndindices, boolean_arrays, or tuples -short_shapes = shared(_short_shapes) +short_shapes = shared(_short_shapes(0)) _integer_arrays = arrays(intp, short_shapes) integer_scalars = arrays(intp, ()).map(lambda x: x[()]) @@ -101,7 +101,7 @@ def _doesnt_raise(idx): # lambda broadcastable_shapes: prod([i for i in broadcastable_shapes.result_shape if i]) < MAX_ARRAY_SIZE))) @composite -def _mutually_broadcastable_shapes(draw, min_shapes=0, max_shapes=32, min_side=0): +def _mutually_broadcastable_shapes(draw, *, shapes=short_shapes, min_shapes=0, max_shapes=32, min_side=0): # mutually_broadcastable_shapes() with the default inputs doesn't generate # very interesting examples (see # https://github.com/HypothesisWorks/hypothesis/issues/3170). It's very @@ -116,7 +116,7 @@ def _mutually_broadcastable_shapes(draw, min_shapes=0, max_shapes=32, min_side=0 # like. But it generates enough "real" interesting shapes that both of # these workarounds are worth doing (plus I don't know if any other better # way of handling the situation). - base_shape = draw(short_shapes) + base_shape = draw(shapes) input_shapes, result_shape = draw( mbs( @@ -162,15 +162,14 @@ def _skip_axes_st(draw, return () negative = draw(booleans(), label='skip_axes < 0') N = len(min(shapes, key=len)) - if N == 0: - assume(num_skip_axes is None) - return () if num_skip_axes is not None: min_size = max_size = num_skip_axes - assume(len(s) >= num_skip_axes for s in shapes) + assume(N >= num_skip_axes) else: min_size = 0 max_size = None + if N == 0: + return () if negative: axes = draw(lists(integers(-N, -1), min_size=min_size, max_size=max_size, unique=True)) else: @@ -218,13 +217,22 @@ def mutually_broadcastable_shapes_with_skipped_axes(draw, skip_axes_st=skip_axes assume(prod([i for i in shape if i]) < SHORT_MAX_ARRAY_SIZE) return BroadcastableShapes(shapes_, result_shape_) -two_mutually_broadcastable_shapes = shared(_mutually_broadcastable_shapes( +two_mutually_broadcastable_shapes_1 = shared(_mutually_broadcastable_shapes( + shapes=_short_shapes(1), min_shapes=2, max_shapes=2, min_side=1)) one_skip_axes = shared(_skip_axes_st( - mutually_broadcastable_shapes=two_mutually_broadcastable_shapes, + mutually_broadcastable_shapes=two_mutually_broadcastable_shapes_1, num_skip_axes=1)) +two_mutually_broadcastable_shapes_2 = shared(_mutually_broadcastable_shapes( + shapes=_short_shapes(2), + min_shapes=2, + max_shapes=2, + min_side=2)) +two_skip_axes = shared(_skip_axes_st( + mutually_broadcastable_shapes=two_mutually_broadcastable_shapes_2, + num_skip_axes=2)) def assert_equal(actual, desired, err_msg='', verbose=True): """ diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 32849c97..cbdb958e 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -16,8 +16,9 @@ from ..tuple import Tuple from .helpers import (prod, mutually_broadcastable_shapes_with_skipped_axes, skip_axes_st, mutually_broadcastable_shapes, tuples, - shapes, two_mutually_broadcastable_shapes, - one_skip_axes, assert_equal) + shapes, two_mutually_broadcastable_shapes_1, + two_mutually_broadcastable_shapes_2, one_skip_axes, + two_skip_axes, assert_equal) @example([((1, 1), (1, 1)), (None, 1)], (0,)) @example([((0,), (0,)), (None,)], (0,)) @@ -148,12 +149,12 @@ def _move_slices_to_end(idx): assert set(vals) == set(correct_vals) cross_shapes = mutually_broadcastable_shapes_with_skipped_axes( - mutually_broadcastable_shapes=two_mutually_broadcastable_shapes, + mutually_broadcastable_shapes=two_mutually_broadcastable_shapes_1, skip_axes_st=one_skip_axes, skip_axes_values=integers(3, 3)) @composite -def cross_arrays(draw): +def cross_arrays_st(draw): broadcastable_shapes = draw(cross_shapes) shapes, broadcasted_shape = broadcastable_shapes @@ -169,7 +170,7 @@ def cross_arrays(draw): return a, b -@given(cross_arrays(), cross_shapes, one_skip_axes) +@given(cross_arrays_st(), cross_shapes, one_skip_axes) def test_iter_indices_cross(cross_arrays, broadcastable_shapes, skip_axes): # Test iter_indices behavior against np.cross, which effectively skips the # crossed axis. Note that we don't test against cross products of size 2 @@ -196,6 +197,69 @@ def test_iter_indices_cross(cross_arrays, broadcastable_shapes, skip_axes): b[idx2.raw]), res[idx3.raw]) + +@composite +def _matmul_shapes(draw): + broadcastable_shapes = draw(mutually_broadcastable_shapes_with_skipped_axes( + mutually_broadcastable_shapes=two_mutually_broadcastable_shapes_2, + skip_axes_st=two_skip_axes, + skip_axes_values=just(None), + )) + shapes, broadcasted_shape = broadcastable_shapes + skip_axes = draw(two_skip_axes) + # (n, m) @ (m, k) -> (n, k) + n, m, k = draw(hypothesis_tuples(integers(0, 10), integers(0, 10), + integers(0, 10))) + + shape1, shape2 = map(list, shapes) + ax1, ax2 = skip_axes + shape1[ax1] = n + shape1[ax2] = m + shape2[ax1] = m + shape2[ax2] = k + broadcasted_shape = list(broadcasted_shape) + broadcasted_shape[ax1] = n + broadcasted_shape[ax2] = k + return [tuple(shape1), tuple(shape2)], tuple(broadcasted_shape) + +matmul_shapes = shared(_matmul_shapes()) + +@composite +def matmul_arrays_st(draw): + broadcastable_shapes = draw(matmul_shapes) + shapes, broadcasted_shape = broadcastable_shapes + + # Sanity check + assert len(shapes) == 2 + a = draw(arrays(dtype=int, shape=shapes[0], elements=integers(-100, 100))) + b = draw(arrays(dtype=int, shape=shapes[1], elements=integers(-100, 100))) + + return a, b + +@given(matmul_arrays_st(), matmul_shapes, two_skip_axes) +def test_iter_indices_matmul(matmul_arrays, broadcastable_shapes, skip_axes): + # Test iter_indices behavior against np.matmul, which effectively skips the + # contracted axis (they aren't broadcasted together, even when they are + # broadcast compatible). + a, b = matmul_arrays + shapes, broadcasted_shape = broadcastable_shapes + + ax1, ax2 = skip_axes + n, m, k = shapes[0][ax1], shapes[0][ax2], shapes[1][ax2] + + res = np.matmul(a, b, axes=[skip_axes, skip_axes, skip_axes]) + assert res.shape == broadcasted_shape + + for idx1, idx2, idx3 in iter_indices(*shapes, broadcasted_shape, skip_axes=skip_axes): + assert a[idx1.raw].shape == (n, m) if ax1 <= ax2 else (m, n) + assert b[idx2.raw].shape == (m, k) if ax1 <= ax2 else (k, m) + if ax1 <= ax2: + sub_res = np.matmul(a[idx1.raw], b[idx2.raw]) + else: + sub_res = np.matmul(a[idx1.raw], b[idx2.raw], + axes=[(1, 0), (1, 0), (1, 0)]) + assert_equal(sub_res, res[idx3.raw]) + def test_iter_indices_errors(): try: list(iter_indices((10,), skip_axes=(2,))) From 3d9977f2f2d3e1aa8beaa26f2865dac6c0fe5794 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 20 Apr 2023 14:08:19 -0600 Subject: [PATCH 144/218] Fix a test failure with older versions of NumPy --- ndindex/tests/test_shapetools.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index cbdb958e..8c83f616 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -264,7 +264,7 @@ def test_iter_indices_errors(): try: list(iter_indices((10,), skip_axes=(2,))) except AxisError as e: - msg1 = str(e) + ndindex_msg = str(e) else: raise RuntimeError("iter_indices did not raise AxisError") # pragma: no cover @@ -272,27 +272,30 @@ def test_iter_indices_errors(): try: np.sum(np.arange(10), axis=2) except np.AxisError as e: - msg2 = str(e) + np_msg = str(e) else: raise RuntimeError("np.sum() did not raise AxisError") # pragma: no cover - assert msg1 == msg2 + assert ndindex_msg == np_msg try: list(iter_indices((2, 3), (3, 2))) except BroadcastError as e: - msg1 = str(e) + ndindex_msg = str(e) else: raise RuntimeError("iter_indices did not raise BroadcastError") # pragma: no cover try: np.broadcast_shapes((2, 3), (3, 2)) except ValueError as e: - msg2 = str(e) + np_msg = str(e) else: raise RuntimeError("np.broadcast_shapes() did not raise ValueError") # pragma: no cover - assert msg1 == msg2 + + if 'Mismatch' in str(np_msg): + # Older versions of NumPy do not have the more helpful error message + assert ndindex_msg == np_msg raises(NotImplementedError, lambda: list(iter_indices((1, 2), skip_axes=(0, -1)))) From e19be21837f6ce58418d7030c2f4dc2859abe416 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 20 Apr 2023 15:15:10 -0600 Subject: [PATCH 145/218] Add some pragma no covers --- ndindex/tests/test_shapetools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 8c83f616..801e35d6 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -293,7 +293,7 @@ def test_iter_indices_errors(): raise RuntimeError("np.broadcast_shapes() did not raise ValueError") # pragma: no cover - if 'Mismatch' in str(np_msg): + if 'Mismatch' in str(np_msg): # pragma: no cover # Older versions of NumPy do not have the more helpful error message assert ndindex_msg == np_msg @@ -389,7 +389,7 @@ def test_broadcast_shapes_errors(shapes): except ValueError as np_exc: # Check that they do in fact not broadcast, and the error messages are # the same modulo the different arg positions. - if 'Mismatch' in str(np_exc): + if 'Mismatch' in str(np_exc): # pragma: no cover # Older versions of NumPy do not have the more helpful error message assert str(BroadcastError(0, e.shape1, 1, e.shape2)) == str(np_exc) else: # pragma: no cover From c9933d0a802e77def4062518bbfeb127824b3d10 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 20 Apr 2023 15:34:35 -0600 Subject: [PATCH 146/218] Fix missing import --- ndindex/tests/helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index 112f62be..12f2fc72 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -1,6 +1,7 @@ import sys from itertools import chain import warnings +from functools import wraps from numpy import intp, bool_, array, broadcast_shapes import numpy.testing From 2bb486f5a260e9cf5bdec097ca9fe73cdd47087d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 20 Apr 2023 15:50:35 -0600 Subject: [PATCH 147/218] NumPy doesn't actually have BroadcastError --- ndindex/shapetools.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index e9806a32..2f5c4367 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -9,9 +9,6 @@ class BroadcastError(ValueError): """ Exception raised by :func:`iter_indices()` when the input shapes are not broadcast compatible. - - This is used instead of the NumPy exception of the same name so that - `iter_indices` does not need to depend on NumPy. """ __slots__ = ("arg1", "shape1", "arg2", "shape2") From 6cdce6b10038db50a8f027473590d4a3e9fee69b Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 20 Apr 2023 15:50:49 -0600 Subject: [PATCH 148/218] Add changelog entries for the next release --- docs/changelog.md | 49 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 34d7af2f..bea8c0ce 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,52 @@ # ndindex Changelog +## Version 1.7 (2023-??-??) + +## Major Changes + +- **Breaking** the `skip_axes` argument {func}`~.iter_indices` function now + applies the skipped axes *before* broadcasting, not after. This behavior is + more generally useful and matches how functions with stacking work (e.g., + `np.cross` or `np.matmul`). The best way to get the old behavior is to + broadcast the arrays/shapes together first. `skip_axes` in `iter_indices` + must be either all negative or all nonnegative. A future version may add + support for specifying different skip axes for each shape. + +- {func}`~.iter_indices` no longer requires the skipped axes specified by + `skip_axes` to be broadcast compatible. + +- New method {meth}`~.isvalid` to check if an index is valid on a given shape. + +- New function {func}`~.broadcast_shapes` which is the same as + `np.broadcast_shapes()` except it also allows specifying a set of + `skip_axes` which will be ignored when broadcasting. + +- New exceptions {class}`~.BroadcastError` and {class}`~.AxisError` which are + used by {func}`~.iter_indices()` and {func}`~.broadcast_shapes`. + +## Minor Changes + +- The documentation theme has been changed to + [Furo](https://pradyunsg.me/furo/), which has a more clean color scheme + based on the ndindex logo, better navigation and layout, mobile support, and + dark mode support. + +- Fix some test failures with the latest version of NumPy. + +- Fix some tests that didn't work properly when run against the sdist. + +- The sdist now includes relevant testing files. + +- Automatically deploy docs from CI again. + +- Add a documentation preview CI job. + +- Test Python 3.11 in CI. + +- Minor improvements to some documentation. + +- Fix a typo in the type confusion docs. (@ruancomelli) + ## Version 1.6 (2022-01-24) ### Major Changes @@ -12,7 +59,7 @@ ndindex objects still match NumPy indexing semantics everywhere. Note that NumPy is still a hard requirement for all tests in the ndindex test suite. -- Added a new function {any}`iter_indices` which is a generalization of the +- Added a new function {func}`~.iter_indices` which is a generalization of the `np.ndindex()` function (which is otherwise unrelated) to allow multiple broadcast compatible shapes, and to allow skipping axes. From 670484bae8572a9544b6157161961721716c53ea Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 20 Apr 2023 15:55:09 -0600 Subject: [PATCH 149/218] Formatting fixes --- docs/changelog.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index bea8c0ce..626fb4fe 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -4,13 +4,14 @@ ## Major Changes -- **Breaking** the `skip_axes` argument {func}`~.iter_indices` function now +- **Breaking:** the `skip_axes` argument {func}`~.iter_indices` function now applies the skipped axes *before* broadcasting, not after. This behavior is more generally useful and matches how functions with stacking work (e.g., `np.cross` or `np.matmul`). The best way to get the old behavior is to - broadcast the arrays/shapes together first. `skip_axes` in `iter_indices` - must be either all negative or all nonnegative. A future version may add - support for specifying different skip axes for each shape. + broadcast the arrays/shapes together first. The `skip_axes` in + `iter_indices` must be either all negative or all nonnegative to avoid + ambiguity. A future version may add support for specifying different skip + axes for each shape. - {func}`~.iter_indices` no longer requires the skipped axes specified by `skip_axes` to be broadcast compatible. @@ -22,7 +23,7 @@ `skip_axes` which will be ignored when broadcasting. - New exceptions {class}`~.BroadcastError` and {class}`~.AxisError` which are - used by {func}`~.iter_indices()` and {func}`~.broadcast_shapes`. + used by {func}`~.iter_indices` and {func}`~.broadcast_shapes`. ## Minor Changes @@ -45,7 +46,7 @@ - Minor improvements to some documentation. -- Fix a typo in the type confusion docs. (@ruancomelli) +- Fix a typo in the [type confusion](type-confusion) docs. (@ruancomelli) ## Version 1.6 (2022-01-24) From 4a3402a1116690e22802b5c9ef3581008272b543 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 20 Apr 2023 15:55:28 -0600 Subject: [PATCH 150/218] Set the 1.7 release date to today --- docs/changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 626fb4fe..71bc5c34 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,6 +1,6 @@ # ndindex Changelog -## Version 1.7 (2023-??-??) +## Version 1.7 (2023-04-20) ## Major Changes From c7443981f8379cba157b2568df2287b5374f11d3 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 20 Apr 2023 15:56:57 -0600 Subject: [PATCH 151/218] Fix docs build failure --- docs/changelog.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 71bc5c34..93d4a555 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -16,7 +16,8 @@ - {func}`~.iter_indices` no longer requires the skipped axes specified by `skip_axes` to be broadcast compatible. -- New method {meth}`~.isvalid` to check if an index is valid on a given shape. +- New method {meth}`~ndindex.ndindex.NDIndex.isvalid` to check if an index is valid on + a given shape. - New function {func}`~.broadcast_shapes` which is the same as `np.broadcast_shapes()` except it also allows specifying a set of From 50335a3fbc2700e303360928b74ac55056d0bf8e Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 20 Apr 2023 15:59:37 -0600 Subject: [PATCH 152/218] Correct a statement in the isvalid() docstring --- ndindex/ndindex.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ndindex/ndindex.py b/ndindex/ndindex.py index 6075a031..47e3da93 100644 --- a/ndindex/ndindex.py +++ b/ndindex/ndindex.py @@ -307,13 +307,17 @@ def isvalid(self, shape): >>> ndindex(3).isvalid((2,)) False - Some indices can never be valid and will raise a `TypeError` if you - attempt to construct them. + Note that some indices can never be valid and will raise a + `IndexError` or `TypeError` if you attempt to construct them. >>> ndindex((..., 0, ...)) Traceback (most recent call last): ... IndexError: an index can only have a single ellipsis ('...') + >>> ndindex(slice(True)) + Traceback (most recent call last): + ... + TypeError: 'bool' object cannot be interpreted as an integer See Also ======== From 0d5fa38772631df5d2464f2d59d498c19b987de1 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 20 Apr 2023 16:37:51 -0600 Subject: [PATCH 153/218] Add an @example for coverage --- ndindex/tests/test_isvalid.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ndindex/tests/test_isvalid.py b/ndindex/tests/test_isvalid.py index 5bad0ac2..3258aa77 100644 --- a/ndindex/tests/test_isvalid.py +++ b/ndindex/tests/test_isvalid.py @@ -5,6 +5,7 @@ from .helpers import ndindices, shapes, MAX_ARRAY_SIZE, check_same, prod +@example((0, 1), (2, 2)) @example((0,), ()) @example([[1]], (0, 0, 1)) @example(None, ()) From 7d8da1d3351ec7f8372e70f4d0af8e7df33be823 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 20 Apr 2023 17:42:56 -0600 Subject: [PATCH 154/218] Improve some hypothesis notes messages --- ndindex/tests/helpers.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index 12f2fc72..9213b253 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -147,7 +147,7 @@ def _mutually_broadcastable_shapes(draw, *, shapes=short_shapes, min_shapes=0, m # is already somewhat limited by the mutually_broadcastable_shapes # defaults, and pretty unlikely, but we filter again here just to be safe. if not prod([i for i in final_result_shape if i]) < SHORT_MAX_ARRAY_SIZE: # pragma: no cover - note(f"Filtering {result_shape}") + note(f"Filtering the shape {result_shape} (too many elements)") assume(False) return BroadcastableShapes(final_input_shapes, final_result_shape) @@ -216,7 +216,9 @@ def mutually_broadcastable_shapes_with_skipped_axes(draw, skip_axes_st=skip_axes assert remove_indices(result_shape_, skip_axes_) == result_shape for shape in shapes_: - assume(prod([i for i in shape if i]) < SHORT_MAX_ARRAY_SIZE) + if prod([i for i in shape if i]) >= SHORT_MAX_ARRAY_SIZE: + note(f"Filtering the shape {shape} (too many elements)") + assume(False) return BroadcastableShapes(shapes_, result_shape_) two_mutually_broadcastable_shapes_1 = shared(_mutually_broadcastable_shapes( From 662648cad0f2f75ebdf17346c7450b023a92b324 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 20 Apr 2023 19:07:55 -0600 Subject: [PATCH 155/218] Fix docs build in rever script --- rever.xsh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rever.xsh b/rever.xsh index b9b21282..33e66b3b 100644 --- a/rever.xsh +++ b/rever.xsh @@ -25,7 +25,8 @@ def run_tests(): @activity def build_docs(): - with run_in_conda_env(['python=3.10', 'sphinx', 'myst-parser', 'numpy']): + with run_in_conda_env(['python=3.10', 'sphinx', 'myst-parser', 'numpy', + 'sphinx-copybutton', 'furo']): cd docs make html cd .. From 7af76a3486628113e39f04aad2d8327aa89eb228 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 20 Apr 2023 19:19:35 -0600 Subject: [PATCH 156/218] Add an example for coverage --- ndindex/tests/test_shapetools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 801e35d6..a7d810ba 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -404,6 +404,7 @@ def test_broadcast_shapes_skip_axes(broadcastable_shapes, skip_axes): shapes, broadcasted_shape = broadcastable_shapes assert broadcast_shapes(*shapes, skip_axes=skip_axes) == broadcasted_shape +@example([[], ()], (0,)) @example([[(0, 1)], (0, 1)], (2,)) @example([[(0, 1)], (0, 1)], (0, -1)) @example([[(0, 1, 0, 0, 0), (2, 0, 0, 0)], (0, 2, 0, 0, 0)], [1]) From 5db02414e6a27e9ee5bb6cf4b1f0561c151e7789 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sun, 23 Apr 2023 00:48:27 -0600 Subject: [PATCH 157/218] Add a negative_int flag to reduce() This makes it return negative Integer or IntegerArray when given a shape (it doesn't so anything to any other index types or when a shape isn't given). Remove unused unfinished function --- ndindex/booleanarray.py | 2 +- ndindex/ellipsis.py | 2 +- ndindex/integer.py | 10 ++++- ndindex/integerarray.py | 15 +++++-- ndindex/ndindex.py | 2 +- ndindex/newaxis.py | 2 +- ndindex/slice.py | 2 +- ndindex/tests/helpers.py | 2 + ndindex/tests/test_booleanarray.py | 24 +++++------ ndindex/tests/test_ellipsis.py | 14 +++---- ndindex/tests/test_integer.py | 59 ++++++++++++++++---------- ndindex/tests/test_integerarray.py | 45 +++++++++++++------- ndindex/tests/test_newaxis.py | 14 +++---- ndindex/tests/test_slice.py | 30 ++++++------- ndindex/tests/test_tuple.py | 67 ++++++++++++++++++------------ ndindex/tuple.py | 10 ++--- 16 files changed, 181 insertions(+), 119 deletions(-) diff --git a/ndindex/booleanarray.py b/ndindex/booleanarray.py index 325f209f..1b99c26a 100644 --- a/ndindex/booleanarray.py +++ b/ndindex/booleanarray.py @@ -111,7 +111,7 @@ def _raise_indexerror(self, shape, axis=0): raise IndexError(f"boolean index did not match indexed array along dimension {i}; dimension is {shape[i]} but corresponding boolean dimension is {self.shape[i-axis]}") - def reduce(self, shape=None, axis=0): + def reduce(self, shape=None, *, axis=0, negative_int=False): """ Reduce a `BooleanArray` index on an array of shape `shape`. diff --git a/ndindex/ellipsis.py b/ndindex/ellipsis.py index 07755ed1..81cf2291 100644 --- a/ndindex/ellipsis.py +++ b/ndindex/ellipsis.py @@ -46,7 +46,7 @@ class ellipsis(NDIndex): def _typecheck(self): return () - def reduce(self, shape=None): + def reduce(self, shape=None, *, negative_int=False): """ Reduce an ellipsis index diff --git a/ndindex/integer.py b/ndindex/integer.py index 9e454572..18899934 100644 --- a/ndindex/integer.py +++ b/ndindex/integer.py @@ -62,13 +62,16 @@ def _raise_indexerror(self, shape, axis=0): size = shape[axis] raise IndexError(f"index {self.raw} is out of bounds for axis {axis} with size {size}") - def reduce(self, shape=None, axis=0): + def reduce(self, shape=None, *, axis=0, negative_int=False): """ Reduce an Integer index on an array of shape `shape`. The result will either be `IndexError` if the index is invalid for the given shape, or an Integer index where the value is nonnegative. + If `negative_int` is `True` and a `shape` is provided, then the result + will be an Integer index where the value is negative. + >>> from ndindex import Integer >>> idx = Integer(-5) >>> idx.reduce((3,)) @@ -96,9 +99,12 @@ def reduce(self, shape=None, axis=0): shape = asshape(shape, axis=axis) self._raise_indexerror(shape, axis) - if self.raw < 0: + if self.raw < 0 and not negative_int: size = shape[axis] return self.__class__(size + self.raw) + elif self.raw >= 0 and negative_int: + size = shape[axis] + return self.__class__(self.raw - size) return self diff --git a/ndindex/integerarray.py b/ndindex/integerarray.py index df2d87af..67a6094e 100644 --- a/ndindex/integerarray.py +++ b/ndindex/integerarray.py @@ -57,7 +57,7 @@ def _raise_indexerror(self, shape, axis=0): if out_of_bounds.any(): raise IndexError(f"index {self.array[out_of_bounds].flat[0]} is out of bounds for axis {axis} with size {size}") - def reduce(self, shape=None, axis=0): + def reduce(self, shape=None, *, axis=0, negative_int=False): """ Reduce an `IntegerArray` index on an array of shape `shape`. @@ -66,6 +66,10 @@ def reduce(self, shape=None, axis=0): nonnegative, or, if `self` is a scalar array index (`self.shape == ()`), an `Integer` whose value is nonnegative. + If `negative_int` is `True` and a `shape` is provided, the result will + be an `IntegerArray` with negative entries instead of positive + entries. + >>> from ndindex import IntegerArray >>> idx = IntegerArray([-5, 2]) >>> idx.reduce((3,)) @@ -74,6 +78,8 @@ def reduce(self, shape=None, axis=0): IndexError: index -5 is out of bounds for axis 0 with size 3 >>> idx.reduce((9,)) IntegerArray([4, 2]) + >>> idx.reduce((9,), negative_int=True) + IntegerArray([-5, -7]) See Also ======== @@ -88,7 +94,7 @@ def reduce(self, shape=None, axis=0): """ if self.shape == (): - return Integer(self.array).reduce(shape, axis=axis) + return Integer(self.array).reduce(shape, axis=axis, negative_int=negative_int) if shape is None: return self @@ -99,7 +105,10 @@ def reduce(self, shape=None, axis=0): size = shape[axis] new_array = self.array.copy() - new_array[new_array < 0] += size + if negative_int: + new_array[new_array >= 0] -= size + else: + new_array[new_array < 0] += size return IntegerArray(new_array) def newshape(self, shape): diff --git a/ndindex/ndindex.py b/ndindex/ndindex.py index 47e3da93..c9e04e03 100644 --- a/ndindex/ndindex.py +++ b/ndindex/ndindex.py @@ -255,7 +255,7 @@ def __hash__(self): # as hash(self.args) return hash(self.raw) - def reduce(self, shape=None): + def reduce(self, shape=None, *, negative_int=False): """ Simplify an index given that it will be applied to an array of a given shape. diff --git a/ndindex/newaxis.py b/ndindex/newaxis.py index 18b2300a..120c0ef2 100644 --- a/ndindex/newaxis.py +++ b/ndindex/newaxis.py @@ -43,7 +43,7 @@ def _typecheck(self): def raw(self): return None - def reduce(self, shape=None, axis=0): + def reduce(self, shape=None, *, axis=0, negative_int=False): """ Reduce a `Newaxis` index diff --git a/ndindex/slice.py b/ndindex/slice.py index 70405a28..a36dbb72 100644 --- a/ndindex/slice.py +++ b/ndindex/slice.py @@ -200,7 +200,7 @@ def __len__(self): return len(range(start, stop, step)) - def reduce(self, shape=None, axis=0): + def reduce(self, shape=None, *, axis=0, negative_int=False): """ `Slice.reduce` returns a slice that is canonicalized for an array of the given shape, or for any shape if `shape` is `None` (the default). diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index 9213b253..8f2366cf 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -238,6 +238,8 @@ def mutually_broadcastable_shapes_with_skipped_axes(draw, skip_axes_st=skip_axes mutually_broadcastable_shapes=two_mutually_broadcastable_shapes_2, num_skip_axes=2)) +reduce_kwargs = sampled_from([{}, {'negative_int': False}, {'negative_int': True}]) + def assert_equal(actual, desired, err_msg='', verbose=True): """ Same as numpy.testing.assert_equal except it also requires the shapes and diff --git a/ndindex/tests/test_booleanarray.py b/ndindex/tests/test_booleanarray.py index f18f2e9c..0ee19e1d 100644 --- a/ndindex/tests/test_booleanarray.py +++ b/ndindex/tests/test_booleanarray.py @@ -5,7 +5,7 @@ from pytest import raises -from .helpers import boolean_arrays, short_shapes, check_same, assert_equal +from .helpers import boolean_arrays, short_shapes, check_same, assert_equal, reduce_kwargs from ..booleanarray import BooleanArray @@ -38,8 +38,8 @@ def test_booleanarray_hypothesis(idx, shape): a = arange(prod(shape)).reshape(shape) check_same(a, idx) -@given(boolean_arrays, one_of(short_shapes, integers(0, 10))) -def test_booleanarray_reduce_no_shape_hypothesis(idx, shape): +@given(boolean_arrays, one_of(short_shapes, integers(0, 10)), reduce_kwargs) +def test_booleanarray_reduce_no_shape_hypothesis(idx, shape, kwargs): if isinstance(shape, int): a = arange(shape) else: @@ -47,12 +47,12 @@ def test_booleanarray_reduce_no_shape_hypothesis(idx, shape): index = BooleanArray(idx) - check_same(a, index.raw, ndindex_func=lambda a, x: a[x.reduce().raw]) + check_same(a, index.raw, ndindex_func=lambda a, x: a[x.reduce(**kwargs).raw]) -@example(full((1, 9), True), (3, 3)) -@example(full((1, 9), False), (3, 3)) -@given(boolean_arrays, one_of(short_shapes, integers(0, 10))) -def test_booleanarray_reduce_hypothesis(idx, shape): +@example(full((1, 9), True), (3, 3), {}) +@example(full((1, 9), False), (3, 3), {}) +@given(boolean_arrays, one_of(short_shapes, integers(0, 10)), reduce_kwargs) +def test_booleanarray_reduce_hypothesis(idx, shape, kwargs): if isinstance(shape, int): a = arange(shape) else: @@ -60,10 +60,10 @@ def test_booleanarray_reduce_hypothesis(idx, shape): index = BooleanArray(idx) - check_same(a, index.raw, ndindex_func=lambda a, x: a[x.reduce(shape).raw]) + check_same(a, index.raw, ndindex_func=lambda a, x: a[x.reduce(shape, **kwargs).raw]) try: - reduced = index.reduce(shape) + reduced = index.reduce(shape, **kwargs) except IndexError: pass else: @@ -72,8 +72,8 @@ def test_booleanarray_reduce_hypothesis(idx, shape): assert reduced == index # Idempotency - assert reduced.reduce() == reduced - assert reduced.reduce(shape) == reduced + assert reduced.reduce(**kwargs) == reduced + assert reduced.reduce(shape, **kwargs) == reduced @given(boolean_arrays, one_of(short_shapes, integers(0, 10))) def test_booleanarray_isempty_hypothesis(idx, shape): diff --git a/ndindex/tests/test_ellipsis.py b/ndindex/tests/test_ellipsis.py index 1b005172..4912cc6f 100644 --- a/ndindex/tests/test_ellipsis.py +++ b/ndindex/tests/test_ellipsis.py @@ -4,7 +4,7 @@ from hypothesis.strategies import one_of, integers from ..ndindex import ndindex -from .helpers import check_same, prod, shapes, ellipses +from .helpers import check_same, prod, shapes, ellipses, reduce_kwargs def test_ellipsis_exhaustive(): for n in range(10): @@ -21,20 +21,20 @@ def test_ellipsis_reduce_exhaustive(): a = arange(n) check_same(a, ..., ndindex_func=lambda a, x: a[x.reduce((n,)).raw]) -@given(ellipses(), shapes) -def test_ellipsis_reduce_hypothesis(idx, shape): +@given(ellipses(), shapes, reduce_kwargs) +def test_ellipsis_reduce_hypothesis(idx, shape, kwargs): a = arange(prod(shape)).reshape(shape) - check_same(a, idx, ndindex_func=lambda a, x: a[x.reduce(shape).raw]) + check_same(a, idx, ndindex_func=lambda a, x: a[x.reduce(shape, **kwargs).raw]) def test_ellipsis_reduce_no_shape_exhaustive(): for n in range(10): a = arange(n) check_same(a, ..., ndindex_func=lambda a, x: a[x.reduce().raw]) -@given(ellipses(), shapes) -def test_ellipsis_reduce_no_shape_hypothesis(idx, shape): +@given(ellipses(), shapes, reduce_kwargs) +def test_ellipsis_reduce_no_shape_hypothesis(idx, shape, kwargs): a = arange(prod(shape)).reshape(shape) - check_same(a, idx, ndindex_func=lambda a, x: a[x.reduce().raw]) + check_same(a, idx, ndindex_func=lambda a, x: a[x.reduce(**kwargs).raw]) @given(ellipses(), one_of(shapes, integers(0, 10))) def test_ellipsis_isempty_hypothesis(idx, shape): diff --git a/ndindex/tests/test_integer.py b/ndindex/tests/test_integer.py index fd354ee4..53c3407a 100644 --- a/ndindex/tests/test_integer.py +++ b/ndindex/tests/test_integer.py @@ -7,7 +7,7 @@ from ..integer import Integer from ..slice import Slice -from .helpers import check_same, ints, prod, shapes, iterslice, assert_equal +from .helpers import check_same, ints, prod, shapes, iterslice, assert_equal, reduce_kwargs def test_integer_args(): zero = Integer(0) @@ -46,51 +46,66 @@ def test_integer_len_hypothesis(i): idx = Integer(i) assert len(idx) == 1 - def test_integer_reduce_exhaustive(): a = arange(10) for i in range(-12, 12): - check_same(a, i, ndindex_func=lambda a, x: a[x.reduce((10,)).raw]) + for kwargs in [{'negative_int': False}, {'negative_int': True}, {}]: + check_same(a, i, ndindex_func=lambda a, x: a[x.reduce((10,), **kwargs).raw]) - try: - reduced = Integer(i).reduce(10) - except IndexError: - pass - else: - assert reduced.raw >= 0 + negative_int = kwargs.get('negative_int', False) + + try: + reduced = Integer(i).reduce(10, **kwargs) + except IndexError: + pass + else: + if negative_int: + assert reduced.raw < 0 + else: + assert reduced.raw >= 0 - # Idempotency - assert reduced.reduce() == reduced - assert reduced.reduce(10) == reduced + # Idempotency + assert reduced.reduce(**kwargs) == reduced + assert reduced.reduce(10, **kwargs) == reduced -@given(ints(), shapes) -def test_integer_reduce_hypothesis(i, shape): +@given(ints(), shapes, reduce_kwargs) +def test_integer_reduce_hypothesis(i, shape, kwargs): a = arange(prod(shape)).reshape(shape) # The axis argument is tested implicitly in the Tuple.reduce test. It is # difficult to test here because we would have to pass in a Tuple to # check_same. - check_same(a, i, ndindex_func=lambda a, x: a[x.reduce(shape).raw]) + check_same(a, i, ndindex_func=lambda a, x: a[x.reduce(shape, **kwargs).raw]) + + negative_int = kwargs.get('negative_int', False) try: - reduced = Integer(i).reduce(shape) + reduced = Integer(i).reduce(shape, **kwargs) except IndexError: pass else: - assert reduced.raw >= 0 + if negative_int: + assert reduced.raw < 0 + else: + assert reduced.raw >= 0 # Idempotency - assert reduced.reduce() == reduced - assert reduced.reduce(shape) == reduced + assert reduced.reduce(**kwargs) == reduced + assert reduced.reduce(shape, **kwargs) == reduced def test_integer_reduce_no_shape_exhaustive(): a = arange(10) for i in range(-12, 12): check_same(a, i, ndindex_func=lambda a, x: a[x.reduce().raw]) -@given(ints(), shapes) -def test_integer_reduce_no_shape_hypothesis(i, shape): +@given(ints(), shapes, reduce_kwargs) +def test_integer_reduce_no_shape_hypothesis(i, shape, kwargs): a = arange(prod(shape)).reshape(shape) - check_same(a, i, ndindex_func=lambda a, x: a[x.reduce().raw]) + check_same(a, i, ndindex_func=lambda a, x: a[x.reduce(**kwargs).raw]) + +@given(ints()) +def test_integer_reduce_no_shape_unchanged(i): + idx = Integer(i) + assert idx.reduce() == idx.reduce(negative_int=False) == idx.reduce(negative_int=True) == i def test_integer_newshape_exhaustive(): shape = 5 diff --git a/ndindex/tests/test_integerarray.py b/ndindex/tests/test_integerarray.py index 615c38af..bb73e196 100644 --- a/ndindex/tests/test_integerarray.py +++ b/ndindex/tests/test_integerarray.py @@ -5,7 +5,7 @@ from pytest import raises -from .helpers import integer_arrays, short_shapes, check_same, assert_equal +from .helpers import integer_arrays, short_shapes, check_same, assert_equal, reduce_kwargs from ..integer import Integer from ..integerarray import IntegerArray @@ -39,8 +39,8 @@ def test_integerarray_hypothesis(idx, shape): a = arange(prod(shape)).reshape(shape) check_same(a, idx) -@given(integer_arrays, one_of(short_shapes, integers(0, 10))) -def test_integerarray_reduce_no_shape_hypothesis(idx, shape): +@given(integer_arrays, one_of(short_shapes, integers(0, 10)), reduce_kwargs) +def test_integerarray_reduce_no_shape_hypothesis(idx, shape, kwargs): if isinstance(shape, int): a = arange(shape) else: @@ -48,13 +48,20 @@ def test_integerarray_reduce_no_shape_hypothesis(idx, shape): index = IntegerArray(idx) - check_same(a, index.raw, ndindex_func=lambda a, x: a[x.reduce().raw]) + check_same(a, index.raw, ndindex_func=lambda a, x: a[x.reduce(**kwargs).raw]) -@example(array([2, 0]), (1, 0)) -@example(array(0), 1) -@example(array([], dtype=intp), 0) -@given(integer_arrays, one_of(short_shapes, integers(0, 10))) -def test_integerarray_reduce_hypothesis(idx, shape): +@given(integer_arrays) +def test_integerarray_reduce_no_shape_unchanged(idx): + index = IntegerArray(idx) + assert index.reduce() == index.reduce(negative_int=False) == index.reduce(negative_int=True) + if index.ndim != 0: + assert index.reduce() == index + +@example(array([2, 0]), (1, 0), {}) +@example(array(0), 1, {}) +@example(array([], dtype=intp), 0, {}) +@given(integer_arrays, one_of(short_shapes, integers(0, 10)), reduce_kwargs) +def test_integerarray_reduce_hypothesis(idx, shape, kwargs): if isinstance(shape, int): a = arange(shape) else: @@ -62,22 +69,30 @@ def test_integerarray_reduce_hypothesis(idx, shape): index = IntegerArray(idx) - check_same(a, index.raw, ndindex_func=lambda a, x: a[x.reduce(shape).raw]) + check_same(a, index.raw, ndindex_func=lambda a, x: a[x.reduce(shape, **kwargs).raw]) + + negative_int = kwargs.get('negative_int', False) try: - reduced = index.reduce(shape) + reduced = index.reduce(shape, **kwargs) except IndexError: pass else: if isinstance(reduced, Integer): - assert reduced.raw >= 0 + if negative_int: + assert reduced.raw < 0 + else: + assert reduced.raw >= 0 else: assert isinstance(reduced, IntegerArray) - assert (reduced.raw >= 0).all() + if negative_int: + assert (reduced.raw < 0).all() + else: + assert (reduced.raw >= 0).all() # Idempotency - assert reduced.reduce() == reduced - assert reduced.reduce(shape) == reduced + assert reduced.reduce(**kwargs) == reduced + assert reduced.reduce(shape, **kwargs) == reduced @example([], (1,)) @example([0], (1, 0)) diff --git a/ndindex/tests/test_newaxis.py b/ndindex/tests/test_newaxis.py index bc655a7d..abe44503 100644 --- a/ndindex/tests/test_newaxis.py +++ b/ndindex/tests/test_newaxis.py @@ -4,7 +4,7 @@ from hypothesis.strategies import one_of, integers from ..ndindex import ndindex -from .helpers import check_same, prod, shapes, newaxes +from .helpers import check_same, prod, shapes, newaxes, reduce_kwargs def test_newaxis_exhaustive(): for n in range(10): @@ -24,10 +24,10 @@ def test_newaxis_reduce_exhaustive(): check_same(a, newaxis, ndindex_func=lambda a, x: a[x.reduce((n,)).raw]) -@given(newaxes(), shapes) -def test_newaxis_reduce_hypothesis(idx, shape): +@given(newaxes(), shapes, reduce_kwargs) +def test_newaxis_reduce_hypothesis(idx, shape, kwargs): a = arange(prod(shape)).reshape(shape) - check_same(a, idx, ndindex_func=lambda a, x: a[x.reduce(shape).raw]) + check_same(a, idx, ndindex_func=lambda a, x: a[x.reduce(shape, **kwargs).raw]) def test_newaxis_reduce_no_shape_exhaustive(): @@ -35,10 +35,10 @@ def test_newaxis_reduce_no_shape_exhaustive(): a = arange(n) check_same(a, newaxis, ndindex_func=lambda a, x: a[x.reduce().raw]) -@given(newaxes(), shapes) -def test_newaxis_reduce_no_shape_hypothesis(idx, shape): +@given(newaxes(), shapes, reduce_kwargs) +def test_newaxis_reduce_no_shape_hypothesis(idx, shape, kwargs): a = arange(prod(shape)).reshape(shape) - check_same(a, idx, ndindex_func=lambda a, x: a[x.reduce().raw]) + check_same(a, idx, ndindex_func=lambda a, x: a[x.reduce(**kwargs).raw]) @given(newaxes(), one_of(shapes, integers(0, 10))) def test_newaxis_isempty_hypothesis(idx, shape): diff --git a/ndindex/tests/test_slice.py b/ndindex/tests/test_slice.py index 7d37427c..ee50e3c8 100644 --- a/ndindex/tests/test_slice.py +++ b/ndindex/tests/test_slice.py @@ -9,7 +9,7 @@ from ..integer import Integer from ..ellipsis import ellipsis from ..shapetools import asshape -from .helpers import check_same, slices, prod, shapes, iterslice, assert_equal +from .helpers import check_same, slices, prod, shapes, iterslice, assert_equal, reduce_kwargs def test_slice_args(): # Test the behavior when not all three arguments are given @@ -144,8 +144,8 @@ def test_slice_reduce_no_shape_exhaustive(): slices[B] = reduced -@given(slices(), one_of(integers(0, 100), shapes)) -def test_slice_reduce_no_shape_hypothesis(s, shape): +@given(slices(), one_of(integers(0, 100), shapes), reduce_kwargs) +def test_slice_reduce_no_shape_hypothesis(s, shape, kwargs): if isinstance(shape, int): a = arange(shape) else: @@ -158,10 +158,10 @@ def test_slice_reduce_no_shape_hypothesis(s, shape): # The axis argument is tested implicitly in the Tuple.reduce test. It is # difficult to test here because we would have to pass in a Tuple to # check_same. - check_same(a, S.raw, ndindex_func=lambda a, x: a[x.reduce().raw]) + check_same(a, S.raw, ndindex_func=lambda a, x: a[x.reduce(**kwargs).raw]) # Check the conditions stated by the Slice.reduce() docstring - reduced = S.reduce() + reduced = S.reduce(**kwargs) assert reduced.start != None if S.start != None and S.start >= 0: assert reduced.start >= 0 @@ -171,7 +171,7 @@ def test_slice_reduce_no_shape_hypothesis(s, shape): if reduced.stop is None: assert S.stop is None # Idempotency - assert reduced.reduce() == reduced, S + assert reduced.reduce(**kwargs) == reduced, S def test_slice_reduce_exhaustive(): for n in range(30): @@ -232,11 +232,11 @@ def test_slice_reduce_exhaustive(): assert reduced.reduce() == reduced, S assert reduced.reduce((n,)) == reduced, S -@example(slice(None, None, -1), 2) -@example(slice(-10, 11, 3), 10) -@example(slice(-1, 3, -3), 10) -@given(slices(), one_of(integers(0, 100), shapes)) -def test_slice_reduce_hypothesis(s, shape): +@example(slice(None, None, -1), 2, {}) +@example(slice(-10, 11, 3), 10, {}) +@example(slice(-1, 3, -3), 10, {}) +@given(slices(), one_of(integers(0, 100), shapes), reduce_kwargs) +def test_slice_reduce_hypothesis(s, shape, kwargs): if isinstance(shape, int): a = arange(shape) else: @@ -250,11 +250,11 @@ def test_slice_reduce_hypothesis(s, shape): # The axis argument is tested implicitly in the Tuple.reduce test. It is # difficult to test here because we would have to pass in a Tuple to # check_same. - check_same(a, S.raw, ndindex_func=lambda a, x: a[x.reduce(shape).raw]) + check_same(a, S.raw, ndindex_func=lambda a, x: a[x.reduce(shape, **kwargs).raw]) # Check the conditions stated by the Slice.reduce() docstring try: - reduced = S.reduce(shape) + reduced = S.reduce(shape, **kwargs) except IndexError: # shape == () return @@ -294,8 +294,8 @@ def test_slice_reduce_hypothesis(s, shape): assert reduced == Slice(reduced.start, reduced.start+1, 1) # Idempotency - assert reduced.reduce() == reduced, S - assert reduced.reduce(shape) == reduced, S + assert reduced.reduce(**kwargs) == reduced, S + assert reduced.reduce(shape, **kwargs) == reduced, S def test_slice_newshape_exhaustive(): def raw_func(a, idx): diff --git a/ndindex/tests/test_tuple.py b/ndindex/tests/test_tuple.py index b1ecdffa..12125748 100644 --- a/ndindex/tests/test_tuple.py +++ b/ndindex/tests/test_tuple.py @@ -10,7 +10,8 @@ from ..ndindex import ndindex from ..tuple import Tuple from ..integer import Integer -from .helpers import check_same, Tuples, prod, short_shapes, iterslice +from ..integerarray import IntegerArray +from .helpers import check_same, Tuples, prod, short_shapes, iterslice, reduce_kwargs def test_tuple_constructor(): # Test things in the Tuple constructor that are not tested by the other @@ -81,10 +82,10 @@ def ndindex_func(a, index): check_same(a, t, ndindex_func=ndindex_func) -@example((True, 0, False), 1) -@example((..., None), ()) -@given(Tuples, one_of(short_shapes, integers(0, 10))) -def test_tuple_reduce_no_shape_hypothesis(t, shape): +@example((True, 0, False), 1, {}) +@example((..., None), (), {}) +@given(Tuples, one_of(short_shapes, integers(0, 10)), reduce_kwargs) +def test_tuple_reduce_no_shape_hypothesis(t, shape, kwargs): if isinstance(shape, int): a = arange(shape) else: @@ -92,31 +93,31 @@ def test_tuple_reduce_no_shape_hypothesis(t, shape): index = Tuple(*t) - check_same(a, index.raw, ndindex_func=lambda a, x: a[x.reduce().raw], + check_same(a, index.raw, ndindex_func=lambda a, x: a[x.reduce(**kwargs).raw], same_exception=False) - reduced = index.reduce() + reduced = index.reduce(**kwargs) if isinstance(reduced, Tuple): assert len(reduced.args) != 1 assert reduced == () or reduced.args[-1] != ... # Idempotency - assert reduced.reduce() == reduced - -@example((..., None), ()) -@example((..., empty((0, 0), dtype=bool)), (0, 0)) -@example((empty((0, 0), dtype=bool), 0), (0, 0, 1)) -@example((array([], dtype=intp), 0), (0, 0)) -@example((array([], dtype=intp), array(0)), (0, 0)) -@example((array([], dtype=intp), [0]), (0, 0)) -@example((0, 1, ..., 2, 3), (2, 3, 4, 5, 6, 7)) -@example((0, slice(None), ..., slice(None), 3), (2, 3, 4, 5, 6, 7)) -@example((0, ..., slice(None)), (2, 3, 4, 5, 6, 7)) -@example((slice(None, None, -1),), (2,)) -@example((..., slice(None, None, -1),), (2, 3, 4)) -@example((..., False, slice(None)), 0) -@given(Tuples, one_of(short_shapes, integers(0, 10))) -def test_tuple_reduce_hypothesis(t, shape): + assert reduced.reduce(**kwargs) == reduced + +@example((..., None), (), {}) +@example((..., empty((0, 0), dtype=bool)), (0, 0), {}) +@example((empty((0, 0), dtype=bool), 0), (0, 0, 1), {}) +@example((array([], dtype=intp), 0), (0, 0), {}) +@example((array([], dtype=intp), array(0)), (0, 0), {}) +@example((array([], dtype=intp), [0]), (0, 0), {}) +@example((0, 1, ..., 2, 3), (2, 3, 4, 5, 6, 7), {}) +@example((0, slice(None), ..., slice(None), 3), (2, 3, 4, 5, 6, 7), {}) +@example((0, ..., slice(None)), (2, 3, 4, 5, 6, 7), {}) +@example((slice(None, None, -1),), (2,), {}) +@example((..., slice(None, None, -1),), (2, 3, 4), {}) +@example((..., False, slice(None)), 0, {}) +@given(Tuples, one_of(short_shapes, integers(0, 10)), reduce_kwargs) +def test_tuple_reduce_hypothesis(t, shape, kwargs): if isinstance(shape, int): a = arange(shape) else: @@ -124,11 +125,13 @@ def test_tuple_reduce_hypothesis(t, shape): index = Tuple(*t) - check_same(a, index.raw, ndindex_func=lambda a, x: a[x.reduce(shape).raw], + check_same(a, index.raw, ndindex_func=lambda a, x: a[x.reduce(shape, **kwargs).raw], same_exception=False) + negative_int = kwargs.get('negative_int', False) + try: - reduced = index.reduce(shape) + reduced = index.reduce(shape, **kwargs) except IndexError: pass else: @@ -138,11 +141,23 @@ def test_tuple_reduce_hypothesis(t, shape): # TODO: Check the other properties from the Tuple.reduce docstring. # Idempotency - assert reduced.reduce() == reduced + assert reduced.reduce(**kwargs) == reduced # This is currently not implemented, for example, (..., False, :) # takes two steps to remove the redundant slice. # assert reduced.reduce(shape) == reduced + for arg in reduced.args: + if isinstance(arg, Integer): + if negative_int: + assert arg.raw < 0 + else: + assert arg.raw >= 0 + elif isinstance(arg, IntegerArray): + if negative_int: + assert all(arg.raw < 0) + else: + assert all(arg.raw >= 0) + def test_tuple_reduce_explicit(): # Some aspects of Tuple.reduce are hard to test as properties, so include # some explicit tests here. diff --git a/ndindex/tuple.py b/ndindex/tuple.py index ea6229dc..61052ac3 100644 --- a/ndindex/tuple.py +++ b/ndindex/tuple.py @@ -181,7 +181,7 @@ def ellipsis_index(self): def raw(self): return tuple(i.raw for i in self.args) - def reduce(self, shape=None): + def reduce(self, shape=None, *, negative_int=False): r""" Reduce a Tuple index on an array of shape `shape` @@ -278,7 +278,7 @@ def reduce(self, shape=None): seen_boolean_scalar = True else: _args.append(s) - return type(self)(*_args).reduce(shape) + return type(self)(*_args).reduce(shape, negative_int=negative_int) arrays = [] for i in args: @@ -340,7 +340,7 @@ def reduce(self, shape=None): elif isinstance(s, BooleanArray): begin_offset += s.ndim - 1 axis = ellipsis_i - i - begin_offset - reduced = s.reduce(shape, axis=axis) + reduced = s.reduce(shape, axis=axis, negative_int=negative_int) if (removable and isinstance(reduced, Slice) and reduced == Slice(0, shape[axis], 1)): @@ -350,7 +350,7 @@ def reduce(self, shape=None): preargs.insert(0, reduced) if shape is None: - endargs = [s.reduce() for s in args[ellipsis_i+1:]] + endargs = [s.reduce(negative_int=negative_int) for s in args[ellipsis_i+1:]] else: endargs = [] end_offset = 0 @@ -363,7 +363,7 @@ def reduce(self, shape=None): if not (isinstance(s, IntegerArray) and (0 in broadcast_shape or False in args)): # Array bounds are not checked when the broadcast shape is empty - s = s.reduce(shape, axis=axis) + s = s.reduce(shape, axis=axis, negative_int=negative_int) endargs.insert(0, s) if shape is not None: From e0e83eefc51f560c44bb78d630f7cd0bae1bfaa2 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sun, 23 Apr 2023 00:51:14 -0600 Subject: [PATCH 158/218] Use reduce(negative_int=True) in some places in the code --- ndindex/shapetools.py | 2 +- ndindex/tests/test_shapetools.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index 2f5c4367..bec43130 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -83,7 +83,7 @@ def broadcast_shapes(*shapes, skip_axes=()): return () dims = [len(shape) for shape in shapes] - shape_skip_axes = [[ndindex(i).reduce(n).raw - n for i in skip_axes] for n in dims] + shape_skip_axes = [[ndindex(i).reduce(n, negative_int=True) for i in skip_axes] for n in dims] N = max(dims) broadcasted_skip_axes = [ndindex(i).reduce(N) for i in skip_axes] diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index a7d810ba..9575eb04 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -517,8 +517,8 @@ def test_associated_axis(broadcastable_shapes, skip_axes): if _skip_axes[0] >= 0: assert ndindex(i).reduce(n) == ndindex(idx).reduce(ndim) in normalized_skip_axes else: - assert ndindex(i).reduce(n).raw - n == \ - ndindex(idx).reduce(ndim).raw - ndim in _skip_axes + assert ndindex(i).reduce(n, negative_int=True) == \ + ndindex(idx).reduce(ndim, negative_int=True) in _skip_axes else: assert val == 1 or bval == val From a40fcb21978f994871a9df199ca431ce8d3eb76d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 24 Apr 2023 00:23:11 -0600 Subject: [PATCH 159/218] Add @examples for coverage --- ndindex/tests/test_integerarray.py | 1 + ndindex/tests/test_isvalid.py | 2 ++ ndindex/tests/test_tuple.py | 1 + 3 files changed, 4 insertions(+) diff --git a/ndindex/tests/test_integerarray.py b/ndindex/tests/test_integerarray.py index bb73e196..e3c05cdd 100644 --- a/ndindex/tests/test_integerarray.py +++ b/ndindex/tests/test_integerarray.py @@ -57,6 +57,7 @@ def test_integerarray_reduce_no_shape_unchanged(idx): if index.ndim != 0: assert index.reduce() == index +@example(array(2), (4,), {'negative_int': True}) @example(array([2, 0]), (1, 0), {}) @example(array(0), 1, {}) @example(array([], dtype=intp), 0, {}) diff --git a/ndindex/tests/test_isvalid.py b/ndindex/tests/test_isvalid.py index 3258aa77..2b8c4551 100644 --- a/ndindex/tests/test_isvalid.py +++ b/ndindex/tests/test_isvalid.py @@ -5,6 +5,8 @@ from .helpers import ndindices, shapes, MAX_ARRAY_SIZE, check_same, prod +@example(slice(0, 1), ()) +@example(slice(0, 1), (1,)) @example((0, 1), (2, 2)) @example((0,), ()) @example([[1]], (0, 0, 1)) diff --git a/ndindex/tests/test_tuple.py b/ndindex/tests/test_tuple.py index 12125748..bcd854aa 100644 --- a/ndindex/tests/test_tuple.py +++ b/ndindex/tests/test_tuple.py @@ -104,6 +104,7 @@ def test_tuple_reduce_no_shape_hypothesis(t, shape, kwargs): # Idempotency assert reduced.reduce(**kwargs) == reduced +@example((1, -1, [1, -1]), (3, 3, 3), {'negative_int': True}) @example((..., None), (), {}) @example((..., empty((0, 0), dtype=bool)), (0, 0), {}) @example((empty((0, 0), dtype=bool), 0), (0, 0, 1), {}) From e44f7a2d3f80581a18a800b43fb678659e72caf5 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 24 Apr 2023 01:44:20 -0600 Subject: [PATCH 160/218] Add basic constructor type checking and tests for AxisError --- ndindex/shapetools.py | 3 +++ ndindex/tests/test_shapetools.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index bec43130..33a152e9 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -34,6 +34,9 @@ class AxisError(ValueError, IndexError): __slots__ = ("axis", "ndim") def __init__(self, axis, ndim): + # NumPy allows axis=-1 for 0-d arrays + if (ndim < 0 or -ndim <= axis < ndim) and not (ndim == 0 and axis == -1): + raise ValueError(f"Invalid AxisError ({axis}, {ndim})") self.axis = axis self.ndim = ndim diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 9575eb04..c786eb5f 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -553,3 +553,32 @@ def test_asshape(): raises(TypeError, lambda: asshape(-1, allow_int=False)) raises(TypeError, lambda: asshape(-1, allow_negative=True, allow_int=False)) raises(TypeError, lambda: asshape(np.int64(1), allow_int=False)) + +@given(integers(), integers()) +def test_axiserror(axis, ndim): + if ndim == 0 and axis in [0, -1]: + # NumPy allows axis=0 or -1 for 0-d arrays + AxisError(axis, ndim) + return + + try: + if ndim >= 0: + range(ndim)[axis] + except IndexError: + e = AxisError(axis, ndim) + else: + raises(ValueError, lambda: AxisError(axis, ndim)) + return + + try: + raise e + except AxisError as e2: + assert e2.args == (axis, ndim) + if ndim <= 32 and -1000 < axis < 1000: + a = np.empty((0,)*ndim) + try: + np.sum(a, axis=axis) + except np.AxisError as e3: + assert str(e2) == str(e3) + else: + raise RuntimeError("numpy didn't raise AxisError") # pragma: no cover From 9c3a75fff997edf95cb21c0fe80212a15f3ff88d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 24 Apr 2023 01:44:48 -0600 Subject: [PATCH 161/218] Add (but don't use yet) the canonical_skip_axes() helper function --- ndindex/integer.py | 11 ++++- ndindex/shapetools.py | 51 +++++++++++++++++++++ ndindex/tests/test_shapetools.py | 78 +++++++++++++++++++++++++++++++- 3 files changed, 137 insertions(+), 3 deletions(-) diff --git a/ndindex/integer.py b/ndindex/integer.py index 18899934..0ba7d89f 100644 --- a/ndindex/integer.py +++ b/ndindex/integer.py @@ -1,5 +1,5 @@ from .ndindex import NDIndex, operator_index -from .shapetools import asshape +from .shapetools import AxisError, asshape class Integer(NDIndex): """ @@ -62,7 +62,7 @@ def _raise_indexerror(self, shape, axis=0): size = shape[axis] raise IndexError(f"index {self.raw} is out of bounds for axis {axis} with size {size}") - def reduce(self, shape=None, *, axis=0, negative_int=False): + def reduce(self, shape=None, *, axis=0, negative_int=False, axiserror=False): """ Reduce an Integer index on an array of shape `shape`. @@ -96,7 +96,14 @@ def reduce(self, shape=None, *, axis=0, negative_int=False): if shape is None: return self + if axiserror: + if not isinstance(shape, int): # pragma: no cover + raise TypeError("axiserror=True requires shape to be an integer") + if not self.isvalid(shape): + raise AxisError(self.raw, shape) + shape = asshape(shape, axis=axis) + self._raise_indexerror(shape, axis) if self.raw < 0 and not negative_int: diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index 33a152e9..e984cbaa 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -475,3 +475,54 @@ def __repr__(self): def __iter__(self): return itertools.chain.from_iterable(itertools.repeat(tuple(self.iterable), self.n)) + +def canonical_skip_axes(shapes, skip_axes): + """ + Return a canonical form of `skip_axes` corresponding to `shapes`. + + A canonical form of `skip_axes` is a list of tuples of integers, one for + each shape in `shapes`, which are a unique set of axes for each + corresponding shape. + + If `skip_axes` is an integer, this is basically `[(skip_axes,) for s + in shapes]`. If `skip_axes is a tuple, it is like `[skip_axes for s in + shapes]`. + + The `skip_axes` must always refer to unique axes in each shape. + + The returned `skip_axes` will always be negative integers. + + This function is only intended for internal usage. + + """ + if isinstance(skip_axes, Sequence): + if skip_axes and all(isinstance(i, Sequence) for i in skip_axes): + if not all(isinstance(skip_axis, Sequence) for skip_axis in + skip_axes): + raise TypeError("skip_axes must all be tuples of integers") + if len(skip_axes) != len(shapes): + raise ValueError(f"Expected {len(shapes)} skip_axes") + return [canonical_skip_axes([shape], skip_axis)[0] for shape, skip_axis in zip(shapes, skip_axes)] + else: + try: + [operator_index(i) for i in skip_axes] + except TypeError: + raise TypeError("skip_axes must be an integer, a tuple of integers, or a list of tuples of integers") + + skip_axes = asshape(skip_axes, allow_negative=True) + + # From here, skip_axes is a single tuple of integers + + if not shapes and skip_axes: + raise ValueError(f"Expected {len(shapes)} skip_axes") + + new_skip_axes = [] + for shape in shapes: + s = tuple(ndindex(i).reduce(len(shape), negative_int=True, axiserror=True).raw for i in skip_axes) + if len(s) != len(set(s)): + err = ValueError(f"skip_axes {skip_axes} are not unique for shape {shape}") + # For testing + err.skip_axes = skip_axes + err.shape = shape + new_skip_axes.append(s) + return new_skip_axes diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index c786eb5f..0c1a552f 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -11,7 +11,8 @@ from ..ndindex import ndindex from ..shapetools import (asshape, iter_indices, ncycles, BroadcastError, AxisError, broadcast_shapes, remove_indices, - unremove_indices, associated_axis) + unremove_indices, associated_axis, + canonical_skip_axes) from ..integer import Integer from ..tuple import Tuple from .helpers import (prod, mutually_broadcastable_shapes_with_skipped_axes, @@ -554,6 +555,81 @@ def test_asshape(): raises(TypeError, lambda: asshape(-1, allow_negative=True, allow_int=False)) raises(TypeError, lambda: asshape(np.int64(1), allow_int=False)) +@given(lists(tuples(integers(0))), + one_of(integers(), tuples(integers()), lists(tuples(integers())))) +def test_canonical_skip_axes(shapes, skip_axes): + if not shapes: + if skip_axes in [(), []]: + assert canonical_skip_axes(shapes, skip_axes) == [] + else: + raises(ValueError, lambda: canonical_skip_axes(shapes, skip_axes)) + return + + min_dim = min(len(shape) for shape in shapes) + + if isinstance(skip_axes, int): + if not (-min_dim <= skip_axes < min_dim): + raises(AxisError, lambda: canonical_skip_axes(shapes, skip_axes)) + return + _skip_axes = [(skip_axes,)]*len(shapes) + skip_len = 1 + elif isinstance(skip_axes, tuple): + if not all(-min_dim <= s < min_dim for s in skip_axes): + raises(AxisError, lambda: canonical_skip_axes(shapes, skip_axes)) + return + _skip_axes = [skip_axes]*len(shapes) + skip_len = len(skip_axes) + elif not skip_axes: + # empty list will be interpreted as a single skip_axes tuple + assert canonical_skip_axes(shapes, skip_axes) == [()]*len(shapes) + return + else: + if len(shapes) != len(skip_axes): + raises(ValueError, lambda: canonical_skip_axes(shapes, skip_axes)) + return + _skip_axes = skip_axes + skip_len = len(skip_axes[0]) + + try: + res = canonical_skip_axes(shapes, skip_axes) + except AxisError as e: + axis, ndim = e.args + assert any(axis in s for s in _skip_axes) + assert any(ndim == len(shape) for shape in shapes) + assert axis < -ndim or axis >= ndim + return + except ValueError as e: + if 'not unique' in str(e): + bad_skip_axes, bad_shape = e.skip_axes, e.shape + assert str(bad_skip_axes) in str(e) + assert str(bad_shape) in str(e) + assert bad_skip_axes in _skip_axes + assert bad_shape in shapes + indexed = [s[i] for s, i in zip(shapes, bad_skip_axes)] + assert len(indexed) != len(indexed) + return + else: + raise + + assert isinstance(res, list) + assert all(isinstance(x, tuple) for x in res) + assert all(isinstance(i, int) for x in res for i in x) + + assert len(res) == len(shapes) + for shape, new_skip_axes in zip(shapes, res): + assert len(new_skip_axes) == len(set(new_skip_axes)) == skip_len + for i in new_skip_axes: + assert ndindex(i).reduce(len(shape), negative_int=True) == i + + # TODO: Assert the order is maintained (doesn't actually matter for now + # but could for future applications) + +def test_canonical_skip_axes_errors(): + raises(TypeError, lambda: canonical_skip_axes([(1,)], {0: 1})) + raises(TypeError, lambda: canonical_skip_axes([(1,)], {0})) + raises(TypeError, lambda: canonical_skip_axes([(1,)], [(0,), 0])) + raises(TypeError, lambda: canonical_skip_axes([(1,)], [0, (0,)])) + @given(integers(), integers()) def test_axiserror(axis, ndim): if ndim == 0 and axis in [0, -1]: From 15bcade720e2253131d1a0e9018482e5db58967b Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 24 Apr 2023 17:02:24 -0600 Subject: [PATCH 162/218] Fix some bugs in canonical_skip_axes --- ndindex/shapetools.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index e984cbaa..70366355 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -497,9 +497,6 @@ def canonical_skip_axes(shapes, skip_axes): """ if isinstance(skip_axes, Sequence): if skip_axes and all(isinstance(i, Sequence) for i in skip_axes): - if not all(isinstance(skip_axis, Sequence) for skip_axis in - skip_axes): - raise TypeError("skip_axes must all be tuples of integers") if len(skip_axes) != len(shapes): raise ValueError(f"Expected {len(shapes)} skip_axes") return [canonical_skip_axes([shape], skip_axis)[0] for shape, skip_axis in zip(shapes, skip_axes)] @@ -524,5 +521,6 @@ def canonical_skip_axes(shapes, skip_axes): # For testing err.skip_axes = skip_axes err.shape = shape + raise err new_skip_axes.append(s) return new_skip_axes From 428f1f5b8063bd368de8b4b019e0f92bde9540ad Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 24 Apr 2023 17:02:42 -0600 Subject: [PATCH 163/218] Add some @examples for coverage --- ndindex/tests/test_shapetools.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 0c1a552f..1fb2e8f7 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -555,6 +555,12 @@ def test_asshape(): raises(TypeError, lambda: asshape(-1, allow_negative=True, allow_int=False)) raises(TypeError, lambda: asshape(np.int64(1), allow_int=False)) + +@example([(0, 1)], 0) +@example([(2, 3), (2, 3, 4)], [(3,), (0,)]) +@example([(0, 1)], 0) +@example([(2, 3)], (0, -2)) +@example([(2, 4), (2, 3, 4)], [(0,), (-3,)]) @given(lists(tuples(integers(0))), one_of(integers(), tuples(integers()), lists(tuples(integers())))) def test_canonical_skip_axes(shapes, skip_axes): From de7661ea32c1310c30f9d305b0cca0fdfd616c78 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 24 Apr 2023 17:02:50 -0600 Subject: [PATCH 164/218] Fix some logic in test_canonical_skip_axes() --- ndindex/tests/test_shapetools.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 1fb2e8f7..9468bd6e 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -611,10 +611,10 @@ def test_canonical_skip_axes(shapes, skip_axes): assert str(bad_shape) in str(e) assert bad_skip_axes in _skip_axes assert bad_shape in shapes - indexed = [s[i] for s, i in zip(shapes, bad_skip_axes)] - assert len(indexed) != len(indexed) + indexed = [bad_shape[i] for i in bad_skip_axes] + assert len(indexed) != len(set(indexed)) return - else: + else: # pragma: no cover raise assert isinstance(res, list) From 46b0831c40a0ffe29261dd11b013c46bc83e0f58 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 24 Apr 2023 17:04:59 -0600 Subject: [PATCH 165/218] Add a test for coverage --- ndindex/tests/test_shapetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 9468bd6e..d9f28a33 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -554,7 +554,7 @@ def test_asshape(): raises(TypeError, lambda: asshape(-1, allow_int=False)) raises(TypeError, lambda: asshape(-1, allow_negative=True, allow_int=False)) raises(TypeError, lambda: asshape(np.int64(1), allow_int=False)) - + raises(IndexError, lambda: asshape((2, 3), 3)) @example([(0, 1)], 0) @example([(2, 3), (2, 3, 4)], [(3,), (0,)]) From d59f0004b8b9491399c7bff6061445eccccaf7dd Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 24 Apr 2023 17:05:59 -0600 Subject: [PATCH 166/218] Add a TODO comment --- ndindex/tests/test_shapetools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index d9f28a33..1e176b75 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -523,6 +523,7 @@ def test_associated_axis(broadcastable_shapes, skip_axes): else: assert val == 1 or bval == val +# TODO: add a hypothesis test for asshape def test_asshape(): assert asshape(1) == (1,) assert asshape(np.int64(2)) == (2,) From 2f2399e8b6290706ad16ec0313c081ba271388c7 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 2 May 2023 02:46:23 -0600 Subject: [PATCH 167/218] Update mutually_broadcastable_shapes_with_skip_axes to generate lists of skip_axes This changes the way the strategy is generated to construct skip_axes and a result shape first, then generate shapes that match those skip axes and broadcast to the result shape. It doesn't attempt to directly use mutually_broadcastable_shapes. This is unfortunate, and it does require a little bit of filtering, but it is the only way that I could figure out how to generate the full range of possible examples. skip_axes_st is now always generated in conjunction with this strategy, as 1) generating the skip axes first and then the shapes doesn't seem to work, and 2) the two strategies are always used together anyway. This doesn't yet update all the tests that use this (nor are the functions themselves updated), but test_mutually_broadcastable_shapes_with_skip_axes() passes. --- ndindex/tests/helpers.py | 228 +++++++++++++++++++++---------- ndindex/tests/test_shapetools.py | 40 +++--- 2 files changed, 175 insertions(+), 93 deletions(-) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index 8f2366cf..3562be12 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -10,13 +10,12 @@ from hypothesis import assume, note from hypothesis.strategies import (integers, none, one_of, lists, just, - builds, shared, composite, sampled_from, - booleans) + builds, shared, composite, sampled_from) from hypothesis.extra.numpy import (arrays, mutually_broadcastable_shapes as - mbs, BroadcastableShapes) + mbs, BroadcastableShapes, valid_tuple_axes) from ..ndindex import ndindex -from ..shapetools import remove_indices, unremove_indices +from ..shapetools import remove_indices from .._crt import prod # Hypothesis strategies for generating indices. Note that some of these @@ -154,89 +153,170 @@ def _mutually_broadcastable_shapes(draw, *, shapes=short_shapes, min_shapes=0, m mutually_broadcastable_shapes = shared(_mutually_broadcastable_shapes()) -@composite -def _skip_axes_st(draw, - mutually_broadcastable_shapes=mutually_broadcastable_shapes, - num_skip_axes=None): - shapes, result_shape = draw(mutually_broadcastable_shapes) - if result_shape == (): - assume(num_skip_axes is None) - return () - negative = draw(booleans(), label='skip_axes < 0') - N = len(min(shapes, key=len)) - if num_skip_axes is not None: - min_size = max_size = num_skip_axes - assume(N >= num_skip_axes) - else: - min_size = 0 - max_size = None - if N == 0: - return () - if negative: - axes = draw(lists(integers(-N, -1), min_size=min_size, max_size=max_size, unique=True)) - else: - axes = draw(lists(integers(0, N-1), min_size=min_size, max_size=max_size, unique=True)) - axes = tuple(axes) - # Sometimes return an integer - if num_skip_axes is None and len(axes) == 1 and draw(booleans(), label='skip_axes integer'): # pragma: no cover - return axes[0] - return axes +def _fill_shape(draw, + *, + result_shape, + skip_axes, + skip_axes_values): + max_n = max([i + 1 if i >= 0 else -i for i in skip_axes], default=0) + assume(max_n <= len(skip_axes) + len(result_shape)) + dim = draw(integers(min_value=max_n, max_value=len(skip_axes) + len(result_shape))) + new_shape = ['placeholder']*dim + for i in skip_axes: + assume(new_shape[i] is not None) # skip_axes must be unique + new_shape[i] = None + j = -1 + for i in range(-1, -dim - 1, -1): + if new_shape[i] is None: + new_shape[i] = draw(skip_axes_values) + else: + new_shape[i] = draw(sampled_from([result_shape[j], 1])) + j -= 1 + while new_shape and new_shape[0] == 'placeholder': + # Can happen if positive and negative skip_axes refer to the same + # entry + new_shape.pop(0) + + # This will happen if the skip axes are too large + assume('placeholder' not in new_shape) + + if prod([i for i in new_shape if i]) >= SHORT_MAX_ARRAY_SIZE: + note(f"Filtering the shape {new_shape} (too many elements)") + assume(False) -skip_axes_st = shared(_skip_axes_st()) + return tuple(new_shape) + + +def _fill_result_shape(draw, + *, + result_shape, + skip_axes): + dim = len(result_shape) + len(skip_axes) + assume(all(-dim <= i < dim for i in skip_axes)) + new_shape = ['placeholder']*dim + for i in skip_axes: + assume(new_shape[i] is not None) # skip_axes must be unique + new_shape[i] = None + j = -1 + for i in range(-1, -dim - 1, -1): + if new_shape[i] is not None: + new_shape[i] = result_shape[j] + j -= 1 + while new_shape[:1] == ['placeholder']: + # Can happen if positive and negative skip_axes refer to the same + # entry + new_shape.pop(0) + + # This will happen if the skip axes are too large + assume('placeholder' not in new_shape) + + if prod([i for i in new_shape if i]) >= SHORT_MAX_ARRAY_SIZE: + note(f"Filtering the shape {new_shape} (too many elements)") + assume(False) + + return tuple(new_shape) + +skip_axes_with_broadcasted_shape_type = shared(sampled_from([int, tuple, list])) @composite -def mutually_broadcastable_shapes_with_skipped_axes(draw, skip_axes_st=skip_axes_st, mutually_broadcastable_shapes=mutually_broadcastable_shapes, -skip_axes_values=integers(0)): +def _mbs_and_skip_axes( + draw, + shapes=short_shapes, + min_shapes=0, + max_shapes=32, + skip_axes_type_st=skip_axes_with_broadcasted_shape_type, + skip_axes_values=integers(0, 20), + num_skip_axes=None, +): """ mutually_broadcastable_shapes except skip_axes() axes might not be broadcastable The result_shape will be None in the position of skip_axes. """ - skip_axes_ = draw(skip_axes_st) - shapes, result_shape = draw(mutually_broadcastable_shapes) - if isinstance(skip_axes_, int): - skip_axes_ = (skip_axes_,) - - # Randomize the shape values in the skipped axes - shapes_ = [] - for shape in shapes: - _shape = list(unremove_indices(shape, skip_axes_)) - # sanity check - assert remove_indices(_shape, skip_axes_) == shape, (_shape, skip_axes_, shape) - - # Replace None values with random values - for j in range(len(_shape)): - if _shape[j] is None: - _shape[j] = draw(skip_axes_values) - shapes_.append(tuple(_shape)) - - result_shape_ = unremove_indices(result_shape, skip_axes_) - # sanity check - assert remove_indices(result_shape_, skip_axes_) == result_shape - - for shape in shapes_: - if prod([i for i in shape if i]) >= SHORT_MAX_ARRAY_SIZE: - note(f"Filtering the shape {shape} (too many elements)") - assume(False) - return BroadcastableShapes(shapes_, result_shape_) - -two_mutually_broadcastable_shapes_1 = shared(_mutually_broadcastable_shapes( + skip_axes_type = draw(skip_axes_type_st) + _result_shape = draw(shapes) + if _result_shape == (): + assume(num_skip_axes is None) + + ndim = len(_result_shape) + num_shapes = draw(integers(min_value=min_shapes, max_value=max_shapes)) + if not ndim: + return BroadcastableShapes([()]*num_shapes, ()), () + + if num_skip_axes is not None: + min_skip_axes = max_skip_axes = num_skip_axes + else: + min_skip_axes = 0 + max_skip_axes = None + + # int and single tuple cases must be limited to N to ensure that they are + # correct for all shapes + if skip_axes_type == int: + skip_axes = draw(valid_tuple_axes(ndim, min_size=1, max_size=1))[0] + _skip_axes = [(skip_axes,)]*(num_shapes+1) + elif skip_axes_type == tuple: + skip_axes = draw(tuples(integers(-ndim, ndim-1), min_size=min_skip_axes, + max_size=max_skip_axes, unique=True)) + _skip_axes = [skip_axes]*(num_shapes+1) + elif skip_axes_type == list: + skip_axes = [] + for i in range(num_shapes): + skip_axes.append(draw(tuples(integers(-ndim, ndim+1), min_size=min_skip_axes, + max_size=max_skip_axes, unique=True))) + skip_axes.append(draw(lists(integers(-2*ndim, 2*ndim+1), + min_size=min_skip_axes, + max_size=max_skip_axes, unique=True))) + _skip_axes = skip_axes + + shapes = [] + for i in range(num_shapes): + shapes.append(_fill_shape(draw, result_shape=_result_shape, skip_axes=_skip_axes[i], + skip_axes_values=skip_axes_values)) + + non_skip_shapes = [remove_indices(shape, sk) for shape, sk in + zip(shapes, _skip_axes)] + # Broadcasting the result _fill_shape may produce a shape different from + # _result_shape because it might not have filled all dimensions, or it + # might have chosen 1 for a dimension every time. Ideally we would just be + # using shapes from mutually_broadcastable_shapes, but I don't know how to + # reverse inject skip axes into shapes in general (see the comment in + # unremove_indices). So for now, we just use the actual broadcast of the + # non-skip shapes. Note that we use np.broadcast_shapes here instead of + # ndindex.broadcast_shapes because test_broadcast_shapes itself uses this + # strategy. + broadcasted_shape = broadcast_shapes(*non_skip_shapes) + if _skip_axes: + _result_skip_axes = _skip_axes[-1] + result_shape = _fill_result_shape(draw, result_shape=broadcasted_shape, + skip_axes=_result_skip_axes) + assert remove_indices(result_shape, _result_skip_axes) == broadcasted_shape, (result_shape, _result_skip_axes, broadcasted_shape) + else: + result_shape = broadcasted_shape + + return BroadcastableShapes(shapes, result_shape), skip_axes + +mbs_and_skip_axes = shared(_mbs_and_skip_axes()) + +mutually_broadcastable_shapes_with_skipped_axes = mbs_and_skip_axes.map( + lambda i: i[0]) +skip_axes_st = mbs_and_skip_axes.map(lambda i: i[1]) + + +one_mbs_and_skip_axes = shared(_mbs_and_skip_axes( shapes=_short_shapes(1), min_shapes=2, - max_shapes=2, - min_side=1)) -one_skip_axes = shared(_skip_axes_st( - mutually_broadcastable_shapes=two_mutually_broadcastable_shapes_1, - num_skip_axes=1)) -two_mutually_broadcastable_shapes_2 = shared(_mutually_broadcastable_shapes( + max_shapes=2)) +one_mutually_broadcastable_shapes = one_mbs_and_skip_axes.map( + lambda i: i[0]) +one_skip_axes = one_mbs_and_skip_axes.map(lambda i: i[1]) +two_mbs_and_skip_axes = shared(_mbs_and_skip_axes( shapes=_short_shapes(2), min_shapes=2, - max_shapes=2, - min_side=2)) -two_skip_axes = shared(_skip_axes_st( - mutually_broadcastable_shapes=two_mutually_broadcastable_shapes_2, - num_skip_axes=2)) + max_shapes=2)) +two_mutually_broadcastable_shapes = two_mbs_and_skip_axes.map( + lambda i: i[0]) +two_skip_axes = two_mbs_and_skip_axes.map(lambda i: i[1]) reduce_kwargs = sampled_from([{}, {'negative_int': False}, {'negative_int': True}]) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 1e176b75..491e8af4 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -17,14 +17,14 @@ from ..tuple import Tuple from .helpers import (prod, mutually_broadcastable_shapes_with_skipped_axes, skip_axes_st, mutually_broadcastable_shapes, tuples, - shapes, two_mutually_broadcastable_shapes_1, - two_mutually_broadcastable_shapes_2, one_skip_axes, + shapes, two_mutually_broadcastable_shapes, + two_mutually_broadcastable_shapes, one_skip_axes, two_skip_axes, assert_equal) @example([((1, 1), (1, 1)), (None, 1)], (0,)) @example([((0,), (0,)), (None,)], (0,)) @example([((1, 2), (2, 1)), (2, None)], 1) -@given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes_st) +@given(mutually_broadcastable_shapes_with_skipped_axes, skip_axes_st) def test_iter_indices(broadcastable_shapes, skip_axes): # broadcasted_shape will contain None on the skip_axes, as those axes # might not be broadcast compatible @@ -149,10 +149,10 @@ def _move_slices_to_end(idx): else: assert set(vals) == set(correct_vals) -cross_shapes = mutually_broadcastable_shapes_with_skipped_axes( - mutually_broadcastable_shapes=two_mutually_broadcastable_shapes_1, - skip_axes_st=one_skip_axes, - skip_axes_values=integers(3, 3)) +# cross_shapes = mutually_broadcastable_shapes_with_skipped_axes( +# mutually_broadcastable_shapes=two_mutually_broadcastable_shapes, +# skip_axes_st=one_skip_axes, +# skip_axes_values=integers(3, 3)) @composite def cross_arrays_st(draw): @@ -171,7 +171,7 @@ def cross_arrays_st(draw): return a, b -@given(cross_arrays_st(), cross_shapes, one_skip_axes) +# @given(cross_arrays_st(), cross_shapes, one_skip_axes) def test_iter_indices_cross(cross_arrays, broadcastable_shapes, skip_axes): # Test iter_indices behavior against np.cross, which effectively skips the # crossed axis. Note that we don't test against cross products of size 2 @@ -202,7 +202,7 @@ def test_iter_indices_cross(cross_arrays, broadcastable_shapes, skip_axes): @composite def _matmul_shapes(draw): broadcastable_shapes = draw(mutually_broadcastable_shapes_with_skipped_axes( - mutually_broadcastable_shapes=two_mutually_broadcastable_shapes_2, + mutually_broadcastable_shapes=two_mutually_broadcastable_shapes, skip_axes_st=two_skip_axes, skip_axes_values=just(None), )) @@ -400,7 +400,7 @@ def test_broadcast_shapes_errors(shapes): raises(TypeError, lambda: broadcast_shapes(1, 2, (2, 2))) raises(TypeError, lambda: broadcast_shapes([(1, 2), (2, 2)])) -@given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes_st) +@given(mutually_broadcastable_shapes_with_skipped_axes, skip_axes_st) def test_broadcast_shapes_skip_axes(broadcastable_shapes, skip_axes): shapes, broadcasted_shape = broadcastable_shapes assert broadcast_shapes(*shapes, skip_axes=skip_axes) == broadcasted_shape @@ -412,9 +412,9 @@ def test_broadcast_shapes_skip_axes(broadcastable_shapes, skip_axes): @given(mutually_broadcastable_shapes, lists(integers(-20, 20), max_size=20)) def test_broadcast_shapes_skip_axes_errors(broadcastable_shapes, skip_axes): shapes, broadcasted_shape = broadcastable_shapes - if any(i < 0 for i in skip_axes) and any(i >= 0 for i in skip_axes): - raises(NotImplementedError, lambda: broadcast_shapes(*shapes, skip_axes=skip_axes)) - return + # if any(i < 0 for i in skip_axes) and any(i >= 0 for i in skip_axes): + # raises(NotImplementedError, lambda: broadcast_shapes(*shapes, skip_axes=skip_axes)) + # return try: if not shapes and skip_axes: @@ -476,19 +476,21 @@ def test_remove_indices(n, idxes): raises(NotImplementedError, lambda: unremove_indices(b, idxes)) # Meta-test for the hypothesis strategy -@given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes_st) +@given(mutually_broadcastable_shapes_with_skipped_axes, skip_axes_st) def test_mutually_broadcastable_shapes_with_skipped_axes(broadcastable_shapes, skip_axes): # pragma: no cover shapes, broadcasted_shape = broadcastable_shapes - _skip_axes = (skip_axes,) if isinstance(skip_axes, int) else skip_axes + _skip_axes = canonical_skip_axes(shapes + [broadcasted_shape], skip_axes) + + assert len(_skip_axes) == len(shapes) + 1 for shape in shapes: assert None not in shape - for i in _skip_axes: + for i in _skip_axes[-1]: assert broadcasted_shape[i] is None - _shapes = [remove_indices(shape, skip_axes) for shape in shapes] - _broadcasted_shape = remove_indices(broadcasted_shape, skip_axes) + _shapes = [remove_indices(shape, sk) for shape, sk in zip(shapes, _skip_axes)] + _broadcasted_shape = remove_indices(broadcasted_shape, _skip_axes[-1]) assert None not in _broadcasted_shape assert broadcast_shapes(*_shapes) == _broadcasted_shape @@ -498,7 +500,7 @@ def test_mutually_broadcastable_shapes_with_skipped_axes(broadcastable_shapes, (1, None, 1, 0, None, 2, 3, 4)], (1, 4)) @example([[(2, 0, 3, 4)], (2, None, 3, 4)], (1,)) @example([[(0, 0, 0, 0, 0), (0, 0, 0, 0, 0, 0)], (0, None, None, 0, 0, 0)], (1, 2)) -@given(mutually_broadcastable_shapes_with_skipped_axes(), skip_axes_st) +@given(mutually_broadcastable_shapes_with_skipped_axes, skip_axes_st) def test_associated_axis(broadcastable_shapes, skip_axes): _skip_axes = (skip_axes,) if isinstance(skip_axes, int) else skip_axes From 29b2122a83c2d377b414f8de768a7ad9d86ba387 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sun, 7 May 2023 00:45:50 -0600 Subject: [PATCH 168/218] No longer add None indices in the return value of broadcast_shapes with skip_axes This is impossible to do in general if we allow both negative and nonnegative skip_axes, and it's also too complex to check the result shape skip axes to make sure they are consistent. Unfortunately, this is a backwards compatibility break, but hopefully no one is actually using this function yet. --- ndindex/shapetools.py | 40 ++++++++++--------------- ndindex/tests/helpers.py | 50 +++++--------------------------- ndindex/tests/test_shapetools.py | 35 +++++++++------------- 3 files changed, 36 insertions(+), 89 deletions(-) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index 70366355..8826048a 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -65,52 +65,41 @@ def broadcast_shapes(*shapes, skip_axes=()): Axes in `skip_axes` apply to each shape *before* being broadcasted. Each shape will be broadcasted together with these axes removed. The dimensions in skip_axes do not need to be equal or broadcast compatible with one - another. The final broadcasted shape will have `None` in each `skip_axes` - location, and the broadcasted remaining `shapes` axes elsewhere. + another. The final broadcasted shape be the result of broadcasting all the + non-skip axes. >>> broadcast_shapes((10, 3, 2), (20, 2), skip_axes=(0,)) - (None, 3, 2) + (3, 2) """ - skip_axes = asshape(skip_axes, allow_negative=True) shapes = [asshape(shape, allow_int=False) for shape in shapes] - - if any(i >= 0 for i in skip_axes) and any(i < 0 for i in skip_axes): - # See the comments in remove_indices and iter_indices - raise NotImplementedError("Mixing both negative and nonnegative skip_axes is not yet supported") + skip_axes = canonical_skip_axes(shapes, skip_axes) if not shapes: - if skip_axes: - # Raise IndexError - ndindex(skip_axes[0]).reduce(0) return () - dims = [len(shape) for shape in shapes] - shape_skip_axes = [[ndindex(i).reduce(n, negative_int=True) for i in skip_axes] for n in dims] + non_skip_shapes = [remove_indices(shape, skip_axis) for shape, skip_axis in zip(shapes, skip_axes)] + dims = [len(shape) for shape in non_skip_shapes] N = max(dims) - broadcasted_skip_axes = [ndindex(i).reduce(N) for i in skip_axes] - broadcasted_shape = [None if i in broadcasted_skip_axes else 1 for i in range(N)] + broadcasted_shape = [1]*N arg = None for i in range(-1, -N-1, -1): for j in range(len(shapes)): if dims[j] < -i: continue - shape = shapes[j] - idx = associated_axis(shape, broadcasted_shape, i, skip_axes) - broadcasted_side = broadcasted_shape[idx] + shape = non_skip_shapes[j] + broadcasted_side = broadcasted_shape[i] shape_side = shape[i] - if i in shape_skip_axes[j]: - continue - elif shape_side == 1: + if shape_side == 1: continue elif broadcasted_side == 1: broadcasted_side = shape_side arg = j elif shape_side != broadcasted_side: raise BroadcastError(arg, shapes[arg], j, shapes[j]) - broadcasted_shape[idx] = broadcasted_side + broadcasted_shape[i] = broadcasted_side return tuple(broadcasted_shape) @@ -415,8 +404,8 @@ def unremove_indices(x, idxes, *, val=None): This function is only intended for internal usage. """ if any(i >= 0 for i in idxes) and any(i < 0 for i in idxes): - # A mix of positive and negative indices provides a fundamental - # problem. Sometimes, the result is not unique: for example, x = [0]; + # A mix of positive and negative indices presents a fundamental + # problem: sometimes the result is not unique. For example, x = [0]; # idxes = [1, -1] could be satisfied by both [0, None] or [0, None, # None], depending on whether each index refers to a separate None or # not (note that both cases are supported by remove_indices(), because @@ -495,6 +484,7 @@ def canonical_skip_axes(shapes, skip_axes): This function is only intended for internal usage. """ + # Note: we assume asshape has already been called on the shapes in shapes if isinstance(skip_axes, Sequence): if skip_axes and all(isinstance(i, Sequence) for i in skip_axes): if len(skip_axes) != len(shapes): @@ -511,7 +501,7 @@ def canonical_skip_axes(shapes, skip_axes): # From here, skip_axes is a single tuple of integers if not shapes and skip_axes: - raise ValueError(f"Expected {len(shapes)} skip_axes") + raise ValueError("skip_axes must be empty if there are no shapes") new_skip_axes = [] for shape in shapes: diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index 3562be12..64e1ec39 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -186,36 +186,6 @@ def _fill_shape(draw, return tuple(new_shape) - -def _fill_result_shape(draw, - *, - result_shape, - skip_axes): - dim = len(result_shape) + len(skip_axes) - assume(all(-dim <= i < dim for i in skip_axes)) - new_shape = ['placeholder']*dim - for i in skip_axes: - assume(new_shape[i] is not None) # skip_axes must be unique - new_shape[i] = None - j = -1 - for i in range(-1, -dim - 1, -1): - if new_shape[i] is not None: - new_shape[i] = result_shape[j] - j -= 1 - while new_shape[:1] == ['placeholder']: - # Can happen if positive and negative skip_axes refer to the same - # entry - new_shape.pop(0) - - # This will happen if the skip axes are too large - assume('placeholder' not in new_shape) - - if prod([i for i in new_shape if i]) >= SHORT_MAX_ARRAY_SIZE: - note(f"Filtering the shape {new_shape} (too many elements)") - assume(False) - - return tuple(new_shape) - skip_axes_with_broadcasted_shape_type = shared(sampled_from([int, tuple, list])) @composite @@ -241,6 +211,9 @@ def _mbs_and_skip_axes( ndim = len(_result_shape) num_shapes = draw(integers(min_value=min_shapes, max_value=max_shapes)) + if not num_shapes: + assume(num_skip_axes is None) + num_skip_axes = 0 if not ndim: return BroadcastableShapes([()]*num_shapes, ()), () @@ -253,20 +226,18 @@ def _mbs_and_skip_axes( # int and single tuple cases must be limited to N to ensure that they are # correct for all shapes if skip_axes_type == int: + assume(num_skip_axes in [None, 1]) skip_axes = draw(valid_tuple_axes(ndim, min_size=1, max_size=1))[0] - _skip_axes = [(skip_axes,)]*(num_shapes+1) + _skip_axes = [(skip_axes,)]*num_shapes elif skip_axes_type == tuple: skip_axes = draw(tuples(integers(-ndim, ndim-1), min_size=min_skip_axes, max_size=max_skip_axes, unique=True)) - _skip_axes = [skip_axes]*(num_shapes+1) + _skip_axes = [skip_axes]*num_shapes elif skip_axes_type == list: skip_axes = [] for i in range(num_shapes): skip_axes.append(draw(tuples(integers(-ndim, ndim+1), min_size=min_skip_axes, max_size=max_skip_axes, unique=True))) - skip_axes.append(draw(lists(integers(-2*ndim, 2*ndim+1), - min_size=min_skip_axes, - max_size=max_skip_axes, unique=True))) _skip_axes = skip_axes shapes = [] @@ -286,15 +257,8 @@ def _mbs_and_skip_axes( # ndindex.broadcast_shapes because test_broadcast_shapes itself uses this # strategy. broadcasted_shape = broadcast_shapes(*non_skip_shapes) - if _skip_axes: - _result_skip_axes = _skip_axes[-1] - result_shape = _fill_result_shape(draw, result_shape=broadcasted_shape, - skip_axes=_result_skip_axes) - assert remove_indices(result_shape, _result_skip_axes) == broadcasted_shape, (result_shape, _result_skip_axes, broadcasted_shape) - else: - result_shape = broadcasted_shape - return BroadcastableShapes(shapes, result_shape), skip_axes + return BroadcastableShapes(shapes, broadcasted_shape), skip_axes mbs_and_skip_axes = shared(_mbs_and_skip_axes()) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 491e8af4..7f2d4379 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -18,8 +18,7 @@ from .helpers import (prod, mutually_broadcastable_shapes_with_skipped_axes, skip_axes_st, mutually_broadcastable_shapes, tuples, shapes, two_mutually_broadcastable_shapes, - two_mutually_broadcastable_shapes, one_skip_axes, - two_skip_axes, assert_equal) + one_skip_axes, two_skip_axes, assert_equal) @example([((1, 1), (1, 1)), (None, 1)], (0,)) @example([((0,), (0,)), (None,)], (0,)) @@ -409,38 +408,32 @@ def test_broadcast_shapes_skip_axes(broadcastable_shapes, skip_axes): @example([[(0, 1)], (0, 1)], (2,)) @example([[(0, 1)], (0, 1)], (0, -1)) @example([[(0, 1, 0, 0, 0), (2, 0, 0, 0)], (0, 2, 0, 0, 0)], [1]) -@given(mutually_broadcastable_shapes, lists(integers(-20, 20), max_size=20)) +@given(mutually_broadcastable_shapes, + one_of( + integers(-20, 20), + tuples(integers(-20, 20), max_size=20), + lists(tuples(integers(-20, 20), max_size=20), max_size=32))) def test_broadcast_shapes_skip_axes_errors(broadcastable_shapes, skip_axes): shapes, broadcasted_shape = broadcastable_shapes - # if any(i < 0 for i in skip_axes) and any(i >= 0 for i in skip_axes): - # raises(NotImplementedError, lambda: broadcast_shapes(*shapes, skip_axes=skip_axes)) - # return + # All errors should come from canonical_skip_axes, which is tested + # separately below. try: - if not shapes and skip_axes: - raise IndexError - for shape in shapes: - for i in skip_axes: - shape[i] - except IndexError: - error = True - else: - error = False + canonical_skip_axes(shapes, skip_axes) + except (TypeError, ValueError, IndexError) as e: + raises(type(e), lambda: broadcast_shapes(*shapes, + skip_axes=skip_axes)) + return try: broadcast_shapes(*shapes, skip_axes=skip_axes) except IndexError: - if not error: # pragma: no cover - raise RuntimeError("broadcast_shapes raised but should not have") - return + raise RuntimeError("broadcast_shapes raised but should not have") except BroadcastError: # Broadcastable shapes can become unbroadcastable after skipping axes # (see the @example above). pass - if error: # pragma: no cover - raise RuntimeError("broadcast_shapes did not raise but should have") - remove_indices_n = shared(integers(0, 100)) @given(remove_indices_n, From 2a6c9f16eb017da96911871c482191c3b9629cc1 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sun, 7 May 2023 00:54:31 -0600 Subject: [PATCH 169/218] Update test_mutually_broadcastable_shapes_with_skipped_axes --- ndindex/tests/test_shapetools.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 7f2d4379..df4587cf 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -473,20 +473,17 @@ def test_remove_indices(n, idxes): def test_mutually_broadcastable_shapes_with_skipped_axes(broadcastable_shapes, skip_axes): # pragma: no cover shapes, broadcasted_shape = broadcastable_shapes - _skip_axes = canonical_skip_axes(shapes + [broadcasted_shape], skip_axes) + _skip_axes = canonical_skip_axes(shapes, skip_axes) - assert len(_skip_axes) == len(shapes) + 1 + assert len(_skip_axes) == len(shapes) for shape in shapes: assert None not in shape - for i in _skip_axes[-1]: - assert broadcasted_shape[i] is None + assert None not in broadcasted_shape _shapes = [remove_indices(shape, sk) for shape, sk in zip(shapes, _skip_axes)] - _broadcasted_shape = remove_indices(broadcasted_shape, _skip_axes[-1]) - assert None not in _broadcasted_shape - assert broadcast_shapes(*_shapes) == _broadcasted_shape + assert broadcast_shapes(*_shapes) == broadcasted_shape @example([[(2, 10, 3, 4), (10, 3, 4)], (2, None, 3, 4)], (-3,)) @example([[(0, 10, 2, 3, 10, 4), (1, 10, 1, 0, 10, 2, 3, 4)], From 6d4202e688269de7a976324d287ffadad256caf1 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sun, 7 May 2023 01:03:33 -0600 Subject: [PATCH 170/218] Rename canonical_skip_axes() to normalize_skip_axes() --- ndindex/shapetools.py | 6 +++--- ndindex/tests/test_shapetools.py | 32 ++++++++++++++++---------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index 8826048a..cd05d85b 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -73,7 +73,7 @@ def broadcast_shapes(*shapes, skip_axes=()): """ shapes = [asshape(shape, allow_int=False) for shape in shapes] - skip_axes = canonical_skip_axes(shapes, skip_axes) + skip_axes = normalize_skip_axes(shapes, skip_axes) if not shapes: return () @@ -465,7 +465,7 @@ def __repr__(self): def __iter__(self): return itertools.chain.from_iterable(itertools.repeat(tuple(self.iterable), self.n)) -def canonical_skip_axes(shapes, skip_axes): +def normalize_skip_axes(shapes, skip_axes): """ Return a canonical form of `skip_axes` corresponding to `shapes`. @@ -489,7 +489,7 @@ def canonical_skip_axes(shapes, skip_axes): if skip_axes and all(isinstance(i, Sequence) for i in skip_axes): if len(skip_axes) != len(shapes): raise ValueError(f"Expected {len(shapes)} skip_axes") - return [canonical_skip_axes([shape], skip_axis)[0] for shape, skip_axis in zip(shapes, skip_axes)] + return [normalize_skip_axes([shape], skip_axis)[0] for shape, skip_axis in zip(shapes, skip_axes)] else: try: [operator_index(i) for i in skip_axes] diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index df4587cf..1a939a65 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -12,7 +12,7 @@ from ..shapetools import (asshape, iter_indices, ncycles, BroadcastError, AxisError, broadcast_shapes, remove_indices, unremove_indices, associated_axis, - canonical_skip_axes) + normalize_skip_axes) from ..integer import Integer from ..tuple import Tuple from .helpers import (prod, mutually_broadcastable_shapes_with_skipped_axes, @@ -419,7 +419,7 @@ def test_broadcast_shapes_skip_axes_errors(broadcastable_shapes, skip_axes): # All errors should come from canonical_skip_axes, which is tested # separately below. try: - canonical_skip_axes(shapes, skip_axes) + normalize_skip_axes(shapes, skip_axes) except (TypeError, ValueError, IndexError) as e: raises(type(e), lambda: broadcast_shapes(*shapes, skip_axes=skip_axes)) @@ -473,7 +473,7 @@ def test_remove_indices(n, idxes): def test_mutually_broadcastable_shapes_with_skipped_axes(broadcastable_shapes, skip_axes): # pragma: no cover shapes, broadcasted_shape = broadcastable_shapes - _skip_axes = canonical_skip_axes(shapes, skip_axes) + _skip_axes = normalize_skip_axes(shapes, skip_axes) assert len(_skip_axes) == len(shapes) @@ -556,41 +556,41 @@ def test_asshape(): @example([(2, 4), (2, 3, 4)], [(0,), (-3,)]) @given(lists(tuples(integers(0))), one_of(integers(), tuples(integers()), lists(tuples(integers())))) -def test_canonical_skip_axes(shapes, skip_axes): +def test_normalize_skip_axes(shapes, skip_axes): if not shapes: if skip_axes in [(), []]: - assert canonical_skip_axes(shapes, skip_axes) == [] + assert normalize_skip_axes(shapes, skip_axes) == [] else: - raises(ValueError, lambda: canonical_skip_axes(shapes, skip_axes)) + raises(ValueError, lambda: normalize_skip_axes(shapes, skip_axes)) return min_dim = min(len(shape) for shape in shapes) if isinstance(skip_axes, int): if not (-min_dim <= skip_axes < min_dim): - raises(AxisError, lambda: canonical_skip_axes(shapes, skip_axes)) + raises(AxisError, lambda: normalize_skip_axes(shapes, skip_axes)) return _skip_axes = [(skip_axes,)]*len(shapes) skip_len = 1 elif isinstance(skip_axes, tuple): if not all(-min_dim <= s < min_dim for s in skip_axes): - raises(AxisError, lambda: canonical_skip_axes(shapes, skip_axes)) + raises(AxisError, lambda: normalize_skip_axes(shapes, skip_axes)) return _skip_axes = [skip_axes]*len(shapes) skip_len = len(skip_axes) elif not skip_axes: # empty list will be interpreted as a single skip_axes tuple - assert canonical_skip_axes(shapes, skip_axes) == [()]*len(shapes) + assert normalize_skip_axes(shapes, skip_axes) == [()]*len(shapes) return else: if len(shapes) != len(skip_axes): - raises(ValueError, lambda: canonical_skip_axes(shapes, skip_axes)) + raises(ValueError, lambda: normalize_skip_axes(shapes, skip_axes)) return _skip_axes = skip_axes skip_len = len(skip_axes[0]) try: - res = canonical_skip_axes(shapes, skip_axes) + res = normalize_skip_axes(shapes, skip_axes) except AxisError as e: axis, ndim = e.args assert any(axis in s for s in _skip_axes) @@ -623,11 +623,11 @@ def test_canonical_skip_axes(shapes, skip_axes): # TODO: Assert the order is maintained (doesn't actually matter for now # but could for future applications) -def test_canonical_skip_axes_errors(): - raises(TypeError, lambda: canonical_skip_axes([(1,)], {0: 1})) - raises(TypeError, lambda: canonical_skip_axes([(1,)], {0})) - raises(TypeError, lambda: canonical_skip_axes([(1,)], [(0,), 0])) - raises(TypeError, lambda: canonical_skip_axes([(1,)], [0, (0,)])) +def test_normalize_skip_axes_errors(): + raises(TypeError, lambda: normalize_skip_axes([(1,)], {0: 1})) + raises(TypeError, lambda: normalize_skip_axes([(1,)], {0})) + raises(TypeError, lambda: normalize_skip_axes([(1,)], [(0,), 0])) + raises(TypeError, lambda: normalize_skip_axes([(1,)], [0, (0,)])) @given(integers(), integers()) def test_axiserror(axis, ndim): From 8007c64d8660e08b453702b21c68d803c16bbd64 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 16 May 2023 02:01:18 -0600 Subject: [PATCH 171/218] Sort the axes in normalize_skip_axes --- ndindex/shapetools.py | 5 +++-- ndindex/tests/test_shapetools.py | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index cd05d85b..7d024253 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -479,7 +479,8 @@ def normalize_skip_axes(shapes, skip_axes): The `skip_axes` must always refer to unique axes in each shape. - The returned `skip_axes` will always be negative integers. + The returned `skip_axes` will always be negative integers and will be + sorted. This function is only intended for internal usage. @@ -505,7 +506,7 @@ def normalize_skip_axes(shapes, skip_axes): new_skip_axes = [] for shape in shapes: - s = tuple(ndindex(i).reduce(len(shape), negative_int=True, axiserror=True).raw for i in skip_axes) + s = tuple(sorted(ndindex(i).reduce(len(shape), negative_int=True, axiserror=True).raw for i in skip_axes)) if len(s) != len(set(s)): err = ValueError(f"skip_axes {skip_axes} are not unique for shape {shape}") # For testing diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 1a939a65..de4cad9e 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -617,7 +617,9 @@ def test_normalize_skip_axes(shapes, skip_axes): assert len(res) == len(shapes) for shape, new_skip_axes in zip(shapes, res): assert len(new_skip_axes) == len(set(new_skip_axes)) == skip_len + assert new_skip_axes == tuple(sorted(new_skip_axes)) for i in new_skip_axes: + assert i < 0 assert ndindex(i).reduce(len(shape), negative_int=True) == i # TODO: Assert the order is maintained (doesn't actually matter for now From a45b3ca844d7febc581a69939aca7218d456c2be Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 25 May 2023 00:40:21 -0600 Subject: [PATCH 172/218] Update iter_indices to support arbitrary skip_axes This also changes the definition of the internal associated_axis() function. --- ndindex/shapetools.py | 87 ++++++++++++------------------------------- 1 file changed, 24 insertions(+), 63 deletions(-) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index 7d024253..424b4c10 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -1,7 +1,7 @@ import numbers import itertools -from collections import defaultdict from collections.abc import Sequence +from ._crt import prod from .ndindex import ndindex, operator_index @@ -208,28 +208,9 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): Tuple(1, 2) """ - skip_axes = asshape(skip_axes, allow_negative=True) + skip_axes = normalize_skip_axes(shapes, skip_axes) shapes = [asshape(shape, allow_int=False) for shape in shapes] - if any(i >= 0 for i in skip_axes) and any(i < 0 for i in skip_axes): - # Mixing positive and negative skip_axes is too difficult to deal with - # (see the comment in unremove_indices). It's a bit of an unusual - # thing to support, at least in the general case, because a positive - # and negative index can index the same element, but only for shapes - # that are a specific size. So while, in principle something like - # iter_indices((2, 10, 20, 4), (2, 30, 4), skip_axes=(1, -2)) could - # make sense, it's a bit odd to do so. Of course, there's no reason we - # couldn't support cases like that, but they complicate the - # implementation and, especially, complicate the test generation in - # the hypothesis strategies. Given that I'm not completely sure how to - # implement it correctly, and I don't actually need support for it, - # I'm leaving it as not implemented for now. - raise NotImplementedError("Mixing both negative and nonnegative skip_axes is not yet supported") - - n = len(skip_axes) - if len(set(skip_axes)) != n: - raise ValueError("skip_axes should not contain duplicate axes") - if not shapes: if skip_axes: raise AxisError(skip_axes[0], 0) @@ -238,37 +219,29 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): shapes = [asshape(shape) for shape in shapes] ndim = len(max(shapes, key=len)) - min_ndim = len(min(shapes, key=len)) - - _skip_axes = defaultdict(list) - for shape in shapes: - for a in skip_axes: - try: - a = ndindex(a).reduce(len(shape)).raw - except IndexError: - raise AxisError(a, min_ndim) - _skip_axes[shape].append(a) - _skip_axes[shape].sort() iters = [[] for i in range(len(shapes))] broadcasted_shape = broadcast_shapes(*shapes, skip_axes=skip_axes) - non_skip_broadcasted_shape = remove_indices(broadcasted_shape, skip_axes) for i in range(-1, -ndim-1, -1): - for it, shape in zip(iters, shapes): + for it, shape, sk in zip(iters, shapes, skip_axes): + val = associated_axis(i, broadcasted_shape, sk) if -i > len(shape): # for every dimension prepended by broadcasting, repeat the # indices that many times for j in range(len(it)): - if non_skip_broadcasted_shape[i+n] not in [0, 1]: - it[j] = ncycles(it[j], non_skip_broadcasted_shape[i+n]) + if val not in [None, 0, 1]: + it[j] = ncycles(it[j], val) break - elif len(shape) + i in _skip_axes[shape]: - it.insert(0, [slice(None)]) + elif i in sk: + if len(shape) == ndim and len(sk) == len(shape) and val: + # The whole shape is skipped. This normally would be + # cycled by the previous block but in this case it isn't + # because the shape already has ndim dimensions. + it.insert(0, ncycles([slice(None)], val)) + else: + it.insert(0, [slice(None)]) else: - idx = associated_axis(shape, broadcasted_shape, i, skip_axes) - val = broadcasted_shape[idx] - assert val is not None if val == 0: return elif val != 1 and shape[i] == 1: @@ -345,7 +318,7 @@ def asshape(shape, axis=None, *, allow_int=True, allow_negative=False): return tuple(newshape) -def associated_axis(shape, broadcasted_shape, i, skip_axes): +def associated_axis(i, broadcasted_shape, skip_axes): """ Return the associated index into `broadcast_shape` corresponding to `shape[i]` given `skip_axes`. @@ -354,30 +327,18 @@ def associated_axis(shape, broadcasted_shape, i, skip_axes): designed for internal use. """ - n = len(shape) - N = len(broadcasted_shape) skip_axes = sorted(skip_axes, reverse=True) if i >= 0: raise NotImplementedError - if not skip_axes: - return i - # We assume skip_axes are either all negative or all nonnegative - if skip_axes[0] < 0: - return i - elif skip_axes[0] >= 0: - invmapping = [None]*N - for s in skip_axes: - invmapping[s] = s - - for j in range(-1, i-1, -1): - if j + n in skip_axes: - k = j + n #- N - continue - for k in range(-1, -N-1, -1): - if invmapping[k] is None: - invmapping[k] = j - break - return k + # We assume skip_axes are all negative and sorted + for sk in skip_axes: + if sk >= i: + i += 1 + else: + break + if ndindex(i).isvalid(len(broadcasted_shape)): + return broadcasted_shape[i] + return None def remove_indices(x, idxes): """ From 2eb09327c29680e191b11fbad26927b2bb004d2f Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 25 May 2023 00:42:59 -0600 Subject: [PATCH 173/218] Update test_iter_indices to handle arbitrary skip_axes The testing strategy is changed a little bit. We just test against the broadcasted arrays with the skip axes removed, since the number of skip axes is not necessarily uniform across the shapes anymore. --- ndindex/tests/test_shapetools.py | 85 +++++++++++++------------------- 1 file changed, 34 insertions(+), 51 deletions(-) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index de4cad9e..2ee0418b 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -20,9 +20,9 @@ shapes, two_mutually_broadcastable_shapes, one_skip_axes, two_skip_axes, assert_equal) -@example([((1, 1), (1, 1)), (None, 1)], (0,)) -@example([((0,), (0,)), (None,)], (0,)) -@example([((1, 2), (2, 1)), (2, None)], 1) +@example([[(1, 1), (1, 1)], (1,)], (0,)) +@example([[(0,), (0,)], ()], (0,)) +@example([[(1, 2), (2, 1)], (2,)], 1) @given(mutually_broadcastable_shapes_with_skipped_axes, skip_axes_st) def test_iter_indices(broadcastable_shapes, skip_axes): # broadcasted_shape will contain None on the skip_axes, as those axes @@ -33,94 +33,76 @@ def test_iter_indices(broadcastable_shapes, skip_axes): assume(len(broadcasted_shape) < 32) # 1. Normalize inputs - _skip_axes = (skip_axes,) if isinstance(skip_axes, int) else skip_axes - - # Double check the mutually_broadcastable_shapes_with_skipped_axes - # strategy - for i in _skip_axes: - assert broadcasted_shape[i] is None + _skip_axes = normalize_skip_axes(shapes, skip_axes) + _skip_axes_kwarg_default = [()]*len(shapes) # Skipped axes may not be broadcast compatible. Since the index for a # skipped axis should always be a slice(None), the result should be the # same if the skipped axes are all moved to the end of the shape. canonical_shapes = [] - for s in shapes: - c = remove_indices(s, _skip_axes) - c = c + tuple(1 for i in _skip_axes) + for s, sk in zip(shapes, _skip_axes): + c = remove_indices(s, sk) canonical_shapes.append(c) - canonical_skip_axes = list(range(-1, -len(_skip_axes) - 1, -1)) - broadcasted_canonical_shape = list(broadcast_shapes(*canonical_shapes, - skip_axes=canonical_skip_axes)) - for i in range(len(broadcasted_canonical_shape)): - if broadcasted_canonical_shape[i] is None: - broadcasted_canonical_shape[i] = 1 - - skip_shapes = [tuple(1 for i in _skip_axes) for shape in shapes] - non_skip_shapes = [remove_indices(shape, skip_axes) for shape in shapes] - broadcasted_non_skip_shape = remove_indices(broadcasted_shape, skip_axes) - assert None not in broadcasted_non_skip_shape - assert broadcast_shapes(*non_skip_shapes) == broadcasted_non_skip_shape - - nitems = prod(broadcasted_non_skip_shape) - - if _skip_axes == (): + + non_skip_shapes = [remove_indices(shape, sk) for shape, sk in zip(shapes, _skip_axes)] + assert np.broadcast_shapes(*non_skip_shapes) == broadcasted_shape + + nitems = prod(broadcasted_shape) + + if skip_axes == (): # kwarg default res = iter_indices(*shapes) - broadcasted_res = iter_indices(broadcast_shapes(*shapes)) else: res = iter_indices(*shapes, skip_axes=skip_axes) - broadcasted_res = iter_indices(broadcasted_canonical_shape, - skip_axes=canonical_skip_axes) + broadcasted_res = iter_indices(broadcasted_shape) sizes = [prod(shape) for shape in shapes] arrays = [np.arange(size).reshape(shape) for size, shape in zip(sizes, shapes)] canonical_sizes = [prod(shape) for shape in canonical_shapes] canonical_arrays = [np.arange(size).reshape(shape) for size, shape in zip(canonical_sizes, canonical_shapes)] + canonical_broadcasted_array = np.arange(nitems).reshape(broadcasted_shape) # 2. Check that iter_indices is the same whether or not the shapes are # broadcasted together first. Also check that every iterated index is the # expected type and there are as many as expected. vals = [] + bvals = [] n = -1 - def _move_slices_to_end(idx): + def _remove_slices(idx): assert isinstance(idx, Tuple) - idx2 = list(idx.args) - slices = [i for i in range(len(idx2)) if idx2[i] == slice(None)] - idx2 = remove_indices(idx2, slices) - idx2 = idx2 + (slice(None),)*len(slices) + idx2 = [i for i in idx.args if i != slice(None)] return Tuple(*idx2) for n, (idxes, bidxes) in enumerate(zip(res, broadcasted_res)): assert len(idxes) == len(shapes) assert len(bidxes) == 1 - for idx, shape in zip(idxes, shapes): + for idx, shape, sk in zip(idxes, shapes, _skip_axes): assert isinstance(idx, Tuple) assert len(idx.args) == len(shape) - normalized_skip_axes = sorted(ndindex(i).reduce(len(shape)).raw for i in _skip_axes) - for i in range(len(idx.args)): - if i in normalized_skip_axes: + for i in range(-1, -len(idx.args) - 1, -1): + if i in sk: assert idx.args[i] == slice(None) else: assert isinstance(idx.args[i], Integer) - canonical_idxes = [_move_slices_to_end(idx) for idx in idxes] + canonical_idxes = [_remove_slices(idx) for idx in idxes] a_indexed = tuple([a[idx.raw] for a, idx in zip(arrays, idxes)]) canonical_a_indexed = tuple([a[idx.raw] for a, idx in zip(canonical_arrays, canonical_idxes)]) + canonical_b_indexed = canonical_broadcasted_array[bidxes[0].raw] - for c_indexed, skip_shape in zip(canonical_a_indexed, skip_shapes): - assert c_indexed.shape == skip_shape + for c_indexed in canonical_a_indexed: + assert c_indexed.shape == () + assert canonical_b_indexed.shape == () - if _skip_axes: - # If there are skipped axes, recursively call iter_indices to - # get each individual element of the resulting subarrays. - for subidxes in iter_indices(*[x.shape for x in canonical_a_indexed]): - items = [x[i.raw] for x, i in zip(canonical_a_indexed, subidxes)] - vals.append(tuple(items)) + if _skip_axes != _skip_axes_kwarg_default: + vals.append(tuple(canonical_a_indexed)) else: vals.append(a_indexed) + bvals.append(canonical_b_indexed) + # assert both iterators have the same length raises(StopIteration, lambda: next(res)) raises(StopIteration, lambda: next(broadcasted_res)) @@ -137,16 +119,17 @@ def _move_slices_to_end(idx): if not arrays: assert vals == [()] else: - correct_vals = [tuple(i) for i in np.stack(np.broadcast_arrays(*canonical_arrays), axis=-1).reshape((nitems, len(arrays)))] + correct_vals = list(zip(*[x.flat for x in np.broadcast_arrays(*canonical_arrays)])) # Also test that the indices are produced in a lexicographic order # (even though this isn't strictly guaranteed by the iter_indices # docstring) in the case when there are no skip axes. The order when # there are skip axes is more complicated because the skipped axes are # iterated together. - if not _skip_axes: + if _skip_axes == _skip_axes_kwarg_default: assert vals == correct_vals else: assert set(vals) == set(correct_vals) + assert bvals == list(canonical_broadcasted_array.flat) # cross_shapes = mutually_broadcastable_shapes_with_skipped_axes( # mutually_broadcastable_shapes=two_mutually_broadcastable_shapes, From 053c14af88e5d79544258f1d7515b561693267f6 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 25 May 2023 00:55:53 -0600 Subject: [PATCH 174/218] Fix an issue in iter_indices Only generate slices for completely skipped shapes once. --- ndindex/shapetools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index 424b4c10..9d934d6b 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -234,11 +234,11 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): it[j] = ncycles(it[j], val) break elif i in sk: - if len(shape) == ndim and len(sk) == len(shape) and val: + if len(shape) == ndim and len(sk) == len(shape) and i == -1: # The whole shape is skipped. This normally would be # cycled by the previous block but in this case it isn't # because the shape already has ndim dimensions. - it.insert(0, ncycles([slice(None)], val)) + it.insert(0, ncycles([slice(None)], prod(broadcasted_shape))) else: it.insert(0, [slice(None)]) else: From 7da16dad61c5a88ed66343054ba9d98791076502 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 25 May 2023 00:58:01 -0600 Subject: [PATCH 175/218] Fix a bug in associated_axis --- ndindex/shapetools.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index 9d934d6b..e4fa6960 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -331,13 +331,14 @@ def associated_axis(i, broadcasted_shape, skip_axes): if i >= 0: raise NotImplementedError # We assume skip_axes are all negative and sorted + j = i for sk in skip_axes: if sk >= i: - i += 1 + j += 1 else: break - if ndindex(i).isvalid(len(broadcasted_shape)): - return broadcasted_shape[i] + if ndindex(j).isvalid(len(broadcasted_shape)): + return broadcasted_shape[j] return None def remove_indices(x, idxes): From a75e1bc5e18e0af4ad26c89c969d9bb046108d63 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 8 Jun 2023 01:05:12 -0600 Subject: [PATCH 176/218] Change the order of the arguments to associated_axis() --- ndindex/shapetools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index e4fa6960..3e7be1ad 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -225,7 +225,7 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): for i in range(-1, -ndim-1, -1): for it, shape, sk in zip(iters, shapes, skip_axes): - val = associated_axis(i, broadcasted_shape, sk) + val = associated_axis(broadcasted_shape, i, sk) if -i > len(shape): # for every dimension prepended by broadcasting, repeat the # indices that many times @@ -318,7 +318,7 @@ def asshape(shape, axis=None, *, allow_int=True, allow_negative=False): return tuple(newshape) -def associated_axis(i, broadcasted_shape, skip_axes): +def associated_axis(broadcasted_shape, i, skip_axes): """ Return the associated index into `broadcast_shape` corresponding to `shape[i]` given `skip_axes`. From 38b784d1954d6f576602d296069e4e2c2a667771 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 8 Jun 2023 01:05:32 -0600 Subject: [PATCH 177/218] Fix the docstring of associated_axis() --- ndindex/shapetools.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index 3e7be1ad..061c97a3 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -320,8 +320,9 @@ def asshape(shape, axis=None, *, allow_int=True, allow_negative=False): def associated_axis(broadcasted_shape, i, skip_axes): """ - Return the associated index into `broadcast_shape` corresponding to - `shape[i]` given `skip_axes`. + Return the associated element of `broadcasted_shape` corresponding to + `shape[i]` given `skip_axes`. If there is not such element (i.e., it's out + of bounds), returns None. This function makes implicit assumptions about its input and is only designed for internal use. From c6ca2a655f660d38b2c59c3bbe690bfd46006b8b Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 8 Jun 2023 01:05:52 -0600 Subject: [PATCH 178/218] Always return None if the index is skipped in associated_axis() --- ndindex/shapetools.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index 061c97a3..9976a14e 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -331,6 +331,8 @@ def associated_axis(broadcasted_shape, i, skip_axes): skip_axes = sorted(skip_axes, reverse=True) if i >= 0: raise NotImplementedError + if i in skip_axes: + return None # We assume skip_axes are all negative and sorted j = i for sk in skip_axes: From 06b00bd66726c10b0542f03b655d77883123c84b Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 8 Jun 2023 01:06:09 -0600 Subject: [PATCH 179/218] Update the test for associated_axis() --- ndindex/tests/test_shapetools.py | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 2ee0418b..177121b2 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -468,35 +468,26 @@ def test_mutually_broadcastable_shapes_with_skipped_axes(broadcastable_shapes, assert broadcast_shapes(*_shapes) == broadcasted_shape -@example([[(2, 10, 3, 4), (10, 3, 4)], (2, None, 3, 4)], (-3,)) +@example([[(2, 10, 3, 4), (10, 3, 4)], (2, 3, 4)], (-3,)) @example([[(0, 10, 2, 3, 10, 4), (1, 10, 1, 0, 10, 2, 3, 4)], - (1, None, 1, 0, None, 2, 3, 4)], (1, 4)) -@example([[(2, 0, 3, 4)], (2, None, 3, 4)], (1,)) -@example([[(0, 0, 0, 0, 0), (0, 0, 0, 0, 0, 0)], (0, None, None, 0, 0, 0)], (1, 2)) + (1, 1, 0, 2, 3, 4)], (1, 4)) +@example([[(2, 0, 3, 4)], (2, 3, 4)], (1,)) +@example([[(0, 0, 0, 0, 0), (0, 0, 0, 0, 0, 0)], (0, 0, 0, 0)], (1, 2)) @given(mutually_broadcastable_shapes_with_skipped_axes, skip_axes_st) def test_associated_axis(broadcastable_shapes, skip_axes): - _skip_axes = (skip_axes,) if isinstance(skip_axes, int) else skip_axes - shapes, broadcasted_shape = broadcastable_shapes - ndim = len(broadcasted_shape) - - normalized_skip_axes = [ndindex(i).reduce(ndim) for i in _skip_axes] + _skip_axes = normalize_skip_axes(shapes, skip_axes) - for shape in shapes: + for shape, sk in zip(shapes, _skip_axes): n = len(shape) for i in range(-len(shape), 0): val = shape[i] - idx = associated_axis(shape, broadcasted_shape, i, _skip_axes) - bval = broadcasted_shape[idx] + bval = associated_axis(broadcasted_shape, i, sk) if bval is None: - if _skip_axes[0] >= 0: - assert ndindex(i).reduce(n) == ndindex(idx).reduce(ndim) in normalized_skip_axes - else: - assert ndindex(i).reduce(n, negative_int=True) == \ - ndindex(idx).reduce(ndim, negative_int=True) in _skip_axes + assert ndindex(i).reduce(n, negative_int=True) in sk, (shape, i) else: - assert val == 1 or bval == val + assert val == 1 or bval == val, (shape, i) # TODO: add a hypothesis test for asshape def test_asshape(): From 2f13d7d8a9a5787b3e5d8615ae98db067ed77780 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Sun, 16 Jul 2023 13:39:14 +0200 Subject: [PATCH 180/218] Use configparser.ConfigParser instead of SafeConfigParser The latter is a compat alias (since python 3.2) that was finally removed in python 3.12. https://bugs.python.org/issue45173 --- versioneer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versioneer.py b/versioneer.py index 13901fcd..1e461ba0 100644 --- a/versioneer.py +++ b/versioneer.py @@ -339,9 +339,9 @@ def get_config_from_root(root): # configparser.NoOptionError (if it lacks "VCS="). See the docstring at # the top of versioneer.py for instructions on writing your setup.cfg . setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.SafeConfigParser() + parser = configparser.ConfigParser() with open(setup_cfg, "r") as f: - parser.readfp(f) + parser.read_file(f) VCS = parser.get("versioneer", "VCS") # mandatory def get(parser, name): From 3763937b7c986a35c17c816699fdc81c1859191c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Sun, 16 Jul 2023 13:46:40 +0200 Subject: [PATCH 181/218] setup.py: specify cython language_level Cython still defaults to compat with python2, but ndindex requires >=3.7, so this is not useful. Specifying the level allows slightly shorter code and gets rid of a bunch of warnings: /usr/lib64/python3.12/site-packages/Cython/Compiler/Main.py:369: FutureWarning: Cython directive 'language_level' not set, using 2 for now (Py2). This will change in a later release! File: /builddir/build/BUILD/ndindex-1.7/ndindex/_crt.py tree = Parsing.p_module(s, pxd, full_module_name) ... --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 5d7e7efe..f39b71f9 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,9 @@ def check_cython(): from Cython.Build import cythonize sys.argv = argv_org[:1] + ["build_ext"] setuptools.setup(name="foo", version="1.0.0", - ext_modules=cythonize(["ndindex/__init__.py"])) + ext_modules=cythonize( + ["ndindex/__init__.py"], + language_level="3")) except: return False finally: @@ -37,7 +39,8 @@ def check_cython(): if use_cython: from Cython.Build import cythonize - ext_modules = cythonize(["ndindex/*.py"]) + ext_modules = cythonize(["ndindex/*.py"], + language_level="3") else: ext_modules = [] From c7c9e21679c21beacbcc057b43a7daa12f251e3b Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sun, 16 Jul 2023 09:31:17 -0500 Subject: [PATCH 182/218] Use pip instead of conda on CI --- .github/workflows/tests.yml | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 68a0327e..1f0691b9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,38 +17,13 @@ jobs: set -x set -e # $CONDA is an environment variable pointing to the root of the miniconda directory - echo $CONDA/bin >> $GITHUB_PATH - conda config --set always_yes yes --set changeps1 no - conda config --add channels conda-forge - conda update -q conda - conda info -a - conda create -n test-environment python=${{ matrix.python-version }} pyflakes pytest pytest-doctestplus numpy hypothesis pytest-cov pytest-flakes packaging - conda init + python -m pip install pyflakes pytest pytest-doctestplus numpy hypothesis pytest-cov pytest-flakes packaging - name: Run Tests run: | - # Copied from .bashrc. We can't just source .bashrc because it exits - # when the shell isn't interactive. - - # >>> conda initialize >>> - # !! Contents within this block are managed by 'conda init' !! - __conda_setup="$('/usr/share/miniconda/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" - if [ $? -eq 0 ]; then - eval "$__conda_setup" - else - if [ -f "/usr/share/miniconda/etc/profile.d/conda.sh" ]; then - . "/usr/share/miniconda/etc/profile.d/conda.sh" - else - export PATH="/usr/share/miniconda/bin:$PATH" - fi - fi - unset __conda_setup - # <<< conda initialize <<< - set -x set -e - conda activate test-environment python -We:invalid -We::SyntaxWarning -m compileall -f -q ndindex/ # The coverage requirement check is done by the coverage report line below PYTEST_FLAGS="$PYTEST_FLAGS -v --cov-fail-under=0"; From 34b3b6448b1835abb8fe650a6a2fca797cbdfceb Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sun, 16 Jul 2023 09:31:26 -0500 Subject: [PATCH 183/218] Add Python 3.12-dev to CI --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1f0691b9..d463a58b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,7 +5,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12-dev'] fail-fast: false steps: - uses: actions/checkout@v2 From ed29010d1b884b6e306689093beaaab7c70a24b0 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sun, 16 Jul 2023 09:37:17 -0500 Subject: [PATCH 184/218] Add @examples for coverage --- ndindex/tests/test_integerarray.py | 2 ++ ndindex/tests/test_isvalid.py | 1 + 2 files changed, 3 insertions(+) diff --git a/ndindex/tests/test_integerarray.py b/ndindex/tests/test_integerarray.py index e3c05cdd..27ae73cd 100644 --- a/ndindex/tests/test_integerarray.py +++ b/ndindex/tests/test_integerarray.py @@ -57,6 +57,8 @@ def test_integerarray_reduce_no_shape_unchanged(idx): if index.ndim != 0: assert index.reduce() == index + +@example(array([2, -2]), (4,), {'negative_int': True}) @example(array(2), (4,), {'negative_int': True}) @example(array([2, 0]), (1, 0), {}) @example(array(0), 1, {}) diff --git a/ndindex/tests/test_isvalid.py b/ndindex/tests/test_isvalid.py index 2b8c4551..e307c345 100644 --- a/ndindex/tests/test_isvalid.py +++ b/ndindex/tests/test_isvalid.py @@ -5,6 +5,7 @@ from .helpers import ndindices, shapes, MAX_ARRAY_SIZE, check_same, prod +@example(..., (1, 2, 3)) @example(slice(0, 1), ()) @example(slice(0, 1), (1,)) @example((0, 1), (2, 2)) From 68914896b40eb8e9659b68208ce81bdfcf3ef4b7 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sun, 16 Jul 2023 09:39:02 -0500 Subject: [PATCH 185/218] Remove Travis CI files --- .travis.yml | 61 -------------------- github_deploy_key_quansight_labs_ndindex.enc | 1 - 2 files changed, 62 deletions(-) delete mode 100644 .travis.yml delete mode 100644 github_deploy_key_quansight_labs_ndindex.enc diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 3d870151..00000000 --- a/.travis.yml +++ /dev/null @@ -1,61 +0,0 @@ -language: python - -dist: xenial - -matrix: - include: - - python: 3.7 - env: - - TESTS=true - - python: 3.8 - env: - - TESTS=true - - DOCS=true - # Doctr deploy key for Quansight-Labs/ndindex - - secure: "oLAqRes+Qylu3xU753wlruJH/V8904rDfQfutk1SFvpxPHDZlxj8RbwpvkVKRPBU0nucIs75z9WRm2XdeJB/bdwMcx0gkL5N99L0BbPxKgicsPI6IPTe7aI5KmStEyQw65xO3Vu9vlGxMtDP2HIEEpq7brDvVAg/lRBcPn2ahujW0BUPVnoch3CZKLsE7f05Yeyc5aBCsOVCUHpV2riptabjAqjgJJVYa1BdlyUals99oRB671kFqyiBSzoeSAriII/joLGwNDaUlYc0EmUoSZZmNKc0I5xAXGwIwhgmvhZ2dwqiy2G253apnHaFyf/wqLyvQPqf8Fr6MVW3hc/EujDqE3y7pwR+UANXvEfbDqCxchhyRfNHswyARwD+DqailOt5voL4q/GMh8NnqMT1aEedAByZ/d+iX3npxwGSHx0qoBJs0HivPuN8t9qJGufI/ux66oASwJQeOZLfed0vVH5A/P3tmEV0IichCfj4horvr1A+h7tZcgN313MI0Lap2+6WnodF1b2AvIR/02OMWBna+P8UCfG4RR7i5Bm8S/jWKV/GVZyN3ACWciwV/NjDr5dxayDYrrm+s3akfLYuZzQGF/gmaNjbDgIN+lUn7Zm4Ayp4yoKz7tBeS8Uhcg5tcSJmoJj77X6XriL9uO9JIkmNRq3pIYHGwlOv8q83fJI=" - - name: python 3.9 - env: - - PYTHON_VERSION=3.9 - - TESTS=true - -install: - - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh; - - bash miniconda.sh -b -p $HOME/miniconda - - export PATH="$HOME/miniconda/bin:$PATH" - - hash -r - - conda config --set always_yes yes --set changeps1 no - - conda config --add channels conda-forge - - conda update -q conda - - conda info -a - - conda create -n test-environment python=${PYTHON_VERSION:-$TRAVIS_PYTHON_VERSION} pyflakes pytest pytest-doctestplus hypothesis doctr sphinx myst-parser sphinx_rtd_theme pytest-cov pytest-flakes - - source activate test-environment - - pip install --pre numpy>=1.20 - -script: - - set -e - - python -We:invalid -We::SyntaxWarning -m compileall -f -q ndindex/ - # The coverage requirement check is done by the coverage report line below - - PYTEST_FLAGS="$PYTEST_FLAGS -v --cov-fail-under=0"; - - pytest $PYTEST_FLAGS - - ./run_doctests - # Make sure it installs - - python setup.py install - - if [[ "${DOCS}" == "true" ]]; then - cd docs; - make html; - cd ..; - if [[ "${TRAVIS_BRANCH}" == "master" ]]; then - doctr deploy .; - else - doctr deploy --no-require-master "_docs-$TRAVIS_BRANCH"; - fi - fi - # Coverage. This also sets the failing status if the - # coverage is not 100%. Travis sometimes cuts off the last command, which is - # why we print stuff at the end. - - if ! coverage report -m; then - echo "Coverage failed"; - false; - else - echo "Coverage passed"; - fi; diff --git a/github_deploy_key_quansight_labs_ndindex.enc b/github_deploy_key_quansight_labs_ndindex.enc deleted file mode 100644 index 78d11c5e..00000000 --- a/github_deploy_key_quansight_labs_ndindex.enc +++ /dev/null @@ -1 +0,0 @@ -gAAAAABgeI3n7QXDzdcsx7Px8UM3QsoRMz8gUpAHctdQWzrdxKqBPgAkC_iF9RWdFESBjpxWHRhnldS3iwSqPhvHttDjcg4lZVJOfTROwe0SepRlCCf2IrsJng4aXxDms4UAXc8XtwqaZCXAuEMl017D_C7ce9VoIuIfJqer5cXKU662I-56dtfYWGvpcYgjmTrehSnrlBh0b_ZDOEhqgWauqQo0o23eaglszV9434o2TkKj2w-sESc9Fc92goA5WO8G6Y4EgMAlJNpByHIJ6ryFVN4-NL-ZO1YQ0PN6NVzlN1oNt5djoRXSvd9Yraz5L73myzHNZJVWAlKHMbOHuXcLdSY8crn4uJ7QTeDtoaesDU3qywkTEDC-CV82o5GJ9WNLrrKjPlOjdgUWVxUdpGGLh3nhWad_z2fTIkMerRxirUyHO0jL32tJttmSYLi49kmGJ-2GPLtgynm1hxQ66rGMrha7FseSHVRoUStnwEBECBtzVClkY6PQK_5aywGXywOSIwB9sgD0golfCawZkP4uAfyjBSkEdwRb4_f3chJv7Wh5WFguCFgVp78NilyEEyUSf20_lsWwgEfQRdXO_GJFyJqVTUQpekc2Iv_D2LQA01W7FbrXhg6gfARG_1uiCzacU8YUyrGfqCCAbeR9YgCoDbca6KM63vNyCMW328nGz_NvA8Vz0iKOCf10EsUjbr8lVufVav1125mvNvNCQzpAWpcjJDUw242Ztv0kiKVd0QiUmolDnXnY_2nNsY2VCXpSD6AottoaZkXaVBQXjQm8yW1cuSWoOSdw4HvyK8ibdV4Y8IAVgA8U7l4a_fMfrzawjd3KIkgCOYS_SwXT8o-Vc6D3zoWvUWDTtlPDl5V6PwggAdHvYThmDyScDacyIQ0HeZC93jrmtr71JrZXzo41zzGojQrgUx8Q7xqKIN2tlOEpeQJd9IaSGQvsJ9wslxOqQzrUCiWpm10bdwZB73cQDyNzX1gkVPUPUBco7A6cBIaa6iUwhdkUoE549DicLYNbpXvCPQhfpEzw9AP6APhGGH03z4sy2PbwDsi-Js3EVu78ahE34H_GuMQ94KjMPLD9_x3qQpZQKE8dmKJeakCOlmXUhtunfWzkUnroP1RPbuk7B_WDTsOCKPaMv32tSNQ06J4mGIGRQchPKQvptUHRZMQP6NANqarUUbrvzzKKFowRL3ZDfxjdUaeq0cKyCAq5fOy4IZgchGXvVKDcZ8ij2m0Lxeg5PiKJPm-nyLROJ2O3HWivLT_Id-rfddivP8yaCaB72DUI1MsUT6dlmN6jrDPKN92H3bYCx1FbnoaXBQ3j6BCt6dp2mydBZjh2vdDs2oeaL4Nlk3QQXbCF6dLPwoT058Ws2V1VPVvgVDFxPJe5ZUILf85BN_zVHkL89DGqbBRwsftrwILHaBrJBBSHB8wNjG3meAY0DEPwKki4ya4N1Av3Z_rCdbtAaMmUbuQNayZZBExTfp5qw2eJ5QRo0KGkPKjmiGpnno8u-Poy2Ybn3wFDkIdd7Vk_7xohALRXAG2SSKpQDOsvAC3aheXMzqxpxpKN4_j6YBhF-TGmZfAVHYa11TVsDv5nQ9hKMfnqcuEtPt7fI78kWLebwRWNiAm1kaFoG_Mv8e6DOo2se7aqTtwT7vDwmzir4R25CvyrlqboVER_oapeBi18j0sCMBCJMTn54F05PFb2KnnrRlluxH_sZj4prvIRNGz706rCMewLe2hjtxx7PkxMguXozbSWWpmgnPk5XZIcfT46jIuDMVQH5WNbpJEgkQcBWJxD8CXwdmGF38JTVIAu4x_jkZ8fHBKUHmn1qYQwX1gAkKQcqm0SluRSYkHnPLavf26hqdbjFSL-G2oColkiT2kELB1ZYBzwxeGKKCaBqFMEY0Rmjd7Ob7Vcd7DUCnK-b_h60F6bKnvLIupIhP5rbN0EmWf3SCeUh0t1RbRD_LGfNY3yLS_pz1qLkmS3qAijg6v5upOKpvciCx43U9mew3jn6zvAO_SfKKh4OmfrkKDG3geaV86qzGK3Rg93IEqt_M8jKAIF0boFnLWtrQjTCil3W6-9kk3SWLJtUJoMZ_kggMDTkkqg_oIE1IgYbCBoXvHXoaHvV4q5n7Uso9Ds5ag4b4uv8x9N710OtkNFEzjtvHALGEyYT9xfQ5x47bniYSz4WbJ0O8EwukH66NeXJHQ1KUc2S8A9iV0wJSoeZfVa8IkNt04hxxRNrXfBvp66R_ndPCFRulS56AgF4iYczgE5O2QvdhXZu8fdVErD91Vq0ck8moKCIVkrri1kWpSVx2vfjK0EJtixTHprpKpVBRKJXsKnVpRqaFnDAK9SILVUAHWRM5Jmw1zcyR36ozV2YoqQubDkQ0UGUBXKbbXHzIUndqCw0EMmQGPcY_iZ0WTAt-rlX5nYMy-GlHhab3Vwb9NBbUMhJ6hjSwwQqpSb_rspOKQU8k2C5sr-nKDk5UEsqqG_QNZcOkioN0Wu3zJ95RcirfBOWVuOymW5dW2VUBP4qvA8m5gnPQiEk7_hAoOPEIncSRnqVSg47JboRLIt6LcedOzMeOiF9WY7mKqxwK_XRgxDCC3Jxv4RtDkECY78qePSb1WiS1kKcozxPtK96FZFu3CGzb4lz5iouG0yXXQY0H6Vnu7hVD-U5fL5qv1g_4v4aEluZKI5p2IS5_Nz1Rc9AFTQ_9EFnEhL0bPgXUAoxDhBgkRgkg7zL-4PkxnLTpMwIsP6l8Rb7tcqD5vj-FZd2v3ty4iPdiFPj9tlMw0d79npiaMSCb4F7F1D37OeCaIGvAvrHHl-G7HgcnDgcYlTp9EGdJXYd8iyc59Mip7-3DdQazRukK7g7gNN0ZiGyXfElUR3VPIQKCyZ45y3rLEwt-4oyHLyLSZwI-xUsPMXJ1yRopR9VgZCMXUGkgCJ9DvkFdO7PaKW-8qo3aJA0dACSNnfiuiMrwaRNm8Jnyzun34uOAVwTLuYB3XEd0hrIu6_brwdvU6JKpYFu9pLVwuzBNLmE5ftiiD8yZjn5WFqvlBtX4SUJG9zY-aJeKHwkt8XzllTwGdiN0Zhm5rlhvnWP18m7NMGNJzYnkaWOSuUyZYryYzJaTTcq1a-E-ikEtWxf_PMxwi5QLjhTjSx5SBXElb46OdHfS3gjAmap3qKF6qUK8b0DCRlBExIfNumBRKqO8K_ivNOCwGDk8620dJ33jSaeX4uU38EVXBxPO2znkXtOX0K31u4tq4H7cYMRSqsaiojOrBoD4tOJvmdHV0gmhV1oWcedA7MIpkToRVUM-mjY5-0GFFYWzTVq8H_ZP2gSWT0tEE_2sr_IrTZcQaMq5SGd6aLrLdedv-WLlA1WQ-_aYx5Zeme8jEM9z6hlI5Y__5EqO6jHpKVm4uLRJZCSMobD0zL8EHevkMtJxjgdEmnpHAumKIpRhSYOnPBp_fSZM7mIIclSvRfD6pb2fs9dMVtKZUlPSyoNlDszYbDFR_RlXD47tSd8MOnYLBHo6sG-hBd-vz-yUIDyGGcKGXuDNJ8kk0jdOwmx8BKvQiAqfEw8AjY5TL06XETSheNPKAj7rTdAbrBb70e_9wAx056wZUnwez-N4IIT8xB_8OT3NYu00d3AN4SK2a7NJX1tSYef4ofkYSXOPcb9eF8vyU-3IE1b2ySTMQeedvLiyngPp0E0_RWv_P4p0LHdYhP_d68DbHHYpcmBpTBlc3n0mGS4uCW9-cf3PKQJqkkJ6ubCi_cBX5kX5Qppq2ZlYI_QXn23g4uHQjZc-YGSVXeijFNH1rn06TnHz8tucz9OdTQN1eCtspUjC863K3VN9ChdnfNWSom6knYqN0PfagkgoyxpStDxcadyWZp0mmf2PKc6V5ofj_i6NNXgqoBeWiz5be9xnpm0JOS4ANDhxEIWEIhXmJsTo0EeWxXerABbDH6wSIBPyMPqoyZRNWO9mFAioBnQQYaZa4iHBdusSr1E3XDZli6uUoBkFAU_bidUF0o-ed36r7efx1EzUOq1JMbnItVVHz6ztMiO3A3XoD1up8F6f_BGOK3NQ_spa-khBHxMQFAPgOX9oY5jHziDhMu47sjNB9MYLkzFrhJFsXPES9rtLACrHlAy21IHV5Zqn1f1LAq2abbUTQK6Z0YZFrwWigfeWoYKwB7IcnhlttZ76LgaDwpwfsO0ks1x8_UfzDyCsd-z6B4Gsmo4GtDcw2Xdr_fs8Km61yfAxbohDHerEaMByz382AT36OGkesqEndoW1V2bIbaHuveBmLOssjZ5-cflKZ38dc_Dj_JlCIwXjUJxrNdtxjWZMmW9CQviWXJkI8eW8ngpTztrEl14eH4XtSI5J66bgVbxAzr69AEj81SE6mbb8S3du4uHTWJmtP_RkgC1o4BiYAcP0Uk3aH9F9GYlRKort7YIq7qvy8CSQ-A_whPhFdbg9G4q5cFEI9zifTr3akE6g== \ No newline at end of file From 8439d9a3fbe9b0599908b834f15e9a3b801cb38d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sun, 16 Jul 2023 09:41:39 -0500 Subject: [PATCH 186/218] Drop support for Python 3.7 It has reached end-of-life https://devguide.python.org/versions/ --- .github/workflows/tests.yml | 2 +- ndindex/shapetools.py | 6 ++---- ndindex/tests/doctest.py | 14 +------------- setup.py | 2 +- 4 files changed, 5 insertions(+), 19 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d463a58b..38e9db9e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,7 +5,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12-dev'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12-dev'] fail-fast: false steps: - uses: actions/checkout@v2 diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index bec43130..7e035d0f 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -187,7 +187,7 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): >>> b array([[100], [110]]) - >>> for idx1, idx2 in iter_indices((1, 3), (2, 1)): # doctest: +SKIP37 + >>> for idx1, idx2 in iter_indices((1, 3), (2, 1)): ... print(f"{idx1 = }; {idx2 = }; {(a[idx1.raw], b[idx2.raw]) = }") idx1 = Tuple(0, 0); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (0, 100) idx1 = Tuple(0, 1); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (1, 100) @@ -285,9 +285,7 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): it.insert(0, range(shape[i])) if _debug: # pragma: no cover - print(iters) - # Use this instead when we drop Python 3.7 support - # print(f"{iters = }") + print(f"{iters = }") for idxes in itertools.zip_longest(*[itertools.product(*i) for i in iters], fillvalue=()): yield tuple(ndindex(idx) for idx in idxes) diff --git a/ndindex/tests/doctest.py b/ndindex/tests/doctest.py index d6917141..c110c60e 100644 --- a/ndindex/tests/doctest.py +++ b/ndindex/tests/doctest.py @@ -3,12 +3,6 @@ This runs the doctests but ignores trailing ``` in Markdown documents. -This also adds the flag SKIP37 to allow skipping doctests in Python 3.7. - ->>> import sys ->>> sys.version_info[1] > 7 # doctest: +SKIP37 -True - Running this separately from pytest also allows us to not include the doctests in the coverage. It also allows us to force a separate namespace for each docstring's doctest, which the pytest doctest integration does not allow. @@ -28,11 +22,7 @@ import os from contextlib import contextmanager from doctest import (DocTestRunner, DocFileSuite, DocTestSuite, - NORMALIZE_WHITESPACE, register_optionflag, SKIP) - -SKIP37 = register_optionflag("SKIP37") - -PY37 = sys.version_info[:2] == (3, 7) + NORMALIZE_WHITESPACE) @contextmanager def patch_doctest(): @@ -45,8 +35,6 @@ def patch_doctest(): def run(self, test, **kwargs): for example in test.examples: - if PY37 and SKIP37 in example.options: - example.options[SKIP] = True # Remove ``` example.want = example.want.replace('```\n', '') example.exc_msg = example.exc_msg and example.exc_msg.replace('```\n', '') diff --git a/setup.py b/setup.py index f39b71f9..376e75cc 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ def check_cython(): "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], - python_requires='>=3.7', + python_requires='>=3.8', ) print("CYTHONIZE_NDINDEX: %r" % CYTHONIZE_NDINDEX) From b9248b9844b83ec4fa08b3b64e12799689298501 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sun, 16 Jul 2023 14:47:36 -0500 Subject: [PATCH 187/218] Install the numpy nightly wheel in 3.12 on CI --- .github/workflows/tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 38e9db9e..09ce08a0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,7 +18,9 @@ jobs: set -e # $CONDA is an environment variable pointing to the root of the miniconda directory python -m pip install pyflakes pytest pytest-doctestplus numpy hypothesis pytest-cov pytest-flakes packaging - + if [[ ${{ matrix.python-version }} == "3.12-dev" ]]; then + pip install --pre --upgrade --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy + fi; - name: Run Tests run: | set -x From 2db8245f7e10ddf9f949a43011e3285ca909f9b0 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sun, 16 Jul 2023 14:50:04 -0500 Subject: [PATCH 188/218] Don't install the release numpy in 3.12 --- .github/workflows/tests.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 09ce08a0..db29e8c5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,10 +16,11 @@ jobs: run: | set -x set -e - # $CONDA is an environment variable pointing to the root of the miniconda directory - python -m pip install pyflakes pytest pytest-doctestplus numpy hypothesis pytest-cov pytest-flakes packaging + python -m pip install pyflakes pytest pytest-doctestplus hypothesis pytest-cov pytest-flakes packaging if [[ ${{ matrix.python-version }} == "3.12-dev" ]]; then - pip install --pre --upgrade --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy + python -m pip install --pre --upgrade --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy + else + python -m pip install numpy fi; - name: Run Tests run: | From 3244de1189b29e0d566cf11f68f5cd81f92919ad Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Sun, 16 Jul 2023 15:04:19 -0500 Subject: [PATCH 189/218] Make hash(Slice) == hash(Slice.raw) in Python 3.12+ Fixes #158 --- ndindex/slice.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ndindex/slice.py b/ndindex/slice.py index a36dbb72..bda8e26d 100644 --- a/ndindex/slice.py +++ b/ndindex/slice.py @@ -76,9 +76,11 @@ def _typecheck(self, start, stop=default, step=None): return args def __hash__(self): - # We can't use the default hash(self.raw) because slices are not - # hashable - return hash(self.args) + # Slices are only hashable in Python 3.12+ + try: + return hash(self.raw) + except TypeError: + return hash(self.args) @property def raw(self): From d81d08108f6f5ce285cc29b989f773c5a2002d42 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 27 Jul 2023 17:28:04 -0600 Subject: [PATCH 190/218] Rewrite iter_indices() It is (hopefully) actually correct now with the new definition for skip_axes. --- ndindex/shapetools.py | 46 ++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index 9976a14e..9ba052d7 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -218,29 +218,38 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): return shapes = [asshape(shape) for shape in shapes] - ndim = len(max(shapes, key=len)) + S = len(shapes) - iters = [[] for i in range(len(shapes))] + iters = [[] for i in range(S)] broadcasted_shape = broadcast_shapes(*shapes, skip_axes=skip_axes) - for i in range(-1, -ndim-1, -1): - for it, shape, sk in zip(iters, shapes, skip_axes): - val = associated_axis(broadcasted_shape, i, sk) + idxes = [-1]*S + + while any(i is not None for i in idxes): + for s, it, shape, sk in zip(range(S), iters, shapes, skip_axes): + i = idxes[s] + if i is None: + continue if -i > len(shape): - # for every dimension prepended by broadcasting, repeat the - # indices that many times - for j in range(len(it)): - if val not in [None, 0, 1]: - it[j] = ncycles(it[j], val) - break - elif i in sk: - if len(shape) == ndim and len(sk) == len(shape) and i == -1: - # The whole shape is skipped. This normally would be - # cycled by the previous block but in this case it isn't - # because the shape already has ndim dimensions. - it.insert(0, ncycles([slice(None)], prod(broadcasted_shape))) + if not shape: + pass + elif len(shape) == len(sk): + # The whole shape is skipped. Just repeat the most recent slice + it[0] = ncycles(it[0], prod(broadcasted_shape)) else: - it.insert(0, [slice(None)]) + # Find the first non-skipped axis and repeat by however + # many implicit axes are left in the broadcasted shape + for j in range(-len(shape), 0): + if j not in sk: + break + it[j] = ncycles(it[j], prod(broadcasted_shape[:len(sk)-len(shape)+len(broadcasted_shape)])) + + idxes[s] = None + continue + + val = associated_axis(broadcasted_shape, i, sk) + if i in sk: + it.insert(0, [slice(None)]) else: if val == 0: return @@ -248,6 +257,7 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): it.insert(0, ncycles(range(shape[i]), val)) else: it.insert(0, range(shape[i])) + idxes[s] -= 1 if _debug: # pragma: no cover print(iters) From 6fe3ce3c3a05204b0bdf98d7a6d95e10a2989c9e Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 31 Jul 2023 15:58:56 -0600 Subject: [PATCH 191/218] Update cross and matmul iter_indices tests --- ndindex/tests/helpers.py | 120 +++++++++++++++++++++++---- ndindex/tests/test_shapetools.py | 135 +++++++++++-------------------- 2 files changed, 153 insertions(+), 102 deletions(-) diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index 64e1ec39..51388118 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -10,12 +10,13 @@ from hypothesis import assume, note from hypothesis.strategies import (integers, none, one_of, lists, just, - builds, shared, composite, sampled_from) + builds, shared, composite, sampled_from, + nothing, tuples as hypothesis_tuples) from hypothesis.extra.numpy import (arrays, mutually_broadcastable_shapes as mbs, BroadcastableShapes, valid_tuple_axes) from ..ndindex import ndindex -from ..shapetools import remove_indices +from ..shapetools import remove_indices, unremove_indices from .._crt import prod # Hypothesis strategies for generating indices. Note that some of these @@ -266,21 +267,108 @@ def _mbs_and_skip_axes( lambda i: i[0]) skip_axes_st = mbs_and_skip_axes.map(lambda i: i[1]) +@composite +def _cross_shapes_and_skip_axes(draw): + (shapes, _broadcasted_shape), skip_axes = draw(_mbs_and_skip_axes( + shapes=_short_shapes(2), + min_shapes=2, + max_shapes=2, + num_skip_axes=1, + # TODO: Test other skip axes types + skip_axes_type_st=just(list), + skip_axes_values=just(3), + )) + + broadcasted_skip_axis = draw(integers(-len(_broadcasted_shape)-1, len(_broadcasted_shape))) + broadcasted_shape = unremove_indices(_broadcasted_shape, + [broadcasted_skip_axis], val=3) + skip_axes.append((broadcasted_skip_axis,)) -one_mbs_and_skip_axes = shared(_mbs_and_skip_axes( - shapes=_short_shapes(1), - min_shapes=2, - max_shapes=2)) -one_mutually_broadcastable_shapes = one_mbs_and_skip_axes.map( - lambda i: i[0]) -one_skip_axes = one_mbs_and_skip_axes.map(lambda i: i[1]) -two_mbs_and_skip_axes = shared(_mbs_and_skip_axes( - shapes=_short_shapes(2), - min_shapes=2, - max_shapes=2)) -two_mutually_broadcastable_shapes = two_mbs_and_skip_axes.map( - lambda i: i[0]) -two_skip_axes = two_mbs_and_skip_axes.map(lambda i: i[1]) + return BroadcastableShapes(shapes, broadcasted_shape), skip_axes + +cross_shapes_and_skip_axes = shared(_cross_shapes_and_skip_axes()) +cross_shapes = cross_shapes_and_skip_axes.map(lambda i: i[0]) +cross_skip_axes = cross_shapes_and_skip_axes.map(lambda i: i[1]) + +@composite +def cross_arrays_st(draw): + broadcastable_shapes = draw(cross_shapes) + shapes, broadcasted_shape = broadcastable_shapes + + # Sanity check + assert len(shapes) == 2 + # We need to generate fairly random arrays. Otherwise, if they are too + # similar to each other, like two arange arrays would be, the cross + # product will be 0. We also disable the fill feature in arrays() for the + # same reason, as it would otherwise generate too many vectors that are + # colinear. + a = draw(arrays(dtype=int, shape=shapes[0], elements=integers(-100, 100), fill=nothing())) + b = draw(arrays(dtype=int, shape=shapes[1], elements=integers(-100, 100), fill=nothing())) + + return a, b + +@composite +def _matmul_shapes_and_skip_axes(draw): + (shapes, _broadcasted_shape), skip_axes = draw(_mbs_and_skip_axes( + shapes=_short_shapes(2), + min_shapes=2, + max_shapes=2, + num_skip_axes=2, + # TODO: Test other skip axes types + skip_axes_type_st=just(list), + skip_axes_values=just(None), + )) + + broadcasted_skip_axes = draw(hypothesis_tuples(*[ + integers(-len(_broadcasted_shape)-1, len(_broadcasted_shape)) + ]*2)) + + try: + broadcasted_shape = unremove_indices(_broadcasted_shape, + broadcasted_skip_axes) + except NotImplementedError: + # TODO: unremove_indices only works with both positive or both negative + assume(False) + # Make sure the indices are unique + assume(len(set(broadcasted_skip_axes)) == len(broadcasted_skip_axes)) + + skip_axes.append(broadcasted_skip_axes) + + # (n, m) @ (m, k) -> (n, k) + n, m, k = draw(hypothesis_tuples(integers(0, 10), integers(0, 10), + integers(0, 10))) + shape1, shape2 = map(list, shapes) + ax1, ax2 = skip_axes[0] + shape1[ax1] = n + shape1[ax2] = m + ax1, ax2 = skip_axes[1] + shape2[ax1] = m + shape2[ax2] = k + broadcasted_shape = list(broadcasted_shape) + ax1, ax2 = skip_axes[2] + broadcasted_shape[ax1] = n + broadcasted_shape[ax2] = k + + shapes = (tuple(shape1), tuple(shape2)) + broadcasted_shape = tuple(broadcasted_shape) + + return BroadcastableShapes(shapes, broadcasted_shape), skip_axes + +matmul_shapes_and_skip_axes = shared(_matmul_shapes_and_skip_axes()) +matmul_shapes = matmul_shapes_and_skip_axes.map(lambda i: i[0]) +matmul_skip_axes = matmul_shapes_and_skip_axes.map(lambda i: i[1]) + +@composite +def matmul_arrays_st(draw): + broadcastable_shapes = draw(matmul_shapes) + shapes, broadcasted_shape = broadcastable_shapes + + # Sanity check + assert len(shapes) == 2 + a = draw(arrays(dtype=int, shape=shapes[0], elements=integers(-100, 100))) + b = draw(arrays(dtype=int, shape=shapes[1], elements=integers(-100, 100))) + + return a, b reduce_kwargs = sampled_from([{}, {'negative_int': False}, {'negative_int': True}]) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 177121b2..f3e77e6c 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -3,8 +3,7 @@ from hypothesis import assume, given, example from hypothesis.strategies import (one_of, integers, tuples as hypothesis_tuples, just, lists, shared, - composite, nothing) -from hypothesis.extra.numpy import arrays + ) from pytest import raises @@ -17,8 +16,9 @@ from ..tuple import Tuple from .helpers import (prod, mutually_broadcastable_shapes_with_skipped_axes, skip_axes_st, mutually_broadcastable_shapes, tuples, - shapes, two_mutually_broadcastable_shapes, - one_skip_axes, two_skip_axes, assert_equal) + shapes, assert_equal, cross_shapes, cross_skip_axes, + cross_arrays_st, matmul_shapes, matmul_skip_axes, + matmul_arrays_st) @example([[(1, 1), (1, 1)], (1,)], (0,)) @example([[(0,), (0,)], ()], (0,)) @@ -131,30 +131,8 @@ def _remove_slices(idx): assert set(vals) == set(correct_vals) assert bvals == list(canonical_broadcasted_array.flat) -# cross_shapes = mutually_broadcastable_shapes_with_skipped_axes( -# mutually_broadcastable_shapes=two_mutually_broadcastable_shapes, -# skip_axes_st=one_skip_axes, -# skip_axes_values=integers(3, 3)) - -@composite -def cross_arrays_st(draw): - broadcastable_shapes = draw(cross_shapes) - shapes, broadcasted_shape = broadcastable_shapes - - # Sanity check - assert len(shapes) == 2 - # We need to generate fairly random arrays. Otherwise, if they are too - # similar to each other, like two arange arrays would be, the cross - # product will be 0. We also disable the fill feature in arrays() for the - # same reason, as it would otherwise generate too many vectors that are - # colinear. - a = draw(arrays(dtype=int, shape=shapes[0], elements=integers(-100, 100), fill=nothing())) - b = draw(arrays(dtype=int, shape=shapes[1], elements=integers(-100, 100), fill=nothing())) - - return a, b - -# @given(cross_arrays_st(), cross_shapes, one_skip_axes) -def test_iter_indices_cross(cross_arrays, broadcastable_shapes, skip_axes): +@given(cross_arrays_st(), cross_shapes, cross_skip_axes) +def test_iter_indices_cross(cross_arrays, broadcastable_shapes, _skip_axes): # Test iter_indices behavior against np.cross, which effectively skips the # crossed axis. Note that we don't test against cross products of size 2 # because a 2 x 2 cross product just returns the z-axis (i.e., it doesn't @@ -162,17 +140,17 @@ def test_iter_indices_cross(cross_arrays, broadcastable_shapes, skip_axes): # going to be removed in NumPy 2.0. a, b = cross_arrays shapes, broadcasted_shape = broadcastable_shapes - skip_axis = skip_axes[0] - broadcasted_shape = list(broadcasted_shape) - # Remove None from the shape for iter_indices - broadcasted_shape[skip_axis] = 3 - broadcasted_shape = tuple(broadcasted_shape) + # Sanity check + skip_axes = normalize_skip_axes([*shapes, broadcasted_shape], _skip_axes) + for sh, sk in zip([*shapes, broadcasted_shape], skip_axes): + assert len(sk) == 1 + assert sh[sk[0]] == 3 - res = np.cross(a, b, axisa=skip_axis, axisb=skip_axis, axisc=skip_axis) + res = np.cross(a, b, axisa=skip_axes[0][0], axisb=skip_axes[1][0], axisc=skip_axes[2][0]) assert res.shape == broadcasted_shape - for idx1, idx2, idx3 in iter_indices(*shapes, broadcasted_shape, skip_axes=skip_axes): + for idx1, idx2, idx3 in iter_indices(*shapes, broadcasted_shape, skip_axes=_skip_axes): assert a[idx1.raw].shape == (3,) assert b[idx2.raw].shape == (3,) assert_equal(np.cross( @@ -180,46 +158,7 @@ def test_iter_indices_cross(cross_arrays, broadcastable_shapes, skip_axes): b[idx2.raw]), res[idx3.raw]) - -@composite -def _matmul_shapes(draw): - broadcastable_shapes = draw(mutually_broadcastable_shapes_with_skipped_axes( - mutually_broadcastable_shapes=two_mutually_broadcastable_shapes, - skip_axes_st=two_skip_axes, - skip_axes_values=just(None), - )) - shapes, broadcasted_shape = broadcastable_shapes - skip_axes = draw(two_skip_axes) - # (n, m) @ (m, k) -> (n, k) - n, m, k = draw(hypothesis_tuples(integers(0, 10), integers(0, 10), - integers(0, 10))) - - shape1, shape2 = map(list, shapes) - ax1, ax2 = skip_axes - shape1[ax1] = n - shape1[ax2] = m - shape2[ax1] = m - shape2[ax2] = k - broadcasted_shape = list(broadcasted_shape) - broadcasted_shape[ax1] = n - broadcasted_shape[ax2] = k - return [tuple(shape1), tuple(shape2)], tuple(broadcasted_shape) - -matmul_shapes = shared(_matmul_shapes()) - -@composite -def matmul_arrays_st(draw): - broadcastable_shapes = draw(matmul_shapes) - shapes, broadcasted_shape = broadcastable_shapes - - # Sanity check - assert len(shapes) == 2 - a = draw(arrays(dtype=int, shape=shapes[0], elements=integers(-100, 100))) - b = draw(arrays(dtype=int, shape=shapes[1], elements=integers(-100, 100))) - - return a, b - -@given(matmul_arrays_st(), matmul_shapes, two_skip_axes) +@given(matmul_arrays_st(), matmul_shapes, matmul_skip_axes) def test_iter_indices_matmul(matmul_arrays, broadcastable_shapes, skip_axes): # Test iter_indices behavior against np.matmul, which effectively skips the # contracted axis (they aren't broadcasted together, even when they are @@ -227,20 +166,44 @@ def test_iter_indices_matmul(matmul_arrays, broadcastable_shapes, skip_axes): a, b = matmul_arrays shapes, broadcasted_shape = broadcastable_shapes - ax1, ax2 = skip_axes - n, m, k = shapes[0][ax1], shapes[0][ax2], shapes[1][ax2] + # Note, we don't use normalize_skip_axes here because it sorts the skip + # axes - res = np.matmul(a, b, axes=[skip_axes, skip_axes, skip_axes]) + ax1, ax2 = skip_axes[0] + ax3 = skip_axes[1][1] + n, m, k = shapes[0][ax1], shapes[0][ax2], shapes[1][ax3] + + # Sanity check + sk0, sk1, sk2 = skip_axes + shape1, shape2 = shapes + assert a.shape == shape1 + assert b.shape == shape2 + assert shape1[sk0[0]] == n + assert shape1[sk0[1]] == m + assert shape2[sk1[0]] == m + assert shape2[sk1[1]] == k + assert broadcasted_shape[sk2[0]] == n + assert broadcasted_shape[sk2[1]] == k + + res = np.matmul(a, b, axes=skip_axes) assert res.shape == broadcasted_shape + is_ordered = lambda sk, shape: (Integer(sk[0]).reduce(len(shape)).raw <= Integer(sk[1]).reduce(len(shape)).raw) + orders = [ + is_ordered(sk0, shapes[0]), + is_ordered(sk1, shapes[1]), + is_ordered(sk2, broadcasted_shape), + ] + for idx1, idx2, idx3 in iter_indices(*shapes, broadcasted_shape, skip_axes=skip_axes): - assert a[idx1.raw].shape == (n, m) if ax1 <= ax2 else (m, n) - assert b[idx2.raw].shape == (m, k) if ax1 <= ax2 else (k, m) - if ax1 <= ax2: - sub_res = np.matmul(a[idx1.raw], b[idx2.raw]) - else: - sub_res = np.matmul(a[idx1.raw], b[idx2.raw], - axes=[(1, 0), (1, 0), (1, 0)]) + assert a[idx1.raw].shape == (n, m) if orders[0] else (m, n) + assert b[idx2.raw].shape == (m, k) if orders[1] else (k, m) + sub_res_axes = [ + (0, 1) if orders[0] else (1, 0), + (0, 1) if orders[1] else (1, 0), + (0, 1) if orders[2] else (1, 0), + ] + sub_res = np.matmul(a[idx1.raw], b[idx2.raw], axes=sub_res_axes) assert_equal(sub_res, res[idx3.raw]) def test_iter_indices_errors(): @@ -399,7 +362,7 @@ def test_broadcast_shapes_skip_axes(broadcastable_shapes, skip_axes): def test_broadcast_shapes_skip_axes_errors(broadcastable_shapes, skip_axes): shapes, broadcasted_shape = broadcastable_shapes - # All errors should come from canonical_skip_axes, which is tested + # All errors should come from normalize_skip_axes, which is tested # separately below. try: normalize_skip_axes(shapes, skip_axes) From c0537c14659368b245cfd661aeaa807cb7852bc9 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 31 Jul 2023 16:02:00 -0600 Subject: [PATCH 192/218] Update test_iter_indices_errors --- ndindex/tests/test_shapetools.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index f3e77e6c..e19014bd 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -243,12 +243,11 @@ def test_iter_indices_errors(): # Older versions of NumPy do not have the more helpful error message assert ndindex_msg == np_msg - raises(NotImplementedError, lambda: list(iter_indices((1, 2), skip_axes=(0, -1)))) - - with raises(ValueError, match=r"duplicate axes"): + with raises(ValueError, match=r"not unique"): list(iter_indices((1, 2), skip_axes=(0, 1, 0))) - raises(AxisError, lambda: list(iter_indices(skip_axes=(0,)))) + raises(AxisError, lambda: list(iter_indices((0,), skip_axes=(3,)))) + raises(ValueError, lambda: list(iter_indices(skip_axes=(0,)))) raises(TypeError, lambda: list(iter_indices(1, 2))) raises(TypeError, lambda: list(iter_indices(1, 2, (2, 2)))) raises(TypeError, lambda: list(iter_indices([(1, 2), (2, 2)]))) From 38a6ab0ea3fc905a97eea4557587b9b11dcf02b9 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 31 Jul 2023 19:34:11 -0600 Subject: [PATCH 193/218] Improve test coverage --- ndindex/shapetools.py | 2 -- ndindex/tests/helpers.py | 2 +- ndindex/tests/test_shapetools.py | 9 ++++++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index 9ba052d7..cdf75b50 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -212,8 +212,6 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): shapes = [asshape(shape, allow_int=False) for shape in shapes] if not shapes: - if skip_axes: - raise AxisError(skip_axes[0], 0) yield () return diff --git a/ndindex/tests/helpers.py b/ndindex/tests/helpers.py index 51388118..1cceaff9 100644 --- a/ndindex/tests/helpers.py +++ b/ndindex/tests/helpers.py @@ -173,7 +173,7 @@ def _fill_shape(draw, else: new_shape[i] = draw(sampled_from([result_shape[j], 1])) j -= 1 - while new_shape and new_shape[0] == 'placeholder': + while new_shape and new_shape[0] == 'placeholder': # pragma: no cover # Can happen if positive and negative skip_axes refer to the same # entry new_shape.pop(0) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index e19014bd..50184057 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -373,7 +373,7 @@ def test_broadcast_shapes_skip_axes_errors(broadcastable_shapes, skip_axes): try: broadcast_shapes(*shapes, skip_axes=skip_axes) except IndexError: - raise RuntimeError("broadcast_shapes raised but should not have") + raise RuntimeError("broadcast_shapes raised but should not have") # pragma: no cover except BroadcastError: # Broadcastable shapes can become unbroadcastable after skipping axes # (see the @example above). @@ -389,6 +389,8 @@ def test_remove_indices(n, idxes): assume(min(idxes) >= -n) a = tuple(range(n)) b = remove_indices(a, idxes) + if len(idxes) == 1: + assert remove_indices(a, idxes[0]) == b A = list(a) for i in idxes: @@ -451,6 +453,11 @@ def test_associated_axis(broadcastable_shapes, skip_axes): else: assert val == 1 or bval == val, (shape, i) + + sk = max(_skip_axes, key=len, default=()) + for i in range(-len(broadcasted_shape)-len(sk)-10, -len(broadcasted_shape)-len(sk)): + assert associated_axis(broadcasted_shape, i, sk) is None + # TODO: add a hypothesis test for asshape def test_asshape(): assert asshape(1) == (1,) From 9f89f6c1b87ce5458521ec89a06180913a17893b Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 31 Jul 2023 23:01:44 -0600 Subject: [PATCH 194/218] Fix deprecation warning from newer NumPys in the tests --- ndindex/tests/test_tuple.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ndindex/tests/test_tuple.py b/ndindex/tests/test_tuple.py index bcd854aa..8297ed7d 100644 --- a/ndindex/tests/test_tuple.py +++ b/ndindex/tests/test_tuple.py @@ -1,6 +1,6 @@ from itertools import product -from numpy import arange, array, intp, empty +from numpy import arange, array, intp, empty, all as np_all from hypothesis import given, example from hypothesis.strategies import integers, one_of @@ -104,6 +104,7 @@ def test_tuple_reduce_no_shape_hypothesis(t, shape, kwargs): # Idempotency assert reduced.reduce(**kwargs) == reduced +@example((..., empty((1, 0), dtype=intp)), (1, 0), {}) @example((1, -1, [1, -1]), (3, 3, 3), {'negative_int': True}) @example((..., None), (), {}) @example((..., empty((0, 0), dtype=bool)), (0, 0), {}) @@ -155,9 +156,9 @@ def test_tuple_reduce_hypothesis(t, shape, kwargs): assert arg.raw >= 0 elif isinstance(arg, IntegerArray): if negative_int: - assert all(arg.raw < 0) + assert np_all(arg.raw < 0) else: - assert all(arg.raw >= 0) + assert np_all(arg.raw >= 0) def test_tuple_reduce_explicit(): # Some aspects of Tuple.reduce are hard to test as properties, so include From 1397242feda25ed7991485c98a83b7f2d2d0f8ee Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 1 Aug 2023 22:36:08 -0600 Subject: [PATCH 195/218] Update documentation --- docs/api.rst | 10 ++++++++++ ndindex/shapetools.py | 14 ++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 9ff9e894..97162042 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -62,6 +62,14 @@ the index objects. .. autofunction:: ndindex.broadcast_shapes +Exceptions +========== + +These are some custom exceptions that are raised by a few functions in +ndindex. Note that most functions in ndindex will raise `IndexError` +(if the index would be invalid), or `TypeError` or `ValueError` (if the input +types or values are incorrect). + .. autoexception:: ndindex.BroadcastError .. autoexception:: ndindex.AxisError @@ -106,3 +114,5 @@ relied on as they may be removed or changed. .. autofunction:: ndindex.shapetools.remove_indices .. autofunction:: ndindex.shapetools.unremove_indices + +.. autofunction:: ndindex.shapetools.normalize_skip_axes diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index 63d71ac3..c15766b6 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -7,8 +7,10 @@ class BroadcastError(ValueError): """ - Exception raised by :func:`iter_indices()` when the input shapes are not - broadcast compatible. + Exception raised by :func:`iter_indices()` and + :func:`broadcast_shapes()` when the input shapes are not broadcast + compatible. + """ __slots__ = ("arg1", "shape1", "arg2", "shape2") @@ -24,8 +26,8 @@ def __str__(self): class AxisError(ValueError, IndexError): """ - Exception raised by :func:`iter_indices()` when the `skip_axes` argument - is out of bounds. + Exception raised by :func:`iter_indices()` and + :func:`broadcast_shapes()` when the `skip_axes` argument is out-of-bounds. This is used instead of the NumPy exception of the same name so that `iter_indices` does not need to depend on NumPy. @@ -52,7 +54,7 @@ def broadcast_shapes(*shapes, skip_axes=()): shape with `skip_axes`. If the shapes are not broadcast compatible (excluding `skip_axes`), - `BroadcastError` is raised. + :class:`BroadcastError` is raised. >>> from ndindex import broadcast_shapes >>> broadcast_shapes((2, 3), (3,), (4, 2, 1)) @@ -445,7 +447,7 @@ def normalize_skip_axes(shapes, skip_axes): corresponding shape. If `skip_axes` is an integer, this is basically `[(skip_axes,) for s - in shapes]`. If `skip_axes is a tuple, it is like `[skip_axes for s in + in shapes]`. If `skip_axes` is a tuple, it is like `[skip_axes for s in shapes]`. The `skip_axes` must always refer to unique axes in each shape. From b163c2c36269107dc2718c9b3799a2597ca48edf Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 1 Aug 2023 22:38:59 -0600 Subject: [PATCH 196/218] Add an example for coverage --- ndindex/tests/test_shapetools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 50184057..711eabd2 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -492,6 +492,7 @@ def test_asshape(): raises(TypeError, lambda: asshape(np.int64(1), allow_int=False)) raises(IndexError, lambda: asshape((2, 3), 3)) +@example([()], []) @example([(0, 1)], 0) @example([(2, 3), (2, 3, 4)], [(3,), (0,)]) @example([(0, 1)], 0) From 1166f11423166b01249e7a56f8576240d6e58fd6 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 1 Aug 2023 22:47:21 -0600 Subject: [PATCH 197/218] Add an example for coverage --- ndindex/tests/test_shapetools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 711eabd2..b91f9fe2 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -575,6 +575,7 @@ def test_normalize_skip_axes_errors(): raises(TypeError, lambda: normalize_skip_axes([(1,)], [(0,), 0])) raises(TypeError, lambda: normalize_skip_axes([(1,)], [0, (0,)])) +@example(10, 100) @given(integers(), integers()) def test_axiserror(axis, ndim): if ndim == 0 and axis in [0, -1]: From abde062b757b33c047af0ce5d671edcef560cdd8 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 1 Aug 2023 22:50:13 -0600 Subject: [PATCH 198/218] Fix coverage example --- ndindex/tests/test_shapetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index b91f9fe2..c24cd33c 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -575,7 +575,7 @@ def test_normalize_skip_axes_errors(): raises(TypeError, lambda: normalize_skip_axes([(1,)], [(0,), 0])) raises(TypeError, lambda: normalize_skip_axes([(1,)], [0, (0,)])) -@example(10, 100) +@example(10, 5) @given(integers(), integers()) def test_axiserror(axis, ndim): if ndim == 0 and axis in [0, -1]: From cbd4bf5ac274979995149fb8f63a46fe5e89afb2 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 1 Aug 2023 23:03:05 -0600 Subject: [PATCH 199/218] Fix coverage --- ndindex/tests/test_chunking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndindex/tests/test_chunking.py b/ndindex/tests/test_chunking.py index 4439ab7c..6e6eaccc 100644 --- a/ndindex/tests/test_chunking.py +++ b/ndindex/tests/test_chunking.py @@ -50,7 +50,7 @@ def test_ChunkSize_args(chunk_size_tuple, idx): try: ndindex(idx) - except ValueError: + except ValueError: # pragma: no cover # Filter out invalid slices (TODO: do this in the strategy) assume(False) From 16566b20b4de78a7c8c8d78d5e5bc44771df3bd0 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 4 Aug 2023 14:59:38 -0600 Subject: [PATCH 200/218] Install the dev numpy in every CI build --- .github/workflows/tests.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index db29e8c5..69dd9ae9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,11 +17,7 @@ jobs: set -x set -e python -m pip install pyflakes pytest pytest-doctestplus hypothesis pytest-cov pytest-flakes packaging - if [[ ${{ matrix.python-version }} == "3.12-dev" ]]; then - python -m pip install --pre --upgrade --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy - else - python -m pip install numpy - fi; + python -m pip install --pre --upgrade --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy - name: Run Tests run: | set -x From 33164e47eea93b5ee604668156db0051c7bf9f4f Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 4 Aug 2023 14:59:52 -0600 Subject: [PATCH 201/218] Fix some doctest failures with the dev NumPy version NumPy changed how scalars print. Note that the doctests will now only pass properly with the development version of NumPy. --- ndindex/shapetools.py | 12 ++++++------ ndindex/tuple.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index 7e035d0f..cbd94e71 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -189,12 +189,12 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): [110]]) >>> for idx1, idx2 in iter_indices((1, 3), (2, 1)): ... print(f"{idx1 = }; {idx2 = }; {(a[idx1.raw], b[idx2.raw]) = }") - idx1 = Tuple(0, 0); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (0, 100) - idx1 = Tuple(0, 1); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (1, 100) - idx1 = Tuple(0, 2); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (2, 100) - idx1 = Tuple(0, 0); idx2 = Tuple(1, 0); (a[idx1.raw], b[idx2.raw]) = (0, 110) - idx1 = Tuple(0, 1); idx2 = Tuple(1, 0); (a[idx1.raw], b[idx2.raw]) = (1, 110) - idx1 = Tuple(0, 2); idx2 = Tuple(1, 0); (a[idx1.raw], b[idx2.raw]) = (2, 110) + idx1 = Tuple(0, 0); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (np.int64(0), np.int64(100)) + idx1 = Tuple(0, 1); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (np.int64(1), np.int64(100)) + idx1 = Tuple(0, 2); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (np.int64(2), np.int64(100)) + idx1 = Tuple(0, 0); idx2 = Tuple(1, 0); (a[idx1.raw], b[idx2.raw]) = (np.int64(0), np.int64(110)) + idx1 = Tuple(0, 1); idx2 = Tuple(1, 0); (a[idx1.raw], b[idx2.raw]) = (np.int64(1), np.int64(110)) + idx1 = Tuple(0, 2); idx2 = Tuple(1, 0); (a[idx1.raw], b[idx2.raw]) = (np.int64(2), np.int64(110)) >>> a + b array([[100, 101, 102], [110, 111, 112]]) diff --git a/ndindex/tuple.py b/ndindex/tuple.py index 61052ac3..80bfcd15 100644 --- a/ndindex/tuple.py +++ b/ndindex/tuple.py @@ -247,7 +247,7 @@ def reduce(self, shape=None, *, negative_int=False): >>> a[..., 1] array(1) >>> a[1] - 1 + np.int64(1) See https://github.com/Quansight-Labs/ndindex/issues/22. From 67238e9c58acdb7a5800c54d1b02f396468713e1 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 4 Aug 2023 16:44:35 -0600 Subject: [PATCH 202/218] Fix coverage in Python 3.12 --- ndindex/slice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndindex/slice.py b/ndindex/slice.py index bda8e26d..c73aca43 100644 --- a/ndindex/slice.py +++ b/ndindex/slice.py @@ -79,7 +79,7 @@ def __hash__(self): # Slices are only hashable in Python 3.12+ try: return hash(self.raw) - except TypeError: + except TypeError: # pragma: no cover return hash(self.args) @property From a8636c802b7bf55739a85b0cacf7a5aa8b76afec Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Fri, 4 Aug 2023 16:52:35 -0600 Subject: [PATCH 203/218] Skip NumPy 1.25 doctests for Python 3.8 NumPy 1.25 doesn't support Python 3.8 but we haven't dropped support yet. --- ndindex/shapetools.py | 2 +- ndindex/tests/doctest.py | 15 ++++++++++++++- ndindex/tuple.py | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index cbd94e71..4ea2c94a 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -188,7 +188,7 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): array([[100], [110]]) >>> for idx1, idx2 in iter_indices((1, 3), (2, 1)): - ... print(f"{idx1 = }; {idx2 = }; {(a[idx1.raw], b[idx2.raw]) = }") + ... print(f"{idx1 = }; {idx2 = }; {(a[idx1.raw], b[idx2.raw]) = }") # doctest: +SKIP38 idx1 = Tuple(0, 0); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (np.int64(0), np.int64(100)) idx1 = Tuple(0, 1); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (np.int64(1), np.int64(100)) idx1 = Tuple(0, 2); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (np.int64(2), np.int64(100)) diff --git a/ndindex/tests/doctest.py b/ndindex/tests/doctest.py index c110c60e..6c06f0a1 100644 --- a/ndindex/tests/doctest.py +++ b/ndindex/tests/doctest.py @@ -22,7 +22,14 @@ import os from contextlib import contextmanager from doctest import (DocTestRunner, DocFileSuite, DocTestSuite, - NORMALIZE_WHITESPACE) + NORMALIZE_WHITESPACE, register_optionflag) + +SKIP38 = register_optionflag("SKIP38") +PY38 = sys.version_info[1] == 8 +if PY38: + SKIP_THIS_VERSION = SKIP38 +else: + SKIP_THIS_VERSION = 0 @contextmanager def patch_doctest(): @@ -34,11 +41,17 @@ def patch_doctest(): orig_run = DocTestRunner.run def run(self, test, **kwargs): + filtered_examples = [] + for example in test.examples: + if SKIP_THIS_VERSION not in example.options: + filtered_examples.append(example) + # Remove ``` example.want = example.want.replace('```\n', '') example.exc_msg = example.exc_msg and example.exc_msg.replace('```\n', '') + test.examples = filtered_examples return orig_run(self, test, **kwargs) try: diff --git a/ndindex/tuple.py b/ndindex/tuple.py index 80bfcd15..b9438e5d 100644 --- a/ndindex/tuple.py +++ b/ndindex/tuple.py @@ -246,7 +246,7 @@ def reduce(self, shape=None, *, negative_int=False): Integer(1) >>> a[..., 1] array(1) - >>> a[1] + >>> a[1] # doctest: +SKIP38 np.int64(1) See https://github.com/Quansight-Labs/ndindex/issues/22. From 92ff4ab7c010f29d27e83fbdd23aa7e49b1cf239 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 4 Oct 2023 01:28:05 -0600 Subject: [PATCH 204/218] Skip doctests that require NumPy 2.0 in NumPy 1 --- ndindex/shapetools.py | 2 +- ndindex/tests/doctest.py | 10 ++++++---- ndindex/tuple.py | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/ndindex/shapetools.py b/ndindex/shapetools.py index 4ea2c94a..e3c5854b 100644 --- a/ndindex/shapetools.py +++ b/ndindex/shapetools.py @@ -188,7 +188,7 @@ def iter_indices(*shapes, skip_axes=(), _debug=False): array([[100], [110]]) >>> for idx1, idx2 in iter_indices((1, 3), (2, 1)): - ... print(f"{idx1 = }; {idx2 = }; {(a[idx1.raw], b[idx2.raw]) = }") # doctest: +SKIP38 + ... print(f"{idx1 = }; {idx2 = }; {(a[idx1.raw], b[idx2.raw]) = }") # doctest: +SKIPNP1 idx1 = Tuple(0, 0); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (np.int64(0), np.int64(100)) idx1 = Tuple(0, 1); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (np.int64(1), np.int64(100)) idx1 = Tuple(0, 2); idx2 = Tuple(0, 0); (a[idx1.raw], b[idx2.raw]) = (np.int64(2), np.int64(100)) diff --git a/ndindex/tests/doctest.py b/ndindex/tests/doctest.py index 6c06f0a1..55a0a02e 100644 --- a/ndindex/tests/doctest.py +++ b/ndindex/tests/doctest.py @@ -16,6 +16,8 @@ """ +import numpy + import sys import unittest import glob @@ -24,10 +26,10 @@ from doctest import (DocTestRunner, DocFileSuite, DocTestSuite, NORMALIZE_WHITESPACE, register_optionflag) -SKIP38 = register_optionflag("SKIP38") -PY38 = sys.version_info[1] == 8 -if PY38: - SKIP_THIS_VERSION = SKIP38 +SKIPNP1 = register_optionflag("SKIPNP1") +NP1 = numpy.__version__.startswith('1') +if NP1: + SKIP_THIS_VERSION = SKIPNP1 else: SKIP_THIS_VERSION = 0 diff --git a/ndindex/tuple.py b/ndindex/tuple.py index b9438e5d..30aa9646 100644 --- a/ndindex/tuple.py +++ b/ndindex/tuple.py @@ -246,7 +246,7 @@ def reduce(self, shape=None, *, negative_int=False): Integer(1) >>> a[..., 1] array(1) - >>> a[1] # doctest: +SKIP38 + >>> a[1] # doctest: +SKIPNP1 np.int64(1) See https://github.com/Quansight-Labs/ndindex/issues/22. From d6ad910980b1fcab8582f43068bf1c09e39a7c6d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 4 Oct 2023 13:22:08 -0600 Subject: [PATCH 205/218] Fix an import in NumPy 2.0 --- ndindex/array.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ndindex/array.py b/ndindex/array.py index 179b7433..4eafd990 100644 --- a/ndindex/array.py +++ b/ndindex/array.py @@ -20,9 +20,13 @@ class ArrayIndex(NDIndex): def _typecheck(self, idx, shape=None, _copy=True): try: - from numpy import ndarray, asarray, integer, bool_, empty, intp, VisibleDeprecationWarning + from numpy import ndarray, asarray, integer, bool_, empty, intp except ImportError: # pragma: no cover raise ImportError("NumPy must be installed to create array indices") + try: + from numpy import VisibleDeprecationWarning + except ImportError: # pragma: no cover + from numpy.exceptions import VisibleDeprecationWarning if self.dtype is None: raise TypeError("Do not instantiate the superclass ArrayIndex directly") From a9a3c77c8c22029fd17acea7983fd64b9e4071e1 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 4 Oct 2023 13:23:45 -0600 Subject: [PATCH 206/218] Test against different numpy versions on CI --- .github/workflows/tests.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 69dd9ae9..0e10c3d3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,6 +6,8 @@ jobs: strategy: matrix: python-version: ['3.8', '3.9', '3.10', '3.11', '3.12-dev'] + # https://numpy.org/neps/nep-0029-deprecation_policy.html + numpy-version: ['1.22', 'latest', 'dev'] fail-fast: false steps: - uses: actions/checkout@v2 @@ -17,7 +19,13 @@ jobs: set -x set -e python -m pip install pyflakes pytest pytest-doctestplus hypothesis pytest-cov pytest-flakes packaging - python -m pip install --pre --upgrade --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy + if [[ ${{ matrix.numpy-version }} == 'latest' ]]; then + python -m pip install --pre --upgrade numpy + elif [[ ${{ matrix.numpy-version }} == 'dev' ]]; then + python -m pip install --pre --upgrade --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy + else + python -m pip install --upgrade numpy==$NUMPY_VERSION + fi - name: Run Tests run: | set -x From 56e9053c2712358082ab00eeb9d9725a559fa381 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 4 Oct 2023 22:58:09 -0600 Subject: [PATCH 207/218] Fix NumPy 1.22 install on CI --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0e10c3d3..02736160 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,7 +24,7 @@ jobs: elif [[ ${{ matrix.numpy-version }} == 'dev' ]]; then python -m pip install --pre --upgrade --extra-index https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy else - python -m pip install --upgrade numpy==$NUMPY_VERSION + python -m pip install --upgrade numpy==${{ matrix.numpy-version }}.* fi - name: Run Tests run: | From 5ba6a135d43eab57ff660f9906d3a07bf5480922 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 4 Oct 2023 23:00:28 -0600 Subject: [PATCH 208/218] Fix a test failure with numpy 2.0 --- ndindex/tests/test_shapetools.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 9575eb04..6eea536b 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -1,4 +1,8 @@ import numpy as np +try: + from numpy import AxisError as np_AxisError +except ImportError: + from numpy.exceptions import AxisError as np_AxisError from hypothesis import assume, given, example from hypothesis.strategies import (one_of, integers, tuples as @@ -271,7 +275,7 @@ def test_iter_indices_errors(): # Check that the message is the same one used by NumPy try: np.sum(np.arange(10), axis=2) - except np.AxisError as e: + except np_AxisError as e: np_msg = str(e) else: raise RuntimeError("np.sum() did not raise AxisError") # pragma: no cover From 6a4a4fbcaa4c5d70d009833da3f1d727c84ebcaa Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 4 Oct 2023 23:06:39 -0600 Subject: [PATCH 209/218] Fix the CircleCI docs preview --- .github/workflows/docs-preview.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docs-preview.yml b/.github/workflows/docs-preview.yml index fce6fc2a..70b1f2e4 100644 --- a/.github/workflows/docs-preview.yml +++ b/.github/workflows/docs-preview.yml @@ -14,6 +14,7 @@ jobs: artifact-path: 0/docs/_build/html/index.html circleci-jobs: Build Docs Preview job-title: Click here to see a preview of the documentation. + api-token: ${{ secrets.CIRCLECI_TOKEN }} - name: Check the URL if: github.event.status != 'pending' run: | From 5ca685a7988cfdd0daed402001625686d4df841e Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Wed, 4 Oct 2023 23:08:38 -0600 Subject: [PATCH 210/218] Don't install numpy 1.12 on Python 3.12 --- .github/workflows/tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 02736160..0c683fc7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,6 +8,9 @@ jobs: python-version: ['3.8', '3.9', '3.10', '3.11', '3.12-dev'] # https://numpy.org/neps/nep-0029-deprecation_policy.html numpy-version: ['1.22', 'latest', 'dev'] + exclude: + - python-version: '3.12-dev' + numpy-version: '1.22' fail-fast: false steps: - uses: actions/checkout@v2 From 898b6568e3c59eb2f8bd2fbce8d6f79071c6687d Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 5 Oct 2023 00:21:59 -0600 Subject: [PATCH 211/218] Fix coverage --- ndindex/tests/test_shapetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 6eea536b..17531deb 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -1,7 +1,7 @@ import numpy as np try: from numpy import AxisError as np_AxisError -except ImportError: +except ImportError: # pragma: no cover from numpy.exceptions import AxisError as np_AxisError from hypothesis import assume, given, example From d155cff8ef51a94cfd19ff2752995d37366b5ee6 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 4 Jan 2024 02:24:22 -0700 Subject: [PATCH 212/218] Fix test failure with dev numpy --- ndindex/tests/test_shapetools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index ddaed8e8..29c0a83a 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -604,7 +604,7 @@ def test_axiserror(axis, ndim): a = np.empty((0,)*ndim) try: np.sum(a, axis=axis) - except np.AxisError as e3: + except np_AxisError as e3: assert str(e2) == str(e3) else: raise RuntimeError("numpy didn't raise AxisError") # pragma: no cover From 012792d2ca6a93e933535d574298f835f5cddaee Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 4 Jan 2024 02:25:14 -0700 Subject: [PATCH 213/218] Add an @example for coverage --- ndindex/tests/test_isvalid.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ndindex/tests/test_isvalid.py b/ndindex/tests/test_isvalid.py index e307c345..13995b81 100644 --- a/ndindex/tests/test_isvalid.py +++ b/ndindex/tests/test_isvalid.py @@ -5,6 +5,7 @@ from .helpers import ndindices, shapes, MAX_ARRAY_SIZE, check_same, prod +@example([0], (1,)) @example(..., (1, 2, 3)) @example(slice(0, 1), ()) @example(slice(0, 1), (1,)) From a565cd07a8ea5b252cd7f97b195743f9f810aa20 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 29 Jan 2024 16:16:27 -0700 Subject: [PATCH 214/218] Add requirements-dev.txt --- requirements-dev.txt | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 requirements-dev.txt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..7d960665 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,8 @@ +hypothesis +numpy +packaging +pyflakes +pytest +pytest-cov +pytest-doctestplus +pytest-flakes From 9bc92da6130a9947eea1e85640144d18b4749826 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Mon, 29 Jan 2024 16:22:38 -0700 Subject: [PATCH 215/218] Update boolean array IndexError to match NumPy 2.0 --- ndindex/booleanarray.py | 3 +-- ndindex/tests/test_booleanarray.py | 8 ++++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ndindex/booleanarray.py b/ndindex/booleanarray.py index 1b99c26a..c4ea5be4 100644 --- a/ndindex/booleanarray.py +++ b/ndindex/booleanarray.py @@ -108,8 +108,7 @@ def _raise_indexerror(self, shape, axis=0): for i in range(axis, axis+self.ndim): if self.shape[i-axis] != 0 and shape[i] != self.shape[i-axis]: - - raise IndexError(f"boolean index did not match indexed array along dimension {i}; dimension is {shape[i]} but corresponding boolean dimension is {self.shape[i-axis]}") + raise IndexError(f'boolean index did not match indexed array along axis {i}; size of axis is {shape[i]} but size of corresponding boolean axis is {self.shape[i-axis]}') def reduce(self, shape=None, *, axis=0, negative_int=False): """ diff --git a/ndindex/tests/test_booleanarray.py b/ndindex/tests/test_booleanarray.py index 0ee19e1d..171140b6 100644 --- a/ndindex/tests/test_booleanarray.py +++ b/ndindex/tests/test_booleanarray.py @@ -1,4 +1,6 @@ -from numpy import prod, arange, array, bool_, empty, full +from numpy import prod, arange, array, bool_, empty, full, __version__ as np_version + +NP1 = np_version.startswith('1') from hypothesis import given, example from hypothesis.strategies import one_of, integers @@ -60,7 +62,9 @@ def test_booleanarray_reduce_hypothesis(idx, shape, kwargs): index = BooleanArray(idx) - check_same(a, index.raw, ndindex_func=lambda a, x: a[x.reduce(shape, **kwargs).raw]) + same_exception = not NP1 + check_same(a, index.raw, ndindex_func=lambda a, x: a[x.reduce(shape, **kwargs).raw], + same_exception=same_exception) try: reduced = index.reduce(shape, **kwargs) From a6cfbbb0935b40190769a3848a311ee69529ca58 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Tue, 30 Jan 2024 16:17:52 -0700 Subject: [PATCH 216/218] Fix doctests --- ndindex/booleanarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndindex/booleanarray.py b/ndindex/booleanarray.py index c4ea5be4..a4620a9c 100644 --- a/ndindex/booleanarray.py +++ b/ndindex/booleanarray.py @@ -124,7 +124,7 @@ def reduce(self, shape=None, *, axis=0, negative_int=False): >>> idx.reduce((3,)) Traceback (most recent call last): ... - IndexError: boolean index did not match indexed array along dimension 0; dimension is 3 but corresponding boolean dimension is 2 + IndexError: boolean index did not match indexed array along axis 0; size of axis is 3 but size of corresponding boolean axis is 2 >>> idx.reduce((2,)) BooleanArray([True, False]) From 72b71ca0bc1471ad6a51eb913ee519674912872f Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 1 Feb 2024 16:03:43 -0700 Subject: [PATCH 217/218] Add an example for coverage --- ndindex/tests/test_shapetools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ndindex/tests/test_shapetools.py b/ndindex/tests/test_shapetools.py index 29c0a83a..ba8333e3 100644 --- a/ndindex/tests/test_shapetools.py +++ b/ndindex/tests/test_shapetools.py @@ -496,6 +496,7 @@ def test_asshape(): raises(TypeError, lambda: asshape(np.int64(1), allow_int=False)) raises(IndexError, lambda: asshape((2, 3), 3)) +@example([], []) @example([()], []) @example([(0, 1)], 0) @example([(2, 3), (2, 3, 4)], [(3,), (0,)]) From 704d4858cb1730dda6ad20bfc3dc91b3d100aa50 Mon Sep 17 00:00:00 2001 From: Aaron Meurer Date: Thu, 1 Feb 2024 16:59:09 -0700 Subject: [PATCH 218/218] Change references from 'master' to 'main' --- .github/workflows/docs.yml | 2 +- docs/changelog.md | 4 ++-- docs/index.md | 4 ++-- docs/slices.md | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 73ef4193..0ffef95e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -57,7 +57,7 @@ jobs: # https://github.com/JamesIves/github-pages-deploy-action/tree/dev#using-an-ssh-deploy-key- - name: Deploy uses: JamesIves/github-pages-deploy-action@v4 - if: ${{ github.ref == 'refs/heads/master' }} + if: ${{ github.ref == 'refs/heads/main' }} with: folder: docs/_build/html ssh-key: ${{ secrets.DEPLOY_KEY }} diff --git a/docs/changelog.md b/docs/changelog.md index 93d4a555..11f3c2af 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -110,9 +110,9 @@ ### Minor Changes - Added - [CODE_OF_CONDUCT.md](https://github.com/Quansight-Labs/ndindex/blob/master/CODE_OF_CONDUCT.md) + [CODE_OF_CONDUCT.md](https://github.com/Quansight-Labs/ndindex/blob/main/CODE_OF_CONDUCT.md) to the ndindex repository. ndindex follows the [Quansight Code of - Conduct](https://github.com/Quansight/.github/blob/master/CODE_OF_CONDUCT.md). + Conduct](https://github.com/Quansight/.github/blob/main/CODE_OF_CONDUCT.md). - Avoid precomputing all iterated values for slices with large steps in {any}`ChunkSize.as_subchunks()`. diff --git a/docs/index.md b/docs/index.md index 26ec156a..16806dab 100644 --- a/docs/index.md +++ b/docs/index.md @@ -277,7 +277,7 @@ There are two primary types of tests that we employ to verify this: - Hypothesis tests. Hypothesis is a library that can intelligently check a combinatorial search space of inputs. This requires writing hypothesis strategies that can generate all the relevant types of indices (see - [ndindex/tests/helpers.py](https://github.com/Quansight-Labs/ndindex/blob/master/ndindex/tests/helpers.py)). + [ndindex/tests/helpers.py](https://github.com/Quansight-Labs/ndindex/blob/main/ndindex/tests/helpers.py)). For more information on hypothesis, see . All tests have hypothesis tests, even if they are also tested exhaustively. @@ -307,7 +307,7 @@ Benchmarks for ndindex are published ## License -[MIT License](https://github.com/Quansight-Labs/ndindex/blob/master/LICENSE) +[MIT License](https://github.com/Quansight-Labs/ndindex/blob/main/LICENSE) (acknowledgments)= ## Acknowledgments diff --git a/docs/slices.md b/docs/slices.md index 3cb8832d..0c76d625 100644 --- a/docs/slices.md +++ b/docs/slices.md @@ -1693,7 +1693,7 @@ hard to write slice arithmetic. The arithmetic is already hard enough due to the modular nature of `step`, but the discontinuous aspect of `start` and `stop` increases this tenfold. If you are unconvinced of this, take a look at the [source -code](https://github.com/Quansight-labs/ndindex/blob/master/ndindex/slice.py) for +code](https://github.com/Quansight-labs/ndindex/blob/main/ndindex/slice.py) for `ndindex.Slice()`. You will see lots of nested `if` blocks.[^source-footnote] This is because slices have *fundamentally* different definitions if the `start` or `stop` are `None`, negative, or nonnegative. Furthermore, `None` is