From f901843a82572731550078e3cd0d8981e4b8965b Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 6 Feb 2023 17:25:42 +0000 Subject: [PATCH 001/141] dev --- cf/cfimplementation.py | 4 + cf/data/array/cfanetcdfarray.py | 312 +++++++++++++---------- cf/data/data.py | 72 +++++- cf/data/fragment/fullfragmentarray.py | 93 +++++++ cf/data/fragment/missingfragmentarray.py | 113 ++++++-- cf/read_write/netcdf/netcdfread.py | 277 ++++++++++++++------ 6 files changed, 633 insertions(+), 238 deletions(-) create mode 100644 cf/data/fragment/fullfragmentarray.py diff --git a/cf/cfimplementation.py b/cf/cfimplementation.py index 5a4a96e1ab..e273ef22c0 100644 --- a/cf/cfimplementation.py +++ b/cf/cfimplementation.py @@ -90,6 +90,7 @@ def initialise_CFANetCDFArray( units=False, calendar=False, instructions=None, + term=None, ): """Return a `CFANetCDFArray` instance. @@ -111,6 +112,8 @@ def initialise_CFANetCDFArray( instructions: `str`, optional + term: `str`, optional + :Returns: `CFANetCDFArray` @@ -126,6 +129,7 @@ def initialise_CFANetCDFArray( units=units, calendar=calendar, instructions=instructions, + term=term, ) diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index bbf5e3f8a0..29eb02aaa2 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -3,6 +3,7 @@ from ...functions import abspath from ..fragment import ( + FullFragmentArray, MissingFragmentArray, NetCDFFragmentArray, UMFragmentArray, @@ -27,6 +28,7 @@ def __new__(cls, *args, **kwargs): instance._FragmentArray = { "nc": NetCDFFragmentArray, "um": UMFragmentArray, + "full": FullFragmentArray, None: MissingFragmentArray, } return instance @@ -42,6 +44,7 @@ def __init__( units=False, calendar=False, instructions=None, + term=None, source=None, copy=True, ): @@ -104,15 +107,18 @@ def __init__( The calendar of the aggregated data. Set to `None` to indicate the CF default calendar, if applicable. - {{init source: optional}} - - {{init copy: `bool`, optional}} - instructions: `str`, optional The ``aggregated_data`` attribute value found on the CFA netCDF variable. If set then this will be used by `__dask_tokenize__` to improve performance. + term: `str`, optional + TODOCFADOCS + + {{init source: optional}} + + {{init copy: `bool`, optional}} + """ if source is not None: super().__init__(source=source, copy=copy) @@ -131,6 +137,12 @@ def __init__( aggregated_data = source.get_aggregated_data(copy=False) except AttributeError: aggregated_data = {} + + try: + term = source.get_term() + except AttributeError: + term = None + elif filename is not None: from CFAPython import CFAFileFormat from CFAPython.CFADataset import CFADataset @@ -173,7 +185,9 @@ def __init__( set_fragment = self._set_fragment compute( *[ - delayed(set_fragment(var, loc, aggregated_data, filename)) + delayed( + set_fragment(var, loc, aggregated_data, filename, term) + ) for loc in product(*[range(i) for i in fragment_shape]) ] ) @@ -195,10 +209,12 @@ def __init__( fragment_shape = None aggregated_data = None instructions = None + term = None self._set_component("fragment_shape", fragment_shape, copy=False) self._set_component("aggregated_data", aggregated_data, copy=False) self._set_component("instructions", instructions, copy=False) + self._set_component("term", term, copy=False) def __dask_tokenize__(self): """Used by `dask.base.tokenize`. @@ -222,7 +238,9 @@ def __getitem__(self, indices): """x.__getitem__(indices) <==> x[indices]""" return NotImplemented # pragma: no cover - def _set_fragment(self, var, frag_loc, aggregated_data, cfa_filename): + def _set_fragment( + self, var, frag_loc, aggregated_data, cfa_filename, term + ): """Create a new key/value pair in the *aggregated_data* dictionary. @@ -244,34 +262,169 @@ def _set_fragment(self, var, frag_loc, aggregated_data, cfa_filename): aggregated_data: `dict` The aggregated data dictionary to be updated in-place. + cfa_filename: `str` + TODOCFADOCS + + term: `str` or `None` + TODOCFADOCS + :Returns: `None` """ fragment = var.getFrag(frag_loc=frag_loc) - + location = fragment.location + + if term is not None: + aggregated_data[frag_loc] = { + "file": cfa_filename, + "address": self.get_ncvar(), + "format": "full", + "location": location, + "fill_value": getattr(fragment, term), + } + return + + # Still here? filename = fragment.file fmt = fragment.format address = fragment.address if address is not None: if filename is None: - # This fragment is in the CFA-netCDF file + # This fragment is contained in the CFA-netCDF + # file filename = cfa_filename fmt = "nc" - else: - # This fragment is in its own file - filename = abspath(fragment.file) aggregated_data[frag_loc] = { "file": filename, "address": address, "format": fmt, - "location": fragment.location, + "location": location, } - def _subarray_shapes(self, shapes): + def get_aggregated_data(self, copy=True): + """Get the aggregation data dictionary. + + The aggregation data dictionary contains the definitions of + the fragments and the instructions on how to aggregate them. + The keys are indices of the CFA fragment dimensions, + e.g. ``(1, 0, 0 ,0)``. + + .. versionadded:: 3.14.0 + + :Parameters: + + copy: `bool`, optional + Whether or not to return a copy of the aggregation + dictionary. By default a deep copy is returned. + + .. warning:: If False then changing the returned + dictionary in-place will change the + aggregation dictionary stored in the + {{class}} instance, **as well as in any + copies of it**. + + :Returns: + + `dict` + The aggregation data dictionary. + + **Examples** + + >>> a.shape + (12, 1, 73, 144) + >>> a.get_fragment_shape() + (2, 1, 1, 1) + >>> a.get_aggregated_data() + {(0, 0, 0, 0): {'file': 'January-June.nc', + 'address': 'temp', + 'format': 'nc', + 'location': [(0, 6), (0, 1), (0, 73), (0, 144)]}, + (1, 0, 0, 0): {'file': 'July-December.nc', + 'address': 'temp', + 'format': 'nc', + 'location': [(6, 12), (0, 1), (0, 73), (0, 144)]}} + + """ + aggregated_data = self._get_component("aggregated_data") + if copy: + aggregated_data = deepcopy(aggregated_data) + + return aggregated_data + + def get_FragmentArray(self, fragment_format): + """Return a Fragment class. + + .. versionadded:: 3.14.0 + + :Parameters: + + fragment_format: `str` + The dataset format of the fragment. Either ``'nc'``, + ``'um'``, or `None`. + + :Returns: + + `FragmentArray` + The class for representing a fragment array of the + given format. + + """ + try: + return self._FragmentArray[fragment_format] + except KeyError: + raise ValueError( + "Can't get FragmentArray class for unknown " + f"fragment dataset format: {fragment_format!r}" + ) + + def get_fragmented_dimensions(self): + """Get the positions dimension that have two or more fragments. + + .. versionadded:: 3.14.0 + + :Returns: + + `list` + The dimension positions. + + **Examples** + + >>> a.get_fragment_shape() + (20, 1, 40, 1) + >>> a.get_fragmented_dimensions() + [0, 2] + + >>> a.get_fragment_shape() + (1, 1, 1) + >>> a.get_fragmented_dimensions() + [] + + """ + return [ + i for i, size in enumerate(self.get_fragment_shape()) if size > 1 + ] + + def get_fragment_shape(self): + """Get the sizes of the fragment dimensions. + + The fragment dimension sizes are given in the same order as + the aggregated dimension sizes given by `shape` + + .. versionadded:: 3.14.0 + + :Returns: + + `tuple` + The shape of the fragment dimensions. + + """ + return self._get_component("fragment_shape") + + def subarray_shapes(self, shapes): """Create the subarray shapes. .. versionadded:: 3.14.0 @@ -374,7 +527,7 @@ def _subarray_shapes(self, shapes): return normalize_chunks(chunks, shape=shape, dtype=self.dtype) - def _subarrays(self, subarray_shapes): + def subarrays(self, subarray_shapes): """Return descriptors for every subarray. .. versionadded:: 3.14.0 @@ -522,125 +675,6 @@ def _subarrays(self, subarray_shapes): product(*f_shapes), ) - def get_aggregated_data(self, copy=True): - """Get the aggregation data dictionary. - - The aggregation data dictionary contains the definitions of - the fragments and the instructions on how to aggregate them. - The keys are indices of the CFA fragment dimensions, - e.g. ``(1, 0, 0 ,0)``. - - .. versionadded:: 3.14.0 - - :Parameters: - - copy: `bool`, optional - Whether or not to return a copy of the aggregation - dictionary. By default a deep copy is returned. - - .. warning:: If False then changing the returned - dictionary in-place will change the - aggregation dictionary stored in the - {{class}} instance, **as well as in any - copies of it**. - - :Returns: - - `dict` - The aggregation data dictionary. - - **Examples** - - >>> a.shape - (12, 1, 73, 144) - >>> a.get_fragment_shape() - (2, 1, 1, 1) - >>> a.get_aggregated_data() - {(0, 0, 0, 0): {'file': 'January-June.nc', - 'address': 'temp', - 'format': 'nc', - 'location': [(0, 6), (0, 1), (0, 73), (0, 144)]}, - (1, 0, 0, 0): {'file': 'July-December.nc', - 'address': 'temp', - 'format': 'nc', - 'location': [(6, 12), (0, 1), (0, 73), (0, 144)]}} - - """ - aggregated_data = self._get_component("aggregated_data") - if copy: - aggregated_data = deepcopy(aggregated_data) - - return aggregated_data - - def get_FragmentArray(self, fragment_format): - """Return a Fragment class. - - .. versionadded:: 3.14.0 - - :Parameters: - - fragment_format: `str` - The dataset format of the fragment. Either ``'nc'``, - ``'um'``, or `None`. - - :Returns: - - `FragmentArray` - The class for representing a fragment array of the - given format. - - """ - try: - return self._FragmentArray[fragment_format] - except KeyError: - raise ValueError( - "Can't get FragmentArray class for unknown " - f"fragment dataset format: {fragment_format!r}" - ) - - def get_fragmented_dimensions(self): - """Get the positions dimension that have two or more fragments. - - .. versionadded:: 3.14.0 - - :Returns: - - `list` - The dimension positions. - - **Examples** - - >>> a.get_fragment_shape() - (20, 1, 40, 1) - >>> a.get_fragmented_dimensions() - [0, 2] - - >>> a.get_fragment_shape() - (1, 1, 1) - >>> a.get_fragmented_dimensions() - [] - - """ - return [ - i for i, size in enumerate(self.get_fragment_shape()) if size > 1 - ] - - def get_fragment_shape(self): - """Get the sizes of the fragment dimensions. - - The fragment dimension sizes are given in the same order as - the aggregated dimension sizes given by `shape` - - .. versionadded:: 3.14.0 - - :Returns: - - `tuple` - The shape of the fragment dimensions. - - """ - return self._get_component("fragment_shape") - def to_dask_array(self, chunks="auto"): """Create a dask array with `FragmentArray` chunks. @@ -675,7 +709,7 @@ def to_dask_array(self, chunks="auto"): aggregated_data = self.get_aggregated_data(copy=False) # Set the chunk sizes for the dask array - chunks = self._subarray_shapes(chunks) + chunks = self.subarray_shapes(chunks) # Create a FragmentArray for each chunk get_FragmentArray = self.get_FragmentArray @@ -688,18 +722,18 @@ def to_dask_array(self, chunks="auto"): chunk_location, fragment_location, fragment_shape, - ) in zip(*self._subarrays(chunks)): - d = aggregated_data[fragment_location] + ) in zip(*self.subarrays(chunks)): + kwargs = aggregated_data[fragment_location].copy() + kwargs.pop("location", None) - FragmentArray = get_FragmentArray(d["format"]) + FragmentArray = get_FragmentArray(kwargs.pop("format", None)) fragment_array = FragmentArray( - filename=d["file"], - address=d["address"], dtype=dtype, shape=fragment_shape, aggregated_units=units, aggregated_calendar=calendar, + **kwargs, ) key = f"{fragment_array.__class__.__name__}-{tokenize(fragment_array)}" diff --git a/cf/data/data.py b/cf/data/data.py index 747f606711..6a56536e76 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1257,13 +1257,13 @@ def _conform_after_dask_update(self): self._del_Array(None) self._del_cached_elements() - def _set_dask(self, array, copy=False, conform=True): + def _set_dask(self, array, copy=False, conform=True, clear_cfa=True): """Set the dask array. .. versionadded:: 3.14.0 - .. seealso:: `to_dask_array`, `_conform_after_dask_update`, - `_del_dask` + .. seealso:: `get_cfa_write`, `to_dask_array`, + `_conform_after_dask_update`, `_del_dask` :Parameters: @@ -1279,6 +1279,18 @@ def _set_dask(self, array, copy=False, conform=True): invalid by updating the `dask` array. See `_conform_after_dask_update` for details. + clear_cfa: `bool`, optional + If True, the default, then set the CFA write status to + `False`. If False then the CFA write status is + unchanged. + + If and only if the CFA write status is `True` (as + determined by `get_cfa_write`), then this `Data` + instance has the potential to be written to a + CFA-netCDF file as aggregated data. + + .. versionadded:: TODOCFAVER + :Returns: `None` @@ -1313,6 +1325,10 @@ def _set_dask(self, array, copy=False, conform=True): # array self._conform_after_dask_update() + if clear_cfa: + # Set the CFA write status to `False` + self._set_cfa_write(False) + def _del_dask(self, default=ValueError(), conform=True): """Remove the dask array. @@ -1428,6 +1444,20 @@ def _set_cached_elements(self, elements): """ self._custom.update(elements) + def _set_cfa_write(self, status): + """Set the CFA write status of the data. + + .. versionadded:: TODOCFAVER + + .. seealso:: `get_cfa_write` + + :Returns: + + `None` + + """ + self._custom["cfa_write"] = bool(status) + @_inplace_enabled(default=False) def diff(self, axis=-1, n=1, inplace=False): """Calculate the n-th discrete difference along the given axis. @@ -4274,7 +4304,9 @@ def Units(self, value): partial(cf_units, from_units=old_units, to_units=value), dtype=dtype, ) - self._set_dask(dx) + + # Changing the units does not affect the CFA write status + self._set_dask(dx, clear_cfa=False) self._Units = value @@ -5781,6 +5813,32 @@ def convert_reference_time( return d + def get_cfa_write(self): + """The CFA write status of the data. + + If and only if the CFA write status is `True`, then this + `Data` instance has the potential to be written to a + CFA-netCDF file as aggregated data. + + .. versionadded:: TODOCFAVER + + :Returns: + + `bool` + + **Examples** + + A sufficient, but not necessary, condition for the CFA write + status to be `False` is if any chunk of the data is specified + by an array in memory, rather than by an array in a file. + + >>> d = cf.Data([1, 2]) + >>> d.get_cfa_write() + False + + """ + return self._custom.get("cfa_write", False) + def get_data(self, default=ValueError(), _units=None, _fill_value=None): """Returns the data. @@ -5788,7 +5846,7 @@ def get_data(self, default=ValueError(), _units=None, _fill_value=None): :Returns: - `Data` + `Data` """ return self @@ -7558,7 +7616,9 @@ def insert_dimension(self, position=0, inplace=False): dx = d.to_dask_array() dx = dx.reshape(shape) - d._set_dask(dx) + + # Inserting a dimension does not affect the CFA write status + d._set_dask(dx, clear_cfa=False) # Expand _axes axis = new_axis_identifier(d._axes) diff --git a/cf/data/fragment/fullfragmentarray.py b/cf/data/fragment/fullfragmentarray.py new file mode 100644 index 0000000000..1f1e3337e4 --- /dev/null +++ b/cf/data/fragment/fullfragmentarray.py @@ -0,0 +1,93 @@ +from ..array.fullarray import FullArray +from .abstract import FragmentArray + + +class FullFragmentArray(FragmentArray): + """A CFA fragment array TODOCFADOCS. + + .. versionadded:: TODOCFAVER + + """ + + def __init__( + self, + fill_value=None, + filename=None, + address=None, + dtype=None, + shape=None, + aggregated_units=False, + aggregated_calendar=False, + units=None, + calendar=None, + source=None, + copy=True, + ): + """**Initialisation** + + :Parameters: + + fill_value: + TODOCFADOCS + + filename: `str` or `None` + The name of the netCDF fragment file containing the + array. + + address: `str`, optional + The name of the netCDF variable containing the + fragment array. + + dtype: `numpy.dtype` + The data type of the aggregated array. May be `None` + if the numpy data-type is not known (which can be the + case for netCDF string types, for example). This may + differ from the data type of the netCDF fragment + variable. + + shape: `tuple` + The shape of the fragment within the aggregated + array. This may differ from the shape of the netCDF + fragment variable in that the latter may have fewer + size 1 dimensions. + + units: `str` or `None`, optional + The units of the fragment data. Set to `None` to + indicate that there are no units. + + calendar: `str` or `None`, optional + The calendar of the fragment data. Set to `None` to + indicate the CF default calendar, if applicable. + + {{aggregated_units: `str` or `None`, optional}} + + {{aggregated_calendar: `str` or `None`, optional}} + + {{init source: optional}} + + {{init copy: `bool`, optional}} + + """ + if source is not None: + super().__init__(source=source, copy=copy) + return + + array = FullArray( + fill_value=fill_value, + dtype=dtype, + shape=shape, + units=None, + calendar=None, + copy=False, + ) + + super().__init__( + filename=filename, + address=address, + dtype=dtype, + shape=shape, + aggregated_units=aggregated_units, + aggregated_calendar=aggregated_calendar, + array=array, + copy=False, + ) diff --git a/cf/data/fragment/missingfragmentarray.py b/cf/data/fragment/missingfragmentarray.py index 3d702efb0e..6679981d9d 100644 --- a/cf/data/fragment/missingfragmentarray.py +++ b/cf/data/fragment/missingfragmentarray.py @@ -1,10 +1,9 @@ import numpy as np -from ..array.fullarray import FullArray -from .abstract import FragmentArray +from .fragment import FullFragmentArray -class MissingFragmentArray(FragmentArray): +class MissingFragmentArray(FullFragmentArray): """A CFA fragment array that is wholly missing data. .. versionadded:: 3.14.0 @@ -57,7 +56,7 @@ def __init__( The calendar of the fragment data. Ignored, as the data are all missing values. - {{aggregated_units: `str` or `None`, optional}}" + {{aggregated_units: `str` or `None`, optional}} {{aggregated_calendar: `str` or `None`, optional}} @@ -66,27 +65,103 @@ def __init__( {{init copy: `bool`, optional}} """ - if source is not None: - super().__init__(source=source, copy=copy) - return - - array = FullArray( - fill_value=np.ma.masked, - dtype=dtype, - shape=shape, - units=None, - calendar=None, - copy=False, - ) - super().__init__( + fill_value=np.ma.masked, filename=filename, address=address, dtype=dtype, shape=shape, aggregated_units=aggregated_units, aggregated_calendar=aggregated_calendar, - array=array, + units=units, + calendar=calendar, source=source, - copy=False, + copy=copy, ) + + +# class MissingFragmentArray(FragmentArray): +# """A CFA fragment array that is wholly missing data. +# +# .. versionadded:: 3.14.0 +# +# """ +# +# def __init__( +# self, +# filename=None, +# address=None, +# dtype=None, +# shape=None, +# aggregated_units=False, +# aggregated_calendar=False, +# units=False, +# calendar=False, +# source=None, +# copy=True, +# ): +# """**Initialisation** +# +# :Parameters: +# +# filename: `str` or `None` +# The name of the netCDF fragment file containing the +# array. +# +# address: `str`, optional +# The name of the netCDF variable containing the +# fragment array. Required unless *varid* is set. +# +# dtype: `numpy.dtype` +# The data type of the aggregated array. May be `None` +# if the numpy data-type is not known (which can be the +# case for netCDF string types, for example). This may +# differ from the data type of the netCDF fragment +# variable. +# +# shape: `tuple` +# The shape of the fragment within the aggregated +# array. This may differ from the shape of the netCDF +# fragment variable in that the latter may have fewer +# size 1 dimensions. +# +# units: `str` or `None`, optional +# The units of the fragment data. Ignored, as the data +# are all missing values. +# +# calendar: `str` or `None`, optional +# The calendar of the fragment data. Ignored, as the data +# are all missing values. +# +# {{aggregated_units: `str` or `None`, optional}} +# +# {{aggregated_calendar: `str` or `None`, optional}} +# +# {{init source: optional}} +# +# {{init copy: `bool`, optional}} +# +# """ +# if source is not None: +# super().__init__(source=source, copy=copy) +# return +# +# array = FullArray( +# fill_value=np.ma.masked, +# dtype=dtype, +# shape=shape, +# units=None, +# calendar=None, +# copy=False, +# ) +# +# super().__init__( +# filename=filename, +# address=address, +# dtype=dtype, +# shape=shape, +# aggregated_units=aggregated_units, +# aggregated_calendar=aggregated_calendar, +# array=array, +# copy=False, +# ) diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index b041fdc30c..167c8a10ae 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -1,21 +1,12 @@ import cfdm import numpy as np +from packaging.version import Version """ TODOCFA: remove aggregation_* properties from constructs TODOCFA: Create auxiliary coordinates from non-standardised terms -TODOCFA: Reference instruction variables (and/or set as - "do_not_create_field") - -TODOCFA: Create auxiliary coordinates from non-standardised terms - -TODOCFA: Consider scanning for cfa variables to the top (e.g. where - scanning for geometry varables is). This will probably need a - change in cfdm so that a customizable hook can be overlaoded - (like `_customize_read_vars` does). - TODOCFA: What about groups/netcdf_flattener? """ @@ -144,7 +135,9 @@ def _get_domain_axes(self, ncvar, allow_external=False, parent_ncvar=None): parent_ncvar=parent_ncvar, ) - # Still here? Then we have a CFA variable. + # ------------------------------------------------------------ + # Still here? Then we have a CFA-netCDF variable. + # ------------------------------------------------------------ g = self.read_vars ncdimensions = g["variable_attributes"][ncvar][ @@ -168,6 +161,7 @@ def _create_data( uncompress_override=None, parent_ncvar=None, coord_ncvar=None, + cfa_term=None, ): """Create data for a netCDF or CFA-netCDF variable. @@ -176,7 +170,8 @@ def _create_data( :Parameters: ncvar: `str` - The name of the netCDF variable that contains the data. + The name of the netCDF variable that contains the + data. See the *cfa_term* parameter. construct: optional @@ -188,14 +183,20 @@ def _create_data( coord_ncvar: `str`, optional - .. versionadded:: TODO + cfa_term: `str`, optional + The name of a non-standard aggregation instruction + term from which to create the data. If set then + *ncvar* must be the value of the term in the + ``aggregation_data`` attribute. + + .. versionadded:: TODOCFAVER :Returns: `Data` """ - if not self._is_cfa_variable(ncvar): + if cfa_term is None and not self._is_cfa_variable(ncvar): # Create data for a normal netCDF variable return super()._create_data( ncvar=ncvar, @@ -207,15 +208,21 @@ def _create_data( ) # ------------------------------------------------------------ - # Still here? Then create data for a CFA-netCDF variable + # Still here? Create data for a CFA variable # ------------------------------------------------------------ + # Remove the aggregation attributes from the construct + # properties + if construct is not None: + for attr in ("aggregation_dimensions", "aggregation_data"): + self.implementation.del_property(construct, attr, None) + cfa_array, kwargs = self._create_cfanetcdfarray( ncvar, unpacked_dtype=unpacked_dtype, coord_ncvar=coord_ncvar, + term=cfa_term, ) - # Return the data return self._create_Data( cfa_array, ncvar, @@ -244,35 +251,7 @@ def _is_cfa_variable(self, ncvar): if not g["cfa"] or ncvar in g["external_variables"]: return False - attributes = g["variable_attributes"][ncvar] - - # TODOCFA: test on the version of CFA given by g["cfa"]. See - # also `_customize_read_vars`. - cfa = "aggregated_dimensions" in attributes - if cfa: - # TODOCFA: Modify this message for v4.0.0 - raise ValueError( - "The reading of CFA files has been temporarily disabled, " - "but will return for CFA-0.6 files at version 4.0.0. " - "CFA-0.4 functionality is still available at version 3.13.1." - ) - - # TODOCFA: The 'return' remains when the exception is - # removed at v4.0.0. - return True - - cfa_04 = attributes.get("cf_role") == "cfa_variable" - if cfa_04: - # TODOCFA: Modify this message for v4.0.0. - raise ValueError( - "The reading of CFA-0.4 files was permanently disabled at " - "version 3.14.0. However, CFA-0.4 functionality is " - "still available at version 3.13.1. " - "The reading and writing of CFA-0.6 files will become " - "available at version 4.0.0." - ) - - return False + return "aggregated_dimensions" in g["variable_attributes"][ncvar] def _create_Data( self, @@ -298,9 +277,7 @@ def _create_Data( ncdimensions: sequence of `str`, optional The netCDF dimensions spanned by the array. - .. versionadded:: 3.14.0 - - units: `str`, optional + .. versionadded:: 3.14.fill_value: The units of *array*. By default, or if `None`, it is assumed that there are no units. @@ -337,6 +314,9 @@ def _create_Data( ) self._cache_data_elements(data, ncvar) + # Set the CFA write status to `True` + data._set_cfa_write(True) + return data def _customize_read_vars(self): @@ -345,41 +325,66 @@ def _customize_read_vars(self): .. versionadded:: 3.0.0 """ + from re import split + super()._customize_read_vars() g = self.read_vars - # ------------------------------------------------------------ - # Find out if this is a CFA file - # ------------------------------------------------------------ - g["cfa"] = "CFA" in g["global_attributes"].get("Conventions", ()) + # Check the 'Conventions' for CFA + Conventions = g["global_attributes"].get("Conventions", "") - if g["cfa"]: - attributes = g["variable_attributes"] - dimensions = g["variable_dimensions"] + all_conventions = split(",\s*", Conventions) + if all_conventions[0] == Conventions: + all_conventions = Conventions.split() - # Do not create fields from CFA private - # variables. TODOCFA: get private variables from - # CFANetCDFArray instances - for ncvar in g["variables"]: - if attributes[ncvar].get("cf_role", None) == "cfa_private": - g["do_not_create_field"].add(ncvar) + CFA_version = None + for c in all_conventions: + if c.startswith("CFA-"): + CFA_version = c.replace("CFA-", "", 1) + break - for ncvar, ncdims in tuple(dimensions.items()): - if ncdims != (): - continue + if c == "CFA": + # Versions <= 3.13.1 wrote CFA-0.4 files with a plain + # 'CFA' in the Conventions string + CFA_version = "0.4" + break - if not ( - ncvar not in g["external_variables"] - and "aggregated_dimensions" in attributes[ncvar] - ): + g["cfa"] = CFA_version is not None + if g["cfa"]: + # -------------------------------------------------------- + # This is a CFA-netCDF file, so check the CFA version and + # process the variables aggregated dimensions. + # -------------------------------------------------------- + g["CFA_version"] = Version(CFA_version) + if g["CFA_version"] < Version("0.6.2"): + raise ValueError( + f"Can not read file {g['filename']} that uses " + f"CFA-{CFA_version}. Only CFA-0.6.2 or newer files " + "are handled. Use version 3.13.1 to read and write " + f"CFA-0.4 files." + ) + + dimensions = g["variable_dimensions"] + attributes = g["variable_attributes"] + for ncvar, attributes in attributes.items(): + if "aggregate_dimensions" not in attributes: + # This is not an aggregated variable continue - ncdimensions = attributes[ncvar][ - "aggregated_dimensions" - ].split() - if ncdimensions: - dimensions[ncvar] = tuple(map(str, ncdimensions)) + # Set the aggregated variable's dimensions as its + # aggregated dimensions + ncdimensions = attributes["aggregated_dimensions"].split() + dimensions[ncvar] = tuple(map(str, ncdimensions)) + + # Do not create fields/domains from the aggregation + # instruction variables + parsed_aggregated_data = self._parse_aggregated_data( + ncvar, attributes.get("aggregated_data") + ) + for x in parsed_aggregated_data: + variable = tuple(x.items())[0][1] + g["do_not_create_field"].add(variable) def _cache_data_elements(self, data, ncvar): """Cache selected element values. @@ -467,10 +472,11 @@ def _create_cfanetcdfarray( ncvar, unpacked_dtype=False, coord_ncvar=None, + term=None, ): """Create a CFA-netCDF variable array. - .. versionadded:: (cfdm) 1.10.0.1 + .. versionadded:: 3.14.0 :Parameters: @@ -480,6 +486,8 @@ def _create_cfanetcdfarray( coord_ncvar: `str`, optional + term: `str`, optional + :Returns: (`CFANetCDFArray`, `dict`) @@ -499,6 +507,10 @@ def _create_cfanetcdfarray( # Get rid of the incorrect shape kwargs.pop("shape", None) + # Specify a non-standardised term from which to create the + # data + kwargs["term"] = term + # Add the aggregated_data attribute (that can be used by # dask.base.tokenize). kwargs["instructions"] = self.read_vars["variable_attributes"][ @@ -575,3 +587,120 @@ def _parse_chunks(self, ncvar): chunks = chunks2 return chunks + + def _parse_aggregated_data(self, ncvar, aggregated_data): + """Parse a CFA-netCDF aggregated_data attribute. + + .. versionadded:: TODOCFAVER + + :Parameters: + + ncvar: `str` + The netCDF variable name. + + aggregated_data: `str` or `None` + The CFA-netCDF ``aggregated_data`` attribute. + + :Returns: + + `list` + + """ + if not aggregated_data: + return [] + + return self._parse_x( + ncvar, + aggregated_data, + keys_are_variables=True, + keys_are_dimensions=False, + ) + + def _customize_auxiliary_coordinates(self, parent_ncvar, f): + """Create auxiliary coordinate constructs from CFA terms. + + This method is primarily aimed at providing a customisation + entry point for subclasses. + + This method currently creates: + + * Auxiliary coordinate constructs derived from + non-standardised terms in CFA aggregation instructions. Each + auxiliary coordinate construct spans the same domain axes as + the parent field construct. No auxiliary coordinate + constructs are ever created for `Domain` instances. + + .. versionadded:: TODODASKCFA + + :Parameters: + + parent_ncvar: `str` + The netCDF variable name of the parent variable. + + f: `Field` or `Domain` + The parent field or domain construct. + + :Returns: + + `dict` + A mapping of netCDF variable names to newly-created + auxiliary coordinate construct identifiers. + + **Examples** + + >>> _customize_auxiliary_coordinates('tas', f) + {} + + >>> _customize_auxiliary_coordinates('pr', f) + {'tracking_id': 'auxiliarycoordinate2'} + + """ + if self.implementation.is_domain(f) or not self._is_cfa_variable( + parent_ncvar + ): + return {} + + # ------------------------------------------------------------ + # Still here? Then we have a CFA-netCDF variable. + # ------------------------------------------------------------ + g = self.read_vars + + out = {} + + attributes = g["variable_attributes"]["parent_ncvar"] + parsed_aggregated_data = self._parse_aggregated_data( + parent_ncvar, attributes.get("aggregated_data") + ) + standardised_terms = ("location", "file", "address", "format") + for x in parsed_aggregated_data: + term, ncvar = tuple(x.items())[0] + if term in standardised_terms: + # Ignore standardised aggregation terms + continue + + # Still here? Then we have a non-standardised aggregation + # term that we want to convert to an auxiliary coordinate + # construct that spans the same domain axes as the parent + # field. + coord = self.implementation.initialise_AuxiliaryCoordinate() + + properties = g["variable_attributes"][ncvar].copy() + properties.setdefault("long_name", term) + self.implementation.set_properties(coord, properties) + + data = self._create_data( + ncvar, coord, parent_ncvar=parent_ncvar, cfa_term=term + ) + self.implementation.set_data(coord, data, copy=False) + + self.implementation.nc_set_variable(coord, ncvar) + + key = self.implementation.set_auxiliary_coordinate( + f, + coord, + axes=self.implementation.get_field_data_axes(f), + copy=False, + ) + out[ncvar] = key + + return out From a1ea1a21ff0e46fac75ffce57319753f6549fae5 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 7 Feb 2023 11:32:13 +0000 Subject: [PATCH 002/141] dev --- cf/data/array/fullarray.py | 35 ++++++++- cf/data/data.py | 79 ++++++++++++++------ cf/data/fragment/__init__.py | 1 + cf/data/fragment/fullfragmentarray.py | 34 ++++++--- cf/data/fragment/missingfragmentarray.py | 91 +----------------------- cf/read_write/netcdf/netcdfread.py | 29 +++++--- cf/read_write/write.py | 58 ++++++++++----- 7 files changed, 176 insertions(+), 151 deletions(-) diff --git a/cf/data/array/fullarray.py b/cf/data/array/fullarray.py index ee1a73f658..796e98dfaf 100644 --- a/cf/data/array/fullarray.py +++ b/cf/data/array/fullarray.py @@ -8,6 +8,8 @@ class FullArray(Array): The array may be empty or all missing values. + .. versionadded:: 3.14.0 + """ def __init__( @@ -194,14 +196,43 @@ def shape(self): """Tuple of array dimension sizes.""" return self._get_component("shape") - def get_fill_value(self): + def get_fill_value(self, default=ValueError()): """Return the data array fill value. .. versionadded:: 3.14.0 + .. seealso:: `set_fill_value` + + :Parameters: + + default: optional + Return the value of the *default* parameter if the + fill value has not been set. If set to an `Exception` + instance then it will be raised instead. + :Returns: The fill value. """ - return self._get_component("fill_value", None) + return self._get_component("fill_value", default=default) + + def set_fill_value(self, fill_value): + """Set the data array fill value. + + .. versionadded:: TODOCFAVER + + .. seealso:: `get_fill_value` + + :Parameters: + + fill_value : scalar, optional + The fill value for the array. May be set to + `cf.masked` or `np.ma.masked`. + + :Returns: + + `None` + + """ + self._set_component("fill_value", fill_value, copy=False) diff --git a/cf/data/data.py b/cf/data/data.py index 6a56536e76..0cb5976d54 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1257,7 +1257,7 @@ def _conform_after_dask_update(self): self._del_Array(None) self._del_cached_elements() - def _set_dask(self, array, copy=False, conform=True, clear_cfa=True): + def _set_dask(self, array, copy=False, conform=True, cfa_clear=True): """Set the dask array. .. versionadded:: 3.14.0 @@ -1279,7 +1279,7 @@ def _set_dask(self, array, copy=False, conform=True, clear_cfa=True): invalid by updating the `dask` array. See `_conform_after_dask_update` for details. - clear_cfa: `bool`, optional + cfa_clear: `bool`, optional If True, the default, then set the CFA write status to `False`. If False then the CFA write status is unchanged. @@ -1325,8 +1325,8 @@ def _set_dask(self, array, copy=False, conform=True, clear_cfa=True): # array self._conform_after_dask_update() - if clear_cfa: - # Set the CFA write status to `False` + if cfa_clear: + # Set the CFA write status to False self._set_cfa_write(False) def _del_dask(self, default=ValueError(), conform=True): @@ -1337,7 +1337,6 @@ def _del_dask(self, default=ValueError(), conform=True): .. seealso:: `to_dask_array`, `_conform_after_dask_update`, `_set_dask` - :Parameters: default: optional @@ -1451,6 +1450,11 @@ def _set_cfa_write(self, status): .. seealso:: `get_cfa_write` + :Parameters: + + status: `bool` + The new CFA write status. + :Returns: `None` @@ -3662,14 +3666,38 @@ def concatenate(cls, data, axis=0, cull_graph=True): # Get data as dask arrays and apply concatenation operation dxs = [d.to_dask_array() for d in processed_data] - data0._set_dask(da.concatenate(dxs, axis=axis)) + dx = da.concatenate(dxs, axis=axis) + + # Set the CFA write status + cfa_clear = False + for d in processed_data: + if not d.get_cfa_write(): + # Set the CFA write status to False when any of the + # input data instances have False status + cfa_clear = True + break + + if not cfa_clear: + non_concat_axis_chunks0 = list(processed_data[0].chunks) + non_concat_axis_chunks0.pop(axis) + for d in processed_data[1:]: + non_concat_axis_chunks = list(d.chunks) + non_concat_axis_chunks.pop(axis) + if non_concat_axis_chunks != non_concat_axis_chunks0: + # Set the CFA write status to False when input + # data instances have different chunk patterns for + # the non-concatenation axes + cfa_clear = True + break + + data0._set_dask(dx, cfa_clear=cfa_clear) # Manage cyclicity of axes: if join axis was cyclic, it is no longer axis = data0._parse_axes(axis)[0] if axis in data0.cyclic(): logger.warning( f"Concatenating along a cyclic axis ({axis}) therefore the " - f"axis has been set as non-cyclic in the output." + "axis has been set as non-cyclic in the output." ) data0.cyclic(axes=axis, iscyclic=False) @@ -4306,7 +4334,7 @@ def Units(self, value): ) # Changing the units does not affect the CFA write status - self._set_dask(dx, clear_cfa=False) + self._set_dask(dx, cfa_clear=False) self._Units = value @@ -5818,10 +5846,14 @@ def get_cfa_write(self): If and only if the CFA write status is `True`, then this `Data` instance has the potential to be written to a - CFA-netCDF file as aggregated data. + CFA-netCDF file as aggregated data. In this case it is the + choice of parameters to `cf.write` that determines if the data + is actually written as aggregated data. .. versionadded:: TODOCFAVER + .. seealso:: `cf.write` + :Returns: `bool` @@ -7618,7 +7650,7 @@ def insert_dimension(self, position=0, inplace=False): dx = dx.reshape(shape) # Inserting a dimension does not affect the CFA write status - d._set_dask(dx, clear_cfa=False) + d._set_dask(dx, cfa_clear=False) # Expand _axes axis = new_axis_identifier(d._axes) @@ -10591,29 +10623,30 @@ def squeeze(self, axes=None, inplace=False, i=False): shape = d.shape if axes is None: - axes = [i for i, n in enumerate(shape) if n == 1] + iaxes = tuple([i for i, n in enumerate(shape) if n == 1]) else: - axes = d._parse_axes(axes) + iaxes = d._parse_axes(axes) # Check the squeeze axes - for i in axes: + for i in iaxes: if shape[i] > 1: raise ValueError( f"Can't squeeze {d.__class__.__name__}: " f"Can't remove axis of size {shape[i]}" ) - if not axes: + if not iaxes: + # Short circuit if the squeeze is a null operation return d # Still here? Then the data array is not scalar and at least # one size 1 axis needs squeezing. dx = d.to_dask_array() - dx = dx.squeeze(axis=tuple(axes)) + dx = dx.squeeze(axis=iaxes) d._set_dask(dx) # Remove the squeezed axes names - d._axes = [axis for i, axis in enumerate(d._axes) if i not in axes] + d._axes = [axis for i, axis in enumerate(d._axes) if i not in iaxes] return d @@ -10766,15 +10799,18 @@ def transpose(self, axes=None, inplace=False, i=False): ndim = d.ndim if axes is None: - if ndim <= 1: - return d iaxes = tuple(range(ndim - 1, -1, -1)) else: iaxes = d._parse_axes(axes) - # Note: _axes attribute is still important/utilised post-Daskification - # because e.g. axes labelled as cyclic by the _cyclic attribute use it - # to determine their position (see #discussion_r694096462 on PR #247). + if iaxes == tuple(range(ndim)): + # Short circuit if the transpose is a null operation + return d + + # Note: The _axes attribute is important because e.g. axes + # labelled as cyclic by the _cyclic attribute use it to + # determine their position (see #discussion_r694096462 + # on PR #247). data_axes = d._axes d._axes = [data_axes[i] for i in iaxes] @@ -10785,6 +10821,7 @@ def transpose(self, axes=None, inplace=False, i=False): raise ValueError( f"Can't transpose: Axes don't match array: {axes}" ) + d._set_dask(dx) return d diff --git a/cf/data/fragment/__init__.py b/cf/data/fragment/__init__.py index 6d23df6b46..b13f7bef19 100644 --- a/cf/data/fragment/__init__.py +++ b/cf/data/fragment/__init__.py @@ -1,3 +1,4 @@ +from .fullfragmentarray import FullFragmentArray from .missingfragmentarray import MissingFragmentArray from .netcdffragmentarray import NetCDFFragmentArray from .umfragmentarray import UMFragmentArray diff --git a/cf/data/fragment/fullfragmentarray.py b/cf/data/fragment/fullfragmentarray.py index 1f1e3337e4..0342d76783 100644 --- a/cf/data/fragment/fullfragmentarray.py +++ b/cf/data/fragment/fullfragmentarray.py @@ -3,7 +3,7 @@ class FullFragmentArray(FragmentArray): - """A CFA fragment array TODOCFADOCS. + """A CFA fragment array that is filled with a value. .. versionadded:: TODOCFAVER @@ -18,8 +18,8 @@ def __init__( shape=None, aggregated_units=False, aggregated_calendar=False, - units=None, - calendar=None, + units=False, + calendar=False, source=None, copy=True, ): @@ -27,8 +27,8 @@ def __init__( :Parameters: - fill_value: - TODOCFADOCS + fill_value: scalar + The fill value. filename: `str` or `None` The name of the netCDF fragment file containing the @@ -53,11 +53,15 @@ def __init__( units: `str` or `None`, optional The units of the fragment data. Set to `None` to - indicate that there are no units. + indicate that there are no units. If unset then the + units will be set to `None` during the first + `__getitem__` call. calendar: `str` or `None`, optional The calendar of the fragment data. Set to `None` to - indicate the CF default calendar, if applicable. + indicate the CF default calendar, if applicable. If + unset then the calendar will be set to `None` during + the first `__getitem__` call. {{aggregated_units: `str` or `None`, optional}} @@ -76,8 +80,8 @@ def __init__( fill_value=fill_value, dtype=dtype, shape=shape, - units=None, - calendar=None, + units=units, + calendar=calendar, copy=False, ) @@ -91,3 +95,15 @@ def __init__( array=array, copy=False, ) + + def get_fill_value(self, default=ValueError()): + """The fragment array fill value. + + .. versionadded:: TODOCFAVER + + :Returns: + + The array fill value. + + """ + return self._get_component("fill_value", default=default) diff --git a/cf/data/fragment/missingfragmentarray.py b/cf/data/fragment/missingfragmentarray.py index 6679981d9d..59495e2288 100644 --- a/cf/data/fragment/missingfragmentarray.py +++ b/cf/data/fragment/missingfragmentarray.py @@ -1,5 +1,3 @@ -import numpy as np - from .fragment import FullFragmentArray @@ -65,6 +63,8 @@ def __init__( {{init copy: `bool`, optional}} """ + import numpy as np + super().__init__( fill_value=np.ma.masked, filename=filename, @@ -78,90 +78,3 @@ def __init__( source=source, copy=copy, ) - - -# class MissingFragmentArray(FragmentArray): -# """A CFA fragment array that is wholly missing data. -# -# .. versionadded:: 3.14.0 -# -# """ -# -# def __init__( -# self, -# filename=None, -# address=None, -# dtype=None, -# shape=None, -# aggregated_units=False, -# aggregated_calendar=False, -# units=False, -# calendar=False, -# source=None, -# copy=True, -# ): -# """**Initialisation** -# -# :Parameters: -# -# filename: `str` or `None` -# The name of the netCDF fragment file containing the -# array. -# -# address: `str`, optional -# The name of the netCDF variable containing the -# fragment array. Required unless *varid* is set. -# -# dtype: `numpy.dtype` -# The data type of the aggregated array. May be `None` -# if the numpy data-type is not known (which can be the -# case for netCDF string types, for example). This may -# differ from the data type of the netCDF fragment -# variable. -# -# shape: `tuple` -# The shape of the fragment within the aggregated -# array. This may differ from the shape of the netCDF -# fragment variable in that the latter may have fewer -# size 1 dimensions. -# -# units: `str` or `None`, optional -# The units of the fragment data. Ignored, as the data -# are all missing values. -# -# calendar: `str` or `None`, optional -# The calendar of the fragment data. Ignored, as the data -# are all missing values. -# -# {{aggregated_units: `str` or `None`, optional}} -# -# {{aggregated_calendar: `str` or `None`, optional}} -# -# {{init source: optional}} -# -# {{init copy: `bool`, optional}} -# -# """ -# if source is not None: -# super().__init__(source=source, copy=copy) -# return -# -# array = FullArray( -# fill_value=np.ma.masked, -# dtype=dtype, -# shape=shape, -# units=None, -# calendar=None, -# copy=False, -# ) -# -# super().__init__( -# filename=filename, -# address=address, -# dtype=dtype, -# shape=shape, -# aggregated_units=aggregated_units, -# aggregated_calendar=aggregated_calendar, -# array=array, -# copy=False, -# ) diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 167c8a10ae..835235dc31 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -228,6 +228,7 @@ def _create_data( ncvar, units=kwargs["units"], calendar=kwargs["calendar"], + cfa_write=cfa_term is None, ) def _is_cfa_variable(self, ncvar): @@ -260,6 +261,7 @@ def _create_Data( units=None, calendar=None, ncdimensions=(), + cfa_write=True, **kwargs, ): """Create a Data object from a netCDF variable. @@ -285,6 +287,11 @@ def _create_Data( The calendar of *array*. By default, or if `None`, it is assumed that there is no calendar. + cfa_write: `bool`, optional + The CFA write status. + + .. versionadded:: TODOCFAVER + kwargs: optional Extra parameters to pass to the initialisation of the returned `Data` object. @@ -314,8 +321,8 @@ def _create_Data( ) self._cache_data_elements(data, ncvar) - # Set the CFA write status to `True` - data._set_cfa_write(True) + # Set the CFA write status + data._set_cfa_write(cfa_write) return data @@ -627,8 +634,8 @@ def _customize_auxiliary_coordinates(self, parent_ncvar, f): * Auxiliary coordinate constructs derived from non-standardised terms in CFA aggregation instructions. Each auxiliary coordinate construct spans the same domain axes as - the parent field construct. No auxiliary coordinate - constructs are ever created for `Domain` instances. + the parent field construct. Auxiliary coordinate constructs + are never created for `Domain` instances. .. versionadded:: TODODASKCFA @@ -648,10 +655,10 @@ def _customize_auxiliary_coordinates(self, parent_ncvar, f): **Examples** - >>> _customize_auxiliary_coordinates('tas', f) + >>> n._customize_auxiliary_coordinates('tas', f) {} - >>> _customize_auxiliary_coordinates('pr', f) + >>> n._customize_auxiliary_coordinates('pr', f) {'tracking_id': 'auxiliarycoordinate2'} """ @@ -661,7 +668,10 @@ def _customize_auxiliary_coordinates(self, parent_ncvar, f): return {} # ------------------------------------------------------------ - # Still here? Then we have a CFA-netCDF variable. + # Still here? Then we have a CFA-netCDF variable: Loop round + # the aggregation instruction terms and convert each + # non-standard term into an auxiliary coordinate construct + # that spans the same domain axes as the parent field. # ------------------------------------------------------------ g = self.read_vars @@ -678,10 +688,7 @@ def _customize_auxiliary_coordinates(self, parent_ncvar, f): # Ignore standardised aggregation terms continue - # Still here? Then we have a non-standardised aggregation - # term that we want to convert to an auxiliary coordinate - # construct that spans the same domain axes as the parent - # field. + # Still here? Then it's a non-standard aggregation term coord = self.implementation.initialise_AuxiliaryCoordinate() properties = g["variable_attributes"][ncvar].copy() diff --git a/cf/read_write/write.py b/cf/read_write/write.py index 0ff086f655..56c2726269 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -368,26 +368,46 @@ def write( A dictionary giving parameters for configuring the output CFA-netCDF file: - ========== =============================================== - Key Value - ========== =============================================== - ``'base'`` * If ``None`` (the default) then file names - within CFA-netCDF files are stored with - absolute paths. - - * If set to an empty string then file names - within CFA-netCDF files are given relative to - the directory or URL base containing the - output CFA-netCDF file. - - * If set to a string then file names within - CFA-netCDF files are given relative to the - directory or URL base described by the - value. For example: ``'../archive'``. - ========== =============================================== - - By default no parameters are specified. + ================= ======================================= + Key Value + ================= ======================================= + + ``???`` ---------- The types of construct to be CFA-ed. By + default field data and the data Nd + metadata constructs. What about UGRID, + for which the 1-d coords are, combined, + muchlarger than the data .... + + ``properties``---- A (sequence of) `str` defining one or + more field or domain properties whose + values are to be written to the output + CFA-netCDF file as non-standardised + aggregation instructions. When the the + output file is read in with `cf.read` + these terms are converted to auxiliary + coordinate constructs. + + ``substitutions``- A dictionary whose key/value pairs + define text substitutions to be applied + to the fragment file URIs when the + output CFA-netCDF file is subsequently + read. Each key must be of the form + ``'${...}'``, where ``...`` represents + one or more letters, digits, and + underscores. The substitutions are + stored in the output file in the + ``substitutions`` attribute of the + ``file`` aggregation instruction + variable. + + ``'base'`` Deprecated at version 3.14.0. + ================= ======================================= + *Parameter example:* + ``cfa_options={'properties': 'tracking_id'}`` + + *Parameter example:* + ``cfa_options={'substitutions': {'${base}': '/home/data/'}`` endian: `str`, optional The endian-ness of the output file. Valid values are From 4210fc04a5de10c405d4a4f74f5e0988d05d9167 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 7 Feb 2023 11:57:39 +0000 Subject: [PATCH 003/141] dev --- cf/read_write/write.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cf/read_write/write.py b/cf/read_write/write.py index 56c2726269..d056e8c2e6 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -372,11 +372,13 @@ def write( Key Value ================= ======================================= - ``???`` ---------- The types of construct to be CFA-ed. By + ``???`` ---------- The types of construct to be written as + CFA-netCDF aggregated variables. By default field data and the data Nd metadata constructs. What about UGRID, for which the 1-d coords are, combined, - muchlarger than the data .... + muchlarger than the data .... What + about DSG and compression in general? ``properties``---- A (sequence of) `str` defining one or more field or domain properties whose @@ -403,6 +405,8 @@ def write( ``'base'`` Deprecated at version 3.14.0. ================= ======================================= + The *cfa_options* default to ``{'???': ['field', 'N-d']}`` + *Parameter example:* ``cfa_options={'properties': 'tracking_id'}`` From 6335b2e66944626a00018e37326097e622e02759 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 7 Feb 2023 15:11:15 +0000 Subject: [PATCH 004/141] dev --- cf/data/data.py | 6 ++++-- cf/read_write/write.py | 14 +++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/cf/data/data.py b/cf/data/data.py index 0cb5976d54..0f86310998 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -352,7 +352,9 @@ def __init__( except (AttributeError, TypeError): pass else: - self._set_dask(array, copy=copy, conform=False) + self._set_dask( + array, copy=copy, conform=False, clear_cfa=False + ) else: self._del_dask(None) @@ -458,7 +460,7 @@ def __init__( self._Units = units # Store the dask array - self._set_dask(array, conform=False) + self._set_dask(array, conform=False, clear_cfa=False) # Override the data type if dtype is not None: diff --git a/cf/read_write/write.py b/cf/read_write/write.py index d056e8c2e6..5e1a9418c4 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -365,8 +365,8 @@ def write( no such constructs. cfa_options: `dict`, optional - A dictionary giving parameters for configuring the output - CFA-netCDF file: + A dictionary defining parameters for configuring the + output CFA-netCDF file: ================= ======================================= Key Value @@ -384,10 +384,10 @@ def write( more field or domain properties whose values are to be written to the output CFA-netCDF file as non-standardised - aggregation instructions. When the the - output file is read in with `cf.read` - these terms are converted to auxiliary - coordinate constructs. + aggregation instruction variables. When + the output file is read in with + `cf.read` these variables are converted + to auxiliary coordinate constructs. ``substitutions``- A dictionary whose key/value pairs define text substitutions to be applied @@ -411,7 +411,7 @@ def write( ``cfa_options={'properties': 'tracking_id'}`` *Parameter example:* - ``cfa_options={'substitutions': {'${base}': '/home/data/'}`` + ``cfa_options={'substitutions': {'${base}': '/home/data/'}}`` endian: `str`, optional The endian-ness of the output file. Valid values are From 0e759dfef73c455c315e49b8a031b47ad081c348 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 7 Feb 2023 22:37:54 +0000 Subject: [PATCH 005/141] dev --- cf/data/data.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/cf/data/data.py b/cf/data/data.py index 0f86310998..ff6aa83a37 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1259,7 +1259,8 @@ def _conform_after_dask_update(self): self._del_Array(None) self._del_cached_elements() - def _set_dask(self, array, copy=False, conform=True, cfa_clear=True): + # def _set_dask(self, array, copy=False, conform=True, cfa_clear=True): + def _set_dask(self, array, copy=False, conform=True): """Set the dask array. .. versionadded:: 3.14.0 @@ -1322,15 +1323,30 @@ def _set_dask(self, array, copy=False, conform=True, cfa_clear=True): self._custom["dask"] = array - if conform: - # Remove elements made invalid by updating the `dask` - # array - self._conform_after_dask_update() + if conform is True: + self._set_cfa_write(False) + self._del_Array(None) + self._del_cached_elements() + return - if cfa_clear: - # Set the CFA write status to False + if conform & _CFA: self._set_cfa_write(False) + if conform & _ARRAY: + self._del_Array(None) + + if conform & _CACHE: + self._del_cached_elements() + + # if conform: + # # Remove elements made invalid by updating the `dask` + # # array + # self._conform_after_dask_update() + # + # if cfa_clear: + # # Set the CFA write status to False + # self._set_cfa_write(False) + def _del_dask(self, default=ValueError(), conform=True): """Remove the dask array. From 4853c9fa1272d57658b25b5581a8504ded6caf03 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 8 Feb 2023 11:58:17 +0000 Subject: [PATCH 006/141] dev --- cf/data/data.py | 177 ++++++++++++++++------------- cf/read_write/netcdf/netcdfread.py | 11 +- 2 files changed, 102 insertions(+), 86 deletions(-) diff --git a/cf/data/data.py b/cf/data/data.py index ff6aa83a37..da7f5ed332 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -86,6 +86,13 @@ _DEFAULT_CHUNKS = "auto" _DEFAULT_HARDMASK = True +# +_NONE = 0 +_ARRAY = 1 +_CACHE = 2 +_CFA = 4 +_ACTIVE = 8 # ACTIVE +_ALL = _ARRAY | _CACHE | _CFA | _ACTIVE # ACTIVE class Data(DataClassDeprecationsMixin, Container, cfdm.Data): """An N-dimensional data array with units and masked values. @@ -352,9 +359,7 @@ def __init__( except (AttributeError, TypeError): pass else: - self._set_dask( - array, copy=copy, conform=False, clear_cfa=False - ) + self._set_dask(array, copy=copy, conform=_NONE) else: self._del_dask(None) @@ -460,7 +465,7 @@ def __init__( self._Units = units # Store the dask array - self._set_dask(array, conform=False, clear_cfa=False) + self._set_dask(array, conform=_NONE) # Override the data type if dtype is not None: @@ -1240,33 +1245,65 @@ def __keepdims_indexing__(self): def __keepdims_indexing__(self, value): self._custom["__keepdims_indexing__"] = bool(value) - def _conform_after_dask_update(self): - """Remove elements made invalid by updating the `dask` array. + def _conform_after_dask_update(self, conform=_ALL): + """Remove components invalidated by updating the `dask` array. Removes or modifies components that can't be guaranteed to be - consistent with an updated `dask` array`: - - * Deletes a source array. - * Deletes cached element values. + consistent with an updated `dask` array`. See the *conform* + parameter for details. .. versionadded:: 3.14.0 + .. seealso:: `_del_Array`, `_del_cached_elements`, + `_del_dask`, `_set_cfa_write`, `_set_dask` + + :Parameters: + + conform: `int`, optional + Specify which components should be removed. The value + of *conform* is sequentially combined with the + ``_ARRAY``, ``_CACHE`` and ``_CFA`` contants, using + the bitwise AND operator, to determine which + components should be removed: + + * If ``conform & _ARRAY`` is True then delete a source + array. + + * If ``conform & _CACHE`` is True then delete cached + element values. + + * If ``conform & _CFA`` is True then set the CFA write + status to `False`. + + By default *conform* is the ``_ALL`` contstant, which + results in all components being removed. + + .. versionadded:: TODOCFAVER + :Returns: `None` """ - self._del_Array(None) - self._del_cached_elements() + if conform & _ARRAY: + # Delete a source array + self._del_Array(None) + + if conform & _CACHE: + # Delete cached element values + self._del_cached_elements() + + if conform & _CFA: + # Set the CFA write status to False + self._set_cfa_write(False) - # def _set_dask(self, array, copy=False, conform=True, cfa_clear=True): - def _set_dask(self, array, copy=False, conform=True): + def _set_dask(self, array, copy=False, conform=_ALL): """Set the dask array. .. versionadded:: 3.14.0 - .. seealso:: `get_cfa_write`, `to_dask_array`, - `_conform_after_dask_update`, `_del_dask` + .. seealso:: `to_dask_array`, `_conform_after_dask_update`, + `_del_dask` :Parameters: @@ -1277,22 +1314,12 @@ def _set_dask(self, array, copy=False, conform=True): If True then copy *array* before setting it. By default it is not copied. - conform: `bool`, optional - If True, the default, then remove elements made - invalid by updating the `dask` array. See - `_conform_after_dask_update` for details. + conform: `int`, optional + Specify which components should be removed. By default + *conform* is the ``_ALL`` contstant, which results in + all components being removed. - cfa_clear: `bool`, optional - If True, the default, then set the CFA write status to - `False`. If False then the CFA write status is - unchanged. - - If and only if the CFA write status is `True` (as - determined by `get_cfa_write`), then this `Data` - instance has the potential to be written to a - CFA-netCDF file as aggregated data. - - .. versionadded:: TODOCFAVER + See `_conform_after_dask_update` for further details. :Returns: @@ -1322,32 +1349,9 @@ def _set_dask(self, array, copy=False, conform=True): array = array.copy() self._custom["dask"] = array - - if conform is True: - self._set_cfa_write(False) - self._del_Array(None) - self._del_cached_elements() - return - - if conform & _CFA: - self._set_cfa_write(False) - - if conform & _ARRAY: - self._del_Array(None) - - if conform & _CACHE: - self._del_cached_elements() - - # if conform: - # # Remove elements made invalid by updating the `dask` - # # array - # self._conform_after_dask_update() - # - # if cfa_clear: - # # Set the CFA write status to False - # self._set_cfa_write(False) - - def _del_dask(self, default=ValueError(), conform=True): + self._conform_after_dask_update(conform) + + def _del_dask(self, default=ValueError(), conform=_ALL): """Remove the dask array. .. versionadded:: 3.14.0 @@ -1363,10 +1367,12 @@ def _del_dask(self, default=ValueError(), conform=True): {{default Exception}} - conform: `bool`, optional - If True, the default, then remove elements made - invalid by updating the `dask` array. See - `_conform_after_dask_update` for details. + conform: `int`, optional + Specify which components should be removed. By default + *conform* is the ``_ALL`` contstant, which results in + all components being removed. + + See `_conform_after_dask_update` for further details. :Returns: @@ -1396,11 +1402,7 @@ def _del_dask(self, default=ValueError(), conform=True): default, f"{self.__class__.__name__!r} has no dask array" ) - if conform: - # Remove elements made invalid by deleting the `dask` - # array - self._conform_after_dask_update() - + self._conform_after_dask_update(conform) return out def _del_cached_elements(self): @@ -1464,6 +1466,14 @@ def _set_cached_elements(self, elements): def _set_cfa_write(self, status): """Set the CFA write status of the data. + This should only be set to `True` if it is known that the dask + array is compatible with the requirements of a CFA-netCDF + aggregation variable's aggregated data. Conversely, it should + be set to `False` if it that compaibility can not be + guaranteed. + + If unset then the CFA write status defaults to `False`. + .. versionadded:: TODOCFAVER .. seealso:: `get_cfa_write` @@ -2339,7 +2349,7 @@ def persist(self, inplace=False): dx = self.to_dask_array() dx = dx.persist() - d._set_dask(dx, conform=False) + d._set_dask(dx, conform=_ALL ^ _ARRAY ^ _CACHE) return d @@ -2790,7 +2800,7 @@ def rechunk( dx = d.to_dask_array() dx = dx.rechunk(chunks, threshold, block_size_limit, balance) - d._set_dask(dx, conform=False) + d._set_dask(dx, conform=_ALL ^ _ARRAY ^ _CACHE) return d @@ -3687,15 +3697,15 @@ def concatenate(cls, data, axis=0, cull_graph=True): dx = da.concatenate(dxs, axis=axis) # Set the CFA write status - cfa_clear = False + cfa = _NONE for d in processed_data: if not d.get_cfa_write(): # Set the CFA write status to False when any of the # input data instances have False status - cfa_clear = True + cfa = _CFA break - if not cfa_clear: + if not cfa: non_concat_axis_chunks0 = list(processed_data[0].chunks) non_concat_axis_chunks0.pop(axis) for d in processed_data[1:]: @@ -3705,10 +3715,10 @@ def concatenate(cls, data, axis=0, cull_graph=True): # Set the CFA write status to False when input # data instances have different chunk patterns for # the non-concatenation axes - cfa_clear = True + cfa = _CFA break - data0._set_dask(dx, cfa_clear=cfa_clear) + data0._set_dask(dx, conform=_ALL ^ cfa) # Manage cyclicity of axes: if join axis was cyclic, it is no longer axis = data0._parse_axes(axis)[0] @@ -4352,7 +4362,7 @@ def Units(self, value): ) # Changing the units does not affect the CFA write status - self._set_dask(dx, cfa_clear=False) + self._set_dask(dx, conform=_ALL ^ _CFA) self._Units = value @@ -5865,8 +5875,8 @@ def get_cfa_write(self): If and only if the CFA write status is `True`, then this `Data` instance has the potential to be written to a CFA-netCDF file as aggregated data. In this case it is the - choice of parameters to `cf.write` that determines if the data - is actually written as aggregated data. + choice of parameters to the `cf.write` function that + determines if the data is actually written as aggregated data. .. versionadded:: TODOCFAVER @@ -7667,8 +7677,9 @@ def insert_dimension(self, position=0, inplace=False): dx = d.to_dask_array() dx = dx.reshape(shape) - # Inserting a dimension does not affect the CFA write status - d._set_dask(dx, cfa_clear=False) + # Inserting a dimension does not affect the cached elements + # nor the CFA write status + d._set_dask(dx, conform=_ALL ^ _CACHE ^ CFA) # Expand _axes axis = new_axis_identifier(d._axes) @@ -8054,7 +8065,7 @@ def harden_mask(self): """ dx = self.to_dask_array() dx = dx.map_blocks(cf_harden_mask, dtype=self.dtype) - self._set_dask(dx, conform=False) + self._set_dask(dx, conform=_NONE) self.hardmask = True def has_calendar(self): @@ -8151,7 +8162,7 @@ def soften_mask(self): """ dx = self.to_dask_array() dx = dx.map_blocks(cf_soften_mask, dtype=self.dtype) - self._set_dask(dx, conform=False) + self._set_dask(dx, conform=_NONE) self.hardmask = False @_inplace_enabled(default=False) @@ -10464,7 +10475,7 @@ def cull_graph(self): dx = self.to_dask_array() dsk, _ = cull(dx.dask, dx.__dask_keys__()) dx = da.Array(dsk, name=dx.name, chunks=dx.chunks, dtype=dx.dtype) - self._set_dask(dx, conform=False) + self._set_dask(dx, conform=_NONE) @_deprecated_kwarg_check("i", version="3.0.0", removed_at="4.0.0") @_inplace_enabled(default=False) @@ -10661,7 +10672,9 @@ def squeeze(self, axes=None, inplace=False, i=False): # one size 1 axis needs squeezing. dx = d.to_dask_array() dx = dx.squeeze(axis=iaxes) - d._set_dask(dx) + + # Squeezing a dimension does not affect the cached elements + d._set_dask(dx, conform=_ALL ^ _CACHE) # Remove the squeezed axes names d._axes = [axis for i, axis in enumerate(d._axes) if i not in iaxes] diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 835235dc31..e8eb346218 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -324,6 +324,9 @@ def _create_Data( # Set the CFA write status data._set_cfa_write(cfa_write) + # Set the active storage status # ACTIVE + data._set_active_storage(True) # ACTIVE + return data def _customize_read_vars(self): @@ -367,9 +370,9 @@ def _customize_read_vars(self): if g["CFA_version"] < Version("0.6.2"): raise ValueError( f"Can not read file {g['filename']} that uses " - f"CFA-{CFA_version}. Only CFA-0.6.2 or newer files " - "are handled. Use version 3.13.1 to read and write " - f"CFA-0.4 files." + f"CFA conventions version CFA-{CFA_version}. " + "Only CFA-0.6.2 or newer files are allowed. Version " + "3.13.1 can be used to read and write CFA-0.4 files." ) dimensions = g["variable_dimensions"] @@ -384,7 +387,7 @@ def _customize_read_vars(self): ncdimensions = attributes["aggregated_dimensions"].split() dimensions[ncvar] = tuple(map(str, ncdimensions)) - # Do not create fields/domains from the aggregation + # Do not create fields/domains from aggregation # instruction variables parsed_aggregated_data = self._parse_aggregated_data( ncvar, attributes.get("aggregated_data") From c6ba761f42080b613b580c980681b988ca1f9a20 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 8 Feb 2023 21:24:28 +0000 Subject: [PATCH 007/141] dev --- cf/data/array/cfanetcdfarray.py | 100 +++++++++++----- cf/data/array/fullarray.py | 22 ++-- cf/data/data.py | 109 ++++++++++-------- cf/data/fragment/abstract/fragmentarray.py | 52 +-------- cf/data/fragment/fullfragmentarray.py | 25 ++-- cf/data/fragment/missingfragmentarray.py | 14 +-- cf/data/fragment/mixin/__init__.py | 1 + .../fragment/mixin/fragmentfilearraymixin.py | 43 +++++++ cf/data/fragment/netcdffragmentarray.py | 5 +- cf/data/fragment/umfragmentarray.py | 5 +- cf/read_write/netcdf/netcdfread.py | 56 +++++---- cf/read_write/netcdf/netcdfwrite.py | 3 +- cf/test/create_test_files.py | 73 ++++++++++++ 13 files changed, 312 insertions(+), 196 deletions(-) create mode 100644 cf/data/fragment/mixin/__init__.py create mode 100644 cf/data/fragment/mixin/fragmentfilearraymixin.py diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index 29eb02aaa2..64566bd707 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -44,7 +44,7 @@ def __init__( units=False, calendar=False, instructions=None, - term=None, + non_standard_term=None, source=None, copy=True, ): @@ -108,12 +108,24 @@ def __init__( indicate the CF default calendar, if applicable. instructions: `str`, optional - The ``aggregated_data`` attribute value found on the - CFA netCDF variable. If set then this will be used by - `__dask_tokenize__` to improve performance. + The ``aggregated_data`` attribute value as found on + the CFA netCDF variable. If set then this will be used + to improve the performance of `__dask_tokenize__`. + + non_standard_term: `str`, optional + The name of a non-standard aggregation instruction + term from which the array is to be created, instead of + the creating the aggregated data in the usual + manner. If set then *ncvar* must be the name of the + term's CFA-netCDF aggregation instruction variable, + which must be defined on the fragment dimensions and + no others. Each value of the aggregation instruction + variable will be broadcast across the shape of the + corresponding fragment. - term: `str`, optional - TODOCFADOCS + *Parameter example:* + ``non_standard_term='tracking_id', + ncvar='aggregation_id'`` {{init source: optional}} @@ -139,9 +151,9 @@ def __init__( aggregated_data = {} try: - term = source.get_term() + non_standard_term = source.get_non_standard_term() except AttributeError: - term = None + non_standard_term = None elif filename is not None: from CFAPython import CFAFileFormat @@ -186,7 +198,13 @@ def __init__( compute( *[ delayed( - set_fragment(var, loc, aggregated_data, filename, term) + set_fragment( + var, + loc, + aggregated_data, + filename, + non_standard_term, + ) ) for loc in product(*[range(i) for i in fragment_shape]) ] @@ -209,12 +227,12 @@ def __init__( fragment_shape = None aggregated_data = None instructions = None - term = None + non_standard_term = None self._set_component("fragment_shape", fragment_shape, copy=False) self._set_component("aggregated_data", aggregated_data, copy=False) self._set_component("instructions", instructions, copy=False) - self._set_component("term", term, copy=False) + self._set_component("non_standard_term", non_standard_term, copy=False) def __dask_tokenize__(self): """Used by `dask.base.tokenize`. @@ -239,7 +257,7 @@ def __getitem__(self, indices): return NotImplemented # pragma: no cover def _set_fragment( - self, var, frag_loc, aggregated_data, cfa_filename, term + self, var, frag_loc, aggregated_data, cfa_filename, non_standard_term ): """Create a new key/value pair in the *aggregated_data* dictionary. @@ -265,8 +283,15 @@ def _set_fragment( cfa_filename: `str` TODOCFADOCS - term: `str` or `None` - TODOCFADOCS + non_standard_term: `str` or `None` + The name of a non-standard aggregation instruction + term from which the array is to be created, instead of + the creating the aggregated data in the usual + manner. Each value of the aggregation instruction + variable will be broadcast across the shape of the + corresponding fragment. + + .. versionadded:: TODOCFAVER :Returns: @@ -276,13 +301,12 @@ def _set_fragment( fragment = var.getFrag(frag_loc=frag_loc) location = fragment.location - if term is not None: + if non_standard_term is not None: + # This fragment contains a constant value aggregated_data[frag_loc] = { - "file": cfa_filename, - "address": self.get_ncvar(), "format": "full", "location": location, - "fill_value": getattr(fragment, term), + "full_value": getattr(fragment, non_standard_term), } return @@ -292,19 +316,25 @@ def _set_fragment( address = fragment.address if address is not None: + # This fragment is contained in a file if filename is None: - # This fragment is contained in the CFA-netCDF - # file + # This fragment is contained in the CFA-netCDF file filename = cfa_filename fmt = "nc" - aggregated_data[frag_loc] = { - "file": filename, - "address": address, - "format": fmt, - "location": location, - } - + aggregated_data[frag_loc] = { + "file": filename, + "address": address, + "format": fmt, + "location": location, + } + elif filename is None: + # This fragment contains wholly missing values + aggregated_data[frag_loc] = { + "format": None, + "location": location, + } + def get_aggregated_data(self, copy=True): """Get the aggregation data dictionary. @@ -424,6 +454,22 @@ def get_fragment_shape(self): """ return self._get_component("fragment_shape") + def get_non_standard_term(self, default=ValueError()): + """Get the sizes of the fragment dimensions. + + The fragment dimension sizes are given in the same order as + the aggregated dimension sizes given by `shape` + + .. versionadded:: TODOCFAVER + + :Returns: + + `tuple` + The shape of the fragment dimensions. + + """ + return self._get_component("non_standard_term", default=default) + def subarray_shapes(self, shapes): """Create the subarray shapes. diff --git a/cf/data/array/fullarray.py b/cf/data/array/fullarray.py index 796e98dfaf..74dbd2d4b4 100644 --- a/cf/data/array/fullarray.py +++ b/cf/data/array/fullarray.py @@ -58,7 +58,7 @@ def __init__( if source is not None: try: - fill_value = source._get_component("fill_value", None) + fill_value = source._get_component("full_value", None) except AttributeError: fill_value = None @@ -82,7 +82,7 @@ def __init__( except AttributeError: calendar = None - self._set_component("fill_value", fill_value, copy=False) + self._set_component("full_value", fill_value, copy=False) self._set_component("dtype", dtype, copy=False) self._set_component("shape", shape, copy=False) self._set_component("units", units, copy=False) @@ -119,7 +119,7 @@ def __getitem__(self, indices): apply_indices = True array_shape = self.shape - fill_value = self.get_fill_value() + fill_value = self.get_full_value() if fill_value is np.ma.masked: array = np.ma.masked_all(array_shape, dtype=self.dtype) elif fill_value is not None: @@ -150,7 +150,7 @@ def __str__(self): x.__str__() <==> str(x) """ - fill_value = self.get_fill_value() + fill_value = self.get_full_value() if fill_value is None: return "Uninitialised" @@ -196,12 +196,12 @@ def shape(self): """Tuple of array dimension sizes.""" return self._get_component("shape") - def get_fill_value(self, default=ValueError()): + def get_full_value(self, default=AttributeError()): """Return the data array fill value. .. versionadded:: 3.14.0 - .. seealso:: `set_fill_value` + .. seealso:: `set_full_value` :Parameters: @@ -215,14 +215,14 @@ def get_fill_value(self, default=ValueError()): The fill value. """ - return self._get_component("fill_value", default=default) + return self._get_component("full_value", default=default) - def set_fill_value(self, fill_value): + def set_full_value(self, fill_value): """Set the data array fill value. - .. versionadded:: TODOCFAVER + .. versionadded:: 3.14.0 - .. seealso:: `get_fill_value` + .. seealso:: `get_full_value` :Parameters: @@ -235,4 +235,4 @@ def set_fill_value(self, fill_value): `None` """ - self._set_component("fill_value", fill_value, copy=False) + self._set_component("full_value", fill_value, copy=False) diff --git a/cf/data/data.py b/cf/data/data.py index da7f5ed332..3b2ba53167 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -86,13 +86,15 @@ _DEFAULT_CHUNKS = "auto" _DEFAULT_HARDMASK = True -# +# Contstants used to specify which components should be removed when +# the dask array is updated. See `Data._conform_after_dask_update` for +# details. _NONE = 0 _ARRAY = 1 _CACHE = 2 _CFA = 4 -_ACTIVE = 8 # ACTIVE -_ALL = _ARRAY | _CACHE | _CFA | _ACTIVE # ACTIVE +_ALL = _ARRAY | _CACHE | _CFA + class Data(DataClassDeprecationsMixin, Container, cfdm.Data): """An N-dimensional data array with units and masked values. @@ -1262,21 +1264,25 @@ def _conform_after_dask_update(self, conform=_ALL): conform: `int`, optional Specify which components should be removed. The value of *conform* is sequentially combined with the - ``_ARRAY``, ``_CACHE`` and ``_CFA`` contants, using - the bitwise AND operator, to determine which - components should be removed: + ``_ARRAY``, ``_CACHE`` and ``_CFA`` integer-valued + contants, using the bitwise AND operator, to determine + which components should be removed: * If ``conform & _ARRAY`` is True then delete a source array. - + * If ``conform & _CACHE`` is True then delete cached element values. - + * If ``conform & _CFA`` is True then set the CFA write status to `False`. - - By default *conform* is the ``_ALL`` contstant, which - results in all components being removed. + + By default *conform* is the ``_ALL`` integer-valued + constant, which results in all components being + removed. + + If *conform* is the ``_NONE`` integer-valued constant + then no components are removed. .. versionadded:: TODOCFAVER @@ -1316,8 +1322,8 @@ def _set_dask(self, array, copy=False, conform=_ALL): conform: `int`, optional Specify which components should be removed. By default - *conform* is the ``_ALL`` contstant, which results in - all components being removed. + *conform* is the ``_ALL`` integer-valued constant, + which results in all components being removed. See `_conform_after_dask_update` for further details. @@ -1350,7 +1356,7 @@ def _set_dask(self, array, copy=False, conform=_ALL): self._custom["dask"] = array self._conform_after_dask_update(conform) - + def _del_dask(self, default=ValueError(), conform=_ALL): """Remove the dask array. @@ -1369,8 +1375,8 @@ def _del_dask(self, default=ValueError(), conform=_ALL): conform: `int`, optional Specify which components should be removed. By default - *conform* is the ``_ALL`` contstant, which results in - all components being removed. + *conform* is the ``_ALL`` integer-valued constant, + which results in all components being removed. See `_conform_after_dask_update` for further details. @@ -1476,7 +1482,7 @@ def _set_cfa_write(self, status): .. versionadded:: TODOCFAVER - .. seealso:: `get_cfa_write` + .. seealso:: `cfa_write` :Parameters: @@ -3699,9 +3705,9 @@ def concatenate(cls, data, axis=0, cull_graph=True): # Set the CFA write status cfa = _NONE for d in processed_data: - if not d.get_cfa_write(): - # Set the CFA write status to False when any of the - # input data instances have False status + if not d.cfa_write: + # Set the CFA write status to False when any input + # data instance has False status cfa = _CFA break @@ -4373,6 +4379,37 @@ def Units(self): "Consider using the override_units method instead." ) + @property + def cfa_write(self): + """The CFA write status of the data. + + If and only if the CFA write status is `True`, then this + `Data` instance has the potential to be written to a + CFA-netCDF file as aggregated data. In this case it is the + choice of parameters to the `cf.write` function that + determines if the data is actually written as aggregated data. + + .. versionadded:: TODOCFAVER + + .. seealso:: `cf.write` + + :Returns: + + `bool` + + **Examples** + + A sufficient, but not necessary, condition for the CFA write + status to be `False` is if any chunk of the data is specified + by an array in memory, rather than by an array in a file. + + >>> d = cf.Data([1, 2]) + >>> d.cfa_write + False + + """ + return self._custom.get("cfa_write", False) + @property def data(self): """The data as an object identity. @@ -5869,36 +5906,6 @@ def convert_reference_time( return d - def get_cfa_write(self): - """The CFA write status of the data. - - If and only if the CFA write status is `True`, then this - `Data` instance has the potential to be written to a - CFA-netCDF file as aggregated data. In this case it is the - choice of parameters to the `cf.write` function that - determines if the data is actually written as aggregated data. - - .. versionadded:: TODOCFAVER - - .. seealso:: `cf.write` - - :Returns: - - `bool` - - **Examples** - - A sufficient, but not necessary, condition for the CFA write - status to be `False` is if any chunk of the data is specified - by an array in memory, rather than by an array in a file. - - >>> d = cf.Data([1, 2]) - >>> d.get_cfa_write() - False - - """ - return self._custom.get("cfa_write", False) - def get_data(self, default=ValueError(), _units=None, _fill_value=None): """Returns the data. @@ -7679,7 +7686,7 @@ def insert_dimension(self, position=0, inplace=False): # Inserting a dimension does not affect the cached elements # nor the CFA write status - d._set_dask(dx, conform=_ALL ^ _CACHE ^ CFA) + d._set_dask(dx, conform=_ALL ^ _CACHE ^ _CFA) # Expand _axes axis = new_axis_identifier(d._axes) diff --git a/cf/data/fragment/abstract/fragmentarray.py b/cf/data/fragment/abstract/fragmentarray.py index 14f60bda90..706c6ed849 100644 --- a/cf/data/fragment/abstract/fragmentarray.py +++ b/cf/data/fragment/abstract/fragmentarray.py @@ -1,11 +1,10 @@ from numbers import Integral from ....units import Units -from ...array.abstract import FileArray -class FragmentArray(FileArray): - """A CFA fragment array. +class FragmentArray: + """Abstract base class for a CFA fragment array. .. versionadded:: 3.14.0 @@ -13,8 +12,6 @@ class FragmentArray(FileArray): def __init__( self, - filename=None, - address=None, dtype=None, shape=None, aggregated_units=False, @@ -27,14 +24,6 @@ def __init__( :Parameters: - filename: `str` - The name of the netCDF fragment file containing the - array. - - address: `str`, optional - The name of the netCDF variable containing the - fragment array. Required unless *varid* is set. - dtype: `numpy.dtype` The data type of the aggregated array. May be `None` if the numpy data-type is not known (which can be the @@ -66,16 +55,6 @@ def __init__( super().__init__(source=source, copy=copy) if source is not None: - try: - filename = source._get_component("filename", None) - except AttributeError: - filename = None - - try: - address = source._get_component("address", None) - except AttributeError: - address = None - try: dtype = source._get_component("dtype", None) except AttributeError: @@ -105,8 +84,6 @@ def __init__( except AttributeError: array = None - self._set_component("filename", filename, copy=False) - self._set_component("address", address, copy=False) self._set_component("dtype", dtype, copy=False) self._set_component("shape", shape, copy=False) self._set_component("aggregated_units", aggregated_units, copy=False) @@ -263,23 +240,6 @@ def aggregated_Units(self): self.get_aggregated_units(), self.get_aggregated_calendar(None) ) - def close(self): - """Close the dataset containing the data.""" - return NotImplemented # pragma: no cover - - def get_address(self): - """The address of the fragment in the file. - - .. versionadded:: 3.14.0 - - :Returns: - - The file address of the fragment, or `None` if there - isn't one. - - """ - return self._get_component("address", None) - def get_aggregated_calendar(self, default=ValueError()): """The calendar of the aggregated array. @@ -351,13 +311,13 @@ def get_aggregated_units(self, default=ValueError()): return units def get_array(self): - """The fragment array stored in a file. + """The fragment array. .. versionadded:: 3.14.0 :Returns: - `Array` + Subclass of `Array` The object defining the fragment array. """ @@ -384,7 +344,3 @@ def get_units(self, default=ValueError()): """ return self.get_array().get_units(default) - - def open(self): - """Returns an open dataset containing the data array.""" - return NotImplemented # pragma: no cover diff --git a/cf/data/fragment/fullfragmentarray.py b/cf/data/fragment/fullfragmentarray.py index 0342d76783..f33b762d18 100644 --- a/cf/data/fragment/fullfragmentarray.py +++ b/cf/data/fragment/fullfragmentarray.py @@ -12,8 +12,6 @@ class FullFragmentArray(FragmentArray): def __init__( self, fill_value=None, - filename=None, - address=None, dtype=None, shape=None, aggregated_units=False, @@ -30,14 +28,6 @@ def __init__( fill_value: scalar The fill value. - filename: `str` or `None` - The name of the netCDF fragment file containing the - array. - - address: `str`, optional - The name of the netCDF variable containing the - fragment array. - dtype: `numpy.dtype` The data type of the aggregated array. May be `None` if the numpy data-type is not known (which can be the @@ -86,8 +76,6 @@ def __init__( ) super().__init__( - filename=filename, - address=address, dtype=dtype, shape=shape, aggregated_units=aggregated_units, @@ -96,14 +84,21 @@ def __init__( copy=False, ) - def get_fill_value(self, default=ValueError()): + def get_full_value(self, default=AttributeError()): """The fragment array fill value. .. versionadded:: TODOCFAVER + :Parameters: + + default: optional + Return the value of the *default* parameter if the + fill value has not been set. If set to an `Exception` + instance then it will be raised instead. + :Returns: - The array fill value. + The fill value. """ - return self._get_component("fill_value", default=default) + return self.get_array().get_full_value(default=default) diff --git a/cf/data/fragment/missingfragmentarray.py b/cf/data/fragment/missingfragmentarray.py index 59495e2288..b52ff0b20b 100644 --- a/cf/data/fragment/missingfragmentarray.py +++ b/cf/data/fragment/missingfragmentarray.py @@ -1,4 +1,4 @@ -from .fragment import FullFragmentArray +from .fullfragmentarray import FullFragmentArray class MissingFragmentArray(FullFragmentArray): @@ -10,8 +10,6 @@ class MissingFragmentArray(FullFragmentArray): def __init__( self, - filename=None, - address=None, dtype=None, shape=None, aggregated_units=False, @@ -25,14 +23,6 @@ def __init__( :Parameters: - filename: `str` or `None` - The name of the netCDF fragment file containing the - array. - - address: `str`, optional - The name of the netCDF variable containing the - fragment array. Required unless *varid* is set. - dtype: `numpy.dtype` The data type of the aggregated array. May be `None` if the numpy data-type is not known (which can be the @@ -67,8 +57,6 @@ def __init__( super().__init__( fill_value=np.ma.masked, - filename=filename, - address=address, dtype=dtype, shape=shape, aggregated_units=aggregated_units, diff --git a/cf/data/fragment/mixin/__init__.py b/cf/data/fragment/mixin/__init__.py new file mode 100644 index 0000000000..3b995a260d --- /dev/null +++ b/cf/data/fragment/mixin/__init__.py @@ -0,0 +1 @@ +from .fragmentfilearraymixin import FragmentFileArrayMixin diff --git a/cf/data/fragment/mixin/fragmentfilearraymixin.py b/cf/data/fragment/mixin/fragmentfilearraymixin.py new file mode 100644 index 0000000000..66a2801cd8 --- /dev/null +++ b/cf/data/fragment/mixin/fragmentfilearraymixin.py @@ -0,0 +1,43 @@ +class FragmentFileArrayMixin: + """Mixin class for a fragment array stored in a file. + + .. versionadded:: TODOCFAVER + + """ + + def get_address(self): + """The address of the fragment in the file. + + .. versionadded:: TODOCFAVER + + :Returns: + + The file address of the fragment, or `None` if there + isn't one. + + """ + try: + return self.get_array().get_address() + except AttributeError: + return + + def get_filename(self): + """Return the name of the file containing the array. + + .. versionadded:: TODOCFAVER + + :Returns: + + `str` or `None` + The filename, or `None` if there isn't one. + + **Examples** + + >>> a.get_filename() + 'file.nc' + + """ + try: + return self.get_array().get_filename() + except AttributeError: + return diff --git a/cf/data/fragment/netcdffragmentarray.py b/cf/data/fragment/netcdffragmentarray.py index 4bf56408ac..008fab6edd 100644 --- a/cf/data/fragment/netcdffragmentarray.py +++ b/cf/data/fragment/netcdffragmentarray.py @@ -1,8 +1,9 @@ from ..array.netcdfarray import NetCDFArray from .abstract import FragmentArray +from .mixin import FragmentFileArrayMixin -class NetCDFFragmentArray(FragmentArray): +class NetCDFFragmentArray(FragmentFileArrayMixin, FragmentArray): """A CFA fragment array stored in a netCDF file. .. versionadded:: 3.14.0 @@ -95,8 +96,6 @@ def __init__( ) super().__init__( - filename=filename, - address=address, dtype=dtype, shape=shape, aggregated_units=aggregated_units, diff --git a/cf/data/fragment/umfragmentarray.py b/cf/data/fragment/umfragmentarray.py index 4ecff13303..cc957669e8 100644 --- a/cf/data/fragment/umfragmentarray.py +++ b/cf/data/fragment/umfragmentarray.py @@ -1,8 +1,9 @@ from ..array.umarray import UMArray from .abstract import FragmentArray +from .mixin import FragmentFileArrayMixin -class UMFragmentArray(FragmentArray): +class UMFragmentArray(FragmentFileArrayMixin, FragmentArray): """A CFA fragment array stored in a UM or PP file. .. versionadded:: 3.14.0 @@ -80,8 +81,6 @@ def __init__( ) super().__init__( - filename=filename, - address=address, dtype=dtype, shape=shape, aggregated_units=aggregated_units, diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index e8eb346218..d3f6f7945b 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -198,7 +198,7 @@ def _create_data( """ if cfa_term is None and not self._is_cfa_variable(ncvar): # Create data for a normal netCDF variable - return super()._create_data( + data = super()._create_data( ncvar=ncvar, construct=construct, unpacked_dtype=unpacked_dtype, @@ -207,12 +207,21 @@ def _create_data( coord_ncvar=coord_ncvar, ) + self._cache_data_elements(data, ncvar) + + if data.npartitions == 1: + # Set the CFA write status to True when there is + # exactly one dask chunk + data._set_cfa_write(True) + + return data + # ------------------------------------------------------------ # Still here? Create data for a CFA variable # ------------------------------------------------------------ - # Remove the aggregation attributes from the construct - # properties if construct is not None: + # Remove the aggregation attributes from the construct + # properties for attr in ("aggregation_dimensions", "aggregation_data"): self.implementation.del_property(construct, attr, None) @@ -223,16 +232,23 @@ def _create_data( term=cfa_term, ) - return self._create_Data( + data = self._create_Data( cfa_array, ncvar, units=kwargs["units"], calendar=kwargs["calendar"], - cfa_write=cfa_term is None, ) + if cfa_term is not None: + self._cache_data_elements(data, ncvar) + + # Set the CFA write status to True + data._set_cfa_write(True) + + return data + def _is_cfa_variable(self, ncvar): - """Return True if *ncvar* is a CFA variable. + """Return True if *ncvar* is a CFA aggregated variable. .. versionadded:: 3.14.0 @@ -248,7 +264,6 @@ def _is_cfa_variable(self, ncvar): """ g = self.read_vars - if not g["cfa"] or ncvar in g["external_variables"]: return False @@ -261,7 +276,6 @@ def _create_Data( units=None, calendar=None, ncdimensions=(), - cfa_write=True, **kwargs, ): """Create a Data object from a netCDF variable. @@ -287,11 +301,6 @@ def _create_Data( The calendar of *array*. By default, or if `None`, it is assumed that there is no calendar. - cfa_write: `bool`, optional - The CFA write status. - - .. versionadded:: TODOCFAVER - kwargs: optional Extra parameters to pass to the initialisation of the returned `Data` object. @@ -319,13 +328,6 @@ def _create_Data( chunks=chunks, **kwargs, ) - self._cache_data_elements(data, ncvar) - - # Set the CFA write status - data._set_cfa_write(cfa_write) - - # Set the active storage status # ACTIVE - data._set_active_storage(True) # ACTIVE return data @@ -369,10 +371,10 @@ def _customize_read_vars(self): g["CFA_version"] = Version(CFA_version) if g["CFA_version"] < Version("0.6.2"): raise ValueError( - f"Can not read file {g['filename']} that uses " + f"Can't read file {g['filename']} that uses obselete " f"CFA conventions version CFA-{CFA_version}. " - "Only CFA-0.6.2 or newer files are allowed. Version " - "3.13.1 can be used to read and write CFA-0.4 files." + "(Note that version 3.13.1 can be used to read and " + "write CFA-0.4 files.)" ) dimensions = g["variable_dimensions"] @@ -491,12 +493,20 @@ def _create_cfanetcdfarray( :Parameters: ncvar: `str` + The name of the CFA-netCDF aggregated variable. See + the *term* parameter. unpacked_dtype: `False` or `numpy.dtype`, optional coord_ncvar: `str`, optional term: `str`, optional + The name of a non-standard aggregation instruction + term from which to create the array. If set then + *ncvar* must be the value of the term in the + ``aggregation_data`` attribute. + + .. versionadded:: TODOCFAVER :Returns: diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 149cd1cc46..bd96d28c54 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -231,8 +231,7 @@ def _write_scalar_coordinate( The updated list of netCDF auxiliary coordinate names. """ - # Unsafe to set mutable '{}' as default in the func signature. - if extra is None: # distinguish from falsy '{}' + if extra is None: extra = {} coord_1d = self._change_reference_datetime(coord_1d) diff --git a/cf/test/create_test_files.py b/cf/test/create_test_files.py index fe49d28e63..0cd4f5dd42 100644 --- a/cf/test/create_test_files.py +++ b/cf/test/create_test_files.py @@ -15539,6 +15539,77 @@ def _make_regrid_file(filename): ] +def _make_cfa_file(filename): + n = netCDF4.Dataset(filename, "w", format="NETCDF4") + + n.Conventions = f"CF-{VN} CFA-0.6.2" + n.comment = ( + "A CFA-netCDF file with non-standarised aggregation instructions" + ) + + n.createDimension("time", 12) + level = n.createDimension("level", 1) + lat = n.createDimension("lat", 73) + lon = n.createDimension("lon", 144) + n.createDimension("f_time", 2) + n.createDimension("f_level", 1) + n.createDimension("f_lat", 1) + n.createDimension("f_lon", 1) + n.createDimension("i", 4) + n.createDimension("j", 2) + + lon = n.createVariable("lon", "f4", ("lon",)) + lon.standard_name = "longitude" + lon.units = "degrees_east" + + lat = n.createVariable("lat", "f4", ("lat",)) + lat.standard_name = "latitude" + lat.units = "degrees_north" + + time = n.createVariable("time", "f4", ("time",)) + time.standard_name = "time" + time.units = "days since 2000-01-01" + + level = n.createVariable("level", "f4", ("level",)) + + tas = n.createVariable("tas", "f4", ()) + tas.standard_name = "air_temperature" + tas.units = "K" + tas.aggregated_dimensions = "time level lat lon" + tas.aggregated_data = "location: aggregation_location file: aggregation_file format: aggregation_format address: aggregation_address tracking_id: aggregation_tracking_id" + + loc = n.createVariable("aggregation_location", "i4", ("i", "j")) + loc[0, :] = 6 + loc[1, 0] = level.size + loc[2, 0] = lat.size + loc[3, 0] = lon.size + + fil = n.createVariable( + "aggregation_file", str, ("f_time", "f_level", "f_lat", "f_lon") + ) + fil[0, 0, 0, 0] = "January-June.nc" + fil[1, 0, 0, 0] = "July-December.nc" + + add = n.createVariable( + "aggregation_address", str, ("f_time", "f_level", "f_lat", "f_lon") + ) + add[0, 0, 0, 0] = "tas0" + add[1, 0, 0, 0] = "tas1" + + fmt = n.createVariable("aggregation_format", str, ()) + fmt[()] = "nc" + + tid = n.createVariable( + "aggregation_tracking_id", str, ("f_time", "f_level", "f_lat", "f_lon") + ) + tid[0, 0, 0, 0] = "tracking_id0" + tid[1, 0, 0, 0] = "tracking_id1" + + n.close() + + return filename + + contiguous_file = _make_contiguous_file("DSG_timeSeries_contiguous.nc") indexed_file = _make_indexed_file("DSG_timeSeries_indexed.nc") indexed_contiguous_file = _make_indexed_contiguous_file( @@ -15572,6 +15643,8 @@ def _make_regrid_file(filename): regrid_file = _make_regrid_file("regrid.nc") +cfa_file = _make_cfa_file("cfa.nc") + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) cf.environment() From 71361ccd15c7f9f21ab99a6250ae76c88fc74356 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 9 Feb 2023 09:36:43 +0000 Subject: [PATCH 008/141] dev --- cf/data/array/cfanetcdfarray.py | 2 +- cf/data/data.py | 109 ++++++++++++++++---------------- 2 files changed, 55 insertions(+), 56 deletions(-) diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index 64566bd707..ddf7d94f8d 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -334,7 +334,7 @@ def _set_fragment( "format": None, "location": location, } - + def get_aggregated_data(self, copy=True): """Get the aggregation data dictionary. diff --git a/cf/data/data.py b/cf/data/data.py index 3b2ba53167..0f333ebb14 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -86,13 +86,13 @@ _DEFAULT_CHUNKS = "auto" _DEFAULT_HARDMASK = True -# Contstants used to specify which components should be removed when -# the dask array is updated. See `Data._conform_after_dask_update` for -# details. -_NONE = 0 -_ARRAY = 1 -_CACHE = 2 -_CFA = 4 +# Contstants used to specify which `Data` components should be cleared +# when the dask array is updated. See `Data._clear_after_dask_update` +# for details. +_NONE = 0 # = 0b000 +_ARRAY = 1 # = 0b001 +_CACHE = 2 # = 0b010 +_CFA = 4 # = 0b100 _ALL = _ARRAY | _CACHE | _CFA @@ -361,7 +361,7 @@ def __init__( except (AttributeError, TypeError): pass else: - self._set_dask(array, copy=copy, conform=_NONE) + self._set_dask(array, copy=copy, clear=_NONE) else: self._del_dask(None) @@ -467,7 +467,7 @@ def __init__( self._Units = units # Store the dask array - self._set_dask(array, conform=_NONE) + self._set_dask(array, clear=_NONE) # Override the data type if dtype is not None: @@ -1129,7 +1129,7 @@ def __setitem__(self, indices, value): self[indices] = reset # Remove elements made invalid by updating the `dask` array - self._conform_after_dask_update() + self._clear_after_dask_update(_ALL) return @@ -1247,11 +1247,11 @@ def __keepdims_indexing__(self): def __keepdims_indexing__(self, value): self._custom["__keepdims_indexing__"] = bool(value) - def _conform_after_dask_update(self, conform=_ALL): + def _clear_after_dask_update(self, clear=_ALL): """Remove components invalidated by updating the `dask` array. Removes or modifies components that can't be guaranteed to be - consistent with an updated `dask` array`. See the *conform* + consistent with an updated `dask` array`. See the *clear* parameter for details. .. versionadded:: 3.14.0 @@ -1261,27 +1261,27 @@ def _conform_after_dask_update(self, conform=_ALL): :Parameters: - conform: `int`, optional + clear: `int`, optional Specify which components should be removed. The value - of *conform* is sequentially combined with the + of *clear* is sequentially combined with the ``_ARRAY``, ``_CACHE`` and ``_CFA`` integer-valued contants, using the bitwise AND operator, to determine which components should be removed: - * If ``conform & _ARRAY`` is True then delete a source + * If ``clear & _ARRAY`` is True then delete a source array. - * If ``conform & _CACHE`` is True then delete cached + * If ``clear & _CACHE`` is True then delete cached element values. - * If ``conform & _CFA`` is True then set the CFA write + * If ``clear & _CFA`` is True then set the CFA write status to `False`. - By default *conform* is the ``_ALL`` integer-valued + By default *clear* is the ``_ALL`` integer-valued constant, which results in all components being removed. - If *conform* is the ``_NONE`` integer-valued constant + If *clear* is the ``_NONE`` integer-valued constant then no components are removed. .. versionadded:: TODOCFAVER @@ -1291,24 +1291,24 @@ def _conform_after_dask_update(self, conform=_ALL): `None` """ - if conform & _ARRAY: + if clear & _ARRAY: # Delete a source array self._del_Array(None) - if conform & _CACHE: + if clear & _CACHE: # Delete cached element values self._del_cached_elements() - if conform & _CFA: + if clear & _CFA: # Set the CFA write status to False self._set_cfa_write(False) - def _set_dask(self, array, copy=False, conform=_ALL): + def _set_dask(self, array, copy=False, clear=_ALL): """Set the dask array. .. versionadded:: 3.14.0 - .. seealso:: `to_dask_array`, `_conform_after_dask_update`, + .. seealso:: `to_dask_array`, `_clear_after_dask_update`, `_del_dask` :Parameters: @@ -1320,12 +1320,12 @@ def _set_dask(self, array, copy=False, conform=_ALL): If True then copy *array* before setting it. By default it is not copied. - conform: `int`, optional + clear: `int`, optional Specify which components should be removed. By default - *conform* is the ``_ALL`` integer-valued constant, - which results in all components being removed. + *clear* is the ``_ALL`` integer-valued constant, which + results in all components being removed. - See `_conform_after_dask_update` for further details. + See `_clear_after_dask_update` for further details. :Returns: @@ -1355,14 +1355,14 @@ def _set_dask(self, array, copy=False, conform=_ALL): array = array.copy() self._custom["dask"] = array - self._conform_after_dask_update(conform) + self._clear_after_dask_update(clear) - def _del_dask(self, default=ValueError(), conform=_ALL): + def _del_dask(self, default=ValueError(), clear=_ALL): """Remove the dask array. .. versionadded:: 3.14.0 - .. seealso:: `to_dask_array`, `_conform_after_dask_update`, + .. seealso:: `to_dask_array`, `_clear_after_dask_update`, `_set_dask` :Parameters: @@ -1373,12 +1373,12 @@ def _del_dask(self, default=ValueError(), conform=_ALL): {{default Exception}} - conform: `int`, optional + clear: `int`, optional Specify which components should be removed. By default - *conform* is the ``_ALL`` integer-valued constant, - which results in all components being removed. + *clear* is the ``_ALL`` integer-valued constant, which + results in all components being removed. - See `_conform_after_dask_update` for further details. + See `_clear_after_dask_update` for further details. :Returns: @@ -1408,7 +1408,7 @@ def _del_dask(self, default=ValueError(), conform=_ALL): default, f"{self.__class__.__name__!r} has no dask array" ) - self._conform_after_dask_update(conform) + self._clear_after_dask_update(clear) return out def _del_cached_elements(self): @@ -1474,15 +1474,18 @@ def _set_cfa_write(self, status): This should only be set to `True` if it is known that the dask array is compatible with the requirements of a CFA-netCDF - aggregation variable's aggregated data. Conversely, it should - be set to `False` if it that compaibility can not be + aggregation variable (or non-stan... TODOCFADOCS). Conversely, + it should be set to `False` if it that compaibility can not be guaranteed. + The CFA status may be set to `True` in `cf.read`. See + `NetCDFRead._create_data` for details. + If unset then the CFA write status defaults to `False`. .. versionadded:: TODOCFAVER - .. seealso:: `cfa_write` + .. seealso:: `cfa_write`, `cf.read`, `cf.write` :Parameters: @@ -2355,7 +2358,7 @@ def persist(self, inplace=False): dx = self.to_dask_array() dx = dx.persist() - d._set_dask(dx, conform=_ALL ^ _ARRAY ^ _CACHE) + d._set_dask(dx, clear=_ALL ^ _ARRAY ^ _CACHE) return d @@ -2806,7 +2809,7 @@ def rechunk( dx = d.to_dask_array() dx = dx.rechunk(chunks, threshold, block_size_limit, balance) - d._set_dask(dx, conform=_ALL ^ _ARRAY ^ _CACHE) + d._set_dask(dx, clear=_ALL ^ _ARRAY ^ _CACHE) return d @@ -3703,12 +3706,12 @@ def concatenate(cls, data, axis=0, cull_graph=True): dx = da.concatenate(dxs, axis=axis) # Set the CFA write status - cfa = _NONE + cfa = _CFA for d in processed_data: if not d.cfa_write: # Set the CFA write status to False when any input # data instance has False status - cfa = _CFA + cfa = _NONE break if not cfa: @@ -3721,10 +3724,10 @@ def concatenate(cls, data, axis=0, cull_graph=True): # Set the CFA write status to False when input # data instances have different chunk patterns for # the non-concatenation axes - cfa = _CFA + cfa = _NONE break - data0._set_dask(dx, conform=_ALL ^ cfa) + data0._set_dask(dx, clear=_ALL ^ cfa) # Manage cyclicity of axes: if join axis was cyclic, it is no longer axis = data0._parse_axes(axis)[0] @@ -4368,7 +4371,7 @@ def Units(self, value): ) # Changing the units does not affect the CFA write status - self._set_dask(dx, conform=_ALL ^ _CFA) + self._set_dask(dx, clear=_ALL ^ _CFA) self._Units = value @@ -4399,10 +4402,6 @@ def cfa_write(self): **Examples** - A sufficient, but not necessary, condition for the CFA write - status to be `False` is if any chunk of the data is specified - by an array in memory, rather than by an array in a file. - >>> d = cf.Data([1, 2]) >>> d.cfa_write False @@ -7686,7 +7685,7 @@ def insert_dimension(self, position=0, inplace=False): # Inserting a dimension does not affect the cached elements # nor the CFA write status - d._set_dask(dx, conform=_ALL ^ _CACHE ^ _CFA) + d._set_dask(dx, clear=_ALL ^ _CACHE ^ _CFA) # Expand _axes axis = new_axis_identifier(d._axes) @@ -8072,7 +8071,7 @@ def harden_mask(self): """ dx = self.to_dask_array() dx = dx.map_blocks(cf_harden_mask, dtype=self.dtype) - self._set_dask(dx, conform=_NONE) + self._set_dask(dx, clear=_NONE) self.hardmask = True def has_calendar(self): @@ -8169,7 +8168,7 @@ def soften_mask(self): """ dx = self.to_dask_array() dx = dx.map_blocks(cf_soften_mask, dtype=self.dtype) - self._set_dask(dx, conform=_NONE) + self._set_dask(dx, clear=_NONE) self.hardmask = False @_inplace_enabled(default=False) @@ -10482,7 +10481,7 @@ def cull_graph(self): dx = self.to_dask_array() dsk, _ = cull(dx.dask, dx.__dask_keys__()) dx = da.Array(dsk, name=dx.name, chunks=dx.chunks, dtype=dx.dtype) - self._set_dask(dx, conform=_NONE) + self._set_dask(dx, clear=_NONE) @_deprecated_kwarg_check("i", version="3.0.0", removed_at="4.0.0") @_inplace_enabled(default=False) @@ -10681,7 +10680,7 @@ def squeeze(self, axes=None, inplace=False, i=False): dx = dx.squeeze(axis=iaxes) # Squeezing a dimension does not affect the cached elements - d._set_dask(dx, conform=_ALL ^ _CACHE) + d._set_dask(dx, clear=_ALL ^ _CACHE) # Remove the squeezed axes names d._axes = [axis for i, axis in enumerate(d._axes) if i not in iaxes] From bb7738073a6259e1f3675968ac895daf29efe94a Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 9 Feb 2023 15:17:14 +0000 Subject: [PATCH 009/141] dev --- cf/data/fragment/abstract/fragmentarray.py | 102 ++++++++++++++++++-- cf/data/fragment/netcdffragmentarray.py | 107 ++++++++++++--------- 2 files changed, 156 insertions(+), 53 deletions(-) diff --git a/cf/data/fragment/abstract/fragmentarray.py b/cf/data/fragment/abstract/fragmentarray.py index 706c6ed849..80c4b1d9b1 100644 --- a/cf/data/fragment/abstract/fragmentarray.py +++ b/cf/data/fragment/abstract/fragmentarray.py @@ -103,19 +103,89 @@ def __getitem__(self, indices): differences: * A dimension's index can't be rank-reducing, i.e. it can't - be an integer, nor a scalar `numpy` or `dask` array. + be an integer, a scalar `numpy` array, nor a scalar `dask` + array. * When two or more dimension's indices are sequences of integers then these indices work independently along each dimension (similar to the way vector subscripts work in Fortran). + **Performance** + + If the fragment variable has fewer than `ndim` dimensions then + the entire array is realised in memory before the requested + subspace of it is returned. + + .. versionadded:: 3.14.0 + + """ + return self._getitem( self.get_array(), indices) + +# indices = self._parse_indices(indices) +# array = self.get_array() +# +# try: +# array = array[indices] +# except ValueError: +# # A value error is raised if indices has at least ndim +# # elements but the fragment variable has fewer than ndim +# # dimensions. In this case we get the entire fragment +# # array, insert the missing size 1 dimensions, and then +# # apply the requested slice. See the CFA conventions for +# # details. +# array = array[Ellipsis] +# if array.ndim < self.ndim: +# array = array.reshape(self.shape) +# +# array = array[indices] +# +# array = self._conform_units(array) +# return array + + def _getitem(self, array, indices): + """Returns a subspace of the fragment as a numpy array. + + x.__getitem__(indices) <==> x[indices] + + Indexing is similar to numpy indexing, with the following + differences: + + * A dimension's index can't be rank-reducing, i.e. it can't + be an integer, a scalar `numpy` array, nor a scalar `dask` + array. + + * When two or more dimension's indices are sequences of + integers then these indices work independently along each + dimension (similar to the way vector subscripts work in + Fortran). + + **Performance** + + If the fragment variable has fewer than `ndim` dimensions then + the entire array is realised in memory before the requested + subspace of it is returned. + .. versionadded:: 3.14.0 """ - array = self.get_array() indices = self._parse_indices(indices) - array = array[indices] + + try: + array = array[indices] + except ValueError: + # A value error is raised if indices has at least ndim + # elements but the fragment variable has fewer than ndim + # dimensions. In this case we get the entire fragment + # array, insert the missing size 1 dimensions, and then + # apply the requested slice. See the CFA conventions for + # details. + array = array[Ellipsis] + if array.ndim < self.ndim: + array = array.reshape(self.shape) + + array = array[indices] + array = self._conform_units(array) return array @@ -196,17 +266,21 @@ def _conform_units(self, array): :Parameters: - array: `numpy.ndarray` - The array to be conformed. + array: `numpy.ndarray` or `dict` + The array to be conformed. If *array* is a `dict` with + `numpy` array values then each value is conformed. :Returns: - `numpy.ndarray` + `numpy.ndarray` or `dict` The conformed array. The returned array may or may not be the input array updated in-place, depending on its data type and the nature of its units and the aggregated units. + If *array* is a `dict` then a dictionary of conformed + arrays is returned. + """ units = self.Units if units: @@ -218,9 +292,19 @@ def _conform_units(self, array): ) if units != aggregated_units: - array = Units.conform( - array, units, aggregated_units, inplace=True - ) + if isinstance(array, dict): + # 'array' is a dictionary + array = { + key: Units.conform( + value, units, aggregated_units, inplace=True + ) + for key, value in array.items() + } + else: + # 'array' is a numpy array + array = Units.conform( + array, units, aggregated_units, inplace=True + ) return array diff --git a/cf/data/fragment/netcdffragmentarray.py b/cf/data/fragment/netcdffragmentarray.py index 008fab6edd..46a1e5a1b9 100644 --- a/cf/data/fragment/netcdffragmentarray.py +++ b/cf/data/fragment/netcdffragmentarray.py @@ -79,7 +79,7 @@ def __init__( ncvar = address varid = None - # TODO set groups from ncvar + # TODOCFA set groups from ncvar group = None array = NetCDFArray( @@ -105,47 +105,66 @@ def __init__( copy=False, ) - def __getitem__(self, indices): - """Returns a subspace of the fragment as a numpy array. - x.__getitem__(indices) <==> x[indices] - - Indexing is similar to numpy indexing, with the following - differences: - - * A dimension's index can't be rank-reducing, i.e. it can't - be an integer, nor a scalar `numpy` or `dask` array. - - * When two or more dimension's indices are sequences of - integers then these indices work independently along each - dimension (similar to the way vector subscripts work in - Fortran). - - **Performance** - - If the netCDF fragment variable has fewer than `ndim` - dimensions then the entire array is read into memory before - the requested subspace of it is returned. - - .. versionadded:: 3.14.0 - - """ - indices = self._parse_indices(indices) - array = self.get_array() - - try: - array = array[indices] - except ValueError: - # A value error is raised if indices has at least ndim - # elements but the netCDF fragment variable has fewer than - # ndim dimensions. In this case we get the entire fragment - # array, insert the missing size 1 dimensions, and then - # apply the requested slice. - array = array[Ellipsis] - if array.ndim < self.ndim: - array = array.reshape(self.shape) - - array = array[indices] - - array = self._conform_units(array) - return array +# def __getitem__(self, indices): +# """Returns a subspace of the fragment as a numpy array. +# +# x.__getitem__(indices) <==> x[indices] +# +# Indexing is similar to numpy indexing, with the following +# differences: +# +# * A dimension's index can't be rank-reducing, i.e. it can't +# be an integer, nor a scalar `numpy` or `dask` array. +# +# * When two or more dimension's indices are sequences of +# integers then these indices work independently along each +# dimension (similar to the way vector subscripts work in +# Fortran). +# +# **Performance** +# +# If the netCDF fragment variable has fewer than `ndim` +# dimensions then the entire array is read into memory before +# the requested subspace of it is returned. +# +# .. versionadded:: 3.14.0 +# +# """ +# try: +# return super().__getitem__(indices) +# except ValueError: +# # A value error is raised if indices has at least ndim +# # elements but the netCDF fragment variable has fewer than +# # ndim dimensions. In this case we get the entire fragment +# # array, insert the missing size 1 dimensions, and then +# # apply the requested slice. +# indices = self._parse_indices(indices) +# array = self.get_array() +# array = array[Ellipsis] +# if array.ndim < self.ndim: +# array = array.reshape(self.shape) +# +# array = array[indices] +# array = self._conform_units(array) +# return array +# +# indices = self._parse_indices(indices) +# array = self.get_array() +# +# try: +# array = array[indices] +# except ValueError: +# # A value error is raised if indices has at least ndim +# # elements but the netCDF fragment variable has fewer than +# # ndim dimensions. In this case we get the entire fragment +# # array, insert the missing size 1 dimensions, and then +# # apply the requested slice. +# array = array[Ellipsis] +# if array.ndim < self.ndim: +# array = array.reshape(self.shape) +# +# array = array[indices] +# +# array = self._conform_units(array) +# return array From 40f81d10fa7c4168d8d2ab139717ef8df7937c5b Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 10 Feb 2023 16:20:09 +0000 Subject: [PATCH 010/141] dev --- cf/data/creation.py | 42 +++++++++++++++++++++++++++++++++++++ cf/data/data.py | 51 ++++++++++++++++++++++++--------------------- 2 files changed, 69 insertions(+), 24 deletions(-) diff --git a/cf/data/creation.py b/cf/data/creation.py index 0db52403c0..49d7354152 100644 --- a/cf/data/creation.py +++ b/cf/data/creation.py @@ -3,8 +3,12 @@ import dask.array as da import numpy as np + +from cfdm import Array from dask.base import is_dask_collection +from .array.mixin import FileArrayMixin + def to_dask(array, chunks, **from_array_options): """Create a `dask` array. @@ -116,3 +120,41 @@ def generate_axis_identifiers(n): """ return [f"dim{i}" for i in range(n)] + + +def is_file_array(array): + """Whether or not an array is stored on disk. + + .. versionaddedd: TODOCFAVER + + :Parameters: + + array: + TODOCFADOCS + + :Returns: + + `bool` + TODOCFADOCS + + """ + return isinstance(array, FileArrayMixin) + + +def is_abstract_Array_subclass(array): + """Whether or not an array is a type of abstract Array. + + .. versionaddedd: TODOCFAVER + + :Parameters: + + array: + TODOCFADOCS + + :Returns: + + `bool` + TODOCFADOCS + + """ + return isinstance(array, Array) diff --git a/cf/data/data.py b/cf/data/data.py index 646fd6262c..5e348d0ff0 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -38,7 +38,7 @@ from ..mixin_container import Container from ..units import Units from .collapse import Collapse -from .creation import generate_axis_identifiers, to_dask +from .creation import generate_axis_identifiers, to_dask, is_file_array, is_abstract_Array_subclass from .dask_utils import ( _da_ma_allclose, cf_contains, @@ -411,28 +411,29 @@ def __init__( # Still here? Then create a dask array and store it. - # Find out if the input data is compressed by convention try: compressed = array.get_compression_type() except AttributeError: - compressed = "" - - if compressed: + pass + else: + # The input data is compressed by convention if init_options.get("from_array"): raise ValueError( "Can't define 'from_array' initialisation options " "for compressed input arrays" ) - # Bring the compressed data into memory without - # decompressing it + if is_file_array(array): if to_memory: try: array = array.to_memory() except AttributeError: pass + else: + # Allow the possibilty of CFA writing + self._set_cfa_write(True) - if self._is_abstract_Array_subclass(array): + if is_abstract_Array_subclass(array): # Save the input array in case it's useful later. For # compressed input arrays this will contain extra information, # such as a count or index variable. @@ -622,20 +623,6 @@ def _rtol(self): """Return the current value of the `cf.rtol` function.""" return rtol().value - def _is_abstract_Array_subclass(self, array): - """Whether or not an array is a type of abstract Array. - - :Parameters: - - array: - - :Returns: - - `bool` - - """ - return isinstance(array, cfdm.Array) - def __data__(self): """Returns a new reference to self.""" return self @@ -1302,7 +1289,7 @@ def _clear_after_dask_update(self, clear=_ALL): if clear & _CFA: # Set the CFA write status to False - self._set_cfa_write(False) + self._del_cfa_write() def _set_dask(self, array, copy=False, clear=_ALL): """Set the dask array. @@ -1441,6 +1428,21 @@ def _del_cached_elements(self): for element in ("first_element", "second_element", "last_element"): custom.pop(element, None) + def _del_cfa_write(self, status): + """Set the CFA write status of the data to `False`. + + .. versionadded:: TODOCFAVER + + .. seealso:: `cfa_write`, `_set_cfa_write` + + :Returns: + + `bool` + The CFA status prior to deletion. + + """ + return self._custom.pop("cfa_write", False) + def _set_cached_elements(self, elements): """Cache selected element values. @@ -1486,7 +1488,8 @@ def _set_cfa_write(self, status): .. versionadded:: TODOCFAVER - .. seealso:: `cfa_write`, `cf.read`, `cf.write` + .. seealso:: `cfa_write`, `cf.read`, `cf.write`, + `_del_cfa_write` :Parameters: From e1e9dbbe9dadd72cbc832068584af9d326053a6a Mon Sep 17 00:00:00 2001 From: David Hassell Date: Sat, 11 Feb 2023 11:46:33 +0000 Subject: [PATCH 011/141] dev --- cf/data/fragment/abstract/fragmentarray.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cf/data/fragment/abstract/fragmentarray.py b/cf/data/fragment/abstract/fragmentarray.py index 80c4b1d9b1..a417aded50 100644 --- a/cf/data/fragment/abstract/fragmentarray.py +++ b/cf/data/fragment/abstract/fragmentarray.py @@ -294,12 +294,11 @@ def _conform_units(self, array): if units != aggregated_units: if isinstance(array, dict): # 'array' is a dictionary - array = { - key: Units.conform( - value, units, aggregated_units, inplace=True - ) - for key, value in array.items() - } + for key, value in array.items(): + if key in _active_chunk_methds: + array[key] = Units.conform( + value, units, aggregated_units, inplace=True + ) else: # 'array' is a numpy array array = Units.conform( From 4bf3dc814af5b51aa96756d08d4e3efbe28c7ce1 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 13 Feb 2023 11:35:15 +0000 Subject: [PATCH 012/141] dev --- cf/data/fragment/abstract/fragmentarray.py | 234 +++++++++++---------- 1 file changed, 122 insertions(+), 112 deletions(-) diff --git a/cf/data/fragment/abstract/fragmentarray.py b/cf/data/fragment/abstract/fragmentarray.py index a417aded50..daa6ba361a 100644 --- a/cf/data/fragment/abstract/fragmentarray.py +++ b/cf/data/fragment/abstract/fragmentarray.py @@ -119,80 +119,82 @@ def __getitem__(self, indices): .. versionadded:: 3.14.0 - """ - return self._getitem( self.get_array(), indices) - -# indices = self._parse_indices(indices) -# array = self.get_array() -# -# try: -# array = array[indices] -# except ValueError: -# # A value error is raised if indices has at least ndim -# # elements but the fragment variable has fewer than ndim -# # dimensions. In this case we get the entire fragment -# # array, insert the missing size 1 dimensions, and then -# # apply the requested slice. See the CFA conventions for -# # details. -# array = array[Ellipsis] -# if array.ndim < self.ndim: -# array = array.reshape(self.shape) -# -# array = array[indices] -# -# array = self._conform_units(array) -# return array - - def _getitem(self, array, indices): - """Returns a subspace of the fragment as a numpy array. - - x.__getitem__(indices) <==> x[indices] - - Indexing is similar to numpy indexing, with the following - differences: - - * A dimension's index can't be rank-reducing, i.e. it can't - be an integer, a scalar `numpy` array, nor a scalar `dask` - array. - - * When two or more dimension's indices are sequences of - integers then these indices work independently along each - dimension (similar to the way vector subscripts work in - Fortran). - - **Performance** - - If the fragment variable has fewer than `ndim` dimensions then - the entire array is realised in memory before the requested - subspace of it is returned. - - .. versionadded:: 3.14.0 - """ indices = self._parse_indices(indices) try: - array = array[indices] + array = super().__getitem__[indices] except ValueError: - # A value error is raised if indices has at least ndim - # elements but the fragment variable has fewer than ndim - # dimensions. In this case we get the entire fragment - # array, insert the missing size 1 dimensions, and then - # apply the requested slice. See the CFA conventions for - # details. - array = array[Ellipsis] - if array.ndim < self.ndim: - array = array.reshape(self.shape) + # A ValueError is raised if 'indices' has ndim elements + # but the fragment variable has fewer than ndim + # dimensions. See the CFA conventions for details. + iaxis = self._missing_size_1_axis(indices) + if iaxis is not None: + # There is a unique size 1 index: Remove it from the + # indices; get the fragment array with the new + # indices; and then insert the missing size one + # dimension. + indices = list(indices) + indices.pop(iaxis) + array = super().__getitem__[tuple(indices)] + array = np.expand_dims(array, iaxis) + else: + # There are multiple size 1 indices: Get the full + # fragment array; and then reshape it to the shape of + # the storage chunk. + array = super().__getitem__[Ellipsis] + if array.size != self.size: + raise ValueError( + "Can't get CFA fragment data from " + f"{self.get_filename()} ({self.get_address()}) when " + "the fragment has two or more missing size 1 " + "dimensions and spans two or more storage chunks" + "\n\n" + "Consider recreating the data with one storage chunk " + "per fragment." + ) - array = array[indices] + array = array.reshape(self.shape) array = self._conform_units(array) return array + def _missing_size_1_axis(self, indices): + """TODOCFADOCS. + + .. versionadded:: TODOCFAVER + + """ + iaxis = None + + n = 0 + for i, index in enumerate(indices): + try: + if index.stop - index.start == 1: + # Index is a slice + n += 1 + iaxis = i + except AttributeError: + try: + if index.size == 1: + # Index is a numpy or dask array + n += 1 + iaxis = i + except AttributeError: + if len(index) == 1: + # Index is a list + n += 1 + iaxis = i + + if n > 1: + iaxis = None + + return iaxis + def _parse_indices(self, indices): """Parse the indices that retrieve the fragment data. - Ellipses are replaced with the approriate number `slice(None)` + Ellipses are replaced with the approriate number `slice` instances, and rank-reducing indices (such as an integer or scalar array) are disallowed. @@ -213,39 +215,43 @@ def _parse_indices(self, indices): >>> a.shape (12, 1, 73, 144) >>> a._parse_indices(([2, 4, 5], Ellipsis, slice(45, 67)) - ([2, 4, 5], slice(None), slice(None), slice(45, 67)) + ([2, 4, 5], slice(0, 1), slice(0, 73), slice(45, 67)) """ - ndim = self.ndim - if indices is Ellipsis: - return (slice(None),) * ndim + shape = self.shape + if indices is Ellipsis: + return tuple([slice(0, n) for n in shape]) # Check indices has_ellipsis = False - for i in indices: - if isinstance(i, slice): + indices = list(indices) + for i, (index, n) in enumerate(zip(indices, shape)): + if isinstance(index, slice): + if index == slice(None): + indices[i] = slice(0, n) + continue - if i is Ellipsis: + if index is Ellipsis: has_ellipsis = True continue - if isinstance(i, Integral) or not getattr(i, "ndim", True): + if isinstance(index, Integral) or not getattr(index, "ndim", True): # TODOCFA: what about [] or np.array([])? - # 'i' is an integer or a scalar numpy/dask array + # 'index' is an integer or a scalar numpy/dask array raise ValueError( f"Can't subspace {self.__class__.__name__} with a " - f"rank-reducing index: {i!r}" + f"rank-reducing index: {index!r}" ) if has_ellipsis: - # Replace Ellipsis with one or more slice(None) + # Replace Ellipsis with one or more slices indices2 = [] length = len(indices) - n = ndim - for i in indices: - if i is Ellipsis: + n = self.ndim + for index in indices: + if index is Ellipsis: m = n - length + 1 indices2.extend([slice(None)] * m) n -= m @@ -255,9 +261,13 @@ def _parse_indices(self, indices): length -= 1 - indices = tuple(indices2) - - return indices + indices = indices2 + + for i, (index, n) in enumerate(zip(indices, shape)): + if index == slice(None): + indices[i] = slice(0, n) + + return tuple(indices) def _conform_units(self, array): """Conform the array to have the aggregated units. @@ -393,37 +403,37 @@ def get_aggregated_units(self, default=ValueError()): return units - def get_array(self): - """The fragment array. - - .. versionadded:: 3.14.0 - - :Returns: - - Subclass of `Array` - The object defining the fragment array. - - """ - return self._get_component("array") - - def get_units(self, default=ValueError()): - """The units of the netCDF variable. - - .. versionadded:: (cfdm) 1.10.0.1 - - .. seealso:: `get_calendar` - - :Parameters: - - default: optional - Return the value of the *default* parameter if the - units have not been set. If set to an `Exception` - instance then it will be raised instead. - - :Returns: - - `str` or `None` - The units value. - - """ - return self.get_array().get_units(default) +# def get_array(self): +# """The fragment array. +# +# .. versionadded:: 3.14.0 +# +# :Returns: +# +# Subclass of `Array` +# The object defining the fragment array. +# +# """ +# return self._get_component("array") +# +# def get_units(self, default=ValueError()): +# """The units of the netCDF variable. +# +# .. versionadded:: (cfdm) 1.10.0.1 +# +# .. seealso:: `get_calendar` +# +# :Parameters: +# +# default: optional +# Return the value of the *default* parameter if the +# units have not been set. If set to an `Exception` +# instance then it will be raised instead. +# +# :Returns: +# +# `str` or `None` +# The units value. +# +# """ +# return self.get_array().get_units(default) From c793c8280021182b1fa7aec68059515e707f8e8f Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 13 Feb 2023 16:00:33 +0000 Subject: [PATCH 013/141] dev --- cf/data/creation.py | 11 +- cf/data/data.py | 26 +- cf/data/fragment/abstract/__init__.py | 2 +- cf/data/fragment/abstract/fragmentarray.py | 839 +++++++++++---------- cf/data/fragment/netcdffragmentarray.py | 7 +- cf/data/fragment/umfragmentarray.py | 5 +- cf/read_write/netcdf/netcdfread.py | 23 +- 7 files changed, 469 insertions(+), 444 deletions(-) diff --git a/cf/data/creation.py b/cf/data/creation.py index 49d7354152..66972965ea 100644 --- a/cf/data/creation.py +++ b/cf/data/creation.py @@ -3,7 +3,6 @@ import dask.array as da import numpy as np - from cfdm import Array from dask.base import is_dask_collection @@ -124,11 +123,11 @@ def generate_axis_identifiers(n): def is_file_array(array): """Whether or not an array is stored on disk. - + .. versionaddedd: TODOCFAVER - + :Parameters: - + array: TODOCFADOCS @@ -145,9 +144,9 @@ def is_abstract_Array_subclass(array): """Whether or not an array is a type of abstract Array. .. versionaddedd: TODOCFAVER - + :Parameters: - + array: TODOCFADOCS diff --git a/cf/data/data.py b/cf/data/data.py index 5e348d0ff0..911f478752 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -38,7 +38,12 @@ from ..mixin_container import Container from ..units import Units from .collapse import Collapse -from .creation import generate_axis_identifiers, to_dask, is_file_array, is_abstract_Array_subclass +from .creation import ( + generate_axis_identifiers, + is_abstract_Array_subclass, + is_file_array, + to_dask, +) from .dask_utils import ( _da_ma_allclose, cf_contains, @@ -411,17 +416,17 @@ def __init__( # Still here? Then create a dask array and store it. + # Find out if the input data is compressed by convention try: compressed = array.get_compression_type() except AttributeError: - pass - else: - # The input data is compressed by convention - if init_options.get("from_array"): - raise ValueError( - "Can't define 'from_array' initialisation options " - "for compressed input arrays" - ) + compressed = "" + + if compressed and init_options.get("from_array"): + raise ValueError( + "Can't define 'from_array' initialisation options " + "for compressed input arrays" + ) if is_file_array(array): if to_memory: @@ -429,9 +434,6 @@ def __init__( array = array.to_memory() except AttributeError: pass - else: - # Allow the possibilty of CFA writing - self._set_cfa_write(True) if is_abstract_Array_subclass(array): # Save the input array in case it's useful later. For diff --git a/cf/data/fragment/abstract/__init__.py b/cf/data/fragment/abstract/__init__.py index e91f73b8de..b771e745ab 100644 --- a/cf/data/fragment/abstract/__init__.py +++ b/cf/data/fragment/abstract/__init__.py @@ -1 +1 @@ -from .fragmentarray import FragmentArray +# from .fragmentarray import FragmentArray diff --git a/cf/data/fragment/abstract/fragmentarray.py b/cf/data/fragment/abstract/fragmentarray.py index daa6ba361a..c1c68a5225 100644 --- a/cf/data/fragment/abstract/fragmentarray.py +++ b/cf/data/fragment/abstract/fragmentarray.py @@ -1,408 +1,437 @@ -from numbers import Integral - -from ....units import Units - - -class FragmentArray: - """Abstract base class for a CFA fragment array. - - .. versionadded:: 3.14.0 - - """ - - def __init__( - self, - dtype=None, - shape=None, - aggregated_units=False, - aggregated_calendar=None, - array=None, - source=None, - copy=True, - ): - """**Initialisation** - - :Parameters: - - dtype: `numpy.dtype` - The data type of the aggregated array. May be `None` - if the numpy data-type is not known (which can be the - case for netCDF string types, for example). This may - differ from the data type of the netCDF fragment - variable. - - shape: `tuple` - The shape of the fragment within the aggregated - array. This may differ from the shape of the netCDF - fragment variable in that the latter may have fewer - size 1 dimensions. - - {{aggregated_units: `str` or `None`, optional}} - - {{aggregated_calendar: `str` or `None`, optional}} - - array: `Array` - The fragment array stored in a file. - - source: optional - Initialise the array from the given object. - - {{init source}} - - {{deep copy}} - - """ - super().__init__(source=source, copy=copy) - - if source is not None: - try: - dtype = source._get_component("dtype", None) - except AttributeError: - dtype = None - - try: - shape = source._get_component("shape", None) - except AttributeError: - shape = None - - try: - aggregated_units = source._get_component( - "aggregated_units", False - ) - except AttributeError: - aggregated_units = False - - try: - aggregated_calendar = source._get_component( - "aggregated_calendar", False - ) - except AttributeError: - aggregated_calendar = False - - try: - array = source._get_component("array", None) - except AttributeError: - array = None - - self._set_component("dtype", dtype, copy=False) - self._set_component("shape", shape, copy=False) - self._set_component("aggregated_units", aggregated_units, copy=False) - self._set_component( - "aggregated_calendar", aggregated_calendar, copy=False - ) - - if array is not None: - self._set_component("array", array, copy=copy) - - def __getitem__(self, indices): - """Returns a subspace of the fragment as a numpy array. - - x.__getitem__(indices) <==> x[indices] - - Indexing is similar to numpy indexing, with the following - differences: - - * A dimension's index can't be rank-reducing, i.e. it can't - be an integer, a scalar `numpy` array, nor a scalar `dask` - array. - - * When two or more dimension's indices are sequences of - integers then these indices work independently along each - dimension (similar to the way vector subscripts work in - Fortran). - - **Performance** - - If the fragment variable has fewer than `ndim` dimensions then - the entire array is realised in memory before the requested - subspace of it is returned. - - .. versionadded:: 3.14.0 - - """ - indices = self._parse_indices(indices) - - try: - array = super().__getitem__[indices] - except ValueError: - # A ValueError is raised if 'indices' has ndim elements - # but the fragment variable has fewer than ndim - # dimensions. See the CFA conventions for details. - iaxis = self._missing_size_1_axis(indices) - if iaxis is not None: - # There is a unique size 1 index: Remove it from the - # indices; get the fragment array with the new - # indices; and then insert the missing size one - # dimension. - indices = list(indices) - indices.pop(iaxis) - array = super().__getitem__[tuple(indices)] - array = np.expand_dims(array, iaxis) - else: - # There are multiple size 1 indices: Get the full - # fragment array; and then reshape it to the shape of - # the storage chunk. - array = super().__getitem__[Ellipsis] - if array.size != self.size: - raise ValueError( - "Can't get CFA fragment data from " - f"{self.get_filename()} ({self.get_address()}) when " - "the fragment has two or more missing size 1 " - "dimensions and spans two or more storage chunks" - "\n\n" - "Consider recreating the data with one storage chunk " - "per fragment." - ) - - array = array.reshape(self.shape) - - array = self._conform_units(array) - return array - - def _missing_size_1_axis(self, indices): - """TODOCFADOCS. - - .. versionadded:: TODOCFAVER - - """ - iaxis = None - - n = 0 - for i, index in enumerate(indices): - try: - if index.stop - index.start == 1: - # Index is a slice - n += 1 - iaxis = i - except AttributeError: - try: - if index.size == 1: - # Index is a numpy or dask array - n += 1 - iaxis = i - except AttributeError: - if len(index) == 1: - # Index is a list - n += 1 - iaxis = i - - if n > 1: - iaxis = None - - return iaxis - - def _parse_indices(self, indices): - """Parse the indices that retrieve the fragment data. - - Ellipses are replaced with the approriate number `slice` - instances, and rank-reducing indices (such as an integer or - scalar array) are disallowed. - - .. versionadded:: 3.14.0 - - :Parameters: - - indices: `tuple` or `Ellipsis` - The array indices to be parsed. - - :Returns: - - `tuple` - The parsed indices. - - **Examples** - - >>> a.shape - (12, 1, 73, 144) - >>> a._parse_indices(([2, 4, 5], Ellipsis, slice(45, 67)) - ([2, 4, 5], slice(0, 1), slice(0, 73), slice(45, 67)) - - """ - shape = self.shape - if indices is Ellipsis: - return tuple([slice(0, n) for n in shape]) - - # Check indices - has_ellipsis = False - indices = list(indices) - for i, (index, n) in enumerate(zip(indices, shape)): - if isinstance(index, slice): - if index == slice(None): - indices[i] = slice(0, n) - - continue - - if index is Ellipsis: - has_ellipsis = True - continue - - if isinstance(index, Integral) or not getattr(index, "ndim", True): - # TODOCFA: what about [] or np.array([])? - - # 'index' is an integer or a scalar numpy/dask array - raise ValueError( - f"Can't subspace {self.__class__.__name__} with a " - f"rank-reducing index: {index!r}" - ) - - if has_ellipsis: - # Replace Ellipsis with one or more slices - indices2 = [] - length = len(indices) - n = self.ndim - for index in indices: - if index is Ellipsis: - m = n - length + 1 - indices2.extend([slice(None)] * m) - n -= m - else: - indices2.append(i) - n -= 1 - - length -= 1 - - indices = indices2 - - for i, (index, n) in enumerate(zip(indices, shape)): - if index == slice(None): - indices[i] = slice(0, n) - - return tuple(indices) - - def _conform_units(self, array): - """Conform the array to have the aggregated units. - - .. versionadded:: 3.14.0 - - :Parameters: - - array: `numpy.ndarray` or `dict` - The array to be conformed. If *array* is a `dict` with - `numpy` array values then each value is conformed. - - :Returns: - - `numpy.ndarray` or `dict` - The conformed array. The returned array may or may not - be the input array updated in-place, depending on its - data type and the nature of its units and the - aggregated units. - - If *array* is a `dict` then a dictionary of conformed - arrays is returned. - - """ - units = self.Units - if units: - aggregated_units = self.aggregated_Units - if not units.equivalent(aggregated_units): - raise ValueError( - f"Can't convert fragment data with units {units!r} to " - f"have aggregated units {aggregated_units!r}" - ) - - if units != aggregated_units: - if isinstance(array, dict): - # 'array' is a dictionary - for key, value in array.items(): - if key in _active_chunk_methds: - array[key] = Units.conform( - value, units, aggregated_units, inplace=True - ) - else: - # 'array' is a numpy array - array = Units.conform( - array, units, aggregated_units, inplace=True - ) - - return array - - @property - def aggregated_Units(self): - """The units of the aggregated data. - - .. versionadded:: 3.14.0 - - :Returns: - - `Units` - The units of the aggregated data. - - """ - return Units( - self.get_aggregated_units(), self.get_aggregated_calendar(None) - ) - - def get_aggregated_calendar(self, default=ValueError()): - """The calendar of the aggregated array. - - If the calendar is `None` then the CF default calendar is - assumed, if applicable. - - .. versionadded:: 3.14.0 - - :Parameters: - - default: optional - Return the value of the *default* parameter if the - calendar has not been set. If set to an `Exception` - instance then it will be raised instead. - - :Returns: - - `str` or `None` - The calendar value. - - """ - calendar = self._get_component("aggregated_calendar", False) - if calendar is False: - if default is None: - return - - return self._default( - default, - f"{self.__class__.__name__} 'aggregated_calendar' has not " - "been set", - ) - - return calendar - - def get_aggregated_units(self, default=ValueError()): - """The units of the aggregated array. - - If the units are `None` then the aggregated array has no - defined units. - - .. versionadded:: 3.14.0 - - .. seealso:: `get_aggregated_calendar` - - :Parameters: - - default: optional - Return the value of the *default* parameter if the - units have not been set. If set to an `Exception` - instance then it will be raised instead. - - :Returns: - - `str` or `None` - The units value. - - """ - units = self._get_component("aggregated_units", False) - if units is False: - if default is None: - return - - return self._default( - default, - f"{self.__class__.__name__} 'aggregated_units' have not " - "been set", - ) - - return units - +# from numbers import Integral +# +# from ....units import Units +# +# +# class FragmentArrayMixin: +# """Mixin for a CFA fragment array. +# +# .. versionadded:: 3.14.0 +# +# """ +# +# +# def __init__( +# self, +# dtype=None, +# shape=None, +# aggregated_units=False, +# aggregated_calendar=None, +# array=None, +# source=None, +# copy=True, +# ): +# """**Initialisation** +# +# :Parameters: +# +# dtype: `numpy.dtype` +# The data type of the aggregated array. May be `None` +# if the numpy data-type is not known (which can be the +# case for netCDF string types, for example). This may +# differ from the data type of the netCDF fragment +# variable. +# +# shape: `tuple` +# The shape of the fragment within the aggregated +# array. This may differ from the shape of the netCDF +# fragment variable in that the latter may have fewer +# size 1 dimensions. +# +# {{aggregated_units: `str` or `None`, optional}} +# +# {{aggregated_calendar: `str` or `None`, optional}} +# +# array: `Array` +# The fragment array stored in a file. +# +# source: optional +# Initialise the array from the given object. +# +# {{init source}} +# +# {{deep copy}} +# +# """ +# super().__init__(source=source, copy=copy) +# +# if source is not None: +# try: +# dtype = source._get_component("dtype", None) +# except AttributeError: +# dtype = None +# +# try: +# shape = source._get_component("shape", None) +# except AttributeError: +# shape = None +# +# try: +# aggregated_units = source._get_component( +# "aggregated_units", False +# ) +# except AttributeError: +# aggregated_units = False +# +# try: +# aggregated_calendar = source._get_component( +# "aggregated_calendar", False +# ) +# except AttributeError: +# aggregated_calendar = False +# +# try: +# array = source._get_component("array", None) +# except AttributeError: +# array = None +# +# self._set_component("dtype", dtype, copy=False) +# self._set_component("shape", shape, copy=False) +# self._set_component("aggregated_units", aggregated_units, copy=False) +# self._set_component( +# "aggregated_calendar", aggregated_calendar, copy=False +# ) +# +# if array is not None: +# self._set_component("array", array, copy=copy) +# +# def __getitem__(self, indices): +# """Returns a subspace of the fragment as a numpy array. +# +# x.__getitem__(indices) <==> x[indices] +# +# Indexing is similar to numpy indexing, with the following +# differences: +# +# * A dimension's index can't be rank-reducing, i.e. it can't +# be an integer, a scalar `numpy` array, nor a scalar `dask` +# array. +# +# * When two or more dimension's indices are sequences of +# integers then these indices work independently along each +# dimension (similar to the way vector subscripts work in +# Fortran). +# +# .. versionadded:: TODOCFAVER +# +# """ +# indices = self._parse_indices(indices) +# +# try: +# array = super().__getitem__(indices) +# except ValueError: +# # A ValueError is expected to be raised when the fragment +# # variable has fewer than 'self.ndim' dimensions (given +# # that 'indices' now has 'self.ndim' elements). +# axis = self._missing_size_1_axis(indices) +# if axis is not None: +# # There is a unique size 1 index, that must correspond +# # to the missing dimension => Remove it from the +# # indices, get the fragment array with the new +# # indices; and then insert the missing size one +# # dimension. +# indices = list(indices) +# indices.pop(axis) +# array = super().__getitem__(tuple(indices)) +# array = np.expand_dims(array, axis) +# else: +# # There are multiple size 1 indices, so we don't know +# # how many missing dimensions there are nor their +# # positions => Get the full fragment array; and then +# # reshape it to the shape of the storage chunk. +# array = super().__getitem__(Ellipsis) +# if array.size != self.size: +# raise ValueError( +# "Can't get CFA fragment data from " +# f"{self.get_filename()} ({self.get_address()}) when " +# "the fragment has two or more missing size 1 " +# "dimensions whilst also spanning two or more " +# "storage chunks" +# "\n\n" +# "Consider recreating the data with exactly one" +# "storage chunk per fragment." +# ) +# +# array = array.reshape(self.shape) +# +# array = self._conform_units(array) +# return array +# +# def _missing_size_1_axis(self, indices): +# """Find the position of a unique size 1 index. +# +# .. versionadded:: TODOCFAVER +# +# .. seealso:: `_parse_indices` +# +# :Parameters: +# +# indices: `tuple` +# The array indices to be parsed, as returned by +# `_parse_indices`. +# +# :Returns: +# +# `int` or `None` +# The position of the unique size 1 index, or `None` if +# there isn't one. +# +# **Examples** +# +# >>> a._missing_size_1_axis(([2, 4, 5], slice(0, 1), slice(0, 73))) +# 1 +# >>> a._missing_size_1_axis(([2, 4, 5], slice(3, 4), slice(0, 73))) +# 1 +# >>> a._missing_size_1_axis(([2, 4, 5], [0], slice(0, 73))) +# 1 +# >>> a._missing_size_1_axis(([2, 4, 5], slice(0, 144), slice(0, 73))) +# None +# >>> a._missing_size_1_axis(([2, 4, 5], slice(3, 7), [0, 1])) +# None +# >>> a._missing_size_1_axis(([2, 4, 5], slice(0, 1), [0])) +# None +# +# """ +# axis = None # Position of unique size 1 index +# +# n = 0 # Number of size 1 indices +# for i, index in enumerate(indices): +# try: +# if index.stop - index.start == 1: +# # Index is a size 1 slice +# n += 1 +# axis = i +# except AttributeError: +# try: +# if index.size == 1: +# # Index is a size 1 numpy or dask array +# n += 1 +# axis = i +# except AttributeError: +# if len(index) == 1: +# # Index is a size 1 list +# n += 1 +# axis = i +# +# if n > 1: +# # There are two or more size 1 indices +# axis = None +# +# return axis +# +# def _parse_indices(self, indices): +# """Parse the indices that retrieve the fragment data. +# +# Ellipses are replaced with the approriate number `slice` +# instances, and rank-reducing indices (such as an integer or +# scalar array) are disallowed. +# +# .. versionadded:: 3.14.0 +# +# :Parameters: +# +# indices: `tuple` or `Ellipsis` +# The array indices to be parsed. +# +# :Returns: +# +# `tuple` +# The parsed indices. +# +# **Examples** +# +# >>> a.shape +# (12, 1, 73, 144) +# >>> a._parse_indices(([2, 4, 5], Ellipsis, slice(45, 67)) +# ([2, 4, 5], slice(0, 1), slice(0, 73), slice(45, 67)) +# +# """ +# shape = self.shape +# if indices is Ellipsis: +# return tuple([slice(0, n) for n in shape]) +# +# # Check indices +# has_ellipsis = False +# indices = list(indices) +# for i, (index, n) in enumerate(zip(indices, shape)): +# if isinstance(index, slice): +# if index == slice(None): +# indices[i] = slice(0, n) +# +# continue +# +# if index is Ellipsis: +# has_ellipsis = True +# continue +# +# if isinstance(index, Integral) or not getattr(index, "ndim", True): +# # TODOCFA: what about [] or np.array([])? +# +# # 'index' is an integer or a scalar numpy/dask array +# raise ValueError( +# f"Can't subspace {self.__class__.__name__} with a " +# f"rank-reducing index: {index!r}" +# ) +# +# if has_ellipsis: +# # Replace Ellipsis with one or more slices +# indices2 = [] +# length = len(indices) +# n = self.ndim +# for index in indices: +# if index is Ellipsis: +# m = n - length + 1 +# indices2.extend([slice(None)] * m) +# n -= m +# else: +# indices2.append(i) +# n -= 1 +# +# length -= 1 +# +# indices = indices2 +# +# for i, (index, n) in enumerate(zip(indices, shape)): +# if index == slice(None): +# indices[i] = slice(0, n) +# +# return tuple(indices) +# +# def _conform_units(self, array): +# """Conform the array to have the aggregated units. +# +# .. versionadded:: 3.14.0 +# +# :Parameters: +# +# array: `numpy.ndarray` or `dict` +# The array to be conformed. If *array* is a `dict` with +# `numpy` array values then each value is conformed. +# +# :Returns: +# +# `numpy.ndarray` or `dict` +# The conformed array. The returned array may or may not +# be the input array updated in-place, depending on its +# data type and the nature of its units and the +# aggregated units. +# +# If *array* is a `dict` then a dictionary of conformed +# arrays is returned. +# +# """ +# units = self.Units +# if units: +# aggregated_units = self.aggregated_Units +# if not units.equivalent(aggregated_units): +# raise ValueError( +# f"Can't convert fragment data with units {units!r} to " +# f"have aggregated units {aggregated_units!r}" +# ) +# +# if units != aggregated_units: +# if isinstance(array, dict): +# # 'array' is a dictionary +# for key, value in array.items(): +# if key in _active_chunk_methds: +# array[key] = Units.conform( +# value, units, aggregated_units, inplace=True +# ) +# else: +# # 'array' is a numpy array +# array = Units.conform( +# array, units, aggregated_units, inplace=True +# ) +# +# return array +# +# @property +# def aggregated_Units(self): +# """The units of the aggregated data. +# +# .. versionadded:: 3.14.0 +# +# :Returns: +# +# `Units` +# The units of the aggregated data. +# +# """ +# return Units( +# self.get_aggregated_units(), self.get_aggregated_calendar(None) +# ) +# +# def get_aggregated_calendar(self, default=ValueError()): +# """The calendar of the aggregated array. +# +# If the calendar is `None` then the CF default calendar is +# assumed, if applicable. +# +# .. versionadded:: 3.14.0 +# +# :Parameters: +# +# default: optional +# Return the value of the *default* parameter if the +# calendar has not been set. If set to an `Exception` +# instance then it will be raised instead. +# +# :Returns: +# +# `str` or `None` +# The calendar value. +# +# """ +# calendar = self._get_component("aggregated_calendar", False) +# if calendar is False: +# if default is None: +# return +# +# return self._default( +# default, +# f"{self.__class__.__name__} 'aggregated_calendar' has not " +# "been set", +# ) +# +# return calendar +# +# def get_aggregated_units(self, default=ValueError()): +# """The units of the aggregated array. +# +# If the units are `None` then the aggregated array has no +# defined units. +# +# .. versionadded:: 3.14.0 +# +# .. seealso:: `get_aggregated_calendar` +# +# :Parameters: +# +# default: optional +# Return the value of the *default* parameter if the +# units have not been set. If set to an `Exception` +# instance then it will be raised instead. +# +# :Returns: +# +# `str` or `None` +# The units value. +# +# """ +# units = self._get_component("aggregated_units", False) +# if units is False: +# if default is None: +# return +# +# return self._default( +# default, +# f"{self.__class__.__name__} 'aggregated_units' have not " +# "been set", +# ) +# +# return units +# +# # def get_array(self): # """The fragment array. # diff --git a/cf/data/fragment/netcdffragmentarray.py b/cf/data/fragment/netcdffragmentarray.py index 46a1e5a1b9..519e738594 100644 --- a/cf/data/fragment/netcdffragmentarray.py +++ b/cf/data/fragment/netcdffragmentarray.py @@ -1,9 +1,10 @@ from ..array.netcdfarray import NetCDFArray -from .abstract import FragmentArray -from .mixin import FragmentFileArrayMixin +# from .abstract import FragmentArray +from .mixin import FragmentArrayMixin -class NetCDFFragmentArray(FragmentFileArrayMixin, FragmentArray): + +class NetCDFFragmentArray(FragmentArrayMixin, NetCDFArray): """A CFA fragment array stored in a netCDF file. .. versionadded:: 3.14.0 diff --git a/cf/data/fragment/umfragmentarray.py b/cf/data/fragment/umfragmentarray.py index cc957669e8..87536f8671 100644 --- a/cf/data/fragment/umfragmentarray.py +++ b/cf/data/fragment/umfragmentarray.py @@ -1,9 +1,8 @@ from ..array.umarray import UMArray -from .abstract import FragmentArray -from .mixin import FragmentFileArrayMixin +from .mixin import FragmentArrayMixin -class UMFragmentArray(FragmentFileArrayMixin, FragmentArray): +class UMFragmentArray(FragmentArrayMixin, UMArray): """A CFA fragment array stored in a UM or PP file. .. versionadded:: 3.14.0 diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index d3f6f7945b..3681bd1657 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -3,10 +3,6 @@ from packaging.version import Version """ -TODOCFA: remove aggregation_* properties from constructs - -TODOCFA: Create auxiliary coordinates from non-standardised terms - TODOCFA: What about groups/netcdf_flattener? """ @@ -207,13 +203,13 @@ def _create_data( coord_ncvar=coord_ncvar, ) - self._cache_data_elements(data, ncvar) - if data.npartitions == 1: # Set the CFA write status to True when there is # exactly one dask chunk data._set_cfa_write(True) + self._cache_data_elements(data, ncvar) + return data # ------------------------------------------------------------ @@ -225,6 +221,8 @@ def _create_data( for attr in ("aggregation_dimensions", "aggregation_data"): self.implementation.del_property(construct, attr, None) + # get shape from parent_ncvar + cfa_array, kwargs = self._create_cfanetcdfarray( ncvar, unpacked_dtype=unpacked_dtype, @@ -239,11 +237,8 @@ def _create_data( calendar=kwargs["calendar"], ) - if cfa_term is not None: - self._cache_data_elements(data, ncvar) - - # Set the CFA write status to True - data._set_cfa_write(True) + # if cfa_term is not None or (data.numblocks == 1 for each non-aggreged dimension): + # data._set_cfa_write(True) return data @@ -484,7 +479,7 @@ def _create_cfanetcdfarray( ncvar, unpacked_dtype=False, coord_ncvar=None, - term=None, + non_standard_term=None, ): """Create a CFA-netCDF variable array. @@ -500,7 +495,7 @@ def _create_cfanetcdfarray( coord_ncvar: `str`, optional - term: `str`, optional + non_standard_term: `str`, optional The name of a non-standard aggregation instruction term from which to create the array. If set then *ncvar* must be the value of the term in the @@ -529,7 +524,7 @@ def _create_cfanetcdfarray( # Specify a non-standardised term from which to create the # data - kwargs["term"] = term + kwargs["term"] = non_standard_term # Add the aggregated_data attribute (that can be used by # dask.base.tokenize). From c16585d56f6cc0bc40e902833867fc5046a75d63 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 13 Feb 2023 19:10:26 +0000 Subject: [PATCH 014/141] dev --- cf/cfimplementation.py | 4 + cf/data/array/cfanetcdfarray.py | 1 + cf/data/fragment/netcdffragmentarray.py | 109 +++++------------------- cf/read_write/netcdf/netcdfread.py | 65 +++++++++++--- 4 files changed, 77 insertions(+), 102 deletions(-) diff --git a/cf/cfimplementation.py b/cf/cfimplementation.py index e273ef22c0..78cf57eda2 100644 --- a/cf/cfimplementation.py +++ b/cf/cfimplementation.py @@ -86,6 +86,7 @@ def initialise_CFANetCDFArray( ncvar=None, group=None, dtype=None, + shape=None, mask=True, units=False, calendar=False, @@ -104,6 +105,8 @@ def initialise_CFANetCDFArray( dytpe: `numpy.dtype` + shape: `tuple` + mask: `bool`, optional units: `str` or `None`, optional @@ -125,6 +128,7 @@ def initialise_CFANetCDFArray( ncvar=ncvar, group=group, dtype=dtype, + shape=shape, mask=mask, units=units, calendar=calendar, diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index ddf7d94f8d..9c5a4d79fd 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -169,6 +169,7 @@ def __init__( f"CFA variable {ncvar} not found in file {filename}" ) +# if term is None: shape = tuple([d.len for d in var.getDims()]) super().__init__( diff --git a/cf/data/fragment/netcdffragmentarray.py b/cf/data/fragment/netcdffragmentarray.py index 519e738594..c261e77a98 100644 --- a/cf/data/fragment/netcdffragmentarray.py +++ b/cf/data/fragment/netcdffragmentarray.py @@ -1,6 +1,4 @@ from ..array.netcdfarray import NetCDFArray - -# from .abstract import FragmentArray from .mixin import FragmentArrayMixin @@ -69,21 +67,7 @@ def __init__( {{init copy: `bool`, optional}} """ - if source is not None: - super().__init__(source=source, copy=copy) - return - - if isinstance(address, int): - ncvar = None - varid = address - else: - ncvar = address - varid = None - - # TODOCFA set groups from ncvar - group = None - - array = NetCDFArray( + super().__init__( filename=filename, ncvar=ncvar, varid=varid, @@ -93,79 +77,28 @@ def __init__( mask=True, units=units, calendar=calendar, + source=source, copy=False, ) - super().__init__( - dtype=dtype, - shape=shape, - aggregated_units=aggregated_units, - aggregated_calendar=aggregated_calendar, - array=array, - source=source, - copy=False, + if source is not None: + try: + aggregated_units = source._get_component( + "aggregated_units", False + ) + except AttributeError: + aggregated_units = False + + try: + aggregated_calendar = source._get_component( + "aggregated_calendar", False + ) + except AttributeError: + aggregated_calendar = False + + + self._set_component("aggregated_units", aggregated_units, copy=False) + self._set_component( + "aggregated_calendar", aggregated_calendar, copy=False ) - -# def __getitem__(self, indices): -# """Returns a subspace of the fragment as a numpy array. -# -# x.__getitem__(indices) <==> x[indices] -# -# Indexing is similar to numpy indexing, with the following -# differences: -# -# * A dimension's index can't be rank-reducing, i.e. it can't -# be an integer, nor a scalar `numpy` or `dask` array. -# -# * When two or more dimension's indices are sequences of -# integers then these indices work independently along each -# dimension (similar to the way vector subscripts work in -# Fortran). -# -# **Performance** -# -# If the netCDF fragment variable has fewer than `ndim` -# dimensions then the entire array is read into memory before -# the requested subspace of it is returned. -# -# .. versionadded:: 3.14.0 -# -# """ -# try: -# return super().__getitem__(indices) -# except ValueError: -# # A value error is raised if indices has at least ndim -# # elements but the netCDF fragment variable has fewer than -# # ndim dimensions. In this case we get the entire fragment -# # array, insert the missing size 1 dimensions, and then -# # apply the requested slice. -# indices = self._parse_indices(indices) -# array = self.get_array() -# array = array[Ellipsis] -# if array.ndim < self.ndim: -# array = array.reshape(self.shape) -# -# array = array[indices] -# array = self._conform_units(array) -# return array -# -# indices = self._parse_indices(indices) -# array = self.get_array() -# -# try: -# array = array[indices] -# except ValueError: -# # A value error is raised if indices has at least ndim -# # elements but the netCDF fragment variable has fewer than -# # ndim dimensions. In this case we get the entire fragment -# # array, insert the missing size 1 dimensions, and then -# # apply the requested slice. -# array = array[Ellipsis] -# if array.ndim < self.ndim: -# array = array.reshape(self.shape) -# -# array = array[indices] -# -# array = self._conform_units(array) -# return array diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 3681bd1657..856efd363e 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -221,14 +221,19 @@ def _create_data( for attr in ("aggregation_dimensions", "aggregation_data"): self.implementation.del_property(construct, attr, None) - # get shape from parent_ncvar - - cfa_array, kwargs = self._create_cfanetcdfarray( - ncvar, - unpacked_dtype=unpacked_dtype, - coord_ncvar=coord_ncvar, - term=cfa_term, - ) + if cfa_term is None: + cfa_array, kwargs = self._create_cfanetcdfarray( + ncvar, + unpacked_dtype=unpacked_dtype, + coord_ncvar=coord_ncvar, + ) + else: + cfa_array, kwargs = self._create_cfanetcdfarray( + parent_ncvar, + unpacked_dtype=unpacked_dtype, + coord_ncvar=coord_ncvar, + term=cfa_term + ) data = self._create_Data( cfa_array, @@ -237,6 +242,8 @@ def _create_data( calendar=kwargs["calendar"], ) + # Note: We don't cache elements from aggregated data + # if cfa_term is not None or (data.numblocks == 1 for each non-aggreged dimension): # data._set_cfa_write(True) @@ -479,6 +486,7 @@ def _create_cfanetcdfarray( ncvar, unpacked_dtype=False, coord_ncvar=None, + parent_ncvar=None, non_standard_term=None, ): """Create a CFA-netCDF variable array. @@ -495,6 +503,11 @@ def _create_cfanetcdfarray( coord_ncvar: `str`, optional + parent_shape: `tuple`, optional + TODOCFADOCS. + + .. versionadded:: TODOCFAVER + non_standard_term: `str`, optional The name of a non-standard aggregation instruction term from which to create the array. If set then @@ -519,13 +532,16 @@ def _create_cfanetcdfarray( return_kwargs_only=True, ) - # Get rid of the incorrect shape - kwargs.pop("shape", None) - + # Specify a non-standardised term from which to create the - # data - kwargs["term"] = non_standard_term - + # data, which will have the shape of the parent variable. + if non_standard_term is not None: + kwargs["term"] = non_standard_term + kwargs['shape'] = parent_shape + else: + # Get rid of the incorrect shape + kwargs.pop("shape", None) + # Add the aggregated_data attribute (that can be used by # dask.base.tokenize). kwargs["instructions"] = self.read_vars["variable_attributes"][ @@ -719,3 +735,24 @@ def _customize_auxiliary_coordinates(self, parent_ncvar, f): out[ncvar] = key return out + + def _cfa(self, ncvar, f): + """TODOCFADOCS. + + .. versionadded:: TODOCFAVER + + :Parameters: + + ncvar: `str` + The netCDF variable name. + + f: `Field` or `Domain` + TODOCFADOCS. + + :Returns: + + TODOCFADOCS. + + """ + x = self._parse_x( ncvar, aggregated_data, + keys_are_variables=True) From 801b54941a1a430a19ec41659a6a5a5e2ea75bf7 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 14 Feb 2023 09:43:34 +0000 Subject: [PATCH 015/141] dev --- cf/data/array/cfanetcdfarray.py | 9 ++-- cf/data/data.py | 69 ++++++++++++++------------- cf/data/fragment/fullfragmentarray.py | 53 +++++++++----------- cf/data/fragment/umfragmentarray.py | 35 +++++++++----- 4 files changed, 86 insertions(+), 80 deletions(-) diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index 9c5a4d79fd..6f08198e76 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -4,7 +4,7 @@ from ...functions import abspath from ..fragment import ( FullFragmentArray, - MissingFragmentArray, +# MissingFragmentArray, NetCDFFragmentArray, UMFragmentArray, ) @@ -29,7 +29,7 @@ def __new__(cls, *args, **kwargs): "nc": NetCDFFragmentArray, "um": UMFragmentArray, "full": FullFragmentArray, - None: MissingFragmentArray, +# None: MissingFragmentArray, } return instance @@ -324,16 +324,17 @@ def _set_fragment( fmt = "nc" aggregated_data[frag_loc] = { + "format": fmt, "file": filename, "address": address, - "format": fmt, "location": location, } elif filename is None: # This fragment contains wholly missing values aggregated_data[frag_loc] = { - "format": None, + "format": "full", "location": location, + "full_value": np.ma.masked, } def get_aggregated_data(self, copy=True): diff --git a/cf/data/data.py b/cf/data/data.py index 911f478752..9d751d4645 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1246,17 +1246,17 @@ def _clear_after_dask_update(self, clear=_ALL): .. versionadded:: 3.14.0 - .. seealso:: `_del_Array`, `_del_cached_elements`, - `_del_dask`, `_set_cfa_write`, `_set_dask` + .. seealso:: `_del_Array`, `_set_dask` :Parameters: clear: `int`, optional - Specify which components should be removed. The value - of *clear* is sequentially combined with the - ``_ARRAY``, ``_CACHE`` and ``_CFA`` integer-valued - contants, using the bitwise AND operator, to determine - which components should be removed: + + Specify which components should be removed. Which + components are removed is determined by sequentially + combining *clear* with the ``_ARRAY``, ``_CACHE`` and + ``_CFA`` integer-valued contants, using the bitwise + AND operator: * If ``clear & _ARRAY`` is True then delete a source array. @@ -1273,6 +1273,12 @@ def _clear_after_dask_update(self, clear=_ALL): If *clear* is the ``_NONE`` integer-valued constant then no components are removed. + + To retain a component and remove all others, use + ``_ALL`` with the bitwise OR operator. For instance, + if *clear* is ``_ALL ^ _CACHE`` then the cached + element values will be kept but all other components + removed. .. versionadded:: TODOCFAVER @@ -1313,9 +1319,8 @@ def _set_dask(self, array, copy=False, clear=_ALL): clear: `int`, optional Specify which components should be removed. By default *clear* is the ``_ALL`` integer-valued constant, which - results in all components being removed. - - See `_clear_after_dask_update` for further details. + results in all components being removed. See + `_clear_after_dask_update` for details. :Returns: @@ -1359,16 +1364,17 @@ def _del_dask(self, default=ValueError(), clear=_ALL): default: optional Return the value of the *default* parameter if the - dask array axes has not been set. - - {{default Exception}} + dask array axes has not been set. If set to an + `Exception` instance then it will be raised instead. clear: `int`, optional Specify which components should be removed. By default *clear* is the ``_ALL`` integer-valued constant, which - results in all components being removed. + results in all components being removed. See + `_clear_after_dask_update` for details. - See `_clear_after_dask_update` for further details. + If there is no dask array then no components are + removed, regardless of the value of *clear*. :Returns: @@ -3712,15 +3718,15 @@ def concatenate(cls, data, axis=0, cull_graph=True): dx = da.concatenate(dxs, axis=axis) # Set the CFA write status - cfa = _CFA + CFA = _CFA for d in processed_data: if not d.cfa_write: # Set the CFA write status to False when any input # data instance has False status - cfa = _NONE + CFA = _NONE break - if not cfa: + if not CFA: non_concat_axis_chunks0 = list(processed_data[0].chunks) non_concat_axis_chunks0.pop(axis) for d in processed_data[1:]: @@ -3729,11 +3735,12 @@ def concatenate(cls, data, axis=0, cull_graph=True): if non_concat_axis_chunks != non_concat_axis_chunks0: # Set the CFA write status to False when input # data instances have different chunk patterns for - # the non-concatenation axes - cfa = _NONE + # the non-concatenated axes + CFA = _NONE break - data0._set_dask(dx, clear=_ALL ^ cfa) + # Set the new dask array + data0._set_dask(dx, clear=_ALL ^ CFA) # Manage cyclicity of axes: if join axis was cyclic, it is no longer axis = data0._parse_axes(axis)[0] @@ -4376,7 +4383,7 @@ def Units(self, value): dtype=dtype, ) - # Changing the units does not affect the CFA write status + # Setting equivalent units doesn't affect the CFA write status self._set_dask(dx, clear=_ALL ^ _CFA) self._Units = value @@ -7689,8 +7696,8 @@ def insert_dimension(self, position=0, inplace=False): dx = d.to_dask_array() dx = dx.reshape(shape) - # Inserting a dimension does not affect the cached elements - # nor the CFA write status + # Inserting a dimension doesn't affect the cached elements nor + # the CFA write status d._set_dask(dx, clear=_ALL ^ _CACHE ^ _CFA) # Expand _axes @@ -9058,9 +9065,8 @@ def del_calendar(self, default=ValueError()): default: optional Return the value of the *default* parameter if the - calendar has not been set. - - {{default Exception}} + calendar has not been set. If set to an `Exception` + instance then it will be raised instead. :Returns: @@ -9107,10 +9113,9 @@ def del_units(self, default=ValueError()): :Parameters: default: optional - Return the value of the *default* parameter if the units - has not been set. - - {{default Exception}} + Return the value of the *default* parameter if the + units has not been set. If set to an `Exception` + instance then it will be raised instead. :Returns: @@ -10685,7 +10690,7 @@ def squeeze(self, axes=None, inplace=False, i=False): dx = d.to_dask_array() dx = dx.squeeze(axis=iaxes) - # Squeezing a dimension does not affect the cached elements + # Squeezing a dimension doesn't affect the cached elements d._set_dask(dx, clear=_ALL ^ _CACHE) # Remove the squeezed axes names diff --git a/cf/data/fragment/fullfragmentarray.py b/cf/data/fragment/fullfragmentarray.py index f33b762d18..022efa985f 100644 --- a/cf/data/fragment/fullfragmentarray.py +++ b/cf/data/fragment/fullfragmentarray.py @@ -62,43 +62,34 @@ def __init__( {{init copy: `bool`, optional}} """ - if source is not None: - super().__init__(source=source, copy=copy) - return - - array = FullArray( + super().__init__( fill_value=fill_value, dtype=dtype, shape=shape, units=units, calendar=calendar, + source=source, copy=False, ) - - super().__init__( - dtype=dtype, - shape=shape, - aggregated_units=aggregated_units, - aggregated_calendar=aggregated_calendar, - array=array, - copy=False, + + if source is not None: + try: + aggregated_units = source._get_component( + "aggregated_units", False + ) + except AttributeError: + aggregated_units = False + + try: + aggregated_calendar = source._get_component( + "aggregated_calendar", False + ) + except AttributeError: + aggregated_calendar = False + + + self._set_component("aggregated_units", aggregated_units, copy=False) + self._set_component( + "aggregated_calendar", aggregated_calendar, copy=False ) - def get_full_value(self, default=AttributeError()): - """The fragment array fill value. - - .. versionadded:: TODOCFAVER - - :Parameters: - - default: optional - Return the value of the *default* parameter if the - fill value has not been set. If set to an `Exception` - instance then it will be raised instead. - - :Returns: - - The fill value. - - """ - return self.get_array().get_full_value(default=default) diff --git a/cf/data/fragment/umfragmentarray.py b/cf/data/fragment/umfragmentarray.py index 87536f8671..21db614b2a 100644 --- a/cf/data/fragment/umfragmentarray.py +++ b/cf/data/fragment/umfragmentarray.py @@ -65,26 +65,35 @@ def __init__( {{init copy: `bool`, optional}} """ - if source is not None: - super().__init__(source=source, copy=copy) - return - - array = UMArray( + super().__init__( filename=filename, header_offset=address, dtype=dtype, shape=shape, units=units, calendar=calendar, + source=source, copy=False, ) - super().__init__( - dtype=dtype, - shape=shape, - aggregated_units=aggregated_units, - aggregated_calendar=aggregated_calendar, - array=array, - source=source, - copy=False, + if source is not None: + try: + aggregated_units = source._get_component( + "aggregated_units", False + ) + except AttributeError: + aggregated_units = False + + try: + aggregated_calendar = source._get_component( + "aggregated_calendar", False + ) + except AttributeError: + aggregated_calendar = False + + + self._set_component("aggregated_units", aggregated_units, copy=False) + self._set_component( + "aggregated_calendar", aggregated_calendar, copy=False ) + From e841bc31108df3e5add43ed520b197cdaa2004e2 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 14 Feb 2023 18:30:16 +0000 Subject: [PATCH 016/141] dev --- cf/read_write/netcdf/netcdfread.py | 356 +++++++++++++++++++++++++++++ 1 file changed, 356 insertions(+) diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 856efd363e..313d786159 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -736,6 +736,362 @@ def _customize_auxiliary_coordinates(self, parent_ncvar, f): return out + + def _customize_auxiliary_coordinates(self, parent_ncvar, f): + """Create auxiliary coordinate constructs from CFA terms. + + This method is primarily aimed at providing a customisation + entry point for subclasses. + + This method currently creates: + + * Auxiliary coordinate constructs derived from + non-standardised terms in CFA aggregation instructions. Each + auxiliary coordinate construct spans the same domain axes as + the parent field construct. Auxiliary coordinate constructs + are never created for `Domain` instances. + + .. versionadded:: TODODASKCFA + + :Parameters: + + parent_ncvar: `str` + The netCDF variable name of the parent variable. + + f: `Field` or `Domain` + The parent field or domain construct. + + :Returns: + + `dict` + A mapping of netCDF variable names to newly-created + auxiliary coordinate construct identifiers. + + **Examples** + + >>> n._customize_auxiliary_coordinates('tas', f) + {} + + >>> n._customize_auxiliary_coordinates('pr', f) + {'tracking_id': 'auxiliarycoordinate2'} + + """ + if self.implementation.is_domain(f) or not self._is_cfa_variable( + parent_ncvar + ): + return {} + + # ------------------------------------------------------------ + # Still here? Then we have a CFA-netCDF variable: Loop round + # the aggregation instruction terms and convert each + # non-standard term into an auxiliary coordinate construct + # that spans the same domain axes as the parent field. + # ------------------------------------------------------------ + g = self.read_vars + + out = {} + + attributes = g["variable_attributes"]["parent_ncvar"] + parsed_aggregated_data = self._parse_aggregated_data( + parent_ncvar, attributes.get("aggregated_data") + ) + standardised_terms = ("location", "file", "address", "format") + for x in parsed_aggregated_data: + term, ncvar = tuple(x.items())[0] + if term in standardised_terms: + # Ignore standardised aggregation terms + continue + + # Still here? Then it's a non-standard aggregation term + coord = self.implementation.initialise_AuxiliaryCoordinate() + + properties = g["variable_attributes"][ncvar].copy() + properties.setdefault("long_name", term) + self.implementation.set_properties(coord, properties) + + data = self._create_data( + ncvar, coord, parent_ncvar=parent_ncvar, cfa_term=term + ) + self.implementation.set_data(coord, data, copy=False) + + self.implementation.nc_set_variable(coord, ncvar) + + key = self.implementation.set_auxiliary_coordinate( + f, + coord, + axes=self.implementation.get_field_data_axes(f), + copy=False, + ) + out[ncvar] = key + + return out + + + def _customize_auxiliary_coordinates(self, parent_ncvar, f): + """Create auxiliary coordinate constructs from CFA terms. + + This method is primarily aimed at providing a customisation + entry point for subclasses. + + This method currently creates: + + * Auxiliary coordinate constructs derived from + non-standardised terms in CFA aggregation instructions. Each + auxiliary coordinate construct spans the same domain axes as + the parent field construct. Auxiliary coordinate constructs + are never created for `Domain` instances. + + .. versionadded:: TODODASKCFA + + :Parameters: + + parent_ncvar: `str` + The netCDF variable name of the parent variable. + + f: `Field` or `Domain` + The parent field or domain construct. + + :Returns: + + `dict` + A mapping of netCDF variable names to newly-created + auxiliary coordinate construct identifiers. + + **Examples** + + >>> n._customize_auxiliary_coordinates('tas', f) + {} + + >>> n._customize_auxiliary_coordinates('pr', f) + {'tracking_id': 'auxiliarycoordinate2'} + + """ + if self.implementation.is_domain(f) or not self._is_cfa_variable( + parent_ncvar + ): + return {} + + # ------------------------------------------------------------ + # Still here? Then we have a CFA-netCDF variable: Loop round + # the aggregation instruction terms and convert each + # non-standard term into an auxiliary coordinate construct + # that spans the same domain axes as the parent field. + # ------------------------------------------------------------ + g = self.read_vars + + out = {} + + attributes = g["variable_attributes"]["parent_ncvar"] + parsed_aggregated_data = self._parse_aggregated_data( + parent_ncvar, attributes.get("aggregated_data") + ) + standardised_terms = ("location", "file", "address", "format") + for x in parsed_aggregated_data: + term, ncvar = tuple(x.items())[0] + if term in standardised_terms: + # Ignore standardised aggregation terms + continue + + # Still here? Then it's a non-standard aggregation term + coord = self.implementation.initialise_AuxiliaryCoordinate() + + properties = g["variable_attributes"][ncvar].copy() + properties.setdefault("long_name", term) + self.implementation.set_properties(coord, properties) + + data = self._create_data( + ncvar, coord, parent_ncvar=parent_ncvar, cfa_term=term + ) + self.implementation.set_data(coord, data, copy=False) + + self.implementation.nc_set_variable(coord, ncvar) + + key = self.implementation.set_auxiliary_coordinate( + f, + coord, + axes=self.implementation.get_field_data_axes(f), + copy=False, + ) + out[ncvar] = key + + return out + + + def _customize_auxiliary_coordinates(self, parent_ncvar, f): + """Create auxiliary coordinate constructs from CFA terms. + + This method is primarily aimed at providing a customisation + entry point for subclasses. + + This method currently creates: + + * Auxiliary coordinate constructs derived from + non-standardised terms in CFA aggregation instructions. Each + construct spans the same domain axes as the parent field + construct. Constructs are never created for `Domain` + instances. + + .. versionadded:: TODODASKCFA + + :Parameters: + + parent_ncvar: `str` + The netCDF variable name of the parent variable. + + f: `Field` or `Domain` + The parent field or domain construct. + + :Returns: + + `dict` + A mapping of netCDF variable names to newly-created + auxiliary coordinate construct identifiers. + + **Examples** + + >>> n._customize_auxiliary_coordinates('tas', f) + {} + + >>> n._customize_auxiliary_coordinates('pr', f) + {'tracking_id': 'auxiliarycoordinate2'} + + """ + if self.implementation.is_domain(f) or not self._is_cfa_variable( + parent_ncvar + ): + return {} + + # ------------------------------------------------------------ + # Still here? Then we have a CFA-netCDF variable: Loop round + # the aggregation instruction terms and convert each + # non-standard term into an auxiliary coordinate construct + # that spans the same domain axes as the parent field. + # ------------------------------------------------------------ + g = self.read_vars + + out = {} + + attributes = g["variable_attributes"]["parent_ncvar"] + parsed_aggregated_data = self._parse_aggregated_data( + parent_ncvar, attributes.get("aggregated_data") + ) + standardised_terms = ("location", "file", "address", "format") + for x in parsed_aggregated_data: + term, ncvar = tuple(x.items())[0] + if term in standardised_terms: + # Ignore standardised aggregation terms + continue + + # Still here? Then it's a non-standard aggregation term + coord = self.implementation.initialise_AuxiliaryCoordinate() + + properties = g["variable_attributes"][ncvar].copy() + properties.setdefault("long_name", term) + self.implementation.set_properties(coord, properties) + + data = self._create_data( + ncvar, coord, parent_ncvar=parent_ncvar, cfa_term=term + ) + self.implementation.set_data(coord, data, copy=False) + + self.implementation.nc_set_variable(coord, ncvar) + + key = self.implementation.set_auxiliary_coordinate( + f, + coord, + axes=self.implementation.get_field_data_axes(f), + copy=False, + ) + out[ncvar] = key + + return out + + def _customize_field_ancillaries(self, parent_ncvar, f): + """Create field ancillary constructs from CFA terms. + + This method is primarily aimed at providing a customisation + entry point for subclasses. + + This method currently creates: + + * Field ancillary constructs derived from non-standardised + terms in CFA aggregation instructions. Each construct spans + the same domain axes as the parent field construct. + Constructs are never created for `Domain` instances. + + .. versionadded:: TODODASKCFA + + :Parameters: + + parent_ncvar: `str` + The netCDF variable name of the parent variable. + + f: `Field` + The parent field construct. + + :Returns: + + `dict` + A mapping of netCDF variable names to newly-created + construct identifiers. + + **Examples** + + >>> n._customize_field_ancillaries('tas', f) + {} + + >>> n._customize_field_ancillaries('pr', f) + {'tracking_id': 'fieldancillary1'} + + """ + if not self._is_cfa_variable(parent_ncvar): + return {} + + # ------------------------------------------------------------ + # Still here? Then we have a CFA-netCDF variable: Loop round + # the aggregation instruction terms and convert each + # non-standard term into a field ancillary construct that + # spans the same domain axes as the parent field. + # ------------------------------------------------------------ + g = self.read_vars + + out = {} + + attributes = g["variable_attributes"]["parent_ncvar"] + parsed_aggregated_data = self._parse_aggregated_data( + parent_ncvar, attributes.get("aggregated_data") + ) + standardised_terms = ("location", "file", "address", "format") + for x in parsed_aggregated_data: + term, ncvar = tuple(x.items())[0] + if term in standardised_terms: + # Ignore standardised aggregation terms + continue + + # Still here? Then it's a non-standard aggregation term + anc = self.implementation.initialise_FieldAncillary() + + properties = g["variable_attributes"][ncvar].copy() + properties.setdefault("long_name", term) + self.implementation.set_properties(anc, properties) + + data = self._create_data( + ncvar, anc, parent_ncvar=parent_ncvar, cfa_term=term + ) + self.implementation.set_data(anc, data, copy=False) + + self.implementation.nc_set_variable(anc, ncvar) + + key = self.implementation.set_field_ancillary( + f, + anc, + axes=self.implementation.get_field_data_axes(f), + copy=False, + ) + out[ncvar] = key + + return out + def _cfa(self, ncvar, f): """TODOCFADOCS. From f76c979b1efae724d321c5becd2f1b8edd59aa5a Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 16 Feb 2023 11:54:04 +0000 Subject: [PATCH 017/141] mixin --- cf/data/fragment/mixin/fragmentarraymixin.py | 350 +++++++++++++++++++ 1 file changed, 350 insertions(+) create mode 100644 cf/data/fragment/mixin/fragmentarraymixin.py diff --git a/cf/data/fragment/mixin/fragmentarraymixin.py b/cf/data/fragment/mixin/fragmentarraymixin.py new file mode 100644 index 0000000000..f88458ec88 --- /dev/null +++ b/cf/data/fragment/mixin/fragmentarraymixin.py @@ -0,0 +1,350 @@ +from numbers import Integral + +from ....units import Units + + +class FragmentArrayMixin: + """Mixin for a CFA fragment array. + + .. versionadded:: TODOCFAVER + + """ + + def __getitem__(self, indices): + """Returns a subspace of the fragment as a numpy array. + + x.__getitem__(indices) <==> x[indices] + + Indexing is similar to numpy indexing, with the following + differences: + + * A dimension's index can't be rank-reducing, i.e. it can't + be an integer, a scalar `numpy` array, nor a scalar `dask` + array. + + * When two or more dimension's indices are sequences of + integers then these indices work independently along each + dimension (similar to the way vector subscripts work in + Fortran). + + .. versionadded:: TODOCFAVER + + """ + indices = self._parse_indices(indices) + + try: + array = super().__getitem__(tuple(indices)) + except ValueError: + # A ValueError is expected to be raised when the fragment + # variable has fewer than 'self.ndim' dimensions (given + # that 'indices' now has 'self.ndim' elements). + axis = self._size_1_axis(indices) + if axis is not None: + # There is a unique size 1 index, that must correspond + # to the missing dimension => Remove it from the + # indices, get the fragment array with the new + # indices; and then insert the missing size one + # dimension. + indices.pop(axis) + array = super().__getitem__(tuple(indices)) + array = np.expand_dims(array, axis) + else: + # There are multiple size 1 indices, so we don't know + # how many missing dimensions there are nor their + # positions => Get the full fragment array; and then + # reshape it to the shape of the storage chunk. + array = super().__getitem__(Ellipsis) + if array.size != self.size: + raise ValueError( + "Can't get CFA fragment data from " + f"{self.get_filename()} ({self.get_address()}) when " + "the fragment has two or more missing size 1 " + "dimensions whilst also spanning two or more " + "storage chunks." + "\n\n" + "Consider recreating the data with exactly one" + "storage chunk per fragment." + ) + + array = array.reshape(self.shape) + + array = self._conform_to_aggregated_units(array) + return array + + def _size_1_axis(self, indices): + """Find the position of a unique size 1 index. + + .. versionadded:: TODOCFAVER + + .. seealso:: `_parse_indices` + + :Parameters: + + indices: sequence of index + The array indices to be parsed, as returned by + `_parse_indices`. + + :Returns: + + `int` or `None` + The position of the unique size 1 index, or `None` if + there isn't one. + + **Examples** + + >>> a._size_1_axis(([2, 4, 5], slice(0, 1), slice(0, 73))) + 1 + >>> a._size_1_axis(([2, 4, 5], slice(3, 4), slice(0, 73))) + 1 + >>> a._size_1_axis(([2, 4, 5], [0], slice(0, 73))) + 1 + >>> a._size_1_axis(([2, 4, 5], slice(0, 144), slice(0, 73))) + None + >>> a._size_1_axis(([2, 4, 5], slice(3, 7), [0, 1])) + None + >>> a._size_1_axis(([2, 4, 5], slice(0, 1), [0])) + None + + """ + axis = None # Position of unique size 1 index + + n = 0 # Number of size 1 indices + for i, index in enumerate(indices): + try: + if index.stop - index.start == 1: + # Index is a size 1 slice + n += 1 + axis = i + except AttributeError: + try: + if index.size == 1: + # Index is a size 1 numpy or dask array + n += 1 + axis = i + except AttributeError: + if len(index) == 1: + # Index is a size 1 list + n += 1 + axis = i + + if n > 1: + # There are two or more size 1 indices + axis = None + + return axis + + def _parse_indices(self, indices): + """Parse the indices that retrieve the fragment data. + + Ellipses are replaced with the approriate number `slice` + instances, and rank-reducing indices (such as an integer or + scalar array) are disallowed. + + .. versionadded:: TODOCFAVER + + :Parameters: + + indices: `tuple` or `Ellipsis` + The array indices to be parsed. + + :Returns: + + `list` + The parsed indices. + + **Examples** + + >>> a.shape + (12, 1, 73, 144) + >>> a._parse_indices(([2, 4, 5], Ellipsis, slice(45, 67)) + [[2, 4, 5], slice(0, 1), slice(0, 73), slice(45, 67)] + >>> a._parse_indices(([2, 4, 5], [0], slice(None), slice(45, 67)) + [[2, 4, 5], [0], slice(0, 73), slice(45, 67)] + + """ + shape = self.shape + if indices is Ellipsis: + return [slice(0, n) for n in shape] + + indices = list(indices) + + # Check indices + has_ellipsis = False + for i, (index, n) in enumerate(zip(indices, shape)): + if isinstance(index, slice): + if index == slice(None): + indices[i] = slice(0, n) + + continue + + if index is Ellipsis: + has_ellipsis = True + continue + + if isinstance(index, Integral) or not getattr(index, "ndim", True): + # TODOCFA: what about [] or np.array([])? + + # 'index' is an integer or a scalar numpy/dask array + raise ValueError( + f"Can't subspace {self.__class__.__name__} with a " + f"rank-reducing index: {index!r}" + ) + + if has_ellipsis: + # Replace Ellipsis with one or more slices + indices2 = [] + length = len(indices) + n = self.ndim + for index in indices: + if index is Ellipsis: + m = n - length + 1 + indices2.extend([slice(None)] * m) + n -= m + else: + indices2.append(i) + n -= 1 + + length -= 1 + + indices = indices2 + + for i, (index, n) in enumerate(zip(indices, shape)): + if index == slice(None): + indices[i] = slice(0, n) + + return indices + + def _conform_to_aggregated_units(self, array): + """Conform the array to have the aggregated units. + + .. versionadded:: TODOCFAVER + + :Parameters: + + array: `numpy.ndarray` or `dict` + The array to be conformed. If *array* is a `dict` with + `numpy` array values then each value is conformed. + + :Returns: + + `numpy.ndarray` or `dict` + The conformed array. The returned array may or may not + be the input array updated in-place, depending on its + data type and the nature of its units and the + aggregated units. + + If *array* is a `dict` then a dictionary of conformed + arrays is returned. + + """ + units = self.Units + if units: + aggregated_units = self.aggregated_Units + if not units.equivalent(aggregated_units): + raise ValueError( + f"Can't convert fragment data with units {units!r} to " + f"have aggregated units {aggregated_units!r}" + ) + + if units != aggregated_units: + if isinstance(array, dict): + # 'array' is a dictionary + for key, value in array.items(): + if key in _active_chunk_methds: + array[key] = Units.conform( + value, units, aggregated_units, inplace=True + ) + else: + # 'array' is a numpy array + array = Units.conform( + array, units, aggregated_units, inplace=True + ) + + return array + + @property + def aggregated_Units(self): + """The units of the aggregated data. + + .. versionadded:: TODOCFAVER + + :Returns: + + `Units` + The units of the aggregated data. + + """ + return Units( + self.get_aggregated_units(), self.get_aggregated_calendar(None) + ) + + def get_aggregated_calendar(self, default=ValueError()): + """The calendar of the aggregated array. + + If the calendar is `None` then the CF default calendar is + assumed, if applicable. + + .. versionadded:: TODOCFAVER + + :Parameters: + + default: optional + Return the value of the *default* parameter if the + aggregated calendar has not been set. If set to an + `Exception` instance then it will be raised instead. + + :Returns: + + `str` or `None` + The calendar value. + + """ + calendar = self._get_component("aggregated_calendar", False) + if calendar is False: + if default is None: + return + + return self._default( + default, + f"{self.__class__.__name__} 'aggregated_calendar' has not " + "been set", + ) + + return calendar + + def get_aggregated_units(self, default=ValueError()): + """The units of the aggregated array. + + If the units are `None` then the aggregated array has no + defined units. + + .. versionadded:: TODOCFAVER + + .. seealso:: `get_aggregated_calendar` + + :Parameters: + + default: optional + Return the value of the *default* parameter if the + aggregated units have not been set. If set to an + `Exception` instance then it will be raised instead. + + :Returns: + + `str` or `None` + The units value. + + """ + units = self._get_component("aggregated_units", False) + if units is False: + if default is None: + return + + return self._default( + default, + f"{self.__class__.__name__} 'aggregated_units' have not " + "been set", + ) + + return units + From f5967f08b1595ec9323df49f690fed81d1454c48 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 16 Feb 2023 15:20:46 +0000 Subject: [PATCH 018/141] dev --- cf/data/array/cfanetcdfarray.py | 10 +- cf/data/data.py | 16 +- cf/data/fragment/fullfragmentarray.py | 4 +- cf/data/fragment/mixin/fragmentarraymixin.py | 3 +- cf/data/fragment/netcdffragmentarray.py | 4 +- cf/data/fragment/umfragmentarray.py | 4 +- cf/read_write/netcdf/netcdfread.py | 413 ++----------------- 7 files changed, 54 insertions(+), 400 deletions(-) diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index 6f08198e76..160d56faa6 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -2,9 +2,8 @@ from itertools import accumulate, product from ...functions import abspath -from ..fragment import ( +from ..fragment import ( # MissingFragmentArray, FullFragmentArray, -# MissingFragmentArray, NetCDFFragmentArray, UMFragmentArray, ) @@ -29,7 +28,7 @@ def __new__(cls, *args, **kwargs): "nc": NetCDFFragmentArray, "um": UMFragmentArray, "full": FullFragmentArray, -# None: MissingFragmentArray, + # None: MissingFragmentArray, } return instance @@ -169,7 +168,7 @@ def __init__( f"CFA variable {ncvar} not found in file {filename}" ) -# if term is None: + # if term is None: shape = tuple([d.len for d in var.getDims()]) super().__init__( @@ -337,6 +336,9 @@ def _set_fragment( "full_value": np.ma.masked, } + def aggregated_dimensions(self): + """;kklknk;l""" + def get_aggregated_data(self, copy=True): """Get the aggregation data dictionary. diff --git a/cf/data/data.py b/cf/data/data.py index 9d751d4645..5e49289a81 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -38,10 +38,9 @@ from ..mixin_container import Container from ..units import Units from .collapse import Collapse -from .creation import ( +from .creation import ( # is_file_array, generate_axis_identifiers, is_abstract_Array_subclass, - is_file_array, to_dask, ) from .dask_utils import ( @@ -428,12 +427,11 @@ def __init__( "for compressed input arrays" ) - if is_file_array(array): - if to_memory: - try: - array = array.to_memory() - except AttributeError: - pass + if to_memory: # and is_file_array(array): + try: + array = array.to_memory() + except AttributeError: + pass if is_abstract_Array_subclass(array): # Save the input array in case it's useful later. For @@ -1273,7 +1271,7 @@ def _clear_after_dask_update(self, clear=_ALL): If *clear* is the ``_NONE`` integer-valued constant then no components are removed. - + To retain a component and remove all others, use ``_ALL`` with the bitwise OR operator. For instance, if *clear* is ``_ALL ^ _CACHE`` then the cached diff --git a/cf/data/fragment/fullfragmentarray.py b/cf/data/fragment/fullfragmentarray.py index 022efa985f..8b179c17be 100644 --- a/cf/data/fragment/fullfragmentarray.py +++ b/cf/data/fragment/fullfragmentarray.py @@ -71,7 +71,7 @@ def __init__( source=source, copy=False, ) - + if source is not None: try: aggregated_units = source._get_component( @@ -87,9 +87,7 @@ def __init__( except AttributeError: aggregated_calendar = False - self._set_component("aggregated_units", aggregated_units, copy=False) self._set_component( "aggregated_calendar", aggregated_calendar, copy=False ) - diff --git a/cf/data/fragment/mixin/fragmentarraymixin.py b/cf/data/fragment/mixin/fragmentarraymixin.py index f88458ec88..a9461c08c9 100644 --- a/cf/data/fragment/mixin/fragmentarraymixin.py +++ b/cf/data/fragment/mixin/fragmentarraymixin.py @@ -167,7 +167,7 @@ def _parse_indices(self, indices): return [slice(0, n) for n in shape] indices = list(indices) - + # Check indices has_ellipsis = False for i, (index, n) in enumerate(zip(indices, shape)): @@ -347,4 +347,3 @@ def get_aggregated_units(self, default=ValueError()): ) return units - diff --git a/cf/data/fragment/netcdffragmentarray.py b/cf/data/fragment/netcdffragmentarray.py index c261e77a98..fea14c55af 100644 --- a/cf/data/fragment/netcdffragmentarray.py +++ b/cf/data/fragment/netcdffragmentarray.py @@ -77,7 +77,7 @@ def __init__( mask=True, units=units, calendar=calendar, - source=source, + source=source, copy=False, ) @@ -96,9 +96,7 @@ def __init__( except AttributeError: aggregated_calendar = False - self._set_component("aggregated_units", aggregated_units, copy=False) self._set_component( "aggregated_calendar", aggregated_calendar, copy=False ) - diff --git a/cf/data/fragment/umfragmentarray.py b/cf/data/fragment/umfragmentarray.py index 21db614b2a..c64d234d07 100644 --- a/cf/data/fragment/umfragmentarray.py +++ b/cf/data/fragment/umfragmentarray.py @@ -72,7 +72,7 @@ def __init__( shape=shape, units=units, calendar=calendar, - source=source, + source=source, copy=False, ) @@ -91,9 +91,7 @@ def __init__( except AttributeError: aggregated_calendar = False - self._set_component("aggregated_units", aggregated_units, copy=False) self._set_component( "aggregated_calendar", aggregated_calendar, copy=False ) - diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 313d786159..bae5567896 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -15,6 +15,16 @@ class NetCDFRead(cfdm.read_write.netcdf.NetCDFRead): """ + def cfa_standard_terms(self): + """Standardised CFA aggregation instruction terms. + + These are found in the ``aggregation_data`` attributes. + + .. versionadded:: TODOCFAVER + + """ + return ("location", "file", "address", "format") + def _ncdimensions(self, ncvar, ncdimensions=None, parent_ncvar=None): """Return a list of the netCDF dimensions corresponding to a netCDF variable. @@ -221,7 +231,7 @@ def _create_data( for attr in ("aggregation_dimensions", "aggregation_data"): self.implementation.del_property(construct, attr, None) - if cfa_term is None: + if cfa_term is None: cfa_array, kwargs = self._create_cfanetcdfarray( ncvar, unpacked_dtype=unpacked_dtype, @@ -232,7 +242,7 @@ def _create_data( parent_ncvar, unpacked_dtype=unpacked_dtype, coord_ncvar=coord_ncvar, - term=cfa_term + term=cfa_term, ) data = self._create_Data( @@ -242,10 +252,19 @@ def _create_data( calendar=kwargs["calendar"], ) + if cfa_term is None: + cfa_write = True + for n, numblocks in zip( + cfa_array.get_fragment_shape(), data.numblocks + ): + if n == 1 and numblocks > 1: + # Each fragment spans multiple compute chunks + cfa_write = False + break + + data._set_cfa_write(cfa_write) + # Note: We don't cache elements from aggregated data - - # if cfa_term is not None or (data.numblocks == 1 for each non-aggreged dimension): - # data._set_cfa_write(True) return data @@ -348,6 +367,8 @@ def _customize_read_vars(self): # Check the 'Conventions' for CFA Conventions = g["global_attributes"].get("Conventions", "") + # If the string contains any commas, it is assumed to be a + # comma-separated list. all_conventions = split(",\s*", Conventions) if all_conventions[0] == Conventions: all_conventions = Conventions.split() @@ -532,16 +553,15 @@ def _create_cfanetcdfarray( return_kwargs_only=True, ) - # Specify a non-standardised term from which to create the - # data, which will have the shape of the parent variable. + # data, which will have the shape of the parent variable. if non_standard_term is not None: kwargs["term"] = non_standard_term - kwargs['shape'] = parent_shape - else: + kwargs["shape"] = parent_shape + else: # Get rid of the incorrect shape kwargs.pop("shape", None) - + # Add the aggregated_data attribute (that can be used by # dask.base.tokenize). kwargs["instructions"] = self.read_vars["variable_attributes"][ @@ -647,365 +667,6 @@ def _parse_aggregated_data(self, ncvar, aggregated_data): keys_are_dimensions=False, ) - def _customize_auxiliary_coordinates(self, parent_ncvar, f): - """Create auxiliary coordinate constructs from CFA terms. - - This method is primarily aimed at providing a customisation - entry point for subclasses. - - This method currently creates: - - * Auxiliary coordinate constructs derived from - non-standardised terms in CFA aggregation instructions. Each - auxiliary coordinate construct spans the same domain axes as - the parent field construct. Auxiliary coordinate constructs - are never created for `Domain` instances. - - .. versionadded:: TODODASKCFA - - :Parameters: - - parent_ncvar: `str` - The netCDF variable name of the parent variable. - - f: `Field` or `Domain` - The parent field or domain construct. - - :Returns: - - `dict` - A mapping of netCDF variable names to newly-created - auxiliary coordinate construct identifiers. - - **Examples** - - >>> n._customize_auxiliary_coordinates('tas', f) - {} - - >>> n._customize_auxiliary_coordinates('pr', f) - {'tracking_id': 'auxiliarycoordinate2'} - - """ - if self.implementation.is_domain(f) or not self._is_cfa_variable( - parent_ncvar - ): - return {} - - # ------------------------------------------------------------ - # Still here? Then we have a CFA-netCDF variable: Loop round - # the aggregation instruction terms and convert each - # non-standard term into an auxiliary coordinate construct - # that spans the same domain axes as the parent field. - # ------------------------------------------------------------ - g = self.read_vars - - out = {} - - attributes = g["variable_attributes"]["parent_ncvar"] - parsed_aggregated_data = self._parse_aggregated_data( - parent_ncvar, attributes.get("aggregated_data") - ) - standardised_terms = ("location", "file", "address", "format") - for x in parsed_aggregated_data: - term, ncvar = tuple(x.items())[0] - if term in standardised_terms: - # Ignore standardised aggregation terms - continue - - # Still here? Then it's a non-standard aggregation term - coord = self.implementation.initialise_AuxiliaryCoordinate() - - properties = g["variable_attributes"][ncvar].copy() - properties.setdefault("long_name", term) - self.implementation.set_properties(coord, properties) - - data = self._create_data( - ncvar, coord, parent_ncvar=parent_ncvar, cfa_term=term - ) - self.implementation.set_data(coord, data, copy=False) - - self.implementation.nc_set_variable(coord, ncvar) - - key = self.implementation.set_auxiliary_coordinate( - f, - coord, - axes=self.implementation.get_field_data_axes(f), - copy=False, - ) - out[ncvar] = key - - return out - - - def _customize_auxiliary_coordinates(self, parent_ncvar, f): - """Create auxiliary coordinate constructs from CFA terms. - - This method is primarily aimed at providing a customisation - entry point for subclasses. - - This method currently creates: - - * Auxiliary coordinate constructs derived from - non-standardised terms in CFA aggregation instructions. Each - auxiliary coordinate construct spans the same domain axes as - the parent field construct. Auxiliary coordinate constructs - are never created for `Domain` instances. - - .. versionadded:: TODODASKCFA - - :Parameters: - - parent_ncvar: `str` - The netCDF variable name of the parent variable. - - f: `Field` or `Domain` - The parent field or domain construct. - - :Returns: - - `dict` - A mapping of netCDF variable names to newly-created - auxiliary coordinate construct identifiers. - - **Examples** - - >>> n._customize_auxiliary_coordinates('tas', f) - {} - - >>> n._customize_auxiliary_coordinates('pr', f) - {'tracking_id': 'auxiliarycoordinate2'} - - """ - if self.implementation.is_domain(f) or not self._is_cfa_variable( - parent_ncvar - ): - return {} - - # ------------------------------------------------------------ - # Still here? Then we have a CFA-netCDF variable: Loop round - # the aggregation instruction terms and convert each - # non-standard term into an auxiliary coordinate construct - # that spans the same domain axes as the parent field. - # ------------------------------------------------------------ - g = self.read_vars - - out = {} - - attributes = g["variable_attributes"]["parent_ncvar"] - parsed_aggregated_data = self._parse_aggregated_data( - parent_ncvar, attributes.get("aggregated_data") - ) - standardised_terms = ("location", "file", "address", "format") - for x in parsed_aggregated_data: - term, ncvar = tuple(x.items())[0] - if term in standardised_terms: - # Ignore standardised aggregation terms - continue - - # Still here? Then it's a non-standard aggregation term - coord = self.implementation.initialise_AuxiliaryCoordinate() - - properties = g["variable_attributes"][ncvar].copy() - properties.setdefault("long_name", term) - self.implementation.set_properties(coord, properties) - - data = self._create_data( - ncvar, coord, parent_ncvar=parent_ncvar, cfa_term=term - ) - self.implementation.set_data(coord, data, copy=False) - - self.implementation.nc_set_variable(coord, ncvar) - - key = self.implementation.set_auxiliary_coordinate( - f, - coord, - axes=self.implementation.get_field_data_axes(f), - copy=False, - ) - out[ncvar] = key - - return out - - - def _customize_auxiliary_coordinates(self, parent_ncvar, f): - """Create auxiliary coordinate constructs from CFA terms. - - This method is primarily aimed at providing a customisation - entry point for subclasses. - - This method currently creates: - - * Auxiliary coordinate constructs derived from - non-standardised terms in CFA aggregation instructions. Each - auxiliary coordinate construct spans the same domain axes as - the parent field construct. Auxiliary coordinate constructs - are never created for `Domain` instances. - - .. versionadded:: TODODASKCFA - - :Parameters: - - parent_ncvar: `str` - The netCDF variable name of the parent variable. - - f: `Field` or `Domain` - The parent field or domain construct. - - :Returns: - - `dict` - A mapping of netCDF variable names to newly-created - auxiliary coordinate construct identifiers. - - **Examples** - - >>> n._customize_auxiliary_coordinates('tas', f) - {} - - >>> n._customize_auxiliary_coordinates('pr', f) - {'tracking_id': 'auxiliarycoordinate2'} - - """ - if self.implementation.is_domain(f) or not self._is_cfa_variable( - parent_ncvar - ): - return {} - - # ------------------------------------------------------------ - # Still here? Then we have a CFA-netCDF variable: Loop round - # the aggregation instruction terms and convert each - # non-standard term into an auxiliary coordinate construct - # that spans the same domain axes as the parent field. - # ------------------------------------------------------------ - g = self.read_vars - - out = {} - - attributes = g["variable_attributes"]["parent_ncvar"] - parsed_aggregated_data = self._parse_aggregated_data( - parent_ncvar, attributes.get("aggregated_data") - ) - standardised_terms = ("location", "file", "address", "format") - for x in parsed_aggregated_data: - term, ncvar = tuple(x.items())[0] - if term in standardised_terms: - # Ignore standardised aggregation terms - continue - - # Still here? Then it's a non-standard aggregation term - coord = self.implementation.initialise_AuxiliaryCoordinate() - - properties = g["variable_attributes"][ncvar].copy() - properties.setdefault("long_name", term) - self.implementation.set_properties(coord, properties) - - data = self._create_data( - ncvar, coord, parent_ncvar=parent_ncvar, cfa_term=term - ) - self.implementation.set_data(coord, data, copy=False) - - self.implementation.nc_set_variable(coord, ncvar) - - key = self.implementation.set_auxiliary_coordinate( - f, - coord, - axes=self.implementation.get_field_data_axes(f), - copy=False, - ) - out[ncvar] = key - - return out - - - def _customize_auxiliary_coordinates(self, parent_ncvar, f): - """Create auxiliary coordinate constructs from CFA terms. - - This method is primarily aimed at providing a customisation - entry point for subclasses. - - This method currently creates: - - * Auxiliary coordinate constructs derived from - non-standardised terms in CFA aggregation instructions. Each - construct spans the same domain axes as the parent field - construct. Constructs are never created for `Domain` - instances. - - .. versionadded:: TODODASKCFA - - :Parameters: - - parent_ncvar: `str` - The netCDF variable name of the parent variable. - - f: `Field` or `Domain` - The parent field or domain construct. - - :Returns: - - `dict` - A mapping of netCDF variable names to newly-created - auxiliary coordinate construct identifiers. - - **Examples** - - >>> n._customize_auxiliary_coordinates('tas', f) - {} - - >>> n._customize_auxiliary_coordinates('pr', f) - {'tracking_id': 'auxiliarycoordinate2'} - - """ - if self.implementation.is_domain(f) or not self._is_cfa_variable( - parent_ncvar - ): - return {} - - # ------------------------------------------------------------ - # Still here? Then we have a CFA-netCDF variable: Loop round - # the aggregation instruction terms and convert each - # non-standard term into an auxiliary coordinate construct - # that spans the same domain axes as the parent field. - # ------------------------------------------------------------ - g = self.read_vars - - out = {} - - attributes = g["variable_attributes"]["parent_ncvar"] - parsed_aggregated_data = self._parse_aggregated_data( - parent_ncvar, attributes.get("aggregated_data") - ) - standardised_terms = ("location", "file", "address", "format") - for x in parsed_aggregated_data: - term, ncvar = tuple(x.items())[0] - if term in standardised_terms: - # Ignore standardised aggregation terms - continue - - # Still here? Then it's a non-standard aggregation term - coord = self.implementation.initialise_AuxiliaryCoordinate() - - properties = g["variable_attributes"][ncvar].copy() - properties.setdefault("long_name", term) - self.implementation.set_properties(coord, properties) - - data = self._create_data( - ncvar, coord, parent_ncvar=parent_ncvar, cfa_term=term - ) - self.implementation.set_data(coord, data, copy=False) - - self.implementation.nc_set_variable(coord, ncvar) - - key = self.implementation.set_auxiliary_coordinate( - f, - coord, - axes=self.implementation.get_field_data_axes(f), - copy=False, - ) - out[ncvar] = key - - return out - def _customize_field_ancillaries(self, parent_ncvar, f): """Create field ancillary constructs from CFA terms. @@ -1061,18 +722,19 @@ def _customize_field_ancillaries(self, parent_ncvar, f): parsed_aggregated_data = self._parse_aggregated_data( parent_ncvar, attributes.get("aggregated_data") ) - standardised_terms = ("location", "file", "address", "format") + standardised_terms = self.cfa_standard_terms() for x in parsed_aggregated_data: term, ncvar = tuple(x.items())[0] if term in standardised_terms: - # Ignore standardised aggregation terms continue - # Still here? Then it's a non-standard aggregation term + # Still here? Then we've got a non-standard aggregation + # term from which we can create a field + # ancillary construct. anc = self.implementation.initialise_FieldAncillary() properties = g["variable_attributes"][ncvar].copy() - properties.setdefault("long_name", term) + properties["long_name"] = term self.implementation.set_properties(anc, properties) data = self._create_data( @@ -1091,7 +753,7 @@ def _customize_field_ancillaries(self, parent_ncvar, f): out[ncvar] = key return out - + def _cfa(self, ncvar, f): """TODOCFADOCS. @@ -1110,5 +772,4 @@ def _cfa(self, ncvar, f): TODOCFADOCS. """ - x = self._parse_x( ncvar, aggregated_data, - keys_are_variables=True) + x = self._parse_x(ncvar, aggregated_data, keys_are_variables=True) From 8a110331fa5f49db8e96f27e87ccb7c54e158a37 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 16 Feb 2023 18:12:18 +0000 Subject: [PATCH 019/141] dev --- cf/data/array/cfanetcdfarray.py | 27 ++-- cf/data/fragment/__init__.py | 3 +- cf/data/fragment/fullfragmentarray.py | 4 +- cf/data/fragment/missingfragmentarray.py | 136 +++++++++--------- cf/data/fragment/mixin/__init__.py | 2 +- cf/data/fragment/mixin/fragmentarraymixin.py | 6 + .../fragment/mixin/fragmentfilearraymixin.py | 87 +++++------ cf/data/fragment/netcdffragmentarray.py | 5 +- cf/read_write/netcdf/netcdfread.py | 34 ++--- 9 files changed, 147 insertions(+), 157 deletions(-) diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index 160d56faa6..e503eeb693 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -1,12 +1,10 @@ from copy import deepcopy from itertools import accumulate, product +import numpy as np + from ...functions import abspath -from ..fragment import ( # MissingFragmentArray, - FullFragmentArray, - NetCDFFragmentArray, - UMFragmentArray, -) +from ..fragment import FullFragmentArray, NetCDFFragmentArray, UMFragmentArray from .netcdfarray import NetCDFArray @@ -28,7 +26,6 @@ def __new__(cls, *args, **kwargs): "nc": NetCDFFragmentArray, "um": UMFragmentArray, "full": FullFragmentArray, - # None: MissingFragmentArray, } return instance @@ -168,7 +165,6 @@ def __init__( f"CFA variable {ncvar} not found in file {filename}" ) - # if term is None: shape = tuple([d.len for d in var.getDims()]) super().__init__( @@ -336,9 +332,6 @@ def _set_fragment( "full_value": np.ma.masked, } - def aggregated_dimensions(self): - """;kklknk;l""" - def get_aggregated_data(self, copy=True): """Get the aggregation data dictionary. @@ -390,7 +383,7 @@ def get_aggregated_data(self, copy=True): return aggregated_data def get_FragmentArray(self, fragment_format): - """Return a Fragment class. + """Return a fragment array class. .. versionadded:: 3.14.0 @@ -398,11 +391,10 @@ def get_FragmentArray(self, fragment_format): fragment_format: `str` The dataset format of the fragment. Either ``'nc'``, - ``'um'``, or `None`. + ``'um'``, or ``'full'``. :Returns: - `FragmentArray` The class for representing a fragment array of the given format. @@ -459,17 +451,14 @@ def get_fragment_shape(self): return self._get_component("fragment_shape") def get_non_standard_term(self, default=ValueError()): - """Get the sizes of the fragment dimensions. - - The fragment dimension sizes are given in the same order as - the aggregated dimension sizes given by `shape` + """TODOCFADOCS. .. versionadded:: TODOCFAVER :Returns: - `tuple` - The shape of the fragment dimensions. + `str` + TODOCFADOCS. """ return self._get_component("non_standard_term", default=default) diff --git a/cf/data/fragment/__init__.py b/cf/data/fragment/__init__.py index b13f7bef19..f7c3465b7f 100644 --- a/cf/data/fragment/__init__.py +++ b/cf/data/fragment/__init__.py @@ -1,4 +1,5 @@ from .fullfragmentarray import FullFragmentArray -from .missingfragmentarray import MissingFragmentArray + +# from .missingfragmentarray import MissingFragmentArray from .netcdffragmentarray import NetCDFFragmentArray from .umfragmentarray import UMFragmentArray diff --git a/cf/data/fragment/fullfragmentarray.py b/cf/data/fragment/fullfragmentarray.py index 8b179c17be..f456dae2b4 100644 --- a/cf/data/fragment/fullfragmentarray.py +++ b/cf/data/fragment/fullfragmentarray.py @@ -1,8 +1,8 @@ from ..array.fullarray import FullArray -from .abstract import FragmentArray +from .mixin import FragmentArrayMixin -class FullFragmentArray(FragmentArray): +class FullFragmentArray(FragmentArrayMixin, FullArray): """A CFA fragment array that is filled with a value. .. versionadded:: TODOCFAVER diff --git a/cf/data/fragment/missingfragmentarray.py b/cf/data/fragment/missingfragmentarray.py index b52ff0b20b..ef1f2d763d 100644 --- a/cf/data/fragment/missingfragmentarray.py +++ b/cf/data/fragment/missingfragmentarray.py @@ -1,68 +1,68 @@ -from .fullfragmentarray import FullFragmentArray - - -class MissingFragmentArray(FullFragmentArray): - """A CFA fragment array that is wholly missing data. - - .. versionadded:: 3.14.0 - - """ - - def __init__( - self, - dtype=None, - shape=None, - aggregated_units=False, - aggregated_calendar=False, - units=False, - calendar=False, - source=None, - copy=True, - ): - """**Initialisation** - - :Parameters: - - dtype: `numpy.dtype` - The data type of the aggregated array. May be `None` - if the numpy data-type is not known (which can be the - case for netCDF string types, for example). This may - differ from the data type of the netCDF fragment - variable. - - shape: `tuple` - The shape of the fragment within the aggregated - array. This may differ from the shape of the netCDF - fragment variable in that the latter may have fewer - size 1 dimensions. - - units: `str` or `None`, optional - The units of the fragment data. Ignored, as the data - are all missing values. - - calendar: `str` or `None`, optional - The calendar of the fragment data. Ignored, as the data - are all missing values. - - {{aggregated_units: `str` or `None`, optional}} - - {{aggregated_calendar: `str` or `None`, optional}} - - {{init source: optional}} - - {{init copy: `bool`, optional}} - - """ - import numpy as np - - super().__init__( - fill_value=np.ma.masked, - dtype=dtype, - shape=shape, - aggregated_units=aggregated_units, - aggregated_calendar=aggregated_calendar, - units=units, - calendar=calendar, - source=source, - copy=copy, - ) +# from .fullfragmentarray import FullFragmentArray +# +# +# class MissingFragmentArray(FullFragmentArray): +# """A CFA fragment array that is wholly missing data. +# +# .. versionadded:: 3.14.0 +# +# """ +# +# def __init__( +# self, +# dtype=None, +# shape=None, +# aggregated_units=False, +# aggregated_calendar=False, +# units=False, +# calendar=False, +# source=None, +# copy=True, +# ): +# """**Initialisation** +# +# :Parameters: +# +# dtype: `numpy.dtype` +# The data type of the aggregated array. May be `None` +# if the numpy data-type is not known (which can be the +# case for netCDF string types, for example). This may +# differ from the data type of the netCDF fragment +# variable. +# +# shape: `tuple` +# The shape of the fragment within the aggregated +# array. This may differ from the shape of the netCDF +# fragment variable in that the latter may have fewer +# size 1 dimensions. +# +# units: `str` or `None`, optional +# The units of the fragment data. Ignored, as the data +# are all missing values. +# +# calendar: `str` or `None`, optional +# The calendar of the fragment data. Ignored, as the data +# are all missing values. +# +# {{aggregated_units: `str` or `None`, optional}} +# +# {{aggregated_calendar: `str` or `None`, optional}} +# +# {{init source: optional}} +# +# {{init copy: `bool`, optional}} +# +# """ +# import numpy as np +# +# super().__init__( +# fill_value=np.ma.masked, +# dtype=dtype, +# shape=shape, +# aggregated_units=aggregated_units, +# aggregated_calendar=aggregated_calendar, +# units=units, +# calendar=calendar, +# source=source, +# copy=copy, +# ) diff --git a/cf/data/fragment/mixin/__init__.py b/cf/data/fragment/mixin/__init__.py index 3b995a260d..a4a35a1129 100644 --- a/cf/data/fragment/mixin/__init__.py +++ b/cf/data/fragment/mixin/__init__.py @@ -1 +1 @@ -from .fragmentfilearraymixin import FragmentFileArrayMixin +from .fragmentarraymixin import FragmentArrayMixin diff --git a/cf/data/fragment/mixin/fragmentarraymixin.py b/cf/data/fragment/mixin/fragmentarraymixin.py index a9461c08c9..4ab731bde1 100644 --- a/cf/data/fragment/mixin/fragmentarraymixin.py +++ b/cf/data/fragment/mixin/fragmentarraymixin.py @@ -1,5 +1,7 @@ from numbers import Integral +import numpy as np + from ....units import Units @@ -249,6 +251,10 @@ def _conform_to_aggregated_units(self, array): if units != aggregated_units: if isinstance(array, dict): # 'array' is a dictionary + + # TODOACTIVE: '_active_chunk_methds = {}' is a + # placeholder for the real thing + _active_chunk_methds = {} for key, value in array.items(): if key in _active_chunk_methds: array[key] = Units.conform( diff --git a/cf/data/fragment/mixin/fragmentfilearraymixin.py b/cf/data/fragment/mixin/fragmentfilearraymixin.py index 66a2801cd8..f40cf33d29 100644 --- a/cf/data/fragment/mixin/fragmentfilearraymixin.py +++ b/cf/data/fragment/mixin/fragmentfilearraymixin.py @@ -1,43 +1,44 @@ -class FragmentFileArrayMixin: - """Mixin class for a fragment array stored in a file. - - .. versionadded:: TODOCFAVER - - """ - - def get_address(self): - """The address of the fragment in the file. - - .. versionadded:: TODOCFAVER - - :Returns: - - The file address of the fragment, or `None` if there - isn't one. - - """ - try: - return self.get_array().get_address() - except AttributeError: - return - - def get_filename(self): - """Return the name of the file containing the array. - - .. versionadded:: TODOCFAVER - - :Returns: - - `str` or `None` - The filename, or `None` if there isn't one. - - **Examples** - - >>> a.get_filename() - 'file.nc' - - """ - try: - return self.get_array().get_filename() - except AttributeError: - return +# class FragmentFileArrayMixin: +# """Mixin class for a fragment array stored in a file. +# +# .. versionadded:: TODOCFAVER +# +# """ +# +# def get_address(self): +# """The address of the fragment in the file. +# +# .. versionadded:: TODOCFAVER +# +# :Returns: +# +# The file address of the fragment, or `None` if there +# isn't one. +# +# """ +# try: +# return self.get_array().get_address() +# except AttributeError: +# return +# +# def get_filename(self): +# """Return the name of the file containing the array. +# +# .. versionadded:: TODOCFAVER +# +# :Returns: +# +# `str` or `None` +# The filename, or `None` if there isn't one. +# +# **Examples** +# +# >>> a.get_filename() +# 'file.nc' +# +# """ +# try: +# return self.get_array().get_filename() +# except AttributeError: +# return +# diff --git a/cf/data/fragment/netcdffragmentarray.py b/cf/data/fragment/netcdffragmentarray.py index fea14c55af..2efa46d0b7 100644 --- a/cf/data/fragment/netcdffragmentarray.py +++ b/cf/data/fragment/netcdffragmentarray.py @@ -67,10 +67,11 @@ def __init__( {{init copy: `bool`, optional}} """ + group = None # TODO ??? + super().__init__( filename=filename, - ncvar=ncvar, - varid=varid, + ncvar=address, group=group, dtype=dtype, shape=shape, diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index bae5567896..aa71c1ed64 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -165,7 +165,6 @@ def _create_data( construct=None, unpacked_dtype=False, uncompress_override=None, - parent_ncvar=None, coord_ncvar=None, cfa_term=None, ): @@ -185,11 +184,9 @@ def _create_data( uncompress_override: `bool`, optional - parent_ncvar: `str`, optional - coord_ncvar: `str`, optional - cfa_term: `str`, optional + cfa_term: `dict`, optional The name of a non-standard aggregation instruction term from which to create the data. If set then *ncvar* must be the value of the term in the @@ -209,7 +206,6 @@ def _create_data( construct=construct, unpacked_dtype=unpacked_dtype, uncompress_override=uncompress_override, - parent_ncvar=parent_ncvar, coord_ncvar=coord_ncvar, ) @@ -231,7 +227,7 @@ def _create_data( for attr in ("aggregation_dimensions", "aggregation_data"): self.implementation.del_property(construct, attr, None) - if cfa_term is None: + if not cfa_term: cfa_array, kwargs = self._create_cfanetcdfarray( ncvar, unpacked_dtype=unpacked_dtype, @@ -253,6 +249,7 @@ def _create_data( ) if cfa_term is None: + # Set the CFA write status cfa_write = True for n, numblocks in zip( cfa_array.get_fragment_shape(), data.numblocks @@ -507,7 +504,6 @@ def _create_cfanetcdfarray( ncvar, unpacked_dtype=False, coord_ncvar=None, - parent_ncvar=None, non_standard_term=None, ): """Create a CFA-netCDF variable array. @@ -524,11 +520,6 @@ def _create_cfanetcdfarray( coord_ncvar: `str`, optional - parent_shape: `tuple`, optional - TODOCFADOCS. - - .. versionadded:: TODOCFAVER - non_standard_term: `str`, optional The name of a non-standard aggregation instruction term from which to create the array. If set then @@ -554,13 +545,13 @@ def _create_cfanetcdfarray( ) # Specify a non-standardised term from which to create the - # data, which will have the shape of the parent variable. + # data if non_standard_term is not None: kwargs["term"] = non_standard_term - kwargs["shape"] = parent_shape - else: - # Get rid of the incorrect shape - kwargs.pop("shape", None) + + # Get rid of the incorrect shape - this will get set by the + # CFAnetCDFArray instance. + kwargs.pop("shape", None) # Add the aggregated_data attribute (that can be used by # dask.base.tokenize). @@ -737,9 +728,7 @@ def _customize_field_ancillaries(self, parent_ncvar, f): properties["long_name"] = term self.implementation.set_properties(anc, properties) - data = self._create_data( - ncvar, anc, parent_ncvar=parent_ncvar, cfa_term=term - ) + data = self._create_data(parent_ncvar, anc, non_standard_term=term) self.implementation.set_data(anc, data, copy=False) self.implementation.nc_set_variable(anc, ncvar) @@ -772,4 +761,7 @@ def _cfa(self, ncvar, f): TODOCFADOCS. """ - x = self._parse_x(ncvar, aggregated_data, keys_are_variables=True) + pass + + +# x = self._parse_x(ncvar, aggregated_data, keys_are_variables=True) From 4a773a99ed7fd93b0950df9072c710cef72c4c85 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 17 Feb 2023 17:30:51 +0000 Subject: [PATCH 020/141] dev --- Changelog.rst | 12 ++ cf/__init__.py | 3 +- cf/data/array/cfanetcdfarray.py | 5 +- cf/data/data.py | 136 ++++++++++++++----- cf/data/fragment/__init__.py | 2 - cf/data/fragment/mixin/fragmentarraymixin.py | 31 +++-- cf/data/utils.py | 78 ++++++++++- cf/functions.py | 29 +++- cf/read_write/netcdf/netcdfread.py | 11 +- cf/read_write/netcdf/netcdfwrite.py | 131 ++++++++++-------- cf/read_write/write.py | 70 ++++++---- cf/test/test_functions.py | 3 + 12 files changed, 374 insertions(+), 137 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index c1a9b902ac..2146df31d8 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -1,3 +1,15 @@ +version ????? +------------- + +**2023-??-??** + +* New function: `cf.CFA` +* New method: `cf.Data.get_cfa_write` +* New method: `cf.Data.set_cfa_write` +* Changed dependency: ``?????<=cfdm>> d = cf.Data([1, 2]) - >>> d.cfa_write - False - - """ - return self._custom.get("cfa_write", False) - @property def data(self): """The data as an object identity. @@ -5916,6 +5889,32 @@ def convert_reference_time( return d + def get_cfa_write(self): + """The CFA write status of the data. + + If and only if the CFA write status is `True`, then this + `Data` instance has the potential to be written to a + CFA-netCDF file as aggregated data. In this case it is the + choice of parameters to the `cf.write` function that + determines if the data is actually written as aggregated data. + + .. versionadded:: TODOCFAVER + + .. seealso:: `set_cfa_write`, `cf.read`, `cf.write` + + :Returns: + + `bool` + + **Examples** + + >>> d = cf.Data([1, 2]) + >>> d.get_cfa_write() + False + + """ + return self._custom.get("cfa_write", False) + def get_data(self, default=ValueError(), _units=None, _fill_value=None): """Returns the data. @@ -5928,7 +5927,7 @@ def get_data(self, default=ValueError(), _units=None, _fill_value=None): """ return self - def get_filenames(self): + def get_filenames(self, address=False): """The names of files containing parts of the data array. Returns the names of any files that are required to deliver @@ -5944,12 +5943,21 @@ def get_filenames(self): object has a callable `get_filename` method, the output of which is added to the returned `set`. + :Parameters: + + address: `bool`, optional + TODOCFADOCS + + .. versionadded:: TODOCFAVER + :Returns: `set` The file names. If no files are required to compute the data then an empty `set` is returned. + TODOCFADOCS + **Examples** >>> d = cf.Data.full((5, 8), 1, chunks=4) @@ -5986,6 +5994,8 @@ def get_filenames(self): >>> d[2, 3].get_filenames() {'file_A.nc'} + TODOCFADOCS: address example + """ from dask.base import collections_to_dsk @@ -5993,9 +6003,14 @@ def get_filenames(self): dsk = collections_to_dsk((self.to_dask_array(),), optimize_graph=True) for a in dsk.values(): try: - out.add(a.get_filename()) + filename = a.get_filename() except AttributeError: pass + else: + if address: + out.add((filename, a.get_address())) + else: + out.add(filename) return out @@ -6094,6 +6109,33 @@ def set_calendar(self, calendar): """ self.Units = Units(self.get_units(default=None), calendar) + def set_cfa_write(self, status): + """Set the CFA write status of the data. + + TODOCFADOCS.ppp + + .. versionadded:: TODOCFAVER + + .. seealso:: `get_cfa_write`, `cf.read`, `cf.write` + + :Parameters: + + status: `bool` + The new CFA write status. + + :Returns: + + `None` + + """ + if status: + raise ValueError( + "'set_cfa_write' only allows the CFA write status to be " + "set to False" + ) + + self._del_cfa_write() + def set_units(self, value): """Set the units. @@ -11929,6 +11971,32 @@ def var( return d + def ggg(self): + """ + + f = cf.example_field(0) + cf.write(f, "file_A.nc") + cf.write(f, "file_B.nc") + + a = cf.read("file_A.nc", chunks=4)[0].data + b = cf.read("file_B.nc", chunks=4)[0].data + c = cf.Data(b.array, units=b.Units, chunks=4) + d = cf.Data.concatenate([a, a.copy(), b, c], axis=1) + + + """ + from .utils import chunk_indices, chunk_locations, chunk_positions + + chunks = self.chunks + for position, location, indices in zip( + chunk_positions(chunks), + chunk_locations(chunks), + chunk_indices(chunks), + ): + print( + position, location, self[indices].get_filenames(address=True) + ) + def section( self, axes, stop=None, chunks=False, min_step=1, mode="dictionary" ): diff --git a/cf/data/fragment/__init__.py b/cf/data/fragment/__init__.py index f7c3465b7f..2ce2dafa60 100644 --- a/cf/data/fragment/__init__.py +++ b/cf/data/fragment/__init__.py @@ -1,5 +1,3 @@ from .fullfragmentarray import FullFragmentArray - -# from .missingfragmentarray import MissingFragmentArray from .netcdffragmentarray import NetCDFFragmentArray from .umfragmentarray import UMFragmentArray diff --git a/cf/data/fragment/mixin/fragmentarraymixin.py b/cf/data/fragment/mixin/fragmentarraymixin.py index 4ab731bde1..daebc4c724 100644 --- a/cf/data/fragment/mixin/fragmentarraymixin.py +++ b/cf/data/fragment/mixin/fragmentarraymixin.py @@ -52,9 +52,10 @@ def __getitem__(self, indices): array = np.expand_dims(array, axis) else: # There are multiple size 1 indices, so we don't know - # how many missing dimensions there are nor their - # positions => Get the full fragment array; and then - # reshape it to the shape of the storage chunk. + # how many missing dimensions the fragment has, nor + # their positions => Get the full fragment array and + # then reshape it to the shape of the storage chunk, + # assuming that it has teh correct size. array = super().__getitem__(Ellipsis) if array.size != self.size: raise ValueError( @@ -65,7 +66,8 @@ def __getitem__(self, indices): "storage chunks." "\n\n" "Consider recreating the data with exactly one" - "storage chunk per fragment." + "storage chunk per fragment (e.g. set the " + "parameter 'chunks=None' to cf.read)." ) array = array.reshape(self.shape) @@ -78,7 +80,7 @@ def _size_1_axis(self, indices): .. versionadded:: TODOCFAVER - .. seealso:: `_parse_indices` + .. seealso:: `_parse_indices`, `__getitem__` :Parameters: @@ -90,7 +92,7 @@ def _size_1_axis(self, indices): `int` or `None` The position of the unique size 1 index, or `None` if - there isn't one. + there are zero or at least two of them. **Examples** @@ -110,26 +112,27 @@ def _size_1_axis(self, indices): """ axis = None # Position of unique size 1 index - n = 0 # Number of size 1 indices - for i, index in enumerate(indices): + n_size_1 = 0 # Number of size 1 indices + for i, (index, n) in enumerate(zip(indices, self.shape)): try: - if index.stop - index.start == 1: + x = index.indices(n) + if abs(x[1] - x[0]) == 1: # Index is a size 1 slice - n += 1 + n_size_1 += 1 axis = i except AttributeError: try: if index.size == 1: # Index is a size 1 numpy or dask array - n += 1 + n_size_1 += 1 axis = i except AttributeError: if len(index) == 1: # Index is a size 1 list - n += 1 + n_size_1 += 1 axis = i - if n > 1: + if n_size_1 > 1: # There are two or more size 1 indices axis = None @@ -203,7 +206,7 @@ def _parse_indices(self, indices): indices2.extend([slice(None)] * m) n -= m else: - indices2.append(i) + indices2.append(index) n -= 1 length -= 1 diff --git a/cf/data/utils.py b/cf/data/utils.py index 3b501fb538..dd9c3720b9 100644 --- a/cf/data/utils.py +++ b/cf/data/utils.py @@ -423,7 +423,7 @@ def chunk_positions(chunks): .. versionadded:: 3.14.0 - .. seealso:: `chunk_shapes` + .. seealso:: `chunk_indices`, `chunk_locations`, `chunk_shapes` :Parameters: @@ -453,7 +453,7 @@ def chunk_shapes(chunks): .. versionadded:: 3.14.0 - .. seealso:: `chunk_positions` + .. seealso:: `chunk_indices`, `chunk_locations`, `chunk_positions` :Parameters: @@ -478,6 +478,80 @@ def chunk_shapes(chunks): return product(*chunks) +def chunk_locations(chunks): + """Find the shape of each chunk. + + .. versionadded:: TODOCFAVER + + .. seealso:: `chunk_indices`, `chunk_positions`, `chunk_shapes` + + :Parameters: + + chunks: `tuple` + The chunk sizes along each dimension, as output by + `dask.array.Array.chunks`. + + **Examples** + + >>> chunks = ((1, 2), (9,), (4, 5, 6)) + >>> for location in cf.data.utils.chunk_locations(chunks): + ... print(location) + ... + ((0, 1), (0, 9), (0, 4)) + ((0, 1), (0, 9), (4, 9)) + ((0, 1), (0, 9), (9, 15)) + ((1, 3), (0, 9), (0, 4)) + ((1, 3), (0, 9), (4, 9)) + ((1, 3), (0, 9), (9, 15)) + + """ + from dask.utils import cached_cumsum + + cumdims = [cached_cumsum(bds, initial_zero=True) for bds in chunks] + locations = [ + [(s, s + dim) for s, dim in zip(starts, shapes)] + for starts, shapes in zip(cumdims, chunks) + ] + return product(*locations) + + +def chunk_indices(chunks): + """Find the shape of each chunk. + + .. versionadded:: TODOCFAVER + + .. seealso:: `chunk_locations`, `chunk_positions`, `chunk_shapes` + + :Parameters: + + chunks: `tuple` + The chunk sizes along each dimension, as output by + `dask.array.Array.chunks`. + + **Examples** + + >>> chunks = ((1, 2), (9,), (4, 5, 6)) + >>> for index in cf.data.utils.chunk_indices(chunks): + ... print(index) + ... + (slice(0, 1, None), slice(0, 9, None), slice(0, 4, None)) + (slice(0, 1, None), slice(0, 9, None), slice(4, 9, None)) + (slice(0, 1, None), slice(0, 9, None), slice(9, 15, None)) + (slice(1, 3, None), slice(0, 9, None), slice(0, 4, None)) + (slice(1, 3, None), slice(0, 9, None), slice(4, 9, None)) + (slice(1, 3, None), slice(0, 9, None), slice(9, 15, None)) + + """ + from dask.utils import cached_cumsum + + cumdims = [cached_cumsum(bds, initial_zero=True) for bds in chunks] + indices = [ + [slice(s, s + dim) for s, dim in zip(starts, shapes)] + for starts, shapes in zip(cumdims, chunks) + ] + return product(*indices) + + def scalar_masked_array(dtype=float): """Return a scalar masked array. diff --git a/cf/functions.py b/cf/functions.py index bdac5380ba..1f3c7a774b 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -30,7 +30,7 @@ from dask.utils import parse_bytes from psutil import virtual_memory -from . import __file__, __version__ +from . import __cfa_version__, __file__, __version__ from .constants import ( CONSTANTS, OperandBoundsCombination, @@ -1149,6 +1149,33 @@ def CF(): CF.__doc__ = cfdm.CF.__doc__.replace("cfdm.", "cf.") + +def CFA(): + """The version of the CFA conventions. + + This indicates which version of the CFA conventions are + represented by this release of the cf package, and therefore the + version can not be changed. + + .. versionadded:: TODOCFAVER + + .. seealso:: `cf.CF` + + :Returns: + + `str` + The version of the CFA conventions represented by this + release of the cf package. + + **Examples** + + >>> cf.CFA() + '0.6.2' + + """ + return __cfa_version__ + + # Module-level alias to avoid name clashes with function keyword # arguments (corresponding to 'import atol as cf_atol' etc. in other # modules) diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index aa71c1ed64..91a4ee5e7f 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -165,6 +165,7 @@ def _create_data( construct=None, unpacked_dtype=False, uncompress_override=None, + parent_ncvar=None, coord_ncvar=None, cfa_term=None, ): @@ -206,12 +207,13 @@ def _create_data( construct=construct, unpacked_dtype=unpacked_dtype, uncompress_override=uncompress_override, + parent_ncvar=parent_ncvar, coord_ncvar=coord_ncvar, ) + # Set the CFA write status to True when there is exactly + # one dask chunk if data.npartitions == 1: - # Set the CFA write status to True when there is - # exactly one dask chunk data._set_cfa_write(True) self._cache_data_elements(data, ncvar) @@ -235,7 +237,7 @@ def _create_data( ) else: cfa_array, kwargs = self._create_cfanetcdfarray( - parent_ncvar, + ncvar, unpacked_dtype=unpacked_dtype, coord_ncvar=coord_ncvar, term=cfa_term, @@ -248,8 +250,8 @@ def _create_data( calendar=kwargs["calendar"], ) + # Set the CFA write status if cfa_term is None: - # Set the CFA write status cfa_write = True for n, numblocks in zip( cfa_array.get_fragment_shape(), data.numblocks @@ -729,6 +731,7 @@ def _customize_field_ancillaries(self, parent_ncvar, f): self.implementation.set_properties(anc, properties) data = self._create_data(parent_ncvar, anc, non_standard_term=term) + data._custom["cfa_term"] = True self.implementation.set_data(anc, data, copy=False) self.implementation.nc_set_variable(anc, ncvar) diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index bd96d28c54..301db3c21f 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -1,19 +1,9 @@ -import random -from string import hexdigits - import cfdm import dask.array as da import numpy as np -from ... import Bounds, Coordinate, DomainAncillary from .netcdfread import NetCDFRead -_cfa_message = ( - "Writing CFA files has been temporarily disabled, " - "and will return at version 4.0.0. " - "CFA-0.4 functionality is still available at version 3.13.x." -) - class NetCDFWrite(cfdm.read_write.netcdf.NetCDFWrite): """A container for writing Fields to a netCDF dataset.""" @@ -30,31 +20,53 @@ def __new__(cls, *args, **kwargs): instance._NetCDFRead = NetCDFRead return instance - def _write_as_cfa(self, cfvar): - """True if the variable should be written as a CFA variable. + def _use_cfa(self, cfvar, construct_type): + """Whether or not to write as a CFA variable. .. versionadded:: 3.0.0 + :Parameters: + + cfvar: cf instance that contains data + + construct_type: `str` + The construct type of the *cfvar*, or its parent if + *cfvar* is not a construct. + + :Returns: + + `bool` + True if the variable is to be should be written as a + CFA variable. + """ - if not self.write_vars["cfa"]: + g = self.write_vars + if not g["cfa"]: return False data = self.implementation.get_data(cfvar, None) if data is None: return False + if not data.get_cfa_write(): + return False + if data.size == 1: return False - if isinstance(cfvar, (Coordinate, DomainAncillary)): - return cfvar.ndim > 1 + if construct_type == "field": + return True - if isinstance(cfvar, Bounds): - return cfvar.ndim > 2 + for ctype, ndim in g["cfa_options"]["metadata"]: + if ctype in ("all", construct_type): + if ndim is None: + return True - return True + return ndim == data.ndim - def _customize_createVariable(self, cfvar, kwargs): + return False + + def _customize_createVariable(self, cfvar, construct_type, kwargs): """Customise keyword arguments for `netCDF4.Dataset.createVariable`. @@ -73,11 +85,11 @@ def _customize_createVariable(self, cfvar, kwargs): `netCDF4.Dataset.createVariable`. """ - kwargs = super()._customize_createVariable(cfvar, kwargs) - - if self._write_as_cfa(cfvar): - raise ValueError(_cfa_message) + kwargs = super()._customize_createVariable( + cfvar, construct_type, kwargs + ) + if self._use_cfa(cfvar, construct_type): kwargs["dimensions"] = () kwargs["chunksizes"] = None @@ -92,6 +104,7 @@ def _write_data( unset_values=(), compressed=False, attributes={}, + construct_type=None, ): """Write a Data object. @@ -109,12 +122,24 @@ def _write_data( unset_values: sequence of numbers + attributes: `dict`, optional + The netCDF attributes for the constructs that have been + written to the file. + + construct_type: `str`, optional + TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Returns: + + `None` + """ g = self.write_vars - if self._write_as_cfa(cfvar): - raise ValueError(_cfa_message) - + if self._use_cfa(cfvar, construct_type): + # Write the data as CFA aggregated data self._write_cfa_data(ncvar, ncdimensions, data, cfvar) return @@ -326,33 +351,33 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): `None` """ - raise ValueError(_cfa_message) - - def _random_hex_string(self, size=10): - """Return a random hexadecimal string with the given number of - characters. - - .. versionadded:: 3.0.0 - - :Parameters: - - size: `int`, optional - The number of characters in the generated string. - - :Returns: - - `str` - The hexadecimal string. - - **Examples:** - - >>> _random_hex_string() - 'C3eECbBBcf' - >>> _random_hex_string(6) - '7a4acc' - - """ - return "".join(random.choice(hexdigits) for i in range(size)) + raise ValueError("_cfa_message") + + # def _random_hex_string(self, size=10): + # """Return a random hexadecimal string with the given number of + # characters. + # + # .. versionadded:: 3.0.0 + # + # :Parameters: + # + # size: `int`, optional + # The number of characters in the generated string. + # + # :Returns: + # + # `str` + # The hexadecimal string. + # + # **Examples:** + # + # >>> _random_hex_string() + # 'C3eECbBBcf' + # >>> _random_hex_string(6) + # '7a4acc' + # + # """ + # return "".join(random.choice(hexdigits) for i in range(size)) def _convert_to_builtin_type(self, x): """Convert a non-JSON-encodable object to a JSON-encodable diff --git a/cf/read_write/write.py b/cf/read_write/write.py index 5e1a9418c4..ecaa64f19a 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -4,7 +4,7 @@ from ..cfimplementation import implementation from ..decorators import _manage_log_level_via_verbosity -from ..functions import _DEPRECATION_ERROR_FUNCTION_KWARGS, flat +from ..functions import _DEPRECATION_ERROR_FUNCTION_KWARGS, CFA, flat from .netcdf import NetCDFWrite # from . import mpi_on @@ -723,9 +723,7 @@ def write( raise ValueError("Can't set datatype and double") if single is not None and double is not None: - raise ValueError( - "Can't set both the single and double " "parameters" - ) + raise ValueError("Can't set both the single and double parameters") if single is not None and not single: double = True @@ -745,32 +743,58 @@ def write( numpy.dtype("int32"): numpy.dtype(int), } - extra_write_vars = { - "cfa": False, - "cfa_options": {}, - "reference_datetime": reference_datetime, - } + # Extra write variables + extra_write_vars = {"reference_datetime": reference_datetime} - # CFA options + # ------------------------------------------------------------ + # CFA + # ------------------------------------------------------------ if fmt in ("CFA", "CFA4"): - extra_write_vars["cfa"] = True + cfa = True fmt = "NETCDF4" - if cfa_options: - extra_write_vars["cfa_options"] = cfa_options elif fmt == "CFA3": - extra_write_vars["cfa"] = True + cfa = True fmt = "NETCDF3_CLASSIC" - if cfa_options: - extra_write_vars["cfa_options"] = cfa_options - - if extra_write_vars["cfa"]: - if Conventions: - if isinstance(Conventions, str): - Conventions = (Conventions,) + else: + cfa = False + + if cfa: + # Add CFA to the Conventions + if not Conventions: + Conventions = CFA() + elif isinstance(Conventions, str): + Conventions = (Conventions, CFA()) + else: + Conventions = tuple(Conventions) + (CFA(),) - Conventions = tuple(Conventions) + ("CFA",) + # Parse the 'cfa_options' parameter + if not cfa_options: + cfa_options = {} else: - Conventions = "CFA" + cfa_options = cfa_options.copy() + keys = ("paths", "metadata", "group") + if not set(cfa_options).issubset(keys): + raise ValueError( + "Invalid dictionary key to the 'cfa_options' " + f"parameter. Valid keys are {keys}. " + f"Got: {tuple(cfa_options)}" + ) + + if "metadata" not in cfa_options: + cfa_options["metadata"] = (("all", None),) + else: + metadata = cfa_options["metadata"] + if isinstance(metadata, str): + cfa_options["metadata"] = ((metadata, None),) + elif isinstance(metadata[0], str): + cfa_options["metadata"] = (metadata,) + + cfa_options.setdefault("paths", "relative") + else: + cfa_options = {} + + extra_write_vars["cfa"] = cfa + extra_write_vars["cfa_options"] = cfa_options netcdf.write( fields, diff --git a/cf/test/test_functions.py b/cf/test/test_functions.py index 7a32b0d859..7210e35263 100644 --- a/cf/test/test_functions.py +++ b/cf/test/test_functions.py @@ -368,6 +368,9 @@ def test_size(self): x = da.arange(9) self.assertEqual(cf.size(x), x.size) + def test_CFA(self): + self.assertEqual(cf.CFA(), cf.__cfa_version__) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) From fdb0998c84bbe276a6766d8f39499dfc524b3fe4 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 20 Feb 2023 23:28:19 +0000 Subject: [PATCH 021/141] dev --- cf/data/array/cfanetcdfarray.py | 2 +- cf/data/data.py | 163 +++++++++++++++----- cf/read_write/netcdf/netcdfread.py | 9 +- cf/read_write/netcdf/netcdfwrite.py | 229 +++++++++++++++++++++------- cf/read_write/write.py | 58 ++++--- 5 files changed, 345 insertions(+), 116 deletions(-) diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index 7567a19b84..66bea87302 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -186,7 +186,7 @@ def __init__( # the 'aggregated_data' dictionary for massive # aggretations (e.g. with O(10e6) fragments) will be # slow, hence the parallelisation of the process - # with delayed + compute, and that the + # with delayed + compute; and that the # parallelisation overheads won't be noticeable for # small aggregations (e.g. O(10) fragments). aggregated_data = {} diff --git a/cf/data/data.py b/cf/data/data.py index 2575ba17fe..022479ce8f 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -94,11 +94,11 @@ # Contstants used to specify which `Data` components should be cleared # when the dask array is updated. See `Data._clear_after_dask_update` # for details. -_NONE = 0 # = 0b000 -_ARRAY = 1 # = 0b001 -_CACHE = 2 # = 0b010 -_CFA = 4 # = 0b100 -_ALL = _ARRAY | _CACHE | _CFA +_NONE = 0 # = 0b000000 +_ARRAY = 1 # = 0b000001 +_CACHE = 2 # = 0b000010 +_CFA = 4 # = 0b000100 +_ALL = 63 # = 0b111111 class Data(DataClassDeprecationsMixin, Container, cfdm.Data): @@ -1116,7 +1116,8 @@ def __setitem__(self, indices, value): self[indices] = reset - # Remove elements made invalid by updating the `dask` array + # Remove componenets made invalid by updating the `dask` array + # in-place self._clear_after_dask_update(_ALL) return @@ -1244,26 +1245,29 @@ def _clear_after_dask_update(self, clear=_ALL): .. versionadded:: 3.14.0 - .. seealso:: `_del_Array`, `_set_dask` + .. seealso:: `_del_Array`, `_del_cached_elements`, + `_del_cfa_write`, `_set_dask` :Parameters: clear: `int`, optional - Specify which components should be removed. Which components are removed is determined by sequentially combining *clear* with the ``_ARRAY``, ``_CACHE`` and ``_CFA`` integer-valued contants, using the bitwise AND operator: - * If ``clear & _ARRAY`` is True then delete a source - array. + * If ``clear & _ARRAY`` is non-zero then a source + array is deleted. + + * If ``clear & _CACHE`` is non-zero then cached + element values are deleted. - * If ``clear & _CACHE`` is True then delete cached - element values. + * If ``clear & _CFA`` is non-zero then the CFA write + status is set to `False`. - * If ``clear & _CFA`` is True then set the CFA write - status to `False`. + * If ``clear`` is non-zero then the CFA term status is + set to False. By default *clear* is the ``_ALL`` integer-valued constant, which results in all components being @@ -1276,7 +1280,7 @@ def _clear_after_dask_update(self, clear=_ALL): ``_ALL`` with the bitwise OR operator. For instance, if *clear* is ``_ALL ^ _CACHE`` then the cached element values will be kept but all other components - removed. + will be removed. .. versionadded:: TODOCFAVER @@ -1285,6 +1289,9 @@ def _clear_after_dask_update(self, clear=_ALL): `None` """ + if not clear: + return + if clear & _ARRAY: # Delete a source array self._del_Array(None) @@ -1297,6 +1304,10 @@ def _clear_after_dask_update(self, clear=_ALL): # Set the CFA write status to False self._del_cfa_write() + # Always set the CFA term status to False + if "cfa_term" in self._custom: + del self._custom["cfa_term"] + def _set_dask(self, array, copy=False, clear=_ALL): """Set the dask array. @@ -5915,19 +5926,19 @@ def get_cfa_write(self): """ return self._custom.get("cfa_write", False) - def get_data(self, default=ValueError(), _units=None, _fill_value=None): - """Returns the data. - - .. versionadded:: 3.0.0 - - :Returns: - - `Data` - - """ + # def get_data(self, default=ValueError(), _units=None, _fill_value=None): + # """Returns the data.## + # + # .. versionadded:: 3.0.0# + # + # :Returns:## + # + # `Data`## + # + # """ return self - def get_filenames(self, address=False): + def get_filenames(self, address_format=False): """The names of files containing parts of the data array. Returns the names of any files that are required to deliver @@ -5994,7 +6005,7 @@ def get_filenames(self, address=False): >>> d[2, 3].get_filenames() {'file_A.nc'} - TODOCFADOCS: address example + TODOCFADOCS: address_format example """ from dask.base import collections_to_dsk @@ -6003,14 +6014,13 @@ def get_filenames(self, address=False): dsk = collections_to_dsk((self.to_dask_array(),), optimize_graph=True) for a in dsk.values(): try: - filename = a.get_filename() + f = a.get_filename() + if address_format: + f = ((f, a.get_address(), a.get_format()),) except AttributeError: pass else: - if address: - out.add((filename, a.get_address())) - else: - out.add(filename) + out.add(f) return out @@ -11971,7 +11981,19 @@ def var( return d - def ggg(self): + def url_or_file_uri(x): + from urllib.parse import urlparse + + result = urlparse(x) + return all([result.scheme in ("file", "http", "https"), result.netloc]) + + def is_file_uri(x): + from urllib.parse import urlparse + + result = urlparse(x) + return all([result.scheme in ("file"), result.netloc]) + + def ggg(self, substitions=None): """ f = cf.example_field(0) @@ -11985,17 +12007,76 @@ def ggg(self): """ - from .utils import chunk_indices, chunk_locations, chunk_positions + from .os.path import abspath + from .utils import chunk_indices, chunk_positions + + if substitutions: + substitions = tuple(substitions.items())[::-1] chunks = self.chunks - for position, location, indices in zip( - chunk_positions(chunks), - chunk_locations(chunks), - chunk_indices(chunks), + shape = self.numblocks + + faf = [] + max_file = 0 + max_address = 0 + max_format = 0 + for indices in chunk_indices(chunks): + a = self[indices].get_filenames(address_format=True) + if len(a) != 1: + raise ValueError("TODOCFADOCS") + + filename, address, fmt = a.pop() + + if relative is not None: + pass + # To what ? The path given by 'relaitve? or the path of the original CFA file, if there was one ...? + + if not url_or_file_uri(filename): + filename = PurePath( + abspath(filename) + ).as_uri() ## ?? see above + + if substitions: + for base, sub in substitions: + filename = filename.replace(sub, base) + + faf.append((filename, address, fmt)) + + max_file = max(max_file, len(filename)) + max_address = max(max_address, len(address)) + max_format = max(max_format, len(fmt)) + + aggregation_file = np.empty(shape, dtype=f"U{max_file}") + aggregation_address = np.empty(shape, dtype=f"U{max_address}") + aggregation_format = np.empty(shape, dtype=f"U{max_format}") + + for position, (filename, address, fmt) in zip( + chunk_positions(chunks), faf ): - print( - position, location, self[indices].get_filenames(address=True) - ) + aggregation_file[position] = filename + aggregation_address[position] = address + aggregation_format[position] = fmt + + # Location + dtype = np.dtype(np.int32) + if max(self.to_dask_array().chunksize) > np.iinfo(dtype).max: + dtype = np.dtype(np.int64) + + aggregation_location = np.ma.masked_all( + (self.ndim, max(shape)), dtype=dtype + ) + + for j, c in enumerate(chunks): + aggregation_location[j, : len(c)] = c + + # Return Data objects + data = partial(type(self), chunks=-1) + return { + "aggregation_location": data(aggregation_location), + "aggregation_file": data(aggregation_file), + "aggregation_format": data(aggregation_format), + "aggregation_address": data(aggregation_address), + } def section( self, axes, stop=None, chunks=False, min_step=1, mode="dictionary" diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 91a4ee5e7f..f80cbe18e2 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -730,12 +730,19 @@ def _customize_field_ancillaries(self, parent_ncvar, f): properties["long_name"] = term self.implementation.set_properties(anc, properties) + # Store the term name as the 'id' attribute. This will be + # used as the term name if the field field ancillary is + # written to disk as a non-standard CFA term. + anc.id = term + data = self._create_data(parent_ncvar, anc, non_standard_term=term) - data._custom["cfa_term"] = True self.implementation.set_data(anc, data, copy=False) self.implementation.nc_set_variable(anc, ncvar) + # Set the CFA term status + anc._custom["cfa_term"] = True + key = self.implementation.set_field_ancillary( f, anc, diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 301db3c21f..00b37b03ba 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -33,6 +33,8 @@ def _use_cfa(self, cfvar, construct_type): The construct type of the *cfvar*, or its parent if *cfvar* is not a construct. + .. versionadded:: TODOCFAVER + :Returns: `bool` @@ -58,7 +60,10 @@ def _use_cfa(self, cfvar, construct_type): return True for ctype, ndim in g["cfa_options"]["metadata"]: + # Write as CFA if it has an appropriate construct type ... if ctype in ("all", construct_type): + # ... and then only if it satisfies the number of + # dimenions criterion if ndim is None: return True @@ -127,7 +132,8 @@ def _write_data( written to the file. construct_type: `str`, optional - TODOCFADOCS + The construct type of the *cfvar*, or its parent if + *cfvar* is not a construct. .. versionadded:: TODOCFAVER @@ -139,15 +145,17 @@ def _write_data( g = self.write_vars if self._use_cfa(cfvar, construct_type): + # -------------------------------------------------------- # Write the data as CFA aggregated data + # -------------------------------------------------------- self._write_cfa_data(ncvar, ncdimensions, data, cfvar) return - # Still here? + # ------------------------------------------------------------ + # Still here? The write a normal (non-CFA) variable + # ------------------------------------------------------------ if compressed: - # -------------------------------------------------------- # Write data in its compressed form - # -------------------------------------------------------- data = data.source().source() # Get the dask array @@ -351,33 +359,59 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): `None` """ - raise ValueError("_cfa_message") - - # def _random_hex_string(self, size=10): - # """Return a random hexadecimal string with the given number of - # characters. - # - # .. versionadded:: 3.0.0 - # - # :Parameters: - # - # size: `int`, optional - # The number of characters in the generated string. - # - # :Returns: - # - # `str` - # The hexadecimal string. - # - # **Examples:** - # - # >>> _random_hex_string() - # 'C3eECbBBcf' - # >>> _random_hex_string(6) - # '7a4acc' - # - # """ - # return "".join(random.choice(hexdigits) for i in range(size)) + ggg = data.ggg() + + location = ggg["location"] + location_ncdimensions = [ + self._netcdf_name(f"cfa{size}", dimsize=size, role="cfa_location") + for size in location.shape + ] + + address = ggg["address"] + fragment_ncdimensions = [ + self._netcdf_name(f"f_{ncdim}", dimsize=size, role="cfa_fragment") + for ncdim, size in zip(ncdimensions, address.shape) + ] + + aggregated_data = [] + for term, d in ggg.items(): + if term == "location": + dimensions = location_ncdimensions + else: + dimensions = fragment_ncdimensions + if term == "format": + u = d.unique().persist() + if u.size == 1: + # Collapse fragment formats to a common scalar + d = u.squeeze() + dimensions = () + + term_ncvar = self._cfa_write_term_variable( + d, + f"cfa_{term}", + dimensions, + ) + + aggregated_data.append(f"{term}: {term_ncvar}") + + # Look for non-standard CFA terms stored as field ancillaries + # on a field + if self.implementation.is_field(cfvar): + aggregated_data.extend( + self._cfa_write_non_standard_terms( + cfvar, fragment_ncdimensions + ) + ) + + # Add the CFA aggreation variable attributes + self._write_attributes( + None, + ncvar, + extra={ + "aggregated_dimensions": " ".join(ncdimensions), + "aggregated_data": " ".join(aggregated_data), + }, + ) def _convert_to_builtin_type(self, x): """Convert a non-JSON-encodable object to a JSON-encodable @@ -479,25 +513,116 @@ def _filled_string_array(self, array, fill_value=""): return array + def _write_field_ancillary(self, f, key, anc): + """Write a field ancillary to the netCDF file. + + If an equal field ancillary has already been written to the file + then it is not re-written. + + .. versionadded:: TODOCFAVER + + :Parameters: + + f : `Field` + + key : `str` + + anc : `FieldAncillary` + + :Returns: + + `str` + The netCDF variable name of the field ancillary + object. If no ancillary variable was written then an + empty string is returned. + + **Examples** + + >>> ncvar = _write_field_ancillary(f, 'fieldancillary2', anc) + + """ + if anc._custom.get("cfa_term", False): + # This field ancillary construct is to be written as a + # non-standard CFA term belonging to the parent field, or + # not at all. + return "" + + return super()._write_field_ancillary(f, key, anc) + + def _cfa_write_term_variable(self, data, ncvar, ncdimensions): + """TODOCFADOCS.""" + create = not self._already_in_file(data, ncdimensions) + + if not create: + ncvar = self.write_vars["seen"][id(data)]["ncvar"] + else: + ncvar = self._netcdf_name(ncvar) + + # Create a new CFA term variable + self._write_netcdf_variable(ncvar, ncdimensions, data) + + return ncvar + + def _cfa_write_non_standard_terms(self, cfvar, fragment_ncdimensions): + """TODOCFADOCS""" + # Look for non-standard CFA terms stored as field ancillaries + from dask.array import blockwise + + aggregated_data = [] + non_standard_terms = [] + for key, field_anc in self.implementation.get_field_ancillaries( + cfvar + ).items(): + if not field_anc._custom.get("cfa_term", False): + continue + + data = self.implementation.get_data(field_anc) + if not data.get_cfa_write(): + continue + + if cfvar.get_data_axes(key) != cfvar.get_data_axes(): + continue + + # Still here? Then convert the data to span the fragment + # dimensions, with one value per fragment, and then write + # it to disk. + dx = data.to_dask_array() + dx_ind = tuple(range(dx.ndim)) + out_ind = dx_ind + dx = blockwise( + self._cfa_unique, + out_ind, + dx, + dx_ind, + adjust_chunks={i: 1 for i in out_ind}, + dtype=dx.dtype, + ) + field_anc.set_data(dx) + + # Get the non-standard term name from the field + # ancillary's 'id' attribute + term = getattr(field_anc, "id", "term") + term = term.replace(" ", "_") + base = term + n = 0 + while term in non_standard_terms: + n += 1 + term = f"{base}_{n}" + + term_ncvar = self._cfa_write_term_variable( + field_anc.data, f"cfa_{term}", fragment_ncdimensions + ) + + aggregated_data.append(f"{term}: {term_ncvar}") + + return aggregated_data + + @classmethod + def _cfa_unique(cls, a): + """TODOCFADOCS.""" + out_shape = (1,) * a.ndim + a = np.unique(a) + if a.size == 1: + return a.reshape(out_shape) -# def _convert_dtype(self, array, new_dtype=None): -# """Convert the data type of a numpy array. -# -# .. versionadded:: 3.14.0 -# -# :Parameters: -# -# array: `numpy.ndarray` -# The `numpy` array -# -# new_dtype: data-type -# The new data type. -# -# :Returns: -# -# `numpy.ndarray` -# The array with converted data type. -# -# """ -# return array.astype(new_dtype) -# + return np.ma.masked_all(out_shape, dtype=a.dtype) diff --git a/cf/read_write/write.py b/cf/read_write/write.py index ecaa64f19a..28c6ae3353 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -372,23 +372,25 @@ def write( Key Value ================= ======================================= - ``???`` ---------- The types of construct to be written as + ``paths`` -------- How to write fragment file names. Set + to ``'relative'`` for them to be + written as relative to the CF_netCDF + file being created, or else set to + ``'absolute'`` for them to be written + as file URIs. Note that in either case, + fragment file names defined by fully + qualified URLs will always be written + as such. + + ``metadata`` ----- The types of construct to be written as CFA-netCDF aggregated variables. By - default field data and the data Nd - metadata constructs. What about UGRID, - for which the 1-d coords are, combined, - muchlarger than the data .... What + default all metadata constructs field + data and the data Nd metadata + constructs. What about UGRID, for which + the 1-d coords are, combined, + muchlarger than the data .... TODO What about DSG and compression in general? - ``properties``---- A (sequence of) `str` defining one or - more field or domain properties whose - values are to be written to the output - CFA-netCDF file as non-standardised - aggregation instruction variables. When - the output file is read in with - `cf.read` these variables are converted - to auxiliary coordinate constructs. - ``substitutions``- A dictionary whose key/value pairs define text substitutions to be applied to the fragment file URIs when the @@ -397,21 +399,35 @@ def write( ``'${...}'``, where ``...`` represents one or more letters, digits, and underscores. The substitutions are - stored in the output file in the + stored in the output file by the ``substitutions`` attribute of the ``file`` aggregation instruction variable. - ``'base'`` Deprecated at version 3.14.0. + ``properties``---- A (sequence of) `str` defining one or + more properties of the fields + represented by each file frgament. For + each property specified, the fragment + values are written to the output + CFA-netCDF file as non-standardised + aggregation instruction variables whose + term name is the same as the property + name. When the output file is read in + with `cf.read` these variables are + converted to field ancillary + constructs. + + ``'base'`` Deprecated at version 3.14.0 and no + longer available. ================= ======================================= - The *cfa_options* default to ``{'???': ['field', 'N-d']}`` + The default of *cfa_options* is ``{'paths': 'relative'}``. *Parameter example:* - ``cfa_options={'properties': 'tracking_id'}`` + ``cfa_options={'substitutions': {'${base}': '/home/data/'}}`` *Parameter example:* - ``cfa_options={'substitutions': {'${base}': '/home/data/'}}`` + ``cfa_options={'properties': 'tracking_id'}`` endian: `str`, optional The endian-ness of the output file. Valid values are @@ -772,7 +788,7 @@ def write( cfa_options = {} else: cfa_options = cfa_options.copy() - keys = ("paths", "metadata", "group") + keys = ("paths", "metadata", "group", "substitutions") if not set(cfa_options).issubset(keys): raise ValueError( "Invalid dictionary key to the 'cfa_options' " @@ -781,7 +797,7 @@ def write( ) if "metadata" not in cfa_options: - cfa_options["metadata"] = (("all", None),) + cfa_options["metadata"] = ((None, None),) else: metadata = cfa_options["metadata"] if isinstance(metadata, str): From ce98e3bcccadaff72988307a58760b367758a3ff Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 22 Feb 2023 10:05:04 +0000 Subject: [PATCH 022/141] dev --- cf/data/data.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cf/data/data.py b/cf/data/data.py index 022479ce8f..4d991d34b5 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -12020,6 +12020,9 @@ def ggg(self, substitions=None): max_file = 0 max_address = 0 max_format = 0 + + filenames = [] + for indices in chunk_indices(chunks): a = self[indices].get_filenames(address_format=True) if len(a) != 1: @@ -12027,6 +12030,8 @@ def ggg(self, substitions=None): filename, address, fmt = a.pop() + filenames.append(filename) + if relative is not None: pass # To what ? The path given by 'relaitve? or the path of the original CFA file, if there was one ...? @@ -12042,11 +12047,11 @@ def ggg(self, substitions=None): faf.append((filename, address, fmt)) - max_file = max(max_file, len(filename)) +# max_file = max(max_file, len(filename)) max_address = max(max_address, len(address)) max_format = max(max_format, len(fmt)) - aggregation_file = np.empty(shape, dtype=f"U{max_file}") + aggregation_file = np.array(filenames).reshape(shape) aggregation_address = np.empty(shape, dtype=f"U{max_address}") aggregation_format = np.empty(shape, dtype=f"U{max_format}") From 71e571dc50821aa3c0c62f3a60ea22c2acbf2894 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 22 Feb 2023 16:56:04 +0000 Subject: [PATCH 023/141] dev --- cf/data/data.py | 231 ++++++++++++++++------------ cf/read_write/netcdf/netcdfwrite.py | 161 ++++++++++++++++--- 2 files changed, 272 insertions(+), 120 deletions(-) diff --git a/cf/data/data.py b/cf/data/data.py index 4d991d34b5..9913aaea68 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -11981,107 +11981,136 @@ def var( return d - def url_or_file_uri(x): - from urllib.parse import urlparse - - result = urlparse(x) - return all([result.scheme in ("file", "http", "https"), result.netloc]) - - def is_file_uri(x): - from urllib.parse import urlparse - - result = urlparse(x) - return all([result.scheme in ("file"), result.netloc]) - - def ggg(self, substitions=None): - """ - - f = cf.example_field(0) - cf.write(f, "file_A.nc") - cf.write(f, "file_B.nc") - - a = cf.read("file_A.nc", chunks=4)[0].data - b = cf.read("file_B.nc", chunks=4)[0].data - c = cf.Data(b.array, units=b.Units, chunks=4) - d = cf.Data.concatenate([a, a.copy(), b, c], axis=1) - - - """ - from .os.path import abspath - from .utils import chunk_indices, chunk_positions - - if substitutions: - substitions = tuple(substitions.items())[::-1] - - chunks = self.chunks - shape = self.numblocks - - faf = [] - max_file = 0 - max_address = 0 - max_format = 0 - - filenames = [] - - for indices in chunk_indices(chunks): - a = self[indices].get_filenames(address_format=True) - if len(a) != 1: - raise ValueError("TODOCFADOCS") - - filename, address, fmt = a.pop() - - filenames.append(filename) - - if relative is not None: - pass - # To what ? The path given by 'relaitve? or the path of the original CFA file, if there was one ...? - - if not url_or_file_uri(filename): - filename = PurePath( - abspath(filename) - ).as_uri() ## ?? see above - - if substitions: - for base, sub in substitions: - filename = filename.replace(sub, base) - - faf.append((filename, address, fmt)) - -# max_file = max(max_file, len(filename)) - max_address = max(max_address, len(address)) - max_format = max(max_format, len(fmt)) - - aggregation_file = np.array(filenames).reshape(shape) - aggregation_address = np.empty(shape, dtype=f"U{max_address}") - aggregation_format = np.empty(shape, dtype=f"U{max_format}") - - for position, (filename, address, fmt) in zip( - chunk_positions(chunks), faf - ): - aggregation_file[position] = filename - aggregation_address[position] = address - aggregation_format[position] = fmt - - # Location - dtype = np.dtype(np.int32) - if max(self.to_dask_array().chunksize) > np.iinfo(dtype).max: - dtype = np.dtype(np.int64) - - aggregation_location = np.ma.masked_all( - (self.ndim, max(shape)), dtype=dtype - ) - - for j, c in enumerate(chunks): - aggregation_location[j, : len(c)] = c - - # Return Data objects - data = partial(type(self), chunks=-1) - return { - "aggregation_location": data(aggregation_location), - "aggregation_file": data(aggregation_file), - "aggregation_format": data(aggregation_format), - "aggregation_address": data(aggregation_address), - } + # def url_or_file_uri(x): + # from urllib.parse import urlparse + # + # result = urlparse(x) + # return all([result.scheme in ("file", "http", "https"), result.netloc]) + # + # def is_url(x): + # from urllib.parse import urlparse + # + # result = urlparse(x) + # return all([result.scheme in ("http", "https"), result.netloc]) + # + # def is_file_uri(x): + # from urllib.parse import urlparse + # + # result = urlparse(x) + # return all([result.scheme in ("file"), result.netloc]) + + # def ggg( + # self, + # absolute=False, + # relative=True, + # cfa_filename=None, + # substitions=None, + # ): + # """ + # + # f = cf.example_field(0) + # cf.write(f, "file_A.nc") + # cf.write(f, "file_B.nc") + # + # a = cf.read("file_A.nc", chunks=4)[0].data + # b = cf.read("file_B.nc", chunks=4)[0].data + # c = cf.Data(b.array, units=b.Units, chunks=4) + # d = cf.Data.concatenate([a, a.copy(), b, c], axis=1) + # + # + # """ + # from os.path import abspath, relpath + # from pathlib import PurePath + # from urllib.parse import urlparse + # + # from .utils import chunk_indices # , chunk_positions + # + # if substitutions: + # substitions = tuple(substitutions.items())[::-1] + # + # if relative: + # cfa_dir = PurePath(abspath(cfa_filename)).parent + # + # chunks = self.chunks + # + # # faf = [] + # # max_file = 0 + # # max_address = 0 + # # max_format = 0 + # + # filenames = [] + # address = [] + # formats = [] + # + # for indices in chunk_indices(chunks): + # a = self[indices].get_filenames(address_format=True) + # if len(a) != 1: + # raise ValueError("TODOCFADOCS") + # + # filename, address, fmt = a.pop() + # + # parsed_filename = urlparse(filename) + # scheme = parsed_filename.scheme + # if scheme not in ("http", "https"): + # path = parsed_filename.path + # if absolute: + # filename = PurePath(abspath(path)).as_uri() + # elif relative or scheme != "file": + # filename = relpath(abspath(path), start=cfa_dir) + # + # if substitutions: + # for base, sub in substitutions: + # filename = filename.replace(sub, base) + # + # filenames.append(filename) + # addresses.append(address) + # formats.append(fmt) + # + # # faf.append((filename, address, fmt)) + # # + # # max_file = max(max_file, len(filename)) + # # max_address = max(max_address, len(address)) + # # max_format = max(max_format, len(fmt)) + # + # aggregation_file = np.array(filenames).reshape(shape) + # aggregation_address = np.array(addresses).reshape( + # shape + # ) # , dtype=f"U{max_address}") + # aggregation_format = np.array(formats).reshape( + # shape + # ) # , dtype=f"U{max_format}") + # del filenames + # del address + # del formats + # + # # for position, (filename, address, fmt) in zip( + # # chunk_positions(chunks), faf + # # ): + # # aggregation_file[position] = filename + # # aggregation_address[position] = address + # # aggregation_format[position] = fmt + # + # # Location + # dtype = np.dtype(np.int32) + # if max(self.to_dask_array().chunksize) > np.iinfo(dtype).max: + # dtype = np.dtype(np.int64) + # + # aggregation_location = np.ma.masked_all( + # (self.ndim, max(shape)), dtype=dtype + # ) + # + # for j, c in enumerate(chunks): + # aggregation_location[j, : len(c)] = c + # + # # Return Data objects + # # data = partial(type(self), chunks=-1) + # data = type(self) + # return { + # "aggregation_location": data(aggregation_location), + # "aggregation_file": data(aggregation_file), + # "aggregation_format": data(aggregation_format), + # "aggregation_address": data(aggregation_address), + # } def section( self, axes, stop=None, chunks=False, min_step=1, mode="dictionary" diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 00b37b03ba..8ee569899f 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -359,19 +359,31 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): `None` """ - ggg = data.ggg() + ggg = self._ggg(data) - location = ggg["location"] - location_ncdimensions = [ - self._netcdf_name(f"cfa{size}", dimsize=size, role="cfa_location") - for size in location.shape - ] + # Get the location netCDF dimensions + location_ncdimensions = [] + for size in ggg["location"].shape: + l_ncdim = self._netcdf_name( + f"cfa_{size}", dimsize=size, role="cfa_location" + ) + if l_ncdim not in g["dimensions"]: + # Create a new location dimension + self._write_dimension(l_ncdim, None, size=size) + + location_ncdimensions.append(l_ncdim) - address = ggg["address"] - fragment_ncdimensions = [ - self._netcdf_name(f"f_{ncdim}", dimsize=size, role="cfa_fragment") - for ncdim, size in zip(ncdimensions, address.shape) - ] + # Get the fragment netCDF dimensions + fragment_ncdimensions = [] + for ncdim, size in zip(ncdimensions, ggg["address"].shape): + f_ncdim = self._netcdf_name( + f"f_{ncdim}", dimsize=size, role="cfa_fragment" + ) + if f_ncdim not in g["dimensions"]: + # Create a new fragement dimension + self._write_dimension(f_ncdim, None, size=size) + + fragment_ncdimensions.append(f_ncdim) aggregated_data = [] for term, d in ggg.items(): @@ -382,7 +394,7 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): if term == "format": u = d.unique().persist() if u.size == 1: - # Collapse fragment formats to a common scalar + # Collapse formats to a common scalar d = u.squeeze() dimensions = () @@ -403,7 +415,7 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): ) ) - # Add the CFA aggreation variable attributes + # Add the CFA aggregation variable attributes self._write_attributes( None, ncvar, @@ -550,7 +562,10 @@ def _write_field_ancillary(self, f, key, anc): return super()._write_field_ancillary(f, key, anc) def _cfa_write_term_variable(self, data, ncvar, ncdimensions): - """TODOCFADOCS.""" + """TODOCFADOCS. + + .. versionadded:: TODOCFAVER + """ create = not self._already_in_file(data, ncdimensions) if not create: @@ -564,10 +579,11 @@ def _cfa_write_term_variable(self, data, ncvar, ncdimensions): return ncvar def _cfa_write_non_standard_terms(self, cfvar, fragment_ncdimensions): - """TODOCFADOCS""" - # Look for non-standard CFA terms stored as field ancillaries - from dask.array import blockwise + """TODOCFADOCS + .. versionadded:: TODOCFAVER + """ + # Look for non-standard CFA terms stored as field ancillaries aggregated_data = [] non_standard_terms = [] for key, field_anc in self.implementation.get_field_ancillaries( @@ -589,7 +605,7 @@ def _cfa_write_non_standard_terms(self, cfvar, fragment_ncdimensions): dx = data.to_dask_array() dx_ind = tuple(range(dx.ndim)) out_ind = dx_ind - dx = blockwise( + dx = da.blockwise( self._cfa_unique, out_ind, dx, @@ -619,10 +635,117 @@ def _cfa_write_non_standard_terms(self, cfvar, fragment_ncdimensions): @classmethod def _cfa_unique(cls, a): - """TODOCFADOCS.""" + """TODOCFADOCS. + + .. versionadded:: TODOCFAVER + """ out_shape = (1,) * a.ndim a = np.unique(a) if a.size == 1: return a.reshape(out_shape) return np.ma.masked_all(out_shape, dtype=a.dtype) + + def _ggg(self, data): + """ + + f = cf.example_field(0) + cf.write(f, "file_A.nc") + cf.write(f, "file_B.nc") + + a = cf.read("file_A.nc", chunks=4)[0].data + b = cf.read("file_B.nc", chunks=4)[0].data + c = cf.Data(b.array, units=b.Units, chunks=4) + d = cf.Data.concatenate([a, a.copy(), b, c], axis=1) + + + """ + from os.path import abspath, relpath + from pathlib import PurePath + from urllib.parse import urlparse + + g = self.write_vars + + substitutions = g["cfa_options"].get("substitutions") + if substitutions: + # TODO move this to global once + substitutions = tuple(substitutions.items())[::-1] + + relative = g["cfa_options"].get("relative", None) + if relative: + absolute = False + cfa_dir = PurePath(abspath(g["filename"])).parent + elif relative is not None: + absolute = True + else: + absolute =None + + aggregation_file = [] + aggregation_address = [] + aggregation_format = [] + for indices in data.chunk_indices(): + a = self[indices].get_filenames(address_format=True) + if len(a) != 1: + raise ValueError("TODOCFADOCS") + + filename, address, fmt = a.pop() + + parsed_filename = urlparse(filename) + scheme = parsed_filename.scheme + if scheme not in ("http", "https"): + path = parsed_filename.path + if absolute: + filename = PurePath(abspath(path)).as_uri() + elif relative or scheme != "file": + filename = relpath(abspath(path), start=cfa_dir) + + if substitutions: + for base, sub in substitutions: + filename = filename.replace(sub, base) + + aggregation_file.append(filename) + aggregation_address.append(address) + aggregation_format.append(fmt) + + shape = data.numblocks + aggregation_file = np.array(aggregation_file).reshape(shape) + aggregation_address = np.array(aggregation_address).reshape(shape) + aggregation_format = np.array(aggregation_format).reshape(shape) + + # Location + dtype = np.dtype(np.int32) + if max(data.to_dask_array().chunksize) > np.iinfo(dtype).max: + dtype = np.dtype(np.int64) + + aggregation_location = np.ma.masked_all( + (self.ndim, max(shape)), dtype=dtype + ) + + for i, c in enumerate(data.chunks): + aggregation_location[i, : len(c)] = c + + # Return Data objects + data = type(data) + return { + "aggregation_location": data(aggregation_location), + "aggregation_file": data(aggregation_file), + "aggregation_format": data(aggregation_format), + "aggregation_address": data(aggregation_address), + } + + def _customize_write_vars(self): + """Customise the write parameters. + + .. versionadded:: TODOCFAVER + + """ + g = self.write_vars + + if g.get('cfa'): + from os.path import abspath + from pathlib import PurePath + + g['cfa_dir'] = PurePath(abspath(g["filename"])).parent + + + Need to know about this on read, too. From d2fcd1985fc38c3620b69d2de2416079bc5e5f14 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 23 Feb 2023 18:25:03 +0000 Subject: [PATCH 024/141] dev --- cf/data/array/cfanetcdfarray.py | 64 +++++++++++++++++++++++++++-- cf/read_write/netcdf/netcdfread.py | 59 ++++++++++++++++++++++++-- cf/read_write/netcdf/netcdfwrite.py | 17 ++++---- 3 files changed, 123 insertions(+), 17 deletions(-) diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index 66bea87302..898854836a 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -1,5 +1,7 @@ from copy import deepcopy from itertools import accumulate, product +from os.path import join as os_join +from urllib.parse import urlparse import numpy as np @@ -40,6 +42,7 @@ def __init__( units=False, calendar=False, instructions=None, + substitutions=None, non_standard_term=None, source=None, copy=True, @@ -108,6 +111,11 @@ def __init__( the CFA netCDF variable. If set then this will be used to improve the performance of `__dask_tokenize__`. + substitutions: `dict`, optional + TODOCFADOCS + + .. versionadded:: TODOCFAVER + non_standard_term: `str`, optional The name of a non-standard aggregation instruction term from which the array is to be created, instead of @@ -123,6 +131,8 @@ def __init__( ``non_standard_term='tracking_id', ncvar='aggregation_id'`` + .. versionadded:: TODOCFAVER + {{init source: optional}} {{init copy: `bool`, optional}} @@ -146,12 +156,19 @@ def __init__( except AttributeError: aggregated_data = {} + try: + substitutions = source.get_substitutions() + except AttributeError: + substitutions = None + try: non_standard_term = source.get_non_standard_term() except AttributeError: non_standard_term = None elif filename is not None: + from pathlib import PurePath + from CFAPython import CFAFileFormat from CFAPython.CFADataset import CFADataset from CFAPython.CFAExceptions import CFAException @@ -182,6 +199,12 @@ def __init__( fragment_shape = tuple(var.getFragDef()) + parsed_filename = urlparse(filename) + if parsed_filename.scheme in ("file", "http", "https"): + directory = str(PurePath(filename).parent) + else: + directory = PurePath(abspath(parsed_filename).path).parent + # Note: It is an as-yet-untested hypothesis that creating # the 'aggregated_data' dictionary for massive # aggretations (e.g. with O(10e6) fragments) will be @@ -198,6 +221,8 @@ def __init__( loc, aggregated_data, filename, + directory, + substitutions, non_standard_term, ) ) @@ -229,6 +254,11 @@ def __init__( self._set_component("instructions", instructions, copy=False) self._set_component("non_standard_term", non_standard_term, copy=False) + if substitutions is not None: + self._set_component( + "substitutions", substitutions.copy(), copy=False + ) + def __dask_tokenize__(self): """Used by `dask.base.tokenize`. @@ -252,7 +282,14 @@ def __getitem__(self, indices): return NotImplemented # pragma: no cover def _set_fragment( - self, var, frag_loc, aggregated_data, cfa_filename, non_standard_term + self, + var, + frag_loc, + aggregated_data, + cfa_filename, + directory, + substitutions, + non_standard_term, ): """Create a new key/value pair in the *aggregated_data* dictionary. @@ -278,6 +315,16 @@ def _set_fragment( cfa_filename: `str` TODOCFADOCS + directory: `str` + TODOCFADOCS + + .. versionadded:: TODOCFAVER + + substitutions: `dict` + TODOCFADOCS + + .. versionadded:: TODOCFAVER + non_standard_term: `str` or `None` The name of a non-standard aggregation instruction term from which the array is to be created, instead of @@ -301,11 +348,10 @@ def _set_fragment( aggregated_data[frag_loc] = { "format": "full", "location": location, - "full_value": getattr(fragment, non_standard_term), + "full_value": fragment.non_standard_term(non_standard_term), } return - # Still here? filename = fragment.file fmt = fragment.format address = fragment.address @@ -316,6 +362,18 @@ def _set_fragment( # This fragment is contained in the CFA-netCDF file filename = cfa_filename fmt = "nc" + else: + if substitutions: + # Apply string substitutions to the fragment + # filename + for base, sub in substitutions.items(): + filename = filename.replace(base, sub) + + parsed_filename = urlparse(filename) + if parsed_filename.scheme not in ("file", "http", "https"): + # Find the full path of a relative fragment + # filename + filename = os_join(directory, parsed_filename.path) aggregated_data[frag_loc] = { "format": fmt, diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index f80cbe18e2..7a5e534e8e 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -354,6 +354,10 @@ def _create_Data( def _customize_read_vars(self): """Customize the read parameters. + Take the opportunity to apply CFA updates to + `read_vars['variable_dimensions']` and + `read_vars['do_not_create_field']`. + .. versionadded:: 3.0.0 """ @@ -387,9 +391,10 @@ def _customize_read_vars(self): g["cfa"] = CFA_version is not None if g["cfa"]: # -------------------------------------------------------- - # This is a CFA-netCDF file, so check the CFA version and - # process the variables aggregated dimensions. + # This is a CFA-netCDF file # -------------------------------------------------------- + + # Check the CFA version g["CFA_version"] = Version(CFA_version) if g["CFA_version"] < Version("0.6.2"): raise ValueError( @@ -399,6 +404,15 @@ def _customize_read_vars(self): "write CFA-0.4 files.)" ) + # Get the pdirectory path of the CFA-netCDF file being + # read + from os.path import abspath + from pathlib import PurePath + + g["cfa_dir"] = PurePath(abspath(g["filename"])).parent + + # Process the aggregation instruction variables, and the + # aggregated dimensions. dimensions = g["variable_dimensions"] attributes = g["variable_attributes"] for ncvar, attributes in attributes.items(): @@ -506,6 +520,7 @@ def _create_cfanetcdfarray( ncvar, unpacked_dtype=False, coord_ncvar=None, + substitutions=None, non_standard_term=None, ): """Create a CFA-netCDF variable array. @@ -522,6 +537,11 @@ def _create_cfanetcdfarray( coord_ncvar: `str`, optional + substitutions: `dict`, optional + TODOCFADOCS + + .. versionadded:: TODOCFAVER + non_standard_term: `str`, optional The name of a non-standard aggregation instruction term from which to create the array. If set then @@ -537,6 +557,8 @@ def _create_cfanetcdfarray( kwargs used to create it. """ + g = self.read_vars + # Get the kwargs needed to instantiate a general NetCDFArray # instance kwargs = self._create_netcdfarray( @@ -561,6 +583,36 @@ def _create_cfanetcdfarray( ncvar ].get("aggregated_data") + # Find URI substitutions + parsed_aggregated_data = self._parse_aggregated_data( + ncvar, g["variable_attributes"][ncvar].get("aggregated_data") + ) + subs = {} + for x in parsed_aggregated_data: + term, term_ncvar = tuple(x.items())[0] + if term != "file": + continue + + subs = g["variable_attributes"][term_ncvar].get("substitutions") + if subs is None: + subs = {} + else: + # Convert, e.g., "${BASE}: a" to {"${BASE}": "a"} + subs = self.parse_x(term_ncvar, subs) + subs = { + key: value[0] for d in subs for key, value in d.items() + } + + break + + if substitutions: + # Include user-defined substitutions, which will overwrite + # any defined in the file with the same base name. + subs = subs.update(substitutions) + + if subs: + kwargs["substitutions"] = subs + # Use the kwargs to create a specialised CFANetCDFArray # instance array = self.implementation.initialise_CFANetCDFArray(**kwargs) @@ -657,7 +709,6 @@ def _parse_aggregated_data(self, ncvar, aggregated_data): ncvar, aggregated_data, keys_are_variables=True, - keys_are_dimensions=False, ) def _customize_field_ancillaries(self, parent_ncvar, f): @@ -711,7 +762,7 @@ def _customize_field_ancillaries(self, parent_ncvar, f): out = {} - attributes = g["variable_attributes"]["parent_ncvar"] + attributes = g["variable_attributes"][parent_ncvar] parsed_aggregated_data = self._parse_aggregated_data( parent_ncvar, attributes.get("aggregated_data") ) diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 8ee569899f..8faeafe8a0 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -671,15 +671,15 @@ def _ggg(self, data): # TODO move this to global once substitutions = tuple(substitutions.items())[::-1] - relative = g["cfa_options"].get("relative", None) + relative = g["cfa_options"].get("relative", None) if relative: absolute = False cfa_dir = PurePath(abspath(g["filename"])).parent elif relative is not None: absolute = True else: - absolute =None - + absolute = None + aggregation_file = [] aggregation_address = [] aggregation_format = [] @@ -738,14 +738,11 @@ def _customize_write_vars(self): .. versionadded:: TODOCFAVER - """ + """ g = self.write_vars - - if g.get('cfa'): + + if g.get("cfa"): from os.path import abspath from pathlib import PurePath - - g['cfa_dir'] = PurePath(abspath(g["filename"])).parent - - Need to know about this on read, too. + g["cfa_dir"] = PurePath(abspath(g["filename"])).parent From 273ec739e4398955de928ffc8047cef1b635171c Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 24 Feb 2023 16:06:36 +0000 Subject: [PATCH 025/141] dev --- cf/data/array/abstract/filearray.py | 30 ++-- cf/data/data.py | 41 +++++- .../fragment/mixin/fragmentfilearraymixin.py | 128 ++++++++++++------ cf/data/fragment/netcdffragmentarray.py | 43 +++++- cf/read_write/netcdf/netcdfwrite.py | 69 ++++++++-- 5 files changed, 231 insertions(+), 80 deletions(-) diff --git a/cf/data/array/abstract/filearray.py b/cf/data/array/abstract/filearray.py index 2f48b709fa..725b74ca45 100644 --- a/cf/data/array/abstract/filearray.py +++ b/cf/data/array/abstract/filearray.py @@ -72,21 +72,21 @@ def get_address(self): f"Must implement {self.__class__.__name__}.get_address" ) # pragma: no cover - def get_filename(self): - """Return the name of the file containing the array. - - :Returns: - - `str` or `None` - The filename, or `None` if there isn't one. - - **Examples** - - >>> a.get_filename() - 'file.nc' - - """ - return self._get_component("filename", None) + # def get_filename(self): + # """Return the name of the file containing the array. + # + # :Returns: + # + # `str` or `None` + # The filename, or `None` if there isn't one. + # + # **Examples** + # + # >>> a.get_filename() + # 'file.nc' + # + # """ + # return self._get_component("filename", None) def open(self): """Returns an open dataset containing the data array.""" diff --git a/cf/data/data.py b/cf/data/data.py index 9913aaea68..e8a6a5d373 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -2421,6 +2421,41 @@ def ceil(self, inplace=False, i=False): d._set_dask(da.ceil(dx)) return d + def cfa_add_fragment_locations(self, location): + """TODOCFADOCS + + :Parameters: + + location: `str` + + """ + if not self.get_cfa_write(): + raise ValueError("TODOCFA") + + from dask.base import collections_to_dsk + + dx = self.to_dask_array() + + dsk = collections_to_dsk((dx,), optimize_graph=True) + for key, a in dsk.items(): + try: + f = a.get_filenames() + except AttributeError: + continue + + # Switch out directory for new location in appended file + + # If multiple basenames exits, add one new lcoation for + # each + dsk[key] = a.add_fragment_lcoations(location, + inplace=False) + + # TODOCFA - don't do this inplace - see active storage + # for creating new dask array + + dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) + self._set_dask(dx, clear=_NONE) + def compute(self): # noqa: F811 """A numpy view the data. @@ -6014,13 +6049,13 @@ def get_filenames(self, address_format=False): dsk = collections_to_dsk((self.to_dask_array(),), optimize_graph=True) for a in dsk.values(): try: - f = a.get_filename() + f = a.get_filenames() if address_format: - f = ((f, a.get_address(), a.get_format()),) + f = ((f, a.get_addresses(), a.get_formats()),) except AttributeError: pass else: - out.add(f) + out.update(f) return out diff --git a/cf/data/fragment/mixin/fragmentfilearraymixin.py b/cf/data/fragment/mixin/fragmentfilearraymixin.py index f40cf33d29..e11068c99c 100644 --- a/cf/data/fragment/mixin/fragmentfilearraymixin.py +++ b/cf/data/fragment/mixin/fragmentfilearraymixin.py @@ -1,44 +1,84 @@ -# class FragmentFileArrayMixin: -# """Mixin class for a fragment array stored in a file. -# -# .. versionadded:: TODOCFAVER -# -# """ -# -# def get_address(self): -# """The address of the fragment in the file. -# -# .. versionadded:: TODOCFAVER -# -# :Returns: -# -# The file address of the fragment, or `None` if there -# isn't one. -# -# """ -# try: -# return self.get_array().get_address() -# except AttributeError: -# return -# -# def get_filename(self): -# """Return the name of the file containing the array. -# -# .. versionadded:: TODOCFAVER -# -# :Returns: -# -# `str` or `None` -# The filename, or `None` if there isn't one. -# -# **Examples** -# -# >>> a.get_filename() -# 'file.nc' -# -# """ -# try: -# return self.get_array().get_filename() -# except AttributeError: -# return -# +from ....decorators import ( + _inplace_enabled, + _inplace_enabled_define_and_cleanup, +) + + +class FragmentFileArrayMixin: + """Mixin class for a fragment array stored in a file. + + .. versionadded:: TODOCFAVER + + """ + + @_inplace_enabled(default=False) + def add_fragment_location(location, inplace=False): + """TODOCFADOCS""" + from os.path import basename, dirname, join + + a = _inplace_enabled_define_and_cleanup(self) + + # Note - it is assumed that all filenames are absolute paths + filenames = a.get_filenames() + addresses = a.get_addresses() + + new_filenames = tuple([join(location, basename(f)) + for f in filenames + if dirname(f) != location]) + + a._set_component('filename', filenames + new_filenames, copy=False) + a._set_component( + 'address', + addresses + addresses[-1] * len(new_filenames), + copy=False + ) + + return a + + def get_addresses(self): + """TODOCFADOCS Return the names of any files containing the data array. + + .. versionadded:: TODOCFAVER + + :Returns: + + `tuple` + The file names in normalised, absolute + form. TODOCFADOCS then an empty `set` is returned. + + """ + try: + return self._get_component("address") + except ValueError: + return () + + def get_filenames(self): + """TODOCFADOCS Return the names of any files containing the data array. + + .. versionadded:: TODOCFAVER + + :Returns: + + `tuple` + The file names in normalised, absolute + form. TODOCFADOCS then an empty `set` is returned. + + """ + try: + return self._get_component("filename") + except ValueError: + return () + + def get_formats(self): + """TODOCFADOCS Return the names of any files containing the data array. + + .. versionadded:: TODOCFAVER + + :Returns: + + `set` + The file names in normalised, absolute + form. TODOCFADOCS then an empty `set` is returned. + + """ + raise NotImplementedError diff --git a/cf/data/fragment/netcdffragmentarray.py b/cf/data/fragment/netcdffragmentarray.py index 2efa46d0b7..094b01c95e 100644 --- a/cf/data/fragment/netcdffragmentarray.py +++ b/cf/data/fragment/netcdffragmentarray.py @@ -1,3 +1,5 @@ +from urllib.parse import urlparse + from ..array.netcdfarray import NetCDFArray from .mixin import FragmentArrayMixin @@ -26,8 +28,8 @@ def __init__( :Parameters: - filename: `str` - The name of the netCDF fragment file containing the + filenames: `tuple` + The names of the netCDF fragment files containing the array. address: `str`, optional @@ -70,7 +72,7 @@ def __init__( group = None # TODO ??? super().__init__( - filename=filename, + filename=filenames, ncvar=address, group=group, dtype=dtype, @@ -101,3 +103,38 @@ def __init__( self._set_component( "aggregated_calendar", aggregated_calendar, copy=False ) + + def open(self): + """Returns an open dataset containing the data array. + + When multiple fragment files have been provided an attempt is + made to open each one, in arbitrary order, and the + `netCDF4.Dataset` is returned from the first success. + + .. versionadded:: TODOCFAVER + + :Returns: + + `netCDF4.Dataset` + + """ + filenames = self.get_filenames() + for filename, address in zip(filenames, self.get_addresses()): + url = urlparse(filename) + if url.scheme == "file": + # Convert file URI into an absolute path + filename = url.path + + try: + nc = netCDF4.Dataset(filename, "r") + except FileNotFoundError: + continue + except RuntimeError as error: + raise RuntimeError(f"{error}: {filename}") + + self._set_component("ncvar", address, copy=False) + return nc + + raise FileNotFoundError( + f"No such netCDF fragment files: {tuple(filenames)}" + ) diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 8faeafe8a0..410d7565ea 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -683,35 +683,74 @@ def _ggg(self, data): aggregation_file = [] aggregation_address = [] aggregation_format = [] + + # Maximum number of files defined on a fragments + max_files = 0 + for indices in data.chunk_indices(): a = self[indices].get_filenames(address_format=True) if len(a) != 1: raise ValueError("TODOCFADOCS") - filename, address, fmt = a.pop() + filenames, addresses, formats = a.pop() + + if len(filenames) > max_files: + max_files = len(filenames) + + filenames2 = [] + for filename in filenames: + parsed_filename = urlparse(filename) + scheme = parsed_filename.scheme + if scheme not in ("http", "https"): + path = parsed_filename.path + if absolute: + filename = PurePath(abspath(path)).as_uri() + elif relative or scheme != "file": + filename = relpath(abspath(path), start=cfa_dir) - parsed_filename = urlparse(filename) - scheme = parsed_filename.scheme - if scheme not in ("http", "https"): - path = parsed_filename.path - if absolute: - filename = PurePath(abspath(path)).as_uri() - elif relative or scheme != "file": - filename = relpath(abspath(path), start=cfa_dir) + if substitutions: + for base, sub in substitutions: + filename = filename.replace(sub, base) - if substitutions: - for base, sub in substitutions: - filename = filename.replace(sub, base) + filenames2.append(filename) - aggregation_file.append(filename) - aggregation_address.append(address) - aggregation_format.append(fmt) + aggregation_file.append(tuple(filenames2)) + aggregation_address.append(addresses) + aggregation_format.append(formats) shape = data.numblocks + + padded = False + if max_files > 1: + # Pad the ... + for i, (filenames, addresses, formats) in enumerate( + zip(aggregation_file, aggregation_address, aggregation_format) + ): + n = max_files - len(filenames) + if n: + pad = ("",) * n + aggregation_file[i] = filenames + pad + aggregation_address[i] = ( + addresses + pad + ) # worry about pad datatype + aggregation_format[i] = formats + pad + padded = True + + shape += (max_files,) + aggregation_file = np.array(aggregation_file).reshape(shape) aggregation_address = np.array(aggregation_address).reshape(shape) aggregation_format = np.array(aggregation_format).reshape(shape) + if padded: + # Mask padded elements + aggregation_file = np.ma.where( + aggregation_file == "", np.ma.masked, aggregation_file + ) + mask = aggregation_file.mask + aggregation_address = np.ma.array(aggregation_address, mask=mask) + aggregation_format = np.ma.array(aggregation_format, mask=mask) + # Location dtype = np.dtype(np.int32) if max(data.to_dask_array().chunksize) > np.iinfo(dtype).max: From 6f46805164ba12f745b1d18aeb0eeafce7b3cc63 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 24 Feb 2023 19:35:51 +0000 Subject: [PATCH 026/141] dev --- cf/data/data.py | 46 ++++---- .../fragment/mixin/fragmentfilearraymixin.py | 30 +++--- cf/data/fragment/netcdffragmentarray.py | 15 ++- cf/field.py | 31 ++++++ cf/mixin/propertiesdata.py | 30 ++++++ cf/mixin/propertiesdatabounds.py | 31 ++++++ cf/read_write/netcdf/netcdfwrite.py | 100 ++++++++++++++---- 7 files changed, 226 insertions(+), 57 deletions(-) diff --git a/cf/data/data.py b/cf/data/data.py index e8a6a5d373..7dce5088d5 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -29,6 +29,7 @@ from ..functions import ( _DEPRECATION_ERROR_KWARGS, _section, + abspath, atol, default_netCDF_fillvals, free_memory, @@ -2421,40 +2422,47 @@ def ceil(self, inplace=False, i=False): d._set_dask(da.ceil(dx)) return d - def cfa_add_fragment_locations(self, location): - """TODOCFADOCS + def cfa_add_fragment_location(self, location): + """TODOCFADOCS in-place. + + .. versionadded:: TODOCFAVER :Parameters: location: `str` + TODOCFADOCS - """ - if not self.get_cfa_write(): - raise ValueError("TODOCFA") + :Returns: + + `None` + + **Examples** + >>> d.cfa_add_fragment_location('/data/model') + + """ from dask.base import collections_to_dsk + location = abspath(location) + dx = self.to_dask_array() - dsk = collections_to_dsk((dx,), optimize_graph=True) + updated = False + dsk = collections_to_dsk((dx,), optimize_graph=True) for key, a in dsk.items(): try: - f = a.get_filenames() + a.get_filenames() except AttributeError: + # This chunk doesn't contain a CFA fragment file continue + else: + # This chunk contains a CFA fragment file + dsk[key] = a.add_fragment_location(location, inplace=False) + updated = True - # Switch out directory for new location in appended file - - # If multiple basenames exits, add one new lcoation for - # each - dsk[key] = a.add_fragment_lcoations(location, - inplace=False) - - # TODOCFA - don't do this inplace - see active storage - # for creating new dask array - - dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) - self._set_dask(dx, clear=_NONE) + if updated: + dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) + self._set_dask(dx, clear=_NONE) def compute(self): # noqa: F811 """A numpy view the data. diff --git a/cf/data/fragment/mixin/fragmentfilearraymixin.py b/cf/data/fragment/mixin/fragmentfilearraymixin.py index e11068c99c..5642c743d6 100644 --- a/cf/data/fragment/mixin/fragmentfilearraymixin.py +++ b/cf/data/fragment/mixin/fragmentfilearraymixin.py @@ -12,27 +12,31 @@ class FragmentFileArrayMixin: """ @_inplace_enabled(default=False) - def add_fragment_location(location, inplace=False): + def add_fragment_location(self, location, inplace=False): """TODOCFADOCS""" - from os.path import basename, dirname, join - + from os.path import basename, dirname, join + a = _inplace_enabled_define_and_cleanup(self) # Note - it is assumed that all filenames are absolute paths filenames = a.get_filenames() addresses = a.get_addresses() - - new_filenames = tuple([join(location, basename(f)) - for f in filenames - if dirname(f) != location]) - a._set_component('filename', filenames + new_filenames, copy=False) + new_filenames = tuple( + [ + join(location, basename(f)) + for f in filenames + if dirname(f) != location + ] + ) + + a._set_component("filename", filenames + new_filenames, copy=False) a._set_component( - 'address', + "address", addresses + addresses[-1] * len(new_filenames), - copy=False + copy=False, ) - + return a def get_addresses(self): @@ -51,7 +55,7 @@ def get_addresses(self): return self._get_component("address") except ValueError: return () - + def get_filenames(self): """TODOCFADOCS Return the names of any files containing the data array. @@ -68,7 +72,7 @@ def get_filenames(self): return self._get_component("filename") except ValueError: return () - + def get_formats(self): """TODOCFADOCS Return the names of any files containing the data array. diff --git a/cf/data/fragment/netcdffragmentarray.py b/cf/data/fragment/netcdffragmentarray.py index 094b01c95e..34a171e3d4 100644 --- a/cf/data/fragment/netcdffragmentarray.py +++ b/cf/data/fragment/netcdffragmentarray.py @@ -1,5 +1,7 @@ from urllib.parse import urlparse +import netCDF4 + from ..array.netcdfarray import NetCDFArray from .mixin import FragmentArrayMixin @@ -13,7 +15,7 @@ class NetCDFFragmentArray(FragmentArrayMixin, NetCDFArray): def __init__( self, - filename=None, + filenames=None, address=None, dtype=None, shape=None, @@ -85,6 +87,12 @@ def __init__( ) if source is not None: + try: + address = source._get_component( "address", False + ) + except AttributeError: + address = None + try: aggregated_units = source._get_component( "aggregated_units", False @@ -99,6 +107,9 @@ def __init__( except AttributeError: aggregated_calendar = False + if address is not None: + self._set_component("address", address, copy=False) + self._set_component("aggregated_units", aggregated_units, copy=False) self._set_component( "aggregated_calendar", aggregated_calendar, copy=False @@ -134,7 +145,7 @@ def open(self): self._set_component("ncvar", address, copy=False) return nc - + raise FileNotFoundError( f"No such netCDF fragment files: {tuple(filenames)}" ) diff --git a/cf/field.py b/cf/field.py index 551f10ac7a..ffa9746c46 100644 --- a/cf/field.py +++ b/cf/field.py @@ -3641,6 +3641,37 @@ def cell_area( return w + @_inplace_enabled(default=False) + def cfa_add_fragment_location(self, location, inplace=False): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + location: `str` + TODOCFADOCS + + {{inplace: `bool`, optional}} + + :Returns: + + `None` + + **Examples** + + >>> f.cfa_add_fragment_location('/data/model') + + """ + f = _inplace_enabled_define_and_cleanup(self) + + super().add_fragment_location(location, inplace=True) + + for c in f.constructs.filter_by_data(todict=True).values(): + c.add_fragment_location(location, inplace=True) + + return f + def radius(self, default=None): """Return the radius of a latitude-longitude plane defined in spherical polar coordinates. diff --git a/cf/mixin/propertiesdata.py b/cf/mixin/propertiesdata.py index 2721211530..35e08bc7e8 100644 --- a/cf/mixin/propertiesdata.py +++ b/cf/mixin/propertiesdata.py @@ -2473,6 +2473,36 @@ def ceil(self, inplace=False, i=False): delete_props=True, ) + @_inplace_enabled(default=False) + def cfa_add_fragment_location(self, location, inplace=False): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + location: `str` + TODOCFADOCS + + {{inplace: `bool`, optional}} + + :Returns: + + `None` + + **Examples** + + >>> f.cfa_add_fragment_location('/data/model') + + """ + f = _inplace_enabled_define_and_cleanup(self) + + data = f.get_data(None) + if data is not None: + data.add_fragment_location(location, inplace=True) + + return f + def chunk(self, chunksize=None): """Partition the data array. diff --git a/cf/mixin/propertiesdatabounds.py b/cf/mixin/propertiesdatabounds.py index 0fa5d716bc..d14a95325b 100644 --- a/cf/mixin/propertiesdatabounds.py +++ b/cf/mixin/propertiesdatabounds.py @@ -1182,6 +1182,37 @@ def ceil(self, bounds=True, inplace=False, i=False): i=i, ) + @_inplace_enabled(default=False) + def cfa_add_fragment_location(self, location, inplace=False): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + location: `str` + TODOCFADOCS + + {{inplace: `bool`, optional}} + + :Returns: + + `None` + + **Examples** + + >>> c.cfa_add_fragment_location('/data/model') + + """ + return self._apply_superclass_data_oper( + _inplace_enabled_define_and_cleanup(self), + "cfa_add_fragment_location", + (location,), + bounds=True, + interior_ring=True, + inplace=inplace, + ) + def chunk(self, chunksize=None): """Partition the data array. diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 410d7565ea..4cb212ee30 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -359,6 +359,8 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): `None` """ + g = self.write_vars + ggg = self._ggg(data) # Get the location netCDF dimensions @@ -374,8 +376,9 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): location_ncdimensions.append(l_ncdim) # Get the fragment netCDF dimensions + aggregation_address = ggg["aggregation_address"] fragment_ncdimensions = [] - for ncdim, size in zip(ncdimensions, ggg["address"].shape): + for ncdim, size in zip(ncdimensions, aggregation_address.shape): f_ncdim = self._netcdf_name( f"f_{ncdim}", dimsize=size, role="cfa_fragment" ) @@ -385,37 +388,56 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): fragment_ncdimensions.append(f_ncdim) + ndim = aggregation_address.ndim + if ndim == len(ncdimensions) + 1: + # Include an extra trailing dimension for the aggregation + # instruction variables + size = aggregation_address.shape[-1] + f_ncdim = self._netcdf_name( + "f_extra", dimsize=size, role="cfa_fragment" + ) + self._write_dimension(f_ncdim, None, size=size) + fragment_ncdimensions.append(f_ncdim) + + # ------------------------------------------------------------ + # Write the standardised aggregation instruction variables to + # the CFA-netCDF file + # ------------------------------------------------------------ aggregated_data = [] - for term, d in ggg.items(): + for term, data in ggg.items(): if term == "location": dimensions = location_ncdimensions else: dimensions = fragment_ncdimensions + + # Attempt to reduce formats to a common scalar value if term == "format": - u = d.unique().persist() + u = data.unique().compressed().persist() if u.size == 1: - # Collapse formats to a common scalar - d = u.squeeze() + data = u.squeeze() dimensions = () term_ncvar = self._cfa_write_term_variable( - d, + data, f"cfa_{term}", dimensions, ) aggregated_data.append(f"{term}: {term_ncvar}") + # ------------------------------------------------------------ # Look for non-standard CFA terms stored as field ancillaries - # on a field + # on a field and write them to the CFA-netCDF file + # ------------------------------------------------------------ if self.implementation.is_field(cfvar): - aggregated_data.extend( - self._cfa_write_non_standard_terms( - cfvar, fragment_ncdimensions - ) + non_standard_terms = self._cfa_write_non_standard_terms( + cfvar, fragment_ncdimensions[:ndim] ) + aggregated_data.extend(non_standard_terms) + # ------------------------------------------------------------ # Add the CFA aggregation variable attributes + # ------------------------------------------------------------ self._write_attributes( None, ncvar, @@ -565,6 +587,11 @@ def _cfa_write_term_variable(self, data, ncvar, ncdimensions): """TODOCFADOCS. .. versionadded:: TODOCFAVER + + :Returns: + + `list` + """ create = not self._already_in_file(data, ncdimensions) @@ -581,9 +608,11 @@ def _cfa_write_term_variable(self, data, ncvar, ncdimensions): def _cfa_write_non_standard_terms(self, cfvar, fragment_ncdimensions): """TODOCFADOCS + Look for non-standard CFA terms stored as field ancillaries + .. versionadded:: TODOCFAVER + """ - # Look for non-standard CFA terms stored as field ancillaries aggregated_data = [] non_standard_terms = [] for key, field_anc in self.implementation.get_field_ancillaries( @@ -596,12 +625,18 @@ def _cfa_write_non_standard_terms(self, cfvar, fragment_ncdimensions): if not data.get_cfa_write(): continue + # Check that the field ancillary has the same axes as the + # parent field, and in the same order. if cfvar.get_data_axes(key) != cfvar.get_data_axes(): continue - # Still here? Then convert the data to span the fragment - # dimensions, with one value per fragment, and then write - # it to disk. + # Still here? Then this field ancillary represent a + # non-standard aggregation term. + + # Then transform the data so that it spans the fragment + # dimensions, with one value per fragment. If a chunk has + # more than one unique value then the fragment's value is + # missing data. dx = data.to_dask_array() dx_ind = tuple(range(dx.ndim)) out_ind = dx_ind @@ -613,7 +648,7 @@ def _cfa_write_non_standard_terms(self, cfvar, fragment_ncdimensions): adjust_chunks={i: 1 for i in out_ind}, dtype=dx.dtype, ) - field_anc.set_data(dx) + array = dx.compute() # Get the non-standard term name from the field # ancillary's 'id' attribute @@ -625,8 +660,9 @@ def _cfa_write_non_standard_terms(self, cfvar, fragment_ncdimensions): n += 1 term = f"{base}_{n}" + # Create the new CFA term variable term_ncvar = self._cfa_write_term_variable( - field_anc.data, f"cfa_{term}", fragment_ncdimensions + type(data)(array), f"cfa_{term}", fragment_ncdimensions ) aggregated_data.append(f"{term}: {term_ncvar}") @@ -638,9 +674,25 @@ def _cfa_unique(cls, a): """TODOCFADOCS. .. versionadded:: TODOCFAVER + + :Parameters: + + a: `numpy.ndarray` + The array. + + :Returns: + + `numpy.ndarray` + A size 1 array containg the unique value, or missing + data if there is not a unique unique value. + """ out_shape = (1,) * a.ndim a = np.unique(a) + if np.ma.isMA(a): + # Remove a masked element + a = a.compressed() + if a.size == 1: return a.reshape(out_shape) @@ -649,6 +701,8 @@ def _cfa_unique(cls, a): def _ggg(self, data): """ + .. versionadded:: TODOCFAVER + f = cf.example_field(0) cf.write(f, "file_A.nc") cf.write(f, "file_B.nc") @@ -720,7 +774,7 @@ def _ggg(self, data): shape = data.numblocks - padded = False + pad = None if max_files > 1: # Pad the ... for i, (filenames, addresses, formats) in enumerate( @@ -730,11 +784,11 @@ def _ggg(self, data): if n: pad = ("",) * n aggregation_file[i] = filenames + pad - aggregation_address[i] = ( - addresses + pad - ) # worry about pad datatype aggregation_format[i] = formats + pad - padded = True + if isinstance(addresses[0], int): + pad = (-1,) * n + + aggregation_address[i] = addresses + pad shape += (max_files,) @@ -742,7 +796,7 @@ def _ggg(self, data): aggregation_address = np.array(aggregation_address).reshape(shape) aggregation_format = np.array(aggregation_format).reshape(shape) - if padded: + if pad: # Mask padded elements aggregation_file = np.ma.where( aggregation_file == "", np.ma.masked, aggregation_file From 7cbb5b3274e3b3acfcd265cc3532e9b65f38d344 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 27 Feb 2023 16:31:34 +0000 Subject: [PATCH 027/141] dev --- cf/aggregate.py | 1 - cf/data/data.py | 8 +- cf/data/fragment/netcdffragmentarray.py | 5 +- cf/mixin2/__init__.py | 2 + cf/mixin2/container.py | 45 +++++ cf/mixin2/netcdf.py | 241 ++++++++++++++++++++++++ cf/read_write/netcdf/netcdfread.py | 31 ++- cf/read_write/netcdf/netcdfwrite.py | 36 +++- 8 files changed, 350 insertions(+), 19 deletions(-) create mode 100644 cf/mixin2/__init__.py create mode 100644 cf/mixin2/container.py create mode 100644 cf/mixin2/netcdf.py diff --git a/cf/aggregate.py b/cf/aggregate.py index a9f4325ed0..dd2c3f6654 100644 --- a/cf/aggregate.py +++ b/cf/aggregate.py @@ -398,7 +398,6 @@ def __init__( "coordrefs": self.find_coordrefs(axis), } ) - # 'size' : None}) # Find the 1-d auxiliary coordinates which span this axis aux_coords = { diff --git a/cf/data/data.py b/cf/data/data.py index 7dce5088d5..e45629680d 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -36,7 +36,9 @@ parse_indices, rtol, ) -from ..mixin_container import Container + +# from ..mixin_container import Container +from ..mixin2 import CFANetCDF, Container from ..units import Units from .collapse import Collapse from .creation import ( # is_file_array, @@ -99,10 +101,10 @@ _ARRAY = 1 # = 0b000001 _CACHE = 2 # = 0b000010 _CFA = 4 # = 0b000100 -_ALL = 63 # = 0b111111 +_ALL = 7 # = 0b000111 -class Data(DataClassDeprecationsMixin, Container, cfdm.Data): +class Data(DataClassDeprecationsMixin, CFANetCDF, Container, cfdm.Data): """An N-dimensional data array with units and masked values. * Contains an N-dimensional, indexable and broadcastable array with diff --git a/cf/data/fragment/netcdffragmentarray.py b/cf/data/fragment/netcdffragmentarray.py index 34a171e3d4..a34b5207ab 100644 --- a/cf/data/fragment/netcdffragmentarray.py +++ b/cf/data/fragment/netcdffragmentarray.py @@ -88,8 +88,7 @@ def __init__( if source is not None: try: - address = source._get_component( "address", False - ) + address = source._get_component("address", False) except AttributeError: address = None @@ -109,7 +108,7 @@ def __init__( if address is not None: self._set_component("address", address, copy=False) - + self._set_component("aggregated_units", aggregated_units, copy=False) self._set_component( "aggregated_calendar", aggregated_calendar, copy=False diff --git a/cf/mixin2/__init__.py b/cf/mixin2/__init__.py new file mode 100644 index 0000000000..4394d414c6 --- /dev/null +++ b/cf/mixin2/__init__.py @@ -0,0 +1,2 @@ +from .netcdf import CFANetCDF +from .container import Container diff --git a/cf/mixin2/container.py b/cf/mixin2/container.py new file mode 100644 index 0000000000..460ce61220 --- /dev/null +++ b/cf/mixin2/container.py @@ -0,0 +1,45 @@ +"""This class is not in the cf.mixin package because it needs to be +imported by cf.Data, and some of the other mixin classes in cf.mixin +themsleves import cf.Data, which would lead to a circular import +situation. + +""" +from .docstring import _docstring_substitution_definitions + + +class Container: + """Mixin class for storing components. + + .. versionadded:: 3.7.0 + + """ + + def __docstring_substitutions__(self): + """Define docstring substitutions that apply to this class and + all of its subclasses. + + These are in addtion to, and take precendence over, docstring + substitutions defined by the base classes of this class. + + See `_docstring_substitutions` for details. + + .. versionadded:: 3.7.0 + + .. seealso:: `_docstring_substitutions` + + :Returns: + + `dict` + The docstring substitutions that have been applied. + + """ + return _docstring_substitution_definitions + + def __docstring_package_depth__(self): + """Return the package depth for {{package}} docstring + substitutions. + + See `_docstring_package_depth` for details. + + """ + return 0 diff --git a/cf/mixin2/netcdf.py b/cf/mixin2/netcdf.py new file mode 100644 index 0000000000..8e28da2a5e --- /dev/null +++ b/cf/mixin2/netcdf.py @@ -0,0 +1,241 @@ +"""This class is not in the cf.mixin package because it needs to be +imported by cf.Data, and some of the other mixin classes in cf.mixin +themsleves import cf.Data, which would lead to a circular import +situation. + +""" + + +class CFANetCDF: + """Mixin class for accessing CFA-netCDF aggregation instruction terms. + + .. versionadded:: TODOCFAVER + + """ + + def nc_del_cfa_aggregation_data(self, default=ValueError()): + """Remove the CFA-netCDF aggregation instruction terms. + + .. versionadded:: TODOCFAVER + + .. seealso:: `nc_get_cfa_aggregation_data`, + `nc_has_cfa_aggregation_data`, + `nc_set_cfa_aggregation_data` + + :Parameters: + + default: optional + Return the value of the *default* parameter if the + CFA-netCDF aggregation terms have not been set. If set + to an `Exception` instance then it will be raised + instead. + + :Returns: + + `dict` + The removed CFA-netCDF aggregation instruction terms. + + **Examples** + + >>> f.nc_set_cfa_aggregation_data( + ... {'location': 'cfa_location', + ... 'file': 'cfa_file', + ... 'address': 'cfa_address', + ... 'format': 'cfa_format', + ... 'tracking_id': 'tracking_id'} + ... ) + >>> f.nc_has_cfa_aggregation_data() + True + >>> f.nc_get_cfa_aggregation_data() + {'location': 'cfa_location', + 'file': 'cfa_file', + 'address': 'cfa_address', + 'format': 'cfa_format', + 'tracking_id': 'tracking_id'} + >>> f.nc_del_cfa_aggregation_data() + {'location': 'cfa_location', + 'file': 'cfa_file', + 'address': 'cfa_address', + 'format': 'cfa_format', + 'tracking_id': 'tracking_id'} + >>> f.nc_has_cfa_aggregation_data() + False + >>> print(f.nc_get_cfa_aggregation_data(None)) + None + >>> print(f.nc_del_cfa_aggregation_data(None)) + None + + """ + return self._nc_del("dimension", default=default) + + def nc_get_cfa_aggregation_data(self, default=ValueError()): + """Return the CFA-netCDF aggregation instruction terms. + + .. versionadded:: TODOCFAVER + + .. seealso:: `nc_del_cfa_aggregation_data`, + `nc_has_cfa_aggregation_data`, + `nc_set_cfa_aggregation_data` + + :Parameters: + + default: optional + Return the value of the *default* parameter if the + CFA-netCDF aggregation terms have not been set. If set + to an `Exception` instance then it will be raised + instead. + + :Returns: + + `dict` + The aggregation instruction terms and their + corresponding netCDF variable names. + + **Examples** + + >>> f.nc_set_cfa_aggregation_data( + ... {'location': 'cfa_location', + ... 'file': 'cfa_file', + ... 'address': 'cfa_address', + ... 'format': 'cfa_format', + ... 'tracking_id': 'tracking_id'} + ... ) + >>> f.nc_has_cfa_aggregation_data() + True + >>> f.nc_get_cfa_aggregation_data() + {'location': 'cfa_location', + 'file': 'cfa_file', + 'address': 'cfa_address', + 'format': 'cfa_format', + 'tracking_id': 'tracking_id'} + >>> f.nc_del_cfa_aggregation_data() + {'location': 'cfa_location', + 'file': 'cfa_file', + 'address': 'cfa_address', + 'format': 'cfa_format', + 'tracking_id': 'tracking_id'} + >>> f.nc_has_cfa_aggregation_data() + False + >>> print(f.nc_get_cfa_aggregation_data(None)) + None + >>> print(f.nc_del_cfa_aggregation_data(None)) + None + + """ + out = self._nc_get("cfa_aggregation_data", default=None) + if out is not None: + return out.copy() + + if default is None: + return default + + return self._default( + default, + f"{self.__class__.__name__} has no CFA-netCDF aggregation terms", + ) + + def nc_has_cfa_aggregation_data(self): + """Whether the CFA-netCDF aggregation instruction terms have been set. + + .. versionadded:: TODOCFAVER + + .. seealso:: `nc_del_cfa_aggregation_data`, + `nc_get_cfa_aggregation_data`, + `nc_set_cfa_aggregation_data` + + :Returns: + + `bool` + `True` if the CFA-netCDF aggregation instruction terms + have been set, otherwise `False`. + + **Examples** + + >>> f.nc_set_cfa_aggregation_data( + ... {'location': 'cfa_location', + ... 'file': 'cfa_file', + ... 'address': 'cfa_address', + ... 'format': 'cfa_format', + ... 'tracking_id': 'tracking_id'} + ... ) + >>> f.nc_has_cfa_aggregation_data() + True + >>> f.nc_get_cfa_aggregation_data() + {'location': 'cfa_location', + 'file': 'cfa_file', + 'address': 'cfa_address', + 'format': 'cfa_format', + 'tracking_id': 'tracking_id'} + >>> f.nc_del_cfa_aggregation_data() + {'location': 'cfa_location', + 'file': 'cfa_file', + 'address': 'cfa_address', + 'format': 'cfa_format', + 'tracking_id': 'tracking_id'} + >>> f.nc_has_cfa_aggregation_data() + False + >>> print(f.nc_get_cfa_aggregation_data(None)) + None + >>> print(f.nc_del_cfa_aggregation_data(None)) + None + + """ + return self._nc_has("cfa_aggregation_data") + + def nc_set_cfa_aggregation_data(self, value): + """Set the CFA-netCDF aggregation instruction terms. + + If there are any ``/`` (slash) characters in the netCDF + variable names then these act as delimiters for a group + hierarchy. By default, or if the name starts with a ``/`` + character and contains no others, the name is assumed to be in + the root group. + + .. versionadded:: TODOCFAVER + + .. seealso:: `nc_del_cfa_aggregation_data`, + `nc_get_cfa_aggregation_data`, + `nc_has_cfa_aggregation_data` + + :Parameters: + + value: `dict` + The aggregation instruction terms and their + corresponding netCDF variable names. + + :Returns: + + `None` + + **Examples** + + >>> f.nc_set_cfa_aggregation_data( + ... {'location': 'cfa_location', + ... 'file': 'cfa_file', + ... 'address': 'cfa_address', + ... 'format': 'cfa_format', + ... 'tracking_id': 'tracking_id'} + ... ) + >>> f.nc_has_cfa_aggregation_data() + True + >>> f.nc_get_cfa_aggregation_data() + {'location': 'cfa_location', + 'file': 'cfa_file', + 'address': 'cfa_address', + 'format': 'cfa_format', + 'tracking_id': 'tracking_id'} + >>> f.nc_del_cfa_aggregation_data() + {'location': 'cfa_location', + 'file': 'cfa_file', + 'address': 'cfa_address', + 'format': 'cfa_format', + 'tracking_id': 'tracking_id'} + >>> f.nc_has_cfa_aggregation_data() + False + >>> print(f.nc_get_cfa_aggregation_data(None)) + None + >>> print(f.nc_del_cfa_aggregation_data(None)) + None + + """ + return self._nc_set("cfa_aggregation_data", value.copy()) diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 7a5e534e8e..5d3291dc90 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -1,3 +1,5 @@ +from re import split + import cfdm import numpy as np from packaging.version import Version @@ -226,8 +228,14 @@ def _create_data( if construct is not None: # Remove the aggregation attributes from the construct # properties - for attr in ("aggregation_dimensions", "aggregation_data"): - self.implementation.del_property(construct, attr, None) + # for attr in ("aggregation_dimensions", "aggregation_data"): + # self.implementation.del_property(construct, attr, None) + self.implementation.del_property( + construct, "aggregation_dimensions", None + ) + aggregation_data = self.implementation.del_property( + construct, "aggregation_data", None + ) if not cfa_term: cfa_array, kwargs = self._create_cfanetcdfarray( @@ -257,12 +265,27 @@ def _create_data( cfa_array.get_fragment_shape(), data.numblocks ): if n == 1 and numblocks > 1: - # Each fragment spans multiple compute chunks + # Each fragment spans multiple compute + # chunks. + # + # Note: We test on 'n == 1' because we're assuming + # that each fragment already spans one chunk + # along those axes for whioch 'n > 1'. See + # `CFANetCDFArray.to_dask_array` for + # details. cfa_write = False break data._set_cfa_write(cfa_write) + # Store the 'aggregation_data' attribute + if aggregation_data: + ad = split("\s+", aggregation_data) + aggregation_data = { + k[:-1]: v for k, v in zip(ad[::2], ad[1::2]) + } + data.nc_set_cfa_aggregation_data(aggregation_data) + # Note: We don't cache elements from aggregated data return data @@ -767,8 +790,10 @@ def _customize_field_ancillaries(self, parent_ncvar, f): parent_ncvar, attributes.get("aggregated_data") ) standardised_terms = self.cfa_standard_terms() + # cfa_terms = {} for x in parsed_aggregated_data: term, ncvar = tuple(x.items())[0] + # cfa_terms[term] = ncvar if term in standardised_terms: continue diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 4cb212ee30..7624c08078 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -396,13 +396,17 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): f_ncdim = self._netcdf_name( "f_extra", dimsize=size, role="cfa_fragment" ) - self._write_dimension(f_ncdim, None, size=size) + if f_ncdim not in g["dimensions"]: + self._write_dimension(f_ncdim, None, size=size) + fragment_ncdimensions.append(f_ncdim) # ------------------------------------------------------------ # Write the standardised aggregation instruction variables to # the CFA-netCDF file # ------------------------------------------------------------ + aggregation_data = data.nc_get_cfa_aggregation_data(default={}) + aggregated_data = [] for term, data in ggg.items(): if term == "location": @@ -419,7 +423,7 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): term_ncvar = self._cfa_write_term_variable( data, - f"cfa_{term}", + aggregation_data.get(term, f"cfa_{term}"), dimensions, ) @@ -431,7 +435,7 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): # ------------------------------------------------------------ if self.implementation.is_field(cfvar): non_standard_terms = self._cfa_write_non_standard_terms( - cfvar, fragment_ncdimensions[:ndim] + cfvar, fragment_ncdimensions[:ndim], aggregation_data ) aggregated_data.extend(non_standard_terms) @@ -605,18 +609,28 @@ def _cfa_write_term_variable(self, data, ncvar, ncdimensions): return ncvar - def _cfa_write_non_standard_terms(self, cfvar, fragment_ncdimensions): + def _cfa_write_non_standard_terms( + self, field, fragment_ncdimensions, aggregation_data + ): """TODOCFADOCS Look for non-standard CFA terms stored as field ancillaries .. versionadded:: TODOCFAVER + :Parameters: + + field: `Field` + + fragment_ncdimensions: `list` of `str` + + aggregation_data: `dict` + """ aggregated_data = [] - non_standard_terms = [] + terms = ["location", "file", "address", "format"] for key, field_anc in self.implementation.get_field_ancillaries( - cfvar + field ).items(): if not field_anc._custom.get("cfa_term", False): continue @@ -627,7 +641,7 @@ def _cfa_write_non_standard_terms(self, cfvar, fragment_ncdimensions): # Check that the field ancillary has the same axes as the # parent field, and in the same order. - if cfvar.get_data_axes(key) != cfvar.get_data_axes(): + if field.get_data_axes(key) != field.get_data_axes(): continue # Still here? Then this field ancillary represent a @@ -656,13 +670,17 @@ def _cfa_write_non_standard_terms(self, cfvar, fragment_ncdimensions): term = term.replace(" ", "_") base = term n = 0 - while term in non_standard_terms: + while term in terms: n += 1 term = f"{base}_{n}" + terms.append(term) + # Create the new CFA term variable term_ncvar = self._cfa_write_term_variable( - type(data)(array), f"cfa_{term}", fragment_ncdimensions + type(data)(array), + aggregation_data.get(term, f"cfa_{term}"), + fragment_ncdimensions, ) aggregated_data.append(f"{term}: {term_ncvar}") From 60aeb353b71ae4f44504c3a1540213620d3ae261 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 27 Feb 2023 19:07:01 +0000 Subject: [PATCH 028/141] dev --- cf/data/data.py | 50 +++--- cf/mixin2/netcdf.py | 261 ++++++++++++++++++++++------ cf/read_write/netcdf/netcdfread.py | 7 +- cf/read_write/netcdf/netcdfwrite.py | 47 +++-- 4 files changed, 269 insertions(+), 96 deletions(-) diff --git a/cf/data/data.py b/cf/data/data.py index e45629680d..1a9edcd2cd 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -3772,31 +3772,47 @@ def concatenate(cls, data, axis=0, cull_graph=True): dx = da.concatenate(dxs, axis=axis) # Set the CFA write status - CFA = _CFA + # + # Assume at first that all input data instance have True + # status, but ... + cfa = _CFA for d in processed_data: if not d.get_cfa_write(): - # Set the CFA write status to False when any input + # ... the CFA write status is False when any input # data instance has False status - CFA = _NONE + cfa = _NONE break - if not CFA: + if cfa != _NONE: non_concat_axis_chunks0 = list(processed_data[0].chunks) non_concat_axis_chunks0.pop(axis) for d in processed_data[1:]: non_concat_axis_chunks = list(d.chunks) non_concat_axis_chunks.pop(axis) if non_concat_axis_chunks != non_concat_axis_chunks0: - # Set the CFA write status to False when input - # data instances have different chunk patterns for - # the non-concatenated axes - CFA = _NONE + # ... the CFA write status is False when any two + # input data instances have different chunk + # patterns for the non-concatenated axes + cfa = _NONE break # Set the new dask array - data0._set_dask(dx, clear=_ALL ^ CFA) - - # Manage cyclicity of axes: if join axis was cyclic, it is no longer + data0._set_dask(dx, clear=_ALL ^ cfa) + + # Set the CFA-netCDF aggregated_data instructions, giving + # precedence to those towards the left hand side of the input + # list. + if data0.get_cfa_write(): + aggregated_data = {} + for d in processed_data[::-1]: + if d.get_cfa_write(): + aggregated_data.update(d.nc_get_cfa_aggregated_data({})) + + if aggregated_data: + data0.nc_set_cfa_aggregated_data(aggregated_data) + + # Manage cyclicity of axes: if join axis was cyclic, it is no + # longer. axis = data0._parse_axes(axis)[0] if axis in data0.cyclic(): logger.warning( @@ -5971,18 +5987,6 @@ def get_cfa_write(self): """ return self._custom.get("cfa_write", False) - # def get_data(self, default=ValueError(), _units=None, _fill_value=None): - # """Returns the data.## - # - # .. versionadded:: 3.0.0# - # - # :Returns:## - # - # `Data`## - # - # """ - return self - def get_filenames(self, address_format=False): """The names of files containing parts of the data array. diff --git a/cf/mixin2/netcdf.py b/cf/mixin2/netcdf.py index 8e28da2a5e..71952f2623 100644 --- a/cf/mixin2/netcdf.py +++ b/cf/mixin2/netcdf.py @@ -13,14 +13,14 @@ class CFANetCDF: """ - def nc_del_cfa_aggregation_data(self, default=ValueError()): + def nc_del_cfa_aggregated_data(self, default=ValueError()): """Remove the CFA-netCDF aggregation instruction terms. .. versionadded:: TODOCFAVER - .. seealso:: `nc_get_cfa_aggregation_data`, - `nc_has_cfa_aggregation_data`, - `nc_set_cfa_aggregation_data` + .. seealso:: `nc_get_cfa_aggregated_data`, + `nc_has_cfa_aggregated_data`, + `nc_set_cfa_aggregated_data` :Parameters: @@ -37,45 +37,45 @@ def nc_del_cfa_aggregation_data(self, default=ValueError()): **Examples** - >>> f.nc_set_cfa_aggregation_data( + >>> f.nc_set_cfa_aggregated_data( ... {'location': 'cfa_location', ... 'file': 'cfa_file', ... 'address': 'cfa_address', ... 'format': 'cfa_format', ... 'tracking_id': 'tracking_id'} ... ) - >>> f.nc_has_cfa_aggregation_data() + >>> f.nc_has_cfa_aggregated_data() True - >>> f.nc_get_cfa_aggregation_data() + >>> f.nc_get_cfa_aggregated_data() {'location': 'cfa_location', 'file': 'cfa_file', 'address': 'cfa_address', 'format': 'cfa_format', 'tracking_id': 'tracking_id'} - >>> f.nc_del_cfa_aggregation_data() + >>> f.nc_del_cfa_aggregated_data() {'location': 'cfa_location', 'file': 'cfa_file', 'address': 'cfa_address', 'format': 'cfa_format', 'tracking_id': 'tracking_id'} - >>> f.nc_has_cfa_aggregation_data() + >>> f.nc_has_cfa_aggregated_data() False - >>> print(f.nc_get_cfa_aggregation_data(None)) + >>> print(f.nc_get_cfa_aggregated_data(None)) None - >>> print(f.nc_del_cfa_aggregation_data(None)) + >>> print(f.nc_del_cfa_aggregated_data(None)) None """ - return self._nc_del("dimension", default=default) + return self._nc_del("cfa_aggregated_data", default=default) - def nc_get_cfa_aggregation_data(self, default=ValueError()): + def nc_get_cfa_aggregated_data(self, default=ValueError()): """Return the CFA-netCDF aggregation instruction terms. .. versionadded:: TODOCFAVER - .. seealso:: `nc_del_cfa_aggregation_data`, - `nc_has_cfa_aggregation_data`, - `nc_set_cfa_aggregation_data` + .. seealso:: `nc_del_cfa_aggregated_data`, + `nc_has_cfa_aggregated_data`, + `nc_set_cfa_aggregated_data` :Parameters: @@ -93,36 +93,36 @@ def nc_get_cfa_aggregation_data(self, default=ValueError()): **Examples** - >>> f.nc_set_cfa_aggregation_data( + >>> f.nc_set_cfa_aggregated_data( ... {'location': 'cfa_location', ... 'file': 'cfa_file', ... 'address': 'cfa_address', ... 'format': 'cfa_format', ... 'tracking_id': 'tracking_id'} ... ) - >>> f.nc_has_cfa_aggregation_data() + >>> f.nc_has_cfa_aggregated_data() True - >>> f.nc_get_cfa_aggregation_data() + >>> f.nc_get_cfa_aggregated_data() {'location': 'cfa_location', 'file': 'cfa_file', 'address': 'cfa_address', 'format': 'cfa_format', 'tracking_id': 'tracking_id'} - >>> f.nc_del_cfa_aggregation_data() + >>> f.nc_del_cfa_aggregated_data() {'location': 'cfa_location', 'file': 'cfa_file', 'address': 'cfa_address', 'format': 'cfa_format', 'tracking_id': 'tracking_id'} - >>> f.nc_has_cfa_aggregation_data() + >>> f.nc_has_cfa_aggregated_data() False - >>> print(f.nc_get_cfa_aggregation_data(None)) + >>> print(f.nc_get_cfa_aggregated_data(None)) None - >>> print(f.nc_del_cfa_aggregation_data(None)) + >>> print(f.nc_del_cfa_aggregated_data(None)) None """ - out = self._nc_get("cfa_aggregation_data", default=None) + out = self._nc_get("cfa_aggregated_data", default=None) if out is not None: return out.copy() @@ -134,14 +134,14 @@ def nc_get_cfa_aggregation_data(self, default=ValueError()): f"{self.__class__.__name__} has no CFA-netCDF aggregation terms", ) - def nc_has_cfa_aggregation_data(self): - """Whether the CFA-netCDF aggregation instruction terms have been set. + def nc_has_cfa_aggregated_data(self): + """Whether any CFA-netCDF aggregation instruction terms have been set. .. versionadded:: TODOCFAVER - .. seealso:: `nc_del_cfa_aggregation_data`, - `nc_get_cfa_aggregation_data`, - `nc_set_cfa_aggregation_data` + .. seealso:: `nc_del_cfa_aggregated_data`, + `nc_get_cfa_aggregated_data`, + `nc_set_cfa_aggregated_data` :Returns: @@ -151,38 +151,38 @@ def nc_has_cfa_aggregation_data(self): **Examples** - >>> f.nc_set_cfa_aggregation_data( + >>> f.nc_set_cfa_aggregated_data( ... {'location': 'cfa_location', ... 'file': 'cfa_file', ... 'address': 'cfa_address', ... 'format': 'cfa_format', ... 'tracking_id': 'tracking_id'} ... ) - >>> f.nc_has_cfa_aggregation_data() + >>> f.nc_has_cfa_aggregated_data() True - >>> f.nc_get_cfa_aggregation_data() + >>> f.nc_get_cfa_aggregated_data() {'location': 'cfa_location', 'file': 'cfa_file', 'address': 'cfa_address', 'format': 'cfa_format', 'tracking_id': 'tracking_id'} - >>> f.nc_del_cfa_aggregation_data() + >>> f.nc_del_cfa_aggregated_data() {'location': 'cfa_location', 'file': 'cfa_file', 'address': 'cfa_address', 'format': 'cfa_format', 'tracking_id': 'tracking_id'} - >>> f.nc_has_cfa_aggregation_data() + >>> f.nc_has_cfa_aggregated_data() False - >>> print(f.nc_get_cfa_aggregation_data(None)) + >>> print(f.nc_get_cfa_aggregated_data(None)) None - >>> print(f.nc_del_cfa_aggregation_data(None)) + >>> print(f.nc_del_cfa_aggregated_data(None)) None """ - return self._nc_has("cfa_aggregation_data") + return self._nc_has("cfa_aggregated_data") - def nc_set_cfa_aggregation_data(self, value): + def nc_set_cfa_aggregated_data(self, value): """Set the CFA-netCDF aggregation instruction terms. If there are any ``/`` (slash) characters in the netCDF @@ -193,9 +193,9 @@ def nc_set_cfa_aggregation_data(self, value): .. versionadded:: TODOCFAVER - .. seealso:: `nc_del_cfa_aggregation_data`, - `nc_get_cfa_aggregation_data`, - `nc_has_cfa_aggregation_data` + .. seealso:: `nc_del_cfa_aggregated_data`, + `nc_get_cfa_aggregated_data`, + `nc_has_cfa_aggregated_data` :Parameters: @@ -209,33 +209,196 @@ def nc_set_cfa_aggregation_data(self, value): **Examples** - >>> f.nc_set_cfa_aggregation_data( + >>> f.nc_set_cfa_aggregated_data( ... {'location': 'cfa_location', ... 'file': 'cfa_file', ... 'address': 'cfa_address', ... 'format': 'cfa_format', ... 'tracking_id': 'tracking_id'} ... ) - >>> f.nc_has_cfa_aggregation_data() + >>> f.nc_has_cfa_aggregated_data() True - >>> f.nc_get_cfa_aggregation_data() + >>> f.nc_get_cfa_aggregated_data() {'location': 'cfa_location', 'file': 'cfa_file', 'address': 'cfa_address', 'format': 'cfa_format', 'tracking_id': 'tracking_id'} - >>> f.nc_del_cfa_aggregation_data() + >>> f.nc_del_cfa_aggregated_data() {'location': 'cfa_location', 'file': 'cfa_file', 'address': 'cfa_address', 'format': 'cfa_format', 'tracking_id': 'tracking_id'} - >>> f.nc_has_cfa_aggregation_data() + >>> f.nc_has_cfa_aggregated_data() False - >>> print(f.nc_get_cfa_aggregation_data(None)) + >>> print(f.nc_get_cfa_aggregated_data(None)) None - >>> print(f.nc_del_cfa_aggregation_data(None)) + >>> print(f.nc_del_cfa_aggregated_data(None)) None """ - return self._nc_set("cfa_aggregation_data", value.copy()) + return self._nc_set("cfa_aggregated_data", value.copy()) + + def nc_del_cfa_file_substitutions(self, value, default=ValueError()): + """Remove the CFA-netCDF file name substitutions. + + .. versionadded:: TODOCFAVER + + .. seealso:: `nc_get_cfa_file_substitutions`, + `nc_has_cfa_file_substitutions`, + `nc_set_cfa_file_substitutions` + + :Parameters: + + default: optional + Return the value of the *default* parameter if + CFA-netCDF file name substitutions have not been + set. If set to an `Exception` instance then it will be + raised instead. + + :Returns: + + `dict` + The removed CFA-netCDF file name substitutions. + + **Examples** + + >>> f.nc_set_cfa_file_substitutions({'${base}': 'file:///data/'}) + >>> f.nc_has_cfa_file_substitutions() + True + >>> f.nc_get_cfa_file_substitutions() + {'${base}': 'file:///data/'} + >>> f.nc_del_cfa_file_substitutions() + {'${base}': 'file:///data/'} + >>> f.nc_has_cfa_file_substitutions() + False + >>> print(f.nc_get_cfa_file_substitutions(None)) + None + >>> print(f.nc_del_cfa_file_substitutions(None)) + None + + """ + return self._nc_del("cfa_file_substitutions", default=default) + + def nc_get_cfa_file_substitutions(self, default=ValueError()): + """Return the CFA-netCDF file name substitutions. + + .. versionadded:: TODOCFAVER + + .. seealso:: `nc_del_cfa_file_substitutions`, + `nc_get_cfa_file_substitutions`, + `nc_set_cfa_file_substitutions` + + :Parameters: + + default: optional + Return the value of the *default* parameter if + CFA-netCDF file name substitutions have not been + set. If set to an `Exception` instance then it will be + raised instead. + + :Returns: + + value: `dict` + The CFA-netCDF file name substitutions. + + **Examples** + + >>> f.nc_set_cfa_file_substitutions({'${base}': 'file:///data/'}) + >>> f.nc_has_cfa_file_substitutions() + True + >>> f.nc_get_cfa_file_substitutions() + {'${base}': 'file:///data/'} + >>> f.nc_del_cfa_file_substitutions() + {'${base}': 'file:///data/'} + >>> f.nc_has_cfa_file_substitutions() + False + >>> print(f.nc_get_cfa_file_substitutions(None)) + None + >>> print(f.nc_del_cfa_file_substitutions(None)) + None + + """ + out = self._nc_get("cfa_file_substitutions", default=None) + if out is not None: + return out.copy() + + if default is None: + return default + + return self._default( + default, + f"{self.__class__.__name__} has no CFA-netCDF file name substitutions", + ) + + def nc_has_cfa_file_substitutions(self): + """Whether any CFA-netCDF file name substitutions have been set. + + .. versionadded:: TODOCFAVER + + .. seealso:: `nc_del_cfa_file_substitutions`, + `nc_get_cfa_file_substitutions`, + `nc_set_cfa_file_substitutions` + + :Returns: + + `bool` + `True` if any CFA-netCDF file name substitutions have + been set, otherwise `False`. + + **Examples** + + >>> f.nc_set_cfa_file_substitutions({'${base}': 'file:///data/'}) + >>> f.nc_has_cfa_file_substitutions() + True + >>> f.nc_get_cfa_file_substitutions() + {'${base}': 'file:///data/'} + >>> f.nc_del_cfa_file_substitutions() + {'${base}': 'file:///data/'} + >>> f.nc_has_cfa_file_substitutions() + False + >>> print(f.nc_get_cfa_file_substitutions(None)) + None + >>> print(f.nc_del_cfa_file_substitutions(None)) + None + + """ + return self._nc_has("cfa_file_substitutions") + + def nc_set_cfa_file_substitutions(self, value): + """Set the CFA-netCDF file name substitutions. + + .. versionadded:: TODOCFAVER + + .. seealso:: `nc_del_cfa_file_substitutions`, + `nc_get_cfa_file_substitutions`, + `nc_has_cfa_file_substitutions` + + :Parameters: + + value: `dict` + The CFA-netCDF file name substitutions. + + :Returns: + + `None` + + **Examples** + + >>> f.nc_set_cfa_file_substitutions({'${base}': 'file:///data/'}) + >>> f.nc_has_cfa_file_substitutions() + True + >>> f.nc_get_cfa_file_substitutions() + {'${base}': 'file:///data/'} + >>> f.nc_del_cfa_file_substitutions() + {'${base}': 'file:///data/'} + >>> f.nc_has_cfa_file_substitutions() + False + >>> print(f.nc_get_cfa_file_substitutions(None)) + None + >>> print(f.nc_del_cfa_file_substitutions(None)) + None + + """ + return self._nc_set("cfa_file_substitutions", value.copy()) diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 5d3291dc90..fdaa0f18ef 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -20,7 +20,7 @@ class NetCDFRead(cfdm.read_write.netcdf.NetCDFRead): def cfa_standard_terms(self): """Standardised CFA aggregation instruction terms. - These are found in the ``aggregation_data`` attributes. + These are found in the ``aggregation_data`` attribute. .. versionadded:: TODOCFAVER @@ -227,9 +227,6 @@ def _create_data( # ------------------------------------------------------------ if construct is not None: # Remove the aggregation attributes from the construct - # properties - # for attr in ("aggregation_dimensions", "aggregation_data"): - # self.implementation.del_property(construct, attr, None) self.implementation.del_property( construct, "aggregation_dimensions", None ) @@ -282,7 +279,7 @@ def _create_data( if aggregation_data: ad = split("\s+", aggregation_data) aggregation_data = { - k[:-1]: v for k, v in zip(ad[::2], ad[1::2]) + term[:-1]: var for term, var in zip(ad[::2], ad[1::2]) } data.nc_set_cfa_aggregation_data(aggregation_data) diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 7624c08078..642f111693 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -363,7 +363,10 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): ggg = self._ggg(data) - # Get the location netCDF dimensions + # ------------------------------------------------------------ + # Get the location netCDF dimensions. These always start with + # "cfa_". + # ------------------------------------------------------------ location_ncdimensions = [] for size in ggg["location"].shape: l_ncdim = self._netcdf_name( @@ -375,7 +378,10 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): location_ncdimensions.append(l_ncdim) - # Get the fragment netCDF dimensions + # ------------------------------------------------------------ + # Get the fragment netCDF dimensions. These always start with + # "f_". + # ------------------------------------------------------------ aggregation_address = ggg["aggregation_address"] fragment_ncdimensions = [] for ncdim, size in zip(ncdimensions, aggregation_address.shape): @@ -405,9 +411,9 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): # Write the standardised aggregation instruction variables to # the CFA-netCDF file # ------------------------------------------------------------ - aggregation_data = data.nc_get_cfa_aggregation_data(default={}) + aggregated_data = data.nc_get_cfa_aggregated_data(default={}) - aggregated_data = [] + aggregated_data_attr = [] for term, data in ggg.items(): if term == "location": dimensions = location_ncdimensions @@ -423,11 +429,11 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): term_ncvar = self._cfa_write_term_variable( data, - aggregation_data.get(term, f"cfa_{term}"), + aggregated_data.get(term, f"cfa_{term}"), dimensions, ) - aggregated_data.append(f"{term}: {term_ncvar}") + aggregated_data_attr.append(f"{term}: {term_ncvar}") # ------------------------------------------------------------ # Look for non-standard CFA terms stored as field ancillaries @@ -435,9 +441,9 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): # ------------------------------------------------------------ if self.implementation.is_field(cfvar): non_standard_terms = self._cfa_write_non_standard_terms( - cfvar, fragment_ncdimensions[:ndim], aggregation_data + cfvar, fragment_ncdimensions[:ndim], aggregated_data ) - aggregated_data.extend(non_standard_terms) + aggregated_data_attr.extend(non_standard_terms) # ------------------------------------------------------------ # Add the CFA aggregation variable attributes @@ -447,7 +453,7 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): ncvar, extra={ "aggregated_dimensions": " ".join(ncdimensions), - "aggregated_data": " ".join(aggregated_data), + "aggregated_data": " ".join(aggregated_data_attr), }, ) @@ -717,19 +723,19 @@ def _cfa_unique(cls, a): return np.ma.masked_all(out_shape, dtype=a.dtype) def _ggg(self, data): - """ + """TODOCFADOCS .. versionadded:: TODOCFAVER - f = cf.example_field(0) - cf.write(f, "file_A.nc") - cf.write(f, "file_B.nc") + :Parameters: - a = cf.read("file_A.nc", chunks=4)[0].data - b = cf.read("file_B.nc", chunks=4)[0].data - c = cf.Data(b.array, units=b.Units, chunks=4) - d = cf.Data.concatenate([a, a.copy(), b, c], axis=1) + data: `Data` + TODOCFADOCS + :Returns: + + `dict` + TODOCFADOCS """ from os.path import abspath, relpath @@ -756,13 +762,16 @@ def _ggg(self, data): aggregation_address = [] aggregation_format = [] - # Maximum number of files defined on a fragments + # Maximum number of files defined for any one fragment max_files = 0 for indices in data.chunk_indices(): a = self[indices].get_filenames(address_format=True) if len(a) != 1: - raise ValueError("TODOCFADOCS") + raise ValueError( + "Can't write CFA variable when a dask storage chunk " + "spans two or more fragment files" + ) filenames, addresses, formats = a.pop() From 95c26223c84a7ed22a6af1020e2ac09b63c45d5b Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 28 Feb 2023 15:28:12 +0000 Subject: [PATCH 029/141] dev --- cf/data/array/mixin/filearraymixin.py | 4 +- cf/data/data.py | 226 ++++++++----- cf/data/fragment/mixin/fragmentarraymixin.py | 25 +- .../fragment/mixin/fragmentfilearraymixin.py | 65 ++-- cf/data/fragment/netcdffragmentarray.py | 43 ++- cf/data/fragment/umfragmentarray.py | 95 +++++- cf/domain.py | 126 +++++++ cf/field.py | 118 ++++++- cf/mixin/propertiesdata.py | 114 ++++++- cf/mixin/propertiesdatabounds.py | 139 +++++++- cf/mixin2/netcdf.py | 314 ++++++++++++------ cf/read_write/netcdf/netcdfread.py | 7 +- cf/read_write/netcdf/netcdfwrite.py | 128 +++---- 13 files changed, 1069 insertions(+), 335 deletions(-) diff --git a/cf/data/array/mixin/filearraymixin.py b/cf/data/array/mixin/filearraymixin.py index 03c0afd283..13397e424d 100644 --- a/cf/data/array/mixin/filearraymixin.py +++ b/cf/data/array/mixin/filearraymixin.py @@ -1,7 +1,9 @@ import numpy as np +# import cfdm -class FileArrayMixin: + +class FileArrayMixin: # (cfdm.FileArrayMixin): """Mixin class for an array stored in a file. .. versionadded:: 3.14.0 diff --git a/cf/data/data.py b/cf/data/data.py index 1a9edcd2cd..a5874296e0 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1249,7 +1249,7 @@ def _clear_after_dask_update(self, clear=_ALL): .. versionadded:: 3.14.0 .. seealso:: `_del_Array`, `_del_cached_elements`, - `_del_cfa_write`, `_set_dask` + `_cfa_del_write`, `_set_dask` :Parameters: @@ -1305,7 +1305,7 @@ def _clear_after_dask_update(self, clear=_ALL): if clear & _CFA: # Set the CFA write status to False - self._del_cfa_write() + self._cfa_del_write() # Always set the CFA term status to False if "cfa_term" in self._custom: @@ -1448,12 +1448,12 @@ def _del_cached_elements(self): for element in ("first_element", "second_element", "last_element"): custom.pop(element, None) - def _del_cfa_write(self): + def _cfa_del_write(self): """Set the CFA write status of the data to `False`. .. versionadded:: TODOCFAVER - .. seealso:: `get_cfa_write`, `_set_cfa_write` + .. seealso:: `cfa_get_write`, `_cfa_set_write` :Returns: @@ -1508,8 +1508,8 @@ def _set_cfa_write(self, status): .. versionadded:: TODOCFAVER - .. seealso:: `get_cfa_write`, `set_cfa_write`, - `_del_cfa_write`, `cf.read`, `cf.write`, + .. seealso:: `cfa_get_write`, `cfa_set_write`, + `_cfa_del_write`, `cf.read`, `cf.write`, :Parameters: @@ -2424,8 +2424,9 @@ def ceil(self, inplace=False, i=False): d._set_dask(da.ceil(dx)) return d - def cfa_add_fragment_location(self, location): - """TODOCFADOCS in-place. + @_inplace_enabled(default=False) + def cfa_add_fragment_location(self, location, inplace=False): + """TODOCFADOCS .. versionadded:: TODOCFAVER @@ -2434,37 +2435,136 @@ def cfa_add_fragment_location(self, location): location: `str` TODOCFADOCS + {{inplace: `bool`, optional}} + :Returns: - `None` + `Data` or `None` + TODOCFADOCS **Examples** - >>> d.cfa_add_fragment_location('/data/model') + >>> e = d.cfa_add_fragment_location('/data/model') """ from dask.base import collections_to_dsk + d = _inplace_enabled_define_and_cleanup(self) + location = abspath(location) - dx = self.to_dask_array() + dx = d.to_dask_array() updated = False dsk = collections_to_dsk((dx,), optimize_graph=True) for key, a in dsk.items(): try: - a.get_filenames() + dsk[key] = a.add_fragment_location(location) except AttributeError: - # This chunk doesn't contain a CFA fragment file + # This chunk doesn't contain CFA fragment continue else: - # This chunk contains a CFA fragment file - dsk[key] = a.add_fragment_location(location, inplace=False) + # This chunk contains a CFA fragment updated = True if updated: dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) - self._set_dask(dx, clear=_NONE) + d._set_dask(dx, clear=_NONE) + + return d + + @_inplace_enabled(default=False) + def cfa_add_file_substitution(self, base, location, inplace=False): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + base: `str`, optional + TODOCFADOCS + + location: `str` + TODOCFADOCS + + {{inplace: `bool`, optional}} + + :Returns: + + `Data` or `None` + TODOCFADOCS + + **Examples** + + >>> e = d.cfa_add_fragment_location('/data/model') + + """ + d = _inplace_enabled_define_and_cleanup(self) + + base = f"${{base}}" + subs = d.cfa_get_file_substitutions({}) + if base in subs and subs[base] != location: + raise ValueError( + "Can't overwrite existing CFA file name substitution " + f"{base}: {subs[base]!r}" + ) + + d.cfa_set_file_substitutions({base: location}) + + return d + + def cfa_get_write(self): + """The CFA write status of the data. + + If and only if the CFA write status is `True`, then this + `Data` instance has the potential to be written to a + CFA-netCDF file as aggregated data. In this case it is the + choice of parameters to the `cf.write` function that + determines if the data is actually written as aggregated data. + + .. versionadded:: TODOCFAVER + + .. seealso:: `cfa_set_write`, `cf.read`, `cf.write` + + :Returns: + + `bool` + + **Examples** + + >>> d = cf.Data([1, 2]) + >>> d.cfa_get_write() + False + + """ + return self._custom.get("cfa_write", False) + + def cfa_set_write(self, status): + """Set the CFA write status of the data. + + TODOCFADOCS.ppp + + .. versionadded:: TODOCFAVER + + .. seealso:: `cfa_get_write`, `cf.read`, `cf.write` + + :Parameters: + + status: `bool` + The new CFA write status. + + :Returns: + + `None` + + """ + if status: + raise ValueError( + "'cfa_set_write' only allows the CFA write status to be " + "set to False" + ) + + self._cfa_del_write() def compute(self): # noqa: F811 """A numpy view the data. @@ -3777,9 +3877,9 @@ def concatenate(cls, data, axis=0, cull_graph=True): # status, but ... cfa = _CFA for d in processed_data: - if not d.get_cfa_write(): + if not d.cfa_get_write(): # ... the CFA write status is False when any input - # data instance has False status + # data instance has False status ... cfa = _NONE break @@ -3790,26 +3890,31 @@ def concatenate(cls, data, axis=0, cull_graph=True): non_concat_axis_chunks = list(d.chunks) non_concat_axis_chunks.pop(axis) if non_concat_axis_chunks != non_concat_axis_chunks0: - # ... the CFA write status is False when any two - # input data instances have different chunk - # patterns for the non-concatenated axes + # ... or the CFA write status is False when any + # two input data instances have different chunk + # patterns for the non-concatenated axes. cfa = _NONE break - # Set the new dask array - data0._set_dask(dx, clear=_ALL ^ cfa) + # Set the new dask array, retaining the cached elements ... + data0._set_dask(dx, clear=_ALL ^ _CACHE ^ cfa) - # Set the CFA-netCDF aggregated_data instructions, giving - # precedence to those towards the left hand side of the input - # list. - if data0.get_cfa_write(): + # ... now delete the cached second element, which might now be + # incorrect. + data0._custom.pop("second_element", None) + + # Set the CFA-netCDF aggregated_data instructions and + # substitutions, giving precedence to those towards the left + # hand side of the input list. + if data0.cfa_get_write(): aggregated_data = {} + substitutions = {} for d in processed_data[::-1]: - if d.get_cfa_write(): - aggregated_data.update(d.nc_get_cfa_aggregated_data({})) + aggregated_data.update(d.cfa_get_aggregated_data({})) + substitutions.update(d.cfa_get_file_substitutions({})) - if aggregated_data: - data0.nc_set_cfa_aggregated_data(aggregated_data) + data0.cfa_set_aggregated_data(aggregated_data) + data0.cfa_set_file_substitutions(substitutions) # Manage cyclicity of axes: if join axis was cyclic, it is no # longer. @@ -5961,32 +6066,6 @@ def convert_reference_time( return d - def get_cfa_write(self): - """The CFA write status of the data. - - If and only if the CFA write status is `True`, then this - `Data` instance has the potential to be written to a - CFA-netCDF file as aggregated data. In this case it is the - choice of parameters to the `cf.write` function that - determines if the data is actually written as aggregated data. - - .. versionadded:: TODOCFAVER - - .. seealso:: `set_cfa_write`, `cf.read`, `cf.write` - - :Returns: - - `bool` - - **Examples** - - >>> d = cf.Data([1, 2]) - >>> d.get_cfa_write() - False - - """ - return self._custom.get("cfa_write", False) - def get_filenames(self, address_format=False): """The names of files containing parts of the data array. @@ -6067,9 +6146,9 @@ def get_filenames(self, address_format=False): if address_format: f = ((f, a.get_addresses(), a.get_formats()),) except AttributeError: - pass - else: - out.update(f) + continue + + out.update(f) return out @@ -6168,33 +6247,6 @@ def set_calendar(self, calendar): """ self.Units = Units(self.get_units(default=None), calendar) - def set_cfa_write(self, status): - """Set the CFA write status of the data. - - TODOCFADOCS.ppp - - .. versionadded:: TODOCFAVER - - .. seealso:: `get_cfa_write`, `cf.read`, `cf.write` - - :Parameters: - - status: `bool` - The new CFA write status. - - :Returns: - - `None` - - """ - if status: - raise ValueError( - "'set_cfa_write' only allows the CFA write status to be " - "set to False" - ) - - self._del_cfa_write() - def set_units(self, value): """Set the units. diff --git a/cf/data/fragment/mixin/fragmentarraymixin.py b/cf/data/fragment/mixin/fragmentarraymixin.py index daebc4c724..5fd28713e6 100644 --- a/cf/data/fragment/mixin/fragmentarraymixin.py +++ b/cf/data/fragment/mixin/fragmentarraymixin.py @@ -38,8 +38,9 @@ def __getitem__(self, indices): array = super().__getitem__(tuple(indices)) except ValueError: # A ValueError is expected to be raised when the fragment - # variable has fewer than 'self.ndim' dimensions (given - # that 'indices' now has 'self.ndim' elements). + # variable has fewer than 'self.ndim' dimensions (we know + # this becuase because 'indices' has 'self.ndim' + # elements). axis = self._size_1_axis(indices) if axis is not None: # There is a unique size 1 index, that must correspond @@ -287,6 +288,26 @@ def aggregated_Units(self): self.get_aggregated_units(), self.get_aggregated_calendar(None) ) + def add_fragment_location(self, location, inplace=False): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + location: `str` + TODOCFADOCS + + :Returns: + + TODOCFADOCS + + """ + raise ValueError( + "Can't add a file location to fragment represented by a " + f"{self.__class__.__name__} instance" + ) + def get_aggregated_calendar(self, default=ValueError()): """The calendar of the aggregated array. diff --git a/cf/data/fragment/mixin/fragmentfilearraymixin.py b/cf/data/fragment/mixin/fragmentfilearraymixin.py index 5642c743d6..3fea3a98ad 100644 --- a/cf/data/fragment/mixin/fragmentfilearraymixin.py +++ b/cf/data/fragment/mixin/fragmentfilearraymixin.py @@ -1,9 +1,3 @@ -from ....decorators import ( - _inplace_enabled, - _inplace_enabled_define_and_cleanup, -) - - class FragmentFileArrayMixin: """Mixin class for a fragment array stored in a file. @@ -11,14 +5,28 @@ class FragmentFileArrayMixin: """ - @_inplace_enabled(default=False) - def add_fragment_location(self, location, inplace=False): - """TODOCFADOCS""" + def add_fragment_location(self, location): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + location: `str` + TODOCFADOCS + + :Returns: + + `{{class}}` + TODOCFADOCS + + """ from os.path import basename, dirname, join - a = _inplace_enabled_define_and_cleanup(self) + a = self.copy() - # Note - it is assumed that all filenames are absolute paths + # Note: It is assumed that each existing file name is either + # an absolute path or a file URI. filenames = a.get_filenames() addresses = a.get_addresses() @@ -30,16 +38,16 @@ def add_fragment_location(self, location, inplace=False): ] ) - a._set_component("filename", filenames + new_filenames, copy=False) + a._set_component("filenames", filenames + new_filenames, copy=False) a._set_component( - "address", + "addresses", addresses + addresses[-1] * len(new_filenames), copy=False, ) return a - def get_addresses(self): + def get_addresses(self, default=AttributeError()): """TODOCFADOCS Return the names of any files containing the data array. .. versionadded:: TODOCFAVER @@ -51,12 +59,9 @@ def get_addresses(self): form. TODOCFADOCS then an empty `set` is returned. """ - try: - return self._get_component("address") - except ValueError: - return () + return self._get_component("addresses", default) - def get_filenames(self): + def get_filenames(self, default=AttributeError()): """TODOCFADOCS Return the names of any files containing the data array. .. versionadded:: TODOCFAVER @@ -68,21 +73,29 @@ def get_filenames(self): form. TODOCFADOCS then an empty `set` is returned. """ - try: - return self._get_component("filename") - except ValueError: - return () + filenames = self._get_component("filenames", None) + if filenames is None: + if default is None: + return + + return self._default( + default, f"{self.__class__.__name__} has no fragement files" + ) - def get_formats(self): + return filenames + + def get_formats(self, default=AttributeError()): """TODOCFADOCS Return the names of any files containing the data array. .. versionadded:: TODOCFAVER + .. seealso:: `get_filenames`, `get_addresses` + :Returns: - `set` + `tuple` The file names in normalised, absolute form. TODOCFADOCS then an empty `set` is returned. """ - raise NotImplementedError + return (self.get_format(),) * len(self.get_filenames(default)) diff --git a/cf/data/fragment/netcdffragmentarray.py b/cf/data/fragment/netcdffragmentarray.py index a34b5207ab..3d99affa7d 100644 --- a/cf/data/fragment/netcdffragmentarray.py +++ b/cf/data/fragment/netcdffragmentarray.py @@ -3,10 +3,12 @@ import netCDF4 from ..array.netcdfarray import NetCDFArray -from .mixin import FragmentArrayMixin +from .mixin import FragmentArrayMixin, FragmentFileArrayMixin -class NetCDFFragmentArray(FragmentArrayMixin, NetCDFArray): +class NetCDFFragmentArray( + FragmentFileArrayMixin, FragmentArrayMixin, NetCDFArray +): """A CFA fragment array stored in a netCDF file. .. versionadded:: 3.14.0 @@ -16,7 +18,7 @@ class NetCDFFragmentArray(FragmentArrayMixin, NetCDFArray): def __init__( self, filenames=None, - address=None, + addresses=None, dtype=None, shape=None, aggregated_units=False, @@ -30,22 +32,22 @@ def __init__( :Parameters: - filenames: `tuple` + filenames: sequence of `str`, optional The names of the netCDF fragment files containing the array. - address: `str`, optional + addresses: sequence of `str`, optional The name of the netCDF variable containing the fragment array. Required unless *varid* is set. - dtype: `numpy.dtype` + dtype: `numpy.dtype`, optional The data type of the aggregated array. May be `None` if the numpy data-type is not known (which can be the case for netCDF string types, for example). This may differ from the data type of the netCDF fragment variable. - shape: `tuple` + shape: `tuple`, optional The shape of the fragment within the aggregated array. This may differ from the shape of the netCDF fragment variable in that the latter may have fewer @@ -74,23 +76,25 @@ def __init__( group = None # TODO ??? super().__init__( - filename=filenames, - ncvar=address, - group=group, dtype=dtype, shape=shape, mask=True, units=units, calendar=calendar, source=source, - copy=False, + copy=copy, ) if source is not None: try: - address = source._get_component("address", False) + filenames = source._get_component("filenames", None) except AttributeError: - address = None + filenames = None + + try: + addresses = source._get_component("addresses ", None) + except AttributeError: + addresses = None try: aggregated_units = source._get_component( @@ -106,8 +110,11 @@ def __init__( except AttributeError: aggregated_calendar = False - if address is not None: - self._set_component("address", address, copy=False) + if filenames: + self._set_component("filenames", tuple(filenames), copy=False) + + if addresses: + self._set_component("addresses ", tuple(addresses), copy=False) self._set_component("aggregated_units", aggregated_units, copy=False) self._set_component( @@ -128,6 +135,8 @@ def open(self): `netCDF4.Dataset` """ + # Loop round the files, returning as soon as we find one that + # works. filenames = self.get_filenames() for filename, address in zip(filenames, self.get_addresses()): url = urlparse(filename) @@ -145,6 +154,4 @@ def open(self): self._set_component("ncvar", address, copy=False) return nc - raise FileNotFoundError( - f"No such netCDF fragment files: {tuple(filenames)}" - ) + raise FileNotFoundError(f"No such netCDF fragment files: {filenames}") diff --git a/cf/data/fragment/umfragmentarray.py b/cf/data/fragment/umfragmentarray.py index c64d234d07..c33fe74abf 100644 --- a/cf/data/fragment/umfragmentarray.py +++ b/cf/data/fragment/umfragmentarray.py @@ -1,8 +1,8 @@ from ..array.umarray import UMArray -from .mixin import FragmentArrayMixin +from .mixin import FragmentArrayMixin, FragmentFileArrayMixin -class UMFragmentArray(FragmentArrayMixin, UMArray): +class UMFragmentArray(FragmentFileArrayMixin, FragmentArrayMixin, UMArray): """A CFA fragment array stored in a UM or PP file. .. versionadded:: 3.14.0 @@ -11,8 +11,8 @@ class UMFragmentArray(FragmentArrayMixin, UMArray): def __init__( self, - filename=None, - address=None, + filenames=None, + addresses=None, dtype=None, shape=None, aggregated_units=False, @@ -26,11 +26,11 @@ def __init__( :Parameters: - filename: `str` - The name of the UM or PP file containing the fragment. + filenames: sequence of `str`, optional + The names of the UM or PP file containing the fragment. - address: `int`, optional - The start word in the file of the header. + addresses: sequence of `str`, optional + The start words in the files of the header. dtype: `numpy.dtype` The data type of the aggregated array. May be `None` @@ -66,8 +66,6 @@ def __init__( """ super().__init__( - filename=filename, - header_offset=address, dtype=dtype, shape=shape, units=units, @@ -77,6 +75,16 @@ def __init__( ) if source is not None: + try: + filenames = source._get_component("filenames", None) + except AttributeError: + filenames = None + + try: + addresses = source._get_component("addresses ", None) + except AttributeError: + addresses = None + try: aggregated_units = source._get_component( "aggregated_units", False @@ -91,7 +99,74 @@ def __init__( except AttributeError: aggregated_calendar = False + if filenames: + self._set_component("filenames", tuple(filenames), copy=False) + + if addresses: + self._set_component("addresses ", tuple(addresses), copy=False) + self._set_component("aggregated_units", aggregated_units, copy=False) self._set_component( "aggregated_calendar", aggregated_calendar, copy=False ) + + def get_formats(self): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + .. seealso:: `get_filenames`, `get_addresses` + + :Returns: + + `tuple` + + """ + return ("um",) * len(self.get_filenames()) + + def open(self): + """Returns an open dataset containing the data array. + + When multiple fragment files have been provided an attempt is + made to open each one, in arbitrary order, and the + `umfile_lib.File` is returned from the first success. + + .. versionadded:: TODOCFAVER + + :Returns: + + `umfile_lib.File` + + """ + # Loop round the files, returning as soon as we find one that + # works. + filenames = self.get_filenames() + for filename, address in zip(filenames, self.get_addresses()): + url = urlparse(filename) + if url.scheme == "file": + # Convert file URI into an absolute path + filename = url.path + + try: + f = File( + path=filename, + byte_ordering=None, + word_size=None, + fmt=None, + ) + except FileNotFoundError: + continue + except Exception as error: + try: + f.close_fd() + except Exception: + pass + + raise Exception(f"{error}: {filename}") + + self._set_component("header_offset", address, copy=False) + return f + + raise FileNotFoundError( + f"No such PP or UM fragment files: {filenames}" + ) diff --git a/cf/domain.py b/cf/domain.py index 9604c4e3ed..ad605c1a0d 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -126,6 +126,132 @@ def size(self): [domain_axis.get_size(0) for domain_axis in domain_axes.values()] ) + def cfa_add_fragment_location( + self, + location, + ): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + location: `str` + TODOCFADOCS + + :Returns: + + `None` + + **Examples** + + >>> f.cfa_add_fragment_location('/data/model') + + """ + for c in self.constructs.filter_by_data(todict=True).values(): + c.cfa_add_fragment_location( + location, + ) + + def cfa_clear_file_substitutions( + self, + ): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Returns: + + `dict` + + **Examples** + + >>> f.cfa_clear_file_substitutions() + {} + + """ + out = {} + for c in self.constructs.filter_by_data(todict=True).values(): + out.update(c.cfa_clear_file_substitutions()) + + return out + + def cfa_get_file_substitutions(self): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Returns: + + `dict` + + **Examples** + + >>> f.cfa_get_file_substitutions() + {} + + """ + out = {} + for c in self.constructs.filter_by_data(todict=True).values(): + out.update(c.cfa_get_file_substitutions()) + + return out + + def cfa_del_file_substitution( + self, + base, + ): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + base: `str` + TODOCFADOCS + + :Returns: + + `None` + + **Examples** + + >>> f.cfa_del_file_substitution('base', '/data/model') + + """ + for c in self.constructs.filter_by_data(todict=True).values(): + c.cfa_del_file_substitution( + base, + ) + + def cfa_set_file_substitutions( + self, + value, + ): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + base: `str` + TODOCFADOCS + + sub: `str` + TODOCFADOCS + + :Returns: + + `None` + + **Examples** + + >>> f.cfa_set_file_substitution({'base': '/data/model'}) + + """ + for c in self.constructs.filter_by_data(todict=True).values(): + c.cfa_set_file_substitutions(value) + def close(self): """Close all files referenced by the domain construct. diff --git a/cf/field.py b/cf/field.py index ffa9746c46..c5cd4e0181 100644 --- a/cf/field.py +++ b/cf/field.py @@ -3641,8 +3641,11 @@ def cell_area( return w - @_inplace_enabled(default=False) - def cfa_add_fragment_location(self, location, inplace=False): + def cfa_add_fragment_location( + self, + location, + constructs=True, + ): """TODOCFADOCS .. versionadded:: TODOCFAVER @@ -3652,8 +3655,6 @@ def cfa_add_fragment_location(self, location, inplace=False): location: `str` TODOCFADOCS - {{inplace: `bool`, optional}} - :Returns: `None` @@ -3663,14 +3664,113 @@ def cfa_add_fragment_location(self, location, inplace=False): >>> f.cfa_add_fragment_location('/data/model') """ - f = _inplace_enabled_define_and_cleanup(self) + super().add_fragment_location( + location, + ) + if constructs: + for c in self.constructs.filter_by_data(todict=True).values(): + c.add_fragment_location( + location, + ) - super().add_fragment_location(location, inplace=True) + def cfa_get_file_substitutions(self, constructs=True): + """TODOCFADOCS - for c in f.constructs.filter_by_data(todict=True).values(): - c.add_fragment_location(location, inplace=True) + .. versionadded:: TODOCFAVER - return f + :Returns: + + `dict` + + **Examples** + + >>> f.cfa_get_file_substitutions() + {} + + """ + out = super().cfa_get_file_substitutions() + if constructs: + for c in self.constructs.filter_by_data(todict=True).values(): + out.update(c.cfa_set_file_substitution()) + + return out + + def cfa_del_file_substitution( + self, + base, + constructs=True, + ): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + base: `str` + TODOCFADOCS + + constructs: `bool` + If True, the default, then metadata constructs are + also transposed so that their axes are in the same + relative order as in the transposed data array of the + field. By default metadata constructs are not + altered. TODOCFADOCS + + :Returns: + + `None` + + **Examples** + + >>> f.cfa_del_file_substitution('base', '/data/model') + + """ + super().cfa_del_file_substitution( + base, + ) + if constructs: + for c in self.constructs.filter_by_data(todict=True).values(): + c.cfa_del_file_substitution( + base, + ) + + def cfa_set_file_substitutions( + self, + value, + constructs=True, + ): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + base: `str` + TODOCFADOCS + + sub: `str` + TODOCFADOCS + + constructs: `bool` + If True, the default, then metadata constructs are + also transposed so that their axes are in the same + relative order as in the transposed data array of the + field. By default metadata constructs are not + altered. TODOCFADOCS + + :Returns: + + `None` + + **Examples** + + >>> f.cfa_set_file_substitution({'base': '/data/model'}) + + """ + super().cfa_set_file_substitutions(value) + if constructs: + for c in self.constructs.filter_by_data(todict=True).values(): + c.cfa_set_file_substitutions(value) def radius(self, default=None): """Return the radius of a latitude-longitude plane defined in diff --git a/cf/mixin/propertiesdata.py b/cf/mixin/propertiesdata.py index 35e08bc7e8..9c172179a9 100644 --- a/cf/mixin/propertiesdata.py +++ b/cf/mixin/propertiesdata.py @@ -2473,18 +2473,18 @@ def ceil(self, inplace=False, i=False): delete_props=True, ) - @_inplace_enabled(default=False) - def cfa_add_fragment_location(self, location, inplace=False): + def cfa_set_file_substitutions(self, value): """TODOCFADOCS .. versionadded:: TODOCFAVER :Parameters: - location: `str` + base: `str` TODOCFADOCS - {{inplace: `bool`, optional}} + sub: `str` + TODOCFADOCS :Returns: @@ -2492,16 +2492,112 @@ def cfa_add_fragment_location(self, location, inplace=False): **Examples** - >>> f.cfa_add_fragment_location('/data/model') + >>> f.cfa_set_file_substitution('base', '/data/model') """ - f = _inplace_enabled_define_and_cleanup(self) + data = self.get_data(None, _fill_value=False, _units=False) + if data is not None: + data.cfa_set_file_substitutions(value) + + @_inplace_enabled(default=False) + def cfa_clear_file_substitutions(self, inplace=False): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + {{inplace: `bool`, optional}} + + :Returns: + + `dict` + + **Examples** + + >>> f.cfa_clear_file_substitutions() + {} + + """ + data = self.get_data(None) + if data is None: + return {} + + return data.cfa_clear_file_substitutions({}) + + def cfa_del_file_substitution( + self, + base, + ): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + base: `str` + TODOCFADOCS + + :Returns: + + `None` + + **Examples** + + >>> f.cfa_del_file_substitution('base') - data = f.get_data(None) + """ + data = self.get_data(None, _fill_value=False, _units=False) if data is not None: - data.add_fragment_location(location, inplace=True) + data.cfa_del_file_substitutions(base) + + def cfa_get_file_substitutions( + self, + ): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Returns: - return f + `dict` + + **Examples** + + >>> g = f.cfa_get_file_substitutions() + + """ + data = self.get_data(None) + if data is None: + return {} + + return data.cfa_get_file_substitutions({}) + + def cfa_add_fragment_location( + self, + location, + ): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + location: `str` + TODOCFADOCS + + :Returns: + + `None` + + **Examples** + + >>> f.cfa_add_fragment_location('/data/model') + + """ + data = self.get_data(None, _fill_value=False, _units=False) + if data is not None: + data.cfa_add_fragment_location(location) def chunk(self, chunksize=None): """Partition the data array. diff --git a/cf/mixin/propertiesdatabounds.py b/cf/mixin/propertiesdatabounds.py index d14a95325b..d15000e84c 100644 --- a/cf/mixin/propertiesdatabounds.py +++ b/cf/mixin/propertiesdatabounds.py @@ -1182,18 +1182,132 @@ def ceil(self, bounds=True, inplace=False, i=False): i=i, ) - @_inplace_enabled(default=False) - def cfa_add_fragment_location(self, location, inplace=False): + def cfa_set_file_substitutions(self, value): """TODOCFADOCS .. versionadded:: TODOCFAVER :Parameters: - location: `str` + base: `str` TODOCFADOCS - {{inplace: `bool`, optional}} + sub: `str` + TODOCFADOCS + + :Returns: + + `None` + + **Examples** + + >>> c.cfa_add_file_substitution('base', '/data/model') + + """ + super().cfa_set_file_substitutions(value) + + bounds = self.get_bounds(None) + if bounds is not None: + bounds.cfa_set_file_substitutions(value) + + interior_ring = self.get_interior_ring(None) + if interior_ring is not None: + interior_ring.cfa_set_file_substitutions(value) + + def cfa_clear_file_substitutions( + self, + ): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Returns: + + `dict` + + **Examples** + + >>> f.cfa_clear_file_substitutions() + {} + + """ + out = super().cfa_clear_file_substitutions() + + bounds = self.get_bounds(None) + if bounds is not None: + out.update(bounds.cfa_clear_file_substitutions()) + + interior_ring = self.get_interior_ring(None) + if interior_ring is not None: + out.update(interior_ring.cfa_clear_file_substitutions()) + + return out + + def cfa_del_file_substitution(self, base): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + base: `str` + TODOCFADOCS + + :Returns: + + `None` + + **Examples** + + >>> c.cfa_del_file_substitution('base') + + """ + super().cfa_del_file_substitution(base) + + bounds = self.get_bounds(None) + if bounds is not None: + bounds.cfa_del_file_substitution(base) + + interior_ring = self.get_interior_ring(None) + if interior_ring is not None: + interior_ring.cfa_del_file_substitution(base) + + def cfa_get_file_substitutions(self): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Returns: + + `dict` + + **Examples** + + >>> c.cfa_get_file_substitutions() + {} + + """ + out = super().cfa_get_file_substitutions() + + bounds = self.get_bounds(None) + if bounds is not None: + out.update(bounds.cfa_get_file_substitutions({})) + + interior_ring = self.get_interior_ring(None) + if interior_ring is not None: + out.update(interior_ring.cfa_get_file_substitutions({})) + + return out + + def cfa_add_fragment_location(self, location): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + location: `str` + TODOCFADOCS :Returns: @@ -1204,14 +1318,15 @@ def cfa_add_fragment_location(self, location, inplace=False): >>> c.cfa_add_fragment_location('/data/model') """ - return self._apply_superclass_data_oper( - _inplace_enabled_define_and_cleanup(self), - "cfa_add_fragment_location", - (location,), - bounds=True, - interior_ring=True, - inplace=inplace, - ) + super().cfa_add_fragment_location(location) + + bounds = self.get_bounds(None) + if bounds is not None: + bounds.cfa_add_fragment_location(location) + + interior_ring = self.get_interior_ring(None) + if interior_ring is not None: + interior_ring.cfa_add_fragment_location(location) def chunk(self, chunksize=None): """Partition the data array. diff --git a/cf/mixin2/netcdf.py b/cf/mixin2/netcdf.py index 71952f2623..b6d4cdcf4c 100644 --- a/cf/mixin2/netcdf.py +++ b/cf/mixin2/netcdf.py @@ -4,23 +4,26 @@ situation. """ +from cfdm.mixin import NetCDFMixin -class CFANetCDF: +class CFANetCDF(NetCDFMixin): """Mixin class for accessing CFA-netCDF aggregation instruction terms. + Must be used in conjunction with `NetCDF` + .. versionadded:: TODOCFAVER """ - def nc_del_cfa_aggregated_data(self, default=ValueError()): + def cfa_del_aggregated_data(self, default=ValueError()): """Remove the CFA-netCDF aggregation instruction terms. .. versionadded:: TODOCFAVER - .. seealso:: `nc_get_cfa_aggregated_data`, - `nc_has_cfa_aggregated_data`, - `nc_set_cfa_aggregated_data` + .. seealso:: `cfa_get_aggregated_data`, + `cfa_has_aggregated_data`, + `cfa_set_aggregated_data` :Parameters: @@ -37,45 +40,44 @@ def nc_del_cfa_aggregated_data(self, default=ValueError()): **Examples** - >>> f.nc_set_cfa_aggregated_data( + >>> f.cfa_set_aggregated_data( ... {'location': 'cfa_location', ... 'file': 'cfa_file', ... 'address': 'cfa_address', ... 'format': 'cfa_format', ... 'tracking_id': 'tracking_id'} ... ) - >>> f.nc_has_cfa_aggregated_data() + >>> f.cfa_has_aggregated_data() True - >>> f.nc_get_cfa_aggregated_data() + >>> f.cfa_get_aggregated_data() {'location': 'cfa_location', 'file': 'cfa_file', 'address': 'cfa_address', - 'format': 'cfa_format', + 'format': 'c ', 'tracking_id': 'tracking_id'} - >>> f.nc_del_cfa_aggregated_data() + >>> f.cfa_del_aggregated_data() {'location': 'cfa_location', 'file': 'cfa_file', 'address': 'cfa_address', 'format': 'cfa_format', 'tracking_id': 'tracking_id'} - >>> f.nc_has_cfa_aggregated_data() + >>> f.cfa_has_aggregated_data() False - >>> print(f.nc_get_cfa_aggregated_data(None)) + >>> print(f.cfa_get_aggregated_data(None)) None - >>> print(f.nc_del_cfa_aggregated_data(None)) None """ return self._nc_del("cfa_aggregated_data", default=default) - def nc_get_cfa_aggregated_data(self, default=ValueError()): + def cfa_get_aggregated_data(self, default=ValueError()): """Return the CFA-netCDF aggregation instruction terms. - .. versionadded:: TODOCFAVER + .. versifragement onadsded:: TODOCFAVER - .. seealso:: `nc_del_cfa_aggregated_data`, - `nc_has_cfa_aggregated_data`, - `nc_set_cfa_aggregated_data` + .. seealso:: `scfa_del_aggregated_data`, + `cfa_has_aggregated_data`, + `cfa_set_aggregated_data` :Parameters: @@ -88,37 +90,37 @@ def nc_get_cfa_aggregated_data(self, default=ValueError()): :Returns: `dict` - The aggregation instruction terms and their + he aggregation instruction terms and their corresponding netCDF variable names. **Examples** - >>> f.nc_set_cfa_aggregated_data( + >>> f.cfa_set_aggregated_data( ... {'location': 'cfa_location', ... 'file': 'cfa_file', ... 'address': 'cfa_address', ... 'format': 'cfa_format', ... 'tracking_id': 'tracking_id'} ... ) - >>> f.nc_has_cfa_aggregated_data() + >>> f.cfa_has_aggregated_data() True - >>> f.nc_get_cfa_aggregated_data() + >>> f.cfa_get_aggregated_data() {'location': 'cfa_location', 'file': 'cfa_file', 'address': 'cfa_address', 'format': 'cfa_format', 'tracking_id': 'tracking_id'} - >>> f.nc_del_cfa_aggregated_data() + >>> f.cfa_del_aggregated_data() {'location': 'cfa_location', 'file': 'cfa_file', 'address': 'cfa_address', 'format': 'cfa_format', 'tracking_id': 'tracking_id'} - >>> f.nc_has_cfa_aggregated_data() + >>> f.cfa_has_aggregated_data() False - >>> print(f.nc_get_cfa_aggregated_data(None)) + >>> print(f.cfa_get_aggregated_data(None)) None - >>> print(f.nc_del_cfa_aggregated_data(None)) + >>> print(f.cfa_del_aggregated_data(None)) None """ @@ -134,14 +136,14 @@ def nc_get_cfa_aggregated_data(self, default=ValueError()): f"{self.__class__.__name__} has no CFA-netCDF aggregation terms", ) - def nc_has_cfa_aggregated_data(self): + def cfa_has_aggregated_data(self): """Whether any CFA-netCDF aggregation instruction terms have been set. .. versionadded:: TODOCFAVER - .. seealso:: `nc_del_cfa_aggregated_data`, - `nc_get_cfa_aggregated_data`, - `nc_set_cfa_aggregated_data` + .. seealso:: `cfa_del_aggregated_data`, + `cfa_get_aggregated_data`, + `cfa_set_aggregated_data` :Returns: @@ -151,38 +153,38 @@ def nc_has_cfa_aggregated_data(self): **Examples** - >>> f.nc_set_cfa_aggregated_data( + >>> f.cfa_set_aggregated_data( ... {'location': 'cfa_location', ... 'file': 'cfa_file', ... 'address': 'cfa_address', ... 'format': 'cfa_format', ... 'tracking_id': 'tracking_id'} ... ) - >>> f.nc_has_cfa_aggregated_data() + >>> f.cfa_has_aggregated_data() True - >>> f.nc_get_cfa_aggregated_data() + >>> f.cfa_get_aggregated_data() {'location': 'cfa_location', 'file': 'cfa_file', 'address': 'cfa_address', 'format': 'cfa_format', 'tracking_id': 'tracking_id'} - >>> f.nc_del_cfa_aggregated_data() + >>> f.cfa_del_aggregated_data() {'location': 'cfa_location', 'file': 'cfa_file', 'address': 'cfa_address', 'format': 'cfa_format', 'tracking_id': 'tracking_id'} - >>> f.nc_has_cfa_aggregated_data() + >>> f.cfa_has_aggregated_data() False - >>> print(f.nc_get_cfa_aggregated_data(None)) + >>> print(f.cfa_get_aggregated_data(None)) None - >>> print(f.nc_del_cfa_aggregated_data(None)) + >>> print(f.cfa_del_aggregated_data(None)) None """ return self._nc_has("cfa_aggregated_data") - def nc_set_cfa_aggregated_data(self, value): + def cfa_set_aggregated_data(self, value): """Set the CFA-netCDF aggregation instruction terms. If there are any ``/`` (slash) characters in the netCDF @@ -193,9 +195,9 @@ def nc_set_cfa_aggregated_data(self, value): .. versionadded:: TODOCFAVER - .. seealso:: `nc_del_cfa_aggregated_data`, - `nc_get_cfa_aggregated_data`, - `nc_has_cfa_aggregated_data` + .. seealso:: `cfa_del_aggregated_data`, + `cfa_get_aggregated_data`, + `cfa_has_aggregated_data` :Parameters: @@ -209,45 +211,46 @@ def nc_set_cfa_aggregated_data(self, value): **Examples** - >>> f.nc_set_cfa_aggregated_data( + >>> f.cfa_set_aggregated_data( ... {'location': 'cfa_location', ... 'file': 'cfa_file', ... 'address': 'cfa_address', ... 'format': 'cfa_format', ... 'tracking_id': 'tracking_id'} ... ) - >>> f.nc_has_cfa_aggregated_data() + >>> f.cfa_has_aggregated_data() True - >>> f.nc_get_cfa_aggregated_data() + >>> f.cfa_get_aggregated_data() {'location': 'cfa_location', 'file': 'cfa_file', 'address': 'cfa_address', 'format': 'cfa_format', 'tracking_id': 'tracking_id'} - >>> f.nc_del_cfa_aggregated_data() + >>> f.cfa_del_aggregated_data() {'location': 'cfa_location', 'file': 'cfa_file', 'address': 'cfa_address', 'format': 'cfa_format', 'tracking_id': 'tracking_id'} - >>> f.nc_has_cfa_aggregated_data() + >>> f.cfa_has_aggregated_data() False - >>> print(f.nc_get_cfa_aggregated_data(None)) + >>> print(f.cfa_get_aggregated_data(None)) None - >>> print(f.nc_del_cfa_aggregated_data(None)) + >>> print(f.cfa_del_aggregated_data(None)) None """ - return self._nc_set("cfa_aggregated_data", value.copy()) + if value: + self._nc_set("cfa_aggregated_data", value.copy()) - def nc_del_cfa_file_substitutions(self, value, default=ValueError()): + def cfa_clear_file_substitutions(self): """Remove the CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER - .. seealso:: `nc_get_cfa_file_substitutions`, - `nc_has_cfa_file_substitutions`, - `nc_set_cfa_file_substitutions` + .. seealso:: `cfa_get_file_substitutions`, + `cfa_has_file_substitutions`, + `cfa_set_file_substitutions` :Parameters: @@ -264,31 +267,92 @@ def nc_del_cfa_file_substitutions(self, value, default=ValueError()): **Examples** - >>> f.nc_set_cfa_file_substitutions({'${base}': 'file:///data/'}) - >>> f.nc_has_cfa_file_substitutions() + >>> f.cfa_set_file_substitutions({'${base}': 'file:///data/'}) + >>> f.cfa_has_file_substitutions() True - >>> f.nc_get_cfa_file_substitutions() + >>> f.cfa_get_file_substitutions() {'${base}': 'file:///data/'} - >>> f.nc_del_cfa_file_substitutions() + >>> f.cfa_del_file_substitutions() {'${base}': 'file:///data/'} - >>> f.nc_has_cfa_file_substitutions() + >>> f.cfa_has_file_substitutions() + False + >>> print(f.cfa_get_file_substitutions(None)) + None + >>> print(f.cfa_del_file_substitutions(None)) + None + + """ + return self._nc_del("cfa_file_substitutions", {}).copy() + + def cfa_del_file_substitution(self, base, default=ValueError()): + """Remove the CFA-netCDF file name substitutions. + + .. versionadded:: TODOCFAVER + + .. seealso:: `cfa_get_file_substitutions`, + `cfa_has_file_substitutions`, + `cfa_set_file_substitutions` + + :Parameters: + + default: optional + Return the value of the *default* parameter if + CFA-netCDF file name substitutions have not been + set. If set to an `Exception` instance then it will be + raised instead. + + :Returns: + + `dict` + The removed CFA-netCDF file name substitutions. + + **Examples** + + >>> f.cfa_set_file_substitutions({'${base}': 'file:///data/'}) + >>> f.cfa_has_file_substitutions() + True + >>> f.cfa_get_file_substitutions() + {'base': 'file:///data/'} + >>> f.cfa_del_file_substitutions() + {'base': 'file:///data/'} + >>> f.cfa_has_file_substitutions() False - >>> print(f.nc_get_cfa_file_substitutions(None)) + >>> print(f.cfa_get_file_substitutions(None)) None - >>> print(f.nc_del_cfa_file_substitutions(None)) + >>> print(f.cfa_del_file_substitutions(None)) None """ - return self._nc_del("cfa_file_substitutions", default=default) + if base.startswith("${") and base.endswith("}"): + base = base[2:-1] + + subs = self.cfa_file_substitutions({}) + if base not in subs: + if default is None: + return + + return self._default( + default, + f"{self.__class__.__name__} has no netCDF {base!r} " + "CFA file substitution", + ) - def nc_get_cfa_file_substitutions(self, default=ValueError()): + out = {base: subs.pop(base)} + if subs: + self._nc_set("cfa_file_substitutions", subs) + else: + self._nc_del("cfa_file_substitutions", None) + + return out + + def cfa_get_file_substitutions(self): """Return the CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER - .. seealso:: `nc_del_cfa_file_substitutions`, - `nc_get_cfa_file_substitutions`, - `nc_set_cfa_file_substitutions` + .. seealso:: `cfa_del_file_substitutions`, + `cfa_get_file_substitutions`, + `cfa_set_file_substitutions` :Parameters: @@ -305,18 +369,18 @@ def nc_get_cfa_file_substitutions(self, default=ValueError()): **Examples** - >>> f.nc_set_cfa_file_substitutions({'${base}': 'file:///data/'}) - >>> f.nc_has_cfa_file_substitutions() + >>> f.cfa_set_file_substitutions({'${base}': 'file:///data/'}) + >>> f.cfa_has_file_substitutions() True - >>> f.nc_get_cfa_file_substitutions() + >>> f.cfa_get_file_substitutions() {'${base}': 'file:///data/'} - >>> f.nc_del_cfa_file_substitutions() + >>> f.cfa_del_file_substitutions() {'${base}': 'file:///data/'} - >>> f.nc_has_cfa_file_substitutions() + >>> f.cfa_has_file_substitutions() False - >>> print(f.nc_get_cfa_file_substitutions(None)) + >>> print(f.cfa_get_file_substitutions(None)) None - >>> print(f.nc_del_cfa_file_substitutions(None)) + >>> print(f.cfa_del_file_substitutions(None)) None """ @@ -324,22 +388,16 @@ def nc_get_cfa_file_substitutions(self, default=ValueError()): if out is not None: return out.copy() - if default is None: - return default + return {} - return self._default( - default, - f"{self.__class__.__name__} has no CFA-netCDF file name substitutions", - ) - - def nc_has_cfa_file_substitutions(self): + def cfa_has_file_substitutions(self): """Whether any CFA-netCDF file name substitutions have been set. .. versionadded:: TODOCFAVER - .. seealso:: `nc_del_cfa_file_substitutions`, - `nc_get_cfa_file_substitutions`, - `nc_set_cfa_file_substitutions` + .. seealso:: `cfa_del_file_substitutions`, + `cfa_get_file_substitutions`, + `cfa_set_file_substitutions` :Returns: @@ -349,36 +407,77 @@ def nc_has_cfa_file_substitutions(self): **Examples** - >>> f.nc_set_cfa_file_substitutions({'${base}': 'file:///data/'}) - >>> f.nc_has_cfa_file_substitutions() + >>> f.cfa_set_file_substitutions({'${base}': 'file:///data/'}) + >>> f.cfa_has_file_substitutions() True - >>> f.nc_get_cfa_file_substitutions() + >>> f.cfa_get_file_substitutions() {'${base}': 'file:///data/'} - >>> f.nc_del_cfa_file_substitutions() + >>> f.cfa_del_file_substitutions() {'${base}': 'file:///data/'} - >>> f.nc_has_cfa_file_substitutions() + >>> f.cfa_has_file_substitutions() False - >>> print(f.nc_get_cfa_file_substitutions(None)) + >>> print(f.cfa_get_file_substitutions(None)) None - >>> print(f.nc_del_cfa_file_substitutions(None)) + >>> print(f.cfa_del_file_substitutions(None)) None """ return self._nc_has("cfa_file_substitutions") - def nc_set_cfa_file_substitutions(self, value): + # def cfa_set_file_substitution(self, base, value): + # """Set the CFA-netCDF file name substitutions. + # + # .. versionadded:: TODOCFAVER + # + # .. seealso:: `cfa_del_file_substitutions`, + # `cfa_get_file_substitutions`, + # `cfa_has_file_substitutions` + # + # :Parameters: + # + # value: `dict` + # The new CFA-netCDF file name substitutions. + # + # :Returns: + # + # `None` + # + # **Examples** + # + # >>> f.cfa_set_file_substitutions({'${base}': 'file:///data/'}) + # >>> f.cfa_has_file_substitutions() + # True + # >>> f.cfa_get_file_substitutions() + # {'${base}': 'file:///data/'} + # >>> f.cfa_del_file_substitutions() + # {'${base}': 'file:///data/'} + # >>> f.cfa_has_file_substitutions() + # False + # >>> print(f.cfa_get_file_substitutions(None)) + # None + # >>> print(f.cfa_del_file_substitutions(None)) + # None + # + # """ + # if base.startswith("${") and base.endswith("}"): + # base = base [2:-1] + # + # subs = self._nc_get("cfa_file_substitutions", {}) + # subs.update({base: value})) + + def cfa_set_file_substitutions(self, value): """Set the CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER - .. seealso:: `nc_del_cfa_file_substitutions`, - `nc_get_cfa_file_substitutions`, - `nc_has_cfa_file_substitutions` + .. seealso:: `cfa_del_file_substitutions`, + `cfa_get_file_substitutions`, + `cfa_has_file_substitutions` :Parameters: value: `dict` - The CFA-netCDF file name substitutions. + The new CFA-netCDF file name substitutions. :Returns: @@ -386,19 +485,28 @@ def nc_set_cfa_file_substitutions(self, value): **Examples** - >>> f.nc_set_cfa_file_substitutions({'${base}': 'file:///data/'}) - >>> f.nc_has_cfa_file_substitutions() + >>> f.cfa_set_file_substitutions({'${base}': 'file:///data/'}) + >>> f.cfa_has_file_substitutions() True - >>> f.nc_get_cfa_file_substitutions() + >>> f.cfa_get_file_substitutions() {'${base}': 'file:///data/'} - >>> f.nc_del_cfa_file_substitutions() + >>> f.cfa_del_file_substitutions() {'${base}': 'file:///data/'} - >>> f.nc_has_cfa_file_substitutions() + >>> f.cfa_has_file_substitutions() False - >>> print(f.nc_get_cfa_file_substitutions(None)) + >>> print(f.cfa_get_file_substitutions(None)) None - >>> print(f.nc_del_cfa_file_substitutions(None)) + >>> print(f.cfa_del_file_substitutions(None)) None """ - return self._nc_set("cfa_file_substitutions", value.copy()) + if not value: + return + + for base, sub in value.items(): + if base.startswith("${") and base.endswith("}"): + value[base[2:-1]] = value.pop(base) + + subs = self.cfa_get_file_substitutions) + subs.update(value) + self._nc_set("cfa_file_substitutions", subs) diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index fdaa0f18ef..d0ab246fa2 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -281,7 +281,10 @@ def _create_data( aggregation_data = { term[:-1]: var for term, var in zip(ad[::2], ad[1::2]) } - data.nc_set_cfa_aggregation_data(aggregation_data) + data.cfa_set_aggregation_data(aggregation_data) + + # Store the file substitutions + data.cfa_set_file_substitutions(kwargs.get("substitutions")) # Note: We don't cache elements from aggregated data @@ -617,7 +620,7 @@ def _create_cfanetcdfarray( if subs is None: subs = {} else: - # Convert, e.g., "${BASE}: a" to {"${BASE}": "a"} + # Convert "${base}: value" to {"${base}": "value"} subs = self.parse_x(term_ncvar, subs) subs = { key: value[0] for d in subs for key, value in d.items() diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 642f111693..3d0dc112dd 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -361,16 +361,18 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): """ g = self.write_vars - ggg = self._ggg(data) + ndim = data.ndim + + ggg = self._ggg(data, cfvar) # ------------------------------------------------------------ # Get the location netCDF dimensions. These always start with - # "cfa_". + # "f_loc_". # ------------------------------------------------------------ location_ncdimensions = [] for size in ggg["location"].shape: l_ncdim = self._netcdf_name( - f"cfa_{size}", dimsize=size, role="cfa_location" + f"f_loc_{size}", dimsize=size, role="cfa_location" ) if l_ncdim not in g["dimensions"]: # Create a new location dimension @@ -378,13 +380,18 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): location_ncdimensions.append(l_ncdim) + location_ncdimensions = tuple(location_ncdimensions) + # ------------------------------------------------------------ # Get the fragment netCDF dimensions. These always start with # "f_". # ------------------------------------------------------------ aggregation_address = ggg["aggregation_address"] fragment_ncdimensions = [] - for ncdim, size in zip(ncdimensions, aggregation_address.shape): + for ncdim, size in zip( + ncdimensions + ("extra",) * (aggregation_address.ndim - ndim), + aggregation_address.shape, + ): f_ncdim = self._netcdf_name( f"f_{ncdim}", dimsize=size, role="cfa_fragment" ) @@ -394,18 +401,7 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): fragment_ncdimensions.append(f_ncdim) - ndim = aggregation_address.ndim - if ndim == len(ncdimensions) + 1: - # Include an extra trailing dimension for the aggregation - # instruction variables - size = aggregation_address.shape[-1] - f_ncdim = self._netcdf_name( - "f_extra", dimsize=size, role="cfa_fragment" - ) - if f_ncdim not in g["dimensions"]: - self._write_dimension(f_ncdim, None, size=size) - - fragment_ncdimensions.append(f_ncdim) + fragment_ncdimensions = tuple(fragment_ncdimensions) # ------------------------------------------------------------ # Write the standardised aggregation instruction variables to @@ -567,11 +563,11 @@ def _write_field_ancillary(self, f, key, anc): :Parameters: - f : `Field` + f: `Field` - key : `str` + key: `str` - anc : `FieldAncillary` + anc: `FieldAncillary` :Returns: @@ -580,10 +576,6 @@ def _write_field_ancillary(self, f, key, anc): object. If no ancillary variable was written then an empty string is returned. - **Examples** - - >>> ncvar = _write_field_ancillary(f, 'fieldancillary2', anc) - """ if anc._custom.get("cfa_term", False): # This field ancillary construct is to be written as a @@ -598,20 +590,30 @@ def _cfa_write_term_variable(self, data, ncvar, ncdimensions): .. versionadded:: TODOCFAVER + :Parameters: + + data `Data` + + ncvar: `str` + + ncdimensions: `tuple` of `str` + :Returns: - `list` + `str` + The netCDF variable name of the CFA term variable. """ create = not self._already_in_file(data, ncdimensions) - if not create: - ncvar = self.write_vars["seen"][id(data)]["ncvar"] - else: + if create: + # Create a new CFA term variable in the file ncvar = self._netcdf_name(ncvar) - - # Create a new CFA term variable self._write_netcdf_variable(ncvar, ncdimensions, data) + else: + # This CFA term variable has already been written to the + # file + ncvar = self.write_vars["seen"][id(data)]["ncvar"] return ncvar @@ -722,7 +724,7 @@ def _cfa_unique(cls, a): return np.ma.masked_all(out_shape, dtype=a.dtype) - def _ggg(self, data): + def _ggg(self, data, cfvar): """TODOCFADOCS .. versionadded:: TODOCFAVER @@ -735,7 +737,9 @@ def _ggg(self, data): :Returns: `dict` - TODOCFADOCS + A dictionary whose keys are the sandardised CFA + aggregation instruction terms, keyed by `Data` + instances containing the corresponding variables. """ from os.path import abspath, relpath @@ -744,11 +748,10 @@ def _ggg(self, data): g = self.write_vars - substitutions = g["cfa_options"].get("substitutions") - if substitutions: - # TODO move this to global once - substitutions = tuple(substitutions.items())[::-1] - + # Define the CFA file susbstitutions + substitutions = data.cfa_get_file_substitutions() + substitutions.update(g["cfa_options"].get("substitutions")) + relative = g["cfa_options"].get("relative", None) if relative: absolute = False @@ -758,25 +761,30 @@ def _ggg(self, data): else: absolute = None + # Size of the trailing dimension + n_trailing = 0 + aggregation_file = [] aggregation_address = [] aggregation_format = [] - - # Maximum number of files defined for any one fragment - max_files = 0 - for indices in data.chunk_indices(): a = self[indices].get_filenames(address_format=True) if len(a) != 1: + if a: + raise ValueError( + f"Can't write CFA variable from {cfvar!r} when a " + "dask storage chunk spans two or more fragment files" + ) + raise ValueError( - "Can't write CFA variable when a dask storage chunk " - "spans two or more fragment files" + f"Can't write CFA variable from {cfvar!r} when a " + "dask storage chunk spans zero fragment files" ) filenames, addresses, formats = a.pop() - if len(filenames) > max_files: - max_files = len(filenames) + if len(filenames) > n_trailing: + n_trailing = len(filenames) filenames2 = [] for filename in filenames: @@ -790,7 +798,8 @@ def _ggg(self, data): filename = relpath(abspath(path), start=cfa_dir) if substitutions: - for base, sub in substitutions: + # Apply the CFA file susbstitutions + for base, sub in substitutions.items(): filename = filename.replace(sub, base) filenames2.append(filename) @@ -799,15 +808,18 @@ def _ggg(self, data): aggregation_address.append(addresses) aggregation_format.append(formats) - shape = data.numblocks - + # Pad each aggregation instruction array value so that it has + # 'n_trailing' elements + a_shape = data.numblocks pad = None - if max_files > 1: + if n_trailing > 1: + a_shape += (n_trailing,) + # Pad the ... for i, (filenames, addresses, formats) in enumerate( zip(aggregation_file, aggregation_address, aggregation_format) ): - n = max_files - len(filenames) + n = n_trailing - len(filenames) if n: pad = ("",) * n aggregation_file[i] = filenames + pad @@ -817,14 +829,14 @@ def _ggg(self, data): aggregation_address[i] = addresses + pad - shape += (max_files,) - - aggregation_file = np.array(aggregation_file).reshape(shape) - aggregation_address = np.array(aggregation_address).reshape(shape) - aggregation_format = np.array(aggregation_format).reshape(shape) + # Reshape the 1-d arrays to span the data dimensions, plus the + # extra trailing dimension if there is one. + aggregation_file = np.array(aggregation_file).reshape(a_shape) + aggregation_address = np.array(aggregation_address).reshape(a_shape) + aggregation_format = np.array(aggregation_format).reshape(a_shape) + # Mask any padded elements if pad: - # Mask padded elements aggregation_file = np.ma.where( aggregation_file == "", np.ma.masked, aggregation_file ) @@ -832,7 +844,9 @@ def _ggg(self, data): aggregation_address = np.ma.array(aggregation_address, mask=mask) aggregation_format = np.ma.array(aggregation_format, mask=mask) - # Location + # ------------------------------------------------------------ + # Create the location array + # ------------------------------------------------------------ dtype = np.dtype(np.int32) if max(data.to_dask_array().chunksize) > np.iinfo(dtype).max: dtype = np.dtype(np.int64) @@ -844,7 +858,9 @@ def _ggg(self, data): for i, c in enumerate(data.chunks): aggregation_location[i, : len(c)] = c + # ------------------------------------------------------------ # Return Data objects + # ------------------------------------------------------------ data = type(data) return { "aggregation_location": data(aggregation_location), From 79753dd993f36d63b6b38536856d78bf257d43c1 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 28 Feb 2023 15:31:40 +0000 Subject: [PATCH 030/141] dev --- cf/read_write/netcdf/netcdfwrite.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 3d0dc112dd..ba023a94d6 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -808,8 +808,8 @@ def _ggg(self, data, cfvar): aggregation_address.append(addresses) aggregation_format.append(formats) - # Pad each aggregation instruction array value so that it has - # 'n_trailing' elements + # Pad each value of the aggregation instruction arrays so that + # it has 'n_trailing' elements a_shape = data.numblocks pad = None if n_trailing > 1: @@ -829,8 +829,9 @@ def _ggg(self, data, cfvar): aggregation_address[i] = addresses + pad - # Reshape the 1-d arrays to span the data dimensions, plus the - # extra trailing dimension if there is one. + # Reshape the 1-d aggregation instruction arrays to span the + # data dimensions, plus the extra trailing dimension if there + # is one. aggregation_file = np.array(aggregation_file).reshape(a_shape) aggregation_address = np.array(aggregation_address).reshape(a_shape) aggregation_format = np.array(aggregation_format).reshape(a_shape) From 3fbb7c8ae08f60373782ae773b4658d6d5b050ce Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 28 Feb 2023 19:15:16 +0000 Subject: [PATCH 031/141] dev --- cf/mixin2/__init__.py | 2 +- cf/mixin2/{netcdf.py => cfanetcdf.py} | 0 cf/read_write/netcdf/netcdfwrite.py | 13 +- cf/read_write/write.py | 227 +++++++++++++++++--------- 4 files changed, 155 insertions(+), 87 deletions(-) rename cf/mixin2/{netcdf.py => cfanetcdf.py} (100%) diff --git a/cf/mixin2/__init__.py b/cf/mixin2/__init__.py index 4394d414c6..3dc304f232 100644 --- a/cf/mixin2/__init__.py +++ b/cf/mixin2/__init__.py @@ -1,2 +1,2 @@ -from .netcdf import CFANetCDF +from .cfanetcdf import CFANetCDF from .container import Container diff --git a/cf/mixin2/netcdf.py b/cf/mixin2/cfanetcdf.py similarity index 100% rename from cf/mixin2/netcdf.py rename to cf/mixin2/cfanetcdf.py diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index ba023a94d6..a18a772e23 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -38,8 +38,8 @@ def _use_cfa(self, cfvar, construct_type): :Returns: `bool` - True if the variable is to be should be written as a - CFA variable. + True if the variable is to be written as a CFA + variable. """ g = self.write_vars @@ -56,10 +56,7 @@ def _use_cfa(self, cfvar, construct_type): if data.size == 1: return False - if construct_type == "field": - return True - - for ctype, ndim in g["cfa_options"]["metadata"]: + for ctype, ndim in g["cfa_options"]["constructs"]: # Write as CFA if it has an appropriate construct type ... if ctype in ("all", construct_type): # ... and then only if it satisfies the number of @@ -748,7 +745,9 @@ def _ggg(self, data, cfvar): g = self.write_vars - # Define the CFA file susbstitutions + # Define the CFA file susbstitutions, giving precedence over + # those set on the Data object to those provided by the CFA + # options. substitutions = data.cfa_get_file_substitutions() substitutions.update(g["cfa_options"].get("substitutions")) diff --git a/cf/read_write/write.py b/cf/read_write/write.py index 28c6ae3353..c1dfebd1d4 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -364,71 +364,6 @@ def write( external to the named external file. Ignored if there are no such constructs. - cfa_options: `dict`, optional - A dictionary defining parameters for configuring the - output CFA-netCDF file: - - ================= ======================================= - Key Value - ================= ======================================= - - ``paths`` -------- How to write fragment file names. Set - to ``'relative'`` for them to be - written as relative to the CF_netCDF - file being created, or else set to - ``'absolute'`` for them to be written - as file URIs. Note that in either case, - fragment file names defined by fully - qualified URLs will always be written - as such. - - ``metadata`` ----- The types of construct to be written as - CFA-netCDF aggregated variables. By - default all metadata constructs field - data and the data Nd metadata - constructs. What about UGRID, for which - the 1-d coords are, combined, - muchlarger than the data .... TODO What - about DSG and compression in general? - - ``substitutions``- A dictionary whose key/value pairs - define text substitutions to be applied - to the fragment file URIs when the - output CFA-netCDF file is subsequently - read. Each key must be of the form - ``'${...}'``, where ``...`` represents - one or more letters, digits, and - underscores. The substitutions are - stored in the output file by the - ``substitutions`` attribute of the - ``file`` aggregation instruction - variable. - - ``properties``---- A (sequence of) `str` defining one or - more properties of the fields - represented by each file frgament. For - each property specified, the fragment - values are written to the output - CFA-netCDF file as non-standardised - aggregation instruction variables whose - term name is the same as the property - name. When the output file is read in - with `cf.read` these variables are - converted to field ancillary - constructs. - - ``'base'`` Deprecated at version 3.14.0 and no - longer available. - ================= ======================================= - - The default of *cfa_options* is ``{'paths': 'relative'}``. - - *Parameter example:* - ``cfa_options={'substitutions': {'${base}': '/home/data/'}}`` - - *Parameter example:* - ``cfa_options={'properties': 'tracking_id'}`` - endian: `str`, optional The endian-ness of the output file. Valid values are ``'little'``, ``'big'`` or ``'native'``. By default the @@ -657,6 +592,124 @@ def write( .. versionadded:: 3.14.0 + cfa_options: `dict`, optional + A dictionary defining parameters for configuring the + output CFA-netCDF file: + + =================== ===================================== + Key Value + =================== ===================================== + + ``'paths'`` -------- How to write fragment file names. Set + to ``'relative'`` (the default) for + them to be written as relative to the + CFA-netCDF file being created, or + else set to ``'absolute'`` for them + to be written as file URIs. Note that + in both cases, fragment file names + defined by fully qualified URLs will + always be written as such. + + ``'constructs'`` --- The types of construct to be written + as CFA-netCDF aggregated + variables. By default only field + constructs are written as CFA-netCDF + aggregated variables. + + The types are given as a (sequence + of) `str`, which may take the same + values as allowed by the *omit_data* + parameter. + + Alternatively, the types may be given + as keys to a `dict`, whose values + specify the number of dimensions that + the construct must also have if it is + to be written as CFA-netCDF + aggregated variable. A value of + `None` means no restriction on the + number of dimensions, which is + equivalent to a value of + ``cf.ge(0)``. + + Note that size 1 data arrays are + never written as CFA-netCDF + aggregated variables, regardless of + the whether or not this has been + requested. + + ``'substitutions'``- A dictionary whose key/value pairs + define text substitutions to be + applied to the fragment file URIs + when the output CFA-netCDF file is + subsequently read. Each key must be a + string of one or more letters, + digits, and underscores. These + substitutions take precendence over + any that are also defined on + individual constructs. + + Substitutions are stored in the + output file by the ``substitutions`` + attribute of the ``file`` aggregation + instruction variable. + + ``'properties'``---- For fragments that define a field + construct's data, a (sequence of) + `str` defining one or more properties + of the file fragments. For each + property specified, the value of that + property from each fragment is + written to the output CFA-netCDF file + in a non-standardised aggregation + instruction variable whose term name + is the same as the property name. + + When the output file is read in with + `cf.read` these variables are + converted to field ancillary + constructs. + + ``'base'`` Deprecated at version 3.14.0 and no + longer available. + =================== ===================================== + + The default of *cfa_options* is ``{'path': 'relative', + 'construct': 'field'}``. + + *Parameter example:* + ``cfa_options={'substitution': {'base': '/home/data/'}}`` + + *Parameter example:* + ``cfa_options={'property': 'tracking_id'}`` + + *Parameter example:* + Equivalent ways to only write cell measure constructs as + CFA-netCDF variables: ``cfa_options={'constructs': + 'cell_measure'}`` and ``cfa_options={'constructs': + ['cell_measure']}`` and ``cfa_options={'constructs': + {'cell_measure': None}}`` + + *Parameter example:* + Equivalent ways to only write field and auxiliary + coordinate constructs as CFA-netCDF variables: + ``cfa_options={'constructs': ['field', + 'auxiliary_coordinate']}`` and + ``cfa_options={'constructs': {'field': None, + 'auxiliary_coordinate': None}}`` + + *Parameter example:* + Only write two dimensional auxiliary coordinate + constructs as CFA-netCDF variables: + ``cfa_options={'constructs': {'auxiliary_coordinate': + 2}}`` + + *Parameter example:* + Only write auxiliary coordinate constructs with two or + more dimensions as CFA-netCDF variables: + ``cfa_options={'constructs': {'auxiliary_coordinate': + cf.ge(2)}}`` + HDF_chunksizes: deprecated at version 3.0.0 HDF chunk sizes may be set for individual constructs prior to writing, instead. See `cf.Data.nc_set_hdf5_chunksizes`. @@ -774,7 +827,9 @@ def write( else: cfa = False - if cfa: + if not cfa: + cfa_options = {} + else: # Add CFA to the Conventions if not Conventions: Conventions = CFA() @@ -788,7 +843,7 @@ def write( cfa_options = {} else: cfa_options = cfa_options.copy() - keys = ("paths", "metadata", "group", "substitutions") + keys = ("paths", "constructs", "substitutions", "properties") if not set(cfa_options).issubset(keys): raise ValueError( "Invalid dictionary key to the 'cfa_options' " @@ -796,19 +851,33 @@ def write( f"Got: {tuple(cfa_options)}" ) - if "metadata" not in cfa_options: - cfa_options["metadata"] = ((None, None),) - else: - metadata = cfa_options["metadata"] - if isinstance(metadata, str): - cfa_options["metadata"] = ((metadata, None),) - elif isinstance(metadata[0], str): - cfa_options["metadata"] = (metadata,) - cfa_options.setdefault("paths", "relative") - else: - cfa_options = {} - + cfa_options.setdefault("constructs", "field") + cfa_options.setdefault("substitutions", {}) + cfa_options.setdefault("properties", ()) + + constructs = cfa_options["constructs"] + if isinstance(constructs, dict): + cfa_options["constructs"] = constructs.copy() + else: + if isinstance(constructs, str): + constructs = (constructs,) + + cfa_options["constructs"] = {c: None for c in constructs} + + substitutions = cfa_options["substitutions"].copy() + for base, sub in substitutions.items(): + if base.startswith("${") and base.endswith("}"): + substitutions[base[2:-1]] = substitutions.pop(base) + + cfa_options["substitutions"] = substitutions + + properties = cfa_options["properties"] + if isinstance(properties, str): + properties = (properties,) + + cfa_options["properties"] = tuple(properties) + extra_write_vars["cfa"] = cfa extra_write_vars["cfa_options"] = cfa_options From 98aaa85184dcfc4e4102f3c6c5a107177d5602e4 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 1 Mar 2023 00:11:46 +0000 Subject: [PATCH 032/141] dev --- cf/data/data.py | 68 ++------- .../fragment/mixin/fragmentfilearraymixin.py | 12 +- cf/data/fragment/netcdffragmentarray.py | 2 - cf/data/fragment/umfragmentarray.py | 3 + cf/mixin2/cfanetcdf.py | 54 +------ cf/read_write/netcdf/netcdfwrite.py | 132 ++++++++++++------ cf/read_write/write.py | 18 +-- 7 files changed, 125 insertions(+), 164 deletions(-) diff --git a/cf/data/data.py b/cf/data/data.py index a5874296e0..b66d24278c 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -2424,8 +2424,7 @@ def ceil(self, inplace=False, i=False): d._set_dask(da.ceil(dx)) return d - @_inplace_enabled(default=False) - def cfa_add_fragment_location(self, location, inplace=False): + def cfa_add_fragment_location(self, location): """TODOCFADOCS .. versionadded:: TODOCFAVER @@ -2435,25 +2434,21 @@ def cfa_add_fragment_location(self, location, inplace=False): location: `str` TODOCFADOCS - {{inplace: `bool`, optional}} - :Returns: - `Data` or `None` + `None` TODOCFADOCS **Examples** - >>> e = d.cfa_add_fragment_location('/data/model') + >>> d.cfa_add_fragment_location('/data/model') """ from dask.base import collections_to_dsk - d = _inplace_enabled_define_and_cleanup(self) - location = abspath(location) - dx = d.to_dask_array() + dx = self.to_dask_array() updated = False dsk = collections_to_dsk((dx,), optimize_graph=True) @@ -2469,49 +2464,7 @@ def cfa_add_fragment_location(self, location, inplace=False): if updated: dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) - d._set_dask(dx, clear=_NONE) - - return d - - @_inplace_enabled(default=False) - def cfa_add_file_substitution(self, base, location, inplace=False): - """TODOCFADOCS - - .. versionadded:: TODOCFAVER - - :Parameters: - - base: `str`, optional - TODOCFADOCS - - location: `str` - TODOCFADOCS - - {{inplace: `bool`, optional}} - - :Returns: - - `Data` or `None` - TODOCFADOCS - - **Examples** - - >>> e = d.cfa_add_fragment_location('/data/model') - - """ - d = _inplace_enabled_define_and_cleanup(self) - - base = f"${{base}}" - subs = d.cfa_get_file_substitutions({}) - if base in subs and subs[base] != location: - raise ValueError( - "Can't overwrite existing CFA file name substitution " - f"{base}: {subs[base]!r}" - ) - - d.cfa_set_file_substitutions({base: location}) - - return d + self._set_dask(dx, clear=_NONE) def cfa_get_write(self): """The CFA write status of the data. @@ -2542,7 +2495,7 @@ def cfa_get_write(self): def cfa_set_write(self, status): """Set the CFA write status of the data. - TODOCFADOCS.ppp + TODOCFADOCS .. versionadded:: TODOCFAVER @@ -3903,15 +3856,16 @@ def concatenate(cls, data, axis=0, cull_graph=True): # incorrect. data0._custom.pop("second_element", None) - # Set the CFA-netCDF aggregated_data instructions and - # substitutions, giving precedence to those towards the left - # hand side of the input list. + # Set the CFA-netCDF aggregated_data instructions + # substitutions by combining them from all of the input data + # instances, giving precedence to those towards the left hand + # side of the input list. if data0.cfa_get_write(): aggregated_data = {} substitutions = {} for d in processed_data[::-1]: aggregated_data.update(d.cfa_get_aggregated_data({})) - substitutions.update(d.cfa_get_file_substitutions({})) + substitutions.update(d.cfa_get_file_substitutions()) data0.cfa_set_aggregated_data(aggregated_data) data0.cfa_set_file_substitutions(substitutions) diff --git a/cf/data/fragment/mixin/fragmentfilearraymixin.py b/cf/data/fragment/mixin/fragmentfilearraymixin.py index 3fea3a98ad..ff32111a28 100644 --- a/cf/data/fragment/mixin/fragmentfilearraymixin.py +++ b/cf/data/fragment/mixin/fragmentfilearraymixin.py @@ -37,6 +37,7 @@ def add_fragment_location(self, location): if dirname(f) != location ] ) + # TODOCFA - how does this work out with URLs and file URIs? a._set_component("filenames", filenames + new_filenames, copy=False) a._set_component( @@ -55,8 +56,7 @@ def get_addresses(self, default=AttributeError()): :Returns: `tuple` - The file names in normalised, absolute - form. TODOCFADOCS then an empty `set` is returned. + TODOCFADOCS """ return self._get_component("addresses", default) @@ -69,8 +69,7 @@ def get_filenames(self, default=AttributeError()): :Returns: `tuple` - The file names in normalised, absolute - form. TODOCFADOCS then an empty `set` is returned. + The fragment file names. """ filenames = self._get_component("filenames", None) @@ -85,7 +84,7 @@ def get_filenames(self, default=AttributeError()): return filenames def get_formats(self, default=AttributeError()): - """TODOCFADOCS Return the names of any files containing the data array. + """Return the format of each fragment file. .. versionadded:: TODOCFAVER @@ -94,8 +93,7 @@ def get_formats(self, default=AttributeError()): :Returns: `tuple` - The file names in normalised, absolute - form. TODOCFADOCS then an empty `set` is returned. + The fragment file formats. """ return (self.get_format(),) * len(self.get_filenames(default)) diff --git a/cf/data/fragment/netcdffragmentarray.py b/cf/data/fragment/netcdffragmentarray.py index 3d99affa7d..351debd554 100644 --- a/cf/data/fragment/netcdffragmentarray.py +++ b/cf/data/fragment/netcdffragmentarray.py @@ -73,8 +73,6 @@ def __init__( {{init copy: `bool`, optional}} """ - group = None # TODO ??? - super().__init__( dtype=dtype, shape=shape, diff --git a/cf/data/fragment/umfragmentarray.py b/cf/data/fragment/umfragmentarray.py index c33fe74abf..5cdd67a0f2 100644 --- a/cf/data/fragment/umfragmentarray.py +++ b/cf/data/fragment/umfragmentarray.py @@ -1,3 +1,6 @@ +from urllib.parse import urlparse + +from ...umread_lib.umfile import File from ..array.umarray import UMArray from .mixin import FragmentArrayMixin, FragmentFileArrayMixin diff --git a/cf/mixin2/cfanetcdf.py b/cf/mixin2/cfanetcdf.py index b6d4cdcf4c..78833c4043 100644 --- a/cf/mixin2/cfanetcdf.py +++ b/cf/mixin2/cfanetcdf.py @@ -323,8 +323,8 @@ def cfa_del_file_substitution(self, base, default=ValueError()): None """ - if base.startswith("${") and base.endswith("}"): - base = base[2:-1] + if not (base.startswith("${") and base.endswith("}")): + base = f"${{{base}}}" subs = self.cfa_file_substitutions({}) if base not in subs: @@ -424,47 +424,6 @@ def cfa_has_file_substitutions(self): """ return self._nc_has("cfa_file_substitutions") - # def cfa_set_file_substitution(self, base, value): - # """Set the CFA-netCDF file name substitutions. - # - # .. versionadded:: TODOCFAVER - # - # .. seealso:: `cfa_del_file_substitutions`, - # `cfa_get_file_substitutions`, - # `cfa_has_file_substitutions` - # - # :Parameters: - # - # value: `dict` - # The new CFA-netCDF file name substitutions. - # - # :Returns: - # - # `None` - # - # **Examples** - # - # >>> f.cfa_set_file_substitutions({'${base}': 'file:///data/'}) - # >>> f.cfa_has_file_substitutions() - # True - # >>> f.cfa_get_file_substitutions() - # {'${base}': 'file:///data/'} - # >>> f.cfa_del_file_substitutions() - # {'${base}': 'file:///data/'} - # >>> f.cfa_has_file_substitutions() - # False - # >>> print(f.cfa_get_file_substitutions(None)) - # None - # >>> print(f.cfa_del_file_substitutions(None)) - # None - # - # """ - # if base.startswith("${") and base.endswith("}"): - # base = base [2:-1] - # - # subs = self._nc_get("cfa_file_substitutions", {}) - # subs.update({base: value})) - def cfa_set_file_substitutions(self, value): """Set the CFA-netCDF file name substitutions. @@ -485,7 +444,7 @@ def cfa_set_file_substitutions(self, value): **Examples** - >>> f.cfa_set_file_substitutions({'${base}': 'file:///data/'}) + >>> f.cfa_set_file_substitutions({'base': 'file:///data/'}) >>> f.cfa_has_file_substitutions() True >>> f.cfa_get_file_substitutions() @@ -503,10 +462,11 @@ def cfa_set_file_substitutions(self, value): if not value: return + value = value.copy() for base, sub in value.items(): - if base.startswith("${") and base.endswith("}"): - value[base[2:-1]] = value.pop(base) + if not (base.startswith("${") and base.endswith("}")): + value[f"${{{base}}}"] = value.pop(base) - subs = self.cfa_get_file_substitutions) + subs = self.cfa_get_file_substitutions() subs.update(value) self._nc_set("cfa_file_substitutions", subs) diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index a18a772e23..e93b87e7c9 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -53,6 +53,9 @@ def _use_cfa(self, cfvar, construct_type): if not data.get_cfa_write(): return False + if construct_type is None: + return False + if data.size == 1: return False @@ -404,29 +407,65 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): # Write the standardised aggregation instruction variables to # the CFA-netCDF file # ------------------------------------------------------------ - aggregated_data = data.nc_get_cfa_aggregated_data(default={}) + aggregated_data = data.cfa_get_aggregated_data(default={}) + substitutions = data.cfa_get_file_substitutions() aggregated_data_attr = [] - for term, data in ggg.items(): - if term == "location": - dimensions = location_ncdimensions - else: - dimensions = fragment_ncdimensions - - # Attempt to reduce formats to a common scalar value - if term == "format": - u = data.unique().compressed().persist() - if u.size == 1: - data = u.squeeze() - dimensions = () - term_ncvar = self._cfa_write_term_variable( - data, - aggregated_data.get(term, f"cfa_{term}"), - dimensions, - ) + # Location + term = "location" + term_ncvar = self._cfa_write_term_variable( + ggg[term], + aggregated_data.get(term, f"cfa_{term}"), + location_ncdimensions, + ) + aggregated_data_attr.append(f"{term}: {term_ncvar}") - aggregated_data_attr.append(f"{term}: {term_ncvar}") + # File + term = "file" + if substitutions: + subs = [] + for base, sub in substitutions.items(): + subs.append(f"${{base}}: {sub}") + + attributes = {"substitutions": " ".join(substitutions)} + else: + attributes = None + + term_ncvar = self._cfa_write_term_variable( + ggg[term], + aggregated_data.get(term, f"cfa_{term}"), + fragment_ncdimensions, + attributes=attributes, + ) + aggregated_data_attr.append(f"{term}: {term_ncvar}") + + # Address + term = "address" + term_ncvar = self._cfa_write_term_variable( + ggg[term], + aggregated_data.get(term, f"cfa_{term}"), + fragment_ncdimensions, + ) + aggregated_data_attr.append(f"{term}: {term_ncvar}") + + # Format + term = "format" + dimensions = fragment_ncdimensions + + # Attempt to reduce formats to a common scalar value + if term == "format": + u = ggg[term].unique().compressed().persist() + if u.size == 1: + ggg[term] = u.squeeze() + dimensions = () + + term_ncvar = self._cfa_write_term_variable( + ggg[term], + aggregated_data.get(term, f"cfa_{term}"), + dimensions, + ) + aggregated_data_attr.append(f"{term}: {term_ncvar}") # ------------------------------------------------------------ # Look for non-standard CFA terms stored as field ancillaries @@ -577,12 +616,14 @@ def _write_field_ancillary(self, f, key, anc): if anc._custom.get("cfa_term", False): # This field ancillary construct is to be written as a # non-standard CFA term belonging to the parent field, or - # not at all. + # else not at all. return "" return super()._write_field_ancillary(f, key, anc) - def _cfa_write_term_variable(self, data, ncvar, ncdimensions): + def _cfa_write_term_variable( + self, data, ncvar, ncdimensions, attributes=None + ): """TODOCFADOCS. .. versionadded:: TODOCFAVER @@ -595,6 +636,8 @@ def _cfa_write_term_variable(self, data, ncvar, ncdimensions): ncdimensions: `tuple` of `str` + attributes: `dict`, optional + :Returns: `str` @@ -606,7 +649,9 @@ def _cfa_write_term_variable(self, data, ncvar, ncdimensions): if create: # Create a new CFA term variable in the file ncvar = self._netcdf_name(ncvar) - self._write_netcdf_variable(ncvar, ncdimensions, data) + self._write_netcdf_variable( + ncvar, ncdimensions, cfvar=data, extra=attributes + ) else: # This CFA term variable has already been written to the # file @@ -615,7 +660,7 @@ def _cfa_write_term_variable(self, data, ncvar, ncdimensions): return ncvar def _cfa_write_non_standard_terms( - self, field, fragment_ncdimensions, aggregation_data + self, field, fragment_ncdimensions, aggregated_data ): """TODOCFADOCS @@ -629,10 +674,10 @@ def _cfa_write_non_standard_terms( fragment_ncdimensions: `list` of `str` - aggregation_data: `dict` + aggregated_data: `dict` """ - aggregated_data = [] + aggregated_data_attr = [] terms = ["location", "file", "address", "format"] for key, field_anc in self.implementation.get_field_ancillaries( field @@ -640,17 +685,17 @@ def _cfa_write_non_standard_terms( if not field_anc._custom.get("cfa_term", False): continue - data = self.implementation.get_data(field_anc) - if not data.get_cfa_write(): + data = self.implementation.get_data(field_anc, None) + if data is None: continue - # Check that the field ancillary has the same axes as the + # Check that the field ancillary has the same axes as its # parent field, and in the same order. if field.get_data_axes(key) != field.get_data_axes(): continue - # Still here? Then this field ancillary represent a - # non-standard aggregation term. + # Still here? Then this field ancillary can be represented + # by a non-standard aggregation term. # Then transform the data so that it spans the fragment # dimensions, with one value per fragment. If a chunk has @@ -667,7 +712,6 @@ def _cfa_write_non_standard_terms( adjust_chunks={i: 1 for i in out_ind}, dtype=dx.dtype, ) - array = dx.compute() # Get the non-standard term name from the field # ancillary's 'id' attribute @@ -683,14 +727,14 @@ def _cfa_write_non_standard_terms( # Create the new CFA term variable term_ncvar = self._cfa_write_term_variable( - type(data)(array), - aggregation_data.get(term, f"cfa_{term}"), - fragment_ncdimensions, + data=type(data)(dx), + ncvar=aggregated_data.get(term, f"cfa_{term}"), + ncdimensions=fragment_ncdimensions, ) - aggregated_data.append(f"{term}: {term_ncvar}") + aggregated_data_attr.append(f"{term}: {term_ncvar}") - return aggregated_data + return aggregated_data_attr @classmethod def _cfa_unique(cls, a): @@ -731,6 +775,9 @@ def _ggg(self, data, cfvar): data: `Data` TODOCFADOCS + cfvar: construct + TODOCFADOCS + :Returns: `dict` @@ -748,9 +795,9 @@ def _ggg(self, data, cfvar): # Define the CFA file susbstitutions, giving precedence over # those set on the Data object to those provided by the CFA # options. - substitutions = data.cfa_get_file_substitutions() - substitutions.update(g["cfa_options"].get("substitutions")) - + data.cfa_set_file_substitutions(g["cfa_options"]["substitutions"]) + substitutions = data.cfa_get_file_substitutions() + relative = g["cfa_options"].get("relative", None) if relative: absolute = False @@ -851,12 +898,13 @@ def _ggg(self, data, cfvar): if max(data.to_dask_array().chunksize) > np.iinfo(dtype).max: dtype = np.dtype(np.int64) + ndim = data.ndim aggregation_location = np.ma.masked_all( - (self.ndim, max(shape)), dtype=dtype + (ndim, max(a_shape[:ndim])), dtype=dtype ) - for i, c in enumerate(data.chunks): - aggregation_location[i, : len(c)] = c + for i, chunks in enumerate(data.chunks): + aggregation_location[i, : len(chunks)] = chunks # ------------------------------------------------------------ # Return Data objects diff --git a/cf/read_write/write.py b/cf/read_write/write.py index c1dfebd1d4..a97c6eea93 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -615,12 +615,12 @@ def write( variables. By default only field constructs are written as CFA-netCDF aggregated variables. - + The types are given as a (sequence of) `str`, which may take the same values as allowed by the *omit_data* parameter. - + Alternatively, the types may be given as keys to a `dict`, whose values specify the number of dimensions that @@ -631,7 +631,7 @@ def write( number of dimensions, which is equivalent to a value of ``cf.ge(0)``. - + Note that size 1 data arrays are never written as CFA-netCDF aggregated variables, regardless of @@ -698,7 +698,7 @@ def write( ``cfa_options={'constructs': {'field': None, 'auxiliary_coordinate': None}}`` - *Parameter example:* + *Parameter example:* Only write two dimensional auxiliary coordinate constructs as CFA-netCDF variables: ``cfa_options={'constructs': {'auxiliary_coordinate': @@ -855,29 +855,29 @@ def write( cfa_options.setdefault("constructs", "field") cfa_options.setdefault("substitutions", {}) cfa_options.setdefault("properties", ()) - + constructs = cfa_options["constructs"] if isinstance(constructs, dict): cfa_options["constructs"] = constructs.copy() else: if isinstance(constructs, str): constructs = (constructs,) - + cfa_options["constructs"] = {c: None for c in constructs} substitutions = cfa_options["substitutions"].copy() for base, sub in substitutions.items(): if base.startswith("${") and base.endswith("}"): substitutions[base[2:-1]] = substitutions.pop(base) - + cfa_options["substitutions"] = substitutions properties = cfa_options["properties"] if isinstance(properties, str): properties = (properties,) - cfa_options["properties"] = tuple(properties) - + cfa_options["properties"] = tuple(properties) + extra_write_vars["cfa"] = cfa extra_write_vars["cfa_options"] = cfa_options From fa91e6a7f3f810e54a6434340f5b268003a79931 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 1 Mar 2023 09:53:17 +0000 Subject: [PATCH 033/141] dev --- cf/data/array/cfanetcdfarray.py | 13 ++-- cf/mixin2/cfanetcdf.py | 37 +++++++++-- cf/read_write/netcdf/netcdfread.py | 103 +++++++++-------------------- cf/read_write/write.py | 4 +- 4 files changed, 74 insertions(+), 83 deletions(-) diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index 898854836a..aa271aa318 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -1,6 +1,6 @@ from copy import deepcopy from itertools import accumulate, product -from os.path import join as os_join +from os.path import join from urllib.parse import urlparse import numpy as np @@ -344,7 +344,9 @@ def _set_fragment( location = fragment.location if non_standard_term is not None: + # -------------------------------------------------------- # This fragment contains a constant value + # -------------------------------------------------------- aggregated_data[frag_loc] = { "format": "full", "location": location, @@ -357,15 +359,16 @@ def _set_fragment( address = fragment.address if address is not None: + # -------------------------------------------------------- # This fragment is contained in a file + # -------------------------------------------------------- if filename is None: # This fragment is contained in the CFA-netCDF file filename = cfa_filename fmt = "nc" else: + # Apply string substitutions to the fragment filename if substitutions: - # Apply string substitutions to the fragment - # filename for base, sub in substitutions.items(): filename = filename.replace(base, sub) @@ -373,7 +376,7 @@ def _set_fragment( if parsed_filename.scheme not in ("file", "http", "https"): # Find the full path of a relative fragment # filename - filename = os_join(directory, parsed_filename.path) + filename = join(directory, parsed_filename.path) aggregated_data[frag_loc] = { "format": fmt, @@ -382,7 +385,9 @@ def _set_fragment( "location": location, } elif filename is None: + # -------------------------------------------------------- # This fragment contains wholly missing values + # -------------------------------------------------------- aggregated_data[frag_loc] = { "format": "full", "location": location, diff --git a/cf/mixin2/cfanetcdf.py b/cf/mixin2/cfanetcdf.py index 78833c4043..15327ecad5 100644 --- a/cf/mixin2/cfanetcdf.py +++ b/cf/mixin2/cfanetcdf.py @@ -4,6 +4,8 @@ situation. """ +from re import split + from cfdm.mixin import NetCDFMixin @@ -90,8 +92,11 @@ def cfa_get_aggregated_data(self, default=ValueError()): :Returns: `dict` - he aggregation instruction terms and their - corresponding netCDF variable names. + The aggregation instruction terms and their + corresponding netCDF variable names in a dictionary + whose key/value pairs are the aggregation instruction + terms and their corresponding variable names. + **Examples** @@ -201,9 +206,13 @@ def cfa_set_aggregated_data(self, value): :Parameters: - value: `dict` + value: `str` or `dict` The aggregation instruction terms and their - corresponding netCDF variable names. + corresponding netCDF variable names. Either a + CFA-netCDF-compliant string value of an + ``aggregated_data`` attribute, or a dictionary whose + key/value pairs are the aggregation instruction terms + and their corresponding variable names. :Returns: @@ -241,7 +250,14 @@ def cfa_set_aggregated_data(self, value): """ if value: - self._nc_set("cfa_aggregated_data", value.copy()) + if isinstance(value, str): + v = split("\s+", value) + value = {term[:-1]: var for term, var in zip(v[::2], v[1::2])} + else: + # 'value' is a dictionary + value = value.copy() + + self._nc_set("cfa_aggregated_data", value) def cfa_clear_file_substitutions(self): """Remove the CFA-netCDF file name substitutions. @@ -435,8 +451,15 @@ def cfa_set_file_substitutions(self, value): :Parameters: - value: `dict` - The new CFA-netCDF file name substitutions. + value: `str` or `dict` + The substition definitions in a dictionary whose + key/value pairs are the file URI parts to be + substituted and their corresponding substitution text. + + The file URI parts to be substituted may be specified + with or without the ``${...}`` syntax. For instance, + the following are equivalent: ``{'base': 'sub'}``, + ``{'${base}': 'sub'}``. :Returns: diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index d0ab246fa2..7b33ca9430 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -1,5 +1,3 @@ -from re import split - import cfdm import numpy as np from packaging.version import Version @@ -17,16 +15,6 @@ class NetCDFRead(cfdm.read_write.netcdf.NetCDFRead): """ - def cfa_standard_terms(self): - """Standardised CFA aggregation instruction terms. - - These are found in the ``aggregation_data`` attribute. - - .. versionadded:: TODOCFAVER - - """ - return ("location", "file", "address", "format") - def _ncdimensions(self, ncvar, ncdimensions=None, parent_ncvar=None): """Return a list of the netCDF dimensions corresponding to a netCDF variable. @@ -233,20 +221,15 @@ def _create_data( aggregation_data = self.implementation.del_property( construct, "aggregation_data", None ) - - if not cfa_term: - cfa_array, kwargs = self._create_cfanetcdfarray( - ncvar, - unpacked_dtype=unpacked_dtype, - coord_ncvar=coord_ncvar, - ) else: - cfa_array, kwargs = self._create_cfanetcdfarray( - ncvar, - unpacked_dtype=unpacked_dtype, - coord_ncvar=coord_ncvar, - term=cfa_term, - ) + aggregation_data = None + + cfa_array, kwargs = self._create_cfanetcdfarray( + ncvar, + unpacked_dtype=unpacked_dtype, + coord_ncvar=coord_ncvar, + non_standard_term=cfa_term, + ) data = self._create_Data( cfa_array, @@ -255,21 +238,15 @@ def _create_data( calendar=kwargs["calendar"], ) - # Set the CFA write status + # Set the CFA write status to True iff each non-aggregated + # axis has exactly one dask storage chunk if cfa_term is None: cfa_write = True for n, numblocks in zip( cfa_array.get_fragment_shape(), data.numblocks ): if n == 1 and numblocks > 1: - # Each fragment spans multiple compute - # chunks. - # - # Note: We test on 'n == 1' because we're assuming - # that each fragment already spans one chunk - # along those axes for whioch 'n > 1'. See - # `CFANetCDFArray.to_dask_array` for - # details. + # Note: 'n == 1' is True for non-aggregated axes cfa_write = False break @@ -277,10 +254,6 @@ def _create_data( # Store the 'aggregation_data' attribute if aggregation_data: - ad = split("\s+", aggregation_data) - aggregation_data = { - term[:-1]: var for term, var in zip(ad[::2], ad[1::2]) - } data.cfa_set_aggregation_data(aggregation_data) # Store the file substitutions @@ -543,7 +516,6 @@ def _create_cfanetcdfarray( ncvar, unpacked_dtype=False, coord_ncvar=None, - substitutions=None, non_standard_term=None, ): """Create a CFA-netCDF variable array. @@ -560,16 +532,11 @@ def _create_cfanetcdfarray( coord_ncvar: `str`, optional - substitutions: `dict`, optional - TODOCFADOCS - - .. versionadded:: TODOCFAVER - non_standard_term: `str`, optional The name of a non-standard aggregation instruction term from which to create the array. If set then - *ncvar* must be the value of the term in the - ``aggregation_data`` attribute. + *ncvar* must be the value of the non-standard term in + the ``aggregation_data`` attribute. .. versionadded:: TODOCFAVER @@ -601,17 +568,16 @@ def _create_cfanetcdfarray( kwargs.pop("shape", None) # Add the aggregated_data attribute (that can be used by - # dask.base.tokenize). - kwargs["instructions"] = self.read_vars["variable_attributes"][ - ncvar - ].get("aggregated_data") - - # Find URI substitutions - parsed_aggregated_data = self._parse_aggregated_data( - ncvar, g["variable_attributes"][ncvar].get("aggregated_data") + # dask.base.tokenize) + aggregated_data = self.read_vars["variable_attributes"][ncvar].get( + "aggregated_data" ) + kwargs["instructions"] = aggregated_data + + # Find URI substitutions that may be stored in the CFA file + # instruction variable's "substitutions" attribute subs = {} - for x in parsed_aggregated_data: + for x in self._parse_aggregated_data(ncvar, aggregated_data): term, term_ncvar = tuple(x.items())[0] if term != "file": continue @@ -620,7 +586,8 @@ def _create_cfanetcdfarray( if subs is None: subs = {} else: - # Convert "${base}: value" to {"${base}": "value"} + # Convert the string "${base}: value" to the + # dictionary {"${base}": "value"} subs = self.parse_x(term_ncvar, subs) subs = { key: value[0] for d in subs for key, value in d.items() @@ -628,16 +595,13 @@ def _create_cfanetcdfarray( break - if substitutions: - # Include user-defined substitutions, which will overwrite - # any defined in the file with the same base name. - subs = subs.update(substitutions) - + # Apply user-defined substitutions, which take precedence over + # those defined in the file. + subs = subs.update(g["cfa_options"]["substitutions"]) if subs: kwargs["substitutions"] = subs - # Use the kwargs to create a specialised CFANetCDFArray - # instance + # Use the kwargs to create a CFANetCDFArray instance array = self.implementation.initialise_CFANetCDFArray(**kwargs) return array, kwargs @@ -789,11 +753,9 @@ def _customize_field_ancillaries(self, parent_ncvar, f): parsed_aggregated_data = self._parse_aggregated_data( parent_ncvar, attributes.get("aggregated_data") ) - standardised_terms = self.cfa_standard_terms() - # cfa_terms = {} + standardised_terms = ("location", "file", "address", "format") for x in parsed_aggregated_data: term, ncvar = tuple(x.items())[0] - # cfa_terms[term] = ncvar if term in standardised_terms: continue @@ -802,16 +764,17 @@ def _customize_field_ancillaries(self, parent_ncvar, f): # ancillary construct. anc = self.implementation.initialise_FieldAncillary() - properties = g["variable_attributes"][ncvar].copy() - properties["long_name"] = term - self.implementation.set_properties(anc, properties) + self.implementation.set_properties( + anc, g["variable_attributes"][ncvar] + ) + anc.set_property("long_name", term) # Store the term name as the 'id' attribute. This will be # used as the term name if the field field ancillary is # written to disk as a non-standard CFA term. anc.id = term - data = self._create_data(parent_ncvar, anc, non_standard_term=term) + data = self._create_data(parent_ncvar, anc, cfa_term=term) self.implementation.set_data(anc, data, copy=False) self.implementation.nc_set_variable(anc, ncvar) diff --git a/cf/read_write/write.py b/cf/read_write/write.py index a97c6eea93..756f1bb3b3 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -867,8 +867,8 @@ def write( substitutions = cfa_options["substitutions"].copy() for base, sub in substitutions.items(): - if base.startswith("${") and base.endswith("}"): - substitutions[base[2:-1]] = substitutions.pop(base) + if not (base.startswith("${") and base.endswith("}")): + substitutions[f"${{{base}}}"] = substitutions.pop(base) cfa_options["substitutions"] = substitutions From a49fb123700908806eb7725b61fdfd8bd9708785 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 1 Mar 2023 17:03:56 +0000 Subject: [PATCH 034/141] dev --- cf/data/data.py | 15 +- cf/read_write/netcdf/netcdfread.py | 2 +- cf/read_write/netcdf/netcdfwrite.py | 52 +++++- cf/read_write/read.py | 1 + cf/read_write/write.py | 257 +++++++++++----------------- 5 files changed, 148 insertions(+), 179 deletions(-) diff --git a/cf/data/data.py b/cf/data/data.py index afa65b3590..1998984d80 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -6050,7 +6050,7 @@ def convert_reference_time( return d - def get_filenames(self, address_format=False): + def get_filenames(self): """The names of files containing parts of the data array. Returns the names of any files that are required to deliver @@ -6066,21 +6066,12 @@ def get_filenames(self, address_format=False): object has a callable `get_filename` method, the output of which is added to the returned `set`. - :Parameters: - - address: `bool`, optional - TODOCFADOCS - - .. versionadded:: TODOCFAVER - :Returns: `set` The file names. If no files are required to compute the data then an empty `set` is returned. - TODOCFADOCS - **Examples** >>> d = cf.Data.full((5, 8), 1, chunks=4) @@ -6117,8 +6108,6 @@ def get_filenames(self, address_format=False): >>> d[2, 3].get_filenames() {'file_A.nc'} - TODOCFADOCS: address_format example - """ from dask.base import collections_to_dsk @@ -6127,8 +6116,6 @@ def get_filenames(self, address_format=False): for a in dsk.values(): try: f = a.get_filenames() - if address_format: - f = ((f, a.get_addresses(), a.get_formats()),) except AttributeError: continue diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 7b33ca9430..07f536107a 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -597,7 +597,7 @@ def _create_cfanetcdfarray( # Apply user-defined substitutions, which take precedence over # those defined in the file. - subs = subs.update(g["cfa_options"]["substitutions"]) + subs = subs.update(g["cfa_substitutions"]) if subs: kwargs["substitutions"] = subs diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index e93b87e7c9..f792cc0777 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -42,6 +42,11 @@ def _use_cfa(self, cfvar, construct_type): variable. """ + if construct_type is None: + # This prrevents recursion whilst writing CFA-netCDF term + # variables. + return False + g = self.write_vars if not g["cfa"]: return False @@ -53,12 +58,6 @@ def _use_cfa(self, cfvar, construct_type): if not data.get_cfa_write(): return False - if construct_type is None: - return False - - if data.size == 1: - return False - for ctype, ndim in g["cfa_options"]["constructs"]: # Write as CFA if it has an appropriate construct type ... if ctype in ("all", construct_type): @@ -814,7 +813,7 @@ def _ggg(self, data, cfvar): aggregation_address = [] aggregation_format = [] for indices in data.chunk_indices(): - a = self[indices].get_filenames(address_format=True) + a = self._cfa_get_filenames(data[indices]) if len(a) != 1: if a: raise ValueError( @@ -930,3 +929,42 @@ def _customize_write_vars(self): from pathlib import PurePath g["cfa_dir"] = PurePath(abspath(g["filename"])).parent + + def _cfa_get_filenames(self, data): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + data: `Data` + The array. + + :Returns: + + `set` + The file names. If no files are required to compute + the data then an empty `set` is returned. + + """ + from dask.base import collections_to_dsk + + out = set() + dsk = collections_to_dsk((data.to_dask_array(),), optimize_graph=True) + for a in dsk.values(): + try: + f = a.get_filenames() + except AttributeError: + continue + + try: + f = ((f, a.get_addresses(), a.get_formats()),) + except AttributeError: + try: + f = ((f, (a.get_address(),), (a.get_format(),)),) + except AttributeError: + continue + + out.update(f) + + return out diff --git a/cf/read_write/read.py b/cf/read_write/read.py index a6449d09b7..16f6921636 100644 --- a/cf/read_write/read.py +++ b/cf/read_write/read.py @@ -59,6 +59,7 @@ def read( warn_valid=False, chunks="auto", domain=False, + cfa_substitutions=None, ): """Read field constructs from netCDF, CDL, PP or UM fields datasets. diff --git a/cf/read_write/write.py b/cf/read_write/write.py index 16763fb817..c8104afc1b 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -36,9 +36,6 @@ def write( group=True, coordinates=False, omit_data=None, - HDF_chunksizes=None, - no_shuffle=None, - unlimited=None, ): """Write field constructs to a netCDF file. @@ -164,9 +161,6 @@ def write( By default the format is ``'NETCDF4'``. - All formats support large files (i.e. those greater than - 2GB) except ``'NETCDF3_CLASSIC'``. - ``'NETCDF3_64BIT_DATA'`` is a format that requires version 4.4.0 or newer of the C library (use `cf.environment` to see which version if the netCDF-C library is in use). It @@ -415,9 +409,6 @@ def write( `_ for more details. - This parameter replaces the deprecated *no_shuffle* - parameter. - datatype: `dict`, optional Specify data type conversions to be applied prior to writing data to disk. This may be useful as a means of @@ -585,132 +576,106 @@ def write( .. versionadded:: 3.14.0 cfa_options: `dict`, optional - A dictionary defining parameters for configuring the - output CFA-netCDF file: - - =================== ===================================== - Key Value - =================== ===================================== - - ``'paths'`` -------- How to write fragment file names. Set - to ``'relative'`` (the default) for - them to be written as relative to the - CFA-netCDF file being created, or - else set to ``'absolute'`` for them - to be written as file URIs. Note that - in both cases, fragment file names - defined by fully qualified URLs will - always be written as such. - - ``'constructs'`` --- The types of construct to be written - as CFA-netCDF aggregated - variables. By default only field - constructs are written as CFA-netCDF - aggregated variables. - - The types are given as a (sequence - of) `str`, which may take the same - values as allowed by the *omit_data* - parameter. - - Alternatively, the types may be given - as keys to a `dict`, whose values - specify the number of dimensions that - the construct must also have if it is - to be written as CFA-netCDF - aggregated variable. A value of - `None` means no restriction on the - number of dimensions, which is - equivalent to a value of - ``cf.ge(0)``. - - Note that size 1 data arrays are - never written as CFA-netCDF - aggregated variables, regardless of - the whether or not this has been - requested. - - ``'substitutions'``- A dictionary whose key/value pairs - define text substitutions to be - applied to the fragment file URIs - when the output CFA-netCDF file is - subsequently read. Each key must be a - string of one or more letters, - digits, and underscores. These - substitutions take precendence over - any that are also defined on - individual constructs. - - Substitutions are stored in the - output file by the ``substitutions`` - attribute of the ``file`` aggregation - instruction variable. - - ``'properties'``---- For fragments that define a field - construct's data, a (sequence of) - `str` defining one or more properties - of the file fragments. For each - property specified, the value of that - property from each fragment is - written to the output CFA-netCDF file - in a non-standardised aggregation - instruction variable whose term name - is the same as the property name. - - When the output file is read in with - `cf.read` these variables are - converted to field ancillary - constructs. - - ``'base'`` Deprecated at version 3.14.0 and no - longer available. - =================== ===================================== - - The default of *cfa_options* is ``{'path': 'relative', - 'construct': 'field'}``. - - *Parameter example:* - ``cfa_options={'substitution': {'base': '/home/data/'}}`` - - *Parameter example:* - ``cfa_options={'property': 'tracking_id'}`` - - *Parameter example:* - Equivalent ways to only write cell measure constructs as - CFA-netCDF variables: ``cfa_options={'constructs': - 'cell_measure'}`` and ``cfa_options={'constructs': - ['cell_measure']}`` and ``cfa_options={'constructs': - {'cell_measure': None}}`` - - *Parameter example:* - Equivalent ways to only write field and auxiliary - coordinate constructs as CFA-netCDF variables: - ``cfa_options={'constructs': ['field', - 'auxiliary_coordinate']}`` and - ``cfa_options={'constructs': {'field': None, - 'auxiliary_coordinate': None}}`` - - *Parameter example:* - Only write two dimensional auxiliary coordinate - constructs as CFA-netCDF variables: - ``cfa_options={'constructs': {'auxiliary_coordinate': - 2}}`` - - *Parameter example:* - Only write auxiliary coordinate constructs with two or - more dimensions as CFA-netCDF variables: - ``cfa_options={'constructs': {'auxiliary_coordinate': - cf.ge(2)}}`` - - HDF_chunksizes: deprecated at version 3.0.0 - HDF chunk sizes may be set for individual constructs prior - to writing, instead. See `cf.Data.nc_set_hdf5_chunksizes`. - - no_shuffle: deprecated at version 3.0.0 - Use keyword *shuffle* instead. - - unlimited: deprecated at version 3.0.0 - Use method `DomainAxis.nc_set_unlimited` instead. + Parameters for configuring the output CFA-netCDF file. By + default *cfa_options* is ``{'paths': 'relative', + 'constructs': 'field'}`` and may have any subset of the + following keys (and value types): + + * ``'paths'`` (`str`) + + How to write fragment file names. Set to ``'relative'`` + (the default) for them to be written as relative to the + CFA-netCDF file being created, or else set to + ``'absolute'`` for them to be written as file URIs. Note + that in both cases, fragment file defined by fully + qualified URLs will always be written as such. + + * ``'constructs'`` (`dict` or (sequence of) `str`) + + The types of construct to be written as CFA-netCDF + aggregated variables. By default only field constructs + are written in this way. The types are given as a + (sequence of) `str`, which may take any of the values + allowed by the *omit_data* parameter. Alternatively, the + same types may be given as keys to a `dict`, whose + values specify the number of dimensions that the + construct must also have if it is to be written as + CFA-netCDF aggregated variable. A value of `None` means + no restriction on the number of dimensions, which is + equivalent to a value of ``cf.ge(0)``. + + Note that size 1 data arrays are never written as + CFA-netCDF aggregated variables, regardless of the + whether or not this has been requested. + + *Parameter example:* + Equivalent ways to only write cell measure constructs + as CFA-netCDF variables: ``'cell_measure``, + ``['cell_measure']``, and ``{'cell_measure': None}``. + + *Parameter example:* + Equivalent ways to only write field and auxiliary + coordinate constructs as CFA-netCDF variables: + ``('field', 'auxiliary_coordinate')`` and ``{'field': + None, 'auxiliary_coordinate': None}``. + + *Parameter example:* + Only write two dimensional auxiliary coordinate + constructs as CFA-netCDF variables: + ``{'auxiliary_coordinate': 2}}``. + + *Parameter example:* + Only write field constructs, and auxiliary coordinate + constructs with two or more dimensions as CFA-netCDF + variables: ``{'field': None, 'auxiliary_coordinate': + cf.ge(2)}}``. + + * ``'substitutions'`` (`dict`) + + A dictionary whose key/value pairs define text + substitutions to be applied to the fragment file + URIs. Each key must be a string of one or more letters, + digits, and underscores. These substitutions take + precendence over any that are also defined on individual + constructs. + + Substitutions are stored in the output file by the + ``substitutions`` attribute of the ``file`` aggregation + instruction variable. + + *Parameter example:* + ``{'base': 'file:///data/'}}`` + + * ``'properties'`` ((sequence of) `str`) + + For fragments of a field construct's data, a (sequence + of) `str` defining one or more properties of the file + fragments. For each property specified, the value of + that property from each fragment is written to the + output CFA-netCDF file in a non-standardised aggregation + instruction variable whose term name is the same as the + property name. + + When the output file is read in with `cf.read` these + variables are converted to field ancillary constructs. + + *Parameter example:* + ``'tracking_id'`` + + *Parameter example:* + ``('tracking_id', 'model_name')`` + + * ``'strict'`` (`bool`) + + A `bool` that determines whether or not to raise an + `Exception` if it is not possible to write as a CFA + aggregated variable a identified by the ``'constructs'`` + key If True, the default, then an `Exception` an + exception is raised, otherwise a warning is logged. + + * ``'base'`` + + Deprecated at version 3.14.0 and no longer available. :Returns: @@ -730,28 +695,6 @@ def write( >>> cf.write(f, 'file.nc', Conventions='CMIP-6.2') """ - if unlimited is not None: - _DEPRECATION_ERROR_FUNCTION_KWARGS( - "cf.write", - {"unlimited": unlimited}, - "Use method 'DomainAxis.nc_set_unlimited' instead.", - ) # pragma: no cover - - if no_shuffle is not None: - _DEPRECATION_ERROR_FUNCTION_KWARGS( - "cf.write", - {"no_shuffle": no_shuffle}, - "Use keyword 'shuffle' instead.", - ) # pragma: no cover - - if HDF_chunksizes is not None: - _DEPRECATION_ERROR_FUNCTION_KWARGS( - "cf.write", - {"HDF_chunksizes": HDF_chunksizes}, - "HDF chunk sizes may be set for individual field constructs " - "prior to writing, instead.", - ) # pragma: no cover - # Flatten the sequence of intput fields fields = tuple(flat(fields)) if fields: From f9ca4f6cfaa05897ba4e352a6ddf1449a91d2729 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 1 Mar 2023 17:06:28 +0000 Subject: [PATCH 035/141] dev --- cf/read_write/netcdf/netcdfwrite.py | 14 +++++++------- cf/read_write/read.py | 2 +- cf/read_write/write.py | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index f792cc0777..732879b66e 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -46,7 +46,7 @@ def _use_cfa(self, cfvar, construct_type): # This prrevents recursion whilst writing CFA-netCDF term # variables. return False - + g = self.write_vars if not g["cfa"]: return False @@ -945,7 +945,7 @@ def _cfa_get_filenames(self, data): `set` The file names. If no files are required to compute the data then an empty `set` is returned. - + """ from dask.base import collections_to_dsk @@ -960,11 +960,11 @@ def _cfa_get_filenames(self, data): try: f = ((f, a.get_addresses(), a.get_formats()),) except AttributeError: - try: - f = ((f, (a.get_address(),), (a.get_format(),)),) - except AttributeError: - continue - + try: + f = ((f, (a.get_address(),), (a.get_format(),)),) + except AttributeError: + continue + out.update(f) return out diff --git a/cf/read_write/read.py b/cf/read_write/read.py index 16f6921636..329e6ddbc3 100644 --- a/cf/read_write/read.py +++ b/cf/read_write/read.py @@ -59,7 +59,7 @@ def read( warn_valid=False, chunks="auto", domain=False, - cfa_substitutions=None, + cfa_substitutions=None, ): """Read field constructs from netCDF, CDL, PP or UM fields datasets. diff --git a/cf/read_write/write.py b/cf/read_write/write.py index c8104afc1b..303535f1ed 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -2,7 +2,7 @@ from ..cfimplementation import implementation from ..decorators import _manage_log_level_via_verbosity -from ..functions import _DEPRECATION_ERROR_FUNCTION_KWARGS, CFA, flat +from ..functions import CFA, flat from .netcdf import NetCDFWrite netcdf = NetCDFWrite(implementation()) @@ -612,18 +612,18 @@ def write( Equivalent ways to only write cell measure constructs as CFA-netCDF variables: ``'cell_measure``, ``['cell_measure']``, and ``{'cell_measure': None}``. - + *Parameter example:* Equivalent ways to only write field and auxiliary coordinate constructs as CFA-netCDF variables: ``('field', 'auxiliary_coordinate')`` and ``{'field': None, 'auxiliary_coordinate': None}``. - + *Parameter example:* Only write two dimensional auxiliary coordinate constructs as CFA-netCDF variables: ``{'auxiliary_coordinate': 2}}``. - + *Parameter example:* Only write field constructs, and auxiliary coordinate constructs with two or more dimensions as CFA-netCDF From de5ae6562ca77e9f2b29957ac6e5dfa562258818 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 1 Mar 2023 23:26:15 +0000 Subject: [PATCH 036/141] dev --- cf/data/array/abstract/filearray.py | 3 +- cf/data/array/mixin/filearraymixin.py | 84 ++++++++++++++++++++++++++- cf/data/array/umarray.py | 32 ++++++++-- 3 files changed, 110 insertions(+), 9 deletions(-) diff --git a/cf/data/array/abstract/filearray.py b/cf/data/array/abstract/filearray.py index 725b74ca45..47573df4b9 100644 --- a/cf/data/array/abstract/filearray.py +++ b/cf/data/array/abstract/filearray.py @@ -69,7 +69,8 @@ def get_address(self): """ raise NotImplementedError( - f"Must implement {self.__class__.__name__}.get_address" + f"Must implement {self.__class__.__name__}.get_address " + "in subclasses" ) # pragma: no cover # def get_filename(self): diff --git a/cf/data/array/mixin/filearraymixin.py b/cf/data/array/mixin/filearraymixin.py index 13397e424d..aabe8d7ea9 100644 --- a/cf/data/array/mixin/filearraymixin.py +++ b/cf/data/array/mixin/filearraymixin.py @@ -1,9 +1,7 @@ import numpy as np -# import cfdm - -class FileArrayMixin: # (cfdm.FileArrayMixin): +class FileArrayMixin: """Mixin class for an array stored in a file. .. versionadded:: 3.14.0 @@ -23,3 +21,83 @@ def _dask_meta(self): """ return np.array((), dtype=self.dtype) + + def __getitem__(self, indices): + """Return a subspace of the array. + + x.__getitem__(indices) <==> x[indices] + + Returns a subspace of the array as an independent numpy array. + + """ + raise NotImplementedError( + f"Must implement {self.__class__.__name__}.__getitem__" + ) # pragma: no cover + + def __repr__(self): + """x.__repr__() <==> repr(x)""" + return f"" + + def __str__(self): + """x.__str__() <==> str(x)""" + return f"{self.get_filename()}, {self.get_address()}" + + @property + def dtype(self): + """Data-type of the array.""" + return self._get_component("dtype") + + @property + def filename(self): + """The name of the file containing the array. + + Deprecated at version 3.14.0. Use method `get_filename` instead. + + """ + _DEPRECATION_ERROR_ATTRIBUTE( + self, + "filename", + message="Use method 'get_filename' instead.", + version="3.14.0", + removed_at="5.0.0", + ) # pragma: no cover + + @property + def shape(self): + """Shape of the array.""" + return self._get_component("shape") + + def close(self): + """Close the dataset containing the data. + + .. versionadded:: TODOCFAVER + + """ + raise NotImplementedError( + f"Must implement {self.__class__.__name__}.close" + ) # pragma: no cover + + def get_address(self): + """The address in the file of the variable. + + .. versionadded:: 3.14.0 + + :Returns: + + `str` or `None` + The address, or `None` if there isn't one. + + """ + raise NotImplementedError( + f"Must implement {self.__class__.__name__}.get_address" + ) # pragma: no cover + + def open(self): + """Returns an open dataset containing the data array. + + .. versionadded:: TODOCFAVER + + """ + raise NotImplementedError( + f"Must implement {self.__class__.__name__}.open" + ) # pragma: no cover diff --git a/cf/data/array/umarray.py b/cf/data/array/umarray.py index 87c08d5fb1..4522ebdcfb 100644 --- a/cf/data/array/umarray.py +++ b/cf/data/array/umarray.py @@ -8,10 +8,11 @@ parse_indices, ) from ...umread_lib.umfile import File, Rec -from .abstract import FileArray +from .mixin import FileArrayMixin +from .abstract import Array -class UMArray(FileArray): +class UMArray(FileArrrayMixin, Array): """A sub-array stored in a PP or UM fields file.""" def __init__( @@ -473,10 +474,11 @@ def header_offset(self): :Returns: - `int` + `int` or `None` + The address, or `None` if there isn't one. """ - return self._get_component("header_offset") + return self._get_component("header_offset", None) @property def data_offset(self): @@ -590,7 +592,7 @@ def get_address(self): :Returns: - `str` or `None` + `int` or `None` The address, or `None` if there isn't one. """ @@ -633,6 +635,26 @@ def get_fmt(self): """ return self._get_component("fmt", None) + def get_format(self): + """TODOCFADOCS + + .. versionadded:: (cfdm) TODOCFAVER + + .. seealso:: `get_filename`, `get_address` + + :Returns: + + `str` + The file format. Always ``'um'``, signifying PP/UM. + + **Examples** + + >>> a.get_format() + 'um' + + """ + return "um" + def get_word_size(self): """Word size in bytes. From 6dbb490a76d817acf5aaf226dedaf302d15a53e3 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 2 Mar 2023 09:29:07 +0000 Subject: [PATCH 037/141] dev --- cf/data/array/mixin/filearraymixin.py | 47 ------------ cf/data/array/umarray.py | 4 +- cf/data/fragment/mixin/__init__.py | 1 + cf/mixin2/container.py | 2 +- cf/read_write/netcdf/netcdfread.py | 105 ++++++++++---------------- 5 files changed, 44 insertions(+), 115 deletions(-) diff --git a/cf/data/array/mixin/filearraymixin.py b/cf/data/array/mixin/filearraymixin.py index aabe8d7ea9..78fdd8e081 100644 --- a/cf/data/array/mixin/filearraymixin.py +++ b/cf/data/array/mixin/filearraymixin.py @@ -22,18 +22,6 @@ def _dask_meta(self): """ return np.array((), dtype=self.dtype) - def __getitem__(self, indices): - """Return a subspace of the array. - - x.__getitem__(indices) <==> x[indices] - - Returns a subspace of the array as an independent numpy array. - - """ - raise NotImplementedError( - f"Must implement {self.__class__.__name__}.__getitem__" - ) # pragma: no cover - def __repr__(self): """x.__repr__() <==> repr(x)""" return f"" @@ -66,38 +54,3 @@ def filename(self): def shape(self): """Shape of the array.""" return self._get_component("shape") - - def close(self): - """Close the dataset containing the data. - - .. versionadded:: TODOCFAVER - - """ - raise NotImplementedError( - f"Must implement {self.__class__.__name__}.close" - ) # pragma: no cover - - def get_address(self): - """The address in the file of the variable. - - .. versionadded:: 3.14.0 - - :Returns: - - `str` or `None` - The address, or `None` if there isn't one. - - """ - raise NotImplementedError( - f"Must implement {self.__class__.__name__}.get_address" - ) # pragma: no cover - - def open(self): - """Returns an open dataset containing the data array. - - .. versionadded:: TODOCFAVER - - """ - raise NotImplementedError( - f"Must implement {self.__class__.__name__}.open" - ) # pragma: no cover diff --git a/cf/data/array/umarray.py b/cf/data/array/umarray.py index 4522ebdcfb..86baee9874 100644 --- a/cf/data/array/umarray.py +++ b/cf/data/array/umarray.py @@ -8,11 +8,11 @@ parse_indices, ) from ...umread_lib.umfile import File, Rec -from .mixin import FileArrayMixin from .abstract import Array +from .mixin import FileArrayMixin -class UMArray(FileArrrayMixin, Array): +class UMArray(FileArrayMixin, Array): """A sub-array stored in a PP or UM fields file.""" def __init__( diff --git a/cf/data/fragment/mixin/__init__.py b/cf/data/fragment/mixin/__init__.py index a4a35a1129..dad50f7451 100644 --- a/cf/data/fragment/mixin/__init__.py +++ b/cf/data/fragment/mixin/__init__.py @@ -1 +1,2 @@ from .fragmentarraymixin import FragmentArrayMixin +from .fragmentfilearraymixin import FragmentFileArrayMixin diff --git a/cf/mixin2/container.py b/cf/mixin2/container.py index 460ce61220..c8ca130d3e 100644 --- a/cf/mixin2/container.py +++ b/cf/mixin2/container.py @@ -4,7 +4,7 @@ situation. """ -from .docstring import _docstring_substitution_definitions +from ..docstring import _docstring_substitution_definitions class Container: diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 07f536107a..f0cd61d10a 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -362,73 +362,48 @@ def _customize_read_vars(self): super()._customize_read_vars() g = self.read_vars + if not g["cfa"]: + return - # Check the 'Conventions' for CFA - Conventions = g["global_attributes"].get("Conventions", "") - - # If the string contains any commas, it is assumed to be a - # comma-separated list. - all_conventions = split(",\s*", Conventions) - if all_conventions[0] == Conventions: - all_conventions = Conventions.split() - - CFA_version = None - for c in all_conventions: - if c.startswith("CFA-"): - CFA_version = c.replace("CFA-", "", 1) - break - - if c == "CFA": - # Versions <= 3.13.1 wrote CFA-0.4 files with a plain - # 'CFA' in the Conventions string - CFA_version = "0.4" - break - - g["cfa"] = CFA_version is not None - if g["cfa"]: - # -------------------------------------------------------- - # This is a CFA-netCDF file - # -------------------------------------------------------- - - # Check the CFA version - g["CFA_version"] = Version(CFA_version) - if g["CFA_version"] < Version("0.6.2"): - raise ValueError( - f"Can't read file {g['filename']} that uses obselete " - f"CFA conventions version CFA-{CFA_version}. " - "(Note that version 3.13.1 can be used to read and " - "write CFA-0.4 files.)" - ) - - # Get the pdirectory path of the CFA-netCDF file being - # read - from os.path import abspath - from pathlib import PurePath - - g["cfa_dir"] = PurePath(abspath(g["filename"])).parent - - # Process the aggregation instruction variables, and the - # aggregated dimensions. - dimensions = g["variable_dimensions"] - attributes = g["variable_attributes"] - for ncvar, attributes in attributes.items(): - if "aggregate_dimensions" not in attributes: - # This is not an aggregated variable - continue + # ------------------------------------------------------------ + # Still here? Then this is a CFA-netCDF file + # ------------------------------------------------------------ + if g["CFA_version"] < Version("0.6.2"): + raise ValueError( + f"Can't read file {g['filename']} that uses obselete " + f"CFA conventions version CFA-{g['CFA_version']}. " + "(Note that version 3.13.1 can be used to read and " + "write CFA-0.4 files.)" + ) + + # Get the directory of the CFA-netCDF file being read + from os.path import abspath + from pathlib import PurePath - # Set the aggregated variable's dimensions as its - # aggregated dimensions - ncdimensions = attributes["aggregated_dimensions"].split() - dimensions[ncvar] = tuple(map(str, ncdimensions)) - - # Do not create fields/domains from aggregation - # instruction variables - parsed_aggregated_data = self._parse_aggregated_data( - ncvar, attributes.get("aggregated_data") - ) - for x in parsed_aggregated_data: - variable = tuple(x.items())[0][1] - g["do_not_create_field"].add(variable) + g["cfa_dir"] = PurePath(abspath(g["filename"])).parent + + # Process the aggregation instruction variables, and the + # aggregated dimensions. + dimensions = g["variable_dimensions"] + attributes = g["variable_attributes"] + for ncvar, attributes in attributes.items(): + if "aggregate_dimensions" not in attributes: + # This is not an aggregated variable + continue + + # Set the aggregated variable's dimensions as its + # aggregated dimensions + ncdimensions = attributes["aggregated_dimensions"].split() + dimensions[ncvar] = tuple(map(str, ncdimensions)) + + # Do not create fields/domains from aggregation + # instruction variables + parsed_aggregated_data = self._parse_aggregated_data( + ncvar, attributes.get("aggregated_data") + ) + for x in parsed_aggregated_data: + term_ncvar = tuple(x.items())[0][1] + g["do_not_create_field"].add(term_ncvar) def _cache_data_elements(self, data, ncvar): """Cache selected element values. From 405b928dc8ba7f6b5890b6e1bf5a47d885802360 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 2 Mar 2023 14:58:13 +0000 Subject: [PATCH 038/141] dev --- cf/cfimplementation.py | 8 +++++ cf/read_write/netcdf/netcdfread.py | 2 +- cf/read_write/netcdf/netcdfwrite.py | 40 +++++++++++++------------ cf/read_write/read.py | 45 +++++++++++++++++++++++------ cf/read_write/write.py | 42 ++++++++++++++++----------- 5 files changed, 91 insertions(+), 46 deletions(-) diff --git a/cf/cfimplementation.py b/cf/cfimplementation.py index 78cf57eda2..be30955413 100644 --- a/cf/cfimplementation.py +++ b/cf/cfimplementation.py @@ -91,7 +91,9 @@ def initialise_CFANetCDFArray( units=False, calendar=False, instructions=None, + substitutions=None, term=None, + **kwargs, ): """Return a `CFANetCDFArray` instance. @@ -115,8 +117,13 @@ def initialise_CFANetCDFArray( instructions: `str`, optional + substitutions: `dict`, optional + term: `str`, optional + kwargs: optional + Ignored. + :Returns: `CFANetCDFArray` @@ -133,6 +140,7 @@ def initialise_CFANetCDFArray( units=units, calendar=calendar, instructions=instructions, + substitutions=substitutions, term=term, ) diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index f0cd61d10a..ae0b9f8e9e 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -572,7 +572,7 @@ def _create_cfanetcdfarray( # Apply user-defined substitutions, which take precedence over # those defined in the file. - subs = subs.update(g["cfa_substitutions"]) + subs = subs.update(g["cfa_options"].get("substitutions", {})) if subs: kwargs["substitutions"] = subs diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 732879b66e..b66aea544d 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -58,7 +58,7 @@ def _use_cfa(self, cfvar, construct_type): if not data.get_cfa_write(): return False - for ctype, ndim in g["cfa_options"]["constructs"]: + for ctype, ndim in g["cfa_options"].get("constructs", {}): # Write as CFA if it has an appropriate construct type ... if ctype in ("all", construct_type): # ... and then only if it satisfies the number of @@ -785,7 +785,7 @@ def _ggg(self, data, cfvar): instances containing the corresponding variables. """ - from os.path import abspath, relpath + from os.path import abspath, join, relpath from pathlib import PurePath from urllib.parse import urlparse @@ -794,18 +794,18 @@ def _ggg(self, data, cfvar): # Define the CFA file susbstitutions, giving precedence over # those set on the Data object to those provided by the CFA # options. - data.cfa_set_file_substitutions(g["cfa_options"]["substitutions"]) + data.cfa_set_file_substitutions( + g["cfa_options"].get("substitutions", {}) + ) substitutions = data.cfa_get_file_substitutions() - relative = g["cfa_options"].get("relative", None) - if relative: - absolute = False - cfa_dir = PurePath(abspath(g["filename"])).parent - elif relative is not None: - absolute = True - else: - absolute = None + # TODOCFA - review this! + paths = g["cfa_options"].get("paths") + relative = paths == "relative" + + cfa_dir = g['cfa_dir'] + # Size of the trailing dimension n_trailing = 0 @@ -833,14 +833,16 @@ def _ggg(self, data, cfvar): filenames2 = [] for filename in filenames: - parsed_filename = urlparse(filename) - scheme = parsed_filename.scheme - if scheme not in ("http", "https"): - path = parsed_filename.path - if absolute: - filename = PurePath(abspath(path)).as_uri() - elif relative or scheme != "file": - filename = relpath(abspath(path), start=cfa_dir) + uri = urlparse(filename) + uri_scheme = uri.scheme + if not uri_scheme: + filename = abspath(join(cfa_dir, filename)) + if relative: + filename = relpath(filename, start=cfa_dir) + else: + filename = PurePath(filename).as_uri() + elif relative and uri_scheme == "file": + filename = relpath(furi.path, start=cfa_dir) if substitutions: # Apply the CFA file susbstitutions diff --git a/cf/read_write/read.py b/cf/read_write/read.py index 329e6ddbc3..72ba412988 100644 --- a/cf/read_write/read.py +++ b/cf/read_write/read.py @@ -59,12 +59,16 @@ def read( warn_valid=False, chunks="auto", domain=False, - cfa_substitutions=None, + cfa_options=None, ): - """Read field constructs from netCDF, CDL, PP or UM fields datasets. + """Read field or domain constructs from files. - Input datasets are mapped to field constructs in memory which are - returned as elements of a `FieldList`. + The following file formats are supported: CF-netCDF, CFA-netCDF, + CDL, PP and UM fields datasets. + + Input datasets are mapped to constructs in memory which are + returned as elements of a `FieldList` or if the *domain* parameter + is True, a `DomainList`. NetCDF files may be on disk or on an OPeNDAP server. @@ -769,6 +773,28 @@ def read( f"when recursive={recursive!r}" ) + # Parse the 'cfa_options' parameter + if not cfa_options: + cfa_options = {} + else: + cfa_options = cfa_options.copy() + keys = ("substitutions",) + if not set(cfa_options).issubset(keys): + raise ValueError( + "Invalid dictionary key to the 'cfa_options' " + f"parameter. Valid keys are {keys}. Got: {cfa_options}" + ) + + cfa_options.setdefault("substitutions", {}) + + substitutions = cfa_options["substitutions"].copy() + for base, sub in substitutions.items(): + if not (base.startswith("${") and base.endswith("}")): + # Add missing ${...} + substitutions[f"${{{base}}}"] = substitutions.pop(base) + + cfa_options["substitutions"] = substitutions + # Initialise the output list of fields/domains if domain: out = DomainList() @@ -898,6 +924,7 @@ def read( warn_valid=warn_valid, select=select, domain=domain, + cfa_options=cfa_options, ) # -------------------------------------------------------- @@ -1009,6 +1036,7 @@ def _read_a_file( chunks="auto", select=None, domain=False, + cfa_options=None, ): """Read the contents of a single file into a field list. @@ -1039,6 +1067,9 @@ def _read_a_file( domain: `bool`, optional See `cf.read` for details. + cfa_options: `dict`, optional + See `cf.read` for details. + :Returns: `FieldList` or `DomainList` @@ -1072,11 +1103,7 @@ def _read_a_file( "chunks": chunks, "fmt": selected_fmt, "ignore_read_error": ignore_read_error, - # 'cfa' defaults to False. If the file has - # "CFA" in its Conventions global attribute - # then 'cfa' will be changed to True in - # netcdf.read - "cfa": False, + "cfa_options": cfa_options, } # ---------------------------------------------------------------- diff --git a/cf/read_write/write.py b/cf/read_write/write.py index 303535f1ed..293ad73bda 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -577,18 +577,18 @@ def write( cfa_options: `dict`, optional Parameters for configuring the output CFA-netCDF file. By - default *cfa_options* is ``{'paths': 'relative', + default *cfa_options* is ``{'paths': 'absolute', 'constructs': 'field'}`` and may have any subset of the following keys (and value types): * ``'paths'`` (`str`) - How to write fragment file names. Set to ``'relative'`` - (the default) for them to be written as relative to the - CFA-netCDF file being created, or else set to - ``'absolute'`` for them to be written as file URIs. Note - that in both cases, fragment file defined by fully - qualified URLs will always be written as such. + How to write fragment file names. Set to ``'absolute'`` + (the default) for them to be written as fully qualified + URIs, or else set to ``'relative'`` for them to be + relative as paths relative to the CFA-netCDF file being + created. Note that in both cases, fragment file defined + by fully qualified URLs will always be written as such. * ``'constructs'`` (`dict` or (sequence of) `str`) @@ -762,14 +762,21 @@ def write( raise ValueError( "Invalid dictionary key to the 'cfa_options' " f"parameter. Valid keys are {keys}. " - f"Got: {tuple(cfa_options)}" + f"Got: {cfa_options}" ) - cfa_options.setdefault("paths", "relative") + cfa_options.setdefault("paths", "absolute") cfa_options.setdefault("constructs", "field") cfa_options.setdefault("substitutions", {}) - cfa_options.setdefault("properties", ()) - +# cfa_options.setdefault("properties", ()) + + paths = ("relative", "absolute") + if cfa_options['paths'] not in paths: + raise ValueError( + "Invalid value of 'paths' CFA option. Valid paths " + f"are {paths}. Got: {cfa_options['paths']!r}" + ) + constructs = cfa_options["constructs"] if isinstance(constructs, dict): cfa_options["constructs"] = constructs.copy() @@ -782,16 +789,17 @@ def write( substitutions = cfa_options["substitutions"].copy() for base, sub in substitutions.items(): if not (base.startswith("${") and base.endswith("}")): + # Add missing ${...} substitutions[f"${{{base}}}"] = substitutions.pop(base) cfa_options["substitutions"] = substitutions - properties = cfa_options["properties"] - if isinstance(properties, str): - properties = (properties,) - - cfa_options["properties"] = tuple(properties) - + # properties = cfa_options["properties"] + # if isinstance(properties, str): + # properties = (properties,) + # + # cfa_options["properties"] = tuple(properties) + extra_write_vars["cfa"] = cfa extra_write_vars["cfa_options"] = cfa_options From d83e839cdef83152e722fd65d067d3e6ad9e531a Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 3 Mar 2023 19:34:09 +0000 Subject: [PATCH 039/141] dev --- cf/aggregate.py | 45 +++++--- cf/data/array/fullarray.py | 57 +++++++++- cf/data/array/mixin/filearraymixin.py | 2 + cf/data/data.py | 87 ++++++++++++-- .../fragment/mixin/fragmentfilearraymixin.py | 106 ++++++++++++++---- cf/field.py | 101 ++++++++++------- cf/read_write/netcdf/netcdfread.py | 2 - cf/read_write/netcdf/netcdfwrite.py | 10 +- cf/read_write/read.py | 15 ++- cf/read_write/write.py | 16 +-- 10 files changed, 344 insertions(+), 97 deletions(-) diff --git a/cf/aggregate.py b/cf/aggregate.py index 721368f580..7e1ac0ee56 100644 --- a/cf/aggregate.py +++ b/cf/aggregate.py @@ -2,19 +2,19 @@ from collections import namedtuple from operator import itemgetter +import numpy as np from cfdm import is_log_level_debug, is_log_level_detail, is_log_level_info -from numpy import argsort as numpy_argsort -from numpy import dtype as numpy_dtype -from numpy import sort as numpy_sort from .auxiliarycoordinate import AuxiliaryCoordinate -from .data.data import Data +from .data import Data +from .data.array import FullArray from .decorators import ( _manage_log_level_via_verbose_attr, _manage_log_level_via_verbosity, _reset_log_emergence_level, ) from .domainaxis import DomainAxis +from .fieldancillary import FieldAncillary from .fieldlist import FieldList from .functions import _DEPRECATION_ERROR_FUNCTION_KWARGS, _numpy_allclose from .functions import atol as cf_atol @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) -_dtype_float = numpy_dtype(float) +_dtype_float = np.dtype(float) # # -------------------------------------------------------------------- # # Global properties, as defined in Appendix A of the CF conventions. @@ -138,6 +138,7 @@ def __init__( relaxed_identities=False, ncvar_identities=False, field_identity=None, + field_ancillaries=(), copy=True, ): """**initialisation** @@ -322,8 +323,25 @@ def __init__( f.del_property(prop) - if dimension: - construct_axes = f.constructs.data_axes() + # ------------------------------------------------------------ + # Create field ancillaries from properties + # ------------------------------------------------------------ + for prop in field_ancillaries: + print (99999, prop) + value = f.get_property(prop, None) + if value is None: + continue + + data = FullArray(value, shape=f.shape, dtype=np.array(value).dtype) + + field_anc = FieldAncillary( + data=Data(data), properties={"long_name": prop} + ) + field_anc.id = prop + print (field_anc.dump(), Data(data).array) + f.set_construct(field_anc, axes=f.get_data_axes(), copy=False) + + construct_axes = f.constructs.data_axes() self.units = self.canonical_units( f, self.identity, relaxed_units=relaxed_units @@ -545,10 +563,10 @@ def __init__( # Field ancillaries # ------------------------------------------------------------ self.field_anc = {} - field_ancillaries = f.constructs.filter_by_type( + field_ancs = f.constructs.filter_by_type( "field_ancillary", todict=True ) - for key, field_anc in field_ancillaries.items(): + for key, field_anc in field_ancs.items(): # Find this field ancillary's identity identity = self.field_ancillary_has_identity_and_data(field_anc) if identity is None: @@ -1422,6 +1440,7 @@ def aggregate( no_overlap=False, shared_nc_domain=False, field_identity=None, + field_ancillaries=(), info=False, ): """Aggregate field constructs into as few field constructs as @@ -1806,7 +1825,7 @@ def aggregate( relaxed_identities=relaxed_identities, ncvar_identities=ncvar_identities, field_identity=field_identity, - respect_valid=respect_valid, + respect_valid=respect_valid,field_ancillaries=field_ancillaries, copy=copy, ) @@ -2219,7 +2238,7 @@ def _create_hash_and_first_values( # ... or which doesn't have a dimension coordinate but # does have one or more 1-d auxiliary coordinates aux = m_axis_identity["keys"][0] - sort_indices = numpy_argsort(field.constructs[aux].array) + sort_indices = np.argsort(field.constructs[aux].array) m_sort_keys[axis] = aux null_sort = False @@ -2661,8 +2680,8 @@ def _get_hfl( if create_flb: # Record the bounds of the first and last (sorted) cells - first = numpy_sort(array[0, ...]) - last = numpy_sort(array[-1, ...]) + first = np.sort(array[0, ...]) + last = np.sort(array[-1, ...]) hfl_cache.flb[key] = (first, last) if first_and_last_values or first_and_last_bounds: diff --git a/cf/data/array/fullarray.py b/cf/data/array/fullarray.py index 74dbd2d4b4..14e427c277 100644 --- a/cf/data/array/fullarray.py +++ b/cf/data/array/fullarray.py @@ -2,6 +2,8 @@ from .abstract import Array +_FULLARRAY_HANDLED_FUNCTIONS = {} + class FullArray(Array): """A array filled with a given value. @@ -88,6 +90,17 @@ def __init__( self._set_component("units", units, copy=False) self._set_component("calendar", calendar, copy=False) + def __array_function__(self, func, types, args, kwargs): + if func not in _FULLARRAY_HANDLED_FUNCTIONS: + return NotImplemented + + # Note: this allows subclasses that don't override + # __array_function__ to handle MyArray objects + if not all(issubclass(t, self.__class__) for t in types): + return NotImplemented + + return _FULLARRAY_HANDLED_FUNCTIONS[func](*args, **kwargs) + def __getitem__(self, indices): """x.__getitem__(indices) <==> x[indices] @@ -104,7 +117,7 @@ def __getitem__(self, indices): array_shape = self.shape else: array_shape = [] - for i, size, i in zip(indices, self.shape): + for i, size in zip(indices, self.shape): if not isinstance(i, slice): continue @@ -236,3 +249,45 @@ def set_full_value(self, fill_value): """ self._set_component("full_value", fill_value, copy=False) + + +def fullarray_implements(numpy_function): + """Register an __array_function__ implementation for FullArray objects. + + .. versionadded:: TODOCFAVER + + """ + + def decorator(func): + _FULLARRAY_HANDLED_FUNCTIONS[numpy_function] = func + return func + + return decorator + + +@fullarray_implements(np.unique) +def unique( + a, return_index=False, return_inverse=False, return_counts=False, axis=None +): + """Version of `np.unique` that is optimised for `FullArray` objects. + + .. versionadded:: TODOCFAVER + + """ + if return_index or return_inverse or return_counts or axis is not None: + # Fall back to the slow unique. (I'm sure we could probably do + # something more clever here, but there is no use case at + # present.) + return np.unique( + a[...], + return_index=return_index, + return_inverse=return_inverse, + return_counts=return_counts, + axis=axis, + ) + + x = a.get_full_value() + if x is np.ma.masked: + return np.ma.masked_all((1,), dtype=a.dtype) + + return np.array([x], dtype=a.dtype) diff --git a/cf/data/array/mixin/filearraymixin.py b/cf/data/array/mixin/filearraymixin.py index 78fdd8e081..27d0b64b06 100644 --- a/cf/data/array/mixin/filearraymixin.py +++ b/cf/data/array/mixin/filearraymixin.py @@ -1,5 +1,7 @@ import numpy as np +from ....functions import _DEPRECATION_ERROR_ATTRIBUTE + class FileArrayMixin: """Mixin class for an array stored in a file. diff --git a/cf/data/data.py b/cf/data/data.py index 1998984d80..17c54f07bc 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -29,7 +29,6 @@ from ..functions import ( _DEPRECATION_ERROR_KWARGS, _section, - abspath, atol, default_netCDF_fillvals, free_memory, @@ -2424,11 +2423,14 @@ def ceil(self, inplace=False, i=False): d._set_dask(da.ceil(dx)) return d - def cfa_add_fragment_location(self, location): + def cfa_del_fragment_location(self, location): """TODOCFADOCS .. versionadded:: TODOCFAVER + .. seealso:: `cfa_add_fragment_location`, + `cfa_fragment_locations` + :Parameters: location: `str` @@ -2437,16 +2439,87 @@ def cfa_add_fragment_location(self, location): :Returns: `None` - TODOCFADOCS **Examples** - >>> d.cfa_add_fragment_location('/data/model') + >>> d.cfa_del_fragment_location('/data/model') + + """ + from dask.base import collections_to_dsk + + dx = self.to_dask_array() + + updated = False + dsk = collections_to_dsk((dx,), optimize_graph=True) + for key, a in dsk.items(): + try: + dsk[key] = a.del_fragment_location(location) + except AttributeError: + # This chunk doesn't contain a CFA fragment + continue + else: + # This chunk contains a CFA fragment + updated = True + + if updated: + dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) + self._set_dask(dx, clear=_NONE) + + def cfa_fragment_locations(self, location): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + .. seealso:: `cfa_del_fragment_location`, + `cfa_set_fragment_location` + + :Returns: + + `set` + + **Examples** + + >>> d.cfa_fragment_locations() + {'/home/data1', 'file:///data2'} """ from dask.base import collections_to_dsk - location = abspath(location) + out = set() + + dsk = collections_to_dsk((self.to_dask_array(),), optimize_graph=True) + for key, a in dsk.items(): + try: + out.update(a.fragment_locations()) + except AttributeError: + # This chunk doesn't contain a CFA fragment + pass + + return out + + def cfa_set_fragment_location(self, location): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + .. seealso:: `cfa_del_fragment_location`, + `cfa_fragment_locations` + + :Parameters: + + location: `str` + TODOCFADOCS + + :Returns: + + `None` + + **Examples** + + >>> d.cfa_set_fragment_location('/data/model') + + """ + from dask.base import collections_to_dsk dx = self.to_dask_array() @@ -2454,9 +2527,9 @@ def cfa_add_fragment_location(self, location): dsk = collections_to_dsk((dx,), optimize_graph=True) for key, a in dsk.items(): try: - dsk[key] = a.add_fragment_location(location) + dsk[key] = a.set_fragment_location(location) except AttributeError: - # This chunk doesn't contain CFA fragment + # This chunk doesn't contain a CFA fragment continue else: # This chunk contains a CFA fragment diff --git a/cf/data/fragment/mixin/fragmentfilearraymixin.py b/cf/data/fragment/mixin/fragmentfilearraymixin.py index ff32111a28..90e21934a4 100644 --- a/cf/data/fragment/mixin/fragmentfilearraymixin.py +++ b/cf/data/fragment/mixin/fragmentfilearraymixin.py @@ -5,7 +5,7 @@ class FragmentFileArrayMixin: """ - def add_fragment_location(self, location): + def del_fragment_location(self, location): """TODOCFADOCS .. versionadded:: TODOCFAVER @@ -21,33 +21,48 @@ def add_fragment_location(self, location): TODOCFADOCS """ - from os.path import basename, dirname, join + from os import sep + from os.path import dirname a = self.copy() + location = location.rstrip(sep) # Note: It is assumed that each existing file name is either # an absolute path or a file URI. - filenames = a.get_filenames() - addresses = a.get_addresses() - - new_filenames = tuple( - [ - join(location, basename(f)) - for f in filenames - if dirname(f) != location - ] - ) - # TODOCFA - how does this work out with URLs and file URIs? + new_filenames = [] + new_addresses = [] + for filename, address in zip(a.get_filenames(), a.get_addresses()): + if dirname(filename) != location: + new_filenames.append(filename) + new_addresses.append(address) - a._set_component("filenames", filenames + new_filenames, copy=False) - a._set_component( - "addresses", - addresses + addresses[-1] * len(new_filenames), - copy=False, - ) + a._set_component("filenames", new_filenames, copy=False) + a._set_component("addresses", new_addresses, copy=False) return a + def fragment_locations(self): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + location: `str` + TODOCFADOCS + + :Returns: + + `{{class}}` + TODOCFADOCS + + """ + from os.path import dirname + + # Note: It is assumed that each existing file name is either + # an absolute path or a file URI. + return set([dirname(f) for f in self.get_filenames()]) + def get_addresses(self, default=AttributeError()): """TODOCFADOCS Return the names of any files containing the data array. @@ -97,3 +112,56 @@ def get_formats(self, default=AttributeError()): """ return (self.get_format(),) * len(self.get_filenames(default)) + + def set_fragment_location(self, location): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + location: `str` + TODOCFADOCS + + :Returns: + + `{{class}}` + TODOCFADOCS + + """ + from os import sep + from os.path import basename, dirname, join + + a = self.copy() + location = location.rstrip(sep) + + filenames = a.get_filenames() + addresses = a.get_addresses() + + # Note: It is assumed that each existing file name is either + # an absolute path or a fully qualified URI. + new_filenames = [] + new_addresses = [] + basenames = [] + for filename, address in zip(filenames, addresses): + if dirname(filename) == location: + continue + + basename = basename(filename) + if basename in basenames: + continue + + basenames.append(filename) + new_filenames.append(join(location, basename)) + new_addresses.append(address) + + a._set_component( + "filenames", filenames + tuple(new_filenames), copy=False + ) + a._set_component( + "addresses", + addresses + tuple(new_addresses), + copy=False, + ) + + return a diff --git a/cf/field.py b/cf/field.py index a532910ae5..1f173232b2 100644 --- a/cf/field.py +++ b/cf/field.py @@ -3642,38 +3642,6 @@ def cell_area( return w - def cfa_add_fragment_location( - self, - location, - constructs=True, - ): - """TODOCFADOCS - - .. versionadded:: TODOCFAVER - - :Parameters: - - location: `str` - TODOCFADOCS - - :Returns: - - `None` - - **Examples** - - >>> f.cfa_add_fragment_location('/data/model') - - """ - super().add_fragment_location( - location, - ) - if constructs: - for c in self.constructs.filter_by_data(todict=True).values(): - c.add_fragment_location( - location, - ) - def cfa_get_file_substitutions(self, constructs=True): """TODOCFADOCS @@ -3690,6 +3658,7 @@ def cfa_get_file_substitutions(self, constructs=True): """ out = super().cfa_get_file_substitutions() + if constructs: for c in self.constructs.filter_by_data(todict=True).values(): out.update(c.cfa_set_file_substitution()) @@ -3726,14 +3695,40 @@ def cfa_del_file_substitution( >>> f.cfa_del_file_substitution('base', '/data/model') """ - super().cfa_del_file_substitution( - base, - ) + super().cfa_del_file_substitution(base) + if constructs: for c in self.constructs.filter_by_data(todict=True).values(): - c.cfa_del_file_substitution( - base, - ) + c.cfa_del_file_substitution(base) + + def cfa_del_fragment_location( + self, + location, + constructs=True, + ): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + location: `str` + TODOCFADOCS + + :Returns: + + `None` + + **Examples** + + >>> f.cfa_set_fragment_location('/data/model') + + """ + super().cfa_del_fragment_location(location) + + if constructs: + for c in self.constructs.filter_by_data(todict=True).values(): + c.cfa_del_fragment_location(location) def cfa_set_file_substitutions( self, @@ -3769,10 +3764,40 @@ def cfa_set_file_substitutions( """ super().cfa_set_file_substitutions(value) + if constructs: for c in self.constructs.filter_by_data(todict=True).values(): c.cfa_set_file_substitutions(value) + def cfa_set_fragment_location( + self, + location, + constructs=True, + ): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + location: `str` + TODOCFADOCS + + :Returns: + + `None` + + **Examples** + + >>> f.cfa_set_fragment_location('/data/model') + + """ + super().cfa_set_fragment_location(location) + + if constructs: + for c in self.constructs.filter_by_data(todict=True).values(): + c.cfa_set_fragment_location(location) + def radius(self, default=None): """Return the radius of a latitude-longitude plane defined in spherical polar coordinates. diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index ae0b9f8e9e..38619b0fa0 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -357,8 +357,6 @@ def _customize_read_vars(self): .. versionadded:: 3.0.0 """ - from re import split - super()._customize_read_vars() g = self.read_vars diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index b66aea544d..c864fd102a 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -799,13 +799,9 @@ def _ggg(self, data, cfvar): ) substitutions = data.cfa_get_file_substitutions() + relative = g["cfa_options"].get("relative_paths") + cfa_dir = g["cfa_dir"] - # TODOCFA - review this! - paths = g["cfa_options"].get("paths") - relative = paths == "relative" - - cfa_dir = g['cfa_dir'] - # Size of the trailing dimension n_trailing = 0 @@ -842,7 +838,7 @@ def _ggg(self, data, cfvar): else: filename = PurePath(filename).as_uri() elif relative and uri_scheme == "file": - filename = relpath(furi.path, start=cfa_dir) + filename = relpath(uri.path, start=cfa_dir) if substitutions: # Apply the CFA file susbstitutions diff --git a/cf/read_write/read.py b/cf/read_write/read.py index 72ba412988..f051708f57 100644 --- a/cf/read_write/read.py +++ b/cf/read_write/read.py @@ -778,7 +778,7 @@ def read( cfa_options = {} else: cfa_options = cfa_options.copy() - keys = ("substitutions",) + keys = ("substitutions", "field_ancillaries") if not set(cfa_options).issubset(keys): raise ValueError( "Invalid dictionary key to the 'cfa_options' " @@ -786,15 +786,22 @@ def read( ) cfa_options.setdefault("substitutions", {}) - + substitutions = cfa_options["substitutions"].copy() for base, sub in substitutions.items(): if not (base.startswith("${") and base.endswith("}")): # Add missing ${...} substitutions[f"${{{base}}}"] = substitutions.pop(base) - + cfa_options["substitutions"] = substitutions + field_ancillaries = cfa_options.pop("field_ancillaries", None) + if field_ancillaries: + if "field_ancillaries" in aggregate_options: + raise ValueError("TODOCFA") + + aggregate_options["field_ancillaries"] = field_ancillaries + # Initialise the output list of fields/domains if domain: out = DomainList() @@ -1070,6 +1077,8 @@ def _read_a_file( cfa_options: `dict`, optional See `cf.read` for details. + .. versionadded:: TODOCFAVER + :Returns: `FieldList` or `DomainList` diff --git a/cf/read_write/write.py b/cf/read_write/write.py index 293ad73bda..c00a1f9389 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -768,15 +768,17 @@ def write( cfa_options.setdefault("paths", "absolute") cfa_options.setdefault("constructs", "field") cfa_options.setdefault("substitutions", {}) -# cfa_options.setdefault("properties", ()) + # cfa_options.setdefault("properties", ()) - paths = ("relative", "absolute") - if cfa_options['paths'] not in paths: - raise ValueError( + paths = cfa_options.pop("paths") + if paths not in ("relative", "absolute"): + raise ValueError( "Invalid value of 'paths' CFA option. Valid paths " - f"are {paths}. Got: {cfa_options['paths']!r}" + f"are 'relative' and 'absolute'. Got: {paths!r}" ) - + + cfa_options["relative_paths"] = paths == "relative" + constructs = cfa_options["constructs"] if isinstance(constructs, dict): cfa_options["constructs"] = constructs.copy() @@ -799,7 +801,7 @@ def write( # properties = (properties,) # # cfa_options["properties"] = tuple(properties) - + extra_write_vars["cfa"] = cfa extra_write_vars["cfa_options"] = cfa_options From fe3867f1afbbaf27798e7b09e3a7e89c7eed9936 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 3 Mar 2023 23:04:20 +0000 Subject: [PATCH 040/141] dev --- cf/aggregate.py | 7 ++++--- cf/data/data.py | 6 +++--- cf/read_write/read.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/cf/aggregate.py b/cf/aggregate.py index 7e1ac0ee56..9434d08805 100644 --- a/cf/aggregate.py +++ b/cf/aggregate.py @@ -327,7 +327,7 @@ def __init__( # Create field ancillaries from properties # ------------------------------------------------------------ for prop in field_ancillaries: - print (99999, prop) + print(99999, prop) value = f.get_property(prop, None) if value is None: continue @@ -338,7 +338,7 @@ def __init__( data=Data(data), properties={"long_name": prop} ) field_anc.id = prop - print (field_anc.dump(), Data(data).array) + print(field_anc.dump(), Data(data).array) f.set_construct(field_anc, axes=f.get_data_axes(), copy=False) construct_axes = f.constructs.data_axes() @@ -1825,7 +1825,8 @@ def aggregate( relaxed_identities=relaxed_identities, ncvar_identities=ncvar_identities, field_identity=field_identity, - respect_valid=respect_valid,field_ancillaries=field_ancillaries, + respect_valid=respect_valid, + field_ancillaries=field_ancillaries, copy=copy, ) diff --git a/cf/data/data.py b/cf/data/data.py index 17c54f07bc..ebfabb5fda 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1306,9 +1306,9 @@ def _clear_after_dask_update(self, clear=_ALL): # Set the CFA write status to False self._cfa_del_write() - # Always set the CFA term status to False - if "cfa_term" in self._custom: - del self._custom["cfa_term"] + # # Always set the CFA term status to False + # if "cfa_term" in self._custom: + # del self._custom["cfa_term"] def _set_dask(self, array, copy=False, clear=_ALL): """Set the dask array. diff --git a/cf/read_write/read.py b/cf/read_write/read.py index f051708f57..bb1b3c5be1 100644 --- a/cf/read_write/read.py +++ b/cf/read_write/read.py @@ -801,7 +801,7 @@ def read( raise ValueError("TODOCFA") aggregate_options["field_ancillaries"] = field_ancillaries - + # Initialise the output list of fields/domains if domain: out = DomainList() From 1786eaf71ffa69f27178a04b725330a52f904c1a Mon Sep 17 00:00:00 2001 From: David Hassell Date: Sat, 4 Mar 2023 11:44:45 +0000 Subject: [PATCH 041/141] dev --- cf/aggregate.py | 167 +++++++++++++++++++++++++++---------- cf/data/array/fullarray.py | 1 + cf/read_write/read.py | 9 +- 3 files changed, 124 insertions(+), 53 deletions(-) diff --git a/cf/aggregate.py b/cf/aggregate.py index 9434d08805..81e9b16e3a 100644 --- a/cf/aggregate.py +++ b/cf/aggregate.py @@ -134,7 +134,7 @@ def __init__( equal=None, exist=None, ignore=None, - dimension=(), + dimension=None, relaxed_identities=False, ncvar_identities=False, field_identity=None, @@ -208,6 +208,11 @@ def __init__( coordinate whose datum is the property's value and the property itself is deleted from that field. + field_ancillaries: (sequence of) `str`, optional + TODOCFADOCS. See `cf.aggregate` for details. + + .. versionadded:: TODOCFAVER + copy: `bool` optional If False then do not copy fields prior to aggregation. Setting this option to False may change input fields in @@ -290,56 +295,20 @@ def __init__( "no identity; consider setting " "relaxed_identities" ) return - # elif not self.has_data: - # self.message = "{} has no data".format(f.__class__.__name__) - # return # ------------------------------------------------------------ # Promote selected properties to 1-d, size 1 auxiliary - # coordinates + # coordinates with new independent domain axes # ------------------------------------------------------------ - _copy = copy - for prop in dimension: - value = f.get_property(prop, None) - if value is None: - continue - - aux_coord = AuxiliaryCoordinate( - properties={"long_name": prop}, - data=Data([value], units=""), - copy=False, - ) - aux_coord.nc_set_variable(prop) - aux_coord.id = prop - - if _copy: - # Copy the field, as we're about to change it. - f = f.copy() - self.field = f - _copy = False - - axis = f.set_construct(DomainAxis(1)) - f.set_construct(aux_coord, axes=[axis], copy=False) - - f.del_property(prop) + if dimension: + f = self.promote_to_auxiliary_coordinate(dimension) # ------------------------------------------------------------ - # Create field ancillaries from properties + # Promote selected properties to field ancillaries that span + # the same domain axes as the field # ------------------------------------------------------------ - for prop in field_ancillaries: - print(99999, prop) - value = f.get_property(prop, None) - if value is None: - continue - - data = FullArray(value, shape=f.shape, dtype=np.array(value).dtype) - - field_anc = FieldAncillary( - data=Data(data), properties={"long_name": prop} - ) - field_anc.id = prop - print(field_anc.dump(), Data(data).array) - f.set_construct(field_anc, axes=f.get_data_axes(), copy=False) + if field_ancillaries: + f = self.promote_to_field_ancillary(field_ancillaries) construct_axes = f.constructs.data_axes() @@ -1412,6 +1381,105 @@ def find_coordrefs(self, key): return tuple(sorted(names)) + def promote_to_auxiliary_coordinate(self, properties): + """Promote properties to auxilliary coordinate constructs. + + Each property is converted to a 1-d auxilliary coordinate + construct that spans a new independent size 1 domain axis the + field, and the property is deleted. + + ... versionadded:: TODOCFAVER + + :Parameters: + + properties: sequence of `str` + TODOCFADOCS + + :Returns: + + `Field` or `Domain` + TODOCFADOCS + + """ + f = self.field + + copy = True + for prop in properties: + value = f.get_property(prop, None) + if value is None: + continue + + aux_coord = AuxiliaryCoordinate( + properties={"long_name": prop}, + data=Data([value], units=""), + copy=False, + ) + aux_coord.nc_set_variable(prop) + aux_coord.id = prop + + if copy: + # Copy the field as we're about to change it + f = f.copy() + copy = False + + axis = f.set_construct(DomainAxis(1)) + f.set_construct(aux_coord, axes=[axis], copy=False) + f.del_property(prop) + + self.field = f + return f + + def promote_to_field_ancillary(self, properties): + """Promote properties to field ancillary constructs. + + Each property is converted to a field ancillary construct that + span the same domain axes as the field, and property the is + deleted. + + If a domain construct is being aggregated then it is always + returned unchanged. + + ... versionadded:: TODOCFAVER + + :Parameters: + + properties: sequence of `str` + TODOCFADOCS + + :Returns: + + `Field` or `Domain` + TODOCFADOCS + + """ + f = self.field + if f.construct_type != "field": + return f + + copy = True + for prop in properties: + value = f.get_property(prop, None) + if value is None: + continue + + data = FullArray(value, shape=f.shape, dtype=np.array(value).dtype) + + field_anc = FieldAncillary( + data=Data(data), properties={"long_name": prop} + ) + field_anc.id = prop + + if copy: + # Copy the field as we're about to change it + f = f.copy() + copy = False + + f.set_construct(field_anc, axes=f.get_data_axes(), copy=False) + f.del_property(prop) + + self.field = f + return f + @_manage_log_level_via_verbosity def aggregate( @@ -1440,7 +1508,7 @@ def aggregate( no_overlap=False, shared_nc_domain=False, field_identity=None, - field_ancillaries=(), + field_ancillaries=None, info=False, ): """Aggregate field constructs into as few field constructs as @@ -1667,6 +1735,11 @@ def aggregate( numbers. The default value is set by the `cf.rtol` function. + field_ancillaries: (sequence of) `str`, optional + TODOCFADOCS + + .. versionadded:: TODOCFAVER + no_overlap: Use the *overlap* parameter instead. @@ -1723,6 +1796,7 @@ def aggregate( "\ninfo=2 maps to verbose=3" "\ninfo=3 maps to verbose=-1", version="3.5.0", + removed_at="4.0.0", ) # pragma: no cover # Initialise the cache for coordinate and cell measure hashes, @@ -1756,6 +1830,9 @@ def aggregate( if isinstance(dimension, str): dimension = (dimension,) + if isinstance(field_ancillaries, str): + field_ancillaries = (field_ancillaries,) + if exist_all and equal_all: raise ValueError( "Only one of 'exist_all' and 'equal_all' can be True, since " diff --git a/cf/data/array/fullarray.py b/cf/data/array/fullarray.py index 14e427c277..3de8a7804d 100644 --- a/cf/data/array/fullarray.py +++ b/cf/data/array/fullarray.py @@ -91,6 +91,7 @@ def __init__( self._set_component("calendar", calendar, copy=False) def __array_function__(self, func, types, args, kwargs): + """TODOCFADOCS""" if func not in _FULLARRAY_HANDLED_FUNCTIONS: return NotImplemented diff --git a/cf/read_write/read.py b/cf/read_write/read.py index bb1b3c5be1..50298b44fc 100644 --- a/cf/read_write/read.py +++ b/cf/read_write/read.py @@ -778,7 +778,7 @@ def read( cfa_options = {} else: cfa_options = cfa_options.copy() - keys = ("substitutions", "field_ancillaries") + keys = ("substitutions",) if not set(cfa_options).issubset(keys): raise ValueError( "Invalid dictionary key to the 'cfa_options' " @@ -795,13 +795,6 @@ def read( cfa_options["substitutions"] = substitutions - field_ancillaries = cfa_options.pop("field_ancillaries", None) - if field_ancillaries: - if "field_ancillaries" in aggregate_options: - raise ValueError("TODOCFA") - - aggregate_options["field_ancillaries"] = field_ancillaries - # Initialise the output list of fields/domains if domain: out = DomainList() From da0747bbf1ed0beb1a2b0daca0c0649db2b6203b Mon Sep 17 00:00:00 2001 From: David Hassell Date: Sat, 4 Mar 2023 14:05:22 +0000 Subject: [PATCH 042/141] dev --- cf/aggregate.py | 18 +++++++--- cf/data/data.py | 94 +++++++++++++++++++++++++++++++++---------------- 2 files changed, 78 insertions(+), 34 deletions(-) diff --git a/cf/aggregate.py b/cf/aggregate.py index 81e9b16e3a..9ac56d1550 100644 --- a/cf/aggregate.py +++ b/cf/aggregate.py @@ -1436,8 +1436,15 @@ def promote_to_field_ancillary(self, properties): span the same domain axes as the field, and property the is deleted. + The `Data` of any the new field ancillary construct is marked + as a CFA term, meaning that it will only be written to disk if + the parent field construct is written as CFA aggregation + variable, and in that case the field ancillary is written as + non-standard CFA aggregation instruction variable, rather than + a CF-netCDF ancillary variable. + If a domain construct is being aggregated then it is always - returned unchanged. + returned unchanged ... versionadded:: TODOCFAVER @@ -1449,7 +1456,7 @@ def promote_to_field_ancillary(self, properties): :Returns: `Field` or `Domain` - TODOCFADOCS + The TODOCFADOCS """ f = self.field @@ -1462,10 +1469,13 @@ def promote_to_field_ancillary(self, properties): if value is None: continue - data = FullArray(value, shape=f.shape, dtype=np.array(value).dtype) + data = Data( + FullArray(value, shape=f.shape, dtype=np.array(value).dtype) + ) + data._cfa_set_term(True) field_anc = FieldAncillary( - data=Data(data), properties={"long_name": prop} + data=data, properties={"long_name": prop} ) field_anc.id = prop diff --git a/cf/data/data.py b/cf/data/data.py index ebfabb5fda..ff6ae14e9e 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1306,10 +1306,6 @@ def _clear_after_dask_update(self, clear=_ALL): # Set the CFA write status to False self._cfa_del_write() - # # Always set the CFA term status to False - # if "cfa_term" in self._custom: - # del self._custom["cfa_term"] - def _set_dask(self, array, copy=False, clear=_ALL): """Set the dask array. @@ -1462,6 +1458,25 @@ def _cfa_del_write(self): """ return self._custom.pop("cfa_write", False) + def _cfa_set_term(self, value): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + .. seealso:: `_cfa_get_term` + + :Parameters: + + value: `bool` + TODOCFADOCS + + :Returns: + + `None` + + """ + self._custom["cfa_term"] = bool(value) + def _set_cached_elements(self, elements): """Cache selected element values. @@ -2449,6 +2464,10 @@ def cfa_del_fragment_location(self, location): dx = self.to_dask_array() + # TODOCFA what if the data definitions are FileArray, rather + # than FragmentArray? Perhaps allow extra locations to be + # addby cf.write cfa_options? + updated = False dsk = collections_to_dsk((dx,), optimize_graph=True) for key, a in dsk.items(): @@ -2485,6 +2504,9 @@ def cfa_fragment_locations(self, location): """ from dask.base import collections_to_dsk + # TODOCFA what if the data definitions are FileArray, rather + # than FragmentArray? + out = set() dsk = collections_to_dsk((self.to_dask_array(),), optimize_graph=True) @@ -2497,6 +2519,44 @@ def cfa_fragment_locations(self, location): return out + def cfa_get_term(self): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Returns: + + `bool` + + """ + return bool(self._custom.get("cfa_term", False)) + + def cfa_get_write(self): + """The CFA write status of the data. + + If and only if the CFA write status is `True`, then this + `Data` instance has the potential to be written to a + CFA-netCDF file as aggregated data. In this case it is the + choice of parameters to the `cf.write` function that + determines if the data is actually written as aggregated data. + + .. versionadded:: TODOCFAVER + + .. seealso:: `cfa_set_write`, `cf.read`, `cf.write` + + :Returns: + + `bool` + + **Examples** + + >>> d = cf.Data([1, 2]) + >>> d.cfa_get_write() + False + + """ + return bool(self._custom.get("cfa_write", False)) + def cfa_set_fragment_location(self, location): """TODOCFADOCS @@ -2539,32 +2599,6 @@ def cfa_set_fragment_location(self, location): dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) self._set_dask(dx, clear=_NONE) - def cfa_get_write(self): - """The CFA write status of the data. - - If and only if the CFA write status is `True`, then this - `Data` instance has the potential to be written to a - CFA-netCDF file as aggregated data. In this case it is the - choice of parameters to the `cf.write` function that - determines if the data is actually written as aggregated data. - - .. versionadded:: TODOCFAVER - - .. seealso:: `cfa_set_write`, `cf.read`, `cf.write` - - :Returns: - - `bool` - - **Examples** - - >>> d = cf.Data([1, 2]) - >>> d.cfa_get_write() - False - - """ - return self._custom.get("cfa_write", False) - def cfa_set_write(self, status): """Set the CFA write status of the data. From 7e6e0cb3d2125d47e7e717e33d095956776cd1c4 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Sun, 5 Mar 2023 11:03:42 +0000 Subject: [PATCH 043/141] dev --- cf/data/data.py | 124 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 91 insertions(+), 33 deletions(-) diff --git a/cf/data/data.py b/cf/data/data.py index ff6ae14e9e..6876e3c4ea 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1238,6 +1238,21 @@ def __keepdims_indexing__(self): def __keepdims_indexing__(self, value): self._custom["__keepdims_indexing__"] = bool(value) + def _cfa_del_write(self): + """Set the CFA write status of the data to `False`. + + .. versionadded:: TODOCFAVER + + .. seealso:: `cfa_get_write`, `_cfa_set_write` + + :Returns: + + `bool` + The CFA status prior to deletion. + + """ + return self._custom.pop("cfa_write", False) + def _clear_after_dask_update(self, clear=_ALL): """Remove components invalidated by updating the `dask` array. @@ -1443,39 +1458,36 @@ def _del_cached_elements(self): for element in ("first_element", "second_element", "last_element"): custom.pop(element, None) - def _cfa_del_write(self): - """Set the CFA write status of the data to `False`. - - .. versionadded:: TODOCFAVER - - .. seealso:: `cfa_get_write`, `_cfa_set_write` - - :Returns: - - `bool` - The CFA status prior to deletion. - - """ - return self._custom.pop("cfa_write", False) + def _get_cached_elements(self, elements): + """Cache selected element values. - def _cfa_set_term(self, value): - """TODOCFADOCS + Updates *data* in-place to store the given element values + within its ``custom`` dictionary. .. versionadded:: TODOCFAVER - .. seealso:: `_cfa_get_term` + .. seealso:: `_del_cached_elements`, `_set_cached_elements` :Parameters: - value: `bool` - TODOCFADOCS + elements: `dict` + Zero or more element values to be cached, each keyed by + a unique identifier to allow unambiguous retrieval. + Existing cached elements not specified by *elements* + will not be removed. :Returns: `None` + **Examples** + + >>> d._set_cached_elements({'first_element': 273.15}) + """ - self._custom["cfa_term"] = bool(value) + custom = self._custom + return { + key, custom[key] for key in ("first_element", "second_element", "last_element")} def _set_cached_elements(self, elements): """Cache selected element values. @@ -2520,14 +2532,22 @@ def cfa_fragment_locations(self, location): return out def cfa_get_term(self): - """TODOCFADOCS + """The CFA aggregation instruction term status. .. versionadded:: TODOCFAVER + .. seealso:: `cfa_set_term` + :Returns: `bool` + **Examples** + + >>> d = cf.Data([1, 2]) + >>> d.cfa_get_term() + False + """ return bool(self._custom.get("cfa_term", False)) @@ -2599,6 +2619,31 @@ def cfa_set_fragment_location(self, location): dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) self._set_dask(dx, clear=_NONE) + def cfa_set_term(self, status): + """Set the CFA aggregation instruction term status. + + .. versionadded:: TODOCFAVER + + .. seealso:: `cfa_get_term` + + :Parameters: + + status: `bool` + The new CFA aggregation instruction term status. + + :Returns: + + `None` + + """ + if status: + raise ValueError( + "'cfa_set_term' only allows the CFA aggregation instruction " + "term write status to be set to False" + ) + + self._custom.pop("cfa_term", False) + def cfa_set_write(self, status): """Set the CFA write status of the data. @@ -3980,23 +4025,29 @@ def concatenate(cls, data, axis=0, cull_graph=True, relaxed_units=False): non_concat_axis_chunks = list(d.chunks) non_concat_axis_chunks.pop(axis) if non_concat_axis_chunks != non_concat_axis_chunks0: - # ... or the CFA write status is False when any - # two input data instances have different chunk + # ... the CFA write status is False when any two + # input data instances have different chunk # patterns for the non-concatenated axes. cfa = _NONE break - # Set the new dask array, retaining the cached elements ... - data0._set_dask(dx, clear=_ALL ^ _CACHE ^ cfa) + # Set the new dask array + data0._set_dask(dx, clear=_ALL ^ cfa) - # ... now delete the cached second element, which might now be - # incorrect. - data0._custom.pop("second_element", None) - - # Set the CFA-netCDF aggregated_data instructions - # substitutions by combining them from all of the input data - # instances, giving precedence to those towards the left hand - # side of the input list. + # Retain valid cached elements + cache = processed_data[0]._get_cached_elements() + last_element = processed_data[-1]._custom.get("last_element", None) + if last_element is None: + cache.pop("last_element", None) + else: + cache["last_element"] = last_element + + data0._set_cached_elements(cache) + + # Set the CFA-netCDF aggregated data instructions and file + # name substitutions by combining them from all of the input + # data instances, giving precedence to those towards the left + # hand side of the input list. if data0.cfa_get_write(): aggregated_data = {} substitutions = {} @@ -4007,6 +4058,13 @@ def concatenate(cls, data, axis=0, cull_graph=True, relaxed_units=False): data0.cfa_set_aggregated_data(aggregated_data) data0.cfa_set_file_substitutions(substitutions) + # Set the CFA aggregation instruction term status + if data0.cfa_get_term(): + for d in processed_data[1:]: + if not d.cfa_get_term(): + data0.cfa_set_term(False) + break + # Manage cyclicity of axes: if join axis was cyclic, it is no # longer. axis = data0._parse_axes(axis)[0] From 758f840e2e85ba548ab357850985f7a73829bff8 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Sun, 5 Mar 2023 12:02:16 +0000 Subject: [PATCH 044/141] dev --- cf/data/data.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/cf/data/data.py b/cf/data/data.py index 6876e3c4ea..ac5f505be6 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1458,7 +1458,7 @@ def _del_cached_elements(self): for element in ("first_element", "second_element", "last_element"): custom.pop(element, None) - def _get_cached_elements(self, elements): + def _get_cached_elements(self): """Cache selected element values. Updates *data* in-place to store the given element values @@ -1468,14 +1468,6 @@ def _get_cached_elements(self, elements): .. seealso:: `_del_cached_elements`, `_set_cached_elements` - :Parameters: - - elements: `dict` - Zero or more element values to be cached, each keyed by - a unique identifier to allow unambiguous retrieval. - Existing cached elements not specified by *elements* - will not be removed. - :Returns: `None` @@ -1487,7 +1479,7 @@ def _get_cached_elements(self, elements): """ custom = self._custom return { - key, custom[key] for key in ("first_element", "second_element", "last_element")} + key: custom[key] for key in ("first_element", "second_element", "last_element")} def _set_cached_elements(self, elements): """Cache selected element values. From 9e4600abd5badfbc31353579549e73dccfc1ad6e Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 6 Mar 2023 14:17:30 +0000 Subject: [PATCH 045/141] dev --- cf/data/array/mixin/filearraymixin.py | 205 ++++++++++- cf/data/array/netcdfarray.py | 6 + cf/data/array/umarray.py | 244 +++++++------ cf/data/data.py | 24 +- cf/data/fragment/mixin/__init__.py | 3 +- .../fragment/mixin/fragmentfilearraymixin.py | 339 +++++++++--------- cf/data/fragment/netcdffragmentarray.py | 97 +++-- cf/data/fragment/umfragmentarray.py | 153 ++++---- cf/read_write/netcdf/netcdfread.py | 4 +- cf/read_write/netcdf/netcdfwrite.py | 24 +- cf/read_write/um/umread.py | 18 +- 11 files changed, 650 insertions(+), 467 deletions(-) diff --git a/cf/data/array/mixin/filearraymixin.py b/cf/data/array/mixin/filearraymixin.py index 27d0b64b06..84366d66df 100644 --- a/cf/data/array/mixin/filearraymixin.py +++ b/cf/data/array/mixin/filearraymixin.py @@ -2,8 +2,11 @@ from ....functions import _DEPRECATION_ERROR_ATTRIBUTE +# import cfdm -class FileArrayMixin: + + +class FileArrayMixin: # (cfdm.data.mixin.FileArrayMixin): """Mixin class for an array stored in a file. .. versionadded:: 3.14.0 @@ -32,10 +35,10 @@ def __str__(self): """x.__str__() <==> str(x)""" return f"{self.get_filename()}, {self.get_address()}" - @property - def dtype(self): - """Data-type of the array.""" - return self._get_component("dtype") + # @property + # def dtype(self): + # """Data-type of the array.""" + # return self._get_component("dtype") @property def filename(self): @@ -52,7 +55,191 @@ def filename(self): removed_at="5.0.0", ) # pragma: no cover - @property - def shape(self): - """Shape of the array.""" - return self._get_component("shape") + # @property + # def shape(self): + # """Shape of the array.""" + # return self._get_component("shape") + + def del_file_location(self, location): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + location: `str` + TODOCFADOCS + + :Returns: + + `{{class}}` + TODOCFADOCS + + """ + from os import sep + from os.path import dirname + + location = location.rstrip(sep) + + # Note: It is assumed that each existing file name is either + # an absolute path or a file URI. + new_filenames = [] + new_addresses = [] + for filename, address in zip( + self.get_filenames(), self.get_addresses() + ): + if dirname(filename) != location: + new_filenames.append(filename) + new_addresses.append(address) + + # if not new_filenames: + # raise ValueError( + # f"Can't remove location {location} when doing so " + # "results in there being no defined files" + # ) + + a = self.copy() + a._set_component("filename", tuple(new_filenames), copy=False) + a._set_component("address", tuple(new_addresses), copy=False) + return a + + def file_locations(self): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Returns: + + `set` + TODOCFADOCS + + """ + from os.path import dirname + + # Note: It is assumed that each existing file name is either + # an absolute path or a file URI. + return set([dirname(f) for f in self.get_filenames()]) + + # def get_addresses(self): + # """TODOCFADOCS Return the names of any files containing the data array. + # + # .. versionadded:: TODOCFAVER + # + # :Returns: + # + # `tuple` + # TODOCFADOCS + # + # """ + # out = self._get_component("address", None) + # if not out: + # return () + # + # return (out,) + # + # def get_formats(self): + # """Return the format of the file. + # + # .. versionadded:: TODOCFAVER + # + # .. seealso:: `get_format`, `get_filenames`, `get_addresses` + # + # :Returns: + # + # `tuple` + # The fragment file formats. + # + # """ + # return (self.get_format(),) + # + # def open(self): + # """Returns an open dataset containing the data array. + # + # When multiple fragment files have been provided an attempt is + # made to open each one, in arbitrary order, and the + # `netCDF4.Dataset` is returned from the first success. + # + # .. versionadded:: TODOCFAVER + # + # :Returns: + # + # `netCDF4.Dataset` + # + # """ + # # Loop round the files, returning as soon as we find one that + # # works. + # filenames = self.get_filenames() + # for filename, address in zip(filenames, self.get_addresses()): + # url = urlparse(filename) + # if url.scheme == "file": + # # Convert a file URI into an absolute path + # filename = url.path + # + # try: + # nc = netCDF4.Dataset(filename, "r") + # except FileNotFoundError: + # continue + # except RuntimeError as error: + # raise RuntimeError(f"{error}: {filename}") + # + # if isisntance(address, str): + # self._set_component("ncvar", address, copy=False) + # else: + # self._set_component("varid", address, copy=False) + # + # return nc + # + # if len(filenames) == 1: + # raise FileNotFoundError(f"No such netCDF file: {filenames[0]}") + # + # raise FileNotFoundError(f"No such netCDF files: {filenames}") + + def set_file_location(self, location): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + :Parameters: + + location: `str` + TODOCFADOCS + + :Returns: + + `{{class}}` + TODOCFADOCS + + """ + from os import sep + from os.path import basename, dirname, join + + location = location.rstrip(sep) + + filenames = self.get_filenames() + addresses = self.get_addresses() + + # Note: It is assumed that each existing file name is either + # an absolute path or a fully qualified URI. + new_filenames = [] + new_addresses = [] + for filename, address in zip(filenames, addresses): + if dirname(filename) == location: + continue + + new_filename = join(location, basename(filename)) + if new_filename in new_filenames: + continue + + new_filenames.append(new_filename) + new_addresses.append(address) + + a = self.copy() + a._set_component( + "filename", filenames + tuple(new_filenames), copy=False + ) + a._set_component( + "address", + addresses + tuple(new_addresses), + copy=False, + ) + return a diff --git a/cf/data/array/netcdfarray.py b/cf/data/array/netcdfarray.py index d6042909f9..95bccbe9f9 100644 --- a/cf/data/array/netcdfarray.py +++ b/cf/data/array/netcdfarray.py @@ -1,4 +1,5 @@ import cfdm +import netCDF4 from dask.utils import SerializableLock from ...mixin_container import Container @@ -38,3 +39,8 @@ def _dask_lock(self): return False return _lock + + +# def open(self): +# """TODOCFADOCS.""" +# return super().open(netCDF4.Dataset, mode="r") diff --git a/cf/data/array/umarray.py b/cf/data/array/umarray.py index 86baee9874..b6584823bf 100644 --- a/cf/data/array/umarray.py +++ b/cf/data/array/umarray.py @@ -1,3 +1,4 @@ +import cfdm import numpy as np from ...constants import _stash2standard_name @@ -7,24 +8,23 @@ load_stash2standard_name, parse_indices, ) -from ...umread_lib.umfile import File, Rec +from ...umread_lib.umfile import File # , Rec from .abstract import Array from .mixin import FileArrayMixin -class UMArray(FileArrayMixin, Array): +class UMArray(FileArrayMixin, cfdm.data.mixin.FileArrayMixin, Array): """A sub-array stored in a PP or UM fields file.""" def __init__( self, filename=None, + address=None, dtype=None, - ndim=None, shape=None, - size=None, - header_offset=None, - data_offset=None, - disk_length=None, + # size=None, + # data_offset=None, + # disk_length=None, fmt=None, word_size=None, byte_ordering=None, @@ -112,24 +112,29 @@ def __init__( filename = None try: - fmt = source._get_component("fmt", None) + address = source._get_component("address", None) except AttributeError: - fmt = None + address = None try: - disk_length = source._get_component("disk_length", None) + fmt = source._get_component("fmt", None) except AttributeError: - disk_length = None + fmt = None - try: - header_offset = source._get_component("header_offset", None) - except AttributeError: - header_offset = None - - try: - data_offset = source._get_component("data_offset", None) - except AttributeError: - data_offset = None + # try: + # disk_length = source._get_component("disk_length", None) + # except AttributeError: + # disk_length = None + # + # try: + # header_offset = source._get_component("header_offset", None) + # except AttributeError: + # header_offset = None + # + # try: + # data_offset = source._get_component("data_offset", None) + # except AttributeError: + # data_offset = None try: dtype = source._get_component("dtype", None) @@ -156,12 +161,25 @@ def __init__( except AttributeError: calendar = False + if filename is not None: + if isinstance(filename, str): + filename = (filename,) + + self._set_component("filename", filename, copy=False) + + if address is not None: + if isinstance(address, (str, int)): + address = (address,) + + self._set_component("address", address, copy=False) + self._set_component("shape", shape, copy=False) - self._set_component("filename", filename, copy=False) + # self._set_component("filename", filename, copy=False) + # self._set_component("address", address, copy=False) self._set_component("dtype", dtype, copy=False) - self._set_component("header_offset", header_offset, copy=False) - self._set_component("data_offset", data_offset, copy=False) - self._set_component("disk_length", disk_length, copy=False) + # self._set_component("header_offset", header_offset, copy=False) + # self._set_component("data_offset", data_offset, copy=False) + # self._set_component("disk_length", disk_length, copy=False) self._set_component("units", units, copy=False) self._set_component("calendar", calendar, copy=False) @@ -185,15 +203,15 @@ def __getitem__(self, indices): Returns a subspace of the array as an independent numpy array. """ - f = self.open() - rec = self._get_rec(f) + f, header_offset = self.open() + rec = self._get_rec(f, header_offset) int_hdr = rec.int_hdr real_hdr = rec.real_hdr array = rec.get_data().reshape(self.shape) self.close(f) - del f + del f, rec if indices is not Ellipsis: indices = parse_indices(array.shape, indices) @@ -253,7 +271,7 @@ def __getitem__(self, indices): # Return the numpy array return array - def _get_rec(self, f): + def _get_rec(self, f, header_offset): """Get a container for a record. This includes the lookup header and file offsets. @@ -267,27 +285,30 @@ def _get_rec(self, f): f: `umread_lib.umfile.File` The open PP or FF file. + header_offset: `int` + :Returns: `umread_lib.umfile.Rec` The record container. """ - header_offset = self.header_offset - data_offset = self.data_offset - disk_length = self.disk_length - if data_offset is None or disk_length is None: - # This method doesn't require data_offset and disk_length, - # so plays nicely with CFA. Is it fast enough that we can - # use this method always? - for v in f.vars: - for r in v.recs: - if r.hdr_offset == header_offset: - return r - else: - return Rec.from_file_and_offsets( - f, header_offset, data_offset, disk_length - ) + # header_offset = self.header_offset + # data_offset = self.data_offset + # disk_length = self.disk_length + # if data_offset is None or disk_length is None: + # This method doesn't require data_offset and disk_length, + # so plays nicely with CFA. Is it fast enough that we can + # use this method always? + for v in f.vars: + for r in v.recs: + if r.hdr_offset == header_offset: + return r + + # else: + # return Rec.from_file_and_offsets( + # f, header_offset, data_offset, disk_length + # ) def _set_units(self, int_hdr): """The units and calendar properties. @@ -468,39 +489,39 @@ def file_address(self): removed_at="5.0.0", ) # pragma: no cover - @property - def header_offset(self): - """The start position in the file of the header. - - :Returns: - - `int` or `None` - The address, or `None` if there isn't one. - - """ - return self._get_component("header_offset", None) - - @property - def data_offset(self): - """The start position in the file of the data array. - - :Returns: - - `int` - - """ - return self._get_component("data_offset") - - @property - def disk_length(self): - """The number of words on disk for the data array. - - :Returns: - - `int` - - """ - return self._get_component("disk_length") + # @property + # def header_offset(self): + # """The start position in the file of the header. + # + # :Returns: + # + # `int` or `None` + # The address, or `None` if there isn't one. + # + # """ + # return self._get_component("header_offset", None) + # + # @property + # def data_offset(self): + # """The start position in the file of the data array. + # + # :Returns: + # + # `int` + # + # """ + # return self._get_component("data_offset") + # + # @property + # def disk_length(self): + # """The number of words on disk for the data array. + # + # :Returns: + # + # `int` + # + # """ + # return self._get_component("disk_length") @property def fmt(self): @@ -583,20 +604,20 @@ def close(self, f): if self._get_component("close"): f.close_fd() - def get_address(self): - """The address in the file of the variable. - - The address is the word offset of the lookup header. - - .. versionadded:: 3.14.0 - - :Returns: - - `int` or `None` - The address, or `None` if there isn't one. - - """ - return self.header_offset + # def get_address(self): + # """The address in the file of the variable. + # + # The address is the word offset of the lookup header. + # + # .. versionadded:: 3.14.0 + # + # :Returns: + # + # `int` or `None` + # The address, or `None` if there isn't one. + # + # """ + # return self.header_offset def get_byte_ordering(self): """The endianness of the data. @@ -678,26 +699,35 @@ def open(self): :Returns: - `umfile_lib.File` + `umfile_lib.File`, `int` **Examples** >>> f.open() """ - try: - f = File( - path=self.get_filename(), - byte_ordering=self.get_byte_ordering(), - word_size=self.get_word_size(), - fmt=self.get_fmt(), - ) - except Exception as error: - try: - f.close_fd() - except Exception: - pass - - raise Exception(error) - else: - return f + return super().open( + File, + byte_ordering=self.get_byte_ordering(), + word_size=self.get_word_size(), + fmt=self.get_fmt(), + ) + + +# try: +# f = File( +# path=self.get_filename(), +# byte_ordering=self.get_byte_ordering(), +# word_size=self.get_word_size(), +# fmt=self.get_fmt(), +# ) +# except Exception as error: +# try: +# f.close_fd() +# except Exception: +# pass +# +# raise Exception(error) +# else: +# return f +# diff --git a/cf/data/data.py b/cf/data/data.py index ac5f505be6..7e8e4bd2cc 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1479,7 +1479,9 @@ def _get_cached_elements(self): """ custom = self._custom return { - key: custom[key] for key in ("first_element", "second_element", "last_element")} + key: custom[key] + for key in ("first_element", "second_element", "last_element") + } def _set_cached_elements(self, elements): """Cache selected element values. @@ -4026,16 +4028,6 @@ def concatenate(cls, data, axis=0, cull_graph=True, relaxed_units=False): # Set the new dask array data0._set_dask(dx, clear=_ALL ^ cfa) - # Retain valid cached elements - cache = processed_data[0]._get_cached_elements() - last_element = processed_data[-1]._custom.get("last_element", None) - if last_element is None: - cache.pop("last_element", None) - else: - cache["last_element"] = last_element - - data0._set_cached_elements(cache) - # Set the CFA-netCDF aggregated data instructions and file # name substitutions by combining them from all of the input # data instances, giving precedence to those towards the left @@ -4056,7 +4048,7 @@ def concatenate(cls, data, axis=0, cull_graph=True, relaxed_units=False): if not d.cfa_get_term(): data0.cfa_set_term(False) break - + # Manage cyclicity of axes: if join axis was cyclic, it is no # longer. axis = data0._parse_axes(axis)[0] @@ -6220,7 +6212,7 @@ def get_filenames(self): A `dask` chunk that contributes to the computed array is assumed to reference data within a file if that chunk's array - object has a callable `get_filename` method, the output of + object has a callable `get_filenames` method, the output of which is added to the returned `set`. :Returns: @@ -6272,11 +6264,9 @@ def get_filenames(self): dsk = collections_to_dsk((self.to_dask_array(),), optimize_graph=True) for a in dsk.values(): try: - f = a.get_filenames() + out.update(a.get_filenames()) except AttributeError: - continue - - out.update(f) + pass return out diff --git a/cf/data/fragment/mixin/__init__.py b/cf/data/fragment/mixin/__init__.py index dad50f7451..b02b1f717d 100644 --- a/cf/data/fragment/mixin/__init__.py +++ b/cf/data/fragment/mixin/__init__.py @@ -1,2 +1,3 @@ from .fragmentarraymixin import FragmentArrayMixin -from .fragmentfilearraymixin import FragmentFileArrayMixin + +# from .fragmentfilearraymixin import FragmentFileArrayMixin diff --git a/cf/data/fragment/mixin/fragmentfilearraymixin.py b/cf/data/fragment/mixin/fragmentfilearraymixin.py index 90e21934a4..404c1b1373 100644 --- a/cf/data/fragment/mixin/fragmentfilearraymixin.py +++ b/cf/data/fragment/mixin/fragmentfilearraymixin.py @@ -1,167 +1,172 @@ -class FragmentFileArrayMixin: - """Mixin class for a fragment array stored in a file. - - .. versionadded:: TODOCFAVER - - """ - - def del_fragment_location(self, location): - """TODOCFADOCS - - .. versionadded:: TODOCFAVER - - :Parameters: - - location: `str` - TODOCFADOCS - - :Returns: - - `{{class}}` - TODOCFADOCS - - """ - from os import sep - from os.path import dirname - - a = self.copy() - location = location.rstrip(sep) - - # Note: It is assumed that each existing file name is either - # an absolute path or a file URI. - new_filenames = [] - new_addresses = [] - for filename, address in zip(a.get_filenames(), a.get_addresses()): - if dirname(filename) != location: - new_filenames.append(filename) - new_addresses.append(address) - - a._set_component("filenames", new_filenames, copy=False) - a._set_component("addresses", new_addresses, copy=False) - - return a - - def fragment_locations(self): - """TODOCFADOCS - - .. versionadded:: TODOCFAVER - - :Parameters: - - location: `str` - TODOCFADOCS - - :Returns: - - `{{class}}` - TODOCFADOCS - - """ - from os.path import dirname - - # Note: It is assumed that each existing file name is either - # an absolute path or a file URI. - return set([dirname(f) for f in self.get_filenames()]) - - def get_addresses(self, default=AttributeError()): - """TODOCFADOCS Return the names of any files containing the data array. - - .. versionadded:: TODOCFAVER - - :Returns: - - `tuple` - TODOCFADOCS - - """ - return self._get_component("addresses", default) - - def get_filenames(self, default=AttributeError()): - """TODOCFADOCS Return the names of any files containing the data array. - - .. versionadded:: TODOCFAVER - - :Returns: - - `tuple` - The fragment file names. - - """ - filenames = self._get_component("filenames", None) - if filenames is None: - if default is None: - return - - return self._default( - default, f"{self.__class__.__name__} has no fragement files" - ) - - return filenames - - def get_formats(self, default=AttributeError()): - """Return the format of each fragment file. - - .. versionadded:: TODOCFAVER - - .. seealso:: `get_filenames`, `get_addresses` - - :Returns: - - `tuple` - The fragment file formats. - - """ - return (self.get_format(),) * len(self.get_filenames(default)) - - def set_fragment_location(self, location): - """TODOCFADOCS - - .. versionadded:: TODOCFAVER - - :Parameters: - - location: `str` - TODOCFADOCS - - :Returns: - - `{{class}}` - TODOCFADOCS - - """ - from os import sep - from os.path import basename, dirname, join - - a = self.copy() - location = location.rstrip(sep) - - filenames = a.get_filenames() - addresses = a.get_addresses() - - # Note: It is assumed that each existing file name is either - # an absolute path or a fully qualified URI. - new_filenames = [] - new_addresses = [] - basenames = [] - for filename, address in zip(filenames, addresses): - if dirname(filename) == location: - continue - - basename = basename(filename) - if basename in basenames: - continue - - basenames.append(filename) - new_filenames.append(join(location, basename)) - new_addresses.append(address) - - a._set_component( - "filenames", filenames + tuple(new_filenames), copy=False - ) - a._set_component( - "addresses", - addresses + tuple(new_addresses), - copy=False, - ) - - return a +# class FragmentFileArrayMixin: +# """Mixin class for a fragment array stored in a file. +# +# .. versionadded:: TODOCFAVER +# +# """ +# +# def del_fragment_location(self, location): +# """TODOCFADOCS +# +# .. versionadded:: TODOCFAVER +# +# :Parameters: +# +# location: `str` +# TODOCFADOCS +# +# :Returns: +# +# `{{class}}` +# TODOCFADOCS +# +# """ +# from os import sep +# from os.path import dirname +# +# a = self.copy() +# location = location.rstrip(sep) +# +# # Note: It is assumed that each existing file name is either +# # an absolute path or a file URI. +# new_filenames = [] +# new_addresses = [] +# for filename, address in zip(a.get_filenames(), a.get_addresses()): +# if dirname(filename) != location: +# new_filenames.append(filename) +# new_addresses.append(address) +# +# if not new_filenames: +# raise ValueError( +# f"Can't remove fragment location {location} when doing so " +# "results in there being no defined fragment files") +# +# a._set_component("filenames", new_filenames, copy=False) +# a._set_component("addresses", new_addresses, copy=False) +# +# return a +# +# def fragment_locations(self): +# """TODOCFADOCS +# +# .. versionadded:: TODOCFAVER +# +# :Parameters: +# +# location: `str` +# TODOCFADOCS +# +# :Returns: +# +# `{{class}}` +# TODOCFADOCS +# +# """ +# from os.path import dirname +# +# # Note: It is assumed that each existing file name is either +# # an absolute path or a file URI. +# return set([dirname(f) for f in self.get_filenames()])# +# +# def get_addresses(self, default=AttributeError()): +# """TODOCFADOCS Return the names of any files containing the data array. +# +# .. versionadded:: TODOCFAVER +# +# :Returns: +# +# `tuple` +# TODOCFADOCS +# +# """ +# return self._get_component("addresses", default) +# +# def get_filenames(self, default=AttributeError()): +# """TODOCFADOCS Return the names of any files containing the data array. +# +# .. versionadded:: TODOCFAVER +# +# :Returns: +# +# `tuple` +# The fragment file names. +# +# """ +# filenames = self._get_component("filenames", None) +# if filenames is None: +# if default is None: +# return +# +# return self._default( +# default, f"{self.__class__.__name__} has no fragement files" +# ) +# +# return filenames +# +# def get_formats(self, default=AttributeError()): +# """Return the format of each fragment file. +# +# .. versionadded:: TODOCFAVER +# +# .. seealso:: `get_filenames`, `get_addresses` +# +# :Returns: +# +# `tuple` +# The fragment file formats. +# +# """ +# return (self.get_format(),) * len(self.get_filenames(default)) +# +# def set_fragment_location(self, location): +# """TODOCFADOCS +# +# .. versionadded:: TODOCFAVER +# +# :Parameters: +# +# location: `str` +# TODOCFADOCS +# +# :Returns: +# +# `{{class}}` +# TODOCFADOCS +# +# """ +# from os import sep +# from os.path import basename, dirname, join +# +# a = self.copy() +# location = location.rstrip(sep) +# +# filenames = a.get_filenames() +# addresses = a.get_addresses() +# +# # Note: It is assumed that each existing file name is either +# # an absolute path or a fully qualified URI. +# new_filenames = [] +# new_addresses = [] +# basenames = [] +# for filename, address in zip(filenames, addresses): +# if dirname(filename) == location: +# continue +# +# basename = basename(filename) +# if basename in basenames: +# continue +# +# basenames.append(filename) +# new_filenames.append(join(location, basename)) +# new_addresses.append(address) +# +# a._set_component( +# "filenames", filenames + tuple(new_filenames), copy=False +# ) +# a._set_component( +# "addresses", +# addresses + tuple(new_addresses), +# copy=False, +# ) +# +# return a diff --git a/cf/data/fragment/netcdffragmentarray.py b/cf/data/fragment/netcdffragmentarray.py index 351debd554..30da4e318b 100644 --- a/cf/data/fragment/netcdffragmentarray.py +++ b/cf/data/fragment/netcdffragmentarray.py @@ -1,14 +1,12 @@ -from urllib.parse import urlparse +# from urllib.parse import urlparse -import netCDF4 +# import netCDF4 from ..array.netcdfarray import NetCDFArray -from .mixin import FragmentArrayMixin, FragmentFileArrayMixin +from .mixin import FragmentArrayMixin -class NetCDFFragmentArray( - FragmentFileArrayMixin, FragmentArrayMixin, NetCDFArray -): +class NetCDFFragmentArray(FragmentArrayMixin, NetCDFArray): """A CFA fragment array stored in a netCDF file. .. versionadded:: 3.14.0 @@ -32,11 +30,11 @@ def __init__( :Parameters: - filenames: sequence of `str`, optional + filename: (sequence of `str`), optional The names of the netCDF fragment files containing the array. - addresses: sequence of `str`, optional + address: (sequence of `str`0, optional The name of the netCDF variable containing the fragment array. Required unless *varid* is set. @@ -84,16 +82,6 @@ def __init__( ) if source is not None: - try: - filenames = source._get_component("filenames", None) - except AttributeError: - filenames = None - - try: - addresses = source._get_component("addresses ", None) - except AttributeError: - addresses = None - try: aggregated_units = source._get_component( "aggregated_units", False @@ -108,48 +96,43 @@ def __init__( except AttributeError: aggregated_calendar = False - if filenames: - self._set_component("filenames", tuple(filenames), copy=False) - - if addresses: - self._set_component("addresses ", tuple(addresses), copy=False) - self._set_component("aggregated_units", aggregated_units, copy=False) self._set_component( "aggregated_calendar", aggregated_calendar, copy=False ) - def open(self): - """Returns an open dataset containing the data array. - - When multiple fragment files have been provided an attempt is - made to open each one, in arbitrary order, and the - `netCDF4.Dataset` is returned from the first success. - - .. versionadded:: TODOCFAVER - - :Returns: - - `netCDF4.Dataset` - - """ - # Loop round the files, returning as soon as we find one that - # works. - filenames = self.get_filenames() - for filename, address in zip(filenames, self.get_addresses()): - url = urlparse(filename) - if url.scheme == "file": - # Convert file URI into an absolute path - filename = url.path - - try: - nc = netCDF4.Dataset(filename, "r") - except FileNotFoundError: - continue - except RuntimeError as error: - raise RuntimeError(f"{error}: {filename}") - - self._set_component("ncvar", address, copy=False) - return nc - raise FileNotFoundError(f"No such netCDF fragment files: {filenames}") +# def open(self): +# """Returns an open dataset containing the data array. +# +# When multiple fragment files have been provided an attempt is +# made to open each one, in arbitrary order, and the +# `netCDF4.Dataset` is returned from the first success. +# +# .. versionadded:: TODOCFAVER +# +# :Returns: +# +# `netCDF4.Dataset` +# +# """ +# # Loop round the files, returning as soon as we find one that +# # works. +# filenames = self.get_filenames() +# for filename, address in zip(filenames, self.get_addresses()): +# url = urlparse(filename) +# if url.scheme == "file": +# # Convert file URI into an absolute path +# filename = url.path +# +# try: +# nc = netCDF4.Dataset(filename, "r") +# except FileNotFoundError: +# continue +# except RuntimeError as error: +# raise RuntimeError(f"{error}: {filename}") +# +# self._set_component("ncvar", address, copy=False) +# return nc +# +# raise FileNotFoundError(f"No such netCDF fragment files: {filenames}") diff --git a/cf/data/fragment/umfragmentarray.py b/cf/data/fragment/umfragmentarray.py index 5cdd67a0f2..2858cb6afc 100644 --- a/cf/data/fragment/umfragmentarray.py +++ b/cf/data/fragment/umfragmentarray.py @@ -1,11 +1,11 @@ -from urllib.parse import urlparse +# from urllib.parse import urlparse -from ...umread_lib.umfile import File +# from ...umread_lib.umfile import File from ..array.umarray import UMArray -from .mixin import FragmentArrayMixin, FragmentFileArrayMixin +from .mixin import FragmentArrayMixin -class UMFragmentArray(FragmentFileArrayMixin, FragmentArrayMixin, UMArray): +class UMFragmentArray(FragmentArrayMixin, UMArray): """A CFA fragment array stored in a UM or PP file. .. versionadded:: 3.14.0 @@ -14,8 +14,8 @@ class UMFragmentArray(FragmentFileArrayMixin, FragmentArrayMixin, UMArray): def __init__( self, - filenames=None, - addresses=None, + filename=None, + address=None, dtype=None, shape=None, aggregated_units=False, @@ -29,10 +29,10 @@ def __init__( :Parameters: - filenames: sequence of `str`, optional + filenames: (sequence of `str`), optional The names of the UM or PP file containing the fragment. - addresses: sequence of `str`, optional + addresses: (sequence of `str`), optional The start words in the files of the header. dtype: `numpy.dtype` @@ -69,6 +69,8 @@ def __init__( """ super().__init__( + filename=filename, + address=address, dtype=dtype, shape=shape, units=units, @@ -78,16 +80,6 @@ def __init__( ) if source is not None: - try: - filenames = source._get_component("filenames", None) - except AttributeError: - filenames = None - - try: - addresses = source._get_component("addresses ", None) - except AttributeError: - addresses = None - try: aggregated_units = source._get_component( "aggregated_units", False @@ -102,74 +94,69 @@ def __init__( except AttributeError: aggregated_calendar = False - if filenames: - self._set_component("filenames", tuple(filenames), copy=False) - - if addresses: - self._set_component("addresses ", tuple(addresses), copy=False) - self._set_component("aggregated_units", aggregated_units, copy=False) self._set_component( "aggregated_calendar", aggregated_calendar, copy=False ) - def get_formats(self): - """TODOCFADOCS - - .. versionadded:: TODOCFAVER - - .. seealso:: `get_filenames`, `get_addresses` - - :Returns: - `tuple` - - """ - return ("um",) * len(self.get_filenames()) - - def open(self): - """Returns an open dataset containing the data array. - - When multiple fragment files have been provided an attempt is - made to open each one, in arbitrary order, and the - `umfile_lib.File` is returned from the first success. - - .. versionadded:: TODOCFAVER - - :Returns: - - `umfile_lib.File` - - """ - # Loop round the files, returning as soon as we find one that - # works. - filenames = self.get_filenames() - for filename, address in zip(filenames, self.get_addresses()): - url = urlparse(filename) - if url.scheme == "file": - # Convert file URI into an absolute path - filename = url.path - - try: - f = File( - path=filename, - byte_ordering=None, - word_size=None, - fmt=None, - ) - except FileNotFoundError: - continue - except Exception as error: - try: - f.close_fd() - except Exception: - pass - - raise Exception(f"{error}: {filename}") - - self._set_component("header_offset", address, copy=False) - return f - - raise FileNotFoundError( - f"No such PP or UM fragment files: {filenames}" - ) +# def get_formats(self): +# """TODOCFADOCS +# +# .. versionadded:: TODOCFAVER +# +# .. seealso:: `get_filenames`, `get_addresses` +# +# :Returns: +# +# `tuple` +# +# """ +# return ("um",) * len(self.get_filenames()) +# +# def open(self): +# """Returns an open dataset containing the data array. +# +# When multiple fragment files have been provided an attempt is +# made to open each one, in arbitrary order, and the +# `umfile_lib.File` is returned from the first success. +# +# .. versionadded:: TODOCFAVER +# +# :Returns: +# +# `umfile_lib.File` +# +# """ +# # Loop round the files, returning as soon as we find one that +# # works. +# filenames = self.get_filenames() +# for filename, address in zip(filenames, self.get_addresses()): +# url = urlparse(filename) +# if url.scheme == "file": +# # Convert file URI into an absolute path +# filename = url.path +# +# try: +# f = File( +# path=filename, +# byte_ordering=None, +# word_size=None, +# fmt=None, +# ) +# except FileNotFoundError: +# continue +# except Exception as error: +# try: +# f.close_fd() +# except Exception: +# pass +# +# raise Exception(f"{error}: {filename}") +# +# self._set_component("header_offset", address, copy=False) +# return f +# +# raise FileNotFoundError( +# f"No such PP or UM fragment files: {filenames}" +# ) diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 38619b0fa0..80bc298ade 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -536,8 +536,8 @@ def _create_cfanetcdfarray( if non_standard_term is not None: kwargs["term"] = non_standard_term - # Get rid of the incorrect shape - this will get set by the - # CFAnetCDFArray instance. + # Get rid of the incorrect shape of () - this will get set + # correctly by the CFAnetCDFArray instance. kwargs.pop("shape", None) # Add the aggregated_data attribute (that can be used by diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index c864fd102a..45741bf7a2 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -809,7 +809,7 @@ def _ggg(self, data, cfvar): aggregation_address = [] aggregation_format = [] for indices in data.chunk_indices(): - a = self._cfa_get_filenames(data[indices]) + a = self._cfa_get_file_details(data[indices]) if len(a) != 1: if a: raise ValueError( @@ -926,9 +926,11 @@ def _customize_write_vars(self): from os.path import abspath from pathlib import PurePath - g["cfa_dir"] = PurePath(abspath(g["filename"])).parent + g["cfa_dir"] = PurePath( + abspath(g["filename"]) + ).parent # TODOCFA??? - def _cfa_get_filenames(self, data): + def _cfa_get_file_details(self, data): """TODOCFADOCS .. versionadded:: TODOCFAVER @@ -951,18 +953,10 @@ def _cfa_get_filenames(self, data): dsk = collections_to_dsk((data.to_dask_array(),), optimize_graph=True) for a in dsk.values(): try: - f = a.get_filenames() - except AttributeError: - continue - - try: - f = ((f, a.get_addresses(), a.get_formats()),) + out.update( + ((a.get_filenames(), a.get_addresses(), a.get_formats()),) + ) except AttributeError: - try: - f = ((f, (a.get_address(),), (a.get_format(),)),) - except AttributeError: - continue - - out.update(f) + pass return out diff --git a/cf/read_write/um/umread.py b/cf/read_write/um/umread.py index b5e6ab6e3e..06ce490b2f 100644 --- a/cf/read_write/um/umread.py +++ b/cf/read_write/um/umread.py @@ -1881,9 +1881,9 @@ def create_data(self): filename=filename, shape=yx_shape, dtype=data_type_in_file(rec), - header_offset=rec.hdr_offset, - data_offset=rec.data_offset, - disk_length=rec.disk_length, + address=rec.hdr_offset, + # data_offset=rec.data_offset, + # disk_length=rec.disk_length, fmt=self.fmt, word_size=self.word_size, byte_ordering=self.byte_ordering, @@ -1930,9 +1930,9 @@ def create_data(self): filename=filename, shape=shape, dtype=file_data_type, - header_offset=rec.hdr_offset, - data_offset=rec.data_offset, - disk_length=rec.disk_length, + address=rec.hdr_offset, + # data_offset=rec.data_offset, + # disk_length=rec.disk_length, fmt=fmt, word_size=word_size, byte_ordering=byte_ordering, @@ -1977,9 +1977,9 @@ def create_data(self): filename=filename, shape=shape, dtype=file_data_type, - header_offset=rec.hdr_offset, - data_offset=rec.data_offset, - disk_length=rec.disk_length, + address=rec.hdr_offset, + # data_offset=rec.data_offset, + # disk_length=rec.disk_length, fmt=fmt, word_size=word_size, byte_ordering=byte_ordering, From 1e9a91fcd59493e8b14213ebb90a70173c4d4776 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 6 Mar 2023 17:00:14 +0000 Subject: [PATCH 046/141] dev --- cf/data/array/cfanetcdfarray.py | 26 +-- cf/data/array/mixin/filearraymixin.py | 217 ++++++++++++-------------- cf/data/array/netcdfarray.py | 6 - cf/data/array/umarray.py | 203 +++++++++--------------- cf/functions.py | 14 +- cf/test/test_NetCDFArray.py | 76 +++++++++ 6 files changed, 272 insertions(+), 270 deletions(-) create mode 100644 cf/test/test_NetCDFArray.py diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index aa271aa318..0a94d1455e 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -34,9 +34,7 @@ def __new__(cls, *args, **kwargs): def __init__( self, filename=None, - ncvar=None, - varid=None, - group=None, + address=None, dtype=None, mask=True, units=False, @@ -174,21 +172,26 @@ def __init__( from CFAPython.CFAExceptions import CFAException from dask import compute, delayed + if not isinstance(filename, str): + if len(filename) != 1: + raise ValueError("TODOCFADOCS") + + filename = filename[0] + + filename = filename[0] cfa = CFADataset(filename, CFAFileFormat.CFANetCDF, "r") try: - var = cfa.getVar(ncvar) + var = cfa.getVar(address) except CFAException: raise ValueError( - f"CFA variable {ncvar} not found in file {filename}" + f"CFA variable {address!r} not found in file {filename}" ) shape = tuple([d.len for d in var.getDims()]) super().__init__( filename=filename, - ncvar=ncvar, - varid=varid, - group=group, + address=address, shape=shape, dtype=dtype, mask=mask, @@ -234,9 +237,7 @@ def __init__( else: super().__init__( filename=filename, - ncvar=ncvar, - varid=varid, - group=group, + address=address, dtype=dtype, mask=mask, units=units, @@ -272,8 +273,7 @@ def __dask_tokenize__(self): return ( self.__class__.__name__, abspath(self.get_filename()), - self.get_ncvar(), - self.get_group(), + self.get_address(), aggregated_data, ) diff --git a/cf/data/array/mixin/filearraymixin.py b/cf/data/array/mixin/filearraymixin.py index 84366d66df..2be263b2f5 100644 --- a/cf/data/array/mixin/filearraymixin.py +++ b/cf/data/array/mixin/filearraymixin.py @@ -1,12 +1,11 @@ -import numpy as np - -from ....functions import _DEPRECATION_ERROR_ATTRIBUTE +from os.path import dirname -# import cfdm +import numpy as np +from ....functions import _DEPRECATION_ERROR_ATTRIBUTE, abspath -class FileArrayMixin: # (cfdm.data.mixin.FileArrayMixin): +class FileArrayMixin: """Mixin class for an array stored in a file. .. versionadded:: 3.14.0 @@ -35,11 +34,6 @@ def __str__(self): """x.__str__() <==> str(x)""" return f"{self.get_filename()}, {self.get_address()}" - # @property - # def dtype(self): - # """Data-type of the array.""" - # return self._get_component("dtype") - @property def filename(self): """The name of the file containing the array. @@ -55,11 +49,6 @@ def filename(self): removed_at="5.0.0", ) # pragma: no cover - # @property - # def shape(self): - # """Shape of the array.""" - # return self._get_component("shape") - def del_file_location(self, location): """TODOCFADOCS @@ -75,11 +64,32 @@ def del_file_location(self, location): `{{class}}` TODOCFADOCS + **Examples** + + >>> a.get_filenames() + ('/data1/file1', '/data2/file2') + >>> a.get_addresses() + ('tas1', 'tas2') + >>> b = a.del_file_location('/data1') + >>> b = get_filenames() + ('/data2/file2',) + >>> b.get_addresses() + ('tas2',) + + >>> a.get_filenames() + ('/data1/file1', '/data2/file1', '/data2/file2') + >>> a.get_addresses() + ('tas1', 'tas1', 'tas2') + >>> b = a.del_file_location('/data2') + >>> b.get_filenames() + ('/data1/file1',) + >>> b.get_addresses() + ('tas1',) + """ from os import sep - from os.path import dirname - location = location.rstrip(sep) + location = abspath(location).rstrip(sep) # Note: It is assumed that each existing file name is either # an absolute path or a file URI. @@ -92,11 +102,8 @@ def del_file_location(self, location): new_filenames.append(filename) new_addresses.append(address) - # if not new_filenames: - # raise ValueError( - # f"Can't remove location {location} when doing so " - # "results in there being no defined files" - # ) + if not new_filenames: + raise ValueError("TODOCFADOCS") a = self.copy() a._set_component("filename", tuple(new_filenames), copy=False) @@ -110,89 +117,28 @@ def file_locations(self): :Returns: - `set` + `tuple` TODOCFADOCS - """ - from os.path import dirname + **Examples** - # Note: It is assumed that each existing file name is either - # an absolute path or a file URI. - return set([dirname(f) for f in self.get_filenames()]) - - # def get_addresses(self): - # """TODOCFADOCS Return the names of any files containing the data array. - # - # .. versionadded:: TODOCFAVER - # - # :Returns: - # - # `tuple` - # TODOCFADOCS - # - # """ - # out = self._get_component("address", None) - # if not out: - # return () - # - # return (out,) - # - # def get_formats(self): - # """Return the format of the file. - # - # .. versionadded:: TODOCFAVER - # - # .. seealso:: `get_format`, `get_filenames`, `get_addresses` - # - # :Returns: - # - # `tuple` - # The fragment file formats. - # - # """ - # return (self.get_format(),) - # - # def open(self): - # """Returns an open dataset containing the data array. - # - # When multiple fragment files have been provided an attempt is - # made to open each one, in arbitrary order, and the - # `netCDF4.Dataset` is returned from the first success. - # - # .. versionadded:: TODOCFAVER - # - # :Returns: - # - # `netCDF4.Dataset` - # - # """ - # # Loop round the files, returning as soon as we find one that - # # works. - # filenames = self.get_filenames() - # for filename, address in zip(filenames, self.get_addresses()): - # url = urlparse(filename) - # if url.scheme == "file": - # # Convert a file URI into an absolute path - # filename = url.path - # - # try: - # nc = netCDF4.Dataset(filename, "r") - # except FileNotFoundError: - # continue - # except RuntimeError as error: - # raise RuntimeError(f"{error}: {filename}") - # - # if isisntance(address, str): - # self._set_component("ncvar", address, copy=False) - # else: - # self._set_component("varid", address, copy=False) - # - # return nc - # - # if len(filenames) == 1: - # raise FileNotFoundError(f"No such netCDF file: {filenames[0]}") - # - # raise FileNotFoundError(f"No such netCDF files: {filenames}") + >>> a.get_filenames() + ('/data1/file1',) + >>> a.file_locations() + ('/data1,) + + >>> a.get_filenames() + ('/data1/file1', '/data2/file2') + >>> a.file_locations() + ('/data1', '/data2') + + >>> a.get_filenames() + ('/data1/file1', '/data2/file2', '/data1/file2') + >>> a.file_locations() + ('/data1', '/data2', '/data1') + + """ + return tuple(map(dirname, self.get_filenames())) def set_file_location(self, location): """TODOCFADOCS @@ -209,37 +155,72 @@ def set_file_location(self, location): `{{class}}` TODOCFADOCS + **Examples** + + >>> a.get_filenames() + ('/data1/file1',) + >>> a.get_addresses() + ('tas',) + >>> b = a.set_file_location('/home/user') + >>> b.get_filenames() + ('/data1/file1', '/home/user/file1') + >>> b.get_addresses() + ('tas', 'tas') + + >>> a.get_filenames() + ('/data1/file1', '/data2/file2',) + >>> a.get_addresses() + ('tas', 'tas') + >>> b = a.set_file_location('/home/user') + >>> b = get_filenames() + ('/data1/file1', '/data2/file2', '/home/user/file1', '/home/user/file2') + >>> b.get_addresses() + ('tas', 'tas', 'tas', 'tas') + + >>> a.get_filenames() + ('/data1/file1', '/data2/file1',) + >>> a.get_addresses() + ('tas1', 'tas2') + >>> b = a.set_file_location('/home/user') + >>> b.get_filenames() + ('/data1/file1', '/data2/file1', '/home/user/file1') + >>> b.get_addresses() + ('tas1', 'tas2', 'tas1') + + >>> a.get_filenames() + ('/data1/file1', '/data2/file1',) + >>> a.get_addresses() + ('tas1', 'tas2') + >>> b = a.set_file_location('/data1') + >>> b.get_filenames() + ('/data1/file1', '/data2/file1') + >>> b.get_addresses() + ('tas1', 'tas2') + """ from os import sep - from os.path import basename, dirname, join + from os.path import basename, join - location = location.rstrip(sep) + location = abspath(location).rstrip(sep) filenames = self.get_filenames() addresses = self.get_addresses() # Note: It is assumed that each existing file name is either # an absolute path or a fully qualified URI. - new_filenames = [] - new_addresses = [] + new_filenames = list(filenames) + new_addresses = list(addresses) for filename, address in zip(filenames, addresses): - if dirname(filename) == location: - continue - new_filename = join(location, basename(filename)) - if new_filename in new_filenames: - continue - - new_filenames.append(new_filename) - new_addresses.append(address) + if new_filename not in new_filenames: + new_filenames.append(new_filename) + new_addresses.append(address) a = self.copy() - a._set_component( - "filename", filenames + tuple(new_filenames), copy=False - ) + a._set_component("filename", tuple(new_filenames), copy=False) a._set_component( "address", - addresses + tuple(new_addresses), + tuple(new_addresses), copy=False, ) return a diff --git a/cf/data/array/netcdfarray.py b/cf/data/array/netcdfarray.py index 95bccbe9f9..d6042909f9 100644 --- a/cf/data/array/netcdfarray.py +++ b/cf/data/array/netcdfarray.py @@ -1,5 +1,4 @@ import cfdm -import netCDF4 from dask.utils import SerializableLock from ...mixin_container import Container @@ -39,8 +38,3 @@ def _dask_lock(self): return False return _lock - - -# def open(self): -# """TODOCFADOCS.""" -# return super().open(netCDF4.Dataset, mode="r") diff --git a/cf/data/array/umarray.py b/cf/data/array/umarray.py index b6584823bf..89983e1264 100644 --- a/cf/data/array/umarray.py +++ b/cf/data/array/umarray.py @@ -8,7 +8,7 @@ load_stash2standard_name, parse_indices, ) -from ...umread_lib.umfile import File # , Rec +from ...umread_lib.umfile import File from .abstract import Array from .mixin import FileArrayMixin @@ -22,9 +22,6 @@ def __init__( address=None, dtype=None, shape=None, - # size=None, - # data_offset=None, - # disk_length=None, fmt=None, word_size=None, byte_ordering=None, @@ -37,8 +34,13 @@ def __init__( :Parameters: - filename: `str` - The file name in normalized, absolute form. + filename: (sequence of) `str`, optional + The file name(s). + + address: (sequence of) `int`, optional + The start position in the file(s) of the header(s) + + .. versionadded:: TODOCFAVER dtype: `numpy.dtype` The data type of the data array on disk. @@ -50,17 +52,6 @@ def __init__( dimensions. When read, the data on disk is reshaped to *shape*. - header_offset: `int` - The start position in the file of the header. - - data_offset: `int`, optional - The start position in the file of the data array. - - disk_length: `int`, optional - The number of words on disk for the data array, - usually LBLREC-LBEXT. If set to ``0`` then `!size` is - used. - fmt: `str`, optional ``'PP'`` or ``'FF'`` @@ -70,18 +61,6 @@ def __init__( byte_ordering: `str`, optional ``'little_endian'`` or ``'big_endian'`` - size: `int` - Deprecated at version 3.14.0. If set will be - ignored. - - Number of elements in the uncompressed array. - - ndim: `int` - Deprecated at version 3.14.0. If set will be - ignored. - - The number of uncompressed array dimensions. - units: `str` or `None`, optional The units of the fragment data. Set to `None` to indicate that there are no units. If unset then the @@ -97,6 +76,22 @@ def __init__( {{init copy: `bool`, optional}} + size: `int` + Deprecated at version 3.14.0. + + ndim: `int` + Deprecated at version 3.14.0. + + header_offset: `int` + Deprecated at version TODOCFAVER. use the *address* + parameter instead. + + data_offset: `int`, optional + Deprecated at version TODOCFAVER. + + disk_length: `int`, optional + Deprecated at version TODOCFAVER. + """ super().__init__(source=source, copy=copy) @@ -121,21 +116,6 @@ def __init__( except AttributeError: fmt = None - # try: - # disk_length = source._get_component("disk_length", None) - # except AttributeError: - # disk_length = None - # - # try: - # header_offset = source._get_component("header_offset", None) - # except AttributeError: - # header_offset = None - # - # try: - # data_offset = source._get_component("data_offset", None) - # except AttributeError: - # data_offset = None - try: dtype = source._get_component("dtype", None) except AttributeError: @@ -168,18 +148,13 @@ def __init__( self._set_component("filename", filename, copy=False) if address is not None: - if isinstance(address, (str, int)): + if isinstance(address, int): address = (address,) self._set_component("address", address, copy=False) self._set_component("shape", shape, copy=False) - # self._set_component("filename", filename, copy=False) - # self._set_component("address", address, copy=False) self._set_component("dtype", dtype, copy=False) - # self._set_component("header_offset", header_offset, copy=False) - # self._set_component("data_offset", data_offset, copy=False) - # self._set_component("disk_length", disk_length, copy=False) self._set_component("units", units, copy=False) self._set_component("calendar", calendar, copy=False) @@ -293,10 +268,6 @@ def _get_rec(self, f, header_offset): The record container. """ - # header_offset = self.header_offset - # data_offset = self.data_offset - # disk_length = self.disk_length - # if data_offset is None or disk_length is None: # This method doesn't require data_offset and disk_length, # so plays nicely with CFA. Is it fast enough that we can # use this method always? @@ -305,11 +276,6 @@ def _get_rec(self, f, header_offset): if r.hdr_offset == header_offset: return r - # else: - # return Rec.from_file_and_offsets( - # f, header_offset, data_offset, disk_length - # ) - def _set_units(self, int_hdr): """The units and calendar properties. @@ -489,39 +455,55 @@ def file_address(self): removed_at="5.0.0", ) # pragma: no cover - # @property - # def header_offset(self): - # """The start position in the file of the header. - # - # :Returns: - # - # `int` or `None` - # The address, or `None` if there isn't one. - # - # """ - # return self._get_component("header_offset", None) - # - # @property - # def data_offset(self): - # """The start position in the file of the data array. - # - # :Returns: - # - # `int` - # - # """ - # return self._get_component("data_offset") - # - # @property - # def disk_length(self): - # """The number of words on disk for the data array. - # - # :Returns: - # - # `int` - # - # """ - # return self._get_component("disk_length") + @property + def header_offset(self): + """The start position in the file of the header. + + :Returns: + + `int` or `None` + The address, or `None` if there isn't one. + + """ + _DEPRECATION_ERROR_ATTRIBUTE( + self, + "header_offset", + "Use method 'get_address' instead.", + version="TODOCFAVER", + removed_at="5.0.0", + ) # pragma: no cover + + @property + def data_offset(self): + """The start position in the file of the data array. + + :Returns: + + `int` + + """ + _DEPRECATION_ERROR_ATTRIBUTE( + self, + "data_offset", + version="TODOCFAVER", + removed_at="5.0.0", + ) # pragma: no cover + + @property + def disk_length(self): + """The number of words on disk for the data array. + + :Returns: + + `int` + + """ + _DEPRECATION_ERROR_ATTRIBUTE( + self, + "disk_length", + version="TODOCFAVER", + removed_at="5.0.0", + ) # pragma: no cover @property def fmt(self): @@ -604,21 +586,6 @@ def close(self, f): if self._get_component("close"): f.close_fd() - # def get_address(self): - # """The address in the file of the variable. - # - # The address is the word offset of the lookup header. - # - # .. versionadded:: 3.14.0 - # - # :Returns: - # - # `int` or `None` - # The address, or `None` if there isn't one. - # - # """ - # return self.header_offset - def get_byte_ordering(self): """The endianness of the data. @@ -659,7 +626,7 @@ def get_fmt(self): def get_format(self): """TODOCFADOCS - .. versionadded:: (cfdm) TODOCFAVER + .. versionadded:: TODOCFAVER .. seealso:: `get_filename`, `get_address` @@ -704,6 +671,7 @@ def open(self): **Examples** >>> f.open() + (, 44567) """ return super().open( @@ -712,22 +680,3 @@ def open(self): word_size=self.get_word_size(), fmt=self.get_fmt(), ) - - -# try: -# f = File( -# path=self.get_filename(), -# byte_ordering=self.get_byte_ordering(), -# word_size=self.get_word_size(), -# fmt=self.get_fmt(), -# ) -# except Exception as error: -# try: -# f.close_fd() -# except Exception: -# pass -# -# raise Exception(error) -# else: -# return f -# diff --git a/cf/functions.py b/cf/functions.py index 6ce0db8a5c..d20856a54a 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -21,6 +21,7 @@ from os.path import expandvars as _os_path_expandvars from os.path import join as _os_path_join from os.path import relpath as _os_path_relpath +from urllib.parse import urlparse import cfdm import netCDF4 @@ -2500,14 +2501,15 @@ def abspath(filename): 'http://data/archive/file.nc' """ - if filename is None: - return + u = urlparse(filename) + scheme = u.scheme + if not scheme: + return _os_path_abspath(filename) - u = urllib.parse.urlparse(filename) - if u.scheme != "": - return filename + if scheme == "file": + return u.path - return _os_path_abspath(filename) + return filename def relpath(filename, start=None): diff --git a/cf/test/test_NetCDFArray.py b/cf/test/test_NetCDFArray.py new file mode 100644 index 0000000000..ee8a58c148 --- /dev/null +++ b/cf/test/test_NetCDFArray.py @@ -0,0 +1,76 @@ +import datetime +import faulthandler +import unittest + +faulthandler.enable() # to debug seg faults and timeouts + +import cf + + +class NetCDFArrayTest(unittest.TestCase): + def test_NetCDFArray_del_file_location(self): + a = cf.NetCDFArray(("/data1/file1", "/data2/file2"), ("tas1", "tas2")) + b = a.del_file_location("/data1") + self.assertIsNot(b, a) + self.assertEqual(b.get_filenames(), ("/data2/file2",)) + self.assertEqual(b.get_addresses(), ("tas2",)) + + a = cf.NetCDFArray( + ("/data1/file1", "/data2/file1", "/data2/file2"), + ("tas1", "tas1", "tas2"), + ) + b = a.del_file_location("/data2") + self.assertEqual(b.get_filenames(), ("/data1/file1",)) + self.assertEqual(b.get_addresses(), ("tas1",)) + + def test_NetCDFArray_file_locations(self): + a = cf.NetCDFArray("/data1/file1") + self.assertEqual(a.file_locations(), ("/data1",)) + + a = cf.NetCDFArray(("/data1/file1", "/data2/file2")) + self.assertEqual(a.file_locations(), ("/data1", "/data2")) + + a = cf.NetCDFArray(("/data1/file1", "/data2/file2", "/data1/file2")) + self.assertEqual(a.file_locations(), ("/data1", "/data2", "/data1")) + + def test_NetCDFArray_set_file_location(self): + a = cf.NetCDFArray("/data1/file1", "tas") + b = a.set_file_location("/home/user") + self.assertIsNot(b, a) + self.assertEqual( + b.get_filenames(), ("/data1/file1", "/home/user/file1") + ) + self.assertEqual(b.get_addresses(), ("tas", "tas")) + + a = cf.NetCDFArray(("/data1/file1", "/data2/file2"), ("tas1", "tas2")) + b = a.set_file_location("/home/user") + self.assertEqual( + b.get_filenames(), + ( + "/data1/file1", + "/data2/file2", + "/home/user/file1", + "/home/user/file2", + ), + ) + self.assertEqual(b.get_addresses(), ("tas1", "tas2", "tas1", "tas2")) + + a = cf.NetCDFArray(("/data1/file1", "/data2/file1"), ("tas1", "tas2")) + b = a.set_file_location("/home/user") + self.assertEqual( + b.get_filenames(), + ("/data1/file1", "/data2/file1", "/home/user/file1"), + ) + self.assertEqual(b.get_addresses(), ("tas1", "tas2", "tas1")) + + a = cf.NetCDFArray(("/data1/file1", "/data2/file1"), ("tas1", "tas2")) + b = a.set_file_location("/data1") + self.assertEqual(b.get_filenames(), a.get_filenames()) + self.assertEqual(b.get_addresses(), a.get_addresses()) + + +if __name__ == "__main__": + print("Run date:", datetime.datetime.now()) + cf.environment() + print() + unittest.main(verbosity=2) From d4b6d0fbcebfa564361673a03a45484ad80aa56f Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 6 Mar 2023 22:48:56 +0000 Subject: [PATCH 047/141] dev --- cf/data/data.py | 221 +++++++------------------------------------ cf/test/test_Data.py | 10 +- 2 files changed, 42 insertions(+), 189 deletions(-) diff --git a/cf/data/data.py b/cf/data/data.py index fe3dbc1d27..f10f7a5034 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1474,31 +1474,6 @@ def _get_cached_elements(self): return cache.copy() - def _get_cached_elements(self): - """Cache selected element values. - - Updates *data* in-place to store the given element values - within its ``custom`` dictionary. - - .. versionadded:: TODOCFAVER - - .. seealso:: `_del_cached_elements`, `_set_cached_elements` - - :Returns: - - `None` - - **Examples** - - >>> d._set_cached_elements({'first_element': 273.15}) - - """ - custom = self._custom - return { - key: custom[key] - for key in ("first_element", "second_element", "last_element") - } - def _set_cached_elements(self, elements): """Cache selected element values. @@ -2470,13 +2445,12 @@ def ceil(self, inplace=False, i=False): d._set_dask(da.ceil(dx)) return d - def cfa_del_fragment_location(self, location): + def cfa_del_file_location(self, location): """TODOCFADOCS .. versionadded:: TODOCFAVER - .. seealso:: `cfa_add_fragment_location`, - `cfa_fragment_locations` + .. seealso:: `cfa_set_file_location`, `cfa_file_locations` :Parameters: @@ -2489,40 +2463,36 @@ def cfa_del_fragment_location(self, location): **Examples** - >>> d.cfa_del_fragment_location('/data/model') + >>> d.cfa_del_file_location('/data/model') """ from dask.base import collections_to_dsk dx = self.to_dask_array() - # TODOCFA what if the data definitions are FileArray, rather - # than FragmentArray? Perhaps allow extra locations to be - # addby cf.write cfa_options? - updated = False dsk = collections_to_dsk((dx,), optimize_graph=True) for key, a in dsk.items(): try: - dsk[key] = a.del_fragment_location(location) + dsk[key] = a.del_file_location(location) except AttributeError: - # This chunk doesn't contain a CFA fragment + # This chunk doesn't contain a file array continue - else: - # This chunk contains a CFA fragment - updated = True + + # This chunk contains a file array and the dask graph has + # been updated + updated = True if updated: dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) self._set_dask(dx, clear=_NONE) - def cfa_fragment_locations(self, location): + def cfa_file_locations(self, location): """TODOCFADOCS .. versionadded:: TODOCFAVER - .. seealso:: `cfa_del_fragment_location`, - `cfa_set_fragment_location` + .. seealso:: `cfa_del_file_location`, `cfa_set_file_location` :Returns: @@ -2530,23 +2500,20 @@ def cfa_fragment_locations(self, location): **Examples** - >>> d.cfa_fragment_locations() + >>> d.cfa_file_locations() {'/home/data1', 'file:///data2'} """ from dask.base import collections_to_dsk - # TODOCFA what if the data definitions are FileArray, rather - # than FragmentArray? - out = set() dsk = collections_to_dsk((self.to_dask_array(),), optimize_graph=True) for key, a in dsk.items(): try: - out.update(a.fragment_locations()) + out.update(a.file_locations()) except AttributeError: - # This chunk doesn't contain a CFA fragment + # This chunk doesn't contain a file array pass return out @@ -2597,13 +2564,12 @@ def cfa_get_write(self): """ return bool(self._custom.get("cfa_write", False)) - def cfa_set_fragment_location(self, location): + def cfa_set_file_location(self, location): """TODOCFADOCS .. versionadded:: TODOCFAVER - .. seealso:: `cfa_del_fragment_location`, - `cfa_fragment_locations` + .. seealso:: `cfa_del_file_location`, `cfa_file_locations` :Parameters: @@ -2616,7 +2582,7 @@ def cfa_set_fragment_location(self, location): **Examples** - >>> d.cfa_set_fragment_location('/data/model') + >>> d.cfa_set_file_location('/data/model') """ from dask.base import collections_to_dsk @@ -2627,13 +2593,14 @@ def cfa_set_fragment_location(self, location): dsk = collections_to_dsk((dx,), optimize_graph=True) for key, a in dsk.items(): try: - dsk[key] = a.set_fragment_location(location) + dsk[key] = a.set_file_location(location) except AttributeError: - # This chunk doesn't contain a CFA fragment + # This chunk doesn't contain a file array continue - else: - # This chunk contains a CFA fragment - updated = True + + # This chunk contains a file array and the dask graph has + # been updated + updated = True if updated: dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) @@ -4075,6 +4042,15 @@ def concatenate(cls, data, axis=0, cull_graph=True, relaxed_units=False): data0.cfa_set_term(False) break + # Set the appropriate cached elements + cached_elements = processed_data[0]._get_cached_elements() + cached_elements.pop(-1, None) + last_element = processed_data[-1]._get_cached_elements().get(-1) + if last_element is not None: + cached_elements[-1] = last_element + + data0._set_cached_elements(cached_elements) + # Manage cyclicity of axes: if join axis was cyclic, it is no # longer. axis = data0._parse_axes(axis)[0] @@ -7968,7 +7944,7 @@ def insert_dimension(self, position=0, inplace=False): **Examples** """ - # TODODASKAPI bring back expand_dime alias (or rather alias this to + # TODODASKAPI bring back expand_dims alias (or rather alias this to # that) d = _inplace_enabled_define_and_cleanup(self) @@ -12226,137 +12202,6 @@ def var( return d - # def url_or_file_uri(x): - # from urllib.parse import urlparse - # - # result = urlparse(x) - # return all([result.scheme in ("file", "http", "https"), result.netloc]) - # - # def is_url(x): - # from urllib.parse import urlparse - # - # result = urlparse(x) - # return all([result.scheme in ("http", "https"), result.netloc]) - # - # def is_file_uri(x): - # from urllib.parse import urlparse - # - # result = urlparse(x) - # return all([result.scheme in ("file"), result.netloc]) - - # def ggg( - # self, - # absolute=False, - # relative=True, - # cfa_filename=None, - # substitions=None, - # ): - # """ - # - # f = cf.example_field(0) - # cf.write(f, "file_A.nc") - # cf.write(f, "file_B.nc") - # - # a = cf.read("file_A.nc", chunks=4)[0].data - # b = cf.read("file_B.nc", chunks=4)[0].data - # c = cf.Data(b.array, units=b.Units, chunks=4) - # d = cf.Data.concatenate([a, a.copy(), b, c], axis=1) - # - # - # """ - # from os.path import abspath, relpath - # from pathlib import PurePath - # from urllib.parse import urlparse - # - # from .utils import chunk_indices # , chunk_positions - # - # if substitutions: - # substitions = tuple(substitutions.items())[::-1] - # - # if relative: - # cfa_dir = PurePath(abspath(cfa_filename)).parent - # - # chunks = self.chunks - # - # # faf = [] - # # max_file = 0 - # # max_address = 0 - # # max_format = 0 - # - # filenames = [] - # address = [] - # formats = [] - # - # for indices in chunk_indices(chunks): - # a = self[indices].get_filenames(address_format=True) - # if len(a) != 1: - # raise ValueError("TODOCFADOCS") - # - # filename, address, fmt = a.pop() - # - # parsed_filename = urlparse(filename) - # scheme = parsed_filename.scheme - # if scheme not in ("http", "https"): - # path = parsed_filename.path - # if absolute: - # filename = PurePath(abspath(path)).as_uri() - # elif relative or scheme != "file": - # filename = relpath(abspath(path), start=cfa_dir) - # - # if substitutions: - # for base, sub in substitutions: - # filename = filename.replace(sub, base) - # - # filenames.append(filename) - # addresses.append(address) - # formats.append(fmt) - # - # # faf.append((filename, address, fmt)) - # # - # # max_file = max(max_file, len(filename)) - # # max_address = max(max_address, len(address)) - # # max_format = max(max_format, len(fmt)) - # - # aggregation_file = np.array(filenames).reshape(shape) - # aggregation_address = np.array(addresses).reshape( - # shape - # ) # , dtype=f"U{max_address}") - # aggregation_format = np.array(formats).reshape( - # shape - # ) # , dtype=f"U{max_format}") - # del filenames - # del address - # del formats - # - # # for position, (filename, address, fmt) in zip( - # # chunk_positions(chunks), faf - # # ): - # # aggregation_file[position] = filename - # # aggregation_address[position] = address - # # aggregation_format[position] = fmt - # - # # Location - # dtype = np.dtype(np.int32) - # if max(self.to_dask_array().chunksize) > np.iinfo(dtype).max: - # dtype = np.dtype(np.int64) - # - # aggregation_location = np.ma.masked_all( - # (self.ndim, max(shape)), dtype=dtype - # ) - # - # for j, c in enumerate(chunks): - # aggregation_location[j, : len(c)] = c - # - # # Return Data objects - # # data = partial(type(self), chunks=-1) - # data = type(self) - # return { - # "aggregation_location": data(aggregation_location), - # "aggregation_file": data(aggregation_file), - # "aggregation_format": data(aggregation_format), - # "aggregation_address": data(aggregation_address), - # } - def section( self, axes, stop=None, chunks=False, min_step=1, mode="dictionary" ): diff --git a/cf/test/test_Data.py b/cf/test/test_Data.py index 012090338b..8577d3ae4f 100644 --- a/cf/test/test_Data.py +++ b/cf/test/test_Data.py @@ -1173,6 +1173,14 @@ def test_Data_concatenate(self): self.assertEqual(f.shape, f_np.shape) self.assertTrue((f.array == f_np).all()) + # Check cached elements + str(d) + str(e) + f = cf.Data.concatenate([d, e], axis=1) + cached = f._get_cached_elements() + self.assertEqual(cached[0], d.first_element()) + self.assertEqual(cached[-1], e.last_element()) + # Check concatenation with one invalid units d.override_units(cf.Units("foo"), inplace=1) with self.assertRaises(ValueError): @@ -4383,7 +4391,7 @@ def test_Data__init__datetime(self): def test_Data_get_filenames(self): """Test `Data.get_filenames`.""" - d = cf.Data.full((5, 8), 1, chunks=4) + d = cf.Data.ones((5, 8), float, chunks=4) self.assertEqual(d.get_filenames(), set()) f = cf.example_field(0) From fc1631ba6355fad719b27ec1c9d94e240d83594e Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 7 Mar 2023 15:22:03 +0000 Subject: [PATCH 048/141] dev --- cf/cfimplementation.py | 14 +- cf/data/array/cfanetcdfarray.py | 71 +++--- cf/data/array/mixin/filearraymixin.py | 18 +- cf/data/array/netcdfarray.py | 4 +- cf/data/data.py | 312 +++++++++++++++--------- cf/data/fragment/netcdffragmentarray.py | 6 +- cf/data/utils.py | 37 --- cf/functions.py | 18 ++ cf/mixin2/cfanetcdf.py | 237 ++++++++++++------ cf/read_write/netcdf/netcdfread.py | 56 ++--- cf/read_write/netcdf/netcdfwrite.py | 45 ++-- cf/read_write/read.py | 32 ++- cf/read_write/write.py | 162 ++++++------ cf/test/file2.nc | Bin 24533 -> 0 bytes cf/test/test_Data.py | 82 +++++++ cf/test/test_NetCDFArray.py | 5 + cf/test/test_read_write.py | 30 +++ docs/source/class/cf.Data.rst | 1 + 18 files changed, 657 insertions(+), 473 deletions(-) delete mode 100644 cf/test/file2.nc diff --git a/cf/cfimplementation.py b/cf/cfimplementation.py index be30955413..8f3861e035 100644 --- a/cf/cfimplementation.py +++ b/cf/cfimplementation.py @@ -83,10 +83,8 @@ def set_construct(self, parent, construct, axes=None, copy=True, **kwargs): def initialise_CFANetCDFArray( self, filename=None, - ncvar=None, - group=None, + address=None, dtype=None, - shape=None, mask=True, units=False, calendar=False, @@ -101,14 +99,10 @@ def initialise_CFANetCDFArray( filename: `str` - ncvar: `str` - - group: `None` or sequence of str` + address: (sequence of) `str or `int`` dytpe: `numpy.dtype` - shape: `tuple` - mask: `bool`, optional units: `str` or `None`, optional @@ -132,10 +126,8 @@ def initialise_CFANetCDFArray( cls = self.get_class("CFANetCDFArray") return cls( filename=filename, - ncvar=ncvar, - group=group, + address=address, dtype=dtype, - shape=shape, mask=mask, units=units, calendar=calendar, diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index 0a94d1455e..025905b855 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -1,6 +1,6 @@ from copy import deepcopy from itertools import accumulate, product -from os.path import join +from os.path import dirname, join from urllib.parse import urlparse import numpy as np @@ -41,7 +41,7 @@ def __init__( calendar=False, instructions=None, substitutions=None, - non_standard_term=None, + term=None, source=None, copy=True, ): @@ -114,11 +114,11 @@ def __init__( .. versionadded:: TODOCFAVER - non_standard_term: `str`, optional + term: `str`, optional The name of a non-standard aggregation instruction term from which the array is to be created, instead of the creating the aggregated data in the usual - manner. If set then *ncvar* must be the name of the + manner. If set then *address* must be the name of the term's CFA-netCDF aggregation instruction variable, which must be defined on the fragment dimensions and no others. Each value of the aggregation instruction @@ -126,8 +126,7 @@ def __init__( corresponding fragment. *Parameter example:* - ``non_standard_term='tracking_id', - ncvar='aggregation_id'`` + ``address='cfa_tracking_id', term='tracking_id'`` .. versionadded:: TODOCFAVER @@ -160,9 +159,9 @@ def __init__( substitutions = None try: - non_standard_term = source.get_non_standard_term() + term = source.get_term() except AttributeError: - non_standard_term = None + term = None elif filename is not None: from pathlib import PurePath @@ -178,7 +177,6 @@ def __init__( filename = filename[0] - filename = filename[0] cfa = CFADataset(filename, CFAFileFormat.CFANetCDF, "r") try: var = cfa.getVar(address) @@ -204,9 +202,9 @@ def __init__( parsed_filename = urlparse(filename) if parsed_filename.scheme in ("file", "http", "https"): - directory = str(PurePath(filename).parent) + cfa_directory = str(PurePath(filename).parent) else: - directory = PurePath(abspath(parsed_filename).path).parent + cfa_directory = dirname(abspath(filename)) # Note: It is an as-yet-untested hypothesis that creating # the 'aggregated_data' dictionary for massive @@ -219,14 +217,14 @@ def __init__( compute( *[ delayed( - self.set_fragment( + self._set_fragment( var, loc, aggregated_data, filename, - directory, + cfa_directory, substitutions, - non_standard_term, + term, ) ) for loc in product(*[range(i) for i in fragment_shape]) @@ -248,12 +246,12 @@ def __init__( fragment_shape = None aggregated_data = None instructions = None - non_standard_term = None + term = None self._set_component("fragment_shape", fragment_shape, copy=False) self._set_component("aggregated_data", aggregated_data, copy=False) self._set_component("instructions", instructions, copy=False) - self._set_component("non_standard_term", non_standard_term, copy=False) + self._set_component("term", term, copy=False) if substitutions is not None: self._set_component( @@ -287,9 +285,9 @@ def _set_fragment( frag_loc, aggregated_data, cfa_filename, - directory, + cfa_directory, substitutions, - non_standard_term, + term, ): """Create a new key/value pair in the *aggregated_data* dictionary. @@ -315,7 +313,7 @@ def _set_fragment( cfa_filename: `str` TODOCFADOCS - directory: `str` + cfa_directory: `str` TODOCFADOCS .. versionadded:: TODOCFAVER @@ -325,7 +323,7 @@ def _set_fragment( .. versionadded:: TODOCFAVER - non_standard_term: `str` or `None` + term: `str` or `None` The name of a non-standard aggregation instruction term from which the array is to be created, instead of the creating the aggregated data in the usual @@ -343,14 +341,14 @@ def _set_fragment( fragment = var.getFrag(frag_loc=frag_loc) location = fragment.location - if non_standard_term is not None: + if term is not None: # -------------------------------------------------------- # This fragment contains a constant value # -------------------------------------------------------- aggregated_data[frag_loc] = { "format": "full", "location": location, - "full_value": fragment.non_standard_term(non_standard_term), + "full_value": fragment.non_standard_term(term), } return @@ -372,15 +370,12 @@ def _set_fragment( for base, sub in substitutions.items(): filename = filename.replace(base, sub) - parsed_filename = urlparse(filename) - if parsed_filename.scheme not in ("file", "http", "https"): - # Find the full path of a relative fragment - # filename - filename = join(directory, parsed_filename.path) + if not urlparse(filename).scheme: + filename = join(cfa_directory, filename) aggregated_data[frag_loc] = { "format": fmt, - "file": filename, + "filename": filename, "address": address, "location": location, } @@ -512,7 +507,7 @@ def get_fragment_shape(self): """ return self._get_component("fragment_shape") - def get_non_standard_term(self, default=ValueError()): + def get_term(self, default=ValueError()): """TODOCFADOCS. .. versionadded:: TODOCFAVER @@ -523,7 +518,7 @@ def get_non_standard_term(self, default=ValueError()): TODOCFADOCS. """ - return self._get_component("non_standard_term", default=default) + return self._get_component("term", default=default) def subarray_shapes(self, shapes): """Create the subarray shapes. @@ -828,8 +823,7 @@ def to_dask_array(self, chunks="auto"): kwargs.pop("location", None) FragmentArray = get_FragmentArray(kwargs.pop("format", None)) - - fragment_array = FragmentArray( + fragment = FragmentArray( dtype=dtype, shape=fragment_shape, aggregated_units=units, @@ -837,16 +831,9 @@ def to_dask_array(self, chunks="auto"): **kwargs, ) - key = f"{fragment_array.__class__.__name__}-{tokenize(fragment_array)}" - dsk[key] = fragment_array - - dsk[name + chunk_location] = ( - getter, - key, - f_indices, - False, - False, - ) + key = f"{fragment.__class__.__name__}-{tokenize(fragment)}" + dsk[key] = fragment + dsk[name + chunk_location] = (getter, key, f_indices, False, False) # Return the dask array return da.Array(dsk, name[0], chunks=chunks, dtype=dtype) diff --git a/cf/data/array/mixin/filearraymixin.py b/cf/data/array/mixin/filearraymixin.py index 2be263b2f5..53aa1adcf5 100644 --- a/cf/data/array/mixin/filearraymixin.py +++ b/cf/data/array/mixin/filearraymixin.py @@ -1,4 +1,5 @@ -from os.path import dirname +from os import sep +from os.path import basename, dirname, join import numpy as np @@ -26,14 +27,6 @@ def _dask_meta(self): """ return np.array((), dtype=self.dtype) - def __repr__(self): - """x.__repr__() <==> repr(x)""" - return f"" - - def __str__(self): - """x.__str__() <==> str(x)""" - return f"{self.get_filename()}, {self.get_address()}" - @property def filename(self): """The name of the file containing the array. @@ -87,12 +80,8 @@ def del_file_location(self, location): ('tas1',) """ - from os import sep - location = abspath(location).rstrip(sep) - # Note: It is assumed that each existing file name is either - # an absolute path or a file URI. new_filenames = [] new_addresses = [] for filename, address in zip( @@ -198,9 +187,6 @@ def set_file_location(self, location): ('tas1', 'tas2') """ - from os import sep - from os.path import basename, join - location = abspath(location).rstrip(sep) filenames = self.get_filenames() diff --git a/cf/data/array/netcdfarray.py b/cf/data/array/netcdfarray.py index d6042909f9..795d918777 100644 --- a/cf/data/array/netcdfarray.py +++ b/cf/data/array/netcdfarray.py @@ -2,13 +2,13 @@ from dask.utils import SerializableLock from ...mixin_container import Container -from .mixin import FileArrayMixin +from .mixin import ArrayMixin, FileArrayMixin # Global lock for netCDF file access _lock = SerializableLock() -class NetCDFArray(FileArrayMixin, Container, cfdm.NetCDFArray): +class NetCDFArray(FileArrayMixin, ArrayMixin, Container, cfdm.NetCDFArray): """An array stored in a netCDF file.""" def __repr__(self): diff --git a/cf/data/data.py b/cf/data/data.py index f10f7a5034..2b07fde4e4 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -5,6 +5,7 @@ from itertools import product from numbers import Integral from operator import mul +from os import sep import cfdm import cftime @@ -13,7 +14,7 @@ from dask import compute, delayed # noqa: F401 from dask.array import Array from dask.array.core import normalize_chunks -from dask.base import is_dask_collection, tokenize +from dask.base import collections_to_dsk, is_dask_collection, tokenize from dask.highlevelgraph import HighLevelGraph from dask.optimization import cull @@ -29,6 +30,7 @@ from ..functions import ( _DEPRECATION_ERROR_KWARGS, _section, + abspath, atol, default_netCDF_fillvals, free_memory, @@ -1513,7 +1515,7 @@ def _set_cached_elements(self, elements): self._custom["cached_elements"] = cache - def _set_cfa_write(self, status): + def _cfa_set_write(self, status): """Set the CFA write status of the data. This should only be set to `True` if it is known that the dask @@ -2445,79 +2447,6 @@ def ceil(self, inplace=False, i=False): d._set_dask(da.ceil(dx)) return d - def cfa_del_file_location(self, location): - """TODOCFADOCS - - .. versionadded:: TODOCFAVER - - .. seealso:: `cfa_set_file_location`, `cfa_file_locations` - - :Parameters: - - location: `str` - TODOCFADOCS - - :Returns: - - `None` - - **Examples** - - >>> d.cfa_del_file_location('/data/model') - - """ - from dask.base import collections_to_dsk - - dx = self.to_dask_array() - - updated = False - dsk = collections_to_dsk((dx,), optimize_graph=True) - for key, a in dsk.items(): - try: - dsk[key] = a.del_file_location(location) - except AttributeError: - # This chunk doesn't contain a file array - continue - - # This chunk contains a file array and the dask graph has - # been updated - updated = True - - if updated: - dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) - self._set_dask(dx, clear=_NONE) - - def cfa_file_locations(self, location): - """TODOCFADOCS - - .. versionadded:: TODOCFAVER - - .. seealso:: `cfa_del_file_location`, `cfa_set_file_location` - - :Returns: - - `set` - - **Examples** - - >>> d.cfa_file_locations() - {'/home/data1', 'file:///data2'} - - """ - from dask.base import collections_to_dsk - - out = set() - - dsk = collections_to_dsk((self.to_dask_array(),), optimize_graph=True) - for key, a in dsk.items(): - try: - out.update(a.file_locations()) - except AttributeError: - # This chunk doesn't contain a file array - pass - - return out - def cfa_get_term(self): """The CFA aggregation instruction term status. @@ -2564,48 +2493,6 @@ def cfa_get_write(self): """ return bool(self._custom.get("cfa_write", False)) - def cfa_set_file_location(self, location): - """TODOCFADOCS - - .. versionadded:: TODOCFAVER - - .. seealso:: `cfa_del_file_location`, `cfa_file_locations` - - :Parameters: - - location: `str` - TODOCFADOCS - - :Returns: - - `None` - - **Examples** - - >>> d.cfa_set_file_location('/data/model') - - """ - from dask.base import collections_to_dsk - - dx = self.to_dask_array() - - updated = False - dsk = collections_to_dsk((dx,), optimize_graph=True) - for key, a in dsk.items(): - try: - dsk[key] = a.set_file_location(location) - except AttributeError: - # This chunk doesn't contain a file array - continue - - # This chunk contains a file array and the dask graph has - # been updated - updated = True - - if updated: - dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) - self._set_dask(dx, clear=_NONE) - def cfa_set_term(self, status): """Set the CFA aggregation instruction term status. @@ -4030,7 +3917,7 @@ def concatenate(cls, data, axis=0, cull_graph=True, relaxed_units=False): substitutions = {} for d in processed_data[::-1]: aggregated_data.update(d.cfa_get_aggregated_data({})) - substitutions.update(d.cfa_get_file_substitutions()) + substitutions.update(d.cfa_file_substitutions()) data0.cfa_set_aggregated_data(aggregated_data) data0.cfa_set_file_substitutions(substitutions) @@ -6260,11 +6147,8 @@ def get_filenames(self): {'file_A.nc'} """ - from dask.base import collections_to_dsk - out = set() - dsk = collections_to_dsk((self.to_dask_array(),), optimize_graph=True) - for a in dsk.values(): + for a in self.todict().values(): try: out.update(a.get_filenames()) except AttributeError: @@ -6367,6 +6251,50 @@ def set_calendar(self, calendar): """ self.Units = Units(self.get_units(default=None), calendar) + def set_file_location(self, location): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + .. seealso:: `del_file_location`, `file_locations` + + :Parameters: + + location: `str` + TODOCFADOCS + + :Returns: + + `None` + + **Examples** + + >>> d.set_file_location('/data/model') + + """ + + location = abspath(location).rstrip(sep) + + updated = False + dsk = self.todict() + for key, a in dsk.items(): + try: + dsk[key] = a.set_file_location(location) + except AttributeError: + # This chunk doesn't contain a file array + continue + + # This chunk contains a file array and the dask graph has + # been updated + updated = True + + if not updated: + raise ValueError("TODOCFADOCS") + + dx = self.to_dask_array() + dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) + self._set_dask(dx, clear=_NONE) + def set_units(self, value): """Set the units. @@ -8455,6 +8383,34 @@ def soften_mask(self): self._set_dask(dx, clear=_NONE) self.hardmask = False + def file_locations(self): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + .. seealso:: `del_file_location`, `set_file_location` + + :Returns: + + `set` + + **Examples** + + >>> d.file_locations() + {'/home/data1', 'file:///data2'} + + """ + out = set() + + for key, a in self.todict().items(): + try: + out.update(a.file_locations()) + except AttributeError: + # This chunk doesn't contain a file array + pass + + return out + @_inplace_enabled(default=False) def filled(self, fill_value=None, inplace=False): """Replace masked elements with a fill value. @@ -9006,6 +8962,41 @@ def change_calendar(self, calendar, inplace=False, i=False): return d + def chunk_indices(self): + """TODOCFADOCS ind the shape of each chunk. + + .. versionadded:: TODOCFAVER + + :Returns: + + TODOCFAVER + + **Examples** + + >>> d = cf.Data(np.arange(405).reshape(3, 9, 15), + ... chunks=((1, 2), (9,), (4, 5, 6))) + >>> for index in d.chunk_indices(): + ... print(index) + ... + (slice(0, 1, None), slice(0, 9, None), slice(0, 4, None)) + (slice(0, 1, None), slice(0, 9, None), slice(4, 9, None)) + (slice(0, 1, None), slice(0, 9, None), slice(9, 15, None)) + (slice(1, 3, None), slice(0, 9, None), slice(0, 4, None)) + (slice(1, 3, None), slice(0, 9, None), slice(4, 9, None)) + (slice(1, 3, None), slice(0, 9, None), slice(9, 15, None)) + + """ + from dask.utils import cached_cumsum + + chunks = self.chunks + + cumdims = [cached_cumsum(bds, initial_zero=True) for bds in chunks] + indices = [ + [slice(s, s + dim) for s, dim in zip(starts, shapes)] + for starts, shapes in zip(cumdims, chunks) + ] + return product(*indices) + @_deprecated_kwarg_check("i", version="3.0.0", removed_at="4.0.0") @_inplace_enabled(default=False) def override_units(self, units, inplace=False, i=False): @@ -9375,6 +9366,47 @@ def del_calendar(self, default=ValueError()): self.override_calendar(None, inplace=True) return calendar + def del_file_location(self, location): + """TODOCFADOCS + + .. versionadded:: TODOCFAVER + + .. seealso:: `set_file_location`, `file_locations` + + :Parameters: + + location: `str` + TODOCFADOCS + + :Returns: + + `None` + + **Examples** + + >>> d.del_file_location('/data/model') + + """ + location = abspath(location).rstrip(sep) + + updated = False + dsk = self.todict() + for key, a in dsk.items(): + try: + dsk[key] = a.del_file_location(location) + except AttributeError: + # This chunk doesn't contain a file array + continue + + # This chunk contains a file array and the dask graph has + # been updated + updated = True + + if updated: + dx = self.to_dask_array() + dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) + self._set_dask(dx, clear=_NONE) + def del_units(self, default=ValueError()): """Delete the units. @@ -11030,6 +11062,42 @@ def tan(self, inplace=False, i=False): return d + def todict(self): + """Return a dictionary of the dask graph key/value pairs. + + Prior to being converted to a dictionary, the graph is + optimised to remove unused chunks. + + .. versionadded:: TODOCFAVER + + :Returns: + + `dict` + The dictionary of the dask graph key/value pairs. + + **Examples** + + >>> d = cf.Data([1, 2, 3, 4], chunks=2) + >>> d.todict() + {('array-7daac373ba27474b6df0af70aab14e49', 0): array([1, 2]), + ('array-7daac373ba27474b6df0af70aab14e49', 1): array([3, 4])} + >>> e = d[0] + >>> e.todict() + {('getitem-14d8301a3deec45c98569d73f7a2239c', + 0): (, ('array-7daac373ba27474b6df0af70aab14e49', + 0), (slice(0, 1, 1),)), + ('array-7daac373ba27474b6df0af70aab14e49', 0): array([1, 2])} + >>> dict(e.to_dask_array().dask) + {('array-7daac373ba27474b6df0af70aab14e49', 0): array([1, 2]), + ('array-7daac373ba27474b6df0af70aab14e49', 1): array([3, 4]), + ('getitem-14d8301a3deec45c98569d73f7a2239c', + 0): (, ('array-7daac373ba27474b6df0af70aab14e49', + 0), (slice(0, 1, 1),))} + + """ + dx = self.to_dask_array() + return collections_to_dsk((dx,), optimize_graph=True) + def tolist(self): """Return the data as a scalar or (nested) list. diff --git a/cf/data/fragment/netcdffragmentarray.py b/cf/data/fragment/netcdffragmentarray.py index 30da4e318b..e45a739201 100644 --- a/cf/data/fragment/netcdffragmentarray.py +++ b/cf/data/fragment/netcdffragmentarray.py @@ -15,8 +15,8 @@ class NetCDFFragmentArray(FragmentArrayMixin, NetCDFArray): def __init__( self, - filenames=None, - addresses=None, + filename=None, + address=None, dtype=None, shape=None, aggregated_units=False, @@ -72,6 +72,8 @@ def __init__( """ super().__init__( + filename=filename, + address=address, dtype=dtype, shape=shape, mask=True, diff --git a/cf/data/utils.py b/cf/data/utils.py index dd9c3720b9..abb1a835a4 100644 --- a/cf/data/utils.py +++ b/cf/data/utils.py @@ -515,43 +515,6 @@ def chunk_locations(chunks): return product(*locations) -def chunk_indices(chunks): - """Find the shape of each chunk. - - .. versionadded:: TODOCFAVER - - .. seealso:: `chunk_locations`, `chunk_positions`, `chunk_shapes` - - :Parameters: - - chunks: `tuple` - The chunk sizes along each dimension, as output by - `dask.array.Array.chunks`. - - **Examples** - - >>> chunks = ((1, 2), (9,), (4, 5, 6)) - >>> for index in cf.data.utils.chunk_indices(chunks): - ... print(index) - ... - (slice(0, 1, None), slice(0, 9, None), slice(0, 4, None)) - (slice(0, 1, None), slice(0, 9, None), slice(4, 9, None)) - (slice(0, 1, None), slice(0, 9, None), slice(9, 15, None)) - (slice(1, 3, None), slice(0, 9, None), slice(0, 4, None)) - (slice(1, 3, None), slice(0, 9, None), slice(4, 9, None)) - (slice(1, 3, None), slice(0, 9, None), slice(9, 15, None)) - - """ - from dask.utils import cached_cumsum - - cumdims = [cached_cumsum(bds, initial_zero=True) for bds in chunks] - indices = [ - [slice(s, s + dim) for s, dim in zip(starts, shapes)] - for starts, shapes in zip(cumdims, chunks) - ] - return product(*indices) - - def scalar_masked_array(dtype=float): """Return a scalar masked array. diff --git a/cf/functions.py b/cf/functions.py index d20856a54a..22556d61ae 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -3236,6 +3236,24 @@ def _DEPRECATION_ERROR_FUNCTION_KWARGS( ) +def _DEPRECATION_ERROR_FUNCTION_KWARG_VALUE( + func, + kwarg, + value, + message="", + version=None, + removed_at=None, +): + if removed_at: + removed_at = f" and will be removed at version {removed_at}" + + raise DeprecationError( + f"Value {value!r} of keyword {kwarg!r} of fcuntion {func!r} " + f"has been deprecated at version {version} and is no longer " + "available{removed_at}. {message}" + ) + + def _DEPRECATION_ERROR_KWARGS( instance, method, diff --git a/cf/mixin2/cfanetcdf.py b/cf/mixin2/cfanetcdf.py index 15327ecad5..279d07d158 100644 --- a/cf/mixin2/cfanetcdf.py +++ b/cf/mixin2/cfanetcdf.py @@ -10,9 +10,7 @@ class CFANetCDF(NetCDFMixin): - """Mixin class for accessing CFA-netCDF aggregation instruction terms. - - Must be used in conjunction with `NetCDF` + """Mixin class for CFA-netCDF. .. versionadded:: TODOCFAVER @@ -21,6 +19,10 @@ class CFANetCDF(NetCDFMixin): def cfa_del_aggregated_data(self, default=ValueError()): """Remove the CFA-netCDF aggregation instruction terms. + The aggregation instructions are stored in the + `aggregation_data` attribute of a CFA-netCDF aggregation + variable. + .. versionadded:: TODOCFAVER .. seealso:: `cfa_get_aggregated_data`, @@ -75,9 +77,13 @@ def cfa_del_aggregated_data(self, default=ValueError()): def cfa_get_aggregated_data(self, default=ValueError()): """Return the CFA-netCDF aggregation instruction terms. - .. versifragement onadsded:: TODOCFAVER + The aggregation instructions are stored in the + `aggregation_data` attribute of a CFA-netCDF aggregation + variable. - .. seealso:: `scfa_del_aggregated_data`, + .. versionadded:: TODOCFAVER + + .. seealso:: `cfa_del_aggregated_data`, `cfa_has_aggregated_data`, `cfa_set_aggregated_data` @@ -144,6 +150,10 @@ def cfa_get_aggregated_data(self, default=ValueError()): def cfa_has_aggregated_data(self): """Whether any CFA-netCDF aggregation instruction terms have been set. + The aggregation instructions are stored in the + `aggregation_data` attribute of a CFA-netCDF aggregation + variable. + .. versionadded:: TODOCFAVER .. seealso:: `cfa_del_aggregated_data`, @@ -192,6 +202,10 @@ def cfa_has_aggregated_data(self): def cfa_set_aggregated_data(self, value): """Set the CFA-netCDF aggregation instruction terms. + The aggregation instructions are stored in the + `aggregation_data` attribute of a CFA-netCDF aggregation + variable. + If there are any ``/`` (slash) characters in the netCDF variable names then these act as delimiters for a group hierarchy. By default, or if the name starts with a ``/`` @@ -262,20 +276,17 @@ def cfa_set_aggregated_data(self, value): def cfa_clear_file_substitutions(self): """Remove the CFA-netCDF file name substitutions. + The file substitutions are stored in the `substitutions` + attribute of a CFA-netCDF `file` aggregation aggregation + instruction term. + .. versionadded:: TODOCFAVER - .. seealso:: `cfa_get_file_substitutions`, + .. seealso:: `cfa_del_file_substitution`, + `cfa_file_substitutions`, `cfa_has_file_substitutions`, `cfa_set_file_substitutions` - :Parameters: - - default: optional - Return the value of the *default* parameter if - CFA-netCDF file name substitutions have not been - set. If set to an `Exception` instance then it will be - raised instead. - :Returns: `dict` @@ -283,18 +294,28 @@ def cfa_clear_file_substitutions(self): **Examples** - >>> f.cfa_set_file_substitutions({'${base}': 'file:///data/'}) + >>> f.`cfa_set_file_substitutions({'base': 'file:///data/'}) >>> f.cfa_has_file_substitutions() True - >>> f.cfa_get_file_substitutions() - {'${base}': 'file:///data/'} - >>> f.cfa_del_file_substitutions() + >>> f.cfa_file_substitutions() {'${base}': 'file:///data/'} + >>> f.`cfa_set_file_substitutions({'${base2}': '/home/data/'}) + >>> f.cfa_file_substitutions() + {'${base}': 'file:///data/', '${base2}': '/home/data/'} + >>> f.`cfa_set_file_substitutions({'${base}': '/new/location/'}) + >>> f.cfa_file_substitutions() + {'${base}': '/new/location/', '${base2}': '/home/data/'} + >>> f.cfa_del_file_substitution('${base}') + {'${base}': '/new/location/'} + >>> f.cfa_clear_file_substitutions() + {'${base2}': '/home/data/'} >>> f.cfa_has_file_substitutions() False - >>> print(f.cfa_get_file_substitutions(None)) - None - >>> print(f.cfa_del_file_substitutions(None)) + >>> f.cfa_file_substitutions() + {} + >>> f.cfa_clear_file_substitutions() + {} + >>> print(f.cfa_del_file_substitution('base', None)) None """ @@ -303,19 +324,29 @@ def cfa_clear_file_substitutions(self): def cfa_del_file_substitution(self, base, default=ValueError()): """Remove the CFA-netCDF file name substitutions. + The file substitutions are stored in the `substitutions` + attribute of a CFA-netCDF `file` aggregation aggregation + instruction term. + .. versionadded:: TODOCFAVER - .. seealso:: `cfa_get_file_substitutions`, + .. seealso:: `cfa_clear_file_substitutions`, + `cfa_file_substitutions`, `cfa_has_file_substitutions`, `cfa_set_file_substitutions` :Parameters: + base: `str` + The substition definition to be removed. May be + specified with or without the ``${...}`` syntax. For + instance, the following are equivalent: ``'base'``, + ``'${base}'``. + default: optional - Return the value of the *default* parameter if - CFA-netCDF file name substitutions have not been - set. If set to an `Exception` instance then it will be - raised instead. + Return the value of the *default* parameter if file + name substitution has not been set. If set to an + `Exception` instance then it will be raised instead. :Returns: @@ -324,25 +355,35 @@ def cfa_del_file_substitution(self, base, default=ValueError()): **Examples** - >>> f.cfa_set_file_substitutions({'${base}': 'file:///data/'}) + >>> f.cfa_set_file_substitutions({'base': 'file:///data/'}) >>> f.cfa_has_file_substitutions() True - >>> f.cfa_get_file_substitutions() - {'base': 'file:///data/'} - >>> f.cfa_del_file_substitutions() - {'base': 'file:///data/'} + >>> f.cfa_file_substitutions() + {'${base}': 'file:///data/'} + >>> f.`cfa_set_file_substitutions({'${base2}': '/home/data/'}) + >>> f.cfa_file_substitutions() + {'${base}': 'file:///data/', '${base2}': '/home/data/'} + >>> f.`cfa_set_file_substitutions({'${base}': '/new/location/'}) + >>> f.cfa_file_substitutions() + {'${base}': '/new/location/', '${base2}': '/home/data/'} + >>> f.cfa_del_file_substitution('${base}') + {'${base}': '/new/location/'} + >>> f.cfa_clear_file_substitutions() + {'${base2}': '/home/data/'} >>> f.cfa_has_file_substitutions() False - >>> print(f.cfa_get_file_substitutions(None)) - None - >>> print(f.cfa_del_file_substitutions(None)) + >>> f.cfa_file_substitutions() + {} + >>> f.cfa_clear_file_substitutions() + {} + >>> print(f.cfa_del_file_substitution('base', None)) None """ if not (base.startswith("${") and base.endswith("}")): base = f"${{{base}}}" - subs = self.cfa_file_substitutions({}) + subs = self.cfa_file_substitutions() if base not in subs: if default is None: return @@ -361,23 +402,19 @@ def cfa_del_file_substitution(self, base, default=ValueError()): return out - def cfa_get_file_substitutions(self): + def cfa_file_substitutions(self): """Return the CFA-netCDF file name substitutions. - .. versionadded:: TODOCFAVER + The file substitutions are stored in the `substitutions` + attribute of a CFA-netCDF `file` aggregation aggregation + instruction term. - .. seealso:: `cfa_del_file_substitutions`, - `cfa_get_file_substitutions`, - `cfa_set_file_substitutions` - - :Parameters: - - default: optional - Return the value of the *default* parameter if - CFA-netCDF file name substitutions have not been - set. If set to an `Exception` instance then it will be - raised instead. + .. versionadded:: TODOCFAVER + .. seealso:: `cfa_clear_file_substitutions`, + `cfa_del_file_substitution`, + `cfa_file_substitutions`, + `cfa_set_file_substitution` :Returns: value: `dict` @@ -385,18 +422,28 @@ def cfa_get_file_substitutions(self): **Examples** - >>> f.cfa_set_file_substitutions({'${base}': 'file:///data/'}) + >>> f.`cfa_set_file_substitutions({'base': 'file:///data/'}) >>> f.cfa_has_file_substitutions() True - >>> f.cfa_get_file_substitutions() - {'${base}': 'file:///data/'} - >>> f.cfa_del_file_substitutions() + >>> f.cfa_file_substitutions() {'${base}': 'file:///data/'} + >>> f.`cfa_set_file_substitutions({'${base2}': '/home/data/'}) + >>> f.cfa_file_substitutions() + {'${base}': 'file:///data/', '${base2}': '/home/data/'} + >>> f.`cfa_set_file_substitutions({'${base}': '/new/location/'}) + >>> f.cfa_file_substitutions() + {'${base}': '/new/location/', '${base2}': '/home/data/'} + >>> f.cfa_del_file_substitution('${base}') + {'${base}': '/new/location/'} + >>> f.cfa_clear_file_substitutions() + {'${base2}': '/home/data/'} >>> f.cfa_has_file_substitutions() False - >>> print(f.cfa_get_file_substitutions(None)) - None - >>> print(f.cfa_del_file_substitutions(None)) + >>> f.cfa_file_substitutions() + {} + >>> f.cfa_clear_file_substitutions() + {} + >>> print(f.cfa_del_file_substitution('base', None)) None """ @@ -409,10 +456,15 @@ def cfa_get_file_substitutions(self): def cfa_has_file_substitutions(self): """Whether any CFA-netCDF file name substitutions have been set. + The file substitutions are stored in the `substitutions` + attribute of a CFA-netCDF `file` aggregation aggregation + instruction term. + .. versionadded:: TODOCFAVER - .. seealso:: `cfa_del_file_substitutions`, - `cfa_get_file_substitutions`, + .. seealso:: `cfa_clear_file_substitutions`, + `cfa_del_file_substitution`, + `cfa_file_substitutions`, `cfa_set_file_substitutions` :Returns: @@ -423,42 +475,57 @@ def cfa_has_file_substitutions(self): **Examples** - >>> f.cfa_set_file_substitutions({'${base}': 'file:///data/'}) + >>> f.`cfa_set_file_substitutions({'base': 'file:///data/'}) >>> f.cfa_has_file_substitutions() True - >>> f.cfa_get_file_substitutions() - {'${base}': 'file:///data/'} - >>> f.cfa_del_file_substitutions() + >>> f.cfa_file_substitutions() {'${base}': 'file:///data/'} + >>> f.`cfa_set_file_substitutions({'${base2}': '/home/data/'}) + >>> f.cfa_file_substitutions() + {'${base}': 'file:///data/', '${base2}': '/home/data/'} + >>> f.`cfa_set_file_substitutions({'${base}': '/new/location/'}) + >>> f.cfa_file_substitutions() + {'${base}': '/new/location/', '${base2}': '/home/data/'} + >>> f.cfa_del_file_substitution('${base}') + {'${base}': '/new/location/'} + >>> f.cfa_clear_file_substitutions() + {'${base2}': '/home/data/'} >>> f.cfa_has_file_substitutions() False - >>> print(f.cfa_get_file_substitutions(None)) - None - >>> print(f.cfa_del_file_substitutions(None)) + >>> f.cfa_file_substitutions() + {} + >>> f.cfa_clear_file_substitutions() + {} + >>> print(f.cfa_del_file_substitution('base', None)) None """ return self._nc_has("cfa_file_substitutions") def cfa_set_file_substitutions(self, value): - """Set the CFA-netCDF file name substitutions. + """Set CFA-netCDF file name substitutions. + + The file substitutions are stored in the `substitutions` + attribute of a CFA-netCDF `file` aggregation aggregation + instruction term. .. versionadded:: TODOCFAVER - .. seealso:: `cfa_del_file_substitutions`, - `cfa_get_file_substitutions`, + .. seealso:: `cfa_clear_file_substitutions`, + `cfa_del_file_substitution`, + `cfa_file_substitutions`, `cfa_has_file_substitutions` :Parameters: value: `str` or `dict` The substition definitions in a dictionary whose - key/value pairs are the file URI parts to be + key/value pairs are the file name parts to be substituted and their corresponding substitution text. - The file URI parts to be substituted may be specified - with or without the ``${...}`` syntax. For instance, - the following are equivalent: ``{'base': 'sub'}``, + The substition definition may be specified with or + without the ``${...}`` syntax. For instance, the + following are equivalent: ``{'base': 'sub'}``, ``{'${base}': 'sub'}``. :Returns: @@ -467,18 +534,28 @@ def cfa_set_file_substitutions(self, value): **Examples** - >>> f.cfa_set_file_substitutions({'base': 'file:///data/'}) + >>> f.`cfa_set_file_substitutions({'base': 'file:///data/'}) >>> f.cfa_has_file_substitutions() True - >>> f.cfa_get_file_substitutions() - {'${base}': 'file:///data/'} - >>> f.cfa_del_file_substitutions() + >>> f.cfa_file_substitutions() {'${base}': 'file:///data/'} + >>> f.`cfa_set_file_substitutions({'${base2}': '/home/data/'}) + >>> f.cfa_file_substitutions() + {'${base}': 'file:///data/', '${base2}': '/home/data/'} + >>> f.`cfa_set_file_substitutions({'${base}': '/new/location/'}) + >>> f.cfa_file_substitutions() + {'${base}': '/new/location/', '${base2}': '/home/data/'} + >>> f.cfa_del_file_substitution('${base}') + {'${base}': '/new/location/'} + >>> f.cfa_clear_file_substitutions() + {'${base2}': '/home/data/'} >>> f.cfa_has_file_substitutions() False - >>> print(f.cfa_get_file_substitutions(None)) - None - >>> print(f.cfa_del_file_substitutions(None)) + >>> f.cfa_file_substitutions() + {} + >>> f.cfa_clear_file_substitutions() + {} + >>> print(f.cfa_del_file_substitution('base', None)) None """ @@ -486,10 +563,10 @@ def cfa_set_file_substitutions(self, value): return value = value.copy() - for base, sub in value.items(): + for base, sub in tuple(value.items()): if not (base.startswith("${") and base.endswith("}")): value[f"${{{base}}}"] = value.pop(base) - subs = self.cfa_get_file_substitutions() + subs = self.cfa_file_substitutions() subs.update(value) self._nc_set("cfa_file_substitutions", subs) diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 6436277d9d..3e1fd4a53a 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -204,7 +204,7 @@ def _create_data( # Set the CFA write status to True when there is exactly # one dask chunk if data.npartitions == 1: - data._set_cfa_write(True) + data._cfa_set_write(True) self._cache_data_elements(data, ncvar) @@ -216,13 +216,13 @@ def _create_data( if construct is not None: # Remove the aggregation attributes from the construct self.implementation.del_property( - construct, "aggregation_dimensions", None + construct, "aggregated_dimensions", None ) - aggregation_data = self.implementation.del_property( - construct, "aggregation_data", None + aggregated_data = self.implementation.del_property( + construct, "aggregated_data", None ) else: - aggregation_data = None + aggregated_data = None cfa_array, kwargs = self._create_cfanetcdfarray( ncvar, @@ -238,6 +238,8 @@ def _create_data( calendar=kwargs["calendar"], ) + # Note: We don't cache elements from CFA variables + # Set the CFA write status to True iff each non-aggregated # axis has exactly one dask storage chunk if cfa_term is None: @@ -250,17 +252,15 @@ def _create_data( cfa_write = False break - data._set_cfa_write(cfa_write) + data._cfa_set_write(cfa_write) - # Store the 'aggregation_data' attribute - if aggregation_data: - data.cfa_set_aggregation_data(aggregation_data) + # Store the 'aggregated_data' attribute + if aggregated_data: + data.cfa_set_aggregated_data(aggregated_data) # Store the file substitutions data.cfa_set_file_substitutions(kwargs.get("substitutions")) - # Note: We don't cache elements from aggregated data - return data def _is_cfa_variable(self, ncvar): @@ -358,7 +358,6 @@ def _customize_read_vars(self): """ super()._customize_read_vars() - g = self.read_vars if not g["cfa"]: return @@ -384,8 +383,9 @@ def _customize_read_vars(self): # aggregated dimensions. dimensions = g["variable_dimensions"] attributes = g["variable_attributes"] + for ncvar, attributes in attributes.items(): - if "aggregate_dimensions" not in attributes: + if "aggregated_dimensions" not in attributes: # This is not an aggregated variable continue @@ -400,7 +400,8 @@ def _customize_read_vars(self): ncvar, attributes.get("aggregated_data") ) for x in parsed_aggregated_data: - term_ncvar = tuple(x.items())[0][1] + term, term_ncvar = tuple(x.items())[0] + term_ncvar = term_ncvar[0] g["do_not_create_field"].add(term_ncvar) def _cache_data_elements(self, data, ncvar): @@ -552,6 +553,8 @@ def _create_cfanetcdfarray( if term != "file": continue + term_ncvar = term_ncvar[0] + subs = g["variable_attributes"][term_ncvar].get("substitutions") if subs is None: subs = {} @@ -729,6 +732,8 @@ def _customize_field_ancillaries(self, parent_ncvar, f): if term in standardised_terms: continue + ncvar = ncvar[0] + # Still here? Then we've got a non-standard aggregation # term from which we can create a field # ancillary construct. @@ -761,26 +766,3 @@ def _customize_field_ancillaries(self, parent_ncvar, f): out[ncvar] = key return out - - def _cfa(self, ncvar, f): - """TODOCFADOCS. - - .. versionadded:: TODOCFAVER - - :Parameters: - - ncvar: `str` - The netCDF variable name. - - f: `Field` or `Domain` - TODOCFADOCS. - - :Returns: - - TODOCFADOCS. - - """ - pass - - -# x = self._parse_x(ncvar, aggregated_data, keys_are_variables=True) diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 45741bf7a2..fdea8d3db7 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -55,10 +55,10 @@ def _use_cfa(self, cfvar, construct_type): if data is None: return False - if not data.get_cfa_write(): + if not data.cfa_get_write(): return False - for ctype, ndim in g["cfa_options"].get("constructs", {}): + for ctype, ndim in g["cfa_options"].get("constructs", {}).items(): # Write as CFA if it has an appropriate construct type ... if ctype in ("all", construct_type): # ... and then only if it satisfies the number of @@ -109,6 +109,7 @@ def _write_data( compressed=False, attributes={}, construct_type=None, + warn_invalid=None, ): """Write a Data object. @@ -177,9 +178,14 @@ def _write_data( # Check for out-of-range values if g["warn_valid"]: + if construct_type: + var = cfvar + else: + var = None + dx = dx.map_blocks( self._check_valid, - cfvar=cfvar, + cfvar=var, attributes=attributes, meta=np.array((), dx.dtype), ) @@ -385,7 +391,7 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): # Get the fragment netCDF dimensions. These always start with # "f_". # ------------------------------------------------------------ - aggregation_address = ggg["aggregation_address"] + aggregation_address = ggg["address"] fragment_ncdimensions = [] for ncdim, size in zip( ncdimensions + ("extra",) * (aggregation_address.ndim - ndim), @@ -406,9 +412,10 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): # Write the standardised aggregation instruction variables to # the CFA-netCDF file # ------------------------------------------------------------ - aggregated_data = data.cfa_get_aggregated_data(default={}) - substitutions = data.cfa_get_file_substitutions() + substitutions = data.cfa_file_substitutions() + substitutions.update(g["cfa_options"].get("substitutions", {})) + aggregated_data = data.cfa_get_aggregated_data(default={}) aggregated_data_attr = [] # Location @@ -425,7 +432,7 @@ def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): if substitutions: subs = [] for base, sub in substitutions.items(): - subs.append(f"${{base}}: {sub}") + subs.append(f"{base}: {sub}") attributes = {"substitutions": " ".join(substitutions)} else: @@ -794,12 +801,10 @@ def _ggg(self, data, cfvar): # Define the CFA file susbstitutions, giving precedence over # those set on the Data object to those provided by the CFA # options. - data.cfa_set_file_substitutions( - g["cfa_options"].get("substitutions", {}) - ) - substitutions = data.cfa_get_file_substitutions() + substitutions = data.cfa_file_substitutions() + substitutions.update(g["cfa_options"].get("substitutions", {})) - relative = g["cfa_options"].get("relative_paths") + absolute_paths = g["cfa_options"].get("absolute_paths") cfa_dir = g["cfa_dir"] # Size of the trailing dimension @@ -833,11 +838,11 @@ def _ggg(self, data, cfvar): uri_scheme = uri.scheme if not uri_scheme: filename = abspath(join(cfa_dir, filename)) - if relative: - filename = relpath(filename, start=cfa_dir) - else: + if absolute_paths: filename = PurePath(filename).as_uri() - elif relative and uri_scheme == "file": + else: + filename = relpath(filename, start=cfa_dir) + elif not absolute_paths and uri_scheme == "file": filename = relpath(uri.path, start=cfa_dir) if substitutions: @@ -908,10 +913,10 @@ def _ggg(self, data, cfvar): # ------------------------------------------------------------ data = type(data) return { - "aggregation_location": data(aggregation_location), - "aggregation_file": data(aggregation_file), - "aggregation_format": data(aggregation_format), - "aggregation_address": data(aggregation_address), + "location": data(aggregation_location), + "file": data(aggregation_file), + "format": data(aggregation_format), + "address": data(aggregation_address), } def _customize_write_vars(self): diff --git a/cf/read_write/read.py b/cf/read_write/read.py index 50298b44fc..989038a677 100644 --- a/cf/read_write/read.py +++ b/cf/read_write/read.py @@ -59,7 +59,7 @@ def read( warn_valid=False, chunks="auto", domain=False, - cfa_options=None, + cfa=None, ): """Read field or domain constructs from files. @@ -640,6 +640,11 @@ def read( .. versionadded:: 3.11.0 + cfa: `dict`, optional + TODOCFADOCS + + .. versionadded:: TODOCFAVER + umversion: deprecated at version 3.0.0 Use the *um* parameter instead. @@ -773,25 +778,26 @@ def read( f"when recursive={recursive!r}" ) - # Parse the 'cfa_options' parameter - if not cfa_options: + # Parse the 'cfa' parameter + if cfa is None: cfa_options = {} else: - cfa_options = cfa_options.copy() + cfa_options = cfa.copy() keys = ("substitutions",) if not set(cfa_options).issubset(keys): raise ValueError( - "Invalid dictionary key to the 'cfa_options' " - f"parameter. Valid keys are {keys}. Got: {cfa_options}" + "Invalid dictionary key to the 'cfa' parameter." + f"Valid keys are {keys}. Got: {cfa_options}" ) - cfa_options.setdefault("substitutions", {}) - - substitutions = cfa_options["substitutions"].copy() - for base, sub in substitutions.items(): - if not (base.startswith("${") and base.endswith("}")): - # Add missing ${...} - substitutions[f"${{{base}}}"] = substitutions.pop(base) + if "substitutions" in cfa_options: + substitutions = cfa_options["substitutions"].copy() + for base, sub in tuple(substitutions.items()): + if not (base.startswith("${") and base.endswith("}")): + # Add missing ${...} + substitutions[f"${{{base}}}"] = substitutions.pop(base) + else: + substitutions = {} cfa_options["substitutions"] = substitutions diff --git a/cf/read_write/write.py b/cf/read_write/write.py index c00a1f9389..20af3e93a2 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -2,7 +2,7 @@ from ..cfimplementation import implementation from ..decorators import _manage_log_level_via_verbosity -from ..functions import CFA, flat +from ..functions import _DEPRECATION_ERROR_FUNCTION_KWARG_VALUE, CFA, flat from .netcdf import NetCDFWrite netcdf = NetCDFWrite(implementation()) @@ -27,7 +27,8 @@ def write( shuffle=True, reference_datetime=None, verbose=None, - cfa_options=None, + cfa=False, + # cfa_options=None, single=None, double=None, variable_attributes=None, @@ -575,20 +576,17 @@ def write( .. versionadded:: 3.14.0 - cfa_options: `dict`, optional - Parameters for configuring the output CFA-netCDF file. By - default *cfa_options* is ``{'paths': 'absolute', - 'constructs': 'field'}`` and may have any subset of the - following keys (and value types): + cfa: `bool` or `dict`, otional + If True or a (possibly empty) dictionary then write the + constructs as CFA-netCDF aggregated variables, where + possible and where requested. - * ``'paths'`` (`str`) - - How to write fragment file names. Set to ``'absolute'`` - (the default) for them to be written as fully qualified - URIs, or else set to ``'relative'`` for them to be - relative as paths relative to the CFA-netCDF file being - created. Note that in both cases, fragment file defined - by fully qualified URLs will always be written as such. + If *cfa* is a dictionary then it is used to configure the + CFA write process. The default options when CFA writing is + enabled are ``{'constructs': 'field', 'absolute_paths': + True, 'strict': True, 'substitutions': {}}``, and the + dictionary may have any subset of the following key/value + pairs: * ``'constructs'`` (`dict` or (sequence of) `str`) @@ -630,52 +628,49 @@ def write( variables: ``{'field': None, 'auxiliary_coordinate': cf.ge(2)}}``. + * ``'absolute_paths'`` (`bool`) + + How to write fragment file names. Set to ``'absolute'`` + (the default) for them to be written as fully qualified + URIs, or else set to ``'relative'`` for them to be + relative to the CFA-netCDF file being created. Note that + in both cases, fragment files defined by fully qualified + URLs will always be written as such. + + * ``'absolute_paths'`` (`bool`) + + How to write fragment file names. Set to ``'absolute'`` + (the default) for them to be written as fully qualified + URIs, or else set to ``'relative'`` for them to be + relative to the CFA-netCDF file being created. Note that + in both cases, fragment files defined by fully qualified + URLs will always be written as such. + + * ``'strict'`` (`bool`) + + If True (the default) then raise an exception if it is + not possible to write a data identified by the + ``'constructs'`` key as a CFA aggregated variable. If + False then a warning is logged, and the is written as a + normal netCDF variable. + * ``'substitutions'`` (`dict`) A dictionary whose key/value pairs define text substitutions to be applied to the fragment file URIs. Each key must be a string of one or more letters, - digits, and underscores. These substitutions take - precendence over any that are also defined on individual - constructs. + digits, and underscores. These substitutions are used in + conjunction with, and take precendence over, any that + are also defined on individual constructs. Substitutions are stored in the output file by the - ``substitutions`` attribute of the ``file`` aggregation - instruction variable. + ``substitutions`` attribute of the ``file`` CFA + aggregation instruction variable. *Parameter example:* ``{'base': 'file:///data/'}}`` - * ``'properties'`` ((sequence of) `str`) - - For fragments of a field construct's data, a (sequence - of) `str` defining one or more properties of the file - fragments. For each property specified, the value of - that property from each fragment is written to the - output CFA-netCDF file in a non-standardised aggregation - instruction variable whose term name is the same as the - property name. - - When the output file is read in with `cf.read` these - variables are converted to field ancillary constructs. - - *Parameter example:* - ``'tracking_id'`` - - *Parameter example:* - ``('tracking_id', 'model_name')`` - - * ``'strict'`` (`bool`) - - A `bool` that determines whether or not to raise an - `Exception` if it is not possible to write as a CFA - aggregated variable a identified by the ``'constructs'`` - key If True, the default, then an `Exception` an - exception is raised, otherwise a warning is logged. - - * ``'base'`` - - Deprecated at version 3.14.0 and no longer available. + .. versionadded:: TODOCFAVER :Returns: @@ -695,6 +690,15 @@ def write( >>> cf.write(f, 'file.nc', Conventions='CMIP-6.2') """ + if fmt in ("CFA", "CFA4", "CFA3"): + return _DEPRECATION_ERROR_FUNCTION_KWARG_VALUE( + "cf.write", + {"fmt": fmt}, + "Use keyword 'cfa' instead", + version="TODOCFAVER", + removed_at="5.0.0", + ) # pragma: no cover + # Flatten the sequence of intput fields fields = tuple(flat(fields)) if fields: @@ -732,52 +736,34 @@ def write( # ------------------------------------------------------------ # CFA # ------------------------------------------------------------ - if fmt in ("CFA", "CFA4"): - cfa = True - fmt = "NETCDF4" - elif fmt == "CFA3": + if isinstance(cfa, dict): + cfa_options = cfa.copy() cfa = True - fmt = "NETCDF3_CLASSIC" else: - cfa = False - - if not cfa: cfa_options = {} - else: + cfa = bool(cfa) + + if cfa: # Add CFA to the Conventions + cfa_conventions = f"CFA-{CFA()}" if not Conventions: - Conventions = CFA() + Conventions = cfa_conventions elif isinstance(Conventions, str): - Conventions = (Conventions, CFA()) - else: - Conventions = tuple(Conventions) + (CFA(),) - - # Parse the 'cfa_options' parameter - if not cfa_options: - cfa_options = {} + Conventions = (Conventions, cfa_conventions) else: - cfa_options = cfa_options.copy() - keys = ("paths", "constructs", "substitutions", "properties") - if not set(cfa_options).issubset(keys): - raise ValueError( - "Invalid dictionary key to the 'cfa_options' " - f"parameter. Valid keys are {keys}. " - f"Got: {cfa_options}" - ) - - cfa_options.setdefault("paths", "absolute") - cfa_options.setdefault("constructs", "field") - cfa_options.setdefault("substitutions", {}) - # cfa_options.setdefault("properties", ()) + Conventions = tuple(Conventions) + (cfa_conventions,) - paths = cfa_options.pop("paths") - if paths not in ("relative", "absolute"): + keys = ("constructs", "absolute_paths", "strict", "substitutions") + if not set(cfa_options).issubset(keys): raise ValueError( - "Invalid value of 'paths' CFA option. Valid paths " - f"are 'relative' and 'absolute'. Got: {paths!r}" + "Invalid dictionary key to the 'cfa_options' " + f"parameter. Valid keys are {keys}. Got: {cfa_options}" ) - cfa_options["relative_paths"] = paths == "relative" + cfa_options.setdefault("constructs", "field") + cfa_options.setdefault("absolute_paths", True) + cfa_options.setdefault("strict", True) + cfa_options.setdefault("substitutions", {}) constructs = cfa_options["constructs"] if isinstance(constructs, dict): @@ -796,12 +782,6 @@ def write( cfa_options["substitutions"] = substitutions - # properties = cfa_options["properties"] - # if isinstance(properties, str): - # properties = (properties,) - # - # cfa_options["properties"] = tuple(properties) - extra_write_vars["cfa"] = cfa extra_write_vars["cfa_options"] = cfa_options diff --git a/cf/test/file2.nc b/cf/test/file2.nc deleted file mode 100644 index 8cf7800912960c73c0b2e859a1849d8ba04022b9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24533 zcmeG^33yaR(mlx#Vi-bzKtdq!I7AM?1j8YU(~uA{6O2d{&oxZQKt_{Em`o5{1QpL! z*Zamr@jiZC&)=K4uCBW(qT-1n$RYfoqW&y+ty|Sq^)h)mAdvmV?@z-^O?Ov!SJ(CW zb=P~B7Ubo3PRdA9m=Y6}Tcwzv^5+*$@0C!AllVFZ)nlnZ~Q7&5Moqte*~l*drvQpqY2{*@HvPK#7k2P4r?um&)?!!;Hz z(*eMF!AMQ0Dm)>3VK^9_RW`dKR4PR|Bgc%)URYi>dvr`;_Q>q4k)xCcNOQqIj~OJl zDm*V3jzT{;1Pzh^PzK4(ACWzBtY?T)NI8;~sH!7XX9QX3*vadFg;SogdcL#ZRU zd5=CITczSVD-od+da|%r7Z8VY$elW|xES)tRFs8Kk>@*Ra#6AG*rJl++=){sD^DJA ztAlA(H1U|pfNWX~ifI*yR#w$imj@%klJW(mkx*Glc`!7mJX$iVswz?z3J0P=z+RK^ z&8%Z{3krqS_-1KUV{jDb3~1(9(;$&@t24Rq`5`CYj-wFvC|1CQTZhvTlRkP-6M^?`MyY zmZ;JeL#44e{e$GJDbnrG#sVjrby%`zB%H({nU!6((ZYKo#5bs|`x8)|5>U z_V5(rroT~I&-n~g5J*!9sDups;c`o(j-^$#;j)@hLC|oMSflZGFN0LP>luu0~R=f+`N3NEL0f`qi?WEYok`UDq3GPFAxbel;J3KTk)qwwhV_r$I3;b zutyLX%7~J0oz3pzW~bl~kVK-*+X2oW=h+2pG+ou(=2Tzt@k;fAI<(z1xKK;Pvj}wp z4b`kh9S{#L(*S^BdJN~&*ZuM)%22fMDI+zdR&i%Y5qiS5qbL=qh^43pbng3|SNJQ(6*fS1H3tFZbcQ&i4vbr`JEGwyr2Et{5NLfiZPzjz2ouXl= z8C9u3MKlzxEem=I(LKoh$+Wp5!`q@*0x3zOhW|Ds`1F8Cb@-NxVC zp9%y|pp+lvz?Xjb9$@z)<((Y((vQbcF8G}tE{rz?c5fo-xUM1)uM2cDe9}R~wDW z2U9n8xyXgpt5NxoX7GnP%4J-17np=jDIZS7jmi&olnXs09OaW4k(cv!KT>{}17G-) z!zV+f5;DEJG@MZCb-*s z+LeFVIpnl>{qU6aXQw{=i&x-X)f19FxO2Bx;4L>}`o=4;--X+bd%Eu?ufU&kBJ&&I*A{2iLna_45;tx0A=&&hQhNmm<7Rc--wg-cHW{w1WEWUbh{n zzi?Yu8&bQJI)QJ#fyq-RumIUJ<)Au&F9X-E{7Z*Afyk}u32}7-=b9fsuMjxfq<^ju zh@&6(^j9mW|88}rc~aLpBaSPRIp3+yu=&D{btEr$-S2^l^=B)l`}-A!to7fnp!#FA zf#=T9OaE&9IkMZYYp+dr&-j;Lzx0ngCO;KlNacxzdh6iq#D_eE`fr6}FP)fDs1L`~ zy-;^sQumKLdKBuiBR@O#inKy~#f@Po@@qr)tA72~uHp@MzwW1YtNhxKv)ZqdM?7}h z*B|)x>CgH+aclhg6zt$rzka9>{_^X^XFMOh_7bM&kAD5bA}sLh_vQm7H~F=9G_+jm z*GD|@+PoK+`Sm@w!UUH4wVD6je*M2cub#j49zW4D&99Nn$us;q5eGQcPxGGX*Vt=% znV;yN7y~ANSEb{@bUGzCP#EaY*2c zK8@Yn|B_E5xw~HW>3^a+zTzYPRQoit!9CBXNnd<4kHtQc^Aev%aou~7Ps8A==U?h0 zxnAbeMs2TPdd4w*6MXv8i7>HAK8@lXKE+3JF7%Op75VhPKs@aHC+L)uP~$v$!`+~8SzR@%!G?oV$v*m| zk$$IUki0yZ+Sq%XrKcMMcIg+Fq2N1`zWnGRIi1;61D7uDd2*^Q#Q_S^v_T)2!G0{0 z?4z^Abh}8OpXx|_S=5>Oy{oG}IN`kOb1zFJdx)fwew~`3cY_(;nwLp>m1a?YSH$U0 z2VsLm)~n9?EJI*dJrK>{d7jtOG^#%*gY0-_rZ#@0zeRR7IG*%uWg_cOGV`IEHsowi zCH;9djqNpq@iIv-hFd&dy#5x%pK{Vcq@O#J^-ac|bYuSZAbb2OP2cw_IHH#_Xq@XZ zbrCWy$I_X`k+^vt2eJKj(Y=A|3Fmd={(5M?VR*X!>3&eS&ofxgnKaJ{7R_(Ho9auG z^i*(rGdFe7<>&yO>_&PwzX#EMRJulYKu3CL3fb?L zZY;MRR6i?SSD{1g-<$ZnE|ceDk-t)^G#Cwl%bpXctOQsVtiWurx+YXn6+WBl=&y>F zn;)Sn)8NcNbu~6LDOD33$#4+ELm7?-7<(Y~K5tFSPM)VgsZREp%EBpQrV z)+jnku>iW@!W0;91+3EXR;g7LshTt1s;Q044$KOc$O#V~h9;@4ClYo~X?7ju54^ui zWFQ&uGbVq=JY-tK?-C^Z(yv2PMPg)KuliT?C*#lee7>?06zyMdCB7g;I-_HDK z_4?(p<7GS(ZJXzhw`}Zz5SKEaxT!k+6Bu;$9<(~`hT?=FvUCHqqAV8Yxex_rrX|83{ zyxr6)BQg|i>mEFC^hM9kUL(i9a8AyrC%u>=51g9d0}Vy8)P_S*F?Bpn6tc^Lb0WcD zjRzVxeZ-0k5#$9Wr4WnoW5L@;#7^i{o{*g8d1lDDGltMK!IY>Z{z4c4R0t| zI5;zc?3ejF3mU1@$9-pFaO8Y{5ufNy!%9dVI6-Ooh#QYcWr}k!)Tbgqr{m~56Vxv! znSRt5!C6lE1O4wGuAOjF(|%NxUSR9|!Ouf!=0WC%_|Y~G)EaFk945*piv>z4zpaElY=3B?NJDG34rXR)AdLuUWqvn)8#*e z>3sAL%P^BXaMaqe14#xvaSr4_s={+jd<})7{h^q88Z1b#`eFf4#R2m-{7UULYRA&E ze~Oi%MB_<_gXd|9n2JIS*YwHxlc!HE%AH(7FtTJZK#7KjFFg3&!rO=7+55I1hI zCk8c&Y-}b9b>flKq455gLk|8OnfoP<<)xUpz}A`fk_j}nu5hJ1SrrGp@1aHhTNLgS(ja3s0&k90# z4j^3^I@KxfK0LE&7c5H$alOub>tr;VUEM`M0Va`A3-OA^X*abpLyBY;yfP~(LXpw} z5!X{q#nYi$OEXz#K(t(adVj+Ur{_;Em}Uy&j45%YsL?QvNt3m%?Mslcm>8m{_9g+k zs63Nq0W9U)l}lZ<^A?v`sU;vD28h8*jpJVGS8Bn_A9*6b7}j&@c%GkhDo|)~t7&S! zvLH){5?9N+;!5D7oz&Jz$4|{y9gezkdc4B5;RLnloM1=w9Q;gGKOF7pqAq|wdaFAh zJ->_U4nH%MulLg4Dhq!0RqvfplcjpV@B`JtBQ}py%b{5>VucPdFlc{$X1CLzkiZi2A!p<>tEivSgkZa-+AoD>r^@X?4!2b z_sIRK0(uyu2Cwj~RYg#mq3)gg@;bG_jCSI8pKMl-Luqf7Q*^=>H4ggir|!M*p&!&+ zfRL&Ru3qr7S<&`Tx8Ab&Oekwl^&Bwrx5KhoK)^yuq!PF~Fp1>A)}dJ3_UeX|#=fnP zc*L);FvPvEQs7gGNu*jH$&z~EbAw4FbX*0PM4EwiViIX6-V#kBZIZA_Rt<@kWCiOk z$fUffk|pJ_?V6Ohr__?lN&qD5)Ux|d1e5&DCbQy8beRpYF6uyBZ-{;goH3*!vp*U^ zd2j(HJwBFyFu2oCKcqgXs9}AN=?&|js9ENXU7h`Bw-0Z8IAwLy&)@dOitnhtpV}ll zXM5P#x1P=~$ZPmO@$D^(5}JQ)!sQnG9<~<&=e!3ol&#+77H=?cFB(gbrOPc&BZ2l! zk_R%=-6GcdJtLP}oQ7+@L|wYv;#U&rVM4T^navNETU<+$ECNchZQi-X(R6}+XL^t* z$rinFxyA2vtcgY{g5j|eg6wjO3pyxJJ3mwr3Pcvzt6#VHqeR)lIqN8yqxdFQV8PK+ zCJrjGlsSAMAL_c?;;dYyH%=6E@j7wQz@Lw&Li;bR>`rsJjAh81D8fW>xz7CFuI%Ko z?IMn}miIc^dTlq^152Kk*LFAsF=JbDo|*Er)K+i2Ag|3ZtaG@h=id1Bl31_M=ppt) zeB{I}*#pssip>fX>w$@xWLpO3#TGcLy6l+#C23h7$1|O*tOc@G?eU#8dT^gJoJMdy z)3xxt01p5tomRe+IFnSX@sWWxy}=(y`}bBD-gXYY<(wCYPs1G6C{`$Za^~aB8=JT; zLCur*`{bPWUn3?p1@>kwahDdp z*o3x)CfRoHZsFcftZm`z9!F%Z-`w=msZb_2EN8y8K5VH4!$A^767c*f1k^~zjl)g>e{+4+{*{xu^=0MeY zS=$P<*UCH#d$G%Tx%-Mdoz`Hl1Q|c)?`2zTjc8NrxN{N>!OXYsC@uIyP+T3L z(*}?SHBSPAc&d8@Q*Y|>1R>?yhXTy^ea0H1~Ojc{#&YZR2-0oPM--3-@NP=9gm;vdIXfAz?6y}$5;tYP;b z{^)YOzwou2p1UFIvE_Py;naJQ@;5<0`wJ@}Hu1Q}m+Sq7%iet8ty<`3f8qYpA?^2< zFVcR0}1v75(68z9hLCu>u|C*PArC;J;@O54aa{k;=ztu~@$e-Ko$*Naz zyE+ao|G-PZ&o6TMDlY{=f8wPe=nw4RZVC?n$V)-f?|V%U?FZYq{Yozdw{G;B;Mbj7 zn9jGk{qr2${sqI&ye3#Vb+ea(xYskCuXs)H^WhtKUjM`WZs&0}dnpKe4UhAoO)mwL zf5GjyasO-Wc?!K01pcXAPv!42|F+n4(Ks7yzR0-5$1Ps!=S`a)3YK5R`y}glURyc8 ziuth7&J*8PGJRigF#UV>c*IYc_gcF@3X0#({Q21C5Ak!Y-9O3yW2SF|EiVdc|J-gT zd6Mr|E?;fSiRfHw=ZUVrGT%2c-J5uvZT5T!Z#C0HT>JmK$p8nZO0NoWAr8gdz~6TM1wH3Php1`Y*5qBay0z?BWCM?Wt`zKxWp@ zVP;}ZvVrByi<#}|qhzwF)tyhzoZHGRJfIK6`88!`@ur++e`e*Vw%@%PA_K6g42683 z9hE!f88r#WIxu%;!_5GkldTUBO56qU9x#Kv}4^2Fo#aiA1C^+w5-AiZKCGFPhQ|;sw%` z2{8*-VVBia6+zoQ8C3Yw)ED=#PQIokvk*w_a<#bbjHXACqbNe(w^8cmX#7GHh8A@* zd%k8amGJsRsPW#fy%!53Ht&i@J@UZCB-xO|6*T7p#3i)W6>Q**ZKo^P-#c!h2Rd%~ zWC=H6MO=;>jb-1V&5cHc*t!i6xOW)6e5TtHr|0)iw zg-9FM<}^Omw6|i0vsZE1mc5QE$+qmHa?d2(*+rL)hf%SP2|6*o?VUsNKmZ3G zfux##0^wPqiV8T1FOd&5)>GGlRXdiCo>}-Ng3!i1h?ZBCna+??809LAT`=C7Ki-0M zZP+>sKVb@oX5lhxNGJ?1k zLkPQ%!-X);=Q#4C9r-bkzx9x37a8{gkv|8ndr1IFF$t7?bt*}`m$Q7Cr9`Yqy{zmcFDpC27bh(B z!ZmTBDG%ONMz5P!YlE|;nqFgGIG6GtR@3YBMO=Rs!}(kuo=dN3zgtbP(_DW2Uhp^% zcxIf1+<%nu2Xgyc>+#JF*P3c=c3kFi`<{#k_@;aT=QnV>bD7>mF8@9Ee=7Im<8ik$ z%;WN#c^>n4oFeZ3nfi8yo+XTTG>=ox`9JbFmoWr-%nsmYZvS^4FNgbI#ChO@nTN|k zZ#?H=JR=V+e5vo>@Uaim*_G*ejr(;opD$qiGZ}9y_mfheH*~(p<>?|HhQJ>)@8Wv; zP5wL1Pvv>8WcXG6cm}UK&qMhCJ-1t0PluUbIrHTn?kB%~{$`$eOz$1cuZf(0iTj!6 X&=1g?c6|oPw(GO)$4HNUyZZcZP}yv2 diff --git a/cf/test/test_Data.py b/cf/test/test_Data.py index 8577d3ae4f..2f0ff6c58c 100644 --- a/cf/test/test_Data.py +++ b/cf/test/test_Data.py @@ -4506,6 +4506,88 @@ def test_Data_convert_reference_time(self): self.assertEqual(e.Units, units) self.assertTrue((e.array == [72, 48, 24, 0]).all()) + def test_Data_cfa_aggregated_data(self): + """Test `Data` CFA aggregated_data methods""" + d = cf.Data(9) + aggregated_data = { + "location": "cfa_location", + "file": "cfa_file", + "address": "cfa_address", + "format": "cfa_format", + "tracking_id": "tracking_id", + } + + self.assertFalse(d.cfa_has_aggregated_data()) + self.assertIsNone(d.cfa_set_aggregated_data(aggregated_data)) + self.assertTrue(d.cfa_has_aggregated_data()) + self.assertEqual(d.cfa_get_aggregated_data(), aggregated_data) + self.assertEqual(d.cfa_del_aggregated_data(), aggregated_data) + self.assertFalse(d.cfa_has_aggregated_data()) + self.assertIsNone(d.cfa_get_aggregated_data(None)) + self.assertIsNone(d.cfa_del_aggregated_data(None)) + + def test_Data_cfa_file_substitutions(self): + """Test `Data` CFA file_substitutions methods""" + d = cf.Data(9) + self.assertFalse(d.cfa_has_file_substitutions()) + self.assertIsNone( + d.cfa_set_file_substitutions({"base": "file:///data/"}) + ) + self.assertTrue(d.cfa_has_file_substitutions()) + self.assertEqual( + d.cfa_file_substitutions(), {"${base}": "file:///data/"} + ) + + d.cfa_set_file_substitutions({"${base2}": "/home/data/"}) + self.assertEqual( + d.cfa_file_substitutions(), + {"${base}": "file:///data/", "${base2}": "/home/data/"}, + ) + + d.cfa_set_file_substitutions({"${base}": "/new/location/"}) + self.assertEqual( + d.cfa_file_substitutions(), + {"${base}": "/new/location/", "${base2}": "/home/data/"}, + ) + self.assertEqual( + d.cfa_del_file_substitution("${base}"), + {"${base}": "/new/location/"}, + ) + self.assertEqual( + d.cfa_clear_file_substitutions(), {"${base2}": "/home/data/"} + ) + self.assertFalse(d.cfa_has_file_substitutions()) + self.assertEqual(d.cfa_file_substitutions(), {}) + self.assertEqual(d.cfa_clear_file_substitutions(), {}) + self.assertIsNone(d.cfa_del_file_substitution("base", None)) + + def test_Data_file_location(self): + """Test `Data` file location methods""" + f = cf.example_field(0) + + # Can't set file locations when no data is in a file + with self.assertRaises(ValueError): + f.data.set_file_location("/data/model/") + + cf.write(f, file_A) + d = cf.read(file_A, chunks=4)[0].data + self.assertGreater(d.npartitions, 1) + + e = d.copy() + location = os.path.dirname(os.path.abspath(file_A)) + + self.assertEqual(d.file_locations(), set((location,))) + self.assertIsNone(d.set_file_location("/data/model/")) + self.assertEqual(d.file_locations(), set((location, "/data/model"))) + + # Check that we haven't changed 'e' + self.assertEqual(e.file_locations(), set((location,))) + + self.assertIsNone(d.del_file_location("/data/model/")) + self.assertEqual(d.file_locations(), set((location,))) + d.del_file_location("/invalid") + self.assertEqual(d.file_locations(), set((location,))) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) diff --git a/cf/test/test_NetCDFArray.py b/cf/test/test_NetCDFArray.py index ee8a58c148..3089f00943 100644 --- a/cf/test/test_NetCDFArray.py +++ b/cf/test/test_NetCDFArray.py @@ -23,6 +23,11 @@ def test_NetCDFArray_del_file_location(self): self.assertEqual(b.get_filenames(), ("/data1/file1",)) self.assertEqual(b.get_addresses(), ("tas1",)) + # Can't be left with no files + self.assertEqual(b.file_locations(), ("/data1",)) + with self.assertRaises(ValueError): + b.del_file_location("/data1/") + def test_NetCDFArray_file_locations(self): a = cf.NetCDFArray("/data1/file1") self.assertEqual(a.file_locations(), ("/data1",)) diff --git a/cf/test/test_read_write.py b/cf/test/test_read_write.py index 64c5434e98..c4d424232e 100644 --- a/cf/test/test_read_write.py +++ b/cf/test/test_read_write.py @@ -920,6 +920,36 @@ def test_write_omit_data(self): self.assertFalse(g.array.count()) self.assertTrue(g.construct("grid_latitude").array.count()) + def test_read_write_cfa(self): + """TODOCFADOCS""" + f = cf.example_field(0) + # tmpfile1 = 'f1.nc' + # tmpfile2 = 'f2.nc' + # tmpfileh = 'n.nc' + # tmpfileh2 = 'c.nc' + + cf.write(f[:2], tmpfile1) + cf.write(f[2:], tmpfile2) + + a = cf.read([tmpfile1, tmpfile2]) + self.assertEqual(len(a), 1) + a = a[0] + + nc_file = tmpfileh + cfa_file = tmpfileh2 + cf.write(a, nc_file) + cf.write(a, cfa_file, cfa=True) + + n = cf.read(nc_file) + c = cf.read(cfa_file) + self.assertEqual(len(n), 1) + self.assertEqual(len(c), 1) + + n = n[0] + c = c[0] + self.assertTrue(c.equals(f)) + self.assertTrue(c.equals(n)) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) diff --git a/docs/source/class/cf.Data.rst b/docs/source/class/cf.Data.rst index 3a01228d82..feb33ab2c0 100644 --- a/docs/source/class/cf.Data.rst +++ b/docs/source/class/cf.Data.rst @@ -70,6 +70,7 @@ Dask ~cf.Data.cull_graph ~cf.Data.dask_compressed_array ~cf.Data.rechunk + ~cf.Data.todict ~cf.Data.to_dask_array .. rubric:: Attributes From 76eef2c610c8e314e96f8f131e3770b80c826472 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 7 Mar 2023 17:59:48 +0000 Subject: [PATCH 049/141] dev --- cf/read_write/netcdf/netcdfwrite.py | 11 ++++----- cf/test/test_read_write.py | 35 +---------------------------- 2 files changed, 7 insertions(+), 39 deletions(-) diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index fdea8d3db7..d436fc75eb 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -43,7 +43,7 @@ def _use_cfa(self, cfvar, construct_type): """ if construct_type is None: - # This prrevents recursion whilst writing CFA-netCDF term + # This prevents recursion whilst writing CFA-netCDF term # variables. return False @@ -63,11 +63,12 @@ def _use_cfa(self, cfvar, construct_type): if ctype in ("all", construct_type): # ... and then only if it satisfies the number of # dimenions criterion - if ndim is None: - return True - - return ndim == data.ndim + ok = ndim is None or ndim == data.ndim + if ok and cfa_options.get('strict', True) and not data.cfa_get_write(): + return False + return ok + return False def _customize_createVariable(self, cfvar, construct_type, kwargs): diff --git a/cf/test/test_read_write.py b/cf/test/test_read_write.py index c4d424232e..20cee549ab 100644 --- a/cf/test/test_read_write.py +++ b/cf/test/test_read_write.py @@ -42,8 +42,6 @@ def _remove_tmpfiles(): atexit.register(_remove_tmpfiles) -TEST_DASKIFIED_ONLY = True - class read_writeTest(unittest.TestCase): filename = os.path.join( @@ -72,7 +70,6 @@ class read_writeTest(unittest.TestCase): netcdf4_fmts = ["NETCDF4", "NETCDF4_CLASSIC"] netcdf_fmts = netcdf3_fmts + netcdf4_fmts - # @unittest.skipIf(TEST_DASKIFIED_ONLY, "KeyError: 'q'") def test_write_filename(self): f = self.f0 a = f.array @@ -920,37 +917,7 @@ def test_write_omit_data(self): self.assertFalse(g.array.count()) self.assertTrue(g.construct("grid_latitude").array.count()) - def test_read_write_cfa(self): - """TODOCFADOCS""" - f = cf.example_field(0) - # tmpfile1 = 'f1.nc' - # tmpfile2 = 'f2.nc' - # tmpfileh = 'n.nc' - # tmpfileh2 = 'c.nc' - - cf.write(f[:2], tmpfile1) - cf.write(f[2:], tmpfile2) - - a = cf.read([tmpfile1, tmpfile2]) - self.assertEqual(len(a), 1) - a = a[0] - - nc_file = tmpfileh - cfa_file = tmpfileh2 - cf.write(a, nc_file) - cf.write(a, cfa_file, cfa=True) - - n = cf.read(nc_file) - c = cf.read(cfa_file) - self.assertEqual(len(n), 1) - self.assertEqual(len(c), 1) - - n = n[0] - c = c[0] - self.assertTrue(c.equals(f)) - self.assertTrue(c.equals(n)) - - + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) cf.environment() From c27bbac2b69377f21fdb35d2d753993032f0ee97 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 7 Mar 2023 23:13:38 +0000 Subject: [PATCH 050/141] dev --- cf/read_write/netcdf/netcdfwrite.py | 10 +++++++--- cf/test/test_read_write.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index d436fc75eb..b68cd9e582 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -64,11 +64,15 @@ def _use_cfa(self, cfvar, construct_type): # ... and then only if it satisfies the number of # dimenions criterion ok = ndim is None or ndim == data.ndim - if ok and cfa_options.get('strict', True) and not data.cfa_get_write(): +TODO if ( + ok + and cfa_options.get("strict", True) + and not data.cfa_get_write() + ): return False - +TODO return ok - + return False def _customize_createVariable(self, cfvar, construct_type, kwargs): diff --git a/cf/test/test_read_write.py b/cf/test/test_read_write.py index 20cee549ab..0720ef4173 100644 --- a/cf/test/test_read_write.py +++ b/cf/test/test_read_write.py @@ -917,7 +917,7 @@ def test_write_omit_data(self): self.assertFalse(g.array.count()) self.assertTrue(g.construct("grid_latitude").array.count()) - + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) cf.environment() From 8cce776f0852df291dd8f6608199fe6c85779ef6 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 8 Mar 2023 17:07:11 +0000 Subject: [PATCH 051/141] dev --- cf/aggregate.py | 4 +- cf/cfimplementation.py | 4 + cf/data/array/cfanetcdfarray.py | 184 ++++++++++++----- cf/data/data.py | 17 ++ cf/read_write/netcdf/netcdfread.py | 302 +++++++++++++++++++--------- cf/read_write/netcdf/netcdfwrite.py | 61 +++--- cf/read_write/write.py | 104 +++++----- cf/test/test_CFA.py | 167 +++++++++++++++ cf/test/test_aggregate.py | 19 ++ 9 files changed, 630 insertions(+), 232 deletions(-) create mode 100644 cf/test/test_CFA.py diff --git a/cf/aggregate.py b/cf/aggregate.py index 9ac56d1550..3c4e18679c 100644 --- a/cf/aggregate.py +++ b/cf/aggregate.py @@ -1411,7 +1411,7 @@ def promote_to_auxiliary_coordinate(self, properties): aux_coord = AuxiliaryCoordinate( properties={"long_name": prop}, - data=Data([value], units=""), + data=Data([value]), copy=False, ) aux_coord.nc_set_variable(prop) @@ -1475,7 +1475,7 @@ def promote_to_field_ancillary(self, properties): data._cfa_set_term(True) field_anc = FieldAncillary( - data=data, properties={"long_name": prop} + data=data, properties={"long_name": prop}, copy=False ) field_anc.id = prop diff --git a/cf/cfimplementation.py b/cf/cfimplementation.py index 8f3861e035..8bba5ca84e 100644 --- a/cf/cfimplementation.py +++ b/cf/cfimplementation.py @@ -91,6 +91,7 @@ def initialise_CFANetCDFArray( instructions=None, substitutions=None, term=None, + x=None, **kwargs, ): """Return a `CFANetCDFArray` instance. @@ -115,6 +116,8 @@ def initialise_CFANetCDFArray( term: `str`, optional + x: `dict`, optional + kwargs: optional Ignored. @@ -134,6 +137,7 @@ def initialise_CFANetCDFArray( instructions=instructions, substitutions=substitutions, term=term, + x=x, ) diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index 025905b855..e1a29c562c 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -7,6 +7,7 @@ from ...functions import abspath from ..fragment import FullFragmentArray, NetCDFFragmentArray, UMFragmentArray +from ..utils import chunk_locations, chunk_positions from .netcdfarray import NetCDFArray @@ -44,6 +45,7 @@ def __init__( term=None, source=None, copy=True, + x=None, ): """**Initialisation** @@ -164,28 +166,69 @@ def __init__( term = None elif filename is not None: - from pathlib import PurePath - - from CFAPython import CFAFileFormat - from CFAPython.CFADataset import CFADataset - from CFAPython.CFAExceptions import CFAException - from dask import compute, delayed - - if not isinstance(filename, str): - if len(filename) != 1: - raise ValueError("TODOCFADOCS") - - filename = filename[0] - - cfa = CFADataset(filename, CFAFileFormat.CFANetCDF, "r") - try: - var = cfa.getVar(address) - except CFAException: - raise ValueError( - f"CFA variable {address!r} not found in file {filename}" - ) + aggregated_data = {} - shape = tuple([d.len for d in var.getDims()]) + location = x["location"] + ndim = location.shape[0] + + chunks = [i.compressed().tolist() for i in location] + shape = [sum(c) for c in chunks] + positions = chunk_positions(chunks) + locations = chunk_locations(chunks) + + if term is not None: + # -------------------------------------------------------- + # This fragment contains a constant value + # -------------------------------------------------------- + term = x[term] + fragment_shape = term.shape + aggregated_data = { + frag_loc: { + "location": loc, + "fill_value": term[frag_loc].item(), + "format": "full", + } + for frag_loc, loc in zip(positions, locations) + } + else: + a = x["address"] + f = x["file"] + fmt = x["format"] + if not a.ndim: + if f.ndim == ndim: + a = np.full(f.shape, a) + else: + a = np.full(f.shape[:-1], a) + + if not fmt.ndim: + fmt = fmt.astype("U") + if f.ndim == ndim: + fmt = np.full(f.shape, fmt) + else: + fmt = np.full(f.shape[:-1], fmt) + + if f.ndim == ndim: + fragment_shape = f.shape + aggregated_data = { + frag_loc: { + "location": loc, + "filename": f[frag_loc].item(), + "address": a[frag_loc].item(), + "format": fmt[frag_loc].item(), + } + for frag_loc, loc in zip(positions, locations) + } + else: + fragment_shape = f.shape[:-1] + aggregated_data = { + frag_loc: { + "location": loc, + "filename": f[frag_loc].tolist(), + "address": a[frag_loc].tolist(), + "format": fmt[frag_loc].item(), + } + for frag_loc, loc in zip(positions, locations) + } super().__init__( filename=filename, @@ -198,40 +241,77 @@ def __init__( copy=copy, ) - fragment_shape = tuple(var.getFragDef()) + if False: + # CFAPython vesion + from pathlib import PurePath - parsed_filename = urlparse(filename) - if parsed_filename.scheme in ("file", "http", "https"): - cfa_directory = str(PurePath(filename).parent) - else: - cfa_directory = dirname(abspath(filename)) - - # Note: It is an as-yet-untested hypothesis that creating - # the 'aggregated_data' dictionary for massive - # aggretations (e.g. with O(10e6) fragments) will be - # slow, hence the parallelisation of the process - # with delayed + compute; and that the - # parallelisation overheads won't be noticeable for - # small aggregations (e.g. O(10) fragments). - aggregated_data = {} - compute( - *[ - delayed( - self._set_fragment( - var, - loc, - aggregated_data, - filename, - cfa_directory, - substitutions, - term, - ) + from CFAPython import CFAFileFormat + from CFAPython.CFADataset import CFADataset + from CFAPython.CFAExceptions import CFAException + from dask import compute, delayed + + if not isinstance(filename, str): + if len(filename) != 1: + raise ValueError("TODOCFADOCS") + + filename = filename[0] + + cfa = CFADataset(filename, CFAFileFormat.CFANetCDF, "r") + try: + var = cfa.getVar(address) + except CFAException: + raise ValueError( + f"CFA variable {address!r} not found in file " + f"{filename}" ) - for loc in product(*[range(i) for i in fragment_shape]) - ] - ) - del cfa + shape = tuple([d.len for d in var.getDims()]) + + super().__init__( + filename=filename, + address=address, + shape=shape, + dtype=dtype, + mask=mask, + units=units, + calendar=calendar, + copy=copy, + ) + + fragment_shape = tuple(var.getFragDef()) + + parsed_filename = urlparse(filename) + if parsed_filename.scheme in ("file", "http", "https"): + cfa_directory = str(PurePath(filename).parent) + else: + cfa_directory = dirname(abspath(filename)) + + # Note: It is an as-yet-untested hypothesis that creating + # the 'aggregated_data' dictionary for massive + # aggretations (e.g. with O(10e6) fragments) will be + # slow, hence the parallelisation of the process + # with delayed + compute; and that the + # parallelisation overheads won't be noticeable for + # small aggregations (e.g. O(10) fragments). + aggregated_data = {} + compute( + *[ + delayed( + self._set_fragment( + var, + loc, + aggregated_data, + filename, + cfa_directory, + substitutions, + term, + ) + ) + for loc in product(*[range(i) for i in fragment_shape]) + ] + ) + + del cfa else: super().__init__( filename=filename, diff --git a/cf/data/data.py b/cf/data/data.py index 2b07fde4e4..12b11a2603 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1255,6 +1255,23 @@ def _cfa_del_write(self): """ return self._custom.pop("cfa_write", False) + def _cfa_set_term(self, value): + """TODOCFADOCS Set the CFA write status of the data to `False`. + + .. versionadded:: TODOCFAVER + + .. seealso:: `cfa_get_term`, `cfa_set_term` + + :Returns: + + `None` + + """ + if not value: + self._custom.pop("cfa_term", None) + + self._custom["cfa_term"] = bool(value) + def _clear_after_dask_update(self, clear=_ALL): """Remove components invalidated by updating the `dask` array. diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 3e1fd4a53a..9a5cee9aa7 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -1,4 +1,5 @@ import cfdm +import netCDF4 import numpy as np from packaging.version import Version @@ -157,7 +158,7 @@ def _create_data( uncompress_override=None, parent_ncvar=None, coord_ncvar=None, - cfa_term=None, + cfa_term=None, ): """Create data for a netCDF or CFA-netCDF variable. @@ -190,7 +191,7 @@ def _create_data( `Data` """ - if cfa_term is None and not self._is_cfa_variable(ncvar): + if not cfa_term and not self._is_cfa_variable(ncvar): # Create data for a normal netCDF variable data = super()._create_data( ncvar=ncvar, @@ -224,12 +225,17 @@ def _create_data( else: aggregated_data = None - cfa_array, kwargs = self._create_cfanetcdfarray( - ncvar, - unpacked_dtype=unpacked_dtype, - coord_ncvar=coord_ncvar, - non_standard_term=cfa_term, - ) + if cfa_term: + term, term_ncvar = tuple(cfa_term.items())[0] + cfa_array, kwargs = self._create_cfanetcdfarray_term( + ncvar, term, term_ncvar + ) + else: + cfa_array, kwargs = self._create_cfanetcdfarray( + ncvar, + unpacked_dtype=unpacked_dtype, + coord_ncvar=coord_ncvar, + ) data = self._create_Data( cfa_array, @@ -237,12 +243,14 @@ def _create_data( units=kwargs["units"], calendar=kwargs["calendar"], ) - + # Note: We don't cache elements from CFA variables # Set the CFA write status to True iff each non-aggregated # axis has exactly one dask storage chunk - if cfa_term is None: + if cfa_term: + data._cfa_set_term(True) + else: cfa_write = True for n, numblocks in zip( cfa_array.get_fragment_shape(), data.numblocks @@ -260,7 +268,7 @@ def _create_data( # Store the file substitutions data.cfa_set_file_substitutions(kwargs.get("substitutions")) - + return data def _is_cfa_variable(self, ncvar): @@ -280,10 +288,11 @@ def _is_cfa_variable(self, ncvar): """ g = self.read_vars - if not g["cfa"] or ncvar in g["external_variables"]: - return False - - return "aggregated_dimensions" in g["variable_attributes"][ncvar] + return ( + g["cfa"] + and ncvar in g["aggregated_data"] + and ncvar not in g["external_variables"] + ) def _create_Data( self, @@ -359,9 +368,13 @@ def _customize_read_vars(self): """ super()._customize_read_vars() g = self.read_vars + if not g["cfa"]: return + g["aggregated_data"] = {} + g["aggregation_instructions"] = {} + # ------------------------------------------------------------ # Still here? Then this is a CFA-netCDF file # ------------------------------------------------------------ @@ -399,9 +412,9 @@ def _customize_read_vars(self): parsed_aggregated_data = self._parse_aggregated_data( ncvar, attributes.get("aggregated_data") ) - for x in parsed_aggregated_data: - term, term_ncvar = tuple(x.items())[0] - term_ncvar = term_ncvar[0] + for term_ncvar in parsed_aggregated_data.values(): + # term, term_ncvar = tuple(x.items())[0] + # term_ncvar = term_ncvar[0] g["do_not_create_field"].add(term_ncvar) def _cache_data_elements(self, data, ncvar): @@ -487,7 +500,7 @@ def _create_cfanetcdfarray( ncvar, unpacked_dtype=False, coord_ncvar=None, - non_standard_term=None, + term=None, ): """Create a CFA-netCDF variable array. @@ -503,7 +516,7 @@ def _create_cfanetcdfarray( coord_ncvar: `str`, optional - non_standard_term: `str`, optional + term: `str`, optional The name of a non-standard aggregation instruction term from which to create the array. If set then *ncvar* must be the value of the non-standard term in @@ -529,44 +542,42 @@ def _create_cfanetcdfarray( return_kwargs_only=True, ) - # Specify a non-standardised term from which to create the - # data - if non_standard_term is not None: - kwargs["term"] = non_standard_term - - # Get rid of the incorrect shape of () - this will get set - # correctly by the CFAnetCDFArray instance. + # Get rid of the incorrect shape of (). This will end up + # getting set correctly by the CFANetCDFArray instance. kwargs.pop("shape", None) - # Add the aggregated_data attribute (that can be used by - # dask.base.tokenize) - aggregated_data = self.read_vars["variable_attributes"][ncvar].get( - "aggregated_data" - ) - kwargs["instructions"] = aggregated_data + aggregated_data = g["aggregated_data"][ncvar] - # Find URI substitutions that may be stored in the CFA file - # instruction variable's "substitutions" attribute + standardised_terms = ("location", "file", "address", "format") + + instructions = [] + aggregation_instructions = {} subs = {} - for x in self._parse_aggregated_data(ncvar, aggregated_data): - term, term_ncvar = tuple(x.items())[0] - if term != "file": + for t, term_ncvar in aggregated_data.items(): + if t not in standardised_terms: continue - term_ncvar = term_ncvar[0] - - subs = g["variable_attributes"][term_ncvar].get("substitutions") - if subs is None: - subs = {} - else: - # Convert the string "${base}: value" to the - # dictionary {"${base}": "value"} - subs = self.parse_x(term_ncvar, subs) - subs = { - key: value[0] for d in subs for key, value in d.items() - } - - break + aggregation_instructions[t] = g["aggregation_instructions"][ + term_ncvar + ] + instructions.append(f"{t}: {ncvar}") + + if t == "file": + # Find URI substitutions that may be stored in the CFA + # file instruction variable's "substitutions" + # attribute + subs = g["variable_attributes"][term_ncvar].get( + "substitutions" + ) + if subs is None: + subs = {} + else: + # Convert the string "${base}: value" to the + # dictionary {"${base}": "value"} + subs = self.parse_x(term_ncvar, subs) + subs = { + key: value[0] for d in subs for key, value in d.items() + } # Apply user-defined substitutions, which take precedence over # those defined in the file. @@ -574,6 +585,72 @@ def _create_cfanetcdfarray( if subs: kwargs["substitutions"] = subs + kwargs["x"] = aggregation_instructions + kwargs["instructions"] = " ".join(sorted(instructions)) + + # Use the kwargs to create a CFANetCDFArray instance + array = self.implementation.initialise_CFANetCDFArray(**kwargs) + + return array, kwargs + + def _create_cfanetcdfarray_term( + self, + parent_ncvar, + term, + ncvar, + ): + """Create a CFA-netCDF variable array. + + .. versionadded:: 3.14.0 + + :Parameters: + + parent_ncvar: `str` + The name of the CFA-netCDF aggregated variable. See + the *term* parameter. + + term: `str`, optional + The name of a non-standard aggregation instruction + term from which to create the array. If set then + *ncvar* must be the value of the non-standard term in + the ``aggregation_data`` attribute. + + .. versionadded:: TODOCFAVER + + ncvar: `str` + The name of the CFA-netCDF aggregated variable. See + the *term* parameter. + + :Returns: + + (`CFANetCDFArray`, `dict`) + The new `NetCDFArray` instance and dictionary of the + kwargs used to create it. + + """ + g = self.read_vars + + # Get the kwargs needed to instantiate a general NetCDFArray + # instance + kwargs = self._create_netcdfarray( + ncvar, + return_kwargs_only=True, + ) + + instructions = [] + aggregation_instructions = {} + for t, term_ncvar in g["aggregated_data"][parent_ncvar].items(): + if t in ("location", term): + aggregation_instructions[t] = g["aggregation_instructions"][ + term_ncvar + ] + instructions.append(f"{t}: {ncvar}") + + kwargs["term"] = term + kwargs["dtype"] = aggregation_instructions[term].dtype + kwargs["x"] = aggregation_instructions + kwargs["instructions"] = " ".join(sorted(instructions)) + # Use the kwargs to create a CFANetCDFArray instance array = self.implementation.initialise_CFANetCDFArray(**kwargs) @@ -644,33 +721,6 @@ def _parse_chunks(self, ncvar): return chunks - def _parse_aggregated_data(self, ncvar, aggregated_data): - """Parse a CFA-netCDF aggregated_data attribute. - - .. versionadded:: TODOCFAVER - - :Parameters: - - ncvar: `str` - The netCDF variable name. - - aggregated_data: `str` or `None` - The CFA-netCDF ``aggregated_data`` attribute. - - :Returns: - - `list` - - """ - if not aggregated_data: - return [] - - return self._parse_x( - ncvar, - aggregated_data, - keys_are_variables=True, - ) - def _customize_field_ancillaries(self, parent_ncvar, f): """Create field ancillary constructs from CFA terms. @@ -720,27 +770,20 @@ def _customize_field_ancillaries(self, parent_ncvar, f): # ------------------------------------------------------------ g = self.read_vars - out = {} - - attributes = g["variable_attributes"][parent_ncvar] - parsed_aggregated_data = self._parse_aggregated_data( - parent_ncvar, attributes.get("aggregated_data") - ) standardised_terms = ("location", "file", "address", "format") - for x in parsed_aggregated_data: - term, ncvar = tuple(x.items())[0] + + out = {} + for term, term_ncvar in g["aggregated_data"][parent_ncvar].items(): if term in standardised_terms: continue - ncvar = ncvar[0] - # Still here? Then we've got a non-standard aggregation # term from which we can create a field # ancillary construct. anc = self.implementation.initialise_FieldAncillary() self.implementation.set_properties( - anc, g["variable_attributes"][ncvar] + anc, g["variable_attributes"][term_ncvar] ) anc.set_property("long_name", term) @@ -749,13 +792,11 @@ def _customize_field_ancillaries(self, parent_ncvar, f): # written to disk as a non-standard CFA term. anc.id = term - data = self._create_data(parent_ncvar, anc, cfa_term=term) + data = self._create_data( + parent_ncvar, anc, cfa_term={term: term_ncvar} + ) self.implementation.set_data(anc, data, copy=False) - - self.implementation.nc_set_variable(anc, ncvar) - - # Set the CFA term status - anc._custom["cfa_term"] = True + self.implementation.nc_set_variable(anc, term_ncvar) key = self.implementation.set_field_ancillary( f, @@ -763,6 +804,73 @@ def _customize_field_ancillaries(self, parent_ncvar, f): axes=self.implementation.get_field_data_axes(f), copy=False, ) - out[ncvar] = key + out[term_ncvar] = key return out + + def _parse_aggregated_data(self, ncvar, aggregated_data): + """Parse a CFA-netCDF aggregated_data attribute. + + .. versionadded:: TODOCFAVER + + :Parameters: + + ncvar: `str` + The netCDF variable name. + + aggregated_data: `str` or `None` + The CFA-netCDF ``aggregated_data`` attribute. + + :Returns: + + `dict` + + """ + if not aggregated_data: + return {} + + g = self.read_vars + aggregation_instructions = g["aggregation_instructions"] + + out = {} + for x in self._parse_x( + ncvar, + aggregated_data, + keys_are_variables=True, + ): + term, term_ncvar = tuple(x.items())[0] + term_ncvar = term_ncvar[0] + out[term] = term_ncvar + + if term_ncvar not in aggregation_instructions: + array = self._conform_array(g["variables"][term_ncvar][...]) + aggregation_instructions[term_ncvar] = array + + g["aggregated_data"][ncvar] = out + return out + + def _conform_array(self, array): + """TODOCFADOCS""" + if isinstance(array, str): + # string + return np.array(array, dtype=f"S{len(array)}").astype("U") + + kind = array.dtype.kind + if kind == "O": + # string + return array.astype("U") + + if kind in "SU": + # char + if kind == "U": + array = array.astype("S") + + array = netCDF4.chartostring(array) + shape = array.shape + array = np.array([x.rstrip() for x in array.flat], dtype="S") + array = np.reshape(array, shape) + array = np.ma.masked_where(array == b"", array) + return array.astype("U") + + # number + return array diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index b68cd9e582..1f8bc2d4f3 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -1,3 +1,5 @@ +from os import remove + import cfdm import dask.array as da import numpy as np @@ -20,7 +22,7 @@ def __new__(cls, *args, **kwargs): instance._NetCDFRead = NetCDFRead return instance - def _use_cfa(self, cfvar, construct_type): + def _write_as_cfa(self, cfvar, construct_type): """Whether or not to write as a CFA variable. .. versionadded:: 3.0.0 @@ -55,23 +57,28 @@ def _use_cfa(self, cfvar, construct_type): if data is None: return False - if not data.cfa_get_write(): - return False - - for ctype, ndim in g["cfa_options"].get("constructs", {}).items(): + cfa_options = g["cfa_options"] + for ctype, ndim in cfa_options.get("constructs", {}).items(): # Write as CFA if it has an appropriate construct type ... if ctype in ("all", construct_type): - # ... and then only if it satisfies the number of - # dimenions criterion - ok = ndim is None or ndim == data.ndim -TODO if ( - ok - and cfa_options.get("strict", True) - and not data.cfa_get_write() - ): - return False -TODO - return ok + # ... and then only if it satisfies the + # number-of-dimenions criterion and the data is + # flagged as OK. + if ndim is None or ndim == data.ndim: + cfa_get_write = data.cfa_get_write() + if not cfa_get_write and cfa_options["strict"]: + if g["mode"] == "w": + remove(g["filename"]) + + raise ValueError( + f"Can't write {cfvar!r} as a CFA-netCDF " + "aggregation variable. Consider setting " + "cfa_options={'strict': False}" + ) + + return cfa_get_write + + break return False @@ -98,7 +105,7 @@ def _customize_createVariable(self, cfvar, construct_type, kwargs): cfvar, construct_type, kwargs ) - if self._use_cfa(cfvar, construct_type): + if self._write_as_cfa(cfvar, construct_type): kwargs["dimensions"] = () kwargs["chunksizes"] = None @@ -149,11 +156,11 @@ def _write_data( """ g = self.write_vars - if self._use_cfa(cfvar, construct_type): + if self._write_as_cfa(cfvar, construct_type): # -------------------------------------------------------- # Write the data as CFA aggregated data # -------------------------------------------------------- - self._write_cfa_data(ncvar, ncdimensions, data, cfvar) + self._create_cfa_data(ncvar, ncdimensions, data, cfvar) return # ------------------------------------------------------------ @@ -345,7 +352,7 @@ def _change_reference_datetime(self, coord): else: return coord2 - def _write_cfa_data(self, ncvar, ncdimensions, data, cfvar): + def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar): """Write a CFA variable to the netCDF file. Any CFA private variables required will be autmatically created @@ -624,7 +631,7 @@ def _write_field_ancillary(self, f, key, anc): empty string is returned. """ - if anc._custom.get("cfa_term", False): + if anc.data.cfa_get_term(): # This field ancillary construct is to be written as a # non-standard CFA term belonging to the parent field, or # else not at all. @@ -693,7 +700,7 @@ def _cfa_write_non_standard_terms( for key, field_anc in self.implementation.get_field_ancillaries( field ).items(): - if not field_anc._custom.get("cfa_term", False): + if not field_anc.data.cfa_get_term(): continue data = self.implementation.get_data(field_anc, None) @@ -823,13 +830,15 @@ def _ggg(self, data, cfvar): if len(a) != 1: if a: raise ValueError( - f"Can't write CFA variable from {cfvar!r} when a " - "dask storage chunk spans two or more fragment files" + f"Can't write CFA variable from {cfvar!r} when the " + f"dask storage chunk defined by indices {indices} " + "spans two or more external files" ) raise ValueError( - f"Can't write CFA variable from {cfvar!r} when a " - "dask storage chunk spans zero fragment files" + f"Can't write CFA variable from {cfvar!r} when the " + f"dask storage chunk defined by indices {indices} spans " + "zero external files" ) filenames, addresses, formats = a.pop() diff --git a/cf/read_write/write.py b/cf/read_write/write.py index 20af3e93a2..def08804a3 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -578,7 +578,7 @@ def write( cfa: `bool` or `dict`, otional If True or a (possibly empty) dictionary then write the - constructs as CFA-netCDF aggregated variables, where + constructs as CFA-netCDF aggregation variables, where possible and where requested. If *cfa* is a dictionary then it is used to configure the @@ -586,88 +586,82 @@ def write( enabled are ``{'constructs': 'field', 'absolute_paths': True, 'strict': True, 'substitutions': {}}``, and the dictionary may have any subset of the following key/value - pairs: + pairs to override thes: - * ``'constructs'`` (`dict` or (sequence of) `str`) + * ``'constructs'``: `dict` or (sequence of) `str` The types of construct to be written as CFA-netCDF - aggregated variables. By default only field constructs - are written in this way. The types are given as a - (sequence of) `str`, which may take any of the values - allowed by the *omit_data* parameter. Alternatively, the - same types may be given as keys to a `dict`, whose - values specify the number of dimensions that the - construct must also have if it is to be written as - CFA-netCDF aggregated variable. A value of `None` means - no restriction on the number of dimensions, which is - equivalent to a value of ``cf.ge(0)``. + aggregation variables. By default only field constructs + are written as CFA-netCDF aggregation variables. + + The types may be given as a (sequence of) `str`, which + may take any of the values allowed by the *omit_data* + parameter. Alternatively, the same types may be given as + keys to a `dict`, whose values specify the number of + dimensions that a construct must also have if it is to + be written as CFA-netCDF aggregation variable. A value + of `None` means no restriction on the number of + dimensions, which is equivalent to a value of + ``cf.ge(0)``. Note that size 1 data arrays are never written as - CFA-netCDF aggregated variables, regardless of the + CFA-netCDF aggregation variables, regardless of the whether or not this has been requested. - *Parameter example:* + *Example:* Equivalent ways to only write cell measure constructs - as CFA-netCDF variables: ``'cell_measure``, - ``['cell_measure']``, and ``{'cell_measure': None}``. + as CFA-netCDF aggregation variables: + ``'cell_measure``, ``['cell_measure']``, + ``{'cell_measure': None}``, ``{'cell_measure': + cf.ge(0)}`` - *Parameter example:* + *Example:* Equivalent ways to only write field and auxiliary - coordinate constructs as CFA-netCDF variables: - ``('field', 'auxiliary_coordinate')`` and ``{'field': - None, 'auxiliary_coordinate': None}``. + coordinate constructs as CFA-netCDF aggregation + variables: ``('field', 'auxiliary_coordinate')`` and + ``{'field': None, 'auxiliary_coordinate': None}``. - *Parameter example:* - Only write two dimensional auxiliary coordinate - constructs as CFA-netCDF variables: + *Example:* + Only write two-dimensional auxiliary coordinate + constructs as CFA-netCDF aggregation variables: ``{'auxiliary_coordinate': 2}}``. - *Parameter example:* - Only write field constructs, and auxiliary coordinate - constructs with two or more dimensions as CFA-netCDF - variables: ``{'field': None, 'auxiliary_coordinate': - cf.ge(2)}}``. + *Example:* + Only write auxiliary coordinate constructs with two or + more dimensions, and all field constructs as + CFA-netCDF variables: ``{'field': None, + 'auxiliary_coordinate': cf.ge(2)}}``. - * ``'absolute_paths'`` (`bool`) + * ``'absolute_paths'``: `bool` - How to write fragment file names. Set to ``'absolute'`` - (the default) for them to be written as fully qualified - URIs, or else set to ``'relative'`` for them to be - relative to the CFA-netCDF file being created. Note that - in both cases, fragment files defined by fully qualified - URLs will always be written as such. + How to write fragment file names. Set to True (the + default) for them to be written as fully qualified URIs, + or else set to False for them to be written as local + paths relative to the location of the CFA-netCDF file + being created. - * ``'absolute_paths'`` (`bool`) + * ``'strict'``:`bool` - How to write fragment file names. Set to ``'absolute'`` - (the default) for them to be written as fully qualified - URIs, or else set to ``'relative'`` for them to be - relative to the CFA-netCDF file being created. Note that - in both cases, fragment files defined by fully qualified - URLs will always be written as such. + If True (the default) then an exception is raised if it + is not possible to create a CFA aggregation variable + from data identified by the ``'constructs'`` option. If + False then a normal CF-netCDF variable in this case. - * ``'strict'`` (`bool`) - - If True (the default) then raise an exception if it is - not possible to write a data identified by the - ``'constructs'`` key as a CFA aggregated variable. If - False then a warning is logged, and the is written as a - normal netCDF variable. - - * ``'substitutions'`` (`dict`) + * ``'substitutions'``: `dict` A dictionary whose key/value pairs define text substitutions to be applied to the fragment file - URIs. Each key must be a string of one or more letters, + names. Each key must be a string of one or more letters, digits, and underscores. These substitutions are used in conjunction with, and take precendence over, any that - are also defined on individual constructs. + are also defined on individual constructs (see + `cf.Data.cfa_set_file_substitutions` for details). Substitutions are stored in the output file by the ``substitutions`` attribute of the ``file`` CFA aggregation instruction variable. - *Parameter example:* + *Example:* ``{'base': 'file:///data/'}}`` .. versionadded:: TODOCFAVER diff --git a/cf/test/test_CFA.py b/cf/test/test_CFA.py new file mode 100644 index 0000000000..213c17f6a2 --- /dev/null +++ b/cf/test/test_CFA.py @@ -0,0 +1,167 @@ +import atexit +import datetime +import faulthandler +import os +import tempfile +import unittest + +faulthandler.enable() # to debug seg faults and timeouts + +import cf + +n_tmpfiles = 5 +tmpfiles = [ + tempfile.mkstemp("_test_CFA.nc", dir=os.getcwd())[1] + for i in range(n_tmpfiles) +] +( + tmpfile1, + tmpfile2, + tmpfile3, + tmpfile4, + tmpfile5, +) = tmpfiles + + +def _remove_tmpfiles(): + """Try to remove defined temporary files by deleting their paths.""" + for f in tmpfiles: + try: + os.remove(f) + except OSError: + pass + + +atexit.register(_remove_tmpfiles) + + +class CFATest(unittest.TestCase): + netcdf3_fmts = [ + "NETCDF3_CLASSIC", + "NETCDF3_64BIT", + "NETCDF3_64BIT_OFFSET", + "NETCDF3_64BIT_DATA", + ] + netcdf4_fmts = ["NETCDF4", "NETCDF4_CLASSIC"] + netcdf_fmts = netcdf3_fmts + netcdf4_fmts + + def test_CFA_fmt(self): + f = cf.example_field(0) + cf.write(f, tmpfile1) + f = cf.read(tmpfile1)[0] + + for fmt in ("NETCDF4",): + cf.write(f, tmpfile2, fmt=fmt, cfa=True) + g = cf.read(tmpfile2) + self.assertEqual(len(g), 1) + g = g[0] + + self.assertTrue(f.equals(g)) + + def test_CFA(self): + f = cf.example_field(0) + + cf.write(f[:2], tmpfile1) + cf.write(f[2:], tmpfile2) + + a = cf.read([tmpfile1, tmpfile2]) + self.assertEqual(len(a), 1) + a = a[0] + + nc_file = tmpfile3 + cfa_file = tmpfile4 + cf.write(a, nc_file) + cf.write(a, cfa_file, cfa=True) + + n = cf.read(nc_file) + c = cf.read(cfa_file) + self.assertEqual(len(n), 1) + self.assertEqual(len(c), 1) + + n = n[0] + c = c[0] + self.assertTrue(c.equals(f)) + self.assertTrue(c.equals(n)) + + def test_CFA_term_strict(self): + f = cf.example_field(0) + + # By default, can't write as CF-netCDF those variables + # selected for CFA treatment, but which aren't suitable. + with self.assertRaises(ValueError): + cf.write(f, tmpfile1, cfa=True) + + # The previous line should have deleted the file + self.assertFalse(os.path.exists(tmpfile1)) + + cf.write(f, tmpfile1, cfa={"strict": False}) + g = cf.read(tmpfile1) + self.assertEqual(len(g), 1) + self.assertTrue(g[0].equals(f)) + + cf.write(g, tmpfile2, cfa={"strict": True}) + g = cf.read(tmpfile2) + self.assertEqual(len(g), 1) + self.assertTrue(g[0].equals(f)) + + def test_CFA_field_ancillaries(self): + import cf + + f = cf.example_field(0) + self.assertFalse(f.field_ancillaries()) + + tmpfile1 = "delme1.nc" + tmpfile2 = "delme2.nc" + tmpfile3 = "delme3.nc" + tmpfile4 = "delme4.nc" + + a = f[:2] + b = f[2:] + a.set_property("foo", "bar_a") + b.set_property("foo", "bar_b") + cf.write(a, tmpfile1) + cf.write(b, tmpfile2) + + c = cf.read( + [tmpfile1, tmpfile2], aggregate={"field_ancillaries": "foo"} + ) + self.assertEqual(len(c), 1) + c = c[0] + self.assertEqual(len(c.field_ancillaries()), 1) + anc = c.field_ancillary() + self.assertTrue(anc.data.cfa_get_term()) + self.assertFalse(anc.data.cfa_get_write()) + + cf.write(c, tmpfile3, cfa=False) + c2 = cf.read(tmpfile3) + self.assertEqual(len(c2), 1) + self.assertFalse(c2[0].field_ancillaries()) + + cf.write(c, tmpfile4, cfa=True) + d = cf.read(tmpfile4) + self.assertEqual(len(d), 1) + d = d[0] + + self.assertEqual(len(d.field_ancillaries()), 1) + anc = d.field_ancillary() + self.assertTrue(anc.data.cfa_get_term()) + self.assertFalse(anc.data.cfa_get_write()) + self.assertTrue(d.equals(c)) + + cf.write(d, tmpfile5, cfa=False) + e = cf.read(tmpfile5) + self.assertEqual(len(e), 1) + self.assertFalse(e[0].field_ancillaries()) + + cf.write(d, tmpfile5, cfa=True) + e = cf.read(tmpfile5) + self.assertEqual(len(e), 1) + e = e[0] + self.assertTrue(e.equals(d)) + + +if __name__ == "__main__": + print("Run date:", datetime.datetime.now()) + cf.environment() + print() + unittest.main(verbosity=2) diff --git a/cf/test/test_aggregate.py b/cf/test/test_aggregate.py index e80a98878d..975458be38 100644 --- a/cf/test/test_aggregate.py +++ b/cf/test/test_aggregate.py @@ -334,6 +334,25 @@ def test_aggregate_relaxed_units(self): self.assertEqual(i.Units.__dict__, bad_units.__dict__) self.assertTrue((i.array == f.array).all()) + def test_aggregate_field_ancillaries(self): + f = cf.example_field(0) + self.assertFalse(f.field_ancillaries()) + + a = f[:2] + b = f[2:] + a.set_property("foo", "bar_a") + b.set_property("foo", "bar_b") + + c = cf.aggregate([a, b], field_ancillaries="foo") + self.assertEqual(len(c), 1) + c = c[0] + self.assertTrue(len(c.field_ancillaries()), 1) + + anc = c.field_ancillary() + self.assertEqual(anc.shape, c.shape) + self.assertTrue((anc[:2] == "bar_a").all()) + self.assertTrue((anc[2:] == "bar_b").all()) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) From 1d9d309b1d13431b8704682a04ea190f0e707bfd Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 8 Mar 2023 22:05:46 +0000 Subject: [PATCH 052/141] dev --- cf/data/array/umarray.py | 8 ++ cf/read_write/netcdf/netcdfread.py | 6 +- cf/read_write/um/umread.py | 116 ++++++++++++++++++++--------- cf/test/file2.nc | Bin 0 -> 24533 bytes cf/test/test_CFA.py | 9 ++- 5 files changed, 99 insertions(+), 40 deletions(-) create mode 100644 cf/test/file2.nc diff --git a/cf/data/array/umarray.py b/cf/data/array/umarray.py index 89983e1264..2a05e9ce2d 100644 --- a/cf/data/array/umarray.py +++ b/cf/data/array/umarray.py @@ -246,6 +246,14 @@ def __getitem__(self, indices): # Return the numpy array return array + def __str__(self): + """Called by the `str` built-in function. + + x.__str__() <==> str(x) + + """ + return f"{self.get_filename(None)}, {self.get_address()}" + def _get_rec(self, f, header_offset): """Get a container for a record. diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 9a5cee9aa7..f000dc8788 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -158,7 +158,7 @@ def _create_data( uncompress_override=None, parent_ncvar=None, coord_ncvar=None, - cfa_term=None, + cfa_term=None, ): """Create data for a netCDF or CFA-netCDF variable. @@ -243,7 +243,7 @@ def _create_data( units=kwargs["units"], calendar=kwargs["calendar"], ) - + # Note: We don't cache elements from CFA variables # Set the CFA write status to True iff each non-aggregated @@ -268,7 +268,7 @@ def _create_data( # Store the file substitutions data.cfa_set_file_substitutions(kwargs.get("substitutions")) - + return data def _is_cfa_variable(self, ncvar): diff --git a/cf/read_write/um/umread.py b/cf/read_write/um/umread.py index 06ce490b2f..e25046fa0a 100644 --- a/cf/read_write/um/umread.py +++ b/cf/read_write/um/umread.py @@ -1068,12 +1068,13 @@ def __init__( for cm in cell_methods: self.implementation.set_cell_method(field, cm) - # Check for decreasing axes that aren't decreasing - down_axes = self.down_axes - logger.info(f"down_axes = {down_axes}") # pragma: no cover + # # Check for decreasing axes that aren't decreasing + # down_axes = self.down_axes + logger.info(f"down_axes = {self.down_axes}") # pragma: no cover - if down_axes: - field.flip(down_axes, inplace=True) + # print (down_axes ) + # if down_axes: + # field.flip(down_axes, inplace=True) # Force cyclic X axis for particular values of LBHEM if xkey is not None and int_hdr[lbhem] in (0, 1, 2, 4): @@ -1126,6 +1127,23 @@ def __str__(self): return "\n".join(out) + def _reorder_z_axis(self, indices, z_axis, pmaxes): + """TODOCFADOCS""" + indices_new = [] + pos = pmaxes.index(z_axis) + aaa0 = next(indices) + indices2 = [aaa0] + for aaa in indices: + if aaa[pos] > aaa0[pos]: + indices2.append(aaa) + else: + indices_new.extend(indices2[::-1]) + aaa0 = aaa + indices2 = [aaa0] + + indices_new.extend(indices2[::-1]) + return indices_new + def atmosphere_hybrid_height_coordinate(self, axiscode): """`atmosphere_hybrid_height_coordinate` when not an array axis. @@ -1635,10 +1653,12 @@ def coord_data( """ if array is not None: array = Data(array, units=units, fill_value=fill_value) + array._cfa_set_write(True) self.implementation.set_data(c, array, copy=False) if bounds is not None: bounds_data = Data(bounds, units=units, fill_value=fill_value) + bounds_data._cfa_set_write(True) bounds = self.implementation.initialise_Bounds() self.implementation.set_data(bounds, bounds_data, copy=False) self.implementation.set_bounds(c, bounds, copy=False) @@ -1676,7 +1696,8 @@ def coord_positive(self, c, axiscode, domain_axis_key): :Parameters: - c: Coordinate construct + c: `Coordinate` + A 1-d coordinate construct axiscode: `int` @@ -1692,6 +1713,7 @@ def coord_positive(self, c, axiscode, domain_axis_key): c.positive = positive if positive == "down" and axiscode != 4: self.down_axes.add(domain_axis_key) + c.flip(inplace=True) return c @@ -1861,6 +1883,9 @@ def create_data(self): name = (UMArray().__class__.__name__ + "-" + token,) dsk = {} full_slice = Ellipsis + klass_name = UMArray().__class__.__name__ + + fmt = self.fmt if len(recs) == 1: # -------------------------------------------------------- @@ -1879,19 +1904,19 @@ def create_data(self): subarray = UMArray( filename=filename, + address=rec.hdr_offset, shape=yx_shape, dtype=data_type_in_file(rec), - address=rec.hdr_offset, - # data_offset=rec.data_offset, - # disk_length=rec.disk_length, - fmt=self.fmt, + fmt=fmt, word_size=self.word_size, byte_ordering=self.byte_ordering, units=units, calendar=calendar, ) - dsk[name + (0, 0)] = (getter, subarray, full_slice, False, False) + key = f"{klass_name}-{tokenize(subarray)}" + dsk[key] = subarray + dsk[name + (0, 0)] = (getter, key, full_slice, False, False) dtype = data_type_in_file(rec) chunks = normalize_chunks((-1, -1), shape=data_shape, dtype=dtype) @@ -1908,18 +1933,23 @@ def create_data(self): # ---------------------------------------------------- # 1-d partition matrix # ---------------------------------------------------- + z_axis = _axis[self.z_axis] if nz > 1: - pmaxes = [_axis[self.z_axis]] + pmaxes = [z_axis] data_shape = (nz, LBROW, LBNPT) else: pmaxes = [_axis["t"]] data_shape = (nt, LBROW, LBNPT) - fmt = self.fmt word_size = self.word_size byte_ordering = self.byte_ordering - for i, rec in enumerate(recs): + indices = ((i, rec) for i, rec in enumerate(recs)) + if z_axis in self.down_axes: + indices = self._reorder_z_axis(indices, z_axis, pmaxes) + + # for i, rec in enumerate(recs): + for i, rec in indices: # Find the data type of the array in the file file_data_type = data_type_in_file(rec) file_data_types.add(file_data_type) @@ -1928,11 +1958,9 @@ def create_data(self): subarray = UMArray( filename=filename, + address=rec.hdr_offset, shape=shape, dtype=file_data_type, - address=rec.hdr_offset, - # data_offset=rec.data_offset, - # disk_length=rec.disk_length, fmt=fmt, word_size=word_size, byte_ordering=byte_ordering, @@ -1940,9 +1968,11 @@ def create_data(self): calendar=calendar, ) + key = f"{klass_name}-{tokenize(subarray)}" + dsk[key] = subarray dsk[name + (i, 0, 0)] = ( getter, - subarray, + key, full_slice, False, False, @@ -1956,17 +1986,41 @@ def create_data(self): # ---------------------------------------------------- # 2-d partition matrix # ---------------------------------------------------- - pmaxes = [_axis["t"], _axis[self.z_axis]] + z_axis = _axis[self.z_axis] + pmaxes = [_axis["t"], z_axis] + data_shape = (nt, nz, LBROW, LBNPT) - fmt = self.fmt word_size = self.word_size byte_ordering = self.byte_ordering - for i, rec in enumerate(recs): + indices = ( + divmod(i, nz) + (rec,) for i, rec in enumerate(recs) + ) + if z_axis in self.down_axes: + indices = self._reorder_z_axis(indices, z_axis, pmaxes) +# +# indices_new = [] +# pos = pmaxes.index(z_axis) +# aaa0 = next(indices) +# indices2 = [aaa0] +# for aaa in indices: +# if aaa[pos] > aaa0[pos]: +# indices2.append(aaa) +# else: +# indices_new.extend(indices2[::-1]) +# aaa0 = aaa +# indices2 = [aaa0] +# +# indices_new.extend(indices2[::-1]) +# indices = indices_new + + for t, z, rec in indices: + # for i, rec in enumerate(recs): # Find T and Z axis indices - t, z = divmod(i, nz) + # t, z = divmod(i, nz) + #print (t, z) # Find the data type of the array in the file file_data_type = data_type_in_file(rec) file_data_types.add(file_data_type) @@ -1975,11 +2029,9 @@ def create_data(self): subarray = UMArray( filename=filename, + address=rec.hdr_offset, shape=shape, dtype=file_data_type, - address=rec.hdr_offset, - # data_offset=rec.data_offset, - # disk_length=rec.disk_length, fmt=fmt, word_size=word_size, byte_ordering=byte_ordering, @@ -1987,9 +2039,11 @@ def create_data(self): calendar=calendar, ) + key = f"{klass_name}-{tokenize(subarray)}" + dsk[key] = subarray dsk[name + (t, z, 0, 0)] = ( getter, - subarray, + key, full_slice, False, False, @@ -2012,6 +2066,7 @@ def create_data(self): # Create the Data object data = Data(array, units=um_Units, fill_value=fill_value) + data._cfa_set_write(True) self.data = data self.data_axes = data_axes @@ -3115,12 +3170,6 @@ def z_coordinate(self, axiscode): if _coord_positive.get(axiscode, None) == "down": bounds0, bounds1 = bounds1, bounds0 - # key = (axiscode, array, bounds0, bounds1) - # dc = _cached_z_coordinate.get(key, None) - - # if dc is not None: - # copy = True - # else: copy = False array = np.array(array, dtype=float) bounds0 = np.array(bounds0, dtype=float) @@ -3207,7 +3256,7 @@ def z_reference_coordinate(self, axiscode): dc = self.coord_names(dc, axiscode) if not dc.get("positive", True): # ppp - dc.flip(i=True) + dc.flip(inplace=True) _cached_z_reference_coordinate[key] = dc copy = False @@ -3593,7 +3642,6 @@ def file_open(self, filename): fmt=g.get("fmt"), ) - """ Problems: diff --git a/cf/test/file2.nc b/cf/test/file2.nc new file mode 100644 index 0000000000000000000000000000000000000000..8cf7800912960c73c0b2e859a1849d8ba04022b9 GIT binary patch literal 24533 zcmeG^33yaR(mlx#Vi-bzKtdq!I7AM?1j8YU(~uA{6O2d{&oxZQKt_{Em`o5{1QpL! z*Zamr@jiZC&)=K4uCBW(qT-1n$RYfoqW&y+ty|Sq^)h)mAdvmV?@z-^O?Ov!SJ(CW zb=P~B7Ubo3PRdA9m=Y6}Tcwzv^5+*$@0C!AllVFZ)nlnZ~Q7&5Moqte*~l*drvQpqY2{*@HvPK#7k2P4r?um&)?!!;Hz z(*eMF!AMQ0Dm)>3VK^9_RW`dKR4PR|Bgc%)URYi>dvr`;_Q>q4k)xCcNOQqIj~OJl zDm*V3jzT{;1Pzh^PzK4(ACWzBtY?T)NI8;~sH!7XX9QX3*vadFg;SogdcL#ZRU zd5=CITczSVD-od+da|%r7Z8VY$elW|xES)tRFs8Kk>@*Ra#6AG*rJl++=){sD^DJA ztAlA(H1U|pfNWX~ifI*yR#w$imj@%klJW(mkx*Glc`!7mJX$iVswz?z3J0P=z+RK^ z&8%Z{3krqS_-1KUV{jDb3~1(9(;$&@t24Rq`5`CYj-wFvC|1CQTZhvTlRkP-6M^?`MyY zmZ;JeL#44e{e$GJDbnrG#sVjrby%`zB%H({nU!6((ZYKo#5bs|`x8)|5>U z_V5(rroT~I&-n~g5J*!9sDups;c`o(j-^$#;j)@hLC|oMSflZGFN0LP>luu0~R=f+`N3NEL0f`qi?WEYok`UDq3GPFAxbel;J3KTk)qwwhV_r$I3;b zutyLX%7~J0oz3pzW~bl~kVK-*+X2oW=h+2pG+ou(=2Tzt@k;fAI<(z1xKK;Pvj}wp z4b`kh9S{#L(*S^BdJN~&*ZuM)%22fMDI+zdR&i%Y5qiS5qbL=qh^43pbng3|SNJQ(6*fS1H3tFZbcQ&i4vbr`JEGwyr2Et{5NLfiZPzjz2ouXl= z8C9u3MKlzxEem=I(LKoh$+Wp5!`q@*0x3zOhW|Ds`1F8Cb@-NxVC zp9%y|pp+lvz?Xjb9$@z)<((Y((vQbcF8G}tE{rz?c5fo-xUM1)uM2cDe9}R~wDW z2U9n8xyXgpt5NxoX7GnP%4J-17np=jDIZS7jmi&olnXs09OaW4k(cv!KT>{}17G-) z!zV+f5;DEJG@MZCb-*s z+LeFVIpnl>{qU6aXQw{=i&x-X)f19FxO2Bx;4L>}`o=4;--X+bd%Eu?ufU&kBJ&&I*A{2iLna_45;tx0A=&&hQhNmm<7Rc--wg-cHW{w1WEWUbh{n zzi?Yu8&bQJI)QJ#fyq-RumIUJ<)Au&F9X-E{7Z*Afyk}u32}7-=b9fsuMjxfq<^ju zh@&6(^j9mW|88}rc~aLpBaSPRIp3+yu=&D{btEr$-S2^l^=B)l`}-A!to7fnp!#FA zf#=T9OaE&9IkMZYYp+dr&-j;Lzx0ngCO;KlNacxzdh6iq#D_eE`fr6}FP)fDs1L`~ zy-;^sQumKLdKBuiBR@O#inKy~#f@Po@@qr)tA72~uHp@MzwW1YtNhxKv)ZqdM?7}h z*B|)x>CgH+aclhg6zt$rzka9>{_^X^XFMOh_7bM&kAD5bA}sLh_vQm7H~F=9G_+jm z*GD|@+PoK+`Sm@w!UUH4wVD6je*M2cub#j49zW4D&99Nn$us;q5eGQcPxGGX*Vt=% znV;yN7y~ANSEb{@bUGzCP#EaY*2c zK8@Yn|B_E5xw~HW>3^a+zTzYPRQoit!9CBXNnd<4kHtQc^Aev%aou~7Ps8A==U?h0 zxnAbeMs2TPdd4w*6MXv8i7>HAK8@lXKE+3JF7%Op75VhPKs@aHC+L)uP~$v$!`+~8SzR@%!G?oV$v*m| zk$$IUki0yZ+Sq%XrKcMMcIg+Fq2N1`zWnGRIi1;61D7uDd2*^Q#Q_S^v_T)2!G0{0 z?4z^Abh}8OpXx|_S=5>Oy{oG}IN`kOb1zFJdx)fwew~`3cY_(;nwLp>m1a?YSH$U0 z2VsLm)~n9?EJI*dJrK>{d7jtOG^#%*gY0-_rZ#@0zeRR7IG*%uWg_cOGV`IEHsowi zCH;9djqNpq@iIv-hFd&dy#5x%pK{Vcq@O#J^-ac|bYuSZAbb2OP2cw_IHH#_Xq@XZ zbrCWy$I_X`k+^vt2eJKj(Y=A|3Fmd={(5M?VR*X!>3&eS&ofxgnKaJ{7R_(Ho9auG z^i*(rGdFe7<>&yO>_&PwzX#EMRJulYKu3CL3fb?L zZY;MRR6i?SSD{1g-<$ZnE|ceDk-t)^G#Cwl%bpXctOQsVtiWurx+YXn6+WBl=&y>F zn;)Sn)8NcNbu~6LDOD33$#4+ELm7?-7<(Y~K5tFSPM)VgsZREp%EBpQrV z)+jnku>iW@!W0;91+3EXR;g7LshTt1s;Q044$KOc$O#V~h9;@4ClYo~X?7ju54^ui zWFQ&uGbVq=JY-tK?-C^Z(yv2PMPg)KuliT?C*#lee7>?06zyMdCB7g;I-_HDK z_4?(p<7GS(ZJXzhw`}Zz5SKEaxT!k+6Bu;$9<(~`hT?=FvUCHqqAV8Yxex_rrX|83{ zyxr6)BQg|i>mEFC^hM9kUL(i9a8AyrC%u>=51g9d0}Vy8)P_S*F?Bpn6tc^Lb0WcD zjRzVxeZ-0k5#$9Wr4WnoW5L@;#7^i{o{*g8d1lDDGltMK!IY>Z{z4c4R0t| zI5;zc?3ejF3mU1@$9-pFaO8Y{5ufNy!%9dVI6-Ooh#QYcWr}k!)Tbgqr{m~56Vxv! znSRt5!C6lE1O4wGuAOjF(|%NxUSR9|!Ouf!=0WC%_|Y~G)EaFk945*piv>z4zpaElY=3B?NJDG34rXR)AdLuUWqvn)8#*e z>3sAL%P^BXaMaqe14#xvaSr4_s={+jd<})7{h^q88Z1b#`eFf4#R2m-{7UULYRA&E ze~Oi%MB_<_gXd|9n2JIS*YwHxlc!HE%AH(7FtTJZK#7KjFFg3&!rO=7+55I1hI zCk8c&Y-}b9b>flKq455gLk|8OnfoP<<)xUpz}A`fk_j}nu5hJ1SrrGp@1aHhTNLgS(ja3s0&k90# z4j^3^I@KxfK0LE&7c5H$alOub>tr;VUEM`M0Va`A3-OA^X*abpLyBY;yfP~(LXpw} z5!X{q#nYi$OEXz#K(t(adVj+Ur{_;Em}Uy&j45%YsL?QvNt3m%?Mslcm>8m{_9g+k zs63Nq0W9U)l}lZ<^A?v`sU;vD28h8*jpJVGS8Bn_A9*6b7}j&@c%GkhDo|)~t7&S! zvLH){5?9N+;!5D7oz&Jz$4|{y9gezkdc4B5;RLnloM1=w9Q;gGKOF7pqAq|wdaFAh zJ->_U4nH%MulLg4Dhq!0RqvfplcjpV@B`JtBQ}py%b{5>VucPdFlc{$X1CLzkiZi2A!p<>tEivSgkZa-+AoD>r^@X?4!2b z_sIRK0(uyu2Cwj~RYg#mq3)gg@;bG_jCSI8pKMl-Luqf7Q*^=>H4ggir|!M*p&!&+ zfRL&Ru3qr7S<&`Tx8Ab&Oekwl^&Bwrx5KhoK)^yuq!PF~Fp1>A)}dJ3_UeX|#=fnP zc*L);FvPvEQs7gGNu*jH$&z~EbAw4FbX*0PM4EwiViIX6-V#kBZIZA_Rt<@kWCiOk z$fUffk|pJ_?V6Ohr__?lN&qD5)Ux|d1e5&DCbQy8beRpYF6uyBZ-{;goH3*!vp*U^ zd2j(HJwBFyFu2oCKcqgXs9}AN=?&|js9ENXU7h`Bw-0Z8IAwLy&)@dOitnhtpV}ll zXM5P#x1P=~$ZPmO@$D^(5}JQ)!sQnG9<~<&=e!3ol&#+77H=?cFB(gbrOPc&BZ2l! zk_R%=-6GcdJtLP}oQ7+@L|wYv;#U&rVM4T^navNETU<+$ECNchZQi-X(R6}+XL^t* z$rinFxyA2vtcgY{g5j|eg6wjO3pyxJJ3mwr3Pcvzt6#VHqeR)lIqN8yqxdFQV8PK+ zCJrjGlsSAMAL_c?;;dYyH%=6E@j7wQz@Lw&Li;bR>`rsJjAh81D8fW>xz7CFuI%Ko z?IMn}miIc^dTlq^152Kk*LFAsF=JbDo|*Er)K+i2Ag|3ZtaG@h=id1Bl31_M=ppt) zeB{I}*#pssip>fX>w$@xWLpO3#TGcLy6l+#C23h7$1|O*tOc@G?eU#8dT^gJoJMdy z)3xxt01p5tomRe+IFnSX@sWWxy}=(y`}bBD-gXYY<(wCYPs1G6C{`$Za^~aB8=JT; zLCur*`{bPWUn3?p1@>kwahDdp z*o3x)CfRoHZsFcftZm`z9!F%Z-`w=msZb_2EN8y8K5VH4!$A^767c*f1k^~zjl)g>e{+4+{*{xu^=0MeY zS=$P<*UCH#d$G%Tx%-Mdoz`Hl1Q|c)?`2zTjc8NrxN{N>!OXYsC@uIyP+T3L z(*}?SHBSPAc&d8@Q*Y|>1R>?yhXTy^ea0H1~Ojc{#&YZR2-0oPM--3-@NP=9gm;vdIXfAz?6y}$5;tYP;b z{^)YOzwou2p1UFIvE_Py;naJQ@;5<0`wJ@}Hu1Q}m+Sq7%iet8ty<`3f8qYpA?^2< zFVcR0}1v75(68z9hLCu>u|C*PArC;J;@O54aa{k;=ztu~@$e-Ko$*Naz zyE+ao|G-PZ&o6TMDlY{=f8wPe=nw4RZVC?n$V)-f?|V%U?FZYq{Yozdw{G;B;Mbj7 zn9jGk{qr2${sqI&ye3#Vb+ea(xYskCuXs)H^WhtKUjM`WZs&0}dnpKe4UhAoO)mwL zf5GjyasO-Wc?!K01pcXAPv!42|F+n4(Ks7yzR0-5$1Ps!=S`a)3YK5R`y}glURyc8 ziuth7&J*8PGJRigF#UV>c*IYc_gcF@3X0#({Q21C5Ak!Y-9O3yW2SF|EiVdc|J-gT zd6Mr|E?;fSiRfHw=ZUVrGT%2c-J5uvZT5T!Z#C0HT>JmK$p8nZO0NoWAr8gdz~6TM1wH3Php1`Y*5qBay0z?BWCM?Wt`zKxWp@ zVP;}ZvVrByi<#}|qhzwF)tyhzoZHGRJfIK6`88!`@ur++e`e*Vw%@%PA_K6g42683 z9hE!f88r#WIxu%;!_5GkldTUBO56qU9x#Kv}4^2Fo#aiA1C^+w5-AiZKCGFPhQ|;sw%` z2{8*-VVBia6+zoQ8C3Yw)ED=#PQIokvk*w_a<#bbjHXACqbNe(w^8cmX#7GHh8A@* zd%k8amGJsRsPW#fy%!53Ht&i@J@UZCB-xO|6*T7p#3i)W6>Q**ZKo^P-#c!h2Rd%~ zWC=H6MO=;>jb-1V&5cHc*t!i6xOW)6e5TtHr|0)iw zg-9FM<}^Omw6|i0vsZE1mc5QE$+qmHa?d2(*+rL)hf%SP2|6*o?VUsNKmZ3G zfux##0^wPqiV8T1FOd&5)>GGlRXdiCo>}-Ng3!i1h?ZBCna+??809LAT`=C7Ki-0M zZP+>sKVb@oX5lhxNGJ?1k zLkPQ%!-X);=Q#4C9r-bkzx9x37a8{gkv|8ndr1IFF$t7?bt*}`m$Q7Cr9`Yqy{zmcFDpC27bh(B z!ZmTBDG%ONMz5P!YlE|;nqFgGIG6GtR@3YBMO=Rs!}(kuo=dN3zgtbP(_DW2Uhp^% zcxIf1+<%nu2Xgyc>+#JF*P3c=c3kFi`<{#k_@;aT=QnV>bD7>mF8@9Ee=7Im<8ik$ z%;WN#c^>n4oFeZ3nfi8yo+XTTG>=ox`9JbFmoWr-%nsmYZvS^4FNgbI#ChO@nTN|k zZ#?H=JR=V+e5vo>@Uaim*_G*ejr(;opD$qiGZ}9y_mfheH*~(p<>?|HhQJ>)@8Wv; zP5wL1Pvv>8WcXG6cm}UK&qMhCJ-1t0PluUbIrHTn?kB%~{$`$eOz$1cuZf(0iTj!6 X&=1g?c6|oPw(GO)$4HNUyZZcZP}yv2 literal 0 HcmV?d00001 diff --git a/cf/test/test_CFA.py b/cf/test/test_CFA.py index 213c17f6a2..d8a6005986 100644 --- a/cf/test/test_CFA.py +++ b/cf/test/test_CFA.py @@ -50,7 +50,7 @@ def test_CFA_fmt(self): cf.write(f, tmpfile1) f = cf.read(tmpfile1)[0] - for fmt in ("NETCDF4",): + for fmt in self.netcdf_fmts: cf.write(f, tmpfile2, fmt=fmt, cfa=True) g = cf.read(tmpfile2) self.assertEqual(len(g), 1) @@ -58,7 +58,7 @@ def test_CFA_fmt(self): self.assertTrue(f.equals(g)) - def test_CFA(self): + def test_CFA_general(self): f = cf.example_field(0) cf.write(f[:2], tmpfile1) @@ -83,7 +83,7 @@ def test_CFA(self): self.assertTrue(c.equals(f)) self.assertTrue(c.equals(n)) - def test_CFA_term_strict(self): + def test_CFA_strict(self): f = cf.example_field(0) # By default, can't write as CF-netCDF those variables @@ -159,6 +159,9 @@ def test_CFA_field_ancillaries(self): e = e[0] self.assertTrue(e.equals(d)) + def test_CFA_PP(self): + pass + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) From 347f074609d5c2b1f62e8fb92990d369a06fd9f5 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 8 Mar 2023 23:23:36 +0000 Subject: [PATCH 053/141] dev --- cf/data/array/umarray.py | 12 ++++-------- cf/read_write/um/umread.py | 38 +++++++++++++++----------------------- 2 files changed, 19 insertions(+), 31 deletions(-) diff --git a/cf/data/array/umarray.py b/cf/data/array/umarray.py index 2a05e9ce2d..fa5002af42 100644 --- a/cf/data/array/umarray.py +++ b/cf/data/array/umarray.py @@ -144,12 +144,16 @@ def __init__( if filename is not None: if isinstance(filename, str): filename = (filename,) + else: + filename = tuple(filename) self._set_component("filename", filename, copy=False) if address is not None: if isinstance(address, int): address = (address,) + else: + address = tuple(address) self._set_component("address", address, copy=False) @@ -246,14 +250,6 @@ def __getitem__(self, indices): # Return the numpy array return array - def __str__(self): - """Called by the `str` built-in function. - - x.__str__() <==> str(x) - - """ - return f"{self.get_filename(None)}, {self.get_address()}" - def _get_rec(self, f, header_offset): """Get a container for a record. diff --git a/cf/read_write/um/umread.py b/cf/read_write/um/umread.py index e25046fa0a..9ff5ce76bc 100644 --- a/cf/read_write/um/umread.py +++ b/cf/read_write/um/umread.py @@ -1131,18 +1131,24 @@ def _reorder_z_axis(self, indices, z_axis, pmaxes): """TODOCFADOCS""" indices_new = [] pos = pmaxes.index(z_axis) - aaa0 = next(indices) + aaa0 = indices[0] indices2 = [aaa0] - for aaa in indices: + for aaa in indices[1:]: if aaa[pos] > aaa0[pos]: indices2.append(aaa) else: indices_new.extend(indices2[::-1]) aaa0 = aaa indices2 = [aaa0] - + indices_new.extend(indices2[::-1]) - return indices_new + # print (indices_new) + + indices = [a[:-1] + b[-1:] for a, b in zip(indices, indices_new)] + # print () + # print (indices) + + return indices def atmosphere_hybrid_height_coordinate(self, axiscode): """`atmosphere_hybrid_height_coordinate` when not an array axis. @@ -1944,7 +1950,7 @@ def create_data(self): word_size = self.word_size byte_ordering = self.byte_ordering - indices = ((i, rec) for i, rec in enumerate(recs)) + indices = [(i, rec) for i, rec in enumerate(recs)] if z_axis in self.down_axes: indices = self._reorder_z_axis(indices, z_axis, pmaxes) @@ -1994,33 +2000,18 @@ def create_data(self): word_size = self.word_size byte_ordering = self.byte_ordering - indices = ( + indices = [ divmod(i, nz) + (rec,) for i, rec in enumerate(recs) - ) + ] if z_axis in self.down_axes: indices = self._reorder_z_axis(indices, z_axis, pmaxes) -# -# indices_new = [] -# pos = pmaxes.index(z_axis) -# aaa0 = next(indices) -# indices2 = [aaa0] -# for aaa in indices: -# if aaa[pos] > aaa0[pos]: -# indices2.append(aaa) -# else: -# indices_new.extend(indices2[::-1]) -# aaa0 = aaa -# indices2 = [aaa0] -# -# indices_new.extend(indices2[::-1]) -# indices = indices_new for t, z, rec in indices: # for i, rec in enumerate(recs): # Find T and Z axis indices # t, z = divmod(i, nz) - #print (t, z) + # print (t, z) # Find the data type of the array in the file file_data_type = data_type_in_file(rec) file_data_types.add(file_data_type) @@ -3642,6 +3633,7 @@ def file_open(self, filename): fmt=g.get("fmt"), ) + """ Problems: From 855f9926f6f72f703975659cab2d3bc6a07bab1b Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 9 Mar 2023 07:51:06 +0000 Subject: [PATCH 054/141] dev --- cf/functions.py | 4 +-- cf/read_write/write.py | 5 ++-- scripts/cfa | 56 +++++++++++++++++++++++++++++++++++++++--- 3 files changed, 57 insertions(+), 8 deletions(-) diff --git a/cf/functions.py b/cf/functions.py index 22556d61ae..c08f3c5d11 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -3248,9 +3248,9 @@ def _DEPRECATION_ERROR_FUNCTION_KWARG_VALUE( removed_at = f" and will be removed at version {removed_at}" raise DeprecationError( - f"Value {value!r} of keyword {kwarg!r} of fcuntion {func!r} " + f"Value {value!r} of keyword {kwarg!r} of function {func!r} " f"has been deprecated at version {version} and is no longer " - "available{removed_at}. {message}" + f"available{removed_at}. {message}" ) diff --git a/cf/read_write/write.py b/cf/read_write/write.py index def08804a3..9f2c5695d0 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -687,8 +687,9 @@ def write( if fmt in ("CFA", "CFA4", "CFA3"): return _DEPRECATION_ERROR_FUNCTION_KWARG_VALUE( "cf.write", - {"fmt": fmt}, - "Use keyword 'cfa' instead", + "fmt", + fmt, + "Use keywords 'fmt' and 'cfa' instead.", version="TODOCFAVER", removed_at="5.0.0", ) # pragma: no cover diff --git a/scripts/cfa b/scripts/cfa index e5d637d48c..6a398c7ba4 100755 --- a/scripts/cfa +++ b/scripts/cfa @@ -313,6 +313,26 @@ absolute paths. Ignored for output files of any other format. . . .TP +.B \-\-cfa_paths=[value] +For output CFA\-netCDF files only. File names referenced by an output +CFA\-netCDF file have relative, as opposed to absolute, paths or URL +bases. This may be useful when relocating a CFA\-netCDF file together +with the datasets referenced by it. +.PP +.RS +If set with no value (\-\-cfa_base=) or the value is empty then file +names are given relative to the directory or URL base containing the +output CFA\-netCDF file. If set with a non\-empty value then file +names are given relative to the directory or URL base described by the +value. +.PP +By default, file names within CFA\-netCDF files are stored with +absolute paths. Ignored for output files of any other format. +.RE +.RE +. +. +.TP .B \-\-compress=N Regulate the speed and efficiency of compression. Must be an integer between @@ -1070,6 +1090,7 @@ Written by David Hassell [--overwrite] Overwrite pre-existing output files [--unlimited=axis Create an unlimited dimension [--cfa_base=[value]] Configure CFA-netCDF output files + [--cfa_paths=[value]] Configure CFA-netCDF output files [--single] Write out as single precision [--double] Write out as double precision [--compress=N] Compress the output data @@ -1183,7 +1204,8 @@ Using cf-python library version {cf.__version__} at {library_path}""" aggregate_options = {} read_options = {} # Keyword parameters to cf.read write_options = {} # Keyword parameters to cf.write - + cfa_options = {} + for option, arg in opts: if option in ("-h", "--help"): print_help() @@ -1274,7 +1296,21 @@ Using cf-python library version {cf.__version__} at {library_path}""" write_options["single"] = True elif option == "--double": write_options["double"] = True + elif option == "--cfa_paths": + cfa_options["absolute_paths"] = option == "absolute" + if option not in ("absolute", "relative"): + print( + f"{iam} ERROR: The {option} option must have a value " + "of either 'absolute' or 'relative'.", + file=sys.stderr, + ) + sys.exit(2) elif option == "--cfa_base": + print( + f"{iam} ERROR: The {option} option has been deprecated.", + file=sys.stderr, + ) + sys.exit(2) write_options["cfa_options"] = {"base": arg} elif option == "--unlimited": unlimited.append(arg) @@ -1381,12 +1417,22 @@ Using cf-python library version {cf.__version__} at {library_path}""" ) sys.exit(2) - write_options["fmt"] = fmt - if unlimited: write_options["unlimited"] = unlimited - if fmt == "CFA": + if fmt in ("CFA3", "CFA4"): + read_options["chunks"] = -1 + if fmt == "CFA4": + fmt = "NETCDF4" + else: + fmt = "NETCDF3_CLASSIC" + + if cfa_options: + write_options["cfa"] = cfa_options + else: + write_options["cfa"] = True + + elif fmt == "CFA": print( f"{iam} ERROR: '-f CFA' has been replaced by '-f CFA3' or " "'-f CFA4' for netCDF3 classic and netCDF4 CFA output formats " @@ -1395,6 +1441,8 @@ Using cf-python library version {cf.__version__} at {library_path}""" ) sys.exit(2) + write_options["fmt"] = fmt + if not infiles: print( f"{iam} ERROR: Must provide at least one input file", From 6e253d1e5f4a4df30b94facf02a9d1eedeaf7aa4 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 9 Mar 2023 09:54:07 +0000 Subject: [PATCH 055/141] dev --- scripts/cfa | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/scripts/cfa b/scripts/cfa index 6a398c7ba4..0a8ce9c5d7 100755 --- a/scripts/cfa +++ b/scripts/cfa @@ -1305,21 +1305,20 @@ Using cf-python library version {cf.__version__} at {library_path}""" file=sys.stderr, ) sys.exit(2) + elif option == "--unlimited": + unlimited.append(arg) elif option == "--cfa_base": print( f"{iam} ERROR: The {option} option has been deprecated.", file=sys.stderr, ) sys.exit(2) - write_options["cfa_options"] = {"base": arg} - elif option == "--unlimited": - unlimited.append(arg) elif option in ("-v", "--view"): view = arg if view not in "smc": print( f"{iam} ERROR: The {option} option must have a value " - "of either s, m or c", + "of either 's', 'm' or 'c'.", file=sys.stderr, ) sys.exit(2) @@ -1420,26 +1419,27 @@ Using cf-python library version {cf.__version__} at {library_path}""" if unlimited: write_options["unlimited"] = unlimited - if fmt in ("CFA3", "CFA4"): - read_options["chunks"] = -1 - if fmt == "CFA4": + if fmt.startswith("CFA"): + if fmt in ("CFA", "CFA4"): fmt = "NETCDF4" - else: + elif fmt == "CFA3": fmt = "NETCDF3_CLASSIC" - + + read_options["chunks"] = -1 + if cfa_options: write_options["cfa"] = cfa_options else: write_options["cfa"] = True - elif fmt == "CFA": - print( - f"{iam} ERROR: '-f CFA' has been replaced by '-f CFA3' or " - "'-f CFA4' for netCDF3 classic and netCDF4 CFA output formats " - "respectively", - file=sys.stderr, - ) - sys.exit(2) +# elif fmt == "CFA": +# print( +# f"{iam} ERROR: '-f CFA' has been replaced by '-f CFA3' or " + # "'-f CFA4' respectively for netCDF3 classic and netCDF4 CFA " + # "output formats.", + # file=sys.stderr, + # ) + # sys.exit(2) write_options["fmt"] = fmt @@ -1520,7 +1520,7 @@ Using cf-python library version {cf.__version__} at {library_path}""" # ------------------------------------------------------------ if view is None and one_to_one: outfile = re_sub("(\.pp|\.nc|\.nca)$", ".nc", infile) - if fmt in ("CFA3", "CFA4"): + if write_options["cfa"]: outfile += "a" if directory is not None: From 0ed1fd84903e7c44bcc438a5282b69584da32b48 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 9 Mar 2023 16:01:51 +0000 Subject: [PATCH 056/141] dev --- cf/data/fragment/mixin/fragmentarraymixin.py | 17 ++++++------- scripts/cfa | 26 ++++++++++---------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/cf/data/fragment/mixin/fragmentarraymixin.py b/cf/data/fragment/mixin/fragmentarraymixin.py index 5fd28713e6..3cbba0bbf4 100644 --- a/cf/data/fragment/mixin/fragmentarraymixin.py +++ b/cf/data/fragment/mixin/fragmentarraymixin.py @@ -39,11 +39,11 @@ def __getitem__(self, indices): except ValueError: # A ValueError is expected to be raised when the fragment # variable has fewer than 'self.ndim' dimensions (we know - # this becuase because 'indices' has 'self.ndim' + # this because because 'indices' has 'self.ndim' # elements). axis = self._size_1_axis(indices) if axis is not None: - # There is a unique size 1 index, that must correspond + # There is a unique size 1 index that must correspond # to the missing dimension => Remove it from the # indices, get the fragment array with the new # indices; and then insert the missing size one @@ -52,22 +52,21 @@ def __getitem__(self, indices): array = super().__getitem__(tuple(indices)) array = np.expand_dims(array, axis) else: - # There are multiple size 1 indices, so we don't know + # There are multiple size 1 indices so we don't know # how many missing dimensions the fragment has, nor # their positions => Get the full fragment array and - # then reshape it to the shape of the storage chunk, - # assuming that it has teh correct size. + # then reshape it to the shape of the dask compute + # chunk, assuming that it has the correct size. array = super().__getitem__(Ellipsis) if array.size != self.size: raise ValueError( - "Can't get CFA fragment data from " - f"{self.get_filename()} ({self.get_address()}) when " + f"Can't get CFA fragment data from ({self}) when " "the fragment has two or more missing size 1 " "dimensions whilst also spanning two or more " - "storage chunks." + "dask compute chunks." "\n\n" "Consider recreating the data with exactly one" - "storage chunk per fragment (e.g. set the " + "dask compute chunk per fragment (e.g. set the " "parameter 'chunks=None' to cf.read)." ) diff --git a/scripts/cfa b/scripts/cfa index 0a8ce9c5d7..ca6b9ef434 100755 --- a/scripts/cfa +++ b/scripts/cfa @@ -1205,7 +1205,7 @@ Using cf-python library version {cf.__version__} at {library_path}""" read_options = {} # Keyword parameters to cf.read write_options = {} # Keyword parameters to cf.write cfa_options = {} - + for option, arg in opts: if option in ("-h", "--help"): print_help() @@ -1420,26 +1420,26 @@ Using cf-python library version {cf.__version__} at {library_path}""" write_options["unlimited"] = unlimited if fmt.startswith("CFA"): - if fmt in ("CFA", "CFA4"): + if fmt in ("CFA", "CFA4"): fmt = "NETCDF4" elif fmt == "CFA3": fmt = "NETCDF3_CLASSIC" - + read_options["chunks"] = -1 - + if cfa_options: write_options["cfa"] = cfa_options else: write_options["cfa"] = True - -# elif fmt == "CFA": -# print( -# f"{iam} ERROR: '-f CFA' has been replaced by '-f CFA3' or " - # "'-f CFA4' respectively for netCDF3 classic and netCDF4 CFA " - # "output formats.", - # file=sys.stderr, - # ) - # sys.exit(2) + + # elif fmt == "CFA": + # print( + # f"{iam} ERROR: '-f CFA' has been replaced by '-f CFA3' or " + # "'-f CFA4' respectively for netCDF3 classic and netCDF4 CFA " + # "output formats.", + # file=sys.stderr, + # ) + # sys.exit(2) write_options["fmt"] = fmt From d56ae8f8fbdc0a6886cc212c84e2f2d85524e4b2 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Sun, 12 Mar 2023 14:15:33 +0000 Subject: [PATCH 057/141] dev --- cf/data/data.py | 24 ++- cf/read_write/netcdf/netcdfread.py | 2 +- cf/read_write/netcdf/netcdfwrite.py | 47 +++--- cf/read_write/write.py | 2 +- cf/test/test_CFA.py | 46 +++++- cf/test/test_Data.py | 20 +++ docs/source/field_analysis.py | 231 ++++++++++++++++------------ 7 files changed, 239 insertions(+), 133 deletions(-) diff --git a/cf/data/data.py b/cf/data/data.py index 25fca64671..312b2f2bca 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -11113,14 +11113,20 @@ def tan(self, inplace=False, i=False): return d - def todict(self): + def todict(self, optimize_graph=True): """Return a dictionary of the dask graph key/value pairs. - Prior to being converted to a dictionary, the graph is - optimised to remove unused chunks. - .. versionadded:: TODOCFAVER + .. seealso:: `to_dask_array`, `tolist` + + :Parameters: + + `optimize_graph`: `bool` + If True, the default, then prior to being converted to + a dictionary, the graph is optimised to remove unused + chunks. + :Returns: `dict` @@ -11138,7 +11144,7 @@ def todict(self): 0): (, ('array-7daac373ba27474b6df0af70aab14e49', 0), (slice(0, 1, 1),)), ('array-7daac373ba27474b6df0af70aab14e49', 0): array([1, 2])} - >>> dict(e.to_dask_array().dask) + >>> e.todict(optimize_graph=False) {('array-7daac373ba27474b6df0af70aab14e49', 0): array([1, 2]), ('array-7daac373ba27474b6df0af70aab14e49', 1): array([3, 4]), ('getitem-14d8301a3deec45c98569d73f7a2239c', @@ -11147,7 +11153,11 @@ def todict(self): """ dx = self.to_dask_array() - return collections_to_dsk((dx,), optimize_graph=True) + + if optimize_graph: + return collections_to_dsk((dx,), optimize_graph=True) + + return dict(collections_to_dsk((dx,), optimize_graph=False)) def tolist(self): """Return the data as a scalar or (nested) list. @@ -11158,6 +11168,8 @@ def tolist(self): If ``N`` is 0 then, since the depth of the nested list is 0, it will not be a list at all, but a simple Python scalar. + .. sealso:: `todict` + :Returns: `list` or scalar diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index f000dc8788..4a7885ad7d 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -574,7 +574,7 @@ def _create_cfanetcdfarray( else: # Convert the string "${base}: value" to the # dictionary {"${base}": "value"} - subs = self.parse_x(term_ncvar, subs) + subs = self._parse_x(term_ncvar, subs) subs = { key: value[0] for d in subs for key, value in d.items() } diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 1f8bc2d4f3..49bb181b96 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -225,9 +225,10 @@ def _write_dimension_coordinate( ncdim: `str` or `None` The name of the netCDF dimension for this dimension - coordinate construct, including any groups structure. Note - that the group structure may be different to the - corodinate variable, and the basename. + coordinate construct, including any groups + structure. Note that the group structure may be + different to the corodinate variable, and the + basename. .. versionadded:: 3.6.0 @@ -442,11 +443,12 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar): # File term = "file" if substitutions: + # Create the "substitutions" netCDF attribute subs = [] for base, sub in substitutions.items(): subs.append(f"{base}: {sub}") - attributes = {"substitutions": " ".join(substitutions)} + attributes = {"substitutions": " ".join(subs)} else: attributes = None @@ -460,23 +462,32 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar): # Address term = "address" + + # Attempt to reduce addresses to a common scalar value + u = ggg[term].unique().compressed().persist() + if u.size == 1: + ggg[term] = u.squeeze() + dimensions = () + else: + dimensions = fragment_ncdimensions + term_ncvar = self._cfa_write_term_variable( ggg[term], aggregated_data.get(term, f"cfa_{term}"), - fragment_ncdimensions, + ncdimensions, ) aggregated_data_attr.append(f"{term}: {term_ncvar}") # Format term = "format" - dimensions = fragment_ncdimensions - - # Attempt to reduce formats to a common scalar value - if term == "format": - u = ggg[term].unique().compressed().persist() - if u.size == 1: - ggg[term] = u.squeeze() - dimensions = () + + # Attempt to reduce addresses to a common scalar value + u = ggg[term].unique().compressed().persist() + if u.size == 1: + ggg[term] = u.squeeze() + dimensions = () + else: + dimensions = fragment_ncdimensions term_ncvar = self._cfa_write_term_variable( ggg[term], @@ -735,11 +746,11 @@ def _cfa_write_non_standard_terms( # ancillary's 'id' attribute term = getattr(field_anc, "id", "term") term = term.replace(" ", "_") - base = term + name = term n = 0 while term in terms: n += 1 - term = f"{base}_{n}" + term = f"{name}_{n}" terms.append(term) @@ -966,11 +977,8 @@ def _cfa_get_file_details(self, data): the data then an empty `set` is returned. """ - from dask.base import collections_to_dsk - out = set() - dsk = collections_to_dsk((data.to_dask_array(),), optimize_graph=True) - for a in dsk.values(): + for a in data.todict().values(): try: out.update( ((a.get_filenames(), a.get_addresses(), a.get_formats()),) @@ -979,3 +987,4 @@ def _cfa_get_file_details(self, data): pass return out + diff --git a/cf/read_write/write.py b/cf/read_write/write.py index 9f2c5695d0..722268aee7 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -770,7 +770,7 @@ def write( cfa_options["constructs"] = {c: None for c in constructs} substitutions = cfa_options["substitutions"].copy() - for base, sub in substitutions.items(): + for base, sub in tuple(substitutions.items()): if not (base.startswith("${") and base.endswith("}")): # Add missing ${...} substitutions[f"${{{base}}}"] = substitutions.pop(base) diff --git a/cf/test/test_CFA.py b/cf/test/test_CFA.py index d8a6005986..6360d714aa 100644 --- a/cf/test/test_CFA.py +++ b/cf/test/test_CFA.py @@ -5,6 +5,8 @@ import tempfile import unittest +import netCDF4 + faulthandler.enable() # to debug seg faults and timeouts import cf @@ -105,16 +107,9 @@ def test_CFA_strict(self): self.assertTrue(g[0].equals(f)) def test_CFA_field_ancillaries(self): - import cf - f = cf.example_field(0) self.assertFalse(f.field_ancillaries()) - tmpfile1 = "delme1.nc" - tmpfile2 = "delme2.nc" - tmpfile3 = "delme3.nc" - tmpfile4 = "delme4.nc" - a = f[:2] b = f[2:] a.set_property("foo", "bar_a") @@ -159,6 +154,43 @@ def test_CFA_field_ancillaries(self): e = e[0] self.assertTrue(e.equals(d)) + def test_substitutions(self): + f = cf.example_field(0) + cf.write(f, tmpfile1) + f = cf.read(tmpfile1)[0] + + tmpfile2 = "delme2.nc" + cwd = os.getcwd() + for base in ("base", "${base}"): + cf.write(f, tmpfile2, cfa={"substitutions": {base: cwd}}) + nc = netCDF4.Dataset(tmpfile2, "r") + self.assertEqual( + nc.variables["cfa_file"].getncattr("substitutions"), + f"${{base}}: {cwd}", + ) + nc.close() + + g = cf.read(tmpfile2) + self.assertEqual(len(g), 1) + g = g[0] + self.assertTrue(f.equals(g)) + + # From python 3.4 pathlib is available. + # + # In [1]: from pathlib import Path + # + # In [2]: Path('..').is_absolute() + # Out[2]: False + # + # In [3]: Path('C:/').is_absolute() + # Out[3]: True + # + # In [4]: Path('..').resolve() + # Out[4]: WindowsPath('C:/the/complete/path') + # + # In [5]: Path('C:/').resolve() + # Out[5]: WindowsPath('C:/') + def test_CFA_PP(self): pass diff --git a/cf/test/test_Data.py b/cf/test/test_Data.py index fa205163cd..590a099eb0 100644 --- a/cf/test/test_Data.py +++ b/cf/test/test_Data.py @@ -4619,6 +4619,26 @@ def test_Data_file_location(self): d.del_file_location("/invalid") self.assertEqual(d.file_locations(), set((location,))) + def test_Data_todict(self): + """Test Data.todict""" + d = cf.Data([1, 2, 3, 4], chunks=2) + key = "array-7daac373ba27474b6df0af70aab14e49" + + x = d.todict() + self.assertIsInstance(x, dict) + self.assertIn((key, 0), x) + self.assertIn((key, 1), x) + + e = d[0] + x = e.todict() + self.assertIn((key, 0), x) + self.assertNotIn((key, 1), x) + + x = e.todict(optimize_graph=False) + self.assertIsInstance(x, dict) + self.assertIn((key, 0), x) + self.assertIn((key, 1), x) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) diff --git a/docs/source/field_analysis.py b/docs/source/field_analysis.py index 13b3126278..e6d35d4d5f 100644 --- a/docs/source/field_analysis.py +++ b/docs/source/field_analysis.py @@ -1,78 +1,96 @@ import cf -a = cf.read('timeseries.nc')[0] + +a = cf.read("timeseries.nc")[0] print(a) -b = a.collapse('minimum') +b = a.collapse("minimum") print(b) print(b.array) -b = a.collapse('maximum', axes='T') -b = a.collapse('T: maximum') +b = a.collapse("maximum", axes="T") +b = a.collapse("T: maximum") print(b) print(b.array) -b = a.collapse('maximum', axes=['X', 'Y']) -b = a.collapse('X: Y: maximum') +b = a.collapse("maximum", axes=["X", "Y"]) +b = a.collapse("X: Y: maximum") print(b) -b = a.collapse('area: maximum') +b = a.collapse("area: maximum") print(b) -b = a.collapse('T: mean', weights=True) +b = a.collapse("T: mean", weights=True) print(b) -print (b.array) +print(b.array) w = a.weights(True) print(w) print(w.array) -b = a.collapse('T: Y: mean', weights='Y') +b = a.collapse("T: Y: mean", weights="Y") print(b) -print (b.array) -b = a.collapse('area: mean', weights=True) +print(b.array) +b = a.collapse("area: mean", weights=True) print(b) -b = a.collapse('area: mean', weights=a.weights('area')) +b = a.collapse("area: mean", weights=a.weights("area")) print(b) -b = a.collapse('area: mean', weights=True).collapse('T: maximum') +b = a.collapse("area: mean", weights=True).collapse("T: maximum") print(b) print(b.array) -b = a.collapse('area: mean T: maximum', weights=True) +b = a.collapse("area: mean T: maximum", weights=True) print(b.array) y = cf.Y(month=12) y -b = a.collapse('T: maximum', group=y) +b = a.collapse("T: maximum", group=y) print(b) -b = a.collapse('T: maximum', group=6) +b = a.collapse("T: maximum", group=6) print(b) -b = a.collapse('T: maximum', group=cf.djf()) +b = a.collapse("T: maximum", group=cf.djf()) print(b) c = cf.seasons() c -b = a.collapse('T: maximum', group=c) +b = a.collapse("T: maximum", group=c) print(b) -b = a.collapse('X: mean', group=cf.Data(180, 'degrees')) +b = a.collapse("X: mean", group=cf.Data(180, "degrees")) print(b) -b = a.collapse('T: mean within years T: mean over years', - within_years=cf.seasons(), weights=True) +b = a.collapse( + "T: mean within years T: mean over years", + within_years=cf.seasons(), + weights=True, +) print(b) -print(b.coordinate('T').bounds.datetime_array) -b = a.collapse('T: minimum within years T: variance over years', - within_years=cf.seasons(), weights=True) +print(b.coordinate("T").bounds.datetime_array) +b = a.collapse( + "T: minimum within years T: variance over years", + within_years=cf.seasons(), + weights=True, +) print(b) -print(b.coordinate('T').bounds.datetime_array) -b = a.collapse('T: mean within years T: mean over years', weights=True, - within_years=cf.seasons(), over_years=cf.Y(5)) +print(b.coordinate("T").bounds.datetime_array) +b = a.collapse( + "T: mean within years T: mean over years", + weights=True, + within_years=cf.seasons(), + over_years=cf.Y(5), +) print(b) -print(b.coordinate('T').bounds.datetime_array) -b = a.collapse('T: mean within years T: mean over years', weights=True, - within_years=cf.seasons(), over_years=cf.year(cf.wi(1963, 1968))) +print(b.coordinate("T").bounds.datetime_array) +b = a.collapse( + "T: mean within years T: mean over years", + weights=True, + within_years=cf.seasons(), + over_years=cf.year(cf.wi(1963, 1968)), +) print(b) -print(b.coordinate('T').bounds.datetime_array) -b = a.collapse('T: standard_deviation within years', - within_years=cf.seasons(), weights=True) +print(b.coordinate("T").bounds.datetime_array) +b = a.collapse( + "T: standard_deviation within years", + within_years=cf.seasons(), + weights=True, +) print(b) -c = b.collapse('T: maximum over years') +c = b.collapse("T: maximum over years") print(c) -a = cf.read('timeseries.nc')[0] +a = cf.read("timeseries.nc")[0] print(a) -b = a.cumsum('T') +b = a.cumsum("T") print(b) -print(a.coordinate('T').bounds[-1].dtarray) -print(b.coordinate('T').bounds[-1].dtarray) -q, t = cf.read('file.nc') +print(a.coordinate("T").bounds[-1].dtarray) +print(b.coordinate("T").bounds[-1].dtarray) +q, t = cf.read("file.nc") print(q.array) indices, bins = q.digitize(10, return_bins=True) print(indices) @@ -81,34 +99,35 @@ h = cf.histogram(indices) print(h) print(h.array) -print(h.coordinate('specific_humidity').bounds.array) +print(h.coordinate("specific_humidity").bounds.array) g = q.copy() -g.standard_name = 'air_temperature' +g.standard_name = "air_temperature" import numpy + g[...] = numpy.random.normal(loc=290, scale=10, size=40).reshape(5, 8) -g.override_units('K', inplace=True) +g.override_units("K", inplace=True) print(g) indices_t = g.digitize(5) h = cf.histogram(indices, indices_t) print(h) print(h.array) h.sum() -q, t = cf.read('file.nc') +q, t = cf.read("file.nc") print(q.array) indices = q.digitize(5) -b = q.bin('range', digitized=indices) +b = q.bin("range", digitized=indices) print(b) print(b.array) -print(b.coordinate('specific_humidity').bounds.array) -p, t = cf.read('file2.nc') +print(b.coordinate("specific_humidity").bounds.array) +p, t = cf.read("file2.nc") print(t) print(p) t_indices = t.digitize(4) p_indices = p.digitize(6) -b = q.bin('mean', digitized=[t_indices, p_indices], weights='area') +b = q.bin("mean", digitized=[t_indices, p_indices], weights="area") print(b) print(b.array) -q, t = cf.read('file.nc') +q, t = cf.read("file.nc") print(q) print(q.array) p = q.percentile([20, 40, 50, 60, 80]) @@ -116,57 +135,69 @@ print(p.array) p80 = q.percentile(80) print(p80) -g = q.where(q<=p80, cf.masked) +g = q.where(q <= p80, cf.masked) print(g.array) -g.collapse('standard_deviation', weights=True).data -p45 = q.percentile(45, axes='X') +g.collapse("standard_deviation", weights=True).data +p45 = q.percentile(45, axes="X") print(p45.array) -g = q.where(q<=p45, cf.masked) +g = q.where(q <= p45, cf.masked) print(g.array) -print(g.collapse('X: mean', weights=True).array) +print(g.collapse("X: mean", weights=True).array) bins = q.percentile([0, 10, 50, 90, 100], squeeze=True) print(bins.array) i = q.digitize(bins, closed_ends=True) print(i.array) -a = cf.read('air_temperature.nc')[0] -b = cf.read('precipitation_flux.nc')[0] +a = cf.read("air_temperature.nc")[0] +b = cf.read("precipitation_flux.nc")[0] print(a) print(b) -c = a.regrids(b, method='conservative') +c = a.regrids(b, method="conservative") print(c) import numpy -lat = cf.DimensionCoordinate(data=cf.Data(numpy.arange(-90, 92.5, 2.5), 'degrees_north')) -lon = cf.DimensionCoordinate(data=cf.Data(numpy.arange(0, 360, 5.0), 'degrees_east')) -c = a.regrids([lat, lon], method='linear') + +lat = cf.DimensionCoordinate( + data=cf.Data(numpy.arange(-90, 92.5, 2.5), "degrees_north") +) +lon = cf.DimensionCoordinate( + data=cf.Data(numpy.arange(0, 360, 5.0), "degrees_east") +) +c = a.regrids([lat, lon], method="linear") time = cf.DimensionCoordinate() -time.standard_name='time' -time.set_data(cf.Data(numpy.arange(0.5, 60, 1), - units='days since 1860-01-01', calendar='360_day')) +time.standard_name = "time" +time.set_data( + cf.Data( + numpy.arange(0.5, 60, 1), + units="days since 1860-01-01", + calendar="360_day", + ) +) time -c = a.regridc([time], axes='T', method='linear') +c = a.regridc([time], axes="T", method="linear") try: - c = a.regridc([time], axes='T', method='conservative') # Raises Exception + c = a.regridc([time], axes="T", method="conservative") # Raises Exception except Exception: pass bounds = time.create_bounds() time.set_bounds(bounds) -c = a.regridc([time], axes='T', method='conservative') +c = a.regridc([time], axes="T", method="conservative") print(c) -v = cf.read('vertical.nc')[0] +v = cf.read("vertical.nc")[0] print(v) -z_p = v.construct('Z') +z_p = v.construct("Z") print(z_p.array) z_ln_p = z_p.log() -z_ln_p.axis = 'Z' +z_ln_p.axis = "Z" print(z_ln_p.array) -_ = v.replace_construct('Z', new=z_ln_p) -new_z_p = cf.DimensionCoordinate(data=cf.Data([800, 705, 632, 510, 320.], 'hPa')) +_ = v.replace_construct("Z", new=z_ln_p) +new_z_p = cf.DimensionCoordinate( + data=cf.Data([800, 705, 632, 510, 320.0], "hPa") +) new_z_ln_p = new_z_p.log() -new_z_ln_p.axis = 'Z' -new_v = v.regridc([new_z_ln_p], axes='Z', method='linear') -new_v.replace_construct('Z', new=new_z_p) +new_z_ln_p.axis = "Z" +new_v = v.regridc([new_z_ln_p], axes="Z", method="linear") +new_v.replace_construct("Z", new=new_z_p) print(new_v) -q, t = cf.read('file.nc') +q, t = cf.read("file.nc") t.data.stats() x = t + t x @@ -174,28 +205,29 @@ (t - 2).min() (2 + t).min() (t * list(range(9))).min() -(t + cf.Data(numpy.arange(20, 29), '0.1 K')).min() +(t + cf.Data(numpy.arange(20, 29), "0.1 K")).min() u = t.copy() u.transpose(inplace=True) u.Units -= 273.15 u[0] t + u[0] t.identities() -u = t * cf.Data(10, 'm s-1') +u = t * cf.Data(10, "m s-1") u.identities() -x = q.dimension_coordinate('X') +x = q.dimension_coordinate("X") x.dump() (x + x).dump() (x + 50).dump() -old = cf.bounds_combination_mode('OR') +old = cf.bounds_combination_mode("OR") (x + 50).dump() x2 = x.copy() x2.del_bounds() (x2 + x).dump() cf.bounds_combination_mode(old) -with cf.bounds_combination_mode('OR'): - (x2 + x).dump() +with cf.bounds_combination_mode("OR"): + (x2 + x).dump() import numpy + t = cf.example_field(0) a = numpy.array(1000) type(t * a) @@ -205,11 +237,11 @@ type(b * t) type(t - cf.Data(b)) type(cf.Data(b) * t) -q, t = cf.read('file.nc') +q, t = cf.read("file.nc") print(q.array) print(-q.array) print(abs(-q).array) -q, t = cf.read('file.nc') +q, t = cf.read("file.nc") print(q.array) print((q == q).array) print((q < 0.05).array) @@ -217,7 +249,7 @@ q.identities() r = q > q.mean() r.identities() -y = q.coordinate('Y') +y = q.coordinate("Y") y.identity(strict=True) del y.standard_name y.identity(strict=True) @@ -231,8 +263,8 @@ u.min() t.data -= t.data t.min() -q, t = cf.read('file.nc') -lat = q.dimension_coordinate('latitude') +q, t = cf.read("file.nc") +lat = q.dimension_coordinate("latitude") lat.data sin_lat = lat.sin() sin_lat.data @@ -250,29 +282,30 @@ t.exp() # Raises Exception except Exception: pass -q, t = cf.read('file.nc') +q, t = cf.read("file.nc") print(q) print(q.array) -print(q.coordinate('X').bounds.array) -q.iscyclic('X') -g = q.moving_window('mean', 3, axis='X', weights=True) +print(q.coordinate("X").bounds.array) +q.iscyclic("X") +g = q.moving_window("mean", 3, axis="X", weights=True) print(g) print(g.array) -print(g.coordinate('X').bounds.array) +print(g.coordinate("X").bounds.array) print(q) -q.iscyclic('X') -r = q.convolution_filter([0.1, 0.15, 0.5, 0.15, 0.1], axis='X') +q.iscyclic("X") +r = q.convolution_filter([0.1, 0.15, 0.5, 0.15, 0.1], axis="X") print(r) -print(q.dimension_coordinate('X').bounds.array) -print(r.dimension_coordinate('X').bounds.array) +print(q.dimension_coordinate("X").bounds.array) +print(r.dimension_coordinate("X").bounds.array) from scipy.signal import windows + exponential_window = windows.exponential(3) print(exponential_window) -r = q.convolution_filter(exponential_window, axis='Y') +r = q.convolution_filter(exponential_window, axis="Y") print(r.array) -r = q.derivative('X') -r = q.derivative('Y', one_sided_at_boundary=True) -u, v = cf.read('wind_components.nc') +r = q.derivative("X") +r = q.derivative("Y", one_sided_at_boundary=True) +u, v = cf.read("wind_components.nc") zeta = cf.relative_vorticity(u, v) print(zeta) print(zeta.array.round(8)) From ee0df71981f88a5b2e4b12e9f211eb9b90046b8c Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 13 Mar 2023 15:44:31 +0000 Subject: [PATCH 058/141] dev --- cf/__init__.py | 2 +- cf/data/array/cfanetcdfarray.py | 17 +- cf/data/array/gatheredarray.py | 8 +- cf/data/array/mixin/compressedarraymixin.py | 7 +- cf/data/array/netcdfarray.py | 17 +- cf/data/creation.py | 6 +- cf/functions.py | 20 ++ cf/read_write/netcdf/netcdfread.py | 12 +- cf/read_write/netcdf/netcdfwrite.py | 45 +-- cf/read_write/um/umread.py | 42 ++- cf/read_write/write.py | 49 ++-- cf/test/file1.pp | Bin 0 -> 187648 bytes cf/test/test_CFA.py | 301 +++++++++++++++++--- 13 files changed, 410 insertions(+), 116 deletions(-) create mode 100644 cf/test/file1.pp diff --git a/cf/__init__.py b/cf/__init__.py index 1ab784a745..b7ca44787d 100644 --- a/cf/__init__.py +++ b/cf/__init__.py @@ -199,7 +199,7 @@ ) # Check the version of dask -_minimum_vn = "2022.12.1" +_minimum_vn = "2022.02.1" if Version(dask.__version__) < Version(_minimum_vn): raise RuntimeError( f"Bad dask version: cf requires dask>={_minimum_vn}. " diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index e1a29c562c..08cbd6eb40 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -230,6 +230,15 @@ def __init__( for frag_loc, loc in zip(positions, locations) } + # Apply string substitutions to the fragment filename + if substitutions: + for xx in aggregated_data.values(): + filename = xx["filename"] + for base, sub in substitutions.items(): + filename = filename.replace(base, sub) + + xx["filename"] = filename + super().__init__( filename=filename, address=address, @@ -913,7 +922,13 @@ def to_dask_array(self, chunks="auto"): key = f"{fragment.__class__.__name__}-{tokenize(fragment)}" dsk[key] = fragment - dsk[name + chunk_location] = (getter, key, f_indices, False, False) + dsk[name + chunk_location] = ( + getter, + key, + f_indices, + False, + getattr(fragment, "_lock", False), + ) # Return the dask array return da.Array(dsk, name[0], chunks=chunks, dtype=dtype) diff --git a/cf/data/array/gatheredarray.py b/cf/data/array/gatheredarray.py index ab777738ab..f8e66117aa 100644 --- a/cf/data/array/gatheredarray.py +++ b/cf/data/array/gatheredarray.py @@ -111,13 +111,7 @@ def to_dask_array(self, chunks="auto"): key = f"{subarray_name}-{tokenize(subarray)}" dsk[key] = subarray - dsk[name + chunk_location] = ( - getter, - key, - Ellipsis, - False, - False, - ) + dsk[name + chunk_location] = (getter, key, Ellipsis, False, False) # Return the dask array return da.Array(dsk, name[0], chunks=chunks, dtype=dtype) diff --git a/cf/data/array/mixin/compressedarraymixin.py b/cf/data/array/mixin/compressedarraymixin.py index 48c731e166..456d2cd919 100644 --- a/cf/data/array/mixin/compressedarraymixin.py +++ b/cf/data/array/mixin/compressedarraymixin.py @@ -26,6 +26,11 @@ def _lock_file_read(self, array): couldn't be ascertained how to form the `dask` array. """ + try: + return array.to_dask_array() + except AttributeError: + pass + try: chunks = array.chunks except AttributeError: @@ -37,7 +42,7 @@ def _lock_file_read(self, array): pass try: - array.get_filename() + array.get_filenames() except AttributeError: pass else: diff --git a/cf/data/array/netcdfarray.py b/cf/data/array/netcdfarray.py index 795d918777..ea78bc75e1 100644 --- a/cf/data/array/netcdfarray.py +++ b/cf/data/array/netcdfarray.py @@ -20,21 +20,16 @@ def __repr__(self): return super().__repr__().replace("<", ">> _reorder_z_axis([(0, ), (1, )], 0, [0]) + [(0, ), (1, )] + + >>> _reorder_z_axis( + ... [(0, 0, ), (0, 1, ), (1, 0, ), (1, 1, )], + ... 1, [0, 1] + ... ) + [(0, 0, ), (0, 1, ), (1, 0, ), (1, 1, )] + + + """ indices_new = [] - pos = pmaxes.index(z_axis) + zpos = pmaxes.index(z_axis) aaa0 = indices[0] indices2 = [aaa0] for aaa in indices[1:]: - if aaa[pos] > aaa0[pos]: + if aaa[zpos] > aaa0[zpos]: indices2.append(aaa) else: indices_new.extend(indices2[::-1]) @@ -1142,12 +1167,8 @@ def _reorder_z_axis(self, indices, z_axis, pmaxes): indices2 = [aaa0] indices_new.extend(indices2[::-1]) - # print (indices_new) indices = [a[:-1] + b[-1:] for a, b in zip(indices, indices_new)] - # print () - # print (indices) - return indices def atmosphere_hybrid_height_coordinate(self, axiscode): @@ -1951,10 +1972,10 @@ def create_data(self): byte_ordering = self.byte_ordering indices = [(i, rec) for i, rec in enumerate(recs)] + if z_axis in self.down_axes: indices = self._reorder_z_axis(indices, z_axis, pmaxes) - # for i, rec in enumerate(recs): for i, rec in indices: # Find the data type of the array in the file file_data_type = data_type_in_file(rec) @@ -2007,11 +2028,6 @@ def create_data(self): indices = self._reorder_z_axis(indices, z_axis, pmaxes) for t, z, rec in indices: - # for i, rec in enumerate(recs): - # Find T and Z axis indices - - # t, z = divmod(i, nz) - # print (t, z) # Find the data type of the array in the file file_data_type = data_type_in_file(rec) file_data_types.add(file_data_type) diff --git a/cf/read_write/write.py b/cf/read_write/write.py index 722268aee7..4312ee4820 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -2,7 +2,12 @@ from ..cfimplementation import implementation from ..decorators import _manage_log_level_via_verbosity -from ..functions import _DEPRECATION_ERROR_FUNCTION_KWARG_VALUE, CFA, flat +from ..functions import ( + _DEPRECATION_ERROR_FUNCTION_KWARG, + _DEPRECATION_ERROR_FUNCTION_KWARG_VALUE, + CFA, + flat, +) from .netcdf import NetCDFWrite netcdf = NetCDFWrite(implementation()) @@ -28,7 +33,6 @@ def write( reference_datetime=None, verbose=None, cfa=False, - # cfa_options=None, single=None, double=None, variable_attributes=None, @@ -37,6 +41,7 @@ def write( group=True, coordinates=False, omit_data=None, + cfa_options=None, ): """Write field constructs to a netCDF file. @@ -583,10 +588,10 @@ def write( If *cfa* is a dictionary then it is used to configure the CFA write process. The default options when CFA writing is - enabled are ``{'constructs': 'field', 'absolute_paths': - True, 'strict': True, 'substitutions': {}}``, and the - dictionary may have any subset of the following key/value - pairs to override thes: + enabled, by any means, are ``{'constructs': 'field', + 'absolute_paths': True, 'strict': True, 'substitutions': + {}}``, and the dictionary may have any subset of the + following key/value pairs to override these defaults: * ``'constructs'``: `dict` or (sequence of) `str` @@ -597,17 +602,13 @@ def write( The types may be given as a (sequence of) `str`, which may take any of the values allowed by the *omit_data* parameter. Alternatively, the same types may be given as - keys to a `dict`, whose values specify the number of + keys to a `dict` whose values specify the number of dimensions that a construct must also have if it is to be written as CFA-netCDF aggregation variable. A value of `None` means no restriction on the number of dimensions, which is equivalent to a value of ``cf.ge(0)``. - Note that size 1 data arrays are never written as - CFA-netCDF aggregation variables, regardless of the - whether or not this has been requested. - *Example:* Equivalent ways to only write cell measure constructs as CFA-netCDF aggregation variables: @@ -622,14 +623,15 @@ def write( ``{'field': None, 'auxiliary_coordinate': None}``. *Example:* - Only write two-dimensional auxiliary coordinate - constructs as CFA-netCDF aggregation variables: - ``{'auxiliary_coordinate': 2}}``. + Equivalent ways to only write two-dimensional + auxiliary coordinate constructs as CFA-netCDF + aggregation variables: ``{'auxiliary_coordinate': + 2}}`` and ``{'auxiliary_coordinate': cf.eq(2)}}``. *Example:* Only write auxiliary coordinate constructs with two or - more dimensions, and all field constructs as - CFA-netCDF variables: ``{'field': None, + more dimensions as CFA-netCDF variables, and also all + field constructs: ``{'field': None, 'auxiliary_coordinate': cf.ge(2)}}``. * ``'absolute_paths'``: `bool` @@ -645,7 +647,8 @@ def write( If True (the default) then an exception is raised if it is not possible to create a CFA aggregation variable from data identified by the ``'constructs'`` option. If - False then a normal CF-netCDF variable in this case. + False then a normal CF-netCDF variable will be written + in this case. * ``'substitutions'``: `dict` @@ -666,6 +669,9 @@ def write( .. versionadded:: TODOCFAVER + cfa_options: Deprecated at version TODOCFAVER + Use the *cfa* parameter instead. + :Returns: `None` @@ -694,6 +700,15 @@ def write( removed_at="5.0.0", ) # pragma: no cover + if cfa_options is not None: + return _DEPRECATION_ERROR_FUNCTION_KWARG( + "cf.write", + "cfa_options", + "Use keyword 'cfa' instead.", + version="TODOCFAVER", + removed_at="5.0.0", + ) # pragma: no cover + # Flatten the sequence of intput fields fields = tuple(flat(fields)) if fields: diff --git a/cf/test/file1.pp b/cf/test/file1.pp new file mode 100644 index 0000000000000000000000000000000000000000..04d26869c152dd405dd6ee40f32872721a2ccf8f GIT binary patch literal 187648 zcmZ6y2{ct-{5Ea~Au|=3Qj|o9B)R9=WoXnyNP~)I%@ak4$UG!e=Ak4iQc0Y>E0ttO z(SSs0k|s2(|NXA@d*Ao}zGtny&c6GebMC(TbM|LHpU?Ab5m6D53Mmm02@w$yVfB9> zzQWr7Y8U(8ts)|R!uo&TL0&{8%2-6iTUZwtmaW3FNm%~Z@!vgD|K~%X@I3T?>Y@Lg z_p6GCn2Lym2yguFcNhQnfdA)k9r8c@`R`icvD)VUT=4(z=l^N_uXg)}XrQy*|J^on zhKPt@=6`JmRI5m_V`o)taKTLBSs)@3ST8Kxe}CNcvWQ5@aq+6Z?xzGXt$U=1Ijc9YLf8%l2sA4V5N52M3wt5WCRCR#l& zoF*2f(|KW~^niOkt!#Nq7g~w2<{8RtS^H>q=GtU-UEhH%{Orc+g0`^kIf2Y!=uviS zQUY`Qn!@5+(^;T$4h#B~!weE~*#YZJ=3;!3DbysgxZ@GbJtB~O9KD(8k9TIbsV!@K zF^rCr&OJ(mpq9nsh|G6Wy{_JWRCkM{d+IHfojH%{B)thNO% z46%U3=P9VKwMF~EIfxOgfc*BYP>c;ksb2)7o<(BMx^V2-8iLg~gHU-r5Jj7U@c8dx zgy)7se|Q+|K7JKg%<|)&DqZC~Uk*XYjVaiyyb}*jAA@^SG3LK7L33;=^3+R^6m}67 zP8V@r_SpF#PoZDS@&uTNnY zGGkfKmk^eAc_$l|@4?nd+p(CqiR_B<2xcWJ##;8a(HT>&P#5C_>hZ&sM!CvTLz}(4 z$?yPy=Dsp6sHr0F3f^#hrOj(QE`HS%W_5>&%bHlwI zRh(z~AAzyz(yHk4HN0hYI^R`tm)EWR$Qzm{P|H{oy8VzFZGRF>)pn%Pp(ifUl}nqb z^~|r79g|@nj5S#`ox~*O&Sa&&uIx(P7PkINAbT?}lD#rZVyb)6*~ia$EHeKbyR_{B z+a6TH9A}p>Bos3f`vT@Clf`O>B(s+#k*wfD04o;pW(S|nWiysdX3_Z?tg%djS+9Ia z-_N>E+l!9Vq}ny~@*6cupT+SaCZV>;S28*E-6L?yYZokn1jH3cF3M=ho4L5 zpt{QiQU|@UQQQwRZNo99EfST%;YceBL9%ZU)>H@JbA_-7$5p+_4&2rK{(-xs(2K`*wPveWmdI*$|4MRk3(n6^GKsW0?6Y4rAtpp{#2M-fZzh+r%)0 zj1Izdhl!|)(nXK;I9NSZ$JDy#obvGiZmoUuAOBMpJXtzx&+3DhOktz z%}j0UJXZX05_^71jTN8!Nkc>K(>dQW>36*?biI-~jdqIT3qqv?8!Inx-3Ror5gTB4 zHUVRX6d~@z4dm7|VEms4P%Wy(D@lf=fXi51TMXM5XOJ`@2iIcK5zS8Ei(E3!`6ZzK zUle*3gk#kf1SxHQlmz=h`p`lAatp@u)km@TZ9M)?K8d&P8PFa~$K<$EST*znvc9LF zBlaY!%W@DgCIg$V9E9@WU8ojLN^PN?q8pOACo0#uw-1NF`I!^!16CtG)C9jO=fUj3 z3IzOgLZY|8yEs9DAn}#HIy%vcMu+RD6HqPAg~rU zA2dU}x(%m|I|IZDww?i;Y5sg}`@vYjp3+BE zYYQe7Q?WmP`1KFx@Ed2B2z(b+af4b$@UGj72g@>{uXq!V6D5yR z<;e$Qb#h5bmkjsPBZg+Wm8VmvDz z;n(NK@lJ+*{Q8b<{H0qLc8*1#gJV)3>H2|=FLyue7*7^ zKA=;)%4n=1_bKuQw?{`4Gjo>XU3nDdd^w9p`Zcio)s4-9f1sAqWS5izk?~L>@dJwF zRJJ_XGftKi_DB-{$zVMaY>=DCInw{D zs)S&@K~Da6Dd#fpsld8&9Uo}Y&(E$LN&j|h)2koVXnk-W|J3O&KjH0VevEh-AH1WQ zKi%|^{}-o5x8%>DnNR%b?YK;8wdf{YA^(=%XpmwX+BI4JqRH&4&m7kI%#+!*?PjA^ z9A;WRk!;qP1eQM~g`N4H${y+^v&m+0Y~s-)ENt99mi66}DL39juc+T#9tTDLoQ*25OXiya2QJ`9EE*R7}`Uo zV4=Z|s$uJNd6!G~1u_T6qJMiQ*Kn(ZyZf-2W7_51^Xz19ZY{V^6()kb!J&N3jZi+> zlJR5vKk}JSH`QDb(eDCUNUOo0FzrRqAz7cb#JGH{8vDjH!)K*7t z%`$MeH0~gXu->uyX$l_33}HpiZ0^ z%1RO6RnjD7kqjA_FGG@aq=?pdG2;C0CvKU3!0wN&7}s$Jp;NBn+ub6h{mp~&;~eb! zoP*7&d61q_2*b8A?4{T7bU_i+-ge^Hp7%JU^&I9=FA*5p1F`PU7?kS9^_2~{@6ifv zmj?)2Q4fujM)Xc<06Uk8r&rUFq@4z_hg+f67tMWZuoblHs&Q|7br4%o&jp$*;KWRA zC`-vB^5kbO>Ps2tF1L(3XAvye^U0}dLXaB2XOTPKv?`Q8{`DwdoVkndesqBUG(Cl1 z)y4QS3sD-YJCR0i^Q4?bEZy_)BAt`jLZz|?shg(?8)`P5O)#C#EGwLt&iPgBXSWYa zFW%3d?+awbESPQG62wa8>}R`sHn2CujomJs#XdZo#@r%|Sy_rU>!=&X79RRcJ>K4+ z7UhZb;BhBIIUQS+7{+eV3y)`tVI z8UF@fkF{ZWa1BK7TtU#QLKJPwMRH^|jGgk(_@w~Sofok><2pu;s==wI8Wg5?!FRj} zncvw5)2z4nJ^URKTzlbttOH&*?xFj_XUuT^0;_$Wpnm@gZa96$FQsh!yOxL3E}5{> z3dQ=@>i7|Qf%`Nup8J+2i+9r%pkZJPD|1sct{aQ$4~qCnK5|jr7r7*rE!>?uGMw$- zqk`z_MB8a?idFuOrB$vohwyPHZ26}TLU|{VYkaMNDD~2qMBT|c+Quc)^R1Vu-GVmy z%~zCZw5c)!-Er*xB`bFSs{?C)J2jV7EstWF8`R zY+>J2CMj#euB(k^3W17jvBXb0eAIm^rhkemymF(WrDAmP=PkVEo)c9g-Yu_sHd30? zY&av(Rg&Y@pO)m@Tvl_Z9>3s1!)=i*>xbieQgOb%6jc*yaL(!}3Wbee=+={&Y

zX-pGhz*#|;v1Zgw7{9%b3g0%UFX=&8&R4iRmLezXiay!*i5@kudl$lS{Xm(=zBz9YJI=i{n zky$q`WZMc|+0eda?Ce7~cKPNaW^X&21#c&8+;?l{S!KrlDU4$gmLu6APYI@O_JT&0 zmC~2xVYJuXmJUvM&yNa86;)!K2I?|L@6>SC6wpvL^MAYj%yZh1>9$J!=h zr`IlUil<<5=Ms`H*Px~L8FVE+A-eiE7IcXc7aeipX(>vsG!A0;cu}H7g(dDE7S8>F zQBEJwnbv`vpe9`Ta2M5Ys$iya1&&T7FbcW^v|q=&&O7jveFAH%*Qgxm!*XL0qIFe_ zWGoy?hQCuKg%)aLX}u&F6b$0U%eOG?eT2J-D#Y7Wn|SBykY4XGBri#iOw&%rcws+> zm!zXNHUaN~4?;5E8MAFBVM(+)l(|Iu}a9#UhI0$NQ%=;JR7=<0_PsldXAj)^=?-^i|`K4Zx)S+|n8^6`&2aT{$^+;N7BNCe`(W+1rXGR!93MZm^q@IKRroMmEU z-Cil8Qzb>Re~OS1-~K?%{x53Pe!)xVv$V@UAZA`Cl18tgrRR=z%=wDf-U~yOPd5SHkKh94Tlj=3wh!~l}PDJb+TG{I9ZV+N%l$hquTKW z+#^SmQ`069?N~E%JJXW9X|y1<8qtuL9}np}iRkh_irqTuHViI zw0`Zcte>UAUv$poTY@F&{D1cJ^v6(|e5!!DT(70oP49(cD#qrGk!Ma%Mlg3(UAAoJ zIMy-4gnd14&T4cm*q4PCOt;XIxjIf|kq@j{p667ycZ)eoIA*}I($rW{lL(6ye?;B9 z&(da}7&_!(2$fUKp$V?1*SG$K0E=Za@IV?})ZYdLw zd#dDIt}4+Qt445Eg`}Bj5RZ9sB&}M6)Ti`<-kM09R-gI?0h4T z@HOX-NS)$3T@$!|?|AO(hIlUUK`iGuV>g$dH-fVbwzA!zV9uAN-{)iJkEbVXwo-0= zDxEyHoCZH>rp_Zj(tn>s*uedvj29@dyo+kgynhr+Gt_1CXOCf;cgL}G4-*D^OXe;z zjcxa^VR<1|%v{HqiH_D{e|8LINvAt$W_3CB-&sQUZ(#I#h9n!StiWC^eNUZ>vuLJ_ z6aDp0mQFp8&C`Stw(WC#IPEJ%ob+VIc^&$|xpWMLoT4!t6=p+0@d##?W@AvBA|vz( zO1`{E%sgMz+A^?h&6kHr9I8i6nP5w{jIoV*o}hS zUxax57hXIYLNwkFBhwBjla!IFq+Mk=*|l&a8F6YjQJbblPQFnjJ>nxsk*Ol->y{wt zR=*GxGKHjkb0lS@^GHU(Vsib`B4TQ@5uqkKFp#?g!}Xnz+iiv1pAHBqutSBa1!^~H z;@X;_I1vAxYxU~pdK?9zVVHVx$a-EJ?*hIJg>!LAZKT)5W zpY)81IGd|0%`~;-nZB7St8CR|CT8Q<;*X|GcluQJDRDZx_1=bwWmqtY7X~abScNs6 z_(fmsc}hcnzNhZb#n`c`iEPwAX@<;~^oh$Q`ZskuU8HP4^N!T;->ddlHJQ!h!j1B| z*9rC9OR1aOuBu8d^p^}IUYX%!l{en542P*h9%M&U;n&UwuwMQgzTbrQTJ;|37kcnG zrW@;4cEEG+F|sWmVa1eY^d4yeOxoacsSAs(d+^GC0I#=-5SK}k#8yp?q&F&)=rzNM zf1VogU#CVAZ>o~J+f>LI2_>TQMTKM+E0FYYQslO}DCzoQOTyPKBt?&1NJGhTlJ?Pq zT;8?-K1h2|U|E43N%cWtv6$&bt)UVO1Mr1TCqAiW0usW$%w5C%YFElPB za?4hamhsr&Hx;&b zC*#fnUF>&}g~0nKXKnnNo6^+C8T%G-l`lQGx%+CX>SkT!M-~V3du@1u-@3zGSS ze-wlJ%kuDd;2fHI*5TV;86;)caxV36s%9u(=k4V+XqU@E`gDH)-NeOF7qyf0U2`@~ z4lSexotLTl>zmZS=pmI@{Fd(8DZ(xul4Fa8X)t|JeU|*xm>H*;vy`1??5(B|+nlPy z?3XLEsA2`yd_tcU%A2!sgCkkp_70l$rhzI3<DcAfkbRo4{~%Yb91Z z<+1EC#oA*xaa;NhHV)KaR7)em#cw z@@e4`54wDv3|%dq&cC|QTh+Sbx!}>=B<}j92b`BeHgMR5pvx?gajV_2a!MD zV3Q|GDy3(Uqm$i8`yzLe7UV|C-@1}t`^KVup)UM)sbcfmpIpzXznsK1O;l%3Ldqp` zG}akF-f%eV;$`qeeF$2o{p9w3f6DnhOW`DZ6{}_~7Nyyb26%7nUsda}N-KRHzvq_O z+(1}O7k*xNgMr>VxTO#b`!Cw?+my_0{jyB3=Fj{!f)&+e?u?eyh-*1z8zo1LvK^_H!zO;Dh6^u|I|4&CeSn9|E1U|yjICWMsC~W_1-mTqH|`Z@l(?BI zy6Y+sCQz$%2fg^;a>@Kc^#Xpt{t~~Cn|Nf=AWEAUsKa(Ak1e4rn%q-)y*x4bntn+IFHLXmf zSLcH6Ol{-;&WPfF-x|xGNk3a@q@vATZ7$<%-ijdNkpZ@7EWp7uPZ(X;gd=kbijKc-}Xe?Ha#WX%1`a_RHA}SSu6Ed(x`7{uH9ZGArsmtC8=xWX}9?9?n?H$jhjq$|-^rlxd&9OfgWm7((58doQ2 zU|QxBsQ;YuPWYFD)%&wkJi0@Rgw%3w~=2(&^wQ5-WemD#t zjf8QD9424ZMxLG__BtBl@(K%FPn?2VE6vgT$^@6yCPCE81WEq-aJw%H<0;K*26=& zX?Y&vJie{ru8v;IB{c2kHoOnvsx(5l-oHyZuhj#B6+Q=TRYv4h$%Pv5$sc#{x}Ved zA2V+7ZWW#UQDqV8oHw4Pua2cVip1IJtrM87yB<68O^X@y$+E`_TItqLMfA?W1Jp6~ z3tw|^Gru+KPgS(ayeiqZ5wk~Ms5Ipms>8k$o9bGgZvLx#Y?&0;f;?3@|sqOw)2Jf z{pkZ4s27 zQ5^jKekmld5o|%H9R5sKIM}jH25%Avgteb>9H9S zcP~NB<~8uHUXS%tcfmDdD`Hk}#elL8%C7mK;JFWa^S5FC;C4i>+5)NMtq^$n;)F#A zyt`viJv0>q!_rZ|G9SZ}ityCE1Oll`=vZ0`JBv~b>XqSj{5723Kq0mI4iqbDp);Zp zIwPAgXHgd($$vs7{{!~_$j!LT;c$_#3>*8i(?Xx+sPFjdJ;jXY9aL4R#i%_XjqtMJ?YMuoiY=NP zufLkho*ThQJ+9{V2M&k*u(>FV*o2b0?KrS%1F}c$N8NpYr04nLfUX}r!gfG@mk;9B zu7SFY7j_%0MV^~CnrpUU<&r~4wv0gDgd|j$pMYpY20ot6#meKSackyTr2Z?!@s1)C z&pwY>`%(;#El0d8L-v>3_*!3wvg#I;+yT1GlEu5f%Z^&7+t){#i)jGhJv-+g^f|%%-nliSe%kj4=@#82baLd z*AtuMR^#PM4;1UVVh>#K_UKaNrLVyDotv@H_YjT-EDgp)i4Z(`5P+PW!O%4n zYPFVyL(3uzCM|*RTCxuVEvq48x&mJHE8+go3nTVy#fVM&F>vt+CO(UUcgAtZTsnd8 zDrsn}N=M<)EbI} zod-^5ufgxv-cXF#2&cc>aieNKJp26+>bMK>D?Fip%N}V@^l|)+1p3Whab=aSxP2<( zSk|NiX_*;#lC%;b%{wvfm_H299zx(Ae~9HCz~HWfa2Xp2`%fVlJu@1sN2TJ#oHMZ8 zU5;zhYS8`nA=cGCLGj>IOlfF^%V;4s>iLch_A;cwWF(2cs6l>rN)YqV96VxfT(HbD z{#>&?byoJE^COo~|6XHy+U+-=*Z8+;_QRpVKBaQ{Wo?{+sSIpG3^4uQ3{<{d3ya+$ zP`nWb`|$~AjEDof9xu!R9TWcFL>$VGht-q_II0~+UcwfPwpk9_o@F>#<_URcABe2q zk0j^A(A1Aa?$~%ZwWE*XCJjw4B^lO3;m67`ESut7Hq_MQ27H2yqv_g==s)+$t& z--YC;M`*Bpfu2L}(Wv|puM9uqbje>~9Hc~KZb*~r$D(BMrU5)J9>9UOmN=gP`AE%t*ye(N4W!Iog0w+_Z589&$xb74L<^YFvS z90o$&*exM;-)N~L)F)bERn1};MQ_8{j$p{wM8F|67Gg`{F?L)$;%FS^Z;Z$4+li1- zO+{UA7A7SYaL1sa*}c|CMOIJ1VEj$M?73&#pvn<+N_j`-?pG z{xpZzH8-qC_C@K42$Wf;ploV3S~GHlSRw~w{j;&8B^$o^8A$n>ifY$*v}7EG<5+)G zwy#D0id8T&UJIwBEr`75i#e)+NLmw$2N@CAbTby$6XMY?w0E=+Blf>aMDfT}?45E7 z*1xl%f)Yp=UPIRGn~-X}j|rz*@a<;@cGbKliKF>#V5PFoluFJc^yCI~frYF;RA+>8;9d+|0c5XC=(kz^EtFUCipMZ+*s z;Ru}02f)@~1IDj$!mt!8yjw8=Z3)Jxd;rvCEJyk11K7<)3c3Cy3{5(P!N@FJ(8m2+%l(pzW7%C6!`W>jU&hzk#&;KVh6LL&zl+Qub4w^l#P{@^uD; zpKC!9pMpHwZ9y&`o=CJJRmtbC_b^k#9cP68AlphA&IYqM-9RZps&$8;+&+d|bTpeA zH~IsoFXYa`Zkr+Zx*MKZ1*7KkF@%iG#?OHw{62aGJx%54@h!vLrc#`WE5#k5pIW{< z8&Wwb@X(Axbg)o6thpKT!!}}m(-wps+X0VLet3HMAj*;gvArtE^lTLXRR2At-b&^Epi!{cj&{b|HI z^dQ3S3AAnRqO$242If~lMV`RE#R@GVrf~T_0oFI?Vx_J(>Spdj#>5~Diig9!F#;*s zk$9ae)K9&OLj3)s$czX=h218hy%r#Y*kI$i$$0bH7A}=bA#b%0M`EK;wCE%}#^fXD zOA*{}mtxQMD>zYg6`C1W;a_nT0b8%(+Mz1wtgD9RnFi!*b-+@rAHm-wh*Q5389b{= z;*XCdmR*KqyoDKAVmz7jaEe5;I~{}ZX;`@8Eizoo(cRz&zx)|^ad8u3mh0j5`!=p> zfO>KsakDSOHo6KAp4>)pQY{LP-a$`D z722+r;_33MuxV^SpVoB@6TZV=&ZPrHh)>dIK-qOF7SEi5n_o>Zq+>1;OxB}&tshb! z96|5%7~GzmfPK#suvIk?vt}eQP@V)YI%}M#exJp1ywz^<%$~ zYciDBMXQikCfY!zw z!uX9?Bh-RhmI(FxO9GHk7l0pTL1@tkgY@B81guEHBhhq_MW?Ynld)1`e1%ognY7 z6E0M4LAx-Asn?kye0~Ys3~u1CXgwr#A7e#uJ4#GC5gp%!V;8zGk9EQG@=I8mzQ$nH zdwAaciamY*Aki?CTrpK9(+f37N186tEE`L{iW(BRn3A;4xk)wZQy96B72`fNAjs^!BD>-^ysHJ_*I3ZyKBf#I*dDQ5zs#yiy!i-IQA_UYp)k! z@MEcP%_&E%>NV(BU&0il0%Yo*KfjR6Dvax2!|wP;q4u;7*-qcEd4f=% z>--&FH@{>4#~*m*@Eh9{M2OF!A*BDR49WkYNTkoIk-WVl$&p$u@-SMLgv?PVsvr6g z)ZK-X(!6~;<>0n( z-;&$m+psrlMQGUv#J&27id!P2_pAsh3;m9jLT&4Nt_^NQ&)~AD1))doVDE^__{;L} z%P|S>KSjd3KLmy>K#1LU;6UnT1ZcXzc>7{p(OreW`5Un2*EZCx-;K(oeXyN;5U+GY zaK1AfmQ#f}UHdGYpL78^9#?TDq8#BlrFd~J7nK@G_$Bn0U5b0*Q|*sA0TGDSUX2}r zCirR4&pq|~&Uw^(V>9u@JT*6D1v^2}*$XiALtb|{)?QA+SJO~|%?T~;EO zQiqd98BL-urbUh&9!2z`jfsYVDR~yINwVwzL3hj(d>?fm%V(9tVbxXKyjO;a{pEPt za0QzDW!xUThL`VdA$(;sMwPt7l_`I5Awr5w-z(Hz1rHf==UFyi+TO|EL1kd7eUfW*ihskDy>|kTA~Of-!Uz#tmD7fvZl)4s(S|*Ggc7 zH!8(8qu6m9PW~0fBei}|&j`lBrWgpGr(ub~Ih+e9MgPeQxOg`g?XRQJ?COVsU#n58 zwFrOSx}#vLFn^__jjX0h&iu48x2#>At10xw=O?>_dBM$?uzU@+Z`py1GlGyD9))A2 zshB06hc!nFVU<~g6?R3~Rd`nD=X23LJQWM(N8(nxFV0_HgLY95IQgza+PxrTq#Vas z;eI>$s%j)nYsFUDiyK1+p%^-ZT$Pa~KlcnHr-U5l^a@2HH&2NiU#diMFDjA6`<01e zm{8X-UX`eOsgU3270JRTIkIqzB6%pMPU`3C5X%*l$kNkRBy`JUqL8XgK1=k28{2}< zPwR1Z{%srxet^fHpF(tfJ9<-}Bd+>6>V-bGJMKLUt$(3YS7@I@iezWX2r{F2Ba!qK%0p!eAY?Gu+{t}vgwux}%L9kxRK%`QZ*IRuMCQJ4^zhTRL#px3Mb zJHKUMlT9p)9Jk`sH3x{QSt8QJ0Bt7?G3Ce@oNmqHtTqG*Hi{VX@Epx66(50`%V9Lh z`=eTY7u>B5V%)k=yikgR|AI6Wc@P3tnc?n8JmxP$-B2M`fqV+{4w00)K)hPs8 z$=gt}+Z%=+TXAoF2v+E(K-~H~?mfSa8jE&_XnjGPq9|FQJCr<~Cr?^aREfk)4HB_l zi+ruuB32q&WOR-uX|&KJDhEfAt!?VWJ%2bEc%ei_8p@IJZNtd993|qTtVH}KXp$FQ z%Bo%k!Yga$|I){+CTct=nCrK9SOOU|%lEinCH1RW1 zB=41lIRBq6IiR9XCVw77#2;&ufIn)aKTeh;&;N_>m*3!X^-F9x+=2IVn{a$FkEZqK zP-l>a;Y$-x_~Zz_4(@}o+!}mIUx=Br?Lqd~VnhzXy}sG7Ipd5e0V{B!&KohBn-H>V z8;(^S#C)j;{BsuaAaxmV@j3x}y$IMJ-;9;}9I*DSHpmqzSYPSkwp4X-VjHe-&lS^b z-z?b0_ka1wt9^UV+k3^Ks3HpLonhz`?maw|6@qZj2<%lphRyw%n3hn4pX)EfE#w+< zCRgBxN;$STl_KoqjV@iHx<>NtihOqoD#ZZRb{70gM#w+WHZ9Y@}t9YyBPS0Eo9 z#L1(<5BON|3O$!P@ld@PeNs2k++KvHl2k}Nj)vyR!!Xa>h8Z@?ka>R=R9;PorGy1a zeJr3g%m!b7%mOC4pft!6fsN}>G0X?YTn}R1;&ALmA{J-F!)bjO-bt;;!|tirv~M`} zochR3^eE$=xrA{;%$9MdB~J-ne2L(9K9QnPhiB4Ly9m|nNk#D2Bor0K^wkQq2=j`QYVsixcO0P$!qBfE-1Fvh z0PaGLCL|{r+wY%4^s<|ns_+zlg}Tuj>jsh6HIzL2u0s4>$CKJhYf?SkimdrIg;bPU zl9w8j$@8#DL~5fE32!zaavr0}9X-IE9FotCDv% zeK3ys2K&GAB;lzZS#oM3$zEqmj*OW@EHh`3%@;Y6z0iuR^fD){YUX62kjs%awjoao z1;i~yKxBVek${QDlcCP0;L`n(CszCr%y(>c-~-3hMvL}eC|1joI^(- zA&f)q4jzKsM0Y4}HASSS9MYz?bI*3>at&!yxY`LJ0^_*E%DP>P_}+E$)Y)wjO;ZY^ zU&M~k%8xmyi56m`2PfgyBHVYVbqZ>?PQqTueaZbi3q|!SxU+=e!?ruvkWzz`l)DHG zxdmCDN<6N=h=0jhSScHi0ozbWT@YgViZGlIa-F@21t=fK`ozv3JB4Pu_ZxG0&;nbfE2H{A)3pq$mM8rQoX~Nw6=^R*M{klv;VY6;%aTO zD|8&W@Ov8BmOYDXdt^h_h^v#Y=VZv6$13FA)d}QUFh^LQ6Ukq-iexu$Bofx^$^MV- z#AU1#*=*)WzPs6z%7u<3Z0H;^*KjUTkDWzwD+Q$T$0SnMsY_nGRU($@;^gGL0XW|8 z#-FNI%%{yralQ$)#%?p0}1 zDwT$&Bq&RZIkV-W7QF`!sGT2#9YrS7mpUfxnz43#3SE^&df0lT! z#KAJ&6-u=P*D)f@3w1&4U?;?c+vEEKE6Cn6N8#sD7(T}cp2JNc?QV-b7rgLvX%Ido zxalDL5dz65IdG0XI4pac{yAlr|C1W=w>^ zo-l~7pMW?a0K@mVqQ89%Qe}rjeX|NA(xp(;_JfRvwy`YKQ!1#vPk4TXPB$E*=H=UI z{oqVm=s%l85h3)h&xJgm*^<-RG4xSn!hGq*bak)^>pRENTuqKnsRYui%Mnz{O{Jun z2~=JlPieKG6#CYPM#jA5cAqHVeqR@Hw-(fj!rs1fK734ZYO=l%W4r$~+yY$Z1c z@3!n03eHvwVJ9yMw(F`%VcaFUdi(+%-g|+{V=j^ZW^I!8;7UZTv4bu`cNE`3pb zM1My$(`^Ih`Rru9#rFI3ul^=k#9XA@#zXA<*-ly?^C)EIM!KC>Kv4tB>C1#lnrCy3 zT#r7X6F1tZKj{ORbq~PtO?o)6z!FW{oH1~b6=tol#-J8^xSVmsisufuY^dJbutvb zQ>^<3D*x{p{V8U%w|AFm_s~;xNxGCCXk?Mn+yvIg2h(|LjuiJXow~}JoKKFSj%Ew8 z_83joW)_q+!GW6M*$iZ{ADvfb^&t7s3a05fXc|*-_8V^Ro;2>>p2wn_*UO#Mca+v- z`+4(WH}~>X*vFgrn+UGfvxU8j%Y?4z3xcWNeW7V$qmZRjPu3ds?994NC**HZy7e`> zk2-SPQ%55Go3wZAJyKutgesI?Qn%R~I)^v3qvRF2CcU7Fphxt3!A%NeJBcP znvpG=N$qq2jk;Ju>D5Q5-IAw_vk&NTR2xmoYoqnN7&LaOp)+$BzIG2sMXDA)EgK0* z6APS3wn4DBJ>nDWA@RT#UplQ3cfuMQcGzRrJwmFXA6|?M!;xnZ_^2O+C5}^Z>ghD3 z&z%7)VKyGBEy7&OrO0bZ#k2((sNcR4<*v#2b$bD<{N`cz-&qLki9v)(D84HC!fq0q zyPKQAt86H`&nm$Angnzn{Gd0R+DY^MbNXC#m)e3algjN=>>k)jOSY_`a~l)LLw6$a z<*b%AccF#@f)aJ zW^nUER*K%8Sy^NJMxKA_HkE(D*YLSt|L{3BqlAM(h+r}?N3c43TnG~Cg|5Nx1qtg` zVO(kp+0JgE!wt{L{L&-xkH1TTaE~4@dq4(?kE!;~a|)_`OJ!BYVk7dhUND ztC`GSeftePzx0@N&Rr+R@-vh-pY`qfJIJ%3hz7Q7r;1evNLq$zdC`q@bzU32EBs6f z(|Sp-PZGZlOW~cmG_F3A!|bmbD9$oKtF<}ImRrN&vK_SV+hgksdsN3cAk)zabq;{% zYHtkk2*J(0lOY*69hdjdLRQi&9MhVGwt0yN%~=e6emQ=8OGQfVYHZM1g_3(IP9_jKp%Sk|je$mB4%%3d&f_@p_rGtvK zq`UVdiT-S(5MdR|H^-Bgb0lrE4I;Z#PqJh4Uw^@w=9Su0c&rt9_?eTqodv1QF{Q8i zBglHY5(&$@xsC^QoO4wH7kS{SNas#-&6!_w`PR9`ya9iKKchK7=vp*N2n(GcbZ%KG zRI2V2D#z9erqM5jVx>Pqwa0Ja+}bXRGVWrYqt7&@q>X70&sg^9IX&L}il)T3(zBM2 z6!yD|nngd!P+&PymEW}dUpKuw`kmTGf1%4?TS4x;``9X4(s!2ZHXc2UDcv1H{?iXB6_Eg~Ak>6T->c_XXC)3#CV-YZE3)*NT*TsJ>YYna9O2 zV{Q)}8UBfucDGaFfX}3JN9vwY@RG5_>1uo2|KtqAH-s1V?yySs!rXQr6jn23s-F&maFm-K+=A50pXfI$50kAcoHud#Jj;mmYM9q0U4KSAu1c_goHX z!xZo%Qyw~{17NpO1{uauP!`0{_53HbzigwQ*P7@le}OiQzd)h`$0#=71c7;dI}cV< zabqKe{Ai*P2U=)F_%kZ$V|mHR-`E_oi(JzFQbmkBgq$HT`=E~}J4Qp!!WN1*o#7qe zhEcWSQ3Ze4$p!;ULUC$+1g6SOL;u^E(0e-@*%#-bXkjuaaV0KSu7>%^41DRyz}D*- z$b7vT3vZ+$)nF0I7f!>Rv_Pym?F{F$qfz`tANmd22%fKo5sMTs^sEG$_V!TO%P!_$ zen+0O@6(RNT8av*Acs4JU}>Q;>=zBfTIIp$zpo1M2P#lkRK_q>1>7u@#jCdx_|^QIwhwKm z(v^4Vqxx++ev0+OJ1)>I%QG}x{yeF_zCscUm`BvKnU3o=(`AoVN(<_yM3zAr+4i0M zUrEBCPX(Q9hSZc}j7V`ySg4Of@d;PhZt=m-rNJomn1u1YQ?TMkEc%xv;E%*y6i-`( zb@j{e*gXvsN>@W#YYkootiiwZ3?xlj4gCjc(Al?)=>wrmoz!gdS8k1$@^t!qNUpU1+hBke+F|3Bep$bi}qse!eQ zhm>m<8h)mhxBb+1N(RAQO3*YK1f8n`QK+qq(xb{qTsR1Bqtr1jb0}`}T4=P@Mo_6H znzJ=v_&^m(ca-7UFNbeGnD@-5hy2Ul)32A$>GbCZXO;CMQA7rJ21tuEUGDZn=rKKP>qK8Vhj}f>knan5S~Ihn zTiR^H1)035mO2;4YpS2-$!>6!)x&mV*eX$7dJNJ1FhM^Zjtsb$6o z%8qNMPP1F2QF5K!8fwVkEBp7e(bSsZID!@$&67_}WuauenZBJx|cB;{RA(S47rMo2j93GfnC%VBEn1(v99g z^>XRdqCbPICXS~vcN5xOB}JPSALXXnD{#$$$7<5n zEkf7z{eqg&1>wb=+d@$CGokK!yAZMIn{YAwhwx~DSnZenvb9nfO0_D(lxn3neW6)* z#o^f~4>Re(&>h9v>}H7|qj`~~Q;O2pWCGZB9}2A}4zKDgW$XAW|R zd29o_G3K}oL)0=|KeuEkdUI957s=tlIC1E+_htJR#*G-?LDdJIlUCym$~bk3_WapP zX3We)1p2@%F6BIDwTF|1(&{y4DxT) zbj~Z|^(6lBGBeDC{<2tMo9qTbDRPgnzWS7~NB*WD?fz6qJM>Y|OaCRj`rap4Z2c?T z$`h}BIetLx>$^&|rPGya!-jsRt4>nLl2t~9xCTyc8HRrcbWvfYhrJGZc)6W<5-yL# z%1@&(NZuT8g2$rZizVXSERh#zfknGau(Qgf`Oms-ye2G6o)#LE6(^dbsgB zDQ#ruUg;;=H>`!6?QYS9mUCpZ^wOJSCYQMB9*B)D_T&p~+gIZXCBe z%iXp8qVOAH*ytdGTK0eQR~d>g2lbF`Wdgq$mhi7}#7aju+#DQ$PM=8#$clmH?RY4g z&Vh>lY^00N#F0bOVAU6k1#QuoWH|*{nPI4N3q(9W9wwE99dqmT0(orgpz8RIr(OZF+Sx^E;I64Z%W4cdV(c(- zEQio9-sn3x0p|up;)TLAd|o|+u@+)+_WER$w@<>mYZKA0GZDpX)>?KV0EL<3abX$Y z@xc-27Ft5}-6$L?*JE}1P$aYZa&d|R8k42q;v5lY^6P4o>55M zZK^tXi57L7pjAf?lFW)-)V_*wL8g?D(xdHkU{@i%a9KynvsX|-K?JRLu%$juo@SLE z<4*iC<`%u(?9#MSk-v9gH!r?lT5uTUD&$LM2!+o}g&UvF3G)U&6nfkl6L)y8;KRw( z%I{OCy>~^bcFsqsT3h`AwVn2|wJjHAY7-|uqtiQn(V6!$nA}=VV>}AWwwmFa zvN=YjTfnt(EF3Lu&^E;Z!|ynw_BA2tu^YrxJRp_ij_Y56?gAHtj<<)yr7rmsmlZ+r(m1+L0vdC~vBgaS$CD)SG4dWD7T>9pK_%mZ6gpEQ1yH8IaAA$@2RFJb$4px>DSiblVxny_H z;>cH|_22~^lxd+0+h3CZ*{Af%=nhq?ULw)osL!jqPS*!2Myqxyjb3{m%~lhsL3EIERe&ZYcBgLh~tatY7Ja z>G|HMIO~aprvRrhPB`Uc0~y{NFEf}Rn&&d<{?<4Hyrz>4uOZ zEujGa!HQUSND(o8>^mB$im1^;vFoi4zNC*tv4$BuYR95C)CStBPVgvnLx!R^Mi&J@ zq9+8dS0nMHbTYP12*uc{VEkF)kK8^VjQit_lOYUKQt>?11?yb*6e*y}f z1NwSvN^0VuZMSOaSZ#B5Je_15jfn=pD z=|+kSJ?YuSb*rgx7jLAxtZ}_w6TLr??=io{BXWeWcl|735Wh*N>n;-%${EYpEq^@ca*_xj^fNH79YC!o_U6ptoO!0VDA^dB zdBB!A&Y1Mqfw7eApmx*}B5`v>iL-BHxe2~S8e@ZxA+FXLpps|5Ge;kj?i(QVs{sTp zBfRW4f$@J9u#2;W|1W1G%8L-XmEgbE4Vl#*nEuKav;PZ%MoS3xtqI1n)j^mv!yAE* z9bw9JEAEOZHheOMRhc2C)~Ms$Zb>L9vfO!jKW$pqO8M*^)wo?lyZbItqjD8pSG+(v zbFR`Y-)4F=|1GKNzarbc4HQ6(0~)@cvK`Av^=v6MVmGPJW3$Uod*~U;>Co7%bS5i@ zTBMiIo}kI(Jj0%jzf_@bJr_CKWuv%rKh3HGjFtJ73$ywC$Le|Y^+SZxpCLl~#WcZi z>v};we23twbVO)gUL!mf6^>y5_dGE*canIQPIDaI&Upz{)|PokaRDFvJx;|cX6-Y9MMg5^#xNL2b_U!*tE z@3=wor8@?Pc%$);2b!WBv6|hjq1t+wuB(Ro%FJKJm<@H-KWMJlU+O;do#b~lk?h?{ zs`*+*7w+z$6xXwK;Ou>>Uip^F!@pBQ?GH*k`ko}%_mrY>njW)!`(L9S6g#(sME>Q} z$2>lP|CLkHlu~;BdkZxy4dW5Nav|aTp%IXQxE%6qwA3Li*7t<{ybfc604v{7<;(#B8-dh#OVI<$XXW& z^~_-Cgt6yy*B7Hh{owN16GmPlY#b}X1q)YX2`-Rjo`-=uN8qceIsVMLX zsoUKqE6tm9YDp!X(>+Dz&BZh0)s)RoLcH(%Ut(eBXSUX*ro($GFqjhLJQ* z<`t*DT9dO%m*g|VPxDDXy7<1u(t?qjybyCuR&bT;2o*@ol3}{%#v7nVh7m?>8wOI%D@18lV<1Lu}QFPnt~LHL2t=abS{{N?pafD ze@Zk4cZIY0*%X}l6$uIN2+UCrfzx3>EI|NL2l~SG`FI#6xM6m%i1j|>*fVuT@@WzL zZCx>^+Z!5-gHW+P7*2P>P-HO?hlU1W(*+-d8hT0(tjz zlf=kj5Hr@sEOQAQGr2>lfA&-BfFeql+f1_p)Q#lyps(U1{_2EU28a%%$A zVnblKGyqx>zWA)<1+jWB4E;SGzs`?``gMQw=1+jm)=+GI8it=60wFgw7#)8-kljz{ z{Avp!(+qkm44^e-2*TtBL1Okm9I*LKwo6{HxxxW<7OkbyqjShCA)2bchm*8;0%a~> z+7Z(v_RLvFAKBiFPVpPmm;H#62j8XCD{P+pht1782jZGf6FnaA9|eC}Pt~K>lYdn% z>6UDwu%rT7bs?83hGbIufQ3|{5KED}Lg?8HAL`%iMj3LW$=32a*Hvi175N45iT;0i zC)q(lf%1EPwP7?bwnmHF5G6xiPi}Mf`tFKOCHV5vvquW;v+@O+T}evF`< z5--HX&J+&qif7F9^)xhbH|cJ!rgPt)l3DO~dbmUiO+JiwDK`XPXAZ~Rvm=r7pD7ka zj)h^JEo9$}gXDV-UUD92i}b;x>w!o$4})ydWNe%kgPW(Oqj>X7q+E`N&6)(19-jp( z!3=@h z@9&E>8s12GFb*e8foHKI#GM|8bN@_HSE&VW2^HM^AO+RH@3dszbIRFvjuMRbk?qM8 zdgy0OkME42Au38#p{hlBgI(xkUKq;)W|Jz*Ia>~Ip@pmuSJf1Q*Y^&Zs`P}q9(GdE zR%NIawvx@M9kg};W4+s?6Sr$Ex!5ruUu_O)xUOMrmHA|?I+?Z>c+u{wPGoP!kZN8D= z8cC3z62j-HBQtmyl-BAaXVWP7M_AzVLu>e3JL30l5k}a#hDWww^fF6Eok*%s9B)%)r~ssZiIOin7>fC=^V=stIha zkQEN^jT7)l)feh3JyG550-ZD7FToPk}|NE6ib8em@=^tHec!%=cK^4KtgMr0%Ee^LlU+$K+Lutv?)7|RRapQ{Qp$BhzN zw-^em9!Lx38YWEJ{l_V6)8US-yTR}H7A{=#=7jVs0|bwqN^Zqs4Tc1aJH^==M#Z1r3& zDxr-l4f3Z6=eJTFV{YtY`lQJK1+-6BL5!j_TrK3WuTc&mRktbf{Q)}ZSV?k~he@l5 zd45mLp?A!W*S;}^{*8;J%9=2GZ{z`;8$Nb&VN!l&i_&Q#7Wg?Q1+s6Wb-GU?H6EK=Y@w!x=f%?zK^IZx`URp ze&b!JEXrOhVZ-ym2)eEb&GWjbdClgu38OI;)(D$D4v*IZrm^f^xfOs>asI#l9anuQ~4A&Yp+d%8>`TH*1D+!dey1vq^%h z^Z8x#N=B$j4vrF4ZtW6%ADhi}epjV}=roe~Q$-DHKhnWSmiKQTjC#hD45?B;70VDb z{k~84CwCGz^cXmOzm0gNqOHQiXQ4uMOHqvOU8quM_E(-4;}I^P@!#~ zmB?{{3fXt4(EY8l6ftBV{d(5H*&pA+z0fZd?KvsM-+YtBSB3q@-+I24k8)VV8}8TP zkA*+0HV=~KtUitAKAn!`-gQ2$SwCJ|I6pvKn5Z7a?{#^A6C1IhSZ(Xd_*G@{TeyyXkeJI6fK3Lg}y~>gF-t-;JRt zxUY-VJB?A;!DbhW9dIp^^+qKDmtVk@6kH$aA7<70>A}+UHmHm7UsfsG- z>ou0?_%4Ng8tPCSI1DRyYhdoL0cg0|MTLz#HT9k%+cmoxt9TEcpR|mG2U(=aa%Z7e zR?}F9>DR!70CEHj~y`4Ix?GmTvQQe&P^nR}Ts4N|Rt4K4Fm1*Ew8A@3Dkb7*u zh}&EI+hw?~6u)SzJKx;K&c88pYff*;71dwx;6B|hS>O*S@ysk!iLl1 zVaCqDNR3c5`cFc{zzFPL6M?mb;aL1L47X2)GT(hL#@zLWdAb|=mw01uLm2vIhv1%y z6AH5oFyvl0osX%f3m5j%wV1V>rFkt^Gi@W+qoBo&NF2qjFPz3bSvi8j9nXr+H!tK? zW;syH+>_jlA~!mF)r+>LE}@ro$H>U(1+|`$z>{oM9Q>yVRb6eATpf%#hb58CvQC-h z)imMb8Zv#AO{YIjAeHDj)H2J0;+1C5wrK_QMt1{!9<-2Z$NACyNk%k_F_EIW#mRrq zM=sl}gOgS1=WK^Do>i#=>xl=DFyb2*U0TVVQP$?BZ606qahw_7qCJXVCt6TbqWD47 ze5#Q1TJn?|^6fXb{Y@)Zz3eNuWqhq@&JlBdj_xHs%w`845Zh2)(4gXSUQD(6m!o2J zSLt$>`%RMEsq>lK{;{vPFgGKznBhw&ZYR<7VazMOzl6lO3OfJf61Aq@BtN$YB;WCr z93H%;f*&8rTkSW+GOlU;CsnKq(1Qc>0jGZ$hrgfQvD4BIW@*7lWgel;X<<;e4MQ$_ zZY!dKu{9_NpNoBwv<_JM+ZwMDnZ}gmf)ye*cd#{uOVuEh-M&r>MsA{O;xD+P#ns%W zmAAN|%121%@g2_a#ab@UxtBBJqDb}{+qYzxM*hD_Smu#2Hl`mSTjsg;9?!I-2h2yM zKN!LHhTyQQ4)*V5bNEOP`OhvWy7=65B!?JGgvSD$fB ziTRww9&M4om>utPa2@ZlDS>~Xw52BL;z(|l#uYB`INKSKr9tlp30!K%98P6%PK|$x z5C8355ufUr!z-;%5|vgqh%^pXic*K35`}zG<8@YO)f7y>DmuWj1JkB`<^1h+NY+zC zcU)r0R(3U2c5WiKo-!Jmc9?dRR+0_-#wvH7qYVpdsE={F7Y)2m+O3Qw&^Z|0tBj!T z!~8tOZs?xkhaAZebVr9`tx723lS2^qIS@`|0SMg`fYSNC_!^pvTtONKcTs}PJ}Qr^ zq?vW6NbK<$wu7dE(hr=W0d^9I9X=FxKSx0KxE@Ma&AL5E1>2vCqwYd8-84Q%^=CHH zWvK*`?g(KT+;sZ(Er6Czm`$zkvuNVITpGW59r+rhQ0u=?YTji=56`KRVuuC|nmm|H zWpt?GqcPpTVosDYioW-2(($XZBzC)%J2+@U0Dvz4@a#+kor9YIRipqlo*6NPJUQ> zJ^(M~2BF(F2s&@Ppf=qaN?N1v!EV`U|2ysf8)TK6d@x1PQ4b#z2uAhoFTWE^&oOk9uAjHWNNGFBeH zPBCUky&_VU48W{BF{J4+pGMFb`m4O2_%nXgGdPki%Pgmvv!+td-I=7Svy^e&m(sEC z)9KE0cbfLmkktPwQB>?el3k!n|$3pfK!Xu%?;s4Qb6xaI^VcV^lHabQNG1Rk*-r^&EQW4 z{EIC-?=n-LullUP)sLLcy?m6(nLo(nmIj^V<1GvMq;o!es#`@(QS1RxVSf^5;#>rFJ^=%1cn~_cTKI|aXTYqU` z+F&g3F^9OkJ2ZU#vFN!E*30-|TvY(>Ffacm^Co%Sleh#S^pCN&h5XrM#iO6nQc!Vho{rA z`>V;Ha{PlcM)-YaWq7LVDB#k>WvXGP1E#%(H zJm)*lUF7Gk+{!E63g^3`t7<&2P85Z#a^*ITE92UZH*ykF`?wp7IXW*#ht7-iXi~}$ zx?nJv6d!Kj9ya_9Yq? z$ovS07gNE;aJv6`CXLhHN?~Vrk=7SEyvSg6fxaC+*t=ki=|H?ld&qLE2dG$u<&Ec= zu$;+i?#gABff>1mjFijh;P-kOa+7)L{S+VAG2@ zWNSfN3YAD<*Llu;$WiW0;4AJxzYT@Gil;GGvYFPjjVgEVBY|j6>%dyo{08e)Z{+5 zeyA2nzN#rtT+A<6*<5pSbOyKL!!>T9V>01V4)~}PJ>*-&(YZtUB_0me()=|NDR-Y;9@fSMQ*+u)lNF({N75t9^`UZjM zoEVLY{MqP8Ujf@sQ{nIG1hsENp%%<|OiYKGIl&gQBlY0iD2=7Lzv=A?1=N03$LLLp zP-$Sf`L&W5e_0n-7CD12WZv-+%=4L3Ny&MuXj#5F+xdH$GtOHjau3rN#fj-rml`{} zi;8H%*~fH25J%G==10rC#kk02ROebl!_@cC#IYMGu+Wg+&Me_ZAC2ZNF5bo+z9>y^ zmiST%tA%A>GqzOK0UD&md`M49X_pq;b8Bix6)F3<6b*kZ&qXc` z<>GwTaoa*mI0u~roZC^BLdlxtGNz}3ySzw}ByXwFh;$>WeymRm{wY$szAot|zvc|@ zKIF{rPNPdrQi070`mpgh9qjy% z(tppTx2h^MByb-muW?jl_I`hjaM!TrM_?X5f7w(1nnwXY_Ggmlfzui8OxhsQR#m0z zcRzEDn}?7pV=x(}baDZA^0*h@^+fR@>D;30##A!SjjErykuTio@eV7&b(xVcx=B;G zQ>iHQW&h&$P2~B;1#5Z#+|hjVvST%6?%fv;UOFUNR;tgnt&QRm2jz1Sqjqve>AGBE zkp%mDtZCRXBBd_Iha6}``-eD^$8CGMHN%Y5vKe>wnlJrM+(V^vswvxVKUt_;p<^Zo zXphQd>R!O+n48bjwO|D}m9h-xeJA+V_+sXnP<-o+gxKCVym5|)^y+vtEu4+dZYwY# zVGZ5}u0w@!44WN|fx!xCWQ}-7BR>Lug^t6;oj9ZeoxOkyppA9N3;taK-O z{aJ!0%0_VdIYnGg$wSV*u7`Ff{G&q#DpFB_#c9A1$vvNUI|&DA;@-eLP!0*B?xznyh}Vpyv#mt6Fl6<8rI}r_{QP z3=VgC=zQMA*K!tDKkgTo_R*R#ro2e~BhjiRU9$S?M48XnPGhgxv~!9xP10g}fIfcU zbZ$kEi}n&qHczI-hmx6Y=_4F?@+PQ6CbGSpUJGpl)#oU6am$@&=HWctKg&HQN(49naI=f$& z4t^U4h=+nJCdiXR}_;z5j;39+y zyGHv7CL6~IXI|+D(_JKlmy;gwM`rHgS6&Y155%fa!B zS;op@`wwzkZqdfujG;gM21V3ACL3`Hc75-MX_Ve;{H)OVioW%OTDaWvz7vYB&U>qkBFY4>Ee~2bS~vJrMvH-+$}Mb zY*9ec1IlQCZ#i|WXYAD-ONF^Z7YZ(ulZ0~#GX)d*Ny3zWPQp9V6ZR^~2r`SCcpdGX zypg9n-zM5#GiA&a7mGS&PXGK4Zd&{UF8C1Z_Zsyn@{=xU)@U

    ;e1oJA{!MNq@k z1R7K^nL4e_srZ*YefbbY9}gB&YySyK4-=?#M?KvxWP7Qeex#UU#<)5DhGZ4~kd~?v z6dOih$3J@vP4Gjdb|ljI8Iah=@}2xj=saA9uf2KLup$i)%wk}3l%RIP2y!_ZSj{{G zM(xVT8=-<*i*1oR#|`Q-dT3<%)a>H#fJTjILKBrve8}_W?U$1SfX?|6FaY(oqXYi_m>&(8(=}TzP zn=|^PUSP(&*kfqpFdxcwU@Yl}v#G#g7TK(xNj~geFboW)`TsoGDmcdVYcD3l_f<5@ z{x)s8_l~m0eM@Z1PlRX82NqQlg7=t)zMW~X zHdv1kZZXp=qcL}Z4>&D*lr|cpu22_BgSF7FJp?<86tH==38YHwarl809%@%o)A`+$ zKXV(UCg#y&pDZ#=OQQ=*E6C{EOZv}M8a^d5SQ~eT)-*5{jYbvu9;>CnXRp%RYdnot zJxhfL50SjlfAo0j7V61c%sfETSuV(hLW%?F{!=gV|FDuS-)Fq2joCCgtC;dzcGJ9X zCu!K=Ym_>BF3H)Ar@t|5ULbLtt|ncha)W-l@Jb$i3aSXdAqk_ttJG)uAGHrWPqJ=2 zE&E$dwj*oEEM*JJNo^K>dglu}zHbn+ax#UARr7@=uL(lOH)~;>t*S6S;u+s=xPkW( zU9Pc<_7ka#d^xWj$2jlL-CRMoG36C7zD2q--F5@%y!IsLdl8f*xtNT@=dt~N^GJey zR}L?AN#bA|*X(IPrK}E{x?~r5e!57}Y`3QJ?T=*c@r~?%exvQB->KZ@7genmL#w(R zu6U|3y+aRPsix4~WsRzvwy-Jm!z86N?92^B-Dx+3_B)~Tl_l;g7&Atd4mPK={CeRK z6sXH0O#c^~fg7S~z8;(DU7<5Kj*?u*0aAFilNz#esbOdmXj*vjV zB=gA`+@b|)H)urKUCM5_OZAL(IZX2s+59_7Vwbj&LLSp~^%KbOegb9bO{FL2!YGsZ zSCUm@>3_P;JesQL{r~1M^PFVJSW%HCx%=56sgwrIGnzHhtfW*zp->^Eq=`mJard)@ zN})ofL4zbkgQ!&U+n?{S*6;Vn@2+*6d(XYN>)vzr+2?uQ@7Fu(X#mV#8VxaKSrGK@ zCg`jd9Gh>&a2aDE@xgkKSEz-n-`gQrbAoTXjAWEeVTq*Mg7NbEtF`8KrrG z73uf9x*8aN%iG zny*JA_y2-9A@8Be^%0D8sDXpOtH3wu9t>I{IPtHpfcb8MZ_8 z1h)t;P6T}M@rTN&(a`X56C6Lg5BzPf!Q>-lFg+tqF#7L9<7fq{T&zh`KX(_JL^B$# z-HXb}noti{1KQO?j#lj|gSNIps3?C1i_Pm`#;{s=B=m$eZ$$2n_E~gTd{+yT0N0kFZ(DAzl7Q858y&jHZnSuIJovA$}y@L0UZEJ25>$~qs^}07CGDqY~ zs@TH5x%My)oxq|^0>Us1?LSAt#3;c)zd8npV-GM|-5K>nB{ z_}zR6=H5FCM(sIJqIC&+2`(89%LA)rqUIea_?0CO;qd8K5b@^=cwP7@nEWbq*3h1G z?iB_4W|h#Te^H=^3l!*TsqlG6%h8NlSt>hKp1!0i!k?o~-;S0Se!4CgxX6H(_kIit zVWr?W6rs}TCd^hWgk@hC{M^A|;Nu!Fi&moV-HmCo#}61brh z$8j0j)^15nUpY`OV@Ddg$BEj1v!y{@_S7b>2R)y!O--uXAvcJ_sc*NTM)DDy(nW^y zs0Q%!YXFrk$1!~VDZFoV9hVdsHF$xiqcjW>p@$q0aWKTK~Tmgu+sbv zFTZ!fMhgYnV5>++n!kc?M4zTA*MsefYRG!4-*9Dj8CTNFAVYtxGzKOgEgWqMs=f8r3Te1^+f7k)|cCR?5 z_JP)+7qBkxG|0}%0H41%;q;1YuvA}$8kD-xDgDOMf!-sj{W1ylA}%yo$)2{0@9y6o z6Pi?|NL?qq0QWs&%+tIFuV=Tyvf+QAPem)3-fx8r!Ib!$cMLydp2CGb=P<}H1+&I% zMa^Yn@%(iwjO_f)K6&P|snVV7)o%|LobJZF=F6}>OT48;FtzYg{c1^2=>*byz!9=< z-4}A-%mOOLJ3!ODUP9CMiFBU&MV|do1S@DJUq2~AWTh&|DJsC^IdU*CqKm9KCJ)WN z+VH@sCk%Kw0P^)m!c?EB@bcI~Fzdb!CKnUDUt3pvbBVI=UKQ-Jv=d zEEqGX6CXj%ta6yAF6J{46)-ic8sul*0g1^w=s!=HPMgW#=JR7HEp3F*F<-zTU5(E45xnR$cj`5K zBCQS;I>z|%RNl;;4xQVZCK~jhX5+Q!1CeLEbHqDP5Z4Yvq7TE1-g+hK@dkVU5W+amRWrsUL;u4YpXZpc^KZe`B-fzGP-&pEHy4 zCiZx;91c#C$1rg}WzY?_&}s);rGG?PR~AGzuKh?x%+rLjGZv6$sseBCwvqW?s>!f# zm&sIegUp_An)ECzctUS!f`ZzlJ^c>L57b8Kepx*u|I!5UPG39-l1}OzrzX))YaH zAxEKA=re1MJ_MVF6!n~j|LD*9SjkT39F7-Jmh%8CR%b%gb^#>H{cZn<_dD`|| zQD`>=(>6e#HlVl$Y<8rhYKKuLrIFP1{x~}7`&8kT@S*)4_M^vJ1y9pkff|2(Ex6C0 zpf>Na;1ET_wcnAT|EC1@oo|EO^)ggdp#ydcmcXb9@ff)x5gUt+plZ=E4Dv}vf4R*V zI%qQ9Gjzv?vQGGAj4cKYv%{z({qfIL3AR;?L)R5kG4_BT`Yk3%a{6OtTO*sfXc1HN zm|4*G^d3^ET|~uwZzG62$>VOoXqbXPP~n_kv{!GiO1cYD3{_u#{}yD*@t9KPJGf?%^!Sg%N(KRenW7;??LCl z7ogwR46k?mg;S!Q930b)uAQh)dpB9oZeIoSaeY7fYxw|LZZ(V+XnW9!7Cv;__%UK0 zG?I=y=1p4-htPduPTEi^PsgmO1-XKB7`6|g#_=GOn}@(fk1%-Mnh*Pg#$n8>@1Qu!Z!E>KW5Q20#2;7qO~g05C*!}BGcoT% zAf~)sj!QyTVcX7yIA*>dj_v1$WAuBVzM&=-c0OVOYW^(Hp;9`?PKgY)n?VxxYLUb{ zv&prty~H3pkqm!xlsuBlBa(x)WSytT4+^k=-L@iKIBxf1+To8y0TW4Y3W`8W(;frcz9yn~4D+c*mVzY}gzBSgtb@%dFX8B## z^UWQ$YQ}mtHfM@-^@?aoW53@Lr{%sRs#_YNSEOX^iD%@}+Q%eW`!D%rY$Z65W8p9^ zfCsUg#CRSKB?sf7Q}#H_z9#B48;*eb^rMh@{S;`tJ`GbpWWtQ+B1b{+919<03-*K* zs^e;)#{4Bzmp+BpQROh^E{AWEF2W7>6o8#az-C1j%$u44S2eStuJ9VXd0q%vF1KK& zdMR}OQ39iS7sIc;MeuvmEzr!k11~Zjfs@eeOdatIZk9g+8}CYRyHN>`YHI*Ky@O>V zTft{*7o3e#reO(MblY(wy0X)pUYuo5jH_!cUAIm;tt|e zdV=`ws3M6DKS)LVClXctmiRB$6TPURu%T+8(0O{m*%40v#d3WDx(;Q6^}nb{@aTr*L0i@}*f8fh^cqnpa)OGX zx%?iCJn{fKqaMJ?ZjYdDbv1YzKLd%yE1~Q83Hok-AVx4~a<8k>)GTeP5^O|=_wPwJ z9k!+?d-bMXyBz6)Fh_c((t$>{TG0|66M8yL=sCZ(!Kc(pm~ER4&hZDJ|F+$5cc4GG-u^5)L z1zT|mram8uSNb|){W3Ld{(OfWOJC1I4U47ie-tDg4j&|5(tc!#|7}Uc;=j^{@;j_b zIfeypj%Dv}d9Zt1UP;r_q9oq3aguxZL1K`YEJ<2PiN=B`@?b&^iRe*8biIF&!vl0+ zw3Q0v?^cK8`SOC5_L{U#MPk3j94=2=1M{|whutb>FlL)Rv`(^vtl+`m2tpIO)*ntP zhC`SADTs2s0aaxPe|;FZ+{}l>{+GdM=Q+`P%7WDogy&D>t+-w}4PokMAyVZWs3)F< z=V_-w*YOm*y?hdC8cssNxGeb6d=7RlxeQHf@=#U2x%eEv6wTpzffm~Toa4w@TDZKE@LRHlpmPtDQ7Oco_Qe=vu=XPIYR7i(>M%e>S$ zi(078-qv1pH@dDyA~&rg^Jl*#D;yL==Ful&d#!{#Jja9&EJ4SXC-NAVA(8K zx5_D?XPg2@kEFoBq!bw0c?6dF92Pa}B)Hr?0fwL7CvwhW!F$3!*cBWP9|YHSUrh>F z9LRt)F=wd{xdbix`LIB=e4J+7fpdEv!TfA7&Q5;1nZu9bRs`#n5l!7aCW-HsRP`XopWb@dgg43XqtV;DJF)tf0s zy=D{EYGc_)3AzvIgBH(p@S?gu-rYhlxLyw(^zvDI)Kpeowoe*x@Rj=-oz+Br(+<)& z-HP0vF-lThSSfiIzn)x@KT1MfdXZ%#iwJ5Bgi)htJ)b0Kx=o>#yiVp3S%`=KkgIC%YVmO}Goy;e+8Z`8d0>?zr z%gP@G2HzZ@8FL#R5i32YKnCxmgsPz5AOdw8J8HE=C*DxD><~C zUERKc^>1jF=FVF$SwH2cB%<7xwDjFe9PY+Rel|H2-ZgDvRm%;qMEK`!tUJq`8tR#I z#1Ce$wT0DKU1hbtTiAd|3)UH_%ck6qm41%$l{)Ecl(w9FDlOk|UaC8*vmjpYh-Bug zcarccpQYtlKctSb{mJZ(J|sQSofO})a~tsXdST1w{_cNLu1Rz~w~);A3#2Krf~3_p z6H$8w**Ig!tabnkIuwQv@q!s!#=^7aao}4&7WVmg!!6;Tx-xdcU(Ffp@YZ7d1IRriW zq(X^tI%s|o$8u>ZJpOu6)Mobx=Kfm9S~U@NPj-f$p$6cl_k)Za#fX3M9`f1Z&?@IN1$*`R+ zwygUEGj`7nr3%{@NPUh|sYzI=w8dJ5brKbpwYx&<_=ri0AN3_0Wwc0@o*@a-tCsj( zyDz!CcZ3e` zrj8ewymEllDv=jFw2mYmKS4gk`IGFP<&w>9(Klxw_m=8RnkWqko*)graz{E^>$*hk zN1DjcXd}a-6$Ll0FIcY=8HnPz8h3dqsK+lAxhhZCi^l8hT-*t!oqd$~3|r5FPF-bw z^ZR1-<5hTL^(54NVTI4$K4B#xvCL*pBnv$o&kSIeWV7vFN#Fg6B=DgIM4zfBI~Vwp zu5ohY^nfGnClku={;lng+1gGuK&N<6rjJTMrZW zDWZGiAC`aRt=Lw;8ajlwZNyaO8ueD%;u}+-+fgK$F@HGu^kWOTv^;^RUrZ$}rs-t& zqr;?ddn|EE4dV`k@2>$MPC!%^?5_Kh--M`EskWi z!B6*N6@}8jc~-1k#+B_@;=^8cpT$llKjaUu;Lb9Le!M@^= zLfhgCpSnhgY^+i6{ZuYHyy!UlL=%|QdlV zB~f2kh5mK6YRWbi?xM-+tHry{Us5vC* zfAXTl(?E}?k8&qDcN0nMx_RW_@?}C_@PRzEFaz?dFZ``?1NnRxc5xt;O z*#<0XdV!BycZk2M1n)-GkX}0R#A})sksa~0uv^JsrW>5VrdwZPU4~a#!js*sx_&q- z9Z8w3fsSPTYEyFXMFiP*R!RaleIfm64e>eii`?4P8yc$m!N@rdaCfN#T*^4c!fzjB zE!BJ2EV6)|T~sZZH^ZJZ2U2Nv!YB6JXf0c{E{0uo9K?J?X0(>ImDHeLAL$*BNJ-7p zTypCEelqu?9hvz$mXx0>B7Jt&62+#6BoiN#D=9iKZLcLLmzl%MK`rF4?QOEH=oFcF zU?UlEOp~lx5MDSmU@2P^|B2meH^H!jcGz6)h#vbzKc#dKX2v+6{6|x~JJtkqSx+nq zv%xcFebD8|P#nY8VPVl>Y|J=<^PCfKpjs@(W=CSnv-P;?#eBSDG6v@n+{e}qS+ zkSTrL#nd)>u(3;wn5dUYtDDQDkwcQD?cr3~@iDjXuI+303qNJZu*SQRj)HCE<+FU^ z*zt~}tx|$+#+J~0%LOiKNT6z)BOJ`Kg2PlWV0&vq{3=;k{NyRQ?V3(j2{#zbNH;s&!$~d2vDhQv@sgN6F zUe^QiW578wtH&J@WTyo>m3ojSygFApy20X?huM;GNvtR!o~6H?!Ez6}lapf-iLbiGefcW5;iQ?7zbgFYTIyts7>bvrGUQRcu4Q`NvSE^$s35 z{uFEUEAU%q5hljx<3ZI6*gy6H+WgAI5Zh?1oxTXYw-3e*1GKP2QqQv7Z?g7=ROWno zD+`Y%?D%_Gmic?9^wu{S$wg*Kq&IxXn4z~xgYcjEPZmA3(!QW@+6k_n902B#_HbsJ zF|_E(gST%Z@zO3J)6a?~j=^$?U7ayYo_2?p$qxQqQg2#A-ZOhR| zt1*5!Qhzdrd4ytY<#aT9;DJp`hvIWxZ=7f9hg-_rFjiw2HaNSXpW`sp9X=FeH+W%N z(r7&I>5I7+Jn=+{7mk#lggxU{VCDX3Oi(|HU-qY>W7I+HnVXE(4N2&t7LT*P?7@WE zo!A)?i8i@=u*Pa1?l_!`cYB<}fjReaM9Oyza#i5-4*$lQ^iNo9(}c6vHDYwaN0dGO z4BZWHqqOimzC0X*sVC;4<7pSnHPFS5o?n>3Dk;<2x0l_nw`b@!Lz?*~TXLXB1Swv1 zi!6B9L|RUDkl2jhWS-kElJ(&mN$L54JgKW7buQ;g-K%hN%+rZHwmB>rv@%J$@BV(a z%=HI5yS6*pt#(A`J_IA<{P6PUX(+RLF}_&qi<%93SbM3CMZEsOj{W|?vX9MRx^h~q zZs!)3l5t4V^VI-y|IHZkWX4Cyw4kws8eAjYYQ!<^pG}-nvdO9B(`@{QtE@Aph*|aD z%2JX8$)>Hzq`uloV#bPD;vrMae7qF>cO>G>1v$9dEeF}dEFAkP6Rpcr&?GAX4`d&} z*)i$(&n^pfolan1ADNE< zmbcL+?V-@3l_39~hb8-R@mco^*zaE!1~_D3@6j3fH7pCYn{sgI_iH%zM=`D+_!Ofo z+R!~tjZf~a&66&v@b>j`d_kie*O;ixRUMVN`b!!9bNe4`v3`xWD-nH^j^PBeaEy66 z8oQkrT#d*dEG00TZ3^*Zdrx{uM?bkPIr4J>`5eEN9GpIkeEwA{8L(lpB&zI~dwamT z!sU;>q)A(@N`0Q|vMtj+nEIKGtlzsVHaoqYg-o`_dF}**`igG$yqS2s*L3`DIu3Ko zJaN`Tz-9LK_-vI18k)RjG5`H4J496=Rv}N;~#_@8kkId4I{04C zo*TChgDTEq;vFg4T)l(-y7w^t`fY3wQE0^XbTi<;cPjCSg>ATJQ8fl7T)?#j(b#S>7hOg=V#(8gOhF-&g&U7$ z=arS&(q?TYqqc*M+gZpuniR3=t~rih?1)Ql55}YK+|hlXC!YU38qIc(#yCqq9NIDs z+Xk=17mp%vncPkcyc~fBZX57s_gSbfm|#f-J#hOQHPki`bDNP`Xt3`SOZ;@7jhp|O zoj+@Y3%<+XvyfBlTmLMU%?>|(=KJbUg&bo-IVlU;~pw<-BCvIlVRfY!99J*=g)MqtF)aHNR&Q-a1>bK72Rm@mOj#b5tH^uC zsPJL$)cMMAE#A-CfCn5jRVUO&>F2d?VJKP+(Oh3otB9-nNvQlJTc z^g)X|-j?D2roBdu)IzMbJ&vPZ>_(Ya3$WiK3GQ>YMdLL#cy-eVJTrd@>h_4jhRx}C z?bLZpd2$6$uFAu@b=T3#CLg_m^Kd}wMbu6`i!VD*;`HO0=sF||rxqQ>;9vW39Be>) zQ(w#&V~L59e+=$cv3Gsnuq?CBtajlerm(!2?e@9AHuRLirhzVa>x%=LYsjEXP$kpO zO=Ao89cD=nu1bp|PDn1rT#(o;*dU4Aw^@?8x|S7Qt7liIePD?-32a_K5-AG)L)P4V zLPE`YF|pk`p;!JSu|ue+KMXe-Zn8t{X(WE%OllQ_!xN|zFf}*pW?xx3s z=WFt$BN`mHYI3(fT0BYo&5)kD{9>FQmuocO<|mAJgpLW{eBPAXjI-n|VZC|NnSp%$ z;-P%nqaplF^bp>uG@Kvx8^KG{D1Tc&Oz0MB-dcZ zuVU2Oa07p=y@da)E@Q-IDXNrJ;A-oAv1_ zIe5)d65R2MJ<@JsPOODpeRzy{XKW^>Bb8wG0e$e%XeP08)?|*;3Fi4W5UUifU}^m` z%-_|FW1`!z&Ak&_zANw|9Sv^QWXPkKCI9lb505_Q!mSI3@KLJ6x&9&vw^aoGvIF?6 z)xe+3mGCjnZv5@y!Cb29!pojGat#xEetmc^zN5;5UmR%3=ghU@o8+zd7)^7|4NZB| zVq-pLf-!G(HQ|-_P5B-JOMbfDp2uHy<|@U*d9Ir$SKjKyXF7ZFJAX#<`K_b5Z0b1f z`q!7Adpn9p>wEA+6W#eH>p@&^g(ElKZ_S(b^x&a-hFsNLhg+#>@^({AZnjXHA4%8e zw}zN-lQ>gu}UKw)#!+QMwbWL9Cs>~f8{>3vl8<8G(h?_PPVCsX@7$Ula z4nS;s_smX;}%>T3Ni%<#EiD1TBaRa@f6 zT<7m&dy6wzWT)8eSJ$wia&Bznht-ACEq@JPKkb}*r~761W72w-@I}leQ@*mxF?mc9 zn=eUuagzis?IKHFwUIp*d&nbi4c7CIJ!Xd_VEVX1e3ny<6_?(KeXs?un*6~LhD!Xo zsWzXjYsw1~Y`EhPCtkU3Ab(Iggqyu_ZZ@oYx%g!C$2q@^(XQo_SK8-wIUY8dd5%WS%x3B5TZ* z#P^ui=EyyN4CBG`M)JI!WBBs1zPzg6I9}2^j&JDg$EPIt@yUhb`GUIf{M!^i?(XTw z1A2|;C2vRbEyF$d#=7BLv28H-RUgRd!vXwPX+PdN(viE)>CLY{x8+k}t@x>LX53(u z313h3`LZ$%Zm~^~`~CcZ@5^2x4!Mn!(sD3jd@>rugrjZzL>zp%7b?#Dz;YkNuv-h2 z*yv7I>9QqJ1qo&a?g<$&lE~L4Brhb0*r)9w-{d1n1$vN(lADs$4R;F1E$q*9AEvPM zMmrYq>S>|HNVUSC10Ds-wi^~asea3z_=r5onlCIhuZGQAF_K+btuD3Luw9arA1fL5 zaE>&7`*FsuIN-bA+t91!80yqr!%x;F_-OeM-fXImHMi#!PHRvqpfYQl3J z%z0y{6<2&}!`EE1=Wz;sc<_ZjJlmr;&#kfL6Xsa)%04~$6h{+2$4rm!G1TBU=Bx0| zg^JwkqCDS|C(C_T{Smn!t$5Dj6T12};ggup*goPn{@$d>J!G|c&>vHt*WZE1S`Oki zcO>FG<-t#td-2vfFaBBCi+9ZTj9{Ca9AAXKk#+Kn% zvpoFfa1uj5$BWM)9D|(a-~+E=xM7tc7G*Us+w=^kAf3QWAAgWm-0UT7@#%1{{$3@~ zQE?-+*CL2-MH;yvfa=v=M@XZnS$W(2kpwK?UszJwi+OM9l$K^2NxKC+FC3SsEPe4< zQF`jb6Sh6Pmett4VJ5>GS>o~!EX%E(`E)AajU#%f6Jw7v>pZZeR{*Xu-i|S!_v6c? zbWG`=gI~=H#QVI9LED~S+sp=>UeGFPg8xuzsla=LD)GJ7l(}oE3hyXT<13`y`1tOc z{M%YhzOAzx_qndj`%jkR3WgnMHt!S8aeIlq8y;Y6aw$&y%`hy01f;8#j@xs`iFWfq}2f43l z^>?3>de-ghshq-&b8Do*+NY)0$|g&*<0eRD?3YO`e3nWt-M-6uj4EZWxwqMs$~$aq z*Atfe>pMF%RUNNbl ztH;3`zT&`!zv#AKj(=aGz}=!0#Wjo)_dKM+^Pj76i^6Vvww5N(h}7UZ_m%m;LRtRv z_76<+`GDioYms{2M~^Coi+*0k#;D8axa=}gqbpdp=?ea8xPrdk*RW_*KF<4CfI&sK zabR=}jv7{vPHx|DvsM=_P*vntZmaTZ*&2M0uNI%_qQze>(Bf@BwD@d4ZNB@F7QX^o zJnyb1udeIHyBjO>Mek+#(BZ#vS?)KK$!Wsw%5PCR`Z+GYU5$A^O0bmYV&J|^oINTD zLxpZ6u6PsrOjv+9JALp=jw|+Xw@0-Zf_3vs10`vK3HJ9X+qd-^J6IUU%udc_&QCqq zGUCK^78szon<}bL}-8@VFFD%ziBPtp*&b*@p65hG&H+@LR)____pT zzVe|8cQRM!gE};LO{O+?`KZfp&CunGzpHaeA4Pt2&0j3v(t^Pw>(Fm~C3>_Y9+16) z%6HG9%AZW!_vaY;Xs6<+KFRpY;sD0Y5QG3=E@Ig`TZK{-}6j5q$o#f zt#?e?bLc_oj`e$`jVB_dbq>3wVT)p=hC5TFk*e9!x^1_l{vqpF}L zosYc7STsF32z&cbe8Hw+cKmCrE2mHnDnR48^pcAviwRXv_$-#kr(nbRXq;%j4#U1p!M=NjV&y0YWIau=NKO?OeEG^IEG%QDuE&{X zTqv`Na$-^A+oieP9!X8AE=dhC4oDqutd)iYO_BavI94hhI97UO)fB1I??CCANg>iV z+xJU-);CHO!pvF1K`&OaVj+vs-NA1Bj$r+-$;ima%E;VQmXT5X@4MpXf8REW)&Fjn z``?XXy-6(pc@8vXWMUj;WP-$fd9kbz%TlrY@9jUoGvt3Bgp0Mx|J;xK&-%2!j7)zS znVsSn|MPzf{_lVAKd+mr@LzB3zh5i1&3F5sC&Ws;<^RvEv`AK_pXdMkr11zD8HvY# zwr$pz(j4zM()|%0Ik6|bzuAy3v^Au+H<;1=FYIVyRBt+3 z==XPyv7%m!&1l6tkso6F4!B7*JoSGDva=rvVL}lUhvh(eWfIuDj)3QOQ=ssRnb1+Z zCI?!N6Zf?iBxtd$#9)3$VM*^5QVf_Q)$iRP-8MjnO||x6$?>OHV8lC?c(6MTnLQTc zmTbb69}eTe&Rl$Y;vs6h_>5a475VKRdff4n1wZ5I$V(p#;ROkvywHCvPrE*u2Ys5x z%}&hZ$8~0L_139eCVLW()bimnPu+RN-G2P*UUQx^Qp3uNer|+$%Gxg1rS^R05;Ef3t2b6fa|B< z;MFQit?d=4*?L8KeWGAIC@ItDovK!9Dnc zgg*S#kYT(e$eVxaKZ$$j%;7VGgLuoIWxVFV3LYi1l9!Y&=Oum%x#!S1+`VN2&&~GW z9ic9~e!3;sv=g~1)7o+S(QN?T9yE@N6x{#T?i_kRnT+vJ7}8q4UWzK1Gy_@sh*uYovfljTfQn#-x@{wYLycG zR-!~JTa{>2rxHCTr%dNGDACgGid37+QvsT${jKGw>oPgI-&%q0kWm&nMQYS~l?Hv^ zQ{@i)4T4v^sco5wf$^F%PI|N@l{>w!gXlcTP^xzkUBk_ zsZPgj(WNN~rqm|Qf-0(+iA)$xy7h_B;!UoE#CJKM5t;x|i#CbJYr)hw1Z@15!;gd+ zFw}P>1nB9(gxc#w-OZcK_?;t(w=s0@BzvSYzUZ;|^eKXm6T`Gtm$Ich6ws!|0i(yv zL6`Ix%)FY7d)gkM=o#X!XEN6}!- zD%4zTi_aHH#dTYql&&%&s;BRgzAG%CcaItH*KQAtIeiSaEXV^Ic@O&ZegV-pKSGY~ zPtervg!v0(sELsry%{S{tI8GVb9E)EVWv!%6)4l23MzD?z6y=1RHj{0MXGj0p8iml zqh^C;Y2^-C+Ku))2n z^n~=mS10!J)=s86qkt*y6}7%LJ6!R`AGgfjh5mm};^mXK(RWGAGo|D{{PgtPG)2zQ^;!8xs&2e~jxED6Xy=K9o^6Wr2d-7K94KWz+3faNy!Fb3K z2zYc2!0;is-+K=Ur+>lrW%9JKLX1&0s#H-)jm{N*&*>{w==%m0>NHx7x`?c^kJrR! z`$LN^=qL8~=Xx|jFb96U(W6g}>e5m1I z-^V@a(E5OqurVLe2`vz|=KZScm#o*+89>y+D0tf%?@WnU)O#1qP@gQHQ zju-?j@4Jg}PzI(;UXUDvQ{;J50FgBd<@2%#cQ8W3#a1Wk%#(^(?t--y*Ug7aQ$FO982ad|#%e=KG5zSHd z@LOose3s?G-+py4#{D1sS*k&uK6Iy%`F1q;jtf1TC!y9yy{SgIFOA#lM^h}uQ~&io zbdtbK{nc}&mRs!U#AQ8bx`rNA($}D^?rKzat{RQ+ra|Sjbf}-P0Ufi~SY%U~(&d+W zifjNYDmU7WuASDG4oew8Q@ahOqpuIAD}qPRJzCy$R?cYZuQZOv6#3Ek6%*-3bALKY zaRvoxxNdGQT#ch}y6k%FP(Ot8$xY05Yrr-2s{Dzi6(90$ z7|-#U#A`1uFCcJj}PauS8aGc z(dM`N{Xo*U3{~2XpxS|nxZkgqZQov27#Dq$?9Lqyzm7yf{i_S07WM$tQ=4JCU`Y=D zsX-e=&PkAEFS`6(Uz(dXkRI?JMm=oZ>EqVnlwKc7`_#J7xySm@iZDxRRAES$PSv3L zOO)vA(<0ZlLzb>SC{Kgkl<085_%EEHPLGJp(aQ^UXxwK5x^%fIy`Eu7!-m?^&9zQ+ ztjOdroa{z*J18ys?M)Zx`qF^y6R2=q)53*QXqw?PI`)ppkndqci!bO<^EPeT+@?hp z2MK1#QyqGtT9?ifMKBvRNh5G>4u7(CL z3JnQDM`mjSzj~O#4O1CdF!v_OP)#TI?k^>qo!&`e6I7)0rn)m!Jj23|{9yX2y)gOF zXjF~}#qzj=Sl)aU|4ey_7P(5i?3V@CO?2ZG%l!G0CBgi>{dRsLAd0V>wvW#-+|Q?x zXkKor*&aIfVOUOB*lKM67B%E#5YX}kn&JVu^25OJscGO3RG?1aa#;Q^2O+7lb(3CbLS<(^D ztZ7=F4IRGUnzGB5luj_C$tgy3n}rUYXRl7zCMwbX?ebK9iad2_lBY#Zih^~jME5LL zqUnW7^!RdR8nr-`4lC?NJuP+Us-1?kPG0zv_x7UArM9$ioFn~eGKgmU7)}ekN6@Gf zUiA5LZ(6A4LoaGN)1;6d)bo}Roqo=Uk_(2^+{c7IPBEvc-o0p8tR<~E*n>7d7rg8p zs&v1uBHauM)H(V!{HwSH$Gmf3tEI@P-W&o2ZWG{Yln-cRkA=y^3%FY!xEkI@oP!RN zB@~F)^CU^NTZUA|U@^OX<`xS$uZ8Ec32r>J5=RY*$5r2QPRFWw?T#=V&Z|1;H)lMrqG z@^Uvmy0;3SzgLFK#Wmrd{ddtQJwx<*S71h)5gu)gWyLGLxrco{MYIN5!`n|w;pF}# zfRHN?6H@^V6C0pH+5s)ol&DXhCY>VYS&<#a^ySj-R8hY>y*@{YI z+|=l$o62;#M2TJ&^S$(IiZt_`B5fM3NZ%&NQ~yj^TG!YGcA@{Ee1#0<fyv z`9Uva^{GaZ88w<>N|(yo(A9bUXyW-nG`V6JJ?P_3*IpzvrJpM;e`igvXj#%~C6Sk- zZb2;-1go*M4;|2kiYv*#V|8fI=9udX|Pu#>0Hi+X# zZywil^W|CNhVc!7z4_bS=G;i9JKr5<%w@+L^W=AiyvHzIz9U$b&rJD+5nhk6s`)(T zMug*_8%~(xT*LSo1NLaiFGq)B5zht?n0rJFbF(wpD4X_vAVm6~eMi5t|YTBZsO^-`t>jw#WlR*JN1lRV8n zD@zaF?*zLgKOxSn6`Tirfu`ACp{BeYS_5V1qzGmDc%Tw>>S;iCy)mY1%FXGynSJP- z>n?Q0^&xcH7dINbSV9#y4X1lY^ryd69BISXzI0=IAKEL&mO9<&PZOWI(Is6#i<8{w z>ixaxj5u9-^!Ps**u4SvDwaa7RW`(KOoQA32{3R{6hOsBPUe%(eCW`2Y*UjXKE64Ml{V4C4>cYK-+VfQhEqQ-#&Zlcy z^2YO4+{ws_ulZ@lT{H~2MyeVwT=f$NeSe5~^D?n+?Ft+^#2(uh{b0GL;u$!%6-KU~ zLF~?UkgEMd;i%$TkWGn)#O|jd?^GfDE`JDpGU{Q;gH{+{BtsvJR-&#xYSi#yH@ftQ z2354xpw};ojI}-LH2;c*kw&aW9{fNgZ^}bbO=3QBcU25^#5>l-r-!n?;A&C zltlL4GLsO=`+c3HqR`NeG!W9(_>_uBl7@y#q$x^UQh47_rKpUCsAwn}A`LC6-~Ijl z(H|X0%7hvj(i0c8;*iql?b8?4@}+efry>Mu=p=$OezgVSfe|#hwyB< z3?Cf$Iv8pFG%;mq51p!LCsE*2zE^dNMw}?3@eO%2WmG2B@IE8YcNSgrv8L*)-Qw=L zBC%sdg7~7hg;-?#pNKR9$~FW$Nzz2Q5_y^4>`k2)o6v6~`;d8@9Y0*bPVH)ERfS5z zNhc#ANY+X4ttH{?2tQ%L@Bo2X_zOqhj1rz)77O=cT!pl2_JW?CmC#XdCS)Bm7x>b( z@Z-6y@ZH`{SlweK+~emqf2Fz*o6uVbi2lG1Ye?AWep^||IdAs*rZH<;@?FwmTqTiy z%9$P|HX;x0gW}h3xL5GsIBH9cr&^7@)T=O`%*U7VKDUa(Y@U+(s}{Oo`<1e^{?YqW zvdFrofJIQk-drV^JXFGkJBlzJ$InTpHyU2aAoYF^&x!9M**P6#>i>bZO?Xeudw9;& z>u;1)Cxt7K3NX`B!>emXJcN-mtqWbCo#uhep5gGD!_ViK4+H3S%~y3p#G*Le(ovVRfvX@IJ#y(1>smT30&>9WFM))%k|PEt&qpnVcRr z?B@eEwK|I}oIRNpE_Gxbik7T!x(YLhc_%48a9T3{sEp)wgSPm3=m&9AwmW&QT1Kl} zc2d);WAt)IAsst?gH%0h$=LG=Y0JN+_Q-a6;`4*{R!V_n<)CaVk11{nSf8Z;^$T1# zN{~e%N#k#NH=XI#MOkM)QGb~Z8nO8^$z9^RM~8c%oY(LfydJ2E=ghn}4#+bej=gh7 zqRlZ7?n>j~_+l*PHIGHk(6MORJQ_NQq3F;JL(BB}7;$AGMxBbmv2t6?%e2Ad;(?qI zIt1%<$KvWe@SfQgMPD?q<+cJ6R{o(n&TYMK(?I3nH_65JG+AVBBin^DsjAU{562U@)8C{>u&UtNPvqH|XVX3vOu(pG} zm@X%Tn5zmu5_E(q?+pZHBO}3bv!RfdY9RQ2(-)p=83-|vhQe<}6Jc_og>X;GPN+(A z7NjCQgsqXog$d4lMt;ObxO7xc$d6YPwrRg*r_u3H(yZn z?p87``bb{CzfjZYZtB=84WXzP?v9c}vAi5k$@W6?87agn^ib2mU$oBl7a2eNO}8HM z9Hhlu-}CB^oN2s<{B8yBfzFT(7GqhBAFf=Q1pV?Em}?mdrHC15zZwLUIvh9U0 z!JoS`#_ocpK0;xAAH}vhkm}5 z!WUl|{L_|!kCin3tCvF2QO=oUTo$s@1iE!2?bLwk1h zrs9~hV$tL#VnI|Iv?ZQr}PPhx%t0n%BjKhWurhhsy~QsIMS?FhKahy*#y}ECk(^ zgN0BHkr0#ZBlHrF5RNAe6CMTF2w6n~g!V!0toH0VR=#5$t6m?+3Q~^pjJ~tX!*L7O zV7WfE#G3hU?8U}J6-dsxX-l;Bb&3wGdnon`aidFTm(k@3+bPL9n~G8kNIJiSrsYBP76IR`9O(%KGC?B->B34H)Z25x$Ts~vlG&Acj<+srM>aMt`BmL zaW1d3F4iZS!LiyNWl!9&(Pjh=FAG4|=&ATQHWcPB!jPpCg_P;hIHD4TptEx^X39)7 zzny@C4T+c$umd=`8(|Ok;mWvu7}8;ZN!QHqdWZ=|O)!IfqbnL;dq8`)GYUgFr|q8x zzB|ezd0`h-41Z2Z<8RWej$`z>JCPoFjUs2O=i+ANh2p=?e?%1rnW*#YX3_6+6(SX@ zbWxnMZdq4HsAQYkCyBhA8*6*Nj0HT|#U5-q$$C96Vmrz|TZ=k*nwvowTF=ZpnAok2ovrl(+~Gg8Rw9wGd-@DTD(SPKF7 zxo$hAiKV_i%0{Fov1G@SOl4^`>z#Rp>Hj&v^vss9gzxSwZnY{q>ro|f-!WGbn_cSZ zl6qMDx<`{vters4wFwk#y^CykUThA}!_)pQp_k|H(ynE7G)%RTHd?+Swe(hM=9+A& zUk63he5J~ipR`n_hm_=`Q8HE*`A&++>$^z4ny{54!j?k z!^qSW8pjOrpx6MBD{Rp)+X<)E@f_Ou<`}zJ1Ai-f<4fmP`t8_Aqvd(#%jqLr-%Fsj zqh7T1O}RMec8_Sr&umYZy6WL^?>CFASA7+^e^M6jPDv8&4S7-^g~Ru`2?G8bc(gM7BD%}^Q=^k`x+ty_Ht}3i;=uyEqPyA>^d1?@zFlQ zo=aN7&bwB^Oa(W=^CyG|884yfw40!P%u;BG>LUc(KV%7)kFt~n$5}J?hv|zO*`(S+ z_GL{rTc@1FdS;AgnrF3HwERm+Sky*||AyRBkw?7PO7E|@;@wb6dbXI3w{9cZ;A}F< z$fw}kLJAuvP|Ah7q`U7CeaLu5O|h>?>cd;=DE~lllRIcq&KHv1_=ENs{3Y9cz2N^& z5!R#CFlvTA&hl@G%eNu8If`HyJqCkKgV0ZUCRR+DgWI3N5z!uryfM*OcRC8P`y$~h ziNOA&^I-QY7}v*aN3l~D>fRs4uiqzeYyC-_-)9Dk3}dKI)92cS4qpE>LB78=wqCQw zjddpI=@|geTsZ{o`b?L*o=}e$qvNuNscACLzKHdpF)we4Z_PUCxtZe2!kl_Zo_(Gm zdShuLDy*&+Ej=iRrr1W6?Y^`@vbRu*$w-E>o64c=lFeE+a!4}w*>7bx>bA4BbJJO0 z+r4bBVit3nev;YkE@HChDp)4>kjz-|o<$t|&5~9q3f?1h1(i};p=;4lA<4sCn49D% zthjF={4$Uc3^(86J1`emlL ze`H2ZhTMUfm}xK@>c#Wm^*9`7x+1WyH3F9uBQRl21ja>0;Q60<@LRPDXSW^Zc>*V4 z@jV|&3HgZZW5T~dMmW%057(}1;flUKMtw9z-6{+Gi8jJq&U*DRH_)@|%lww$2vuO*y1WiC8FJy2kk#zKp?vS3=*%sxt8VdZlwSfS}Z=JH2g zSXN!emhL#g%=x}c(8~n2)@BAHKYmpaGyL7x)+^cT25H3GX^2W6Y<~LX&5dSjOtCZFqd=nR94Nw z!(MZ7S8guK@6SVe(R|qZ#2{%#24dxpVQlCzOc{0B?j2tpo^nJv@vCl zF3xxxqw%*HRQ-*hVaI1Gdt`C0se^V;t|yUSDOnk3QEcFUq*6o_mi}1WY4l&2^~M@W z)m70?-}#A>nxV5L2InIrUBi+jZe!0$PObVOnX*lbJ=-~u83a4A?YD+9 zc6S(See20G?MJW>zNeJ8XAbi}y_(GvrEp*1ZnjA^n=`~uviZk~*`lY9SZ+ZV8yu)8 z^iu9G9kcaRk}Y-bTY>)9~<2`s>E3uQbARhMAfW8LKA|W%CS`?4cPWSV)!=s3{Ou0g#nkDpA z{{|`b4@Bl3URK>n65sQhm$Mb>?$QWc)fl`4-!kt(G3>S5YWOJsAl zd5k4z5-lBx_G|u7wVeoIHqWg;JQW3X)A9N94A{8MgnwKx7CFzx-*@v-Fe?^u=l0=X zeHJQo4nX6{URb2=Mc@$wObj%@l&g9OOVhm^>v-} z?@R-=>@TAVo-fva+bX&?!k;92RjBY>fymGyP7*)Rf{nUi!Pc37mq;yeWcz%4nc{FO z_P6b!M21dEhJ3vtiL-wv+4k+91j~D~v*wEIwTu!o9@K{kx+-i4Kg02l%vta<7xp<| z6x%&$3iIDGhfS)FVV&v;ETMf5%O7^0^&V5rZftnP{Cb|T2ie8!^^meP==x6=iBH~A_pE-WLte4cf;>^e>R?=~G7T}8d6@6*Ykht#z9F&&@K zNDh^6XhUfyWl#J^d*;hSC8|Hl?Q~%hX^vId4lphpiikupvh#fKZ@nL)a{b{`8;Hbj zY-i~lMM6N`$VmjE4U1&YH9hV1h#hLG^xYNfFue=S>|D8VK zM(JbPbR#S-EI*|HtqkWl{|jHJzhh%OU`b}me@C(mRwtY zNz#8`p~T|b1if<~=omJ&9k)%J>;ppJi;|w5#kx<8gMyYd4!I-pJNvN3r9^ z{>=Fy*ZFoyv(&&d62(($lH;CpMf>~5h@EECh~*|2Q1r*C^h-O1eqPNYEz$bwBvBmhDCw#*lN4Ogl>7-(lBje@Nm3(! zl&!n|qfBpdFGo2xX6@x)YCl`6}uWXdJxEJCu}c|cjqRXg$G zkEg_=T7HSU1FXqy`wW`xm`uaW_LCaVJ>C~}ff`0%ql30LXmE7}6;7+6dY6YZPQIQ- zzJE$b{a%t)O)C}rY$x|^KWMA5476kwVZi6(p3gN=>ci(%b*5Oh)e8T1*g{j)5lin3 z!SOk+ke%g*_>sf#CUiJj#)%=h>V<);{h*mS6=9+%tXr}UjzMc+cw`w`JQtu!bpZxG zHbqYWKO0pe{PyK_IiGV(x?qdaPlK_i#sT`DO<)tF3dbWdNH+XP9WNh{>#WOk@AN)0 zAHIP+Gs5|^G>vq=_)~+=XnHfqf@YhZ;~W-+($CLlO62}*mRvjFC0UY^S+?-psuGpD z1D+v+*L&_!*7N*PzSuK$?Jv)noq?hqn+}UK&b$+~n(B$QJ4h_v5+b%;yTG<#*D+)GI zqVzt_PdG|9zn>xbDMcjvu#CC_Z&7qY4Ry|aOv)dg(v66hv^4cC_50O9XV!hEew{q) z`;aXDj8a10Gwx}y)WB!XMjSI+ACKl5LZ2C9rIR_fAGN}qAvXB4%@*glrn+{M1Im@1 z5PNbcN(=$F$>XrpDiqDOQLtV-4_tu(?PiGvUriWXl}DJtcN(4ANE>P;TL$tO%gZDFMWI zMo34DDWzSyCjR-pm$);b-1D@`aM99{?V|l{gT>{C#)x&A{KVVae8j8O1H}8BL&fcv z)`(-=cZ=l)Ula$Qs27*%{uL`$_oq{NM&zO4K*v1Aq{9CWHM3}1voMZyUdPe6YxAjm z-dM`*twU?9(|J&Xf<*6Hp~U4uo@DZ~bCM6fHzenqZb&NEmPvfyW=P`h43(xVPes9Y%EUq9;8cIfdeim(vutWbz-dmpmPg&~%LgvdX_q_n(xKzWyyr zJzPy<%X(6J-AE&TH`C&UZPdlnSk7lu{+-q(u$5S{^p9(*UtEJ${N|%xdw8(FH{Ec4CN`U z^kH`eWwqpz&(h62lXez;%=4m6f(IRr^QPwaW9Z675yiQxQ~tHP;*K%<#UU~W#7gGp z#8oCY#Eqg#@uT@S#J69TiX8@A7w4+n6Ysp-D1KD5?P&Ll0iF6B}Y6~m6e(+h{{?9i1j}tidSnih-;#Csb|#)>N`J_x~?pv!laG- z=UZvVfjuO*F^6ndouV73i)iU}Mo)KE&`ejJPv>1vX>S{;$o4hoowbqU@~_nPxtsbQ zmciFSy)lz>M*H0EgOcigc*6D9e4tFyQk+oSB@&0l+GqIB_c($qktuxeqHQc)y9a>wYMgxQUa$jv3h@20X$>LFp=&SWzg&m`SkTl2}ORYrLKl|)V4$h-#BNdSfqop1sXW8Q~|{^ zTgck)D7{g4CAHo6#OYUdiMw|X6Sujiik5j~ms~!SB(i^?FFxlUFSbj)BF-rNB>s6y zkJj3cplMx^)PHk4ZHV7Ym-4oe+^0QcGd+{0f5@Q_&bbOQyh1JkHz`b|nhcsAP`yb# z{XO!O%Ge9~7V?(5Z9nmh_8+uDQwsY1WKo;Z8!-tASZS+-@MSA+qLB79wC54{d!2wycJXVMoB z!uo>u4!ERdf|#AAQ13Fw8E{Q%tTVo^bVF92E9Sni$HpcT_$z2aYoQX7ihff}z%!a1 zRz$Ar(n(8o9Gh$YeMJ6mM$l!RRrAq-^QZm` zpnrcx(2$H_q*CQVLvmc{@+1-GDF;%pB$A{zZX&O@nWQ-VBCWb`haxY%qUvwI$5=tDinZ%vx6mFM6hpG?ImEyznbImb&`}i2$HOl9? zc9&_-hnv(?cb}T>HIPG}1{#(1n8GJNp|Nexd7fb_)#iUBWpvSg&Z7t)DuZ#=y>Rik zERGuR*`losI-~zmu{rl%%>6+k_n%bM_>-L2Z<_BYjfQGP{G6(Z1x@CVTHy%a3>Wlo za^T)GZCuw;Li9C01LnEzQXf80|4Rm#zsLjuYs^q`!2*w83_{Xa7d(O+o@{o-B;7$6 zTyKtXk-C_{c>{&_WRNnhm6k5OPTRb*sMu{4eVyw^ThiUArf4{Y>y4q2_oq=o`dpgU zFrQY3#Za|jEUAB6OB*I`pb^*O=*{BgbnIgUy>Sj9VcK+>-w;Ag{MQ+nZ=^Sec9RsJ zt%hH{LF*?r(S!a!s4-O$7nbN?cbF~e)_Ov5;wVH;@`3FPYq%`w2b=C@>Zv`&Sppj< zCDoS()?0H%rX2P8@LgQ)Aw{0y11PrBj)KOGrDT@{^tE*(*$F$SDe54-Vmvp}ESLPR zpCsj~0x~>xmZCCG6Bphnyz~+!THc_kb82bnoF}w-)njVUy-((IAJAf@I&$mvl(6On zDOR=8Gda!`yuq0OR^22s;}69+|E3(dF0v2kpcMIc#3J92q38`MYP66{*n4`U@ttBV z<#DfE1J*;#;KXx5GuB$c*hUw|DKeN~{*Ic1AJRm*GAh4cMn1)csAxAuO{zKe)mz|f zn*(kibLM=~q4+r16=~j%`24^MN2~a+t5L`8rFI8i z?`;`NmtRh$1oJ5JdAORw#%!dFKbvXip>3r3FP)5D?52^4yU2IYc3RxJnSAahQ1_fT z3Q~xpx7#+-=a*^p(>;rR%{@;ytt*#&= zeR_2EF3$+Br15QcsQmXm+VJr{spvP*+9@x|E9@QFpZ-XZdAE7XVlWIexF;sa45X=xkLU;G zKL036qlq+3ipjuq9~F;VPZKVO(ZtiUX~ew+^q_DZ&0oKn^Wb;T&0G7Zg@5a>)gPmD zaV~XvpP&h%BQ#R=Ag$i8hx2Q8(EOcgWR$#}st<5K)7&G}CzJOsXK&G_&5h)=?+a}{ zAjh8*O&sdBMc8K%!gl#0Jz@wNxrhE(#7DB_|91PsB_!zOQ}cfrWZP>K8Mbl$`S5w1 zsToN{-Lcd(Fqw92%b>{VC&}&%*TP5e%+`6gNFlqDd@kLl#dmK~iPA03?YTws&IxpJ z$5rxjE~dkuBy=FHlDuB>Eb86&$ie(3&C!>To^l!W-d#raFL@sFrAnSHexH6>H&7SX zC_YYZqDe~6DD>Q8n*V|uKm6}-K5RL)|1U@RS0$C*sUtP+W$>2ijpVCp$n(-fV?sZa zb^M_V*7a1d@gnuV!+F#4@$~-pBx+NfN&}6}aAx>GRF+#~zq~at#Y914^+3|HVt@qg35dI8T_I3)NZzl&ZKUjsr(%) zsozD$FS1BqJ(r)!S?bX!r1F?6^m8KDpcMyuGzv`L&Hu0Xeu>jr+Bda~*6q7PEc+pya^-sJrAE3q z@(B&t$#dcBYG_hRHRXP)phrH}Nq2i0C9fAqpJ%D24!BDV;kPL#s+5lP6p}yB)m_B3 z+SpAz?{h{8&3Sa4I&W6cVt$sx&R5Zdjd$sjR0aJ!euI8QmeH$ESE=D>3HhZ633RdS4%UDPSf{K+sR^FI8Cl|p*7FsDauHRdczpErVYfE zSN7P)YlaW%4ybZ=LId~yRhPTr)_+6sG;9dm-0iWn*&I0^ba2qBFRp9+Bn`hhGE(Qy zW?Lqus%@n-^G(#`kV1{R>EyC7i}vvOKw9NR?m;M}cR@F4flMVOO}j(A)>hDwD>vw< zLMi2>^WXoO=i2AwlYVs`{cAr>E3aOpb3@BWNpgqcEb8gafe+;6*c<5+b)mT37(M)+ zkUjF3uCzX;AhjyG@T!95b7qywz!D-!5hZ=#Y#!}=s`oukIy>{Y#&dzDN|n*EM|Wt% zgU6I@`-+O(-chVc2L*B8iT%_M6dn1JF7J3k7O9*K(_KUHZ53p`<|fJfxk+qb1r3v{ zAcM^7G;-J#iZeS)|Ex8gvHCPgFF#K`7mH~4w-RcIWF(!#sOxMg{a0K}XU-MT zkC}y3aPlJ6bQY4XcM0u1dx!c+zotar-?ZV76#h1L()zyD)Xndjd($`3eT0xm>X6B+VS0L32iKr%h@*`5nHWuBRWT0ihRYc^cO;Csxux z^9S^JT?0itc}6c>8!3M56B?5FkfuJlOM!cD(16t?K2u5E~CWK3uJdKk6OZxQBv$tYI|{l4lX!FE1#Vs z%hiRnxUPtnDHKuO4bBeFJwvkEJX3$b8M6OaKy$s%Q(oK^&Re-n&3s0jmDEN#l2)3N zRz>MrxpY+_p1yDnu6C^vBiWJx9#1wm?9DHVU^WV2bxATGUxXx7HWZ-QGtiVA(#N zm%N{z4?0XnZ%)&Ou43wXe~U6dJfNw8O*HCI3mLLDQh)S`N~=Fn``0#N2j6f;z%#m} z&)HK}cS%@#n~YO#)2n%R$yo6L$!vN;&AZ?5KJ*W5J*fhVSE{)7=noBg^n^6qZd0jY z6+P{KKxyK7YI^sShAX|GiBp@%;#?D%*Sw(?sSXZa_(fHTz2K;L zp`e75#fm7JAcsu3Zt}R#xj)9QY3nWCM?2KhAIU@d-CRp!dh>TS@HW|T6!XvEU%s#@VR8|*e8;+ zXY_H+$_hnrMEr9%WaNu*>3lp%0cS2!`%uR9zACD`|AbC0Y@zz^pXjalHw~F0 z1BrtyO0;_6W$j;@r_n|0PJH0()>rgUwvkM-p3t27r<6YH1>L;zhSD_KsI2WD1@gVf zqkN~RdhIXLn)ZkW*WIOW`p+mJ@Gb4OZKDvw&y=!;6BIxEq{NipRM0Ms%op+~kW=CJ zwkC8B>!Gn*AM+0BVlCgPd^B1e-ZiQy$?gl8=e-~o^_}Dm-qXv5*Ytkt3p&Wp%j)P8 z@;=VbOQn*$XI-V3{R=t*00=+3K?)F<~WEx37}R(PMGv;4ap z5S2qE(T6B&|6ywXb(9P{^XTf73p8kS33=YQNk7DldV=$4hIbOZpAHHLlI2X8eDTzX zo1&nE5Q+8VuadJ1-Pyy(uB^<+5Rtytc&Ri5w?__#@)s}IP92HHN4}6&4ZyMU<8a?A z5J^LQ;dE3CwWC82yVU}>k7&SeoE*AkKhhqzT1pvxm3AtfA@zT!$j_F4&v#xWAKP0z ztN8($xAQub_iB#ce$aefIaH-9L;08*surlABD4<<@}7On5zZdX|3#tWzfiMu2c2?g zr~NITXtl*RI4Q06qd1+Uig%Fe^<7kxe}L*O zj!})lX*w|dEafVlqRNKDH1_ml=xbdI>=J8{OeE2O;uu&k_{~A`8t+#-x!i(cCfG-idCx#1=6GN zwA&AlX9Z$TS`em;pNibJDexLQ0ee@B#@kj8#LcqjJ5vU**W$ZU4c#<`b5x2;DrmgI zHOl;cnf%nRQt2Im4jbH~(71Z~ysep1pK{OjNGW6xpONg>!#Tc#(6dnsXFm^sNlO|6uHXQn#n zvlZq|G)Mb4RF%KrNv`#`#qGXCHmNm_~P3Z>I0+TgbduIvqAVNZU$|QDV&rI&&z8JXi8wz+($N z*qcldV^eAPrtS2yeJ^$B9ikV*4sy?I8u?C$rqf%jC~5K)@pJPMk)y7ZWc=tnNy>9e z_I5@bGaGt@O__3?9aj-p*&qWHY_UiD-{Gjg>h0MkYE^!{^8-6Cd`M{ z;4t*+&F3LHfm|OH!)w7HteI+r^OpS)%zcJ;mwuqz1E0`*`&t?|sG2-S->16Ldh)b+ zN!7#O)42Sv?qdCG1J;j{!HfI3-a*fm}nEQ_> zcTwg4zL{M=>6g@ZI`#T3S!q8c^FRq{85K~(tW0W(NF&RXWU_18%;!6tS?IN&s>=BJ z=p3Q)4$d^FPNTV#6X=)o20GF=k;+FWlkvvnsd+u-|Wca&TCVv>3gY@SVp zrbP(imWQF~$^xugzZlUo7NSKz96z^C!_S3&X#eSs=x5fL%{>*bpY(y}?;d*j`vX;l zzo5E(PidCfGtLxvLCU(VROZpazstXQ?;;O2K@W#l7-6@%9=^3`qRvwpI~)7rPL2xv zU#g;(4Zz-f4YcZLqbE%dd*&F!KFpGHSNPrh-5$S&Iq>_@3CA~kAkozuM|O|G_j6;h zzt1G}GM|Y>X|thmU=H4Eg}|)u4D1P>i6L#lh*>og7bB+PeEdZ0&-RDSNiS&gZ}NYG zEKt2hA78r$pu9p68{4E%X4p-^t}_>G->#C`p*4gW@emeQoNl` zq->@-yA&!i-A-E38Pw#LMH_cy(WZra=+5(G`sTHY-dHW8V(ukbr@or{4O~y>b9r6o zxP+GL_|UVyN))*(P2BYC*;SzDWh<4{Uls|GZTNfS4r($6g18+#s2b@*sZq`dsG%f#$g`1!X{zHU?2Ef z@;ip#f0H^j;c#9N#c%)8Dc#SsKmR>duX{^5)$i%qs}7pr{GD80^FHNBA3U6)g*+{7 zTuI`7kF36!EXbp)QHf_I^yA)6ReaFZK-ME|WR~b-pQ#Cw*IOdGjDP!AIPv+LE9T7_ zhSwhnd((Uo79EJHt>dw3(PTv23c-`aaCk*WBjH&L>JLU^&4T%`Hkpr(l5p(#H5=O; zXCQBX5Jqeaz{6c5P?GF|JufU#wNf9BTxZc()d!j?vasS>wqdd~%8$w*!ky3DR(_(t zd$_-77N2o&O{?7TFul;;MO!>LGx+uvik`fkZmif#g&Q(y(fI=;dw3`PRM<$S7K^C8 zcs>R3IY6w;Y|f9HNt$#+theQJxcu({IP&k{)1CJs8HXO>xy<15aiv;p;9LIIDG2!No4}JpP?Jl)I^@ zLkjkMpLL@>ukY&AVXv-=7)50q{nH0IHGQz&UKy;UAKqs0-l&Y9Q@k$1mT}D?k^5<6 zZ7^ET5pmqJxYNV~Q!D`4yQ3j58js7qlOgqI3f``n3H847aN$Zc(z+Mo*QX`i7qSe; zM=rt8qD5dS(TJ9whbEI5C^4Og2@A&JQnWX6I$ZF%-U>$w^`Ts#j^+!>Sm~<(xm$AR z*S|M*p5=R>PQ9Sx@Rb_mpONvvTl9JCCBn?3l>K%OH5zWC#@ww`qqUv)Lc3_1?ml{Y zEQ9XUZKtNs8)*5yNLuGJo5IvX$UEGRB4!VzU~?xL^;M7F%QlO19cGFnHbs}6k;|6& z%d4>4eD!&a{34zye26{dbgDB2pV;+Id0|zoyzpCd0QTjWBY2<-zFZ%HCrc(G?%6Et zQHw&{(`8Uw6^EFZ1XS$Vh~XRJ@pANPy!jUmgD*1?bjTlSi6VT@<+BiGf=hoj;C@gU z1(SH*$e&&qdXU#`Lu3(tpWpkseULSxKP+?9P{Hehqz(O{F+d$v&r~sy?^L?p?2jYM z)o{X;&-i8aFlC`JTE1D}=K))IB@Kp3$WS<`c;ZNpE=$3;E6MoDfLy^(B)9mKYe4`ynui7c?PfHnF)Vf#Tc>dcHKJ z&f+Y!aa7aahB7vHiT8{u7Z0k>7F!+i5i|CsH1hHx$>8&<%} z?H;Y{a=M(5F;z=Y{-!CE-&e!);pRAT-vwX4`=Dq4I6Qt6jPlNCoSVKHhLs73zmkm0 zOSd53bPLurY(lRk@eo~#MdqytOxivT`+Ub>cZMfic%SmDoYzmT+-oM*!m_LCh^be_ z@&5f`_d^9O%elAhg&Ov9&$^wQHX8aHVn>w$Hc9L9`J@&)6|~T2gf zbunVBHgY5bAib?GmTU2RqK-Dw(tSjxt4k@(C!b2TWz*cb8Ps)Z7met*n@o=HCUdjB z^!najniI}*g=Qp^=b+`Jxo{4h4-MpuGc!8e_)uKdXNP$6z%k-32SJpR`>yQs#Tv;6 z2RruV*L?Qh=@fRLcQ#`uZZR+KFYMv^KEjJLI>PTD9U)_%8qyD%AxLn>zPUcg`xXep z&oj||I~o_V|HA;&L}Xr1fjB4)uVuHvx;Pb5X36lfiN_wFSm<4dfNRGz+)N(}rDKHh z;vsliYX`mA=BOKIh`>f|jGnHJ4kcA+Y*NL=wd&aVTLW)y>Epb_6hBWI;3 zEmNT}b~di<=liRP3sEp}DI(7-!#U?=s27$XM|m;Myo^E9mnis3h2w-M80sI!qo-yh z4u}AKwc*g8=!}E29AF)4i^+1fSohQpFYE_l$Xf?2_p-tB!=~^bt&5<)s@U3J5ysp9 zQ0K@u^q}5`D|L7noVBYchj)B+(+OU zK?!&Ls5#k~GN;#wEw0QG-&k}_G+()QSzK$lL{?RbS)3ii#LCI+-T!zVOU|*#m$j_J z>Kj`zSW&ndrX~C^)e<~T^+)GFV;Joi3>Pmi-b)7H_tqIm2#MlZaQ{J3ISC8>n-50$;~_BVHx!}C?hxh-<=In% zpf-^2SYFYC#z1u(=+zgV_oU%_vyD3Vj$!h)Qkr@4G<^xoAyujU6cU<2MkDr9Q0_sB zy^uq%(~i>o{h1_0ZKjs`FnTuDiwrAuDERS1u`^F)9hrYk^hQx!miymQN%Nh*l3LDT zQFKgUf1e#@*|^A#zpZAXGo7q%v%G-U8p5;FJj=ejAG&3XP&|4NGCqk>!F#~b_ogFO zI}*mFye1Dz#MF(4`0zd5Mv8;nNdNmw{%3{LL!M(@j>@bBf$IrJ{LYT<&Lf8Fr8 zp9l*g0V5wT)Y*8VHQotR08jvyRhlb&@kQ(}#&!wNxf6_N-g#QIZeHS-ro?fabF$r*e4?4^SM_*)5qV6%p#~+!=aXBH#TwOT+djQXn?+cG` zeZ)K3!6$b(tgny3n#gJR6&Qi~(JRp2dm}`5w&29+o#<@Zi>+n*vDxwfXyHCI+wH;W zFFWvuzi*9iHo;`g2FQI}h1e^LF`{!W_s>j*YXav}o4G^Id=O-8tZ?D7HJ-cJBjCI( z_8+i=)*^d2qz=NxZbuy7>x9DGA-JF6in^j-A8^{7sno>8`BRF8=gilhs`EA z84K#7H{uId4v78MkyyzrPxShmN!j{G|3}ezhhzD^VH}Z=k;ux7Bq@nX;<-*UiJ~Nl zB$7fY5|W)gDl?Rs7EK!BeN~!YZEX#b7L}B!^t*pYfAmksd&v9T&wY*a{G67&ut|4# z?uTdbdmlvd+jkc8kA$A!-@bB<|Mc8@K1xTm?AMoV8Ih80iDmWk$0}ufxi$r*OtU9% zya2-Gj9XRei~Vw2A*mjR1??#aQOkknoP4a?un!{cCHSscg8B3JL1KCVJk0mPF*yqt zW72T^dLjlt?!=tkTj2B94Hz#`BAWAr%qX3u84gV{)7wMJ9qOuXl2B8lZI2%{r&%vSoc~JG{kl<&H!T0uPtX+(2^Ifs0aV5r8c%ydE50{=WzvQ)Gn6tk7 zlS3eMd^SK}Qvd{b>tU(skEaQ~7~;H<syA8E+gjYe($56T<=k`8RRN$27k zNk;oP9lKXeYp))lQ`M#9nO#b@#`_8FrDS@bkhX41C8r7xmhG%a-x*77yNDzATj!~T zwt=X{zeH+o3!lVG^*zdSKFj>5YF_*d`y9S$`C-0GR15!WMHfHQNw6jUf^>`DQR$Yb zk+HbSyxG^5>0;(@V|=Z)!sRtC_}c7^B85%pJs*we6}vEWHWT%0b8+NI5$s2mLfUyh z>r+b6EL{RolOnvQJV+Yu#_yO6NH*+(`h!?(;kF@Z{{}ctTn%3FQiPsZ2pfG{gjQQX zS=kJ?rp-g#HdAQXn<7Wa3^z2)F=3ef9_1`i9Bd2cxem}~ToStMiqn@?Lceb{p8oQK zqe38rTtW~(JrpjdwjfnM7=LLaii+5ri_N-}z62ob(>iD?twGNzcc%4qfx=}6n7pyU zi&1QL-eU>Lv<0wUy#!asF9)4j4rNI;r(izX0DTU%b7msypbqN1RiR%#0b^!Kqa%am z+wU46&$bS#@4ZRk#!Ymv`6TVweTahBl#@kfDcN7zM``&5WIu5)i5O+Dx-gdNf4k5> zxp5@%yo}3SB*@(wId7hb_e}%6dz(L|jpupj?c(J=`NZ>mYQk3%+rf8DD&gm;oaHMO zJ?3xp{mP%=C(!bIqd?285|_{n-J?dLDp5O)1*$?Sr>l0Y>KSMa;x3Ea9b~U~2;0f_9*K z&lXsA`omaZHQseC$1Ei$1xRAw)0XWhdKiwL%*{~q2!w)VAjI|t!b2z!TBhsK z((aAW?{1ivy#zA~9UyYe4yhY$F=oX=3|w`=-CTEESm^=zk`-9p<_u4#1$gv#KGt*? z0!&}9{=`J^732}VOcLB@#^C+*jqbjCPN&#;%40f}!Skm`=k8%DGOQqSE}^~d`P879 zP4-`t=xJI6=^gT?Hz#aK<&rF2+kKqt+&F;~TIyon^~}5Z$I#toh0TnQ5V(-%{kWLt z^IVW$*lNxHXq?DDa_|6u?tupW{WW*^wFN}?QDnPu?$wx4m(N}xNc z2)j?^V&TVZ1kFf?@%df&dOr?gRS|IX3WeN`0L=Qf8VVsE*pcjpA{AG-Zd-eM zbugl|f?@xI)fRWyJkNDKe%$cJmx*quX1kd9KMT>?Z_oHOi?IE>GdeolF??Y)vLjdF zk?%6hYIDF3IV=4BI}h4&Gg0_Y2fN}}FZWCqmOOFzz7a&|^*;I-@|5WUZ_t2wGu3;Z zA$QxOWcRFsY%=$ek>GAB3QHm>oh@X-TSac8oXFtJeA=O?LFX&JaxzZ~Ig6K~oYrQ) zi&ywVeP|R`0zEJX_q3AWWE(9 z4#5y!y8(H&>!G2=W^-}tAS>XH3g-Vd6A6Hq^hR6}WV@8nA*eUlim$Umq3gE=?J3(J zus9Mv!BJ>E9*yU7cHqp#aJ*Frg@#QCbe9I9_B^XGhJ4{s?*kcY=6$8*FspEaSgj+H zuDCGI>`K%Ic%iRlHBt>%;M{$7FPU26*%x!hd76hrRU_PG+Te&|O4vQb_ANg|nYVm| ztV=(!TyCa=)MDC`j3(N?q@L`IkCDxza&q=&d~m-sl86bXI<}*|b{_wyTAcHfP6WAK~Ju%9Mrfd5e+-;d5l(KMiP!XmseTY;!$e&~C$1&57e zq0qAn(Q7g>uyHS}>I3Brz#=_!v6t;Lo;`Yu6y#BZiLvO>F)@&QJ?}R}ie;ZnFgyZ~~ z?Fe9JrXytslGjGStug{51`*JI8Nqtn9nfK)QK1qAiIE7{hlE3Q9NWE2^vCn^H3(z( zQ_|Np=vuiJ3g5ioIB6w%0zDzQWi96V`5-@JB|fGt!n|QZ(}*cd1Lxr;;}w23n*}R3 z=80hY{Lvm`5ho%7d*9z=Uw>=`&lxx(hev#(soT1F7Ba~^u_UW6mX;efkElc#J zuOSO)X69VF31eCfjy9(|(wIIsdMWEf3fI+WXF>(H^KFm$+44KhD+HXIxA!l!n0^12 zg@}ojMeBZh-bEv6{t(ZYzvtfyexq45zqMv3KR-tZE97KwXR0cKSj{)W8Dj(MS=I&H z;p;bN3_SA0$bk(oxgQSKgm@(UPQipJ*;s!s7qd?lp`oZ0vC(CCzPn6HuucgFKE55Y=OUnSAsTYM zI}r9X48|u|FJ;T-UghkphWkUJI{*>p!7xhLfSTbAII%8(F;+dX)qC%)SI7kB#JxtT^h(6;Qwel`dxYZpRte(qxLxbW5ykcR;Q8ayY2^ptCF(?U7NKHaQXTzG*OY$idvoJZv-Fhe^Lm z(e$DW^Iuos%)3emysgB9-U?h$E{FW;{Wz~u!e$;tSpKyDvlg?O@oNqSXJj)TT_z45 zPD34+il~_>(BG4cO4D5^JCKOZka(#5j>UYlI2cF7L11$%T3*JYKsXwQhqfW>#CB-> z*p4f#j))SAKt=aXXl#f>%F$T(SMOkTLIi5qvvuQTI7Ti+V3}_ideQ@-`qmd&s!Q?K zb^%N#T0p(Z2p5*=KyUE>9s^4#d|7}FQxhCNs*FL#3RhGUge&)lyo|n6?7&;fThUIv z6Rwe~&IJnVsim;vWpw&VHrWhE&;j8UEc+bPs4#~H^JkNwp9OW%Lb`k0o%{~@(A>pq zNNWZtzj8G3r|;#~&QRpK?<;e0?{l~-uO4v8m5;gqQcSte58`+hEp@zrhfjF7!-jZ% zxqo;Qj&zcX*Kg{aB@Tmminzej#XR>}=q{R%1tB)L_}dA+yWQbG#SaqRAvoF*4Z+Fm z986Eg(32df4CXogZH+FB8vYL>$A73|@!hPF5 zgtZl*Xdo9e?e;>O)$*CW*?1|Lg%0Mw7g9(=Q+*14UfYGM4-?_(l86nn5+D^8hZiw% z$PABxnq&lG{kFqBe>>`qN1!+~8f!;lk#sl#9mkUK!!ZSp_Y%=!m4L*U1f0!DMDV|8 zzEMT` z7v?eiOcT;Osr|xz+Go~EL80fU-r)#ceYuZ%oKvXQHh}p~7Lb1UY-&GbOuO{vlI78P zl=s4%I{iSG)|yku1QV)vQ>CMIf4SF{7r9iu-Q2sJLQdqzORl|2nw;BY>HhRe?oV=a zvunc`o>Q4IucF44r(f&A%gDb&e$hP?G+6+;`=oG7SRDqo`gmhv0y(n4R~tL5?OOty z7!S5LTL+mNTUgDt6SpFhV6ZG5qU|~Gewv3|hkbaaw4e2}6jgv zf>^U;giqRq#|DYuL=x~|b}X{4$FMnZ1kNvy#EF<45WF0RM>mp?IV%}meyPaHPKSPH z3c9jV@QCfr=Ef$l+F=Jm`vZ{T!uFum*051DgZTQ{kZzle((ChK^w$ar5AEQ{`i0#G zoS-w>7!%frp?`Td4gY#W(sEt&nR!%Jp1(s#xQAJjF_EOp5IC>M~NpHq; z#7WPj!PY% zl$dTvvt*}}hs-1Hh*h>lc=K6{s5O}utz#T4Y-Efrx)(Rn)iw7iOZpT2ix5UuG}C(h z)WqxXX}HtP5_%q3K=r8|+7g`EGwFeLR-5@8-2&|mJ28KJ5`z9q!}j?((6h;7JHsM8 zZz{##&t-_AO7xXiAyvB?^TZCKY@`}e?o}8&uL5J(eO4cQfYnv|S&v`B7#78llq$mJ zKly0eor?)e_5ho*;mQ~n-XW>@7@rIm&s|WhNyM4IF{rnTg~#nkBzi^Twq+bzLzAGO zl7^sZ8TfD^4eiOv2-As26Lw%p9jiyCZ9?AKbr==kh9<=YNIq_Xq+|17tUVt#iwK!_ zSzjgThylxmsE@OO*h+%=8|F_IQG$74A9>GuOv`RPrqKou$#3xuGV(u9{{)VZ>xF%E zBs+~%^uh=ST*;_^9`$`PBz0Ssxo&Jg(E-{tXOKwob1c(+=u6E?(wsc zXwpOK-_%dXYQ@pNQGs!PnFi5$8rCc~!3$9f7!BE?s9*`pBv=W_Lw@*sZ!;pScfj;r z0^_WwB4ung>Za$i-9;fb_Le|_E5oYh3cRzbVl%jEC|s*XfMGS(X;eXQc_mD$Dp0?n z9G~pU5D|C)=POI`&8`@yn+h>%Q$GAp@4@xdEad-AgJgCx+_oiS?S(jOSR2dE=ngn^ zMxu619E{47z+07s=$<&(b%w%gAOMoeYart5!oEvu$o^y6-dT(BHQy0SAK0MKjYF6S zhqP!b_{>{?kDnZ2x_c>t?OhS&>IA{@=GYT%fSLvk#Kg+M>ESRnIkeNP5C2hp*9|(i zu9-aJ>!{&(3HeRVAhnSw3i`E{M8urv@c8*;CT2oL#mozCu1Bl<6zI~eQS>selRNdU ziPMWp<7`%2aUm}zb8hDUEb?4ho22Ta%w!Ho(<#QTUgpcufIc*B* zDJqgj`7qZ~^B-3nRKr;mMREpu^EfBz_1x|?C%8Quhq=59>Lg{PNVbDI^j>5l#lQW- zB`baBL{n8rG@r3kmW-$OIpfHFZ#*?A<&v^|4LxK&f2r0k`Ypt~za8SpJ}8eC6?H7y zuM6cfGmz3b2RgAV#wE`lCk>Xu&)XC3M(goBWGkZn?nIG!61pFxp@P-G2ComO;d!9F@+61j^*fWE<=cZ8Il(6N4jh=^2QdTbubsg z7Fme;vNEb)mDRtwAE zxbjq-`@%F3-7L#u*8*%6T7oHUZs^$Kh4#WVC}49a{*=|YIcXJ?(wC!c+5*VVnva-r zmfavD3BT(cjUtS=Srdo1uD5F)m>PduPbTzLd%1l!Nu0N6E;r4P$GvPG5B0f!=C>gof(*3$z~VF^Kj*PAx?0`?A=s?;Bot5zwQ7kSCrwgNEuE~JAk%^{rK3v z9}lOL;={@!2%O5pu8|yA?o46q3&!@bibJVJAl~Y2#`_=c&<&h}Hs-N;@q_WP!nBYc zcAZY@w9p=(_p~cw6bku_rBOK@_okWS{fvcpaDOS=w|GM3gD-+C)?=gcI%uhTvpZ@9 z6r7x~q00_MI&;weMjsD7$K%7@QMmH)5zDK&Nq4`Vp|IH}X^i7BQdBxXLyI!0bnRB6 z-?p^<&NTYcGMUoP&tT6uk>RE#)N#~<{3@4IP?{Z`_+d!pbL2>A!yspVqJtaz@ILn~ z@izBh{e7;guagtg6eX=kQ^+uKgzK*nVm;O`&L!**H!)s}`HhI4PH-Z>d?$)~zKAv+ z0I7KT(&j};^ik{pJ@|W(_NUz=hqP|`x_pSv++wVIaVeZHlgE^fiRci}MkMo{A6q^P zv5ccVF2aUoJS>L#&XtIlT8E_VA#gjj1FrWHpwOQJ<6W7s-?$sMkK|$zR{#x#BE0&s z51U!dcg(01^?@Z=kywmsmtu6zD28lKKD1~zD&D4Ib51N0zegi{ojm!{|D`$gzah7*dJPTLZj=gT>I5cPeAgd>DNY~^d{a#W>e8E~Or79Bc zEurPiOI-XqlpdBkQs%PBwDPMwNhYY!h>an26gZHjfFC&qZ6R&(P1N?=n?BulpjWxG z=;Sy}lE_k{%a1k5EJ%X}qZDb$29~S6MuS`@5%nGzOI!Dfk&dMVmAVVj%9|oof5?iy zhpwWZN4+UZeKq51sFQU&NXlb9^LS*@Wcj01I`7nDxo-I69XUCv)U-8Vx?Rr z)Y91dFnTZEYveQUeIbnA7UA!wB2IOZ+qzP;C-5LLL1SsQ=lTIhxRS{5H3)_ie`0)44L5hN+;}$ zTZSDr?rdJ@j+LIvvG&^No)KLsOrS51Xfur`VwC?TR-7#ll(lkw0eS`_n`GVWcY zAK&XKuI~u7&nu@-C-SI2hUvsLJ*fPe5gE71QIE+u(!(U0qXnj$Tua(1F(j*%OgbNv z$WkMQM&vis#&B=?sOmGz9t&=jfT-GAa zOM>nSiqi5k_N?y;ptciRN!xx4iAeq8#QfFBoKAPYJ3UG@!Fy4=1vWa8}y_nzt9iOkf2{I#|th zG7wb{!ZG7xEZT#UFn)h3UR=sVt`EKxxW_#r)PE1b&934Lcdp*^0gym^%e=Z5ro8ovK#PViV|Dm(9 z|I*XCin4rYH5N8M+A zD3JL9@i?1mr{z=Kr#z~;vzzuaove0NI@#E!(lob1YC2R&Mom?8d&2?xa=Vyz|CdPV z;(ny_N}odR7*gY5YbrUZO0LB+bjins^xZ z)po79h!D$#YwvEReNM);rmeVlYYlo&%tzQo4 zX@%C>>B5rbkZoEz72+N<;d^r)E)LJf@ZR|lHnYb|EP>@Mc8=0a@ZT6!jJYp@hs*nE zdha_jZvRME9(B`=-S6m3=XYA|@`REyu2QLF3#~D`NRo^T)7@E3wVt`OY;G)-^{t?V zNl4*Kq9?8wBB$=Hj$e4GLq{m*T4&?{*EU<%4 zt!Eit%qR5W<`Au#IvP*Mh$C>KG~6algt+H)l*e14>+w<)PxnG_xj#NQZGrJHn+a@8 zK<=Xy6f$j0_@pexrpQ5W%^p}E&qW-Ytv^%AfumCr)1_>I=|exf5@9>{#Y@04wsBHL z3ybTfF)pMQ8nv{M`O6Rk<7Z)mrUCkrrr>YOWZZO|hF(8YxOds$*nbwNzhsP{sdHG) znK=v|7~$0|6^JhFr@D1~3ih}_KTcgI263d;*|$kt;w9C`JRtck%~UvamIUMK>Cxso zveqdjA(6dQua`m*+qRMKT{jBV@+Or9QA{hAMN3*rXv5$^a?L(YVV-B{G0Oszj!L6a z150|l;5nx=mB-z3S0IC7dy4SdKyBJS6z1#oBe`tdvB#J&Zr{CZCE&ZK6#J zrc$<%H;FNC#~${_wPiOM%U`w_8nEM<4EJ$vK3!Zz(3DcP ztqrQ6pQmf-mGpTkK75HbkGerKEbb8R@hcM6`9`u=`)N{?K>Y3xrE+ZJSx>|XN zbKtJ0&-UQD@vP@{dlTeXrB0qrQ+XPAnw3 zX@{xw;Xw*=JxKKiyC~Mzh~C)xkZ;v4^0`?+jav%H_4A_UsP85g*K?)0IU1SV(^Fr# z%a(Ff`dpt}n`}t1#+yEDiX?gcRGRo9m-;`JQ7GdN$$mINHSFgba=by67oIY2#xt7z z{v`#ueA`4;_smMuBLU=?9UYt)uUPdzZoQpw{+IkEa_@m(NDopfWj5&|&ndiU>8&f?P zqs1GA2fT2fX(?u$oe%Rnv(PAJ%5oOXA$M&K0!A~hA@hFcUS(Q`At|h@6-C&7aTFbS zOHF9$52Uh8oAczkmk8;#_!Fdu#0<0 zB|n!Oy~^24wT?Wc8>n@46CLeqAFJH|E&r#~GzwD1LcmsXWggvTOY zQ{k89D<G^)6cTWG2NZ>`{i}1)$@DK$o-9+-j z+4O^X6PwMCQ?cT4QY}2o(~aH3Yi@AmHLBG&Tc3PvvFONB?n+i3_q6*4C)gxFt3p($ zK*NMa8!e_e%hr=y-3}6IPo%eh66iClTS)3Cod~!Bwfa88Jc#D4(41Vg>|bZ#_K{^)Gl$~YonOY2E`J)deF_%vy+5P#=LFK?C8f4l`AdAu;i>&+X& zY|PU~KexCR|klbk>s*o+WG;(G0&>{KYSQY%GH^XDh91T^=yn<+KZ9X1 zFzg~F`byERt@Nd-g7Fpd2{YKcEg+YUg(Ne*cot1M9ZyH)c9Gy1#s;#@qulHRBzNyH z?GZXhVlqwicit7sVE589sUu|ATTSlMj!?PwQBpT-q=BGoBv*c){)96BRP#$RSL`6M z2|Y9of;g)yj?-Q;_|Hj-`56S!nb=2vr{5)|ht;Hg_8b+>WpkEY_i0wpecEtIi@*51 zBLBk^DSp`S058ZfkJn*m#EUY?Y(BPTk%g;@A=mqOFXv`)i@P%VFW1U)L+VYZQf0z? zvSYo3RN+Q?9TiO;KB@GUJ&)5qouSR+?$Rprm({Q4mSZ&z<4a=|)UT z-W36}%~7zOxC}>Y&GA=eAyh-$FzKKdhV6VX`#hW99a)AAdo5A8L>IcFrEt1J44)gM z;M6M3>hv+FyZf7RB|cJrb}tD&V!yj!_I+LHp!p>SY5$djG;Xk(R`ndFQy&kqOm~)> z*|&#sw&&5K^`+E1?)2l8)Jo6OwTpK?y8NMu zvw_b1U`(Q?btLxq47G1LN3J$UDT?t=MN^N{e7AE{!q`-Al5df{5X-IBe#n@M?X;`= z9z~iwq}9tG&^(`qH1EO#ra^i}suew?GH-x#uMN_tufIsr`4hdK+CvGQtcKaI1cUQ4 zA!5TJ{Jt5yJeVHYL<(l-`bhTX9df?;hF)@gboljmvT0%)RSRE!c-~6>q8ev@<0axJ z-;n1wXASUHZENP)ZQH>!8~?4jO+<0Nh)k!2_*-{wyH!5--Sa9}X8MDZ*JOJWQ$3pV z*n&E5tRV9jA>^qR%l;p!B%`{A{9f&+1@{kAY5GGUp(b6~-^9BlQ8v7fB)Sb8A}l9obBXf;k8_d<4q z6-t>VQgfRDK^95nL}NBA);^EqWp^ya+ycCTJIN0=y#GR%WvzH`$MN! z3c%f3fce$q`P~LP_;$;-^1q6$<$nvC!~Z#DBH#0=Aiw?f6`slHMBW(@86M9~vDqX77nfBhNfLYw(py7Rr0z*yZw6A8YBV)gr;~hG9!c7i(Q}`p zv~CxV;<`G>{w&M1a#w&uh8lG5OhW4kU1(WNh5LVc5L`Wl`6XCBoQf`1it8b>bQ;2h zW+MHA38qY#kHRXZ<1ccA_7`W|cwvefaYM}h0!-%|k-2RNbe1?F(s=<+MiEAxn2iNX zrlCn%6aO_T;Qmi`A2fd@YsN`(?HZ;NjIS7Wyqm5zJf}Zn=J(Ujkkof3k+GyWO z<~IdMy(h=HU#RN-56V>>q?fvbG-C9dBpWioaHk<4;?u z&A$*Q!oRHhA5Tks4=-NBf)^cL-z*^DY#w;8-r`f1GZ%B|C}+L%6?fN7nl`_lNfTze zQo;4jxEAkQWh07bnTMe=HN* zR}QL6|^;3<%4GQc4vmf!tg9&9F>G0)-@h{{cd?V&k% z$~^EfXDxB`BiqyZ%|rXdSvX`k6+iB2V(uOVh)tD6-+mE1KL3?G*uKVi!aq8D?K=(n zd?G(${MOIkX`{hcvRD63^470N$E1z6uwGE)IO_%9uo~k?KWPLFkVMWPDL)Szf50y{TBsJV+lDTT3?Jtfg=DG7<*LcXEX z0e)a(34e!U0l%{$j=%Dm5C40DE#FLF3g0JGjQ`O48E?SOqM4Fl670S4W5EY9>i9w@Tz*m5^glH6m(|NRgxL;U44FF_lkClS_;N}JF;s&+ zW3rrGq>H7`C!yO;8S)8gFe}h!8g!Ojz;?IV_ol;PpB}akFpuyG=KJlG$NHpkh|iOP z|BXN7W5ZYw-G6A$X#vzr|EBKc-$`MppX@FT(EfivDZ2POZJG6*%neyynpHPldEHOG zKmSpMA)NnO8u#APRx{LAnr_T)cfN33u-d>A43cHrRVnZt@mZ~C8c8dq4B=_woY6A9#uWQ z#Nb$dvRoJMzQ;kH4L^ZbwnB^7Uu@gl_b|~yc!~uVE#AO=xhO=E>a*#JU=V%Q&Z6A2 zrS!43l#=S}NaI{9y;^*W#GZAKqslwl)^(RQ6}FQ0k`~5SxJ+w@Zj$qYyA&zTX7ev! zQS6xyRJ@0A!n*&_@y#MwUd*)B3eu?HWRZAG4(+dH(OM^s8W9EDlU7IeM@<+>X`-f3 zm3e>^5n&;RuZ1#joHrJ4qs3Teh&bBL{-hQDg6K~CO(87L{b=Q1${IUNOI8li?M=g! z-u{a*SO&;w%mA&87C^_getNAsKt;z!K{rDfrZYtlUMz|=K@xCE9f!Ad6X1SJ6-jK* zAMed_rNbD@YKAn^QYmA_VgqQ|upHF}_712wMfLY(*tj(aIjrwMz)r_&~!#w2%%Fwx?1hqOP9IbKZKMV-u+gym_ixp+_ zrH!*$F3c|e6|Zo9()(5XTy^5t-k8MS*dxjxy!3##L9K#!>1Q%;PlP@XNeRvNwZF_a zzcA)jO4V~sFU6?8V-7u@8OSnR8S`u)n_Bd1=*50Mt-W%Kc6Z#Rl$EU{*;G%rvuY`h zv8!73>j%RQc=NWT0E_dRE|HUN%uO*on-;-l?3G?k2v)R3g1o({6G21{GdvZk(dU+JiDUO2LpMMlxF-TQr z!z3X+LieBirI8aObd8;@)|GK{E`Ap{{I5kxBrqN#{+{tYG2Y9$HZ z^(=cZOdgNEbSuB+_;0zRi^PdtC5Z@>jc=G+KS8oU(Bp!{kfzA#$N`mv${uejXFAp zWx!`=Gl($&>K|&b8Gst*Y-D*1w*~mwI^-7GI%{`nM@5nC11X zctbfYAIYPS&D>Z`x}sK)@ymqKe_}NDCys%QgBU)27em`$aa4Oq;xD^*Mmb61@E|RdQ19|VS~l{N%|Cxp@1tRo-c>sT(?d=&*qvhQXjOBI*xHK4>Yx4b^+!e4zFocofIjA4Wz*MHmHct)5E`_yt zTk48AwJz}8J{$W_7(wr{0lu?)xO=@M?wyv0%lnB~zFQ5|3q@RHI``THa^+_eS6NJYjDs;UyM_whogn4ndg}Unfdp##^wZ-i<=Nk+ zPxl^^(a0Mz*!zhh+I~>L)Dik9E{tujMd2tZfj@^OAwN?FQ$91+&VURgEXJX}Uk2l+ z$YNm0c(~cgLC#DLFPdZ^R4f4p8@8*sHbTX2zsNDQk4Bn5(QoB`>Jk4=o*w-aaD9*@ zRu2>D{$+DHHltA(&F&EitYEyJpKB+;ELsJFLu!azr-i0%x{&PEL*8Y5)bE)AlaX0S z+%^|FZ!DmzwE&6_mSLc9Eu`LW#lhrwsK=(EjA^(Z{>S#6jrq)DP=G$O0$g2_hj$xx zW8bYb^o&V>%DoWqc6h?o#SxQ^S>bb=6=q$TiJ9*WuqJ9Uy57s-=-)9gFByl@OeI`D zr-Bw~Rn&c&h-&L8yz)vf-gT24UdNhpUZTu?UggbF-f8uNJjaA0-d|5!-pVes=9(N~ zuG_nUYuYM6P2Z=oEUU#7|0t4ly7S51`Y`=C+(H42tuW30F?EK$ruumw=%{-y-3|Xt zN~&Mjx%ojOb;INxB!IEWLin+CG}PH{A?}A18tY`C!gRQj;R<+Lpv)Mg6Je>P3WstP z)J>U)nPtj|Ggid=*Ark;sers7Wi)(I!R+J85Z);d>2*^0xQH?1SRbnpH$>K21N3_A z0NpD4NycFVlxO^lQr-TL!p#xd)+2~PjnR1MCjl?UGqTlE0N+;?Yu`@7(@Z_o`>^NQ z-;ntQjA8d>4hy|D$80?-BpWY8zl|%DKX@amU=xB@#vmj#8A(T(F8*pR>@OCg)@vWO zKP$pS&jN`2+k><0Tuo*Cji-6xaEtSWO_mEn!mQzCG!J(tvd^zK22X7|=49w1Z<;#( z9hb%;ZDEv4vUdeL&x6j2Sfi|rF@s8&{6whvmcgm!4uMb2M)hNOp^Bl+q7s?rYB6gp z_|-2h-mXaDRO|Y0Z?1H9gBroEpRXe``oke z{K_8WZO?;|Rz6g=0Xp+!FpS31#vf9;0mDT${f727)F-R^QhZvUQz|Wh2imek+G>UAEFq| z(YWj`#PausNZNcfT54rs%ws&}HM;o4GLvnh4d7F5fSrt=YpXdEb!^wlcbf_KQPaTR zr4Q3@`ndN(50itYVy==No=#@DW0zHNR9qGEG3t!*B9EQ&jQN%%fzKrp@E;xv`C=K& zw^f9~g^BprAdldC5?ItA3=084$R>$G!(iXlGh&Go1{`F>eshHUqcUr?PiiA~sh?q1HDL<9k+NM&BjNE z>2Qvl0@|yMHd8GONHE>*VGYdnVVO=(mGEJT6zWU<(5I=N=%a%W99e&6dtVxAhsU9i z^#R7aeYx&-WA5xL9WHOxZ;Ot53EZjMpE-*uYIH+j9^(V=q(zohlso$>DVubX)Rtc~ za~0#LZWY7yN8*sG5Qp(}#&R|OOIo+yvTTihQavV&{GX$7e1!z;ACAG18g}L(KHi+DsvImY=L|w=CzJ<0&naJ zcrWvTt8F0KKZiniLo{Qy#6qS&4myQQw)STYQsOKp@k)|@G#S$1I9rlx-bHnn z*?g}4KYISAha9_pQQP7_RI}(e?a%EagICYULaCih@{NY?eQsabtZ(A-9r;d?5a&Zia}_7Tm53LGrgv?B55n+2}fiFdejy*HUDa z+aNb&4k%C`?=cD2->BiQt}0CWCSr#h%O~cu=dfNC4X&yX{l{{U$4^9=m@=N;RK(2B z;?Pd`Mu{$Ws89V0yt|MSsYg-^b}K_`sd-dd&T(s^@H^I=I%^ zWiV&0xGI=QigoQ^lp1D46!D%MH`W>-l!;0jePIYF;jefMTEn~gIDr+7vQ zb+0O<+RMACt|E`r7?(I>St%W`six4!)zsHgNek<0C~Vbfx-8Mic494*xsmbPZQE(Z zs4g<15&FTjfu>q=kZzU4m6UHRyW=ula;&1K7jj57rht?u93+ip4XiI}rgym)X|DEp z3VwW^Wv&H!mETD>_%AKqISLO~jKK)2-{-LG_4X+W_}8w2{-qjt z+@y_%3-wT%%)Er#jd7@V9z1RUN9q@#>Gfj#kFGb5sww*a|C{GYbJCzxqLLI+-TmB= zDU~V7Jd`mRQz0Y`nhceY2!)WLP@#MFF_S`sP=+W;rV1Iq`~CiXK5PBHe|+y+XPxWb z)LrM?bN1e^*Yo*!YQ{~Xx7SRillfXSO=$*g)1OAGErRJt^+39CZvg#4M^c;h68a{< zh5AL?(4_Y6G?|afH9z&J-C#pn`pAU#A8$r?U$>ycDlF*-b4%L5`~Tnhbu`$kJN?WV zt3EHZXf&^vnKvY z`jJ%ZmV<3~dBl&_$w|?pYFqvWd$%BR01vD zTbK~}0!9PBp=_WcmG7-ej~r8=$|X;swwBK&S8PN>$Sm}_wg|tc#NspO5bl__6U~NM zcpJSN$37m!9-lM+Jy#a*?j!ETOZeRVhziMXoZF&6YmTVVnz1@`+Y+9&^wxx4Eaur{ zU6xcuwihkzX-D&_`q0#?PPAf)3%yk4Mtc+wrsKl7_r0FdU%cNh8+upq}@P=#gw)nmLw_9kYJo&+n&rey9x7>hiFM_uDJx@NBT__p#!|LsWgN zMOJeyF4pwMwD?}gwCRq|Q)F?k<`rMF-Y2itUnOsp?~}3;Lv$Q=h59FN*nIUt%{@Q# zs^lJSgHL4nF$*{Z4}^}=Jj8R&hG$$NsPZ1j3ddpVdlHgc$KbjlAFdDgA;M}eHXPl9 z{T6%ief@sepUVGl-6+}eJPe0lMf2{5(BWBE^-Egtas=m4zvY=6z6as9We#lKnPY0a zK9+bp!oW2EPLCI%`TZKi&W}UK*EIBBn~N1(XE))R6bcio;eNgzk$G+SIO-4d`YF-< zA2lfUbB?CI5mgz+>u;la&`ojt{xidzUNyAfSrry^#Ucx;{F~pKwsL;{Lvt$C;q%g- zwsileKJ-#cU;6x9U)pws=Lv7`LvMKWp>A{SsX>J;9XZ^J_TzQFP=g*+%rK|oduz_a zx2H)Red%9c-dD-u%I1DHv_P<+VM^Vp|9(B%qlX4{doN3mOWP1``xNuVau{&F*w2?f0vg0S z>M+^lf0l&pP3PPLH!`H48yP-gANjCJ1^wok!>if~KOKf);+47Z>5NCe8@tfA`#~5C zJ&Nb{N8nbJholMnu+%FD#qv35i`oktuGMLLoQv=^d2qX000ZCS(6T)PRfltEJ6MWP zb^??6XMXjM;=%Ig#A{a?nVxfq_-^OA-OarrziS-&HZR7;t8s8X!u7!)^RP7M6dv8V zgu7+8F!$6G6a=(jb6FQU+7&pziL+@p=}?5|(W(XdG@3K859aC8&Ke!MV5Jsa)I*(m zuU6u`0y$bdR)*U0UX|)!W!h57*PGVe>6Tz~dN|pFs!r@l&4PQ;hA*5wXX-!?jdr3_ z=JK4MnNHLyuOBUa)t^p(??O9oJJZ`ueQ9n8&$o~W@_0GBMX z!MV@BN$2V~@>%PjB&MRkbEq&{>Sj5Hto(OHGH80L*RwQl;$`4Kn)cX}{qIYOm#G%6 zzWPB@3)HZ8iX)~+jpX?w!Mt}i11a$f@tv;VtoC_uVhhk(5DLoy3vl+w0`9SzkF(Yb zF++PPocFGVt4}l{2d;&`^*TgWufXQbqw)NMEZR6M?@>ptXTal)UNauMkih(#WXX0@ zc#QSu9M6R)jEzEP|7|Gt+>c&8_;(`w6yyubvA2xtb_drXapQaZCo^S5vK)P@sl+)y zN;Dusk^ae-qj|x9(N*{j=GGsPcJU3`bDzWD3!fA9ti#)VUlILLnSObpN2ltU(y;}W zbnAU<`r!v()BSa%fBblc+0h|%Ko-|$=z7t%e0TctgezUZb-<|$Y-!6MGkWHrA?;SF zO-D4SQwtk4`mbJz>Z!`n#W&mGJ@h3;Rop@Dh;kfM&qdJUILw?jiC;f6Aon*OpS$NE z>}wvz@S5BAhFz%o!oxVnkKo_&LD=%l8Z&A>5I@cCXX@Y@^(D|aja5~jjvUGxi zP&i|p5V68g@>*4;BJ=vBij>psp64d{lYf~mWBPFhrvpty3bqY%ib zgnQ(n=S?!D^a`0glaWW=u9HFfcZk9L2jr|#6DeB$onZW5VwdxsbaZ|umAyWawe}at zZ&8={_Fbit~UktjVj0d^^&c)ffxeEy|i zUfy<$UcMKdu}86gMhPygxrw<-k8mOJ1+M*RM110FjI*wX)#67urEw2&Uv9%}{7sm& z-Gbe%8l2T=ME#W?Xw1^4(F-^We@1UQ`!K(M?RDU+1xNapI@8Mju5@R}AbQrxoz`v~ zNHxs*(YiS{bj88$bnto|s=1M8qE1()))RQWF+zd%d!g{(eXq}Qba%{OGz@qTzlaBT zRW6|J*kQnB9b9+KM{?Y7oY_1Mp{Lg2qBI$PJ5mv6mV}JvSfu`n#hLXZV39BY$Jdx* zZ|gnc*fLS_OkPLOaL5rlk{<{;0TYEU8;b?&jDtert69Rf!3%|f(+)_@6BbqkkJ?aS zGyS2o-T1Lozu~-e_l^>&woQq&>$8aZN}VM@qZCeyphIts|92otMti_%4muWG5sk92CCZm1p`9Hq4>Hl)Yp31iu6w;nk^| zlJy_9kPd%Fx{rEA?1O$1%l?XpP*=wK7!&lG;0U3Y6B1emq0WJz;o%HSaoPgKq#S(n zIRecMCoz|6R652Kp^bANmP8jrefcHS@Qga6LroZboO9PRb*W9371d>q^yx+yTHe^7 z&iv{`f9~x|-(KlW57^q#Q93?!#XAy{^Ni@5BtDytrh}Z&k%>rw<}g34iN4QJ}#K#wFyo4 z4+?5`&k5%ytr1!tUXof~nGd>AL)++4B1bsFw z*p^job7k6|p3IWtc%laQu=h)bvPmTp_TqIv_K{hzzxNE8YJwiS_r;8THg;oIYruxI zPh_geE7^*P8`-m>MNHV{#KOMc7Od{s2^$7iR}4GrCRs8xLUMA`K}p`)Hi>nF0vXc$ zQnE2uh75n_K}P*uO@f#2Cx)sbxfUajalZEOd**}3Pv@ehVm*ZFEr{Kah0;nscB);0 z-|I#sz$!akFMc$ z``HS#H}Ab?YW_us)^GIs@Cy?Q{$jD6JWXv;rrj55(zw^%sH_2JvU6Q<)<+fE7A#9O zZ?$5c@go#?m!f@d7JMQTutRnwYPCkgOn)5YZieCb>^1mPI2lQ~qv0wg$PF8S`nUmz z4(^9<8eC$9fyuGL8l(upb_v2l&uN00c0XZuN`IkrpNrr(NGkPz5GzTHnLyGf<&!x1 z$Hd269_t_4!1HYYI=3u^vP~+M>^chXuxgyl{|OylcfX$2i<UM{XF;8e!?h_R zuv{Y;lI3ghadZr3_2a%lOM6^t>w&HuZ6t9&{HR28#NJXzjzJSiI=hJsIC4+=Wk|5# zB8?E19F7xmeGdx?EAoUoix}bc`tE}D?sL-4uNs1{<2J!j?V}(s^k#|oXRvd>lbFjw zejZ9svZ6N^m~+;3HgxcPw!ZTbJE~O2BI=*8i%V;nYrs9`XI9OE%B$IzTMyVE%O|WQ z{0Tc=`ha;Kyw09_7O_c4VJdURvt{een028nlXUMCOfUTr5_ZV4$ak{r>e&uqPtgM* zP3DZyb7a1-w>3rR{A41mRI!ve>O_&=HI<|-u8XW3V~ut7v!K5*k=JDRKp1)sb)Oz! zQ`dLI@XY3)yl207x)t}dcc;DiTsdg_Pi)%z1}9AGG52Q!e$8kAhwEZ_!(*7mJ;K|A zbqG4xfE_MPa1Qy5S*pMBHA{gu4^ZWdd<{C)M2BXC>(jT|M%1RU8|^=u_xq(h7h3Tv z_XpJC!{8DGFWLdC@XaVaybiuSrop-11!o_7qWaThxJ_7xS~c$TxZVc^A?ir{)g6pMC!nu^~!lRz~!lxfcgiXeW1v8Z_ z;l<84Vc3x@L1lBbV79XxgZ*$;Td;wxl-tX+vPzhO`876p?gLg7_L6DcXk|x^e`SLz zJDB3`4wlHiu(O-n*!)ZHnd!%Oto3v=8`=DtYX={*VV&1lQvOM1J~)$|yc)sGH3QfU zgMlpNVsF-Yq9=>LX2Dh_n6vKZO_+E{ixsW>DP(MaD4bhXB3PeG5RR06ljdK_m9%|3 zKsL-*M4!QZ@GCzURr$#fI`=_7=p-gBvcEBJzC3-Z zr$YbDQK!=fYSD@gE&7wQ>6Zs7Q)h*LP{0Q~iFky#{8C)h%|L$2JR}IC5dX^;AGiBr zOMf?X*Wl;s#c(WqHXob6hTz0NJyhntBL7xMNyFnJvMRcWG@B^E^5a|b$?g)FRXCnV zeSS+lDmM!L5BCVB6N`kcPfrSEbtQsR_bb9__uGQ%fLnso^ZUZ(o1MZqLo=2(X$;ft z8Ov&a?_s{Rr&y>wV>6AanQPrMmh05aww-8W_XdArI)~cW$p`P)i#@Mct9KpqD7nY% z|6OP2lPCPh$?}C$O*fzAR^`2fMP-mgy^-u)h&H?EQOX z*30OZuSU4_8h@518mqPDAC7PmnAE)FcGA}h_$J|fQ^7@WZ`+lQZ^B;5`|3Ut$HAFVfy5gBF3v(2=jy2b*NlRc5TMuJ;Bxv-Lz zllxIwWcTe@GSnrO9O=_RzKp0P(>sgE=b^($Q%Je=@`6y|_NQIK_s&v5u)i*xzI{jd zJF-sbXZT8p(R?cu)VBz}WnDtR9&Kh5)Q^pd3t~Rw*Rzu;>8yOf9>z(dEbiH97QgcX z`&n)D*6^Vl>CUzRWRcTyrfIB7PJI<<5Hq?F%Ww!>T!4tDz%!4)K)cdv1|L zxQ!)O=adqIRmzwa-~tuzV5ojwhsId$g&Cg--Q~Lw?YalIXXe7(=n%$#<{5NcuWRw( zFsk9$Ag3Ogkx089B=lk%>F<7vMD0FI z#P)Gyz+V4#WjwK(-8eLojTc9=s$(PA+cVzm%0hS69&68b-qmAi z6@P^&`{%-$lnSBb{$8QYZh_Es%urZiE=Uzhi==hKnx(a$4F!p;zMyhgl$w`rkOn#| zkj^vTCS8^;l^%+$mKOcVmL82st}vAyFX=m2nV4#Yk@WJD#47nCx!Y=hsd~=P?iz*C z<+CuzG#mDK*Uc_P5s~GI4jDS>#LeY&FEV&Vj{U-7F_9ziKhFc)> zeG9(x%)#RIsfaL4gOwW3(&&+jn7PT=QksN|-x6?Z(PlL5*oeYok?hk9ArLg;sCGxK6NudV7oSSQ&gaD zAcJi!tweZVL)5g($nQ0|q_n*QQ#j-)&)g3xv!%<`SZ_|P z3T;zo=q<}M$G;GWO}TKZcDr!ebFMI}-9ZLwR-c=@_8xjidVt>L==v@uESiPSacR8U{Q1`XNhk|ckabZ zQ`w14i+Co`o$cuQmj#$vY6C93j#&4GdOn91z zjkQ~_$1(-Rok?&`ipO5%SOhmjA$`^|INh6%dy{yLGj#$g)Q3YSZy@}XZMiO6A5;8g z@b>#d@+aUhu|K+yyd5G>{O8Y;ghb|E95-gBblT#FQrBbMg@xBg2p3Pz5GML97yJ`f z3XAS~BmTPt!HMp8=;Vw()fTvsp^4Bz?WFS74WiFM7K&3Q68rQQl7!+a&*!5`q+?`V z1j{FhLhitFVO7Nk;ihsowj_npFVVUds~{+4?6zE)9qG)=Nz)0V6<-%NHaKTGEHYa~~jlyP3i0H+7^#F7vP zume1cZ1gZ>oEV3BcW1!u-(nm$TZx70QRs?}!TEEsu-zVyU;KJl=(QECp_y3eoQ=k^ zEM#ym@s_w9XdJc!Yn3u_(_km26ldePunU@Y*@%9Xi6tMmbM8Po+~SfDUm1%W+X%Q_ z2!+$Zxfo>|hK?Pp5czo>Mis~6Lu?$poA|XgJOQUO5@Fl634T>;vFz_M#0a67T{9KE zD#qhz*+`y$=Y@%y{Sj4SiAh~r2rc+d9_^?iZ@26tBP@f+)z6P47P=*uwDN99GolQI zy9b5}!Bc{T<6FXn`oSB7Wpk2*!O2^NKl`=`&gW%hWMpMzE-K5&DDtOreEZ+W8vf*e zUzh*i7x{iI-=g_8SxZJH)lo(!lt2H^`{8_B#JB%_{O5NK{-1wrc^?0teNtZ; z8D|-p&HO+9=kJdH|Nnvi`MDs4|9-Up?`QdIvpxRj4Zh%O1gDZ;ZVMp$jEH0<{SCXFZ|E3*l@uEc6LGdJCz@MWWvX-2qPs| z@Z-W0%&&L>?|!_us`(E2OPb+y`2z|}e_(!~3=K)+HNA!Ew5_%qRob9I{k|*Gd)!ND zcIGMaC%AL%!D8~(V}vApZkY7&jgi8q$!S8ypHkuA#gD?n)rRbFe;-!BYi8PuGMUkt zV)i`z9=kUF6Vo(O7RQzti0O@1qL;t37(aH1IKq3FICte3v25;marV~9;=*N9#Fov| z#H7@zqVey^qWPtXV)t@C(d&wjIK|LIES%R*4BT!dPVLkeC%se>txkSr&(Az&@$aQf zZt+pJC?|ti$!=nfLqb{YgHi0QpEH}GsmngwJQef?CJXD^+ND`J&Jw3dlgN{t3*=(B z0(9gZU^Zkde6^RO_mC|}U7Ldh<#Vt)eit2!-XnC89R0FXn{L}-${ANS^uiJ++Tb&g z-n4V4mjc{q(Hq_aGVez}J2+A;Wd}NIFXy|3*wZWHd(-#Rd(j`fm#011m>M7IMzdl$ z6Ze}OO)=(qq}$)Z?E4#3y%~lh$L3+S$|mfq+kuGEgD`U`!{c$)_{(#(FPptasBQ!B z;t@vY-^cvKyReX{g>Ugwls$NaXBHRnPVEq0-AP2m_DyhYjzZPN$3$n-bmABLR-&5o zu3~eoyijU3SDL1#ctxVJ8}B0D<_PYISx{_m6&9@1)f9pp+c<` zNsI5nChR>%g~`#{AZ@zxt|@);(T>h)bfHn>J*eMGPtHglLT!}?bMJ;LwT*J&oL*q*OJ!H<|(9<({QH%&e< zfGRe6(d$zr^zj-`x-)hN^*A<|o;Dst1s6AZ?C1cxfU~hS-0e@D{y5P#n?6*jz8AH& zu%O?1o6t=I^yo5kODZr4 zxmru~xosgj2lNwj1HDB3mXV@c62E2+1&Jr!ricR%O%YEIoh+)gjTeP5Uva%F6+_!x z#V=ZRV!XAnczTSASUUI%yWoG1%{M;HW~|=HcE4G}0(JaZ#|v8)JynLS>v>hUoRTW& zdG``Z+XJLF_v0ncnnsekCC5qC=62G~4LbwB`@rh%BG?>A#7wt6@C!TvBToTS-#>t* z<$Ltx+|N%#)#;=cJnxlrs_9QYqjYwm{ey?lz&t`v_XZtK2=#H1&^0{kW_%A%s@=zf z{`kOqc$tG}@BvrqxWt)8Z0buZI5Q`Ec2DZtZc0_P4XNI89qN&&POqzSJ$wCc-00ql zIqC1=7U2eqs8MK9g3NoVcU`lDfbFx6-Hsvl%bfsU=TJA|H^rXU8LCC z3N<6Tf#{wlZsxV*K!X`>{&dHZv!x`l@gupR^NrYK<`eTC3gpMe(UO01T2j9xOJQ(V zsxb9HqcH4JANF_hLY90vlbJT3X9414R{8t~EBK))&I+>-LsFeZ^(G=Zx%r8Xz4&=y zGsXIeA>!tzp`yo(5OL0uSz`O7DPo=ec(MG^2yy8OPjN|rvlvLN#e}{F;)W}VV&U3W zc6VhZyK}sdP52tm%EG-_$j}ZU&}FLN9aiRb`gFIxy>7Hj3sU%?>4b=#sA#cV|#g@@Dq-z9Cc}eNan>=HCx+8U;YeReF zSy1(@rnEAI`#`y7H*u{NEhowMX|%b_LS-BRo&XEr_Z_Orohp zJPX`;JdIl(!2L^pbkh=FdN{*}`W_xkuQoW)-A=YteUlZHBbM~^$sY7C&xBoLr$=?o z)#;7liZoj77u;s@8v5%N6ofjVnlnCgKKbMQ3J=sys3(VtOGvJr5}Gb`!;aO%aI0?s z)~+6a%uaoDJQvA8zy74~M03fA7 zJ924`adL%q#zZN3cui_Q$ydm;t`J;a8#0@y39RFO5#`dQ!e4IIXgzM)0 zhR|DfL#WEjA#_EtCyhw~ef8Og9vbIM4VMJauvcSgdwKw^UonP;9UDofHx8%o#*U;P z^+(V*pN7z#*ZRP3RQ;slvx2wuR@IRTs}-X>ti2w1y`N_#NmetLEY=P6QeWF&n%vb> z$i@+&-(nT!K8i9tjA7L;_cNcSD@?BV1B(pQ6u0iS5fAL~5SK3-Bl@@piPcA@iu31A z5p%r)Ma_T!F;CW4j2Ssnyl{P(=r>g&YR+^OC(gGMj~W__>uZ(7%D{K**GiE^wC`bB zx5Ai0ac`zSvr;g+*CAb;V@dSB-y@Pex=>l`f^lcYA=o?`n`^g0_Fq1nN0;N{ojVv& z^a4TqzoOfCIjUBzMtvvg(abxhROTVq#1-4pC!AAayPs#b+S}1jGJUAyQzv>y&5ceQ z@5!?whtklQBWaHdqv@*YBWY2iHx-*aY0DFL`nbfC8b71FpE`;vWcA~@H5N2N*_;Ls zv7j%5xkj(ukp9rqrnh$~(@IZyn)#aZ1=cmgK&KhKwx(l!%_02DD1~GC5g1&s!JVgd zq^R*H*=9Twe_ru=cOQK`4zj}q+nyMtWr`AcO}w`GOvcKU5SMwYNJmba`l8~bcBs{uHa%NbGgzLM!WPUYD&0JE1>H%%SF3mTmR=0c5u)`+w+C*JC{gfJQ+Nem2TV<*5 z@^9GU(S&WaZ{am53nnXyFzcp>{uSl?{v3_AwS!?>Yl}k56kA@{=d3-P_8f!}~ zEZrqJDFf-MVPk}}0hfgZQ}tQ-CqK4hW)kan;1rWBu4ki9sfg9zEyX({`imhxgT*k{ z!Q!y91H_@ZPGYygj^gA&{l%LX28reaJj9zNgT&f02k~#2sW^VLl4!H)8O!oI!j{P` zWQIGnSfgx$pfja+McAr|jUc*{ukd-;Jt^DLf7d!W+Qx99yP_B!t-_wW6XnD{^ z6Gz&>nXo0D#&lK>1A0GPlLjnSqI2C9Xg38}TJ)p?=H_p)So#L%E@k2K(;~cf=AMS( zB4!64gvOP4_ymPup?@@#<>q0>$PHNP7t8BwtC78HE_YV=MOoAjBX&a z-LH~}6&J~j@(U#1<{}9ldY%M&pCG+9>?MxJV#q;fD>Cu5m1KB!owT24x=_6Ji%_c9 zpM5>Cf_Zk&WAm?8v3>nz#94z3#aB^Q;--E!;(%K{Ma!ZdVuik`xZBQBES=X!e5Bzj zT3r|@TDSERqjy@0LCWf4*Us0h>cTO$`)MdEfAvp@ueTM9onA@m%lDDG)K-$2OF1`e zGL-8!pv@y0-miB;U;Z%EPL*Td?$WuaS=Jx2JA4Ry6XU1N3b zYxl|J+a+Xoa{^iX!iWsh`QiEWM^E9!moq|eu{Jwx8o=WEq_IJHlv}uD1Na@mZ?WG(eB`9&AA0bC!MC1r6GKR-Sqs{f_F&4rKrSgjqeCV6*=TzJ0BN z@#k7xe$x#HpzD`bB##lvnU zu$6R2ajiLSqbK4&TO#bQ8L~DS!|g;j{5q(D=CpQlH~0q0OH3os*CwVriYmf;D0yBk*GmaPt= zrf(lHx5HE{@mCQ4hE%gh?c3N9JtyXSG)pimIO3H$E}nQ#TF!N?8->GyfmdzbXfk@A zH4LIB!#5xj`W-t^7kmH~FUs+*p&A$S-yjC<=$xWR!?$Zw{l$hROJl! z)wWsFEd^%uT!#sDyKhY0wwv(#q$!=!-Gbie-jj~pW<~Fn8&NGSCE7ZM^HMT@BGX5X z4%;q6{awFd)SebZjA=p9hjgs#lZTrsB~Xm5z@W10$QyVUK3z9K50|5*vk*Ju*996v?AvU`r(#{={R~|eI&jXs52Vu^Deo%a2 zfljK5m*bw0FV-1E!(55zTewTFU0=iV3EPFhG6@SEk;rsj7O~?B*IC%hO6I3@i~SgS zi%sU5t=;{;uxFyW7`n<#R5!O3D|Dq+g<4Oz6^4g zKO$(HCij$fr`wP9q*W_zscw@E-8s^V-u`V)N4lHPQ_uBjh>Jc&r2)M%z=+NqZ$wKI z45{B79qO8_Lr09^43DNR%t-%^&?p(|`r;2x?`ucliDncoZbtjZMEs51g#)jT;M}(& z=w$F((W~3s%U=b1nJc(je;Oz6?7^X^I9TORK~b0+)R`%0o;uQXE|Y(ya&WWg3;9MT z=q(`V?C?iN1G9AecrDizUuPP?_1POTed|u5JO8)D$8CqS*mI}w z=Z!k^cMD`2Kcq16%04#kDA$t=I>co6A7dRJQYLu6U<2ODi!rhW;)2yZ#P6|2;>s0D zqFLw@_QQNHGnhA&tsYzZMVaaq43w#ZjN(U3?ERB_snqCZW6o;z(BrHUJ?a$GjaI0s)0s%3vgq6Tfo1nEXD>}Qvh)YW?4;LD!Lc}ASlON@L>tC2yUFn^ ze5WI8aV!udj7h=_j+54MWz@{}!R~8IF;gKOulgUvAB(eSjk%52GVh=;y%Tmp3bZ3% zi5~u{P9vUZ(i0sTbe@(9t+xM%b*iu7e)BRS9kMYkaU+%w--RLSH5fHmfu4xcrEgqJ zsQqzM+NNMi86eC|13hw_QT&OiL-J;;JTdPrilgoF8?KOFKs8Ezv_}|`5H23&s$>Ew>#Xwr@}Aj zFweo`%xvQu=+=1-ZoYYFiA%ylNVaBJ?7|kGg`Va=KjWESGRG#|>p-_%ziz;$v zgfd-Mu0-!FlBFBxeZ=Hh_mDf|C}w?1K&2Je_2gW};1pTvXkb7C#+cLDcP(k3983Ch zhZ!~OYfP`-ia8_=M{k^l_FdY zaj*UcO`7xd7v|h}j-A^sB3yGHWcqA@QOPoRbsK{dCYET*eNE=H#F6@*`z1T7)=Hml zHWf_PhY3FB$AmvouLVUL4K`55j?Me-$q(@HwLED}8m|<`i1}6@KR;D6Y zX&;otIxPCCMmLV>K?hv2rokI+s8UET>fgtbCghvYfJJ)LpoIIbHPvXosyh8ys7|x_ zJ#^_`Sz7y|3pqbNL6`Sx`&HFK_i`ot60hURx?4E7yau}-Rbs29i1Ay_;rk`d!(vvb zD|biejj{07TY{4GRM;HLMc;-ah@V#kwb93+v1vb&{I}uZlugKVUkqhefA~(Ym zMA6DzXjv92#5~WHOqH`Hn`O07%X1BHBz3{9T$RQ(Dbq_^USiUn>qyyl9C`BDs7Omh zN>n(M^LZA0rYWSmUy&X0{2plCi?qh2ORk;mRUuz_R%$X`Px$)6K`0;TF8rwJD{N-1 z(m^L$D>~(UBzq!XNNT)B5Kr^tWRzD0S(P+im@T2qr+GW8N}a)W(1Sws;P;ZC`IkuM z`kv5TFo)moW3a|J4fV5f;qd(k%u>!E&0NIOj#@N zmizZNbzDH=o0B;7<{%&c4k5Qo9dou>&_r+W*!1$GEl7&kG@g!5oSQ~_*ZwBN?FJUUniC(zexKTJsi4Yin3TE=+5Kw$6yN-I)5g!x>aR{8&QhXAEx$1A`CHV^yo)uyh#Qj!fli(ZzlYpqv?2jFGZJ8TZX;aR ztVF+6b75wth~&tb>~?}t{?PK2CH!TOYSDB$bo;npLNQQ(ZwndXSOVTjq6ETDLG2o?sRD`h0v?qiEB=_kUQ4uxu}au6J0!s%G}pvq)89VI_Zfm zrb|HY<>1WcgBVkL91p_FxGtL^*ZMMIUti|A19xz9{8L<={0`&@FdcHrWzcmz7lLRW+}PA@MZ;m`9VJCc*7pML)zj?OzA z>+k>LAtaHERA_5SsVF7)`+3SpX-a8m7Y)@%no=q=g^XnHvLh=Q_aP%m$gYe;5;CG` z{Lc4x^}lPB)~D+bu#$&~OAR4We+Jp!_u#7>j9st z$KX5gH-2P4^NF1`bx;v{0ZHC2yzjijU66O+^7f5#0j@KV+@z0;iB@pC?~ZbpC%7A$ zh;4#dNK$--FAqvlAYTqe`*&C~`2(ise8G(m{kS+V4EY-(q*zjvNb87_>MKG-H;}E7 z$xm4Gd)Q|+AW)w1Egz)7b$%IycQB5%pd<;^l_S@7P9{!YCKLC38FE=ek~mb3BlU-b zNw2>!DN|sc%nRemGZQg#`O{cZ>ch_N%616;s)p%*B`7uKqq2?h0Ov&F?#+kT%=VyY zZAjjlkIsTw+B9_w$+r6s~JSLFzh+v~NMj!Unp&x8WD)g55G+FrDFt z(qDn7UKxzHf&qB?-UCuhgXzE76iKHRLF!T)H*3yau2gn6r@ZL~rz-w}TNWb?3)Rhd zSA7eApLru+Aq0h{;h4n_LtsKMY_9pC?TjZ5GM(*`E~c;5{SW8A9YNfsgBZL1ES|h@ z#I=uscz-Pg%Z7{5y0M96fBb}bjR^797biF;NJ3{-u=^VGUjJSOjoFGA=d2Bx+3Qi# zw;K;wuf?t+7>_j)@v|cnj}8?hJ+KNfJL_@I`aNpU2=`m8r#hw=8aIC7JG+Q4*ApU3 zpNu8*1;-L&=5@6b{slFy9>^YQL7#jzdv?BK$uS z$v?vhq~*E@;eTa!u>KLuX1vjsXF}wqjxd?7AViL={)hUU4px)bqHIeME{49q^+!q2 zNQlJ9U0?J_-Nl|O=TX0Y7xM(L9^5hoyeU_N`xrZJVR9ii%Ul#E3|U4yJA3-WuHlf4 z6?|7;XZ=NI=uY#3dpYBl42QwWA`+UBVJOh|L;VoTJ)U+2$$r~lze^eVmjsaU?hB_o z(#N@;9FNB_D)7I&3md-LV)kFAA-ll5`Flfgg@&POI21Ek9jSAi457aT?X06RbYSgz>Z9vovo7yG#e(p|+^DW3@AtS69}LNS)zxgP8j zBx?7^la2A>i4~4e=Irm@Qg} zM#&7cTuH#bX^bhN?u#WE_uw+?D%?el5i+}KvzBuA*9qb7 zZw-XCT|l(|73OKN1aIRt*jPDXPnj3;9|hykz9*O{7=y>&k5RoZ5CS*dP}^sNQ*3_N zZnzr%+Gjyvof52E=OU$V8D8!-K*nuzEInd}6HLc6hSejVii2UK5{BE;!ZFew2H_2% zP^t`sx91}mc6uPy+7;tE@4->h1GTIVQa>++d4k@;cE%&>24$Qvu1^0EoL4SWQq~!ZyJF5dZ zQMHg`oS>H>Nmz89Wyp{~-12(}r&lhpVSCuIs(naavl0yntbS5%;z+wJimyduQr%n;f$cPhlu|f#%;V7DyA|y=%=5`I4(&zmVc!mKh}Zk$Kei7Q%nHMU zs&MrDj$nC*k(g=|4xNAyv^@<(O{PB%SNTKfzfde%`xM5!0xp%bvjSU>u%CrJ>fK7<)Ic zUKz_TIB(jID#rcPN@P9HO9MEo)dw4vO_hD33qfp-AbIKwT69YXGSoX7)^UZMQ zjF(7ooP#XqVzi%IeL0FtvFYGc>K7op{XZ1t-9X4aYm830jk>qYGr!7@`Ewq_vpfNf zjO+QVndvc?1mW&JS1jzkih}if5NSw|JzW#6UwM$`Z$V1=QEWPF^Zz{1;CgS|_X>cq zO9=E5*cwSkV2V#9Zizg>5W7#mNqhpMu}|PV?+Kc0qOik{@x!+zBg!EUed|6zp0W7V zk{BqfvmbFLc)H0MUziDwAkM)+*AR-k1P}OVG=SsVj&v)6vt1epz&V< z<_l-wgTyN+Mb{&+uM-Ws1~548CnV~A;G$?BZVPpxwWkeBJ~u;oQv<})-{bekN*LcR zhQv$?``Aome0u@EoH&%PX0roTFUTB<#ckua2%6l6pqD+kxTYVC#=S5tW$%n^6Ru9G zLAiE0N|nm-`fde+XO`mG7mB{v1c+D%K<=^|?j$)P@wqiHdIndQ?8Ul)HE=SJ!l_Mp z+;aIFT<@WFUe)TlY`=E@>_;+dIh$c`?rzn4Zt|a*&>uX5hvRK=;I=I~>sie=?g5fY zg0QDQ2G(biQ1B-a%B()_hZ2J>9^5rn&;6}nuAOu}Qtk-Y<98-pxF{)MoyAd{< zR>?-OR5}(5KLhnmhw$+{?6s&sPJIi@)9A+LeLWC8)W!0l*giOm0?VzBCJ9Bju$FkY4?JjT0X!>T8k{-Xdb!ynm9;|r#WeMOjF z8@i&KF;TA;p}D2_x4i%>HFA+Vj*o`_GT`wl9&aWG;y~~{SeoC2*8KDEHaQBLubXjk z(<&&*$>F$62^U#m$Sr#0#{0GIYEEg80smkJ;jcL;&8fY;$t6B{$0>R$pyRL^iWG0+ z!lm1g^>W5wo;NPJg(KT60bkUU@##tuq6?z&I3N%cI$e=ddjlUG*mvfz2|qp7VrI1hnIYdhQ;3m2!to%xnClrKPRI0OA_{)Cu70ERAfAVj`2>} z$dPyn!#2kCX{%%&`4-re{l>f?F`}zIfdodp!xr<|_<7te&CWFj7Yihy@~Fd8&MvEo}0tQWBv& zZlBInyf8X~(=6vNB;*cwt@j|dgVp0gQMkG{2~9UsFd--jx?a&JV|~bkPH$!N;k)S;X9Z)NA|M>#@*nTR;GO=Wm>v3jU4Tg-^x>=T>-Zcja6;GjJ5)QvyU+BnrV1m9A%&IP+ z&1)N?&Z@(uZw6F9P5|Wlxl;9)oJIZxt~YoAFM33YueQvKA4_ZbK5r%H-!gG(ny{B^ zbqe4f^wx0wkETK3_5r;6cNGHr?cf~khRMf*5HT3bvfYzW?VXByWl3<)i^lNuV5EQY zz!r8Nw(~g07*ZxUc*_tw+jg*Qy_2Z5xq|}U16Z;qYJIHcS2aT~wLIZcF%44U-DEGR$fQvuw#hrV(ipx!w=WTG1fQyg^u zredm74hB2&p>9+R!-sFsXkW(8n|H8TQjOcIYM{t;_&erT;-oO+Mk-e@KWjMzpD>@% z4?Y%2$D&%~F^W@DQM&v&`nNH@!bqGakgS*e_vj@1udhHUHR}gvi1~#Acfo5(hPAo2h;o-NKG4(A(WC}4siw}u$84$S= zkL~g78O^^7#{g@rzG05MgiR0*R6+N|3CyF@$W^yIL+ z1U721Y)_8O4Xe{3Czp&jnX$+o$Fkc~9$`_{J={KW6Gyh4K-m}Ov9~pb>igrMQ*0so z?LLIx_`-2Ld-h#o@q0-MOf0fdx~UMn@KRi5xwc}btFit`4Ssjj!2D1(2Fogt{qqgn zeiz|^bslDUW|rx<1xI@KV?g&j7Kbu6-c47ic`;7InMWYz%ooef z;xGF=kW=c8n7yu;f95WB|8RhKpgmg`XUs9QgR7Z2+siM(;~rzpUQWT3y@e zg@e^?cr)L0$y6W66+VI9vlJxAzd(yh29&p_Al@+^6?u`s`T(Z4aL17~c8vLW4%oaO z%Oy;3x$+pQ_g-iFt_!HoLqtytf%D8L*vTbgQc?y^`xG$EUj-Ie*TI}Ilp-=}Vfn2J zhu)XN%%%jh+wwu;S-o~W6^;_|P&SG{^Hw+5>~q77@^G9POoyYx3s}4kf_Yvzgbydf zzBCJ!%U&SACmj~r$w+AQ$CXUBcU$d&ki0sR`OL!;wWXNa$6j@5BfQ&s6lWJ(BK7s-6E@zkgZJ@U*z9eKs?qCEIeG#2_U?mt&}rm}FjlWa zGO|||;L*84oGj(T?Z0HyB}KzOCKO2(525zTj^)lCMxX_=W11+#f}O2fE|qf^Cg10D zUB_}q_D|HFclUG7%pfoR(@CHBvcKf0cjg-EIMtM%Z@5EcA`rQa^0HAA_&I((t+~6SL?GOq`Sk|G}q77mCL5B_a5G z)*JS!&Zw8T3ia{FFkG<@1-i!|efA0(1XvcYvj=Qj{os)j3LEuk2<}S7?-MWagK=Vm z^_p!7sr9TxYM;5T7w_#qqL#nT{J9fuR~VQ8`NMZ4@3T;a3tZpa1ky?ne< zeU6ggI24p7AjOrvqk3$=nUahxV-k?67=~l!j)<}|flkkSMAV6+C3ynU4W~nQ&TUB=VfW3)mu;DD@iwY=-bCzvYvc!AM(cVr+?!^C zIoCHq{Er!SFdgx$mC0zU&O?T19_Ed(`a|>yrWW{Pqn8J?G#$|&a}um|4ddkLc(hp% zg^WiNHR8lokId#eOG>rgk5%RmJNWYlE4uk-Z_cE)Pxa{ZGNuWc>qzYn2GGY2iF8%x zODgv?j~Z69ntib%r2o!i*(~c({`)$juik~qx(A3&48%ICc=YQq-FaI!?xtiR{rGbT zvAwV1c^tg=MZn~rFYe7_-a^}3Xe~JdCFjG4bw3XK2N!W?)DAL?fAFZ+6F(TIWo}vs zbWCHRt5%GE&K+<}Z-cjf3ue6hfE(8&L9I~}B^4)7KFrQ9-(;MSq}a8LaqBgnLf}{c1ZLi6y`tMVwB`%~ z9rdx+MhT*ye{&tv^0+mZoVf#W6S%jt&aN$9H-q0K6vA)+*~d?5pFvxuZl+gvU8LQ| zU1_6r2(3PtO6!gk)5?O6ber7=+Ob_3|14%=AXyoqVmcV?WbBD$_pr^+3!SGzq34~5 zu)It>R;DnP;G>4igl%#f)JqZ(oD>D&kRU8l@w*KL&b2GW~(_eE-jEA{H9#h$DdNCpj7>lt&75k3PL-6`pkkOipo^`4y(_Mva zvD-2E=6;+VItUHZGq8PT4x{bH5L&qbsY;7+@SZBFJC5Po{zv$$#rB4se1uqJ;AG_! z#1=e&YNZW)P94LS^9B%>SBBby-(0q68n-k*Q_&>Ae z(A9Hw=#mXqwARF(4$Xf|5Bg`&Ho3R7T&{)gNbaNiEPJWzj5Xkj)G;Ap1+stXF*el| zlqb5dxup+oszjnyG=*{4b8sm;4^hRrsJxKF_JJ3SJDUo(t?{T042SjgNBAUq4>5ag zK_lf7ti3M6c%= zU_4w0uYcvJT3!O5v_d!pCAo>~5mMVmuaUqT!%!WGCgw8Zc!cnCd zq>r#(!8|r^QNM@mug6d>JPlg8shoteKNouH74rj(MYY)sm=!I7&tWxu?P2;3dqsG2 zQV8lFkIHTGnD%!8Ovh;VOaAb@l6jP62!FWBqD_H=^kAhIX@`4*KHRpUaI(X8zh1bq)tcr1?`^qo5GK1>f+dzwCPtrw3cd1fLD7846L0@fsOFMf$Q_)Y~sh9k3THo@MF4x$K z{euSZ++xV~b7ScLy9m8ocd@A85jJ;3!LmIaI$gP_<`=-TF&|Zfx%ifygEx5@xNPtY zr4yo&zby!w3LdyG=7j4hx6w80Hr@)b8Y#z_%@W-pXX?p(RE&ARvxUXJ6v*YY;gzTW z*%B?p*!n$8kI(dl{4ZG9_X&T8K0^6iJ=WCMK=ODcKL0C)Txt=b8uFnt{1PV9@)5N? z4@VzlhqkrjVm5<3HtB3phHslv3enV zj8>p=#$wDdnF+%^@|ZYV3`h5~Ju#kXWnWLnXv2K$U#o$(c5MueTZe6GdbsG$J`d_B zx-}cqot9#z_c_MF_eXJd8ah035IQ{(jkA0qb>J$x>J8ZYxDe0QkH^F4LM}-50#{o{ zd6tnhJKIx$zwPO5zDwX6{{8?ts$C13w*DyHL+t7Py$`9OV;r5oFrOYcS4&0bw$p8Y z`e~xkFr}(PG}7xB9y5)?shLL^ll2&Ezh1(4;k%fB!w=r<47$1?3mbC^5VD{c|FQXS zJFAI*ioV3bfE?6^vwf#20fKfB*yj8QnZvGFta}&BC*Osh6w`3u^1!_ZzAT#}1oPPp z>wjFg;4CQqTZzwen76e+6-z$NgiN?RBKZa}l&+ z3GTjJhEfBjnGahI|HF%5*)|jB)@$I`WGmPY`{O+S83qNO!~Mh)3>;&6y2r=y{_6_- zzAJ@Bsd{e7*t?uyYCG?rb6K{knFRmru}l0x(E|Qhe=+(wZw0mB8`183OImZsjS625 zqLRAL=$)-aR8F9ds;q9O@rpmGVAPo06s0k_A=)f^tp5tEZe2#oB}<5BTVr8Ad!HV# zjFFRx_;ZWmLrXEFzQ2KQT?vwHixGdc2wOv6;xC)WOb|-_{~oQUd@#lrd*it8131q2 z!2VHIx8?>y^H&tEcPGRDQ#$V7WAl&?iBL6ofm0^W;mT#8>=Ao6#@1r`MaDK|n%!aR zK8Sto$H=dKsM+`8PyaXkw~FOw-+PaE!FL#0{2HG7a}gBDdK>T9yR7ezNfqv}F?E1? z@eK%cZDgLiKU_nr7boNG!fkn5!?i7B?6H%p5E#4>)=HaDTdIj?yBO<~X+ka?n+BU~ zSvcO8!@7KVGjwW~Z9E@Ap=t64A!nGLlIvrxeNC}NT`a6C#A;%*nPt)1yi>SCZR zkbv9jtbRzkg~QGU7~rNdjcOnFy*h#uzc7nCwLv&rWadP^Qm`%GVSF;bZR>Bor}P49 zq_Leg?mk7IX4ukwj1L^JA%cqRN~0h8Us12#I=V{eE7g_%MWcQQlx%qRPkTu|d z$BL{sKK2&)Qg=`!c>XY*c&;!Efc*8d-*gW~KN{nIh19~w_`LfC6nbByA*T#tt0+@5mT9tGm>eb zd^FkqwY?J5_zS$#`miqV7dFNJ#espp=t%#C*6v=!4z)vSMI*jkXS&tF*Vx3okC9A& zsK|6I;!^^kE$;~tN!D|ieHArlb&<;=|{qJQdYe9PK}e_bblkQ0y( z+zzKrJh;*oSW&PDze;8!=c^*pji=$5i3~)yPGtM^L_|)SgwM&7*!@BpP5UR~r08te zyJ@38>OAHh^~Co>VYvAv40{i;J^i%>PBp4RJoOJJ^fHrMl6s8$5Oar@|D+{n&o*oR z^s8z7&h_8;`?ck$$y^>iaO?!t@wrWf>m8}BR{*Vwi>3}2UQn4&B~*9D2ga!Fpws(* z(Y;=RxdJl;bN3y0gTeh}J`#mr}I}BZt=}=8@cwIi^4HN;t2yyv#NNi!de zE)Fqf$Ns&xFo>~7{yRJD_PvPh^H@Fid<*R719|t)-3uA)J1mCNA;Nem zD}+C8f=s_AjNq1usQfkusk3!pkog}RSGr@hxj$wW_+XQk6P!Eu;l6?rgx8mIe?QuD zZi~in|9s1`&*|&&PtsVvM^h`m@2UiC-JnV}mu{t}S6!xKJYA`n&;#1?HtrqSltC6*%3L6<~bgT7S)>C+e932X~*QprwiA7IC7#bHOWA?J= z7>y}Jn@c5JKYYU3flhSPHo-@@n)LL%!+rwwzq5@rC6TOGC$nE&-jBo-H`JBHUtly1vlRWzZrX>{KN_@ zojnVGL?_|(eU_n-{+BDt=;0RHcXQ$;9%fA51lLL{^t!s>*%GFhBa^0gddApzW<^NYTj;@NDLq9PO(8##s^p~O? zRWZ@9hWNKHZ6I{{gr(|3b@|pO~%JhmM*K*q-}@^C5L;N-V?9s6s4N$i{ZoyT5E0 z&Gyj%y!T>#I*l8UP1%DCMJ;?W;^F?&GtiyS)kKX;@R$Q^+nu04*A^G7593RTIH*5bzZ{{&qmjz8{e% znE$w{2eA?z*p}Uhtj1RmzVZfXZ&^lN`DbW}c0o??JDjsRVO;VBB4?X%_jEl>8!8az zRE+84C}uJJP!ZGVs6~ZCcE}5Xa(0+mdjeTEHzRb7Ax5UKdW;Le?f+ijPFg9hPbtTI ztupu-@^O6lDeGlMqxW$jTrJ$tHsuEPd76T=S&wm_7Gd52xqQL5 zoN0F(XKyqfZ?>!8vCMWvvf6(#>-mXox`~~$&LKZ<3;5$_LjB(dt|k3Ex4QlSPb>Rd zj{bc&zK&=k|4EVz4G&yFZMSct#aoZkk^^_>p?DuECmlsMv?kHQt}I%uUr5hwsGuLj z8)=U27g~Lyg9bhRPOYu`X?aB-eY)QR3)pifH^&D@%^yOMd9SLOU+wnA7#OnW)-0Q*SM#=9R&E%S)8EvOa+)MZw!QkV~$G zVE8AbmRT`S=}Hg;%3@eiIJ3e@2{S3pynm z;hEnDz>4q)%@}hVds1s(gU<^b*#~&SV*f$(Sz{g|VM}5cvEKmOVd$Qxf`2x4Q&6 ztZuM5ILJ-WsN%ZM=W_zW6`YdUcTVHSL_DETuDitLX~IM%ps#Gc8}!Ml)y!)%e{-rH#AjzI9&MZ}$kIoBgnu&4n9G z{ou^lV49nw(QlOrfs`_A7;V5+xmK*JZAC}SXIQTMj6}5JmwX%RA$-OK*JhM*_28Em zLH6`(R8$n3v$e-w{wjgDPWp6p>URIW7dU#WzER5 zZ@@rMGqN5xqA;JaYd*0a($NyOR)s8wkuuiZb1YJMikXh#m~-C;+l1L(_vSX7UCp5R zNFNDffLz&w(4F@Hd{$$`8P_6Y9MgWy8idrNUWg~PLSkbz!fvql+#(0pRg+=!Cm3Ec zov_fz9L5U`@iTk{?u5%h^!O;Z#-x!e4=v|<7k}h33x_#($is4(I?m*7$7P>mEPuib z%Q6q(^y*FQJyd~0^)Tl*Hi-K(rIYu<+BL_!z?yGvSHlmXvb5~XN?I~sh3ip)H$opOcm*R3X^}T_-YAJZDYH{^t6Xbk8q4;elW`6kzH|D*|D3Bvc z+7d+Yln7}t8^RyOKD6EJWIgS#NPXW3Q@3}hpY#fca@hJ@c>#}w&+xb62`rlfa6j7} zy07nmDxF6lJCF9euft696-+;~5$#s?aF2Wj+4%Qx*8T~fdxGR0{&W{tIQWSRJlDkqmx|#d zyKC{UYvRzPZ79EFiWN&uVX)2!`qA1DC{@7ntk0b9Z6_|(&X4!Ky(h=wtuOybPapsM zo7r>&F`)Y#Ptggx@6Z}oPx@$UIF)ruqRO^>dTd2Gwd}2=iT&?st#2dE-qJ#!)VI)u z$}Ke0v6V`#`$Fx)+o+!syOXlnW?oq^q~`=<$;4ocYA~Jt{z$afq+tKPVi>SH=I(*d zn6KIn1%5kbuKWu5@!w!!){R=lUW9Gwh4I90rr~V^gA6gIQ#pplmSe_@I#_tI8ZV>` zTUK?U&GQ$&d=e+Rc8qDUQk*;w7bRN1g~**XV+gBaF<<)!^5k2gyR!=BW`)RjkcHir z$>_NF1VTE#P&(iO<;I)%Wq$(i-WWoB#ac8i*Tm7On~~~c2eqt3##3p+ge;bo!*YXb zwI`Bgo5abY|5zrK++T!k=)|IgCYDQ34efD^$YYv-rMtYaNb(j+9L@0NtseTvDC6CS z@#w7j$<8rpm?@Zu3&J&`|BC6D&U?*(|wWd2hyHfM~AbPkYp58f}PKOl=>2j`?j@rDZ z4w=o2@AR4GcYUEZ>b}ssBA@Ap!>v@fw3T)&Zl&g@T_L~U2i0}{II0{71({$3_y&Pj zABH986Onx;AAW^35M{OXD)v6r27iT(z&FqpU9i&b#f^{wSXKOlRNMgMPIqI>^G{53 zUJv20_XxlJ3Gb$VW!l1SBp&R?I;IJB6(2|V|3rzLgD6?LW*q6CI*$Aa5g`-Gg~-Aa zLr^E*A=%S_B`-@M%{-Zp?ViDNW*8F3x}*Kq4P+*tfa%{IIJR&D{v2R>s-LG|QSE^v z=d)qIs2g@sKFmw- zf%{w+^oLsGVWknAe=dO}yF;a@iQwLOVKkIWp!59p?cy!Y+KL#;(}~VI=-Nzs~zj6bmR7{eza}=iN=`U2viXuGIqjb%>1!rWPu1# z*ffshri>@ObHqvWsc}T;sSv5P9ftF?9@J~H+C}9JmP@gD`mqSKv3YKr3iCh9oq=Tf zVH}!z4fA{4Fe;vcX2%Z@XkuB5Tc?n#67$Hbjf+Xadlh2Zu1qRS|HssjB+X3M@2tq) zDW-|6lKYHJH%f5kEYocK@dfXE{Ua7chQ^=rE=-zRZB=0QB1i(97~8y`K$XpUn`? zE*ph*Jlp3U3lo<(AtKi$LX4fnh|S>%Btlb?{9YwNj7`PK*E!?JaHt@Oef9&~t!A_( zmcZ%?(;e>&#nqkO=(6-;`5+-E4aQ zH!n*iC%Hg}Kd(7~f23)I@2<3vzH%|4O2W>RW<*kxLz(o^f>$(Yq?A5WDx+<$-qMob z8oK&OJyr7iNb^RUsEK+Ly?woc3LYq-m&d-Q!WP9e>ZB9I4?e&y#^BXpeF)D=KcL4C zjpGCG;#@dZvYF{(y8=vKU5)%NO_;u;9qu2xpmMzryZ8J=p!N`ciV2Xp?9A>`6(r|h zGnPokDE6KkL-sR{)87#xvSrp-vdWg_O1g@X=3649`hX}2V?O;0uO!ILqKQPfeiE6^ zn48LlGNfKdhQyRelKwYrVbIfCm8jM_E`L1lOjU^a^Xv_kV z)VP{R8S0Xwo*Rhl2*~Bc73A!ZMMR=}A^F~@Oypvwk;&u4NM`F0mRt5BIJgl_fBEQU z`qT}r@hp4n8M>E+qUEm*Zavb%ynZEIxjq|`kt=ccvktZ%(ns-Df)ndCAv~}M0&8aD zmG@M{Kb?SIFMBxUpXuB?i_4s7QV(x))c5SKPZshu7+>T-TO{+q#Lt}X@g!N6|Tym?^6n?L{BkQKl+v$dA*~RgYW2qngZIH%%@AkbLbQC z9NK%`5ySm%c+=sH?zMhwh8};3qp{Ttp^Bj~pjBxZSIi8u2sC~RTuxM{+qpjnvlI7P|ZDlw9JUYsPe9MCai#K}@; zadKk2Bq_+AL?qN@iJE~NF?lLa{)9~>by`Y9dDe7dr9X|tdru~QHPS2}ej+*MAVD4n zN)ojll0->Ej;v()qStn^>_qPs#8s3>WLE2vq_KL$^ph^B8sSJ);u4}^G@TfP%aP;h zl0;{_6sf*0PBLx`!F^jdn}u<<=eBVg6W{Un$P8(z zMEuH8cQ@z1D30gL9qi&ic)f@!g&d{7TwSR2ln}bVC!Rh$oI3S>WST$v zoMpe|(AH_MXunP|JtLh(tIE@;O-m{rzcH2WvcCn3+1;oIs|q9Iv zy!nLRHjju{%_aHQ=8!*XbBU|Ue4^g4n8X+_BLy`pNv*y%dC6Z#X6hP{>~q`5xcOU2 zv!@=B4qiv@8|V`Ei|fccYjrZyb`H_{BuS#T|GEhw?7(;w|x(CyO_MngT^-lXO$6Rd%*idmu z$MkU~{_nWL+-Kay0Xy#RDRpk^lp3B~{xV*e!Ll5IFiF1PeoKDf1Im}&B2TN5cd%SH z8@ht$Lc==U>00#%^ygm}sKq*qIl1FjQ^|JF(50+C@}J2?{QE>9xAO(U{J z%wzm|8nLgONlZS@CE@0C$%4{(WTwMHQX`~FG?;QcDpQr5`?G}HU$}xye4<4Lf^^6l z`Hf^{!cMZn)|AYXJxs!U&B&pbr^ue#qvUw15fQ21NM?m?ASx$8p2aOC7t|HVxdP_l zoACu&)$cHd`9sa(L(#q52hk_&*sMALJ(=lvFIIrY$xom;ISo%M^B})F7nAi_4JBv` zjX*QTIzEWkM@`VsvL11n^HG~83Ek;qn3l4U6Zn|Ko%(#2%l^!9VTpp=<7F{Cv9(6L zX4`GrnhTEPNIq5Mhgt^kSJ;oG7vJj81C|z4s^ALMDz>DX(@m*`zdpTawVHN|Z=&i- zwzPeAF#U2elUf@T(vmMPsDRKj+Ljkd-!6`%&aKCBM9Bu1zdIu!#Rsb8VR&8_jtNY+ z`%5nr!g7ycyg!l6cyi!9ne~EVm={j56~j|luA1u~cvJ`w?-&uHp~NyJqb14qUs6Qq zvNZAeB|{>s<%n#mJbB+JPbw3XNXGYBq+;J}679uuuIJ7r#|2afUv)8=`fdr4lv+v@ zTbGjdb<4<=2O4DK(>3JDWwmsNX@&$ybAtkD&%9XAJb{hxQBz24&q_PdJM}igM*L~o4bv2R*Rdt zQ#FNL+k78xR*eC-ZA~f9$ylHFecpiftF-spF0H4u&z>8b!&HYkTk_2K8X7fxIlaa7 zLaz}W_jL#5jS{NiHH|JfS;xPmxR&3c^nf3>Q;JIE9i<Z!NAKgAzAgi+E)*heMJ1G1 zHKA3g6Ju+BGVY}S$z$Gd58nwSOm7kqjguvHRg+1Y`xGMKGL7WFQ6%NbiX=j94!L4G zm$7K(k%JTFkragmWU1Z~a`T!Ru^}tSl%f^n@HP!{VA(1%)TB+;dFzr#?S|x&`W~WT zcAT_UoF=<1tcZ>2C6d6qNn)uDfuRlgAb)|JHatSi)eK4K?8Ri=3ORCdr39Hj@dy4* z{fKL;Utq$5Na$Gk;iZ!g8vX>JZfOeB=T^X0vje9HoK zD5)abx8zE;in?~raWlUhe$@p2t^T|GoZ_GSx;Kh+NSH9~;0=v9*s7-@ER`Zvq7;`5SVp;H8%i|#p>$~!pKL@(-82cJDlS9T zGal37Sqh||F@DrqXOIN9*(7_3GSNOapM<@dM^q;*AeTNbWb7DK!m<2Ty(2&V`(q*n+g;0 z<>JIuR)l0OAB1pUD-N712aklpi}B1i+irp!)4aw#a7N1FbCBOW1v$ZeoLpEtXKGZ# z?TJg}l*58KkM~xb*6D9N^TDhvrNNY(jsxFwCQ_Fi;qXN{>SsE#D>Bov6CN+giM^Da z(`I>$U!C=Yf2`h|A5*wGN2r9)?N}rU$!*`Vj&EOzT0cMT8^>L*yL7Ymo?NjN81rq2ui@d$a?!SW*i1geMjP38ll$~`jZG8^^ z3~zjVyd4*^72zu-GdK$JMo*)`@y9w4jjF7fs}HT>`j)oo6|Lk#XF+$?e+t zGHyAWjV1Y~>LmC)@wl9Sb1v~ZyR@?H4F75C9QEO8w{GT*JEfpK!O($Mk)y~P?s$+@ zxMBv?yuh+a543V8TU4;%y8;?-a-3=5B3f^EkXB{s(n-N;R6<3K`c7U5o5O4txu5l8 z{xDsnng_f#2cV73-DBrJhVPCj=oZJoO!6tyFQ-D&gpbJ^U&C*64eU>TW}1UO)@K(W z+M!~^SVfu`icTXDX*0<#t$Ac^)gq$gy@V_eUq;#^RuEG$4RTyjgZ$gOip+P@CKG$t zk$(~TBsJ8K7?kWF!ObS5?*}_;(#(it@;Nf~`gx+GU`YxbPZOzqC&?U#{iJ*RVbZ>1 z4>53EM*{B366?SsobU`n_cNyRv3|g%9O>ZQiD>5OJOAKB2TtLRQxWdh3X)&nrjU+j zs$@dKN^JH51U+Ejek3G?x zHFH`*7AKs`(?7P36RbMRwRW8624o#M!QQp31_{fKRI16bxxw+2SPP@XSk<1gYHFKd&c-i)CZmSgR$;z1jPAI zP_-Zia=meg_e{d_kmtBon~UDMH}HB`kAziU5#RnBarweT;_yUb6FY_c6PrbD=r17p zCzlX27V>)4RR)$P=nomj{w1{h;F8S@iIPh-j=&~Ua!n-H)6#(3^z&kc_AOyHuo{2xhY z8jj`jzhSbJM0RaT3QT zj18z2$zuC~qe9u*Ho>K|Kv<-}1l1%#IH>rWSQ!n)W#d^$)^)(6+nzY*9|*OP5fH2J z&bce+5gq;iXR7@s@ATNE@qSen43Fl)d}S&8pVh*6OgqjGm!u_?^3->?1`V;)rqu%s zsFok+4nLbt2Su9FCL2PpDbJ;wzs#flz4NJ5%pw}nYDtZ}ms6#=wzR5d3srR5PsKJz zXnfl~+S;&(>b-KHR`r&2T(cQfij}9LIf_`&W=#f{Mw5lIyUFi{%Otw(0g+z$MYLFN zGO7FeMkMXMn;fdj#1roUH1XPWdSZ>W!8_D zX1~Fe#o3@cLy@t~7m@w#v9e?jtbGP^m;F1U`@x!+rN1OIPYmbGss3<$`-wbXUqjA~ zyhAMVE|3ICM`CqxDv389L@G_6iahH5M1SU*i+1G>6Uht@6`irsB*#X35vw)%#N<^s zX}YWmrx01Z9(30fiIbyZ1;3X`Lc^#Vf_z1y zkU#J#nR~h)_u}bb&%K451#%R%&A!Oz*^t1q5pZse!3wE(Y>Q9Cx~oYTp?(X!R(BB2 zJIcj!{H!sA=Oc0(;5YCOzrz|pf2OF>`ah$n#ZLp;?LCD~zc-79Iua@qIFG)zTR@N1 zE~K^77SXSgOK47%CA}AJ#kpE5=)!0_nsQ}3mG<0C9eTFXTchmh<%YSmm+#XLW^`j& zT^5;fDUJMeJxZ>(7K-Ku*pl5L9^~TJ2D194EZIKn5E(8R2K6f&;Njeh3oED7Q#B5B zqS9`27rrFTQnE6zB@MN0kGL%Z6`-iNw5;(k$!_RTsVEttcl&4E!hszkq zCx;Q^Cn2Ptdn3tPt%#S0c@OBAde+NaL-0WM6L}DOK1_%zrH*jv>>Cv4#$r zKVONoj%^d!oGKJu_*oz-_^L?Uz8ogi29Jq}t}GT*=%JxUk8^MSlGeAy5hOURQ>pPnlM}rzsb0eez;rG4@PIYN!-U~ z^7h(CVioX!XvkeAdHY<+`vO~XVD@4nE~Z4=Z4Oy-!i2nhVL zeZnjF;~$14Gomo{(tpSycaZ$~8*Ycn(x1ti^yU(MnqWMQPHPoW=hF*l<&(wqg{>w1 zP-;or?px7M`4R?P#4BesK2-qi2XQXRTx7Ye~dg4^7)jElyL5~(IXwk&wW2j=% zYAUV1gEsu#;!Fzc3$P2Cwo;W+N63IGU z_&9bXEjl-a9v>y5)hP? zq|Y8Kr=ONCrrS--XxI}za^KN>d2RvLNZ0+2I}()=)t{5PMGxg$OhDZhLC`M62qGmjCC z7r{j4nk<=hT#>x~@=>G`cZuZPnStWWKzwTqAod0`iB|etQHzBvxtkbHPIi}>RnGPl zJkRdk?IGiebWF|r^nogu_H=OwZ4Q$y;}>5!9M29@)iK`3_!Ek8XVQ|gU_ z4}Ce?=&)J(9F|x+)!#IF8N_okKT`TtNGN zQyTVmK3(0ugsz~==?-^0>XWpYE_u3x`VQDZ$Et3m>B$>t_7*$peQqIDj9p8kN|fnS zrQ48kK7o`E>IfdnvkdFTP%B#v`l&^l?s_?teh8RDr9)0qL)Bn<^k5*pB;`Y8j1Evc z>18zX`UGm!+6(o)oLjT>2~uinP$qo9+=sH%iF3x)t`4QYSMvAhi~;mnem2PM3z#<| z4%;VO!`HXI$aCMuc{h_`aCrdsKQ17te_xU>Ij_jj3+-h0fg+-B|CE$Y%Oa_F;|O($ zA_)_ch|={Eq9rW>_xm-(?A9I<(r>yr>~gfQKjxOO?L-JKg7@b>^Yv!F9UVFP;xy9T zq(?F`2NU}@PGnT|ERkC2W@7&MF>!HPEn-VOgaz^!1cQM=f=DSyxKR=)(k&fIelFcj z7VtIn$H^ZgF#_r&G@Erk)e_(vYT8RR8-4S~`9w)qJQ&v-&I1wjOyZ7b{I|6}a!9)`m`=vxe?6 zTS|Mrt)cxjXVM?aPIRf$DQbAgpN^OwN&O^!Y5S55RJ+)iR%uDoxJUPSw_q_YP4Psx z)>T9%-{%?2Hsqa>qtdlQ=v$w`ba(DYIPnhL{eck>By%}$Ed#T)cu!`e11x&SLFHE` zF+(F+@vfR&Yxqju52z=5pB9q@?xM&q$|mv~Gf1&fA<1)WAwH6v1JSUH{4F~pitCXT zR90OT@;7NSk(&X_d#}J;jPr%={fWYWXUoaPLouX&wGnYSkVy7aO(F}$o+QSpgJk+= zi~e?53lr8U3GX{%#h=Pz#P42Zn%UjW6OFT&MEWlHkbL9Eq&)T)3Hz&xDOS8E6gU@u zHEeKX)@H=-+J*j_JMqkB9rg`12N|`JoYm1pd#xfqywibs^bO=(4?@TtdF+j^A`#KO z#HV6A>=&mZ`Ar-Cef*3FwXZ0%;_kh3GigZA68hlUYC2JBC)L^MMvs5?rJ9y!X_J-< zy{BzOZFEiPYQObVtIVB_u=1j}Wc}%FKVN!S!0%M2?cBju%i!{fC^CHJG6cMi#jq!d5Pjfzjv=ztqWT}s znB?(Z8o$qeEx>i-U5xmdh_{0dW5y9(WJ?Z!h303H^rneet2dHC%VZ!dY9ODRpOeFG zPe^WE4*5^Bm{^{@Lh^rp6UF^pK=usN6Ds~46sGJ|Vz%=g*xe1AS>2^+ta;;5=D+xc zaQx4uoP&Ixk;pr2f98H5Im@1t+$XAtw%mlJU6myM=p3@k{v2sv7flid$B@^lvSP>1 z4zn#GWunP@mykE=SI7X)203c^jkt-Fp#ND90qIU@DH7{(cRPP1nM;)mbDqJd=!=@LqJbU_Nonu|h=IBRFdRhmj#VnC5Z{ zs!r=s&3opeks0vN7(oTGEp?ULMu)oE(@XAa=$Kuc36`*)Du@qKH47h_`oo_Nhzq9m zo?-N#WCT507(~?%deNxU+o;*ah18rZpdP(uwAYn?ZeKmA+`c7LuUdRlL`f-RO?! zu0=5UpbL}R3RqwJmTal~NCuj|Bk@sXL~Gwe-e12)Hm;pRYWDOKH6`p8g?Qf<`!ps9 z+xiLw!xyue^BgbsrOlU}bmx9Iu{P7XcUsu6-$;mfsYDiDQAOB-v55UQ8dm13I0rHW zZiDUkUDG6llDuz^&*^=36#Xv{4&CP)!EqCVamN@Q_=8U6W@N_Cind~&+XxMsLWc23{PojU-(Om4|I@_m(R$r zF^S}&Pdv#SG#I+pKA6dwdmb->peDzkGueBK+`CY!Fq+cC-#fMA zsb8^(=6zj7SI^td_Zo-k-SHlD+2?b#bNxkHw%U)rK6QzTJ5JGAo$F}OiJ4UEoGw*q z;p-%KogL?YFHYD(m-bAd!bt_{lv0b1kvH+p$_HEQEU-qsgxuMCn!GtyM0BSpQGMw? zByjeBe~on1wC=(a-?5M_x`@&`4xLu`U=;_GTn+lY-?^RVc5}fvcP+M8|Eh zN@S0>;R`TDRvW{8J4o233*@`zX0j$!Y_?VXk(tl*DME>1yl_b6hT!$?g;1hs$ZpZ8 z%)DYFTiEKw{0_Raq}AJ5oaRJ!(S5COO?e4fJk5j5i0LD`+kG>QX)IwZX%Q0BFI>~ zJhEWaTe5W77jmcj7b%TuCbdx&#Jr%LTsylEQM#ru*)|ZBW#37_17kd_Sq_7Z+T5cz z6|4Qtq2c|Mj6YsV;u{uY+@MGlDEx;T<*|7G@&A~MIatO29j&^YUvxSjIdU&h=h}=A zO(}Zt1o!Yb>Cwz>+&i{!E{*Y6M8_T=RC0znJ!!O;heKA;u|;#}ZufE2yh?@kju=Qo zx)rJVQyrQYX-4(u?4-F03+UL_n)K(3X4FpQyen}6^6Epe=1vs;ZHPtW+-L-Sd4ZHg zfARHM5kkJ4!nBjp*m!6NmY&^$`6b5?`f5K8jN62Z%7ptUv=JQENW#}SkgWDlQTzmd zvCqR5LTXHu5R&&=m?9d;{HCh0_&Kv!P{MR}Y3B-7ntPJPOF6Ltueg^bFiQx1l_~y{ ztW1>TwiDaKQ?WfG4CXyqkUL$?bEJ~=K-&O1lb^vWRHbNQgEZ~@!Tlbq6_}z&m!MT# zA!y4-3jh5ZAp|yii_;wXnYo)}i2Twfl25%yiKSXNIVjvDL&IZ;%ESN2jGm7~vQrLY ztY^TkXf8@iXJDl}cL3g53{#`Cs1I?)@ny%cYViV?SoOy-nf_2Ysg3?pTkxO8SnBd-VBhEADtd`7!oUywL zE*;i9E2o3BkTzoR;y7od65&&mBIO{&5*#iQa}-E^}( zss*B<$z#dr1)GSl*MgMZj3b^YpUJm(U&tbL9q3J%58cSE=-hG?@$@2`rzRqQU=lV= z;XDVPHM=s$6ECl=hGCf=mPm_`FKGqem0Pg^e#qgug{<_qFkaJvYmfW*en*O0&*d4i zR|Dzx0wp@BOoO&R7*4kz8cEF`Xw&7tId@-dMBCD4(wU~Z)Skc3hJ`3ot93lf#Is-5 zjtrusAF9%Zz1&4|dKfML%I_qnOsBG;{ixE7yEyE318y`-V` zWufpJKSz!A!>8(fa7eV`yre_Wsy~1z9nSxb3d0m9rtonqZrpGB%K zq)_!?7ZPH!F>4w>7c|P!Ve1v?>js|h(KerbG-fni`ptMvHm3#%5o6eatonA&~KeR zantz~g4 zR;)2;2D|8>&pc9|2sP8<#d8c3MTvR`Nrfs;OU}NIFN;4w`;G*4DpRE<<{C6VemLDX zYzn83?@#Xm@ z%-fWV7Ug*8-rfq&!7gxLwh1B1_S{9lIq!kW&>QiT%${vcyjq%aB0n7x7wnZ3norsZ zN1|^CXUE91ADTu?>-R1;RMw5H8DYRysEua3-VbK)f)tpuco0+XbrzZQ^fMbW(uWL6 zw?Wq5+sKRf1JygS^t`GDom{O(Cx|rYtpsCQJ##YknrukFP9H;0&L2b7HLfzl$&pMW z@iH^Y^I?O1H!~*}!U}7&nC|H=Vf^&_LRI`hp?_$XxN`JVGn?_ABC2(bw6zR?{V_w_ zj5C4FCl5@z5)RGK1lY#j!0=&#H$Owt!cjK638ySJcnzMPu*)n(CuSTe*A2`0ID1c}vry z)%iUCnu(I}sp#x?6I-q)VH9URxA^*@6R|Xh`c4qGRfFhDnr&Sb8VfD{%sUGBYsVsGBp#WT=y;B3+ur{SICV^x%?7 z)TP>l?)f;0s@sgC%eIW8>W9Nv!o&zRKqrdb`5ewJkMd+6E<3PSY9`DeU!H9|$)DHc zeS+t+JK{&);i3%-WJvYDXGC?B4pu&$4^fXj(jEuEvn&ejD>x^4Cg*m|F?@_Y+?$|2xd^H5+?Qb6%z4#c5qSFt#!7$1-m%RvQEkB66Se61 z^9p7oUSQ0+XNa%JgY$_8xSo}Xg$;M``OgDfC=}4m*>0j`3`&xPDEU{0jG;B~YHS27 zKXWGI4=h*c1E)ptaC#dG#mAh< z_jeKRm0w53;`Im})j+h=hhgWpKcqhH5pgOGA)AjbAt5S)==J=Y;y(o{LeaMq;-R+l zg)s*%2@YW=g!rq+MK4C37D8^P2-96=2*))2g+T|5#j~WI5-E)oQqixLyPz##Zx)RG z{5$rkqa9k;dEVYqlDf$LgX|_*>g8xe_qChQIsHxO+v7%brk@e@8WYUi247|;%feWa zPbf=qImg__ZDq64XR@AbIktY*J>m7m*}}GR-<*U_3LRU+mVDBo!OYq^9=94m*D|-yzSL| zkG?nWp}hPZl0&M{zV11yjSJ!L%b&F?VgzpEY>J^&5XzQD_}4X-P|aNf~Z{OtaYbAP{}zL&dAGkDi`^<(^- zmW8WRIb-x~60rF?b06&OVJ=1E#9L1mOW70 z5wEUbvaw$(>9Ko5_HI^y$C2^4xM306439!8G7R~RHy~S=fSigj_K)d8Xh9C2u?7t5MQ_;1hLwhE24P)-NcMLlJvff!qhXIV>ToJ+sk9%${BfA{i-Sm@zW-_fa zgV=Y4JmLF!TVV!WFP2XVgM7BN)S#IP)Tfb;tIIioBM zl0QMJp$g=fVjs!_bN z4gq>i7@pP6``sULZTBCnHEGAnE-{8TKEcNo*|3q|?h=``A$1awc8Vbm+26@Pr_rb=22Omu2$wyH zXk7jfNn=Y<@Ba#Z?|D}w;~Q2V;orT(uMjCCLuJls)6X5F>7Rkxw0G+Ws%$@kHX5H{ zm`_{-a+BIDy-q1h_u=mxHu-_lObPgO73%h zHs?=@IOoHZ?+4!B#|-lfC_PO@a6l?DKJzU5;0I9DdyJ{eOK?ED98FH;&^lIuJ=-{k z`Fu4NU#Ld=nQ{z$%I7@QFK}u28*~h6K=|4gT%7z7x2Ex&*D*dP;^)`;P5J1^jDpR*zX& zG0zZ94~O8_t^qKKZXwpPFNxovT;ew_gJeadlS`cs$>v9eBv?xzt_|sA)rl9R-9`$L zGxT8ccqyD^-S}LMvmWd2;9S!a989X>xuSR2bmARev}B;k@EgW8a4rYWgIC__!_*K} znpmkuFXgFF>B~xVyN41@vE9dZ&vj+?)q9!CX$NNgYyzu{t{479xeIn(PsFP>$C&A8 z^ob7dwj*~61gYv=xbnG?b?jZ_9lV9R%VUsU8HeUow{g-b z6KgMU|Jw01NZ-i9!EpjCmk205$^9XLN!XFhxxjuYxEv|QAO3!N;#kG~X0^}`eGj+v zchI(dj%5oUqOZIzH!CIgL)l1SSk#qUbx5wTSr zsR`Pc=rsv#I~GE{ZYv7rbMKpH1d?a-HN^8N&bYmS1osaF$9zUfMl)91ai>wOfF*rz z(3Dz>{Z{o*b8N(4Lm7I}kd<{UW_Gg|GE-eMW+|i0_WpeK7`)GTUhWa0T<6jV#szcR9GEC z|Is!mvA0EI?|yzZ4aDf}sfavzALBUp_3q9joFBw}UB!N|k~#zPLuZkj;}4S|$*8%T zi}ClKai&i(&qx&`S*{RvnPSLT+{UMsVW0|F8g9%0XH{j zI-_OsO2qt{1d{`kp&n+$Id~(n#Zna|yY10heGh-~oULF-07TJ;6QIcMO6 zej<0QUPb1|TlgN9jRhBY&+Nf{?x#sZ-|U-srGE{ETSG8lmlw?L?ZcIl^|+?63<;?g z_?^UOkVVU2T(KIp^_yU`Wgngl#P&o{B$D;A%5 z25azQKUl~d!uW_C+#$LPKf7EJ{vTgAXZdq4826jF-Q+%@Z1e^`#YF=?>(*~V@|Sj0 z-swQw-C8pbyqJ^I^?0h&E*)0W<`_l%X(?ABvi%ih-*Z{G!Y;n+VJ#@CKMSXL&M)s51*hZO+d4G` zV=Yn<_VG5JMcu?r?>Jnv3d7D_7tx~Ri>s}^DC~C$&y&MpJ3Jm6-lXF44Kef+p5fKz zO5PRWK7(u7_<8)u?LwIE&O!e8dvLXnL&x|)Sa0U{{6^g2WV->b zdpJXXPZ9FZJi^OO4sPJihRh44cOL3*#}y&iILQO9 zhTG8l)f%l&7a+gW6#p%nhQRlea3pFXv{EPGZuMkr-8>BzJ7+*5p8tN~T%2BOiQ)3!JV*fmXdlTyAxdlQZQZ6K@3B zmy1xbZwE>XJ@JHdnI)du!tduK-Um^F*?dpp^kNw4oUeps{H$T~ z!V{Z!yK}bsF+Auzik;I>Avv7$itT-&_9hU&#$N#&!};IxaTvBI9=-G9VD>QvKbJ+| zeo-KjSNdRziwAJzB)oF=;r)xnIA$Xc!(x8E@hv426L@Z^i?iUBFT-nK0B(hOBhktm z13zYn4wR1}t{dl(7-f)B`4Pl%uOwNhr)8FGa!)ke`_`OzYZbFvr5e$USJTOmyvZb1 z@FH1{uMr11O8UxHkhHZX=pQ7HY5Z<*$cZQ9_yEp=`(+;P z#?5X=NM^5tji)nroNzwzyBO%w97G$JAp1uyT6(Ud_@D>R`YcC_HekMaE>_8`gTfkT zj1BhU@8e6j{wo+V!?=g+gF9^g>_-S)2Gx?$P*nR%Mwyq9ntVpy-N+^D(~7vCwT84L za4uVH4>>JJajv=y&ksn$DNO^OP9T@6ToA&&YKozO0_akDP!eHzWAGG{qtbGiU~TU=3df ze(uc0m#8xMaQ1|We?2nhy@Jtz$MAZ72Ys^&ps=tFkCP?nFvw8&%g~kCzhPGP0*b4m zk(jy-8Xl&&VlWQp=8VICxij(OFVJf+598)@ciZTx_&H`OoN7m-a&$jTTq}?fi_ViR zU0aE7y9-%-$eY+ciX=+k(#gLOg=D4vD`Fa7N3yc&h(mA{aj7aLPs0kx5%D8pA(un9@g!*w$ z_PJQncRiDAZ?7b33BO41G&wAPq>Pl4>i9WJ12;4VW2u%5#2dep^l$Hp)VnHT*zkyC zY6OvdG0&1Ob|shN)yNeub<#6!2s!%BfQ*;dffFn_KOIOtH3<&?CL(Lu7BXr01>!I> zp4@D>M|>3?kzW&TlAC8KNgm`SwoI>z*x?F=r@!W z1P`Zes>7*`#1J}rgeuJr<{9PkE?{jLWGk*gf|pzMXX?YIa)W;O(cP45h~+iwCM?!hIW(v4eM<<>Sf14H-oF zXb!m=Es(!^bI4M+`(%>mUDCFKXNd2o5~q!6M9{cLT%AkES(P_LW_BmB`kqY;I(*6e zi~zFlxCIHBeOvVKg^b85X0dqq%tPYxmzm;Abs=J-_fMHLX)^Lzr>J*q2MJ!Hg~=Jpd|t_On}_?zz-%vauRWaT8>W(gHBZRNMNPzZP%*LZ zPA65hl;kJnkk<7Mq))#t$2LF#X49iYh06m(Kb<|v^m`-G{AMubSIvW>Ja-{V+(cli z7;%pt;z>H^A-9x4KDHH3OqS+`E76_B>U7miO}d^tf!}7RQIq}3)Tv6Imf1+r?SWrm z`nwbj_Y$C@w2$vcILCTT85x)FLs}jA9He;!d9ZCTv74buJ{}oCj+dMjt+xu!dHAk0 z$L`HA@urg(#hvBB;*32?V%^zlBG2jHMS&r%#Qks>x#ki_Uf10qak)2$!bILR>bpUn zTHPX>8`8+wdk=`d&l6(h$aDDJy<}xG?+rftK;n#x$hnw3WOz@<9IL-tLU{+zr`CoF zd5YPB(xnVRef}MxvL;#Zx*sBJT5wP}zk8Zs^;=nR^;8y)eI6>DNzxY-8g~jU<@<%X z8%_w_&Rd1oY^zW?dzcWfH%hchYMMCijg~Mn_lV&1FiMm#M4EgFeJ@J1J|qfMNFcol zAIVnhI`VZu35h;fObRS75K|ICPB699cGQ>zbEDtI{j2I+NC zBvz{VbKVXK5vwOj2y5*}2)COD3w5P3f>oS>VB~RD7{v+XZt}4L9$yj8JoXk={97n| zpoT(&M}qkA&3MtRM<&F*co(TVdyIGw@*r+Ue8>Q`U{b6fOA5U1kQ>bc$@Y6rth*bD zrtLQ}=T{+lFus_WHV5;Z_#n}LH@t-i?+#(|TqR~aQ;T)Z7{hdhajdCBhnYBNvl}6r zjDI3*h+aP?UeF^%tm+X8e@L*?-ZHF&$TQbCdFJ9E%}T}yg4W$=Vbi`8L1|mNxYK@( zcv#p~p|?erZQS$B)THs2sFKg%SF9avcGAm<#QsPok?#}8*}gEcIWvq%q@O2Se+Lp4 zf0g*eJR|#S6i_0ej-5lNBKFTZgot;*Y}tHlZUc@^-U^%dt5B(Y3Ul&Lqm$2F-@fGC zjTrj-!Mv$3YE>v{n z3up3*1<%pH1kv=t>`0yzbJTAa5`R<(qdIO2A>v%YEHOmL8Ma>VGcFR_n97+MS=);2 zeC~*rzLF%(j+2P>i?yVx$(_`Fxk#R_3?f_OE)Y$Z?ZmKLia1^N&vD%DDu`eH6G~Rj zWE0};*nGEb%st-dRhsmYRt87@Rs#*wUjZAm`XTN2%;3HC;5|ej+i&(k%2vhm;OSZwQ(-P9E(~(kSh&7Y+(7bRY@>l9$ zfRqWk#S3uOXF2C{twGX88aiT{Nh~&YB6Iy{$d2fZW@+EG*@hWYSwqttc4eC(`?N`w z`LxNg=Nr_RZpTRWWrH#^`A?0RzW632f4nE?n)wSBkJkuyDpiDWH)F&<(*EYe^`12y zxIc4_x7R@N+pZ3=j-i{7Jo1xZ-lflE?rmnK2QRRg_86u^lUd=*+iYxiGJBqSlgUg> zVEYWC*tBOMEM4*vyQO%Z9oFz<%1Ni0Op-er`oxu;qdSQg07N$PG zjWcICPe09Z%vigNgZ>3q)G^GBLWFL{1dkA@T}0 ziT|K*;=J}G*=}n_0_Qc0dZWLX&AbyY_8qZI$a&x^#OLM<58#&815AGwH05Ie?} zIy3ul7nZVUFVFPsWDXyd zu}xq@{G*xqjBut`>d&04y;xhtQ8v(JExTVdgQ*T2#GGVq3Yt<|g)dqWLfxAF?3sZS ziwlkvbY2z-4TX;c`|DSPk`jO6dAO%I=i7O+Pl*m@*(vU3PfL8v?2~Soxow8Xy*f&i zJb5eeImc&f63cP^z-hQ1jz(+he^|4Y-|a17Ad(MI;(h~d7yNKj-3gJ#lW}xOFR9%c zOUiG}AhUM_h-|WJvd2en6yH3MB%WrHBR*%IDIT0&FOFHGFR*r3p=s|8VNCFA!Bt;^ z*+*%z<M?aSPCYXIQ3u7~8hckukVa#E0D63c*#LCzD zvtxI>S#+ES+dlFXD{}T=Im1t}Ya0(SKbe!PBP5s|v$??rtxaZT{wb_lA&DKolfWwX zUS;w9f?3Xli|l~%MYf{Zk40dn}V{HC8^i zqE#`szDOyzEkH5%Wc|S0pw+UuiPO88wn8hj8dk?#c2=(|8fwLHZK15z zJ)HFvUSU5cUSRg}c3CanD|{CxKN= zva6b@4ya|0|C(4;S{uuZ?qZd%WpXDlh1@82)!fs{nz`#^hvb^(56xBBGBnpUP$SnR zLM1onfkN(i-Tt|9`Mqp)T?^B&sbS+2%UG{LE{j@zn_cUQU`91(S^V*>%;oP~h84q^ z?y45y>-SV4(PXPItns3_|6LQJBxIA$wHL{+^yMOh6CfbgYHG zy)we7eVRg!RE5~D_qkZ}(j=j-U0s+T_DMX)e8Qai$HBx_TN28OMo7)ELEM{@sN=J) zF+rTEzswg&)12V9RUavZuZh)^ZN$%ZgJ^M0uJ{w%A}mo$5|$QJ3KysR5~djRXZJKz z*=)Tr%vF0fJ9^oUO&;#a7VV2-W@qj*my4xr)Q)$oZS@!SZEYv3xZcG~&vY@x?Z4RY z@^9>uX$y0Vt!JIdRqS@qbM~UVh+VHLWQLRSS=l~;73eY;_d0`eUA8LbTAL2eUEHOZtNuqJciH|yxyp|Q<{nzsKX+`PWUlCb z2irXQJ)`lZtg$GEeI1^{CZu0wC7x$lT$COAreVmOcxJV8M~aXp>=3?gmlt-eS|L_j z;x1aKe46Ck>?YN3roy>)5jxBbv6B{)il3Tf+}Z!cIX^E5Cf8OAHs$?Mpt5uylpo>*Wf$))519=|Gv>xqdxt?$UNP?G#1!g~EgQH7WvU&vzy7rcN zT5-;*?l1Pcu7_1;O60D|l+4}xOfq--h<>>xwYGRhzl#v5!shAwvqtrJ79uHT zBj&zhKAO$!j`-l%pRW14Gb&N=sU zFRvG=TDgw+=D3gtGP=aE;WwKRaf6k38p0l5dXKLucFpCdOKp9kw%|9@4i=#DbhaYZ(pv?YZt6urRO8=PewKAvWe>z!wriWK(o z`77+s{(LrLw32l@`-D|~{DrMc5hd1p#}oM>J#yo?75#lqCvrhENyvfuWLl9Q2@wb+ zk=3h6RnsPNfNtj-_kIqol(2AZgvPjaW3SAUn(E5idie zpZPShTW~5#H=RPJSWO{b6^_KL56G64bBORZf3hfJBPkJ!B(l_Tb@F{GDLRx-S`Stb zh2t#Qy`3H(qaKnr*Jcv<>^XU-+)hf&Uz3OL-;z6(-DJh?_e9;en`nu4kRIn2va5%F zKgqXAXI(W>O{*j?zL%00_4&lqbHU1Kc6rn?IBw54U@PwN*+H$ltmuvB?3~WGY~%Egto_a}Z0C|+?2$fk z!faM1uk;MaypJ}d^9_&G4SAB>8;eO^`C6j-dkdL;YX_MTw~Opk2_wb(!pYc-NMi6X zifE@$9{#&yWGpF?XsYibmOp~Yz1oGOwRr})r)*1h=<5;PLV0pGUYhh@k|OJVN)e~U z(&Vk54Ed?7K$IV;5Ra+4#8=sl_?_|~XAi9=+P@-*Px^6k?&Nv0x<8$Kjm{^}UtJ~S zb0ztBvYM>AR!hot8c6Hud&IcCiOAGGCT>HI$tnH&88LsSjv7Q9iJiWUg3By!7V~GZ}kQv`~wD926#U zZI9R{zXY}>WFi|8qQYNObd;w!QqJ=>sOEhRzQxP@Rmcn36vMk&gS?g%EzTl>OP%yr zq}2@b-1y5crShkzweWwJiL>9j4A{X}Q`u{~=dp)sR~(mz5b*AywJdBzo;E0>jJ6L4jT5d_@!~`x8%Ee_SA+7NwKUupE+nrkKdG(Pd9+ZN9PV&r<6 z6nG{QgYGyYyEK|`HzP@tY&cz0?<8k0uOlf!3rOUc9kC?x=MjgcyyFqyyF6U$uW^#V=qF=!+*2ouX|a`hn?)#t1sCW(^l5Xrj?B& zFWG7Jo$O z>Z}A3xFen%{Ca}aNgO8z`NxUIg_Go(L;^AYe2)C`JwxX7MAP^pj5^GN$V^pFa&WRe zIqj%Lu51?}?*478?SUt(1YM^W$FQurXC)idkk1BorLwUCr`ZFq53_y=8EnvSGb<1y zNJMSbiI$iRIp6C>qL(irUxK!h*p@>?W_~OQtV$qV%E{!NLIz1T&mqBF0m)ijO6o)^ zNQhAxSy@v;vYrbSNQd&##i1Oa;lPy-F?@;^aS%q%O818y?D$b7wxViR-ViK_~aHGxkfeOTBIQBASvso@WN{pKAus zS>+;cuGTT0eyJ}nRqu_nUvFjgC1Y#;i|vK{$jyKF3*0r?oO#n(x0Sotkj(S!ZIvRn z{VAXAFnGwmZGFJnR0;?P2nq;Pi3GeLkh0)DlRzTo{ zrGP*%z5SoZH`8qc-TwFTpVylD-+vsW*W>dPU#z|Kn0$D=1*?`hP#DHbX#w=klNX_9^kPc;;Jvd}N*r zz2*rB9DGPO=07j^pU>h~%yiUsos9piCu9C5CwzL}1vgvIKwb{u;` z%yFpsQ2-fM1baG#u|a&8I?3LEN)ikAi}K+4r36UR*a`{X8Bo2cNqYicFflt1Fg@9S zc#Zyhop;|&=5IUZ%}Tww!sgYsv(s-=O52AyM91I|Ni9exCoNcFQPfEqYsY9@Cc`!Q zs&jjijJO(qOD^n&J@@vdBbTb~#07@(xT9O0IHT-ooLb^!u4cb2mzHDB9Xn~rJ$BRP zG#!+=2Wm3h#byz%)#^7fmi+$z4vW)0=?1&q|0*jo+E{`-1xbbBC#(%#heoLq=Xy$zvT+kmM4fD;Qv z@nV+(s!!Izuv9brBRd7Vp3cCT<(?=-nGzdkEkgn4Rk(QXN>o_49KQzyVq(e?R99V$ zSBCsBq0t9pkIYA<`Z*Y);f}FG4B|mYtgW-dyhrA^tJn~;)@Y;bW-VNKV-o6HPQoBj zdmIpPz<*cmDCgE5m)x9=`th!Kv%?<42lVmzXGO|P6-PVgQP>V2;AUPEJm5;9L-8sc zeOCf4{>iYv`XGGSvIw5nOn?!GF2*bAG?PW^QDy>foI~sG@I?hzv-gIJ*{a!s?7oE*=RiQ<(+*|?Q#zBx=BA4qe$-!-^f9>(0$yLQ}<<&NB5Cx&Y6 zW^izi#|>Yc#!Xgo;7(t);_@?$IWt#1PJ6x@*HSCbd3229=1mafTqAyx@QQ9Ca{D}yRjWy0ehKLxxk4)KFOcVD(WLM3YBHAKKzedT$#_xX&S+hTZ6FshI zNXp>uY3lgM*9cDp*`kS_Gmi3SVWZhX%xhhS^Cz#uB~ybDiq=rR@>&dRUW?tk*5G}? zU`)+lg}wnn7;tVm9w}dpJ)3>eq-7q~eVvVq+Fh_?gEQ8^WNdkDiRp@lID3Kt3hlJO zwKCRtPRasH-cG^|Mh;l_z!uF+>~Paj8~nT19Iy82;ByUW3^e=#LF*raX53X+5R(Mj zDkot4yIA;H9Rq%*M?vfPL73*T9M-!A7eL~Xj4H9KfS*}M*kE>d1&GlD0abrd9+`=4B&S+>J_fc#< z_jcHe>*INF0^P1$hbWJWI6IjOH?iW*1{!f`653pDuoAaopEReqT7>IZH%vN?d?1IO zJR=F~Idc0<5ph?%NHSI(A&P!94m@B>w$us|;|rJA(zfUP)X-nN#dZ~p*-bU-Gg=IX zHpfx7U=eIx`vBg^_JZOxVbmU#$371oJl$r7!gSU>;XTkM01w<*;qCr7 zX6%Dm2j^nzA2(dsG!0+>u*N{aiI}2ngtzt?U;v%5_cb!XjlvF?Jk1Ty**fDLt4SEQ zz!D{|8{&RLCA2#shz=PqA%6dLIO&!I#k5by$Zscj{n`Wa?mHmp*?Ne2;0qUS@!?*wr?60B$R?*y=$e-Fk7Ui5H&sA@csa>y$Pp1$! zWmt)uR%yst%(3H$t}}O`$c+oH^W^4D_U3B-`f^3P7ICwy{kWUke7ME}FK(uqJ2x!v z%-uLPiR-U7<+O)%xf7dIIKftFZheaocVGJ}alG}6nD4z#eElzy?wVa>;l^o1bm}0> zcU{VAA3e!iJB4HJ(Y~bC?a`1!Yw|Tqy5Zj7IBapy!H~Rkql& z&KN5WO~kPc>e&227k%y-VSt_mDwo@0-zF=x%`?Iu^EI*So-Bsw2~)0jKRo&N5UQx- z6rY`debYkVg+u_zDFndCt3}Y%?FODxE#YvOChT7-4qY)%7|-J;nGoACRr=?tz%VMss%ZIah=)w7B1LsvRnR`HE4VPu~``x3-m6a-TaTOBW zm%smrq0}ehxVnivQY@xk-Z)}$cnMMeqeZs%UuStaH~9rhE=(4=p!t z!px)Z;2@p#(-Kuh({&T^rHDNm?*SCl^2DBpez;S81unY07BiDW@KnrptU0p_`*(+7 znA<-5VYr`ucKh+e!+khf5{3e7D7F-D!6J`!cvpTU&W>4v9ejV}y#3MFXEECKFGYL7 z<#=NKa;#{hT+6IL^cGoxHd{O~Pu>|TCfVT!KNCE$(-_w)DB;0(+SGSrj9Yu@eC$4B zTv2O)$vT>NNKhKHBq_6a(ofJ6z7K&+8FeHifk*ry&|BgOfp=*C^*cN0J^d*Wxpb&=&kZJWaqDcjtKS{Dj?XUKA)&e4d-UVN4=v`7FACtio_TW}^Jj6v zZcdzmf-PtC)R^9<%T-=i<$nB><>DhnxnP|k(mec@geBc0Jq;yf-{5ItwQUVK7-UBF zK7GR){94TFWef5mTF)@cZ^=T#(}f`M=OoPDT@Duxzl7E8qj2@HG;^HB!vRo!v--8p#f!U8;G;E%kKWvES7qwHbhbuULXr)gJiJ z&>3Yb?9gb~1ZU}(puxBCc#sTbW*A^Iqk(O2l(9BK8Mk|hU`YOZ zD1CDmQj;?w;_iO<9x@wVt)33k4j?_Zx`C_5bWr?l0KE$&pf9(FnR&mQk+k(^Dpnol zF^_KYV^2o2d)(i#=WTR}RP8*P2RKY7=46sK&qgxl_k~24h;!dMl(<3zU2dbP2^Uvs z%hg8oQ>WWJoJuyz$ z7Y~dr!65x09Ccrd=1(?YX2~WDkllni!s}7|{X*33n2En{I$`%uJ9G~=#ndw8n= zl49`tnrX<}KpCoUbn$709&V3N!lv30Xl!l*fB$@l$vqBs!5bjeb1qnfF;KnD0mLRw zg>&Z|KvTsO=7lN1+J?`J`HfQM<-3*4gNFOOZ4#z@nKgdwZu18A=pIGl^4oeWaR*P{Hl#i@a9NUau)Gziy^jI%Slg-H5o<`qVOP> z-D+aRW*e?>PG2~Usi`kyI)_9cViE%dY2h$7Ar)rxs^CI2^>jr4gt1&Y=X^pAbJnWi zf>V0P*EYtxOU?1sDk}_5wZ#avsaO<_XxTCc&8mFx;GxCXEU^L&d8=@J+-h{BF-214 z3KTlC6itr$W6_*NSg<_+OFu3`J=bN}ci9h((!Ehl#uv}~`e9;&57xY&hqIq~;@}xq z3>%q_VqfhrD&7=7*HQk?N(H>rWPmM?CS#nQJ2pgk;?`M!VucoX?1MInrc2|G2|r=- zk%tiew-j!lya=5c$Dp=)Czzb2Ga-p?ATe_~{BX2|knx7pRVD|=BR???I$LhteIqVYLxb}cm*e#8MYxvA@5ImUB@r6p zlWegoE^v54-*Dt|}!6nESS%PN@{P6+ni*9}1c&dE?9-;A^wCEzd_hB)% zi!H_rPTuJMY5|U&@J90)^H4NoHmYoQ#eNsWUW4hl=dvC8?K8!MnTYigtkbO-4$-gFRAM_I0(qXcS5#y{a#&elHdfZ5?IoIK0!-ev#xDTNw z+)i6v?l&rN(|yKqUp$6La7jCHRON`u>P*skE{fFLnMvMU8D`%pc(P3&O!}Q`KC(fJWBxfnan3xz0a`|Q>Qc*ofn3t#%9v%Vi5dORP)BlCS}t_UtQH(o%xRC0u9MPLjW<%Jw`mXWRM=!O499gn?#&!BIO~k z$nLrQWdCYm&SsYa*JP&6MZGfMo;VtDLZkXzSHA`~wMw2FG8W-7l4;*Y<^yuVypSB8 z7)!3K6?@HX@y<3=TzS_7FDFUh;JER) zi`B-WDU;A4&lf{iEJUS*8Q8bS0<&CoQSE~=ehHI7cL5Q6IPn{VtZ#;=XQ+EAES|2L z)&iViVC<~{Bu6VkKMaziO$AEBI&w>q(2&CMH;B1VC*n+#o;A0nk@x;)HLDI z&8bk4JOi2>eW3fqc2H2J9th7g;JX$1!*ie2^i@G@OJSKyW6;)9Coi6U$V2+IiQ&Dfk1&bm)@Qxqlk~Vvy+I0_1 zbD4#|=euF{zZvM;jTlJ11EJSl@X??ht-?#4f}GKJ(Dj-yKJ@op6K4BvkF4 zjK#qY`1*wvo_9CKn9_-O=YR-K)uW!h9?A=zY>MH|be&+jfaVBYu~F0>lkAM~LmXue zy-`BZdP({|M zT_Z0G?~$?i_e9xMkW*BU;fB?fIqT!99Hfrtj2tAnBCp>hd+l=)5>iOKi^53YWF^8x z+R(c1K_=X~kIB3t3hs8=;Ca>r+7I#I%1j@Ke!USE`$og&%p}M^QVb*hln=4K2~=*j zLFmt3$hz?h_yVKQBQ^qV&jw-j`2n~%djJyr2VhyoD0C(Z;c)Ia)Nzx?W0N%Txuhv- z%1=S54~QEk%|beyiHbdQP*T_fT~pj~!LAt?ae&6y%ctS-m}%%D>5S?TfMl&J{@XVj z)rIC@kMtb$U+sc>{T*>es5Poe+Mx_XL5icPmpr;$(nX8o z3{KfG8+9#RF|%tbMjf_9mlFn*{ilvm(`3=x>n}82cnRCyl!MckIOy2A23D;`*upc1 z?it!};=2ZHchm;^*V=GKN)sNNj0d%OBJe%=CFAIw$OJgZFo~ys*C<;6pRC=!+ZgMkc~M2 z`h5xTAt?u%l5Wu%rbl3E*a6-fK7cB9ZrWS?2Bll05I1ia8g0MB*sssfI`cDhocRLc zW<$`R{}1k(iJ8&DIW{u6`7%=ff4rC z)4a%H9n5(A9SUvf@mEL^JJM7zFxeKVdI6^-0_FZqqw%{f-d$mgpSNgXtGyiREExlV zh0no@&IS&11e1Undzwu9gl3+Rv1f-zY!=t+3U zMC^`ahOKV%I{roS)s2GLy>srcjrYY!^|J}2oU^1a9ri`8VKP6W}9K23Zh@`>8Z zdg8vbi+oBMB7O1y$XSPBBDnb-c|LTTywFT0PFuZ6{Ad;XLZq745g-aZ1rEU0DZ!MS zzf9sxaY!oBg{U4!kkj>tkx6uR*We<+w_?~cSPS8^pTUo|cOd`qE9A@^fV`mJkoa*J zv^9T$V|pJP+0_HHC-gv8bRP_){{%Fdu-O4iigAKeh@VUSE)?KppBF8 zO_dcE%TTsctPu{)FuhLmYlY80^MCPIvaMA`?9u2Bc5A0F zv0JS{E`GBi+A_0lcXTBPy@VXPl1ctD`5Vuqp-{>9VRZm0io3o zAeGk!=G#8NH}{{=`Fj{P?*9!#^@Bioltr@mJyfQ@279|MxOt)%hUC7%+VDa6w^#u4 zVx@8V6K#|avceB5r(ymxz*J!xQ*UxZm3CK5DL_n{H64wv*-~eh3BElv0nOeSpu|>F zTq|vj9)@<3=A?g+uE7nFf~|2T~cH3&0y$&s7)erIwoORgE3LDrzrO9l*NF$p%I>4GV$&xEaOo@@W7vU#uC-+NYiM+sh z5-5~Nc(+cFv1yx$m!}o^HM55OB@|fmK&^^-c-|B)`Y(oOkuxBEh81i#Qh}rm8t@_0 z6ol3Q44m2shezVTds!~5)31fF)aQ`K>juXMUqN&75Pb6)g8+{|kdr?Q=1VCjYUl$T z8R!D3&K}sV@fmD3e}kT`0m#`cgt2ARdAQ9G71vPn`+OeCc)H*^c{kiQ*$pqBb-|g- zXpZUU6qIbT!2N0yu;aTH{-E_lan1n6Hke}Fb#p8XHbe1C6H)Ws1T6cjhh;i?$g9!9 zLnF#qW~YFbw$iAZDvrUS;<&D%8D{Ht!=0%=L0?e_`v>GupiUpNENt-0Gkcu6zy_CF zOvIo&YFIK`0&iM=gV7K7V1q^;aPy8qKJVLW1N3*iIB7wRVt6nZ+_TfybDoyDb^h*sDCJurg;_8|R0QvV6&TcI?2c zgX~m=%WT8z26krHAp2^M9BJ9DL)<H9<66xy z!N(C}C>u7Z#uYup-B9_x3%=1u%usYfOGA5%NjE{i^V;~NUk%?HUHqG6h_5MI z_I;fJo-)+My*v#ZKd6eq)vBo5t%Pmm(&!y1jvLksVRG^)EIB+1S+#eevHBJ4`_Ts# zGsocWI!Od8EzFU$KyM*CJo~{Ci$V-gpD3f$5izuT*$0B^4IoIaz`~VBA=hsn7>j*l zIv>qq94?)y2{?0)mmo@*YeUx<3&~7IwLX*SNa|$vpOygU{W8$EOAu~~w=t?OuQH!f zPcpR-%qT8*V$LnoWRhmL^DbR-<|(8}));M{#&>GF%>Oi_gD(*GmcK*gI{(~&KVMxu zt-36EC+}90HseJZzJxCdp6*`YEJMADi3cH{w-Bbz28fb$2bniBVA?|-TojxQGhJ7M z!Hr0W{c;`>Vv0d}PCdvxc?G7b{SdTf6qfxHL~Si$?4ig1ybVHFdf_ju`1}ichQ7j` zP2V9ZiuM`AjevK&06L3FW1F20UhuG{p33Q1A}|A&+Pk8oo(qnB1=Od`l-;3Iu+Q2W zd;O^ARY3#ab}Hkf05!~}b@ZdV^znPd1U%5HgHJB0;hTlyu_{g;%}>jrId$Qr)(N4w z%pYi#{Q-4TK0@9?%CeCnuq34!a<=w>)cs+QDHcZ~ado_CXo|n0t#L}AIp&?z!?>@C zn6OaiA>bDqs&DJW9-)K zXG9hqV_Ni+nINwkCPMx_BT**;rjZIzK{;S?j3J1++QSmrrSRrc6ik;t24|ifhQ@DO zpm_aKsPyxNI?f-6r4RI!EP~QUTR^TV8baS)pzM?)xGH%Y_6Wa#t!yvU%%tA#MIy*5 zi=mL5I4W%v!>R4Un3X7i=I*1gXvZH|7WfakO9b#c%}=bBR>bj#4RG}yTYOXRh^Je5 zc+2SvnZiuZ|(q z=^Or%GRhB2ppm0E&NLLlRC$^UG3bTW-(G{$t`^X%Z-G9VBhGBN5594)VZd+z3UkD; zE>0DX*%)K@B1_ckF~Ov-+L-l79z$1A_Rx%8SlQPIdFOJ#VdNxC^g0A7xl1Td%MwJ} zO(F6I!cZoHX2UdyFn-T$aEfA56;Ck|S8JKF<=+^O8G@7>_Lq@aJiv6hQ_lFoKBlg& zkJ)HX9X)wp8S$a-OoQtf6RIf%A#1eZXN^5v^YjLJvn}wf{}4P5hz8}?$KXZgQSh~n zf;s$?AnkDxxE5ObS3O0!39(>wIuh>7?S_}rHbc_@ohOtDft&>qp#S9r+@Q~M+*bre zcL}VNeMUcv_h2dT8>AA2@r1%SjO33)ZD|QSB_ob$mqjozR|t=F3uB472)3mNQU>(v?cbbkI7+*$P%s**+V!A2Ey+h~M7Cg#{oYl&Z)HIYh{Fg=~to>S?drSD@{ z^Rbl9?j^!+hgh&N+Yd2*!C;zx5Gq0#M)EUr_s0vyNcJZq z|5*ea6ePg>s3e42$UxCHd1zgt1cSTOp;1>4w)&YtS@ATeZCn7W6*j{@w`kbE@FMNK z$OGl#Yv4|2p6*0mr>@^phc0x60k>hVWD9-&AD~Zh7-|NEQ19e8e8JK0C_oY=lqE3f zgBY$g7sCn>aSS~zj!dyA&GSiM=W7-8Ofg1oqaD&N3>-Xck9E$}w>xT&$+C9Xo?(IJ zs)kt9rjB=Ws5FI;|IJpOC>iAyJ%fKMG+YKVVMcC&&`- z0O1XfAYpO?81+`u@8>E6{J9F@l)tm?83zaIT43NoFUa1edAt}U+-PWk?wKaIf;#6n z1km5W898(r5W!N)Ox<|06~c~DJ`tuuNly%Hbqt4~BRjw(^AL1upMe|NrLg*DEd&&l zg2Ts1XkI-Dp4rJl(o8AJNmqox<7%*GR2y2A4S{hqgS8iJpmF0=kc)GHR1aS`cYibd z7K#Qb+Y8XODIZRauYst-2hb|h3dbk4gID7#xLfc9sx~)*{ul?nx37cT#e6txk_*vY z+0Z0(1@^nAz-{3KXi`1_ZkciLaojmDK79$sL<+&}?@f55b{`HMZHEdzUBg<8LXd|D zp0kj^Nm|kvy-^B9R!ZUaT@rZq)i^w{R{{%UCGf^PF}$8I4xbEDW_+j-R^GNolR#TM zy4(h<^=)wV9cwIDYKi^-jB!paUHjCJNBl(VSaQ_mv_%@1G>^x%%hWJVT?JcGWwG`V z?Xl|^g4t=GK+)zkgnn!R0(o0r_DlANK{y z%iqDW56!SY>o&|tD1o+$Ecm>LakIILZJdWmFuI%OLN>*Y>5`E1v+P&qwYB~bXjhKoSp${H>zXV7Fm2~DTY=@M9}7> z1eQOSM_U~w+>s=QTiQgi({K)1Sa8!!*N_xQm66n{9~69`7PS3!dCMt~=~p=$6LT)CYL*Uy)L>>m!?-Zev@@;g}T z{1cKwgz%oDBu+Rbg>rqu)HgB!ItATOlXwr@0!qP6IR~m~|HhtW`CyTi4N5*&;QHfq z*!?aOm)!^2dCfGR@)p_-eFJU#G0H6$Mwvb0Sl>Plhnpm@Zi^&}uaZRJ z7%7ail0h{`SseQ=fx?SLaHv!m%{pXIxkn2lE=efQJY;E^Eer3xUQM~`n?rO@+_G^RR>V@=~QoD1oK%=3?6-(n7?9ls85hl*je zDif;Cr$E!bvrx|*1I50r@YiAojGWAc>yA|ry6_=LE_?^umybZ36#XsfsbTO1npdJR zL{qRj)|-#V>)}%9(jthhS-mi^^B$;d%ZAdL!!WYU7li*YFt~aqi0iC_2QKm8w5pJv zV{SpXb2+RTp8<34$HU93hoNiSJ_xee1zn?|&}XzQ*rz*l4VUY4^1-szwHPDGOk~%4hH$c?3`J8i@YR zf}2YD@USHwbR8#Ia9O3G3uFQBYS48$8wVm9Y}GpbY-<7D3gI-@tcDGc0&p49>c7aD2^r z=n3@!>uz5NE?x)9Qz;kVViMGu7C`p15-`*+fL)Y77UG%&*~w?2sOuE8Ts#T#S583m zomd$5i-F*W$6$)g2}ln~gc}ig5bjDxtZ^~*q8>S- ziU#sR_?PBSc{$CHoKp=M)I<2>dp0C_7Qnw4%2wWV1>7rh;laAAASl9z;*k5`a_1!k zP{+@a=Y#Ny_PT9fE`~?9OW-nA67#1^{f}E_;d0ue{X-1(X#613DuKyUWHIrdB%VDa zglnh&g~toU@u#j5UR$M&*EJ`g=r=UYVC)t#sgguvJF*ivJ zw+anWM#D?!H@pF-oi9M9)*8$o<@2W;v%1oPgVgiYD0u=`LheEVJm6Wod+ z%C-<9qkTt%y|T`*2s z0MF^D;3Y8|blU)UXD@?mi8-EpB7$oi-$CoWN041}2aFv{pz&`ONCXu?(Z*c3W}XkD zw~OG~(i@;fIa7h}o`A-?H*lPC9Ic}NK(M|D_NR`+f9g`G`c(=Y3#70zND@cQ$6-K@ zD9Y>*!7T#fxF$pjH?NocpJUkci=%LQq8J{g=YdcU4V0nvx#^Q9pt}166e!lgYrU$d zQ7DVOgTgoCs`bV+u=ZnKBf-%$nVl}mHL=i#bC3^Xm+4+m{G zz}=;bAfOka*4q^BNXWn*C+hIzcQFHjSD?429F7;%L-XlYDAwx(<*S1DaI*}~*shGf zuB)L1&1syyrG)v<<eKLb`gx5qJ@^P?NL{G7Ij|D#^O9n99kfVmwYIn@nQqq4XTBk zGW0lWNxiwh3&7z}0X$n&3@XXxAa<`B#Au%Anc`EZ;JpPx*B|K%|4^Yd=7!K*S_%WF@n+)3*h&vKwSA4vq-v*S+V{YW8rg>$qY#cW$r4RD7^!6 zAD)8Y-EMIBJ_i3l3d3n!8GeT5vDT_$nJ8V`2`S*`jkH$y?jNLm=z^g;9C*yT0w22L zz#;7j%|+Am%dev_AvhX3L}TFR^LTJkNrwE@IWRVG6`YsegfRUYV8d$wN9kv(R0-}8 zC9s*gTK*JezeINv~m72lG<0CwB5JAT}4YX6XLf6-hsD0Q9W84%l z>-$?c{ronV##F;r%}U6dRSDguC2)K~A!QmC!i6U#z*D{sKmSyN^{(6CRr3IPC%=Fx zY!4U-|Ac6XfAC6O1jp{sHIgKaA-u-nym#X0J3|cL?GVO&qXKyOl^|*ri(x4Bm&oe~ zV2sBIWyp$Rud6(M8n22v1zNbRTnBaBwee4-Dkhf8VoI7Y*64o&&GL3|jiL3eop&K) z+ih4tC?`0QGC)19z=X+X;aulo2(w=gD+TAkzzQ2EidLs_xDZq&JYx(^^BBX4s~C^! zX5J5j&%BQBY4GuEDU1l+0sU#saNYV1oHH7s3^Ym9)gOdcLD!D-c%n0K6LybY1N|pu5OTZ(HaskZrhpRI|0o}{!wO;9fkLQT zUJgoG*MZf#1@X67%6q&EqiN3|_|#j<>-_>#Muwrk@gM9wL)Rk}LMTLY4tw&2u+=~i zzr7fPWi(bHD2i)`L@_$;FIesxfu>}igGpQ=rZKa7kQ*9Jc)x^kClr=U}5~ZGu zg6EAccrwxiAvOd)O}P$>JxXEW>|E&6O93O}IB5I04;s9e!5P~B7~MuWJt2~y!geux zyNj5n`@5OnM3-s!xtFI_aELE*-IM>jBM(Fpu0w=+JwyyQgRw#<2&@~1<)0;R?aA?I zB&dc}ht;rRKn44kj>qEp(s(#V0ORJqg}s#B?WTDdc1NFp3ynu$R(v!p42c82`ze@q z@D%LeQRm*1M40sR63lMPg^D1$9&)(>O)sh-E3pciO>V%}^fHL9Eu>zVT%eP*U{qZU zQzzYkqYbribypKK9DM;zRUhH&TAItgDn|RVhM@C(8|*bI0*5JSFwyoh*m!5bn|$g{ z(#(can{%PkJR3rJxpbY81FZ`RLDa7Zdgm6wifg6Nb=n30PN$j?`W{7bimx2D1gqdX z8x4Htt%;%r>Zml4^4C|2<96wvuyp@(7<|NsckZPiY>^F_lx^kk>kRA@JqFzywnNc# zZ&=9M!>a{qFp%+^Ni4X{I7G%XflXG-bLSXd&|eXL*byCelr3hJx8|@9-c&%u-dYG+ z_z3z}y@1-QuOZU;H*Btz#O>ddQ1_}j9$2A)PJA`gDWW}>tTZ}F3SfuZYw$Iwfl%2? zFccaKU(64K>hfrC+7k~(sb>L3&w-$LBHUV-1cUc4f!owu?7fP@)#P;+aPIv2V`{If$%)K zMvsq$S$Xl$taJ_%U6Nqe;3ZJ^Pls6#FN5DU>iCnQG0OEz^!;6iUs>reXn7f?bY;?< zZ2`EwEr($%7Nnjw(jMNYFi)KNWM;g9hGy#i@%aKrS_goqIz&C?L-0fSFXWnvU|qKe z2C-xCIClgt>=MPp1+sX*T?xIDXie8z0~>}_(CLUAK3z&ZQEU1@{n35+=uifsb{TN= z#CfLxljQS?MXS@S9gn*hMi``Ri$R zb7c?v?1~<7s?s7qcMzy?dO$s!)H@*k27YPufMv%Y7z>xiA~9MMQP;xoPgR&f#hBePb3xx)*|TWI4D!sfLvecYvL7ADSLKh5^Y|V1B%T zfn&YU9QqCTD!*W7@fbX(wT{E3B6K$6FZ{Nojx(A4efpRYubZI=Stc1Z< z(zt-uVx0FU0iFgJU%c)mo(gz5e`x&Op z5X6$#vN$1D9bbRYL5EIVOv%;88Cq%>wnP?Z`U#+o_iI>0^MuAusqk1b8qP%?gx6DnyVu}4 zb&Uv=7r+RY4`Otk>bN8i_R-vG>T9~6&~xa(nj8>oErI%yOqd~f0yM5i)B446aQk`& zekmkEdh}Vynso+TrYFGhM#|Coe3-s&BsdI5!OWK@VD8MbVE!%{)ciBx$@k?X zLllctWU$9y5sTWDk)h6s9r1EF?xHw~&!v3{KiXmP!&_jSnhJpdF<{=e9b{hlLDe`O z@Y^PU1nryKlyi$Q)?Ui2v$)LL%HPO8eqbv5;%pkbd-n(SkDd;hZ#9p|FNh!)L=(vg zk#po$@Dn&5&<1y+d!T72?Opr&6WSw1{trjz9naPKzj0eaMn*`Pm8_OXIIl~#w4^0% zNup^~ABD0bi3&+%w2YKQcppMV8q!cvl(e*$Lcja_`=fsvkLq#Gx$oDwuIGi}2?~g1 zz5k6{+Q|5+1Mes;9Af8Rrh+`=+l0}zvzyM^2`HO=Pp=fC=!{Gl`Mi%M-MML`EWu_| z?7mp2d4`e=3rKm8M6hHq0$*5nYiGA0|E|b;LsWr6Qwt}V~DWxBt=jrT` z0=m(Die{#spefl$C~(?AvS%FT27@$u?R|=V-(&aT;$t+dk^Ow#VR}CI2vt62nFdo2 z)3@dWWZ@r6y?6IgZ_!3l>)1ebKEb4MHH!A`I7D&-X{7%>i{-@R(us`IG(LxY!v>`k zy!i^<2@ud&*BkV%t%b_}Jfg1RZrURCp5)*Eq?C6;Sdk@!yQKrPYoid%d?j%!TNX-Q z3JAHch?0f!`1VH%L+QdW-~Ea9dEF-!#|!kUKbZ_Hce1Qe52}wdXZ6AuvfL*^R!xnZ z-qEexD*14m`{Fli7Mko7RK;BryvmT^z0Vu-FP<&s11+NXQ*v4SzmZq?g9?}V{PsuG z)ANENcfTW{%CE%IRLHMd65$h-;B;gRMuzKQ!XJH%_)iz71wie>NahaTFo3cNvb^7M#&+3i8SJ-h(YVkPuj?4cJbTTooba&cMe5U zw$BQB`PH0~7OBx{ZDC5?+|2DeAICX*_S(4IORUidjSz(Yd?+ZNF_O2=oyqGsc=IZU zSVr{vQ~WTSYQ7`r5kDCCfEV8SloA|!SPs)i5?ab;M&3f$5x_KJt5x8ATnjC&Z2wd~ z0ayITA!e8sx-Kgtuu~GMGM{Lq#!U*{oqOzuX~g^l~j`9&hNSZ2^vpDemMA(QepWU&2T8Wr26k`~K;)Hgpwo)4ob;#wd* zu$@gmnIUROHKXi_^RkFIJsGRH+1@upz zy)%27Y4@7Dv}NZ*vOm&I1=gR4F8-pT;y>g#dx&Bz#G&6MgP-#9xKk#F7!MgJ$Fu&c zZ-D-ayd$G0Ar#b% zGd;C1nzd!n^F|eBN;>$F&Gcl7%uix%fI(+n+~2Q;%@#~&G3+-n{^`ZN z`)FHpBAxBbBAvbxTH#ki%ZBR7&EXp9#a|_*+=~<+&h}kX>&YVL4izlALF+6V$nYxL z*F_2Fr1n)ha`qDCB^1*1tNEm%kww9(M_5kg0ou4FhRSLasPTLXStX~Fr228jhfk-Q z&PS>3coH4knLu-Av)*8CByAb;CVveJI<#dh74Miv`Twn;O5JcW4o#rf>yFTine6T? z%A?XN=VX`af36}fPE&-d6g^G|E!jnf3TGKKj0XD9I+d{^^#hGO_awQRm(Y9&9^^MF^p z_JL1K5w5*>TB!Dn>nqZH@QHi}evyCX0JT39!Sa7H*iknM{ewC<)n@?lKSucT$PigY zZ%c=O>b+d^B}?C(+28j0x&pLK=4k6n3$RWreg*lIcy- zD!)cS%d1H2L=E+|RFNh7&a_P`NJR1yC7-@R4~DK%m`o*=Z!4paTgCKJ{|vSGWz$T} zG`fBznM~P_ae*oHbQkNJQ_?B8yIEeB`@u5$`LUdqOsXPboq965e~V^D-6Mg* z1DcWam^#&8P_h4aBEu=xNB#exYv4jZBj0Ms%~2{PY|ZwBAD?&g`Wv$ ze%p&(d{E>OUeCUmcQL=gZ=TS_%iH(!@3M#0Dm5~`Ui~{dv-v0Ws1H%w9w7vTh+^wD zSrkoGg9_WFe7!#r-eD$4o@4@JwTZ~{)j?2)3JM?zsr*-Dr*@fTJEhQehj(`DM13_iWHPFS{Nf+b5{rlA^sSBdhcweAyGmN0Qb;91r|2r1LAVb)Pse>KDEYpC zCa&bE^hPzchgML~hl>=otC0AW`E(*Nixw@&AX%Ao8tIoo`~POrhGi#6c}xcFkvvQT zzv3z5TqKS7yn}>iIFX~17Tr5LhEm%dDDc&G@_H3R`&rMOXT|1+H%`&;KDLKjQBKb! zs>ztmr}WM1spVV~=`L-h2JuIPs?7{X{m>gVg;*6z*)cnN%YSK^9}4 zv;Ij$X*g_IEtv4_3)^jV&;z{&YE)wTuQdlr{_<*iZ$E+UB|dSPt|z!~5#ls!+HJn~ zdRd#j|5CG4=cqtu!f%1csfqleWab;b6vaPI%i^8xm-DK}?(!!z-tm@w!nNP_jHqqA z$QXd}UrF40kXpP&aE;}w_pcGd^)Irxkf{buQ9WoMHH79}Q|wqa3C=r=@oK_&Ol@T| z(TB3EKL0`wD|xz9ew-Stp(2NeOcPO2 zrU#u+Bh*ivjN|`JLH_kg(CagV3#-!|x|Hy@;vckQDGC!8wY*0y>zU>% z?-6x8enNhpFDNwT741LznpViYCAT%7Xi8x}%exnW+n^*?M9RQ=IeSh|N8r)d;W*~b z&OsZtS5X@ZwL<>cP(44f;5q;P=pWv&PpVe`vPx|V^Hxeo z3*qT33AA|*$GS1n!06$qUaEk#moy;PqL1g5##nuG3T*C7L&C&q$S<9QBx3_qIIl}OEOGvA< zls?y&(Z`5tnt$^;{g>TF5?&o-`sg{m9Nk65axcmFRWEg}`#>ho`sljeXWDc72YsJ6 zL=T0<;QMG8*1VFzE;%{mWy#{C5jN<0hQlud}%f+Ke^65hAN*_(O z8FxA9jsWhldZmrHRz>wX`@xzirJDt#HdP8H)QI!t71Md&hLwD=d?Y{0>^MLC zR5Sne!E65fW1-sQIN4g+>FTv(a@A@>OW8b_>0wN7%3{nIdBoewp^AB(5}Gy9vur%- zmYG79JtNy)=ICJDoyz@Y__^H_D?aHmkCr^73j4@hvW{Hhi>Q+3Cmrd^o|-l z-jdX!_w@G5N19>sjm~*|XMT4!lWk%0|))*^}{~kvX>Qx5W43Gx2(-CEApRiwf8EGs&05`XtL#T7iF;`PsHgg?t_Om3&_*2mOw>Jfd;ddPC? zA5yMx8^x?>A&Hx}N&DGd+I8*$#r8ZQ<6T7a)-Qin(6boo0KQh z%rvfSr^7!aOYbM7U)oL{ns-^2V--be=aG$mCVhN$mNu%@)AxWU^!eEV7cJT7pW-qgZfr}Cr{t6v|tjOy(avkG5i0}rO|)M+hK^lvb})#S`l2f zV|$FwVaW26!7b(i6S0wDaxX~~j{HX*JKmDfm)mry;sOnwI84oN{AlmY$@G=`#)&E= zaF@hh*_>T?rutdsFhSw(2tiogb-}#J!+FkcGQT3ri(gf=pTC%x#?SOD<|P{&_J&7sTPEA*;J!~e5Qd?={K2X5oQz}SdbMd2Un$Y&s#W&_@T)B8E zw9e1K;5BRXmvdN>!Jz|I*k3ahV-$3;*Jn6RD88Z!maDjHSQAxGyh~!)_oy*`|kLcQT){#9Jyc>ZBQq?$Ki1yDT5$4o%qdfSz4qK4$&} zd0pzJVmXnIE)tp8OKuC_(#Z`kN&mrp`h2FEq)!); zh+Qe2UCqAJ)o-XdcaVtf?6O)!k**HXjH!wjBVBAeETbZ;-GNtTDjr1QpF0jk3AG>@5zWmZJjKt z*>|+!pc=eGMq}WP7XEmR!@NVrXm>TkZ7C~E9VEPcI}1+Tc1U<-Z)fNFLyCRVMgt2Tk@vZ$w6c$7fPQ0|_Qaob)qH@yPyb0a zXWx*;-p3^0)=D~e?on^-L-GuIMs@1lByZJ2x!!DUWA=)!cE6@ZT#bf6Op6zJE1OPsBtCU-&HrK+H$ zuV!$pH=D1P3A9Iu@~v6M{LAbm{KMnn{D)mfc*;A+uZyVV)Bk(OOI7voXX-xlhO__i z85bmL=ZnkLo~#^En`0-5vHfyL^-@LZHReSdr-hC{J$QN=;6Sx0d=$-5t!l&O6SFY% z#SwC2Tu_(i3>#NFM6WVONuf5*uVWdG)^CK#agh_SH~t_MH~~8fvXeK>*a{y#Gp8KR1e3i+7XaGCl7BidGyPS zK=E%@&uwN}ki(Mr!!%-RYKFs@?J~NVjy3D$aO^G@friQ_IxW*if1j06N(Iy6eD$Z6 zJ~OJhB23zwk8z{dx7eg5EUSsj8YdX-5-qsa+8`*hy?N;;QT&D5>3rF@ zVqWA%1K)D}37_oohIc*piO=X7;2Y&7YKw$rYrU_@)XrnNjTse+sMxNK(?`cb_pdIl z&Yl35BqL~wO@Y*93w*EQU^K@8jS;Tw|1lSR>hn-@d^XDd0;v~FQ2CXyb*72n^W8qW zl+!~FkVILYKz&(MdMLm?Az5HS@*c_-~N<>wZvx)d%Vu)kD)$ zx@dc97x{?4q&>Q?$+4}MX+PeON!J_ttn{9$&-RgX=STV>-$!9DU(@3~9c-q0m%4V< zQ%Nz)nBISz^yadh-bKSPs5Kh%78$_*{ba_mF~hKdX%Ou<#hHZhh>>S!`Z+~}m<&g{ zuLSaXrEtn!0d{v)aN>t5jNU6FI6x8BMkAnKG8{`3rQp{n1q(JSYr8)J8|>t8e9LgO zMX@@W>m@_~o3!ZXX;O@3I^ecNWYwxmgG+n3RWJVIj`gm#QGX(+5#8Y?xZib5aO2i} zfp363@2+dZ``7yM0yZc5;B=hVTwB76m^ShnKF|2n{&#$s|7X5s>i{39D_(1sEnREv zFI_8|D+TN8BVnPb2?M54t}YypCrgd6TWJ#Fqo-lm0!z3L5h8^d1HpL?_Dpq$&leAv z9$yHV%-NV^Vuk#EJqSt5K}AgjjuSu9H>NLfnD&HZnc3uAWe52@c|#)NOmFBciT3Z( zXn7`%7eT_Xe>OxPm;52M-0vi0@tNAk^pXAE_td@Q1Bsn`Pp|&Gr56@&XPo8#cR}%T}Nm3-ld~%AL-r0;V4!a3og?X3GJ46wgWg>YX_SN zcF4}Q!LfvCh>;kNp;@DFpLzAx+YE!K;RqP2s3Pg)XcSLo&v1=8TD+9;Kp=;$_0pK9 zIvlRz((pbg3u!}nbal%?^@J3B<^Gb)$`0xsRZYisq*LGHZIqN_Nduv>l(4FS+pyJ* zE8385ZSqe-urPMJz;VnOf#u{Ef(s=|e8+!ae9cY#VvQJn#rSl-$hwdh2w&qzYCh#V zpMT(|ZTia3QyS!lzKhlVy*RwqKZgC7BZcaakvPTrrDucMkawMkUxt&w-JFUtcXQN< zTcPALaQuT4N`A0!$$SySFEg&yRZm>VScC>&XACw>!@}`nz`c-xdYKTk1fOZ)@OR{` z@Qys#H*npypT<0An)jECwfI67TkcE3FjE}UpNk`ZsW|=}7DY0ve-}I&ppRz329mUIdIlew_|- z&CGDgc@`G@a)nyrJXlD(qg`PE78SeVE8~4R&tk03CmOiQ?wOhEr13s)B%1q2W0r+B z7=;I>FGiywOa(Lk$zkxWG>i@pXZ}rT#1xG{p|%3Fl;tsxX_7QgF#qMjR{DLhm@2-< zk&KZ$nJpbnYZ6{_X1OQ0p9|Evm2byXuV1Stkm@=hXvn!L=q&y$=qMe5&Kmw@vl$0%2^ykSVBf^mc=&iVB$uv$XtD=hs5oHo{3QGv zJsRfH@{n9EkD5j`Oqrnrp54jSb~<>VtqJ>lWlXz00!e1l=ou>wnj?#As`41DRKV?y z5zJd7h8Nf0(oKT~dM$dA@_z52OOaD(xA$-EgWw!@_?ag+(({5%%MRz7iZDCD3h`{g zm7oWLcfMl$zFIBDUf}qYkDj~@tM^1?BYCr!!@PA%0Y8*l%Wqill)vrP%d3_5@M~my zc~8yve1Q0SzRG(THf>PE7GZUCvUkej5WAOb%@8u)5>3k1$ZX?K@ZT&L_c-BJ-(2ub zi?MF*N=Uv~2V>*)a7bT;!>in3T)_O^%|>u|tAVdZ@_6}73RV+`L0oP)oJ`oe?m7~| zhN_U19Er#2j8)OAfB_k%v5io{GbdHZ7O0?7N(G@zciR%Dh-~)WW=tP}%<+ub6UDTU zY__Fi`iIOFzmpczYg zf`oo6Nukaf$Knvdp@jlL#PeSQxaU*5hQg1Gx?e1-B1-ZgCzf9czLp7-$K zSC3uJmoIVQXDQk7720lm&$A7@c-cYz$7IHp_bTQ;CzkM0&&&8x;+Odo%oDoSMFA@m z)v@A<4pQ|@@ZSt`n0i`6NDkOzJqw?-?4hLVgl@*}d4Fyp#@TuyPvFgZ^-ZXL6oBgA z8}T806{Z=@N5VZ0r8<-FqD~LlIb+ZqsEM4yvABF!3%MJ|Ahcc+DSl(IM_C(sXZ3K= z-T*=ijqq}s3FIS8@%E<)#&0%;f2lFH?KOe#T@&aXGC|2X6UO*7!LP=tcpzns=o>C5 zd%X;gcKV|KPB2_vhod$w3S0DJF*p>9^$D@~`Y#$y8vCG@yA5h;3laaFdH!DMz=-+J zum918M1TSIJTb9`|UdtaF! zC2PfdO33TR#oAmj=WeWVn4mG~K+>H-;n8@FDPT^gX zQu)a1>Ado=4F1}9QEbwd!*DBgkd7V#cbTGT&>X=sHrPH0OmMM7@O^ufzIVi*AFdeK zv;a3M7d3k2Q@zV?lKFowWd79 zJ23W^w;bNHJ;q0Cb`CBVN4wu=I=S)|P6DDju&n;_WIp zZ+bvrZ;>Tvn{`QWH~h2USE)WPzG^=IeN`yGs5OJnkuT;y&OXcciKOzGbpd=|p#gu} z;ETW_u2s;mek3n6(~mDP58}%OL402QUS4Wi82`Qd5B=9Q4BcKzSa?Jm2do$)+ra|& zR@tB`)E3d-?67H*1Kxdbgx*IN=**vofv=0P=g%tGI&6gFxNWG54#N10foL(=4BPIt z`0s%ScGWndVuTg8ewqrytCLXh(-eB1lkvUF6q~o0BAPKF-HfK8x5^xeWCiaN971Jn z(YJdR%7WNE^2-^08|K32`XYp8da=549R@S~vBq&bQhIh_(d7NGJP`|{U5OBpNWs|b zbZo25LiznH%=b)3Tk;`fZ`_Y5OMS5AKU>I&>LXyeI@WE_LD)?rWNKL=Ox*_A+B0Ai zW{N{TlhAp`6jNtRz+|yen9BOuw>q*YpEv?bV`X7NEW3Kj0GYQwCjrx5zwk{Vg}aL= zHE=k6;`VWi&6BLGI_(6Zr?Uh*63YaC7hVzU5Y!3ms(uJQf6(SXTyy5T2epd#r&8YmDIWV>-$|TBGr>Egt*X!&1@_8(f^wv&#jy*d6sS&;v%wD^OGI z10{*ANEjW0q3OFJHGe1eGY0YaUp}z*^hDzm#^N@#!~6r5@angO^Z{#-tTi5V7cKRWZ^FfmrZx ze%@}DLu@C?U+Q@;$k^24iZ{dJi0*24h}!0CuLV#gBDM zkQ(fYM0xg%C0wvu*cB@GUGc5c74uiR!PIaLF80sEGJ!kVjxI*^Sx<=58l(jH;^vmE z$QTt0jXnEu$}J9)S0!Qg!xR`jO-H=>35fnWg~hU`G2`%A+;lwy&7Cx5!K%e+g1zS#4Egj-pHDH{s}hgQ}L+@?PkG}reC;$kg$j}FGWrz-wvQw9G%q=1kA zzJuQwK9e_p{Y7x@XPID)d7(gLQHH?HBtani!&~6|WrINRcq_fX$NbY8|LE405jbzE zjY?}1C~dQVnG|7A$sV$*PHZ;hjFvoC6fT*AVM7bp{Ad~8YqM+uAAfXg3Pw`FUdEY= zKtsSjJUSAA>~UdmO$|X&TL8SSdgH;n6=;fH3bQ#&vA)m~X`5DH`;(RE5M713j@9_Q zd>!`h-hjd>{xBH51Fn~Lqgyiq?uyaalzsq7iw`5Y@fgfbWkBpm4tg)-V|2(lSY@2U z*r^3Lqg?8m2!}!JgmDTj>0U0-RsdzfHH<-nWdnt?A4Q z6iDV3Q)#jA8}8_&wcPS&vepNVPZB7a#0tt269upRVg;AycnPX^E)mo#ZV+s6oG*y3 zGZ*mw4+PbZ*6>c}5AiOmkMLaQJa$hJKM-!rC%t+l&{}v%@NToX;Q8DGHJ9$i*OaBH z)a>1;Q*(@CT#}bePses-vQZLfN*e_?<~4P!n1a1Q?A};B3-wiwShUjxV~);-iTYfa z$+$z(XelOVu15V$KkN<^Ni-IlMq#4#K6ozO3+KoXsJU!| z>%jm_S-lyHb+_Z!lR$WD1Y@_uPP~fQg*B5y(W(><9moApy%~+yyW=tG5~~kZC!_B{ z8VY}=qh}}!Vf%AoT3LY2eg!bNn1vq^N$_$H#Y+Vbd||A+8LVD02p$b_uhE#lwHmYl?*(6ObRRjo``58*@t%3x2R22is?4 zIWwPY&nGfF`j8TB1r(mg^tvMsk(w&whr}@N{=?VY@2m)pePcF3%foBDJrxATJvM@_ zkxBv^rC&7>yKmJTzf)DyzkYO0;_Hc5z1Hp!oH#a}Uwnq;KwV7aKTgl)4>U>e0prdH z`1k(_+_q{7qKu<#bnhv0Wm9FiFVjYGW4|hK`oAtR|K)Ys`kJv6_VqK}nk?czj7E;6 zA$AFyhPmMV)7c1NT-tA@?pU6(1TXB_{V{qY45kD^d2AR|^dn*F7K>Ty z4xrN^5uWXdup6F;4vzy2ARLRfWzkr0YCn3-_TkKu{oocxA-OOb*AK+ND?JwHkH%ri zk^^X4n23|f$%vLc46pmi2tRfN4_2PQ+uRdaem(~-JF-!~G@Z@zsge#%3AHanhLXtA*s5*63(i2xbI@Vv;xZ6t9O}=NgzjT88q!Q|dSr9+ zHJAK6iQB8C#C81fwVCm{)GGJHzv}7dmR3EipJ3fIGuHaWlP%VHPs41?V(#12Z+}_i z|5k$cUjbfusww|t;~Rl|j+Mah&QY5&&D(7ry?tVBueOcbsD7HOv$()za;03ecM0dK zkVldk<+QJ+nXY>EkVl9J>opV+cuO0BOlS1=g(bA_+G4?Y#%b(v#i(I(aroc@#IIkB zf9@;b$}$W(1>4|rc@G@8NSw=!!)fP(@Dw|YQ>CfcW_}bI$B!a;;0ScXQ`vcX7!$+} zq5tDS7QYy}M9KqFx?Dy*&!;!ri@ZWm`Ci{{gZ+(=_rn4aPI157c z$KaqF!?bkkF*tE1+-i-{VyK3VW5uD#^in_MRGHpzD)hsbW6$OgFh43xEcRme{$04a zWIK|heIc}HIl}bjpqZWP2`i?ePi{QYj%&a-RSg|Ms>o*^`1&ioMk%e$CahhQ3ty(q&3qeFy>xZ4;Nykwg3Rz!0)C@~<&6`=xYU8mT-c`Lob#kP z-0|{q&g0r&PUH13s-7^6QpZcupP^Jb6L^|tuBf7nkT%MQ`%GGl+x`UJGIkE3O620DJHqf$8o_qOA2XHOy0EE_v3 zvfz2`Fy3^>)Z_)TK5qj=rItW>wIjwbp5*TqL$u%4!VeQw8273m z^SLB$TmB)%Gq=gyi+Qi3E>mB18A6H3W<0~Suk!sZK_kqjaR?L}w z3gh-x&*dbX+_`^?mT>J?yt%i3{kdB|mT)5zr*pU0PU04vn#v_8+u1Y)I|v5Sq68t& z6lz*VU*LS4?s54(!)eVP#(bJ5;2ON%a&2{^DEiJAy06FbbDUM_fgp;4t<%U1%x@Rb zKprX2sVHrL#(Rx`xzZTee>6mix;e&qbC}23V+CWOD~q`yLwY_oGLF3^%kEGP^nqF6 zb~v95!%cRMZA(qWd!1Bl7@2|nQ%>TRb1p*X6kzq!(o2kl4u&Ip{d_QA<2#+r^Wg!lg#s2g-4 zR_264zsX1ncfxmjPuwYYht!BL{54C0rh6*vrzYcJRXomh?nk0!Fy0TWhDoIxT03SU zP}2n09QCjaqcO<12YFf&_`U2o)lIrZuFTiy;h#qe$BXEnUk<5mNv15%Xj&JujmG*c zqZ>MAG$UV`>_oqFFMH}Z@519;&zo4zu`rhFls&}h=cjX)momAls}6Cls++k9?k=3x z0YlE%HOQtq`a#WAqX#vq0p^@ylNvp`piIMVFdf;+i4+$;oXlo?C?0+_iO;EKZ>Oi@e?#_Cu8dL+@Uh<#w zJ641ZhQ;{kQ4C+dLPV}P1G`nZ$al!bgP45AwLQ;RT4#}#b`sgn@o?q#!RPB%q*}!w zOeq>SHn`#E?I|d{KMvDfrl9l6dIX$c_4|~~@D2|~=aU_n=C&6PMkK;nJ`LI4saW(b z5mqsg_^TI)tf@=!K#Xwfwh@{;wPCY?dDQ2Pgjk6%ip)Mxbm(L9Q*B{-<_4mVWi%-~ zkB(@jP&)I5x90yxbMn?xZpds>Q8c0xMVySvd9GNkgloG~ z&PCm*;{4wfb5m7fxYYGaxT?jjF_2`i2Zy3 zr>at*XmuQu|K#9l#c8DK7Gi%`2}Y@1#N***7;*a&-Yvg?=|$(zJtZIa{^i4}zX+1i z=WzSTNvQb8K}_BsD&0{CYdwXC9DnRlT#25*`3Q+zf^HA?jQY0X@6`X$E(k&Er$ETM z@5EdGJ!m_%4{3)EBjLs|7`UXMLpcs*Dj_f)RnC+9xnF?Yx#llxKO&y~Lz#)V5Rw;3-SYtyQc zz=_O|rDblbD7IrM^`2W#H-GM=OrzyAe4IE9&6lMn3u8)U%<-dIAgApdi40#%4iL^N>WmTeOB_il!c{c0G<1fy`+ zK2*JpKvr@Xs_*QEfp7@ytAe2Axf@(k1Qt0&q1Gx9(_SXC+2lcZ`$gdUj#bEi$mU`r z)UcK9F26nyg3yW~+Ohl(jT!ly4lqrJeaaOIK6RdMM`hCi_f*;-6vOz-A=H|*o=z1l zrn7&oX}ZY-=9yC`dxw!^)F?xBFC<7kM~M7x^mFp^pE=#-jAJLSOWPiep~4?x^jj;9 zJ6@*Bd6`()l%3Y#{_Jew3}uu^u4*E=+E~--eaq-yTqMnF+DqxHl_@Y;oj7rhmZ>Z! z$0AQME+q;+>p>=uHq!$4Xez5d&OC68akrt3af3cm;&)NVpOb^j3U#E^>Oyxv^KA@F zg7IuKXzjDX=u)N`)3w2Zse~-pqp5HaqAD0WT!qiIN|?1=#q>SZSSVbB60g(9mrg_;>&FU5rD4OL(};CF ziggAFaCje%9mUZY*?I&)LML!CDGeD32a&oi5?hAEd>e8%AD zY5A&&6xCxw_H*1Bn_(+;T-{22*+h%W7E-}WUlR8Yqd}!4x;p7J1rD#HU)fCKQ1pd* zN*TwpTNV?om?kNBEPfg3!}zNKmN&9&0X8cOXtIFw&1d`xaB#L}l1;2l>6=lhiyJ+TJ%1GRYn_Xd17G^4q?9x)57fA0@LS9dch!Vp4T6&Kxd9`S>E#Rvw2Y$9giO zo$#6(1cQyc@YOH`IUCpGUgk19*yM%5wJd9s<;%=U&_~_HL2`O~n~p5IPxk2@ZZkGBu&VPxI&4vY5Hsv9boRo7Wr=KT5 zYNqlubY~egh=!BW=;I`jaf~!~Skt*d2U4urNd@y#sNE@r{B1cpa&R8wr>vzqzhFA| zig}_vGRRu*0_7ZQq(;%Fv}MOfirysz`z6c)d{6;{g{tV$7=u8oahR}nBFopEipZyC zsF$9G)%zzwD|b3O*Ic3HwHyKeHbbguKkmOz#Mz2ul)OxZ>!D+qdm$Saiq1kK`x5%k zUd7Px1|&E(qpZCN25+t*Q=G@#dDZ9&szpp#Gc=^*l_OkdKQM zg-DoG3eky|aq)Bsb}1c0eb8?BdHTW5#21>1zSy~VJ){mTf^dxkjxjAkuhKLq%S^+X z4Av*=bdt)XYjmvcHr+mXleY5rsQkLd42-mY}Yn|*=KGv-C`R3Ew)7eU`tGO1%tDGhBc zrRv!;X^o^aEn2md9$el;Ez`rP=Wr7BGylrGnHOpN#(HuWc|hw%_t2WDKS+~l6GKFp zhwg_w-i=VjyjHgJn_>XlC<8>_(#3$3E?y28F>aL&Eblp^pw%7HbzAXpYz)k&#$(yX z7&PqIkE_kGFkroKU3@zH{PSQfTZx|UHJJUe5=&w$a4x?RLaH^eyHN+_rUtCPPzU|3 z)fj)}G8X+V0;hEf?Rr@d*UN_M$ULkzVCSO484NAW!Qk%*=+0b@^ndg5mCYUMCd|Wd z|9SWr?+VfV(~uK77Q$wFm?Emg>W|^@P-`Wp7M{esuF$}L4Rnn$L>7-fO@$*9sJzjS z{C~1L$%;tf(_}ikX);}11>(t>tX%9UY%NFCL(^&NPeZ!wYe?8If#m%fsmP*&@u3-m zqd1$ucw({=%zJ5chs=LWrA?=7=!)WOx^U8qLKC-<$Gu2u`;biMcb%k`Sw*D(@hY7> zdy`hsV^Xr{rPhY;G&e&8V?&u1al%MMFrAfSw>p;QYT#(IHuAq1;fJz0G}C8crSfc$ zs~6V!@4$_-VbB}c$M&U>_|G8{xBGS@S8oSOSe>^2z%kIw0<4e7!>S1dXlOixo}Yyn z_q7BzRu}N)PceQ~6=HnbIg~}^L;Xnx689!!{nJBGH%fu9_YwF#KZ@wSBaruw!!;9@ z^E}%ehMUZw`C$eEnWp#DNeh%LorEK;EH6n&9qN5@j6=y7*Md*ve7l8$YAcCY9p&{i zm10&io@W0(x?~njHiBR>cK2br83)qowx9*)Eh+LMD7VCxqR(5CkBJ(0EEtbAZw+{>=pwqy2(@g7KgNi$C;N4w7+q6tx-Ek7B&fVia$uZ_eGLg#1@j?JD+?kENSZ)rg^QLOdGD5 zlhisDGTtalp4A^XGv7=rn=3qZy93o1>ZaU5iyYyl`=0+^wZZaP`5E&lbc> zZi9Yu5bkUV#}bQZ2wO!%yq(QzV`JehbpV!)i7@$>1ZBTujP6fH#foICYCee7a}v>Q z9tA`1VC=aTh=P)xaMcLK%!s}CmAM;lleXZulo!$_n_+D&;}8YOK|g;qMjjl8zb^Wa z71M;uSOw@9%A&vMH>F(^#+-uB6j6VTB)2f%J>&QY$1paM^GRCJahRH-$NNPUABU)%{X4emGWJ%<00qiO zq2s9nUX4(M>ibokUSkp$y7(fu)un^;TPj1&7)QM z@;*aypDQTL^#;xU&U`R}Zj$=M{9KA+FjADpz%muQDA$3O(s_FO!ooEQzg}(c{+3tev86(0m^l~3WM@L~NtG}1* ziDCTJ7|@4kEESDGe?TO3EO(;YE&zkH3C59|5F6(Y#mAd4{)ZnF&DWs$&_eWe+9F}0 z4t8E)TILb5_%M!TTrjOMBx&`V02m z@F-h<)S2C0B^JM0U?-B?93k2ok}8g29f%sD&AFw%3HJ#wnLY_W8fW zd8=o#;8pSL=DEXcRYX1uT5y`RztDq9_!jJ6nuw9c`Iu+<5Zx3pC8z2IGmQ?Cq3u3VQMr0COD?wIG;ZrZ3gifWJWJ7@M-dY$3R`#S;T&OPZV3oLS#X76<4buPSSd zmF&JyD!El`C`>UvA_-O)C9yG&6F>YdVr2o_*{^_|ENR?*alGLLmZuqsD`txj$Y(`6 z2B*Pe=WS?(%G3J*ebTTtrQl75beFqrUN@N2eRXRp7KTyh-(j@cze%uFx*#lnd_;)Z z8Y`HOZkEXY^$=IMda>-KMNFaWJL@;!7_wVD;F~=geUER#JkHyg#CIVwJI>+8;;Seg z_5h})A0XfM3oCodQ$?c+9ek%jS!Ped+I(yLI#p7t)gb7qt=ufL-CsT6I0Q$upeyJz?DY@F0wu{D*QayjRnr1{Zq$(tSrT-zN7 zG5fw?e}8mHmc5m-{jpQA^sfuTb=noDQ z8|_T2(S`5FTeUZ9QwYP{`zlmww z1D;9J1+@=lq&MD;2JmOHe3=6sSnj|*Ft(IB#D+@wU3;tCVETS!5V;u{^RACEo#z=h zi&+M=$6kl_^chGga}}t=kTW`ebM6h3C)bOAVYROvme-!4*y0Xm4{bm>=Sa+YQ2|-r z)$U`LkMe0b*l(JOpJ{8cbJbJ~zioqUi)C>+=oFjJhO=E3L&fi$9!gZ!O%(EGTM3aB zN+O+L19q*dP<(rDFF}6(0-?*wSeV>HMwFzf!93iDb|Eq??YARG353-i;cW* z8G3JeR3PJ`#lwkCVL3_$az=xrSX) zf6dwhRj_`71N`ssu9J8MXo&ly8MB#T32cSvJll7^ zotY)8!RO~NM6MdgT}v}D&|*ErTX$m0uv}=WmE)mFBg%$-Gx@JWv1Pxqwg z8$7eUMuT3RQ=`U=J`|awNGA;xDRQ#9y$R^C~+}eSl&6a~wT#4;K|1F-I)Llr5)W@+l8$H}av%xi9jg zGEn&~5d#fYLti=_!bR?NP3(`wJ`dO_^UVySK8rJhZ&$r>))2bhOcdT9@)ukzjtW(h zer%hE4tuC}Reb8HIxCh*XZ=0?v6cX1^tKiwEjtDY<2d(wSv zA$VA*5VG=*q;8jt#3X2fc+dhF<~Aspt=n~oJ==DZMRm(?Zj%AjxF^H9dKT||$Kq>V zBJ8C*@rrZallyS~R>B?p;5*_?wePU+zYYvcYKPAz&fX4u2mQ|X`1$k`?ihTBc;jCL zdCE~~s{(1vP@ym08l3aY^9mwE+7n|+&YOnt47(*giLs)~_bq90g(aPuZb^exEr{_u zz`mL8d&wq}jKSG@=gM z$5%tG?^#TLejLw`g}|cyP*kv=!!#B}av|8b$P=P2Gb9G}!uMNsO#9qQwxZ>z zcy^hOtoGaJ z0roqu!-Su6=65{79r}oSMnABP^NzdhIS z3uj6`=}bn0oT>cTNP5V3%(Z`=XrYkiYVsCzwb__H_0gs(&MqwSQ=~>iIZB%P z6>rmD!9s!ef8DrSL#RhhGUq4MUckh~<)|3KT^_;_^!l*}9Wp!dv@D*pZx`}e%q-9? zFTCDpi$kWG*uCc`3z5IdmQ*IQ5u^Gt{Q-kSot;n^mv(>2@2P>}_~{;O z49N9Knh&oWZ#N8}fXW>C#jK+I(p!&1ul$eK329eB(q;N~7qy z?-W|MJcumM^E~ibKEwGto_3HoxjbQ#Z}TE0von(=0T0_QnCH)`I{G(FV%;Sp>^rO_ z$eGCrZC`UGJ6D)jc}zMiE}us%EGLiUH@C8WxAlej29ziSiF|vF(lO_5MPEP-fE1UiybGZW9sHjoq!~S$pnY$|U zjR`>(H20<*?QL?QVWsZ0w_pruCVJC4kq^my_NH^I$MdeoSo*wf48?bNlJ!`ku~(f* zlXq3eNCs2TEAE+?z~6UFfqHoV#u1yh@LkJ$^j^2niYqW)BVevtC5*k#KqapTsWtgH zy!{Z$Yuo0^}7Qtbu}bw`37(z!3 zoXBQ8&*N^eqpU_yQ=k_;j^i_HJ3i#~3#12IC(^C1akTT*C^|dHgG_xI#h<*&#VuCP z#oxUnSZ|G8?E29yOi6Dtd*5xv9M^d=Z<#nYYE>nBJ-Y|KEU-e{+aP@Fj03f1A-?1S zip2L3EVLskUY4RitJ2@ST4YpWNFVuoD0HMg)wOHW{#OI(Mh{K;m7qyp>$GST_mua) zZbBuNR#frcmMT|sF3&MXa^$<>7zt4>?}sFXj3XuSMB+>a%H{pQiI+ln4myZuEd%Ln zP5{Yf`qP9rJfFM6i{w9uXn(aG-CbZtR-CP?Vb48fC%NZuW(Qs#eh$pJi*vWGV%$Xu zq?0NzXz3}Oay)@2lk@S_EeDyN2XJ@fF5J{efa&&CsJS=?T8e&HqQWp^zBNWaG{B^W zfoM%rg8XqM$emX}n2kIh4pqQmt-dIjXo59LqtMfKG2+8^b4F$<=8e9G(#T)9W2Z{T zS8~tYuA$VUYb33|IEn^r@}apw0n}kVk!lxAp~u0Yl#@J01 ze^(0faiy7o2iS(T{cM4D8JqRz8+*NoGfXor(ao7;m%L2Svfmz}H$E`Rjz;mHU2tqY ziITSKSU&zECgjP{oMC>i(H70`ADn5i$a`yXpgKfi7R}`_KP>MwDI)j z%UCj>IhHQ&9z#DYJn6{eQ8Z(o2VLjS|WpYi-m-_ca^eKZxdjiZ_?<0&-4osPGTppr}6*@!$DmDBX&~(!^EV zvo{Bmil*Urh&P7M^FV>r1AqUDI8WaTnLlPAsc|Wi>b79&xC}4@&QUVB2odl4f8G8Q zcMX&&B~6##1udy#(+Ki9;7;D1<4L7+65q>Br=OOysQ=zMbh3US*=}9T=V7a;+aiXh zdq>kuKVQ zk@hLn9jwEPQ*RJZCPx8*{i$@632ArRa&7@5-M_B1W7r7tQQ{eYpCFp_dlpr0j->Cs z=8^N}`Q&dhk1T#f(BwrCw02QAnKp;eYsqA?YW61LcY`+@MRecQiK4@;Nv(1aCEemZ z!?3|rF@?LD2Rc!ivK{UIYD15Q+EDHEp+xg6=xwbvmH9f88>bA$bN^ffcLaA|(IZE{ zzT}-DPyLU6Lq)Hb-S?Rc``y}6S*HW79soAAm3p%Oky@c#wH#^EYfh`DL*GAS0QM6Gi+5p zBg?iIUHGU$zB7zzZnzC;=DCpSc~5%E@9c97rjU+VFok>#qlh_?bZ^1}T3x=Fk{?CW ztL5v+<@!dRYvsB8uFd>sb`@<=A48HDXS(pjma3F3XgOzzE{_%Am76EVI824XoaN9M z$$61=CCKV0MZ0AUI(ffb|JZvJPmrPENoq8Fu_3MfI*j(e7t_b6v2^h31X7*mN7r0H z5$3iuqsf(GE{>sp`Ml2?KZaChc+zO^(X@5%SnBb0JgvMro_@seU+-d6^q(6Ym^G4A z%7@Y24ig$VNQYFtbSdpWJ-T>-Gda@u{NjN*{T$8D;C4oQH=<3m7xky*vYKSrPoH-^ zEcmx+Fg!y$k$NU`VCUVa0_)O@=W-iBePdP`7|I9uxHlh44wsbX2M6dnF(0~&YDJyCkrDV>eZQkM3uWSxA+Rdk( z0Sjr4`n3_dw3pAJ zDsxqN-=;U^zEUQ~2b?{(N|U}ds8E2F4B1$=LcioO?vJ>Io%gOFZle_a_n*g&{b%9+ zy9@(L{skkxDv^w)4h2lHphKKj%fAOpP(euj-;bMkjD8=ri=S= z&Jv&dh^%Rcg&nb*RiaB{9eGcwG?`qf79lz69+V#n`pz1ZQ3z$8wbdJmw{Rg{pjfOgM(1(}i&P zd;-t;JMjv46FhuSjL?9y_-9m)S8F)~uJbQ~f2h)h4+eCCJB=oB{z+K22YI%5Q}Vb; z9eVOZ8+5g^A3VuB-QXd&Cl9MvQ_1Kf$y zX8uR&KJ#hhbZ+-5m_+s*Ik>P^k9yhh&h%wPipu3Xj$1O^t6PTgJ*(imPRiYo_sa1Bbw?qKZIkNDR6J4&s-pi}-MQhKx_uak4TcHV{FnTw#WrAT_1gR-DW z7-Sv`yP>HVIHwfum+DaQ{2`{l>cWl9Dl{Zqn_^z@eEn)uQoL?TS|<(3XoU_rZqOko z2VJTPD}yFeZ;vT-QzwY( zMg&vYxe%&w458D$(`nnvDZJ}7nQHD%BA-nY$vSmBT{7Ys)tC`fG<*oX;QobZW!|+v z`2$Mh-oa$+eVi|@hq2N%6zE+=`oMbR?{0wdt|r{s^#-B;eT0lXXL0?hgq%S!igG#E zUAhs&n!FG)%N`5lryxyr9o!Um!l98nX+K|uQtW%kYst~*>zucksYOMz2GOnGL#Vxy zXZHKqQ0JW?RHJN8v2H`D-zqy=aGTGE`E1oAXefQIw;;bpGpgHYNKs8%G(lg3jC!b3 zKT@OR%E}ZlpS#VvzQF$EeN@Vv!`{pT$nLcn2~!sHZeI}2I{QNHwLesEMnFMrBPMyL z;FxtLLRRKsfnzb)sx#baU54AQ&){`?3D&GR&RsS~u=L;|d0){|XS? z!M#%j@34P`JOwuJ`QTtv`dMI4;cYJDbC!`>A(6}yqN!s+q57aQBD$GBio)h`)=T_Y z8YuB56`o(*-|A0#&jaY`_5cb<@Ta!l-ek6M3@xtoq+9%K_4^;AS?@>Eno0IF_?R&@ z8t`{Wwi3BEcf(2Y5LMOnxLbQ3ey@sn1{TaIQNmZvAm^5pSF zo~+c>$dhM9+(v2BA$MJRvPPTQV|oAbf-#+T9!yn^oZDPuN_z6f1Spwt0Jzti1+Ble}Tw+a5-0!x1Luio5GQ@M+3q z_;oGd?}N>-UY`b^6wYDZlZRxD<7hd40{1p>$KKZhtXIy(!9E8NzA_!Rns!3`YzNZj z?nc1*Yy@@j?@{mjuo?0bL*6QpuU$XND%2*AR1<1^XGw7Z!zk>69qAq5&#jPW7VkLn z1h6xWOAwKw1(C_iQIsa}q;D6!D1FB`l3C$Rvb)A{E)?eqtryX*HfKuN;XsZn9LQ#x zJyl;Yqj{VUIEyo}Bu;Xqu;nZ73$~zpqXc>lCt;YK59?EhklH632fkdxP|qvy?#*|z z6EEXiG|#^LxrRfLRk(IL8=D90N5SUpNMFkR7Gq8z){SR89#&yc)&sQP?}GcGj|gr4 zfTEeLm{#=)uAHUkI_(zLzj=bfi*0y%v<+U1KjA6g$%TCX2aC-;DgKHAt=_9hAuHwS zzXs0I*Zu?dluwvB{{^CD9^i)OEo{GDi*3#2@KMc%{FnqZ9}31=Rcqc&kl{PkdKQ1M zk&W-{WLy69!fm{gI-dQox3#6wK92OFlyhOU8I_oiq7~OY_&I$v1s@ng)H0Sh$$~4CPwGXCy)6n%P9#s)@Fmc^TI9%+7iCt${ z`Z-_LvaMSDZrEn=&4OOcLK?#cPpxE)yZ*96Gjw1!+yZ}oyW#9=A1ExJk7oy0qfvVm zjyWts%F9T+mJ3Cpdl1g_4uV*I77T_gLodAqbidpSm96}|&K(Ad!}xqZ=NWtl@H^v_ zcH~=i;lCR_NF`N)oHwY^jLd;#xZ8k+)SA+{E-Q+kKb(?1TqwbjEB>~RB17pY63y`> zv-x94l;A-=7H*`QxMJY_;V^$=1;b+2WD=30ar+?2th_MK)E zMYmbU*KU?LTNfP$Cdgc7jG2ChD77<&@kuje1Y08`UIfpv0l2=1JL&&ytnn+x zfOG-VwQpdO=L7U>c@DAZJ0!Stz|xb)HJb0n_|7Wl1Xj_S7bE zp_YYW8hb@Vj=GGybcuTcx!PURg>H=)PU(&|)OCb&T%+`;k~^b9FLmQd-zH3Id}aHB&25|8x%)3{S+Ag&Qy^Y#9!AEyG}q0!VKi#P?Ub@Wv_`Q?{qW zeb9dHh|a`&WW96f7iA|G6ySf_n7QUTJ5z?J*2#(|a2*sxrrG0(mrFSF02!0Org8aBW zf@gxekpJMBB=e}Tr0MEl@rnK_Y+}?jHdA#s+xxG8<@6|E*0v=q`?{1}d)C6f?&t~g zR$WXG98r@S2){ltxcQEEIXm*;F`yhf8?GQr`99j$y@A5OPI#^R2|YdTKB2=jum~b2i+RG7RTg$(t*U@c%9Y47{r(BeR_i<~noGn0b z-^r8?y3ZVOF;SGZ(}n)o}~-ixY5sK|E}KFN00}LL6Ke4i%p%kUa3i zq#nNTs~L+0N+M*PHRrx-WlVeXm@N<;VAJm~mj7*@*wQXv@_1E_%Zp4(9n2a z&~$qwY!f{cdR>1aypw+^tR4DEi1z3Z+J3zftTG-5pOv2qMH&Bu4K}*c0dXUxeqSd@ zr(K#MofI@fT0PTGDu3TaI)42iskwY_Y4-b9g6MjwkQ<#W?C)v)ZTEo!_c;k(}|%v9eAxed9vwW1WQH)_xn zdY?O_TH)NU8w)ae(zp^u+EUPmv~rYb`$Nu2?CwYJ-f7ZWK40C@YD89izr3)4^Rl{Z zDJIaC&V}*4N|7B&GHvN^h6SliFroBs2IQ@yMe9%Wp|vA_;mp!UIGB1B*G>so`>6t} zrTl);SPkojYlyy8hvE0n;CE;|zfa7?qXhv-7mvlv{bLcqcd(_imcivy7_LnY#Ogi4 zh-#mO>Pevxee}nM5_gO|<%%y}!*I4)2lCFUu&C>WD{s1)UtJ56GcRST`rHI)(JFS> z{-J7IuB$+I&k9fXsY&mCahKjxSSWR?+9d6cO_av!CP^PUCrY2|B}zveN|YMKBuUxJ z1nI};o24tYVx=>tZkGOX+$J3my;~X+nz{;)HK5272lwG^l0Az+A>UsM5~O*r;BrV23pbjw4t<4btuJgSAyq4GpavhL}$Zv zsCJ+RP2+#RBB~n>e4m^%isxVFSEJ^rfbTNZSas~KDmtBt3zVd=} zf=r|Iu*x0j(zLtMtW|fV7o~Tl#gCh%jlJ$kU#z_=U3R!h`p595bZ6-e>9>Ge()N)} z(%`i>q>hWPNY7PPN~1OuNGtbcNZ()EBK?XeX)yUnKe$;+EnVfLWA{}EWIJ1k3qL8D zf5t&f)9l%F`TcCr-bVKJ(obfntBnFTXC&4J;xymQ4DYoSQwQcEOJIJbj!snXr;86G($3B0@?+?G=&d;K+w{+u2Ko2rYmZJgR6-fT1GUe6vrI$_v zX+WtSnVm8sb;UtcHJoRiS`4U=-`%6XXj0Qgb#mqRj48XiQRU3{Kz$w}W&?K@y_G^f zxso&S%McyHXO1WHaOhPQa(eH@vkUy}&@>JF$O$tC8`R$$gpBS%xS!z;A5ACxdSs5a zdR^T1)WW-?{gC9S!1punS+DFGmUQz7yYeWC*}YR`=g)2yFOs?97IVd1qQ9e1GJ1M% z!A5?n5MiAseDCTMd}W48O$RQNx;AG@rx#q5Ub*s8+T-9a>G*AO)%81jRqr{WSpDE{ z@9Nh7RH|iU)T;dlt5siquTt&0rcZT|v10Y89zCm<)^|($Yk!s+_IoZ}XwWEa39OJx zZ|6wmKPOA)NtQ@`kByd&cF>jv2iy_-t(FQsMTaCO4(t|ZI0v#(vrn)`Ki;!%Yt#_0 zH5fhY+)(vtA_A=Ez|1HHi;nR8#;G(uD>?v;$|LBzs1P@0PQ%6HEUpDhps=at1*28?3kej5&UP&|D`+)~A%{q(qGxh76#qPqk?IcpXai)TWV} zG-=fg?w5S7Odkz<(du>G@Ojh@GuP+*J97tP>aQZ}>Um_h6=8aGHhK@(jiXZ%;n)%b z*QHCaz$^@Dm2Sv3vVv!T4$h2G2Kg$ZHrxWb?K;T)sfuGyWU**YC)>IIG28uB%0kn# znVL=_D;yrilAH#x;$var-?lv@7Zt8Z%yat)&yxlTef>rV`$K|+lddU3@N=m!X_=f< zb&;L4fARvUSIQpg6ZLW_HQkU7=A8TPe{Ir%L%&P!9{nc`D3q&C|0rLrY0#_s+Tfnm zWj6n$s+}EDrx7jE6s-oS+vKxSUCmtSjp*&tv6B}_YivmRtDmMc!u6hDmmM!eSG|=q z+U^qVXpm=Rx92h^A(z>nXkezM9qji(1sw42kJfhv=oMyxS}g~JZ)Dub z5-G!1AnH2rdi_m6a8?TbyxNVAqD)K=$;F2wCos8-dt1k!hvOe9mM^PAvg%#D;TfeV zb#18g_=yAQvec=fNV(&bDXXU%pW&-f;2TvslFhT*yqC?C5pVtf&I z8k9Do<_MqTm!1MUo{fnUcOYl^1{4pCKxYp>_#7n|8#_QbBUta@n(WV=$Kq7A&0@zjS)xUI zmtJUE;ab%tzqu;%$cd^=9&f75Y;7cWTsBKoODiO$mhBQZkG{e%9Y-NSf2MGnGcDR@ z9S}bJEEbl$z95`;z9?LdED;i#Q-!QA!NP|1YQp=LaS~bCNO52<8}?If2RruX0=uj6 zj+I|hM8zdtm~FCxVx9}%MT|j$pFirt{E^Z!4sK;YaHbP%-E1+w+!6&IW_UQ?2tL*Z zQ2cIyeh-b|ahU(!Dl5)laKPLnt9yAU0V`M&(bepJ_G zWB=NGY|1@}(2#Q&`0p~L?{DDVg$H=cpOtLZdniPFhQ)<0Jed6xtNF}2YUVFQEc}Un z=0C79qzj8bbfU(;6{h+x(4+M(5}(&XHKl@k75Sa=*k1fzn~0osOYp~d3NCDO$F^~{ zaJ*-T_kYx3pD2TXocH|P)5yG*oMGoJ&aeU1TG+R%FA6s(VA_#y?C8t~?9b)1?BAm_ z7BSzSy{(XEE-~ri5yO6nYP`IwoGW}J{Rc-#b{ou<+>KZznU$U>Ilq06#5W;dVw!hO z(lWC_k`!1f`4|}`dHd>FRauVYT;`IEB9+e-qR!pTBKr(Eu~Otbaam$-_Ih&!bDNdJ zPG4+dcN=7}b&@{V_YqinYa;GchM~`*|B$E6=c;!$L;h(Z9^@y(`e7<&oZO8cwW-)! zx((Yt#&LiAIy~F66k~79gZ=C<)TB>C)6@V&TnIpG*%TPh4ubqqo-@2L7n^!6N8b$_ z5y|fvrA0e1`|TchF3&<-UJha}9fjB7VDzjK62D7W*W7?H5AWied_OkeIE&-)1fnIDiH%;$r$FEGmg0mf~(hBX-%(9Iok zr|J*lrP59~tH;73A{>rWyl~~+P@Kxpg+^^3tQ`N7t)BmwDV0{U$+5>--rikI)_e`? z7qyzjnh!v;M_&vd*#}{VWRWM=$<&WNV9%%U%=*ww*5tU9rI-w1!OADZ9;*T`*e4fB z{%%kZ+}wK$uY**1zPZ2fxK3BlId3AA{~IcdIyPJ=f6-f5W_dues9|qa?tnhx(2;w@ z$wm$0pRc})Eerax;BAbJ$Vz4FU2B>1=HJYL|EvT)cR>Boi8vj+2=1Qoh{)K64&7AP zPv3*aTn;P>_fPlKSOl*D`qxK<7`%h7@eA(feT4Vk z%}{i@iTVGoV&!ZpW;m5%kSBls6Zb-tycO1Nt1&xc2JU@ec)8OAOg~RGG=~Nuf@^6uf%z$`{VgM4g6lI4vCL4d~6kv6ZeljVXe%& z_zt`ItCEe0Ilv71u4JVfZCJ>b*J8Ph1hIbcMp5bpBgw5{S0y4>6CtHMLh$s75`I37 z7Us7a2^+WVlw8spBGQsQA$CkrXA6J1Fq7dknfj6#w&%|wR-5#Qow4YHV9sHPD)vN_ zemH)G$HL|8PH1jBgc&6ZM@hMq)zS|R+6@?qSZ14I6eE^*^Kbv`G%&v)rpjvd8@{6Z+_^X%Htv)DFFz!%qx z2>5Xs*JfP7My)!?EUm-B16MK4@d|p_*I>x*t7x#fiI^q#uzd1UL?pb#qq^5fUh)PR zBY0Ls@ilyZJ;mP<_j#84Cgz;xoro3H7#nvExBokhJUWIo?gw$gB^4%aarj-dh`U@S zn}TZ-bf{#7k^la<(( H_vrCIL$|!@ literal 0 HcmV?d00001 diff --git a/cf/test/test_CFA.py b/cf/test/test_CFA.py index 6360d714aa..4c6fc2214b 100644 --- a/cf/test/test_CFA.py +++ b/cf/test/test_CFA.py @@ -4,6 +4,7 @@ import os import tempfile import unittest +from pathlib import PurePath import netCDF4 @@ -56,11 +57,9 @@ def test_CFA_fmt(self): cf.write(f, tmpfile2, fmt=fmt, cfa=True) g = cf.read(tmpfile2) self.assertEqual(len(g), 1) - g = g[0] + self.assertTrue(f.equals(g[0])) - self.assertTrue(f.equals(g)) - - def test_CFA_general(self): + def test_CFA_multiple_fragments(self): f = cf.example_field(0) cf.write(f[:2], tmpfile1) @@ -79,11 +78,8 @@ def test_CFA_general(self): c = cf.read(cfa_file) self.assertEqual(len(n), 1) self.assertEqual(len(c), 1) - - n = n[0] - c = c[0] - self.assertTrue(c.equals(f)) - self.assertTrue(c.equals(n)) + self.assertTrue(c[0].equals(f)) + self.assertTrue(n[0].equals(c[0])) def test_CFA_strict(self): f = cf.example_field(0) @@ -93,7 +89,7 @@ def test_CFA_strict(self): with self.assertRaises(ValueError): cf.write(f, tmpfile1, cfa=True) - # The previous line should have deleted the file + # The previous line should have deleted the output file self.assertFalse(os.path.exists(tmpfile1)) cf.write(f, tmpfile1, cfa={"strict": False}) @@ -151,48 +147,283 @@ def test_CFA_field_ancillaries(self): cf.write(d, tmpfile5, cfa=True) e = cf.read(tmpfile5) self.assertEqual(len(e), 1) - e = e[0] - self.assertTrue(e.equals(d)) + self.assertTrue(e[0].equals(d)) + + def test_CFA_substitutions_0(self): + f = cf.example_field(0) + cf.write(f, tmpfile1) + f = cf.read(tmpfile1)[0] + + cwd = os.getcwd() + + f.data.cfa_set_file_substitutions({"base": cwd}) - def test_substitutions(self): + cf.write( + f, + tmpfile2, + cfa={"absolute_paths": True}, + ) + + nc = netCDF4.Dataset(tmpfile2, "r") + cfa_file = nc.variables["cfa_file"] + self.assertEqual( + cfa_file.getncattr("substitutions"), + f"${{base}}: {cwd}", + ) + self.assertEqual( + cfa_file[...], f"file://${{base}}/{os.path.basename(tmpfile1)}" + ) + nc.close() + + g = cf.read(tmpfile2) + self.assertEqual(len(g), 1) + self.assertTrue(f.equals(g[0])) + + def test_CFA_substitutions_1(self): f = cf.example_field(0) cf.write(f, tmpfile1) f = cf.read(tmpfile1)[0] - tmpfile2 = "delme2.nc" cwd = os.getcwd() for base in ("base", "${base}"): - cf.write(f, tmpfile2, cfa={"substitutions": {base: cwd}}) + cf.write( + f, + tmpfile2, + cfa={"absolute_paths": True, "substitutions": {base: cwd}}, + ) + nc = netCDF4.Dataset(tmpfile2, "r") + cfa_file = nc.variables["cfa_file"] self.assertEqual( - nc.variables["cfa_file"].getncattr("substitutions"), + cfa_file.getncattr("substitutions"), f"${{base}}: {cwd}", ) + self.assertEqual( + cfa_file[...], f"file://${{base}}/{os.path.basename(tmpfile1)}" + ) nc.close() g = cf.read(tmpfile2) self.assertEqual(len(g), 1) - g = g[0] - self.assertTrue(f.equals(g)) - - # From python 3.4 pathlib is available. - # - # In [1]: from pathlib import Path - # - # In [2]: Path('..').is_absolute() - # Out[2]: False - # - # In [3]: Path('C:/').is_absolute() - # Out[3]: True - # - # In [4]: Path('..').resolve() - # Out[4]: WindowsPath('C:/the/complete/path') - # - # In [5]: Path('C:/').resolve() - # Out[5]: WindowsPath('C:/') + self.assertTrue(f.equals(g[0])) + + def test_CFA_substitutions_2(self): + f = cf.example_field(0) + cf.write(f, tmpfile1) + f = cf.read(tmpfile1)[0] + + cwd = os.getcwd() + + # RRRRRRRRRRRRRRRRRRR + f.data.cfa_clear_file_substitutions() + f.data.cfa_set_file_substitutions({"base": cwd}) + + cf.write( + f, + tmpfile2, + cfa={ + "absolute_paths": True, + "substitutions": {"base2": "/bad/location"}, + }, + ) + + nc = netCDF4.Dataset(tmpfile2, "r") + cfa_file = nc.variables["cfa_file"] + self.assertEqual( + cfa_file.getncattr("substitutions"), + f"${{base2}}: /bad/location ${{base}}: {cwd}", + ) + self.assertEqual( + cfa_file[...], f"file://${{base}}/{os.path.basename(tmpfile1)}" + ) + nc.close() + + g = cf.read(tmpfile2) + self.assertEqual(len(g), 1) + self.assertTrue(f.equals(g[0])) + + # RRRRRRRRRRRRRRRRRRR + f.data.cfa_clear_file_substitutions() + f.data.cfa_set_file_substitutions({"base": "/bad/location"}) + + cf.write( + f, + tmpfile2, + cfa={"absolute_paths": True, "substitutions": {"base": cwd}}, + ) + + nc = netCDF4.Dataset(tmpfile2, "r") + cfa_file = nc.variables["cfa_file"] + self.assertEqual( + cfa_file.getncattr("substitutions"), + f"${{base}}: {cwd}", + ) + self.assertEqual( + cfa_file[...], f"file://${{base}}/{os.path.basename(tmpfile1)}" + ) + nc.close() + + g = cf.read(tmpfile2) + self.assertEqual(len(g), 1) + self.assertTrue(f.equals(g[0])) + + # RRRRRRRRRRRRRRRR + f.data.cfa_clear_file_substitutions() + f.data.cfa_set_file_substitutions({"base2": "/bad/location"}) + + cf.write( + f, + tmpfile2, + cfa={"absolute_paths": True, "substitutions": {"base": cwd}}, + ) + + nc = netCDF4.Dataset(tmpfile2, "r") + cfa_file = nc.variables["cfa_file"] + self.assertEqual( + cfa_file.getncattr("substitutions"), + f"${{base2}}: /bad/location ${{base}}: {cwd}", + ) + self.assertEqual( + cfa_file[...], f"file://${{base}}/{os.path.basename(tmpfile1)}" + ) + nc.close() + + g = cf.read(tmpfile2) + self.assertEqual(len(g), 1) + self.assertTrue(f.equals(g[0])) + + def test_CFA_absolute_paths(self): + f = cf.example_field(0) + cf.write(f, tmpfile1) + f = cf.read(tmpfile1)[0] + + for absolute_paths, filename in zip( + (True, False), + ( + PurePath(os.path.abspath(tmpfile1)).as_uri(), + os.path.basename(tmpfile1), + ), + ): + cf.write(f, tmpfile2, cfa={"absolute_paths": absolute_paths}) + + nc = netCDF4.Dataset(tmpfile2, "r") + cfa_file = nc.variables["cfa_file"] + self.assertEqual(cfa_file[...], filename) + nc.close() + + g = cf.read(tmpfile2) + self.assertEqual(len(g), 1) + self.assertTrue(f.equals(g[0])) + + def test_CFA_constructs(self): + f = cf.example_field(1) + f.del_construct("T") + f.del_construct("long_name=Grid latitude name") + cf.write(f, tmpfile1) + f = cf.read(tmpfile1)[0] + + # No constructs + cf.write(f, tmpfile2, cfa={"constructs": []}) + nc = netCDF4.Dataset(tmpfile2, "r") + for var in nc.variables.values(): + attrs = var.ncattrs() + self.assertNotIn("aggregated_dimensions", attrs) + self.assertNotIn("aggregated_data", attrs) + + nc.close() + + # Field construct + cf.write(f, tmpfile2, cfa={"constructs": "field"}) + nc = netCDF4.Dataset(tmpfile2, "r") + for ncvar, var in nc.variables.items(): + attrs = var.ncattrs() + if ncvar in ("ta",): + self.assertFalse(var.ndim) + self.assertIn("aggregated_dimensions", attrs) + self.assertIn("aggregated_data", attrs) + else: + self.assertNotIn("aggregated_dimensions", attrs) + self.assertNotIn("aggregated_data", attrs) + + nc.close() + + # Dimension construct + for constructs in ( + "dimension_coordinate", + ["dimension_coordinate"], + {"dimension_coordinate": None}, + {"dimension_coordinate": 1}, + {"dimension_coordinate": cf.eq(1)}, + ): + cf.write(f, tmpfile2, cfa={"constructs": constructs}) + nc = netCDF4.Dataset(tmpfile2, "r") + for ncvar, var in nc.variables.items(): + attrs = var.ncattrs() + if ncvar in ( + "x", + "x_bnds", + "y", + "y_bnds", + "atmosphere_hybrid_height_coordinate", + "atmosphere_hybrid_height_coordinate_bounds", + ): + self.assertFalse(var.ndim) + self.assertIn("aggregated_dimensions", attrs) + self.assertIn("aggregated_data", attrs) + else: + self.assertNotIn("aggregated_dimensions", attrs) + self.assertNotIn("aggregated_data", attrs) + + nc.close() + + # Dimension and auxiliary constructs + for constructs in ( + ["dimension_coordinate", "auxiliary_coordinate"], + {"dimension_coordinate": None, "auxiliary_coordinate": cf.ge(2)}, + ): + cf.write(f, tmpfile2, cfa={"constructs": constructs}) + nc = netCDF4.Dataset(tmpfile2, "r") + for ncvar, var in nc.variables.items(): + attrs = var.ncattrs() + if ncvar in ( + "x", + "x_bnds", + "y", + "y_bnds", + "atmosphere_hybrid_height_coordinate", + "atmosphere_hybrid_height_coordinate_bounds", + "latitude_1", + "longitude_1", + ): + self.assertFalse(var.ndim) + self.assertIn("aggregated_dimensions", attrs) + self.assertIn("aggregated_data", attrs) + else: + self.assertNotIn("aggregated_dimensions", attrs) + self.assertNotIn("aggregated_data", attrs) + + nc.close() def test_CFA_PP(self): - pass + f = cf.read("file1.pp")[0] + cf.write(f, tmpfile1, cfa=True) + + nc = netCDF4.Dataset(tmpfile1, "r") + for ncvar, var in nc.variables.items(): + attrs = var.ncattrs() + if ncvar in ("UM_m01s15i201_vn405",): + self.assertFalse(var.ndim) + self.assertIn("aggregated_dimensions", attrs) + self.assertIn("aggregated_data", attrs) + else: + self.assertNotIn("aggregated_dimensions", attrs) + self.assertNotIn("aggregated_data", attrs) + + nc.close() + + g = cf.read(tmpfile1) + self.assertEqual(len(g), 1) + self.assertTrue(f.equals(g[0])) if __name__ == "__main__": From 0f46fb902558cf00ca10455cba649e485fd86375 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 13 Mar 2023 17:58:12 +0000 Subject: [PATCH 059/141] dev --- cf/data/array/fullarray.py | 57 ++++++++++++++++++-------------------- cf/data/data.py | 1 + 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/cf/data/array/fullarray.py b/cf/data/array/fullarray.py index 3de8a7804d..66f7bcb1c3 100644 --- a/cf/data/array/fullarray.py +++ b/cf/data/array/fullarray.py @@ -91,7 +91,6 @@ def __init__( self._set_component("calendar", calendar, copy=False) def __array_function__(self, func, types, args, kwargs): - """TODOCFADOCS""" if func not in _FULLARRAY_HANDLED_FUNCTIONS: return NotImplemented @@ -170,35 +169,33 @@ def __str__(self): return f"Filled with {fill_value!r}" - def _set_units(self): - """The units and calendar properties. - - These are the values set during initialisation, defaulting to - `None` if either was not set at that time. - - .. versionadded:: 3.14.0 - - :Returns: - - `tuple` - The units and calendar values, either of which may be - `None`. - - """ - # TODOCFA: Consider moving _set_units to cfdm.Array, or some - # other common ancestor so that this, and other, - # subclasses can access it. - units = self.get_units(False) - if units is False: - units = None - self._set_component("units", units, copy=False) - - calendar = self.get_calendar(False) - if calendar is False: - calendar = None - self._set_component("calendar", calendar, copy=False) - - return units, calendar + # def _set_units(self): + # """The units and calendar properties. + # + # These are the values set during initialisation, defaulting to + # `None` if either was not set at that time. + # + # .. versionadded:: 3.14.0 + # + # :Returns: + # + # `tuple` + # The units and calendar values, either of which may be + # `None`. + # + # """ + # # TODOCFA: Consider moving _set_units to cfdm.Array, or some + # # other common ancestor so that this, and other, + # # subclasses can access it. + # units = self.get_units(False) + # if units is False: + # self._set_component("units", None, copy=False) + # + # calendar = self.get_calendar(False) + # if calendar is False: + # self._set_component("calendar", None, copy=False) + # + # return units, calendar @property def dtype(self): diff --git a/cf/data/data.py b/cf/data/data.py index 312b2f2bca..585c6579cb 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -6343,6 +6343,7 @@ def set_file_location(self, location): raise ValueError("TODOCFADOCS") dx = self.to_dask_array() +# name = tokenize(dx.name, location) dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) self._set_dask(dx, clear=_NONE) From 43fca49c315c72bb154565abf16d9295c4c220f5 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 15 Mar 2023 13:32:39 +0000 Subject: [PATCH 060/141] dev --- cf/data/array/fullarray.py | 60 ++++++++++++++++++------------------ cf/data/array/netcdfarray.py | 14 +++++++++ cf/data/array/umarray.py | 13 ++++++++ cf/data/data.py | 23 +++++++------- cf/test/test_CFA.py | 1 + cf/test/test_Data.py | 2 +- cf/test/test_NetCDFArray.py | 9 ++++++ scripts/cfa | 9 ------ 8 files changed, 80 insertions(+), 51 deletions(-) diff --git a/cf/data/array/fullarray.py b/cf/data/array/fullarray.py index 66f7bcb1c3..c362a627fb 100644 --- a/cf/data/array/fullarray.py +++ b/cf/data/array/fullarray.py @@ -90,12 +90,39 @@ def __init__( self._set_component("units", units, copy=False) self._set_component("calendar", calendar, copy=False) + def __array__(self, *dtype): + """The numpy array interface. + + .. versionadded:: TODOCFAVER + + :Parameters: + + dtype: optional + Typecode or data-type to which the array is cast. + + :Returns: + + `numpy.ndarray` + An independent numpy array of the data. + + """ + array = self[...] + if not dtype: + return array + else: + return array.astype(dtype[0], copy=False) + def __array_function__(self, func, types, args, kwargs): + """The `numpy` `__array_function__` protocol. + + .. versionadded:: TODOCFAVER + + """ if func not in _FULLARRAY_HANDLED_FUNCTIONS: return NotImplemented - # Note: this allows subclasses that don't override - # __array_function__ to handle MyArray objects + # Note: This allows subclasses that don't override + # __array_function__ to handle FullArray objects if not all(issubclass(t, self.__class__) for t in types): return NotImplemented @@ -169,34 +196,6 @@ def __str__(self): return f"Filled with {fill_value!r}" - # def _set_units(self): - # """The units and calendar properties. - # - # These are the values set during initialisation, defaulting to - # `None` if either was not set at that time. - # - # .. versionadded:: 3.14.0 - # - # :Returns: - # - # `tuple` - # The units and calendar values, either of which may be - # `None`. - # - # """ - # # TODOCFA: Consider moving _set_units to cfdm.Array, or some - # # other common ancestor so that this, and other, - # # subclasses can access it. - # units = self.get_units(False) - # if units is False: - # self._set_component("units", None, copy=False) - # - # calendar = self.get_calendar(False) - # if calendar is False: - # self._set_component("calendar", None, copy=False) - # - # return units, calendar - @property def dtype(self): """Data-type of the data elements.""" @@ -284,6 +283,7 @@ def unique( axis=axis, ) + # Fast unique based in the full value x = a.get_full_value() if x is np.ma.masked: return np.ma.masked_all((1,), dtype=a.dtype) diff --git a/cf/data/array/netcdfarray.py b/cf/data/array/netcdfarray.py index ea78bc75e1..b20ca6b070 100644 --- a/cf/data/array/netcdfarray.py +++ b/cf/data/array/netcdfarray.py @@ -11,6 +11,20 @@ class NetCDFArray(FileArrayMixin, ArrayMixin, Container, cfdm.NetCDFArray): """An array stored in a netCDF file.""" + def __dask_tokenize__(self): + """Return a value fully representative of the object. + + .. versionadded:: TODOCFAVER + + """ + return ( + self.__class__, + self.shape, + self.get_filenames(), + self.get_addresses(), + self.get_mask(), + ) + def __repr__(self): """Called by the `repr` built-in function. diff --git a/cf/data/array/umarray.py b/cf/data/array/umarray.py index fa5002af42..75b8c1c66e 100644 --- a/cf/data/array/umarray.py +++ b/cf/data/array/umarray.py @@ -174,6 +174,19 @@ def __init__( # By default, close the UM file after data array access self._set_component("close", True, copy=False) + def __dask_tokenize__(self): + """Return a value fully representative of the object. + + .. versionadded:: TODOCFAVER + + """ + return ( + self.__class__, + self.shape, + self.get_filenames(), + self.get_addresses(), + ) + def __getitem__(self, indices): """Return a subspace of the array. diff --git a/cf/data/data.py b/cf/data/data.py index 585c6579cb..b9dbf9e673 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -6343,7 +6343,7 @@ def set_file_location(self, location): raise ValueError("TODOCFADOCS") dx = self.to_dask_array() -# name = tokenize(dx.name, location) + # name = tokenize(dx.name, location) dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) self._set_dask(dx, clear=_NONE) @@ -11126,7 +11126,8 @@ def todict(self, optimize_graph=True): `optimize_graph`: `bool` If True, the default, then prior to being converted to a dictionary, the graph is optimised to remove unused - chunks. + chunks. Note that optimising the graph can add a + considerable performance overhead. :Returns: @@ -11137,19 +11138,19 @@ def todict(self, optimize_graph=True): >>> d = cf.Data([1, 2, 3, 4], chunks=2) >>> d.todict() - {('array-7daac373ba27474b6df0af70aab14e49', 0): array([1, 2]), - ('array-7daac373ba27474b6df0af70aab14e49', 1): array([3, 4])} + {('array-2f41b21b4cd29f757a7bfa932bf67832', 0): array([1, 2]), + ('array-2f41b21b4cd29f757a7bfa932bf67832', 1): array([3, 4])} >>> e = d[0] >>> e.todict() - {('getitem-14d8301a3deec45c98569d73f7a2239c', - 0): (, ('array-7daac373ba27474b6df0af70aab14e49', + {('getitem-153fd24082bc067cf438a0e213b41ce6', + 0): (, ('array-2f41b21b4cd29f757a7bfa932bf67832', 0), (slice(0, 1, 1),)), - ('array-7daac373ba27474b6df0af70aab14e49', 0): array([1, 2])} + ('array-2f41b21b4cd29f757a7bfa932bf67832', 0): array([1, 2])} >>> e.todict(optimize_graph=False) - {('array-7daac373ba27474b6df0af70aab14e49', 0): array([1, 2]), - ('array-7daac373ba27474b6df0af70aab14e49', 1): array([3, 4]), - ('getitem-14d8301a3deec45c98569d73f7a2239c', - 0): (, ('array-7daac373ba27474b6df0af70aab14e49', + {('array-2f41b21b4cd29f757a7bfa932bf67832', 0): array([1, 2]), + ('array-2f41b21b4cd29f757a7bfa932bf67832', 1): array([3, 4]), + ('getitem-153fd24082bc067cf438a0e213b41ce6', + 0): (, ('array-2f41b21b4cd29f757a7bfa932bf67832', 0), (slice(0, 1, 1),))} """ diff --git a/cf/test/test_CFA.py b/cf/test/test_CFA.py index 4c6fc2214b..350b6dab65 100644 --- a/cf/test/test_CFA.py +++ b/cf/test/test_CFA.py @@ -408,6 +408,7 @@ def test_CFA_PP(self): f = cf.read("file1.pp")[0] cf.write(f, tmpfile1, cfa=True) + # Check that only the fields have been aggregated nc = netCDF4.Dataset(tmpfile1, "r") for ncvar, var in nc.variables.items(): attrs = var.ncattrs() diff --git a/cf/test/test_Data.py b/cf/test/test_Data.py index 590a099eb0..febc840a7e 100644 --- a/cf/test/test_Data.py +++ b/cf/test/test_Data.py @@ -4622,7 +4622,7 @@ def test_Data_file_location(self): def test_Data_todict(self): """Test Data.todict""" d = cf.Data([1, 2, 3, 4], chunks=2) - key = "array-7daac373ba27474b6df0af70aab14e49" + key = d.to_dask_array().name x = d.todict() self.assertIsInstance(x, dict) diff --git a/cf/test/test_NetCDFArray.py b/cf/test/test_NetCDFArray.py index 3089f00943..16f5f1e84c 100644 --- a/cf/test/test_NetCDFArray.py +++ b/cf/test/test_NetCDFArray.py @@ -2,6 +2,8 @@ import faulthandler import unittest +from dask.base import tokenize + faulthandler.enable() # to debug seg faults and timeouts import cf @@ -73,6 +75,13 @@ def test_NetCDFArray_set_file_location(self): self.assertEqual(b.get_filenames(), a.get_filenames()) self.assertEqual(b.get_addresses(), a.get_addresses()) + def test_NetCDFArray__dask_tokenize__(self): + a = cf.NetCDFArray("/data1/file1", "tas", shape=(12, 2), mask=False) + self.assertEqual(tokenize(a), tokenize(a.copy())) + + b = cf.NetCDFArray("/home/file2", "tas", shape=(12, 2)) + self.assertNotEqual(tokenize(a), tokenize(b)) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) diff --git a/scripts/cfa b/scripts/cfa index ca6b9ef434..80b0b925da 100755 --- a/scripts/cfa +++ b/scripts/cfa @@ -1432,15 +1432,6 @@ Using cf-python library version {cf.__version__} at {library_path}""" else: write_options["cfa"] = True - # elif fmt == "CFA": - # print( - # f"{iam} ERROR: '-f CFA' has been replaced by '-f CFA3' or " - # "'-f CFA4' respectively for netCDF3 classic and netCDF4 CFA " - # "output formats.", - # file=sys.stderr, - # ) - # sys.exit(2) - write_options["fmt"] = fmt if not infiles: From f180defe14fa1edc7efb065fd01218cd2f5a37b0 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Wed, 15 Mar 2023 22:21:44 +0000 Subject: [PATCH 061/141] correct bounds and units of PP hybrid height coordinates --- cf/read_write/um/umread.py | 147 ++++++++++++++++++++----------------- 1 file changed, 78 insertions(+), 69 deletions(-) diff --git a/cf/read_write/um/umread.py b/cf/read_write/um/umread.py index 70943a1cd3..2e4df56b49 100644 --- a/cf/read_write/um/umread.py +++ b/cf/read_write/um/umread.py @@ -38,7 +38,7 @@ _cached_time = {} _cached_ctime = {} _cached_size_1_height_coordinate = {} -_cached_z_reference_coordinate = {} +# _cached_z_reference_coordinate = {} _cached_date2num = {} _cached_model_level_number_coordinate = {} @@ -1130,10 +1130,11 @@ def __str__(self): def _reorder_z_axis(self, indices, z_axis, pmaxes): """Reorder the Z axis `Rec` instances. - :Parmaeters: + :Parameters: indices: `list` - Aggregation axis indices. See `create_data` for details. + Aggregation axis indices. See `create_data` for + details. z_axis: `int` The identifier of the Z axis @@ -1141,18 +1142,24 @@ def _reorder_z_axis(self, indices, z_axis, pmaxes): pmaxes: sequence of `int` The aggregation axes, which include the Z axis. + :Returns: + + `list` + **Examples** >>> _reorder_z_axis([(0, ), (1, )], 0, [0]) [(0, ), (1, )] >>> _reorder_z_axis( - ... [(0, 0, ), (0, 1, ), (1, 0, ), (1, 1, )], + ... [(0, 0, ), + ... (0, 1, ), + ... (1, 0, ), + ... (1, 1, )], ... 1, [0, 1] ... ) [(0, 0, ), (0, 1, ), (1, 0, ), (1, 1, )] - """ indices_new = [] zpos = pmaxes.index(z_axis) @@ -1282,8 +1289,9 @@ def atmosphere_hybrid_height_coordinate(self, axiscode): dc = None else: array = array / toa_height + bounds = bounds / toa_height dc = self.implementation.initialise_DimensionCoordinate() - dc = self.coord_data(dc, array, bounds, units=_Units[""]) + dc = self.coord_data(dc, array, bounds, units=_Units["1"]) self.implementation.set_properties( dc, {"standard_name": "atmosphere_hybrid_height_coordinate"}, @@ -3215,70 +3223,71 @@ def z_coordinate(self, axiscode): return dc - @_manage_log_level_via_verbose_attr - def z_reference_coordinate(self, axiscode): - """Create and return the Z reference coordinates.""" - logger.info( - "Creating Z reference coordinates from BRLEV" - ) # pragma: no cover - - array = np.array( - [rec.real_hdr.item(brlev) for rec in self.z_recs], dtype=float - ) - - LBVC = self.lbvc - atol = self.atol - - key = (axiscode, LBVC, array) - dc = _cached_z_reference_coordinate.get(key, None) - - if dc is not None: - copy = True - else: - if not 128 <= LBVC <= 139: - bounds = [] - for rec in self.z_recs: - BRLEV = rec.real_hdr.item(brlev) - BRSVD1 = rec.real_hdr.item(brsvd1) - - if abs(BRSVD1 - BRLEV) >= atol: - bounds = None - break - - bounds.append((BRLEV, BRSVD1)) - else: - bounds = None - - if bounds: - bounds = np.array((bounds,), dtype=float) - - dc = self.implementation.initialise_DimensionCoordinate() - dc = self.coord_data( - dc, - array, - bounds, - units=_axiscode_to_Units.setdefault(axiscode, None), - ) - dc = self.coord_axis(dc, axiscode) - dc = self.coord_names(dc, axiscode) - - if not dc.get("positive", True): # ppp - dc.flip(inplace=True) - - _cached_z_reference_coordinate[key] = dc - copy = False - - self.implementation.set_dimension_coordinate( - self.field, - dc, - axes=[_axis["z"]], - copy=copy, - autocyclic=_autocyclic_false, - ) - - return dc - +# @_manage_log_level_via_verbose_attr +# def z_reference_coordinate(self, axiscode): +# """Create and return the Z reference coordinates.""" +# logger.info( +# "Creating Z reference coordinates from BRLEV" +# ) # pragma: no cover +# +# array = np.array( +# [rec.real_hdr.item(brlev) for rec in self.z_recs], dtype=float +# ) +# +# LBVC = self.lbvc +# atol = self.atol +# print(99999) +# key = (axiscode, LBVC, array) +# dc = _cached_z_reference_coordinate.get(key) +# +# if dc is not None: +# copy = True +# else: +# if not 128 <= LBVC <= 139: +# bounds = [] +# for rec in self.z_recs: +# BRLEV = rec.real_hdr.item(brlev) +# BRSVD1 = rec.real_hdr.item(brsvd1) +# +# if abs(BRSVD1 - BRLEV) >= atol: +# bounds = None +# break +# +# bounds.append((BRLEV, BRSVD1)) +# else: +# bounds = None +# +# if bounds: +# bounds = np.array((bounds,), dtype=float) +# +# dc = self.implementation.initialise_DimensionCoordinate() +# dc = self.coord_data( +# dc, +# array, +# bounds, +# units=_axiscode_to_Units.setdefault(axiscode, None), +# ) +# dc = self.coord_axis(dc, axiscode) +# dc = self.coord_names(dc, axiscode) +# +# if not dc.get("positive", True): # ppp +# dc.flip(inplace=True) +# +# _cached_z_reference_coordinate[key] = dc +# copy = False +# +# self.implementation.set_dimension_coordinate( +# self.field, +# dc, +# axes=[_axis["z"]], +# copy=copy, +# autocyclic=_autocyclic_false, +# ) +# +# return dc +# +# # _stash2standard_name = {} # # def load_stash2standard_name(table=None, delimiter='!', merge=True): From 3663b151177b992898708690af0fe0af59f8388d Mon Sep 17 00:00:00 2001 From: David Hassell Date: Sat, 18 Mar 2023 09:09:35 +0000 Subject: [PATCH 062/141] dev --- cf/data/fragment/missingfragmentarray.py | 68 ------- cf/data/fragment/mixin/__init__.py | 2 - cf/data/fragment/mixin/fragmentarraymixin.py | 187 +++++++++--------- .../fragment/mixin/fragmentfilearraymixin.py | 172 ---------------- 4 files changed, 94 insertions(+), 335 deletions(-) delete mode 100644 cf/data/fragment/missingfragmentarray.py delete mode 100644 cf/data/fragment/mixin/fragmentfilearraymixin.py diff --git a/cf/data/fragment/missingfragmentarray.py b/cf/data/fragment/missingfragmentarray.py deleted file mode 100644 index ef1f2d763d..0000000000 --- a/cf/data/fragment/missingfragmentarray.py +++ /dev/null @@ -1,68 +0,0 @@ -# from .fullfragmentarray import FullFragmentArray -# -# -# class MissingFragmentArray(FullFragmentArray): -# """A CFA fragment array that is wholly missing data. -# -# .. versionadded:: 3.14.0 -# -# """ -# -# def __init__( -# self, -# dtype=None, -# shape=None, -# aggregated_units=False, -# aggregated_calendar=False, -# units=False, -# calendar=False, -# source=None, -# copy=True, -# ): -# """**Initialisation** -# -# :Parameters: -# -# dtype: `numpy.dtype` -# The data type of the aggregated array. May be `None` -# if the numpy data-type is not known (which can be the -# case for netCDF string types, for example). This may -# differ from the data type of the netCDF fragment -# variable. -# -# shape: `tuple` -# The shape of the fragment within the aggregated -# array. This may differ from the shape of the netCDF -# fragment variable in that the latter may have fewer -# size 1 dimensions. -# -# units: `str` or `None`, optional -# The units of the fragment data. Ignored, as the data -# are all missing values. -# -# calendar: `str` or `None`, optional -# The calendar of the fragment data. Ignored, as the data -# are all missing values. -# -# {{aggregated_units: `str` or `None`, optional}} -# -# {{aggregated_calendar: `str` or `None`, optional}} -# -# {{init source: optional}} -# -# {{init copy: `bool`, optional}} -# -# """ -# import numpy as np -# -# super().__init__( -# fill_value=np.ma.masked, -# dtype=dtype, -# shape=shape, -# aggregated_units=aggregated_units, -# aggregated_calendar=aggregated_calendar, -# units=units, -# calendar=calendar, -# source=source, -# copy=copy, -# ) diff --git a/cf/data/fragment/mixin/__init__.py b/cf/data/fragment/mixin/__init__.py index b02b1f717d..a4a35a1129 100644 --- a/cf/data/fragment/mixin/__init__.py +++ b/cf/data/fragment/mixin/__init__.py @@ -1,3 +1 @@ from .fragmentarraymixin import FragmentArrayMixin - -# from .fragmentfilearraymixin import FragmentFileArrayMixin diff --git a/cf/data/fragment/mixin/fragmentarraymixin.py b/cf/data/fragment/mixin/fragmentarraymixin.py index 3cbba0bbf4..00db6d6ba5 100644 --- a/cf/data/fragment/mixin/fragmentarraymixin.py +++ b/cf/data/fragment/mixin/fragmentarraymixin.py @@ -6,7 +6,7 @@ class FragmentArrayMixin: - """Mixin for a CFA fragment array. + """Mixin class for a CFA fragment array. .. versionadded:: TODOCFAVER @@ -32,6 +32,10 @@ def __getitem__(self, indices): .. versionadded:: TODOCFAVER """ + # TODOACTIVE: modify this the for case when + # super().__getitem__(tuple(indices)) returns a + # dictionary + indices = self._parse_indices(indices) try: @@ -56,18 +60,18 @@ def __getitem__(self, indices): # how many missing dimensions the fragment has, nor # their positions => Get the full fragment array and # then reshape it to the shape of the dask compute - # chunk, assuming that it has the correct size. + # chunk. array = super().__getitem__(Ellipsis) if array.size != self.size: raise ValueError( f"Can't get CFA fragment data from ({self}) when " "the fragment has two or more missing size 1 " - "dimensions whilst also spanning two or more " + "dimensions, whilst also spanning two or more " "dask compute chunks." "\n\n" - "Consider recreating the data with exactly one" - "dask compute chunk per fragment (e.g. set the " - "parameter 'chunks=None' to cf.read)." + "Consider re-creating the data with exactly one " + "dask compute chunk per fragment (e.g. by setting " + "'chunks=None' as a keyword to cf.read)." ) array = array.reshape(self.shape) @@ -75,68 +79,54 @@ def __getitem__(self, indices): array = self._conform_to_aggregated_units(array) return array - def _size_1_axis(self, indices): - """Find the position of a unique size 1 index. + def _conform_to_aggregated_units(self, array): + """Conform the array to have the aggregated units. .. versionadded:: TODOCFAVER - .. seealso:: `_parse_indices`, `__getitem__` - :Parameters: - indices: sequence of index - The array indices to be parsed, as returned by - `_parse_indices`. + array: `numpy.ndarray` or `dict` + The array to be conformed. If *array* is a `dict` with + `numpy` array values then selected values are + conformed. :Returns: - `int` or `None` - The position of the unique size 1 index, or `None` if - there are zero or at least two of them. - - **Examples** + `numpy.ndarray` or `dict` + The conformed array. The returned array may or may not + be the input array updated in-place, depending on its + data type and the nature of its units and the + aggregated units. - >>> a._size_1_axis(([2, 4, 5], slice(0, 1), slice(0, 73))) - 1 - >>> a._size_1_axis(([2, 4, 5], slice(3, 4), slice(0, 73))) - 1 - >>> a._size_1_axis(([2, 4, 5], [0], slice(0, 73))) - 1 - >>> a._size_1_axis(([2, 4, 5], slice(0, 144), slice(0, 73))) - None - >>> a._size_1_axis(([2, 4, 5], slice(3, 7), [0, 1])) - None - >>> a._size_1_axis(([2, 4, 5], slice(0, 1), [0])) - None + If *array* is a `dict` then a dictionary of conformed + arrays is returned. """ - axis = None # Position of unique size 1 index - - n_size_1 = 0 # Number of size 1 indices - for i, (index, n) in enumerate(zip(indices, self.shape)): - try: - x = index.indices(n) - if abs(x[1] - x[0]) == 1: - # Index is a size 1 slice - n_size_1 += 1 - axis = i - except AttributeError: - try: - if index.size == 1: - # Index is a size 1 numpy or dask array - n_size_1 += 1 - axis = i - except AttributeError: - if len(index) == 1: - # Index is a size 1 list - n_size_1 += 1 - axis = i + units = self.Units + if units: + aggregated_units = self.aggregated_Units + if not units.equivalent(aggregated_units): + raise ValueError( + f"Can't convert fragment data with units {units!r} to " + f"have aggregated units {aggregated_units!r}" + ) - if n_size_1 > 1: - # There are two or more size 1 indices - axis = None + if units != aggregated_units: + if isinstance(array, dict): + # 'array' is a dictionary. + raise ValueError( + "TODOACTIVE. This error is notification of an " + "unreplaced placeholder for dealing with active " + "storage reductions on CFA fragments." + ) + else: + # 'array' is a numpy array + array = Units.conform( + array, units, aggregated_units, inplace=True + ) - return axis + return array def _parse_indices(self, indices): """Parse the indices that retrieve the fragment data. @@ -219,57 +209,68 @@ def _parse_indices(self, indices): return indices - def _conform_to_aggregated_units(self, array): - """Conform the array to have the aggregated units. + def _size_1_axis(self, indices): + """Find the position of a unique size 1 index. .. versionadded:: TODOCFAVER + .. seealso:: `_parse_indices`, `__getitem__` + :Parameters: - array: `numpy.ndarray` or `dict` - The array to be conformed. If *array* is a `dict` with - `numpy` array values then each value is conformed. + indices: sequence of index + The array indices to be parsed, as returned by + `_parse_indices`. :Returns: - `numpy.ndarray` or `dict` - The conformed array. The returned array may or may not - be the input array updated in-place, depending on its - data type and the nature of its units and the - aggregated units. + `int` or `None` + The position of the unique size 1 index, or `None` if + there are zero or at least two of them. - If *array* is a `dict` then a dictionary of conformed - arrays is returned. + **Examples** + + >>> a._size_1_axis(([2, 4, 5], slice(0, 1), slice(0, 73))) + 1 + >>> a._size_1_axis(([2, 4, 5], slice(3, 4), slice(0, 73))) + 1 + >>> a._size_1_axis(([2, 4, 5], [0], slice(0, 73))) + 1 + >>> a._size_1_axis(([2, 4, 5], slice(0, 144), slice(0, 73))) + None + >>> a._size_1_axis(([2, 4, 5], slice(3, 7), [0, 1])) + None + >>> a._size_1_axis(([2, 4, 5], slice(0, 1), [0])) + None """ - units = self.Units - if units: - aggregated_units = self.aggregated_Units - if not units.equivalent(aggregated_units): - raise ValueError( - f"Can't convert fragment data with units {units!r} to " - f"have aggregated units {aggregated_units!r}" - ) + axis = None - if units != aggregated_units: - if isinstance(array, dict): - # 'array' is a dictionary - - # TODOACTIVE: '_active_chunk_methds = {}' is a - # placeholder for the real thing - _active_chunk_methds = {} - for key, value in array.items(): - if key in _active_chunk_methds: - array[key] = Units.conform( - value, units, aggregated_units, inplace=True - ) - else: - # 'array' is a numpy array - array = Units.conform( - array, units, aggregated_units, inplace=True - ) + n_size_1 = 0 # Number of size 1 indices + for i, (index, n) in enumerate(zip(indices, self.shape)): + try: + x = index.indices(n) + if abs(x[1] - x[0]) == 1: + # Index is a size 1 slice + n_size_1 += 1 + axis = i + except AttributeError: + try: + if index.size == 1: + # Index is a size 1 numpy or dask array + n_size_1 += 1 + axis = i + except AttributeError: + if len(index) == 1: + # Index is a size 1 list + n_size_1 += 1 + axis = i - return array + if n_size_1 > 1: + # There are two or more size 1 indices + axis = None + + return axis @property def aggregated_Units(self): diff --git a/cf/data/fragment/mixin/fragmentfilearraymixin.py b/cf/data/fragment/mixin/fragmentfilearraymixin.py deleted file mode 100644 index 404c1b1373..0000000000 --- a/cf/data/fragment/mixin/fragmentfilearraymixin.py +++ /dev/null @@ -1,172 +0,0 @@ -# class FragmentFileArrayMixin: -# """Mixin class for a fragment array stored in a file. -# -# .. versionadded:: TODOCFAVER -# -# """ -# -# def del_fragment_location(self, location): -# """TODOCFADOCS -# -# .. versionadded:: TODOCFAVER -# -# :Parameters: -# -# location: `str` -# TODOCFADOCS -# -# :Returns: -# -# `{{class}}` -# TODOCFADOCS -# -# """ -# from os import sep -# from os.path import dirname -# -# a = self.copy() -# location = location.rstrip(sep) -# -# # Note: It is assumed that each existing file name is either -# # an absolute path or a file URI. -# new_filenames = [] -# new_addresses = [] -# for filename, address in zip(a.get_filenames(), a.get_addresses()): -# if dirname(filename) != location: -# new_filenames.append(filename) -# new_addresses.append(address) -# -# if not new_filenames: -# raise ValueError( -# f"Can't remove fragment location {location} when doing so " -# "results in there being no defined fragment files") -# -# a._set_component("filenames", new_filenames, copy=False) -# a._set_component("addresses", new_addresses, copy=False) -# -# return a -# -# def fragment_locations(self): -# """TODOCFADOCS -# -# .. versionadded:: TODOCFAVER -# -# :Parameters: -# -# location: `str` -# TODOCFADOCS -# -# :Returns: -# -# `{{class}}` -# TODOCFADOCS -# -# """ -# from os.path import dirname -# -# # Note: It is assumed that each existing file name is either -# # an absolute path or a file URI. -# return set([dirname(f) for f in self.get_filenames()])# -# -# def get_addresses(self, default=AttributeError()): -# """TODOCFADOCS Return the names of any files containing the data array. -# -# .. versionadded:: TODOCFAVER -# -# :Returns: -# -# `tuple` -# TODOCFADOCS -# -# """ -# return self._get_component("addresses", default) -# -# def get_filenames(self, default=AttributeError()): -# """TODOCFADOCS Return the names of any files containing the data array. -# -# .. versionadded:: TODOCFAVER -# -# :Returns: -# -# `tuple` -# The fragment file names. -# -# """ -# filenames = self._get_component("filenames", None) -# if filenames is None: -# if default is None: -# return -# -# return self._default( -# default, f"{self.__class__.__name__} has no fragement files" -# ) -# -# return filenames -# -# def get_formats(self, default=AttributeError()): -# """Return the format of each fragment file. -# -# .. versionadded:: TODOCFAVER -# -# .. seealso:: `get_filenames`, `get_addresses` -# -# :Returns: -# -# `tuple` -# The fragment file formats. -# -# """ -# return (self.get_format(),) * len(self.get_filenames(default)) -# -# def set_fragment_location(self, location): -# """TODOCFADOCS -# -# .. versionadded:: TODOCFAVER -# -# :Parameters: -# -# location: `str` -# TODOCFADOCS -# -# :Returns: -# -# `{{class}}` -# TODOCFADOCS -# -# """ -# from os import sep -# from os.path import basename, dirname, join -# -# a = self.copy() -# location = location.rstrip(sep) -# -# filenames = a.get_filenames() -# addresses = a.get_addresses() -# -# # Note: It is assumed that each existing file name is either -# # an absolute path or a fully qualified URI. -# new_filenames = [] -# new_addresses = [] -# basenames = [] -# for filename, address in zip(filenames, addresses): -# if dirname(filename) == location: -# continue -# -# basename = basename(filename) -# if basename in basenames: -# continue -# -# basenames.append(filename) -# new_filenames.append(join(location, basename)) -# new_addresses.append(address) -# -# a._set_component( -# "filenames", filenames + tuple(new_filenames), copy=False -# ) -# a._set_component( -# "addresses", -# addresses + tuple(new_addresses), -# copy=False, -# ) -# -# return a From 2e6a3bd04dd83d4fb5afca6bd832bc082f4a3b9d Mon Sep 17 00:00:00 2001 From: David Hassell Date: Sun, 19 Mar 2023 11:41:33 +0000 Subject: [PATCH 063/141] dev --- cf/data/array/abstract/filearray.py | 16 --- cf/data/array/cfanetcdfarray.py | 172 ++++++++++++-------------- cf/data/array/mixin/filearraymixin.py | 13 ++ cf/data/array/netcdfarray.py | 8 +- cf/data/array/umarray.py | 13 -- cf/data/data.py | 1 + cf/read_write/netcdf/netcdfread.py | 123 ++++++++++-------- 7 files changed, 164 insertions(+), 182 deletions(-) diff --git a/cf/data/array/abstract/filearray.py b/cf/data/array/abstract/filearray.py index 47573df4b9..750a7f8f31 100644 --- a/cf/data/array/abstract/filearray.py +++ b/cf/data/array/abstract/filearray.py @@ -73,22 +73,6 @@ def get_address(self): "in subclasses" ) # pragma: no cover - # def get_filename(self): - # """Return the name of the file containing the array. - # - # :Returns: - # - # `str` or `None` - # The filename, or `None` if there isn't one. - # - # **Examples** - # - # >>> a.get_filename() - # 'file.nc' - # - # """ - # return self._get_component("filename", None) - def open(self): """Returns an open dataset containing the data array.""" raise NotImplementedError( diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index 08cbd6eb40..4414ddc41d 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -1,4 +1,5 @@ from copy import deepcopy +from functools import partial from itertools import accumulate, product from os.path import dirname, join from urllib.parse import urlparse @@ -10,6 +11,13 @@ from ..utils import chunk_locations, chunk_positions from .netcdfarray import NetCDFArray +# Store fragment array classes. +_FragmentArray = { + "nc": NetCDFFragmentArray, + "um": UMFragmentArray, + "full": FullFragmentArray, +} + class CFANetCDFArray(NetCDFArray): """A CFA aggregated array stored in a netCDF file. @@ -18,20 +26,6 @@ class CFANetCDFArray(NetCDFArray): """ - def __new__(cls, *args, **kwargs): - """Store fragment array classes. - - .. versionadded:: 3.14.0 - - """ - instance = super().__new__(cls) - instance._FragmentArray = { - "nc": NetCDFFragmentArray, - "um": UMFragmentArray, - "full": FullFragmentArray, - } - return instance - def __init__( self, filename=None, @@ -51,43 +45,21 @@ def __init__( :Parameters: - filename: `str` - The name of the netCDF file containing the array. - - ncvar: `str`, optional - The name of the netCDF variable containing the - array. Required unless *varid* is set. + filename: (sequence of `str`), optional + The name of the CFA-netCDF file containing the + array. If a sequence then it must contain one element. - varid: `int`, optional - The UNIDATA netCDF interface ID of the variable - containing the array. Required if *ncvar* is not set, - ignored if *ncvar* is set. + address: (sequence of `str`0, optional + The name of the CFA-netCDF aggregation variable the + array. If a sequence then it must contain one element. - group: `None` or sequence of `str`, optional - Specify the netCDF4 group to which the netCDF variable - belongs. By default, or if *group* is `None` or an - empty sequence, it assumed to be in the root - group. The last element in the sequence is the name of - the group in which the variable lies, with other - elements naming any parent groups (excluding the root - group). - - *Parameter example:* - To specify that a variable is in the root group: - ``group=()`` or ``group=None`` - - *Parameter example:* - To specify that a variable is in the group '/forecasts': - ``group=['forecasts']`` - - *Parameter example:* - To specify that a variable is in the group - '/forecasts/model2': ``group=['forecasts', 'model2']`` + shape: `tuple` of `int` + The shape of the aggregated data array. dtype: `numpy.dtype` - The data type of the array in the netCDF file. May be - `None` if the numpy data-type is not known (which can be - the case for netCDF string types, for example). + The data type of the aggregated data array. May be + `None` if the numpy data-type is not known (which can + be the case for netCDF string types, for example). mask: `bool` If True (the default) then mask by convention when @@ -119,8 +91,8 @@ def __init__( term: `str`, optional The name of a non-standard aggregation instruction term from which the array is to be created, instead of - the creating the aggregated data in the usual - manner. If set then *address* must be the name of the + the creating the aggregated data in the standard + terms. If set then *address* must be the name of the term's CFA-netCDF aggregation instruction variable, which must be defined on the fragment dimensions and no others. Each value of the aggregation instruction @@ -178,7 +150,8 @@ def __init__( if term is not None: # -------------------------------------------------------- - # This fragment contains a constant value + # This fragment contains a constant value, not file + # locations. # -------------------------------------------------------- term = x[term] fragment_shape = term.shape @@ -194,18 +167,12 @@ def __init__( a = x["address"] f = x["file"] fmt = x["format"] + if not a.ndim: - if f.ndim == ndim: - a = np.full(f.shape, a) - else: - a = np.full(f.shape[:-1], a) + a = np.full(f.shape, a, dtype=a.dtype) if not fmt.ndim: - fmt = fmt.astype("U") - if f.ndim == ndim: - fmt = np.full(f.shape, fmt) - else: - fmt = np.full(f.shape[:-1], fmt) + fmt = np.full(f.shape, fmt, dtype=fmt.dtype) if f.ndim == ndim: fragment_shape = f.shape @@ -225,19 +192,19 @@ def __init__( "location": loc, "filename": f[frag_loc].tolist(), "address": a[frag_loc].tolist(), - "format": fmt[frag_loc].item(), + "format": fmt[frag_loc].tolist(), } for frag_loc, loc in zip(positions, locations) } - # Apply string substitutions to the fragment filename + # Apply string substitutions to the fragment filenames if substitutions: - for xx in aggregated_data.values(): - filename = xx["filename"] + for value in aggregated_data.values(): + filename = value["filename"] for base, sub in substitutions.items(): filename = filename.replace(base, sub) - xx["filename"] = filename + value["filename"] = filename super().__init__( filename=filename, @@ -353,16 +320,12 @@ def __dask_tokenize__(self): .. versionadded:: 3.14.0 """ + out = super().__dask_tokenize__() aggregated_data = self._get_component("instructions", None) if aggregated_data is None: aggregated_data = self.get_aggregated_data(copy=False) - return ( - self.__class__.__name__, - abspath(self.get_filename()), - self.get_address(), - aggregated_data, - ) + return out + (aggregated_data,) def __getitem__(self, indices): """x.__getitem__(indices) <==> x[indices]""" @@ -528,30 +491,30 @@ def get_aggregated_data(self, copy=True): return aggregated_data - def get_FragmentArray(self, fragment_format): - """Return a fragment array class. - - .. versionadded:: 3.14.0 - - :Parameters: - - fragment_format: `str` - The dataset format of the fragment. Either ``'nc'``, - ``'um'``, or ``'full'``. - - :Returns: - - The class for representing a fragment array of the - given format. - - """ - try: - return self._FragmentArray[fragment_format] - except KeyError: - raise ValueError( - "Can't get FragmentArray class for unknown " - f"fragment dataset format: {fragment_format!r}" - ) + # def get_FragmentArray(self, fragment_format): + # """Return a fragment array class. + # + # .. versionadded:: 3.14.0 + # + # :Parameters: + # + # fragment_format: `str` + # The dataset format of the fragment. Either ``'nc'``, + # ``'um'``, or ``'full'``. + # + # :Returns: + # + # The class for representing a fragment array of the + # given format. + # + # """ + # try: + # return _FragmentArray[fragment_format] + # except KeyError: + # raise ValueError( + # "Can't get FragmentArray class for unknown " + # f"fragment dataset format: {fragment_format!r}" + # ) def get_fragmented_dimensions(self): """Get the positions dimension that have two or more fragments. @@ -896,8 +859,14 @@ def to_dask_array(self, chunks="auto"): # Set the chunk sizes for the dask array chunks = self.subarray_shapes(chunks) + if self.get_mask(): + fragment_arrays = _FragmentArray + else: + fragment_arrays = _FragmentArray.copy() + fragment_arrays["nc"] = partial(_FragmentArray["nc"], mask=False) + # Create a FragmentArray for each chunk - get_FragmentArray = self.get_FragmentArray + # get_FragmentArray = self.get_FragmentArray dsk = {} for ( @@ -911,7 +880,18 @@ def to_dask_array(self, chunks="auto"): kwargs = aggregated_data[chunk_location].copy() kwargs.pop("location", None) - FragmentArray = get_FragmentArray(kwargs.pop("format", None)) + # FragmentArray = get_FragmentArray(kwargs.pop("format", None)) + + # FragmentArray = FragmentArray[kwargs.pop("format", None)] + fragment_format = kwargs.pop("format", None) + try: + FragmentArray = fragment_arrays[fragment_format] + except KeyError: + raise ValueError( + "Can't get FragmentArray class for unknown " + f"fragment dataset format: {fragment_format!r}" + ) + fragment = FragmentArray( dtype=dtype, shape=fragment_shape, diff --git a/cf/data/array/mixin/filearraymixin.py b/cf/data/array/mixin/filearraymixin.py index 53aa1adcf5..7c9991f368 100644 --- a/cf/data/array/mixin/filearraymixin.py +++ b/cf/data/array/mixin/filearraymixin.py @@ -13,6 +13,19 @@ class FileArrayMixin: """ + def __dask_tokenize__(self): + """Return a value fully representative of the object. + + .. versionadded:: TODOCFAVER + + """ + return ( + self.__class__, + self.shape, + self.get_filenames(), + self.get_addresses(), + ) + @property def _dask_meta(self): """The metadata for the containing dask array. diff --git a/cf/data/array/netcdfarray.py b/cf/data/array/netcdfarray.py index b20ca6b070..e602cec16f 100644 --- a/cf/data/array/netcdfarray.py +++ b/cf/data/array/netcdfarray.py @@ -17,13 +17,7 @@ def __dask_tokenize__(self): .. versionadded:: TODOCFAVER """ - return ( - self.__class__, - self.shape, - self.get_filenames(), - self.get_addresses(), - self.get_mask(), - ) + return super().__dask_tokenize__() + (self.get_mask(),) def __repr__(self): """Called by the `repr` built-in function. diff --git a/cf/data/array/umarray.py b/cf/data/array/umarray.py index 75b8c1c66e..fa5002af42 100644 --- a/cf/data/array/umarray.py +++ b/cf/data/array/umarray.py @@ -174,19 +174,6 @@ def __init__( # By default, close the UM file after data array access self._set_component("close", True, copy=False) - def __dask_tokenize__(self): - """Return a value fully representative of the object. - - .. versionadded:: TODOCFAVER - - """ - return ( - self.__class__, - self.shape, - self.get_filenames(), - self.get_addresses(), - ) - def __getitem__(self, indices): """Return a subspace of the array. diff --git a/cf/data/data.py b/cf/data/data.py index b9dbf9e673..f6bb5078cf 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -458,6 +458,7 @@ def __init__( dt = True first_value = None + if not dt and array.dtype.kind == "O": kwargs = init_options.get("first_non_missing_value", {}) first_value = first_non_missing_value(array, **kwargs) diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 34c4487c28..4c7558b986 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -3,11 +3,6 @@ import numpy as np from packaging.version import Version -""" -TODOCFA: What about groups/netcdf_flattener? - -""" - class NetCDFRead(cfdm.read_write.netcdf.NetCDFRead): """A container for instantiating Fields from a netCDF dataset. @@ -290,7 +285,7 @@ def _is_cfa_variable(self, ncvar): g = self.read_vars return ( g["cfa"] - and ncvar in g["aggregated_data"] + and ncvar in g["cfa_aggregated_data"] and ncvar not in g["external_variables"] ) @@ -372,8 +367,9 @@ def _customize_read_vars(self): if not g["cfa"]: return - g["aggregated_data"] = {} - g["aggregation_instructions"] = {} + g["cfa_aggregated_data"] = {} + g["cfa_aggregation_instructions"] = {} + g["cfa_file_substitutions"] = {} # ------------------------------------------------------------ # Still here? Then this is a CFA-netCDF file @@ -409,7 +405,7 @@ def _customize_read_vars(self): # Do not create fields/domains from aggregation # instruction variables - parsed_aggregated_data = self._parse_aggregated_data( + parsed_aggregated_data = self._cfa_parse_aggregated_data( ncvar, attributes.get("aggregated_data") ) for term_ncvar in parsed_aggregated_data.values(): @@ -546,42 +542,25 @@ def _create_cfanetcdfarray( # getting set correctly by the CFANetCDFArray instance. kwargs.pop("shape", None) - aggregated_data = g["aggregated_data"][ncvar] + aggregated_data = g["cfa_aggregated_data"][ncvar] standardised_terms = ("location", "file", "address", "format") instructions = [] aggregation_instructions = {} - subs = {} for t, term_ncvar in aggregated_data.items(): if t not in standardised_terms: continue - aggregation_instructions[t] = g["aggregation_instructions"][ + aggregation_instructions[t] = g["cfa_aggregation_instructions"][ term_ncvar ] - instructions.append(f"{t}: {ncvar}") + instructions.append(f"{t}: {term_ncvar}") if t == "file": - # Find URI substitutions that may be stored in the CFA - # file instruction variable's "substitutions" - # attribute - subs = g["variable_attributes"][term_ncvar].get( - "substitutions", {} + kwargs["substitutions"] = g["cfa_file_substitutions"].get( + term_ncvar ) - if subs: - # Convert the string "${base}: value" to the - # dictionary {"${base}": "value"} - s = subs.split() - subs = { - base[:-1]: sub for base, sub in zip(s[::2], s[1::2]) - } - - # Apply user-defined substitutions, which take precedence over - # those defined in the file. - subs.update(g["cfa_options"].get("substitutions", {})) - if subs: - kwargs["substitutions"] = subs kwargs["x"] = aggregation_instructions kwargs["instructions"] = " ".join(sorted(instructions)) @@ -637,11 +616,11 @@ def _create_cfanetcdfarray_term( instructions = [] aggregation_instructions = {} - for t, term_ncvar in g["aggregated_data"][parent_ncvar].items(): + for t, term_ncvar in g["cfa_aggregated_data"][parent_ncvar].items(): if t in ("location", term): - aggregation_instructions[t] = g["aggregation_instructions"][ - term_ncvar - ] + aggregation_instructions[t] = g[ + "cfa_aggregation_instructions" + ][term_ncvar] instructions.append(f"{t}: {ncvar}") kwargs["term"] = term @@ -720,10 +699,7 @@ def _parse_chunks(self, ncvar): return chunks def _customize_field_ancillaries(self, parent_ncvar, f): - """Create field ancillary constructs from CFA terms. - - This method is primarily aimed at providing a customisation - entry point for subclasses. + """Create customised field ancillary constructs. This method currently creates: @@ -732,7 +708,7 @@ def _customize_field_ancillaries(self, parent_ncvar, f): the same domain axes as the parent field construct. Constructs are never created for `Domain` instances. - .. versionadded:: TODODASKCFA + .. versionadded:: TODOCFAVER :Parameters: @@ -771,10 +747,15 @@ def _customize_field_ancillaries(self, parent_ncvar, f): standardised_terms = ("location", "file", "address", "format") out = {} - for term, term_ncvar in g["aggregated_data"][parent_ncvar].items(): + for term, term_ncvar in g["cfa_aggregated_data"][parent_ncvar].items(): if term in standardised_terms: continue + if g["variables"][term_ncvar].ndim != f.ndim: + # Can only create field ancillaries with the same rank + # as the field + continue + # Still here? Then we've got a non-standard aggregation # term from which we can create a field # ancillary construct. @@ -793,6 +774,7 @@ def _customize_field_ancillaries(self, parent_ncvar, f): data = self._create_data( parent_ncvar, anc, cfa_term={term: term_ncvar} ) + self.implementation.set_data(anc, data, copy=False) self.implementation.nc_set_variable(anc, term_ncvar) @@ -806,8 +788,8 @@ def _customize_field_ancillaries(self, parent_ncvar, f): return out - def _parse_aggregated_data(self, ncvar, aggregated_data): - """Parse a CFA-netCDF aggregated_data attribute. + def _cfa_parse_aggregated_data(self, ncvar, aggregated_data): + """Parse a CFA-netCDF ``aggregated_data`` attribute. .. versionadded:: TODOCFAVER @@ -822,13 +804,15 @@ def _parse_aggregated_data(self, ncvar, aggregated_data): :Returns: `dict` + The parsed attribute. """ if not aggregated_data: return {} g = self.read_vars - aggregation_instructions = g["aggregation_instructions"] + aggregation_instructions = g["cfa_aggregation_instructions"] + variable_attributes = g["variable_attributes"] out = {} for x in self._parse_x( @@ -840,15 +824,54 @@ def _parse_aggregated_data(self, ncvar, aggregated_data): term_ncvar = term_ncvar[0] out[term] = term_ncvar - if term_ncvar not in aggregation_instructions: - array = self._conform_array(g["variables"][term_ncvar][...]) - aggregation_instructions[term_ncvar] = array + if term_ncvar in aggregation_instructions: + # Already processed this term + continue + + array = g["variables"][term_ncvar][...] + aggregation_instructions[term_ncvar] = self._cfa_conform_array( + array + ) - g["aggregated_data"][ncvar] = out + if term == "file": + # Find URI substitutions that may be stored in the + # CFA file instruction variable's "substitutions" + # attribute + subs = variable_attributes[term_ncvar].get( + "substitutions", + ) + if subs: + # Convert the string "${base}: value" to the + # dictionary {"${base}": "value"} + s = subs.split() + subs = { + base[:-1]: sub for base, sub in zip(s[::2], s[1::2]) + } + + # Apply user-defined substitutions, which take + # precedence over those defined in the file. + subs.update(g["cfa_options"].get("substitutions", {})) + g["cfa_file_substitutions"][term_ncvar] = subs + + g["cfa_aggregated_data"][ncvar] = out return out - def _conform_array(self, array): - """TODOCFADOCS""" + def _cfa_conform_array(self, array): + """Conform an array so that it is suitable for CFA processing. + + .. versionadded: TODOCFAVER + + :Parameters: + + array: `np.ndarray` + The array to conform. + + :Returns: + + array: `np.ndarray` + The conformed array. + + """ if isinstance(array, str): # string return np.array(array, dtype=f"S{len(array)}").astype("U") From 5bb7e487146bbe331bbfd43d62123654b976c0e0 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 20 Mar 2023 14:02:31 +0000 Subject: [PATCH 064/141] docs --- cf/aggregate.py | 27 +- cf/data/array/cfanetcdfarray.py | 263 +++---------------- cf/data/array/mixin/filearraymixin.py | 44 ++-- cf/data/array/umarray.py | 4 +- cf/data/creation.py | 41 --- cf/data/data.py | 171 ++++++------ cf/data/fragment/mixin/fragmentarraymixin.py | 20 -- cf/data/fragment/umfragmentarray.py | 62 ----- cf/docstring/docstring.py | 31 +++ cf/domain.py | 132 ++++++++-- cf/field.py | 201 +++++++++----- cf/mixin/propertiesdata.py | 160 +++++++---- cf/mixin/propertiesdatabounds.py | 183 +++++++++---- cf/mixin2/cfanetcdf.py | 78 ++---- cf/read_write/netcdf/netcdfwrite.py | 95 ++++--- cf/read_write/read.py | 19 +- cf/read_write/write.py | 28 +- cf/test/test_CFA.py | 20 +- cf/test/test_Data.py | 12 +- cf/test/test_NetCDFArray.py | 49 +++- 20 files changed, 863 insertions(+), 777 deletions(-) diff --git a/cf/aggregate.py b/cf/aggregate.py index 3c4e18679c..907a9e0d67 100644 --- a/cf/aggregate.py +++ b/cf/aggregate.py @@ -209,7 +209,7 @@ def __init__( property itself is deleted from that field. field_ancillaries: (sequence of) `str`, optional - TODOCFADOCS. See `cf.aggregate` for details. + See `cf.aggregate` for details. .. versionadded:: TODOCFAVER @@ -1393,12 +1393,13 @@ def promote_to_auxiliary_coordinate(self, properties): :Parameters: properties: sequence of `str` - TODOCFADOCS + The names of the properties to be promoted. :Returns: `Field` or `Domain` - TODOCFADOCS + The field or domain with the new auxiliary coordinate + constructs. """ f = self.field @@ -1432,9 +1433,9 @@ def promote_to_auxiliary_coordinate(self, properties): def promote_to_field_ancillary(self, properties): """Promote properties to field ancillary constructs. - Each property is converted to a field ancillary construct that - span the same domain axes as the field, and property the is - deleted. + For each input field, each property is converted to a field + ancillary construct that spans the entire domain, with the + constant value of the property. The `Data` of any the new field ancillary construct is marked as a CFA term, meaning that it will only be written to disk if @@ -1450,13 +1451,14 @@ def promote_to_field_ancillary(self, properties): :Parameters: - properties: sequence of `str` - TODOCFADOCS + properties: sequnce of `str` + The names of the properties to be promoted. :Returns: `Field` or `Domain` - The TODOCFADOCS + The field or domain with the new field ancillary + constructs. """ f = self.field @@ -1746,7 +1748,12 @@ def aggregate( `cf.rtol` function. field_ancillaries: (sequence of) `str`, optional - TODOCFADOCS + Create new field ancillary constructs for each input field + which has one or more of the given properties. For each + input field, each property is converted to a field + ancillary construct that spans the entire domain, with the + constant value of the propertyand the property itself is + deleted. .. versionadded:: TODOCFAVER diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index 4414ddc41d..3b695f9adb 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -1,12 +1,9 @@ from copy import deepcopy from functools import partial from itertools import accumulate, product -from os.path import dirname, join -from urllib.parse import urlparse import numpy as np -from ...functions import abspath from ..fragment import FullFragmentArray, NetCDFFragmentArray, UMFragmentArray from ..utils import chunk_locations, chunk_positions from .netcdfarray import NetCDFArray @@ -84,7 +81,10 @@ def __init__( to improve the performance of `__dask_tokenize__`. substitutions: `dict`, optional - TODOCFADOCS + A dictionary whose key/value pairs define text + substitutions to be applied to the fragment file + names. Each key must be specified with the ``${...}`` + syntax, for instance ``{'${base}': 'sub'}``. .. versionadded:: TODOCFAVER @@ -168,31 +168,36 @@ def __init__( f = x["file"] fmt = x["format"] + extra_dimension = f.ndim > ndim + if extra_dimension: + # There is a extra non-fragment dimension + fragment_shape = f.shape[:-1] + else: + fragment_shape = f.shape + if not a.ndim: a = np.full(f.shape, a, dtype=a.dtype) if not fmt.ndim: - fmt = np.full(f.shape, fmt, dtype=fmt.dtype) + fmt = np.full(fragment_shape, fmt, dtype=fmt.dtype) - if f.ndim == ndim: - fragment_shape = f.shape + if extra_dimension: aggregated_data = { frag_loc: { "location": loc, - "filename": f[frag_loc].item(), - "address": a[frag_loc].item(), + "filename": f[frag_loc].tolist(), + "address": a[frag_loc].tolist(), "format": fmt[frag_loc].item(), } for frag_loc, loc in zip(positions, locations) } else: - fragment_shape = f.shape[:-1] aggregated_data = { frag_loc: { "location": loc, - "filename": f[frag_loc].tolist(), - "address": a[frag_loc].tolist(), - "format": fmt[frag_loc].tolist(), + "filename": (f[frag_loc].item(),), + "address": (a[frag_loc].item(),), + "format": fmt[frag_loc].item(), } for frag_loc, loc in zip(positions, locations) } @@ -200,11 +205,14 @@ def __init__( # Apply string substitutions to the fragment filenames if substitutions: for value in aggregated_data.values(): - filename = value["filename"] - for base, sub in substitutions.items(): - filename = filename.replace(base, sub) + filenames2 = [] + for filename in value["filename"]: + for base, sub in substitutions.items(): + filename = filename.replace(base, sub) + + filenames2.append(filename) - value["filename"] = filename + value["filename"] = filenames2 super().__init__( filename=filename, @@ -216,78 +224,6 @@ def __init__( calendar=calendar, copy=copy, ) - - if False: - # CFAPython vesion - from pathlib import PurePath - - from CFAPython import CFAFileFormat - from CFAPython.CFADataset import CFADataset - from CFAPython.CFAExceptions import CFAException - from dask import compute, delayed - - if not isinstance(filename, str): - if len(filename) != 1: - raise ValueError("TODOCFADOCS") - - filename = filename[0] - - cfa = CFADataset(filename, CFAFileFormat.CFANetCDF, "r") - try: - var = cfa.getVar(address) - except CFAException: - raise ValueError( - f"CFA variable {address!r} not found in file " - f"{filename}" - ) - - shape = tuple([d.len for d in var.getDims()]) - - super().__init__( - filename=filename, - address=address, - shape=shape, - dtype=dtype, - mask=mask, - units=units, - calendar=calendar, - copy=copy, - ) - - fragment_shape = tuple(var.getFragDef()) - - parsed_filename = urlparse(filename) - if parsed_filename.scheme in ("file", "http", "https"): - cfa_directory = str(PurePath(filename).parent) - else: - cfa_directory = dirname(abspath(filename)) - - # Note: It is an as-yet-untested hypothesis that creating - # the 'aggregated_data' dictionary for massive - # aggretations (e.g. with O(10e6) fragments) will be - # slow, hence the parallelisation of the process - # with delayed + compute; and that the - # parallelisation overheads won't be noticeable for - # small aggregations (e.g. O(10) fragments). - aggregated_data = {} - compute( - *[ - delayed( - self._set_fragment( - var, - loc, - aggregated_data, - filename, - cfa_directory, - substitutions, - term, - ) - ) - for loc in product(*[range(i) for i in fragment_shape]) - ] - ) - - del cfa else: super().__init__( filename=filename, @@ -331,116 +267,6 @@ def __getitem__(self, indices): """x.__getitem__(indices) <==> x[indices]""" return NotImplemented # pragma: no cover - def _set_fragment( - self, - var, - frag_loc, - aggregated_data, - cfa_filename, - cfa_directory, - substitutions, - term, - ): - """Create a new key/value pair in the *aggregated_data* - dictionary. - - The *aggregated_data* dictionary contains the definitions of - the fragments and the instructions on how to aggregate them, - and is updated in-place. - - .. versionadded:: 3.14.0 - - :Parameters: - - var: `CFAPython.CFAVariable.CFAVariable` - The CFA aggregation variable. - - frag_loc: `tuple` of `int` - The new key, that must be index of the CFA fragment - dimensions, e.g. ``(1, 0, 0, 0)``. - - aggregated_data: `dict` - The aggregated data dictionary to be updated in-place. - - cfa_filename: `str` - TODOCFADOCS - - cfa_directory: `str` - TODOCFADOCS - - .. versionadded:: TODOCFAVER - - substitutions: `dict` - TODOCFADOCS - - .. versionadded:: TODOCFAVER - - term: `str` or `None` - The name of a non-standard aggregation instruction - term from which the array is to be created, instead of - the creating the aggregated data in the usual - manner. Each value of the aggregation instruction - variable will be broadcast across the shape of the - corresponding fragment. - - .. versionadded:: TODOCFAVER - - :Returns: - - `None` - - """ - fragment = var.getFrag(frag_loc=frag_loc) - location = fragment.location - - if term is not None: - # -------------------------------------------------------- - # This fragment contains a constant value - # -------------------------------------------------------- - aggregated_data[frag_loc] = { - "format": "full", - "location": location, - "full_value": fragment.non_standard_term(term), - } - return - - filename = fragment.file - fmt = fragment.format - address = fragment.address - - if address is not None: - # -------------------------------------------------------- - # This fragment is contained in a file - # -------------------------------------------------------- - if filename is None: - # This fragment is contained in the CFA-netCDF file - filename = cfa_filename - fmt = "nc" - else: - # Apply string substitutions to the fragment filename - if substitutions: - for base, sub in substitutions.items(): - filename = filename.replace(base, sub) - - if not urlparse(filename).scheme: - filename = join(cfa_directory, filename) - - aggregated_data[frag_loc] = { - "format": fmt, - "filename": filename, - "address": address, - "location": location, - } - elif filename is None: - # -------------------------------------------------------- - # This fragment contains wholly missing values - # -------------------------------------------------------- - aggregated_data[frag_loc] = { - "format": "full", - "location": location, - "full_value": np.ma.masked, - } - def get_aggregated_data(self, copy=True): """Get the aggregation data dictionary. @@ -491,31 +317,6 @@ def get_aggregated_data(self, copy=True): return aggregated_data - # def get_FragmentArray(self, fragment_format): - # """Return a fragment array class. - # - # .. versionadded:: 3.14.0 - # - # :Parameters: - # - # fragment_format: `str` - # The dataset format of the fragment. Either ``'nc'``, - # ``'um'``, or ``'full'``. - # - # :Returns: - # - # The class for representing a fragment array of the - # given format. - # - # """ - # try: - # return _FragmentArray[fragment_format] - # except KeyError: - # raise ValueError( - # "Can't get FragmentArray class for unknown " - # f"fragment dataset format: {fragment_format!r}" - # ) - def get_fragmented_dimensions(self): """Get the positions dimension that have two or more fragments. @@ -560,14 +361,21 @@ def get_fragment_shape(self): return self._get_component("fragment_shape") def get_term(self, default=ValueError()): - """TODOCFADOCS. + """The CFA aggregation instruction term for the data, if set. .. versionadded:: TODOCFAVER + :Parameters: + + default: optional + Return the value of the *default* parameter if the + term has not been set. If set to an `Exception` + instance then it will be raised instead. + :Returns: `str` - TODOCFADOCS. + The CFA aggregation instruction term name. """ return self._get_component("term", default=default) @@ -880,9 +688,6 @@ def to_dask_array(self, chunks="auto"): kwargs = aggregated_data[chunk_location].copy() kwargs.pop("location", None) - # FragmentArray = get_FragmentArray(kwargs.pop("format", None)) - - # FragmentArray = FragmentArray[kwargs.pop("format", None)] fragment_format = kwargs.pop("format", None) try: FragmentArray = fragment_arrays[fragment_format] diff --git a/cf/data/array/mixin/filearraymixin.py b/cf/data/array/mixin/filearraymixin.py index 7c9991f368..5aa6f493af 100644 --- a/cf/data/array/mixin/filearraymixin.py +++ b/cf/data/array/mixin/filearraymixin.py @@ -56,19 +56,20 @@ def filename(self): ) # pragma: no cover def del_file_location(self, location): - """TODOCFADOCS + """Remove reference to files in the given location. .. versionadded:: TODOCFAVER :Parameters: location: `str` - TODOCFADOCS + The file location to remove. :Returns: `{{class}}` - TODOCFADOCS + A new {{class}} with reference to files in *location* + removed. **Examples** @@ -105,7 +106,10 @@ def del_file_location(self, location): new_addresses.append(address) if not new_filenames: - raise ValueError("TODOCFADOCS") + raise ValueError( + "Can't delete a file location when it results in there " + "being no files" + ) a = self.copy() a._set_component("filename", tuple(new_filenames), copy=False) @@ -113,14 +117,16 @@ def del_file_location(self, location): return a def file_locations(self): - """TODOCFADOCS + """The locations of the files, any of which may contain the data. .. versionadded:: TODOCFAVER :Returns: `tuple` - TODOCFADOCS + The file locations, one for each file, as absolute + paths with no trailing separate pathname component + separator. **Examples** @@ -142,20 +148,24 @@ def file_locations(self): """ return tuple(map(dirname, self.get_filenames())) - def set_file_location(self, location): - """TODOCFADOCS + def add_file_location(self, location): + """Add a new file location. + + All existing files are additionally referenced from the given + location. .. versionadded:: TODOCFAVER :Parameters: location: `str` - TODOCFADOCS + The new location. :Returns: `{{class}}` - TODOCFADOCS + A new {{class}} with all previous files additionally + referenced from *location*. **Examples** @@ -163,9 +173,9 @@ def set_file_location(self, location): ('/data1/file1',) >>> a.get_addresses() ('tas',) - >>> b = a.set_file_location('/home/user') + >>> b = a.add_file_location('/home') >>> b.get_filenames() - ('/data1/file1', '/home/user/file1') + ('/data1/file1', '/home/file1') >>> b.get_addresses() ('tas', 'tas') @@ -173,9 +183,9 @@ def set_file_location(self, location): ('/data1/file1', '/data2/file2',) >>> a.get_addresses() ('tas', 'tas') - >>> b = a.set_file_location('/home/user') + >>> b = a.add_file_location('/home/') >>> b = get_filenames() - ('/data1/file1', '/data2/file2', '/home/user/file1', '/home/user/file2') + ('/data1/file1', '/data2/file2', '/home/file1', '/home/file2') >>> b.get_addresses() ('tas', 'tas', 'tas', 'tas') @@ -183,9 +193,9 @@ def set_file_location(self, location): ('/data1/file1', '/data2/file1',) >>> a.get_addresses() ('tas1', 'tas2') - >>> b = a.set_file_location('/home/user') + >>> b = a.add_file_location('/home/') >>> b.get_filenames() - ('/data1/file1', '/data2/file1', '/home/user/file1') + ('/data1/file1', '/data2/file1', '/home/file1') >>> b.get_addresses() ('tas1', 'tas2', 'tas1') @@ -193,7 +203,7 @@ def set_file_location(self, location): ('/data1/file1', '/data2/file1',) >>> a.get_addresses() ('tas1', 'tas2') - >>> b = a.set_file_location('/data1') + >>> b = a.add_file_location('/data1') >>> b.get_filenames() ('/data1/file1', '/data2/file1') >>> b.get_addresses() diff --git a/cf/data/array/umarray.py b/cf/data/array/umarray.py index fa5002af42..1ac874f5b8 100644 --- a/cf/data/array/umarray.py +++ b/cf/data/array/umarray.py @@ -628,11 +628,11 @@ def get_fmt(self): return self._get_component("fmt", None) def get_format(self): - """TODOCFADOCS + """The format of the files. .. versionadded:: TODOCFAVER - .. seealso:: `get_filename`, `get_address` + .. seealso:: `get_address`, `get_filename`, `get_formats` :Returns: diff --git a/cf/data/creation.py b/cf/data/creation.py index c3b00ed71b..0f2ebb94e7 100644 --- a/cf/data/creation.py +++ b/cf/data/creation.py @@ -3,11 +3,8 @@ import dask.array as da import numpy as np -from cfdm import Array from dask.base import is_dask_collection -from .array.mixin import FileArrayMixin - def to_dask(array, chunks, **from_array_options): """Create a `dask` array. @@ -119,41 +116,3 @@ def generate_axis_identifiers(n): """ return [f"dim{i}" for i in range(n)] - - -def is_file_array(array): - """Whether or not an array is stored on disk. - - .. versionaddedd: TODOCFAVER - - :Parameters: - - array: - TODOCFADOCS - - :Returns: - - `bool` - TODOCFADOCS - - """ - return isinstance(array, FileArrayMixin) - - -def is_abstract_Array_subclass(array): - """Whether or not an array is a type of abstract Array. - - .. versionaddedd: TODOCFAVER - - :Parameters: - - array: - TODOCFADOCS - - :Returns: - - `bool` - TODOCFADOCS - - """ - return isinstance(array, Array) diff --git a/cf/data/data.py b/cf/data/data.py index f6bb5078cf..0cff6d0a86 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -37,16 +37,10 @@ parse_indices, rtol, ) - -# from ..mixin_container import Container from ..mixin2 import CFANetCDF, Container from ..units import Units from .collapse import Collapse -from .creation import ( # is_file_array, - generate_axis_identifiers, - is_abstract_Array_subclass, - to_dask, -) +from .creation import generate_axis_identifiers, to_dask from .dask_utils import ( _da_ma_allclose, cf_contains, @@ -437,11 +431,17 @@ def __init__( except AttributeError: pass - if is_abstract_Array_subclass(array): - # Save the input array in case it's useful later. For - # compressed input arrays this will contain extra information, - # such as a count or index variable. + try: + array.get_filenames() + except AttributeError: + pass + else: self._set_Array(array) + # if is_abstract_Array_subclass(array): + # # Save the input array in case it's useful later. For + # # compressed input arrays this will contain extra information, + # # such as a count or index variable. + # self._set_Array(array) # Cast the input data as a dask array kwargs = init_options.get("from_array", {}) @@ -1257,43 +1257,16 @@ def _cfa_del_write(self): return self._custom.pop("cfa_write", False) def _cfa_set_term(self, value): - """TODOCFADOCS Set the CFA write status of the data to `False`. + """Set the CFA aggregation instruction term status. .. versionadded:: TODOCFAVER .. seealso:: `cfa_get_term`, `cfa_set_term` - .. seealso:: `_del_Array`, `_del_cached_elements`, `_set_dask` - :Parameters: - clear: `int`, optional - Specify which components should be removed. Which - components are removed is determined by sequentially - combining *clear* with the ``_ARRAY`` and ``_CACHE`` - integer-valued contants, using the bitwise AND - operator: - - * If ``clear & _ARRAY`` is non-zero then a source - array is deleted. - - * If ``clear & _CACHE`` is non-zero then cached - element values are deleted. - - By default *clear* is the ``_ALL`` integer-valued - constant, which results in all components being - removed. - - If *clear* is the ``_NONE`` integer-valued constant - then no components are removed. - - To retain a component and remove all others, use - ``_ALL`` with the bitwise OR operator. For instance, - if *clear* is ``_ALL ^ _CACHE`` then the cached - element values will be kept but all other components - will be removed. - - .. versionadded:: 3.14.1 + status: `bool` + The new CFA aggregation instruction term status. :Returns: @@ -1570,16 +1543,9 @@ def _set_cached_elements(self, elements): def _cfa_set_write(self, status): """Set the CFA write status of the data. - This should only be set to `True` if it is known that the dask - array is compatible with the requirements of a CFA-netCDF - aggregation variable (or non-stan... TODOCFADOCS). Conversely, - it should be set to `False` if it that compaibility can not be - guaranteed. - - The CFA status may be set to `True` in `cf.read`. See - `NetCDFRead._create_data` for details. - - If unset then the CFA write status defaults to `False`. + If and only if the CFA write status is True then it may be + possible to write the data as an aggregation variable to a + CFA-netCDF file. .. versionadded:: TODOCFAVER @@ -2502,6 +2468,9 @@ def ceil(self, inplace=False, i=False): def cfa_get_term(self): """The CFA aggregation instruction term status. + If True then the data represents that of a non-standard CFA + aggregation instruction variable. + .. versionadded:: TODOCFAVER .. seealso:: `cfa_set_term` @@ -2522,11 +2491,9 @@ def cfa_get_term(self): def cfa_get_write(self): """The CFA write status of the data. - If and only if the CFA write status is `True`, then this - `Data` instance has the potential to be written to a - CFA-netCDF file as aggregated data. In this case it is the - choice of parameters to the `cf.write` function that - determines if the data is actually written as aggregated data. + If and only if the CFA write status is True then it may be + possible to write the data as an aggregation variable to a + CFA-netCDF file. .. versionadded:: TODOCFAVER @@ -2548,6 +2515,9 @@ def cfa_get_write(self): def cfa_set_term(self, status): """Set the CFA aggregation instruction term status. + If True then the data represents that of a non-standard CFA + aggregation instruction variable. + .. versionadded:: TODOCFAVER .. seealso:: `cfa_get_term` @@ -2573,7 +2543,9 @@ def cfa_set_term(self, status): def cfa_set_write(self, status): """Set the CFA write status of the data. - TODOCFADOCS + If and only if the CFA write status is True then it may be + possible to write the data as an aggregation variable to a + CFA-netCDF file. .. versionadded:: TODOCFAVER @@ -3708,8 +3680,6 @@ def _regrid( The regridded data. """ - from dask import delayed - from .dask_regrid import regrid, regrid_weights shape = self.shape @@ -3980,8 +3950,11 @@ def concatenate(cls, data, axis=0, cull_graph=True, relaxed_units=False): aggregated_data.update(d.cfa_get_aggregated_data({})) substitutions.update(d.cfa_file_substitutions()) - data0.cfa_set_aggregated_data(aggregated_data) - data0.cfa_set_file_substitutions(substitutions) + if aggregated_data: + data0.cfa_set_aggregated_data(aggregated_data) + + if substitutions: + data0.cfa_set_file_substitutions(substitutions) # Set the CFA aggregation instruction term status if data0.cfa_get_term(): @@ -6143,11 +6116,11 @@ def convert_reference_time( def get_filenames(self): """The names of files containing parts of the data array. - Returns the names of any files that are required to deliver - the computed data array. This list may contain fewer names - than the collection of file names that defined the data when - it was first instantiated, as could be the case after the data - has been subspaced. + Returns the names of any files that may be required to deliver + the computed data array. This set may contain fewer names than + the collection of file names that defined the data when it was + first instantiated, as could be the case after the data has + been subspaced. **Implementation** @@ -6303,8 +6276,11 @@ def set_calendar(self, calendar): """ self.Units = Units(self.get_units(default=None), calendar) - def set_file_location(self, location): - """TODOCFADOCS + def add_file_location(self, location): + """Add a new file location in-place. + + All data definitions that reference files are additionally + referenced from the given location. .. versionadded:: TODOCFAVER @@ -6313,25 +6289,27 @@ def set_file_location(self, location): :Parameters: location: `str` - TODOCFADOCS + The new location. :Returns: - `None` + `str` + The new location as an absolute path with no trailing + separate pathname component separator. **Examples** - >>> d.set_file_location('/data/model') + >>> d.add_file_location('/data/model/') + '/data/model' """ - location = abspath(location).rstrip(sep) updated = False dsk = self.todict() for key, a in dsk.items(): try: - dsk[key] = a.set_file_location(location) + dsk[key] = a.add_file_location(location) except AttributeError: # This chunk doesn't contain a file array continue @@ -6340,13 +6318,12 @@ def set_file_location(self, location): # been updated updated = True - if not updated: - raise ValueError("TODOCFADOCS") + if updated: + dx = self.to_dask_array() + dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) + self._set_dask(dx, clear=_NONE) - dx = self.to_dask_array() - # name = tokenize(dx.name, location) - dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) - self._set_dask(dx, clear=_NONE) + return location def set_units(self, value): """Set the units. @@ -8437,15 +8414,20 @@ def soften_mask(self): self.hardmask = False def file_locations(self): - """TODOCFADOCS + """The locations of files containing parts of the data. + + Returns the locations of any files that may be required to + deliver the computed data array. .. versionadded:: TODOCFAVER - .. seealso:: `del_file_location`, `set_file_location` + .. seealso:: `add_file_location`, `del_file_location` :Returns: `set` + The unique file locations as absolute paths with no + trailing separate pathname component separator. **Examples** @@ -9016,18 +8998,23 @@ def change_calendar(self, calendar, inplace=False, i=False): return d def chunk_indices(self): - """TODOCFADOCS ind the shape of each chunk. + """Return indices that define each dask compute chunk. .. versionadded:: TODOCFAVER + .. seealso:: `chunks` + :Returns: - TODOCFAVER + `itertools.product` + An iterator over tuples of indices of the data array. **Examples** >>> d = cf.Data(np.arange(405).reshape(3, 9, 15), ... chunks=((1, 2), (9,), (4, 5, 6))) + >>> d.npartitions + 6 >>> for index in d.chunk_indices(): ... print(index) ... @@ -9420,24 +9407,30 @@ def del_calendar(self, default=ValueError()): return calendar def del_file_location(self, location): - """TODOCFADOCS + """Remove a file location in-place. + + All data definitions that reference files will have references + to files in the given location removed from them. .. versionadded:: TODOCFAVER - .. seealso:: `set_file_location`, `file_locations` + .. seealso:: `add_file_location`, `file_locations` :Parameters: location: `str` - TODOCFADOCS + The file location to remove. :Returns: - `None` + `str` + The removed location as an absolute path with no + trailing separate pathname component separator. **Examples** - >>> d.del_file_location('/data/model') + >>> d.del_file_location('/data/model/') + '/data/model' """ location = abspath(location).rstrip(sep) @@ -9460,6 +9453,8 @@ def del_file_location(self, location): dx = da.Array(dsk, dx.name, dx.chunks, dx.dtype, dx._meta) self._set_dask(dx, clear=_NONE) + return location + def del_units(self, default=ValueError()): """Delete the units. diff --git a/cf/data/fragment/mixin/fragmentarraymixin.py b/cf/data/fragment/mixin/fragmentarraymixin.py index 00db6d6ba5..1e5107b639 100644 --- a/cf/data/fragment/mixin/fragmentarraymixin.py +++ b/cf/data/fragment/mixin/fragmentarraymixin.py @@ -288,26 +288,6 @@ def aggregated_Units(self): self.get_aggregated_units(), self.get_aggregated_calendar(None) ) - def add_fragment_location(self, location, inplace=False): - """TODOCFADOCS - - .. versionadded:: TODOCFAVER - - :Parameters: - - location: `str` - TODOCFADOCS - - :Returns: - - TODOCFADOCS - - """ - raise ValueError( - "Can't add a file location to fragment represented by a " - f"{self.__class__.__name__} instance" - ) - def get_aggregated_calendar(self, default=ValueError()): """The calendar of the aggregated array. diff --git a/cf/data/fragment/umfragmentarray.py b/cf/data/fragment/umfragmentarray.py index 2858cb6afc..f47e212977 100644 --- a/cf/data/fragment/umfragmentarray.py +++ b/cf/data/fragment/umfragmentarray.py @@ -98,65 +98,3 @@ def __init__( self._set_component( "aggregated_calendar", aggregated_calendar, copy=False ) - - -# def get_formats(self): -# """TODOCFADOCS -# -# .. versionadded:: TODOCFAVER -# -# .. seealso:: `get_filenames`, `get_addresses` -# -# :Returns: -# -# `tuple` -# -# """ -# return ("um",) * len(self.get_filenames()) -# -# def open(self): -# """Returns an open dataset containing the data array. -# -# When multiple fragment files have been provided an attempt is -# made to open each one, in arbitrary order, and the -# `umfile_lib.File` is returned from the first success. -# -# .. versionadded:: TODOCFAVER -# -# :Returns: -# -# `umfile_lib.File` -# -# """ -# # Loop round the files, returning as soon as we find one that -# # works. -# filenames = self.get_filenames() -# for filename, address in zip(filenames, self.get_addresses()): -# url = urlparse(filename) -# if url.scheme == "file": -# # Convert file URI into an absolute path -# filename = url.path -# -# try: -# f = File( -# path=filename, -# byte_ordering=None, -# word_size=None, -# fmt=None, -# ) -# except FileNotFoundError: -# continue -# except Exception as error: -# try: -# f.close_fd() -# except Exception: -# pass -# -# raise Exception(f"{error}: {filename}") -# -# self._set_component("header_offset", address, copy=False) -# return f -# -# raise FileNotFoundError( -# f"No such PP or UM fragment files: {filenames}" -# ) diff --git a/cf/docstring/docstring.py b/cf/docstring/docstring.py index c07262f3fc..243e8d8096 100644 --- a/cf/docstring/docstring.py +++ b/cf/docstring/docstring.py @@ -491,6 +491,22 @@ culled. See `dask.optimization.cull` for details. .. versionadded:: 3.14.0""", + # cfa substitutions + "{{cfa substitutions: `dict`}}": """substitutions: `dict` + The substitution definitions in a dictionary whose + key/value pairs are the file name parts to be + substituted and their corresponding substitution text. + + Each substitution definition may be specified with or + without the ``${...}`` syntax. For instance, the + following are equivalent: ``{'base': 'sub'}``, + ``{'${base}': 'sub'}``.""", + # cfa base + "{{cfa base: `str`}}": """base: `str` + The substitution definition to be removed. May be + specified with or without the ``${...}`` syntax. For + instance, the following are equivalent: ``'base'`` and + ``'${base}'``.""", # ---------------------------------------------------------------- # Method description substitutions (4 levels of indentation) # ---------------------------------------------------------------- @@ -523,4 +539,19 @@ checked. The coordinates check will be carried out, however, if the *check_coordinates* parameter is True.""", + # Returns cfa_file_substitutions + "{{Returns cfa_file_substitutions}}": """The CFA-netCDF file name substitutions in a dictionary + whose key/value pairs are the file name parts to be + substituted and their corresponding substitution + text.""", + # Returns cfa_clear_file_substitutions + "{{Returns cfa_clear_file_substitutions}}": """The removed CFA-netCDF file name substitutions in a + dictionary whose key/value pairs are the file name + parts to be substituted and their corresponding + substitution text.""", + # Returns cfa_clear_file_substitutions + "{{Returns cfa_del_file_substitution}}": """ + The removed CFA-netCDF file name substitution. If the + substitution was not defined then an empty dictionary + is returned.""", } diff --git a/cf/domain.py b/cf/domain.py index ad3898c9d1..56ad16081c 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -1,4 +1,5 @@ from math import prod +from os import sep import cfdm @@ -12,6 +13,7 @@ from .functions import ( _DEPRECATION_ERROR_ARG, _DEPRECATION_ERROR_METHOD, + abspath, indices_shape, parse_indices, ) @@ -130,47 +132,58 @@ def size(self): [domain_axis.get_size(0) for domain_axis in domain_axes.values()] ) - def cfa_add_fragment_location( + def add_file_location( self, location, ): - """TODOCFADOCS + """Add a new file location in-place. + + All data definitions that reference files are additionally + referenced from the given location. .. versionadded:: TODOCFAVER + .. seealso:: `del_file_location`, `file_locations` + :Parameters: location: `str` - TODOCFADOCS + The new location. :Returns: - `None` + `str` + The new location as an absolute path with no trailing + separate pathname component separator. **Examples** - >>> f.cfa_add_fragment_location('/data/model') + >>> f.add_file_location('/data/model/') + '/data/model' """ + location = abspath(location).rstrip(sep) + for c in self.constructs.filter_by_data(todict=True).values(): - c.cfa_add_fragment_location( - location, - ) + c.add_file_location(location) + + return location def cfa_clear_file_substitutions( self, ): - """TODOCFADOCS + """Remove all of the CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER :Returns: `dict` + {{Returns cfa_clear_file_substitutions}} **Examples** - >>> f.cfa_clear_file_substitutions() + >>> d.cfa_clear_file_substitutions() {} """ @@ -180,24 +193,25 @@ def cfa_clear_file_substitutions( return out - def cfa_get_file_substitutions(self): - """TODOCFADOCS + def cfa_file_substitutions(self): + """Return the CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER :Returns: `dict` + {{Returns cfa_file_substitutions}} **Examples** - >>> f.cfa_get_file_substitutions() + >>> d.cfa_file_substitutions() {} """ out = {} for c in self.constructs.filter_by_data(todict=True).values(): - out.update(c.cfa_get_file_substitutions()) + out.update(c.cfa_file_substitutions()) return out @@ -205,22 +219,23 @@ def cfa_del_file_substitution( self, base, ): - """TODOCFADOCS + """Remove a CFA-netCDF file name substitution. .. versionadded:: TODOCFAVER :Parameters: base: `str` - TODOCFADOCS + {{cfa base: `str`}} :Returns: - `None` + `dict` + {{Returns cfa_del_file_substitution}} **Examples** - >>> f.cfa_del_file_substitution('base', '/data/model') + >>> f.cfa_del_file_substitution('base') """ for c in self.constructs.filter_by_data(todict=True).values(): @@ -230,19 +245,15 @@ def cfa_del_file_substitution( def cfa_set_file_substitutions( self, - value, + substitutions, ): - """TODOCFADOCS + """Set CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER :Parameters: - base: `str` - TODOCFADOCS - - sub: `str` - TODOCFADOCS + {{cfa substitutions: `dict`}} :Returns: @@ -250,11 +261,11 @@ def cfa_set_file_substitutions( **Examples** - >>> f.cfa_set_file_substitution({'base': '/data/model'}) + >>> d.cfa_set_file_substitutions({'base': '/data/model'}) """ for c in self.constructs.filter_by_data(todict=True).values(): - c.cfa_set_file_substitutions(value) + c.cfa_set_file_substitutions(substitutions) def close(self): """Close all files referenced by the domain construct. @@ -282,6 +293,73 @@ def close(self): removed_at="5.0.0", ) # pragma: no cover + def del_file_location( + self, + location, + ): + """Remove a file location in-place. + + All data definitions that reference files will have references + to files in the given location removed from them. + + .. versionadded:: TODOCFAVER + + .. seealso:: `add_file_location`, `file_locations` + + :Parameters: + + location: `str` + The file location to remove. + + :Returns: + + `str` + The removed location as an absolute path with no + trailing separate pathname component separator. + + **Examples** + + >>> d.del_file_location('/data/model/') + '/data/model' + + """ + location = abspath(location).rstrip(sep) + + for c in self.constructs.filter_by_data(todict=True).values(): + c.del_file_location(location) + + return location + + def file_locations( + self, + ): + """The locations of files containing parts of the data. + + Returns the locations of any files that may be required to + deliver the computed data array. + + .. versionadded:: TODOCFAVER + + .. seealso:: `add_file_location`, `del_file_location` + + :Returns: + + `set` + The unique file locations as absolute paths with no + trailing separate pathname component separator. + + **Examples** + + >>> d.file_locations() + {'/home/data1', 'file:///data2'} + + """ + out = set() + for c in self.constructs.filter_by_data(todict=True).values(): + out.update(c.file_locations()) + + return out + @_inplace_enabled(default=False) def flip(self, axes=None, inplace=False): """Flip (reverse the direction of) domain axes. diff --git a/cf/field.py b/cf/field.py index ef01bd2202..afa8b82e7a 100644 --- a/cf/field.py +++ b/cf/field.py @@ -2,6 +2,7 @@ from collections import namedtuple from functools import reduce from operator import mul as operator_mul +from os import sep import cfdm import numpy as np @@ -46,6 +47,7 @@ _DEPRECATION_ERROR_METHOD, DeprecationError, _section, + abspath, flat, parse_indices, ) @@ -3643,26 +3645,28 @@ def cell_area( return w - def cfa_get_file_substitutions(self, constructs=True): - """TODOCFADOCS + def cfa_clear_file_substitutions( + self, + ): + """Remove all of the CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER :Returns: `dict` + {{Returns cfa_clear_file_substitutions}} **Examples** - >>> f.cfa_get_file_substitutions() + >>> f.cfa_clear_file_substitutions() {} """ - out = super().cfa_get_file_substitutions() + out = super().cfa_clear_file_substitution() - if constructs: - for c in self.constructs.filter_by_data(todict=True).values(): - out.update(c.cfa_set_file_substitution()) + for c in self.constructs.filter_by_data(todict=True).values(): + out.update(c.cfa_clear_file_substitutions()) return out @@ -3671,29 +3675,26 @@ def cfa_del_file_substitution( base, constructs=True, ): - """TODOCFADOCS + """Remove a CFA-netCDF file name substitution. .. versionadded:: TODOCFAVER :Parameters: - base: `str` - TODOCFADOCS + {{cfa base: `str`}} - constructs: `bool` - If True, the default, then metadata constructs are - also transposed so that their axes are in the same - relative order as in the transposed data array of the - field. By default metadata constructs are not - altered. TODOCFADOCS + constructs: `bool`, optional + If True (the default) then metadata constructs also + have the file substitutions removed from them. :Returns: - `None` + `dict` + {{Returns cfa_del_file_substitution}} **Examples** - >>> f.cfa_del_file_substitution('base', '/data/model') + >>> f.cfa_del_file_substitution('base') """ super().cfa_del_file_substitution(base) @@ -3702,87 +3703,90 @@ def cfa_del_file_substitution( for c in self.constructs.filter_by_data(todict=True).values(): c.cfa_del_file_substitution(base) - def cfa_del_fragment_location( - self, - location, - constructs=True, - ): - """TODOCFADOCS + def cfa_file_substitutions(self, constructs=True): + """Return the CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER - :Parameters: - - location: `str` - TODOCFADOCS - :Returns: - `None` + `dict` + {{Returns cfa_file_substitutions}} **Examples** - >>> f.cfa_set_fragment_location('/data/model') + >>> f.cfa_file_substitutions() + {} """ - super().cfa_del_fragment_location(location) + out = super().cfa_file_substitutions() if constructs: for c in self.constructs.filter_by_data(todict=True).values(): - c.cfa_del_fragment_location(location) + out.update(c.cfa_file_substitutions()) - def cfa_set_file_substitutions( + return out + + def del_file_location( self, - value, + location, constructs=True, ): - """TODOCFADOCS + """Remove a file location in-place. + + All data definitions that reference files will have references + to files in the given location removed from them. .. versionadded:: TODOCFAVER - :Parameters: + .. seealso:: `add_file_location`, `file_locations` - base: `str` - TODOCFADOCS + :Parameters: - sub: `str` - TODOCFADOCS + location: `str` + The file location to remove. - constructs: `bool` - If True, the default, then metadata constructs are - also transposed so that their axes are in the same - relative order as in the transposed data array of the - field. By default metadata constructs are not - altered. TODOCFADOCS + constructs: `bool`, optional + If True (the default) then metadata constructs also + have the new file location removed from them. :Returns: - `None` + `str` + The removed location as an absolute path with no + trailing separate pathname component separator. **Examples** - >>> f.cfa_set_file_substitution({'base': '/data/model'}) + >>> d.del_file_location('/data/model/') + '/data/model' """ - super().cfa_set_file_substitutions(value) + location = abspath(location).rstrip(sep) + super().del_file_location(location) if constructs: for c in self.constructs.filter_by_data(todict=True).values(): - c.cfa_set_file_substitutions(value) + c.del_file_location(location, inplace=True) + + return location - def cfa_set_fragment_location( + def cfa_set_file_substitutions( self, - location, + substitutions, constructs=True, ): - """TODOCFADOCS + """Set CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER :Parameters: - location: `str` - TODOCFADOCS + {{cfa substitutions: `dict`}} + + constructs: `bool`, optional + If True (the default) then metadata constructs also + have the file substitutions set on them. :Returns: @@ -3790,14 +3794,14 @@ def cfa_set_fragment_location( **Examples** - >>> f.cfa_set_fragment_location('/data/model') + >>> f.cfa_set_file_substitutions({'base': '/data/model'}) """ - super().cfa_set_fragment_location(location) + super().cfa_set_file_substitutions(substitutions) if constructs: for c in self.constructs.filter_by_data(todict=True).values(): - c.cfa_set_fragment_location(location) + c.cfa_set_file_substitutions(substitutions) def radius(self, default=None): """Return the radius of a latitude-longitude plane defined in @@ -11594,6 +11598,41 @@ def cumsum( return f + def file_locations(self, constructs=True): + """The locations of files containing parts of the data. + + Returns the locations of any files that may be required to + deliver the computed data array. + + .. versionadded:: TODOCFAVER + + .. seealso:: `add_file_location`, `del_file_location` + + :Parameters: + + constructs: `bool`, optional + If True (the default) then the file locations from + metadata constructs are also returned. + + :Returns: + + `set` + The unique file locations as absolute paths with no + trailing separate pathname component separator. + + **Examples** + + >>> f.file_locations() + {'/home/data1', 'file:///data2'} + + """ + out = super().file_locations() + if constructs: + for c in self.constructs.filter_by_data(todict=True).values(): + out.update(c.file_locations()) + + return out + @_deprecated_kwarg_check("i", version="3.0.0", removed_at="4.0.0") @_inplace_enabled(default=False) def flip(self, axes=None, inplace=False, i=False, **kwargs): @@ -11937,7 +11976,7 @@ def transpose( provided, or if no axes are specified then the axis order is reversed. - constructs: `bool` + constructs: `bool`, optional If True then metadata constructs are also transposed so that their axes are in the same relative order as in the transposed data array of the field. By default metadata @@ -13987,6 +14026,48 @@ def subspace(self): """ return SubspaceField(self) + def add_file_location( + self, + location, + constructs=True, + ): + """Add a new file location in-place. + + All data definitions that reference files are additionally + referenced from the given location. + + .. versionadded:: TODOCFAVER + + .. seealso:: `del_file_location`, `file_locations` + + :Parameters: + + location: `str` + The new location. + + constructs: `bool`, optional + If True (the default) then metadata constructs also + have the new file location added to them. + + :Returns: + + `str` + The new location as an absolute path with no trailing + separate pathname component separator. + + **Examples** + + >>> f.add_file_location('/data/model/') + '/data/model' + + """ + location = super().add_file_location(location) + if constructs: + for c in self.constructs.filter_by_data(todict=True).values(): + c.add_file_location(location) + + return location + def section(self, axes=None, stop=None, min_step=1, **kwargs): """Return a FieldList of m dimensional sections of a Field of n dimensions, where M <= N. diff --git a/cf/mixin/propertiesdata.py b/cf/mixin/propertiesdata.py index 6b9d2a7ab6..f3bf4a492d 100644 --- a/cf/mixin/propertiesdata.py +++ b/cf/mixin/propertiesdata.py @@ -1,5 +1,6 @@ import logging from itertools import chain +from os import sep import numpy as np @@ -16,6 +17,7 @@ _DEPRECATION_ERROR_ATTRIBUTE, _DEPRECATION_ERROR_KWARGS, _DEPRECATION_ERROR_METHOD, + abspath, default_netCDF_fillvals, ) from ..functions import equivalent as cf_equivalent @@ -1593,6 +1595,39 @@ def units(self): self.Units = Units(None, getattr(self, "calendar", None)) + def add_file_location(self, location): + """Add a new file location in-place. + + All data definitions that reference files are additionally + referenced from the given location. + + .. versionadded:: TODOCFAVER + + .. seealso:: `del_file_location`, `file_locations` + + :Parameters: + + location: `str` + The new location. + + :Returns: + + `str` + The new location as an absolute path with no trailing + separate pathname component separator. + + **Examples** + + >>> d.add_file_location('/data/model/') + '/data/model' + + """ + data = self.get_data(None, _fill_value=False, _units=False) + if data is not None: + return data.add_file_location(location) + + return abspath(location).rstrip(sep) + @_deprecated_kwarg_check("i", version="3.0.0", removed_at="4.0.0") @_inplace_enabled(default=False) def mask_invalid(self, inplace=False, i=False): @@ -2473,18 +2508,14 @@ def ceil(self, inplace=False, i=False): delete_props=True, ) - def cfa_set_file_substitutions(self, value): - """TODOCFADOCS + def cfa_set_file_substitutions(self, substitutions): + """Set CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER :Parameters: - base: `str` - TODOCFADOCS - - sub: `str` - TODOCFADOCS + {{cfa substitutions: `dict`}} :Returns: @@ -2492,16 +2523,16 @@ def cfa_set_file_substitutions(self, value): **Examples** - >>> f.cfa_set_file_substitution('base', '/data/model') + >>> f.cfa_set_file_substitutions({'base', '/data/model'}) """ data = self.get_data(None, _fill_value=False, _units=False) if data is not None: - data.cfa_set_file_substitutions(value) + data.cfa_set_file_substitutions(substitutions) @_inplace_enabled(default=False) def cfa_clear_file_substitutions(self, inplace=False): - """TODOCFADOCS + """Remove all of the CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER @@ -2512,6 +2543,7 @@ def cfa_clear_file_substitutions(self, inplace=False): :Returns: `dict` + {{Returns cfa_clear_file_substitutions}} **Examples** @@ -2529,18 +2561,14 @@ def cfa_del_file_substitution( self, base, ): - """TODOCFADOCS + """Remove a CFA-netCDF file name substitution. .. versionadded:: TODOCFAVER :Parameters: - base: `str` - TODOCFADOCS - - :Returns: - - `None` + `dict` + {{Returns cfa_del_file_substitution}} **Examples** @@ -2549,55 +2577,30 @@ def cfa_del_file_substitution( """ data = self.get_data(None, _fill_value=False, _units=False) if data is not None: - data.cfa_del_file_substitutions(base) + data.cfa_del_file_substitution(base) - def cfa_get_file_substitutions( + def cfa_file_substitutions( self, ): - """TODOCFADOCS + """Return the CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER :Returns: `dict` + {{Returns cfa_file_substitutions}} **Examples** - >>> g = f.cfa_get_file_substitutions() + >>> g = f.cfa_file_substitutions() """ data = self.get_data(None) if data is None: return {} - return data.cfa_get_file_substitutions({}) - - def cfa_add_fragment_location( - self, - location, - ): - """TODOCFADOCS - - .. versionadded:: TODOCFAVER - - :Parameters: - - location: `str` - TODOCFADOCS - - :Returns: - - `None` - - **Examples** - - >>> f.cfa_add_fragment_location('/data/model') - - """ - data = self.get_data(None, _fill_value=False, _units=False) - if data is not None: - data.cfa_add_fragment_location(location) + return data.cfa_file_substitutions({}) def chunk(self, chunksize=None): """Partition the data array. @@ -3025,6 +3028,39 @@ def datum(self, *index): return data.datum(*index) + def del_file_location(self, location): + """Remove a file location in-place. + + All data definitions that reference files will have references + to files in the given location removed from them. + + .. versionadded:: TODOCFAVER + + .. seealso:: `add_file_location`, `file_locations` + + :Parameters: + + location: `str` + The file location to remove. + + :Returns: + + `str` + The removed location as an absolute path with no + trailing separate pathname component separator. + + **Examples** + + >>> f.del_file_location('/data/model/') + '/data/model' + + """ + data = self.get_data(None, _fill_value=False, _units=False) + if data is not None: + return data.del_file_location(location) + + return abspath(location).rstrip(sep) + @_manage_log_level_via_verbosity def equals( self, @@ -3352,6 +3388,34 @@ def convert_reference_time( calendar_years=calendar_years, ) + def file_locations(self): + """The locations of files containing parts of the data. + + Returns the locations of any files that may be required to + deliver the computed data array. + + .. versionadded:: TODOCFAVER + + .. seealso:: `add_file_location`, `del_file_location` + + :Returns: + + `set` + The unique file locations as absolute paths with no + trailing separate pathname component separator. + + **Examples** + + >>> d.file_locations() + {'/home/data1', 'file:///data2'} + + """ + data = self.get_data(None, _fill_value=False, _units=False) + if data is not None: + return data.file_locations() + + return set() + @_inplace_enabled(default=False) def flatten(self, axes=None, inplace=False): """Flatten axes of the data. diff --git a/cf/mixin/propertiesdatabounds.py b/cf/mixin/propertiesdatabounds.py index d15000e84c..ea80fe9353 100644 --- a/cf/mixin/propertiesdatabounds.py +++ b/cf/mixin/propertiesdatabounds.py @@ -1135,6 +1135,45 @@ def dtype(self): if data is not None: del data.dtype + def add_file_location(self, location): + """Add a new file location in-place. + + All data definitions that reference files are additionally + referenced from the given location. + + .. versionadded:: TODOCFAVER + + .. seealso:: `del_file_location`, `file_locations` + + :Parameters: + + location: `str` + The new location. + + :Returns: + + `str` + The new location as an absolute path with no trailing + separate pathname component separator. + + **Examples** + + >>> d.add_file_location('/data/model/') + '/data/model' + + """ + location = super().add_file_location(location) + + bounds = self.get_bounds(None) + if bounds is not None: + bounds.add_file_location(location) + + interior_ring = self.get_interior_ring(None) + if interior_ring is not None: + interior_ring.add_file_location(location) + + return location + @_deprecated_kwarg_check("i", version="3.0.0", removed_at="4.0.0") @_inplace_enabled(default=False) def ceil(self, bounds=True, inplace=False, i=False): @@ -1182,48 +1221,17 @@ def ceil(self, bounds=True, inplace=False, i=False): i=i, ) - def cfa_set_file_substitutions(self, value): - """TODOCFADOCS - - .. versionadded:: TODOCFAVER - - :Parameters: - - base: `str` - TODOCFADOCS - - sub: `str` - TODOCFADOCS - - :Returns: - - `None` - - **Examples** - - >>> c.cfa_add_file_substitution('base', '/data/model') - - """ - super().cfa_set_file_substitutions(value) - - bounds = self.get_bounds(None) - if bounds is not None: - bounds.cfa_set_file_substitutions(value) - - interior_ring = self.get_interior_ring(None) - if interior_ring is not None: - interior_ring.cfa_set_file_substitutions(value) - def cfa_clear_file_substitutions( self, ): - """TODOCFADOCS + """Remove all of the CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER :Returns: `dict` + {{Returns cfa_clear_file_substitutions}} **Examples** @@ -1244,18 +1252,18 @@ def cfa_clear_file_substitutions( return out def cfa_del_file_substitution(self, base): - """TODOCFADOCS + """Remove a CFA-netCDF file name substitution. .. versionadded:: TODOCFAVER :Parameters: - base: `str` - TODOCFADOCS + {{cfa base: `str`}} :Returns: - `None` + `dict` + {{Returns cfa_del_file_substitution}} **Examples** @@ -1272,42 +1280,42 @@ def cfa_del_file_substitution(self, base): if interior_ring is not None: interior_ring.cfa_del_file_substitution(base) - def cfa_get_file_substitutions(self): - """TODOCFADOCS + def cfa_file_substitutions(self): + """Return the CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER :Returns: `dict` + {{Returns cfa_file_substitutions}} **Examples** - >>> c.cfa_get_file_substitutions() + >>> c.cfa_file_substitutions() {} """ - out = super().cfa_get_file_substitutions() + out = super().cfa_file_substitutions() bounds = self.get_bounds(None) if bounds is not None: - out.update(bounds.cfa_get_file_substitutions({})) + out.update(bounds.cfa_file_substitutions({})) interior_ring = self.get_interior_ring(None) if interior_ring is not None: - out.update(interior_ring.cfa_get_file_substitutions({})) + out.update(interior_ring.cfa_file_substitutions({})) return out - def cfa_add_fragment_location(self, location): - """TODOCFADOCS + def cfa_set_file_substitutions(self, substitutions): + """Set CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER :Parameters: - location: `str` - TODOCFADOCS + {{cfa substitutions: `dict`}} :Returns: @@ -1315,18 +1323,18 @@ def cfa_add_fragment_location(self, location): **Examples** - >>> c.cfa_add_fragment_location('/data/model') + >>> c.cfa_add_file_substitutions({'base', '/data/model'}) """ - super().cfa_add_fragment_location(location) + super().cfa_set_file_substitutions(substitutions) bounds = self.get_bounds(None) if bounds is not None: - bounds.cfa_add_fragment_location(location) + bounds.cfa_set_file_substitutions(substitutions) interior_ring = self.get_interior_ring(None) if interior_ring is not None: - interior_ring.cfa_add_fragment_location(location) + interior_ring.cfa_set_file_substitutions(substitutions) def chunk(self, chunksize=None): """Partition the data array. @@ -2045,6 +2053,40 @@ def get_property(self, prop, default=ValueError(), bounds=False): return super().get_property(prop, default) + def file_locations(self): + """The locations of files containing parts of the data. + + Returns the locations of any files that may be required to + deliver the computed data array. + + .. versionadded:: TODOCFAVER + + .. seealso:: `add_file_location`, `del_file_location` + + :Returns: + + `set` + The unique file locations as absolute paths with no + trailing separate pathname component separator. + + **Examples** + + >>> d.file_locations() + {'/home/data1', 'file:///data2'} + + """ + out = super().file_locations() + + bounds = self.get_bounds(None) + if bounds is not None: + out.update(bounds.file_locations()) + + interior_ring = self.get_interior_ring(None) + if interior_ring is not None: + out.update(interior_ring.file_locations()) + + return out + @_inplace_enabled(default=False) def flatten(self, axes=None, inplace=False): """Flatten axes of the data. @@ -2116,6 +2158,45 @@ def flatten(self, axes=None, inplace=False): return v + def del_file_location(self, location): + """Remove a file location in-place. + + All data definitions that reference files will have references + to files in the given location removed from them. + + .. versionadded:: TODOCFAVER + + .. seealso:: `add_file_location`, `file_locations` + + :Parameters: + + location: `str` + The file location to remove. + + :Returns: + + `str` + The removed location as an absolute path with no + trailing separate pathname component separator. + + **Examples** + + >>> c.del_file_location('/data/model/') + '/data/model' + + """ + location = super().del_file_location(location) + + bounds = self.get_bounds(None) + if bounds is not None: + bounds.del_file_location(location) + + interior_ring = self.get_interior_ring(None) + if interior_ring is not None: + interior_ring.del_file_location(location) + + return location + @_deprecated_kwarg_check("i", version="3.0.0", removed_at="4.0.0") @_inplace_enabled(default=False) def floor(self, bounds=True, inplace=False, i=False): diff --git a/cf/mixin2/cfanetcdf.py b/cf/mixin2/cfanetcdf.py index 279d07d158..9a038e074c 100644 --- a/cf/mixin2/cfanetcdf.py +++ b/cf/mixin2/cfanetcdf.py @@ -274,11 +274,7 @@ def cfa_set_aggregated_data(self, value): self._nc_set("cfa_aggregated_data", value) def cfa_clear_file_substitutions(self): - """Remove the CFA-netCDF file name substitutions. - - The file substitutions are stored in the `substitutions` - attribute of a CFA-netCDF `file` aggregation aggregation - instruction term. + """Remove all of the CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER @@ -290,7 +286,7 @@ def cfa_clear_file_substitutions(self): :Returns: `dict` - The removed CFA-netCDF file name substitutions. + {{Returns cfa_clear_file_substitutions}} **Examples** @@ -321,12 +317,8 @@ def cfa_clear_file_substitutions(self): """ return self._nc_del("cfa_file_substitutions", {}).copy() - def cfa_del_file_substitution(self, base, default=ValueError()): - """Remove the CFA-netCDF file name substitutions. - - The file substitutions are stored in the `substitutions` - attribute of a CFA-netCDF `file` aggregation aggregation - instruction term. + def cfa_del_file_substitution(self, base): + """Remove a CFA-netCDF file name substitution. .. versionadded:: TODOCFAVER @@ -337,21 +329,12 @@ def cfa_del_file_substitution(self, base, default=ValueError()): :Parameters: - base: `str` - The substition definition to be removed. May be - specified with or without the ``${...}`` syntax. For - instance, the following are equivalent: ``'base'``, - ``'${base}'``. - - default: optional - Return the value of the *default* parameter if file - name substitution has not been set. If set to an - `Exception` instance then it will be raised instead. + {{cfa base: `str`}} :Returns: `dict` - The removed CFA-netCDF file name substitutions. + {{Returns cfa_del_file_substitution}} **Examples** @@ -376,8 +359,8 @@ def cfa_del_file_substitution(self, base, default=ValueError()): {} >>> f.cfa_clear_file_substitutions() {} - >>> print(f.cfa_del_file_substitution('base', None)) - None + >>> print(f.cfa_del_file_substitution('base')) + {} """ if not (base.startswith("${") and base.endswith("}")): @@ -385,14 +368,7 @@ def cfa_del_file_substitution(self, base, default=ValueError()): subs = self.cfa_file_substitutions() if base not in subs: - if default is None: - return - - return self._default( - default, - f"{self.__class__.__name__} has no netCDF {base!r} " - "CFA file substitution", - ) + return {} out = {base: subs.pop(base)} if subs: @@ -405,10 +381,6 @@ def cfa_del_file_substitution(self, base, default=ValueError()): def cfa_file_substitutions(self): """Return the CFA-netCDF file name substitutions. - The file substitutions are stored in the `substitutions` - attribute of a CFA-netCDF `file` aggregation aggregation - instruction term. - .. versionadded:: TODOCFAVER .. seealso:: `cfa_clear_file_substitutions`, @@ -417,7 +389,7 @@ def cfa_file_substitutions(self): `cfa_set_file_substitution` :Returns: - value: `dict` + `dict` The CFA-netCDF file name substitutions. **Examples** @@ -456,10 +428,6 @@ def cfa_file_substitutions(self): def cfa_has_file_substitutions(self): """Whether any CFA-netCDF file name substitutions have been set. - The file substitutions are stored in the `substitutions` - attribute of a CFA-netCDF `file` aggregation aggregation - instruction term. - .. versionadded:: TODOCFAVER .. seealso:: `cfa_clear_file_substitutions`, @@ -502,13 +470,9 @@ def cfa_has_file_substitutions(self): """ return self._nc_has("cfa_file_substitutions") - def cfa_set_file_substitutions(self, value): + def cfa_set_file_substitutions(self, substitutions): """Set CFA-netCDF file name substitutions. - The file substitutions are stored in the `substitutions` - attribute of a CFA-netCDF `file` aggregation aggregation - instruction term. - .. versionadded:: TODOCFAVER .. seealso:: `cfa_clear_file_substitutions`, @@ -518,15 +482,7 @@ def cfa_set_file_substitutions(self, value): :Parameters: - value: `str` or `dict` - The substition definitions in a dictionary whose - key/value pairs are the file name parts to be - substituted and their corresponding substitution text. - - The substition definition may be specified with or - without the ``${...}`` syntax. For instance, the - following are equivalent: ``{'base': 'sub'}``, - ``{'${base}': 'sub'}``. + {{cfa substitutions: `dict`}} :Returns: @@ -559,14 +515,14 @@ def cfa_set_file_substitutions(self, value): None """ - if not value: + if not substitutions: return - value = value.copy() - for base, sub in tuple(value.items()): + substitutions = substitutions.copy() + for base, sub in tuple(substitutions.items()): if not (base.startswith("${") and base.endswith("}")): - value[f"${{{base}}}"] = value.pop(base) + substitutions[f"${{{base}}}"] = substitutions.pop(base) subs = self.cfa_file_substitutions() - subs.update(value) + subs.update(substitutions) self._nc_set("cfa_file_substitutions", subs) diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 9354441460..687d0b5c23 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -391,14 +391,14 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar): ndim = data.ndim - ggg = self._ggg(data, cfvar) + cfa = self._cfa_aggregation_instructions(data, cfvar) # ------------------------------------------------------------ # Get the location netCDF dimensions. These always start with - # "f_loc_". + # "f_{size}_loc". # ------------------------------------------------------------ location_ncdimensions = [] - for size in ggg["location"].shape: + for size in cfa["location"].shape: l_ncdim = f"f_{size}_loc" if l_ncdim not in g["dimensions"]: # Create a new location dimension @@ -412,7 +412,7 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar): # Get the fragment netCDF dimensions. These always start with # "f_". # ------------------------------------------------------------ - aggregation_address = ggg["address"] + aggregation_address = cfa["address"] fragment_ncdimensions = [] for ncdim, size in zip( ncdimensions + ("extra",) * (aggregation_address.ndim - ndim), @@ -440,7 +440,7 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar): # Location term = "location" term_ncvar = self._cfa_write_term_variable( - ggg[term], + cfa[term], aggregated_data.get(term, f"cfa_{term}"), location_ncdimensions, ) @@ -459,7 +459,7 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar): attributes = None term_ncvar = self._cfa_write_term_variable( - ggg[term], + cfa[term], aggregated_data.get(term, f"cfa_{term}"), fragment_ncdimensions, attributes=attributes, @@ -470,15 +470,15 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar): term = "address" # Attempt to reduce addresses to a common scalar value - u = ggg[term].unique().compressed().persist() + u = cfa[term].unique().compressed().persist() if u.size == 1: - ggg[term] = u.squeeze() + cfa[term] = u.squeeze() dimensions = () else: dimensions = fragment_ncdimensions term_ncvar = self._cfa_write_term_variable( - ggg[term], + cfa[term], aggregated_data.get(term, f"cfa_{term}"), dimensions, ) @@ -488,15 +488,15 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar): term = "format" # Attempt to reduce addresses to a common scalar value - u = ggg[term].unique().compressed().persist() + u = cfa[term].unique().compressed().persist() if u.size == 1: - ggg[term] = u.squeeze() + cfa[term] = u.squeeze() dimensions = () else: dimensions = fragment_ncdimensions term_ncvar = self._cfa_write_term_variable( - ggg[term], + cfa[term], aggregated_data.get(term, f"cfa_{term}"), dimensions, ) @@ -659,19 +659,23 @@ def _write_field_ancillary(self, f, key, anc): def _cfa_write_term_variable( self, data, ncvar, ncdimensions, attributes=None ): - """TODOCFADOCS. + """Write a CFA aggregation instruction term variable .. versionadded:: TODOCFAVER :Parameters: data `Data` + The data to write. ncvar: `str` + The netCDF variable name. ncdimensions: `tuple` of `str` + The variable's netCDF dimensions. attributes: `dict`, optional + Any attributes to attach to the variable. :Returns: @@ -697,9 +701,9 @@ def _cfa_write_term_variable( def _cfa_write_non_standard_terms( self, field, fragment_ncdimensions, aggregated_data ): - """TODOCFADOCS + """ "Write a non-standard CFA aggregation instruction term variable - Look for non-standard CFA terms stored as field ancillaries + Wites non-standard CFA terms stored as field ancillaries .. versionadded:: TODOCFAVER @@ -773,7 +777,10 @@ def _cfa_write_non_standard_terms( @classmethod def _cfa_unique(cls, a): - """TODOCFADOCS. + """Return the unique value of an array. + + If there are multipl unique vales then missing data is + returned. .. versionadded:: TODOCFAVER @@ -785,7 +792,7 @@ def _cfa_unique(cls, a): :Returns: `numpy.ndarray` - A size 1 array containg the unique value, or missing + A size 1 array containing the unique value, or missing data if there is not a unique unique value. """ @@ -800,26 +807,35 @@ def _cfa_unique(cls, a): return np.ma.masked_all(out_shape, dtype=a.dtype) - def _ggg(self, data, cfvar): - """TODOCFADOCS + def _cfa_aggregation_instructions(self, data, cfvar): + """Convert data to standardised CFA aggregation instruction terms. .. versionadded:: TODOCFAVER :Parameters: data: `Data` - TODOCFADOCS + The data to be converted to standardised CFA + aggregation instruction terms. cfvar: construct - TODOCFADOCS + The construct that contains the *data*. :Returns: `dict` - A dictionary whose keys are the sandardised CFA + A dictionary whose keys are the standardised CFA aggregation instruction terms, keyed by `Data` instances containing the corresponding variables. + **Examples** + + >>> n._cfa_aggregation_instructions(data, cfvar) + {'location': , + 'file': , + 'format': , + 'address': } + """ from os.path import abspath, join, relpath from pathlib import PurePath @@ -843,22 +859,24 @@ def _ggg(self, data, cfvar): aggregation_address = [] aggregation_format = [] for indices in data.chunk_indices(): - a = self._cfa_get_file_details(data[indices]) - if len(a) != 1: - if a: + file_details = self._cfa_get_file_details(data[indices]) + if len(file_details) != 1: + if file_details: raise ValueError( - f"Can't write CFA variable from {cfvar!r} when the " + "Can't write CFA-netCDF aggregation variable from " + f"{cfvar!r} when the " f"dask storage chunk defined by indices {indices} " - "spans two or more external files" + "spans two or more files" ) raise ValueError( - f"Can't write CFA variable from {cfvar!r} when the " + "Can't write CFA-netCDF aggregation variable from " + f"{cfvar!r} when the " f"dask storage chunk defined by indices {indices} spans " - "zero external files" + "zero files" ) - filenames, addresses, formats = a.pop() + filenames, addresses, formats = file_details.pop() if len(filenames) > n_trailing: n_trailing = len(filenames) @@ -967,20 +985,29 @@ def _customize_write_vars(self): ).parent # TODOCFA??? def _cfa_get_file_details(self, data): - """TODOCFADOCS + """Get the details of all files referenced by the data. .. versionadded:: TODOCFAVER :Parameters: data: `Data` - The array. + The data :Returns: `set` - The file names. If no files are required to compute - the data then an empty `set` is returned. + The file names, the addresses in the files, and the + file formats. If no files are required to compute the + data then an empty `set` is returned. + + **Examples** + + >>> n._cfa_get_file_details(data): + {(('/home/file.nc',), ('tas',), ('nc',))} + + >>> n._cfa_get_file_details(data): + {(('/home/file.pp',), (34556,), ('um',))} """ out = set() diff --git a/cf/read_write/read.py b/cf/read_write/read.py index f71ef58760..bbe6fdd2ee 100644 --- a/cf/read_write/read.py +++ b/cf/read_write/read.py @@ -641,7 +641,24 @@ def read( .. versionadded:: 3.11.0 cfa: `dict`, optional - TODOCFADOCS + Configure the reading of CFA-netCDF files. The dictionary + may have any subset of the following key/value pairs to + override the information read from the file: + + * ``'substitutions'``: `dict` + + A dictionary whose key/value pairs define text + substitutions to be applied to the fragment file + names. Each key may be specified with or without the + ``${...}`` syntax. For instance, the following are + equivalent: ``{'base': 'sub'}``, ``{'${base}': 'sub'}``. + The substitutions are used in conjunction with, and take + precedence over, any that are stored in the CFA-netCDF + file by the ``substitutions`` attribute of the ``file`` + CFA aggregation instruction variable. + + *Example:* + ``{'base': 'file:///data/'}}`` .. versionadded:: TODOCFAVER diff --git a/cf/read_write/write.py b/cf/read_write/write.py index 4312ee4820..c6e62c2741 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -160,9 +160,11 @@ def write( ``'NETCDF3_64BIT_DATA'`` NetCDF3 64-bit offset format file with extensions (see below) - ``'CFA'`` or ``'CFA4'`` CFA-netCDF4 format file + ``'CFA'`` or ``'CFA4'`` Deprecated at version TODOCFAVER. + Use the *cfa* parameter instead. - ``'CFA3'`` CFA-netCDF3 classic format file + ``'CFA3'`` Deprecated at version TODOCFAVER. + Use the *cfa* parameter instead. ========================== ================================ By default the format is ``'NETCDF4'``. @@ -588,10 +590,10 @@ def write( If *cfa* is a dictionary then it is used to configure the CFA write process. The default options when CFA writing is - enabled, by any means, are ``{'constructs': 'field', - 'absolute_paths': True, 'strict': True, 'substitutions': - {}}``, and the dictionary may have any subset of the - following key/value pairs to override these defaults: + enabled are ``{'constructs': 'field', 'absolute_paths': + True, 'strict': True, 'substitutions': {}}``, and the + dictionary may have any subset of the following key/value + pairs to override these defaults: * ``'constructs'``: `dict` or (sequence of) `str` @@ -654,11 +656,13 @@ def write( A dictionary whose key/value pairs define text substitutions to be applied to the fragment file - names. Each key must be a string of one or more letters, - digits, and underscores. These substitutions are used in - conjunction with, and take precendence over, any that - are also defined on individual constructs (see - `cf.Data.cfa_set_file_substitutions` for details). + names. Each key may be specified with or without the + ``${...}`` syntax. For instance, the following are + equivalent: ``{'base': 'sub'}``, ``{'${base}': 'sub'}``. + The substitutions are used in conjunction with, and take + precedence over, any that are also defined on individual + constructs (see `cf.Data.cfa_set_file_substitutions` for + details). Substitutions are stored in the output file by the ``substitutions`` attribute of the ``file`` CFA @@ -695,7 +699,7 @@ def write( "cf.write", "fmt", fmt, - "Use keywords 'fmt' and 'cfa' instead.", + "Use the 'cfa' keyword instead.", version="TODOCFAVER", removed_at="5.0.0", ) # pragma: no cover diff --git a/cf/test/test_CFA.py b/cf/test/test_CFA.py index 350b6dab65..711a67bdd1 100644 --- a/cf/test/test_CFA.py +++ b/cf/test/test_CFA.py @@ -214,7 +214,6 @@ def test_CFA_substitutions_2(self): cwd = os.getcwd() - # RRRRRRRRRRRRRRRRRRR f.data.cfa_clear_file_substitutions() f.data.cfa_set_file_substitutions({"base": cwd}) @@ -242,7 +241,6 @@ def test_CFA_substitutions_2(self): self.assertEqual(len(g), 1) self.assertTrue(f.equals(g[0])) - # RRRRRRRRRRRRRRRRRRR f.data.cfa_clear_file_substitutions() f.data.cfa_set_file_substitutions({"base": "/bad/location"}) @@ -267,7 +265,6 @@ def test_CFA_substitutions_2(self): self.assertEqual(len(g), 1) self.assertTrue(f.equals(g[0])) - # RRRRRRRRRRRRRRRR f.data.cfa_clear_file_substitutions() f.data.cfa_set_file_substitutions({"base2": "/bad/location"}) @@ -426,6 +423,23 @@ def test_CFA_PP(self): self.assertEqual(len(g), 1) self.assertTrue(f.equals(g[0])) + def test_CFA_multiple_files(self): + tmpfile1 = "delme1.nc" + tmpfile2 = "delme2.nc" + f = cf.example_field(0) + cf.write(f, tmpfile1) + f = cf.read(tmpfile1)[0] + f.add_file_location("/new/location") + + cf.write(f, tmpfile2, cfa=True) + g = cf.read(tmpfile2) + self.assertEqual(len(g), 1) + g = g[0] + self.assertTrue(f.equals(g)) + + self.assertEqual(len(g.data.get_filenames()), 2) + self.assertEqual(len(g.get_filenames()), 3) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) diff --git a/cf/test/test_Data.py b/cf/test/test_Data.py index febc840a7e..2c99e3cb27 100644 --- a/cf/test/test_Data.py +++ b/cf/test/test_Data.py @@ -4590,15 +4590,15 @@ def test_Data_cfa_file_substitutions(self): self.assertFalse(d.cfa_has_file_substitutions()) self.assertEqual(d.cfa_file_substitutions(), {}) self.assertEqual(d.cfa_clear_file_substitutions(), {}) - self.assertIsNone(d.cfa_del_file_substitution("base", None)) + self.assertEqual(d.cfa_del_file_substitution("base"), {}) def test_Data_file_location(self): """Test `Data` file location methods""" f = cf.example_field(0) - # Can't set file locations when no data is in a file - with self.assertRaises(ValueError): - f.data.set_file_location("/data/model/") + self.assertEqual( + f.data.add_file_location("/data/model/"), "/data/model" + ) cf.write(f, file_A) d = cf.read(file_A, chunks=4)[0].data @@ -4608,13 +4608,13 @@ def test_Data_file_location(self): location = os.path.dirname(os.path.abspath(file_A)) self.assertEqual(d.file_locations(), set((location,))) - self.assertIsNone(d.set_file_location("/data/model/")) + self.assertEqual(d.add_file_location("/data/model/"), "/data/model") self.assertEqual(d.file_locations(), set((location, "/data/model"))) # Check that we haven't changed 'e' self.assertEqual(e.file_locations(), set((location,))) - self.assertIsNone(d.del_file_location("/data/model/")) + self.assertEqual(d.del_file_location("/data/model/"), "/data/model") self.assertEqual(d.file_locations(), set((location,))) d.del_file_location("/invalid") self.assertEqual(d.file_locations(), set((location,))) diff --git a/cf/test/test_NetCDFArray.py b/cf/test/test_NetCDFArray.py index 16f5f1e84c..c69a4654e7 100644 --- a/cf/test/test_NetCDFArray.py +++ b/cf/test/test_NetCDFArray.py @@ -1,5 +1,8 @@ +import atexit import datetime import faulthandler +import os +import tempfile import unittest from dask.base import tokenize @@ -8,6 +11,25 @@ import cf +n_tmpfiles = 1 +tmpfiles = [ + tempfile.mkstemp("_test_NetCDFArray.nc", dir=os.getcwd())[1] + for i in range(n_tmpfiles) +] +(tmpfile1,) = tmpfiles + + +def _remove_tmpfiles(): + """Try to remove defined temporary files by deleting their paths.""" + for f in tmpfiles: + try: + os.remove(f) + except OSError: + pass + + +atexit.register(_remove_tmpfiles) + class NetCDFArrayTest(unittest.TestCase): def test_NetCDFArray_del_file_location(self): @@ -40,9 +62,9 @@ def test_NetCDFArray_file_locations(self): a = cf.NetCDFArray(("/data1/file1", "/data2/file2", "/data1/file2")) self.assertEqual(a.file_locations(), ("/data1", "/data2", "/data1")) - def test_NetCDFArray_set_file_location(self): + def test_NetCDFArray_add_file_location(self): a = cf.NetCDFArray("/data1/file1", "tas") - b = a.set_file_location("/home/user") + b = a.add_file_location("/home/user") self.assertIsNot(b, a) self.assertEqual( b.get_filenames(), ("/data1/file1", "/home/user/file1") @@ -50,7 +72,7 @@ def test_NetCDFArray_set_file_location(self): self.assertEqual(b.get_addresses(), ("tas", "tas")) a = cf.NetCDFArray(("/data1/file1", "/data2/file2"), ("tas1", "tas2")) - b = a.set_file_location("/home/user") + b = a.add_file_location("/home/user") self.assertEqual( b.get_filenames(), ( @@ -63,7 +85,7 @@ def test_NetCDFArray_set_file_location(self): self.assertEqual(b.get_addresses(), ("tas1", "tas2", "tas1", "tas2")) a = cf.NetCDFArray(("/data1/file1", "/data2/file1"), ("tas1", "tas2")) - b = a.set_file_location("/home/user") + b = a.add_file_location("/home/user") self.assertEqual( b.get_filenames(), ("/data1/file1", "/data2/file1", "/home/user/file1"), @@ -71,7 +93,7 @@ def test_NetCDFArray_set_file_location(self): self.assertEqual(b.get_addresses(), ("tas1", "tas2", "tas1")) a = cf.NetCDFArray(("/data1/file1", "/data2/file1"), ("tas1", "tas2")) - b = a.set_file_location("/data1") + b = a.add_file_location("/data1/") self.assertEqual(b.get_filenames(), a.get_filenames()) self.assertEqual(b.get_addresses(), a.get_addresses()) @@ -82,6 +104,23 @@ def test_NetCDFArray__dask_tokenize__(self): b = cf.NetCDFArray("/home/file2", "tas", shape=(12, 2)) self.assertNotEqual(tokenize(a), tokenize(b)) + def test_NetCDFArray_multiple_files(self): + f = cf.example_field(0) + cf.write(f, tmpfile1) + + # Create instance with non-existent file + n = cf.NetCDFArray( + filename=os.path.join("/bad/location", os.path.basename(tmpfile1)), + address=f.nc_get_variable(), + shape=f.shape, + dtype=f.dtype, + ) + # Add file that exists + n = n.add_file_location(os.path.dirname(tmpfile1)) + + self.assertEqual(len(n.get_filenames()), 2) + self.assertTrue((n[...] == f.array).all()) + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) From 271fba914f8311fa1107e8430f6b85425140d549 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 23 Mar 2023 14:46:58 +0000 Subject: [PATCH 065/141] cfanetcdfarray.to_dask_array bugfix --- cf/data/array/cfanetcdfarray.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index 3b695f9adb..2a8c791274 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -673,9 +673,6 @@ def to_dask_array(self, chunks="auto"): fragment_arrays = _FragmentArray.copy() fragment_arrays["nc"] = partial(_FragmentArray["nc"], mask=False) - # Create a FragmentArray for each chunk - # get_FragmentArray = self.get_FragmentArray - dsk = {} for ( u_indices, @@ -685,7 +682,7 @@ def to_dask_array(self, chunks="auto"): fragment_location, fragment_shape, ) in zip(*self.subarrays(chunks)): - kwargs = aggregated_data[chunk_location].copy() + kwargs = aggregated_data[fragment_location].copy() kwargs.pop("location", None) fragment_format = kwargs.pop("format", None) From c680bc52bfe651fad69a2352f51b5a991efc2b06 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 27 Mar 2023 18:49:53 +0100 Subject: [PATCH 066/141] dev --- cf/__init__.py | 4 +- cf/data/data.py | 4 +- cf/domain.py | 6 +- cf/field.py | 8 +-- cf/mixin/propertiesdata.py | 6 +- cf/mixin/propertiesdatabounds.py | 8 +-- cf/mixin2/cfanetcdf.py | 103 +++++++++++----------------- cf/read_write/netcdf/netcdfread.py | 2 +- cf/read_write/netcdf/netcdfwrite.py | 2 +- cf/read_write/write.py | 4 +- cf/test/test_CFA.py | 8 +-- cf/test/test_Data.py | 10 +-- requirements.txt | 2 +- 13 files changed, 72 insertions(+), 95 deletions(-) diff --git a/cf/__init__.py b/cf/__init__.py index b7ca44787d..ef037e2d0b 100644 --- a/cf/__init__.py +++ b/cf/__init__.py @@ -189,8 +189,8 @@ ) # Check the version of cfdm -_minimum_vn = "1.10.0.3" -_maximum_vn = "1.10.1.0" +_minimum_vn = "1.10.1.0" +_maximum_vn = "1.10.2.0" _cfdm_version = Version(cfdm.__version__) if not Version(_minimum_vn) <= _cfdm_version < Version(_maximum_vn): raise RuntimeError( diff --git a/cf/data/data.py b/cf/data/data.py index 0cff6d0a86..d3225c6e0b 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -3947,14 +3947,14 @@ def concatenate(cls, data, axis=0, cull_graph=True, relaxed_units=False): aggregated_data = {} substitutions = {} for d in processed_data[::-1]: - aggregated_data.update(d.cfa_get_aggregated_data({})) + aggregated_data.update(d.cfa_get_aggregated_data()) substitutions.update(d.cfa_file_substitutions()) if aggregated_data: data0.cfa_set_aggregated_data(aggregated_data) if substitutions: - data0.cfa_set_file_substitutions(substitutions) + data0.cfa_update_file_substitutions(substitutions) # Set the CFA aggregation instruction term status if data0.cfa_get_term(): diff --git a/cf/domain.py b/cf/domain.py index 56ad16081c..9e551431a7 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -243,7 +243,7 @@ def cfa_del_file_substitution( base, ) - def cfa_set_file_substitutions( + def cfa_update_file_substitutions( self, substitutions, ): @@ -261,11 +261,11 @@ def cfa_set_file_substitutions( **Examples** - >>> d.cfa_set_file_substitutions({'base': '/data/model'}) + >>> d.cfa_update_file_substitutions({'base': '/data/model'}) """ for c in self.constructs.filter_by_data(todict=True).values(): - c.cfa_set_file_substitutions(substitutions) + c.cfa_update_file_substitutions(substitutions) def close(self): """Close all files referenced by the domain construct. diff --git a/cf/field.py b/cf/field.py index afa8b82e7a..812976d88a 100644 --- a/cf/field.py +++ b/cf/field.py @@ -3771,7 +3771,7 @@ def del_file_location( return location - def cfa_set_file_substitutions( + def cfa_update_file_substitutions( self, substitutions, constructs=True, @@ -3794,14 +3794,14 @@ def cfa_set_file_substitutions( **Examples** - >>> f.cfa_set_file_substitutions({'base': '/data/model'}) + >>> f.cfa_update_file_substitutions({'base': '/data/model'}) """ - super().cfa_set_file_substitutions(substitutions) + super().cfa_update_file_substitutions(substitutions) if constructs: for c in self.constructs.filter_by_data(todict=True).values(): - c.cfa_set_file_substitutions(substitutions) + c.cfa_update_file_substitutions(substitutions) def radius(self, default=None): """Return the radius of a latitude-longitude plane defined in diff --git a/cf/mixin/propertiesdata.py b/cf/mixin/propertiesdata.py index f3bf4a492d..6ab2e0bc31 100644 --- a/cf/mixin/propertiesdata.py +++ b/cf/mixin/propertiesdata.py @@ -2508,7 +2508,7 @@ def ceil(self, inplace=False, i=False): delete_props=True, ) - def cfa_set_file_substitutions(self, substitutions): + def cfa_update_file_substitutions(self, substitutions): """Set CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER @@ -2523,12 +2523,12 @@ def cfa_set_file_substitutions(self, substitutions): **Examples** - >>> f.cfa_set_file_substitutions({'base', '/data/model'}) + >>> f.cfa_update_file_substitutions({'base', '/data/model'}) """ data = self.get_data(None, _fill_value=False, _units=False) if data is not None: - data.cfa_set_file_substitutions(substitutions) + data.cfa_update_file_substitutions(substitutions) @_inplace_enabled(default=False) def cfa_clear_file_substitutions(self, inplace=False): diff --git a/cf/mixin/propertiesdatabounds.py b/cf/mixin/propertiesdatabounds.py index ea80fe9353..bc0ad1ac2f 100644 --- a/cf/mixin/propertiesdatabounds.py +++ b/cf/mixin/propertiesdatabounds.py @@ -1308,7 +1308,7 @@ def cfa_file_substitutions(self): return out - def cfa_set_file_substitutions(self, substitutions): + def cfa_update_file_substitutions(self, substitutions): """Set CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER @@ -1326,15 +1326,15 @@ def cfa_set_file_substitutions(self, substitutions): >>> c.cfa_add_file_substitutions({'base', '/data/model'}) """ - super().cfa_set_file_substitutions(substitutions) + super().cfa_update_file_substitutions(substitutions) bounds = self.get_bounds(None) if bounds is not None: - bounds.cfa_set_file_substitutions(substitutions) + bounds.cfa_update_file_substitutions(substitutions) interior_ring = self.get_interior_ring(None) if interior_ring is not None: - interior_ring.cfa_set_file_substitutions(substitutions) + interior_ring.cfa_update_file_substitutions(substitutions) def chunk(self, chunksize=None): """Partition the data array. diff --git a/cf/mixin2/cfanetcdf.py b/cf/mixin2/cfanetcdf.py index 9a038e074c..ad1c97c425 100644 --- a/cf/mixin2/cfanetcdf.py +++ b/cf/mixin2/cfanetcdf.py @@ -16,7 +16,7 @@ class CFANetCDF(NetCDFMixin): """ - def cfa_del_aggregated_data(self, default=ValueError()): + def cfa_del_aggregated_data(self): """Remove the CFA-netCDF aggregation instruction terms. The aggregation instructions are stored in the @@ -29,14 +29,6 @@ def cfa_del_aggregated_data(self, default=ValueError()): `cfa_has_aggregated_data`, `cfa_set_aggregated_data` - :Parameters: - - default: optional - Return the value of the *default* parameter if the - CFA-netCDF aggregation terms have not been set. If set - to an `Exception` instance then it will be raised - instead. - :Returns: `dict` @@ -67,14 +59,15 @@ def cfa_del_aggregated_data(self, default=ValueError()): 'tracking_id': 'tracking_id'} >>> f.cfa_has_aggregated_data() False - >>> print(f.cfa_get_aggregated_data(None)) - None - None + >>> f.cfa_del_aggregated_data() + {} + >>> f.cfa_get_aggregated_data() + {} """ - return self._nc_del("cfa_aggregated_data", default=default) + return self._nc_del("cfa_aggregated_data", {}).copy() - def cfa_get_aggregated_data(self, default=ValueError()): + def cfa_get_aggregated_data(self): """Return the CFA-netCDF aggregation instruction terms. The aggregation instructions are stored in the @@ -87,14 +80,6 @@ def cfa_get_aggregated_data(self, default=ValueError()): `cfa_has_aggregated_data`, `cfa_set_aggregated_data` - :Parameters: - - default: optional - Return the value of the *default* parameter if the - CFA-netCDF aggregation terms have not been set. If set - to an `Exception` instance then it will be raised - instead. - :Returns: `dict` @@ -103,7 +88,6 @@ def cfa_get_aggregated_data(self, default=ValueError()): whose key/value pairs are the aggregation instruction terms and their corresponding variable names. - **Examples** >>> f.cfa_set_aggregated_data( @@ -129,23 +113,17 @@ def cfa_get_aggregated_data(self, default=ValueError()): 'tracking_id': 'tracking_id'} >>> f.cfa_has_aggregated_data() False - >>> print(f.cfa_get_aggregated_data(None)) - None - >>> print(f.cfa_del_aggregated_data(None)) - None + >>> f.cfa_del_aggregated_data() + {} + >>> f.cfa_get_aggregated_data() + {} """ out = self._nc_get("cfa_aggregated_data", default=None) if out is not None: return out.copy() - if default is None: - return default - - return self._default( - default, - f"{self.__class__.__name__} has no CFA-netCDF aggregation terms", - ) + return {} def cfa_has_aggregated_data(self): """Whether any CFA-netCDF aggregation instruction terms have been set. @@ -191,11 +169,10 @@ def cfa_has_aggregated_data(self): 'tracking_id': 'tracking_id'} >>> f.cfa_has_aggregated_data() False - >>> print(f.cfa_get_aggregated_data(None)) - None - >>> print(f.cfa_del_aggregated_data(None)) - None - + >>> f.cfa_del_aggregated_data() + {} + >>> f.cfa_get_aggregated_data() + {} """ return self._nc_has("cfa_aggregated_data") @@ -257,10 +234,10 @@ def cfa_set_aggregated_data(self, value): 'tracking_id': 'tracking_id'} >>> f.cfa_has_aggregated_data() False - >>> print(f.cfa_get_aggregated_data(None)) - None - >>> print(f.cfa_del_aggregated_data(None)) - None + >>> f.cfa_del_aggregated_data() + {} + >>> f.cfa_get_aggregated_data() + {} """ if value: @@ -281,7 +258,7 @@ def cfa_clear_file_substitutions(self): .. seealso:: `cfa_del_file_substitution`, `cfa_file_substitutions`, `cfa_has_file_substitutions`, - `cfa_set_file_substitutions` + `cfa_update_file_substitutions` :Returns: @@ -290,15 +267,15 @@ def cfa_clear_file_substitutions(self): **Examples** - >>> f.`cfa_set_file_substitutions({'base': 'file:///data/'}) + >>> f.cfa_update_file_substitutions({'base': 'file:///data/'}) >>> f.cfa_has_file_substitutions() True >>> f.cfa_file_substitutions() {'${base}': 'file:///data/'} - >>> f.`cfa_set_file_substitutions({'${base2}': '/home/data/'}) + >>> f.cfa_update_file_substitutions({'${base2}': '/home/data/'}) >>> f.cfa_file_substitutions() {'${base}': 'file:///data/', '${base2}': '/home/data/'} - >>> f.`cfa_set_file_substitutions({'${base}': '/new/location/'}) + >>> f.cfa_update_file_substitutions({'${base}': '/new/location/'}) >>> f.cfa_file_substitutions() {'${base}': '/new/location/', '${base2}': '/home/data/'} >>> f.cfa_del_file_substitution('${base}') @@ -325,7 +302,7 @@ def cfa_del_file_substitution(self, base): .. seealso:: `cfa_clear_file_substitutions`, `cfa_file_substitutions`, `cfa_has_file_substitutions`, - `cfa_set_file_substitutions` + `cfa_update_file_substitutions` :Parameters: @@ -338,15 +315,15 @@ def cfa_del_file_substitution(self, base): **Examples** - >>> f.cfa_set_file_substitutions({'base': 'file:///data/'}) + >>> f.cfa_update_file_substitutions({'base': 'file:///data/'}) >>> f.cfa_has_file_substitutions() True >>> f.cfa_file_substitutions() {'${base}': 'file:///data/'} - >>> f.`cfa_set_file_substitutions({'${base2}': '/home/data/'}) + >>> f.cfa_update_file_substitutions({'${base2}': '/home/data/'}) >>> f.cfa_file_substitutions() {'${base}': 'file:///data/', '${base2}': '/home/data/'} - >>> f.`cfa_set_file_substitutions({'${base}': '/new/location/'}) + >>> f.cfa_update_file_substitutions({'${base}': '/new/location/'}) >>> f.cfa_file_substitutions() {'${base}': '/new/location/', '${base2}': '/home/data/'} >>> f.cfa_del_file_substitution('${base}') @@ -386,7 +363,7 @@ def cfa_file_substitutions(self): .. seealso:: `cfa_clear_file_substitutions`, `cfa_del_file_substitution`, `cfa_file_substitutions`, - `cfa_set_file_substitution` + `cfa_update_file_substitution` :Returns: `dict` @@ -394,15 +371,15 @@ def cfa_file_substitutions(self): **Examples** - >>> f.`cfa_set_file_substitutions({'base': 'file:///data/'}) + >>> f.cfa_update_file_substitutions({'base': 'file:///data/'}) >>> f.cfa_has_file_substitutions() True >>> f.cfa_file_substitutions() {'${base}': 'file:///data/'} - >>> f.`cfa_set_file_substitutions({'${base2}': '/home/data/'}) + >>> f.cfa_update_file_substitutions({'${base2}': '/home/data/'}) >>> f.cfa_file_substitutions() {'${base}': 'file:///data/', '${base2}': '/home/data/'} - >>> f.`cfa_set_file_substitutions({'${base}': '/new/location/'}) + >>> f.cfa_update_file_substitutions({'${base}': '/new/location/'}) >>> f.cfa_file_substitutions() {'${base}': '/new/location/', '${base2}': '/home/data/'} >>> f.cfa_del_file_substitution('${base}') @@ -433,7 +410,7 @@ def cfa_has_file_substitutions(self): .. seealso:: `cfa_clear_file_substitutions`, `cfa_del_file_substitution`, `cfa_file_substitutions`, - `cfa_set_file_substitutions` + `cfa_update_file_substitutions` :Returns: @@ -443,15 +420,15 @@ def cfa_has_file_substitutions(self): **Examples** - >>> f.`cfa_set_file_substitutions({'base': 'file:///data/'}) + >>> f.cfa_update_file_substitutions({'base': 'file:///data/'}) >>> f.cfa_has_file_substitutions() True >>> f.cfa_file_substitutions() {'${base}': 'file:///data/'} - >>> f.`cfa_set_file_substitutions({'${base2}': '/home/data/'}) + >>> f.cfa_update_file_substitutions({'${base2}': '/home/data/'}) >>> f.cfa_file_substitutions() {'${base}': 'file:///data/', '${base2}': '/home/data/'} - >>> f.`cfa_set_file_substitutions({'${base}': '/new/location/'}) + >>> f.cfa_update_file_substitutions({'${base}': '/new/location/'}) >>> f.cfa_file_substitutions() {'${base}': '/new/location/', '${base2}': '/home/data/'} >>> f.cfa_del_file_substitution('${base}') @@ -470,7 +447,7 @@ def cfa_has_file_substitutions(self): """ return self._nc_has("cfa_file_substitutions") - def cfa_set_file_substitutions(self, substitutions): + def cfa_update_file_substitutions(self, substitutions): """Set CFA-netCDF file name substitutions. .. versionadded:: TODOCFAVER @@ -490,15 +467,15 @@ def cfa_set_file_substitutions(self, substitutions): **Examples** - >>> f.`cfa_set_file_substitutions({'base': 'file:///data/'}) + >>> f.cfa_update_file_substitutions({'base': 'file:///data/'}) >>> f.cfa_has_file_substitutions() True >>> f.cfa_file_substitutions() {'${base}': 'file:///data/'} - >>> f.`cfa_set_file_substitutions({'${base2}': '/home/data/'}) + >>> f.cfa_update_file_substitutions({'${base2}': '/home/data/'}) >>> f.cfa_file_substitutions() {'${base}': 'file:///data/', '${base2}': '/home/data/'} - >>> f.`cfa_set_file_substitutions({'${base}': '/new/location/'}) + >>> f.cfa_update_file_substitutions({'${base}': '/new/location/'}) >>> f.cfa_file_substitutions() {'${base}': '/new/location/', '${base2}': '/home/data/'} >>> f.cfa_del_file_substitution('${base}') diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 4c7558b986..3784f9c142 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -262,7 +262,7 @@ def _create_data( data.cfa_set_aggregated_data(aggregated_data) # Store the file substitutions - data.cfa_set_file_substitutions(kwargs.get("substitutions")) + data.cfa_update_file_substitutions(kwargs.get("substitutions")) return data diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 687d0b5c23..d9e38c49ae 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -434,7 +434,7 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar): substitutions = data.cfa_file_substitutions() substitutions.update(g["cfa_options"].get("substitutions", {})) - aggregated_data = data.cfa_get_aggregated_data(default={}) + aggregated_data = data.cfa_get_aggregated_data() aggregated_data_attr = [] # Location diff --git a/cf/read_write/write.py b/cf/read_write/write.py index c6e62c2741..cea6be035a 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -661,8 +661,8 @@ def write( equivalent: ``{'base': 'sub'}``, ``{'${base}': 'sub'}``. The substitutions are used in conjunction with, and take precedence over, any that are also defined on individual - constructs (see `cf.Data.cfa_set_file_substitutions` for - details). + constructs (see `cf.Data.cfa_update_file_substitutions` + for details). Substitutions are stored in the output file by the ``substitutions`` attribute of the ``file`` CFA diff --git a/cf/test/test_CFA.py b/cf/test/test_CFA.py index 711a67bdd1..f51715f92d 100644 --- a/cf/test/test_CFA.py +++ b/cf/test/test_CFA.py @@ -156,7 +156,7 @@ def test_CFA_substitutions_0(self): cwd = os.getcwd() - f.data.cfa_set_file_substitutions({"base": cwd}) + f.data.cfa_update_file_substitutions({"base": cwd}) cf.write( f, @@ -215,7 +215,7 @@ def test_CFA_substitutions_2(self): cwd = os.getcwd() f.data.cfa_clear_file_substitutions() - f.data.cfa_set_file_substitutions({"base": cwd}) + f.data.cfa_update_file_substitutions({"base": cwd}) cf.write( f, @@ -242,7 +242,7 @@ def test_CFA_substitutions_2(self): self.assertTrue(f.equals(g[0])) f.data.cfa_clear_file_substitutions() - f.data.cfa_set_file_substitutions({"base": "/bad/location"}) + f.data.cfa_update_file_substitutions({"base": "/bad/location"}) cf.write( f, @@ -266,7 +266,7 @@ def test_CFA_substitutions_2(self): self.assertTrue(f.equals(g[0])) f.data.cfa_clear_file_substitutions() - f.data.cfa_set_file_substitutions({"base2": "/bad/location"}) + f.data.cfa_update_file_substitutions({"base2": "/bad/location"}) cf.write( f, diff --git a/cf/test/test_Data.py b/cf/test/test_Data.py index 2c99e3cb27..373e9d7248 100644 --- a/cf/test/test_Data.py +++ b/cf/test/test_Data.py @@ -4554,28 +4554,28 @@ def test_Data_cfa_aggregated_data(self): self.assertEqual(d.cfa_get_aggregated_data(), aggregated_data) self.assertEqual(d.cfa_del_aggregated_data(), aggregated_data) self.assertFalse(d.cfa_has_aggregated_data()) - self.assertIsNone(d.cfa_get_aggregated_data(None)) - self.assertIsNone(d.cfa_del_aggregated_data(None)) + self.assertEqual(d.cfa_get_aggregated_data(), {}) + self.assertEqual(d.cfa_del_aggregated_data(), {}) def test_Data_cfa_file_substitutions(self): """Test `Data` CFA file_substitutions methods""" d = cf.Data(9) self.assertFalse(d.cfa_has_file_substitutions()) self.assertIsNone( - d.cfa_set_file_substitutions({"base": "file:///data/"}) + d.cfa_update_file_substitutions({"base": "file:///data/"}) ) self.assertTrue(d.cfa_has_file_substitutions()) self.assertEqual( d.cfa_file_substitutions(), {"${base}": "file:///data/"} ) - d.cfa_set_file_substitutions({"${base2}": "/home/data/"}) + d.cfa_update_file_substitutions({"${base2}": "/home/data/"}) self.assertEqual( d.cfa_file_substitutions(), {"${base}": "file:///data/", "${base2}": "/home/data/"}, ) - d.cfa_set_file_substitutions({"${base}": "/new/location/"}) + d.cfa_update_file_substitutions({"${base}": "/new/location/"}) self.assertEqual( d.cfa_file_substitutions(), {"${base}": "/new/location/", "${base2}": "/home/data/"}, diff --git a/requirements.txt b/requirements.txt index 050ec56d78..cc13a7e9f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ netCDF4>=1.5.4 cftime>=1.6.0 numpy>=1.22 -cfdm>=1.10.0.3, <1.10.1.0 +cfdm>=1.10.1.0, <1.10.2.0 psutil>=0.6.0 cfunits>=3.3.5 dask>=2022.12.1 From 9c2c6ff3f7deea7511d464c70c89eb8c4adf127a Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 17 Apr 2023 19:14:27 +0100 Subject: [PATCH 067/141] customize -> customise --- cf/read_write/netcdf/netcdfread.py | 12 ++++++------ cf/read_write/netcdf/netcdfwrite.py | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 3784f9c142..be83a60a42 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -351,8 +351,8 @@ def _create_Data( return data - def _customize_read_vars(self): - """Customize the read parameters. + def _customise_read_vars(self): + """Customise the read parameters. Take the opportunity to apply CFA updates to `read_vars['variable_dimensions']` and @@ -361,7 +361,7 @@ def _customize_read_vars(self): .. versionadded:: 3.0.0 """ - super()._customize_read_vars() + super()._customise_read_vars() g = self.read_vars if not g["cfa"]: @@ -698,7 +698,7 @@ def _parse_chunks(self, ncvar): return chunks - def _customize_field_ancillaries(self, parent_ncvar, f): + def _customise_field_ancillaries(self, parent_ncvar, f): """Create customised field ancillary constructs. This method currently creates: @@ -726,10 +726,10 @@ def _customize_field_ancillaries(self, parent_ncvar, f): **Examples** - >>> n._customize_field_ancillaries('tas', f) + >>> n._customise_field_ancillaries('tas', f) {} - >>> n._customize_field_ancillaries('pr', f) + >>> n._customise_field_ancillaries('pr', f) {'tracking_id': 'fieldancillary1'} """ diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index d9e38c49ae..ad8a631fa6 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -84,7 +84,7 @@ def _write_as_cfa(self, cfvar, construct_type, domain_axes): return False - def _customize_createVariable( + def _customise_createVariable( self, cfvar, construct_type, domain_axes, kwargs ): """Customise keyword arguments for @@ -105,7 +105,7 @@ def _customize_createVariable( `netCDF4.Dataset.createVariable`. """ - kwargs = super()._customize_createVariable( + kwargs = super()._customise_createVariable( cfvar, construct_type, domain_axes, kwargs ) @@ -968,7 +968,7 @@ def _cfa_aggregation_instructions(self, data, cfvar): "address": data(aggregation_address), } - def _customize_write_vars(self): + def _customise_write_vars(self): """Customise the write parameters. .. versionadded:: TODOCFAVER From d5e0bbfae5a9c5df9a5d46447d39408baa770873 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 17 Apr 2023 19:39:47 +0100 Subject: [PATCH 068/141] domain_axes docs --- cf/read_write/netcdf/netcdfwrite.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index ad8a631fa6..cc703ebcb5 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -37,7 +37,10 @@ def _write_as_cfa(self, cfvar, construct_type, domain_axes): .. versionadded:: TODOCFAVER - domain-axes: `None`, or `tuple` of `int` + domain_axes: `None`, or `tuple` of `str` + The domain axis construct identidifiers for *cfvar*. + + .. versionadded:: TODOCFAVER :Returns: @@ -96,6 +99,17 @@ def _customise_createVariable( cfvar: cf instance that contains data + construct_type: `str` + The construct type of the *cfvar*, or its parent if + *cfvar* is not a construct. + + .. versionadded:: TODOCFAVER + + domain_axes: `None`, or `tuple` of `str` + The domain axis construct identidifiers for *cfvar*. + + .. versionadded:: TODOCFAVER + kwargs: `dict` :Returns: @@ -142,6 +156,11 @@ def _write_data( ncdimensions: `tuple` of `str` + domain_axes: `None`, or `tuple` of `str` + The domain axis construct identidifiers for *cfvar*. + + .. versionadded:: TODOCFAVER + unset_values: sequence of numbers attributes: `dict`, optional From 4ba58733afc499772fd3ba95ec6887c711b16cbc Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 18 Apr 2023 08:43:31 +0100 Subject: [PATCH 069/141] remove obselete file --- cf/data/fragment/abstract/__init__.py | 1 - cf/data/fragment/abstract/fragmentarray.py | 468 --------------------- 2 files changed, 469 deletions(-) delete mode 100644 cf/data/fragment/abstract/__init__.py delete mode 100644 cf/data/fragment/abstract/fragmentarray.py diff --git a/cf/data/fragment/abstract/__init__.py b/cf/data/fragment/abstract/__init__.py deleted file mode 100644 index b771e745ab..0000000000 --- a/cf/data/fragment/abstract/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# from .fragmentarray import FragmentArray diff --git a/cf/data/fragment/abstract/fragmentarray.py b/cf/data/fragment/abstract/fragmentarray.py deleted file mode 100644 index c1c68a5225..0000000000 --- a/cf/data/fragment/abstract/fragmentarray.py +++ /dev/null @@ -1,468 +0,0 @@ -# from numbers import Integral -# -# from ....units import Units -# -# -# class FragmentArrayMixin: -# """Mixin for a CFA fragment array. -# -# .. versionadded:: 3.14.0 -# -# """ -# -# -# def __init__( -# self, -# dtype=None, -# shape=None, -# aggregated_units=False, -# aggregated_calendar=None, -# array=None, -# source=None, -# copy=True, -# ): -# """**Initialisation** -# -# :Parameters: -# -# dtype: `numpy.dtype` -# The data type of the aggregated array. May be `None` -# if the numpy data-type is not known (which can be the -# case for netCDF string types, for example). This may -# differ from the data type of the netCDF fragment -# variable. -# -# shape: `tuple` -# The shape of the fragment within the aggregated -# array. This may differ from the shape of the netCDF -# fragment variable in that the latter may have fewer -# size 1 dimensions. -# -# {{aggregated_units: `str` or `None`, optional}} -# -# {{aggregated_calendar: `str` or `None`, optional}} -# -# array: `Array` -# The fragment array stored in a file. -# -# source: optional -# Initialise the array from the given object. -# -# {{init source}} -# -# {{deep copy}} -# -# """ -# super().__init__(source=source, copy=copy) -# -# if source is not None: -# try: -# dtype = source._get_component("dtype", None) -# except AttributeError: -# dtype = None -# -# try: -# shape = source._get_component("shape", None) -# except AttributeError: -# shape = None -# -# try: -# aggregated_units = source._get_component( -# "aggregated_units", False -# ) -# except AttributeError: -# aggregated_units = False -# -# try: -# aggregated_calendar = source._get_component( -# "aggregated_calendar", False -# ) -# except AttributeError: -# aggregated_calendar = False -# -# try: -# array = source._get_component("array", None) -# except AttributeError: -# array = None -# -# self._set_component("dtype", dtype, copy=False) -# self._set_component("shape", shape, copy=False) -# self._set_component("aggregated_units", aggregated_units, copy=False) -# self._set_component( -# "aggregated_calendar", aggregated_calendar, copy=False -# ) -# -# if array is not None: -# self._set_component("array", array, copy=copy) -# -# def __getitem__(self, indices): -# """Returns a subspace of the fragment as a numpy array. -# -# x.__getitem__(indices) <==> x[indices] -# -# Indexing is similar to numpy indexing, with the following -# differences: -# -# * A dimension's index can't be rank-reducing, i.e. it can't -# be an integer, a scalar `numpy` array, nor a scalar `dask` -# array. -# -# * When two or more dimension's indices are sequences of -# integers then these indices work independently along each -# dimension (similar to the way vector subscripts work in -# Fortran). -# -# .. versionadded:: TODOCFAVER -# -# """ -# indices = self._parse_indices(indices) -# -# try: -# array = super().__getitem__(indices) -# except ValueError: -# # A ValueError is expected to be raised when the fragment -# # variable has fewer than 'self.ndim' dimensions (given -# # that 'indices' now has 'self.ndim' elements). -# axis = self._missing_size_1_axis(indices) -# if axis is not None: -# # There is a unique size 1 index, that must correspond -# # to the missing dimension => Remove it from the -# # indices, get the fragment array with the new -# # indices; and then insert the missing size one -# # dimension. -# indices = list(indices) -# indices.pop(axis) -# array = super().__getitem__(tuple(indices)) -# array = np.expand_dims(array, axis) -# else: -# # There are multiple size 1 indices, so we don't know -# # how many missing dimensions there are nor their -# # positions => Get the full fragment array; and then -# # reshape it to the shape of the storage chunk. -# array = super().__getitem__(Ellipsis) -# if array.size != self.size: -# raise ValueError( -# "Can't get CFA fragment data from " -# f"{self.get_filename()} ({self.get_address()}) when " -# "the fragment has two or more missing size 1 " -# "dimensions whilst also spanning two or more " -# "storage chunks" -# "\n\n" -# "Consider recreating the data with exactly one" -# "storage chunk per fragment." -# ) -# -# array = array.reshape(self.shape) -# -# array = self._conform_units(array) -# return array -# -# def _missing_size_1_axis(self, indices): -# """Find the position of a unique size 1 index. -# -# .. versionadded:: TODOCFAVER -# -# .. seealso:: `_parse_indices` -# -# :Parameters: -# -# indices: `tuple` -# The array indices to be parsed, as returned by -# `_parse_indices`. -# -# :Returns: -# -# `int` or `None` -# The position of the unique size 1 index, or `None` if -# there isn't one. -# -# **Examples** -# -# >>> a._missing_size_1_axis(([2, 4, 5], slice(0, 1), slice(0, 73))) -# 1 -# >>> a._missing_size_1_axis(([2, 4, 5], slice(3, 4), slice(0, 73))) -# 1 -# >>> a._missing_size_1_axis(([2, 4, 5], [0], slice(0, 73))) -# 1 -# >>> a._missing_size_1_axis(([2, 4, 5], slice(0, 144), slice(0, 73))) -# None -# >>> a._missing_size_1_axis(([2, 4, 5], slice(3, 7), [0, 1])) -# None -# >>> a._missing_size_1_axis(([2, 4, 5], slice(0, 1), [0])) -# None -# -# """ -# axis = None # Position of unique size 1 index -# -# n = 0 # Number of size 1 indices -# for i, index in enumerate(indices): -# try: -# if index.stop - index.start == 1: -# # Index is a size 1 slice -# n += 1 -# axis = i -# except AttributeError: -# try: -# if index.size == 1: -# # Index is a size 1 numpy or dask array -# n += 1 -# axis = i -# except AttributeError: -# if len(index) == 1: -# # Index is a size 1 list -# n += 1 -# axis = i -# -# if n > 1: -# # There are two or more size 1 indices -# axis = None -# -# return axis -# -# def _parse_indices(self, indices): -# """Parse the indices that retrieve the fragment data. -# -# Ellipses are replaced with the approriate number `slice` -# instances, and rank-reducing indices (such as an integer or -# scalar array) are disallowed. -# -# .. versionadded:: 3.14.0 -# -# :Parameters: -# -# indices: `tuple` or `Ellipsis` -# The array indices to be parsed. -# -# :Returns: -# -# `tuple` -# The parsed indices. -# -# **Examples** -# -# >>> a.shape -# (12, 1, 73, 144) -# >>> a._parse_indices(([2, 4, 5], Ellipsis, slice(45, 67)) -# ([2, 4, 5], slice(0, 1), slice(0, 73), slice(45, 67)) -# -# """ -# shape = self.shape -# if indices is Ellipsis: -# return tuple([slice(0, n) for n in shape]) -# -# # Check indices -# has_ellipsis = False -# indices = list(indices) -# for i, (index, n) in enumerate(zip(indices, shape)): -# if isinstance(index, slice): -# if index == slice(None): -# indices[i] = slice(0, n) -# -# continue -# -# if index is Ellipsis: -# has_ellipsis = True -# continue -# -# if isinstance(index, Integral) or not getattr(index, "ndim", True): -# # TODOCFA: what about [] or np.array([])? -# -# # 'index' is an integer or a scalar numpy/dask array -# raise ValueError( -# f"Can't subspace {self.__class__.__name__} with a " -# f"rank-reducing index: {index!r}" -# ) -# -# if has_ellipsis: -# # Replace Ellipsis with one or more slices -# indices2 = [] -# length = len(indices) -# n = self.ndim -# for index in indices: -# if index is Ellipsis: -# m = n - length + 1 -# indices2.extend([slice(None)] * m) -# n -= m -# else: -# indices2.append(i) -# n -= 1 -# -# length -= 1 -# -# indices = indices2 -# -# for i, (index, n) in enumerate(zip(indices, shape)): -# if index == slice(None): -# indices[i] = slice(0, n) -# -# return tuple(indices) -# -# def _conform_units(self, array): -# """Conform the array to have the aggregated units. -# -# .. versionadded:: 3.14.0 -# -# :Parameters: -# -# array: `numpy.ndarray` or `dict` -# The array to be conformed. If *array* is a `dict` with -# `numpy` array values then each value is conformed. -# -# :Returns: -# -# `numpy.ndarray` or `dict` -# The conformed array. The returned array may or may not -# be the input array updated in-place, depending on its -# data type and the nature of its units and the -# aggregated units. -# -# If *array* is a `dict` then a dictionary of conformed -# arrays is returned. -# -# """ -# units = self.Units -# if units: -# aggregated_units = self.aggregated_Units -# if not units.equivalent(aggregated_units): -# raise ValueError( -# f"Can't convert fragment data with units {units!r} to " -# f"have aggregated units {aggregated_units!r}" -# ) -# -# if units != aggregated_units: -# if isinstance(array, dict): -# # 'array' is a dictionary -# for key, value in array.items(): -# if key in _active_chunk_methds: -# array[key] = Units.conform( -# value, units, aggregated_units, inplace=True -# ) -# else: -# # 'array' is a numpy array -# array = Units.conform( -# array, units, aggregated_units, inplace=True -# ) -# -# return array -# -# @property -# def aggregated_Units(self): -# """The units of the aggregated data. -# -# .. versionadded:: 3.14.0 -# -# :Returns: -# -# `Units` -# The units of the aggregated data. -# -# """ -# return Units( -# self.get_aggregated_units(), self.get_aggregated_calendar(None) -# ) -# -# def get_aggregated_calendar(self, default=ValueError()): -# """The calendar of the aggregated array. -# -# If the calendar is `None` then the CF default calendar is -# assumed, if applicable. -# -# .. versionadded:: 3.14.0 -# -# :Parameters: -# -# default: optional -# Return the value of the *default* parameter if the -# calendar has not been set. If set to an `Exception` -# instance then it will be raised instead. -# -# :Returns: -# -# `str` or `None` -# The calendar value. -# -# """ -# calendar = self._get_component("aggregated_calendar", False) -# if calendar is False: -# if default is None: -# return -# -# return self._default( -# default, -# f"{self.__class__.__name__} 'aggregated_calendar' has not " -# "been set", -# ) -# -# return calendar -# -# def get_aggregated_units(self, default=ValueError()): -# """The units of the aggregated array. -# -# If the units are `None` then the aggregated array has no -# defined units. -# -# .. versionadded:: 3.14.0 -# -# .. seealso:: `get_aggregated_calendar` -# -# :Parameters: -# -# default: optional -# Return the value of the *default* parameter if the -# units have not been set. If set to an `Exception` -# instance then it will be raised instead. -# -# :Returns: -# -# `str` or `None` -# The units value. -# -# """ -# units = self._get_component("aggregated_units", False) -# if units is False: -# if default is None: -# return -# -# return self._default( -# default, -# f"{self.__class__.__name__} 'aggregated_units' have not " -# "been set", -# ) -# -# return units -# -# -# def get_array(self): -# """The fragment array. -# -# .. versionadded:: 3.14.0 -# -# :Returns: -# -# Subclass of `Array` -# The object defining the fragment array. -# -# """ -# return self._get_component("array") -# -# def get_units(self, default=ValueError()): -# """The units of the netCDF variable. -# -# .. versionadded:: (cfdm) 1.10.0.1 -# -# .. seealso:: `get_calendar` -# -# :Parameters: -# -# default: optional -# Return the value of the *default* parameter if the -# units have not been set. If set to an `Exception` -# instance then it will be raised instead. -# -# :Returns: -# -# `str` or `None` -# The units value. -# -# """ -# return self.get_array().get_units(default) From 5ae26a7981a8b7f8ac6dab6645b6a324a5ce51e6 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 18 Apr 2023 08:54:20 +0100 Subject: [PATCH 070/141] Improved docstring --- cf/domain.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cf/domain.py b/cf/domain.py index 9e551431a7..78eff09ab9 100644 --- a/cf/domain.py +++ b/cf/domain.py @@ -333,10 +333,12 @@ def del_file_location( def file_locations( self, ): - """The locations of files containing parts of the data. + """The locations of files containing parts of the components data. Returns the locations of any files that may be required to - deliver the computed data array. + deliver the computed data arrays of any of the component + constructs (such as dimension coordinate constructs, cell + measure constructs, etc.). .. versionadded:: TODOCFAVER From 37ff8a69fc17d01a927eed4350ce044032abaa4b Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 18 Apr 2023 17:49:29 +0100 Subject: [PATCH 071/141] Remove dead code Co-authored-by: Sadie L. Bartholomew --- cf/data/fragment/umfragmentarray.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cf/data/fragment/umfragmentarray.py b/cf/data/fragment/umfragmentarray.py index f47e212977..4eae7c95d5 100644 --- a/cf/data/fragment/umfragmentarray.py +++ b/cf/data/fragment/umfragmentarray.py @@ -1,6 +1,3 @@ -# from urllib.parse import urlparse - -# from ...umread_lib.umfile import File from ..array.umarray import UMArray from .mixin import FragmentArrayMixin From f76821fa04ea190ab6a187f19164168c38c0774a Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 18 Apr 2023 17:49:44 +0100 Subject: [PATCH 072/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/aggregate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/aggregate.py b/cf/aggregate.py index 907a9e0d67..7249f36d70 100644 --- a/cf/aggregate.py +++ b/cf/aggregate.py @@ -1382,7 +1382,7 @@ def find_coordrefs(self, key): return tuple(sorted(names)) def promote_to_auxiliary_coordinate(self, properties): - """Promote properties to auxilliary coordinate constructs. + """Promote properties to auxiliary coordinate constructs. Each property is converted to a 1-d auxilliary coordinate construct that spans a new independent size 1 domain axis the From 702f5b2cce91f13742744591a0ccb6b41a63be0e Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 18 Apr 2023 17:49:57 +0100 Subject: [PATCH 073/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/data/array/umarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/array/umarray.py b/cf/data/array/umarray.py index 1ac874f5b8..973c8529d6 100644 --- a/cf/data/array/umarray.py +++ b/cf/data/array/umarray.py @@ -38,7 +38,7 @@ def __init__( The file name(s). address: (sequence of) `int`, optional - The start position in the file(s) of the header(s) + The start position in the file(s) of the header(s). .. versionadded:: TODOCFAVER From bd95b686b99691aec3d55d3223436b3e8c6d1730 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 18 Apr 2023 17:51:54 +0100 Subject: [PATCH 074/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/aggregate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cf/aggregate.py b/cf/aggregate.py index 7249f36d70..31d28138ea 100644 --- a/cf/aggregate.py +++ b/cf/aggregate.py @@ -1385,8 +1385,8 @@ def promote_to_auxiliary_coordinate(self, properties): """Promote properties to auxiliary coordinate constructs. Each property is converted to a 1-d auxilliary coordinate - construct that spans a new independent size 1 domain axis the - field, and the property is deleted. + construct that spans a new independent size 1 domain axis of + the field, and the property is deleted. ... versionadded:: TODOCFAVER From 91bde24a2b123734504ca45dc85ebdf7c88afde1 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 18 Apr 2023 17:52:25 +0100 Subject: [PATCH 075/141] Remove dead code Co-authored-by: Sadie L. Bartholomew --- cf/data/fragment/netcdffragmentarray.py | 34 ------------------------- 1 file changed, 34 deletions(-) diff --git a/cf/data/fragment/netcdffragmentarray.py b/cf/data/fragment/netcdffragmentarray.py index e45a739201..7fb4e30a42 100644 --- a/cf/data/fragment/netcdffragmentarray.py +++ b/cf/data/fragment/netcdffragmentarray.py @@ -104,37 +104,3 @@ def __init__( ) -# def open(self): -# """Returns an open dataset containing the data array. -# -# When multiple fragment files have been provided an attempt is -# made to open each one, in arbitrary order, and the -# `netCDF4.Dataset` is returned from the first success. -# -# .. versionadded:: TODOCFAVER -# -# :Returns: -# -# `netCDF4.Dataset` -# -# """ -# # Loop round the files, returning as soon as we find one that -# # works. -# filenames = self.get_filenames() -# for filename, address in zip(filenames, self.get_addresses()): -# url = urlparse(filename) -# if url.scheme == "file": -# # Convert file URI into an absolute path -# filename = url.path -# -# try: -# nc = netCDF4.Dataset(filename, "r") -# except FileNotFoundError: -# continue -# except RuntimeError as error: -# raise RuntimeError(f"{error}: {filename}") -# -# self._set_component("ncvar", address, copy=False) -# return nc -# -# raise FileNotFoundError(f"No such netCDF fragment files: {filenames}") From b28fe8ac2e218f08f7592bc571a1c3ad565d9b4a Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 18 Apr 2023 17:52:39 +0100 Subject: [PATCH 076/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/data/fragment/mixin/fragmentarraymixin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/fragment/mixin/fragmentarraymixin.py b/cf/data/fragment/mixin/fragmentarraymixin.py index 1e5107b639..443c127acc 100644 --- a/cf/data/fragment/mixin/fragmentarraymixin.py +++ b/cf/data/fragment/mixin/fragmentarraymixin.py @@ -131,7 +131,7 @@ def _conform_to_aggregated_units(self, array): def _parse_indices(self, indices): """Parse the indices that retrieve the fragment data. - Ellipses are replaced with the approriate number `slice` + Ellipses are replaced with the approriate number of `slice` instances, and rank-reducing indices (such as an integer or scalar array) are disallowed. From 5f4ac3581f542ef20b549a2d64a2843cdee5a3bc Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 18 Apr 2023 17:53:55 +0100 Subject: [PATCH 077/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/aggregate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/aggregate.py b/cf/aggregate.py index 31d28138ea..c98d9f9354 100644 --- a/cf/aggregate.py +++ b/cf/aggregate.py @@ -1752,7 +1752,7 @@ def aggregate( which has one or more of the given properties. For each input field, each property is converted to a field ancillary construct that spans the entire domain, with the - constant value of the propertyand the property itself is + constant value of the property, and the property itself is deleted. .. versionadded:: TODOCFAVER From 5f2cba294927965749305d39874b75fa256f2de4 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 18 Apr 2023 17:56:56 +0100 Subject: [PATCH 078/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/aggregate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/aggregate.py b/cf/aggregate.py index c98d9f9354..a55b04d05f 100644 --- a/cf/aggregate.py +++ b/cf/aggregate.py @@ -1445,7 +1445,7 @@ def promote_to_field_ancillary(self, properties): a CF-netCDF ancillary variable. If a domain construct is being aggregated then it is always - returned unchanged + returned unchanged. ... versionadded:: TODOCFAVER From 2e70f1605486d726ac7852ac0eb380f150b1ef7b Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 18 Apr 2023 17:57:14 +0100 Subject: [PATCH 079/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/aggregate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cf/aggregate.py b/cf/aggregate.py index a55b04d05f..5f2ecd9951 100644 --- a/cf/aggregate.py +++ b/cf/aggregate.py @@ -1439,8 +1439,8 @@ def promote_to_field_ancillary(self, properties): The `Data` of any the new field ancillary construct is marked as a CFA term, meaning that it will only be written to disk if - the parent field construct is written as CFA aggregation - variable, and in that case the field ancillary is written as + the parent field construct is written as a CFA aggregation + variable, and in that case the field ancillary is written as a non-standard CFA aggregation instruction variable, rather than a CF-netCDF ancillary variable. From 77907915c924c46b78bc446d657d5f138f04b820 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 18 Apr 2023 17:57:32 +0100 Subject: [PATCH 080/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/aggregate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/aggregate.py b/cf/aggregate.py index 5f2ecd9951..9b17100214 100644 --- a/cf/aggregate.py +++ b/cf/aggregate.py @@ -1437,7 +1437,7 @@ def promote_to_field_ancillary(self, properties): ancillary construct that spans the entire domain, with the constant value of the property. - The `Data` of any the new field ancillary construct is marked + The `Data` of any new field ancillary construct is marked as a CFA term, meaning that it will only be written to disk if the parent field construct is written as a CFA aggregation variable, and in that case the field ancillary is written as a From 0c7f7ab7e022abd3c00c916243439e1dfcc2cee2 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 18 Apr 2023 18:04:34 +0100 Subject: [PATCH 081/141] byte string -> unicode --- cf/test/setup_create_field.py | 37 ++++++++++++++++------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/cf/test/setup_create_field.py b/cf/test/setup_create_field.py index 6440bd372f..5f6280d07d 100644 --- a/cf/test/setup_create_field.py +++ b/cf/test/setup_create_field.py @@ -68,26 +68,23 @@ def test_create_field(self): ) aux3.standard_name = "longitude" - aux4 = cf.AuxiliaryCoordinate( - data=cf.Data( - numpy.array( - [ - "alpha", - "beta", - "gamma", - "delta", - "epsilon", - "zeta", - "eta", - "theta", - "iota", - "kappa", - ], - dtype="S", - ) - ) + array = numpy.ma.array( + [ + "alpha", + "beta", + "gamma", + "delta", + "epsilon", + "zeta", + "eta", + "theta", + "iota", + "kappa", + ], ) - aux4.standard_name = "greek_letters" + array[0] = numpy.ma.masked + aux4 = cf.AuxiliaryCoordinate(data=cf.Data(array)) + aux4.set_property("long_name", "greek_letters") # ------------------------------------------------------------ # TODO: Re-instate this line when @@ -203,7 +200,7 @@ def test_create_field(self): g = cf.read(self.filename, squeeze=True, verbose=0)[0] self.assertTrue( - g.equals(f, verbose=0), "Field not equal to itself read back in" + g.equals(f, verbose=1), "Field not equal to itself read back in" ) x = g.dump(display=False) From ca18292b1b3eb71a9e05dd0b7449754cbece9bcb Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 18 Apr 2023 18:06:35 +0100 Subject: [PATCH 082/141] byte string -> unicode --- cf/test/setup_create_field.py | 37 +++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/cf/test/setup_create_field.py b/cf/test/setup_create_field.py index 5f6280d07d..34e4bdcd58 100644 --- a/cf/test/setup_create_field.py +++ b/cf/test/setup_create_field.py @@ -68,23 +68,26 @@ def test_create_field(self): ) aux3.standard_name = "longitude" - array = numpy.ma.array( - [ - "alpha", - "beta", - "gamma", - "delta", - "epsilon", - "zeta", - "eta", - "theta", - "iota", - "kappa", - ], + aux4 = cf.AuxiliaryCoordinate( + data=cf.Data( + numpy.array( + [ + "alpha", + "beta", + "gamma", + "delta", + "epsilon", + "zeta", + "eta", + "theta", + "iota", + "kappa", + ], + dtype="U", + ) + ) ) - array[0] = numpy.ma.masked - aux4 = cf.AuxiliaryCoordinate(data=cf.Data(array)) - aux4.set_property("long_name", "greek_letters") + aux4.standard_name = "greek_letters" # ------------------------------------------------------------ # TODO: Re-instate this line when @@ -200,7 +203,7 @@ def test_create_field(self): g = cf.read(self.filename, squeeze=True, verbose=0)[0] self.assertTrue( - g.equals(f, verbose=1), "Field not equal to itself read back in" + g.equals(f, verbose=0), "Field not equal to itself read back in" ) x = g.dump(display=False) From eedf03378aa6c31b9f1bde7e2313003cf951baa3 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 18 Apr 2023 19:08:45 +0100 Subject: [PATCH 083/141] dev --- cf/data/fragment/netcdffragmentarray.py | 2 -- cf/test/individual_tests.sh | 9 +++++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cf/data/fragment/netcdffragmentarray.py b/cf/data/fragment/netcdffragmentarray.py index 7fb4e30a42..0c369f6289 100644 --- a/cf/data/fragment/netcdffragmentarray.py +++ b/cf/data/fragment/netcdffragmentarray.py @@ -102,5 +102,3 @@ def __init__( self._set_component( "aggregated_calendar", aggregated_calendar, copy=False ) - - diff --git a/cf/test/individual_tests.sh b/cf/test/individual_tests.sh index 67bbfd91b8..f02a941197 100755 --- a/cf/test/individual_tests.sh +++ b/cf/test/individual_tests.sh @@ -1,12 +1,16 @@ #!/bin/bash -python create_test_files.py +file=create_test_files.py +echo "Running $file" +python $file rc=$? if [[ $rc != 0 ]]; then exit $rc fi -python setup_create_field.py +file=setup_create_field.py +echo "Running $file" +python $file rc=$? if [[ $rc != 0 ]]; then exit $rc @@ -14,6 +18,7 @@ fi for file in test_*.py do + echo "Running $file" python $file rc=$? if [[ $rc != 0 ]]; then From 5261e063e4972259bb4bba96a6e1c024ef83d0ac Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:08:41 +0100 Subject: [PATCH 084/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/cfimplementation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/cfimplementation.py b/cf/cfimplementation.py index 8bba5ca84e..a5f3035971 100644 --- a/cf/cfimplementation.py +++ b/cf/cfimplementation.py @@ -100,7 +100,7 @@ def initialise_CFANetCDFArray( filename: `str` - address: (sequence of) `str or `int`` + address: (sequence of) `str` or `int` dytpe: `numpy.dtype` From 5ba66cfd948f17e850f2933a0193041a0a561c61 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:09:05 +0100 Subject: [PATCH 085/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/mixin2/cfanetcdf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/mixin2/cfanetcdf.py b/cf/mixin2/cfanetcdf.py index ad1c97c425..9dcb002448 100644 --- a/cf/mixin2/cfanetcdf.py +++ b/cf/mixin2/cfanetcdf.py @@ -363,7 +363,7 @@ def cfa_file_substitutions(self): .. seealso:: `cfa_clear_file_substitutions`, `cfa_del_file_substitution`, `cfa_file_substitutions`, - `cfa_update_file_substitution` + `cfa_update_file_substitutions` :Returns: `dict` From 42391c234ae87b9aff6c65ba0e04571682b2969a Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:09:21 +0100 Subject: [PATCH 086/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/read_write/write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/read_write/write.py b/cf/read_write/write.py index cea6be035a..253d9d6467 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -644,7 +644,7 @@ def write( paths relative to the location of the CFA-netCDF file being created. - * ``'strict'``:`bool` + * ``'strict'``: `bool` If True (the default) then an exception is raised if it is not possible to create a CFA aggregation variable From b3a71c711a8b69d75b3c77bbc0755ab82ab5694f Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:09:41 +0100 Subject: [PATCH 087/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/read_write/write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/read_write/write.py b/cf/read_write/write.py index 253d9d6467..49f034cb46 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -634,7 +634,7 @@ def write( Only write auxiliary coordinate constructs with two or more dimensions as CFA-netCDF variables, and also all field constructs: ``{'field': None, - 'auxiliary_coordinate': cf.ge(2)}}``. + 'auxiliary_coordinate': cf.ge(2)}``. * ``'absolute_paths'``: `bool` From 785ea0e2d1a89254c8b42bcbfbf84bd728b5d370 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:09:53 +0100 Subject: [PATCH 088/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/mixin2/cfanetcdf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/mixin2/cfanetcdf.py b/cf/mixin2/cfanetcdf.py index 9dcb002448..df01b4ef08 100644 --- a/cf/mixin2/cfanetcdf.py +++ b/cf/mixin2/cfanetcdf.py @@ -362,7 +362,7 @@ def cfa_file_substitutions(self): .. seealso:: `cfa_clear_file_substitutions`, `cfa_del_file_substitution`, - `cfa_file_substitutions`, + `cfa_has_file_substitutions`, `cfa_update_file_substitutions` :Returns: From c0fdf72f5e5a93cbd12e9e98d8cdda84a71b3bab Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:10:04 +0100 Subject: [PATCH 089/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/read_write/write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/read_write/write.py b/cf/read_write/write.py index 49f034cb46..0732219f45 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -628,7 +628,7 @@ def write( Equivalent ways to only write two-dimensional auxiliary coordinate constructs as CFA-netCDF aggregation variables: ``{'auxiliary_coordinate': - 2}}`` and ``{'auxiliary_coordinate': cf.eq(2)}}``. + 2}`` and ``{'auxiliary_coordinate': cf.eq(2)}``. *Example:* Only write auxiliary coordinate constructs with two or From af589ce7f65951d0364b8d2b3063db15615e64b6 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:10:21 +0100 Subject: [PATCH 090/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/read_write/write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/read_write/write.py b/cf/read_write/write.py index 0732219f45..16eff20aca 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -606,7 +606,7 @@ def write( parameter. Alternatively, the same types may be given as keys to a `dict` whose values specify the number of dimensions that a construct must also have if it is to - be written as CFA-netCDF aggregation variable. A value + be written as a CFA-netCDF aggregation variable. A value of `None` means no restriction on the number of dimensions, which is equivalent to a value of ``cf.ge(0)``. From 16ad8dbac0a809fdcc7d36b1816eddd87a22d16a Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:10:49 +0100 Subject: [PATCH 091/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/read_write/read.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/read_write/read.py b/cf/read_write/read.py index bbe6fdd2ee..852220b199 100644 --- a/cf/read_write/read.py +++ b/cf/read_write/read.py @@ -658,7 +658,7 @@ def read( CFA aggregation instruction variable. *Example:* - ``{'base': 'file:///data/'}}`` + ``{'base': 'file:///data/'}`` .. versionadded:: TODOCFAVER From 283911f45f9110544440f566f0dfd937d12c74ca Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:11:20 +0100 Subject: [PATCH 092/141] Remove dead code Co-authored-by: Sadie L. Bartholomew --- cf/data/data.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cf/data/data.py b/cf/data/data.py index 52db13c17d..2a3d2dadfb 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -437,11 +437,6 @@ def __init__( pass else: self._set_Array(array) - # if is_abstract_Array_subclass(array): - # # Save the input array in case it's useful later. For - # # compressed input arrays this will contain extra information, - # # such as a count or index variable. - # self._set_Array(array) # Cast the input data as a dask array kwargs = init_options.get("from_array", {}) From c8598d39078db23e2f6dc0cf4272de3e65b1e26c Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:11:43 +0100 Subject: [PATCH 093/141] Remove dead code Co-authored-by: Sadie L. Bartholomew --- cf/data/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/data.py b/cf/data/data.py index 2a3d2dadfb..cace5877b7 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -425,7 +425,7 @@ def __init__( "for compressed input arrays" ) - if to_memory: # and is_file_array(array): + if to_memory: try: array = array.to_memory() except AttributeError: From 7553a535a6afdeca53b0c3e086ba84bf505dd289 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:12:03 +0100 Subject: [PATCH 094/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/data/fragment/mixin/fragmentarraymixin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/fragment/mixin/fragmentarraymixin.py b/cf/data/fragment/mixin/fragmentarraymixin.py index 443c127acc..e49873c41a 100644 --- a/cf/data/fragment/mixin/fragmentarraymixin.py +++ b/cf/data/fragment/mixin/fragmentarraymixin.py @@ -151,7 +151,7 @@ def _parse_indices(self, indices): >>> a.shape (12, 1, 73, 144) - >>> a._parse_indices(([2, 4, 5], Ellipsis, slice(45, 67)) + >>> a._parse_indices([2, 4, 5], Ellipsis, slice(45, 67)) [[2, 4, 5], slice(0, 1), slice(0, 73), slice(45, 67)] >>> a._parse_indices(([2, 4, 5], [0], slice(None), slice(45, 67)) [[2, 4, 5], [0], slice(0, 73), slice(45, 67)] From 44244ab0529dc5e0eb12b28e4e7b2a7ab0eaa540 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:12:19 +0100 Subject: [PATCH 095/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/data/fragment/mixin/fragmentarraymixin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/fragment/mixin/fragmentarraymixin.py b/cf/data/fragment/mixin/fragmentarraymixin.py index e49873c41a..7b77a91fbe 100644 --- a/cf/data/fragment/mixin/fragmentarraymixin.py +++ b/cf/data/fragment/mixin/fragmentarraymixin.py @@ -153,7 +153,7 @@ def _parse_indices(self, indices): (12, 1, 73, 144) >>> a._parse_indices([2, 4, 5], Ellipsis, slice(45, 67)) [[2, 4, 5], slice(0, 1), slice(0, 73), slice(45, 67)] - >>> a._parse_indices(([2, 4, 5], [0], slice(None), slice(45, 67)) + >>> a._parse_indices([2, 4, 5], [0], slice(None), slice(45, 67)) [[2, 4, 5], [0], slice(0, 73), slice(45, 67)] """ From 1031ad2f06dde427f839a7edb8ac7fdd14649db8 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:13:11 +0100 Subject: [PATCH 096/141] Remove dead code Co-authored-by: Sadie L. Bartholomew --- cf/data/fragment/netcdffragmentarray.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cf/data/fragment/netcdffragmentarray.py b/cf/data/fragment/netcdffragmentarray.py index 0c369f6289..1405fc3608 100644 --- a/cf/data/fragment/netcdffragmentarray.py +++ b/cf/data/fragment/netcdffragmentarray.py @@ -1,7 +1,3 @@ -# from urllib.parse import urlparse - -# import netCDF4 - from ..array.netcdfarray import NetCDFArray from .mixin import FragmentArrayMixin From ac40c4c0941e9c2698a97f51d031ae788e75f03a Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:15:23 +0100 Subject: [PATCH 097/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/data/fragment/netcdffragmentarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/fragment/netcdffragmentarray.py b/cf/data/fragment/netcdffragmentarray.py index 1405fc3608..578412f124 100644 --- a/cf/data/fragment/netcdffragmentarray.py +++ b/cf/data/fragment/netcdffragmentarray.py @@ -30,7 +30,7 @@ def __init__( The names of the netCDF fragment files containing the array. - address: (sequence of `str`0, optional + address: (sequence of `str`), optional The name of the netCDF variable containing the fragment array. Required unless *varid* is set. From 7b63ca437c9fba53884cb297592eb4734bf38f52 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:16:46 +0100 Subject: [PATCH 098/141] Docs Co-authored-by: Sadie L. Bartholomew --- cf/data/fragment/umfragmentarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/fragment/umfragmentarray.py b/cf/data/fragment/umfragmentarray.py index 4eae7c95d5..07bc6df753 100644 --- a/cf/data/fragment/umfragmentarray.py +++ b/cf/data/fragment/umfragmentarray.py @@ -27,7 +27,7 @@ def __init__( :Parameters: filenames: (sequence of `str`), optional - The names of the UM or PP file containing the fragment. + The names of the UM or PP files containing the fragment. addresses: (sequence of `str`), optional The start words in the files of the header. From acdfbc0cae295508365058d587d8c0f5fc8abf22 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:17:07 +0100 Subject: [PATCH 099/141] docs --- cf/data/fragment/umfragmentarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/fragment/umfragmentarray.py b/cf/data/fragment/umfragmentarray.py index 07bc6df753..a971b9bd4b 100644 --- a/cf/data/fragment/umfragmentarray.py +++ b/cf/data/fragment/umfragmentarray.py @@ -26,7 +26,7 @@ def __init__( :Parameters: - filenames: (sequence of `str`), optional + filename: (sequence of `str`), optional The names of the UM or PP files containing the fragment. addresses: (sequence of `str`), optional From e19bc87e24da28bcddda3cdf623b55bcd42490b7 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:17:21 +0100 Subject: [PATCH 100/141] docs --- cf/data/fragment/umfragmentarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/fragment/umfragmentarray.py b/cf/data/fragment/umfragmentarray.py index a971b9bd4b..a30737f46d 100644 --- a/cf/data/fragment/umfragmentarray.py +++ b/cf/data/fragment/umfragmentarray.py @@ -29,7 +29,7 @@ def __init__( filename: (sequence of `str`), optional The names of the UM or PP files containing the fragment. - addresses: (sequence of `str`), optional + address: (sequence of `str`), optional The start words in the files of the header. dtype: `numpy.dtype` From b70d054788a10208f49b4582f336af370a3edbf1 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:19:50 +0100 Subject: [PATCH 101/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/read_write/write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/read_write/write.py b/cf/read_write/write.py index 16eff20aca..88d54be804 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -669,7 +669,7 @@ def write( aggregation instruction variable. *Example:* - ``{'base': 'file:///data/'}}`` + ``{'base': 'file:///data/'}`` .. versionadded:: TODOCFAVER From 7e34daf4d41ee38e9c1bd534f8d7207373eba6e5 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 08:20:10 +0100 Subject: [PATCH 102/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/read_write/write.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/read_write/write.py b/cf/read_write/write.py index 88d54be804..9333e78c3c 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -583,7 +583,7 @@ def write( .. versionadded:: 3.14.0 - cfa: `bool` or `dict`, otional + cfa: `bool` or `dict`, optional If True or a (possibly empty) dictionary then write the constructs as CFA-netCDF aggregation variables, where possible and where requested. From b87733ef960cf222a9cea109b0d6512c587a9132 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 09:03:34 +0100 Subject: [PATCH 103/141] CFA format docs --- cf/read_write/write.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/cf/read_write/write.py b/cf/read_write/write.py index 9333e78c3c..bba9f173d0 100644 --- a/cf/read_write/write.py +++ b/cf/read_write/write.py @@ -138,14 +138,14 @@ def write( fmt: `str`, optional The format of the output file. One of: - ========================== ================================ + ========================== ============================== *fmt* Output file type - ========================== ================================ - ``'NETCDF4'`` NetCDF4 format file. This is the - default. + ========================== ============================== + ``'NETCDF4'`` NetCDF4 format file. This is + the default. - ``'NETCDF4_CLASSIC'`` NetCDF4 classic format file (see - below) + ``'NETCDF4_CLASSIC'`` NetCDF4 classic format file + (see below) ``'NETCDF3_CLASSIC'`` NetCDF3 classic format file (limited to file sizes less @@ -158,14 +158,17 @@ def write( ``'NETCDF3_64BIT_OFFSET'`` ``'NETCDF3_64BIT_DATA'`` NetCDF3 64-bit offset format - file with extensions (see below) + file with extensions (see + below) - ``'CFA'`` or ``'CFA4'`` Deprecated at version TODOCFAVER. - Use the *cfa* parameter instead. + ``'CFA'`` or ``'CFA4'`` Deprecated at version + TODOCFAVER. See the *cfa* + parameter. - ``'CFA3'`` Deprecated at version TODOCFAVER. - Use the *cfa* parameter instead. - ========================== ================================ + ``'CFA3'`` Deprecated at version + TODOCFAVER. See the *cfa* + parameter. + ========================== ============================== By default the format is ``'NETCDF4'``. @@ -588,6 +591,9 @@ def write( constructs as CFA-netCDF aggregation variables, where possible and where requested. + The netCDF format of the CFA-netCDF file is determined by + the *fmt* parameter, as usual. + If *cfa* is a dictionary then it is used to configure the CFA write process. The default options when CFA writing is enabled are ``{'constructs': 'field', 'absolute_paths': From 0db992e36ef915a5422a0153f967aba190496c14 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 11:32:55 +0100 Subject: [PATCH 104/141] deprecate cfa_base --- scripts/cfa | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/scripts/cfa b/scripts/cfa index 80b0b925da..9d61f4e915 100755 --- a/scripts/cfa +++ b/scripts/cfa @@ -294,22 +294,7 @@ containing the source property values. . .TP .B \-\-cfa_base=[value] -For output CFA\-netCDF files only. File names referenced by an output -CFA\-netCDF file have relative, as opposed to absolute, paths or URL -bases. This may be useful when relocating a CFA\-netCDF file together -with the datasets referenced by it. -.PP -.RS -If set with no value (\-\-cfa_base=) or the value is empty then file -names are given relative to the directory or URL base containing the -output CFA\-netCDF file. If set with a non\-empty value then file -names are given relative to the directory or URL base described by the -value. -.PP -By default, file names within CFA\-netCDF files are stored with -absolute paths. Ignored for output files of any other format. -.RE -.RE +Deprecated. Use \-\-cfa_paths instead. . . .TP @@ -1089,7 +1074,6 @@ Written by David Hassell [--reference_datetime] Override coordinate reference date-times [--overwrite] Overwrite pre-existing output files [--unlimited=axis Create an unlimited dimension - [--cfa_base=[value]] Configure CFA-netCDF output files [--cfa_paths=[value]] Configure CFA-netCDF output files [--single] Write out as single precision [--double] Write out as double precision From 8c5848c67328c0d910318469c542a2c593644970 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Thu, 20 Apr 2023 11:45:42 +0100 Subject: [PATCH 105/141] deprecate cfa_base, fix cfa_paths --- scripts/cfa | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/cfa b/scripts/cfa index 9d61f4e915..c411cd7bf9 100755 --- a/scripts/cfa +++ b/scripts/cfa @@ -298,14 +298,14 @@ Deprecated. Use \-\-cfa_paths instead. . . .TP -.B \-\-cfa_paths=[value] +.B \-\-cfa_paths=value For output CFA\-netCDF files only. File names referenced by an output CFA\-netCDF file have relative, as opposed to absolute, paths or URL bases. This may be useful when relocating a CFA\-netCDF file together with the datasets referenced by it. .PP .RS -If set with no value (\-\-cfa_base=) or the value is empty then file +If set with no value (\-\-cfa_paths=) or the value is empty then file names are given relative to the directory or URL base containing the output CFA\-netCDF file. If set with a non\-empty value then file names are given relative to the directory or URL base described by the @@ -1074,7 +1074,7 @@ Written by David Hassell [--reference_datetime] Override coordinate reference date-times [--overwrite] Overwrite pre-existing output files [--unlimited=axis Create an unlimited dimension - [--cfa_paths=[value]] Configure CFA-netCDF output files + [--cfa_paths=value] Configure CFA-netCDF output files [--single] Write out as single precision [--double] Write out as double precision [--compress=N] Compress the output data @@ -1107,7 +1107,7 @@ Using cf-python library version {cf.__version__} at {library_path}""" "1aDd:f:hino:r:s:uv:x:", longopts=[ "axis=", - "cfa_base=", + "cfa_paths=", "compress=", "contiguous", "Directory", @@ -1155,6 +1155,7 @@ Using cf-python library version {cf.__version__} at {library_path}""" "read=", "write=", "verbose=", + "cfa_base=", ], ) except GetoptError as err: From 18683a3a2c0c17c871756d0fbdbb192aaee22f7b Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:04:47 +0100 Subject: [PATCH 106/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/data/fragment/mixin/fragmentarraymixin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/fragment/mixin/fragmentarraymixin.py b/cf/data/fragment/mixin/fragmentarraymixin.py index 7b77a91fbe..c2c549423b 100644 --- a/cf/data/fragment/mixin/fragmentarraymixin.py +++ b/cf/data/fragment/mixin/fragmentarraymixin.py @@ -32,7 +32,7 @@ def __getitem__(self, indices): .. versionadded:: TODOCFAVER """ - # TODOACTIVE: modify this the for case when + # TODOACTIVE: modify this for the case when # super().__getitem__(tuple(indices)) returns a # dictionary From 6dc72d3620e70ef61dc75d80ed27729bacd891dd Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:07:16 +0100 Subject: [PATCH 107/141] Mark comment with TODOCFA Co-authored-by: Sadie L. Bartholomew --- cf/data/array/umarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/array/umarray.py b/cf/data/array/umarray.py index 973c8529d6..18f5cb30e9 100644 --- a/cf/data/array/umarray.py +++ b/cf/data/array/umarray.py @@ -272,7 +272,7 @@ def _get_rec(self, f, header_offset): The record container. """ - # This method doesn't require data_offset and disk_length, + # TODOCFA: This method doesn't require data_offset and disk_length, # so plays nicely with CFA. Is it fast enough that we can # use this method always? for v in f.vars: From 4755345e9e4e3f8c4dc295eee64523f6f61ae0cc Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:07:50 +0100 Subject: [PATCH 108/141] Docs clarity Co-authored-by: Sadie L. Bartholomew --- cf/read_write/netcdf/netcdfwrite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index cc703ebcb5..9b7a23fb9b 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -1015,7 +1015,7 @@ def _cfa_get_file_details(self, data): :Returns: - `set` + `set` of 3-tuples The file names, the addresses in the files, and the file formats. If no files are required to compute the data then an empty `set` is returned. From d52741ecd0e94b045c3f8bec05a04ada64bac651 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:09:07 +0100 Subject: [PATCH 109/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/read_write/netcdf/netcdfwrite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 9b7a23fb9b..468fb757e5 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -844,7 +844,7 @@ def _cfa_aggregation_instructions(self, data, cfvar): `dict` A dictionary whose keys are the standardised CFA - aggregation instruction terms, keyed by `Data` + aggregation instruction terms, with values of `Data` instances containing the corresponding variables. **Examples** From 912a50dfb5aadd1461008f37a3010980639a7685 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:10:34 +0100 Subject: [PATCH 110/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/data/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/data.py b/cf/data/data.py index cace5877b7..add0381acb 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1277,7 +1277,7 @@ def _clear_after_dask_update(self, clear=_ALL): """Remove components invalidated by updating the `dask` array. Removes or modifies components that can't be guaranteed to be - consistent with an updated `dask` array`. See the *clear* + consistent with an updated `dask` array. See the *clear* parameter for details. .. versionadded:: 3.14.0 From 03c2fade70a2545e385b3b7acf89501d10de84b0 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:10:58 +0100 Subject: [PATCH 111/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/read_write/netcdf/netcdfwrite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 468fb757e5..2fc8f2ea05 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -812,7 +812,7 @@ def _cfa_unique(cls, a): `numpy.ndarray` A size 1 array containing the unique value, or missing - data if there is not a unique unique value. + data if there is not a unique value. """ out_shape = (1,) * a.ndim From 5a5620f676d3d52a3d6330e460641bcb1c4461e2 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:11:55 +0100 Subject: [PATCH 112/141] Docs clarity Co-authored-by: Sadie L. Bartholomew --- cf/data/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/data.py b/cf/data/data.py index add0381acb..08e7893065 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -1304,7 +1304,7 @@ def _clear_after_dask_update(self, clear=_ALL): status is set to `False`. * If ``clear`` is non-zero then the CFA term status is - set to False. + set to `False`. By default *clear* is the ``_ALL`` integer-valued constant, which results in all components being From 7cba5400d0570a72369cbebad8b34231030bb4ab Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:12:20 +0100 Subject: [PATCH 113/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/data/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/data.py b/cf/data/data.py index 08e7893065..3ab25ba6e4 100644 --- a/cf/data/data.py +++ b/cf/data/data.py @@ -3899,7 +3899,7 @@ def concatenate(cls, data, axis=0, cull_graph=True, relaxed_units=False): # Set the CFA write status # - # Assume at first that all input data instance have True + # Assume at first that all input data instances have True # status, but ... cfa = _CFA for d in processed_data: From 9901c47d864e9690db5aa290b4dad70391fb5a4f Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:14:37 +0100 Subject: [PATCH 114/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/data/array/fullarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/array/fullarray.py b/cf/data/array/fullarray.py index c362a627fb..9cf58ce577 100644 --- a/cf/data/array/fullarray.py +++ b/cf/data/array/fullarray.py @@ -283,7 +283,7 @@ def unique( axis=axis, ) - # Fast unique based in the full value + # Fast unique based on the full value x = a.get_full_value() if x is np.ma.masked: return np.ma.masked_all((1,), dtype=a.dtype) From f702bf5f64e84f3b26ea0ae5d610e1d99cd03312 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:26:16 +0100 Subject: [PATCH 115/141] Docs improvement Co-authored-by: Sadie L. Bartholomew --- cf/read_write/netcdf/netcdfwrite.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 2fc8f2ea05..f4ff503cf3 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -1016,9 +1016,10 @@ def _cfa_get_file_details(self, data): :Returns: `set` of 3-tuples - The file names, the addresses in the files, and the - file formats. If no files are required to compute the - data then an empty `set` is returned. + A set containing 3-tuples giving the file names, + the addresses in the files, and the file formats. If + no files are required to compute the data then + an empty `set` is returned. **Examples** From 73cd14e56382fa4b9cc778d738451ed9b8ea83d2 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:27:04 +0100 Subject: [PATCH 116/141] Remove dead code Co-authored-by: Sadie L. Bartholomew --- cf/read_write/um/umread.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cf/read_write/um/umread.py b/cf/read_write/um/umread.py index 829d6e01e1..a54203cc98 100644 --- a/cf/read_write/um/umread.py +++ b/cf/read_write/um/umread.py @@ -38,7 +38,6 @@ _cached_time = {} _cached_ctime = {} _cached_size_1_height_coordinate = {} -# _cached_z_reference_coordinate = {} _cached_date2num = {} _cached_model_level_number_coordinate = {} From f4f15aad4fcb90c3c8d656410e3343a5e0f9e6b3 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:27:36 +0100 Subject: [PATCH 117/141] Remove dead code Co-authored-by: Sadie L. Bartholomew --- cf/read_write/um/umread.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cf/read_write/um/umread.py b/cf/read_write/um/umread.py index a54203cc98..8980d75082 100644 --- a/cf/read_write/um/umread.py +++ b/cf/read_write/um/umread.py @@ -1076,8 +1076,6 @@ def __init__( for cm in cell_methods: self.implementation.set_cell_method(field, cm) - # # Check for decreasing axes that aren't decreasing - # down_axes = self.down_axes logger.info(f"down_axes = {self.down_axes}") # pragma: no cover # print (down_axes ) From 98e8d23ba76684bf8a6b08dde15c93b0d35d18a5 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:28:00 +0100 Subject: [PATCH 118/141] Remove dead code Co-authored-by: Sadie L. Bartholomew --- cf/read_write/um/umread.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cf/read_write/um/umread.py b/cf/read_write/um/umread.py index 8980d75082..99a90134fc 100644 --- a/cf/read_write/um/umread.py +++ b/cf/read_write/um/umread.py @@ -1078,9 +1078,6 @@ def __init__( logger.info(f"down_axes = {self.down_axes}") # pragma: no cover - # print (down_axes ) - # if down_axes: - # field.flip(down_axes, inplace=True) # Force cyclic X axis for particular values of LBHEM if xkey is not None and int_hdr[lbhem] in (0, 1, 2, 4): From 1e543ad86f7d44688811bd295ee4e8cabb98b949 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:28:37 +0100 Subject: [PATCH 119/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/read_write/um/umread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/read_write/um/umread.py b/cf/read_write/um/umread.py index 99a90134fc..f072a96917 100644 --- a/cf/read_write/um/umread.py +++ b/cf/read_write/um/umread.py @@ -1140,7 +1140,7 @@ def _reorder_z_axis(self, indices, z_axis, pmaxes): details. z_axis: `int` - The identifier of the Z axis + The identifier of the Z axis. pmaxes: sequence of `int` The aggregation axes, which include the Z axis. From c8591c33531eea2df97f8d5a8172c57f5ff933d1 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:29:59 +0100 Subject: [PATCH 120/141] Remove dead code Co-authored-by: Sadie L. Bartholomew --- cf/read_write/um/umread.py | 64 -------------------------------------- 1 file changed, 64 deletions(-) diff --git a/cf/read_write/um/umread.py b/cf/read_write/um/umread.py index f072a96917..8e18fbb4f4 100644 --- a/cf/read_write/um/umread.py +++ b/cf/read_write/um/umread.py @@ -3220,70 +3220,6 @@ def z_coordinate(self, axiscode): return dc -# @_manage_log_level_via_verbose_attr -# def z_reference_coordinate(self, axiscode): -# """Create and return the Z reference coordinates.""" -# logger.info( -# "Creating Z reference coordinates from BRLEV" -# ) # pragma: no cover -# -# array = np.array( -# [rec.real_hdr.item(brlev) for rec in self.z_recs], dtype=float -# ) -# -# LBVC = self.lbvc -# atol = self.atol -# print(99999) -# key = (axiscode, LBVC, array) -# dc = _cached_z_reference_coordinate.get(key) -# -# if dc is not None: -# copy = True -# else: -# if not 128 <= LBVC <= 139: -# bounds = [] -# for rec in self.z_recs: -# BRLEV = rec.real_hdr.item(brlev) -# BRSVD1 = rec.real_hdr.item(brsvd1) -# -# if abs(BRSVD1 - BRLEV) >= atol: -# bounds = None -# break -# -# bounds.append((BRLEV, BRSVD1)) -# else: -# bounds = None -# -# if bounds: -# bounds = np.array((bounds,), dtype=float) -# -# dc = self.implementation.initialise_DimensionCoordinate() -# dc = self.coord_data( -# dc, -# array, -# bounds, -# units=_axiscode_to_Units.setdefault(axiscode, None), -# ) -# dc = self.coord_axis(dc, axiscode) -# dc = self.coord_names(dc, axiscode) -# -# if not dc.get("positive", True): # ppp -# dc.flip(inplace=True) -# -# _cached_z_reference_coordinate[key] = dc -# copy = False -# -# self.implementation.set_dimension_coordinate( -# self.field, -# dc, -# axes=[_axis["z"]], -# copy=copy, -# autocyclic=_autocyclic_false, -# ) -# -# return dc -# -# # _stash2standard_name = {} # # def load_stash2standard_name(table=None, delimiter='!', merge=True): From 3fb241584ab0272c36082feec448ed3790f9d3a0 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:33:03 +0100 Subject: [PATCH 121/141] docs clarity --- cf/data/array/cfanetcdfarray.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index 2a8c791274..df31d15325 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -42,11 +42,11 @@ def __init__( :Parameters: - filename: (sequence of `str`), optional + filename: (sequence of) `str`, optional The name of the CFA-netCDF file containing the array. If a sequence then it must contain one element. - address: (sequence of `str`0, optional + address: (sequence of) `str`, optional The name of the CFA-netCDF aggregation variable the array. If a sequence then it must contain one element. From 10a7d444e509e24815f8d961a811972fadbde379 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:34:30 +0100 Subject: [PATCH 122/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/data/array/cfanetcdfarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index df31d15325..63b7fe335d 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -47,7 +47,7 @@ def __init__( array. If a sequence then it must contain one element. address: (sequence of) `str`, optional - The name of the CFA-netCDF aggregation variable the + The name of the CFA-netCDF aggregation variable for the array. If a sequence then it must contain one element. shape: `tuple` of `int` From 7f040fa225bddef4f06f811464810e855eb66d4a Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:35:09 +0100 Subject: [PATCH 123/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/data/array/cfanetcdfarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index 63b7fe335d..dd1baa0031 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -91,7 +91,7 @@ def __init__( term: `str`, optional The name of a non-standard aggregation instruction term from which the array is to be created, instead of - the creating the aggregated data in the standard + creating the aggregated data in the standard terms. If set then *address* must be the name of the term's CFA-netCDF aggregation instruction variable, which must be defined on the fragment dimensions and From 3254e55f070d1bce06e8a9863077bbd43ea28292 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:35:51 +0100 Subject: [PATCH 124/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/data/array/cfanetcdfarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index dd1baa0031..661b25b285 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -170,7 +170,7 @@ def __init__( extra_dimension = f.ndim > ndim if extra_dimension: - # There is a extra non-fragment dimension + # There is an extra non-fragment dimension fragment_shape = f.shape[:-1] else: fragment_shape = f.shape From f85981f2fd58a8e85a2d55751e69714b2b5485d7 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:40:45 +0100 Subject: [PATCH 125/141] docs clarity --- cf/data/array/cfanetcdfarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index 661b25b285..9b5cc1ad55 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -318,7 +318,7 @@ def get_aggregated_data(self, copy=True): return aggregated_data def get_fragmented_dimensions(self): - """Get the positions dimension that have two or more fragments. + """Get the positions of dimensions that have two or more fragments. .. versionadded:: 3.14.0 From d18b8dbe90ddbb2227e9da986b371fa844a07c0a Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:41:00 +0100 Subject: [PATCH 126/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/data/array/cfanetcdfarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index 661b25b285..8efb532b3a 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -348,7 +348,7 @@ def get_fragment_shape(self): """Get the sizes of the fragment dimensions. The fragment dimension sizes are given in the same order as - the aggregated dimension sizes given by `shape` + the aggregated dimension sizes given by `shape`. .. versionadded:: 3.14.0 From d0ad8cec4132299ff4e1299df33575bfc85a90cf Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:45:14 +0100 Subject: [PATCH 127/141] comment clarity --- cf/read_write/netcdf/netcdfread.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index be83a60a42..b31b643737 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -239,7 +239,10 @@ def _create_data( calendar=kwargs["calendar"], ) - # Note: We don't cache elements from CFA variables + # Note: We don't cache elements from CFA variables, because + # the data are in fragment files which have not been + # opened; and may not not even be openable, such as + # could be the case if a fragement was on tape storage. # Set the CFA write status to True iff each non-aggregated # axis has exactly one dask storage chunk From 84e2f1d07fa60a8173a25e1894351f91180bf25a Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:49:01 +0100 Subject: [PATCH 128/141] docs fix --- cf/read_write/netcdf/netcdfread.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index b31b643737..f67c3c3902 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -313,10 +313,7 @@ def _create_Data( ncvar: `str` The netCDF variable containing the array. - ncdimensions: sequence of `str`, optional - The netCDF dimensions spanned by the array. - - .. versionadded:: 3.14.fill_value: + units: `str`, optional The units of *array*. By default, or if `None`, it is assumed that there are no units. @@ -324,6 +321,11 @@ def _create_Data( The calendar of *array*. By default, or if `None`, it is assumed that there is no calendar. + ncdimensions: sequence of `str`, optional + The netCDF dimensions spanned by the array. + + .. versionadded:: 3.14.0 + kwargs: optional Extra parameters to pass to the initialisation of the returned `Data` object. From af45ce54e8d6e588eeae21c6d94a235c399493bd Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:52:03 +0100 Subject: [PATCH 129/141] Typo Co-authored-by: Sadie L. Bartholomew --- cf/read_write/netcdf/netcdfread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index f67c3c3902..7060425999 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -381,7 +381,7 @@ def _customise_read_vars(self): # ------------------------------------------------------------ if g["CFA_version"] < Version("0.6.2"): raise ValueError( - f"Can't read file {g['filename']} that uses obselete " + f"Can't read file {g['filename']} that uses obsolete " f"CFA conventions version CFA-{g['CFA_version']}. " "(Note that version 3.13.1 can be used to read and " "write CFA-0.4 files.)" From ce6a1bdcdcdac8a509d13ee36cea4b9cc4ea48f4 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:52:36 +0100 Subject: [PATCH 130/141] Exception message clarity Co-authored-by: Sadie L. Bartholomew --- cf/read_write/netcdf/netcdfread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 7060425999..3561810c29 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -383,7 +383,7 @@ def _customise_read_vars(self): raise ValueError( f"Can't read file {g['filename']} that uses obsolete " f"CFA conventions version CFA-{g['CFA_version']}. " - "(Note that version 3.13.1 can be used to read and " + "(Note that cf version 3.13.1 can be used to read and " "write CFA-0.4 files.)" ) From a7901e89017a8c00e700dcc931ed4f0080ef8328 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 10:57:27 +0100 Subject: [PATCH 131/141] Typos Co-authored-by: Sadie L. Bartholomew --- cf/read_write/netcdf/netcdfread.py | 2 -- cf/read_write/netcdf/netcdfwrite.py | 14 +++++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/cf/read_write/netcdf/netcdfread.py b/cf/read_write/netcdf/netcdfread.py index 3561810c29..2f3eed94d0 100644 --- a/cf/read_write/netcdf/netcdfread.py +++ b/cf/read_write/netcdf/netcdfread.py @@ -414,8 +414,6 @@ def _customise_read_vars(self): ncvar, attributes.get("aggregated_data") ) for term_ncvar in parsed_aggregated_data.values(): - # term, term_ncvar = tuple(x.items())[0] - # term_ncvar = term_ncvar[0] g["do_not_create_field"].add(term_ncvar) def _cache_data_elements(self, data, ncvar): diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index f4ff503cf3..69dc937e24 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -38,7 +38,7 @@ def _write_as_cfa(self, cfvar, construct_type, domain_axes): .. versionadded:: TODOCFAVER domain_axes: `None`, or `tuple` of `str` - The domain axis construct identidifiers for *cfvar*. + The domain axis construct identifiers for *cfvar*. .. versionadded:: TODOCFAVER @@ -106,7 +106,7 @@ def _customise_createVariable( .. versionadded:: TODOCFAVER domain_axes: `None`, or `tuple` of `str` - The domain axis construct identidifiers for *cfvar*. + The domain axis construct identifiers for *cfvar*. .. versionadded:: TODOCFAVER @@ -157,7 +157,7 @@ def _write_data( ncdimensions: `tuple` of `str` domain_axes: `None`, or `tuple` of `str` - The domain axis construct identidifiers for *cfvar*. + The domain axis construct identifiers for *cfvar*. .. versionadded:: TODOCFAVER @@ -256,7 +256,7 @@ def _write_dimension_coordinate( The name of the netCDF dimension for this dimension coordinate construct, including any groups structure. Note that the group structure may be - different to the corodinate variable, and the + different to the coordinate variable, and the basename. .. versionadded:: 3.6.0 @@ -720,9 +720,9 @@ def _cfa_write_term_variable( def _cfa_write_non_standard_terms( self, field, fragment_ncdimensions, aggregated_data ): - """ "Write a non-standard CFA aggregation instruction term variable + """Write a non-standard CFA aggregation instruction term variable. - Wites non-standard CFA terms stored as field ancillaries + Writes non-standard CFA terms stored as field ancillaries. .. versionadded:: TODOCFAVER @@ -798,7 +798,7 @@ def _cfa_write_non_standard_terms( def _cfa_unique(cls, a): """Return the unique value of an array. - If there are multipl unique vales then missing data is + If there are multiple unique vales then missing data is returned. .. versionadded:: TODOCFAVER From 9b9ea5c2dec7b4d6082a7cf2a35df485c22f4ed0 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 11:49:26 +0100 Subject: [PATCH 132/141] CFA --- Changelog.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index 128681e201..398f317dc9 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -1,16 +1,20 @@ -version 3.14.2 +version 3.15.0 -------------- -**2023-??-??** +**2023-04-??** +* Re-introduction of CFA-netCDF functionality for CFA-0.6 + (https://github.com/NCAS-CMS/cf-python/issues/451, + https://github.com/NCAS-CMS/cf-python/issues/475, + https://github.com/NCAS-CMS/cf-python/issues/637) * New function: `cf.CFA` * New method: `cf.Data.get_cfa_write` * New method: `cf.Data.set_cfa_write` -* Changed dependency: ``?????<=cfdm Date: Mon, 24 Apr 2023 11:50:15 +0100 Subject: [PATCH 133/141] cfdm version --- docs/source/installation.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/installation.rst b/docs/source/installation.rst index a7a5be668b..353354d821 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -201,8 +201,8 @@ Required * `cftime `_, version 1.6.0 or newer (note that this package may be installed with netCDF4). -* `cfdm `_, version 1.10.0.3 or up to, - but not including, 1.10.1.0. +* `cfdm `_, version 1.10.1.0 or up to, + but not including, 1.10.2.0. * `cfunits `_, version 3.3.5 or newer. From a70e08e9fc7293f1bbf7d916a6cfd914592d2d95 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 12:01:39 +0100 Subject: [PATCH 134/141] Remove redundant code Co-authored-by: Sadie L. Bartholomew --- cf/read_write/netcdf/netcdfwrite.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 69dc937e24..6eba4ca026 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -140,7 +140,6 @@ def _write_data( compressed=False, attributes={}, construct_type=None, - warn_invalid=None, ): """Write a Data object. From 5eda8e9b46ab08bb720510879eed8ed3a2eccb9a Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 12:09:38 +0100 Subject: [PATCH 135/141] test method docstrings --- cf/test/test_CFA.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cf/test/test_CFA.py b/cf/test/test_CFA.py index f51715f92d..0fbe4b4996 100644 --- a/cf/test/test_CFA.py +++ b/cf/test/test_CFA.py @@ -49,6 +49,7 @@ class CFATest(unittest.TestCase): netcdf_fmts = netcdf3_fmts + netcdf4_fmts def test_CFA_fmt(self): + """Test the cf.read 'fmt' and 'cfa' keywords.""" f = cf.example_field(0) cf.write(f, tmpfile1) f = cf.read(tmpfile1)[0] @@ -60,6 +61,7 @@ def test_CFA_fmt(self): self.assertTrue(f.equals(g[0])) def test_CFA_multiple_fragments(self): + """Test CFA with more than one fragment.""" f = cf.example_field(0) cf.write(f[:2], tmpfile1) @@ -82,6 +84,7 @@ def test_CFA_multiple_fragments(self): self.assertTrue(n[0].equals(c[0])) def test_CFA_strict(self): + """Test CFA 'strict' option to the cfa.write 'cfa' keyword.""" f = cf.example_field(0) # By default, can't write as CF-netCDF those variables @@ -103,6 +106,7 @@ def test_CFA_strict(self): self.assertTrue(g[0].equals(f)) def test_CFA_field_ancillaries(self): + """Test creation of field ancillaries from non-standard CFA terms.""" f = cf.example_field(0) self.assertFalse(f.field_ancillaries()) @@ -150,6 +154,7 @@ def test_CFA_field_ancillaries(self): self.assertTrue(e[0].equals(d)) def test_CFA_substitutions_0(self): + """Test CFA substitution URI substitutions (0).""" f = cf.example_field(0) cf.write(f, tmpfile1) f = cf.read(tmpfile1)[0] @@ -180,6 +185,7 @@ def test_CFA_substitutions_0(self): self.assertTrue(f.equals(g[0])) def test_CFA_substitutions_1(self): + """Test CFA substitution URI substitutions (1).""" f = cf.example_field(0) cf.write(f, tmpfile1) f = cf.read(tmpfile1)[0] @@ -208,6 +214,7 @@ def test_CFA_substitutions_1(self): self.assertTrue(f.equals(g[0])) def test_CFA_substitutions_2(self): + """Test CFA substitution URI substitutions (2).""" f = cf.example_field(0) cf.write(f, tmpfile1) f = cf.read(tmpfile1)[0] @@ -290,6 +297,7 @@ def test_CFA_substitutions_2(self): self.assertTrue(f.equals(g[0])) def test_CFA_absolute_paths(self): + """Test CFA 'absolute_paths' option to the cfa.write 'cfa' keyword.""" f = cf.example_field(0) cf.write(f, tmpfile1) f = cf.read(tmpfile1)[0] @@ -313,6 +321,7 @@ def test_CFA_absolute_paths(self): self.assertTrue(f.equals(g[0])) def test_CFA_constructs(self): + """Test choice of constructs to write as CFA-netCDF variables.""" f = cf.example_field(1) f.del_construct("T") f.del_construct("long_name=Grid latitude name") @@ -402,6 +411,7 @@ def test_CFA_constructs(self): nc.close() def test_CFA_PP(self): + """Test writing CFA-netCDF with PP format fragments.""" f = cf.read("file1.pp")[0] cf.write(f, tmpfile1, cfa=True) @@ -424,6 +434,7 @@ def test_CFA_PP(self): self.assertTrue(f.equals(g[0])) def test_CFA_multiple_files(self): + """Test storing multiple CFA frgament locations.""" tmpfile1 = "delme1.nc" tmpfile2 = "delme2.nc" f = cf.example_field(0) From 5f32490b04640f3a341ab33827c287b7e64a88dd Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 12:27:00 +0100 Subject: [PATCH 136/141] comment on CFA-netCDF file directory path --- cf/read_write/netcdf/netcdfwrite.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py index 6eba4ca026..757ac91fcc 100644 --- a/cf/read_write/netcdf/netcdfwrite.py +++ b/cf/read_write/netcdf/netcdfwrite.py @@ -998,9 +998,9 @@ def _customise_write_vars(self): from os.path import abspath from pathlib import PurePath - g["cfa_dir"] = PurePath( - abspath(g["filename"]) - ).parent # TODOCFA??? + # Find the absolute directory path of the output + # CFA-netCDF file URI + g["cfa_dir"] = PurePath(abspath(g["filename"])).parent def _cfa_get_file_details(self, data): """Get the details of all files referenced by the data. From 36a6313dd0cf0462fca65eff2d5a481417e0e9e2 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 12:38:41 +0100 Subject: [PATCH 137/141] Remove redundant docstring Co-authored-by: Sadie L. Bartholomew --- cf/data/array/cfanetcdfarray.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cf/data/array/cfanetcdfarray.py b/cf/data/array/cfanetcdfarray.py index cbfd703e08..bef6fbffb3 100644 --- a/cf/data/array/cfanetcdfarray.py +++ b/cf/data/array/cfanetcdfarray.py @@ -50,9 +50,6 @@ def __init__( The name of the CFA-netCDF aggregation variable for the array. If a sequence then it must contain one element. - shape: `tuple` of `int` - The shape of the aggregated data array. - dtype: `numpy.dtype` The data type of the aggregated data array. May be `None` if the numpy data-type is not known (which can From b815e735c2dadd31c6e826eeb3f62273ae06a565 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 12:55:55 +0100 Subject: [PATCH 138/141] CFA methods docs --- docs/source/class/cf.AuxiliaryCoordinate.rst | 18 +++++++++++++ docs/source/class/cf.Bounds.rst | 17 ++++++++++++ docs/source/class/cf.CellMeasure.rst | 17 ++++++++++++ docs/source/class/cf.Count.rst | 18 +++++++++++++ docs/source/class/cf.Data.rst | 27 ++++++++++++++++++++ docs/source/class/cf.DimensionCoordinate.rst | 18 +++++++++++++ docs/source/class/cf.Domain.rst | 18 +++++++++++++ docs/source/class/cf.DomainAncillary.rst | 20 ++++++++++++++- docs/source/class/cf.Field.rst | 20 ++++++++++++++- docs/source/class/cf.FieldAncillary.rst | 17 ++++++++++++ docs/source/class/cf.Index.rst | 17 ++++++++++++ docs/source/class/cf.List.rst | 17 ++++++++++++ docs/source/class/cf.NetCDFArray.rst | 11 +++++++- 13 files changed, 232 insertions(+), 3 deletions(-) diff --git a/docs/source/class/cf.AuxiliaryCoordinate.rst b/docs/source/class/cf.AuxiliaryCoordinate.rst index 228394b7ef..f697317e41 100644 --- a/docs/source/class/cf.AuxiliaryCoordinate.rst +++ b/docs/source/class/cf.AuxiliaryCoordinate.rst @@ -508,6 +508,24 @@ Groups ~cf.AuxiliaryCoordinate.nc_set_variable_groups ~cf.AuxiliaryCoordinate.nc_clear_variable_groups +CFA +--- + +.. rubric:: Methods + +.. autosummary:: + :nosignatures: + :toctree: ../method/ + :template: method.rst + + ~cf.AuxiliaryCoordinate.add_file_location + ~cf.AuxiliaryCoordinate.cfa_clear_file_substitutions + ~cf.AuxiliaryCoordinate.cfa_del_file_substitution + ~cf.AuxiliaryCoordinate.cfa_file_substitutions + ~cf.AuxiliaryCoordinate.cfa_update_file_substitutions + ~cf.AuxiliaryCoordinate.del_file_location + ~cf.AuxiliaryCoordinate.file_locations + Aliases ------- diff --git a/docs/source/class/cf.Bounds.rst b/docs/source/class/cf.Bounds.rst index 2cdb1214c1..94b2bf2f42 100644 --- a/docs/source/class/cf.Bounds.rst +++ b/docs/source/class/cf.Bounds.rst @@ -405,6 +405,23 @@ NetCDF ~cf.Bounds.nc_has_dimension ~cf.Bounds.nc_set_dimension +CFA +--- + +.. rubric:: Methods + +.. autosummary:: + :nosignatures: + :toctree: ../method/ + :template: method.rst + + ~cf.Bounds.add_file_location + ~cf.Bounds.cfa_clear_file_substitutions + ~cf.Bounds.cfa_del_file_substitution + ~cf.Bounds.cfa_file_substitutions + ~cf.Bounds.cfa_update_file_substitutions + ~cf.Bounds.del_file_location + ~cf.Bounds.file_locations Aliases ------- diff --git a/docs/source/class/cf.CellMeasure.rst b/docs/source/class/cf.CellMeasure.rst index 466a7e348c..cff23ee73e 100644 --- a/docs/source/class/cf.CellMeasure.rst +++ b/docs/source/class/cf.CellMeasure.rst @@ -425,6 +425,23 @@ NetCDF ~cf.CellMeasure.nc_get_external ~cf.CellMeasure.nc_set_external +CFA +--- + +.. rubric:: Methods + +.. autosummary:: + :nosignatures: + :toctree: ../method/ + :template: method.rst + + ~cf.CellMeasure.add_file_location + ~cf.CellMeasure.cfa_clear_file_substitutions + ~cf.CellMeasure.cfa_del_file_substitution + ~cf.CellMeasure.cfa_file_substitutions + ~cf.CellMeasure.cfa_update_file_substitutions + ~cf.CellMeasure.del_file_location + ~cf.CellMeasure.file_locations Aliases ------- diff --git a/docs/source/class/cf.Count.rst b/docs/source/class/cf.Count.rst index f35a115757..47d9c4509c 100644 --- a/docs/source/class/cf.Count.rst +++ b/docs/source/class/cf.Count.rst @@ -402,6 +402,24 @@ NetCDF ~cf.Count.nc_has_sample_dimension ~cf.Count.nc_set_sample_dimension +CFA +--- + +.. rubric:: Methods + +.. autosummary:: + :nosignatures: + :toctree: ../method/ + :template: method.rst + + ~cf.Count.add_file_location + ~cf.Count.cfa_clear_file_substitutions + ~cf.Count.cfa_del_file_substitution + ~cf.Count.cfa_file_substitutions + ~cf.Count.cfa_update_file_substitutions + ~cf.Count.del_file_location + ~cf.Count.file_locations + Aliases ------- diff --git a/docs/source/class/cf.Data.rst b/docs/source/class/cf.Data.rst index feb33ab2c0..afe0f56dcb 100644 --- a/docs/source/class/cf.Data.rst +++ b/docs/source/class/cf.Data.rst @@ -70,6 +70,7 @@ Dask ~cf.Data.cull_graph ~cf.Data.dask_compressed_array ~cf.Data.rechunk + ~cf.Data.chunk_indices ~cf.Data.todict ~cf.Data.to_dask_array @@ -646,6 +647,32 @@ Performance ~cf.Data.npartitions ~cf.Data.numblocks + +CFA +--- + +.. autosummary:: + :nosignatures: + :toctree: ../method/ + :template: method.rst + + ~cf.Data.add_file_location + ~cf.Data.cfa_clear_file_substitutions + ~cf.Data.cfa_del_aggregated_data + ~cf.Data.cfa_del_file_substitution + ~cf.Data.cfa_file_substitutions + ~cf.Data.cfa_get_aggregated_data + ~cf.Data.cfa_get_term + ~cf.Data.cfa_get_write + ~cf.Data.cfa_has_aggregated_data + ~cf.Data.cfa_has_file_substitutions + ~cf.Data.cfa_set_aggregated_data + ~cf.Data.cfa_set_term + ~cf.Data.cfa_set_write + ~cf.Data.cfa_update_file_substitutions + ~cf.Data.del_file_location + ~cf.Data.file_locations + Element-wise arithmetic, bit and comparison operations ------------------------------------------------------ diff --git a/docs/source/class/cf.DimensionCoordinate.rst b/docs/source/class/cf.DimensionCoordinate.rst index 5b1d81a3d4..94b372776e 100644 --- a/docs/source/class/cf.DimensionCoordinate.rst +++ b/docs/source/class/cf.DimensionCoordinate.rst @@ -516,6 +516,24 @@ Groups ~cf.DimensionCoordinate.nc_set_variable_groups ~cf.DimensionCoordinate.nc_clear_variable_groups +CFA +--- + +.. rubric:: Methods + +.. autosummary:: + :nosignatures: + :toctree: ../method/ + :template: method.rst + + ~cf.DimensionCoordinate.add_file_location + ~cf.DimensionCoordinate.cfa_clear_file_substitutions + ~cf.DimensionCoordinate.cfa_del_file_substitution + ~cf.DimensionCoordinate.cfa_file_substitutions + ~cf.DimensionCoordinate.cfa_update_file_substitutions + ~cf.DimensionCoordinate.del_file_location + ~cf.DimensionCoordinate.file_locations + Aliases ------- diff --git a/docs/source/class/cf.Domain.rst b/docs/source/class/cf.Domain.rst index 6a8d424c01..0655568037 100644 --- a/docs/source/class/cf.Domain.rst +++ b/docs/source/class/cf.Domain.rst @@ -251,6 +251,24 @@ Groups ~cf.Domain.nc_set_group_attribute ~cf.Domain.nc_set_group_attributes +CFA +--- + +.. rubric:: Methods + +.. autosummary:: + :nosignatures: + :toctree: ../method/ + :template: method.rst + + ~cf.Domain.add_file_location + ~cf.Domain.cfa_clear_file_substitutions + ~cf.Domain.cfa_del_file_substitution + ~cf.Domain.cfa_file_substitutions + ~cf.Domain.cfa_update_file_substitutions + ~cf.Domain.del_file_location + ~cf.Domain.file_locations + Geometries ^^^^^^^^^^ diff --git a/docs/source/class/cf.DomainAncillary.rst b/docs/source/class/cf.DomainAncillary.rst index a478ef869d..2313f367eb 100644 --- a/docs/source/class/cf.DomainAncillary.rst +++ b/docs/source/class/cf.DomainAncillary.rst @@ -452,8 +452,26 @@ NetCDF ~cf.DomainAncillary.nc_del_variable ~cf.DomainAncillary.nc_get_variable ~cf.DomainAncillary.nc_has_variable - ~cf.DomainAncillary.nc_set_variable + ~cf.DomainAncillary.nc_set_variable +CFA +--- + +.. rubric:: Methods + +.. autosummary:: + :nosignatures: + :toctree: ../method/ + :template: method.rst + + ~cf.DomainAncillary.add_file_location + ~cf.DomainAncillary.cfa_clear_file_substitutions + ~cf.DomainAncillary.cfa_del_file_substitution + ~cf.DomainAncillary.cfa_file_substitutions + ~cf.DomainAncillary.cfa_update_file_substitutions + ~cf.DomainAncillary.del_file_location + ~cf.DomainAncillary.file_locations + Aliases ------- diff --git a/docs/source/class/cf.Field.rst b/docs/source/class/cf.Field.rst index 28a6133615..b82562117c 100644 --- a/docs/source/class/cf.Field.rst +++ b/docs/source/class/cf.Field.rst @@ -426,7 +426,25 @@ Groups ~cf.Field.nc_clear_group_attributes ~cf.Field.nc_set_group_attribute ~cf.Field.nc_set_group_attributes - + +CFA +^^^ + +.. rubric:: Methods + +.. autosummary:: + :nosignatures: + :toctree: ../method/ + :template: method.rst + + ~cf.Field.add_file_location + ~cf.Field.cfa_clear_file_substitutions + ~cf.Field.cfa_del_file_substitution + ~cf.Field.cfa_file_substitutions + ~cf.Field.cfa_update_file_substitutions + ~cf.Field.del_file_location + ~cf.Field.file_locations + Geometries ^^^^^^^^^^ diff --git a/docs/source/class/cf.FieldAncillary.rst b/docs/source/class/cf.FieldAncillary.rst index a9c14b3a13..0e53ca16f5 100644 --- a/docs/source/class/cf.FieldAncillary.rst +++ b/docs/source/class/cf.FieldAncillary.rst @@ -399,6 +399,23 @@ NetCDF ~cf.FieldAncillary.nc_has_variable ~cf.FieldAncillary.nc_set_variable +CFA +--- + +.. rubric:: Methods + +.. autosummary:: + :nosignatures: + :toctree: ../method/ + :template: method.rst + + ~cf.FieldAncillary.add_file_location + ~cf.FieldAncillary.cfa_clear_file_substitutions + ~cf.FieldAncillary.cfa_del_file_substitution + ~cf.FieldAncillary.cfa_file_substitutions + ~cf.FieldAncillary.cfa_update_file_substitutions + ~cf.FieldAncillary.del_file_location + ~cf.FieldAncillary.file_locations Aliases ------- diff --git a/docs/source/class/cf.Index.rst b/docs/source/class/cf.Index.rst index 81405432b2..1f680f9477 100644 --- a/docs/source/class/cf.Index.rst +++ b/docs/source/class/cf.Index.rst @@ -403,6 +403,23 @@ NetCDF ~cf.Index.nc_has_sample_dimension ~cf.Index.nc_set_sample_dimension +CFA +--- + +.. rubric:: Methods + +.. autosummary:: + :nosignatures: + :toctree: ../method/ + :template: method.rst + + ~cf.Index.add_file_location + ~cf.Index.cfa_clear_file_substitutions + ~cf.Index.cfa_del_file_substitution + ~cf.Index.cfa_file_substitutions + ~cf.Index.cfa_update_file_substitutions + ~cf.Index.del_file_location + ~cf.Index.file_locations Aliases ------- diff --git a/docs/source/class/cf.List.rst b/docs/source/class/cf.List.rst index 1a27978da7..3bcc834a01 100644 --- a/docs/source/class/cf.List.rst +++ b/docs/source/class/cf.List.rst @@ -395,6 +395,23 @@ NetCDF ~cf.List.nc_has_variable ~cf.List.nc_set_variable +CFA +--- + +.. rubric:: Methods + +.. autosummary:: + :nosignatures: + :toctree: ../method/ + :template: method.rst + + ~cf.List.add_file_location + ~cf.List.cfa_clear_file_substitutions + ~cf.List.cfa_del_file_substitution + ~cf.List.cfa_file_substitutions + ~cf.List.cfa_update_file_substitutions + ~cf.List.del_file_location + ~cf.List.file_locations Aliases ------- diff --git a/docs/source/class/cf.NetCDFArray.rst b/docs/source/class/cf.NetCDFArray.rst index 9df7e9fda3..34d7bf0d65 100644 --- a/docs/source/class/cf.NetCDFArray.rst +++ b/docs/source/class/cf.NetCDFArray.rst @@ -17,14 +17,22 @@ cf.NetCDFArray :toctree: ../method/ :template: method.rst + ~cf.NetCDFArray.add_file_location ~cf.NetCDFArray.close ~cf.NetCDFArray.copy + ~cf.NetCDFArray.del_file_location + ~cf.NetCDFArray.file_locations + ~cf.NetCDFArray.filename ~cf.NetCDFArray.get_address + ~cf.NetCDFArray.get_addresses + ~cf.NetCDFArray.get_format + ~cf.NetCDFArray.get_formats ~cf.NetCDFArray.get_calendar ~cf.NetCDFArray.get_compression_type ~cf.NetCDFArray.get_filename ~cf.NetCDFArray.get_filenames ~cf.NetCDFArray.get_group + ~cf.NetCDFArray.get_groups ~cf.NetCDFArray.get_mask ~cf.NetCDFArray.get_missing_values ~cf.NetCDFArray.get_ncvar @@ -33,7 +41,8 @@ cf.NetCDFArray ~cf.NetCDFArray.get_varid ~cf.NetCDFArray.open ~cf.NetCDFArray.to_memory - + ~cf.NetCDFArray.Units + .. rubric:: Attributes .. autosummary:: From a51a24c026207647e0461af5308faecb6794e2a5 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 12:56:14 +0100 Subject: [PATCH 139/141] linting --- cf/read_write/um/umread.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cf/read_write/um/umread.py b/cf/read_write/um/umread.py index 8e18fbb4f4..04ef52d8bf 100644 --- a/cf/read_write/um/umread.py +++ b/cf/read_write/um/umread.py @@ -1078,7 +1078,6 @@ def __init__( logger.info(f"down_axes = {self.down_axes}") # pragma: no cover - # Force cyclic X axis for particular values of LBHEM if xkey is not None and int_hdr[lbhem] in (0, 1, 2, 4): field.cyclic( From c6d7fec12e9d54cd50014b55f7959d07472b68a1 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 13:47:47 +0100 Subject: [PATCH 140/141] Typos --- cf/mixin2/cfanetcdf.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cf/mixin2/cfanetcdf.py b/cf/mixin2/cfanetcdf.py index df01b4ef08..0eb80c9d1f 100644 --- a/cf/mixin2/cfanetcdf.py +++ b/cf/mixin2/cfanetcdf.py @@ -20,7 +20,7 @@ def cfa_del_aggregated_data(self): """Remove the CFA-netCDF aggregation instruction terms. The aggregation instructions are stored in the - `aggregation_data` attribute of a CFA-netCDF aggregation + ``aggregation_data`` attribute of a CFA-netCDF aggregation variable. .. versionadded:: TODOCFAVER @@ -71,7 +71,7 @@ def cfa_get_aggregated_data(self): """Return the CFA-netCDF aggregation instruction terms. The aggregation instructions are stored in the - `aggregation_data` attribute of a CFA-netCDF aggregation + ``aggregation_data`` attribute of a CFA-netCDF aggregation variable. .. versionadded:: TODOCFAVER @@ -129,7 +129,7 @@ def cfa_has_aggregated_data(self): """Whether any CFA-netCDF aggregation instruction terms have been set. The aggregation instructions are stored in the - `aggregation_data` attribute of a CFA-netCDF aggregation + ``aggregation_data`` attribute of a CFA-netCDF aggregation variable. .. versionadded:: TODOCFAVER @@ -173,6 +173,7 @@ def cfa_has_aggregated_data(self): {} >>> f.cfa_get_aggregated_data() {} + """ return self._nc_has("cfa_aggregated_data") @@ -180,7 +181,7 @@ def cfa_set_aggregated_data(self, value): """Set the CFA-netCDF aggregation instruction terms. The aggregation instructions are stored in the - `aggregation_data` attribute of a CFA-netCDF aggregation + ``aggregation_data`` attribute of a CFA-netCDF aggregation variable. If there are any ``/`` (slash) characters in the netCDF From 31a53c42e1a83c7a3419b6b72b5731822a5f146c Mon Sep 17 00:00:00 2001 From: David Hassell Date: Mon, 24 Apr 2023 14:31:55 +0100 Subject: [PATCH 141/141] basic CFA references --- docs/source/tutorial.rst | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index a49e844acd..2ca8f7f1cb 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -119,10 +119,8 @@ elements. The following file types can be read: -* All formats of netCDF3 and netCDF4 files (including `CFA-netCDF - `_ files) - can be read, containing datasets for any version of CF up to and - including CF-|version|. +* All formats of netCDF3 and netCDF4 files can be read, containing + datasets for any version of CF up to and including CF-|version|. .. @@ -132,6 +130,12 @@ The following file types can be read: .. +* `CFA-netCDF + `_ + files at version 0.6 or later. + +.. + * :ref:`PP and UM fields files `, whose contents are mapped into field constructs. @@ -5256,6 +5260,10 @@ The `cf.write` function has optional parameters to * append to the netCDF file rather than over-writing it by default; +* write as a `CFA-netCDF + `_ + file. + * specify which field construct properties should become netCDF data variable attributes and which should become netCDF global attributes; @@ -5275,6 +5283,8 @@ The `cf.write` function has optional parameters to * set the endian-ness of the output data. +* omit the data arrays of selected constructs. + For example, to use the `mode` parameter to append a new field, or fields, to a netCDF file whilst preserving the field or fields already contained in that file: