Merge pull request #53 from podaac/release/1.3.1

Release/1.3.1

Author: Stepheny Perez

.github/workflows/build-pipeline.yml

@@ -127,7 +127,7 @@ jobs:
           tag: ${{ env.software_version }}
           message: "Version ${{ env.software_version }}"
       - name: Publish UMM-S with new version
-        uses: podaac/cmr-umm-updater@0.1.1
+        uses: podaac/cmr-umm-updater@0.2.1
         if: |
           github.ref == 'refs/heads/main'    ||
           startsWith(github.ref, 'refs/heads/release')
@@ -136,9 +136,11 @@ jobs:
           provider: 'POCLOUD'
           env: ${{ env.venue }}
           version: ${{ env.software_version }}
+          timeout: 60
         env:
-          cmr_user: ${{secrets.CMR_USER}}
-          cmr_pass: ${{secrets.CMR_PASS}}
+          LAUNCHPAD_TOKEN_SIT: ${{secrets.LAUNCHPAD_TOKEN_SIT}}
+          LAUNCHPAD_TOKEN_UAT: ${{secrets.LAUNCHPAD_TOKEN_UAT}}
+          LAUNCHPAD_TOKEN_OPS: ${{secrets.LAUNCHPAD_TOKEN_OPS}}
       - name: Build Docs
         run: |
           poetry run sphinx-build -b html ./docs docs/_build/

CHANGELOG.md

@@ -12,6 +12,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 ### Security
 
+## [1.3.1]
+### Added
+- [issues/50](https://github.com/podaac/l2ss-py/issues/50): Spatial bounds are computed correctly for grouped empty subset operations
+- Added `timeout` option to `cmr-umm-updater`
+### Changed 
+- Upgraded `cmr-umm-updater` to 0.2.1
+### Deprecated 
+### Removed
+### Fixed
+- [issues/48](https://github.com/podaac/l2ss-py/issues/48): get_epoch_time_var was not able to pick up the 'time' variable for the TROPOMI CH4 collection. Extra elif statement was added to get the full time variable returned.
+- [issues/54](https://github.com/podaac/l2ss-py/issues/54): Skip encoding when xr dataset is empty
+### Security
+
 ## [1.3.0]
 ### Added
 - [issues/27](https://github.com/podaac/l2ss-py/issues/27): Xarray is unable to handle variables with duplicate dimensions. Module dimension_cleanup.py added to handle variables that may have duplicate dimensions. Method remove_duplicate_dims() creates a new dimension identical dimension to the dimensions originally duplicated so the dimension does not need to be duplicated and can have the same shape and values.

cmr/ops_associations.txt

@@ -31,3 +31,7 @@ C2183155461-POCLOUD
 C2158344213-POCLOUD
 C2205618215-POCLOUD
 C2205618339-POCLOUD
+C2158348170-POCLOUD
+C2158348264-POCLOUD
+C2158350299-POCLOUD
+C2205121400-POCLOUD

cmr/uat_associations.txt

@@ -19,3 +19,6 @@ C1238538241-POCLOUD
 C1238658052-POCLOUD
 C1240739577-POCLOUD
 C1240739709-POCLOUD
+C1240739691-POCLOUD
+C1238621091-POCLOUD
+C1240739526-POCLOUD

podaac/subsetter/subset.py

@@ -525,6 +525,13 @@ def get_time_epoch_var(dataset, time_var_name):
         epoch_var_name = time_var.attrs['comment'].split('plus')[0].strip()
     elif 'time' in dataset.variables.keys() and time_var_name != 'time':
         epoch_var_name = 'time'
+    elif any('time' in s for s in list(dataset.variables.keys())) and time_var_name != 'time':
+        for i in list(dataset.variables.keys()):
+            group_list = i.split(GROUP_DELIM)
+            if group_list[-1] == 'time':
+                epoch_var_name = i
+                break
+        return epoch_var_name
     else:
         raise ValueError('Unable to determine time variables')
 
@@ -977,6 +984,8 @@ def _rename_variables(dataset, base_dataset):
 
         if variable.dtype == object:
             var_group.createVariable(new_var_name, 'S1', var_dims, fill_value=fill_value)
+        elif variable.dtype == 'timedelta64[ns]':
+            var_group.createVariable(new_var_name, 'i4', var_dims, fill_value=fill_value)
         else:
             var_group.createVariable(new_var_name, variable.dtype, var_dims, fill_value=fill_value)
 
@@ -1110,7 +1119,8 @@ def subset(file_to_subset, bbox, output_file, variables=None,  # pylint: disable
                 encoding = {}
                 compression = dict(zlib=True, complevel=5, _FillValue=None)
 
-                if (min_time or max_time) and any(dataset.dims.values()):
+                if (min_time or max_time) and not all(
+                        dim_size == 1 for dim_size in dataset.dims.values()):
                     encoding = {
                         var_name: {
                             'units': nc_dataset.variables[var_name].__dict__['units'],
@@ -1131,6 +1141,10 @@ def subset(file_to_subset, bbox, output_file, variables=None,  # pylint: disable
 
         if has_groups:
             recombine_grouped_datasets(datasets, output_file)
+            # Check if the spatial bounds are all 'None'. This means the
+            # subset result is empty.
+            if any(bound is None for bound in spatial_bounds):
+                return None
             return np.array([[
                 min(lon[0][0][0] for lon in zip(spatial_bounds)),
                 max(lon[0][0][1] for lon in zip(spatial_bounds))

pyproject.toml

@@ -12,7 +12,7 @@
 
 [tool.poetry]
 name = "l2ss-py"
-version = "1.3.0"
+version = "1.3.1-rc.9"
 description = "L2 Subsetter Service"
 authors = ["podaac-tva <[email protected]>"]
 license = "Apache-2.0"

tests/data/tropomi/S5P_OFFL_L2__CH4____20190319T110835_20190319T125006_07407_01_010202_20190325T125810_subset.nc4

@@ -1434,7 +1434,14 @@ def test_variable_dims_matched_tropomi(self):
             var_name: [dim.split(subset.GROUP_DELIM)[-1] for dim in var.dimensions]
             for var_name, var in in_nc.groups['PRODUCT'].variables.items()
         }
-
+        
+        # Get variables from METADATA group
+        in_var_dims.update(
+            {
+                var_name: [dim.split(subset.GROUP_DELIM)[-1] for dim in var.dimensions]
+                for var_name, var in in_nc.groups['METADATA'].groups['QA_STATISTICS'].variables.items()
+            }
+        )
         # Include PRODUCT>SUPPORT_DATA>GEOLOCATIONS location
         in_var_dims.update(
             {
@@ -1502,6 +1509,42 @@ def test_temporal_merged_topex(self):
         assert (out_ds.time.values >= start_delta_dt).all()
         assert (out_ds.time.values <= end_delta_dt).all()
 
+    def test_get_time_epoch_var(self):
+        """
+        Test that get_time_epoch_var method returns the 'time' variable for the tropomi CH4 granule"
+        """
+        bbox = np.array(((-180, 180), (-90, 90)))
+        tropomi_file = 'S5P_OFFL_L2__CH4____20190319T110835_20190319T125006_07407_01_010202_20190325T125810_subset.nc4'
+
+        shutil.copyfile(os.path.join(self.test_data_dir, 'tropomi', tropomi_file),
+                        os.path.join(self.subset_output_dir, tropomi_file))
+
+
+        nc_dataset = nc.Dataset(os.path.join(self.subset_output_dir, tropomi_file), mode='r')
+
+        nc_dataset = subset.transform_grouped_dataset(nc_dataset, os.path.join(self.subset_output_dir, tropomi_file))
+
+        args = {
+            'decode_coords': False,
+            'mask_and_scale': False,
+            'decode_times': False
+        }
+
+        with xr.open_dataset(
+                xr.backends.NetCDF4DataStore(nc_dataset),
+                **args
+        ) as dataset:
+
+            lat_var_names, lon_var_names = subset.get_coord_variable_names(dataset)
+            time_var_names = [
+                subset.get_time_variable_name(
+                    dataset, dataset[lat_var_name]
+                ) for lat_var_name in lat_var_names
+            ]
+            epoch_time_var = subset.get_time_epoch_var(dataset, time_var_names[0])
+            
+            assert epoch_time_var.split('__')[-1] == 'time'
+
     def test_temporal_variable_subset(self):
         """
         Test that both a temporal and variable subset can be executed
@@ -1580,3 +1623,54 @@ def test_temporal_subset_lines(self):
         )
 
         assert ds.time.dims != ds.latitude.dims
+
+    def test_grouped_empty_subset(self):
+        """
+        Test that an empty subset of a grouped dataset returns 'None'
+        spatial bounds.
+        """
+        bbox = np.array(((-10, 10), (-10, 10)))
+        file = 'S6A_P4_2__LR_STD__ST_002_140_20201207T011501_20201207T013023_F00.nc'
+        output_file = "{}_{}".format(self._testMethodName, file)
+
+        shutil.copyfile(os.path.join(self.test_data_dir, 'sentinel_6', file),
+                        os.path.join(self.subset_output_dir, file))
+
+        spatial_bounds = subset.subset(
+            file_to_subset=join(self.subset_output_dir, file),
+            bbox=bbox,
+            output_file=join(self.subset_output_dir, output_file)
+        )
+
+        assert spatial_bounds is None
+
+    def test_empty_temporal_subset(self):
+        """
+        Test the edge case where a subsetted empty granule
+        (due to bbox) is temporally subset, which causes the encoding
+        step to fail due to size '1' data for each dimension.
+        """
+        #  37.707:38.484
+        bbox = np.array(((37.707, 38.484), (-13.265, -12.812)))
+        file = '20190927000500-JPL-L2P_GHRSST-SSTskin-MODIS_A-D-v02.0-fv01.0.nc'
+        output_file = "{}_{}".format(self._testMethodName, file)
+        min_time = '2019-09-01'
+        max_time = '2019-09-30'
+
+        subset.subset(
+            file_to_subset=join(self.test_data_dir, file),
+            bbox=bbox,
+            output_file=join(self.subset_output_dir, output_file),
+            min_time=min_time,
+            max_time=max_time
+        )
+
+        # Check that all times are within the given bounds. Open
+        # dataset using 'decode_times=True' for auto-conversions to
+        # datetime
+        ds = xr.open_dataset(
+            join(self.subset_output_dir, output_file),
+            decode_coords=False
+        )
+
+        assert all(dim_size == 1 for dim_size in ds.dims.values())

tests/data/tropomi/S5P_OFFL_L2__SO2____20200713T002730_20200713T020900_14239_01_020103_20200721T191355_subset.nc4

@@ -21,13 +21,24 @@
 import json
 import os.path
 import sys
+import pytest
+import shutil
+import tempfile
 from unittest.mock import patch, MagicMock
 
 import podaac.subsetter
 from podaac.subsetter.subset_harmony import L2SubsetterService
 from harmony.util import config
 
 
+@pytest.fixture()
+def temp_dir():
+    test_data_dir = os.path.dirname(os.path.realpath(__file__))
+    temp_dir = tempfile.mkdtemp(dir=test_data_dir)
+    yield temp_dir
+    shutil.rmtree(temp_dir)
+
+
 def spy_on(method):
     """
     Creates a spy for the given object instance method which records results
@@ -61,7 +72,7 @@ def wrapper(self, *args, **kwargs):
     return wrapper
 
 
-def test_service_invoke(mock_environ):
+def test_service_invoke(mock_environ, temp_dir):
     test_dir = os.path.dirname(os.path.realpath(__file__))
     input_json = json.load(
         open(os.path.join(test_dir, 'data', 'test_subset_harmony', 'test_service_invoke.input.json')))
@@ -101,12 +112,34 @@ def test_service_invoke(mock_environ):
     # When subset function returns 'None', bbox should not be passed to
     # the Harmony service lib 'async_add_local_file_partial_result'
     # function
+    base_file_name = 'S6A_P4_2__LR_STD__ST_002_140_20201207T011501_20201207T013023_F00.nc'
+    test_granule = os.path.join(
+        test_dir,
+        'data',
+        'sentinel_6',
+        base_file_name
+    )
+
+    shutil.copyfile(
+        test_granule,
+        os.path.join(temp_dir, base_file_name)
+    )
+
+    test_granule_temp = os.path.join(temp_dir, base_file_name)
+
+    input_json['sources'][0]['granules'][0]['url'] = f'file://{test_granule_temp}'
+    input_json['sources'][0]['variables'][0]['name'] = '/data_01/wind_speed_alt'
+    input_json['subset']['bbox'] = [-10, -10, 10, 10]
+
+    test_args = [
+        podaac.subsetter.subset_harmony.__file__,
+        "--harmony-action", "invoke",
+        "--harmony-input", json.dumps(input_json)
+    ]
+
     process_item_spy = spy_on(L2SubsetterService.process_item)
     with patch.object(sys, 'argv', test_args), \
-         patch.object(L2SubsetterService, 'process_item', process_item_spy), \
-         patch('podaac.subsetter.subset.subset') as mock_subsetter:
-
-        mock_subsetter.return_value = None
+         patch.object(L2SubsetterService, 'process_item', process_item_spy):
 
         podaac.subsetter.subset_harmony.main(config(False))
 
@@ -118,5 +151,5 @@ def test_service_invoke(mock_environ):
         assert result.bbox == [-1, -2, 3, 4]
 
         # Uses a filename that indicates no spatial subsetting
-        filename = 'JA1_GPN_2PeP001_002_20020115_060706_20020115_070316_bathymetry.nc4'
-        assert result.assets['data'].title == filename
+        # filename = 'JA1_GPN_2PeP001_002_20020115_060706_20020115_070316_bathymetry.nc4'
+        assert 'subsetted' not in result.assets['data'].title