Update Variable Subsetter notebook with no UAT collections.

* Update most test suites to Python 3.10. * Update all test suites to notebook, instead of jupyter. * Other dependency version bumps.
nasa · May 12, 2023 · 3f60004 · 3f60004
1 parent 417a339
commit 3f60004
Show file tree

Hide file tree

Showing 26 changed files with 100 additions and 124 deletions.
diff --git a/test/harmony-regression/environment.yaml b/test/harmony-regression/environment.yaml
@@ -3,14 +3,14 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.9
-  - jupyter
+  - python=3.10
+  - notebook=6.5.4
   - python-dateutil == 2.8.2
   - python-dateutil ~= 2.7
   - sat-stac == 0.4.1
   - intake-stac == 0.3.0 # increasing causes problems with additional install of pystac which has problems with urllib/requests.py
-  - papermill
-  - requests
+  - papermill==2.3.4
+  - requests==2.29.0
   - Pillow
   - CacheControl
   - matplotlib

diff --git a/test/harmony-regression/version.txt b/test/harmony-regression/version.txt
@@ -1 +1 @@
-0.1.1
+0.1.2
diff --git a/test/harmony/environment.yaml b/test/harmony/environment.yaml
@@ -3,9 +3,9 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.9
-  - jupyter==1.0.0
-  - requests==2.28.1
+  - python=3.10
+  - notebook==6.5.4
+  - requests==2.29.0
   - papermill==2.3.4
   - pip:
     - harmony-py
diff --git a/test/harmony/version.txt b/test/harmony/version.txt
@@ -1 +1 @@
-0.1.1
+0.1.2
diff --git a/test/hga/environment.yaml b/test/hga/environment.yaml
@@ -3,9 +3,9 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.9
-  - jupyter
-  - papermill
+  - python=3.10
+  - notebook=6.5.4
+  - papermill=2.3.4
   - pyproj
   - numpy
   - gdal=3.4.2

diff --git a/test/hga/version.txt b/test/hga/version.txt
@@ -1 +1 @@
-0.1.1
+0.1.2
diff --git a/test/hoss/environment.yaml b/test/hoss/environment.yaml
@@ -3,12 +3,12 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.9
-  - jupyter=1.0.0
+  - python=3.10
+  - notebook=6.5.4
   - papermill=2.3.4
   - python-dateutil=2.7.5
-  - numpy=1.24.1
-  - netCDF4=1.6.2
+  - numpy=1.24.3
+  - netCDF4=1.6.3
   - xarray=2022.12.0
   - pip:
     - harmony-py
diff --git a/test/hoss/utilities.py b/test/hoss/utilities.py
@@ -57,8 +57,9 @@ def submit_and_download(harmony_client: Client, request: Request,
             replace(downloaded_filename, output_file_name)
             print(f'Saved output to: {output_file_name}')
 
-    except ProcessingFailedException:
+    except ProcessingFailedException as exception:
         print_error('Harmony request failed to complete successfully.')
+        raise exception
 
 
 def remove_results_files() -> None:

diff --git a/test/hoss/version.txt b/test/hoss/version.txt
@@ -1 +1 @@
-0.1.1
+0.1.2
diff --git a/test/n2z/environment.yaml b/test/n2z/environment.yaml
@@ -4,14 +4,14 @@ channels:
   - defaults
 dependencies:
   - python=3.9
-  - boto3
-  - jupyter
-  - papermill
+  - boto3=1.26.133
+  - notebook=6.5.4
+  - papermill=2.3.4
   - xarray == 2022.9.0
-  - numpy
+  - numpy=1.24.3
   - pandas == 1.3.0
-  - packaging
-  - zarr
-  - cftime
+  - packaging=23.1
+  - zarr=2.14.2
+  - cftime=1.6.2
   - pip:
     - harmony-py
diff --git a/test/n2z/version.txt b/test/n2z/version.txt
@@ -1 +1 @@
-0.1.1
+0.1.2
diff --git a/test/regridder/environment.yaml b/test/regridder/environment.yaml
@@ -4,11 +4,11 @@ channels:
   - defaults
 dependencies:
   - python=3.9
-  - jupyter
-  - papermill
+  - notebook=6.5.4
+  - papermill=2.3.4
   - xarray == 2022.12.0
-  - numpy
-  - netCDF4
+  - numpy=1.24.3
+  - netCDF4=1.6.3
   - pandas == 1.3.0
   - pip:
     - harmony-py
diff --git a/test/regridder/version.txt b/test/regridder/version.txt
@@ -1 +1 @@
-0.1.1
+0.1.2
diff --git a/test/swath-projector/SwathProjector_Regression.ipynb b/test/swath-projector/SwathProjector_Regression.ipynb
@@ -278,7 +278,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.15"
+   "version": "3.9.13"
   },
   "name": "SDS_Regression.ipynb"
  },

diff --git a/test/swath-projector/environment.yaml b/test/swath-projector/environment.yaml
@@ -3,12 +3,12 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.9
-  - jupyter=1.0.0
+  - python=3.10
+  - notebook=6.5.4
   - papermill=2.3.4
   - python-dateutil=2.7.5
-  - numpy=1.24.1
-  - netCDF4=1.6.2
+  - numpy=1.24.3
+  - netCDF4=1.6.3
   - xarray=2022.12.0
   - pip:
     - harmony-py
diff --git a/test/swath-projector/utilities.py b/test/swath-projector/utilities.py
@@ -52,8 +52,9 @@ def submit_and_download(harmony_client: Client, request: Request,
             replace(downloaded_filename, output_file_name)
             print(f'Saved output to: {output_file_name}')
 
-    except ProcessingFailedException:
+    except ProcessingFailedException as exception:
         print_error('Harmony request failed to complete successfully.')
+        raise exception
 
 
 def remove_results_files() -> None:

diff --git a/test/swath-projector/version.txt b/test/swath-projector/version.txt
@@ -1 +1 @@
-0.1.1
+0.1.2
diff --git a/test/trajectory-subsetter/environment.yaml b/test/trajectory-subsetter/environment.yaml
@@ -3,12 +3,12 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.9
-  - jupyter=1.0.0
+  - python=3.10
+  - notebook=6.5.4
   - papermill=2.3.4
   - python-dateutil=2.7.5
-  - numpy=1.24.1
-  - netCDF4=1.6.2
+  - numpy=1.24.3
+  - netCDF4=1.6.3
   - xarray=2022.12.0
   - pip:
     - harmony-py
diff --git a/test/trajectory-subsetter/utilities.py b/test/trajectory-subsetter/utilities.py
@@ -53,8 +53,9 @@ def submit_and_download(harmony_client: Client, request: Request,
             replace(downloaded_filename, output_file_name)
             print(f'Saved output to: {output_file_name}')
 
-    except ProcessingFailedException:
+    except ProcessingFailedException as exception:
         print_error('Harmony request failed to complete successfully.')
+        raise exception
 
 
 def remove_results_files() -> None:

diff --git a/test/trajectory-subsetter/version.txt b/test/trajectory-subsetter/version.txt
@@ -1 +1 @@
-0.1.1
+0.1.2
diff --git a/test/variable-subsetter/VariableSubsetter_Regression.ipynb b/test/variable-subsetter/VariableSubsetter_Regression.ipynb
@@ -110,7 +110,16 @@
     "\n",
     "The variable subsetter is currently only configured for collections in UAT.\n",
     "\n",
-    "The granule selected is the smallest in the ATL08 collection, to improve performance of tests."
+    "**2023-05-11 - The tests in this notebook been disabled as there are currently no collections associated with the Variable Subsetter. ATL03 and ATL08 have recently been associated with the Trajectory Subsetter instead.**\n",
+    "\n",
+    "To re-enable these tests:\n",
+    "\n",
+    "* Associate a suitable UAT collection to the [UMM-S record for the variable-subsetter](https://mmt.uat.earthdata.nasa.gov/services/S1237976118-EEDTEST). This collection will likely be L2, and must be enabled for OPeNDAP (including related URLs in granule UMM-G records).\n",
+    "* Revert the change in the next cell to `var_subsetter_env`.\n",
+    "* Add the collection concept ID and granule concept ID for testing to the `var_subsetter_non_prod_information` cell.\n",
+    "* Enter a variable name in the single-variable `harmony.Request` instance.\n",
+    "* Create reference files for both requests and save them in the `reference_files` subdirectory (with the expected file names).\n",
+    "* Run the notebook locally before commiting any changes."
    ]
   },
   {
@@ -120,12 +129,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "var_subsetter_non_prod_information = {'collection': Collection(id='C1234714698-EEDTEST'),\n",
-    "                                      'granule_id': 'G1238479209-EEDTEST'}\n",
+    "var_subsetter_non_prod_information = {'collection': Collection(id='<enter collection concept ID>'),\n",
+    "                                      'granule_id': '<enter granule concept ID>'}\n",
     "\n",
-    "var_subsetter_env = {Environment.LOCAL: var_subsetter_non_prod_information,\n",
-    "                     Environment.SIT: var_subsetter_non_prod_information,\n",
-    "                     Environment.UAT: var_subsetter_non_prod_information}\n",
+    "var_subsetter_env = {}\n",
+    "# var_subsetter_env = {Environment.LOCAL: var_subsetter_non_prod_information,\n",
+    "#                      Environment.SIT: var_subsetter_non_prod_information,\n",
+    "#                      Environment.UAT: var_subsetter_non_prod_information}\n",
     "\n",
     "if harmony_environment in var_subsetter_env:\n",
     "    var_subsetter_info = var_subsetter_env[harmony_environment]\n",
@@ -138,13 +148,11 @@
    "id": "e8f58ea4-124f-4dc4-8039-81fb20cd02c7",
    "metadata": {},
    "source": [
-    "### Variable Subsetter request, no Int64 variables\n",
+    "### Variable Subsetter, single-variable request:\n",
     "\n",
-    "This request should retrieve the requested `/gt1l/land_segments/dem_h` variable alongside the following supporting variables:\n",
+    "**2023-05-11 - When this test is active a sample reference file will need to be created and added as `reference_files/var_subsetter_single_var_reference.nc4`.**\n",
     "\n",
-    "* `/gt1l/land_segments/delta_time`\n",
-    "* `/gt1l/land_segments/latitude`\n",
-    "* `/gt1l/land_segments/longitude`"
+    "This request should retrieve the requested variable and any required variables (e.g., coordinates, or dimension variables). When this test is reactivated for a new collection, the expected output variables should be listed here."
    ]
   },
   {
@@ -155,97 +163,61 @@
    "outputs": [],
    "source": [
     "if var_subsetter_info is not None:\n",
-    "    no_int64_file_name = 'var_subsetter_no_int64.nc4'\n",
-    "    no_int64_request = Request(collection=var_subsetter_info['collection'],\n",
-    "                               granule_id=[var_subsetter_info['granule_id']],\n",
-    "                               variables=['/gt1l/land_segments/dem_h'])\n",
+    "    single_var_file_name = 'var_subsetter.nc4'\n",
+    "    single_var_request = Request(collection=var_subsetter_info['collection'],\n",
+    "                                 granule_id=[var_subsetter_info['granule_id']],\n",
+    "                                 variables=['<enter variable full name>'])\n",
     "\n",
-    "    submit_and_download(harmony_client, no_int64_request, no_int64_file_name)\n",
-    "    assert exists(no_int64_file_name), 'Unsuccessful non-Int64 Variable Subsetter request.'\n",
+    "    submit_and_download(harmony_client, single_var_request, single_var_file_name)\n",
+    "    assert exists(single_var_file_name), 'Unsuccessful single-variable Variable Subsetter request.'\n",
     "\n",
-    "    compare_results_to_reference_file(no_int64_file_name,\n",
-    "                              'reference_files/var_subsetter_no_int64_reference.nc4')\n",
+    "    compare_results_to_reference_file(\n",
+    "        single_var_file_name,\n",
+    "        'reference_files/var_subsetter_single_var_reference.nc4'\n",
+    "    )\n",
     "\n",
-    "    print_success('Variable subsetter synchronous request.')\n",
+    "    print_success('Variable subsetter single variable request.')\n",
     "else:\n",
     "    print(f'The Variable Subsetter is not configured for environment: \"{harmony_environment}\" - skipping test.')"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "40e40229-3ff7-4d82-945b-af33703eb11c",
+   "id": "aaeb21d3-ce95-4068-b6aa-28b9f9b93b5a",
    "metadata": {},
    "source": [
-    "### Variable Subsetter request and Int64 variables\n",
+    "### Variable Subsetter, all variables\n",
     "\n",
-    "This request should retrieve the `/gt1l/signal_photons/classed_pc_flag` variable and 6 supporting variables, some of which are Int64, which is not supported by the DAP2 protocol:\n",
+    "Make a request for \"all\" variables. This should retrieve the entire file, with all the variables from the original source granule.\n",
     "\n",
-    "* `/gt1l/signal_photons/delta_time`\n",
-    "* `/gt1l/land_segments/ph_ndx`\n",
-    "* `/gt1l/land_segments/n_seg_ph`\n",
-    "* `/gt1l/land_segments/delta_time`\n",
-    "* `/gt1l/land_segments/latitude`\n",
-    "* `/gt1l/land_segments/longitude`"
+    "**2023-05-11 - When this test is active a sample reference file will need to be created and added as `reference_files/var_subsetter_all_vars_reference.nc4`.**"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "41a3f8fd",
+   "id": "a5b11ceb",
    "metadata": {},
    "outputs": [],
    "source": [
     "if var_subsetter_info is not None:\n",
-    "    int64_file_name = 'var_subsetter_int64.nc4'\n",
-    "    int64_request = Request(collection=var_subsetter_info['collection'],\n",
-    "                            granule_id=[var_subsetter_info['granule_id']],\n",
-    "                            variables=['/gt1l/signal_photons/classed_pc_flag'])\n",
+    "    all_variables_file_name = 'var_subsetter_all_vars.nc4'\n",
+    "    all_variables_request = Request(collection=var_subsetter_info['collection'],\n",
+    "                                    granule_id=[var_subsetter_info['granule_id']])\n",
     "\n",
-    "    submit_and_download(harmony_client, int64_request, int64_file_name)\n",
-    "    assert exists(int64_file_name), 'Unsuccessful Int64 Variable Subsetter request.'\n",
+    "    submit_and_download(harmony_client, all_variables_request, all_variables_file_name)\n",
+    "    assert exists(all_variables_file_name), 'Unsuccessful Variable Subsetter all-variable request.'\n",
     "\n",
-    "    compare_results_to_reference_file(int64_file_name,\n",
-    "                              'reference_files/var_subsetter_int64_reference.nc4')\n",
+    "    compare_results_to_reference_file(\n",
+    "        all_variables_file_name,\n",
+    "        'reference_files/var_subsetter_all_vars_reference.nc4'\n",
+    "    )\n",
     "\n",
-    "    print_success('Variable Subsetter Int64 request.')\n",
+    "    print_success('Variable Subsetter all variable request.')\n",
     "else:\n",
     "    print(f'The Variable Subsetter is not configured for environment: \"{harmony_environment}\" - skipping test.')"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "id": "aaeb21d3-ce95-4068-b6aa-28b9f9b93b5a",
-   "metadata": {},
-   "source": [
-    "### Variable Subsetter, all variables\n",
-    "\n",
-    "Make a request for \"all\" variables. This should retrieve the entire file, with all the variables from the original source granule.\n",
-    "\n",
-    "**2023-01-12: This test is currently disabled as the output contains a variable type for `/ancillary_data/control` that is not currently handled by OPeNDAP (an array of string types).**"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "a5b11ceb",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# if var_subsetter_info is not None:\n",
-    "#     all_variables_file_name = 'var_subsetter_all_vars.nc4'\n",
-    "#     all_variables_request = Request(collection=var_subsetter_info['collection'],\n",
-    "#                                     granule_id=[var_subsetter_info['granule_id']])\n",
-    "#\n",
-    "#     submit_and_download(harmony_client, all_variables_request, all_variables_file_name)\n",
-    "#     assert exists(all_variables_file_name), 'Unsuccessful Variable Subsetter all variable request.'\n",
-    "# \n",
-    "#     compare_results_to_reference_file(all_variables_file_name, 'reference_files/to_be_retrieved...')\n",
-    "# \n",
-    "#     print_success('Variable Subsetter all variable request.')\n",
-    "# else:\n",
-    "#     print(f'The Variable Subsetter is not configured for environment: \"{harmony_environment}\" - skipping test.')"
-   ]
-  },
   {
    "cell_type": "markdown",
    "id": "08dc1eb9",

diff --git a/test/variable-subsetter/environment.yaml b/test/variable-subsetter/environment.yaml
@@ -3,10 +3,10 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.9
-  - jupyter=1.0.0
+  - python=3.10
+  - notebook=6.5.4
   - papermill=2.3.4
-  - numpy=1.24.1
-  - netCDF4=1.6.2
+  - numpy=1.24.3
+  - netCDF4=1.6.3
   - pip:
     - harmony-py
diff --git a/test/variable-subsetter/reference_files/var_subsetter_int64_reference.nc4 b/test/variable-subsetter/reference_files/var_subsetter_int64_reference.nc4
diff --git a/test/variable-subsetter/reference_files/var_subsetter_no_int64_reference.nc4 b/test/variable-subsetter/reference_files/var_subsetter_no_int64_reference.nc4