Das 1867 (#69)

* Updated Harmony service test notebooks to reference the correct environment in the documentation. * DAS-1867: Updated reference files. Previous one had a typo in a string within a global parameter. * DAS-1867: Updated utilties.py function to use DataTrees instead of Datasets. * DAS-1867: Updated version number. * Remove wild whitespace. --------- Co-authored-by: Josie Lyon <[email protected]>
nasa · Sep 12, 2023 · a21f3f0 · a21f3f0
1 parent 432fb74
commit a21f3f0
Show file tree

Hide file tree

Showing 16 changed files with 122 additions and 33 deletions.
diff --git a/test/harmony/Harmony.ipynb b/test/harmony/Harmony.ipynb
@@ -22,6 +22,21 @@
     "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## Prerequisites\n",
+    "\n",
+    "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n",
+    "\n",
+    "`conda env create -f ./environment.yaml && conda activate papermill-harmony`\n",
+    "\n",
+    "A `.netrc` file must also be located in the `test` directory of this repository."
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

diff --git a/test/hga/HGA_Regression.ipynb b/test/hga/HGA_Regression.ipynb
@@ -43,6 +43,22 @@
     "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e8240032",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## Prerequisites\n",
+    "\n",
+    "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n",
+    "\n",
+    "`conda env create -f ./environment.yaml && conda activate papermill-hga`\n",
+    "\n",
+    "A `.netrc` file must also be located in the `test` directory of this repository."
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "7f0766b7",

diff --git a/test/hoss/HOSS_Regression.ipynb b/test/hoss/HOSS_Regression.ipynb
@@ -21,7 +21,7 @@
     "\n",
     "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n",
     "\n",
-    "`conda env create -f ./environment.yaml && conda activate papermill`\n",
+    "`conda env create -f ./environment.yaml && conda activate papermill-hoss`\n",
     "\n",
     "A `.netrc` file must also be located in the `test` directory of this repository."
    ]

diff --git a/test/hybig/HyBIG_Regression.ipynb b/test/hybig/HyBIG_Regression.ipynb
@@ -43,6 +43,22 @@
     "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a6e210ef",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## Prerequisites\n",
+    "\n",
+    "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n",
+    "\n",
+    "`conda env create -f ./environment.yaml && conda activate papermill-hybig`\n",
+    "\n",
+    "A `.netrc` file must also be located in the `test` directory of this repository."
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "802241b5",

diff --git a/test/n2z/N2Z_Regression.ipynb b/test/n2z/N2Z_Regression.ipynb
@@ -43,6 +43,22 @@
     "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9bdda441",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## Prerequisites\n",
+    "\n",
+    "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n",
+    "\n",
+    "`conda env create -f ./environment.yaml && conda activate papermill-n2z`\n",
+    "\n",
+    "A `.netrc` file must also be located in the `test` directory of this repository."
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "4979305a",

diff --git a/test/regridder/Regridder_Regression.ipynb b/test/regridder/Regridder_Regression.ipynb
@@ -41,6 +41,22 @@
     "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0ca1e5bb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## Prerequisites\n",
+    "\n",
+    "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n",
+    "\n",
+    "`conda env create -f ./environment.yaml && conda activate papermill-regridder`\n",
+    "\n",
+    "A `.netrc` file must also be located in the `test` directory of this repository."
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "4979305a",

diff --git a/test/swath-projector/SwathProjector_Regression.ipynb b/test/swath-projector/SwathProjector_Regression.ipynb
@@ -23,7 +23,7 @@
     "\n",
     "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n",
     "\n",
-    "`conda env create -f ./environment.yaml && conda activate papermill`\n",
+    "`conda env create -f ./environment.yaml && conda activate papermill-swath-projector`\n",
     "\n",
     "A `.netrc` file must also be located in the `test` directory of this repository."
    ]

diff --git a/test/trajectory-subsetter/TrajectorySubsetter_Regression.ipynb b/test/trajectory-subsetter/TrajectorySubsetter_Regression.ipynb
@@ -21,7 +21,7 @@
     "\n",
     "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n",
     "\n",
-    "`conda env create -f ./environment.yaml && conda activate papermill`\n",
+    "`conda env create -f ./environment.yaml && conda activate papermill-trajectory-subsetter`\n",
     "\n",
     "A `.netrc` file must also be located in the `test` directory of this repository."
    ]
@@ -172,7 +172,11 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "74bf1567",
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "raises-exception"
+    ]
+   },
    "outputs": [],
    "source": [
     "if trajectory_subsetter_info is not None:\n",
@@ -185,8 +189,7 @@
     "    assert exists(ts_variable_file_name), 'Unsuccessful Trajectory Subsetter variable subset request.'\n",
     "\n",
     "    compare_results_to_reference_file(ts_variable_file_name,\n",
-    "                                      'reference_files/trajectory_subsetter_variable_reference.h5',\n",
-    "                                      '/BEAM0000')\n",
+    "                                      'reference_files/trajectory_subsetter_variable_reference.h5')\n",
     "\n",
     "    print_success('Trajectory Subsetter variable subset request.')\n",
     "else:\n",
@@ -207,7 +210,11 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "c51eb82c",
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "raises-exception"
+    ]
+   },
    "outputs": [],
    "source": [
     "if trajectory_subsetter_info is not None:\n",
@@ -222,8 +229,7 @@
     "    assert exists(ts_temporal_file_name), 'Unsuccessful Trajectory Subsetter temporal subset request.'\n",
     "\n",
     "    compare_results_to_reference_file(ts_temporal_file_name,\n",
-    "                                      'reference_files/trajectory_subsetter_temporal_reference.h5',\n",
-    "                                      '/BEAM0000')\n",
+    "                                      'reference_files/trajectory_subsetter_temporal_reference.h5')\n",
     "\n",
     "    print_success('Trajectory Subsetter temporal subset request.')\n",
     "else:\n",
@@ -247,7 +253,11 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "ffb9742c",
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "raises-exception"
+    ]
+   },
    "outputs": [],
    "source": [
     "if trajectory_subsetter_info is not None:\n",
@@ -262,8 +272,7 @@
     "    assert exists(ts_bbox_file_name), 'Unsuccessful Trajectory Subsetter bounding box subset request.'\n",
     "\n",
     "    compare_results_to_reference_file(ts_bbox_file_name,\n",
-    "                                      'reference_files/trajectory_subsetter_bbox_reference.h5',\n",
-    "                                      '/BEAM0000')\n",
+    "                                      'reference_files/trajectory_subsetter_bbox_reference.h5')\n",
     "\n",
     "    print_success('Trajectory Subsetter bounding box spatial subset request.')\n",
     "else:\n",
@@ -284,7 +293,11 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "a10d6af4",
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "raises-exception"
+    ]
+   },
    "outputs": [],
    "source": [
     "if trajectory_subsetter_info is not None:\n",
@@ -298,8 +311,7 @@
     "    assert exists(ts_polygon_file_name), 'Unsuccessful Trajectory Subsetter polygon spatial subset request.'\n",
     "\n",
     "    compare_results_to_reference_file(ts_polygon_file_name,\n",
-    "                                      'reference_files/trajectory_subsetter_polygon_reference.h5',\n",
-    "                                      '/BEAM0000')\n",
+    "                                      'reference_files/trajectory_subsetter_polygon_reference.h5')\n",
     "\n",
     "    print_success('Trajectory Subsetter polygon spatial subset request.')\n",
     "else:\n",
@@ -338,7 +350,7 @@
  "metadata": {
   "celltoolbar": "Tags",
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "papermill-trajectory-subsetter",
    "language": "python",
    "name": "python3"
   },
@@ -352,7 +364,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.15"
+   "version": "3.10.12"
   },
   "name": "SDS_Regression.ipynb"
  },

diff --git a/test/trajectory-subsetter/environment.yaml b/test/trajectory-subsetter/environment.yaml
@@ -9,6 +9,6 @@ dependencies:
   - python-dateutil=2.7.5
   - numpy=1.24.3
   - netCDF4=1.6.3
-  - xarray=2022.12.0
+  - xarray-datatree=0.0.12
   - pip:
     - harmony-py
diff --git a/test/trajectory-subsetter/reference_files/trajectory_subsetter_bbox_reference.h5 b/test/trajectory-subsetter/reference_files/trajectory_subsetter_bbox_reference.h5
diff --git a/test/trajectory-subsetter/reference_files/trajectory_subsetter_polygon_reference.h5 b/test/trajectory-subsetter/reference_files/trajectory_subsetter_polygon_reference.h5
diff --git a/test/trajectory-subsetter/reference_files/trajectory_subsetter_temporal_reference.h5 b/test/trajectory-subsetter/reference_files/trajectory_subsetter_temporal_reference.h5
diff --git a/test/trajectory-subsetter/reference_files/trajectory_subsetter_variable_reference.h5 b/test/trajectory-subsetter/reference_files/trajectory_subsetter_variable_reference.h5
diff --git a/test/trajectory-subsetter/utilities.py b/test/trajectory-subsetter/utilities.py
@@ -7,28 +7,26 @@
 
 from harmony import Client, Request
 from harmony.harmony import ProcessingFailedException
-import xarray as xr
+import datatree
 
 
 def compare_results_to_reference_file(results_file_name: str,
-                                      reference_file_name: str,
-                                      group_path: str) -> None:
-    """ Use native `xarray` functionality to compare data values and metadata
-        attributes of downloaded results to a reference file.
+                                      reference_file_name: str) -> None:
+    """ Use `DataTree` functionality to compare data values, variables, 
+        coordinates, metadata, and all their corresponding attributes of
+        downloaded results to a reference file.
 
     """
-    reference_data =  xr.open_dataset(reference_file_name, group=group_path)
-    results_data = xr.open_dataset(results_file_name, group=group_path)
+    reference_data =  datatree.open_datatree(reference_file_name)
+    results_data = datatree.open_datatree(results_file_name)
 
-    assert results_data.equals(reference_data), ('Output and reference files '
-                                                 'do not match.')
+    assert results_data.identical(reference_data), ('Output and reference files '
+                                                    'do not match.')
 
-    reference_data.close()
-    results_data.close()
     reference_data = None
     results_data = None
-
-
+ 
+ 
 def submit_and_download(harmony_client: Client, request: Request,
                         output_file_name: str):
     """ Submit a Harmony request via a `harmony-py` client. Wait for the

diff --git a/test/trajectory-subsetter/version.txt b/test/trajectory-subsetter/version.txt
@@ -1 +1 @@
-0.1.3
+0.1.4
diff --git a/test/variable-subsetter/VariableSubsetter_Regression.ipynb b/test/variable-subsetter/VariableSubsetter_Regression.ipynb
@@ -17,7 +17,7 @@
     "\n",
     "The dependencies for this notebook are listed in the [environment.yaml](./environment.yaml). To test or install locally, create the papermill environment used in the automated regression testing suite:\n",
     "\n",
-    "`conda env create -f ./environment.yaml && conda activate papermill`\n",
+    "`conda env create -f ./environment.yaml && conda activate papermill-variable-subsetter`\n",
     "\n",
     "A `.netrc` file must also be located in the `test` directory of this repository."
    ]