HARMONY-1649 - Remove no-op Swath Projector and HOSS requests.

nasa · Jan 29, 2024 · b9c121b · b9c121b
1 parent f75472f
commit b9c121b
Show file tree

Hide file tree

Showing 8 changed files with 146 additions and 88 deletions.
diff --git a/test/harmony-regression/HarmonyRegression.ipynb b/test/harmony-regression/HarmonyRegression.ipynb
@@ -61,7 +61,10 @@
    "source": [
     "coverages_root = '{root}/{collection}/ogc-api-coverages/1.0.0/collections/{variable}/coverage/rangeset'\n",
     "\n",
-    "is_not_prod = harmony_host_url == 'https://harmony.uat.earthdata.nasa.gov' or harmony_host_url == 'https://harmony.sit.earthdata.nasa.gov' or re.search(\"http:\\/\\/internal-harmony-.*us-west-2\\.elb.amazonaws\\.com\", harmony_host_url)\n",
+    "if harmony_host_url == 'https://harmony.earthdata.nasa.gov':\n",
+    "    is_not_prod = False\n",
+    "else:\n",
+    "    is_not_prod = True\n",
     "\n",
     "if is_not_prod:\n",
     "    l3_collection = 'C1234088182-EEDTEST'\n",
@@ -70,15 +73,48 @@
     "    shapefile_collection = 'C1234530533-EEDTEST'\n",
     "    l2ss_collection = 'C1234208438-POCLOUD'\n",
     "    var_collection = 'C1234714698-EEDTEST'\n",
-    "elif harmony_host_url == 'https://harmony.earthdata.nasa.gov':\n",
+    "    download_collection = 'C1238392622-EEDTEST'\n",
+    "    download_link_substring = 'harmony-uat-eedtest-data'\n",
+    "else:\n",
     "    l3_collection = 'C1756916832-XYZ_PROV'\n",
     "    l3_zarr_collection = 'C1940468263-POCLOUD'\n",
     "    l2_collection = 'C1756916832-XYZ_PROV'\n",
     "    l2ss_collection = 'C1940473819-POCLOUD'\n",
+    "    download_collection = 'C1996546500-GHRC_DAAC'\n",
+    "    download_link_substring = 'data.ghrc.earthdata.nasa.gov'\n",
     "\n",
     "example_vars = ['red_var', 'green_var', 'blue_var', 'alpha_var']"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Set up Harmony Client\n",
+    "\n",
+    "This is for use with requests, which will eventually replace some of the functionality in `notebook_helpers`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from harmony import Client, Collection, Environment, Request\n",
+    "\n",
+    "host_environment = {'http://localhost:3000': Environment.LOCAL,\n",
+    "                    'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n",
+    "                    'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n",
+    "                    'https://harmony.earthdata.nasa.gov': Environment.PROD}\n",
+    "\n",
+    "\n",
+    "harmony_environment = host_environment.get(harmony_host_url)\n",
+    "\n",
+    "if harmony_environment is not None:\n",
+    "    harmony_client = Client(env=harmony_environment)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -102,14 +138,18 @@
     "        'subset': [\n",
     "            'lon(-20:90)', \n",
     "            'lat(0:60)', \n",
-    "            'time(\"2020-01-15T00:00:00Z\":\"2020-01-15T01:00:00Z\")']\n",
+    "            'time(\"2020-01-15T00:00:00Z\":\"2020-01-15T01:00:00Z\")'\n",
+    "        ]\n",
     "    }\n",
+    "\n",
     "    response = get(\n",
     "        coverages_root.format(\n",
     "            root=harmony_host_url,\n",
-    "            collection=l3_collection, \n",
-    "            variable='all'), \n",
-    "        params=params)\n",
+    "            collection=l3_collection,\n",
+    "            variable='all'\n",
+    "        ),\n",
+    "        params=params\n",
+    "    )\n",
     "\n",
     "    show(response, example_vars)"
    ]
@@ -124,11 +164,13 @@
    "source": [
     "if is_not_prod:\n",
     "    response = get(\n",
-    "    coverages_root.format(\n",
-    "        root=harmony_host_url,\n",
-    "        collection=l3_collection, \n",
-    "        variable='green_var'), \n",
-    "    params=params)\n",
+    "        coverages_root.format(\n",
+    "            root=harmony_host_url,\n",
+    "            collection=l3_collection,\n",
+    "            variable='green_var'\n",
+    "        ),\n",
+    "        params=params\n",
+    "    )\n",
     "\n",
     "    show(response, color_index=1)"
    ]
@@ -155,12 +197,16 @@
     "        coverages_root.format(\n",
     "            root=harmony_host_url,\n",
     "            collection=l3_collection, \n",
-    "            variable='all'), \n",
+    "            variable='all'\n",
+    "        ), \n",
     "        params={\n",
     "            'subset': [\n",
     "                'lon(-20:90)', \n",
     "                'lat(0:60)', \n",
-    "                'time(\"2020-01-01T00:00:00Z\":\"2020-01-05T01:00:00Z\")']})\n",
+    "                'time(\"2020-01-01T00:00:00Z\":\"2020-01-05T01:00:00Z\")'\n",
+    "            ]\n",
+    "        }\n",
+    "    )\n",
     "    show_async_condensed(response)"
    ]
   },
@@ -184,9 +230,20 @@
    "source": [
     "if is_not_prod:\n",
     "    #Add 3 requests\n",
-    "    response1 = get(coverages_root.format(root=harmony_host_url, collection=l3_collection, variable='all'), params={'format': 'image/tiff', 'maxResults': '20'})\n",
-    "    response2 = get(coverages_root.format(root=harmony_host_url, collection=l3_collection, variable='all'), params={'format': 'image/tiff', 'maxResults': '20'})\n",
-    "    response3 = get(coverages_root.format(root=harmony_host_url, collection=l3_collection, variable='all'), params={'format': 'image/tiff', 'maxResults': '20'})\n",
+    "    response1 = get(\n",
+    "        coverages_root.format(root=harmony_host_url, collection=l3_collection, variable='all'),\n",
+    "        params={'format': 'image/tiff', 'maxResults': '20'}\n",
+    "    )\n",
+    "\n",
+    "    response2 = get(\n",
+    "        coverages_root.format(root=harmony_host_url, collection=l3_collection, variable='all'),\n",
+    "        params={'format': 'image/tiff', 'maxResults': '20'}\n",
+    "    )\n",
+    "\n",
+    "    response3 = get(\n",
+    "        coverages_root.format(root=harmony_host_url, collection=l3_collection, variable='all'),\n",
+    "        params={'format': 'image/tiff', 'maxResults': '20'}\n",
+    "    )\n",
     "\n",
     "    #List the requests\n",
     "\n",
@@ -229,18 +286,20 @@
    },
    "outputs": [],
    "source": [
-    "response = get(\n",
+    "zarr_response = get(\n",
     "    coverages_root.format(\n",
     "        root=harmony_host_url,\n",
     "        collection=l3_zarr_collection, \n",
-    "        variable='all'), \n",
+    "        variable='all'\n",
+    "    ), \n",
     "    params={\n",
     "        'subset': 'time(\"2020-01-15T00:00:00Z\":\"2020-01-16T01:00:00Z\")',\n",
     "        'maxResults': '3'\n",
-    "        },\n",
-    "    headers = {'accept': 'application/x-zarr'})\n",
+    "    },\n",
+    "    headers = {'accept': 'application/x-zarr'}\n",
+    ")\n",
     "\n",
-    "zarr_response = show_async(response)"
+    "show_async(zarr_response)"
    ]
   },
   {
@@ -262,7 +321,11 @@
     "\n",
     "stac_root = '{root}/stac/{jobID}/{item}'\n",
     "\n",
-    "stac_cat = intake.open_stac_catalog(stac_root.format(root=harmony_host_url,jobID=job,item=''),name='Harmony output')\n",
+    "stac_cat = intake.open_stac_catalog(\n",
+    "    stac_root.format(root=harmony_host_url, jobID=job, item=''),\n",
+    "    name='Harmony output'\n",
+    ")\n",
+    "\n",
     "display(list(stac_cat))"
    ]
   },
@@ -306,12 +369,17 @@
    "source": [
     "### Data Services:\n",
     "\n",
-    "The following services are tested in the `SDS_Regression.ipynb` Jupyter notebook:\n",
+    "The following services are tested in service-specific Jupyter notebooks:\n",
     "\n",
-    "* Swath Projector (a.k.a. \"SWOT Reprojection Tool\").\n",
-    "* MaskFill.\n",
-    "* Variable Subsetter.\n",
-    "* Harmony OPeNDAP SubSetter (HOSS)."
+    "* Geoloco\n",
+    "* Harmony Browse Image Generator (HyBIG)\n",
+    "* Harmony GDAL Adapter (HGA)\n",
+    "* Harmony OPeNDAP SubSetter (HOSS)\n",
+    "* Harmony Regridding Service\n",
+    "* NetCDF-to-Zarr\n",
+    "* Swath Projector\n",
+    "* Trajectory Subsetter\n",
+    "* Variable Subsetter"
    ]
   },
   {
@@ -340,14 +408,17 @@
     "    coverages_root.format(\n",
     "        root=harmony_host_url,\n",
     "        collection=l2ss_collection, \n",
-    "        variable='all'), \n",
+    "        variable='all'\n",
+    "    ), \n",
     "    params={\n",
     "        'maxResults': '1',\n",
     "        'subset': [\n",
     "           'lon(-160:160)', \n",
     "           'lat(-80:80)',\n",
-    "           'time(\"2012-03-03T12:17:00\":\"2012-03-03T12:18:00\")']\n",
-    "        })\n",
+    "           'time(\"2012-03-03T12:17:00\":\"2012-03-03T12:18:00\")'\n",
+    "        ]\n",
+    "    }\n",
+    ")\n",
     "\n",
     "if is_not_prod:\n",
     "    show(response, ['ssha'])\n",
@@ -374,22 +445,55 @@
     "    coverages_root.format(\n",
     "        root=harmony_host_url,\n",
     "        collection=l2ss_collection, \n",
-    "        variable='all'), \n",
+    "        variable='all'\n",
+    "    ), \n",
     "    params={\n",
     "        'maxResults': '3',\n",
     "        'format': 'application/x-netcdf4',\n",
     "        'subset': [\n",
     "            'lon(-160:160)', \n",
     "            'lat(-80:80)',\n",
     "             'time(\"2012-03-03T00:00:00Z\":\"2012-03-03T02:59:59Z\")'\n",
-    "            ]})\n",
+    "        ]\n",
+    "    }\n",
+    ")\n",
     "\n",
     "if is_not_prod:\n",
     "    show_async_condensed(response, ['ssha'])\n",
     "else:\n",
     "    show_async_condensed(response, ['sea_surface_temperature'])"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## harmony/download service\n",
+    "\n",
+    "[HARMONY-1649](https://bugs.earthdata.nasa.gov/browse/HARMONY-1649) added the `harmony/download` service, which will be called when no transformation is specified in the request parameters (e.g., subsetting, reformatting, reprojections).\n",
+    "\n",
+    "The request below establishes that the service is used, rather than one of the other Harmony backend services. Because there is nothing in the `/jobs` output indicating the service that was used, this test asserts that at least one of the output links from Harmony contains a substring that is tied to the original source data."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "download_service_request = Request(collection=Collection(download_collection),\n",
+    "                                   max_results=1)\n",
+    "\n",
+    "download_service_job_id = harmony_client.submit(download_service_request)\n",
+    "download_service_json = harmony_client.result_json(download_service_job_id)\n",
+    "\n",
+    "assert any(\n",
+    "    download_link_substring in link['href']\n",
+    "    for link in download_service_json.get('links')\n",
+    "    if link['rel'] == 'data'\n",
+    "), 'Download service did not return expected link.'"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -402,7 +506,7 @@
   "celltoolbar": "Tags",
   "file_extension": ".py",
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -416,7 +520,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.7"
+   "version": "3.9.13"
   },
   "mimetype": "text/x-python",
   "name": "python",

diff --git a/test/harmony-regression/version.txt b/test/harmony-regression/version.txt
@@ -1 +1 @@
-0.1.4
+0.1.5
diff --git a/test/hoss/HOSS_Regression.ipynb b/test/hoss/HOSS_Regression.ipynb
@@ -335,34 +335,9 @@
    "source": [
     "### HOSS request all variables, no bounding box\n",
     "\n",
-    "If no variables and no bounding box are specified, the entire original granule should be retrieved. (This will run the Variable Subsetter branch of the `sds/variable-subsetter` Docker image, skipping any spatial subsetting portion of the service)\n",
+    "**This request has been migrated to the harmony-regression suite.**\n",
     "\n",
-    "The plotted image should cover the entire Earth (landmasses will be masked)."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "55c0da12",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "if test_is_configured(hoss_info, 'collection'):\n",
-    "    all_no_bbox_file_name = 'hoss_all_no_bbox.nc4'\n",
-    "    all_no_bbox_request = Request(collection=hoss_info['collection'],\n",
-    "                                  granule_id=hoss_info['granule_id'])\n",
-    "\n",
-    "\n",
-    "    submit_and_download(harmony_client, all_no_bbox_request, all_no_bbox_file_name)\n",
-    "    assert exists(all_no_bbox_file_name), 'Unsuccessful HOSS all-variable, no bounding box request.'\n",
-    "\n",
-    "    compare_results_to_reference_file(all_no_bbox_file_name,\n",
-    "                                      'reference_files/hoss_all_no_bbox_reference.nc4',\n",
-    "                                      '/')\n",
-    "\n",
-    "    print_success('HOSS all-variable, no bounding box request.')\n",
-    "else:\n",
-    "    print(f'Skipping - HOSS is not configured for this test in {harmony_environment}.')"
+    "[HARMONY-1649](https://bugs.earthdata.nasa.gov/browse/HARMONY-1649) added a service will return original download links for services that do not explicitly request a transformation in the request parameters. The all-variable, no-bounding-box request will now be routed to that service instead of placing load on HOSS and OPeNDAP."
    ]
   },
   {
@@ -404,7 +379,7 @@
     "\n",
     "\n",
     "    submit_and_download(harmony_client, hoss_temporal_request, hoss_temporal_file_name)\n",
-    "    assert exists(all_no_bbox_file_name), 'Unsuccessful HOSS temporal request.'\n",
+    "    assert exists(hoss_temporal_file_name), 'Unsuccessful HOSS temporal request.'\n",
     "\n",
     "    compare_results_to_reference_file(hoss_temporal_file_name,\n",
     "                                      'reference_files/hoss_temporal_reference.nc4',\n",

diff --git a/test/hoss/reference_files/hoss_all_no_bbox_reference.nc4 b/test/hoss/reference_files/hoss_all_no_bbox_reference.nc4
diff --git a/test/hoss/version.txt b/test/hoss/version.txt
@@ -1 +1 @@
-0.1.6
+0.1.7
diff --git a/test/swath-projector/SwathProjector_Regression.ipynb b/test/swath-projector/SwathProjector_Regression.ipynb
@@ -145,30 +145,9 @@
    "source": [
     "### Swath Projector request with defaults:\n",
     "\n",
-    "Make a request that only specifies the collection and an appropriate granule. This should rely on the default target Coordinate Reference System (CRS) and interpolation method."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ebe08e96",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "if swath_projector_info is not None:\n",
-    "    defaults_file_name = 'swath_projector_defaults.nc4'\n",
-    "    defaults_request = Request(collection=swath_projector_info['collection'],\n",
-    "                               granule_id=[swath_projector_info['granule_id']])\n",
-    "\n",
-    "    submit_and_download(harmony_client, defaults_request, defaults_file_name)\n",
-    "    assert exists(defaults_file_name), 'Unsuccessful Swath Projector defaults request.'\n",
+    "**This test has been removed.**\n",
     "\n",
-    "    compare_results_to_reference_file(defaults_file_name,\n",
-    "                                      'reference_files/swath_projector_defaults_reference.nc4')\n",
-    "\n",
-    "    print_success('Swath projector with default parameters.')\n",
-    "else:\n",
-    "    print(f'The Swath Projector is not configured for environment: \"{harmony_environment}\" - skipping test.')"
+    "[HARMONY-1649](https://bugs.earthdata.nasa.gov/browse/HARMONY-1649) added a service to Harmony that means transformation services are called if a transformation has been requested in the parameters (e.g., subsetting, reformatting, reprojection). For a request that only specifies the collection and granule, the original download links for the source data are returned."
    ]
   },
   {

diff --git a/test/swath-projector/reference_files/swath_projector_defaults_reference.nc4 b/test/swath-projector/reference_files/swath_projector_defaults_reference.nc4
diff --git a/test/swath-projector/version.txt b/test/swath-projector/version.txt
@@ -1 +1 @@
-0.1.4
+0.1.5