Harmony 1523 (#80)

Co-authored-by: Christine Brown-Stevenson <[email protected]>
nasa · Mar 11, 2024 · f13f90e · f13f90e
1 parent b9c121b
commit f13f90e
Show file tree

Hide file tree

Showing 12 changed files with 451 additions and 3 deletions.
diff --git a/.github/workflows/build-all-images.yml b/.github/workflows/build-all-images.yml
@@ -42,6 +42,9 @@ jobs:
             image: "variable-subsetter"
             notebook: "VariableSubsetter_Regression.ipynb"
 
+          - image: "geoloco"
+            notebook: "Geoloco_Regression.ipynb"
+
     uses: ./.github/workflows/build-target-image.yml
     with:
       image-short-name: ${{ matrix.targets.image }}

diff --git a/.gitignore b/.gitignore
@@ -8,5 +8,6 @@ output/*
 .identity
 .netrc
 .DS_Store
+.idea
 
 **/pulled-images.txt
diff --git a/script/test-in-bamboo.sh b/script/test-in-bamboo.sh
@@ -42,7 +42,7 @@ echo "harmony host url: ${harmony_host_url}"
 ## e.g. if REGRESSION_TESTS_N2Z_IMAGE environment was set, the value would be used instead of the default.
 
 image_names=()
-all_tests=(harmony harmony-regression hoss hga n2z swath-projector trajectory-subsetter variable-subsetter regridder hybig)
+all_tests=(harmony harmony-regression hoss hga n2z swath-projector trajectory-subsetter variable-subsetter regridder hybig geoloco)
 for image in "${all_tests[@]}"; do
     image_names+=($(image_name "$image" true))
 done

diff --git a/test/Makefile b/test/Makefile
@@ -28,4 +28,7 @@ trajectory-subsetter-image: Dockerfile trajectory-subsetter/environment.yaml
 variable-subsetter-image: Dockerfile variable-subsetter/environment.yaml
 	docker build -t ghcr.io/nasa/regression-tests-variable-subsetter:latest -f ./Dockerfile --build-arg notebook=VariableSubsetter_Regression.ipynb --build-arg sub_dir=variable-subsetter .
 
-images: harmony-image harmony-regression-image hga-image hoss-image hybig-image n2z-image regridder-image swath-projector-image trajectory-subsetter-image variable-subsetter-image
+geoloco-image: Dockerfile geoloco/environment.yaml
+	docker build -t ghcr.io/nasa/regression-tests-geoloco:latest -f ./Dockerfile --build-arg notebook=Geoloco_Regression.ipynb --build-arg sub_dir=geoloco .
+
+images: harmony-image harmony-regression-image hga-image hoss-image hybig-image n2z-image regridder-image swath-projector-image trajectory-subsetter-image variable-subsetter-image geoloco-image
diff --git a/test/geoloco/Geoloco_Regression.ipynb b/test/geoloco/Geoloco_Regression.ipynb
@@ -0,0 +1,305 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "cfc3aea0-3e7c-42b0-a013-f8b9897fc707",
+   "metadata": {},
+   "source": [
+    "# LAADS DAAC Geoloco Regression Tests\n",
+    "\n",
+    "This notebook contains contains a suite of regression tests against LAADS DAAC Geoloco Harmony Service against reference data generated on premises. \n",
+    "\n",
+    "Geoloco ideally operates on Levels 3 & 4 data. Level 1B and 2 data can be operated, but the outputs will be automatically reprojected to `Geographic` and regridded to its raster resolution if no `proj4` string is supplied. \n",
+    "\n",
+    "Although geoloco can perform variable and location subsetting, reprojection, resampling and regridding, this regression test suite will focus on reprojection, resampling and regridding. \n",
+    "\n",
+    "## Prerequisites\n",
+    "\n",
+    "The dependencies for this notebook are listed in the environment.yaml. To test or install locally, create the papermill environment used in the automated regression testing suite:\n",
+    "\n",
+    "`conda env create -f ./environment.yaml && conda activate papermill-geoloco`\n",
+    "\n",
+    "A `.netrc` file must also be located in the test directory of this repository."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "509a1aa5-2a5f-4223-9bfe-f6c222ffffb2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from os.path import exists\n",
+    "\n",
+    "from harmony import Client, Collection, Environment, Request\n",
+    "\n",
+    "from utilities import (submit_and_download, get_dim_sizes, get_sds_data,\n",
+    "                       remove_results_files, print_error, print_success,\n",
+    "                       compare_dimensions, compare_data)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e0caf2f1-1a52-4db3-8a6c-bc5c2911fb5e",
+   "metadata": {},
+   "source": [
+    "## Set Default Parameters\n",
+    "\n",
+    "`papermill` requires default values for parameters used on the workflow. In this case, `harmony_host_url`\n",
+    "\n",
+    "The following are the valid values\n",
+    "- Production: https://harmony.earthdata.nasa.gov\n",
+    "- UAT: https://harmony.uat.earthdata.nasa.gov\n",
+    "- SIT: https://harmony.sit.earthdata.nasa.gov\n",
+    "- Local: http://localhost:3000"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "68b5e491-086c-48c0-b9bd-42ae068aa8f4",
+   "metadata": {
+    "editable": true,
+    "slideshow": {
+     "slide_type": ""
+    },
+    "tags": [
+     "parameters"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "harmony_host_url = 'https://harmony.uat.earthdata.nasa.gov'\n",
+    "# harmony_host_url = 'https://harmony.sit.earthdata.nasa.gov'\n",
+    "# harmony_host_url = 'http://localhost:3000'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "37cfa0f5-aa39-453b-b69e-1398472154b7",
+   "metadata": {},
+   "source": [
+    "## Identify Harmony Environment"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9e8b905a-b08c-4288-80b2-42af52ed0241",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "host_environment = {'http://localhost:3000': Environment.LOCAL,\n",
+    "                    'https://harmony.sit.earthdata.nasa.gov': Environment.SIT,\n",
+    "                    'https://harmony.uat.earthdata.nasa.gov': Environment.UAT,\n",
+    "                    'https://harmony.earthdata.nasa.gov': Environment.PROD}\n",
+    "\n",
+    "\n",
+    "harmony_environment = host_environment.get(harmony_host_url)\n",
+    "\n",
+    "if harmony_environment is not None:\n",
+    "    harmony_client = Client(env=harmony_environment)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4ce4a5f7-7f82-455c-b693-52715eed525d",
+   "metadata": {},
+   "source": [
+    "## Setting up Collection Environment Variables\n",
+    "\n",
+    "The cell below sets up the Collection, Granule and other necessary variables for each tested dataset. The datasets provided are in the `UAT` environment. There is one dataset for Level 1B, Level 2 and Level 3.\n",
+    "\n",
+    "- Level 1B: MOD021KM\n",
+    "- Level 2: MOD35_L2\n",
+    "- Level 3: MOD08_D3\n",
+    "\n",
+    "Also provided are `proj4` strings for UTM and Geographic tranformations for reprojections and downscale sizing for regridding."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "da8fbd43-8232-496b-a57a-6156447a965c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mod021km_non_production_info = {'collection' : Collection(id='C1256826282-LAADSCDUAT'),\n",
+    "                               'granule_id' : 'G1259320275-LAADSCDUAT',\n",
+    "                               'variable' : ['EV_250_Aggr1km_RefSB'],\n",
+    "                               'downscale_size' : [0.01802, 0.01802]}\n",
+    "\n",
+    "mod35l2_non_production_info = {'collection' : Collection(id='C1257437479-LAADSCDUAT'),\n",
+    "                               'granule_id' : 'G1261599141-LAADSCDUAT',\n",
+    "                               'variable' : ['Cloud_Mask'],\n",
+    "                               'downscale_size' : [0.01802, 0.01802]}\n",
+    "\n",
+    "mod08d3_non_production_info = {'collection' : Collection(id='C1257773477-LAADSCDUAT'),\n",
+    "                               'granule_id' : 'G1259320277-LAADSCDUAT',\n",
+    "                               'variable' : ['Aerosol_Optical_Depth_Land_Ocean_Mean'],\n",
+    "                               'downscale_size' : [2, 2]}\n",
+    "\n",
+    "geo_proj4_string = '+a=6378137.0 +b=6356752.3142451793 +no_defs +proj=latlong'\n",
+    "\n",
+    "resampling_string = 'NN'\n",
+    "\n",
+    "file_indicators = {'MOD021KM': 'EV_250_Aggr1km_RefSB_1.hdf',\n",
+    "                   'MOD35_L2': 'Cloud_Mask_1.hdf',\n",
+    "                   'MOD08_D3': 'Aerosol_Optical_Depth_Land_Ocean_Mean.hdf'}\n",
+    "\n",
+    "reference_data = {'MOD021KM': 'reference_data/MOD021KM.A2023001.0020.061.psrpcs_001701802061.EV_250_Aggr1km_RefSB_1.hdf',\n",
+    "                  'MOD35_L2': 'reference_data/MOD35_L2.A2023001.0020.061.psrpcs_001701881013.Cloud_Mask_1.hdf',\n",
+    "                  'MOD08_D3': 'reference_data/MOD08_D3.A2023001.061.psrpcs_001701881265.Aerosol_Optical_Depth_Land_Ocean_Mean.hdf'}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "722b554f-d69d-45bf-acfe-2ad255f875c0",
+   "metadata": {},
+   "source": [
+    "These selected collections and granules are only available in UAT environment. To minimize the output, all requests will utilize variable subsetting."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9864c2f7-2263-467f-8b33-4810e609cd41",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "mod021km_geoloco_env = {Environment.LOCAL: mod021km_non_production_info,\n",
+    "                       Environment.UAT: mod021km_non_production_info,\n",
+    "                       Environment.SIT: mod021km_non_production_info}\n",
+    "mod35l2_geoloco_env = {Environment.LOCAL: mod35l2_non_production_info,\n",
+    "                       Environment.UAT: mod35l2_non_production_info,\n",
+    "                       Environment.SIT: mod35l2_non_production_info}\n",
+    "mod08d3_geoloco_env = {Environment.LOCAL: mod08d3_non_production_info,\n",
+    "                       Environment.UAT: mod08d3_non_production_info,\n",
+    "                       Environment.SIT: mod08d3_non_production_info}\n",
+    "\n",
+    "if harmony_environment in mod021km_geoloco_env:\n",
+    "    mod021km_geoloco_info =  mod021km_geoloco_env[harmony_environment]\n",
+    "else:\n",
+    "    mod021km_geoloco_info = None\n",
+    "\n",
+    "if harmony_environment in mod35l2_geoloco_env:\n",
+    "    mod35l2_geoloco_info =  mod35l2_geoloco_env[harmony_environment]\n",
+    "else:\n",
+    "    mod35l2_geoloco_info = None\n",
+    "\n",
+    "if harmony_environment in mod08d3_geoloco_env:\n",
+    "    mod08d3_geoloco_info =  mod08d3_geoloco_env[harmony_environment]\n",
+    "else:\n",
+    "    mod08d3_geoloco_info = None"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4640dde7-36cc-4c98-881d-9cbe6ec63d14",
+   "metadata": {},
+   "source": [
+    "## Reprojection/Resampling/Regridding Regression Tests\n",
+    "\n",
+    "In the cell below, reprojection is tested using a Geographic `proj4` string, 2x resolution scale size, and nearest neighbor resampling for the Level 1B, Level 2 and Level 3 Collections/Granules. Outputs are considerably reduced using variable subsetting. Dimension sizes are checked between the reference data and the output data."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ad598301-433b-41ae-850e-3caa8968a873",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if (mod021km_geoloco_info is not None and \n",
+    "    mod35l2_geoloco_info is not None and\n",
+    "    mod08d3_geoloco_info is not None):\n",
+    "\n",
+    "    mod021km_request = Request(collection=mod021km_geoloco_info['collection'],\n",
+    "                               granule_id=mod021km_geoloco_info['granule_id'],\n",
+    "                               variables=mod021km_geoloco_info['variable'],\n",
+    "                               scale_size=mod021km_geoloco_info['downscale_size'],\n",
+    "                               crs=geo_proj4_string,\n",
+    "                               interpolation=resampling_string)\n",
+    "\n",
+    "    mod35l2_request = Request(collection=mod35l2_geoloco_info['collection'],\n",
+    "                              granule_id=mod35l2_geoloco_info['granule_id'],\n",
+    "                              variables=mod35l2_geoloco_info['variable'],\n",
+    "                              scale_size=mod35l2_geoloco_info['downscale_size'],\n",
+    "                              crs=geo_proj4_string,\n",
+    "                              interpolation=resampling_string)\n",
+    "\n",
+    "    mod08d3_request = Request(collection=mod08d3_geoloco_info['collection'],\n",
+    "                              granule_id=mod08d3_geoloco_info['granule_id'],\n",
+    "                              variables=mod08d3_geoloco_info['variable'],\n",
+    "                              scale_size=mod08d3_geoloco_info['downscale_size'],\n",
+    "                              crs=geo_proj4_string,\n",
+    "                              interpolation=resampling_string)\n",
+    "\n",
+    "    mod021km_compare_file = submit_and_download(harmony_client, mod021km_request, file_indicators['MOD021KM'])\n",
+    "    mod35l2_compare_file = submit_and_download(harmony_client, mod35l2_request, file_indicators['MOD35_L2'])\n",
+    "    mod08d3_compare_file = submit_and_download(harmony_client, mod08d3_request, file_indicators['MOD08_D3'])\n",
+    "\n",
+    "    mod021km_test = True\n",
+    "    mod35l2_test = True\n",
+    "    mod08d3_test = True\n",
+    "    \n",
+    "    if compare_dimensions(reference_data['MOD021KM'], mod021km_compare_file):\n",
+    "        if not compare_data(reference_data['MOD021KM'], mod021km_compare_file):\n",
+    "            print_error('MOD021KM data mismatch.')\n",
+    "            mod021km_test = False\n",
+    "    else:\n",
+    "        print_error('MOD021KM data dimension mismatch.')\n",
+    "        mod021km_test = False\n",
+    "    \n",
+    "    if compare_dimensions(reference_data['MOD35_L2'], mod35l2_compare_file):\n",
+    "        if not compare_data(reference_data['MOD35_L2'], mod35l2_compare_file):\n",
+    "            print_error('MOD35_L2 data mismatch.')\n",
+    "            mod35l2_test = False\n",
+    "    else:\n",
+    "        print_error('MOD35_L2 data dimension mismatch.')\n",
+    "        mod35l2_test = False\n",
+    "    \n",
+    "    if compare_dimensions(reference_data['MOD08_D3'], mod08d3_compare_file):\n",
+    "        if not compare_data(reference_data['MOD08_D3'], mod08d3_compare_file):\n",
+    "            print_error('MOD08_D3 data mismatch.')\n",
+    "            mod08d3_test = False\n",
+    "    else:\n",
+    "        print_error('MOD08_D3 data dimension mismatch.')\n",
+    "        mod08d3_test = False\n",
+    "\n",
+    "    remove_results_files()\n",
+    "\n",
+    "    geoloco_tests = mod021km_test and mod35l2_test and mod08d3_test\n",
+    "\n",
+    "    if geoloco_tests:\n",
+    "        print_success('Geoloco Reprojection/Resampling/Regridding requests.')\n",
+    "    else:\n",
+    "        raise Exception('Geoloco test suite failed')\n",
+    "        \n",
+    "else:\n",
+    "    print(f'Geoloco is not configured for this environment: \"{harmony_environment}\" - skipping test.')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/test/geoloco/environment.yaml b/test/geoloco/environment.yaml
@@ -0,0 +1,16 @@
+name: papermill-geoloco
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.11.0
+  - notebook=7.0.6
+  - papermill=2.3.4
+  - python-dateutil=2.8.2
+  - hdf4=4.2.15
+  - gcc=13.2.0
+  - zlib=1.2.13
+  - pip:
+    - harmony-py==0.4.10
+    - pyhdf==0.11.3
+    - numpy==1.26.2
diff --git a/.../reference_data/MOD021KM.A2023001.0020.061.psrpcs_001701802061.EV_250_Aggr1km_RefSB_1.hdf b/.../reference_data/MOD021KM.A2023001.0020.061.psrpcs_001701802061.EV_250_Aggr1km_RefSB_1.hdf
diff --git a/..._data/MOD08_D3.A2023001.061.psrpcs_001701881265.Aerosol_Optical_Depth_Land_Ocean_Mean.hdf b/..._data/MOD08_D3.A2023001.061.psrpcs_001701881265.Aerosol_Optical_Depth_Land_Ocean_Mean.hdf
diff --git a/test/geoloco/reference_data/MOD35_L2.A2023001.0020.061.psrpcs_001701881013.Cloud_Mask_1.hdf b/test/geoloco/reference_data/MOD35_L2.A2023001.0020.061.psrpcs_001701881013.Cloud_Mask_1.hdf