Merge branch 'main' into 112-add-documentation

CHANGELOG.md

@@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.3.6] - 2024-02-07
+
+### Added
+* Adds an exception to catch an error when a corrupted file is encountered while building
+the Kerchunk reference file using `SingleHdf5ToZarr`.
+* The behavior determining whether to raise an exception is controlled by the
+`ignore_missing_file` flag.
+
+
 ## [0.3.5] - 2023-12-18
 
 ### Added

README.md

@@ -38,8 +38,8 @@ $ poetry add git+https://github.com/RTIInternational/teehr.git#[BRANCH TAG]
 
 Use Docker
 ```bash
-$ docker build -t teehr:v0.3.5 .
-$ docker run -it --rm --volume $HOME:$HOME -p 8888:8888 teehr:v0.3.5 jupyter lab --ip 0.0.0.0 $HOME
+$ docker build -t teehr:v0.3.6 .
+$ docker run -it --rm --volume $HOME:$HOME -p 8888:8888 teehr:v0.3.6 jupyter lab --ip 0.0.0.0 $HOME
 ```
 
 ## Examples

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "teehr"
-version = "0.3.5"
+version = "0.3.6"
 description = "Tools for Exploratory Evaluation in Hydrologic Research"
 authors = [
     "RTI International",

src/teehr/loading/nwm/utils.py

@@ -297,9 +297,16 @@ def gen_json(
             date = p[3]
             fname = p[5]
             outf = str(Path(json_dir, f"{date}.{fname}.json"))
-            h5chunks = SingleHdf5ToZarr(infile,
-                                        remote_path,
-                                        inline_threshold=300)
+            try:
+                h5chunks = SingleHdf5ToZarr(infile,
+                                            remote_path,
+                                            inline_threshold=300)
+            except OSError as err:
+                if not ignore_missing_file:
+                    raise Exception(f"Corrupt file: {remote_path}") from err
+                else:
+                    # TODO: log missing file?
+                    return None
             with open(outf, "wb") as f:
                 f.write(ujson.dumps(h5chunks.translate()).encode())
     except FileNotFoundError as e:
@@ -358,8 +365,9 @@ def build_zarr_references(
     json_paths.extend(existing_jsons)
 
     if not any(json_paths):
-        raise FileNotFoundError("No NWM files for specified input \
-                                configuration were found in GCS!")
+        raise FileNotFoundError(
+            "No NWM files for specified input configuration were found in GCS!"
+        )
 
     json_paths = [path for path in json_paths if path is not None]
 

teehr-hub/helm-chart/config.yaml

@@ -48,19 +48,19 @@ jupyterhub:
         mem_limit: null
         node_selector:
           node.kubernetes.io/instance-type: r5.4xlarge
-    - display_name: "TEEHR Evaluation System (4 vCPU/32GB memory) v0.3.5"
+    - display_name: "TEEHR Evaluation System (4 vCPU/32GB memory) v0.3.6"
       default: True
       description: "A r5.xlarge EC2 instance $0.252/hour"
       kubespawner_override:
-        image: 935462133478.dkr.ecr.us-east-2.amazonaws.com/teehr:v0.3.5
+        image: 935462133478.dkr.ecr.us-east-2.amazonaws.com/teehr:v0.3.6
         cpu_limit: null
         mem_limit: null
         node_selector:
           node.kubernetes.io/instance-type: r5.xlarge
-    - display_name: "TEEHR Evaluation System (16 vCPU/128GB memory) v0.3.5"
+    - display_name: "TEEHR Evaluation System (16 vCPU/128GB memory) v0.3.6"
       description: "A r5.4xlarge EC2 instance @ $1.008/hour"
       kubespawner_override:
-        image: 935462133478.dkr.ecr.us-east-2.amazonaws.com/teehr:v0.3.5
+        image: 935462133478.dkr.ecr.us-east-2.amazonaws.com/teehr:v0.3.6
         cpu_limit: null
         mem_limit: null
         node_selector:
@@ -96,7 +96,7 @@ dask-gateway:
     backend:
       image:
         name: 935462133478.dkr.ecr.us-east-2.amazonaws.com/teehr  # Image to use for singleuser environment. Must include dask-gateway.
-        tag: v0.3.5
+        tag: v0.3.6
       scheduler:
         extraPodConfig:
           # serviceAccountName: user-sa

tests/test_nwm_loading_utils.py

@@ -1,6 +1,8 @@
 """Test NWM loading utils."""
 from pathlib import Path
 
+import pytest
+
 from teehr.loading.nwm.utils import (
     build_zarr_references,
     check_dates_against_nwm_version,
@@ -132,9 +134,27 @@ def test_generate_json_paths():
     pass
 
 
+def test_generate_json_for_bad_file():
+    """Test generating json paths for a corrupt GCS file."""
+    kerchunk_method = "local"
+    gcs_component_paths = \
+        ['gcs://national-water-model/nwm.20240125/forcing_medium_range/nwm.t18z.medium_range.forcing.f104.conus.nc'] # noqa
+    json_dir = ""
+    ignore_missing_file = False
+
+    with pytest.raises(Exception):
+        _ = generate_json_paths(
+            kerchunk_method,
+            gcs_component_paths,
+            json_dir,
+            ignore_missing_file
+        )
+
+
 if __name__ == "__main__":
     test_dates_and_nwm_version()
     test_building_nwm30_gcs_paths()
     test_building_nwm22_gcs_paths()
     test_generate_json_paths()
     test_point_zarr_reference_file()
+    test_generate_json_for_bad_file()

version.txt

@@ -1 +1 @@
-0.3.5
+0.3.6