Merge pull request #107 from iwensu0313/nan_depth

Climatology Test: Handing masked depths without zspan

.pre-commit-config.yaml

@@ -24,3 +24,14 @@ repos:
           )$
       args:
         - --ignore-words-list=pres,ba,ot
+
+ci:
+    autofix_commit_msg: |
+        [pre-commit.ci] auto fixes from pre-commit.com hooks
+
+        for more information, see https://pre-commit.ci
+    autofix_prs: false
+    autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate'
+    autoupdate_schedule: monthly
+    skip: []
+    submodules: false

ioos_qc/qartod.py

@@ -340,9 +340,6 @@ def check(self, tinp, inp, zinp):
         flag_arr = np.ma.empty(inp.size, dtype='uint8')
         flag_arr.fill(QartodFlags.UNKNOWN)
 
-        # If the value is masked set the flag to MISSING
-        flag_arr[inp.mask] = QartodFlags.MISSING
-
         # Iterate over each member and apply its spans on the input data.
         # Member spans are applied in order and any data points that fall into
         # more than one member are flagged by each one.
@@ -377,8 +374,9 @@ def check(self, tinp, inp, zinp):
                 with np.errstate(invalid='ignore'):
                     z_idx = (~zinp.mask) & (zinp >= m.zspan.minv) & (zinp <= m.zspan.maxv)
             else:
-                # Only test the values with masked Z, ie values with no Z
-                z_idx = zinp.mask
+                # If there is no z data in the config, don't try to filter by depth!
+                # Set z_idx to all True to prevent filtering
+                z_idx = np.ones(inp.size, dtype=bool)
 
             # Combine the T and Z indexes
             values_idx = (t_idx & z_idx)
@@ -398,6 +396,9 @@ def check(self, tinp, inp, zinp):
                 flag_arr[(values_idx & ~fail_idx & suspect_idx)] = QartodFlags.SUSPECT
                 flag_arr[(values_idx & ~fail_idx & ~suspect_idx)] = QartodFlags.GOOD
 
+        # If the value is masked set the flag to MISSING
+        flag_arr[inp.mask] = QartodFlags.MISSING
+
         return flag_arr
 
     @staticmethod
@@ -427,11 +428,11 @@ def climatology_test(config : Union[ClimatologyConfig, Sequence[Dict[str, Tuple]
         config: A ClimatologyConfig object or a list of dicts containing tuples
             that can be used to create a ClimatologyConfig object. See ClimatologyConfig
             docs for more info.
+        inp: Input data as a numeric numpy array or a list of numbers.
         tinp: Time data as a sequence of datetime objects compatible with pandas DatetimeIndex.
           This includes numpy datetime64, python datetime objects and pandas Timestamp object.
           ie. pd.DatetimeIndex([datetime.utcnow(), np.datetime64(), pd.Timestamp.now()]
           If anything else is passed in the format is assumed to be seconds since the unix epoch.
-        vinp: Input data as a numeric numpy array or a list of numbers.
         zinp: Z (depth) data, in meters positive down, as a numeric numpy array or a list of numbers.
 
     Returns:

tests/test_qartod.py

@@ -720,6 +720,60 @@ def test_climatology_test_all_unknown(self):
         self._run_test(test_inputs, expected_result)
 
 
+class QartodClimatologyMissingTest(unittest.TestCase):
+    def setUp(self):
+        self.cc = qartod.ClimatologyConfig()
+        # different time range, no depth
+        self.cc.add(
+            tspan=(np.datetime64('2021-07'), np.datetime64('2021-09')),
+            vspan=(3.4, 5)
+        )
+
+    def _run_test(self, test_inputs, expected_result):
+        times, values, depths = zip(*test_inputs)
+        inputs = [
+            values,
+            np.asarray(values, dtype=np.float64),
+            dask_arr(np.asarray(values, dtype=np.float64))
+        ]
+
+        for i in inputs:
+            results = qartod.climatology_test(
+                config=self.cc,
+                tinp=times,
+                inp=i,
+                zinp=depths
+            )
+            npt.assert_array_equal(
+                results,
+                np.ma.array(expected_result)
+            )
+
+    def test_climatology_missing_values(self):
+        test_inputs = [
+            # Not missing value or depth, value out of bounds
+            (
+                np.datetime64('2021-07-16'),
+                0,
+                0
+            ),
+            # Missing value and depth
+            (
+                np.datetime64('2021-07-16'),
+                np.nan,
+                np.nan
+            ),
+            # Not missing value and depth, value within bounds
+            (
+                np.datetime64('2021-07-16'),
+                4.16743,
+                0.08931513
+            )
+        ]
+        expected_result = [3, 9, 1]
+        self._run_test(test_inputs, expected_result)
+
+
 class QartodClimatologyTest(unittest.TestCase):
 
     def setUp(self):
@@ -857,7 +911,7 @@ def test_climatology_test_depths(self):
                 101
             )
         ]
-        expected_result = [1, 1, 1, 3, 3, 2]
+        expected_result = [1, 1, 1, 3, 3, 3]
         self._run_test(test_inputs, expected_result)