diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8430946..989a95a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,3 +24,14 @@ repos: )$ args: - --ignore-words-list=pres,ba,ot + +ci: + autofix_commit_msg: | + [pre-commit.ci] auto fixes from pre-commit.com hooks + + for more information, see https://pre-commit.ci + autofix_prs: false + autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate' + autoupdate_schedule: monthly + skip: [] + submodules: false diff --git a/ioos_qc/qartod.py b/ioos_qc/qartod.py index 80336cf..027a49f 100644 --- a/ioos_qc/qartod.py +++ b/ioos_qc/qartod.py @@ -340,9 +340,6 @@ def check(self, tinp, inp, zinp): flag_arr = np.ma.empty(inp.size, dtype='uint8') flag_arr.fill(QartodFlags.UNKNOWN) - # If the value is masked set the flag to MISSING - flag_arr[inp.mask] = QartodFlags.MISSING - # Iterate over each member and apply its spans on the input data. # Member spans are applied in order and any data points that fall into # more than one member are flagged by each one. @@ -377,8 +374,9 @@ def check(self, tinp, inp, zinp): with np.errstate(invalid='ignore'): z_idx = (~zinp.mask) & (zinp >= m.zspan.minv) & (zinp <= m.zspan.maxv) else: - # Only test the values with masked Z, ie values with no Z - z_idx = zinp.mask + # If there is no z data in the config, don't try to filter by depth! + # Set z_idx to all True to prevent filtering + z_idx = np.ones(inp.size, dtype=bool) # Combine the T and Z indexes values_idx = (t_idx & z_idx) @@ -398,6 +396,9 @@ def check(self, tinp, inp, zinp): flag_arr[(values_idx & ~fail_idx & suspect_idx)] = QartodFlags.SUSPECT flag_arr[(values_idx & ~fail_idx & ~suspect_idx)] = QartodFlags.GOOD + # If the value is masked set the flag to MISSING + flag_arr[inp.mask] = QartodFlags.MISSING + return flag_arr @staticmethod @@ -427,11 +428,11 @@ def climatology_test(config : Union[ClimatologyConfig, Sequence[Dict[str, Tuple] config: A ClimatologyConfig object or a list of dicts containing tuples that can be used to create a ClimatologyConfig object. See ClimatologyConfig docs for more info. + inp: Input data as a numeric numpy array or a list of numbers. tinp: Time data as a sequence of datetime objects compatible with pandas DatetimeIndex. This includes numpy datetime64, python datetime objects and pandas Timestamp object. ie. pd.DatetimeIndex([datetime.utcnow(), np.datetime64(), pd.Timestamp.now()] If anything else is passed in the format is assumed to be seconds since the unix epoch. - vinp: Input data as a numeric numpy array or a list of numbers. zinp: Z (depth) data, in meters positive down, as a numeric numpy array or a list of numbers. Returns: diff --git a/tests/test_qartod.py b/tests/test_qartod.py index 51c2bcc..d3f25b9 100644 --- a/tests/test_qartod.py +++ b/tests/test_qartod.py @@ -720,6 +720,60 @@ def test_climatology_test_all_unknown(self): self._run_test(test_inputs, expected_result) +class QartodClimatologyMissingTest(unittest.TestCase): + def setUp(self): + self.cc = qartod.ClimatologyConfig() + # different time range, no depth + self.cc.add( + tspan=(np.datetime64('2021-07'), np.datetime64('2021-09')), + vspan=(3.4, 5) + ) + + def _run_test(self, test_inputs, expected_result): + times, values, depths = zip(*test_inputs) + inputs = [ + values, + np.asarray(values, dtype=np.float64), + dask_arr(np.asarray(values, dtype=np.float64)) + ] + + for i in inputs: + results = qartod.climatology_test( + config=self.cc, + tinp=times, + inp=i, + zinp=depths + ) + npt.assert_array_equal( + results, + np.ma.array(expected_result) + ) + + def test_climatology_missing_values(self): + test_inputs = [ + # Not missing value or depth, value out of bounds + ( + np.datetime64('2021-07-16'), + 0, + 0 + ), + # Missing value and depth + ( + np.datetime64('2021-07-16'), + np.nan, + np.nan + ), + # Not missing value and depth, value within bounds + ( + np.datetime64('2021-07-16'), + 4.16743, + 0.08931513 + ) + ] + expected_result = [3, 9, 1] + self._run_test(test_inputs, expected_result) + + class QartodClimatologyTest(unittest.TestCase): def setUp(self): @@ -857,7 +911,7 @@ def test_climatology_test_depths(self): 101 ) ] - expected_result = [1, 1, 1, 3, 3, 2] + expected_result = [1, 1, 1, 3, 3, 3] self._run_test(test_inputs, expected_result)