Merge pull request #129 from Sakshamgupta90/fix/inconsistency

Fix flagging logic for missing and unknown data in QARTOD spike test

Author: ocefpaf

ioos_qc/qartod.py

@@ -612,8 +612,19 @@ def spike_test(
     flag_arr[0] = QartodFlags.UNKNOWN
     flag_arr[-1] = QartodFlags.UNKNOWN
 
-    # If the value is masked or nan set the flag to MISSING
-    flag_arr[diff.mask] = QartodFlags.MISSING
+    # Check if the original data was masked
+    for i in range(inp.size):
+
+       # Check if inp is masked (original data missing)
+        if inp.mask[i]:
+            flag_arr[i] = QartodFlags.MISSING
+        
+       # Check if diff is masked but not in inp (this indicates that original data is not missing, 
+       # but the data point got masked in diff lines 575-580 due to trying to calculate a value 
+       # using a valid value and a missing value; and because of that, we are not able to apply QARTOD
+       # thus the UNKNOWN flag)
+        elif (diff.mask[i] and not inp.mask[i]):
+            flag_arr[i] = QartodFlags.UNKNOWN
 
     return flag_arr.reshape(original_shape)
 

tests/test_qartod.py

@@ -1018,7 +1018,7 @@ def test_spike_masked(self):
 
         # First and last elements should always be good data, unless someone
         # has set a threshold to zero.
-        expected = [2, 4, 4, 4, 1, 3, 1, 9, 9, 9, 4, 4, 9, 9]
+        expected = [2, 4, 4, 4, 1, 3, 1, 2, 9, 2, 4, 4, 2, 9]
 
         inputs = [
             arr,
@@ -1126,46 +1126,8 @@ def test_spike_methods(self):
         inp = [3, 4.99, 5, 6, 8, 6, 6, 6.75, 6, 6, 5.3, 6, 6, 9, 5, None, 4, 4]
         suspect_threshold = 0.5
         fail_threshold = 1
-        average_method_expected = [
-            2,
-            3,
-            1,
-            1,
-            4,
-            3,
-            1,
-            3,
-            1,
-            1,
-            3,
-            1,
-            4,
-            4,
-            9,
-            9,
-            9,
-            2,
-        ]
-        diff_method_expected = [
-            2,
-            1,
-            1,
-            1,
-            4,
-            1,
-            1,
-            3,
-            1,
-            1,
-            3,
-            1,
-            1,
-            4,
-            9,
-            9,
-            9,
-            2,
-        ]
+        average_method_expected = [2, 3, 1, 1, 4, 3, 1, 3, 1, 1, 3, 1, 4, 4, 2, 9, 2, 2]
+        diff_method_expected = [2, 1, 1, 1, 4, 1, 1, 3, 1, 1, 3, 1, 1, 4, 2, 9, 2, 2]
 
         # Test average method
         npt.assert_array_equal(
@@ -1222,46 +1184,8 @@ def test_spike_test_bad_method(self):
 
     def test_spike_test_inputs(self):
         inp = [3, 4.99, 5, 6, 8, 6, 6, 6.75, 6, 6, 5.3, 6, 6, 9, 5, None, 4, 4]
-        expected_suspect_only = [
-            2,
-            3,
-            1,
-            1,
-            3,
-            3,
-            1,
-            3,
-            1,
-            1,
-            3,
-            1,
-            3,
-            3,
-            9,
-            9,
-            9,
-            2,
-        ]
-        expected_fail_only = [
-            2,
-            1,
-            1,
-            1,
-            4,
-            1,
-            1,
-            1,
-            1,
-            1,
-            1,
-            1,
-            4,
-            4,
-            9,
-            9,
-            9,
-            2,
-        ]
+        expected_suspect_only = [2, 3, 1, 1, 3, 3, 1, 3, 1, 1, 3, 1, 3, 3, 2, 9, 2, 2]
+        expected_fail_only = [2, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1, 4, 4, 2, 9, 2, 2]
         suspect_threshold = 0.5
         fail_threshold = 1