Skip to content

Commit c2fe7ed

Browse files
authored
Merge pull request #129 from Sakshamgupta90/fix/inconsistency
Fix flagging logic for missing and unknown data in QARTOD spike test
2 parents 58546ea + 421ee5d commit c2fe7ed

File tree

2 files changed

+18
-83
lines changed

2 files changed

+18
-83
lines changed

ioos_qc/qartod.py

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -612,8 +612,19 @@ def spike_test(
612612
flag_arr[0] = QartodFlags.UNKNOWN
613613
flag_arr[-1] = QartodFlags.UNKNOWN
614614

615-
# If the value is masked or nan set the flag to MISSING
616-
flag_arr[diff.mask] = QartodFlags.MISSING
615+
# Check if the original data was masked
616+
for i in range(inp.size):
617+
618+
# Check if inp is masked (original data missing)
619+
if inp.mask[i]:
620+
flag_arr[i] = QartodFlags.MISSING
621+
622+
# Check if diff is masked but not in inp (this indicates that original data is not missing,
623+
# but the data point got masked in diff lines 575-580 due to trying to calculate a value
624+
# using a valid value and a missing value; and because of that, we are not able to apply QARTOD
625+
# thus the UNKNOWN flag)
626+
elif (diff.mask[i] and not inp.mask[i]):
627+
flag_arr[i] = QartodFlags.UNKNOWN
617628

618629
return flag_arr.reshape(original_shape)
619630

tests/test_qartod.py

Lines changed: 5 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -1018,7 +1018,7 @@ def test_spike_masked(self):
10181018

10191019
# First and last elements should always be good data, unless someone
10201020
# has set a threshold to zero.
1021-
expected = [2, 4, 4, 4, 1, 3, 1, 9, 9, 9, 4, 4, 9, 9]
1021+
expected = [2, 4, 4, 4, 1, 3, 1, 2, 9, 2, 4, 4, 2, 9]
10221022

10231023
inputs = [
10241024
arr,
@@ -1126,46 +1126,8 @@ def test_spike_methods(self):
11261126
inp = [3, 4.99, 5, 6, 8, 6, 6, 6.75, 6, 6, 5.3, 6, 6, 9, 5, None, 4, 4]
11271127
suspect_threshold = 0.5
11281128
fail_threshold = 1
1129-
average_method_expected = [
1130-
2,
1131-
3,
1132-
1,
1133-
1,
1134-
4,
1135-
3,
1136-
1,
1137-
3,
1138-
1,
1139-
1,
1140-
3,
1141-
1,
1142-
4,
1143-
4,
1144-
9,
1145-
9,
1146-
9,
1147-
2,
1148-
]
1149-
diff_method_expected = [
1150-
2,
1151-
1,
1152-
1,
1153-
1,
1154-
4,
1155-
1,
1156-
1,
1157-
3,
1158-
1,
1159-
1,
1160-
3,
1161-
1,
1162-
1,
1163-
4,
1164-
9,
1165-
9,
1166-
9,
1167-
2,
1168-
]
1129+
average_method_expected = [2, 3, 1, 1, 4, 3, 1, 3, 1, 1, 3, 1, 4, 4, 2, 9, 2, 2]
1130+
diff_method_expected = [2, 1, 1, 1, 4, 1, 1, 3, 1, 1, 3, 1, 1, 4, 2, 9, 2, 2]
11691131

11701132
# Test average method
11711133
npt.assert_array_equal(
@@ -1222,46 +1184,8 @@ def test_spike_test_bad_method(self):
12221184

12231185
def test_spike_test_inputs(self):
12241186
inp = [3, 4.99, 5, 6, 8, 6, 6, 6.75, 6, 6, 5.3, 6, 6, 9, 5, None, 4, 4]
1225-
expected_suspect_only = [
1226-
2,
1227-
3,
1228-
1,
1229-
1,
1230-
3,
1231-
3,
1232-
1,
1233-
3,
1234-
1,
1235-
1,
1236-
3,
1237-
1,
1238-
3,
1239-
3,
1240-
9,
1241-
9,
1242-
9,
1243-
2,
1244-
]
1245-
expected_fail_only = [
1246-
2,
1247-
1,
1248-
1,
1249-
1,
1250-
4,
1251-
1,
1252-
1,
1253-
1,
1254-
1,
1255-
1,
1256-
1,
1257-
1,
1258-
4,
1259-
4,
1260-
9,
1261-
9,
1262-
9,
1263-
2,
1264-
]
1187+
expected_suspect_only = [2, 3, 1, 1, 3, 3, 1, 3, 1, 1, 3, 1, 3, 3, 2, 9, 2, 2]
1188+
expected_fail_only = [2, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1, 4, 4, 2, 9, 2, 2]
12651189
suspect_threshold = 0.5
12661190
fail_threshold = 1
12671191

0 commit comments

Comments
 (0)