ULTRA l1b coinph valid and backtof valid flags (#2467)

* add more flags l1b

Author: lacoak21

imap_processing/quality_flags.py

@@ -46,6 +46,7 @@ class ImapDEOutliersUltraFlags(FlagNameMixin):
     COINPH = 2**2  # bit 2 # Event validity
     INVALID_ENERGY = 2**3  # bit 3
     DURINGREPOINT = 2**4  # bit 4 # event during a repointing
+    BACKTOF = 2**5  # bit 5 # Back TOF outlier
 
 
 class ImapHkUltraFlags(FlagNameMixin):

imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py

@@ -734,15 +734,24 @@ def test_is_back_tof_valid(test_fixture, ancillary_files):
     df_filt, _, _, de_dataset = test_fixture
     df_ph = df_filt[np.isin(df_filt["StopType"], [StopType.PH.value])]
 
-    valid = is_back_tof_valid(
-        de_dataset,
-        df_filt.Xf.astype("float").values,
+    # Get the ph inds
+    ph_mask = (
+        (de_dataset["stop_type"] == StopType.Top.value)
+        | (de_dataset["stop_type"] == StopType.Bottom.value)
+    ).values
+    quality_flags = np.zeros(len(np.nonzero(ph_mask)[0]), dtype=np.uint16)
+    valid, quality_flags = is_back_tof_valid(
+        de_dataset.isel(epoch=np.nonzero(ph_mask)[0]),
+        df_filt.Xf.astype("float").values[ph_mask],
         "ultra45",
         ancillary_files,
+        quality_flags,
     )
+
     back_tof_valid_bool = df_ph["BackTOFValid"].astype(int).astype(bool).values
 
     np.testing.assert_equal(back_tof_valid_bool, valid)
+    assert np.any(quality_flags)
 
 
 @pytest.mark.external_test_data
@@ -764,7 +773,7 @@ def test_is_coin_ph_valid(test_fixture, ancillary_files):
         len(ctof), ImapDEOutliersUltraFlags.NONE.value, dtype=np.uint16
     )
 
-    combined_mask = is_coin_ph_valid(
+    combined_mask, quality_flags = is_coin_ph_valid(
         etof,
         xc,
         xb,

imap_processing/ultra/l1b/de.py

@@ -209,13 +209,17 @@ def calculate_de(
         f"ultra{sensor}",
         ancillary_files,
     )
-    backtofvalid = is_back_tof_valid(
-        de_dataset,
-        xf,
+    backtofvalid_quality_flags = np.zeros(len(ph_indices), dtype=quality_flags.dtype)
+    backtofvalid, backtofvalid_quality_flags = is_back_tof_valid(
+        de_dataset.isel(epoch=ph_indices),
+        xf[ph_indices],
         f"ultra{sensor}",
         ancillary_files,
+        backtofvalid_quality_flags,
     )
-    coinphvalid = is_coin_ph_valid(
+
+    coinphvalid_quality_flags = np.zeros(len(ph_indices), dtype=quality_flags.dtype)
+    coinphvalid, coinphvalid_quality_flags = is_coin_ph_valid(
         etof[ph_indices],
         xc[ph_indices],
         xb[ph_indices],
@@ -225,8 +229,11 @@ def calculate_de(
         de_dataset["stop_west_tdc"][ph_indices].values,
         f"ultra{sensor}",
         ancillary_files,
-        quality_flags[ph_indices],
+        coinphvalid_quality_flags,
     )
+    quality_flags[ph_indices] |= coinphvalid_quality_flags
+    quality_flags[ph_indices] |= backtofvalid_quality_flags
+
     e_bin[ph_indices] = determine_ebin_pulse_height(
         energy[ph_indices],
         tof[ph_indices],

imap_processing/ultra/l1b/quality_flag_filters.py

@@ -19,6 +19,8 @@
     "quality_outliers": [
         ImapDEOutliersUltraFlags.FOV,
         ImapDEOutliersUltraFlags.DURINGREPOINT,
+        ImapDEOutliersUltraFlags.COINPH,
+        ImapDEOutliersUltraFlags.BACKTOF,
     ],
     "quality_scattering": [
         ImapDEScatteringUltraFlags.ABOVE_THRESHOLD,

imap_processing/ultra/l1b/ultra_l1b_extended.py

@@ -1342,7 +1342,8 @@ def is_back_tof_valid(
     xf: NDArray,
     sensor: str,
     ancillary_files: dict,
-) -> NDArray:
+    quality_flags: NDArray,
+) -> tuple[NDArray, NDArray]:
     """
     Determine whether back TOF is valid based on stop type.
 
@@ -1357,11 +1358,15 @@ def is_back_tof_valid(
         Sensor name: "ultra45" or "ultra90".
     ancillary_files : dict
         Ancillary files for lookup.
+    quality_flags : NDArray
+        Quality flag to set when there is an outlier.
 
     Returns
     -------
     valid_mask : NDArray
         Boolean array indicating whether back TOF is valid.
+    quality_flags : NDArray
+        Updated quality flags.
 
     Notes
     -----
@@ -1372,15 +1377,10 @@ def is_back_tof_valid(
     )
     diff = tofy - tofx
 
-    indices = np.nonzero(
-        np.isin(de_dataset["stop_type"], [StopType.Top.value, StopType.Bottom.value])
-    )[0]
-    de_ph = de_dataset.isel(epoch=indices)
-
-    top_mask = de_ph["stop_type"] == StopType.Top.value
-    bottom_mask = de_ph["stop_type"] == StopType.Bottom.value
+    top_mask = de_dataset["stop_type"] == StopType.Top.value
+    bottom_mask = de_dataset["stop_type"] == StopType.Bottom.value
 
-    valid = np.zeros_like(diff, dtype=bool)
+    valid = np.zeros(len(top_mask), dtype=bool)
 
     diff_tp_min = get_image_params("TOFDiffTpMin", sensor, ancillary_files)
     diff_tp_max = get_image_params("TOFDiffTpMax", sensor, ancillary_files)
@@ -1391,8 +1391,8 @@ def is_back_tof_valid(
     valid[bottom_mask] = (diff[bottom_mask] >= diff_bt_min) & (
         diff[bottom_mask] <= diff_bt_max
     )
-
-    return valid
+    quality_flags[~valid] |= ImapDEOutliersUltraFlags.BACKTOF.value
+    return valid, quality_flags
 
 
 def is_coin_ph_valid(
@@ -1406,7 +1406,7 @@ def is_coin_ph_valid(
     sensor: str,
     ancillary_files: dict,
     quality_flags: NDArray,
-) -> NDArray:
+) -> tuple[NDArray, NDArray]:
     """
     Determine event validity.
 
@@ -1438,6 +1438,8 @@ def is_coin_ph_valid(
     -------
     combined_mask : NDArray
         Boolean array indicating whether back TOF is valid.
+    quality_flags : NDArray
+        Updated quality flags.
 
     Notes
     -----
@@ -1486,4 +1488,4 @@ def is_coin_ph_valid(
 
     quality_flags[~combined_mask] |= ImapDEOutliersUltraFlags.COINPH.value
 
-    return combined_mask
+    return combined_mask, quality_flags