add new validation tables. Vectorize lookup

Author: lacoak21

imap_processing/codice/codice_l2.py

@@ -93,7 +93,9 @@ def get_mpq_calc_energy_conversion_vals(
     esa_kev : np.ndarray
         An array of energy in keV for each esa step.
     """
-    mpq_calc_lut_file = dependencies.get_file_paths(descriptor="lo-mpq-cal")[0]
+    mpq_calc_lut_file = dependencies.get_file_paths(descriptor="l2-lo-onboard-mpq-cal")[
+        0
+    ]
     mpq_df = pd.read_csv(mpq_calc_lut_file, header=None)
     k_factor = float(mpq_df.loc[0, 10])
     esa_v = mpq_df.loc[4, 4:].to_numpy().astype(np.float64)
@@ -1017,12 +1019,14 @@ def process_lo_direct_events(dependencies: ProcessingInputCollection) -> xr.Data
     apd_energy_shape = l2_dataset["apd_energy"].shape
 
     # The energy table lookup columns are ordered by apd_id and gain
-    # E.g. APD-0-LG, APD-0-HG, APD-1-LG, APD-1-HG, ..., APD-29-LG
+    # E.g. APD-1-LG, APD-1-HG, ..., APD-29-LG
     # So we can get the col index like so: ind = apd_id * 2 + gain
     col_inds = apd_ids * 2 + gains
     # Get a mask of valid indices
-    valid_mask = (apd_energy < energy_table.shape[0]) & (
-        col_inds < energy_table.shape[1]
+    valid_mask = (
+        (apd_energy < energy_table.shape[0])
+        & (col_inds < energy_table.shape[1])
+        & (apd_ids > 0)
     )
     # Initialize output array with NaNs
     energy_bins_inds = np.full(apd_energy.shape, np.nan)
@@ -1106,23 +1110,6 @@ def process_lo_direct_events(dependencies: ProcessingInputCollection) -> xr.Data
         attrs=cdf_attrs.get_variable_attributes("priority_label", check_schema=False),
     )
 
-    print("\n=== DATA TYPE CHECK ===")
-    print(f"event_num_label dtype: {l2_dataset['event_num_label'].dtype}")
-    print(f"priority_label dtype: {l2_dataset['priority_label'].dtype}")
-    print(f"elevation_angle dtype: {l2_dataset['elevation_angle'].dtype}")
-    print(f"spin_angle dtype: {l2_dataset['spin_angle'].dtype}")
-    print(f"apd_energy dtype: {l2_dataset['apd_energy'].dtype}")
-    print(f"energy_per_charge dtype: {l2_dataset['energy_per_charge'].dtype}")
-    print(f"tof dtype: {l2_dataset['tof'].dtype}")
-    print(f"epoch_delta_minus dtype: {l2_dataset['epoch_delta_minus'].dtype}")
-    print(f"epoch_delta_plus dtype: {l2_dataset['epoch_delta_plus'].dtype}")
-
-    # Check for any unexpected dtypes
-    for var in l2_dataset.data_vars:
-        if "U" in str(l2_dataset[var].dtype) and "|S" not in str(l2_dataset[var].dtype):
-            if "<U" not in str(l2_dataset[var].dtype):
-                print(f"⚠️  WARNING: {var} has dtype {l2_dataset[var].dtype}")
-
     return l2_dataset
 
 

imap_processing/tests/codice/conftest.py

@@ -9,7 +9,7 @@
 TEST_L0_FILE = TEST_DATA_L0_PATH / "imap_codice_l0_raw_20241110_v001.pkts"
 
 VALIDATION_FILE_DATE = "20250814"
-VALIDATION_FILE_VERSION = "v011"
+VALIDATION_FILE_VERSION = "v012"
 
 
 @pytest.fixture(scope="session")

imap_processing/tests/codice/test_codice_l2.py

@@ -48,10 +48,7 @@
 def processing_dependencies(codice_lut_path):
     eff_file = "imap_codice_l2-lo-efficiency_20251008_v001.csv"
     gf_file = "imap_codice_l2-lo-gfactor_20251008_v001.csv"
-    mpq_file = "imap_codice_lo-mpq-cal_20250101_v001.csv"
-    return ProcessingInputCollection(
-        AncillaryInput(gf_file), AncillaryInput(eff_file), AncillaryInput(mpq_file)
-    )
+    return ProcessingInputCollection(AncillaryInput(gf_file), AncillaryInput(eff_file))
 
 
 @pytest.fixture
@@ -181,9 +178,9 @@ def test_get_efficiency_lut(processing_dependencies, mock_get_file_paths):
 def test_get_tof_ns_from_mpq_lut(processing_dependencies, mock_get_file_paths):
     tof_ns = get_mpq_calc_tof_conversion_vals(processing_dependencies)
     assert tof_ns.shape == (1024,)
-    mpq_calc_lut_file = processing_dependencies.get_file_paths(descriptor="lo-mpq-cal")[
-        0
-    ]
+    mpq_calc_lut_file = processing_dependencies.get_file_paths(
+        descriptor="l2-lo-onboard-mpq-cal"
+    )[0]
     mpq_df = pd.read_csv(mpq_calc_lut_file, header=None)
     expected_tof_ns = mpq_df.loc[6:, 1].to_numpy().astype(np.float64)
     # Calculated values should be more precise than LUT but should be close
@@ -193,9 +190,9 @@ def test_get_tof_ns_from_mpq_lut(processing_dependencies, mock_get_file_paths):
 def test_get_energy_kev_from_mpq_lut(processing_dependencies, mock_get_file_paths):
     energy_kev = get_mpq_calc_energy_conversion_vals(processing_dependencies)
     assert energy_kev.shape == (128,)
-    mpq_calc_lut_file = processing_dependencies.get_file_paths(descriptor="lo-mpq-cal")[
-        0
-    ]
+    mpq_calc_lut_file = processing_dependencies.get_file_paths(
+        descriptor="l2-lo-onboard-mpq-cal"
+    )[0]
     mpq_df = pd.read_csv(mpq_calc_lut_file, header=None)
     expected_tof_ns = mpq_df.loc[5, 4:].to_numpy().astype(np.float64)
     # Calculated values should be more precise than LUT but should be close
@@ -524,6 +521,7 @@ def test_codice_l2_lo_de(mock_get_file_paths, codice_lut_path):
         codice_lut_path(descriptor="l2-lo-onboard-energy-table"),
         codice_lut_path(descriptor="l2-lo-onboard-energy-bins"),
         codice_lut_path(descriptor="l2-lo-onboard-mpq-cal"),
+        codice_lut_path(descriptor="l2-lo-onboard-mpq-cal"),
     ]
 
     processed_l2_ds = process_codice_l2("lo-direct-events", ProcessingInputCollection())
@@ -538,7 +536,9 @@ def test_codice_l2_lo_de(mock_get_file_paths, codice_lut_path):
             f"_{VALIDATION_FILE_VERSION}.cdf"
         )
     )
+
     l2_val_data = load_cdf(l2_val_data)
+
     for variable in l2_val_data.data_vars:
         if variable in ["spin_angle"]:
             # TODO remove this block when joey fixes spin_angle calculation
@@ -555,11 +555,11 @@ def test_codice_l2_lo_de(mock_get_file_paths, codice_lut_path):
                 l2_val_data[variable].values,
                 rtol=5e-5,
                 err_msg=f"Mismatch in variable '{variable}'",
+                equal_nan=True,
             )
-
     processed_l2_ds.attrs["Data_version"] = "001"
     assert processed_l2_ds.attrs["Logical_source"] == "imap_codice_l2_lo-direct-events"
     file = write_cdf(processed_l2_ds)
-    errors = CDFValidator().validate_raw(file)
+    errors = CDFValidator().validate(file)
     assert not errors
     load_cdf(file)