diff --git a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml index c9523aaf6..42864b5bd 100644 --- a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml @@ -518,11 +518,6 @@ lo-low_tof_cutoff: CATDESC: Low TOF Cutoff FIELDNAM: Low TOF Cutoff -lo-low_tof_cutoff: - <<: *lo_counters_aggregated_default - CATDESC: Low TOF Cutoff - FIELDNAM: Low TOF Cutoff - lo-asic1_flag_invalid: <<: *lo_counters_aggregated_default CATDESC: ASIC 1 Flag Invalid Count diff --git a/imap_processing/cdf/config/imap_codice_l2-lo-direct-events_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l2-lo-direct-events_variable_attrs.yaml new file mode 100644 index 000000000..82cda9c07 --- /dev/null +++ b/imap_processing/cdf/config/imap_codice_l2-lo-direct-events_variable_attrs.yaml @@ -0,0 +1,323 @@ +# ----------------------------- Useful variables ----------------------------- +uint8_fillval: &uint8_fillval 255 +uint16_fillval: &uint16_fillval 65535 +real_fillval: &real_fillval -1.0e+31 +min_int: &min_int -9223372036854775808 +max_int: &max_int 9223372036854775807 +# ------------------------------- Coordinates ------------------------------- +priority: + CATDESC: Priority Level + FIELDNAM: Priority Level + FILLVAL: *uint8_fillval + FORMAT: I1 + LABLAXIS: Priority + SCALETYP: linear + UNITS: " " + VALIDMIN: 0 + VALIDMAX: 7 + VAR_TYPE: support_data + +event_num: + CATDESC: Event Number + FIELDNAM: Event Number + FILLVAL: *uint16_fillval + FORMAT: I5 + LABLAXIS: Event Number + SCALETYP: linear + UNITS: " " + VALIDMIN: 0 + VALIDMAX: 10000 + VAR_TYPE: support_data + +epoch_delta_minus: + CATDESC: Time from acquisition start to acquisition center + FIELDNAM: epoch delta minus + DEPEND_0: epoch + FILLVAL: -9223372036854775808 + FORMAT: I18 + LABLAXIS: Epoch Delta Minus + SCALETYP: linear + UNITS: ns + VALIDMIN: *min_int + VALIDMAX: *max_int + VAR_TYPE: support_data + DICT_KEY: SPASE>Support>SupportQuantity:Temporal,Qualifier:Uncertainty + +epoch_delta_plus: + CATDESC: Time from acquisition center to acquisition end + FIELDNAM: epoch delta plus + DEPEND_0: epoch + FILLVAL: -9223372036854775808 + FORMAT: I18 + LABLAXIS: Epoch Delta Plus + SCALETYP: linear + UNITS: ns + VALIDMIN: *min_int + VALIDMAX: *max_int + VAR_TYPE: support_data + DICT_KEY: SPASE>Support>SupportQuantity:Temporal,Qualifier:Uncertainty + +# ------------------------------- Data Variables ------------------------------- +num_events: + CATDESC: Number of Events per Priority + DEPEND_0: epoch + DEPEND_1: priority + FIELDNAM: Number of Events + FILLVAL: *uint16_fillval + FORMAT: I5 + LABLAXIS: Number of Events + LABL_PTR_1: priority_label + SCALETYP: linear + UNITS: " " + VALIDMAX: 10000 + VALIDMIN: 0 + VAR_TYPE: data + +data_quality: + CATDESC: Data Quality Flag per Priority + DEPEND_0: epoch + DEPEND_1: priority + FIELDNAM: Data Quality + FILLVAL: *uint8_fillval + FORMAT: I3 + LABLAXIS: Data Quality + LABL_PTR_1: priority_label + SCALETYP: linear + UNITS: " " + VALIDMAX: 255 + VALIDMIN: 0 + VAR_TYPE: data + +energy_step: + CATDESC: Energy Step Index + DEPEND_0: epoch + DEPEND_1: priority + DEPEND_2: event_num + FIELDNAM: Energy Step + FILLVAL: *uint8_fillval + FORMAT: I3 + LABLAXIS: Energy Step + LABL_PTR_1: priority_label + LABL_PTR_2: event_num_label + SCALETYP: linear + UNITS: " " + VALIDMAX: 127 + VALIDMIN: 0 + VAR_TYPE: data + +energy_per_charge: + CATDESC: Energy per Charge + DEPEND_0: epoch + DEPEND_1: priority + DEPEND_2: event_num + FIELDNAM: Energy per Charge + FILLVAL: *real_fillval + FORMAT: F12.4 + LABLAXIS: Energy per Charge + LABL_PTR_1: priority_label + LABL_PTR_2: event_num_label + SCALETYP: linear + UNITS: keV/q + VALIDMAX: 200.0 + VALIDMIN: 0.0 + VAR_TYPE: data + +apd_energy: + CATDESC: APD Energy + DEPEND_0: epoch + DEPEND_1: priority + DEPEND_2: event_num + FIELDNAM: APD Energy + FILLVAL: *real_fillval + FORMAT: F12.4 + LABLAXIS: APD Energy + LABL_PTR_1: priority_label + LABL_PTR_2: event_num_label + SCALETYP: linear + UNITS: keV + VALIDMAX: 512.0 + VALIDMIN: 0.0 + VAR_TYPE: data + +gain: + CATDESC: Gain Setting + DEPEND_0: epoch + DEPEND_1: priority + DEPEND_2: event_num + FIELDNAM: Gain + FILLVAL: *uint8_fillval + FORMAT: I1 + LABLAXIS: Gain + LABL_PTR_1: priority_label + LABL_PTR_2: event_num_label + SCALETYP: linear + UNITS: " " + VALIDMAX: 1 + VALIDMIN: 0 + VAR_TYPE: data + +apd_id: + CATDESC: APD Identifier + DEPEND_0: epoch + DEPEND_1: priority + DEPEND_2: event_num + FIELDNAM: APD ID + FILLVAL: *uint8_fillval + FORMAT: I2 + LABLAXIS: APD ID + LABL_PTR_1: priority_label + LABL_PTR_2: event_num_label + SCALETYP: linear + UNITS: " " + VALIDMAX: 31 + VALIDMIN: 0 + VAR_TYPE: data + +position: + CATDESC: Position Index + DEPEND_0: epoch + DEPEND_1: priority + DEPEND_2: event_num + FIELDNAM: Position + FILLVAL: *uint8_fillval + FORMAT: I2 + LABLAXIS: Position + LABL_PTR_1: priority_label + LABL_PTR_2: event_num_label + SCALETYP: linear + UNITS: " " + VALIDMAX: 31 + VALIDMIN: 0 + VAR_TYPE: data + +multi_flag: + CATDESC: Multiple Event Flag + DEPEND_0: epoch + DEPEND_1: priority + DEPEND_2: event_num + FIELDNAM: Multi Flag + FILLVAL: *uint8_fillval + FORMAT: I1 + LABLAXIS: Multi Flag + LABL_PTR_1: priority_label + LABL_PTR_2: event_num_label + SCALETYP: linear + UNITS: " " + VALIDMAX: 1 + VALIDMIN: 0 + VAR_TYPE: data + +type: + CATDESC: Event Type + DEPEND_0: epoch + DEPEND_1: priority + DEPEND_2: event_num + FIELDNAM: Type + FILLVAL: *uint8_fillval + FORMAT: I1 + LABLAXIS: Type + LABL_PTR_1: priority_label + LABL_PTR_2: event_num_label + SCALETYP: linear + UNITS: " " + VALIDMAX: 4 + VALIDMIN: 0 + VAR_TYPE: data + +tof: + CATDESC: Time of Flight + DEPEND_0: epoch + DEPEND_1: priority + DEPEND_2: event_num + FIELDNAM: Time of Flight + FILLVAL: *real_fillval + FORMAT: F12.4 + LABLAXIS: TOF + LABL_PTR_1: priority_label + LABL_PTR_2: event_num_label + SCALETYP: linear + UNITS: ns + VALIDMAX: 1024.0 + VALIDMIN: 0.0 + VAR_TYPE: data + +spin_sector: + CATDESC: Spin Sector Index + FIELDNAM: Spin Sector Index + DEPEND_0: epoch + DEPEND_1: priority + DEPEND_2: event_num + LABL_PTR_1: priority_label + LABL_PTR_2: event_num_label + FILLVAL: *uint8_fillval + FORMAT: I2 + LABLAXIS: Spin Sector + SCALETYP: linear + UNITS: " " + VALIDMIN: 0 + VALIDMAX: 23 + VAR_TYPE: support_data + +spin_angle: + VAR_TYPE: data + CATDESC: Spin Angle + FIELDNAM: Spin Angle + DEPEND_0: epoch + DEPEND_1: priority + DEPEND_2: event_num + LABL_PTR_1: priority_label + LABL_PTR_2: event_num_label + SCALETYP: linear + UNITS: degrees + FILLVAL: *real_fillval + VALIDMAX: 360.0 + VALIDMIN: 0.0 + FORMAT: F8.2 + DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:ArrivalDirection,Qualifier:DirectionAngle.AzimuthAngle + +elevation_angle: + CATDESC: Elevation Angle + FIELDNAM: Elevation Angle + DEPEND_0: epoch + DEPEND_1: priority + DEPEND_2: event_num + LABL_PTR_1: priority_label + LABL_PTR_2: event_num_label + FORMAT: F8.2 + FILLVAL: *real_fillval + LABLAXIS: Elevation Angle + SCALETYP: linear + UNITS: degrees + VALIDMAX: 180.0 + VALIDMIN: 0.0 + VAR_TYPE: data + +esa_step: + CATDESC: Energy per charge (E/q) sweeping step + FIELDNAM: Energy Index + DEPEND_0: epoch + DEPEND_1: priority + DEPEND_2: event_num + LABL_PTR_1: priority_label + LABL_PTR_2: event_num_label + FILLVAL: *uint8_fillval + FORMAT: I3 + LABLAXIS: Energy Index + SCALETYP: linear + UNITS: " " + VALIDMIN: 0 + VALIDMAX: 127 + VAR_TYPE: data +# ------------------------------- labels ------------------------------- + +event_num_label: + CATDESC: Event Number Label + FIELDNAM: Event Number + FORMAT: A5 + VAR_TYPE: metadata + +priority_label: + CATDESC: Priority Level Label + FIELDNAM: Priority Level + FORMAT: A1 + VAR_TYPE: metadata \ No newline at end of file diff --git a/imap_processing/codice/codice_l2.py b/imap_processing/codice/codice_l2.py index 3df91eca8..ba5ba85c1 100644 --- a/imap_processing/codice/codice_l2.py +++ b/imap_processing/codice/codice_l2.py @@ -10,12 +10,12 @@ """ import logging -from pathlib import Path import numpy as np import pandas as pd import xarray as xr from imap_data_access import ProcessingInputCollection, ScienceFilePath +from numpy._typing import NDArray from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes from imap_processing.cdf.utils import load_cdf @@ -44,6 +44,97 @@ logger = logging.getLogger(__name__) +def get_lo_de_energy_luts( + dependencies: ProcessingInputCollection, +) -> tuple[NDArray, NDArray]: + """ + Get the LO DE lookup tables for energy conversions. + + Parameters + ---------- + dependencies : ProcessingInputCollection + The collection of processing input files. + + Returns + ------- + energy_lut : np.ndarray + An array of energy in keV for each energy table index. + energy_bins_lut : np.ndarray + An array of energy bins. + """ + # Get lookup tables + energy_table_file = dependencies.get_file_paths( + descriptor="l2-lo-onboard-energy-table" + )[0] + energy_bins_file = dependencies.get_file_paths( + descriptor="l2-lo-onboard-energy-bins" + )[0] + energy_lut = pd.read_csv(energy_table_file, header=None, skiprows=1).to_numpy() + energy_bins_lut = pd.read_csv(energy_bins_file, header=None, skiprows=1).to_numpy()[ + :, 1 + ] + + return energy_lut, energy_bins_lut + + +def get_mpq_calc_energy_conversion_vals( + dependencies: ProcessingInputCollection, +) -> np.ndarray: + """ + Get the MPQ calculation esa step to energy kev conversion lookup table values. + + Parameters + ---------- + dependencies : ProcessingInputCollection + The collection of processing input files. + + Returns + ------- + esa_kev : np.ndarray + An array of energy in keV for each esa step. + """ + mpq_calc_lut_file = dependencies.get_file_paths(descriptor="l2-lo-onboard-mpq-cal")[ + 0 + ] + mpq_df = pd.read_csv(mpq_calc_lut_file, header=None) + k_factor = float(mpq_df.loc[0, 10]) + esa_v = mpq_df.loc[4, 4:].to_numpy().astype(np.float64) + # Calculate the energy in keV for each esa step + esa_kev = esa_v * k_factor / 1000 + return esa_kev + + +def get_mpq_calc_tof_conversion_vals( + dependencies: ProcessingInputCollection, +) -> np.ndarray: + """ + Get the MPQ calculation tof to ns conversion lookup table values. + + Parameters + ---------- + dependencies : ProcessingInputCollection + The collection of processing input files. + + Returns + ------- + tof_ns : np.ndarray + Tof in ns for each TOF bit. + """ + mpq_calc_lut_file = dependencies.get_file_paths(descriptor="l2-lo-onboard-mpq-cal")[ + 0 + ] + mpq_df = pd.read_csv(mpq_calc_lut_file, header=None) + ns_channel_sq = float(mpq_df.loc[2, 1]) + ns_channel = float(mpq_df.loc[3, 1]) + tof_offset = float(mpq_df.loc[4, 1]) + # Get the TOF bit to ns lookup + tof_bits = mpq_df.loc[6:, 0].to_numpy().astype(np.int64) + # Calculate the TOF in ns for each TOF bit + tof_ns = tof_bits**2 * ns_channel_sq + tof_bits * ns_channel + tof_offset + + return tof_ns + + def get_geometric_factor_lut( dependencies: ProcessingInputCollection | None, path: Path | None = None, @@ -866,6 +957,162 @@ def process_hi_sectored(dependencies: ProcessingInputCollection) -> xr.Dataset: return l2_dataset +def process_lo_direct_events(dependencies: ProcessingInputCollection) -> xr.Dataset: + """ + Process the lo-direct-events L1A dataset to convert variables to physical units. + + See section 11.2.1 of the CoDICE algorithm document for details. + + Parameters + ---------- + dependencies : ProcessingInputCollection + The collection of processing input files. + + Returns + ------- + xarray.Dataset + The updated L2 dataset with variables converted to physical units. + """ + file_path = dependencies.get_file_paths(descriptor="lo-direct-events")[0] + l1a_dataset = load_cdf(file_path) + + # Update global CDF attributes + cdf_attrs = ImapCdfAttributes() + cdf_attrs.add_instrument_global_attrs("codice") + cdf_attrs.add_instrument_variable_attrs("codice", "l2-lo-direct-events") + energy_table, energy_bins = get_lo_de_energy_luts(dependencies) + # Convert from position to elevation angle in degrees relative to the spacecraft + # axis + l2_dataset = l1a_dataset.copy(deep=True) + # Create a new coordinate for elevation_angle based on inst_az + pos_to_els = ( + LO_POSITION_TO_ELEVATION_ANGLE["sw"] | LO_POSITION_TO_ELEVATION_ANGLE["nsw"] + ) + elevation_angle_shape = l2_dataset["position"].shape + elevation_angle = np.array( + [pos_to_els.get(pos, np.nan) for pos in l2_dataset["position"].values.flat] + ).reshape(elevation_angle_shape) + l2_dataset["elevation_angle"] = ( + l2_dataset["position"].dims, + elevation_angle.astype(np.float32), + ) + # Convert spin_sector to spin_angle in degrees + # Use equation from section 11.2.2 of algorithm document + # Shift all spin sectors for all positions 13 - 24 adding 12 and mod 24 + l2_dataset["spin_sector"] = xr.where( + (l2_dataset["position"] >= 13) & (l2_dataset["position"] <= 24), + (l2_dataset["spin_sector"] + 12) % 24, + l2_dataset["spin_sector"], + ) + l2_dataset["spin_angle"] = xr.DataArray( + data=l2_dataset["spin_sector"].data * 15.0 + 7.5, + dims=l2_dataset["spin_sector"].dims, + ).astype(np.float32) + l2_dataset["spin_angle"] = xr.where( + (l2_dataset["spin_sector"] > 23), np.nan, l2_dataset["spin_angle"] + ) + # convert apd energy to physical units + # Set the gain labels based on gain values + gains = l2_dataset["gain"].values.ravel() + apd_ids = l2_dataset["apd_id"].values.ravel() + apd_energy = l2_dataset["apd_energy"].values.ravel() + apd_energy_shape = l2_dataset["apd_energy"].shape + + # The energy table lookup columns are ordered by apd_id and gain + # E.g. APD-1-LG, APD-1-HG, ..., APD-29-LG + # So we can get the col index like so: ind = apd_id * 2 + gain + col_inds = apd_ids * 2 + gains + # Get a mask of valid indices + valid_mask = ( + (apd_energy < energy_table.shape[0]) + & (col_inds < energy_table.shape[1]) + & (apd_ids > 0) + ) + # Initialize output array with NaNs + energy_bins_inds = np.full(apd_energy.shape, np.nan) + energy_kev = np.full(apd_energy.shape, np.nan) + # The rows are apd_energy bins + energy_bins_inds[valid_mask] = energy_table[ + apd_energy[valid_mask], col_inds[valid_mask] + ] + energy_kev[valid_mask] = energy_bins[energy_bins_inds[valid_mask].astype(int)] + + l2_dataset["apd_energy"].data = ( + np.array(energy_kev).astype(np.float32).reshape(apd_energy_shape) + ) + + # Calculate TOF in nanoseconds + tof_bit_to_ns = get_mpq_calc_tof_conversion_vals(dependencies) + tof_bits = l2_dataset["tof"].values.flatten() + # Create output array + tof_ns = np.full(tof_bits.shape, np.nan, dtype=np.float64) + # Get only valid TOF bits between 0 and 1023 + valid_mask = (tof_bits >= 0) & (tof_bits < 1024) + tof_ns[valid_mask] = tof_bit_to_ns[tof_bits[valid_mask]] + # Reshape back to original shape + l2_dataset["tof"].data = tof_ns.astype(np.float32).reshape(l2_dataset["tof"].shape) + + # Convert energy step to energy in keV + esa_kev = get_mpq_calc_energy_conversion_vals(dependencies) + energy_steps = l2_dataset["energy_step"].values.flatten() + # Create output array + kev = np.full(energy_steps.shape, np.nan, dtype=np.float64) + # Get only valid energy_steps between 0 and 128 + valid_mask = (energy_steps >= 0) & (energy_steps < 128) + kev[valid_mask] = esa_kev[energy_steps[valid_mask]] + # Reshape back to original shape + l2_dataset["energy_per_charge"] = ( + l2_dataset["energy_step"].dims, + kev.astype(np.float32).reshape(l2_dataset["energy_step"].shape), + ) + # Drop unused variables + vars_to_drop = ["spare", "sw_bias_gain_mode", "st_bias_gain_mode", "k_factor"] + l2_dataset = l2_dataset.drop_vars(vars_to_drop) + # Update variable attributes + l2_dataset.attrs.update( + cdf_attrs.get_global_attributes("imap_codice_l2_lo-direct-events") + ) + for var in l2_dataset.data_vars: + l2_dataset[var].attrs.update(cdf_attrs.get_variable_attributes(var)) + # Update coord attributes + l2_dataset["priority"].attrs.update( + cdf_attrs.get_variable_attributes("priority", check_schema=False) + ) + l2_dataset["event_num"].attrs.update( + cdf_attrs.get_variable_attributes("event_num", check_schema=False) + ) + l2_dataset["epoch"] = xr.DataArray( + l2_dataset["epoch"].data, + dims="epoch", + attrs=cdf_attrs.get_variable_attributes("epoch", check_schema=False), + ) + l2_dataset["epoch_delta_minus"] = xr.DataArray( + data=l2_dataset["epoch_delta_minus"].data.astype(np.int64), + dims="epoch", + attrs=cdf_attrs.get_variable_attributes( + "epoch_delta_minus", check_schema=False + ), + ) + l2_dataset["epoch_delta_plus"] = xr.DataArray( + l2_dataset["epoch_delta_plus"].data.astype(np.int64), + dims="epoch", + attrs=cdf_attrs.get_variable_attributes("epoch_delta_plus", check_schema=False), + ) + # Add labels + l2_dataset["event_num_label"] = xr.DataArray( + l2_dataset["event_num"].values.astype(str).astype(" xr.Dataset: @@ -1022,7 +1269,7 @@ def process_codice_l2( # These converted variables are *in addition* to the existing L1 variables # The other data variables require no changes # See section 11.1.2 of algorithm document - pass + l2_dataset = process_lo_direct_events(dependencies) # logger.info(f"\nFinal data product:\n{l2_dataset}\n") diff --git a/imap_processing/tests/codice/conftest.py b/imap_processing/tests/codice/conftest.py index c705c4252..049ff16ba 100644 --- a/imap_processing/tests/codice/conftest.py +++ b/imap_processing/tests/codice/conftest.py @@ -9,7 +9,7 @@ TEST_L0_FILE = TEST_DATA_L0_PATH / "imap_codice_l0_raw_20241110_v001.pkts" VALIDATION_FILE_DATE = "20250814" -VALIDATION_FILE_VERSION = "v011" +VALIDATION_FILE_VERSION = "v012" @pytest.fixture(scope="session") @@ -256,6 +256,21 @@ def _side_effect(descriptor: str = None, data_type: str = None) -> list[Path]: return [ TEST_DATA_PATH / "l2_lut/imap_codice_l2-lo-gfactor_20251008_v001.csv" ] + elif descriptor == "l2-lo-onboard-mpq-cal": + return [ + TEST_DATA_PATH + / "l2_lut/imap_codice_l2-lo-onboard-mpq-cal_20250101_v001.csv" + ] + elif descriptor == "l2-lo-onboard-energy-bins": + return [ + TEST_DATA_PATH + / "l2_lut/imap_codice_l2-lo-onboard-energy-bins_20250101_v001.csv" + ] + elif descriptor == "l2-lo-onboard-energy-table": + return [ + TEST_DATA_PATH + / "l2_lut/imap_codice_l2-lo-onboard-energy-table_20250101_v001.csv" + ] else: raise ValueError(f"Unknown descriptor: {descriptor}") diff --git a/imap_processing/tests/codice/test_codice_l2.py b/imap_processing/tests/codice/test_codice_l2.py index 720641cb3..7d08181f3 100644 --- a/imap_processing/tests/codice/test_codice_l2.py +++ b/imap_processing/tests/codice/test_codice_l2.py @@ -8,6 +8,7 @@ import pytest import xarray as xr from imap_data_access import AncillaryInput, ProcessingInputCollection +from sammi.validation import CDFValidator from imap_processing import imap_module_directory from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes @@ -18,6 +19,8 @@ compute_geometric_factors, get_efficiency_lut, get_geometric_factor_lut, + get_mpq_calc_energy_conversion_vals, + get_mpq_calc_tof_conversion_vals, process_codice_l2, process_lo_angular_intensity, process_lo_species_intensity, @@ -172,6 +175,30 @@ def test_get_efficiency_lut(processing_dependencies, mock_get_file_paths): assert col in efficiency_lut.columns, f"Missing column {col} in efficiency LUT" +def test_get_tof_ns_from_mpq_lut(processing_dependencies, mock_get_file_paths): + tof_ns = get_mpq_calc_tof_conversion_vals(processing_dependencies) + assert tof_ns.shape == (1024,) + mpq_calc_lut_file = processing_dependencies.get_file_paths( + descriptor="l2-lo-onboard-mpq-cal" + )[0] + mpq_df = pd.read_csv(mpq_calc_lut_file, header=None) + expected_tof_ns = mpq_df.loc[6:, 1].to_numpy().astype(np.float64) + # Calculated values should be more precise than LUT but should be close + np.testing.assert_allclose(tof_ns, expected_tof_ns, atol=1e-5) + + +def test_get_energy_kev_from_mpq_lut(processing_dependencies, mock_get_file_paths): + energy_kev = get_mpq_calc_energy_conversion_vals(processing_dependencies) + assert energy_kev.shape == (128,) + mpq_calc_lut_file = processing_dependencies.get_file_paths( + descriptor="l2-lo-onboard-mpq-cal" + )[0] + mpq_df = pd.read_csv(mpq_calc_lut_file, header=None) + expected_tof_ns = mpq_df.loc[5, 4:].to_numpy().astype(np.float64) + # Calculated values should be more precise than LUT but should be close + np.testing.assert_allclose(energy_kev, expected_tof_ns, rtol=0.01) + + def test_process_lo_species_intensity(mock_get_file_paths, codice_lut_path): mock_get_file_paths.side_effect = [ codice_lut_path(descriptor="lo-sw-species", data_type="l0"), @@ -476,3 +503,63 @@ def test_codice_l2_sw_angular_intensity(mock_get_file_paths, codice_lut_path): processed_2_ds.attrs["Data_version"] = "001" assert processed_2_ds.attrs["Logical_source"] == "imap_codice_l2_lo-sw-angular" write_cdf(processed_2_ds) + + +@patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") +def test_codice_l2_lo_de(mock_get_file_paths, codice_lut_path): + mock_get_file_paths.side_effect = [ + codice_lut_path(descriptor="lo-direct-events", data_type="l0") + ] + l1a_cdf = process_l1a(ProcessingInputCollection())[0] + + processed_l1a_file = write_cdf(l1a_cdf) + file_path = processed_l1a_file.as_posix() + # Mock get_files for l2 + mock_get_file_paths.side_effect = [ + [file_path], + [file_path], + codice_lut_path(descriptor="l2-lo-onboard-energy-table"), + codice_lut_path(descriptor="l2-lo-onboard-energy-bins"), + codice_lut_path(descriptor="l2-lo-onboard-mpq-cal"), + codice_lut_path(descriptor="l2-lo-onboard-mpq-cal"), + ] + + processed_l2_ds = process_codice_l2("lo-direct-events", ProcessingInputCollection()) + l2_val_data = ( + imap_module_directory + / "tests" + / "codice" + / "data" + / "l2_validation" + / ( + f"imap_codice_l2_lo-direct-events_{VALIDATION_FILE_DATE}" + f"_{VALIDATION_FILE_VERSION}.cdf" + ) + ) + + l2_val_data = load_cdf(l2_val_data) + + for variable in l2_val_data.data_vars: + if variable in ["spin_angle"]: + # TODO remove this block when joey fixes spin_angle calculation + continue # skip spin_angle + if "label" in variable: + np.testing.assert_array_equal( + processed_l2_ds[variable].values, + l2_val_data[variable].values, + err_msg=f"Mismatch in variable '{variable}'", + ) + else: + np.testing.assert_allclose( + processed_l2_ds[variable].values, + l2_val_data[variable].values, + rtol=5e-5, + err_msg=f"Mismatch in variable '{variable}'", + equal_nan=True, + ) + processed_l2_ds.attrs["Data_version"] = "001" + assert processed_l2_ds.attrs["Logical_source"] == "imap_codice_l2_lo-direct-events" + file = write_cdf(processed_l2_ds) + errors = CDFValidator().validate(file) + assert not errors + load_cdf(file) diff --git a/imap_processing/tests/external_test_data_config.py b/imap_processing/tests/external_test_data_config.py index 489720858..8d5aa69e9 100644 --- a/imap_processing/tests/external_test_data_config.py +++ b/imap_processing/tests/external_test_data_config.py @@ -83,6 +83,9 @@ ("imap_codice_l2-hi-ialirt-efficiency_20251008_v001.csv", "codice/data/l2_lut/"), ("imap_codice_l2-lo-gfactor_20251008_v001.csv", "codice/data/l2_lut/"), ("imap_codice_l2-lo-efficiency_20251008_v001.csv", "codice/data/l2_lut/"), + ("imap_codice_l2-lo-onboard-energy-bins_20250101_v001.csv", "codice/data/l2_lut/"), + ("imap_codice_l2-lo-onboard-energy-table_20250101_v001.csv", "codice/data/l2_lut/"), + ("imap_codice_l2-lo-onboard-mpq-cal_20250101_v001.csv", "codice/data/l2_lut/"), # L2 Validation data (f"imap_codice_l2_hi-omni_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l2_validation/"), @@ -91,6 +94,8 @@ (f"imap_codice_l2_lo-sw-angular_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l2_validation/"), (f"imap_codice_l2_lo-nsw-species_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l2_validation/"), (f"imap_codice_l2_lo-sw-species_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l2_validation/"), + (f"imap_codice_l2_lo-direct-events_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l2_validation/"), + (f"imap_codice_l2_hi-direct-events_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l2_validation/"), (f"imap_codice_l2_hi-ialirt_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l2_validation/"), (f"imap_codice_l2_lo-ialirt_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l2_validation/"),