From 1eb52011bc8bfd3550d471d3152e1c5ee644ea69 Mon Sep 17 00:00:00 2001 From: Tim Plummer Date: Wed, 4 Nov 2026 14:32:55 -0700 Subject: [PATCH 1/2] Fixed goodtimes calculation of qualified events mask to take gain configuration into account --- imap_processing/hi/hi_goodtimes.py | 143 ++++++++++--- imap_processing/tests/hi/test_hi_goodtimes.py | 194 +++++++++++++++++- 2 files changed, 302 insertions(+), 35 deletions(-) diff --git a/imap_processing/hi/hi_goodtimes.py b/imap_processing/hi/hi_goodtimes.py index fabcdfa813..c7983aa0cd 100644 --- a/imap_processing/hi/hi_goodtimes.py +++ b/imap_processing/hi/hi_goodtimes.py @@ -89,8 +89,11 @@ def hi_goodtimes( l1b_de_datasets : list[xr.Dataset] L1B DE datasets for surrounding pointings. Typically, includes current plus 4 preceding and 4 following pointings (9 total). - Statistical filters 0 and 1 use all datasets; other filters use - only the current pointing. + Statistical filter 0 uses all datasets. Statistical filter 1 uses + the current pointing plus the surrounding pointings whose gain state + matches a gain_config_id in the cal-prod config (each pointing's + events are qualified against its own matched gain config). Other + filters use only the current pointing. l1b_hk : xr.Dataset L1B housekeeping dataset containing DRF status. l1a_diagfee : xr.Dataset @@ -254,28 +257,10 @@ def _apply_goodtimes_filters( stats = goodtimes_ds.goodtimes.get_cull_statistics() logger.info(f"Initial good bins: {stats['good_bins']}/{stats['total_bins']}") - # Pre-compute qualified event masks for each dataset - # These masks check BOTH coincidence_type AND TOF windows - for l1b_de in l1b_de_datasets: - ccsds_index = l1b_de["ccsds_index"].values - - # Handle invalid events (FILLVAL trigger_id) to avoid IndexError - # For pointings with no valid events, trigger_id will be at FILLVAL - trigger_id_fillval = l1b_de["trigger_id"].attrs.get("FILLVAL", 65535) - valid_events = l1b_de["trigger_id"].values != trigger_id_fillval - - # Initialize with -1 (won't match any config row since ESA energy steps > 0) - esa_energy_steps: np.ndarray = np.full(len(ccsds_index), -1, dtype=np.int32) - if np.any(valid_events): - esa_energy_steps[valid_events] = l1b_de["esa_energy_step"].values[ - ccsds_index[valid_events] - ] - - l1b_de["qualified_mask"] = xr.DataArray( - compute_qualified_event_mask(l1b_de, cal_product_config, esa_energy_steps), - dims=["event_met"], - ) - logger.info("Pre-computed qualified event masks for all datasets") + # Pre-compute qualified event masks for each dataset, using each + # pointing's own matched gain config. These masks check BOTH + # coincidence_type AND TOF windows. + matched_indices = _add_qualified_event_masks(l1b_de_datasets, cal_product_config) # === Apply culling filters === @@ -309,11 +294,18 @@ def _apply_goodtimes_filters( mark_statistical_filter_0(goodtimes_ds, l1b_de_datasets, current_index) # 6. Statistical Filter 1 - isotropic count rate increases + # Surrounding pointings that match no gain config have no qualified + # events and are excluded so they don't bias the per-ESA medians low. This + # is not expected to occur and would indicate a problem with commanding or + # an instrumental anomaly. + # The current pointing is always kept (if it is unmatched it has already + # been fully culled by mark_unmatched_gain_config). logger.info("Applying filter: mark_statistical_filter_1") + filter_1_indices = sorted(set(matched_indices) | {current_index}) mark_statistical_filter_1( goodtimes_ds, - l1b_de_datasets, - current_index, + [l1b_de_datasets[i] for i in filter_1_indices], + filter_1_indices.index(current_index), ) # 7. Statistical Filter 2 - short-lived event pulses @@ -324,6 +316,97 @@ def _apply_goodtimes_filters( ) +def _get_gain_match_values(l1b_de: xr.Dataset) -> dict[str, float]: + """ + Get a pointing's gain match values from its L1B DE global attributes. + + Parameters + ---------- + l1b_de : xarray.Dataset + L1B Direct Event data with the gain match global attributes set by + hi_l1b.de_gain_test_filter(). + + Returns + ------- + dict[str, float] + Mapping of CalibrationProductConfig.GAIN_MATCH_FIELDS field names to + the pointing's values. A missing attribute is returned as NaN. + """ + return { + field: float(l1b_de.attrs.get(field, np.nan)) + for field in CalibrationProductConfig.GAIN_MATCH_FIELDS + } + + +def _add_qualified_event_masks( + l1b_de_datasets: list[xr.Dataset], + cal_product_config: pd.DataFrame, +) -> list[int]: + """ + Add a "qualified_mask" variable to each L1B DE dataset. + + Each pointing's events are qualified against only the calibration product + rows of the gain config that pointing ran (selected with + CalibrationProductConfig.select_gain_config() from the pointing's gain + match attributes), so coincidence/TOF windows of other gain configs never + qualify its events. A pointing that matches no single gain config gets an + all-False mask. + + Parameters + ---------- + l1b_de_datasets : list[xarray.Dataset] + L1B DE datasets (current and surrounding pointings). Modified in place. + cal_product_config : pandas.DataFrame + Full (multi-gain-config) calibration product configuration. Use + CalibrationProductConfig.from_csv() to load. + + Returns + ------- + list[int] + Indices into l1b_de_datasets of the pointings that matched a gain + config. + """ + matched_indices = [] + for i, l1b_de in enumerate(l1b_de_datasets): + ccsds_index = l1b_de["ccsds_index"].values + n_events = len(ccsds_index) + + gain_config_df = cal_product_config.cal_prod_config.select_gain_config( + _get_gain_match_values(l1b_de) + ) + if gain_config_df is None: + logger.warning( + f"Pointing {l1b_de.attrs.get('Repointing')} matches no single " + f"gain_config_id; it has no qualified events." + ) + qualified_mask: np.ndarray = np.zeros(n_events, dtype=bool) + else: + matched_indices.append(i) + # Handle invalid events (FILLVAL trigger_id) to avoid IndexError + # For pointings with no valid events, trigger_id will be at FILLVAL + trigger_id_fillval = l1b_de["trigger_id"].attrs.get("FILLVAL", 65535) + valid_events = l1b_de["trigger_id"].values != trigger_id_fillval + + # Initialize with -1 (won't match any config row since ESA energy + # steps > 0) + esa_energy_steps: np.ndarray = np.full(n_events, -1, dtype=np.int32) + if np.any(valid_events): + esa_energy_steps[valid_events] = l1b_de["esa_energy_step"].values[ + ccsds_index[valid_events] + ] + qualified_mask = compute_qualified_event_mask( + l1b_de, gain_config_df, esa_energy_steps + ) + + l1b_de["qualified_mask"] = xr.DataArray(qualified_mask, dims=["event_met"]) + + logger.info( + f"Pre-computed qualified event masks for all datasets; " + f"{len(matched_indices)}/{len(l1b_de_datasets)} matched a gain config" + ) + return matched_indices + + def create_goodtimes_dataset(l1b_de: xr.Dataset) -> xr.Dataset: """ Create goodtimes dataset from L1B Direct Event data. @@ -1077,10 +1160,7 @@ def mark_unmatched_gain_config( """ logger.info("Running mark_unmatched_gain_config culling") - hv_deltas = { - field: float(l1b_de.attrs.get(field, np.nan)) - for field in CalibrationProductConfig.GAIN_MATCH_FIELDS - } + hv_deltas = _get_gain_match_values(l1b_de) gain_config_id = cal_product_config.cal_prod_config.match_gain_config_id(hv_deltas) if gain_config_id is not None: logger.info(f"Pointing gain state matches gain_config_id={gain_config_id}") @@ -2273,7 +2353,8 @@ def mark_statistical_filter_1( ------ ValueError If current_index is out of range or if fewer than min_pointings - datasets are provided. + datasets are provided (e.g. because too many surrounding Pointings + matched no gain config and were excluded by the caller). Notes ----- diff --git a/imap_processing/tests/hi/test_hi_goodtimes.py b/imap_processing/tests/hi/test_hi_goodtimes.py index 4b65706775..dda29a62b8 100644 --- a/imap_processing/tests/hi/test_hi_goodtimes.py +++ b/imap_processing/tests/hi/test_hi_goodtimes.py @@ -10,6 +10,7 @@ from imap_processing.hi.hi_goodtimes import ( INTERVAL_DTYPE, CullCode, + _add_qualified_event_masks, _add_sweep_indices, _apply_goodtimes_filters, _build_per_sweep_datasets, @@ -4417,6 +4418,10 @@ def test_loads_cal_config(self, tmp_path): patch( "imap_processing.hi.utils.CalibrationProductConfig.from_csv" ) as mock_cal_load, + patch( + "imap_processing.hi.hi_goodtimes._add_qualified_event_masks", + return_value=list(range(9)), + ), patch("imap_processing.hi.hi_goodtimes.mark_bad_voltage"), patch("imap_processing.hi.hi_goodtimes.mark_unmatched_gain_config"), patch("imap_processing.hi.hi_goodtimes.mark_incomplete_spin_sets"), @@ -4431,7 +4436,7 @@ def test_loads_cal_config(self, tmp_path): _apply_goodtimes_filters( mock_goodtimes, - [mock_l1b_de], + [mock_l1b_de] * 9, current_index=0, l1b_hk=mock_hk, l1a_diagfee=MagicMock(), @@ -4456,6 +4461,10 @@ def test_calls_all_filters(self, tmp_path): "imap_processing.hi.utils.CalibrationProductConfig.from_csv", return_value=mock_cal, ), + patch( + "imap_processing.hi.hi_goodtimes._add_qualified_event_masks", + return_value=list(range(9)), + ), patch("imap_processing.hi.hi_goodtimes.mark_bad_voltage") as mock_f0, patch( "imap_processing.hi.hi_goodtimes.mark_unmatched_gain_config" @@ -4478,7 +4487,7 @@ def test_calls_all_filters(self, tmp_path): ): _apply_goodtimes_filters( mock_goodtimes, - [mock_l1b_de], + [mock_l1b_de] * 9, current_index=0, l1b_hk=mock_hk, l1a_diagfee=MagicMock(), @@ -4511,6 +4520,10 @@ def test_raises_statistical_filter_0_errors(self, tmp_path): "imap_processing.hi.utils.CalibrationProductConfig.from_csv", return_value=mock_cal, ), + patch( + "imap_processing.hi.hi_goodtimes._add_qualified_event_masks", + return_value=list(range(9)), + ), patch("imap_processing.hi.hi_goodtimes.mark_bad_voltage"), patch("imap_processing.hi.hi_goodtimes.mark_unmatched_gain_config"), patch("imap_processing.hi.hi_goodtimes.mark_incomplete_spin_sets"), @@ -4525,7 +4538,7 @@ def test_raises_statistical_filter_0_errors(self, tmp_path): with pytest.raises(ValueError, match="filter 0 error"): _apply_goodtimes_filters( mock_goodtimes, - [mock_l1b_de], + [mock_l1b_de] * 9, current_index=0, l1b_hk=mock_hk, l1a_diagfee=MagicMock(), @@ -4548,6 +4561,10 @@ def test_raises_statistical_filter_1_errors(self, tmp_path): "imap_processing.hi.utils.CalibrationProductConfig.from_csv", return_value=mock_cal, ), + patch( + "imap_processing.hi.hi_goodtimes._add_qualified_event_masks", + return_value=list(range(9)), + ), patch("imap_processing.hi.hi_goodtimes.mark_bad_voltage"), patch("imap_processing.hi.hi_goodtimes.mark_unmatched_gain_config"), patch("imap_processing.hi.hi_goodtimes.mark_incomplete_spin_sets"), @@ -4563,7 +4580,7 @@ def test_raises_statistical_filter_1_errors(self, tmp_path): with pytest.raises(ValueError, match="filter 1 error"): _apply_goodtimes_filters( mock_goodtimes, - [mock_l1b_de], + [mock_l1b_de] * 9, current_index=0, l1b_hk=mock_hk, l1a_diagfee=MagicMock(), @@ -4964,3 +4981,172 @@ def test_custom_cull_code(self, goodtimes, cal_config): cull_code=CullCode.DRF, ) assert np.all(goodtimes["cull_flags"].values == CullCode.DRF) + + +# HV deltas within tolerance of the second gain config added by +# two_gain_cal_config below. +GAIN_1_HV_DELTAS = { + "mcp_delta_v": 500.0, + "cem_a_delta_v": 1000.0, + "cem_b_delta_v": 1000.0, + "tof_v": -5000.0, +} + + +@pytest.fixture +def two_gain_cal_config(hi_test_cal_prod_config_path): + """ + Cal-prod config with two gain configs whose AB TOF windows don't overlap. + + gain_config_id=0 is the test file as-is (tof_ab window 15-55 ns). + gain_config_id=1 has tof_ab window 60-100 ns and matches GAIN_1_HV_DELTAS. + """ + config_0 = CalibrationProductConfig.from_csv(hi_test_cal_prod_config_path) + config_1 = config_0.rename(index={0: 1}, level="gain_config_id") + config_1["tof_ab_low"] = 60 + config_1["tof_ab_high"] = 100 + for field, value in GAIN_1_HV_DELTAS.items(): + config_1[field] = value + config_1[f"{field}_tol"] = 50.0 + return pd.concat([config_0, config_1]) + + +def _qualified_mask_l1b_de(hv_deltas, repointing="repoint00001"): + """ + Build a minimal L1B DE dataset for qualified-mask tests. + + Four ABC1C2 events at ESA energy step 1. Events 0 and 1 have tof_ab inside + only gain config 0's window, events 2 and 3 inside only gain config 1's. + """ + n_events = 4 + ds = xr.Dataset( + { + "ccsds_index": (["event_met"], np.zeros(n_events, dtype=np.int64)), + "trigger_id": (["event_met"], np.ones(n_events, dtype=np.uint16)), + "coincidence_type": (["event_met"], np.full(n_events, 15)), + "tof_ab": (["event_met"], np.array([30, 40, 70, 80])), + "tof_ac1": (["event_met"], np.full(n_events, 30)), + "tof_bc1": (["event_met"], np.full(n_events, 0)), + "tof_c1c2": (["event_met"], np.full(n_events, 10)), + "esa_energy_step": (["epoch"], np.array([1])), + }, + coords={"event_met": np.arange(n_events, dtype=float)}, + attrs={"Repointing": repointing, **hv_deltas}, + ) + ds["trigger_id"].attrs["FILLVAL"] = 65535 + for tof_var in ["tof_ab", "tof_ac1", "tof_bc1", "tof_c1c2"]: + ds[tof_var].attrs["FILLVAL"] = -9999 + return ds + + +class TestAddQualifiedEventMasks: + """Tests for _add_qualified_event_masks.""" + + def test_uses_each_pointings_gain_config(self, two_gain_cal_config): + """Events qualify only by the TOF windows of their pointing's gain config.""" + datasets = [ + _qualified_mask_l1b_de(NOMINAL_HV_DELTAS, "repoint00001"), + _qualified_mask_l1b_de(GAIN_1_HV_DELTAS, "repoint00002"), + ] + matched = _add_qualified_event_masks(datasets, two_gain_cal_config) + + assert matched == [0, 1] + np.testing.assert_array_equal( + datasets[0]["qualified_mask"].values, [True, True, False, False] + ) + np.testing.assert_array_equal( + datasets[1]["qualified_mask"].values, [False, False, True, True] + ) + + def test_unmatched_pointing_has_no_qualified_events(self, two_gain_cal_config): + """A pointing matching no gain config gets an all-False mask.""" + unmatched = dict(NOMINAL_HV_DELTAS, tof_v=0.0) + datasets = [ + _qualified_mask_l1b_de(NOMINAL_HV_DELTAS, "repoint00001"), + _qualified_mask_l1b_de(unmatched, "repoint00002"), + _qualified_mask_l1b_de({}, "repoint00003"), # missing attributes + ] + matched = _add_qualified_event_masks(datasets, two_gain_cal_config) + + assert matched == [0] + assert datasets[0]["qualified_mask"].values.any() + for ds in datasets[1:]: + assert ds["qualified_mask"].dims == ("event_met",) + assert not ds["qualified_mask"].values.any() + + def test_invalid_trigger_id_events_not_qualified(self, two_gain_cal_config): + """Events with FILLVAL trigger_id never qualify.""" + ds = _qualified_mask_l1b_de(NOMINAL_HV_DELTAS) + ds["trigger_id"].values[0] = 65535 + _add_qualified_event_masks([ds], two_gain_cal_config) + np.testing.assert_array_equal( + ds["qualified_mask"].values, [False, True, False, False] + ) + + +class TestApplyGoodtimesFiltersGainMatching: + """Tests for how _apply_goodtimes_filters handles unmatched pointings.""" + + @staticmethod + def _run(matched_indices, current_index, n_datasets=9): + mock_goodtimes = MagicMock() + mock_goodtimes.goodtimes.get_cull_statistics.return_value = { + "good_bins": 100, + "total_bins": 100, + } + datasets = [MagicMock(name=f"ds{i}") for i in range(n_datasets)] + with ( + patch("imap_processing.hi.utils.CalibrationProductConfig.from_csv"), + patch( + "imap_processing.hi.hi_goodtimes._add_qualified_event_masks", + return_value=matched_indices, + ), + patch("imap_processing.hi.hi_goodtimes.mark_bad_voltage"), + patch("imap_processing.hi.hi_goodtimes.mark_unmatched_gain_config"), + patch("imap_processing.hi.hi_goodtimes.mark_incomplete_spin_sets"), + patch("imap_processing.hi.hi_goodtimes.mark_drf_times"), + patch("imap_processing.hi.hi_goodtimes.mark_bad_tdc_cal"), + patch("imap_processing.hi.hi_goodtimes.mark_overflow_packets"), + patch( + "imap_processing.hi.hi_goodtimes.mark_statistical_filter_0" + ) as mock_f0, + patch( + "imap_processing.hi.hi_goodtimes.mark_statistical_filter_1" + ) as mock_f1, + patch("imap_processing.hi.hi_goodtimes.mark_statistical_filter_2"), + ): + _apply_goodtimes_filters( + mock_goodtimes, + datasets, + current_index=current_index, + l1b_hk=MagicMock(), + l1a_diagfee=MagicMock(), + cal_product_config_path=MagicMock(), + ) + return datasets, mock_f0, mock_f1 + + def test_filter_1_excludes_unmatched_surrounding_pointings(self): + """Unmatched surrounding pointings are dropped from filter 1's set.""" + datasets, mock_f0, mock_f1 = self._run([0, 2, 4, 5, 8], current_index=4) + + # Filter 0 does not use qualified masks and still sees all pointings + assert mock_f0.call_args.args[1] is datasets + _, f1_datasets, f1_current_index = mock_f1.call_args.args + assert f1_datasets == [datasets[i] for i in [0, 2, 4, 5, 8]] + assert f1_current_index == 2 + + def test_filter_1_keeps_unmatched_current_pointing(self): + """The current pointing stays in filter 1's set even when unmatched.""" + datasets, _, mock_f1 = self._run([0, 1, 2, 6, 7], current_index=4) + + _, f1_datasets, f1_current_index = mock_f1.call_args.args + assert f1_datasets == [datasets[i] for i in [0, 1, 2, 4, 6, 7]] + assert f1_datasets[f1_current_index] is datasets[4] + + def test_filter_1_gets_reduced_set_when_few_matched(self): + """Filter 1 receives the reduced set and enforces the minimum itself.""" + datasets, _, mock_f1 = self._run([4, 5], current_index=4) + + _, f1_datasets, f1_current_index = mock_f1.call_args.args + assert f1_datasets == [datasets[4], datasets[5]] + assert f1_current_index == 0 From ca0709b48e7e02f6e3b27573be1a1bae39ff1b2e Mon Sep 17 00:00:00 2001 From: Tim Plummer Date: Wed, 4 Nov 2026 15:04:55 -0700 Subject: [PATCH 2/2] Use pointing's matched gain config for overflow packet qualification --- imap_processing/hi/hi_goodtimes.py | 17 ++++- imap_processing/tests/hi/test_hi_goodtimes.py | 74 ++++++++++++++++--- 2 files changed, 77 insertions(+), 14 deletions(-) diff --git a/imap_processing/hi/hi_goodtimes.py b/imap_processing/hi/hi_goodtimes.py index c7983aa0cd..68c845e020 100644 --- a/imap_processing/hi/hi_goodtimes.py +++ b/imap_processing/hi/hi_goodtimes.py @@ -1398,10 +1398,12 @@ def mark_overflow_packets( - ccsds_index: Index mapping each event to its packet - coincidence_type: Coincidence type bitmap for each event - event_met: MET timestamp for each event + - gain match global attributes set by hi_l1b.de_gain_test_filter() config_df : pandas.DataFrame Calibration product configuration DataFrame with coincidence_type_values column containing tuples of valid coincidence type integers for each calibration product. Use CalibrationProductConfig.from_csv() to load. + Only the rows of the pointing's matched gain config are used. cull_code : int, optional Cull code to use for marking bad times (default: CullCode.LOOSE). @@ -1411,11 +1413,20 @@ def mark_overflow_packets( for MET timestamps with overflow packets containing qualified final events. The check for qualified events uses the coincidence_type_values from the - calibration product configuration, which defines which coincidence types - are considered valid for science analysis. + pointing's matched gain config in the calibration product configuration, + which defines which coincidence types are considered valid for science + analysis. If the pointing matches no gain config, nothing is marked here; + mark_unmatched_gain_config() culls the whole pointing in that case. """ logger.info("Running mark_overflow_packets culling") + gain_config_df = config_df.cal_prod_config.select_gain_config( + _get_gain_match_values(l1b_de) + ) + if gain_config_df is None: + logger.info("Pointing matches no gain config; skipping overflow check") + return + ccsds_indices = l1b_de["ccsds_index"].values coincidence_types = l1b_de["coincidence_type"].values event_mets = l1b_de["event_met"].values @@ -1443,7 +1454,7 @@ def mark_overflow_packets( # Build set of all valid coincidence types from calibration products all_valid_coin_types = set() - for coin_types in config_df["coincidence_type_values"]: + for coin_types in gain_config_df["coincidence_type_values"]: all_valid_coin_types.update(coin_types) # Find the last event index for each packet (vectorized) diff --git a/imap_processing/tests/hi/test_hi_goodtimes.py b/imap_processing/tests/hi/test_hi_goodtimes.py index dda29a62b8..d14a22fec3 100644 --- a/imap_processing/tests/hi/test_hi_goodtimes.py +++ b/imap_processing/tests/hi/test_hi_goodtimes.py @@ -46,6 +46,14 @@ "cem_b_delta_v": 2150.0, "tof_v": -8000.0, } +# HV deltas within tolerance of the second gain config used in the +# two-gain-config tests (two_gain_cal_config, overflow packet tests). +GAIN_1_HV_DELTAS = { + "mcp_delta_v": 500.0, + "cem_a_delta_v": 1000.0, + "cem_b_delta_v": 1000.0, + "tof_v": -5000.0, +} @pytest.fixture @@ -1838,10 +1846,14 @@ def mock_config_df(self): "tof_c1c2_low": [0, 0], "tof_c1c2_high": [100, 100], } + for field, value in NOMINAL_HV_DELTAS.items(): + data[field] = [value, value] + data[f"{field}_tol"] = [1.0, 1.0] df = pd.DataFrame( data, index=pd.MultiIndex.from_tuples( - [(1, 1), (2, 1)], names=["calibration_prod", "esa_energy_step"] + [(0, 1, 1), (0, 2, 1)], + names=["gain_config_id", "calibration_prod", "esa_energy_step"], ), ) # Add coincidence_type_values column (converted from strings to ints) @@ -1880,6 +1892,7 @@ def test_no_full_packets(self, mock_goodtimes, mock_config_df): ), }, coords={"event_met": np.linspace(1000.0, 1010.0, n_events)}, + attrs=NOMINAL_HV_DELTAS, ) mark_overflow_packets(mock_goodtimes, l1b_de, mock_config_df) @@ -1902,6 +1915,7 @@ def test_full_packet_with_qualified_event(self, mock_goodtimes, mock_config_df): ), }, coords={"event_met": event_mets}, + attrs=NOMINAL_HV_DELTAS, ) mark_overflow_packets(mock_goodtimes, l1b_de, mock_config_df) @@ -1925,6 +1939,7 @@ def test_full_packet_with_unqualified_event(self, mock_goodtimes, mock_config_df ), }, coords={"event_met": event_mets}, + attrs=NOMINAL_HV_DELTAS, ) mark_overflow_packets(mock_goodtimes, l1b_de, mock_config_df) @@ -1964,6 +1979,7 @@ def test_multiple_full_packets(self, mock_goodtimes, mock_config_df): "coincidence_type": (["event_met"], coincidence_types), }, coords={"event_met": event_mets}, + attrs=NOMINAL_HV_DELTAS, ) mark_overflow_packets(mock_goodtimes, l1b_de, mock_config_df) @@ -1981,6 +1997,7 @@ def test_empty_de_data(self, mock_goodtimes, mock_config_df): "coincidence_type": (["event_met"], np.array([], dtype=np.uint8)), }, coords={"event_met": np.array([])}, + attrs=NOMINAL_HV_DELTAS, ) # Should not raise, just return without culling @@ -2000,6 +2017,7 @@ def test_custom_cull_code(self, mock_goodtimes, mock_config_df): ), }, coords={"event_met": event_mets}, + attrs=NOMINAL_HV_DELTAS, ) custom_cull = 5 @@ -2025,6 +2043,7 @@ def test_final_event_is_last_in_list(self, mock_goodtimes, mock_config_df): "coincidence_type": (["event_met"], coincidence_types), }, coords={"event_met": event_mets}, + attrs=NOMINAL_HV_DELTAS, ) mark_overflow_packets(mock_goodtimes, l1b_de, mock_config_df) @@ -2032,6 +2051,49 @@ def test_final_event_is_last_in_list(self, mock_goodtimes, mock_config_df): # Should be culled because the final event (last in list) is qualified assert np.sum(mock_goodtimes["cull_flags"].values > 0) > 0 + @staticmethod + def _full_packet_l1b_de(final_coincidence_type, hv_deltas): + n_events = 664 + coincidence_types = np.full(n_events, 3, dtype=np.uint8) + coincidence_types[-1] = final_coincidence_type + return xr.Dataset( + { + "ccsds_index": (["event_met"], np.zeros(n_events, dtype=np.uint16)), + "coincidence_type": (["event_met"], coincidence_types), + }, + coords={"event_met": np.linspace(1005.0, 1006.0, n_events)}, + attrs=hv_deltas, + ) + + def test_uses_pointings_gain_config(self, mock_goodtimes, mock_config_df): + """Only the matched gain config's coincidence types qualify.""" + # gain_config_id=1 qualifies only coincidence type 8 (A) + other = mock_config_df.rename(index={0: 1}, level="gain_config_id") + other["coincidence_type_list"] = [("A",), ("A",)] + for field, value in GAIN_1_HV_DELTAS.items(): + other[field] = value + config = pd.concat([mock_config_df, other]) + + # Type 8 is only in gain config 1, so a gain-0 pointing is not culled + mark_overflow_packets( + mock_goodtimes, self._full_packet_l1b_de(8, NOMINAL_HV_DELTAS), config + ) + assert np.all(mock_goodtimes["cull_flags"].values == 0) + + # ...but a gain-1 pointing is + mark_overflow_packets( + mock_goodtimes, self._full_packet_l1b_de(8, GAIN_1_HV_DELTAS), config + ) + assert np.all(mock_goodtimes["cull_flags"].values[0, :] == CullCode.OVERFLOW) + + def test_unmatched_gain_config_skipped(self, mock_goodtimes, mock_config_df): + """A pointing matching no gain config is not marked by this filter.""" + unmatched = dict(NOMINAL_HV_DELTAS, tof_v=0.0) + mark_overflow_packets( + mock_goodtimes, self._full_packet_l1b_de(15, unmatched), mock_config_df + ) + assert np.all(mock_goodtimes["cull_flags"].values == 0) + class TestGetSweepIndices: """Test suite for _get_sweep_indices() helper function.""" @@ -4983,16 +5045,6 @@ def test_custom_cull_code(self, goodtimes, cal_config): assert np.all(goodtimes["cull_flags"].values == CullCode.DRF) -# HV deltas within tolerance of the second gain config added by -# two_gain_cal_config below. -GAIN_1_HV_DELTAS = { - "mcp_delta_v": 500.0, - "cem_a_delta_v": 1000.0, - "cem_b_delta_v": 1000.0, - "tof_v": -5000.0, -} - - @pytest.fixture def two_gain_cal_config(hi_test_cal_prod_config_path): """