From 4a0cf1aefd9e9d6007e8e7386e516a718e6ffb6e Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Thu, 29 Oct 2026 14:42:40 -0600 Subject: [PATCH 1/7] remove ne from desired products list --- imap_processing/codice/codice_l1a_lo_species.py | 1 + imap_processing/codice/constants.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/codice/codice_l1a_lo_species.py b/imap_processing/codice/codice_l1a_lo_species.py index 187d0aa2c2..7cee76633e 100644 --- a/imap_processing/codice/codice_l1a_lo_species.py +++ b/imap_processing/codice/codice_l1a_lo_species.py @@ -83,6 +83,7 @@ def l1a_lo_species( ] + LO_SW_SPECIES_VARIABLE_NAMES ) + print(desired_species_names) logical_source_id = "imap_codice_l1a_lo-sw-species" elif view_tab_obj.apid == CODICEAPID.COD_LO_IAL: actual_species_names = sci_lut_data["data_product_lo_tab"]["0"]["ialirt"]["sw"][ diff --git a/imap_processing/codice/constants.py b/imap_processing/codice/constants.py index e0604bc561..c5c3822df3 100644 --- a/imap_processing/codice/constants.py +++ b/imap_processing/codice/constants.py @@ -824,7 +824,6 @@ class CODICEAPID(IntEnum): "oplus6", "oplus7", "oplus8", - "ne", "mg", "si", "fe_loq", From 7b9f22b12e198bbb689bcb7539abe0b97edd5223 Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Thu, 29 Oct 2026 14:45:23 -0600 Subject: [PATCH 2/7] remove species name --- imap_processing/codice/codice_l1a_lo_species.py | 1 - 1 file changed, 1 deletion(-) diff --git a/imap_processing/codice/codice_l1a_lo_species.py b/imap_processing/codice/codice_l1a_lo_species.py index 7cee76633e..187d0aa2c2 100644 --- a/imap_processing/codice/codice_l1a_lo_species.py +++ b/imap_processing/codice/codice_l1a_lo_species.py @@ -83,7 +83,6 @@ def l1a_lo_species( ] + LO_SW_SPECIES_VARIABLE_NAMES ) - print(desired_species_names) logical_source_id = "imap_codice_l1a_lo-sw-species" elif view_tab_obj.apid == CODICEAPID.COD_LO_IAL: actual_species_names = sci_lut_data["data_product_lo_tab"]["0"]["ialirt"]["sw"][ From fc163af87c96cefe8dd0349570b67db2e99fad71 Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Fri, 30 Oct 2026 14:03:31 -0600 Subject: [PATCH 3/7] use one desired species list --- imap_processing/codice/codice_l1a_lo_species.py | 14 ++------------ imap_processing/codice/codice_l2.py | 1 - imap_processing/codice/constants.py | 2 -- imap_processing/tests/codice/test_codice_l1a.py | 3 +++ imap_processing/tests/codice/test_codice_l1b.py | 3 +++ imap_processing/tests/codice/test_codice_l2.py | 9 +++------ 6 files changed, 11 insertions(+), 21 deletions(-) diff --git a/imap_processing/codice/codice_l1a_lo_species.py b/imap_processing/codice/codice_l1a_lo_species.py index 187d0aa2c2..e325680eb0 100644 --- a/imap_processing/codice/codice_l1a_lo_species.py +++ b/imap_processing/codice/codice_l1a_lo_species.py @@ -77,23 +77,13 @@ def l1a_lo_species( actual_species_names = sci_lut_data["data_product_lo_tab"]["0"]["species"][ "sw" ]["species_names"] - desired_species_names = set( - sci_lut_data["data_product_lo_tab"]["0"]["species"]["sw"][ - "desired_species_names" - ] - + LO_SW_SPECIES_VARIABLE_NAMES - ) + desired_species_names = LO_SW_SPECIES_VARIABLE_NAMES logical_source_id = "imap_codice_l1a_lo-sw-species" elif view_tab_obj.apid == CODICEAPID.COD_LO_IAL: actual_species_names = sci_lut_data["data_product_lo_tab"]["0"]["ialirt"]["sw"][ "species_names" ] - desired_species_names = set( - sci_lut_data["data_product_lo_tab"]["0"]["ialirt"]["sw"][ - "desired_species_names" - ] - + LO_IALIRT_VARIABLE_NAMES - ) + desired_species_names = LO_IALIRT_VARIABLE_NAMES # Note: ialirt does not produce a cdf for l1a so this is arbitrary. logical_source_id = "imap_codice_l1a_lo-sw-species" else: diff --git a/imap_processing/codice/codice_l2.py b/imap_processing/codice/codice_l2.py index 4bb79c9f3e..0904fe0014 100644 --- a/imap_processing/codice/codice_l2.py +++ b/imap_processing/codice/codice_l2.py @@ -65,7 +65,6 @@ "oplus6": "O6+", "oplus7": "O7+", "oplus8": "O8+", - "ne": "Ne", "mg": "Mg", "si": "Si", "fe_loq": "Fe LoQ", diff --git a/imap_processing/codice/constants.py b/imap_processing/codice/constants.py index c5c3822df3..e8f78a596b 100644 --- a/imap_processing/codice/constants.py +++ b/imap_processing/codice/constants.py @@ -803,7 +803,6 @@ class CODICEAPID(IntEnum): # ------L1B Constants------ HI_ACQUISITION_TIME = 0.59916 -# TODO: in the future, read from sci-lut LO_SW_ANGULAR_VARIABLE_NAMES = ["hplus", "heplusplus", "oplus6", "fe_loq", "heplus"] LO_NSW_ANGULAR_VARIABLE_NAMES = ["heplusplus", "heplus"] LO_SW_PRIORITY_VARIABLE_NAMES = [ @@ -909,7 +908,6 @@ class CODICEAPID(IntEnum): "oplus6", "oplus7", "oplus8", - "ne", "mg", "si", "fe_hiq", diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index c94b11582b..a85092c78b 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -331,6 +331,9 @@ def test_lo_sw_species(mock_get_file_paths, codice_lut_path): processed_data = filter_day_boundary_data(processed_data, VALIDATION_FILE_DATE) # Compare only the common variables for variable in val_data.data_vars: + if variable in ["ne", "unc_ne"]: + # Ne is no longer being processed. + continue assert_allclose_fillaware( processed_data[variable].values, val_data[variable], diff --git a/imap_processing/tests/codice/test_codice_l1b.py b/imap_processing/tests/codice/test_codice_l1b.py index 72348add69..ef8fecd7bc 100644 --- a/imap_processing/tests/codice/test_codice_l1b.py +++ b/imap_processing/tests/codice/test_codice_l1b.py @@ -52,6 +52,9 @@ def test_l1b_lo_sw_species(mock_get_file_paths, codice_lut_path): l1b_val_data = load_cdf(l1b_val_data) processed_data = process_codice_l1b(processed_l1a_file) for variable in l1b_val_data.data_vars: + if variable in ["ne", "unc_ne"]: + # Ne is no longer being processed. + continue np.testing.assert_allclose( processed_data[variable].values, l1b_val_data[variable].values, diff --git a/imap_processing/tests/codice/test_codice_l2.py b/imap_processing/tests/codice/test_codice_l2.py index 119e962aaa..8ce48b511e 100644 --- a/imap_processing/tests/codice/test_codice_l2.py +++ b/imap_processing/tests/codice/test_codice_l2.py @@ -479,6 +479,9 @@ def test_codice_l2_sw_species_intensity(mock_get_file_paths, codice_lut_path): l2_val_data = load_cdf(l2_val_data) for variable in l2_val_data.data_vars: + # Ne is no longer being processed. + if variable in ["ne", "unc_ne"]: + continue np.testing.assert_allclose( processed_2_ds[variable].values, l2_val_data[variable].values, @@ -517,9 +520,6 @@ def test_codice_l2_sw_species_intensity(mock_get_file_paths, codice_lut_path): assert "DEPEND_1" not in energy_per_charge_attrs -@pytest.mark.xfail( - reason="SPDF ISTP validation API is down. Remove this xfail marker when fixed." -) @patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") def test_codice_l2_lo_de(mock_get_file_paths, codice_lut_path): mock_get_file_paths.side_effect = [ @@ -618,9 +618,6 @@ def test_codice_l2_lo_de(mock_get_file_paths, codice_lut_path): load_cdf(file) -@pytest.mark.xfail( - reason="SPDF ISTP validation API is down. Remove this xfail marker when fixed." -) @patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") def test_codice_l2_hi_de(mock_get_file_paths, codice_lut_path): mock_get_file_paths.side_effect = [ From d7409861d04ff03e711e024a0fa163288db8a89e Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Fri, 30 Oct 2026 14:10:43 -0600 Subject: [PATCH 4/7] add comment about sci lut desired species --- imap_processing/codice/codice_l1a_lo_species.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/imap_processing/codice/codice_l1a_lo_species.py b/imap_processing/codice/codice_l1a_lo_species.py index e325680eb0..94baf7692e 100644 --- a/imap_processing/codice/codice_l1a_lo_species.py +++ b/imap_processing/codice/codice_l1a_lo_species.py @@ -77,6 +77,9 @@ def l1a_lo_species( actual_species_names = sci_lut_data["data_product_lo_tab"]["0"]["species"][ "sw" ]["species_names"] + # The "desired_species" from the sci lut is no longer used. + # All codice lo sw CDFs should contain all the species in + # LO_SW_SPECIES_VARIABLE_NAMES regardless of what sci lut is used. desired_species_names = LO_SW_SPECIES_VARIABLE_NAMES logical_source_id = "imap_codice_l1a_lo-sw-species" elif view_tab_obj.apid == CODICEAPID.COD_LO_IAL: From 00b77d0d7059934a6489dc453eb12517d654828b Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Fri, 30 Oct 2026 14:49:04 -0600 Subject: [PATCH 5/7] add neon assert --- imap_processing/tests/codice/test_codice_l1a.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index a85092c78b..d012eeb682 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -354,6 +354,9 @@ def test_lo_sw_species(mock_get_file_paths, codice_lut_path): rtol=1e-5, err_msg=f"Mismatch in coordinate '{variable}'", ) + # assert that neon is not in the processed data + assert "ne" not in processed_data.data_vars + assert "unc_ne" not in processed_data.data_vars processed_data.attrs["Data_version"] = "002" cdf_file = write_cdf(processed_data, terminate_on_warning=True) assert ( From 5810a6da7b3e15c5c1c752f55565fad7a464b298 Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Fri, 30 Oct 2026 14:53:28 -0600 Subject: [PATCH 6/7] update docs --- .../codice/ancillary.rst | 18 ++++++++++----- .../codice/data-products.rst | 22 ++++++++++++------- .../codice/implementation-status.rst | 15 ++++++++++--- .../codice/l1a.rst | 11 ++++++---- .../codice/l2.rst | 4 ++++ .../codice/overview.rst | 6 ++--- 6 files changed, 53 insertions(+), 23 deletions(-) diff --git a/docs/source/algorithm-code-documentation/codice/ancillary.rst b/docs/source/algorithm-code-documentation/codice/ancillary.rst index d276feff52..214fcde861 100644 --- a/docs/source/algorithm-code-documentation/codice/ancillary.rst +++ b/docs/source/algorithm-code-documentation/codice/ancillary.rst @@ -125,11 +125,19 @@ Things to know: * ``collapse_*[id]["variables"]`` is used only by the aggregated-counters products, where each row is one counter and a row of zeros means "counter turned off in flight". -* ``species_names`` is what the instrument actually sent; - ``desired_species_names`` is what the team wants written. L1A takes the union - of ``desired_species_names`` and the hard-coded ``*_VARIABLE_NAMES``, and - NaN-fills anything that is not in ``species_names``. This is how the P3 Fe - highQ/lowQ label swap is absorbed. +* ``species_names`` is the slot map: position *i* is the species in slot *i* + of the decompressed data. It is how mislabeled telemetry is corrected - the + FSW "off by one" tables prepend a ``junk`` slot, and the P3 Fe highQ/lowQ + swap is a reordering of ``fe_hiq`` / ``fe_loq`` here. +* For ``species.sw`` and ``ialirt.sw``, L1A does **not** read + ``desired_species_names``. The output variables are exactly + ``LO_SW_SPECIES_VARIABLE_NAMES`` / ``LO_IALIRT_VARIABLE_NAMES``; each is + taken from its slot in ``species_names`` by name, or NaN-filled if it has no + slot. This keeps the variable set identical across SCI-LUT versions, which + L1B and L2 rely on since they only know the constants. Older SCI-LUTs still + list ``ne`` in ``species.sw.desired_species_names`` (it is ignored), and + ``imap_codice_l1a-sci-lut_20250731_20251123_v001`` has no + ``desired_species_names`` field at all. * ``lo_stepping_tab["row_number"]["data"]`` is the half-spin index per ESA step and can be **shorter than 128** (it is under the post-2025-12-18 scheme). L1A pads with ``HALF_SPIN_FILLVAL = 63`` and masks those steps to NaN. diff --git a/docs/source/algorithm-code-documentation/codice/data-products.rst b/docs/source/algorithm-code-documentation/codice/data-products.rst index 697eb55bb4..0abd05ba18 100644 --- a/docs/source/algorithm-code-documentation/codice/data-products.rst +++ b/docs/source/algorithm-code-documentation/codice/data-products.rst @@ -80,7 +80,8 @@ the science and housekeeping APIDs are defined in the two XTCE files in - 0x484 - ``COD_LO_SW_SPECIES_COUNTS`` - yes - - Sunward species counts, 16 species, 128 ESA steps. + - Sunward species counts, 16 telemetry slots (15 species written to + the CDF; ``ne`` is dropped), 128 ESA steps. * - 1157 - 0x485 - ``COD_LO_NSW_SPECIES_COUNTS`` @@ -452,9 +453,12 @@ See :ref:`codice-ancillary` for the contents of each of these. Species inventories ------------------- -**[CODE]** ``constants.py``. These lists drive which CDF variables exist; the -**actual** species ordering in the telemetry comes from the SCI-LUT and is -cross-checked at L1A. +**[CODE]** ``constants.py``. These lists drive which CDF variables exist, and +they are the same for every date and SCI-LUT version. The **actual** species +ordering in the telemetry comes from the SCI-LUT ``species_names`` and is used +at L1A to find each species' slot; a species with no slot is NaN-filled. For +``lo-sw-species`` and I-ALiRT Lo, the SCI-LUT ``desired_species_names`` is no +longer read (see :ref:`codice-ancillary`). .. list-table:: :header-rows: 1 @@ -462,11 +466,13 @@ cross-checked at L1A. * - Constant - Members - * - ``LO_SW_SPECIES_VARIABLE_NAMES`` (16) + * - ``LO_SW_SPECIES_VARIABLE_NAMES`` (15) - ``hplus``, ``heplusplus``, ``cplus4``, ``cplus5``, ``cplus6``, - ``oplus5``, ``oplus6``, ``oplus7``, ``oplus8``, ``ne``, ``mg``, ``si``, - ``fe_loq``, ``fe_hiq``, ``heplus``, ``cnoplus`` - * - ``LO_SW_SOLAR_WIND_SPECIES_VARIABLE_NAMES`` (14) + ``oplus5``, ``oplus6``, ``oplus7``, ``oplus8``, ``mg``, ``si``, + ``fe_loq``, ``fe_hiq``, ``heplus``, ``cnoplus``. ``ne`` is in the + telemetry and in some SCI-LUT ``desired_species_names`` lists, but is + not a product. + * - ``LO_SW_SOLAR_WIND_SPECIES_VARIABLE_NAMES`` (13) - the above minus ``heplus`` and ``cnoplus`` * - ``LO_SW_PICKUP_ION_SPECIES_VARIABLE_NAMES`` (2) - ``heplus``, ``cnoplus`` diff --git a/docs/source/algorithm-code-documentation/codice/implementation-status.rst b/docs/source/algorithm-code-documentation/codice/implementation-status.rst index 09896fddcf..91b25e3334 100644 --- a/docs/source/algorithm-code-documentation/codice/implementation-status.rst +++ b/docs/source/algorithm-code-documentation/codice/implementation-status.rst @@ -251,6 +251,18 @@ read from the SCI-LUT, so turning one off is handled - but selecting a *different* rate type in flight would require a code change. The same applies to the seven Hi aggregated counters. +Ne is not a Lo sunward species product +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The document lists Ne among the sunward solar-wind species. The code drops it: +``ne`` is not in ``LO_SW_SPECIES_VARIABLE_NAMES`` or +``LO_SW_SOLAR_WIND_SPECIES_VARIABLE_NAMES``, so ``lo-sw-species`` has 15 +species at L1A/L1B and 13 solar-wind species at L2, for every date. The ``ne`` +slot is still in the telemetry and the SCI-LUT ``species_names``, and some +SCI-LUT ``desired_species_names`` lists still include it; L1A ignores +``desired_species_names`` so that every SCI-LUT version produces the same +variables. + Direct events skip L1B ^^^^^^^^^^^^^^^^^^^^^^ @@ -316,9 +328,6 @@ Complete TODO inventory - Update the list of datasets that need geometric factors. * - ``codice_l2.py:1515`` - "May not even need L2 files for these products" (the broken pass-through). - * - ``constants.py:806`` - - Read the angular/priority/species variable name lists from the SCI-LUT - instead of hard-coding them. Test coverage ------------- diff --git a/docs/source/algorithm-code-documentation/codice/l1a.rst b/docs/source/algorithm-code-documentation/codice/l1a.rst index 90cdfc9ef5..e180b0ea2d 100644 --- a/docs/source/algorithm-code-documentation/codice/l1a.rst +++ b/docs/source/algorithm-code-documentation/codice/l1a.rst @@ -337,10 +337,13 @@ intensities. This is an on-board issue; the ground cannot undo it. See is normally ``(1,)``. Output dims ``(epoch, esa_step, spin_sector)`` with ``spin_sector`` of length 1. -Species selection is defensive: the module takes the union of the SCI-LUT's -``desired_species_names`` and the hard-coded list, then for each wanted species -either takes its index in ``actual_species_names`` or logs a warning and fills -NaN. This absorbs the Fe highQ/lowQ label swap. +Species selection: the output species are the hard-coded +``LO_SW_SPECIES_VARIABLE_NAMES`` (or ``LO_IALIRT_VARIABLE_NAMES``), the same for +every SCI-LUT version. For each one the module either takes its index in the +SCI-LUT ``species_names`` (``actual_species_names``) or logs a warning and fills +NaN. ``desired_species_names`` is not read. The Fe highQ/lowQ label swap and the +FSW "off by one" mislabeling are absorbed by ``species_names``, not by the +species list. Uncertainty is ``sqrt(counts)``, matching **[DOC]** :math:`\sigma_j(l) = \sqrt{C_j(l)}`. diff --git a/docs/source/algorithm-code-documentation/codice/l2.rst b/docs/source/algorithm-code-documentation/codice/l2.rst index 25e14bf2fa..0afe6604a1 100644 --- a/docs/source/algorithm-code-documentation/codice/l2.rst +++ b/docs/source/algorithm-code-documentation/codice/l2.rst @@ -163,6 +163,10 @@ position 1, so use the geometric factor for **one** sector: Species: H+, He++, O5+, O6+, O7+, O8+, C4+, C5+, C6+, Ne, Mg, Si, Fe(low Q), Fe(high Q). +**[CODE]** ``LO_SW_SOLAR_WIND_SPECIES_VARIABLE_NAMES`` omits Ne, so the code +produces 13 solar-wind species. See +:ref:`codice-implementation-status`. + **Sunward pickup ions** - isotropic over the sunward range, which is **5** azimuthal bins: diff --git a/docs/source/algorithm-code-documentation/codice/overview.rst b/docs/source/algorithm-code-documentation/codice/overview.rst index 1140c20479..9b90a483cb 100644 --- a/docs/source/algorithm-code-documentation/codice/overview.rst +++ b/docs/source/algorithm-code-documentation/codice/overview.rst @@ -700,9 +700,9 @@ instrument team for later changes. .. warning:: **[CODE]** The Fe highQ/lowQ label swap is handled indirectly: L1A reads the - species ordering from the SCI-LUT (``desired_species_names`` vs - ``actual_species_names``) and warns + fills with NaN when a wanted species is - absent. The comment in ``codice_l1a_lo_species.py`` calls this out as + species ordering (slot map) from the SCI-LUT ``species_names`` and looks up + each species in ``LO_SW_SPECIES_VARIABLE_NAMES`` by name, warning and filling + with NaN when one has no slot. The comment in ``codice_l1a_lo_species.py`` calls this out as handling "the bug in which the spacecraft was sending data down 'off by one' and getting mislabeled". If you see unexplained NaN species columns, check the SCI-LUT version first. From a61d1668b30b9b7fc7ea0e1077cb3c9690d5fc27 Mon Sep 17 00:00:00 2001 From: Luisa Coakley Date: Wed, 4 Nov 2026 10:16:01 -0700 Subject: [PATCH 7/7] add mods attrs for species ne removal --- .../cdf/config/imap_codice_global_cdf_attrs.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml b/imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml index 50d86a1ac5..80b2df8700 100644 --- a/imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml @@ -104,6 +104,9 @@ imap_codice_l1a_lo-sw-species: Data_type: L1A_lo-sw-species>Level-1A Lo Sunward Species Counts Data Logical_source: imap_codice_l1a_lo-sw-species Logical_source_description: IMAP Mission CoDICE Lo Level-1A Sunward Species Counts Data + MODS: + - >- + Major version 2, Nov. 18, 2026 - Remove species data for "ne" and "unc_ne". imap_codice_l1a_lo-nsw-species: <<: *instrument_base @@ -208,6 +211,9 @@ imap_codice_l1b_lo-sw-species: Data_type: L1B_lo-sw-species>Level-1B Lo Sunward Species Rates Data Logical_source: imap_codice_l1b_lo-sw-species Logical_source_description: IMAP Mission CoDICE Lo Level-1B Sunward Species Rates Data + MODS: + - >- + Major version 2, Nov. 18, 2026 - Remove species data for "ne" and "unc_ne". imap_codice_l1b_lo-nsw-species: <<: *instrument_base @@ -240,6 +246,9 @@ imap_codice_l2_lo-sw-species: Data_type: L2_lo-sw-species>Level-2 Lo Sunward Species Intensities Data Logical_source: imap_codice_l2_lo-sw-species Logical_source_description: IMAP Mission CoDICE Lo Level-2 Sunward Species Intensity Data + MODS: + - >- + Major version 2, Nov. 18, 2026 - Remove species data for "ne" and "unc_ne". imap_codice_l2_lo-nsw-species: <<: *instrument_base