From bdc5e3a826de69cdf8180383c9e24ba9b72e028a Mon Sep 17 00:00:00 2001 From: Leo Werneck Date: Thu, 8 Oct 2026 16:47:42 -0400 Subject: [PATCH 1/2] Significantly improved GLOWS integration test runtime --- .../test_glows_processor_integration.py | 56 ++++++++++++++++--- ...orce-reprocessing-config_20250101_v000.csv | 12 ++-- 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/tests/integration/test_glows_processor_integration.py b/tests/integration/test_glows_processor_integration.py index b31fa8268..ae8f2b150 100644 --- a/tests/integration/test_glows_processor_integration.py +++ b/tests/integration/test_glows_processor_integration.py @@ -6,7 +6,7 @@ import subprocess import unittest from datetime import timedelta, datetime -from functools import wraps +from functools import wraps, partial from pathlib import Path from typing import Callable from unittest import skipIf @@ -29,6 +29,10 @@ create_glows_l3a_from_dictionary, ) from imap_l3_processing.glows.l3d.utils import PATH_TO_L3D_TOOLKIT +from imap_l3_processing.glows.l3e.glows_l3e_initializer import ( + identify_versions_for_l3e_output_files, +) +from imap_l3_processing.glows.l3e.glows_l3e_utils import GlowsL3eVersionsForRepointings from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.utils import save_data from tests.integration.integration_test_helpers import ( @@ -110,8 +114,9 @@ def decorated(self): f"type=bind,src={l3_processing_dir}/run_local_input_data,dst=/run_local_input_data", ] - if imap_api_key := os.getenv("IMAP_API_KEY"): - args += ["-e", f"IMAP_API_KEY={imap_api_key}"] + # Pass by name so docker reads the value from the environment and the key isn't in error messages + if os.getenv("IMAP_API_KEY"): + args += ["-e", "IMAP_API_KEY"] if imap_data_access_url := os.getenv("IMAP_DATA_ACCESS_URL"): args += ["-e", f"IMAP_DATA_ACCESS_URL={imap_data_access_url}"] @@ -123,6 +128,29 @@ def decorated(self): return decorated +def only_process_l3e_for(repointings_to_keep: set[int]): + # Each repointing runs five survival probability executables and needs SPICE kernels covering it, + # so only select the repointings the test checks + def identify_subset(*args, **kwargs) -> GlowsL3eVersionsForRepointings: + def keep(versions: dict) -> dict: + return {n: v for n, v in versions.items() if n in repointings_to_keep} + + r = identify_versions_for_l3e_output_files(*args, **kwargs) + return GlowsL3eVersionsForRepointings( + [n for n in r.repointing_numbers if n in repointings_to_keep], + keep(r.hi_90_repointings), + keep(r.hi_45_repointings), + keep(r.lo_repointings), + keep(r.ultra_sf_repointings), + keep(r.ultra_hf_repointings), + ) + + return patch( + "imap_l3_processing.glows.l3e.glows_l3e_initializer.identify_versions_for_l3e_output_files", + side_effect=identify_subset, + ) + + class TestGlowsProcessorIntegration(unittest.TestCase): @with_tempdir def test_glows_l3a(self, tmp_dir): @@ -399,7 +427,8 @@ def test_l3bcde_first_time_processing(self): ) processor = GlowsProcessor(processing_input, input_metadata) - processor.process() + with only_process_l3e_for({36, 49}): + processor.process() expected_files = [ ScienceFilePath( @@ -572,7 +601,8 @@ def test_l3bcde_automatic_reprocessing(self): ) processor = GlowsProcessor(processing_input, input_metadata) - processor.process() + with only_process_l3e_for({36}): + processor.process() expected_files = [ ScienceFilePath( @@ -770,16 +800,24 @@ def test_run_glows_l3be_against_prod(self): "dependency": json.loads(processing_input.serialize()), "version": { desc: { - "major_version": 1, + "major_version": 2, "minor_version": 1, } for desc in descriptors_to_produce }, } - with patch( - "imap_l3_data_processor._parse_cli_arguments" - ) as mock_parse_cli_arguments: + # Nothing is uploaded, so reruns compute the same output names as earlier local runs; overwrite them + with ( + patch( + "imap_l3_data_processor._parse_cli_arguments" + ) as mock_parse_cli_arguments, + patch( + "imap_l3_processing.glows.glows_processor.save_data", + side_effect=partial(save_data, delete_if_present=True), + ), + only_process_l3e_for({36}), + ): mock_arguments = Mock() mock_arguments.instrument = "glows" mock_arguments.data_level = "l3b" diff --git a/tests/test_data/glows/imap_glows_force-reprocessing-config_20250101_v000.csv b/tests/test_data/glows/imap_glows_force-reprocessing-config_20250101_v000.csv index 29c5c7835..c307eb120 100644 --- a/tests/test_data/glows/imap_glows_force-reprocessing-config_20250101_v000.csv +++ b/tests/test_data/glows/imap_glows_force-reprocessing-config_20250101_v000.csv @@ -1,6 +1,6 @@ -ignore-after: 2026-08-13 12:39:07.208204+00:00 -l3e_survival-probability-hi-45 repoint36, repoint37, repoint38, repoint39, repoint40, repoint41, repoint42, repoint43, repoint44, repoint45, repoint46, repoint47, repoint48, repoint49, repoint50, repoint51, repoint52, repoint53, repoint54, repoint55, repoint56, repoint57, repoint58, repoint59, repoint60, repoint61, repoint62, repoint63, repoint64, repoint65, repoint66, repoint67, repoint68, repoint69, repoint70, repoint71, repoint72, repoint73, repoint74, repoint75, repoint76, repoint77, repoint78, repoint79, repoint80, repoint81, repoint82, repoint83, repoint84, repoint85, repoint86, repoint87, repoint88, repoint89, repoint90, repoint91, repoint92, repoint93, repoint94, repoint95, repoint96, repoint97, repoint98, repoint99, repoint100, repoint101, repoint102, repoint103, repoint104, repoint105, repoint106, repoint107, repoint108, repoint109, repoint110, repoint111, repoint112, repoint113, repoint114, repoint115, repoint116, repoint117, repoint118, repoint119, repoint120, repoint121, repoint122, repoint123, repoint124, repoint125, repoint126, repoint127, repoint128, repoint129, repoint130, repoint131, repoint132, repoint133, repoint134, repoint135, repoint136, repoint137, repoint138, repoint139, repoint140, repoint141, repoint142, repoint143, repoint144, repoint145, repoint146, repoint147, repoint148, repoint149, repoint150, repoint151, repoint152, repoint153, repoint154, repoint155, repoint156, repoint157, repoint158, repoint159, repoint160, repoint161 -l3e_survival-probability-hi-90 repoint36, repoint37, repoint38, repoint39, repoint40, repoint41, repoint42, repoint43, repoint44, repoint45, repoint46, repoint47, repoint48, repoint49, repoint50, repoint51, repoint52, repoint53, repoint54, repoint55, repoint56, repoint57, repoint58, repoint59, repoint60, repoint61, repoint62, repoint63, repoint64, repoint65, repoint66, repoint67, repoint68, repoint69, repoint70, repoint71, repoint72, repoint73, repoint74, repoint75, repoint76, repoint77, repoint78, repoint79, repoint80, repoint81, repoint82, repoint83, repoint84, repoint85, repoint86, repoint87, repoint88, repoint89, repoint90, repoint91, repoint92, repoint93, repoint94, repoint95, repoint96, repoint97, repoint98, repoint99, repoint100, repoint101, repoint102, repoint103, repoint104, repoint105, repoint106, repoint107, repoint108, repoint109, repoint110, repoint111, repoint112, repoint113, repoint114, repoint115, repoint116, repoint117, repoint118, repoint119, repoint120, repoint121, repoint122, repoint123, repoint124, repoint125, repoint126, repoint127, repoint128, repoint129, repoint130, repoint131, repoint132, repoint133, repoint134, repoint135, repoint136, repoint137, repoint138, repoint139, repoint140, repoint141, repoint142, repoint143, repoint144, repoint145, repoint146, repoint147, repoint148, repoint149, repoint150, repoint151, repoint152, repoint153, repoint154, repoint155, repoint156, repoint157, repoint158, repoint159, repoint160, repoint161 -l3e_survival-probability-lo repoint36, repoint37, repoint38, repoint39, repoint40, repoint41, repoint42, repoint43, repoint44, repoint45, repoint46, repoint47, repoint48, repoint49, repoint50, repoint51, repoint52, repoint53, repoint54, repoint55, repoint56, repoint57, repoint58, repoint59, repoint60, repoint61, repoint62, repoint63, repoint64, repoint65, repoint66, repoint67, repoint68, repoint69, repoint70, repoint71, repoint72, repoint73, repoint74, repoint75, repoint76, repoint77, repoint78, repoint79, repoint80, repoint81, repoint82, repoint83, repoint84, repoint85, repoint86, repoint87, repoint88, repoint89, repoint90, repoint91, repoint92, repoint93, repoint94, repoint95, repoint96, repoint97, repoint98, repoint99, repoint100, repoint101, repoint102, repoint103, repoint104, repoint105, repoint106, repoint107, repoint108, repoint109, repoint110, repoint111, repoint112, repoint113, repoint114, repoint115, repoint116, repoint117, repoint118, repoint119, repoint120, repoint121, repoint122, repoint123, repoint124, repoint125, repoint126, repoint127, repoint128, repoint129, repoint130, repoint131, repoint132, repoint133, repoint134, repoint135, repoint136, repoint137, repoint138, repoint139, repoint140, repoint141, repoint142, repoint143, repoint144, repoint145, repoint146, repoint147, repoint148, repoint149, repoint150, repoint151, repoint152, repoint153, repoint154, repoint155, repoint156, repoint157, repoint158, repoint159, repoint160, repoint161 -l3e_survival-probability-ul-hf repoint36, repoint37, repoint38, repoint39, repoint40, repoint41, repoint42, repoint43, repoint44, repoint45, repoint46, repoint47, repoint48, repoint49, repoint50, repoint51, repoint52, repoint53, repoint54, repoint55, repoint56, repoint57, repoint58, repoint59, repoint60, repoint61, repoint62, repoint63, repoint64, repoint65, repoint66, repoint67, repoint68, repoint69, repoint70, repoint71, repoint72, repoint73, repoint74, repoint75, repoint76, repoint77, repoint78, repoint79, repoint80, repoint81, repoint82, repoint83, repoint84, repoint85, repoint86, repoint87, repoint88, repoint89, repoint90, repoint91, repoint92, repoint93, repoint94, repoint95, repoint96, repoint97, repoint98, repoint99, repoint100, repoint101, repoint102, repoint103, repoint104, repoint105, repoint106, repoint107, repoint108, repoint109, repoint110, repoint111, repoint112, repoint113, repoint114, repoint115, repoint116, repoint117, repoint118, repoint119, repoint120, repoint121, repoint122, repoint123, repoint124, repoint125, repoint126, repoint127, repoint128, repoint129, repoint130, repoint131, repoint132, repoint133, repoint134, repoint135, repoint136, repoint137, repoint138, repoint139, repoint140, repoint141, repoint142, repoint143, repoint144, repoint145, repoint146, repoint147, repoint148, repoint149, repoint150, repoint151, repoint152, repoint153, repoint154, repoint155, repoint156, repoint157, repoint158, repoint159, repoint160, repoint161 -l3e_survival-probability-ul-sf repoint36, repoint37, repoint38, repoint39, repoint40, repoint41, repoint42, repoint43, repoint44, repoint45, repoint46, repoint47, repoint48, repoint49, repoint50, repoint51, repoint52, repoint53, repoint54, repoint55, repoint56, repoint57, repoint58, repoint59, repoint60, repoint61, repoint62, repoint63, repoint64, repoint65, repoint66, repoint67, repoint68, repoint69, repoint70, repoint71, repoint72, repoint73, repoint74, repoint75, repoint76, repoint77, repoint78, repoint79, repoint80, repoint81, repoint82, repoint83, repoint84, repoint85, repoint86, repoint87, repoint88, repoint89, repoint90, repoint91, repoint92, repoint93, repoint94, repoint95, repoint96, repoint97, repoint98, repoint99, repoint100, repoint101, repoint102, repoint103, repoint104, repoint105, repoint106, repoint107, repoint108, repoint109, repoint110, repoint111, repoint112, repoint113, repoint114, repoint115, repoint116, repoint117, repoint118, repoint119, repoint120, repoint121, repoint122, repoint123, repoint124, repoint125, repoint126, repoint127, repoint128, repoint129, repoint130, repoint131, repoint132, repoint133, repoint134, repoint135, repoint136, repoint137, repoint138, repoint139, repoint140, repoint141, repoint142, repoint143, repoint144, repoint145, repoint146, repoint147, repoint148, repoint149, repoint150, repoint151, repoint152, repoint153, repoint154, repoint155, repoint156, repoint157, repoint158, repoint159, repoint160, repoint161 \ No newline at end of file +ignore-after: 2100-01-01 00:00:00+00:00 +l3e_survival-probability-hi-45 repoint36 +l3e_survival-probability-hi-90 repoint36 +l3e_survival-probability-lo repoint36 +l3e_survival-probability-ul-hf repoint36 +l3e_survival-probability-ul-sf repoint36 \ No newline at end of file From 936c4a83f062702d6efb9caabcd8214a651d2774 Mon Sep 17 00:00:00 2001 From: Leo Werneck Date: Thu, 22 Oct 2026 12:40:58 -0400 Subject: [PATCH 2/2] Address PR comments --- .../test_glows_processor_integration.py | 36 ++++++++++++++++--- ...orce-reprocessing-config_20250101_v000.csv | 2 +- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_glows_processor_integration.py b/tests/integration/test_glows_processor_integration.py index ae8f2b150..5094b09af 100644 --- a/tests/integration/test_glows_processor_integration.py +++ b/tests/integration/test_glows_processor_integration.py @@ -5,6 +5,7 @@ import shutil import subprocess import unittest +from contextlib import contextmanager from datetime import timedelta, datetime from functools import wraps, partial from pathlib import Path @@ -128,14 +129,19 @@ def decorated(self): return decorated +@contextmanager def only_process_l3e_for(repointings_to_keep: set[int]): # Each repointing runs five survival probability executables and needs SPICE kernels covering it, - # so only select the repointings the test checks + # so only select the repointings the test checks. Yields the unfiltered results so tests can + # still check which repointings would have been processed. + unfiltered_results = [] + def identify_subset(*args, **kwargs) -> GlowsL3eVersionsForRepointings: def keep(versions: dict) -> dict: return {n: v for n, v in versions.items() if n in repointings_to_keep} r = identify_versions_for_l3e_output_files(*args, **kwargs) + unfiltered_results.append(r) return GlowsL3eVersionsForRepointings( [n for n in r.repointing_numbers if n in repointings_to_keep], keep(r.hi_90_repointings), @@ -145,10 +151,11 @@ def keep(versions: dict) -> dict: keep(r.ultra_hf_repointings), ) - return patch( + with patch( "imap_l3_processing.glows.l3e.glows_l3e_initializer.identify_versions_for_l3e_output_files", side_effect=identify_subset, - ) + ): + yield unfiltered_results class TestGlowsProcessorIntegration(unittest.TestCase): @@ -427,9 +434,22 @@ def test_l3bcde_first_time_processing(self): ) processor = GlowsProcessor(processing_input, input_metadata) - with only_process_l3e_for({36, 49}): + with only_process_l3e_for({36, 49}) as identified: processor.process() + [unfiltered] = identified + repointings_to_update = set(range(36, 116)) + # Repoint 20 is not otherwise updated, so it is only there because the force reprocessing config lists it + self.assertEqual(repointings_to_update | {20}, set(unfiltered.repointing_numbers)) + self.assertEqual(repointings_to_update | {20}, set(unfiltered.ultra_sf_repointings)) + for versions in [ + unfiltered.hi_90_repointings, + unfiltered.hi_45_repointings, + unfiltered.lo_repointings, + unfiltered.ultra_hf_repointings, + ]: + self.assertEqual(repointings_to_update, set(versions)) + expected_files = [ ScienceFilePath( "imap_glows_l3b_ion-rate-profile_20251102-cr02304_v000.0001.cdf" @@ -601,9 +621,15 @@ def test_l3bcde_automatic_reprocessing(self): ) processor = GlowsProcessor(processing_input, input_metadata) - with only_process_l3e_for({36}): + with only_process_l3e_for({36}) as identified: processor.process() + [unfiltered] = identified + all_repointings = set(range(1, 116)) + self.assertEqual(all_repointings, set(unfiltered.repointing_numbers)) + for versions in dataclasses.astuple(unfiltered)[1:]: + self.assertEqual(all_repointings, set(versions)) + expected_files = [ ScienceFilePath( "imap_glows_l3b_ion-rate-profile_20251102-cr02304_v001.0002.cdf" diff --git a/tests/test_data/glows/imap_glows_force-reprocessing-config_20250101_v000.csv b/tests/test_data/glows/imap_glows_force-reprocessing-config_20250101_v000.csv index c307eb120..da9a12742 100644 --- a/tests/test_data/glows/imap_glows_force-reprocessing-config_20250101_v000.csv +++ b/tests/test_data/glows/imap_glows_force-reprocessing-config_20250101_v000.csv @@ -3,4 +3,4 @@ l3e_survival-probability-hi-45 repoint36 l3e_survival-probability-hi-90 repoint36 l3e_survival-probability-lo repoint36 l3e_survival-probability-ul-hf repoint36 -l3e_survival-probability-ul-sf repoint36 \ No newline at end of file +l3e_survival-probability-ul-sf repoint20, repoint36 \ No newline at end of file