From e451191cd772d9effbec8d64fe5d1e68a9e9e954 Mon Sep 17 00:00:00 2001 From: Jonathan Niehof Date: Fri, 23 Oct 2026 07:56:57 -0600 Subject: [PATCH 1/4] Safe auto-fixes from ruff check --- .../figure_src/plot_pui_cutoff_staircase.py | 2 +- docs/swapi/figure_src/plot_real_data_fit.py | 4 +- .../science/geometric_factor_lookup.py | 2 +- imap_l3_processing/codice/l3/lo/models.py | 2 - .../l3/lo/science/codice_lo_calculations.py | 2 +- imap_l3_processing/data_utils.py | 2 +- .../glows_toolkit/l1a_data_direct_events.py | 3 - .../l3a/glows_toolkit/l1b_data_histogram.py | 1 - .../l3a/glows_toolkit/l2_data_histogram.py | 2 - .../glows/l3bc/dependency_validator.py | 1 - .../glows/l3bc/l3bc_toolkit/funcs.py | 3 +- imap_l3_processing/glows/l3bc/utils.py | 15 - .../glows/l3d/science/toolkit/funcs.py | 3 +- .../glows/l3e/glows_l3e_dependencies.py | 2 +- imap_l3_processing/hi/hi_sp_initializer.py | 4 +- imap_l3_processing/hit/l3/hit_processor.py | 2 +- imap_l3_processing/maps/mpfit.py | 8 +- imap_l3_processing/models.py | 2 +- imap_l3_processing/processor.py | 1 - .../alpha/fit_solar_wind_alpha_model.py | 1 - .../swe/l3/swe_l3_dependencies.py | 4 +- imap_l3_processing/ultra/ultra_processor.py | 1 - imap_l3_processing/utils.py | 6 +- scripts/get_batch_jobs_status.py | 2 +- .../glows/create_l2_glows_cdf_from_json.py | 2 +- scripts/hi/create_all_maps_from_inputs.py | 2 +- scripts/hi/create_cdf_from_ibex.py | 4 +- scripts/hi/create_hi_full_spin_deps.py | 4 +- scripts/hi/generate_fake_l1c_cdfs.py | 2 +- ...eate_l2_hit_sectored_rates_cdf_from_csv.py | 2 +- ...ombine_existing_maps_to_fix_frozen_data.py | 16 - scripts/lo/create_l2_from_pointing_sets.py | 12 +- scripts/swapi/fake_data_generator.py | 2 +- scripts/swapi/fit_and_plot_pui.py | 1 - scripts/swapi/upload_swapi_l3_dependencies.py | 1 - scripts/swe/compare_outputs.py | 2 +- .../ultra/create_example_ultra_l1c_pset.py | 2 +- scripts/ultra/create_l2_dependency_files.py | 4 +- scripts/ultra/create_l2_from_pointing_sets.py | 2 +- .../validation/create_l3_validation_maps.py | 2 +- scripts/ultra/validation/upload_files.py | 2 +- ..._codice_hi_mass_correction_lookup_table.py | 1 - tests/codice/l3/hi/test_models.py | 4 +- .../lo/science/test_codice_lo_calculations.py | 2 +- ...ce_lo_l3b_3d_distributions_dependencies.py | 2 +- .../codice/l3/lo/test_codice_lo_processor.py | 5 +- tests/glows/l3bc/science/test_time_parsing.py | 2 +- tests/glows/l3bc/test_dependency_validator.py | 1 - tests/glows/l3bc/test_utils.py | 2 +- tests/glows/l3d/test_glows_l3d_initializer.py | 2 - tests/glows/l3e/test_reprocess_info.py | 2 +- tests/glows/test_glows_processor.py | 42 +-- tests/hi/test_hi_combined_initializer.py | 26 +- tests/hi/test_hi_processor.py | 2 +- tests/hi/test_hi_sp_initializer.py | 22 +- .../hit/l3/pha/science/test_calculate_pha.py | 1 - tests/hit/l3/test_hit_processor.py | 2 +- tests/hit/l3/test_l3_sectored_dependencies.py | 10 +- tests/integration/integration_test_helpers.py | 1 - tests/lo/l3/test_lo_combined_dependencies.py | 1 - .../test_hilo_l3_survival_dependencies.py | 2 +- tests/maps/test_map_models.py | 1 - tests/maps/test_spectral_fit.py | 6 +- .../alpha/test_fit_solar_wind_alpha_model.py | 2 - .../solar_wind/test_integration_limits.py | 1 - .../test_efficiency_calibration_table.py | 1 - .../l3/science/test_moments_calculation.py | 4 +- .../3S5_l1b_ram_maps/archive/map_SCFrame.py | 4 - .../3S5_l1b_ram_maps/csv_to_soc.py | 1 - .../3S5_l1b_ram_maps/l1b_to_spin.py | 5 +- .../3S5_l1b_ram_maps/map_SCFrame.py | 5 - .../3S5_l1b_ram_maps/new_patch_arc2_linear.py | 1 - .../3S5_l1b_ram_maps/plot_map_DN.py | 4 - tests/test_imap_l3_data_processor.py | 1 - tests/ultra/test_models.py | 1 - .../test_ultra_combined_nsp_initializer.py | 136 ++++---- .../test_ultra_l3_survival_probabillity.py | 2 - tests/ultra/test_ultra_processor.py | 8 +- tests/ultra/test_ultra_sp_initializer.py | 298 +++++++++--------- 79 files changed, 336 insertions(+), 414 deletions(-) diff --git a/docs/swapi/figure_src/plot_pui_cutoff_staircase.py b/docs/swapi/figure_src/plot_pui_cutoff_staircase.py index 9d16ad2a4..30093152b 100644 --- a/docs/swapi/figure_src/plot_pui_cutoff_staircase.py +++ b/docs/swapi/figure_src/plot_pui_cutoff_staircase.py @@ -191,7 +191,7 @@ def sampled_distribution(grid, params, apply_cutoff): axis.plot( _CUTOFF_SWEEP_KMS, rate_reference, "-", color="0.4", lw=1, - label=f"fine-grid reference", + label="fine-grid reference", ) axis.scatter( _CUTOFF_SWEEP_KMS, rate_without, color="tab:red", s=10, diff --git a/docs/swapi/figure_src/plot_real_data_fit.py b/docs/swapi/figure_src/plot_real_data_fit.py index 70c11f713..61c943a39 100644 --- a/docs/swapi/figure_src/plot_real_data_fit.py +++ b/docs/swapi/figure_src/plot_real_data_fit.py @@ -133,9 +133,9 @@ def main(): ) print("\n--- Fit results ---") - print(f" Science bins (1..71, includes fine):") + print(" Science bins (1..71, includes fine):") _print_fit(science_result, sc_velocity_rtn) - print(f" Coarse-only bins (1..62):") + print(" Coarse-only bins (1..62):") _print_fit(coarse_result, sc_velocity_rtn) print("\n--- Bootstrap σ on the all-bins fit (B=300, warm-LM, xtol=1e-3) ---") diff --git a/imap_l3_processing/codice/l3/lo/direct_events/science/geometric_factor_lookup.py b/imap_l3_processing/codice/l3/lo/direct_events/science/geometric_factor_lookup.py index b3ac43f59..b5cd463d3 100644 --- a/imap_l3_processing/codice/l3/lo/direct_events/science/geometric_factor_lookup.py +++ b/imap_l3_processing/codice/l3/lo/direct_events/science/geometric_factor_lookup.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field -from datetime import datetime, date +from datetime import date from pathlib import Path from typing import TypeVar diff --git a/imap_l3_processing/codice/l3/lo/models.py b/imap_l3_processing/codice/l3/lo/models.py index 4f0d94810..4dd74ad30 100644 --- a/imap_l3_processing/codice/l3/lo/models.py +++ b/imap_l3_processing/codice/l3/lo/models.py @@ -3,14 +3,12 @@ from collections import namedtuple from dataclasses import dataclass, field from pathlib import Path -from typing import TypeVar import numpy as np from numpy import ndarray from spacepy.pycdf import CDF from imap_l3_processing.cdf.cdf_utils import read_numeric_variable, read_variable_and_mask_fill_values -from imap_l3_processing.codice.l3.lo.direct_events.science.mass_species_bin_lookup import MassSpeciesBinLookup from imap_l3_processing.models import DataProductVariable, DataProduct CODICE_LO_L2_NUM_PRIORITIES = 7 diff --git a/imap_l3_processing/codice/l3/lo/science/codice_lo_calculations.py b/imap_l3_processing/codice/l3/lo/science/codice_lo_calculations.py index 4d9a3d375..688d0d7cb 100644 --- a/imap_l3_processing/codice/l3/lo/science/codice_lo_calculations.py +++ b/imap_l3_processing/codice/l3/lo/science/codice_lo_calculations.py @@ -12,7 +12,7 @@ from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import EnergyLookup from imap_l3_processing.codice.l3.lo.direct_events.science.mass_coefficient_lookup import MassCoefficientLookup from imap_l3_processing.codice.l3.lo.direct_events.science.mass_species_bin_lookup import MassSpeciesBinLookup -from imap_l3_processing.codice.l3.lo.models import EnergyAndSpinAngle, CodiceLoDirectEventData +from imap_l3_processing.codice.l3.lo.models import CodiceLoDirectEventData def calculate_partial_densities(intensities: np.ndarray, esa_steps: np.ndarray, mass_per_charge: float): diff --git a/imap_l3_processing/data_utils.py b/imap_l3_processing/data_utils.py index a02101f32..a49d5ea9d 100644 --- a/imap_l3_processing/data_utils.py +++ b/imap_l3_processing/data_utils.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import timedelta import numpy as np diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/l1a_data_direct_events.py b/imap_l3_processing/glows/l3a/glows_toolkit/l1a_data_direct_events.py index ed6c9b454..bcb8da31d 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/l1a_data_direct_events.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/l1a_data_direct_events.py @@ -2,12 +2,9 @@ Author: Marek Strumik, maro at cbk.waw.pl """ import json -import os -import struct import copy import numpy as np from .constants import VERSION, SUBSECOND_LIMIT -from .time import Time from .funcs import time_sec_subsec_to_utc from .direct_event import DirectEvent diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/l1b_data_histogram.py b/imap_l3_processing/glows/l3a/glows_toolkit/l1b_data_histogram.py index e20ff9a5e..b6443198a 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/l1b_data_histogram.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/l1b_data_histogram.py @@ -3,7 +3,6 @@ """ import json import copy -import sys import numpy as np import pandas as pd from astropy.coordinates import SkyCoord diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/l2_data_histogram.py b/imap_l3_processing/glows/l3a/glows_toolkit/l2_data_histogram.py index dc7222f7a..ad2578c3e 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/l2_data_histogram.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/l2_data_histogram.py @@ -4,8 +4,6 @@ import json import copy import numpy as np -import time -import sys from .constants import VERSION, L1B_L2_HIST_FLAG_FIELDS, SCAN_CIRCLE_ANGULAR_RADIUS from .funcs import time_sec_subsec_to_utc, calibration_factor from .geom_fun_libr_box import spin2lb diff --git a/imap_l3_processing/glows/l3bc/dependency_validator.py b/imap_l3_processing/glows/l3bc/dependency_validator.py index bdf58db2e..e45038b04 100644 --- a/imap_l3_processing/glows/l3bc/dependency_validator.py +++ b/imap_l3_processing/glows/l3bc/dependency_validator.py @@ -3,7 +3,6 @@ import netCDF4 import numpy as np -from astropy.time import Time, TimeDelta from numpy import ndarray YEAR_COLUMN = 0 diff --git a/imap_l3_processing/glows/l3bc/l3bc_toolkit/funcs.py b/imap_l3_processing/glows/l3bc/l3bc_toolkit/funcs.py index e669c40f5..02ea12ea8 100644 --- a/imap_l3_processing/glows/l3bc/l3bc_toolkit/funcs.py +++ b/imap_l3_processing/glows/l3bc/l3bc_toolkit/funcs.py @@ -11,7 +11,6 @@ from astropy.time import Time from astropy.coordinates import get_sun -from imap_l3_processing.glows.quality_flags import NOMINAL_ALPHA_PROTON_RATIO_VALUE from .constants import PHISICAL_CONSTANTS, CONST_ALPHA_TO_PROTON import logging @@ -288,7 +287,7 @@ def get_spher_coord_rad(vec): if np.shape(vec)[0]!=3: raise "get_spher_coord_rad() error: np.shape(vec)[0]!=3" r=np.sqrt(np.dot(vec,vec)) - if r==0.0: b=0.0; + if r==0.0: b=0.0 else: b=np.arcsin(vec[2]/r) # compute latitude # compute longitude if vec[0]==0.0 and vec[1]==0.0: l=0.0 diff --git a/imap_l3_processing/glows/l3bc/utils.py b/imap_l3_processing/glows/l3bc/utils.py index 230e1bb70..492725f63 100644 --- a/imap_l3_processing/glows/l3bc/utils.py +++ b/imap_l3_processing/glows/l3bc/utils.py @@ -1,26 +1,11 @@ from datetime import timedelta, datetime -from datetime import timedelta, datetime from pathlib import Path from typing import Optional -import imap_data_access import numpy as np import pandas as pd from imap_processing.spice.repoint import get_repoint_data -from spacepy.pycdf import CDF -from imap_l3_processing.glows.l3a.models import GlowsL3LightCurve, PHOTON_FLUX_UNCERTAINTY_CDF_VAR_NAME, \ - PHOTON_FLUX_CDF_VAR_NAME, RAW_HISTOGRAM_CDF_VAR_NAME, EXPOSURE_TIMES_CDF_VAR_NAME, EPOCH_CDF_VAR_NAME, \ - EPOCH_DELTA_CDF_VAR_NAME, SPIN_ANGLE_CDF_VAR_NAME, SPIN_ANGLE_DELTA_CDF_VAR_NAME, LATITUDE_CDF_VAR_NAME, \ - LONGITUDE_CDF_VAR_NAME, EXTRA_HELIOSPHERIC_BACKGROUND_CDF_VAR_NAME, TIME_DEPENDENT_BACKGROUND_CDF_VAR_NAME, \ - SPACECRAFT_VELOCITY_STD_DEV_CDF_VAR_NAME, FILTER_TEMPERATURE_AVERAGE_CDF_VAR_NAME, \ - FILTER_TEMPERATURE_STD_DEV_CDF_VAR_NAME, HV_VOLTAGE_AVERAGE_CDF_VAR_NAME, HV_VOLTAGE_STD_DEV_CDF_VAR_NAME, \ - SPIN_PERIOD_AVERAGE_CDF_VAR_NAME, SPIN_PERIOD_STD_DEV_CDF_VAR_NAME, SPIN_PERIOD_GROUND_AVERAGE_CDF_VAR_NAME, \ - SPIN_PERIOD_GROUND_STD_DEV_CDF_VAR_NAME, PULSE_LENGTH_AVERAGE_CDF_VAR_NAME, PULSE_LENGTH_STD_DEV_CDF_VAR_NAME, \ - POSITION_ANGLE_OFFSET_AVERAGE_CDF_VAR_NAME, POSITION_ANGLE_OFFSET_STD_DEV_CDF_VAR_NAME, \ - SPIN_AXIS_ORIENTATION_AVERAGE_CDF_VAR_NAME, SPIN_AXIS_ORIENTATION_STD_DEV_CDF_VAR_NAME, \ - SPACECRAFT_LOCATION_AVERAGE_CDF_VAR_NAME, SPACECRAFT_LOCATION_STD_DEV_CDF_VAR_NAME, \ - SPACECRAFT_VELOCITY_AVERAGE_CDF_VAR_NAME, NUM_OF_BINS_CDF_VAR_NAME def get_best_ancillary(start_date: datetime, end_date: datetime, ancillary_query_results: list[dict]) -> Optional[str]: diff --git a/imap_l3_processing/glows/l3d/science/toolkit/funcs.py b/imap_l3_processing/glows/l3d/science/toolkit/funcs.py index e669c40f5..02ea12ea8 100644 --- a/imap_l3_processing/glows/l3d/science/toolkit/funcs.py +++ b/imap_l3_processing/glows/l3d/science/toolkit/funcs.py @@ -11,7 +11,6 @@ from astropy.time import Time from astropy.coordinates import get_sun -from imap_l3_processing.glows.quality_flags import NOMINAL_ALPHA_PROTON_RATIO_VALUE from .constants import PHISICAL_CONSTANTS, CONST_ALPHA_TO_PROTON import logging @@ -288,7 +287,7 @@ def get_spher_coord_rad(vec): if np.shape(vec)[0]!=3: raise "get_spher_coord_rad() error: np.shape(vec)[0]!=3" r=np.sqrt(np.dot(vec,vec)) - if r==0.0: b=0.0; + if r==0.0: b=0.0 else: b=np.arcsin(vec[2]/r) # compute latitude # compute longitude if vec[0]==0.0 and vec[1]==0.0: l=0.0 diff --git a/imap_l3_processing/glows/l3e/glows_l3e_dependencies.py b/imap_l3_processing/glows/l3e/glows_l3e_dependencies.py index 809e3cb4a..f00ea77b7 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_dependencies.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_dependencies.py @@ -1,7 +1,7 @@ import json import logging import shutil -from dataclasses import dataclass, field +from dataclasses import dataclass from datetime import datetime from pathlib import Path diff --git a/imap_l3_processing/hi/hi_sp_initializer.py b/imap_l3_processing/hi/hi_sp_initializer.py index bfaad67ad..ab8cdf206 100644 --- a/imap_l3_processing/hi/hi_sp_initializer.py +++ b/imap_l3_processing/hi/hi_sp_initializer.py @@ -75,7 +75,7 @@ def __init__(self): sp_hi45_query_result = imap_data_access.query( instrument='glows', data_level='l3e', - descriptor=f'survival-probability-hi-45', + descriptor='survival-probability-hi-45', version='latest' ) self.glows_hi45_file_by_repoint = {int(r["repointing"]): Path(r["file_path"]).name for r in @@ -84,7 +84,7 @@ def __init__(self): sp_hi90_query_result = imap_data_access.query( instrument='glows', data_level='l3e', - descriptor=f'survival-probability-hi-90', + descriptor='survival-probability-hi-90', version='latest' ) self.glows_hi90_file_by_repoint = {int(r["repointing"]): Path(r["file_path"]).name for r in diff --git a/imap_l3_processing/hit/l3/hit_processor.py b/imap_l3_processing/hit/l3/hit_processor.py index f2a9b2593..3b83da0ba 100644 --- a/imap_l3_processing/hit/l3/hit_processor.py +++ b/imap_l3_processing/hit/l3/hit_processor.py @@ -52,7 +52,7 @@ def process_direct_event_product(self, direct_event_dependencies.hit_l1_data.event_binary): try: event_raw_pha_events = PHAEventReader.read_all_pha_events(event_binary) - except Exception as e: + except Exception: logger.warning(f"Could not read all pha events for epoch {epoch}", exc_info=True) continue diff --git a/imap_l3_processing/maps/mpfit.py b/imap_l3_processing/maps/mpfit.py index 767c24f89..6ccbf214d 100644 --- a/imap_l3_processing/maps/mpfit.py +++ b/imap_l3_processing/maps/mpfit.py @@ -1333,7 +1333,7 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, # if ct GT 0 OR finite(ratio) EQ 0 then begin if self.status != 0: - break; + break # End of outer loop. catch_msg = 'in the termination phase' @@ -1756,8 +1756,8 @@ def qrfac(self, a, pivot=0): # exchange occurs later, within the body of MPFIT, after the # extraneous columns of the matrix have been shed. if kmax != j: - temp = ipvt[j]; - ipvt[j] = ipvt[kmax]; + temp = ipvt[j] + ipvt[j] = ipvt[kmax] ipvt[kmax] = temp rdiag[kmax] = rdiag[j] wa[kmax] = wa[j] @@ -2135,7 +2135,7 @@ def lmpar(self, r, ipvt, diag, qtb, delta, x, sdiag, par=None): if (numpy.abs(fp) <= 0.1 * delta) or \ ((parl == 0) and (fp <= temp) and (temp < 0)) or \ (iter == 10): - break; + break # Compute the newton correction wa1 = diag[ipvt] * wa2[ipvt] / dxnorm diff --git a/imap_l3_processing/models.py b/imap_l3_processing/models.py index c90866d6e..dcf44b4d5 100644 --- a/imap_l3_processing/models.py +++ b/imap_l3_processing/models.py @@ -72,7 +72,7 @@ class DataProduct(abc.ABC, Generic[D]): @abc.abstractmethod def to_data_product_variables(self) -> list[DataProductVariable]: - raise NotImplemented + raise NotImplementedError def add_paths_to_parents(self, paths: list[Path]): self.add_filenames_to_parents([path.name for path in paths]) diff --git a/imap_l3_processing/processor.py b/imap_l3_processing/processor.py index 15d2dd1a4..bf539b94d 100644 --- a/imap_l3_processing/processor.py +++ b/imap_l3_processing/processor.py @@ -1,7 +1,6 @@ import abc from pathlib import Path -import spiceypy from imap_data_access.processing_input import ProcessingInputCollection from imap_l3_processing.models import InputMetadata diff --git a/imap_l3_processing/swapi/l3a/science/solar_wind/alpha/fit_solar_wind_alpha_model.py b/imap_l3_processing/swapi/l3a/science/solar_wind/alpha/fit_solar_wind_alpha_model.py index a28d39c06..4c39674c3 100644 --- a/imap_l3_processing/swapi/l3a/science/solar_wind/alpha/fit_solar_wind_alpha_model.py +++ b/imap_l3_processing/swapi/l3a/science/solar_wind/alpha/fit_solar_wind_alpha_model.py @@ -6,7 +6,6 @@ from numpy import ndarray from uncertainties import UFloat, covariance_matrix, ufloat -from imap_l3_processing.constants import ALPHA_MASS_PER_CHARGE_M_P_PER_E from imap_l3_processing.swapi.l3a.science.solar_wind.alpha.calculate_initial_guess import ( calculate_initial_guess, ) diff --git a/imap_l3_processing/swe/l3/swe_l3_dependencies.py b/imap_l3_processing/swe/l3/swe_l3_dependencies.py index 1de137846..535744bfe 100644 --- a/imap_l3_processing/swe/l3/swe_l3_dependencies.py +++ b/imap_l3_processing/swe/l3/swe_l3_dependencies.py @@ -35,12 +35,12 @@ def fetch_dependencies(cls, dependencies: ProcessingInputCollection) -> SweL3Dep swe_l2_dependency = next( d.imap_file_paths[0] for d in science_files if d.source == "swe" and d.data_type == "l2") except StopIteration: - raise ValueError(f"Missing SWE l2 dependency.") + raise ValueError("Missing SWE l2 dependency.") try: swe_l1b_dependency = next( d.imap_file_paths[0] for d in science_files if d.source == "swe" and d.data_type == "l1b") except StopIteration: - raise ValueError(f"Missing SWE l1b dependency.") + raise ValueError("Missing SWE l1b dependency.") mag_file, mag_level = select_mag_path(dependencies, MAG_DESPUN_L1D_DESCRIPTOR) if mag_file is None: raise ValueError(f"Missing MAG {MAG_DESPUN_L1D_DESCRIPTOR} dependency.") diff --git a/imap_l3_processing/ultra/ultra_processor.py b/imap_l3_processing/ultra/ultra_processor.py index 7022bfccf..1b42def05 100644 --- a/imap_l3_processing/ultra/ultra_processor.py +++ b/imap_l3_processing/ultra/ultra_processor.py @@ -2,7 +2,6 @@ import pickle import re from dataclasses import dataclass -from datetime import datetime, timedelta from hashlib import sha256 from typing import Optional diff --git a/imap_l3_processing/utils.py b/imap_l3_processing/utils.py index 48016f00f..4a6e1bb56 100644 --- a/imap_l3_processing/utils.py +++ b/imap_l3_processing/utils.py @@ -164,11 +164,11 @@ def generate_global_metadata_for_undefined_logical_source(input_metadata: InputM level = input_metadata.data_level.replace('l', '') data_type_string = f"Level-{level}" if "spx" in input_metadata.descriptor: - data_type_string += f" Spectral Fit Index Map" + data_type_string += " Spectral Fit Index Map" elif "ena" in input_metadata.descriptor and "-sp-" in input_metadata.descriptor: - data_type_string += f" Survival Corrected" + data_type_string += " Survival Corrected" elif "ena" in input_metadata.descriptor: - data_type_string += f" ENA Intensity" + data_type_string += " ENA Intensity" logical_source_global_attrs = { "Data_level": level, diff --git a/scripts/get_batch_jobs_status.py b/scripts/get_batch_jobs_status.py index 55cbff752..991f33811 100644 --- a/scripts/get_batch_jobs_status.py +++ b/scripts/get_batch_jobs_status.py @@ -51,7 +51,7 @@ class Command(enum.Enum): imap_dev_url = os.getenv("IMAP_DATA_ACCESS_URL") headers = {'x-api-key': os.getenv("IMAP_API_KEY")} -if not "log_id" in vars(args): +if "log_id" not in vars(args): query_params = { key: val for key, val in vars(args).items() if val is not None and key not in ['command']} response = requests.get(url=f"{imap_dev_url}/processing-jobs", headers=headers, params=query_params) processing_jobs = response.json() diff --git a/scripts/glows/create_l2_glows_cdf_from_json.py b/scripts/glows/create_l2_glows_cdf_from_json.py index eab48bd8c..ad09e91d9 100644 --- a/scripts/glows/create_l2_glows_cdf_from_json.py +++ b/scripts/glows/create_l2_glows_cdf_from_json.py @@ -123,5 +123,5 @@ def fill_official_l2_cdf_with_json_values(output_folder: Path, input_path: Path) for file_path in json_directory.iterdir(): try: fill_official_l2_cdf_with_json_values(output_directory, file_path) - except Exception as e: + except Exception: traceback.print_exc() diff --git a/scripts/hi/create_all_maps_from_inputs.py b/scripts/hi/create_all_maps_from_inputs.py index 2a777c591..4b781653d 100644 --- a/scripts/hi/create_all_maps_from_inputs.py +++ b/scripts/hi/create_all_maps_from_inputs.py @@ -42,7 +42,7 @@ def run_all_maps(mock_parse_cli_arguments, *, output_dir, input_dir,): try: input_dir = Path(sys.argv[1]) except: - input_dir = Path(f'hi_validation_input') + input_dir = Path('hi_validation_input') print(f"Could not open input file, using default input path of {input_dir} instead") run_all_maps(output_dir=output_dir, input_dir=input_dir) diff --git a/scripts/hi/create_cdf_from_ibex.py b/scripts/hi/create_cdf_from_ibex.py index 309fd14ac..3102c2018 100644 --- a/scripts/hi/create_cdf_from_ibex.py +++ b/scripts/hi/create_cdf_from_ibex.py @@ -1,5 +1,5 @@ import argparse -from datetime import datetime, timedelta +from datetime import datetime from pathlib import Path import numpy as np @@ -27,7 +27,7 @@ def extract_data(lines: list[str]): variances = [] for file in folder.iterdir(): - if file.parts[-1].endswith("flux.txt") and not "mono" in file.parts[-1]: + if file.parts[-1].endswith("flux.txt") and "mono" not in file.parts[-1]: group_file_name_prefix = '-'.join(file.parts[-1].split('-')[:-1]) with open(file.parent / (group_file_name_prefix + "-flux.txt"), "r") as flux_file: diff --git a/scripts/hi/create_hi_full_spin_deps.py b/scripts/hi/create_hi_full_spin_deps.py index 3b36eee3c..f2a9e3713 100644 --- a/scripts/hi/create_hi_full_spin_deps.py +++ b/scripts/hi/create_hi_full_spin_deps.py @@ -25,7 +25,7 @@ def create_hi_full_spin_deps( if l2_map_dir is None: l2_map_dir = get_test_instrument_team_data_path(f"hi/hi{sensor}-6months") if glows_dir is None: - glows_dir = get_test_data_path(f"hi/fake_l3e_survival_probabilities/90") + glows_dir = get_test_data_path("hi/fake_l3e_survival_probabilities/90") l1c_folder = output_dir / "l1c" l1c_folder.mkdir(parents=True, exist_ok=True) @@ -47,7 +47,7 @@ def create_hi_full_spin_deps( f"h{sensor}-ena-h-sf-nsp-anti-hae-4deg-6mo") input_metadata = InputMetadata("hi", "l3", datetime(year=2025, month=4, day=15), None, "v001", - f"h90-ena-h-sf-sp-full-hae-4deg-6mo") + "h90-ena-h-sf-sp-full-hae-4deg-6mo") ram_data = survival_probability_processing.process_survival_probabilities(ramified_map_deps, SpiceFrame.ECLIPJ2000) ram_exposure_is_zero = (np.isnan(ram_data.intensity_map_data.ena_intensity) diff --git a/scripts/hi/generate_fake_l1c_cdfs.py b/scripts/hi/generate_fake_l1c_cdfs.py index 295bf0711..00f229c73 100644 --- a/scripts/hi/generate_fake_l1c_cdfs.py +++ b/scripts/hi/generate_fake_l1c_cdfs.py @@ -25,4 +25,4 @@ def generate_fake_l1c(start_date: datetime, num_days: int, output_dir: Path) -> if __name__ == "__main__": generate_fake_l1c(datetime(month=4, year=2025, day=16, hour=12), num_days=90, - output_dir=get_test_data_path(f"hi/fake_l1c/90")) + output_dir=get_test_data_path("hi/fake_l1c/90")) diff --git a/scripts/hit/create_l2_hit_sectored_rates_cdf_from_csv.py b/scripts/hit/create_l2_hit_sectored_rates_cdf_from_csv.py index e10e217fa..8233eecf4 100644 --- a/scripts/hit/create_l2_hit_sectored_rates_cdf_from_csv.py +++ b/scripts/hit/create_l2_hit_sectored_rates_cdf_from_csv.py @@ -83,5 +83,5 @@ def create_l2_hit_sectored_rates_cdf_from_csv(csv_file_path: str, output_filenam try: create_l2_hit_sectored_rates_cdf_from_csv(str(input_path), str(cdf_file_path)) - except Exception as e: + except Exception: traceback.print_exc() diff --git a/scripts/lo/combine_existing_maps_to_fix_frozen_data.py b/scripts/lo/combine_existing_maps_to_fix_frozen_data.py index 63321188f..eb20d9889 100644 --- a/scripts/lo/combine_existing_maps_to_fix_frozen_data.py +++ b/scripts/lo/combine_existing_maps_to_fix_frozen_data.py @@ -1,31 +1,15 @@ -import abc -import dataclasses import logging -import re import shutil -from concurrent.futures import ThreadPoolExecutor from datetime import datetime from pathlib import Path -from typing import Self, Optional, override, Literal import imap_data_access -import numpy as np -import pandas -import xarray as xr from imap_data_access import ProcessingInputCollection, ScienceFilePath from imap_data_access.file_validation import Version from imap_data_access.processing_input import ScienceInput -from imap_processing.cdf.utils import write_cdf as write_l2_cdf -from imap_processing.ena_maps.ena_maps import RectangularSkyMap -from imap_processing.hit.l1b.constants import FILLVAL_INT64 -from imap_processing.spice.geometry import SpiceFrame -from spacepy.pycdf import CDF -from imap_l3_processing.constants import TT2000_EPOCH -from imap_l3_processing.lo.l3.lo_sp_initializer import LO_SP_MAP_KERNELS from imap_l3_processing.lo.lo_processor import LoProcessor from imap_l3_processing.models import InputMetadata, VersionMap -from imap_l3_processing.utils import furnished_metakernel from tests.test_helpers import get_run_local_data_path diff --git a/scripts/lo/create_l2_from_pointing_sets.py b/scripts/lo/create_l2_from_pointing_sets.py index 5a14304ea..80d2c8c42 100644 --- a/scripts/lo/create_l2_from_pointing_sets.py +++ b/scripts/lo/create_l2_from_pointing_sets.py @@ -1,7 +1,7 @@ import os import shutil import subprocess -from datetime import datetime, timedelta +from datetime import datetime from pathlib import Path import imap_data_access @@ -35,7 +35,7 @@ def create_input_file() -> str: inputs.add([generate_imap_input(f) for f in static_files]) - dependency_file = f"imap_lo_l2_90sensor-de-20250415-20250504_20250415_v001.json" + dependency_file = "imap_lo_l2_90sensor-de-20250415-20250504_20250415_v001.json" dependency_path = generate_imap_file_path(dependency_file).construct_path() dependency_path.parent.mkdir(parents=True, exist_ok=True) dependency_path.write_text(inputs.serialize()) @@ -72,10 +72,10 @@ def generate_map(descriptor: str, start: datetime): start_date = datetime(2025, 4, 15) descriptors = [ - f"l090-ena-h-hf-nsp-ram-hae-6deg-1yr", - f"l090-ena-h-sf-nsp-ram-hae-6deg-1yr", - f"l090-enanbs-h-hf-nsp-ram-hae-6deg-1yr", - f"l090-enanbs-h-sf-nsp-ram-hae-6deg-1yr", + "l090-ena-h-hf-nsp-ram-hae-6deg-1yr", + "l090-ena-h-sf-nsp-ram-hae-6deg-1yr", + "l090-enanbs-h-hf-nsp-ram-hae-6deg-1yr", + "l090-enanbs-h-sf-nsp-ram-hae-6deg-1yr", ] for descriptor in descriptors: generate_map(descriptor, start_date) \ No newline at end of file diff --git a/scripts/swapi/fake_data_generator.py b/scripts/swapi/fake_data_generator.py index 51fe72923..5af6de3e0 100644 --- a/scripts/swapi/fake_data_generator.py +++ b/scripts/swapi/fake_data_generator.py @@ -1,4 +1,4 @@ -from random import random, randint +from random import randint import numpy as np diff --git a/scripts/swapi/fit_and_plot_pui.py b/scripts/swapi/fit_and_plot_pui.py index 880bb13e2..f187be398 100644 --- a/scripts/swapi/fit_and_plot_pui.py +++ b/scripts/swapi/fit_and_plot_pui.py @@ -35,7 +35,6 @@ from imap_l3_processing.constants import ( HE_PUI_PARTICLE_MASS_PER_CHARGE_M_P_PER_E, - ONE_AU_IN_KM, ONE_SECOND_IN_NANOSECONDS, ) from imap_l3_processing.models import InputMetadata diff --git a/scripts/swapi/upload_swapi_l3_dependencies.py b/scripts/swapi/upload_swapi_l3_dependencies.py index 2510bef1a..4a7fa0111 100644 --- a/scripts/swapi/upload_swapi_l3_dependencies.py +++ b/scripts/swapi/upload_swapi_l3_dependencies.py @@ -1,5 +1,4 @@ import shutil -from pathlib import Path import imap_data_access from imap_data_access.file_validation import generate_imap_file_path diff --git a/scripts/swe/compare_outputs.py b/scripts/swe/compare_outputs.py index 2258c8567..1979a4e85 100644 --- a/scripts/swe/compare_outputs.py +++ b/scripts/swe/compare_outputs.py @@ -174,7 +174,7 @@ def compare_parallel_perp(): ratio = heritage_data / modern_data smooth_ratio = np.convolve(ratio, np.ones(5) / 5, mode="same") plt.plot(smooth_ratio) - plt.savefig(OUTPUT_PATH / f"density_ratio.png") + plt.savefig(OUTPUT_PATH / "density_ratio.png") plt.clf() l3_output_cdf.close() diff --git a/scripts/ultra/create_example_ultra_l1c_pset.py b/scripts/ultra/create_example_ultra_l1c_pset.py index 020187d65..d23d237f6 100644 --- a/scripts/ultra/create_example_ultra_l1c_pset.py +++ b/scripts/ultra/create_example_ultra_l1c_pset.py @@ -267,7 +267,7 @@ def _write_ultra_l1c_cdf_with_parents( def _add_depends(var: Var, depends: list[str], depend_0: str = None): if depend_0 is not None: - var.attrs[f"DEPEND_0"] = depend_0 + var.attrs["DEPEND_0"] = depend_0 for i, dep in enumerate(depends): var.attrs[f"DEPEND_{i + 1}"] = dep diff --git a/scripts/ultra/create_l2_dependency_files.py b/scripts/ultra/create_l2_dependency_files.py index e047c644f..afcec421d 100644 --- a/scripts/ultra/create_l2_dependency_files.py +++ b/scripts/ultra/create_l2_dependency_files.py @@ -1,4 +1,4 @@ -from datetime import timedelta, datetime +from datetime import datetime from pathlib import Path import imap_data_access @@ -7,7 +7,7 @@ from imap_data_access.file_validation import generate_imap_file_path from imap_data_access.processing_input import generate_imap_input -from imap_l3_processing.utils import get_spice_kernels_file_names, SpiceKernelTypes +from imap_l3_processing.utils import SpiceKernelTypes REFERENCE_FRAMES = ['sf', 'hf'] POINTING_SET_DESCRIPTORS_FROM_REF_FRAME = { diff --git a/scripts/ultra/create_l2_from_pointing_sets.py b/scripts/ultra/create_l2_from_pointing_sets.py index 4d7568154..b37f8abc0 100644 --- a/scripts/ultra/create_l2_from_pointing_sets.py +++ b/scripts/ultra/create_l2_from_pointing_sets.py @@ -70,7 +70,7 @@ def generate_map(descriptor: str, start: datetime, end: datetime): end_date = datetime(2025, 10, 15) duration = '6mo' - descriptor = f"u90-ena-h-hf-nsp-full-hae-2deg-6mo" + descriptor = "u90-ena-h-hf-nsp-full-hae-2deg-6mo" generate_map(descriptor, start_date, end_date) # for sensor in [45, 90]: # for side in ['ram', 'anti', 'full']: diff --git a/scripts/ultra/validation/create_l3_validation_maps.py b/scripts/ultra/validation/create_l3_validation_maps.py index ec3276922..537ee63d0 100644 --- a/scripts/ultra/validation/create_l3_validation_maps.py +++ b/scripts/ultra/validation/create_l3_validation_maps.py @@ -46,7 +46,7 @@ def run_all_maps(mock_parse_cli_arguments, *, output_dir, input_dir,): try: input_dir = Path(sys.argv[1]) except: - input_dir = Path(fr'C:\Users\Harrison\AppData\Local\cava\cava\data') + input_dir = Path(r'C:\Users\Harrison\AppData\Local\cava\cava\data') print(f"Could not open input file, using default input path of {input_dir} instead") run_all_maps(output_dir=output_dir, input_dir=input_dir) \ No newline at end of file diff --git a/scripts/ultra/validation/upload_files.py b/scripts/ultra/validation/upload_files.py index f39a0cf0c..18cfb9cf3 100644 --- a/scripts/ultra/validation/upload_files.py +++ b/scripts/ultra/validation/upload_files.py @@ -30,7 +30,7 @@ def upload_file(f): uploaded.append(f.name) print("uploaded", f.name) return - except Exception as e: + except Exception: failures += 1 print(f"failed to upload file: {f} after {failures} attempts, retrying...") diff --git a/tests/codice/l3/hi/direct_event/test_codice_hi_mass_correction_lookup_table.py b/tests/codice/l3/hi/direct_event/test_codice_hi_mass_correction_lookup_table.py index 968d09989..cabb8b3c1 100644 --- a/tests/codice/l3/hi/direct_event/test_codice_hi_mass_correction_lookup_table.py +++ b/tests/codice/l3/hi/direct_event/test_codice_hi_mass_correction_lookup_table.py @@ -1,6 +1,5 @@ import unittest -import numpy as np from imap_l3_processing.codice.l3.hi.direct_event.codice_hi_mass_correction_lookup_table import \ CodiceHiMassCorrectionLookupTable diff --git a/tests/codice/l3/hi/test_models.py b/tests/codice/l3/hi/test_models.py index aa0f16ad2..02a285222 100644 --- a/tests/codice/l3/hi/test_models.py +++ b/tests/codice/l3/hi/test_models.py @@ -40,7 +40,7 @@ def test_codice_hi_l2_data_read_from_instrument_team_cdf(self, tempdir): # @formatter:off np.testing.assert_array_equal(l2_direct_event_data.epoch, cdf["epoch"]) np.testing.assert_array_equal(l2_direct_event_data.epoch_delta_plus, cdf["epoch_delta_plus"]) - np.testing.assert_array_equal(l2_direct_event_data.ssd_energy, cdf[f"ssd_energy"]) + np.testing.assert_array_equal(l2_direct_event_data.ssd_energy, cdf["ssd_energy"]) expected_ssd_id = cdf["ssd_id"][...] np.testing.assert_array_equal(l2_direct_event_data.ssd_id, expected_ssd_id) @@ -63,7 +63,7 @@ def test_codice_hi_l2_data_read_from_instrument_team_cdf(self, tempdir): np.testing.assert_array_equal(l2_direct_event_data.data_quality, cdf["data_quality"]) np.testing.assert_array_equal(l2_direct_event_data.multi_flag, cdf["multi_flag"]) - np.testing.assert_array_equal(l2_direct_event_data.type, cdf[f"type"]) + np.testing.assert_array_equal(l2_direct_event_data.type, cdf["type"]) expected_elevation_angle = cdf["elevation_angle"][...] expected_elevation_angle[expected_elevation_angle == -1e31] = np.nan diff --git a/tests/codice/l3/lo/science/test_codice_lo_calculations.py b/tests/codice/l3/lo/science/test_codice_lo_calculations.py index dd677731e..28d463f13 100644 --- a/tests/codice/l3/lo/science/test_codice_lo_calculations.py +++ b/tests/codice/l3/lo/science/test_codice_lo_calculations.py @@ -8,7 +8,7 @@ from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import EnergyLookup from imap_l3_processing.codice.l3.lo.direct_events.science.mass_coefficient_lookup import MassCoefficientLookup from imap_l3_processing.codice.l3.lo.direct_events.science.mass_species_bin_lookup import MassSpeciesBinLookup -from imap_l3_processing.codice.l3.lo.models import EnergyAndSpinAngle, CodiceLoDirectEventData +from imap_l3_processing.codice.l3.lo.models import CodiceLoDirectEventData from imap_l3_processing.codice.l3.lo.science.codice_lo_calculations import calculate_partial_densities, \ calculate_total_number_of_events, calculate_mass, calculate_mass_per_charge, \ rebin_to_counts_by_species_elevation_and_spin_sector, rebin_direct_events_by_energy_and_spin_sector, \ diff --git a/tests/codice/l3/lo/test_codice_lo_l3b_3d_distributions_dependencies.py b/tests/codice/l3/lo/test_codice_lo_l3b_3d_distributions_dependencies.py index 75529dd73..77fabb35e 100644 --- a/tests/codice/l3/lo/test_codice_lo_l3b_3d_distributions_dependencies.py +++ b/tests/codice/l3/lo/test_codice_lo_l3b_3d_distributions_dependencies.py @@ -28,7 +28,7 @@ def _test_fetch_dependencies(self, species: str, mock_3d_distribution_deps_from_ efficiency_factors_lut_name = f"imap_codice_{EFFICIENCY_FACTOR_LOOKUP_DESCRIPTOR}_20100105_v010.csv" energy_per_charge_lut_name = f"imap_codice_{ESA_TO_ENERGY_PER_CHARGE_LOOKUP_DESCRIPTOR}_20100105_v010.csv" - unused_science_input = ScienceInput(f"imap_codice_l2_lo-direct-events_20100105_v010.cdf") + unused_science_input = ScienceInput("imap_codice_l2_lo-direct-events_20100105_v010.cdf") processing_input_collection = ProcessingInputCollection(AncillaryInput(mass_species_lut_name), AncillaryInput(geometric_factors_lut_name), diff --git a/tests/codice/l3/lo/test_codice_lo_processor.py b/tests/codice/l3/lo/test_codice_lo_processor.py index 25cf7bd38..5fe7a4529 100644 --- a/tests/codice/l3/lo/test_codice_lo_processor.py +++ b/tests/codice/l3/lo/test_codice_lo_processor.py @@ -7,7 +7,6 @@ import numpy as np from imap_data_access.processing_input import ProcessingInputCollection -from imap_processing.spice.geometry import get_spacecraft_to_instrument_spin_phase_offset from imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies import \ CodiceLoL3b3dDistributionsDependencies @@ -96,7 +95,7 @@ def test_process_ratios(self, mock_spiceypy, mock_save_data, mock_process_l3a_ra @patch(f'{MODULE}.CodiceLoL3aRatiosDependencies.fetch_dependencies') @patch(f'{MODULE}.CodiceLoProcessor.process_l3a_charge_state_distributions') @patch(f'{MODULE}.save_data') - @patch(f'imap_l3_processing.processor.spiceypy') + @patch('imap_l3_processing.processor.spiceypy') def test_process_abundances(self, mock_spiceypy, mock_save_data, mock_process_l3a_abundances, mock_fetch_dependencies): input_collection = MagicMock() @@ -313,7 +312,7 @@ def test_process_abundances_uses_safe_divide(self): @patch(f'{MODULE}.CodiceLoL3aDirectEventsDependencies.fetch_dependencies') @patch(f'{MODULE}.CodiceLoProcessor.process_l3a_direct_event_data_product') @patch(f'{MODULE}.save_data') - @patch(f'imap_l3_processing.processor.spiceypy') + @patch('imap_l3_processing.processor.spiceypy') def test_process_direct_events(self, mock_spiceypy, mock_save_data, mock_process_direct_event, mock_fetch_dependencies): input_collection = MagicMock() diff --git a/tests/glows/l3bc/science/test_time_parsing.py b/tests/glows/l3bc/science/test_time_parsing.py index 8d3803f51..0a7d72226 100644 --- a/tests/glows/l3bc/science/test_time_parsing.py +++ b/tests/glows/l3bc/science/test_time_parsing.py @@ -18,7 +18,7 @@ def test_works_with_fast_parser(self): try: result_l3bc = time_from_yday_l3bc(time_rows) result_l3d = time_from_yday_l3d(time_rows) - except ValueError as e: + except ValueError: self.fail('Fast parser should not crash, check formatting of date string.') np.testing.assert_array_equal(result_l3bc, Time([ diff --git a/tests/glows/l3bc/test_dependency_validator.py b/tests/glows/l3bc/test_dependency_validator.py index d6c3cf2db..8d7a2fe39 100644 --- a/tests/glows/l3bc/test_dependency_validator.py +++ b/tests/glows/l3bc/test_dependency_validator.py @@ -4,7 +4,6 @@ from tempfile import TemporaryDirectory from unittest.mock import patch, Mock -from astropy.time import Time, TimeDelta from imap_l3_processing.glows.l3bc.dependency_validator import validate_omni2_dependency, \ validate_dependencies, validate_f107_fluxtable_dependency, validate_lyman_alpha_dependency diff --git a/tests/glows/l3bc/test_utils.py b/tests/glows/l3bc/test_utils.py index 306a3745f..332da0030 100644 --- a/tests/glows/l3bc/test_utils.py +++ b/tests/glows/l3bc/test_utils.py @@ -67,7 +67,7 @@ def test_get_repoint_date_range_handles_no_pointing(self): repointing_number = 5998 with self.assertRaises(ValueError) as err: _, _ = get_pointing_date_range(repointing_number) - self.assertEqual(str(err.exception), f"No pointing found for pointing: 5998") + self.assertEqual(str(err.exception), "No pointing found for pointing: 5998") def create_imap_data_access_json(file_path: str, data_level: str, start_date: str, diff --git a/tests/glows/l3d/test_glows_l3d_initializer.py b/tests/glows/l3d/test_glows_l3d_initializer.py index 232d52c04..c7bd77b72 100644 --- a/tests/glows/l3d/test_glows_l3d_initializer.py +++ b/tests/glows/l3d/test_glows_l3d_initializer.py @@ -1,6 +1,4 @@ import unittest -from itertools import product -from math import prod from unittest.mock import patch, call, Mock, sentinel from imap_data_access.file_validation import Version diff --git a/tests/glows/l3e/test_reprocess_info.py b/tests/glows/l3e/test_reprocess_info.py index 92bf4c174..9e3287825 100644 --- a/tests/glows/l3e/test_reprocess_info.py +++ b/tests/glows/l3e/test_reprocess_info.py @@ -10,7 +10,7 @@ GLOWS_L3E_LO_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, GLOWS_L3E_ULTRA_HF_DESCRIPTOR, GLOWS_L3B_DESCRIPTOR, \ GLOWS_L3C_DESCRIPTOR, GLOWS_L3D_DESCRIPTOR, GLOWS_REPROCESSING_DESCRIPTOR from imap_l3_processing.glows.l3e.reprocess_info import ReprocessInfo, ReprocessTargets, fetch_reprocess_info -from tests.test_helpers import get_test_data_folder, get_test_data_path, create_mock_query_results +from tests.test_helpers import get_test_data_folder, get_test_data_path class TestReprocessInfo(unittest.TestCase): diff --git a/tests/glows/test_glows_processor.py b/tests/glows/test_glows_processor.py index f524e15a1..2035c5016 100644 --- a/tests/glows/test_glows_processor.py +++ b/tests/glows/test_glows_processor.py @@ -1273,28 +1273,28 @@ def test_process_l3e_invoked_from_top_level_process(self, mock_process_l3d, mock ) expected_lo_files = [ - [f'imap_glows_l3e_survival-probability-lo_20250101-repoint02902_v001.cdf', sentinel.lo_dat_1], - [f'imap_glows_l3e_survival-probability-lo_20250101-repoint02905_v003.0013.cdf', sentinel.lo_dat_2], + ['imap_glows_l3e_survival-probability-lo_20250101-repoint02902_v001.cdf', sentinel.lo_dat_1], + ['imap_glows_l3e_survival-probability-lo_20250101-repoint02905_v003.0013.cdf', sentinel.lo_dat_2], ] mock_process_lo.side_effect = expected_lo_files expected_hi90_files = [ [ - f"imap_glows_l3e_survival-probability-hi-90_20250101-repoint02902_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20250101-repoint02902_v001.cdf", sentinel.hi90_dat_1, ], [ - f"imap_glows_l3e_survival-probability-hi-90_20250101-repoint02903_v001.0011.cdf", + "imap_glows_l3e_survival-probability-hi-90_20250101-repoint02903_v001.0011.cdf", sentinel.hi90_dat_2, ], ] expected_hi45_files = [ [ - f"imap_glows_l3e_survival-probability-hi-45_20250101-repoint02902_v001.cdf", + "imap_glows_l3e_survival-probability-hi-45_20250101-repoint02902_v001.cdf", sentinel.hi45_dat_1, ], [ - f"imap_glows_l3e_survival-probability-hi-45_20250101-repoint02904_v002.0012.cdf", + "imap_glows_l3e_survival-probability-hi-45_20250101-repoint02904_v002.0012.cdf", sentinel.hi45_dat_2, ], ] @@ -1307,11 +1307,11 @@ def test_process_l3e_invoked_from_top_level_process(self, mock_process_l3d, mock expected_ul_sf_files = [ [ - f"imap_glows_l3e_survival-probability-ul-sf_20250101-repoint02902_v001.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20250101-repoint02902_v001.cdf", sentinel.ul_sf_dat_1, ], [ - f"imap_glows_l3e_survival-probability-ul-sf_20250101-repoint02906_v004.0014.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20250101-repoint02906_v004.0014.cdf", sentinel.ul_sf_dat_2, ], ] @@ -1319,11 +1319,11 @@ def test_process_l3e_invoked_from_top_level_process(self, mock_process_l3d, mock expected_ul_hf_files = [ [ - f"imap_glows_l3e_survival-probability-ul-hf_20250101-repoint02902_v001.cdf", + "imap_glows_l3e_survival-probability-ul-hf_20250101-repoint02902_v001.cdf", sentinel.ul_hf_dat_1, ], [ - f"imap_glows_l3e_survival-probability-ul-hf_20250101-repoint02907_v005.0015.cdf", + "imap_glows_l3e_survival-probability-ul-hf_20250101-repoint02907_v005.0015.cdf", sentinel.ul_hf_dat_2, ], ] @@ -1332,25 +1332,25 @@ def test_process_l3e_invoked_from_top_level_process(self, mock_process_l3d, mock expected_products = [ sentinel.l3d_text_file_paths, sentinel.l3d_cdf_file_path, - f"imap_glows_l3e_survival-probability-lo_20250101-repoint02902_v001.cdf", + "imap_glows_l3e_survival-probability-lo_20250101-repoint02902_v001.cdf", sentinel.lo_dat_1, - f"imap_glows_l3e_survival-probability-hi-90_20250101-repoint02902_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20250101-repoint02902_v001.cdf", sentinel.hi90_dat_1, - f"imap_glows_l3e_survival-probability-hi-45_20250101-repoint02902_v001.cdf", + "imap_glows_l3e_survival-probability-hi-45_20250101-repoint02902_v001.cdf", sentinel.hi45_dat_1, - f"imap_glows_l3e_survival-probability-ul-sf_20250101-repoint02902_v001.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20250101-repoint02902_v001.cdf", sentinel.ul_sf_dat_1, - f"imap_glows_l3e_survival-probability-ul-hf_20250101-repoint02902_v001.cdf", + "imap_glows_l3e_survival-probability-ul-hf_20250101-repoint02902_v001.cdf", sentinel.ul_hf_dat_1, - f"imap_glows_l3e_survival-probability-hi-90_20250101-repoint02903_v001.0011.cdf", + "imap_glows_l3e_survival-probability-hi-90_20250101-repoint02903_v001.0011.cdf", sentinel.hi90_dat_2, - f"imap_glows_l3e_survival-probability-hi-45_20250101-repoint02904_v002.0012.cdf", + "imap_glows_l3e_survival-probability-hi-45_20250101-repoint02904_v002.0012.cdf", sentinel.hi45_dat_2, - f"imap_glows_l3e_survival-probability-lo_20250101-repoint02905_v003.0013.cdf", + "imap_glows_l3e_survival-probability-lo_20250101-repoint02905_v003.0013.cdf", sentinel.lo_dat_2, - f"imap_glows_l3e_survival-probability-ul-sf_20250101-repoint02906_v004.0014.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20250101-repoint02906_v004.0014.cdf", sentinel.ul_sf_dat_2, - f"imap_glows_l3e_survival-probability-ul-hf_20250101-repoint02907_v005.0015.cdf", + "imap_glows_l3e_survival-probability-ul-hf_20250101-repoint02907_v005.0015.cdf", sentinel.ul_hf_dat_2, ] @@ -1865,7 +1865,7 @@ def test_process_l3e_skips_repointing_on_exception(self, mock_determine_spacecra @patch("imap_l3_processing.glows.glows_processor.ZipFile") def test_archive_dependencies(self, mock_zip, mock_json): version_number = Version(1, 1) - expected_filepath = TEMP_CDF_FOLDER_PATH / f"imap_glows_l3b-archive_20250314_v001.zip" + expected_filepath = TEMP_CDF_FOLDER_PATH / "imap_glows_l3b-archive_20250314_v001.zip" expected_json_filename = "cr_to_process.json" l3bc_dependencies = GlowsL3BCDependencies( diff --git a/tests/hi/test_hi_combined_initializer.py b/tests/hi/test_hi_combined_initializer.py index 970f5d0bd..e7108d5ab 100644 --- a/tests/hi/test_hi_combined_initializer.py +++ b/tests/hi/test_hi_combined_initializer.py @@ -89,8 +89,8 @@ def test_get_maps_that_should_be_produced_new_major_version(self, mock_read_cdf_ *l3_sp_input_maps } - existing_file_without_major_number = f'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.cdf' - existing_file_with_lower_major_number = f'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.0001.cdf' + existing_file_without_major_number = 'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.cdf' + existing_file_with_lower_major_number = 'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.0001.cdf' for existing_file in [existing_file_without_major_number, existing_file_with_lower_major_number]: with self.subTest(existing_file=existing_file): mock_read_cdf_parents.reset_mock() @@ -98,24 +98,24 @@ def test_get_maps_that_should_be_produced_new_major_version(self, mock_read_cdf_ self.hi_combined_initializer_mock_query.side_effect = [ create_mock_query_results([ - f'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', - f'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', - f'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', - f'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', + 'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', + 'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', + 'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', + 'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', existing_file, ]), create_mock_query_results([]) ] - descriptor = f'hic-ena-h-hf-sp-full-hae-6deg-1yr' + descriptor = 'hic-ena-h-hf-sp-full-hae-6deg-1yr' expected_maps_to_produce = [ PossibleMapToProduce( input_files={ - f'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', - f'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', + 'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', + 'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', - f'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', - f'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', + 'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', + 'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', }, input_metadata=InputMetadata( instrument='hi', @@ -130,7 +130,7 @@ def test_get_maps_that_should_be_produced_new_major_version(self, mock_read_cdf_ initializer = HiCombinedInitializer() actual_maps_to_produce = initializer.get_maps_that_should_be_produced( - f'hic-ena-h-hf-sp-full-hae-6deg-1yr', 2) + 'hic-ena-h-hf-sp-full-hae-6deg-1yr', 2) mock_read_cdf_parents.assert_called_once_with(existing_file) self.hi_combined_initializer_mock_query.assert_has_calls([ @@ -351,7 +351,7 @@ def test_get_maps_that_should_be_produced_can_produce_multiple_maps(self, mock_r ] initializer = HiCombinedInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced(f'hic-ena-h-hf-nsp-full-hae-6deg-1yr', + actual_maps_to_produce = initializer.get_maps_that_should_be_produced('hic-ena-h-hf-nsp-full-hae-6deg-1yr', None) mock_read_cdf_parents.assert_not_called() diff --git a/tests/hi/test_hi_processor.py b/tests/hi/test_hi_processor.py index 1e9704703..341da49c4 100644 --- a/tests/hi/test_hi_processor.py +++ b/tests/hi/test_hi_processor.py @@ -449,7 +449,7 @@ def test_process_combined_sensor_map(self, mock_uncertainty_weighted_combination start_date=datetime.now(), end_date=datetime.now() + timedelta(days=1), version="", - descriptor=f"hic-ena-h-hf-sp-full-hae-4deg-6mo") + descriptor="hic-ena-h-hf-sp-full-hae-4deg-6mo") dependencies = ProcessingInputCollection(*[ScienceInput(map_name) for map_name in parent_map_names]) data_product = HiProcessor(dependencies, input_metadata).process() diff --git a/tests/hi/test_hi_sp_initializer.py b/tests/hi/test_hi_sp_initializer.py index 304d435ec..c9b98a140 100644 --- a/tests/hi/test_hi_sp_initializer.py +++ b/tests/hi/test_hi_sp_initializer.py @@ -339,13 +339,13 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): actual_possible_maps = initializer.get_maps_that_should_be_produced(descriptor, None) mock_read_cdf_parents.assert_has_calls([ - call(f'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf'), - call(f'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf'), - call(f'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf'), - call(f'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20101001_v001.cdf'), + call('imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf'), + call('imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf'), + call('imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf'), + call('imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20101001_v001.cdf'), - call(f'imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf'), - call(f'imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100401_v001.cdf'), + call('imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf'), + call('imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100401_v001.cdf'), ]) self.assertEqual(expected_possible_maps, actual_possible_maps) @@ -412,9 +412,9 @@ def test_get_maps_that_should_be_produced_remakes_when_major_version_changes(sel actual_possible_maps = initializer.get_maps_that_should_be_produced(descriptor, major_version) mock_read_cdf_parents.assert_has_calls([ - call(f'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf'), + call('imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf'), - call(f'imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf'), + call('imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf'), ]) self.assertEqual(expected_possible_maps, actual_possible_maps) @@ -487,7 +487,7 @@ def fake_read_cdf_parents(file_name: str) -> set[str]: f'imap_hi_l1c_{sensor}sensor-pset_20100101-repoint00001_v001.cdf', f'imap_hi_l1c_{sensor}sensor-pset_20100102-repoint00002_v001.cdf', f'imap_hi_l1c_{sensor}sensor-pset_20100103-repoint00003_v001.cdf', - f'imap_science_0001.tf', + 'imap_science_0001.tf', }, f'imap_hi_l2_h{sensor}-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf': { f'imap_hi_l1c_{sensor}sensor-pset_20100401-repoint00101_v001.cdf', @@ -512,7 +512,7 @@ def fake_read_cdf_parents(file_name: str) -> set[str]: f'imap_glows_l3e_survival-probability-hi-{sensor}_20100101-repoint00001_v001.cdf', f'imap_glows_l3e_survival-probability-hi-{sensor}_20100102-repoint00002_v001.cdf', f'imap_glows_l3e_survival-probability-hi-{sensor}_20100103-repoint00003_v001.cdf', - f'imap_science_0001.tf', + 'imap_science_0001.tf', }, f'imap_hi_l3_h{sensor}-ena-h-sf-sp-anti-hae-4deg-3mo_20100401_v001.cdf': { f'imap_hi_l2_h{sensor}-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf', @@ -522,7 +522,7 @@ def fake_read_cdf_parents(file_name: str) -> set[str]: f'imap_glows_l3e_survival-probability-hi-{sensor}_20100401-repoint00101_v001.cdf', f'imap_glows_l3e_survival-probability-hi-{sensor}_20100402-repoint00102_v001.cdf', f'imap_glows_l3e_survival-probability-hi-{sensor}_20100403-repoint00103_v001.cdf', - f'imap_science_0001.tf', + 'imap_science_0001.tf', }, } return mapping[file_name] diff --git a/tests/hit/l3/pha/science/test_calculate_pha.py b/tests/hit/l3/pha/science/test_calculate_pha.py index 9f2394291..ec7b3105a 100644 --- a/tests/hit/l3/pha/science/test_calculate_pha.py +++ b/tests/hit/l3/pha/science/test_calculate_pha.py @@ -1,6 +1,5 @@ import math import unittest -from collections import defaultdict from unittest.mock import patch, Mock, sentinel import numpy as np diff --git a/tests/hit/l3/test_hit_processor.py b/tests/hit/l3/test_hit_processor.py index 5d307f5e0..7603f6180 100644 --- a/tests/hit/l3/test_hit_processor.py +++ b/tests/hit/l3/test_hit_processor.py @@ -675,6 +675,6 @@ def test_raise_error_if_descriptor_doesnt_match(self): processor.process() self.assertEqual(e.exception.args[0], - f"Don't know how to generate 'spectral-index' /n Known HIT l3 data products: 'macropixel', 'direct-events'.") + "Don't know how to generate 'spectral-index' /n Known HIT l3 data products: 'macropixel', 'direct-events'.") T = TypeVar("T") diff --git a/tests/hit/l3/test_l3_sectored_dependencies.py b/tests/hit/l3/test_l3_sectored_dependencies.py index 479488e41..6790e2932 100644 --- a/tests/hit/l3/test_l3_sectored_dependencies.py +++ b/tests/hit/l3/test_l3_sectored_dependencies.py @@ -14,13 +14,13 @@ class TestHITL3SectoredDependencies(TestCase): @patch('imap_l3_processing.hit.l3.hit_l3_sectored_dependencies.imap_data_access.download') def test_fetch_dependencies(self, mock_download, mock_read_hit_data, mock_read_mag_data): hit_l2_data_dependency = ScienceInput(f"imap_hit_l2_{HIT_L2_DESCRIPTOR}_20240908_v001.cdf") - mag_data_dependency = ScienceInput(f"imap_mag_l1d_norm-dsrf_20240906_v001.cdf") + mag_data_dependency = ScienceInput("imap_mag_l1d_norm-dsrf_20240906_v001.cdf") hit_data_dir = imap_data_access.config["DATA_DIR"] / 'imap' / 'hit' / 'l2' / '2024' / '09' mag_data_dir = imap_data_access.config["DATA_DIR"] / 'imap' / 'mag' / 'l1d' / '2024' / '09' expected_file_paths = [hit_data_dir / f"imap_hit_l2_{HIT_L2_DESCRIPTOR}_20240908_v001.cdf", - mag_data_dir / f"imap_mag_l1d_norm-dsrf_20240906_v001.cdf"] + mag_data_dir / "imap_mag_l1d_norm-dsrf_20240906_v001.cdf"] processing_input_collection = ProcessingInputCollection(hit_l2_data_dependency, mag_data_dependency) hit_data_path = Path("hit") @@ -49,14 +49,14 @@ def test_fetch_dependencies(self, mock_download, mock_read_hit_data, mock_read_m @patch('imap_l3_processing.hit.l3.hit_l3_sectored_dependencies.imap_data_access.download') def test_fetch_dependencies_uses_mag_l2_if_available(self, mock_download, mock_read_hit_data, mock_read_mag_data): hit_l2_data_dependency = ScienceInput(f"imap_hit_l2_{HIT_L2_DESCRIPTOR}_20240908_v001.cdf") - mag_data_dependency = ScienceInput(f"imap_mag_l1d_norm-dsrf_20240906_v001.cdf") - mag_l2_data_dependency = ScienceInput(f"imap_mag_l2_norm-dsrf_20240906_v001.cdf") + mag_data_dependency = ScienceInput("imap_mag_l1d_norm-dsrf_20240906_v001.cdf") + mag_l2_data_dependency = ScienceInput("imap_mag_l2_norm-dsrf_20240906_v001.cdf") hit_data_dir = imap_data_access.config["DATA_DIR"] / 'imap' / 'hit' / 'l2' / '2024' / '09' mag_data_dir = imap_data_access.config["DATA_DIR"] / 'imap' / 'mag' / 'l2' / '2024' / '09' expected_file_paths = [hit_data_dir / f"imap_hit_l2_{HIT_L2_DESCRIPTOR}_20240908_v001.cdf", - mag_data_dir / f"imap_mag_l2_norm-dsrf_20240906_v001.cdf"] + mag_data_dir / "imap_mag_l2_norm-dsrf_20240906_v001.cdf"] processing_input_collection = ProcessingInputCollection(hit_l2_data_dependency, mag_data_dependency, mag_l2_data_dependency) diff --git a/tests/integration/integration_test_helpers.py b/tests/integration/integration_test_helpers.py index 21c3231d5..9312a9579 100644 --- a/tests/integration/integration_test_helpers.py +++ b/tests/integration/integration_test_helpers.py @@ -14,7 +14,6 @@ generate_imap_file_path, ScienceFilePath, SPICEFilePath, - Version, ) from requests import Response diff --git a/tests/lo/l3/test_lo_combined_dependencies.py b/tests/lo/l3/test_lo_combined_dependencies.py index 139acf89b..7ffe25b7c 100644 --- a/tests/lo/l3/test_lo_combined_dependencies.py +++ b/tests/lo/l3/test_lo_combined_dependencies.py @@ -4,7 +4,6 @@ from imap_data_access import ScienceInput, ProcessingInputCollection from imap_l3_processing.lo.l3.lo_combined_dependencies import LoCombinedDependencies -from imap_l3_processing.maps.map_models import RectangularIntensityMapData class TestLoCombinedDependencies(unittest.TestCase): diff --git a/tests/maps/test_hilo_l3_survival_dependencies.py b/tests/maps/test_hilo_l3_survival_dependencies.py index 8ca95ed56..220547c64 100644 --- a/tests/maps/test_hilo_l3_survival_dependencies.py +++ b/tests/maps/test_hilo_l3_survival_dependencies.py @@ -30,7 +30,7 @@ def test_fetch_dependencies(self, mock_read_from_path, mock_read_l1c, mock_read_ f"imap_glows_l3e_survival-probability-{instrument.value}_20201002_v002.cdf", f"imap_glows_l3e_survival-probability-{instrument.value}_20201003_v003.cdf"] - l2_map_descriptor = f"ena-h-sf-nsp-ram-hae-6deg-1yr" + l2_map_descriptor = "ena-h-sf-nsp-ram-hae-6deg-1yr" l2_filename = f"imap_{instrument.value}_l2_{l2_map_descriptor}_20250415_v001.cdf" dependencies = ProcessingInputCollection( diff --git a/tests/maps/test_map_models.py b/tests/maps/test_map_models.py index a36127f23..2298dc033 100644 --- a/tests/maps/test_map_models.py +++ b/tests/maps/test_map_models.py @@ -42,7 +42,6 @@ from tests.test_helpers import ( get_test_data_folder, get_integration_test_data_path, - NumpyArrayMatcher, ) diff --git a/tests/maps/test_spectral_fit.py b/tests/maps/test_spectral_fit.py index 269ad4ba3..97ad5b804 100644 --- a/tests/maps/test_spectral_fit.py +++ b/tests/maps/test_spectral_fit.py @@ -6,7 +6,7 @@ import numpy as np from scipy.stats import linregress -from imap_l3_processing.maps.map_models import IntensityMapData, RectangularIntensityMapData, QUALITY_FLAGS_VAR_NAME +from imap_l3_processing.maps.map_models import IntensityMapData, RectangularIntensityMapData from imap_l3_processing.maps.mpfit import mpfit from imap_l3_processing.maps.quality_flags import MapL3Flags from imap_l3_processing.maps.spectral_fit import power_law, fit_arrays_to_power_law, fit_spectral_index_map, \ @@ -132,7 +132,7 @@ def test_spectral_fit_map(self): expected_energy_midpoint = np.sqrt(0.5 * 10.5) expected_energy_minus = expected_energy_midpoint - 0.5 expected_energy_plus = 10.5 - expected_energy_midpoint - expected_energy_label = f"0.5 - 10.5" + expected_energy_label = "0.5 - 10.5" np.testing.assert_array_equal(spectral_intensity_map.ena_spectral_index, np.array(true_gamma).reshape(1, 1, 1, 1)) @@ -283,7 +283,7 @@ def test_spectral_fit_map_negative_gammas(self): expected_energy_midpoint = np.sqrt(0.5 * 10.5) expected_energy_minus = expected_energy_midpoint - 0.5 expected_energy_plus = 10.5 - expected_energy_midpoint - expected_energy_label = f"0.5 - 10.5" + expected_energy_label = "0.5 - 10.5" np.testing.assert_array_equal(spectral_intensity_map.ena_spectral_index_stat_uncert, np.full(shape=(1, 1, 1, 1), fill_value=np.nan)) np.testing.assert_array_equal(spectral_intensity_map.ena_spectral_index_scalar_coefficient, diff --git a/tests/swapi/l3a/science/solar_wind/alpha/test_fit_solar_wind_alpha_model.py b/tests/swapi/l3a/science/solar_wind/alpha/test_fit_solar_wind_alpha_model.py index 03511f3b3..a8ff4bf46 100644 --- a/tests/swapi/l3a/science/solar_wind/alpha/test_fit_solar_wind_alpha_model.py +++ b/tests/swapi/l3a/science/solar_wind/alpha/test_fit_solar_wind_alpha_model.py @@ -1,13 +1,11 @@ import unittest from unittest.mock import MagicMock, patch -import numba import numpy as np from uncertainties import ufloat from imap_l3_processing.constants import ( ALPHA_MASS_PER_CHARGE_M_P_PER_E, - ALPHA_PARTICLE_CHARGE_COULOMBS, ALPHA_PARTICLE_MASS_KG, PROTON_CHARGE_COULOMBS, PROTON_MASS_KG, diff --git a/tests/swapi/l3a/science/solar_wind/test_integration_limits.py b/tests/swapi/l3a/science/solar_wind/test_integration_limits.py index b043bd6cf..c7f8f4145 100644 --- a/tests/swapi/l3a/science/solar_wind/test_integration_limits.py +++ b/tests/swapi/l3a/science/solar_wind/test_integration_limits.py @@ -15,7 +15,6 @@ speed_window_misses_passband, ) from imap_l3_processing.swapi.l3a.science.solar_wind.params import ( - SolarWindParams, bulk_speed, thermal_speed, thermal_speed_to_temperature, diff --git a/tests/swapi/response/test_efficiency_calibration_table.py b/tests/swapi/response/test_efficiency_calibration_table.py index 283ce7389..536803c05 100644 --- a/tests/swapi/response/test_efficiency_calibration_table.py +++ b/tests/swapi/response/test_efficiency_calibration_table.py @@ -1,7 +1,6 @@ import unittest from datetime import datetime -from scipy.constants import milli from spacepy import pycdf from imap_l3_processing.swapi.response.efficiency_calibration_table import EfficiencyCalibrationTable diff --git a/tests/swe/l3/science/test_moments_calculation.py b/tests/swe/l3/science/test_moments_calculation.py index a0202c42a..a7dc65d4b 100644 --- a/tests/swe/l3/science/test_moments_calculation.py +++ b/tests/swe/l3/science/test_moments_calculation.py @@ -960,8 +960,8 @@ def test_check_temperature_outlier_flag(self): for i in np.arange(len(tof_flags)): self.assertEqual(expected_values[i], tof_flags[i]) - @patch(f"spiceypy.datetime2et") - @patch(f"spiceypy.pxform") + @patch("spiceypy.datetime2et") + @patch("spiceypy.pxform") @patch(f"{MODULE}.PredictedEphemerisTracker") def test_rotate_rtn_vectors_to_dps(self, mock_predicted_ephemeris_tracker_class, mock_pxform, mock_datetime2et): pred_tracker_1 = create_autospec(PredictedEphemerisTracker, used_predict=False) diff --git a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/archive/map_SCFrame.py b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/archive/map_SCFrame.py index f734506f8..87ff94626 100755 --- a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/archive/map_SCFrame.py +++ b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/archive/map_SCFrame.py @@ -3,10 +3,6 @@ #!/usr/bin/python import numpy as np import argparse -from scipy.optimize import curve_fit -import random -import datetime -from spacepy.pycdf import CDF import pandas as pd import os from pathlib import Path diff --git a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/csv_to_soc.py b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/csv_to_soc.py index 38aed9b3c..319521947 100755 --- a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/csv_to_soc.py +++ b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/csv_to_soc.py @@ -7,7 +7,6 @@ """ import os -import glob from pathlib import Path import pandas as pd import numpy as np diff --git a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/l1b_to_spin.py b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/l1b_to_spin.py index 4e75345fe..2060a937b 100755 --- a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/l1b_to_spin.py +++ b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/l1b_to_spin.py @@ -11,10 +11,11 @@ import numpy as np import os import sys -import argparse from datetime import datetime from pathlib import Path -import pandas as pd, re, shutil +import pandas as pd +import re +import shutil def radec2cart(ra,th): diff --git a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/map_SCFrame.py b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/map_SCFrame.py index 91f7f8853..922a1983b 100755 --- a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/map_SCFrame.py +++ b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/map_SCFrame.py @@ -7,11 +7,6 @@ """ import numpy as np -import argparse -from scipy.optimize import curve_fit -import random -import datetime -from spacepy.pycdf import CDF import pandas as pd import os from pathlib import Path diff --git a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/new_patch_arc2_linear.py b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/new_patch_arc2_linear.py index 9f495a89e..24a93d562 100755 --- a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/new_patch_arc2_linear.py +++ b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/new_patch_arc2_linear.py @@ -3,7 +3,6 @@ import matplotlib.pyplot as plt import cartopy.crs as ccrs from scipy.spatial.transform import Rotation as R -from matplotlib.colors import LogNorm PI = np.pi diff --git a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/plot_map_DN.py b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/plot_map_DN.py index dacb5f0f6..cb2a0f01a 100755 --- a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/plot_map_DN.py +++ b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/plot_map_DN.py @@ -7,12 +7,8 @@ """ import numpy as np -import pandas as pd import os -import matplotlib.pyplot as plt -import cartopy.crs as ccrs from scipy.spatial.transform import Rotation as R -from matplotlib.colors import LogNorm import new_patch_arc2 as patch import new_patch_arc2_linear as pchlin diff --git a/tests/test_imap_l3_data_processor.py b/tests/test_imap_l3_data_processor.py index 4b8f08704..1e529e053 100644 --- a/tests/test_imap_l3_data_processor.py +++ b/tests/test_imap_l3_data_processor.py @@ -1,4 +1,3 @@ -import itertools import json from datetime import datetime from pathlib import Path diff --git a/tests/ultra/test_models.py b/tests/ultra/test_models.py index 0d8785f36..715067635 100644 --- a/tests/ultra/test_models.py +++ b/tests/ultra/test_models.py @@ -1,4 +1,3 @@ -import traceback from datetime import datetime import numpy as np diff --git a/tests/ultra/test_ultra_combined_nsp_initializer.py b/tests/ultra/test_ultra_combined_nsp_initializer.py index 729fa0c70..b0adbfd6d 100644 --- a/tests/ultra/test_ultra_combined_nsp_initializer.py +++ b/tests/ultra/test_ultra_combined_nsp_initializer.py @@ -80,17 +80,17 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): for major_version, expected_version in cases: with self.subTest(major_version=major_version): self.mock_query.side_effect = ImapQueryPatcher([ - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v002.cdf', + 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v002.cdf', - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', + 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', + 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf', + 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf', f'imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_{Version(major_version, 1)}.cdf', f'imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_{Version(major_version, 1)}.cdf', @@ -99,60 +99,60 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): mock_read_cdf_parents.side_effect = [ [ - f"imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", - f"imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", - f"imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf", ], [ - f"imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", - f"imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", - f"imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf" + "imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf" ], [ - f'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf' + 'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf' ], [ - f'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v002.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v002.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v002.cdf' + 'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v002.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v002.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v002.cdf' ], [ - f'imap_ultra_l1c_45sensor-spacecraftpset_20100701-repoint00201_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100702-repoint00202_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100703-repoint00203_v001.cdf' + 'imap_ultra_l1c_45sensor-spacecraftpset_20100701-repoint00201_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100702-repoint00202_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100703-repoint00203_v001.cdf' ], [ - f'imap_ultra_l1c_90sensor-spacecraftpset_20100701-repoint00201_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100702-repoint00202_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100703-repoint00203_v001.cdf' + 'imap_ultra_l1c_90sensor-spacecraftpset_20100701-repoint00201_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100702-repoint00202_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100703-repoint00203_v001.cdf' ], [ - f"imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", - f"imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", - f"imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf", - f"imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", - f"imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", - f"imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf", - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + "imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf", + 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv', ], [ - f'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v001.cdf', + 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv', ] ] initializer = UltraCombinedNSPInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced(f"ulc-ena-h-sf-nsp-full-hae-4deg-3mo", + actual_maps_to_produce = initializer.get_maps_that_should_be_produced("ulc-ena-h-sf-nsp-full-hae-4deg-3mo", major_version) self.mock_query.assert_has_calls([ @@ -163,12 +163,12 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): ]) mock_read_cdf_parents.assert_has_calls([ - call(f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), - call(f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), - call(f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), - call(f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v002.cdf'), - call(f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), - call(f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), + call('imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), + call('imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), + call('imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), + call('imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v002.cdf'), + call('imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), + call('imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), call(f'imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_{Version(major_version, 1)}.cdf'), call(f'imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_{Version(major_version, 1)}.cdf'), @@ -186,15 +186,15 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): descriptor=descriptor ), input_files={ - f'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v002.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v002.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v002.cdf', - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v002.cdf', - f'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' + 'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v002.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v002.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v002.cdf', + 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v002.cdf', + 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' } ) expected_possible_map_to_produce_with_no_previous_version = PossibleMapToProduce( @@ -207,15 +207,15 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): descriptor=descriptor ), input_files={ - f'imap_ultra_l1c_45sensor-spacecraftpset_20100701-repoint00201_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100702-repoint00202_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100703-repoint00203_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100701-repoint00201_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100702-repoint00202_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100703-repoint00203_v001.cdf', - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', - f'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' + 'imap_ultra_l1c_45sensor-spacecraftpset_20100701-repoint00201_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100702-repoint00202_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100703-repoint00203_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100701-repoint00201_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100702-repoint00202_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100703-repoint00203_v001.cdf', + 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', + 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', + 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' } ) expected_maps_to_produce = [expected_possible_map_to_produce_with_previous_version, @@ -225,5 +225,5 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): def test_initializer_raises_when_given_invalid_map_descriptor(self): initializer = UltraCombinedNSPInitializer() with self.assertRaises(ValueError) as e: - initializer.get_maps_that_should_be_produced(f"ulc-invalid", None) + initializer.get_maps_that_should_be_produced("ulc-invalid", None) self.assertEqual(e.exception.args, ("Invalid map descriptor: ulc-invalid",)) \ No newline at end of file diff --git a/tests/ultra/test_ultra_l3_survival_probabillity.py b/tests/ultra/test_ultra_l3_survival_probabillity.py index 128ea5ba5..12e6dbead 100644 --- a/tests/ultra/test_ultra_l3_survival_probabillity.py +++ b/tests/ultra/test_ultra_l3_survival_probabillity.py @@ -5,7 +5,6 @@ import numpy as np from astropy_healpix import HEALPix from imap_processing.ena_maps.ena_maps import UltraPointingSet, HealpixSkyMap -from imap_processing.ena_maps.utils.coordinates import CoordNames from imap_processing.spice import geometry from imap_processing.ultra.constants import UltraConstants @@ -15,7 +14,6 @@ SECONDS_PER_DAY, ) from imap_l3_processing.glows.quality_flags import GlowsL3Flags -from imap_l3_processing.maps.quality_flags import MapL3Flags from imap_l3_processing.ultra.models import UltraL1CPSet, UltraGlowsL3eData from imap_l3_processing.ultra.science.ultra_survival_probability import UltraSurvivalProbability, \ UltraSurvivalProbabilitySkyMap diff --git a/tests/ultra/test_ultra_processor.py b/tests/ultra/test_ultra_processor.py index c6f1999ae..b21bb1e51 100644 --- a/tests/ultra/test_ultra_processor.py +++ b/tests/ultra/test_ultra_processor.py @@ -734,7 +734,7 @@ def test_defaults_to_ECLIPJ2000_spice_frame(self, mock_fetch_dependencies, start_date=datetime.now(), end_date=datetime.now() + timedelta(days=1), version="", - descriptor=f"u90-ena-h-sf-sp-full-hae-2deg-6mo" + descriptor="u90-ena-h-sf-sp-full-hae-2deg-6mo" ) mock_healpix_skymap = Mock() @@ -836,7 +836,7 @@ def test_process_spectral_index_with_custom_energy_bin_range(self, start_date=datetime.now(), end_date=datetime.now() + timedelta(days=1), version="v000", - descriptor=f"u90-spx0510-h-sf-sp-full-hae-4deg-6mo") + descriptor="u90-spx0510-h-sf-sp-full-hae-4deg-6mo") dependencies = mock_fetch_dependencies.return_value processor = UltraProcessor(input_deps, input_metadata) @@ -858,7 +858,7 @@ def test_process_raises_exception_when_generating_a_healpix_map(self): start_date=datetime.now(), end_date=datetime.now() + timedelta(days=1), version="v000", - descriptor=f"u90-spx-h-sf-sp-full-hae-nside8-6mo") + descriptor="u90-spx-h-sf-sp-full-hae-nside8-6mo") processor = UltraProcessor(ProcessingInputCollection(), input_metadata) @@ -873,7 +873,7 @@ def test_process_spectral_index_validating_output_values(self, mock_fetch_depend start_date=datetime.now(), end_date=datetime.now() + timedelta(days=1), version="v000", - descriptor=f"u90-spx-h-sf-sp-full-hae-6deg-6mo") + descriptor="u90-spx-h-sf-sp-full-hae-6deg-6mo") input_map_path = get_test_data_path('ultra/fake_ultra_map_data_with_breakpoint_at_15keV.cdf') fit_energy_ranges_path = get_test_data_path('ultra/imap_ultra_ulc-spx-energy-ranges_20250407_v000.dat') dependencies = UltraL3SpectralIndexDependencies.from_file_paths(input_map_path, fit_energy_ranges_path) diff --git a/tests/ultra/test_ultra_sp_initializer.py b/tests/ultra/test_ultra_sp_initializer.py index 1f7e5e69e..106008cc3 100644 --- a/tests/ultra/test_ultra_sp_initializer.py +++ b/tests/ultra/test_ultra_sp_initializer.py @@ -145,7 +145,7 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): f'imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100402-repoint00102_v001.cdf', f'imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100403-repoint00103_v001.cdf', f'imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' + 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' } ) self.assertIn(expected_possible_map_to_produce, actual_maps_to_produce) @@ -165,96 +165,96 @@ def test_get_maps_that_should_be_produced_lists_combined_maps(self, mock_read_cd 'imap_glows_l3e_survival-probability-ul-sf_20100703-repoint00201_v001.cdf', - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v000.cdf', - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf', + 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v000.cdf', + 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', + 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v000.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf', + 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v000.cdf', + 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', + 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf', - f'imap_ultra_l3_ulc-ena-h-sf-sp-full-hae-4deg-3mo_20100101_v001.cdf', - f'imap_ultra_l3_ulc-ena-h-sf-sp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l3_ulc-ena-h-sf-sp-full-hae-4deg-3mo_20100101_v001.cdf', + 'imap_ultra_l3_ulc-ena-h-sf-sp-full-hae-4deg-3mo_20100401_v001.cdf', 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' ]) mock_read_cdf_parents.side_effect = [ [ - f"imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", - f"imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", - f"imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf" + "imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf" ], [ - f"imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", - f"imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", - f"imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf" + "imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf" ], [ - f'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf' + 'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf' ], [ - f'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v001.cdf' + 'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v001.cdf' ], [ - f'imap_ultra_l1c_45sensor-spacecraftpset_20100701-repoint00201_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100702-repoint00202_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100703-repoint00203_v001.cdf' + 'imap_ultra_l1c_45sensor-spacecraftpset_20100701-repoint00201_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100702-repoint00202_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100703-repoint00203_v001.cdf' ], [ - f'imap_ultra_l1c_90sensor-spacecraftpset_20100701-repoint00201_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100702-repoint00202_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100703-repoint00203_v001.cdf' + 'imap_ultra_l1c_90sensor-spacecraftpset_20100701-repoint00201_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100702-repoint00202_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100703-repoint00203_v001.cdf' ], [ - f'imap_ultra_l1c_45sensor-spacecraftpset_20101001-repoint00301_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20101002-repoint00302_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20101003-repoint00303_v001.cdf' + 'imap_ultra_l1c_45sensor-spacecraftpset_20101001-repoint00301_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20101002-repoint00302_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20101003-repoint00303_v001.cdf' ], [ - f'imap_ultra_l1c_90sensor-spacecraftpset_20101001-repoint00301_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20101002-repoint00302_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20101003-repoint00303_v001.cdf' + 'imap_ultra_l1c_90sensor-spacecraftpset_20101001-repoint00301_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20101002-repoint00302_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20101003-repoint00303_v001.cdf' ], [ 'imap_glows_l3e_survival-probability-ul-sf_20100101-repoint00001_v001.cdf', 'imap_glows_l3e_survival-probability-ul-sf_20100102-repoint00002_v001.cdf', 'imap_glows_l3e_survival-probability-ul-sf_20100103-repoint00003_v001.cdf', - f"imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", - f"imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", - f"imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf", - f"imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", - f"imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", - f"imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf", - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + "imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf", + 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv', ], [ 'imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v001.cdf', 'imap_glows_l3e_survival-probability-ul-sf_20100402-repoint00102_v001.cdf', 'imap_glows_l3e_survival-probability-ul-sf_20100403-repoint00103_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v001.cdf', + 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv', ] ] initializer = UltraSPInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced(f"ulc-ena-h-sf-sp-full-hae-4deg-3mo", + actual_maps_to_produce = initializer.get_maps_that_should_be_produced("ulc-ena-h-sf-sp-full-hae-4deg-3mo", None) self.mock_query.assert_has_calls([ @@ -267,19 +267,19 @@ def test_get_maps_that_should_be_produced_lists_combined_maps(self, mock_read_cd ]) mock_read_cdf_parents.assert_has_calls([ - call(f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), - call(f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), - call(f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), - call(f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), - call(f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), - call(f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), - call(f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf'), - call(f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf'), - - call(f'imap_ultra_l3_ulc-ena-h-sf-sp-full-hae-4deg-3mo_20100101_v001.cdf'), - call(f'imap_ultra_l3_ulc-ena-h-sf-sp-full-hae-4deg-3mo_20100401_v001.cdf'), + call('imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), + call('imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), + call('imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), + call('imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), + call('imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), + call('imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), + call('imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf'), + call('imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf'), + + call('imap_ultra_l3_ulc-ena-h-sf-sp-full-hae-4deg-3mo_20100101_v001.cdf'), + call('imap_ultra_l3_ulc-ena-h-sf-sp-full-hae-4deg-3mo_20100401_v001.cdf'), ]) - descriptor = f"ulc-ena-h-sf-sp-full-hae-4deg-3mo" + descriptor = "ulc-ena-h-sf-sp-full-hae-4deg-3mo" expected_possible_map_to_produce = PossibleMapToProduce( input_metadata=InputMetadata( @@ -294,15 +294,15 @@ def test_get_maps_that_should_be_produced_lists_combined_maps(self, mock_read_cd 'imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v002.cdf', 'imap_glows_l3e_survival-probability-ul-sf_20100402-repoint00102_v002.cdf', 'imap_glows_l3e_survival-probability-ul-sf_20100403-repoint00103_v002.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - f'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' + 'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', + 'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v001.cdf', + 'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v001.cdf', + 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' } ) self.assertIn(expected_possible_map_to_produce, actual_maps_to_produce) @@ -322,96 +322,96 @@ def test_get_maps_that_should_be_produced_lists_cg_corrected_maps(self, mock_rea 'imap_glows_l3e_survival-probability-ul-hf_20100703-repoint00201_v001.cdf', - f'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - f'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v000.cdf', - f'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', - f'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20101001_v001.cdf', + 'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + 'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v000.cdf', + 'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', + 'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20101001_v001.cdf', - f'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - f'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v000.cdf', - f'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', - f'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20101001_v001.cdf', + 'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + 'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v000.cdf', + 'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', + 'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20101001_v001.cdf', - f'imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-4deg-3mo_20100101_v001.cdf', - f'imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-4deg-3mo_20100101_v001.cdf', + 'imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-4deg-3mo_20100401_v001.cdf', 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' ]) mock_read_cdf_parents.side_effect = [ [ - f"imap_ultra_l1c_45sensor-heliopset_20100101-repoint00001_v001.cdf", - f"imap_ultra_l1c_45sensor-heliopset_20100102-repoint00002_v001.cdf", - f"imap_ultra_l1c_45sensor-heliopset_20100103-repoint00003_v001.cdf" + "imap_ultra_l1c_45sensor-heliopset_20100101-repoint00001_v001.cdf", + "imap_ultra_l1c_45sensor-heliopset_20100102-repoint00002_v001.cdf", + "imap_ultra_l1c_45sensor-heliopset_20100103-repoint00003_v001.cdf" ], [ - f"imap_ultra_l1c_90sensor-heliopset_20100101-repoint00001_v001.cdf", - f"imap_ultra_l1c_90sensor-heliopset_20100102-repoint00002_v001.cdf", - f"imap_ultra_l1c_90sensor-heliopset_20100103-repoint00003_v001.cdf" + "imap_ultra_l1c_90sensor-heliopset_20100101-repoint00001_v001.cdf", + "imap_ultra_l1c_90sensor-heliopset_20100102-repoint00002_v001.cdf", + "imap_ultra_l1c_90sensor-heliopset_20100103-repoint00003_v001.cdf" ], [ - f'imap_ultra_l1c_45sensor-heliopset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_45sensor-heliopset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_45sensor-heliopset_20100403-repoint00103_v001.cdf' + 'imap_ultra_l1c_45sensor-heliopset_20100401-repoint00101_v001.cdf', + 'imap_ultra_l1c_45sensor-heliopset_20100402-repoint00102_v001.cdf', + 'imap_ultra_l1c_45sensor-heliopset_20100403-repoint00103_v001.cdf' ], [ - f'imap_ultra_l1c_90sensor-heliopset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_90sensor-heliopset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_90sensor-heliopset_20100403-repoint00103_v001.cdf' + 'imap_ultra_l1c_90sensor-heliopset_20100401-repoint00101_v001.cdf', + 'imap_ultra_l1c_90sensor-heliopset_20100402-repoint00102_v001.cdf', + 'imap_ultra_l1c_90sensor-heliopset_20100403-repoint00103_v001.cdf' ], [ - f'imap_ultra_l1c_45sensor-heliopset_20100701-repoint00201_v001.cdf', - f'imap_ultra_l1c_45sensor-heliopset_20100702-repoint00202_v001.cdf', - f'imap_ultra_l1c_45sensor-heliopset_20100703-repoint00203_v001.cdf' + 'imap_ultra_l1c_45sensor-heliopset_20100701-repoint00201_v001.cdf', + 'imap_ultra_l1c_45sensor-heliopset_20100702-repoint00202_v001.cdf', + 'imap_ultra_l1c_45sensor-heliopset_20100703-repoint00203_v001.cdf' ], [ - f'imap_ultra_l1c_90sensor-heliopset_20100701-repoint00201_v001.cdf', - f'imap_ultra_l1c_90sensor-heliopset_20100702-repoint00202_v001.cdf', - f'imap_ultra_l1c_90sensor-heliopset_20100703-repoint00203_v001.cdf' + 'imap_ultra_l1c_90sensor-heliopset_20100701-repoint00201_v001.cdf', + 'imap_ultra_l1c_90sensor-heliopset_20100702-repoint00202_v001.cdf', + 'imap_ultra_l1c_90sensor-heliopset_20100703-repoint00203_v001.cdf' ], [ - f'imap_ultra_l1c_45sensor-heliopset_20101001-repoint00301_v001.cdf', - f'imap_ultra_l1c_45sensor-heliopset_20101002-repoint00302_v001.cdf', - f'imap_ultra_l1c_45sensor-heliopset_20101003-repoint00303_v001.cdf' + 'imap_ultra_l1c_45sensor-heliopset_20101001-repoint00301_v001.cdf', + 'imap_ultra_l1c_45sensor-heliopset_20101002-repoint00302_v001.cdf', + 'imap_ultra_l1c_45sensor-heliopset_20101003-repoint00303_v001.cdf' ], [ - f'imap_ultra_l1c_90sensor-heliopset_20101001-repoint00301_v001.cdf', - f'imap_ultra_l1c_90sensor-heliopset_20101002-repoint00302_v001.cdf', - f'imap_ultra_l1c_90sensor-heliopset_20101003-repoint00303_v001.cdf' + 'imap_ultra_l1c_90sensor-heliopset_20101001-repoint00301_v001.cdf', + 'imap_ultra_l1c_90sensor-heliopset_20101002-repoint00302_v001.cdf', + 'imap_ultra_l1c_90sensor-heliopset_20101003-repoint00303_v001.cdf' ], [ 'imap_glows_l3e_survival-probability-ul-hf_20100101-repoint00001_v001.cdf', 'imap_glows_l3e_survival-probability-ul-hf_20100102-repoint00002_v001.cdf', 'imap_glows_l3e_survival-probability-ul-hf_20100103-repoint00003_v001.cdf', - f"imap_ultra_l1c_45sensor-heliopset_20100101-repoint00001_v001.cdf", - f"imap_ultra_l1c_45sensor-heliopset_20100102-repoint00002_v001.cdf", - f"imap_ultra_l1c_45sensor-heliopset_20100103-repoint00003_v001.cdf", - f"imap_ultra_l1c_90sensor-heliopset_20100101-repoint00001_v001.cdf", - f"imap_ultra_l1c_90sensor-heliopset_20100102-repoint00002_v001.cdf", - f"imap_ultra_l1c_90sensor-heliopset_20100103-repoint00003_v001.cdf", - f'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - f'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + "imap_ultra_l1c_45sensor-heliopset_20100101-repoint00001_v001.cdf", + "imap_ultra_l1c_45sensor-heliopset_20100102-repoint00002_v001.cdf", + "imap_ultra_l1c_45sensor-heliopset_20100103-repoint00003_v001.cdf", + "imap_ultra_l1c_90sensor-heliopset_20100101-repoint00001_v001.cdf", + "imap_ultra_l1c_90sensor-heliopset_20100102-repoint00002_v001.cdf", + "imap_ultra_l1c_90sensor-heliopset_20100103-repoint00003_v001.cdf", + 'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + 'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv', ], [ 'imap_glows_l3e_survival-probability-ul-hf_20100401-repoint00101_v001.cdf', 'imap_glows_l3e_survival-probability-ul-hf_20100402-repoint00102_v001.cdf', 'imap_glows_l3e_survival-probability-ul-hf_20100403-repoint00103_v001.cdf', - f'imap_ultra_l1c_45sensor-heliopset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_45sensor-heliopset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_45sensor-heliopset_20100403-repoint00103_v001.cdf', - f'imap_ultra_l1c_90sensor-heliopset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_90sensor-heliopset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_90sensor-heliopset_20100403-repoint00103_v001.cdf', - f'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l1c_45sensor-heliopset_20100401-repoint00101_v001.cdf', + 'imap_ultra_l1c_45sensor-heliopset_20100402-repoint00102_v001.cdf', + 'imap_ultra_l1c_45sensor-heliopset_20100403-repoint00103_v001.cdf', + 'imap_ultra_l1c_90sensor-heliopset_20100401-repoint00101_v001.cdf', + 'imap_ultra_l1c_90sensor-heliopset_20100402-repoint00102_v001.cdf', + 'imap_ultra_l1c_90sensor-heliopset_20100403-repoint00103_v001.cdf', + 'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv', ] ] initializer = UltraSPInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced(f"ulc-ena-h-hf-sp-full-hae-4deg-3mo", + actual_maps_to_produce = initializer.get_maps_that_should_be_produced("ulc-ena-h-hf-sp-full-hae-4deg-3mo", None) self.mock_query.assert_has_calls([ @@ -424,20 +424,20 @@ def test_get_maps_that_should_be_produced_lists_cg_corrected_maps(self, mock_rea ]) mock_read_cdf_parents.assert_has_calls([ - call(f'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), - call(f'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), - call(f'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), - call(f'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), - call(f'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), - call(f'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), - call(f'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20101001_v001.cdf'), - call(f'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20101001_v001.cdf'), - - call(f'imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-4deg-3mo_20100101_v001.cdf'), - call(f'imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-4deg-3mo_20100401_v001.cdf'), + call('imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), + call('imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), + call('imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), + call('imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), + call('imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), + call('imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), + call('imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20101001_v001.cdf'), + call('imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20101001_v001.cdf'), + + call('imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-4deg-3mo_20100101_v001.cdf'), + call('imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-4deg-3mo_20100401_v001.cdf'), ]) - descriptor = f"ulc-ena-h-hf-sp-full-hae-4deg-3mo" + descriptor = "ulc-ena-h-hf-sp-full-hae-4deg-3mo" expected_possible_map_to_produce = PossibleMapToProduce( input_metadata=InputMetadata( instrument="ultra", @@ -451,15 +451,15 @@ def test_get_maps_that_should_be_produced_lists_cg_corrected_maps(self, mock_rea 'imap_glows_l3e_survival-probability-ul-hf_20100401-repoint00101_v002.cdf', 'imap_glows_l3e_survival-probability-ul-hf_20100402-repoint00102_v002.cdf', 'imap_glows_l3e_survival-probability-ul-hf_20100403-repoint00103_v002.cdf', - f'imap_ultra_l1c_45sensor-heliopset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_45sensor-heliopset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_45sensor-heliopset_20100403-repoint00103_v001.cdf', - f'imap_ultra_l1c_90sensor-heliopset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_90sensor-heliopset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_90sensor-heliopset_20100403-repoint00103_v001.cdf', - f'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' + 'imap_ultra_l1c_45sensor-heliopset_20100401-repoint00101_v001.cdf', + 'imap_ultra_l1c_45sensor-heliopset_20100402-repoint00102_v001.cdf', + 'imap_ultra_l1c_45sensor-heliopset_20100403-repoint00103_v001.cdf', + 'imap_ultra_l1c_90sensor-heliopset_20100401-repoint00101_v001.cdf', + 'imap_ultra_l1c_90sensor-heliopset_20100402-repoint00102_v001.cdf', + 'imap_ultra_l1c_90sensor-heliopset_20100403-repoint00103_v001.cdf', + 'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', + 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' } ) self.assertIn(expected_possible_map_to_produce, actual_maps_to_produce) From d4572b0384ffdc009a8195c698b3de4e2532020b Mon Sep 17 00:00:00 2001 From: Jonathan Niehof Date: Fri, 23 Oct 2026 11:18:59 -0600 Subject: [PATCH 2/4] Auto-fixes from ruff format --- docs/swapi/figure_src/figure_utils.py | 70 +- .../figure_src/plot_alpha_peak_finding.py | 12 +- .../plot_collapsed_response_grid.py | 6 +- docs/swapi/figure_src/plot_fit_accuracy.py | 6 +- .../figure_src/plot_passband_boundaries.py | 4 +- .../figure_src/plot_pui_cutoff_staircase.py | 63 +- .../swapi/figure_src/plot_pui_distribution.py | 4 +- .../plot_pui_reference_comparison.py | 38 +- docs/swapi/figure_src/plot_real_data_fit.py | 16 +- .../figure_src/plot_validation_scatter.py | 9 +- imap_l3_data_processor.py | 160 +- imap_l3_processing/cdf/cdf_utils.py | 67 +- .../cdf/imap_attribute_manager.py | 45 +- .../codice/l3/hi/codice_hi_processor.py | 224 +- ...odice_hi_l3a_direct_events_dependencies.py | 53 +- .../codice_hi_mass_correction_lookup_table.py | 15 +- imap_l3_processing/codice/l3/hi/models.py | 183 +- .../codice_pitch_angle_dependencies.py | 27 +- ...odice_lo_l3a_direct_events_dependencies.py | 99 +- ...e_lo_l3a_partial_densities_dependencies.py | 49 +- .../lo/codice_lo_l3a_ratios_dependencies.py | 10 +- ...ce_lo_l3b_3d_distributions_dependencies.py | 98 +- .../codice/l3/lo/codice_lo_processor.py | 458 ++- imap_l3_processing/codice/l3/lo/constants.py | 2 +- .../lo/direct_events/science/angle_lookup.py | 29 +- .../science/efficiency_lookup.py | 11 +- .../lo/direct_events/science/energy_lookup.py | 12 +- .../science/geometric_factor_lookup.py | 132 +- .../science/mass_coefficient_lookup.py | 2 +- .../science/mass_species_bin_lookup.py | 9 +- imap_l3_processing/codice/l3/lo/models.py | 327 +- .../l3/lo/science/codice_lo_calculations.py | 239 +- imap_l3_processing/codice/quality_flags.py | 2 +- imap_l3_processing/constants.py | 26 +- imap_l3_processing/data_utils.py | 37 +- imap_l3_processing/glows/descriptors.py | 14 +- imap_l3_processing/glows/glows_processor.py | 624 +++- .../glows/l3a/glows_l3a_dependencies.py | 45 +- .../glows/l3a/glows_toolkit/ccsds.py | 85 +- .../glows/l3a/glows_toolkit/constants.py | 32 +- .../glows_toolkit/constants_arch20241201.py | 29 +- .../glows/l3a/glows_toolkit/direct_event.py | 34 +- .../glows/l3a/glows_toolkit/funcs.py | 63 +- .../geom_fun_libr_box/__init__.py | 193 +- .../glows/l3a/glows_toolkit/imap_time.py | 5 +- .../glows_toolkit/l0_data_direct_events.py | 39 +- .../l3a/glows_toolkit/l0_data_histogram.py | 61 +- .../glows_toolkit/l1a_data_direct_events.py | 239 +- .../l3a/glows_toolkit/l1a_data_histogram.py | 144 +- .../glows_toolkit/l1b_data_direct_events.py | 219 +- .../l3a/glows_toolkit/l1b_data_histogram.py | 839 +++-- .../l3a/glows_toolkit/l2_data_histogram.py | 569 ++-- .../glows/l3a/glows_toolkit/l3a_data.py | 437 ++- .../glows/l3a/glows_toolkit/reader.py | 30 +- .../glows/l3a/glows_toolkit/time.py | 3 + imap_l3_processing/glows/l3a/models.py | 105 +- .../science/bad_angle_flag_configuration.py | 12 +- .../l3a/science/calculate_daily_lightcurve.py | 54 +- ...ime_independent_background_lookup_table.py | 18 +- imap_l3_processing/glows/l3a/utils.py | 279 +- ...annot_process_carrington_rotation_error.py | 1 - .../glows/l3bc/dependency_validator.py | 57 +- .../glows/l3bc/glows_l3bc_dependencies.py | 33 +- .../glows/l3bc/glows_l3bc_initializer.py | 168 +- .../glows/l3bc/l3bc_toolkit/constants.py | 45 +- .../glows/l3bc/l3bc_toolkit/funcs.py | 844 +++-- .../l3bc_toolkit/l3b_CarringtonIonRate.py | 346 +- .../l3bc/l3bc_toolkit/l3b_DailyIonRate.py | 302 +- .../l3bc_toolkit/l3c_CarringtonSolarWind.py | 203 +- .../l3bc_toolkit/l3c_EclipticSolarWind.py | 102 +- imap_l3_processing/glows/l3bc/models.py | 205 +- .../glows/l3bc/science/filter_out_bad_days.py | 32 +- .../glows/l3bc/science/generate_l3bc.py | 75 +- imap_l3_processing/glows/l3bc/utils.py | 41 +- .../glows/l3d/glows_l3d_dependencies.py | 95 +- .../glows/l3d/glows_l3d_initializer.py | 137 +- imap_l3_processing/glows/l3d/models.py | 18 +- .../glows/l3d/science/generate_l3d.py | 57 +- .../glows/l3d/science/toolkit/constants.py | 45 +- .../glows/l3d/science/toolkit/funcs.py | 844 +++-- .../science/toolkit/l3d_SolarParamHistory.py | 713 ++-- imap_l3_processing/glows/l3d/utils.py | 153 +- .../glows/l3e/glows_l3e_call_arguments.py | 5 +- .../glows/l3e/glows_l3e_dependencies.py | 146 +- .../glows/l3e/glows_l3e_hi_model.py | 99 +- .../glows/l3e/glows_l3e_initializer.py | 193 +- .../glows/l3e/glows_l3e_lo_model.py | 68 +- .../glows/l3e/glows_l3e_ultra_model.py | 67 +- .../glows/l3e/glows_l3e_utils.py | 108 +- .../glows/l3e/reprocess_info.py | 33 +- imap_l3_processing/glows/quality_flags.py | 6 +- .../hi/hi_combined_initializer.py | 74 +- .../hi/hi_combined_sensor_dependencies.py | 29 +- imap_l3_processing/hi/hi_processor.py | 201 +- imap_l3_processing/hi/hi_sp_initializer.py | 88 +- .../hi/hi_spectral_fit_dependencies.py | 23 +- imap_l3_processing/hi/utils.py | 58 +- .../hit/l3/hit_l3_sectored_dependencies.py | 23 +- imap_l3_processing/hit/l3/hit_processor.py | 392 ++- imap_l3_processing/hit/l3/models.py | 6 +- .../hit/l3/pha/hit_l3_pha_dependencies.py | 135 +- .../hit/l3/pha/pha_event_reader.py | 77 +- .../hit/l3/pha/science/calculate_pha.py | 163 +- .../science/cosine_correction_lookup_table.py | 75 +- .../l3/pha/science/hit_event_type_lookup.py | 31 +- .../hit/l3/pha/science/range_fit_lookup.py | 36 +- .../hit/l3/sectored_products/models.py | 293 +- .../science/sectored_products_algorithms.py | 17 +- imap_l3_processing/hit/quality_flags.py | 2 +- .../lo/l3/lo_combined_dependencies.py | 10 +- .../lo/l3/lo_l3_spectral_fit_dependencies.py | 8 +- imap_l3_processing/lo/l3/lo_sp_initializer.py | 34 +- imap_l3_processing/lo/lo_processor.py | 161 +- .../lo/lo_txt_to_cdf_converter_spike.py | 238 +- .../maps/hilo_l3_survival_dependencies.py | 101 +- imap_l3_processing/maps/map_combination.py | 217 +- imap_l3_processing/maps/map_descriptors.py | 117 +- imap_l3_processing/maps/map_initializer.py | 62 +- imap_l3_processing/maps/map_models.py | 295 +- imap_l3_processing/maps/map_processor.py | 7 +- imap_l3_processing/maps/mpfit.py | 1956 ++++++----- .../maps/rectangular_survival_probability.py | 198 +- imap_l3_processing/maps/sp_map_initializer.py | 92 +- imap_l3_processing/maps/spectral_fit.py | 144 +- .../maps/survival_probability_processing.py | 102 +- imap_l3_processing/models.py | 14 +- imap_l3_processing/pitch_angles.py | 184 +- imap_l3_processing/processor.py | 16 +- imap_l3_processing/swapi/constants.py | 4 +- .../pickup_ion/collapsed_response_grid.py | 45 +- .../density_of_neutral_helium_lookup_table.py | 11 +- .../l3a/science/pickup_ion/goodness_of_fit.py | 11 +- .../l3a/science/pickup_ion/inflow_vector.py | 6 +- .../swapi/l3a/science/pickup_ion/moments.py | 1 + .../swapi/l3a/science/pickup_ion/utils.py | 3 +- .../vasyliunas_siscoe_distribution.py | 27 +- .../alpha/fit_solar_wind_alpha_model.py | 18 +- .../l3a/science/solar_wind/forward_model.py | 19 +- .../science/solar_wind/integration_limits.py | 8 +- .../proton/calculate_initial_guess.py | 9 +- .../proton/fit_solar_wind_proton_model.py | 5 +- .../optimize_solar_wind_proton_params.py | 4 +- .../science/solar_wind/trim_open_aperture.py | 30 +- .../swapi/l3a/science/uncertainties.py | 8 +- .../swapi/l3a/swapi_l3a_dependencies.py | 87 +- imap_l3_processing/swapi/l3b/models.py | 48 +- .../calculate_solar_wind_differential_flux.py | 17 +- .../l3b/science/calculate_solar_wind_vdf.py | 102 +- .../geometric_factor_calibration_table.py | 2 +- .../swapi/l3b/swapi_l3b_dependencies.py | 60 +- .../swapi/response/azimuthal_transmission.py | 5 +- .../response/efficiency_calibration_table.py | 23 +- .../swapi/response/swapi_response.py | 21 +- imap_l3_processing/swe/l3/models.py | 641 ++-- .../swe/l3/science/moment_calculations.py | 598 ++-- .../swe/l3/science/pitch_calculations.py | 452 ++- .../swe/l3/swe_l3_dependencies.py | 72 +- imap_l3_processing/swe/l3/utils.py | 47 +- imap_l3_processing/swe/quality_flags.py | 22 +- imap_l3_processing/swe/swe_processor.py | 98 +- imap_l3_processing/ultra/models.py | 32 +- .../science/ultra_survival_probability.py | 118 +- .../ultra/ultra_combined_nsp_initializer.py | 87 +- .../ultra/ultra_l3_dependencies.py | 163 +- .../ultra_l3_to_rectangular_dependencies.py | 4 +- imap_l3_processing/ultra/ultra_processor.py | 199 +- .../ultra/ultra_sp_initializer.py | 85 +- imap_l3_processing/utils.py | 163 +- imap_l3_processing/version.py | 2 +- run_local.py | 1353 +++++--- run_local_with_upload_and_download.py | 712 ++-- scripts/batch_download_parents.py | 16 +- scripts/clean_metadata.py | 13 +- .../create_fake_efficiency_ancillary.py | 21 +- .../create_more_accurate_l3a_direct_event.py | 35 +- ...te_more_accurate_l3a_direct_event_input.py | 81 +- scripts/compare_l2_map_metadata.py | 2 - scripts/confirm_reprocessing.py | 20 +- scripts/convert_product_definition_to_yaml.py | 110 +- scripts/convert_variable_yaml_to_csv.py | 14 +- scripts/fetch_latest_data.py | 4 +- scripts/generate_hi_yaml.py | 45 +- scripts/generate_lo_yaml.py | 52 +- scripts/generate_repointing_lines.py | 12 +- scripts/generate_ultra_yaml.py | 41 +- scripts/get_batch_jobs_status.py | 61 +- ...le_glows_l3e_survival_probabilities_cdf.py | 38 +- .../glows/create_l2_glows_cdf_from_json.py | 15 +- .../glows/create_l3a_glows_cdf_from_json.py | 27 +- scripts/glows/download_l2_prod_data.py | 17 +- scripts/glows/download_validation_data.py | 26 +- scripts/glows/upload_l3a_ancillary_files.py | 10 +- scripts/hi/create_all_maps_from_inputs.py | 31 +- scripts/hi/create_cdf_from_ibex.py | 20 +- .../create_cdf_from_instrument_team_data.py | 112 +- scripts/hi/create_hi_full_spin_deps.py | 120 +- scripts/hi/create_l2_from_pointing_sets.py | 33 +- scripts/hi/generate_fake_l1c_cdfs.py | 25 +- ...tack_arrays_to_match_number_of_energies.py | 23 +- scripts/hit/create_l1a_direct-event_data.py | 15 +- ...eate_l2_hit_sectored_rates_cdf_from_csv.py | 22 +- scripts/hit/create_mag_from_csv.py | 18 +- scripts/hit/hit_plots.py | 13 +- ...ombine_existing_maps_to_fix_frozen_data.py | 26 +- scripts/lo/create_l2_from_pointing_sets.py | 6 +- scripts/make_cdf_all_fill.py | 7 +- scripts/poll_batch_jobs_status.py | 30 +- scripts/reprocess_all_in_situ.py | 69 +- scripts/run_spectral_index.py | 13 +- ...termine_swapi_files_with_outdated_spice.py | 7 +- ...ha_sw_temperature_density_lut_generator.py | 7 +- scripts/swapi/fake_data_generator.py | 19 +- scripts/swapi/fake_uncertainties_for_cdf.py | 3 +- scripts/swapi/fit_and_plot_alpha.py | 259 +- scripts/swapi/fit_and_plot_proton.py | 263 +- scripts/swapi/fit_and_plot_pui.py | 381 ++- scripts/swapi/generate_cdf_from_dat.py | 23 +- ...erate_collapsed_response_grid_reference.py | 85 +- scripts/swapi/generate_reference_integrals.py | 8 +- .../generate_with_predicted_ephemeris.py | 62 +- scripts/swapi/pui_xarray_reference.py | 165 +- scripts/swapi/pui_xarray_reference_50sweep.py | 46 +- .../report_out_of_range_uncertainties.py | 2 +- ...alpha_sw_speed_temperature_density_demo.py | 121 +- scripts/swapi/upload_swapi_l3_dependencies.py | 14 +- scripts/swapi/view_one_pui_spectrum.py | 297 +- scripts/swe/calculate_rebin_by_pitch_angle.py | 103 +- scripts/swe/compare_outputs.py | 81 +- scripts/swe/create_fake_l2_cdf.py | 239 +- .../swe/filter_good_times_from_l1b_sdc_cdf.py | 76 +- .../swe/filter_good_times_from_l2_sdc_cdf.py | 28 +- ...ci_using_swapi_with_predicted_ephemeris.py | 65 +- scripts/swe/plot_swe.py | 16 +- scripts/swe/read_ace.py | 180 +- scripts/swe/swe_l2.py | 10 +- scripts/swe/swe_l2_old.py | 12 +- scripts/time_shift_cdf.py | 23 +- scripts/truncate_cdf.py | 8 +- .../ultra/create_example_ultra_l1c_pset.py | 99 +- scripts/ultra/create_example_ultra_l2_map.py | 117 +- ...ate_example_ultra_l2_map_with_power_law.py | 51 +- ...ple_ultra_l3_survival_probabilities_cdf.py | 48 +- .../ultra/create_glows_l3e_cdf_from_dat.py | 25 +- ...c_l2_and_glows_with_matching_date_range.py | 52 +- ...create_l2_cdf_from_instrument_team_data.py | 79 +- scripts/ultra/create_l2_dependency_files.py | 62 +- scripts/ultra/create_l2_from_pointing_sets.py | 13 +- scripts/ultra/plot_l2_uncertainties.py | 40 +- .../validation/create_L2_validation_maps.py | 43 +- .../validation/create_l3_validation_maps.py | 30 +- .../prepare_glows_for_ultra_validation.py | 15 +- ...e_ul_repoint_files_for_ultra_validation.py | 14 +- scripts/ultra/validation/upload_files.py | 20 +- tests/cdf/test_cdf_utils.py | 520 ++- tests/cdf/test_imap_attribute_manager.py | 221 +- ..._codice_hi_mass_correction_lookup_table.py | 50 +- ...t_codice_l3a_direct_events_dependencies.py | 157 +- .../test_codice_pitch_angle_dependencies.py | 113 +- .../codice/l3/hi/test_codice_hi_processor.py | 576 +++- tests/codice/l3/hi/test_models.py | 394 ++- .../science/test_angle_lookups.py | 61 +- .../science/test_efficiency_lookup.py | 36 +- .../science/test_energy_lookup.py | 29 +- .../science/test_geometric_factor_lookup.py | 309 +- .../science/test_mass_coefficient_lookup.py | 13 +- .../science/test_mass_species_bin_lookup.py | 31 +- .../lo/science/test_codice_lo_calculations.py | 661 ++-- .../test_mass_per_charge_lookup.py | 14 +- ...odice_lo_l3a_direct_events_dependencies.py | 161 +- ...e_lo_l3a_partial_densities_dependencies.py | 125 +- .../test_codice_lo_l3a_ratios_dependencies.py | 30 +- ...ce_lo_l3b_3d_distributions_dependencies.py | 129 +- .../codice/l3/lo/test_codice_lo_processor.py | 1663 ++++++--- tests/codice/l3/lo/test_models.py | 906 +++-- tests/debug_utils/profile_wrappers.py | 2 +- .../test_bad_angle_flag_configuration.py | 45 +- .../test_calculate_daily_lightcurve.py | 226 +- ...ime_independent_background_lookup_table.py | 30 +- .../glows/l3a/test_glows_l3a_dependencies.py | 88 +- tests/glows/l3a/test_models.py | 311 +- tests/glows/l3a/test_utils.py | 436 ++- tests/glows/l3bc/l3bc_toolkit/test_funcs.py | 17 +- .../l3bc/science/test_filter_out_bad_days.py | 14 +- .../glows/l3bc/science/test_generate_l3bc.py | 137 +- tests/glows/l3bc/science/test_time_parsing.py | 64 +- tests/glows/l3bc/test_dependency_validator.py | 94 +- .../l3bc/test_glows_l3bc_dependencies.py | 86 +- .../glows/l3bc/test_glows_l3bc_initializer.py | 606 +++- tests/glows/l3bc/test_models.py | 533 ++- tests/glows/l3bc/test_utils.py | 128 +- .../glows/l3d/test_glows_l3d_dependencies.py | 150 +- tests/glows/l3d/test_glows_l3d_initializer.py | 661 ++-- tests/glows/l3d/test_l3d_utils.py | 513 ++- tests/glows/l3d/test_models.py | 77 +- .../l3e/test_glows_l3e_call_arguments.py | 10 +- .../glows/l3e/test_glows_l3e_dependencies.py | 327 +- tests/glows/l3e/test_glows_l3e_hi_model.py | 1011 +++++- tests/glows/l3e/test_glows_l3e_initializer.py | 903 +++-- tests/glows/l3e/test_glows_l3e_lo_model.py | 719 +++- tests/glows/l3e/test_glows_l3e_ultra_model.py | 374 +- tests/glows/l3e/test_glows_l3e_utils.py | 324 +- tests/glows/l3e/test_reprocess_info.py | 129 +- tests/glows/test_glows_processor.py | 2649 ++++++++++----- tests/hi/test_hi_combined_dependencies.py | 69 +- tests/hi/test_hi_combined_initializer.py | 467 +-- tests/hi/test_hi_processor.py | 662 ++-- tests/hi/test_hi_sp_initializer.py | 721 ++-- tests/hi/test_hi_spectral_fit_dependencies.py | 53 +- tests/hi/test_utils.py | 126 +- tests/hit/l3/hit_test_builders.py | 58 +- .../hit/l3/pha/science/test_calculate_pha.py | 735 ++-- .../test_cosine_correction_lookup_table.py | 111 +- .../l3/pha/science/test_gain_lookup_table.py | 12 +- .../pha/science/test_hit_event_type_lookup.py | 54 +- .../l3/pha/science/test_range_fit_lookup.py | 76 +- tests/hit/l3/pha/test_l3_pha_dependencies.py | 218 +- tests/hit/l3/pha/test_pha_event_reader.py | 314 +- .../test_sectored_products_algorithms.py | 23 +- tests/hit/l3/sectored_products/test_models.py | 387 ++- tests/hit/l3/test_hit_processor.py | 1182 +++++-- tests/hit/l3/test_l3_sectored_dependencies.py | 90 +- tests/hit/l3/test_models.py | 173 +- tests/hit/l3/test_utils.py | 197 +- tests/integration/integration_test_helpers.py | 57 +- .../test_codice_processor_integration.py | 65 +- .../test_glows_processor_integration.py | 8 +- .../test_hit_processor_integration.py | 24 +- .../test_map_processor_integration.py | 429 ++- .../test_swe_processor_integration.py | 15 +- tests/lo/l3/test_lo_combined_dependencies.py | 29 +- ..._isn_background_subtracted_dependencies.py | 22 +- .../test_lo_l3_spectral_fit_dependencies.py | 42 +- tests/lo/l3/test_lo_sp_initializer.py | 174 +- tests/lo/test_lo_processor.py | 406 ++- tests/maps/test_builders.py | 173 +- .../test_hilo_l3_survival_dependencies.py | 185 +- tests/maps/test_map_combination.py | 460 ++- tests/maps/test_map_descriptors.py | 539 ++- tests/maps/test_map_models.py | 567 +++- .../test_rectangular_survival_probability.py | 938 +++-- tests/maps/test_spectral_fit.py | 819 +++-- .../test_survival_probability_processing.py | 526 ++- ...test_convert_product_definition_to_yaml.py | 171 +- tests/swapi/_helpers.py | 1 + tests/swapi/cdf_model_test_case.py | 32 +- .../test_build_collapsed_response_grid.py | 13 +- .../test_calculate_coincidence_rate.py | 17 +- ..._density_of_neutral_helium_lookup_table.py | 5 +- .../pickup_ion/test_goodness_of_fit.py | 76 +- .../l3a/science/pickup_ion/test_utils.py | 31 +- .../test_vasyliunas_siscoe_distribution.py | 14 +- .../alpha/test_fit_solar_wind_alpha_model.py | 94 +- .../proton/test_calculate_initial_guess.py | 35 +- .../proton/test_escape_local_minimum.py | 112 +- .../test_fit_solar_wind_proton_model.py | 141 +- .../test_optimize_solar_wind_proton_params.py | 2 - .../solar_wind/test_azimuthal_regions.py | 3 +- .../science/solar_wind/test_fit_context.py | 20 +- .../science/solar_wind/test_forward_model.py | 6 +- .../solar_wind/test_integration_limits.py | 32 +- .../l3a/science/solar_wind/test_params.py | 4 +- .../solar_wind/test_trim_open_aperture.py | 23 +- .../test_uncertainty_calibration.py | 19 +- .../l3a/science/solar_wind/test_utils.py | 8 +- tests/swapi/l3a/science/test_inflow_vector.py | 16 +- .../swapi/l3a/test_swapi_l3a_dependencies.py | 238 +- ..._calculate_solar_wind_differential_flux.py | 22 +- .../science/test_calculate_solar_wind_vdf.py | 45 +- ...test_geometric_factor_calibration_table.py | 18 +- tests/swapi/l3b/test_models.py | 82 +- .../swapi/l3b/test_swapi_l3b_dependencies.py | 138 +- .../response/test_azimuthal_transmission.py | 20 +- .../test_efficiency_calibration_table.py | 133 +- tests/swapi/response/test_passband_grid.py | 21 +- tests/swapi/response/test_swapi_response.py | 91 +- tests/swe/l3/science/test_models.py | 713 +++- .../l3/science/test_moments_calculation.py | 1782 +++++++--- .../swe/l3/science/test_pitch_calculations.py | 773 +++-- tests/swe/l3/test_swe_l3_dependencies.py | 113 +- tests/swe/l3/test_utils.py | 69 +- tests/swe/test_swe_processor.py | 3005 ++++++++++++----- .../3S5_l1b_ram_maps/archive/map_SCFrame.py | 206 +- .../3S5_l1b_ram_maps/crap/new_patch_arc2.py | 419 ++- .../3S5_l1b_ram_maps/csv2yaml.py | 170 +- .../3S5_l1b_ram_maps/csv_to_soc.py | 75 +- .../3S5_l1b_ram_maps/l1b_to_spin.py | 301 +- .../3S5_l1b_ram_maps/map_SCFrame.py | 300 +- .../3S5_l1b_ram_maps/new_patch_arc2.py | 436 ++- .../3S5_l1b_ram_maps/new_patch_arc2_linear.py | 427 ++- .../3S5_l1b_ram_maps/old/csv2yaml.py | 131 +- .../3S5_l1b_ram_maps/plot_map_DN.py | 203 +- tests/test_data_utils.py | 364 +- tests/test_helpers.py | 151 +- tests/test_imap_l3_data_processor.py | 760 +++-- tests/test_models.py | 94 +- tests/test_pitch_angles.py | 395 ++- tests/test_predicted_ephemeris_tracker.py | 10 +- tests/test_processor.py | 22 +- tests/ultra/test_models.py | 66 +- .../test_ultra_combined_nsp_initializer.py | 250 +- tests/ultra/test_ultra_l3_dependencies.py | 860 +++-- .../test_ultra_l3_survival_probabillity.py | 507 ++- ...st_ultra_l3_to_rectangular_dependencies.py | 34 +- tests/ultra/test_ultra_processor.py | 981 ++++-- tests/ultra/test_ultra_sp_initializer.py | 707 ++-- 405 files changed, 48544 insertions(+), 23144 deletions(-) diff --git a/docs/swapi/figure_src/figure_utils.py b/docs/swapi/figure_src/figure_utils.py index 9e9dc7b2a..15cd0e8c5 100644 --- a/docs/swapi/figure_src/figure_utils.py +++ b/docs/swapi/figure_src/figure_utils.py @@ -42,14 +42,68 @@ def load_swapi_response() -> SwapiResponse: COARSE_SWEEP_VOLTAGES_MEAN_V = np.array( [ - 9895.52, 9088.69, 8348.80, 7667.55, 7042.16, 6469.31, 5941.77, 5457.31, - 5013.22, 4603.65, 4230.77, 3886.92, 3569.16, 3278.72, 3011.13, 2766.25, - 2539.54, 2333.83, 2144.24, 1969.31, 1808.74, 1660.86, 1525.75, 1401.82, - 1287.58, 1182.24, 1085.15, 995.55, 914.31, 839.94, 771.70, 709.46, - 651.59, 598.47, 549.91, 505.12, 463.89, 425.92, 391.18, 359.35, - 329.94, 303.02, 278.25, 255.55, 234.77, 215.61, 197.95, 181.82, - 167.04, 153.46, 140.91, 129.50, 118.91, 109.20, 100.30, 92.11, - 84.61, 77.73, 71.40, 65.59, 60.23, 55.34, + 9895.52, + 9088.69, + 8348.80, + 7667.55, + 7042.16, + 6469.31, + 5941.77, + 5457.31, + 5013.22, + 4603.65, + 4230.77, + 3886.92, + 3569.16, + 3278.72, + 3011.13, + 2766.25, + 2539.54, + 2333.83, + 2144.24, + 1969.31, + 1808.74, + 1660.86, + 1525.75, + 1401.82, + 1287.58, + 1182.24, + 1085.15, + 995.55, + 914.31, + 839.94, + 771.70, + 709.46, + 651.59, + 598.47, + 549.91, + 505.12, + 463.89, + 425.92, + 391.18, + 359.35, + 329.94, + 303.02, + 278.25, + 255.55, + 234.77, + 215.61, + 197.95, + 181.82, + 167.04, + 153.46, + 140.91, + 129.50, + 118.91, + 109.20, + 100.30, + 92.11, + 84.61, + 77.73, + 71.40, + 65.59, + 60.23, + 55.34, ] ) diff --git a/docs/swapi/figure_src/plot_alpha_peak_finding.py b/docs/swapi/figure_src/plot_alpha_peak_finding.py index 0b8446d58..a4e9466cc 100644 --- a/docs/swapi/figure_src/plot_alpha_peak_finding.py +++ b/docs/swapi/figure_src/plot_alpha_peak_finding.py @@ -187,8 +187,12 @@ def _read_5_sweep_block( def _coarse_measurement_times_ns(epoch_ns: np.ndarray) -> np.ndarray: coarse_bins = np.arange(SWAPI_COARSE_SWEEP_BINS.start, SWAPI_COARSE_SWEEP_BINS.stop) - seconds_into_sweep = coarse_bins * SWAPI_BIN_PERIOD_S + SWAPI_LIVETIME_CENTER_OFFSET_S - return (epoch_ns[:, None] + seconds_into_sweep * ONE_SECOND_IN_NANOSECONDS).flatten() + seconds_into_sweep = ( + coarse_bins * SWAPI_BIN_PERIOD_S + SWAPI_LIVETIME_CENTER_OFFSET_S + ) + return ( + epoch_ns[:, None] + seconds_into_sweep * ONE_SECOND_IN_NANOSECONDS + ).flatten() # --------------------------------------------------------------------------- # @@ -261,9 +265,7 @@ def _plot_case(ax, swapi_response, cdf_path, mag_data, chunk_index): proton_moments=proton_moments, magnetic_field_direction=b_hat, ) - alpha_velocity_rtn = np.array( - [c.nominal_value for c in alpha_moments.velocity_rtn] - ) + alpha_velocity_rtn = np.array([c.nominal_value for c in alpha_moments.velocity_rtn]) alpha_sw = SolarWindParams( density=alpha_moments.density.nominal_value, velocity_rtn=alpha_velocity_rtn, diff --git a/docs/swapi/figure_src/plot_collapsed_response_grid.py b/docs/swapi/figure_src/plot_collapsed_response_grid.py index 53be77b27..21f9a6b14 100644 --- a/docs/swapi/figure_src/plot_collapsed_response_grid.py +++ b/docs/swapi/figure_src/plot_collapsed_response_grid.py @@ -32,7 +32,11 @@ from imap_l3_processing.constants import HE_PUI_PARTICLE_MASS_PER_CHARGE_M_P_PER_E _REFERENCE_CSV_PATH = ( - REPO_ROOT / "tests" / "test_data" / "swapi" / "collapsed_response_grid_reference.csv" + REPO_ROOT + / "tests" + / "test_data" + / "swapi" + / "collapsed_response_grid_reference.csv" ) _ESA_VOLTAGE = 5000.0 diff --git a/docs/swapi/figure_src/plot_fit_accuracy.py b/docs/swapi/figure_src/plot_fit_accuracy.py index 795583dee..040aa57f4 100644 --- a/docs/swapi/figure_src/plot_fit_accuracy.py +++ b/docs/swapi/figure_src/plot_fit_accuracy.py @@ -315,7 +315,7 @@ def _plot_results(data: pd.DataFrame) -> None: marker="o", zorder=2, label="Initial guess", - rasterized=True + rasterized=True, ) ax.errorbar( truth[good], @@ -344,7 +344,7 @@ def _plot_results(data: pd.DataFrame) -> None: edgecolors="k", linewidths=0.4, zorder=4, - rasterized=True + rasterized=True, ) ax.scatter( truth[~good], @@ -356,7 +356,7 @@ def _plot_results(data: pd.DataFrame) -> None: edgecolors="k", linewidths=0.4, zorder=4, - rasterized=True + rasterized=True, ) ax.set_xlabel(f"True {label}", fontsize=9) diff --git a/docs/swapi/figure_src/plot_passband_boundaries.py b/docs/swapi/figure_src/plot_passband_boundaries.py index 862d543ee..57abb5c84 100644 --- a/docs/swapi/figure_src/plot_passband_boundaries.py +++ b/docs/swapi/figure_src/plot_passband_boundaries.py @@ -85,9 +85,7 @@ def plot_region_panel( grid, grid.max_boundary, active_elevations, np.maximum ) - image = draw_transmission_heatmap( - axis, grid.values, elevations, speed_ratios - ) + image = draw_transmission_heatmap(axis, grid.values, elevations, speed_ratios) draw_integration_window_outline( axis, active_elevations, lower_speed_ratios, upper_speed_ratios ) diff --git a/docs/swapi/figure_src/plot_pui_cutoff_staircase.py b/docs/swapi/figure_src/plot_pui_cutoff_staircase.py index 30093152b..bf4a20211 100644 --- a/docs/swapi/figure_src/plot_pui_cutoff_staircase.py +++ b/docs/swapi/figure_src/plot_pui_cutoff_staircase.py @@ -37,7 +37,10 @@ from figure_utils import FIGURES_DIR, load_swapi_response -from imap_l3_processing.constants import HE_PUI_PARTICLE_MASS_PER_CHARGE_M_P_PER_E, ONE_AU_IN_KM +from imap_l3_processing.constants import ( + HE_PUI_PARTICLE_MASS_PER_CHARGE_M_P_PER_E, + ONE_AU_IN_KM, +) from imap_l3_processing.swapi.l3a.science.pickup_ion.calculate_coincidence_rate import ( apply_partial_heaviside_at_cutoff, calculate_coincidence_rate, @@ -53,7 +56,9 @@ FittingParameters, VasyliunasSiscoeDistribution, ) -from imap_l3_processing.swapi.l3a.utils import velocity_components_to_angles_in_instrument_frame +from imap_l3_processing.swapi.l3a.utils import ( + velocity_components_to_angles_in_instrument_frame, +) _SW_SPEED_KMS = 450.0 _SW_AZIMUTH_DEG = 0.0 @@ -90,9 +95,13 @@ def _params(cutoff_speed_kms: float) -> FittingParameters: def main(): - voltages = pd.read_csv( - REPO_ROOT / "tests" / "test_data" / "swapi" / "pui_count_rate_reference.csv" - ).iloc[:, 0].to_numpy() + voltages = ( + pd.read_csv( + REPO_ROOT / "tests" / "test_data" / "swapi" / "pui_count_rate_reference.csv" + ) + .iloc[:, 0] + .to_numpy() + ) voltage_index = int(np.argmin(np.abs(voltages - _ESA_VOLTAGE_V))) swapi_response = load_swapi_response() @@ -109,11 +118,13 @@ def main(): azimuth_rad = np.radians(_SW_AZIMUTH_DEG) elevation_rad = np.radians(_SW_ELEVATION_DEG) - bulk_vec = _SW_SPEED_KMS * np.array([ - -np.cos(elevation_rad) * np.sin(azimuth_rad), - -np.cos(elevation_rad) * np.cos(azimuth_rad), - -np.sin(elevation_rad), - ]) + bulk_vec = _SW_SPEED_KMS * np.array( + [ + -np.cos(elevation_rad) * np.sin(azimuth_rad), + -np.cos(elevation_rad) * np.cos(azimuth_rad), + -np.sin(elevation_rad), + ] + ) bulk_per_bin = np.broadcast_to(bulk_vec, (1, voltages.size, 3)).copy() chunk_response = build_chunk_collapsed_response( @@ -130,8 +141,10 @@ def main(): bin_weights = chunk_response.bin_weights[0, voltage_index, :] # Fine-grid reference response for the same voltage step. - bulk_azimuth_deg, bulk_elevation_deg = velocity_components_to_angles_in_instrument_frame( - bulk_vec[0], bulk_vec[1], bulk_vec[2] + bulk_azimuth_deg, bulk_elevation_deg = ( + velocity_components_to_angles_in_instrument_frame( + bulk_vec[0], bulk_vec[1], bulk_vec[2] + ) ) fine_grid = np.linspace(1.0, _GRID_MAX_KMS, _FINE_GRID_POINTS) fine_collapsed = build_collapsed_response_grid( @@ -145,9 +158,7 @@ def main(): bulk_elevation_deg, speed_in_sw_frame=fine_grid, ) - fine_weights = ( - fine_collapsed.values * fine_grid**2 * (fine_grid[1] - fine_grid[0]) - ) + fine_weights = fine_collapsed.values * fine_grid**2 * (fine_grid[1] - fine_grid[0]) # Below the neutral-helium LUT support the distribution is physically zero; # the LUT extrapolation returns non-finite values there, so clamp them. @@ -190,15 +201,26 @@ def sampled_distribution(grid, params, apply_cutoff): axis.axvline(grid_edges[0], color="0.85", lw=0.8, zorder=0, label="$v'$ grid edges") axis.plot( - _CUTOFF_SWEEP_KMS, rate_reference, "-", color="0.4", lw=1, + _CUTOFF_SWEEP_KMS, + rate_reference, + "-", + color="0.4", + lw=1, label="fine-grid reference", ) axis.scatter( - _CUTOFF_SWEEP_KMS, rate_without, color="tab:red", s=10, + _CUTOFF_SWEEP_KMS, + rate_without, + color="tab:red", + s=10, label="no correction", ) axis.scatter( - _CUTOFF_SWEEP_KMS, rate_with, color="tab:blue", s=10, marker='x', + _CUTOFF_SWEEP_KMS, + rate_with, + color="tab:blue", + s=10, + marker="x", label="with correction", ) @@ -207,7 +229,10 @@ def sampled_distribution(grid, params, apply_cutoff): axis.set_xlim(_CUTOFF_SWEEP_KMS[0], _CUTOFF_SWEEP_KMS[-1]) axis.annotate( rf"grid $\Delta v' = {delta_v_prime:.2f}$ km/s", - xy=(0.02, 0.04), xycoords="axes fraction", fontsize=9, color="0.3", + xy=(0.02, 0.04), + xycoords="axes fraction", + fontsize=9, + color="0.3", ) axis.grid(True, alpha=0.3) axis.legend(loc="upper left") diff --git a/docs/swapi/figure_src/plot_pui_distribution.py b/docs/swapi/figure_src/plot_pui_distribution.py index 79275342f..03a3fe520 100644 --- a/docs/swapi/figure_src/plot_pui_distribution.py +++ b/docs/swapi/figure_src/plot_pui_distribution.py @@ -92,7 +92,9 @@ def main(): label=r"cutoff speed $v_b$", ) axis.set_yscale("log") - axis.set_xlabel(r"Comoving speed $v' = \|\mathbf{v} - \mathbf{v}_\text{sw}\|$ [km/s]") + axis.set_xlabel( + r"Comoving speed $v' = \|\mathbf{v} - \mathbf{v}_\text{sw}\|$ [km/s]" + ) axis.set_ylabel(r"$f_\text{PUI}(v')$ [s$^3$ km$^{-6}$]") axis.set_xlim(speed_in_sw_frame[0], speed_in_sw_frame[-1]) axis.set_ylim(1e0, 1e4) diff --git a/docs/swapi/figure_src/plot_pui_reference_comparison.py b/docs/swapi/figure_src/plot_pui_reference_comparison.py index 9b32aaab5..c0546f371 100644 --- a/docs/swapi/figure_src/plot_pui_reference_comparison.py +++ b/docs/swapi/figure_src/plot_pui_reference_comparison.py @@ -95,14 +95,14 @@ def compute_production_rates(voltages_v: np.ndarray) -> np.ndarray: sw_az_rad = np.radians(_SW_AZIMUTH_DEG) sw_el_rad = np.radians(_SW_ELEVATION_DEG) - bulk_vec = _SW_SPEED_KMS * np.array([ - -np.cos(sw_el_rad) * np.sin(sw_az_rad), - -np.cos(sw_el_rad) * np.cos(sw_az_rad), - -np.sin(sw_el_rad), - ]) - bulk_sw_per_bin = np.broadcast_to( - bulk_vec, (1, voltages_v.size, 3) - ).copy() + bulk_vec = _SW_SPEED_KMS * np.array( + [ + -np.cos(sw_el_rad) * np.sin(sw_az_rad), + -np.cos(sw_el_rad) * np.cos(sw_az_rad), + -np.sin(sw_el_rad), + ] + ) + bulk_sw_per_bin = np.broadcast_to(bulk_vec, (1, voltages_v.size, 3)).copy() swapi_response = load_swapi_response() swapi_response.warm_cache(voltages_v) @@ -115,7 +115,9 @@ def compute_production_rates(voltages_v: np.ndarray) -> np.ndarray: min_speed_kms=min_speed_kms, central_effective_area_scale=_HELIUM_EFFICIENCY_RATIO, ) - return calculate_coincidence_rate(chunk_response, vasyliunas_siscoe_distribution, fitting_params)[0] + return calculate_coincidence_rate( + chunk_response, vasyliunas_siscoe_distribution, fitting_params + )[0] def main(): @@ -133,11 +135,17 @@ def main(): ) sw_az_rad = np.radians(_SW_AZIMUTH_DEG) sw_el_rad = np.radians(_SW_ELEVATION_DEG) - bulk_vec = _SW_SPEED_KMS * np.array([ - -np.cos(sw_el_rad) * np.sin(sw_az_rad), - -np.cos(sw_el_rad) * np.cos(sw_az_rad), - -np.sin(sw_el_rad), - ]) + 0.0 # normalize -0.0 to +0.0 for display + bulk_vec = ( + _SW_SPEED_KMS + * np.array( + [ + -np.cos(sw_el_rad) * np.sin(sw_az_rad), + -np.cos(sw_el_rad) * np.cos(sw_az_rad), + -np.sin(sw_el_rad), + ] + ) + + 0.0 + ) # normalize -0.0 to +0.0 for display axes[0].set_title( f"PUI He coincidence rate — m/q={_HELIUM_MASS_PER_CHARGE_M_P_PER_E:.0f}, " f"efficiency ratio={_HELIUM_EFFICIENCY_RATIO:.2f}\n" @@ -165,7 +173,7 @@ def main(): ".", color="red", label="Optimized", - markersize=4 + markersize=4, ) rel_err = np.abs(production_rate_hz - count_rate_hz) / count_rate_hz axes[1].semilogy(voltage_v, rel_err, ".", color="red") diff --git a/docs/swapi/figure_src/plot_real_data_fit.py b/docs/swapi/figure_src/plot_real_data_fit.py index 61c943a39..af29fe159 100644 --- a/docs/swapi/figure_src/plot_real_data_fit.py +++ b/docs/swapi/figure_src/plot_real_data_fit.py @@ -120,15 +120,11 @@ def main(): sc_velocity_rtn = get_spacecraft_velocity_rtn(chunk_center_ns) science_result["velocity_rtn_sun"] = ( - np.array( - [c.nominal_value for c in science_result["fit_result"].velocity_rtn] - ) + np.array([c.nominal_value for c in science_result["fit_result"].velocity_rtn]) + sc_velocity_rtn ) coarse_result["velocity_rtn_sun"] = ( - np.array( - [c.nominal_value for c in coarse_result["fit_result"].velocity_rtn] - ) + np.array([c.nominal_value for c in coarse_result["fit_result"].velocity_rtn]) + sc_velocity_rtn ) @@ -283,7 +279,9 @@ def _read_5_sweep_block( def _measurement_times_ns(epoch_ns: np.ndarray, bin_slice: slice) -> np.ndarray: bins = np.arange(bin_slice.start, bin_slice.stop) seconds_into_sweep = bins * SWAPI_BIN_PERIOD_S + SWAPI_LIVETIME_CENTER_OFFSET_S - return (epoch_ns[:, None] + seconds_into_sweep * ONE_SECOND_IN_NANOSECONDS).flatten() + return ( + epoch_ns[:, None] + seconds_into_sweep * ONE_SECOND_IN_NANOSECONDS + ).flatten() # --------------------------------------------------------------------------- # @@ -561,9 +559,7 @@ def _model_rates_with_deadtime( ) sw = SolarWindParams( density=fit_result.density.nominal_value, - velocity_rtn=np.array( - [c.nominal_value for c in fit_result.velocity_rtn] - ), + velocity_rtn=np.array([c.nominal_value for c in fit_result.velocity_rtn]), temperature=fit_result.temperature.nominal_value, mass=PROTON_MASS_KG, ) diff --git a/docs/swapi/figure_src/plot_validation_scatter.py b/docs/swapi/figure_src/plot_validation_scatter.py index ea117ff02..fd97aee7c 100644 --- a/docs/swapi/figure_src/plot_validation_scatter.py +++ b/docs/swapi/figure_src/plot_validation_scatter.py @@ -123,8 +123,13 @@ def _plot_scatter(reference: np.ndarray, optimized: np.ndarray) -> None: zorder=4, ) ax.plot( - line, line, color="black", linestyle="--", linewidth=0.8, - label="1:1", zorder=5, + line, + line, + color="black", + linestyle="--", + linewidth=0.8, + label="1:1", + zorder=5, ) ax.set_xscale("log") diff --git a/imap_l3_data_processor.py b/imap_l3_data_processor.py index 3b7af7cfb..b664297e2 100644 --- a/imap_l3_data_processor.py +++ b/imap_l3_data_processor.py @@ -13,21 +13,36 @@ from imap_l3_processing.codice.l3.hi.codice_hi_processor import CodiceHiProcessor from imap_l3_processing.codice.l3.lo.codice_lo_processor import CodiceLoProcessor from imap_l3_processing.glows.glows_processor import GlowsProcessor -from imap_l3_processing.hi.hi_combined_initializer import HI_COMBINED_DESCRIPTORS, HiCombinedInitializer +from imap_l3_processing.hi.hi_combined_initializer import ( + HI_COMBINED_DESCRIPTORS, + HiCombinedInitializer, +) from imap_l3_processing.hi.hi_processor import HiProcessor -from imap_l3_processing.hi.hi_sp_initializer import HiSPInitializer, HI_SP_MAP_DESCRIPTORS +from imap_l3_processing.hi.hi_sp_initializer import ( + HiSPInitializer, + HI_SP_MAP_DESCRIPTORS, +) from imap_l3_processing.hit.l3.hit_processor import HitProcessor -from imap_l3_processing.lo.l3.lo_sp_initializer import LoSPInitializer, LO_SP_MAP_DESCRIPTORS +from imap_l3_processing.lo.l3.lo_sp_initializer import ( + LoSPInitializer, + LO_SP_MAP_DESCRIPTORS, +) from imap_l3_processing.lo.lo_processor import LoProcessor from imap_l3_processing.maps.map_descriptors import parse_map_descriptor from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.swapi.swapi_processor import SwapiProcessor from imap_l3_processing.swe.swe_processor import SweProcessor -from imap_l3_processing.ultra.ultra_combined_nsp_initializer import UltraCombinedNSPInitializer, \ - ULTRA_COMBINED_NSP_DESCRIPTORS +from imap_l3_processing.ultra.ultra_combined_nsp_initializer import ( + UltraCombinedNSPInitializer, + ULTRA_COMBINED_NSP_DESCRIPTORS, +) from imap_l3_processing.ultra.ultra_processor import UltraProcessor -from imap_l3_processing.ultra.ultra_sp_initializer import UltraSPInitializer, ULTRA_45_DESCRIPTORS, \ - ULTRA_90_DESCRIPTORS, ULTRA_COMBINED_SP_DESCRIPTORS +from imap_l3_processing.ultra.ultra_sp_initializer import ( + UltraSPInitializer, + ULTRA_45_DESCRIPTORS, + ULTRA_90_DESCRIPTORS, + ULTRA_COMBINED_SP_DESCRIPTORS, +) logger = logging.getLogger(__name__) @@ -75,32 +90,68 @@ def imap_l3_processor(): processing_input_collection.deserialize(args.dependency) version_map = VersionMap({}, Version(None, args.version)) else: - processing_input_collection.deserialize(json.dumps(parsed_dependency["dependency"])) - version_map = VersionMap({k: Version(v["major_version"], v["minor_version"]) for k, v in parsed_dependency["version"].items()}) + processing_input_collection.deserialize( + json.dumps(parsed_dependency["dependency"]) + ) + version_map = VersionMap( + { + k: Version(v["major_version"], v["minor_version"]) + for k, v in parsed_dependency["version"].items() + } + ) repointing_number = None if args.repointing is not None: - repointing_number_match = re.match(r"repoint(?P\d{5})", args.repointing) + repointing_number_match = re.match( + r"repoint(?P\d{5})", args.repointing + ) if repointing_number_match is None: raise ValueError("Unexpected repointing number command line format!") repointing_number = int(repointing_number_match["repoint"]) _furnish_spice_kernels(processing_input_collection) - input_dependency = InputMetadata(args.instrument, - args.data_level, - _convert_to_datetime(args.start_date), - _convert_to_datetime(args.end_date or args.start_date), - version_map, descriptor=args.descriptor, repointing=repointing_number) - if args.instrument in ["hi", "lo", "ultra"] and args.data_level == 'l3' and not parse_map_descriptor( - args.descriptor): + input_dependency = InputMetadata( + args.instrument, + args.data_level, + _convert_to_datetime(args.start_date), + _convert_to_datetime(args.end_date or args.start_date), + version_map, + descriptor=args.descriptor, + repointing=repointing_number, + ) + if ( + args.instrument in ["hi", "lo", "ultra"] + and args.data_level == "l3" + and not parse_map_descriptor(args.descriptor) + ): initializer_class, processor_class, descriptors = { ("hi", "sp-maps"): (HiSPInitializer, HiProcessor, HI_SP_MAP_DESCRIPTORS), - ("hi", "hic-maps"): (HiCombinedInitializer, HiProcessor, HI_COMBINED_DESCRIPTORS), + ("hi", "hic-maps"): ( + HiCombinedInitializer, + HiProcessor, + HI_COMBINED_DESCRIPTORS, + ), ("lo", "all-maps"): (LoSPInitializer, LoProcessor, LO_SP_MAP_DESCRIPTORS), - ("ultra", "u45-maps"): (UltraSPInitializer, UltraProcessor, ULTRA_45_DESCRIPTORS), - ("ultra", "u90-maps"): (UltraSPInitializer, UltraProcessor, ULTRA_90_DESCRIPTORS), - ("ultra", "ulc-sp-maps"): (UltraSPInitializer, UltraProcessor, ULTRA_COMBINED_SP_DESCRIPTORS), - ("ultra", "ulc-nsp-maps"): (UltraCombinedNSPInitializer, UltraProcessor, ULTRA_COMBINED_NSP_DESCRIPTORS), + ("ultra", "u45-maps"): ( + UltraSPInitializer, + UltraProcessor, + ULTRA_45_DESCRIPTORS, + ), + ("ultra", "u90-maps"): ( + UltraSPInitializer, + UltraProcessor, + ULTRA_90_DESCRIPTORS, + ), + ("ultra", "ulc-sp-maps"): ( + UltraSPInitializer, + UltraProcessor, + ULTRA_COMBINED_SP_DESCRIPTORS, + ), + ("ultra", "ulc-nsp-maps"): ( + UltraCombinedNSPInitializer, + UltraProcessor, + ULTRA_COMBINED_NSP_DESCRIPTORS, + ), }[args.instrument, args.descriptor] initializer = initializer_class() @@ -108,57 +159,75 @@ def imap_l3_processor(): maps_to_produce = [] for map_descriptor in descriptors: major_version_from_dependency = version_map.lookup(map_descriptor).major - maps_to_produce.extend(initializer.get_maps_that_should_be_produced(map_descriptor, major_version_from_dependency)) + maps_to_produce.extend( + initializer.get_maps_that_should_be_produced( + map_descriptor, major_version_from_dependency + ) + ) - logger.info(f"maps to produce {[m.input_metadata.descriptor for m in maps_to_produce]}") + logger.info( + f"maps to produce {[m.input_metadata.descriptor for m in maps_to_produce]}" + ) if len(maps_to_produce) == 0: - logger.info(f"Did not find any maps to produce for instrument {args.instrument}") + logger.info( + f"Did not find any maps to produce for instrument {args.instrument}" + ) for map_to_produce in maps_to_produce: logger.info( - f"Processing map {map_to_produce.input_metadata.descriptor}, {map_to_produce.input_metadata.start_date}") + f"Processing map {map_to_produce.input_metadata.descriptor}, {map_to_produce.input_metadata.start_date}" + ) initializer.furnish_spice_dependencies(map_to_produce) try: - processor = processor_class(map_to_produce.processing_input_collection, map_to_produce.input_metadata) + processor = processor_class( + map_to_produce.processing_input_collection, + map_to_produce.input_metadata, + ) paths.extend(processor.process()) except Exception as e: logger.exception( f"Failed to produce map {map_to_produce.input_metadata.descriptor}, {map_to_produce.input_metadata.start_date}!", - exc_info=e) - elif args.instrument == 'swapi' and (args.data_level == 'l3a' or args.data_level == 'l3b'): + exc_info=e, + ) + elif args.instrument == "swapi" and ( + args.data_level == "l3a" or args.data_level == "l3b" + ): processor = SwapiProcessor(processing_input_collection, input_dependency) paths = processor.process() - elif args.instrument == 'glows' and args.data_level in ['l3a', 'l3b']: + elif args.instrument == "glows" and args.data_level in ["l3a", "l3b"]: processor = GlowsProcessor(processing_input_collection, input_dependency) paths = processor.process() - elif args.instrument == 'swe' and args.data_level == 'l3': + elif args.instrument == "swe" and args.data_level == "l3": processor = SweProcessor(processing_input_collection, input_dependency) paths = processor.process() - elif args.instrument == 'hit' and args.data_level == 'l3': + elif args.instrument == "hit" and args.data_level == "l3": processor = HitProcessor(processing_input_collection, input_dependency) paths = processor.process() - elif args.instrument == 'hi' and args.data_level == 'l3': + elif args.instrument == "hi" and args.data_level == "l3": processor = HiProcessor(processing_input_collection, input_dependency) paths = processor.process() - elif args.instrument == 'ultra' and args.data_level == 'l3': + elif args.instrument == "ultra" and args.data_level == "l3": processor = UltraProcessor(processing_input_collection, input_dependency) paths = processor.process() - elif args.instrument == 'lo' and args.data_level == 'l3': + elif args.instrument == "lo" and args.data_level == "l3": processor = LoProcessor(processing_input_collection, input_dependency) paths = processor.process() - elif args.instrument == 'codice': - if args.descriptor.startswith("hi") and args.data_level in ['l3a', 'l3b']: + elif args.instrument == "codice": + if args.descriptor.startswith("hi") and args.data_level in ["l3a", "l3b"]: processor = CodiceHiProcessor(processing_input_collection, input_dependency) paths = processor.process() - elif args.descriptor.startswith("lo") and args.data_level in ['l3a', 'l3b']: + elif args.descriptor.startswith("lo") and args.data_level in ["l3a", "l3b"]: processor = CodiceLoProcessor(processing_input_collection, input_dependency) paths = processor.process() else: - raise NotImplementedError(f"Unknown descriptor '{args.descriptor}' for codice instrument") + raise NotImplementedError( + f"Unknown descriptor '{args.descriptor}' for codice instrument" + ) else: raise NotImplementedError( - f'Level {args.data_level} data processing has not yet been implemented for {args.instrument}') + f"Level {args.data_level} data processing has not yet been implemented for {args.instrument}" + ) if args.upload_to_sdc: exceptions = [] @@ -172,16 +241,19 @@ def imap_l3_processor(): def _furnish_spice_kernels(processing_input_collection): - spice_kernel_paths = processing_input_collection.get_file_paths(data_type='spice') + spice_kernel_paths = processing_input_collection.get_file_paths(data_type="spice") for kernel in spice_kernel_paths: kernel_path = imap_data_access.download(kernel) spiceypy.furnsh(str(kernel_path)) -if __name__ == '__main__': +if __name__ == "__main__": with TemporaryDirectory() as dir: - logging.basicConfig(force=True, level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + logging.basicConfig( + force=True, + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) try: imap_l3_processor() diff --git a/imap_l3_processing/cdf/cdf_utils.py b/imap_l3_processing/cdf/cdf_utils.py index 230b435e5..39490bf8d 100644 --- a/imap_l3_processing/cdf/cdf_utils.py +++ b/imap_l3_processing/cdf/cdf_utils.py @@ -14,11 +14,15 @@ } -def write_cdf(file_path: str, data: DataProduct, attribute_manager: ImapAttributeManager): - with CDF(file_path, '') as cdf: +def write_cdf( + file_path: str, data: DataProduct, attribute_manager: ImapAttributeManager +): + with CDF(file_path, "") as cdf: cdf.col_major(True) try: - global_attrs = attribute_manager.get_global_attributes(data.input_metadata.logical_source) + global_attrs = attribute_manager.get_global_attributes( + data.input_metadata.logical_source + ) except KeyError: trimmed_source = re.sub(r"-\d+$", "-", data.input_metadata.logical_source) global_attrs = attribute_manager.get_global_attributes(trimmed_source) @@ -27,9 +31,16 @@ def write_cdf(file_path: str, data: DataProduct, attribute_manager: ImapAttribut cdf.attrs[k] = v map_descriptor = None - if data.input_metadata.instrument in ("lo", "hi", "ultra", "glows",): + if data.input_metadata.instrument in ( + "lo", + "hi", + "ultra", + "glows", + ): try: - map_descriptor = naming.MapDescriptor.from_string(data.input_metadata.descriptor) + map_descriptor = naming.MapDescriptor.from_string( + data.input_metadata.descriptor + ) except ValueError: # Assume this is not actually a map pass @@ -54,48 +65,58 @@ def write_cdf(file_path: str, data: DataProduct, attribute_manager: ImapAttribut data_array = None else: target_float_dtype = _CDF_FLOAT_NUMPY_DTYPE.get(data_type.value) - if target_float_dtype is not None and np.issubdtype(data_array.dtype, np.floating): + if target_float_dtype is not None and np.issubdtype( + data_array.dtype, np.floating + ): data_array = data_array.astype(target_float_dtype) - if 'FILLVAL' in variable_attributes: + if "FILLVAL" in variable_attributes: if np.issubdtype(data_array.dtype, np.floating): data_array = np.ma.masked_invalid(data_array) - data_array = np.ma.filled(data_array, variable_attributes['FILLVAL']) + data_array = np.ma.filled( + data_array, variable_attributes["FILLVAL"] + ) else: assert not np.ma.isMaskedArray(data_array) if ( - data_type - not in (pycdf.const.CDF_TIME_TT2000, pycdf.const.CDF_EPOCH, - pycdf.const.CDF_EPOCH16) - and record_varying + data_type + not in ( + pycdf.const.CDF_TIME_TT2000, + pycdf.const.CDF_EPOCH, + pycdf.const.CDF_EPOCH16, + ) + and record_varying ): compress = pycdf.const.GZIP_COMPRESSION compress_param = 7 else: compress = None compress_param = None - cdf.new(var_name, data_array, - recVary=record_varying, - type=data_type, - dims=dims, - compress=compress, - compress_param=compress_param) + cdf.new( + var_name, + data_array, + recVary=record_varying, + type=data_type, + dims=dims, + compress=compress, + compress_param=compress_param, + ) for k, v in variable_attributes.items(): - if v == '': + if v == "": continue - if k in ['DATA_TYPE', 'RECORD_VARYING']: + if k in ["DATA_TYPE", "RECORD_VARYING"]: continue - if k == 'FILLVAL' and data_type is not None: + if k == "FILLVAL" and data_type is not None: cdf[var_name].attrs.new(k, v, data_type) else: cdf[var_name].attrs[k] = v def read_variable_and_mask_fill_values(var: pycdf.Var) -> np.ma.masked_array: - return np.ma.masked_equal(var[...], var.attrs['FILLVAL']) + return np.ma.masked_equal(var[...], var.attrs["FILLVAL"]) def read_numeric_variable(var: pycdf.Var) -> np.ndarray: assert np.issubdtype(var.dtype, np.number) - return np.where(var[...] == var.attrs['FILLVAL'], np.nan, var[...]) + return np.where(var[...] == var.attrs["FILLVAL"], np.nan, var[...]) diff --git a/imap_l3_processing/cdf/imap_attribute_manager.py b/imap_l3_processing/cdf/imap_attribute_manager.py index 25c93e923..349354f48 100644 --- a/imap_l3_processing/cdf/imap_attribute_manager.py +++ b/imap_l3_processing/cdf/imap_attribute_manager.py @@ -6,15 +6,25 @@ class ImapAttributeManager(CdfAttributeManager): def __init__(self): - super().__init__(variable_schema_layers=[ - Path(f'{Path(__file__).parent.resolve()}/config/imap_l3_variable_cdf_attrs_schema.yaml'), - Path(f'{Path(__file__).parent.resolve()}/config/default_variable_cdf_attrs_schema.yaml') - ], - global_schema_layers=[Path(__file__).parent / "config/default_global_cdf_attrs_schema.yaml"], - use_defaults=True) - self.config_folder_path = Path(f'{Path(__file__).parent.resolve()}/config') - - self.load_global_attributes(self.config_folder_path / 'imap_default_global_cdf_attrs.yaml') + super().__init__( + variable_schema_layers=[ + Path( + f"{Path(__file__).parent.resolve()}/config/imap_l3_variable_cdf_attrs_schema.yaml" + ), + Path( + f"{Path(__file__).parent.resolve()}/config/default_variable_cdf_attrs_schema.yaml" + ), + ], + global_schema_layers=[ + Path(__file__).parent / "config/default_global_cdf_attrs_schema.yaml" + ], + use_defaults=True, + ) + self.config_folder_path = Path(f"{Path(__file__).parent.resolve()}/config") + + self.load_global_attributes( + self.config_folder_path / "imap_default_global_cdf_attrs.yaml" + ) def try_load_global_metadata(self, logical_source: str) -> Optional[dict]: try: @@ -24,16 +34,23 @@ def try_load_global_metadata(self, logical_source: str) -> Optional[dict]: def add_instrument_attrs(self, instrument: str, level: str, descriptor: str): self._load_variable_attributes_if_file_exists( - self.config_folder_path / f"imap_{instrument}_{level}_variable_attrs.yaml") + self.config_folder_path / f"imap_{instrument}_{level}_variable_attrs.yaml" + ) self._load_variable_attributes_if_file_exists( - self.config_folder_path / f"imap_{instrument}_{level}_{descriptor}_variable_attrs.yaml") + self.config_folder_path + / f"imap_{instrument}_{level}_{descriptor}_variable_attrs.yaml" + ) self._load_global_attributes_if_file_exists( - self.config_folder_path / f"imap_{instrument}_global_cdf_attrs.yaml") + self.config_folder_path / f"imap_{instrument}_global_cdf_attrs.yaml" + ) self._load_global_attributes_if_file_exists( - self.config_folder_path / f"imap_{instrument}_{level}_global_cdf_attrs.yaml") + self.config_folder_path / f"imap_{instrument}_{level}_global_cdf_attrs.yaml" + ) self._load_global_attributes_if_file_exists( - self.config_folder_path / f"imap_{instrument}_{level}_{descriptor}_global_cdf_attrs.yaml") + self.config_folder_path + / f"imap_{instrument}_{level}_{descriptor}_global_cdf_attrs.yaml" + ) def _load_variable_attributes_if_file_exists(self, path: Path): if path.exists(): diff --git a/imap_l3_processing/codice/l3/hi/codice_hi_processor.py b/imap_l3_processing/codice/l3/hi/codice_hi_processor.py index fc9a48337..421d1a6ef 100644 --- a/imap_l3_processing/codice/l3/hi/codice_hi_processor.py +++ b/imap_l3_processing/codice/l3/hi/codice_hi_processor.py @@ -4,40 +4,60 @@ import numpy as np from imap_data_access.processing_input import ProcessingInputCollection -from imap_l3_processing.codice.l3.hi.direct_event.codice_hi_l3a_direct_events_dependencies import \ - CodiceHiL3aDirectEventsDependencies +from imap_l3_processing.codice.l3.hi.direct_event.codice_hi_l3a_direct_events_dependencies import ( + CodiceHiL3aDirectEventsDependencies, +) from imap_l3_processing.codice.l3.hi.models import CodiceHiL3PitchAngleDataProduct from imap_l3_processing.codice.l3.hi.models import CodiceL3HiDirectEvents -from imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies import CodicePitchAngleDependencies -from imap_l3_processing.codice.l3.lo.constants import CODICE_INSTRUMENT_TO_SC_FRAME_SPIN_ANGLE_OFFSET +from imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies import ( + CodicePitchAngleDependencies, +) +from imap_l3_processing.codice.l3.lo.constants import ( + CODICE_INSTRUMENT_TO_SC_FRAME_SPIN_ANGLE_OFFSET, +) from imap_l3_processing.codice.quality_flags import CodiceL3Flags -from imap_l3_processing.hit.l3.sectored_products.science.sectored_products_algorithms import \ - get_sector_unit_vectors +from imap_l3_processing.hit.l3.sectored_products.science.sectored_products_algorithms import ( + get_sector_unit_vectors, +) from imap_l3_processing.models import InputMetadata -from imap_l3_processing.pitch_angles import calculate_unit_vector, calculate_pitch_angle, calculate_gyrophase, \ - rebin_by_pitch_angle_and_gyrophase +from imap_l3_processing.pitch_angles import ( + calculate_unit_vector, + calculate_pitch_angle, + calculate_gyrophase, + rebin_by_pitch_angle_and_gyrophase, +) from imap_l3_processing.processor import Processor from imap_l3_processing.utils import save_data class CodiceHiProcessor(Processor): - def __init__(self, dependencies: ProcessingInputCollection, input_metadata: InputMetadata): + def __init__( + self, dependencies: ProcessingInputCollection, input_metadata: InputMetadata + ): super().__init__(dependencies, input_metadata) def process(self) -> list[Path]: if self.input_metadata.data_level == "l3a": - dependencies = CodiceHiL3aDirectEventsDependencies.fetch_dependencies(self.dependencies) + dependencies = CodiceHiL3aDirectEventsDependencies.fetch_dependencies( + self.dependencies + ) data_product = self.process_l3a_direct_event(dependencies) elif self.input_metadata.data_level == "l3b": - dependencies = CodicePitchAngleDependencies.fetch_dependencies(self.dependencies) + dependencies = CodicePitchAngleDependencies.fetch_dependencies( + self.dependencies + ) data_product = self.process_l3b(dependencies) else: - raise NotImplementedError(f"Unknown data level for CoDICE: {self.input_metadata.data_level}") + raise NotImplementedError( + f"Unknown data level for CoDICE: {self.input_metadata.data_level}" + ) data_product.parent_file_names = self.get_parent_file_names() return [save_data(data_product)] - def process_l3a_direct_event(self, dependencies: CodiceHiL3aDirectEventsDependencies) -> CodiceL3HiDirectEvents: + def process_l3a_direct_event( + self, dependencies: CodiceHiL3aDirectEventsDependencies + ) -> CodiceL3HiDirectEvents: l2_data = dependencies.codice_l2_hi_data l1a_data = dependencies.codice_l1a_hi_data hi_mass_correction_lookup = dependencies.codice_hi_mass_correction_lut_ancillary @@ -59,20 +79,18 @@ def process_l3a_direct_event(self, dependencies: CodiceHiL3aDirectEventsDependen correction_factors[high_gain] = [ hi_mass_correction_lookup.lookup_correction_factor( - ssd_id = ssd, - energy_channel = energy, - tof = tof_channel + ssd_id=ssd, energy_channel=energy, tof=tof_channel ) for ssd, energy, tof_channel in zip( - ssd_id[high_gain], - energy_channel[high_gain], - tof[high_gain] + ssd_id[high_gain], energy_channel[high_gain], tof[high_gain] ) ] incident_ion_energy[i, j, :npts] = l2_energy * correction_factors - estimated_mass[i, j, :npts] = incident_ion_energy[i, j, :npts] / l2_data.energy_per_nuc[i, j, :npts] - + estimated_mass[i, j, :npts] = ( + incident_ion_energy[i, j, :npts] + / l2_data.energy_per_nuc[i, j, :npts] + ) return CodiceL3HiDirectEvents( input_metadata=self.input_metadata, @@ -95,21 +113,36 @@ def process_l3a_direct_event(self, dependencies: CodiceHiL3aDirectEventsDependen spin_sector=l2_data.spin_sector, ) - def process_l3b(self, dependencies: CodicePitchAngleDependencies) -> CodiceHiL3PitchAngleDataProduct: + def process_l3b( + self, dependencies: CodicePitchAngleDependencies + ) -> CodiceHiL3PitchAngleDataProduct: mag_data = dependencies.mag_data sectored_intensities = dependencies.codice_sectored_intensities_data epochs = dependencies.codice_sectored_intensities_data.epoch h_energy_bins = dependencies.codice_sectored_intensities_data.energy_h.shape[0] - he3he4_energy_bins = dependencies.codice_sectored_intensities_data.energy_he3he4.shape[0] - cno_energy_bins = dependencies.codice_sectored_intensities_data.energy_cno.shape[0] - fe_energy_bins = dependencies.codice_sectored_intensities_data.energy_fe.shape[0] - rebinned_mag_data = mag_data.rebin_to(sectored_intensities.epoch, sectored_intensities.epoch_delta_plus) + he3he4_energy_bins = ( + dependencies.codice_sectored_intensities_data.energy_he3he4.shape[0] + ) + cno_energy_bins = ( + dependencies.codice_sectored_intensities_data.energy_cno.shape[0] + ) + fe_energy_bins = dependencies.codice_sectored_intensities_data.energy_fe.shape[ + 0 + ] + rebinned_mag_data = mag_data.rebin_to( + sectored_intensities.epoch, sectored_intensities.epoch_delta_plus + ) - spin_angles_in_dps_elevation_by_spin_angle = (sectored_intensities.spin_angles.T + CODICE_INSTRUMENT_TO_SC_FRAME_SPIN_ANGLE_OFFSET) % 360 + spin_angles_in_dps_elevation_by_spin_angle = ( + sectored_intensities.spin_angles.T + + CODICE_INSTRUMENT_TO_SC_FRAME_SPIN_ANGLE_OFFSET + ) % 360 mag_unit_vectors = calculate_unit_vector(rebinned_mag_data) - sector_unit = get_sector_unit_vectors(sectored_intensities.elevation_angle, - spin_angles_in_dps_elevation_by_spin_angle).transpose(1,0,2) + sector_unit = get_sector_unit_vectors( + sectored_intensities.elevation_angle, + spin_angles_in_dps_elevation_by_spin_angle, + ).transpose(1, 0, 2) sector_unit_vectors = calculate_unit_vector(sector_unit) particle_unit_vectors = -1 * sector_unit_vectors @@ -117,27 +150,68 @@ def process_l3b(self, dependencies: CodicePitchAngleDependencies) -> CodiceHiL3P num_gyrophase_bins = 12 pitch_angle_delta_value = (180 / num_pitch_angle_bins) / 2 gyrophase_delta_value = (360 / num_gyrophase_bins) / 2 - pitch_angle_bins = np.linspace(0, 180, num_pitch_angle_bins, endpoint=False) + pitch_angle_delta_value - gyrophase_bins = np.linspace(0, 360, num_gyrophase_bins, endpoint=False) + gyrophase_delta_value + pitch_angle_bins = ( + np.linspace(0, 180, num_pitch_angle_bins, endpoint=False) + + pitch_angle_delta_value + ) + gyrophase_bins = ( + np.linspace(0, 360, num_gyrophase_bins, endpoint=False) + + gyrophase_delta_value + ) h_pa_shape = (len(epochs), h_energy_bins, num_pitch_angle_bins) he3he4_pa_shape = (len(epochs), he3he4_energy_bins, num_pitch_angle_bins) cno_pa_shape = (len(epochs), cno_energy_bins, num_pitch_angle_bins) fe_pa_shape = (len(epochs), fe_energy_bins, num_pitch_angle_bins) - h_gyro_shape = (len(epochs), h_energy_bins, num_pitch_angle_bins, num_gyrophase_bins) - he3he4_gyro_shape = (len(epochs), he3he4_energy_bins, num_pitch_angle_bins, num_gyrophase_bins) - cno_gyro_shape = (len(epochs), cno_energy_bins, num_pitch_angle_bins, num_gyrophase_bins) - fe_gyro_shape = (len(epochs), fe_energy_bins, num_pitch_angle_bins, num_gyrophase_bins) + h_gyro_shape = ( + len(epochs), + h_energy_bins, + num_pitch_angle_bins, + num_gyrophase_bins, + ) + he3he4_gyro_shape = ( + len(epochs), + he3he4_energy_bins, + num_pitch_angle_bins, + num_gyrophase_bins, + ) + cno_gyro_shape = ( + len(epochs), + cno_energy_bins, + num_pitch_angle_bins, + num_gyrophase_bins, + ) + fe_gyro_shape = ( + len(epochs), + fe_energy_bins, + num_pitch_angle_bins, + num_gyrophase_bins, + ) - SpeciesIntensity = namedtuple("SpeciesIntensity", - ["l2_intensity", "intensity_by_pa", "intensity_by_pa_and_gyro"]) + SpeciesIntensity = namedtuple( + "SpeciesIntensity", + ["l2_intensity", "intensity_by_pa", "intensity_by_pa_and_gyro"], + ) # @formatter:off species_intensities: dict[str, SpeciesIntensity] = { - "h": SpeciesIntensity(sectored_intensities.h_intensities, *_create_pa_and_gyro_nan_arrays(h_pa_shape, h_gyro_shape)), - "he4": SpeciesIntensity(sectored_intensities.he3he4_intensities, *_create_pa_and_gyro_nan_arrays(he3he4_pa_shape, he3he4_gyro_shape)), - "o": SpeciesIntensity(sectored_intensities.cno_intensities, *_create_pa_and_gyro_nan_arrays(cno_pa_shape, cno_gyro_shape)), - "fe": SpeciesIntensity(sectored_intensities.fe_intensities,*_create_pa_and_gyro_nan_arrays(fe_pa_shape, fe_gyro_shape))} + "h": SpeciesIntensity( + sectored_intensities.h_intensities, + *_create_pa_and_gyro_nan_arrays(h_pa_shape, h_gyro_shape), + ), + "he4": SpeciesIntensity( + sectored_intensities.he3he4_intensities, + *_create_pa_and_gyro_nan_arrays(he3he4_pa_shape, he3he4_gyro_shape), + ), + "o": SpeciesIntensity( + sectored_intensities.cno_intensities, + *_create_pa_and_gyro_nan_arrays(cno_pa_shape, cno_gyro_shape), + ), + "fe": SpeciesIntensity( + sectored_intensities.fe_intensities, + *_create_pa_and_gyro_nan_arrays(fe_pa_shape, fe_gyro_shape), + ), + } # @formatter:on codice_flags = np.full(len(epochs), CodiceL3Flags.NONE) @@ -145,8 +219,12 @@ def process_l3b(self, dependencies: CodicePitchAngleDependencies) -> CodiceHiL3P codice_flags |= CodiceL3Flags.PRELIMINARY_MAG for time_index in range(len(epochs)): - pitch_angles = calculate_pitch_angle(particle_unit_vectors, mag_unit_vectors[time_index]) - gyrophases = calculate_gyrophase(particle_unit_vectors, mag_unit_vectors[time_index]) + pitch_angles = calculate_pitch_angle( + particle_unit_vectors, mag_unit_vectors[time_index] + ) + gyrophases = calculate_gyrophase( + particle_unit_vectors, mag_unit_vectors[time_index] + ) for species, species_intensity in species_intensities.items(): intensity = species_intensity.l2_intensity[time_index] @@ -154,13 +232,29 @@ def process_l3b(self, dependencies: CodicePitchAngleDependencies) -> CodiceHiL3P intensities_delta_plus = np.full_like(intensity, 0) intensities_delta_minus = np.full_like(intensity, 0) - rebinned_intensities_by_pa_and_gyro, _, _, rebinned_intensities_by_pa, _, _ = rebin_by_pitch_angle_and_gyrophase( - intensity_data=intensity, intensity_delta_plus=intensities_delta_plus, - intensity_delta_minus=intensities_delta_minus, pitch_angles=pitch_angles, gyrophases=gyrophases, - number_of_pitch_angle_bins=num_pitch_angle_bins, number_of_gyrophase_bins=num_gyrophase_bins) + ( + rebinned_intensities_by_pa_and_gyro, + _, + _, + rebinned_intensities_by_pa, + _, + _, + ) = rebin_by_pitch_angle_and_gyrophase( + intensity_data=intensity, + intensity_delta_plus=intensities_delta_plus, + intensity_delta_minus=intensities_delta_minus, + pitch_angles=pitch_angles, + gyrophases=gyrophases, + number_of_pitch_angle_bins=num_pitch_angle_bins, + number_of_gyrophase_bins=num_gyrophase_bins, + ) - species_intensity.intensity_by_pa[time_index] = rebinned_intensities_by_pa - species_intensity.intensity_by_pa_and_gyro[time_index] = rebinned_intensities_by_pa_and_gyro + species_intensity.intensity_by_pa[time_index] = ( + rebinned_intensities_by_pa + ) + species_intensity.intensity_by_pa_and_gyro[time_index] = ( + rebinned_intensities_by_pa_and_gyro + ) return CodiceHiL3PitchAngleDataProduct( input_metadata=self.input_metadata, @@ -182,17 +276,29 @@ def process_l3b(self, dependencies: CodicePitchAngleDependencies) -> CodiceHiL3P pitch_angle_delta=np.repeat(pitch_angle_delta_value, num_pitch_angle_bins), gyrophase=gyrophase_bins, gyrophase_delta=np.repeat(gyrophase_delta_value, num_gyrophase_bins), - h_intensity_by_pitch_angle=species_intensities['h'].intensity_by_pa, - h_intensity_by_pitch_angle_and_gyrophase=species_intensities['h'].intensity_by_pa_and_gyro, - he3he4_intensity_by_pitch_angle=species_intensities['he4'].intensity_by_pa, - he3he4_intensity_by_pitch_angle_and_gyrophase=species_intensities['he4'].intensity_by_pa_and_gyro, - cno_intensity_by_pitch_angle=species_intensities['o'].intensity_by_pa, - cno_intensity_by_pitch_angle_and_gyrophase=species_intensities['o'].intensity_by_pa_and_gyro, - fe_intensity_by_pitch_angle=species_intensities['fe'].intensity_by_pa, - fe_intensity_by_pitch_angle_and_gyrophase=species_intensities['fe'].intensity_by_pa_and_gyro, + h_intensity_by_pitch_angle=species_intensities["h"].intensity_by_pa, + h_intensity_by_pitch_angle_and_gyrophase=species_intensities[ + "h" + ].intensity_by_pa_and_gyro, + he3he4_intensity_by_pitch_angle=species_intensities["he4"].intensity_by_pa, + he3he4_intensity_by_pitch_angle_and_gyrophase=species_intensities[ + "he4" + ].intensity_by_pa_and_gyro, + cno_intensity_by_pitch_angle=species_intensities["o"].intensity_by_pa, + cno_intensity_by_pitch_angle_and_gyrophase=species_intensities[ + "o" + ].intensity_by_pa_and_gyro, + fe_intensity_by_pitch_angle=species_intensities["fe"].intensity_by_pa, + fe_intensity_by_pitch_angle_and_gyrophase=species_intensities[ + "fe" + ].intensity_by_pa_and_gyro, codice_flags=codice_flags, ) -def _create_pa_and_gyro_nan_arrays(pitch_angle_shape, pitch_angle_and_gyrophase_shape) -> np.array: - return np.full(pitch_angle_shape, fill_value=np.nan), np.full(pitch_angle_and_gyrophase_shape, fill_value=np.nan) +def _create_pa_and_gyro_nan_arrays( + pitch_angle_shape, pitch_angle_and_gyrophase_shape +) -> np.array: + return np.full(pitch_angle_shape, fill_value=np.nan), np.full( + pitch_angle_and_gyrophase_shape, fill_value=np.nan + ) diff --git a/imap_l3_processing/codice/l3/hi/direct_event/codice_hi_l3a_direct_events_dependencies.py b/imap_l3_processing/codice/l3/hi/direct_event/codice_hi_l3a_direct_events_dependencies.py index 621bd3d72..4ac6fca7d 100644 --- a/imap_l3_processing/codice/l3/hi/direct_event/codice_hi_l3a_direct_events_dependencies.py +++ b/imap_l3_processing/codice/l3/hi/direct_event/codice_hi_l3a_direct_events_dependencies.py @@ -4,9 +4,13 @@ from imap_data_access import download from imap_data_access.processing_input import ProcessingInputCollection -from imap_l3_processing.codice.l3.hi.direct_event.codice_hi_mass_correction_lookup_table import \ - CodiceHiMassCorrectionLookupTable -from imap_l3_processing.codice.l3.hi.models import CodiceL2HiDirectEventData, CodiceL1aHiDirectEvents +from imap_l3_processing.codice.l3.hi.direct_event.codice_hi_mass_correction_lookup_table import ( + CodiceHiMassCorrectionLookupTable, +) +from imap_l3_processing.codice.l3.hi.models import ( + CodiceL2HiDirectEventData, + CodiceL1aHiDirectEvents, +) @dataclass @@ -17,19 +21,46 @@ class CodiceHiL3aDirectEventsDependencies: @classmethod def fetch_dependencies(cls, dependencies: ProcessingInputCollection): - l2_hi_direct_events_file_paths = dependencies.get_file_paths("codice", "hi-direct-events", data_type="l2") - l1a_hi_direct_events_file_paths = dependencies.get_file_paths("codice", "hi-direct-events", data_type="l1a") - ancillary_file_paths = dependencies.get_file_paths("codice", "l3-hi-mass-correction-lut") + l2_hi_direct_events_file_paths = dependencies.get_file_paths( + "codice", "hi-direct-events", data_type="l2" + ) + l1a_hi_direct_events_file_paths = dependencies.get_file_paths( + "codice", "hi-direct-events", data_type="l1a" + ) + ancillary_file_paths = dependencies.get_file_paths( + "codice", "l3-hi-mass-correction-lut" + ) - for download_location_file_path in [*l2_hi_direct_events_file_paths, *l1a_hi_direct_events_file_paths, *ancillary_file_paths]: + for download_location_file_path in [ + *l2_hi_direct_events_file_paths, + *l1a_hi_direct_events_file_paths, + *ancillary_file_paths, + ]: download(download_location_file_path) - return cls.from_file_paths(l2_hi_direct_events_file_paths[0], l1a_hi_direct_events_file_paths[0], ancillary_file_paths[0]) + return cls.from_file_paths( + l2_hi_direct_events_file_paths[0], + l1a_hi_direct_events_file_paths[0], + ancillary_file_paths[0], + ) @classmethod - def from_file_paths(cls, codice_l2_hi_cdf: Path, codice_l1a_hi_cdf: Path, codice_hi_mass_correction_lut_ancillary: Path): + def from_file_paths( + cls, + codice_l2_hi_cdf: Path, + codice_l1a_hi_cdf: Path, + codice_hi_mass_correction_lut_ancillary: Path, + ): codice_l2_hi_data = CodiceL2HiDirectEventData.read_from_cdf(codice_l2_hi_cdf) codice_l1a_hi_data = CodiceL1aHiDirectEvents.read_from_cdf(codice_l1a_hi_cdf) - codice_hi_mass_correction_lut_ancillary = CodiceHiMassCorrectionLookupTable.from_file(codice_hi_mass_correction_lut_ancillary) + codice_hi_mass_correction_lut_ancillary = ( + CodiceHiMassCorrectionLookupTable.from_file( + codice_hi_mass_correction_lut_ancillary + ) + ) - return cls(codice_l2_hi_data, codice_l1a_hi_data, codice_hi_mass_correction_lut_ancillary) + return cls( + codice_l2_hi_data, + codice_l1a_hi_data, + codice_hi_mass_correction_lut_ancillary, + ) diff --git a/imap_l3_processing/codice/l3/hi/direct_event/codice_hi_mass_correction_lookup_table.py b/imap_l3_processing/codice/l3/hi/direct_event/codice_hi_mass_correction_lookup_table.py index df0c5292d..2c5133b93 100644 --- a/imap_l3_processing/codice/l3/hi/direct_event/codice_hi_mass_correction_lookup_table.py +++ b/imap_l3_processing/codice/l3/hi/direct_event/codice_hi_mass_correction_lookup_table.py @@ -8,6 +8,7 @@ SSD_SHEET_NAME_FORMAT = "ssd(\d+)" + @dataclass class CodiceHiMassCorrectionLookupTable: correction_factors: dict[int, np.ndarray] @@ -23,15 +24,21 @@ def from_file(cls, file_path: Path): ssd = int(ssd_name_match.group(1)) data[ssd] = file[sheet_name].to_numpy() else: - logging.warning(f"Not using sheet {sheet_name} from {file_path.name} ancillary! Expected sheet names in the format: {SSD_SHEET_NAME_FORMAT}") + logging.warning( + f"Not using sheet {sheet_name} from {file_path.name} ancillary! Expected sheet names in the format: {SSD_SHEET_NAME_FORMAT}" + ) - assert len(data) > 0, f"Failed to read any correction factors from: {file_path.name}" + assert len(data) > 0, ( + f"Failed to read any correction factors from: {file_path.name}" + ) return cls(correction_factors=data) - def lookup_correction_factor(self, ssd_id: int, energy_channel: int, tof: int) -> float: + def lookup_correction_factor( + self, ssd_id: int, energy_channel: int, tof: int + ) -> float: data_for_ssd = self.correction_factors.get(ssd_id) if data_for_ssd is not None: return data_for_ssd[energy_channel, tof] else: - return 1.0 \ No newline at end of file + return 1.0 diff --git a/imap_l3_processing/codice/l3/hi/models.py b/imap_l3_processing/codice/l3/hi/models.py index abae7993b..5cd5fd0c6 100644 --- a/imap_l3_processing/codice/l3/hi/models.py +++ b/imap_l3_processing/codice/l3/hi/models.py @@ -5,7 +5,10 @@ from numpy import ndarray from spacepy.pycdf import CDF -from imap_l3_processing.cdf.cdf_utils import read_numeric_variable, read_variable_and_mask_fill_values +from imap_l3_processing.cdf.cdf_utils import ( + read_numeric_variable, + read_variable_and_mask_fill_values, +) from imap_l3_processing.models import DataProduct, DataProductVariable CODICE_HI_NUM_L2_PRIORITIES = 6 @@ -39,20 +42,46 @@ def read_from_cdf(cls, filename): with CDF(str(filename)) as cdf: return cls( epoch=cdf["epoch"][...], - epoch_delta_plus=cdf['epoch_delta_plus'][...], - data_quality=read_variable_and_mask_fill_values(cdf["data_quality"])[:, :CODICE_HI_NUM_L2_PRIORITIES, ...], - multi_flag=read_variable_and_mask_fill_values(cdf["multi_flag"])[:, :CODICE_HI_NUM_L2_PRIORITIES, ...], - number_of_events=read_variable_and_mask_fill_values(cdf["num_events"])[:, :CODICE_HI_NUM_L2_PRIORITIES, ...], - ssd_energy=read_variable_and_mask_fill_values(cdf["ssd_energy"])[:, :CODICE_HI_NUM_L2_PRIORITIES, ...], - ssd_id=read_variable_and_mask_fill_values(cdf["ssd_id"])[:, :CODICE_HI_NUM_L2_PRIORITIES, ...], - spin_angle=read_numeric_variable(cdf["spin_angle"])[:, :CODICE_HI_NUM_L2_PRIORITIES, ...], - spin_number=read_variable_and_mask_fill_values(cdf["spin_number"])[:, :CODICE_HI_NUM_L2_PRIORITIES, ...], - time_of_flight=read_numeric_variable(cdf["tof"])[:, :CODICE_HI_NUM_L2_PRIORITIES, ...], - type=read_variable_and_mask_fill_values(cdf["type"])[:, :CODICE_HI_NUM_L2_PRIORITIES, ...], - energy_per_nuc=read_numeric_variable(cdf["energy_per_nuc"])[:, :CODICE_HI_NUM_L2_PRIORITIES, ...], - elevation_angle=read_numeric_variable(cdf[ELEVATION_ANGLE_VAR_NAME])[:, :CODICE_HI_NUM_L2_PRIORITIES, ...], - gain=read_variable_and_mask_fill_values(cdf[GAIN_VAR_NAME])[:, :CODICE_HI_NUM_L2_PRIORITIES, ...], - spin_sector=read_variable_and_mask_fill_values(cdf[SPIN_SECTOR_VAR_NAME])[:, :CODICE_HI_NUM_L2_PRIORITIES, ...], + epoch_delta_plus=cdf["epoch_delta_plus"][...], + data_quality=read_variable_and_mask_fill_values(cdf["data_quality"])[ + :, :CODICE_HI_NUM_L2_PRIORITIES, ... + ], + multi_flag=read_variable_and_mask_fill_values(cdf["multi_flag"])[ + :, :CODICE_HI_NUM_L2_PRIORITIES, ... + ], + number_of_events=read_variable_and_mask_fill_values(cdf["num_events"])[ + :, :CODICE_HI_NUM_L2_PRIORITIES, ... + ], + ssd_energy=read_variable_and_mask_fill_values(cdf["ssd_energy"])[ + :, :CODICE_HI_NUM_L2_PRIORITIES, ... + ], + ssd_id=read_variable_and_mask_fill_values(cdf["ssd_id"])[ + :, :CODICE_HI_NUM_L2_PRIORITIES, ... + ], + spin_angle=read_numeric_variable(cdf["spin_angle"])[ + :, :CODICE_HI_NUM_L2_PRIORITIES, ... + ], + spin_number=read_variable_and_mask_fill_values(cdf["spin_number"])[ + :, :CODICE_HI_NUM_L2_PRIORITIES, ... + ], + time_of_flight=read_numeric_variable(cdf["tof"])[ + :, :CODICE_HI_NUM_L2_PRIORITIES, ... + ], + type=read_variable_and_mask_fill_values(cdf["type"])[ + :, :CODICE_HI_NUM_L2_PRIORITIES, ... + ], + energy_per_nuc=read_numeric_variable(cdf["energy_per_nuc"])[ + :, :CODICE_HI_NUM_L2_PRIORITIES, ... + ], + elevation_angle=read_numeric_variable(cdf[ELEVATION_ANGLE_VAR_NAME])[ + :, :CODICE_HI_NUM_L2_PRIORITIES, ... + ], + gain=read_variable_and_mask_fill_values(cdf[GAIN_VAR_NAME])[ + :, :CODICE_HI_NUM_L2_PRIORITIES, ... + ], + spin_sector=read_variable_and_mask_fill_values( + cdf[SPIN_SECTOR_VAR_NAME] + )[:, :CODICE_HI_NUM_L2_PRIORITIES, ...], ) # @formatter:on @@ -95,13 +124,21 @@ def read_from_cdf(cls, filename): GYROPHASE_VAR_NAME = "gyrophase" GYROPHASE_DELTA_VAR_NAME = "gyrophase_delta" H_INTENSITY_BY_PITCH_ANGLE_VAR_NAME = "h_intensity_by_pitch_angle" -H_INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_VAR_NAME = "h_intensity_by_pitch_angle_and_gyrophase" +H_INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_VAR_NAME = ( + "h_intensity_by_pitch_angle_and_gyrophase" +) HE3HE4_INTENSITY_BY_PITCH_ANGLE_VAR_NAME = "he3he4_intensity_by_pitch_angle" -HE3HE4_INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_VAR_NAME = "he3he4_intensity_by_pitch_angle_and_gyrophase" +HE3HE4_INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_VAR_NAME = ( + "he3he4_intensity_by_pitch_angle_and_gyrophase" +) CNO_INTENSITY_BY_PITCH_ANGLE_VAR_NAME = "cno_intensity_by_pitch_angle" -CNO_INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_VAR_NAME = "cno_intensity_by_pitch_angle_and_gyrophase" +CNO_INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_VAR_NAME = ( + "cno_intensity_by_pitch_angle_and_gyrophase" +) FE_INTENSITY_BY_PITCH_ANGLE_VAR_NAME = "fe_intensity_by_pitch_angle" -FE_INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_VAR_NAME = "fe_intensity_by_pitch_angle_and_gyrophase" +FE_INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_VAR_NAME = ( + "fe_intensity_by_pitch_angle_and_gyrophase" +) ENERGY_H_LABEL_VAR_NAME = "energy_h_label" ENERGY_CNO_LABEL_VAR_NAME = "energy_cno_label" ENERGY_FE_LABEL_VAR_NAME = "energy_fe_label" @@ -138,8 +175,12 @@ class CodiceL3HiDirectEvents(DataProduct): def __post_init__(self): self.priority_index = np.arange(CODICE_HI_NUM_L2_PRIORITIES) self.event_index = np.arange(self.ssd_id.shape[-1]) - self.priority_index_label = np.array([str(i) for i in range(CODICE_HI_NUM_L2_PRIORITIES)]) - self.event_index_label = np.array([str(i) for i in range(len(self.event_index))]) + self.priority_index_label = np.array( + [str(i) for i in range(CODICE_HI_NUM_L2_PRIORITIES)] + ) + self.event_index_label = np.array( + [str(i) for i in range(len(self.event_index))] + ) def to_data_product_variables(self) -> list[DataProductVariable]: return [ @@ -162,7 +203,9 @@ def to_data_product_variables(self) -> list[DataProductVariable]: DataProductVariable(SPIN_SECTOR_VAR_NAME, self.spin_sector), DataProductVariable(PRIORITY_INDEX_VAR_NAME, self.priority_index), DataProductVariable(EVENT_INDEX_VAR_NAME, self.event_index), - DataProductVariable(PRIORITY_INDEX_LABEL_VAR_NAME, self.priority_index_label), + DataProductVariable( + PRIORITY_INDEX_LABEL_VAR_NAME, self.priority_index_label + ), DataProductVariable(EVENT_INDEX_LABEL_VAR_NAME, self.event_index_label), ] @@ -215,7 +258,10 @@ def __post_init__(self): def to_data_product_variables(self) -> list[DataProductVariable]: return [ DataProductVariable(EPOCH_VAR_NAME, self.epoch), - DataProductVariable(EPOCH_DELTA_VAR_NAME, np.array([t.total_seconds() for t in self.epoch_delta]) * 1e9), + DataProductVariable( + EPOCH_DELTA_VAR_NAME, + np.array([t.total_seconds() for t in self.epoch_delta]) * 1e9, + ), DataProductVariable(ENERGY_H_VAR_NAME, self.energy_h), DataProductVariable(ENERGY_H_PLUS_VAR_NAME, self.energy_h_plus), DataProductVariable(ENERGY_H_MINUS_VAR_NAME, self.energy_h_minus), @@ -232,18 +278,35 @@ def to_data_product_variables(self) -> list[DataProductVariable]: DataProductVariable(PITCH_ANGLE_DELTA_VAR_NAME, self.pitch_angle_delta), DataProductVariable(GYROPHASE_VAR_NAME, self.gyrophase), DataProductVariable(GYROPHASE_DELTA_VAR_NAME, self.gyrophase_delta), - DataProductVariable(H_INTENSITY_BY_PITCH_ANGLE_VAR_NAME, self.h_intensity_by_pitch_angle), - DataProductVariable(H_INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_VAR_NAME, - self.h_intensity_by_pitch_angle_and_gyrophase), - DataProductVariable(HE3HE4_INTENSITY_BY_PITCH_ANGLE_VAR_NAME, self.he3he4_intensity_by_pitch_angle), - DataProductVariable(HE3HE4_INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_VAR_NAME, - self.he3he4_intensity_by_pitch_angle_and_gyrophase), - DataProductVariable(CNO_INTENSITY_BY_PITCH_ANGLE_VAR_NAME, self.cno_intensity_by_pitch_angle), - DataProductVariable(CNO_INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_VAR_NAME, - self.cno_intensity_by_pitch_angle_and_gyrophase), - DataProductVariable(FE_INTENSITY_BY_PITCH_ANGLE_VAR_NAME, self.fe_intensity_by_pitch_angle), - DataProductVariable(FE_INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_VAR_NAME, - self.fe_intensity_by_pitch_angle_and_gyrophase), + DataProductVariable( + H_INTENSITY_BY_PITCH_ANGLE_VAR_NAME, self.h_intensity_by_pitch_angle + ), + DataProductVariable( + H_INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_VAR_NAME, + self.h_intensity_by_pitch_angle_and_gyrophase, + ), + DataProductVariable( + HE3HE4_INTENSITY_BY_PITCH_ANGLE_VAR_NAME, + self.he3he4_intensity_by_pitch_angle, + ), + DataProductVariable( + HE3HE4_INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_VAR_NAME, + self.he3he4_intensity_by_pitch_angle_and_gyrophase, + ), + DataProductVariable( + CNO_INTENSITY_BY_PITCH_ANGLE_VAR_NAME, self.cno_intensity_by_pitch_angle + ), + DataProductVariable( + CNO_INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_VAR_NAME, + self.cno_intensity_by_pitch_angle_and_gyrophase, + ), + DataProductVariable( + FE_INTENSITY_BY_PITCH_ANGLE_VAR_NAME, self.fe_intensity_by_pitch_angle + ), + DataProductVariable( + FE_INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_VAR_NAME, + self.fe_intensity_by_pitch_angle_and_gyrophase, + ), DataProductVariable(ENERGY_H_LABEL_VAR_NAME, self.energy_h_label), DataProductVariable(ENERGY_CNO_LABEL_VAR_NAME, self.energy_cno_label), DataProductVariable(ENERGY_FE_LABEL_VAR_NAME, self.energy_fe_label), @@ -281,28 +344,32 @@ class CodiceHiL2SectoredIntensitiesData: @classmethod def read_from_cdf(cls, l2_sectored_intensities_cdf): with CDF(str(l2_sectored_intensities_cdf)) as cdf: - return cls(epoch=cdf["epoch"][...], - epoch_delta_plus=np.array([timedelta(seconds=ns / 1e9) for ns in cdf["epoch_delta_plus"][...]]), - spin_angles=cdf['spin_angle'][...], - elevation_angle=cdf['elevation_angle'][...], - data_quality=cdf['data_quality'][...], - h_intensities=read_numeric_variable(cdf['h']), - energy_h=cdf['energy_h'][...], - energy_h_plus=cdf['energy_h_plus'][...], - energy_h_minus=cdf['energy_h_minus'][...], - cno_intensities=read_numeric_variable(cdf['cno']), - energy_cno=cdf['energy_cno'][...], - energy_cno_plus=cdf['energy_cno_plus'][...], - energy_cno_minus=cdf['energy_cno_minus'][...], - fe_intensities=read_numeric_variable(cdf['fe']), - energy_fe=cdf['energy_fe'][...], - energy_fe_plus=cdf['energy_fe_plus'][...], - energy_fe_minus=cdf['energy_fe_minus'][...], - he3he4_intensities=read_numeric_variable(cdf['he3he4']), - energy_he3he4=cdf['energy_he3he4'][...], - energy_he3he4_plus=cdf['energy_he3he4_plus'][...], - energy_he3he4_minus=cdf['energy_he3he4_minus'][...], - ) + return cls( + epoch=cdf["epoch"][...], + epoch_delta_plus=np.array( + [timedelta(seconds=ns / 1e9) for ns in cdf["epoch_delta_plus"][...]] + ), + spin_angles=cdf["spin_angle"][...], + elevation_angle=cdf["elevation_angle"][...], + data_quality=cdf["data_quality"][...], + h_intensities=read_numeric_variable(cdf["h"]), + energy_h=cdf["energy_h"][...], + energy_h_plus=cdf["energy_h_plus"][...], + energy_h_minus=cdf["energy_h_minus"][...], + cno_intensities=read_numeric_variable(cdf["cno"]), + energy_cno=cdf["energy_cno"][...], + energy_cno_plus=cdf["energy_cno_plus"][...], + energy_cno_minus=cdf["energy_cno_minus"][...], + fe_intensities=read_numeric_variable(cdf["fe"]), + energy_fe=cdf["energy_fe"][...], + energy_fe_plus=cdf["energy_fe_plus"][...], + energy_fe_minus=cdf["energy_fe_minus"][...], + he3he4_intensities=read_numeric_variable(cdf["he3he4"]), + energy_he3he4=cdf["energy_he3he4"][...], + energy_he3he4_plus=cdf["energy_he3he4_plus"][...], + energy_he3he4_minus=cdf["energy_he3he4_minus"][...], + ) + @dataclass class CodiceL1aHiDirectEvents: @@ -316,5 +383,5 @@ def read_from_cdf(cls, l1a_hi_direct_events_cdf): return cls( epoch=cdf["epoch"][...], ssd_energy=cdf["ssd_energy"][...], - tof=cdf["tof"][...] + tof=cdf["tof"][...], ) diff --git a/imap_l3_processing/codice/l3/hi/pitch_angle/codice_pitch_angle_dependencies.py b/imap_l3_processing/codice/l3/hi/pitch_angle/codice_pitch_angle_dependencies.py index 89709ac89..8209251d4 100644 --- a/imap_l3_processing/codice/l3/hi/pitch_angle/codice_pitch_angle_dependencies.py +++ b/imap_l3_processing/codice/l3/hi/pitch_angle/codice_pitch_angle_dependencies.py @@ -17,19 +17,34 @@ class CodicePitchAngleDependencies: @classmethod def fetch_dependencies(cls, input_collection: ProcessingInputCollection): codice_file_paths = input_collection.get_file_paths("codice", "hi-sectored") - mag_l2_paths = input_collection.get_file_paths("mag", data_type="l2", descriptor="norm-dsrf") - mag_l1d_paths = input_collection.get_file_paths("mag", data_type="l1d", descriptor="norm-dsrf") + mag_l2_paths = input_collection.get_file_paths( + "mag", data_type="l2", descriptor="norm-dsrf" + ) + mag_l1d_paths = input_collection.get_file_paths( + "mag", data_type="l1d", descriptor="norm-dsrf" + ) mag_dependency = [*mag_l2_paths, *mag_l1d_paths][0] mag_is_preliminary = len(mag_l2_paths) == 0 for download_location_file_path in [*codice_file_paths, mag_dependency]: imap_data_access.download(download_location_file_path) - return cls.from_file_paths(mag_dependency, codice_file_paths[0], mag_is_preliminary=mag_is_preliminary) + return cls.from_file_paths( + mag_dependency, codice_file_paths[0], mag_is_preliminary=mag_is_preliminary + ) @classmethod - def from_file_paths(cls, mag_file_path, codice_l2_sectored_intensities_path, mag_is_preliminary: bool = False): + def from_file_paths( + cls, + mag_file_path, + codice_l2_sectored_intensities_path, + mag_is_preliminary: bool = False, + ): mag_data = read_mag_data(mag_file_path) - sectored_intensities = CodiceHiL2SectoredIntensitiesData.read_from_cdf(codice_l2_sectored_intensities_path) + sectored_intensities = CodiceHiL2SectoredIntensitiesData.read_from_cdf( + codice_l2_sectored_intensities_path + ) - return cls(mag_data, sectored_intensities, mag_is_preliminary=mag_is_preliminary) + return cls( + mag_data, sectored_intensities, mag_is_preliminary=mag_is_preliminary + ) diff --git a/imap_l3_processing/codice/l3/lo/codice_lo_l3a_direct_events_dependencies.py b/imap_l3_processing/codice/l3/lo/codice_lo_l3a_direct_events_dependencies.py index bf6d11a60..73d28bd69 100644 --- a/imap_l3_processing/codice/l3/lo/codice_lo_l3a_direct_events_dependencies.py +++ b/imap_l3_processing/codice/l3/lo/codice_lo_l3a_direct_events_dependencies.py @@ -4,16 +4,23 @@ from imap_data_access import download from imap_data_access.processing_input import ProcessingInputCollection, ScienceInput -from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import EnergyLookup, \ - ESA_TO_ENERGY_PER_CHARGE_LOOKUP_DESCRIPTOR -from imap_l3_processing.codice.l3.lo.direct_events.science.mass_coefficient_lookup import MassCoefficientLookup -from imap_l3_processing.codice.l3.lo.models import CodiceLoL2DirectEventData, CodiceLoL1aSWPriorityRates, \ - CodiceLoL1aNSWPriorityRates +from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import ( + EnergyLookup, + ESA_TO_ENERGY_PER_CHARGE_LOOKUP_DESCRIPTOR, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.mass_coefficient_lookup import ( + MassCoefficientLookup, +) +from imap_l3_processing.codice.l3.lo.models import ( + CodiceLoL2DirectEventData, + CodiceLoL1aSWPriorityRates, + CodiceLoL1aNSWPriorityRates, +) -SW_PRIORITY_DESCRIPTOR = 'lo-sw-priority' -NSW_PRIORITY_DESCRIPTOR = 'lo-nsw-priority' -DIRECT_EVENTS_DESCRIPTOR = 'lo-direct-events' -MASS_COEFFICIENT_DESCRIPTOR = 'mass-coefficient-lookup' +SW_PRIORITY_DESCRIPTOR = "lo-sw-priority" +NSW_PRIORITY_DESCRIPTOR = "lo-nsw-priority" +DIRECT_EVENTS_DESCRIPTOR = "lo-direct-events" +MASS_COEFFICIENT_DESCRIPTOR = "mass-coefficient-lookup" @dataclass @@ -31,14 +38,17 @@ def fetch_dependencies(cls, dependencies: ProcessingInputCollection): direct_events_paths = [] for science_input in dependencies.get_science_inputs(): match science_input: - case ScienceInput(data_type="l1a", source="codice", - descriptor=descriptor) if descriptor == SW_PRIORITY_DESCRIPTOR: + case ScienceInput( + data_type="l1a", source="codice", descriptor=descriptor + ) if descriptor == SW_PRIORITY_DESCRIPTOR: sw_priority_rates_paths.extend(science_input.filename_list) - case ScienceInput(data_type="l1a", source="codice", - descriptor=descriptor) if descriptor == NSW_PRIORITY_DESCRIPTOR: + case ScienceInput( + data_type="l1a", source="codice", descriptor=descriptor + ) if descriptor == NSW_PRIORITY_DESCRIPTOR: nsw_priority_rates_paths.extend(science_input.filename_list) - case ScienceInput(data_type="l2", source="codice", - descriptor=descriptor) if descriptor == DIRECT_EVENTS_DESCRIPTOR: + case ScienceInput( + data_type="l2", source="codice", descriptor=descriptor + ) if descriptor == DIRECT_EVENTS_DESCRIPTOR: direct_events_paths.extend(science_input.filename_list) assert len(sw_priority_rates_paths) == 1 @@ -48,38 +58,59 @@ def fetch_dependencies(cls, dependencies: ProcessingInputCollection): assert len(direct_events_paths) == 1 direct_events_path = direct_events_paths[0] - [mass_coefficients_file_path] = dependencies.get_file_paths(source='codice', - descriptor=MASS_COEFFICIENT_DESCRIPTOR) - [energy_per_charge_file_path] = dependencies.get_file_paths(source='codice', - descriptor=ESA_TO_ENERGY_PER_CHARGE_LOOKUP_DESCRIPTOR) + [mass_coefficients_file_path] = dependencies.get_file_paths( + source="codice", descriptor=MASS_COEFFICIENT_DESCRIPTOR + ) + [energy_per_charge_file_path] = dependencies.get_file_paths( + source="codice", descriptor=ESA_TO_ENERGY_PER_CHARGE_LOOKUP_DESCRIPTOR + ) sw_priority_rates_downloaded_path = download(sw_priority_rates_path) nsw_priority_rates_downloaded_path = download(nsw_priority_rates_path) direct_events_downloaded_path = download(direct_events_path) - mass_coefficients_file_downloaded_path = download(mass_coefficients_file_path.name) - energy_per_charge_lookup_downloaded_path = download(energy_per_charge_file_path.name) + mass_coefficients_file_downloaded_path = download( + mass_coefficients_file_path.name + ) + energy_per_charge_lookup_downloaded_path = download( + energy_per_charge_file_path.name + ) return cls.from_file_paths( sw_priority_rates_downloaded_path, nsw_priority_rates_downloaded_path, direct_events_downloaded_path, mass_coefficients_file_downloaded_path, - energy_per_charge_lookup_downloaded_path + energy_per_charge_lookup_downloaded_path, ) @classmethod - def from_file_paths(cls, - sw_priority_rates_cdf: Path, - nsw_priority_rates_cdf: Path, - direct_event_path: Path, - mass_coefficients_file_path: Path, - esa_to_energy_per_charge_file_path: Path): - mass_coefficients_file_path = MassCoefficientLookup.read_from_csv(mass_coefficients_file_path) - sw_priority_rates = CodiceLoL1aSWPriorityRates.read_from_cdf(sw_priority_rates_cdf) - nsw_priority_rates = CodiceLoL1aNSWPriorityRates.read_from_cdf(nsw_priority_rates_cdf) - codice_l2_direct_events = CodiceLoL2DirectEventData.read_from_cdf(direct_event_path) + def from_file_paths( + cls, + sw_priority_rates_cdf: Path, + nsw_priority_rates_cdf: Path, + direct_event_path: Path, + mass_coefficients_file_path: Path, + esa_to_energy_per_charge_file_path: Path, + ): + mass_coefficients_file_path = MassCoefficientLookup.read_from_csv( + mass_coefficients_file_path + ) + sw_priority_rates = CodiceLoL1aSWPriorityRates.read_from_cdf( + sw_priority_rates_cdf + ) + nsw_priority_rates = CodiceLoL1aNSWPriorityRates.read_from_cdf( + nsw_priority_rates_cdf + ) + codice_l2_direct_events = CodiceLoL2DirectEventData.read_from_cdf( + direct_event_path + ) energy_lookup = EnergyLookup.read_from_csv(esa_to_energy_per_charge_file_path) - return cls(sw_priority_rates, nsw_priority_rates, codice_l2_direct_events, mass_coefficients_file_path, - energy_lookup) + return cls( + sw_priority_rates, + nsw_priority_rates, + codice_l2_direct_events, + mass_coefficients_file_path, + energy_lookup, + ) diff --git a/imap_l3_processing/codice/l3/lo/codice_lo_l3a_partial_densities_dependencies.py b/imap_l3_processing/codice/l3/lo/codice_lo_l3a_partial_densities_dependencies.py index 0bc002486..910cdc6c1 100644 --- a/imap_l3_processing/codice/l3/lo/codice_lo_l3a_partial_densities_dependencies.py +++ b/imap_l3_processing/codice/l3/lo/codice_lo_l3a_partial_densities_dependencies.py @@ -5,10 +5,12 @@ from imap_data_access.processing_input import ProcessingInputCollection, ScienceInput from imap_l3_processing.codice.l3.lo.models import CodiceLoL2SWSpeciesData -from imap_l3_processing.codice.l3.lo.sectored_intensities.science.mass_per_charge_lookup import MassPerChargeLookup +from imap_l3_processing.codice.l3.lo.sectored_intensities.science.mass_per_charge_lookup import ( + MassPerChargeLookup, +) -SW_SPECIES_DESCRIPTOR = 'lo-sw-species' -MASS_PER_CHARGE_DESCRIPTOR = 'mass-per-charge' +SW_SPECIES_DESCRIPTOR = "lo-sw-species" +MASS_PER_CHARGE_DESCRIPTOR = "mass-per-charge" @dataclass @@ -22,28 +24,39 @@ def fetch_dependencies(cls, dependencies: ProcessingInputCollection): for science_input in dependencies.get_science_inputs(): match science_input: - case ScienceInput(data_type="l2", source="codice", - descriptor=descriptor) if descriptor == SW_SPECIES_DESCRIPTOR: + case ScienceInput( + data_type="l2", source="codice", descriptor=descriptor + ) if descriptor == SW_SPECIES_DESCRIPTOR: sectored_intensities_file_paths.extend(science_input.filename_list) assert len(sectored_intensities_file_paths) == 1 sectored_intensities_file_path = sectored_intensities_file_paths[0] - mass_per_charge_ancillary_file_paths = dependencies.get_file_paths(source='codice', - descriptor=MASS_PER_CHARGE_DESCRIPTOR) + mass_per_charge_ancillary_file_paths = dependencies.get_file_paths( + source="codice", descriptor=MASS_PER_CHARGE_DESCRIPTOR + ) assert len(mass_per_charge_ancillary_file_paths) == 1 - mass_per_charge_ancillary_file_path = mass_per_charge_ancillary_file_paths[0].name - - sectored_intensities_downloaded_path = imap_data_access.download(sectored_intensities_file_path) - mass_per_charge_ancillary_file_downloaded_path = imap_data_access.download(mass_per_charge_ancillary_file_path) - - return cls.from_file_paths(sectored_intensities_downloaded_path, mass_per_charge_ancillary_file_downloaded_path) + mass_per_charge_ancillary_file_path = mass_per_charge_ancillary_file_paths[ + 0 + ].name + + sectored_intensities_downloaded_path = imap_data_access.download( + sectored_intensities_file_path + ) + mass_per_charge_ancillary_file_downloaded_path = imap_data_access.download( + mass_per_charge_ancillary_file_path + ) + + return cls.from_file_paths( + sectored_intensities_downloaded_path, + mass_per_charge_ancillary_file_downloaded_path, + ) @classmethod - def from_file_paths(cls, codice_l2_lo_cdf: Path, - mass_per_charge_lookup_path: Path): - mass_per_charge_lookup = MassPerChargeLookup.read_from_file(mass_per_charge_lookup_path) + def from_file_paths(cls, codice_l2_lo_cdf: Path, mass_per_charge_lookup_path: Path): + mass_per_charge_lookup = MassPerChargeLookup.read_from_file( + mass_per_charge_lookup_path + ) codice_l2_lo_data = CodiceLoL2SWSpeciesData.read_from_cdf(codice_l2_lo_cdf) - return cls(codice_l2_lo_data, - mass_per_charge_lookup) + return cls(codice_l2_lo_data, mass_per_charge_lookup) diff --git a/imap_l3_processing/codice/l3/lo/codice_lo_l3a_ratios_dependencies.py b/imap_l3_processing/codice/l3/lo/codice_lo_l3a_ratios_dependencies.py index f4600b007..89df62b0f 100644 --- a/imap_l3_processing/codice/l3/lo/codice_lo_l3a_ratios_dependencies.py +++ b/imap_l3_processing/codice/l3/lo/codice_lo_l3a_ratios_dependencies.py @@ -6,7 +6,7 @@ from imap_l3_processing.codice.l3.lo.models import CodiceLoPartialDensityData -PARTIAL_DENSITY_DESCRIPTOR = 'lo-partial-densities' +PARTIAL_DENSITY_DESCRIPTOR = "lo-partial-densities" @dataclass @@ -15,7 +15,9 @@ class CodiceLoL3aRatiosDependencies: @classmethod def fetch_dependencies(cls, dependencies: ProcessingInputCollection): - input_file_paths = dependencies.get_file_paths(source='codice', descriptor=PARTIAL_DENSITY_DESCRIPTOR) + input_file_paths = dependencies.get_file_paths( + source="codice", descriptor=PARTIAL_DENSITY_DESCRIPTOR + ) assert len(input_file_paths) == 1 input_file_name = input_file_paths[0].name @@ -25,6 +27,8 @@ def fetch_dependencies(cls, dependencies: ProcessingInputCollection): @classmethod def from_file_paths(cls, codice_l2_lo_cdf: Path | str): - partial_density_data = CodiceLoPartialDensityData.read_from_cdf(codice_l2_lo_cdf) + partial_density_data = CodiceLoPartialDensityData.read_from_cdf( + codice_l2_lo_cdf + ) return cls(partial_density_data) diff --git a/imap_l3_processing/codice/l3/lo/codice_lo_l3b_3d_distributions_dependencies.py b/imap_l3_processing/codice/l3/lo/codice_lo_l3b_3d_distributions_dependencies.py index d4dcf61b5..d0aa4110a 100644 --- a/imap_l3_processing/codice/l3/lo/codice_lo_l3b_3d_distributions_dependencies.py +++ b/imap_l3_processing/codice/l3/lo/codice_lo_l3b_3d_distributions_dependencies.py @@ -4,12 +4,22 @@ import imap_data_access from imap_data_access.processing_input import ProcessingInputCollection, ScienceInput -from imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies import DIRECT_EVENTS_DESCRIPTOR -from imap_l3_processing.codice.l3.lo.direct_events.science.efficiency_lookup import EfficiencyLookup -from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import EnergyLookup, \ - ESA_TO_ENERGY_PER_CHARGE_LOOKUP_DESCRIPTOR -from imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup import GeometricFactorLookup -from imap_l3_processing.codice.l3.lo.direct_events.science.mass_species_bin_lookup import MassSpeciesBinLookup +from imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies import ( + DIRECT_EVENTS_DESCRIPTOR, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.efficiency_lookup import ( + EfficiencyLookup, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import ( + EnergyLookup, + ESA_TO_ENERGY_PER_CHARGE_LOOKUP_DESCRIPTOR, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup import ( + GeometricFactorLookup, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.mass_species_bin_lookup import ( + MassSpeciesBinLookup, +) from imap_l3_processing.codice.l3.lo.models import CodiceLoDirectEventData MASS_SPECIES_BIN_LOOKUP_DESCRIPTOR = "lo-mass-species-bin-lookup" @@ -27,58 +37,80 @@ class CodiceLoL3b3dDistributionsDependencies: species: str @classmethod - def fetch_dependencies(cls, processing_input_collection: ProcessingInputCollection, species: str): + def fetch_dependencies( + cls, processing_input_collection: ProcessingInputCollection, species: str + ): l3a_direct_event_paths = [] for input in processing_input_collection.get_science_inputs(): match input: - case ScienceInput(data_type="l3a", source="codice", descriptor=descriptor) \ - if descriptor == DIRECT_EVENTS_DESCRIPTOR: + case ScienceInput( + data_type="l3a", source="codice", descriptor=descriptor + ) if descriptor == DIRECT_EVENTS_DESCRIPTOR: l3a_direct_event_paths.extend(input.filename_list) [direct_events_path] = l3a_direct_event_paths [mass_species_bin_path] = processing_input_collection.get_file_paths( - source='codice', + source="codice", descriptor=MASS_SPECIES_BIN_LOOKUP_DESCRIPTOR, ) [geometric_factor_path] = processing_input_collection.get_file_paths( - source='codice', + source="codice", descriptor=GEOMETRIC_FACTOR_LOOKUP_DESCRIPTOR, ) [efficiency_factor_path] = processing_input_collection.get_file_paths( - source='codice', + source="codice", descriptor=EFFICIENCY_FACTOR_LOOKUP_DESCRIPTOR, ) [esa_to_energy_per_charge_path] = processing_input_collection.get_file_paths( - source='codice', + source="codice", descriptor=ESA_TO_ENERGY_PER_CHARGE_LOOKUP_DESCRIPTOR, ) direct_events_downloaded_path = imap_data_access.download(direct_events_path) - mass_species_bin_downloaded_path = imap_data_access.download(mass_species_bin_path.name) - geometric_factor_downloaded_path = imap_data_access.download(geometric_factor_path.name) - efficiency_factor_downloaded_path = imap_data_access.download(efficiency_factor_path.name) - esa_to_energy_per_charge_downloaded_path = imap_data_access.download(esa_to_energy_per_charge_path.name) + mass_species_bin_downloaded_path = imap_data_access.download( + mass_species_bin_path.name + ) + geometric_factor_downloaded_path = imap_data_access.download( + geometric_factor_path.name + ) + efficiency_factor_downloaded_path = imap_data_access.download( + efficiency_factor_path.name + ) + esa_to_energy_per_charge_downloaded_path = imap_data_access.download( + esa_to_energy_per_charge_path.name + ) - return cls.from_file_paths(l3a_file_path=direct_events_downloaded_path, - mass_species_bin_lut=mass_species_bin_downloaded_path, - geometric_factors_lut=geometric_factor_downloaded_path, - efficiency_factors_lut=efficiency_factor_downloaded_path, - energy_per_charge_lut=esa_to_energy_per_charge_downloaded_path, - species=species) + return cls.from_file_paths( + l3a_file_path=direct_events_downloaded_path, + mass_species_bin_lut=mass_species_bin_downloaded_path, + geometric_factors_lut=geometric_factor_downloaded_path, + efficiency_factors_lut=efficiency_factor_downloaded_path, + energy_per_charge_lut=esa_to_energy_per_charge_downloaded_path, + species=species, + ) @classmethod - def from_file_paths(cls, l3a_file_path: Path, - mass_species_bin_lut: Path, - geometric_factors_lut: Path, - efficiency_factors_lut: Path, - energy_per_charge_lut: Path, - species: str): - mass_species_bin_lookup = MassSpeciesBinLookup.read_from_csv(mass_species_bin_lut) + def from_file_paths( + cls, + l3a_file_path: Path, + mass_species_bin_lut: Path, + geometric_factors_lut: Path, + efficiency_factors_lut: Path, + energy_per_charge_lut: Path, + species: str, + ): + mass_species_bin_lookup = MassSpeciesBinLookup.read_from_csv( + mass_species_bin_lut + ) return cls( l3a_direct_event_data=CodiceLoDirectEventData.read_from_cdf(l3a_file_path), mass_species_bin_lookup=mass_species_bin_lookup, - geometric_factors_lookup=GeometricFactorLookup.read_from_csv(geometric_factors_lut), - efficiency_factors_lut=EfficiencyLookup.read_from_csv(efficiency_factors_lut, species), + geometric_factors_lookup=GeometricFactorLookup.read_from_csv( + geometric_factors_lut + ), + efficiency_factors_lut=EfficiencyLookup.read_from_csv( + efficiency_factors_lut, species + ), energy_per_charge_lut=EnergyLookup.read_from_csv(energy_per_charge_lut), - species=species + species=species, ) diff --git a/imap_l3_processing/codice/l3/lo/codice_lo_processor.py b/imap_l3_processing/codice/l3/lo/codice_lo_processor.py index 547a91d44..ade699b84 100644 --- a/imap_l3_processing/codice/l3/lo/codice_lo_processor.py +++ b/imap_l3_processing/codice/l3/lo/codice_lo_processor.py @@ -2,60 +2,98 @@ import numpy as np from imap_data_access.processing_input import ProcessingInputCollection -from imap_processing.spice.geometry import get_spacecraft_to_instrument_spin_phase_offset, SpiceFrame - -from imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies import \ - CodiceLoL3b3dDistributionsDependencies -from imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies import CodiceLoL3aDirectEventsDependencies -from imap_l3_processing.codice.l3.lo.codice_lo_l3a_partial_densities_dependencies import \ - CodiceLoL3aPartialDensitiesDependencies -from imap_l3_processing.codice.l3.lo.codice_lo_l3a_ratios_dependencies import CodiceLoL3aRatiosDependencies -from imap_l3_processing.codice.l3.lo.direct_events.science.angle_lookup import SpinAngleLookup, \ - PositionToElevationLookup -from imap_l3_processing.codice.l3.lo.models import CodiceLoL3aPartialDensityDataProduct, \ - CodiceLoL3aDirectEventDataProduct, CodiceLoPartialDensityData, CodiceLoL3aRatiosDataProduct, \ - CodiceLoL3ChargeStateDistributionsDataProduct, CodiceLoL3b3dDistributionDataProduct -from imap_l3_processing.codice.l3.lo.science.codice_lo_calculations import calculate_partial_densities, \ - calculate_mass, calculate_mass_per_charge,convert_counts_to_intensity, \ - rebin_to_counts_by_species_elevation_and_spin_sector, combine_priorities_for_species_and_calculate_count_uncertainty, \ - calculate_normalization_factor, lookup_normalization_per_event, rebin_3d_distribution_azimuth_to_elevation +from imap_processing.spice.geometry import ( + get_spacecraft_to_instrument_spin_phase_offset, + SpiceFrame, +) + +from imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies import ( + CodiceLoL3b3dDistributionsDependencies, +) +from imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies import ( + CodiceLoL3aDirectEventsDependencies, +) +from imap_l3_processing.codice.l3.lo.codice_lo_l3a_partial_densities_dependencies import ( + CodiceLoL3aPartialDensitiesDependencies, +) +from imap_l3_processing.codice.l3.lo.codice_lo_l3a_ratios_dependencies import ( + CodiceLoL3aRatiosDependencies, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.angle_lookup import ( + SpinAngleLookup, + PositionToElevationLookup, +) +from imap_l3_processing.codice.l3.lo.models import ( + CodiceLoL3aPartialDensityDataProduct, + CodiceLoL3aDirectEventDataProduct, + CodiceLoPartialDensityData, + CodiceLoL3aRatiosDataProduct, + CodiceLoL3ChargeStateDistributionsDataProduct, + CodiceLoL3b3dDistributionDataProduct, +) +from imap_l3_processing.codice.l3.lo.science.codice_lo_calculations import ( + calculate_partial_densities, + calculate_mass, + calculate_mass_per_charge, + convert_counts_to_intensity, + rebin_to_counts_by_species_elevation_and_spin_sector, + combine_priorities_for_species_and_calculate_count_uncertainty, + calculate_normalization_factor, + lookup_normalization_per_event, + rebin_3d_distribution_azimuth_to_elevation, +) from imap_l3_processing.data_utils import safe_divide from imap_l3_processing.models import InputMetadata from imap_l3_processing.processor import Processor from imap_l3_processing.utils import save_data -PriorityRate = namedtuple('PriorityRate', ('epoch', 'energy_table', 'priority_count')) +PriorityRate = namedtuple("PriorityRate", ("epoch", "energy_table", "priority_count")) class CodiceLoProcessor(Processor): - def __init__(self, dependencies: ProcessingInputCollection, input_metadata: InputMetadata): + def __init__( + self, dependencies: ProcessingInputCollection, input_metadata: InputMetadata + ): super().__init__(dependencies, input_metadata) def process(self): if self.input_metadata.descriptor == "lo-partial-densities": - dependencies = CodiceLoL3aPartialDensitiesDependencies.fetch_dependencies(self.dependencies) + dependencies = CodiceLoL3aPartialDensitiesDependencies.fetch_dependencies( + self.dependencies + ) data_product = self.process_l3a_partial_densities(dependencies) elif self.input_metadata.descriptor == "lo-direct-events": - dependencies = CodiceLoL3aDirectEventsDependencies.fetch_dependencies(self.dependencies) + dependencies = CodiceLoL3aDirectEventsDependencies.fetch_dependencies( + self.dependencies + ) data_product = self.process_l3a_direct_event_data_product(dependencies) elif self.input_metadata.descriptor == "lo-sw-ratios": - dependencies = CodiceLoL3aRatiosDependencies.fetch_dependencies(self.dependencies) + dependencies = CodiceLoL3aRatiosDependencies.fetch_dependencies( + self.dependencies + ) data_product = self.process_l3a_ratios(dependencies) elif self.input_metadata.descriptor == "lo-sw-charge-state-distributions": - dependencies = CodiceLoL3aRatiosDependencies.fetch_dependencies(self.dependencies) + dependencies = CodiceLoL3aRatiosDependencies.fetch_dependencies( + self.dependencies + ) data_product = self.process_l3a_charge_state_distributions(dependencies) elif "3d-distribution" in self.input_metadata.descriptor: - species = self.input_metadata.descriptor.split('-')[1] - dependencies = CodiceLoL3b3dDistributionsDependencies.fetch_dependencies(self.dependencies, species) + species = self.input_metadata.descriptor.split("-")[1] + dependencies = CodiceLoL3b3dDistributionsDependencies.fetch_dependencies( + self.dependencies, species + ) data_product = self.process_l3b_3d_distribution_product(dependencies) else: raise NotImplementedError( - f"Unknown data level and descriptor for CoDICE: {self.input_metadata.data_level}, {self.input_metadata.descriptor}") + f"Unknown data level and descriptor for CoDICE: {self.input_metadata.data_level}, {self.input_metadata.descriptor}" + ) data_product.parent_file_names = self.get_parent_file_names() return [save_data(data_product)] - def process_l3a_ratios(self, dependencies: CodiceLoL3aRatiosDependencies) -> CodiceLoL3aRatiosDataProduct: + def process_l3a_ratios( + self, dependencies: CodiceLoL3aRatiosDependencies + ) -> CodiceLoL3aRatiosDataProduct: input_data = dependencies.partial_density_data c4 = _average_over_block(input_data.cplus4_partial_density, 3) c5 = _average_over_block(input_data.cplus5_partial_density, 3) @@ -81,8 +119,9 @@ def process_l3a_ratios(self, dependencies: CodiceLoL3aRatiosDependencies) -> Cod felo_to_fehi_ratio=safe_divide(feloq, fehiq), ) - def process_l3a_charge_state_distributions(self, - dependencies: CodiceLoL3aRatiosDependencies) -> CodiceLoL3ChargeStateDistributionsDataProduct: + def process_l3a_charge_state_distributions( + self, dependencies: CodiceLoL3aRatiosDependencies + ) -> CodiceLoL3ChargeStateDistributionsDataProduct: o5 = dependencies.partial_density_data.oplus5_partial_density o6 = dependencies.partial_density_data.oplus6_partial_density @@ -92,21 +131,31 @@ def process_l3a_charge_state_distributions(self, c5 = dependencies.partial_density_data.cplus5_partial_density c6 = dependencies.partial_density_data.cplus6_partial_density - o_densities = _average_over_block(np.column_stack((o5, o6, o7, o8)), block_size=3) - o_distribution = safe_divide(o_densities, np.sum(o_densities, axis=1, keepdims=True)) + o_densities = _average_over_block( + np.column_stack((o5, o6, o7, o8)), block_size=3 + ) + o_distribution = safe_divide( + o_densities, np.sum(o_densities, axis=1, keepdims=True) + ) c_densities = _average_over_block(np.column_stack((c4, c5, c6)), block_size=3) - c_distribution = safe_divide(c_densities, np.sum(c_densities, axis=1, keepdims=True)) + c_distribution = safe_divide( + c_densities, np.sum(c_densities, axis=1, keepdims=True) + ) return CodiceLoL3ChargeStateDistributionsDataProduct( self.input_metadata, epoch=_average_dates_over_block(dependencies.partial_density_data.epoch, 3), - epoch_delta=_sum_over_block(dependencies.partial_density_data.epoch_delta, 3), + epoch_delta=_sum_over_block( + dependencies.partial_density_data.epoch_delta, 3 + ), oxygen_charge_state_distribution=o_distribution, - carbon_charge_state_distribution=c_distribution + carbon_charge_state_distribution=c_distribution, ) - def process_l3a_partial_densities(self, dependencies: CodiceLoL3aPartialDensitiesDependencies): + def process_l3a_partial_densities( + self, dependencies: CodiceLoL3aPartialDensitiesDependencies + ): codice_lo_l2_data = dependencies.codice_l2_lo_data mass_per_charge_lookup = dependencies.mass_per_charge_lookup h_plus_partial_density = calculate_partial_densities( @@ -114,42 +163,71 @@ def process_l3a_partial_densities(self, dependencies: CodiceLoL3aPartialDensitie codice_lo_l2_data.energy_per_charge, mass_per_charge_lookup.hplus, ) - heplusplus_partial_density = calculate_partial_densities(codice_lo_l2_data.heplusplus, - codice_lo_l2_data.energy_per_charge, - mass_per_charge_lookup.heplusplus) - cplus4_partial_density = calculate_partial_densities(codice_lo_l2_data.cplus4, - codice_lo_l2_data.energy_per_charge, - mass_per_charge_lookup.cplus4) - cplus5_partial_density = calculate_partial_densities(codice_lo_l2_data.cplus5, - codice_lo_l2_data.energy_per_charge, - mass_per_charge_lookup.cplus5) - cplus6_partial_density = calculate_partial_densities(codice_lo_l2_data.cplus6, - codice_lo_l2_data.energy_per_charge, - mass_per_charge_lookup.cplus6) - oplus5_partial_density = calculate_partial_densities(codice_lo_l2_data.oplus5, - codice_lo_l2_data.energy_per_charge, - mass_per_charge_lookup.oplus5) - oplus6_partial_density = calculate_partial_densities(codice_lo_l2_data.oplus6, - codice_lo_l2_data.energy_per_charge, - mass_per_charge_lookup.oplus6) - oplus7_partial_density = calculate_partial_densities(codice_lo_l2_data.oplus7, - codice_lo_l2_data.energy_per_charge, - mass_per_charge_lookup.oplus7) - oplus8_partial_density = calculate_partial_densities(codice_lo_l2_data.oplus8, - codice_lo_l2_data.energy_per_charge, - mass_per_charge_lookup.oplus8) - ne_partial_density = calculate_partial_densities(codice_lo_l2_data.ne, codice_lo_l2_data.energy_per_charge, - mass_per_charge_lookup.ne) - mg_partial_density = calculate_partial_densities(codice_lo_l2_data.mg, codice_lo_l2_data.energy_per_charge, - mass_per_charge_lookup.mg) - si_partial_density = calculate_partial_densities(codice_lo_l2_data.si, codice_lo_l2_data.energy_per_charge, - mass_per_charge_lookup.si) - fe_loq_partial_density = calculate_partial_densities(codice_lo_l2_data.fe_loq, - codice_lo_l2_data.energy_per_charge, - mass_per_charge_lookup.fe_loq) - fe_hiq_partial_density = calculate_partial_densities(codice_lo_l2_data.fe_hiq, - codice_lo_l2_data.energy_per_charge, - mass_per_charge_lookup.fe_hiq) + heplusplus_partial_density = calculate_partial_densities( + codice_lo_l2_data.heplusplus, + codice_lo_l2_data.energy_per_charge, + mass_per_charge_lookup.heplusplus, + ) + cplus4_partial_density = calculate_partial_densities( + codice_lo_l2_data.cplus4, + codice_lo_l2_data.energy_per_charge, + mass_per_charge_lookup.cplus4, + ) + cplus5_partial_density = calculate_partial_densities( + codice_lo_l2_data.cplus5, + codice_lo_l2_data.energy_per_charge, + mass_per_charge_lookup.cplus5, + ) + cplus6_partial_density = calculate_partial_densities( + codice_lo_l2_data.cplus6, + codice_lo_l2_data.energy_per_charge, + mass_per_charge_lookup.cplus6, + ) + oplus5_partial_density = calculate_partial_densities( + codice_lo_l2_data.oplus5, + codice_lo_l2_data.energy_per_charge, + mass_per_charge_lookup.oplus5, + ) + oplus6_partial_density = calculate_partial_densities( + codice_lo_l2_data.oplus6, + codice_lo_l2_data.energy_per_charge, + mass_per_charge_lookup.oplus6, + ) + oplus7_partial_density = calculate_partial_densities( + codice_lo_l2_data.oplus7, + codice_lo_l2_data.energy_per_charge, + mass_per_charge_lookup.oplus7, + ) + oplus8_partial_density = calculate_partial_densities( + codice_lo_l2_data.oplus8, + codice_lo_l2_data.energy_per_charge, + mass_per_charge_lookup.oplus8, + ) + ne_partial_density = calculate_partial_densities( + codice_lo_l2_data.ne, + codice_lo_l2_data.energy_per_charge, + mass_per_charge_lookup.ne, + ) + mg_partial_density = calculate_partial_densities( + codice_lo_l2_data.mg, + codice_lo_l2_data.energy_per_charge, + mass_per_charge_lookup.mg, + ) + si_partial_density = calculate_partial_densities( + codice_lo_l2_data.si, + codice_lo_l2_data.energy_per_charge, + mass_per_charge_lookup.si, + ) + fe_loq_partial_density = calculate_partial_densities( + codice_lo_l2_data.fe_loq, + codice_lo_l2_data.energy_per_charge, + mass_per_charge_lookup.fe_loq, + ) + fe_hiq_partial_density = calculate_partial_densities( + codice_lo_l2_data.fe_hiq, + codice_lo_l2_data.energy_per_charge, + mass_per_charge_lookup.fe_hiq, + ) return CodiceLoL3aPartialDensityDataProduct( input_metadata=self.input_metadata, @@ -170,15 +248,18 @@ def process_l3a_partial_densities(self, dependencies: CodiceLoL3aPartialDensitie si_partial_density=si_partial_density, fe_loq_partial_density=fe_loq_partial_density, fe_hiq_partial_density=fe_hiq_partial_density, - ) + ), ) def process_l3a_direct_event_data_product( - self, - dependencies: CodiceLoL3aDirectEventsDependencies + self, dependencies: CodiceLoL3aDirectEventsDependencies ) -> CodiceLoL3aDirectEventDataProduct: - codice_sw_priority_counts_l1a_data = dependencies.codice_lo_l1a_sw_priority_rates - codice_nsw_priority_counts_l1a_data = dependencies.codice_lo_l1a_nsw_priority_rates + codice_sw_priority_counts_l1a_data = ( + dependencies.codice_lo_l1a_sw_priority_rates + ) + codice_nsw_priority_counts_l1a_data = ( + dependencies.codice_lo_l1a_nsw_priority_rates + ) codice_direct_events = dependencies.codice_l2_direct_events esa_energy_per_charge_lookup = dependencies.energy_lookup @@ -186,8 +267,14 @@ def process_l3a_direct_event_data_product( spin_angle_lut = SpinAngleLookup() - mass_per_charge = calculate_mass_per_charge(codice_direct_events.energy_per_charge, codice_direct_events.tof) - mass = calculate_mass(codice_direct_events.apd_energy, codice_direct_events.tof, mass_coefficient_lookup) + mass_per_charge = calculate_mass_per_charge( + codice_direct_events.energy_per_charge, codice_direct_events.tof + ) + mass = calculate_mass( + codice_direct_events.apd_energy, + codice_direct_events.tof, + mass_coefficient_lookup, + ) direct_event_epochs = codice_direct_events.epoch @@ -209,49 +296,84 @@ def process_l3a_direct_event_data_product( axis=1, ) sw_aligned, sw_missing = _align_priority_counts_variable_to_direct_event_epochs( - sw_priority_counts, codice_sw_priority_counts_l1a_data.epoch, direct_event_epochs + sw_priority_counts, + codice_sw_priority_counts_l1a_data.epoch, + direct_event_epochs, ) - nsw_aligned, nsw_missing = _align_priority_counts_variable_to_direct_event_epochs( - nsw_priority_counts, codice_nsw_priority_counts_l1a_data.epoch, direct_event_epochs + nsw_aligned, nsw_missing = ( + _align_priority_counts_variable_to_direct_event_epochs( + nsw_priority_counts, + codice_nsw_priority_counts_l1a_data.epoch, + direct_event_epochs, + ) ) - acquisition_time_per_esa_aligned, _ = _align_priority_counts_variable_to_direct_event_epochs( - codice_sw_priority_counts_l1a_data.acquisition_time_per_esa_step, codice_sw_priority_counts_l1a_data.epoch, - direct_event_epochs + acquisition_time_per_esa_aligned, _ = ( + _align_priority_counts_variable_to_direct_event_epochs( + codice_sw_priority_counts_l1a_data.acquisition_time_per_esa_step, + codice_sw_priority_counts_l1a_data.epoch, + direct_event_epochs, + ) ) - rgfo_half_spin_aligned, _ = _align_priority_counts_variable_to_direct_event_epochs( - codice_sw_priority_counts_l1a_data.rgfo_half_spin, codice_sw_priority_counts_l1a_data.epoch, - direct_event_epochs + rgfo_half_spin_aligned, _ = ( + _align_priority_counts_variable_to_direct_event_epochs( + codice_sw_priority_counts_l1a_data.rgfo_half_spin, + codice_sw_priority_counts_l1a_data.epoch, + direct_event_epochs, + ) ) - rgfo_spin_sector_aligned, _ = _align_priority_counts_variable_to_direct_event_epochs( - codice_sw_priority_counts_l1a_data.rgfo_spin_sector, codice_sw_priority_counts_l1a_data.epoch, - direct_event_epochs + rgfo_spin_sector_aligned, _ = ( + _align_priority_counts_variable_to_direct_event_epochs( + codice_sw_priority_counts_l1a_data.rgfo_spin_sector, + codice_sw_priority_counts_l1a_data.epoch, + direct_event_epochs, + ) + ) + rgfo_esa_step_aligned, _ = ( + _align_priority_counts_variable_to_direct_event_epochs( + codice_sw_priority_counts_l1a_data.rgfo_esa_step, + codice_sw_priority_counts_l1a_data.epoch, + direct_event_epochs, + ) ) - rgfo_esa_step_aligned, _ = _align_priority_counts_variable_to_direct_event_epochs( - codice_sw_priority_counts_l1a_data.rgfo_esa_step, codice_sw_priority_counts_l1a_data.epoch, - direct_event_epochs + half_spin_per_esa_step_aligned, _ = ( + _align_priority_counts_variable_to_direct_event_epochs( + codice_sw_priority_counts_l1a_data.half_spin_per_esa_step, + codice_sw_priority_counts_l1a_data.epoch, + direct_event_epochs, + ) ) - half_spin_per_esa_step_aligned, _ = _align_priority_counts_variable_to_direct_event_epochs( - codice_sw_priority_counts_l1a_data.half_spin_per_esa_step, codice_sw_priority_counts_l1a_data.epoch, - direct_event_epochs + nso_spin_sector_aligned, _ = ( + _align_priority_counts_variable_to_direct_event_epochs( + codice_sw_priority_counts_l1a_data.nso_spin_sector, + codice_sw_priority_counts_l1a_data.epoch, + direct_event_epochs, + ) ) - nso_spin_sector_aligned, _ = _align_priority_counts_variable_to_direct_event_epochs( - codice_sw_priority_counts_l1a_data.nso_spin_sector, codice_sw_priority_counts_l1a_data.epoch, - direct_event_epochs + nso_esa_step_aligned, _ = ( + _align_priority_counts_variable_to_direct_event_epochs( + codice_sw_priority_counts_l1a_data.nso_esa_step, + codice_sw_priority_counts_l1a_data.epoch, + direct_event_epochs, + ) ) - nso_esa_step_aligned, _ = _align_priority_counts_variable_to_direct_event_epochs( - codice_sw_priority_counts_l1a_data.nso_esa_step, codice_sw_priority_counts_l1a_data.epoch, - direct_event_epochs + nso_half_spin_aligned, _ = ( + _align_priority_counts_variable_to_direct_event_epochs( + codice_sw_priority_counts_l1a_data.nso_half_spin, + codice_sw_priority_counts_l1a_data.epoch, + direct_event_epochs, + ) ) - nso_half_spin_aligned, _ = _align_priority_counts_variable_to_direct_event_epochs( - codice_sw_priority_counts_l1a_data.nso_half_spin, codice_sw_priority_counts_l1a_data.epoch, - direct_event_epochs + stacked_priorities = np.concatenate( + [np.ma.filled(sw_aligned, np.nan), np.ma.filled(nsw_aligned, np.nan)], + axis=1, ) - stacked_priorities = np.concatenate([ - np.ma.filled(sw_aligned, np.nan), np.ma.filled(nsw_aligned, np.nan)], axis=1) - normalization = calculate_normalization_factor(stacked_priorities, codice_direct_events.num_events, - codice_direct_events.energy_step, - codice_direct_events.spin_sector) + normalization = calculate_normalization_factor( + stacked_priorities, + codice_direct_events.num_events, + codice_direct_events.energy_step, + codice_direct_events.spin_sector, + ) normalization_per_event = lookup_normalization_per_event( normalization, codice_direct_events.num_events, @@ -262,7 +384,8 @@ def process_l3a_direct_event_data_product( num_sw_priorities = sw_aligned.shape[1] num_nsw_priorities = nsw_aligned.shape[1] priority_missing_mask = np.zeros( - (len(direct_event_epochs), num_sw_priorities + num_nsw_priorities), dtype=bool + (len(direct_event_epochs), num_sw_priorities + num_nsw_priorities), + dtype=bool, ) priority_missing_mask[sw_missing, :num_sw_priorities] = True priority_missing_mask[nsw_missing, num_sw_priorities:] = True @@ -306,39 +429,68 @@ def process_l3a_direct_event_data_product( type=codice_direct_events.type, ) - def process_l3b_3d_distribution_product(self, dependencies: CodiceLoL3b3dDistributionsDependencies): + def process_l3b_3d_distribution_product( + self, dependencies: CodiceLoL3b3dDistributionsDependencies + ): mass_species_bin_lookup = dependencies.mass_species_bin_lookup position_elevation_lut = PositionToElevationLookup() energy_lut = dependencies.energy_per_charge_lut geometric_factor_lut = dependencies.geometric_factors_lookup counts_3d_data = rebin_to_counts_by_species_elevation_and_spin_sector( - direct_event_data=dependencies.l3a_direct_event_data, mass_species_bin_lookup=mass_species_bin_lookup) + direct_event_data=dependencies.l3a_direct_event_data, + mass_species_bin_lookup=mass_species_bin_lookup, + ) species_index = mass_species_bin_lookup.get_species_index(dependencies.species) counts_for_species = counts_3d_data[species_index] - normalized_counts, counts_uncertainty = combine_priorities_for_species_and_calculate_count_uncertainty(counts_for_species) + normalized_counts, counts_uncertainty = ( + combine_priorities_for_species_and_calculate_count_uncertainty( + counts_for_species + ) + ) geometric_factors = geometric_factor_lut.get_geometric_factors( - dependencies.l3a_direct_event_data.rgfo_half_spin, dependencies.l3a_direct_event_data.rgfo_spin_sector, - dependencies.l3a_direct_event_data.rgfo_esa_step, dependencies.l3a_direct_event_data.half_spin_per_esa_step, - self.input_metadata.start_date.date() - ) - intensities = convert_counts_to_intensity(normalized_counts, dependencies.l3a_direct_event_data.acquisition_time_per_esa_step, - dependencies.energy_per_charge_lut, - dependencies.efficiency_factors_lut, - geometric_factors) - intensities_uncertainty = convert_counts_to_intensity(counts_uncertainty, dependencies.l3a_direct_event_data.acquisition_time_per_esa_step, - dependencies.energy_per_charge_lut, - dependencies.efficiency_factors_lut, - geometric_factors) - - rebinned_intensity = rebin_3d_distribution_azimuth_to_elevation(intensities, np.arange(1, 25), position_elevation_lut, - dependencies.l3a_direct_event_data.half_spin_per_esa_step) - - rebinned_intensity_uncertainty = rebin_3d_distribution_azimuth_to_elevation(intensities_uncertainty, np.arange(1, 25), position_elevation_lut, - dependencies.l3a_direct_event_data.half_spin_per_esa_step) - offset_degrees = get_spacecraft_to_instrument_spin_phase_offset(SpiceFrame.IMAP_CODICE)*360 - 180 - spin_angle_bins_in_dps = (dependencies.l3a_direct_event_data.spin_angle_bin + offset_degrees) % 360 + dependencies.l3a_direct_event_data.rgfo_half_spin, + dependencies.l3a_direct_event_data.rgfo_spin_sector, + dependencies.l3a_direct_event_data.rgfo_esa_step, + dependencies.l3a_direct_event_data.half_spin_per_esa_step, + self.input_metadata.start_date.date(), + ) + intensities = convert_counts_to_intensity( + normalized_counts, + dependencies.l3a_direct_event_data.acquisition_time_per_esa_step, + dependencies.energy_per_charge_lut, + dependencies.efficiency_factors_lut, + geometric_factors, + ) + intensities_uncertainty = convert_counts_to_intensity( + counts_uncertainty, + dependencies.l3a_direct_event_data.acquisition_time_per_esa_step, + dependencies.energy_per_charge_lut, + dependencies.efficiency_factors_lut, + geometric_factors, + ) + + rebinned_intensity = rebin_3d_distribution_azimuth_to_elevation( + intensities, + np.arange(1, 25), + position_elevation_lut, + dependencies.l3a_direct_event_data.half_spin_per_esa_step, + ) + + rebinned_intensity_uncertainty = rebin_3d_distribution_azimuth_to_elevation( + intensities_uncertainty, + np.arange(1, 25), + position_elevation_lut, + dependencies.l3a_direct_event_data.half_spin_per_esa_step, + ) + offset_degrees = ( + get_spacecraft_to_instrument_spin_phase_offset(SpiceFrame.IMAP_CODICE) * 360 + - 180 + ) + spin_angle_bins_in_dps = ( + dependencies.l3a_direct_event_data.spin_angle_bin + offset_degrees + ) % 360 return CodiceLoL3b3dDistributionDataProduct( input_metadata=self.input_metadata, epoch=dependencies.l3a_direct_event_data.epoch, @@ -356,14 +508,16 @@ def process_l3b_3d_distribution_product(self, dependencies: CodiceLoL3b3dDistrib rgfo_esa_step=dependencies.l3a_direct_event_data.rgfo_esa_step, rgfo_spin_sector=dependencies.l3a_direct_event_data.rgfo_spin_sector, rgfo_half_spin=dependencies.l3a_direct_event_data.rgfo_half_spin, - half_spin_per_esa_step=np.flip(dependencies.l3a_direct_event_data.half_spin_per_esa_step), + half_spin_per_esa_step=np.flip( + dependencies.l3a_direct_event_data.half_spin_per_esa_step + ), ) def _align_priority_counts_variable_to_direct_event_epochs( - priority_variable: np.ndarray, - priority_epochs: np.ndarray, - direct_event_epochs: np.ndarray, + priority_variable: np.ndarray, + priority_epochs: np.ndarray, + direct_event_epochs: np.ndarray, ) -> tuple[np.ndarray, np.ndarray]: num_direct_event_epochs = len(direct_event_epochs) aligned_shape = (num_direct_event_epochs,) + priority_variable.shape[1:] @@ -378,14 +532,32 @@ def _align_priority_counts_variable_to_direct_event_epochs( def _average_over_block(data_array: np.ndarray, block_size: int): - return np.array([data_array[i:i + block_size].mean(axis=0) for i in range(0, len(data_array), block_size)]) + return np.array( + [ + data_array[i : i + block_size].mean(axis=0) + for i in range(0, len(data_array), block_size) + ] + ) def _sum_over_block(data_array: np.ndarray, block_size: int): - return np.array([data_array[i:i + block_size].sum() for i in range(0, len(data_array), block_size)]) + return np.array( + [ + data_array[i : i + block_size].sum() + for i in range(0, len(data_array), block_size) + ] + ) def _average_dates_over_block(data_array: np.ndarray, block_size: int): - dates_as_int = data_array.astype('datetime64[us]').astype('int64') - return np.array([dates_as_int[i:i + block_size].mean() for i in range(0, len(dates_as_int), block_size)]).astype( - "datetime64[us]").astype('O') + dates_as_int = data_array.astype("datetime64[us]").astype("int64") + return ( + np.array( + [ + dates_as_int[i : i + block_size].mean() + for i in range(0, len(dates_as_int), block_size) + ] + ) + .astype("datetime64[us]") + .astype("O") + ) diff --git a/imap_l3_processing/codice/l3/lo/constants.py b/imap_l3_processing/codice/l3/lo/constants.py index 47edd09a6..38a2bf777 100644 --- a/imap_l3_processing/codice/l3/lo/constants.py +++ b/imap_l3_processing/codice/l3/lo/constants.py @@ -18,4 +18,4 @@ CODICE_INSTRUMENT_TO_SC_FRAME_SPIN_ANGLE_OFFSET = 46.0 CODICE_SECOND_ERA = date(2025, 11, 24) -CODICE_THIRD_ERA = date(2026, 1, 29) \ No newline at end of file +CODICE_THIRD_ERA = date(2026, 1, 29) diff --git a/imap_l3_processing/codice/l3/lo/direct_events/science/angle_lookup.py b/imap_l3_processing/codice/l3/lo/direct_events/science/angle_lookup.py index 517ba36d4..d4ca3e163 100644 --- a/imap_l3_processing/codice/l3/lo/direct_events/science/angle_lookup.py +++ b/imap_l3_processing/codice/l3/lo/direct_events/science/angle_lookup.py @@ -10,6 +10,7 @@ class SpinAngleLookup: baseline: np.ndarray offset = 270 + def __init__(self): num_bins = 24 self.baseline = np.linspace(0, 360, num_bins, endpoint=False) @@ -42,7 +43,33 @@ def __init__(self): self.lower_bin_edges = self.bin_centers - self.bin_deltas self.elevation_indices_by_apd = np.array( - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1]) + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 11, + 10, + 9, + 8, + 7, + 6, + 5, + 4, + 3, + 2, + 1, + ] + ) @property def num_bins(self): diff --git a/imap_l3_processing/codice/l3/lo/direct_events/science/efficiency_lookup.py b/imap_l3_processing/codice/l3/lo/direct_events/science/efficiency_lookup.py index b37496077..77b3e1bcd 100644 --- a/imap_l3_processing/codice/l3/lo/direct_events/science/efficiency_lookup.py +++ b/imap_l3_processing/codice/l3/lo/direct_events/science/efficiency_lookup.py @@ -10,6 +10,7 @@ POSITION = TypeVar("POSITION") ENERGIES = TypeVar("ENERGIES") + @dataclass class EfficiencyLookup: efficiency_data: np.ndarray[(ENERGIES, POSITION)] @@ -17,9 +18,11 @@ class EfficiencyLookup: @classmethod def read_from_csv(cls, path: Path, species: str) -> EfficiencyLookup: df = pd.read_csv(path) - efficiency_data = (df[(df['species'] == species) & (df['product'] == 'sw')] - .sort_values('esa_step') - .drop(['species', 'product', 'esa_step'], axis=1) - .to_numpy()) + efficiency_data = ( + df[(df["species"] == species) & (df["product"] == "sw")] + .sort_values("esa_step") + .drop(["species", "product", "esa_step"], axis=1) + .to_numpy() + ) return cls(efficiency_data=efficiency_data) diff --git a/imap_l3_processing/codice/l3/lo/direct_events/science/energy_lookup.py b/imap_l3_processing/codice/l3/lo/direct_events/science/energy_lookup.py index a4951e697..2742b4b9e 100644 --- a/imap_l3_processing/codice/l3/lo/direct_events/science/energy_lookup.py +++ b/imap_l3_processing/codice/l3/lo/direct_events/science/energy_lookup.py @@ -14,11 +14,15 @@ class EnergyLookup: @classmethod def read_from_csv(cls, path_to_csv: Path): - energy_lowers, energy_centers, energy_uppers = np.loadtxt(path_to_csv, delimiter=',', skiprows=1, usecols=(4, 5, 6)).T + energy_lowers, energy_centers, energy_uppers = np.loadtxt( + path_to_csv, delimiter=",", skiprows=1, usecols=(4, 5, 6) + ).T - return cls(bin_centers=energy_centers, - delta_minus=energy_centers - energy_lowers, - delta_plus=energy_uppers - energy_centers) + return cls( + bin_centers=energy_centers, + delta_minus=energy_centers - energy_lowers, + delta_plus=energy_uppers - energy_centers, + ) @property def num_bins(self): diff --git a/imap_l3_processing/codice/l3/lo/direct_events/science/geometric_factor_lookup.py b/imap_l3_processing/codice/l3/lo/direct_events/science/geometric_factor_lookup.py index b5cd463d3..7e7e0163c 100644 --- a/imap_l3_processing/codice/l3/lo/direct_events/science/geometric_factor_lookup.py +++ b/imap_l3_processing/codice/l3/lo/direct_events/science/geometric_factor_lookup.py @@ -6,56 +6,105 @@ import numpy as np import pandas as pd -from imap_l3_processing.codice.l3.lo.constants import CODICE_LO_NUM_SPIN_SECTORS, CODICE_LO_NUM_ESA_STEPS, \ - CODICE_LO_NUM_AZIMUTH_BINS, CODICE_SECOND_ERA, CODICE_THIRD_ERA +from imap_l3_processing.codice.l3.lo.constants import ( + CODICE_LO_NUM_SPIN_SECTORS, + CODICE_LO_NUM_ESA_STEPS, + CODICE_LO_NUM_AZIMUTH_BINS, + CODICE_SECOND_ERA, + CODICE_THIRD_ERA, +) def _half_spin_to_esa_step_lookup(): - return np.array([ - 0, 1, 2, 3, 5, 7, 9, 11, 14, 17, 20, 23, 27, 31, 35, 39, 44, 49, 54, 59, 64, 69, 74, 79, 85, 91, 97, 103, 109, - 115, 121, 127 - ]) - - -POSITION = TypeVar('POSITION') -ESA_STEP = TypeVar('ESA_STEP') + return np.array( + [ + 0, + 1, + 2, + 3, + 5, + 7, + 9, + 11, + 14, + 17, + 20, + 23, + 27, + 31, + 35, + 39, + 44, + 49, + 54, + 59, + 64, + 69, + 74, + 79, + 85, + 91, + 97, + 103, + 109, + 115, + 121, + 127, + ] + ) + + +POSITION = TypeVar("POSITION") +ESA_STEP = TypeVar("ESA_STEP") @dataclass class GeometricFactorLookup: _full_factor: np.ndarray[(ESA_STEP, POSITION)] _reduced_factor: np.ndarray[(ESA_STEP, POSITION)] - _esa_step_end_index: np.ndarray = field(default_factory=_half_spin_to_esa_step_lookup) + _esa_step_end_index: np.ndarray = field( + default_factory=_half_spin_to_esa_step_lookup + ) @classmethod def read_from_csv(cls, filepath: Path): - df = (pd.read_csv(filepath) - .sort_values(['mode', 'esa_step']) - .groupby('mode')) + df = pd.read_csv(filepath).sort_values(["mode", "esa_step"]).groupby("mode") - full = df.get_group('full').drop(['mode', 'esa_step'], axis=1).to_numpy() - reduced = df.get_group('reduced').drop(['mode', 'esa_step'], axis=1).to_numpy() + full = df.get_group("full").drop(["mode", "esa_step"], axis=1).to_numpy() + reduced = df.get_group("reduced").drop(["mode", "esa_step"], axis=1).to_numpy() return cls(full, reduced) - def get_geometric_factors(self, - rgfo_half_spin: np.ma.masked_array, - rgfo_spin_sector: np.ma.masked_array, - rgfo_esa_step: np.ma.masked_array, - half_spin: np.ma.masked_array, - epoch: date, - ) -> np.ndarray: - - full_shape = (rgfo_half_spin.shape[0], CODICE_LO_NUM_ESA_STEPS, CODICE_LO_NUM_SPIN_SECTORS, CODICE_LO_NUM_AZIMUTH_BINS) - reduced_factor_full_shape = np.broadcast_to(self._reduced_factor[None, :, None, :], full_shape) - full_factor_full_shape = np.broadcast_to(self._full_factor[None, :, None, :], full_shape) + def get_geometric_factors( + self, + rgfo_half_spin: np.ma.masked_array, + rgfo_spin_sector: np.ma.masked_array, + rgfo_esa_step: np.ma.masked_array, + half_spin: np.ma.masked_array, + epoch: date, + ) -> np.ndarray: + + full_shape = ( + rgfo_half_spin.shape[0], + CODICE_LO_NUM_ESA_STEPS, + CODICE_LO_NUM_SPIN_SECTORS, + CODICE_LO_NUM_AZIMUTH_BINS, + ) + reduced_factor_full_shape = np.broadcast_to( + self._reduced_factor[None, :, None, :], full_shape + ) + full_factor_full_shape = np.broadcast_to( + self._full_factor[None, :, None, :], full_shape + ) if epoch < CODICE_SECOND_ERA: use_reduced = self._rgfo_half_spin_compare_only(rgfo_half_spin, half_spin) elif epoch < CODICE_THIRD_ERA: use_reduced = False else: - use_reduced = self._is_past_rgfo(rgfo_half_spin, rgfo_spin_sector, rgfo_esa_step, half_spin) + use_reduced = self._is_past_rgfo( + rgfo_half_spin, rgfo_spin_sector, rgfo_esa_step, half_spin + ) return np.where( use_reduced, reduced_factor_full_shape, @@ -64,27 +113,28 @@ def get_geometric_factors(self, @staticmethod def _rgfo_half_spin_compare_only( - rgfo_half_spin: np.ma.masked_array, - half_spin: np.ma.masked_array - ): + rgfo_half_spin: np.ma.masked_array, half_spin: np.ma.masked_array + ): rgfo_half_spin_full_shape = rgfo_half_spin[:, None, None, None] half_spin_full_shape = half_spin[:, :, None, None] return half_spin_full_shape > rgfo_half_spin_full_shape - @staticmethod - def _is_past_rgfo(rgfo_half_spin: np.ma.masked_array, - rgfo_spin_sector: np.ma.masked_array, - rgfo_esa_step: np.ma.masked_array, - half_spin: np.ma.masked_array, - ) -> np.ndarray: + def _is_past_rgfo( + rgfo_half_spin: np.ma.masked_array, + rgfo_spin_sector: np.ma.masked_array, + rgfo_esa_step: np.ma.masked_array, + half_spin: np.ma.masked_array, + ) -> np.ndarray: rgfo_half_spin_e = rgfo_half_spin[:, None, None, None] rgfo_esa_step_e = rgfo_esa_step[:, None, None, None] rgfo_spin_sector_mod = (rgfo_spin_sector % 12)[:, None, None, None] half_spin_e = half_spin[:, :, None, None] esa_step_axis = np.arange(CODICE_LO_NUM_ESA_STEPS)[None, :, None, None] - spin_sector_axis_mod = np.arange(CODICE_LO_NUM_SPIN_SECTORS)[None, None, :, None] % 12 + spin_sector_axis_mod = ( + np.arange(CODICE_LO_NUM_SPIN_SECTORS)[None, None, :, None] % 12 + ) half_spin_past = half_spin_e > rgfo_half_spin_e half_spin_match = half_spin_e == rgfo_half_spin_e @@ -93,7 +143,7 @@ def _is_past_rgfo(rgfo_half_spin: np.ma.masked_array, esa_step_past = esa_step_axis > rgfo_esa_step_e return ( - half_spin_past - | (half_spin_match & spin_sector_past) - | (half_spin_match & spin_sector_match & esa_step_past) + half_spin_past + | (half_spin_match & spin_sector_past) + | (half_spin_match & spin_sector_match & esa_step_past) ) diff --git a/imap_l3_processing/codice/l3/lo/direct_events/science/mass_coefficient_lookup.py b/imap_l3_processing/codice/l3/lo/direct_events/science/mass_coefficient_lookup.py index 91685aa24..9feb8c0ef 100644 --- a/imap_l3_processing/codice/l3/lo/direct_events/science/mass_coefficient_lookup.py +++ b/imap_l3_processing/codice/l3/lo/direct_events/science/mass_coefficient_lookup.py @@ -13,5 +13,5 @@ def __getitem__(self, item): @classmethod def read_from_csv(cls, path: Path): - mass_coefficients = np.loadtxt(path, dtype=float, delimiter=',') + mass_coefficients = np.loadtxt(path, dtype=float, delimiter=",") return cls(mass_coefficients) diff --git a/imap_l3_processing/codice/l3/lo/direct_events/science/mass_species_bin_lookup.py b/imap_l3_processing/codice/l3/lo/direct_events/science/mass_species_bin_lookup.py index d667cadd9..fdd9e9a5c 100644 --- a/imap_l3_processing/codice/l3/lo/direct_events/science/mass_species_bin_lookup.py +++ b/imap_l3_processing/codice/l3/lo/direct_events/science/mass_species_bin_lookup.py @@ -15,7 +15,9 @@ class MassSpeciesBinLookup: def read_from_csv(cls, path: Path | str): loaded_csv = np.genfromtxt(path, delimiter=",", dtype=None, skip_header=1) range_data = list(zip(*loaded_csv)) - [species, mpq_min, mpq_max, mass_min, mass_max] = [list(column) for column in range_data] + [species, mpq_min, mpq_max, mass_min, mass_max] = [ + list(column) for column in range_data + ] return cls( species=species, @@ -27,7 +29,10 @@ def get_species(self, mass: float, mass_per_charge: float) -> Optional[str]: for i in range(len(self.species)): lower_mass, upper_mass = self.mass_ranges[i] lower_mass_per_charge, upper_mass_per_charge = self.mass_per_charge[i] - if lower_mass <= mass < upper_mass and lower_mass_per_charge <= mass_per_charge < upper_mass_per_charge: + if ( + lower_mass <= mass < upper_mass + and lower_mass_per_charge <= mass_per_charge < upper_mass_per_charge + ): return self.species[i] def get_species_index(self, species: str) -> int: diff --git a/imap_l3_processing/codice/l3/lo/models.py b/imap_l3_processing/codice/l3/lo/models.py index 4dd74ad30..454ef345b 100644 --- a/imap_l3_processing/codice/l3/lo/models.py +++ b/imap_l3_processing/codice/l3/lo/models.py @@ -8,7 +8,10 @@ from numpy import ndarray from spacepy.pycdf import CDF -from imap_l3_processing.cdf.cdf_utils import read_numeric_variable, read_variable_and_mask_fill_values +from imap_l3_processing.cdf.cdf_utils import ( + read_numeric_variable, + read_variable_and_mask_fill_values, +) from imap_l3_processing.models import DataProductVariable, DataProduct CODICE_LO_L2_NUM_PRIORITIES = 7 @@ -87,7 +90,9 @@ def read_from_cdf(cls, l2_lo_sw_species_cdf: Path): ) -EnergyAndSpinAngle = namedtuple(typename="EnergyAndSpinAngle", field_names=["energy", "spin_angle"]) +EnergyAndSpinAngle = namedtuple( + typename="EnergyAndSpinAngle", field_names=["energy", "spin_angle"] +) @dataclass @@ -117,26 +122,48 @@ def read_from_cdf(cls, l2_direct_event_cdf: Path): epoch=cdf["epoch"][...], epoch_delta_plus=cdf["epoch_delta_plus"][...], epoch_delta_minus=cdf["epoch_delta_minus"][...], - apd_energy=read_variable_and_mask_fill_values(cdf["apd_energy"])[:, :CODICE_LO_L2_NUM_PRIORITIES, ...], - gain=read_variable_and_mask_fill_values(cdf["gain"])[:, :CODICE_LO_L2_NUM_PRIORITIES, ...], - apd_id=read_variable_and_mask_fill_values(cdf["apd_id"])[:, :CODICE_LO_L2_NUM_PRIORITIES, ...], - data_quality=read_variable_and_mask_fill_values(cdf["data_quality"])[:, :CODICE_LO_L2_NUM_PRIORITIES, - ...], - energy_step=read_variable_and_mask_fill_values(cdf["energy_step"])[:, :CODICE_LO_L2_NUM_PRIORITIES, - ...], - energy_per_charge=read_variable_and_mask_fill_values(cdf["energy_per_charge"])[:, - :CODICE_LO_L2_NUM_PRIORITIES, - ...], - multi_flag=read_variable_and_mask_fill_values(cdf["multi_flag"])[:, :CODICE_LO_L2_NUM_PRIORITIES, ...], - num_events=read_variable_and_mask_fill_values(cdf["num_events"])[:, :CODICE_LO_L2_NUM_PRIORITIES, ...], - spin_angle=read_variable_and_mask_fill_values(cdf["spin_angle"])[:, :CODICE_LO_L2_NUM_PRIORITIES, ...], - spin_sector=read_variable_and_mask_fill_values(cdf["spin_sector"])[:, :CODICE_LO_L2_NUM_PRIORITIES, - ...].astype(int), - elevation_angle=read_variable_and_mask_fill_values(cdf["elevation_angle"])[:, - :CODICE_LO_L2_NUM_PRIORITIES, ...], - tof=read_variable_and_mask_fill_values(cdf["tof"])[:, :CODICE_LO_L2_NUM_PRIORITIES, ...], - type=read_variable_and_mask_fill_values(cdf["type"])[:, :CODICE_LO_L2_NUM_PRIORITIES, ...], - position=read_variable_and_mask_fill_values(cdf["position"])[:, :CODICE_LO_L2_NUM_PRIORITIES, ...], + apd_energy=read_variable_and_mask_fill_values(cdf["apd_energy"])[ + :, :CODICE_LO_L2_NUM_PRIORITIES, ... + ], + gain=read_variable_and_mask_fill_values(cdf["gain"])[ + :, :CODICE_LO_L2_NUM_PRIORITIES, ... + ], + apd_id=read_variable_and_mask_fill_values(cdf["apd_id"])[ + :, :CODICE_LO_L2_NUM_PRIORITIES, ... + ], + data_quality=read_variable_and_mask_fill_values(cdf["data_quality"])[ + :, :CODICE_LO_L2_NUM_PRIORITIES, ... + ], + energy_step=read_variable_and_mask_fill_values(cdf["energy_step"])[ + :, :CODICE_LO_L2_NUM_PRIORITIES, ... + ], + energy_per_charge=read_variable_and_mask_fill_values( + cdf["energy_per_charge"] + )[:, :CODICE_LO_L2_NUM_PRIORITIES, ...], + multi_flag=read_variable_and_mask_fill_values(cdf["multi_flag"])[ + :, :CODICE_LO_L2_NUM_PRIORITIES, ... + ], + num_events=read_variable_and_mask_fill_values(cdf["num_events"])[ + :, :CODICE_LO_L2_NUM_PRIORITIES, ... + ], + spin_angle=read_variable_and_mask_fill_values(cdf["spin_angle"])[ + :, :CODICE_LO_L2_NUM_PRIORITIES, ... + ], + spin_sector=read_variable_and_mask_fill_values(cdf["spin_sector"])[ + :, :CODICE_LO_L2_NUM_PRIORITIES, ... + ].astype(int), + elevation_angle=read_variable_and_mask_fill_values( + cdf["elevation_angle"] + )[:, :CODICE_LO_L2_NUM_PRIORITIES, ...], + tof=read_variable_and_mask_fill_values(cdf["tof"])[ + :, :CODICE_LO_L2_NUM_PRIORITIES, ... + ], + type=read_variable_and_mask_fill_values(cdf["type"])[ + :, :CODICE_LO_L2_NUM_PRIORITIES, ... + ], + position=read_variable_and_mask_fill_values(cdf["position"])[ + :, :CODICE_LO_L2_NUM_PRIORITIES, ... + ], ) result.tof = np.ma.masked_less(result.tof, 0) return result @@ -176,10 +203,16 @@ def read_from_cdf(cls, cdf_path: Path): epoch_delta_minus=cdf["epoch_delta_minus"][...], acquisition_time_per_esa_step=cdf["acquisition_time_per_esa_step"][...], spin_sector_index=cdf["spin_sector"][...], - rgfo_half_spin=read_variable_and_mask_fill_values(cdf["rgfo_half_spin"]), + rgfo_half_spin=read_variable_and_mask_fill_values( + cdf["rgfo_half_spin"] + ), nso_half_spin=read_variable_and_mask_fill_values(cdf["nso_half_spin"]), - sw_bias_gain_mode=read_variable_and_mask_fill_values(cdf["sw_bias_gain_mode"]), - st_bias_gain_mode=read_variable_and_mask_fill_values(cdf["st_bias_gain_mode"]), + sw_bias_gain_mode=read_variable_and_mask_fill_values( + cdf["sw_bias_gain_mode"] + ), + st_bias_gain_mode=read_variable_and_mask_fill_values( + cdf["st_bias_gain_mode"] + ), data_quality=read_variable_and_mask_fill_values(cdf["data_quality"]), spin_period=read_numeric_variable(cdf["spin_period"]), p0_tcrs=read_variable_and_mask_fill_values(cdf["p0_tcrs"]), @@ -187,10 +220,16 @@ def read_from_cdf(cls, cdf_path: Path): p2_heplusplus=read_variable_and_mask_fill_values(cdf["p2_heplusplus"]), p3_heavies=read_variable_and_mask_fill_values(cdf["p3_heavies"]), p4_dcrs=read_variable_and_mask_fill_values(cdf["p4_dcrs"]), - half_spin_per_esa_step=read_variable_and_mask_fill_values(cdf["half_spin_per_esa_step"]), - rgfo_spin_sector=read_variable_and_mask_fill_values(cdf["rgfo_spin_sector"]), + half_spin_per_esa_step=read_variable_and_mask_fill_values( + cdf["half_spin_per_esa_step"] + ), + rgfo_spin_sector=read_variable_and_mask_fill_values( + cdf["rgfo_spin_sector"] + ), rgfo_esa_step=read_variable_and_mask_fill_values(cdf["rgfo_esa_step"]), - nso_spin_sector=read_variable_and_mask_fill_values(cdf["nso_spin_sector"]), + nso_spin_sector=read_variable_and_mask_fill_values( + cdf["nso_spin_sector"] + ), nso_esa_step=read_variable_and_mask_fill_values(cdf["nso_esa_step"]), esa_step=read_variable_and_mask_fill_values(cdf["esa_step"]), ) @@ -221,13 +260,21 @@ def read_from_cdf(cls, cdf_path: Path): epoch_delta_minus=cdf["epoch_delta_minus"][...], acquisition_time_per_esa_step=cdf["acquisition_time_per_esa_step"][...], spin_sector_index=cdf["spin_sector"][...], - rgfo_half_spin=read_variable_and_mask_fill_values(cdf["rgfo_half_spin"]), + rgfo_half_spin=read_variable_and_mask_fill_values( + cdf["rgfo_half_spin"] + ), data_quality=read_variable_and_mask_fill_values(cdf["data_quality"]), p5_heavies=read_variable_and_mask_fill_values(cdf["p5_heavies"]), - p6_hplus_heplusplus=read_variable_and_mask_fill_values(cdf["p6_hplus_heplusplus"]), + p6_hplus_heplusplus=read_variable_and_mask_fill_values( + cdf["p6_hplus_heplusplus"] + ), nso_half_spin=read_variable_and_mask_fill_values(cdf["nso_half_spin"]), - sw_bias_gain_mode=read_variable_and_mask_fill_values(cdf["sw_bias_gain_mode"]), - st_bias_gain_mode=read_variable_and_mask_fill_values(cdf["st_bias_gain_mode"]), + sw_bias_gain_mode=read_variable_and_mask_fill_values( + cdf["sw_bias_gain_mode"] + ), + st_bias_gain_mode=read_variable_and_mask_fill_values( + cdf["st_bias_gain_mode"] + ), spin_period=read_numeric_variable(cdf["spin_period"]), ) @@ -257,20 +304,48 @@ def read_from_cdf(cls, cdf_path: Path | str): return cls( epoch=cdf[EPOCH_VAR_NAME][...], epoch_delta=cdf[EPOCH_DELTA_VAR_NAME][...], - hplus_partial_density=read_numeric_variable(cdf[H_PARTIAL_DENSITY_VAR_NAME]), - heplusplus_partial_density=read_numeric_variable(cdf[HE_PARTIAL_DENSITY_VAR_NAME]), - cplus4_partial_density=read_numeric_variable(cdf[C4_PARTIAL_DENSITY_VAR_NAME]), - cplus5_partial_density=read_numeric_variable(cdf[C5_PARTIAL_DENSITY_VAR_NAME]), - cplus6_partial_density=read_numeric_variable(cdf[C6_PARTIAL_DENSITY_VAR_NAME]), - oplus5_partial_density=read_numeric_variable(cdf[O5_PARTIAL_DENSITY_VAR_NAME]), - oplus6_partial_density=read_numeric_variable(cdf[O6_PARTIAL_DENSITY_VAR_NAME]), - oplus7_partial_density=read_numeric_variable(cdf[O7_PARTIAL_DENSITY_VAR_NAME]), - oplus8_partial_density=read_numeric_variable(cdf[O8_PARTIAL_DENSITY_VAR_NAME]), - ne_partial_density=read_numeric_variable(cdf[NE_PARTIAL_DENSITY_VAR_NAME]), - mg_partial_density=read_numeric_variable(cdf[MG_PARTIAL_DENSITY_VAR_NAME]), - si_partial_density=read_numeric_variable(cdf[SI_PARTIAL_DENSITY_VAR_NAME]), - fe_loq_partial_density=read_numeric_variable(cdf[FE_LOW_PARTIAL_DENSITY_VAR_NAME]), - fe_hiq_partial_density=read_numeric_variable(cdf[FE_HIGH_PARTIAL_DENSITY_VAR_NAME]), + hplus_partial_density=read_numeric_variable( + cdf[H_PARTIAL_DENSITY_VAR_NAME] + ), + heplusplus_partial_density=read_numeric_variable( + cdf[HE_PARTIAL_DENSITY_VAR_NAME] + ), + cplus4_partial_density=read_numeric_variable( + cdf[C4_PARTIAL_DENSITY_VAR_NAME] + ), + cplus5_partial_density=read_numeric_variable( + cdf[C5_PARTIAL_DENSITY_VAR_NAME] + ), + cplus6_partial_density=read_numeric_variable( + cdf[C6_PARTIAL_DENSITY_VAR_NAME] + ), + oplus5_partial_density=read_numeric_variable( + cdf[O5_PARTIAL_DENSITY_VAR_NAME] + ), + oplus6_partial_density=read_numeric_variable( + cdf[O6_PARTIAL_DENSITY_VAR_NAME] + ), + oplus7_partial_density=read_numeric_variable( + cdf[O7_PARTIAL_DENSITY_VAR_NAME] + ), + oplus8_partial_density=read_numeric_variable( + cdf[O8_PARTIAL_DENSITY_VAR_NAME] + ), + ne_partial_density=read_numeric_variable( + cdf[NE_PARTIAL_DENSITY_VAR_NAME] + ), + mg_partial_density=read_numeric_variable( + cdf[MG_PARTIAL_DENSITY_VAR_NAME] + ), + si_partial_density=read_numeric_variable( + cdf[SI_PARTIAL_DENSITY_VAR_NAME] + ), + fe_loq_partial_density=read_numeric_variable( + cdf[FE_LOW_PARTIAL_DENSITY_VAR_NAME] + ), + fe_hiq_partial_density=read_numeric_variable( + cdf[FE_HIGH_PARTIAL_DENSITY_VAR_NAME] + ), ) @@ -282,24 +357,52 @@ def to_data_product_variables(self) -> list[DataProductVariable]: return [ DataProductVariable(EPOCH_VAR_NAME, self.data.epoch), DataProductVariable(EPOCH_DELTA_VAR_NAME, self.data.epoch_delta), - DataProductVariable(H_PARTIAL_DENSITY_VAR_NAME, self.data.hplus_partial_density), - DataProductVariable(HE_PARTIAL_DENSITY_VAR_NAME, self.data.heplusplus_partial_density), - DataProductVariable(C4_PARTIAL_DENSITY_VAR_NAME, self.data.cplus4_partial_density), - DataProductVariable(C5_PARTIAL_DENSITY_VAR_NAME, self.data.cplus5_partial_density), - DataProductVariable(C6_PARTIAL_DENSITY_VAR_NAME, self.data.cplus6_partial_density), - DataProductVariable(O5_PARTIAL_DENSITY_VAR_NAME, self.data.oplus5_partial_density), - DataProductVariable(O6_PARTIAL_DENSITY_VAR_NAME, self.data.oplus6_partial_density), - DataProductVariable(O7_PARTIAL_DENSITY_VAR_NAME, self.data.oplus7_partial_density), - DataProductVariable(O8_PARTIAL_DENSITY_VAR_NAME, self.data.oplus8_partial_density), - DataProductVariable(NE_PARTIAL_DENSITY_VAR_NAME, self.data.ne_partial_density), - DataProductVariable(MG_PARTIAL_DENSITY_VAR_NAME, self.data.mg_partial_density), - DataProductVariable(SI_PARTIAL_DENSITY_VAR_NAME, self.data.si_partial_density), - DataProductVariable(FE_LOW_PARTIAL_DENSITY_VAR_NAME, self.data.fe_loq_partial_density), - DataProductVariable(FE_HIGH_PARTIAL_DENSITY_VAR_NAME, self.data.fe_hiq_partial_density) + DataProductVariable( + H_PARTIAL_DENSITY_VAR_NAME, self.data.hplus_partial_density + ), + DataProductVariable( + HE_PARTIAL_DENSITY_VAR_NAME, self.data.heplusplus_partial_density + ), + DataProductVariable( + C4_PARTIAL_DENSITY_VAR_NAME, self.data.cplus4_partial_density + ), + DataProductVariable( + C5_PARTIAL_DENSITY_VAR_NAME, self.data.cplus5_partial_density + ), + DataProductVariable( + C6_PARTIAL_DENSITY_VAR_NAME, self.data.cplus6_partial_density + ), + DataProductVariable( + O5_PARTIAL_DENSITY_VAR_NAME, self.data.oplus5_partial_density + ), + DataProductVariable( + O6_PARTIAL_DENSITY_VAR_NAME, self.data.oplus6_partial_density + ), + DataProductVariable( + O7_PARTIAL_DENSITY_VAR_NAME, self.data.oplus7_partial_density + ), + DataProductVariable( + O8_PARTIAL_DENSITY_VAR_NAME, self.data.oplus8_partial_density + ), + DataProductVariable( + NE_PARTIAL_DENSITY_VAR_NAME, self.data.ne_partial_density + ), + DataProductVariable( + MG_PARTIAL_DENSITY_VAR_NAME, self.data.mg_partial_density + ), + DataProductVariable( + SI_PARTIAL_DENSITY_VAR_NAME, self.data.si_partial_density + ), + DataProductVariable( + FE_LOW_PARTIAL_DENSITY_VAR_NAME, self.data.fe_loq_partial_density + ), + DataProductVariable( + FE_HIGH_PARTIAL_DENSITY_VAR_NAME, self.data.fe_hiq_partial_density + ), ] -ACQUISITION_TIME_PER_ESA_STEP_VAR_NAME = 'acquisition_time_per_esa_step' +ACQUISITION_TIME_PER_ESA_STEP_VAR_NAME = "acquisition_time_per_esa_step" EVENT_INDEX_VAR_NAME = "event_index" SPIN_ANGLE_DELTA_BIN_VAR_NAME = "spin_angle_bin_delta" SPIN_ANGLE_BIN_VAR_NAME = "spin_angle_bin" @@ -331,10 +434,10 @@ def to_data_product_variables(self) -> list[DataProductVariable]: RGFO_SPIN_SECTOR_VAR_NAME = "rgfo_spin_sector" RGFO_HALF_SPIN_VAR_NAME = "rgfo_half_spin" RGFO_ESA_STEP_VAR_NAME = "rgfo_esa_step" -NSO_HALF_SPIN_NAME = 'nso_half_spin' +NSO_HALF_SPIN_NAME = "nso_half_spin" NSO_SPIN_SECTOR_VAR_NAME = "nso_spin_sector" NSO_ESA_STEP_VAR_NAME = "nso_esa_step" -RGFO_HALF_SPIN_NAME = 'rgfo_half_spin' +RGFO_HALF_SPIN_NAME = "rgfo_half_spin" ESA_STEP_VAR_NAME = "esa_step" ENERGY_PER_CHARGE_VAR_NAME = "energy_per_charge" TYPE_VAR_NAME = "type" @@ -376,29 +479,49 @@ def read_from_cdf(cls, cdf_path: Path | str): return cls( epoch=cdf[EPOCH_VAR_NAME][...], epoch_delta=cdf[EPOCH_DELTA_VAR_NAME][...], - acquisition_time_per_esa_step=read_numeric_variable(cdf[ACQUISITION_TIME_PER_ESA_STEP_VAR_NAME]), + acquisition_time_per_esa_step=read_numeric_variable( + cdf[ACQUISITION_TIME_PER_ESA_STEP_VAR_NAME] + ), apd_energy=read_numeric_variable(cdf[APD_ENERGY_VAR_NAME]), apd_id=read_variable_and_mask_fill_values(cdf[APD_ID_VAR_NAME]), data_quality=cdf[DATA_QUALITY_VAR_NAME][...], elevation=read_numeric_variable(cdf[ELEVATION_VAR_NAME]), - energy_per_charge=read_numeric_variable(cdf[ENERGY_PER_CHARGE_VAR_NAME]), - energy_step=read_variable_and_mask_fill_values(cdf[ENERGY_STEP_VAR_NAME]), + energy_per_charge=read_numeric_variable( + cdf[ENERGY_PER_CHARGE_VAR_NAME] + ), + energy_step=read_variable_and_mask_fill_values( + cdf[ENERGY_STEP_VAR_NAME] + ), gain=read_variable_and_mask_fill_values(cdf[GAIN_VAR_NAME]), mass=read_numeric_variable(cdf[MASS_VAR_NAME]), mass_per_charge=read_numeric_variable(cdf[MASS_PER_CHARGE_VAR_NAME]), multi_flag=read_variable_and_mask_fill_values(cdf[MULTI_FLAG_VAR_NAME]), normalization=read_numeric_variable(cdf[NORMALIZATION_VAR_NAME]), - normalization_per_event=read_numeric_variable(cdf[NORMALIZATION_PER_EVENT_VAR_NAME]), + normalization_per_event=read_numeric_variable( + cdf[NORMALIZATION_PER_EVENT_VAR_NAME] + ), num_events=read_variable_and_mask_fill_values(cdf[NUM_EVENTS_VAR_NAME]), position=read_variable_and_mask_fill_values(cdf[POSITION_VAR_NAME]), spin_angle=read_numeric_variable(cdf[SPIN_ANGLE_VAR_NAME]), spin_angle_bin=read_numeric_variable(cdf[SPIN_ANGLE_BIN_VAR_NAME]), - spin_angle_bin_delta=read_numeric_variable(cdf[SPIN_ANGLE_DELTA_BIN_VAR_NAME]), - spin_sector=read_variable_and_mask_fill_values(cdf[SPIN_SECTOR_VAR_NAME]), - half_spin_per_esa_step=read_variable_and_mask_fill_values(cdf[HALF_SPIN_PER_ESA_STEP_VAR_NAME]), - rgfo_half_spin=read_variable_and_mask_fill_values(cdf[RGFO_HALF_SPIN_VAR_NAME]), - rgfo_spin_sector=read_variable_and_mask_fill_values(cdf[RGFO_SPIN_SECTOR_VAR_NAME]), - rgfo_esa_step=read_variable_and_mask_fill_values(cdf[RGFO_ESA_STEP_VAR_NAME]), + spin_angle_bin_delta=read_numeric_variable( + cdf[SPIN_ANGLE_DELTA_BIN_VAR_NAME] + ), + spin_sector=read_variable_and_mask_fill_values( + cdf[SPIN_SECTOR_VAR_NAME] + ), + half_spin_per_esa_step=read_variable_and_mask_fill_values( + cdf[HALF_SPIN_PER_ESA_STEP_VAR_NAME] + ), + rgfo_half_spin=read_variable_and_mask_fill_values( + cdf[RGFO_HALF_SPIN_VAR_NAME] + ), + rgfo_spin_sector=read_variable_and_mask_fill_values( + cdf[RGFO_SPIN_SECTOR_VAR_NAME] + ), + rgfo_esa_step=read_variable_and_mask_fill_values( + cdf[RGFO_ESA_STEP_VAR_NAME] + ), tof=read_variable_and_mask_fill_values(cdf[TOF_VAR_NAME]), type=read_variable_and_mask_fill_values(cdf[TYPE_VAR_NAME]), ) @@ -435,14 +558,21 @@ def to_data_product_variables(self) -> list[DataProductVariable]: return [ DataProductVariable(EPOCH_VAR_NAME, self.epoch), DataProductVariable(EPOCH_DELTA_VAR_NAME, self.epoch_delta), - DataProductVariable(ACQUISITION_TIME_PER_ESA_STEP_VAR_NAME, self.acquisition_time_per_esa_step), + DataProductVariable( + ACQUISITION_TIME_PER_ESA_STEP_VAR_NAME, + self.acquisition_time_per_esa_step, + ), DataProductVariable(APD_ENERGY_VAR_NAME, self.apd_energy), DataProductVariable(APD_ID_VAR_NAME, self.apd_id), DataProductVariable(DATA_QUALITY_VAR_NAME, self.data_quality), DataProductVariable(ELEVATION_VAR_NAME, self.elevation), DataProductVariable(ENERGY_BIN_VAR_NAME, self.energy_bin), - DataProductVariable(ENERGY_BIN_DELTA_MINUS_VAR_NAME, self.energy_bin_delta_minus), - DataProductVariable(ENERGY_BIN_DELTA_PLUS_VAR_NAME, self.energy_bin_delta_plus), + DataProductVariable( + ENERGY_BIN_DELTA_MINUS_VAR_NAME, self.energy_bin_delta_minus + ), + DataProductVariable( + ENERGY_BIN_DELTA_PLUS_VAR_NAME, self.energy_bin_delta_plus + ), DataProductVariable(ENERGY_BIN_LABEL_VAR_NAME, self.energy_bin_label), DataProductVariable(ENERGY_PER_CHARGE_VAR_NAME, self.energy_per_charge), DataProductVariable(ENERGY_STEP_VAR_NAME, self.energy_step), @@ -450,26 +580,36 @@ def to_data_product_variables(self) -> list[DataProductVariable]: DataProductVariable(EVENT_INDEX_VAR_NAME, self.event_index), DataProductVariable(EVENT_INDEX_LABEL_VAR_NAME, self.event_index_label), DataProductVariable(GAIN_VAR_NAME, self.gain), - DataProductVariable(HALF_SPIN_PER_ESA_STEP_VAR_NAME, self.half_spin_per_esa_step), + DataProductVariable( + HALF_SPIN_PER_ESA_STEP_VAR_NAME, self.half_spin_per_esa_step + ), DataProductVariable(MASS_VAR_NAME, self.mass), DataProductVariable(MASS_PER_CHARGE_VAR_NAME, self.mass_per_charge), DataProductVariable(MULTI_FLAG_VAR_NAME, self.multi_flag), DataProductVariable(NORMALIZATION_VAR_NAME, self.normalization), - DataProductVariable(NORMALIZATION_PER_EVENT_VAR_NAME, self.normalization_per_event), + DataProductVariable( + NORMALIZATION_PER_EVENT_VAR_NAME, self.normalization_per_event + ), DataProductVariable(NSO_ESA_STEP_VAR_NAME, self.nso_esa_step), DataProductVariable(NSO_SPIN_SECTOR_VAR_NAME, self.nso_spin_sector), DataProductVariable(NSO_HALF_SPIN_NAME, self.nso_half_spin), DataProductVariable(NUM_EVENTS_VAR_NAME, self.num_events), DataProductVariable(POSITION_VAR_NAME, self.position), DataProductVariable(PRIORITY_INDEX_VAR_NAME, self.priority_index), - DataProductVariable(PRIORITY_INDEX_LABEL_VAR_NAME, self.priority_index_label), + DataProductVariable( + PRIORITY_INDEX_LABEL_VAR_NAME, self.priority_index_label + ), DataProductVariable(RGFO_ESA_STEP_VAR_NAME, self.rgfo_esa_step), DataProductVariable(RGFO_SPIN_SECTOR_VAR_NAME, self.rgfo_spin_sector), DataProductVariable(RGFO_HALF_SPIN_NAME, self.rgfo_half_spin), DataProductVariable(SPIN_ANGLE_VAR_NAME, self.spin_angle), DataProductVariable(SPIN_ANGLE_BIN_VAR_NAME, self.spin_angle_bin), - DataProductVariable(SPIN_ANGLE_DELTA_BIN_VAR_NAME, self.spin_angle_bin_delta), - DataProductVariable(SPIN_ANGLE_BIN_LABEL_VAR_NAME, self.spin_angle_bin_label), + DataProductVariable( + SPIN_ANGLE_DELTA_BIN_VAR_NAME, self.spin_angle_bin_delta + ), + DataProductVariable( + SPIN_ANGLE_BIN_LABEL_VAR_NAME, self.spin_angle_bin_label + ), DataProductVariable(SPIN_SECTOR_VAR_NAME, self.spin_sector), DataProductVariable(TOF_VAR_NAME, self.tof), DataProductVariable(TYPE_VAR_NAME, self.type), @@ -507,7 +647,7 @@ def to_data_product_variables(self) -> list[DataProductVariable]: DataProductVariable(C6_TO_C5_RATIO_VAR_NAME, self.c6_to_c5_ratio), DataProductVariable(C6_TO_C4_RATIO_VAR_NAME, self.c6_to_c4_ratio), DataProductVariable(O7_TO_O6_RATIO_VAR_NAME, self.o7_to_o6_ratio), - DataProductVariable(FELO_TO_FEHI_RATIO_VAR_NAME, self.felo_to_fehi_ratio) + DataProductVariable(FELO_TO_FEHI_RATIO_VAR_NAME, self.felo_to_fehi_ratio), ] @@ -522,12 +662,19 @@ def to_data_product_variables(self) -> list[DataProductVariable]: return [ DataProductVariable("epoch", self.epoch), DataProductVariable("epoch_delta", self.epoch_delta), - DataProductVariable("oxygen_charge_state_distribution", self.oxygen_charge_state_distribution), - DataProductVariable("carbon_charge_state_distribution", self.carbon_charge_state_distribution), + DataProductVariable( + "oxygen_charge_state_distribution", + self.oxygen_charge_state_distribution, + ), + DataProductVariable( + "carbon_charge_state_distribution", + self.carbon_charge_state_distribution, + ), DataProductVariable("oxygen_charge_state", np.array([5, 6, 7, 8])), DataProductVariable("carbon_charge_state", np.array([4, 5, 6])), ] + ENERGY_VAR_NAME = "energy" ENERGY_DELTA_PLUS_VAR_NAME = "energy_delta_plus" ENERGY_DELTA_MINUS_VAR_NAME = "energy_delta_minus" @@ -572,10 +719,16 @@ def to_data_product_variables(self) -> list[DataProductVariable]: DataProductVariable(self.species, self.species_data), DataProductVariable(ENERGY_LABEL_VAR_NAME, self.energy.astype(str)), DataProductVariable(SPIN_ANGLE_LABEL_VAR_NAME, self.spin_angle.astype(str)), - DataProductVariable(ELEVATION_ANGLE_LABEL_VAR_NAME, self.elevation.astype(str)), - DataProductVariable(f"{self.species}_stat_uncert", self.species_data_stat_uncert), + DataProductVariable( + ELEVATION_ANGLE_LABEL_VAR_NAME, self.elevation.astype(str) + ), + DataProductVariable( + f"{self.species}_stat_uncert", self.species_data_stat_uncert + ), DataProductVariable(RGFO_ESA_STEP_VAR_NAME, self.rgfo_esa_step), DataProductVariable(RGFO_SPIN_SECTOR_VAR_NAME, self.rgfo_spin_sector), DataProductVariable(RGFO_HALF_SPIN_VAR_NAME, self.rgfo_half_spin), - DataProductVariable(HALF_SPIN_PER_ESA_STEP_VAR_NAME, self.half_spin_per_esa_step), + DataProductVariable( + HALF_SPIN_PER_ESA_STEP_VAR_NAME, self.half_spin_per_esa_step + ), ] diff --git a/imap_l3_processing/codice/l3/lo/science/codice_lo_calculations.py b/imap_l3_processing/codice/l3/lo/science/codice_lo_calculations.py index 688d0d7cb..29224a7dc 100644 --- a/imap_l3_processing/codice/l3/lo/science/codice_lo_calculations.py +++ b/imap_l3_processing/codice/l3/lo/science/codice_lo_calculations.py @@ -4,62 +4,125 @@ import numpy as np -from imap_l3_processing.codice.l3.lo.constants import AZIMUTH_STEP_SIZE, ELEVATION_STEP_SIZE, ENERGY_STEP_SIZE, \ - ENERGY_LOST_IN_CARBON_FOIL, POST_ACCELERATION_VOLTAGE_IN_KV, CONVERSION_CONSTANT_K, CODICE_LO_NUM_AZIMUTH_BINS, \ - CONSTANT_C_FROM_INSTRUMENT_TEAM, CODICE_LO_NUM_ESA_STEPS, CODICE_LO_NUM_SPIN_SECTORS -from imap_l3_processing.codice.l3.lo.direct_events.science.angle_lookup import PositionToElevationLookup -from imap_l3_processing.codice.l3.lo.direct_events.science.efficiency_lookup import EfficiencyLookup -from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import EnergyLookup -from imap_l3_processing.codice.l3.lo.direct_events.science.mass_coefficient_lookup import MassCoefficientLookup -from imap_l3_processing.codice.l3.lo.direct_events.science.mass_species_bin_lookup import MassSpeciesBinLookup +from imap_l3_processing.codice.l3.lo.constants import ( + AZIMUTH_STEP_SIZE, + ELEVATION_STEP_SIZE, + ENERGY_STEP_SIZE, + ENERGY_LOST_IN_CARBON_FOIL, + POST_ACCELERATION_VOLTAGE_IN_KV, + CONVERSION_CONSTANT_K, + CODICE_LO_NUM_AZIMUTH_BINS, + CONSTANT_C_FROM_INSTRUMENT_TEAM, + CODICE_LO_NUM_ESA_STEPS, + CODICE_LO_NUM_SPIN_SECTORS, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.angle_lookup import ( + PositionToElevationLookup, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.efficiency_lookup import ( + EfficiencyLookup, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import ( + EnergyLookup, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.mass_coefficient_lookup import ( + MassCoefficientLookup, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.mass_species_bin_lookup import ( + MassSpeciesBinLookup, +) from imap_l3_processing.codice.l3.lo.models import CodiceLoDirectEventData -def calculate_partial_densities(intensities: np.ndarray, esa_steps: np.ndarray, mass_per_charge: float): +def calculate_partial_densities( + intensities: np.ndarray, esa_steps: np.ndarray, mass_per_charge: float +): return np.nansum( - CONSTANT_C_FROM_INSTRUMENT_TEAM * AZIMUTH_STEP_SIZE * ELEVATION_STEP_SIZE * ENERGY_STEP_SIZE * intensities * np.sqrt( - esa_steps[np.newaxis, :, np.newaxis]) * np.sqrt(mass_per_charge), axis=(1, 2)) - - -def calculate_total_number_of_events(priority_rate_variable: np.ndarray, acquisition_time: np.ndarray) -> np.ndarray[ - int]: + CONSTANT_C_FROM_INSTRUMENT_TEAM + * AZIMUTH_STEP_SIZE + * ELEVATION_STEP_SIZE + * ENERGY_STEP_SIZE + * intensities + * np.sqrt(esa_steps[np.newaxis, :, np.newaxis]) + * np.sqrt(mass_per_charge), + axis=(1, 2), + ) + + +def calculate_total_number_of_events( + priority_rate_variable: np.ndarray, acquisition_time: np.ndarray +) -> np.ndarray[int]: acquisition_time_in_seconds = acquisition_time / 1_000_000 - counts = np.multiply(priority_rate_variable, acquisition_time_in_seconds[np.newaxis, :, np.newaxis]) + counts = np.multiply( + priority_rate_variable, acquisition_time_in_seconds[np.newaxis, :, np.newaxis] + ) return np.sum(counts, axis=(1, 2), dtype=int) -def calculate_mass(apd_energy: np.ndarray, tof: np.ndarray, mass_coefficients: MassCoefficientLookup) -> np.ndarray: +def calculate_mass( + apd_energy: np.ndarray, tof: np.ndarray, mass_coefficients: MassCoefficientLookup +) -> np.ndarray: energy = np.log(apd_energy) tof = np.log(tof) - mass_calculation = mass_coefficients[0] + (mass_coefficients[1] * energy) + (mass_coefficients[2] * tof) + ( - mass_coefficients[3] * energy * tof) + (mass_coefficients[4] * np.power(energy, 2)) + ( - mass_coefficients[5] * np.power(tof, 3)) - return np.e ** mass_calculation - - -def calculate_mass_per_charge(energy_per_charge: np.ndarray, tof: np.ndarray) -> np.ndarray: - return (energy_per_charge + POST_ACCELERATION_VOLTAGE_IN_KV - ENERGY_LOST_IN_CARBON_FOIL) * ( - tof ** 2) * CONVERSION_CONSTANT_K - - -def rebin_direct_events_for_normalization(num_events: np.ndarray, spin_sector: np.ndarray, energy_step: np.ndarray, - num_spin_sectors: int, num_energies: int) -> np.ndarray: - base_counts = rebin_direct_events_by_energy_and_spin_sector(num_events, spin_sector, energy_step, num_spin_sectors, - num_energies) + mass_calculation = ( + mass_coefficients[0] + + (mass_coefficients[1] * energy) + + (mass_coefficients[2] * tof) + + (mass_coefficients[3] * energy * tof) + + (mass_coefficients[4] * np.power(energy, 2)) + + (mass_coefficients[5] * np.power(tof, 3)) + ) + return np.e**mass_calculation + + +def calculate_mass_per_charge( + energy_per_charge: np.ndarray, tof: np.ndarray +) -> np.ndarray: + return ( + ( + energy_per_charge + + POST_ACCELERATION_VOLTAGE_IN_KV + - ENERGY_LOST_IN_CARBON_FOIL + ) + * (tof**2) + * CONVERSION_CONSTANT_K + ) + + +def rebin_direct_events_for_normalization( + num_events: np.ndarray, + spin_sector: np.ndarray, + energy_step: np.ndarray, + num_spin_sectors: int, + num_energies: int, +) -> np.ndarray: + base_counts = rebin_direct_events_by_energy_and_spin_sector( + num_events, spin_sector, energy_step, num_spin_sectors, num_energies + ) half_spin = num_spin_sectors // 2 result = np.zeros_like(base_counts) - result[:, :, :, 0:half_spin] = base_counts[:, :, :, 0:half_spin] + base_counts[:, :, :, half_spin:num_spin_sectors] + result[:, :, :, 0:half_spin] = ( + base_counts[:, :, :, 0:half_spin] + + base_counts[:, :, :, half_spin:num_spin_sectors] + ) result[:, :, :, half_spin:num_spin_sectors] = result[:, :, :, 0:half_spin] return result -def rebin_direct_events_by_energy_and_spin_sector(num_events: np.ndarray, spin_sector: np.ndarray, - energy_step: np.ndarray, num_spin_sectors: int, num_energies: int) -> np.ndarray: +def rebin_direct_events_by_energy_and_spin_sector( + num_events: np.ndarray, + spin_sector: np.ndarray, + energy_step: np.ndarray, + num_spin_sectors: int, + num_energies: int, +) -> np.ndarray: num_epochs = num_events.shape[0] num_priorities = num_events.shape[1] - rebinned_output = np.zeros((num_epochs, num_priorities, num_energies, num_spin_sectors)) + rebinned_output = np.zeros( + (num_epochs, num_priorities, num_energies, num_spin_sectors) + ) for time_index in range(num_epochs): for priority_index in range(num_priorities): @@ -77,17 +140,26 @@ def rebin_direct_events_by_energy_and_spin_sector(num_events: np.ndarray, spin_s valid_spin_sectors = spin_sectors[valid_events_mask] valid_energy_indices = energy_indices[valid_events_mask] - np.add.at(rebinned_output[time_index, priority_index], (valid_energy_indices, valid_spin_sectors), 1) + np.add.at( + rebinned_output[time_index, priority_index], + (valid_energy_indices, valid_spin_sectors), + 1, + ) return rebinned_output -def calculate_normalization_factor(priority_counts: np.ndarray, num_events: np.ndarray, energy_steps: np.ndarray, - spin_sectors: np.ndarray) -> np.ndarray: +def calculate_normalization_factor( + priority_counts: np.ndarray, + num_events: np.ndarray, + energy_steps: np.ndarray, + spin_sectors: np.ndarray, +) -> np.ndarray: numerator = np.concatenate((priority_counts, priority_counts), axis=3) num_energies = priority_counts.shape[2] num_spin_sectors = 2 * priority_counts.shape[3] - denominator = rebin_direct_events_for_normalization(num_events, spin_sectors, energy_steps, num_spin_sectors, - num_energies) + denominator = rebin_direct_events_for_normalization( + num_events, spin_sectors, energy_steps, num_spin_sectors, num_energies + ) division_result = np.zeros(numerator.shape, dtype=float) np.divide(numerator, denominator, out=division_result, where=denominator != 0) @@ -99,8 +171,12 @@ def calculate_normalization_factor(priority_counts: np.ndarray, num_events: np.n return output -def lookup_normalization_per_event(normalization: np.ndarray, num_events: np.ndarray, energy_steps: np.ndarray, - spin_sectors: np.ndarray) -> np.ndarray: +def lookup_normalization_per_event( + normalization: np.ndarray, + num_events: np.ndarray, + energy_steps: np.ndarray, + spin_sectors: np.ndarray, +) -> np.ndarray: results = np.full(spin_sectors.shape, np.nan) for (epoch, priority), count in np.ma.ndenumerate(num_events, compressed=True): energy_step = energy_steps[epoch, priority, :count] @@ -111,13 +187,16 @@ def lookup_normalization_per_event(normalization: np.ndarray, num_events: np.nda good_events = ~(bad_spin_sectors | bad_energy_steps) results[epoch, priority, :count][good_events] = normalization[ - epoch, priority, energy_step[good_events], spin_sector[good_events]] + epoch, priority, energy_step[good_events], spin_sector[good_events] + ] return results -def rebin_to_counts_by_species_elevation_and_spin_sector(direct_event_data: CodiceLoDirectEventData, - mass_species_bin_lookup: MassSpeciesBinLookup) -> np.ndarray: +def rebin_to_counts_by_species_elevation_and_spin_sector( + direct_event_data: CodiceLoDirectEventData, + mass_species_bin_lookup: MassSpeciesBinLookup, +) -> np.ndarray: mass = direct_event_data.mass mass_per_charge = direct_event_data.mass_per_charge spin_sector = direct_event_data.spin_sector @@ -128,8 +207,17 @@ def rebin_to_counts_by_species_elevation_and_spin_sector(direct_event_data: Codi num_epochs = mass.shape[0] num_priorities = mass.shape[1] - output = np.full((mass_species_bin_lookup.get_num_species(), num_epochs, num_priorities, - CODICE_LO_NUM_ESA_STEPS, CODICE_LO_NUM_SPIN_SECTORS, CODICE_LO_NUM_AZIMUTH_BINS), 0.0) + output = np.full( + ( + mass_species_bin_lookup.get_num_species(), + num_epochs, + num_priorities, + CODICE_LO_NUM_ESA_STEPS, + CODICE_LO_NUM_SPIN_SECTORS, + CODICE_LO_NUM_AZIMUTH_BINS, + ), + 0.0, + ) for epoch_i in range(num_epochs): for priority_i in range(num_priorities): @@ -148,16 +236,22 @@ def rebin_to_counts_by_species_elevation_and_spin_sector(direct_event_data: Codi if apd_id_of_event < 1 or apd_id_of_event > CODICE_LO_NUM_AZIMUTH_BINS: continue - species = mass_species_bin_lookup.get_species(mass[indices_of_event], - mass_per_charge[indices_of_event]) + species = mass_species_bin_lookup.get_species( + mass[indices_of_event], mass_per_charge[indices_of_event] + ) if species is not None: energy_i = energy_step[indices_of_event] species_i = mass_species_bin_lookup.get_species_index(species) spin_sector_i = spin_sector[indices_of_event] apd_id_i = apd_id_of_event - 1 output[ - species_i, epoch_i, priority_i, energy_i, spin_sector_i, apd_id_i] += \ - direct_event_data.normalization_per_event[indices_of_event] + species_i, + epoch_i, + priority_i, + energy_i, + spin_sector_i, + apd_id_i, + ] += direct_event_data.normalization_per_event[indices_of_event] return output @@ -171,15 +265,18 @@ def rebin_to_counts_by_species_elevation_and_spin_sector(direct_event_data: Codi def combine_priorities_for_species_and_calculate_count_uncertainty( - counts: np.ndarray[(EPOCH, PRIORITY, POSITION, SPIN_ANGLE, ENERGY)]) -> tuple[np.ndarray, np.ndarray]: + counts: np.ndarray[(EPOCH, PRIORITY, POSITION, SPIN_ANGLE, ENERGY)], +) -> tuple[np.ndarray, np.ndarray]: combined_counts = np.sum(counts, axis=1) return combined_counts, np.sqrt(combined_counts) -def rebin_3d_distribution_azimuth_to_elevation(intensity_data: np.ndarray, - azimuths: np.ndarray, - position_to_elevation_lut: PositionToElevationLookup, - half_spin) -> np.ndarray: +def rebin_3d_distribution_azimuth_to_elevation( + intensity_data: np.ndarray, + azimuths: np.ndarray, + position_to_elevation_lut: PositionToElevationLookup, + half_spin, +) -> np.ndarray: num_epochs = intensity_data.shape[0] num_elevations = len(position_to_elevation_lut.bin_centers) num_spin_angles = intensity_data.shape[2] @@ -202,15 +299,27 @@ def rebin_3d_distribution_azimuth_to_elevation(intensity_data: np.ndarray, return rebinned -def convert_counts_to_intensity(counts: np.ndarray, - acquisition_times: np.ndarray[(EPOCH, ENERGY,)], - energy_per_charge: EnergyLookup, - efficiency_lookup: EfficiencyLookup, - geometric_factor: np.ndarray[(EPOCH, ENERGY, SPIN_ANGLE, POSITION)]) -> np.ndarray: +def convert_counts_to_intensity( + counts: np.ndarray, + acquisition_times: np.ndarray[ + ( + EPOCH, + ENERGY, + ) + ], + energy_per_charge: EnergyLookup, + efficiency_lookup: EfficiencyLookup, + geometric_factor: np.ndarray[(EPOCH, ENERGY, SPIN_ANGLE, POSITION)], +) -> np.ndarray: count_rates = counts / acquisition_times[:, :, np.newaxis, np.newaxis] - reshaped_efficiency_data = efficiency_lookup.efficiency_data[np.newaxis, :, np.newaxis, :] - denominator = geometric_factor * energy_per_charge.bin_centers[np.newaxis, :, np.newaxis, - np.newaxis] * reshaped_efficiency_data + reshaped_efficiency_data = efficiency_lookup.efficiency_data[ + np.newaxis, :, np.newaxis, : + ] + denominator = ( + geometric_factor + * energy_per_charge.bin_centers[np.newaxis, :, np.newaxis, np.newaxis] + * reshaped_efficiency_data + ) intensities = count_rates / denominator return intensities diff --git a/imap_l3_processing/codice/quality_flags.py b/imap_l3_processing/codice/quality_flags.py index f4133dcf5..5bb577a9c 100644 --- a/imap_l3_processing/codice/quality_flags.py +++ b/imap_l3_processing/codice/quality_flags.py @@ -3,4 +3,4 @@ class CodiceL3Flags(FlagNameMixin): NONE = CommonFlags.NONE - PRELIMINARY_MAG = 2 ** 2 + PRELIMINARY_MAG = 2**2 diff --git a/imap_l3_processing/constants.py b/imap_l3_processing/constants.py index 8a7d25a37..f8bf6eed7 100644 --- a/imap_l3_processing/constants.py +++ b/imap_l3_processing/constants.py @@ -14,14 +14,22 @@ HE_PUI_PARTICLE_MASS_KG = 6.64556828337e-27 PROTON_CHARGE_OVER_MASS_C_PER_KG = PROTON_CHARGE_COULOMBS / PROTON_MASS_KG -ALPHA_CHARGE_OVER_MASS_C_PER_KG = ALPHA_PARTICLE_CHARGE_COULOMBS / ALPHA_PARTICLE_MASS_KG +ALPHA_CHARGE_OVER_MASS_C_PER_KG = ( + ALPHA_PARTICLE_CHARGE_COULOMBS / ALPHA_PARTICLE_MASS_KG +) PROTON_MASS_PER_CHARGE_M_P_PER_E = 1.0 -ALPHA_MASS_PER_CHARGE_M_P_PER_E = (ALPHA_PARTICLE_MASS_KG / PROTON_MASS_KG) / (ALPHA_PARTICLE_CHARGE_COULOMBS / PROTON_CHARGE_COULOMBS) -HE_PUI_PARTICLE_MASS_PER_CHARGE_M_P_PER_E = (HE_PUI_PARTICLE_MASS_KG / PROTON_MASS_KG) / (PUI_PARTICLE_CHARGE_COULOMBS / PROTON_CHARGE_COULOMBS) +ALPHA_MASS_PER_CHARGE_M_P_PER_E = (ALPHA_PARTICLE_MASS_KG / PROTON_MASS_KG) / ( + ALPHA_PARTICLE_CHARGE_COULOMBS / PROTON_CHARGE_COULOMBS +) +HE_PUI_PARTICLE_MASS_PER_CHARGE_M_P_PER_E = ( + HE_PUI_PARTICLE_MASS_KG / PROTON_MASS_KG +) / (PUI_PARTICLE_CHARGE_COULOMBS / PROTON_CHARGE_COULOMBS) BOLTZMANN_CONSTANT_JOULES_PER_KELVIN = 1.380_649e-23 -EV_TO_KELVIN = PROTON_CHARGE_COULOMBS / BOLTZMANN_CONSTANT_JOULES_PER_KELVIN # ≈ 11604.52 K/eV +EV_TO_KELVIN = ( + PROTON_CHARGE_COULOMBS / BOLTZMANN_CONSTANT_JOULES_PER_KELVIN +) # ≈ 11604.52 K/eV ONE_SECOND_IN_MICROSECONDS = 1_000_000 ONE_SECOND_IN_NANOSECONDS = 1_000_000_000 @@ -32,7 +40,9 @@ CARRINGTON_ROTATION_IN_DAYS = 27.2753 CARRINGTON_ROTATION_IN_SECONDS = CARRINGTON_ROTATION_IN_DAYS * SECONDS_PER_DAY -CARRINGTON_ROTATION_IN_NANOSECONDS = CARRINGTON_ROTATION_IN_SECONDS * ONE_SECOND_IN_NANOSECONDS +CARRINGTON_ROTATION_IN_NANOSECONDS = ( + CARRINGTON_ROTATION_IN_SECONDS * ONE_SECOND_IN_NANOSECONDS +) METERS_PER_KILOMETER = 1_000 CENTIMETERS_PER_METER = 100 @@ -42,8 +52,8 @@ TT2000_EPOCH = datetime(2000, 1, 1, 11, 58, 55, 816) -FLOAT4_FILL_VALUE = -1.0E31 -FLOAT8_FILL_VALUE = -1.0E31 +FLOAT4_FILL_VALUE = -1.0e31 +FLOAT8_FILL_VALUE = -1.0e31 INT1_FILL_VALUE = -128 INT2_FILL_VALUE = -32768 INT4_FILL_VALUE = -2147483648 @@ -52,4 +62,4 @@ UNSIGNED_INT4_FILL_VALUE = 4294967295 INT8_FILL_VALUE = -9223372036854775808 -TTJ2000_EPOCH = datetime.fromisoformat("2000-01-01T11:58:55.816") \ No newline at end of file +TTJ2000_EPOCH = datetime.fromisoformat("2000-01-01T11:58:55.816") diff --git a/imap_l3_processing/data_utils.py b/imap_l3_processing/data_utils.py index a49d5ea9d..ab096f38f 100644 --- a/imap_l3_processing/data_utils.py +++ b/imap_l3_processing/data_utils.py @@ -3,8 +3,12 @@ import numpy as np -def rebin(from_epoch: np.ndarray[float], from_data: np.ndarray[float], to_epoch: np.ndarray[float], - to_epoch_delta: np.ndarray[float]) -> np.ndarray[float]: +def rebin( + from_epoch: np.ndarray[float], + from_data: np.ndarray[float], + to_epoch: np.ndarray[float], + to_epoch_delta: np.ndarray[float], +) -> np.ndarray[float]: output_shape = to_epoch.shape + from_data.shape[1:] vector_sums = np.zeros(shape=output_shape, dtype=float) vector_counts = np.zeros(shape=output_shape, dtype=float) @@ -23,17 +27,22 @@ def rebin(from_epoch: np.ndarray[float], from_data: np.ndarray[float], to_epoch: vector_counts[i] += 1 current_epoch, current_vec = next(input_data_iter, (None, None)) - return np.divide(vector_sums, vector_counts, out=np.full_like(vector_sums, fill_value=np.nan), - where=vector_counts != 0) + return np.divide( + vector_sums, + vector_counts, + out=np.full_like(vector_sums, fill_value=np.nan), + where=vector_counts != 0, + ) class NearestInterpolator: - def __init__(self, - from_epoch: np.ndarray, - from_data: np.ndarray, - to_epoch: np.ndarray, - maximum_distance: timedelta | np.timedelta64 - ): + def __init__( + self, + from_epoch: np.ndarray, + from_data: np.ndarray, + to_epoch: np.ndarray, + maximum_distance: timedelta | np.timedelta64, + ): if len(from_data.shape) == 1: from_data = from_data.reshape(-1, 1) from_data_nan_mask = np.any(np.isnan(from_data), axis=1) @@ -49,13 +58,17 @@ def __init__(self, right_delta = np.abs(from_epoch_as_dt64[right_indices] - to_epoch_as_dt64) left_delta = np.abs(from_epoch_as_dt64[left_indices] - to_epoch_as_dt64) - self.best_indices = np.where(right_delta < left_delta, right_indices, left_indices) + self.best_indices = np.where( + right_delta < left_delta, right_indices, left_indices + ) min_deltas = np.abs(from_epoch_as_dt64[self.best_indices] - to_epoch_as_dt64) self.min_outside_range = min_deltas > maximum_distance def interpolate_data(self) -> np.ndarray: - closest_data = self.only_valid_from_data[self.best_indices].astype(float, copy=True) + closest_data = self.only_valid_from_data[self.best_indices].astype( + float, copy=True + ) closest_data[self.min_outside_range] = np.nan return closest_data diff --git a/imap_l3_processing/glows/descriptors.py b/imap_l3_processing/glows/descriptors.py index 47a605345..75d1f4661 100644 --- a/imap_l3_processing/glows/descriptors.py +++ b/imap_l3_processing/glows/descriptors.py @@ -7,13 +7,13 @@ GLOWS_L3A_DESCRIPTOR = "hist" GLOWS_L3B_DESCRIPTOR = "ion-rate-profile" GLOWS_L3C_DESCRIPTOR = "sw-profile" -GLOWS_L3D_DESCRIPTOR = 'solar-hist' +GLOWS_L3D_DESCRIPTOR = "solar-hist" WAW_HELIOION_DESCRIPTOR = "WawHelioIonMP" BAD_DAYS_LIST_DESCRIPTOR = "bad-days-list" UV_ANISOTROPY_1CR_DESCRIPTOR = "uv-anisotropy-1CR" -PLASMA_SPEED_DESCRIPTOR = 'plasma-speed-2026d' +PLASMA_SPEED_DESCRIPTOR = "plasma-speed-2026d" PROTON_DENSITY_DESCRIPTOR = "proton-density-2026d" UV_ANISOTROPY_DESCRIPTOR = "uv-anisotropy-2026d" PHOTOION_DESCRIPTOR = "photoion-2026d" @@ -21,9 +21,9 @@ ELECTRON_DENSITY_DESCRIPTOR = "electron-density-2026d" PIPELINE_SETTINGS_L3BCDE_DESCRIPTOR = "pipeline-settings-l3bcde" -GLOWS_L3E_HI_45_DESCRIPTOR = 'survival-probability-hi-45' -GLOWS_L3E_HI_90_DESCRIPTOR = 'survival-probability-hi-90' -GLOWS_L3E_LO_DESCRIPTOR = 'survival-probability-lo' +GLOWS_L3E_HI_45_DESCRIPTOR = "survival-probability-hi-45" +GLOWS_L3E_HI_90_DESCRIPTOR = "survival-probability-hi-90" +GLOWS_L3E_LO_DESCRIPTOR = "survival-probability-lo" GLOWS_L3E_ULTRA_SF_DESCRIPTOR = "survival-probability-ul-sf" GLOWS_L3E_ULTRA_HF_DESCRIPTOR = "survival-probability-ul-hf" @@ -37,7 +37,7 @@ GLOWS_L3E_HI_90_DESCRIPTOR, GLOWS_L3E_LO_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, - GLOWS_L3E_ULTRA_HF_DESCRIPTOR + GLOWS_L3E_ULTRA_HF_DESCRIPTOR, ] GLOWS_L3E_DESCRIPTORS = [ @@ -46,4 +46,4 @@ GLOWS_L3E_LO_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, GLOWS_L3E_ULTRA_HF_DESCRIPTOR, -] \ No newline at end of file +] diff --git a/imap_l3_processing/glows/glows_processor.py b/imap_l3_processing/glows/glows_processor.py index 01611bb1b..e9600bc16 100644 --- a/imap_l3_processing/glows/glows_processor.py +++ b/imap_l3_processing/glows/glows_processor.py @@ -13,36 +13,67 @@ from zipfile import ZipFile, ZIP_DEFLATED import numpy as np -from imap_data_access.file_validation import generate_imap_file_path, ScienceFilePath, AncillaryFilePath, Version +from imap_data_access.file_validation import ( + generate_imap_file_path, + ScienceFilePath, + AncillaryFilePath, + Version, +) from imap_data_access.processing_input import ProcessingInputCollection from imap_l3_processing.constants import TEMP_CDF_FOLDER_PATH -from imap_l3_processing.glows.descriptors import GLOWS_L3A_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, \ - GLOWS_L3E_ULTRA_HF_DESCRIPTOR, GLOWS_L3D_DESCRIPTOR, GLOWS_L3B_DESCRIPTOR, GLOWS_L3C_DESCRIPTOR, \ - GLOWS_L3E_LO_DESCRIPTOR +from imap_l3_processing.glows.descriptors import ( + GLOWS_L3A_DESCRIPTOR, + GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + GLOWS_L3E_ULTRA_HF_DESCRIPTOR, + GLOWS_L3D_DESCRIPTOR, + GLOWS_L3B_DESCRIPTOR, + GLOWS_L3C_DESCRIPTOR, + GLOWS_L3E_LO_DESCRIPTOR, +) from imap_l3_processing.glows.l3a.glows_l3a_dependencies import GlowsL3ADependencies from imap_l3_processing.glows.l3a.glows_toolkit.l3a_data import L3aData from imap_l3_processing.glows.l3a.models import GlowsL3LightCurve from imap_l3_processing.glows.l3a.utils import create_glows_l3a_from_dictionary from imap_l3_processing.glows.l3bc.glows_l3bc_dependencies import GlowsL3BCDependencies -from imap_l3_processing.glows.l3bc.glows_l3bc_initializer import GlowsL3BCInitializer, GlowsL3BCInitializerData -from imap_l3_processing.glows.l3bc.models import GlowsL3BIonizationRate, GlowsL3CSolarWind, GlowsL3BCProcessorOutput, \ - ExternalDependencies +from imap_l3_processing.glows.l3bc.glows_l3bc_initializer import ( + GlowsL3BCInitializer, + GlowsL3BCInitializerData, +) +from imap_l3_processing.glows.l3bc.models import ( + GlowsL3BIonizationRate, + GlowsL3CSolarWind, + GlowsL3BCProcessorOutput, + ExternalDependencies, +) from imap_l3_processing.glows.l3bc.science.filter_out_bad_days import filter_l3a_files from imap_l3_processing.glows.l3bc.science.generate_l3bc import generate_l3bc from imap_l3_processing.glows.l3bc.utils import get_pointing_date_range from imap_l3_processing.glows.l3d.glows_l3d_dependencies import GlowsL3DDependencies from imap_l3_processing.glows.l3d.glows_l3d_initializer import GlowsL3DInitializer from imap_l3_processing.glows.l3d.models import GlowsL3DProcessorOutput -from imap_l3_processing.glows.l3d.utils import create_glows_l3b_json_file_from_cdf, create_glows_l3c_json_file_from_cdf, \ - PATH_TO_L3D_TOOLKIT, convert_json_to_l3d_data_product, get_parent_file_names_from_l3d_json, rename_l3d_text_outputs +from imap_l3_processing.glows.l3d.utils import ( + create_glows_l3b_json_file_from_cdf, + create_glows_l3c_json_file_from_cdf, + PATH_TO_L3D_TOOLKIT, + convert_json_to_l3d_data_product, + get_parent_file_names_from_l3d_json, + rename_l3d_text_outputs, +) from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import GlowsL3eSpacecraftInfo from imap_l3_processing.glows.l3e.glows_l3e_hi_model import GlowsL3EHiData -from imap_l3_processing.glows.l3e.glows_l3e_initializer import GlowsL3EInitializer, GlowsL3EInitializerOutput +from imap_l3_processing.glows.l3e.glows_l3e_initializer import ( + GlowsL3EInitializer, + GlowsL3EInitializerOutput, +) from imap_l3_processing.glows.l3e.glows_l3e_lo_model import GlowsL3ELoData from imap_l3_processing.glows.l3e.glows_l3e_ultra_model import GlowsL3EUltraData -from imap_l3_processing.glows.l3e.glows_l3e_utils import determine_call_args_for_l3e_executable, get_lo_pivot_angles, \ - compute_glows_flags_for_repoint, determine_spacecraft_info_using_predict_if_needed +from imap_l3_processing.glows.l3e.glows_l3e_utils import ( + determine_call_args_for_l3e_executable, + get_lo_pivot_angles, + compute_glows_flags_for_repoint, + determine_spacecraft_info_using_predict_if_needed, +) from imap_l3_processing.glows.l3e.reprocess_info import fetch_reprocess_info from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.processor import Processor @@ -52,12 +83,16 @@ class GlowsProcessor(Processor): - def __init__(self, dependencies: ProcessingInputCollection, input_metadata: InputMetadata): + def __init__( + self, dependencies: ProcessingInputCollection, input_metadata: InputMetadata + ): super().__init__(dependencies, input_metadata) def process(self): if self.input_metadata.data_level == "l3a": - l3a_dependencies = GlowsL3ADependencies.fetch_dependencies(self.dependencies) + l3a_dependencies = GlowsL3ADependencies.fetch_dependencies( + self.dependencies + ) products = [] l3a_output = self.process_l3a(l3a_dependencies) @@ -67,14 +102,22 @@ def process(self): return products elif self.input_metadata.data_level == "l3b": products_list = [] - l3bc_major_version = self.input_metadata.version.lookup(GLOWS_L3B_DESCRIPTOR).major - l3bc_initializer_data: GlowsL3BCInitializerData = GlowsL3BCInitializer.get_crs_to_process(self.dependencies, l3bc_major_version) + l3bc_major_version = self.input_metadata.version.lookup( + GLOWS_L3B_DESCRIPTOR + ).major + l3bc_initializer_data: GlowsL3BCInitializerData = ( + GlowsL3BCInitializer.get_crs_to_process( + self.dependencies, l3bc_major_version + ) + ) if len(l3bc_initializer_data.l3bc_dependencies) > 0: logger.info("Found CRs to Process L3BC:") for dep in l3bc_initializer_data.l3bc_dependencies: l3a_file_names = [l3a_d["filename"] for l3a_d in dep.l3a_data] - logger.info(f"\t{dep.carrington_rotation_number}, {dep.version}: {l3a_file_names}") + logger.info( + f"\t{dep.carrington_rotation_number}, {dep.version}: {l3a_file_names}" + ) else: logger.info("No CRs to process for B/C") @@ -83,13 +126,32 @@ def process(self): reprocess_info = fetch_reprocess_info(self.dependencies) - l3d_major_version = self.input_metadata.version.lookup(GLOWS_L3D_DESCRIPTOR).major - l3bs = list({**l3bc_initializer_data.l3bs_by_cr, **glows_l3bc_output_data.l3bs_by_cr}.values()) - l3cs = list({**l3bc_initializer_data.l3cs_by_cr, **glows_l3bc_output_data.l3cs_by_cr}.values()) - l3d_initializer_result = GlowsL3DInitializer.should_process_l3d(l3bc_initializer_data.external_dependencies, - l3bs, l3cs, reprocess_info, l3d_major_version) + l3d_major_version = self.input_metadata.version.lookup( + GLOWS_L3D_DESCRIPTOR + ).major + l3bs = list( + { + **l3bc_initializer_data.l3bs_by_cr, + **glows_l3bc_output_data.l3bs_by_cr, + }.values() + ) + l3cs = list( + { + **l3bc_initializer_data.l3cs_by_cr, + **glows_l3bc_output_data.l3cs_by_cr, + }.values() + ) + l3d_initializer_result = GlowsL3DInitializer.should_process_l3d( + l3bc_initializer_data.external_dependencies, + l3bs, + l3cs, + reprocess_info, + l3d_major_version, + ) if l3d_initializer_result is None: - logger.info("No inputs to L3d have changed. Skipping processing of L3d and L3e!") + logger.info( + "No inputs to L3d have changed. Skipping processing of L3d and L3e!" + ) return products_list version_number, glows_l3d_dependency, old_l3d = l3d_initializer_result @@ -100,12 +162,21 @@ def process(self): logger.info("Not enough L3B/C data to produce L3d.") return products_list - logger.info(f"Finished processing L3d up to CR: {process_l3d_result.last_processed_cr}") - logger.info(f"Saved L3d CDF output to: {process_l3d_result.l3d_cdf_file_path}") + logger.info( + f"Finished processing L3d up to CR: {process_l3d_result.last_processed_cr}" + ) + logger.info( + f"Saved L3d CDF output to: {process_l3d_result.l3d_cdf_file_path}" + ) for txt_file in process_l3d_result.l3d_text_file_paths: logger.info(f"Saved L3d text file output to: {txt_file}") - products_list.extend([*process_l3d_result.l3d_text_file_paths, process_l3d_result.l3d_cdf_file_path]) + products_list.extend( + [ + *process_l3d_result.l3d_text_file_paths, + process_l3d_result.l3d_cdf_file_path, + ] + ) l3e_initializer_output = GlowsL3EInitializer.get_repointings_to_process( process_l3d_result, @@ -116,91 +187,149 @@ def process(self): ) if l3e_initializer_output is not None: - logger.info(f"Processing L3e for repointings: {l3e_initializer_output.repointings.repointing_numbers}") + logger.info( + f"Processing L3e for repointings: {l3e_initializer_output.repointings.repointing_numbers}" + ) l3e_products = process_l3e(l3e_initializer_output) products_list.extend(l3e_products) else: - logger.info(f"There are no changes between: {old_l3d} and the newly produced L3d. Not uploading L3d or processing L3e!") + logger.info( + f"There are no changes between: {old_l3d} and the newly produced L3d. Not uploading L3d or processing L3e!" + ) return products_list - def process_l3a(self, dependencies: GlowsL3ADependencies) -> Optional[GlowsL3LightCurve]: + def process_l3a( + self, dependencies: GlowsL3ADependencies + ) -> Optional[GlowsL3LightCurve]: data = dependencies.data l3_data = L3aData(dependencies.ancillary_files) l3_data.process_l2_data_file(data) l3_data.generate_l3a_data(dependencies.ancillary_files) - if len(l3_data.data['daily_lightcurve']['exposure_times']) != 0: - data_with_spin_angle = self.add_spin_angle_delta(l3_data.data, dependencies.ancillary_files) - return create_glows_l3a_from_dictionary(data_with_spin_angle, - replace(self.input_metadata, descriptor=GLOWS_L3A_DESCRIPTOR)) + if len(l3_data.data["daily_lightcurve"]["exposure_times"]) != 0: + data_with_spin_angle = self.add_spin_angle_delta( + l3_data.data, dependencies.ancillary_files + ) + return create_glows_l3a_from_dictionary( + data_with_spin_angle, + replace(self.input_metadata, descriptor=GLOWS_L3A_DESCRIPTOR), + ) return None @staticmethod def add_spin_angle_delta(data: dict, ancillary_files: dict) -> dict: - with open(ancillary_files['settings']) as f: + with open(ancillary_files["settings"]) as f: settings_file = json.load(f) - number_of_bins = settings_file['l3a_nominal_number_of_bins'] + number_of_bins = settings_file["l3a_nominal_number_of_bins"] delta = 360 / number_of_bins / 2 - data['daily_lightcurve']['spin_angle_delta'] = np.full_like(data['daily_lightcurve']['spin_angle'], delta) + data["daily_lightcurve"]["spin_angle_delta"] = np.full_like( + data["daily_lightcurve"]["spin_angle"], delta + ) return data @staticmethod - def archive_dependencies(l3bc_deps: GlowsL3BCDependencies, external_dependencies: ExternalDependencies) -> Path: + def archive_dependencies( + l3bc_deps: GlowsL3BCDependencies, external_dependencies: ExternalDependencies + ) -> Path: start_date = l3bc_deps.start_date.strftime("%Y%m%d") - minor_version_str = str(Version(None,l3bc_deps.version.minor)) - zip_path = TEMP_CDF_FOLDER_PATH / f"imap_glows_l3b-archive_{start_date}_{minor_version_str}.zip" + minor_version_str = str(Version(None, l3bc_deps.version.minor)) + zip_path = ( + TEMP_CDF_FOLDER_PATH + / f"imap_glows_l3b-archive_{start_date}_{minor_version_str}.zip" + ) json_filename = "cr_to_process.json" with ZipFile(zip_path, "w", ZIP_DEFLATED) as file: - file.write(external_dependencies.lyman_alpha_path, "lyman_alpha_composite.nc") + file.write( + external_dependencies.lyman_alpha_path, "lyman_alpha_composite.nc" + ) file.write(external_dependencies.omni2_data_path, "omni2_all_years.dat") file.write(external_dependencies.f107_index_file_path, "f107_fluxtable.txt") - cr = {"cr_rotation_number": l3bc_deps.carrington_rotation_number, - "l3a_paths": [l3a['filename'] for l3a in l3bc_deps.l3a_data], - "cr_start_date": str(l3bc_deps.start_date), - "cr_end_date": str(l3bc_deps.end_date), - "bad_days_list": l3bc_deps.ancillary_files['bad_days_list'].name, - "pipeline_settings": l3bc_deps.ancillary_files['pipeline_settings'].name, - "waw_helioion_mp": l3bc_deps.ancillary_files['WawHelioIonMP_parameters'].name, - "uv_anisotropy": l3bc_deps.ancillary_files['uv_anisotropy'].name, - "repointing_file": l3bc_deps.repointing_file_path.name, - } + cr = { + "cr_rotation_number": l3bc_deps.carrington_rotation_number, + "l3a_paths": [l3a["filename"] for l3a in l3bc_deps.l3a_data], + "cr_start_date": str(l3bc_deps.start_date), + "cr_end_date": str(l3bc_deps.end_date), + "bad_days_list": l3bc_deps.ancillary_files["bad_days_list"].name, + "pipeline_settings": l3bc_deps.ancillary_files[ + "pipeline_settings" + ].name, + "waw_helioion_mp": l3bc_deps.ancillary_files[ + "WawHelioIonMP_parameters" + ].name, + "uv_anisotropy": l3bc_deps.ancillary_files["uv_anisotropy"].name, + "repointing_file": l3bc_deps.repointing_file_path.name, + } json_string = json.dumps(cr) file.writestr(json_filename, json_string) return zip_path + def process_l3bc(processor, initializer_data: GlowsL3BCInitializerData): l3bs_by_cr = {} l3cs_by_cr = {} data_products = [] for dependency in initializer_data.l3bc_dependencies: logger.info(f"Processing L3BC for CR: {dependency.carrington_rotation_number}") - zip_path = GlowsProcessor.archive_dependencies(l3bc_deps=dependency, - external_dependencies=initializer_data.external_dependencies) + zip_path = GlowsProcessor.archive_dependencies( + l3bc_deps=dependency, + external_dependencies=initializer_data.external_dependencies, + ) logger.info(f"Archived L3BC Dependencies to {zip_path}") - filtered_days = filter_l3a_files(dependency.l3a_data, dependency.ancillary_files['bad_days_list'], - dependency.carrington_rotation_number) - with SwallowExceptionAndLog(f"Exception caught in L3BC science code, skipping CR {dependency.carrington_rotation_number}") as manager: - dependency_filtering_out_bad_days = replace(dependency, l3a_data=filtered_days) + filtered_days = filter_l3a_files( + dependency.l3a_data, + dependency.ancillary_files["bad_days_list"], + dependency.carrington_rotation_number, + ) + with SwallowExceptionAndLog( + f"Exception caught in L3BC science code, skipping CR {dependency.carrington_rotation_number}" + ) as manager: + dependency_filtering_out_bad_days = replace( + dependency, l3a_data=filtered_days + ) l3b_data, l3c_data = generate_l3bc(dependency_filtering_out_bad_days) if manager.exception_caught: continue - l3b_metadata = InputMetadata("glows", "l3b", dependency.start_date, dependency.end_date, - VersionMap({GLOWS_L3B_DESCRIPTOR: dependency.version}), GLOWS_L3B_DESCRIPTOR) - l3c_metadata = InputMetadata("glows", "l3c", dependency.start_date, dependency.end_date, - VersionMap({GLOWS_L3C_DESCRIPTOR: dependency.version}), GLOWS_L3C_DESCRIPTOR) + l3b_metadata = InputMetadata( + "glows", + "l3b", + dependency.start_date, + dependency.end_date, + VersionMap({GLOWS_L3B_DESCRIPTOR: dependency.version}), + GLOWS_L3B_DESCRIPTOR, + ) + l3c_metadata = InputMetadata( + "glows", + "l3c", + dependency.start_date, + dependency.end_date, + VersionMap({GLOWS_L3C_DESCRIPTOR: dependency.version}), + GLOWS_L3C_DESCRIPTOR, + ) - l3b_data_product = GlowsL3BIonizationRate.from_instrument_team_dictionary(l3b_data, l3b_metadata) - l3c_data_product = GlowsL3CSolarWind.from_instrument_team_dictionary(l3c_data, l3c_metadata) + l3b_data_product = GlowsL3BIonizationRate.from_instrument_team_dictionary( + l3b_data, l3b_metadata + ) + l3c_data_product = GlowsL3CSolarWind.from_instrument_team_dictionary( + l3c_data, l3c_metadata + ) - l3b_data_product.parent_file_names += processor.get_parent_file_names([*dependency.l3a_file_names, zip_path]) - l3b_cdf = save_data(l3b_data_product, cr_number=dependency.carrington_rotation_number) + l3b_data_product.parent_file_names += processor.get_parent_file_names( + [*dependency.l3a_file_names, zip_path] + ) + l3b_cdf = save_data( + l3b_data_product, cr_number=dependency.carrington_rotation_number + ) - l3c_data_product.parent_file_names += processor.get_parent_file_names([zip_path, Path(l3b_cdf)]) - l3c_cdf = save_data(l3c_data_product, cr_number=dependency.carrington_rotation_number) + l3c_data_product.parent_file_names += processor.get_parent_file_names( + [zip_path, Path(l3b_cdf)] + ) + l3c_cdf = save_data( + l3c_data_product, cr_number=dependency.carrington_rotation_number + ) logger.info(f"Finished processing CR: {dependency.carrington_rotation_number}") for path in [l3b_cdf, l3c_cdf]: @@ -214,22 +343,44 @@ def process_l3bc(processor, initializer_data: GlowsL3BCInitializerData): l3bs_by_cr=l3bs_by_cr, l3cs_by_cr=l3cs_by_cr, data_products=data_products ) + def process_l3d( dependencies: GlowsL3DDependencies, version: Version ) -> Optional[GlowsL3DProcessorOutput]: - collections.deque((create_glows_l3b_json_file_from_cdf(l3b) for l3b in dependencies.l3b_file_paths), maxlen=0) - collections.deque((create_glows_l3c_json_file_from_cdf(l3c) for l3c in dependencies.l3c_file_paths), maxlen=0) + collections.deque( + ( + create_glows_l3b_json_file_from_cdf(l3b) + for l3b in dependencies.l3b_file_paths + ), + maxlen=0, + ) + collections.deque( + ( + create_glows_l3c_json_file_from_cdf(l3c) + for l3c in dependencies.l3c_file_paths + ), + maxlen=0, + ) os.makedirs(PATH_TO_L3D_TOOLKIT / "data_l3d", exist_ok=True) os.makedirs(PATH_TO_L3D_TOOLKIT / "data_l3d_txt", exist_ok=True) start_date = datetime(1947, 3, 3) - data_product_metadata = InputMetadata(instrument="glows", data_level="l3d", descriptor=GLOWS_L3D_DESCRIPTOR, - start_date=start_date, end_date=start_date, - version=VersionMap({GLOWS_L3D_DESCRIPTOR: version})) + data_product_metadata = InputMetadata( + instrument="glows", + data_level="l3d", + descriptor=GLOWS_L3D_DESCRIPTOR, + start_date=start_date, + end_date=start_date, + version=VersionMap({GLOWS_L3D_DESCRIPTOR: version}), + ) - output_l3d_cdf_filename = data_product_metadata.to_science_file_path(cr_number=dependencies.end_cr).construct_path().name + output_l3d_cdf_filename = ( + data_product_metadata.to_science_file_path(cr_number=dependencies.end_cr) + .construct_path() + .name + ) file_manifest = { "external_files": { @@ -242,7 +393,7 @@ def process_l3d( for key, val in dependencies.ancillary_files["WawHelioIon"].items() }, }, - "l3d_cdf_filename": output_l3d_cdf_filename + "l3d_cdf_filename": output_l3d_cdf_filename, } last_processed_cr = None @@ -264,95 +415,205 @@ def process_l3d( logger.warning("generate_l3d.py error: %s /// %s", e.output, e.stderr) raise if output.stdout: - last_processed_cr = int(output.stdout.split('= ')[-1]) + last_processed_cr = int(output.stdout.split("= ")[-1]) if last_processed_cr == dependencies.end_cr: output_text_files = [] - for text_file in os.listdir(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt'): + for text_file in os.listdir(PATH_TO_L3D_TOOLKIT / "data_l3d_txt"): if str(last_processed_cr) in text_file: - output_text_files.append(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt' / text_file) + output_text_files.append( + PATH_TO_L3D_TOOLKIT / "data_l3d_txt" / text_file + ) - txt_files_with_correct_version = rename_l3d_text_outputs(output_text_files, str(Version(None, version.minor))) + txt_files_with_correct_version = rename_l3d_text_outputs( + output_text_files, str(Version(None, version.minor)) + ) for txt_file in txt_files_with_correct_version: - shutil.copy(txt_file, generate_imap_file_path(txt_file.name).construct_path()) + shutil.copy( + txt_file, generate_imap_file_path(txt_file.name).construct_path() + ) - file_name = f'imap_glows_l3d_solar-params-history_19470303-cr0{last_processed_cr}_v00.json' + file_name = f"imap_glows_l3d_solar-params-history_19470303-cr0{last_processed_cr}_v00.json" - parent_file_names = get_parent_file_names_from_l3d_json(PATH_TO_L3D_TOOLKIT / 'data_l3d') + parent_file_names = get_parent_file_names_from_l3d_json( + PATH_TO_L3D_TOOLKIT / "data_l3d" + ) - l3d_data_product = convert_json_to_l3d_data_product(PATH_TO_L3D_TOOLKIT / 'data_l3d' / file_name, - data_product_metadata, parent_file_names) - l3d_data_product_path = save_data(l3d_data_product, cr_number=dependencies.end_cr) + l3d_data_product = convert_json_to_l3d_data_product( + PATH_TO_L3D_TOOLKIT / "data_l3d" / file_name, + data_product_metadata, + parent_file_names, + ) + l3d_data_product_path = save_data( + l3d_data_product, cr_number=dependencies.end_cr + ) - return GlowsL3DProcessorOutput(l3d_data_product_path, txt_files_with_correct_version, last_processed_cr) + return GlowsL3DProcessorOutput( + l3d_data_product_path, txt_files_with_correct_version, last_processed_cr + ) return None + def process_l3e(initializer_data: GlowsL3EInitializerOutput): products_list = [] - lo_pivot_angles = get_lo_pivot_angles(initializer_data.repointings.repointing_numbers) + lo_pivot_angles = get_lo_pivot_angles( + initializer_data.repointings.repointing_numbers + ) for repointing in initializer_data.repointings.repointing_numbers: - with SwallowExceptionAndLog(f"Exception encountered when processing L3e for repointing {repointing}"): + with SwallowExceptionAndLog( + f"Exception encountered when processing L3e for repointing {repointing}" + ): start_repointing, end_repointing = get_pointing_date_range(repointing) epoch_delta: timedelta = (end_repointing - start_repointing) / 2 repointing_midpoint = start_repointing + epoch_delta - glows_flags = compute_glows_flags_for_repoint(initializer_data.l3d_cdf_path, repointing_midpoint) - spacecraft_info, predict_flag, kernel_names = determine_spacecraft_info_using_predict_if_needed(repointing_midpoint, initializer_data.metakernel_with_predict_ephem, initializer_data.metakernel_without_predict_ephem) + glows_flags = compute_glows_flags_for_repoint( + initializer_data.l3d_cdf_path, repointing_midpoint + ) + spacecraft_info, predict_flag, kernel_names = ( + determine_spacecraft_info_using_predict_if_needed( + repointing_midpoint, + initializer_data.metakernel_with_predict_ephem, + initializer_data.metakernel_without_predict_ephem, + ) + ) glows_flags |= predict_flag - with SwallowExceptionAndLog(f"Exception encountered when processing L3e lo for repointing {repointing}"): + with SwallowExceptionAndLog( + f"Exception encountered when processing L3e lo for repointing {repointing}" + ): if repointing in initializer_data.repointings.lo_repointings: - lo_parent_file_names = initializer_data.dependencies.get_lo_parents() + kernel_names + lo_parent_file_names = ( + initializer_data.dependencies.get_lo_parents() + kernel_names + ) pivot_info = lo_pivot_angles[repointing] if pivot_info.parent_filename is not None: - lo_parent_file_names = lo_parent_file_names + [pivot_info.parent_filename] + lo_parent_file_names = lo_parent_file_names + [ + pivot_info.parent_filename + ] lo_version = initializer_data.repointings.lo_repointings[repointing] - products_list.extend(process_l3e_lo(lo_parent_file_names, repointing, start_repointing, epoch_delta, pivot_info.pivot_angle, lo_version, glows_flags, spacecraft_info)) + products_list.extend( + process_l3e_lo( + lo_parent_file_names, + repointing, + start_repointing, + epoch_delta, + pivot_info.pivot_angle, + lo_version, + glows_flags, + spacecraft_info, + ) + ) - with SwallowExceptionAndLog(f"Exception encountered when processing L3e hi-90 for repointing {repointing}"): + with SwallowExceptionAndLog( + f"Exception encountered when processing L3e hi-90 for repointing {repointing}" + ): if repointing in initializer_data.repointings.hi_90_repointings: - hi_parent_file_names = initializer_data.dependencies.get_hi_parents() + kernel_names - hi_90_version = initializer_data.repointings.hi_90_repointings[repointing] - products_list.extend(process_l3e_hi(hi_parent_file_names, repointing, start_repointing, epoch_delta, 90, hi_90_version, glows_flags, spacecraft_info)) + hi_parent_file_names = ( + initializer_data.dependencies.get_hi_parents() + kernel_names + ) + hi_90_version = initializer_data.repointings.hi_90_repointings[ + repointing + ] + products_list.extend( + process_l3e_hi( + hi_parent_file_names, + repointing, + start_repointing, + epoch_delta, + 90, + hi_90_version, + glows_flags, + spacecraft_info, + ) + ) - with SwallowExceptionAndLog(f"Exception encountered when processing L3e hi-45 for repointing {repointing}"): + with SwallowExceptionAndLog( + f"Exception encountered when processing L3e hi-45 for repointing {repointing}" + ): if repointing in initializer_data.repointings.hi_45_repointings: - hi_parent_file_names = initializer_data.dependencies.get_hi_parents() + kernel_names - hi_45_version = initializer_data.repointings.hi_45_repointings[repointing] - products_list.extend(process_l3e_hi(hi_parent_file_names, repointing, start_repointing, epoch_delta, 135, hi_45_version, glows_flags, spacecraft_info)) - - ul_parent_file_names = initializer_data.dependencies.get_ul_parents() + kernel_names + hi_parent_file_names = ( + initializer_data.dependencies.get_hi_parents() + kernel_names + ) + hi_45_version = initializer_data.repointings.hi_45_repointings[ + repointing + ] + products_list.extend( + process_l3e_hi( + hi_parent_file_names, + repointing, + start_repointing, + epoch_delta, + 135, + hi_45_version, + glows_flags, + spacecraft_info, + ) + ) + + ul_parent_file_names = ( + initializer_data.dependencies.get_ul_parents() + kernel_names + ) with SwallowExceptionAndLog( - f"Exception encountered when processing L3e ultra SF for repointing {repointing}"): + f"Exception encountered when processing L3e ultra SF for repointing {repointing}" + ): if repointing in initializer_data.repointings.ultra_sf_repointings: - ul_sf_version = initializer_data.repointings.ultra_sf_repointings[repointing] + ul_sf_version = initializer_data.repointings.ultra_sf_repointings[ + repointing + ] products_list.extend( - process_l3e_ul_sf(ul_parent_file_names, repointing, start_repointing, epoch_delta, ul_sf_version, glows_flags, spacecraft_info)) + process_l3e_ul_sf( + ul_parent_file_names, + repointing, + start_repointing, + epoch_delta, + ul_sf_version, + glows_flags, + spacecraft_info, + ) + ) with SwallowExceptionAndLog( - f"Exception encountered when processing L3e ultra HF for repointing {repointing}"): + f"Exception encountered when processing L3e ultra HF for repointing {repointing}" + ): if repointing in initializer_data.repointings.ultra_hf_repointings: - ul_hf_version = initializer_data.repointings.ultra_hf_repointings[repointing] + ul_hf_version = initializer_data.repointings.ultra_hf_repointings[ + repointing + ] products_list.extend( - process_l3e_ul_hf(ul_parent_file_names, repointing, start_repointing, epoch_delta, ul_hf_version, glows_flags, spacecraft_info)) + process_l3e_ul_hf( + ul_parent_file_names, + repointing, + start_repointing, + epoch_delta, + ul_hf_version, + glows_flags, + spacecraft_info, + ) + ) return products_list + def process_l3e_lo( - parent_file_names: list[str], - repointing: int, - repointing_start: datetime, - epoch_delta: timedelta, - elongation_value: float, - version: Version, - glows_flags: int, - spacecraft_info: GlowsL3eSpacecraftInfo + parent_file_names: list[str], + repointing: int, + repointing_start: datetime, + epoch_delta: timedelta, + elongation_value: float, + version: Version, + glows_flags: int, + spacecraft_info: GlowsL3eSpacecraftInfo, ) -> list[Path]: repointing_midpoint = repointing_start + epoch_delta - l3e_args = determine_call_args_for_l3e_executable(repointing_start, repointing_midpoint, float(elongation_value), - spacecraft_info=spacecraft_info) + l3e_args = determine_call_args_for_l3e_executable( + repointing_start, + repointing_midpoint, + float(elongation_value), + spacecraft_info=spacecraft_info, + ) call_args = l3e_args.to_argument_list() logger.info(f"Processing L3e Lo, calling survProbLo with {call_args}") @@ -371,11 +632,19 @@ def process_l3e_lo( elongation_value_as_int = int(elongation_value) elongation_in_filename = f"{elongation_value_as_int}." - elongation_in_filename += "0" * (5-len(elongation_in_filename)) + elongation_in_filename += "0" * (5 - len(elongation_in_filename)) - output_path = Path(f'probSur.Imap.Lo_{l3e_args.formatted_date}_{l3e_args.decimal_date[:8]}_{elongation_in_filename}.dat') - lo_data = GlowsL3ELoData.convert_dat_to_glows_l3e_lo_product(input_metadata, output_path, - repointing_midpoint, epoch_delta, elongation_value_as_int, l3e_args) + output_path = Path( + f"probSur.Imap.Lo_{l3e_args.formatted_date}_{l3e_args.decimal_date[:8]}_{elongation_in_filename}.dat" + ) + lo_data = GlowsL3ELoData.convert_dat_to_glows_l3e_lo_product( + input_metadata, + output_path, + repointing_midpoint, + epoch_delta, + elongation_value_as_int, + l3e_args, + ) lo_data.parent_file_names = parent_file_names lo_data.glows_flags = np.array([glows_flags], dtype=np.uint16) @@ -389,7 +658,7 @@ def process_l3e_lo( start_time=cdf_science_file_path.start_date, repointing=repointing, version=str(Version(None, version.minor)), - extension="dat" + extension="dat", ).construct_path() shutil.move(output_path, new_dat_path) @@ -397,11 +666,19 @@ def process_l3e_lo( return [lo_cdf, new_dat_path] -def process_l3e_ul_sf(parent_file_names: list[str], repointing: int, repointing_start: datetime, epoch_delta: timedelta, - version: Version, glows_flags: int, spacecraft_info: GlowsL3eSpacecraftInfo) -> list[Path]: +def process_l3e_ul_sf( + parent_file_names: list[str], + repointing: int, + repointing_start: datetime, + epoch_delta: timedelta, + version: Version, + glows_flags: int, + spacecraft_info: GlowsL3eSpacecraftInfo, +) -> list[Path]: repointing_midpoint = repointing_start + epoch_delta - call_args_object = determine_call_args_for_l3e_executable(repointing_start, repointing_midpoint, 30, - spacecraft_info=spacecraft_info) + call_args_object = determine_call_args_for_l3e_executable( + repointing_start, repointing_midpoint, 30, spacecraft_info=spacecraft_info + ) call_args = call_args_object.to_argument_list() logger.info(f"Processing L3e Ultra, calling survProbUltra with {call_args}") @@ -418,9 +695,10 @@ def process_l3e_ul_sf(parent_file_names: list[str], repointing: int, repointing_ repointing=repointing, ) - output_path = Path(f'probSur.Imap.Ul_{call_args[0]}_{call_args[1][:8]}.dat') - ul_data = GlowsL3EUltraData.convert_dat_to_glows_l3e_ul_product(input_metadata, output_path, - repointing_midpoint, epoch_delta, call_args_object) + output_path = Path(f"probSur.Imap.Ul_{call_args[0]}_{call_args[1][:8]}.dat") + ul_data = GlowsL3EUltraData.convert_dat_to_glows_l3e_ul_product( + input_metadata, output_path, repointing_midpoint, epoch_delta, call_args_object + ) ul_data.parent_file_names = parent_file_names ul_data.glows_flags = np.array([glows_flags], dtype=np.uint16) @@ -435,7 +713,7 @@ def process_l3e_ul_sf(parent_file_names: list[str], repointing: int, repointing_ start_time=cdf_science_file_path.start_date, repointing=repointing, version=str(Version(None, version.minor)), - extension="dat" + extension="dat", ).construct_path() shutil.move(output_path, new_dat_path) @@ -443,17 +721,26 @@ def process_l3e_ul_sf(parent_file_names: list[str], repointing: int, repointing_ return [ul_cdf, new_dat_path] -def process_l3e_ul_hf(parent_file_names: list[str], repointing: int, repointing_start: datetime, epoch_delta: timedelta, - version: Version, glows_flags: int, spacecraft_info: GlowsL3eSpacecraftInfo) -> list[Path]: +def process_l3e_ul_hf( + parent_file_names: list[str], + repointing: int, + repointing_start: datetime, + epoch_delta: timedelta, + version: Version, + glows_flags: int, + spacecraft_info: GlowsL3eSpacecraftInfo, +) -> list[Path]: repointing_midpoint = repointing_start + epoch_delta - call_args_object = determine_call_args_for_l3e_executable(repointing_start, repointing_midpoint, 30, - spacecraft_info=spacecraft_info) + call_args_object = determine_call_args_for_l3e_executable( + repointing_start, repointing_midpoint, 30, spacecraft_info=spacecraft_info + ) - call_args_object.spacecraft_info = replace(call_args_object.spacecraft_info, - spacecraft_velocity_x=0, - spacecraft_velocity_y=0, - spacecraft_velocity_z=0, - ) + call_args_object.spacecraft_info = replace( + call_args_object.spacecraft_info, + spacecraft_velocity_x=0, + spacecraft_velocity_y=0, + spacecraft_velocity_z=0, + ) call_args = call_args_object.to_argument_list() logger.info(f"Processing L3e Ultra, calling survProbUltra with {call_args}") @@ -469,9 +756,10 @@ def process_l3e_ul_hf(parent_file_names: list[str], repointing: int, repointing_ repointing=repointing, ) - output_path = Path(f'probSur.Imap.Ul.V0_{call_args[0]}_{call_args[1][:8]}.dat') - ul_data = GlowsL3EUltraData.convert_dat_to_glows_l3e_ul_product(input_metadata, output_path, - repointing_midpoint, epoch_delta, call_args_object) + output_path = Path(f"probSur.Imap.Ul.V0_{call_args[0]}_{call_args[1][:8]}.dat") + ul_data = GlowsL3EUltraData.convert_dat_to_glows_l3e_ul_product( + input_metadata, output_path, repointing_midpoint, epoch_delta, call_args_object + ) ul_data.parent_file_names = parent_file_names ul_data.glows_flags = np.array([glows_flags], dtype=np.uint16) @@ -486,36 +774,54 @@ def process_l3e_ul_hf(parent_file_names: list[str], repointing: int, repointing_ start_time=cdf_science_file_path.start_date, repointing=repointing, version=str(Version(None, version.minor)), - extension="dat" + extension="dat", ).construct_path() shutil.move(output_path, new_dat_path) return [ul_cdf, new_dat_path] -def process_l3e_hi(parent_file_names: list[str], repointing: int, repointing_start: datetime, epoch_delta: timedelta, elongation: int, version: Version, glows_flags: int, spacecraft_info: GlowsL3eSpacecraftInfo) -> list[Path]: + +def process_l3e_hi( + parent_file_names: list[str], + repointing: int, + repointing_start: datetime, + epoch_delta: timedelta, + elongation: int, + version: Version, + glows_flags: int, + spacecraft_info: GlowsL3eSpacecraftInfo, +) -> list[Path]: repointing_midpoint = repointing_start + epoch_delta - l3e_hi_args = determine_call_args_for_l3e_executable(repointing_start, repointing_midpoint, elongation, - spacecraft_info=spacecraft_info) + l3e_hi_args = determine_call_args_for_l3e_executable( + repointing_start, + repointing_midpoint, + elongation, + spacecraft_info=spacecraft_info, + ) call_args = l3e_hi_args.to_argument_list() logger.info(f"Processing L3e Hi, calling survProbHi with {call_args}") run(["./survProbHi"] + call_args) - descriptor = f'survival-probability-hi-{180-elongation}' - input_metadata = InputMetadata(instrument='glows', descriptor=descriptor, - version=VersionMap({descriptor: version}), start_date=repointing_start, end_date=repointing_start + epoch_delta * 2, - repointing=repointing, data_level='l3e') + descriptor = f"survival-probability-hi-{180 - elongation}" + input_metadata = InputMetadata( + instrument="glows", + descriptor=descriptor, + version=VersionMap({descriptor: version}), + start_date=repointing_start, + end_date=repointing_start + epoch_delta * 2, + repointing=repointing, + data_level="l3e", + ) - output_path = Path(f'probSur.Imap.Hi_{call_args[0]}_{call_args[1][:8]}_{call_args[-1][:5]}.dat') + output_path = Path( + f"probSur.Imap.Hi_{call_args[0]}_{call_args[1][:8]}_{call_args[-1][:5]}.dat" + ) hi_data = GlowsL3EHiData.convert_dat_to_glows_l3e_hi_product( - input_metadata, - output_path, - repointing_midpoint, - epoch_delta, - l3e_hi_args + input_metadata, output_path, repointing_midpoint, epoch_delta, l3e_hi_args ) hi_data.parent_file_names = parent_file_names hi_data.glows_flags = np.array([glows_flags], dtype=np.uint16) @@ -529,7 +835,7 @@ def process_l3e_hi(parent_file_names: list[str], repointing: int, repointing_sta start_time=cdf_science_file_path.start_date, repointing=repointing, version=str(Version(None, version.minor)), - extension="dat" + extension="dat", ).construct_path() shutil.move(output_path, new_dat_path) @@ -551,4 +857,4 @@ def __exit__(self, exc_type, exc_val, exc_tb) -> bool: elif exc_type is not None: logger.exception(self.message, exc_info=exc_val) self.exception_caught = True - return True \ No newline at end of file + return True diff --git a/imap_l3_processing/glows/l3a/glows_l3a_dependencies.py b/imap_l3_processing/glows/l3a/glows_l3a_dependencies.py index 660d27488..dfa57d3ef 100644 --- a/imap_l3_processing/glows/l3a/glows_l3a_dependencies.py +++ b/imap_l3_processing/glows/l3a/glows_l3a_dependencies.py @@ -5,9 +5,13 @@ from imap_data_access.processing_input import ProcessingInputCollection, ScienceInput from spacepy.pycdf import CDF -from imap_l3_processing.glows.descriptors import GLOWS_L2_DESCRIPTOR, GLOWS_TIME_DEPENDENT_BACKGROUND_DESCRIPTOR, \ - GLOWS_CALIBRATION_DATA_DESCRIPTOR, \ - GLOWS_EXTRA_HELIOSPHERIC_BACKGROUND_DESCRIPTOR, GLOWS_PIPELINE_SETTINGS_DESCRIPTOR +from imap_l3_processing.glows.descriptors import ( + GLOWS_L2_DESCRIPTOR, + GLOWS_TIME_DEPENDENT_BACKGROUND_DESCRIPTOR, + GLOWS_CALIBRATION_DATA_DESCRIPTOR, + GLOWS_EXTRA_HELIOSPHERIC_BACKGROUND_DESCRIPTOR, + GLOWS_PIPELINE_SETTINGS_DESCRIPTOR, +) from imap_l3_processing.glows.l3a.models import GlowsL2Data from imap_l3_processing.glows.l3a.utils import read_l2_glows_data @@ -20,8 +24,12 @@ class GlowsL3ADependencies: @classmethod def fetch_dependencies(cls, dependencies: ProcessingInputCollection): science_inputs: list[ScienceInput] = dependencies.get_science_inputs() - desired_input = next(science_input for science_input in science_inputs - if science_input.descriptor == GLOWS_L2_DESCRIPTOR and science_input.source == 'glows') + desired_input = next( + science_input + for science_input in science_inputs + if science_input.descriptor == GLOWS_L2_DESCRIPTOR + and science_input.source == "glows" + ) l2_cdf_file_path: Path = desired_input.imap_file_paths[0].construct_path() @@ -29,17 +37,26 @@ def fetch_dependencies(cls, dependencies: ProcessingInputCollection): cdf = CDF(str(l2_cdf)) l2_glows_data = read_l2_glows_data(cdf) - calibration_dependency = dependencies.get_file_paths(source='glows', - descriptor=GLOWS_CALIBRATION_DATA_DESCRIPTOR) - time_dependent_background_dependency = dependencies.get_file_paths(source='glows', - descriptor=GLOWS_TIME_DEPENDENT_BACKGROUND_DESCRIPTOR) - extra_heliospheric_background_dependency = dependencies.get_file_paths(source='glows', - descriptor=GLOWS_EXTRA_HELIOSPHERIC_BACKGROUND_DESCRIPTOR) - settings_dependency = dependencies.get_file_paths(source='glows', descriptor=GLOWS_PIPELINE_SETTINGS_DESCRIPTOR) + calibration_dependency = dependencies.get_file_paths( + source="glows", descriptor=GLOWS_CALIBRATION_DATA_DESCRIPTOR + ) + time_dependent_background_dependency = dependencies.get_file_paths( + source="glows", descriptor=GLOWS_TIME_DEPENDENT_BACKGROUND_DESCRIPTOR + ) + extra_heliospheric_background_dependency = dependencies.get_file_paths( + source="glows", descriptor=GLOWS_EXTRA_HELIOSPHERIC_BACKGROUND_DESCRIPTOR + ) + settings_dependency = dependencies.get_file_paths( + source="glows", descriptor=GLOWS_PIPELINE_SETTINGS_DESCRIPTOR + ) calibration_path = download(calibration_dependency[0]) - extra_heliospheric_background = download(extra_heliospheric_background_dependency[0]) - time_dependent_background_path = download(time_dependent_background_dependency[0]) + extra_heliospheric_background = download( + extra_heliospheric_background_dependency[0] + ) + time_dependent_background_path = download( + time_dependent_background_dependency[0] + ) settings_path = download(settings_dependency[0]) ancillary_files = { diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/ccsds.py b/imap_l3_processing/glows/l3a/glows_toolkit/ccsds.py index eea07bce0..97d519117 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/ccsds.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/ccsds.py @@ -3,7 +3,8 @@ Class for CCSDS headers """ -class CCSDSHeader(): + +class CCSDSHeader: """ Class for CCSDS headers """ @@ -25,55 +26,73 @@ def deserialize(self, reader): aux = reader.read_uint16_be() if reader.overrun: return - bit_seq_as_string = format(aux, 'b').zfill(16) - #print('first', bit_seq_as_string) - self.data['ccsds_ver'] = int(bit_seq_as_string[:3], 2) - assert self.data['ccsds_ver'] == 0, \ - 'Field ccsds_ver in CCSDS header has incorrect value' + ' %d' % self.data['ccsds_ver'] - self.data['pkt_type'] = bool(int(bit_seq_as_string[3])) - self.data['sec_header_flag'] = bool(int(bit_seq_as_string[4])) - self.data['apid'] = int(bit_seq_as_string[5:], 2) + bit_seq_as_string = format(aux, "b").zfill(16) + # print('first', bit_seq_as_string) + self.data["ccsds_ver"] = int(bit_seq_as_string[:3], 2) + assert self.data["ccsds_ver"] == 0, ( + "Field ccsds_ver in CCSDS header has incorrect value" + + " %d" % self.data["ccsds_ver"] + ) + self.data["pkt_type"] = bool(int(bit_seq_as_string[3])) + self.data["sec_header_flag"] = bool(int(bit_seq_as_string[4])) + self.data["apid"] = int(bit_seq_as_string[5:], 2) - #print(bit_seq_as_string, end=' ') - assert self.data['apid'] in [0x5a1, 0x5a2, 0x5a3, 0x5a4, 0x5a5, 0x5a8, 0x5a9, 0x5b1, 0x5c8, 0x5c9], \ - 'error deserialize(): wrong apid %d 0x%x' % (self.data['apid'], self.data['apid']) + # print(bit_seq_as_string, end=' ') + assert self.data["apid"] in [ + 0x5A1, + 0x5A2, + 0x5A3, + 0x5A4, + 0x5A5, + 0x5A8, + 0x5A9, + 0x5B1, + 0x5C8, + 0x5C9, + ], "error deserialize(): wrong apid %d 0x%x" % ( + self.data["apid"], + self.data["apid"], + ) aux = reader.read_uint16_be() - bit_seq_as_string = format(aux, 'b').zfill(16) - #print('second', bit_seq_as_string) - self.data['grouping_flag'] = int(bit_seq_as_string[:2], 2) - self.data['seq_count'] = int(bit_seq_as_string[2:], 2) + bit_seq_as_string = format(aux, "b").zfill(16) + # print('second', bit_seq_as_string) + self.data["grouping_flag"] = int(bit_seq_as_string[:2], 2) + self.data["seq_count"] = int(bit_seq_as_string[2:], 2) - #print(bit_seq_as_string, end=' ') + # print(bit_seq_as_string, end=' ') - self.data['packet_length'] = reader.read_uint16_be() - #print('third', format(self.data['packet_length'], 'b').zfill(16)) + self.data["packet_length"] = reader.read_uint16_be() + # print('third', format(self.data['packet_length'], 'b').zfill(16)) - #print(self.data['apid'], self.data['packet_length'], 0x5a1) + # print(self.data['apid'], self.data['packet_length'], 0x5a1) - #bit_seq_as_string = format(self.data['packet_length'], 'b').zfill(16) - #print(bit_seq_as_string) + # bit_seq_as_string = format(self.data['packet_length'], 'b').zfill(16) + # print(bit_seq_as_string) -# if self.data['packet_length'] == 289: -# self.data['packet_length'] = 288 -# if self.data['apid'] == 0x5c8 and self.data['packet_length'] == 289: -# print('error deserialize(): bad histogram', self.data['apid'], self.data['packet_length']) -# exit() + # if self.data['packet_length'] == 289: + # self.data['packet_length'] = 288 + # if self.data['apid'] == 0x5c8 and self.data['packet_length'] == 289: + # print('error deserialize(): bad histogram', self.data['apid'], self.data['packet_length']) + # exit() if not reader.overrun: - assert self.data['packet_length'] >= 3 and self.data['packet_length'] <= 4089, \ - 'Field packet_length in CCSDS header has incorrect value' + \ - ' %d' % self.data['packet_length'] + assert ( + self.data["packet_length"] >= 3 and self.data["packet_length"] <= 4089 + ), ( + "Field packet_length in CCSDS header has incorrect value" + + " %d" % self.data["packet_length"] + ) # SCLK/MET field is expected to be included in every CCSDS packet for GLOWS so it is # safe to always read it - self.data['sclk_seconds'] = reader.read_uint32_be() - #print(self.data['sclk_seconds']) + self.data["sclk_seconds"] = reader.read_uint32_be() + # print(self.data['sclk_seconds']) def print_data_keys(self): """ Print data keys for CCSDS header """ - print('\n\nCCSDS header keys:') + print("\n\nCCSDS header keys:") for key in self.data: print(key) diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/constants.py b/imap_l3_processing/glows/l3a/glows_toolkit/constants.py index f2f348aa7..598e8dfff 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/constants.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/constants.py @@ -3,25 +3,25 @@ Constants (broad-sense understood) are grouped together for convenience in this file """ -VERSION = "0.3" # version of ground processing software +VERSION = "0.3" # version of ground processing software # ancillary input expected from SDC (based on general IMAP telemetry) ANC_INPUT_FROM_SDC = { # only one file is here now but in the real pipeline there will be probably several files - 'anc_data_file_name': 'data_ancillary/imap_l1_anc_sc_Merged_2010-2030_mockByGlowsTeam001.csv' + "anc_data_file_name": "data_ancillary/imap_l1_anc_sc_Merged_2010-2030_mockByGlowsTeam001.csv" } # dictionary with names of ancillary input files expected from IMAP/GLOWS Instrument Team ANC_INPUT_FROM_INSTRUMENT_TEAM = { - 'conversion_table_for_anc_data': 'data_ancillary/imap_glows_conversion_table_for_anc_data_v002.json', - 'calibration_data': 'data_ancillary/imap_glows_calibration_data_v002.dat', - 'settings': 'data_ancillary/imap_glows_pipeline_settings_v002.json', - 'map_of_uv_sources': 'data_ancillary/imap_glows_map_of_uv_sources_v002.dat', - 'map_of_excluded_regions': 'data_ancillary/imap_glows_map_of_excluded_regions_v002.dat', - 'exclusions_by_instr_team': 'data_ancillary/imap_glows_exclusions_by_instr_team_v002.dat', - 'suspected_transients': 'data_ancillary/imap_glows_suspected_transients_v002.dat', - 'extra_heliospheric_bckgrd': 'data_ancillary/imap_glows_map_of_extra_helio_bckgrd_v001.dat', - 'time_dependent_bckgrd': 'data_ancillary/imap_glows_time_dep_bckgrd_v001.dat' + "conversion_table_for_anc_data": "data_ancillary/imap_glows_conversion_table_for_anc_data_v002.json", + "calibration_data": "data_ancillary/imap_glows_calibration_data_v002.dat", + "settings": "data_ancillary/imap_glows_pipeline_settings_v002.json", + "map_of_uv_sources": "data_ancillary/imap_glows_map_of_uv_sources_v002.dat", + "map_of_excluded_regions": "data_ancillary/imap_glows_map_of_excluded_regions_v002.dat", + "exclusions_by_instr_team": "data_ancillary/imap_glows_exclusions_by_instr_team_v002.dat", + "suspected_transients": "data_ancillary/imap_glows_suspected_transients_v002.dat", + "extra_heliospheric_bckgrd": "data_ancillary/imap_glows_map_of_extra_helio_bckgrd_v001.dat", + "time_dependent_bckgrd": "data_ancillary/imap_glows_time_dep_bckgrd_v001.dat", } # subsecond limit for GLOWS clock (and consequently also onboard-interpolated IMAP clock) @@ -34,14 +34,14 @@ # numbers indicate bitwise left_shift for a given field L1B_L2_HIST_FLAG_FIELDS = { # bad-angle bits (related to individual bins) - 'is_close_to_uv_source': 0, - 'is_inside_excluded_region': 1, - 'is_excluded_by_instr_team': 2, - 'is_suspected_transient': 3 + "is_close_to_uv_source": 0, + "is_inside_excluded_region": 1, + "is_excluded_by_instr_team": 2, + "is_suspected_transient": 3, } # instrument emulator sometimes does not set the imap_start_time properly, then we have to use # glows_start_time, MAIN_TIME_FIELD was defined to switch quickly between imap_start_time and # glows_start_time for some places in the code where correct values of imap_start_time are crucial # TMP_SOLUTION, NEEDS TO BE CHANGED WHEN imap_start_time will be correct -MAIN_TIME_FIELD = 'imap_start_time' +MAIN_TIME_FIELD = "imap_start_time" diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/constants_arch20241201.py b/imap_l3_processing/glows/l3a/glows_toolkit/constants_arch20241201.py index 86532562d..348ba6088 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/constants_arch20241201.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/constants_arch20241201.py @@ -3,24 +3,23 @@ Constants (broad-sense understood) are grouped together for convenience in this file """ -VERSION = "0.2" # version of ground processing software +VERSION = "0.2" # version of ground processing software # ancillary input expected from SDC (based on general IMAP telemetry) ANC_INPUT_FROM_SDC = { # only one file is here now but in the real pipeline there will be probably several files - 'anc_data_file_name': 'data_ancillary/imap_l1_anc_sc_Merged_2010-2030_mockByGlowsTeam001.csv' + "anc_data_file_name": "data_ancillary/imap_l1_anc_sc_Merged_2010-2030_mockByGlowsTeam001.csv" } # ancillary input expected from IMAP/GLOWS Instrument Team ANC_INPUT_FROM_INSTRUMENT_TEAM = { - 'conversion_table_for_anc_data': - 'data_ancillary/imap_glows_conversion_table_for_anc_data_v001.json', - 'calibration_data': 'data_ancillary/imap_glows_calibration_data_v001.json', - 'settings': 'data_ancillary/imap_glows_pipeline_settings_v001.json', - 'map_of_uv_sources': 'data_ancillary/imap_glows_map_of_uv_sources_v001.dat', - 'map_of_excluded_regions': 'data_ancillary/imap_glows_map_of_excluded_regions_v001.dat', - 'exclusions_by_instr_team': 'data_ancillary/imap_glows_exclusions_by_instr_team_v001.dat', - 'suspected_transients': 'data_ancillary/imap_glows_suspected_transients_v001.dat' + "conversion_table_for_anc_data": "data_ancillary/imap_glows_conversion_table_for_anc_data_v001.json", + "calibration_data": "data_ancillary/imap_glows_calibration_data_v001.json", + "settings": "data_ancillary/imap_glows_pipeline_settings_v001.json", + "map_of_uv_sources": "data_ancillary/imap_glows_map_of_uv_sources_v001.dat", + "map_of_excluded_regions": "data_ancillary/imap_glows_map_of_excluded_regions_v001.dat", + "exclusions_by_instr_team": "data_ancillary/imap_glows_exclusions_by_instr_team_v001.dat", + "suspected_transients": "data_ancillary/imap_glows_suspected_transients_v001.dat", } # subsecond limit for GLOWS clock (and consequently also onboard-interpolated IMAP clock) @@ -33,17 +32,17 @@ # numbers indicate bitwise left_shift for a given field L1B_L2_HIST_FLAG_FIELDS = { # bad-angle bits (related to individual bins) - 'is_close_to_uv_source': 0, - 'is_inside_excluded_region': 1, - 'is_excluded_by_instr_team': 2, - 'is_suspected_transient': 3 + "is_close_to_uv_source": 0, + "is_inside_excluded_region": 1, + "is_excluded_by_instr_team": 2, + "is_suspected_transient": 3, } # instrument emulator sometimes does not set the imap_start_time properly, then we have to use # glows_start_time, MAIN_TIME_FIELD was defined to switch quickly between imap_start_time and # glows_start_time for some places in the code where correct values of imap_start_time are crucial # TMP_SOLUTION, NEEDS TO BE CHANGED WHEN imap_start_time will be correct -MAIN_TIME_FIELD = 'imap_start_time' +MAIN_TIME_FIELD = "imap_start_time" # number of bins for helioglow lightcurve at L3 L3_N_BINS = 90 diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/direct_event.py b/imap_l3_processing/glows/l3a/glows_toolkit/direct_event.py index 3eea1dd07..032d9a86c 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/direct_event.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/direct_event.py @@ -1,36 +1,45 @@ """@package docstring Author: Marek Strumik, maro at cbk.waw.pl """ + import struct from dataclasses import dataclass from .constants import SUBSECOND_LIMIT + @dataclass(frozen=True) class DirectEvent: """ DirectEvent() class for IMAP/GLOWS """ + seconds: int subseconds: int impulse_length: int multi_event: bool = False # based on deserialize() in glows_appsw-master/tooling/science/models/direct_event.py by KPLabs - def build_event_from_uncompressed_data(raw) -> 'DirectEvent': + def build_event_from_uncompressed_data(raw) -> "DirectEvent": """ Build direct event from raw binary 8-byte array assuming that it contains uncompressed timestamps """ assert len(raw) == 8 - values = struct.unpack('>II', raw) - return DirectEvent( # see glows_appsw/-/blob/master/libs/science/data/include/science/science_direct_event.h (bits 22,23 are not used) + values = struct.unpack(">II", raw) + return DirectEvent( # see glows_appsw/-/blob/master/libs/science/data/include/science/science_direct_event.h (bits 22,23 are not used) seconds=values[0], - subseconds=values[1] & 0x1FFFFF, # subsecond encoding on the least significant 21 bits - impulse_length=(values[1] >> 24) & 0xFF, # first byte encodes the impulse length - multi_event=bool((values[1] >> 23) & 0b1), # KPLabs says it is set by FPGA and currently not used by AppSW at all + subseconds=values[1] + & 0x1FFFFF, # subsecond encoding on the least significant 21 bits + impulse_length=(values[1] >> 24) + & 0xFF, # first byte encodes the impulse length + multi_event=bool( + (values[1] >> 23) & 0b1 + ), # KPLabs says it is set by FPGA and currently not used by AppSW at all ) # based on _build_event() in glows_appsw-master/tooling/science/models/direct_event_file_decompressor.py by KPLabs - def build_event_from_compressed_data(diff: int, length: int, current_event: 'DirectEvent') -> 'DirectEvent': + def build_event_from_compressed_data( + diff: int, length: int, current_event: "DirectEvent" + ) -> "DirectEvent": """ Build direct event assuming that it contains timestamps compressed as timedeltas """ @@ -38,8 +47,13 @@ def build_event_from_compressed_data(diff: int, length: int, current_event: 'Dir seconds = current_event.seconds if subseconds >= SUBSECOND_LIMIT: - full_seconds = int(subseconds/SUBSECOND_LIMIT) + full_seconds = int(subseconds / SUBSECOND_LIMIT) seconds += full_seconds - subseconds -= full_seconds*SUBSECOND_LIMIT + subseconds -= full_seconds * SUBSECOND_LIMIT - return DirectEvent(seconds=seconds, subseconds=subseconds, impulse_length=length, multi_event=False) + return DirectEvent( + seconds=seconds, + subseconds=subseconds, + impulse_length=length, + multi_event=False, + ) diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/funcs.py b/imap_l3_processing/glows/l3a/glows_toolkit/funcs.py index 44c30d31b..41af3d46d 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/funcs.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/funcs.py @@ -2,11 +2,13 @@ Author: Marek Strumik, maro at cbk.waw.pl Various functions that are not methods to classes """ + import datetime import numpy as np from .imap_time import SpacecraftTime from .constants import SUBSECOND_LIMIT + def calibration_factor(calibration_file_name, date_string): """ Compute calibration factor cps_per_R to convert cps to the absolute physical units of Rayleigh @@ -22,6 +24,7 @@ def calibration_factor(calibration_file_name, date_string): idx = np.nonzero(date_array <= np.datetime64(date_string))[0][-1] return cps_per_R_array[idx] + def check_if_contains_actual_data(file_name): """ Check if there is any actual data in a file apart from header @@ -29,17 +32,18 @@ def check_if_contains_actual_data(file_name): Args: file_name: name of file to be checked """ - f = open(file_name, 'r') + f = open(file_name, "r") line = f.readline() is_data_in_file = False while line: - if line.lstrip()[0] != '#' and len(line.lstrip()) > 0: + if line.lstrip()[0] != "#" and len(line.lstrip()) > 0: is_data_in_file = True break line = f.readline() f.close() return is_data_in_file + def decode_ancillary_parameter(name, decoding_table, encoded_value): """ Decode parameters collected onboard, which are encoded as int at @@ -49,17 +53,18 @@ def decode_ancillary_parameter(name, decoding_table, encoded_value): decoding_table: dictionary created from JSON file provided by GLOWS Instrument Team l1a_data: L1a data provided as dictionary """ - params = decoding_table[name] # extract decoding params for a given parameter name + params = decoding_table[name] # extract decoding params for a given parameter name # compute parameters A and B - p_a = (2**params['n_bits'] - 1)/(params['max'] - params['min']) - p_b = -params['min']*p_a + p_a = (2 ** params["n_bits"] - 1) / (params["max"] - params["min"]) + p_b = -params["min"] * p_a # decode int-encoded value - decoded_value = (encoded_value-p_b)/p_a + decoded_value = (encoded_value - p_b) / p_a return decoded_value + def decode_ancillary_parameters_avg_stddev(name, decoding_table, l1a_data): """ Decode parameters collected onboard (average and variance), which are encoded as int at @@ -70,22 +75,23 @@ def decode_ancillary_parameters_avg_stddev(name, decoding_table, l1a_data): decoding_table: dictionary created from JSON file provided by GLOWS Instrument Team l1a_data: L1a data provided as dictionary """ - params = decoding_table[name] # extract decoding params for a given parameter name + params = decoding_table[name] # extract decoding params for a given parameter name # compute parameters A and B from Sec. "On-board encoding and ground decoding of measured quantities" # in the algorithm document - p_a = (2**params['n_bits'] - 1)/(params['max'] - params['min']) - p_b = -params['min']*p_a + p_a = (2 ** params["n_bits"] - 1) / (params["max"] - params["min"]) + p_b = -params["min"] * p_a # decode int values for the average and variance - average = (l1a_data[name + '_average']-p_b)/p_a - variance = l1a_data[name + '_variance']/(p_a*p_a) + average = (l1a_data[name + "_average"] - p_b) / p_a + variance = l1a_data[name + "_variance"] / (p_a * p_a) # compute standard deviation std_deviation = np.sqrt(variance) return average, std_deviation + def flags_deserialize(raw): """ Convert raw-int-number-encoded GLOWS flags to a dictionary @@ -93,20 +99,21 @@ def flags_deserialize(raw): Args: raw: flags encoded as bit fields of an integer """ - assert raw < 65536, 'Argument of flags_deserialize() is wrong' + assert raw < 65536, "Argument of flags_deserialize() is wrong" flags = {} - flags['is_pps_missing'] = bool((raw >> 0) & 1) - flags['is_time_status_missing'] = bool((raw >> 1) & 1) - flags['is_phase_missing'] = bool((raw >> 2) & 1) - flags['is_spin_period_missing'] = bool((raw >> 3) & 1) - flags['is_overexposed'] = bool((raw >> 4) & 1) - flags['is_direct_event_non_monotonic'] = bool((raw >> 5) & 1) - flags['is_night'] = bool((raw >> 6) & 1) - flags['is_hv_test_in_progress'] = bool((raw >> 7) & 1) - flags['is_test_pulse_in_progress'] = bool((raw >> 8) & 1) - flags['is_memory_error_detected'] = bool((raw >> 9) & 1) + flags["is_pps_missing"] = bool((raw >> 0) & 1) + flags["is_time_status_missing"] = bool((raw >> 1) & 1) + flags["is_phase_missing"] = bool((raw >> 2) & 1) + flags["is_spin_period_missing"] = bool((raw >> 3) & 1) + flags["is_overexposed"] = bool((raw >> 4) & 1) + flags["is_direct_event_non_monotonic"] = bool((raw >> 5) & 1) + flags["is_night"] = bool((raw >> 6) & 1) + flags["is_hv_test_in_progress"] = bool((raw >> 7) & 1) + flags["is_test_pulse_in_progress"] = bool((raw >> 8) & 1) + flags["is_memory_error_detected"] = bool((raw >> 9) & 1) return flags + def read_l0_data(pkts_file_name): """ Read L0 data as an input data for L1a @@ -114,11 +121,12 @@ def read_l0_data(pkts_file_name): pkts_file_name: name of of the file with CCSDS packets """ # open binary pkts file containing CCSDS packets with histograms - file_handler = open('%s' % pkts_file_name, mode='rb') + file_handler = open("%s" % pkts_file_name, mode="rb") # read binary file with histograms (CCSDS headers are assumed to be included) histogram_telemetry_packets = file_handler.read() file_handler.close() - return histogram_telemetry_packets # byte array + return histogram_telemetry_packets # byte array + def time_sec_subsec_to_utc(seconds: int, subseconds: int) -> datetime: """ @@ -128,8 +136,9 @@ def time_sec_subsec_to_utc(seconds: int, subseconds: int) -> datetime: subseconds: """ result = SpacecraftTime.spacecraft_epoch_to_date(seconds) - result = result + datetime.timedelta(microseconds=subseconds/2) - return result # datatime object + result = result + datetime.timedelta(microseconds=subseconds / 2) + return result # datatime object + def time_sec_subsec_to_float64(seconds: int, subseconds: int): """ @@ -141,5 +150,5 @@ def time_sec_subsec_to_float64(seconds: int, subseconds: int): subseconds: """ # python default floating-point type is 64-bit double precision - result = seconds + subseconds/SUBSECOND_LIMIT + result = seconds + subseconds / SUBSECOND_LIMIT return result diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/geom_fun_libr_box/__init__.py b/imap_l3_processing/glows/l3a/glows_toolkit/geom_fun_libr_box/__init__.py index a1aa58757..a94a5eb57 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/geom_fun_libr_box/__init__.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/geom_fun_libr_box/__init__.py @@ -5,8 +5,10 @@ Function names and functionality are intentionally possibly similar to geomFunLibrBox Functions are listed in alphabetical order """ + import numpy as np + def angle_coord(lon_1, lat_1, lon_2, lat_2): """ Compute angle between two directions lon_1,lat_1 and lon_2,lat_2 (longitude l and latitude b) @@ -44,6 +46,7 @@ def angle_coord(lon_1, lat_1, lon_2, lat_2): angle = angle_vec(v_1, v_2) return angle + def angle_vec(vec1, vec2): """ Compute angle between two vectors vec1 and vec2 @@ -78,37 +81,42 @@ def angle_vec(vec1, vec2): [ 0. 40. 80. 120. 160. 160. 120. 80. 40. 0.] """ if np.shape(vec1) != np.shape(vec2): - raise Exception('angle_vec() error: np.shape(vec1)!=np.shape(vec2)') + raise Exception("angle_vec() error: np.shape(vec1)!=np.shape(vec2)") if vec1.ndim == 1: if np.shape(vec1)[0] != 3: - raise Exception('angle_vec() error: np.shape(vec1)[0]!=3') + raise Exception("angle_vec() error: np.shape(vec1)[0]!=3") r_1 = np.sqrt(np.dot(vec1, vec1)) r_2 = np.sqrt(np.dot(vec2, vec2)) - cos = np.dot(vec1, vec2)/(r_1*r_2) + cos = np.dot(vec1, vec2) / (r_1 * r_2) # handle a case where cos is shifted below -1 due to numerical errors - if cos < -1.0 and np.fabs(cos+1.0) < 1.0e-14: + if cos < -1.0 and np.fabs(cos + 1.0) < 1.0e-14: cos = -1.0 # handle a case where cos is shifted above 1 due to numerical errors - if cos > 1.0 and np.fabs(cos-1.0) < 1.0e-14: + if cos > 1.0 and np.fabs(cos - 1.0) < 1.0e-14: cos = 1.0 if r_1 != 0.0 and r_2 != 0.0: return np.arccos(cos) return 0.0 if np.shape(vec1)[1] != 3: - raise Exception('angle_vec() error: np.shape(vec1)[1]!=3') + raise Exception("angle_vec() error: np.shape(vec1)[1]!=3") r_1 = np.sqrt((vec1 * vec1).sum(axis=1)) r_2 = np.sqrt((vec2 * vec2).sum(axis=1)) idxs0 = np.nonzero(np.logical_and(r_1 != 0.0, r_2 != 0.0)) - cos = (vec1[idxs0[0], :] * vec2[idxs0[0], :]).sum(axis=1)/(r_1[idxs0[0]]*r_2[idxs0[0]]) - idxs = np.nonzero(np.logical_and(cos < -1.0, np.fabs(cos+1.0) < 1.0e-14)) - cos[idxs[0]] = -1.0 # just in case if cos is shifted below -1 due to numerical errors - idxs = np.nonzero(np.logical_and(cos > 1.0, np.fabs(cos-1.0) < 1.0e-14)) - cos[idxs[0]] = 1.0 # just in case if cos is shifted above 1 due to numerical errors + cos = (vec1[idxs0[0], :] * vec2[idxs0[0], :]).sum(axis=1) / ( + r_1[idxs0[0]] * r_2[idxs0[0]] + ) + idxs = np.nonzero(np.logical_and(cos < -1.0, np.fabs(cos + 1.0) < 1.0e-14)) + cos[ + idxs[0] + ] = -1.0 # just in case if cos is shifted below -1 due to numerical errors + idxs = np.nonzero(np.logical_and(cos > 1.0, np.fabs(cos - 1.0) < 1.0e-14)) + cos[idxs[0]] = 1.0 # just in case if cos is shifted above 1 due to numerical errors angle = np.zeros((np.shape(vec1)[0]), dtype=float) angle[idxs0[0]] = np.arccos(cos) return angle + def get_spher_coord_rad(vec): """ Compute radius r, longitude l and latitude b for a vector vec @@ -159,23 +167,28 @@ def get_spher_coord_rad(vec): if vec_m == 0.0: lat = 0.0 else: - lat = np.arcsin(vec[2]/vec_m) # compute latitude + lat = np.arcsin(vec[2] / vec_m) # compute latitude # compute longitude if vec[0] == 0.0 and vec[1] == 0.0: lon = 0.0 else: - lon = np.arctan2(vec[1], vec[0])%(2*np.pi) - return np.array([vec_m, lon, lat]) # return one array similarly as in Mathematica + lon = np.arctan2(vec[1], vec[0]) % (2 * np.pi) + return np.array( + [vec_m, lon, lat] + ) # return one array similarly as in Mathematica if np.shape(vec)[1] != 3: raise Exception("get_spher_coord_rad() error: np.shape(vec)[1]!=3") vec_m = np.sqrt((vec * vec).sum(axis=1)) idxs = np.nonzero(vec_m != 0.0) lat = np.zeros((len(vec_m)), float) - lat[idxs[0]] = np.arcsin(vec[idxs[0], 2]/vec_m[idxs[0]]) # compute latitude - lon = np.arctan2(vec[:, 1], vec[:, 0])%(2*np.pi) # compute longitude + lat[idxs[0]] = np.arcsin(vec[idxs[0], 2] / vec_m[idxs[0]]) # compute latitude + lon = np.arctan2(vec[:, 1], vec[:, 0]) % (2 * np.pi) # compute longitude idxs = np.nonzero(np.logical_and(vec[:, 0] == 0.0, vec[:, 1] == 0.0)) - lon[idxs[0]] = 0.0 # set lon=0 if vec[:,0]==0.0 and vec[:,1]==0.0 - return np.column_stack([vec_m, lon, lat]) # return one array similarly as in Mathematica + lon[idxs[0]] = 0.0 # set lon=0 if vec[:,0]==0.0 and vec[:,1]==0.0 + return np.column_stack( + [vec_m, lon, lat] + ) # return one array similarly as in Mathematica + def lb2spin(l_0, b_0, soff, l_q, b_q): """ @@ -231,11 +244,12 @@ def lb2spin(l_0, b_0, soff, l_q, b_q): [ 40. 270.]] """ tht = angle_coord(l_0, b_0, l_q, b_q) - spin = (-position_angle(l_q, b_q, l_0, b_0)-soff)%(2.*np.pi) + spin = (-position_angle(l_q, b_q, l_0, b_0) - soff) % (2.0 * np.pi) if np.isscalar(l_0): return np.array([tht, spin]) return np.column_stack([tht, spin]) + def make_co_vec(l_q, b_q): """ Compute vectors for given values of longitude l_q and co-latitude b_q @@ -279,8 +293,13 @@ def make_co_vec(l_q, b_q): if np.shape(l_q) != np.shape(b_q): raise Exception("make_co_vec() error: np.shape(l_q)!=np.shape(b_q)") if np.isscalar(l_q): - return np.array([np.cos(l_q)*np.sin(b_q), np.sin(l_q)*np.sin(b_q), np.cos(b_q)]) - return np.column_stack([np.cos(l_q)*np.sin(b_q), np.sin(l_q)*np.sin(b_q), np.cos(b_q)]) + return np.array( + [np.cos(l_q) * np.sin(b_q), np.sin(l_q) * np.sin(b_q), np.cos(b_q)] + ) + return np.column_stack( + [np.cos(l_q) * np.sin(b_q), np.sin(l_q) * np.sin(b_q), np.cos(b_q)] + ) + def make_vec(l_q, b_q): """ @@ -325,8 +344,13 @@ def make_vec(l_q, b_q): if np.shape(l_q) != np.shape(b_q): raise Exception("make_vec() error: np.shape(l_q)!=np.shape(b_q)") if np.isscalar(l_q): - return np.array([np.cos(l_q)*np.cos(b_q), np.sin(l_q)*np.cos(b_q), np.sin(b_q)]) - return np.column_stack([np.cos(l_q)*np.cos(b_q), np.sin(l_q)*np.cos(b_q), np.sin(b_q)]) + return np.array( + [np.cos(l_q) * np.cos(b_q), np.sin(l_q) * np.cos(b_q), np.sin(b_q)] + ) + return np.column_stack( + [np.cos(l_q) * np.cos(b_q), np.sin(l_q) * np.cos(b_q), np.sin(b_q)] + ) + def position_angle(a_1, d_1, a_2, d_2): """ @@ -368,10 +392,13 @@ def position_angle(a_1, d_1, a_2, d_2): [ 0. 6.60406655 15.06329832 25.50555026 37.6074435 50.43084061 62.72683044 73.55865702 82.60570856 90. ] """ - delta_a = a_1-a_2 - pos_a = np.arctan2(np.sin(delta_a), np.cos(d_2)*np.tan(d_1)-np.sin(d_2)*np.cos(delta_a)) + delta_a = a_1 - a_2 + pos_a = np.arctan2( + np.sin(delta_a), np.cos(d_2) * np.tan(d_1) - np.sin(d_2) * np.cos(delta_a) + ) return pos_a + def rot_two_vec_frame(l_c, b_c, l_x, b_x, l_q, b_q): """ rotate (l_q,b_q) to a frame centered at (l_c,b_c) and x-axis coaligned with (l_c,b_c),(l_x,b_x) @@ -390,17 +417,18 @@ def rot_two_vec_frame(l_c, b_c, l_x, b_x, l_q, b_q): else: pos_ang = lb2spin(l_c, b_c, 0.0, l_x, b_x)[:, 1] # compute rotation matrix - rot_mtrx = rzc_ryb_rza(-(2.0*np.pi-l_c), -(b_c-0.5*np.pi), pos_ang+np.pi) - vec0 = make_vec(l_q, b_q) # temporary vectors to rotate + rot_mtrx = rzc_ryb_rza(-(2.0 * np.pi - l_c), -(b_c - 0.5 * np.pi), pos_ang + np.pi) + vec0 = make_vec(l_q, b_q) # temporary vectors to rotate if np.isscalar(l_q): - vec = np.matmul(rot_mtrx, vec0) # rotate vec0 + vec = np.matmul(rot_mtrx, vec0) # rotate vec0 else: - vec = np.einsum('lij,lj->li', rot_mtrx, vec0) + vec = np.einsum("lij,lj->li", rot_mtrx, vec0) rlb = get_spher_coord_rad(vec) if np.isscalar(l_q): return np.array([-rlb[1], rlb[2]]) return np.column_stack([-rlb[:, 1], rlb[:, 2]]) + def rot_x(ang): """ Construct a rotation matrix that rotates vectors by the angle ang about @@ -441,21 +469,26 @@ def rot_x(ang): [ 0.00000000e+00 -1.00000000e+00 6.12323400e-17]]] """ if np.isscalar(ang): - rot_mtrx = [[1.0, 0.0, 0.0], - [0.0, np.cos(ang), np.sin(ang)], - [0.0, -np.sin(ang), np.cos(ang)]] + rot_mtrx = [ + [1.0, 0.0, 0.0], + [0.0, np.cos(ang), np.sin(ang)], + [0.0, -np.sin(ang), np.cos(ang)], + ] rot_mtrx = np.array(rot_mtrx) else: n_ang = len(ang) zers = np.zeros((n_ang), float) ones = np.ones((n_ang), float) - rot_mtrx = [[ones, zers, zers], - [zers, np.cos(ang), np.sin(ang)], - [zers, -np.sin(ang), np.cos(ang)]] + rot_mtrx = [ + [ones, zers, zers], + [zers, np.cos(ang), np.sin(ang)], + [zers, -np.sin(ang), np.cos(ang)], + ] rot_mtrx = np.array(rot_mtrx) rot_mtrx = rot_mtrx.transpose(2, 0, 1) return rot_mtrx + def rot_y(ang): """ Construct a rotation matrix that rotates vectors by the angle ang about @@ -496,21 +529,26 @@ def rot_y(ang): [ 1.00000000e+00 0.00000000e+00 6.12323400e-17]]] """ if np.isscalar(ang): - rot_mtrx = [[np.cos(ang), 0.0, -np.sin(ang)], - [0.0, 1.0, 0.0], - [np.sin(ang), 0.0, np.cos(ang)]] + rot_mtrx = [ + [np.cos(ang), 0.0, -np.sin(ang)], + [0.0, 1.0, 0.0], + [np.sin(ang), 0.0, np.cos(ang)], + ] rot_mtrx = np.array(rot_mtrx) else: n_ang = len(ang) zers = np.zeros((n_ang), float) ones = np.ones((n_ang), float) - rot_mtrx = [[np.cos(ang), zers, -np.sin(ang)], - [zers, ones, zers], - [np.sin(ang), zers, np.cos(ang)]] + rot_mtrx = [ + [np.cos(ang), zers, -np.sin(ang)], + [zers, ones, zers], + [np.sin(ang), zers, np.cos(ang)], + ] rot_mtrx = np.array(rot_mtrx) rot_mtrx = rot_mtrx.transpose(2, 0, 1) return rot_mtrx + def rot_z(ang): """ Construct a rotation matrix that rotates vectors by the angle ang about @@ -551,21 +589,26 @@ def rot_z(ang): [ 0.00000000e+00 0.00000000e+00 1.00000000e+00]]] """ if np.isscalar(ang): - rot_mtrx = [[np.cos(ang), np.sin(ang), 0.0], - [-np.sin(ang), np.cos(ang), 0.0], - [0.0, 0.0, 1.0]] + rot_mtrx = [ + [np.cos(ang), np.sin(ang), 0.0], + [-np.sin(ang), np.cos(ang), 0.0], + [0.0, 0.0, 1.0], + ] rot_mtrx = np.array(rot_mtrx) else: n_ang = len(ang) zers = np.zeros((n_ang), float) ones = np.ones((n_ang), float) - rot_mtrx = [[np.cos(ang), np.sin(ang), zers], - [-np.sin(ang), np.cos(ang), zers], - [zers, zers, ones]] + rot_mtrx = [ + [np.cos(ang), np.sin(ang), zers], + [-np.sin(ang), np.cos(ang), zers], + [zers, zers, ones], + ] rot_mtrx = np.array(rot_mtrx) rot_mtrx = rot_mtrx.transpose(2, 0, 1) return rot_mtrx + def rxc_ryb_rza(ang_a, ang_b, ang_c): """ Construct a rotation matrix that rotates vectors by the @@ -595,6 +638,7 @@ def rxc_ryb_rza(ang_a, ang_b, ang_c): rot_mtrx = np.matmul(rot_x(ang_c), np.matmul(rot_y(ang_b), rot_z(ang_a))) return rot_mtrx + def rxc_rzb_rxa(ang_a, ang_b, ang_c): """ Construct a rotation matrix that rotates vectors by the @@ -624,6 +668,7 @@ def rxc_rzb_rxa(ang_a, ang_b, ang_c): rot_mtrx = np.matmul(rot_x(ang_c), np.matmul(rot_z(ang_b), rot_x(ang_a))) return rot_mtrx + def rzb_rya(ang_a, ang_b): """ Construct a rotation matrix that rotates vectors by the @@ -650,6 +695,7 @@ def rzb_rya(ang_a, ang_b): rot_mtrx = np.matmul(rot_z(ang_b), rot_y(ang_a)) return rot_mtrx + def rzc_ryb_rza(ang_a, ang_b, ang_c): """ Construct a rotation matrix that rotates vectors by the @@ -698,6 +744,7 @@ def rzc_ryb_rza(ang_a, ang_b, ang_c): rot_mtrx = np.matmul(rot_z(ang_c), np.matmul(rot_y(ang_b), rot_z(ang_a))) return rot_mtrx + def spin2lb(l_0, b_0, spin_ang, spin_off, tht): """ Computes longitude,latitude (l,b) for a given spin angle spin_ang. @@ -749,18 +796,23 @@ def spin2lb(l_0, b_0, spin_ang, spin_off, tht): [ 1.00000000e+02 -4.00000000e+01] [ 1.40000000e+02 -4.07782511e-15]] """ - rot_mtrx = rzc_ryb_rza(-spin_ang-spin_off, b_0-0.5*np.pi, -l_0) # compute rotation matrix + rot_mtrx = rzc_ryb_rza( + -spin_ang - spin_off, b_0 - 0.5 * np.pi, -l_0 + ) # compute rotation matrix if np.isscalar(tht): - vec0 = np.array([-np.sin(tht), 0.0, np.cos(tht)]) # temporary vector to rotate - vec = np.matmul(rot_mtrx, vec0) # rotate vec0 + vec0 = np.array([-np.sin(tht), 0.0, np.cos(tht)]) # temporary vector to rotate + vec = np.matmul(rot_mtrx, vec0) # rotate vec0 else: - vec0 = np.column_stack([-np.sin(tht), np.zeros((len(tht)), dtype=float), np.cos(tht)]) - vec = np.einsum('lij,lj->li', rot_mtrx, vec0) + vec0 = np.column_stack( + [-np.sin(tht), np.zeros((len(tht)), dtype=float), np.cos(tht)] + ) + vec = np.einsum("lij,lj->li", rot_mtrx, vec0) rlb = get_spher_coord_rad(vec) if np.isscalar(l_0): return np.array([rlb[1], rlb[2]]) return np.column_stack([rlb[:, 1], rlb[:, 2]]) + def spin2lb_test_on_grid(spin_off, tht): """ Test functions spin2lb() and lb2spin() for l_0,b_0,spin values on a grid of 11x11x11 cases @@ -790,12 +842,12 @@ def spin2lb_test_on_grid(spin_off, tht): # create a grid of cases lin_spc_1 = np.linspace(0.0, 360.0, 11) lin_spc_2 = np.linspace(-90.0, 90.0, 11) -# l_0, b_0, s_0 = np.radians(np.mgrid[0.0:360.0:11*1j, -90.0:90.0:11*1j, 0.0:360.0:11*1j]) + # l_0, b_0, s_0 = np.radians(np.mgrid[0.0:360.0:11*1j, -90.0:90.0:11*1j, 0.0:360.0:11*1j]) l_0, b_0, s_0 = np.radians(np.meshgrid(lin_spc_1, lin_spc_2, lin_spc_1)) l_0 = l_0.flatten() b_0 = b_0.flatten() s_0 = s_0.flatten() - tht_0 = tht*np.ones_like(s_0) + tht_0 = tht * np.ones_like(s_0) lon_lat = spin2lb(l_0, b_0, s_0, spin_off, tht_0) tht1s = lb2spin(l_0, b_0, spin_off, lon_lat[:, 0], lon_lat[:, 1]) @@ -805,23 +857,24 @@ def spin2lb_test_on_grid(spin_off, tht): ist = 0 # check theta - idxs = np.nonzero(np.fabs(tht_1-tht_0) > 1.0e-14) + idxs = np.nonzero(np.fabs(tht_1 - tht_0) > 1.0e-14) if np.shape(idxs[0])[0] > 0: ist = 1 # check spin - err_s = np.fabs(spin_ang-s_0) - idxs = np.nonzero(np.fabs(err_s-2.0*np.pi) < 1.0e-14) - err_s[idxs] = err_s[idxs]-2.0*np.pi + err_s = np.fabs(spin_ang - s_0) + idxs = np.nonzero(np.fabs(err_s - 2.0 * np.pi) < 1.0e-14) + err_s[idxs] = err_s[idxs] - 2.0 * np.pi if err_s.max() > 1.0e-14: ist = 1 if ist == 0: - print('spin2lb_test_on_grid(tht=%g,spin_off=%g): passed' % (tht, spin_off)) + print("spin2lb_test_on_grid(tht=%g,spin_off=%g): passed" % (tht, spin_off)) return 1 - print('spin2lb_test_on_grid(tht=%g,spin_off=%g): failed' % (tht, spin_off)) + print("spin2lb_test_on_grid(tht=%g,spin_off=%g): failed" % (tht, spin_off)) return 0 + def spin2lb_test_on_text_file(filename): """ Test spin2lb() and lb2spin() using content of a text file @@ -848,7 +901,9 @@ def spin2lb_test_on_text_file(filename): 1 """ # load a text file with testing cases - l_0, b_0, s_0, spin_off, tht, l_q, b_q = np.radians(np.loadtxt(filename, unpack=True)) + l_0, b_0, s_0, spin_off, tht, l_q, b_q = np.radians( + np.loadtxt(filename, unpack=True) + ) # back-and-forth transforms ltbt = spin2lb(l_0, b_0, s_0, spin_off, tht) @@ -859,30 +914,30 @@ def spin2lb_test_on_text_file(filename): ist = 0 # check longitude - idxs = np.nonzero(np.fabs(l_q-ltbt[:, 0]) > 1.0e-14) + idxs = np.nonzero(np.fabs(l_q - ltbt[:, 0]) > 1.0e-14) if np.shape(idxs[0])[0] > 0: ist = 1 # check latitude - idxs = np.nonzero(np.fabs(b_q-ltbt[:, 1]) > 1.0e-14) + idxs = np.nonzero(np.fabs(b_q - ltbt[:, 1]) > 1.0e-14) if np.shape(idxs[0])[0] > 0: ist = 1 # check theta - idxs = np.nonzero(np.fabs(tht-thtt) > 1.0e-14) + idxs = np.nonzero(np.fabs(tht - thtt) > 1.0e-14) if np.shape(idxs[0])[0] > 0: ist = 1 # check spin - err_s = np.fabs(s_t-s_0) - idxs = np.nonzero(np.fabs(err_s-2.0*np.pi) < 1.0e-14) - err_s[idxs] = err_s[idxs]-2.0*np.pi + err_s = np.fabs(s_t - s_0) + idxs = np.nonzero(np.fabs(err_s - 2.0 * np.pi) < 1.0e-14) + err_s[idxs] = err_s[idxs] - 2.0 * np.pi idxs = np.nonzero(err_s > 1.0e-14) if np.shape(idxs[0])[0] > 0: ist = 1 if ist == 0: - print('spin2lb_test_on_text_file(\'%s\'): passed' % filename) + print("spin2lb_test_on_text_file('%s'): passed" % filename) return 1 - print('spin2lb_test_on_text_file(\'%s\'): failed' % filename) + print("spin2lb_test_on_text_file('%s'): failed" % filename) return 0 diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/imap_time.py b/imap_l3_processing/glows/l3a/glows_toolkit/imap_time.py index 3bde4a168..15b0d5195 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/imap_time.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/imap_time.py @@ -1,12 +1,15 @@ """@package docstring Author: KPLabs with some modifications by Marek Strumik, maro at cbk.waw.pl """ + from datetime import datetime, timedelta -class SpacecraftTime(): + +class SpacecraftTime: """ SpacecraftTime() class for epoch-datatime conversions """ + @staticmethod def date_to_spacecraft_epoch(time_tag: datetime) -> int: """Get number of seconds after 1 January 2010, midnight diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/l0_data_direct_events.py b/imap_l3_processing/glows/l3a/glows_toolkit/l0_data_direct_events.py index 5b129933a..789274c9b 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/l0_data_direct_events.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/l0_data_direct_events.py @@ -2,10 +2,12 @@ Author: Marek Strumik, maro at cbk.waw.pl """ -class L0DataDirectEvents(): + +class L0DataDirectEvents: """ L0DataDirectEvents class for GLOWS-L0-direct-events data """ + def __init__(self): """ Constructor for this class @@ -21,38 +23,41 @@ def deserialize(self, reader, packet_length): Args: reader: Reader-class object needed for reading binary data stream """ - self.data['imap_start_time_second'] = reader.read_uint32_be() - self.data['number_of_de_packets'] = reader.read_uint16_be() - self.data['seq_num_of_de_packet'] = reader.read_uint16_be() + self.data["imap_start_time_second"] = reader.read_uint32_be() + self.data["number_of_de_packets"] = reader.read_uint16_be() + self.data["seq_num_of_de_packet"] = reader.read_uint16_be() # TMP_SOLUTION # read 4078 bytes if zero-padding is used - #self.data['de_data'] = bytearray(reader.read_buffer(4078)) + # self.data['de_data'] = bytearray(reader.read_buffer(4078)) # TMP_SOLUTION # read packet_length-with-correction bytes if zero-padding is not used # + 1 see definition of CCSDS header, -4 sclk, -4 - 2 - 2 if for the three reads above packet_length_correction = 1 - 4 - 4 - 2 - 2 - assert (packet_length + packet_length_correction) > 0, \ - 'wrong packet length for direct-event packet' - self.data['de_data'] = bytearray(reader.read_buffer(packet_length + packet_length_correction)) + assert (packet_length + packet_length_correction) > 0, ( + "wrong packet length for direct-event packet" + ) + self.data["de_data"] = bytearray( + reader.read_buffer(packet_length + packet_length_correction) + ) # DEBUG TEST - #print() - #print(len(self.data['de_data'])) - #import binascii - #print(binascii.hexlify(self.data['de_data'])) - #print(' '.join(format(x, '02x') for x in self.data['de_data'][:40])) - #print(' '.join(format(x, '02x') for x in self.data['de_data'][40:])) - #if len(self.data['de_data']) != 40: + # print() + # print(len(self.data['de_data'])) + # import binascii + # print(binascii.hexlify(self.data['de_data'])) + # print(' '.join(format(x, '02x') for x in self.data['de_data'][:40])) + # print(' '.join(format(x, '02x') for x in self.data['de_data'][40:])) + # if len(self.data['de_data']) != 40: # print('\nerror: len(self.data[\'de_data\']) != 40') # exit() - #return self.data + # return self.data def print_data_keys(self): """ Print data keys for L0 data """ - print('\n\nL0 data keys:') + print("\n\nL0 data keys:") for key in self.data: print(key) diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/l0_data_histogram.py b/imap_l3_processing/glows/l3a/glows_toolkit/l0_data_histogram.py index a8863eb74..933046cea 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/l0_data_histogram.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/l0_data_histogram.py @@ -2,10 +2,12 @@ Author: Marek Strumik, maro at cbk.waw.pl """ -class L0DataHistogram(): + +class L0DataHistogram: """ L0DataHistogram class (based on HistogramPayload by KPLabs) for GLOWS-L0-histogram data """ + def __init__(self): """ Constructor for this class @@ -22,49 +24,50 @@ def deserialize(self, reader): reader: Reader-class object needed for reading binary data stream """ # start reading the data - self.data['first_spin_id_in_block'] = reader.read_uint32_be() - self.data['diff_spin_id_in_block'] = reader.read_uint16_be() - self.data['histogram_validity_flags'] = reader.read_uint16_be() - self.data['software_version'] = reader.read_uint24_be() # 11 + self.data["first_spin_id_in_block"] = reader.read_uint32_be() + self.data["diff_spin_id_in_block"] = reader.read_uint16_be() + self.data["histogram_validity_flags"] = reader.read_uint16_be() + self.data["software_version"] = reader.read_uint24_be() # 11 # IMAP time - self.data['imap_start_time_second'] = reader.read_uint32_be() - self.data['imap_start_time_subsecond'] = reader.read_uint24_be() - self.data['imap_diff_second'] = reader.read_uint16_be() - self.data['imap_diff_subsecond'] = reader.read_uint24_be() # 23 + self.data["imap_start_time_second"] = reader.read_uint32_be() + self.data["imap_start_time_subsecond"] = reader.read_uint24_be() + self.data["imap_diff_second"] = reader.read_uint16_be() + self.data["imap_diff_subsecond"] = reader.read_uint24_be() # 23 # GLOWS time - self.data['glows_start_time_second'] = reader.read_uint32_be() - self.data['glows_start_time_subsecond'] = reader.read_uint24_be() - self.data['glows_diff_second'] = reader.read_uint16_be() - self.data['glows_diff_subsecond'] = reader.read_uint24_be() # 35 + self.data["glows_start_time_second"] = reader.read_uint32_be() + self.data["glows_start_time_subsecond"] = reader.read_uint24_be() + self.data["glows_diff_second"] = reader.read_uint16_be() + self.data["glows_diff_subsecond"] = reader.read_uint24_be() # 35 # histogram parameters - self.data['number_of_spins_per_block'] = reader.read_uint8() + 1 - self.data['number_of_bins_per_histogram'] = reader.read_uint16_be() # 38 + self.data["number_of_spins_per_block"] = reader.read_uint8() + 1 + self.data["number_of_bins_per_histogram"] = reader.read_uint16_be() # 38 # ancillary parameters collected onboard - self.data['filter_temperature_average'] = reader.read_uint8() - self.data['filter_temperature_variance'] = reader.read_uint16_be() - self.data['hv_voltage_average'] = reader.read_uint16_be() - self.data['hv_voltage_variance'] = reader.read_uint32_be() - self.data['spin_period_average'] = reader.read_uint16_be() - self.data['spin_period_variance'] = reader.read_uint32_be() - self.data['pulse_length_average'] = reader.read_uint8() - self.data['pulse_length_variance'] = reader.read_uint16_be() # 56 + self.data["filter_temperature_average"] = reader.read_uint8() + self.data["filter_temperature_variance"] = reader.read_uint16_be() + self.data["hv_voltage_average"] = reader.read_uint16_be() + self.data["hv_voltage_variance"] = reader.read_uint32_be() + self.data["spin_period_average"] = reader.read_uint16_be() + self.data["spin_period_variance"] = reader.read_uint32_be() + self.data["pulse_length_average"] = reader.read_uint8() + self.data["pulse_length_variance"] = reader.read_uint16_be() # 56 # TMP_SOLUTION # in TLM_GLX_2023_06_22.xls a new field (number of events in histogram) was introduced # the line below needs to be commented out, if earlier versions of mock data are used - self.data['number_of_events'] = reader.read_uint32_be() + self.data["number_of_events"] = reader.read_uint32_be() # histogram array - self.data['histogram'] = bytearray(reader.read_buffer( - self.data['number_of_bins_per_histogram'])) + self.data["histogram"] = bytearray( + reader.read_buffer(self.data["number_of_bins_per_histogram"]) + ) # additional padding when odd number of bins? - if self.data['number_of_bins_per_histogram'] % 2 == 1: - aux = reader.read_uint8() + if self.data["number_of_bins_per_histogram"] % 2 == 1: + aux = reader.read_uint8() return self.data @@ -72,6 +75,6 @@ def print_data_keys(self): """ Print data keys for L0 data """ - print('\n\nL0 data keys:') + print("\n\nL0 data keys:") for key in self.data: print(key) diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/l1a_data_direct_events.py b/imap_l3_processing/glows/l3a/glows_toolkit/l1a_data_direct_events.py index bcb8da31d..c55f5ad36 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/l1a_data_direct_events.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/l1a_data_direct_events.py @@ -1,6 +1,7 @@ """@package docstring Author: Marek Strumik, maro at cbk.waw.pl """ + import json import copy import numpy as np @@ -8,7 +9,8 @@ from .funcs import time_sec_subsec_to_utc from .direct_event import DirectEvent -class L1aDataDirectEvents(): + +class L1aDataDirectEvents: """ L1aDataHistogram() class for GLOWS-L1a-histogram data """ @@ -32,153 +34,217 @@ def create_l1a_from_l0_data(self, l0_data, pkts_file_name: str, packet_number: i # this is supposed to carry information about errors to be passed up at return error_messages = [] - self.data['header'] = { - 'ground_software_version': VERSION, - 'pkts_file_name': pkts_file_name, + self.data["header"] = { + "ground_software_version": VERSION, + "pkts_file_name": pkts_file_name, # note: packet number is seq_count (per apid!) field in CCSDS header - 'seq_count_in_pkts_file': packet_number + "seq_count_in_pkts_file": packet_number, } - self.data['imap_start_time_seconds'] = l0_data['imap_start_time_second'] - self.data['number_of_de_packets'] = l0_data['number_of_de_packets'] - self.data['seq_num_of_de_packet'] = l0_data['seq_num_of_de_packet'] - data = l0_data['de_data'] # data buffer which may contain (data_every_second + DE) data or only DE data + self.data["imap_start_time_seconds"] = l0_data["imap_start_time_second"] + self.data["number_of_de_packets"] = l0_data["number_of_de_packets"] + self.data["seq_num_of_de_packet"] = l0_data["seq_num_of_de_packet"] + data = l0_data[ + "de_data" + ] # data buffer which may contain (data_every_second + DE) data or only DE data # we assume that multi-part DE packets have been merged together before execution of this method # in the merged packet self.data['seq_num_of_de_packet'] should be set to 0 - assert self.data['seq_num_of_de_packet'] == 0 + assert self.data["seq_num_of_de_packet"] == 0 # sanity check, we should have: # >= 51B = 40 (data_every_second) + 8 (first direct event uncompressed) + 3 (the smallest timedelta + pulse_length) # or # 40 B if there is only data_every_second structure and no direct events - #assert len(data) >= 51 or len(data) == 40 # commented out -> see conditions checked later + # assert len(data) >= 51 or len(data) == 40 # commented out -> see conditions checked later # read data_every_second structure # based on Jupyter notebook sent in an email by K. Ber on Sep 20, 2023 # corrected (missing fields filter_temperature and hv_voltage) by looking at # glows_appsw-master/tooling/science/models/direct_event_file_model.py:42 - self.data['data_every_second'] = {} - self.data['data_every_second']['imap_sclk_last_pps'] = int.from_bytes(data[0:4], "big") - self.data['data_every_second']['glows_sclk_last_pps'] = int.from_bytes(data[4:8], "big") - self.data['data_every_second']['glows_ssclk_last_pps'] = int.from_bytes(data[8:12], "big") - self.data['data_every_second']['imap_sclk_next_pps'] = int.from_bytes(data[12:16], "big") - self.data['data_every_second']['catbed_heater_active'] = False if int(data[16]) == 0 else True - self.data['data_every_second']['spin_period_valid'] = False if int(data[17]) == 0 else True - self.data['data_every_second']['spin_phase_at_next_pps_valid'] = False if int(data[18]) == 0 else True - self.data['data_every_second']['spin_period_source'] = False if int(data[19]) == 0 else True - self.data['data_every_second']['spin_period'] = int.from_bytes(data[20:22], "big") - self.data['data_every_second']['spin_phase_at_next_pps'] = int.from_bytes(data[22:24], "big") - self.data['data_every_second']['number_of_completed_spins'] = int.from_bytes(data[24:28], "big") - self.data['data_every_second']['filter_temperature'] = int.from_bytes(data[28:30], "big") - self.data['data_every_second']['hv_voltage'] = int.from_bytes(data[30:32], "big") - self.data['data_every_second']['glows_time_on_pps_valid'] = False if int(data[32]) == 0 else True - self.data['data_every_second']['time_status_valid'] = False if int(data[33]) == 0 else True - self.data['data_every_second']['housekeeping_valid'] = False if int(data[34]) == 0 else True - self.data['data_every_second']['is_pps_autogenerated'] = False if int(data[35]) == 0 else True - self.data['data_every_second']['hv_test_in_progress'] = False if int(data[36]) == 0 else True - self.data['data_every_second']['pulse_test_in_progress'] = False if int(data[37]) == 0 else True - self.data['data_every_second']['memory_error_detected'] = False if int(data[38]) == 0 else True + self.data["data_every_second"] = {} + self.data["data_every_second"]["imap_sclk_last_pps"] = int.from_bytes( + data[0:4], "big" + ) + self.data["data_every_second"]["glows_sclk_last_pps"] = int.from_bytes( + data[4:8], "big" + ) + self.data["data_every_second"]["glows_ssclk_last_pps"] = int.from_bytes( + data[8:12], "big" + ) + self.data["data_every_second"]["imap_sclk_next_pps"] = int.from_bytes( + data[12:16], "big" + ) + self.data["data_every_second"]["catbed_heater_active"] = ( + False if int(data[16]) == 0 else True + ) + self.data["data_every_second"]["spin_period_valid"] = ( + False if int(data[17]) == 0 else True + ) + self.data["data_every_second"]["spin_phase_at_next_pps_valid"] = ( + False if int(data[18]) == 0 else True + ) + self.data["data_every_second"]["spin_period_source"] = ( + False if int(data[19]) == 0 else True + ) + self.data["data_every_second"]["spin_period"] = int.from_bytes( + data[20:22], "big" + ) + self.data["data_every_second"]["spin_phase_at_next_pps"] = int.from_bytes( + data[22:24], "big" + ) + self.data["data_every_second"]["number_of_completed_spins"] = int.from_bytes( + data[24:28], "big" + ) + self.data["data_every_second"]["filter_temperature"] = int.from_bytes( + data[28:30], "big" + ) + self.data["data_every_second"]["hv_voltage"] = int.from_bytes( + data[30:32], "big" + ) + self.data["data_every_second"]["glows_time_on_pps_valid"] = ( + False if int(data[32]) == 0 else True + ) + self.data["data_every_second"]["time_status_valid"] = ( + False if int(data[33]) == 0 else True + ) + self.data["data_every_second"]["housekeeping_valid"] = ( + False if int(data[34]) == 0 else True + ) + self.data["data_every_second"]["is_pps_autogenerated"] = ( + False if int(data[35]) == 0 else True + ) + self.data["data_every_second"]["hv_test_in_progress"] = ( + False if int(data[36]) == 0 else True + ) + self.data["data_every_second"]["pulse_test_in_progress"] = ( + False if int(data[37]) == 0 else True + ) + self.data["data_every_second"]["memory_error_detected"] = ( + False if int(data[38]) == 0 else True + ) # NOTE: there is one-byte padding in the C structure of data_every_seconds, so it has 40 bytes in total # DEBUG TEST - #print(self.data) - #exit() - #print('\nMET', self.data['imap_start_time_seconds']) - #print('XXX', self.data['data_every_second']['imap_sclk_last_pps']) - #exit() + # print(self.data) + # exit() + # print('\nMET', self.data['imap_start_time_seconds']) + # print('XXX', self.data['data_every_second']['imap_sclk_last_pps']) + # exit() # sanity check - #assert self.data['imap_start_time_seconds'] == self.data['data_every_second']['imap_sclk_last_pps'], \ + # assert self.data['imap_start_time_seconds'] == self.data['data_every_second']['imap_sclk_last_pps'], \ # 'error create_l1a_from_l0_data(): imap_start_time_seconds=%d != imap_sclk_last_pps=%d' % \ # (self.data['imap_start_time_seconds'], self.data['data_every_second']['imap_sclk_last_pps']) # check if there are any direct events included in the current packet or it contains only data_every_second if len(data) == 40: - return error_messages + return error_messages # sanity check, the first direct event must be uncompressed 8-byte structure - assert len(data) >= 48, \ - 'error create_l1a_from_l0_data(): len(data)=%d < 48' % len(data) + assert len(data) >= 48, ( + "error create_l1a_from_l0_data(): len(data)=%d < 48" % len(data) + ) # read the first direct event, which is always uncompressed (i.e., not timedelta but full timestamp) current_event = DirectEvent.build_event_from_uncompressed_data(data[40:48]) self.direct_events.append(current_event) # sanity check, the first-direct-event sclk must be within +1 second from data_every_second sclk - #assert self.direct_events[0].seconds <= self.data['data_every_second']['glows_sclk_last_pps']+1, \ + # assert self.direct_events[0].seconds <= self.data['data_every_second']['glows_sclk_last_pps']+1, \ # 'error create_l1a_from_l0_data(): sclk %d %d' % (self.direct_events[0].seconds, self.data['data_every_second']['glows_sclk_last_pps']) # TMP SOLUTION: this if was included instead of the assert above, because in the tests performed in the PTB, some # packets had incorrect sclk in the first direct event included; when the bug is fixed by the GLOWS SW team this if # should be replaced by the assert above - if self.direct_events[0].seconds > self.data['data_every_second']['glows_sclk_last_pps']+1: - print('\n### data_every_second ###') - print(self.data['data_every_second']) - print('---') + if ( + self.direct_events[0].seconds + > self.data["data_every_second"]["glows_sclk_last_pps"] + 1 + ): + print("\n### data_every_second ###") + print(self.data["data_every_second"]) + print("---") print(self.direct_events) - print('###') - error_messages.append('DE: de[0].sec %d > glows_sclk_last_pps+1 %d' % (self.direct_events[0].seconds, self.data['data_every_second']['glows_sclk_last_pps']+1)) + print("###") + error_messages.append( + "DE: de[0].sec %d > glows_sclk_last_pps+1 %d" + % ( + self.direct_events[0].seconds, + self.data["data_every_second"]["glows_sclk_last_pps"] + 1, + ) + ) # sanity check, KPLabs says that multi event field is actually not processed by AppSW, so it is expected to be False # TMP_SOLUTION this assert was commented out but this needs to be discussed with GLOWS FSW team - #assert self.direct_events[0].multi_event != True, \ + # assert self.direct_events[0].multi_event != True, \ # 'error create_l1a_from_l0_data(): multi event occured' # check if there are any compressed direct events included in the current packet if len(data) <= 48: - return error_messages + return error_messages # pack the rest of the binary stream into a separate variable to be processed later binary_stream_of_timedeltas = data[48:] # a test snippet - #current_event = DirectEvent(54232337, 1997808, 6, False) - #binary_stream_of_timedeltas = bytearray.fromhex("033b8511061e7bf0") + bytearray.fromhex("876106") + bytearray.fromhex("E7620687") + bytearray.fromhex("00") + # current_event = DirectEvent(54232337, 1997808, 6, False) + # binary_stream_of_timedeltas = bytearray.fromhex("033b8511061e7bf0") + bytearray.fromhex("876106") + bytearray.fromhex("E7620687") + bytearray.fromhex("00") # convert direct events in the form of compressed timedeltas (in a binary stream) into DirectEvent objects # based on glows_appsw-master/tooling/science/models/direct_event_file_decompressor.py by KPLabs i = 0 - while i < len(binary_stream_of_timedeltas)-1: + while i < len(binary_stream_of_timedeltas) - 1: first_byte = int(binary_stream_of_timedeltas[i]) i += 1 oldest_diff = first_byte & 0x3F marker = first_byte >> 6 - if marker == 0x00 and i <= len(binary_stream_of_timedeltas)-7: # uncompressed time stamp (8-bytes) - rest_bytes = binary_stream_of_timedeltas[i:i+7] + if ( + marker == 0x00 and i <= len(binary_stream_of_timedeltas) - 7 + ): # uncompressed time stamp (8-bytes) + rest_bytes = binary_stream_of_timedeltas[i : i + 7] i += 7 part = bytearray([oldest_diff]) part.extend(rest_bytes) current_event = DirectEvent.build_event_from_uncompressed_data(part) - elif marker == 0x2 and i < len(binary_stream_of_timedeltas)-1: # 2-byte compression of timedelta + elif ( + marker == 0x2 and i < len(binary_stream_of_timedeltas) - 1 + ): # 2-byte compression of timedelta rest = int(binary_stream_of_timedeltas[i]) i += 1 diff = oldest_diff << 8 | rest length = int(binary_stream_of_timedeltas[i]) i += 1 - current_event = DirectEvent.build_event_from_compressed_data(diff, length, current_event) - - elif marker == 0x3 and i < len(binary_stream_of_timedeltas)-2: # 3-byte compression of timedelta - rest = int.from_bytes(binary_stream_of_timedeltas[i:i+2], "big") + current_event = DirectEvent.build_event_from_compressed_data( + diff, length, current_event + ) + + elif ( + marker == 0x3 and i < len(binary_stream_of_timedeltas) - 2 + ): # 3-byte compression of timedelta + rest = int.from_bytes(binary_stream_of_timedeltas[i : i + 2], "big") i += 2 diff = oldest_diff << 16 | rest length = int(binary_stream_of_timedeltas[i]) i += 1 - current_event = DirectEvent.build_event_from_compressed_data(diff, length, current_event) - - else: # wrong-marker or hitting-the-buffer-end case -# print() -# pass -# raise Exception('error create_l1a_from_l0_data(): unexpected marker %d or end of data stream %d %d' % \ -# (marker, i, len(binary_stream_of_timedeltas))) - error_messages.append('DE: direct event decompressing failed at byte %d' % (i-1)) + current_event = DirectEvent.build_event_from_compressed_data( + diff, length, current_event + ) + + else: # wrong-marker or hitting-the-buffer-end case + # print() + # pass + # raise Exception('error create_l1a_from_l0_data(): unexpected marker %d or end of data stream %d %d' % \ + # (marker, i, len(binary_stream_of_timedeltas))) + error_messages.append( + "DE: direct event decompressing failed at byte %d" % (i - 1) + ) break # sanity check if current_event.subseconds >= SUBSECOND_LIMIT: - error_messages.append('DE: de[0].subsec >= SUBSECOND_LIMIT') + error_messages.append("DE: de[0].subsec >= SUBSECOND_LIMIT") break self.direct_events.append(current_event) @@ -187,13 +253,16 @@ def create_l1a_from_l0_data(self, l0_data, pkts_file_name: str, packet_number: i def generate_file_name(self, extension): # file names should be tagged with utc time as suggested in Sec. 1.4.2 of # "IMAP SDC to Instrument Team ICD" - start_time_utc = time_sec_subsec_to_utc(self.data['imap_start_time_seconds'], 0 - ).strftime('%Y%m%d%H%M%S') + start_time_utc = time_sec_subsec_to_utc( + self.data["imap_start_time_seconds"], 0 + ).strftime("%Y%m%d%H%M%S") # the output file name is constructed using a convention defined in Sec. 1.4.2 of # "IMAP SDC to Instrument Team ICD" - file_name = 'data_l1a_direct_events/imap_glows_l1a_%s_orbX_modX_p_v00.%s' % \ - (start_time_utc, extension) + file_name = "data_l1a_direct_events/imap_glows_l1a_%s_orbX_modX_p_v00.%s" % ( + start_time_utc, + extension, + ) return file_name @@ -206,7 +275,7 @@ def read_from_file(self, file_name): # TMP_SOLUTION currently JSON and text format are used, it needs to be changed to CDF when python-cdf # modules for IMAP will be provided by SDC self.read_from_json_file(file_name) - self.read_direct_events(file_name.replace('.json', '.dat')) + self.read_direct_events(file_name.replace(".json", ".dat")) def read_from_json_file(self, file_name): """ @@ -214,11 +283,11 @@ def read_from_json_file(self, file_name): Args: file_name: name of file with L1a data """ - file_handler = open(file_name, 'r') + file_handler = open(file_name, "r") self.data = json.load(file_handler) # additional field to keep track where data come from - self.data['l1a_file_name'] = file_name + self.data["l1a_file_name"] = file_name file_handler.close() def read_direct_events(self, file_name): @@ -230,7 +299,7 @@ def read_direct_events(self, file_name): self.direct_events = np.loadtxt(file_name, dtype=int) # sanity check - #assert np.shape(self.direct_events)[1] == 3 + # assert np.shape(self.direct_events)[1] == 3 def save_data_to_file(self): """ @@ -248,20 +317,20 @@ def save_data_to_json_file(self): """ # generate JSON content as string from self.data dictionary data = copy.deepcopy(self.data) - del data['seq_num_of_de_packet'] + del data["seq_num_of_de_packet"] json_content = json.dumps(data, indent=4, default=vars) - file_name = self.generate_file_name('json') + file_name = self.generate_file_name("json") # sanity check # TMP_SOLUTION due to problems with FSW this needed to be commented out, but # it seems to be ok to check this when the problems will be fixed - #if os.path.exists(file_name) and self.data['number_of_de_packets'] == 1: + # if os.path.exists(file_name) and self.data['number_of_de_packets'] == 1: # print('L1a direct-event file already exists, while it should not', file_name) # print(self.data['number_of_de_packets'], self.data['seq_num_of_de_packet']) # exit() - file_handler = open(file_name, 'w') + file_handler = open(file_name, "w") print(json_content, file=file_handler) file_handler.close() @@ -271,8 +340,10 @@ def save_direct_events(self): Output file name is set automatically here (TBC if it should be method argument) """ - file_name = self.generate_file_name('dat') - file_handler = open(file_name, 'w') + file_name = self.generate_file_name("dat") + file_handler = open(file_name, "w") for i, x in enumerate(self.direct_events): - file_handler.write('%d %7d %3d\n' % (x.seconds, x.subseconds, x.impulse_length)) + file_handler.write( + "%d %7d %3d\n" % (x.seconds, x.subseconds, x.impulse_length) + ) file_handler.close() diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/l1a_data_histogram.py b/imap_l3_processing/glows/l3a/glows_toolkit/l1a_data_histogram.py index 8b016de2d..8474e8f12 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/l1a_data_histogram.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/l1a_data_histogram.py @@ -1,13 +1,15 @@ """@package docstring Author: Marek Strumik, maro at cbk.waw.pl """ + import json import numpy as np from .constants import VERSION, MAIN_TIME_FIELD from .time import Time from .funcs import time_sec_subsec_to_utc -class L1aDataHistogram(): + +class L1aDataHistogram: """ L1aDataHistogram() class for GLOWS-L1a-histogram data """ @@ -30,58 +32,69 @@ def create_l1a_from_l0_data(self, l0_data, pkts_file_name: str, packet_number: i # this is supposed to carry information about errors to be passed up at return error_messages = [] - self.data['first_spin_id_in_block'] = l0_data['first_spin_id_in_block'] - self.data['last_spin_id_in_block'] = self.data['first_spin_id_in_block'] + \ - l0_data['diff_spin_id_in_block'] + self.data["first_spin_id_in_block"] = l0_data["first_spin_id_in_block"] + self.data["last_spin_id_in_block"] = ( + self.data["first_spin_id_in_block"] + l0_data["diff_spin_id_in_block"] + ) # sanity check # TMP_SOLUTION: instrument emulator does not set l0_data['diff_spin_id_in_block'] properly # thus we temporarily comment out this assert - #assert l0_data['diff_spin_id_in_block'] == l0_data['number_of_spins_per_block'], \ + # assert l0_data['diff_spin_id_in_block'] == l0_data['number_of_spins_per_block'], \ # 'Inconsistency between spin-numbering field l0_data[\'diff_spin_id_in_block\'] and' + \ # 'histogram-parameter field l0_data[\'number_of_spins_per_block\']' - self.data['block_header'] = { - 'flight_software_version': l0_data['software_version'], - 'ground_software_version': VERSION, - 'pkts_file_name': pkts_file_name, + self.data["block_header"] = { + "flight_software_version": l0_data["software_version"], + "ground_software_version": VERSION, + "pkts_file_name": pkts_file_name, # note: packet number is seq_count (per apid!) field in CCSDS header - 'seq_count_in_pkts_file': packet_number + "seq_count_in_pkts_file": packet_number, } # times for L1a are kept in the same format as for L0 (i.e., using seconds and subseconds) - self.data['glows_start_time'] = Time(l0_data['glows_start_time_second'], - l0_data['glows_start_time_subsecond']) - self.data['glows_end_time_offset'] = Time(l0_data['glows_diff_second'], - l0_data['glows_diff_subsecond']) - self.data['imap_start_time'] = Time(l0_data['imap_start_time_second'], - l0_data['imap_start_time_subsecond']) - self.data['imap_end_time_offset'] = Time(l0_data['imap_diff_second'], - l0_data['imap_diff_subsecond']) - - self.data['number_of_spins_per_block'] = l0_data['number_of_spins_per_block'] - self.data['number_of_bins_per_histogram'] = l0_data['number_of_bins_per_histogram'] - self.data['histogram'] = l0_data['histogram'] + self.data["glows_start_time"] = Time( + l0_data["glows_start_time_second"], l0_data["glows_start_time_subsecond"] + ) + self.data["glows_end_time_offset"] = Time( + l0_data["glows_diff_second"], l0_data["glows_diff_subsecond"] + ) + self.data["imap_start_time"] = Time( + l0_data["imap_start_time_second"], l0_data["imap_start_time_subsecond"] + ) + self.data["imap_end_time_offset"] = Time( + l0_data["imap_diff_second"], l0_data["imap_diff_subsecond"] + ) + + self.data["number_of_spins_per_block"] = l0_data["number_of_spins_per_block"] + self.data["number_of_bins_per_histogram"] = l0_data[ + "number_of_bins_per_histogram" + ] + self.data["histogram"] = l0_data["histogram"] # TMP_SOLUTION - self.data['number_of_events'] = l0_data['number_of_events'] + self.data["number_of_events"] = l0_data["number_of_events"] # sanity check - if self.data['number_of_events'] != np.sum(self.data['histogram']): - error_messages.append('HIST: number of events is not equal to the sum over histogram %d %d' % \ - (self.data['number_of_events'], np.sum(self.data['histogram']))) - - self.data['filter_temperature_average'] = l0_data['filter_temperature_average'] - self.data['filter_temperature_variance'] = l0_data['filter_temperature_variance'] - self.data['hv_voltage_average'] = l0_data['hv_voltage_average'] - self.data['hv_voltage_variance'] = l0_data['hv_voltage_variance'] - self.data['spin_period_average'] = l0_data['spin_period_average'] - self.data['spin_period_variance'] = l0_data['spin_period_variance'] - self.data['pulse_length_average'] = l0_data['pulse_length_average'] - self.data['pulse_length_variance'] = l0_data['pulse_length_variance'] - - self.data['flags'] = {} - self.data['flags']['flags_set_onboard'] = l0_data['histogram_validity_flags'] - self.data['flags']['is_generated_on_ground'] = False + if self.data["number_of_events"] != np.sum(self.data["histogram"]): + error_messages.append( + "HIST: number of events is not equal to the sum over histogram %d %d" + % (self.data["number_of_events"], np.sum(self.data["histogram"])) + ) + + self.data["filter_temperature_average"] = l0_data["filter_temperature_average"] + self.data["filter_temperature_variance"] = l0_data[ + "filter_temperature_variance" + ] + self.data["hv_voltage_average"] = l0_data["hv_voltage_average"] + self.data["hv_voltage_variance"] = l0_data["hv_voltage_variance"] + self.data["spin_period_average"] = l0_data["spin_period_average"] + self.data["spin_period_variance"] = l0_data["spin_period_variance"] + self.data["pulse_length_average"] = l0_data["pulse_length_average"] + self.data["pulse_length_variance"] = l0_data["pulse_length_variance"] + + self.data["flags"] = {} + self.data["flags"]["flags_set_onboard"] = l0_data["histogram_validity_flags"] + self.data["flags"]["is_generated_on_ground"] = False return error_messages @@ -101,15 +114,16 @@ def read_from_json_file(self, file_name): Args: file_name: name of file with L1a data """ - file_handler = open(file_name, 'r') + file_handler = open(file_name, "r") self.data = json.load(file_handler) # hexadecimal values saved as strings require special treatment to make them integers - self.data['flags']['flags_set_onboard'] = int(self.data['flags']['flags_set_onboard'], - base=16) + self.data["flags"]["flags_set_onboard"] = int( + self.data["flags"]["flags_set_onboard"], base=16 + ) # additional field to keep track where data come from - self.data['l1a_file_name'] = file_name + self.data["l1a_file_name"] = file_name file_handler.close() def save_data_to_file(self): @@ -127,28 +141,29 @@ def save_data_to_json_file(self): """ # histogram needs to be temporarily converted to string, because we want to force that it # is saved as one line (i.e., without \n after each element) by json.dumps() - histogram_tmp = self.data['histogram'] - flag_tmp = self.data['flags']['flags_set_onboard'] - self.data['histogram'] = '[' + ', '.join(map(str, list(histogram_tmp))) + ']' - self.data['flags']['flags_set_onboard'] = '0x' + format(flag_tmp, 'x').zfill(4) + histogram_tmp = self.data["histogram"] + flag_tmp = self.data["flags"]["flags_set_onboard"] + self.data["histogram"] = "[" + ", ".join(map(str, list(histogram_tmp))) + "]" + self.data["flags"]["flags_set_onboard"] = "0x" + format(flag_tmp, "x").zfill(4) json_content = json.dumps(self.data, indent=4, default=vars) - self.data['flags']['flags_set_onboard'] = flag_tmp - self.data['histogram'] = histogram_tmp + self.data["flags"]["flags_set_onboard"] = flag_tmp + self.data["histogram"] = histogram_tmp # modify json_content to make histogram string be seen finally as a number array - json_content = json_content.replace('"[', '[').replace(']"', ']') + json_content = json_content.replace('"[', "[").replace(']"', "]") # file names should be tagged with utc time as suggested in Sec. 1.4.2 of # "IMAP SDC to Instrument Team ICD" - start_time_utc = time_sec_subsec_to_utc(self.data[MAIN_TIME_FIELD].seconds, - self.data[MAIN_TIME_FIELD].subseconds - ).strftime('%Y%m%d%H%M%S') + start_time_utc = time_sec_subsec_to_utc( + self.data[MAIN_TIME_FIELD].seconds, self.data[MAIN_TIME_FIELD].subseconds + ).strftime("%Y%m%d%H%M%S") # the output file name is constructed using a convention defined in Sec. 1.4.2 of # "IMAP SDC to Instrument Team ICD" - file_name = 'data_l1a_histograms/imap_glows_l1a_%s_orbX_modX_p_v00.json' % \ - (start_time_utc) + file_name = "data_l1a_histograms/imap_glows_l1a_%s_orbX_modX_p_v00.json" % ( + start_time_utc + ) - file_handler = open(file_name, 'w') + file_handler = open(file_name, "w") print(json_content, file=file_handler) file_handler.close() @@ -157,15 +172,22 @@ def total_counts_file_init(self, tag): Initialize a file for storing total count numbers for L1a histograms This kind of information is needed when L1b is generated (for one of the filters) """ - file_handler = open('data_l1a_histograms/total_counts_file.dat', 'w') - file_handler.write('%s %s\n' % (tag, self.data['block_header']['pkts_file_name'])) + file_handler = open("data_l1a_histograms/total_counts_file.dat", "w") + file_handler.write( + "%s %s\n" % (tag, self.data["block_header"]["pkts_file_name"]) + ) file_handler.close() def total_counts_file_update(self): """ Update a file for storing total counts numbers for L1a histograms """ - file_handler = open('data_l1a_histograms/total_counts_file.dat', 'a') - file_handler.write('%d %d\n' % (self.data['block_header']['seq_count_in_pkts_file'], - int(np.sum(self.data['histogram'])))) + file_handler = open("data_l1a_histograms/total_counts_file.dat", "a") + file_handler.write( + "%d %d\n" + % ( + self.data["block_header"]["seq_count_in_pkts_file"], + int(np.sum(self.data["histogram"])), + ) + ) file_handler.close() diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/l1b_data_direct_events.py b/imap_l3_processing/glows/l3a/glows_toolkit/l1b_data_direct_events.py index da392f90d..af633c004 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/l1b_data_direct_events.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/l1b_data_direct_events.py @@ -1,13 +1,19 @@ """@package docstring Author: Marek Strumik, maro at cbk.waw.pl """ + import json import copy import numpy as np from .constants import VERSION, SUBSECOND_LIMIT -from .funcs import decode_ancillary_parameter, time_sec_subsec_to_utc, time_sec_subsec_to_float64 +from .funcs import ( + decode_ancillary_parameter, + time_sec_subsec_to_utc, + time_sec_subsec_to_float64, +) + -class L1bDataDirectEvents(): +class L1bDataDirectEvents: """ L1bDataDirectEvents() class for GLOWS-L1b-direct-event data """ @@ -18,20 +24,23 @@ def __init__(self): """ self.data = {} - def _block_header_add_list_of_anc_files(self, anc_input_from_sdc, anc_input_from_instr_team): + def _block_header_add_list_of_anc_files( + self, anc_input_from_sdc, anc_input_from_instr_team + ): """ Add list of used ancillary files to block header Args: anc_input_from_sdc: ancillary input provided by IMAP SDC (general telemetry) anc_input_from_instr_team: ancillary input provided by IMAP/GLOWS Instrument Team """ - self.data['header']['ancillary_data_files'] = [] - for key in ['conversion_table_for_anc_data']: - self.data['header']['ancillary_data_files'].append(anc_input_from_instr_team[key]) - #for key in anc_input_from_sdc: + self.data["header"]["ancillary_data_files"] = [] + for key in ["conversion_table_for_anc_data"]: + self.data["header"]["ancillary_data_files"].append( + anc_input_from_instr_team[key] + ) + # for key in anc_input_from_sdc: # self.data['block_header']['ancillary_data_files'].append(anc_input_from_sdc[key]) - def _convert_ancillary_parameters(self, l1a_data, anc_input_from_instr_team): """ Convert ancillary parameters collected onboard from integer values to physcial @@ -42,21 +51,29 @@ def _convert_ancillary_parameters(self, l1a_data, anc_input_from_instr_team): """ # load conversion/decoding table for ancillary parameters collected onboard - file_handler = open(anc_input_from_instr_team['conversion_table_for_anc_data'], 'r') + file_handler = open( + anc_input_from_instr_team["conversion_table_for_anc_data"], "r" + ) conversion_table = json.load(file_handler) file_handler.close() # ancillary parameters collected onboard that need to be decoded - self.data['filter_temperature'] = decode_ancillary_parameter('filter_temperature', - conversion_table, l1a_data['filter_temperature']) - self.data['hv_voltage'] = decode_ancillary_parameter('hv_voltage', - conversion_table, l1a_data['hv_voltage']) - self.data['spin_period'] = decode_ancillary_parameter('spin_period', - conversion_table, l1a_data['spin_period']) - self.data['spin_phase_at_next_pps'] = decode_ancillary_parameter('spin_phase', - conversion_table, l1a_data['spin_phase_at_next_pps']) - - def create_l1b_from_l1a_data(self, l1a, anc_input_from_sdc, anc_input_from_instr_team): + self.data["filter_temperature"] = decode_ancillary_parameter( + "filter_temperature", conversion_table, l1a_data["filter_temperature"] + ) + self.data["hv_voltage"] = decode_ancillary_parameter( + "hv_voltage", conversion_table, l1a_data["hv_voltage"] + ) + self.data["spin_period"] = decode_ancillary_parameter( + "spin_period", conversion_table, l1a_data["spin_period"] + ) + self.data["spin_phase_at_next_pps"] = decode_ancillary_parameter( + "spin_phase", conversion_table, l1a_data["spin_phase_at_next_pps"] + ) + + def create_l1b_from_l1a_data( + self, l1a, anc_input_from_sdc, anc_input_from_instr_team + ): """ Sets values of L1b fields using L1a data read from a file Args: @@ -64,71 +81,95 @@ def create_l1b_from_l1a_data(self, l1a, anc_input_from_sdc, anc_input_from_instr anc_input_from_sdc: ancillary input provided by IMAP SDC (general telemetry) anc_input_from_instr_team: ancillary input provided by IMAP/GLOWS Instrument Team """ - self.data['header'] = { - 'ground_software_version': VERSION, - 'pkts_file_name': l1a.data['header']['pkts_file_name'], - 'seq_count_in_pkts_file': l1a.data['header']['seq_count_in_pkts_file'], - 'l1a_file_name': l1a.data['l1a_file_name'] + self.data["header"] = { + "ground_software_version": VERSION, + "pkts_file_name": l1a.data["header"]["pkts_file_name"], + "seq_count_in_pkts_file": l1a.data["header"]["seq_count_in_pkts_file"], + "l1a_file_name": l1a.data["l1a_file_name"], } # add list of used ancillary files to block_header - self._block_header_add_list_of_anc_files(anc_input_from_sdc, anc_input_from_instr_team) + self._block_header_add_list_of_anc_files( + anc_input_from_sdc, anc_input_from_instr_team + ) - #self.data['unique_identifier'] = \ + # self.data['unique_identifier'] = \ # time_sec_subsec_to_utc(l1a.data['data_every_second']['imap_sclk_last_pps'], # 0).strftime('%Y-%m-%dT%H:%M:%S') # TMP_SOLUTION: for now we use imap_start_time_seconds for unique_identifier # becasue FSW has a bug that leads to a discrepancy between ['imap_start_time_seconds'] # and ['data_every_second']['imap_sclk_last_pps'] - self.data['unique_identifier'] = \ - time_sec_subsec_to_utc(l1a.data['imap_start_time_seconds'], - 0).strftime('%Y-%m-%dT%H:%M:%S') + self.data["unique_identifier"] = time_sec_subsec_to_utc( + l1a.data["imap_start_time_seconds"], 0 + ).strftime("%Y-%m-%dT%H:%M:%S") # set IMAP and GLOWS time fields self.set_times(l1a.data) # set number of completed spins - self.data['number_of_completed_spins'] = \ - l1a.data['data_every_second']['number_of_completed_spins'] + self.data["number_of_completed_spins"] = l1a.data["data_every_second"][ + "number_of_completed_spins" + ] # convert ancillary parameters collected onboard to physical units - self._convert_ancillary_parameters(l1a.data['data_every_second'], anc_input_from_instr_team) + self._convert_ancillary_parameters( + l1a.data["data_every_second"], anc_input_from_instr_team + ) # add flags set onboard to self.data['flags'] - self.data['flags'] = {} - for key in ['catbed_heater_active', 'spin_period_valid', 'spin_phase_at_next_pps_valid', - 'spin_period_source', 'glows_time_on_pps_valid', 'time_status_valid', - 'housekeeping_valid', 'is_pps_autogenerated', 'hv_test_in_progress', - 'pulse_test_in_progress', 'memory_error_detected']: - if key[:3] != 'is_': - self.data['flags']['is_' + key] = l1a.data['data_every_second'][key] + self.data["flags"] = {} + for key in [ + "catbed_heater_active", + "spin_period_valid", + "spin_phase_at_next_pps_valid", + "spin_period_source", + "glows_time_on_pps_valid", + "time_status_valid", + "housekeeping_valid", + "is_pps_autogenerated", + "hv_test_in_progress", + "pulse_test_in_progress", + "memory_error_detected", + ]: + if key[:3] != "is_": + self.data["flags"]["is_" + key] = l1a.data["data_every_second"][key] else: - self.data['flags'][key] = l1a.data['data_every_second'][key] + self.data["flags"][key] = l1a.data["data_every_second"][key] if np.shape(l1a.direct_events)[0] > 0: - if l1a.direct_events.ndim > 1: # this is supposed to check if we have more than one DE - self.data['direct_event_glows_times'] = (l1a.direct_events[:, 0] + \ - l1a.direct_events[:, 1]/SUBSECOND_LIMIT).astype(float) + if ( + l1a.direct_events.ndim > 1 + ): # this is supposed to check if we have more than one DE + self.data["direct_event_glows_times"] = ( + l1a.direct_events[:, 0] + l1a.direct_events[:, 1] / SUBSECOND_LIMIT + ).astype(float) else: - self.data['direct_event_glows_times'] = (l1a.direct_events[0] + \ - l1a.direct_events[1]/SUBSECOND_LIMIT).astype(float) + self.data["direct_event_glows_times"] = ( + l1a.direct_events[0] + l1a.direct_events[1] / SUBSECOND_LIMIT + ).astype(float) # load conversion/decoding table for ancillary parameters collected onboard - file_handler = open(anc_input_from_instr_team['conversion_table_for_anc_data'], 'r') + file_handler = open( + anc_input_from_instr_team["conversion_table_for_anc_data"], "r" + ) conversion_table = json.load(file_handler) file_handler.close() # convert pulse lengths into physical units - if l1a.direct_events.ndim > 1: # this is supposed to check if we have more than one DE - self.data['direct_event_pulse_lengths'] = decode_ancillary_parameter('pulse_length', - conversion_table, l1a.direct_events[:, 2]) + if ( + l1a.direct_events.ndim > 1 + ): # this is supposed to check if we have more than one DE + self.data["direct_event_pulse_lengths"] = decode_ancillary_parameter( + "pulse_length", conversion_table, l1a.direct_events[:, 2] + ) else: - self.data['direct_event_pulse_lengths'] = decode_ancillary_parameter('pulse_length', - conversion_table, l1a.direct_events[2]) + self.data["direct_event_pulse_lengths"] = decode_ancillary_parameter( + "pulse_length", conversion_table, l1a.direct_events[2] + ) else: - self.data['direct_event_glows_times'] = np.array([]) - self.data['direct_event_pulse_lengths'] = np.array([]) + self.data["direct_event_glows_times"] = np.array([]) + self.data["direct_event_pulse_lengths"] = np.array([]) def save_data_to_file(self): """ @@ -145,49 +186,56 @@ def _save_data_to_json_file(self): """ # define formats for printing values to JSON file fmt = { - 'arrays': { - 'direct_event_glows_times': '%.7f', - 'direct_event_pulse_lengths': '%d', + "arrays": { + "direct_event_glows_times": "%.7f", + "direct_event_pulse_lengths": "%d", }, - 'scalar variables': { - 'glows_time_last_pps': '%.7f', - 'imap_time_last_pps': '%.7f', - 'imap_time_next_pps': '%.7f', - 'filter_temperature': '%.2f', - 'hv_voltage': '%.1f', - 'spin_period': '%.7f', - 'spin_phase_at_next_pps': '%.7f', + "scalar variables": { + "glows_time_last_pps": "%.7f", + "imap_time_last_pps": "%.7f", + "imap_time_next_pps": "%.7f", + "filter_temperature": "%.2f", + "hv_voltage": "%.1f", + "spin_period": "%.7f", + "spin_phase_at_next_pps": "%.7f", }, } # create local copy for printing purposes data = copy.deepcopy(self.data) - for key in fmt['arrays'].keys(): - if type(data[key]) == np.ndarray: # check if we have more than one direct event - data[key] = '[' + ', '.join(map(lambda x: fmt['arrays'][key] % x, list(data[key]))) + ']' + for key in fmt["arrays"].keys(): + if ( + type(data[key]) == np.ndarray + ): # check if we have more than one direct event + data[key] = ( + "[" + + ", ".join(map(lambda x: fmt["arrays"][key] % x, list(data[key]))) + + "]" + ) else: - data[key] = '[' + fmt['arrays'][key] % data[key] + ']' - for key in fmt['scalar variables'].keys(): - data[key] = '|'+ fmt['scalar variables'][key] % data[key] +'|' + data[key] = "[" + fmt["arrays"][key] % data[key] + "]" + for key in fmt["scalar variables"].keys(): + data[key] = "|" + fmt["scalar variables"][key] % data[key] + "|" # generate JSON content to be saved in output file json_content = json.dumps(data, indent=4, default=vars) # some corrections to get numbers back - json_content = json_content.replace('"[', '[').replace(']"', ']') - json_content = json_content.replace('|"', '').replace('"|', '') - #json_content = json_content.replace('\\"', '"') + json_content = json_content.replace('"[', "[").replace(']"', "]") + json_content = json_content.replace('|"', "").replace('"|', "") + # json_content = json_content.replace('\\"', '"') # start_time_utc string for output file name, based on unique_block_identifier - start_time_utc = ''.join(filter(str.isdigit, self.data['unique_identifier'])) + start_time_utc = "".join(filter(str.isdigit, self.data["unique_identifier"])) # the output file name is constructed using a convention defined in Sec. 1.4.2 of # "IMAP SDC to Instrument Team ICD" - file_name = 'data_l1b_direct_events/imap_glows_l1b_%s_orbX_modX_p_v00.json' % \ - (start_time_utc) + file_name = "data_l1b_direct_events/imap_glows_l1b_%s_orbX_modX_p_v00.json" % ( + start_time_utc + ) - file_handler = open(file_name, 'w') + file_handler = open(file_name, "w") print(json_content, file=file_handler) file_handler.close() @@ -197,10 +245,13 @@ def set_times(self, l1a_data): Args: l1a_data: L1a data provided in the form of data field of L1aDataDirectEvents object """ - self.data['imap_time_last_pps'] = \ - time_sec_subsec_to_float64(l1a_data['data_every_second']['imap_sclk_last_pps'], 0) - self.data['imap_time_next_pps'] = \ - time_sec_subsec_to_float64(l1a_data['data_every_second']['imap_sclk_next_pps'], 0) - self.data['glows_time_last_pps'] = \ - time_sec_subsec_to_float64(l1a_data['data_every_second']['glows_sclk_last_pps'], - l1a_data['data_every_second']['glows_ssclk_last_pps']) + self.data["imap_time_last_pps"] = time_sec_subsec_to_float64( + l1a_data["data_every_second"]["imap_sclk_last_pps"], 0 + ) + self.data["imap_time_next_pps"] = time_sec_subsec_to_float64( + l1a_data["data_every_second"]["imap_sclk_next_pps"], 0 + ) + self.data["glows_time_last_pps"] = time_sec_subsec_to_float64( + l1a_data["data_every_second"]["glows_sclk_last_pps"], + l1a_data["data_every_second"]["glows_ssclk_last_pps"], + ) diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/l1b_data_histogram.py b/imap_l3_processing/glows/l3a/glows_toolkit/l1b_data_histogram.py index b6443198a..a14ed10fa 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/l1b_data_histogram.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/l1b_data_histogram.py @@ -1,18 +1,30 @@ """@package docstring Author: Marek Strumik, maro at cbk.waw.pl """ + import json import copy import numpy as np import pandas as pd from astropy.coordinates import SkyCoord from astropy import units as u -from .constants import VERSION, SCAN_CIRCLE_ANGULAR_RADIUS, L1B_L2_HIST_FLAG_FIELDS, MAIN_TIME_FIELD -from .funcs import time_sec_subsec_to_utc, time_sec_subsec_to_float64, flags_deserialize, \ - decode_ancillary_parameters_avg_stddev, check_if_contains_actual_data +from .constants import ( + VERSION, + SCAN_CIRCLE_ANGULAR_RADIUS, + L1B_L2_HIST_FLAG_FIELDS, + MAIN_TIME_FIELD, +) +from .funcs import ( + time_sec_subsec_to_utc, + time_sec_subsec_to_float64, + flags_deserialize, + decode_ancillary_parameters_avg_stddev, + check_if_contains_actual_data, +) from .geom_fun_libr_box import spin2lb -class L1bDataHistogram(): + +class L1bDataHistogram: """ L1bDataHistogram() class for GLOWS-L1b-histogram data """ @@ -27,18 +39,24 @@ def __init__(self): self.block_start_time = None self.block_end_time = None - def _block_header_add_list_of_anc_files(self, anc_input_from_sdc, anc_input_from_instr_team): + def _block_header_add_list_of_anc_files( + self, anc_input_from_sdc, anc_input_from_instr_team + ): """ Add list of used ancillary files to block header Args: anc_input_from_sdc: ancillary input provided by IMAP SDC (general telemetry) anc_input_from_instr_team: ancillary input provided by IMAP/GLOWS Instrument Team """ - self.data['block_header']['ancillary_data_files'] = [] + self.data["block_header"]["ancillary_data_files"] = [] for key in anc_input_from_instr_team: - self.data['block_header']['ancillary_data_files'].append(anc_input_from_instr_team[key]) + self.data["block_header"]["ancillary_data_files"].append( + anc_input_from_instr_team[key] + ) for key in anc_input_from_sdc: - self.data['block_header']['ancillary_data_files'].append(anc_input_from_sdc[key]) + self.data["block_header"]["ancillary_data_files"].append( + anc_input_from_sdc[key] + ) def _compute_spin_period_ground_estimate(self, anc_data): """ @@ -50,13 +68,16 @@ def _compute_spin_period_ground_estimate(self, anc_data): anc_data: Pandas Dataframe with ancillary data from CSV file """ # extract data frames: last before block start and first after block end - df_1 = anc_data.loc[anc_data['imap_time'] <= self.block_start_time].iloc[-1, :] - df_2 = anc_data.loc[anc_data['imap_time'] >= self.block_end_time].iloc[0, :] + df_1 = anc_data.loc[anc_data["imap_time"] <= self.block_start_time].iloc[-1, :] + df_2 = anc_data.loc[anc_data["imap_time"] >= self.block_end_time].iloc[0, :] # generate synthetic series with spin-period data (assumed to arrive every 0.1 s) time = np.arange(self.block_start_time, self.block_end_time, 0.1) - data = np.interp(time, [df_1['imap_time'], df_2['imap_time']], - [df_1['spin_period'], df_2['spin_period']]) + data = np.interp( + time, + [df_1["imap_time"], df_2["imap_time"]], + [df_1["spin_period"], df_2["spin_period"]], + ) # compute spin-period average and standard deviation over block average = np.mean(data) @@ -73,13 +94,16 @@ def _compute_position_angle(self, anc_data): anc_data: Pandas Dataframe with ancillary data from CSV file """ # extract data frames: last before block start and first after block end - df_1 = anc_data.loc[anc_data['imap_time'] <= self.block_start_time].iloc[-1, :] - df_2 = anc_data.loc[anc_data['imap_time'] >= self.block_end_time].iloc[0, :] + df_1 = anc_data.loc[anc_data["imap_time"] <= self.block_start_time].iloc[-1, :] + df_2 = anc_data.loc[anc_data["imap_time"] >= self.block_end_time].iloc[0, :] # generate synthetic series with position-angle data (assumed to arrive every 0.1 s) time = np.arange(self.block_start_time, self.block_end_time, 0.1) - data = np.interp(time, [df_1['imap_time'], df_2['imap_time']], - [df_1['position_angle'], df_2['position_angle']]) + data = np.interp( + time, + [df_1["imap_time"], df_2["imap_time"]], + [df_1["position_angle"], df_2["position_angle"]], + ) # compute position-angle average and standard deviation over block average = np.mean(data) @@ -96,23 +120,31 @@ def _compute_spin_axis_orientation(self, anc_data): anc_data: Pandas Dataframe with ancillary data from CSV file """ # extract data frames: last before block start and first after block end - df_1 = anc_data.loc[anc_data['imap_time'] <= self.block_start_time].iloc[-1, :] - df_2 = anc_data.loc[anc_data['imap_time'] >= self.block_end_time].iloc[0, :] + df_1 = anc_data.loc[anc_data["imap_time"] <= self.block_start_time].iloc[-1, :] + df_2 = anc_data.loc[anc_data["imap_time"] >= self.block_end_time].iloc[0, :] # generate synthetic series with spin-axis-orientation data (assumed to arrive every 0.1 s) time = np.arange(self.block_start_time, self.block_end_time, 0.1) - data_lon = np.interp(time, [df_1['imap_time'], df_2['imap_time']], - [df_1['spin_ax_lon'], df_2['spin_ax_lon']]) - data_lat = np.interp(time, [df_1['imap_time'], df_2['imap_time']], - [df_1['spin_ax_lat'], df_2['spin_ax_lat']]) + data_lon = np.interp( + time, + [df_1["imap_time"], df_2["imap_time"]], + [df_1["spin_ax_lon"], df_2["spin_ax_lon"]], + ) + data_lat = np.interp( + time, + [df_1["imap_time"], df_2["imap_time"]], + [df_1["spin_ax_lat"], df_2["spin_ax_lat"]], + ) # compute spin-axis-orientation average and standard deviation over block average_lon = np.mean(data_lon) std_deviation_lon = np.std(data_lon) average_lat = np.mean(data_lat) std_deviation_lat = np.std(data_lat) - return {'lon': average_lon, 'lat': average_lat}, \ - {'lon': std_deviation_lon, 'lat': std_deviation_lat} + return {"lon": average_lon, "lat": average_lat}, { + "lon": std_deviation_lon, + "lat": std_deviation_lat, + } def _compute_spacecraft_location(self, anc_data): """ @@ -124,20 +156,29 @@ def _compute_spacecraft_location(self, anc_data): anc_data: Pandas Dataframe with ancillary data from CSV file """ # extract data frames: last before block start and first after block end - df_1 = anc_data.loc[anc_data['imap_time'] <= self.block_start_time].iloc[-1, :] - df_2 = anc_data.loc[anc_data['imap_time'] >= self.block_end_time].iloc[0, :] + df_1 = anc_data.loc[anc_data["imap_time"] <= self.block_start_time].iloc[-1, :] + df_2 = anc_data.loc[anc_data["imap_time"] >= self.block_end_time].iloc[0, :] # generate synthetic series with spacecraft location data (assumed to arrive every 0.1 s) time = np.arange(self.block_start_time, self.block_end_time, 0.1) - data_x = np.interp(time, [df_1['imap_time'], df_2['imap_time']], [df_1['x'], df_2['x']]) - data_y = np.interp(time, [df_1['imap_time'], df_2['imap_time']], [df_1['y'], df_2['y']]) - data_z = np.interp(time, [df_1['imap_time'], df_2['imap_time']], [df_1['z'], df_2['z']]) + data_x = np.interp( + time, [df_1["imap_time"], df_2["imap_time"]], [df_1["x"], df_2["x"]] + ) + data_y = np.interp( + time, [df_1["imap_time"], df_2["imap_time"]], [df_1["y"], df_2["y"]] + ) + data_z = np.interp( + time, [df_1["imap_time"], df_2["imap_time"]], [df_1["z"], df_2["z"]] + ) # compute spacecraft-location average and standard deviation over block average = [np.mean(data_x), np.mean(data_y), np.mean(data_z)] std_deviation = [np.std(data_x), np.std(data_y), np.std(data_z)] - return {'x': average[0], 'y': average[1], 'z': average[2]}, \ - {'x': std_deviation[0], 'y': std_deviation[1], 'z': std_deviation[2]} + return {"x": average[0], "y": average[1], "z": average[2]}, { + "x": std_deviation[0], + "y": std_deviation[1], + "z": std_deviation[2], + } def _compute_spacecraft_velocity(self, anc_data): """ @@ -149,41 +190,56 @@ def _compute_spacecraft_velocity(self, anc_data): anc_data: Pandas Dataframe with ancillary data from CSV file """ # extract data frames: last before block start and first after block end - df_1 = anc_data.loc[anc_data['imap_time'] <= self.block_start_time].iloc[-1, :] - df_2 = anc_data.loc[anc_data['imap_time'] >= self.block_end_time].iloc[0, :] + df_1 = anc_data.loc[anc_data["imap_time"] <= self.block_start_time].iloc[-1, :] + df_2 = anc_data.loc[anc_data["imap_time"] >= self.block_end_time].iloc[0, :] # generate synthetic series with spacecraft velocity data (assumed to arrive every 0.1 s) time = np.arange(self.block_start_time, self.block_end_time, 0.1) - data_vx = np.interp(time, [df_1['imap_time'], df_2['imap_time']], [df_1['vx'], df_2['vx']]) - data_vy = np.interp(time, [df_1['imap_time'], df_2['imap_time']], [df_1['vy'], df_2['vy']]) - data_vz = np.interp(time, [df_1['imap_time'], df_2['imap_time']], [df_1['vz'], df_2['vz']]) + data_vx = np.interp( + time, [df_1["imap_time"], df_2["imap_time"]], [df_1["vx"], df_2["vx"]] + ) + data_vy = np.interp( + time, [df_1["imap_time"], df_2["imap_time"]], [df_1["vy"], df_2["vy"]] + ) + data_vz = np.interp( + time, [df_1["imap_time"], df_2["imap_time"]], [df_1["vz"], df_2["vz"]] + ) # compute spacecraft-velocity average and standard deviation over block average = [np.mean(data_vx), np.mean(data_vy), np.mean(data_vz)] std_deviation = [np.std(data_vx), np.std(data_vy), np.std(data_vz)] - return {'x': average[0], 'y': average[1], 'z': average[2]}, \ - {'x': std_deviation[0], 'y': std_deviation[1], 'z': std_deviation[2]} + return {"x": average[0], "y": average[1], "z": average[2]}, { + "x": std_deviation[0], + "y": std_deviation[1], + "z": std_deviation[2], + } def _compute_histogram_bins_ecl_coords(self): """ Compute ecliptic longitude and latitude for histogram bin centers """ - n_bins = self.data['number_of_bins_per_histogram'] # just for convenience + n_bins = self.data["number_of_bins_per_histogram"] # just for convenience # generate spin angle values (for histogram bin centers) measured from the northernmost # point of the scanning circle - spin_angle_from_north = (self.data['imap_spin_angle_bin_cntr'] - \ - self.data['position_angle_offset_average'] + \ - 360.0) % 360.0 + spin_angle_from_north = ( + self.data["imap_spin_angle_bin_cntr"] + - self.data["position_angle_offset_average"] + + 360.0 + ) % 360.0 # compute ecliptic longitude and latitude of histogram bin centers histogram_bins_ecl_coords = spin2lb( - np.radians(self.data['spin_axis_orientation_average']['lon'])*np.ones(n_bins, float), - np.radians(self.data['spin_axis_orientation_average']['lat'])*np.ones(n_bins, float), - np.radians(spin_angle_from_north), 0.0, - np.radians(SCAN_CIRCLE_ANGULAR_RADIUS)*np.ones(n_bins, float)) + np.radians(self.data["spin_axis_orientation_average"]["lon"]) + * np.ones(n_bins, float), + np.radians(self.data["spin_axis_orientation_average"]["lat"]) + * np.ones(n_bins, float), + np.radians(spin_angle_from_north), + 0.0, + np.radians(SCAN_CIRCLE_ANGULAR_RADIUS) * np.ones(n_bins, float), + ) - return np.degrees(histogram_bins_ecl_coords) # in deg units + return np.degrees(histogram_bins_ecl_coords) # in deg units def _convert_ancillary_parameters(self, l1a_data, anc_input_from_instr_team): """ @@ -195,21 +251,38 @@ def _convert_ancillary_parameters(self, l1a_data, anc_input_from_instr_team): """ # load conversion/decoding table for ancillary parameters collected onboard - file_handler = open(anc_input_from_instr_team['conversion_table_for_anc_data'], 'r') + file_handler = open( + anc_input_from_instr_team["conversion_table_for_anc_data"], "r" + ) conversion_table = json.load(file_handler) file_handler.close() # ancillary parameters collected onboard that need to be decoded - self.data['filter_temperature_average'], self.data['filter_temperature_std_dev'] = \ - decode_ancillary_parameters_avg_stddev('filter_temperature', conversion_table, l1a_data) - self.data['hv_voltage_average'], self.data['hv_voltage_std_dev'] = \ - decode_ancillary_parameters_avg_stddev('hv_voltage', conversion_table, l1a_data) - self.data['spin_period_average'], self.data['spin_period_std_dev'] = \ - decode_ancillary_parameters_avg_stddev('spin_period', conversion_table, l1a_data) - self.data['pulse_length_average'], self.data['pulse_length_std_dev'] = \ - decode_ancillary_parameters_avg_stddev('pulse_length', conversion_table, l1a_data) - - def create_l1b_from_l1a_data(self, l1a_data, anc_input_from_sdc, anc_input_from_instr_team): + ( + self.data["filter_temperature_average"], + self.data["filter_temperature_std_dev"], + ) = decode_ancillary_parameters_avg_stddev( + "filter_temperature", conversion_table, l1a_data + ) + self.data["hv_voltage_average"], self.data["hv_voltage_std_dev"] = ( + decode_ancillary_parameters_avg_stddev( + "hv_voltage", conversion_table, l1a_data + ) + ) + self.data["spin_period_average"], self.data["spin_period_std_dev"] = ( + decode_ancillary_parameters_avg_stddev( + "spin_period", conversion_table, l1a_data + ) + ) + self.data["pulse_length_average"], self.data["pulse_length_std_dev"] = ( + decode_ancillary_parameters_avg_stddev( + "pulse_length", conversion_table, l1a_data + ) + ) + + def create_l1b_from_l1a_data( + self, l1a_data, anc_input_from_sdc, anc_input_from_instr_team + ): """ Sets values of L1b fields using L1a data read from a file Args: @@ -217,51 +290,66 @@ def create_l1b_from_l1a_data(self, l1a_data, anc_input_from_sdc, anc_input_from_ anc_input_from_sdc: ancillary input provided by IMAP SDC (general telemetry) anc_input_from_instr_team: ancillary input provided by IMAP/GLOWS Instrument Team """ - self.data['block_header'] = { - 'flight_software_version': l1a_data['block_header']['flight_software_version'], - 'ground_software_version': VERSION, - 'pkts_file_name': l1a_data['block_header']['pkts_file_name'], - 'seq_count_in_pkts_file': l1a_data['block_header']['seq_count_in_pkts_file'], - 'l1a_file_name': l1a_data['l1a_file_name'] + self.data["block_header"] = { + "flight_software_version": l1a_data["block_header"][ + "flight_software_version" + ], + "ground_software_version": VERSION, + "pkts_file_name": l1a_data["block_header"]["pkts_file_name"], + "seq_count_in_pkts_file": l1a_data["block_header"][ + "seq_count_in_pkts_file" + ], + "l1a_file_name": l1a_data["l1a_file_name"], } # add list of used ancillary files to block_header - self._block_header_add_list_of_anc_files(anc_input_from_sdc, anc_input_from_instr_team) + self._block_header_add_list_of_anc_files( + anc_input_from_sdc, anc_input_from_instr_team + ) - self.data['unique_block_identifier'] = \ - time_sec_subsec_to_utc(l1a_data[MAIN_TIME_FIELD]['seconds'], - l1a_data[MAIN_TIME_FIELD]['subseconds'] - ).strftime('%Y-%m-%dT%H:%M:%S') # TBC: cmp with l1a identifier + self.data["unique_block_identifier"] = time_sec_subsec_to_utc( + l1a_data[MAIN_TIME_FIELD]["seconds"], + l1a_data[MAIN_TIME_FIELD]["subseconds"], + ).strftime("%Y-%m-%dT%H:%M:%S") # TBC: cmp with l1a identifier # set IMAP and GLOWS time fields self.set_times(l1a_data) # set histogram-related quantities - self.data['number_of_spins_per_block'] = l1a_data['number_of_spins_per_block'] - self.data['number_of_bins_per_histogram'] = l1a_data['number_of_bins_per_histogram'] + self.data["number_of_spins_per_block"] = l1a_data["number_of_spins_per_block"] + self.data["number_of_bins_per_histogram"] = l1a_data[ + "number_of_bins_per_histogram" + ] # set histogram - self.data['histogram'] = l1a_data['histogram'] - self.data['number_of_events'] = l1a_data['number_of_events'] + self.data["histogram"] = l1a_data["histogram"] + self.data["number_of_events"] = l1a_data["number_of_events"] # set spin angle values for bin centers - spin_offset_first_bin = 0.5*360.0/self.data['number_of_bins_per_histogram'] - self.data['imap_spin_angle_bin_cntr'] = np.array( \ - np.array(range(self.data['number_of_bins_per_histogram']))*360.0/ \ - self.data['number_of_bins_per_histogram'] + spin_offset_first_bin).tolist() + spin_offset_first_bin = 0.5 * 360.0 / self.data["number_of_bins_per_histogram"] + self.data["imap_spin_angle_bin_cntr"] = np.array( + np.array(range(self.data["number_of_bins_per_histogram"])) + * 360.0 + / self.data["number_of_bins_per_histogram"] + + spin_offset_first_bin + ).tolist() # this is to make the key 'histogram_flag_array' to be seen immediately after 'histogram' - self.data['histogram_flag_array'] = None + self.data["histogram_flag_array"] = None # dictionary for some additional debugging data - self.data['debugging_data'] = {} + self.data["debugging_data"] = {} # set block_start_time and block_end_time for funcs invoked in process_anc_input_from_sdc() self.block_start_time = self.data[MAIN_TIME_FIELD] - if MAIN_TIME_FIELD == 'imap_start_time': - self.block_end_time = self.block_start_time + self.data['imap_end_time_offset'] - if MAIN_TIME_FIELD == 'glows_start_time': - self.block_end_time = self.block_start_time + self.data['glows_end_time_offset'] + if MAIN_TIME_FIELD == "imap_start_time": + self.block_end_time = ( + self.block_start_time + self.data["imap_end_time_offset"] + ) + if MAIN_TIME_FIELD == "glows_start_time": + self.block_end_time = ( + self.block_start_time + self.data["glows_end_time_offset"] + ) # convert ancillary parameters collected onboard to physical units self._convert_ancillary_parameters(l1a_data, anc_input_from_instr_team) @@ -270,26 +358,28 @@ def create_l1b_from_l1a_data(self, l1a_data, anc_input_from_sdc, anc_input_from_ self._process_anc_input_from_sdc(anc_input_from_sdc) # deserialize flags generated onboard and save them as self.data['flags'] dictionary - self.data['flags'] = flags_deserialize(l1a_data['flags']['flags_set_onboard']) + self.data["flags"] = flags_deserialize(l1a_data["flags"]["flags_set_onboard"]) # add ground-processing flags to self.data['flags'] - for key in l1a_data['flags']: - if key != 'flags_set_onboard': - self.data['flags'][key] = l1a_data['flags'][key] + for key in l1a_data["flags"]: + if key != "flags_set_onboard": + self.data["flags"][key] = l1a_data["flags"][key] # initialize histogram bin flags - self.data['histogram_flag_array'] = np.zeros_like(self.data['histogram'], np.int16) + self.data["histogram_flag_array"] = np.zeros_like( + self.data["histogram"], np.int16 + ) # apply filters defined by the IMAP/GLOWS Instrument Team self._filters(anc_input_from_instr_team) # move 'debuging_data' dictionary to the end of self.data dictionary - aux = self.data['debugging_data'] - self.data.pop('debugging_data') - self.data['debugging_data'] = aux + aux = self.data["debugging_data"] + self.data.pop("debugging_data") + self.data["debugging_data"] = aux # uncomment the line below to remove 'debugging_data' from JSON - del self.data['debugging_data'] + del self.data["debugging_data"] def _filters(self, anc_input_from_instr_team): """ @@ -298,7 +388,7 @@ def _filters(self, anc_input_from_instr_team): anc_input_from_instr_team: ancillary input provided by IMAP/GLOWS Instrument Team """ # read filter settings - file_handler = open(anc_input_from_instr_team['settings'], 'r') + file_handler = open(anc_input_from_instr_team["settings"], "r") settings = json.load(file_handler) file_handler.close() @@ -309,49 +399,61 @@ def _filters(self, anc_input_from_instr_team): self._set_histogram_flag_array_excl_regions(anc_input_from_instr_team, settings) # set excluded-by-instrument-team bit in histogram flag array - self._set_histogram_flag_array_by_instr_team(anc_input_from_instr_team, 'exclusions_by_instr_team') + self._set_histogram_flag_array_by_instr_team( + anc_input_from_instr_team, "exclusions_by_instr_team" + ) # set suspected_transient bit in histogram flag array - self._set_histogram_flag_array_by_instr_team(anc_input_from_instr_team, 'suspected_transients') + self._set_histogram_flag_array_by_instr_team( + anc_input_from_instr_team, "suspected_transients" + ) # apply filter based on daily statistical error and set related flag self._flag_based_on_daily_statistical_error(settings) # apply filter based on filter-temperature spread and set related flag - self._flag_compare_with_threshold('filter_temperature_std_dev', - settings['filter_based_on_temperature_std_dev'] - ['std_dev_threshold__celsius_deg'], - 'is_temperature_std_dev_beyond_threshold') + self._flag_compare_with_threshold( + "filter_temperature_std_dev", + settings["filter_based_on_temperature_std_dev"][ + "std_dev_threshold__celsius_deg" + ], + "is_temperature_std_dev_beyond_threshold", + ) # apply filter based on HV-voltage spread and set related flag - self._flag_compare_with_threshold('hv_voltage_std_dev', - settings['filter_based_on_hv_voltage_std_dev'] - ['std_dev_threshold__volt'], - 'is_hv_voltage_std_dev_beyond_threshold') + self._flag_compare_with_threshold( + "hv_voltage_std_dev", + settings["filter_based_on_hv_voltage_std_dev"]["std_dev_threshold__volt"], + "is_hv_voltage_std_dev_beyond_threshold", + ) # apply filter based on spin-period spread and set related flag - self._flag_compare_with_threshold('spin_period_std_dev', - settings['filter_based_on_spin_period_std_dev'] - ['std_dev_threshold__sec'], - 'is_spin_period_std_dev_beyond_threshold') + self._flag_compare_with_threshold( + "spin_period_std_dev", + settings["filter_based_on_spin_period_std_dev"]["std_dev_threshold__sec"], + "is_spin_period_std_dev_beyond_threshold", + ) # apply filter based on pulse-length spread and set related flag - self._flag_compare_with_threshold('pulse_length_std_dev', - settings['filter_based_on_pulse_length_std_dev'] - ['std_dev_threshold__usec'], - 'is_pulse_length_std_dev_beyond_threshold') + self._flag_compare_with_threshold( + "pulse_length_std_dev", + settings["filter_based_on_pulse_length_std_dev"]["std_dev_threshold__usec"], + "is_pulse_length_std_dev_beyond_threshold", + ) # apply filter based on relative difference between onboard and ground spin periods self._flag_relative_difference_vs_threshold( - 'spin_period_average', - 'spin_period_ground_average', - settings['filter_based_on_comparison_of_spin_periods'] - ['relative_difference_threshold'], - 'is_spin_period_difference_beyond_threshold') + "spin_period_average", + "spin_period_ground_average", + settings["filter_based_on_comparison_of_spin_periods"][ + "relative_difference_threshold" + ], + "is_spin_period_difference_beyond_threshold", + ) # set a fancy pattern 0x5555 in histogram_flag_array # this is for testing purposes, uncomment only if know exactly what you are doing - #self._set_histogram_flag_array_test_pattern() + # self._set_histogram_flag_array_test_pattern() def _flag_based_on_daily_statistical_error(self, settings): """ @@ -360,25 +462,37 @@ def _flag_based_on_daily_statistical_error(self, settings): Args: settings: filter settings from a file provided by IMAP/GLOWS Instrument Team """ - n_lo = settings['filter_based_on_daily_statistical_error']['n_sigma_threshold_lower'] - n_up = settings['filter_based_on_daily_statistical_error']['n_sigma_threshold_upper'] + n_lo = settings["filter_based_on_daily_statistical_error"][ + "n_sigma_threshold_lower" + ] + n_up = settings["filter_based_on_daily_statistical_error"][ + "n_sigma_threshold_upper" + ] if n_lo < 0 or n_up < 0: # negative value means that filter is turned off, so we can immediately set the flag - self.data['flags']['is_beyond_daily_statistical_error'] = False + self.data["flags"]["is_beyond_daily_statistical_error"] = False else: # filter is turned on, so we need to compare the total number of counts in the current # histogram with thresholds and set the flag accordingly - total_counts = np.sum(self.data['histogram']) - self.data['debugging_data']['total_counts_in_histogram'] = int(total_counts) - total_counts_daily_average, total_counts_daily_std_dev = \ + total_counts = np.sum(self.data["histogram"]) + self.data["debugging_data"]["total_counts_in_histogram"] = int(total_counts) + total_counts_daily_average, total_counts_daily_std_dev = ( self.total_counts_daily_statistics() - self.data['debugging_data']['total_counts_daily_average'] = total_counts_daily_average - self.data['debugging_data']['total_counts_daily_std_dev'] = total_counts_daily_std_dev - if total_counts_daily_average - n_lo*total_counts_daily_std_dev <= total_counts <= \ - total_counts_daily_average + n_up*total_counts_daily_std_dev: - self.data['flags']['is_beyond_daily_statistical_error'] = False + ) + self.data["debugging_data"]["total_counts_daily_average"] = ( + total_counts_daily_average + ) + self.data["debugging_data"]["total_counts_daily_std_dev"] = ( + total_counts_daily_std_dev + ) + if ( + total_counts_daily_average - n_lo * total_counts_daily_std_dev + <= total_counts + <= total_counts_daily_average + n_up * total_counts_daily_std_dev + ): + self.data["flags"]["is_beyond_daily_statistical_error"] = False else: - self.data['flags']['is_beyond_daily_statistical_error'] = True + self.data["flags"]["is_beyond_daily_statistical_error"] = True def _flag_compare_with_threshold(self, parameter_name, threshold, flag_name): """ @@ -390,17 +504,18 @@ def _flag_compare_with_threshold(self, parameter_name, threshold, flag_name): """ if threshold < 0.0: # negative value means that filter is turned off, so we can immediately set the flag - self.data['flags'][flag_name] = False + self.data["flags"][flag_name] = False else: # filter is turned on, so we need to compare the parameter value in the current # histogram with threshold and set the flag accordingly if self.data[parameter_name] < threshold: - self.data['flags'][flag_name] = False + self.data["flags"][flag_name] = False else: - self.data['flags'][flag_name] = True + self.data["flags"][flag_name] = True - def _flag_relative_difference_vs_threshold(self, parameter_1_name, parameter_2_name, threshold, - flag_name): + def _flag_relative_difference_vs_threshold( + self, parameter_1_name, parameter_2_name, threshold, flag_name + ): """ Set flag based on comparison of relative difference of two parameters vs. threshold, set True if beyond threshold @@ -412,19 +527,20 @@ def _flag_relative_difference_vs_threshold(self, parameter_1_name, parameter_2_n """ if threshold < 0.0: # negative value means that filter is turned off, so we can immediately set the flag - self.data['flags'][flag_name] = False + self.data["flags"][flag_name] = False else: # filter is turned on, so we need to compare the relative difference in the current # histogram with threshold and set the flag accordingly - relative_difference = np.fabs(self.data[parameter_1_name] - \ - self.data[parameter_2_name])/ \ - (self.data[parameter_1_name] + \ - self.data[parameter_2_name]) - self.data['debugging_data']['spin_period_relative_difference'] = relative_difference + relative_difference = np.fabs( + self.data[parameter_1_name] - self.data[parameter_2_name] + ) / (self.data[parameter_1_name] + self.data[parameter_2_name]) + self.data["debugging_data"]["spin_period_relative_difference"] = ( + relative_difference + ) if relative_difference < threshold: - self.data['flags'][flag_name] = False + self.data["flags"][flag_name] = False else: - self.data['flags'][flag_name] = True + self.data["flags"][flag_name] = True def _process_anc_input_from_sdc(self, anc_input_from_sdc): """ @@ -435,7 +551,9 @@ def _process_anc_input_from_sdc(self, anc_input_from_sdc): # read ancillary variables provided by SDC (IMAP general telemetry) # note that finally different CSV files can be used for different quantities - anc_data = pd.read_csv(anc_input_from_sdc['anc_data_file_name'], sep=' ', comment="#") + anc_data = pd.read_csv( + anc_input_from_sdc["anc_data_file_name"], sep=" ", comment="#" + ) # TMP_SOLUTION: we use here a csv file with mock ancillary data generated by # GLOWS Team using generate_anc_file.py script # the following procedures invoked below: @@ -446,16 +564,26 @@ def _process_anc_input_from_sdc(self, anc_input_from_sdc): # e.g., it is expected that finally there will be several anc_data files instead # of one containing all required data - self.data['spin_period_ground_average'], self.data['spin_period_ground_std_dev'] = \ - self._compute_spin_period_ground_estimate(anc_data) - self.data['position_angle_offset_average'], self.data['position_angle_offset_std_dev'] = \ - self._compute_position_angle(anc_data) - self.data['spin_axis_orientation_average'], self.data['spin_axis_orientation_std_dev'] = \ - self._compute_spin_axis_orientation(anc_data) - self.data['spacecraft_location_average'], self.data['spacecraft_location_std_dev'] = \ - self._compute_spacecraft_location(anc_data) - self.data['spacecraft_velocity_average'], self.data['spacecraft_velocity_std_dev'] = \ - self._compute_spacecraft_velocity(anc_data) + ( + self.data["spin_period_ground_average"], + self.data["spin_period_ground_std_dev"], + ) = self._compute_spin_period_ground_estimate(anc_data) + ( + self.data["position_angle_offset_average"], + self.data["position_angle_offset_std_dev"], + ) = self._compute_position_angle(anc_data) + ( + self.data["spin_axis_orientation_average"], + self.data["spin_axis_orientation_std_dev"], + ) = self._compute_spin_axis_orientation(anc_data) + ( + self.data["spacecraft_location_average"], + self.data["spacecraft_location_std_dev"], + ) = self._compute_spacecraft_location(anc_data) + ( + self.data["spacecraft_velocity_average"], + self.data["spacecraft_velocity_std_dev"], + ) = self._compute_spacecraft_velocity(anc_data) def read_from_file(self, file_name): """ @@ -473,16 +601,17 @@ def _read_from_json_file(self, file_name): Args: file_name: name of of file with L1b data """ - file_handler = open(file_name, 'r') + file_handler = open(file_name, "r") self.data = json.load(file_handler) file_handler.close() # hexadecimal values saved as strings require special treatment to make them integers - self.data['histogram_flag_array'] = \ - [int(x, base=16) for x in self.data['histogram_flag_array']] + self.data["histogram_flag_array"] = [ + int(x, base=16) for x in self.data["histogram_flag_array"] + ] # additional field to keep track where data come from - self.data['l1b_file_name'] = file_name + self.data["l1b_file_name"] = file_name def save_data_to_file(self): """ @@ -499,38 +628,38 @@ def _save_data_to_json_file(self): """ # define formats for printing values to JSON file fmt = { - 'arrays': { - 'histogram': '%d', - 'imap_spin_angle_bin_cntr': '%.2f', + "arrays": { + "histogram": "%d", + "imap_spin_angle_bin_cntr": "%.2f", }, - 'scalar variables': { - 'glows_start_time': '%.7f', - 'glows_end_time_offset': '%.7f', - 'imap_start_time': '%.7f', - 'imap_end_time_offset': '%.7f', - 'filter_temperature_average': '%.2f', - 'filter_temperature_std_dev': '%.3e', - 'hv_voltage_average': '%.1f', - 'hv_voltage_std_dev': '%.3e', - 'spin_period_average': '%.7f', - 'spin_period_std_dev': '%.3e', - 'spin_period_ground_average': '%.7f', - 'spin_period_ground_std_dev': '%.3e', - 'pulse_length_average': '%.3e', - 'pulse_length_std_dev': '%.3e', - 'position_angle_offset_average': '%.3f', - 'position_angle_offset_std_dev': '%.3e' + "scalar variables": { + "glows_start_time": "%.7f", + "glows_end_time_offset": "%.7f", + "imap_start_time": "%.7f", + "imap_end_time_offset": "%.7f", + "filter_temperature_average": "%.2f", + "filter_temperature_std_dev": "%.3e", + "hv_voltage_average": "%.1f", + "hv_voltage_std_dev": "%.3e", + "spin_period_average": "%.7f", + "spin_period_std_dev": "%.3e", + "spin_period_ground_average": "%.7f", + "spin_period_ground_std_dev": "%.3e", + "pulse_length_average": "%.3e", + "pulse_length_std_dev": "%.3e", + "position_angle_offset_average": "%.3f", + "position_angle_offset_std_dev": "%.3e", }, - 'lon lat variables': { - 'spin_axis_orientation_average': '%.3f', - 'spin_axis_orientation_std_dev': '%.3e' + "lon lat variables": { + "spin_axis_orientation_average": "%.3f", + "spin_axis_orientation_std_dev": "%.3e", + }, + "vector variables": { + "spacecraft_location_average": "%.1f", + "spacecraft_location_std_dev": "%.3e", + "spacecraft_velocity_average": "%.3f", + "spacecraft_velocity_std_dev": "%.3e", }, - 'vector variables': { - 'spacecraft_location_average': '%.1f', - 'spacecraft_location_std_dev': '%.3e', - 'spacecraft_velocity_average': '%.3f', - 'spacecraft_velocity_std_dev': '%.3e' - } } # create local copy for printing purposes @@ -540,68 +669,98 @@ def _save_data_to_json_file(self): # them to be saved as one-liners (i.e., without \n after each element) or we want them to # be printed with a specific format by json.dumps() # so we convert variables to strings as defined in fmt - for key in fmt['arrays'].keys(): - data[key] = '[' + ', '.join(map(lambda x: fmt['arrays'][key] % x, list(data[key]))) + ']' - for key in fmt['scalar variables'].keys(): - data[key] = '|'+ fmt['scalar variables'][key] % data[key] +'|' - for key in fmt['lon lat variables'].keys(): - for sec_key in ['lon', 'lat']: - data[key][sec_key] = '|'+ fmt['lon lat variables'][key] % data[key][sec_key] +'|' - for key in fmt['vector variables'].keys(): - for sec_key in ['x', 'y', 'z']: - data[key][sec_key] = '|'+ fmt['vector variables'][key] % data[key][sec_key] +'|' + for key in fmt["arrays"].keys(): + data[key] = ( + "[" + + ", ".join(map(lambda x: fmt["arrays"][key] % x, list(data[key]))) + + "]" + ) + for key in fmt["scalar variables"].keys(): + data[key] = "|" + fmt["scalar variables"][key] % data[key] + "|" + for key in fmt["lon lat variables"].keys(): + for sec_key in ["lon", "lat"]: + data[key][sec_key] = ( + "|" + fmt["lon lat variables"][key] % data[key][sec_key] + "|" + ) + for key in fmt["vector variables"].keys(): + for sec_key in ["x", "y", "z"]: + data[key][sec_key] = ( + "|" + fmt["vector variables"][key] % data[key][sec_key] + "|" + ) # histogram_flag_array requires special hexadecimal treatment - data['histogram_flag_array'] = '[' + \ - ', '.join(map(lambda x: '"0x' + format(x, 'x').zfill(4) +'"', - list(data['histogram_flag_array']))) + ']' + data["histogram_flag_array"] = ( + "[" + + ", ".join( + map( + lambda x: '"0x' + format(x, "x").zfill(4) + '"', + list(data["histogram_flag_array"]), + ) + ) + + "]" + ) # generate JSON content to be saved in output file json_content = json.dumps(data, indent=4, default=vars) # some corrections to get numbers back - json_content = json_content.replace('"[', '[').replace(']"', ']') - json_content = json_content.replace('|"', '').replace('"|', '') + json_content = json_content.replace('"[', "[").replace(']"', "]") + json_content = json_content.replace('|"', "").replace('"|', "") json_content = json_content.replace('\\"', '"') - if 1==2: + if 1 == 2: # some fields need to be temporarily converted to string arrays, because we want to force # them to be saved as one-liners (i.e., without \n after each element) by json.dumps() - spin_angle_tmp = self.data['imap_spin_angle_bin_cntr'] - histogram_tmp = self.data['histogram'] - flags_tmp = self.data['histogram_flag_array'] - self.data['histogram'] = '[' + ', '.join(map(str, list(histogram_tmp))) + ']' - self.data['imap_spin_angle_bin_cntr'] = \ - '[' + ', '.join(map(lambda x: '%.2f' % x, list(spin_angle_tmp))) + ']' - self.data['histogram_flag_array'] = '[' + \ - ', '.join(map(lambda x: '"0x' + format(x, 'x').zfill(4) +'"', list(flags_tmp))) + ']' + spin_angle_tmp = self.data["imap_spin_angle_bin_cntr"] + histogram_tmp = self.data["histogram"] + flags_tmp = self.data["histogram_flag_array"] + self.data["histogram"] = ( + "[" + ", ".join(map(str, list(histogram_tmp))) + "]" + ) + self.data["imap_spin_angle_bin_cntr"] = ( + "[" + ", ".join(map(lambda x: "%.2f" % x, list(spin_angle_tmp))) + "]" + ) + self.data["histogram_flag_array"] = ( + "[" + + ", ".join( + map( + lambda x: '"0x' + format(x, "x").zfill(4) + '"', list(flags_tmp) + ) + ) + + "]" + ) json_content = json.dumps(self.data, indent=4, default=vars) - self.data['imap_spin_angle_bin_cntr'] = spin_angle_tmp - self.data['histogram'] = histogram_tmp - self.data['histogram_flag_array'] = flags_tmp + self.data["imap_spin_angle_bin_cntr"] = spin_angle_tmp + self.data["histogram"] = histogram_tmp + self.data["histogram_flag_array"] = flags_tmp # modify json_content to make string arrays be seen finally as number arrays - json_content = json_content.replace('"[', '[').replace(']"', ']') + json_content = json_content.replace('"[', "[").replace(']"', "]") json_content = json_content.replace('\\"', '"') # start_time_utc string for output file name, based on unique_block_identifier - start_time_utc = ''.join(filter(str.isdigit, self.data['unique_block_identifier'])) + start_time_utc = "".join( + filter(str.isdigit, self.data["unique_block_identifier"]) + ) # the output file name is constructed using a convention defined in Sec. 1.4.2 of # "IMAP SDC to Instrument Team ICD" - file_name = 'data_l1b_histograms/imap_glows_l1b_%s_orbX_modX_p_v00.json' % \ - (start_time_utc) + file_name = "data_l1b_histograms/imap_glows_l1b_%s_orbX_modX_p_v00.json" % ( + start_time_utc + ) # save data - file_handler = open(file_name, 'w') + file_handler = open(file_name, "w") print(json_content, file=file_handler) file_handler.close() # TO_BE_REMOVED append file name to l1b list needed for the l1b-to-l2 processing - #file_handler = open('l1b_list.dat', 'a') - #print(file_name, file=file_handler) - #file_handler.close() + # file_handler = open('l1b_list.dat', 'a') + # print(file_name, file=file_handler) + # file_handler.close() - def _set_histogram_flag_array_by_instr_team(self, anc_input_from_instr_team, type_of_list): + def _set_histogram_flag_array_by_instr_team( + self, anc_input_from_instr_team, type_of_list + ): """ Set histogram flag array using lists defined by IMAP/GLOWS Instrument Team bin indexes in the list are marked by True @@ -609,31 +768,40 @@ def _set_histogram_flag_array_by_instr_team(self, anc_input_from_instr_team, typ anc_input_from_instr_team: ancillary input provided by IMAP/GLOWS Instrument Team type_of_list: 'exclusions_by_instr_team' or 'suspected_transients' """ - file_handler = open(anc_input_from_instr_team[type_of_list], 'r') + file_handler = open(anc_input_from_instr_team[type_of_list], "r") line = file_handler.readline() while line: - if line[0] != '#': + if line[0] != "#": # sanity check - assert line[:2] == '20' and line[10] == 'T', \ - 'Bad identifier ' + line[:20] + ' in ' + \ - anc_input_from_instr_team[type_of_list] - identifier = line.rstrip().split(' ')[0] - flags = line.rstrip().split(' ')[1] + assert line[:2] == "20" and line[10] == "T", ( + "Bad identifier " + + line[:20] + + " in " + + anc_input_from_instr_team[type_of_list] + ) + identifier = line.rstrip().split(" ")[0] + flags = line.rstrip().split(" ")[1] # sanity check # this check turned out to be inappropriate because number of bins is not always the same - #assert len(flags) == self.data['number_of_bins_per_histogram'], \ + # assert len(flags) == self.data['number_of_bins_per_histogram'], \ # 'Bad length of mask_array for identifier ' + identifier + ' in ' + \ # anc_input_from_instr_team[type_of_list] - if identifier == self.data['unique_block_identifier']: + if identifier == self.data["unique_block_identifier"]: flags = np.array(list(flags)).astype(bool) -# print(np.shape(flags)) - if type_of_list == 'exclusions_by_instr_team': - flags = np.left_shift(flags, L1B_L2_HIST_FLAG_FIELDS['is_excluded_by_instr_team']) - if type_of_list == 'suspected_transients': - flags = np.left_shift(flags, L1B_L2_HIST_FLAG_FIELDS['is_suspected_transient']) - self.data['histogram_flag_array'] = self.data['histogram_flag_array'] | flags + # print(np.shape(flags)) + if type_of_list == "exclusions_by_instr_team": + flags = np.left_shift( + flags, L1B_L2_HIST_FLAG_FIELDS["is_excluded_by_instr_team"] + ) + if type_of_list == "suspected_transients": + flags = np.left_shift( + flags, L1B_L2_HIST_FLAG_FIELDS["is_suspected_transient"] + ) + self.data["histogram_flag_array"] = ( + self.data["histogram_flag_array"] | flags + ) line = file_handler.readline() file_handler.close() @@ -644,26 +812,36 @@ def _set_histogram_flag_array_excl_by_instr_team(self, anc_input_from_instr_team Args: anc_input_from_instr_team: ancillary input provided by IMAP/GLOWS Instrument Team """ - file_handler = open(anc_input_from_instr_team['exclusions_by_instr_team'], 'r') + file_handler = open(anc_input_from_instr_team["exclusions_by_instr_team"], "r") line = file_handler.readline() while line: - if line[0] != '#': + if line[0] != "#": # sanity check - assert line[:2] == '20' and line[10] == '_', \ - 'Bad identifier ' + line[:20] + ' in ' + \ - anc_input_from_instr_team['exclusions_by_instr_team'] - identifier = line.rstrip().split(' ')[0] - flags = line.rstrip().split(' ')[1] + assert line[:2] == "20" and line[10] == "_", ( + "Bad identifier " + + line[:20] + + " in " + + anc_input_from_instr_team["exclusions_by_instr_team"] + ) + identifier = line.rstrip().split(" ")[0] + flags = line.rstrip().split(" ")[1] # sanity check - assert len(flags) == self.data['number_of_bins_per_histogram'], \ - 'Bad length of mask_array for identifier ' + identifier + ' in ' + \ - anc_input_from_instr_team['exclusions_by_instr_team'] - - if identifier == self.data['unique_block_identifier']: + assert len(flags) == self.data["number_of_bins_per_histogram"], ( + "Bad length of mask_array for identifier " + + identifier + + " in " + + anc_input_from_instr_team["exclusions_by_instr_team"] + ) + + if identifier == self.data["unique_block_identifier"]: flags = np.array(list(flags)).astype(bool) - flags = np.left_shift(flags, L1B_L2_HIST_FLAG_FIELDS['is_excluded_by_instr_team']) - self.data['histogram_flag_array'] = self.data['histogram_flag_array'] | flags + flags = np.left_shift( + flags, L1B_L2_HIST_FLAG_FIELDS["is_excluded_by_instr_team"] + ) + self.data["histogram_flag_array"] = ( + self.data["histogram_flag_array"] | flags + ) line = file_handler.readline() file_handler.close() @@ -676,46 +854,68 @@ def _set_histogram_flag_array_uv_sources(self, anc_input_from_instr_team, settin settings: filter settings """ - if check_if_contains_actual_data(anc_input_from_instr_team['map_of_uv_sources']): + if check_if_contains_actual_data( + anc_input_from_instr_team["map_of_uv_sources"] + ): # read ecliptic coords and angular radiuses for the masking of bright uv point sources - uv_sources_coords_and_radiuses = np.loadtxt(anc_input_from_instr_team['map_of_uv_sources'], - usecols=[1, 2, 3]) + uv_sources_coords_and_radiuses = np.loadtxt( + anc_input_from_instr_team["map_of_uv_sources"], usecols=[1, 2, 3] + ) # compute ecliptic coords (lon, lat) for histogram bin centers histogram_bins_ecl_coords = self._compute_histogram_bins_ecl_coords() - flag = np.zeros_like(self.data['histogram'], bool) + flag = np.zeros_like(self.data["histogram"], bool) # a radius_bins grouping is used below for code speed up, because otherwise we would need to use # coord1.separation(coord2) from astropy, which turned out to be slow in tests - n_radius_bins = 10 # actually used bins, see +2 in np.linspace() argument below - radius_bins = np.linspace(uv_sources_coords_and_radiuses[:, 2].min()-1.0e-6, - uv_sources_coords_and_radiuses[:, 2].max()+1.0e-6, n_radius_bins+2) + n_radius_bins = ( + 10 # actually used bins, see +2 in np.linspace() argument below + ) + radius_bins = np.linspace( + uv_sources_coords_and_radiuses[:, 2].min() - 1.0e-6, + uv_sources_coords_and_radiuses[:, 2].max() + 1.0e-6, + n_radius_bins + 2, + ) # digitizing/binning into radius_bins - radius_idxs = np.digitize(uv_sources_coords_and_radiuses[:, 2], bins=radius_bins) + radius_idxs = np.digitize( + uv_sources_coords_and_radiuses[:, 2], bins=radius_bins + ) - for i in range(1, n_radius_bins+2): # looping over all radius_bins groups + for i in range(1, n_radius_bins + 2): # looping over all radius_bins groups rejection_radius = radius_bins[i] idxs = np.nonzero(radius_idxs == i)[0] - if len(idxs) > 0: # check if we have any uv sources to check in the current group - match_catalog = SkyCoord(uv_sources_coords_and_radiuses[idxs, 0], uv_sources_coords_and_radiuses[idxs, 1], - frame='geocentrictrueecliptic', unit='deg') - search_catalog = SkyCoord(histogram_bins_ecl_coords[:, 0], - histogram_bins_ecl_coords[:, 1], - frame='geocentrictrueecliptic', unit='deg') + if ( + len(idxs) > 0 + ): # check if we have any uv sources to check in the current group + match_catalog = SkyCoord( + uv_sources_coords_and_radiuses[idxs, 0], + uv_sources_coords_and_radiuses[idxs, 1], + frame="geocentrictrueecliptic", + unit="deg", + ) + search_catalog = SkyCoord( + histogram_bins_ecl_coords[:, 0], + histogram_bins_ecl_coords[:, 1], + frame="geocentrictrueecliptic", + unit="deg", + ) # find bin vs. map point pairs that are closer than rejection_radius - idxs_match, __, __, __ = \ - match_catalog.search_around_sky(search_catalog, rejection_radius*u.deg) + idxs_match, __, __, __ = match_catalog.search_around_sky( + search_catalog, rejection_radius * u.deg + ) # find indexes of bins that are too close to map points idxs_to_reject = np.unique(idxs_match) # set histogram bin flags flag[idxs_to_reject] = True - flag = np.left_shift(flag, L1B_L2_HIST_FLAG_FIELDS['is_close_to_uv_source']) - self.data['histogram_flag_array'] = self.data['histogram_flag_array'] | flag + flag = np.left_shift(flag, L1B_L2_HIST_FLAG_FIELDS["is_close_to_uv_source"]) + self.data["histogram_flag_array"] = self.data["histogram_flag_array"] | flag - def _set_histogram_flag_array_excl_regions(self, anc_input_from_instr_team, settings): + def _set_histogram_flag_array_excl_regions( + self, anc_input_from_instr_team, settings + ): """ Set histogram flag array using a map of excluded regions, close-to-map-points bins are denoted by True @@ -724,18 +924,23 @@ def _set_histogram_flag_array_excl_regions(self, anc_input_from_instr_team, sett settings: filter settings """ - if check_if_contains_actual_data(anc_input_from_instr_team['map_of_excluded_regions']): + if check_if_contains_actual_data( + anc_input_from_instr_team["map_of_excluded_regions"] + ): # read ecliptic coords of points defining excluded regions - excluded_pts_coords = np.loadtxt(anc_input_from_instr_team['map_of_excluded_regions']) + excluded_pts_coords = np.loadtxt( + anc_input_from_instr_team["map_of_excluded_regions"] + ) # set angular radius below which bins are too close to map points - rejection_radius = settings['filter_based_on_maps'] \ - ['angular_radius_for_excl_regions__deg'] + rejection_radius = settings["filter_based_on_maps"][ + "angular_radius_for_excl_regions__deg" + ] if rejection_radius <= 0.0: # negative or zero value of rejection_radius means that the filter is turned off and we # can immediately set False value in the histogram_flag_array for all bins - flag = np.zeros_like(self.data['histogram'], bool) + flag = np.zeros_like(self.data["histogram"], bool) else: # positive value of rejection_radius means that the filter is turned on and we must # compare bin positions in the sky with map-point positions to set True/False values @@ -744,23 +949,33 @@ def _set_histogram_flag_array_excl_regions(self, anc_input_from_instr_team, sett # compute ecliptic coords (lon, lat) for histogram bin centers histogram_bins_ecl_coords = self._compute_histogram_bins_ecl_coords() - search_catalog = SkyCoord(excluded_pts_coords[:, 0], excluded_pts_coords[:, 1], - frame='geocentrictrueecliptic', unit='deg') - match_catalog = SkyCoord(histogram_bins_ecl_coords[:, 0], - histogram_bins_ecl_coords[:, 1], - frame='geocentrictrueecliptic', unit='deg') + search_catalog = SkyCoord( + excluded_pts_coords[:, 0], + excluded_pts_coords[:, 1], + frame="geocentrictrueecliptic", + unit="deg", + ) + match_catalog = SkyCoord( + histogram_bins_ecl_coords[:, 0], + histogram_bins_ecl_coords[:, 1], + frame="geocentrictrueecliptic", + unit="deg", + ) # find bin vs. map point pairs that are closer than rejection_radius - __, idxs_match, __, __ = \ - match_catalog.search_around_sky(search_catalog, rejection_radius*u.deg) + __, idxs_match, __, __ = match_catalog.search_around_sky( + search_catalog, rejection_radius * u.deg + ) # find indexes of bins that are too close to map points idxs_to_reject = np.unique(idxs_match) # set histogram bin flags - flag = np.zeros_like(self.data['histogram'], bool) + flag = np.zeros_like(self.data["histogram"], bool) flag[idxs_to_reject] = True - flag = np.left_shift(flag, L1B_L2_HIST_FLAG_FIELDS['is_inside_excluded_region']) - self.data['histogram_flag_array'] = self.data['histogram_flag_array'] | flag + flag = np.left_shift( + flag, L1B_L2_HIST_FLAG_FIELDS["is_inside_excluded_region"] + ) + self.data["histogram_flag_array"] = self.data["histogram_flag_array"] | flag def _set_histogram_flag_array_test_pattern(self): """ @@ -768,11 +983,11 @@ def _set_histogram_flag_array_test_pattern(self): """ for i in range(15): if i % 2 == 0: - flag = np.ones_like(self.data['histogram'], bool) + flag = np.ones_like(self.data["histogram"], bool) else: - flag = np.zeros_like(self.data['histogram'], bool) + flag = np.zeros_like(self.data["histogram"], bool) flag = np.left_shift(flag, i) - self.data['histogram_flag_array'] = self.data['histogram_flag_array'] | flag + self.data["histogram_flag_array"] = self.data["histogram_flag_array"] | flag def set_times(self, l1a_data): """ @@ -780,32 +995,42 @@ def set_times(self, l1a_data): Args: l1a_data: L1a data provided in the form of data field of L1aDataHistogram object """ - self.data['glows_start_time'] = \ - time_sec_subsec_to_float64(l1a_data['glows_start_time']['seconds'], - l1a_data['glows_start_time']['subseconds']) - self.data['glows_end_time_offset'] = \ - time_sec_subsec_to_float64(l1a_data['glows_end_time_offset']['seconds'], - l1a_data['glows_end_time_offset']['subseconds']) - self.data['imap_start_time'] = \ - time_sec_subsec_to_float64(l1a_data['imap_start_time']['seconds'], - l1a_data['imap_start_time']['subseconds']) - self.data['imap_end_time_offset'] = \ - time_sec_subsec_to_float64(l1a_data['imap_end_time_offset']['seconds'], - l1a_data['imap_end_time_offset']['subseconds']) + self.data["glows_start_time"] = time_sec_subsec_to_float64( + l1a_data["glows_start_time"]["seconds"], + l1a_data["glows_start_time"]["subseconds"], + ) + self.data["glows_end_time_offset"] = time_sec_subsec_to_float64( + l1a_data["glows_end_time_offset"]["seconds"], + l1a_data["glows_end_time_offset"]["subseconds"], + ) + self.data["imap_start_time"] = time_sec_subsec_to_float64( + l1a_data["imap_start_time"]["seconds"], + l1a_data["imap_start_time"]["subseconds"], + ) + self.data["imap_end_time_offset"] = time_sec_subsec_to_float64( + l1a_data["imap_end_time_offset"]["seconds"], + l1a_data["imap_end_time_offset"]["subseconds"], + ) def total_counts_daily_statistics(self): """ Compute daily average for the total number of counts in L1a histograms """ - file_name = 'data_l1a_histograms/total_counts_file.dat' - file_handler = open(file_name, 'r') + file_name = "data_l1a_histograms/total_counts_file.dat" + file_handler = open(file_name, "r") line = file_handler.readline() # sanity check - assert line.rstrip().split(' ')[1] == self.data['block_header']['pkts_file_name'], \ - 'Packets file name ' + line.rstrip().split(' ')[1] + ' in ' + file_name + \ - ' is different from currently processed packets file ' + \ - self.data['block_header']['pkts_file_name'] + assert ( + line.rstrip().split(" ")[1] == self.data["block_header"]["pkts_file_name"] + ), ( + "Packets file name " + + line.rstrip().split(" ")[1] + + " in " + + file_name + + " is different from currently processed packets file " + + self.data["block_header"]["pkts_file_name"] + ) # the first column of data array below: 'seq_count_in_pkts_file' # the second column: total counts in a L1a histogram diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/l2_data_histogram.py b/imap_l3_processing/glows/l3a/glows_toolkit/l2_data_histogram.py index ad2578c3e..aa2e6e326 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/l2_data_histogram.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/l2_data_histogram.py @@ -1,6 +1,7 @@ """@package docstring Author: Marek Strumik, maro at cbk.waw.pl """ + import json import copy import numpy as np @@ -8,7 +9,8 @@ from .funcs import time_sec_subsec_to_utc, calibration_factor from .geom_fun_libr_box import spin2lb -class L2DataHistogram(): + +class L2DataHistogram: """ L2DataHistogram() class for GLOWS-L2-histogram data """ @@ -19,57 +21,74 @@ def __init__(self, anc_input_from_instr_team): """ # self.data is a dictionary to be saved as L2 data self.data = {} - self.data['header'] = {} - self.data['identifier'] = None - self.data['start_time'] = None - self.data['end_time'] = None - self.data['daily_lightcurve'] = {} + self.data["header"] = {} + self.data["identifier"] = None + self.data["start_time"] = None + self.data["end_time"] = None + self.data["daily_lightcurve"] = {} # self.var is for various variables needed for computations, but not to be saved in L2 output file self.var = {} # initialize counters - self.var['l1b_file_counter_all'] = 0 # all l1b files processed for a given day - self.var['l1b_file_counter_good_time'] = 0 # only good-time files for a given day + self.var["l1b_file_counter_all"] = 0 # all l1b files processed for a given day + self.var["l1b_file_counter_good_time"] = ( + 0 # only good-time files for a given day + ) # read pipeline settings to be used for L2 - file_handler = open(anc_input_from_instr_team['settings'], 'r') - self.var['settings'] = json.load(file_handler) + file_handler = open(anc_input_from_instr_team["settings"], "r") + self.var["settings"] = json.load(file_handler) file_handler.close() # sanity check for bad-angle flags (all L1B_L2_HIST_FLAG_FIELDS flags must be included in # self.var['settings'] in the correct order) - list_1 = list(self.var['settings']['active_bad_angle_flags'].keys()) + list_1 = list(self.var["settings"]["active_bad_angle_flags"].keys()) list_2 = list(L1B_L2_HIST_FLAG_FIELDS.keys()) - assert len(list_1) == len(list_2), \ - 'Different numbers of keys in self.var[\'settings\'][\'active_bad_angle_flags\'] ' + \ - 'and in L1B_L2_HIST_FLAG_FIELDS' + assert len(list_1) == len(list_2), ( + "Different numbers of keys in self.var['settings']['active_bad_angle_flags'] " + + "and in L1B_L2_HIST_FLAG_FIELDS" + ) for i, __ in enumerate(list_1): - assert list_1[i] == list_2[i], \ - 'Different keys "' + list_1[i] + '" and "' + list_2[i] + \ - '" in self.var[\'settings\'][\'active_bad_angle_flags\'] and L1B_L2_HIST_FLAG_FIELDS' + assert list_1[i] == list_2[i], ( + 'Different keys "' + + list_1[i] + + '" and "' + + list_2[i] + + "\" in self.var['settings']['active_bad_angle_flags'] and L1B_L2_HIST_FLAG_FIELDS" + ) # define a structure to keep track of occurences of bad-time flags - self.var['bad_time_flag_occurences'] = {} - for key in self.var['settings']['active_bad_time_flags'].keys(): - self.var['bad_time_flag_occurences'][key] = 0 + self.var["bad_time_flag_occurences"] = {} + for key in self.var["settings"]["active_bad_time_flags"].keys(): + self.var["bad_time_flag_occurences"][key] = 0 # define a bad-time mask which is used to decide which L1b files are accepted # as a whole during generation of L2 data product - self.var['bad_time_mask'] = \ - ''.join(['%d' % self.var['settings']['active_bad_time_flags'][key] - for key in self.var['settings']['active_bad_time_flags']]) - self.var['bad_time_mask'] = '0b' + self.var['bad_time_mask'][::-1] + self.var["bad_time_mask"] = "".join( + [ + "%d" % self.var["settings"]["active_bad_time_flags"][key] + for key in self.var["settings"]["active_bad_time_flags"] + ] + ) + self.var["bad_time_mask"] = "0b" + self.var["bad_time_mask"][::-1] # define a bad-angle mask which is used to decide which bins of L1b histograms are accepted # during generation of L2 data product - self.var['bad_angle_mask'] = \ - ''.join(['%d' % self.var['settings']['active_bad_angle_flags'][key] - for key in self.var['settings']['active_bad_angle_flags']]) - self.var['bad_angle_mask'] = '0b' + self.var['bad_angle_mask'][::-1] - - self.var['is_night_for_previous_l1b'] = -1 # stores is-night-flag state from the previously processed L1b file - self.var['is_night_counter'] = 0 # counter of L1b histograms for which is_night flag has been raised + self.var["bad_angle_mask"] = "".join( + [ + "%d" % self.var["settings"]["active_bad_angle_flags"][key] + for key in self.var["settings"]["active_bad_angle_flags"] + ] + ) + self.var["bad_angle_mask"] = "0b" + self.var["bad_angle_mask"][::-1] + + self.var[ + "is_night_for_previous_l1b" + ] = -1 # stores is-night-flag state from the previously processed L1b file + self.var["is_night_counter"] = ( + 0 # counter of L1b histograms for which is_night flag has been raised + ) def _averages_accumulate(self, l1b_data): """ @@ -78,15 +97,17 @@ def _averages_accumulate(self, l1b_data): Args: l1b_data: L1b data provided as data field of L1bDataHistogram object """ - for key in self.var['l1b_averages'].keys(): - if key == 'counter': + for key in self.var["l1b_averages"].keys(): + if key == "counter": continue - if isinstance(self.var['l1b_averages'][key], dict): - for second_key in self.var['l1b_averages'][key].keys(): - self.var['l1b_averages'][key][second_key].append(l1b_data[key][second_key]) + if isinstance(self.var["l1b_averages"][key], dict): + for second_key in self.var["l1b_averages"][key].keys(): + self.var["l1b_averages"][key][second_key].append( + l1b_data[key][second_key] + ) else: - self.var['l1b_averages'][key].append(l1b_data[key]) - self.var['l1b_averages']['counter'] += 1 + self.var["l1b_averages"][key].append(l1b_data[key]) + self.var["l1b_averages"]["counter"] += 1 def _averages_init(self, l1b_data): """ @@ -97,16 +118,16 @@ def _averages_init(self, l1b_data): """ # generate dictionary for computing averages over GOOD TIMES for L1b-file ancillary parameters - self.var['l1b_averages'] = {} + self.var["l1b_averages"] = {} for key in l1b_data.keys(): - if key.endswith('average'): + if key.endswith("average"): if isinstance(l1b_data[key], dict): - self.var['l1b_averages'][key] = {} + self.var["l1b_averages"][key] = {} for second_key in l1b_data[key].keys(): - self.var['l1b_averages'][key][second_key] = [] + self.var["l1b_averages"][key][second_key] = [] else: - self.var['l1b_averages'][key] = [] - self.var['l1b_averages']['counter'] = 0 + self.var["l1b_averages"][key] = [] + self.var["l1b_averages"]["counter"] = 0 def _check_l1b_bad_time_flags(self, l1b_data): """ @@ -118,37 +139,47 @@ def _check_l1b_bad_time_flags(self, l1b_data): """ # sanity check, all bad-time flags in L1b file must be included in self.var['settings'] # in the correct order, otherwise we can assume that an error occurred - list_1 = list(self.var['settings']['active_bad_time_flags'].keys()) - list_2 = list(l1b_data['flags'].keys()) - assert len(list_1) == len(list_2), \ - 'Different number of keys in self.var[\'settings\'][\'active_bad_time_flags\'] ' + \ - 'and in l1b_data[\'flags\']' + list_1 = list(self.var["settings"]["active_bad_time_flags"].keys()) + list_2 = list(l1b_data["flags"].keys()) + assert len(list_1) == len(list_2), ( + "Different number of keys in self.var['settings']['active_bad_time_flags'] " + + "and in l1b_data['flags']" + ) for i, __ in enumerate(list_1): - assert list_1[i] == list_2[i], \ - 'Different keys "' + list_1[i] + '" and "' + list_2[i] + \ - '" in self.var[\'settings\'][\'active_bad_time_flags\'] and l1b_data[\'flags\']' + assert list_1[i] == list_2[i], ( + 'Different keys "' + + list_1[i] + + '" and "' + + list_2[i] + + "\" in self.var['settings']['active_bad_time_flags'] and l1b_data['flags']" + ) # set bad-time-flag bits to compare it with self.var['bad_time_mask'] - l1b_bad_time_flags = ''.join(['%d' % l1b_data['flags'][key] for key in l1b_data['flags']]) - l1b_bad_time_flags = '0b' + l1b_bad_time_flags[::-1] + l1b_bad_time_flags = "".join( + ["%d" % l1b_data["flags"][key] for key in l1b_data["flags"]] + ) + l1b_bad_time_flags = "0b" + l1b_bad_time_flags[::-1] # compare bad_time_flags for L1b with mask - cmp_result = int(l1b_bad_time_flags, 2) & int(self.var['bad_time_mask'], 2) + cmp_result = int(l1b_bad_time_flags, 2) & int(self.var["bad_time_mask"], 2) # if self.var['bad_time_mask'] has inactive is_night field we have to account for # self.var['is_night_counter'] and correct the cmp_result - if cmp_result == 0 and \ - self.var['is_night_counter'] >= self.var['settings']['number_of_good_histograms_at_night']: + if ( + cmp_result == 0 + and self.var["is_night_counter"] + >= self.var["settings"]["number_of_good_histograms_at_night"] + ): cmp_result = 1 - if l1b_data['flags']['is_night'] == True: - print('(night', self.var['is_night_counter'], cmp_result, end=') ') + if l1b_data["flags"]["is_night"] == True: + print("(night", self.var["is_night_counter"], cmp_result, end=") ") if cmp_result != 0: - print('-> bad time') + print("-> bad time") return False - print('-> good time') + print("-> good time") return True def _compute_anc_parms_averages_and_spreads(self): @@ -156,23 +187,26 @@ def _compute_anc_parms_averages_and_spreads(self): Compute averages and spread (std dev) for accumulated L1b data. Mean value is computed over all accumulated L1b averages. Spread is computed as standard deviation of block averages. """ - if self.var['l1b_averages']['counter'] == 0: # there is nothing to compute + if self.var["l1b_averages"]["counter"] == 0: # there is nothing to compute return - for key in self.var['l1b_averages'].keys(): - spread_key = '_'.join(key.split('_')[:-1]) + '_std_dev' - if key == 'counter': + for key in self.var["l1b_averages"].keys(): + spread_key = "_".join(key.split("_")[:-1]) + "_std_dev" + if key == "counter": continue - if isinstance(self.var['l1b_averages'][key], dict): + if isinstance(self.var["l1b_averages"][key], dict): self.data[key] = {} self.data[spread_key] = {} - for second_key in self.var['l1b_averages'][key].keys(): - self.data[key][second_key] = np.mean(self.var['l1b_averages'][key][second_key]) - self.data[spread_key][second_key] = \ - np.std(self.var['l1b_averages'][key][second_key]) + for second_key in self.var["l1b_averages"][key].keys(): + self.data[key][second_key] = np.mean( + self.var["l1b_averages"][key][second_key] + ) + self.data[spread_key][second_key] = np.std( + self.var["l1b_averages"][key][second_key] + ) else: - self.data[key] = np.mean(self.var['l1b_averages'][key]) - self.data[spread_key] = np.std(self.var['l1b_averages'][key]) + self.data[key] = np.mean(self.var["l1b_averages"][key]) + self.data[spread_key] = np.std(self.var["l1b_averages"][key]) def generate_l2_data(self, anc_input_from_instr_team): """ @@ -182,91 +216,122 @@ def generate_l2_data(self, anc_input_from_instr_team): and MOC see ANC_INPUT_FROM_INSTRUMENT_TEAM in constants.py """ - self.data['header']['number_of_l1b_files_used'] = self.var['l1b_file_counter_good_time'] - self.data['header']['number_of_all_l1b_files'] = self.var['l1b_file_counter_all'] - - if self.data['header']['number_of_l1b_files_used'] > 0: - self.data['start_time'] = \ - time_sec_subsec_to_utc(int(self.var['l1b_start_time']), - 0).strftime('%Y-%m-%d %H:%M:%S') - self.data['end_time'] = \ - time_sec_subsec_to_utc(int(self.var['l1b_end_time']), - 0).strftime('%Y-%m-%d %H:%M:%S') + self.data["header"]["number_of_l1b_files_used"] = self.var[ + "l1b_file_counter_good_time" + ] + self.data["header"]["number_of_all_l1b_files"] = self.var[ + "l1b_file_counter_all" + ] + + if self.data["header"]["number_of_l1b_files_used"] > 0: + self.data["start_time"] = time_sec_subsec_to_utc( + int(self.var["l1b_start_time"]), 0 + ).strftime("%Y-%m-%d %H:%M:%S") + self.data["end_time"] = time_sec_subsec_to_utc( + int(self.var["l1b_end_time"]), 0 + ).strftime("%Y-%m-%d %H:%M:%S") else: # good times have not been found - self.data['start_time'] = \ - time_sec_subsec_to_utc(int(self.var['l1b_start_time_all']), - 0).strftime('%Y-%m-%d %H:%M:%S') - self.data['end_time'] = \ - time_sec_subsec_to_utc(int(self.var['l1b_end_time_all']), - 0).strftime('%Y-%m-%d %H:%M:%S') - self.data['daily_lightcurve']['exposure_times'] = \ - np.zeros(self.var['number_of_bins_per_histogram'], float) - self.var['raw_histogram'] = \ - np.zeros(self.var['number_of_bins_per_histogram'], int) - self.data['daily_lightcurve']['photon_flux'] = \ - np.zeros(self.var['number_of_bins_per_histogram'], float) - self.data['daily_lightcurve']['flux_uncertainties'] = \ - np.zeros(self.var['number_of_bins_per_histogram'], float) + self.data["start_time"] = time_sec_subsec_to_utc( + int(self.var["l1b_start_time_all"]), 0 + ).strftime("%Y-%m-%d %H:%M:%S") + self.data["end_time"] = time_sec_subsec_to_utc( + int(self.var["l1b_end_time_all"]), 0 + ).strftime("%Y-%m-%d %H:%M:%S") + self.data["daily_lightcurve"]["exposure_times"] = np.zeros( + self.var["number_of_bins_per_histogram"], float + ) + self.var["raw_histogram"] = np.zeros( + self.var["number_of_bins_per_histogram"], int + ) + self.data["daily_lightcurve"]["photon_flux"] = np.zeros( + self.var["number_of_bins_per_histogram"], float + ) + self.data["daily_lightcurve"]["flux_uncertainties"] = np.zeros( + self.var["number_of_bins_per_histogram"], float + ) # compute cps from raw histogram and exposure times - idxs = np.nonzero(self.data['daily_lightcurve']['exposure_times'] > 0.0)[0] - self.data['daily_lightcurve']['photon_flux'][idxs] = \ - self.var['raw_histogram'][idxs]/self.data['daily_lightcurve']['exposure_times'][idxs] + idxs = np.nonzero(self.data["daily_lightcurve"]["exposure_times"] > 0.0)[0] + self.data["daily_lightcurve"]["photon_flux"][idxs] = ( + self.var["raw_histogram"][idxs] + / self.data["daily_lightcurve"]["exposure_times"][idxs] + ) # add raw histogram counts - self.data['daily_lightcurve']['raw_histogram'] = self.var['raw_histogram'] + self.data["daily_lightcurve"]["raw_histogram"] = self.var["raw_histogram"] # compute uncertainties - self.var['raw_uncertainties'] = np.sqrt(self.var['raw_histogram']) - self.data['daily_lightcurve']['flux_uncertainties'][idxs] = \ - self.var['raw_uncertainties'][idxs]/ \ - self.data['daily_lightcurve']['exposure_times'][idxs] + self.var["raw_uncertainties"] = np.sqrt(self.var["raw_histogram"]) + self.data["daily_lightcurve"]["flux_uncertainties"][idxs] = ( + self.var["raw_uncertainties"][idxs] + / self.data["daily_lightcurve"]["exposure_times"][idxs] + ) - if self.data['header']['number_of_l1b_files_used'] > 0: + if self.data["header"]["number_of_l1b_files_used"] > 0: # compute averages and spreads for parameters for L2 self._compute_anc_parms_averages_and_spreads() # generate spin angle values (for histogram bin centers) measured from the northernmost # point of the scanning circle - spin_angle_from_north = (self.data['daily_lightcurve']['spin_angle'] - \ - self.data['position_angle_offset_average'] + 360.0) % 360.0 + spin_angle_from_north = ( + self.data["daily_lightcurve"]["spin_angle"] + - self.data["position_angle_offset_average"] + + 360.0 + ) % 360.0 # redefine the spin angle, for L2 it is not the imap spin angle anymore but angle measured # from the northernmost part of the scanning circle - self.data['daily_lightcurve']['spin_angle'] = spin_angle_from_north + self.data["daily_lightcurve"]["spin_angle"] = spin_angle_from_north # find the index of the smallest value of spin_angle_from_north, it indicates the shift # value for rolling elements of self.data['daily_lightcurve'] arrays idx = np.argsort(spin_angle_from_north)[0] # we have the shift value, so we can roll all arrays in self.data['daily_lightcurve'] - for key in self.data['daily_lightcurve'].keys(): - self.data['daily_lightcurve'][key] = np.roll(self.data['daily_lightcurve'][key], -idx) + for key in self.data["daily_lightcurve"].keys(): + self.data["daily_lightcurve"][key] = np.roll( + self.data["daily_lightcurve"][key], -idx + ) # compute ecliptic longitude and latitude of histogram bin centers - ones = np.ones(len(self.data['daily_lightcurve']['spin_angle']), float) - histogram_bins_ecl_coords = np.degrees(spin2lb( - np.radians(self.data['spin_axis_orientation_average']['lon'])*ones, - np.radians(self.data['spin_axis_orientation_average']['lat'])*ones, - np.radians(self.data['daily_lightcurve']['spin_angle']), 0.0, - np.radians(SCAN_CIRCLE_ANGULAR_RADIUS)*ones)) - self.data['daily_lightcurve']['ecliptic_lon'] = histogram_bins_ecl_coords[:, 0] - self.data['daily_lightcurve']['ecliptic_lat'] = histogram_bins_ecl_coords[:, 1] + ones = np.ones(len(self.data["daily_lightcurve"]["spin_angle"]), float) + histogram_bins_ecl_coords = np.degrees( + spin2lb( + np.radians(self.data["spin_axis_orientation_average"]["lon"]) + * ones, + np.radians(self.data["spin_axis_orientation_average"]["lat"]) + * ones, + np.radians(self.data["daily_lightcurve"]["spin_angle"]), + 0.0, + np.radians(SCAN_CIRCLE_ANGULAR_RADIUS) * ones, + ) + ) + self.data["daily_lightcurve"]["ecliptic_lon"] = histogram_bins_ecl_coords[ + :, 0 + ] + self.data["daily_lightcurve"]["ecliptic_lat"] = histogram_bins_ecl_coords[ + :, 1 + ] # apply calibration to get helioglow intensity in physical units of Rayleigh - cps_per_R = calibration_factor(anc_input_from_instr_team['calibration_data'], - self.data['start_time'].replace(' ', 'T')) - self.data['daily_lightcurve']['photon_flux'] = \ - self.data['daily_lightcurve']['photon_flux'] / cps_per_R - self.data['daily_lightcurve']['flux_uncertainties'] = \ - self.data['daily_lightcurve']['flux_uncertainties'] / cps_per_R + cps_per_R = calibration_factor( + anc_input_from_instr_team["calibration_data"], + self.data["start_time"].replace(" ", "T"), + ) + self.data["daily_lightcurve"]["photon_flux"] = ( + self.data["daily_lightcurve"]["photon_flux"] / cps_per_R + ) + self.data["daily_lightcurve"]["flux_uncertainties"] = ( + self.data["daily_lightcurve"]["flux_uncertainties"] / cps_per_R + ) # add the number of bins as a key to be saved in L2 output file - self.data['daily_lightcurve']['number_of_bins'] = \ - len(self.data['daily_lightcurve']['spin_angle']) + self.data["daily_lightcurve"]["number_of_bins"] = len( + self.data["daily_lightcurve"]["spin_angle"] + ) - self.data['bad_time_flag_occurences'] = self.var['bad_time_flag_occurences'] + self.data["bad_time_flag_occurences"] = self.var["bad_time_flag_occurences"] def is_new_observational_day(self, l1b_data): """ @@ -278,14 +343,19 @@ def is_new_observational_day(self, l1b_data): l1b_data: L1b data provided in the form of data field of L1bDataHistogram object """ - is_new_observ_day = False # init + is_new_observ_day = False # init # check if is_night flag at L1b goes from True to False, if so then we have a new observational day - if self.var['is_night_for_previous_l1b'] >= 0: # we are at the second (or later) l1b file - if self.var['is_night_for_previous_l1b'] == 1 and l1b_data['flags']['is_night'] == False: # new observational day + if ( + self.var["is_night_for_previous_l1b"] >= 0 + ): # we are at the second (or later) l1b file + if ( + self.var["is_night_for_previous_l1b"] == 1 + and l1b_data["flags"]["is_night"] == False + ): # new observational day is_new_observ_day = True - self.var['is_night_for_previous_l1b'] = l1b_data['flags']['is_night'] + self.var["is_night_for_previous_l1b"] = l1b_data["flags"]["is_night"] return is_new_observ_day def process_l1b_data_files(self, l1b_data, pointing_counter): @@ -295,88 +365,109 @@ def process_l1b_data_files(self, l1b_data, pointing_counter): l1b_data: L1b data provided in the form of data field of L1bDataHistogram object """ - if self.var['l1b_file_counter_all'] == 0: + if self.var["l1b_file_counter_all"] == 0: # the start time of the first L1b file - self.var['l1b_start_time_all'] = l1b_data['imap_start_time'] + self.var["l1b_start_time_all"] = l1b_data["imap_start_time"] # generate spin-angle values for L2 for which spin=0 is measured from the # northernmost point of lightcurve, however it must have the same resolution as # l1b histograms so we can use L1b imap_spin_angle values for angular-grid generation - self.data['daily_lightcurve']['spin_angle'] = l1b_data['imap_spin_angle_bin_cntr'] - self.var['l1b_end_time_all'] = l1b_data['imap_start_time'] + l1b_data['imap_end_time_offset'] - self.var['number_of_bins_per_histogram'] = l1b_data['number_of_bins_per_histogram'] + self.data["daily_lightcurve"]["spin_angle"] = l1b_data[ + "imap_spin_angle_bin_cntr" + ] + self.var["l1b_end_time_all"] = ( + l1b_data["imap_start_time"] + l1b_data["imap_end_time_offset"] + ) + self.var["number_of_bins_per_histogram"] = l1b_data[ + "number_of_bins_per_histogram" + ] # increment all-l1b-files counter - self.var['l1b_file_counter_all'] += 1 + self.var["l1b_file_counter_all"] += 1 - for key in self.var['settings']['active_bad_time_flags'].keys(): - if l1b_data['flags'][key]: - self.var['bad_time_flag_occurences'][key] += 1 + for key in self.var["settings"]["active_bad_time_flags"].keys(): + if l1b_data["flags"][key]: + self.var["bad_time_flag_occurences"][key] += 1 if self._check_l1b_bad_time_flags(l1b_data): - - if self.var['l1b_file_counter_good_time'] <= 0: + if self.var["l1b_file_counter_good_time"] <= 0: # things to do when we have the first good-time L1b data file - self.data['header'] = { - 'flight_software_version': l1b_data['block_header']['flight_software_version'], - 'ground_software_version': VERSION, - 'pkts_file_name': l1b_data['block_header']['pkts_file_name'], - 'ancillary_data_files': l1b_data['block_header']['ancillary_data_files'] + self.data["header"] = { + "flight_software_version": l1b_data["block_header"][ + "flight_software_version" + ], + "ground_software_version": VERSION, + "pkts_file_name": l1b_data["block_header"]["pkts_file_name"], + "ancillary_data_files": l1b_data["block_header"][ + "ancillary_data_files" + ], } # we are here at the first good-time L1b file, so we set the current state of # the pointing counter as an unique identifier in L2 - self.data['identifier'] = pointing_counter + self.data["identifier"] = pointing_counter # the start time of the first good-time L1b file is needed for start time of L2 - self.var['l1b_start_time'] = l1b_data['imap_start_time'] + self.var["l1b_start_time"] = l1b_data["imap_start_time"] # initialize dictionary for computing averages over L1b files self._averages_init(l1b_data) # initialize daily helioglow-flux-lightcurve related stuff - self.data['daily_lightcurve']['photon_flux'] = \ - np.zeros_like(self.data['daily_lightcurve']['spin_angle'], float) - self.var['raw_histogram'] = \ - np.zeros_like(self.data['daily_lightcurve']['spin_angle'], int) - self.data['daily_lightcurve']['exposure_times'] = \ - np.zeros_like(self.data['daily_lightcurve']['spin_angle'], float) - self.data['daily_lightcurve']['flux_uncertainties'] = \ - np.zeros_like(self.data['daily_lightcurve']['spin_angle'], float) - self.data['daily_lightcurve']['histogram_flag_array'] = \ - np.zeros_like(self.data['daily_lightcurve']['spin_angle'], np.int16) + self.data["daily_lightcurve"]["photon_flux"] = np.zeros_like( + self.data["daily_lightcurve"]["spin_angle"], float + ) + self.var["raw_histogram"] = np.zeros_like( + self.data["daily_lightcurve"]["spin_angle"], int + ) + self.data["daily_lightcurve"]["exposure_times"] = np.zeros_like( + self.data["daily_lightcurve"]["spin_angle"], float + ) + self.data["daily_lightcurve"]["flux_uncertainties"] = np.zeros_like( + self.data["daily_lightcurve"]["spin_angle"], float + ) + self.data["daily_lightcurve"]["histogram_flag_array"] = np.zeros_like( + self.data["daily_lightcurve"]["spin_angle"], np.int16 + ) # keep track of the last end_time for good-time L1b files, it is needed for # the end time of L2 file - self.var['l1b_end_time'] = l1b_data['imap_start_time'] + l1b_data['imap_end_time_offset'] + self.var["l1b_end_time"] = ( + l1b_data["imap_start_time"] + l1b_data["imap_end_time_offset"] + ) # update dictionary for computing averages over L1b files self._averages_accumulate(l1b_data) # generate bad-angle mask for L2, if any L1b file has bitfield set to True, L2 does too - self.data['daily_lightcurve']['histogram_flag_array'] = \ - np.bitwise_and(int(self.var['bad_angle_mask'], 2), - np.bitwise_or(self.data['daily_lightcurve']['histogram_flag_array'], - l1b_data['histogram_flag_array'])) + self.data["daily_lightcurve"]["histogram_flag_array"] = np.bitwise_and( + int(self.var["bad_angle_mask"], 2), + np.bitwise_or( + self.data["daily_lightcurve"]["histogram_flag_array"], + l1b_data["histogram_flag_array"], + ), + ) # compute the exposure time per bin for currently processed L1b-data file # we must do it here because it can vary from one block to another - l1b_exposure_time_per_bin = l1b_data['spin_period_average'] * \ - l1b_data['number_of_spins_per_block'] / \ - l1b_data['number_of_bins_per_histogram'] + l1b_exposure_time_per_bin = ( + l1b_data["spin_period_average"] + * l1b_data["number_of_spins_per_block"] + / l1b_data["number_of_bins_per_histogram"] + ) # update raw L2 histogram and exposure times, note that block-to-block variations of the # position angle offset are not taken into account here, but if they will turn to be # significant we must include them somehow here - self.var['raw_histogram'] += np.array(l1b_data['histogram']) - self.data['daily_lightcurve']['exposure_times'] += l1b_exposure_time_per_bin + self.var["raw_histogram"] += np.array(l1b_data["histogram"]) + self.data["daily_lightcurve"]["exposure_times"] += l1b_exposure_time_per_bin # increment good-time counter for L1b files - self.var['l1b_file_counter_good_time'] += 1 + self.var["l1b_file_counter_good_time"] += 1 - if l1b_data['flags']['is_night'] == True: + if l1b_data["flags"]["is_night"] == True: # increment is-night-flag-raised counter for L1b files - self.var['is_night_counter'] += 1 + self.var["is_night_counter"] += 1 def read_from_file(self, file_name): """ @@ -394,16 +485,18 @@ def _read_from_json_file(self, file_name): Args: file_name: name of of file with L1b data """ - file_handler = open(file_name, 'r') + file_handler = open(file_name, "r") self.data = json.load(file_handler) file_handler.close() # hexadecimal values saved as strings require special treatment to make them integers - self.data['daily_lightcurve']['histogram_flag_array'] = \ - [int(x, base=16) for x in self.data['daily_lightcurve']['histogram_flag_array']] + self.data["daily_lightcurve"]["histogram_flag_array"] = [ + int(x, base=16) + for x in self.data["daily_lightcurve"]["histogram_flag_array"] + ] # additional field to keep track where data come from - self.data['l2_file_name'] = file_name + self.data["l2_file_name"] = file_name def save_data_to_file(self): """ @@ -413,8 +506,8 @@ def save_data_to_file(self): # modules for IMAP will be provided by SDC self._save_data_to_json_file() - #sys.stdout.flush() - #time.sleep(5) + # sys.stdout.flush() + # time.sleep(5) def _save_data_to_json_file(self): """ @@ -423,39 +516,39 @@ def _save_data_to_json_file(self): """ # define formats for printing values to JSON file fmt = { - 'daily_lightcurve': { - 'spin_angle': '%.3f', - 'photon_flux': '%.1f', - 'raw_histogram': '%d', - 'exposure_times': '%.3e', - 'flux_uncertainties': '%.3e', - 'ecliptic_lon': '%.3f', - 'ecliptic_lat': '%.3f' + "daily_lightcurve": { + "spin_angle": "%.3f", + "photon_flux": "%.1f", + "raw_histogram": "%d", + "exposure_times": "%.3e", + "flux_uncertainties": "%.3e", + "ecliptic_lon": "%.3f", + "ecliptic_lat": "%.3f", + }, + "scalar variables": { + "filter_temperature_average": "%.2f", + "filter_temperature_std_dev": "%.3e", + "hv_voltage_average": "%.1f", + "hv_voltage_std_dev": "%.3e", + "spin_period_average": "%.7f", + "spin_period_std_dev": "%.3e", + "spin_period_ground_average": "%.7f", + "spin_period_ground_std_dev": "%.3e", + "pulse_length_average": "%.3e", + "pulse_length_std_dev": "%.3e", + "position_angle_offset_average": "%.3f", + "position_angle_offset_std_dev": "%.3e", }, - 'scalar variables': { - 'filter_temperature_average': '%.2f', - 'filter_temperature_std_dev': '%.3e', - 'hv_voltage_average': '%.1f', - 'hv_voltage_std_dev': '%.3e', - 'spin_period_average': '%.7f', - 'spin_period_std_dev': '%.3e', - 'spin_period_ground_average': '%.7f', - 'spin_period_ground_std_dev': '%.3e', - 'pulse_length_average': '%.3e', - 'pulse_length_std_dev': '%.3e', - 'position_angle_offset_average': '%.3f', - 'position_angle_offset_std_dev': '%.3e' + "lon lat variables": { + "spin_axis_orientation_average": "%.3f", + "spin_axis_orientation_std_dev": "%.3e", }, - 'lon lat variables': { - 'spin_axis_orientation_average': '%.3f', - 'spin_axis_orientation_std_dev': '%.3e' + "vector variables": { + "spacecraft_location_average": "%.1f", + "spacecraft_location_std_dev": "%.3e", + "spacecraft_velocity_average": "%.3f", + "spacecraft_velocity_std_dev": "%.3e", }, - 'vector variables': { - 'spacecraft_location_average': '%.1f', - 'spacecraft_location_std_dev': '%.3e', - 'spacecraft_velocity_average': '%.3f', - 'spacecraft_velocity_std_dev': '%.3e' - } } # create local copy for printing purposes @@ -465,37 +558,57 @@ def _save_data_to_json_file(self): # them to be saved as one-liners (i.e., without \n after each element) or we want them to # be printed with a specific format by json.dumps() # so we convert variables to strings as defined in fmt - for key in fmt['daily_lightcurve'].keys(): - data['daily_lightcurve'][key] = \ - '[' + ', '.join(map(lambda x: fmt['daily_lightcurve'][key] % x, - list(data['daily_lightcurve'][key]))) + ']' - for key in fmt['scalar variables'].keys(): - data[key] = '|'+ fmt['scalar variables'][key] % data[key] +'|' - for key in fmt['lon lat variables'].keys(): - for sec_key in ['lon', 'lat']: - data[key][sec_key] = '|'+ fmt['lon lat variables'][key] % data[key][sec_key] +'|' - for key in fmt['vector variables'].keys(): - for sec_key in ['x', 'y', 'z']: - data[key][sec_key] = '|'+ fmt['vector variables'][key] % data[key][sec_key] +'|' + for key in fmt["daily_lightcurve"].keys(): + data["daily_lightcurve"][key] = ( + "[" + + ", ".join( + map( + lambda x: fmt["daily_lightcurve"][key] % x, + list(data["daily_lightcurve"][key]), + ) + ) + + "]" + ) + for key in fmt["scalar variables"].keys(): + data[key] = "|" + fmt["scalar variables"][key] % data[key] + "|" + for key in fmt["lon lat variables"].keys(): + for sec_key in ["lon", "lat"]: + data[key][sec_key] = ( + "|" + fmt["lon lat variables"][key] % data[key][sec_key] + "|" + ) + for key in fmt["vector variables"].keys(): + for sec_key in ["x", "y", "z"]: + data[key][sec_key] = ( + "|" + fmt["vector variables"][key] % data[key][sec_key] + "|" + ) # histogram_flag_array requires special hexadecimal treatment - data['daily_lightcurve']['histogram_flag_array'] = '[' + \ - ', '.join(map(lambda x: '"0x' + format(x, 'x').zfill(4) +'"', - list(data['daily_lightcurve']['histogram_flag_array']))) + ']' + data["daily_lightcurve"]["histogram_flag_array"] = ( + "[" + + ", ".join( + map( + lambda x: '"0x' + format(x, "x").zfill(4) + '"', + list(data["daily_lightcurve"]["histogram_flag_array"]), + ) + ) + + "]" + ) # generate JSON content to be saved in output file json_content = json.dumps(data, indent=4, default=vars) # some corrections to get numbers back - json_content = json_content.replace('"[', '[').replace(']"', ']') - json_content = json_content.replace('|"', '').replace('"|', '') + json_content = json_content.replace('"[', "[").replace(']"', "]") + json_content = json_content.replace('|"', "").replace('"|', "") json_content = json_content.replace('\\"', '"') # the output file name is constructed using a convention defined in Sec. 1.4.2 of # "IMAP SDC to Instrument Team ICD" - date_utc = ''.join(filter(str.isdigit, self.data['start_time'])) - file_name = 'data_l2_histograms/imap_glows_l2_%s_orbX_modX_p_v00.json' % date_utc + date_utc = "".join(filter(str.isdigit, self.data["start_time"])) + file_name = ( + "data_l2_histograms/imap_glows_l2_%s_orbX_modX_p_v00.json" % date_utc + ) - file_handler = open(file_name, 'w') + file_handler = open(file_name, "w") print(json_content, file=file_handler) file_handler.close() diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/l3a_data.py b/imap_l3_processing/glows/l3a/glows_toolkit/l3a_data.py index 451788084..a0d91a276 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/l3a_data.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/l3a_data.py @@ -1,6 +1,7 @@ """@package docstring Author: Marek Strumik, maro at cbk.waw.pl """ + import copy import json @@ -13,7 +14,7 @@ from .geom_fun_libr_box import spin2lb -class L3aData(): +class L3aData: """ L3aData() class for GLOWS-L3a data """ @@ -24,20 +25,20 @@ def __init__(self, anc_input_from_instr_team): """ # self.data is a dictionary to be saved as L2 data self.data = {} - self.data['header'] = {} - self.data['identifier'] = None - self.data['start_time'] = None - self.data['end_time'] = None - self.data['daily_lightcurve'] = {} + self.data["header"] = {} + self.data["identifier"] = None + self.data["start_time"] = None + self.data["end_time"] = None + self.data["daily_lightcurve"] = {} # self.var is for various variables needed for computations, but not to be saved in L3a output file self.var = {} # read pipeline settings to be used for L3a - file_handler = open(anc_input_from_instr_team['settings'], 'r') + file_handler = open(anc_input_from_instr_team["settings"], "r") settings = json.load(file_handler) file_handler.close() - self.var['nominal_number_of_bins'] = settings['l3a_nominal_number_of_bins'] + self.var["nominal_number_of_bins"] = settings["l3a_nominal_number_of_bins"] def _compute_extra_heliospheric_bckgrd(self, anc_input_from_instr_team): """ @@ -48,14 +49,17 @@ def _compute_extra_heliospheric_bckgrd(self, anc_input_from_instr_team): see ANC_INPUT_FROM_INSTRUMENT_TEAM in constants.py """ - if check_if_contains_actual_data(anc_input_from_instr_team['extra_heliospheric_bckgrd']): - - fh = open(anc_input_from_instr_team['extra_heliospheric_bckgrd'], 'r') + if check_if_contains_actual_data( + anc_input_from_instr_team["extra_heliospheric_bckgrd"] + ): + fh = open(anc_input_from_instr_team["extra_heliospheric_bckgrd"], "r") for i in range(3): line = fh.readline() # sanity check - if line.split()[4] != 'NSIDE': - raise Exception('error _compute_extra_heliospheric_bckgrd(): NSIDE string not found') + if line.split()[4] != "NSIDE": + raise Exception( + "error _compute_extra_heliospheric_bckgrd(): NSIDE string not found" + ) # read NSIDE parameter for healpix grid nside = int(line.split()[6]) # read ecliptic coords and background values span on a healpix grid @@ -63,23 +67,30 @@ def _compute_extra_heliospheric_bckgrd(self, anc_input_from_instr_team): fh.close() # generate Healpix grid internally here for a sanity check performed later - ahp = HEALPix(nside=nside, order='ring') + ahp = HEALPix(nside=nside, order="ring") lon, lat = ahp.healpix_to_lonlat(range(ahp.npix)) lon = np.degrees(lon.value) lat = np.degrees(lat.value) # sanity check for the Healpix grid read from anc_input_from_instr_team['extra_heliospheric_bckgrd'] # it should match the Healpix grid generated internally, otherwise raise error - if np.fabs(lon - lon_lat_bckgrd[:, 0]).max() > 1.0e-6 or \ - np.fabs(lat - lon_lat_bckgrd[:, 1]).max() > 1.0e-6: - raise Exception('error _compute_extra_heliospheric_bckgrd(): ' + \ - 'healpix grid mismatch (NSIDE wrong? NESTED ordering instead of RING?)') + if ( + np.fabs(lon - lon_lat_bckgrd[:, 0]).max() > 1.0e-6 + or np.fabs(lat - lon_lat_bckgrd[:, 1]).max() > 1.0e-6 + ): + raise Exception( + "error _compute_extra_heliospheric_bckgrd(): " + + "healpix grid mismatch (NSIDE wrong? NESTED ordering instead of RING?)" + ) # interpolate the background estimate from the healpix map into the centers of the scanning circle bins - self.data['daily_lightcurve']['extra_heliospheric_bckgrd'] = \ - ahp.interpolate_bilinear_lonlat(self.data['daily_lightcurve']['ecliptic_lon'] * u.deg, - self.data['daily_lightcurve']['ecliptic_lat'] * u.deg, - lon_lat_bckgrd[:, 2]) + self.data["daily_lightcurve"]["extra_heliospheric_bckgrd"] = ( + ahp.interpolate_bilinear_lonlat( + self.data["daily_lightcurve"]["ecliptic_lon"] * u.deg, + self.data["daily_lightcurve"]["ecliptic_lat"] * u.deg, + lon_lat_bckgrd[:, 2], + ) + ) def _compute_time_dependent_bckgrd(self, anc_input_from_instr_team): """ @@ -90,54 +101,85 @@ def _compute_time_dependent_bckgrd(self, anc_input_from_instr_team): see ANC_INPUT_FROM_INSTRUMENT_TEAM in constants.py """ - if check_if_contains_actual_data(anc_input_from_instr_team['time_dependent_bckgrd']): - + if check_if_contains_actual_data( + anc_input_from_instr_team["time_dependent_bckgrd"] + ): # read the size of the background-estimate array - fh = open(anc_input_from_instr_team['time_dependent_bckgrd'], 'r') + fh = open(anc_input_from_instr_team["time_dependent_bckgrd"], "r") for i in range(2): line = fh.readline() # sanity check - if line.split()[5] != 'array' or line.split()[6] != 'size' or line.split()[7] != '=': + if ( + line.split()[5] != "array" + or line.split()[6] != "size" + or line.split()[7] != "=" + ): raise Exception( - 'error _compute_time_dependent_bckgrd(): input file header does not contain expected keywords (1)') + "error _compute_time_dependent_bckgrd(): input file header does not contain expected keywords (1)" + ) # read the number of bins bckgrd_array_size = int(line.split()[8]) # sanity check if bckgrd_array_size != 3600: raise Exception( - 'error _compute_time_dependent_bckgrd(): array size in input file header is not equal to 3600') + "error _compute_time_dependent_bckgrd(): array size in input file header is not equal to 3600" + ) # read the array of spin angles for which the background was estimated line = fh.readline() # sanity check - if line.split()[1] != 'spin' or line.split()[2] != 'angle' or line.split()[3] != 'grid:': + if ( + line.split()[1] != "spin" + or line.split()[2] != "angle" + or line.split()[3] != "grid:" + ): raise Exception( - 'error _compute_time_dependent_bckgrd(): input file header does not contain expected keywords (2)') - bckgrd_array_spin_angle = np.fromstring(line[18:], dtype=float, sep=' ') + "error _compute_time_dependent_bckgrd(): input file header does not contain expected keywords (2)" + ) + bckgrd_array_spin_angle = np.fromstring(line[18:], dtype=float, sep=" ") fh.close() # read L3a lightcurve start dates and array of corrections - identifiers = np.loadtxt(anc_input_from_instr_team['time_dependent_bckgrd'], usecols=[0], dtype=str) - start_dates = np.loadtxt(anc_input_from_instr_team['time_dependent_bckgrd'], usecols=[1], dtype=str, ndmin=1) - bckgrd_arrays = np.loadtxt(anc_input_from_instr_team['time_dependent_bckgrd'], - usecols=range(2, bckgrd_array_size + 2), ndmin=2) - idx = np.nonzero(self.data['start_time'].replace(' ', 'T') == start_dates)[0] - print(' tdbckgrd idx:', idx, ' ', end='') + identifiers = np.loadtxt( + anc_input_from_instr_team["time_dependent_bckgrd"], + usecols=[0], + dtype=str, + ) + start_dates = np.loadtxt( + anc_input_from_instr_team["time_dependent_bckgrd"], + usecols=[1], + dtype=str, + ndmin=1, + ) + bckgrd_arrays = np.loadtxt( + anc_input_from_instr_team["time_dependent_bckgrd"], + usecols=range(2, bckgrd_array_size + 2), + ndmin=2, + ) + idx = np.nonzero(self.data["start_time"].replace(" ", "T") == start_dates)[ + 0 + ] + print(" tdbckgrd idx:", idx, " ", end="") if idx.size == 0: # corrections has not been found for the current date, so we set zeros - self.data['daily_lightcurve']['time_dependent_bckgrd'] = \ - np.zeros_like(self.data['daily_lightcurve']['spin_angle'], float) + self.data["daily_lightcurve"]["time_dependent_bckgrd"] = np.zeros_like( + self.data["daily_lightcurve"]["spin_angle"], float + ) else: idx = idx[0] if idx.size != 1: raise Exception( - 'error _compute_time_dependent_bckgrd(): not-found or multiple-definition error for ' + - 'the start date %s' % self.data['start_time'].replace(' ', 'T')) + "error _compute_time_dependent_bckgrd(): not-found or multiple-definition error for " + + "the start date %s" + % self.data["start_time"].replace(" ", "T") + ) # interpolate the background estimate from the bckgrd_array into the centers of the scanning circle bins - self.data['daily_lightcurve']['time_dependent_bckgrd'] = \ - np.interp(self.data['daily_lightcurve']['spin_angle'], bckgrd_array_spin_angle, - bckgrd_arrays[idx, :]) + self.data["daily_lightcurve"]["time_dependent_bckgrd"] = np.interp( + self.data["daily_lightcurve"]["spin_angle"], + bckgrd_array_spin_angle, + bckgrd_arrays[idx, :], + ) def generate_l3a_data(self, anc_input_from_instr_team): """ @@ -148,39 +190,57 @@ def generate_l3a_data(self, anc_input_from_instr_team): """ # compute count rate for L3a - self.var['count_rate'] = np.zeros_like(self.data['daily_lightcurve']['raw_histogram'], float) - good_angle_idxs = np.nonzero(np.array(self.data['daily_lightcurve']['exposure_times']) > 0)[0] - self.var['count_rate'][good_angle_idxs] = \ - self.data['daily_lightcurve']['raw_histogram'][good_angle_idxs] / \ - self.data['daily_lightcurve']['exposure_times'][good_angle_idxs] + self.var["count_rate"] = np.zeros_like( + self.data["daily_lightcurve"]["raw_histogram"], float + ) + good_angle_idxs = np.nonzero( + np.array(self.data["daily_lightcurve"]["exposure_times"]) > 0 + )[0] + self.var["count_rate"][good_angle_idxs] = ( + self.data["daily_lightcurve"]["raw_histogram"][good_angle_idxs] + / self.data["daily_lightcurve"]["exposure_times"][good_angle_idxs] + ) # compute calibration factor - cps_per_R = calibration_factor(anc_input_from_instr_team['calibration_data'], - self.data['start_time'].replace(' ', 'T')) - print(' cps/R:', cps_per_R, ' ', end='') + cps_per_R = calibration_factor( + anc_input_from_instr_team["calibration_data"], + self.data["start_time"].replace(" ", "T"), + ) + print(" cps/R:", cps_per_R, " ", end="") # convert count rate to photon flux - self.data['daily_lightcurve']['photon_flux'] = self.var['count_rate'] / cps_per_R + self.data["daily_lightcurve"]["photon_flux"] = ( + self.var["count_rate"] / cps_per_R + ) # compute uncertainties - self.var['raw_uncertainties'] = np.sqrt(self.data['daily_lightcurve']['raw_histogram']) - self.data['daily_lightcurve']['flux_uncertainties'] = np.zeros_like( - self.data['daily_lightcurve']['raw_histogram'], float) - self.data['daily_lightcurve']['flux_uncertainties'][good_angle_idxs] = \ - self.var['raw_uncertainties'][good_angle_idxs] / \ - np.array(self.data['daily_lightcurve']['exposure_times'])[good_angle_idxs] - self.data['daily_lightcurve']['flux_uncertainties'] = \ - self.data['daily_lightcurve']['flux_uncertainties'] / cps_per_R + self.var["raw_uncertainties"] = np.sqrt( + self.data["daily_lightcurve"]["raw_histogram"] + ) + self.data["daily_lightcurve"]["flux_uncertainties"] = np.zeros_like( + self.data["daily_lightcurve"]["raw_histogram"], float + ) + self.data["daily_lightcurve"]["flux_uncertainties"][good_angle_idxs] = ( + self.var["raw_uncertainties"][good_angle_idxs] + / np.array(self.data["daily_lightcurve"]["exposure_times"])[good_angle_idxs] + ) + self.data["daily_lightcurve"]["flux_uncertainties"] = ( + self.data["daily_lightcurve"]["flux_uncertainties"] / cps_per_R + ) # compute ecliptic longitude and latitude of histogram bin centers - ones = np.ones_like(self.data['daily_lightcurve']['spin_angle'], float) - histogram_bins_ecl_coords = np.degrees(spin2lb( - np.radians(self.data['spin_axis_orientation_average']['lon']) * ones, - np.radians(self.data['spin_axis_orientation_average']['lat']) * ones, - np.radians(self.data['daily_lightcurve']['spin_angle']), 0.0, - np.radians(SCAN_CIRCLE_ANGULAR_RADIUS) * ones)) - self.data['daily_lightcurve']['ecliptic_lon'] = histogram_bins_ecl_coords[:, 0] - self.data['daily_lightcurve']['ecliptic_lat'] = histogram_bins_ecl_coords[:, 1] + ones = np.ones_like(self.data["daily_lightcurve"]["spin_angle"], float) + histogram_bins_ecl_coords = np.degrees( + spin2lb( + np.radians(self.data["spin_axis_orientation_average"]["lon"]) * ones, + np.radians(self.data["spin_axis_orientation_average"]["lat"]) * ones, + np.radians(self.data["daily_lightcurve"]["spin_angle"]), + 0.0, + np.radians(SCAN_CIRCLE_ANGULAR_RADIUS) * ones, + ) + ) + self.data["daily_lightcurve"]["ecliptic_lon"] = histogram_bins_ecl_coords[:, 0] + self.data["daily_lightcurve"]["ecliptic_lat"] = histogram_bins_ecl_coords[:, 1] # compute an estimate of extra-heliospheric background self._compute_extra_heliospheric_bckgrd(anc_input_from_instr_team) @@ -192,10 +252,11 @@ def generate_l3a_data(self, anc_input_from_instr_team): self._remove_masked_bins() # add the number of bins as a key to be saved in L3a output file - self.data['daily_lightcurve']['number_of_bins'] = \ - len(self.data['daily_lightcurve']['spin_angle']) + self.data["daily_lightcurve"]["number_of_bins"] = len( + self.data["daily_lightcurve"]["spin_angle"] + ) - print('... done') + print("... done") def process_l2_data_file(self, l2_data): """ @@ -205,62 +266,97 @@ def process_l2_data_file(self, l2_data): """ # prepare quality_flag_array for L2, where 0 means bad data and 1 means good data - l2_quality_flag_array = np.ones_like(l2_data['daily_lightcurve']['histogram_flag_array'], dtype=int) - bad_angle_idxs = np.nonzero(np.array(l2_data['daily_lightcurve']['histogram_flag_array']) > 0)[0] + l2_quality_flag_array = np.ones_like( + l2_data["daily_lightcurve"]["histogram_flag_array"], dtype=int + ) + bad_angle_idxs = np.nonzero( + np.array(l2_data["daily_lightcurve"]["histogram_flag_array"]) > 0 + )[0] l2_quality_flag_array[bad_angle_idxs] = 0 # copy ancillary date from L2 to L3a - PART 1 - for key in ['identifier', 'start_time', 'end_time', 'filter_temperature_average', 'filter_temperature_std_dev', - 'hv_voltage_average', 'hv_voltage_std_dev', 'spin_period_average', 'spin_period_std_dev', - 'spin_period_ground_average', 'spin_period_ground_std_dev', 'pulse_length_average', - 'pulse_length_std_dev', - 'position_angle_offset_average', 'position_angle_offset_std_dev']: + for key in [ + "identifier", + "start_time", + "end_time", + "filter_temperature_average", + "filter_temperature_std_dev", + "hv_voltage_average", + "hv_voltage_std_dev", + "spin_period_average", + "spin_period_std_dev", + "spin_period_ground_average", + "spin_period_ground_std_dev", + "pulse_length_average", + "pulse_length_std_dev", + "position_angle_offset_average", + "position_angle_offset_std_dev", + ]: self.data[key] = l2_data[key] # copy ancillary date from L2 to L3a - PART 2 - for key1 in ['spin_axis_orientation_average', 'spin_axis_orientation_std_dev']: + for key1 in ["spin_axis_orientation_average", "spin_axis_orientation_std_dev"]: self.data[key1] = {} - for key2 in ['lon', 'lat']: + for key2 in ["lon", "lat"]: self.data[key1][key2] = l2_data[key1][key2] # copy ancillary date from L2 to L3a - PART 3 - for key1 in ['spacecraft_location_average', 'spacecraft_location_std_dev', - 'spacecraft_velocity_average', 'spacecraft_velocity_std_dev']: + for key1 in [ + "spacecraft_location_average", + "spacecraft_location_std_dev", + "spacecraft_velocity_average", + "spacecraft_velocity_std_dev", + ]: self.data[key1] = {} - for key2 in ['x', 'y', 'z']: + for key2 in ["x", "y", "z"]: self.data[key1][key2] = l2_data[key1][key2] # generate the nominal spin angle grid for L3a - bin_width = 360.0 / self.var['nominal_number_of_bins'] - self.data['daily_lightcurve']['spin_angle'] = np.linspace(0.5 * bin_width, - 360.0 - 0.5 * bin_width, - self.var['nominal_number_of_bins']) + bin_width = 360.0 / self.var["nominal_number_of_bins"] + self.data["daily_lightcurve"]["spin_angle"] = np.linspace( + 0.5 * bin_width, 360.0 - 0.5 * bin_width, self.var["nominal_number_of_bins"] + ) # compute raw counts and exposure times for low-resolution L3a histogram - self.data['daily_lightcurve']['raw_histogram'] = [] - self.data['daily_lightcurve']['exposure_times'] = [] - for i in range(self.var['nominal_number_of_bins']): + self.data["daily_lightcurve"]["raw_histogram"] = [] + self.data["daily_lightcurve"]["exposure_times"] = [] + for i in range(self.var["nominal_number_of_bins"]): # find indexes of l2 bins falling into i-th l3a bin - idxs = np.nonzero(np.logical_and( - l2_data['daily_lightcurve']['spin_angle'] > self.data['daily_lightcurve']['spin_angle'][ - i] - 0.5 * bin_width, - l2_data['daily_lightcurve']['spin_angle'] <= self.data['daily_lightcurve']['spin_angle'][ - i] + 0.5 * bin_width))[0] + idxs = np.nonzero( + np.logical_and( + l2_data["daily_lightcurve"]["spin_angle"] + > self.data["daily_lightcurve"]["spin_angle"][i] - 0.5 * bin_width, + l2_data["daily_lightcurve"]["spin_angle"] + <= self.data["daily_lightcurve"]["spin_angle"][i] + 0.5 * bin_width, + ) + )[0] # compute raw counts - self.data['daily_lightcurve']['raw_histogram'].append( - np.sum(np.array(l2_data['daily_lightcurve']['raw_histogram'])[idxs] * l2_quality_flag_array[idxs])) + self.data["daily_lightcurve"]["raw_histogram"].append( + np.sum( + np.array(l2_data["daily_lightcurve"]["raw_histogram"])[idxs] + * l2_quality_flag_array[idxs] + ) + ) # compute exposure times - self.data['daily_lightcurve']['exposure_times'].append( - np.sum(np.array(l2_data['daily_lightcurve']['exposure_times'])[idxs] * l2_quality_flag_array[idxs])) - self.data['daily_lightcurve']['raw_histogram'] = np.array(self.data['daily_lightcurve']['raw_histogram']) - self.data['daily_lightcurve']['exposure_times'] = np.array(self.data['daily_lightcurve']['exposure_times']) - - self.data['header'] = { - 'flight_software_version': l2_data['header']['flight_software_version'], - 'ground_software_version': VERSION, - 'pkts_file_name': l2_data['header']['pkts_file_name'], - 'ancillary_data_files': l2_data['header']['ancillary_data_files'], - 'l2_input_file_name': l2_data['l2_file_name'] + self.data["daily_lightcurve"]["exposure_times"].append( + np.sum( + np.array(l2_data["daily_lightcurve"]["exposure_times"])[idxs] + * l2_quality_flag_array[idxs] + ) + ) + self.data["daily_lightcurve"]["raw_histogram"] = np.array( + self.data["daily_lightcurve"]["raw_histogram"] + ) + self.data["daily_lightcurve"]["exposure_times"] = np.array( + self.data["daily_lightcurve"]["exposure_times"] + ) + + self.data["header"] = { + "flight_software_version": l2_data["header"]["flight_software_version"], + "ground_software_version": VERSION, + "pkts_file_name": l2_data["header"]["pkts_file_name"], + "ancillary_data_files": l2_data["header"]["ancillary_data_files"], + "l2_input_file_name": l2_data["l2_file_name"], } def read_from_file(self, file_name): @@ -279,20 +375,24 @@ def _read_from_json_file(self, file_name): Args: file_name: name of of file with L1b data """ - file_handler = open(file_name, 'r') + file_handler = open(file_name, "r") self.data = json.load(file_handler) file_handler.close() # additional field to keep track where data come from - self.data['l3a_file_name'] = file_name + self.data["l3a_file_name"] = file_name def _remove_masked_bins(self): """ Remove masked bins from L3a lightcurve """ - good_angle_idxs = np.nonzero(np.array(self.data['daily_lightcurve']['exposure_times']) > 0)[0] - for key in self.data['daily_lightcurve'].keys(): - self.data['daily_lightcurve'][key] = self.data['daily_lightcurve'][key][good_angle_idxs] + good_angle_idxs = np.nonzero( + np.array(self.data["daily_lightcurve"]["exposure_times"]) > 0 + )[0] + for key in self.data["daily_lightcurve"].keys(): + self.data["daily_lightcurve"][key] = self.data["daily_lightcurve"][key][ + good_angle_idxs + ] def save_data_to_file(self): """ @@ -309,41 +409,41 @@ def _save_data_to_json_file(self): """ # define formats for printing values to JSON file fmt = { - 'daily_lightcurve': { - 'spin_angle': '%.3f', - 'photon_flux': '%.1f', - 'raw_histogram': '%d', - 'extra_heliospheric_bckgrd': '%.2f', - 'time_dependent_bckgrd': '%.2f', - 'exposure_times': '%.3e', - 'flux_uncertainties': '%.3e', - 'ecliptic_lon': '%.3f', - 'ecliptic_lat': '%.3f' + "daily_lightcurve": { + "spin_angle": "%.3f", + "photon_flux": "%.1f", + "raw_histogram": "%d", + "extra_heliospheric_bckgrd": "%.2f", + "time_dependent_bckgrd": "%.2f", + "exposure_times": "%.3e", + "flux_uncertainties": "%.3e", + "ecliptic_lon": "%.3f", + "ecliptic_lat": "%.3f", + }, + "scalar variables": { + "filter_temperature_average": "%.2f", + "filter_temperature_std_dev": "%.3e", + "hv_voltage_average": "%.1f", + "hv_voltage_std_dev": "%.3e", + "spin_period_average": "%.7f", + "spin_period_std_dev": "%.3e", + "spin_period_ground_average": "%.7f", + "spin_period_ground_std_dev": "%.3e", + "pulse_length_average": "%.3e", + "pulse_length_std_dev": "%.3e", + "position_angle_offset_average": "%.3f", + "position_angle_offset_std_dev": "%.3e", }, - 'scalar variables': { - 'filter_temperature_average': '%.2f', - 'filter_temperature_std_dev': '%.3e', - 'hv_voltage_average': '%.1f', - 'hv_voltage_std_dev': '%.3e', - 'spin_period_average': '%.7f', - 'spin_period_std_dev': '%.3e', - 'spin_period_ground_average': '%.7f', - 'spin_period_ground_std_dev': '%.3e', - 'pulse_length_average': '%.3e', - 'pulse_length_std_dev': '%.3e', - 'position_angle_offset_average': '%.3f', - 'position_angle_offset_std_dev': '%.3e' + "lon lat variables": { + "spin_axis_orientation_average": "%.3f", + "spin_axis_orientation_std_dev": "%.3e", }, - 'lon lat variables': { - 'spin_axis_orientation_average': '%.3f', - 'spin_axis_orientation_std_dev': '%.3e' + "vector variables": { + "spacecraft_location_average": "%.1f", + "spacecraft_location_std_dev": "%.3e", + "spacecraft_velocity_average": "%.3f", + "spacecraft_velocity_std_dev": "%.3e", }, - 'vector variables': { - 'spacecraft_location_average': '%.1f', - 'spacecraft_location_std_dev': '%.3e', - 'spacecraft_velocity_average': '%.3f', - 'spacecraft_velocity_std_dev': '%.3e' - } } # create local copy for printing purposes @@ -353,33 +453,44 @@ def _save_data_to_json_file(self): # them to be saved as one-liners (i.e., without \n after each element) or we want them to # be printed with a specific format by json.dumps() # so we convert variables to strings as defined in fmt - for key in fmt['daily_lightcurve'].keys(): - data['daily_lightcurve'][key] = \ - '[' + ', '.join(map(lambda x: fmt['daily_lightcurve'][key] % x, - list(data['daily_lightcurve'][key]))) + ']' - for key in fmt['scalar variables'].keys(): - data[key] = '|' + fmt['scalar variables'][key] % data[key] + '|' - for key in fmt['lon lat variables'].keys(): - for sec_key in ['lon', 'lat']: - data[key][sec_key] = '|' + fmt['lon lat variables'][key] % data[key][sec_key] + '|' - for key in fmt['vector variables'].keys(): - for sec_key in ['x', 'y', 'z']: - data[key][sec_key] = '|' + fmt['vector variables'][key] % data[key][sec_key] + '|' + for key in fmt["daily_lightcurve"].keys(): + data["daily_lightcurve"][key] = ( + "[" + + ", ".join( + map( + lambda x: fmt["daily_lightcurve"][key] % x, + list(data["daily_lightcurve"][key]), + ) + ) + + "]" + ) + for key in fmt["scalar variables"].keys(): + data[key] = "|" + fmt["scalar variables"][key] % data[key] + "|" + for key in fmt["lon lat variables"].keys(): + for sec_key in ["lon", "lat"]: + data[key][sec_key] = ( + "|" + fmt["lon lat variables"][key] % data[key][sec_key] + "|" + ) + for key in fmt["vector variables"].keys(): + for sec_key in ["x", "y", "z"]: + data[key][sec_key] = ( + "|" + fmt["vector variables"][key] % data[key][sec_key] + "|" + ) # generate JSON content to be saved in output file json_content = json.dumps(data, indent=4, default=vars) # some corrections to get numbers back - json_content = json_content.replace('"[', '[').replace(']"', ']') - json_content = json_content.replace('|"', '').replace('"|', '') + json_content = json_content.replace('"[', "[").replace(']"', "]") + json_content = json_content.replace('|"', "").replace('"|', "") json_content = json_content.replace('\\"', '"') # the output file name is constructed using a convention defined in Sec. 1.4.2 of # "IMAP SDC to Instrument Team ICD" # date_utc = ''.join(filter(str.isdigit, self.data['identifier'])) - date_utc = ''.join(filter(str.isdigit, self.data['start_time'])) - file_name = 'data_l3a/imap_glows_l3a_%s_orbX_modX_p_v00.json' % date_utc + date_utc = "".join(filter(str.isdigit, self.data["start_time"])) + file_name = "data_l3a/imap_glows_l3a_%s_orbX_modX_p_v00.json" % date_utc - file_handler = open(file_name, 'w') + file_handler = open(file_name, "w") print(json_content, file=file_handler) file_handler.close() diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/reader.py b/imap_l3_processing/glows/l3a/glows_toolkit/reader.py index ee13298fc..0adb5736b 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/reader.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/reader.py @@ -1,9 +1,11 @@ """@package docstring Author: KPLabs with some modifications by Marek Strumik, maro at cbk.waw.pl """ + import struct from typing import Any, cast, Tuple, Union + class Reader: """ Reader() class for reading binary stream with GLOWS data @@ -19,18 +21,18 @@ def __init__(self, buffer: Union[memoryview, bytes]): @property def overrun(self) -> bool: - """ overrun property """ + """overrun property""" return self._overrun @property def remaining_size(self) -> int: - """ remaining_size property""" + """remaining_size property""" if self._overrun: return 0 return len(self._buffer) - self._offset def _read_part(self, length: int) -> bytes: - """ read bytes from binary stream """ + """read bytes from binary stream""" if self._overrun: return bytes([0] * length) @@ -38,33 +40,33 @@ def _read_part(self, length: int) -> bytes: self._overrun = True return bytes([0] * length) - part = self._buffer[self._offset:self._offset + length] + part = self._buffer[self._offset : self._offset + length] self._offset += length return part def read(self, pack_format: str) -> Tuple[Any, ...]: - """ read bytes according to pack_format """ + """read bytes according to pack_format""" length = struct.calcsize(pack_format) part = self._read_part(length) return struct.unpack(pack_format, part) def read_uint8(self) -> int: - """ read one byte as uint """ - return cast(int, self.read(' int: - """ read 2 bytes as big-endian uint """ - return cast(int, self.read('>H')[0]) + """read 2 bytes as big-endian uint""" + return cast(int, self.read(">H")[0]) def read_uint24_be(self) -> int: - """ read 3 bytes as big-endian uint """ - high, low = cast(Tuple[int, int], self.read('>BH')) + """read 3 bytes as big-endian uint""" + high, low = cast(Tuple[int, int], self.read(">BH")) return high << 16 | low def read_uint32_be(self) -> int: - """ read 4 bytes as big-endian uint """ - return cast(int, self.read('>L')[0]) + """read 4 bytes as big-endian uint""" + return cast(int, self.read(">L")[0]) def read_buffer(self, size: int) -> bytes: - """ read bytes from binary stream """ + """read bytes from binary stream""" return self._read_part(size) diff --git a/imap_l3_processing/glows/l3a/glows_toolkit/time.py b/imap_l3_processing/glows/l3a/glows_toolkit/time.py index cb471709e..0863c336f 100644 --- a/imap_l3_processing/glows/l3a/glows_toolkit/time.py +++ b/imap_l3_processing/glows/l3a/glows_toolkit/time.py @@ -1,12 +1,15 @@ """@package docstring Author: Marek Strumik, maro at cbk.waw.pl """ + from dataclasses import dataclass + @dataclass(frozen=True) class Time: """ Time() class for IMAP and GLOWS times (with subseconds) """ + seconds: int subseconds: int diff --git a/imap_l3_processing/glows/l3a/models.py b/imap_l3_processing/glows/l3a/models.py index b4d75894d..64108452b 100644 --- a/imap_l3_processing/glows/l3a/models.py +++ b/imap_l3_processing/glows/l3a/models.py @@ -7,11 +7,11 @@ from imap_l3_processing.models import DataProduct, DataProductVariable REPOINTING_CDF_VAR_NAME = "identifier" -PHOTON_FLUX_CDF_VAR_NAME = 'photon_flux' -PHOTON_FLUX_UNCERTAINTY_CDF_VAR_NAME = 'photon_flux_uncertainty' -RAW_HISTOGRAM_CDF_VAR_NAME = 'raw_histogram' -EXPOSURE_TIMES_CDF_VAR_NAME = 'exposure_times' -NUM_OF_BINS_CDF_VAR_NAME = 'number_of_bins' +PHOTON_FLUX_CDF_VAR_NAME = "photon_flux" +PHOTON_FLUX_UNCERTAINTY_CDF_VAR_NAME = "photon_flux_uncertainty" +RAW_HISTOGRAM_CDF_VAR_NAME = "raw_histogram" +EXPOSURE_TIMES_CDF_VAR_NAME = "exposure_times" +NUM_OF_BINS_CDF_VAR_NAME = "number_of_bins" EPOCH_CDF_VAR_NAME = "epoch" EPOCH_DELTA_CDF_VAR_NAME = "epoch_delta" START_TIME_CDF_VAR_NAME = "start_time" @@ -142,7 +142,9 @@ def to_data_product_variables(self) -> list[DataProductVariable]: return [ DataProductVariable(REPOINTING_CDF_VAR_NAME, [self.identifier]), DataProductVariable(PHOTON_FLUX_CDF_VAR_NAME, self.photon_flux), - DataProductVariable(PHOTON_FLUX_UNCERTAINTY_CDF_VAR_NAME, self.photon_flux_uncertainty), + DataProductVariable( + PHOTON_FLUX_UNCERTAINTY_CDF_VAR_NAME, self.photon_flux_uncertainty + ), DataProductVariable(RAW_HISTOGRAM_CDF_VAR_NAME, self.raw_histogram), DataProductVariable(EXPOSURE_TIMES_CDF_VAR_NAME, self.exposure_times), DataProductVariable(NUM_OF_BINS_CDF_VAR_NAME, self.number_of_bins), @@ -154,28 +156,75 @@ def to_data_product_variables(self) -> list[DataProductVariable]: DataProductVariable(SPIN_ANGLE_DELTA_CDF_VAR_NAME, self.spin_angle_delta), DataProductVariable(LATITUDE_CDF_VAR_NAME, self.latitude), DataProductVariable(LONGITUDE_CDF_VAR_NAME, self.longitude), - DataProductVariable(EXTRA_HELIOSPHERIC_BACKGROUND_CDF_VAR_NAME, self.extra_heliospheric_background), - DataProductVariable(TIME_DEPENDENT_BACKGROUND_CDF_VAR_NAME, self.time_dependent_background), - - DataProductVariable(FILTER_TEMPERATURE_AVERAGE_CDF_VAR_NAME, self.filter_temperature_average), - DataProductVariable(FILTER_TEMPERATURE_STD_DEV_CDF_VAR_NAME, self.filter_temperature_std_dev), - DataProductVariable(HV_VOLTAGE_AVERAGE_CDF_VAR_NAME, self.hv_voltage_average), - DataProductVariable(HV_VOLTAGE_STD_DEV_CDF_VAR_NAME, self.hv_voltage_std_dev), - DataProductVariable(SPIN_PERIOD_AVERAGE_CDF_VAR_NAME, self.spin_period_average), - DataProductVariable(SPIN_PERIOD_STD_DEV_CDF_VAR_NAME, self.spin_period_std_dev), - DataProductVariable(SPIN_PERIOD_GROUND_AVERAGE_CDF_VAR_NAME, self.spin_period_ground_average), - DataProductVariable(SPIN_PERIOD_GROUND_STD_DEV_CDF_VAR_NAME, self.spin_period_ground_std_dev), - DataProductVariable(PULSE_LENGTH_AVERAGE_CDF_VAR_NAME, self.pulse_length_average), - DataProductVariable(PULSE_LENGTH_STD_DEV_CDF_VAR_NAME, self.pulse_length_std_dev), - DataProductVariable(POSITION_ANGLE_OFFSET_AVERAGE_CDF_VAR_NAME, self.position_angle_offset_average), - DataProductVariable(POSITION_ANGLE_OFFSET_STD_DEV_CDF_VAR_NAME, self.position_angle_offset_std_dev), - DataProductVariable(SPIN_AXIS_ORIENTATION_AVERAGE_CDF_VAR_NAME, self.spin_axis_orientation_average), - DataProductVariable(SPIN_AXIS_ORIENTATION_STD_DEV_CDF_VAR_NAME, self.spin_axis_orientation_std_dev), - DataProductVariable(SPACECRAFT_LOCATION_AVERAGE_CDF_VAR_NAME, self.spacecraft_location_average), - DataProductVariable(SPACECRAFT_LOCATION_STD_DEV_CDF_VAR_NAME, self.spacecraft_location_std_dev), - DataProductVariable(SPACECRAFT_VELOCITY_AVERAGE_CDF_VAR_NAME, self.spacecraft_velocity_average), - DataProductVariable(SPACECRAFT_VELOCITY_STD_DEV_CDF_VAR_NAME, self.spacecraft_velocity_std_dev), - + DataProductVariable( + EXTRA_HELIOSPHERIC_BACKGROUND_CDF_VAR_NAME, + self.extra_heliospheric_background, + ), + DataProductVariable( + TIME_DEPENDENT_BACKGROUND_CDF_VAR_NAME, self.time_dependent_background + ), + DataProductVariable( + FILTER_TEMPERATURE_AVERAGE_CDF_VAR_NAME, self.filter_temperature_average + ), + DataProductVariable( + FILTER_TEMPERATURE_STD_DEV_CDF_VAR_NAME, self.filter_temperature_std_dev + ), + DataProductVariable( + HV_VOLTAGE_AVERAGE_CDF_VAR_NAME, self.hv_voltage_average + ), + DataProductVariable( + HV_VOLTAGE_STD_DEV_CDF_VAR_NAME, self.hv_voltage_std_dev + ), + DataProductVariable( + SPIN_PERIOD_AVERAGE_CDF_VAR_NAME, self.spin_period_average + ), + DataProductVariable( + SPIN_PERIOD_STD_DEV_CDF_VAR_NAME, self.spin_period_std_dev + ), + DataProductVariable( + SPIN_PERIOD_GROUND_AVERAGE_CDF_VAR_NAME, self.spin_period_ground_average + ), + DataProductVariable( + SPIN_PERIOD_GROUND_STD_DEV_CDF_VAR_NAME, self.spin_period_ground_std_dev + ), + DataProductVariable( + PULSE_LENGTH_AVERAGE_CDF_VAR_NAME, self.pulse_length_average + ), + DataProductVariable( + PULSE_LENGTH_STD_DEV_CDF_VAR_NAME, self.pulse_length_std_dev + ), + DataProductVariable( + POSITION_ANGLE_OFFSET_AVERAGE_CDF_VAR_NAME, + self.position_angle_offset_average, + ), + DataProductVariable( + POSITION_ANGLE_OFFSET_STD_DEV_CDF_VAR_NAME, + self.position_angle_offset_std_dev, + ), + DataProductVariable( + SPIN_AXIS_ORIENTATION_AVERAGE_CDF_VAR_NAME, + self.spin_axis_orientation_average, + ), + DataProductVariable( + SPIN_AXIS_ORIENTATION_STD_DEV_CDF_VAR_NAME, + self.spin_axis_orientation_std_dev, + ), + DataProductVariable( + SPACECRAFT_LOCATION_AVERAGE_CDF_VAR_NAME, + self.spacecraft_location_average, + ), + DataProductVariable( + SPACECRAFT_LOCATION_STD_DEV_CDF_VAR_NAME, + self.spacecraft_location_std_dev, + ), + DataProductVariable( + SPACECRAFT_VELOCITY_AVERAGE_CDF_VAR_NAME, + self.spacecraft_velocity_average, + ), + DataProductVariable( + SPACECRAFT_VELOCITY_STD_DEV_CDF_VAR_NAME, + self.spacecraft_velocity_std_dev, + ), DataProductVariable("lon_lat", np.arange(2)), DataProductVariable("lon_lat_labels", ["lon", "lat"]), DataProductVariable("x_y_z", np.arange(3)), diff --git a/imap_l3_processing/glows/l3a/science/bad_angle_flag_configuration.py b/imap_l3_processing/glows/l3a/science/bad_angle_flag_configuration.py index 399eda955..835cc4881 100644 --- a/imap_l3_processing/glows/l3a/science/bad_angle_flag_configuration.py +++ b/imap_l3_processing/glows/l3a/science/bad_angle_flag_configuration.py @@ -14,10 +14,14 @@ class BadAngleFlagConfiguration: mask_suspected_transient: bool def evaluate_flags(self, flags: np.ndarray): - configuration = np.array([self.mask_close_to_uv_source, - self.mask_inside_excluded_region, - self.mask_excluded_by_instr_team, - self.mask_suspected_transient]).reshape(4, 1) + configuration = np.array( + [ + self.mask_close_to_uv_source, + self.mask_inside_excluded_region, + self.mask_excluded_by_instr_team, + self.mask_suspected_transient, + ] + ).reshape(4, 1) return np.any(np.logical_and(configuration, flags), axis=-2) diff --git a/imap_l3_processing/glows/l3a/science/calculate_daily_lightcurve.py b/imap_l3_processing/glows/l3a/science/calculate_daily_lightcurve.py index f5189f1c6..c368f7f21 100644 --- a/imap_l3_processing/glows/l3a/science/calculate_daily_lightcurve.py +++ b/imap_l3_processing/glows/l3a/science/calculate_daily_lightcurve.py @@ -1,21 +1,29 @@ import numpy as np from numpy import ndarray -from imap_l3_processing.glows.l3a.science.bad_angle_flag_configuration import BadAngleFlagConfiguration -from imap_l3_processing.glows.l3a.science.time_independent_background_lookup_table import \ - TimeIndependentBackgroundLookupTable - - -def rebin_lightcurve(time_independent_background_table: TimeIndependentBackgroundLookupTable, - bad_angle_flag_configuration: BadAngleFlagConfiguration, - photon_flux: np.ndarray, - latitudes: np.ndarray, - longitudes: np.ndarray, - flags: np.ndarray, - exposure_times: np.ndarray, output_size: int, background: ndarray[float] - ) -> tuple[np.ndarray, np.ndarray]: - photon_flux_minus_time_independent_background = photon_flux - time_independent_background_table.lookup( - lat=latitudes, lon=longitudes) +from imap_l3_processing.glows.l3a.science.bad_angle_flag_configuration import ( + BadAngleFlagConfiguration, +) +from imap_l3_processing.glows.l3a.science.time_independent_background_lookup_table import ( + TimeIndependentBackgroundLookupTable, +) + + +def rebin_lightcurve( + time_independent_background_table: TimeIndependentBackgroundLookupTable, + bad_angle_flag_configuration: BadAngleFlagConfiguration, + photon_flux: np.ndarray, + latitudes: np.ndarray, + longitudes: np.ndarray, + flags: np.ndarray, + exposure_times: np.ndarray, + output_size: int, + background: ndarray[float], +) -> tuple[np.ndarray, np.ndarray]: + photon_flux_minus_time_independent_background = ( + photon_flux + - time_independent_background_table.lookup(lat=latitudes, lon=longitudes) + ) masked = bad_angle_flag_configuration.evaluate_flags(flags) included = np.logical_not(masked) @@ -24,13 +32,21 @@ def rebin_lightcurve(time_independent_background_table: TimeIndependentBackgroun exposure_times_rebinned = np.sum(exposure_time_by_bins, axis=-1) - filtered_photon_flux = photon_flux_minus_time_independent_background * filtered_exposure_times + filtered_photon_flux = ( + photon_flux_minus_time_independent_background * filtered_exposure_times + ) split_by_bins = np.reshape(filtered_photon_flux, (output_size, -1)) - rebinned_photon_flux = np.full(shape=(output_size,), fill_value=np.nan, dtype=filtered_photon_flux.dtype) + rebinned_photon_flux = np.full( + shape=(output_size,), fill_value=np.nan, dtype=filtered_photon_flux.dtype + ) - np.divide(np.sum(split_by_bins, axis=-1), exposure_times_rebinned, out=rebinned_photon_flux, - where=exposure_times_rebinned != 0) + np.divide( + np.sum(split_by_bins, axis=-1), + exposure_times_rebinned, + out=rebinned_photon_flux, + where=exposure_times_rebinned != 0, + ) rebinned_photon_flux = rebinned_photon_flux - background return rebinned_photon_flux, exposure_times_rebinned diff --git a/imap_l3_processing/glows/l3a/science/time_independent_background_lookup_table.py b/imap_l3_processing/glows/l3a/science/time_independent_background_lookup_table.py index 632b4c094..0f5492449 100644 --- a/imap_l3_processing/glows/l3a/science/time_independent_background_lookup_table.py +++ b/imap_l3_processing/glows/l3a/science/time_independent_background_lookup_table.py @@ -5,7 +5,7 @@ import numpy as np import scipy.interpolate -T = TypeVar('T', bound='TimeIndependentBackgroundLookupTable') +T = TypeVar("T", bound="TimeIndependentBackgroundLookupTable") @dataclass @@ -14,13 +14,19 @@ class TimeIndependentBackgroundLookupTable: latitudes: np.ndarray[float] background_values: np.ndarray[float] - def lookup(self, lat: Union[np.ndarray[float], float], lon: Union[np.ndarray[float], float]) -> Union[ - np.ndarray[float], float]: + def lookup( + self, lat: Union[np.ndarray[float], float], lon: Union[np.ndarray[float], float] + ) -> Union[np.ndarray[float], float]: wrapped_lon = np.mod(lon, 360) - background_extended = np.append(self.background_values, self.background_values[:, :1], axis=1) + background_extended = np.append( + self.background_values, self.background_values[:, :1], axis=1 + ) longitudes_extended = np.append(self.longitudes, self.longitudes[0] + 360) - return scipy.interpolate.interpn((self.latitudes, longitudes_extended), background_extended, - np.column_stack([lat, wrapped_lon])) + return scipy.interpolate.interpn( + (self.latitudes, longitudes_extended), + background_extended, + np.column_stack([lat, wrapped_lon]), + ) @classmethod def from_file(cls: type[T], path: Path) -> T: diff --git a/imap_l3_processing/glows/l3a/utils.py b/imap_l3_processing/glows/l3a/utils.py index 77b2d9d0c..574f184af 100644 --- a/imap_l3_processing/glows/l3a/utils.py +++ b/imap_l3_processing/glows/l3a/utils.py @@ -6,8 +6,14 @@ from imap_data_access import AncillaryFilePath from spacepy.pycdf import CDF -from imap_l3_processing.glows.l3a.models import GlowsL2Data, GlowsL2LightCurve, GlowsLatLon, GlowsL3LightCurve, XYZ, \ - GlowsL2Header +from imap_l3_processing.glows.l3a.models import ( + GlowsL2Data, + GlowsL2LightCurve, + GlowsLatLon, + GlowsL3LightCurve, + XYZ, + GlowsL2Header, +) from imap_l3_processing.models import InputMetadata, Instrument MAX_SPIN_ANGLE_BINS = 90 @@ -18,13 +24,13 @@ def _pad_lightcurve(values) -> np.ndarray: values = np.asarray(values) if len(values) > MAX_SPIN_ANGLE_BINS: raise ValueError( - f"GLOWS L3a lightcurve bins ({len(values)}) exceed " - f"the allowed maximum ({MAX_SPIN_ANGLE_BINS})." + f"GLOWS L3a lightcurve bins ({len(values)}) exceed " + f"the allowed maximum ({MAX_SPIN_ANGLE_BINS})." ) if np.issubdtype(values.dtype, np.integer): padded_values = np.ma.masked_all(MAX_SPIN_ANGLE_BINS, dtype=values.dtype) - padded_values[:len(values)] = values + padded_values[: len(values)] = values return padded_values.reshape(1, -1) return np.pad( @@ -35,32 +41,48 @@ def _pad_lightcurve(values) -> np.ndarray: def read_l2_glows_data(cdf: CDF) -> GlowsL2Data: - assert 1 == cdf['photon_flux'].shape[0], "Level 2 file should have only one histogram" - - light_curve = GlowsL2LightCurve(photon_flux=cdf['photon_flux'][0], - spin_angle=cdf['spin_angle'][0], - histogram_flag_array=cdf['histogram_flag_array'][0], - exposure_times=cdf['exposure_times'][0], - flux_uncertainties=cdf['flux_uncertainties'][0], - raw_histogram=cdf['raw_histograms'][0], - ecliptic_lon=cdf['ecliptic_lon'][0], - ecliptic_lat=cdf['ecliptic_lat'][0]) - spin_axis_average = GlowsLatLon(lon=cdf['spin_axis_orientation_average'][0, 0], - lat=cdf['spin_axis_orientation_average'][0, 1]) - spin_axis_std_dev = GlowsLatLon(lon=cdf['spin_axis_orientation_std_dev'][0, 0], - lat=cdf['spin_axis_orientation_std_dev'][0, 1]) - spacecraft_location_average = XYZ(x=cdf['spacecraft_location_average'][0, 0], - y=cdf['spacecraft_location_average'][0, 1], - z=cdf['spacecraft_location_average'][0, 2], ) - spacecraft_location_std_dev = XYZ(x=cdf['spacecraft_location_std_dev'][0, 0], - y=cdf['spacecraft_location_std_dev'][0, 1], - z=cdf['spacecraft_location_std_dev'][0, 2], ) - spacecraft_velocity_average = XYZ(x=cdf['spacecraft_velocity_average'][0, 0], - y=cdf['spacecraft_velocity_average'][0, 1], - z=cdf['spacecraft_velocity_average'][0, 2], ) - spacecraft_velocity_std_dev = XYZ(x=cdf['spacecraft_velocity_std_dev'][0, 0], - y=cdf['spacecraft_velocity_std_dev'][0, 1], - z=cdf['spacecraft_velocity_std_dev'][0, 2], ) + assert 1 == cdf["photon_flux"].shape[0], ( + "Level 2 file should have only one histogram" + ) + + light_curve = GlowsL2LightCurve( + photon_flux=cdf["photon_flux"][0], + spin_angle=cdf["spin_angle"][0], + histogram_flag_array=cdf["histogram_flag_array"][0], + exposure_times=cdf["exposure_times"][0], + flux_uncertainties=cdf["flux_uncertainties"][0], + raw_histogram=cdf["raw_histograms"][0], + ecliptic_lon=cdf["ecliptic_lon"][0], + ecliptic_lat=cdf["ecliptic_lat"][0], + ) + spin_axis_average = GlowsLatLon( + lon=cdf["spin_axis_orientation_average"][0, 0], + lat=cdf["spin_axis_orientation_average"][0, 1], + ) + spin_axis_std_dev = GlowsLatLon( + lon=cdf["spin_axis_orientation_std_dev"][0, 0], + lat=cdf["spin_axis_orientation_std_dev"][0, 1], + ) + spacecraft_location_average = XYZ( + x=cdf["spacecraft_location_average"][0, 0], + y=cdf["spacecraft_location_average"][0, 1], + z=cdf["spacecraft_location_average"][0, 2], + ) + spacecraft_location_std_dev = XYZ( + x=cdf["spacecraft_location_std_dev"][0, 0], + y=cdf["spacecraft_location_std_dev"][0, 1], + z=cdf["spacecraft_location_std_dev"][0, 2], + ) + spacecraft_velocity_average = XYZ( + x=cdf["spacecraft_velocity_average"][0, 0], + y=cdf["spacecraft_velocity_average"][0, 1], + z=cdf["spacecraft_velocity_average"][0, 2], + ) + spacecraft_velocity_std_dev = XYZ( + x=cdf["spacecraft_velocity_std_dev"][0, 0], + y=cdf["spacecraft_velocity_std_dev"][0, 1], + z=cdf["spacecraft_velocity_std_dev"][0, 2], + ) ancillary_files = [] for file in list(cdf.attrs["Parents"][...]): @@ -71,43 +93,46 @@ def read_l2_glows_data(cdf: CDF) -> GlowsL2Data: except AncillaryFilePath.InvalidImapFileError: pass - return GlowsL2Data(identifier=cdf['identifier'][0], - start_time=cdf['start_time'][0], - end_time=cdf['end_time'][0], - daily_lightcurve=light_curve, - number_of_bins=cdf['number_of_bins'][0], - spin_axis_orientation_average=spin_axis_average, - spin_axis_orientation_std_dev=spin_axis_std_dev, - filter_temperature_average=cdf['filter_temperature_average'][0], - filter_temperature_std_dev=cdf['filter_temperature_std_dev'][0], - hv_voltage_average=cdf['hv_voltage_average'][0], - hv_voltage_std_dev=cdf['hv_voltage_std_dev'][0], - spin_period_average=cdf['spin_period_average'][0], - spin_period_std_dev=cdf['spin_period_std_dev'][0], - spin_period_ground_average=cdf['spin_period_ground_average'][0], - spin_period_ground_std_dev=cdf['spin_period_ground_std_dev'][0], - pulse_length_average=cdf['pulse_length_average'][0], - pulse_length_std_dev=cdf['pulse_length_std_dev'][0], - position_angle_offset_average=cdf['position_angle_offset_average'][0], - position_angle_offset_std_dev=cdf['position_angle_offset_std_dev'][0], - spacecraft_location_average=spacecraft_location_average, - spacecraft_location_std_dev=spacecraft_location_std_dev, - spacecraft_velocity_average=spacecraft_velocity_average, - spacecraft_velocity_std_dev=spacecraft_velocity_std_dev, - header=GlowsL2Header( - flight_software_version=str(cdf.attrs["flight_software_version"][0]), - pkts_file_name=cdf.attrs["pkts_file_name"][0], - ancillary_data_files=ancillary_files, - ), - l2_file_name=Path(cdf.pathname.decode('utf-8')).name - ) + return GlowsL2Data( + identifier=cdf["identifier"][0], + start_time=cdf["start_time"][0], + end_time=cdf["end_time"][0], + daily_lightcurve=light_curve, + number_of_bins=cdf["number_of_bins"][0], + spin_axis_orientation_average=spin_axis_average, + spin_axis_orientation_std_dev=spin_axis_std_dev, + filter_temperature_average=cdf["filter_temperature_average"][0], + filter_temperature_std_dev=cdf["filter_temperature_std_dev"][0], + hv_voltage_average=cdf["hv_voltage_average"][0], + hv_voltage_std_dev=cdf["hv_voltage_std_dev"][0], + spin_period_average=cdf["spin_period_average"][0], + spin_period_std_dev=cdf["spin_period_std_dev"][0], + spin_period_ground_average=cdf["spin_period_ground_average"][0], + spin_period_ground_std_dev=cdf["spin_period_ground_std_dev"][0], + pulse_length_average=cdf["pulse_length_average"][0], + pulse_length_std_dev=cdf["pulse_length_std_dev"][0], + position_angle_offset_average=cdf["position_angle_offset_average"][0], + position_angle_offset_std_dev=cdf["position_angle_offset_std_dev"][0], + spacecraft_location_average=spacecraft_location_average, + spacecraft_location_std_dev=spacecraft_location_std_dev, + spacecraft_velocity_average=spacecraft_velocity_average, + spacecraft_velocity_std_dev=spacecraft_velocity_std_dev, + header=GlowsL2Header( + flight_software_version=str(cdf.attrs["flight_software_version"][0]), + pkts_file_name=cdf.attrs["pkts_file_name"][0], + ancillary_data_files=ancillary_files, + ), + l2_file_name=Path(cdf.pathname.decode("utf-8")).name, + ) def _read_xyz(cdf: CDF, variable_name: str) -> XYZ: - return {k: cdf[f'{variable_name}_{k}'][0] for k in "xyz"} + return {k: cdf[f"{variable_name}_{k}"][0] for k in "xyz"} -def create_glows_l3a_from_dictionary(data: dict, input_metadata: InputMetadata) -> GlowsL3LightCurve: +def create_glows_l3a_from_dictionary( + data: dict, input_metadata: InputMetadata +) -> GlowsL3LightCurve: start_time = datetime.fromisoformat(data["start_time"]) end_time = datetime.fromisoformat(data["end_time"]) total_time = end_time - start_time @@ -121,15 +146,21 @@ def create_glows_l3a_from_dictionary(data: dict, input_metadata: InputMetadata) start_time=data["start_time"], end_time=data["end_time"], photon_flux=_pad_lightcurve(data["daily_lightcurve"]["photon_flux"]), - photon_flux_uncertainty=_pad_lightcurve(data["daily_lightcurve"]["flux_uncertainties"]), + photon_flux_uncertainty=_pad_lightcurve( + data["daily_lightcurve"]["flux_uncertainties"] + ), raw_histogram=_pad_lightcurve(data["daily_lightcurve"]["raw_histogram"]), exposure_times=_pad_lightcurve(data["daily_lightcurve"]["exposure_times"]), spin_angle=_pad_lightcurve(data["daily_lightcurve"]["spin_angle"]), spin_angle_delta=_pad_lightcurve(data["daily_lightcurve"]["spin_angle_delta"]), latitude=_pad_lightcurve(data["daily_lightcurve"]["ecliptic_lat"]), longitude=_pad_lightcurve(data["daily_lightcurve"]["ecliptic_lon"]), - extra_heliospheric_background=_pad_lightcurve(data["daily_lightcurve"]["extra_heliospheric_bckgrd"]), - time_dependent_background=_pad_lightcurve(data["daily_lightcurve"]["time_dependent_bckgrd"]), + extra_heliospheric_background=_pad_lightcurve( + data["daily_lightcurve"]["extra_heliospheric_bckgrd"] + ), + time_dependent_background=_pad_lightcurve( + data["daily_lightcurve"]["time_dependent_bckgrd"] + ), filter_temperature_average=np.array([data["filter_temperature_average"]]), filter_temperature_std_dev=np.array([data["filter_temperature_std_dev"]]), hv_voltage_average=np.array([data["hv_voltage_average"]]), @@ -142,77 +173,83 @@ def create_glows_l3a_from_dictionary(data: dict, input_metadata: InputMetadata) pulse_length_std_dev=np.array([data["pulse_length_std_dev"]]), position_angle_offset_average=np.array([data["position_angle_offset_average"]]), position_angle_offset_std_dev=np.array([data["position_angle_offset_std_dev"]]), - spin_axis_orientation_average=get_lon_lat(data["spin_axis_orientation_average"]), - spin_axis_orientation_std_dev=get_lon_lat(data["spin_axis_orientation_std_dev"]), + spin_axis_orientation_average=get_lon_lat( + data["spin_axis_orientation_average"] + ), + spin_axis_orientation_std_dev=get_lon_lat( + data["spin_axis_orientation_std_dev"] + ), spacecraft_location_average=get_xyz(data["spacecraft_location_average"]), spacecraft_location_std_dev=get_xyz(data["spacecraft_location_std_dev"]), spacecraft_velocity_average=get_xyz(data["spacecraft_velocity_average"]), spacecraft_velocity_std_dev=get_xyz(data["spacecraft_velocity_std_dev"]), - number_of_bins=np.array([data['daily_lightcurve']['number_of_bins']]), + number_of_bins=np.array([data["daily_lightcurve"]["number_of_bins"]]), ) def create_glows_l3a_dictionary_from_cdf(cdf_file_path: Path) -> dict: cdf = CDF(str(cdf_file_path)) - time_delta = timedelta(seconds=cdf['epoch_delta'][0] / 1e9) - start_time = cdf['epoch'][0] - time_delta - end_time = cdf['epoch'][0] + time_delta - valid_bin_count = cdf['number_of_bins'][0] + time_delta = timedelta(seconds=cdf["epoch_delta"][0] / 1e9) + start_time = cdf["epoch"][0] - time_delta + end_time = cdf["epoch"][0] + time_delta + valid_bin_count = cdf["number_of_bins"][0] return { - 'filename': f'{os.path.basename(cdf_file_path)}', - 'start_time': start_time.strftime("%Y-%m-%d %H:%M:%S"), - 'end_time': end_time.strftime("%Y-%m-%d %H:%M:%S"), - 'daily_lightcurve': { - 'ecliptic_lat': cdf['ecliptic_lat'][0][:valid_bin_count], - 'ecliptic_lon': cdf['ecliptic_lon'][0][:valid_bin_count], - 'exposure_times': cdf['exposure_times'][0][:valid_bin_count], - 'photon_flux': cdf['photon_flux'][0][:valid_bin_count], - 'flux_uncertainties': cdf['photon_flux_uncertainty'][0][:valid_bin_count], - 'extra_heliospheric_bckgrd': cdf['extra_heliospheric_bckgrd'][0][:valid_bin_count], - 'time_dependent_bckgrd': cdf['time_dependent_bckgrd'][0][:valid_bin_count], - 'spin_angle': cdf['spin_angle'][0][:valid_bin_count], - 'raw_histogram': cdf['raw_histogram'][0][:valid_bin_count], - 'number_of_bins': valid_bin_count + "filename": f"{os.path.basename(cdf_file_path)}", + "start_time": start_time.strftime("%Y-%m-%d %H:%M:%S"), + "end_time": end_time.strftime("%Y-%m-%d %H:%M:%S"), + "daily_lightcurve": { + "ecliptic_lat": cdf["ecliptic_lat"][0][:valid_bin_count], + "ecliptic_lon": cdf["ecliptic_lon"][0][:valid_bin_count], + "exposure_times": cdf["exposure_times"][0][:valid_bin_count], + "photon_flux": cdf["photon_flux"][0][:valid_bin_count], + "flux_uncertainties": cdf["photon_flux_uncertainty"][0][:valid_bin_count], + "extra_heliospheric_bckgrd": cdf["extra_heliospheric_bckgrd"][0][ + :valid_bin_count + ], + "time_dependent_bckgrd": cdf["time_dependent_bckgrd"][0][:valid_bin_count], + "spin_angle": cdf["spin_angle"][0][:valid_bin_count], + "raw_histogram": cdf["raw_histogram"][0][:valid_bin_count], + "number_of_bins": valid_bin_count, }, - 'filter_temperature_average': cdf['filter_temperature_average'][0], - 'filter_temperature_std_dev': cdf['filter_temperature_std_dev'][0], - 'hv_voltage_average': cdf['hv_voltage_average'][0], - 'hv_voltage_std_dev': cdf['hv_voltage_std_dev'][0], - 'spin_period_average': cdf['spin_period_average'][0], - 'spin_period_std_dev': cdf['spin_period_std_dev'][0], - 'spin_period_ground_average': cdf['spin_period_ground_average'][0], - 'spin_period_ground_std_dev': cdf['spin_period_ground_std_dev'][0], - 'pulse_length_average': cdf['pulse_length_average'][0], - 'pulse_length_std_dev': cdf['pulse_length_std_dev'][0], - 'position_angle_offset_average': cdf['position_angle_offset_average'][0], - 'position_angle_offset_std_dev': cdf['position_angle_offset_std_dev'][0], - 'spin_axis_orientation_average': { - 'lon': cdf['spin_axis_orientation_average'][0][0], - 'lat': cdf['spin_axis_orientation_average'][0][1], + "filter_temperature_average": cdf["filter_temperature_average"][0], + "filter_temperature_std_dev": cdf["filter_temperature_std_dev"][0], + "hv_voltage_average": cdf["hv_voltage_average"][0], + "hv_voltage_std_dev": cdf["hv_voltage_std_dev"][0], + "spin_period_average": cdf["spin_period_average"][0], + "spin_period_std_dev": cdf["spin_period_std_dev"][0], + "spin_period_ground_average": cdf["spin_period_ground_average"][0], + "spin_period_ground_std_dev": cdf["spin_period_ground_std_dev"][0], + "pulse_length_average": cdf["pulse_length_average"][0], + "pulse_length_std_dev": cdf["pulse_length_std_dev"][0], + "position_angle_offset_average": cdf["position_angle_offset_average"][0], + "position_angle_offset_std_dev": cdf["position_angle_offset_std_dev"][0], + "spin_axis_orientation_average": { + "lon": cdf["spin_axis_orientation_average"][0][0], + "lat": cdf["spin_axis_orientation_average"][0][1], }, - 'spin_axis_orientation_std_dev': { - 'lon': cdf['spin_axis_orientation_std_dev'][0][0], - 'lat': cdf['spin_axis_orientation_std_dev'][0][1], + "spin_axis_orientation_std_dev": { + "lon": cdf["spin_axis_orientation_std_dev"][0][0], + "lat": cdf["spin_axis_orientation_std_dev"][0][1], }, - 'spacecraft_location_average': { - 'x': cdf['spacecraft_location_average'][0][0], - 'y': cdf['spacecraft_location_average'][0][1], - 'z': cdf['spacecraft_location_average'][0][2], + "spacecraft_location_average": { + "x": cdf["spacecraft_location_average"][0][0], + "y": cdf["spacecraft_location_average"][0][1], + "z": cdf["spacecraft_location_average"][0][2], }, - 'spacecraft_location_std_dev': { - 'x': cdf['spacecraft_location_std_dev'][0][0], - 'y': cdf['spacecraft_location_std_dev'][0][1], - 'z': cdf['spacecraft_location_std_dev'][0][2], + "spacecraft_location_std_dev": { + "x": cdf["spacecraft_location_std_dev"][0][0], + "y": cdf["spacecraft_location_std_dev"][0][1], + "z": cdf["spacecraft_location_std_dev"][0][2], }, - 'spacecraft_velocity_average': { - 'x': cdf['spacecraft_velocity_average'][0][0], - 'y': cdf['spacecraft_velocity_average'][0][1], - 'z': cdf['spacecraft_velocity_average'][0][2], + "spacecraft_velocity_average": { + "x": cdf["spacecraft_velocity_average"][0][0], + "y": cdf["spacecraft_velocity_average"][0][1], + "z": cdf["spacecraft_velocity_average"][0][2], }, - 'spacecraft_velocity_std_dev': { - 'x': cdf['spacecraft_velocity_std_dev'][0][0], - 'y': cdf['spacecraft_velocity_std_dev'][0][1], - 'z': cdf['spacecraft_velocity_std_dev'][0][2], + "spacecraft_velocity_std_dev": { + "x": cdf["spacecraft_velocity_std_dev"][0][0], + "y": cdf["spacecraft_velocity_std_dev"][0][1], + "z": cdf["spacecraft_velocity_std_dev"][0][2], }, } diff --git a/imap_l3_processing/glows/l3bc/cannot_process_carrington_rotation_error.py b/imap_l3_processing/glows/l3bc/cannot_process_carrington_rotation_error.py index 476061749..6eb626984 100644 --- a/imap_l3_processing/glows/l3bc/cannot_process_carrington_rotation_error.py +++ b/imap_l3_processing/glows/l3bc/cannot_process_carrington_rotation_error.py @@ -1,4 +1,3 @@ class CannotProcessCarringtonRotationError(Exception): - def __init__(self, message): super().__init__(message) diff --git a/imap_l3_processing/glows/l3bc/dependency_validator.py b/imap_l3_processing/glows/l3bc/dependency_validator.py index e45038b04..335f74012 100644 --- a/imap_l3_processing/glows/l3bc/dependency_validator.py +++ b/imap_l3_processing/glows/l3bc/dependency_validator.py @@ -14,41 +14,62 @@ SPEED_COLUMN_2 = 8 -def validate_omni2_dependency(cr_start_date: datetime, - cr_end_date_exclusive: datetime, file_path: Path) -> bool: - omni_data: ndarray = np.loadtxt(file_path, usecols=(0, 1, 2, 5, 23, 24, 27, 30, 31, 34)) +def validate_omni2_dependency( + cr_start_date: datetime, cr_end_date_exclusive: datetime, file_path: Path +) -> bool: + omni_data: ndarray = np.loadtxt( + file_path, usecols=(0, 1, 2, 5, 23, 24, 27, 30, 31, 34) + ) row_day_keys = omni_data[:, YEAR_COLUMN] * 1000 + omni_data[:, DOY_COLUMN] start_key = cr_start_date.year * 1000 + cr_start_date.timetuple().tm_yday - end_key = cr_end_date_exclusive.year * 1000 + cr_end_date_exclusive.timetuple().tm_yday + end_key = ( + cr_end_date_exclusive.year * 1000 + cr_end_date_exclusive.timetuple().tm_yday + ) start_idx = np.searchsorted(row_day_keys, start_key) end_idx = np.searchsorted(row_day_keys, end_key) omni_data = omni_data[start_idx:end_idx] for data_point in omni_data: - mask_fill_density_rows = np.all(data_point[[DENSITY_COLUMN_1, DENSITY_COLUMN_2]] < 999.9) - mask_fill_speed_rows = np.all(data_point[[SPEED_COLUMN_1, SPEED_COLUMN_2]] < 9999) + mask_fill_density_rows = np.all( + data_point[[DENSITY_COLUMN_1, DENSITY_COLUMN_2]] < 999.9 + ) + mask_fill_speed_rows = np.all( + data_point[[SPEED_COLUMN_1, SPEED_COLUMN_2]] < 9999 + ) if np.all([mask_fill_density_rows, mask_fill_speed_rows]): return True return False -def validate_dependencies(cr_start_date: datetime, cr_end_date: datetime, buffer: timedelta, - omni2_file_path: Path, - f107_index_path: Path, lyman_alpha_path: Path) -> bool: - omni_condition = validate_omni2_dependency(cr_start_date, cr_end_date, omni2_file_path) - f107_condition = validate_f107_fluxtable_dependency(cr_end_date, buffer, - f107_index_path) - lyman_alpha_condition = validate_lyman_alpha_dependency(cr_end_date, buffer, lyman_alpha_path) +def validate_dependencies( + cr_start_date: datetime, + cr_end_date: datetime, + buffer: timedelta, + omni2_file_path: Path, + f107_index_path: Path, + lyman_alpha_path: Path, +) -> bool: + omni_condition = validate_omni2_dependency( + cr_start_date, cr_end_date, omni2_file_path + ) + + f107_condition = validate_f107_fluxtable_dependency( + cr_end_date, buffer, f107_index_path + ) + lyman_alpha_condition = validate_lyman_alpha_dependency( + cr_end_date, buffer, lyman_alpha_path + ) return omni_condition and f107_condition and lyman_alpha_condition -def validate_f107_fluxtable_dependency(end_date: datetime, - buffer: timedelta, file_path: Path) -> bool: +def validate_f107_fluxtable_dependency( + end_date: datetime, buffer: timedelta, file_path: Path +) -> bool: f107_data = np.loadtxt(file_path, dtype=str) f107_data = f107_data[2:] last_date = f107_data[-1][0] @@ -56,10 +77,12 @@ def validate_f107_fluxtable_dependency(end_date: datetime, return datetime.strptime(last_date, "%Y%m%d") >= end_date + buffer -def validate_lyman_alpha_dependency(end_date: datetime, buffer: timedelta, file_path: Path) -> bool: +def validate_lyman_alpha_dependency( + end_date: datetime, buffer: timedelta, file_path: Path +) -> bool: cdf = netCDF4.Dataset(file_path) - isoformat_time = cdf.time_coverage_end.replace('Z', '+00:00') + isoformat_time = cdf.time_coverage_end.replace("Z", "+00:00") datetime_from_iso = datetime.fromisoformat(isoformat_time).replace(tzinfo=None) diff --git a/imap_l3_processing/glows/l3bc/glows_l3bc_dependencies.py b/imap_l3_processing/glows/l3bc/glows_l3bc_dependencies.py index c71c7236b..78c07edd5 100644 --- a/imap_l3_processing/glows/l3bc/glows_l3bc_dependencies.py +++ b/imap_l3_processing/glows/l3bc/glows_l3bc_dependencies.py @@ -22,21 +22,34 @@ class GlowsL3BCDependencies: @property def l3a_file_names(self): - return [l3a['filename'] for l3a in self.l3a_data] + return [l3a["filename"] for l3a in self.l3a_data] @classmethod - def download_from_cr_to_process(cls, cr_to_process: CRToProcess, version: Version, - external_dependencies: ExternalDependencies, repointing_file_path: Path): + def download_from_cr_to_process( + cls, + cr_to_process: CRToProcess, + version: Version, + external_dependencies: ExternalDependencies, + repointing_file_path: Path, + ): external_files = { - 'f107_raw_data': external_dependencies.f107_index_file_path, - 'omni_raw_data': external_dependencies.omni2_data_path + "f107_raw_data": external_dependencies.f107_index_file_path, + "omni_raw_data": external_dependencies.omni2_data_path, } ancillary_files = { - 'uv_anisotropy': imap_data_access.download(cr_to_process.uv_anisotropy_file_name), - 'WawHelioIonMP_parameters': imap_data_access.download(cr_to_process.waw_helio_ion_mp_file_name), - 'bad_days_list': imap_data_access.download(cr_to_process.bad_days_list_file_name), - 'pipeline_settings': imap_data_access.download(cr_to_process.pipeline_settings_file_name), + "uv_anisotropy": imap_data_access.download( + cr_to_process.uv_anisotropy_file_name + ), + "WawHelioIonMP_parameters": imap_data_access.download( + cr_to_process.waw_helio_ion_mp_file_name + ), + "bad_days_list": imap_data_access.download( + cr_to_process.bad_days_list_file_name + ), + "pipeline_settings": imap_data_access.download( + cr_to_process.pipeline_settings_file_name + ), } l3a_data = [] @@ -52,5 +65,5 @@ def download_from_cr_to_process(cls, cr_to_process: CRToProcess, version: Versio l3a_data=l3a_data, external_files=external_files, ancillary_files=ancillary_files, - repointing_file_path=repointing_file_path + repointing_file_path=repointing_file_path, ) diff --git a/imap_l3_processing/glows/l3bc/glows_l3bc_initializer.py b/imap_l3_processing/glows/l3bc/glows_l3bc_initializer.py index 337ee2450..297378b7e 100644 --- a/imap_l3_processing/glows/l3bc/glows_l3bc_initializer.py +++ b/imap_l3_processing/glows/l3bc/glows_l3bc_initializer.py @@ -10,12 +10,21 @@ from imap_processing.spice.repoint import set_global_repoint_table_paths from spacepy.pycdf import CDF -from imap_l3_processing.glows.descriptors import BAD_DAYS_LIST_DESCRIPTOR, WAW_HELIOION_DESCRIPTOR, \ - UV_ANISOTROPY_1CR_DESCRIPTOR, PIPELINE_SETTINGS_L3BCDE_DESCRIPTOR, GLOWS_L3B_DESCRIPTOR, GLOWS_L3C_DESCRIPTOR +from imap_l3_processing.glows.descriptors import ( + BAD_DAYS_LIST_DESCRIPTOR, + WAW_HELIOION_DESCRIPTOR, + UV_ANISOTROPY_1CR_DESCRIPTOR, + PIPELINE_SETTINGS_L3BCDE_DESCRIPTOR, + GLOWS_L3B_DESCRIPTOR, + GLOWS_L3C_DESCRIPTOR, +) from imap_l3_processing.glows.l3bc.glows_l3bc_dependencies import GlowsL3BCDependencies from imap_l3_processing.glows.l3bc.models import CRToProcess, ExternalDependencies -from imap_l3_processing.glows.l3bc.utils import get_date_range_of_cr, get_best_ancillary, \ - get_cr_for_date_time +from imap_l3_processing.glows.l3bc.utils import ( + get_date_range_of_cr, + get_best_ancillary, + get_cr_for_date_time, +) from imap_l3_processing.utils import read_cdf_parents logger = logging.getLogger(__name__) @@ -38,32 +47,60 @@ class GlowsL3BCInitializerData: class GlowsL3BCInitializer: @staticmethod - def get_crs_to_process(dependencies: ProcessingInputCollection, major_version_to_process: int) -> GlowsL3BCInitializerData: + def get_crs_to_process( + dependencies: ProcessingInputCollection, major_version_to_process: int + ) -> GlowsL3BCInitializerData: [repoint_file] = dependencies.get_file_paths(data_type=RepointInput.data_type) repoint_downloaded_path = imap_data_access.download(repoint_file) set_global_repoint_table_paths([repoint_downloaded_path]) - l3a_query_results = imap_data_access.query(instrument="glows", data_level="l3a", descriptor="hist", - version="latest") - l3a_files_names = [Path(l3a_query_result["file_path"]).name for l3a_query_result in l3a_query_results] + l3a_query_results = imap_data_access.query( + instrument="glows", data_level="l3a", descriptor="hist", version="latest" + ) + l3a_files_names = [ + Path(l3a_query_result["file_path"]).name + for l3a_query_result in l3a_query_results + ] cr_to_l3a_file_names = GlowsL3BCInitializer.group_l3a_by_cr(l3a_files_names) - l3b_query_result = imap_data_access.query(instrument="glows", data_level="l3b", - descriptor=GLOWS_L3B_DESCRIPTOR, version="latest") - l3c_query_result = imap_data_access.query(instrument="glows", data_level="l3c", descriptor=GLOWS_L3C_DESCRIPTOR, - version="latest") - - l3bs_by_cr = {int(result['cr']): Path(result["file_path"]).name for result in l3b_query_result} - l3cs_by_cr = {int(result['cr']): Path(result["file_path"]).name for result in l3c_query_result} + l3b_query_result = imap_data_access.query( + instrument="glows", + data_level="l3b", + descriptor=GLOWS_L3B_DESCRIPTOR, + version="latest", + ) + l3c_query_result = imap_data_access.query( + instrument="glows", + data_level="l3c", + descriptor=GLOWS_L3C_DESCRIPTOR, + version="latest", + ) - uv_anisotropy_query_result = imap_data_access.query(table="ancillary", instrument="glows", - descriptor=UV_ANISOTROPY_1CR_DESCRIPTOR) - waw_helio_ion_mp_query_result = imap_data_access.query(table="ancillary", instrument="glows", - descriptor=WAW_HELIOION_DESCRIPTOR) - bad_days_list_query_result = imap_data_access.query(table="ancillary", instrument="glows", - descriptor=BAD_DAYS_LIST_DESCRIPTOR) - pipeline_settings_query_result = imap_data_access.query(table="ancillary", instrument="glows", - descriptor=PIPELINE_SETTINGS_L3BCDE_DESCRIPTOR) + l3bs_by_cr = { + int(result["cr"]): Path(result["file_path"]).name + for result in l3b_query_result + } + l3cs_by_cr = { + int(result["cr"]): Path(result["file_path"]).name + for result in l3c_query_result + } + + uv_anisotropy_query_result = imap_data_access.query( + table="ancillary", + instrument="glows", + descriptor=UV_ANISOTROPY_1CR_DESCRIPTOR, + ) + waw_helio_ion_mp_query_result = imap_data_access.query( + table="ancillary", instrument="glows", descriptor=WAW_HELIOION_DESCRIPTOR + ) + bad_days_list_query_result = imap_data_access.query( + table="ancillary", instrument="glows", descriptor=BAD_DAYS_LIST_DESCRIPTOR + ) + pipeline_settings_query_result = imap_data_access.query( + table="ancillary", + instrument="glows", + descriptor=PIPELINE_SETTINGS_L3BCDE_DESCRIPTOR, + ) logger.info("Downloading external dependencies...") @@ -71,16 +108,23 @@ def get_crs_to_process(dependencies: ProcessingInputCollection, major_version_to logger.info("Finished downloading external dependencies") - if not all([external_dependencies.f107_index_file_path, external_dependencies.omni2_data_path, - external_dependencies.lyman_alpha_path]): - logger.info(f"Found issues with external dependencies, returning {external_dependencies}") + if not all( + [ + external_dependencies.f107_index_file_path, + external_dependencies.omni2_data_path, + external_dependencies.lyman_alpha_path, + ] + ): + logger.info( + f"Found issues with external dependencies, returning {external_dependencies}" + ) return GlowsL3BCInitializerData( external_dependencies=external_dependencies, l3bc_dependencies=[], l3bs_by_cr=l3bs_by_cr, l3cs_by_cr=l3cs_by_cr, - repoint_file_path=repoint_downloaded_path + repoint_file_path=repoint_downloaded_path, ) all_l3bc_dependencies = [] @@ -89,11 +133,18 @@ def get_crs_to_process(dependencies: ProcessingInputCollection, major_version_to cr_start_date, cr_end_date = get_date_range_of_cr(cr_number) ancillaries = { - "uv-anisotropy-1CR": get_best_ancillary(cr_start_date, cr_end_date, uv_anisotropy_query_result), - "WawHelioIonMP": get_best_ancillary(cr_start_date, cr_end_date, waw_helio_ion_mp_query_result), - "bad-days-list": get_best_ancillary(cr_start_date, cr_end_date, bad_days_list_query_result), - "pipeline-settings-l3bcde": get_best_ancillary(cr_start_date, cr_end_date, - pipeline_settings_query_result), + "uv-anisotropy-1CR": get_best_ancillary( + cr_start_date, cr_end_date, uv_anisotropy_query_result + ), + "WawHelioIonMP": get_best_ancillary( + cr_start_date, cr_end_date, waw_helio_ion_mp_query_result + ), + "bad-days-list": get_best_ancillary( + cr_start_date, cr_end_date, bad_days_list_query_result + ), + "pipeline-settings-l3bcde": get_best_ancillary( + cr_start_date, cr_end_date, pipeline_settings_query_result + ), } if all(ancillaries.values()): @@ -105,14 +156,23 @@ def get_crs_to_process(dependencies: ProcessingInputCollection, major_version_to ancillaries["pipeline-settings-l3bcde"], cr_start_date, cr_end_date, - cr_number + cr_number, ) - if version := GlowsL3BCInitializer.should_process_cr_candidate(cr_candidate, l3bs_by_cr, - external_dependencies, major_version_to_process): - l3bc_dependencies = GlowsL3BCDependencies.download_from_cr_to_process(cr_candidate, version, - external_dependencies, - repoint_downloaded_path) + if version := GlowsL3BCInitializer.should_process_cr_candidate( + cr_candidate, + l3bs_by_cr, + external_dependencies, + major_version_to_process, + ): + l3bc_dependencies = ( + GlowsL3BCDependencies.download_from_cr_to_process( + cr_candidate, + version, + external_dependencies, + repoint_downloaded_path, + ) + ) all_l3bc_dependencies.append(l3bc_dependencies) else: logger.info(f"decided not to process {cr_candidate}") @@ -124,18 +184,26 @@ def get_crs_to_process(dependencies: ProcessingInputCollection, major_version_to l3bc_dependencies=all_l3bc_dependencies, l3bs_by_cr=l3bs_by_cr, l3cs_by_cr=l3cs_by_cr, - repoint_file_path=repoint_downloaded_path + repoint_file_path=repoint_downloaded_path, ) @staticmethod - def should_process_cr_candidate(cr_candidate: CRToProcess, l3bs_by_cr: dict[int, str], - external_dependencies: ExternalDependencies, major_version: int|None) -> Optional[Version]: + def should_process_cr_candidate( + cr_candidate: CRToProcess, + l3bs_by_cr: dict[int, str], + external_dependencies: ExternalDependencies, + major_version: int | None, + ) -> Optional[Version]: if not cr_candidate.buffer_time_has_elapsed_since_cr(): - logger.warning(f"Not enough time has elapsed for cr {cr_candidate.cr_rotation_number}") + logger.warning( + f"Not enough time has elapsed for cr {cr_candidate.cr_rotation_number}" + ) return None if not cr_candidate.has_valid_external_dependencies(external_dependencies): - logger.warning(f"Invalid external dependencies for {cr_candidate.cr_rotation_number}") + logger.warning( + f"Invalid external dependencies for {cr_candidate.cr_rotation_number}" + ) return None match l3bs_by_cr.get(cr_candidate.cr_rotation_number): @@ -144,11 +212,17 @@ def should_process_cr_candidate(cr_candidate: CRToProcess, l3bs_by_cr: dict[int, case l3b_file_name: l3b_parents = read_cdf_parents(l3b_file_name) existing_path = ScienceFilePath(l3b_file_name) - existing_version = Version(existing_path.major_version, existing_path.minor_version) + existing_version = Version( + existing_path.major_version, existing_path.minor_version + ) major_version_updated = major_version != existing_version.major - inputs_changed = not cr_candidate.pipeline_dependency_file_names().issubset(l3b_parents) + inputs_changed = ( + not cr_candidate.pipeline_dependency_file_names().issubset( + l3b_parents + ) + ) if major_version_updated or inputs_changed: - return Version(major_version, existing_version.minor+1) + return Version(major_version, existing_version.minor + 1) return None @staticmethod @@ -157,7 +231,7 @@ def group_l3a_by_cr(l3a_file_names: list[str]) -> dict[int, set[str]]: for l3a_file_name in l3a_file_names: path = imap_data_access.download(l3a_file_name) with CDF(str(path)) as cdf: - epoch = cdf['epoch'][0] + epoch = cdf["epoch"][0] cr_number = get_cr_for_date_time(epoch) grouped_l3a_by_cr[cr_number].add(l3a_file_name) diff --git a/imap_l3_processing/glows/l3bc/l3bc_toolkit/constants.py b/imap_l3_processing/glows/l3bc/l3bc_toolkit/constants.py index 827b4c06c..0082d96da 100644 --- a/imap_l3_processing/glows/l3bc/l3bc_toolkit/constants.py +++ b/imap_l3_processing/glows/l3bc/l3bc_toolkit/constants.py @@ -5,33 +5,34 @@ import astropy.units as u -VERSION = "3.3" # version of ground processing software +VERSION = "3.3" # version of ground processing software # dictionary with names of ancillary input files expected from IMAP/GLOWS Instrument Team ANC_INPUT_FROM_INSTRUMENT_TEAM = { - 'WawHelioIonMP_parameters': 'data_ancillary/imap_glows_WawHelioIonMP_20251113_v007.json', - 'bad_day_list': 'data_ancillary/imap_glows_bad-days-list_20251113_v001.dat', - 'pipeline_settings': 'data_ancillary/imap_glows_pipeline-settings-l3bcde_20251113_v005.json', - 'uv_anisotropy' : 'data_ancillary/imap_glows_uv-anisotropy-1CR_20251113_v002.json', - 'WawHelioIon' : { - 'speed': 'data_ancillary/imap_glows_plasma-speed-2026d_20251113_v003.dat', - 'p-dens': 'data_ancillary/imap_glows_proton-density-2026d_20251113_v003.dat', - 'uv-anis': 'data_ancillary/imap_glows_uv-anisotropy-2026d_20251113_v003.dat', - 'phion': 'data_ancillary/imap_glows_photoion-2026d_20251113_v003.dat', - 'lya': 'data_ancillary/imap_glows_lya-2026d_20251113_v003.dat', - 'e-dens': 'data_ancillary/imap_glows_electron-density-2026d_20251113_v003.dat' - }} + "WawHelioIonMP_parameters": "data_ancillary/imap_glows_WawHelioIonMP_20251113_v007.json", + "bad_day_list": "data_ancillary/imap_glows_bad-days-list_20251113_v001.dat", + "pipeline_settings": "data_ancillary/imap_glows_pipeline-settings-l3bcde_20251113_v005.json", + "uv_anisotropy": "data_ancillary/imap_glows_uv-anisotropy-1CR_20251113_v002.json", + "WawHelioIon": { + "speed": "data_ancillary/imap_glows_plasma-speed-2026d_20251113_v003.dat", + "p-dens": "data_ancillary/imap_glows_proton-density-2026d_20251113_v003.dat", + "uv-anis": "data_ancillary/imap_glows_uv-anisotropy-2026d_20251113_v003.dat", + "phion": "data_ancillary/imap_glows_photoion-2026d_20251113_v003.dat", + "lya": "data_ancillary/imap_glows_lya-2026d_20251113_v003.dat", + "e-dens": "data_ancillary/imap_glows_electron-density-2026d_20251113_v003.dat", + }, +} -EXT_DEPENDENCIES={ - 'f107_raw_data': 'external_dependencies/f107_fluxtable.txt', - 'omni_raw_data': 'external_dependencies/omni2_all_years.dat', - 'lya_raw_data': 'external_dependencies/lyman_alpha_composite.nc' +EXT_DEPENDENCIES = { + "f107_raw_data": "external_dependencies/f107_fluxtable.txt", + "omni_raw_data": "external_dependencies/omni2_all_years.dat", + "lya_raw_data": "external_dependencies/lyman_alpha_composite.nc", } -PHISICAL_CONSTANTS={ - 'm_alpha' : 6.6446573357e-27*u.kg, - 'jd_carrington_first' : 2.398167329*10**6, - 'carrington_time' : 27.2753 +PHISICAL_CONSTANTS = { + "m_alpha": 6.6446573357e-27 * u.kg, + "jd_carrington_first": 2.398167329 * 10**6, + "carrington_time": 27.2753, } -CONST_ALPHA_TO_PROTON=0.04 +CONST_ALPHA_TO_PROTON = 0.04 diff --git a/imap_l3_processing/glows/l3bc/l3bc_toolkit/funcs.py b/imap_l3_processing/glows/l3bc/l3bc_toolkit/funcs.py index 02ea12ea8..09222822e 100644 --- a/imap_l3_processing/glows/l3bc/l3bc_toolkit/funcs.py +++ b/imap_l3_processing/glows/l3bc/l3bc_toolkit/funcs.py @@ -1,7 +1,7 @@ -''' +""" Author: Izabela Kowalska-Leszczynska (ikowalska@cbk.waw.pl) Functions used in L3a->L3b processing that are not class methods -''' +""" import numpy as np import astropy.units as u @@ -14,13 +14,17 @@ from .constants import PHISICAL_CONSTANTS, CONST_ALPHA_TO_PROTON import logging + logging.basicConfig(level=logging.ERROR) + ########################## # Exceptions ########################## class ArrayShapeError(Exception): pass + + class OutOfLimitsError(Exception): pass @@ -40,87 +44,109 @@ def base_funcs_def(i, sin_or_cos): """ if i == 0: + def fun(angle): - return 1.0/(2.0*np.sqrt(np.pi))*np.ones_like(angle) + return 1.0 / (2.0 * np.sqrt(np.pi)) * np.ones_like(angle) + return fun - if sin_or_cos == 's': + if sin_or_cos == "s": + def fun_s(angle): - return np.sin(i*angle)/np.sqrt(np.pi) + return np.sin(i * angle) / np.sqrt(np.pi) + return fun_s - if sin_or_cos == 'c': + if sin_or_cos == "c": + def fun_c(angle): - return np.cos(i*angle)/np.sqrt(np.pi) + return np.cos(i * angle) / np.sqrt(np.pi) + return fun_c # TODO: maybe we need some error if sin_or_cos is anything beside 'c' or 's' - return 'None' + return "None" -def calculate_lya_monthly(t,flux): - ''' - Calculates data that are averaged over each Carrington rotation period. Time is mean value of decimal year in each period. - ''' - t_carr,flux_carr=resize_by_carr(t,flux) - t_mean=Time(np.array([np.mean(t.decimalyear) for t in t_carr]),format='decimalyear') - flux_mean=np.array([f.mean() for f in flux_carr]) - return t_mean,flux_mean +def calculate_lya_monthly(t, flux): + """ + Calculates data that are averaged over each Carrington rotation period. Time is mean value of decimal year in each period. + """ + t_carr, flux_carr = resize_by_carr(t, flux) + t_mean = Time( + np.array([np.mean(t.decimalyear) for t in t_carr]), format="decimalyear" + ) + flux_mean = np.array([f.mean() for f in flux_carr]) + + return t_mean, flux_mean + def calculate_mean_date(fn_list): - daily_dates=[] + daily_dates = [] for f in fn_list: - l3a=read_json(f) - daily_dates.append(l3a['start_time']) - date_mean=Time(daily_dates).mean() + l3a = read_json(f) + daily_dates.append(l3a["start_time"]) + date_mean = Time(daily_dates).mean() return date_mean -def calculate_sw_energy_flux(n,v,p): - ''' - Solar wind energy flux that is latitudinal invariant - Le Chat et al. 2012a - Parameters must be given as a object with unit - Parameters: - -------------- - n - protons number density - v - plasma speed - p - concentration of the alpha particles wrt to the protons +def calculate_sw_energy_flux(n, v, p): + """ + Solar wind energy flux that is latitudinal invariant + Le Chat et al. 2012a + Parameters must be given as a object with unit + + Parameters: + -------------- + n - protons number density + v - plasma speed + p - concentration of the alpha particles wrt to the protons + + Output: + ------------ + energy_flux: energy flux at 1au in the ecliptic plane (invariant) + """ + energy_flux = ( + n + * (const.m_p + p * PHISICAL_CONSTANTS["m_alpha"]) + * v + * (0.5 * v**2 + const.GM_sun / const.R_sun) + ) - Output: - ------------ - energy_flux: energy flux at 1au in the ecliptic plane (invariant) - ''' - energy_flux=n*(const.m_p+p*PHISICAL_CONSTANTS['m_alpha'])*v*(0.5*v**2+const.GM_sun/const.R_sun) + return energy_flux.to("erg.s-1.cm-2") - return energy_flux.to('erg.s-1.cm-2') def carrington(jd): - ''' + """ Converts time in Julian Day format into Carrington Rotation number - ''' - ans=(jd-PHISICAL_CONSTANTS['jd_carrington_first'])/PHISICAL_CONSTANTS['carrington_time']+1 + """ + ans = (jd - PHISICAL_CONSTANTS["jd_carrington_first"]) / PHISICAL_CONSTANTS[ + "carrington_time" + ] + 1 return ans + def check_nan(data): - ''' + """ Checks if there are any NaNs in the output data. If so, raises Exception and data product is not produced Parameters: -------------- data - dictionary with the data product - ''' + """ for key in data: - if np.any((np.isnan(np.array(data[key])))): - raise Exception("There are NaN in the output data") + if np.any((np.isnan(np.array(data[key])))): + raise Exception("There are NaN in the output data") + def cr_from_l3b_fn(fn): - ''' + """ Extracts CR from the L3b filename. This may change if the naming convention will change - ''' - cr=fn.split('_')[5] + """ + cr = fn.split("_")[5] return cr + def cross_section_cx_LS(E): - ''' + """ Lindsay&Stebbings 2005 Returns cross_section as a astro.unit object @@ -132,14 +158,15 @@ def cross_section_cx_LS(E): ------------ sigma: cross-section as a astropy.unit object - ''' - EkeV=E.to('keV').value - out=1e-16*(1-np.exp(-67.3/EkeV))**4.5*(4.15-0.531*np.log(EkeV))**2 - sigma=out*u.cm*u.cm + """ + EkeV = E.to("keV").value + out = 1e-16 * (1 - np.exp(-67.3 / EkeV)) ** 4.5 * (4.15 - 0.531 * np.log(EkeV)) ** 2 + sigma = out * u.cm * u.cm return sigma + def cross_section_cx(E): - ''' + """ Swaczyna et al. 2025 https://doi.org/10.3847/1538-4365/adaf17 Returns cross_section as a astro.unit object @@ -152,24 +179,29 @@ def cross_section_cx(E): ------------ sigma: cross-section as a astropy.unit object - ''' + """ # Parameters - A1=4.8569 - A2=21.906 - A3=31.487 - A4=0.12018 - A5=4.1402E-6 - A6=3.7524 - A7=8.8476E-12 - A8=6.1091 - - EkeV=E.to('keV').value - out=(1e-16)*(A1*np.log(A2/EkeV+A3))/(1+A4*EkeV+A5*EkeV**A6+A7*EkeV**A8) - sigma=out*u.cm*u.cm + A1 = 4.8569 + A2 = 21.906 + A3 = 31.487 + A4 = 0.12018 + A5 = 4.1402e-6 + A6 = 3.7524 + A7 = 8.8476e-12 + A8 = 6.1091 + + EkeV = E.to("keV").value + out = ( + (1e-16) + * (A1 * np.log(A2 / EkeV + A3)) + / (1 + A4 * EkeV + A5 * EkeV**A6 + A7 * EkeV**A8) + ) + sigma = out * u.cm * u.cm return sigma + def derive_sin_cos(order): """ equivalent of Mathematica's deriveSinCos function @@ -183,135 +215,147 @@ def derive_sin_cos(order): """ funcs = [] - for i in range(order+1): + for i in range(order + 1): if i == 0: - funcs.append(base_funcs_def(i, 'None')) + funcs.append(base_funcs_def(i, "None")) else: - funcs.append(base_funcs_def(i, 's')) - funcs.append(base_funcs_def(i, 'c')) + funcs.append(base_funcs_def(i, "s")) + funcs.append(base_funcs_def(i, "c")) return funcs -def f107_daily_data(t,flux): - ''' - Returns adjusted F10.7 flux with daily cadence. In the case of multiple measurements during a day, - the closest to the noon is chosen - - Parameters: - ------------ - t: time array of raw data (astropy.time) - flux: F10.7 flux adjusted to 1au (10^(-22) W/m^2/Hz) - - Output: - ------------ - t_out: daily time array (astropy.time) - flux_out: daily F10.7 flux adjusted to 1au (10^(-22) W/m^2/Hz) - ''' - idx_noon,idx_dup=select_noon(t) # select one measurment per day - idx_no0=np.where(flux[idx_noon]>0)[0] - t_out=t[idx_noon[idx_no0]] - flux_out=flux[idx_noon[idx_no0]] - return t_out,flux_out + +def f107_daily_data(t, flux): + """ + Returns adjusted F10.7 flux with daily cadence. In the case of multiple measurements during a day, + the closest to the noon is chosen + + Parameters: + ------------ + t: time array of raw data (astropy.time) + flux: F10.7 flux adjusted to 1au (10^(-22) W/m^2/Hz) + + Output: + ------------ + t_out: daily time array (astropy.time) + flux_out: daily F10.7 flux adjusted to 1au (10^(-22) W/m^2/Hz) + """ + idx_noon, idx_dup = select_noon(t) # select one measurment per day + idx_no0 = np.where(flux[idx_noon] > 0)[0] + t_out = t[idx_noon[idx_no0]] + flux_out = flux[idx_noon[idx_no0]] + return t_out, flux_out + def find_CR_idx(CR, CR_list): - ''' + """ finds index of the value in the CR_list that is closest to the CR and the next idx it corresponds to the two L3bc files: close to current CR and the next one (for interpolation) - ''' - idx_CR = np.abs(np.array(CR_list)-int(CR)).argmin() - # OLD idx_read=[idx_CR-1,idx_CR,idx_CR+1] - idx_read=[idx_CR,idx_CR+1] + """ + idx_CR = np.abs(np.array(CR_list) - int(CR)).argmin() + # OLD idx_read=[idx_CR-1,idx_CR,idx_CR+1] + idx_read = [idx_CR, idx_CR + 1] return idx_read + def generate_cr_lya(CR, data_lya): - ''' + """ calculates Lyman-alpha composite irradiance interpolated on the center of a given CR - ''' + """ - t_CR = Time(jd_fm_Carrington(CR),format='jd') + t_CR = Time(jd_fm_Carrington(CR), format="jd") CR_list = [int(carrington(t.jd)) for t in data_lya[0]] idx_read = find_CR_idx(CR, CR_list) t_lya = np.array([data_lya[0][i].jd for i in idx_read]) lya = np.array([data_lya[1][i] for i in idx_read]) - lya_CR = np.interp(t_CR.jd,t_lya,lya) + lya_CR = np.interp(t_CR.jd, t_lya, lya) return lya_CR - def get_spher_coord_rad(vec): - """ - Compute radius r, longitude l and latitude b for a vector vec - - Parameters - ---------- - vec: 1D or 2D numpy array - input vector(s) - - Returns - ------- - if vec is an 1D numpy array, return 1D numpy array rlb (see below) - if vec is a 2D numpy array, assume that its rows represent vectors and return 2D array - rlb (see below), where every row of rlb corresponds to every row of vec - - rlb: 1D or 2D numpy array - if 1D then rlb=[r,l,b] where: r - magnitude of vector vec, l - longitude of the direction - pointed by vec in radians, b - latitude of the direction pointed by vec in radians - if 2D then columns of the array contain r,l,b - - Examples - -------- - >>> import geom_fun_libr_box as gflb - >>> import numpy as np - >>> rlb=gflb.get_spher_coord_rad(np.array([0,1,1])) - >>> print(rlb[0],np.degrees(rlb[1]),np.degrees(rlb[2])) - 1.41421356237 90.0 45.0 - >>> n=10 - >>> phi=np.radians(np.linspace(0,360,n)) - >>> tht=np.radians(np.linspace(-90,90,n)) - >>> vecs=np.column_stack([np.cos(phi)*np.cos(tht),np.sin(phi)*np.cos(tht),np.sin(tht)]) - >>> print(gflb.get_spher_coord_rad(vecs)) - [[ 1. 0. -1.57079633] - [ 1. 0.6981317 -1.22173048] - [ 1. 1.3962634 -0.87266463] - [ 1. 2.0943951 -0.52359878] - [ 1. 2.7925268 -0.17453293] - [ 1. 3.4906585 0.17453293] - [ 1. 4.1887902 0.52359878] - [ 1. 4.88692191 0.87266463] - [ 1. 5.58505361 1.22173048] - [ 1. 6.28318531 1.57079633]] - """ - if vec.ndim==1: - if np.shape(vec)[0]!=3: - raise "get_spher_coord_rad() error: np.shape(vec)[0]!=3" - r=np.sqrt(np.dot(vec,vec)) - if r==0.0: b=0.0 - else: b=np.arcsin(vec[2]/r) # compute latitude - # compute longitude - if vec[0]==0.0 and vec[1]==0.0: l=0.0 - else: l=np.arctan2(vec[1],vec[0])%(2*np.pi) - return np.array([r,l,b]) # return one array similarly as in Mathematica - else: - if np.shape(vec)[1]!=3: - raise "get_spher_coord_rad() error: np.shape(vec)[1]!=3" - r=np.sqrt((vec * vec).sum(axis=1)) - idxs=np.nonzero(r!=0.0) - b=np.zeros((len(r)),np.float64) - b[idxs[0]]=np.arcsin(vec[idxs[0],2]/r[idxs[0]]) # compute latitude - l=np.arctan2(vec[:,1],vec[:,0])%(2*np.pi) # compute longitude - idxs=np.nonzero(np.logical_and(vec[:,0]==0.0,vec[:,1]==0.0)) - l[idxs[0]]=0.0 # set l=0 if vec[:,0]==0.0 and vec[:,1]==0.0 - return np.column_stack([r,l,b]) # return one array similarly as in Mathematica + """ + Compute radius r, longitude l and latitude b for a vector vec + + Parameters + ---------- + vec: 1D or 2D numpy array + input vector(s) + + Returns + ------- + if vec is an 1D numpy array, return 1D numpy array rlb (see below) + if vec is a 2D numpy array, assume that its rows represent vectors and return 2D array + rlb (see below), where every row of rlb corresponds to every row of vec + + rlb: 1D or 2D numpy array + if 1D then rlb=[r,l,b] where: r - magnitude of vector vec, l - longitude of the direction + pointed by vec in radians, b - latitude of the direction pointed by vec in radians + if 2D then columns of the array contain r,l,b + + Examples + -------- + >>> import geom_fun_libr_box as gflb + >>> import numpy as np + >>> rlb=gflb.get_spher_coord_rad(np.array([0,1,1])) + >>> print(rlb[0],np.degrees(rlb[1]),np.degrees(rlb[2])) + 1.41421356237 90.0 45.0 + >>> n=10 + >>> phi=np.radians(np.linspace(0,360,n)) + >>> tht=np.radians(np.linspace(-90,90,n)) + >>> vecs=np.column_stack([np.cos(phi)*np.cos(tht),np.sin(phi)*np.cos(tht),np.sin(tht)]) + >>> print(gflb.get_spher_coord_rad(vecs)) + [[ 1. 0. -1.57079633] + [ 1. 0.6981317 -1.22173048] + [ 1. 1.3962634 -0.87266463] + [ 1. 2.0943951 -0.52359878] + [ 1. 2.7925268 -0.17453293] + [ 1. 3.4906585 0.17453293] + [ 1. 4.1887902 0.52359878] + [ 1. 4.88692191 0.87266463] + [ 1. 5.58505361 1.22173048] + [ 1. 6.28318531 1.57079633]] + """ + if vec.ndim == 1: + if np.shape(vec)[0] != 3: + raise "get_spher_coord_rad() error: np.shape(vec)[0]!=3" + r = np.sqrt(np.dot(vec, vec)) + if r == 0.0: + b = 0.0 + else: + b = np.arcsin(vec[2] / r) # compute latitude + # compute longitude + if vec[0] == 0.0 and vec[1] == 0.0: + l = 0.0 + else: + l = np.arctan2(vec[1], vec[0]) % (2 * np.pi) + return np.array([r, l, b]) # return one array similarly as in Mathematica + else: + if np.shape(vec)[1] != 3: + raise "get_spher_coord_rad() error: np.shape(vec)[1]!=3" + r = np.sqrt((vec * vec).sum(axis=1)) + idxs = np.nonzero(r != 0.0) + b = np.zeros((len(r)), np.float64) + b[idxs[0]] = np.arcsin(vec[idxs[0], 2] / r[idxs[0]]) # compute latitude + l = np.arctan2(vec[:, 1], vec[:, 0]) % (2 * np.pi) # compute longitude + idxs = np.nonzero(np.logical_and(vec[:, 0] == 0.0, vec[:, 1] == 0.0)) + l[idxs[0]] = 0.0 # set l=0 if vec[:,0]==0.0 and vec[:,1]==0.0 + return np.column_stack( + [r, l, b] + ) # return one array similarly as in Mathematica + def jd_fm_Carrington(carr): - ''' + """ calculates time in format of Julian Days for a given Carrington Rotation number - ''' - ans=(carr-1)*PHISICAL_CONSTANTS['carrington_time']+PHISICAL_CONSTANTS['jd_carrington_first'] + """ + ans = (carr - 1) * PHISICAL_CONSTANTS["carrington_time"] + PHISICAL_CONSTANTS[ + "jd_carrington_first" + ] return ans + def make_lsq_matrix(psi, base_funcs): """ make data matrix and vector for least-squares method in the matrix form @@ -331,52 +375,56 @@ def make_lsq_matrix(psi, base_funcs): i = i + 1 return lsq_mtrx + def process_omni_param(omni_raw, cr_grid, param_settings): - ''' + """ Processes OMNI2 raw data to obtain averaged solar wind speed, density and alpha-particles abundance - ''' + """ # select from the raw data columns relevant for a given parameter - param_raw=omni_raw[:,param_settings['column_numbers']] + param_raw = omni_raw[:, param_settings["column_numbers"]] # remove gaps where there was no data available based on coded value - param=param_raw[param_raw[:,3] original_last_date - param_v=np.append(param_v,CONST_ALPHA_TO_PROTON) + param_v = np.append(param_v, CONST_ALPHA_TO_PROTON) # split array into 1-Carrington chunks - param_cr, idx_param=np.unique(np.floor(date_cr), return_index=True,axis=0) - param_s=np.split(param_v,idx_param)[1:] + param_cr, idx_param = np.unique(np.floor(date_cr), return_index=True, axis=0) + param_s = np.split(param_v, idx_param)[1:] # calculate averaged over 1 Carrington time and parameter value - param_value=np.array([p.mean() for p in param_s]) + param_value = np.array([p.mean() for p in param_s]) # interpolate into cr_grid - if np.logical_and(param_cr[0]<=cr_grid[0], param_cr[-1]>=cr_grid[-1]): - param_value_interp=np.interp(cr_grid,param_cr,param_value) + if np.logical_and(param_cr[0] <= cr_grid[0], param_cr[-1] >= cr_grid[-1]): + param_value_interp = np.interp(cr_grid, param_cr, param_value) else: - logging.info('There are gaps in the OMNI2 data that cannot be filled by the interpolation') - raise Exception('OMNI Error: not enough data for interpolation') + logging.info( + "There are gaps in the OMNI2 data that cannot be filled by the interpolation" + ) + raise Exception("OMNI Error: not enough data for interpolation") return param_value_interp, used_nominal_per_cr def read_f107_raw_data(fn): - ''' + """ Read raw F10.7 data from file Parameters: @@ -387,61 +435,76 @@ def read_f107_raw_data(fn): ------------ t0: time array (astropy.time) flux_adjusted: F10.7 flux adjusted to 1au (10^(-22) W/m^2/Hz) - ''' + """ data = np.loadtxt(fn) - jd=data[:,2] # time of observation in JD - flux_adjusted=data[:,5] # adjusted do 1AU flux in sfu 1sfu=10^(-22) W/m^2/Hz (taking into accont that Sun-Earth distance is changing) - t0=Time(jd,format='jd') # time converted to astropy.time format + jd = data[:, 2] # time of observation in JD + flux_adjusted = data[ + :, 5 + ] # adjusted do 1AU flux in sfu 1sfu=10^(-22) W/m^2/Hz (taking into accont that Sun-Earth distance is changing) + t0 = Time(jd, format="jd") # time converted to astropy.time format + + return t0, flux_adjusted - return t0,flux_adjusted -def read_hdr_txt(fn,N): - ''' +def read_hdr_txt(fn, N): + """ Reads header from the text file and preapers it to the further analysis - ''' + """ # open file and read header with open(fn) as file: - hdr=[next(file) for _ in range(N)] + hdr = [next(file) for _ in range(N)] # remove hash - hdr_str=remove_hash(hdr) + hdr_str = remove_hash(hdr) return hdr_str + def read_lya_data(fn): - ''' + """ reads daily Lyman-alpha composite irradiance and calculates monthly averages - ''' - t_lya,flux_lya,_= read_lya_raw_data(fn) - t_lya_m,flux_lya_m = calculate_lya_monthly(t_lya,flux_lya) - return [t_lya_m,flux_lya_m] + """ + t_lya, flux_lya, _ = read_lya_raw_data(fn) + t_lya_m, flux_lya_m = calculate_lya_monthly(t_lya, flux_lya) + return [t_lya_m, flux_lya_m] + def read_lya_raw_data(filename): - ''' + """ Reads *.nc file with composite Lyman-alpha solar irradiance (from LAPS) Returns daily product adjusted to 1au. Flux is in ph/(cm^2 s^2) - ''' - - data=netCDF4.Dataset(filename) - - convert_to_ph=data.variables['convert_to_photons'][0].data # Multiplicative factor to convert irr_121 from W/m^2 to ph/(cm^2 s^2) [ph / (cm^2 s^2)] / (W / m^2) - t_d1947=data.variables['time'][:].data # time of observation [days since 1947-01-01 12:00:00 UTC] - flux0=data.variables['irr_121'][:].data # Averaged daily irradiance in band from 121.0 to 122.0 nm at 1 AU [W/m^2] - flux=flux0*convert_to_ph # Flux in ph/(cm^2 s^2) - flux_uncert=data.variables['irr_121_uncertainty'][:].data # Uncertainty (1 standard deviation) of average daily irradiance [W/m^2] + """ - t0=Time('1947-01-01 12:00') + data = netCDF4.Dataset(filename) + + convert_to_ph = data.variables[ + "convert_to_photons" + ][ + 0 + ].data # Multiplicative factor to convert irr_121 from W/m^2 to ph/(cm^2 s^2) [ph / (cm^2 s^2)] / (W / m^2) + t_d1947 = data.variables["time"][ + : + ].data # time of observation [days since 1947-01-01 12:00:00 UTC] + flux0 = data.variables["irr_121"][ + : + ].data # Averaged daily irradiance in band from 121.0 to 122.0 nm at 1 AU [W/m^2] + flux = flux0 * convert_to_ph # Flux in ph/(cm^2 s^2) + flux_uncert = data.variables["irr_121_uncertainty"][ + : + ].data # Uncertainty (1 standard deviation) of average daily irradiance [W/m^2] + + t0 = Time("1947-01-01 12:00") # convertion to jd and adding days since 1947-01-01 - it is more accurate than simple adding days to date - t=Time(t0.jd+t_d1947,format='jd') + t = Time(t0.jd + t_d1947, format="jd") return t, flux, flux_uncert -def read_raw_OMNI_data(ext_dependencies,t_window=None): - ''' +def read_raw_OMNI_data(ext_dependencies, t_window=None): + """ Reads 3 types of data from OMNI2 database: plasma speed, proton density, and alpha-particles abundance in the ecliptic plane Parameters: @@ -450,48 +513,55 @@ def read_raw_OMNI_data(ext_dependencies,t_window=None): list of file names with external dependencies t_window: list [astropy.time, astropy.time] time window of the data that are needed - ''' + """ # Read OMNI2 data (hour resolution) # year, DOY, hour, S/C Id, proton_dens, plasma_speed, alpha_abundance, sigma proton_dens, sigma plasma_speed, sigma alpha_abundance - omni_raw=np.loadtxt(ext_dependencies['omni_raw_data'],usecols=(0,1,2,5,23,24,27,30,31,34)) - omni_raw_date=time_from_yday(omni_raw[:,(0,1,2)]) - if t_window==None: out=omni_raw + omni_raw = np.loadtxt( + ext_dependencies["omni_raw_data"], usecols=(0, 1, 2, 5, 23, 24, 27, 30, 31, 34) + ) + omni_raw_date = time_from_yday(omni_raw[:, (0, 1, 2)]) + if t_window == None: + out = omni_raw else: - idx_ini=np.abs(omni_raw_date-t_window[0]).argmin() - idx_fin=np.abs(omni_raw_date-t_window[1]).argmin() + idx_ini = np.abs(omni_raw_date - t_window[0]).argmin() + idx_fin = np.abs(omni_raw_date - t_window[1]).argmin() - out=omni_raw[idx_ini:idx_fin] + out = omni_raw[idx_ini:idx_fin] return out + def read_json(fn): - ''' + """ Opens, reads and returns the content of the json file - ''' - fp = open(fn, 'r') + """ + fp = open(fn, "r") data = json.load(fp) fp.close() return data + def remove_hash(hdr): - ''' + """ Removes # from the header that is read from the text files and returns string that will be used to generate another header - ''' - hdr_str=[l[2:] for l in hdr] + """ + hdr_str = [l[2:] for l in hdr] return hdr_str -def resize_by_carr(t,flux): - ''' - Reshaping flux array in such a way that each element is a list of fluxes from the same carrington rotation number - ''' - t_carr=carrington(t.jd) - n_carr=np.array([int(t_carr[i]) for i in np.arange(len(t_carr))]) - flux_carr_sort=np.split(flux,np.unique(n_carr,return_index=True)[1][1:]) - t_carr_sort=np.split(t,np.unique(n_carr,return_index=True)[1][1:]) - return t_carr_sort,flux_carr_sort + +def resize_by_carr(t, flux): + """ + Reshaping flux array in such a way that each element is a list of fluxes from the same carrington rotation number + """ + t_carr = carrington(t.jd) + n_carr = np.array([int(t_carr[i]) for i in np.arange(len(t_carr))]) + flux_carr_sort = np.split(flux, np.unique(n_carr, return_index=True)[1][1:]) + t_carr_sort = np.split(t, np.unique(n_carr, return_index=True)[1][1:]) + return t_carr_sort, flux_carr_sort + def scale_density(date, dens): - ''' + """ Scaling density measurements that are taken at different distance than 1au. We need solar wind parameters ajusted to 1au. Proton density decreases as 1/r^2 @@ -502,21 +572,22 @@ def scale_density(date, dens): time of the measurement dens : 1D float vector measured density [#/cm^3] - ''' + """ # Sun-Earth distance at given moment - d_sun=get_sun(date).distance + d_sun = get_sun(date).distance # Sun-L1 distance - r_obs=d_sun*(1-(const.M_earth/(3*const.M_sun))**(1./3)) + r_obs = d_sun * (1 - (const.M_earth / (3 * const.M_sun)) ** (1.0 / 3)) # density ajustment - dens_1au=dens*(r_obs/u.au)**2 + dens_1au = dens * (r_obs / u.au) ** 2 return dens_1au -def select_data_window(data,t_window): - ''' + +def select_data_window(data, t_window): + """ Selects data whitin a given time window Parameters: @@ -526,15 +597,16 @@ def select_data_window(data,t_window): t_ini: astropy.time t_end: astropy.time - ''' - idx=np.where(np.logical_and(data[0]>=t_window[0],data[0]<=t_window[1]))[0] - t=data[0][idx] - data=data[1][idx] + """ + idx = np.where(np.logical_and(data[0] >= t_window[0], data[0] <= t_window[1]))[0] + t = data[0][idx] + data = data[1][idx] + + return [t, data] - return [t,data] def select_noon(t_pen): - ''' + """ Select the dates that are closest to the local noon Parameters: @@ -546,119 +618,203 @@ def select_noon(t_pen): 1. index list of all data taken at noon (20 UT) or the closest one if there is no data at noon 2. a list of indexes of data taken on the same day (if there is only one measurement then this element is just 1 element list np. [[1],[2,3,4],[6]] - ''' - #TODO it should be done in a dynamical way - it should determine how many measurements are in a given day - index=np.array([0]) - index_duplicate=[] - i=0 - while i<(len(t_pen)-3): - - if (t_pen[i].yday[:8]==t_pen[i+1].yday[:8]): - - if(t_pen[i].yday[:8]==t_pen[i+2].yday[:8]): - if(t_pen[i].yday[:8]==t_pen[i+3].yday[:8]): - h=np.array([int(t_pen[i].yday[9:11]),int(t_pen[i+1].yday[9:11]),int(t_pen[i+2].yday[9:11]),int(t_pen[i+3].yday[9:11])]) - k=i+np.abs(h-20).argmin() # index of the hour that is near 20UT (local noon) - d=[i,i+1,i+2,i+3] - i=i+4 + """ + # TODO it should be done in a dynamical way - it should determine how many measurements are in a given day + index = np.array([0]) + index_duplicate = [] + i = 0 + while i < (len(t_pen) - 3): + if t_pen[i].yday[:8] == t_pen[i + 1].yday[:8]: + if t_pen[i].yday[:8] == t_pen[i + 2].yday[:8]: + if t_pen[i].yday[:8] == t_pen[i + 3].yday[:8]: + h = np.array( + [ + int(t_pen[i].yday[9:11]), + int(t_pen[i + 1].yday[9:11]), + int(t_pen[i + 2].yday[9:11]), + int(t_pen[i + 3].yday[9:11]), + ] + ) + k = ( + i + np.abs(h - 20).argmin() + ) # index of the hour that is near 20UT (local noon) + d = [i, i + 1, i + 2, i + 3] + i = i + 4 else: - h=np.array([int(t_pen[i].yday[9:11]),int(t_pen[i+1].yday[9:11]),int(t_pen[i+2].yday[9:11])]) - k=i+np.abs(h-20).argmin() # index of the hour that is near 20UT (local noon) - d=[i,i+1,i+2] - i=i+3 + h = np.array( + [ + int(t_pen[i].yday[9:11]), + int(t_pen[i + 1].yday[9:11]), + int(t_pen[i + 2].yday[9:11]), + ] + ) + k = ( + i + np.abs(h - 20).argmin() + ) # index of the hour that is near 20UT (local noon) + d = [i, i + 1, i + 2] + i = i + 3 else: - h=np.array([int(t_pen[i].yday[9:11]),int(t_pen[i+1].yday[9:11])]) - k=i+np.abs(h-20).argmin() # index of the hour that is near 20UT (local noon) - d=[i,i+1] - i=i+2 + h = np.array([int(t_pen[i].yday[9:11]), int(t_pen[i + 1].yday[9:11])]) + k = ( + i + np.abs(h - 20).argmin() + ) # index of the hour that is near 20UT (local noon) + d = [i, i + 1] + i = i + 2 else: - k=i - d=[i] - i=i+1 - index=np.append(index,k) + k = i + d = [i] + i = i + 1 + index = np.append(index, k) index_duplicate.append(d) - return index[1:],index_duplicate + return index[1:], index_duplicate + def test_WawHelioInMP_file(model): - ''' + """ test if json file has what it should inside - ''' - - vg_points_number=model['header']['vg_points_number'] - hion_filtration=model['params_hion']['hion_filtration'] - hion_order=model['params_hion']['hion_order'] - - hion_const_0=np.array(model['params_hion']['hion_const']).shape[0] - hion_base=np.array(model['params_hion']['hion_base']) - hion_base_0=np.array(model['params_hion']['hion_base']).shape[0] - hion_base_1=np.array(model['params_hion']['hion_base']).shape[1] - - if vg_points_number!=len(model['vg_points']): - logging.error('vg_points_number is incorrect.\nIt is '+str(vg_points_number) - +'\nIt should be '+str(len(model['vg_points']))) - raise ArrayShapeError('Array shape error: Array shape is incorrect') - - elif hion_const_0!=(hion_order-1-hion_filtration): - logging.error('hion_const shape is incorrect.\nIt is '+str(hion_const_0) - +'\nIt should be '+str(hion_order-1-hion_filtration)) - raise ArrayShapeError('Array shape error: Array shape is incorect') - - elif np.logical_or(hion_base_0!=hion_order-1,hion_base_1!=hion_order+1): - logging.error('hion_base shape is incorrect.\nIt is '+str(hion_base.shape) - +'\nIt should be ('+str(hion_order-1)+','+str(hion_order+1)+')') - raise ArrayShapeError('Array shape error: Array shape is incorect') - - - for k in np.arange(model['header']['vg_points_number']): - - lcrv_filtration=model['vg_points'][k]['parameters']['lcrv_filtration'] - lcrv_order=model['vg_points'][k]['parameters']['lcrv_order'] - - model_const_0=np.array(model['vg_points'][k]['parameters']['model_const']).shape[0] - model_matrix=np.array(model['vg_points'][k]['parameters']['model_matrix']) - model_matrix_0=np.array(model['vg_points'][k]['parameters']['model_matrix']).shape[0] - model_matrix_1=np.array(model['vg_points'][k]['parameters']['model_matrix']).shape[1] - - lcrv_Trmatrix=np.array(model['vg_points'][k]['parameters']['lcrv_Trmatrix']) - lcrv_Trmatrix_0=np.array(model['vg_points'][k]['parameters']['lcrv_Trmatrix']).shape[0] - lcrv_Trmatrix_1=np.array(model['vg_points'][k]['parameters']['lcrv_Trmatrix']).shape[1] - - if model_const_0!=hion_filtration: - logging.error('model_const shape is incorrect.\nIt is '+str(model_const_0) - +'\nIt should be '+str(hion_filtration)) - raise ArrayShapeError('Array shape error: Array shape is incorrect') + """ - elif np.logical_or(model_matrix_0!=hion_filtration,model_matrix_1!=lcrv_filtration): - logging.error('model_matrix shape is incorrect.\nIt is '+str(model_matrix.shape) - +'\nIt should be ('+str(hion_filtration)+','+str(lcrv_filtration)+')') - raise ArrayShapeError('Array shape error: Array shape is incorect') + vg_points_number = model["header"]["vg_points_number"] + hion_filtration = model["params_hion"]["hion_filtration"] + hion_order = model["params_hion"]["hion_order"] + + hion_const_0 = np.array(model["params_hion"]["hion_const"]).shape[0] + hion_base = np.array(model["params_hion"]["hion_base"]) + hion_base_0 = np.array(model["params_hion"]["hion_base"]).shape[0] + hion_base_1 = np.array(model["params_hion"]["hion_base"]).shape[1] + + if vg_points_number != len(model["vg_points"]): + logging.error( + "vg_points_number is incorrect.\nIt is " + + str(vg_points_number) + + "\nIt should be " + + str(len(model["vg_points"])) + ) + raise ArrayShapeError("Array shape error: Array shape is incorrect") + + elif hion_const_0 != (hion_order - 1 - hion_filtration): + logging.error( + "hion_const shape is incorrect.\nIt is " + + str(hion_const_0) + + "\nIt should be " + + str(hion_order - 1 - hion_filtration) + ) + raise ArrayShapeError("Array shape error: Array shape is incorect") + + elif np.logical_or(hion_base_0 != hion_order - 1, hion_base_1 != hion_order + 1): + logging.error( + "hion_base shape is incorrect.\nIt is " + + str(hion_base.shape) + + "\nIt should be (" + + str(hion_order - 1) + + "," + + str(hion_order + 1) + + ")" + ) + raise ArrayShapeError("Array shape error: Array shape is incorect") + + for k in np.arange(model["header"]["vg_points_number"]): + lcrv_filtration = model["vg_points"][k]["parameters"]["lcrv_filtration"] + lcrv_order = model["vg_points"][k]["parameters"]["lcrv_order"] + + model_const_0 = np.array( + model["vg_points"][k]["parameters"]["model_const"] + ).shape[0] + model_matrix = np.array(model["vg_points"][k]["parameters"]["model_matrix"]) + model_matrix_0 = np.array( + model["vg_points"][k]["parameters"]["model_matrix"] + ).shape[0] + model_matrix_1 = np.array( + model["vg_points"][k]["parameters"]["model_matrix"] + ).shape[1] + + lcrv_Trmatrix = np.array(model["vg_points"][k]["parameters"]["lcrv_Trmatrix"]) + lcrv_Trmatrix_0 = np.array( + model["vg_points"][k]["parameters"]["lcrv_Trmatrix"] + ).shape[0] + lcrv_Trmatrix_1 = np.array( + model["vg_points"][k]["parameters"]["lcrv_Trmatrix"] + ).shape[1] + + if model_const_0 != hion_filtration: + logging.error( + "model_const shape is incorrect.\nIt is " + + str(model_const_0) + + "\nIt should be " + + str(hion_filtration) + ) + raise ArrayShapeError("Array shape error: Array shape is incorrect") + + elif np.logical_or( + model_matrix_0 != hion_filtration, model_matrix_1 != lcrv_filtration + ): + logging.error( + "model_matrix shape is incorrect.\nIt is " + + str(model_matrix.shape) + + "\nIt should be (" + + str(hion_filtration) + + "," + + str(lcrv_filtration) + + ")" + ) + raise ArrayShapeError("Array shape error: Array shape is incorect") + + elif np.logical_or( + lcrv_Trmatrix_0 != 2 * lcrv_order + 1, lcrv_Trmatrix_1 != 2 * lcrv_order + 1 + ): + logging.error( + "lcrv_Trmatrix shape is incorrect.\nIt is " + + str(lcrv_Trmatrix.shape) + + "\nIt should be (" + + str(2 * lcrv_order + 1) + + "," + + str(2 * lcrv_order + 1) + + ")" + ) + raise ArrayShapeError("Array shape error: Array shape is incorect") - elif np.logical_or(lcrv_Trmatrix_0!=2*lcrv_order+1,lcrv_Trmatrix_1!=2*lcrv_order+1): - logging.error('lcrv_Trmatrix shape is incorrect.\nIt is '+str(lcrv_Trmatrix.shape) - +'\nIt should be ('+str(2*lcrv_order+1)+','+str(2*lcrv_order+1)+')') - raise ArrayShapeError('Array shape error: Array shape is incorect') def time_string_from_l3a_fn(fn): - ''' + """ Extracts date from the L3a filename. This may change if the naming convention will change - ''' - fn_string=fn.split('_')[4] - time_string=fn_string[0:4]+'-'+fn_string[4:6]+'-'+fn_string[6:8]+' '+fn_string[8:10]+':'+fn_string[10:12]+':'+fn_string[12:14] + """ + fn_string = fn.split("_")[4] + time_string = ( + fn_string[0:4] + + "-" + + fn_string[4:6] + + "-" + + fn_string[6:8] + + " " + + fn_string[8:10] + + ":" + + fn_string[10:12] + + ":" + + fn_string[12:14] + ) return time_string + def time_from_l3a(fn): - data_l3a=read_json(fn) - mean_time=Time(data_l3a['start_time'])+0.5*(Time(data_l3a['end_time'])-Time(data_l3a['start_time'])) - return [mean_time, Time(data_l3a['start_time']), Time(data_l3a['end_time'])] + data_l3a = read_json(fn) + mean_time = Time(data_l3a["start_time"]) + 0.5 * ( + Time(data_l3a["end_time"]) - Time(data_l3a["start_time"]) + ) + return [mean_time, Time(data_l3a["start_time"]), Time(data_l3a["end_time"])] + def time_from_yday(yday_list): - string = [f'{int(yday_list[i, 0])}:{int(yday_list[i, 1]):03d}:{int(yday_list[i, 2]):02d}:00' for i in range(len(yday_list))] - date=Time(string,format='yday') - return date + string = [ + f"{int(yday_list[i, 0])}:{int(yday_list[i, 1]):03d}:{int(yday_list[i, 2]):02d}:00" + for i in range(len(yday_list)) + ] + date = Time(string, format="yday") + return date + def v2E(v): - ''' + """ Speed to kinetic energy transformation for hydrogen Parameters: @@ -668,8 +824,8 @@ def v2E(v): Output: ------------ E: energy as an astropy.unit object - ''' + """ - m_H=1.6733e-24*u.g - E=(0.5*m_H*v**2).to('eV') + m_H = 1.6733e-24 * u.g + E = (0.5 * m_H * v**2).to("eV") return E diff --git a/imap_l3_processing/glows/l3bc/l3bc_toolkit/l3b_CarringtonIonRate.py b/imap_l3_processing/glows/l3bc/l3bc_toolkit/l3b_CarringtonIonRate.py index 76c8db678..f2df1dc99 100644 --- a/imap_l3_processing/glows/l3bc/l3bc_toolkit/l3b_CarringtonIonRate.py +++ b/imap_l3_processing/glows/l3bc/l3bc_toolkit/l3b_CarringtonIonRate.py @@ -1,7 +1,7 @@ -''' +""" Author: Izabela Kowalska-Leszczynska (ikowalska@cbk.waw.pl) Carrington averaged ionization rate class -''' +""" import json import logging @@ -16,9 +16,10 @@ logging.basicConfig(level=logging.ERROR) + @dataclass -class CarringtonIonizationRate(): - ''' +class CarringtonIonizationRate: + """ Class for a GLOWS Carrington averaged ionization rate profiles Contains original daily ionization rate profiles, averaged ionization rate profile charge exchange and photoionization profiles @@ -87,47 +88,47 @@ class CarringtonIonizationRate(): Read file with UV latitudinal anisotrpy factor averaged over 1 Carrington rotation period save_to_file(self,fn) saves L3b structure to file - ''' + """ - def __init__(self, anc_input_from_instr_team,ext_dependencies): + def __init__(self, anc_input_from_instr_team, ext_dependencies): """ Class constructor """ - self.header={ - 'software_version': VERSION, - 'filename': None, - 'ancillary_data_files': anc_input_from_instr_team, - 'external_dependeciens': ext_dependencies['f107_raw_data'], + self.header = { + "software_version": VERSION, + "filename": None, + "ancillary_data_files": anc_input_from_instr_team, + "external_dependeciens": ext_dependencies["f107_raw_data"], } # read pipeline settings to be used for L3b - self.settings = fun.read_json(anc_input_from_instr_team['pipeline_settings']) + self.settings = fun.read_json(anc_input_from_instr_team["pipeline_settings"]) - self.sw_ecliptic={} - self.sw_ecliptic['mean_speed']=None - self.sw_ecliptic['mean_proton_density']=None + self.sw_ecliptic = {} + self.sw_ecliptic["mean_speed"] = None + self.sw_ecliptic["mean_proton_density"] = None # All parameters will be read from L3a file or computed later on - self.daily_ion_rate={} - self.daily_ion_rate['date']=None - self.daily_ion_rate['ion_rate']=None - - self.carr_ion_rate={} - self.carr_ion_rate['date']=None - self.carr_ion_rate['CR']=None - self.carr_ion_rate['ion_grid']=self.settings['ion_rate_grid'] - self.carr_ion_rate['ion_rate']=None - self.carr_ion_rate['cx_rate']=None - self.carr_ion_rate['ph_rate']=None - self.carr_ion_rate['ion_rate_uncert']=None - self.carr_ion_rate['cx_rate_uncert']=None - self.carr_ion_rate['ph_rate_uncert']=None - - self.uv_anisotropy=None - self.uv_anisotropy_flag=None - - def _anchor_to_ecliptic(self,carr_ion_rate): - ''' + self.daily_ion_rate = {} + self.daily_ion_rate["date"] = None + self.daily_ion_rate["ion_rate"] = None + + self.carr_ion_rate = {} + self.carr_ion_rate["date"] = None + self.carr_ion_rate["CR"] = None + self.carr_ion_rate["ion_grid"] = self.settings["ion_rate_grid"] + self.carr_ion_rate["ion_rate"] = None + self.carr_ion_rate["cx_rate"] = None + self.carr_ion_rate["ph_rate"] = None + self.carr_ion_rate["ion_rate_uncert"] = None + self.carr_ion_rate["cx_rate_uncert"] = None + self.carr_ion_rate["ph_rate_uncert"] = None + + self.uv_anisotropy = None + self.uv_anisotropy_flag = None + + def _anchor_to_ecliptic(self, carr_ion_rate): + """ normalizes the profile in such a way that the value in its central bin matches the average ionization rate calculated based on the measured parameters of the solar wind in the ecliptic plane @@ -136,120 +137,162 @@ def _anchor_to_ecliptic(self,carr_ion_rate): --------------- carr_ion_rate: mean ionization rate profile calculated by taking average value of all daily ionization rate profiles within considered CR - ''' - idx_middle=int(len(self.carr_ion_rate['ion_grid'])/2) # middle bin (equator) in the latitudinal grid - carr_ion_rate0=carr_ion_rate[idx_middle] - - ph_rate0=self.carr_ion_rate['ph_rate'][idx_middle] # ph_rate at latitude=0 - v0=self.sw_ecliptic['mean_speed']*u.km/u.s - n0=self.sw_ecliptic['mean_proton_density']/(u.cm)**3 - sigma0=fun.cross_section_cx(fun.v2E(v0)) + """ + idx_middle = int( + len(self.carr_ion_rate["ion_grid"]) / 2 + ) # middle bin (equator) in the latitudinal grid + carr_ion_rate0 = carr_ion_rate[idx_middle] - cx_rate0=(v0*sigma0*n0).to('1/s').value + ph_rate0 = self.carr_ion_rate["ph_rate"][idx_middle] # ph_rate at latitude=0 + v0 = self.sw_ecliptic["mean_speed"] * u.km / u.s + n0 = self.sw_ecliptic["mean_proton_density"] / (u.cm) ** 3 + sigma0 = fun.cross_section_cx(fun.v2E(v0)) - ion_rate0=ph_rate0+cx_rate0 + cx_rate0 = (v0 * sigma0 * n0).to("1/s").value - self.carr_ion_rate['ion_rate']=carr_ion_rate/carr_ion_rate0*ion_rate0 + ion_rate0 = ph_rate0 + cx_rate0 + self.carr_ion_rate["ion_rate"] = carr_ion_rate / carr_ion_rate0 * ion_rate0 def calculate_averaged_profile(self): - ''' + """ Mean value of the ionization profile and time. Average taken bin-by-bin on the latitudinal grid. - ''' + """ - carr_ion_rate=np.mean(self.daily_ion_rate['ion_rate'],axis=0) + carr_ion_rate = np.mean(self.daily_ion_rate["ion_rate"], axis=0) # Anchore averaged ionization rate profile to the solar wind measurements in the ecliptic plane self._anchor_to_ecliptic(carr_ion_rate) - self.carr_ion_rate['date']=np.mean(self.daily_ion_rate['date']) + self.carr_ion_rate["date"] = np.mean(self.daily_ion_rate["date"]) # Statistical uncertinity - if len(self.daily_ion_rate['ion_rate'])>1: - self.carr_ion_rate['ion_rate_uncert']=np.std(self.daily_ion_rate['ion_rate'],axis=0,ddof=1) - else: self.carr_ion_rate['ion_rate_uncert']=-1.0E31*np.ones_like(self.carr_ion_rate['ion_rate']) - - + if len(self.daily_ion_rate["ion_rate"]) > 1: + self.carr_ion_rate["ion_rate_uncert"] = np.std( + self.daily_ion_rate["ion_rate"], axis=0, ddof=1 + ) + else: + self.carr_ion_rate["ion_rate_uncert"] = -1.0e31 * np.ones_like( + self.carr_ion_rate["ion_rate"] + ) def calculate_charge_exchange(self): - ''' + """ Charge exchange rate for a stationary atom aproximation assuming that total ionization rate is a sum of photoionization and charge exchange rate. - ''' + """ # ionization rate obtained from the light curve is a sum of charge exchange rate and photoionization rate - self.carr_ion_rate['cx_rate']=self.carr_ion_rate['ion_rate']-self.carr_ion_rate['ph_rate'] + self.carr_ion_rate["cx_rate"] = ( + self.carr_ion_rate["ion_rate"] - self.carr_ion_rate["ph_rate"] + ) # statistical uncertainty - if len(self.carr_ion_rate['cx_rate'])>1: - self.carr_ion_rate['cx_rate_uncert']=self.carr_ion_rate['ion_rate_uncert']+self.carr_ion_rate['ph_rate_uncert'] + if len(self.carr_ion_rate["cx_rate"]) > 1: + self.carr_ion_rate["cx_rate_uncert"] = ( + self.carr_ion_rate["ion_rate_uncert"] + + self.carr_ion_rate["ph_rate_uncert"] + ) else: - self.carr_ion_rate['cx_rate_uncert']=-1.0E31*np.ones_like(self.carr_ion_rate['cx_rate']) + self.carr_ion_rate["cx_rate_uncert"] = -1.0e31 * np.ones_like( + self.carr_ion_rate["cx_rate"] + ) - def calculate_photoion(self, anc_input_from_instr_team,ext_dependencies): - ''' + def calculate_photoion(self, anc_input_from_instr_team, ext_dependencies): + """ Photoionization rate at 1au averaged over a time window Based on daily photoionization calculated from F10.7 using WawHelioIon method - ''' + """ # raw data from the same period as available daily profiles - t_ini=self.daily_ion_rate['date'][0] - t_end=self.daily_ion_rate['date'][-1] - t_mid=t_ini+(t_end-t_ini)/2 + t_ini = self.daily_ion_rate["date"][0] + t_end = self.daily_ion_rate["date"][-1] + t_mid = t_ini + (t_end - t_ini) / 2 # read F10.7 file (the raw data from LASP) - t_raw,flux_raw=fun.read_f107_raw_data(ext_dependencies['f107_raw_data']) - t_daily, flux_daily0=fun.f107_daily_data(t_raw,flux_raw) # zmiana + t_raw, flux_raw = fun.read_f107_raw_data(ext_dependencies["f107_raw_data"]) + t_daily, flux_daily0 = fun.f107_daily_data(t_raw, flux_raw) # zmiana # select data from the window - #data_window=fun.select_data_window([t_raw,flux_raw],[t_ini,t_end]) zmiana - data_window=fun.select_data_window([t_daily,flux_daily0],[t_ini,t_end]) + # data_window=fun.select_data_window([t_raw,flux_raw],[t_ini,t_end]) zmiana + data_window = fun.select_data_window([t_daily, flux_daily0], [t_ini, t_end]) # check if there are at least 2 data points in the window - photoion_daily=[] + photoion_daily = [] - if len(data_window[0])>2: - flux_daily=data_window[1] + if len(data_window[0]) > 2: + flux_daily = data_window[1] # apply correlation between photoion and F10.7 - photoion_daily=self._calculate_photoion_from_f107(flux_daily) + photoion_daily = self._calculate_photoion_from_f107(flux_daily) # average daily photoionization - photoion_mean=photoion_daily.mean() - photoion_std=np.std(photoion_daily,ddof=1) + photoion_mean = photoion_daily.mean() + photoion_std = np.std(photoion_daily, ddof=1) else: # if there are less than 2 data points in the time window, we want to interpolate # between mean values of the previous and next CR - t_ini_next=Time(fun.jd_fm_Carrington(self.carr_ion_rate['CR']+1),format='jd') - t_end_next=Time(fun.jd_fm_Carrington(self.carr_ion_rate['CR']+2),format='jd') - data_window_next=fun.select_data_window([t_raw,flux_raw],[t_ini_next,t_end_next]) - if len(data_window_next[0])>2: + t_ini_next = Time( + fun.jd_fm_Carrington(self.carr_ion_rate["CR"] + 1), format="jd" + ) + t_end_next = Time( + fun.jd_fm_Carrington(self.carr_ion_rate["CR"] + 2), format="jd" + ) + data_window_next = fun.select_data_window( + [t_raw, flux_raw], [t_ini_next, t_end_next] + ) + if len(data_window_next[0]) > 2: # if there are at least data points in the next CR, then we can interpolate - t_ini_prev=Time(fun.jd_fm_Carrington(self.carr_ion_rate['CR']-1),format='jd') - t_end_prev=Time(fun.jd_fm_Carrington(self.carr_ion_rate['CR']),format='jd') - data_window_prev=fun.select_data_window([t_raw,flux_raw],[t_ini_prev,t_end_prev]) - - data_next_mean=[data_window_next[0].mean(),data_window_next[1].mean()] - data_prev_mean=[data_window_prev[0].mean(),data_window_prev[1].mean()] - - data_mean=[t_mid,np.interp(t_mid.jd,np.array([data_next_mean[0].jd,data_prev_mean[0].jd]),np.array([data_next_mean[1],data_prev_mean[1]]))] - - photoion_mean=self._calculate_photoion_from_f107(data_mean[1]) + t_ini_prev = Time( + fun.jd_fm_Carrington(self.carr_ion_rate["CR"] - 1), format="jd" + ) + t_end_prev = Time( + fun.jd_fm_Carrington(self.carr_ion_rate["CR"]), format="jd" + ) + data_window_prev = fun.select_data_window( + [t_raw, flux_raw], [t_ini_prev, t_end_prev] + ) + + data_next_mean = [ + data_window_next[0].mean(), + data_window_next[1].mean(), + ] + data_prev_mean = [ + data_window_prev[0].mean(), + data_window_prev[1].mean(), + ] + + data_mean = [ + t_mid, + np.interp( + t_mid.jd, + np.array([data_next_mean[0].jd, data_prev_mean[0].jd]), + np.array([data_next_mean[1], data_prev_mean[1]]), + ), + ] + + photoion_mean = self._calculate_photoion_from_f107(data_mean[1]) else: # if there are no data in the next CR, then we need to wait for another CR - raise Exception("There are no data to interpolate F10.7 for CR=%d",self.carr_ion_rate['CR']) - + raise Exception( + "There are no data to interpolate F10.7 for CR=%d", + self.carr_ion_rate["CR"], + ) # including latitudinal anisotropy factor - self._read_uv_anisotrpy(anc_input_from_instr_team['uv_anisotropy']) - self.carr_ion_rate['ph_rate']=photoion_mean*self.uv_anisotropy + self._read_uv_anisotrpy(anc_input_from_instr_team["uv_anisotropy"]) + self.carr_ion_rate["ph_rate"] = photoion_mean * self.uv_anisotropy - if len(photoion_daily)>1: - self.carr_ion_rate['ph_rate_uncert']=photoion_std*np.ones_like(self.carr_ion_rate['ph_rate']) + if len(photoion_daily) > 1: + self.carr_ion_rate["ph_rate_uncert"] = photoion_std * np.ones_like( + self.carr_ion_rate["ph_rate"] + ) else: - self.carr_ion_rate['ph_rate_uncert']=-1.0E31*np.ones_like(self.carr_ion_rate['ph_rate']) + self.carr_ion_rate["ph_rate_uncert"] = -1.0e31 * np.ones_like( + self.carr_ion_rate["ph_rate"] + ) - def _calculate_photoion_from_f107(self,flux): - ''' + def _calculate_photoion_from_f107(self, flux): + """ Correlation between photoionization rate and F10.7 (sokol_bzowski:14a, sokol_etal:20a) Parameters: @@ -260,50 +303,56 @@ def _calculate_photoion_from_f107(self,flux): ------------ photoion: the value of the photoionization at 1au on ecliptic calculated from F10.7 - ''' - photoion=-2.9819e-8+2.416e-8*flux**0.4017 + """ + photoion = -2.9819e-8 + 2.416e-8 * flux**0.4017 return photoion - def _read_uv_anisotrpy(self,fn): - ''' + def _read_uv_anisotrpy(self, fn): + """ Read file with UV latitudinal anisotrpy factor averaged over 1 Carrington rotation period. File will be delivered by the instrument team Parameters: -------------- fn: file name (str) - ''' + """ anisotropy = fun.read_json(fn) - self.uv_anisotropy = np.array(anisotropy['anisotropy_factor']) - self.uv_anisotropy_flag = anisotropy['flag'] + self.uv_anisotropy = np.array(anisotropy["anisotropy_factor"]) + self.uv_anisotropy_flag = anisotropy["flag"] def get_dict(self): # Dictionary with the ionization rate profiles (total, photoionization and charge exchange) that will be saved in json file output = {} - output['header'] = self.header - - output['date'] = self.carr_ion_rate['date'].iso - output['CR'] = int(self.carr_ion_rate['CR']) - - output['ion_rate_profile'] = {} - output['ion_rate_profile']['lat_grid'] = self.carr_ion_rate['ion_grid'] - output['ion_rate_profile']['sum_rate'] = self.carr_ion_rate['ion_rate'].tolist() - output['ion_rate_profile']['ph_rate'] = self.carr_ion_rate['ph_rate'].tolist() - output['ion_rate_profile']['cx_rate'] = self.carr_ion_rate['cx_rate'].tolist() - output['ion_rate_profile']['sum_uncert'] = self.carr_ion_rate['ion_rate_uncert'].tolist() - output['ion_rate_profile']['ph_uncert'] = self.carr_ion_rate['ph_rate_uncert'].tolist() - output['ion_rate_profile']['cx_uncert'] = self.carr_ion_rate['cx_rate_uncert'].tolist() - - output['uv_anisotropy_factor'] = self.uv_anisotropy.tolist() - output['uv_anisotropy_flag'] = self.uv_anisotropy_flag + output["header"] = self.header + + output["date"] = self.carr_ion_rate["date"].iso + output["CR"] = int(self.carr_ion_rate["CR"]) + + output["ion_rate_profile"] = {} + output["ion_rate_profile"]["lat_grid"] = self.carr_ion_rate["ion_grid"] + output["ion_rate_profile"]["sum_rate"] = self.carr_ion_rate["ion_rate"].tolist() + output["ion_rate_profile"]["ph_rate"] = self.carr_ion_rate["ph_rate"].tolist() + output["ion_rate_profile"]["cx_rate"] = self.carr_ion_rate["cx_rate"].tolist() + output["ion_rate_profile"]["sum_uncert"] = self.carr_ion_rate[ + "ion_rate_uncert" + ].tolist() + output["ion_rate_profile"]["ph_uncert"] = self.carr_ion_rate[ + "ph_rate_uncert" + ].tolist() + output["ion_rate_profile"]["cx_uncert"] = self.carr_ion_rate[ + "cx_rate_uncert" + ].tolist() + + output["uv_anisotropy_factor"] = self.uv_anisotropy.tolist() + output["uv_anisotropy_flag"] = self.uv_anisotropy_flag return output - def save_to_file(self,fn): - ''' + def save_to_file(self, fn): + """ Carrington ionization rate file is an official GLOWS L3b data product Temporarly it is in json format, but in production code it should be in CDF @@ -312,33 +361,38 @@ def save_to_file(self,fn): fn: str name of the output file - ''' + """ # Dictionary with the ionization rate profiles (total, photoionization and charge exchange) that will be saved in json file - output={} - output['header']=self.header - - output['date']=self.carr_ion_rate['date'].iso - output['CR']=int(self.carr_ion_rate['CR']) - - output['ion_rate_profile']={} - output['ion_rate_profile']['lat_grid']=self.carr_ion_rate['ion_grid'] - output['ion_rate_profile']['sum_rate']=self.carr_ion_rate['ion_rate'].tolist() - output['ion_rate_profile']['ph_rate']=self.carr_ion_rate['ph_rate'].tolist() - output['ion_rate_profile']['cx_rate']=self.carr_ion_rate['cx_rate'].tolist() - output['ion_rate_profile']['sum_uncert']=self.carr_ion_rate['ion_rate_uncert'].tolist() - output['ion_rate_profile']['ph_uncert']=self.carr_ion_rate['ph_rate_uncert'].tolist() - output['ion_rate_profile']['cx_uncert']=self.carr_ion_rate['cx_rate_uncert'].tolist() - - output['uv_anisotropy_factor']=self.uv_anisotropy.tolist() - output['uv_anisotropy_flag'] = self.uv_anisotropy_flag + output = {} + output["header"] = self.header + + output["date"] = self.carr_ion_rate["date"].iso + output["CR"] = int(self.carr_ion_rate["CR"]) + + output["ion_rate_profile"] = {} + output["ion_rate_profile"]["lat_grid"] = self.carr_ion_rate["ion_grid"] + output["ion_rate_profile"]["sum_rate"] = self.carr_ion_rate["ion_rate"].tolist() + output["ion_rate_profile"]["ph_rate"] = self.carr_ion_rate["ph_rate"].tolist() + output["ion_rate_profile"]["cx_rate"] = self.carr_ion_rate["cx_rate"].tolist() + output["ion_rate_profile"]["sum_uncert"] = self.carr_ion_rate[ + "ion_rate_uncert" + ].tolist() + output["ion_rate_profile"]["ph_uncert"] = self.carr_ion_rate[ + "ph_rate_uncert" + ].tolist() + output["ion_rate_profile"]["cx_uncert"] = self.carr_ion_rate[ + "cx_rate_uncert" + ].tolist() + + output["uv_anisotropy_factor"] = self.uv_anisotropy.tolist() + output["uv_anisotropy_flag"] = self.uv_anisotropy_flag # Check validity of the data - fun.check_nan(output['ion_rate_profile']) + fun.check_nan(output["ion_rate_profile"]) json_content = json.dumps(output, indent=3) - output_fp = open(fn, 'w') + output_fp = open(fn, "w") print(json_content, file=output_fp) output_fp.close() - diff --git a/imap_l3_processing/glows/l3bc/l3bc_toolkit/l3b_DailyIonRate.py b/imap_l3_processing/glows/l3bc/l3bc_toolkit/l3b_DailyIonRate.py index 0c8de8087..4e420538c 100644 --- a/imap_l3_processing/glows/l3bc/l3bc_toolkit/l3b_DailyIonRate.py +++ b/imap_l3_processing/glows/l3bc/l3bc_toolkit/l3b_DailyIonRate.py @@ -1,7 +1,7 @@ -''' +""" Author: Izabela Kowalska-Leszczynska (ikowalska@cbk.waw.pl) Daily ionization rate class -''' +""" import json import logging @@ -35,8 +35,8 @@ class WawHelioIonMPError(Exception): @dataclass -class DailyIonizationRate(): - ''' +class DailyIonizationRate: + """ Class for a GLOWS daily ionization rate Contains original light curve, parameters used to convert light curve into ionization profile and ionization rate profile @@ -151,7 +151,7 @@ class DailyIonizationRate(): _total_ion_params_from_lcrv_params() calculates ionization rate Legendre decomposition coeffiecients from the the lightcurve decomposed coeffiecients - ''' + """ def __init__(self, anc_input_from_instr_team): """ @@ -159,62 +159,64 @@ def __init__(self, anc_input_from_instr_team): """ self.header = { - 'software_version': VERSION, - 'ancillary_data_files': anc_input_from_instr_team['WawHelioIonMP_parameters'], - 'l3a_input_filename': None + "software_version": VERSION, + "ancillary_data_files": anc_input_from_instr_team[ + "WawHelioIonMP_parameters" + ], + "l3a_input_filename": None, } # read pipeline settings to be used for L3b - self.settings = fun.read_json(anc_input_from_instr_team['pipeline_settings']) + self.settings = fun.read_json(anc_input_from_instr_team["pipeline_settings"]) # All parameters will be read from L3a file or computed later on self.spacecraft_location = None self.lcrv = {} - self.lcrv['date'] = None - self.lcrv['ecl_lon'] = None - self.lcrv['sc_coord'] = None - self.lcrv['spin_angle'] = None - self.lcrv['photon_flux'] = None - self.lcrv['flux_uncertainties'] = None - self.lcrv['bkg_galaxy'] = None - self.lcrv['bkg_time'] = None - self.lcrv['glow_flux'] = None - self.lcrv['glow_flux_swan'] = None - self.lcrv['decomposition_coeff'] = None + self.lcrv["date"] = None + self.lcrv["ecl_lon"] = None + self.lcrv["sc_coord"] = None + self.lcrv["spin_angle"] = None + self.lcrv["photon_flux"] = None + self.lcrv["flux_uncertainties"] = None + self.lcrv["bkg_galaxy"] = None + self.lcrv["bkg_time"] = None + self.lcrv["glow_flux"] = None + self.lcrv["glow_flux_swan"] = None + self.lcrv["decomposition_coeff"] = None self.model = {} - self.model['params_hion'] = {} - self.model['params_hion']['hion_order'] = None - self.model['params_hion']['hion_filtration'] = None - self.model['params_hion']['hion_const'] = None - self.model['params_hion']['hion_base'] = None - self.model['vg_points'] = {} - self.model['vg_points']['pos_name'] = None - self.model['vg_points']['ecl_lon'] = None - self.model['vg_points']['ecl_lon_ini'] = None - self.model['vg_points']['ecl_lon_fin'] = None - self.model['vg_points']['parameters'] = {} - self.model['vg_points']['parameters']['lcrv_order'] = None - self.model['vg_points']['parameters']['lcrv_filtration'] = None - self.model['vg_points']['parameters']['lcrv_Tmatrix'] = None - self.model['vg_points']['parameters']['lcrv_const'] = None - self.model['vg_points']['parameters']['lcrv_matrix'] = None + self.model["params_hion"] = {} + self.model["params_hion"]["hion_order"] = None + self.model["params_hion"]["hion_filtration"] = None + self.model["params_hion"]["hion_const"] = None + self.model["params_hion"]["hion_base"] = None + self.model["vg_points"] = {} + self.model["vg_points"]["pos_name"] = None + self.model["vg_points"]["ecl_lon"] = None + self.model["vg_points"]["ecl_lon_ini"] = None + self.model["vg_points"]["ecl_lon_fin"] = None + self.model["vg_points"]["parameters"] = {} + self.model["vg_points"]["parameters"]["lcrv_order"] = None + self.model["vg_points"]["parameters"]["lcrv_filtration"] = None + self.model["vg_points"]["parameters"]["lcrv_Tmatrix"] = None + self.model["vg_points"]["parameters"]["lcrv_const"] = None + self.model["vg_points"]["parameters"]["lcrv_matrix"] = None self.ion = {} - self.ion['ion_grid'] = self.settings['ion_rate_grid'] - self.ion['ion_rate'] = None - self.ion['decomposition_coeff'] = None + self.ion["ion_grid"] = self.settings["ion_rate_grid"] + self.ion["ion_rate"] = None + self.ion["decomposition_coeff"] = None def calculate_daily_ionization_rate(self): - ''' + """ Main method that calculates daily ionization rate profile step-by-step - ''' + """ # subtract background self._subtract_background() # rescale to SWAN - self._rescale_to_SWAN(self.settings['swan_scaling_factor']) + self._rescale_to_SWAN(self.settings["swan_scaling_factor"]) # decompose lightcurve into coeffiecents vector self._decompose_light_curve() # calculate function that describes ionization rate profile @@ -223,109 +225,141 @@ def calculate_daily_ionization_rate(self): self._calculate_ion_profile(ion_profile_fun) def _calculate_ion_profile(self, ion_profile_fun): - ''' + """ Generating tabular profile on ionization grid defined in pipeline settings Parameters ----------- ion_profile_fun - ''' + """ bins = np.arange(-90, 100, 10) # latitude in deg # sin, not cos at it is in Legendre's because in astronomy latitude is meaured # from the equator, not from z axis as it is in mathematics ion = ion_profile_fun(np.sin(np.radians(bins))) / 1e6 - self.ion['ion_grid'] = bins - self.ion['ion_rate'] = ion + self.ion["ion_grid"] = bins + self.ion["ion_rate"] = ion def _decompose_light_curve(self): - ''' + """ Decomposition GLOWS light curve rescaled to SWAN units into sin, cos series As a result, the list of coefficients is calculated - ''' - order = self.model['vg_points']['parameters']['lcrv_order'] - uncert = self.lcrv['flux_uncertainties'] - lightcurve = self.lcrv['glow_flux_swan'] - method = self.settings['lcrv_approximation_method'] - alpha = self.settings['lcrv_approximation_alpha'] + """ + order = self.model["vg_points"]["parameters"]["lcrv_order"] + uncert = self.lcrv["flux_uncertainties"] + lightcurve = self.lcrv["glow_flux_swan"] + method = self.settings["lcrv_approximation_method"] + alpha = self.settings["lcrv_approximation_alpha"] # define basis functions base_funcs = fun.derive_sin_cos(order) # compute matrix for the least-squares method - lsq_mtrx = fun.make_lsq_matrix(np.radians(self.lcrv['spin_angle']), base_funcs) + lsq_mtrx = fun.make_lsq_matrix(np.radians(self.lcrv["spin_angle"]), base_funcs) # fit the linear regression model - if method == 'LinearRegression': - lin_model = LinearRegression(fit_intercept=False).fit(lsq_mtrx, lightcurve, - sample_weight=1.0 / (uncert * uncert)) - elif method == 'Lasso': - lin_model = Lasso(alpha=alpha, fit_intercept=False).fit(lsq_mtrx, lightcurve, - sample_weight=1.0 / (uncert * uncert)) - elif method == 'Ridge': - lin_model = Ridge(alpha=alpha, fit_intercept=False).fit(lsq_mtrx, lightcurve, - sample_weight=1.0 / (uncert * uncert)) + if method == "LinearRegression": + lin_model = LinearRegression(fit_intercept=False).fit( + lsq_mtrx, lightcurve, sample_weight=1.0 / (uncert * uncert) + ) + elif method == "Lasso": + lin_model = Lasso(alpha=alpha, fit_intercept=False).fit( + lsq_mtrx, lightcurve, sample_weight=1.0 / (uncert * uncert) + ) + elif method == "Ridge": + lin_model = Ridge(alpha=alpha, fit_intercept=False).fit( + lsq_mtrx, lightcurve, sample_weight=1.0 / (uncert * uncert) + ) else: - raise Exception('Unknown regression method. Use LinearRegression, Lasso or Ridge') + raise Exception( + "Unknown regression method. Use LinearRegression, Lasso or Ridge" + ) - self.lcrv['decomposition_coeff'] = lin_model.coef_ + self.lcrv["decomposition_coeff"] = lin_model.coef_ def _find_vg_ecl(self, model): - ''' + """ find the index of the vg_point that is closest to the lightcurve, but whitin defined boundaries. Parameters ----------- model : dict Structure read from the WawHelioIon-MP json file - ''' + """ - model_ecl_bins = np.array([[model['vg_points'][i]['ecl_longIni'], model['vg_points'][i]['ecl_longFin']] for i in - range(len(model['vg_points']))]) - k = self.settings['no_vg_point_flag'] # if there is no bin in vg_points main loop should be interupted + model_ecl_bins = np.array( + [ + [ + model["vg_points"][i]["ecl_longIni"], + model["vg_points"][i]["ecl_longFin"], + ] + for i in range(len(model["vg_points"])) + ] + ) + k = self.settings[ + "no_vg_point_flag" + ] # if there is no bin in vg_points main loop should be interupted for i, ecl_bin in enumerate(model_ecl_bins): ini, fin = ecl_bin # Case when we are not crossing 0-360 if ini <= fin: - if ini <= self.lcrv['ecl_lon'] <= fin: k = i + if ini <= self.lcrv["ecl_lon"] <= fin: + k = i # Case when we are crossing 0-360 else: - if np.logical_or(self.lcrv['ecl_lon'] >= ini, self.lcrv['ecl_lon'] <= fin): k = i + if np.logical_or( + self.lcrv["ecl_lon"] >= ini, self.lcrv["ecl_lon"] <= fin + ): + k = i return k def read_l3a(self, data_l3a: dict): - ''' + """ Read light curve from the L3a json file Parameters ----------- filename : str Path to the L3a data product file - ''' + """ # Open and load json file with L3a data - self.spacecraft_location = list(data_l3a['spacecraft_location_average'].values()) - self.lcrv['ecl_lon'] = np.degrees(fun.get_spher_coord_rad(np.array(self.spacecraft_location))[1]) - self.lcrv['date'] = Time(data_l3a['start_time']) + 0.5 * ( - Time(data_l3a['end_time']) - Time(data_l3a['start_time'])) - self.lcrv['sc_coords'] = SkyCoord(lon=data_l3a['spin_axis_orientation_average']['lon'], - lat=data_l3a['spin_axis_orientation_average']['lat'], - unit=u.deg, - frame='heliocentrictrueecliptic') # scaning circle centre coordinates - self.lcrv['spin_angle'] = np.array( - data_l3a['daily_lightcurve']['spin_angle']) # spin angle grid [masked bins are missing] - self.lcrv['photon_flux'] = np.array(data_l3a['daily_lightcurve']['photon_flux']) # signal in physical units - self.lcrv['flux_uncertainties'] = np.array( - data_l3a['daily_lightcurve']['flux_uncertainties']) # measurements uncertaintie - self.lcrv['bkg_galaxy'] = np.array( - data_l3a['daily_lightcurve']['extra_heliospheric_bckgrd']) # constant extra-galactic background - self.lcrv['bkg_time'] = np.array( - data_l3a['daily_lightcurve']['time_dependent_bckgrd']) # time-dependend background - self.lcrv['glow_flux'] = self.lcrv['photon_flux'] - self.lcrv['bkg_galaxy'] - self.lcrv[ - 'bkg_time'] # glow flux in physical units + self.spacecraft_location = list( + data_l3a["spacecraft_location_average"].values() + ) + self.lcrv["ecl_lon"] = np.degrees( + fun.get_spher_coord_rad(np.array(self.spacecraft_location))[1] + ) + self.lcrv["date"] = Time(data_l3a["start_time"]) + 0.5 * ( + Time(data_l3a["end_time"]) - Time(data_l3a["start_time"]) + ) + self.lcrv["sc_coords"] = SkyCoord( + lon=data_l3a["spin_axis_orientation_average"]["lon"], + lat=data_l3a["spin_axis_orientation_average"]["lat"], + unit=u.deg, + frame="heliocentrictrueecliptic", + ) # scaning circle centre coordinates + self.lcrv["spin_angle"] = np.array( + data_l3a["daily_lightcurve"]["spin_angle"] + ) # spin angle grid [masked bins are missing] + self.lcrv["photon_flux"] = np.array( + data_l3a["daily_lightcurve"]["photon_flux"] + ) # signal in physical units + self.lcrv["flux_uncertainties"] = np.array( + data_l3a["daily_lightcurve"]["flux_uncertainties"] + ) # measurements uncertaintie + self.lcrv["bkg_galaxy"] = np.array( + data_l3a["daily_lightcurve"]["extra_heliospheric_bckgrd"] + ) # constant extra-galactic background + self.lcrv["bkg_time"] = np.array( + data_l3a["daily_lightcurve"]["time_dependent_bckgrd"] + ) # time-dependend background + self.lcrv["glow_flux"] = ( + self.lcrv["photon_flux"] - self.lcrv["bkg_galaxy"] - self.lcrv["bkg_time"] + ) # glow flux in physical units def read_model_params(self, anc_input_from_instr_team): - ''' + """ Read WawHelioIon-MP parameters Parameters: @@ -337,34 +371,34 @@ def read_model_params(self, anc_input_from_instr_team): Output: returns flag that is non 0 if light curve can't be converted to the ionization rate due to the gap in WawHelioIon-MP - ''' + """ # read file - model = fun.read_json(anc_input_from_instr_team['WawHelioIonMP_parameters']) + model = fun.read_json(anc_input_from_instr_team["WawHelioIonMP_parameters"]) # find vantage point in the WawHelioIon-MP file that isvalid to analyse current light curve vg_idx = self._find_vg_ecl(model) - if vg_idx == self.settings['no_vg_point_flag']: + if vg_idx == self.settings["no_vg_point_flag"]: # Light curve can not be converted to the ionization rate profile - WawHelioIon-MP could not be train in that part of the orbit - return self.settings['no_vg_point_flag'] + return self.settings["no_vg_point_flag"] - self.model['params_hion'] = model['params_hion'] - self.model['vg_points'] = model['vg_points'][vg_idx] + self.model["params_hion"] = model["params_hion"] + self.model["vg_points"] = model["vg_points"][vg_idx] return 0 def _rescale_to_SWAN(self, sf): - ''' + """ GLOWS ligth curve without background needs to be rescaled to the SWAN units, because WaWHelioIon-MP was tuned on SWAN data Parameters ----------- sf : vector coefficients of the linear scaling function - ''' - self.lcrv['glow_flux_swan'] = self.lcrv['glow_flux'] * sf[0] + sf[1] + """ + self.lcrv["glow_flux_swan"] = self.lcrv["glow_flux"] * sf[0] + sf[1] def save_to_file(self, fn): - ''' + """ Daily ionization rate files are not official GLOWS data product. They are needed to calculate average profile, so they will be calculated but not saved to a files. This method is for testing purposes @@ -373,64 +407,72 @@ def save_to_file(self, fn): ----------- fn : str Path to the daily L3b file - ''' + """ # Dictionary with the daily ionization rate profile that will be saved in json file output = {} - output['header'] = self.header + output["header"] = self.header - output['data'] = {} - output['data']['date'] = self.lcrv['date'].iso - output['data']['ion_grid'] = self.ion['ion_grid'].tolist() - output['data']['ion_rate'] = self.ion['ion_rate'].tolist() + output["data"] = {} + output["data"]["date"] = self.lcrv["date"].iso + output["data"]["ion_grid"] = self.ion["ion_grid"].tolist() + output["data"]["ion_rate"] = self.ion["ion_rate"].tolist() json_content = json.dumps(output, indent=3) - output_fp = open(fn, 'w') + output_fp = open(fn, "w") print(json_content, file=output_fp) output_fp.close() def _subtract_background(self): - ''' + """ Subtracts galactic and time-dependent background from the measured photon flux - ''' - self.lcrv['glow_flux'] = self.lcrv['photon_flux'] - (self.lcrv['bkg_galaxy'] + self.lcrv['bkg_time']) + """ + self.lcrv["glow_flux"] = self.lcrv["photon_flux"] - ( + self.lcrv["bkg_galaxy"] + self.lcrv["bkg_time"] + ) def _total_ion_from_lcrv_params(self): - ''' + """ Calculates total ionization rate profile for a set of light curve decomposition parameters. - ''' - hion_base = np.array(self.model['params_hion']['hion_base']) + """ + hion_base = np.array(self.model["params_hion"]["hion_base"]) total_ion_params = self._total_ion_params_from_lcrv_params() # Reconstruction of the total ionization rate profile using base functions and total_ion_params # Base functions are polynomials and hion_base are coefficients c0, c1, c2,... # creating base functions - total_ion_base_functions = np.array([np.polynomial.Polynomial(matrix) - for matrix in hion_base]) + total_ion_base_functions = np.array( + [np.polynomial.Polynomial(matrix) for matrix in hion_base] + ) # creating a function that describes total ion profile total_ion_profile_fun = np.dot(total_ion_params, total_ion_base_functions) return total_ion_profile_fun def _total_ion_params_from_lcrv_params(self): - ''' + """ Calculates ionization rate coeffiecents that allows to reconstruct ionization rate profile in Legandre base - ''' + """ # Read model data from json file - lcrv_Trmatrix = np.array(self.model['vg_points']['parameters']['lcrv_Trmatrix']) - model_const = np.array(self.model['vg_points']['parameters']['model_const']) - model_matrix = np.array(self.model['vg_points']['parameters']['model_matrix']) - hion_const = np.array(self.model['params_hion']['hion_const']) - lcrv_filtration = np.array(self.model['vg_points']['parameters']['lcrv_filtration']) + lcrv_Trmatrix = np.array(self.model["vg_points"]["parameters"]["lcrv_Trmatrix"]) + model_const = np.array(self.model["vg_points"]["parameters"]["model_const"]) + model_matrix = np.array(self.model["vg_points"]["parameters"]["model_matrix"]) + hion_const = np.array(self.model["params_hion"]["hion_const"]) + lcrv_filtration = np.array( + self.model["vg_points"]["parameters"]["lcrv_filtration"] + ) # Transformation PCA (we are taking only the first few components of the light curve decomposition) - lcrv_params_PCA = np.dot(self.lcrv['decomposition_coeff'], lcrv_Trmatrix)[:lcrv_filtration] + lcrv_params_PCA = np.dot(self.lcrv["decomposition_coeff"], lcrv_Trmatrix)[ + :lcrv_filtration + ] # Transformation from light curve parameters into total ion parameters - total_ion_params = np.concatenate((model_const + np.dot(model_matrix, lcrv_params_PCA) - , hion_const)) - self.ion['decomposition_coeff'] = total_ion_params + total_ion_params = np.concatenate( + (model_const + np.dot(model_matrix, lcrv_params_PCA), hion_const) + ) + self.ion["decomposition_coeff"] = total_ion_params return total_ion_params diff --git a/imap_l3_processing/glows/l3bc/l3bc_toolkit/l3c_CarringtonSolarWind.py b/imap_l3_processing/glows/l3bc/l3bc_toolkit/l3c_CarringtonSolarWind.py index d4e9290ec..ca469f7f3 100644 --- a/imap_l3_processing/glows/l3bc/l3bc_toolkit/l3c_CarringtonSolarWind.py +++ b/imap_l3_processing/glows/l3bc/l3bc_toolkit/l3c_CarringtonSolarWind.py @@ -1,7 +1,7 @@ -''' +""" Author: Izabela Kowalska-Leszczynska (ikowalska@cbk.waw.pl) Carrington solar wind class -''' +""" import json from dataclasses import dataclass @@ -13,11 +13,13 @@ from .constants import VERSION, PHISICAL_CONSTANTS import logging + logging.basicConfig(level=logging.ERROR) + @dataclass -class CarringtonSolarWind(): - ''' +class CarringtonSolarWind: + """ Class for a GLOWS Carrington averaged solar wind speed and density Attributes @@ -69,118 +71,137 @@ class CarringtonSolarWind(): reads L3b data product file save_to_file(self,fn) saves L3c data product to the file - ''' + """ def __init__(self, anc_input_from_instr_team): """ Class constructor """ - self.header={ - 'software_version': VERSION, - 'filename' : None, - 'ancillary_data_files': anc_input_from_instr_team, - 'external_dependeciens': None, - 'l3b_input_filename': None + self.header = { + "software_version": VERSION, + "filename": None, + "ancillary_data_files": anc_input_from_instr_team, + "external_dependeciens": None, + "l3b_input_filename": None, } # read pipeline settings to be used for L3b - self.settings = fun.read_json(anc_input_from_instr_team['pipeline_settings']) + self.settings = fun.read_json(anc_input_from_instr_team["pipeline_settings"]) # All parameters will be read from L3b file or computed later on - self.cx_profile={} - self.cx_profile['rate']=None - self.cx_profile['uncert']=None - - self.sw_ecliptic={} - self.sw_ecliptic['mean_speed']=None - self.sw_ecliptic['mean_proton_density']=None - self.sw_ecliptic['mean_alpha_abundance']=None - self.sw_ecliptic['invariant']=None - - self.sw_profile={} - self.sw_profile['date']=None - self.sw_profile['CR']=None - self.sw_profile['grid']=self.settings['solar_wind_grid'] - self.sw_profile['plasma_speed']=None - self.sw_profile['proton_density']=None - + self.cx_profile = {} + self.cx_profile["rate"] = None + self.cx_profile["uncert"] = None + + self.sw_ecliptic = {} + self.sw_ecliptic["mean_speed"] = None + self.sw_ecliptic["mean_proton_density"] = None + self.sw_ecliptic["mean_alpha_abundance"] = None + self.sw_ecliptic["invariant"] = None + + self.sw_profile = {} + self.sw_profile["date"] = None + self.sw_profile["CR"] = None + self.sw_profile["grid"] = self.settings["solar_wind_grid"] + self.sw_profile["plasma_speed"] = None + self.sw_profile["proton_density"] = None def calculate_sw_profile(self): - ''' + """ Main procedure to calculate solar wind parameters profiles. Sets of 2 equations is numerically solved. For detailed disscussion see Documentation Section 13.7 - ''' + """ # initialization of the vectors that will be calculated later on # solar wind speed latitudinal profile - plasma_speed_profile=np.empty(len(self.settings['solar_wind_grid'])) + plasma_speed_profile = np.empty(len(self.settings["solar_wind_grid"])) # solar wind density latitudinal profile - proton_dens_profile=np.empty(len(self.settings['solar_wind_grid'])) - - - invariant=self.sw_ecliptic['invariant'] - p_alpha=self.sw_ecliptic['mean_alpha_abundance'] - - v_grid=np.array(self.settings['plasma_speed_numerical_grid'])*u.km/u.s # numerical grid to solve equations [km/s] - cs_grid=fun.cross_section_cx(fun.v2E(v_grid)) # cross section for charge exchange for speed grid [cm^2] - rho_grid=invariant/(const.m_p+p_alpha*PHISICAL_CONSTANTS['m_alpha'])/(0.5*v_grid**2+const.GM_sun/const.R_sun) + proton_dens_profile = np.empty(len(self.settings["solar_wind_grid"])) + + invariant = self.sw_ecliptic["invariant"] + p_alpha = self.sw_ecliptic["mean_alpha_abundance"] + + v_grid = ( + np.array(self.settings["plasma_speed_numerical_grid"]) * u.km / u.s + ) # numerical grid to solve equations [km/s] + cs_grid = fun.cross_section_cx( + fun.v2E(v_grid) + ) # cross section for charge exchange for speed grid [cm^2] + rho_grid = ( + invariant + / (const.m_p + p_alpha * PHISICAL_CONSTANTS["m_alpha"]) + / (0.5 * v_grid**2 + const.GM_sun / const.R_sun) + ) # calculation of the charge exchange rate for each speed in speed grid - cx_grid=(rho_grid*cs_grid).to('1/s') + cx_grid = (rho_grid * cs_grid).to("1/s") for i in np.arange(len(plasma_speed_profile)): # charge exchane rate obtained from lcrv - cx_obs=self.cx_profile['rate'][i] - if cx_obs bool: @@ -57,11 +70,15 @@ class ExternalDependencies: @classmethod def fetch_dependencies(cls): - f107_index_file_path = download_external_dependency(F107_FLUX_TABLE_URL, - TEMP_CDF_FOLDER_PATH / 'f107_fluxtable.txt') - lyman_alpha_path = download_external_dependency(LYMAN_ALPHA_COMPOSITE_INDEX_URL, - TEMP_CDF_FOLDER_PATH / 'lyman_alpha.txt') - omni2_data_path = download_external_dependency(OMNI2_URL, TEMP_CDF_FOLDER_PATH / 'omni2.txt') + f107_index_file_path = download_external_dependency( + F107_FLUX_TABLE_URL, TEMP_CDF_FOLDER_PATH / "f107_fluxtable.txt" + ) + lyman_alpha_path = download_external_dependency( + LYMAN_ALPHA_COMPOSITE_INDEX_URL, TEMP_CDF_FOLDER_PATH / "lyman_alpha.txt" + ) + omni2_data_path = download_external_dependency( + OMNI2_URL, TEMP_CDF_FOLDER_PATH / "omni2.txt" + ) if f107_index_file_path is not None: ExternalDependencies._comment_headers(f107_index_file_path) @@ -98,26 +115,32 @@ def pipeline_dependency_file_names(self) -> set[str]: self.uv_anisotropy_file_name, self.waw_helio_ion_mp_file_name, self.bad_days_list_file_name, - self.pipeline_settings_file_name + self.pipeline_settings_file_name, } return self.l3a_file_names | ancillary_files def get_buffer_time(self) -> timedelta: if self._buffer_time is None: - pipeline_settings_local_path = imap_data_access.download(self.pipeline_settings_file_name) + pipeline_settings_local_path = imap_data_access.download( + self.pipeline_settings_file_name + ) pipeline_settings = read_pipeline_settings(pipeline_settings_local_path) - self._buffer_time = timedelta(days=pipeline_settings["initializer_time_delta_days"]) + self._buffer_time = timedelta( + days=pipeline_settings["initializer_time_delta_days"] + ) return self._buffer_time - def has_valid_external_dependencies(self, external_deps: ExternalDependencies) -> bool: + def has_valid_external_dependencies( + self, external_deps: ExternalDependencies + ) -> bool: return validate_dependencies( self.cr_start_date, self.cr_end_date, self.get_buffer_time(), external_deps.omni2_data_path, external_deps.f107_index_file_path, - external_deps.lyman_alpha_path + external_deps.lyman_alpha_path, ) def buffer_time_has_elapsed_since_cr(self): @@ -150,26 +173,29 @@ class GlowsL3BIonizationRate(DataProduct): glows_flags: np.ndarray[int] def to_data_product_variables(self) -> list[DataProductVariable]: - return [DataProductVariable("epoch", self.epoch), - DataProductVariable("epoch_delta", self.epoch_delta), - DataProductVariable("mean_time", self.mean_time), - DataProductVariable("cr", self.cr), - DataProductVariable("uv_anisotropy_factor", self.uv_anisotropy_factor), - DataProductVariable("lat_grid", self.lat_grid), - DataProductVariable("sum_rate", self.sum_rate), - DataProductVariable("ph_rate", self.ph_rate), - DataProductVariable("cx_rate", self.cx_rate), - DataProductVariable("sum_uncert", self.sum_uncert), - DataProductVariable("ph_uncert", self.ph_uncert), - DataProductVariable("cx_uncert", self.cx_uncert), - DataProductVariable("lat_grid_label", self.lat_grid_label), - DataProductVariable("uv_anisotropy_flag", self.uv_anisotropy_flag), - DataProductVariable("used_l3a", self.used_l3a), - DataProductVariable("glows_flags", self.glows_flags), - ] + return [ + DataProductVariable("epoch", self.epoch), + DataProductVariable("epoch_delta", self.epoch_delta), + DataProductVariable("mean_time", self.mean_time), + DataProductVariable("cr", self.cr), + DataProductVariable("uv_anisotropy_factor", self.uv_anisotropy_factor), + DataProductVariable("lat_grid", self.lat_grid), + DataProductVariable("sum_rate", self.sum_rate), + DataProductVariable("ph_rate", self.ph_rate), + DataProductVariable("cx_rate", self.cx_rate), + DataProductVariable("sum_uncert", self.sum_uncert), + DataProductVariable("ph_uncert", self.ph_uncert), + DataProductVariable("cx_uncert", self.cx_uncert), + DataProductVariable("lat_grid_label", self.lat_grid_label), + DataProductVariable("uv_anisotropy_flag", self.uv_anisotropy_flag), + DataProductVariable("used_l3a", self.used_l3a), + DataProductVariable("glows_flags", self.glows_flags), + ] @classmethod - def from_instrument_team_dictionary(cls, model: dict, input_metadata: InputMetadata) -> GlowsL3BIonizationRate: + def from_instrument_team_dictionary( + cls, model: dict, input_metadata: InputMetadata + ) -> GlowsL3BIonizationRate: latitude_grid = model["ion_rate_profile"]["lat_grid"] mean_time = datetime.fromisoformat(model["date"]) start_of_cr, end_of_cr = get_date_range_of_cr(model["CR"]) @@ -189,8 +215,10 @@ def from_instrument_team_dictionary(cls, model: dict, input_metadata: InputMetad l3a_file_names += [""] * (MAX_USED_L3A_FILES - len(l3a_file_names)) parent_file_names = [] - parent_file_names += collect_file_names(model['header']['ancillary_data_files']) - parent_file_names += collect_file_names(model['header']['external_dependeciens']) + parent_file_names += collect_file_names(model["header"]["ancillary_data_files"]) + parent_file_names += collect_file_names( + model["header"]["external_dependeciens"] + ) return cls( input_metadata=input_metadata, parent_file_names=parent_file_names, @@ -207,9 +235,9 @@ def from_instrument_team_dictionary(cls, model: dict, input_metadata: InputMetad ph_uncert=np.array([model["ion_rate_profile"]["ph_uncert"]]), cx_uncert=np.array([model["ion_rate_profile"]["cx_uncert"]]), lat_grid_label=[f"{x}°" for x in latitude_grid], - uv_anisotropy_flag=np.array([model['uv_anisotropy_flag']]), + uv_anisotropy_flag=np.array([model["uv_anisotropy_flag"]]), used_l3a=np.array([l3a_file_names]), - glows_flags=np.array([model['glows_flags']], dtype=np.uint16), + glows_flags=np.array([model["glows_flags"]], dtype=np.uint16), ) @@ -237,51 +265,96 @@ class GlowsL3CSolarWind(DataProduct): def to_data_product_variables(self) -> list[DataProductVariable]: return [ - DataProductVariable("epoch", self.epoch, cdf_data_type=pycdf.const.CDF_TIME_TT2000), - DataProductVariable("epoch_delta", self.epoch_delta, cdf_data_type=pycdf.const.CDF_INT8), - DataProductVariable("mean_time", self.mean_time, cdf_data_type=pycdf.const.CDF_TIME_TT2000), + DataProductVariable( + "epoch", self.epoch, cdf_data_type=pycdf.const.CDF_TIME_TT2000 + ), + DataProductVariable( + "epoch_delta", self.epoch_delta, cdf_data_type=pycdf.const.CDF_INT8 + ), + DataProductVariable( + "mean_time", self.mean_time, cdf_data_type=pycdf.const.CDF_TIME_TT2000 + ), DataProductVariable("cr", self.cr, cdf_data_type=pycdf.const.CDF_INT2), - DataProductVariable("lat_grid", self.lat_grid, cdf_data_type=pycdf.const.CDF_FLOAT, record_varying=False), - DataProductVariable("lat_grid_label", self.lat_grid_label, cdf_data_type=pycdf.const.CDF_CHAR, - record_varying=False), - DataProductVariable("plasma_speed_ecliptic", self.plasma_speed_ecliptic, - cdf_data_type=pycdf.const.CDF_FLOAT), - DataProductVariable("proton_density_ecliptic", self.proton_density_ecliptic, - cdf_data_type=pycdf.const.CDF_FLOAT), - DataProductVariable("alpha_abundance_ecliptic", self.alpha_abundance_ecliptic, - cdf_data_type=pycdf.const.CDF_FLOAT), - DataProductVariable("plasma_speed_profile", self.plasma_speed_profile, cdf_data_type=pycdf.const.CDF_FLOAT), - DataProductVariable("proton_density_profile", self.proton_density_profile, - cdf_data_type=pycdf.const.CDF_FLOAT), + DataProductVariable( + "lat_grid", + self.lat_grid, + cdf_data_type=pycdf.const.CDF_FLOAT, + record_varying=False, + ), + DataProductVariable( + "lat_grid_label", + self.lat_grid_label, + cdf_data_type=pycdf.const.CDF_CHAR, + record_varying=False, + ), + DataProductVariable( + "plasma_speed_ecliptic", + self.plasma_speed_ecliptic, + cdf_data_type=pycdf.const.CDF_FLOAT, + ), + DataProductVariable( + "proton_density_ecliptic", + self.proton_density_ecliptic, + cdf_data_type=pycdf.const.CDF_FLOAT, + ), + DataProductVariable( + "alpha_abundance_ecliptic", + self.alpha_abundance_ecliptic, + cdf_data_type=pycdf.const.CDF_FLOAT, + ), + DataProductVariable( + "plasma_speed_profile", + self.plasma_speed_profile, + cdf_data_type=pycdf.const.CDF_FLOAT, + ), + DataProductVariable( + "proton_density_profile", + self.proton_density_profile, + cdf_data_type=pycdf.const.CDF_FLOAT, + ), DataProductVariable("glows_flags", self.glows_flags), ] @classmethod - def from_instrument_team_dictionary(cls, model: dict, input_metadata: InputMetadata) -> GlowsL3CSolarWind: + def from_instrument_team_dictionary( + cls, model: dict, input_metadata: InputMetadata + ) -> GlowsL3CSolarWind: latitude_grid = model["solar_wind_profile"]["lat_grid"] mean_time = datetime.fromisoformat(model["date"]) start_of_cr, end_of_cr = get_date_range_of_cr(model["CR"]) - epoch = start_of_cr + (end_of_cr - start_of_cr)/2 - epoch_delta = (epoch - start_of_cr).total_seconds()*ONE_SECOND_IN_NANOSECONDS + epoch = start_of_cr + (end_of_cr - start_of_cr) / 2 + epoch_delta = (epoch - start_of_cr).total_seconds() * ONE_SECOND_IN_NANOSECONDS parent_file_names = [] - parent_file_names += collect_file_names(model['header']['ancillary_data_files']) - parent_file_names += collect_file_names(model['header']['external_dependeciens']) + parent_file_names += collect_file_names(model["header"]["ancillary_data_files"]) + parent_file_names += collect_file_names( + model["header"]["external_dependeciens"] + ) return cls( input_metadata=input_metadata, epoch=np.array([epoch]), epoch_delta=np.array([epoch_delta]), mean_time=np.array([mean_time]), - cr=np.array([model['CR']]), + cr=np.array([model["CR"]]), lat_grid=np.array(latitude_grid), lat_grid_label=[f"{x}°" for x in latitude_grid], - plasma_speed_ecliptic=np.array([model["solar_wind_ecliptic"]['plasma_speed']]), - proton_density_ecliptic=np.array([model["solar_wind_ecliptic"]['proton_density']]), - alpha_abundance_ecliptic=np.array([model["solar_wind_ecliptic"]['alpha_abundance']]), - plasma_speed_profile=np.array([model["solar_wind_profile"]['plasma_speed']]), - proton_density_profile=np.array([model["solar_wind_profile"]['proton_density']]), - glows_flags=np.array([model['glows_flags']], dtype=np.uint16), + plasma_speed_ecliptic=np.array( + [model["solar_wind_ecliptic"]["plasma_speed"]] + ), + proton_density_ecliptic=np.array( + [model["solar_wind_ecliptic"]["proton_density"]] + ), + alpha_abundance_ecliptic=np.array( + [model["solar_wind_ecliptic"]["alpha_abundance"]] + ), + plasma_speed_profile=np.array( + [model["solar_wind_profile"]["plasma_speed"]] + ), + proton_density_profile=np.array( + [model["solar_wind_profile"]["proton_density"]] + ), + glows_flags=np.array([model["glows_flags"]], dtype=np.uint16), parent_file_names=parent_file_names, ) diff --git a/imap_l3_processing/glows/l3bc/science/filter_out_bad_days.py b/imap_l3_processing/glows/l3bc/science/filter_out_bad_days.py index b2efe6f25..5ae4ecb15 100644 --- a/imap_l3_processing/glows/l3bc/science/filter_out_bad_days.py +++ b/imap_l3_processing/glows/l3bc/science/filter_out_bad_days.py @@ -6,29 +6,39 @@ from imap_l3_processing.glows.l3bc.l3bc_toolkit.funcs import jd_fm_Carrington -def filter_l3a_files(l3a_data: list[dict], bad_day_list_path: Path, cr: int) -> list[dict]: +def filter_l3a_files( + l3a_data: list[dict], bad_day_list_path: Path, cr: int +) -> list[dict]: carrington_start_date = jd_fm_Carrington(float(cr)) - cr_start_time = Time(carrington_start_date, format='jd') - cr_start_time.format = 'iso' + cr_start_time = Time(carrington_start_date, format="jd") + cr_start_time.format = "iso" carrington_end_date_non_inclusive = jd_fm_Carrington(cr + 1) - cr_end_time = Time(carrington_end_date_non_inclusive, format='jd') - cr_end_time.format = 'iso' + cr_end_time = Time(carrington_end_date_non_inclusive, format="jd") + cr_end_time.format = "iso" def mean_time(l3a_dict): - time = Time(l3a_dict['start_time']) + 0.5 * (Time(l3a_dict['end_time']) - Time(l3a_dict['start_time'])) + time = Time(l3a_dict["start_time"]) + 0.5 * ( + Time(l3a_dict["end_time"]) - Time(l3a_dict["start_time"]) + ) return time - l3a_in_cr = [l3a_dict for l3a_dict in l3a_data if cr_start_time <= mean_time(l3a_dict) < cr_end_time] + l3a_in_cr = [ + l3a_dict + for l3a_dict in l3a_data + if cr_start_time <= mean_time(l3a_dict) < cr_end_time + ] - with open(bad_day_list_path, 'r') as bad_day_file: + with open(bad_day_list_path, "r") as bad_day_file: bad_day_list = Time(np.genfromtxt(bad_day_file, dtype=str)[:, 1]) sorted_bad_days = bad_day_list.sort() filtered_data = [] l3a_data_index = 0 bad_day_list_index = 0 - while l3a_data_index < len(l3a_in_cr) and bad_day_list_index < len(sorted_bad_days): - start_time = Time(l3a_in_cr[l3a_data_index]['start_time']) - end_time = Time(l3a_in_cr[l3a_data_index]['end_time']) + while l3a_data_index < len(l3a_in_cr) and bad_day_list_index < len( + sorted_bad_days + ): + start_time = Time(l3a_in_cr[l3a_data_index]["start_time"]) + end_time = Time(l3a_in_cr[l3a_data_index]["end_time"]) if start_time <= sorted_bad_days[bad_day_list_index] < end_time: l3a_data_index += 1 elif sorted_bad_days[bad_day_list_index] > end_time: diff --git a/imap_l3_processing/glows/l3bc/science/generate_l3bc.py b/imap_l3_processing/glows/l3bc/science/generate_l3bc.py index 45c99cd6f..e32f60e5f 100644 --- a/imap_l3_processing/glows/l3bc/science/generate_l3bc.py +++ b/imap_l3_processing/glows/l3bc/science/generate_l3bc.py @@ -1,13 +1,15 @@ -''' +""" Author: Izabela Kowalska-Leszczynska, ikowalska@cbk.waw.pl Modified by Menlo Main procedure to generate GLOWS L3b and L3c data products -''' +""" import numpy as np from astropy.time import Time -from imap_l3_processing.glows.l3bc.cannot_process_carrington_rotation_error import CannotProcessCarringtonRotationError +from imap_l3_processing.glows.l3bc.cannot_process_carrington_rotation_error import ( + CannotProcessCarringtonRotationError, +) from imap_l3_processing.glows.l3bc.glows_l3bc_dependencies import GlowsL3BCDependencies from imap_l3_processing.glows.l3bc.l3bc_toolkit import l3b_CarringtonIonRate as cir from imap_l3_processing.glows.l3bc.l3bc_toolkit import l3b_DailyIonRate as dir @@ -21,7 +23,9 @@ def generate_l3bc(dependencies: GlowsL3BCDependencies): # if all days in CR are bad, then there will be no L3bc data products if len(dependencies.l3a_data) == 0: - raise CannotProcessCarringtonRotationError("All days for Carrington Rotation are in a bad season.") + raise CannotProcessCarringtonRotationError( + "All days for Carrington Rotation are in a bad season." + ) # Daily ionization rate from the daily light curves daily_ion_rate = [] @@ -29,48 +33,65 @@ def generate_l3bc(dependencies: GlowsL3BCDependencies): l3a_used = [] for l3a in dependencies.l3a_data: ion_rate_daily = dir.DailyIonizationRate( - dependencies.ancillary_files) # daily ionization rate object initialization + dependencies.ancillary_files + ) # daily ionization rate object initialization ion_rate_daily.read_l3a(l3a) # reading L3a data (masked lightcurve) flag = ion_rate_daily.read_model_params( - dependencies.ancillary_files) # reading WawHelioIon-MP parameters for lightcurve to ionization rate conversion - if flag == ion_rate_daily.settings['no_vg_point_flag']: continue - l3a_used_date.append(ion_rate_daily.lcrv['date']) # mean observation time - l3a_used.append(l3a['filename']) + dependencies.ancillary_files + ) # reading WawHelioIon-MP parameters for lightcurve to ionization rate conversion + if flag == ion_rate_daily.settings["no_vg_point_flag"]: + continue + l3a_used_date.append(ion_rate_daily.lcrv["date"]) # mean observation time + l3a_used.append(l3a["filename"]) ion_rate_daily.calculate_daily_ionization_rate() # conversion to the daily ionization rate profile - daily_ion_rate.append(ion_rate_daily.ion['ion_rate']) + daily_ion_rate.append(ion_rate_daily.ion["ion_rate"]) if len(daily_ion_rate) == 0: - raise CannotProcessCarringtonRotationError("All days for Carrington Rotation are in a bad season.") + raise CannotProcessCarringtonRotationError( + "All days for Carrington Rotation are in a bad season." + ) # Carrington averaged ionization rate (L3b) and Carrington averaged solar wind parameters objects initialization - ion_rate_Carr = cir.CarringtonIonizationRate(dependencies.ancillary_files, dependencies.external_files) + ion_rate_Carr = cir.CarringtonIonizationRate( + dependencies.ancillary_files, dependencies.external_files + ) solar_wind_Carr = csw.CarringtonSolarWind(dependencies.ancillary_files) # File name of the L3b and L3c passed to the object structure # Save daily ionization rates into CarringtonIonizationRate structure - ion_rate_Carr.daily_ion_rate['date'] = Time(l3a_used_date) - ion_rate_Carr.daily_ion_rate['ion_rate'] = daily_ion_rate + ion_rate_Carr.daily_ion_rate["date"] = Time(l3a_used_date) + ion_rate_Carr.daily_ion_rate["ion_rate"] = daily_ion_rate # ion_rate_Carr.carr_ion_rate['date']=Time(fun.jd_fm_Carrington(CR+0.5),format='jd') # Date fixed, but in principle could be mean from used L3a - ion_rate_Carr.carr_ion_rate['CR'] = CR # Carrington rotation number + ion_rate_Carr.carr_ion_rate["CR"] = CR # Carrington rotation number # Solar wind in the ecliptic plane processing sw_ecliptic = esw.EclipticSolarWind(dependencies.ancillary_files, CR) sw_ecliptic.calculate_invariant(dependencies.external_files) - solar_wind_Carr.sw_ecliptic['invariant'] = sw_ecliptic.invariant - solar_wind_Carr.header['external_dependeciens'] = sw_ecliptic.external_dependeciens + solar_wind_Carr.sw_ecliptic["invariant"] = sw_ecliptic.invariant + solar_wind_Carr.header["external_dependeciens"] = sw_ecliptic.external_dependeciens # Solar wind parameters passed to the CarringtonSolarWind structure for L3c processing - idx_current_CR = np.abs(sw_ecliptic.CR_grid - ion_rate_Carr.carr_ion_rate['CR']).argmin() - solar_wind_Carr.sw_ecliptic['mean_speed'] = sw_ecliptic.mean_speed[idx_current_CR] - solar_wind_Carr.sw_ecliptic['mean_proton_density'] = sw_ecliptic.mean_proton_density[idx_current_CR] - solar_wind_Carr.sw_ecliptic['mean_alpha_abundance'] = sw_ecliptic.mean_alpha_abundance[idx_current_CR] + idx_current_CR = np.abs( + sw_ecliptic.CR_grid - ion_rate_Carr.carr_ion_rate["CR"] + ).argmin() + solar_wind_Carr.sw_ecliptic["mean_speed"] = sw_ecliptic.mean_speed[idx_current_CR] + solar_wind_Carr.sw_ecliptic["mean_proton_density"] = ( + sw_ecliptic.mean_proton_density[idx_current_CR] + ) + solar_wind_Carr.sw_ecliptic["mean_alpha_abundance"] = ( + sw_ecliptic.mean_alpha_abundance[idx_current_CR] + ) # Solar wind parameters passed to the CarringtonIonizationRate structure for L3c processing - ion_rate_Carr.sw_ecliptic['mean_speed'] = sw_ecliptic.mean_speed[idx_current_CR] - ion_rate_Carr.sw_ecliptic['mean_proton_density'] = sw_ecliptic.mean_proton_density[idx_current_CR] + ion_rate_Carr.sw_ecliptic["mean_speed"] = sw_ecliptic.mean_speed[idx_current_CR] + ion_rate_Carr.sw_ecliptic["mean_proton_density"] = sw_ecliptic.mean_proton_density[ + idx_current_CR + ] # Calculate averaged ionization rate profile and save it to the file - ion_rate_Carr.calculate_photoion(dependencies.ancillary_files, dependencies.external_files) + ion_rate_Carr.calculate_photoion( + dependencies.ancillary_files, dependencies.external_files + ) ion_rate_Carr.calculate_averaged_profile() ion_rate_Carr.calculate_charge_exchange() @@ -81,13 +102,13 @@ def generate_l3bc(dependencies: GlowsL3BCDependencies): solar_wind_Carr.read_l3b(l3b_dat) solar_wind_Carr.calculate_sw_profile() l3c_dat = solar_wind_Carr.get_dict() - l3b_dat['header']['l3a_input_files_name'] = l3a_used + l3b_dat["header"]["l3a_input_files_name"] = l3a_used l3b_quality_flags = int(GlowsL3Flags.NONE) l3c_quality_flags = int(GlowsL3Flags.NONE) if bool(sw_ecliptic.used_nominal_alpha_per_cr[idx_current_CR]): l3c_quality_flags |= int(GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO) - l3b_dat['glows_flags'] = l3b_quality_flags - l3c_dat['glows_flags'] = l3c_quality_flags + l3b_dat["glows_flags"] = l3b_quality_flags + l3c_dat["glows_flags"] = l3c_quality_flags return l3b_dat, l3c_dat diff --git a/imap_l3_processing/glows/l3bc/utils.py b/imap_l3_processing/glows/l3bc/utils.py index 492725f63..4cfceea6e 100644 --- a/imap_l3_processing/glows/l3bc/utils.py +++ b/imap_l3_processing/glows/l3bc/utils.py @@ -7,15 +7,21 @@ from imap_processing.spice.repoint import get_repoint_data - -def get_best_ancillary(start_date: datetime, end_date: datetime, ancillary_query_results: list[dict]) -> Optional[str]: +def get_best_ancillary( + start_date: datetime, end_date: datetime, ancillary_query_results: list[dict] +) -> Optional[str]: valid_ancillaries = [] for ancillary_file in ancillary_query_results: ancillary_start_date = datetime.strptime(ancillary_file["start_date"], "%Y%m%d") - ancillary_end_date = datetime.strptime(ancillary_file["end_date"], "%Y%m%d") if ancillary_file[ - "end_date"] else None + ancillary_end_date = ( + datetime.strptime(ancillary_file["end_date"], "%Y%m%d") + if ancillary_file["end_date"] + else None + ) - if ancillary_start_date <= end_date and (ancillary_end_date is None or ancillary_end_date >= start_date): + if ancillary_start_date <= end_date and ( + ancillary_end_date is None or ancillary_end_date >= start_date + ): valid_ancillaries.append(ancillary_file) if len(valid_ancillaries) == 0: @@ -29,26 +35,37 @@ def get_best_ancillary(start_date: datetime, end_date: datetime, ancillary_query first_carrington_start_date = datetime(1853, 11, 9, 19, 53, 45, 600000) carrington_length = timedelta(days=27.2753) + def get_date_range_of_cr(cr_number: int) -> tuple[datetime, datetime]: - start_date = first_carrington_start_date + (cr_number - carrington_first) * carrington_length + start_date = ( + first_carrington_start_date + (cr_number - carrington_first) * carrington_length + ) return start_date, start_date + carrington_length + def get_midpoint_of_cr(cr_number: int) -> datetime: start, _ = get_date_range_of_cr(cr_number) return start + carrington_length / 2 + def get_cr_for_date_time(datetime_to_check: datetime) -> int: - return int(carrington_first + (datetime_to_check - first_carrington_start_date) / carrington_length) + return int( + carrington_first + + (datetime_to_check - first_carrington_start_date) / carrington_length + ) + def get_pointing_date_range(repointing: int) -> (datetime, datetime): repointing_df: pd.DataFrame = get_repoint_data() - matching_rows_start = repointing_df[repointing_df['repoint_id'] == repointing] - matching_rows_end = repointing_df[repointing_df['repoint_id'] == repointing + 1] + matching_rows_start = repointing_df[repointing_df["repoint_id"] == repointing] + matching_rows_end = repointing_df[repointing_df["repoint_id"] == repointing + 1] if len(matching_rows_start) == 0 or len(matching_rows_end) == 0: raise ValueError(f"No pointing found for pointing: {repointing}") repointing_data_start = matching_rows_start.iloc[0] repointing_data_end = matching_rows_end.iloc[0] - start_time = repointing_data_start['repoint_end_utc'] - end_time = repointing_data_end['repoint_start_utc'] + start_time = repointing_data_start["repoint_end_utc"] + end_time = repointing_data_end["repoint_start_utc"] - return np.datetime64(start_time).astype(datetime), np.datetime64(end_time).astype(datetime) + return np.datetime64(start_time).astype(datetime), np.datetime64(end_time).astype( + datetime + ) diff --git a/imap_l3_processing/glows/l3d/glows_l3d_dependencies.py b/imap_l3_processing/glows/l3d/glows_l3d_dependencies.py index 74a5330d3..c2d3d62d8 100644 --- a/imap_l3_processing/glows/l3d/glows_l3d_dependencies.py +++ b/imap_l3_processing/glows/l3d/glows_l3d_dependencies.py @@ -5,9 +5,17 @@ from imap_data_access import ScienceFilePath from imap_data_access.processing_input import ProcessingInputCollection -from imap_l3_processing.glows.descriptors import PROTON_DENSITY_DESCRIPTOR, PLASMA_SPEED_DESCRIPTOR, \ - UV_ANISOTROPY_DESCRIPTOR, PHOTOION_DESCRIPTOR, LYA_DESCRIPTOR, ELECTRON_DENSITY_DESCRIPTOR, \ - PIPELINE_SETTINGS_L3BCDE_DESCRIPTOR, GLOWS_L3B_DESCRIPTOR, GLOWS_L3C_DESCRIPTOR +from imap_l3_processing.glows.descriptors import ( + PROTON_DENSITY_DESCRIPTOR, + PLASMA_SPEED_DESCRIPTOR, + UV_ANISOTROPY_DESCRIPTOR, + PHOTOION_DESCRIPTOR, + LYA_DESCRIPTOR, + ELECTRON_DENSITY_DESCRIPTOR, + PIPELINE_SETTINGS_L3BCDE_DESCRIPTOR, + GLOWS_L3B_DESCRIPTOR, + GLOWS_L3C_DESCRIPTOR, +) from imap_l3_processing.glows.l3bc.models import ExternalDependencies @@ -20,46 +28,75 @@ class GlowsL3DDependencies: end_cr: int @classmethod - def fetch_dependencies(cls, dependencies: ProcessingInputCollection, external_dependencies: ExternalDependencies): - plasma_speed_legendre_path = dependencies.get_file_paths(source='glows', descriptor=PLASMA_SPEED_DESCRIPTOR) - proton_density_legendre_path = dependencies.get_file_paths(source='glows', descriptor=PROTON_DENSITY_DESCRIPTOR) - uv_anisotropy_path = dependencies.get_file_paths(source='glows', descriptor=UV_ANISOTROPY_DESCRIPTOR) - photoion_path = dependencies.get_file_paths(source='glows', descriptor=PHOTOION_DESCRIPTOR) - lya_path = dependencies.get_file_paths(source='glows', descriptor=LYA_DESCRIPTOR) - electron_density_path = dependencies.get_file_paths(source='glows', descriptor=ELECTRON_DENSITY_DESCRIPTOR) - pipeline_settings_l3bc_path = dependencies.get_file_paths(source='glows', descriptor=PIPELINE_SETTINGS_L3BCDE_DESCRIPTOR) + def fetch_dependencies( + cls, + dependencies: ProcessingInputCollection, + external_dependencies: ExternalDependencies, + ): + plasma_speed_legendre_path = dependencies.get_file_paths( + source="glows", descriptor=PLASMA_SPEED_DESCRIPTOR + ) + proton_density_legendre_path = dependencies.get_file_paths( + source="glows", descriptor=PROTON_DENSITY_DESCRIPTOR + ) + uv_anisotropy_path = dependencies.get_file_paths( + source="glows", descriptor=UV_ANISOTROPY_DESCRIPTOR + ) + photoion_path = dependencies.get_file_paths( + source="glows", descriptor=PHOTOION_DESCRIPTOR + ) + lya_path = dependencies.get_file_paths( + source="glows", descriptor=LYA_DESCRIPTOR + ) + electron_density_path = dependencies.get_file_paths( + source="glows", descriptor=ELECTRON_DENSITY_DESCRIPTOR + ) + pipeline_settings_l3bc_path = dependencies.get_file_paths( + source="glows", descriptor=PIPELINE_SETTINGS_L3BCDE_DESCRIPTOR + ) - plasma_speed_legendre = imap_data_access.download(str(plasma_speed_legendre_path[0])) - proton_density_legendre = imap_data_access.download(str(proton_density_legendre_path[0])) + plasma_speed_legendre = imap_data_access.download( + str(plasma_speed_legendre_path[0]) + ) + proton_density_legendre = imap_data_access.download( + str(proton_density_legendre_path[0]) + ) uv_anisotropy = imap_data_access.download(str(uv_anisotropy_path[0])) photoion = imap_data_access.download(str(photoion_path[0])) lya_2010a = imap_data_access.download(str(lya_path[0])) electron_density = imap_data_access.download(str(electron_density_path[0])) - pipeline_settings_l3bc = imap_data_access.download(str(pipeline_settings_l3bc_path[0])) + pipeline_settings_l3bc = imap_data_access.download( + str(pipeline_settings_l3bc_path[0]) + ) - l3b_file_names = dependencies.get_file_paths(source="glows", descriptor=GLOWS_L3B_DESCRIPTOR) - l3c_file_names = dependencies.get_file_paths(source="glows", descriptor=GLOWS_L3C_DESCRIPTOR) + l3b_file_names = dependencies.get_file_paths( + source="glows", descriptor=GLOWS_L3B_DESCRIPTOR + ) + l3c_file_names = dependencies.get_file_paths( + source="glows", descriptor=GLOWS_L3C_DESCRIPTOR + ) l3b_file_paths = [imap_data_access.download(l3b) for l3b in l3b_file_names] l3c_file_paths = [imap_data_access.download(l3c) for l3c in l3c_file_names] ancillary_dict = { - 'pipeline_settings': pipeline_settings_l3bc, - 'WawHelioIon': { - 'speed': plasma_speed_legendre, - 'p-dens': proton_density_legendre, - 'uv-anis': uv_anisotropy, - 'phion': photoion, - 'lya': lya_2010a, - 'e-dens': electron_density - }} - - external_dict = { - 'lya_raw_data': external_dependencies.lyman_alpha_path + "pipeline_settings": pipeline_settings_l3bc, + "WawHelioIon": { + "speed": plasma_speed_legendre, + "p-dens": proton_density_legendre, + "uv-anis": uv_anisotropy, + "phion": photoion, + "lya": lya_2010a, + "e-dens": electron_density, + }, } + external_dict = {"lya_raw_data": external_dependencies.lyman_alpha_path} + last_l3b_cr = max([ScienceFilePath(file).cr for file in l3b_file_paths]) last_l3c_cr = max([ScienceFilePath(file).cr for file in l3c_file_paths]) end_cr = min(last_l3b_cr, last_l3c_cr) + 1 - return cls(external_dict, ancillary_dict, l3b_file_paths, l3c_file_paths, end_cr) + return cls( + external_dict, ancillary_dict, l3b_file_paths, l3c_file_paths, end_cr + ) diff --git a/imap_l3_processing/glows/l3d/glows_l3d_initializer.py b/imap_l3_processing/glows/l3d/glows_l3d_initializer.py index 304f6611e..83dc78c67 100644 --- a/imap_l3_processing/glows/l3d/glows_l3d_initializer.py +++ b/imap_l3_processing/glows/l3d/glows_l3d_initializer.py @@ -3,13 +3,28 @@ from typing import Optional import imap_data_access -from imap_data_access import ProcessingInputCollection, ScienceInput, AncillaryInput, ScienceFilePath +from imap_data_access import ( + ProcessingInputCollection, + ScienceInput, + AncillaryInput, + ScienceFilePath, +) from imap_data_access.file_validation import Version -from imap_l3_processing.glows.descriptors import PLASMA_SPEED_DESCRIPTOR, PROTON_DENSITY_DESCRIPTOR, \ - UV_ANISOTROPY_DESCRIPTOR, PHOTOION_DESCRIPTOR, LYA_DESCRIPTOR, ELECTRON_DENSITY_DESCRIPTOR, \ - PIPELINE_SETTINGS_L3BCDE_DESCRIPTOR, GLOWS_L3D_DESCRIPTOR -from imap_l3_processing.glows.l3bc.models import ExternalDependencies, read_pipeline_settings +from imap_l3_processing.glows.descriptors import ( + PLASMA_SPEED_DESCRIPTOR, + PROTON_DENSITY_DESCRIPTOR, + UV_ANISOTROPY_DESCRIPTOR, + PHOTOION_DESCRIPTOR, + LYA_DESCRIPTOR, + ELECTRON_DENSITY_DESCRIPTOR, + PIPELINE_SETTINGS_L3BCDE_DESCRIPTOR, + GLOWS_L3D_DESCRIPTOR, +) +from imap_l3_processing.glows.l3bc.models import ( + ExternalDependencies, + read_pipeline_settings, +) from imap_l3_processing.glows.l3d.glows_l3d_dependencies import GlowsL3DDependencies from imap_l3_processing.glows.l3d.utils import query_for_most_recent_l3d from imap_l3_processing.glows.l3e.reprocess_info import ReprocessInfo @@ -19,10 +34,14 @@ class GlowsL3DInitializer: - @staticmethod - def should_process_l3d(external_deps: ExternalDependencies, l3bs: list[str], l3cs: list[str], reprocess_info: ReprocessInfo, major_version: int|None) -> Optional[ - tuple[Version, GlowsL3DDependencies, Optional[str]]]: + def should_process_l3d( + external_deps: ExternalDependencies, + l3bs: list[str], + l3cs: list[str], + reprocess_info: ReprocessInfo, + major_version: int | None, + ) -> Optional[tuple[Version, GlowsL3DDependencies, Optional[str]]]: if len(l3bs) == 0 and len(l3cs) == 0: logger.info("Found no L3b and L3c files!") return None @@ -30,52 +49,63 @@ def should_process_l3d(external_deps: ExternalDependencies, l3bs: list[str], l3c most_recent_l3d = query_for_most_recent_l3d(GLOWS_L3D_DESCRIPTOR) [plasma_speed_2026a] = imap_data_access.query( - table='ancillary', - instrument='glows', + table="ancillary", + instrument="glows", descriptor=PLASMA_SPEED_DESCRIPTOR, - version='latest' + version="latest", ) [proton_density_2026a] = imap_data_access.query( - table='ancillary', - instrument='glows', + table="ancillary", + instrument="glows", descriptor=PROTON_DENSITY_DESCRIPTOR, - version='latest' + version="latest", ) [uv_anisotropy_2026a] = imap_data_access.query( - table='ancillary', - instrument='glows', + table="ancillary", + instrument="glows", descriptor=UV_ANISOTROPY_DESCRIPTOR, - version='latest') + version="latest", + ) [photoion_2026a] = imap_data_access.query( - table='ancillary', - instrument='glows', + table="ancillary", + instrument="glows", descriptor=PHOTOION_DESCRIPTOR, - version='latest' + version="latest", ) [lya_2026a] = imap_data_access.query( - table='ancillary', - instrument='glows', + table="ancillary", + instrument="glows", descriptor=LYA_DESCRIPTOR, - version='latest' + version="latest", ) [electron_density_2026a] = imap_data_access.query( - table='ancillary', - instrument='glows', + table="ancillary", + instrument="glows", descriptor=ELECTRON_DENSITY_DESCRIPTOR, - version='latest' + version="latest", ) [pipeline_settings_l3bcde] = imap_data_access.query( - table='ancillary', - instrument='glows', + table="ancillary", + instrument="glows", descriptor=PIPELINE_SETTINGS_L3BCDE_DESCRIPTOR, - version='latest' + version="latest", ) pipeline_settings_filename = Path(pipeline_settings_l3bcde["file_path"]).name - pipeline_settings = read_pipeline_settings(imap_data_access.download(pipeline_settings_filename)) + pipeline_settings = read_pipeline_settings( + imap_data_access.download(pipeline_settings_filename) + ) - l3bs = [l3b for l3b in l3bs if ScienceFilePath(l3b).cr >= pipeline_settings["start_cr"]] - l3cs = [l3c for l3c in l3cs if ScienceFilePath(l3c).cr >= pipeline_settings["start_cr"]] + l3bs = [ + l3b + for l3b in l3bs + if ScienceFilePath(l3b).cr >= pipeline_settings["start_cr"] + ] + l3cs = [ + l3c + for l3c in l3cs + if ScienceFilePath(l3c).cr >= pipeline_settings["start_cr"] + ] if len(l3bs) == 0 and len(l3cs) == 0: logger.info("Found no L3b and L3c files after start CR!") @@ -95,12 +125,12 @@ def should_process_l3d(external_deps: ExternalDependencies, l3bs: list[str], l3c updated_input_files = { *l3bs, *l3cs, - Path(plasma_speed_2026a['file_path']).name, - Path(proton_density_2026a['file_path']).name, - Path(uv_anisotropy_2026a['file_path']).name, - Path(photoion_2026a['file_path']).name, - Path(lya_2026a['file_path']).name, - Path(electron_density_2026a['file_path']).name, + Path(plasma_speed_2026a["file_path"]).name, + Path(proton_density_2026a["file_path"]).name, + Path(uv_anisotropy_2026a["file_path"]).name, + Path(photoion_2026a["file_path"]).name, + Path(lya_2026a["file_path"]).name, + Path(electron_density_2026a["file_path"]).name, } old_l3d = None @@ -118,20 +148,29 @@ def should_process_l3d(external_deps: ExternalDependencies, l3bs: list[str], l3c minor_version_to_generate = most_recent_l3d_version.minor + 1 same_major_version = most_recent_l3d_version.major == major_version - inputs_unchanged = same_major_version and updated_input_files.issubset(l3d_parents) + inputs_unchanged = same_major_version and updated_input_files.issubset( + l3d_parents + ) if inputs_unchanged and not reprocess_info.should_reprocess_l3d(): - return None + return None version_to_generate = Version(major_version, minor_version_to_generate) - return GlowsL3DInitializer.generate_result(version_to_generate, processing_input_collection, external_deps, - old_l3d) + return GlowsL3DInitializer.generate_result( + version_to_generate, processing_input_collection, external_deps, old_l3d + ) @staticmethod - def generate_result(version_to_generate: Version, - processing_input_collection: ProcessingInputCollection, - external_deps: ExternalDependencies, - old_l3d: str | None) -> tuple[Version, GlowsL3DDependencies, str | None]: - return (version_to_generate, - GlowsL3DDependencies.fetch_dependencies(processing_input_collection, external_deps), - old_l3d) + def generate_result( + version_to_generate: Version, + processing_input_collection: ProcessingInputCollection, + external_deps: ExternalDependencies, + old_l3d: str | None, + ) -> tuple[Version, GlowsL3DDependencies, str | None]: + return ( + version_to_generate, + GlowsL3DDependencies.fetch_dependencies( + processing_input_collection, external_deps + ), + old_l3d, + ) diff --git a/imap_l3_processing/glows/l3d/models.py b/imap_l3_processing/glows/l3d/models.py index b22a973ef..114e86183 100644 --- a/imap_l3_processing/glows/l3d/models.py +++ b/imap_l3_processing/glows/l3d/models.py @@ -6,8 +6,8 @@ from imap_l3_processing.constants import CARRINGTON_ROTATION_IN_NANOSECONDS from imap_l3_processing.models import DataProduct, DataProductVariable -EPOCH_CDF_VAR_NAME = 'epoch' -EPOCH_DELTA_CDF_VAR_NAME = 'epoch_delta' +EPOCH_CDF_VAR_NAME = "epoch" +EPOCH_DELTA_CDF_VAR_NAME = "epoch_delta" LATITUDE_CDF_CDF_VAR_NAME = "lat_grid" LATITUDE_LABEL_CDF_VAR_NAME = "lat_grid_label" CR_CDF_VAR_NAME = "cr_grid" @@ -48,7 +48,7 @@ class GlowsL3DSolarParamsHistory(DataProduct): def to_data_product_variables(self) -> list[DataProductVariable]: epoch_delta = np.full_like(self.epoch, CARRINGTON_ROTATION_IN_NANOSECONDS / 2) - latitude_label = [f'{lat:.1f} degrees' for lat in self.latitude] + latitude_label = [f"{lat:.1f} degrees" for lat in self.latitude] return [ DataProductVariable(EPOCH_CDF_VAR_NAME, self.epoch), @@ -58,12 +58,18 @@ def to_data_product_variables(self) -> list[DataProductVariable]: DataProductVariable(CR_CDF_VAR_NAME, self.cr), DataProductVariable(SPEED_CDF_VAR_NAME, self.plasma_speed), DataProductVariable(PROTON_DENSITY_CDF_VAR_NAME, self.proton_density), - DataProductVariable(UV_ANISOTROPY_CDF_VAR_NAME, self.ultraviolet_anisotropy), + DataProductVariable( + UV_ANISOTROPY_CDF_VAR_NAME, self.ultraviolet_anisotropy + ), DataProductVariable(PHION_CDF_VAR_NAME, self.phion), DataProductVariable(LYMAN_ALPHA_CDF_VAR_NAME, self.lyman_alpha), DataProductVariable(ELECTRON_DENSITY_CDF_VAR_NAME, self.electron_density), DataProductVariable(PLASMA_SPEED_FLAG_CDF_VAR_NAME, self.plasma_speed_flag), - DataProductVariable(UV_ANISOTROPY_FLAG_CDF_VAR_NAME, self.uv_anisotropy_flag), - DataProductVariable(PROTON_DENSITY_FLAG_CDF_VAR_NAME, self.proton_density_flag), + DataProductVariable( + UV_ANISOTROPY_FLAG_CDF_VAR_NAME, self.uv_anisotropy_flag + ), + DataProductVariable( + PROTON_DENSITY_FLAG_CDF_VAR_NAME, self.proton_density_flag + ), DataProductVariable(GLOWS_FLAGS_CDF_VAR_NAME, self.glows_flags), ] diff --git a/imap_l3_processing/glows/l3d/science/generate_l3d.py b/imap_l3_processing/glows/l3d/science/generate_l3d.py index 17c53cab9..ce532ef1e 100644 --- a/imap_l3_processing/glows/l3d/science/generate_l3d.py +++ b/imap_l3_processing/glows/l3d/science/generate_l3d.py @@ -1,7 +1,7 @@ -''' +""" Author: Izabela Kowalska-Leszczynska, ikowalska@cbk.waw.pl Main procedure to generate GLOWS L3d data product -''' +""" import glob import json @@ -20,40 +20,61 @@ dependencies = json.loads(sys.argv[2]) -ANC_INPUT_FROM_INSTRUMENT_TEAM = dependencies['ancillary_files'] -EXT_DEPENDENCIES = dependencies['external_files'] -L3D_CDF_FILENAME = dependencies['l3d_cdf_filename'] +ANC_INPUT_FROM_INSTRUMENT_TEAM = dependencies["ancillary_files"] +EXT_DEPENDENCIES = dependencies["external_files"] +L3D_CDF_FILENAME = dependencies["l3d_cdf_filename"] # Create list of L3b/c files and read them -l3b_fn_list = np.array(sorted(glob.glob('data_l3b/imap_glows_l3b*.json'))) -l3c_fn_list = np.array(sorted(glob.glob('data_l3c/imap_glows_l3c*.json'))) +l3b_fn_list = np.array(sorted(glob.glob("data_l3b/imap_glows_l3b*.json"))) +l3c_fn_list = np.array(sorted(glob.glob("data_l3c/imap_glows_l3c*.json"))) # l3d_fn_list = np.array(sorted(glob.glob('data_l3d/imap_glows_l3d*.json'))) data_l3b = [fun.read_json(fn) for fn in l3b_fn_list] data_l3c = [fun.read_json(fn) for fn in l3c_fn_list] -solar_param_hist = sph.SolarParamsHistory(ANC_INPUT_FROM_INSTRUMENT_TEAM, EXT_DEPENDENCIES) -fn_initial = ANC_INPUT_FROM_INSTRUMENT_TEAM['WawHelioIon'] +solar_param_hist = sph.SolarParamsHistory( + ANC_INPUT_FROM_INSTRUMENT_TEAM, EXT_DEPENDENCIES +) +fn_initial = ANC_INPUT_FROM_INSTRUMENT_TEAM["WawHelioIon"] # Start generating from the initial files (not using the last entry from L3d) -solar_param_hist.header['ancillary_data_files'] = [*fn_initial.values(), ANC_INPUT_FROM_INSTRUMENT_TEAM['pipeline_settings']] -solar_param_hist.header['l3d_cdf_filename'] = L3D_CDF_FILENAME +solar_param_hist.header["ancillary_data_files"] = [ + *fn_initial.values(), + ANC_INPUT_FROM_INSTRUMENT_TEAM["pipeline_settings"], +] +solar_param_hist.header["l3d_cdf_filename"] = L3D_CDF_FILENAME solar_param_hist.generate_initial_history(fn_initial) -solar_param_hist.update_solar_params_hist(EXT_DEPENDENCIES, data_l3b, data_l3c, CR_current) +solar_param_hist.update_solar_params_hist( + EXT_DEPENDENCIES, data_l3b, data_l3c, CR_current +) # filename of the data products (main json file and the text version) start_date = solar_param_hist.time_grid[0] -start_date_str = ''.join((start_date.iso[:10]).split('-')) -l3d_fn = 'data_l3d/imap_glows_l3d_solar-params-history_' + start_date_str + '-cr0' + str(CR_current) + '_v00.json' -solar_param_hist.header['filename'] = l3d_fn +start_date_str = "".join((start_date.iso[:10]).split("-")) +l3d_fn = ( + "data_l3d/imap_glows_l3d_solar-params-history_" + + start_date_str + + "-cr0" + + str(CR_current) + + "_v00.json" +) +solar_param_hist.header["filename"] = l3d_fn l3d_txt_fn = {} -for k in solar_param_hist.ini_data['label']: l3d_txt_fn[ - k] = 'data_l3d_txt/imap_glows_l3d_' + k + '_' + start_date_str + '-cr0' + str(CR_current) + '_v00.dat' +for k in solar_param_hist.ini_data["label"]: + l3d_txt_fn[k] = ( + "data_l3d_txt/imap_glows_l3d_" + + k + + "_" + + start_date_str + + "-cr0" + + str(CR_current) + + "_v00.dat" + ) hdr = solar_param_hist.generate_hdr_txt(l3d_txt_fn) solar_param_hist.save_to_file(l3d_fn) solar_param_hist.save_to_txt(l3d_txt_fn, hdr) -print('Processing CR=', CR_current) +print("Processing CR=", CR_current) diff --git a/imap_l3_processing/glows/l3d/science/toolkit/constants.py b/imap_l3_processing/glows/l3d/science/toolkit/constants.py index 827b4c06c..0082d96da 100644 --- a/imap_l3_processing/glows/l3d/science/toolkit/constants.py +++ b/imap_l3_processing/glows/l3d/science/toolkit/constants.py @@ -5,33 +5,34 @@ import astropy.units as u -VERSION = "3.3" # version of ground processing software +VERSION = "3.3" # version of ground processing software # dictionary with names of ancillary input files expected from IMAP/GLOWS Instrument Team ANC_INPUT_FROM_INSTRUMENT_TEAM = { - 'WawHelioIonMP_parameters': 'data_ancillary/imap_glows_WawHelioIonMP_20251113_v007.json', - 'bad_day_list': 'data_ancillary/imap_glows_bad-days-list_20251113_v001.dat', - 'pipeline_settings': 'data_ancillary/imap_glows_pipeline-settings-l3bcde_20251113_v005.json', - 'uv_anisotropy' : 'data_ancillary/imap_glows_uv-anisotropy-1CR_20251113_v002.json', - 'WawHelioIon' : { - 'speed': 'data_ancillary/imap_glows_plasma-speed-2026d_20251113_v003.dat', - 'p-dens': 'data_ancillary/imap_glows_proton-density-2026d_20251113_v003.dat', - 'uv-anis': 'data_ancillary/imap_glows_uv-anisotropy-2026d_20251113_v003.dat', - 'phion': 'data_ancillary/imap_glows_photoion-2026d_20251113_v003.dat', - 'lya': 'data_ancillary/imap_glows_lya-2026d_20251113_v003.dat', - 'e-dens': 'data_ancillary/imap_glows_electron-density-2026d_20251113_v003.dat' - }} + "WawHelioIonMP_parameters": "data_ancillary/imap_glows_WawHelioIonMP_20251113_v007.json", + "bad_day_list": "data_ancillary/imap_glows_bad-days-list_20251113_v001.dat", + "pipeline_settings": "data_ancillary/imap_glows_pipeline-settings-l3bcde_20251113_v005.json", + "uv_anisotropy": "data_ancillary/imap_glows_uv-anisotropy-1CR_20251113_v002.json", + "WawHelioIon": { + "speed": "data_ancillary/imap_glows_plasma-speed-2026d_20251113_v003.dat", + "p-dens": "data_ancillary/imap_glows_proton-density-2026d_20251113_v003.dat", + "uv-anis": "data_ancillary/imap_glows_uv-anisotropy-2026d_20251113_v003.dat", + "phion": "data_ancillary/imap_glows_photoion-2026d_20251113_v003.dat", + "lya": "data_ancillary/imap_glows_lya-2026d_20251113_v003.dat", + "e-dens": "data_ancillary/imap_glows_electron-density-2026d_20251113_v003.dat", + }, +} -EXT_DEPENDENCIES={ - 'f107_raw_data': 'external_dependencies/f107_fluxtable.txt', - 'omni_raw_data': 'external_dependencies/omni2_all_years.dat', - 'lya_raw_data': 'external_dependencies/lyman_alpha_composite.nc' +EXT_DEPENDENCIES = { + "f107_raw_data": "external_dependencies/f107_fluxtable.txt", + "omni_raw_data": "external_dependencies/omni2_all_years.dat", + "lya_raw_data": "external_dependencies/lyman_alpha_composite.nc", } -PHISICAL_CONSTANTS={ - 'm_alpha' : 6.6446573357e-27*u.kg, - 'jd_carrington_first' : 2.398167329*10**6, - 'carrington_time' : 27.2753 +PHISICAL_CONSTANTS = { + "m_alpha": 6.6446573357e-27 * u.kg, + "jd_carrington_first": 2.398167329 * 10**6, + "carrington_time": 27.2753, } -CONST_ALPHA_TO_PROTON=0.04 +CONST_ALPHA_TO_PROTON = 0.04 diff --git a/imap_l3_processing/glows/l3d/science/toolkit/funcs.py b/imap_l3_processing/glows/l3d/science/toolkit/funcs.py index 02ea12ea8..09222822e 100644 --- a/imap_l3_processing/glows/l3d/science/toolkit/funcs.py +++ b/imap_l3_processing/glows/l3d/science/toolkit/funcs.py @@ -1,7 +1,7 @@ -''' +""" Author: Izabela Kowalska-Leszczynska (ikowalska@cbk.waw.pl) Functions used in L3a->L3b processing that are not class methods -''' +""" import numpy as np import astropy.units as u @@ -14,13 +14,17 @@ from .constants import PHISICAL_CONSTANTS, CONST_ALPHA_TO_PROTON import logging + logging.basicConfig(level=logging.ERROR) + ########################## # Exceptions ########################## class ArrayShapeError(Exception): pass + + class OutOfLimitsError(Exception): pass @@ -40,87 +44,109 @@ def base_funcs_def(i, sin_or_cos): """ if i == 0: + def fun(angle): - return 1.0/(2.0*np.sqrt(np.pi))*np.ones_like(angle) + return 1.0 / (2.0 * np.sqrt(np.pi)) * np.ones_like(angle) + return fun - if sin_or_cos == 's': + if sin_or_cos == "s": + def fun_s(angle): - return np.sin(i*angle)/np.sqrt(np.pi) + return np.sin(i * angle) / np.sqrt(np.pi) + return fun_s - if sin_or_cos == 'c': + if sin_or_cos == "c": + def fun_c(angle): - return np.cos(i*angle)/np.sqrt(np.pi) + return np.cos(i * angle) / np.sqrt(np.pi) + return fun_c # TODO: maybe we need some error if sin_or_cos is anything beside 'c' or 's' - return 'None' + return "None" -def calculate_lya_monthly(t,flux): - ''' - Calculates data that are averaged over each Carrington rotation period. Time is mean value of decimal year in each period. - ''' - t_carr,flux_carr=resize_by_carr(t,flux) - t_mean=Time(np.array([np.mean(t.decimalyear) for t in t_carr]),format='decimalyear') - flux_mean=np.array([f.mean() for f in flux_carr]) - return t_mean,flux_mean +def calculate_lya_monthly(t, flux): + """ + Calculates data that are averaged over each Carrington rotation period. Time is mean value of decimal year in each period. + """ + t_carr, flux_carr = resize_by_carr(t, flux) + t_mean = Time( + np.array([np.mean(t.decimalyear) for t in t_carr]), format="decimalyear" + ) + flux_mean = np.array([f.mean() for f in flux_carr]) + + return t_mean, flux_mean + def calculate_mean_date(fn_list): - daily_dates=[] + daily_dates = [] for f in fn_list: - l3a=read_json(f) - daily_dates.append(l3a['start_time']) - date_mean=Time(daily_dates).mean() + l3a = read_json(f) + daily_dates.append(l3a["start_time"]) + date_mean = Time(daily_dates).mean() return date_mean -def calculate_sw_energy_flux(n,v,p): - ''' - Solar wind energy flux that is latitudinal invariant - Le Chat et al. 2012a - Parameters must be given as a object with unit - Parameters: - -------------- - n - protons number density - v - plasma speed - p - concentration of the alpha particles wrt to the protons +def calculate_sw_energy_flux(n, v, p): + """ + Solar wind energy flux that is latitudinal invariant + Le Chat et al. 2012a + Parameters must be given as a object with unit + + Parameters: + -------------- + n - protons number density + v - plasma speed + p - concentration of the alpha particles wrt to the protons + + Output: + ------------ + energy_flux: energy flux at 1au in the ecliptic plane (invariant) + """ + energy_flux = ( + n + * (const.m_p + p * PHISICAL_CONSTANTS["m_alpha"]) + * v + * (0.5 * v**2 + const.GM_sun / const.R_sun) + ) - Output: - ------------ - energy_flux: energy flux at 1au in the ecliptic plane (invariant) - ''' - energy_flux=n*(const.m_p+p*PHISICAL_CONSTANTS['m_alpha'])*v*(0.5*v**2+const.GM_sun/const.R_sun) + return energy_flux.to("erg.s-1.cm-2") - return energy_flux.to('erg.s-1.cm-2') def carrington(jd): - ''' + """ Converts time in Julian Day format into Carrington Rotation number - ''' - ans=(jd-PHISICAL_CONSTANTS['jd_carrington_first'])/PHISICAL_CONSTANTS['carrington_time']+1 + """ + ans = (jd - PHISICAL_CONSTANTS["jd_carrington_first"]) / PHISICAL_CONSTANTS[ + "carrington_time" + ] + 1 return ans + def check_nan(data): - ''' + """ Checks if there are any NaNs in the output data. If so, raises Exception and data product is not produced Parameters: -------------- data - dictionary with the data product - ''' + """ for key in data: - if np.any((np.isnan(np.array(data[key])))): - raise Exception("There are NaN in the output data") + if np.any((np.isnan(np.array(data[key])))): + raise Exception("There are NaN in the output data") + def cr_from_l3b_fn(fn): - ''' + """ Extracts CR from the L3b filename. This may change if the naming convention will change - ''' - cr=fn.split('_')[5] + """ + cr = fn.split("_")[5] return cr + def cross_section_cx_LS(E): - ''' + """ Lindsay&Stebbings 2005 Returns cross_section as a astro.unit object @@ -132,14 +158,15 @@ def cross_section_cx_LS(E): ------------ sigma: cross-section as a astropy.unit object - ''' - EkeV=E.to('keV').value - out=1e-16*(1-np.exp(-67.3/EkeV))**4.5*(4.15-0.531*np.log(EkeV))**2 - sigma=out*u.cm*u.cm + """ + EkeV = E.to("keV").value + out = 1e-16 * (1 - np.exp(-67.3 / EkeV)) ** 4.5 * (4.15 - 0.531 * np.log(EkeV)) ** 2 + sigma = out * u.cm * u.cm return sigma + def cross_section_cx(E): - ''' + """ Swaczyna et al. 2025 https://doi.org/10.3847/1538-4365/adaf17 Returns cross_section as a astro.unit object @@ -152,24 +179,29 @@ def cross_section_cx(E): ------------ sigma: cross-section as a astropy.unit object - ''' + """ # Parameters - A1=4.8569 - A2=21.906 - A3=31.487 - A4=0.12018 - A5=4.1402E-6 - A6=3.7524 - A7=8.8476E-12 - A8=6.1091 - - EkeV=E.to('keV').value - out=(1e-16)*(A1*np.log(A2/EkeV+A3))/(1+A4*EkeV+A5*EkeV**A6+A7*EkeV**A8) - sigma=out*u.cm*u.cm + A1 = 4.8569 + A2 = 21.906 + A3 = 31.487 + A4 = 0.12018 + A5 = 4.1402e-6 + A6 = 3.7524 + A7 = 8.8476e-12 + A8 = 6.1091 + + EkeV = E.to("keV").value + out = ( + (1e-16) + * (A1 * np.log(A2 / EkeV + A3)) + / (1 + A4 * EkeV + A5 * EkeV**A6 + A7 * EkeV**A8) + ) + sigma = out * u.cm * u.cm return sigma + def derive_sin_cos(order): """ equivalent of Mathematica's deriveSinCos function @@ -183,135 +215,147 @@ def derive_sin_cos(order): """ funcs = [] - for i in range(order+1): + for i in range(order + 1): if i == 0: - funcs.append(base_funcs_def(i, 'None')) + funcs.append(base_funcs_def(i, "None")) else: - funcs.append(base_funcs_def(i, 's')) - funcs.append(base_funcs_def(i, 'c')) + funcs.append(base_funcs_def(i, "s")) + funcs.append(base_funcs_def(i, "c")) return funcs -def f107_daily_data(t,flux): - ''' - Returns adjusted F10.7 flux with daily cadence. In the case of multiple measurements during a day, - the closest to the noon is chosen - - Parameters: - ------------ - t: time array of raw data (astropy.time) - flux: F10.7 flux adjusted to 1au (10^(-22) W/m^2/Hz) - - Output: - ------------ - t_out: daily time array (astropy.time) - flux_out: daily F10.7 flux adjusted to 1au (10^(-22) W/m^2/Hz) - ''' - idx_noon,idx_dup=select_noon(t) # select one measurment per day - idx_no0=np.where(flux[idx_noon]>0)[0] - t_out=t[idx_noon[idx_no0]] - flux_out=flux[idx_noon[idx_no0]] - return t_out,flux_out + +def f107_daily_data(t, flux): + """ + Returns adjusted F10.7 flux with daily cadence. In the case of multiple measurements during a day, + the closest to the noon is chosen + + Parameters: + ------------ + t: time array of raw data (astropy.time) + flux: F10.7 flux adjusted to 1au (10^(-22) W/m^2/Hz) + + Output: + ------------ + t_out: daily time array (astropy.time) + flux_out: daily F10.7 flux adjusted to 1au (10^(-22) W/m^2/Hz) + """ + idx_noon, idx_dup = select_noon(t) # select one measurment per day + idx_no0 = np.where(flux[idx_noon] > 0)[0] + t_out = t[idx_noon[idx_no0]] + flux_out = flux[idx_noon[idx_no0]] + return t_out, flux_out + def find_CR_idx(CR, CR_list): - ''' + """ finds index of the value in the CR_list that is closest to the CR and the next idx it corresponds to the two L3bc files: close to current CR and the next one (for interpolation) - ''' - idx_CR = np.abs(np.array(CR_list)-int(CR)).argmin() - # OLD idx_read=[idx_CR-1,idx_CR,idx_CR+1] - idx_read=[idx_CR,idx_CR+1] + """ + idx_CR = np.abs(np.array(CR_list) - int(CR)).argmin() + # OLD idx_read=[idx_CR-1,idx_CR,idx_CR+1] + idx_read = [idx_CR, idx_CR + 1] return idx_read + def generate_cr_lya(CR, data_lya): - ''' + """ calculates Lyman-alpha composite irradiance interpolated on the center of a given CR - ''' + """ - t_CR = Time(jd_fm_Carrington(CR),format='jd') + t_CR = Time(jd_fm_Carrington(CR), format="jd") CR_list = [int(carrington(t.jd)) for t in data_lya[0]] idx_read = find_CR_idx(CR, CR_list) t_lya = np.array([data_lya[0][i].jd for i in idx_read]) lya = np.array([data_lya[1][i] for i in idx_read]) - lya_CR = np.interp(t_CR.jd,t_lya,lya) + lya_CR = np.interp(t_CR.jd, t_lya, lya) return lya_CR - def get_spher_coord_rad(vec): - """ - Compute radius r, longitude l and latitude b for a vector vec - - Parameters - ---------- - vec: 1D or 2D numpy array - input vector(s) - - Returns - ------- - if vec is an 1D numpy array, return 1D numpy array rlb (see below) - if vec is a 2D numpy array, assume that its rows represent vectors and return 2D array - rlb (see below), where every row of rlb corresponds to every row of vec - - rlb: 1D or 2D numpy array - if 1D then rlb=[r,l,b] where: r - magnitude of vector vec, l - longitude of the direction - pointed by vec in radians, b - latitude of the direction pointed by vec in radians - if 2D then columns of the array contain r,l,b - - Examples - -------- - >>> import geom_fun_libr_box as gflb - >>> import numpy as np - >>> rlb=gflb.get_spher_coord_rad(np.array([0,1,1])) - >>> print(rlb[0],np.degrees(rlb[1]),np.degrees(rlb[2])) - 1.41421356237 90.0 45.0 - >>> n=10 - >>> phi=np.radians(np.linspace(0,360,n)) - >>> tht=np.radians(np.linspace(-90,90,n)) - >>> vecs=np.column_stack([np.cos(phi)*np.cos(tht),np.sin(phi)*np.cos(tht),np.sin(tht)]) - >>> print(gflb.get_spher_coord_rad(vecs)) - [[ 1. 0. -1.57079633] - [ 1. 0.6981317 -1.22173048] - [ 1. 1.3962634 -0.87266463] - [ 1. 2.0943951 -0.52359878] - [ 1. 2.7925268 -0.17453293] - [ 1. 3.4906585 0.17453293] - [ 1. 4.1887902 0.52359878] - [ 1. 4.88692191 0.87266463] - [ 1. 5.58505361 1.22173048] - [ 1. 6.28318531 1.57079633]] - """ - if vec.ndim==1: - if np.shape(vec)[0]!=3: - raise "get_spher_coord_rad() error: np.shape(vec)[0]!=3" - r=np.sqrt(np.dot(vec,vec)) - if r==0.0: b=0.0 - else: b=np.arcsin(vec[2]/r) # compute latitude - # compute longitude - if vec[0]==0.0 and vec[1]==0.0: l=0.0 - else: l=np.arctan2(vec[1],vec[0])%(2*np.pi) - return np.array([r,l,b]) # return one array similarly as in Mathematica - else: - if np.shape(vec)[1]!=3: - raise "get_spher_coord_rad() error: np.shape(vec)[1]!=3" - r=np.sqrt((vec * vec).sum(axis=1)) - idxs=np.nonzero(r!=0.0) - b=np.zeros((len(r)),np.float64) - b[idxs[0]]=np.arcsin(vec[idxs[0],2]/r[idxs[0]]) # compute latitude - l=np.arctan2(vec[:,1],vec[:,0])%(2*np.pi) # compute longitude - idxs=np.nonzero(np.logical_and(vec[:,0]==0.0,vec[:,1]==0.0)) - l[idxs[0]]=0.0 # set l=0 if vec[:,0]==0.0 and vec[:,1]==0.0 - return np.column_stack([r,l,b]) # return one array similarly as in Mathematica + """ + Compute radius r, longitude l and latitude b for a vector vec + + Parameters + ---------- + vec: 1D or 2D numpy array + input vector(s) + + Returns + ------- + if vec is an 1D numpy array, return 1D numpy array rlb (see below) + if vec is a 2D numpy array, assume that its rows represent vectors and return 2D array + rlb (see below), where every row of rlb corresponds to every row of vec + + rlb: 1D or 2D numpy array + if 1D then rlb=[r,l,b] where: r - magnitude of vector vec, l - longitude of the direction + pointed by vec in radians, b - latitude of the direction pointed by vec in radians + if 2D then columns of the array contain r,l,b + + Examples + -------- + >>> import geom_fun_libr_box as gflb + >>> import numpy as np + >>> rlb=gflb.get_spher_coord_rad(np.array([0,1,1])) + >>> print(rlb[0],np.degrees(rlb[1]),np.degrees(rlb[2])) + 1.41421356237 90.0 45.0 + >>> n=10 + >>> phi=np.radians(np.linspace(0,360,n)) + >>> tht=np.radians(np.linspace(-90,90,n)) + >>> vecs=np.column_stack([np.cos(phi)*np.cos(tht),np.sin(phi)*np.cos(tht),np.sin(tht)]) + >>> print(gflb.get_spher_coord_rad(vecs)) + [[ 1. 0. -1.57079633] + [ 1. 0.6981317 -1.22173048] + [ 1. 1.3962634 -0.87266463] + [ 1. 2.0943951 -0.52359878] + [ 1. 2.7925268 -0.17453293] + [ 1. 3.4906585 0.17453293] + [ 1. 4.1887902 0.52359878] + [ 1. 4.88692191 0.87266463] + [ 1. 5.58505361 1.22173048] + [ 1. 6.28318531 1.57079633]] + """ + if vec.ndim == 1: + if np.shape(vec)[0] != 3: + raise "get_spher_coord_rad() error: np.shape(vec)[0]!=3" + r = np.sqrt(np.dot(vec, vec)) + if r == 0.0: + b = 0.0 + else: + b = np.arcsin(vec[2] / r) # compute latitude + # compute longitude + if vec[0] == 0.0 and vec[1] == 0.0: + l = 0.0 + else: + l = np.arctan2(vec[1], vec[0]) % (2 * np.pi) + return np.array([r, l, b]) # return one array similarly as in Mathematica + else: + if np.shape(vec)[1] != 3: + raise "get_spher_coord_rad() error: np.shape(vec)[1]!=3" + r = np.sqrt((vec * vec).sum(axis=1)) + idxs = np.nonzero(r != 0.0) + b = np.zeros((len(r)), np.float64) + b[idxs[0]] = np.arcsin(vec[idxs[0], 2] / r[idxs[0]]) # compute latitude + l = np.arctan2(vec[:, 1], vec[:, 0]) % (2 * np.pi) # compute longitude + idxs = np.nonzero(np.logical_and(vec[:, 0] == 0.0, vec[:, 1] == 0.0)) + l[idxs[0]] = 0.0 # set l=0 if vec[:,0]==0.0 and vec[:,1]==0.0 + return np.column_stack( + [r, l, b] + ) # return one array similarly as in Mathematica + def jd_fm_Carrington(carr): - ''' + """ calculates time in format of Julian Days for a given Carrington Rotation number - ''' - ans=(carr-1)*PHISICAL_CONSTANTS['carrington_time']+PHISICAL_CONSTANTS['jd_carrington_first'] + """ + ans = (carr - 1) * PHISICAL_CONSTANTS["carrington_time"] + PHISICAL_CONSTANTS[ + "jd_carrington_first" + ] return ans + def make_lsq_matrix(psi, base_funcs): """ make data matrix and vector for least-squares method in the matrix form @@ -331,52 +375,56 @@ def make_lsq_matrix(psi, base_funcs): i = i + 1 return lsq_mtrx + def process_omni_param(omni_raw, cr_grid, param_settings): - ''' + """ Processes OMNI2 raw data to obtain averaged solar wind speed, density and alpha-particles abundance - ''' + """ # select from the raw data columns relevant for a given parameter - param_raw=omni_raw[:,param_settings['column_numbers']] + param_raw = omni_raw[:, param_settings["column_numbers"]] # remove gaps where there was no data available based on coded value - param=param_raw[param_raw[:,3] original_last_date - param_v=np.append(param_v,CONST_ALPHA_TO_PROTON) + param_v = np.append(param_v, CONST_ALPHA_TO_PROTON) # split array into 1-Carrington chunks - param_cr, idx_param=np.unique(np.floor(date_cr), return_index=True,axis=0) - param_s=np.split(param_v,idx_param)[1:] + param_cr, idx_param = np.unique(np.floor(date_cr), return_index=True, axis=0) + param_s = np.split(param_v, idx_param)[1:] # calculate averaged over 1 Carrington time and parameter value - param_value=np.array([p.mean() for p in param_s]) + param_value = np.array([p.mean() for p in param_s]) # interpolate into cr_grid - if np.logical_and(param_cr[0]<=cr_grid[0], param_cr[-1]>=cr_grid[-1]): - param_value_interp=np.interp(cr_grid,param_cr,param_value) + if np.logical_and(param_cr[0] <= cr_grid[0], param_cr[-1] >= cr_grid[-1]): + param_value_interp = np.interp(cr_grid, param_cr, param_value) else: - logging.info('There are gaps in the OMNI2 data that cannot be filled by the interpolation') - raise Exception('OMNI Error: not enough data for interpolation') + logging.info( + "There are gaps in the OMNI2 data that cannot be filled by the interpolation" + ) + raise Exception("OMNI Error: not enough data for interpolation") return param_value_interp, used_nominal_per_cr def read_f107_raw_data(fn): - ''' + """ Read raw F10.7 data from file Parameters: @@ -387,61 +435,76 @@ def read_f107_raw_data(fn): ------------ t0: time array (astropy.time) flux_adjusted: F10.7 flux adjusted to 1au (10^(-22) W/m^2/Hz) - ''' + """ data = np.loadtxt(fn) - jd=data[:,2] # time of observation in JD - flux_adjusted=data[:,5] # adjusted do 1AU flux in sfu 1sfu=10^(-22) W/m^2/Hz (taking into accont that Sun-Earth distance is changing) - t0=Time(jd,format='jd') # time converted to astropy.time format + jd = data[:, 2] # time of observation in JD + flux_adjusted = data[ + :, 5 + ] # adjusted do 1AU flux in sfu 1sfu=10^(-22) W/m^2/Hz (taking into accont that Sun-Earth distance is changing) + t0 = Time(jd, format="jd") # time converted to astropy.time format + + return t0, flux_adjusted - return t0,flux_adjusted -def read_hdr_txt(fn,N): - ''' +def read_hdr_txt(fn, N): + """ Reads header from the text file and preapers it to the further analysis - ''' + """ # open file and read header with open(fn) as file: - hdr=[next(file) for _ in range(N)] + hdr = [next(file) for _ in range(N)] # remove hash - hdr_str=remove_hash(hdr) + hdr_str = remove_hash(hdr) return hdr_str + def read_lya_data(fn): - ''' + """ reads daily Lyman-alpha composite irradiance and calculates monthly averages - ''' - t_lya,flux_lya,_= read_lya_raw_data(fn) - t_lya_m,flux_lya_m = calculate_lya_monthly(t_lya,flux_lya) - return [t_lya_m,flux_lya_m] + """ + t_lya, flux_lya, _ = read_lya_raw_data(fn) + t_lya_m, flux_lya_m = calculate_lya_monthly(t_lya, flux_lya) + return [t_lya_m, flux_lya_m] + def read_lya_raw_data(filename): - ''' + """ Reads *.nc file with composite Lyman-alpha solar irradiance (from LAPS) Returns daily product adjusted to 1au. Flux is in ph/(cm^2 s^2) - ''' - - data=netCDF4.Dataset(filename) - - convert_to_ph=data.variables['convert_to_photons'][0].data # Multiplicative factor to convert irr_121 from W/m^2 to ph/(cm^2 s^2) [ph / (cm^2 s^2)] / (W / m^2) - t_d1947=data.variables['time'][:].data # time of observation [days since 1947-01-01 12:00:00 UTC] - flux0=data.variables['irr_121'][:].data # Averaged daily irradiance in band from 121.0 to 122.0 nm at 1 AU [W/m^2] - flux=flux0*convert_to_ph # Flux in ph/(cm^2 s^2) - flux_uncert=data.variables['irr_121_uncertainty'][:].data # Uncertainty (1 standard deviation) of average daily irradiance [W/m^2] + """ - t0=Time('1947-01-01 12:00') + data = netCDF4.Dataset(filename) + + convert_to_ph = data.variables[ + "convert_to_photons" + ][ + 0 + ].data # Multiplicative factor to convert irr_121 from W/m^2 to ph/(cm^2 s^2) [ph / (cm^2 s^2)] / (W / m^2) + t_d1947 = data.variables["time"][ + : + ].data # time of observation [days since 1947-01-01 12:00:00 UTC] + flux0 = data.variables["irr_121"][ + : + ].data # Averaged daily irradiance in band from 121.0 to 122.0 nm at 1 AU [W/m^2] + flux = flux0 * convert_to_ph # Flux in ph/(cm^2 s^2) + flux_uncert = data.variables["irr_121_uncertainty"][ + : + ].data # Uncertainty (1 standard deviation) of average daily irradiance [W/m^2] + + t0 = Time("1947-01-01 12:00") # convertion to jd and adding days since 1947-01-01 - it is more accurate than simple adding days to date - t=Time(t0.jd+t_d1947,format='jd') + t = Time(t0.jd + t_d1947, format="jd") return t, flux, flux_uncert -def read_raw_OMNI_data(ext_dependencies,t_window=None): - ''' +def read_raw_OMNI_data(ext_dependencies, t_window=None): + """ Reads 3 types of data from OMNI2 database: plasma speed, proton density, and alpha-particles abundance in the ecliptic plane Parameters: @@ -450,48 +513,55 @@ def read_raw_OMNI_data(ext_dependencies,t_window=None): list of file names with external dependencies t_window: list [astropy.time, astropy.time] time window of the data that are needed - ''' + """ # Read OMNI2 data (hour resolution) # year, DOY, hour, S/C Id, proton_dens, plasma_speed, alpha_abundance, sigma proton_dens, sigma plasma_speed, sigma alpha_abundance - omni_raw=np.loadtxt(ext_dependencies['omni_raw_data'],usecols=(0,1,2,5,23,24,27,30,31,34)) - omni_raw_date=time_from_yday(omni_raw[:,(0,1,2)]) - if t_window==None: out=omni_raw + omni_raw = np.loadtxt( + ext_dependencies["omni_raw_data"], usecols=(0, 1, 2, 5, 23, 24, 27, 30, 31, 34) + ) + omni_raw_date = time_from_yday(omni_raw[:, (0, 1, 2)]) + if t_window == None: + out = omni_raw else: - idx_ini=np.abs(omni_raw_date-t_window[0]).argmin() - idx_fin=np.abs(omni_raw_date-t_window[1]).argmin() + idx_ini = np.abs(omni_raw_date - t_window[0]).argmin() + idx_fin = np.abs(omni_raw_date - t_window[1]).argmin() - out=omni_raw[idx_ini:idx_fin] + out = omni_raw[idx_ini:idx_fin] return out + def read_json(fn): - ''' + """ Opens, reads and returns the content of the json file - ''' - fp = open(fn, 'r') + """ + fp = open(fn, "r") data = json.load(fp) fp.close() return data + def remove_hash(hdr): - ''' + """ Removes # from the header that is read from the text files and returns string that will be used to generate another header - ''' - hdr_str=[l[2:] for l in hdr] + """ + hdr_str = [l[2:] for l in hdr] return hdr_str -def resize_by_carr(t,flux): - ''' - Reshaping flux array in such a way that each element is a list of fluxes from the same carrington rotation number - ''' - t_carr=carrington(t.jd) - n_carr=np.array([int(t_carr[i]) for i in np.arange(len(t_carr))]) - flux_carr_sort=np.split(flux,np.unique(n_carr,return_index=True)[1][1:]) - t_carr_sort=np.split(t,np.unique(n_carr,return_index=True)[1][1:]) - return t_carr_sort,flux_carr_sort + +def resize_by_carr(t, flux): + """ + Reshaping flux array in such a way that each element is a list of fluxes from the same carrington rotation number + """ + t_carr = carrington(t.jd) + n_carr = np.array([int(t_carr[i]) for i in np.arange(len(t_carr))]) + flux_carr_sort = np.split(flux, np.unique(n_carr, return_index=True)[1][1:]) + t_carr_sort = np.split(t, np.unique(n_carr, return_index=True)[1][1:]) + return t_carr_sort, flux_carr_sort + def scale_density(date, dens): - ''' + """ Scaling density measurements that are taken at different distance than 1au. We need solar wind parameters ajusted to 1au. Proton density decreases as 1/r^2 @@ -502,21 +572,22 @@ def scale_density(date, dens): time of the measurement dens : 1D float vector measured density [#/cm^3] - ''' + """ # Sun-Earth distance at given moment - d_sun=get_sun(date).distance + d_sun = get_sun(date).distance # Sun-L1 distance - r_obs=d_sun*(1-(const.M_earth/(3*const.M_sun))**(1./3)) + r_obs = d_sun * (1 - (const.M_earth / (3 * const.M_sun)) ** (1.0 / 3)) # density ajustment - dens_1au=dens*(r_obs/u.au)**2 + dens_1au = dens * (r_obs / u.au) ** 2 return dens_1au -def select_data_window(data,t_window): - ''' + +def select_data_window(data, t_window): + """ Selects data whitin a given time window Parameters: @@ -526,15 +597,16 @@ def select_data_window(data,t_window): t_ini: astropy.time t_end: astropy.time - ''' - idx=np.where(np.logical_and(data[0]>=t_window[0],data[0]<=t_window[1]))[0] - t=data[0][idx] - data=data[1][idx] + """ + idx = np.where(np.logical_and(data[0] >= t_window[0], data[0] <= t_window[1]))[0] + t = data[0][idx] + data = data[1][idx] + + return [t, data] - return [t,data] def select_noon(t_pen): - ''' + """ Select the dates that are closest to the local noon Parameters: @@ -546,119 +618,203 @@ def select_noon(t_pen): 1. index list of all data taken at noon (20 UT) or the closest one if there is no data at noon 2. a list of indexes of data taken on the same day (if there is only one measurement then this element is just 1 element list np. [[1],[2,3,4],[6]] - ''' - #TODO it should be done in a dynamical way - it should determine how many measurements are in a given day - index=np.array([0]) - index_duplicate=[] - i=0 - while i<(len(t_pen)-3): - - if (t_pen[i].yday[:8]==t_pen[i+1].yday[:8]): - - if(t_pen[i].yday[:8]==t_pen[i+2].yday[:8]): - if(t_pen[i].yday[:8]==t_pen[i+3].yday[:8]): - h=np.array([int(t_pen[i].yday[9:11]),int(t_pen[i+1].yday[9:11]),int(t_pen[i+2].yday[9:11]),int(t_pen[i+3].yday[9:11])]) - k=i+np.abs(h-20).argmin() # index of the hour that is near 20UT (local noon) - d=[i,i+1,i+2,i+3] - i=i+4 + """ + # TODO it should be done in a dynamical way - it should determine how many measurements are in a given day + index = np.array([0]) + index_duplicate = [] + i = 0 + while i < (len(t_pen) - 3): + if t_pen[i].yday[:8] == t_pen[i + 1].yday[:8]: + if t_pen[i].yday[:8] == t_pen[i + 2].yday[:8]: + if t_pen[i].yday[:8] == t_pen[i + 3].yday[:8]: + h = np.array( + [ + int(t_pen[i].yday[9:11]), + int(t_pen[i + 1].yday[9:11]), + int(t_pen[i + 2].yday[9:11]), + int(t_pen[i + 3].yday[9:11]), + ] + ) + k = ( + i + np.abs(h - 20).argmin() + ) # index of the hour that is near 20UT (local noon) + d = [i, i + 1, i + 2, i + 3] + i = i + 4 else: - h=np.array([int(t_pen[i].yday[9:11]),int(t_pen[i+1].yday[9:11]),int(t_pen[i+2].yday[9:11])]) - k=i+np.abs(h-20).argmin() # index of the hour that is near 20UT (local noon) - d=[i,i+1,i+2] - i=i+3 + h = np.array( + [ + int(t_pen[i].yday[9:11]), + int(t_pen[i + 1].yday[9:11]), + int(t_pen[i + 2].yday[9:11]), + ] + ) + k = ( + i + np.abs(h - 20).argmin() + ) # index of the hour that is near 20UT (local noon) + d = [i, i + 1, i + 2] + i = i + 3 else: - h=np.array([int(t_pen[i].yday[9:11]),int(t_pen[i+1].yday[9:11])]) - k=i+np.abs(h-20).argmin() # index of the hour that is near 20UT (local noon) - d=[i,i+1] - i=i+2 + h = np.array([int(t_pen[i].yday[9:11]), int(t_pen[i + 1].yday[9:11])]) + k = ( + i + np.abs(h - 20).argmin() + ) # index of the hour that is near 20UT (local noon) + d = [i, i + 1] + i = i + 2 else: - k=i - d=[i] - i=i+1 - index=np.append(index,k) + k = i + d = [i] + i = i + 1 + index = np.append(index, k) index_duplicate.append(d) - return index[1:],index_duplicate + return index[1:], index_duplicate + def test_WawHelioInMP_file(model): - ''' + """ test if json file has what it should inside - ''' - - vg_points_number=model['header']['vg_points_number'] - hion_filtration=model['params_hion']['hion_filtration'] - hion_order=model['params_hion']['hion_order'] - - hion_const_0=np.array(model['params_hion']['hion_const']).shape[0] - hion_base=np.array(model['params_hion']['hion_base']) - hion_base_0=np.array(model['params_hion']['hion_base']).shape[0] - hion_base_1=np.array(model['params_hion']['hion_base']).shape[1] - - if vg_points_number!=len(model['vg_points']): - logging.error('vg_points_number is incorrect.\nIt is '+str(vg_points_number) - +'\nIt should be '+str(len(model['vg_points']))) - raise ArrayShapeError('Array shape error: Array shape is incorrect') - - elif hion_const_0!=(hion_order-1-hion_filtration): - logging.error('hion_const shape is incorrect.\nIt is '+str(hion_const_0) - +'\nIt should be '+str(hion_order-1-hion_filtration)) - raise ArrayShapeError('Array shape error: Array shape is incorect') - - elif np.logical_or(hion_base_0!=hion_order-1,hion_base_1!=hion_order+1): - logging.error('hion_base shape is incorrect.\nIt is '+str(hion_base.shape) - +'\nIt should be ('+str(hion_order-1)+','+str(hion_order+1)+')') - raise ArrayShapeError('Array shape error: Array shape is incorect') - - - for k in np.arange(model['header']['vg_points_number']): - - lcrv_filtration=model['vg_points'][k]['parameters']['lcrv_filtration'] - lcrv_order=model['vg_points'][k]['parameters']['lcrv_order'] - - model_const_0=np.array(model['vg_points'][k]['parameters']['model_const']).shape[0] - model_matrix=np.array(model['vg_points'][k]['parameters']['model_matrix']) - model_matrix_0=np.array(model['vg_points'][k]['parameters']['model_matrix']).shape[0] - model_matrix_1=np.array(model['vg_points'][k]['parameters']['model_matrix']).shape[1] - - lcrv_Trmatrix=np.array(model['vg_points'][k]['parameters']['lcrv_Trmatrix']) - lcrv_Trmatrix_0=np.array(model['vg_points'][k]['parameters']['lcrv_Trmatrix']).shape[0] - lcrv_Trmatrix_1=np.array(model['vg_points'][k]['parameters']['lcrv_Trmatrix']).shape[1] - - if model_const_0!=hion_filtration: - logging.error('model_const shape is incorrect.\nIt is '+str(model_const_0) - +'\nIt should be '+str(hion_filtration)) - raise ArrayShapeError('Array shape error: Array shape is incorrect') + """ - elif np.logical_or(model_matrix_0!=hion_filtration,model_matrix_1!=lcrv_filtration): - logging.error('model_matrix shape is incorrect.\nIt is '+str(model_matrix.shape) - +'\nIt should be ('+str(hion_filtration)+','+str(lcrv_filtration)+')') - raise ArrayShapeError('Array shape error: Array shape is incorect') + vg_points_number = model["header"]["vg_points_number"] + hion_filtration = model["params_hion"]["hion_filtration"] + hion_order = model["params_hion"]["hion_order"] + + hion_const_0 = np.array(model["params_hion"]["hion_const"]).shape[0] + hion_base = np.array(model["params_hion"]["hion_base"]) + hion_base_0 = np.array(model["params_hion"]["hion_base"]).shape[0] + hion_base_1 = np.array(model["params_hion"]["hion_base"]).shape[1] + + if vg_points_number != len(model["vg_points"]): + logging.error( + "vg_points_number is incorrect.\nIt is " + + str(vg_points_number) + + "\nIt should be " + + str(len(model["vg_points"])) + ) + raise ArrayShapeError("Array shape error: Array shape is incorrect") + + elif hion_const_0 != (hion_order - 1 - hion_filtration): + logging.error( + "hion_const shape is incorrect.\nIt is " + + str(hion_const_0) + + "\nIt should be " + + str(hion_order - 1 - hion_filtration) + ) + raise ArrayShapeError("Array shape error: Array shape is incorect") + + elif np.logical_or(hion_base_0 != hion_order - 1, hion_base_1 != hion_order + 1): + logging.error( + "hion_base shape is incorrect.\nIt is " + + str(hion_base.shape) + + "\nIt should be (" + + str(hion_order - 1) + + "," + + str(hion_order + 1) + + ")" + ) + raise ArrayShapeError("Array shape error: Array shape is incorect") + + for k in np.arange(model["header"]["vg_points_number"]): + lcrv_filtration = model["vg_points"][k]["parameters"]["lcrv_filtration"] + lcrv_order = model["vg_points"][k]["parameters"]["lcrv_order"] + + model_const_0 = np.array( + model["vg_points"][k]["parameters"]["model_const"] + ).shape[0] + model_matrix = np.array(model["vg_points"][k]["parameters"]["model_matrix"]) + model_matrix_0 = np.array( + model["vg_points"][k]["parameters"]["model_matrix"] + ).shape[0] + model_matrix_1 = np.array( + model["vg_points"][k]["parameters"]["model_matrix"] + ).shape[1] + + lcrv_Trmatrix = np.array(model["vg_points"][k]["parameters"]["lcrv_Trmatrix"]) + lcrv_Trmatrix_0 = np.array( + model["vg_points"][k]["parameters"]["lcrv_Trmatrix"] + ).shape[0] + lcrv_Trmatrix_1 = np.array( + model["vg_points"][k]["parameters"]["lcrv_Trmatrix"] + ).shape[1] + + if model_const_0 != hion_filtration: + logging.error( + "model_const shape is incorrect.\nIt is " + + str(model_const_0) + + "\nIt should be " + + str(hion_filtration) + ) + raise ArrayShapeError("Array shape error: Array shape is incorrect") + + elif np.logical_or( + model_matrix_0 != hion_filtration, model_matrix_1 != lcrv_filtration + ): + logging.error( + "model_matrix shape is incorrect.\nIt is " + + str(model_matrix.shape) + + "\nIt should be (" + + str(hion_filtration) + + "," + + str(lcrv_filtration) + + ")" + ) + raise ArrayShapeError("Array shape error: Array shape is incorect") + + elif np.logical_or( + lcrv_Trmatrix_0 != 2 * lcrv_order + 1, lcrv_Trmatrix_1 != 2 * lcrv_order + 1 + ): + logging.error( + "lcrv_Trmatrix shape is incorrect.\nIt is " + + str(lcrv_Trmatrix.shape) + + "\nIt should be (" + + str(2 * lcrv_order + 1) + + "," + + str(2 * lcrv_order + 1) + + ")" + ) + raise ArrayShapeError("Array shape error: Array shape is incorect") - elif np.logical_or(lcrv_Trmatrix_0!=2*lcrv_order+1,lcrv_Trmatrix_1!=2*lcrv_order+1): - logging.error('lcrv_Trmatrix shape is incorrect.\nIt is '+str(lcrv_Trmatrix.shape) - +'\nIt should be ('+str(2*lcrv_order+1)+','+str(2*lcrv_order+1)+')') - raise ArrayShapeError('Array shape error: Array shape is incorect') def time_string_from_l3a_fn(fn): - ''' + """ Extracts date from the L3a filename. This may change if the naming convention will change - ''' - fn_string=fn.split('_')[4] - time_string=fn_string[0:4]+'-'+fn_string[4:6]+'-'+fn_string[6:8]+' '+fn_string[8:10]+':'+fn_string[10:12]+':'+fn_string[12:14] + """ + fn_string = fn.split("_")[4] + time_string = ( + fn_string[0:4] + + "-" + + fn_string[4:6] + + "-" + + fn_string[6:8] + + " " + + fn_string[8:10] + + ":" + + fn_string[10:12] + + ":" + + fn_string[12:14] + ) return time_string + def time_from_l3a(fn): - data_l3a=read_json(fn) - mean_time=Time(data_l3a['start_time'])+0.5*(Time(data_l3a['end_time'])-Time(data_l3a['start_time'])) - return [mean_time, Time(data_l3a['start_time']), Time(data_l3a['end_time'])] + data_l3a = read_json(fn) + mean_time = Time(data_l3a["start_time"]) + 0.5 * ( + Time(data_l3a["end_time"]) - Time(data_l3a["start_time"]) + ) + return [mean_time, Time(data_l3a["start_time"]), Time(data_l3a["end_time"])] + def time_from_yday(yday_list): - string = [f'{int(yday_list[i, 0])}:{int(yday_list[i, 1]):03d}:{int(yday_list[i, 2]):02d}:00' for i in range(len(yday_list))] - date=Time(string,format='yday') - return date + string = [ + f"{int(yday_list[i, 0])}:{int(yday_list[i, 1]):03d}:{int(yday_list[i, 2]):02d}:00" + for i in range(len(yday_list)) + ] + date = Time(string, format="yday") + return date + def v2E(v): - ''' + """ Speed to kinetic energy transformation for hydrogen Parameters: @@ -668,8 +824,8 @@ def v2E(v): Output: ------------ E: energy as an astropy.unit object - ''' + """ - m_H=1.6733e-24*u.g - E=(0.5*m_H*v**2).to('eV') + m_H = 1.6733e-24 * u.g + E = (0.5 * m_H * v**2).to("eV") return E diff --git a/imap_l3_processing/glows/l3d/science/toolkit/l3d_SolarParamHistory.py b/imap_l3_processing/glows/l3d/science/toolkit/l3d_SolarParamHistory.py index 58d7e2e82..8373d6cc3 100644 --- a/imap_l3_processing/glows/l3d/science/toolkit/l3d_SolarParamHistory.py +++ b/imap_l3_processing/glows/l3d/science/toolkit/l3d_SolarParamHistory.py @@ -10,8 +10,8 @@ @dataclass -class SolarParamsHistory(): - ''' +class SolarParamsHistory: + """ Class for an Ionization files generator Text files (solar wind speed, density, uv-anisotropy, Lyman-alpha irradiance, photoionization, and electron density) are requaried as a input files for L3e processing @@ -41,9 +41,9 @@ class SolarParamsHistory(): _update_lya_data(self,ext_dependencies,CR) update_solar_params_hist(self,ext_dependencies,data_l3b,data_l3c) - ''' + """ - def __init__(self,anc_input_from_instr_team,ext_dependencies): + def __init__(self, anc_input_from_instr_team, ext_dependencies): """ Class constructor @@ -101,276 +101,386 @@ def __init__(self,anc_input_from_instr_team,ext_dependencies): """ # read pipeline settings to be used for L3d - self.settings = fun.read_json(anc_input_from_instr_team['pipeline_settings']) - - self.header={ - 'software_version': VERSION, - 'filename': None, - 'ancillary_data_files': None, - 'external_dependeciens': ext_dependencies['lya_raw_data'], - 'l3b_input_filename': [], - 'l3c_input_filename': [] + self.settings = fun.read_json(anc_input_from_instr_team["pipeline_settings"]) + + self.header = { + "software_version": VERSION, + "filename": None, + "ancillary_data_files": None, + "external_dependeciens": ext_dependencies["lya_raw_data"], + "l3b_input_filename": [], + "l3c_input_filename": [], } - self.ini_data={} - self.ini_data['CR_last']=None - self.ini_data['label']=list(anc_input_from_instr_team['WawHelioIon'].keys()) - N=len(self.ini_data['label']) - self.ini_data['time']={} - for k in anc_input_from_instr_team['WawHelioIon']: self.ini_data['time'][k]=None - self.ini_data['data']={} - for k in anc_input_from_instr_team['WawHelioIon']: self.ini_data['data'][k]=None - self.ini_data['hdr_txt']={} - for k in anc_input_from_instr_team['WawHelioIon']: self.ini_data['hdr_txt'][k]=None - self.ini_data['flags']={} - for k in anc_input_from_instr_team['WawHelioIon']: self.ini_data['flags'][k]=None - - self.lat_grid=self.settings['solar_wind_grid'] # latitudinal grid [deg] - self.CR_grid=None # regular grid with Carrington rotation period resolution. Bin centered on Carr halves - self.time_grid=None # decimal year corresponding to the regular grid with bins on Carrington halves - self.CR_last=None - - self.solar_params={} # ionization parameters interplolated to the Carrington rotation halves - for k in anc_input_from_instr_team['WawHelioIon']: self.solar_params[k]=None - - self.flags={} # ionization parameters interplolated to the Carrington rotation halves - for k in anc_input_from_instr_team['WawHelioIon']: self.flags[k]=None - - self.glows_flags=None # GLOWS quality-flag bitmask per CR (initialized in generate_initial_history) + self.ini_data = {} + self.ini_data["CR_last"] = None + self.ini_data["label"] = list(anc_input_from_instr_team["WawHelioIon"].keys()) + N = len(self.ini_data["label"]) + self.ini_data["time"] = {} + for k in anc_input_from_instr_team["WawHelioIon"]: + self.ini_data["time"][k] = None + self.ini_data["data"] = {} + for k in anc_input_from_instr_team["WawHelioIon"]: + self.ini_data["data"][k] = None + self.ini_data["hdr_txt"] = {} + for k in anc_input_from_instr_team["WawHelioIon"]: + self.ini_data["hdr_txt"][k] = None + self.ini_data["flags"] = {} + for k in anc_input_from_instr_team["WawHelioIon"]: + self.ini_data["flags"][k] = None + + self.lat_grid = self.settings["solar_wind_grid"] # latitudinal grid [deg] + self.CR_grid = None # regular grid with Carrington rotation period resolution. Bin centered on Carr halves + self.time_grid = None # decimal year corresponding to the regular grid with bins on Carrington halves + self.CR_last = None + + self.solar_params = {} # ionization parameters interplolated to the Carrington rotation halves + for k in anc_input_from_instr_team["WawHelioIon"]: + self.solar_params[k] = None + + self.flags = {} # ionization parameters interplolated to the Carrington rotation halves + for k in anc_input_from_instr_team["WawHelioIon"]: + self.flags[k] = None + + self.glows_flags = None # GLOWS quality-flag bitmask per CR (initialized in generate_initial_history) def _check_ini_data(self): - ''' + """ Check if initial data ends at the same CR - ''' + """ # Resolution of the file should be Carrington rotation # Last CR in every file should be the same - eps=0.001 - diff=np.array([self.ini_data['time'][k][-1]-self.ini_data['time']['lya'][-1] for k in self.ini_data['time']]) - if (diff>eps).any(): - raise Exception('Incorrect format of the input files. Last CR is not the same in all files') - else: return True + eps = 0.001 + diff = np.array( + [ + self.ini_data["time"][k][-1] - self.ini_data["time"]["lya"][-1] + for k in self.ini_data["time"] + ] + ) + if (diff > eps).any(): + raise Exception( + "Incorrect format of the input files. Last CR is not the same in all files" + ) + else: + return True def find_fn_initial(self): - ''' + """ Finds text files from the previous CR that now are initial files for the current CR. For each type of the text file (speed, p-dens, etc.) a file with the highest CR is selected In the final version there should be additional option for version checking - ''' - l3d_txt_fn_list={} - for k in self.ini_data['label']: l3d_txt_fn_list[k]=np.array(sorted(glob.glob('data_l3d_txt/imap_glows_l3d*'+k+'*.dat'))) + """ + l3d_txt_fn_list = {} + for k in self.ini_data["label"]: + l3d_txt_fn_list[k] = np.array( + sorted(glob.glob("data_l3d_txt/imap_glows_l3d*" + k + "*.dat")) + ) # list of the L3d text files from the latest CR - fn_initial={} + fn_initial = {} for k in l3d_txt_fn_list: - idx_latest=np.array([f.split('cr_')[1].split('_')[0] for f in l3d_txt_fn_list[k]]).argmax() - fn_initial[k]=l3d_txt_fn_list[k][idx_latest] + idx_latest = np.array( + [f.split("cr_")[1].split("_")[0] for f in l3d_txt_fn_list[k]] + ).argmax() + fn_initial[k] = l3d_txt_fn_list[k][idx_latest] return fn_initial - - def _generate_cr_solar_params(self,CR, data_l3b, data_l3c): - ''' + def _generate_cr_solar_params(self, CR, data_l3b, data_l3c): + """ Calculates solar parameters (plasma speed, proton density, photoionization, electron density, and uv anisotropy) interpolated on the center of a given CR - ''' + """ - cr_params={} + cr_params = {} # find indexes of the closest to the current CR and the next CR where L3bc are available - t_CR = Time(fun.jd_fm_Carrington(CR),format='jd') + t_CR = Time(fun.jd_fm_Carrington(CR), format="jd") - CR_list_b=[data['CR'] for data in data_l3b] - CR_list_c=[data['CR'] for data in data_l3b] + CR_list_b = [data["CR"] for data in data_l3b] + CR_list_c = [data["CR"] for data in data_l3b] - idx_read_b = fun.find_CR_idx(CR, CR_list_b) # two value vector with idx of the current CR and the next + idx_read_b = fun.find_CR_idx( + CR, CR_list_b + ) # two value vector with idx of the current CR and the next idx_read_c = fun.find_CR_idx(CR, CR_list_c) # if there is only current CR, but not the next one available, then we can't interpolate # but we can make a temporary entry with values copied from L3bc as they are (not exactly at CR.5) - Nmid=int(len(data_l3b[0]['ion_rate_profile']['lat_grid'])/2) # idx of the middle bin in phion - idx_current=idx_read_b[0] + Nmid = int( + len(data_l3b[0]["ion_rate_profile"]["lat_grid"]) / 2 + ) # idx of the middle bin in phion + idx_current = idx_read_b[0] # Patched temporary record of L3d - if int(CR)==CR_list_b[-1]: - idx_read_b=[idx_read_b[0]] - idx_read_c=[idx_read_c[0]] - cr_params['uv-anis']=data_l3b[idx_current]['uv_anisotropy_factor'] - cr_params['phion']=data_l3b[idx_current]['ion_rate_profile']['ph_rate'][Nmid] - cr_params['p-dens']=data_l3c[idx_current]['solar_wind_profile']['proton_density'] - cr_params['speed']=data_l3c[idx_current]['solar_wind_profile']['plasma_speed'] - p_dens_ecl=data_l3c[idx_current]['solar_wind_ecliptic']['proton_density'] - a_abundance_ecl=data_l3c[idx_current]['solar_wind_ecliptic']['alpha_abundance'] - cr_params['e-dens']=p_dens_ecl*(1+2*a_abundance_ecl) + if int(CR) == CR_list_b[-1]: + idx_read_b = [idx_read_b[0]] + idx_read_c = [idx_read_c[0]] + cr_params["uv-anis"] = data_l3b[idx_current]["uv_anisotropy_factor"] + cr_params["phion"] = data_l3b[idx_current]["ion_rate_profile"]["ph_rate"][ + Nmid + ] + cr_params["p-dens"] = data_l3c[idx_current]["solar_wind_profile"][ + "proton_density" + ] + cr_params["speed"] = data_l3c[idx_current]["solar_wind_profile"][ + "plasma_speed" + ] + p_dens_ecl = data_l3c[idx_current]["solar_wind_ecliptic"]["proton_density"] + a_abundance_ecl = data_l3c[idx_current]["solar_wind_ecliptic"][ + "alpha_abundance" + ] + cr_params["e-dens"] = p_dens_ecl * (1 + 2 * a_abundance_ecl) else: - #if idx_read_b[-1]>=len(CR_list_b): raise Exception('L3d not generated: there is not enough L3b data to interpolate') - #if idx_read_c[-1]>=len(CR_list_c): raise Exception('L3d not generated: there is not enough L3c data to interpolate') - + # if idx_read_b[-1]>=len(CR_list_b): raise Exception('L3d not generated: there is not enough L3b data to interpolate') + # if idx_read_c[-1]>=len(CR_list_c): raise Exception('L3d not generated: there is not enough L3c data to interpolate') # mean time based on the light curves that were actually used during L3b processing (after removing bad-days and bad-seasons) - t_b=Time(np.array([data_l3b[i]['date'] for i in idx_read_b])) - t_c=Time(np.array([data_l3b[i]['date'] for i in idx_read_c])) + t_b = Time(np.array([data_l3b[i]["date"] for i in idx_read_b])) + t_c = Time(np.array([data_l3b[i]["date"] for i in idx_read_c])) # time nodes that will be used for interpolation. The first one is the last time stamp from the previous CR, the second and the third are next available L3bc dates # Usually t_b=t_c, but there could be a cases when for a current CR L3b was generated and L3c is not. Then we can still generate L3d if we have both L3b and L3c for later CR - t_nods_b=np.concatenate([[fun.jd_fm_Carrington(self.ini_data['CR_last'])],t_b.jd]) - t_nods_c=np.concatenate([[fun.jd_fm_Carrington(self.ini_data['CR_last'])],t_c.jd]) + t_nods_b = np.concatenate( + [[fun.jd_fm_Carrington(self.ini_data["CR_last"])], t_b.jd] + ) + t_nods_c = np.concatenate( + [[fun.jd_fm_Carrington(self.ini_data["CR_last"])], t_c.jd] + ) # UV-anisotropy - anisotropy=np.concatenate([[self.ini_data['data']['uv-anis'][-1]],np.array([data_l3b[i]['uv_anisotropy_factor'] for i in idx_read_b])]) - cr_params['uv-anis']=np.array([np.interp(t_CR.jd,t_nods_b,anisotropy[:,i]) for i in np.arange(len(anisotropy[0]))]) + anisotropy = np.concatenate( + [ + [self.ini_data["data"]["uv-anis"][-1]], + np.array([data_l3b[i]["uv_anisotropy_factor"] for i in idx_read_b]), + ] + ) + cr_params["uv-anis"] = np.array( + [ + np.interp(t_CR.jd, t_nods_b, anisotropy[:, i]) + for i in np.arange(len(anisotropy[0])) + ] + ) # photoion in the ecliptic plane - Nmid=int(len(data_l3b[0]['ion_rate_profile']['lat_grid'])/2) - ph_b=np.concatenate([[self.ini_data['data']['phion'][-1]],np.array([data_l3b[i]['ion_rate_profile']['ph_rate'][Nmid] for i in idx_read_b])]) - cr_params['phion']=np.interp(t_CR.jd,t_nods_b,ph_b) + Nmid = int(len(data_l3b[0]["ion_rate_profile"]["lat_grid"]) / 2) + ph_b = np.concatenate( + [ + [self.ini_data["data"]["phion"][-1]], + np.array( + [ + data_l3b[i]["ion_rate_profile"]["ph_rate"][Nmid] + for i in idx_read_b + ] + ), + ] + ) + cr_params["phion"] = np.interp(t_CR.jd, t_nods_b, ph_b) # solar wind parameters (plasma speed, proton density) - p_dens=np.concatenate([[self.ini_data['data']['p-dens'][-1]],np.array([data_l3c[i]['solar_wind_profile']['proton_density'] for i in idx_read_c])]) - sw_speed=np.concatenate([[self.ini_data['data']['speed'][-1]],np.array([data_l3c[i]['solar_wind_profile']['plasma_speed'] for i in idx_read_c])]) - cr_params['p-dens']=np.array([np.interp(t_CR.jd,t_nods_c,p_dens[:,i]) for i in np.arange(len(p_dens[0]))]) - cr_params['speed']=np.array([np.interp(t_CR.jd,t_nods_c,sw_speed[:,i]) for i in np.arange(len(sw_speed[0]))]) + p_dens = np.concatenate( + [ + [self.ini_data["data"]["p-dens"][-1]], + np.array( + [ + data_l3c[i]["solar_wind_profile"]["proton_density"] + for i in idx_read_c + ] + ), + ] + ) + sw_speed = np.concatenate( + [ + [self.ini_data["data"]["speed"][-1]], + np.array( + [ + data_l3c[i]["solar_wind_profile"]["plasma_speed"] + for i in idx_read_c + ] + ), + ] + ) + cr_params["p-dens"] = np.array( + [ + np.interp(t_CR.jd, t_nods_c, p_dens[:, i]) + for i in np.arange(len(p_dens[0])) + ] + ) + cr_params["speed"] = np.array( + [ + np.interp(t_CR.jd, t_nods_c, sw_speed[:, i]) + for i in np.arange(len(sw_speed[0])) + ] + ) # electron density in the ecliptic plane calculated from p-dens and alpha-abundance - p_dens_ecl=np.array([data_l3c[i]['solar_wind_ecliptic']['proton_density'] for i in idx_read_c]) - a_abundance_ecl=np.array([data_l3c[i]['solar_wind_ecliptic']['alpha_abundance'] for i in idx_read_c]) - e_dens_ecl=np.concatenate([[self.ini_data['data']['e-dens'][-1]],p_dens_ecl*(1+2*a_abundance_ecl)]) - cr_params['e-dens']=np.interp(t_CR.jd,t_nods_c,e_dens_ecl) + p_dens_ecl = np.array( + [ + data_l3c[i]["solar_wind_ecliptic"]["proton_density"] + for i in idx_read_c + ] + ) + a_abundance_ecl = np.array( + [ + data_l3c[i]["solar_wind_ecliptic"]["alpha_abundance"] + for i in idx_read_c + ] + ) + e_dens_ecl = np.concatenate( + [ + [self.ini_data["data"]["e-dens"][-1]], + p_dens_ecl * (1 + 2 * a_abundance_ecl), + ] + ) + cr_params["e-dens"] = np.interp(t_CR.jd, t_nods_c, e_dens_ecl) return cr_params, idx_read_b, idx_read_c - def generate_initial_history(self,fn_list): - ''' + def generate_initial_history(self, fn_list): + """ Reads text files with a time history. Usually it is the previous CR file Function fills ini_data structure that will be passed to the next text file - ''' - fill_value=self.settings['WawHelioIonGlows_fill_value'] + """ + fill_value = self.settings["WawHelioIonGlows_fill_value"] self._read_files(fn_list) self._check_ini_data() # Find file that has the longest history - k_max=max(self.ini_data['time'],key=lambda k: len(self.ini_data['time'][k])) + k_max = max(self.ini_data["time"], key=lambda k: len(self.ini_data["time"][k])) # Time grid is taken from the file with the logest history - self.time_grid=self.ini_data['time'][k_max] - self.CR_grid=[np.round(fun.carrington(t.jd),2) for t in self.time_grid] - self.ini_data['CR_last']=self.CR_grid[-1] + self.time_grid = self.ini_data["time"][k_max] + self.CR_grid = [np.round(fun.carrington(t.jd), 2) for t in self.time_grid] + self.ini_data["CR_last"] = self.CR_grid[-1] # Dimensions of the ionization parameters on common grid - Nt=len(self.time_grid) - Nl=len(self.lat_grid) + Nt = len(self.time_grid) + Nl = len(self.lat_grid) # Initialization of the ionization parameters on common grid. # At first all values are set at fill value defined in pipeline settings for f in fn_list: - if np.logical_or(np.logical_or(f=='speed',f=='p-dens'),f=='uv-anis'): - self.solar_params[f]=np.ones((Nt,Nl))*fill_value - self.flags[f]=np.ones(Nt)*fill_value - elif np.logical_or(np.logical_or(f=='lya',f=='phion'),f=='e-dens'): - self.solar_params[f]=np.ones(Nt)*fill_value + if np.logical_or( + np.logical_or(f == "speed", f == "p-dens"), f == "uv-anis" + ): + self.solar_params[f] = np.ones((Nt, Nl)) * fill_value + self.flags[f] = np.ones(Nt) * fill_value + elif np.logical_or(np.logical_or(f == "lya", f == "phion"), f == "e-dens"): + self.solar_params[f] = np.ones(Nt) * fill_value # Indexes where actual data starts - idx={} - for k,v in self.ini_data['time'].items(): idx[k]=np.abs(self.time_grid-v[0]).argmin() + idx = {} + for k, v in self.ini_data["time"].items(): + idx[k] = np.abs(self.time_grid - v[0]).argmin() # solar_params structure is filled by values from the ini_data on the uniform time grid - for f in fn_list: - self.solar_params[f][idx[f]:]=self.ini_data['data'][f] - + self.solar_params[f][idx[f] :] = self.ini_data["data"][f] # Flags are in the 3 ionization files - for k in ['speed', 'uv-anis', 'p-dens']: - self.flags[k][idx[k]:]=self.ini_data['flags'][k] + for k in ["speed", "uv-anis", "p-dens"]: + self.flags[k][idx[k] :] = self.ini_data["flags"][k] - self.glows_flags=np.zeros(Nt, dtype=np.uint16) + self.glows_flags = np.zeros(Nt, dtype=np.uint16) - def generate_hdr_txt(self,fn_dict): - ''' + def generate_hdr_txt(self, fn_dict): + """ Generates text header for a text L3d files from prievious L3d text files - ''' - hdr_dict={} + """ + hdr_dict = {} for f in fn_dict: - hdr=self._generate_hdr_template() - hdr_dict[f]=self._generate_hdr_txt_glows(hdr, self.ini_data['hdr_txt'][f],fn_dict[f]) + hdr = self._generate_hdr_template() + hdr_dict[f] = self._generate_hdr_txt_glows( + hdr, self.ini_data["hdr_txt"][f], fn_dict[f] + ) return hdr_dict - def _generate_hdr_txt_glows(self,hdr_temp,hdr_ini_str,fn_out): - ''' + def _generate_hdr_txt_glows(self, hdr_temp, hdr_ini_str, fn_out): + """ Copy some lines from the input header to the output header structure. As an input there is unfied header structure - ''' + """ - hdr_temp[1]='filename: ' + fn_out+'\n' - hdr_temp[2:4]=hdr_ini_str[2:4] - hdr_temp[16:29]=hdr_ini_str[16:29] + hdr_temp[1] = "filename: " + fn_out + "\n" + hdr_temp[2:4] = hdr_ini_str[2:4] + hdr_temp[16:29] = hdr_ini_str[16:29] return hdr_temp def _generate_hdr_template(self): - ''' + """ Generates unified header for the L3d text files - ''' + """ # Elements in the header that are common for all files - N=self.settings['hdr_txt_lines_number'] + N = self.settings["hdr_txt_lines_number"] # initialize uniform header structure - hdr_temp=N*['\n'] - hdr_temp[0]='lines in header: '+str(N)+'\n' - hdr_temp[4]='ground software version: '+VERSION+'\n' - hdr_temp[5]='creation date: '+ Time.now().iso +'\n' - hdr_temp[6]='input files: ' + self.header['l3d_cdf_filename'] + "\n" - hdr_temp[7]='external dependencies: ' + self.header['external_dependeciens'] + '\n' - hdr_temp[8]='last CR: '+ str(self.CR_last) + '\n' - hdr_temp[9]='fill value: '+ str(self.settings['WawHelioIonGlows_fill_value']) + '\n' - hdr_temp[15]='Remarks:\n' - hdr_temp[29]='###############################################' + hdr_temp = N * ["\n"] + hdr_temp[0] = "lines in header: " + str(N) + "\n" + hdr_temp[4] = "ground software version: " + VERSION + "\n" + hdr_temp[5] = "creation date: " + Time.now().iso + "\n" + hdr_temp[6] = "input files: " + self.header["l3d_cdf_filename"] + "\n" + hdr_temp[7] = ( + "external dependencies: " + self.header["external_dependeciens"] + "\n" + ) + hdr_temp[8] = "last CR: " + str(self.CR_last) + "\n" + hdr_temp[9] = ( + "fill value: " + str(self.settings["WawHelioIonGlows_fill_value"]) + "\n" + ) + hdr_temp[15] = "Remarks:\n" + hdr_temp[29] = "###############################################" return hdr_temp - - def _read_ecliptic(self,fn): - ''' + def _read_ecliptic(self, fn): + """ Reads L3d text files that are 1D value in the ecliptic plane - ''' - data=np.loadtxt(fn) + """ + data = np.loadtxt(fn) # time, value - t=Time(data[:,0],format='decimalyear') + t = Time(data[:, 0], format="decimalyear") - return t, data[:,1] + return t, data[:, 1] - - def _read_files(self,fn_list): - ''' + def _read_files(self, fn_list): + """ Reads L3d text files - ''' + """ # Read ionization files - N=self.settings['hdr_txt_lines_number'] + N = self.settings["hdr_txt_lines_number"] for f in fn_list: - self.ini_data['hdr_txt'][f]=fun.read_hdr_txt(fn_list[f],N) - if np.logical_or(np.logical_or(f=='speed',f=='p-dens'),f=='uv-anis'): - self.ini_data['time'][f], self.ini_data['data'][f], self.ini_data['flags'][f]=self._read_profile(fn_list[f]) - elif np.logical_or(np.logical_or(f=='lya',f=='phion'),f=='e-dens'): - self.ini_data['time'][f], self.ini_data['data'][f]=self._read_ecliptic(fn_list[f]) - - def _read_profile(self,fn): - ''' + self.ini_data["hdr_txt"][f] = fun.read_hdr_txt(fn_list[f], N) + if np.logical_or( + np.logical_or(f == "speed", f == "p-dens"), f == "uv-anis" + ): + ( + self.ini_data["time"][f], + self.ini_data["data"][f], + self.ini_data["flags"][f], + ) = self._read_profile(fn_list[f]) + elif np.logical_or(np.logical_or(f == "lya", f == "phion"), f == "e-dens"): + self.ini_data["time"][f], self.ini_data["data"][f] = ( + self._read_ecliptic(fn_list[f]) + ) + + def _read_profile(self, fn): + """ Reads L3d text files that are 2D profiles (on the time, latitude grid) - ''' - data=np.loadtxt(fn) + """ + data = np.loadtxt(fn) # time, value(-90), value(-80), ... value(0), ..., value(90), CR, flag - t=Time(data[:,0],format='decimalyear') - flag=data[:,-1] - param=data[:,1:-2] + t = Time(data[:, 0], format="decimalyear") + flag = data[:, -1] + param = data[:, 1:-2] - return t, param,flag + return t, param, flag - - def save_to_file(self,fn): - ''' + def save_to_file(self, fn): + """ Solar params history file is an official GLOWS L3d data product Temporarly it is in json format, but in production code it should be in CDF @@ -378,182 +488,231 @@ def save_to_file(self,fn): ------------ fn: str name of the output file - ''' - + """ - output={} - output['header']=self.header - output['lat_grid']=self.lat_grid - output['cr_grid']=self.CR_grid.tolist() - output['time_grid']=[t.iso for t in self.time_grid] - output['solar_params']={} - output['flags']={} - for k in self.ini_data['label']: output['solar_params'][k]=self.solar_params[k].tolist() - for k in ['speed','uv-anis','p-dens']: output['flags'][k]=self.flags[k].tolist() - output['glows_flags']=[int(v) for v in self.glows_flags] + output = {} + output["header"] = self.header + output["lat_grid"] = self.lat_grid + output["cr_grid"] = self.CR_grid.tolist() + output["time_grid"] = [t.iso for t in self.time_grid] + output["solar_params"] = {} + output["flags"] = {} + for k in self.ini_data["label"]: + output["solar_params"][k] = self.solar_params[k].tolist() + for k in ["speed", "uv-anis", "p-dens"]: + output["flags"][k] = self.flags[k].tolist() + output["glows_flags"] = [int(v) for v in self.glows_flags] json_content = json.dumps(output, indent=3) - output_fp = open(fn, 'w') + output_fp = open(fn, "w") print(json_content, file=output_fp) output_fp.close() - def save_to_txt(self,fn_out,hdr): - ''' + def save_to_txt(self, fn_out, hdr): + """ Write to the text file - ''' + """ for f in fn_out: - if np.logical_or(np.logical_or(f=='speed',f=='p-dens'),f=='uv-anis'): - self._save_to_txt_profile(fn_out[f],f,hdr[f]) - elif np.logical_or(np.logical_or(f=='lya',f=='phion'),f=='e-dens'): - self._save_to_txt_ecliptic(fn_out[f],f,hdr[f]) - - def _save_to_txt_ecliptic(self,fn,k,hdr): - ''' + if np.logical_or( + np.logical_or(f == "speed", f == "p-dens"), f == "uv-anis" + ): + self._save_to_txt_profile(fn_out[f], f, hdr[f]) + elif np.logical_or(np.logical_or(f == "lya", f == "phion"), f == "e-dens"): + self._save_to_txt_ecliptic(fn_out[f], f, hdr[f]) + + def _save_to_txt_ecliptic(self, fn, k, hdr): + """ Write to the text file parameters in the ecliptic plane (1D) - ''' - Ncol= 3 # number of columns in eliptic-type files (time, value, CR) - Nrow=len(self.solar_params[k]) - output = np.zeros((Nrow,Ncol)) + """ + Ncol = 3 # number of columns in eliptic-type files (time, value, CR) + Nrow = len(self.solar_params[k]) + output = np.zeros((Nrow, Ncol)) - output[:,0]=np.array([t.decimalyear for t in self.time_grid]) - output[:,1]=self.solar_params[k] - output[:,-1]=np.array(self.CR_grid) + output[:, 0] = np.array([t.decimalyear for t in self.time_grid]) + output[:, 1] = self.solar_params[k] + output[:, -1] = np.array(self.CR_grid) - np.savetxt(fn,output,fmt='%4.12f %.15e %4.2f',header=('').join(hdr)) + np.savetxt(fn, output, fmt="%4.12f %.15e %4.2f", header=("").join(hdr)) return 0 - def _save_to_txt_profile(self,fn,k,hdr): - ''' + def _save_to_txt_profile(self, fn, k, hdr): + """ Write to the text file parameters that are profiles on the latitudinal grid - ''' - Ncol=len(self.settings['solar_wind_grid'])+3 # number of columns in profile-type files (time, grid bins, CR) - Nrow=len(self.solar_params[k]) - output = np.zeros((Nrow,Ncol)) - - output[:,0]=np.array([t.decimalyear for t in self.time_grid]) - output[:,1:Ncol-2]=self.solar_params[k] - output[:,-2]=np.array(self.CR_grid) - output[:,-1]=np.array(self.flags[k]) - - np.savetxt(fn,output,fmt='%4.12f '+ (Ncol-3)*'%.15e ' + '%4.2f' + '%7d',header=('').join(hdr)) + """ + Ncol = ( + len(self.settings["solar_wind_grid"]) + 3 + ) # number of columns in profile-type files (time, grid bins, CR) + Nrow = len(self.solar_params[k]) + output = np.zeros((Nrow, Ncol)) + + output[:, 0] = np.array([t.decimalyear for t in self.time_grid]) + output[:, 1 : Ncol - 2] = self.solar_params[k] + output[:, -2] = np.array(self.CR_grid) + output[:, -1] = np.array(self.flags[k]) + + np.savetxt( + fn, + output, + fmt="%4.12f " + (Ncol - 3) * "%.15e " + "%4.2f" + "%7d", + header=("").join(hdr), + ) return 0 - - def _update_l3bc_data(self,data_l3b,data_l3c,CR): - ''' + def _update_l3bc_data(self, data_l3b, data_l3c, CR): + """ Generates new value for given CR of the solar parameters. - ''' - cr_params, idx_read_b, idx_read_c = self._generate_cr_solar_params(CR, data_l3b, data_l3c) + """ + cr_params, idx_read_b, idx_read_c = self._generate_cr_solar_params( + CR, data_l3b, data_l3c + ) - self.header['l3b_input_filename'].extend(data_l3b[i]['header']['filename'] for i in idx_read_b) - self.header['l3c_input_filename'].extend(data_l3c[i]['header']['filename'] for i in idx_read_c) + self.header["l3b_input_filename"].extend( + data_l3b[i]["header"]["filename"] for i in idx_read_b + ) + self.header["l3c_input_filename"].extend( + data_l3c[i]["header"]["filename"] for i in idx_read_c + ) # add a row with values for current CR - for k in self.ini_data['label']: - - if np.logical_or(np.logical_or(k=='speed',k=='p-dens'),k=='uv-anis'): - self.solar_params[k] = np.r_[self.solar_params[k],[cr_params[k]]] - elif np.logical_or(k=='phion',k=='e-dens'): + for k in self.ini_data["label"]: + if np.logical_or( + np.logical_or(k == "speed", k == "p-dens"), k == "uv-anis" + ): + self.solar_params[k] = np.r_[self.solar_params[k], [cr_params[k]]] + elif np.logical_or(k == "phion", k == "e-dens"): self.solar_params[k] = np.append(self.solar_params[k], cr_params[k]) # define source flags for p-dens and speed that are result of the GLOWS analysis # we have want to flag those values that are based on generated L3b and L3c, as well as those that are interpolated - if int(data_l3b[idx_read_b[0]]['CR'])==int(CR): + if int(data_l3b[idx_read_b[0]]["CR"]) == int(CR): # there is L3b for a current CR - self.flags['uv-anis']=np.append(self.flags['uv-anis'],data_l3b[idx_read_b[0]]['uv_anisotropy_flag']) - self.glows_flags=np.append(self.glows_flags,np.uint16(data_l3b[idx_read_b[0]]['glows_flags'])) + self.flags["uv-anis"] = np.append( + self.flags["uv-anis"], data_l3b[idx_read_b[0]]["uv_anisotropy_flag"] + ) + self.glows_flags = np.append( + self.glows_flags, np.uint16(data_l3b[idx_read_b[0]]["glows_flags"]) + ) else: # there is no L3b for a current CR - self.flags['uv-anis']=np.append(self.flags['uv-anis'],self.settings['l3d_source_flags']['uv_anis_interpolated']) - self.glows_flags=np.append(self.glows_flags,np.uint16(0)) + self.flags["uv-anis"] = np.append( + self.flags["uv-anis"], + self.settings["l3d_source_flags"]["uv_anis_interpolated"], + ) + self.glows_flags = np.append(self.glows_flags, np.uint16(0)) - if int(data_l3c[idx_read_c[0]]['CR'])==int(CR): + if int(data_l3c[idx_read_c[0]]["CR"]) == int(CR): # there is L3c for a current CR - self.flags['speed']=np.append(self.flags['speed'],self.settings['l3d_source_flags']['speed_GLOWS']) - self.flags['p-dens']=np.append(self.flags['p-dens'],self.settings['l3d_source_flags']['dens_from_speed']) + self.flags["speed"] = np.append( + self.flags["speed"], self.settings["l3d_source_flags"]["speed_GLOWS"] + ) + self.flags["p-dens"] = np.append( + self.flags["p-dens"], + self.settings["l3d_source_flags"]["dens_from_speed"], + ) else: # there is no L3c for a current CR - self.flags['speed']=np.append(self.flags['speed'],self.settings['l3d_source_flags']['speed_GLOWS_interpolated']) - self.flags['p-dens']=np.append(self.flags['p-dens'],self.settings['l3d_source_flags']['dens_from_speed_interpolated']) + self.flags["speed"] = np.append( + self.flags["speed"], + self.settings["l3d_source_flags"]["speed_GLOWS_interpolated"], + ) + self.flags["p-dens"] = np.append( + self.flags["p-dens"], + self.settings["l3d_source_flags"]["dens_from_speed_interpolated"], + ) - - def _freeze_l3bc_data(self,data_l3b,data_l3c): + def _freeze_l3bc_data(self, data_l3b, data_l3c): # add a row with values copied from the last CR generated from the GLOWS data - for k in self.ini_data['label']: - - if np.logical_or(np.logical_or(k=='speed',k=='p-dens'),k=='uv-anis'): - self.solar_params[k] = np.r_[self.solar_params[k],[self.solar_params[k][-1]]] - - elif np.logical_or(k=='phion',k=='e-dens'): - self.solar_params[k] = np.append(self.solar_params[k], self.solar_params[k][-1]) + for k in self.ini_data["label"]: + if np.logical_or( + np.logical_or(k == "speed", k == "p-dens"), k == "uv-anis" + ): + self.solar_params[k] = np.r_[ + self.solar_params[k], [self.solar_params[k][-1]] + ] + + elif np.logical_or(k == "phion", k == "e-dens"): + self.solar_params[k] = np.append( + self.solar_params[k], self.solar_params[k][-1] + ) # add flags - self.flags['uv-anis']=np.append(self.flags['uv-anis'],self.settings['l3d_source_flags']['freezed']) - self.flags['speed']=np.append(self.flags['speed'],self.settings['l3d_source_flags']['freezed']) - self.flags['p-dens']=np.append(self.flags['p-dens'],self.settings['l3d_source_flags']['freezed']) - self.glows_flags=np.append(self.glows_flags, self.glows_flags[-1] | GlowsL3Flags.PERSISTED_LAST_POINT) - + self.flags["uv-anis"] = np.append( + self.flags["uv-anis"], self.settings["l3d_source_flags"]["freezed"] + ) + self.flags["speed"] = np.append( + self.flags["speed"], self.settings["l3d_source_flags"]["freezed"] + ) + self.flags["p-dens"] = np.append( + self.flags["p-dens"], self.settings["l3d_source_flags"]["freezed"] + ) + self.glows_flags = np.append( + self.glows_flags, self.glows_flags[-1] | GlowsL3Flags.PERSISTED_LAST_POINT + ) def _freeze_lya_data(self): - self.solar_params['lya'] = np.append(self.solar_params['lya'], self.solar_params['lya'][-1]) + self.solar_params["lya"] = np.append( + self.solar_params["lya"], self.solar_params["lya"][-1] + ) - def _update_lya_data(self,ext_dependencies,CR): - ''' + def _update_lya_data(self, ext_dependencies, CR): + """ Generates next value of the Lyman-alpha irradiance based on external daily measurements. - ''' + """ - data_lya = fun.read_lya_data(ext_dependencies['lya_raw_data']) + data_lya = fun.read_lya_data(ext_dependencies["lya_raw_data"]) lya_CR = fun.generate_cr_lya(CR, data_lya) - self.solar_params['lya'] = np.append(self.solar_params['lya'], lya_CR) + self.solar_params["lya"] = np.append(self.solar_params["lya"], lya_CR) - def update_solar_params_hist(self,ext_dependencies,data_l3b,data_l3c,CR_current): - ''' + def update_solar_params_hist( + self, ext_dependencies, data_l3b, data_l3c, CR_current + ): + """ Updates structure by calculating parameters values from last CR in the initial data until currently processed CR - ''' - CR_l3b_last=np.round(fun.carrington(Time(data_l3b[-1]['date']).jd),2) + """ + CR_l3b_last = np.round(fun.carrington(Time(data_l3b[-1]["date"]).jd), 2) - #for CR in np.arange(self.ini_data['CR_last']+1,CR_current+1.5): - CR=self.ini_data['CR_last'] + 1 + # for CR in np.arange(self.ini_data['CR_last']+1,CR_current+1.5): + CR = self.ini_data["CR_last"] + 1 - while(int(CR) GlowsL3DSolarParamsHistory: - with open(json_file_path, 'r') as json_file: +def convert_json_to_l3d_data_product( + json_file_path: Path, input_metadata: InputMetadata, parent_file_names: list[str] +) -> GlowsL3DSolarParamsHistory: + with open(json_file_path, "r") as json_file: l3d_json_dict = json.load(json_file) - new_start = datetime.fromisoformat(l3d_json_dict['time_grid'][0]) + new_start = datetime.fromisoformat(l3d_json_dict["time_grid"][0]) input_metadata.start_date = new_start.date() return GlowsL3DSolarParamsHistory( input_metadata=input_metadata, parent_file_names=parent_file_names, - latitude=l3d_json_dict['lat_grid'], - cr=l3d_json_dict['cr_grid'], - epoch=np.array([datetime.fromisoformat(time) for time in l3d_json_dict['time_grid']]), - plasma_speed=l3d_json_dict['solar_params']['speed'], - proton_density=l3d_json_dict['solar_params']['p-dens'], - ultraviolet_anisotropy=l3d_json_dict['solar_params']['uv-anis'], - phion=l3d_json_dict['solar_params']['phion'], - lyman_alpha=l3d_json_dict['solar_params']['lya'], - electron_density=l3d_json_dict['solar_params']['e-dens'], - plasma_speed_flag=l3d_json_dict['flags']['speed'], - uv_anisotropy_flag=l3d_json_dict['flags']['uv-anis'], - proton_density_flag=l3d_json_dict['flags']['p-dens'], - glows_flags=np.array(l3d_json_dict['glows_flags'], dtype=np.uint16), + latitude=l3d_json_dict["lat_grid"], + cr=l3d_json_dict["cr_grid"], + epoch=np.array( + [datetime.fromisoformat(time) for time in l3d_json_dict["time_grid"]] + ), + plasma_speed=l3d_json_dict["solar_params"]["speed"], + proton_density=l3d_json_dict["solar_params"]["p-dens"], + ultraviolet_anisotropy=l3d_json_dict["solar_params"]["uv-anis"], + phion=l3d_json_dict["solar_params"]["phion"], + lyman_alpha=l3d_json_dict["solar_params"]["lya"], + electron_density=l3d_json_dict["solar_params"]["e-dens"], + plasma_speed_flag=l3d_json_dict["flags"]["speed"], + uv_anisotropy_flag=l3d_json_dict["flags"]["uv-anis"], + proton_density_flag=l3d_json_dict["flags"]["p-dens"], + glows_flags=np.array(l3d_json_dict["glows_flags"], dtype=np.uint16), ) @@ -107,47 +120,65 @@ def get_parent_file_names_from_l3d_json(l3d_folder: Path) -> list[str]: with open(l3d_folder / l3d_file_paths[0]) as l3d: l3d_data = json.load(l3d) - ancillary_files = [Path(file).name for file in l3d_data['header']['ancillary_data_files']] - external_dependencies = Path(l3d_data['header']['external_dependeciens']).name + ancillary_files = [ + Path(file).name for file in l3d_data["header"]["ancillary_data_files"] + ] + external_dependencies = Path(l3d_data["header"]["external_dependeciens"]).name parent_file_names.update(ancillary_files) parent_file_names.add(external_dependencies) for file_path in l3d_file_paths: - with open(l3d_folder / file_path, 'r') as l3d: + with open(l3d_folder / file_path, "r") as l3d: l3d_data = json.load(l3d) - l3b_parents = [Path(file).name for file in l3d_data['header']['l3b_input_filename']] - l3c_parents = [Path(file).name for file in l3d_data['header']['l3c_input_filename']] + l3b_parents = [ + Path(file).name for file in l3d_data["header"]["l3b_input_filename"] + ] + l3c_parents = [ + Path(file).name for file in l3d_data["header"]["l3c_input_filename"] + ] parent_file_names.update(l3b_parents) parent_file_names.update(l3c_parents) return list(parent_file_names) + def get_most_recently_uploaded_ancillary(query_result: list[dict]) -> dict: - query_result = max(query_result, key=lambda x: x['ingestion_date'], default=None) + query_result = max(query_result, key=lambda x: x["ingestion_date"], default=None) return query_result + def rename_l3d_text_outputs(paths: list[Path], version: str) -> list[Path]: out_paths = [] for path in paths: - filename_without_extension, extension = path.name.split('.') - original_name_components = filename_without_extension.split('_') + filename_without_extension, extension = path.name.split(".") + original_name_components = filename_without_extension.split("_") descriptor = original_name_components[3] start_date_with_cr = original_name_components[4] - start_date, cr = start_date_with_cr.split('-cr') + start_date, cr = start_date_with_cr.split("-cr") midpoint_of_cr = get_midpoint_of_cr(int(cr)) - new_path = path.parent / f"imap_glows_{descriptor}_{start_date}_{midpoint_of_cr.strftime('%Y%m%d')}_{version}.{extension}" + new_path = ( + path.parent + / f"imap_glows_{descriptor}_{start_date}_{midpoint_of_cr.strftime('%Y%m%d')}_{version}.{extension}" + ) os.rename(path, new_path) out_paths.append(new_path) return out_paths def query_for_most_recent_l3d(descriptor: str) -> Optional[dict]: - query_result = imap_data_access.query(instrument="glows", data_level="l3d", descriptor=descriptor) + query_result = imap_data_access.query( + instrument="glows", data_level="l3d", descriptor=descriptor + ) sorted_query_result = sorted( query_result, key=lambda qr: ( qr["cr"], - (qr["minor_version"] if "minor_version" in qr else Version.from_version(qr["version"]).minor) + ( + qr["minor_version"] + if "minor_version" in qr + else Version.from_version(qr["version"]).minor + ), ), - reverse=True) + reverse=True, + ) return next(iter(sorted_query_result), None) diff --git a/imap_l3_processing/glows/l3e/glows_l3e_call_arguments.py b/imap_l3_processing/glows/l3e/glows_l3e_call_arguments.py index 69254d2f8..3d108a828 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_call_arguments.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_call_arguments.py @@ -24,6 +24,7 @@ def to_argument_list(self): f"{self.spin_axis_latitude:.4f}", ] + @dataclass class GlowsL3eCallArguments: formatted_date: str @@ -36,5 +37,5 @@ def to_argument_list(self): self.formatted_date, self.decimal_date, *self.spacecraft_info.to_argument_list(), - f"{self.elongation:.3f}" - ] \ No newline at end of file + f"{self.elongation:.3f}", + ] diff --git a/imap_l3_processing/glows/l3e/glows_l3e_dependencies.py b/imap_l3_processing/glows/l3e/glows_l3e_dependencies.py index f00ea77b7..558bb4221 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_dependencies.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_dependencies.py @@ -23,6 +23,7 @@ SpiceKernelTypes.SpacecraftClock, ] + @dataclass class GlowsL3EDependencies: energy_grid_lo: Path @@ -42,42 +43,76 @@ class GlowsL3EDependencies: @classmethod def fetch_dependencies(cls, dependencies: ProcessingInputCollection): - lya_series_dependency = dependencies.get_file_paths(source='glows', descriptor='lya') - solar_uv_anisotropy_dependency = dependencies.get_file_paths(source='glows', descriptor='uv-anis') - speed_3d_dependency = dependencies.get_file_paths(source='glows', descriptor='speed') - density_3d_dependency = dependencies.get_file_paths(source='glows', descriptor='p-dens') - phion_hydrogen_dependency = dependencies.get_file_paths(source='glows', descriptor='phion') - sw_eqtr_electrons_dependency = dependencies.get_file_paths(source='glows', descriptor='e-dens') + lya_series_dependency = dependencies.get_file_paths( + source="glows", descriptor="lya" + ) + solar_uv_anisotropy_dependency = dependencies.get_file_paths( + source="glows", descriptor="uv-anis" + ) + speed_3d_dependency = dependencies.get_file_paths( + source="glows", descriptor="speed" + ) + density_3d_dependency = dependencies.get_file_paths( + source="glows", descriptor="p-dens" + ) + phion_hydrogen_dependency = dependencies.get_file_paths( + source="glows", descriptor="phion" + ) + sw_eqtr_electrons_dependency = dependencies.get_file_paths( + source="glows", descriptor="e-dens" + ) - pipeline_settings_dependency = dependencies.get_file_paths(source='glows', descriptor='pipeline-settings-l3bcde') + pipeline_settings_dependency = dependencies.get_file_paths( + source="glows", descriptor="pipeline-settings-l3bcde" + ) - tess_xyz_dependency = dependencies.get_file_paths(source='glows', descriptor='tess-xyz-8') - tess_ang_dependency = dependencies.get_file_paths(source='glows', descriptor='tess-ang-16') + tess_xyz_dependency = dependencies.get_file_paths( + source="glows", descriptor="tess-xyz-8" + ) + tess_ang_dependency = dependencies.get_file_paths( + source="glows", descriptor="tess-ang-16" + ) - energy_grid_lo_dependency = dependencies.get_file_paths(source='glows', descriptor='energy-grid-lo') - energy_grid_hi_dependency = dependencies.get_file_paths(source='glows', descriptor='energy-grid-hi') - energy_grid_ultra_dependency = dependencies.get_file_paths(source='glows', descriptor='energy-grid-ultra') + energy_grid_lo_dependency = dependencies.get_file_paths( + source="glows", descriptor="energy-grid-lo" + ) + energy_grid_hi_dependency = dependencies.get_file_paths( + source="glows", descriptor="energy-grid-hi" + ) + energy_grid_ultra_dependency = dependencies.get_file_paths( + source="glows", descriptor="energy-grid-ultra" + ) lya_series_path = imap_data_access.download(lya_series_dependency[0]) - solar_uv_anisotropy_path = imap_data_access.download(solar_uv_anisotropy_dependency[0]) + solar_uv_anisotropy_path = imap_data_access.download( + solar_uv_anisotropy_dependency[0] + ) speed_3d_path = imap_data_access.download(speed_3d_dependency[0]) density_3d_path = imap_data_access.download(density_3d_dependency[0]) phion_hydrogen_path = imap_data_access.download(phion_hydrogen_dependency[0]) - sw_eqtr_electrons_path = imap_data_access.download(sw_eqtr_electrons_dependency[0]) + sw_eqtr_electrons_path = imap_data_access.download( + sw_eqtr_electrons_dependency[0] + ) - pipeline_settings_path = imap_data_access.download(pipeline_settings_dependency[0]) + pipeline_settings_path = imap_data_access.download( + pipeline_settings_dependency[0] + ) tess_xyz_path = imap_data_access.download(tess_xyz_dependency[0]) tess_ang_path = imap_data_access.download(tess_ang_dependency[0]) energy_grid_lo_path = imap_data_access.download(energy_grid_lo_dependency[0]) energy_grid_hi_path = imap_data_access.download(energy_grid_hi_dependency[0]) - energy_grid_ultra_path = imap_data_access.download(energy_grid_ultra_dependency[0]) + energy_grid_ultra_path = imap_data_access.download( + energy_grid_ultra_dependency[0] + ) with open(pipeline_settings_path) as f: pipeline_settings = json.load(f) - repoint_file_dependency = dependencies.get_file_paths(data_type=RepointInput.data_type) + repoint_file_dependency = dependencies.get_file_paths( + data_type=RepointInput.data_type + ) repoint_file_path = imap_data_access.download(repoint_file_dependency[0]) return cls( @@ -99,26 +134,60 @@ def fetch_dependencies(cls, dependencies: ProcessingInputCollection): @staticmethod def collect_spice_dependencies(start_date: datetime, end_date: datetime): - logger.info(f"Querying for SPICE data over the range: {start_date} to {end_date}") + logger.info( + f"Querying for SPICE data over the range: {start_date} to {end_date}" + ) kernel_types_with_predicted = GLOWS_L3E_REQUIRED_SPICE_KERNELS - kernel_types_without_predicted = [kernel for kernel in GLOWS_L3E_REQUIRED_SPICE_KERNELS if kernel != SpiceKernelTypes.EphemerisPredicted] + kernel_types_without_predicted = [ + kernel + for kernel in GLOWS_L3E_REQUIRED_SPICE_KERNELS + if kernel != SpiceKernelTypes.EphemerisPredicted + ] - return (furnish_spice_metakernel(start_date=start_date, end_date=end_date, kernel_types=kernel_types_with_predicted, metakernel_file_name="metakernel_with_predict_ephem.txt"), - furnish_spice_metakernel(start_date=start_date, end_date=end_date, kernel_types=kernel_types_without_predicted, metakernel_file_name="metakernel_without_predict_ephem.txt")) + return ( + furnish_spice_metakernel( + start_date=start_date, + end_date=end_date, + kernel_types=kernel_types_with_predicted, + metakernel_file_name="metakernel_with_predict_ephem.txt", + ), + furnish_spice_metakernel( + start_date=start_date, + end_date=end_date, + kernel_types=kernel_types_without_predicted, + metakernel_file_name="metakernel_without_predict_ephem.txt", + ), + ) def copy_dependencies(self): if self.energy_grid_lo is not None: - shutil.copy(self.energy_grid_lo, self.pipeline_settings['executable_dependency_paths']['energy-grid-lo']) + shutil.copy( + self.energy_grid_lo, + self.pipeline_settings["executable_dependency_paths"]["energy-grid-lo"], + ) if self.energy_grid_hi is not None: - shutil.copy(self.energy_grid_hi, self.pipeline_settings['executable_dependency_paths']['energy-grid-hi']) + shutil.copy( + self.energy_grid_hi, + self.pipeline_settings["executable_dependency_paths"]["energy-grid-hi"], + ) if self.energy_grid_ultra is not None: - shutil.copy(self.energy_grid_ultra, - self.pipeline_settings['executable_dependency_paths']['energy-grid-ultra']) + shutil.copy( + self.energy_grid_ultra, + self.pipeline_settings["executable_dependency_paths"][ + "energy-grid-ultra" + ], + ) if self.tess_xyz_8 is not None: - shutil.copy(self.tess_xyz_8, self.pipeline_settings['executable_dependency_paths']['tess-xyz-8']) + shutil.copy( + self.tess_xyz_8, + self.pipeline_settings["executable_dependency_paths"]["tess-xyz-8"], + ) if self.tess_ang16 is not None: - shutil.copy(self.tess_ang16, self.pipeline_settings['executable_dependency_paths']['tess-ang-16']) + shutil.copy( + self.tess_ang16, + self.pipeline_settings["executable_dependency_paths"]["tess-ang-16"], + ) shutil.copy(self.lya_series, self.lya_series.name) shutil.copy(self.solar_uv_anisotropy, self.solar_uv_anisotropy.name) shutil.copy(self.speed_3d_sw, self.speed_3d_sw.name) @@ -126,15 +195,20 @@ def copy_dependencies(self): shutil.copy(self.phion_hydrogen, self.phion_hydrogen.name) shutil.copy(self.sw_eqtr_electrons, self.sw_eqtr_electrons.name) - with open("ionization.files.dat", 'w') as ionization_file: - ionization_file.write("\n".join([ - self.lya_series.name, - self.solar_uv_anisotropy.name, - self.speed_3d_sw.name, - self.density_3d_sw.name, - self.phion_hydrogen.name, - self.sw_eqtr_electrons.name, - ]) + "\n") + with open("ionization.files.dat", "w") as ionization_file: + ionization_file.write( + "\n".join( + [ + self.lya_series.name, + self.solar_uv_anisotropy.name, + self.speed_3d_sw.name, + self.density_3d_sw.name, + self.phion_hydrogen.name, + self.sw_eqtr_electrons.name, + ] + ) + + "\n" + ) def get_hi_parents(self): return [ diff --git a/imap_l3_processing/glows/l3e/glows_l3e_hi_model.py b/imap_l3_processing/glows/l3e/glows_l3e_hi_model.py index cc4c31062..fd0fca959 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_hi_model.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_hi_model.py @@ -54,45 +54,66 @@ class GlowsL3EHiData(DataProduct): glows_flags: np.ndarray @classmethod - def convert_dat_to_glows_l3e_hi_product(cls, input_metadata: InputMetadata, file_path: Path, - epoch: datetime, - epoch_delta: timedelta, - args: GlowsL3eCallArguments): + def convert_dat_to_glows_l3e_hi_product( + cls, + input_metadata: InputMetadata, + file_path: Path, + epoch: datetime, + epoch_delta: timedelta, + args: GlowsL3eCallArguments, + ): with open(file_path) as input_data: lines = input_data.readlines() energy_line = [line for line in lines if line.startswith("#energy_grid")] - energies = np.array([float(i) for i in re.findall(r"\d+.\d+", energy_line[0])]) + energies = np.array( + [float(i) for i in re.findall(r"\d+.\d+", energy_line[0])] + ) - code_version_line = [line for line in lines if line.startswith("# code version")] - code_version = code_version_line[0].split(',')[0][14:].strip() + code_version_line = [ + line for line in lines if line.startswith("# code version") + ] + code_version = code_version_line[0].split(",")[0][14:].strip() - spin_angle_and_survival_probabilities = np.loadtxt(file_path, skiprows=200, dtype=np.float64) + spin_angle_and_survival_probabilities = np.loadtxt( + file_path, skiprows=200, dtype=np.float64 + ) spin_angles = spin_angle_and_survival_probabilities[:, 0] - survival_probabilities = np.array([spin_angle_and_survival_probabilities[:, 1:].T]) + survival_probabilities = np.array( + [spin_angle_and_survival_probabilities[:, 1:].T] + ) energy_delta_plus, energy_delta_minus = calculate_energy_deltas(energies) - return cls(input_metadata, - epoch=np.array([epoch]), - epoch_delta=np.array([epoch_delta.total_seconds() * ONE_SECOND_IN_NANOSECONDS]), - energy=energies, - energy_delta_plus=energy_delta_plus, - energy_delta_minus=energy_delta_minus, - spin_angle=spin_angles, - probability_of_survival=survival_probabilities, - spin_axis_lat=np.array([args.spacecraft_info.spin_axis_latitude]), - spin_axis_lon=np.array([args.spacecraft_info.spin_axis_longitude]), - program_version=np.array([code_version]), - spacecraft_radius=np.array([args.spacecraft_info.spacecraft_radius]), - spacecraft_longitude=np.array([args.spacecraft_info.spacecraft_longitude]), - spacecraft_latitude=np.array([args.spacecraft_info.spacecraft_latitude]), - spacecraft_velocity_x=np.array([args.spacecraft_info.spacecraft_velocity_x]), - spacecraft_velocity_y=np.array([args.spacecraft_info.spacecraft_velocity_y]), - spacecraft_velocity_z=np.array([args.spacecraft_info.spacecraft_velocity_z]), - elongation=np.array([args.elongation]), - glows_flags=np.array([0], dtype=np.uint16), - ) + return cls( + input_metadata, + epoch=np.array([epoch]), + epoch_delta=np.array( + [epoch_delta.total_seconds() * ONE_SECOND_IN_NANOSECONDS] + ), + energy=energies, + energy_delta_plus=energy_delta_plus, + energy_delta_minus=energy_delta_minus, + spin_angle=spin_angles, + probability_of_survival=survival_probabilities, + spin_axis_lat=np.array([args.spacecraft_info.spin_axis_latitude]), + spin_axis_lon=np.array([args.spacecraft_info.spin_axis_longitude]), + program_version=np.array([code_version]), + spacecraft_radius=np.array([args.spacecraft_info.spacecraft_radius]), + spacecraft_longitude=np.array([args.spacecraft_info.spacecraft_longitude]), + spacecraft_latitude=np.array([args.spacecraft_info.spacecraft_latitude]), + spacecraft_velocity_x=np.array( + [args.spacecraft_info.spacecraft_velocity_x] + ), + spacecraft_velocity_y=np.array( + [args.spacecraft_info.spacecraft_velocity_y] + ), + spacecraft_velocity_z=np.array( + [args.spacecraft_info.spacecraft_velocity_z] + ), + elongation=np.array([args.elongation]), + glows_flags=np.array([0], dtype=np.uint16), + ) def to_data_product_variables(self) -> list[DataProductVariable]: spin_angle_labels = [f"{i:.0f}" for i in self.spin_angle] @@ -104,7 +125,9 @@ def to_data_product_variables(self) -> list[DataProductVariable]: DataProductVariable(ENERGY_DELTA_PLUS_VAR_NAME, self.energy_delta_plus), DataProductVariable(ENERGY_DELTA_MINUS_VAR_NAME, self.energy_delta_minus), DataProductVariable(SPIN_ANGLE_VAR_NAME, self.spin_angle), - DataProductVariable(PROBABILITY_OF_SURVIVAL_VAR_NAME, self.probability_of_survival), + DataProductVariable( + PROBABILITY_OF_SURVIVAL_VAR_NAME, self.probability_of_survival + ), DataProductVariable(ENERGY_LABEL_VAR_NAME, energy_labels), DataProductVariable(SPIN_ANGLE_LABEL_VAR_NAME, spin_angle_labels), DataProductVariable(SPIN_AXIS_LATITUDE_VAR_NAME, self.spin_axis_lat), @@ -112,10 +135,18 @@ def to_data_product_variables(self) -> list[DataProductVariable]: DataProductVariable(PROGRAM_VERSION_VAR_NAME, self.program_version), DataProductVariable(SPACECRAFT_RADIUS_VAR_NAME, self.spacecraft_radius), DataProductVariable(SPACECRAFT_LATITUDE_VAR_NAME, self.spacecraft_latitude), - DataProductVariable(SPACECRAFT_LONGITUDE_VAR_NAME, self.spacecraft_longitude), - DataProductVariable(SPACECRAFT_VELOCITY_X_VAR_NAME, self.spacecraft_velocity_x), - DataProductVariable(SPACECRAFT_VELOCITY_Y_VAR_NAME, self.spacecraft_velocity_y), - DataProductVariable(SPACECRAFT_VELOCITY_Z_VAR_NAME, self.spacecraft_velocity_z), + DataProductVariable( + SPACECRAFT_LONGITUDE_VAR_NAME, self.spacecraft_longitude + ), + DataProductVariable( + SPACECRAFT_VELOCITY_X_VAR_NAME, self.spacecraft_velocity_x + ), + DataProductVariable( + SPACECRAFT_VELOCITY_Y_VAR_NAME, self.spacecraft_velocity_y + ), + DataProductVariable( + SPACECRAFT_VELOCITY_Z_VAR_NAME, self.spacecraft_velocity_z + ), DataProductVariable(ELONGATION, self.elongation), DataProductVariable(GLOWS_FLAGS_VAR_NAME, self.glows_flags), ] diff --git a/imap_l3_processing/glows/l3e/glows_l3e_initializer.py b/imap_l3_processing/glows/l3e/glows_l3e_initializer.py index 0706ab508..37ff090bb 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_initializer.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_initializer.py @@ -14,23 +14,35 @@ RepointInput, ) from imap_data_access.file_validation import Version -from imap_processing.spice.repoint import set_global_repoint_table_paths, get_repoint_data +from imap_processing.spice.repoint import ( + set_global_repoint_table_paths, + get_repoint_data, +) from spacepy.pycdf import CDF -from imap_l3_processing.glows.descriptors import GLOWS_L3E_DESCRIPTORS, GLOWS_L3E_HI_90_DESCRIPTOR, \ - GLOWS_L3E_HI_45_DESCRIPTOR, GLOWS_L3E_LO_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, GLOWS_L3E_ULTRA_HF_DESCRIPTOR +from imap_l3_processing.glows.descriptors import ( + GLOWS_L3E_DESCRIPTORS, + GLOWS_L3E_HI_90_DESCRIPTOR, + GLOWS_L3E_HI_45_DESCRIPTOR, + GLOWS_L3E_LO_DESCRIPTOR, + GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + GLOWS_L3E_ULTRA_HF_DESCRIPTOR, +) from imap_l3_processing.glows.l3bc.utils import get_pointing_date_range from imap_l3_processing.glows.l3d.models import GlowsL3DProcessorOutput from imap_l3_processing.glows.l3d.utils import get_most_recently_uploaded_ancillary from imap_l3_processing.glows.l3e.glows_l3e_dependencies import GlowsL3EDependencies from imap_l3_processing.glows.l3e.glows_l3e_utils import ( - GlowsL3eVersionsForRepointings, get_repoint_numbers_within_cr_window, ) + GlowsL3eVersionsForRepointings, + get_repoint_numbers_within_cr_window, +) from imap_l3_processing.glows.l3e.reprocess_info import ReprocessInfo from imap_l3_processing.models import VersionMap from imap_l3_processing.utils import FurnishMetakernelOutput logger = logging.getLogger(__name__) + @dataclass class GlowsL3EInitializerOutput: dependencies: GlowsL3EDependencies @@ -49,12 +61,38 @@ def get_repointings_to_process( version_map: VersionMap, reprocess_info: ReprocessInfo, ) -> Optional[GlowsL3EInitializerOutput]: - pipeline_settings_l3bcde = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='pipeline-settings-l3bcde')) - energy_grid_lo = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='energy-grid-lo')) - tess_xyz_8 = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='tess-xyz-8')) - energy_grid_hi = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='energy-grid-hi')) - energy_grid_ultra = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='energy-grid-ultra')) - tess_ang_16 = get_most_recently_uploaded_ancillary(imap_data_access.query(table='ancillary', instrument='glows', descriptor='tess-ang-16')) + pipeline_settings_l3bcde = get_most_recently_uploaded_ancillary( + imap_data_access.query( + table="ancillary", + instrument="glows", + descriptor="pipeline-settings-l3bcde", + ) + ) + energy_grid_lo = get_most_recently_uploaded_ancillary( + imap_data_access.query( + table="ancillary", instrument="glows", descriptor="energy-grid-lo" + ) + ) + tess_xyz_8 = get_most_recently_uploaded_ancillary( + imap_data_access.query( + table="ancillary", instrument="glows", descriptor="tess-xyz-8" + ) + ) + energy_grid_hi = get_most_recently_uploaded_ancillary( + imap_data_access.query( + table="ancillary", instrument="glows", descriptor="energy-grid-hi" + ) + ) + energy_grid_ultra = get_most_recently_uploaded_ancillary( + imap_data_access.query( + table="ancillary", instrument="glows", descriptor="energy-grid-ultra" + ) + ) + tess_ang_16 = get_most_recently_uploaded_ancillary( + imap_data_access.query( + table="ancillary", instrument="glows", descriptor="tess-ang-16" + ) + ) processing_input_collection = ProcessingInputCollection( ScienceInput(l3d_output.l3d_cdf_file_path.name), @@ -65,7 +103,7 @@ def get_repointings_to_process( AncillaryInput(str(energy_grid_hi["file_path"])), AncillaryInput(str(energy_grid_ultra["file_path"])), AncillaryInput(str(tess_ang_16["file_path"])), - RepointInput(str(repointing_file_path)) + RepointInput(str(repointing_file_path)), ) l3e_deps = GlowsL3EDependencies.fetch_dependencies(processing_input_collection) @@ -75,7 +113,9 @@ def get_repointings_to_process( first_updated_cr = first_cr if previous_l3d is not None: - first_updated_cr = find_first_updated_cr(l3d_output.l3d_cdf_file_path, previous_l3d) + first_updated_cr = find_first_updated_cr( + l3d_output.l3d_cdf_file_path, previous_l3d + ) if first_updated_cr is not None: first_updated_cr -= 1 @@ -111,26 +151,44 @@ def get_repointings_to_process( metakernel_without_predict_ephem=furnished_metakernels[1], ) + def query_existing_l3es_versions(descriptor: str) -> dict[int, Version]: existing_l3es_for_descriptor = {} - l3e_files = imap_data_access.query(instrument='glows', data_level='l3e', version="latest", descriptor=descriptor) + l3e_files = imap_data_access.query( + instrument="glows", data_level="l3e", version="latest", descriptor=descriptor + ) for l3e in l3e_files: - if 'major_version' in l3e.keys() and 'minor_version' in l3e.keys(): - existing_l3es_for_descriptor[int(l3e['repointing'])] = Version(l3e["major_version"], l3e["minor_version"]) - elif 'version' in l3e.keys(): - existing_l3es_for_descriptor[int(l3e['repointing'])] = Version.from_version(l3e['version']) + if "major_version" in l3e.keys() and "minor_version" in l3e.keys(): + existing_l3es_for_descriptor[int(l3e["repointing"])] = Version( + l3e["major_version"], l3e["minor_version"] + ) + elif "version" in l3e.keys(): + existing_l3es_for_descriptor[int(l3e["repointing"])] = Version.from_version( + l3e["version"] + ) else: continue return existing_l3es_for_descriptor -def identify_versions_for_l3e_output_files(start_cr_of_mission: int, end_cr_of_mission: int, first_updated_cr_from_l3d: Optional[int], - repointing_path: Path, version_map: VersionMap, reprocess_info: ReprocessInfo) -> GlowsL3eVersionsForRepointings: + +def identify_versions_for_l3e_output_files( + start_cr_of_mission: int, + end_cr_of_mission: int, + first_updated_cr_from_l3d: Optional[int], + repointing_path: Path, + version_map: VersionMap, + reprocess_info: ReprocessInfo, +) -> GlowsL3eVersionsForRepointings: set_global_repoint_table_paths([repointing_path]) repointing_data = get_repoint_data() - all_pointing_numbers = get_repoint_numbers_within_cr_window(start_cr_of_mission, end_cr_of_mission, repointing_data) - pointing_numbers_updated_by_l3d = get_repoint_numbers_within_cr_window(first_updated_cr_from_l3d, end_cr_of_mission, repointing_data) + all_pointing_numbers = get_repoint_numbers_within_cr_window( + start_cr_of_mission, end_cr_of_mission, repointing_data + ) + pointing_numbers_updated_by_l3d = get_repoint_numbers_within_cr_window( + first_updated_cr_from_l3d, end_cr_of_mission, repointing_data + ) updated_pointings_per_instruments = {} updated_pointing_numbers = {} @@ -143,9 +201,15 @@ def identify_versions_for_l3e_output_files(start_cr_of_mission: int, end_cr_of_m ) existing_file_versions = query_existing_l3es_versions(descriptor) - out_of_date_repointings = find_out_of_date_l3es(existing_file_versions, all_pointing_numbers, new_major_version) + out_of_date_repointings = find_out_of_date_l3es( + existing_file_versions, all_pointing_numbers, new_major_version + ) - repointings_to_process = pointing_numbers_updated_by_l3d | repointings_to_force_processing | out_of_date_repointings + repointings_to_process = ( + pointing_numbers_updated_by_l3d + | repointings_to_force_processing + | out_of_date_repointings + ) new_file_versions = {} for pointing_number in sorted(repointings_to_process): @@ -158,16 +222,21 @@ def identify_versions_for_l3e_output_files(start_cr_of_mission: int, end_cr_of_m updated_pointings_per_instruments[descriptor] = new_file_versions updated_pointing_numbers = updated_pointing_numbers | new_file_versions.keys() - - return GlowsL3eVersionsForRepointings(list(updated_pointing_numbers), - updated_pointings_per_instruments[GLOWS_L3E_HI_90_DESCRIPTOR], - updated_pointings_per_instruments[GLOWS_L3E_HI_45_DESCRIPTOR], - updated_pointings_per_instruments[GLOWS_L3E_LO_DESCRIPTOR], - updated_pointings_per_instruments[GLOWS_L3E_ULTRA_SF_DESCRIPTOR], - updated_pointings_per_instruments[GLOWS_L3E_ULTRA_HF_DESCRIPTOR], - ) - -def find_out_of_date_l3es(existing_l3es: dict[int, Version], repointings_to_check: set[int], current_major_version: int) -> set[int]: + return GlowsL3eVersionsForRepointings( + list(updated_pointing_numbers), + updated_pointings_per_instruments[GLOWS_L3E_HI_90_DESCRIPTOR], + updated_pointings_per_instruments[GLOWS_L3E_HI_45_DESCRIPTOR], + updated_pointings_per_instruments[GLOWS_L3E_LO_DESCRIPTOR], + updated_pointings_per_instruments[GLOWS_L3E_ULTRA_SF_DESCRIPTOR], + updated_pointings_per_instruments[GLOWS_L3E_ULTRA_HF_DESCRIPTOR], + ) + + +def find_out_of_date_l3es( + existing_l3es: dict[int, Version], + repointings_to_check: set[int], + current_major_version: int, +) -> set[int]: out_of_date_l3es = set() for repointing in repointings_to_check: if existing_l3e_version := existing_l3es.get(repointing): @@ -184,22 +253,54 @@ def find_first_updated_cr(new_l3d: Path, old_l3d: str) -> Optional[int]: old_l3d_cdf = CDF(str(downloaded_old_l3d)) new_l3d_cdf = CDF(str(new_l3d)) - for i, cr in enumerate(old_l3d_cdf['cr_grid'][...]): - lya_matches = np.isclose(old_l3d_cdf['lyman_alpha'][i], new_l3d_cdf['lyman_alpha'][i]) - phion_matches = np.isclose(old_l3d_cdf['phion'][i], new_l3d_cdf['phion'][i]) - plasma_speed_flag_matches = np.isclose(old_l3d_cdf['plasma_speed_flag'][i], new_l3d_cdf['plasma_speed_flag'][i]) - proton_density_flag_matches = np.isclose(old_l3d_cdf['proton_density_flag'][i], new_l3d_cdf['proton_density_flag'][i]) - uv_anisotropy_flag_matches = np.isclose(old_l3d_cdf['uv_anisotropy_flag'][i], new_l3d_cdf['uv_anisotropy_flag'][i]) - glows_flags_matches = np.isclose(old_l3d_cdf['glows_flags'][i], new_l3d_cdf['glows_flags'][i]) + for i, cr in enumerate(old_l3d_cdf["cr_grid"][...]): + lya_matches = np.isclose( + old_l3d_cdf["lyman_alpha"][i], new_l3d_cdf["lyman_alpha"][i] + ) + phion_matches = np.isclose(old_l3d_cdf["phion"][i], new_l3d_cdf["phion"][i]) + plasma_speed_flag_matches = np.isclose( + old_l3d_cdf["plasma_speed_flag"][i], new_l3d_cdf["plasma_speed_flag"][i] + ) + proton_density_flag_matches = np.isclose( + old_l3d_cdf["proton_density_flag"][i], new_l3d_cdf["proton_density_flag"][i] + ) + uv_anisotropy_flag_matches = np.isclose( + old_l3d_cdf["uv_anisotropy_flag"][i], new_l3d_cdf["uv_anisotropy_flag"][i] + ) + glows_flags_matches = np.isclose( + old_l3d_cdf["glows_flags"][i], new_l3d_cdf["glows_flags"][i] + ) - plasma_speed_matches = np.all(np.isclose(old_l3d_cdf['plasma_speed'][i], new_l3d_cdf['plasma_speed'][i])) - proton_density_matches = np.all(np.isclose(old_l3d_cdf['proton_density'][i], new_l3d_cdf['proton_density'][i])) - uv_anisotropy_matches = np.all(np.isclose(old_l3d_cdf['uv_anisotropy'][i], new_l3d_cdf['uv_anisotropy'][i])) + plasma_speed_matches = np.all( + np.isclose(old_l3d_cdf["plasma_speed"][i], new_l3d_cdf["plasma_speed"][i]) + ) + proton_density_matches = np.all( + np.isclose( + old_l3d_cdf["proton_density"][i], new_l3d_cdf["proton_density"][i] + ) + ) + uv_anisotropy_matches = np.all( + np.isclose(old_l3d_cdf["uv_anisotropy"][i], new_l3d_cdf["uv_anisotropy"][i]) + ) - if np.any(np.logical_not([lya_matches, phion_matches, plasma_speed_matches, plasma_speed_flag_matches, proton_density_matches, proton_density_flag_matches, uv_anisotropy_matches, uv_anisotropy_flag_matches, glows_flags_matches])): + if np.any( + np.logical_not( + [ + lya_matches, + phion_matches, + plasma_speed_matches, + plasma_speed_flag_matches, + proton_density_matches, + proton_density_flag_matches, + uv_anisotropy_matches, + uv_anisotropy_flag_matches, + glows_flags_matches, + ] + ) + ): return int(cr) - if old_l3d_cdf['cr_grid'].shape != new_l3d_cdf['cr_grid'].shape: - return int(old_l3d_cdf['cr_grid'][-1]) + 1 + if old_l3d_cdf["cr_grid"].shape != new_l3d_cdf["cr_grid"].shape: + return int(old_l3d_cdf["cr_grid"][-1]) + 1 return None diff --git a/imap_l3_processing/glows/l3e/glows_l3e_lo_model.py b/imap_l3_processing/glows/l3e/glows_l3e_lo_model.py index ed48c175c..9cf9515a3 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_lo_model.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_lo_model.py @@ -54,28 +54,44 @@ class GlowsL3ELoData(DataProduct): glows_flags: np.ndarray @classmethod - def convert_dat_to_glows_l3e_lo_product(cls, input_metadata: InputMetadata, file_path: Path, - epoch: datetime, epoch_delta: timedelta, elongation: int, - args: GlowsL3eCallArguments): + def convert_dat_to_glows_l3e_lo_product( + cls, + input_metadata: InputMetadata, + file_path: Path, + epoch: datetime, + epoch_delta: timedelta, + elongation: int, + args: GlowsL3eCallArguments, + ): with open(file_path) as input_data: lines = input_data.readlines() energy_line = [line for line in lines if line.startswith("#energy_grid")] - energies = np.array([float(i) for i in re.findall(r"\d+.\d+", energy_line[0])]) + energies = np.array( + [float(i) for i in re.findall(r"\d+.\d+", energy_line[0])] + ) - code_version_line = [line for line in lines if line.startswith("# code version")] - code_version = code_version_line[0].split(',')[0][14:].strip() + code_version_line = [ + line for line in lines if line.startswith("# code version") + ] + code_version = code_version_line[0].split(",")[0][14:].strip() - spin_angle_and_survival_probabilities = np.loadtxt(file_path, skiprows=200, dtype=np.float64) + spin_angle_and_survival_probabilities = np.loadtxt( + file_path, skiprows=200, dtype=np.float64 + ) spin_angles = spin_angle_and_survival_probabilities[:, 0] - survival_probabilities = np.array([spin_angle_and_survival_probabilities[:, 1:].T]) + survival_probabilities = np.array( + [spin_angle_and_survival_probabilities[:, 1:].T] + ) energy_delta_plus, energy_delta_minus = calculate_energy_deltas(energies) return cls( input_metadata=input_metadata, epoch=np.array([epoch]), - epoch_delta=np.array([epoch_delta.total_seconds() * ONE_SECOND_IN_NANOSECONDS]), + epoch_delta=np.array( + [epoch_delta.total_seconds() * ONE_SECOND_IN_NANOSECONDS] + ), energy=energies, energy_delta_plus=energy_delta_plus, energy_delta_minus=energy_delta_minus, @@ -88,11 +104,17 @@ def convert_dat_to_glows_l3e_lo_product(cls, input_metadata: InputMetadata, file spacecraft_radius=np.array([args.spacecraft_info.spacecraft_radius]), spacecraft_longitude=np.array([args.spacecraft_info.spacecraft_longitude]), spacecraft_latitude=np.array([args.spacecraft_info.spacecraft_latitude]), - spacecraft_velocity_x=np.array([args.spacecraft_info.spacecraft_velocity_x]), - spacecraft_velocity_y=np.array([args.spacecraft_info.spacecraft_velocity_y]), - spacecraft_velocity_z=np.array([args.spacecraft_info.spacecraft_velocity_z]), + spacecraft_velocity_x=np.array( + [args.spacecraft_info.spacecraft_velocity_x] + ), + spacecraft_velocity_y=np.array( + [args.spacecraft_info.spacecraft_velocity_y] + ), + spacecraft_velocity_z=np.array( + [args.spacecraft_info.spacecraft_velocity_z] + ), glows_flags=np.array([0], dtype=np.uint16), - ) + ) def to_data_product_variables(self) -> list[DataProductVariable]: spin_angle_labels = [f"{i:.0f}" for i in self.spin_angle] @@ -105,7 +127,9 @@ def to_data_product_variables(self) -> list[DataProductVariable]: DataProductVariable(ENERGY_DELTA_PLUS_VAR_NAME, self.energy_delta_plus), DataProductVariable(ENERGY_DELTA_MINUS_VAR_NAME, self.energy_delta_minus), DataProductVariable(SPIN_ANGLE_VAR_NAME, self.spin_angle), - DataProductVariable(PROBABILITY_OF_SURVIVAL_VAR_NAME, self.probability_of_survival), + DataProductVariable( + PROBABILITY_OF_SURVIVAL_VAR_NAME, self.probability_of_survival + ), DataProductVariable(ENERGY_LABEL_VAR_NAME, energy_labels), DataProductVariable(SPIN_ANGLE_LABEL_VAR_NAME, spin_angle_labels), DataProductVariable(ELONGATION_VAR_NAME, self.elongation), @@ -114,9 +138,17 @@ def to_data_product_variables(self) -> list[DataProductVariable]: DataProductVariable(PROGRAM_VERSION_VAR_NAME, self.program_version), DataProductVariable(SPACECRAFT_RADIUS_VAR_NAME, self.spacecraft_radius), DataProductVariable(SPACECRAFT_LATITUDE_VAR_NAME, self.spacecraft_latitude), - DataProductVariable(SPACECRAFT_LONGITUDE_VAR_NAME, self.spacecraft_longitude), - DataProductVariable(SPACECRAFT_VELOCITY_X_VAR_NAME, self.spacecraft_velocity_x), - DataProductVariable(SPACECRAFT_VELOCITY_Y_VAR_NAME, self.spacecraft_velocity_y), - DataProductVariable(SPACECRAFT_VELOCITY_Z_VAR_NAME, self.spacecraft_velocity_z), + DataProductVariable( + SPACECRAFT_LONGITUDE_VAR_NAME, self.spacecraft_longitude + ), + DataProductVariable( + SPACECRAFT_VELOCITY_X_VAR_NAME, self.spacecraft_velocity_x + ), + DataProductVariable( + SPACECRAFT_VELOCITY_Y_VAR_NAME, self.spacecraft_velocity_y + ), + DataProductVariable( + SPACECRAFT_VELOCITY_Z_VAR_NAME, self.spacecraft_velocity_z + ), DataProductVariable(GLOWS_FLAGS_VAR_NAME, self.glows_flags), ] diff --git a/imap_l3_processing/glows/l3e/glows_l3e_ultra_model.py b/imap_l3_processing/glows/l3e/glows_l3e_ultra_model.py index 64f0b6e1d..7eef9be5e 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_ultra_model.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_ultra_model.py @@ -58,18 +58,26 @@ class GlowsL3EUltraData(DataProduct): glows_flags: np.ndarray @classmethod - def convert_dat_to_glows_l3e_ul_product(cls, input_metadata: InputMetadata, file_path: Path, - epoch: datetime, - epoch_delta: timedelta, - args: GlowsL3eCallArguments): + def convert_dat_to_glows_l3e_ul_product( + cls, + input_metadata: InputMetadata, + file_path: Path, + epoch: datetime, + epoch_delta: timedelta, + args: GlowsL3eCallArguments, + ): with open(file_path) as input_data: lines = input_data.readlines() energy_line = [line for line in lines if line.startswith("#energy_grid")] - energies = np.array([float(i) for i in re.findall(r"\d+.\d+", energy_line[0])]) + energies = np.array( + [float(i) for i in re.findall(r"\d+.\d+", energy_line[0])] + ) - code_version_line = [line for line in lines if line.startswith("# code version")] - code_version = code_version_line[0].split(',')[0][14:].strip() + code_version_line = [ + line for line in lines if line.startswith("# code version") + ] + code_version = code_version_line[0].split(",")[0][14:].strip() data_table = np.loadtxt(file_path, skiprows=200, dtype=np.float64) @@ -80,12 +88,15 @@ def convert_dat_to_glows_l3e_ul_product(cls, input_metadata: InputMetadata, file pixel_latitude = data_table[:, 1] pixel_longitude = data_table[:, 2] - - probability_of_survival_to_return = np.full((len(energies), len(healpix_indexes)), np.nan, dtype=float) + probability_of_survival_to_return = np.full( + (len(energies), len(healpix_indexes)), np.nan, dtype=float + ) pixel_latitude_to_return = np.full(len(healpix_indexes), np.nan, dtype=float) pixel_longitude_to_return = np.full(len(healpix_indexes), np.nan, dtype=float) - for healpix, prob_sur, pixel_lat, pixel_lon in zip(existing_healpix, probability_of_survival, pixel_latitude, pixel_longitude): + for healpix, prob_sur, pixel_lat, pixel_lon in zip( + existing_healpix, probability_of_survival, pixel_latitude, pixel_longitude + ): probability_of_survival_to_return[:, int(healpix)] = prob_sur pixel_latitude_to_return[int(healpix)] = pixel_lat pixel_longitude_to_return[int(healpix)] = pixel_lon @@ -97,7 +108,9 @@ def convert_dat_to_glows_l3e_ul_product(cls, input_metadata: InputMetadata, file return cls( input_metadata, epoch=np.array([epoch]), - epoch_delta=np.array([epoch_delta.total_seconds() * ONE_SECOND_IN_NANOSECONDS]), + epoch_delta=np.array( + [epoch_delta.total_seconds() * ONE_SECOND_IN_NANOSECONDS] + ), energy=energies, energy_delta_plus=energy_delta_plus, energy_delta_minus=energy_delta_minus, @@ -109,9 +122,15 @@ def convert_dat_to_glows_l3e_ul_product(cls, input_metadata: InputMetadata, file spacecraft_radius=np.array([args.spacecraft_info.spacecraft_radius]), spacecraft_longitude=np.array([args.spacecraft_info.spacecraft_longitude]), spacecraft_latitude=np.array([args.spacecraft_info.spacecraft_latitude]), - spacecraft_velocity_x=np.array([args.spacecraft_info.spacecraft_velocity_x]), - spacecraft_velocity_y=np.array([args.spacecraft_info.spacecraft_velocity_y]), - spacecraft_velocity_z=np.array([args.spacecraft_info.spacecraft_velocity_z]), + spacecraft_velocity_x=np.array( + [args.spacecraft_info.spacecraft_velocity_x] + ), + spacecraft_velocity_y=np.array( + [args.spacecraft_info.spacecraft_velocity_y] + ), + spacecraft_velocity_z=np.array( + [args.spacecraft_info.spacecraft_velocity_z] + ), elongation_excluded=np.array([args.elongation]), pixel_latitude=np.array([pixel_latitude_to_return]), pixel_longitude=np.array([pixel_longitude_to_return]), @@ -128,7 +147,9 @@ def to_data_product_variables(self) -> list[DataProductVariable]: DataProductVariable(ENERGY_DELTA_PLUS_VAR_NAME, self.energy_delta_plus), DataProductVariable(ENERGY_DELTA_MINUS_VAR_NAME, self.energy_delta_minus), DataProductVariable(HEALPIX_INDEX_VAR_NAME, self.healpix_index), - DataProductVariable(PROBABILITY_OF_SURVIVAL_VAR_NAME, self.probability_of_survival), + DataProductVariable( + PROBABILITY_OF_SURVIVAL_VAR_NAME, self.probability_of_survival + ), DataProductVariable(ENERGY_LABEL_VAR_NAME, energy_labels), DataProductVariable(PIXEL_INDEX_LABEL_VAR_NAME, pixel_labels), DataProductVariable(SPIN_AXIS_LATITUDE_VAR_NAME, self.spin_axis_lat), @@ -136,10 +157,18 @@ def to_data_product_variables(self) -> list[DataProductVariable]: DataProductVariable(PROGRAM_VERSION_VAR_NAME, self.program_version), DataProductVariable(SPACECRAFT_RADIUS_VAR_NAME, self.spacecraft_radius), DataProductVariable(SPACECRAFT_LATITUDE_VAR_NAME, self.spacecraft_latitude), - DataProductVariable(SPACECRAFT_LONGITUDE_VAR_NAME, self.spacecraft_longitude), - DataProductVariable(SPACECRAFT_VELOCITY_X_VAR_NAME, self.spacecraft_velocity_x), - DataProductVariable(SPACECRAFT_VELOCITY_Y_VAR_NAME, self.spacecraft_velocity_y), - DataProductVariable(SPACECRAFT_VELOCITY_Z_VAR_NAME, self.spacecraft_velocity_z), + DataProductVariable( + SPACECRAFT_LONGITUDE_VAR_NAME, self.spacecraft_longitude + ), + DataProductVariable( + SPACECRAFT_VELOCITY_X_VAR_NAME, self.spacecraft_velocity_x + ), + DataProductVariable( + SPACECRAFT_VELOCITY_Y_VAR_NAME, self.spacecraft_velocity_y + ), + DataProductVariable( + SPACECRAFT_VELOCITY_Z_VAR_NAME, self.spacecraft_velocity_z + ), DataProductVariable(ELONGATION_EXCLUDED_VAR_NAME, self.elongation_excluded), DataProductVariable(PIXEL_LATITUDE_VAR_NAME, self.pixel_latitude), DataProductVariable(PIXEL_LONGITUDE_VAR_NAME, self.pixel_longitude), diff --git a/imap_l3_processing/glows/l3e/glows_l3e_utils.py b/imap_l3_processing/glows/l3e/glows_l3e_utils.py index b44951991..890f651ac 100644 --- a/imap_l3_processing/glows/l3e/glows_l3e_utils.py +++ b/imap_l3_processing/glows/l3e/glows_l3e_utils.py @@ -17,16 +17,28 @@ from typing import Optional -from imap_l3_processing.constants import ONE_AU_IN_KM, TT2000_EPOCH, ONE_SECOND_IN_NANOSECONDS +from imap_l3_processing.constants import ( + ONE_AU_IN_KM, + TT2000_EPOCH, + ONE_SECOND_IN_NANOSECONDS, +) from imap_l3_processing.glows.l3bc.l3bc_toolkit.funcs import jd_fm_Carrington -from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import GlowsL3eCallArguments, GlowsL3eSpacecraftInfo +from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import ( + GlowsL3eCallArguments, + GlowsL3eSpacecraftInfo, +) from imap_l3_processing.utils import FurnishMetakernelOutput if typing.TYPE_CHECKING: pass -def determine_call_args_for_l3e_executable(start_date: datetime, repointing_midpoint: datetime, elongation: float, - spacecraft_info: GlowsL3eSpacecraftInfo) -> GlowsL3eCallArguments: + +def determine_call_args_for_l3e_executable( + start_date: datetime, + repointing_midpoint: datetime, + elongation: float, + spacecraft_info: GlowsL3eSpacecraftInfo, +) -> GlowsL3eCallArguments: formatted_date = start_date.strftime("%Y%m%d_%H%M%S") decimal_date = _decimal_time(repointing_midpoint) @@ -37,10 +49,15 @@ def determine_call_args_for_l3e_executable(start_date: datetime, repointing_midp spacecraft_info=spacecraft_info, ) -def determine_spacecraft_info_for_l3e_executable(repointing_midpoint: datetime) -> GlowsL3eSpacecraftInfo: + +def determine_spacecraft_info_for_l3e_executable( + repointing_midpoint: datetime, +) -> GlowsL3eSpacecraftInfo: ephemeris_time = spiceypy.datetime2et(repointing_midpoint) - [x, y, z, vx, vy, vz], _ = spiceypy.spkezr("IMAP", ephemeris_time, "ECLIPJ2000", "NONE", "SUN") + [x, y, z, vx, vy, vz], _ = spiceypy.spkezr( + "IMAP", ephemeris_time, "ECLIPJ2000", "NONE", "SUN" + ) radius, longitude, latitude = spiceypy.reclat([x, y, z]) @@ -61,18 +78,24 @@ def determine_spacecraft_info_for_l3e_executable(repointing_midpoint: datetime) ) -def determine_spacecraft_info_using_predict_if_needed(repointing_midpoint: datetime, spice_with_predict: FurnishMetakernelOutput, spice_without_predict: FurnishMetakernelOutput) -> \ - tuple[GlowsL3eSpacecraftInfo, GlowsL3Flags, list[str]]: +def determine_spacecraft_info_using_predict_if_needed( + repointing_midpoint: datetime, + spice_with_predict: FurnishMetakernelOutput, + spice_without_predict: FurnishMetakernelOutput, +) -> tuple[GlowsL3eSpacecraftInfo, GlowsL3Flags, list[str]]: try: with spiceypy.KernelPool([str(spice_without_predict.metakernel_path)]): - spacecraft_info: GlowsL3eSpacecraftInfo = determine_spacecraft_info_for_l3e_executable(repointing_midpoint) + spacecraft_info: GlowsL3eSpacecraftInfo = ( + determine_spacecraft_info_for_l3e_executable(repointing_midpoint) + ) glows_l3_flags: GlowsL3Flags = GlowsL3Flags.NONE kernel_names = [n.name for n in spice_without_predict.spice_kernel_paths] except SpiceyError: with spiceypy.KernelPool([str(spice_with_predict.metakernel_path)]): - spacecraft_info: GlowsL3eSpacecraftInfo = determine_spacecraft_info_for_l3e_executable( - repointing_midpoint) + spacecraft_info: GlowsL3eSpacecraftInfo = ( + determine_spacecraft_info_for_l3e_executable(repointing_midpoint) + ) glows_l3_flags: GlowsL3Flags = GlowsL3Flags.PREDICTIVE_EPHEMERIS kernel_names = [n.name for n in spice_with_predict.spice_kernel_paths] @@ -96,10 +119,12 @@ class GlowsL3eVersionsForRepointings: ultra_hf_repointings: dict[int, Version] -def compute_glows_flags_for_repoint(l3d_cdf_path: Path, repoint_midpoint: datetime) -> int: +def compute_glows_flags_for_repoint( + l3d_cdf_path: Path, repoint_midpoint: datetime +) -> int: with CDF(str(l3d_cdf_path)) as cdf: - cr_epochs = cdf['epoch'][...] - flags = cdf['glows_flags'][...] + cr_epochs = cdf["epoch"][...] + flags = cdf["glows_flags"][...] cr_after_repoint = np.searchsorted(cr_epochs, repoint_midpoint) cr_before_repoint = cr_after_repoint - 1 @@ -115,10 +140,11 @@ def compute_glows_flags_for_repoint(l3d_cdf_path: Path, repoint_midpoint: dateti return int(np.bitwise_or.reduce(selected.astype(np.uint16), initial=0)) + def get_lo_pivot_angle_from_l1b_file(path: Path) -> float: with CDF(str(path)) as cdf: - epoch = cdf['epoch'][...] - angles = cdf['pcc_coarse_pot_pri'][...] + epoch = cdf["epoch"][...] + angles = cdf["pcc_coarse_pot_pri"][...] if len(epoch) == 0: return 90.0 t0 = epoch[0] @@ -130,11 +156,13 @@ def get_lo_pivot_angle_from_l1b_file(path: Path) -> float: return 90.0 return np.round(np.median(angles_to_consider)) + @dataclass class LoPivotAngle: parent_filename: Optional[str] pivot_angle: float + def get_lo_pivot_angles(repointings: list[int]) -> dict[int, LoPivotAngle]: l1b_results = imap_data_access.query( instrument="lo", @@ -142,38 +170,61 @@ def get_lo_pivot_angles(repointings: list[int]) -> dict[int, LoPivotAngle]: descriptor="nhk", version="latest", ) - paths_by_repointing = {f["repointing"]:f["file_path"] for f in l1b_results} + paths_by_repointing = {f["repointing"]: f["file_path"] for f in l1b_results} result = {} for repointing in repointings: if path := paths_by_repointing.get(repointing): downloaded_path = imap_data_access.download(path) - result[repointing] = LoPivotAngle(parent_filename=Path(path).name, pivot_angle=get_lo_pivot_angle_from_l1b_file(downloaded_path)) + result[repointing] = LoPivotAngle( + parent_filename=Path(path).name, + pivot_angle=get_lo_pivot_angle_from_l1b_file(downloaded_path), + ) else: result[repointing] = LoPivotAngle(parent_filename=None, pivot_angle=90.0) return result -def get_repoint_numbers_within_cr_window(start_cr_number: int | None, end_cr_number: int, repointing_data) -> set[int]: + +def get_repoint_numbers_within_cr_window( + start_cr_number: int | None, end_cr_number: int, repointing_data +) -> set[int]: if start_cr_number is None: return set() - first_carrington_start_date = Time(jd_fm_Carrington(float(start_cr_number)), format='jd') - last_cr_end_date = Time(jd_fm_Carrington(float(end_cr_number + 0.5)), format='jd') - - start_ns = (first_carrington_start_date.to_datetime() - TT2000_EPOCH).total_seconds() * ONE_SECOND_IN_NANOSECONDS - end_ns = (last_cr_end_date.to_datetime() - TT2000_EPOCH).total_seconds() * ONE_SECOND_IN_NANOSECONDS - - vectorized_date_conv = np.vectorize(lambda d: (Time(d, format="iso").to_datetime( - leap_second_strict='silent') - TT2000_EPOCH).total_seconds() * ONE_SECOND_IN_NANOSECONDS) + first_carrington_start_date = Time( + jd_fm_Carrington(float(start_cr_number)), format="jd" + ) + last_cr_end_date = Time(jd_fm_Carrington(float(end_cr_number + 0.5)), format="jd") + + start_ns = ( + first_carrington_start_date.to_datetime() - TT2000_EPOCH + ).total_seconds() * ONE_SECOND_IN_NANOSECONDS + end_ns = ( + last_cr_end_date.to_datetime() - TT2000_EPOCH + ).total_seconds() * ONE_SECOND_IN_NANOSECONDS + + vectorized_date_conv = np.vectorize( + lambda d: ( + ( + Time(d, format="iso").to_datetime(leap_second_strict="silent") + - TT2000_EPOCH + ).total_seconds() + * ONE_SECOND_IN_NANOSECONDS + ) + ) repoint_starts = vectorized_date_conv(repointing_data["repoint_start_utc"]) repoint_ends = vectorized_date_conv(repointing_data["repoint_end_utc"]) repoint_ids = repointing_data["repoint_id"] repoint_numbers = set() for i in range(len(repoint_ids)): - if i + 1 < len(repoint_ids) and start_ns < (repoint_starts[i + 1] + repoint_ends[i])/2 < end_ns: + if ( + i + 1 < len(repoint_ids) + and start_ns < (repoint_starts[i + 1] + repoint_ends[i]) / 2 < end_ns + ): repoint_numbers.add(int(repoint_ids[i])) return repoint_numbers + def calculate_energy_deltas(centers: np.ndarray): edges = np.empty_like(centers, shape=(len(centers) + 1,)) edges[1:-1] = np.sqrt(centers[1:] * centers[:-1]) @@ -184,4 +235,3 @@ def calculate_energy_deltas(centers: np.ndarray): delta_minus = centers - edges[:-1] return delta_plus, delta_minus - diff --git a/imap_l3_processing/glows/l3e/reprocess_info.py b/imap_l3_processing/glows/l3e/reprocess_info.py index 4583edd81..20c27421a 100644 --- a/imap_l3_processing/glows/l3e/reprocess_info.py +++ b/imap_l3_processing/glows/l3e/reprocess_info.py @@ -57,16 +57,22 @@ def _parse_header(line: str) -> datetime: ignore_after = ignore_after.replace(tzinfo=timezone.utc) return ignore_after - @staticmethod def _parse_line(line: str) -> tuple[str, str, ReprocessTargets]: product_name, _, remainder = line.strip().partition(" ") data_level, _, descriptor = product_name.partition("_") targets = remainder.replace(",", " ").split() - repoints = [int(target.removeprefix(REPOINT_PREFIX)) for target in targets if target.startswith(REPOINT_PREFIX)] - carrington_rotations = [int(target.removeprefix(CARRINGTON_ROTATION_PREFIX)) for target in targets if - target.startswith(CARRINGTON_ROTATION_PREFIX)] + repoints = [ + int(target.removeprefix(REPOINT_PREFIX)) + for target in targets + if target.startswith(REPOINT_PREFIX) + ] + carrington_rotations = [ + int(target.removeprefix(CARRINGTON_ROTATION_PREFIX)) + for target in targets + if target.startswith(CARRINGTON_ROTATION_PREFIX) + ] return data_level, descriptor, ReprocessTargets(repoints, carrington_rotations) @@ -77,9 +83,7 @@ def should_reprocess_l3d(self) -> bool: ) ) - def get_repoints_for_descriptor( - self, descriptor: str, repointing_data - ) -> set[int]: + def get_repoints_for_descriptor(self, descriptor: str, repointing_data) -> set[int]: reprocess_targets = self.products_to_reprocess.get(descriptor, None) if reprocess_targets is None: return set() @@ -87,11 +91,18 @@ def get_repoints_for_descriptor( repoints = set(reprocess_targets.repoints) for cr in reprocess_targets.carrington_rotations: - repoints.update(get_repoint_numbers_within_cr_window(cr, cr, repointing_data)) + repoints.update( + get_repoint_numbers_within_cr_window(cr, cr, repointing_data) + ) return repoints + def fetch_reprocess_info(input_collection: ProcessingInputCollection) -> ReprocessInfo: - [reprocessing_file_path] = input_collection.get_file_paths(source="glows", descriptor=GLOWS_REPROCESSING_DESCRIPTOR) - path_to_downloaded_ancillary = imap_data_access.download(reprocessing_file_path.name) - return ReprocessInfo.parse_from_ancillary(path_to_downloaded_ancillary) \ No newline at end of file + [reprocessing_file_path] = input_collection.get_file_paths( + source="glows", descriptor=GLOWS_REPROCESSING_DESCRIPTOR + ) + path_to_downloaded_ancillary = imap_data_access.download( + reprocessing_file_path.name + ) + return ReprocessInfo.parse_from_ancillary(path_to_downloaded_ancillary) diff --git a/imap_l3_processing/glows/quality_flags.py b/imap_l3_processing/glows/quality_flags.py index af1878fe8..f6350739b 100644 --- a/imap_l3_processing/glows/quality_flags.py +++ b/imap_l3_processing/glows/quality_flags.py @@ -6,6 +6,6 @@ class GlowsL3Flags(FlagNameMixin): NONE = CommonFlags.NONE - NOMINAL_ALPHA_PROTON_RATIO = 2 ** 2 - PERSISTED_LAST_POINT = 2 ** 3 - PREDICTIVE_EPHEMERIS = 2 ** 15 + NOMINAL_ALPHA_PROTON_RATIO = 2**2 + PERSISTED_LAST_POINT = 2**3 + PREDICTIVE_EPHEMERIS = 2**15 diff --git a/imap_l3_processing/hi/hi_combined_initializer.py b/imap_l3_processing/hi/hi_combined_initializer.py index 4c88bbd0c..6c890071a 100644 --- a/imap_l3_processing/hi/hi_combined_initializer.py +++ b/imap_l3_processing/hi/hi_combined_initializer.py @@ -5,8 +5,13 @@ from dateutil.relativedelta import relativedelta from imap_data_access.file_validation import Version -from imap_l3_processing.maps.map_descriptors import MapDescriptorParts, parse_map_descriptor, Sensor, \ - map_descriptor_parts_to_string, SpinPhase +from imap_l3_processing.maps.map_descriptors import ( + MapDescriptorParts, + parse_map_descriptor, + Sensor, + map_descriptor_parts_to_string, + SpinPhase, +) from imap_l3_processing.maps.map_initializer import PossibleMapToProduce, MapInitializer from imap_l3_processing.models import InputMetadata, VersionMap @@ -25,37 +30,58 @@ class HiCombinedInitializer(MapInitializer): input_map_filenames: dict[str, list[str]] def __init__(self, canonical_map_start_date: datetime = datetime(2026, 1, 1)): - super().__init__('hi') + super().__init__("hi") l2_query_results = imap_data_access.query(instrument="hi", data_level="l2") - l2_by_descriptor_and_start_date = self.get_latest_version_by_descriptor_and_start_date(l2_query_results) + l2_by_descriptor_and_start_date = ( + self.get_latest_version_by_descriptor_and_start_date(l2_query_results) + ) self.input_maps = dict(self.existing_l3_maps, **l2_by_descriptor_and_start_date) self.canonical_map_start_date = canonical_map_start_date def furnish_spice_dependencies(self, map_to_produce: PossibleMapToProduce): pass - def get_maps_that_can_be_produced(self, input_descriptor: str) -> list[PossibleMapToProduce]: + def get_maps_that_can_be_produced( + self, input_descriptor: str + ) -> list[PossibleMapToProduce]: possible_maps_to_produce = [] map_descriptor: MapDescriptorParts = parse_map_descriptor(input_descriptor) hi90_ram_descriptor = map_descriptor_parts_to_string( - dataclasses.replace(map_descriptor, sensor=Sensor.Hi90, spin_phase=SpinPhase.RamOnly)) + dataclasses.replace( + map_descriptor, sensor=Sensor.Hi90, spin_phase=SpinPhase.RamOnly + ) + ) hi45_ram_descriptor = map_descriptor_parts_to_string( - dataclasses.replace(map_descriptor, sensor=Sensor.Hi45, spin_phase=SpinPhase.RamOnly)) + dataclasses.replace( + map_descriptor, sensor=Sensor.Hi45, spin_phase=SpinPhase.RamOnly + ) + ) hi90_anti_descriptor = map_descriptor_parts_to_string( - dataclasses.replace(map_descriptor, sensor=Sensor.Hi90, spin_phase=SpinPhase.AntiRamOnly)) + dataclasses.replace( + map_descriptor, sensor=Sensor.Hi90, spin_phase=SpinPhase.AntiRamOnly + ) + ) hi45_anti_descriptor = map_descriptor_parts_to_string( - dataclasses.replace(map_descriptor, sensor=Sensor.Hi45, spin_phase=SpinPhase.AntiRamOnly)) - - if hi90_ram_descriptor not in self.input_maps or hi90_anti_descriptor not in self.input_maps or hi45_ram_descriptor not in self.input_maps or hi45_anti_descriptor not in self.input_maps: + dataclasses.replace( + map_descriptor, sensor=Sensor.Hi45, spin_phase=SpinPhase.AntiRamOnly + ) + ) + + if ( + hi90_ram_descriptor not in self.input_maps + or hi90_anti_descriptor not in self.input_maps + or hi45_ram_descriptor not in self.input_maps + or hi45_anti_descriptor not in self.input_maps + ): return [] hi90_start_dates = sorted(self.input_maps[hi90_ram_descriptor].keys()) for start_date_str in hi90_start_dates: - start_date = datetime.strptime(start_date_str, '%Y%m%d') + start_date = datetime.strptime(start_date_str, "%Y%m%d") input_files = [ self.input_maps[hi90_ram_descriptor].get(start_date_str), @@ -67,17 +93,19 @@ def get_maps_that_can_be_produced(self, input_descriptor: str) -> list[PossibleM if not all(input_files): continue - possible_maps_to_produce.append(PossibleMapToProduce( - input_files=set(input_files), - input_metadata=InputMetadata( - instrument='hi', - data_level='l3', - start_date=start_date, - end_date=start_date + relativedelta(months=+12), - version=VersionMap({},Version(None,1)), - descriptor=input_descriptor, - repointing=None + possible_maps_to_produce.append( + PossibleMapToProduce( + input_files=set(input_files), + input_metadata=InputMetadata( + instrument="hi", + data_level="l3", + start_date=start_date, + end_date=start_date + relativedelta(months=+12), + version=VersionMap({}, Version(None, 1)), + descriptor=input_descriptor, + repointing=None, + ), ) - )) + ) return possible_maps_to_produce diff --git a/imap_l3_processing/hi/hi_combined_sensor_dependencies.py b/imap_l3_processing/hi/hi_combined_sensor_dependencies.py index 8f71ca572..2992d05cf 100644 --- a/imap_l3_processing/hi/hi_combined_sensor_dependencies.py +++ b/imap_l3_processing/hi/hi_combined_sensor_dependencies.py @@ -15,15 +15,30 @@ class HiL3CombinedMapDependencies: h90_maps: list[RectangularIntensityMapData] @classmethod - def fetch_dependencies(cls, dependencies: ProcessingInputCollection) -> HiL3CombinedMapDependencies: + def fetch_dependencies( + cls, dependencies: ProcessingInputCollection + ) -> HiL3CombinedMapDependencies: input_map_filenames = dependencies.get_file_paths(source="hi") - map_files = [imap_data_access.download(input_map.name) for input_map in input_map_filenames] + map_files = [ + imap_data_access.download(input_map.name) + for input_map in input_map_filenames + ] return cls.from_file_paths(map_files) @classmethod - def from_file_paths(cls, hi_l3_map_paths: list[Path]) -> HiL3CombinedMapDependencies: - return cls([RectangularIntensityMapData.read_from_path(path) for path in hi_l3_map_paths if - 'h45' == path.name.split('_')[3][:3]], - [RectangularIntensityMapData.read_from_path(path) for path in hi_l3_map_paths if - 'h90' == path.name.split('_')[3][:3]]) + def from_file_paths( + cls, hi_l3_map_paths: list[Path] + ) -> HiL3CombinedMapDependencies: + return cls( + [ + RectangularIntensityMapData.read_from_path(path) + for path in hi_l3_map_paths + if "h45" == path.name.split("_")[3][:3] + ], + [ + RectangularIntensityMapData.read_from_path(path) + for path in hi_l3_map_paths + if "h90" == path.name.split("_")[3][:3] + ], + ) diff --git a/imap_l3_processing/hi/hi_processor.py b/imap_l3_processing/hi/hi_processor.py index 5d2a7c052..9436f5fcd 100644 --- a/imap_l3_processing/hi/hi_processor.py +++ b/imap_l3_processing/hi/hi_processor.py @@ -3,98 +3,161 @@ from imap_processing.spice.geometry import SpiceFrame -from imap_l3_processing.hi.hi_combined_sensor_dependencies import HiL3CombinedMapDependencies -from imap_l3_processing.hi.hi_spectral_fit_dependencies import HiSpectralIndexDependencies -from imap_l3_processing.maps.hilo_l3_survival_dependencies import HiLoL3SurvivalDependencies, \ - HiL3SingleSensorFullSpinDependencies -from imap_l3_processing.maps.map_combination import UncertaintyWeightedCombination, UnweightedCombination, \ - CombinationStrategy, ExposureWeightedCombination -from imap_l3_processing.maps.map_descriptors import parse_map_descriptor, MapQuantity, MapDescriptorParts, \ - SurvivalCorrection, \ - SpinPhase, Sensor, ReferenceFrame -from imap_l3_processing.maps.map_models import RectangularSpectralIndexDataProduct, RectangularSpectralIndexMapData, \ - RectangularIntensityMapData, RectangularIntensityDataProduct +from imap_l3_processing.hi.hi_combined_sensor_dependencies import ( + HiL3CombinedMapDependencies, +) +from imap_l3_processing.hi.hi_spectral_fit_dependencies import ( + HiSpectralIndexDependencies, +) +from imap_l3_processing.maps.hilo_l3_survival_dependencies import ( + HiLoL3SurvivalDependencies, + HiL3SingleSensorFullSpinDependencies, +) +from imap_l3_processing.maps.map_combination import ( + UncertaintyWeightedCombination, + UnweightedCombination, + CombinationStrategy, + ExposureWeightedCombination, +) +from imap_l3_processing.maps.map_descriptors import ( + parse_map_descriptor, + MapQuantity, + MapDescriptorParts, + SurvivalCorrection, + SpinPhase, + Sensor, + ReferenceFrame, +) +from imap_l3_processing.maps.map_models import ( + RectangularSpectralIndexDataProduct, + RectangularSpectralIndexMapData, + RectangularIntensityMapData, + RectangularIntensityDataProduct, +) from imap_l3_processing.maps.map_processor import MapProcessor -from imap_l3_processing.maps.spectral_fit import fit_spectral_index_map, slice_energy_range_by_bin -from imap_l3_processing.maps.survival_probability_processing import process_survival_probabilities +from imap_l3_processing.maps.spectral_fit import ( + fit_spectral_index_map, + slice_energy_range_by_bin, +) +from imap_l3_processing.maps.survival_probability_processing import ( + process_survival_probabilities, +) from imap_l3_processing.models import Instrument from imap_l3_processing.utils import save_data class HiProcessor(MapProcessor): - def process(self, spice_frame_name: SpiceFrame = SpiceFrame.ECLIPJ2000) -> list[Path]: + def process( + self, spice_frame_name: SpiceFrame = SpiceFrame.ECLIPJ2000 + ) -> list[Path]: set_of_parent_file_names = set(self.get_parent_file_names()) parsed_descriptor = parse_map_descriptor(self.input_metadata.descriptor) match parsed_descriptor: case MapDescriptorParts(quantity=MapQuantity.SpectralIndex): - hi_l3_spectral_fit_dependencies = HiSpectralIndexDependencies.fetch_dependencies(self.dependencies) - map_data = self.process_spectral_fit_index(hi_l3_spectral_fit_dependencies, parsed_descriptor.spectral_index_energy_range) + hi_l3_spectral_fit_dependencies = ( + HiSpectralIndexDependencies.fetch_dependencies(self.dependencies) + ) + map_data = self.process_spectral_fit_index( + hi_l3_spectral_fit_dependencies, + parsed_descriptor.spectral_index_energy_range, + ) data_product = RectangularSpectralIndexDataProduct( data=map_data, input_metadata=self.input_metadata, - spice_frame_name=spice_frame_name + spice_frame_name=spice_frame_name, + ) + case MapDescriptorParts( + survival_correction=SurvivalCorrection.SurvivalCorrected, + sensor=Sensor.Hi90 | Sensor.Hi45, + spin_phase=SpinPhase.RamOnly | SpinPhase.AntiRamOnly, + ): + hi_l3_survival_probabilities_dependencies = ( + HiLoL3SurvivalDependencies.fetch_dependencies( + self.dependencies, Instrument.IMAP_HI + ) ) - case MapDescriptorParts(survival_correction=SurvivalCorrection.SurvivalCorrected, - sensor=Sensor.Hi90 | Sensor.Hi45, - spin_phase=SpinPhase.RamOnly | SpinPhase.AntiRamOnly): - hi_l3_survival_probabilities_dependencies = HiLoL3SurvivalDependencies.fetch_dependencies( - self.dependencies, Instrument.IMAP_HI) data_product = RectangularIntensityDataProduct( - data=process_survival_probabilities(hi_l3_survival_probabilities_dependencies, spice_frame_name), + data=process_survival_probabilities( + hi_l3_survival_probabilities_dependencies, spice_frame_name + ), input_metadata=self.input_metadata, - spice_frame_name=spice_frame_name + spice_frame_name=spice_frame_name, ) set_of_parent_file_names.update( - p.name for p in hi_l3_survival_probabilities_dependencies.dependency_file_paths) - case MapDescriptorParts(survival_correction=SurvivalCorrection.SurvivalCorrected, - sensor=Sensor.Hi90 | Sensor.Hi45, - spin_phase=SpinPhase.FullSpin, - reference_frame=ReferenceFrame.Spacecraft): - hi_l3_full_spin_dependencies = HiL3SingleSensorFullSpinDependencies.fetch_dependencies( - self.dependencies) + p.name + for p in hi_l3_survival_probabilities_dependencies.dependency_file_paths + ) + case MapDescriptorParts( + survival_correction=SurvivalCorrection.SurvivalCorrected, + sensor=Sensor.Hi90 | Sensor.Hi45, + spin_phase=SpinPhase.FullSpin, + reference_frame=ReferenceFrame.Spacecraft, + ): + hi_l3_full_spin_dependencies = ( + HiL3SingleSensorFullSpinDependencies.fetch_dependencies( + self.dependencies + ) + ) combination_strategy = UnweightedCombination() - combined_map = self.process_full_spin_single_sensor(hi_l3_full_spin_dependencies, spice_frame_name, - combination_strategy) + combined_map = self.process_full_spin_single_sensor( + hi_l3_full_spin_dependencies, spice_frame_name, combination_strategy + ) data_product = RectangularIntensityDataProduct( data=combined_map, input_metadata=self.input_metadata, - spice_frame_name=spice_frame_name + spice_frame_name=spice_frame_name, + ) + set_of_parent_file_names.update( + p.name for p in hi_l3_full_spin_dependencies.dependency_file_paths + ) + case MapDescriptorParts( + survival_correction=SurvivalCorrection.SurvivalCorrected, + sensor=Sensor.Hi90 | Sensor.Hi45, + spin_phase=SpinPhase.FullSpin, + reference_frame=ReferenceFrame.Heliospheric, + ): + hi_l3_full_spin_cg_corrected_dependencies = ( + HiL3SingleSensorFullSpinDependencies.fetch_dependencies( + self.dependencies + ) ) - set_of_parent_file_names.update(p.name for p in hi_l3_full_spin_dependencies.dependency_file_paths) - case MapDescriptorParts(survival_correction=SurvivalCorrection.SurvivalCorrected, - sensor=Sensor.Hi90 | Sensor.Hi45, - spin_phase=SpinPhase.FullSpin, - reference_frame=ReferenceFrame.Heliospheric): - - hi_l3_full_spin_cg_corrected_dependencies = HiL3SingleSensorFullSpinDependencies.fetch_dependencies( - self.dependencies) combination_strategy = UncertaintyWeightedCombination() - combined_map = self.process_full_spin_single_sensor(hi_l3_full_spin_cg_corrected_dependencies, - spice_frame_name, combination_strategy) + combined_map = self.process_full_spin_single_sensor( + hi_l3_full_spin_cg_corrected_dependencies, + spice_frame_name, + combination_strategy, + ) data_product = RectangularIntensityDataProduct( data=combined_map, input_metadata=self.input_metadata, - spice_frame_name=spice_frame_name + spice_frame_name=spice_frame_name, ) case MapDescriptorParts(sensor=Sensor.HiCombined): - dependencies = HiL3CombinedMapDependencies.fetch_dependencies(self.dependencies) + dependencies = HiL3CombinedMapDependencies.fetch_dependencies( + self.dependencies + ) h45_full_spin_12mo = UncertaintyWeightedCombination().combine_rectangular_intensity_map_data( - dependencies.h45_maps) + dependencies.h45_maps + ) h90_full_spin_12mo = UncertaintyWeightedCombination().combine_rectangular_intensity_map_data( - dependencies.h90_maps) + dependencies.h90_maps + ) combined_map_data = ExposureWeightedCombination().combine_rectangular_intensity_map_data( - [h45_full_spin_12mo, h90_full_spin_12mo]) + [h45_full_spin_12mo, h90_full_spin_12mo] + ) data_product = RectangularIntensityDataProduct( data=combined_map_data, input_metadata=self.input_metadata, - spice_frame_name=spice_frame_name + spice_frame_name=spice_frame_name, ) case None: - raise ValueError(f"Could not parse descriptor {self.input_metadata.descriptor}") + raise ValueError( + f"Could not parse descriptor {self.input_metadata.descriptor}" + ) case _: raise NotImplementedError(self.input_metadata.descriptor) @@ -102,23 +165,35 @@ def process(self, spice_frame_name: SpiceFrame = SpiceFrame.ECLIPJ2000) -> list[ return [save_data(data_product)] - def process_full_spin_single_sensor(self, hi_l3_full_spin_dependencies: HiL3SingleSensorFullSpinDependencies, - spice_frame_name: SpiceFrame, combination_strategy: CombinationStrategy) \ - -> RectangularIntensityMapData: - ram_data_product = process_survival_probabilities(hi_l3_full_spin_dependencies.ram_dependencies, - spice_frame_name) - antiram_data_product = process_survival_probabilities(hi_l3_full_spin_dependencies.antiram_dependencies, - spice_frame_name) + def process_full_spin_single_sensor( + self, + hi_l3_full_spin_dependencies: HiL3SingleSensorFullSpinDependencies, + spice_frame_name: SpiceFrame, + combination_strategy: CombinationStrategy, + ) -> RectangularIntensityMapData: + ram_data_product = process_survival_probabilities( + hi_l3_full_spin_dependencies.ram_dependencies, spice_frame_name + ) + antiram_data_product = process_survival_probabilities( + hi_l3_full_spin_dependencies.antiram_dependencies, spice_frame_name + ) - return combination_strategy.combine_rectangular_intensity_map_data([ram_data_product, antiram_data_product]) + return combination_strategy.combine_rectangular_intensity_map_data( + [ram_data_product, antiram_data_product] + ) - def process_spectral_fit_index(self, hi_l3_spectral_fit_dependencies: HiSpectralIndexDependencies, spectral_index_energy_range: Optional[tuple[int,int]] = None) \ - -> RectangularSpectralIndexMapData: + def process_spectral_fit_index( + self, + hi_l3_spectral_fit_dependencies: HiSpectralIndexDependencies, + spectral_index_energy_range: Optional[tuple[int, int]] = None, + ) -> RectangularSpectralIndexMapData: intensity_map_data = hi_l3_spectral_fit_dependencies.map_data.intensity_map_data if spectral_index_energy_range is not None: start_bin, end_bin = spectral_index_energy_range - intensity_map_data = slice_energy_range_by_bin(intensity_map_data, start_bin, end_bin) + intensity_map_data = slice_energy_range_by_bin( + intensity_map_data, start_bin, end_bin + ) return RectangularSpectralIndexMapData( spectral_index_map_data=fit_spectral_index_map(intensity_map_data), - coords=hi_l3_spectral_fit_dependencies.map_data.coords + coords=hi_l3_spectral_fit_dependencies.map_data.coords, ) diff --git a/imap_l3_processing/hi/hi_sp_initializer.py b/imap_l3_processing/hi/hi_sp_initializer.py index ab8cdf206..19824f40a 100644 --- a/imap_l3_processing/hi/hi_sp_initializer.py +++ b/imap_l3_processing/hi/hi_sp_initializer.py @@ -4,7 +4,12 @@ import imap_data_access -from imap_l3_processing.maps.map_descriptors import MapDescriptorParts, Sensor, SurvivalCorrection, SpinPhase +from imap_l3_processing.maps.map_descriptors import ( + MapDescriptorParts, + Sensor, + SurvivalCorrection, + SpinPhase, +) from imap_l3_processing.maps.map_initializer import PossibleMapToProduce from imap_l3_processing.maps.sp_map_initializer import SPMapInitializer from imap_l3_processing.utils import SpiceKernelTypes, furnish_spice_metakernel @@ -54,7 +59,6 @@ "h90-ena-h-sf-sp-ram-hae-6deg-1yr", "h90-ena-h-sf-sp-anti-hae-6deg-1yr", "h90-ena-h-sf-sp-full-hae-6deg-6mo", - "h45-ena-h-hf-sp-ram-hae-4deg-1yr", "h45-ena-h-hf-sp-anti-hae-4deg-1yr", "h90-ena-h-hf-sp-ram-hae-4deg-1yr", @@ -66,54 +70,80 @@ "h90-ena-h-hf-sp-ram-hae-6deg-1yr", "h90-ena-h-hf-sp-anti-hae-6deg-1yr", "h45-ena-h-hf-sp-full-hae-6deg-6mo", - "h90-ena-h-hf-sp-full-hae-6deg-6mo" + "h90-ena-h-hf-sp-full-hae-6deg-6mo", ] class HiSPInitializer(SPMapInitializer): def __init__(self): sp_hi45_query_result = imap_data_access.query( - instrument='glows', - data_level='l3e', - descriptor='survival-probability-hi-45', - version='latest' + instrument="glows", + data_level="l3e", + descriptor="survival-probability-hi-45", + version="latest", ) - self.glows_hi45_file_by_repoint = {int(r["repointing"]): Path(r["file_path"]).name for r in - sp_hi45_query_result} + self.glows_hi45_file_by_repoint = { + int(r["repointing"]): Path(r["file_path"]).name + for r in sp_hi45_query_result + } sp_hi90_query_result = imap_data_access.query( - instrument='glows', - data_level='l3e', - descriptor='survival-probability-hi-90', - version='latest' + instrument="glows", + data_level="l3e", + descriptor="survival-probability-hi-90", + version="latest", ) - self.glows_hi90_file_by_repoint = {int(r["repointing"]): Path(r["file_path"]).name for r in - sp_hi90_query_result} + self.glows_hi90_file_by_repoint = { + int(r["repointing"]): Path(r["file_path"]).name + for r in sp_hi90_query_result + } - hi_l2_query_result = imap_data_access.query(instrument='hi', data_level='l2') + hi_l2_query_result = imap_data_access.query(instrument="hi", data_level="l2") super().__init__("hi", hi_l2_query_result) def furnish_spice_dependencies(self, map_to_produce: PossibleMapToProduce): - furnish_spice_metakernel(start_date=map_to_produce.input_metadata.start_date, - end_date=map_to_produce.input_metadata.end_date, kernel_types=HI_SP_SPICE_KERNELS) + furnish_spice_metakernel( + start_date=map_to_produce.input_metadata.start_date, + end_date=map_to_produce.input_metadata.end_date, + kernel_types=HI_SP_SPICE_KERNELS, + ) - def _get_l2_dependencies(self, descriptor: MapDescriptorParts) -> list[MapDescriptorParts]: + def _get_l2_dependencies( + self, descriptor: MapDescriptorParts + ) -> list[MapDescriptorParts]: match descriptor: - case MapDescriptorParts(survival_correction=SurvivalCorrection.SurvivalCorrected, - spin_phase=SpinPhase.RamOnly | SpinPhase.AntiRamOnly): - return [dataclasses.replace(descriptor, survival_correction=SurvivalCorrection.NotSurvivalCorrected)] - case MapDescriptorParts(survival_correction=SurvivalCorrection.SurvivalCorrected, - spin_phase=SpinPhase.FullSpin): + case MapDescriptorParts( + survival_correction=SurvivalCorrection.SurvivalCorrected, + spin_phase=SpinPhase.RamOnly | SpinPhase.AntiRamOnly, + ): + return [ + dataclasses.replace( + descriptor, + survival_correction=SurvivalCorrection.NotSurvivalCorrected, + ) + ] + case MapDescriptorParts( + survival_correction=SurvivalCorrection.SurvivalCorrected, + spin_phase=SpinPhase.FullSpin, + ): return [ - dataclasses.replace(descriptor, survival_correction=SurvivalCorrection.NotSurvivalCorrected, - spin_phase=SpinPhase.AntiRamOnly), - dataclasses.replace(descriptor, survival_correction=SurvivalCorrection.NotSurvivalCorrected, - spin_phase=SpinPhase.RamOnly), + dataclasses.replace( + descriptor, + survival_correction=SurvivalCorrection.NotSurvivalCorrected, + spin_phase=SpinPhase.AntiRamOnly, + ), + dataclasses.replace( + descriptor, + survival_correction=SurvivalCorrection.NotSurvivalCorrected, + spin_phase=SpinPhase.RamOnly, + ), ] case _: raise ValueError("Expected map to be produced to use a single sensor!") - def _collect_glows_psets_by_repoint(self, descriptor: MapDescriptorParts) -> dict[int, str]: + def _collect_glows_psets_by_repoint( + self, descriptor: MapDescriptorParts + ) -> dict[int, str]: match descriptor: case MapDescriptorParts(sensor=Sensor.Hi45): return self.glows_hi45_file_by_repoint diff --git a/imap_l3_processing/hi/hi_spectral_fit_dependencies.py b/imap_l3_processing/hi/hi_spectral_fit_dependencies.py index 86374c3fb..4462a4aa5 100644 --- a/imap_l3_processing/hi/hi_spectral_fit_dependencies.py +++ b/imap_l3_processing/hi/hi_spectral_fit_dependencies.py @@ -7,16 +7,21 @@ import numpy as np from imap_data_access.processing_input import ProcessingInputCollection -from imap_l3_processing.maps.map_models import RectangularIntensityMapData, SpectralIndexDependencies +from imap_l3_processing.maps.map_models import ( + RectangularIntensityMapData, + SpectralIndexDependencies, +) @dataclass class HiSpectralIndexDependencies(SpectralIndexDependencies): - @classmethod - def fetch_dependencies(cls, dependencies: ProcessingInputCollection) -> HiSpectralIndexDependencies: - input_map_filenames = dependencies.get_file_paths(source="hi", data_type="l2") + \ - dependencies.get_file_paths(source="hi", data_type="l3") + def fetch_dependencies( + cls, dependencies: ProcessingInputCollection + ) -> HiSpectralIndexDependencies: + input_map_filenames = dependencies.get_file_paths( + source="hi", data_type="l2" + ) + dependencies.get_file_paths(source="hi", data_type="l3") if len(input_map_filenames) != 1: raise ValueError("Missing Hi dependency.") @@ -30,8 +35,12 @@ def from_file_paths(cls, hi_l3_path: Path) -> HiSpectralIndexDependencies: def get_fit_energy_ranges(self) -> np.ndarray: energy_min = np.min( - self.map_data.intensity_map_data.energy - self.map_data.intensity_map_data.energy_delta_minus) + self.map_data.intensity_map_data.energy + - self.map_data.intensity_map_data.energy_delta_minus + ) energy_max = np.max( - self.map_data.intensity_map_data.energy + self.map_data.intensity_map_data.energy_delta_plus) + self.map_data.intensity_map_data.energy + + self.map_data.intensity_map_data.energy_delta_plus + ) return np.array([[energy_min, energy_max]]) diff --git a/imap_l3_processing/hi/utils.py b/imap_l3_processing/hi/utils.py index b597bda2a..c66e292c2 100644 --- a/imap_l3_processing/hi/utils.py +++ b/imap_l3_processing/hi/utils.py @@ -13,31 +13,51 @@ SPIN_ANGLE_VAR_NAME, GLOWS_FLAGS_VAR_NAME, ) -from imap_l3_processing.maps.map_models import GlowsL3eRectangularMapInputData, InputRectangularPointingSet +from imap_l3_processing.maps.map_models import ( + GlowsL3eRectangularMapInputData, + InputRectangularPointingSet, +) -def read_l1c_rectangular_pointing_set_data(path: Union[Path, str]) -> InputRectangularPointingSet: +def read_l1c_rectangular_pointing_set_data( + path: Union[Path, str], +) -> InputRectangularPointingSet: repointing = ScienceFilePath(path).repointing with CDF(str(path)) as cdf: - exposure_time_variable = cdf['exposure_time'] if 'exposure_time' in cdf else cdf['exposure_times'] + exposure_time_variable = ( + cdf["exposure_time"] if "exposure_time" in cdf else cdf["exposure_times"] + ) assert ("epoch_delta" in cdf) or ( - "pointing_start_met" in cdf and "pointing_end_met" in cdf), "Expected the pset to have either epoch_delta or pointing_start_met and pointing_end_met defined" - - pointing_start_met = cdf["pointing_start_met"][...] if "pointing_start_met" in cdf else None - pointing_end_met = cdf["pointing_end_met"][...] if "pointing_end_met" in cdf else None - epoch_delta = cdf["epoch_delta"][...] if "epoch_delta" in cdf else met_to_ttj2000ns( - pointing_end_met) - met_to_ttj2000ns(pointing_start_met) - return InputRectangularPointingSet(epoch=cdf["epoch"][0], - epoch_delta=epoch_delta, - epoch_j2000=cdf.raw_var("epoch")[...], - repointing=repointing, - exposure_times=exposure_time_variable[...], - esa_energy_step=cdf["esa_energy_step"][...], - pointing_start_met=pointing_start_met, - pointing_end_met=pointing_end_met, - hae_longitude=cdf['hae_longitude'][...], - hae_latitude=cdf['hae_latitude'][...]) + "pointing_start_met" in cdf and "pointing_end_met" in cdf + ), ( + "Expected the pset to have either epoch_delta or pointing_start_met and pointing_end_met defined" + ) + + pointing_start_met = ( + cdf["pointing_start_met"][...] if "pointing_start_met" in cdf else None + ) + pointing_end_met = ( + cdf["pointing_end_met"][...] if "pointing_end_met" in cdf else None + ) + epoch_delta = ( + cdf["epoch_delta"][...] + if "epoch_delta" in cdf + else met_to_ttj2000ns(pointing_end_met) + - met_to_ttj2000ns(pointing_start_met) + ) + return InputRectangularPointingSet( + epoch=cdf["epoch"][0], + epoch_delta=epoch_delta, + epoch_j2000=cdf.raw_var("epoch")[...], + repointing=repointing, + exposure_times=exposure_time_variable[...], + esa_energy_step=cdf["esa_energy_step"][...], + pointing_start_met=pointing_start_met, + pointing_end_met=pointing_end_met, + hae_longitude=cdf["hae_longitude"][...], + hae_latitude=cdf["hae_latitude"][...], + ) def read_glows_l3e_data(cdf_path: Union[Path, str]) -> GlowsL3eRectangularMapInputData: diff --git a/imap_l3_processing/hit/l3/hit_l3_sectored_dependencies.py b/imap_l3_processing/hit/l3/hit_l3_sectored_dependencies.py index e07fdc78e..002e38cf8 100644 --- a/imap_l3_processing/hit/l3/hit_l3_sectored_dependencies.py +++ b/imap_l3_processing/hit/l3/hit_l3_sectored_dependencies.py @@ -21,12 +21,18 @@ class HITL3SectoredDependencies: mag_is_preliminary: bool = False @classmethod - def fetch_dependencies(cls, dependencies: ProcessingInputCollection) -> HITL3SectoredDependencies: - hit_data_dependency = dependencies.get_file_paths(source="hit", descriptor=HIT_L2_DESCRIPTOR) - mag_l2_dependencies = dependencies.get_file_paths(source="mag", data_type="l2", - descriptor=MAG_L1D_DESCRIPTOR) - mag_l1d_dependencies = dependencies.get_file_paths(source="mag", data_type="l1d", - descriptor=MAG_L1D_DESCRIPTOR) + def fetch_dependencies( + cls, dependencies: ProcessingInputCollection + ) -> HITL3SectoredDependencies: + hit_data_dependency = dependencies.get_file_paths( + source="hit", descriptor=HIT_L2_DESCRIPTOR + ) + mag_l2_dependencies = dependencies.get_file_paths( + source="mag", data_type="l2", descriptor=MAG_L1D_DESCRIPTOR + ) + mag_l1d_dependencies = dependencies.get_file_paths( + source="mag", data_type="l1d", descriptor=MAG_L1D_DESCRIPTOR + ) mag_dependencies = [*mag_l2_dependencies, *mag_l1d_dependencies] mag_is_preliminary = len(mag_l2_dependencies) == 0 @@ -37,5 +43,6 @@ def fetch_dependencies(cls, dependencies: ProcessingInputCollection) -> HITL3Sec mag_data = read_mag_data(mag_data_path) hit_data = read_l2_hit_data(hit_data_path) - return HITL3SectoredDependencies(data=hit_data, mag_data=mag_data, - mag_is_preliminary=mag_is_preliminary) + return HITL3SectoredDependencies( + data=hit_data, mag_data=mag_data, mag_is_preliminary=mag_is_preliminary + ) diff --git a/imap_l3_processing/hit/l3/hit_processor.py b/imap_l3_processing/hit/l3/hit_processor.py index 3b83da0ba..3b20971c5 100644 --- a/imap_l3_processing/hit/l3/hit_processor.py +++ b/imap_l3_processing/hit/l3/hit_processor.py @@ -3,19 +3,31 @@ import numpy as np from imap_data_access.processing_input import ProcessingInputCollection -from imap_l3_processing.constants import UNSIGNED_INT1_FILL_VALUE, UNSIGNED_INT2_FILL_VALUE -from imap_l3_processing.hit.l3.hit_l3_sectored_dependencies import HITL3SectoredDependencies +from imap_l3_processing.constants import ( + UNSIGNED_INT1_FILL_VALUE, + UNSIGNED_INT2_FILL_VALUE, +) +from imap_l3_processing.hit.l3.hit_l3_sectored_dependencies import ( + HITL3SectoredDependencies, +) from imap_l3_processing.hit.l3.models import HitDirectEventDataProduct from imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies import HitL3PhaDependencies from imap_l3_processing.hit.l3.pha.pha_event_reader import PHAEventReader, RawPHAEvent from imap_l3_processing.hit.l3.pha.science.calculate_pha import process_pha_event from imap_l3_processing.hit.l3.sectored_products.models import HitPitchAngleDataProduct -from imap_l3_processing.hit.l3.sectored_products.science.sectored_products_algorithms import get_sector_unit_vectors, \ - get_hit_bin_polar_coordinates +from imap_l3_processing.hit.l3.sectored_products.science.sectored_products_algorithms import ( + get_sector_unit_vectors, + get_hit_bin_polar_coordinates, +) from imap_l3_processing.hit.quality_flags import HitL3Flags from imap_l3_processing.models import InputMetadata -from imap_l3_processing.pitch_angles import calculate_unit_vector, calculate_pitch_angle, calculate_gyrophase, \ - rotate_particle_vectors_from_hit_despun_to_imap_despun, rebin_by_pitch_angle_and_gyrophase +from imap_l3_processing.pitch_angles import ( + calculate_unit_vector, + calculate_pitch_angle, + calculate_gyrophase, + rotate_particle_vectors_from_hit_despun_to_imap_despun, + rebin_by_pitch_angle_and_gyrophase, +) from imap_l3_processing.processor import Processor from imap_l3_processing.utils import save_data @@ -23,37 +35,51 @@ class HitProcessor(Processor): - def __init__(self, dependencies: ProcessingInputCollection, input_metadata: InputMetadata): + def __init__( + self, dependencies: ProcessingInputCollection, input_metadata: InputMetadata + ): super().__init__(dependencies, input_metadata) def process(self): if self.input_metadata.descriptor == "macropixel": - dependencies = HITL3SectoredDependencies.fetch_dependencies(self.dependencies) + dependencies = HITL3SectoredDependencies.fetch_dependencies( + self.dependencies + ) pitch_angle_data_product = self.process_pitch_angle_product(dependencies) pitch_angle_data_product.parent_file_names = self.get_parent_file_names() cdf_file_path = save_data(pitch_angle_data_product) return [cdf_file_path] elif self.input_metadata.descriptor == "direct-events": - direct_event_dependencies = HitL3PhaDependencies.fetch_dependencies(self.dependencies) - direct_event_data_product = self.process_direct_event_product(direct_event_dependencies) + direct_event_dependencies = HitL3PhaDependencies.fetch_dependencies( + self.dependencies + ) + direct_event_data_product = self.process_direct_event_product( + direct_event_dependencies + ) direct_event_data_product.parent_file_names = self.get_parent_file_names() cdf_file_path = save_data(direct_event_data_product) return [cdf_file_path] else: raise ValueError( - f"Don't know how to generate '{self.input_metadata.descriptor}' /n Known HIT l3 data products: 'macropixel', 'direct-events'.") + f"Don't know how to generate '{self.input_metadata.descriptor}' /n Known HIT l3 data products: 'macropixel', 'direct-events'." + ) - def process_direct_event_product(self, - direct_event_dependencies: HitL3PhaDependencies) -> HitDirectEventDataProduct: + def process_direct_event_product( + self, direct_event_dependencies: HitL3PhaDependencies + ) -> HitDirectEventDataProduct: epochs = [] raw_pha_events: list[RawPHAEvent] = [] - for epoch, event_binary in zip(direct_event_dependencies.hit_l1_data.epoch, - direct_event_dependencies.hit_l1_data.event_binary): + for epoch, event_binary in zip( + direct_event_dependencies.hit_l1_data.epoch, + direct_event_dependencies.hit_l1_data.event_binary, + ): try: event_raw_pha_events = PHAEventReader.read_all_pha_events(event_binary) except Exception: - logger.warning(f"Could not read all pha events for epoch {epoch}", exc_info=True) + logger.warning( + f"Could not read all pha events for epoch {epoch}", exc_info=True + ) continue epochs += [epoch] * len(event_raw_pha_events) @@ -64,9 +90,15 @@ def process_direct_event_product(self, e_delta = np.full(shape=(len(raw_pha_events)), fill_value=np.nan) e_prime = np.full(shape=(len(raw_pha_events)), fill_value=np.nan) detected_range = np.full(shape=(len(raw_pha_events)), fill_value=np.nan) - particle_id = np.full(shape=(len(raw_pha_events)), fill_value=UNSIGNED_INT2_FILL_VALUE) - priority_buffer_number = np.full(shape=(len(raw_pha_events)), fill_value=UNSIGNED_INT1_FILL_VALUE) - latency = np.full(shape=(len(raw_pha_events)), fill_value=UNSIGNED_INT1_FILL_VALUE) + particle_id = np.full( + shape=(len(raw_pha_events)), fill_value=UNSIGNED_INT2_FILL_VALUE + ) + priority_buffer_number = np.full( + shape=(len(raw_pha_events)), fill_value=UNSIGNED_INT1_FILL_VALUE + ) + latency = np.full( + shape=(len(raw_pha_events)), fill_value=UNSIGNED_INT1_FILL_VALUE + ) stim_tag = np.full(shape=(len(raw_pha_events)), fill_value=False) long_event_flag = np.full(shape=(len(raw_pha_events)), fill_value=False) haz_tag = np.full(shape=(len(raw_pha_events)), fill_value=False) @@ -74,17 +106,29 @@ def process_direct_event_product(self, has_unread_adcs = np.full(shape=(len(raw_pha_events)), fill_value=False) culling_flag = np.full(shape=(len(raw_pha_events)), fill_value=False) - pha_value = np.full(shape=(len(raw_pha_events), 64), fill_value=UNSIGNED_INT2_FILL_VALUE) + pha_value = np.full( + shape=(len(raw_pha_events), 64), fill_value=UNSIGNED_INT2_FILL_VALUE + ) energy_at_detector = np.full(shape=(len(raw_pha_events), 64), fill_value=np.nan) is_low_gain = np.full(shape=(len(raw_pha_events), 64), fill_value=False) - detector_flags = np.full(shape=(len(raw_pha_events)), fill_value=UNSIGNED_INT2_FILL_VALUE) - deindex = np.full(shape=(len(raw_pha_events)), fill_value=UNSIGNED_INT2_FILL_VALUE) - epindex = np.full(shape=(len(raw_pha_events)), fill_value=UNSIGNED_INT2_FILL_VALUE) + detector_flags = np.full( + shape=(len(raw_pha_events)), fill_value=UNSIGNED_INT2_FILL_VALUE + ) + deindex = np.full( + shape=(len(raw_pha_events)), fill_value=UNSIGNED_INT2_FILL_VALUE + ) + epindex = np.full( + shape=(len(raw_pha_events)), fill_value=UNSIGNED_INT2_FILL_VALUE + ) stim_gain = np.full(shape=(len(raw_pha_events)), fill_value=False) a_l_stim = np.full(shape=(len(raw_pha_events)), fill_value=False) - stim_step = np.full(shape=(len(raw_pha_events)), fill_value=UNSIGNED_INT1_FILL_VALUE) - dac_value = np.full(shape=(len(raw_pha_events)), fill_value=UNSIGNED_INT2_FILL_VALUE) + stim_step = np.full( + shape=(len(raw_pha_events)), fill_value=UNSIGNED_INT1_FILL_VALUE + ) + dac_value = np.full( + shape=(len(raw_pha_events)), fill_value=UNSIGNED_INT2_FILL_VALUE + ) for i, raw_event in enumerate(raw_pha_events): event_output = process_pha_event( @@ -92,7 +136,7 @@ def process_direct_event_product(self, direct_event_dependencies.cosine_correction_lookup, direct_event_dependencies.gain_lookup, direct_event_dependencies.range_fit_lookup, - direct_event_dependencies.event_type_lookup + direct_event_dependencies.event_type_lookup, ) charge[i] = event_output.charge @@ -115,7 +159,9 @@ def process_direct_event_product(self, has_unread_adcs[i] = raw_event.has_unread_adcs culling_flag[i] = raw_event.culling_flag - for event_energy_at_detector, word in zip(event_output.energies, raw_event.pha_words): + for event_energy_at_detector, word in zip( + event_output.energies, raw_event.pha_words + ): pha_value[i, word.detector.address] = word.adc_value energy_at_detector[i, word.detector.address] = event_energy_at_detector is_low_gain[i, word.detector.address] = word.is_low_gain @@ -161,7 +207,9 @@ def process_direct_event_product(self, input_metadata=self.input_metadata, ) - def process_pitch_angle_product(self, dependencies: HITL3SectoredDependencies) -> HitPitchAngleDataProduct: + def process_pitch_angle_product( + self, dependencies: HITL3SectoredDependencies + ) -> HitPitchAngleDataProduct: number_of_pitch_angle_bins = 8 number_of_gyrophase_bins = 15 @@ -170,45 +218,102 @@ def process_pitch_angle_product(self, dependencies: HITL3SectoredDependencies) - input_intensity_data_by_species = { "hydrogen": (hit_data.h, hit_data.delta_plus_h, hit_data.delta_minus_h), - "helium4": (hit_data.he4, hit_data.delta_plus_he4, hit_data.delta_minus_he4), + "helium4": ( + hit_data.he4, + hit_data.delta_plus_he4, + hit_data.delta_minus_he4, + ), "cno": (hit_data.cno, hit_data.delta_plus_cno, hit_data.delta_minus_cno), - "NeMgSi": (hit_data.nemgsi, hit_data.delta_plus_nemgsi, hit_data.delta_minus_nemgsi), - "iron": (hit_data.fe, hit_data.delta_plus_fe, hit_data.delta_minus_fe)} + "NeMgSi": ( + hit_data.nemgsi, + hit_data.delta_plus_nemgsi, + hit_data.delta_minus_nemgsi, + ), + "iron": (hit_data.fe, hit_data.delta_plus_fe, hit_data.delta_minus_fe), + } epoch_count = len(hit_data.epoch) cno_energy_count = hit_data.cno.shape[1] he4_energy_count = hit_data.he4.shape[1] h_energy_count = hit_data.h.shape[1] fe_energy_count = hit_data.fe.shape[1] nemgsi_energy_count = hit_data.nemgsi.shape[1] - rebinned_pa_gyro_intensity_by_species = {"cno": self._create_nan_array( - (epoch_count, cno_energy_count, number_of_pitch_angle_bins, number_of_gyrophase_bins)), - "helium4": self._create_nan_array((epoch_count, he4_energy_count, - number_of_pitch_angle_bins, - number_of_gyrophase_bins)), - "hydrogen": self._create_nan_array((epoch_count, h_energy_count, - number_of_pitch_angle_bins, - number_of_gyrophase_bins)), - "iron": self._create_nan_array((epoch_count, fe_energy_count, - number_of_pitch_angle_bins, - number_of_gyrophase_bins)), - "NeMgSi": self._create_nan_array((epoch_count, nemgsi_energy_count, - number_of_pitch_angle_bins, - number_of_gyrophase_bins))} + rebinned_pa_gyro_intensity_by_species = { + "cno": self._create_nan_array( + ( + epoch_count, + cno_energy_count, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ) + ), + "helium4": self._create_nan_array( + ( + epoch_count, + he4_energy_count, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ) + ), + "hydrogen": self._create_nan_array( + ( + epoch_count, + h_energy_count, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ) + ), + "iron": self._create_nan_array( + ( + epoch_count, + fe_energy_count, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ) + ), + "NeMgSi": self._create_nan_array( + ( + epoch_count, + nemgsi_energy_count, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ) + ), + } rebinned_pa_only_intensity_by_species = { - "cno": self._create_nan_array((epoch_count, cno_energy_count, number_of_pitch_angle_bins)), - "helium4": self._create_nan_array((epoch_count, he4_energy_count, number_of_pitch_angle_bins)), - "hydrogen": self._create_nan_array((epoch_count, h_energy_count, number_of_pitch_angle_bins)), - "iron": self._create_nan_array((epoch_count, fe_energy_count, number_of_pitch_angle_bins)), - "NeMgSi": self._create_nan_array((epoch_count, nemgsi_energy_count, number_of_pitch_angle_bins))} - - sector_unit_vectors = np.transpose(get_sector_unit_vectors(hit_data.zenith, hit_data.azimuth), (1, 0, 2)) + "cno": self._create_nan_array( + (epoch_count, cno_energy_count, number_of_pitch_angle_bins) + ), + "helium4": self._create_nan_array( + (epoch_count, he4_energy_count, number_of_pitch_angle_bins) + ), + "hydrogen": self._create_nan_array( + (epoch_count, h_energy_count, number_of_pitch_angle_bins) + ), + "iron": self._create_nan_array( + (epoch_count, fe_energy_count, number_of_pitch_angle_bins) + ), + "NeMgSi": self._create_nan_array( + (epoch_count, nemgsi_energy_count, number_of_pitch_angle_bins) + ), + } + + sector_unit_vectors = np.transpose( + get_sector_unit_vectors(hit_data.zenith, hit_data.azimuth), (1, 0, 2) + ) particle_unit_vectors = -sector_unit_vectors - rotated_particle_unit_vectors = rotate_particle_vectors_from_hit_despun_to_imap_despun(particle_unit_vectors) + rotated_particle_unit_vectors = ( + rotate_particle_vectors_from_hit_despun_to_imap_despun( + particle_unit_vectors + ) + ) - pitch_angles, gyrophases, pitch_angle_deltas, gyrophase_deltas = get_hit_bin_polar_coordinates( - number_of_pitch_angle_bins, number_of_gyrophase_bins) + pitch_angles, gyrophases, pitch_angle_deltas, gyrophase_deltas = ( + get_hit_bin_polar_coordinates( + number_of_pitch_angle_bins, number_of_gyrophase_bins + ) + ) averaged_mag_data = mag_data.rebin_to(hit_data.epoch, hit_data.epoch_delta) hit_flags = np.full(len(hit_data.epoch), HitL3Flags.NONE) @@ -219,82 +324,115 @@ def process_pitch_angle_product(self, dependencies: HITL3SectoredDependencies) - for time_index, average_mag_vector in enumerate(averaged_mag_data): mag_unit_vector = calculate_unit_vector(average_mag_vector) - input_bin_pitch_angles = calculate_pitch_angle(rotated_particle_unit_vectors, mag_unit_vector) - input_bin_gyrophases = calculate_gyrophase(rotated_particle_unit_vectors, mag_unit_vector) + input_bin_pitch_angles = calculate_pitch_angle( + rotated_particle_unit_vectors, mag_unit_vector + ) + input_bin_gyrophases = calculate_gyrophase( + rotated_particle_unit_vectors, mag_unit_vector + ) measurement_pitch_angle.append(input_bin_pitch_angles) measurement_gyrophase.append(input_bin_gyrophases) for species, intensity in input_intensity_data_by_species.items(): - rebinned_result = rebin_by_pitch_angle_and_gyrophase(intensity[0][time_index], intensity[1][time_index], - intensity[2][time_index], input_bin_pitch_angles, - input_bin_gyrophases, number_of_pitch_angle_bins, - number_of_gyrophase_bins) - - intensity_by_pa_gyro, intensity_delta_plus_by_pa_gyro, intensity_delta_minus_by_pa_gyro = rebinned_result[ - 0:3] - intensity_by_pa, intensity_delta_plus_by_pa, intensity_delta_minus_by_pa = rebinned_result[3:6] - - rebinned_pa_gyro_intensity_by_species[species][0][time_index, ...] = intensity_by_pa_gyro - rebinned_pa_gyro_intensity_by_species[species][1][time_index, ...] = intensity_delta_plus_by_pa_gyro - rebinned_pa_gyro_intensity_by_species[species][2][time_index, ...] = intensity_delta_minus_by_pa_gyro - rebinned_pa_only_intensity_by_species[species][0][time_index, ...] = intensity_by_pa - rebinned_pa_only_intensity_by_species[species][1][time_index, ...] = intensity_delta_plus_by_pa - rebinned_pa_only_intensity_by_species[species][2][time_index, ...] = intensity_delta_minus_by_pa - - return HitPitchAngleDataProduct(self.input_metadata, hit_data.epoch, - hit_data.epoch_delta, pitch_angles, pitch_angle_deltas, - gyrophases, - gyrophase_deltas, - rebinned_pa_gyro_intensity_by_species["hydrogen"][0], - rebinned_pa_gyro_intensity_by_species["hydrogen"][1], - rebinned_pa_gyro_intensity_by_species["hydrogen"][2], - rebinned_pa_only_intensity_by_species["hydrogen"][0], - rebinned_pa_only_intensity_by_species["hydrogen"][1], - rebinned_pa_only_intensity_by_species["hydrogen"][2], - hit_data.h_energy, - hit_data.h_energy_delta_plus, - hit_data.h_energy_delta_minus, - rebinned_pa_gyro_intensity_by_species["helium4"][0], - rebinned_pa_gyro_intensity_by_species["helium4"][1], - rebinned_pa_gyro_intensity_by_species["helium4"][2], - rebinned_pa_only_intensity_by_species["helium4"][0], - rebinned_pa_only_intensity_by_species["helium4"][1], - rebinned_pa_only_intensity_by_species["helium4"][2], - hit_data.he4_energy, - hit_data.he4_energy_delta_plus, - hit_data.he4_energy_delta_minus, - rebinned_pa_gyro_intensity_by_species["cno"][0], - rebinned_pa_gyro_intensity_by_species["cno"][1], - rebinned_pa_gyro_intensity_by_species["cno"][2], - rebinned_pa_only_intensity_by_species["cno"][0], - rebinned_pa_only_intensity_by_species["cno"][1], - rebinned_pa_only_intensity_by_species["cno"][2], - hit_data.cno_energy, - hit_data.cno_energy_delta_plus, - hit_data.cno_energy_delta_minus, - rebinned_pa_gyro_intensity_by_species["NeMgSi"][0], - rebinned_pa_gyro_intensity_by_species["NeMgSi"][1], - rebinned_pa_gyro_intensity_by_species["NeMgSi"][2], - rebinned_pa_only_intensity_by_species["NeMgSi"][0], - rebinned_pa_only_intensity_by_species["NeMgSi"][1], - rebinned_pa_only_intensity_by_species["NeMgSi"][2], - hit_data.nemgsi_energy, - hit_data.nemgsi_energy_delta_plus, - hit_data.nemgsi_energy_delta_minus, - rebinned_pa_gyro_intensity_by_species["iron"][0], - rebinned_pa_gyro_intensity_by_species["iron"][1], - rebinned_pa_gyro_intensity_by_species["iron"][2], - rebinned_pa_only_intensity_by_species["iron"][0], - rebinned_pa_only_intensity_by_species["iron"][1], - rebinned_pa_only_intensity_by_species["iron"][2], - hit_data.fe_energy, - hit_data.fe_energy_delta_plus, - hit_data.fe_energy_delta_minus, - np.array(measurement_pitch_angle), - np.array(measurement_gyrophase), - azimuth=hit_data.azimuth, - zenith=hit_data.zenith, - hit_flags=hit_flags) + rebinned_result = rebin_by_pitch_angle_and_gyrophase( + intensity[0][time_index], + intensity[1][time_index], + intensity[2][time_index], + input_bin_pitch_angles, + input_bin_gyrophases, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ) + + ( + intensity_by_pa_gyro, + intensity_delta_plus_by_pa_gyro, + intensity_delta_minus_by_pa_gyro, + ) = rebinned_result[0:3] + ( + intensity_by_pa, + intensity_delta_plus_by_pa, + intensity_delta_minus_by_pa, + ) = rebinned_result[3:6] + + rebinned_pa_gyro_intensity_by_species[species][0][time_index, ...] = ( + intensity_by_pa_gyro + ) + rebinned_pa_gyro_intensity_by_species[species][1][time_index, ...] = ( + intensity_delta_plus_by_pa_gyro + ) + rebinned_pa_gyro_intensity_by_species[species][2][time_index, ...] = ( + intensity_delta_minus_by_pa_gyro + ) + rebinned_pa_only_intensity_by_species[species][0][time_index, ...] = ( + intensity_by_pa + ) + rebinned_pa_only_intensity_by_species[species][1][time_index, ...] = ( + intensity_delta_plus_by_pa + ) + rebinned_pa_only_intensity_by_species[species][2][time_index, ...] = ( + intensity_delta_minus_by_pa + ) + + return HitPitchAngleDataProduct( + self.input_metadata, + hit_data.epoch, + hit_data.epoch_delta, + pitch_angles, + pitch_angle_deltas, + gyrophases, + gyrophase_deltas, + rebinned_pa_gyro_intensity_by_species["hydrogen"][0], + rebinned_pa_gyro_intensity_by_species["hydrogen"][1], + rebinned_pa_gyro_intensity_by_species["hydrogen"][2], + rebinned_pa_only_intensity_by_species["hydrogen"][0], + rebinned_pa_only_intensity_by_species["hydrogen"][1], + rebinned_pa_only_intensity_by_species["hydrogen"][2], + hit_data.h_energy, + hit_data.h_energy_delta_plus, + hit_data.h_energy_delta_minus, + rebinned_pa_gyro_intensity_by_species["helium4"][0], + rebinned_pa_gyro_intensity_by_species["helium4"][1], + rebinned_pa_gyro_intensity_by_species["helium4"][2], + rebinned_pa_only_intensity_by_species["helium4"][0], + rebinned_pa_only_intensity_by_species["helium4"][1], + rebinned_pa_only_intensity_by_species["helium4"][2], + hit_data.he4_energy, + hit_data.he4_energy_delta_plus, + hit_data.he4_energy_delta_minus, + rebinned_pa_gyro_intensity_by_species["cno"][0], + rebinned_pa_gyro_intensity_by_species["cno"][1], + rebinned_pa_gyro_intensity_by_species["cno"][2], + rebinned_pa_only_intensity_by_species["cno"][0], + rebinned_pa_only_intensity_by_species["cno"][1], + rebinned_pa_only_intensity_by_species["cno"][2], + hit_data.cno_energy, + hit_data.cno_energy_delta_plus, + hit_data.cno_energy_delta_minus, + rebinned_pa_gyro_intensity_by_species["NeMgSi"][0], + rebinned_pa_gyro_intensity_by_species["NeMgSi"][1], + rebinned_pa_gyro_intensity_by_species["NeMgSi"][2], + rebinned_pa_only_intensity_by_species["NeMgSi"][0], + rebinned_pa_only_intensity_by_species["NeMgSi"][1], + rebinned_pa_only_intensity_by_species["NeMgSi"][2], + hit_data.nemgsi_energy, + hit_data.nemgsi_energy_delta_plus, + hit_data.nemgsi_energy_delta_minus, + rebinned_pa_gyro_intensity_by_species["iron"][0], + rebinned_pa_gyro_intensity_by_species["iron"][1], + rebinned_pa_gyro_intensity_by_species["iron"][2], + rebinned_pa_only_intensity_by_species["iron"][0], + rebinned_pa_only_intensity_by_species["iron"][1], + rebinned_pa_only_intensity_by_species["iron"][2], + hit_data.fe_energy, + hit_data.fe_energy_delta_plus, + hit_data.fe_energy_delta_minus, + np.array(measurement_pitch_angle), + np.array(measurement_gyrophase), + azimuth=hit_data.azimuth, + zenith=hit_data.zenith, + hit_flags=hit_flags, + ) @staticmethod def _create_nan_array(shape) -> tuple[np.array, np.array, np.array]: diff --git a/imap_l3_processing/hit/l3/models.py b/imap_l3_processing/hit/l3/models.py index f65b7048f..596f98f48 100644 --- a/imap_l3_processing/hit/l3/models.py +++ b/imap_l3_processing/hit/l3/models.py @@ -118,7 +118,9 @@ def to_data_product_variables(self) -> list[DataProductVariable]: DataProductVariable(E_PRIME_VAR_NAME, self.e_prime), DataProductVariable(DETECTED_RANGE_VAR_NAME, self.detected_range), DataProductVariable(PARTICLE_ID_VAR_NAME, self.particle_id), - DataProductVariable(PRIORITY_BUFFER_NUMBER_VAR_NAME, self.priority_buffer_number), + DataProductVariable( + PRIORITY_BUFFER_NUMBER_VAR_NAME, self.priority_buffer_number + ), DataProductVariable(LATENCY_VAR_NAME, self.latency), DataProductVariable(STIM_TAG_VAR_NAME, self.stim_tag), DataProductVariable(LONG_EVENT_FLAG_VAR_NAME, self.long_event_flag), @@ -137,7 +139,7 @@ def to_data_product_variables(self) -> list[DataProductVariable]: DataProductVariable(STIM_STEP_VAR_NAME, self.stim_step), DataProductVariable(DAC_VALUE_VAR_NAME, self.dac_value), DataProductVariable(HIT_FLAGS_CDF_VAR_NAME, self.hit_flags), - DataProductVariable(DETECTOR_ID_VAR_NAME, np.arange(0, 64)) + DataProductVariable(DETECTOR_ID_VAR_NAME, np.arange(0, 64)), ] diff --git a/imap_l3_processing/hit/l3/pha/hit_l3_pha_dependencies.py b/imap_l3_processing/hit/l3/pha/hit_l3_pha_dependencies.py index fdc322e69..a6489685e 100644 --- a/imap_l3_processing/hit/l3/pha/hit_l3_pha_dependencies.py +++ b/imap_l3_processing/hit/l3/pha/hit_l3_pha_dependencies.py @@ -4,9 +4,13 @@ from imap_data_access.processing_input import ProcessingInputCollection from imap_l3_processing.hit.l3.models import HitL1Data -from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import CosineCorrectionLookupTable +from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import ( + CosineCorrectionLookupTable, +) from imap_l3_processing.hit.l3.pha.science.gain_lookup_table import GainLookupTable -from imap_l3_processing.hit.l3.pha.science.hit_event_type_lookup import HitEventTypeLookup +from imap_l3_processing.hit.l3.pha.science.hit_event_type_lookup import ( + HitEventTypeLookup, +) from imap_l3_processing.hit.l3.pha.science.range_fit_lookup import RangeFitLookup HIT_L1A_EVENT_DESCRIPTOR = "direct-events" @@ -38,8 +42,12 @@ class HitL3PhaDependencies: @classmethod def fetch_dependencies(cls, dependencies: ProcessingInputCollection): try: - l1_events_dependency = next(dependency for dependency in - dependencies.get_file_paths(source="hit", descriptor=HIT_L1A_EVENT_DESCRIPTOR)) + l1_events_dependency = next( + dependency + for dependency in dependencies.get_file_paths( + source="hit", descriptor=HIT_L1A_EVENT_DESCRIPTOR + ) + ) except StopIteration: raise ValueError(f"Missing {HIT_L1A_EVENT_DESCRIPTOR} dependency.") @@ -47,53 +55,106 @@ def fetch_dependencies(cls, dependencies: ProcessingInputCollection): hit_l1_data = HitL1Data.read_from_cdf(l1_events_cdf_path) range_2A_cosine_lookup_path = download( - dependencies.get_file_paths(source="hit", descriptor=HIT_L3_RANGE_2A_COSINE_LOOKUP_DESCRIPTOR)[0]) + dependencies.get_file_paths( + source="hit", descriptor=HIT_L3_RANGE_2A_COSINE_LOOKUP_DESCRIPTOR + )[0] + ) range_3A_cosine_lookup_path = download( - dependencies.get_file_paths(source="hit", descriptor=HIT_L3_RANGE_3A_COSINE_LOOKUP_DESCRIPTOR)[0]) + dependencies.get_file_paths( + source="hit", descriptor=HIT_L3_RANGE_3A_COSINE_LOOKUP_DESCRIPTOR + )[0] + ) range_4A_cosine_lookup_path = download( - dependencies.get_file_paths(source="hit", descriptor=HIT_L3_RANGE_4A_COSINE_LOOKUP_DESCRIPTOR)[0]) + dependencies.get_file_paths( + source="hit", descriptor=HIT_L3_RANGE_4A_COSINE_LOOKUP_DESCRIPTOR + )[0] + ) range_2B_cosine_lookup_path = download( - dependencies.get_file_paths(source="hit", descriptor=HIT_L3_RANGE_2B_COSINE_LOOKUP_DESCRIPTOR)[0]) + dependencies.get_file_paths( + source="hit", descriptor=HIT_L3_RANGE_2B_COSINE_LOOKUP_DESCRIPTOR + )[0] + ) range_3B_cosine_lookup_path = download( - dependencies.get_file_paths(source="hit", descriptor=HIT_L3_RANGE_3B_COSINE_LOOKUP_DESCRIPTOR)[0]) + dependencies.get_file_paths( + source="hit", descriptor=HIT_L3_RANGE_3B_COSINE_LOOKUP_DESCRIPTOR + )[0] + ) range_4B_cosine_lookup_path = download( - dependencies.get_file_paths(source="hit", descriptor=HIT_L3_RANGE_4B_COSINE_LOOKUP_DESCRIPTOR)[0]) + dependencies.get_file_paths( + source="hit", descriptor=HIT_L3_RANGE_4B_COSINE_LOOKUP_DESCRIPTOR + )[0] + ) lo_gain_lookup_path = download( - dependencies.get_file_paths(source="hit", descriptor=HIT_L3_LO_GAIN_LOOKUP_DESCRIPTOR)[0]) + dependencies.get_file_paths( + source="hit", descriptor=HIT_L3_LO_GAIN_LOOKUP_DESCRIPTOR + )[0] + ) hi_gain_lookup_path = download( - dependencies.get_file_paths(source="hit", descriptor=HIT_L3_HI_GAIN_LOOKUP_DESCRIPTOR)[0]) + dependencies.get_file_paths( + source="hit", descriptor=HIT_L3_HI_GAIN_LOOKUP_DESCRIPTOR + )[0] + ) range_2A_charge_fit_lookup_path = download( - dependencies.get_file_paths(source="hit", descriptor=HIT_L3_RANGE_2A_CHARGE_FIT_LOOKUP_DESCRIPTOR)[0]) + dependencies.get_file_paths( + source="hit", descriptor=HIT_L3_RANGE_2A_CHARGE_FIT_LOOKUP_DESCRIPTOR + )[0] + ) range_3A_charge_fit_lookup_path = download( - dependencies.get_file_paths(source="hit", descriptor=HIT_L3_RANGE_3A_CHARGE_FIT_LOOKUP_DESCRIPTOR)[0]) + dependencies.get_file_paths( + source="hit", descriptor=HIT_L3_RANGE_3A_CHARGE_FIT_LOOKUP_DESCRIPTOR + )[0] + ) range_4A_charge_fit_lookup_path = download( - dependencies.get_file_paths(source="hit", descriptor=HIT_L3_RANGE_4A_CHARGE_FIT_LOOKUP_DESCRIPTOR)[0]) + dependencies.get_file_paths( + source="hit", descriptor=HIT_L3_RANGE_4A_CHARGE_FIT_LOOKUP_DESCRIPTOR + )[0] + ) range_2B_charge_fit_lookup_path = download( - dependencies.get_file_paths(source="hit", descriptor=HIT_L3_RANGE_2B_CHARGE_FIT_LOOKUP_DESCRIPTOR)[0]) + dependencies.get_file_paths( + source="hit", descriptor=HIT_L3_RANGE_2B_CHARGE_FIT_LOOKUP_DESCRIPTOR + )[0] + ) range_3B_charge_fit_lookup_path = download( - dependencies.get_file_paths(source="hit", descriptor=HIT_L3_RANGE_3B_CHARGE_FIT_LOOKUP_DESCRIPTOR)[0]) + dependencies.get_file_paths( + source="hit", descriptor=HIT_L3_RANGE_3B_CHARGE_FIT_LOOKUP_DESCRIPTOR + )[0] + ) range_4B_charge_fit_lookup_path = download( - dependencies.get_file_paths(source="hit", descriptor=HIT_L3_RANGE_4B_CHARGE_FIT_LOOKUP_DESCRIPTOR)[0]) + dependencies.get_file_paths( + source="hit", descriptor=HIT_L3_RANGE_4B_CHARGE_FIT_LOOKUP_DESCRIPTOR + )[0] + ) event_type_lookup_path = download( - dependencies.get_file_paths(source="hit", descriptor=HIT_L3_EVENT_TYPE_LOOKUP_DESCRIPTOR)[0]) + dependencies.get_file_paths( + source="hit", descriptor=HIT_L3_EVENT_TYPE_LOOKUP_DESCRIPTOR + )[0] + ) - cosine_lookup_table = CosineCorrectionLookupTable(range_2A_cosine_lookup_path, range_3A_cosine_lookup_path, - range_4A_cosine_lookup_path, - range_2B_cosine_lookup_path, - range_3B_cosine_lookup_path, - range_4B_cosine_lookup_path, ) - gain_lookup_table = GainLookupTable.from_file(hi_gain_lookup_path, lo_gain_lookup_path) - range_fit_lookup_table = RangeFitLookup.from_files(range_2A_charge_fit_lookup_path, - range_3A_charge_fit_lookup_path, - range_4A_charge_fit_lookup_path, - range_2B_charge_fit_lookup_path, - range_3B_charge_fit_lookup_path, - range_4B_charge_fit_lookup_path - ) + cosine_lookup_table = CosineCorrectionLookupTable( + range_2A_cosine_lookup_path, + range_3A_cosine_lookup_path, + range_4A_cosine_lookup_path, + range_2B_cosine_lookup_path, + range_3B_cosine_lookup_path, + range_4B_cosine_lookup_path, + ) + gain_lookup_table = GainLookupTable.from_file( + hi_gain_lookup_path, lo_gain_lookup_path + ) + range_fit_lookup_table = RangeFitLookup.from_files( + range_2A_charge_fit_lookup_path, + range_3A_charge_fit_lookup_path, + range_4A_charge_fit_lookup_path, + range_2B_charge_fit_lookup_path, + range_3B_charge_fit_lookup_path, + range_4B_charge_fit_lookup_path, + ) event_type_lookup_table = HitEventTypeLookup.from_csv(event_type_lookup_path) - return cls(hit_l1_data=hit_l1_data, - cosine_correction_lookup=cosine_lookup_table, - gain_lookup=gain_lookup_table, - range_fit_lookup=range_fit_lookup_table, - event_type_lookup=event_type_lookup_table) + return cls( + hit_l1_data=hit_l1_data, + cosine_correction_lookup=cosine_lookup_table, + gain_lookup=gain_lookup_table, + range_fit_lookup=range_fit_lookup_table, + event_type_lookup=event_type_lookup_table, + ) diff --git a/imap_l3_processing/hit/l3/pha/pha_event_reader.py b/imap_l3_processing/hit/l3/pha/pha_event_reader.py index 5a2f55173..978400d71 100644 --- a/imap_l3_processing/hit/l3/pha/pha_event_reader.py +++ b/imap_l3_processing/hit/l3/pha/pha_event_reader.py @@ -22,7 +22,9 @@ def __str__(self): @classmethod def from_address(cls, address: int): if cls.detector_mapping == {}: - with open(Path(__file__).parent / "address_to_detector.csv") as detector_mapping_file: + with open( + Path(__file__).parent / "address_to_detector.csv" + ) as detector_mapping_file: reader = csv.reader(detector_mapping_file) next(reader) for name, address_str, group in reader: @@ -33,14 +35,22 @@ def from_address(cls, address: int): if detector_name_and_group is not None: detector_name, group = detector_name_and_group layer = int(detector_name[1]) - return cls(layer=layer, - side=detector_name[2], - segment=detector_name[3:], - address=address, - group=group) + return cls( + layer=layer, + side=detector_name[2], + segment=detector_name[3:], + address=address, + group=group, + ) else: - return cls(layer=4, side="A", segment=f"UNKNOWN_{address}", address=address, group="Unknown") + return cls( + layer=4, + side="A", + segment=f"UNKNOWN_{address}", + address=address, + group="Unknown", + ) @dataclass @@ -116,7 +126,9 @@ def read_pha_event(cls, event_bitstream: BitStream) -> RawPHAEvent: delta_e_index = event_bitstream.read("uint:9") e_prime_index = event_bitstream.read("uint:7") detector_flags = event_bitstream.read("uint:8") - extended_header = PHAExtendedHeader(detector_flags, delta_e_index, e_prime_index) + extended_header = PHAExtendedHeader( + detector_flags, delta_e_index, e_prime_index + ) elif stim_tag: if long_event_flag: @@ -128,7 +140,12 @@ def read_pha_event(cls, event_bitstream: BitStream) -> RawPHAEvent: unused = event_bitstream.read("uint:2") stim_gain = event_bitstream.read("uint:1") stim_step = event_bitstream.read("uint:4") - stim_block = StimBlock(stim_step=stim_step, stim_gain=stim_gain, unused=unused, a_l_stim=a_l_stim) + stim_block = StimBlock( + stim_step=stim_step, + stim_gain=stim_gain, + unused=unused, + a_l_stim=a_l_stim, + ) reading_pha = True pha_words = [] @@ -140,30 +157,34 @@ def read_pha_event(cls, event_bitstream: BitStream) -> RawPHAEvent: adc_value = event_bitstream.read("uint:11") reading_pha = not is_last_event - pha_word = PHAWord(adc_overflow=adc_overflow, adc_value=adc_value, - detector=Detector.from_address(adc_detector_address), - is_last_pha=is_last_event, - is_low_gain=is_low_gain) + pha_word = PHAWord( + adc_overflow=adc_overflow, + adc_value=adc_value, + detector=Detector.from_address(adc_detector_address), + is_last_pha=is_last_event, + is_low_gain=is_low_gain, + ) pha_words.append(pha_word) if len(pha_words) % 2 == 1: event_bitstream.read("uint:4") - return RawPHAEvent(particle_id=particle_id, - priority_buffer_num=priority_buffer_num, - stim_tag=stim_tag, - haz_tag=haz_tag, - time_tag=time_tag, - a_b_side_flag=a_b_side_flag, - has_unread_adcs=has_unread_adcs, - long_event_flag=long_event_flag, - culling_flag=culling_flag, - spare=spare, - pha_words=pha_words, - extended_header=extended_header, - extended_stim_header=extended_stim_header, - stim_block=stim_block - ) + return RawPHAEvent( + particle_id=particle_id, + priority_buffer_num=priority_buffer_num, + stim_tag=stim_tag, + haz_tag=haz_tag, + time_tag=time_tag, + a_b_side_flag=a_b_side_flag, + has_unread_adcs=has_unread_adcs, + long_event_flag=long_event_flag, + culling_flag=culling_flag, + spare=spare, + pha_words=pha_words, + extended_header=extended_header, + extended_stim_header=extended_stim_header, + stim_block=stim_block, + ) @classmethod def read_all_pha_events(cls, binary_pha_events: str) -> list[RawPHAEvent]: diff --git a/imap_l3_processing/hit/l3/pha/science/calculate_pha.py b/imap_l3_processing/hit/l3/pha/science/calculate_pha.py index 2c5356613..bc1d54ea1 100644 --- a/imap_l3_processing/hit/l3/pha/science/calculate_pha.py +++ b/imap_l3_processing/hit/l3/pha/science/calculate_pha.py @@ -5,10 +5,19 @@ from numpy import clip from imap_l3_processing.hit.l3.pha.pha_event_reader import PHAWord, RawPHAEvent -from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import DetectedRange, Detector, \ - CosineCorrectionLookupTable, DetectorRange -from imap_l3_processing.hit.l3.pha.science.gain_lookup_table import GainLookupTable, DetectorGain -from imap_l3_processing.hit.l3.pha.science.hit_event_type_lookup import HitEventTypeLookup +from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import ( + DetectedRange, + Detector, + CosineCorrectionLookupTable, + DetectorRange, +) +from imap_l3_processing.hit.l3.pha.science.gain_lookup_table import ( + GainLookupTable, + DetectorGain, +) +from imap_l3_processing.hit.l3.pha.science.hit_event_type_lookup import ( + HitEventTypeLookup, +) from imap_l3_processing.hit.l3.pha.science.range_fit_lookup import RangeFitLookup @@ -39,7 +48,9 @@ def calculate_mev(word: PHAWord, gain_lookup_table: GainLookupTable) -> float: return word.adc_value * gain_coeffs.a + gain_coeffs.b -def analyze_event(event: RawPHAEvent, rule_lookup: HitEventTypeLookup) -> Optional[EventAnalysis]: +def analyze_event( + event: RawPHAEvent, rule_lookup: HitEventTypeLookup +) -> Optional[EventAnalysis]: def get_adc_value(pha_word: PHAWord): return 20 * pha_word.adc_value if pha_word.is_low_gain else pha_word.adc_value @@ -56,33 +67,57 @@ def get_adc_value(pha_word: PHAWord): if rule is not None: highest_value_words_per_group = {} for include_group in rule.included_detector_groups: - unsaturated_words = [word for word in groups_to_words[include_group] if word.adc_value < 2047] + unsaturated_words = [ + word for word in groups_to_words[include_group] if word.adc_value < 2047 + ] if len(unsaturated_words) == 0: return None - highest_value_words_per_group[include_group] = max(unsaturated_words, key=get_adc_value) - - l1_detector = [group for group in rule.included_detector_groups if group[0:3] == f"L1{rule.range.side.name}"][0] - l2_detector = [group for group in rule.included_detector_groups if group[0:3] == f"L2{rule.range.side.name}"][0] + highest_value_words_per_group[include_group] = max( + unsaturated_words, key=get_adc_value + ) + + l1_detector = [ + group + for group in rule.included_detector_groups + if group[0:3] == f"L1{rule.range.side.name}" + ][0] + l2_detector = [ + group + for group in rule.included_detector_groups + if group[0:3] == f"L2{rule.range.side.name}" + ][0] if rule.range.range == DetectorRange.R2: e_prime_group = l2_detector e_delta_group = l1_detector elif rule.range.range == DetectorRange.R3: - e_prime_group = \ - [group for group in rule.included_detector_groups if group[0:3] == f"L3{rule.range.side.name}"][0] + e_prime_group = [ + group + for group in rule.included_detector_groups + if group[0:3] == f"L3{rule.range.side.name}" + ][0] e_delta_group = l2_detector elif rule.range.range == DetectorRange.R4: - opposite_side = 'A' if rule.range.side.name == 'B' else 'B' - e_prime_group = [group for group in rule.included_detector_groups if group[0:3] == f"L3{opposite_side}"][0] - e_delta_group = \ - [group for group in rule.included_detector_groups if group[0:3] == f"L3{rule.range.side.name}"][0] - - return EventAnalysis(range=rule.range, - l1_detector=highest_value_words_per_group[l1_detector].detector, - l2_detector=highest_value_words_per_group[l2_detector].detector, - e_delta_word=highest_value_words_per_group[e_delta_group], - e_prime_word=highest_value_words_per_group[e_prime_group], - words_with_highest_energy=list(highest_value_words_per_group.values())) + opposite_side = "A" if rule.range.side.name == "B" else "B" + e_prime_group = [ + group + for group in rule.included_detector_groups + if group[0:3] == f"L3{opposite_side}" + ][0] + e_delta_group = [ + group + for group in rule.included_detector_groups + if group[0:3] == f"L3{rule.range.side.name}" + ][0] + + return EventAnalysis( + range=rule.range, + l1_detector=highest_value_words_per_group[l1_detector].detector, + l2_detector=highest_value_words_per_group[l2_detector].detector, + e_delta_word=highest_value_words_per_group[e_delta_group], + e_prime_word=highest_value_words_per_group[e_prime_group], + words_with_highest_energy=list(highest_value_words_per_group.values()), + ) @dataclass @@ -93,52 +128,92 @@ class ValidDetectorRange: delta_e_max: float def is_in_range(self, e_prime: float, delta_e: float) -> bool: - return self.e_prime_min <= e_prime <= self.e_prime_max and self.delta_e_min <= delta_e <= self.delta_e_max + return ( + self.e_prime_min <= e_prime <= self.e_prime_max + and self.delta_e_min <= delta_e <= self.delta_e_max + ) valid_ranges = { - DetectorRange.R2: ValidDetectorRange(e_prime_min=0.2, e_prime_max=860, delta_e_min=0.1, delta_e_max=430), - DetectorRange.R3: ValidDetectorRange(e_prime_min=1.0, e_prime_max=4300, delta_e_min=0.2, delta_e_max=860), - DetectorRange.R4: ValidDetectorRange(e_prime_min=1.0, e_prime_max=4300, delta_e_min=1.0, delta_e_max=4300), + DetectorRange.R2: ValidDetectorRange( + e_prime_min=0.2, e_prime_max=860, delta_e_min=0.1, delta_e_max=430 + ), + DetectorRange.R3: ValidDetectorRange( + e_prime_min=1.0, e_prime_max=4300, delta_e_min=0.2, delta_e_max=860 + ), + DetectorRange.R4: ValidDetectorRange( + e_prime_min=1.0, e_prime_max=4300, delta_e_min=1.0, delta_e_max=4300 + ), } -def compute_charge(detected_range: DetectedRange, delta_e: float, e_prime: float, - double_power_law_lookup: RangeFitLookup) -> float: +def compute_charge( + detected_range: DetectedRange, + delta_e: float, + e_prime: float, + double_power_law_lookup: RangeFitLookup, +) -> float: if valid_ranges[detected_range.range].is_in_range(e_prime, delta_e): - charges, deltas = double_power_law_lookup.evaluate_e_prime(detected_range, e_prime) + charges, deltas = double_power_law_lookup.evaluate_e_prime( + detected_range, e_prime + ) assert np.array_equal(deltas, np.sort(deltas)), "values are not increasing" index_2 = clip(np.searchsorted(deltas, delta_e), 1, len(charges) - 1) index_1 = index_2 - 1 - B = np.log(charges[index_2] / charges[index_1]) / np.log(deltas[index_2] / deltas[index_1]) + B = np.log(charges[index_2] / charges[index_1]) / np.log( + deltas[index_2] / deltas[index_1] + ) A = charges[index_1] / (deltas[index_1] ** B) - return A * delta_e ** B + return A * delta_e**B return np.nan -def process_pha_event(event: RawPHAEvent, cosine_table: CosineCorrectionLookupTable, gain_table: GainLookupTable, - range_fit_lookup: RangeFitLookup, rule_lookup: HitEventTypeLookup) -> \ - EventOutput: +def process_pha_event( + event: RawPHAEvent, + cosine_table: CosineCorrectionLookupTable, + gain_table: GainLookupTable, + range_fit_lookup: RangeFitLookup, + rule_lookup: HitEventTypeLookup, +) -> EventOutput: event_analysis = analyze_event(event, rule_lookup) if event_analysis: - correction = cosine_table.get_cosine_correction(event_analysis.range, event_analysis.l1_detector, - event_analysis.l2_detector) + correction = cosine_table.get_cosine_correction( + event_analysis.range, event_analysis.l1_detector, event_analysis.l2_detector + ) def calculate_corrected_energy(word): return correction * calculate_mev(word, gain_table) energies = [calculate_corrected_energy(word) for word in event.pha_words] total_energy = sum( - calculate_corrected_energy(word) for word in event_analysis.words_with_highest_energy) + calculate_corrected_energy(word) + for word in event_analysis.words_with_highest_energy + ) e_delta = calculate_corrected_energy(event_analysis.e_delta_word) e_prime = calculate_corrected_energy(event_analysis.e_prime_word) - charge = compute_charge(event_analysis.range, e_delta, e_prime, range_fit_lookup) - - return EventOutput(original_event=event, charge=charge, total_energy=total_energy, energies=energies, - detected_range=event_analysis.range, e_delta=e_delta, e_prime=e_prime) + charge = compute_charge( + event_analysis.range, e_delta, e_prime, range_fit_lookup + ) + + return EventOutput( + original_event=event, + charge=charge, + total_energy=total_energy, + energies=energies, + detected_range=event_analysis.range, + e_delta=e_delta, + e_prime=e_prime, + ) else: energies = [calculate_mev(word, gain_table) for word in event.pha_words] - return EventOutput(original_event=event, charge=None, total_energy=None, energies=energies, detected_range=None, - e_delta=None, e_prime=None) + return EventOutput( + original_event=event, + charge=None, + total_energy=None, + energies=energies, + detected_range=None, + e_delta=None, + e_prime=None, + ) diff --git a/imap_l3_processing/hit/l3/pha/science/cosine_correction_lookup_table.py b/imap_l3_processing/hit/l3/pha/science/cosine_correction_lookup_table.py index 85869e15a..63407dbe7 100644 --- a/imap_l3_processing/hit/l3/pha/science/cosine_correction_lookup_table.py +++ b/imap_l3_processing/hit/l3/pha/science/cosine_correction_lookup_table.py @@ -24,12 +24,45 @@ class DetectedRange: class CosineCorrectionLookupTable: - _l1_detector_order = ["L1A0a", "L1A0b", "L1A0c", "L1A1a", "L1A1b", "L1A1c", "L1A2a", "L1A2b", "L1A2c", "L1A3a", - "L1A3b", "L1A3c", "L1A4a", "L1A4b", "L1A4c"] - _l2_detector_order = ["L2A0", "L2A1", "L2A2", "L2A3", "L2A4", "L2A5", "L2A6", "L2A7", "L2A8", "L2A9"] + _l1_detector_order = [ + "L1A0a", + "L1A0b", + "L1A0c", + "L1A1a", + "L1A1b", + "L1A1c", + "L1A2a", + "L1A2b", + "L1A2c", + "L1A3a", + "L1A3b", + "L1A3c", + "L1A4a", + "L1A4b", + "L1A4c", + ] + _l2_detector_order = [ + "L2A0", + "L2A1", + "L2A2", + "L2A3", + "L2A4", + "L2A5", + "L2A6", + "L2A7", + "L2A8", + "L2A9", + ] - def __init__(self, range_2A_file_path: Path, range_3A_file_path: Path, range_4A_file_path: Path, - range_2B_file_path: Path, range_3B_file_path: Path, range_4B_file_path: Path): + def __init__( + self, + range_2A_file_path: Path, + range_3A_file_path: Path, + range_4A_file_path: Path, + range_2B_file_path: Path, + range_3B_file_path: Path, + range_4B_file_path: Path, + ): self._range2A_corrections = {} self._range3A_corrections = {} self._range4A_corrections = {} @@ -37,23 +70,31 @@ def __init__(self, range_2A_file_path: Path, range_3A_file_path: Path, range_4A_ self._range3B_corrections = {} self._range4B_corrections = {} - for path, lookup_table in [(range_2A_file_path, self._range2A_corrections), - (range_3A_file_path, self._range3A_corrections), - (range_4A_file_path, self._range4A_corrections), - (range_2B_file_path, self._range2B_corrections), - (range_3B_file_path, self._range3B_corrections), - (range_4B_file_path, self._range4B_corrections)]: - + for path, lookup_table in [ + (range_2A_file_path, self._range2A_corrections), + (range_3A_file_path, self._range3A_corrections), + (range_4A_file_path, self._range4A_corrections), + (range_2B_file_path, self._range2B_corrections), + (range_3B_file_path, self._range3B_corrections), + (range_4B_file_path, self._range4B_corrections), + ]: with open(str(path)) as file: detected_range_reader = csv.reader(file, delimiter=",") for row_num, row in enumerate(detected_range_reader): for col_num, value in enumerate(row): - lookup_key = (self._l1_detector_order[col_num][3:], self._l2_detector_order[row_num][3:]) + lookup_key = ( + self._l1_detector_order[col_num][3:], + self._l2_detector_order[row_num][3:], + ) lookup_table[lookup_key] = float(value) - def get_cosine_correction(self, detected_range: DetectedRange, l1_detector: Detector, - l2_detector: Detector) -> float: + def get_cosine_correction( + self, + detected_range: DetectedRange, + l1_detector: Detector, + l2_detector: Detector, + ) -> float: corrections_by_range = { DetectedRange(DetectorRange.R2, DetectorSide.A): self._range2A_corrections, DetectedRange(DetectorRange.R3, DetectorSide.A): self._range3A_corrections, @@ -62,4 +103,6 @@ def get_cosine_correction(self, detected_range: DetectedRange, l1_detector: Dete DetectedRange(DetectorRange.R3, DetectorSide.B): self._range3A_corrections, DetectedRange(DetectorRange.R4, DetectorSide.B): self._range4A_corrections, } - return corrections_by_range[detected_range][(l1_detector.segment, l2_detector.segment)] + return corrections_by_range[detected_range][ + (l1_detector.segment, l2_detector.segment) + ] diff --git a/imap_l3_processing/hit/l3/pha/science/hit_event_type_lookup.py b/imap_l3_processing/hit/l3/pha/science/hit_event_type_lookup.py index 111417acd..bf06b4b94 100644 --- a/imap_l3_processing/hit/l3/pha/science/hit_event_type_lookup.py +++ b/imap_l3_processing/hit/l3/pha/science/hit_event_type_lookup.py @@ -2,8 +2,11 @@ from dataclasses import dataclass from pathlib import Path -from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import DetectedRange, DetectorRange, \ - DetectorSide +from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import ( + DetectedRange, + DetectorRange, + DetectorSide, +) @dataclass @@ -20,9 +23,13 @@ class HitEventTypeLookup: def lookup_range(self, detectors_groups_to_check: set[str]): for rule in self._rules: has_required_groups = all( - included_group in detectors_groups_to_check for included_group in rule.included_detector_groups) + included_group in detectors_groups_to_check + for included_group in rule.included_detector_groups + ) does_not_have_excluding_groups = all( - excluded_group not in detectors_groups_to_check for excluded_group in rule.excluded_detector_groups) + excluded_group not in detectors_groups_to_check + for excluded_group in rule.excluded_detector_groups + ) if has_required_groups and does_not_have_excluding_groups: return rule @@ -32,7 +39,7 @@ def lookup_range(self, detectors_groups_to_check: set[str]): @classmethod def from_csv(cls, file_path: Path): with open(file_path) as csv_file: - csv_reader = csv.reader(csv_file, delimiter=',') + csv_reader = csv.reader(csv_file, delimiter=",") detector_groups = next(csv_reader) data = list(csv_reader) @@ -43,16 +50,22 @@ def from_csv(cls, file_path: Path): included_detector_groups = [] excluded_detector_groups = [] - for detector_group, is_included in zip(detector_groups[:-1], row[:-1]): + for detector_group, is_included in zip( + detector_groups[:-1], row[:-1] + ): if is_included == "1": included_detector_groups.append(detector_group) elif is_included == "0": excluded_detector_groups.append(detector_group) rules.append( Rule( - range=DetectedRange(range=DetectorRange(int(row[-1][0])), - side=DetectorSide[(row[-1][-1])]), + range=DetectedRange( + range=DetectorRange(int(row[-1][0])), + side=DetectorSide[(row[-1][-1])], + ), included_detector_groups=included_detector_groups, - excluded_detector_groups=excluded_detector_groups)) + excluded_detector_groups=excluded_detector_groups, + ) + ) return cls(_rules=rules) diff --git a/imap_l3_processing/hit/l3/pha/science/range_fit_lookup.py b/imap_l3_processing/hit/l3/pha/science/range_fit_lookup.py index 37d6c806c..69e115312 100644 --- a/imap_l3_processing/hit/l3/pha/science/range_fit_lookup.py +++ b/imap_l3_processing/hit/l3/pha/science/range_fit_lookup.py @@ -6,11 +6,15 @@ import numpy as np from numpy import ndarray -from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import DetectedRange, DetectorRange, DetectorSide +from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import ( + DetectedRange, + DetectorRange, + DetectorSide, +) def double_power_law(e_prime, a1, b1, a2, b2, gamma): - return ((a1 * e_prime ** b1) ** gamma + (a2 * e_prime ** b2) ** gamma) ** (1 / gamma) + return ((a1 * e_prime**b1) ** gamma + (a2 * e_prime**b2) ** gamma) ** (1 / gamma) @dataclass @@ -23,15 +27,23 @@ class RangeFitLookup: range_4B_table: ndarray @classmethod - def from_files(cls, range2A_file: str | Path, range3A_file: str | Path, range4A_file: str | Path, - range2B_file: str | Path, range3B_file: str | Path, range4B_file: str | Path): - return cls(np.loadtxt(range2A_file, delimiter=',', usecols=range(6)), - np.loadtxt(range3A_file, delimiter=',', usecols=range(6)), - np.loadtxt(range4A_file, delimiter=',', usecols=range(6)), - np.loadtxt(range2B_file, delimiter=',', usecols=range(6)), - np.loadtxt(range3B_file, delimiter=',', usecols=range(6)), - np.loadtxt(range4B_file, delimiter=',', usecols=range(6)) - ) + def from_files( + cls, + range2A_file: str | Path, + range3A_file: str | Path, + range4A_file: str | Path, + range2B_file: str | Path, + range3B_file: str | Path, + range4B_file: str | Path, + ): + return cls( + np.loadtxt(range2A_file, delimiter=",", usecols=range(6)), + np.loadtxt(range3A_file, delimiter=",", usecols=range(6)), + np.loadtxt(range4A_file, delimiter=",", usecols=range(6)), + np.loadtxt(range2B_file, delimiter=",", usecols=range(6)), + np.loadtxt(range3B_file, delimiter=",", usecols=range(6)), + np.loadtxt(range4B_file, delimiter=",", usecols=range(6)), + ) def evaluate_e_prime(self, detected_range: DetectedRange, energy): tables = { @@ -40,7 +52,7 @@ def evaluate_e_prime(self, detected_range: DetectedRange, energy): DetectedRange(DetectorRange.R4, DetectorSide.A): self.range_4A_table, DetectedRange(DetectorRange.R2, DetectorSide.B): self.range_2B_table, DetectedRange(DetectorRange.R3, DetectorSide.B): self.range_3B_table, - DetectedRange(DetectorRange.R4, DetectorSide.B): self.range_4B_table + DetectedRange(DetectorRange.R4, DetectorSide.B): self.range_4B_table, } table = tables[detected_range] diff --git a/imap_l3_processing/hit/l3/sectored_products/models.py b/imap_l3_processing/hit/l3/sectored_products/models.py index 9383cb330..302107776 100644 --- a/imap_l3_processing/hit/l3/sectored_products/models.py +++ b/imap_l3_processing/hit/l3/sectored_products/models.py @@ -14,8 +14,12 @@ H_MACROPIXEL_INTENSITY_DELTA_PLUS_CDF_VAR_NAME = "h_macropixel_intensity_delta_plus" H_MACROPIXEL_INTENSITY_DELTA_MINUS_CDF_VAR_NAME = "h_macropixel_intensity_delta_minus" H_MACROPIXEL_INTENSITY_PA_CDF_VAR_NAME = "h_macropixel_intensity_pa" -H_MACROPIXEL_INTENSITY_PA_DELTA_PLUS_CDF_VAR_NAME = "h_macropixel_intensity_pa_delta_plus" -H_MACROPIXEL_INTENSITY_PA_DELTA_MINUS_CDF_VAR_NAME = "h_macropixel_intensity_pa_delta_minus" +H_MACROPIXEL_INTENSITY_PA_DELTA_PLUS_CDF_VAR_NAME = ( + "h_macropixel_intensity_pa_delta_plus" +) +H_MACROPIXEL_INTENSITY_PA_DELTA_MINUS_CDF_VAR_NAME = ( + "h_macropixel_intensity_pa_delta_minus" +) H_ENERGY_CDF_VAR_NAME = "h_energy" H_ENERGY_DELTA_PLUS_CDF_VAR_NAME = "h_energy_delta_plus" H_ENERGY_DELTA_MINUS_CDF_VAR_NAME = "h_energy_delta_minus" @@ -41,8 +45,12 @@ NE_MG_SI_INTENSITY_DELTA_PLUS_CDF_VAR_NAME = "nemgsi_macropixel_intensity_delta_plus" NE_MG_SI_INTENSITY_DELTA_MINUS_CDF_VAR_NAME = "nemgsi_macropixel_intensity_delta_minus" NE_MG_SI_INTENSITY_PA_CDF_VAR_NAME = "nemgsi_macropixel_intensity_pa" -NE_MG_SI_INTENSITY_PA_DELTA_PLUS_CDF_VAR_NAME = "nemgsi_macropixel_intensity_pa_delta_plus" -NE_MG_SI_INTENSITY_PA_DELTA_MINUS_CDF_VAR_NAME = "nemgsi_macropixel_intensity_pa_delta_minus" +NE_MG_SI_INTENSITY_PA_DELTA_PLUS_CDF_VAR_NAME = ( + "nemgsi_macropixel_intensity_pa_delta_plus" +) +NE_MG_SI_INTENSITY_PA_DELTA_MINUS_CDF_VAR_NAME = ( + "nemgsi_macropixel_intensity_pa_delta_minus" +) NE_MG_SI_ENERGY_CDF_VAR_NAME = "nemgsi_energy" NE_MG_SI_ENERGY_DELTA_PLUS_CDF_VAR_NAME = "nemgsi_energy_delta_plus" NE_MG_SI_ENERGY_DELTA_MINUS_CDF_VAR_NAME = "nemgsi_energy_delta_minus" @@ -133,87 +141,220 @@ class HitPitchAngleDataProduct(DataProduct): def to_data_product_variables(self) -> list[DataProductVariable]: return [ DataProductVariable(EPOCH_CDF_VAR_NAME, self.epochs), - DataProductVariable(EPOCH_DELTA_CDF_VAR_NAME, - np.array([t.total_seconds() for t in self.epoch_deltas]) * 1e9), + DataProductVariable( + EPOCH_DELTA_CDF_VAR_NAME, + np.array([t.total_seconds() for t in self.epoch_deltas]) * 1e9, + ), DataProductVariable(PITCH_ANGLE_CDF_VAR_NAME, self.pitch_angles), - DataProductVariable(PITCH_ANGLE_DELTA_CDF_VAR_NAME, self.pitch_angle_deltas), + DataProductVariable( + PITCH_ANGLE_DELTA_CDF_VAR_NAME, self.pitch_angle_deltas + ), DataProductVariable(GYROPHASE_CDF_VAR_NAME, self.gyrophases), DataProductVariable(GYROPHASE_DELTA_CDF_VAR_NAME, self.gyrophase_deltas), - DataProductVariable(H_MACROPIXEL_INTENSITY_CDF_VAR_NAME, self.h_macropixel_intensity), - DataProductVariable(H_MACROPIXEL_INTENSITY_DELTA_PLUS_CDF_VAR_NAME, self.h_macropixel_intensity_delta_plus), - DataProductVariable(H_MACROPIXEL_INTENSITY_DELTA_MINUS_CDF_VAR_NAME, - self.h_macropixel_intensity_delta_minus), - DataProductVariable(H_MACROPIXEL_INTENSITY_PA_CDF_VAR_NAME, self.h_macropixel_intensity_pa), - DataProductVariable(H_MACROPIXEL_INTENSITY_PA_DELTA_PLUS_CDF_VAR_NAME, - self.h_macropixel_intensity_pa_delta_plus), - DataProductVariable(H_MACROPIXEL_INTENSITY_PA_DELTA_MINUS_CDF_VAR_NAME, - self.h_macropixel_intensity_pa_delta_minus), + DataProductVariable( + H_MACROPIXEL_INTENSITY_CDF_VAR_NAME, self.h_macropixel_intensity + ), + DataProductVariable( + H_MACROPIXEL_INTENSITY_DELTA_PLUS_CDF_VAR_NAME, + self.h_macropixel_intensity_delta_plus, + ), + DataProductVariable( + H_MACROPIXEL_INTENSITY_DELTA_MINUS_CDF_VAR_NAME, + self.h_macropixel_intensity_delta_minus, + ), + DataProductVariable( + H_MACROPIXEL_INTENSITY_PA_CDF_VAR_NAME, self.h_macropixel_intensity_pa + ), + DataProductVariable( + H_MACROPIXEL_INTENSITY_PA_DELTA_PLUS_CDF_VAR_NAME, + self.h_macropixel_intensity_pa_delta_plus, + ), + DataProductVariable( + H_MACROPIXEL_INTENSITY_PA_DELTA_MINUS_CDF_VAR_NAME, + self.h_macropixel_intensity_pa_delta_minus, + ), DataProductVariable(H_ENERGY_CDF_VAR_NAME, self.h_energies), - DataProductVariable(H_ENERGY_DELTA_PLUS_CDF_VAR_NAME, self.h_energy_delta_plus), - DataProductVariable(H_ENERGY_DELTA_MINUS_CDF_VAR_NAME, self.h_energy_delta_minus), - DataProductVariable(HE4_INTENSITY_CDF_VAR_NAME, self.he4_macropixel_intensity), - DataProductVariable(HE4_INTENSITY_DELTA_PLUS_CDF_VAR_NAME, self.he4_macropixel_intensity_delta_plus), - DataProductVariable(HE4_INTENSITY_DELTA_MINUS_CDF_VAR_NAME, self.he4_macropixel_intensity_delta_minus), - DataProductVariable(HE4_INTENSITY_PA_CDF_VAR_NAME, self.he4_macropixel_intensity_pa), - DataProductVariable(HE4_INTENSITY_PA_DELTA_PLUS_CDF_VAR_NAME, self.he4_macropixel_intensity_pa_delta_plus), - DataProductVariable(HE4_INTENSITY_PA_DELTA_MINUS_CDF_VAR_NAME, - self.he4_macropixel_intensity_pa_delta_minus), + DataProductVariable( + H_ENERGY_DELTA_PLUS_CDF_VAR_NAME, self.h_energy_delta_plus + ), + DataProductVariable( + H_ENERGY_DELTA_MINUS_CDF_VAR_NAME, self.h_energy_delta_minus + ), + DataProductVariable( + HE4_INTENSITY_CDF_VAR_NAME, self.he4_macropixel_intensity + ), + DataProductVariable( + HE4_INTENSITY_DELTA_PLUS_CDF_VAR_NAME, + self.he4_macropixel_intensity_delta_plus, + ), + DataProductVariable( + HE4_INTENSITY_DELTA_MINUS_CDF_VAR_NAME, + self.he4_macropixel_intensity_delta_minus, + ), + DataProductVariable( + HE4_INTENSITY_PA_CDF_VAR_NAME, self.he4_macropixel_intensity_pa + ), + DataProductVariable( + HE4_INTENSITY_PA_DELTA_PLUS_CDF_VAR_NAME, + self.he4_macropixel_intensity_pa_delta_plus, + ), + DataProductVariable( + HE4_INTENSITY_PA_DELTA_MINUS_CDF_VAR_NAME, + self.he4_macropixel_intensity_pa_delta_minus, + ), DataProductVariable(HE4_ENERGY_CDF_VAR_NAME, self.he4_energies), - DataProductVariable(HE4_ENERGY_DELTA_PLUS_CDF_VAR_NAME, self.he4_energy_delta_plus), - DataProductVariable(HE4_ENERGY_DELTA_MINUS_CDF_VAR_NAME, self.he4_energy_delta_minus), - DataProductVariable(CNO_INTENSITY_CDF_VAR_NAME, self.cno_macropixel_intensity), - DataProductVariable(CNO_INTENSITY_DELTA_PLUS_CDF_VAR_NAME, self.cno_macropixel_intensity_delta_plus), - DataProductVariable(CNO_INTENSITY_DELTA_MINUS_CDF_VAR_NAME, self.cno_macropixel_intensity_delta_minus), - DataProductVariable(CNO_INTENSITY_PA_CDF_VAR_NAME, self.cno_macropixel_intensity_pa), - DataProductVariable(CNO_INTENSITY_PA_DELTA_PLUS_CDF_VAR_NAME, self.cno_macropixel_intensity_pa_delta_plus), - DataProductVariable(CNO_INTENSITY_PA_DELTA_MINUS_CDF_VAR_NAME, - self.cno_macropixel_intensity_pa_delta_minus), + DataProductVariable( + HE4_ENERGY_DELTA_PLUS_CDF_VAR_NAME, self.he4_energy_delta_plus + ), + DataProductVariable( + HE4_ENERGY_DELTA_MINUS_CDF_VAR_NAME, self.he4_energy_delta_minus + ), + DataProductVariable( + CNO_INTENSITY_CDF_VAR_NAME, self.cno_macropixel_intensity + ), + DataProductVariable( + CNO_INTENSITY_DELTA_PLUS_CDF_VAR_NAME, + self.cno_macropixel_intensity_delta_plus, + ), + DataProductVariable( + CNO_INTENSITY_DELTA_MINUS_CDF_VAR_NAME, + self.cno_macropixel_intensity_delta_minus, + ), + DataProductVariable( + CNO_INTENSITY_PA_CDF_VAR_NAME, self.cno_macropixel_intensity_pa + ), + DataProductVariable( + CNO_INTENSITY_PA_DELTA_PLUS_CDF_VAR_NAME, + self.cno_macropixel_intensity_pa_delta_plus, + ), + DataProductVariable( + CNO_INTENSITY_PA_DELTA_MINUS_CDF_VAR_NAME, + self.cno_macropixel_intensity_pa_delta_minus, + ), DataProductVariable(CNO_ENERGY_CDF_VAR_NAME, self.cno_energies), - DataProductVariable(CNO_ENERGY_DELTA_PLUS_CDF_VAR_NAME, self.cno_energy_delta_plus), - DataProductVariable(CNO_ENERGY_DELTA_MINUS_CDF_VAR_NAME, self.cno_energy_delta_minus), - DataProductVariable(NE_MG_SI_INTENSITY_CDF_VAR_NAME, self.ne_mg_si_macropixel_intensity), - DataProductVariable(NE_MG_SI_INTENSITY_DELTA_PLUS_CDF_VAR_NAME, - self.ne_mg_si_macropixel_intensity_delta_plus), - DataProductVariable(NE_MG_SI_INTENSITY_DELTA_MINUS_CDF_VAR_NAME, - self.ne_mg_si_macropixel_intensity_delta_minus), - DataProductVariable(NE_MG_SI_INTENSITY_PA_CDF_VAR_NAME, self.ne_mg_si_macropixel_intensity_pa), - DataProductVariable(NE_MG_SI_INTENSITY_PA_DELTA_PLUS_CDF_VAR_NAME, - self.ne_mg_si_macropixel_intensity_pa_delta_plus), - DataProductVariable(NE_MG_SI_INTENSITY_PA_DELTA_MINUS_CDF_VAR_NAME, - self.ne_mg_si_macropixel_intensity_pa_delta_minus), + DataProductVariable( + CNO_ENERGY_DELTA_PLUS_CDF_VAR_NAME, self.cno_energy_delta_plus + ), + DataProductVariable( + CNO_ENERGY_DELTA_MINUS_CDF_VAR_NAME, self.cno_energy_delta_minus + ), + DataProductVariable( + NE_MG_SI_INTENSITY_CDF_VAR_NAME, self.ne_mg_si_macropixel_intensity + ), + DataProductVariable( + NE_MG_SI_INTENSITY_DELTA_PLUS_CDF_VAR_NAME, + self.ne_mg_si_macropixel_intensity_delta_plus, + ), + DataProductVariable( + NE_MG_SI_INTENSITY_DELTA_MINUS_CDF_VAR_NAME, + self.ne_mg_si_macropixel_intensity_delta_minus, + ), + DataProductVariable( + NE_MG_SI_INTENSITY_PA_CDF_VAR_NAME, + self.ne_mg_si_macropixel_intensity_pa, + ), + DataProductVariable( + NE_MG_SI_INTENSITY_PA_DELTA_PLUS_CDF_VAR_NAME, + self.ne_mg_si_macropixel_intensity_pa_delta_plus, + ), + DataProductVariable( + NE_MG_SI_INTENSITY_PA_DELTA_MINUS_CDF_VAR_NAME, + self.ne_mg_si_macropixel_intensity_pa_delta_minus, + ), DataProductVariable(NE_MG_SI_ENERGY_CDF_VAR_NAME, self.ne_mg_si_energies), - DataProductVariable(NE_MG_SI_ENERGY_DELTA_PLUS_CDF_VAR_NAME, self.ne_mg_si_energy_delta_plus), - DataProductVariable(NE_MG_SI_ENERGY_DELTA_MINUS_CDF_VAR_NAME, self.ne_mg_si_energy_delta_minus), - DataProductVariable(IRON_INTENSITY_CDF_VAR_NAME, self.iron_macropixel_intensity), - DataProductVariable(IRON_INTENSITY_DELTA_PLUS_CDF_VAR_NAME, self.iron_macropixel_intensity_delta_plus), - DataProductVariable(IRON_INTENSITY_DELTA_MINUS_CDF_VAR_NAME, self.iron_macropixel_intensity_delta_minus), - DataProductVariable(IRON_INTENSITY_PA_CDF_VAR_NAME, self.iron_macropixel_intensity_pa), - DataProductVariable(IRON_INTENSITY_PA_DELTA_PLUS_CDF_VAR_NAME, - self.iron_macropixel_intensity_pa_delta_plus), - DataProductVariable(IRON_INTENSITY_PA_DELTA_MINUS_CDF_VAR_NAME, - self.iron_macropixel_intensity_pa_delta_minus), + DataProductVariable( + NE_MG_SI_ENERGY_DELTA_PLUS_CDF_VAR_NAME, self.ne_mg_si_energy_delta_plus + ), + DataProductVariable( + NE_MG_SI_ENERGY_DELTA_MINUS_CDF_VAR_NAME, + self.ne_mg_si_energy_delta_minus, + ), + DataProductVariable( + IRON_INTENSITY_CDF_VAR_NAME, self.iron_macropixel_intensity + ), + DataProductVariable( + IRON_INTENSITY_DELTA_PLUS_CDF_VAR_NAME, + self.iron_macropixel_intensity_delta_plus, + ), + DataProductVariable( + IRON_INTENSITY_DELTA_MINUS_CDF_VAR_NAME, + self.iron_macropixel_intensity_delta_minus, + ), + DataProductVariable( + IRON_INTENSITY_PA_CDF_VAR_NAME, self.iron_macropixel_intensity_pa + ), + DataProductVariable( + IRON_INTENSITY_PA_DELTA_PLUS_CDF_VAR_NAME, + self.iron_macropixel_intensity_pa_delta_plus, + ), + DataProductVariable( + IRON_INTENSITY_PA_DELTA_MINUS_CDF_VAR_NAME, + self.iron_macropixel_intensity_pa_delta_minus, + ), DataProductVariable(IRON_ENERGY_CDF_VAR_NAME, self.iron_energies), - DataProductVariable(IRON_ENERGY_DELTA_PLUS_CDF_VAR_NAME, self.iron_energy_delta_plus), - DataProductVariable(IRON_ENERGY_DELTA_MINUS_CDF_VAR_NAME, self.iron_energy_delta_minus), - DataProductVariable(MEASUREMENT_PITCH_ANGLE_VAR_NAME, self.measurement_pitch_angle), - DataProductVariable(MEASUREMENT_GYROPHASE_VAR_NAME, self.measurement_gyrophase), - DataProductVariable(PITCH_ANGLE_LABEL_VAR_NAME, - [f"Pitch Angle Label {str(i + 1)}" for i in range(len(self.pitch_angles))]), - DataProductVariable(GYROPHASE_LABEL_VAR_NAME, - [f"Gyrophase Label {str(i + 1)}" for i in range(len(self.gyrophases))]), - DataProductVariable(H_ENERGY_LABEL_VAR_NAME, - [f"H Energy Label {str(i + 1)}" for i in range(len(self.h_energies))]), - DataProductVariable(HE4_ENERGY_LABEL_VAR_NAME, - [f"He4 Energy Label {str(i + 1)}" for i in range(len(self.he4_energies))]), - DataProductVariable(CNO_ENERGY_LABEL_VAR_NAME, - [f"CNO Energy Label {str(i + 1)}" for i in range(len(self.cno_energies))]), - DataProductVariable(NE_MG_SI_ENERGY_LABEL_VAR_NAME, - [f"NeMgSi Energy Label {str(i + 1)}" for i in range(len(self.ne_mg_si_energies))]), - DataProductVariable(IRON_ENERGY_LABEL_VAR_NAME, - [f"Fe Energy Label {str(i + 1)}" for i in range(len(self.iron_energies))]), + DataProductVariable( + IRON_ENERGY_DELTA_PLUS_CDF_VAR_NAME, self.iron_energy_delta_plus + ), + DataProductVariable( + IRON_ENERGY_DELTA_MINUS_CDF_VAR_NAME, self.iron_energy_delta_minus + ), + DataProductVariable( + MEASUREMENT_PITCH_ANGLE_VAR_NAME, self.measurement_pitch_angle + ), + DataProductVariable( + MEASUREMENT_GYROPHASE_VAR_NAME, self.measurement_gyrophase + ), + DataProductVariable( + PITCH_ANGLE_LABEL_VAR_NAME, + [ + f"Pitch Angle Label {str(i + 1)}" + for i in range(len(self.pitch_angles)) + ], + ), + DataProductVariable( + GYROPHASE_LABEL_VAR_NAME, + [f"Gyrophase Label {str(i + 1)}" for i in range(len(self.gyrophases))], + ), + DataProductVariable( + H_ENERGY_LABEL_VAR_NAME, + [f"H Energy Label {str(i + 1)}" for i in range(len(self.h_energies))], + ), + DataProductVariable( + HE4_ENERGY_LABEL_VAR_NAME, + [ + f"He4 Energy Label {str(i + 1)}" + for i in range(len(self.he4_energies)) + ], + ), + DataProductVariable( + CNO_ENERGY_LABEL_VAR_NAME, + [ + f"CNO Energy Label {str(i + 1)}" + for i in range(len(self.cno_energies)) + ], + ), + DataProductVariable( + NE_MG_SI_ENERGY_LABEL_VAR_NAME, + [ + f"NeMgSi Energy Label {str(i + 1)}" + for i in range(len(self.ne_mg_si_energies)) + ], + ), + DataProductVariable( + IRON_ENERGY_LABEL_VAR_NAME, + [ + f"Fe Energy Label {str(i + 1)}" + for i in range(len(self.iron_energies)) + ], + ), DataProductVariable(AZIMUTH_VAR_NAME, self.azimuth), DataProductVariable(ZENITH_VAR_NAME, self.zenith), - DataProductVariable(AZIMUTH_LABEL_VAR_NAME, [str(float(azimuth)) for azimuth in self.azimuth]), - DataProductVariable(ZENITH_LABEL_VAR_NAME, [str(float(zenith)) for zenith in self.zenith]), + DataProductVariable( + AZIMUTH_LABEL_VAR_NAME, + [str(float(azimuth)) for azimuth in self.azimuth], + ), + DataProductVariable( + ZENITH_LABEL_VAR_NAME, [str(float(zenith)) for zenith in self.zenith] + ), DataProductVariable(HIT_FLAGS_CDF_VAR_NAME, self.hit_flags), ] diff --git a/imap_l3_processing/hit/l3/sectored_products/science/sectored_products_algorithms.py b/imap_l3_processing/hit/l3/sectored_products/science/sectored_products_algorithms.py index 35cc374f7..56808e398 100644 --- a/imap_l3_processing/hit/l3/sectored_products/science/sectored_products_algorithms.py +++ b/imap_l3_processing/hit/l3/sectored_products/science/sectored_products_algorithms.py @@ -1,12 +1,17 @@ import numpy as np -def get_hit_bin_polar_coordinates(declination_bins=8, inclination_bins=15) -> tuple[ - np.ndarray, np.ndarray, np.ndarray, np.ndarray]: - declination_starts, declination_step = np.linspace(0, 180, declination_bins, endpoint=False, retstep=True) +def get_hit_bin_polar_coordinates( + declination_bins=8, inclination_bins=15 +) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + declination_starts, declination_step = np.linspace( + 0, 180, declination_bins, endpoint=False, retstep=True + ) declination_delta = declination_step / 2 declinations = declination_starts + declination_delta - inclination_starts, inclination_step = np.linspace(0, 360, inclination_bins, endpoint=False, retstep=True) + inclination_starts, inclination_step = np.linspace( + 0, 360, inclination_bins, endpoint=False, retstep=True + ) inclination_delta = inclination_step / 2 inclinations = inclination_starts + inclination_delta declination_deltas = np.array([declination_delta] * declination_bins) @@ -15,7 +20,9 @@ def get_hit_bin_polar_coordinates(declination_bins=8, inclination_bins=15) -> tu return declinations, inclinations, declination_deltas, inclination_deltas -def get_sector_unit_vectors(declinations_degrees: np.ndarray, inclinations_degrees: np.ndarray) -> np.ndarray: +def get_sector_unit_vectors( + declinations_degrees: np.ndarray, inclinations_degrees: np.ndarray +) -> np.ndarray: declinations = np.deg2rad(declinations_degrees) inclinations = np.deg2rad(inclinations_degrees) declinations = declinations[:, np.newaxis] diff --git a/imap_l3_processing/hit/quality_flags.py b/imap_l3_processing/hit/quality_flags.py index adca15030..002bb8bab 100644 --- a/imap_l3_processing/hit/quality_flags.py +++ b/imap_l3_processing/hit/quality_flags.py @@ -3,4 +3,4 @@ class HitL3Flags(FlagNameMixin): NONE = CommonFlags.NONE - PRELIMINARY_MAG = 2 ** 2 + PRELIMINARY_MAG = 2**2 diff --git a/imap_l3_processing/lo/l3/lo_combined_dependencies.py b/imap_l3_processing/lo/l3/lo_combined_dependencies.py index 6bd4fe732..d392f144f 100644 --- a/imap_l3_processing/lo/l3/lo_combined_dependencies.py +++ b/imap_l3_processing/lo/l3/lo_combined_dependencies.py @@ -7,6 +7,7 @@ from imap_l3_processing.maps.map_models import RectangularIntensityMapData + @dataclass class LoCombinedDependencies: map_data: list[RectangularIntensityMapData] @@ -15,7 +16,9 @@ class LoCombinedDependencies: def fetch_dependencies(cls, dependencies: ProcessingInputCollection) -> Self: lo_inputs = dependencies.get_file_paths(source="lo") - assert len(lo_inputs) == 3, f"Expected 3 input lo maps to produce combined map! got {len(lo_inputs)}" + assert len(lo_inputs) == 3, ( + f"Expected 3 input lo maps to produce combined map! got {len(lo_inputs)}" + ) downloaded_paths = [imap_data_access.download(Path(i).name) for i in lo_inputs] @@ -23,5 +26,6 @@ def fetch_dependencies(cls, dependencies: ProcessingInputCollection) -> Self: @classmethod def from_file_paths(cls, file_paths: list[Path]) -> Self: - return cls(map_data=[RectangularIntensityMapData.read_from_path(p) for p in file_paths]) - + return cls( + map_data=[RectangularIntensityMapData.read_from_path(p) for p in file_paths] + ) diff --git a/imap_l3_processing/lo/l3/lo_l3_spectral_fit_dependencies.py b/imap_l3_processing/lo/l3/lo_l3_spectral_fit_dependencies.py index 6d2af6448..bf8574baa 100644 --- a/imap_l3_processing/lo/l3/lo_l3_spectral_fit_dependencies.py +++ b/imap_l3_processing/lo/l3/lo_l3_spectral_fit_dependencies.py @@ -12,8 +12,12 @@ class LoL3SpectralFitDependencies: map_data: RectangularIntensityMapData @classmethod - def fetch_dependencies(cls, dependencies: ProcessingInputCollection) -> LoL3SpectralFitDependencies: - file_names = dependencies.get_file_paths(source="lo", data_type="l3") + dependencies.get_file_paths(source="lo", data_type="l2") + def fetch_dependencies( + cls, dependencies: ProcessingInputCollection + ) -> LoL3SpectralFitDependencies: + file_names = dependencies.get_file_paths( + source="lo", data_type="l3" + ) + dependencies.get_file_paths(source="lo", data_type="l2") if len(file_names) != 1: raise ValueError("Incorrect number of dependencies") diff --git a/imap_l3_processing/lo/l3/lo_sp_initializer.py b/imap_l3_processing/lo/l3/lo_sp_initializer.py index ee5937b4d..7b4ddc5e0 100644 --- a/imap_l3_processing/lo/l3/lo_sp_initializer.py +++ b/imap_l3_processing/lo/l3/lo_sp_initializer.py @@ -3,7 +3,10 @@ import imap_data_access -from imap_l3_processing.maps.map_descriptors import MapDescriptorParts, SurvivalCorrection +from imap_l3_processing.maps.map_descriptors import ( + MapDescriptorParts, + SurvivalCorrection, +) from imap_l3_processing.maps.map_initializer import PossibleMapToProduce from imap_l3_processing.maps.sp_map_initializer import SPMapInitializer from imap_l3_processing.utils import SpiceKernelTypes, furnish_spice_metakernel @@ -25,7 +28,7 @@ LO_ISN_MAP_DESCRIPTORS = [ "l090-isn-h-sf-nsp-ram-hae-6deg-1yr", - "l090-isn-o-sf-nsp-ram-hae-6deg-1yr" + "l090-isn-o-sf-nsp-ram-hae-6deg-1yr", ] LO_SPECTRAL_INDEX_MAP_DESCRIPTORS = [ @@ -45,12 +48,14 @@ class LoSPInitializer(SPMapInitializer): def __init__(self): glows_query_result = imap_data_access.query( - instrument='glows', - data_level='l3e', + instrument="glows", + data_level="l3e", descriptor="survival-probability-lo", - version="latest" + version="latest", ) - self.glows_files_by_repointing = {int(r["repointing"]): Path(r["file_path"]).name for r in glows_query_result} + self.glows_files_by_repointing = { + int(r["repointing"]): Path(r["file_path"]).name for r in glows_query_result + } lo_l2_query_result = imap_data_access.query(instrument="lo", data_level="l2") super().__init__("lo", lo_l2_query_result) @@ -58,12 +63,21 @@ def __init__(self): def _collect_glows_psets_by_repoint(self, descriptor: str) -> dict[int, str]: return self.glows_files_by_repointing - def _get_l2_dependencies(self, descriptor: MapDescriptorParts) -> list[MapDescriptorParts]: - return [dataclasses.replace(descriptor, survival_correction=SurvivalCorrection.NotSurvivalCorrected)] + def _get_l2_dependencies( + self, descriptor: MapDescriptorParts + ) -> list[MapDescriptorParts]: + return [ + dataclasses.replace( + descriptor, survival_correction=SurvivalCorrection.NotSurvivalCorrected + ) + ] def furnish_spice_dependencies(self, map_to_produce: PossibleMapToProduce): - furnish_spice_metakernel(start_date=map_to_produce.input_metadata.start_date, - end_date=map_to_produce.input_metadata.end_date, kernel_types=LO_SP_MAP_KERNELS) + furnish_spice_metakernel( + start_date=map_to_produce.input_metadata.start_date, + end_date=map_to_produce.input_metadata.end_date, + kernel_types=LO_SP_MAP_KERNELS, + ) def _get_ancillary_files(self) -> list[str]: return [] diff --git a/imap_l3_processing/lo/lo_processor.py b/imap_l3_processing/lo/lo_processor.py index 88f88e4a7..2f6b05d08 100644 --- a/imap_l3_processing/lo/lo_processor.py +++ b/imap_l3_processing/lo/lo_processor.py @@ -5,64 +5,119 @@ from imap_processing.spice.geometry import SpiceFrame from imap_l3_processing.lo.l3.lo_combined_dependencies import LoCombinedDependencies -from imap_l3_processing.lo.l3.lo_l3_isn_background_subtracted_dependencies import \ - LoL3ISNBackgroundSubtractedDependencies -from imap_l3_processing.lo.l3.lo_l3_spectral_fit_dependencies import LoL3SpectralFitDependencies -from imap_l3_processing.maps.hilo_l3_survival_dependencies import HiLoL3SurvivalDependencies +from imap_l3_processing.lo.l3.lo_l3_isn_background_subtracted_dependencies import ( + LoL3ISNBackgroundSubtractedDependencies, +) +from imap_l3_processing.lo.l3.lo_l3_spectral_fit_dependencies import ( + LoL3SpectralFitDependencies, +) +from imap_l3_processing.maps.hilo_l3_survival_dependencies import ( + HiLoL3SurvivalDependencies, +) from imap_l3_processing.maps.map_combination import ExposureWeightedCombination -from imap_l3_processing.maps.map_descriptors import parse_map_descriptor, MapDescriptorParts, MapQuantity, \ - SurvivalCorrection, ReferenceFrame, Sensor -from imap_l3_processing.maps.map_models import ISNRateData, ISNBackgroundSubtractedData, ISNBackgroundSubtractedMapData -from imap_l3_processing.maps.map_models import RectangularIntensityMapData, RectangularSpectralIndexDataProduct, \ - RectangularSpectralIndexMapData, RectangularIntensityDataProduct, InputRectangularPointingSet, \ - ISNBackgroundSubtractedDataProduct +from imap_l3_processing.maps.map_descriptors import ( + parse_map_descriptor, + MapDescriptorParts, + MapQuantity, + SurvivalCorrection, + ReferenceFrame, + Sensor, +) +from imap_l3_processing.maps.map_models import ( + ISNRateData, + ISNBackgroundSubtractedData, + ISNBackgroundSubtractedMapData, +) +from imap_l3_processing.maps.map_models import ( + RectangularIntensityMapData, + RectangularSpectralIndexDataProduct, + RectangularSpectralIndexMapData, + RectangularIntensityDataProduct, + InputRectangularPointingSet, + ISNBackgroundSubtractedDataProduct, +) from imap_l3_processing.maps.map_processor import MapProcessor -from imap_l3_processing.maps.spectral_fit import fit_spectral_index_map, slice_energy_range_by_bin -from imap_l3_processing.maps.survival_probability_processing import process_survival_probabilities +from imap_l3_processing.maps.spectral_fit import ( + fit_spectral_index_map, + slice_energy_range_by_bin, +) +from imap_l3_processing.maps.survival_probability_processing import ( + process_survival_probabilities, +) from imap_l3_processing.models import Instrument from imap_l3_processing.utils import save_data class LoProcessor(MapProcessor): - def process(self, spice_frame_name: SpiceFrame = SpiceFrame.ECLIPJ2000) -> list[Path]: + def process( + self, spice_frame_name: SpiceFrame = SpiceFrame.ECLIPJ2000 + ) -> list[Path]: set_of_parent_file_names = set(self.get_parent_file_names()) descriptor = parse_map_descriptor(self.input_metadata.descriptor) match descriptor: case MapDescriptorParts(quantity=MapQuantity.SpectralIndex): deps = LoL3SpectralFitDependencies.fetch_dependencies(self.dependencies) - spectral_fit_data = perform_spectral_fit(deps.map_data, - descriptor.spectral_index_energy_range or (4, 7)) - data_product = RectangularSpectralIndexDataProduct(self.input_metadata, spectral_fit_data, - spice_frame_name=spice_frame_name) + spectral_fit_data = perform_spectral_fit( + deps.map_data, descriptor.spectral_index_energy_range or (4, 7) + ) + data_product = RectangularSpectralIndexDataProduct( + self.input_metadata, + spectral_fit_data, + spice_frame_name=spice_frame_name, + ) case MapDescriptorParts(quantity=MapQuantity.SpectralIndexNBS): deps = LoL3SpectralFitDependencies.fetch_dependencies(self.dependencies) - spectral_fit_data = perform_spectral_fit(deps.map_data, - descriptor.spectral_index_energy_range or (1, 7)) - data_product = RectangularSpectralIndexDataProduct(self.input_metadata, spectral_fit_data, - spice_frame_name=spice_frame_name) + spectral_fit_data = perform_spectral_fit( + deps.map_data, descriptor.spectral_index_energy_range or (1, 7) + ) + data_product = RectangularSpectralIndexDataProduct( + self.input_metadata, + spectral_fit_data, + spice_frame_name=spice_frame_name, + ) case MapDescriptorParts(quantity=MapQuantity.ISNBackgroundSubtracted): - deps = LoL3ISNBackgroundSubtractedDependencies.fetch_dependencies(self.dependencies) + deps = LoL3ISNBackgroundSubtractedDependencies.fetch_dependencies( + self.dependencies + ) background_subtracted = isn_background_subtraction(deps.map_data) - data_product = ISNBackgroundSubtractedDataProduct(self.input_metadata, background_subtracted, - spice_frame_name=spice_frame_name) + data_product = ISNBackgroundSubtractedDataProduct( + self.input_metadata, + background_subtracted, + spice_frame_name=spice_frame_name, + ) case MapDescriptorParts(sensor=Sensor.LoCombined): deps = LoCombinedDependencies.fetch_dependencies(self.dependencies) combination = ExposureWeightedCombination() - combined_map = combination.combine_rectangular_intensity_map_data(deps.map_data) - data_product = RectangularIntensityDataProduct(self.input_metadata, combined_map,spice_frame_name=spice_frame_name) + combined_map = combination.combine_rectangular_intensity_map_data( + deps.map_data + ) + data_product = RectangularIntensityDataProduct( + self.input_metadata, combined_map, spice_frame_name=spice_frame_name + ) - case MapDescriptorParts(survival_correction=SurvivalCorrection.SurvivalCorrected, - reference_frame=ReferenceFrame.Spacecraft | ReferenceFrame.Heliospheric): - deps = HiLoL3SurvivalDependencies.fetch_dependencies(self.dependencies, Instrument.IMAP_LO) + case MapDescriptorParts( + survival_correction=SurvivalCorrection.SurvivalCorrected, + reference_frame=ReferenceFrame.Spacecraft | ReferenceFrame.Heliospheric, + ): + deps = HiLoL3SurvivalDependencies.fetch_dependencies( + self.dependencies, Instrument.IMAP_LO + ) deps.l1c_data = list(map(self._collapse_pset_dimension, deps.l1c_data)) - data = process_survival_probabilities(deps, spice_frame_name, cg_corrected=False) + data = process_survival_probabilities( + deps, spice_frame_name, cg_corrected=False + ) - data_product = RectangularIntensityDataProduct(self.input_metadata, data, - spice_frame_name=spice_frame_name) - set_of_parent_file_names.update(path.name for path in deps.dependency_file_paths) + data_product = RectangularIntensityDataProduct( + self.input_metadata, data, spice_frame_name=spice_frame_name + ) + set_of_parent_file_names.update( + path.name for path in deps.dependency_file_paths + ) case None: - raise ValueError(f"Could not parse descriptor {self.input_metadata.descriptor}") + raise ValueError( + f"Could not parse descriptor {self.input_metadata.descriptor}" + ) case _: raise NotImplementedError(self.input_metadata.descriptor) @@ -70,26 +125,38 @@ def process(self, spice_frame_name: SpiceFrame = SpiceFrame.ECLIPJ2000) -> list[ return [save_data(data_product)] @staticmethod - def _collapse_pset_dimension(pset: InputRectangularPointingSet) -> InputRectangularPointingSet: - return dataclasses.replace(pset, exposure_times=np.sum(pset.exposure_times, axis=3), - hae_longitude=np.mean(pset.hae_longitude, axis=-1), - hae_latitude=np.mean(pset.hae_latitude, axis=-1)) + def _collapse_pset_dimension( + pset: InputRectangularPointingSet, + ) -> InputRectangularPointingSet: + return dataclasses.replace( + pset, + exposure_times=np.sum(pset.exposure_times, axis=3), + hae_longitude=np.mean(pset.hae_longitude, axis=-1), + hae_latitude=np.mean(pset.hae_latitude, axis=-1), + ) -def perform_spectral_fit(data: RectangularIntensityMapData, - spectral_index_range: tuple[int, int]) -> RectangularSpectralIndexMapData: - intensity_data = slice_energy_range_by_bin(data.intensity_map_data, spectral_index_range[0], - spectral_index_range[1]) +def perform_spectral_fit( + data: RectangularIntensityMapData, spectral_index_range: tuple[int, int] +) -> RectangularSpectralIndexMapData: + intensity_data = slice_energy_range_by_bin( + data.intensity_map_data, spectral_index_range[0], spectral_index_range[1] + ) return RectangularSpectralIndexMapData( spectral_index_map_data=fit_spectral_index_map(intensity_data), - coords=data.coords + coords=data.coords, ) -def isn_background_subtraction(isn_rate_data: ISNRateData) -> ISNBackgroundSubtractedMapData: - isn_rate_background_subtracted = isn_rate_data.ena_count_rate - isn_rate_data.bg_rate +def isn_background_subtraction( + isn_rate_data: ISNRateData, +) -> ISNBackgroundSubtractedMapData: + isn_rate_background_subtracted = ( + isn_rate_data.ena_count_rate - isn_rate_data.bg_rate + ) isn_bg_subtracted_stat_err = np.sqrt( - np.square(isn_rate_data.ena_count_rate_stat_uncert) + np.square(isn_rate_data.bg_rate_stat_uncert) + np.square(isn_rate_data.ena_count_rate_stat_uncert) + + np.square(isn_rate_data.bg_rate_stat_uncert) ) map_data = ISNBackgroundSubtractedData( @@ -114,7 +181,7 @@ def isn_background_subtraction(isn_rate_data: ISNRateData) -> ISNBackgroundSubtr longitude_label=isn_rate_data.longitude_label, isn_bg_rate_subtracted=isn_rate_background_subtracted, isn_bg_rate_subtracted_stat_uncert=isn_bg_subtracted_stat_err, - isn_bg_rate_subtracted_sys_err=isn_rate_data.bg_rate_sys_err + isn_bg_rate_subtracted_sys_err=isn_rate_data.bg_rate_sys_err, ) return ISNBackgroundSubtractedMapData(isn_rate_map_data=map_data) diff --git a/imap_l3_processing/lo/lo_txt_to_cdf_converter_spike.py b/imap_l3_processing/lo/lo_txt_to_cdf_converter_spike.py index 42809e833..e6d675b43 100644 --- a/imap_l3_processing/lo/lo_txt_to_cdf_converter_spike.py +++ b/imap_l3_processing/lo/lo_txt_to_cdf_converter_spike.py @@ -28,9 +28,13 @@ from imap_l3_processing.utils import furnished_metakernel from tests.test_helpers import get_run_local_data_path -LO_ENERGIES_IN_KEV = np.array([16.33, 30.47, 55.76, 106.3, 200.0, 405.0, 787.3]) / 1000.0 +LO_ENERGIES_IN_KEV = ( + np.array([16.33, 30.47, 55.76, 106.3, 200.0, 405.0, 787.3]) / 1000.0 +) LO_ENERGY_BIN_LOWERS = np.array([10.9, 20.4, 36.8, 71.6, 135.0, 269.0, 504.7]) / 1000.0 -LO_ENERGY_BIN_UPPERS = np.array([21.7, 40.5, 74.7, 140.9, 265.0, 541.0, 1069.9]) / 1000.0 +LO_ENERGY_BIN_UPPERS = ( + np.array([21.7, 40.5, 74.7, 140.9, 265.0, 541.0, 1069.9]) / 1000.0 +) class CsvNameToProduct: @@ -55,7 +59,7 @@ def get_energy_and_quantity(self, filename: str) -> Optional[tuple[int, str]]: "bflux": "bg_intensity", "bfvar": "bg_intensity_stat_var", "bfunc": "bg_intensity_sys_err", - "expo": "exposure_factor" + "expo": "exposure_factor", } if quantity in csv_to_l2_cdf_mapping: @@ -89,7 +93,9 @@ def __init__(self, correction: Literal["sput", "boot"]): @override def get_energy_and_quantity(self, filename: str) -> Optional[tuple[int, str]]: - if fn_match := re.match("map_flux_(\d{1})_Hy_([a-zA-Z]+)_([a-zA-Z]+).csv", filename): + if fn_match := re.match( + "map_flux_(\d{1})_Hy_([a-zA-Z]+)_([a-zA-Z]+).csv", filename + ): [energy, correction, quantity] = fn_match.groups() csv_to_l2_cdf_mapping = { @@ -140,34 +146,47 @@ class LoProcessingInput: def load_data_dir(data_dir: Path, name_mapping: CsvNameToProduct, **kwargs): temp_data = {} for data_file_path in data_dir.iterdir(): - if energy_and_quantity := name_mapping.get_energy_and_quantity(data_file_path.name): + if energy_and_quantity := name_mapping.get_energy_and_quantity( + data_file_path.name + ): esa_step, data_type = energy_and_quantity if data_type not in temp_data: temp_data[data_type] = np.full((1, 7, 60, 30), np.nan) - temp_data[data_type][0, esa_step - 1] = np.loadtxt(data_file_path, delimiter=",", **kwargs).T + temp_data[data_type][0, esa_step - 1] = np.loadtxt( + data_file_path, delimiter=",", **kwargs + ).T return temp_data @classmethod - def load(cls, - manifest: Manifest, - l2_descriptor: str, - version: int, - input_path: Path, - name_mapping: CsvNameToProduct, - use_masked_data: bool, - **additional_map_data - ) -> Self: + def load( + cls, + manifest: Manifest, + l2_descriptor: str, + version: int, + input_path: Path, + name_mapping: CsvNameToProduct, + use_masked_data: bool, + **additional_map_data, + ) -> Self: if use_masked_data: loaded_data = { - **LoProcessingInput.load_data_dir(input_path / "maps", name_mapping, skiprows=1), - **LoProcessingInput.load_data_dir(input_path / "masked_maps", name_mapping) + **LoProcessingInput.load_data_dir( + input_path / "maps", name_mapping, skiprows=1 + ), + **LoProcessingInput.load_data_dir( + input_path / "masked_maps", name_mapping + ), } else: loaded_data = { - **LoProcessingInput.load_data_dir(input_path / "maps", name_mapping, skiprows=1), + **LoProcessingInput.load_data_dir( + input_path / "maps", name_mapping, skiprows=1 + ), } - start_date_nanoseconds = (manifest.start_date - TT2000_EPOCH).total_seconds() * 1e9 + start_date_nanoseconds = ( + manifest.start_date - TT2000_EPOCH + ).total_seconds() * 1e9 end_date_nanoseconds = (manifest.end_date - TT2000_EPOCH).total_seconds() * 1e9 skymap = RectangularSkyMap(6, SpiceFrame.ECLIPJ2000) @@ -176,16 +195,26 @@ def load(cls, map_data_coords = ["epoch", "energy", "longitude", "latitude"] l2_dataset = xr.Dataset( - data_vars= - { + data_vars={ "obs_date": (map_data_coords, np.full((1, 7, 60, 30), FILLVAL_INT64)), "obs_date_range": (map_data_coords, np.full((1, 7, 60, 30), np.nan)), - "solid_angle": (["epoch", "longitude", "latitude"], np.full((1, 60, 30), np.nan)), - "energy_delta_minus": (["energy"], LO_ENERGIES_IN_KEV - LO_ENERGY_BIN_LOWERS), - "energy_delta_plus": (["energy"], LO_ENERGY_BIN_UPPERS - LO_ENERGIES_IN_KEV), - - **{quantity: (map_data_coords, data) for quantity, data in loaded_data.items()}, - **additional_map_data + "solid_angle": ( + ["epoch", "longitude", "latitude"], + np.full((1, 60, 30), np.nan), + ), + "energy_delta_minus": ( + ["energy"], + LO_ENERGIES_IN_KEV - LO_ENERGY_BIN_LOWERS, + ), + "energy_delta_plus": ( + ["energy"], + LO_ENERGY_BIN_UPPERS - LO_ENERGIES_IN_KEV, + ), + **{ + quantity: (map_data_coords, data) + for quantity, data in loaded_data.items() + }, + **additional_map_data, }, coords={ "epoch": np.array([start_date_nanoseconds]), @@ -196,11 +225,15 @@ def load(cls, ) if "ena_intensity_stat_var" in l2_dataset.data_vars: - l2_dataset["ena_intensity_stat_uncert"] = np.sqrt(l2_dataset["ena_intensity_stat_var"]) + l2_dataset["ena_intensity_stat_uncert"] = np.sqrt( + l2_dataset["ena_intensity_stat_var"] + ) l2_dataset.drop_vars(["ena_intensity_stat_var"]) if "bg_intensity_stat_var" in l2_dataset.data_vars: - l2_dataset["bg_intensity_stat_uncert"] = np.sqrt(l2_dataset["bg_intensity_stat_var"]) + l2_dataset["bg_intensity_stat_uncert"] = np.sqrt( + l2_dataset["bg_intensity_stat_var"] + ) l2_dataset.drop_vars(["bg_intensity_stat_var"]) variables_to_mask = [ @@ -244,7 +277,9 @@ def get_spx_dependencies(self): def get_survival_corrected_dependencies(self, l1c_files: list[Path]): # l1c_results = imap_data_access.query(instrument="lo", data_level="l1c", version="latest") # l1c_inputs = [Path(l1c["file_path"]) for l1c in l1c_results if l1c["repointing"] in self.repoints] - l1c_inputs = [p for p in l1c_files if ScienceFilePath(p.name).repointing in self.repoints] + l1c_inputs = [ + p for p in l1c_files if ScienceFilePath(p.name).repointing in self.repoints + ] glows_query_results = imap_data_access.query( instrument="glows", @@ -253,10 +288,17 @@ def get_survival_corrected_dependencies(self, l1c_files: list[Path]): version="latest", ) glows_paths = [ - Path(glows["file_path"]) for glows in glows_query_results if int(glows["repointing"]) in self.repoints + Path(glows["file_path"]) + for glows in glows_query_results + if int(glows["repointing"]) in self.repoints ] - return ProcessingInputCollection(*(ScienceInput(p.name) for p in [self.l2_cdf_path, *glows_paths, *l1c_inputs])) + return ProcessingInputCollection( + *( + ScienceInput(p.name) + for p in [self.l2_cdf_path, *glows_paths, *l1c_inputs] + ) + ) def make_l3_input_metadata(self, l3_descriptor: str) -> InputMetadata: return InputMetadata( @@ -270,30 +312,32 @@ def make_l3_input_metadata(self, l3_descriptor: str) -> InputMetadata: def copy_to_output_directory_and_rename_for_initial_release( - release_directory: Path, output_maps: list[Path] + release_directory: Path, output_maps: list[Path] ): for generated_path in output_maps: science_file_path = ScienceFilePath(generated_path.name) new_name = ( - "_".join( - [ - "imap", - science_file_path.instrument, - science_file_path.data_level, - science_file_path.descriptor + "-INITIAL", - science_file_path.start_date, - science_file_path.version, - ] - ) - + ".cdf" + "_".join( + [ + "imap", + science_file_path.instrument, + science_file_path.data_level, + science_file_path.descriptor + "-INITIAL", + science_file_path.start_date, + science_file_path.version, + ] + ) + + ".cdf" ) output_dir = release_directory / science_file_path.data_level output_dir.mkdir(parents=True, exist_ok=True) output_path = output_dir / new_name - with CDF(str(output_path), masterpath=str(generated_path), readonly=False) as cdf: + with CDF( + str(output_path), masterpath=str(generated_path), readonly=False + ) as cdf: cdf.attrs["Logical_file_id"] = output_path.stem @@ -301,7 +345,9 @@ def download_all_l1c(path: Path): original_data_dir = imap_data_access.config["DATA_DIR"] try: imap_data_access.config["DATA_DIR"] = path - all_l1c = imap_data_access.query(instrument="lo", data_level="l1c", version="latest") + all_l1c = imap_data_access.query( + instrument="lo", data_level="l1c", version="latest" + ) with ThreadPoolExecutor() as pool: for l1c in all_l1c: @@ -312,7 +358,11 @@ def download_all_l1c(path: Path): if __name__ == "__main__": - logging.basicConfig(force=True, level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + logging.basicConfig( + force=True, + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) output_data_path = get_run_local_data_path("lo_txt_pipeline") shutil.rmtree(output_data_path / "imap" / "lo" / "l2", ignore_errors=True) @@ -361,7 +411,8 @@ def download_all_l1c(path: Path): [spx_nsnbs_map] = LoProcessor( input_metadata=nbs_processing_input.make_l3_input_metadata( - f"l{pivot:03d}-spxnsnbs{descriptor_suffix}-h-sf-nsp-ram-hae-6deg-6mo"), + f"l{pivot:03d}-spxnsnbs{descriptor_suffix}-h-sf-nsp-ram-hae-6deg-6mo" + ), dependencies=nbs_processing_input.get_spx_dependencies(), ).process() @@ -372,25 +423,34 @@ def download_all_l1c(path: Path): cg_corrected_input_path / "outdir" / f"pivot_{pivot}", CGNameMapping(), masked, - exposure_factor=nbs_processing_input.dataset["exposure_factor"] + exposure_factor=nbs_processing_input.dataset["exposure_factor"], ) [spx_cg_nsp_map] = LoProcessor( input_metadata=cg_processing_input.make_l3_input_metadata( - f"l{pivot:03d}-spxsbs{descriptor_suffix}-h-hf-nsp-ram-hae-6deg-6mo"), - dependencies=cg_processing_input.get_spx_dependencies() + f"l{pivot:03d}-spxsbs{descriptor_suffix}-h-hf-nsp-ram-hae-6deg-6mo" + ), + dependencies=cg_processing_input.get_spx_dependencies(), ).process() - with furnished_metakernel(cg_processing_input.start_date, cg_processing_input.end_date, LO_SP_MAP_KERNELS): + with furnished_metakernel( + cg_processing_input.start_date, + cg_processing_input.end_date, + LO_SP_MAP_KERNELS, + ): [sp_map] = LoProcessor( input_metadata=cg_processing_input.make_l3_input_metadata( - f"l{pivot:03d}-enasbs{descriptor_suffix}-h-hf-sp-ram-hae-6deg-6mo"), - dependencies=cg_processing_input.get_survival_corrected_dependencies(l1c_paths), + f"l{pivot:03d}-enasbs{descriptor_suffix}-h-hf-sp-ram-hae-6deg-6mo" + ), + dependencies=cg_processing_input.get_survival_corrected_dependencies( + l1c_paths + ), ).process() [spx_cg_sp_map] = LoProcessor( input_metadata=cg_processing_input.make_l3_input_metadata( - f"l{pivot:03d}-spxsbs{descriptor_suffix}-h-hf-sp-ram-hae-6deg-6mo"), + f"l{pivot:03d}-spxsbs{descriptor_suffix}-h-hf-sp-ram-hae-6deg-6mo" + ), dependencies=ProcessingInputCollection(ScienceInput(sp_map.name)), ).process() @@ -398,32 +458,38 @@ def download_all_l1c(path: Path): manifest=manifest, l2_descriptor=f"l{pivot:03d}-enasnbs{descriptor_suffix}-h-sf-nsp-ram-hae-6deg-6mo", version=1, - input_path=sputter_or_bootstrap_input_path / "outdir" / f"pivot_{pivot}", + input_path=sputter_or_bootstrap_input_path + / "outdir" + / f"pivot_{pivot}", name_mapping=SputterOrBootStrapNameMapping("sput"), use_masked_data=masked, - exposure_factor=nbs_processing_input.dataset["exposure_factor"] + exposure_factor=nbs_processing_input.dataset["exposure_factor"], ) sputter_and_bootstrap_map = LoProcessingInput.load( manifest=manifest, l2_descriptor=f"l{pivot:03d}-enasbs{descriptor_suffix}-h-sf-nsp-ram-hae-6deg-6mo", version=1, - input_path=sputter_or_bootstrap_input_path / "outdir" / f"pivot_{pivot}", + input_path=sputter_or_bootstrap_input_path + / "outdir" + / f"pivot_{pivot}", name_mapping=SputterOrBootStrapNameMapping("boot"), use_masked_data=masked, - exposure_factor=nbs_processing_input.dataset["exposure_factor"] + exposure_factor=nbs_processing_input.dataset["exposure_factor"], ) - output_maps.extend([ - nbs_processing_input.l2_cdf_path, - spx_nsnbs_map, - cg_processing_input.l2_cdf_path, - spx_cg_nsp_map, - sp_map, - spx_cg_sp_map, - sputter_no_bootstrap_map.l2_cdf_path, - sputter_and_bootstrap_map.l2_cdf_path - ]) + output_maps.extend( + [ + nbs_processing_input.l2_cdf_path, + spx_nsnbs_map, + cg_processing_input.l2_cdf_path, + spx_cg_nsp_map, + sp_map, + spx_cg_sp_map, + sputter_no_bootstrap_map.l2_cdf_path, + sputter_and_bootstrap_map.l2_cdf_path, + ] + ) maps_needed_for_combination = [ ( @@ -473,27 +539,32 @@ def download_all_l1c(path: Path): for combined_descriptor, combined_dependencies in maps_needed_for_combination: combined_inputs = [] for output_map in output_maps: - if any([f"_{combined_dep}_" in output_map.name for combined_dep in combined_dependencies]): + if any( + [ + f"_{combined_dep}_" in output_map.name + for combined_dep in combined_dependencies + ] + ): combined_inputs.append(output_map) [combined_sp_map] = LoProcessor( - dependencies=ProcessingInputCollection(*[ScienceInput(map_path.name) for map_path in combined_inputs]), + dependencies=ProcessingInputCollection( + *[ScienceInput(map_path.name) for map_path in combined_inputs] + ), input_metadata=InputMetadata( instrument="lo", data_level="l3", start_date=combined_start_date, end_date=combined_end_date, version=VersionMap({}, Version(None, 2)), - descriptor=combined_descriptor - ) + descriptor=combined_descriptor, + ), ).process() - combined_spx_descriptor = combined_descriptor.replace( - "-enasbsMsk-", "-spxsbsMsk-" - ).replace( - "-enansnbsMsk-", "-spxnsnbsMsk-" - ).replace( - "-enasnbsMsk-", "-spxsnbsMsk-" + combined_spx_descriptor = ( + combined_descriptor.replace("-enasbsMsk-", "-spxsbsMsk-") + .replace("-enansnbsMsk-", "-spxnsnbsMsk-") + .replace("-enasnbsMsk-", "-spxsnbsMsk-") ) [combined_spx_map] = LoProcessor( @@ -508,10 +579,9 @@ def download_all_l1c(path: Path): ), ).process() - output_maps.extend([ - combined_sp_map, - combined_spx_map - ]) + output_maps.extend([combined_sp_map, combined_spx_map]) release_directory = get_run_local_data_path("IMAP-Lo June 2nd 2026 Maps") - copy_to_output_directory_and_rename_for_initial_release(release_directory, output_maps) + copy_to_output_directory_and_rename_for_initial_release( + release_directory, output_maps + ) diff --git a/imap_l3_processing/maps/hilo_l3_survival_dependencies.py b/imap_l3_processing/maps/hilo_l3_survival_dependencies.py index 69ddcb71d..73052d515 100644 --- a/imap_l3_processing/maps/hilo_l3_survival_dependencies.py +++ b/imap_l3_processing/maps/hilo_l3_survival_dependencies.py @@ -7,10 +7,20 @@ from imap_data_access import ScienceFilePath from imap_data_access.processing_input import ProcessingInputCollection -from imap_l3_processing.hi.utils import read_l1c_rectangular_pointing_set_data, read_glows_l3e_data -from imap_l3_processing.maps.map_descriptors import MapDescriptorParts, parse_map_descriptor, SpinPhase -from imap_l3_processing.maps.map_models import RectangularIntensityMapData, GlowsL3eRectangularMapInputData, \ - InputRectangularPointingSet +from imap_l3_processing.hi.utils import ( + read_l1c_rectangular_pointing_set_data, + read_glows_l3e_data, +) +from imap_l3_processing.maps.map_descriptors import ( + MapDescriptorParts, + parse_map_descriptor, + SpinPhase, +) +from imap_l3_processing.maps.map_models import ( + RectangularIntensityMapData, + GlowsL3eRectangularMapInputData, + InputRectangularPointingSet, +) from imap_l3_processing.models import Instrument @@ -24,23 +34,39 @@ class HiLoL3SurvivalDependencies: dependency_file_paths: list[Path] = field(default_factory=list) @classmethod - def fetch_dependencies(cls, dependencies: ProcessingInputCollection, - instrument: Instrument) -> HiLoL3SurvivalDependencies: - [l2_map_path] = dependencies.get_file_paths(source=instrument.value, data_type="l2") - l1c_paths = dependencies.get_file_paths(source=instrument.value, data_type="l1c") + def fetch_dependencies( + cls, dependencies: ProcessingInputCollection, instrument: Instrument + ) -> HiLoL3SurvivalDependencies: + [l2_map_path] = dependencies.get_file_paths( + source=instrument.value, data_type="l2" + ) + l1c_paths = dependencies.get_file_paths( + source=instrument.value, data_type="l1c" + ) glows_paths = dependencies.get_file_paths(source="glows") map_file_path = imap_data_access.download(l2_map_path.name) - l1c_downloaded_paths = [imap_data_access.download(l1c.name) for l1c in l1c_paths] - glows_downloaded_paths = [imap_data_access.download(path.name) for path in glows_paths] + l1c_downloaded_paths = [ + imap_data_access.download(l1c.name) for l1c in l1c_paths + ] + glows_downloaded_paths = [ + imap_data_access.download(path.name) for path in glows_paths + ] l2_descriptor = ScienceFilePath(l2_map_path).descriptor - return cls.from_file_paths(map_file_path, l1c_downloaded_paths, glows_downloaded_paths, l2_descriptor) + return cls.from_file_paths( + map_file_path, l1c_downloaded_paths, glows_downloaded_paths, l2_descriptor + ) @classmethod - def from_file_paths(cls, map_file_path: Path, l1c_paths: list[Path], - glows_l3e_paths: list[Path], l2_descriptor: str) -> HiLoL3SurvivalDependencies: + def from_file_paths( + cls, + map_file_path: Path, + l1c_paths: list[Path], + glows_l3e_paths: list[Path], + l2_descriptor: str, + ) -> HiLoL3SurvivalDependencies: glows_l3e_data = list(map(read_glows_l3e_data, glows_l3e_paths)) l1c_data = list(map(read_l1c_rectangular_pointing_set_data, l1c_paths)) @@ -48,10 +74,13 @@ def from_file_paths(cls, map_file_path: Path, l1c_paths: list[Path], paths = [map_file_path] + l1c_paths + glows_l3e_paths - return cls(l2_data, l1c_data=l1c_data, - glows_l3e_data=glows_l3e_data, - l2_map_descriptor_parts=parse_map_descriptor(l2_descriptor), - dependency_file_paths=paths) + return cls( + l2_data, + l1c_data=l1c_data, + glows_l3e_data=glows_l3e_data, + l2_map_descriptor_parts=parse_map_descriptor(l2_descriptor), + dependency_file_paths=paths, + ) @dataclass @@ -60,21 +89,37 @@ class HiL3SingleSensorFullSpinDependencies: antiram_dependencies: HiLoL3SurvivalDependencies @classmethod - def fetch_dependencies(cls, dependencies: ProcessingInputCollection) -> HiL3SingleSensorFullSpinDependencies: - parsed_descriptors = [parse_map_descriptor(pi.descriptor) for pi in dependencies.processing_input] - - ram_dependencies = [pi for pi, descriptor in zip(dependencies.processing_input, parsed_descriptors) if - descriptor is None or descriptor.spin_phase == SpinPhase.RamOnly] - - antiram_dependencies = [pi for pi, descriptor in zip(dependencies.processing_input, parsed_descriptors) if - descriptor is None or descriptor.spin_phase == SpinPhase.AntiRamOnly] + def fetch_dependencies( + cls, dependencies: ProcessingInputCollection + ) -> HiL3SingleSensorFullSpinDependencies: + parsed_descriptors = [ + parse_map_descriptor(pi.descriptor) for pi in dependencies.processing_input + ] + + ram_dependencies = [ + pi + for pi, descriptor in zip(dependencies.processing_input, parsed_descriptors) + if descriptor is None or descriptor.spin_phase == SpinPhase.RamOnly + ] + + antiram_dependencies = [ + pi + for pi, descriptor in zip(dependencies.processing_input, parsed_descriptors) + if descriptor is None or descriptor.spin_phase == SpinPhase.AntiRamOnly + ] return cls( ram_dependencies=HiLoL3SurvivalDependencies.fetch_dependencies( - ProcessingInputCollection(*ram_dependencies), Instrument.IMAP_HI), + ProcessingInputCollection(*ram_dependencies), Instrument.IMAP_HI + ), antiram_dependencies=HiLoL3SurvivalDependencies.fetch_dependencies( - ProcessingInputCollection(*antiram_dependencies), Instrument.IMAP_HI)) + ProcessingInputCollection(*antiram_dependencies), Instrument.IMAP_HI + ), + ) @property def dependency_file_paths(self) -> list[Path]: - return self.ram_dependencies.dependency_file_paths + self.antiram_dependencies.dependency_file_paths + return ( + self.ram_dependencies.dependency_file_paths + + self.antiram_dependencies.dependency_file_paths + ) diff --git a/imap_l3_processing/maps/map_combination.py b/imap_l3_processing/maps/map_combination.py index b7852d051..568cfab29 100644 --- a/imap_l3_processing/maps/map_combination.py +++ b/imap_l3_processing/maps/map_combination.py @@ -4,31 +4,52 @@ import numpy as np from imap_l3_processing.data_utils import safe_divide -from imap_l3_processing.maps.map_models import IntensityMapData, HealPixIntensityMapData, RectangularIntensityMapData, \ - calculate_datetime_weighted_average +from imap_l3_processing.maps.map_models import ( + IntensityMapData, + HealPixIntensityMapData, + RectangularIntensityMapData, + calculate_datetime_weighted_average, +) class CombinationStrategy(ABC): - def combine_rectangular_intensity_map_data(self, - maps: list[RectangularIntensityMapData]) -> RectangularIntensityMapData: + def combine_rectangular_intensity_map_data( + self, maps: list[RectangularIntensityMapData] + ) -> RectangularIntensityMapData: for m in maps[1:]: - assert np.array_equal(maps[0].coords.latitude_delta, m.coords.latitude_delta) - assert np.array_equal(maps[0].coords.longitude_delta, m.coords.longitude_delta) - assert np.array_equal(maps[0].coords.latitude_label, m.coords.latitude_label) - assert np.array_equal(maps[0].coords.longitude_label, m.coords.longitude_label) + assert np.array_equal( + maps[0].coords.latitude_delta, m.coords.latitude_delta + ) + assert np.array_equal( + maps[0].coords.longitude_delta, m.coords.longitude_delta + ) + assert np.array_equal( + maps[0].coords.latitude_label, m.coords.latitude_label + ) + assert np.array_equal( + maps[0].coords.longitude_label, m.coords.longitude_label + ) self._check_maps_match([m.intensity_map_data for m in maps]) intensity_map_data = self._combine([m.intensity_map_data for m in maps]) - return RectangularIntensityMapData(intensity_map_data=intensity_map_data, coords=maps[0].coords) + return RectangularIntensityMapData( + intensity_map_data=intensity_map_data, coords=maps[0].coords + ) - def combine_healpix_intensity_map_data(self, maps: list[HealPixIntensityMapData]) -> HealPixIntensityMapData: + def combine_healpix_intensity_map_data( + self, maps: list[HealPixIntensityMapData] + ) -> HealPixIntensityMapData: for m in maps[1:]: assert np.array_equal(maps[0].coords.pixel_index, m.coords.pixel_index) - assert np.array_equal(maps[0].coords.pixel_index_label, m.coords.pixel_index_label) + assert np.array_equal( + maps[0].coords.pixel_index_label, m.coords.pixel_index_label + ) self._check_maps_match([m.intensity_map_data for m in maps]) intensity_map_data = self._combine([m.intensity_map_data for m in maps]) - return HealPixIntensityMapData(intensity_map_data=intensity_map_data, coords=maps[0].coords) + return HealPixIntensityMapData( + intensity_map_data=intensity_map_data, coords=maps[0].coords + ) @abstractmethod def _combine(self, maps: list[IntensityMapData]) -> IntensityMapData: @@ -62,20 +83,32 @@ def _check_maps_match(maps: list[IntensityMapData]): for field in dataclasses.fields(first_map): if field.name not in fields_which_may_differ: differing_fields.append(field.name) - supports_nan = np.issubdtype(first_map_dict[field.name].dtype, np.floating) + supports_nan = np.issubdtype( + first_map_dict[field.name].dtype, np.floating + ) assert np.all( - [np.array_equal(dataclasses.asdict(m)[field.name], first_map_dict[field.name], - equal_nan=supports_nan) - for m in maps]), f"{field.name}" + [ + np.array_equal( + dataclasses.asdict(m)[field.name], + first_map_dict[field.name], + equal_nan=supports_nan, + ) + for m in maps + ] + ), f"{field.name}" @staticmethod - def calculated_weighted_uncertainty(values: np.ndarray, weights: np.ndarray) -> np.ndarray: + def calculated_weighted_uncertainty( + values: np.ndarray, weights: np.ndarray + ) -> np.ndarray: masked_values = np.where(weights == 0, 0, values) numerator = np.sum(np.square(masked_values * weights), axis=0) return np.sqrt(safe_divide(numerator, np.square(np.sum(weights, axis=0)))) @staticmethod - def calculate_weighted_sys_err(values: np.ndarray, weights: np.ndarray) -> np.ndarray: + def calculate_weighted_sys_err( + values: np.ndarray, weights: np.ndarray + ) -> np.ndarray: numerator = np.sum(values * weights, axis=0) denominator = np.sum(weights, axis=0) @@ -89,7 +122,9 @@ def _combine(self, maps: list[IntensityMapData]) -> IntensityMapData: intensity_stat_unc = np.array([m.ena_intensity_stat_uncert for m in maps]) exposures = np.array([m.exposure_factor for m in maps]) if any(m.ena_intensity_sys_err_plus is not None for m in maps): - raise NotImplementedError("Unweighted combination does not yet combine asymmetric systemic errors") + raise NotImplementedError( + "Unweighted combination does not yet combine asymmetric systemic errors" + ) mask = np.isnan(intensities) | (exposures == 0) | np.isnan(exposures) @@ -121,7 +156,9 @@ def _combine(self, maps: list[IntensityMapData]) -> IntensityMapData: survival_probabilities = np.array([m.survival_probability for m in maps]) if all(m.survival_probability is not None for m in maps): sp_mask = ( - np.isnan(survival_probabilities) | (exposures == 0) | np.isnan(exposures) + np.isnan(survival_probabilities) + | (exposures == 0) + | np.isnan(exposures) ) masked_survival_probabilities = np.where(sp_mask, 0, survival_probabilities) sp_masked_weights = np.where(sp_mask, 0, 1) @@ -178,32 +215,44 @@ def _combine(self, maps: list[IntensityMapData]): survival_probabilities = np.array([m.survival_probability for m in maps]) if all(m.survival_probability is not None for m in maps): - sp_mask = np.isnan(survival_probabilities) | (exposures == 0) | np.isnan(exposures) + sp_mask = ( + np.isnan(survival_probabilities) + | (exposures == 0) + | np.isnan(exposures) + ) masked_survival_probabilities = np.where(sp_mask, 0, survival_probabilities) sp_masked_exposures = np.where(sp_mask, 0, exposures) - summed_survival_probability = np.sum(masked_survival_probabilities * sp_masked_exposures, axis=0) + summed_survival_probability = np.sum( + masked_survival_probabilities * sp_masked_exposures, axis=0 + ) exposure_weighted_summed_survival_probability = safe_divide( summed_survival_probability, np.sum(sp_masked_exposures, axis=0) ) else: exposure_weighted_summed_survival_probability = None - avg_obs_date = calculate_datetime_weighted_average(np.ma.array([m.obs_date for m in maps]), masked_exposures, 0) + avg_obs_date = calculate_datetime_weighted_average( + np.ma.array([m.obs_date for m in maps]), masked_exposures, 0 + ) bg_intensities = np.array([m.bg_intensity for m in maps]) bg_intensity_sys_err = np.array([m.bg_intensity_sys_err for m in maps]) bg_intensity_stat_uncert = np.array([m.bg_intensity_stat_uncert for m in maps]) - ena_intensity_sys_err_plus = np.array([m.ena_intensity_sys_err_plus for m in maps]) - ena_intensity_sys_err_minus = np.array([m.ena_intensity_sys_err_minus for m in maps]) + ena_intensity_sys_err_plus = np.array( + [m.ena_intensity_sys_err_plus for m in maps] + ) + ena_intensity_sys_err_minus = np.array( + [m.ena_intensity_sys_err_minus for m in maps] + ) def all_elem_defined(array): return all(v is not None for v in array) if ( - all_elem_defined(bg_intensities) - and all_elem_defined(bg_intensity_sys_err) - and all_elem_defined(bg_intensity_stat_uncert) + all_elem_defined(bg_intensities) + and all_elem_defined(bg_intensity_sys_err) + and all_elem_defined(bg_intensity_stat_uncert) ): bg_mask = np.isnan(bg_intensities) | (exposures == 0) | np.isnan(exposures) @@ -212,27 +261,35 @@ def all_elem_defined(array): bg_intensity_stat_uncert = np.where(bg_mask, 0, bg_intensity_stat_uncert) bg_masked_exposures = np.where(bg_mask, 0, exposures) - combined_bg_intensity_stat_uncert = self.calculated_weighted_uncertainty(bg_intensity_stat_uncert, - bg_masked_exposures) - combined_bg_intensity_sys_err = self.calculate_weighted_sys_err(bg_intensity_sys_err, bg_masked_exposures) + combined_bg_intensity_stat_uncert = self.calculated_weighted_uncertainty( + bg_intensity_stat_uncert, bg_masked_exposures + ) + combined_bg_intensity_sys_err = self.calculate_weighted_sys_err( + bg_intensity_sys_err, bg_masked_exposures + ) summed_bg_intensity = np.sum(bg_intensities * bg_masked_exposures, axis=0) summed_exposures = np.sum(bg_masked_exposures, axis=0) - exposure_weighted_summed_bg_intensity = safe_divide(summed_bg_intensity, summed_exposures) + exposure_weighted_summed_bg_intensity = safe_divide( + summed_bg_intensity, summed_exposures + ) else: exposure_weighted_summed_bg_intensity = None combined_bg_intensity_sys_err = None combined_bg_intensity_stat_uncert = None - if ( - all_elem_defined(ena_intensity_sys_err_plus) - and all_elem_defined(ena_intensity_sys_err_minus) + if all_elem_defined(ena_intensity_sys_err_plus) and all_elem_defined( + ena_intensity_sys_err_minus ): assert np.all(np.isfinite(ena_intensity_sys_err_plus) | mask) assert np.all(np.isfinite(ena_intensity_sys_err_minus) | mask) ena_intensity_sys_err_plus = np.where(mask, 0, ena_intensity_sys_err_plus) ena_intensity_sys_err_minus = np.where(mask, 0, ena_intensity_sys_err_minus) - combined_sys_err_plus = self.calculate_weighted_sys_err(ena_intensity_sys_err_plus, masked_exposures) - combined_sys_err_minus = self.calculate_weighted_sys_err(ena_intensity_sys_err_minus, masked_exposures) + combined_sys_err_plus = self.calculate_weighted_sys_err( + ena_intensity_sys_err_plus, masked_exposures + ) + combined_sys_err_minus = self.calculate_weighted_sys_err( + ena_intensity_sys_err_minus, masked_exposures + ) else: combined_sys_err_plus = None @@ -241,20 +298,21 @@ def all_elem_defined(array): quality_flags = np.array([m.quality_flags for m in maps]) combined_quality_flags = np.bitwise_or.reduce(quality_flags) - return dataclasses.replace(maps[0], - ena_intensity=exposure_weighted_summed_intensity, - ena_intensity_sys_err=combined_intensity_sys_err, - ena_intensity_stat_uncert=combined_intensity_stat_unc, - bg_intensity=exposure_weighted_summed_bg_intensity, - bg_intensity_sys_err=combined_bg_intensity_sys_err, - bg_intensity_stat_uncert=combined_bg_intensity_stat_uncert, - exposure_factor=summed_exposures, - obs_date=avg_obs_date, - survival_probability=exposure_weighted_summed_survival_probability, - ena_intensity_sys_err_plus=combined_sys_err_plus, - ena_intensity_sys_err_minus=combined_sys_err_minus, - quality_flags=combined_quality_flags - ) + return dataclasses.replace( + maps[0], + ena_intensity=exposure_weighted_summed_intensity, + ena_intensity_sys_err=combined_intensity_sys_err, + ena_intensity_stat_uncert=combined_intensity_stat_unc, + bg_intensity=exposure_weighted_summed_bg_intensity, + bg_intensity_sys_err=combined_bg_intensity_sys_err, + bg_intensity_stat_uncert=combined_bg_intensity_stat_uncert, + exposure_factor=summed_exposures, + obs_date=avg_obs_date, + survival_probability=exposure_weighted_summed_survival_probability, + ena_intensity_sys_err_plus=combined_sys_err_plus, + ena_intensity_sys_err_minus=combined_sys_err_minus, + quality_flags=combined_quality_flags, + ) class UncertaintyWeightedCombination(CombinationStrategy): @@ -266,10 +324,15 @@ def _combine(self, maps: list[IntensityMapData]) -> IntensityMapData: if any(m.ena_intensity_sys_err_plus is not None for m in maps): raise NotImplementedError( - "Uncertainty-weighted combination does not yet combine asymmetric systemic errors") + "Uncertainty-weighted combination does not yet combine asymmetric systemic errors" + ) - missing_weights_or_exposures = (exposures == 0) | np.isnan(exposures) | np.isnan(intensity_stat_unc) | np.isinf( - intensity_stat_unc) + missing_weights_or_exposures = ( + (exposures == 0) + | np.isnan(exposures) + | np.isnan(intensity_stat_unc) + | np.isinf(intensity_stat_unc) + ) inten_mask = np.isnan(intensities) | missing_weights_or_exposures intensities = np.where(inten_mask, 0, intensities) @@ -279,15 +342,24 @@ def _combine(self, maps: list[IntensityMapData]) -> IntensityMapData: summed_exposures = np.sum(masked_exposures, axis=0) squared_stat_unc = np.square(intensity_stat_unc) - inverse_squared_stat_unc = safe_divide(np.ones_like(squared_stat_unc), squared_stat_unc) + inverse_squared_stat_unc = safe_divide( + np.ones_like(squared_stat_unc), squared_stat_unc + ) uncertainty_weighted_combined_intensity = safe_divide( np.nansum(safe_divide(intensities, squared_stat_unc), axis=0), - np.nansum(inverse_squared_stat_unc, axis=0)) - combined_intensity_stat_unc = safe_divide(np.ones_like(inverse_squared_stat_unc[0]), - np.nansum(inverse_squared_stat_unc, axis=0)) - combined_intensity_sys_err = self.calculate_weighted_sys_err(intensity_sys_err, masked_exposures) + np.nansum(inverse_squared_stat_unc, axis=0), + ) + combined_intensity_stat_unc = safe_divide( + np.ones_like(inverse_squared_stat_unc[0]), + np.nansum(inverse_squared_stat_unc, axis=0), + ) + combined_intensity_sys_err = self.calculate_weighted_sys_err( + intensity_sys_err, masked_exposures + ) - avg_obs_date = calculate_datetime_weighted_average(np.ma.array([m.obs_date for m in maps]), masked_exposures, 0) + avg_obs_date = calculate_datetime_weighted_average( + np.ma.array([m.obs_date for m in maps]), masked_exposures, 0 + ) survival_probability = np.array([m.survival_probability for m in maps]) if all(m.survival_probability is not None for m in maps): @@ -295,11 +367,13 @@ def _combine(self, maps: list[IntensityMapData]) -> IntensityMapData: masked_sp = np.where(sp_mask, 0, survival_probability) sp_masked_stat_unc = np.where(sp_mask, 0, intensity_stat_unc) squared_sp_masked_stat_unc = np.square(sp_masked_stat_unc) - inverse_squared_sp_masked_stat_unc = safe_divide(np.ones_like(squared_stat_unc), squared_sp_masked_stat_unc) + inverse_squared_sp_masked_stat_unc = safe_divide( + np.ones_like(squared_stat_unc), squared_sp_masked_stat_unc + ) uncertainty_weighted_combined_sp = safe_divide( np.nansum(safe_divide(masked_sp, squared_sp_masked_stat_unc), axis=0), - np.nansum(inverse_squared_sp_masked_stat_unc, axis=0) + np.nansum(inverse_squared_sp_masked_stat_unc, axis=0), ) else: uncertainty_weighted_combined_sp = None @@ -307,12 +381,13 @@ def _combine(self, maps: list[IntensityMapData]) -> IntensityMapData: quality_flags = np.array([m.quality_flags for m in maps]) combined_quality_flags = np.bitwise_or.reduce(quality_flags) - return dataclasses.replace(maps[0], - ena_intensity=uncertainty_weighted_combined_intensity, - exposure_factor=summed_exposures, - ena_intensity_sys_err=combined_intensity_sys_err, - ena_intensity_stat_uncert=np.sqrt(combined_intensity_stat_unc), - obs_date=avg_obs_date, - survival_probability=uncertainty_weighted_combined_sp, - quality_flags=combined_quality_flags - ) + return dataclasses.replace( + maps[0], + ena_intensity=uncertainty_weighted_combined_intensity, + exposure_factor=summed_exposures, + ena_intensity_sys_err=combined_intensity_sys_err, + ena_intensity_stat_uncert=np.sqrt(combined_intensity_stat_unc), + obs_date=avg_obs_date, + survival_probability=uncertainty_weighted_combined_sp, + quality_flags=combined_quality_flags, + ) diff --git a/imap_l3_processing/maps/map_descriptors.py b/imap_l3_processing/maps/map_descriptors.py index f94912aea..ec82313fd 100644 --- a/imap_l3_processing/maps/map_descriptors.py +++ b/imap_l3_processing/maps/map_descriptors.py @@ -19,15 +19,19 @@ class Sensor(enum.Enum): @staticmethod def get_sensor_angle(sensor_name): - sensor_angles = {Sensor.Hi45: -45, - Sensor.Hi90: 0, - Sensor.Lo75: 15, - Sensor.Lo90: 0, - Sensor.Lo105: -15, - Sensor.Ultra45: -45, - Sensor.Ultra90: 0} - - assert sensor_name in sensor_angles, "Invalid to attempt to get a sensor angle for " + sensor_name.name + sensor_angles = { + Sensor.Hi45: -45, + Sensor.Hi90: 0, + Sensor.Lo75: 15, + Sensor.Lo90: 0, + Sensor.Lo105: -15, + Sensor.Ultra45: -45, + Sensor.Ultra90: 0, + } + + assert sensor_name in sensor_angles, ( + "Invalid to attempt to get a sensor angle for " + sensor_name.name + ) return sensor_angles[sensor_name] @@ -76,7 +80,7 @@ class MapDescriptorParts: duration: str quantity: MapQuantity quantity_suffix: str - spectral_index_energy_range: Optional[tuple[int,int]] = None + spectral_index_energy_range: Optional[tuple[int, int]] = None sensor_mapping = [ @@ -89,31 +93,31 @@ class MapDescriptorParts: ("ilo", Sensor.LoCombined), ("ulc", Sensor.UltraCombined), ("u45", Sensor.Ultra45), - ("u90", Sensor.Ultra90) + ("u90", Sensor.Ultra90), ] quantity_mapping = [ ("spx", MapQuantity.SpectralIndex), ("spxnbs", MapQuantity.SpectralIndexNBS), ("ena", MapQuantity.Intensity), - ("isn", MapQuantity.ISNBackgroundSubtracted) + ("isn", MapQuantity.ISNBackgroundSubtracted), ] cg_correction_mapping = [ ("sf", ReferenceFrame.Spacecraft), ("hf", ReferenceFrame.Heliospheric), - ("hk", ReferenceFrame.HeliosphericKinematic) + ("hk", ReferenceFrame.HeliosphericKinematic), ] sp_correction_mapping = [ ("sp", SurvivalCorrection.SurvivalCorrected), - ("nsp", SurvivalCorrection.NotSurvivalCorrected) + ("nsp", SurvivalCorrection.NotSurvivalCorrected), ] spin_phase_mapping = [ ("ram", SpinPhase.RamOnly), ("anti", SpinPhase.AntiRamOnly), - ("full", SpinPhase.FullSpin) + ("full", SpinPhase.FullSpin), ] grid_size_mapping = [ @@ -122,7 +126,7 @@ class MapDescriptorParts: ("4deg", PixelSize.FourDegrees), ("6deg", PixelSize.SixDegrees), ("nside8", PixelSize.Nside8), - ("nside16", PixelSize.Nside16) + ("nside16", PixelSize.Nside16), ] @@ -144,51 +148,80 @@ def parse_map_descriptor(descriptor: str) -> Optional[MapDescriptorParts]: return None sensor_part_from_str = {desc: sensor_part for desc, sensor_part in sensor_mapping} - quantity_part_from_str = {desc: quantity_part for desc, quantity_part in quantity_mapping} - cg_correction_part_from_str = {desc: cg_correction_part for desc, cg_correction_part in cg_correction_mapping} - sp_correction_part_from_str = {desc: sp_correction_part for desc, sp_correction_part in sp_correction_mapping} - spin_phase_part_from_str = {desc: spin_phase_part for desc, spin_phase_part in spin_phase_mapping} - grid_size_part_from_str = {desc: grid_size_part for desc, grid_size_part in grid_size_mapping} + quantity_part_from_str = { + desc: quantity_part for desc, quantity_part in quantity_mapping + } + cg_correction_part_from_str = { + desc: cg_correction_part for desc, cg_correction_part in cg_correction_mapping + } + sp_correction_part_from_str = { + desc: sp_correction_part for desc, sp_correction_part in sp_correction_mapping + } + spin_phase_part_from_str = { + desc: spin_phase_part for desc, spin_phase_part in spin_phase_mapping + } + grid_size_part_from_str = { + desc: grid_size_part for desc, grid_size_part in grid_size_mapping + } spectral_index_energy_range = None if descriptor_part_match["spectral_index_range"] is not None: - spectral_index_energy_range = (int(descriptor_part_match["spectral_index_range"][:2]), int(descriptor_part_match["spectral_index_range"][2:])) + spectral_index_energy_range = ( + int(descriptor_part_match["spectral_index_range"][:2]), + int(descriptor_part_match["spectral_index_range"][2:]), + ) return MapDescriptorParts( sensor=sensor_part_from_str[descriptor_part_match["sensor"]], quantity=quantity_part_from_str[descriptor_part_match["quantity"]], quantity_suffix=descriptor_part_match["quantity_suffix"], reference_frame=cg_correction_part_from_str[descriptor_part_match["frame"]], - survival_correction=sp_correction_part_from_str[descriptor_part_match["survival_corrected"]], + survival_correction=sp_correction_part_from_str[ + descriptor_part_match["survival_corrected"] + ], spin_phase=spin_phase_part_from_str[descriptor_part_match["spin_phase"]], coord=descriptor_part_match["coord"], grid=grid_size_part_from_str[descriptor_part_match["grid"]], duration=descriptor_part_match["duration"], - spectral_index_energy_range=spectral_index_energy_range + spectral_index_energy_range=spectral_index_energy_range, ) def map_descriptor_parts_to_string(descriptor_parts: MapDescriptorParts) -> str: sensor_part_to_str = {sensor_part: desc for desc, sensor_part in sensor_mapping} - quantity_part_to_str = {quantity_part: desc for desc, quantity_part in quantity_mapping} - cg_correction_part_to_str = {cg_correction_part: desc for desc, cg_correction_part in cg_correction_mapping} - sp_correction_part_to_str = {sp_correction_part: desc for desc, sp_correction_part in sp_correction_mapping} - spin_phase_part_to_str = {spin_phase_part: desc for desc, spin_phase_part in spin_phase_mapping} - grid_size_part_to_str = {grid_size_part: desc for desc, grid_size_part in grid_size_mapping} + quantity_part_to_str = { + quantity_part: desc for desc, quantity_part in quantity_mapping + } + cg_correction_part_to_str = { + cg_correction_part: desc for desc, cg_correction_part in cg_correction_mapping + } + sp_correction_part_to_str = { + sp_correction_part: desc for desc, sp_correction_part in sp_correction_mapping + } + spin_phase_part_to_str = { + spin_phase_part: desc for desc, spin_phase_part in spin_phase_mapping + } + grid_size_part_to_str = { + grid_size_part: desc for desc, grid_size_part in grid_size_mapping + } spectral_index_energy_range = "" if descriptor_parts.spectral_index_energy_range is not None: spectral_index_energy_range = f"{descriptor_parts.spectral_index_energy_range[0]:02}{descriptor_parts.spectral_index_energy_range[1]:02}" - return "-".join([ - sensor_part_to_str[descriptor_parts.sensor], - quantity_part_to_str[descriptor_parts.quantity] + spectral_index_energy_range + descriptor_parts.quantity_suffix, - "h", - cg_correction_part_to_str[descriptor_parts.reference_frame], - sp_correction_part_to_str[descriptor_parts.survival_correction], - spin_phase_part_to_str[descriptor_parts.spin_phase], - descriptor_parts.coord, - grid_size_part_to_str[descriptor_parts.grid], - descriptor_parts.duration, - ]) + return "-".join( + [ + sensor_part_to_str[descriptor_parts.sensor], + quantity_part_to_str[descriptor_parts.quantity] + + spectral_index_energy_range + + descriptor_parts.quantity_suffix, + "h", + cg_correction_part_to_str[descriptor_parts.reference_frame], + sp_correction_part_to_str[descriptor_parts.survival_correction], + spin_phase_part_to_str[descriptor_parts.spin_phase], + descriptor_parts.coord, + grid_size_part_to_str[descriptor_parts.grid], + descriptor_parts.duration, + ] + ) def get_duration_from_map_descriptor(descriptor: MapDescriptorParts) -> timedelta: @@ -202,4 +235,6 @@ def get_duration_from_map_descriptor(descriptor: MapDescriptorParts) -> timedelt case MapDescriptorParts(duration="1yr" | "12mo"): return timedelta(days=365.25) case _: - raise ValueError(f"Expected a duration in the map descriptor, got: {descriptor} (e.g., '1mo', '3mo')") + raise ValueError( + f"Expected a duration in the map descriptor, got: {descriptor} (e.g., '1mo', '3mo')" + ) diff --git a/imap_l3_processing/maps/map_initializer.py b/imap_l3_processing/maps/map_initializer.py index 98ed0daea..20fe45d11 100644 --- a/imap_l3_processing/maps/map_initializer.py +++ b/imap_l3_processing/maps/map_initializer.py @@ -22,13 +22,19 @@ class PossibleMapToProduce: @property def processing_input_collection(self) -> ProcessingInputCollection: - return ProcessingInputCollection(*[generate_imap_input(file_path) for file_path in sorted(self.input_files)]) + return ProcessingInputCollection( + *[generate_imap_input(file_path) for file_path in sorted(self.input_files)] + ) class MapInitializer(abc.ABC): def __init__(self, instrument: str): - l3_query_results = imap_data_access.query(instrument=instrument, data_level='l3') - self.existing_l3_maps = self.get_latest_version_by_descriptor_and_start_date(l3_query_results) + l3_query_results = imap_data_access.query( + instrument=instrument, data_level="l3" + ) + self.existing_l3_maps = self.get_latest_version_by_descriptor_and_start_date( + l3_query_results + ) self.instrument = instrument @abc.abstractmethod @@ -39,16 +45,24 @@ def furnish_spice_dependencies(self, map_to_produce: PossibleMapToProduce): def get_maps_that_can_be_produced(self, descriptor: str): raise NotImplementedError() - def get_maps_that_should_be_produced(self, descriptor: str, major_version: int|None) -> list[PossibleMapToProduce]: + def get_maps_that_should_be_produced( + self, descriptor: str, major_version: int | None + ) -> list[PossibleMapToProduce]: possible_maps = self.get_maps_that_can_be_produced(descriptor) maps_to_make = [] for possible_map in possible_maps: start_time = possible_map.input_metadata.start_date.strftime("%Y%m%d") - possible_map.input_metadata.version = VersionMap({descriptor: Version(major_version, 1)}) - - if highest_version_path_by_start_date := self.existing_l3_maps.get(descriptor): - if path_with_highest_version := highest_version_path_by_start_date.get(start_time): + possible_map.input_metadata.version = VersionMap( + {descriptor: Version(major_version, 1)} + ) + + if highest_version_path_by_start_date := self.existing_l3_maps.get( + descriptor + ): + if path_with_highest_version := highest_version_path_by_start_date.get( + start_time + ): existing_parents = read_cdf_parents(path_with_highest_version) existing_path = ScienceFilePath(path_with_highest_version) if ( @@ -59,7 +73,9 @@ def get_maps_that_should_be_produced(self, descriptor: str, major_version: int|N new_version = Version( major_version, existing_path.minor_version + 1 ) - possible_map.input_metadata.version = VersionMap({descriptor: new_version}) + possible_map.input_metadata.version = VersionMap( + {descriptor: new_version} + ) maps_to_make.append(possible_map) return maps_to_make @@ -69,21 +85,37 @@ def get_l1c_parents_from_map(map_file_path: str) -> list[str]: for l1 in read_cdf_parents(map_file_path): try: l1c_science_file_path = ScienceFilePath(l1) - if l1c_science_file_path.data_level == 'l1c': + if l1c_science_file_path.data_level == "l1c": l1c_names.append(l1c_science_file_path.filename.name) except ImapFilePath.InvalidImapFileError: continue return l1c_names @staticmethod - def get_latest_version_by_descriptor_and_start_date(query_results: list[dict]) -> defaultdict: + def get_latest_version_by_descriptor_and_start_date( + query_results: list[dict], + ) -> defaultdict: qr_by_descriptor_and_start_date = defaultdict(list) for qr in query_results: - qr_by_descriptor_and_start_date[(qr['descriptor'], qr["start_date"])].append(qr) + qr_by_descriptor_and_start_date[ + (qr["descriptor"], qr["start_date"]) + ].append(qr) file_paths_by_descriptor = defaultdict(dict) - for (descriptor, start_date), query_results in qr_by_descriptor_and_start_date.items(): - highest_version_qr = max(query_results, key=lambda x: x['minor_version'] if 'minor_version' in x else Version.from_version(x['version']).minor) - file_paths_by_descriptor[descriptor][start_date] = Path(highest_version_qr['file_path']).name + for ( + descriptor, + start_date, + ), query_results in qr_by_descriptor_and_start_date.items(): + highest_version_qr = max( + query_results, + key=lambda x: ( + x["minor_version"] + if "minor_version" in x + else Version.from_version(x["version"]).minor + ), + ) + file_paths_by_descriptor[descriptor][start_date] = Path( + highest_version_qr["file_path"] + ).name return file_paths_by_descriptor diff --git a/imap_l3_processing/maps/map_models.py b/imap_l3_processing/maps/map_models.py index 94b72ff06..2ea828f65 100644 --- a/imap_l3_processing/maps/map_models.py +++ b/imap_l3_processing/maps/map_models.py @@ -13,7 +13,10 @@ from imap_processing.spice.geometry import SpiceFrame from spacepy.pycdf import CDF -from imap_l3_processing.cdf.cdf_utils import read_variable_and_mask_fill_values, read_numeric_variable +from imap_l3_processing.cdf.cdf_utils import ( + read_variable_and_mask_fill_values, + read_numeric_variable, +) from imap_l3_processing.constants import TT2000_EPOCH from imap_l3_processing.maps.quality_flags import MapL3Flags from imap_l3_processing.models import DataProduct, DataProductVariable, D @@ -39,7 +42,9 @@ ENA_SPECTRAL_INDEX_VAR_NAME = "ena_spectral_index" ENA_SPECTRAL_INDEX_STAT_UNC_VAR_NAME = "ena_spectral_index_stat_uncert" ENA_SPECTRAL_INDEX_SCALAR_COEFFICIENT_VAR_NAME = "ena_spectral_scalar" -ENA_SPECTRAL_INDEX_SCALAR_COEFFICIENT_STAT_UNCERT_VAR_NAME = "ena_spectral_scalar_stat_uncert" +ENA_SPECTRAL_INDEX_SCALAR_COEFFICIENT_STAT_UNCERT_VAR_NAME = ( + "ena_spectral_scalar_stat_uncert" +) ENA_SPECTRAL_INDEX_CHISQ_VAR_NAME = "ena_spectral_index_chisq" ENA_INTENSITY_VAR_NAME = "ena_intensity" @@ -195,42 +200,56 @@ def read_from_path(cls, cdf_path: Path | str) -> ISNRateData: if "obs_date" in cdf and "obs_date_range" in cdf: masked_obs_date = read_variable_and_mask_fill_values(cdf["obs_date"]) if np.issubdtype(cdf["obs_date"].dtype, np.number): - obs_date = convert_tt2000_time_to_datetime(masked_obs_date.filled(0)) - masked_obs_date = np.ma.masked_array(data=obs_date, mask=masked_obs_date.mask) - obs_date_range = read_variable_and_mask_fill_values(cdf["obs_date_range"]) + obs_date = convert_tt2000_time_to_datetime( + masked_obs_date.filled(0) + ) + masked_obs_date = np.ma.masked_array( + data=obs_date, mask=masked_obs_date.mask + ) + obs_date_range = read_variable_and_mask_fill_values( + cdf["obs_date_range"] + ) else: obs_date_shape = cdf["bg_rate"].shape all_mask_array = np.full(obs_date_shape, True) - masked_obs_date = np.ma.masked_array(np.full(obs_date_shape, TT2000_EPOCH), mask=all_mask_array) - obs_date_range = np.ma.masked_array(np.full(obs_date_shape, 0), - mask=all_mask_array, - dtype=np.int64) + masked_obs_date = np.ma.masked_array( + np.full(obs_date_shape, TT2000_EPOCH), mask=all_mask_array + ) + obs_date_range = np.ma.masked_array( + np.full(obs_date_shape, 0), mask=all_mask_array, dtype=np.int64 + ) return ISNRateData( - epoch=cdf['epoch'][...], - epoch_delta=read_variable_and_mask_fill_values(cdf['epoch_delta']), + epoch=cdf["epoch"][...], + epoch_delta=read_variable_and_mask_fill_values(cdf["epoch_delta"]), obs_date=masked_obs_date, obs_date_range=obs_date_range, - bg_rate=read_numeric_variable(cdf['bg_rate']), - bg_rate_stat_uncert=read_numeric_variable(cdf['bg_rate_stat_uncert']), - bg_rate_sys_err=read_numeric_variable(cdf['bg_rate_sys_err']), - ena_count_rate=read_numeric_variable(cdf['ena_count_rate']), - ena_count_rate_stat_uncert=read_numeric_variable(cdf['ena_count_rate_stat_uncert']), - latitude=(cdf['latitude'][...]), - latitude_delta=(cdf['latitude_delta'][...]), - latitude_label=(cdf['latitude_label'][...]), - longitude=(cdf['longitude'][...]), - longitude_delta=(cdf['longitude_delta'][...]), - longitude_label=(cdf['longitude_label'][...]), - ena_intensity=read_numeric_variable(cdf['ena_intensity']), - ena_intensity_stat_uncert=read_numeric_variable(cdf['ena_intensity_stat_uncert']), - ena_intensity_sys_err=read_numeric_variable(cdf['ena_intensity_sys_err']), - energy=(cdf['energy'][...]), - energy_delta_plus=(cdf['energy_delta_plus'][...]), - energy_delta_minus=(cdf['energy_delta_minus'][...]), - energy_label=(cdf['energy_label'][...]), - exposure_factor=read_numeric_variable(cdf['exposure_factor']), - solid_angle=read_numeric_variable(cdf['solid_angle']) + bg_rate=read_numeric_variable(cdf["bg_rate"]), + bg_rate_stat_uncert=read_numeric_variable(cdf["bg_rate_stat_uncert"]), + bg_rate_sys_err=read_numeric_variable(cdf["bg_rate_sys_err"]), + ena_count_rate=read_numeric_variable(cdf["ena_count_rate"]), + ena_count_rate_stat_uncert=read_numeric_variable( + cdf["ena_count_rate_stat_uncert"] + ), + latitude=(cdf["latitude"][...]), + latitude_delta=(cdf["latitude_delta"][...]), + latitude_label=(cdf["latitude_label"][...]), + longitude=(cdf["longitude"][...]), + longitude_delta=(cdf["longitude_delta"][...]), + longitude_label=(cdf["longitude_label"][...]), + ena_intensity=read_numeric_variable(cdf["ena_intensity"]), + ena_intensity_stat_uncert=read_numeric_variable( + cdf["ena_intensity_stat_uncert"] + ), + ena_intensity_sys_err=read_numeric_variable( + cdf["ena_intensity_sys_err"] + ), + energy=(cdf["energy"][...]), + energy_delta_plus=(cdf["energy_delta_plus"][...]), + energy_delta_minus=(cdf["energy_delta_minus"][...]), + energy_label=(cdf["energy_label"][...]), + exposure_factor=read_numeric_variable(cdf["exposure_factor"]), + solid_angle=read_numeric_variable(cdf["solid_angle"]), ) @@ -292,17 +311,19 @@ def to_healpix_skymap(self) -> HealpixSkyMap: ), "predicted_ephemeris_flag": ( full_shape, - self.intensity_map_data.quality_flags & MapL3Flags.PREDICTIVE_EPHEMERIS, + self.intensity_map_data.quality_flags + & MapL3Flags.PREDICTIVE_EPHEMERIS, ), "nominal_alpha_proton_ratio_flag": ( full_shape, - self.intensity_map_data.quality_flags & MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO, + self.intensity_map_data.quality_flags + & MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO, ), "persisted_last_point_flag": ( full_shape, - self.intensity_map_data.quality_flags & MapL3Flags.PERSISTED_LAST_POINT, - ) - + self.intensity_map_data.quality_flags + & MapL3Flags.PERSISTED_LAST_POINT, + ), }, coords={ CoordNames.TIME.value: self.intensity_map_data.epoch, @@ -313,12 +334,21 @@ def to_healpix_skymap(self) -> HealpixSkyMap: if self.intensity_map_data.survival_probability is not None: data_1d_with_sp = healpix_map.data_1d.assign( - survival_probability=(full_shape, self.intensity_map_data.survival_probability)) + survival_probability=( + full_shape, + self.intensity_map_data.survival_probability, + ) + ) healpix_map.data_1d = data_1d_with_sp - healpix_map.data_1d = healpix_map.data_1d \ - .assign({"obs_date": (full_shape, healpix_map.data_1d["obs_date"].values.astype(np.float64))}) \ - .rename({CoordNames.HEALPIX_INDEX.value: CoordNames.GENERIC_PIXEL.value}) + healpix_map.data_1d = healpix_map.data_1d.assign( + { + "obs_date": ( + full_shape, + healpix_map.data_1d["obs_date"].values.astype(np.float64), + ) + } + ).rename({CoordNames.HEALPIX_INDEX.value: CoordNames.GENERIC_PIXEL.value}) return healpix_map @@ -386,16 +416,26 @@ def _read_intensity_map_data_from_open_cdf(cdf: CDF) -> IntensityMapData: ) if "survival_probability" in cdf: - map_intensity_data.survival_probability = read_numeric_variable(cdf["survival_probability"]) + map_intensity_data.survival_probability = read_numeric_variable( + cdf["survival_probability"] + ) if "bg_intensity" in cdf: map_intensity_data.bg_intensity = read_numeric_variable(cdf["bg_intensity"]) - map_intensity_data.bg_intensity_sys_err = read_numeric_variable(cdf["bg_intensity_sys_err"]) - map_intensity_data.bg_intensity_stat_uncert = read_numeric_variable(cdf["bg_intensity_stat_uncert"]) + map_intensity_data.bg_intensity_sys_err = read_numeric_variable( + cdf["bg_intensity_sys_err"] + ) + map_intensity_data.bg_intensity_stat_uncert = read_numeric_variable( + cdf["bg_intensity_stat_uncert"] + ) if "ena_intensity_sys_err_minus" in cdf: - map_intensity_data.ena_intensity_sys_err_minus = read_numeric_variable(cdf["ena_intensity_sys_err_minus"]) - map_intensity_data.ena_intensity_sys_err_plus = read_numeric_variable(cdf["ena_intensity_sys_err_plus"]) + map_intensity_data.ena_intensity_sys_err_minus = read_numeric_variable( + cdf["ena_intensity_sys_err_minus"] + ) + map_intensity_data.ena_intensity_sys_err_plus = read_numeric_variable( + cdf["ena_intensity_sys_err_plus"] + ) return map_intensity_data @@ -403,7 +443,7 @@ def _read_intensity_map_data_from_open_cdf(cdf: CDF) -> IntensityMapData: def _read_healpix_coords_from_open_cdf(cdf: CDF) -> HealPixCoords: return HealPixCoords( pixel_index=cdf["pixel_index"][...], - pixel_index_label=cdf["pixel_index_label"][...] + pixel_index_label=cdf["pixel_index_label"][...], ) @@ -414,7 +454,9 @@ def _read_intensity_map_data_from_xarray(dataset: xarray.Dataset) -> IntensityMa epoch_delta=_replace_fill_values_in_xarray(dataset, "epoch_delta"), energy=dataset.coords[CoordNames.ENERGY_L2.value].values, energy_delta_plus=_replace_fill_values_in_xarray(dataset, "energy_delta_plus"), - energy_delta_minus=_replace_fill_values_in_xarray(dataset, "energy_delta_minus"), + energy_delta_minus=_replace_fill_values_in_xarray( + dataset, "energy_delta_minus" + ), energy_label=_replace_fill_values_in_xarray(dataset, "energy_label"), latitude=_replace_fill_values_in_xarray(dataset, "latitude"), longitude=_replace_fill_values_in_xarray(dataset, "longitude"), @@ -423,23 +465,30 @@ def _read_intensity_map_data_from_xarray(dataset: xarray.Dataset) -> IntensityMa obs_date_range=_replace_fill_values_in_xarray(dataset, "obs_date_range"), solid_angle=np.squeeze(_replace_fill_values_in_xarray(dataset, "solid_angle")), ena_intensity=intensity, - ena_intensity_stat_uncert=_replace_fill_values_in_xarray(dataset, "ena_intensity_stat_uncert"), - ena_intensity_sys_err=_replace_fill_values_in_xarray(dataset, "ena_intensity_sys_err"), + ena_intensity_stat_uncert=_replace_fill_values_in_xarray( + dataset, "ena_intensity_stat_uncert" + ), + ena_intensity_sys_err=_replace_fill_values_in_xarray( + dataset, "ena_intensity_sys_err" + ), quality_flags=np.full(intensity.shape, MapL3Flags.NONE), ) def _replace_fill_values_in_xarray(dataset, variable): - if 'FILLVAL' in dataset[variable].attrs: - return np.where(dataset[variable].values == dataset[variable].attrs['FILLVAL'], np.nan, - dataset[variable].values) + if "FILLVAL" in dataset[variable].attrs: + return np.where( + dataset[variable].values == dataset[variable].attrs["FILLVAL"], + np.nan, + dataset[variable].values, + ) return dataset[variable].values def _read_healpix_coords_from_xarray(dataset: xarray.Dataset) -> HealPixCoords: return HealPixCoords( pixel_index=dataset[CoordNames.HEALPIX_INDEX.value].values, - pixel_index_label=dataset["pixel_index_label"].values + pixel_index_label=dataset["pixel_index_label"].values, ) @@ -462,50 +511,93 @@ def to_data_product_variables(self) -> list[DataProductVariable]: raise NotImplementedError -class RectangularSpectralIndexDataProduct(MapDataProduct[RectangularSpectralIndexMapData]): +class RectangularSpectralIndexDataProduct( + MapDataProduct[RectangularSpectralIndexMapData] +): data: RectangularSpectralIndexMapData def to_data_product_variables(self) -> list[DataProductVariable]: - return _spectral_index_data_variables(self.data.spectral_index_map_data) \ - + _rectangular_coords_to_variables(self.data.coords) + return _spectral_index_data_variables( + self.data.spectral_index_map_data + ) + _rectangular_coords_to_variables(self.data.coords) class RectangularIntensityDataProduct(MapDataProduct[RectangularIntensityMapData]): data: RectangularIntensityMapData def to_data_product_variables(self) -> list[DataProductVariable]: - return _intensity_data_variables(self.data.intensity_map_data) \ - + _rectangular_coords_to_variables(self.data.coords) + return _intensity_data_variables( + self.data.intensity_map_data + ) + _rectangular_coords_to_variables(self.data.coords) -class ISNBackgroundSubtractedDataProduct(MapDataProduct[ISNBackgroundSubtractedMapData]): +class ISNBackgroundSubtractedDataProduct( + MapDataProduct[ISNBackgroundSubtractedMapData] +): data: ISNBackgroundSubtractedMapData def to_data_product_variables(self) -> list[DataProductVariable]: return [ DataProductVariable(EPOCH_VAR_NAME, self.data.isn_rate_map_data.epoch), - DataProductVariable(EPOCH_DELTA_VAR_NAME, self.data.isn_rate_map_data.epoch_delta), + DataProductVariable( + EPOCH_DELTA_VAR_NAME, self.data.isn_rate_map_data.epoch_delta + ), DataProductVariable(ENERGY_VAR_NAME, self.data.isn_rate_map_data.energy), - DataProductVariable(ENERGY_DELTA_PLUS_VAR_NAME, self.data.isn_rate_map_data.energy_delta_plus), - DataProductVariable(ENERGY_DELTA_MINUS_VAR_NAME, self.data.isn_rate_map_data.energy_delta_minus), - DataProductVariable(ENERGY_LABEL_VAR_NAME, self.data.isn_rate_map_data.energy_label), - DataProductVariable(EXPOSURE_FACTOR_VAR_NAME, self.data.isn_rate_map_data.exposure_factor), - DataProductVariable(SOLID_ANGLE_VAR_NAME, self.data.isn_rate_map_data.solid_angle), + DataProductVariable( + ENERGY_DELTA_PLUS_VAR_NAME, + self.data.isn_rate_map_data.energy_delta_plus, + ), + DataProductVariable( + ENERGY_DELTA_MINUS_VAR_NAME, + self.data.isn_rate_map_data.energy_delta_minus, + ), + DataProductVariable( + ENERGY_LABEL_VAR_NAME, self.data.isn_rate_map_data.energy_label + ), + DataProductVariable( + EXPOSURE_FACTOR_VAR_NAME, self.data.isn_rate_map_data.exposure_factor + ), + DataProductVariable( + SOLID_ANGLE_VAR_NAME, self.data.isn_rate_map_data.solid_angle + ), DataProductVariable(BG_RATE_VAR_NAME, self.data.isn_rate_map_data.bg_rate), - DataProductVariable(BG_RATE_STAT_UNCERT_VAR_NAME, self.data.isn_rate_map_data.bg_rate_stat_uncert), - DataProductVariable(BG_RATE_SYS_ERR_VAR_NAME, self.data.isn_rate_map_data.bg_rate_sys_err), - DataProductVariable(LATITUDE_VAR_NAME, self.data.isn_rate_map_data.latitude), - DataProductVariable(LATITUDE_DELTA_VAR_NAME, self.data.isn_rate_map_data.latitude_delta), - DataProductVariable(LATITUDE_LABEL_VAR_NAME, self.data.isn_rate_map_data.latitude_label), - DataProductVariable(LONGITUDE_VAR_NAME, self.data.isn_rate_map_data.longitude), - DataProductVariable(LONGITUDE_DELTA_VAR_NAME, self.data.isn_rate_map_data.longitude_delta), - DataProductVariable(LONGITUDE_LABEL_VAR_NAME, self.data.isn_rate_map_data.longitude_label), - DataProductVariable(ISN_BG_RATE_SUBTRACTED_VAR_NAME, - self.data.isn_rate_map_data.isn_bg_rate_subtracted), - DataProductVariable(ISN_BG_RATE_SUBTRACTED_VAR_SYS_ERR_NAME, - self.data.isn_rate_map_data.isn_bg_rate_subtracted_sys_err), - DataProductVariable(ISN_BG_RATE_SUBTRACTED_STAT_UNCERT_VAR_NAME, - self.data.isn_rate_map_data.isn_bg_rate_subtracted_stat_uncert) + DataProductVariable( + BG_RATE_STAT_UNCERT_VAR_NAME, + self.data.isn_rate_map_data.bg_rate_stat_uncert, + ), + DataProductVariable( + BG_RATE_SYS_ERR_VAR_NAME, self.data.isn_rate_map_data.bg_rate_sys_err + ), + DataProductVariable( + LATITUDE_VAR_NAME, self.data.isn_rate_map_data.latitude + ), + DataProductVariable( + LATITUDE_DELTA_VAR_NAME, self.data.isn_rate_map_data.latitude_delta + ), + DataProductVariable( + LATITUDE_LABEL_VAR_NAME, self.data.isn_rate_map_data.latitude_label + ), + DataProductVariable( + LONGITUDE_VAR_NAME, self.data.isn_rate_map_data.longitude + ), + DataProductVariable( + LONGITUDE_DELTA_VAR_NAME, self.data.isn_rate_map_data.longitude_delta + ), + DataProductVariable( + LONGITUDE_LABEL_VAR_NAME, self.data.isn_rate_map_data.longitude_label + ), + DataProductVariable( + ISN_BG_RATE_SUBTRACTED_VAR_NAME, + self.data.isn_rate_map_data.isn_bg_rate_subtracted, + ), + DataProductVariable( + ISN_BG_RATE_SUBTRACTED_VAR_SYS_ERR_NAME, + self.data.isn_rate_map_data.isn_bg_rate_subtracted_sys_err, + ), + DataProductVariable( + ISN_BG_RATE_SUBTRACTED_STAT_UNCERT_VAR_NAME, + self.data.isn_rate_map_data.isn_bg_rate_subtracted_stat_uncert, + ), ] @@ -526,7 +618,9 @@ def _map_data_to_variables(data: MapData) -> list[DataProductVariable]: ] -def _spectral_index_data_variables(data: SpectralIndexMapData) -> list[DataProductVariable]: +def _spectral_index_data_variables( + data: SpectralIndexMapData, +) -> list[DataProductVariable]: variables = _map_data_to_variables(data) + [ DataProductVariable(ENA_SPECTRAL_INDEX_VAR_NAME, data.ena_spectral_index), DataProductVariable( @@ -551,7 +645,9 @@ def _spectral_index_data_variables(data: SpectralIndexMapData) -> list[DataProdu def _intensity_data_variables(data: IntensityMapData) -> list[DataProductVariable]: intensity_variables = [ DataProductVariable(ENA_INTENSITY_VAR_NAME, data.ena_intensity), - DataProductVariable(ENA_INTENSITY_STAT_UNCERT_VAR_NAME, data.ena_intensity_stat_uncert), + DataProductVariable( + ENA_INTENSITY_STAT_UNCERT_VAR_NAME, data.ena_intensity_stat_uncert + ), DataProductVariable(ENA_INTENSITY_SYS_ERR_VAR_NAME, data.ena_intensity_sys_err), DataProductVariable(QUALITY_FLAGS_VAR_NAME, data.quality_flags), ] @@ -568,17 +664,31 @@ def _intensity_data_variables(data: IntensityMapData) -> list[DataProductVariabl ] ) if data.survival_probability is not None: - intensity_variables.extend([ - DataProductVariable(SURVIVAL_PROBABILITY_VAR_NAME, data.survival_probability), - ]) + intensity_variables.extend( + [ + DataProductVariable( + SURVIVAL_PROBABILITY_VAR_NAME, data.survival_probability + ), + ] + ) if data.ena_intensity_sys_err_minus is not None: intensity_variables.extend( - [DataProductVariable(ENA_INTENSITY_SYS_ERR_MINUS_VAR_NAME, data.ena_intensity_sys_err_minus), - DataProductVariable(ENA_INTENSITY_SYS_ERR_PLUS_VAR_NAME, data.ena_intensity_sys_err_plus)]) + [ + DataProductVariable( + ENA_INTENSITY_SYS_ERR_MINUS_VAR_NAME, + data.ena_intensity_sys_err_minus, + ), + DataProductVariable( + ENA_INTENSITY_SYS_ERR_PLUS_VAR_NAME, data.ena_intensity_sys_err_plus + ), + ] + ) return _map_data_to_variables(data) + intensity_variables -def _rectangular_coords_to_variables(coords: RectangularCoords) -> list[DataProductVariable]: +def _rectangular_coords_to_variables( + coords: RectangularCoords, +) -> list[DataProductVariable]: return [ DataProductVariable(LATITUDE_DELTA_VAR_NAME, coords.latitude_delta), DataProductVariable(LATITUDE_LABEL_VAR_NAME, coords.latitude_label), @@ -587,16 +697,17 @@ def _rectangular_coords_to_variables(coords: RectangularCoords) -> list[DataProd ] - -def calculate_datetime_weighted_average(data: np.ndarray, weights: np.ndarray, axis: int, - **kwargs) -> np.ma.masked_array: +def calculate_datetime_weighted_average( + data: np.ndarray, weights: np.ndarray, axis: int, **kwargs +) -> np.ma.masked_array: if isinstance(np.ravel(np.ma.getdata(data))[0], datetime): masked_indices = np.ma.getmask(data) masked_weights = weights.copy() masked_weights[masked_indices] = 0 - epoch_based_dates = np.array((np.ma.getdata(data) - TT2000_EPOCH) / timedelta(seconds=1), - dtype=float) + epoch_based_dates = np.array( + (np.ma.getdata(data) - TT2000_EPOCH) / timedelta(seconds=1), dtype=float + ) averaged_dates_as_seconds = np.ma.average( epoch_based_dates, weights=masked_weights, axis=axis, **kwargs diff --git a/imap_l3_processing/maps/map_processor.py b/imap_l3_processing/maps/map_processor.py index 99403f697..1ee3c6acb 100644 --- a/imap_l3_processing/maps/map_processor.py +++ b/imap_l3_processing/maps/map_processor.py @@ -7,9 +7,8 @@ class MapProcessor(Processor): - @abstractmethod - def process(self, spice_reference_frame: SpiceFrame = SpiceFrame.ECLIPJ2000) -> list[Path]: + def process( + self, spice_reference_frame: SpiceFrame = SpiceFrame.ECLIPJ2000 + ) -> list[Path]: pass - - diff --git a/imap_l3_processing/maps/mpfit.py b/imap_l3_processing/maps/mpfit.py index 6ccbf214d..1c4b34e07 100644 --- a/imap_l3_processing/maps/mpfit.py +++ b/imap_l3_processing/maps/mpfit.py @@ -1,31 +1,31 @@ """ Perform Levenberg-Marquardt least-squares minimization, based on MINPACK-1. - AUTHORS + AUTHORS The original version of this software, called LMFIT, was written in FORTRAN as part of the MINPACK-1 package by XXX. Craig Markwardt converted the FORTRAN code to IDL. The information for the IDL version is: - Craig B. Markwardt, NASA/GSFC Code 662, Greenbelt, MD 20770 - craigm@lheamail.gsfc.nasa.gov - UPDATED VERSIONs can be found on my WEB PAGE: - http://cow.physics.wisc.edu/~craigm/idl/idl.html + Craig B. Markwardt, NASA/GSFC Code 662, Greenbelt, MD 20770 + craigm@lheamail.gsfc.nasa.gov + UPDATED VERSIONs can be found on my WEB PAGE: + http://cow.physics.wisc.edu/~craigm/idl/idl.html Mark Rivers created this Python version from Craig's IDL version. - Mark Rivers, University of Chicago - Building 434A, Argonne National Laboratory - 9700 South Cass Avenue, Argonne, IL 60439 - rivers@cars.uchicago.edu - Updated versions can be found at http://cars.uchicago.edu/software + Mark Rivers, University of Chicago + Building 434A, Argonne National Laboratory + 9700 South Cass Avenue, Argonne, IL 60439 + rivers@cars.uchicago.edu + Updated versions can be found at http://cars.uchicago.edu/software Sergey Koposov converted the Mark's Python version from Numeric to numpy - Sergey Koposov, University of Cambridge, Institute of Astronomy, - Madingley road, CB3 0HA, Cambridge, UK - koposov@ast.cam.ac.uk - Updated versions can be found at http://code.google.com/p/astrolibpy/source/browse/trunk/ + Sergey Koposov, University of Cambridge, Institute of Astronomy, + Madingley road, CB3 0HA, Cambridge, UK + koposov@ast.cam.ac.uk + Updated versions can be found at http://code.google.com/p/astrolibpy/source/browse/trunk/ - DESCRIPTION + DESCRIPTION MPFIT uses the Levenberg-Marquardt technique to solve the least-squares problem. In its typical use, MPFIT will be used to @@ -79,7 +79,7 @@ least-squares minimization problem. - USER FUNCTION + USER FUNCTION The user must define a function which returns the appropriate values as specified above. The function should return the weighted @@ -89,15 +89,15 @@ function should be declared in the following way: def myfunct(p, fjac=None, x=None, y=None, err=None) - # Parameter values are passed in "p" - # If fjac==None then partial derivatives should not be - # computed. It will always be None if MPFIT is called with default - # flag. - model = F(x, p) - # Non-negative status value means MPFIT should continue, negative means - # stop the calculation. - status = 0 - return([status, (y-model)/err] + # Parameter values are passed in "p" + # If fjac==None then partial derivatives should not be + # computed. It will always be None if MPFIT is called with default + # flag. + model = F(x, p) + # Non-negative status value means MPFIT should continue, negative means + # stop the calculation. + status = 0 + return([status, (y-model)/err] See below for applications with analytical derivatives. @@ -116,7 +116,7 @@ def myfunct(p, fjac=None, x=None, y=None, err=None) -15 and -1 then MPFIT will stop the calculation and return to the caller. - ANALYTIC DERIVATIVES + ANALYTIC DERIVATIVES In the search for the best-fit solution, MPFIT by default calculates derivatives numerically via a finite difference @@ -131,21 +131,21 @@ def myfunct(p, fjac=None, x=None, y=None, err=None) FJAC, and if FJAC!=None then return the partial derivative array in the return list. def myfunct(p, fjac=None, x=None, y=None, err=None) - # Parameter values are passed in "p" - # If FJAC!=None then partial derivatives must be comptuer. - # FJAC contains an array of len(p), where each entry - # is 1 if that parameter is free and 0 if it is fixed. - model = F(x, p) - Non-negative status value means MPFIT should continue, negative means - # stop the calculation. - status = 0 - if (dojac): - pderiv = zeros([len(x), len(p)], Float) - for j in range(len(p)): - pderiv[:,j] = FGRAD(x, p, j) - else: - pderiv = None - return([status, (y-model)/err, pderiv] + # Parameter values are passed in "p" + # If FJAC!=None then partial derivatives must be comptuer. + # FJAC contains an array of len(p), where each entry + # is 1 if that parameter is free and 0 if it is fixed. + model = F(x, p) + Non-negative status value means MPFIT should continue, negative means + # stop the calculation. + status = 0 + if (dojac): + pderiv = zeros([len(x), len(p)], Float) + for j in range(len(p)): + pderiv[:,j] = FGRAD(x, p, j) + else: + pderiv = None + return([status, (y-model)/err, pderiv] where FGRAD(x, p, i) is a user function which must compute the derivative of the model with respect to parameter P[i] at X. When @@ -170,7 +170,7 @@ def myfunct(p, fjac=None, x=None, y=None, err=None) 50x50 image, "dp" should be 50x50xNPAR. - CONSTRAINING PARAMETER VALUES WITH THE PARINFO KEYWORD + CONSTRAINING PARAMETER VALUES WITH THE PARINFO KEYWORD The behavior of MPFIT can be modified with respect to each parameter to be fitted. A parameter value can be fixed; simple @@ -186,72 +186,72 @@ def myfunct(p, fjac=None, x=None, y=None, err=None) numerical order. The dictionary can have the following keys (none are required, keys are case insensitive): - 'value' - the starting parameter value (but see the START_PARAMS - parameter for more information). - - 'fixed' - a boolean value, whether the parameter is to be held - fixed or not. Fixed parameters are not varied by - MPFIT, but are passed on to MYFUNCT for evaluation. - - 'limited' - a two-element boolean array. If the first/second - element is set, then the parameter is bounded on the - lower/upper side. A parameter can be bounded on both - sides. Both LIMITED and LIMITS must be given - together. - - 'limits' - a two-element float array. Gives the - parameter limits on the lower and upper sides, - respectively. Zero, one or two of these values can be - set, depending on the values of LIMITED. Both LIMITED - and LIMITS must be given together. - - 'parname' - a string, giving the name of the parameter. The - fitting code of MPFIT does not use this tag in any - way. However, the default iterfunct will print the - parameter name if available. - - 'step' - the step size to be used in calculating the numerical - derivatives. If set to zero, then the step size is - computed automatically. Ignored when AUTODERIVATIVE=0. - - 'mpside' - the sidedness of the finite difference when computing - numerical derivatives. This field can take four - values: - - 0 - one-sided derivative computed automatically - 1 - one-sided derivative (f(x+h) - f(x) )/h - -1 - one-sided derivative (f(x) - f(x-h))/h - 2 - two-sided derivative (f(x+h) - f(x-h))/(2*h) - - Where H is the STEP parameter described above. The - "automatic" one-sided derivative method will chose a - direction for the finite difference which does not - violate any constraints. The other methods do not - perform this check. The two-sided method is in - principle more precise, but requires twice as many - function evaluations. Default: 0. - - 'mpmaxstep' - the maximum change to be made in the parameter - value. During the fitting process, the parameter - will never be changed by more than this value in - one iteration. - - A value of 0 indicates no maximum. Default: 0. - - 'tied' - a string expression which "ties" the parameter to other - free or fixed parameters. Any expression involving - constants and the parameter array P are permitted. - Example: if parameter 2 is always to be twice parameter - 1 then use the following: parinfo(2).tied = '2 * p(1)'. - Since they are totally constrained, tied parameters are - considered to be fixed; no errors are computed for them. - [ NOTE: the PARNAME can't be used in expressions. ] - - 'mpprint' - if set to 1, then the default iterfunct will print the - parameter value. If set to 0, the parameter value - will not be printed. This tag can be used to - selectively print only a few parameter values out of - many. Default: 1 (all parameters printed) + 'value' - the starting parameter value (but see the START_PARAMS + parameter for more information). + + 'fixed' - a boolean value, whether the parameter is to be held + fixed or not. Fixed parameters are not varied by + MPFIT, but are passed on to MYFUNCT for evaluation. + + 'limited' - a two-element boolean array. If the first/second + element is set, then the parameter is bounded on the + lower/upper side. A parameter can be bounded on both + sides. Both LIMITED and LIMITS must be given + together. + + 'limits' - a two-element float array. Gives the + parameter limits on the lower and upper sides, + respectively. Zero, one or two of these values can be + set, depending on the values of LIMITED. Both LIMITED + and LIMITS must be given together. + + 'parname' - a string, giving the name of the parameter. The + fitting code of MPFIT does not use this tag in any + way. However, the default iterfunct will print the + parameter name if available. + + 'step' - the step size to be used in calculating the numerical + derivatives. If set to zero, then the step size is + computed automatically. Ignored when AUTODERIVATIVE=0. + + 'mpside' - the sidedness of the finite difference when computing + numerical derivatives. This field can take four + values: + + 0 - one-sided derivative computed automatically + 1 - one-sided derivative (f(x+h) - f(x) )/h + -1 - one-sided derivative (f(x) - f(x-h))/h + 2 - two-sided derivative (f(x+h) - f(x-h))/(2*h) + + Where H is the STEP parameter described above. The + "automatic" one-sided derivative method will chose a + direction for the finite difference which does not + violate any constraints. The other methods do not + perform this check. The two-sided method is in + principle more precise, but requires twice as many + function evaluations. Default: 0. + + 'mpmaxstep' - the maximum change to be made in the parameter + value. During the fitting process, the parameter + will never be changed by more than this value in + one iteration. + + A value of 0 indicates no maximum. Default: 0. + + 'tied' - a string expression which "ties" the parameter to other + free or fixed parameters. Any expression involving + constants and the parameter array P are permitted. + Example: if parameter 2 is always to be twice parameter + 1 then use the following: parinfo(2).tied = '2 * p(1)'. + Since they are totally constrained, tied parameters are + considered to be fixed; no errors are computed for them. + [ NOTE: the PARNAME can't be used in expressions. ] + + 'mpprint' - if set to 1, then the default iterfunct will print the + parameter value. If set to 0, the parameter value + will not be printed. This tag can be used to + selectively print only a few parameter values out of + many. Default: 1 (all parameters printed) Future modifications to the PARINFO structure, if any, will involve @@ -262,7 +262,7 @@ def myfunct(p, fjac=None, x=None, y=None, err=None) PARINFO Example: parinfo = [{'value':0., 'fixed':0, 'limited':[0,0], 'limits':[0.,0.]} - for i in range(5)] + for i in range(5)] parinfo[0]['fixed'] = 1 parinfo[4]['limited'][0] = 1 parinfo[4]['limits'][0] = 50. @@ -275,14 +275,14 @@ def myfunct(p, fjac=None, x=None, y=None, err=None) constrained to be above 50. - EXAMPLE + EXAMPLE import mpfit import numpy.oldnumeric as Numeric x = arange(100, float) p0 = [5.7, 2.2, 500., 1.5, 2000.] y = ( p[0] + p[1]*[x] + p[2]*[x**2] + p[3]*sqrt(x) + - p[4]*log(x)) + p[4]*log(x)) fa = {'x':x, 'y':y, 'err':err} m = mpfit('myfunct', p0, functkw=fa) print 'status = ', m.status @@ -294,16 +294,16 @@ def myfunct(p, fjac=None, x=None, y=None, err=None) results can be obtained from the returned object m. - THEORY OF OPERATION + THEORY OF OPERATION There are many specific strategies for function minimization. One very popular technique is to use function gradient information to realize the local structure of the function. Near a local minimum the function value can be taylor expanded about x0 as follows: - f(x) = f(x0) + f'(x0) . (x-x0) + (1/2) (x-x0) . f''(x0) . (x-x0) - ----- --------------- ------------------------------- (1) - Order 0th 1st 2nd + f(x) = f(x0) + f'(x0) . (x-x0) + (1/2) (x-x0) . f''(x0) . (x-x0) + ----- --------------- ------------------------------- (1) + Order 0th 1st 2nd Here f'(x) is the gradient vector of f at x, and f''(x) is the Hessian matrix of second derivatives of f at x. The vector x is @@ -311,7 +311,7 @@ def myfunct(p, fjac=None, x=None, y=None, err=None) can find the minimum of f, f(xm) using Newton's method, and arrives at the following linear equation: - f''(x0) . (xm-x0) = - f'(x0) (2) + f''(x0) . (xm-x0) = - f'(x0) (2) If an inverse can be found for f''(x0) then one can solve for (xm-x0), the step vector from the current position x0 to the new @@ -323,7 +323,7 @@ def myfunct(p, fjac=None, x=None, y=None, err=None) It adds an additional diagonal term to the equation which may aid the convergence properties: - (f''(x0) + nu I) . (xm-x0) = -f'(x0) (2a) + (f''(x0) + nu I) . (xm-x0) = -f'(x0) (2a) where I is the identity matrix. When nu is large, the overall matrix is diagonally dominant, and the iterations follow steepest @@ -341,14 +341,14 @@ def myfunct(p, fjac=None, x=None, y=None, err=None) which assist in solving eqn (2). The function to be minimized is a sum of squares: - f = Sum(hi^2) (3) + f = Sum(hi^2) (3) where hi is the ith residual out of m residuals as described above. This can be substituted back into eqn (2) after computing the derivatives: - f' = 2 Sum(hi hi') - f'' = 2 Sum(hi' hj') + 2 Sum(hi hi'') (4) + f' = 2 Sum(hi hi') + f'' = 2 Sum(hi' hj') + 2 Sum(hi hi'') (4) If one assumes that the parameters are already close enough to a minimum, then one typically finds that the second term in f'' is @@ -358,7 +358,7 @@ def myfunct(p, fjac=None, x=None, y=None, err=None) In matrix notation, the combination of eqns (2) and (4) becomes: - hT' . h' . dx = - hT' . h (5) + hT' . h' . dx = - hT' . h (5) Where h is the residual vector (length m), hT is its transpose, h' is the Jacobian matrix (dimensions n x m), and dx is (xm-x0). The @@ -374,9 +374,9 @@ def myfunct(p, fjac=None, x=None, y=None, err=None) Q = I, and R is upper right triangular. Using h' = Q . R and the ortogonality of Q, eqn (5) becomes - (RT . QT) . (Q . R) . dx = - (RT . QT) . h - RT . R . dx = - RT . QT . h (6) - R . dx = - QT . h + (RT . QT) . (Q . R) . dx = - (RT . QT) . h + RT . R . dx = - RT . QT . h (6) + R . dx = - QT . h where the last statement follows because R is upper triangular. Here, R, QT and h are known so this is a matter of solving for dx. @@ -384,17 +384,17 @@ def myfunct(p, fjac=None, x=None, y=None, err=None) pivoting, and MPFIT_QRSOLV provides the solution for dx. - REFERENCES + REFERENCES MINPACK-1, Jorge More', available from netlib (www.netlib.org). "Optimization Software Guide," Jorge More' and Stephen Wright, - SIAM, *Frontiers in Applied Mathematics*, Number 14. + SIAM, *Frontiers in Applied Mathematics*, Number 14. More', Jorge J., "The Levenberg-Marquardt Algorithm: - Implementation and Theory," in *Numerical Analysis*, ed. Watson, - G. A., Lecture Notes in Mathematics 630, Springer-Verlag, 1977. + Implementation and Theory," in *Numerical Analysis*, ed. Watson, + G. A., Lecture Notes in Mathematics 630, Springer-Verlag, 1977. - MODIFICATION HISTORY + MODIFICATION HISTORY Translated from MINPACK-1 in FORTRAN, Apr-Jul 1998, CM Copyright (C) 1997-2002, Craig Markwardt @@ -412,431 +412,453 @@ def myfunct(p, fjac=None, x=None, y=None, err=None) import scipy.linalg.blas -# Original FORTRAN documentation -# ********** +# Original FORTRAN documentation +# ********** # -# subroutine lmdif +# subroutine lmdif # -# the purpose of lmdif is to minimize the sum of the squares of -# m nonlinear functions in n variables by a modification of -# the levenberg-marquardt algorithm. the user must provide a -# subroutine which calculates the functions. the jacobian is -# then calculated by a forward-difference approximation. +# the purpose of lmdif is to minimize the sum of the squares of +# m nonlinear functions in n variables by a modification of +# the levenberg-marquardt algorithm. the user must provide a +# subroutine which calculates the functions. the jacobian is +# then calculated by a forward-difference approximation. # -# the subroutine statement is +# the subroutine statement is # -# subroutine lmdif(fcn,m,n,x,fvec,ftol,xtol,gtol,maxfev,epsfcn, -# diag,mode,factor,nprint,info,nfev,fjac, -# ldfjac,ipvt,qtf,wa1,wa2,wa3,wa4) +# subroutine lmdif(fcn,m,n,x,fvec,ftol,xtol,gtol,maxfev,epsfcn, +# diag,mode,factor,nprint,info,nfev,fjac, +# ldfjac,ipvt,qtf,wa1,wa2,wa3,wa4) # -# where +# where # -# fcn is the name of the user-supplied subroutine which -# calculates the functions. fcn must be declared -# in an external statement in the user calling -# program, and should be written as follows. +# fcn is the name of the user-supplied subroutine which +# calculates the functions. fcn must be declared +# in an external statement in the user calling +# program, and should be written as follows. # -# subroutine fcn(m,n,x,fvec,iflag) -# integer m,n,iflag -# double precision x(n),fvec(m) -# ---------- -# calculate the functions at x and -# return this vector in fvec. -# ---------- -# return -# end +# subroutine fcn(m,n,x,fvec,iflag) +# integer m,n,iflag +# double precision x(n),fvec(m) +# ---------- +# calculate the functions at x and +# return this vector in fvec. +# ---------- +# return +# end # -# the value of iflag should not be changed by fcn unless -# the user wants to terminate execution of lmdif. -# in this case set iflag to a negative integer. +# the value of iflag should not be changed by fcn unless +# the user wants to terminate execution of lmdif. +# in this case set iflag to a negative integer. # -# m is a positive integer input variable set to the number -# of functions. +# m is a positive integer input variable set to the number +# of functions. # -# n is a positive integer input variable set to the number -# of variables. n must not exceed m. +# n is a positive integer input variable set to the number +# of variables. n must not exceed m. # -# x is an array of length n. on input x must contain -# an initial estimate of the solution vector. on output x -# contains the final estimate of the solution vector. +# x is an array of length n. on input x must contain +# an initial estimate of the solution vector. on output x +# contains the final estimate of the solution vector. # -# fvec is an output array of length m which contains -# the functions evaluated at the output x. +# fvec is an output array of length m which contains +# the functions evaluated at the output x. # -# ftol is a nonnegative input variable. termination -# occurs when both the actual and predicted relative -# reductions in the sum of squares are at most ftol. -# therefore, ftol measures the relative error desired -# in the sum of squares. +# ftol is a nonnegative input variable. termination +# occurs when both the actual and predicted relative +# reductions in the sum of squares are at most ftol. +# therefore, ftol measures the relative error desired +# in the sum of squares. # -# xtol is a nonnegative input variable. termination -# occurs when the relative error between two consecutive -# iterates is at most xtol. therefore, xtol measures the -# relative error desired in the approximate solution. +# xtol is a nonnegative input variable. termination +# occurs when the relative error between two consecutive +# iterates is at most xtol. therefore, xtol measures the +# relative error desired in the approximate solution. # -# gtol is a nonnegative input variable. termination -# occurs when the cosine of the angle between fvec and -# any column of the jacobian is at most gtol in absolute -# value. therefore, gtol measures the orthogonality -# desired between the function vector and the columns -# of the jacobian. +# gtol is a nonnegative input variable. termination +# occurs when the cosine of the angle between fvec and +# any column of the jacobian is at most gtol in absolute +# value. therefore, gtol measures the orthogonality +# desired between the function vector and the columns +# of the jacobian. # -# maxfev is a positive integer input variable. termination -# occurs when the number of calls to fcn is at least -# maxfev by the end of an iteration. +# maxfev is a positive integer input variable. termination +# occurs when the number of calls to fcn is at least +# maxfev by the end of an iteration. # -# epsfcn is an input variable used in determining a suitable -# step length for the forward-difference approximation. this -# approximation assumes that the relative errors in the -# functions are of the order of epsfcn. if epsfcn is less -# than the machine precision, it is assumed that the relative -# errors in the functions are of the order of the machine -# precision. +# epsfcn is an input variable used in determining a suitable +# step length for the forward-difference approximation. this +# approximation assumes that the relative errors in the +# functions are of the order of epsfcn. if epsfcn is less +# than the machine precision, it is assumed that the relative +# errors in the functions are of the order of the machine +# precision. # -# diag is an array of length n. if mode = 1 (see -# below), diag is internally set. if mode = 2, diag -# must contain positive entries that serve as -# multiplicative scale factors for the variables. +# diag is an array of length n. if mode = 1 (see +# below), diag is internally set. if mode = 2, diag +# must contain positive entries that serve as +# multiplicative scale factors for the variables. # -# mode is an integer input variable. if mode = 1, the -# variables will be scaled internally. if mode = 2, -# the scaling is specified by the input diag. other -# values of mode are equivalent to mode = 1. +# mode is an integer input variable. if mode = 1, the +# variables will be scaled internally. if mode = 2, +# the scaling is specified by the input diag. other +# values of mode are equivalent to mode = 1. # -# factor is a positive input variable used in determining the -# initial step bound. this bound is set to the product of -# factor and the euclidean norm of diag*x if nonzero, or else -# to factor itself. in most cases factor should lie in the -# interval (.1,100.). 100. is a generally recommended value. +# factor is a positive input variable used in determining the +# initial step bound. this bound is set to the product of +# factor and the euclidean norm of diag*x if nonzero, or else +# to factor itself. in most cases factor should lie in the +# interval (.1,100.). 100. is a generally recommended value. # -# nprint is an integer input variable that enables controlled -# printing of iterates if it is positive. in this case, -# fcn is called with iflag = 0 at the beginning of the first -# iteration and every nprint iterations thereafter and -# immediately prior to return, with x and fvec available -# for printing. if nprint is not positive, no special calls -# of fcn with iflag = 0 are made. +# nprint is an integer input variable that enables controlled +# printing of iterates if it is positive. in this case, +# fcn is called with iflag = 0 at the beginning of the first +# iteration and every nprint iterations thereafter and +# immediately prior to return, with x and fvec available +# for printing. if nprint is not positive, no special calls +# of fcn with iflag = 0 are made. # -# info is an integer output variable. if the user has -# terminated execution, info is set to the (negative) -# value of iflag. see description of fcn. otherwise, -# info is set as follows. +# info is an integer output variable. if the user has +# terminated execution, info is set to the (negative) +# value of iflag. see description of fcn. otherwise, +# info is set as follows. # -# info = 0 improper input parameters. +# info = 0 improper input parameters. # -# info = 1 both actual and predicted relative reductions -# in the sum of squares are at most ftol. +# info = 1 both actual and predicted relative reductions +# in the sum of squares are at most ftol. # -# info = 2 relative error between two consecutive iterates -# is at most xtol. +# info = 2 relative error between two consecutive iterates +# is at most xtol. # -# info = 3 conditions for info = 1 and info = 2 both hold. +# info = 3 conditions for info = 1 and info = 2 both hold. # -# info = 4 the cosine of the angle between fvec and any -# column of the jacobian is at most gtol in -# absolute value. +# info = 4 the cosine of the angle between fvec and any +# column of the jacobian is at most gtol in +# absolute value. # -# info = 5 number of calls to fcn has reached or -# exceeded maxfev. +# info = 5 number of calls to fcn has reached or +# exceeded maxfev. # -# info = 6 ftol is too small. no further reduction in -# the sum of squares is possible. +# info = 6 ftol is too small. no further reduction in +# the sum of squares is possible. # -# info = 7 xtol is too small. no further improvement in -# the approximate solution x is possible. +# info = 7 xtol is too small. no further improvement in +# the approximate solution x is possible. # -# info = 8 gtol is too small. fvec is orthogonal to the -# columns of the jacobian to machine precision. +# info = 8 gtol is too small. fvec is orthogonal to the +# columns of the jacobian to machine precision. # -# nfev is an integer output variable set to the number of -# calls to fcn. +# nfev is an integer output variable set to the number of +# calls to fcn. # -# fjac is an output m by n array. the upper n by n submatrix -# of fjac contains an upper triangular matrix r with -# diagonal elements of nonincreasing magnitude such that +# fjac is an output m by n array. the upper n by n submatrix +# of fjac contains an upper triangular matrix r with +# diagonal elements of nonincreasing magnitude such that # -# t t t -# p *(jac *jac)*p = r *r, +# t t t +# p *(jac *jac)*p = r *r, # -# where p is a permutation matrix and jac is the final -# calculated jacobian. column j of p is column ipvt(j) -# (see below) of the identity matrix. the lower trapezoidal -# part of fjac contains information generated during -# the computation of r. +# where p is a permutation matrix and jac is the final +# calculated jacobian. column j of p is column ipvt(j) +# (see below) of the identity matrix. the lower trapezoidal +# part of fjac contains information generated during +# the computation of r. # -# ldfjac is a positive integer input variable not less than m -# which specifies the leading dimension of the array fjac. +# ldfjac is a positive integer input variable not less than m +# which specifies the leading dimension of the array fjac. # -# ipvt is an integer output array of length n. ipvt -# defines a permutation matrix p such that jac*p = q*r, -# where jac is the final calculated jacobian, q is -# orthogonal (not stored), and r is upper triangular -# with diagonal elements of nonincreasing magnitude. -# column j of p is column ipvt(j) of the identity matrix. +# ipvt is an integer output array of length n. ipvt +# defines a permutation matrix p such that jac*p = q*r, +# where jac is the final calculated jacobian, q is +# orthogonal (not stored), and r is upper triangular +# with diagonal elements of nonincreasing magnitude. +# column j of p is column ipvt(j) of the identity matrix. # -# qtf is an output array of length n which contains -# the first n elements of the vector (q transpose)*fvec. +# qtf is an output array of length n which contains +# the first n elements of the vector (q transpose)*fvec. # -# wa1, wa2, and wa3 are work arrays of length n. +# wa1, wa2, and wa3 are work arrays of length n. # -# wa4 is a work array of length m. +# wa4 is a work array of length m. # -# subprograms called +# subprograms called # -# user-supplied ...... fcn +# user-supplied ...... fcn # -# minpack-supplied ... dpmpar,enorm,fdjac2,,qrfac +# minpack-supplied ... dpmpar,enorm,fdjac2,,qrfac # -# fortran-supplied ... dabs,dmax1,dmin1,dsqrt,mod +# fortran-supplied ... dabs,dmax1,dmin1,dsqrt,mod # -# argonne national laboratory. minpack project. march 1980. -# burton s. garbow, kenneth e. hillstrom, jorge j. more +# argonne national laboratory. minpack project. march 1980. +# burton s. garbow, kenneth e. hillstrom, jorge j. more # -# ********** +# ********** + class mpfit: - blas_enorm32, = scipy.linalg.blas.get_blas_funcs(['nrm2'], numpy.array([0], dtype=numpy.float32)) - blas_enorm64, = scipy.linalg.blas.get_blas_funcs(['nrm2'], numpy.array([0], dtype=numpy.float64)) - - def __init__(self, fcn, xall=None, functkw={}, parinfo=None, - ftol=1.e-10, xtol=1.e-10, gtol=1.e-10, - damp=0., maxiter=200, factor=100., nprint=1, - iterfunct='default', iterkw={}, nocovar=0, - rescale=0, autoderivative=1, quiet=0, - diag=None, epsfcn=None, debug=0): + (blas_enorm32,) = scipy.linalg.blas.get_blas_funcs( + ["nrm2"], numpy.array([0], dtype=numpy.float32) + ) + (blas_enorm64,) = scipy.linalg.blas.get_blas_funcs( + ["nrm2"], numpy.array([0], dtype=numpy.float64) + ) + + def __init__( + self, + fcn, + xall=None, + functkw={}, + parinfo=None, + ftol=1.0e-10, + xtol=1.0e-10, + gtol=1.0e-10, + damp=0.0, + maxiter=200, + factor=100.0, + nprint=1, + iterfunct="default", + iterkw={}, + nocovar=0, + rescale=0, + autoderivative=1, + quiet=0, + diag=None, + epsfcn=None, + debug=0, + ): """ - Inputs: - fcn: - The function to be minimized. The function should return the weighted - deviations between the model and the data, as described above. - - xall: - An array of starting values for each of the parameters of the model. - The number of parameters should be fewer than the number of measurements. - - This parameter is optional if the parinfo keyword is used (but see - parinfo). The parinfo keyword provides a mechanism to fix or constrain - individual parameters. - - Keywords: - - autoderivative: - If this is set, derivatives of the function will be computed - automatically via a finite differencing procedure. If not set, then - fcn must provide the (analytical) derivatives. - Default: set (=1) - NOTE: to supply your own analytical derivatives, - explicitly pass autoderivative=0 - - ftol: - A nonnegative input variable. Termination occurs when both the actual - and predicted relative reductions in the sum of squares are at most - ftol (and status is accordingly set to 1 or 3). Therefore, ftol - measures the relative error desired in the sum of squares. - Default: 1E-10 - - functkw: - A dictionary which contains the parameters to be passed to the - user-supplied function specified by fcn via the standard Python - keyword dictionary mechanism. This is the way you can pass additional - data to your user-supplied function without using global variables. - - Consider the following example: - if functkw = {'xval':[1.,2.,3.], 'yval':[1.,4.,9.], - 'errval':[1.,1.,1.] } - then the user supplied function should be declared like this: - def myfunct(p, fjac=None, xval=None, yval=None, errval=None): - - Default: {} No extra parameters are passed to the user-supplied - function. - - gtol: - A nonnegative input variable. Termination occurs when the cosine of - the angle between fvec and any column of the jacobian is at most gtol - in absolute value (and status is accordingly set to 4). Therefore, - gtol measures the orthogonality desired between the function vector - and the columns of the jacobian. - Default: 1e-10 - - iterkw: - The keyword arguments to be passed to iterfunct via the dictionary - keyword mechanism. This should be a dictionary and is similar in - operation to FUNCTKW. - Default: {} No arguments are passed. - - iterfunct: - The name of a function to be called upon each NPRINT iteration of the - MPFIT routine. It should be declared in the following way: - def iterfunct(myfunct, p, iter, fnorm, functkw=None, - parinfo=None, quiet=0, dof=None, [iterkw keywords here]) - # perform custom iteration update - - iterfunct must accept all three keyword parameters (FUNCTKW, PARINFO - and QUIET). - - myfunct: The user-supplied function to be minimized, - p: The current set of model parameters - iter: The iteration number - functkw: The arguments to be passed to myfunct. - fnorm: The chi-squared value. - quiet: Set when no textual output should be printed. - dof: The number of degrees of freedom, normally the number of points - less the number of free parameters. - See below for documentation of parinfo. - - In implementation, iterfunct can perform updates to the terminal or - graphical user interface, to provide feedback while the fit proceeds. - If the fit is to be stopped for any reason, then iterfunct should return a - a status value between -15 and -1. Otherwise it should return None - (e.g. no return statement) or 0. - In principle, iterfunct should probably not modify the parameter values, - because it may interfere with the algorithm's stability. In practice it - is allowed. - - Default: an internal routine is used to print the parameter values. - - Set iterfunct=None if there is no user-defined routine and you don't - want the internal default routine be called. - - maxiter: - The maximum number of iterations to perform. If the number is exceeded, - then the status value is set to 5 and MPFIT returns. - Default: 200 iterations - - nocovar: - Set this keyword to prevent the calculation of the covariance matrix - before returning (see COVAR) - Default: clear (=0) The covariance matrix is returned - - nprint: - The frequency with which iterfunct is called. A value of 1 indicates - that iterfunct is called with every iteration, while 2 indicates every - other iteration, etc. Note that several Levenberg-Marquardt attempts - can be made in a single iteration. - Default value: 1 - - parinfo - Provides a mechanism for more sophisticated constraints to be placed on - parameter values. When parinfo is not passed, then it is assumed that - all parameters are free and unconstrained. Values in parinfo are never - modified during a call to MPFIT. - - See description above for the structure of PARINFO. - - Default value: None All parameters are free and unconstrained. - - quiet: - Set this keyword when no textual output should be printed by MPFIT - - damp: - A scalar number, indicating the cut-off value of residuals where - "damping" will occur. Residuals with magnitudes greater than this - number will be replaced by their hyperbolic tangent. This partially - mitigates the so-called large residual problem inherent in - least-squares solvers (as for the test problem CURVI, - http://www.maxthis.com/curviex.htm). - A value of 0 indicates no damping. - Default: 0 - - Note: DAMP doesn't work with autoderivative=0 - - xtol: - A nonnegative input variable. Termination occurs when the relative error - between two consecutive iterates is at most xtol (and status is - accordingly set to 2 or 3). Therefore, xtol measures the relative error - desired in the approximate solution. - Default: 1E-10 - - Outputs: - - Returns an object of type mpfit. The results are attributes of this class, - e.g. mpfit.status, mpfit.errmsg, mpfit.params, npfit.niter, mpfit.covar. - - .status - An integer status code is returned. All values greater than zero can - represent success (however .status == 5 may indicate failure to - converge). It can have one of the following values: - - -16 - A parameter or function value has become infinite or an undefined - number. This is usually a consequence of numerical overflow in the - user's model function, which must be avoided. - - -15 to -1 - These are error codes that either MYFUNCT or iterfunct may return to - terminate the fitting process. Values from -15 to -1 are reserved - for the user functions and will not clash with MPFIT. - - 0 Improper input parameters. - - 1 Both actual and predicted relative reductions in the sum of squares - are at most ftol. - - 2 Relative error between two consecutive iterates is at most xtol - - 3 Conditions for status = 1 and status = 2 both hold. - - 4 The cosine of the angle between fvec and any column of the jacobian - is at most gtol in absolute value. - - 5 The maximum number of iterations has been reached. + Inputs: + fcn: + The function to be minimized. The function should return the weighted + deviations between the model and the data, as described above. + + xall: + An array of starting values for each of the parameters of the model. + The number of parameters should be fewer than the number of measurements. + + This parameter is optional if the parinfo keyword is used (but see + parinfo). The parinfo keyword provides a mechanism to fix or constrain + individual parameters. + + Keywords: + + autoderivative: + If this is set, derivatives of the function will be computed + automatically via a finite differencing procedure. If not set, then + fcn must provide the (analytical) derivatives. + Default: set (=1) + NOTE: to supply your own analytical derivatives, + explicitly pass autoderivative=0 + + ftol: + A nonnegative input variable. Termination occurs when both the actual + and predicted relative reductions in the sum of squares are at most + ftol (and status is accordingly set to 1 or 3). Therefore, ftol + measures the relative error desired in the sum of squares. + Default: 1E-10 + + functkw: + A dictionary which contains the parameters to be passed to the + user-supplied function specified by fcn via the standard Python + keyword dictionary mechanism. This is the way you can pass additional + data to your user-supplied function without using global variables. + + Consider the following example: + if functkw = {'xval':[1.,2.,3.], 'yval':[1.,4.,9.], + 'errval':[1.,1.,1.] } + then the user supplied function should be declared like this: + def myfunct(p, fjac=None, xval=None, yval=None, errval=None): + + Default: {} No extra parameters are passed to the user-supplied + function. + + gtol: + A nonnegative input variable. Termination occurs when the cosine of + the angle between fvec and any column of the jacobian is at most gtol + in absolute value (and status is accordingly set to 4). Therefore, + gtol measures the orthogonality desired between the function vector + and the columns of the jacobian. + Default: 1e-10 + + iterkw: + The keyword arguments to be passed to iterfunct via the dictionary + keyword mechanism. This should be a dictionary and is similar in + operation to FUNCTKW. + Default: {} No arguments are passed. + + iterfunct: + The name of a function to be called upon each NPRINT iteration of the + MPFIT routine. It should be declared in the following way: + def iterfunct(myfunct, p, iter, fnorm, functkw=None, + parinfo=None, quiet=0, dof=None, [iterkw keywords here]) + # perform custom iteration update + + iterfunct must accept all three keyword parameters (FUNCTKW, PARINFO + and QUIET). + + myfunct: The user-supplied function to be minimized, + p: The current set of model parameters + iter: The iteration number + functkw: The arguments to be passed to myfunct. + fnorm: The chi-squared value. + quiet: Set when no textual output should be printed. + dof: The number of degrees of freedom, normally the number of points + less the number of free parameters. + See below for documentation of parinfo. + + In implementation, iterfunct can perform updates to the terminal or + graphical user interface, to provide feedback while the fit proceeds. + If the fit is to be stopped for any reason, then iterfunct should return a + a status value between -15 and -1. Otherwise it should return None + (e.g. no return statement) or 0. + In principle, iterfunct should probably not modify the parameter values, + because it may interfere with the algorithm's stability. In practice it + is allowed. + + Default: an internal routine is used to print the parameter values. + + Set iterfunct=None if there is no user-defined routine and you don't + want the internal default routine be called. + + maxiter: + The maximum number of iterations to perform. If the number is exceeded, + then the status value is set to 5 and MPFIT returns. + Default: 200 iterations + + nocovar: + Set this keyword to prevent the calculation of the covariance matrix + before returning (see COVAR) + Default: clear (=0) The covariance matrix is returned + + nprint: + The frequency with which iterfunct is called. A value of 1 indicates + that iterfunct is called with every iteration, while 2 indicates every + other iteration, etc. Note that several Levenberg-Marquardt attempts + can be made in a single iteration. + Default value: 1 + + parinfo + Provides a mechanism for more sophisticated constraints to be placed on + parameter values. When parinfo is not passed, then it is assumed that + all parameters are free and unconstrained. Values in parinfo are never + modified during a call to MPFIT. + + See description above for the structure of PARINFO. + + Default value: None All parameters are free and unconstrained. + + quiet: + Set this keyword when no textual output should be printed by MPFIT + + damp: + A scalar number, indicating the cut-off value of residuals where + "damping" will occur. Residuals with magnitudes greater than this + number will be replaced by their hyperbolic tangent. This partially + mitigates the so-called large residual problem inherent in + least-squares solvers (as for the test problem CURVI, + http://www.maxthis.com/curviex.htm). + A value of 0 indicates no damping. + Default: 0 + + Note: DAMP doesn't work with autoderivative=0 + + xtol: + A nonnegative input variable. Termination occurs when the relative error + between two consecutive iterates is at most xtol (and status is + accordingly set to 2 or 3). Therefore, xtol measures the relative error + desired in the approximate solution. + Default: 1E-10 + + Outputs: + + Returns an object of type mpfit. The results are attributes of this class, + e.g. mpfit.status, mpfit.errmsg, mpfit.params, npfit.niter, mpfit.covar. + + .status + An integer status code is returned. All values greater than zero can + represent success (however .status == 5 may indicate failure to + converge). It can have one of the following values: + + -16 + A parameter or function value has become infinite or an undefined + number. This is usually a consequence of numerical overflow in the + user's model function, which must be avoided. + + -15 to -1 + These are error codes that either MYFUNCT or iterfunct may return to + terminate the fitting process. Values from -15 to -1 are reserved + for the user functions and will not clash with MPFIT. + + 0 Improper input parameters. + + 1 Both actual and predicted relative reductions in the sum of squares + are at most ftol. + + 2 Relative error between two consecutive iterates is at most xtol + + 3 Conditions for status = 1 and status = 2 both hold. + + 4 The cosine of the angle between fvec and any column of the jacobian + is at most gtol in absolute value. + + 5 The maximum number of iterations has been reached. - 6 ftol is too small. No further reduction in the sum of squares is - possible. + 6 ftol is too small. No further reduction in the sum of squares is + possible. - 7 xtol is too small. No further improvement in the approximate solution - x is possible. + 7 xtol is too small. No further improvement in the approximate solution + x is possible. - 8 gtol is too small. fvec is orthogonal to the columns of the jacobian - to machine precision. + 8 gtol is too small. fvec is orthogonal to the columns of the jacobian + to machine precision. - .fnorm - The value of the summed squared residuals for the returned parameter - values. + .fnorm + The value of the summed squared residuals for the returned parameter + values. - .covar - The covariance matrix for the set of parameters returned by MPFIT. - The matrix is NxN where N is the number of parameters. The square root - of the diagonal elements gives the formal 1-sigma statistical errors on - the parameters if errors were treated "properly" in fcn. - Parameter errors are also returned in .perror. + .covar + The covariance matrix for the set of parameters returned by MPFIT. + The matrix is NxN where N is the number of parameters. The square root + of the diagonal elements gives the formal 1-sigma statistical errors on + the parameters if errors were treated "properly" in fcn. + Parameter errors are also returned in .perror. - To compute the correlation matrix, pcor, use this example: - cov = mpfit.covar - pcor = cov * 0. - for i in range(n): - for j in range(n): - pcor[i,j] = cov[i,j]/sqrt(cov[i,i]*cov[j,j]) + To compute the correlation matrix, pcor, use this example: + cov = mpfit.covar + pcor = cov * 0. + for i in range(n): + for j in range(n): + pcor[i,j] = cov[i,j]/sqrt(cov[i,i]*cov[j,j]) - If nocovar is set or MPFIT terminated abnormally, then .covar is set to - a scalar with value None. + If nocovar is set or MPFIT terminated abnormally, then .covar is set to + a scalar with value None. - .errmsg - A string error or warning message is returned. + .errmsg + A string error or warning message is returned. - .nfev - The number of calls to MYFUNCT performed. + .nfev + The number of calls to MYFUNCT performed. - .niter - The number of iterations completed. + .niter + The number of iterations completed. - .perror - The formal 1-sigma errors in each parameter, computed from the - covariance matrix. If a parameter is held fixed, or if it touches a - boundary, then the error is reported as zero. + .perror + The formal 1-sigma errors in each parameter, computed from the + covariance matrix. If a parameter is held fixed, or if it touches a + boundary, then the error is reported as zero. - If the fit is unweighted (i.e. no errors were given, or the weights - were uniformly set to unity), then .perror will probably not represent - the true parameter uncertainties. + If the fit is unweighted (i.e. no errors were given, or the weights + were uniformly set to unity), then .perror will probably not represent + the true parameter uncertainties. - *If* you can assume that the true reduced chi-squared value is unity -- - meaning that the fit is implicitly assumed to be of good quality -- - then the estimated parameter uncertainties can be computed by scaling - .perror by the measured chi-squared value. + *If* you can assume that the true reduced chi-squared value is unity -- + meaning that the fit is implicitly assumed to be of good quality -- + then the estimated parameter uncertainties can be computed by scaling + .perror by the measured chi-squared value. - dof = len(x) - len(mpfit.params) # deg of freedom - # scaled uncertainties - pcerror = mpfit.perror * sqrt(mpfit.fnorm / dof) + dof = len(x) - len(mpfit.params) # deg of freedom + # scaled uncertainties + pcerror = mpfit.perror * sqrt(mpfit.fnorm / dof) """ self.niter = 0 @@ -845,7 +867,7 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, self.perror = None self.status = 0 # Invalid input flag set while we check inputs self.debug = debug - self.errmsg = '' + self.errmsg = "" self.nfev = 0 self.damp = damp self.dof = 0 @@ -854,37 +876,39 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, self.errmsg = "Usage: parms = mpfit('myfunt', ... )" return - if iterfunct == 'default': + if iterfunct == "default": iterfunct = self.defiter # Parameter damping doesn't work when user is providing their own # gradients. if (self.damp != 0) and (autoderivative == 0): - self.errmsg = 'ERROR: keywords DAMP and AUTODERIVATIVE are mutually exclusive' + self.errmsg = ( + "ERROR: keywords DAMP and AUTODERIVATIVE are mutually exclusive" + ) return # Parameters can either be stored in parinfo, or x. x takes precedence if it exists if (xall is None) and (parinfo is None): - self.errmsg = 'ERROR: must pass parameters in P or PARINFO' + self.errmsg = "ERROR: must pass parameters in P or PARINFO" return # Be sure that PARINFO is of the right type if parinfo is not None: if type(parinfo) is not list: - self.errmsg = 'ERROR: PARINFO must be a list of dictionaries.' + self.errmsg = "ERROR: PARINFO must be a list of dictionaries." return else: if type(parinfo[0]) is not dict: - self.errmsg = 'ERROR: PARINFO must be a list of dictionaries.' + self.errmsg = "ERROR: PARINFO must be a list of dictionaries." return - if ((xall is not None) and (len(xall) != len(parinfo))): - self.errmsg = 'ERROR: number of elements in PARINFO and P must agree' + if (xall is not None) and (len(xall) != len(parinfo)): + self.errmsg = "ERROR: number of elements in PARINFO and P must agree" return # If the parameters were not specified at the command line, then # extract them from PARINFO if xall is None: - xall = self.parinfo(parinfo, 'value') + xall = self.parinfo(parinfo, "value") if xall is None: self.errmsg = 'ERROR: either P or PARINFO(*)["value"] must be supplied.' return @@ -893,50 +917,52 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, xall = numpy.asarray(xall) # In the case if the xall is not float or if is float but has less # than 64 bits we do convert it into double - if xall.dtype.kind != 'f' or xall.dtype.itemsize <= 4: + if xall.dtype.kind != "f" or xall.dtype.itemsize <= 4: xall = xall.astype(numpy.float64) npar = len(xall) - self.fnorm = -1. - fnorm1 = -1. + self.fnorm = -1.0 + fnorm1 = -1.0 # TIED parameters? - ptied = self.parinfo(parinfo, 'tied', default='', n=npar) + ptied = self.parinfo(parinfo, "tied", default="", n=npar) self.qanytied = 0 for i in range(npar): ptied[i] = ptied[i].strip() - if ptied[i] != '': + if ptied[i] != "": self.qanytied = 1 self.ptied = ptied # FIXED parameters ? - pfixed = self.parinfo(parinfo, 'fixed', default=0, n=npar) - pfixed = (pfixed == 1) + pfixed = self.parinfo(parinfo, "fixed", default=0, n=npar) + pfixed = pfixed == 1 for i in range(npar): - pfixed[i] = pfixed[i] or (ptied[i] != '') # Tied parameters are also effectively fixed + pfixed[i] = pfixed[i] or ( + ptied[i] != "" + ) # Tied parameters are also effectively fixed # Finite differencing step, absolute and relative, and sidedness of deriv. - step = self.parinfo(parinfo, 'step', default=0., n=npar) - dstep = self.parinfo(parinfo, 'relstep', default=0., n=npar) - dside = self.parinfo(parinfo, 'mpside', default=0, n=npar) + step = self.parinfo(parinfo, "step", default=0.0, n=npar) + dstep = self.parinfo(parinfo, "relstep", default=0.0, n=npar) + dside = self.parinfo(parinfo, "mpside", default=0, n=npar) # Maximum and minimum steps allowed to be taken in one iteration - maxstep = self.parinfo(parinfo, 'mpmaxstep', default=0., n=npar) - minstep = self.parinfo(parinfo, 'mpminstep', default=0., n=npar) + maxstep = self.parinfo(parinfo, "mpmaxstep", default=0.0, n=npar) + minstep = self.parinfo(parinfo, "mpminstep", default=0.0, n=npar) qmin = minstep != 0 qmin[:] = False # Remove minstep for now!! qmax = maxstep != 0 if numpy.any(qmin & qmax & (maxstep < minstep)): - self.errmsg = 'ERROR: MPMINSTEP is greater than MPMAXSTEP' + self.errmsg = "ERROR: MPMINSTEP is greater than MPMAXSTEP" return - wh = (numpy.nonzero((qmin != 0.) | (qmax != 0.)))[0] + wh = (numpy.nonzero((qmin != 0.0) | (qmax != 0.0)))[0] qminmax = len(wh > 0) # Finish up the free parameters ifree = (numpy.nonzero(pfixed != 1))[0] nfree = len(ifree) if nfree == 0: - self.errmsg = 'ERROR: no free parameters' + self.errmsg = "ERROR: no free parameters" return # Compose only VARYING parameters @@ -944,18 +970,22 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, x = self.params[ifree] # x is the set of free parameters # LIMITED parameters ? - limited = self.parinfo(parinfo, 'limited', default=[0, 0], n=npar) - limits = self.parinfo(parinfo, 'limits', default=[0., 0.], n=npar) + limited = self.parinfo(parinfo, "limited", default=[0, 0], n=npar) + limits = self.parinfo(parinfo, "limits", default=[0.0, 0.0], n=npar) if (limited is not None) and (limits is not None): # Error checking on limits in parinfo - if numpy.any((limited[:, 0] & (xall < limits[:, 0])) | - (limited[:, 1] & (xall > limits[:, 1]))): - self.errmsg = 'ERROR: parameters are not within PARINFO limits' + if numpy.any( + (limited[:, 0] & (xall < limits[:, 0])) + | (limited[:, 1] & (xall > limits[:, 1])) + ): + self.errmsg = "ERROR: parameters are not within PARINFO limits" return - if numpy.any((limited[:, 0] & limited[:, 1]) & - (limits[:, 0] >= limits[:, 1]) & - (pfixed == 0)): - self.errmsg = 'ERROR: PARINFO parameter limits are not consistent' + if numpy.any( + (limited[:, 0] & limited[:, 1]) + & (limits[:, 0] >= limits[:, 1]) + & (pfixed == 0) + ): + self.errmsg = "ERROR: PARINFO parameter limits are not consistent" return # Transfer structure values to local variables @@ -964,32 +994,38 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, qllim = (limited[:, 0])[ifree] llim = (limits[:, 0])[ifree] - if numpy.any((qulim != 0.) | (qllim != 0.)): + if numpy.any((qulim != 0.0) | (qllim != 0.0)): qanylim = 1 else: qanylim = 0 else: # Fill in local variables with dummy values qulim = numpy.zeros(nfree) - ulim = x * 0. + ulim = x * 0.0 qllim = qulim - llim = x * 0. + llim = x * 0.0 qanylim = 0 n = len(x) # Check input parameters for errors - if (n < 0) or (ftol <= 0) or (xtol <= 0) or (gtol <= 0) \ - or (maxiter < 0) or (factor <= 0): - self.errmsg = 'ERROR: input keywords are inconsistent' + if ( + (n < 0) + or (ftol <= 0) + or (xtol <= 0) + or (gtol <= 0) + or (maxiter < 0) + or (factor <= 0) + ): + self.errmsg = "ERROR: input keywords are inconsistent" return if rescale != 0: - self.errmsg = 'ERROR: DIAG parameter scales are inconsistent' + self.errmsg = "ERROR: DIAG parameter scales are inconsistent" if len(diag) < n: return if numpy.any(diag <= 0): return - self.errmsg = '' + self.errmsg = "" [self.status, fvec] = self.call(fcn, self.params, functkw) @@ -1010,22 +1046,21 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, m = len(fvec) if m < n: - self.errmsg = 'ERROR: number of parameters must not exceed data' + self.errmsg = "ERROR: number of parameters must not exceed data" return self.dof = m - nfree self.fnorm = self.enorm(fvec) # Initialize Levelberg-Marquardt parameter and iteration counter - par = 0. + par = 0.0 self.niter = 1 - qtf = x * 0. + qtf = x * 0.0 self.status = 0 # Beginning of the outer loop - while (1): - + while 1: # If requested, call fcn to enable printing of iterates self.params[ifree] = x if self.qanytied: @@ -1037,15 +1072,25 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, xnew0 = self.params.copy() dof = numpy.max([len(fvec) - len(x), 0]) - status = iterfunct(fcn, self.params, self.niter, self.fnorm ** 2, - functkw=functkw, parinfo=parinfo, quiet=quiet, - dof=dof, **iterkw) + status = iterfunct( + fcn, + self.params, + self.niter, + self.fnorm**2, + functkw=functkw, + parinfo=parinfo, + quiet=quiet, + dof=dof, + **iterkw, + ) if status is not None: self.status = status # Check for user termination if self.status < 0: - self.errmsg = 'WARNING: premature termination by ' + str(iterfunct) + self.errmsg = "WARNING: premature termination by " + str( + iterfunct + ) return # If parameters were changed (grrr..) then re-tie @@ -1056,18 +1101,29 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, # Calculate the jacobian matrix self.status = 2 - catch_msg = 'calling MPFIT_FDJAC2' - fjac = self.fdjac2(fcn, x, fvec, step, qulim, ulim, dside, - epsfcn=epsfcn, - autoderivative=autoderivative, dstep=dstep, - functkw=functkw, ifree=ifree, xall=self.params) + catch_msg = "calling MPFIT_FDJAC2" + fjac = self.fdjac2( + fcn, + x, + fvec, + step, + qulim, + ulim, + dside, + epsfcn=epsfcn, + autoderivative=autoderivative, + dstep=dstep, + functkw=functkw, + ifree=ifree, + xall=self.params, + ) if fjac is None: - self.errmsg = 'WARNING: premature termination by FDJAC2' + self.errmsg = "WARNING: premature termination by FDJAC2" return # Determine if any of the parameters are pegged at the limits if qanylim: - catch_msg = 'zeroing derivatives of pegged parameters' + catch_msg = "zeroing derivatives of pegged parameters" whlpeg = (numpy.nonzero(qllim & (x == llim)))[0] nlpeg = len(whlpeg) whupeg = (numpy.nonzero(qulim & (x == ulim)))[0] @@ -1091,22 +1147,22 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, # On the first iteration if "diag" is unspecified, scale # according to the norms of the columns of the initial jacobian - catch_msg = 'rescaling diagonal elements' + catch_msg = "rescaling diagonal elements" if self.niter == 1: if (rescale == 0) or (len(diag) < n): diag = wa2.copy() - diag[diag == 0] = 1. + diag[diag == 0] = 1.0 # On the first iteration, calculate the norm of the scaled x # and initialize the step bound delta wa3 = diag * x xnorm = self.enorm(wa3) delta = factor * xnorm - if delta == 0.: + if delta == 0.0: delta = factor # Form (q transpose)*fvec and store the first n components in qtf - catch_msg = 'forming (q transpose)*fvec' + catch_msg = "forming (q transpose)*fvec" wa4 = fvec.copy() for j in range(n): lj = ipvt[j] @@ -1134,13 +1190,13 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, # if ct GT 0 then goto, FAIL_OVERFLOW # Compute the norm of the scaled gradient - catch_msg = 'computing the scaled gradient' - gnorm = 0. + catch_msg = "computing the scaled gradient" + gnorm = 0.0 if self.fnorm != 0: for j in range(n): l = ipvt[j] if wa2[l] != 0: - sum0 = sum(fjac[0:j + 1, j] * qtf[0:j + 1]) / self.fnorm + sum0 = sum(fjac[0 : j + 1, j] * qtf[0 : j + 1]) / self.fnorm gnorm = numpy.max([gnorm, numpy.abs(sum0 / wa2[l])]) # Test for convergence of the gradient norm @@ -1156,54 +1212,59 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, diag = numpy.choose(diag > wa2, (wa2, diag)) # Beginning of the inner loop - while (1): - + while 1: # Determine the levenberg-marquardt parameter - catch_msg = 'calculating LM parameter (MPFIT_)' - [fjac, par, wa1, wa2] = self.lmpar(fjac, ipvt, diag, qtf, - delta, wa1, wa2, par=par) + catch_msg = "calculating LM parameter (MPFIT_)" + [fjac, par, wa1, wa2] = self.lmpar( + fjac, ipvt, diag, qtf, delta, wa1, wa2, par=par + ) # Store the direction p and x+p. Calculate the norm of p wa1 = -wa1 if (qanylim == 0) and (qminmax == 0): # No parameter limits, so just move to new position WA2 - alpha = 1. + alpha = 1.0 wa2 = x + wa1 else: - # Respect the limits. If a step were to go out of bounds, then # we should take a step in the same direction but shorter distance. # The step should take us right to the limit in that case. - alpha = 1. + alpha = 1.0 if qanylim: # Do not allow any steps out of bounds - catch_msg = 'checking for a step out of bounds' + catch_msg = "checking for a step out of bounds" if nlpeg > 0: - wa1[whlpeg] = numpy.clip(wa1[whlpeg], 0., numpy.max(wa1)) + wa1[whlpeg] = numpy.clip(wa1[whlpeg], 0.0, numpy.max(wa1)) if nupeg > 0: - wa1[whupeg] = numpy.clip(wa1[whupeg], numpy.min(wa1), 0.) + wa1[whupeg] = numpy.clip(wa1[whupeg], numpy.min(wa1), 0.0) dwa1 = numpy.abs(wa1) > machep - whl = (numpy.nonzero(((dwa1 != 0.) & qllim) & ((x + wa1) < llim)))[0] + whl = ( + numpy.nonzero(((dwa1 != 0.0) & qllim) & ((x + wa1) < llim)) + )[0] if len(whl) > 0: - t = ((llim[whl] - x[whl]) / - wa1[whl]) + t = (llim[whl] - x[whl]) / wa1[whl] alpha = numpy.min([alpha, numpy.min(t)]) - whu = (numpy.nonzero(((dwa1 != 0.) & qulim) & ((x + wa1) > ulim)))[0] + whu = ( + numpy.nonzero(((dwa1 != 0.0) & qulim) & ((x + wa1) > ulim)) + )[0] if len(whu) > 0: - t = ((ulim[whu] - x[whu]) / - wa1[whu]) + t = (ulim[whu] - x[whu]) / wa1[whu] alpha = numpy.min([alpha, numpy.min(t)]) # Obey any max step values. if qminmax: nwa1 = wa1 * alpha - whmax = (numpy.nonzero((qmax[ifree] != 0.) & (maxstep[ifree] > 0)))[0] + whmax = ( + numpy.nonzero((qmax[ifree] != 0.0) & (maxstep[ifree] > 0)) + )[0] if len(whmax) > 0: - mrat = numpy.max(numpy.abs(nwa1[whmax]) / - numpy.abs(maxstep[ifree[whmax]])) + mrat = numpy.max( + numpy.abs(nwa1[whmax]) + / numpy.abs(maxstep[ifree[whmax]]) + ) if mrat > 1: alpha = alpha / mrat @@ -1213,8 +1274,8 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, # Adjust the final output values. If the step put us exactly # on a boundary, make sure it is exact. - sgnu = (ulim >= 0) * 2. - 1. - sgnl = (llim >= 0) * 2. - 1. + sgnu = (ulim >= 0) * 2.0 - 1.0 + sgnl = (llim >= 0) * 2.0 - 1.0 # Handles case of # ... nonzero *LIM ... ...zero * LIM ulim1 = ulim * (1 - sgnu * machep) - (ulim == 0) * machep @@ -1222,7 +1283,7 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, wh = (numpy.nonzero((qulim != 0) & (wa2 >= ulim1)))[0] if len(wh) > 0: wa2[wh] = ulim[wh] - wh = (numpy.nonzero((qllim != 0.) & (wa2 <= llim1)))[0] + wh = (numpy.nonzero((qllim != 0.0) & (wa2 <= llim1)))[0] if len(wh) > 0: wa2[wh] = llim[wh] # endelse @@ -1237,7 +1298,7 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, # Evaluate the function at x+p and calculate its norm mperr = 0 - catch_msg = 'calling ' + str(fcn) + catch_msg = "calling " + str(fcn) [self.status, wa4] = self.call(fcn, self.params, functkw) if self.status < 0: self.errmsg = 'WARNING: premature termination by "' + fcn + '"' @@ -1245,16 +1306,16 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, fnorm1 = self.enorm(wa4) # Compute the scaled actual reduction - catch_msg = 'computing convergence criteria' - actred = -1. + catch_msg = "computing convergence criteria" + actred = -1.0 if (0.1 * fnorm1) < self.fnorm: - actred = - (fnorm1 / self.fnorm) ** 2 + 1. + actred = -((fnorm1 / self.fnorm) ** 2) + 1.0 # Compute the scaled predicted reduction and the scaled directional # derivative for j in range(n): wa3[j] = 0 - wa3[0:j + 1] = wa3[0:j + 1] + fjac[0:j + 1, j] * wa1[ipvt[j]] + wa3[0 : j + 1] = wa3[0 : j + 1] + fjac[0 : j + 1, j] * wa1[ipvt[j]] # Remember, alpha is the fraction of the full LM step actually # taken @@ -1264,24 +1325,24 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, dirder = -(temp1 * temp1 + temp2 * temp2) # Compute the ratio of the actual to the predicted reduction. - ratio = 0. + ratio = 0.0 if prered != 0: ratio = actred / prered # Update the step bound if ratio <= 0.25: if actred >= 0: - temp = .5 + temp = 0.5 else: - temp = .5 * dirder / (dirder + .5 * actred) + temp = 0.5 * dirder / (dirder + 0.5 * actred) if ((0.1 * fnorm1) >= self.fnorm) or (temp < 0.1): temp = 0.1 delta = temp * numpy.min([delta, pnorm / 0.1]) par = par / temp else: if (par == 0) or (ratio >= 0.75): - delta = pnorm / .5 - par = .5 * par + delta = pnorm / 0.5 + par = 0.5 * par # Test for successful iteration if ratio >= 0.0001: @@ -1294,13 +1355,20 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, self.niter = self.niter + 1 # Tests for convergence - if (numpy.abs(actred) <= ftol) and (prered <= ftol) \ - and (0.5 * ratio <= 1): + if ( + (numpy.abs(actred) <= ftol) + and (prered <= ftol) + and (0.5 * ratio <= 1) + ): self.status = 1 if delta <= xtol * xnorm: self.status = 2 - if (numpy.abs(actred) <= ftol) and (prered <= ftol) \ - and (0.5 * ratio <= 1) and (self.status == 2): + if ( + (numpy.abs(actred) <= ftol) + and (prered <= ftol) + and (0.5 * ratio <= 1) + and (self.status == 2) + ): self.status = 3 if self.status != 0: break @@ -1308,8 +1376,11 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, # Tests for termination and stringent tolerances if self.niter >= maxiter: self.status = 5 - if (numpy.abs(actred) <= machep) and (prered <= machep) \ - and (0.5 * ratio <= 1): + if ( + (numpy.abs(actred) <= machep) + and (prered <= machep) + and (0.5 * ratio <= 1) + ): self.status = 6 if delta <= machep * xnorm: self.status = 7 @@ -1323,10 +1394,11 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, break # Check for over/underflow - if ~numpy.all(numpy.isfinite(wa1) & numpy.isfinite(wa2) & \ - numpy.isfinite(x)) or ~numpy.isfinite(ratio): - errmsg = ('''ERROR: parameter or function value(s) have become - 'infinite; check model function for over- 'and underflow''') + if ~numpy.all( + numpy.isfinite(wa1) & numpy.isfinite(wa2) & numpy.isfinite(x) + ) or ~numpy.isfinite(ratio): + errmsg = """ERROR: parameter or function value(s) have become + 'infinite; check model function for over- 'and underflow""" self.status = -16 break # wh = where(finite(wa1) EQ 0 OR finite(wa2) EQ 0 OR finite(x) EQ 0, ct) @@ -1336,7 +1408,7 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, break # End of outer loop. - catch_msg = 'in the termination phase' + catch_msg = "in the termination phase" # Termination, either normal or user imposed. if len(self.params) == 0: return @@ -1345,25 +1417,28 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, else: self.params[ifree] = x if (nprint > 0) and (self.status > 0): - catch_msg = 'calling ' + str(fcn) + catch_msg = "calling " + str(fcn) [status, fvec] = self.call(fcn, self.params, functkw) - catch_msg = 'in the termination phase' + catch_msg = "in the termination phase" self.fnorm = self.enorm(fvec) if (self.fnorm is not None) and (fnorm1 is not None): self.fnorm = numpy.max([self.fnorm, fnorm1]) - self.fnorm = self.fnorm ** 2. + self.fnorm = self.fnorm**2.0 self.covar = None self.perror = None # (very carefully) set the covariance matrix COVAR - if (self.status > 0) and (nocovar == 0) and (n is not None) \ - and (fjac is not None) and (ipvt is not None): + if ( + (self.status > 0) + and (nocovar == 0) + and (n is not None) + and (fjac is not None) + and (ipvt is not None) + ): sz = fjac.shape - if (n > 0) and (sz[0] >= n) and (sz[1] >= n) \ - and (len(ipvt) >= n): - - catch_msg = 'computing the covariance matrix' + if (n > 0) and (sz[0] >= n) and (sz[1] >= n) and (len(ipvt) >= n): + catch_msg = "computing the covariance matrix" cv = self.calc_covar(fjac[0:n, 0:n], ipvt[0:n]) cv.shape = [n, n] nn = len(xall) @@ -1375,7 +1450,7 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, self.covar[ifree, ifree[i]] = cv[:, i] # Compute errors in parameters - catch_msg = 'computing parameter errors' + catch_msg = "computing parameter errors" self.perror = numpy.zeros(nn, dtype=float) d = numpy.diagonal(self.covar) wh = (numpy.nonzero(d >= 0))[0] @@ -1384,27 +1459,39 @@ def iterfunct(myfunct, p, iter, fnorm, functkw=None, return def __str__(self): - return {'params': self.params, - 'niter': self.niter, - 'params': self.params, - 'covar': self.covar, - 'perror': self.perror, - 'status': self.status, - 'debug': self.debug, - 'errmsg': self.errmsg, - 'nfev': self.nfev, - 'damp': self.damp - # ,'machar':self.machar - }.__str__() + return { + "params": self.params, + "niter": self.niter, + "params": self.params, + "covar": self.covar, + "perror": self.perror, + "status": self.status, + "debug": self.debug, + "errmsg": self.errmsg, + "nfev": self.nfev, + "damp": self.damp, + # ,'machar':self.machar + }.__str__() # Default procedure to be called every iteration. It simply prints # the parameter values. - def defiter(self, fcn, x, iter, fnorm=None, functkw=None, - quiet=0, iterstop=None, parinfo=None, - format=None, pformat='%.10g', dof=1): + def defiter( + self, + fcn, + x, + iter, + fnorm=None, + functkw=None, + quiet=0, + iterstop=None, + parinfo=None, + format=None, + pformat="%.10g", + dof=1, + ): if self.debug: - print('Entering defiter...') + print("Entering defiter...") if quiet: return if fnorm is None: @@ -1413,40 +1500,47 @@ def defiter(self, fcn, x, iter, fnorm=None, functkw=None, # Determine which parameters to print nprint = len(x) - print("Iter ", ('%6i' % iter), " CHI-SQUARE = ", ('%.10g' % fnorm), " DOF = ", ('%i' % dof)) + print( + "Iter ", + ("%6i" % iter), + " CHI-SQUARE = ", + ("%.10g" % fnorm), + " DOF = ", + ("%i" % dof), + ) for i in range(nprint): - if (parinfo is not None) and ('parname' in parinfo[i]): - p = ' ' + parinfo[i]['parname'] + ' = ' + if (parinfo is not None) and ("parname" in parinfo[i]): + p = " " + parinfo[i]["parname"] + " = " else: - p = ' P' + str(i) + ' = ' - if (parinfo is not None) and ('mpprint' in parinfo[i]): - iprint = parinfo[i]['mpprint'] + p = " P" + str(i) + " = " + if (parinfo is not None) and ("mpprint" in parinfo[i]): + iprint = parinfo[i]["mpprint"] else: iprint = 1 if iprint: - print(p + (pformat % x[i]) + ' ') + print(p + (pformat % x[i]) + " ") return 0 # DO_ITERSTOP: # if keyword_set(iterstop) then begin - # k = get_kbrd(0) - # if k EQ string(byte(7)) then begin - # message, 'WARNING: minimization not complete', /info - # print, 'Do you want to terminate this procedure? (y/n)', $ - # format='(A,$)' - # k = '' - # read, k - # if strupcase(strmid(k,0,1)) EQ 'Y' then begin - # message, 'WARNING: Procedure is terminating.', /info - # mperr = -1 - # endif - # endif + # k = get_kbrd(0) + # if k EQ string(byte(7)) then begin + # message, 'WARNING: minimization not complete', /info + # print, 'Do you want to terminate this procedure? (y/n)', $ + # format='(A,$)' + # k = '' + # read, k + # if strupcase(strmid(k,0,1)) EQ 'Y' then begin + # message, 'WARNING: Procedure is terminating.', /info + # mperr = -1 + # endif + # endif # endif # Procedure to parse the parameter values in PARINFO, which is a list of dictionaries - def parinfo(self, parinfo=None, key='a', default=None, n=0): + def parinfo(self, parinfo=None, key="a", default=None, n=0): if self.debug: - print('Entering parinfo...') + print("Entering parinfo...") if (n == 0) and (parinfo is not None): n = len(parinfo) if n == 0: @@ -1471,7 +1565,7 @@ def parinfo(self, parinfo=None, key='a', default=None, n=0): # derivatives or not. def call(self, fcn, x, functkw, fjac=None): if self.debug: - print('Entering call...') + print("Entering call...") if self.qanytied: x = self.tie(x, self.ptied) self.nfev = self.nfev + 1 @@ -1490,12 +1584,25 @@ def enorm(self, vec): ans = self.blas_enorm(vec) return ans - def fdjac2(self, fcn, x, fvec, step=None, ulimited=None, ulimit=None, dside=None, - epsfcn=None, autoderivative=1, - functkw=None, xall=None, ifree=None, dstep=None): + def fdjac2( + self, + fcn, + x, + fvec, + step=None, + ulimited=None, + ulimit=None, + dside=None, + epsfcn=None, + autoderivative=1, + functkw=None, + xall=None, + ifree=None, + dstep=None, + ): if self.debug: - print('Entering fdjac2...') + print("Entering fdjac2...") machep = self.machar.machep if epsfcn is None: epsfcn = machep @@ -1504,7 +1611,7 @@ def fdjac2(self, fcn, x, fvec, step=None, ulimited=None, ulimit=None, dside=None if ifree is None: ifree = numpy.arange(len(xall)) if step is None: - step = x * 0. + step = x * 0.0 nall = len(xall) eps = numpy.sqrt(numpy.max([epsfcn, machep])) @@ -1519,7 +1626,7 @@ def fdjac2(self, fcn, x, fvec, step=None, ulimited=None, ulimit=None, dside=None [status, fp] = self.call(fcn, xall, functkw, fjac=fjac) if len(fjac) != m * nall: - print('ERROR: Derivative matrix was not computed properly.') + print("ERROR: Derivative matrix was not computed properly.") return None # This definition is consistent with CURVEFIT @@ -1562,10 +1669,10 @@ def fdjac2(self, fcn, x, fvec, step=None, ulimited=None, ulimit=None, dside=None # varying ones) mask = dside[ifree] == -1 if len(ulimited) > 0 and len(ulimit) > 0: - mask = (mask | ((ulimited != 0) & (x > ulimit - h))) + mask = mask | ((ulimited != 0) & (x > ulimit - h)) wh = (numpy.nonzero(mask))[0] if len(wh) > 0: - h[wh] = - h[wh] + h[wh] = -h[wh] # Loop through parameters, computing the derivative for each for j in range(n): xp = xall.copy() @@ -1592,82 +1699,82 @@ def fdjac2(self, fcn, x, fvec, step=None, ulimited=None, ulimit=None, dside=None fjac[0:, j] = (fp - fm) / (2 * h[j]) return fjac - # Original FORTRAN documentation - # ********** + # Original FORTRAN documentation + # ********** # - # subroutine qrfac + # subroutine qrfac # - # this subroutine uses householder transformations with column - # pivoting (optional) to compute a qr factorization of the - # m by n matrix a. that is, qrfac determines an orthogonal - # matrix q, a permutation matrix p, and an upper trapezoidal - # matrix r with diagonal elements of nonincreasing magnitude, - # such that a*p = q*r. the householder transformation for - # column k, k = 1,2,...,min(m,n), is of the form + # this subroutine uses householder transformations with column + # pivoting (optional) to compute a qr factorization of the + # m by n matrix a. that is, qrfac determines an orthogonal + # matrix q, a permutation matrix p, and an upper trapezoidal + # matrix r with diagonal elements of nonincreasing magnitude, + # such that a*p = q*r. the householder transformation for + # column k, k = 1,2,...,min(m,n), is of the form # - # t - # i - (1/u(k))*u*u + # t + # i - (1/u(k))*u*u # - # where u has zeros in the first k-1 positions. the form of - # this transformation and the method of pivoting first - # appeared in the corresponding linpack subroutine. + # where u has zeros in the first k-1 positions. the form of + # this transformation and the method of pivoting first + # appeared in the corresponding linpack subroutine. # - # the subroutine statement is + # the subroutine statement is # - # subroutine qrfac(m,n,a,lda,pivot,ipvt,lipvt,rdiag,acnorm,wa) + # subroutine qrfac(m,n,a,lda,pivot,ipvt,lipvt,rdiag,acnorm,wa) # - # where + # where # - # m is a positive integer input variable set to the number - # of rows of a. + # m is a positive integer input variable set to the number + # of rows of a. # - # n is a positive integer input variable set to the number - # of columns of a. + # n is a positive integer input variable set to the number + # of columns of a. # - # a is an m by n array. on input a contains the matrix for - # which the qr factorization is to be computed. on output - # the strict upper trapezoidal part of a contains the strict - # upper trapezoidal part of r, and the lower trapezoidal - # part of a contains a factored form of q (the non-trivial - # elements of the u vectors described above). + # a is an m by n array. on input a contains the matrix for + # which the qr factorization is to be computed. on output + # the strict upper trapezoidal part of a contains the strict + # upper trapezoidal part of r, and the lower trapezoidal + # part of a contains a factored form of q (the non-trivial + # elements of the u vectors described above). # - # lda is a positive integer input variable not less than m - # which specifies the leading dimension of the array a. + # lda is a positive integer input variable not less than m + # which specifies the leading dimension of the array a. # - # pivot is a logical input variable. if pivot is set true, - # then column pivoting is enforced. if pivot is set false, - # then no column pivoting is done. + # pivot is a logical input variable. if pivot is set true, + # then column pivoting is enforced. if pivot is set false, + # then no column pivoting is done. # - # ipvt is an integer output array of length lipvt. ipvt - # defines the permutation matrix p such that a*p = q*r. - # column j of p is column ipvt(j) of the identity matrix. - # if pivot is false, ipvt is not referenced. + # ipvt is an integer output array of length lipvt. ipvt + # defines the permutation matrix p such that a*p = q*r. + # column j of p is column ipvt(j) of the identity matrix. + # if pivot is false, ipvt is not referenced. # - # lipvt is a positive integer input variable. if pivot is false, - # then lipvt may be as small as 1. if pivot is true, then - # lipvt must be at least n. + # lipvt is a positive integer input variable. if pivot is false, + # then lipvt may be as small as 1. if pivot is true, then + # lipvt must be at least n. # - # rdiag is an output array of length n which contains the - # diagonal elements of r. + # rdiag is an output array of length n which contains the + # diagonal elements of r. # - # acnorm is an output array of length n which contains the - # norms of the corresponding columns of the input matrix a. - # if this information is not needed, then acnorm can coincide - # with rdiag. + # acnorm is an output array of length n which contains the + # norms of the corresponding columns of the input matrix a. + # if this information is not needed, then acnorm can coincide + # with rdiag. # - # wa is a work array of length n. if pivot is false, then wa - # can coincide with rdiag. + # wa is a work array of length n. if pivot is false, then wa + # can coincide with rdiag. # - # subprograms called + # subprograms called # - # minpack-supplied ... dpmpar,enorm + # minpack-supplied ... dpmpar,enorm # - # fortran-supplied ... dmax1,dsqrt,min0 + # fortran-supplied ... dmax1,dsqrt,min0 # - # argonne national laboratory. minpack project. march 1980. - # burton s. garbow, kenneth e. hillstrom, jorge j. more + # argonne national laboratory. minpack project. march 1980. + # burton s. garbow, kenneth e. hillstrom, jorge j. more # - # ********** + # ********** # # PIVOTING / PERMUTING: # @@ -1688,21 +1795,21 @@ def fdjac2(self, fcn, x, fvec, step=None, ulimited=None, ulimit=None, dside=None # output. # # EXAMPLE: decompose the matrix [[9.,2.,6.],[4.,8.,7.]] - # aa = [[9.,2.,6.],[4.,8.,7.]] - # mpfit_qrfac, aa, aapvt, rdiag, aanorm - # IDL> print, aa - # 1.81818* 0.181818* 0.545455* - # -8.54545+ 1.90160* 0.432573* - # IDL> print, rdiag - # -11.0000+ -7.48166+ + # aa = [[9.,2.,6.],[4.,8.,7.]] + # mpfit_qrfac, aa, aapvt, rdiag, aanorm + # IDL> print, aa + # 1.81818* 0.181818* 0.545455* + # -8.54545+ 1.90160* 0.432573* + # IDL> print, rdiag + # -11.0000+ -7.48166+ # # The components marked with a * are the components of the # reflectors, and those marked with a + are components of R. # # To reconstruct Q and R we proceed as follows. First R. - # r = fltarr(m, n) - # for i = 0, n-1 do r(0:i,i) = aa(0:i,i) # fill in lower diag - # r(lindgen(n)*(m+1)) = rdiag + # r = fltarr(m, n) + # for i = 0, n-1 do r(0:i,i) = aa(0:i,i) # fill in lower diag + # r(lindgen(n)*(m+1)) = rdiag # # Next, Q, which are composed from the reflectors. Each reflector v # is taken from the upper trapezoid of aa, and converted to a matrix @@ -1710,22 +1817,23 @@ def fdjac2(self, fcn, x, fvec, step=None, ulimited=None, ulimit=None, dside=None # # hh = ident # identity matrix # for i = 0, n-1 do begin - # v = aa(*,i) & if i GT 0 then v(0:i-1) = 0 # extract reflector - # hh = hh # (ident - 2*(v # v)/total(v * v)) # generate matrix + # v = aa(*,i) & if i GT 0 then v(0:i-1) = 0 # extract reflector + # hh = hh # (ident - 2*(v # v)/total(v * v)) # generate matrix # endfor # # Test the result: - # IDL> print, hh # transpose(r) - # 9.00000 4.00000 - # 2.00000 8.00000 - # 6.00000 7.00000 + # IDL> print, hh # transpose(r) + # 9.00000 4.00000 + # 2.00000 8.00000 + # 6.00000 7.00000 # # Note that it is usually never necessary to form the Q matrix # explicitly, and MPFIT does not. def qrfac(self, a, pivot=0): - if self.debug: print('Entering qrfac...') + if self.debug: + print("Entering qrfac...") machep = self.machar.machep sz = a.shape m = sz[0] @@ -1793,95 +1901,97 @@ def qrfac(self, a, pivot=0): a[j:, lk] = ajk - ajj * sum(ajk * ajj) / a[j, lj] if (pivot != 0) and (rdiag[k] != 0): temp = a[j, lk] / rdiag[k] - rdiag[k] = rdiag[k] * numpy.sqrt(numpy.max([(1. - temp ** 2), 0.])) + rdiag[k] = rdiag[k] * numpy.sqrt( + numpy.max([(1.0 - temp**2), 0.0]) + ) temp = rdiag[k] / wa[k] if (0.05 * temp * temp) <= machep: - rdiag[k] = self.enorm(a[j + 1:, lk]) + rdiag[k] = self.enorm(a[j + 1 :, lk]) wa[k] = rdiag[k] rdiag[j] = -ajnorm return [a, ipvt, rdiag, acnorm] - # Original FORTRAN documentation - # ********** + # Original FORTRAN documentation + # ********** # - # subroutine qrsolv + # subroutine qrsolv # - # given an m by n matrix a, an n by n diagonal matrix d, - # and an m-vector b, the problem is to determine an x which - # solves the system + # given an m by n matrix a, an n by n diagonal matrix d, + # and an m-vector b, the problem is to determine an x which + # solves the system # - # a*x = b , d*x = 0 , + # a*x = b , d*x = 0 , # - # in the least squares sense. + # in the least squares sense. # - # this subroutine completes the solution of the problem - # if it is provided with the necessary information from the - # factorization, with column pivoting, of a. that is, if - # a*p = q*r, where p is a permutation matrix, q has orthogonal - # columns, and r is an upper triangular matrix with diagonal - # elements of nonincreasing magnitude, then qrsolv expects - # the full upper triangle of r, the permutation matrix p, - # and the first n components of (q transpose)*b. the system - # a*x = b, d*x = 0, is then equivalent to + # this subroutine completes the solution of the problem + # if it is provided with the necessary information from the + # factorization, with column pivoting, of a. that is, if + # a*p = q*r, where p is a permutation matrix, q has orthogonal + # columns, and r is an upper triangular matrix with diagonal + # elements of nonincreasing magnitude, then qrsolv expects + # the full upper triangle of r, the permutation matrix p, + # and the first n components of (q transpose)*b. the system + # a*x = b, d*x = 0, is then equivalent to # - # t t - # r*z = q *b , p *d*p*z = 0 , + # t t + # r*z = q *b , p *d*p*z = 0 , # - # where x = p*z. if this system does not have full rank, - # then a least squares solution is obtained. on output qrsolv - # also provides an upper triangular matrix s such that + # where x = p*z. if this system does not have full rank, + # then a least squares solution is obtained. on output qrsolv + # also provides an upper triangular matrix s such that # - # t t t - # p *(a *a + d*d)*p = s *s . + # t t t + # p *(a *a + d*d)*p = s *s . # - # s is computed within qrsolv and may be of separate interest. + # s is computed within qrsolv and may be of separate interest. # - # the subroutine statement is + # the subroutine statement is # - # subroutine qrsolv(n,r,ldr,ipvt,diag,qtb,x,sdiag,wa) + # subroutine qrsolv(n,r,ldr,ipvt,diag,qtb,x,sdiag,wa) # - # where + # where # - # n is a positive integer input variable set to the order of r. + # n is a positive integer input variable set to the order of r. # - # r is an n by n array. on input the full upper triangle - # must contain the full upper triangle of the matrix r. - # on output the full upper triangle is unaltered, and the - # strict lower triangle contains the strict upper triangle - # (transposed) of the upper triangular matrix s. + # r is an n by n array. on input the full upper triangle + # must contain the full upper triangle of the matrix r. + # on output the full upper triangle is unaltered, and the + # strict lower triangle contains the strict upper triangle + # (transposed) of the upper triangular matrix s. # - # ldr is a positive integer input variable not less than n - # which specifies the leading dimension of the array r. + # ldr is a positive integer input variable not less than n + # which specifies the leading dimension of the array r. # - # ipvt is an integer input array of length n which defines the - # permutation matrix p such that a*p = q*r. column j of p - # is column ipvt(j) of the identity matrix. + # ipvt is an integer input array of length n which defines the + # permutation matrix p such that a*p = q*r. column j of p + # is column ipvt(j) of the identity matrix. # - # diag is an input array of length n which must contain the - # diagonal elements of the matrix d. + # diag is an input array of length n which must contain the + # diagonal elements of the matrix d. # - # qtb is an input array of length n which must contain the first - # n elements of the vector (q transpose)*b. + # qtb is an input array of length n which must contain the first + # n elements of the vector (q transpose)*b. # - # x is an output array of length n which contains the least - # squares solution of the system a*x = b, d*x = 0. + # x is an output array of length n which contains the least + # squares solution of the system a*x = b, d*x = 0. # - # sdiag is an output array of length n which contains the - # diagonal elements of the upper triangular matrix s. + # sdiag is an output array of length n which contains the + # diagonal elements of the upper triangular matrix s. # - # wa is a work array of length n. + # wa is a work array of length n. # - # subprograms called + # subprograms called # - # fortran-supplied ... dabs,dsqrt + # fortran-supplied ... dabs,dsqrt # - # argonne national laboratory. minpack project. march 1980. - # burton s. garbow, kenneth e. hillstrom, jorge j. more + # argonne national laboratory. minpack project. march 1980. + # burton s. garbow, kenneth e. hillstrom, jorge j. more # def qrsolv(self, r, ipvt, diag, qtb, sdiag): if self.debug: - print('Entering qrsolv...') + print("Entering qrsolv...") sz = r.shape m = sz[0] n = sz[1] @@ -1906,17 +2016,17 @@ def qrsolv(self, r, ipvt, diag, qtb, sdiag): # single element of (q transpose)*b beyond the first n, which # is initially zero. - qtbpj = 0. + qtbpj = 0.0 for k in range(j, n): if sdiag[k] == 0: break if numpy.abs(r[k, k]) < numpy.abs(sdiag[k]): cotan = r[k, k] / sdiag[k] - sine = 0.5 / numpy.sqrt(.25 + .25 * cotan * cotan) + sine = 0.5 / numpy.sqrt(0.25 + 0.25 * cotan * cotan) cosine = sine * cotan else: tang = sdiag[k] / r[k, k] - cosine = 0.5 / numpy.sqrt(.25 + .25 * tang * tang) + cosine = 0.5 / numpy.sqrt(0.25 + 0.25 * tang * tang) sine = cosine * tang # Compute the modified diagonal element of r and the @@ -1928,9 +2038,11 @@ def qrsolv(self, r, ipvt, diag, qtb, sdiag): # Accumulate the transformation in the row of s if n > k + 1: - temp = cosine * r[k + 1:n, k] + sine * sdiag[k + 1:n] - sdiag[k + 1:n] = -sine * r[k + 1:n, k] + cosine * sdiag[k + 1:n] - r[k + 1:n, k] = temp + temp = cosine * r[k + 1 : n, k] + sine * sdiag[k + 1 : n] + sdiag[k + 1 : n] = ( + -sine * r[k + 1 : n, k] + cosine * sdiag[k + 1 : n] + ) + r[k + 1 : n, k] = temp sdiag[j] = r[j, j] r[j, j] = x[j] @@ -1946,111 +2058,111 @@ def qrsolv(self, r, ipvt, diag, qtb, sdiag): wa[nsing - 1] = wa[nsing - 1] / sdiag[nsing - 1] # Degenerate case # *** Reverse loop *** for j in range(nsing - 2, -1, -1): - sum0 = sum(r[j + 1:nsing, j] * wa[j + 1:nsing]) + sum0 = sum(r[j + 1 : nsing, j] * wa[j + 1 : nsing]) wa[j] = (wa[j] - sum0) / sdiag[j] # Permute the components of z back to components of x x[ipvt] = wa return (r, x, sdiag) - # Original FORTRAN documentation + # Original FORTRAN documentation # - # subroutine lmpar + # subroutine lmpar # - # given an m by n matrix a, an n by n nonsingular diagonal - # matrix d, an m-vector b, and a positive number delta, - # the problem is to determine a value for the parameter - # par such that if x solves the system + # given an m by n matrix a, an n by n nonsingular diagonal + # matrix d, an m-vector b, and a positive number delta, + # the problem is to determine a value for the parameter + # par such that if x solves the system # - # a*x = b , sqrt(par)*d*x = 0 , + # a*x = b , sqrt(par)*d*x = 0 , # - # in the least squares sense, and dxnorm is the euclidean - # norm of d*x, then either par is zero and + # in the least squares sense, and dxnorm is the euclidean + # norm of d*x, then either par is zero and # - # (dxnorm-delta) .le. 0.1*delta , + # (dxnorm-delta) .le. 0.1*delta , # - # or par is positive and + # or par is positive and # - # abs(dxnorm-delta) .le. 0.1*delta . + # abs(dxnorm-delta) .le. 0.1*delta . # - # this subroutine completes the solution of the problem - # if it is provided with the necessary information from the - # qr factorization, with column pivoting, of a. that is, if - # a*p = q*r, where p is a permutation matrix, q has orthogonal - # columns, and r is an upper triangular matrix with diagonal - # elements of nonincreasing magnitude, then lmpar expects - # the full upper triangle of r, the permutation matrix p, - # and the first n components of (q transpose)*b. on output - # lmpar also provides an upper triangular matrix s such that + # this subroutine completes the solution of the problem + # if it is provided with the necessary information from the + # qr factorization, with column pivoting, of a. that is, if + # a*p = q*r, where p is a permutation matrix, q has orthogonal + # columns, and r is an upper triangular matrix with diagonal + # elements of nonincreasing magnitude, then lmpar expects + # the full upper triangle of r, the permutation matrix p, + # and the first n components of (q transpose)*b. on output + # lmpar also provides an upper triangular matrix s such that # - # t t t - # p *(a *a + par*d*d)*p = s *s . + # t t t + # p *(a *a + par*d*d)*p = s *s . # - # s is employed within lmpar and may be of separate interest. + # s is employed within lmpar and may be of separate interest. # - # only a few iterations are generally needed for convergence - # of the algorithm. if, however, the limit of 10 iterations - # is reached, then the output par will contain the best - # value obtained so far. + # only a few iterations are generally needed for convergence + # of the algorithm. if, however, the limit of 10 iterations + # is reached, then the output par will contain the best + # value obtained so far. # - # the subroutine statement is + # the subroutine statement is # - # subroutine lmpar(n,r,ldr,ipvt,diag,qtb,delta,par,x,sdiag, - # wa1,wa2) + # subroutine lmpar(n,r,ldr,ipvt,diag,qtb,delta,par,x,sdiag, + # wa1,wa2) # - # where + # where # - # n is a positive integer input variable set to the order of r. + # n is a positive integer input variable set to the order of r. # - # r is an n by n array. on input the full upper triangle - # must contain the full upper triangle of the matrix r. - # on output the full upper triangle is unaltered, and the - # strict lower triangle contains the strict upper triangle - # (transposed) of the upper triangular matrix s. + # r is an n by n array. on input the full upper triangle + # must contain the full upper triangle of the matrix r. + # on output the full upper triangle is unaltered, and the + # strict lower triangle contains the strict upper triangle + # (transposed) of the upper triangular matrix s. # - # ldr is a positive integer input variable not less than n - # which specifies the leading dimension of the array r. + # ldr is a positive integer input variable not less than n + # which specifies the leading dimension of the array r. # - # ipvt is an integer input array of length n which defines the - # permutation matrix p such that a*p = q*r. column j of p - # is column ipvt(j) of the identity matrix. + # ipvt is an integer input array of length n which defines the + # permutation matrix p such that a*p = q*r. column j of p + # is column ipvt(j) of the identity matrix. # - # diag is an input array of length n which must contain the - # diagonal elements of the matrix d. + # diag is an input array of length n which must contain the + # diagonal elements of the matrix d. # - # qtb is an input array of length n which must contain the first - # n elements of the vector (q transpose)*b. + # qtb is an input array of length n which must contain the first + # n elements of the vector (q transpose)*b. # - # delta is a positive input variable which specifies an upper - # bound on the euclidean norm of d*x. + # delta is a positive input variable which specifies an upper + # bound on the euclidean norm of d*x. # - # par is a nonnegative variable. on input par contains an - # initial estimate of the levenberg-marquardt parameter. - # on output par contains the final estimate. + # par is a nonnegative variable. on input par contains an + # initial estimate of the levenberg-marquardt parameter. + # on output par contains the final estimate. # - # x is an output array of length n which contains the least - # squares solution of the system a*x = b, sqrt(par)*d*x = 0, - # for the output par. + # x is an output array of length n which contains the least + # squares solution of the system a*x = b, sqrt(par)*d*x = 0, + # for the output par. # - # sdiag is an output array of length n which contains the - # diagonal elements of the upper triangular matrix s. + # sdiag is an output array of length n which contains the + # diagonal elements of the upper triangular matrix s. # - # wa1 and wa2 are work arrays of length n. + # wa1 and wa2 are work arrays of length n. # - # subprograms called + # subprograms called # - # minpack-supplied ... dpmpar,enorm,qrsolv + # minpack-supplied ... dpmpar,enorm,qrsolv # - # fortran-supplied ... dabs,dmax1,dmin1,dsqrt + # fortran-supplied ... dabs,dmax1,dmin1,dsqrt # - # argonne national laboratory. minpack project. march 1980. - # burton s. garbow, kenneth e. hillstrom, jorge j. more + # argonne national laboratory. minpack project. march 1980. + # burton s. garbow, kenneth e. hillstrom, jorge j. more # def lmpar(self, r, ipvt, diag, qtb, delta, x, sdiag, par=None): if self.debug: - print('Entering lmpar...') + print("Entering lmpar...") dwarf = self.machar.minnum machep = self.machar.machep sz = r.shape @@ -2065,7 +2177,7 @@ def lmpar(self, r, ipvt, diag, qtb, delta, x, sdiag, par=None): wh = (numpy.nonzero(numpy.abs(numpy.diagonal(r)) < rthresh))[0] if len(wh) > 0: nsing = wh[0] - wa1[wh[0]:] = 0 + wa1[wh[0] :] = 0 if nsing >= 1: # *** Reverse loop *** for j in range(nsing - 1, -1, -1): @@ -2083,13 +2195,13 @@ def lmpar(self, r, ipvt, diag, qtb, delta, x, sdiag, par=None): dxnorm = self.enorm(wa2) fp = dxnorm - delta if fp <= 0.1 * delta: - return [r, 0., x, sdiag] + return [r, 0.0, x, sdiag] # If the jacobian is not rank deficient, the newton step provides a # lower bound, parl, for the zero of the function. Otherwise set # this bound to zero. - parl = 0. + parl = 0.0 if nsing >= n: wa1 = diag[ipvt] * wa2[ipvt] / dxnorm wa1[0] = wa1[0] / r[0, 0] # Degenerate case @@ -2102,7 +2214,7 @@ def lmpar(self, r, ipvt, diag, qtb, delta, x, sdiag, par=None): # Calculate an upper bound, paru, for the zero of the function for j in range(n): - sum0 = sum(r[0:j + 1, j] * qtb[0:j + 1]) + sum0 = sum(r[0 : j + 1, j] * qtb[0 : j + 1]) wa1[j] = sum0 / diag[ipvt[j]] gnorm = self.enorm(wa1) paru = gnorm / delta @@ -2118,7 +2230,7 @@ def lmpar(self, r, ipvt, diag, qtb, delta, x, sdiag, par=None): par = gnorm / dxnorm # Beginning of an interation - while (1): + while 1: iter = iter + 1 # Evaluate the function at the current value of par @@ -2132,9 +2244,11 @@ def lmpar(self, r, ipvt, diag, qtb, delta, x, sdiag, par=None): temp = fp fp = dxnorm - delta - if (numpy.abs(fp) <= 0.1 * delta) or \ - ((parl == 0) and (fp <= temp) and (temp < 0)) or \ - (iter == 10): + if ( + (numpy.abs(fp) <= 0.1 * delta) + or ((parl == 0) and (fp <= temp) and (temp < 0)) + or (iter == 10) + ): break # Compute the newton correction @@ -2142,7 +2256,7 @@ def lmpar(self, r, ipvt, diag, qtb, delta, x, sdiag, par=None): for j in range(n - 1): wa1[j] = wa1[j] / sdiag[j] - wa1[j + 1:n] = wa1[j + 1:n] - r[j + 1:n, j] * wa1[j] + wa1[j + 1 : n] = wa1[j + 1 : n] - r[j + 1 : n, j] * wa1[j] wa1[n - 1] = wa1[n - 1] / sdiag[n - 1] # Degenerate case temp = self.enorm(wa1) @@ -2165,94 +2279,94 @@ def lmpar(self, r, ipvt, diag, qtb, delta, x, sdiag, par=None): # Procedure to tie one parameter to another. def tie(self, p, ptied=None): if self.debug: - print('Entering tie...') + print("Entering tie...") if ptied is None: return for i in range(len(ptied)): - if ptied[i] == '': + if ptied[i] == "": continue - cmd = 'p[' + str(i) + '] = ' + ptied[i] + cmd = "p[" + str(i) + "] = " + ptied[i] exec(cmd) return p - # Original FORTRAN documentation - # ********** + # Original FORTRAN documentation + # ********** # - # subroutine covar + # subroutine covar # - # given an m by n matrix a, the problem is to determine - # the covariance matrix corresponding to a, defined as + # given an m by n matrix a, the problem is to determine + # the covariance matrix corresponding to a, defined as # - # t - # inverse(a *a) . + # t + # inverse(a *a) . # - # this subroutine completes the solution of the problem - # if it is provided with the necessary information from the - # qr factorization, with column pivoting, of a. that is, if - # a*p = q*r, where p is a permutation matrix, q has orthogonal - # columns, and r is an upper triangular matrix with diagonal - # elements of nonincreasing magnitude, then covar expects - # the full upper triangle of r and the permutation matrix p. - # the covariance matrix is then computed as + # this subroutine completes the solution of the problem + # if it is provided with the necessary information from the + # qr factorization, with column pivoting, of a. that is, if + # a*p = q*r, where p is a permutation matrix, q has orthogonal + # columns, and r is an upper triangular matrix with diagonal + # elements of nonincreasing magnitude, then covar expects + # the full upper triangle of r and the permutation matrix p. + # the covariance matrix is then computed as # - # t t - # p*inverse(r *r)*p . + # t t + # p*inverse(r *r)*p . # - # if a is nearly rank deficient, it may be desirable to compute - # the covariance matrix corresponding to the linearly independent - # columns of a. to define the numerical rank of a, covar uses - # the tolerance tol. if l is the largest integer such that + # if a is nearly rank deficient, it may be desirable to compute + # the covariance matrix corresponding to the linearly independent + # columns of a. to define the numerical rank of a, covar uses + # the tolerance tol. if l is the largest integer such that # - # abs(r(l,l)) .gt. tol*abs(r(1,1)) , + # abs(r(l,l)) .gt. tol*abs(r(1,1)) , # - # then covar computes the covariance matrix corresponding to - # the first l columns of r. for k greater than l, column - # and row ipvt(k) of the covariance matrix are set to zero. + # then covar computes the covariance matrix corresponding to + # the first l columns of r. for k greater than l, column + # and row ipvt(k) of the covariance matrix are set to zero. # - # the subroutine statement is + # the subroutine statement is # - # subroutine covar(n,r,ldr,ipvt,tol,wa) + # subroutine covar(n,r,ldr,ipvt,tol,wa) # - # where + # where # - # n is a positive integer input variable set to the order of r. + # n is a positive integer input variable set to the order of r. # - # r is an n by n array. on input the full upper triangle must - # contain the full upper triangle of the matrix r. on output - # r contains the square symmetric covariance matrix. + # r is an n by n array. on input the full upper triangle must + # contain the full upper triangle of the matrix r. on output + # r contains the square symmetric covariance matrix. # - # ldr is a positive integer input variable not less than n - # which specifies the leading dimension of the array r. + # ldr is a positive integer input variable not less than n + # which specifies the leading dimension of the array r. # - # ipvt is an integer input array of length n which defines the - # permutation matrix p such that a*p = q*r. column j of p - # is column ipvt(j) of the identity matrix. + # ipvt is an integer input array of length n which defines the + # permutation matrix p such that a*p = q*r. column j of p + # is column ipvt(j) of the identity matrix. # - # tol is a nonnegative input variable used to define the - # numerical rank of a in the manner described above. + # tol is a nonnegative input variable used to define the + # numerical rank of a in the manner described above. # - # wa is a work array of length n. + # wa is a work array of length n. # - # subprograms called + # subprograms called # - # fortran-supplied ... dabs + # fortran-supplied ... dabs # - # argonne national laboratory. minpack project. august 1980. - # burton s. garbow, kenneth e. hillstrom, jorge j. more + # argonne national laboratory. minpack project. august 1980. + # burton s. garbow, kenneth e. hillstrom, jorge j. more # - # ********** + # ********** - def calc_covar(self, rr, ipvt=None, tol=1.e-14): + def calc_covar(self, rr, ipvt=None, tol=1.0e-14): if self.debug: - print('Entering calc_covar...') + print("Entering calc_covar...") if numpy.ndim(rr) != 2: - print('ERROR: r must be a two-dimensional matrix') + print("ERROR: r must be a two-dimensional matrix") return -1 s = rr.shape n = s[0] if s[0] != s[1]: - print('ERROR: r must be a square matrix') + print("ERROR: r must be a square matrix") return -1 if ipvt is None: @@ -2266,11 +2380,11 @@ def calc_covar(self, rr, ipvt=None, tol=1.e-14): for k in range(n): if numpy.abs(r[k, k]) <= tolr: break - r[k, k] = 1. / r[k, k] + r[k, k] = 1.0 / r[k, k] for j in range(k): temp = r[k, k] * r[j, k] - r[j, k] = 0. - r[0:j + 1, k] = r[0:j + 1, k] - temp * r[0:j + 1, j] + r[j, k] = 0.0 + r[0 : j + 1, k] = r[0 : j + 1, k] - temp * r[0 : j + 1, j] l = k # Form the full upper triangle of the inverse of (r transpose)*r @@ -2279,9 +2393,9 @@ def calc_covar(self, rr, ipvt=None, tol=1.e-14): for k in range(l + 1): for j in range(k): temp = r[j, k] - r[0:j + 1, j] = r[0:j + 1, j] + temp * r[0:j + 1, k] + r[0 : j + 1, j] = r[0 : j + 1, j] + temp * r[0 : j + 1, k] temp = r[k, k] - r[0:k + 1, k] = temp * r[0:k + 1, k] + r[0 : k + 1, k] = temp * r[0 : k + 1, k] # For the full lower triangle of the covariance matrix # in the strict lower triangle or and in wa @@ -2291,7 +2405,7 @@ def calc_covar(self, rr, ipvt=None, tol=1.e-14): sing = j > l for i in range(j + 1): if sing: - r[i, j] = 0. + r[i, j] = 0.0 ii = ipvt[i] if ii > jj: r[ii, jj] = r[i, j] @@ -2301,7 +2415,7 @@ def calc_covar(self, rr, ipvt=None, tol=1.e-14): # Symmetrize the covariance matrix in r for j in range(n): - r[0:j + 1, j] = r[j, 0:j + 1] + r[0 : j + 1, j] = r[j, 0 : j + 1] r[j, j] = wa[j] return r diff --git a/imap_l3_processing/maps/rectangular_survival_probability.py b/imap_l3_processing/maps/rectangular_survival_probability.py index c942284c8..3fd41f4f5 100644 --- a/imap_l3_processing/maps/rectangular_survival_probability.py +++ b/imap_l3_processing/maps/rectangular_survival_probability.py @@ -2,19 +2,28 @@ import xarray as xr from imap_processing.ena_maps.ena_maps import RectangularSkyMap, PointingSet from imap_processing.ena_maps.utils.coordinates import CoordNames -from imap_processing.ena_maps.utils.corrections import apply_compton_getting_correction, \ - add_spacecraft_position_and_velocity_to_pset, calculate_ram_mask +from imap_processing.ena_maps.utils.corrections import ( + apply_compton_getting_correction, + add_spacecraft_position_and_velocity_to_pset, + calculate_ram_mask, +) from imap_processing.spice.geometry import SpiceFrame from imap_l3_processing.glows.quality_flags import GlowsL3Flags from imap_l3_processing.maps.map_descriptors import Sensor, SpinPhase -from imap_l3_processing.maps.map_models import GlowsL3eRectangularMapInputData, InputRectangularPointingSet +from imap_l3_processing.maps.map_models import ( + GlowsL3eRectangularMapInputData, + InputRectangularPointingSet, +) from imap_l3_processing.maps.quality_flags import MapL3Flags -def interpolate_angular_data_to_nearest_neighbor(input_azimuths: np.array, glows_azimuths: np.array, - glows_data: np.array) -> np.array: - expanded_az = np.concatenate([glows_azimuths - 360, glows_azimuths, glows_azimuths + 360]) +def interpolate_angular_data_to_nearest_neighbor( + input_azimuths: np.array, glows_azimuths: np.array, glows_data: np.array +) -> np.array: + expanded_az = np.concatenate( + [glows_azimuths - 360, glows_azimuths, glows_azimuths + 360] + ) expanded_glows_data = np.concatenate([glows_data, glows_data, glows_data], axis=1) sort = np.argsort(expanded_az) sorted_az = expanded_az[sort] @@ -24,21 +33,31 @@ def interpolate_angular_data_to_nearest_neighbor(input_azimuths: np.array, glows class RectangularSurvivalProbabilityPointingSet(PointingSet): - def __init__(self, l1c_dataset: InputRectangularPointingSet, sensor: Sensor, spin_phase: SpinPhase, - glows_dataset: GlowsL3eRectangularMapInputData, energies: np.ndarray, cg_corrected: bool = False): + def __init__( + self, + l1c_dataset: InputRectangularPointingSet, + sensor: Sensor, + spin_phase: SpinPhase, + glows_dataset: GlowsL3eRectangularMapInputData, + energies: np.ndarray, + cg_corrected: bool = False, + ): num_spin_angle_bins = l1c_dataset.exposure_times.shape[-1] deg_spacing = 360 / num_spin_angle_bins half_bin_width = deg_spacing / 2 - self.azimuths = np.linspace(0, 360, num_spin_angle_bins, - endpoint=False) + half_bin_width + self.azimuths = ( + np.linspace(0, 360, num_spin_angle_bins, endpoint=False) + half_bin_width + ) sensor_angle = Sensor.get_sensor_angle(sensor) self.elevations = np.repeat(sensor_angle, num_spin_angle_bins) hae_az_el_points = xr.DataArray( - np.column_stack([l1c_dataset.hae_longitude[0], l1c_dataset.hae_latitude[0]]), + np.column_stack( + [l1c_dataset.hae_longitude[0], l1c_dataset.hae_latitude[0]] + ), dims=[CoordNames.GENERIC_PIXEL.value, CoordNames.AZ_EL_VECTOR.value], ) @@ -46,36 +65,48 @@ def __init__(self, l1c_dataset: InputRectangularPointingSet, sensor: Sensor, spi if l1c_dataset.epoch_delta is not None: repointing_midpoint = l1c_dataset.epoch_j2000 + l1c_dataset.epoch_delta / 2 - elif l1c_dataset.pointing_start_met is not None and l1c_dataset.pointing_end_met is not None: - pointing_duration = l1c_dataset.pointing_end_met - l1c_dataset.pointing_start_met - repointing_midpoint = l1c_dataset.epoch_j2000 + (1e9 * pointing_duration / 2) - - initial_dataset = xr.Dataset({}, - coords={ - CoordNames.TIME.value: l1c_dataset.epoch_j2000, - CoordNames.ENERGY_ULTRA_L1C.value: l1c_dataset.esa_energy_step, - CoordNames.AZIMUTH_L1C.value: self.azimuths, - }) - - initial_dataset['epoch'] = l1c_dataset.epoch_j2000 - initial_dataset['epoch_delta'] = l1c_dataset.epoch_delta - initial_dataset['hae_longitude'] = xr.DataArray( + elif ( + l1c_dataset.pointing_start_met is not None + and l1c_dataset.pointing_end_met is not None + ): + pointing_duration = ( + l1c_dataset.pointing_end_met - l1c_dataset.pointing_start_met + ) + repointing_midpoint = l1c_dataset.epoch_j2000 + ( + 1e9 * pointing_duration / 2 + ) + + initial_dataset = xr.Dataset( + {}, + coords={ + CoordNames.TIME.value: l1c_dataset.epoch_j2000, + CoordNames.ENERGY_ULTRA_L1C.value: l1c_dataset.esa_energy_step, + CoordNames.AZIMUTH_L1C.value: self.azimuths, + }, + ) + + initial_dataset["epoch"] = l1c_dataset.epoch_j2000 + initial_dataset["epoch_delta"] = l1c_dataset.epoch_delta + initial_dataset["hae_longitude"] = xr.DataArray( l1c_dataset.hae_longitude, dims=[CoordNames.TIME.value, CoordNames.AZIMUTH_L1C.value], ) - initial_dataset['hae_latitude'] = xr.DataArray( + initial_dataset["hae_latitude"] = xr.DataArray( l1c_dataset.hae_latitude, dims=[CoordNames.TIME.value, CoordNames.AZIMUTH_L1C.value], ) - initial_dataset['pointing_start_met'] = l1c_dataset.pointing_start_met - initial_dataset['pointing_end_met'] = l1c_dataset.pointing_end_met + initial_dataset["pointing_start_met"] = l1c_dataset.pointing_start_met + initial_dataset["pointing_end_met"] = l1c_dataset.pointing_end_met if sensor == Sensor.Hi90 or sensor == Sensor.Hi45: - initial_dataset.attrs['Logical_source'] = 'imap_hi' + initial_dataset.attrs["Logical_source"] = "imap_hi" elif sensor in (Sensor.Lo75, Sensor.Lo90, Sensor.Lo105): - initial_dataset.attrs['Logical_source'] = 'imap_lo' + initial_dataset.attrs["Logical_source"] = "imap_lo" else: - raise ValueError("Unexpected sensor when performing survival probability correction!", sensor.name) + raise ValueError( + "Unexpected sensor when performing survival probability correction!", + sensor.name, + ) dataset = add_spacecraft_position_and_velocity_to_pset(initial_dataset) if cg_corrected: @@ -83,21 +114,35 @@ def __init__(self, l1c_dataset: InputRectangularPointingSet, sensor: Sensor, spi dataset = apply_compton_getting_correction( dataset, - xr.DataArray(energy_in_ev, dims=[CoordNames.ENERGY_ULTRA_L1C.value]) + xr.DataArray(energy_in_ev, dims=[CoordNames.ENERGY_ULTRA_L1C.value]), ) self.az_el_points = xr.DataArray( - np.stack([dataset['hae_longitude'].values[0], dataset['hae_latitude'].values[0]], axis=2), - dims=[CoordNames.ENERGY_ULTRA_L1C.value, CoordNames.GENERIC_PIXEL.value, CoordNames.AZ_EL_VECTOR.value], + np.stack( + [ + dataset["hae_longitude"].values[0], + dataset["hae_latitude"].values[0], + ], + axis=2, + ), + dims=[ + CoordNames.ENERGY_ULTRA_L1C.value, + CoordNames.GENERIC_PIXEL.value, + CoordNames.AZ_EL_VECTOR.value, + ], ) spacecraft_frame_energies_in_kev = dataset["energy_sc"].values / 1000.0 exposure = np.full_like(l1c_dataset.exposure_times, np.nan) - for (energy_i, spin_angle_i), cg_energy in np.ndenumerate(dataset['energy_sc'].values[0]): + for (energy_i, spin_angle_i), cg_energy in np.ndenumerate( + dataset["energy_sc"].values[0] + ): distance_to_bins = np.abs(np.log10(energy_in_ev) - np.log10(cg_energy)) closest_energy_bin_index = np.argmin(distance_to_bins) - exposure[0, energy_i, spin_angle_i] = l1c_dataset.exposure_times[0, closest_energy_bin_index, spin_angle_i] + exposure[0, energy_i, spin_angle_i] = l1c_dataset.exposure_times[ + 0, closest_energy_bin_index, spin_angle_i + ] else: self.az_el_points = hae_az_el_points exposure = l1c_dataset.exposure_times @@ -110,8 +155,13 @@ def __init__(self, l1c_dataset: InputRectangularPointingSet, sensor: Sensor, spi dataset["directional_mask"] = ~dataset["ram_mask"] if cg_corrected: - sp_interpolated_to_pset_angles = interpolate_angular_data_to_nearest_neighbor( - self.azimuths, glows_dataset.spin_angle, glows_dataset.probability_of_survival[0]) + sp_interpolated_to_pset_angles = ( + interpolate_angular_data_to_nearest_neighbor( + self.azimuths, + glows_dataset.spin_angle, + glows_dataset.probability_of_survival[0], + ) + ) log_sc_frame_energies = np.log10(spacecraft_frame_energies_in_kev[0]) sp_final = np.empty((1, len(energies), num_spin_angle_bins)) @@ -119,11 +169,13 @@ def __init__(self, l1c_dataset: InputRectangularPointingSet, sensor: Sensor, spi sp_final[0, :, spin_angle_index] = np.interp( log_sc_frame_energies[:, spin_angle_index], np.log10(glows_dataset.energy), - sp_interpolated_to_pset_angles[:, spin_angle_index] + sp_interpolated_to_pset_angles[:, spin_angle_index], ) else: glows_spin_bin_count = len(glows_dataset.spin_angle) - sp_interpolated_to_hi_energies = np.empty(shape=(len(energies), glows_spin_bin_count)) + sp_interpolated_to_hi_energies = np.empty( + shape=(len(energies), glows_spin_bin_count) + ) for spin_angle_index in range(glows_spin_bin_count): sp_interpolated_to_hi_energies[:, spin_angle_index] = np.interp( np.log10(energies), @@ -150,7 +202,7 @@ def __init__(self, l1c_dataset: InputRectangularPointingSet, sensor: Sensor, spi CoordNames.TIME.value, CoordNames.ENERGY_ULTRA_L1C.value, CoordNames.AZIMUTH_L1C.value, - ] + ], ) dataset["exposure"] = xr.DataArray( exposure, @@ -170,24 +222,28 @@ def __init__(self, l1c_dataset: InputRectangularPointingSet, sensor: Sensor, spi CoordNames.AZIMUTH_L1C.value, ], ) - nominal_alpha_proton_flag_set = flag_value & GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO != 0 + nominal_alpha_proton_flag_set = ( + flag_value & GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO != 0 + ) dataset["nominal_alpha_proton_ratio_flag"] = xr.DataArray( exposure * nominal_alpha_proton_flag_set, dims=[ CoordNames.TIME.value, CoordNames.ENERGY_ULTRA_L1C.value, CoordNames.AZIMUTH_L1C.value, - ] + ], ) - persisted_last_point_flag_set = flag_value & GlowsL3Flags.PERSISTED_LAST_POINT != 0 + persisted_last_point_flag_set = ( + flag_value & GlowsL3Flags.PERSISTED_LAST_POINT != 0 + ) dataset["persisted_last_point_flag"] = xr.DataArray( exposure * persisted_last_point_flag_set, dims=[ CoordNames.TIME.value, CoordNames.ENERGY_ULTRA_L1C.value, CoordNames.AZIMUTH_L1C.value, - ] + ], ) frame = SpiceFrame.IMAP_HAE @@ -195,22 +251,44 @@ def __init__(self, l1c_dataset: InputRectangularPointingSet, sensor: Sensor, spi class RectangularSurvivalProbabilitySkyMap(RectangularSkyMap): - def __init__(self, survival_probability_pointing_sets: list[RectangularSurvivalProbabilityPointingSet], - spacing_degree: float, spice_frame: SpiceFrame): + def __init__( + self, + survival_probability_pointing_sets: list[ + RectangularSurvivalProbabilityPointingSet + ], + spacing_degree: float, + spice_frame: SpiceFrame, + ): super().__init__(spacing_degree, spice_frame) - for sp_pset in survival_probability_pointing_sets: - value_keys = ["survival_probability_times_exposure", "exposure", "predicted_ephemeris_flag", "nominal_alpha_proton_ratio_flag", "persisted_last_point_flag"] - self.project_pset_values_to_map(sp_pset, value_keys, pset_valid_mask=sp_pset.data["directional_mask"]) + for sp_pset in survival_probability_pointing_sets: + value_keys = [ + "survival_probability_times_exposure", + "exposure", + "predicted_ephemeris_flag", + "nominal_alpha_proton_ratio_flag", + "persisted_last_point_flag", + ] + self.project_pset_values_to_map( + sp_pset, value_keys, pset_valid_mask=sp_pset.data["directional_mask"] + ) predicted_ephemeris_set = self.data_1d["predicted_ephemeris_flag"] != 0 - nominal_alpha_proton_ratio_set = self.data_1d["nominal_alpha_proton_ratio_flag"] != 0 + nominal_alpha_proton_ratio_set = ( + self.data_1d["nominal_alpha_proton_ratio_flag"] != 0 + ) persisted_last_point_set = self.data_1d["persisted_last_point_flag"] != 0 - quality_flags_1d = (predicted_ephemeris_set * MapL3Flags.PREDICTIVE_EPHEMERIS) | \ - (nominal_alpha_proton_ratio_set * MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO) | \ - (persisted_last_point_set * MapL3Flags.PERSISTED_LAST_POINT) - - self.data_1d = xr.Dataset({ - "exposure_weighted_survival_probabilities": self.data_1d["survival_probability_times_exposure"] / - self.data_1d["exposure"], - "quality_flags": quality_flags_1d, - }) + quality_flags_1d = ( + (predicted_ephemeris_set * MapL3Flags.PREDICTIVE_EPHEMERIS) + | (nominal_alpha_proton_ratio_set * MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO) + | (persisted_last_point_set * MapL3Flags.PERSISTED_LAST_POINT) + ) + + self.data_1d = xr.Dataset( + { + "exposure_weighted_survival_probabilities": self.data_1d[ + "survival_probability_times_exposure" + ] + / self.data_1d["exposure"], + "quality_flags": quality_flags_1d, + } + ) diff --git a/imap_l3_processing/maps/sp_map_initializer.py b/imap_l3_processing/maps/sp_map_initializer.py index 3694bc1fa..08674951b 100644 --- a/imap_l3_processing/maps/sp_map_initializer.py +++ b/imap_l3_processing/maps/sp_map_initializer.py @@ -5,8 +5,13 @@ from imap_data_access import ScienceFilePath from imap_data_access.file_validation import Version -from imap_l3_processing.maps.map_descriptors import MapDescriptorParts, parse_map_descriptor, \ - get_duration_from_map_descriptor, map_descriptor_parts_to_string, Sensor +from imap_l3_processing.maps.map_descriptors import ( + MapDescriptorParts, + parse_map_descriptor, + get_duration_from_map_descriptor, + map_descriptor_parts_to_string, + Sensor, +) from imap_l3_processing.maps.map_initializer import PossibleMapToProduce, MapInitializer from imap_l3_processing.models import InputMetadata, VersionMap @@ -16,10 +21,14 @@ class SPMapInitializer(MapInitializer): def __init__(self, instrument: str, l2_query_results: list[dict[str, str]]): super().__init__(instrument) - self.input_maps_by_descriptor = self.get_latest_version_by_descriptor_and_start_date(l2_query_results) + self.input_maps_by_descriptor = ( + self.get_latest_version_by_descriptor_and_start_date(l2_query_results) + ) @abstractmethod - def _collect_glows_psets_by_repoint(self, descriptor: MapDescriptorParts) -> dict[int, str]: + def _collect_glows_psets_by_repoint( + self, descriptor: MapDescriptorParts + ) -> dict[int, str]: raise NotImplementedError() @abstractmethod @@ -27,25 +36,41 @@ def _get_ancillary_files(self) -> list[str]: raise NotImplementedError() @abstractmethod - def _get_l2_dependencies(self, descriptor: MapDescriptorParts) -> list[MapDescriptorParts]: + def _get_l2_dependencies( + self, descriptor: MapDescriptorParts + ) -> list[MapDescriptorParts]: raise NotImplementedError() - def get_maps_that_can_be_produced(self, descriptor: str) -> list[PossibleMapToProduce]: + def get_maps_that_can_be_produced( + self, descriptor: str + ) -> list[PossibleMapToProduce]: l3_descriptor_parts = parse_map_descriptor(descriptor) map_duration = get_duration_from_map_descriptor(l3_descriptor_parts) - glows_file_by_repointing = self._collect_glows_psets_by_repoint(l3_descriptor_parts) + glows_file_by_repointing = self._collect_glows_psets_by_repoint( + l3_descriptor_parts + ) if len(glows_file_by_repointing) == 0: - logger.info(f"No GLOWS data available for descriptor {descriptor}, no maps will be produced!") + logger.info( + f"No GLOWS data available for descriptor {descriptor}, no maps will be produced!" + ) l2_descriptors = self._get_l2_dependencies(l3_descriptor_parts) - l2_descriptor_strs = [map_descriptor_parts_to_string(parts) for parts in l2_descriptors] - assert l2_descriptor_strs, f"Expected at least one L2 dependency for l3 map: {descriptor}" - - possible_start_dates = set(self.input_maps_by_descriptor[l2_descriptor_strs[0]].keys()) + l2_descriptor_strs = [ + map_descriptor_parts_to_string(parts) for parts in l2_descriptors + ] + assert l2_descriptor_strs, ( + f"Expected at least one L2 dependency for l3 map: {descriptor}" + ) + + possible_start_dates = set( + self.input_maps_by_descriptor[l2_descriptor_strs[0]].keys() + ) for l2_descriptor in l2_descriptor_strs[1:]: - possible_start_dates.intersection_update(self.input_maps_by_descriptor[l2_descriptor].keys()) + possible_start_dates.intersection_update( + self.input_maps_by_descriptor[l2_descriptor].keys() + ) possible_maps = [] for str_start_date in sorted(list(possible_start_dates)): @@ -53,10 +78,16 @@ def get_maps_that_can_be_produced(self, descriptor: str) -> list[PossibleMapToPr l2_file_paths = [] for l2_descriptor in l2_descriptor_strs: - l2_file_paths.append(self.input_maps_by_descriptor[l2_descriptor][str_start_date]) + l2_file_paths.append( + self.input_maps_by_descriptor[l2_descriptor][str_start_date] + ) if l3_descriptor_parts.sensor in [Sensor.UltraCombined, Sensor.HiCombined]: - l1c_names = [l1c for l2 in l2_file_paths for l1c in self.get_l1c_parents_from_map(l2)] + l1c_names = [ + l1c + for l2 in l2_file_paths + for l1c in self.get_l1c_parents_from_map(l2) + ] else: l1c_names = self.get_l1c_parents_from_map(l2_file_paths[0]) @@ -76,19 +107,34 @@ def get_maps_that_can_be_produced(self, descriptor: str) -> list[PossibleMapToPr l1c_repointings = [ScienceFilePath(l1c).repointing for l1c in l1c_names] - glows_files = [glows_file_by_repointing[repoint] for repoint in l1c_repointings if - repoint in glows_file_by_repointing] + glows_files = [ + glows_file_by_repointing[repoint] + for repoint in l1c_repointings + if repoint in glows_file_by_repointing + ] if len(glows_files) == 0: - logger.info(f"No GLOWS data available for l3 map: {descriptor} with start date: {str_start_date}. Skipping") + logger.info( + f"No GLOWS data available for l3 map: {descriptor} with start date: {str_start_date}. Skipping" + ) continue - input_metadata = InputMetadata(instrument=self.instrument, data_level='l3', start_date=start_date, - end_date=start_date + map_duration, version=VersionMap({}, Version(None, 1)), - descriptor=descriptor) + input_metadata = InputMetadata( + instrument=self.instrument, + data_level="l3", + start_date=start_date, + end_date=start_date + map_duration, + version=VersionMap({}, Version(None, 1)), + descriptor=descriptor, + ) possible_map_to_produce = PossibleMapToProduce( - input_files=set(l2_file_paths + glows_files + l1c_names + self._get_ancillary_files()), - input_metadata=input_metadata + input_files=set( + l2_file_paths + + glows_files + + l1c_names + + self._get_ancillary_files() + ), + input_metadata=input_metadata, ) possible_maps.append(possible_map_to_produce) return possible_maps diff --git a/imap_l3_processing/maps/spectral_fit.py b/imap_l3_processing/maps/spectral_fit.py index 039321083..eebaf3e0f 100644 --- a/imap_l3_processing/maps/spectral_fit.py +++ b/imap_l3_processing/maps/spectral_fit.py @@ -3,18 +3,21 @@ import numpy as np import scipy -from imap_l3_processing.maps.map_models import SpectralIndexMapData, IntensityMapData, \ - calculate_datetime_weighted_average +from imap_l3_processing.maps.map_models import ( + SpectralIndexMapData, + IntensityMapData, + calculate_datetime_weighted_average, +) from imap_l3_processing.maps.mpfit import mpfit -def calculate_spectral_index_for_multiple_ranges(intensity_data: IntensityMapData, map_ranges) -> SpectralIndexMapData: +def calculate_spectral_index_for_multiple_ranges( + intensity_data: IntensityMapData, map_ranges +) -> SpectralIndexMapData: spectral_maps = [] for start, end in map_ranges: spectral_maps.append( - fit_spectral_index_map( - slice_energy_range(intensity_data, start, end) - ) + fit_spectral_index_map(slice_energy_range(intensity_data, start, end)) ) return SpectralIndexMapData( epoch=intensity_data.epoch, @@ -48,31 +51,37 @@ def calculate_spectral_index_for_multiple_ranges(intensity_data: IntensityMapDat ena_spectral_index_chisq=np.concat( [m.ena_spectral_index_chisq for m in spectral_maps], axis=1 ), - quality_flags=np.concat( - [m.quality_flags for m in spectral_maps], axis=1 - ), + quality_flags=np.concat([m.quality_flags for m in spectral_maps], axis=1), ) -def slice_energy_range(data: IntensityMapData, start: float, end: float) -> IntensityMapData: +def slice_energy_range( + data: IntensityMapData, start: float, end: float +) -> IntensityMapData: energy_mask = np.logical_and(data.energy >= start, data.energy < end) - return dataclasses.replace(data, - energy=data.energy[energy_mask], - energy_delta_plus=data.energy_delta_plus[energy_mask], - energy_delta_minus=data.energy_delta_minus[energy_mask], - energy_label=data.energy_label[energy_mask], - exposure_factor=data.exposure_factor[:, energy_mask], - obs_date=data.obs_date[:, energy_mask], - obs_date_range=data.obs_date_range[:, energy_mask], - ena_intensity=data.ena_intensity[:, energy_mask], - ena_intensity_sys_err=data.ena_intensity_sys_err[:, energy_mask], - ena_intensity_stat_uncert=data.ena_intensity_stat_uncert[:, energy_mask], - quality_flags=data.quality_flags[:, energy_mask], - ) - -def slice_energy_range_by_bin(data: IntensityMapData, start_bin_id: int, end_bin_id: int) -> IntensityMapData: + return dataclasses.replace( + data, + energy=data.energy[energy_mask], + energy_delta_plus=data.energy_delta_plus[energy_mask], + energy_delta_minus=data.energy_delta_minus[energy_mask], + energy_label=data.energy_label[energy_mask], + exposure_factor=data.exposure_factor[:, energy_mask], + obs_date=data.obs_date[:, energy_mask], + obs_date_range=data.obs_date_range[:, energy_mask], + ena_intensity=data.ena_intensity[:, energy_mask], + ena_intensity_sys_err=data.ena_intensity_sys_err[:, energy_mask], + ena_intensity_stat_uncert=data.ena_intensity_stat_uncert[:, energy_mask], + quality_flags=data.quality_flags[:, energy_mask], + ) + + +def slice_energy_range_by_bin( + data: IntensityMapData, start_bin_id: int, end_bin_id: int +) -> IntensityMapData: max_bin_id = 1 + len(data.energy) - bin_ids_in_range = (1 <= start_bin_id <= max_bin_id) and (1 <= end_bin_id <= max_bin_id) + bin_ids_in_range = (1 <= start_bin_id <= max_bin_id) and ( + 1 <= end_bin_id <= max_bin_id + ) bin_ids_valid = bin_ids_in_range and start_bin_id < end_bin_id if not bin_ids_valid: raise ValueError(f"Error slicing energy bins {start_bin_id},{end_bin_id}") @@ -93,6 +102,7 @@ def slice_energy_range_by_bin(data: IntensityMapData, start_bin_id: int, end_bin quality_flags=data.quality_flags[:, energy_slice], ) + def fit_spectral_index_map(intensity_data: IntensityMapData) -> SpectralIndexMapData: fluxes = intensity_data.ena_intensity uncertainty = intensity_data.ena_intensity_stat_uncert @@ -103,20 +113,36 @@ def fit_spectral_index_map(intensity_data: IntensityMapData) -> SpectralIndexMap mean_energy = np.sqrt(min_energy * max_energy) new_energy_label = f"{min_energy} - {max_energy}" - output_scalar_coefficients, output_scalar_errors, output_gammas, output_gamma_errors, chisq = fit_arrays_to_power_law(fluxes, uncertainty, energy) - mean_obs_date = calculate_datetime_weighted_average(intensity_data.obs_date, - weights=intensity_data.exposure_factor, - axis=1, keepdims=True) - mean_obs_date_range = np.ma.average(intensity_data.obs_date_range, weights=intensity_data.exposure_factor, - axis=1, - keepdims=True) - total_exposure_factor = np.sum(intensity_data.exposure_factor, axis=1, keepdims=True) + ( + output_scalar_coefficients, + output_scalar_errors, + output_gammas, + output_gamma_errors, + chisq, + ) = fit_arrays_to_power_law(fluxes, uncertainty, energy) + mean_obs_date = calculate_datetime_weighted_average( + intensity_data.obs_date, + weights=intensity_data.exposure_factor, + axis=1, + keepdims=True, + ) + mean_obs_date_range = np.ma.average( + intensity_data.obs_date_range, + weights=intensity_data.exposure_factor, + axis=1, + keepdims=True, + ) + total_exposure_factor = np.sum( + intensity_data.exposure_factor, axis=1, keepdims=True + ) positive_gammas = output_gammas < 0 output_gammas[positive_gammas] = np.nan output_scalar_coefficients[positive_gammas] = np.nan output_gamma_errors[positive_gammas] = np.nan chisq[positive_gammas] = np.nan - quality_flags = np.bitwise_or.reduce(intensity_data.quality_flags, axis=1, keepdims=True) + quality_flags = np.bitwise_or.reduce( + intensity_data.quality_flags, axis=1, keepdims=True + ) return SpectralIndexMapData( epoch=intensity_data.epoch, @@ -136,15 +162,16 @@ def fit_spectral_index_map(intensity_data: IntensityMapData) -> SpectralIndexMap ena_spectral_index_scalar_coefficient=output_scalar_coefficients, ena_spectral_index_scalar_coefficient_stat_uncert=output_scalar_errors, ena_spectral_index_chisq=chisq, - quality_flags=quality_flags + quality_flags=quality_flags, ) -def fit_arrays_to_power_law(fluxes: np.ndarray, uncertainties: np.ndarray, energy: np.ndarray) -> tuple[ - np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: +def fit_arrays_to_power_law( + fluxes: np.ndarray, uncertainties: np.ndarray, energy: np.ndarray +) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: par_info = [ - {'limits': [0.0, 1000.0]}, - {'limits': [0.0, 1000.0]}, + {"limits": [0.0, 1000.0]}, + {"limits": [0.0, 1000.0]}, ] output_shape = (fluxes.shape[0], 1, *fluxes.shape[2:]) @@ -168,15 +195,24 @@ def fit_arrays_to_power_law(fluxes: np.ndarray, uncertainties: np.ndarray, energ flux = intensity[:, i] uncertainty = unc[:, i] flux_and_variance_are_zero = np.equal(flux, 0) & np.equal(uncertainty, 0) - flux_or_error_is_invalid = np.isnan(flux) | np.isnan(uncertainty) | flux_and_variance_are_zero + flux_or_error_is_invalid = ( + np.isnan(flux) | np.isnan(uncertainty) | flux_and_variance_are_zero + ) flux = flux[~flux_or_error_is_invalid] uncertainty = uncertainty[~flux_or_error_is_invalid] filtered_energy = energy[~flux_or_error_is_invalid] positive_flux = flux > 0 if np.count_nonzero(positive_flux) > 1: - keywords = {'xval': filtered_energy, 'yval': flux, 'errval': uncertainty} - result = scipy.stats.linregress(np.log10(filtered_energy[positive_flux]), np.log10(flux[positive_flux])) + keywords = { + "xval": filtered_energy, + "yval": flux, + "errval": uncertainty, + } + result = scipy.stats.linregress( + np.log10(filtered_energy[positive_flux]), + np.log10(flux[positive_flux]), + ) initial_parameters = (10**result.intercept, -result.slope) @@ -192,17 +228,27 @@ def fit_arrays_to_power_law(fluxes: np.ndarray, uncertainties: np.ndarray, energ chisqs[i] = fit.fnorm / fit.dof output_gammas[epoch, 0] = gammas.reshape(fluxes.shape[2:]) output_gamma_errors[epoch, 0] = gamma_errors.reshape(fluxes.shape[2:]) - output_scalar_coefficients[epoch, 0] = scalar_coefficients.reshape(fluxes.shape[2:]) - output_scalar_coefficients_errors[epoch, 0] = scalar_coefficient_errors.reshape(fluxes.shape[2:]) + output_scalar_coefficients[epoch, 0] = scalar_coefficients.reshape( + fluxes.shape[2:] + ) + output_scalar_coefficients_errors[epoch, 0] = scalar_coefficient_errors.reshape( + fluxes.shape[2:] + ) output_chisqs[epoch, 0] = chisqs.reshape(fluxes.shape[2:]) - return output_scalar_coefficients, output_scalar_coefficients_errors, output_gammas, output_gamma_errors, output_chisqs + return ( + output_scalar_coefficients, + output_scalar_coefficients_errors, + output_gammas, + output_gamma_errors, + output_chisqs, + ) def power_law(params, **kwargs): A, B = params - x = kwargs['xval'] - y = kwargs['yval'] - err = kwargs['errval'] + x = kwargs["xval"] + y = kwargs["yval"] + err = kwargs["errval"] model = A * np.power(x, -B) diff --git a/imap_l3_processing/maps/survival_probability_processing.py b/imap_l3_processing/maps/survival_probability_processing.py index ebcc2808f..46a9f477d 100644 --- a/imap_l3_processing/maps/survival_probability_processing.py +++ b/imap_l3_processing/maps/survival_probability_processing.py @@ -3,57 +3,93 @@ import numpy as np from imap_processing.spice.geometry import SpiceFrame -from imap_l3_processing.maps.hilo_l3_survival_dependencies import HiLoL3SurvivalDependencies +from imap_l3_processing.maps.hilo_l3_survival_dependencies import ( + HiLoL3SurvivalDependencies, +) from imap_l3_processing.maps.map_descriptors import ReferenceFrame -from imap_l3_processing.maps.map_models import RectangularIntensityMapData, IntensityMapData, \ - InputRectangularPointingSet, GlowsL3eRectangularMapInputData -from imap_l3_processing.maps.rectangular_survival_probability import RectangularSurvivalProbabilityPointingSet, \ - RectangularSurvivalProbabilitySkyMap +from imap_l3_processing.maps.map_models import ( + RectangularIntensityMapData, + IntensityMapData, + InputRectangularPointingSet, + GlowsL3eRectangularMapInputData, +) +from imap_l3_processing.maps.rectangular_survival_probability import ( + RectangularSurvivalProbabilityPointingSet, + RectangularSurvivalProbabilitySkyMap, +) from imap_l3_processing.ultra.models import UltraGlowsL3eData, UltraL1CPSet -L1CPointingSet = TypeVar("L1CPointingSet", bound=Union[InputRectangularPointingSet, UltraL1CPSet]) -GlowsL3eData = TypeVar("GlowsL3eData", bound=Union[GlowsL3eRectangularMapInputData, UltraGlowsL3eData]) +L1CPointingSet = TypeVar( + "L1CPointingSet", bound=Union[InputRectangularPointingSet, UltraL1CPSet] +) +GlowsL3eData = TypeVar( + "GlowsL3eData", bound=Union[GlowsL3eRectangularMapInputData, UltraGlowsL3eData] +) -def combine_glows_l3e_with_l1c_pointing(glows_l3e_data: list[GlowsL3eData], l1c_data: list[L1CPointingSet]) -> list[ - tuple[L1CPointingSet, GlowsL3eData]]: +def combine_glows_l3e_with_l1c_pointing( + glows_l3e_data: list[GlowsL3eData], l1c_data: list[L1CPointingSet] +) -> list[tuple[L1CPointingSet, GlowsL3eData]]: l1c_by_repoint = {l1c.repointing: l1c for l1c in l1c_data} glows_by_repoint = {l3e.repointing: l3e for l3e in glows_l3e_data} - return [(l1c_by_repoint[repoint], glows_by_repoint[repoint]) - for repoint in l1c_by_repoint.keys() if repoint in glows_by_repoint] + return [ + (l1c_by_repoint[repoint], glows_by_repoint[repoint]) + for repoint in l1c_by_repoint.keys() + if repoint in glows_by_repoint + ] + def filter_bad_days(input_psets: list[L1CPointingSet]) -> list[L1CPointingSet]: return [pset for pset in input_psets if not np.all(pset.exposure_times == 0.0)] -def process_survival_probabilities(survival_probabilities_dependencies: HiLoL3SurvivalDependencies, - spice_frame_name: SpiceFrame, cg_corrected: bool = None) \ - -> RectangularIntensityMapData: + +def process_survival_probabilities( + survival_probabilities_dependencies: HiLoL3SurvivalDependencies, + spice_frame_name: SpiceFrame, + cg_corrected: bool = None, +) -> RectangularIntensityMapData: l2_descriptor_parts = survival_probabilities_dependencies.l2_map_descriptor_parts l1c_data = filter_bad_days(survival_probabilities_dependencies.l1c_data) - combined_glows = combine_glows_l3e_with_l1c_pointing(survival_probabilities_dependencies.glows_l3e_data, l1c_data) + combined_glows = combine_glows_l3e_with_l1c_pointing( + survival_probabilities_dependencies.glows_l3e_data, l1c_data + ) pointing_sets = [] if cg_corrected is None: - cg_corrected = l2_descriptor_parts.reference_frame == ReferenceFrame.Heliospheric + cg_corrected = ( + l2_descriptor_parts.reference_frame == ReferenceFrame.Heliospheric + ) for l1c, glows_l3e in combined_glows: - pointing_sets.append(RectangularSurvivalProbabilityPointingSet( - l1c, l2_descriptor_parts.sensor, l2_descriptor_parts.spin_phase, glows_l3e, - survival_probabilities_dependencies.l2_data.intensity_map_data.energy, cg_corrected=cg_corrected)) + pointing_sets.append( + RectangularSurvivalProbabilityPointingSet( + l1c, + l2_descriptor_parts.sensor, + l2_descriptor_parts.spin_phase, + glows_l3e, + survival_probabilities_dependencies.l2_data.intensity_map_data.energy, + cg_corrected=cg_corrected, + ) + ) assert len(pointing_sets) > 0 - survival_sky_map = RectangularSurvivalProbabilitySkyMap(pointing_sets, int(l2_descriptor_parts.grid), - spice_frame_name) + survival_sky_map = RectangularSurvivalProbabilitySkyMap( + pointing_sets, int(l2_descriptor_parts.grid), spice_frame_name + ) survival_dataset = survival_sky_map.to_dataset() input_data = survival_probabilities_dependencies.l2_data.intensity_map_data - survival_probabilities = survival_dataset["exposure_weighted_survival_probabilities"].values + survival_probabilities = survival_dataset[ + "exposure_weighted_survival_probabilities" + ].values survival_corrected_intensity = input_data.ena_intensity / survival_probabilities - corrected_stat_uncert = input_data.ena_intensity_stat_uncert / survival_probabilities + corrected_stat_uncert = ( + input_data.ena_intensity_stat_uncert / survival_probabilities + ) corrected_sys_err = input_data.ena_intensity_sys_err / survival_probabilities map_data = RectangularIntensityMapData( @@ -80,12 +116,22 @@ def process_survival_probabilities(survival_probabilities_dependencies: HiLoL3Su ) if input_data.bg_intensity is not None: - map_data.intensity_map_data.bg_intensity = input_data.bg_intensity / survival_probabilities - map_data.intensity_map_data.bg_intensity_sys_err = input_data.bg_intensity_sys_err / survival_probabilities - map_data.intensity_map_data.bg_intensity_stat_uncert = input_data.bg_intensity_stat_uncert / survival_probabilities + map_data.intensity_map_data.bg_intensity = ( + input_data.bg_intensity / survival_probabilities + ) + map_data.intensity_map_data.bg_intensity_sys_err = ( + input_data.bg_intensity_sys_err / survival_probabilities + ) + map_data.intensity_map_data.bg_intensity_stat_uncert = ( + input_data.bg_intensity_stat_uncert / survival_probabilities + ) if input_data.ena_intensity_sys_err_minus is not None: - map_data.intensity_map_data.ena_intensity_sys_err_minus = input_data.ena_intensity_sys_err_minus / survival_probabilities - map_data.intensity_map_data.ena_intensity_sys_err_plus = input_data.ena_intensity_sys_err_plus / survival_probabilities + map_data.intensity_map_data.ena_intensity_sys_err_minus = ( + input_data.ena_intensity_sys_err_minus / survival_probabilities + ) + map_data.intensity_map_data.ena_intensity_sys_err_plus = ( + input_data.ena_intensity_sys_err_plus / survival_probabilities + ) return map_data diff --git a/imap_l3_processing/models.py b/imap_l3_processing/models.py index dcf44b4d5..9f944539a 100644 --- a/imap_l3_processing/models.py +++ b/imap_l3_processing/models.py @@ -39,7 +39,9 @@ def logical_source(self): return f"imap_{self.instrument}_{self.data_level}_{self.descriptor}" def to_science_file_path(self, cr_number: Optional[int] = None) -> ScienceFilePath: - assert self.repointing is None or cr_number is None, "You cannot call save_data with both a repointing in the metadata while passing in a CR number" + assert self.repointing is None or cr_number is None, ( + "You cannot call save_data with both a repointing in the metadata while passing in a CR number" + ) formatted_start_date = self.start_date.strftime("%Y%m%d") version = self.version.lookup(self.descriptor) @@ -54,6 +56,7 @@ def to_science_file_path(self, cr_number: Optional[int] = None) -> ScienceFilePa cr=cr_number, ) + @dataclass class DataProductVariable: name: str @@ -64,6 +67,7 @@ class DataProductVariable: D = TypeVar("D") + @dataclass class DataProduct(abc.ABC, Generic[D]): input_metadata: InputMetadata @@ -78,7 +82,9 @@ def add_paths_to_parents(self, paths: list[Path]): self.add_filenames_to_parents([path.name for path in paths]) def add_filenames_to_parents(self, filenames): - self.parent_file_names.extend(filename for filename in filenames if filename not in self.parent_file_names) + self.parent_file_names.extend( + filename for filename in filenames if filename not in self.parent_file_names + ) @dataclass @@ -86,7 +92,9 @@ class MagData: epoch: np.ndarray mag_data: np.ndarray - def rebin_to(self, epoch: np.ndarray[float], epoch_delta: np.ndarray[float]) -> np.ndarray[float]: + def rebin_to( + self, epoch: np.ndarray[float], epoch_delta: np.ndarray[float] + ) -> np.ndarray[float]: return rebin(self.epoch, self.mag_data, epoch, epoch_delta) diff --git a/imap_l3_processing/pitch_angles.py b/imap_l3_processing/pitch_angles.py index d56252f32..27b462698 100644 --- a/imap_l3_processing/pitch_angles.py +++ b/imap_l3_processing/pitch_angles.py @@ -2,8 +2,9 @@ from uncertainties.unumpy import nominal_values, std_devs, uarray -def calculate_pitch_angle(particle_vectors: np.ndarray[float], magnetic_field_vector: np.ndarray[float]) -> np.ndarray[ - float]: +def calculate_pitch_angle( + particle_vectors: np.ndarray[float], magnetic_field_vector: np.ndarray[float] +) -> np.ndarray[float]: norm_x = calculate_unit_vector(particle_vectors) norm_y = calculate_unit_vector(magnetic_field_vector) dot = np.sum(norm_x * norm_y, axis=-1) @@ -15,17 +16,23 @@ def calculate_unit_vector(vector: np.ndarray[float]) -> np.ndarray[float]: return vector / norm -def rotate_from_imap_despun_to_hit_despun(vector: np.ndarray[float]) -> np.ndarray[float]: - rotation_matrix_from_imap_despun_frame_to_hit_despun_instrument_frame = [[0.866025, 0.5, 0], - [-0.5, 0.866025, 0], - [0, 0, 1]] - return rotation_matrix_from_imap_despun_frame_to_hit_despun_instrument_frame @ vector +def rotate_from_imap_despun_to_hit_despun( + vector: np.ndarray[float], +) -> np.ndarray[float]: + rotation_matrix_from_imap_despun_frame_to_hit_despun_instrument_frame = [ + [0.866025, 0.5, 0], + [-0.5, 0.866025, 0], + [0, 0, 1], + ] + return ( + rotation_matrix_from_imap_despun_frame_to_hit_despun_instrument_frame @ vector + ) -def rotate_particle_vectors_from_hit_despun_to_imap_despun(vector: np.ndarray[float]) -> np.ndarray[float]: - rotation_matrix = [[0.866025, -0.5, 0], - [0.5, 0.866025, 0], - [0, 0, 1]] +def rotate_particle_vectors_from_hit_despun_to_imap_despun( + vector: np.ndarray[float], +) -> np.ndarray[float]: + rotation_matrix = [[0.866025, -0.5, 0], [0.5, 0.866025, 0], [0, 0, 1]] output = np.full_like(vector, np.nan, dtype=float) @@ -36,99 +43,140 @@ def rotate_particle_vectors_from_hit_despun_to_imap_despun(vector: np.ndarray[fl return output -def calculate_gyrophase(particle_vectors: np.ndarray, magnetic_field_vector: np.ndarray): +def calculate_gyrophase( + particle_vectors: np.ndarray, magnetic_field_vector: np.ndarray +): magnetic_field_plus_z = magnetic_field_vector imap_dps_plus_y = [0, 1, 0] if np.all(np.cross(magnetic_field_plus_z, imap_dps_plus_y) == 0): return np.full(shape=particle_vectors.shape[:-1], fill_value=np.nan) - magnetic_field_plus_x = calculate_unit_vector(np.cross(imap_dps_plus_y, magnetic_field_plus_z)) - magnetic_field_plus_y = calculate_unit_vector(np.cross(magnetic_field_plus_z, magnetic_field_plus_x)) - particle_magnetic_field_x_component = np.sum(particle_vectors * magnetic_field_plus_x, axis=-1) - particle_magnetic_field_y_component = np.sum(particle_vectors * magnetic_field_plus_y, axis=-1) - gyrophases = np.atan2(particle_magnetic_field_x_component, particle_magnetic_field_y_component) + magnetic_field_plus_x = calculate_unit_vector( + np.cross(imap_dps_plus_y, magnetic_field_plus_z) + ) + magnetic_field_plus_y = calculate_unit_vector( + np.cross(magnetic_field_plus_z, magnetic_field_plus_x) + ) + particle_magnetic_field_x_component = np.sum( + particle_vectors * magnetic_field_plus_x, axis=-1 + ) + particle_magnetic_field_y_component = np.sum( + particle_vectors * magnetic_field_plus_y, axis=-1 + ) + gyrophases = np.atan2( + particle_magnetic_field_x_component, particle_magnetic_field_y_component + ) return np.mod(np.degrees(gyrophases), 360) -def rebin_by_pitch_angle_and_gyrophase(intensity_data: np.array, - intensity_delta_plus: np.array, - intensity_delta_minus: np.array, - pitch_angles: np.array, - gyrophases: np.array, - number_of_pitch_angle_bins: int, - number_of_gyrophase_bins: int): +def rebin_by_pitch_angle_and_gyrophase( + intensity_data: np.array, + intensity_delta_plus: np.array, + intensity_delta_minus: np.array, + pitch_angles: np.array, + gyrophases: np.array, + number_of_pitch_angle_bins: int, + number_of_gyrophase_bins: int, +): intensity_with_delta_plus = uarray(intensity_data, intensity_delta_plus) intensity_with_delta_minus = uarray(intensity_data, intensity_delta_minus) - output_shape_pa_and_gyro = (intensity_data.shape[0], number_of_pitch_angle_bins, number_of_gyrophase_bins) + output_shape_pa_and_gyro = ( + intensity_data.shape[0], + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ) output_shape_pa_only = (intensity_data.shape[0], number_of_pitch_angle_bins) - rebinned_summed_by_pa_and_gyro_with_delta_plus = uarray(np.zeros(shape=output_shape_pa_and_gyro), 0) - rebinned_summed_by_pa_and_gyro_with_delta_minus = uarray(np.zeros(shape=output_shape_pa_and_gyro), 0) + rebinned_summed_by_pa_and_gyro_with_delta_plus = uarray( + np.zeros(shape=output_shape_pa_and_gyro), 0 + ) + rebinned_summed_by_pa_and_gyro_with_delta_minus = uarray( + np.zeros(shape=output_shape_pa_and_gyro), 0 + ) - rebinned_summed_pa_only_with_delta_plus = uarray(np.zeros(shape=output_shape_pa_only), 0) - rebinned_summed_pa_only_with_delta_minus = uarray(np.zeros(shape=output_shape_pa_only), 0) + rebinned_summed_pa_only_with_delta_plus = uarray( + np.zeros(shape=output_shape_pa_only), 0 + ) + rebinned_summed_pa_only_with_delta_minus = uarray( + np.zeros(shape=output_shape_pa_only), 0 + ) rebinned_count_by_pa_and_gyro = np.zeros(shape=output_shape_pa_and_gyro) rebinned_count_pa_only = np.zeros(shape=output_shape_pa_only) for i in range(intensity_data.shape[0]): for pitch_angle, gyrophase, intensity_with_plus, intensity_with_minus in zip( - np.ravel(pitch_angles), - np.ravel(gyrophases), - np.ravel(intensity_with_delta_plus[i]), - np.ravel(intensity_with_delta_minus[i])): - if not (np.isnan(intensity_with_plus.nominal_value) or np.isnan(pitch_angle)): + np.ravel(pitch_angles), + np.ravel(gyrophases), + np.ravel(intensity_with_delta_plus[i]), + np.ravel(intensity_with_delta_minus[i]), + ): + if not ( + np.isnan(intensity_with_plus.nominal_value) or np.isnan(pitch_angle) + ): if pitch_angle == 180: pitch_angle_bin = number_of_pitch_angle_bins - 1 else: - pitch_angle_bin = np.floor(pitch_angle / (180 / number_of_pitch_angle_bins)).astype(int) + pitch_angle_bin = np.floor( + pitch_angle / (180 / number_of_pitch_angle_bins) + ).astype(int) if not np.isnan(gyrophase): - bin_size = (360 / number_of_gyrophase_bins) + bin_size = 360 / number_of_gyrophase_bins normalized_gyrophase = gyrophase % 360 gyrophase_bin = int(normalized_gyrophase // bin_size) rebinned_summed_by_pa_and_gyro_with_delta_plus[ - i, pitch_angle_bin, gyrophase_bin] += intensity_with_plus + i, pitch_angle_bin, gyrophase_bin + ] += intensity_with_plus rebinned_summed_by_pa_and_gyro_with_delta_minus[ - i, pitch_angle_bin, gyrophase_bin] += intensity_with_minus - rebinned_count_by_pa_and_gyro[i, pitch_angle_bin, gyrophase_bin] += 1 - - rebinned_summed_pa_only_with_delta_plus[i, pitch_angle_bin] += intensity_with_plus - rebinned_summed_pa_only_with_delta_minus[i, pitch_angle_bin] += intensity_with_minus + i, pitch_angle_bin, gyrophase_bin + ] += intensity_with_minus + rebinned_count_by_pa_and_gyro[ + i, pitch_angle_bin, gyrophase_bin + ] += 1 + + rebinned_summed_pa_only_with_delta_plus[i, pitch_angle_bin] += ( + intensity_with_plus + ) + rebinned_summed_pa_only_with_delta_minus[i, pitch_angle_bin] += ( + intensity_with_minus + ) rebinned_count_pa_only[i, pitch_angle_bin] += 1 averaged_rebinned_intensity_by_pa_and_gyro_with_delta_plus = np.divide( rebinned_summed_by_pa_and_gyro_with_delta_plus, rebinned_count_by_pa_and_gyro, - out=np.full_like( - rebinned_summed_by_pa_and_gyro_with_delta_plus, np.nan), - where=rebinned_count_by_pa_and_gyro != 0) + out=np.full_like(rebinned_summed_by_pa_and_gyro_with_delta_plus, np.nan), + where=rebinned_count_by_pa_and_gyro != 0, + ) averaged_rebinned_intensity_by_pa_and_gyro_with_delta_minus = np.divide( rebinned_summed_by_pa_and_gyro_with_delta_minus, rebinned_count_by_pa_and_gyro, - out=np.full_like( - rebinned_summed_by_pa_and_gyro_with_delta_minus, np.nan), - where=rebinned_count_by_pa_and_gyro != 0) - - averaged_rebinned_intensity_by_pa_only_with_delta_plus = np.divide(rebinned_summed_pa_only_with_delta_plus, - rebinned_count_pa_only, - out=np.full_like( - rebinned_summed_pa_only_with_delta_plus, - np.nan), - where=rebinned_count_pa_only != 0) - averaged_rebinned_intensity_by_pa_only_with_delta_minus = np.divide(rebinned_summed_pa_only_with_delta_minus, - rebinned_count_pa_only, - out=np.full_like( - rebinned_summed_pa_only_with_delta_minus, - np.nan), - where=rebinned_count_pa_only != 0) - - return (nominal_values(averaged_rebinned_intensity_by_pa_and_gyro_with_delta_plus), - std_devs(averaged_rebinned_intensity_by_pa_and_gyro_with_delta_plus), - std_devs(averaged_rebinned_intensity_by_pa_and_gyro_with_delta_minus), - nominal_values(averaged_rebinned_intensity_by_pa_only_with_delta_plus), - std_devs(averaged_rebinned_intensity_by_pa_only_with_delta_plus), - std_devs(averaged_rebinned_intensity_by_pa_only_with_delta_minus)) + out=np.full_like(rebinned_summed_by_pa_and_gyro_with_delta_minus, np.nan), + where=rebinned_count_by_pa_and_gyro != 0, + ) + + averaged_rebinned_intensity_by_pa_only_with_delta_plus = np.divide( + rebinned_summed_pa_only_with_delta_plus, + rebinned_count_pa_only, + out=np.full_like(rebinned_summed_pa_only_with_delta_plus, np.nan), + where=rebinned_count_pa_only != 0, + ) + averaged_rebinned_intensity_by_pa_only_with_delta_minus = np.divide( + rebinned_summed_pa_only_with_delta_minus, + rebinned_count_pa_only, + out=np.full_like(rebinned_summed_pa_only_with_delta_minus, np.nan), + where=rebinned_count_pa_only != 0, + ) + + return ( + nominal_values(averaged_rebinned_intensity_by_pa_and_gyro_with_delta_plus), + std_devs(averaged_rebinned_intensity_by_pa_and_gyro_with_delta_plus), + std_devs(averaged_rebinned_intensity_by_pa_and_gyro_with_delta_minus), + nominal_values(averaged_rebinned_intensity_by_pa_only_with_delta_plus), + std_devs(averaged_rebinned_intensity_by_pa_only_with_delta_plus), + std_devs(averaged_rebinned_intensity_by_pa_only_with_delta_minus), + ) diff --git a/imap_l3_processing/processor.py b/imap_l3_processing/processor.py index bf539b94d..9ea73c8c4 100644 --- a/imap_l3_processing/processor.py +++ b/imap_l3_processing/processor.py @@ -8,8 +8,9 @@ class Processor(abc.ABC): - def __init__(self, dependencies: ProcessingInputCollection, - input_metadata: InputMetadata): + def __init__( + self, dependencies: ProcessingInputCollection, input_metadata: InputMetadata + ): self.input_metadata = input_metadata self.dependencies = dependencies @@ -19,10 +20,13 @@ def process(self) -> list[Path]: def get_parent_file_names(self, file_paths: list[Path | str] = None) -> list[str]: if file_paths: - parent_file_names = [Path(parent_file_name).name for parent_file_name in file_paths] + parent_file_names = [ + Path(parent_file_name).name for parent_file_name in file_paths + ] else: - parent_file_names = [parent_file_name.name for parent_file_name in self.dependencies.get_file_paths()] + parent_file_names = [ + parent_file_name.name + for parent_file_name in self.dependencies.get_file_paths() + ] return parent_file_names + get_spice_parent_file_names() - - diff --git a/imap_l3_processing/swapi/constants.py b/imap_l3_processing/swapi/constants.py index e272eca7e..de72012a6 100644 --- a/imap_l3_processing/swapi/constants.py +++ b/imap_l3_processing/swapi/constants.py @@ -22,7 +22,9 @@ SWAPI_BIN_PERIOD_S = 12 / 72 # duration of one ESA step in the 12 s, 72-bin sweep # Counts accumulate over the 0.145 s livetime that starts after the voltage ramp-up; # sample geometry at the center of that window rather than the bin's start time. -SWAPI_LIVETIME_CENTER_OFFSET_S = (SWAPI_BIN_PERIOD_S - SWAPI_LIVETIME_S) + SWAPI_LIVETIME_S / 2 +SWAPI_LIVETIME_CENTER_OFFSET_S = ( + SWAPI_BIN_PERIOD_S - SWAPI_LIVETIME_S +) + SWAPI_LIVETIME_S / 2 # baseline instrument background is approximately 0.1 Hz SWAPI_BACKGROUND_RATE = 0.1 diff --git a/imap_l3_processing/swapi/l3a/science/pickup_ion/collapsed_response_grid.py b/imap_l3_processing/swapi/l3a/science/pickup_ion/collapsed_response_grid.py index 9fd9de129..c49228ba6 100644 --- a/imap_l3_processing/swapi/l3a/science/pickup_ion/collapsed_response_grid.py +++ b/imap_l3_processing/swapi/l3a/science/pickup_ion/collapsed_response_grid.py @@ -11,22 +11,30 @@ CENTIMETERS_PER_METER, METERS_PER_KILOMETER, ) -from imap_l3_processing.swapi.l3a.science.pickup_ion.uniform_speed_grid import UniformSpeedGrid -from imap_l3_processing.swapi.l3a.utils import velocity_components_to_angles_in_instrument_frame +from imap_l3_processing.swapi.l3a.science.pickup_ion.uniform_speed_grid import ( + UniformSpeedGrid, +) +from imap_l3_processing.swapi.l3a.utils import ( + velocity_components_to_angles_in_instrument_frame, +) from imap_l3_processing.swapi.response.passband_grid import interpolate_passband from imap_l3_processing.swapi.response.swapi_response import ResponseGrid, SwapiResponse from imap_l3_processing.swapi.species import Species -from imap_l3_processing.swapi.response.azimuthal_transmission import interpolate_azimuthal_transmission +from imap_l3_processing.swapi.response.azimuthal_transmission import ( + interpolate_azimuthal_transmission, +) class ChunkCollapsedResponse(NamedTuple): - speed_grid: UniformSpeedGrid # shared v' grid of N cells - bin_weights: NDArray[float] # (n_sweeps, n_steps, N); count_rate = bin_weights @ f(v') + speed_grid: UniformSpeedGrid # shared v' grid of N cells + bin_weights: NDArray[ + float + ] # (n_sweeps, n_steps, N); count_rate = bin_weights @ f(v') class CollapsedResponseGrid(NamedTuple): speed_in_sw_frame: NDArray[float] # (N,) v' samples - values: NDArray[float] # (N,) H(v', V) [km^3/s] at each v' + values: NDArray[float] # (N,) H(v', V) [km^3/s] at each v' _ELEVATION_RESOLUTION = 32 @@ -65,8 +73,10 @@ def build_chunk_collapsed_response( voltage = float(voltages_v[step_index]) bulk_vec = bulk_sw_per_bin_kms[sweep_index, step_index] bulk_speed = float(bulk_speeds[sweep_index, step_index]) - bulk_azimuth_deg, bulk_elevation_deg = velocity_components_to_angles_in_instrument_frame( - bulk_vec[0], bulk_vec[1], bulk_vec[2] + bulk_azimuth_deg, bulk_elevation_deg = ( + velocity_components_to_angles_in_instrument_frame( + bulk_vec[0], bulk_vec[1], bulk_vec[2] + ) ) response_grid = swapi_response.get_response_grid( time_as_tt2000, voltage, species @@ -114,9 +124,7 @@ def build_collapsed_response_grid( bulk_elevation, ) - return CollapsedResponseGrid( - speed_in_sw_frame=speed_in_sw_frame, values=values - ) + return CollapsedResponseGrid(speed_in_sw_frame=speed_in_sw_frame, values=values) def solar_wind_frame_speed_range( @@ -170,20 +178,21 @@ def _collapse_response( continue cell_coeff = ( - delta_speed * delta_elevation * speed**2 * central_effective_area_km2 + delta_speed + * delta_elevation + * speed**2 + * central_effective_area_km2 / (bulk_speed * cos_bulk_elevation) ) for i in range(speed_in_sw_frame.shape[0]): v_prime = speed_in_sw_frame[i] - cos_angle = ( - (speed**2 + bulk_speed**2 - v_prime**2) - / (2.0 * speed * bulk_speed) + cos_angle = (speed**2 + bulk_speed**2 - v_prime**2) / ( + 2.0 * speed * bulk_speed ) - cos_delta_azimuth = ( - (cos_angle - sin_elevation * sin_bulk_elevation) - / (cos_elevation * cos_bulk_elevation) + cos_delta_azimuth = (cos_angle - sin_elevation * sin_bulk_elevation) / ( + cos_elevation * cos_bulk_elevation ) if abs(cos_delta_azimuth) >= 1.0 - 1e-5: diff --git a/imap_l3_processing/swapi/l3a/science/pickup_ion/density_of_neutral_helium_lookup_table.py b/imap_l3_processing/swapi/l3a/science/pickup_ion/density_of_neutral_helium_lookup_table.py index 275addb66..42db7009b 100644 --- a/imap_l3_processing/swapi/l3a/science/pickup_ion/density_of_neutral_helium_lookup_table.py +++ b/imap_l3_processing/swapi/l3a/science/pickup_ion/density_of_neutral_helium_lookup_table.py @@ -21,8 +21,9 @@ def __init__(self, calibration_table: np.ndarray): self.grid = (angle_with_360, distance) self.densities = np.append(self.densities, self.densities[0:1], axis=0) - self._interp = RegularGridInterpolator(self.grid, self.densities, bounds_error=False, fill_value=0, - method='linear') + self._interp = RegularGridInterpolator( + self.grid, self.densities, bounds_error=False, fill_value=0, method="linear" + ) def density(self, angle: Union[ndarray, float], distance: Union[ndarray, float]): if isinstance(distance, float): @@ -39,9 +40,11 @@ def density(self, angle: Union[ndarray, float], distance: Union[ndarray, float]) distance > self.get_minimum_distance(), result, ( - self.density(angle, np.ones_like(distance) * self.get_minimum_distance()) + self.density( + angle, np.ones_like(distance) * self.get_minimum_distance() + ) * (distance / self.get_minimum_distance()) - ) + ), ) return result diff --git a/imap_l3_processing/swapi/l3a/science/pickup_ion/goodness_of_fit.py b/imap_l3_processing/swapi/l3a/science/pickup_ion/goodness_of_fit.py index 4a64cbb68..c4e4e0983 100644 --- a/imap_l3_processing/swapi/l3a/science/pickup_ion/goodness_of_fit.py +++ b/imap_l3_processing/swapi/l3a/science/pickup_ion/goodness_of_fit.py @@ -6,7 +6,9 @@ MAX_CUTOFF_SPEED_KMS = 550.0 # v_cutoff <= 550 km/s MAX_MEAN_RELATIVE_ERROR = 0.12 # Delta_rel <= 0.12 MAX_PAST_PEAK_RATIO = 0.4 # R_past_peak <= 0.4 -CUTOFF_DROP_RATIO = 0.25 # extend up to and including the last step with model rate >= 0.25 * peak +CUTOFF_DROP_RATIO = ( + 0.25 # extend up to and including the last step with model rate >= 0.25 * peak +) MIN_CUTOFF_SPEED_RATIO = 0.5 # v_cutoff >= 0.5 * v_sw MAX_CUTOFF_SPEED_RATIO = 1.5 # v_cutoff <= 1.5 * v_sw MIN_IONIZATION_RATE = 0.6e-9 # beta >= 0.6 * 10^-9 s^-1 @@ -67,13 +69,14 @@ def is_good_fit( in_range = ~past_range mean_absolute_percent_error = ( - np.abs(chunk_mean_model_rates + SWAPI_BACKGROUND_RATE - chunk_mean_observed_rates) + np.abs( + chunk_mean_model_rates + SWAPI_BACKGROUND_RATE - chunk_mean_observed_rates + ) / (chunk_mean_model_rates + SWAPI_BACKGROUND_RATE) )[in_range].mean() past_cutoff_ratio = ( - chunk_mean_observed_rates[past_range].mean() - / chunk_mean_model_rates.max() + chunk_mean_observed_rates[past_range].mean() / chunk_mean_model_rates.max() ) cutoff_speed_ratio = cutoff_speed_kms / sw_speed_kms diff --git a/imap_l3_processing/swapi/l3a/science/pickup_ion/inflow_vector.py b/imap_l3_processing/swapi/l3a/science/pickup_ion/inflow_vector.py index 9b5a5d1b9..ab66bb33f 100644 --- a/imap_l3_processing/swapi/l3a/science/pickup_ion/inflow_vector.py +++ b/imap_l3_processing/swapi/l3a/science/pickup_ion/inflow_vector.py @@ -19,10 +19,12 @@ def from_file(cls, path: PathLike) -> InflowVector: loaded_vector = np.squeeze(np.loadtxt(path, dtype=float)) - assert loaded_vector.shape == (3,), f"Failed to parse Inflow Vector from {path.name}" + assert loaded_vector.shape == (3,), ( + f"Failed to parse Inflow Vector from {path.name}" + ) return cls( speed_km_per_s=(loaded_vector[0]), longitude_deg_eclipj2000=loaded_vector[1], - latitude_deg_eclipj2000=loaded_vector[2] + latitude_deg_eclipj2000=loaded_vector[2], ) diff --git a/imap_l3_processing/swapi/l3a/science/pickup_ion/moments.py b/imap_l3_processing/swapi/l3a/science/pickup_ion/moments.py index ea20d1311..69f8114f8 100644 --- a/imap_l3_processing/swapi/l3a/science/pickup_ion/moments.py +++ b/imap_l3_processing/swapi/l3a/science/pickup_ion/moments.py @@ -36,6 +36,7 @@ def calculate_helium_pui_density( Uncertainty on the fit parameters is propagated through the integral. """ + @uncertainties.wrap def calculate(ionization_rate, cutoff_speed): f_pui = vasyliunas_siscoe_vdf( diff --git a/imap_l3_processing/swapi/l3a/science/pickup_ion/utils.py b/imap_l3_processing/swapi/l3a/science/pickup_ion/utils.py index 74f0f7b55..f44af1c74 100644 --- a/imap_l3_processing/swapi/l3a/science/pickup_ion/utils.py +++ b/imap_l3_processing/swapi/l3a/science/pickup_ion/utils.py @@ -36,8 +36,7 @@ def rotate_rtn_velocity_to_swapi_per_bin( chunk: SwapiL2Data, sw_velocity_rtn_kms: ndarray, ) -> ndarray: - """Apply the IMAP_RTN to IMAP_SWAPI rotation at each point in time. - """ + """Apply the IMAP_RTN to IMAP_SWAPI rotation at each point in time.""" measurement_times_tt2000_ns = measurement_times(chunk.sci_start_time) n_sweeps, n_bins = measurement_times_tt2000_ns.shape ephemeris_times = ttj2000ns_to_et(measurement_times_tt2000_ns.ravel()) diff --git a/imap_l3_processing/swapi/l3a/science/pickup_ion/vasyliunas_siscoe_distribution.py b/imap_l3_processing/swapi/l3a/science/pickup_ion/vasyliunas_siscoe_distribution.py index 9ba699523..e012ee354 100644 --- a/imap_l3_processing/swapi/l3a/science/pickup_ion/vasyliunas_siscoe_distribution.py +++ b/imap_l3_processing/swapi/l3a/science/pickup_ion/vasyliunas_siscoe_distribution.py @@ -3,11 +3,18 @@ import numpy as np from numpy.typing import NDArray -from imap_l3_processing.constants import ONE_AU_IN_KM, CENTIMETERS_PER_METER, METERS_PER_KILOMETER +from imap_l3_processing.constants import ( + ONE_AU_IN_KM, + CENTIMETERS_PER_METER, + METERS_PER_KILOMETER, +) from imap_l3_processing.swapi.constants import SWAPI_PUI_COOLING_INDEX -from imap_l3_processing.swapi.l3a.science.pickup_ion.density_of_neutral_helium_lookup_table import \ - DensityOfNeutralHeliumLookupTable -from imap_l3_processing.swapi.l3a.science.pickup_ion.uniform_speed_grid import UniformSpeedGrid +from imap_l3_processing.swapi.l3a.science.pickup_ion.density_of_neutral_helium_lookup_table import ( + DensityOfNeutralHeliumLookupTable, +) +from imap_l3_processing.swapi.l3a.science.pickup_ion.uniform_speed_grid import ( + UniformSpeedGrid, +) def vasyliunas_siscoe_vdf( @@ -70,10 +77,8 @@ def _filled_shell_vdf_without_cutoff( ) -> NDArray: w = speed_in_sw_frame / cutoff_speed radius_in_au = distance / ONE_AU_IN_KM - neutral_helium_density_per_cm3 = ( - density_of_neutral_helium_lookup_table.density( - inflow_angle, radius_in_au * w**SWAPI_PUI_COOLING_INDEX - ) + neutral_helium_density_per_cm3 = density_of_neutral_helium_lookup_table.density( + inflow_angle, radius_in_au * w**SWAPI_PUI_COOLING_INDEX ) neutral_helium_density_per_km3 = ( neutral_helium_density_per_cm3 @@ -81,9 +86,7 @@ def _filled_shell_vdf_without_cutoff( ) term1 = SWAPI_PUI_COOLING_INDEX / (4 * np.pi) term2 = (ionization_rate * ONE_AU_IN_KM**2) / ( - distance - * solar_wind_speed_inertial_frame - * cutoff_speed**3 + distance * solar_wind_speed_inertial_frame * cutoff_speed**3 ) term3 = w ** (SWAPI_PUI_COOLING_INDEX - 3) term4 = neutral_helium_density_per_km3 @@ -110,4 +113,4 @@ def _apply_partial_heaviside_at_cutoff( bin_min = speed_grid.centers[cutoff_index] - delta_v_prime / 2 fraction_below_cutoff = (cutoff_speed - bin_min) / delta_v_prime f_pui[cutoff_index] *= fraction_below_cutoff - f_pui[cutoff_index + 1:] = 0.0 + f_pui[cutoff_index + 1 :] = 0.0 diff --git a/imap_l3_processing/swapi/l3a/science/solar_wind/alpha/fit_solar_wind_alpha_model.py b/imap_l3_processing/swapi/l3a/science/solar_wind/alpha/fit_solar_wind_alpha_model.py index 4c39674c3..7aa41baa4 100644 --- a/imap_l3_processing/swapi/l3a/science/solar_wind/alpha/fit_solar_wind_alpha_model.py +++ b/imap_l3_processing/swapi/l3a/science/solar_wind/alpha/fit_solar_wind_alpha_model.py @@ -160,9 +160,8 @@ def _construct_alpha_fit_result( delta_v_fit = float(result.x[2]) velocity_rtn = proton_bulk + delta_v_fit * magnetic_field_direction - speed_ratio = ( - np.linalg.norm(velocity_rtn, axis=-1) - / np.linalg.norm(proton_bulk, axis=-1) + speed_ratio = np.linalg.norm(velocity_rtn, axis=-1) / np.linalg.norm( + proton_bulk, axis=-1 ) if speed_ratio < MIN_TOLERABLE_ALPHA_SPEED_RATIO: return _nan_alpha_fit_result(bad_fit_flag | SwapiL3Flags.BAD_FIT) @@ -190,9 +189,7 @@ def _construct_alpha_fit_result( return AlphaSolarWindFitResult( density=ufloat(alpha_density_fit, density_sigma), temperature=ufloat(alpha_temperature_fit, temperature_sigma), - velocity_rtn=make_correlated_velocity( - velocity_rtn, velocity_covariance_rtn - ), + velocity_rtn=make_correlated_velocity(velocity_rtn, velocity_covariance_rtn), delta_v=ufloat(delta_v_fit, delta_v_sigma), quality_flag=int(bad_fit_flag), ) @@ -202,9 +199,7 @@ def _alpha_r_squared( residuals: ndarray, count_rate: ndarray, n_sweeps: int, n_peak_bins: int ) -> float: averaged_count_rate = np.nanmean(count_rate, axis=0) - averaged_residual = np.nanmean( - residuals.reshape(n_sweeps, n_peak_bins), axis=0 - ) + averaged_residual = np.nanmean(residuals.reshape(n_sweeps, n_peak_bins), axis=0) return r_squared(averaged_residual, averaged_count_rate) @@ -231,7 +226,10 @@ def _eval(self, x: ndarray) -> None: alpha_velocity_rtn = self.proton_bulk + delta_v * self.magnetic_field_direction alpha_true, jacobian_alpha_5d = model_solar_wind_ideal_coincidence_rates( SolarWindParams( - alpha_density, alpha_velocity_rtn, alpha_temperature, self.alpha_ctx.mass_kg + alpha_density, + alpha_velocity_rtn, + alpha_temperature, + self.alpha_ctx.mass_kg, ), self.alpha_ctx, ) diff --git a/imap_l3_processing/swapi/l3a/science/solar_wind/forward_model.py b/imap_l3_processing/swapi/l3a/science/solar_wind/forward_model.py index a2f7a03d0..6b32ff7fb 100644 --- a/imap_l3_processing/swapi/l3a/science/solar_wind/forward_model.py +++ b/imap_l3_processing/swapi/l3a/science/solar_wind/forward_model.py @@ -135,10 +135,14 @@ def _integrate_region( for i_speed in range(speed_quadrature.points.shape[0]): speed = speed_quadrature.points[i_speed] exponent = ( - speed ** 2 - + bulk_speed(sw_params) ** 2 - - 2 * speed * velocity_along_direction - ) * 1.0 / (2 * thermal_speed(sw_params) ** 2) + ( + speed**2 + + bulk_speed(sw_params) ** 2 + - 2 * speed * velocity_along_direction + ) + * 1.0 + / (2 * thermal_speed(sw_params) ** 2) + ) weighted_integrand = ( speed_quadrature.weights[i_speed] * speed_quadrature.speed_cubed_times_passband[i_speed] @@ -190,13 +194,13 @@ def _integrate_region( elevation_log_temperature_jacobian * count_rate_factor - 1.5 * rate ) jacobian[VELOCITY_SLICE.start] = ( - elevation_jacobian_r * count_rate_factor / thermal_speed(sw_params) ** 2 + elevation_jacobian_r * count_rate_factor / thermal_speed(sw_params) ** 2 ) jacobian[VELOCITY_SLICE.start + 1] = ( - elevation_jacobian_t * count_rate_factor / thermal_speed(sw_params) ** 2 + elevation_jacobian_t * count_rate_factor / thermal_speed(sw_params) ** 2 ) jacobian[VELOCITY_SLICE.start + 2] = ( - elevation_jacobian_n * count_rate_factor / thermal_speed(sw_params) ** 2 + elevation_jacobian_n * count_rate_factor / thermal_speed(sw_params) ** 2 ) return rate, jacobian @@ -222,4 +226,3 @@ def _direction_in_rtn(angular_quadrature, rotation_xyz_to_rtn, i_elevation, i_az + rotation_xyz_to_rtn[2, 1] * direction_y + rotation_xyz_to_rtn[2, 2] * direction_z, ) - diff --git a/imap_l3_processing/swapi/l3a/science/solar_wind/integration_limits.py b/imap_l3_processing/swapi/l3a/science/solar_wind/integration_limits.py index 74b9dec5d..5f9519fdb 100644 --- a/imap_l3_processing/swapi/l3a/science/solar_wind/integration_limits.py +++ b/imap_l3_processing/swapi/l3a/science/solar_wind/integration_limits.py @@ -113,9 +113,7 @@ def rescale(nodes, weights, lower, upper): ) transmission_azimuth = np.array( [ - interpolate_azimuthal_transmission( - response_grid.azimuthal_transmission, az - ) + interpolate_azimuthal_transmission(response_grid.azimuthal_transmission, az) for az in azimuth_points ] ) @@ -179,7 +177,9 @@ def _angular_limits( half_width = _maxwellian_angular_extent( sw_params, response_grid.central_speed, EPSILON ) - bulk_az, bulk_el = velocity_to_angles_in_instrument_frame(sw_params, rotation_matrix) + bulk_az, bulk_el = velocity_to_angles_in_instrument_frame( + sw_params, rotation_matrix + ) if region.is_sunglasses: el_lo, el_hi = response_grid.sg_passband.elevation_range diff --git a/imap_l3_processing/swapi/l3a/science/solar_wind/proton/calculate_initial_guess.py b/imap_l3_processing/swapi/l3a/science/solar_wind/proton/calculate_initial_guess.py index 10192db9a..ad910c120 100644 --- a/imap_l3_processing/swapi/l3a/science/solar_wind/proton/calculate_initial_guess.py +++ b/imap_l3_processing/swapi/l3a/science/solar_wind/proton/calculate_initial_guess.py @@ -17,8 +17,11 @@ optimal_density_scale, ) from imap_l3_processing.swapi.l3a.science.solar_wind.utils import average_spin_axis_rtn -from imap_l3_processing.swapi.l3a.science.solar_wind.params import SolarWindParams, temperature_to_thermal_speed, \ - thermal_speed_to_temperature +from imap_l3_processing.swapi.l3a.science.solar_wind.params import ( + SolarWindParams, + temperature_to_thermal_speed, + thermal_speed_to_temperature, +) # 1 eV @@ -99,5 +102,3 @@ def gaussian(v, amplitude, mean, sigma): INITIAL_TEMPERATURE_FLOOR_K, ) return float(bulk_speed_fit), float(temperature_fit) - - diff --git a/imap_l3_processing/swapi/l3a/science/solar_wind/proton/fit_solar_wind_proton_model.py b/imap_l3_processing/swapi/l3a/science/solar_wind/proton/fit_solar_wind_proton_model.py index ecb5e6193..2de0b9f39 100644 --- a/imap_l3_processing/swapi/l3a/science/solar_wind/proton/fit_solar_wind_proton_model.py +++ b/imap_l3_processing/swapi/l3a/science/solar_wind/proton/fit_solar_wind_proton_model.py @@ -62,10 +62,7 @@ def _construct_fit_result(final_result, ctx): fit_r_squared = _coarse_peak_averaged_r_squared( final_result.residuals, ctx.count_rate ) - flag_bad = ( - fit_r_squared < 0.9 - or final_result.sw_params.temperature > 5.0e5 - ) + flag_bad = fit_r_squared < 0.9 or final_result.sw_params.temperature > 5.0e5 if flag_bad: return _nan_proton_fit_result(int(SwapiL3Flags.BAD_FIT)) diff --git a/imap_l3_processing/swapi/l3a/science/solar_wind/proton/optimize_solar_wind_proton_params.py b/imap_l3_processing/swapi/l3a/science/solar_wind/proton/optimize_solar_wind_proton_params.py index 09d07ad4d..45c954b88 100644 --- a/imap_l3_processing/swapi/l3a/science/solar_wind/proton/optimize_solar_wind_proton_params.py +++ b/imap_l3_processing/swapi/l3a/science/solar_wind/proton/optimize_solar_wind_proton_params.py @@ -17,8 +17,8 @@ @dataclass class OptimizeSolarWindProtonParamsResult: sw_params: SolarWindParams - residuals: ndarray # count-rate residuals at the solution - jacobian: ndarray # ∂residuals/∂param, columns ordered per the params vector + residuals: ndarray # count-rate residuals at the solution + jacobian: ndarray # ∂residuals/∂param, columns ordered per the params vector success: bool @property diff --git a/imap_l3_processing/swapi/l3a/science/solar_wind/trim_open_aperture.py b/imap_l3_processing/swapi/l3a/science/solar_wind/trim_open_aperture.py index 5d551d1fb..8b748b506 100644 --- a/imap_l3_processing/swapi/l3a/science/solar_wind/trim_open_aperture.py +++ b/imap_l3_processing/swapi/l3a/science/solar_wind/trim_open_aperture.py @@ -5,7 +5,8 @@ from imap_l3_processing.swapi.l3a.science.solar_wind.params import ( SolarWindParams, - bulk_speed, thermal_speed, + bulk_speed, + thermal_speed, ) from imap_l3_processing.swapi.l3a.science.solar_wind.utils import ( count_rate_conversion_factor, @@ -62,9 +63,11 @@ def trim_open_aperture( break dphi_deg = scan_azimuths[1] - scan_azimuths[0] - transmission_az_integral = float(np.trapezoid( - transmission_x_maxwellian[lower_index : upper_index + 1], dx=dphi_deg - )) + transmission_az_integral = float( + np.trapezoid( + transmission_x_maxwellian[lower_index : upper_index + 1], dx=dphi_deg + ) + ) upper_bound = _oa_rate_upper_bound( response_grid, sw_params, @@ -88,15 +91,16 @@ def _evaluate_oa_integrand_along_azimuth( sigma = thermal_speed(sw_params) speed = bulk_speed(sw_params) central_speed = response_grid.central_speed - bulk_az, bulk_el = velocity_to_angles_in_instrument_frame(sw_params, rotation_matrix) + bulk_az, bulk_el = velocity_to_angles_in_instrument_frame( + sw_params, rotation_matrix + ) sin_bulk_el = math.sin(np.radians(bulk_el)) cos_bulk_el = math.cos(np.radians(bulk_el)) sin_scan_el = math.sin(np.radians(scan_elevation)) cos_scan_el = math.cos(np.radians(scan_elevation)) - cos_view_to_bulk = ( - sin_bulk_el * sin_scan_el + cos_bulk_el * cos_scan_el - * np.cos(np.radians(scan_azimuths - bulk_az)) + cos_view_to_bulk = sin_bulk_el * sin_scan_el + cos_bulk_el * cos_scan_el * np.cos( + np.radians(scan_azimuths - bulk_az) ) delta_v_sq = ( central_speed**2 + speed**2 - 2.0 * central_speed * speed * cos_view_to_bulk @@ -125,9 +129,9 @@ def _oa_rate_upper_bound( ratio_lo, ratio_hi = speed_ratio_range_at_elevation(response_grid.oa_passband, 0.0) delta_v = central_speed * (ratio_hi - ratio_lo) return ( - count_rate_conversion_factor(sw_params, response_grid) - * central_speed ** 3 - * delta_theta_deg - * delta_v - * transmission_az_integral + count_rate_conversion_factor(sw_params, response_grid) + * central_speed**3 + * delta_theta_deg + * delta_v + * transmission_az_integral ) diff --git a/imap_l3_processing/swapi/l3a/science/uncertainties.py b/imap_l3_processing/swapi/l3a/science/uncertainties.py index d35c3d2a4..d10c8033a 100644 --- a/imap_l3_processing/swapi/l3a/science/uncertainties.py +++ b/imap_l3_processing/swapi/l3a/science/uncertainties.py @@ -3,9 +3,7 @@ from uncertainties import UFloat, correlated_values, ufloat -def compute_hc3_parameter_covariance( - jacobian: ndarray, residuals: ndarray -) -> ndarray: +def compute_hc3_parameter_covariance(jacobian: ndarray, residuals: ndarray) -> ndarray: n_params = jacobian.shape[1] try: JT_J_pseudoinverse = np.linalg.pinv(jacobian.T @ jacobian) @@ -17,9 +15,7 @@ def compute_hc3_parameter_covariance( ) h_ii_clipped = np.clip(h_ii, 0.0, 0.9999) hc3_weights = (residuals / (1.0 - h_ii_clipped)) ** 2 - sandwich_middle = np.einsum( - "ki,k,kj->ij", jacobian, hc3_weights, jacobian - ) + sandwich_middle = np.einsum("ki,k,kj->ij", jacobian, hc3_weights, jacobian) return JT_J_pseudoinverse @ sandwich_middle @ JT_J_pseudoinverse except np.linalg.LinAlgError: return np.full((n_params, n_params), np.nan) diff --git a/imap_l3_processing/swapi/l3a/swapi_l3a_dependencies.py b/imap_l3_processing/swapi/l3a/swapi_l3a_dependencies.py index da826015f..cf6db7c56 100644 --- a/imap_l3_processing/swapi/l3a/swapi_l3a_dependencies.py +++ b/imap_l3_processing/swapi/l3a/swapi_l3a_dependencies.py @@ -7,18 +7,27 @@ from spacepy.pycdf import CDF from imap_l3_processing.models import MagData -from imap_l3_processing.swapi.descriptors import SWAPI_L2_DESCRIPTOR, \ - DENSITY_OF_NEUTRAL_HELIUM_DESCRIPTOR, EFFICIENCY_LOOKUP_TABLE_DESCRIPTOR, HYDROGEN_INFLOW_VECTOR_DESCRIPTOR, \ - HELIUM_INFLOW_VECTOR_DESCRIPTOR, AZIMUTHAL_TRANSMISSION_DESCRIPTOR, \ - CENTRAL_EFFECTIVE_AREA_DESCRIPTOR, PASSBAND_FIT_COEFFICIENTS_DESCRIPTOR, \ - MAG_RTN_DESCRIPTOR +from imap_l3_processing.swapi.descriptors import ( + SWAPI_L2_DESCRIPTOR, + DENSITY_OF_NEUTRAL_HELIUM_DESCRIPTOR, + EFFICIENCY_LOOKUP_TABLE_DESCRIPTOR, + HYDROGEN_INFLOW_VECTOR_DESCRIPTOR, + HELIUM_INFLOW_VECTOR_DESCRIPTOR, + AZIMUTHAL_TRANSMISSION_DESCRIPTOR, + CENTRAL_EFFECTIVE_AREA_DESCRIPTOR, + PASSBAND_FIT_COEFFICIENTS_DESCRIPTOR, + MAG_RTN_DESCRIPTOR, +) from imap_l3_processing.swapi.l3a.models import SwapiL2Data -from imap_l3_processing.swapi.l3a.science.pickup_ion.density_of_neutral_helium_lookup_table import \ - DensityOfNeutralHeliumLookupTable +from imap_l3_processing.swapi.l3a.science.pickup_ion.density_of_neutral_helium_lookup_table import ( + DensityOfNeutralHeliumLookupTable, +) from imap_l3_processing.swapi.l3a.science.pickup_ion.inflow_vector import InflowVector from imap_l3_processing.swapi.l3a.utils import read_l2_swapi_data, read_mag_rtn_data from imap_l3_processing.utils import select_mag_path -from imap_l3_processing.swapi.response.efficiency_calibration_table import EfficiencyCalibrationTable +from imap_l3_processing.swapi.response.efficiency_calibration_table import ( + EfficiencyCalibrationTable, +) from imap_l3_processing.swapi.response.swapi_response import SwapiResponse @@ -36,14 +45,30 @@ class SwapiL3ADependencies: @classmethod def fetch_dependencies(cls, dependencies: ProcessingInputCollection): # @formatter:off - science_dependency_file = dependencies.get_file_paths(source='swapi', descriptor=SWAPI_L2_DESCRIPTOR) - efficiency_calibration_table = dependencies.get_file_paths(source='swapi', descriptor=EFFICIENCY_LOOKUP_TABLE_DESCRIPTOR) - neutral_helium_table = dependencies.get_file_paths(source='swapi', descriptor=DENSITY_OF_NEUTRAL_HELIUM_DESCRIPTOR) - hydrogen_vector_paths = dependencies.get_file_paths(source='swapi', descriptor=HYDROGEN_INFLOW_VECTOR_DESCRIPTOR) - helium_vector_paths = dependencies.get_file_paths(source='swapi', descriptor=HELIUM_INFLOW_VECTOR_DESCRIPTOR) - azimuthal_transmission_paths = dependencies.get_file_paths(source='swapi', descriptor=AZIMUTHAL_TRANSMISSION_DESCRIPTOR) - central_effective_area_paths = dependencies.get_file_paths(source='swapi', descriptor=CENTRAL_EFFECTIVE_AREA_DESCRIPTOR) - passband_fit_coefficients_paths = dependencies.get_file_paths(source='swapi', descriptor=PASSBAND_FIT_COEFFICIENTS_DESCRIPTOR) + science_dependency_file = dependencies.get_file_paths( + source="swapi", descriptor=SWAPI_L2_DESCRIPTOR + ) + efficiency_calibration_table = dependencies.get_file_paths( + source="swapi", descriptor=EFFICIENCY_LOOKUP_TABLE_DESCRIPTOR + ) + neutral_helium_table = dependencies.get_file_paths( + source="swapi", descriptor=DENSITY_OF_NEUTRAL_HELIUM_DESCRIPTOR + ) + hydrogen_vector_paths = dependencies.get_file_paths( + source="swapi", descriptor=HYDROGEN_INFLOW_VECTOR_DESCRIPTOR + ) + helium_vector_paths = dependencies.get_file_paths( + source="swapi", descriptor=HELIUM_INFLOW_VECTOR_DESCRIPTOR + ) + azimuthal_transmission_paths = dependencies.get_file_paths( + source="swapi", descriptor=AZIMUTHAL_TRANSMISSION_DESCRIPTOR + ) + central_effective_area_paths = dependencies.get_file_paths( + source="swapi", descriptor=CENTRAL_EFFECTIVE_AREA_DESCRIPTOR + ) + passband_fit_coefficients_paths = dependencies.get_file_paths( + source="swapi", descriptor=PASSBAND_FIT_COEFFICIENTS_DESCRIPTOR + ) # @formatter:on mag_path, mag_level = select_mag_path(dependencies, MAG_RTN_DESCRIPTOR) @@ -63,25 +88,35 @@ def fetch_dependencies(cls, dependencies: ProcessingInputCollection): ) @classmethod - def from_file_paths(cls, science_dependency_path: Path, - efficiency_calibration_path: Path, - neutral_helium_path: Path, hydrogen_inflow_vector_path: Path, - helium_inflow_vector_path: Path, azimuthal_transmission_path: Path, - central_effective_area_path: Path, passband_fit_coefficients_path: Path, - mag_path: Optional[Path] = None, - mag_is_preliminary: bool = False): + def from_file_paths( + cls, + science_dependency_path: Path, + efficiency_calibration_path: Path, + neutral_helium_path: Path, + hydrogen_inflow_vector_path: Path, + helium_inflow_vector_path: Path, + azimuthal_transmission_path: Path, + central_effective_area_path: Path, + passband_fit_coefficients_path: Path, + mag_path: Optional[Path] = None, + mag_is_preliminary: bool = False, + ): return cls( data=read_l2_swapi_data(CDF(str(science_dependency_path))), - efficiency_calibration_table=EfficiencyCalibrationTable(efficiency_calibration_path), + efficiency_calibration_table=EfficiencyCalibrationTable( + efficiency_calibration_path + ), density_of_neutral_helium_calibration_table=DensityOfNeutralHeliumLookupTable.from_file( - neutral_helium_path), + neutral_helium_path + ), hydrogen_inflow_vector=InflowVector.from_file(hydrogen_inflow_vector_path), helium_inflow_vector=InflowVector.from_file(helium_inflow_vector_path), swapi_response=SwapiResponse.from_files( azimuthal_transmission_path=azimuthal_transmission_path, central_effective_area_path=central_effective_area_path, passband_fit_coefficients_path=passband_fit_coefficients_path, - efficiency_table_path=efficiency_calibration_path), + efficiency_table_path=efficiency_calibration_path, + ), mag_data=read_mag_rtn_data(mag_path) if mag_path is not None else None, mag_is_preliminary=mag_is_preliminary, ) diff --git a/imap_l3_processing/swapi/l3b/models.py b/imap_l3_processing/swapi/l3b/models.py index 87a54d45f..26e7c34f4 100644 --- a/imap_l3_processing/swapi/l3b/models.py +++ b/imap_l3_processing/swapi/l3b/models.py @@ -7,15 +7,22 @@ from imap_l3_processing.constants import FIVE_MINUTES_IN_NANOSECONDS from imap_l3_processing.models import DataProduct, DataProductVariable -from imap_l3_processing.swapi.l3a.models import EPOCH_CDF_VAR_NAME, EPOCH_DELTA_CDF_VAR_NAME +from imap_l3_processing.swapi.l3a.models import ( + EPOCH_CDF_VAR_NAME, + EPOCH_DELTA_CDF_VAR_NAME, +) SOLAR_WIND_ENERGY_CDF_VAR_NAME = "combined_energy" -SOLAR_WIND_COMBINED_ENERGY_DELTA_MINUS_CDF_VAR_NAME = "combined_energy_stat_uncert_minus" +SOLAR_WIND_COMBINED_ENERGY_DELTA_MINUS_CDF_VAR_NAME = ( + "combined_energy_stat_uncert_minus" +) SOLAR_WIND_COMBINED_ENERGY_DELTA_PLUS_CDF_VAR_NAME = "combined_energy_stat_uncert_plus" SOLAR_WIND_ENERGY_LABEL_CDF_VAR_NAME = "combined_energy_label" COMBINED_SOLAR_WIND_DIFFERENTIAL_FLUX_CDF_VAR_NAME = "combined_differential_flux" -COMBINED_SOLAR_WIND_DIFFERENTIAL_FLUX_DELTA_CDF_VAR_NAME = "combined_differential_flux_stat_uncert" +COMBINED_SOLAR_WIND_DIFFERENTIAL_FLUX_DELTA_CDF_VAR_NAME = ( + "combined_differential_flux_stat_uncert" +) @dataclass @@ -44,16 +51,31 @@ def to_data_product_variables(self) -> list[DataProductVariable]: "ESA %02d", np.arange(start=1, stop=energy_count + 1) ) return [ - DataProductVariable(EPOCH_CDF_VAR_NAME, self.epoch, cdf_data_type=pycdf.const.CDF_TIME_TT2000), - DataProductVariable(EPOCH_DELTA_CDF_VAR_NAME, np.full_like(self.epoch, FIVE_MINUTES_IN_NANOSECONDS)), - + DataProductVariable( + EPOCH_CDF_VAR_NAME, + self.epoch, + cdf_data_type=pycdf.const.CDF_TIME_TT2000, + ), + DataProductVariable( + EPOCH_DELTA_CDF_VAR_NAME, + np.full_like(self.epoch, FIVE_MINUTES_IN_NANOSECONDS), + ), DataProductVariable(SOLAR_WIND_ENERGY_CDF_VAR_NAME, self.combined_energy), - DataProductVariable(SOLAR_WIND_COMBINED_ENERGY_DELTA_MINUS_CDF_VAR_NAME, self.combined_energy_delta_minus), - DataProductVariable(SOLAR_WIND_COMBINED_ENERGY_DELTA_PLUS_CDF_VAR_NAME, self.combined_energy_delta_plus), + DataProductVariable( + SOLAR_WIND_COMBINED_ENERGY_DELTA_MINUS_CDF_VAR_NAME, + self.combined_energy_delta_minus, + ), + DataProductVariable( + SOLAR_WIND_COMBINED_ENERGY_DELTA_PLUS_CDF_VAR_NAME, + self.combined_energy_delta_plus, + ), DataProductVariable(SOLAR_WIND_ENERGY_LABEL_CDF_VAR_NAME, energy_labels), - - DataProductVariable(COMBINED_SOLAR_WIND_DIFFERENTIAL_FLUX_CDF_VAR_NAME, - nominal_values(self.combined_differential_flux)), - DataProductVariable(COMBINED_SOLAR_WIND_DIFFERENTIAL_FLUX_DELTA_CDF_VAR_NAME, - std_devs(self.combined_differential_flux)), + DataProductVariable( + COMBINED_SOLAR_WIND_DIFFERENTIAL_FLUX_CDF_VAR_NAME, + nominal_values(self.combined_differential_flux), + ), + DataProductVariable( + COMBINED_SOLAR_WIND_DIFFERENTIAL_FLUX_DELTA_CDF_VAR_NAME, + std_devs(self.combined_differential_flux), + ), ] diff --git a/imap_l3_processing/swapi/l3b/science/calculate_solar_wind_differential_flux.py b/imap_l3_processing/swapi/l3b/science/calculate_solar_wind_differential_flux.py index ec6695a56..9dbd469ab 100644 --- a/imap_l3_processing/swapi/l3b/science/calculate_solar_wind_differential_flux.py +++ b/imap_l3_processing/swapi/l3b/science/calculate_solar_wind_differential_flux.py @@ -1,15 +1,22 @@ from numpy import ndarray from imap_l3_processing.constants import METERS_PER_KILOMETER, CENTIMETERS_PER_METER -from imap_l3_processing.swapi.l3b.science.geometric_factor_calibration_table import GeometricFactorCalibrationTable +from imap_l3_processing.swapi.l3b.science.geometric_factor_calibration_table import ( + GeometricFactorCalibrationTable, +) -def calculate_combined_solar_wind_differential_flux(energies: ndarray, average_count_rates: ndarray, - efficiency: float, - geometric_factor_table: GeometricFactorCalibrationTable): +def calculate_combined_solar_wind_differential_flux( + energies: ndarray, + average_count_rates: ndarray, + efficiency: float, + geometric_factor_table: GeometricFactorCalibrationTable, +): geometric_factor = geometric_factor_table.lookup_geometric_factor(energies) denominator = energies * geometric_factor * efficiency result_per_square_km = average_count_rates / denominator - result_per_square_cm = result_per_square_km / (METERS_PER_KILOMETER * CENTIMETERS_PER_METER) ** 2 + result_per_square_cm = ( + result_per_square_km / (METERS_PER_KILOMETER * CENTIMETERS_PER_METER) ** 2 + ) return result_per_square_cm diff --git a/imap_l3_processing/swapi/l3b/science/calculate_solar_wind_vdf.py b/imap_l3_processing/swapi/l3b/science/calculate_solar_wind_vdf.py index 96d8e6fcb..f5e361e95 100644 --- a/imap_l3_processing/swapi/l3b/science/calculate_solar_wind_vdf.py +++ b/imap_l3_processing/swapi/l3b/science/calculate_solar_wind_vdf.py @@ -5,15 +5,30 @@ import numpy as np from numpy import ndarray -from imap_l3_processing.constants import PROTON_MASS_KG, PROTON_CHARGE_COULOMBS, ALPHA_PARTICLE_CHARGE_COULOMBS, \ - ALPHA_PARTICLE_MASS_KG, HE_PUI_PARTICLE_MASS_KG, PUI_PARTICLE_CHARGE_COULOMBS, METERS_PER_KILOMETER, \ - CENTIMETERS_PER_METER +from imap_l3_processing.constants import ( + PROTON_MASS_KG, + PROTON_CHARGE_COULOMBS, + ALPHA_PARTICLE_CHARGE_COULOMBS, + ALPHA_PARTICLE_MASS_KG, + HE_PUI_PARTICLE_MASS_KG, + PUI_PARTICLE_CHARGE_COULOMBS, + METERS_PER_KILOMETER, + CENTIMETERS_PER_METER, +) from imap_l3_processing.swapi.l3a.utils import calculate_sw_speed -from imap_l3_processing.swapi.l3b.science.geometric_factor_calibration_table import GeometricFactorCalibrationTable - - -def calculate_vdf(particle_mass, particle_charge, energies: ndarray, average_count_rates: ndarray, - efficiency: float, geometric_factor_table: GeometricFactorCalibrationTable): +from imap_l3_processing.swapi.l3b.science.geometric_factor_calibration_table import ( + GeometricFactorCalibrationTable, +) + + +def calculate_vdf( + particle_mass, + particle_charge, + energies: ndarray, + average_count_rates: ndarray, + efficiency: float, + geometric_factor_table: GeometricFactorCalibrationTable, +): velocities = calculate_sw_speed(particle_mass, particle_charge, energies) geometric_factors = geometric_factor_table.lookup_geometric_factor(energies) geometric_factors *= (METERS_PER_KILOMETER * CENTIMETERS_PER_METER) ** 2 @@ -22,31 +37,59 @@ def calculate_vdf(particle_mass, particle_charge, energies: ndarray, average_cou numerator = 4 * np.pi * proton_mass_per_charge * average_count_rates - denominator = (energies * geometric_factors * efficiency) + denominator = energies * geometric_factors * efficiency probabilities = numerator / denominator return velocities, probabilities -def calculate_proton_solar_wind_vdf(energies: ndarray, average_count_rates: ndarray, - efficiency: float, geometric_factor_table: GeometricFactorCalibrationTable): - return calculate_vdf(PROTON_MASS_KG, PROTON_CHARGE_COULOMBS, energies, average_count_rates, efficiency, - geometric_factor_table) - - -def calculate_alpha_solar_wind_vdf(energies: ndarray, average_count_rates: ndarray, - efficiency: float, geometric_factor_table: GeometricFactorCalibrationTable): - return calculate_vdf(ALPHA_PARTICLE_MASS_KG, ALPHA_PARTICLE_CHARGE_COULOMBS, energies, - average_count_rates, efficiency, - geometric_factor_table) - - -def calculate_pui_solar_wind_vdf(energies: ndarray, average_count_rates: ndarray, efficiency: float, - geometric_factor_table: GeometricFactorCalibrationTable): - return calculate_vdf(HE_PUI_PARTICLE_MASS_KG, PUI_PARTICLE_CHARGE_COULOMBS, energies, - average_count_rates, efficiency, - geometric_factor_table) +def calculate_proton_solar_wind_vdf( + energies: ndarray, + average_count_rates: ndarray, + efficiency: float, + geometric_factor_table: GeometricFactorCalibrationTable, +): + return calculate_vdf( + PROTON_MASS_KG, + PROTON_CHARGE_COULOMBS, + energies, + average_count_rates, + efficiency, + geometric_factor_table, + ) + + +def calculate_alpha_solar_wind_vdf( + energies: ndarray, + average_count_rates: ndarray, + efficiency: float, + geometric_factor_table: GeometricFactorCalibrationTable, +): + return calculate_vdf( + ALPHA_PARTICLE_MASS_KG, + ALPHA_PARTICLE_CHARGE_COULOMBS, + energies, + average_count_rates, + efficiency, + geometric_factor_table, + ) + + +def calculate_pui_solar_wind_vdf( + energies: ndarray, + average_count_rates: ndarray, + efficiency: float, + geometric_factor_table: GeometricFactorCalibrationTable, +): + return calculate_vdf( + HE_PUI_PARTICLE_MASS_KG, + PUI_PARTICLE_CHARGE_COULOMBS, + energies, + average_count_rates, + efficiency, + geometric_factor_table, + ) @dataclass @@ -62,4 +105,7 @@ def calculate_delta_minus_plus(nominal_values: ndarray) -> DeltaMinusPlus: right_edges = nominal_values * [*half_ratios, half_ratios[-1]] lower_bounds = np.minimum(left_edges, right_edges) upper_bounds = np.maximum(left_edges, right_edges) - return DeltaMinusPlus(delta_minus=nominal_values - lower_bounds, delta_plus=upper_bounds - nominal_values) + return DeltaMinusPlus( + delta_minus=nominal_values - lower_bounds, + delta_plus=upper_bounds - nominal_values, + ) diff --git a/imap_l3_processing/swapi/l3b/science/geometric_factor_calibration_table.py b/imap_l3_processing/swapi/l3b/science/geometric_factor_calibration_table.py index 25e0a4f63..076188583 100644 --- a/imap_l3_processing/swapi/l3b/science/geometric_factor_calibration_table.py +++ b/imap_l3_processing/swapi/l3b/science/geometric_factor_calibration_table.py @@ -14,5 +14,5 @@ def lookup_geometric_factor(self, energy): @classmethod def from_file(cls, file_path) -> GeometricFactorCalibrationTable: - data = np.loadtxt(file_path, skiprows=1, delimiter=',') + data = np.loadtxt(file_path, skiprows=1, delimiter=",") return cls(data) diff --git a/imap_l3_processing/swapi/l3b/swapi_l3b_dependencies.py b/imap_l3_processing/swapi/l3b/swapi_l3b_dependencies.py index b4774f320..a04b47921 100644 --- a/imap_l3_processing/swapi/l3b/swapi_l3b_dependencies.py +++ b/imap_l3_processing/swapi/l3b/swapi_l3b_dependencies.py @@ -5,12 +5,19 @@ from imap_data_access.processing_input import ProcessingInputCollection from spacepy.pycdf import CDF -from imap_l3_processing.swapi.descriptors import EFFICIENCY_LOOKUP_TABLE_DESCRIPTOR, \ - GEOMETRIC_FACTOR_SW_LOOKUP_TABLE_DESCRIPTOR, SWAPI_L2_DESCRIPTOR +from imap_l3_processing.swapi.descriptors import ( + EFFICIENCY_LOOKUP_TABLE_DESCRIPTOR, + GEOMETRIC_FACTOR_SW_LOOKUP_TABLE_DESCRIPTOR, + SWAPI_L2_DESCRIPTOR, +) from imap_l3_processing.swapi.l3a.models import SwapiL2Data from imap_l3_processing.swapi.l3a.utils import read_l2_swapi_data -from imap_l3_processing.swapi.l3b.science.calculate_solar_wind_vdf import GeometricFactorCalibrationTable -from imap_l3_processing.swapi.response.efficiency_calibration_table import EfficiencyCalibrationTable +from imap_l3_processing.swapi.l3b.science.calculate_solar_wind_vdf import ( + GeometricFactorCalibrationTable, +) +from imap_l3_processing.swapi.response.efficiency_calibration_table import ( + EfficiencyCalibrationTable, +) @dataclass @@ -21,28 +28,43 @@ class SwapiL3BDependencies: @classmethod def fetch_dependencies(cls, dependencies: ProcessingInputCollection): - science_dependency_file = dependencies.get_file_paths(source='swapi', descriptor=SWAPI_L2_DESCRIPTOR) - geometric_factor_lookup_table_file = dependencies.get_file_paths(source='swapi', - descriptor=GEOMETRIC_FACTOR_SW_LOOKUP_TABLE_DESCRIPTOR) - efficiency_table_lookup_file = dependencies.get_file_paths(source='swapi', - descriptor=EFFICIENCY_LOOKUP_TABLE_DESCRIPTOR) + science_dependency_file = dependencies.get_file_paths( + source="swapi", descriptor=SWAPI_L2_DESCRIPTOR + ) + geometric_factor_lookup_table_file = dependencies.get_file_paths( + source="swapi", descriptor=GEOMETRIC_FACTOR_SW_LOOKUP_TABLE_DESCRIPTOR + ) + efficiency_table_lookup_file = dependencies.get_file_paths( + source="swapi", descriptor=EFFICIENCY_LOOKUP_TABLE_DESCRIPTOR + ) download(science_dependency_file[0]) download(geometric_factor_lookup_table_file[0]) download(efficiency_table_lookup_file[0]) - return cls.from_file_paths(science_dependency_file[0], - geometric_factor_lookup_table_file[0], - efficiency_table_lookup_file[0]) + return cls.from_file_paths( + science_dependency_file[0], + geometric_factor_lookup_table_file[0], + efficiency_table_lookup_file[0], + ) @classmethod - def from_file_paths(cls, science_dependency_path: Path, geometric_factor_calibration_path: Path, - efficiency_calibration_table_path: Path): + def from_file_paths( + cls, + science_dependency_path: Path, + geometric_factor_calibration_path: Path, + efficiency_calibration_table_path: Path, + ): swapi_l2_data = read_l2_swapi_data(CDF(str(science_dependency_path))) geometric_factor_calibration_lookup = GeometricFactorCalibrationTable.from_file( - geometric_factor_calibration_path) - efficiency_calibration_table = EfficiencyCalibrationTable(efficiency_calibration_table_path) + geometric_factor_calibration_path + ) + efficiency_calibration_table = EfficiencyCalibrationTable( + efficiency_calibration_table_path + ) - return cls(swapi_l2_data, - geometric_factor_calibration_lookup, - efficiency_calibration_table) + return cls( + swapi_l2_data, + geometric_factor_calibration_lookup, + efficiency_calibration_table, + ) diff --git a/imap_l3_processing/swapi/response/azimuthal_transmission.py b/imap_l3_processing/swapi/response/azimuthal_transmission.py index c7c84589d..5baa39cbc 100644 --- a/imap_l3_processing/swapi/response/azimuthal_transmission.py +++ b/imap_l3_processing/swapi/response/azimuthal_transmission.py @@ -44,10 +44,7 @@ def interpolate_azimuthal_transmission( weight_lower = float(i_upper) - i_float weight_upper = i_float - float(i_lower) - return ( - grid.values[i_lower] * weight_lower - + grid.values[i_upper] * weight_upper - ) + return grid.values[i_lower] * weight_lower + grid.values[i_upper] * weight_upper def validate_azimuthal_transmission_values(values: ndarray, spacing: float) -> None: diff --git a/imap_l3_processing/swapi/response/efficiency_calibration_table.py b/imap_l3_processing/swapi/response/efficiency_calibration_table.py index 447ff9e2c..7d74dd09f 100644 --- a/imap_l3_processing/swapi/response/efficiency_calibration_table.py +++ b/imap_l3_processing/swapi/response/efficiency_calibration_table.py @@ -15,14 +15,29 @@ class EfficiencyCalibrationTable: def __init__(self, path): - self.data = np.loadtxt(path, dtype=[("time", "M8[ns]"), ("MET", "i8"), (_PROTON_EFFICIENCY_COLUMN, "f8"), (_HELIUM_EFFICIENCY_COLUMN, "f8")], ndmin=1) + self.data = np.loadtxt( + path, + dtype=[ + ("time", "M8[ns]"), + ("MET", "i8"), + (_PROTON_EFFICIENCY_COLUMN, "f8"), + (_HELIUM_EFFICIENCY_COLUMN, "f8"), + ], + ndmin=1, + ) def relative_efficiency(self, time_as_tt2000: int, species: Species) -> float: - return self._get_entry_for(time_as_tt2000, _EFFICIENCY_COLUMN_BY_SPECIES[species]) + return self._get_entry_for( + time_as_tt2000, _EFFICIENCY_COLUMN_BY_SPECIES[species] + ) def _get_entry_for(self, time_as_tt2000, column: str) -> float: for d in reversed(self.data): - if d["time"] < np.datetime64(pycdf.lib.tt2000_to_datetime(int(time_as_tt2000)), "ns"): + if d["time"] < np.datetime64( + pycdf.lib.tt2000_to_datetime(int(time_as_tt2000)), "ns" + ): return float(d[column]) - raise ValueError(f"No efficiency data for {pycdf.lib.tt2000_to_datetime(time_as_tt2000)}") + raise ValueError( + f"No efficiency data for {pycdf.lib.tt2000_to_datetime(time_as_tt2000)}" + ) diff --git a/imap_l3_processing/swapi/response/swapi_response.py b/imap_l3_processing/swapi/response/swapi_response.py index 7f2a2a856..7d5a01d70 100644 --- a/imap_l3_processing/swapi/response/swapi_response.py +++ b/imap_l3_processing/swapi/response/swapi_response.py @@ -14,7 +14,9 @@ AzimuthalTransmissionGrid, validate_azimuthal_transmission_values, ) -from imap_l3_processing.swapi.response.efficiency_calibration_table import EfficiencyCalibrationTable +from imap_l3_processing.swapi.response.efficiency_calibration_table import ( + EfficiencyCalibrationTable, +) from imap_l3_processing.swapi.response.passband_grid import ( PassbandGrid, build_passband_grid, @@ -43,9 +45,9 @@ def __init__( efficiency_table: EfficiencyCalibrationTable, ): self._azimuthal_transmission_grid = AzimuthalTransmissionGrid( - values=np.asarray(azimuthal_transmission, dtype=float), - spacing=float(self.AZIMUTHAL_TRANSMISSION_SPACING_DEG), - ) + values=np.asarray(azimuthal_transmission, dtype=float), + spacing=float(self.AZIMUTHAL_TRANSMISSION_SPACING_DEG), + ) self._central_effective_area_at_voltage = central_effective_area_at_voltage self._passband_fit_coefficients = passband_fit_coefficients self._passband_esa_voltage_limits = passband_esa_voltage_limits @@ -137,7 +139,9 @@ def get_response_grid( esa_voltage: float, species: Species, ) -> ResponseGrid: - relative_efficiency = self._efficiency_table.relative_efficiency(time_as_tt2000, species) + relative_efficiency = self._efficiency_table.relative_efficiency( + time_as_tt2000, species + ) cache_key = ( self._cache_key(float(esa_voltage)), @@ -160,8 +164,11 @@ def get_response_grid( response_grid = ResponseGrid( sg_passband=passband_pair["SG"], oa_passband=passband_pair["OA"], - central_speed=self._central_speed(esa_voltage, species.mass_per_charge_m_p_per_e), - central_effective_area=self._get_central_effective_area(esa_voltage) * relative_efficiency, + central_speed=self._central_speed( + esa_voltage, species.mass_per_charge_m_p_per_e + ), + central_effective_area=self._get_central_effective_area(esa_voltage) + * relative_efficiency, azimuthal_transmission=self._azimuthal_transmission_grid, ) diff --git a/imap_l3_processing/swe/l3/models.py b/imap_l3_processing/swe/l3/models.py index 7a3a776aa..f25102c4f 100644 --- a/imap_l3_processing/swe/l3/models.py +++ b/imap_l3_processing/swe/l3/models.py @@ -17,14 +17,22 @@ GYROPHASE_BINS_CDF_VAR_NAME = "gyrophase" GYROPHASE_DELTA_CDF_VAR_NAME = "gyrophase_delta" PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_CDF_VAR_NAME = "phase_space_density_by_pitch_angle" -PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME = "phase_space_density_by_pitch_angle_and_gyrophase" +PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME = ( + "phase_space_density_by_pitch_angle_and_gyrophase" +) PHASE_SPACE_DENSITY_1D_CDF_VAR_NAME = "phase_space_density_1d" PHASE_SPACE_DENSITY_INWARD_CDF_VAR_NAME = "phase_space_density_inward" PHASE_SPACE_DENSITY_OUTWARD_CDF_VAR_NAME = "phase_space_density_outward" INTENSITY_BY_PITCH_ANGLE_CDF_VAR_NAME = "intensity_by_pitch_angle" -INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME = "intensity_by_pitch_angle_and_gyrophase" -INTENSITY_UNCERTAINTY_BY_PITCH_ANGLE_CDF_VAR_NAME = "intensity_uncertainty_by_pitch_angle" -INTENSITY_UNCERTAINTY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME = "intensity_uncertainty_by_pitch_angle_and_gyrophase" +INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME = ( + "intensity_by_pitch_angle_and_gyrophase" +) +INTENSITY_UNCERTAINTY_BY_PITCH_ANGLE_CDF_VAR_NAME = ( + "intensity_uncertainty_by_pitch_angle" +) +INTENSITY_UNCERTAINTY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME = ( + "intensity_uncertainty_by_pitch_angle_and_gyrophase" +) SPACECRAFT_POTENTIAL_CDF_VAR_NAME = "spacecraft_potential" CORE_HALO_BREAKPOINT_CDF_VAR_NAME = "core_halo_breakpoint" CORE_FIT_NUM_POINTS_CDF_VAR_NAME = "core_fit_num_points" @@ -52,40 +60,70 @@ TOTAL_SPEED_INTEGRATED_CDF_VAR_NAME = "total_speed_integrated" CORE_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME = "core_velocity_vector_rtn_integrated" HALO_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME = "halo_velocity_vector_rtn_integrated" -TOTAL_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME = "total_velocity_vector_rtn_integrated" +TOTAL_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME = ( + "total_velocity_vector_rtn_integrated" +) CORE_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME = "core_heat_flux_magnitude_integrated" CORE_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME = "core_heat_flux_theta_integrated" CORE_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME = "core_heat_flux_phi_integrated" HALO_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME = "halo_heat_flux_magnitude_integrated" HALO_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME = "halo_heat_flux_theta_integrated" HALO_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME = "halo_heat_flux_phi_integrated" -TOTAL_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME = "total_heat_flux_magnitude_integrated" +TOTAL_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME = ( + "total_heat_flux_magnitude_integrated" +) TOTAL_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME = "total_heat_flux_theta_integrated" TOTAL_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME = "total_heat_flux_phi_integrated" CORE_T_PARALLEL_INTEGRATED_CDF_VAR_NAME = "core_t_parallel_integrated" CORE_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME = "core_t_perpendicular_integrated" -CORE_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME = "core_t_ratio_perpendicular_integrated" +CORE_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME = ( + "core_t_ratio_perpendicular_integrated" +) HALO_T_PARALLEL_INTEGRATED_CDF_VAR_NAME = "halo_t_parallel_integrated" HALO_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME = "halo_t_perpendicular_integrated" -HALO_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME = "halo_t_ratio_perpendicular_integrated" +HALO_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME = ( + "halo_t_ratio_perpendicular_integrated" +) TOTAL_T_PARALLEL_INTEGRATED_CDF_VAR_NAME = "total_t_parallel_integrated" TOTAL_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME = "total_t_perpendicular_integrated" -TOTAL_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME = "total_t_ratio_perpendicular_integrated" -CORE_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME = "core_temperature_theta_rtn_integrated" +TOTAL_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME = ( + "total_t_ratio_perpendicular_integrated" +) +CORE_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME = ( + "core_temperature_theta_rtn_integrated" +) CORE_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME = "core_temperature_phi_rtn_integrated" -HALO_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME = "halo_temperature_theta_rtn_integrated" +HALO_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME = ( + "halo_temperature_theta_rtn_integrated" +) HALO_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME = "halo_temperature_phi_rtn_integrated" -TOTAL_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME = "total_temperature_theta_rtn_integrated" -TOTAL_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME = "total_temperature_phi_rtn_integrated" +TOTAL_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME = ( + "total_temperature_theta_rtn_integrated" +) +TOTAL_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME = ( + "total_temperature_phi_rtn_integrated" +) CORE_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME = "core_temperature_parallel_to_mag" -CORE_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME = "core_temperature_perpendicular_to_mag" -CORE_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME = "core_temperature_ratio_perpendicular_to_mag" +CORE_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME = ( + "core_temperature_perpendicular_to_mag" +) +CORE_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME = ( + "core_temperature_ratio_perpendicular_to_mag" +) HALO_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME = "halo_temperature_parallel_to_mag" -HALO_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME = "halo_temperature_perpendicular_to_mag" -HALO_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME = "halo_temperature_ratio_perpendicular_to_mag" +HALO_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME = ( + "halo_temperature_perpendicular_to_mag" +) +HALO_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME = ( + "halo_temperature_ratio_perpendicular_to_mag" +) TOTAL_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME = "total_temperature_parallel_to_mag" -TOTAL_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME = "total_temperature_perpendicular_to_mag" -TOTAL_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME = "total_temperature_ratio_perpendicular_to_mag" +TOTAL_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME = ( + "total_temperature_perpendicular_to_mag" +) +TOTAL_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME = ( + "total_temperature_ratio_perpendicular_to_mag" +) CORE_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME = "core_temperature_tensor_integrated" HALO_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME = "halo_temperature_tensor_integrated" TOTAL_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME = "total_temperature_tensor_integrated" @@ -100,7 +138,9 @@ HALO_VELOCITY_VECTOR_RTN_FIT_LABEL = "halo_velocity_vector_rtn_fit_label" CORE_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL = "core_velocity_vector_rtn_integrated_label" HALO_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL = "halo_velocity_vector_rtn_integrated_label" -TOTAL_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL = "total_velocity_vector_rtn_integrated_label" +TOTAL_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL = ( + "total_velocity_vector_rtn_integrated_label" +) CORE_TEMPERATURE_TENSOR_INTEGRATED_LABEL = "core_temperature_tensor_integrated_label" HALO_TEMPERATURE_TENSOR_INTEGRATED_LABEL = "halo_temperature_tensor_integrated_label" TOTAL_TEMPERATURE_TENSOR_INTEGRATED_LABEL = "total_temperature_tensor_integrated_label" @@ -242,204 +282,385 @@ def to_data_product_variables(self) -> list[DataProductVariable]: DataProductVariable(EPOCH_CDF_VAR_NAME, value=self.epoch), DataProductVariable(EPOCH_DELTA_CDF_VAR_NAME, value=self.epoch_delta), DataProductVariable(ENERGY_CDF_VAR_NAME, value=self.energy), - DataProductVariable(ENERGY_DELTA_PLUS_CDF_VAR_NAME, value=self.energy_delta_plus), - DataProductVariable(ENERGY_DELTA_MINUS_CDF_VAR_NAME, value=self.energy_delta_minus), + DataProductVariable( + ENERGY_DELTA_PLUS_CDF_VAR_NAME, value=self.energy_delta_plus + ), + DataProductVariable( + ENERGY_DELTA_MINUS_CDF_VAR_NAME, value=self.energy_delta_minus + ), DataProductVariable(PITCH_ANGLE_CDF_VAR_NAME, value=self.pitch_angle), - DataProductVariable(PITCH_ANGLE_DELTA_CDF_VAR_NAME, value=self.pitch_angle_delta), - + DataProductVariable( + PITCH_ANGLE_DELTA_CDF_VAR_NAME, value=self.pitch_angle_delta + ), DataProductVariable(GYROPHASE_BINS_CDF_VAR_NAME, value=self.gyrophase_bins), - DataProductVariable(GYROPHASE_DELTA_CDF_VAR_NAME, value=self.gyrophase_delta), - DataProductVariable(PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_CDF_VAR_NAME, - value=self.phase_space_density_by_pitch_angle), - DataProductVariable(PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME, - value=self.phase_space_density_by_pitch_angle_and_gyrophase), - DataProductVariable(PHASE_SPACE_DENSITY_1D_CDF_VAR_NAME, - value=self.phase_space_density_1d), - DataProductVariable(PHASE_SPACE_DENSITY_INWARD_CDF_VAR_NAME, - value=self.phase_space_density_inward), - DataProductVariable(PHASE_SPACE_DENSITY_OUTWARD_CDF_VAR_NAME, - value=self.phase_space_density_outward), - DataProductVariable(INTENSITY_BY_PITCH_ANGLE_CDF_VAR_NAME, - value=self.intensity_by_pitch_angle), - DataProductVariable(INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME, - value=self.intensity_by_pitch_angle_and_gyrophase), - DataProductVariable(INTENSITY_UNCERTAINTY_BY_PITCH_ANGLE_CDF_VAR_NAME, - value=self.intensity_uncertainty_by_pitch_angle), - DataProductVariable(INTENSITY_UNCERTAINTY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME, - value=self.intensity_uncertainty_by_pitch_angle_and_gyrophase), - DataProductVariable(SPACECRAFT_POTENTIAL_CDF_VAR_NAME, - value=self.spacecraft_potential), - DataProductVariable(CORE_HALO_BREAKPOINT_CDF_VAR_NAME, - value=self.core_halo_breakpoint), - DataProductVariable(CORE_FIT_NUM_POINTS_CDF_VAR_NAME, - value=self.moment_data.core_fit_num_points), - DataProductVariable(CHISQ_C_CDF_VAR_NAME, - value=self.moment_data.core_chisq), - DataProductVariable(CHISQ_H_CDF_VAR_NAME, - value=self.moment_data.halo_chisq), - DataProductVariable(CORE_DENSITY_FIT_CDF_VAR_NAME, - value=self.moment_data.core_density_fit), - DataProductVariable(HALO_DENSITY_FIT_CDF_VAR_NAME, - value=self.moment_data.halo_density_fit), - DataProductVariable(CORE_T_PARALLEL_FIT_CDF_VAR_NAME, - value=self.moment_data.core_t_parallel_fit), - DataProductVariable(HALO_T_PARALLEL_FIT_CDF_VAR_NAME, - value=self.moment_data.halo_t_parallel_fit), - DataProductVariable(CORE_T_PERPENDICULAR_FIT_CDF_VAR_NAME, - value=self.moment_data.core_t_perpendicular_fit), - DataProductVariable(HALO_T_PERPENDICULAR_FIT_CDF_VAR_NAME, - value=self.moment_data.halo_t_perpendicular_fit), - DataProductVariable(CORE_TEMPERATURE_PHI_RTN_FIT_CDF_VAR_NAME, - value=self.moment_data.core_temperature_phi_rtn_fit), - DataProductVariable(HALO_TEMPERATURE_PHI_RTN_FIT_CDF_VAR_NAME, - value=self.moment_data.halo_temperature_phi_rtn_fit), - DataProductVariable(CORE_TEMPERATURE_THETA_RTN_FIT_CDF_VAR_NAME, - value=self.moment_data.core_temperature_theta_rtn_fit), - DataProductVariable(HALO_TEMPERATURE_THETA_RTN_FIT_CDF_VAR_NAME, - value=self.moment_data.halo_temperature_theta_rtn_fit), - DataProductVariable(CORE_SPEED_FIT_CDF_VAR_NAME, - value=self.moment_data.core_speed_fit), - DataProductVariable(HALO_SPEED_FIT_CDF_VAR_NAME, - value=self.moment_data.halo_speed_fit), - DataProductVariable(CORE_VELOCITY_VECTOR_RTN_FIT_CDF_VAR_NAME, - value=self.moment_data.core_velocity_vector_rtn_fit), - DataProductVariable(HALO_VELOCITY_VECTOR_RTN_FIT_CDF_VAR_NAME, - value=self.moment_data.halo_velocity_vector_rtn_fit), - DataProductVariable(CORE_DENSITY_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.core_density_integrated), - DataProductVariable(HALO_DENSITY_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.halo_density_integrated), - DataProductVariable(TOTAL_DENSITY_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.total_density_integrated), - DataProductVariable(CORE_SPEED_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.core_speed_integrated), - DataProductVariable(HALO_SPEED_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.halo_speed_integrated), - DataProductVariable(TOTAL_SPEED_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.total_speed_integrated), - DataProductVariable(CORE_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.core_velocity_vector_rtn_integrated), - DataProductVariable(HALO_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.halo_velocity_vector_rtn_integrated), - DataProductVariable(TOTAL_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.total_velocity_vector_rtn_integrated), - DataProductVariable(CORE_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.core_heat_flux_magnitude_integrated), - DataProductVariable(CORE_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.core_heat_flux_theta_integrated), - DataProductVariable(CORE_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.core_heat_flux_phi_integrated), - DataProductVariable(HALO_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.halo_heat_flux_magnitude_integrated), - DataProductVariable(HALO_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.halo_heat_flux_theta_integrated), - DataProductVariable(HALO_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.halo_heat_flux_phi_integrated), - DataProductVariable(TOTAL_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.total_heat_flux_magnitude_integrated), - DataProductVariable(TOTAL_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.total_heat_flux_theta_integrated), - DataProductVariable(TOTAL_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.total_heat_flux_phi_integrated), - DataProductVariable(CORE_T_PARALLEL_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.core_t_parallel_integrated), - DataProductVariable(CORE_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.core_t_perpendicular_integrated[:, 0]), - DataProductVariable(CORE_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.core_t_perpendicular_integrated[:, 1]), - DataProductVariable(HALO_T_PARALLEL_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.halo_t_parallel_integrated), - DataProductVariable(HALO_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.halo_t_perpendicular_integrated[:, 0]), - DataProductVariable(HALO_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.halo_t_perpendicular_integrated[:, 1]), - DataProductVariable(TOTAL_T_PARALLEL_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.total_t_parallel_integrated), - DataProductVariable(TOTAL_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.total_t_perpendicular_integrated[:, 0]), - DataProductVariable(TOTAL_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.total_t_perpendicular_integrated[:, 1]), - DataProductVariable(CORE_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.core_temperature_theta_rtn_integrated), - DataProductVariable(CORE_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.core_temperature_phi_rtn_integrated), - DataProductVariable(HALO_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.halo_temperature_theta_rtn_integrated), - DataProductVariable(HALO_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.halo_temperature_phi_rtn_integrated), - DataProductVariable(TOTAL_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.total_temperature_theta_rtn_integrated), - DataProductVariable(TOTAL_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.total_temperature_phi_rtn_integrated), - DataProductVariable(CORE_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME, - value=self.moment_data.core_temperature_parallel_to_mag), - DataProductVariable(CORE_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, - value=self.moment_data.core_temperature_perpendicular_to_mag[:, 0]), - DataProductVariable(CORE_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, - value=self.moment_data.core_temperature_perpendicular_to_mag[:, 1]), - DataProductVariable(HALO_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME, - value=self.moment_data.halo_temperature_parallel_to_mag), - DataProductVariable(HALO_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, - value=self.moment_data.halo_temperature_perpendicular_to_mag[:, 0]), - DataProductVariable(HALO_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, - value=self.moment_data.halo_temperature_perpendicular_to_mag[:, 1]), - DataProductVariable(TOTAL_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME, - value=self.moment_data.total_temperature_parallel_to_mag), - DataProductVariable(TOTAL_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, - value=self.moment_data.total_temperature_perpendicular_to_mag[:, 0]), - DataProductVariable(TOTAL_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, - value=self.moment_data.total_temperature_perpendicular_to_mag[:, 1]), - DataProductVariable(CORE_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.core_temperature_tensor_integrated), - DataProductVariable(HALO_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.halo_temperature_tensor_integrated), - DataProductVariable(TOTAL_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME, - value=self.moment_data.total_temperature_tensor_integrated), + DataProductVariable( + GYROPHASE_DELTA_CDF_VAR_NAME, value=self.gyrophase_delta + ), + DataProductVariable( + PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_CDF_VAR_NAME, + value=self.phase_space_density_by_pitch_angle, + ), + DataProductVariable( + PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME, + value=self.phase_space_density_by_pitch_angle_and_gyrophase, + ), + DataProductVariable( + PHASE_SPACE_DENSITY_1D_CDF_VAR_NAME, value=self.phase_space_density_1d + ), + DataProductVariable( + PHASE_SPACE_DENSITY_INWARD_CDF_VAR_NAME, + value=self.phase_space_density_inward, + ), + DataProductVariable( + PHASE_SPACE_DENSITY_OUTWARD_CDF_VAR_NAME, + value=self.phase_space_density_outward, + ), + DataProductVariable( + INTENSITY_BY_PITCH_ANGLE_CDF_VAR_NAME, + value=self.intensity_by_pitch_angle, + ), + DataProductVariable( + INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME, + value=self.intensity_by_pitch_angle_and_gyrophase, + ), + DataProductVariable( + INTENSITY_UNCERTAINTY_BY_PITCH_ANGLE_CDF_VAR_NAME, + value=self.intensity_uncertainty_by_pitch_angle, + ), + DataProductVariable( + INTENSITY_UNCERTAINTY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME, + value=self.intensity_uncertainty_by_pitch_angle_and_gyrophase, + ), + DataProductVariable( + SPACECRAFT_POTENTIAL_CDF_VAR_NAME, value=self.spacecraft_potential + ), + DataProductVariable( + CORE_HALO_BREAKPOINT_CDF_VAR_NAME, value=self.core_halo_breakpoint + ), + DataProductVariable( + CORE_FIT_NUM_POINTS_CDF_VAR_NAME, + value=self.moment_data.core_fit_num_points, + ), + DataProductVariable( + CHISQ_C_CDF_VAR_NAME, value=self.moment_data.core_chisq + ), + DataProductVariable( + CHISQ_H_CDF_VAR_NAME, value=self.moment_data.halo_chisq + ), + DataProductVariable( + CORE_DENSITY_FIT_CDF_VAR_NAME, value=self.moment_data.core_density_fit + ), + DataProductVariable( + HALO_DENSITY_FIT_CDF_VAR_NAME, value=self.moment_data.halo_density_fit + ), + DataProductVariable( + CORE_T_PARALLEL_FIT_CDF_VAR_NAME, + value=self.moment_data.core_t_parallel_fit, + ), + DataProductVariable( + HALO_T_PARALLEL_FIT_CDF_VAR_NAME, + value=self.moment_data.halo_t_parallel_fit, + ), + DataProductVariable( + CORE_T_PERPENDICULAR_FIT_CDF_VAR_NAME, + value=self.moment_data.core_t_perpendicular_fit, + ), + DataProductVariable( + HALO_T_PERPENDICULAR_FIT_CDF_VAR_NAME, + value=self.moment_data.halo_t_perpendicular_fit, + ), + DataProductVariable( + CORE_TEMPERATURE_PHI_RTN_FIT_CDF_VAR_NAME, + value=self.moment_data.core_temperature_phi_rtn_fit, + ), + DataProductVariable( + HALO_TEMPERATURE_PHI_RTN_FIT_CDF_VAR_NAME, + value=self.moment_data.halo_temperature_phi_rtn_fit, + ), + DataProductVariable( + CORE_TEMPERATURE_THETA_RTN_FIT_CDF_VAR_NAME, + value=self.moment_data.core_temperature_theta_rtn_fit, + ), + DataProductVariable( + HALO_TEMPERATURE_THETA_RTN_FIT_CDF_VAR_NAME, + value=self.moment_data.halo_temperature_theta_rtn_fit, + ), + DataProductVariable( + CORE_SPEED_FIT_CDF_VAR_NAME, value=self.moment_data.core_speed_fit + ), + DataProductVariable( + HALO_SPEED_FIT_CDF_VAR_NAME, value=self.moment_data.halo_speed_fit + ), + DataProductVariable( + CORE_VELOCITY_VECTOR_RTN_FIT_CDF_VAR_NAME, + value=self.moment_data.core_velocity_vector_rtn_fit, + ), + DataProductVariable( + HALO_VELOCITY_VECTOR_RTN_FIT_CDF_VAR_NAME, + value=self.moment_data.halo_velocity_vector_rtn_fit, + ), + DataProductVariable( + CORE_DENSITY_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.core_density_integrated, + ), + DataProductVariable( + HALO_DENSITY_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.halo_density_integrated, + ), + DataProductVariable( + TOTAL_DENSITY_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.total_density_integrated, + ), + DataProductVariable( + CORE_SPEED_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.core_speed_integrated, + ), + DataProductVariable( + HALO_SPEED_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.halo_speed_integrated, + ), + DataProductVariable( + TOTAL_SPEED_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.total_speed_integrated, + ), + DataProductVariable( + CORE_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.core_velocity_vector_rtn_integrated, + ), + DataProductVariable( + HALO_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.halo_velocity_vector_rtn_integrated, + ), + DataProductVariable( + TOTAL_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.total_velocity_vector_rtn_integrated, + ), + DataProductVariable( + CORE_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.core_heat_flux_magnitude_integrated, + ), + DataProductVariable( + CORE_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.core_heat_flux_theta_integrated, + ), + DataProductVariable( + CORE_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.core_heat_flux_phi_integrated, + ), + DataProductVariable( + HALO_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.halo_heat_flux_magnitude_integrated, + ), + DataProductVariable( + HALO_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.halo_heat_flux_theta_integrated, + ), + DataProductVariable( + HALO_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.halo_heat_flux_phi_integrated, + ), + DataProductVariable( + TOTAL_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.total_heat_flux_magnitude_integrated, + ), + DataProductVariable( + TOTAL_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.total_heat_flux_theta_integrated, + ), + DataProductVariable( + TOTAL_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.total_heat_flux_phi_integrated, + ), + DataProductVariable( + CORE_T_PARALLEL_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.core_t_parallel_integrated, + ), + DataProductVariable( + CORE_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.core_t_perpendicular_integrated[:, 0], + ), + DataProductVariable( + CORE_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.core_t_perpendicular_integrated[:, 1], + ), + DataProductVariable( + HALO_T_PARALLEL_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.halo_t_parallel_integrated, + ), + DataProductVariable( + HALO_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.halo_t_perpendicular_integrated[:, 0], + ), + DataProductVariable( + HALO_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.halo_t_perpendicular_integrated[:, 1], + ), + DataProductVariable( + TOTAL_T_PARALLEL_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.total_t_parallel_integrated, + ), + DataProductVariable( + TOTAL_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.total_t_perpendicular_integrated[:, 0], + ), + DataProductVariable( + TOTAL_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.total_t_perpendicular_integrated[:, 1], + ), + DataProductVariable( + CORE_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.core_temperature_theta_rtn_integrated, + ), + DataProductVariable( + CORE_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.core_temperature_phi_rtn_integrated, + ), + DataProductVariable( + HALO_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.halo_temperature_theta_rtn_integrated, + ), + DataProductVariable( + HALO_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.halo_temperature_phi_rtn_integrated, + ), + DataProductVariable( + TOTAL_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.total_temperature_theta_rtn_integrated, + ), + DataProductVariable( + TOTAL_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.total_temperature_phi_rtn_integrated, + ), + DataProductVariable( + CORE_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME, + value=self.moment_data.core_temperature_parallel_to_mag, + ), + DataProductVariable( + CORE_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, + value=self.moment_data.core_temperature_perpendicular_to_mag[:, 0], + ), + DataProductVariable( + CORE_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, + value=self.moment_data.core_temperature_perpendicular_to_mag[:, 1], + ), + DataProductVariable( + HALO_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME, + value=self.moment_data.halo_temperature_parallel_to_mag, + ), + DataProductVariable( + HALO_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, + value=self.moment_data.halo_temperature_perpendicular_to_mag[:, 0], + ), + DataProductVariable( + HALO_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, + value=self.moment_data.halo_temperature_perpendicular_to_mag[:, 1], + ), + DataProductVariable( + TOTAL_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME, + value=self.moment_data.total_temperature_parallel_to_mag, + ), + DataProductVariable( + TOTAL_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, + value=self.moment_data.total_temperature_perpendicular_to_mag[:, 0], + ), + DataProductVariable( + TOTAL_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, + value=self.moment_data.total_temperature_perpendicular_to_mag[:, 1], + ), + DataProductVariable( + CORE_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.core_temperature_tensor_integrated, + ), + DataProductVariable( + HALO_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.halo_temperature_tensor_integrated, + ), + DataProductVariable( + TOTAL_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME, + value=self.moment_data.total_temperature_tensor_integrated, + ), DataProductVariable(SWE_FLAGS_VAR_NAME, value=self.swe_flags), DataProductVariable("raw_1d_psd_rebinned", value=self.raw_1d_psd_rebinned), - DataProductVariable("raw_psd_by_theta_rebinned", value=self.raw_psd_by_theta_rebinned), - DataProductVariable("raw_psd_by_phi_rebinned", value=self.raw_psd_by_phi_rebinned), + DataProductVariable( + "raw_psd_by_theta_rebinned", value=self.raw_psd_by_theta_rebinned + ), + DataProductVariable( + "raw_psd_by_phi_rebinned", value=self.raw_psd_by_phi_rebinned + ), DataProductVariable("inst_az", value=self.inst_az), DataProductVariable("inst_el", value=self.inst_el), - DataProductVariable(ENERGY_LABEL, - value=np.char.mod("%.1f eV", self.energy), - ), - DataProductVariable(PITCH_ANGLE_LABEL, - value=np.char.mod("PA %.0f", self.pitch_angle), - ), - DataProductVariable(GYROPHASE_LABEL, - value=np.char.mod("Gyrophase %.0f", self.gyrophase_bins), - ), - DataProductVariable(RTN_LABEL, - value=["R", "T", "N"], - ), - DataProductVariable(CORE_VELOCITY_VECTOR_RTN_FIT_LABEL, - value=["Core v fit R", "Core v fit T", "Core v fit N"], - ), - DataProductVariable(HALO_VELOCITY_VECTOR_RTN_FIT_LABEL, - value=["Halo v fit R", "Halo v fit T", "Halo v fit N"], - ), - DataProductVariable(CORE_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL, - value=["Core v int. R", "Core v int. T", "Core v int. N"], - ), - DataProductVariable(HALO_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL, - value=["Halo v int. R", "Halo v int. T", "Halo v int. N"], - ), - DataProductVariable(TOTAL_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL, - value=["Total v int. R", "Total v int. T", "Total v int. N"], - ), - DataProductVariable(CORE_TEMPERATURE_TENSOR_INTEGRATED_LABEL, - value=['Core T_XX', 'Core T_XY', 'Core T_YY', 'Core T_XZ', 'Core T_YZ', 'Core T_ZZ'], - ), - DataProductVariable(HALO_TEMPERATURE_TENSOR_INTEGRATED_LABEL, - value=['Halo T_XX', 'Halo T_XY', 'Halo T_YY', 'Halo T_XZ', 'Halo T_YZ', 'Halo T_ZZ'], - ), - DataProductVariable(TOTAL_TEMPERATURE_TENSOR_INTEGRATED_LABEL, - value=['Total T_XX', 'Total T_XY', 'Total T_YY', 'Total T_XZ', 'Total T_YZ', 'Total T_ZZ'], - ), + DataProductVariable( + ENERGY_LABEL, + value=np.char.mod("%.1f eV", self.energy), + ), + DataProductVariable( + PITCH_ANGLE_LABEL, + value=np.char.mod("PA %.0f", self.pitch_angle), + ), + DataProductVariable( + GYROPHASE_LABEL, + value=np.char.mod("Gyrophase %.0f", self.gyrophase_bins), + ), + DataProductVariable( + RTN_LABEL, + value=["R", "T", "N"], + ), + DataProductVariable( + CORE_VELOCITY_VECTOR_RTN_FIT_LABEL, + value=["Core v fit R", "Core v fit T", "Core v fit N"], + ), + DataProductVariable( + HALO_VELOCITY_VECTOR_RTN_FIT_LABEL, + value=["Halo v fit R", "Halo v fit T", "Halo v fit N"], + ), + DataProductVariable( + CORE_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL, + value=["Core v int. R", "Core v int. T", "Core v int. N"], + ), + DataProductVariable( + HALO_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL, + value=["Halo v int. R", "Halo v int. T", "Halo v int. N"], + ), + DataProductVariable( + TOTAL_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL, + value=["Total v int. R", "Total v int. T", "Total v int. N"], + ), + DataProductVariable( + CORE_TEMPERATURE_TENSOR_INTEGRATED_LABEL, + value=[ + "Core T_XX", + "Core T_XY", + "Core T_YY", + "Core T_XZ", + "Core T_YZ", + "Core T_ZZ", + ], + ), + DataProductVariable( + HALO_TEMPERATURE_TENSOR_INTEGRATED_LABEL, + value=[ + "Halo T_XX", + "Halo T_XY", + "Halo T_YY", + "Halo T_XZ", + "Halo T_YZ", + "Halo T_ZZ", + ], + ), + DataProductVariable( + TOTAL_TEMPERATURE_TENSOR_INTEGRATED_LABEL, + value=[ + "Total T_XX", + "Total T_XY", + "Total T_YY", + "Total T_XZ", + "Total T_YZ", + "Total T_ZZ", + ], + ), DataProductVariable("inst_az_label", value=self.inst_az_label), DataProductVariable("inst_el_label", value=self.inst_el_label), - DataProductVariable(TENSOR_ID, value=np.array([1, 2, 3, 4, 5, 6])) + DataProductVariable(TENSOR_ID, value=np.array([1, 2, 3, 4, 5, 6])), ] diff --git a/imap_l3_processing/swe/l3/science/moment_calculations.py b/imap_l3_processing/swe/l3/science/moment_calculations.py index 665ff5956..4df23fcb1 100644 --- a/imap_l3_processing/swe/l3/science/moment_calculations.py +++ b/imap_l3_processing/swe/l3/science/moment_calculations.py @@ -7,21 +7,31 @@ import numpy as np import spiceypy -from imap_l3_processing.constants import ELECTRON_MASS_KG, \ - BOLTZMANN_CONSTANT_JOULES_PER_KELVIN, METERS_PER_KILOMETER, \ - CENTIMETERS_PER_METER, PROTON_CHARGE_COULOMBS, GRAMS_PER_KILOGRAM +from imap_l3_processing.constants import ( + ELECTRON_MASS_KG, + BOLTZMANN_CONSTANT_JOULES_PER_KELVIN, + METERS_PER_KILOMETER, + CENTIMETERS_PER_METER, + PROTON_CHARGE_COULOMBS, + GRAMS_PER_KILOGRAM, +) from imap_l3_processing.pitch_angles import calculate_unit_vector from imap_l3_processing.predicted_ephemeris_tracker import PredictedEphemerisTracker -ELECTRON_MASS_OVER_BOLTZMANN_IN_CGS_UNITS = ELECTRON_MASS_KG / BOLTZMANN_CONSTANT_JOULES_PER_KELVIN * 1e-4 +ELECTRON_MASS_OVER_BOLTZMANN_IN_CGS_UNITS = ( + ELECTRON_MASS_KG / BOLTZMANN_CONSTANT_JOULES_PER_KELVIN * 1e-4 +) NUMBER_OF_DETECTORS = 7 NUMBER_OF_SPIN_SECTORS = 30 ZMK = 3.2971e-12 -ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR = np.sqrt(2 * PROTON_CHARGE_COULOMBS / ELECTRON_MASS_KG) * 100 +ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR = ( + np.sqrt(2 * PROTON_CHARGE_COULOMBS / ELECTRON_MASS_KG) * 100 +) logger = logging.getLogger(__name__) + @dataclass class Moments: alpha: float @@ -37,17 +47,18 @@ class Moments: @classmethod def construct_all_fill(cls): - return cls(alpha=np.nan, - beta=np.nan, - t_parallel=np.nan, - t_perpendicular=np.nan, - velocity_x=np.nan, - velocity_y=np.nan, - velocity_z=np.nan, - density=np.nan, - aoo=np.nan, - ao=np.nan, - ) + return cls( + alpha=np.nan, + beta=np.nan, + t_parallel=np.nan, + t_perpendicular=np.nan, + velocity_x=np.nan, + velocity_y=np.nan, + velocity_z=np.nan, + density=np.nan, + aoo=np.nan, + ao=np.nan, + ) @dataclass @@ -82,13 +93,15 @@ class ScaleDensityOutput: cdelt: np.ndarray -def core_fit_moments_retrying_on_failure(corrected_energy_bins: np.ndarray, - velocity_vectors: np.ndarray, - phase_space_density: np.ndarray, - weights: np.ndarray, - energy_start: int, - energy_end: int, - density_history: Union[np.ndarray, list[float]]) -> Optional[MomentFitResults]: +def core_fit_moments_retrying_on_failure( + corrected_energy_bins: np.ndarray, + velocity_vectors: np.ndarray, + phase_space_density: np.ndarray, + weights: np.ndarray, + energy_start: int, + energy_end: int, + density_history: Union[np.ndarray, list[float]], +) -> Optional[MomentFitResults]: return _fit_moments_retrying_on_failure( corrected_energy_bins, velocity_vectors, @@ -97,18 +110,21 @@ def core_fit_moments_retrying_on_failure(corrected_energy_bins: np.ndarray, energy_start, energy_end, density_history, - 1.85 + 1.85, ) -def halo_fit_moments_retrying_on_failure(corrected_energy_bins: np.ndarray, velocity_vectors: np.ndarray, - phase_space_density: np.ndarray, - weights: np.ndarray, - energy_start: int, - energy_end: int, - density_history: Union[np.ndarray, list[float]], - spacecraft_potential: Union[np.float64, float], - core_halo_breakpoint: Union[np.float64, float]) -> Optional[MomentFitResults]: +def halo_fit_moments_retrying_on_failure( + corrected_energy_bins: np.ndarray, + velocity_vectors: np.ndarray, + phase_space_density: np.ndarray, + weights: np.ndarray, + energy_start: int, + energy_end: int, + density_history: Union[np.ndarray, list[float]], + spacecraft_potential: Union[np.float64, float], + core_halo_breakpoint: Union[np.float64, float], +) -> Optional[MomentFitResults]: return _fit_moments_retrying_on_failure( corrected_energy_bins, velocity_vectors, @@ -118,41 +134,53 @@ def halo_fit_moments_retrying_on_failure(corrected_energy_bins: np.ndarray, velo energy_end, density_history, 1.35, - halo_correction_parameters=HaloCorrectionParameters(spacecraft_potential, - core_halo_breakpoint) + halo_correction_parameters=HaloCorrectionParameters( + spacecraft_potential, core_halo_breakpoint + ), ) -def _fit_moments_retrying_on_failure(corrected_energy_bins: np.ndarray, - velocity_vectors: np.ndarray, - phase_space_density: np.ndarray, - weights: np.ndarray, - energy_start: int, - energy_end: int, - density_history: Union[np.ndarray, list[float]], - history_scalar: float, - halo_correction_parameters: Optional[HaloCorrectionParameters] = None) -> Optional[ - MomentFitResults]: - filtered_velocity_vectors, filtered_weights, filtered_yreg = filter_and_flatten_regress_parameters( - corrected_energy_bins, - velocity_vectors, - phase_space_density, - weights, - energy_start, - energy_end) +def _fit_moments_retrying_on_failure( + corrected_energy_bins: np.ndarray, + velocity_vectors: np.ndarray, + phase_space_density: np.ndarray, + weights: np.ndarray, + energy_start: int, + energy_end: int, + density_history: Union[np.ndarray, list[float]], + history_scalar: float, + halo_correction_parameters: Optional[HaloCorrectionParameters] = None, +) -> Optional[MomentFitResults]: + filtered_velocity_vectors, filtered_weights, filtered_yreg = ( + filter_and_flatten_regress_parameters( + corrected_energy_bins, + velocity_vectors, + phase_space_density, + weights, + energy_start, + energy_end, + ) + ) - fit_function, chi_squared = regress(filtered_velocity_vectors, filtered_weights, filtered_yreg) + fit_function, chi_squared = regress( + filtered_velocity_vectors, filtered_weights, filtered_yreg + ) moment = calculate_fit_temperature_density_velocity(fit_function) average_density = np.average(density_history) * history_scalar if halo_correction_parameters is not None: - moment.density = halotrunc(moment, halo_correction_parameters.core_halo_breakpoint, - halo_correction_parameters.spacecraft_potential) + moment.density = halotrunc( + moment, + halo_correction_parameters.core_halo_breakpoint, + halo_correction_parameters.spacecraft_potential, + ) - results = MomentFitResults(moments=moment, chisq=chi_squared, - number_of_points=energy_end - energy_start, - regress_result=fit_function - ) + results = MomentFitResults( + moments=moment, + chisq=chi_squared, + number_of_points=energy_end - energy_start, + regress_result=fit_function, + ) if moment.density is not None and 0 < moment.density < average_density: return results elif energy_end - energy_start < 4: @@ -170,21 +198,22 @@ def _fit_moments_retrying_on_failure(corrected_energy_bins: np.ndarray, energy_end - 1, density_history, history_scalar, - halo_correction_parameters + halo_correction_parameters, ) -def regress(velocity_vectors: np.ndarray, weight: np.ndarray, yreg: np.ndarray) -> \ - np.ndarray: +def regress( + velocity_vectors: np.ndarray, weight: np.ndarray, yreg: np.ndarray +) -> np.ndarray: fit_function = np.zeros((len(velocity_vectors), 9)) velocity_xs = velocity_vectors[:, 0] velocity_ys = velocity_vectors[:, 1] velocity_zs = velocity_vectors[:, 2] - fit_function[:, 0] = -1 * ZMK * (velocity_xs ** 2) - fit_function[:, 1] = -1 * ZMK * (velocity_ys ** 2) - fit_function[:, 2] = -1 * ZMK * (velocity_zs ** 2) + fit_function[:, 0] = -1 * ZMK * (velocity_xs**2) + fit_function[:, 1] = -1 * ZMK * (velocity_ys**2) + fit_function[:, 2] = -1 * ZMK * (velocity_zs**2) fit_function[:, 3] = -1 * ZMK * 2 * velocity_xs * velocity_ys fit_function[:, 4] = -1 * ZMK * 2 * velocity_xs * velocity_zs fit_function[:, 5] = -1 * ZMK * 2 * velocity_ys * velocity_zs @@ -261,17 +290,18 @@ def regress(velocity_vectors: np.ndarray, weight: np.ndarray, yreg: np.ndarray) def calculate_fit_temperature_density_velocity(parameters: np.ndarray[float]): - moments = Moments(alpha=0, - beta=0, - t_parallel=0, - t_perpendicular=0, - velocity_x=0, - velocity_y=0, - velocity_z=0, - density=0, - ao=0, - aoo=0 - ) + moments = Moments( + alpha=0, + beta=0, + t_parallel=0, + t_perpendicular=0, + velocity_x=0, + velocity_y=0, + velocity_z=0, + density=0, + ao=0, + aoo=0, + ) moments.ao = parameters[9] if moments.ao == 0: @@ -283,8 +313,15 @@ def calculate_fit_temperature_density_velocity(parameters: np.ndarray[float]): moments.t_perpendicular = 1.0 / parameters[0] moments.t_parallel = 1.0 / parameters[2] else: - moments.t_perpendicular = 1.0 / (parameters[0] - parameters[3] * parameters[4] / parameters[5]) - moments.t_parallel = 1.0 / (parameters[0] + parameters[1] + parameters[2] - 2.0 / moments.t_perpendicular) + moments.t_perpendicular = 1.0 / ( + parameters[0] - parameters[3] * parameters[4] / parameters[5] + ) + moments.t_parallel = 1.0 / ( + parameters[0] + + parameters[1] + + parameters[2] + - 2.0 / moments.t_perpendicular + ) c11 = parameters[1] * parameters[2] - parameters[5] * parameters[5] c12 = parameters[5] * parameters[4] - parameters[3] * parameters[2] @@ -300,16 +337,28 @@ def calculate_fit_temperature_density_velocity(parameters: np.ndarray[float]): moments.velocity_y = 0 moments.velocity_z = 0 else: - moments.velocity_x = -1 * (parameters[6] * c11 + parameters[7] * c12 + parameters[8] * c13) / d - moments.velocity_y = -1 * (parameters[6] * c12 + parameters[7] * c22 + parameters[8] * c23) / d - moments.velocity_z = -1 * (parameters[6] * c13 + parameters[7] * c23 + parameters[8] * c33) / d + moments.velocity_x = ( + -1 * (parameters[6] * c11 + parameters[7] * c12 + parameters[8] * c13) / d + ) + moments.velocity_y = ( + -1 * (parameters[6] * c12 + parameters[7] * c22 + parameters[8] * c23) / d + ) + moments.velocity_z = ( + -1 * (parameters[6] * c13 + parameters[7] * c23 + parameters[8] * c33) / d + ) - fact = parameters[0] * moments.velocity_x ** 2 + parameters[1] * moments.velocity_y ** 2 + parameters[ - 2] * moments.velocity_z ** 2 + fact = ( + parameters[0] * moments.velocity_x**2 + + parameters[1] * moments.velocity_y**2 + + parameters[2] * moments.velocity_z**2 + ) - moments.aoo = fact + parameters[3] * moments.velocity_x * moments.velocity_y + parameters[4] \ - * moments.velocity_x * moments.velocity_z + \ - parameters[5] * moments.velocity_y * moments.velocity_z + moments.aoo = ( + fact + + parameters[3] * moments.velocity_x * moments.velocity_y + + parameters[4] * moments.velocity_x * moments.velocity_z + + parameters[5] * moments.velocity_y * moments.velocity_z + ) moments.velocity_x *= 1e-5 moments.velocity_y *= 1e-5 @@ -318,20 +367,25 @@ def calculate_fit_temperature_density_velocity(parameters: np.ndarray[float]): if (moments.t_parallel < 0) or (moments.aoo < 0) or (moments.aoo > 1e13): moments.density = 0.0 else: - moments.density = (moments.t_perpendicular * np.sqrt(moments.t_parallel) / - ((ZMK / math.pi) * np.sqrt(ZMK / math.pi))) * np.exp( - moments.ao + ZMK * moments.aoo) + moments.density = ( + moments.t_perpendicular + * np.sqrt(moments.t_parallel) + / ((ZMK / math.pi) * np.sqrt(ZMK / math.pi)) + ) * np.exp(moments.ao + ZMK * moments.aoo) return moments LIMIT = np.array([32, 64, 128, 256, 1024, 2048, 3072, 5120, 9216, 17408, 33792]) -SIGMA2 = np.array([0, 0.25, 1.25, 5.25, 21.25, 85.25, 341.25, 1365.25, 5461.25, 21845.25, 87381.25]) +SIGMA2 = np.array( + [0, 0.25, 1.25, 5.25, 21.25, 85.25, 341.25, 1365.25, 5461.25, 21845.25, 87381.25] +) MINIMUM_WEIGHT = 0.8165 MAX_VARIANCE = 349525.25 -def compute_maxwellian_weight_factors(count_rates: np.ndarray, acquisition_durations: np.ndarray) -> \ - np.ndarray[float]: +def compute_maxwellian_weight_factors( + count_rates: np.ndarray, acquisition_durations: np.ndarray +) -> np.ndarray[float]: correction = 1.0 - 1e-9 * LIMIT / acquisition_durations[:, :, np.newaxis] correction[correction < 0.1] = 0.1 xlimits_per_measurement = LIMIT / correction @@ -349,18 +403,21 @@ def compute_maxwellian_weight_factors(count_rates: np.ndarray, acquisition_durat if corrected_count < xlimit: variance = sigma break - weights[energy_i, spin_i, declination_i] = np.sqrt(variance + corrected_count) / corrected_count + weights[energy_i, spin_i, declination_i] = ( + np.sqrt(variance + corrected_count) / corrected_count + ) return weights -def filter_and_flatten_regress_parameters(corrected_energy_bins: np.ndarray, - velocity_vectors: np.ndarray, - phase_space_density: np.ndarray, - weights: np.ndarray, - start_index: int, - end_index: int) -> tuple[ - np.ndarray, np.ndarray, np.ndarray]: +def filter_and_flatten_regress_parameters( + corrected_energy_bins: np.ndarray, + velocity_vectors: np.ndarray, + phase_space_density: np.ndarray, + weights: np.ndarray, + start_index: int, + end_index: int, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: valid_mask = np.full_like(phase_space_density, fill_value=False, dtype=bool) valid_mask[start_index:end_index] = True valid_mask[corrected_energy_bins <= 0] = False @@ -369,7 +426,8 @@ def filter_and_flatten_regress_parameters(corrected_energy_bins: np.ndarray, filtered_phase_space_density = phase_space_density[valid_mask] yreg = np.zeros_like(filtered_phase_space_density) yreg[filtered_phase_space_density > 1e-35] = np.log( - filtered_phase_space_density[filtered_phase_space_density > 1e-35]) + filtered_phase_space_density[filtered_phase_space_density > 1e-35] + ) yreg[filtered_phase_space_density <= 1e-35] = -80.6 return velocity_vectors[valid_mask], weights[valid_mask], yreg @@ -379,18 +437,24 @@ def get_dps_to_rtn_rotation_matrix(epoch: datetime) -> np.ndarray: return spiceypy.pxform("IMAP_DPS", "IMAP_RTN", spiceypy.datetime2et(epoch)) -def apply_rotation_matrix(rotation_matrix: np.ndarray, vector: np.ndarray) -> np.ndarray: +def apply_rotation_matrix( + rotation_matrix: np.ndarray, vector: np.ndarray +) -> np.ndarray: return rotation_matrix @ vector -def rotate_rtn_vectors_to_dps(epochs: np.ndarray, vectors_rtn: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: +def rotate_rtn_vectors_to_dps( + epochs: np.ndarray, vectors_rtn: np.ndarray +) -> Tuple[np.ndarray, np.ndarray]: result = np.full_like(vectors_rtn, np.nan, dtype=float) predicted_ephemeris_flags = np.zeros_like(epochs, dtype=bool) for i, epoch in enumerate(epochs): tracker = PredictedEphemerisTracker() try: et_time = spiceypy.datetime2et(epoch) - rotation_matrix = tracker.run(spiceypy.pxform,"IMAP_RTN", "IMAP_DPS", et_time) + rotation_matrix = tracker.run( + spiceypy.pxform, "IMAP_RTN", "IMAP_DPS", et_time + ) except spiceypy.SpiceyError as e: logger.info(f"Failed to rotate RTN→DPS at epoch {epoch}: {e}") continue @@ -400,7 +464,9 @@ def rotate_rtn_vectors_to_dps(epochs: np.ndarray, vectors_rtn: np.ndarray) -> Tu return result, predicted_ephemeris_flags -def rotate_temperature(rotation_matrix: np.ndarray, alpha: float, beta: float) -> tuple[float, float]: +def rotate_temperature( + rotation_matrix: np.ndarray, alpha: float, beta: float +) -> tuple[float, float]: sin_dec = np.sin(beta) x = sin_dec * np.cos(alpha) y = sin_dec * np.sin(alpha) @@ -414,8 +480,9 @@ def rotate_temperature(rotation_matrix: np.ndarray, alpha: float, beta: float) - return theta, phi -def rotate_vector_to_rtn_spherical_coordinates(rotation_matrix: np.ndarray, heat_flux: np.ndarray) -> tuple[ - float, float, float]: +def rotate_vector_to_rtn_spherical_coordinates( + rotation_matrix: np.ndarray, heat_flux: np.ndarray +) -> tuple[float, float, float]: r, t, n = apply_rotation_matrix(rotation_matrix, heat_flux) magnitude = np.linalg.norm(heat_flux, axis=-1) rt = np.sqrt(r * r + t * t) @@ -424,7 +491,9 @@ def rotate_vector_to_rtn_spherical_coordinates(rotation_matrix: np.ndarray, heat return magnitude, theta, phi -def compute_density_scale(core_electron_energy_range: float, speed: float, temperature: float) -> float: +def compute_density_scale( + core_electron_energy_range: float, speed: float, temperature: float +) -> float: energy_multplier = 11600.0 density_multiplier = 0.88623 ev_speed = 593.097 @@ -446,17 +515,25 @@ def compute_density_scale(core_electron_energy_range: float, speed: float, tempe escalep = math.erfc(scalep) if scalep < 9 else 0 escalem = math.erfc(scalem) if scalem < 9 else 0 - dscalep = density_multiplier / (scalep * np.exp(-scalep2) + density_multiplier * escalep) - dscalem = density_multiplier / (scalem * np.exp(-scalem2) + density_multiplier * escalem) + dscalep = density_multiplier / ( + scalep * np.exp(-scalep2) + density_multiplier * escalep + ) + dscalem = density_multiplier / ( + scalem * np.exp(-scalem2) + density_multiplier * escalem + ) return 0.5 * (dscalep + dscalem) -def halotrunc(moments: Moments, core_halo_breakpoint: float, spacecraft_potential: float) -> Optional[np.float64]: +def halotrunc( + moments: Moments, core_halo_breakpoint: float, spacecraft_potential: float +) -> Optional[np.float64]: htmax = moments.t_parallel htmin = moments.t_perpendicular - speed = math.sqrt(moments.velocity_x ** 2 + moments.velocity_y ** 2 + moments.velocity_z ** 2) + speed = math.sqrt( + moments.velocity_x**2 + moments.velocity_y**2 + moments.velocity_z**2 + ) halod = moments.density @@ -468,7 +545,9 @@ def halotrunc(moments: Moments, core_halo_breakpoint: float, spacecraft_potentia if htmax <= 1e4 or htmin <= 1e4: dscale = 1 elif core_halo_breakpoint - spacecraft_potential > 5 and temp < 1.0e7: - dscale = compute_density_scale(core_halo_breakpoint - spacecraft_potential, speed, temp) + dscale = compute_density_scale( + core_halo_breakpoint - spacecraft_potential, speed, temp + ) else: dscale = 1 @@ -477,10 +556,19 @@ def halotrunc(moments: Moments, core_halo_breakpoint: float, spacecraft_potentia return halod -def integrate(istart, iend, energy: np.ndarray, sintheta: np.ndarray, - costheta: np.ndarray, deltheta: np.ndarray, fv: np.ndarray, phi: np.ndarray, - spacecraft_potential: float, cdelnv: np.ndarray, cdelt: np.ndarray) -> Optional[ - IntegrateOutputs]: +def integrate( + istart, + iend, + energy: np.ndarray, + sintheta: np.ndarray, + costheta: np.ndarray, + deltheta: np.ndarray, + fv: np.ndarray, + phi: np.ndarray, + spacecraft_potential: float, + cdelnv: np.ndarray, + cdelt: np.ndarray, +) -> Optional[IntegrateOutputs]: sumn = 0 sumvx = 0 sumvy = 0 @@ -491,17 +579,24 @@ def integrate(istart, iend, energy: np.ndarray, sintheta: np.ndarray, fv_with_axis_order_energy_cem_spin = np.moveaxis(fv, 1, 2) for i in range(istart, iend + 1): if energy[i] > 0: - v2mid = (ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR ** 2) * energy[i] + v2mid = (ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR**2) * energy[i] v3mid = v2mid * np.sqrt(v2mid) else: v2mid = 0 v3mid = 0 ehigh = 1.175 * energy[i] if i < len(energy) - 1: - ehigh = np.sqrt((energy[i + 1] + spacecraft_potential) * (energy[i] + spacecraft_potential)) - elow = np.sqrt((energy[i] + spacecraft_potential) * (energy[i - 1] + spacecraft_potential)) + ehigh = np.sqrt( + (energy[i + 1] + spacecraft_potential) + * (energy[i] + spacecraft_potential) + ) + elow = np.sqrt( + (energy[i] + spacecraft_potential) * (energy[i - 1] + spacecraft_potential) + ) - delv[i] = ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR * (np.sqrt(ehigh) - np.sqrt(elow)) + delv[i] = ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR * ( + np.sqrt(ehigh) - np.sqrt(elow) + ) if elow < base: base = elow for j in range(7): @@ -510,8 +605,20 @@ def integrate(istart, iend, energy: np.ndarray, sintheta: np.ndarray, delta = delv[i] * deltheta[j] * delphi[j] fact = sintheta[j] * fv_with_axis_order_energy_cem_spin[i, j, k] sumn += delta * v2mid * fact - sumvx += delta * v3mid * fact * sintheta[j] * np.cos(np.deg2rad(phi[i, k])) - sumvy += delta * v3mid * fact * sintheta[j] * np.sin(np.deg2rad(phi[i, k])) + sumvx += ( + delta + * v3mid + * fact + * sintheta[j] + * np.cos(np.deg2rad(phi[i, k])) + ) + sumvy += ( + delta + * v3mid + * fact + * sintheta[j] + * np.sin(np.deg2rad(phi[i, k])) + ) sumvz += delta * v3mid * fact * costheta[j] totden = sumn + cdelnv[0] @@ -521,11 +628,13 @@ def integrate(istart, iend, energy: np.ndarray, sintheta: np.ndarray, base -= spacecraft_potential CM_PER_KM = METERS_PER_KILOMETER * CENTIMETERS_PER_METER KM_PER_CM = 1 / CM_PER_KM - output_velocities = np.array([ - (-KM_PER_CM * sumvx + cdelnv[1]) / totden, - (-KM_PER_CM * sumvy + cdelnv[2]) / totden, - (-KM_PER_CM * sumvz + cdelnv[3]) / totden, - ]) + output_velocities = np.array( + [ + (-KM_PER_CM * sumvx + cdelnv[1]) / totden, + (-KM_PER_CM * sumvy + cdelnv[2]) / totden, + (-KM_PER_CM * sumvz + cdelnv[3]) / totden, + ] + ) sumtxx = 0 sumtxy = 0 @@ -538,25 +647,41 @@ def integrate(istart, iend, energy: np.ndarray, sintheta: np.ndarray, sumqz = 0 for i in range(istart, iend + 1): for j in range(7): - v2mid = (ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR ** 2) * energy[i] + v2mid = (ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR**2) * energy[i] for k in range(28): # again why 28? if energy[i] > 0: angx = sintheta[j] * np.cos(np.deg2rad(phi[i, k])) - vx = -ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR * np.sqrt(energy[i]) * angx - CM_PER_KM * \ - output_velocities[0] + vx = ( + -ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR + * np.sqrt(energy[i]) + * angx + - CM_PER_KM * output_velocities[0] + ) angy = sintheta[j] * np.sin(np.deg2rad(phi[i, k])) - vy = -ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR * np.sqrt(energy[i]) * angy - CM_PER_KM * \ - output_velocities[1] + vy = ( + -ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR + * np.sqrt(energy[i]) + * angy + - CM_PER_KM * output_velocities[1] + ) angz = costheta[j] - vz = -ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR * np.sqrt(energy[i]) * angz - CM_PER_KM * \ - output_velocities[2] + vz = ( + -ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR + * np.sqrt(energy[i]) + * angz + - CM_PER_KM * output_velocities[2] + ) vmag2 = vx * vx + vy * vy + vz * vz delta = delv[i] * deltheta[j] * delphi[j] - fact = v2mid * sintheta[j] * fv_with_axis_order_energy_cem_spin[i, j, k] + fact = ( + v2mid + * sintheta[j] + * fv_with_axis_order_energy_cem_spin[i, j, k] + ) sumtxx += delta * fact * vx * vx sumtxy += delta * fact * vx * vy sumtxz += delta * fact * vx * vz @@ -568,22 +693,34 @@ def integrate(istart, iend, energy: np.ndarray, sintheta: np.ndarray, sumqz += delta * fact * vmag2 * vz TEMPERATURE_SCALING_FACTOR_TO_UNDO_IN_EIGEN = 1e-4 - temperature = (np.array([sumtxx, sumtxy, sumtyy, sumtxz, sumtyz, sumtzz]) * - TEMPERATURE_SCALING_FACTOR_TO_UNDO_IN_EIGEN * ELECTRON_MASS_OVER_BOLTZMANN_IN_CGS_UNITS + cdelt) / totden - - heat_flux = np.array([sumqx, sumqy, sumqz]) * 500 * ELECTRON_MASS_KG * GRAMS_PER_KILOGRAM + temperature = ( + np.array([sumtxx, sumtxy, sumtyy, sumtxz, sumtyz, sumtzz]) + * TEMPERATURE_SCALING_FACTOR_TO_UNDO_IN_EIGEN + * ELECTRON_MASS_OVER_BOLTZMANN_IN_CGS_UNITS + + cdelt + ) / totden + + heat_flux = ( + np.array([sumqx, sumqy, sumqz]) * 500 * ELECTRON_MASS_KG * GRAMS_PER_KILOGRAM + ) return IntegrateOutputs(totden, output_velocities, temperature, heat_flux, base) -def scale_core_density(core_density: float, - core_velocity: np.ndarray, core_temp: np.ndarray, - core_moment_fit: Moments, ifit: int, energy: np.ndarray, - spacecraft_potential: float, cosin_p: np.ndarray, - aperture_field_of_view: list, - phi: np.ndarray, - regress_outputs: np.ndarray, - base_energy: float) -> ScaleDensityOutput: +def scale_core_density( + core_density: float, + core_velocity: np.ndarray, + core_temp: np.ndarray, + core_moment_fit: Moments, + ifit: int, + energy: np.ndarray, + spacecraft_potential: float, + cosin_p: np.ndarray, + aperture_field_of_view: list, + phi: np.ndarray, + regress_outputs: np.ndarray, + base_energy: float, +) -> ScaleDensityOutput: zmk = ELECTRON_MASS_KG / (2 * BOLTZMANN_CONSTANT_JOULES_PER_KELVIN * 1e4) MAX_SPIN_SECTOR_INDEX = 28 # why 28 and not 30? @@ -592,21 +729,37 @@ def scale_core_density(core_density: float, delv = np.zeros((2, NUMBER_OF_DETECTORS)) velocity_in_sc_frame = np.zeros((2, NUMBER_OF_DETECTORS)) for j in range(NUMBER_OF_DETECTORS): - ehigh = np.sqrt((energy[ifit + 1] + spacecraft_potential) * - (energy[ifit] + spacecraft_potential)) - elow = np.sqrt((energy[ifit] + spacecraft_potential) * - (energy[ifit - 1] + spacecraft_potential)) - delv[0, j] = ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR * (np.sqrt(ehigh) - np.sqrt(elow)) - velocity_in_sc_frame[0, j] = ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR * np.sqrt(energy[ifit]) + ehigh = np.sqrt( + (energy[ifit + 1] + spacecraft_potential) + * (energy[ifit] + spacecraft_potential) + ) + elow = np.sqrt( + (energy[ifit] + spacecraft_potential) + * (energy[ifit - 1] + spacecraft_potential) + ) + delv[0, j] = ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR * ( + np.sqrt(ehigh) - np.sqrt(elow) + ) + velocity_in_sc_frame[0, j] = ( + ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR * np.sqrt(energy[ifit]) + ) base_energy = min(base_energy, elow) base_energy -= spacecraft_potential if base_energy > 0: number_of_energies = 2 - velocity_in_sc_frame[1, :] = ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR * np.sqrt(0.5 * base_energy) - delv[1, :] = ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR * np.sqrt(base_energy) + velocity_in_sc_frame[1, :] = ( + ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR * np.sqrt(0.5 * base_energy) + ) + delv[1, :] = ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR * np.sqrt( + base_energy + ) - factor = core_moment_fit.density * zmk / (np.pi * core_moment_fit.t_perpendicular) * np.sqrt( - zmk / (np.pi * core_moment_fit.t_parallel)) + factor = ( + core_moment_fit.density + * zmk + / (np.pi * core_moment_fit.t_perpendicular) + * np.sqrt(zmk / (np.pi * core_moment_fit.t_parallel)) + ) cos_theta = cosin_p[np.newaxis, :, np.newaxis] sin_theta = np.sin(np.arccos(cos_theta)) @@ -620,14 +773,22 @@ def scale_core_density(core_density: float, vy = -velocity_3_dim * sin_theta * sin_phi vz = -velocity_3_dim * cos_theta - fun = -zmk * np.stack(np.broadcast_arrays(vx * vx, vy * vy, vz * vz, vx * vy, vx * vz, vy * vz, vx, vy, vz), - axis=-1) + fun = -zmk * np.stack( + np.broadcast_arrays( + vx * vx, vy * vy, vz * vz, vx * vy, vx * vz, vy * vz, vx, vy, vz + ), + axis=-1, + ) exponent = -zmk * core_moment_fit.aoo + np.dot(fun, regress_outputs[:9]) delt = delv[:, :, np.newaxis] * delta_theta * delta_phi - common_factor = np.square(velocity_3_dim) * delt * sin_theta * factor * np.exp(exponent) + common_factor = ( + np.square(velocity_3_dim) * delt * sin_theta * factor * np.exp(exponent) + ) def integrate(array): - return np.sum(array[:number_of_energies, :NUMBER_OF_DETECTORS, :MAX_SPIN_SECTOR_INDEX]) + return np.sum( + array[:number_of_energies, :NUMBER_OF_DETECTORS, :MAX_SPIN_SECTOR_INDEX] + ) sumint = integrate(common_factor) sumvx = integrate(vx * common_factor) @@ -660,30 +821,44 @@ def integrate(array): cdelnv = np.append(sumint, delta_v) - return ScaleDensityOutput(density=corrected_density, velocity=corrected_core_velocity, - temperature=corrected_core_temp, cdelnv=cdelnv, cdelt=cdelt) + return ScaleDensityOutput( + density=corrected_density, + velocity=corrected_core_velocity, + temperature=corrected_core_temp, + cdelnv=cdelnv, + cdelt=cdelt, + ) -def scale_halo_density(halo_density: float, - halo_velocity: np.ndarray, halo_temp: np.ndarray, - halo_moment_fit: Moments, - spacecraft_potential: float, - core_halo_break: float, - cosin_p: np.ndarray, - aperture_field_of_view: list, - phi: np.ndarray, - regress_outputs: np.ndarray, - base_energy: float) -> ScaleDensityOutput: +def scale_halo_density( + halo_density: float, + halo_velocity: np.ndarray, + halo_temp: np.ndarray, + halo_moment_fit: Moments, + spacecraft_potential: float, + core_halo_break: float, + cosin_p: np.ndarray, + aperture_field_of_view: list, + phi: np.ndarray, + regress_outputs: np.ndarray, + base_energy: float, +) -> ScaleDensityOutput: zmk = ELECTRON_MASS_KG / (2 * BOLTZMANN_CONSTANT_JOULES_PER_KELVIN * 1e4) MAX_SPIN_SECTOR_INDEX = 28 # why 28 and not 30? hchbreak = core_halo_break - spacecraft_potential scval = abs(hchbreak - base_energy) min_energy = min(base_energy, hchbreak) - vsch = ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR * np.sqrt(min_energy + 0.5 * scval) + vsch = ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR * np.sqrt( + min_energy + 0.5 * scval + ) deltav = ENERGY_EV_TO_SPEED_CM_PER_S_CONVERSION_FACTOR * np.sqrt(scval) - factor = halo_moment_fit.density * zmk / (np.pi * halo_moment_fit.t_parallel) * np.sqrt( - zmk / (np.pi * halo_moment_fit.t_perpendicular)) + factor = ( + halo_moment_fit.density + * zmk + / (np.pi * halo_moment_fit.t_parallel) + * np.sqrt(zmk / (np.pi * halo_moment_fit.t_perpendicular)) + ) cos_theta = cosin_p[:, np.newaxis] sin_theta = np.sin(np.arccos(cos_theta)) @@ -696,8 +871,12 @@ def scale_halo_density(halo_density: float, vy = -vsch * sin_theta * sin_phi vz = -vsch * cos_theta - fun = -zmk * np.stack(np.broadcast_arrays(vx * vx, vy * vy, vz * vz, vx * vy, vx * vz, vy * vz, vx, vy, vz), - axis=-1) + fun = -zmk * np.stack( + np.broadcast_arrays( + vx * vx, vy * vy, vz * vz, vx * vy, vx * vz, vy * vz, vx, vy, vz + ), + axis=-1, + ) exponent = np.dot(fun, regress_outputs[:9]) delt = deltav * delta_theta * delta_phi common_factor = np.square(vsch) * delt * sin_theta * factor * np.exp(exponent) @@ -737,20 +916,38 @@ def integrate(array): corrected_halo_temp = halo_temp.copy() corrected_halo_temp *= halo_density / corrected_density - corrected_halo_temp += delta_temperature * 1e-4 * ELECTRON_MASS_OVER_BOLTZMANN_IN_CGS_UNITS / corrected_density + corrected_halo_temp += ( + delta_temperature + * 1e-4 + * ELECTRON_MASS_OVER_BOLTZMANN_IN_CGS_UNITS + / corrected_density + ) - return ScaleDensityOutput(density=corrected_density, velocity=corrected_halo_velocity, - temperature=corrected_halo_temp, cdelnv=None, cdelt=None) + return ScaleDensityOutput( + density=corrected_density, + velocity=corrected_halo_velocity, + temperature=corrected_halo_temp, + cdelnv=None, + cdelt=None, + ) -def calculate_primary_eigenvector(temperature_tensor: np.ndarray) -> tuple[np.ndarray, np.ndarray]: +def calculate_primary_eigenvector( + temperature_tensor: np.ndarray, +) -> tuple[np.ndarray, np.ndarray]: symmetric_temperature_tensor = np.zeros(shape=(3, 3)) symmetric_temperature_tensor[0][0] = temperature_tensor[0] - symmetric_temperature_tensor[0][1] = symmetric_temperature_tensor[1][0] = temperature_tensor[1] + symmetric_temperature_tensor[0][1] = symmetric_temperature_tensor[1][0] = ( + temperature_tensor[1] + ) symmetric_temperature_tensor[1][1] = temperature_tensor[2] - symmetric_temperature_tensor[0][2] = symmetric_temperature_tensor[2][0] = temperature_tensor[3] - symmetric_temperature_tensor[1][2] = symmetric_temperature_tensor[2][1] = temperature_tensor[4] + symmetric_temperature_tensor[0][2] = symmetric_temperature_tensor[2][0] = ( + temperature_tensor[3] + ) + symmetric_temperature_tensor[1][2] = symmetric_temperature_tensor[2][1] = ( + temperature_tensor[4] + ) symmetric_temperature_tensor[2][2] = temperature_tensor[5] nan_array = np.full(3, np.nan) @@ -759,7 +956,11 @@ def calculate_primary_eigenvector(temperature_tensor: np.ndarray) -> tuple[np.nd except np.linalg.LinAlgError: return nan_array, nan_array - if np.any(eigen_values < 0) or np.all(eigen_values == 0) or np.any(np.isnan(eigen_values)): + if ( + np.any(eigen_values < 0) + or np.all(eigen_values == 0) + or np.any(np.isnan(eigen_values)) + ): return nan_array, nan_array eigen_values_mean = np.mean(eigen_values) @@ -777,12 +978,16 @@ def calculate_primary_eigenvector(temperature_tensor: np.ndarray) -> tuple[np.nd other_evals = [v for i, v in enumerate(eigen_values) if i != ipar] max_eval = max(other_evals) min_eval = min(other_evals) - perpendicular_temperature = TEMPERATURE_SCALING_FACTOR * np.sqrt(min_eval * max_eval) + perpendicular_temperature = TEMPERATURE_SCALING_FACTOR * np.sqrt( + min_eval * max_eval + ) if min_eval == 0: gyro = 1 else: gyro = max(other_evals) / min_eval - return primary_evec, np.array([parallel_temperature, perpendicular_temperature, gyro]) + return primary_evec, np.array( + [parallel_temperature, perpendicular_temperature, gyro] + ) def rotation_matrix_builder(mag_vector: np.ndarray) -> np.ndarray(shape=(3, 3)): @@ -795,15 +1000,22 @@ def rotation_matrix_builder(mag_vector: np.ndarray) -> np.ndarray(shape=(3, 3)): return np.stack((normalized_mag_vector, row_1, row_2), axis=0) -def rotate_temperature_tensor_to_mag(temperature_tensor: np.ndarray, mag_vector: np.ndarray) -> tuple[ - float, float, float]: +def rotate_temperature_tensor_to_mag( + temperature_tensor: np.ndarray, mag_vector: np.ndarray +) -> tuple[float, float, float]: symmetric_temperature_tensor = np.zeros(shape=(3, 3)) symmetric_temperature_tensor[0][0] = temperature_tensor[0] - symmetric_temperature_tensor[0][1] = symmetric_temperature_tensor[1][0] = temperature_tensor[1] + symmetric_temperature_tensor[0][1] = symmetric_temperature_tensor[1][0] = ( + temperature_tensor[1] + ) symmetric_temperature_tensor[1][1] = temperature_tensor[2] - symmetric_temperature_tensor[0][2] = symmetric_temperature_tensor[2][0] = temperature_tensor[3] - symmetric_temperature_tensor[1][2] = symmetric_temperature_tensor[2][1] = temperature_tensor[4] + symmetric_temperature_tensor[0][2] = symmetric_temperature_tensor[2][0] = ( + temperature_tensor[3] + ) + symmetric_temperature_tensor[1][2] = symmetric_temperature_tensor[2][1] = ( + temperature_tensor[4] + ) symmetric_temperature_tensor[2][2] = temperature_tensor[5] rotation_matrix = rotation_matrix_builder(mag_vector) diff --git a/imap_l3_processing/swe/l3/science/pitch_calculations.py b/imap_l3_processing/swe/l3/science/pitch_calculations.py index 5fd820ccd..86ed34434 100644 --- a/imap_l3_processing/swe/l3/science/pitch_calculations.py +++ b/imap_l3_processing/swe/l3/science/pitch_calculations.py @@ -5,12 +5,23 @@ import numpy as np import scipy -from imap_l3_processing.constants import ELECTRON_MASS_KG, PROTON_CHARGE_COULOMBS, METERS_PER_KILOMETER -from imap_l3_processing.pitch_angles import calculate_pitch_angle, calculate_unit_vector, calculate_gyrophase +from imap_l3_processing.constants import ( + ELECTRON_MASS_KG, + PROTON_CHARGE_COULOMBS, + METERS_PER_KILOMETER, +) +from imap_l3_processing.pitch_angles import ( + calculate_pitch_angle, + calculate_unit_vector, + calculate_gyrophase, +) from imap_l3_processing.swe.l3.models import SweConfiguration from imap_l3_processing.swe.quality_flags import SweL3Flags -def mec_breakpoint_finder(energies: np.ndarray, averaged_psd: np.ndarray) -> tuple[float, float, SweL3Flags]: + +def mec_breakpoint_finder( + energies: np.ndarray, averaged_psd: np.ndarray +) -> tuple[float, float, SweL3Flags]: """ Input: energies - energy bins @@ -25,14 +36,16 @@ def mec_breakpoint_finder(energies: np.ndarray, averaged_psd: np.ndarray) -> tup # If True, then potential is likely less than 2.7 V (lower than first energy bin) def line_model(params, x): return params[0] + params[1] * x + from scipy import odr + odr_model = odr.Model(line_model) x = log_energy[:4] y = log_psd[:4] - mydata = odr.RealData(x=x,y=y) - myodr = odr.ODR(mydata, odr_model, beta0=[y.max(),-5]) + mydata = odr.RealData(x=x, y=y) + myodr = odr.ODR(mydata, odr_model, beta0=[y.max(), -5]) myodr.set_job(fit_type=2) - myoutput= myodr.run() + myoutput = myodr.run() if myoutput.res_var <= 0.01: FALLBACK_POTENTIAL_ESTIMATE = SweL3Flags.FALLBACK_POTENTIAL_ESTIMATE return_value = 2.5 @@ -43,9 +56,15 @@ def line_model(params, x): # Mirror real point as fake point to left of first energy bin to improve spline concavity ewidth = np.nanmean(log_energy[1:] - log_energy[:-1]) from scipy.interpolate import UnivariateSpline as uspline - spline = uspline(np.concatenate([[log_energy[0]-ewidth],log_energy]), - np.concatenate([[log_psd[2]],log_psd]), s=.25) - spline_energies = np.geomspace(energies.min()*np.exp(-ewidth), energies.max(), 100) + + spline = uspline( + np.concatenate([[log_energy[0] - ewidth], log_energy]), + np.concatenate([[log_psd[2]], log_psd]), + s=0.25, + ) + spline_energies = np.geomspace( + energies.min() * np.exp(-ewidth), energies.max(), 100 + ) spline_derivative = spline.derivative(2) curvature = spline_derivative(np.log(spline_energies)) try: @@ -64,8 +83,11 @@ def piece_wise_model_mec(x, b0, b1, b2, b3): Modified Piecewise to fit Potential and Core-Halo Break separately The breakpoint is b2 """ - return np.piecewise(x, [x<=b2, x>b2], - [lambda x: b0 - b1*x, lambda x: b0 + b2*(b3-b1) - b3*x]) + return np.piecewise( + x, + [x <= b2, x > b2], + [lambda x: b0 - b1 * x, lambda x: b0 + b2 * (b3 - b1) - b3 * x], + ) def refine_breakpoint_value(energy, psd, breakpoint_value, num_points): """ @@ -73,8 +95,12 @@ def refine_breakpoint_value(energy, psd, breakpoint_value, num_points): energy at which the lines intersect is the refined_breakpoint """ # Find Nearest energy bin to breakpoint_value - nearest_energy_idx = np.argmin(np.abs(energy-breakpoint_value)) - if np.abs(breakpoint_value - energy[nearest_energy_idx])/energy[nearest_energy_idx] <= .075: + nearest_energy_idx = np.argmin(np.abs(energy - breakpoint_value)) + if ( + np.abs(breakpoint_value - energy[nearest_energy_idx]) + / energy[nearest_energy_idx] + <= 0.075 + ): # Breakpoint_value was within FWHM of nearest energy bin # return that energy bin as refined_breakpoint return energy[nearest_energy_idx] @@ -90,7 +116,7 @@ def refine_breakpoint_value(energy, psd, breakpoint_value, num_points): z_left = np.polyfit(e_left[-num_points:], psd_left[-num_points:], 1) z_right = np.polyfit(e_right[:num_points], psd_right[:num_points], 1) # Determine energy of their intersection - refined_breakpoint = (z_left[1]-z_right[1]) / (z_right[0]-z_left[0]) + refined_breakpoint = (z_left[1] - z_right[1]) / (z_right[0] - z_left[0]) if (refined_breakpoint < z_left[-1]) | (refined_breakpoint > z_right[0]): # Refined breakpoint lies outside of expected range return breakpoint_value @@ -100,19 +126,27 @@ def refine_breakpoint_value(energy, psd, breakpoint_value, num_points): mask_sc = log_energy <= np.log(30) mask_ch = (log_energy > np.log(30)) & (log_energy < np.log(400)) - log_energy_sc = log_energy[mask_sc]; log_psd_sc = log_psd[mask_sc] - log_energy_ch = log_energy[mask_ch]; log_psd_ch = log_psd[mask_ch] + log_energy_sc = log_energy[mask_sc] + log_psd_sc = log_psd[mask_sc] + log_energy_ch = log_energy[mask_ch] + log_psd_ch = log_psd[mask_ch] fitting_model = piece_wise_model_mec # Try Spacecraft Potential Fit POTENTIAL_FIT_UNCONVERGED = SweL3Flags.NONE if FALLBACK_POTENTIAL_ESTIMATE == SweL3Flags.NONE: try: - initial_guess = [log_psd[0],1,7,1] - z, cov = scipy.optimize.curve_fit(fitting_model, np.exp(log_energy_sc), log_psd_sc, p0=initial_guess) + initial_guess = [log_psd[0], 1, 7, 1] + z, cov = scipy.optimize.curve_fit( + fitting_model, np.exp(log_energy_sc), log_psd_sc, p0=initial_guess + ) # Make sure the fit converged - if ((z[0] == initial_guess[0]) | (z[1] == initial_guess[1]) - | (z[2] == initial_guess[2]) | (z[3] == initial_guess[3])): + if ( + (z[0] == initial_guess[0]) + | (z[1] == initial_guess[1]) + | (z[2] == initial_guess[2]) + | (z[3] == initial_guess[3]) + ): # Fall back on Spline method # Fit did not converge POTENTIAL_FIT_UNCONVERGED = SweL3Flags.POTENTIAL_FIT_UNCONVERGED @@ -121,7 +155,9 @@ def refine_breakpoint_value(energy, psd, breakpoint_value, num_points): # Fit worked # Check whether breakpoint has two points to left and right # If so, then find intersection of linear fits to each side - sc_pot_output = refine_breakpoint_value(np.exp(log_energy_sc), log_psd_sc, z[2], 2) + sc_pot_output = refine_breakpoint_value( + np.exp(log_energy_sc), log_psd_sc, z[2], 2 + ) # spline not used BACKUP_SPLINE_UNRESOLVED = SweL3Flags.NONE except: @@ -134,11 +170,17 @@ def refine_breakpoint_value(energy, psd, breakpoint_value, num_points): # Try Core-Halo Breakpoint Fit BREAKPOINT_FIT_UNCONVERGED = SweL3Flags.NONE try: - initial_guess = [log_psd_ch[0],1,65,1] - z, cov = scipy.optimize.curve_fit(fitting_model, np.exp(log_energy_ch), log_psd_ch, p0=initial_guess) + initial_guess = [log_psd_ch[0], 1, 65, 1] + z, cov = scipy.optimize.curve_fit( + fitting_model, np.exp(log_energy_ch), log_psd_ch, p0=initial_guess + ) # Make sure the fit converged - if ((z[0] == initial_guess[0]) & (z[1] == initial_guess[1]) - & (z[2] == initial_guess[2]) & (z[3] == initial_guess[3])): + if ( + (z[0] == initial_guess[0]) + & (z[1] == initial_guess[1]) + & (z[2] == initial_guess[2]) + & (z[3] == initial_guess[3]) + ): # Fall back on Spline method # Fit did not converge BREAKPOINT_FIT_UNCONVERGED = SweL3Flags.BREAKPOINT_FIT_UNCONVERGED @@ -147,19 +189,33 @@ def refine_breakpoint_value(energy, psd, breakpoint_value, num_points): # Fit worked # Check whether breakpoint has three points to left and right # If so, then find intersection of linear fits to each side - ch_break_output = refine_breakpoint_value(np.exp(log_energy_ch), log_psd_ch, z[2], 3) + ch_break_output = refine_breakpoint_value( + np.exp(log_energy_ch), log_psd_ch, z[2], 3 + ) except: # Fall back on Spline method # Fit did not converge BREAKPOINT_FIT_UNCONVERGED = SweL3Flags.BREAKPOINT_FIT_UNCONVERGED ch_break_output = ch_break - return sc_pot_output, ch_break_output, FALLBACK_POTENTIAL_ESTIMATE | BACKUP_SPLINE_UNRESOLVED | POTENTIAL_FIT_UNCONVERGED | BREAKPOINT_FIT_UNCONVERGED - - -def average_over_look_directions(phase_space_density: np.ndarray, geometric_weights: np.ndarray, - minimum_psd_value: float) -> np.ndarray: + return ( + sc_pot_output, + ch_break_output, + FALLBACK_POTENTIAL_ESTIMATE + | BACKUP_SPLINE_UNRESOLVED + | POTENTIAL_FIT_UNCONVERGED + | BREAKPOINT_FIT_UNCONVERGED, + ) + + +def average_over_look_directions( + phase_space_density: np.ndarray, + geometric_weights: np.ndarray, + minimum_psd_value: float, +) -> np.ndarray: enforced_minimum_psd = np.maximum(phase_space_density, minimum_psd_value) - weighted_average = np.sum(enforced_minimum_psd * geometric_weights, axis=-1) / np.sum(geometric_weights) + weighted_average = np.sum( + enforced_minimum_psd * geometric_weights, axis=-1 + ) / np.sum(geometric_weights) return np.mean(weighted_average, axis=1) @@ -173,9 +229,13 @@ def calculate_look_directions(inst_el: np.ndarray, inst_az: np.ndarray) -> np.nd return np.stack(np.broadcast_arrays(x, y, z), axis=-1) -def calculate_velocity_in_dsp_frame_km_s(energy: np.ndarray, inst_el: np.ndarray, inst_az: np.ndarray) -> np.ndarray: - particle_direction = - calculate_look_directions(inst_el, inst_az) - speed_meters_per_second = np.sqrt(energy * PROTON_CHARGE_COULOMBS * 2 / ELECTRON_MASS_KG) +def calculate_velocity_in_dsp_frame_km_s( + energy: np.ndarray, inst_el: np.ndarray, inst_az: np.ndarray +) -> np.ndarray: + particle_direction = -calculate_look_directions(inst_el, inst_az) + speed_meters_per_second = np.sqrt( + energy * PROTON_CHARGE_COULOMBS * 2 / ELECTRON_MASS_KG + ) speed_km_per_second = speed_meters_per_second / METERS_PER_KILOMETER for i in range(len(particle_direction)): @@ -184,13 +244,19 @@ def calculate_velocity_in_dsp_frame_km_s(energy: np.ndarray, inst_el: np.ndarray return particle_direction -def rebin_by_pitch_angle(flux, pitch_angles, energies, config: SweConfiguration) -> np.ndarray: - all_the_bins = rebin_by_pitch_angle_and_gyrophase(flux, pitch_angles, None, energies, config) +def rebin_by_pitch_angle( + flux, pitch_angles, energies, config: SweConfiguration +) -> np.ndarray: + all_the_bins = rebin_by_pitch_angle_and_gyrophase( + flux, pitch_angles, None, energies, config + ) remove_gyrophase_axis = all_the_bins[:, :, 0] return remove_gyrophase_axis -def _rebin(flux, pitch_angles, energies, config: SweConfiguration, gyrophase: np.ndarray = None) -> np.ndarray: +def _rebin( + flux, pitch_angles, energies, config: SweConfiguration, gyrophase: np.ndarray = None +) -> np.ndarray: pitch_angle_bins = np.array(config["pitch_angle_bins"]) pitch_angle_delta = np.array(config["pitch_angle_deltas"]) energy_bins = config["energy_bins"] @@ -215,20 +281,28 @@ def _rebin(flux, pitch_angles, energies, config: SweConfiguration, gyrophase: np gyrophase_bins_range = [0] num_gyrophase_bins = 1 - rebinned = np.full((num_energy_bins, num_pitch_bins, num_gyrophase_bins), np.nan, dtype=float) + rebinned = np.full( + (num_energy_bins, num_pitch_bins, num_gyrophase_bins), np.nan, dtype=float + ) for g in gyrophase_bins_range: for j in range(num_pitch_bins): - mask_pitch_angle = (pitch_angles_for_masked_psd >= pitch_angle_left_edges[j]) & ( - pitch_angles_for_masked_psd < pitch_angle_right_edges[j]) + mask_pitch_angle = ( + pitch_angles_for_masked_psd >= pitch_angle_left_edges[j] + ) & (pitch_angles_for_masked_psd < pitch_angle_right_edges[j]) if gyrophase is not None: - mask_gyrophase = (gyrophase_for_masked_psd >= gyrophase_left_edges[g]) & ( - gyrophase_for_masked_psd < gyrophase_right_edges[g]) + mask_gyrophase = ( + gyrophase_for_masked_psd >= gyrophase_left_edges[g] + ) & (gyrophase_for_masked_psd < gyrophase_right_edges[g]) else: mask_gyrophase = np.ones_like(mask_pitch_angle, dtype=bool) - psd_by_pitch_angle = psd_greater_than_zero[mask_pitch_angle & mask_gyrophase] - energy_by_pitch_angle = energies_for_masked_psd[mask_pitch_angle & mask_gyrophase] + psd_by_pitch_angle = psd_greater_than_zero[ + mask_pitch_angle & mask_gyrophase + ] + energy_by_pitch_angle = energies_for_masked_psd[ + mask_pitch_angle & mask_gyrophase + ] # maintain in new method if len(energy_by_pitch_angle) < 2: @@ -242,7 +316,9 @@ def _rebin(flux, pitch_angles, energies, config: SweConfiguration, gyrophase: np left = config["energy_bin_low_multiplier"] * center right = config["energy_bin_high_multiplier"] * center - mask_energy = (energy_by_pitch_angle > left) & (energy_by_pitch_angle < right) + mask_energy = (energy_by_pitch_angle > left) & ( + energy_by_pitch_angle < right + ) energy_to_fit = energy_by_pitch_angle[mask_energy] psd_to_fit = psd_by_pitch_angle[mask_energy] @@ -253,23 +329,41 @@ def _rebin(flux, pitch_angles, energies, config: SweConfiguration, gyrophase: np max_within_window = np.max(energy_to_fit) min_within_window = np.min(energy_to_fit) - has_points_on_both_sides = max_within_window > center and min_within_window < center - is_overall_max_within_window = overall_max_energy_for_pitch_angle in energy_to_fit - max_within_case = is_overall_max_within_window and closest_energy_ratio < config[ - "high_energy_proximity_threshold"] - - is_overall_second_min_within_window = overall_second_min_energy_for_pitch_angle in energy_to_fit - - mins_within_case = is_overall_second_min_within_window and (closest_energy_ratio < config[ - "low_energy_proximity_threshold"] or has_points_on_both_sides) - max_and_mins_outside_window_case = (not is_overall_max_within_window - and not is_overall_second_min_within_window - and has_points_on_both_sides) - - if max_within_case or mins_within_case or max_and_mins_outside_window_case: + has_points_on_both_sides = ( + max_within_window > center and min_within_window < center + ) + is_overall_max_within_window = ( + overall_max_energy_for_pitch_angle in energy_to_fit + ) + max_within_case = ( + is_overall_max_within_window + and closest_energy_ratio < config["high_energy_proximity_threshold"] + ) + + is_overall_second_min_within_window = ( + overall_second_min_energy_for_pitch_angle in energy_to_fit + ) + + mins_within_case = is_overall_second_min_within_window and ( + closest_energy_ratio < config["low_energy_proximity_threshold"] + or has_points_on_both_sides + ) + max_and_mins_outside_window_case = ( + not is_overall_max_within_window + and not is_overall_second_min_within_window + and has_points_on_both_sides + ) + + if ( + max_within_case + or mins_within_case + or max_and_mins_outside_window_case + ): log_energy_to_fit = np.log(energy_to_fit) log_psd_to_fit = np.log(psd_to_fit) - intercept, slope = np.polynomial.polynomial.polyfit(log_energy_to_fit, log_psd_to_fit, 1) + intercept, slope = np.polynomial.polynomial.polyfit( + log_energy_to_fit, log_psd_to_fit, 1 + ) log_psd_to_nom = slope * np.log(center) + intercept value = np.exp(log_psd_to_nom) @@ -278,21 +372,27 @@ def _rebin(flux, pitch_angles, energies, config: SweConfiguration, gyrophase: np return rebinned -def rebin_by_pitch_angle_and_gyrophase(psd, pitch_angles, gyrophase, energies, config: SweConfiguration) -> np.ndarray: +def rebin_by_pitch_angle_and_gyrophase( + psd, pitch_angles, gyrophase, energies, config: SweConfiguration +) -> np.ndarray: return _rebin(psd, pitch_angles, energies, config, gyrophase=gyrophase) -def calculate_velocity_in_sw_frame(velocity_in_despun_frame: np.ndarray[(..., 3)], - solar_wind_velocity: np.ndarray[(3,)]) -> np.ndarray[(..., 3)]: +def calculate_velocity_in_sw_frame( + velocity_in_despun_frame: np.ndarray[(..., 3)], + solar_wind_velocity: np.ndarray[(3,)], +) -> np.ndarray[(..., 3)]: return velocity_in_despun_frame - solar_wind_velocity -def calculate_energy_in_ev_from_velocity_in_km_per_second(velocity: np.ndarray[(..., 3)]): +def calculate_energy_in_ev_from_velocity_in_km_per_second( + velocity: np.ndarray[(..., 3)], +): joule_to_ev = 1 / PROTON_CHARGE_COULOMBS velocity_m_s = velocity * 1e3 - speed_squared = np.sum(velocity_m_s ** 2, axis=-1) + speed_squared = np.sum(velocity_m_s**2, axis=-1) energy_joules = 0.5 * ELECTRON_MASS_KG * speed_squared return energy_joules * joule_to_ev @@ -305,42 +405,84 @@ def calculate_energy_in_ev_from_velocity_in_km_per_second(velocity: np.ndarray[( GYROPHASE_BINS = TypeVar("GYROPHASE_BINS") -def correct_and_rebin(flux_or_psd: np.ndarray[(E_BINS, SPIN_SECTORS, CEMS)], - solar_wind_vector: np.ndarray[(3,)], - dsp_velocities: np.ndarray[(E_BINS, SPIN_SECTORS, CEMS, 3,)], - mag_vector: np.ndarray[(E_BINS, SPIN_SECTORS, 3,)], - config: SweConfiguration) -> tuple[np.ndarray, np.ndarray]: - velocity_in_sw_frame = calculate_velocity_in_sw_frame(dsp_velocities, solar_wind_vector) - energy_in_sw_frame = calculate_energy_in_ev_from_velocity_in_km_per_second(velocity_in_sw_frame) - pitch_angle = calculate_pitch_angle(velocity_in_sw_frame, mag_vector[..., np.newaxis, :]) - gyrophase = calculate_gyrophase(velocity_in_sw_frame, mag_vector[..., np.newaxis, :]) - - rebinned_by_pa = rebin_by_pitch_angle(flux_or_psd, pitch_angle, energy_in_sw_frame, config) - rebinned_by_pa_and_gyro = rebin_by_pitch_angle_and_gyrophase(flux_or_psd, pitch_angle, gyrophase, - energy_in_sw_frame, config) +def correct_and_rebin( + flux_or_psd: np.ndarray[(E_BINS, SPIN_SECTORS, CEMS)], + solar_wind_vector: np.ndarray[(3,)], + dsp_velocities: np.ndarray[ + ( + E_BINS, + SPIN_SECTORS, + CEMS, + 3, + ) + ], + mag_vector: np.ndarray[ + ( + E_BINS, + SPIN_SECTORS, + 3, + ) + ], + config: SweConfiguration, +) -> tuple[np.ndarray, np.ndarray]: + velocity_in_sw_frame = calculate_velocity_in_sw_frame( + dsp_velocities, solar_wind_vector + ) + energy_in_sw_frame = calculate_energy_in_ev_from_velocity_in_km_per_second( + velocity_in_sw_frame + ) + pitch_angle = calculate_pitch_angle( + velocity_in_sw_frame, mag_vector[..., np.newaxis, :] + ) + gyrophase = calculate_gyrophase( + velocity_in_sw_frame, mag_vector[..., np.newaxis, :] + ) + + rebinned_by_pa = rebin_by_pitch_angle( + flux_or_psd, pitch_angle, energy_in_sw_frame, config + ) + rebinned_by_pa_and_gyro = rebin_by_pitch_angle_and_gyrophase( + flux_or_psd, pitch_angle, gyrophase, energy_in_sw_frame, config + ) return rebinned_by_pa, rebinned_by_pa_and_gyro -def integrate_distribution_to_get_1d_spectrum(psd_by_energy_and_pitch_angle: np.ndarray[(E_BINS, PITCH_ANGLE_BINS)], - config: SweConfiguration) -> np.ndarray[E_BINS]: - pitch_angle_bin_factors = np.sin(np.deg2rad(config["pitch_angle_bins"])) * 2 * np.deg2rad( - config["pitch_angle_deltas"]) / 2 +def integrate_distribution_to_get_1d_spectrum( + psd_by_energy_and_pitch_angle: np.ndarray[(E_BINS, PITCH_ANGLE_BINS)], + config: SweConfiguration, +) -> np.ndarray[E_BINS]: + pitch_angle_bin_factors = ( + np.sin(np.deg2rad(config["pitch_angle_bins"])) + * 2 + * np.deg2rad(config["pitch_angle_deltas"]) + / 2 + ) return np.nansum(psd_by_energy_and_pitch_angle * pitch_angle_bin_factors, axis=1) def integrate_distribution_to_get_inbound_and_outbound_1d_spectrum( - psd_by_pitch_angle: np.ndarray[(E_BINS, PITCH_ANGLE_BINS)], - config: SweConfiguration) -> (np.ndarray[E_BINS], np.ndarray[E_BINS]): - pitch_angle_bin_factors = np.sin(np.deg2rad(config["pitch_angle_bins"])) * 2 * np.deg2rad( - config["pitch_angle_deltas"]) + psd_by_pitch_angle: np.ndarray[(E_BINS, PITCH_ANGLE_BINS)], config: SweConfiguration +) -> (np.ndarray[E_BINS], np.ndarray[E_BINS]): + pitch_angle_bin_factors = ( + np.sin(np.deg2rad(config["pitch_angle_bins"])) + * 2 + * np.deg2rad(config["pitch_angle_deltas"]) + ) pitch_less_than_90 = np.array(config["pitch_angle_bins"]) < 90 - spectrum_a = np.nansum((psd_by_pitch_angle * pitch_angle_bin_factors)[:, pitch_less_than_90], axis=1) - spectrum_b = np.nansum((psd_by_pitch_angle * pitch_angle_bin_factors)[:, ~pitch_less_than_90], axis=1) - - if spectrum_a[config["in_vs_out_energy_index"]] > spectrum_b[config["in_vs_out_energy_index"]]: + spectrum_a = np.nansum( + (psd_by_pitch_angle * pitch_angle_bin_factors)[:, pitch_less_than_90], axis=1 + ) + spectrum_b = np.nansum( + (psd_by_pitch_angle * pitch_angle_bin_factors)[:, ~pitch_less_than_90], axis=1 + ) + + if ( + spectrum_a[config["in_vs_out_energy_index"]] + > spectrum_b[config["in_vs_out_energy_index"]] + ): inbound = spectrum_b outbound = spectrum_a else: @@ -350,20 +492,40 @@ def integrate_distribution_to_get_inbound_and_outbound_1d_spectrum( return inbound, outbound -def swe_rebin_intensity_by_pitch_angle_and_gyrophase(intensity_data: np.ndarray[(E_BINS, SPIN_SECTORS, CEMS)], - counts: np.ndarray[(E_BINS, SPIN_SECTORS, CEMS)], - dsp_velocities: np.ndarray[(E_BINS, SPIN_SECTORS, CEMS, 3)], - mag_vectors: np.ndarray[([(E_BINS, SPIN_SECTORS, 3,)])], - config: SweConfiguration) -> [np.ndarray]: +def swe_rebin_intensity_by_pitch_angle_and_gyrophase( + intensity_data: np.ndarray[(E_BINS, SPIN_SECTORS, CEMS)], + counts: np.ndarray[(E_BINS, SPIN_SECTORS, CEMS)], + dsp_velocities: np.ndarray[(E_BINS, SPIN_SECTORS, CEMS, 3)], + mag_vectors: np.ndarray[ + ( + [ + ( + E_BINS, + SPIN_SECTORS, + 3, + ) + ] + ) + ], + config: SweConfiguration, +) -> [np.ndarray]: normalized_velocities = calculate_unit_vector(dsp_velocities) normalized_mag_vectors = calculate_unit_vector(mag_vectors) - pitch_angles = calculate_pitch_angle(normalized_velocities, normalized_mag_vectors[..., np.newaxis, :]) - gyrophases = calculate_gyrophase(normalized_velocities, normalized_mag_vectors[..., np.newaxis, :]) + pitch_angles = calculate_pitch_angle( + normalized_velocities, normalized_mag_vectors[..., np.newaxis, :] + ) + gyrophases = calculate_gyrophase( + normalized_velocities, normalized_mag_vectors[..., np.newaxis, :] + ) num_pitch_angle_bins = len(config["pitch_angle_bins"]) num_gyrophase_bins = len(config["gyrophase_bins"]) - output_shape_pa_and_gyro = (intensity_data.shape[0], num_pitch_angle_bins, num_gyrophase_bins) + output_shape_pa_and_gyro = ( + intensity_data.shape[0], + num_pitch_angle_bins, + num_gyrophase_bins, + ) output_shape_pa_only = (intensity_data.shape[0], num_pitch_angle_bins) rebinned_summed_by_pa_and_gyro = np.zeros(shape=output_shape_pa_and_gyro) @@ -377,52 +539,82 @@ def swe_rebin_intensity_by_pitch_angle_and_gyrophase(intensity_data: np.ndarray[ for i in range(intensity_data.shape[0]): for pitch_angle, gyrophase, intensity, count in zip( - np.ravel(pitch_angles[i]), - np.ravel(gyrophases[i]), - np.ravel(intensity_data[i]), - np.ravel(counts[i])): + np.ravel(pitch_angles[i]), + np.ravel(gyrophases[i]), + np.ravel(intensity_data[i]), + np.ravel(counts[i]), + ): if not (np.isnan(intensity) or np.isnan(pitch_angle)): - pitch_angle_bin = next((i for i, (center, delta) in - enumerate(zip(config["pitch_angle_bins"], config["pitch_angle_deltas"])) if - center - delta <= pitch_angle < center + delta), - num_pitch_angle_bins - 1) + pitch_angle_bin = next( + ( + i + for i, (center, delta) in enumerate( + zip( + config["pitch_angle_bins"], config["pitch_angle_deltas"] + ) + ) + if center - delta <= pitch_angle < center + delta + ), + num_pitch_angle_bins - 1, + ) if not np.isnan(gyrophase): - gyrophase_bin = next((i for i, (center, delta) in - enumerate(zip(config["gyrophase_bins"], config["gyrophase_deltas"])) if - center - delta <= gyrophase < center + delta), - num_gyrophase_bins - 1) - - rebinned_summed_by_pa_and_gyro[i, pitch_angle_bin, gyrophase_bin] += intensity - rebinned_count_by_pa_and_gyro[i, pitch_angle_bin, gyrophase_bin] += 1 - - rebinned_summed_counts_by_pa_and_gyro[i, pitch_angle_bin, gyrophase_bin] += count + gyrophase_bin = next( + ( + i + for i, (center, delta) in enumerate( + zip( + config["gyrophase_bins"], config["gyrophase_deltas"] + ) + ) + if center - delta <= gyrophase < center + delta + ), + num_gyrophase_bins - 1, + ) + + rebinned_summed_by_pa_and_gyro[ + i, pitch_angle_bin, gyrophase_bin + ] += intensity + rebinned_count_by_pa_and_gyro[ + i, pitch_angle_bin, gyrophase_bin + ] += 1 + + rebinned_summed_counts_by_pa_and_gyro[ + i, pitch_angle_bin, gyrophase_bin + ] += count rebinned_summed_pa_only[i, pitch_angle_bin] += intensity rebinned_count_pa_only[i, pitch_angle_bin] += 1 rebinned_summed_counts_by_pa_only[i, pitch_angle_bin] += count - rebinned_summed_counts_by_pa_and_gyro[rebinned_summed_counts_by_pa_and_gyro == 0] = np.nan + rebinned_summed_counts_by_pa_and_gyro[ + rebinned_summed_counts_by_pa_and_gyro == 0 + ] = np.nan rebinned_summed_counts_by_pa_only[rebinned_summed_counts_by_pa_only == 0] = np.nan averaged_rebinned_intensity_by_pa_and_gyro = np.divide( rebinned_summed_by_pa_and_gyro, rebinned_count_by_pa_and_gyro, - out=np.full_like( - rebinned_count_by_pa_and_gyro, np.nan), - where=rebinned_count_by_pa_and_gyro != 0) + out=np.full_like(rebinned_count_by_pa_and_gyro, np.nan), + where=rebinned_count_by_pa_and_gyro != 0, + ) - averaged_rebinned_intensity_by_pa_only = np.divide(rebinned_summed_pa_only, - rebinned_count_pa_only, - out=np.full_like( - rebinned_count_pa_only, - np.nan), - where=rebinned_count_pa_only != 0) + averaged_rebinned_intensity_by_pa_only = np.divide( + rebinned_summed_pa_only, + rebinned_count_pa_only, + out=np.full_like(rebinned_count_pa_only, np.nan), + where=rebinned_count_pa_only != 0, + ) uncertainties_by_pa_and_gyro = averaged_rebinned_intensity_by_pa_and_gyro / np.sqrt( - rebinned_summed_counts_by_pa_and_gyro) - uncertainties_by_pa_only = averaged_rebinned_intensity_by_pa_only / np.sqrt(rebinned_summed_counts_by_pa_only) + rebinned_summed_counts_by_pa_and_gyro + ) + uncertainties_by_pa_only = averaged_rebinned_intensity_by_pa_only / np.sqrt( + rebinned_summed_counts_by_pa_only + ) return ( - averaged_rebinned_intensity_by_pa_and_gyro, averaged_rebinned_intensity_by_pa_only, + averaged_rebinned_intensity_by_pa_and_gyro, + averaged_rebinned_intensity_by_pa_only, uncertainties_by_pa_and_gyro, - uncertainties_by_pa_only) + uncertainties_by_pa_only, + ) diff --git a/imap_l3_processing/swe/l3/swe_l3_dependencies.py b/imap_l3_processing/swe/l3/swe_l3_dependencies.py index 535744bfe..81a4bd3db 100644 --- a/imap_l3_processing/swe/l3/swe_l3_dependencies.py +++ b/imap_l3_processing/swe/l3/swe_l3_dependencies.py @@ -7,9 +7,18 @@ from imap_data_access.processing_input import ProcessingInputCollection from imap_l3_processing.models import MagData -from imap_l3_processing.swe.l3.models import SweL2Data, SweConfiguration, SwapiL3aProtonData, SweL1bData -from imap_l3_processing.swe.l3.utils import read_l2_swe_data, read_l3a_swapi_proton_data, read_swe_config, \ - read_l1b_swe_data +from imap_l3_processing.swe.l3.models import ( + SweL2Data, + SweConfiguration, + SwapiL3aProtonData, + SweL1bData, +) +from imap_l3_processing.swe.l3.utils import ( + read_l2_swe_data, + read_l3a_swapi_proton_data, + read_swe_config, + read_l1b_swe_data, +) from imap_l3_processing.utils import read_mag_data, select_mag_path MAG_DESPUN_L1D_DESCRIPTOR = "norm-dsrf" @@ -27,18 +36,28 @@ class SweL3Dependencies: mag_is_preliminary: bool = False @classmethod - def fetch_dependencies(cls, dependencies: ProcessingInputCollection) -> SweL3Dependencies: + def fetch_dependencies( + cls, dependencies: ProcessingInputCollection + ) -> SweL3Dependencies: science_files = dependencies.processing_input - swe_config_dependency = dependencies.get_file_paths(source='swe', descriptor=SWE_CONFIG_DESCRIPTOR)[0] + swe_config_dependency = dependencies.get_file_paths( + source="swe", descriptor=SWE_CONFIG_DESCRIPTOR + )[0] try: swe_l2_dependency = next( - d.imap_file_paths[0] for d in science_files if d.source == "swe" and d.data_type == "l2") + d.imap_file_paths[0] + for d in science_files + if d.source == "swe" and d.data_type == "l2" + ) except StopIteration: raise ValueError("Missing SWE l2 dependency.") try: swe_l1b_dependency = next( - d.imap_file_paths[0] for d in science_files if d.source == "swe" and d.data_type == "l1b") + d.imap_file_paths[0] + for d in science_files + if d.source == "swe" and d.data_type == "l1b" + ) except StopIteration: raise ValueError("Missing SWE l1b dependency.") mag_file, mag_level = select_mag_path(dependencies, MAG_DESPUN_L1D_DESCRIPTOR) @@ -47,8 +66,10 @@ def fetch_dependencies(cls, dependencies: ProcessingInputCollection) -> SweL3Dep mag_is_preliminary = mag_level == "l1d" try: swapi_dependency = next( - d.imap_file_paths[0] for d in science_files if d.source == "swapi" - and d.descriptor == SWAPI_L3A_PROTON_DESCRIPTOR) + d.imap_file_paths[0] + for d in science_files + if d.source == "swapi" and d.descriptor == SWAPI_L3A_PROTON_DESCRIPTOR + ) except StopIteration: raise ValueError(f"Missing SWAPI {SWAPI_L3A_PROTON_DESCRIPTOR} dependency.") @@ -57,19 +78,36 @@ def fetch_dependencies(cls, dependencies: ProcessingInputCollection) -> SweL3Dep swapi_file = download(swapi_dependency.construct_path()) swe_config = download(swe_config_dependency) - return cls.from_file_paths(swe_l2_file, swe_l1b_file, mag_file, swapi_file, swe_config, - mag_is_preliminary=mag_is_preliminary) + return cls.from_file_paths( + swe_l2_file, + swe_l1b_file, + mag_file, + swapi_file, + swe_config, + mag_is_preliminary=mag_is_preliminary, + ) @classmethod - def from_file_paths(cls, swe_l2_file_path: Path, swe_l1b_file_path: Path, mag_file_path: Path, - swapi_file_path: Path, - configuration_file_path: Path, - mag_is_preliminary: bool = False) -> SweL3Dependencies: + def from_file_paths( + cls, + swe_l2_file_path: Path, + swe_l1b_file_path: Path, + mag_file_path: Path, + swapi_file_path: Path, + configuration_file_path: Path, + mag_is_preliminary: bool = False, + ) -> SweL3Dependencies: mag_l1d_data = read_mag_data(mag_file_path) swe_l1b_data = read_l1b_swe_data(swe_l1b_file_path) swe_l2_data = read_l2_swe_data(swe_l2_file_path) swapi_l3a_proton_data = read_l3a_swapi_proton_data(swapi_file_path) configuration = read_swe_config(configuration_file_path) - return cls(swe_l2_data, swe_l1b_data, mag_l1d_data, swapi_l3a_proton_data, configuration, - mag_is_preliminary=mag_is_preliminary) + return cls( + swe_l2_data, + swe_l1b_data, + mag_l1d_data, + swapi_l3a_proton_data, + configuration, + mag_is_preliminary=mag_is_preliminary, + ) diff --git a/imap_l3_processing/swe/l3/utils.py b/imap_l3_processing/swe/l3/utils.py index c52b4e0c6..45e686952 100644 --- a/imap_l3_processing/swe/l3/utils.py +++ b/imap_l3_processing/swe/l3/utils.py @@ -7,7 +7,12 @@ from spacepy.pycdf import CDF from imap_l3_processing.cdf.cdf_utils import read_numeric_variable -from imap_l3_processing.swe.l3.models import SweConfiguration, SweL2Data, SwapiL3aProtonData, SweL1bData +from imap_l3_processing.swe.l3.models import ( + SweConfiguration, + SweL2Data, + SwapiL3aProtonData, + SweL1bData, +) def read_l2_swe_data(swe_l2_data: Path) -> SweL2Data: @@ -20,11 +25,17 @@ def read_l2_swe_data(swe_l2_data: Path) -> SweL2Data: inst_az_label = cdf["inst_az_label"][:] energy = cdf["energy"][:] inst_az_spin_sector = read_numeric_variable(cdf["inst_az_spin_sector"]) - phase_space_density = read_numeric_variable(cdf["phase_space_density_spin_sector"]) + phase_space_density = read_numeric_variable( + cdf["phase_space_density_spin_sector"] + ) acquisition_time_in_MET = read_numeric_variable(cdf["acquisition_time"]) valid_times_mask = np.isfinite(acquisition_time_in_MET) - converted_valid_times = met_to_datetime64(acquisition_time_in_MET[valid_times_mask].ravel()) - acquisition_time_dt64 = np.full(acquisition_time_in_MET.shape, np.datetime64("NaT", 'ns')) + converted_valid_times = met_to_datetime64( + acquisition_time_in_MET[valid_times_mask].ravel() + ) + acquisition_time_dt64 = np.full( + acquisition_time_in_MET.shape, np.datetime64("NaT", "ns") + ) acquisition_time_dt64[valid_times_mask] = converted_valid_times acquisition_duration = cdf["acq_duration"][...] phase_space_density_rebinned = read_numeric_variable(cdf["phase_space_density"]) @@ -52,7 +63,9 @@ def read_l1b_swe_data(swe_l1b_data: Path) -> SweL1bData: count_rates = cdf["science_data"][...] settle_duration = cdf["settle_duration"][...] - return SweL1bData(epoch=epoch, count_rates=count_rates, settle_duration=settle_duration) + return SweL1bData( + epoch=epoch, count_rates=count_rates, settle_duration=settle_duration + ) def read_l3a_swapi_proton_data(swapi_l3a_data: Path) -> SwapiL3aProtonData: @@ -65,16 +78,20 @@ def read_l3a_swapi_proton_data(swapi_l3a_data: Path) -> SwapiL3aProtonData: epoch_delta = np.repeat(timedelta(seconds=epoch_delta_ns), len(epoch)) proton_sw_speed = read_numeric_variable(cdf["proton_sw_speed"]) if "proton_sw_velocity_rtn" in cdf: - proton_sw_velocity_rtn = read_numeric_variable(cdf["proton_sw_velocity_rtn"]) + proton_sw_velocity_rtn = read_numeric_variable( + cdf["proton_sw_velocity_rtn"] + ) else: proton_sw_velocity_rtn = np.full((len(epoch), 3), np.nan) swp_flags = cdf["swp_flags"][...] - return SwapiL3aProtonData(epoch=epoch, - epoch_delta=epoch_delta, - proton_sw_velocity_rtn=proton_sw_velocity_rtn, - proton_sw_speed=proton_sw_speed, - swp_flags=swp_flags) + return SwapiL3aProtonData( + epoch=epoch, + epoch_delta=epoch_delta, + proton_sw_velocity_rtn=proton_sw_velocity_rtn, + proton_sw_speed=proton_sw_speed, + swp_flags=swp_flags, + ) def read_swe_config(swe_config: Path) -> SweConfiguration: @@ -84,14 +101,18 @@ def read_swe_config(swe_config: Path) -> SweConfiguration: return config -def compute_epoch_delta_in_ns(acq_duration: np.ndarray[float], settle_duration: np.ndarray[float]) -> np.ndarray: +def compute_epoch_delta_in_ns( + acq_duration: np.ndarray[float], settle_duration: np.ndarray[float] +) -> np.ndarray: number_of_energies = acq_duration.shape[1] number_of_spins = settle_duration.shape[1] number_of_spin_sectors = acq_duration.shape[2] energies_per_spin = number_of_energies / number_of_spins - total_settle_duration = np.sum(settle_duration, axis=1) * energies_per_spin * number_of_spin_sectors + total_settle_duration = ( + np.sum(settle_duration, axis=1) * energies_per_spin * number_of_spin_sectors + ) total_acq_duration = np.sum(acq_duration, axis=(1, 2)) total_duration_in_microseconds = total_settle_duration + total_acq_duration return total_duration_in_microseconds / 2 * 1000 diff --git a/imap_l3_processing/swe/quality_flags.py b/imap_l3_processing/swe/quality_flags.py index 6ac03d5c8..bf4318aaf 100644 --- a/imap_l3_processing/swe/quality_flags.py +++ b/imap_l3_processing/swe/quality_flags.py @@ -3,15 +3,15 @@ class SweL3Flags(FlagNameMixin): NONE = CommonFlags.NONE - FALLBACK_POTENTIAL_ESTIMATE = 2 ** 2 - BACKUP_SPLINE_UNRESOLVED = 2 ** 3 - POTENTIAL_FIT_UNCONVERGED = 2 ** 4 - BREAKPOINT_FIT_UNCONVERGED = 2 ** 5 - ULTRA_HV_OFF = 2 ** 6 - TEMPERATURE_OUTLIER = 2 ** 7 - PRELIMINARY_MAG = 2 ** 8 - FALLBACK_SWAPI_SPEED = 2 ** 9 - NEGATIVE_MOMENT = 2 ** 10 - UNPHYSICAL_PSD = 2 ** 11 + FALLBACK_POTENTIAL_ESTIMATE = 2**2 + BACKUP_SPLINE_UNRESOLVED = 2**3 + POTENTIAL_FIT_UNCONVERGED = 2**4 + BREAKPOINT_FIT_UNCONVERGED = 2**5 + ULTRA_HV_OFF = 2**6 + TEMPERATURE_OUTLIER = 2**7 + PRELIMINARY_MAG = 2**8 + FALLBACK_SWAPI_SPEED = 2**9 + NEGATIVE_MOMENT = 2**10 + UNPHYSICAL_PSD = 2**11 FALLBACK_CALIBRATION_EXTRAPOLATED = 2**12 - PREDICTIVE_EPHEMERIS = 2 ** 15 + PREDICTIVE_EPHEMERIS = 2**15 diff --git a/imap_l3_processing/swe/swe_processor.py b/imap_l3_processing/swe/swe_processor.py index 031719d00..627832100 100644 --- a/imap_l3_processing/swe/swe_processor.py +++ b/imap_l3_processing/swe/swe_processor.py @@ -73,7 +73,9 @@ def _detect_negative(density: np.ndarray, tensor: np.ndarray) -> np.ndarray: - return (density < 0) | np.any(tensor[:, TEMPERATURE_TENSOR_DIAGONAL_INDICES] < 0, axis=-1) + return (density < 0) | np.any( + tensor[:, TEMPERATURE_TENSOR_DIAGONAL_INDICES] < 0, axis=-1 + ) def check_and_mask_negative_moments(moment_data: SweL3MomentData) -> np.ndarray: @@ -96,23 +98,27 @@ def check_and_mask_negative_moments(moment_data: SweL3MomentData) -> np.ndarray: for field_suffix in INTEGRATED_FIELD_SUFFIXES: getattr(moment_data, "core" + field_suffix)[core_negative] = np.nan getattr(moment_data, "halo" + field_suffix)[halo_negative] = np.nan - getattr(moment_data, "total" + field_suffix)[core_negative | halo_negative | total_negative] = np.nan + getattr(moment_data, "total" + field_suffix)[ + core_negative | halo_negative | total_negative + ] = np.nan - flags[core_negative | halo_negative | total_negative] |= np.uint16(SweL3Flags.NEGATIVE_MOMENT) + flags[core_negative | halo_negative | total_negative] |= np.uint16( + SweL3Flags.NEGATIVE_MOMENT + ) return flags # Temperature Outlier Flag Algorithm def check_temperature_outlier_flag(data: np.ndarray): # Define window duration that is centered, so best to use an odd number - window = 61 # 61 is about 1 hour + window = 61 # 61 is about 1 hour # Initiate Temperature Outliers to be all NONE value TEMPERATURE_OUTLIER = np.zeros(len(data), dtype=np.uint16) TEMPERATURE_OUTLIER[:] = SweL3Flags.NONE for i in np.arange(len(data)): # Get left and right index accounting for edges - left_idx = 0 if i - window//2 < 0 else i - window//2 - right_idx = i + window//2 + 1 + left_idx = 0 if i - window // 2 < 0 else i - window // 2 + right_idx = i + window // 2 + 1 # Loop check the current value while removing every point from the window on each iteration for k in np.arange(len(data[left_idx:right_idx])): temp_data = data[left_idx:right_idx] @@ -120,7 +126,7 @@ def check_temperature_outlier_flag(data: np.ndarray): # Check if current value is beyond 3-sigma of median from data median = np.nanmedian(temp_data) deviation = np.nanstd(temp_data) - if np.abs(median - data[i]) >= deviation*3: + if np.abs(median - data[i]) >= deviation * 3: TEMPERATURE_OUTLIER[i] = SweL3Flags.TEMPERATURE_OUTLIER # Break since we already know the current index i is an outlier break @@ -130,8 +136,8 @@ def check_temperature_outlier_flag(data: np.ndarray): # Don't do this is the current value is already an outlier if TEMPERATURE_OUTLIER[i] == SweL3Flags.NONE: # Get left and right index accounting for edges - left_idx = 0 if i - window//2 < 0 else i - window//2 - right_idx = i + window//2 + 1 + left_idx = 0 if i - window // 2 < 0 else i - window // 2 + right_idx = i + window // 2 + 1 # Loop check the current value while removing every point from the window on each iteration # This time around, we conditionally slice to avoid including already known outliers from # median and standard devation calculations @@ -140,9 +146,17 @@ def check_temperature_outlier_flag(data: np.ndarray): continue temp_data = data[left_idx:right_idx] # Check if current value is beyond 3-sigma of median from data - median = np.nanmedian(np.delete(temp_data,k)[np.delete(tof_copy[left_idx:right_idx],k)==0]) - deviation = np.nanstd(np.delete(temp_data,k)[np.delete(tof_copy[left_idx:right_idx],k)==0]) - if np.abs(median - data[i]) >= deviation * 3.: + median = np.nanmedian( + np.delete(temp_data, k)[ + np.delete(tof_copy[left_idx:right_idx], k) == 0 + ] + ) + deviation = np.nanstd( + np.delete(temp_data, k)[ + np.delete(tof_copy[left_idx:right_idx], k) == 0 + ] + ) + if np.abs(median - data[i]) >= deviation * 3.0: TEMPERATURE_OUTLIER[i] = SweL3Flags.TEMPERATURE_OUTLIER # Break since we already know the current index i is an outlier break @@ -263,13 +277,21 @@ def calculate_products(self, dependencies: SweL3Dependencies) -> SweL3Data: psd_array > UNPHYSICAL_PSD_THRESHOLD, axis=tuple(range(1, psd_array.ndim)), ) - swe_quality_flags[unphysical_psd_per_epoch] |= np.uint16(SweL3Flags.UNPHYSICAL_PSD) + swe_quality_flags[unphysical_psd_per_epoch] |= np.uint16( + SweL3Flags.UNPHYSICAL_PSD + ) if dependencies.mag_is_preliminary: - swe_quality_flags = np.bitwise_or(swe_quality_flags, SweL3Flags.PRELIMINARY_MAG) + swe_quality_flags = np.bitwise_or( + swe_quality_flags, SweL3Flags.PRELIMINARY_MAG + ) - last_cal_interval = (swe_l2_data.data_quality & SweL1bFlags.LAST_CAL_INTERVAL) != 0 - swe_quality_flags[last_cal_interval] |= np.uint16(SweL3Flags.FALLBACK_CALIBRATION_EXTRAPOLATED) + last_cal_interval = ( + swe_l2_data.data_quality & SweL1bFlags.LAST_CAL_INTERVAL + ) != 0 + swe_quality_flags[last_cal_interval] |= np.uint16( + SweL3Flags.FALLBACK_CALIBRATION_EXTRAPOLATED + ) rebinned_mask = np.ma.masked_invalid(swe_l2_data.phase_space_density_rebinned) dist_by_phi_rebinned = np.average( @@ -432,11 +454,17 @@ def calculate_moment_products( predicted_tracker = PredictedEphemerisTracker() try: - dps_to_rtn = predicted_tracker.run(get_dps_to_rtn_rotation_matrix, current_epoch) + dps_to_rtn = predicted_tracker.run( + get_dps_to_rtn_rotation_matrix, current_epoch + ) except SpiceyError: - logger.info(f"No IMAP_DPS to IMAP_RTN rotation available at epoch index {i}. Using fill values.") + logger.info( + f"No IMAP_DPS to IMAP_RTN rotation available at epoch index {i}. Using fill values." + ) dps_to_rtn = np.full((3, 3), np.nan) - quality_flags[i] |= np.uint16(SweL3Flags.PREDICTIVE_EPHEMERIS * predicted_tracker.used_predict) + quality_flags[i] |= np.uint16( + SweL3Flags.PREDICTIVE_EPHEMERIS * predicted_tracker.used_predict + ) if core_moment_fit_result is not None: core_moment = core_moment_fit_result.moments @@ -859,9 +887,10 @@ def calculate_moment_products( halo_temperature_perpendicular_to_mag=halo_temperature_perpendicular_to_mag, total_temperature_parallel_to_mag=total_temperature_parallel_to_mag, total_temperature_perpendicular_to_mag=total_temperature_perpendicular_to_mag, - core_temperature_tensor_integrated = core_temperature_tensor_integrated*1e4, - halo_temperature_tensor_integrated = halo_temperature_tensor_integrated*1e4, - total_temperature_tensor_integrated = total_temperature_tensor_integrated*1e4, + core_temperature_tensor_integrated=core_temperature_tensor_integrated * 1e4, + halo_temperature_tensor_integrated=halo_temperature_tensor_integrated * 1e4, + total_temperature_tensor_integrated=total_temperature_tensor_integrated + * 1e4, quality_flags=quality_flags, ) @@ -885,9 +914,11 @@ def calculate_pitch_angle_products( swapi_l3a_proton_data = dependencies.swapi_l3a_proton_data swapi_epoch = swapi_l3a_proton_data.epoch - solar_wind_vectors, used_predict_to_rotate_solar_wind = rotate_rtn_vectors_to_dps( - swapi_epoch, - swapi_l3a_proton_data.proton_sw_velocity_rtn, + solar_wind_vectors, used_predict_to_rotate_solar_wind = ( + rotate_rtn_vectors_to_dps( + swapi_epoch, + swapi_l3a_proton_data.proton_sw_velocity_rtn, + ) ) proton_sw_speed = swapi_l3a_proton_data.proton_sw_speed fallback_to_speed = np.any(np.isnan(solar_wind_vectors), axis=1) & np.isfinite( @@ -911,15 +942,22 @@ def calculate_pitch_angle_products( maximum_distance=swapi_max_distance, ) - rebinned_solar_wind_vectors = solar_wind_interpolator.interpolate_data() - rebinned_fallback_to_speed = solar_wind_interpolator.interpolate_flags(fallback_to_speed) - rebinned_used_predict_to_rotate_solar_wind = solar_wind_interpolator.interpolate_flags(used_predict_to_rotate_solar_wind) + rebinned_fallback_to_speed = solar_wind_interpolator.interpolate_flags( + fallback_to_speed + ) + rebinned_used_predict_to_rotate_solar_wind = ( + solar_wind_interpolator.interpolate_flags(used_predict_to_rotate_solar_wind) + ) swe_flags = np.full(len(swe_epoch), SweL3Flags.NONE, dtype=np.uint16) - swe_flags[rebinned_fallback_to_speed] |= np.uint16(SweL3Flags.FALLBACK_SWAPI_SPEED) - swe_flags[rebinned_used_predict_to_rotate_solar_wind] |= np.uint16(SweL3Flags.PREDICTIVE_EPHEMERIS) + swe_flags[rebinned_fallback_to_speed] |= np.uint16( + SweL3Flags.FALLBACK_SWAPI_SPEED + ) + swe_flags[rebinned_used_predict_to_rotate_solar_wind] |= np.uint16( + SweL3Flags.PREDICTIVE_EPHEMERIS + ) counts = dependencies.swe_l1b_data.count_rates * ( swe_l2_data.acquisition_duration[..., np.newaxis] / 1e6 diff --git a/imap_l3_processing/ultra/models.py b/imap_l3_processing/ultra/models.py index 43c32b363..8406576c3 100644 --- a/imap_l3_processing/ultra/models.py +++ b/imap_l3_processing/ultra/models.py @@ -12,7 +12,9 @@ from imap_l3_processing.cdf.cdf_utils import read_numeric_variable from imap_l3_processing.constants import TT2000_EPOCH, ONE_SECOND_IN_NANOSECONDS -from imap_l3_processing.glows.l3e.glows_l3e_hi_model import PROBABILITY_OF_SURVIVAL_VAR_NAME +from imap_l3_processing.glows.l3e.glows_l3e_hi_model import ( + PROBABILITY_OF_SURVIVAL_VAR_NAME, +) from imap_l3_processing.glows.l3e.glows_l3e_ultra_model import ( HEALPIX_INDEX_VAR_NAME, ENERGY_VAR_NAME, @@ -40,8 +42,10 @@ def read_from_path(cls, path_to_cdf: Path) -> UltraGlowsL3eData: repointing=repointing, energy=read_numeric_variable(cdf[ENERGY_VAR_NAME]), healpix_index=cdf[HEALPIX_INDEX_VAR_NAME][...], - survival_probability=read_numeric_variable(cdf[PROBABILITY_OF_SURVIVAL_VAR_NAME]), - flags=cdf[GLOWS_FLAGS_VAR_NAME][...] + survival_probability=read_numeric_variable( + cdf[PROBABILITY_OF_SURVIVAL_VAR_NAME] + ), + flags=cdf[GLOWS_FLAGS_VAR_NAME][...], ) @@ -67,8 +71,12 @@ def read_from_path(cls, path: Path) -> UltraL1CPSet: epoch_delta=cdf["epoch_delta"][0], energy=read_numeric_variable(cdf[CoordNames.ENERGY_ULTRA_L1C.value]), exposure=read_numeric_variable(cdf["exposure_factor"]), - latitude=read_numeric_variable(cdf[CoordNames.ELEVATION_L1C.value]).ravel(), - longitude=read_numeric_variable(cdf[CoordNames.AZIMUTH_L1C.value]).ravel(), + latitude=read_numeric_variable( + cdf[CoordNames.ELEVATION_L1C.value] + ).ravel(), + longitude=read_numeric_variable( + cdf[CoordNames.AZIMUTH_L1C.value] + ).ravel(), healpix_index=cdf[CoordNames.HEALPIX_INDEX.value][...], sensitivity=read_numeric_variable(cdf["sensitivity"]), ) @@ -77,9 +85,11 @@ def to_xarray(self): return xr.Dataset( { "exposure_factor": ( - [CoordNames.TIME.value, - CoordNames.ENERGY_ULTRA_L1C.value, - CoordNames.HEALPIX_INDEX.value], + [ + CoordNames.TIME.value, + CoordNames.ENERGY_ULTRA_L1C.value, + CoordNames.HEALPIX_INDEX.value, + ], self.exposure, ), "sensitivity": ( @@ -100,9 +110,11 @@ def to_xarray(self): }, coords={ CoordNames.TIME.value: [ - ((self.epoch - TT2000_EPOCH).total_seconds()) * ONE_SECOND_IN_NANOSECONDS + self.epoch_delta / 2, + ((self.epoch - TT2000_EPOCH).total_seconds()) + * ONE_SECOND_IN_NANOSECONDS + + self.epoch_delta / 2, ], CoordNames.ENERGY_ULTRA_L1C.value: self.energy, CoordNames.HEALPIX_INDEX.value: self.healpix_index, - } + }, ) diff --git a/imap_l3_processing/ultra/science/ultra_survival_probability.py b/imap_l3_processing/ultra/science/ultra_survival_probability.py index 1b0fa34ab..bd9db8548 100644 --- a/imap_l3_processing/ultra/science/ultra_survival_probability.py +++ b/imap_l3_processing/ultra/science/ultra_survival_probability.py @@ -17,26 +17,33 @@ class UltraSurvivalProbability(UltraPointingSet): - def __init__(self, l1c_pset: UltraL1CPSet, l3e_glows: UltraGlowsL3eData, - bin_groups: Optional[np.ndarray] = None): + def __init__( + self, + l1c_pset: UltraL1CPSet, + l3e_glows: UltraGlowsL3eData, + bin_groups: Optional[np.ndarray] = None, + ): coarse_bins = bin_pset_energy_bins(l1c_pset.to_xarray(), bin_groups) super().__init__(coarse_bins, geometry.SpiceFrame.IMAP_DPS) - predicted_ephemeris = ( - l3e_glows.flags & GlowsL3Flags.PREDICTIVE_EPHEMERIS > 0 - ) + predicted_ephemeris = l3e_glows.flags & GlowsL3Flags.PREDICTIVE_EPHEMERIS > 0 nominal_alpha_proton_ratio = ( - l3e_glows.flags & GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO > 0 - ) - persisted_last_point = ( - l3e_glows.flags & GlowsL3Flags.PERSISTED_LAST_POINT > 0 + l3e_glows.flags & GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO > 0 ) + persisted_last_point = l3e_glows.flags & GlowsL3Flags.PERSISTED_LAST_POINT > 0 l1c_epoch_in_et = spiceypy.unitim( - self.data.coords[CoordNames.TIME.value].values[0] / ONE_SECOND_IN_NANOSECONDS, "TT", "ET") - rotated_az_el_points = geometry.frame_transform_az_el(l1c_epoch_in_et, self.az_el_points, - geometry.SpiceFrame.IMAP_DPS, - geometry.SpiceFrame.ECLIPJ2000) + self.data.coords[CoordNames.TIME.value].values[0] + / ONE_SECOND_IN_NANOSECONDS, + "TT", + "ET", + ) + rotated_az_el_points = geometry.frame_transform_az_el( + l1c_epoch_in_et, + self.az_el_points, + geometry.SpiceFrame.IMAP_DPS, + geometry.SpiceFrame.ECLIPJ2000, + ) glows_nside = npix_to_nside(len(l3e_glows.healpix_index)) glows_healpix = HEALPix(nside=glows_nside) @@ -45,78 +52,105 @@ def __init__(self, l1c_pset: UltraL1CPSet, l3e_glows: UltraGlowsL3eData, spatially_interpolated_sp = np.zeros((len(l3e_glows.energy), npixels)) for energy_index in range(len(l3e_glows.energy)): - spatially_interpolated_sp[energy_index, :] = glows_healpix.interpolate_bilinear_lonlat( - Quantity(rotated_az_el_points[:, 0], unit='deg'), - Quantity(rotated_az_el_points[:, 1], unit='deg'), - l3e_glows.survival_probability[0, energy_index, :]) + spatially_interpolated_sp[energy_index, :] = ( + glows_healpix.interpolate_bilinear_lonlat( + Quantity(rotated_az_el_points[:, 0], unit="deg"), + Quantity(rotated_az_el_points[:, 1], unit="deg"), + l3e_glows.survival_probability[0, energy_index, :], + ) + ) - energy_interpolated_sp = np.zeros((len(coarse_bins.energy_bin_geometric_mean), npixels)) + energy_interpolated_sp = np.zeros( + (len(coarse_bins.energy_bin_geometric_mean), npixels) + ) for healpix_index in range(npixels): energy_interpolated_sp[:, healpix_index] = np.interp( np.log10(coarse_bins.energy_bin_geometric_mean), np.log10(l3e_glows.energy), - spatially_interpolated_sp[:, healpix_index] + spatially_interpolated_sp[:, healpix_index], ) self.data["survival_probability_times_exposure"] = ( [ CoordNames.TIME.value, CoordNames.ENERGY_ULTRA_L1C.value, - CoordNames.HEALPIX_INDEX.value + CoordNames.HEALPIX_INDEX.value, ], - energy_interpolated_sp * coarse_bins.exposure_factor.data + energy_interpolated_sp * coarse_bins.exposure_factor.data, ) self.data["predicted_ephemeris_flag"] = ( [ CoordNames.TIME.value, CoordNames.ENERGY_ULTRA_L1C.value, - CoordNames.HEALPIX_INDEX.value + CoordNames.HEALPIX_INDEX.value, ], - predicted_ephemeris * coarse_bins.exposure_factor.data + predicted_ephemeris * coarse_bins.exposure_factor.data, ) self.data["nominal_alpha_proton_ratio_flag"] = ( [ CoordNames.TIME.value, CoordNames.ENERGY_ULTRA_L1C.value, - CoordNames.HEALPIX_INDEX.value + CoordNames.HEALPIX_INDEX.value, ], - nominal_alpha_proton_ratio * coarse_bins.exposure_factor.data + nominal_alpha_proton_ratio * coarse_bins.exposure_factor.data, ) self.data["persisted_last_point_flag"] = ( [ CoordNames.TIME.value, CoordNames.ENERGY_ULTRA_L1C.value, - CoordNames.HEALPIX_INDEX.value + CoordNames.HEALPIX_INDEX.value, ], - persisted_last_point * coarse_bins.exposure_factor.data + persisted_last_point * coarse_bins.exposure_factor.data, ) self.data["epoch_delta"] = [l1c_pset.epoch_delta] class UltraSurvivalProbabilitySkyMap(HealpixSkyMap): - def __init__(self, sp: list[UltraSurvivalProbability], spice_frame: geometry.SpiceFrame, nside: int): + def __init__( + self, + sp: list[UltraSurvivalProbability], + spice_frame: geometry.SpiceFrame, + nside: int, + ): super().__init__(nside, spice_frame) for sp_pset in sp: - values_to_project = ["survival_probability_times_exposure", "exposure_factor", "predicted_ephemeris_flag", - "nominal_alpha_proton_ratio_flag", "persisted_last_point_flag"] - self.project_pset_values_to_map(sp_pset, values_to_project, - pset_valid_mask=np.isfinite( - sp_pset.data["survival_probability_times_exposure"])) + values_to_project = [ + "survival_probability_times_exposure", + "exposure_factor", + "predicted_ephemeris_flag", + "nominal_alpha_proton_ratio_flag", + "persisted_last_point_flag", + ] + self.project_pset_values_to_map( + sp_pset, + values_to_project, + pset_valid_mask=np.isfinite( + sp_pset.data["survival_probability_times_exposure"] + ), + ) predicted_ephemeris_set = self.data_1d["predicted_ephemeris_flag"] > 0 - nominal_proton_alpha_ratio_set = self.data_1d["nominal_alpha_proton_ratio_flag"] > 0 + nominal_proton_alpha_ratio_set = ( + self.data_1d["nominal_alpha_proton_ratio_flag"] > 0 + ) persisted_last_point_set = self.data_1d["persisted_last_point_flag"] > 0 - quality_flags = (predicted_ephemeris_set * MapL3Flags.PREDICTIVE_EPHEMERIS) | \ - (nominal_proton_alpha_ratio_set * MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO) | \ - (persisted_last_point_set * MapL3Flags.PERSISTED_LAST_POINT) + quality_flags = ( + (predicted_ephemeris_set * MapL3Flags.PREDICTIVE_EPHEMERIS) + | (nominal_proton_alpha_ratio_set * MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO) + | (persisted_last_point_set * MapL3Flags.PERSISTED_LAST_POINT) + ) - self.data_1d = Dataset({ - "exposure_weighted_survival_probabilities": self.data_1d["survival_probability_times_exposure"] / - self.data_1d["exposure_factor"], - "quality_flags": quality_flags - }) + self.data_1d = Dataset( + { + "exposure_weighted_survival_probabilities": self.data_1d[ + "survival_probability_times_exposure" + ] + / self.data_1d["exposure_factor"], + "quality_flags": quality_flags, + } + ) diff --git a/imap_l3_processing/ultra/ultra_combined_nsp_initializer.py b/imap_l3_processing/ultra/ultra_combined_nsp_initializer.py index 59bed3a3b..fbd7ae0f0 100644 --- a/imap_l3_processing/ultra/ultra_combined_nsp_initializer.py +++ b/imap_l3_processing/ultra/ultra_combined_nsp_initializer.py @@ -5,8 +5,12 @@ import imap_data_access from imap_data_access.file_validation import Version -from imap_l3_processing.maps.map_descriptors import Sensor, map_descriptor_parts_to_string, \ - parse_map_descriptor, get_duration_from_map_descriptor +from imap_l3_processing.maps.map_descriptors import ( + Sensor, + map_descriptor_parts_to_string, + parse_map_descriptor, + get_duration_from_map_descriptor, +) from imap_l3_processing.maps.map_initializer import MapInitializer, PossibleMapToProduce from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.ultra.ultra_sp_initializer import ULTRA_SP_SPICE_KERNELS @@ -16,37 +20,46 @@ "ulc-ena-h-hf-nsp-full-hae-2deg-3mo", "ulc-ena-h-hf-nsp-full-hae-4deg-3mo", "ulc-ena-h-hf-nsp-full-hae-6deg-3mo", - "ulc-ena-h-hf-nsp-full-hae-2deg-6mo", "ulc-ena-h-hf-nsp-full-hae-4deg-6mo", "ulc-ena-h-hf-nsp-full-hae-6deg-6mo", - "ulc-ena-h-hf-nsp-full-hae-2deg-1yr", "ulc-ena-h-hf-nsp-full-hae-4deg-1yr", "ulc-ena-h-hf-nsp-full-hae-6deg-1yr", ] + class UltraCombinedNSPInitializer(MapInitializer): def __init__(self): - l2_ultra_maps_query_results = imap_data_access.query(instrument="ultra", data_level="l2") - self.l2_maps_by_descriptor_and_start = self.get_latest_version_by_descriptor_and_start_date( - l2_ultra_maps_query_results) + l2_ultra_maps_query_results = imap_data_access.query( + instrument="ultra", data_level="l2" + ) + self.l2_maps_by_descriptor_and_start = ( + self.get_latest_version_by_descriptor_and_start_date( + l2_ultra_maps_query_results + ) + ) energy_bin_group_sizes_files = imap_data_access.query( instrument="ultra", table="ancillary", descriptor="l2-energy-bin-group-sizes", - version="latest") - self.ancillary_dependencies = [Path(qr['file_path']).name for qr in energy_bin_group_sizes_files] + version="latest", + ) + self.ancillary_dependencies = [ + Path(qr["file_path"]).name for qr in energy_bin_group_sizes_files + ] super().__init__("ultra") def furnish_spice_dependencies(self, map_to_produce: PossibleMapToProduce): furnish_spice_metakernel( start_date=map_to_produce.input_metadata.start_date, end_date=map_to_produce.input_metadata.end_date, - kernel_types=ULTRA_SP_SPICE_KERNELS + kernel_types=ULTRA_SP_SPICE_KERNELS, ) - def get_maps_that_can_be_produced(self, descriptor: str) -> list[PossibleMapToProduce]: + def get_maps_that_can_be_produced( + self, descriptor: str + ) -> list[PossibleMapToProduce]: descriptor_parts = parse_map_descriptor(descriptor) if descriptor_parts is None: raise ValueError(f"Invalid map descriptor: {descriptor}") @@ -55,34 +68,56 @@ def get_maps_that_can_be_produced(self, descriptor: str) -> list[PossibleMapToPr dataclasses.replace(descriptor_parts, sensor=Sensor.Ultra45), dataclasses.replace(descriptor_parts, sensor=Sensor.Ultra90), ] - input_map_descriptors = [map_descriptor_parts_to_string(parts) for parts in dependency_descriptors] + input_map_descriptors = [ + map_descriptor_parts_to_string(parts) for parts in dependency_descriptors + ] first_input_descriptor = input_map_descriptors[0] - possible_start_dates = self.l2_maps_by_descriptor_and_start[first_input_descriptor].keys() + possible_start_dates = self.l2_maps_by_descriptor_and_start[ + first_input_descriptor + ].keys() possible_maps_to_produce = [] for str_start_date in possible_start_dates: input_map_filenames = [] for input_descriptor in input_map_descriptors: - if str_start_date in self.l2_maps_by_descriptor_and_start[input_descriptor]: - input_map_filenames.append(self.l2_maps_by_descriptor_and_start[input_descriptor][str_start_date]) + if ( + str_start_date + in self.l2_maps_by_descriptor_and_start[input_descriptor] + ): + input_map_filenames.append( + self.l2_maps_by_descriptor_and_start[input_descriptor][ + str_start_date + ] + ) if len(input_map_filenames) != len(input_map_descriptors): continue - pset_filenames = [l1c for l2 in input_map_filenames for l1c in self.get_l1c_parents_from_map(l2)] + pset_filenames = [ + l1c + for l2 in input_map_filenames + for l1c in self.get_l1c_parents_from_map(l2) + ] start_date = datetime.strptime(str_start_date, "%Y%m%d") - possible_maps_to_produce.append(PossibleMapToProduce( - input_files=set(pset_filenames + input_map_filenames + self.ancillary_dependencies), - input_metadata=InputMetadata( - instrument='ultra', - data_level='l3', - start_date=start_date, - end_date=start_date + get_duration_from_map_descriptor(descriptor_parts), - version=VersionMap({},Version(None,1)), - descriptor=descriptor, + possible_maps_to_produce.append( + PossibleMapToProduce( + input_files=set( + pset_filenames + + input_map_filenames + + self.ancillary_dependencies + ), + input_metadata=InputMetadata( + instrument="ultra", + data_level="l3", + start_date=start_date, + end_date=start_date + + get_duration_from_map_descriptor(descriptor_parts), + version=VersionMap({}, Version(None, 1)), + descriptor=descriptor, + ), ) - )) + ) return possible_maps_to_produce diff --git a/imap_l3_processing/ultra/ultra_l3_dependencies.py b/imap_l3_processing/ultra/ultra_l3_dependencies.py index ad6b71bf2..00e3447a4 100644 --- a/imap_l3_processing/ultra/ultra_l3_dependencies.py +++ b/imap_l3_processing/ultra/ultra_l3_dependencies.py @@ -14,10 +14,16 @@ from imap_data_access.processing_input import ProcessingInputCollection from imap_processing.ultra.l2.ultra_l2 import ultra_l2 -from imap_l3_processing.maps.map_models import HealPixIntensityMapData, RectangularIntensityMapData, \ - SpectralIndexDependencies +from imap_l3_processing.maps.map_models import ( + HealPixIntensityMapData, + RectangularIntensityMapData, + SpectralIndexDependencies, +) from imap_l3_processing.ultra.models import UltraL1CPSet, UltraGlowsL3eData -from imap_l3_processing.utils import get_dependency_paths_by_descriptor, get_temp_cache_dir +from imap_l3_processing.utils import ( + get_dependency_paths_by_descriptor, + get_temp_cache_dir, +) logger = logging.getLogger(__name__) @@ -34,30 +40,47 @@ class UltraL3Dependencies: @classmethod def fetch_dependencies(cls, deps: ProcessingInputCollection) -> UltraL3Dependencies: ultra_l2_names = deps.get_file_paths("ultra", data_type="l2") - assert len(ultra_l2_names) == 1, f"Incorrect number of map dependencies: {len(ultra_l2_names)}" + assert len(ultra_l2_names) == 1, ( + f"Incorrect number of map dependencies: {len(ultra_l2_names)}" + ) ultra_l2_name = ultra_l2_names[0] - ultra_l1c_names = deps.get_file_paths("ultra", data_type="l1c") glows_l3e_names = deps.get_file_paths("glows") - energy_bin_group_sizes_names = deps.get_file_paths(data_type='ancillary', - descriptor='l2-energy-bin-group-sizes') + energy_bin_group_sizes_names = deps.get_file_paths( + data_type="ancillary", descriptor="l2-energy-bin-group-sizes" + ) l2_map_path = imap_data_access.download(ultra_l2_name) - ultra_l1c_downloaded_paths = [imap_data_access.download(l1c) for l1c in ultra_l1c_names] - glows_l3e_download_paths = [imap_data_access.download(path) for path in glows_l3e_names] + ultra_l1c_downloaded_paths = [ + imap_data_access.download(l1c) for l1c in ultra_l1c_names + ] + glows_l3e_download_paths = [ + imap_data_access.download(path) for path in glows_l3e_names + ] if energy_bin_group_sizes_names: - energy_bin_group_sizes_path = imap_data_access.download(energy_bin_group_sizes_names[0]) + energy_bin_group_sizes_path = imap_data_access.download( + energy_bin_group_sizes_names[0] + ) else: energy_bin_group_sizes_path = None - return cls.from_file_paths(l2_map_path, ultra_l1c_downloaded_paths, glows_l3e_download_paths, - energy_bin_group_sizes_path) + return cls.from_file_paths( + l2_map_path, + ultra_l1c_downloaded_paths, + glows_l3e_download_paths, + energy_bin_group_sizes_path, + ) @classmethod - def from_file_paths(cls, l2_map_path: Path, l1c_file_paths: list[Path], glows_file_paths: list[Path], - energy_bin_path: Optional[Path]) -> UltraL3Dependencies: + def from_file_paths( + cls, + l2_map_path: Path, + l1c_file_paths: list[Path], + glows_file_paths: list[Path], + energy_bin_path: Optional[Path], + ) -> UltraL3Dependencies: ultra_l1c_data = [] glows_l3e_data = [] for file_path in l1c_file_paths: @@ -68,14 +91,24 @@ def from_file_paths(cls, l2_map_path: Path, l1c_file_paths: list[Path], glows_fi l2_healpix_map_data = load_or_create_healpix_l2(l2_map_path, l1c_file_paths) - l2_rectangular_map_data = RectangularIntensityMapData.read_from_path(l2_map_path) + l2_rectangular_map_data = RectangularIntensityMapData.read_from_path( + l2_map_path + ) energy_bin_group_sizes = None if energy_bin_path: paths.append(energy_bin_path) - energy_bin_group_sizes = np.loadtxt(energy_bin_path, delimiter=',', dtype=np.uint8) + energy_bin_group_sizes = np.loadtxt( + energy_bin_path, delimiter=",", dtype=np.uint8 + ) - return cls(ultra_l2_healpix_map=l2_healpix_map_data, ultra_l2_rectangular_map=l2_rectangular_map_data, ultra_l1c_pset=ultra_l1c_data, glows_l3e_sp=glows_l3e_data, - dependency_file_paths=paths, energy_bin_group_sizes=energy_bin_group_sizes) + return cls( + ultra_l2_healpix_map=l2_healpix_map_data, + ultra_l2_rectangular_map=l2_rectangular_map_data, + ultra_l1c_pset=ultra_l1c_data, + glows_l3e_sp=glows_l3e_data, + dependency_file_paths=paths, + energy_bin_group_sizes=energy_bin_group_sizes, + ) @dataclass @@ -83,18 +116,28 @@ class UltraL3SpectralIndexDependencies(SpectralIndexDependencies): fit_energy_ranges: np.ndarray @classmethod - def fetch_dependencies(cls, deps: ProcessingInputCollection) -> UltraL3SpectralIndexDependencies: - energy_fit_ranges_ancillary_file_path = deps.get_file_paths(source="ultra", descriptor="spx-energy-ranges") - ultra_map_file_paths = deps.get_file_paths(source="ultra", data_type="l3") + deps.get_file_paths(source="ultra", data_type="l2") + def fetch_dependencies( + cls, deps: ProcessingInputCollection + ) -> UltraL3SpectralIndexDependencies: + energy_fit_ranges_ancillary_file_path = deps.get_file_paths( + source="ultra", descriptor="spx-energy-ranges" + ) + ultra_map_file_paths = deps.get_file_paths( + source="ultra", data_type="l3" + ) + deps.get_file_paths(source="ultra", data_type="l2") if len(ultra_map_file_paths) != 1: - raise ValueError(f"Expected 1 input map, got {len(ultra_map_file_paths)}: [{', '.join(p.name for p in ultra_map_file_paths)}]") + raise ValueError( + f"Expected 1 input map, got {len(ultra_map_file_paths)}: [{', '.join(p.name for p in ultra_map_file_paths)}]" + ) if len(energy_fit_ranges_ancillary_file_path) != 1: raise ValueError("Missing fit energy ranges ancillary file") map_file_path = imap_data_access.download(ultra_map_file_paths[0].name) - energy_ranges_file_path = imap_data_access.download(energy_fit_ranges_ancillary_file_path[0].name) + energy_ranges_file_path = imap_data_access.download( + energy_fit_ranges_ancillary_file_path[0].name + ) return cls.from_file_paths(map_file_path, energy_ranges_file_path) @@ -114,46 +157,72 @@ class UltraL3CombinedDependencies: u90_dependencies: UltraL3Dependencies @classmethod - def fetch_dependencies(cls, deps: ProcessingInputCollection) -> UltraL3CombinedDependencies: + def fetch_dependencies( + cls, deps: ProcessingInputCollection + ) -> UltraL3CombinedDependencies: descriptors = ["u45", "u90", "45sensor", "90sensor", "survival-probability-ul"] - file_paths = get_dependency_paths_by_descriptor(deps=deps, descriptors=descriptors) - - assert len(file_paths['u45']) == 1 - assert len(file_paths['u90']) == 1 - - u45_pset_paths = [imap_data_access.download(pset) for pset in file_paths['45sensor']] - u90_pset_paths = [imap_data_access.download(pset) for pset in file_paths['90sensor']] - glows_l3e_pset_paths = [imap_data_access.download(pset) for pset in file_paths['survival-probability-ul']] - u45_map_path = imap_data_access.download(file_paths['u45'][0]) - u90_map_path = imap_data_access.download(file_paths['u90'][0]) + file_paths = get_dependency_paths_by_descriptor( + deps=deps, descriptors=descriptors + ) - energy_bin_group_sizes_names = deps.get_file_paths(data_type='ancillary', - descriptor='l2-energy-bin-group-sizes') + assert len(file_paths["u45"]) == 1 + assert len(file_paths["u90"]) == 1 + + u45_pset_paths = [ + imap_data_access.download(pset) for pset in file_paths["45sensor"] + ] + u90_pset_paths = [ + imap_data_access.download(pset) for pset in file_paths["90sensor"] + ] + glows_l3e_pset_paths = [ + imap_data_access.download(pset) + for pset in file_paths["survival-probability-ul"] + ] + u45_map_path = imap_data_access.download(file_paths["u45"][0]) + u90_map_path = imap_data_access.download(file_paths["u90"][0]) + + energy_bin_group_sizes_names = deps.get_file_paths( + data_type="ancillary", descriptor="l2-energy-bin-group-sizes" + ) if energy_bin_group_sizes_names: - energy_bin_group_sizes_path = imap_data_access.download(energy_bin_group_sizes_names[0].name) + energy_bin_group_sizes_path = imap_data_access.download( + energy_bin_group_sizes_names[0].name + ) else: energy_bin_group_sizes_path = None - return cls.from_file_paths(u45_pset_paths, u90_pset_paths, glows_l3e_pset_paths, u45_map_path, u90_map_path, - energy_bin_group_sizes_path) + return cls.from_file_paths( + u45_pset_paths, + u90_pset_paths, + glows_l3e_pset_paths, + u45_map_path, + u90_map_path, + energy_bin_group_sizes_path, + ) @classmethod - def from_file_paths(cls, u45_pset_paths: list[Path], u90_pset_paths: list[Path], glows_l3e_pset_paths: list[Path], - u45_map_path: Path, u90_map_path: Path, - energy_bin_group_sizes_path: Optional[Path]) -> UltraL3CombinedDependencies: + def from_file_paths( + cls, + u45_pset_paths: list[Path], + u90_pset_paths: list[Path], + glows_l3e_pset_paths: list[Path], + u45_map_path: Path, + u90_map_path: Path, + energy_bin_group_sizes_path: Optional[Path], + ) -> UltraL3CombinedDependencies: return cls( u45_dependencies=UltraL3Dependencies.from_file_paths( l2_map_path=u45_map_path, l1c_file_paths=u45_pset_paths, glows_file_paths=glows_l3e_pset_paths, - energy_bin_path=energy_bin_group_sizes_path + energy_bin_path=energy_bin_group_sizes_path, ), u90_dependencies=UltraL3Dependencies.from_file_paths( l2_map_path=u90_map_path, l1c_file_paths=u90_pset_paths, glows_file_paths=glows_l3e_pset_paths, - energy_bin_path=energy_bin_group_sizes_path + energy_bin_path=energy_bin_group_sizes_path, ), ) @@ -176,9 +245,11 @@ def load_or_create_healpix_l2( else: logger.info("Cached HEALPix L2 not found; building from pointing sets") l2_healpix_datasets = ultra_l2(l1c_paths_dict, descriptor=l2_descriptor) - l2_healpix_data = HealPixIntensityMapData.read_from_xarray(l2_healpix_datasets[0]) + l2_healpix_data = HealPixIntensityMapData.read_from_xarray( + l2_healpix_datasets[0] + ) logger.info("Saving HEALPix L2 to cache at %s", cache_path) with open(cache_path, "wb") as f: pickle.dump(l2_healpix_data, f) - return l2_healpix_data \ No newline at end of file + return l2_healpix_data diff --git a/imap_l3_processing/ultra/ultra_l3_to_rectangular_dependencies.py b/imap_l3_processing/ultra/ultra_l3_to_rectangular_dependencies.py index ba8961435..0ca0ec47d 100644 --- a/imap_l3_processing/ultra/ultra_l3_to_rectangular_dependencies.py +++ b/imap_l3_processing/ultra/ultra_l3_to_rectangular_dependencies.py @@ -12,7 +12,9 @@ class UltraL3ToRectangularDependencies: healpix_map_data: HealPixIntensityMapData @classmethod - def fetch_dependencies(cls, deps: ProcessingInputCollection) -> UltraL3ToRectangularDependencies: + def fetch_dependencies( + cls, deps: ProcessingInputCollection + ) -> UltraL3ToRectangularDependencies: [l3_ultra_map] = deps.get_file_paths("ultra", None, "l3") l3_map_path = download(l3_ultra_map.name) return cls.from_file_paths(l3_map_path) diff --git a/imap_l3_processing/ultra/ultra_processor.py b/imap_l3_processing/ultra/ultra_processor.py index 1b42def05..a9d503f0e 100644 --- a/imap_l3_processing/ultra/ultra_processor.py +++ b/imap_l3_processing/ultra/ultra_processor.py @@ -32,11 +32,18 @@ ) from imap_l3_processing.maps.map_processor import MapProcessor from imap_l3_processing.maps.quality_flags import MapL3Flags -from imap_l3_processing.maps.spectral_fit import calculate_spectral_index_for_multiple_ranges, \ - slice_energy_range_by_bin, fit_spectral_index_map -from imap_l3_processing.maps.survival_probability_processing import combine_glows_l3e_with_l1c_pointing -from imap_l3_processing.ultra.science.ultra_survival_probability import UltraSurvivalProbabilitySkyMap, \ - UltraSurvivalProbability +from imap_l3_processing.maps.spectral_fit import ( + calculate_spectral_index_for_multiple_ranges, + slice_energy_range_by_bin, + fit_spectral_index_map, +) +from imap_l3_processing.maps.survival_probability_processing import ( + combine_glows_l3e_with_l1c_pointing, +) +from imap_l3_processing.ultra.science.ultra_survival_probability import ( + UltraSurvivalProbabilitySkyMap, + UltraSurvivalProbability, +) from imap_l3_processing.ultra.ultra_l3_dependencies import ( UltraL3Dependencies, UltraL3SpectralIndexDependencies, @@ -56,34 +63,56 @@ def process(self, spice_frame_name: SpiceFrame = SpiceFrame.ECLIPJ2000): parent_file_names = self.get_parent_file_names() match parsed_descriptor: - case MapDescriptorParts(quantity=MapQuantity.SpectralIndex, - grid=PixelSize.OneDegree | PixelSize.TwoDegrees | PixelSize.FourDegrees | PixelSize.SixDegrees): - ultra_l3_spectral_fit_dependencies = UltraL3SpectralIndexDependencies.fetch_dependencies( - self.dependencies) + case MapDescriptorParts( + quantity=MapQuantity.SpectralIndex, + grid=PixelSize.OneDegree + | PixelSize.TwoDegrees + | PixelSize.FourDegrees + | PixelSize.SixDegrees, + ): + ultra_l3_spectral_fit_dependencies = ( + UltraL3SpectralIndexDependencies.fetch_dependencies( + self.dependencies + ) + ) spectral_index_map_data = self._process_spectral_index( ultra_l3_spectral_fit_dependencies, - parsed_descriptor.spectral_index_energy_range + parsed_descriptor.spectral_index_energy_range, ) - data_product = RectangularSpectralIndexDataProduct(input_metadata=self.input_metadata, - data=spectral_index_map_data, - spice_frame_name=spice_frame_name) - case MapDescriptorParts(survival_correction=SurvivalCorrection.SurvivalCorrected, - sensor=Sensor.Ultra45 | Sensor.Ultra90, - grid=PixelSize.OneDegree | PixelSize.TwoDegrees | PixelSize.FourDegrees | PixelSize.SixDegrees): + data_product = RectangularSpectralIndexDataProduct( + input_metadata=self.input_metadata, + data=spectral_index_map_data, + spice_frame_name=spice_frame_name, + ) + case MapDescriptorParts( + survival_correction=SurvivalCorrection.SurvivalCorrected, + sensor=Sensor.Ultra45 | Sensor.Ultra90, + grid=PixelSize.OneDegree + | PixelSize.TwoDegrees + | PixelSize.FourDegrees + | PixelSize.SixDegrees, + ): deps = UltraL3Dependencies.fetch_dependencies(self.dependencies) healpix_intensity_map_data = ( correct_healpix_data_for_survival_probability( deps, spice_frame_name ) ) - data_product = self._process_healpix_intensity_to_rectangular(healpix_intensity_map_data, - deps.ultra_l2_rectangular_map, - parsed_descriptor.grid, - spice_frame_name=spice_frame_name) + data_product = self._process_healpix_intensity_to_rectangular( + healpix_intensity_map_data, + deps.ultra_l2_rectangular_map, + parsed_descriptor.grid, + spice_frame_name=spice_frame_name, + ) data_product.add_paths_to_parents(deps.dependency_file_paths) - case MapDescriptorParts(survival_correction=SurvivalCorrection.SurvivalCorrected, - sensor=Sensor.UltraCombined, - grid=PixelSize.OneDegree | PixelSize.TwoDegrees | PixelSize.FourDegrees | PixelSize.SixDegrees): + case MapDescriptorParts( + survival_correction=SurvivalCorrection.SurvivalCorrected, + sensor=Sensor.UltraCombined, + grid=PixelSize.OneDegree + | PixelSize.TwoDegrees + | PixelSize.FourDegrees + | PixelSize.SixDegrees, + ): combined_deps = UltraL3CombinedDependencies.fetch_dependencies( self.dependencies ) @@ -105,9 +134,14 @@ def process(self, spice_frame_name: SpiceFrame = SpiceFrame.ECLIPJ2000): combined_deps.u90_dependencies.dependency_file_paths ) - case MapDescriptorParts(sensor=Sensor.UltraCombined, - survival_correction=SurvivalCorrection.NotSurvivalCorrected, - grid=PixelSize.OneDegree | PixelSize.TwoDegrees | PixelSize.FourDegrees | PixelSize.SixDegrees): + case MapDescriptorParts( + sensor=Sensor.UltraCombined, + survival_correction=SurvivalCorrection.NotSurvivalCorrected, + grid=PixelSize.OneDegree + | PixelSize.TwoDegrees + | PixelSize.FourDegrees + | PixelSize.SixDegrees, + ): deps = UltraL3CombinedDependencies.fetch_dependencies(self.dependencies) combination_strategy = ExposureWeightedCombination() @@ -119,16 +153,27 @@ def process(self, spice_frame_name: SpiceFrame = SpiceFrame.ECLIPJ2000): ] ) ) - combined_rectangular = combination_strategy.combine_rectangular_intensity_map_data( - [deps.u45_dependencies.ultra_l2_rectangular_map, - deps.u90_dependencies.ultra_l2_rectangular_map]) - - data_product = self._process_healpix_intensity_to_rectangular(combined_healpix, - combined_rectangular, - parsed_descriptor.grid, - spice_frame_name=spice_frame_name) - data_product.add_paths_to_parents(deps.u45_dependencies.dependency_file_paths) - data_product.add_paths_to_parents(deps.u90_dependencies.dependency_file_paths) + combined_rectangular = ( + combination_strategy.combine_rectangular_intensity_map_data( + [ + deps.u45_dependencies.ultra_l2_rectangular_map, + deps.u90_dependencies.ultra_l2_rectangular_map, + ] + ) + ) + + data_product = self._process_healpix_intensity_to_rectangular( + combined_healpix, + combined_rectangular, + parsed_descriptor.grid, + spice_frame_name=spice_frame_name, + ) + data_product.add_paths_to_parents( + deps.u45_dependencies.dependency_file_paths + ) + data_product.add_paths_to_parents( + deps.u90_dependencies.dependency_file_paths + ) case _: raise NotImplementedError @@ -136,27 +181,41 @@ def process(self, spice_frame_name: SpiceFrame = SpiceFrame.ECLIPJ2000): return [save_data(data_product)] def _process_combined_survival_probability( - self, deps: UltraL3CombinedDependencies, spice_frame_name: SpiceFrame, + self, + deps: UltraL3CombinedDependencies, + spice_frame_name: SpiceFrame, ) -> tuple[HealPixIntensityMapData, RectangularIntensityMapData]: - u45_survival_corrected = correct_healpix_data_for_survival_probability(deps.u45_dependencies, spice_frame_name) - u90_survival_corrected = correct_healpix_data_for_survival_probability(deps.u90_dependencies, spice_frame_name) + u45_survival_corrected = correct_healpix_data_for_survival_probability( + deps.u45_dependencies, spice_frame_name + ) + u90_survival_corrected = correct_healpix_data_for_survival_probability( + deps.u90_dependencies, spice_frame_name + ) combination_strategy = UncertaintyWeightedCombination() combined_healpix = combination_strategy.combine_healpix_intensity_map_data( [u45_survival_corrected, u90_survival_corrected] ) - combined_rectangular = combination_strategy.combine_rectangular_intensity_map_data( - [deps.u45_dependencies.ultra_l2_rectangular_map, deps.u90_dependencies.ultra_l2_rectangular_map] + combined_rectangular = ( + combination_strategy.combine_rectangular_intensity_map_data( + [ + deps.u45_dependencies.ultra_l2_rectangular_map, + deps.u90_dependencies.ultra_l2_rectangular_map, + ] + ) ) return combined_healpix, combined_rectangular - def _process_spectral_index(self, - dependencies: UltraL3SpectralIndexDependencies, - spectral_index_range: Optional[tuple[int, int]] - ) -> RectangularSpectralIndexMapData: + def _process_spectral_index( + self, + dependencies: UltraL3SpectralIndexDependencies, + spectral_index_range: Optional[tuple[int, int]], + ) -> RectangularSpectralIndexMapData: if spectral_index_range is not None: start, end = spectral_index_range - sliced = slice_energy_range_by_bin(dependencies.map_data.intensity_map_data, start, end) + sliced = slice_energy_range_by_bin( + dependencies.map_data.intensity_map_data, start, end + ) map_data = fit_spectral_index_map(sliced) else: map_data = calculate_spectral_index_for_multiple_ranges( @@ -191,18 +250,28 @@ def _process_healpix_intensity_to_rectangular( variables_to_convert_to_rectangular.append("survival_probability") healpix_map = healpix_map_data.to_healpix_skymap() - rectangular_map, _ = healpix_map.to_rectangular_skymap(spacing_deg, variables_to_convert_to_rectangular) + rectangular_map, _ = healpix_map.to_rectangular_skymap( + spacing_deg, variables_to_convert_to_rectangular + ) rectangular_map_xarray_dataset = rectangular_map.to_dataset() rect_l2_data = rect_l2_map.intensity_map_data - predicted_ephemeris_set = rectangular_map_xarray_dataset["predicted_ephemeris_flag"].values > 0 - nominal_alpha_proton_ratio_set = rectangular_map_xarray_dataset["nominal_alpha_proton_ratio_flag"].values > 0 - persisted_last_point_set = rectangular_map_xarray_dataset["persisted_last_point_flag"].values > 0 + predicted_ephemeris_set = ( + rectangular_map_xarray_dataset["predicted_ephemeris_flag"].values > 0 + ) + nominal_alpha_proton_ratio_set = ( + rectangular_map_xarray_dataset["nominal_alpha_proton_ratio_flag"].values > 0 + ) + persisted_last_point_set = ( + rectangular_map_xarray_dataset["persisted_last_point_flag"].values > 0 + ) - quality_flags = (predicted_ephemeris_set * MapL3Flags.PREDICTIVE_EPHEMERIS) | \ - (nominal_alpha_proton_ratio_set * MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO) | \ - (persisted_last_point_set * MapL3Flags.PERSISTED_LAST_POINT) + quality_flags = ( + (predicted_ephemeris_set * MapL3Flags.PREDICTIVE_EPHEMERIS) + | (nominal_alpha_proton_ratio_set * MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO) + | (persisted_last_point_set * MapL3Flags.PERSISTED_LAST_POINT) + ) intensity_map_data = IntensityMapData( epoch=rect_l2_data.epoch, @@ -228,7 +297,9 @@ def _process_healpix_intensity_to_rectangular( ) if has_survival_data: - intensity_map_data.survival_probability = rectangular_map_xarray_dataset["survival_probability"].values + intensity_map_data.survival_probability = rectangular_map_xarray_dataset[ + "survival_probability" + ].values rect_intensity_map_data = RectangularIntensityMapData( intensity_map_data, @@ -240,13 +311,18 @@ def _process_healpix_intensity_to_rectangular( ), ) - return RectangularIntensityDataProduct(data=rect_intensity_map_data, input_metadata=self.input_metadata, - spice_frame_name=spice_frame_name) + return RectangularIntensityDataProduct( + data=rect_intensity_map_data, + input_metadata=self.input_metadata, + spice_frame_name=spice_frame_name, + ) + @dataclass class UltraMapDescriptorParts: grid_size: int + def _make_cache_key_for_sp_corrected_data(deps: UltraL3Dependencies) -> str: pset_names = [] glows_names = [] @@ -264,6 +340,7 @@ def _make_cache_key_for_sp_corrected_data(deps: UltraL3Dependencies) -> str: pass return str(sorted(pset_names)) + str(sorted(glows_names)) + l2_descriptor + def correct_healpix_data_for_survival_probability( deps: UltraL3Dependencies, spice_frame_name: SpiceFrame ) -> HealPixIntensityMapData: @@ -277,7 +354,9 @@ def correct_healpix_data_for_survival_probability( logger.info("Cached HEALPix SP-corrected data not found; building from inputs") - combined_psets = combine_glows_l3e_with_l1c_pointing(deps.glows_l3e_sp, deps.ultra_l1c_pset) + combined_psets = combine_glows_l3e_with_l1c_pointing( + deps.glows_l3e_sp, deps.ultra_l1c_pset + ) survival_probability_psets = [ UltraSurvivalProbability(_l1c, _l3e, bin_groups=deps.energy_bin_group_sizes) for _l1c, _l3e in combined_psets @@ -289,7 +368,9 @@ def correct_healpix_data_for_survival_probability( survival_probability_psets, spice_frame_name, coords.nside ) skymap_dataset = corrected_skymap.to_dataset() - survival_probability_map = skymap_dataset["exposure_weighted_survival_probabilities"].values + survival_probability_map = skymap_dataset[ + "exposure_weighted_survival_probabilities" + ].values healpix_intensity_data = HealPixIntensityMapData( intensity_map_data=IntensityMapData( @@ -323,4 +404,4 @@ def correct_healpix_data_for_survival_probability( with open(cache_path, "wb") as f: pickle.dump(healpix_intensity_data, f) - return healpix_intensity_data \ No newline at end of file + return healpix_intensity_data diff --git a/imap_l3_processing/ultra/ultra_sp_initializer.py b/imap_l3_processing/ultra/ultra_sp_initializer.py index 2d20430b7..fba378346 100644 --- a/imap_l3_processing/ultra/ultra_sp_initializer.py +++ b/imap_l3_processing/ultra/ultra_sp_initializer.py @@ -3,8 +3,16 @@ import imap_data_access -from imap_l3_processing.glows.descriptors import GLOWS_L3E_ULTRA_HF_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR -from imap_l3_processing.maps.map_descriptors import MapDescriptorParts, SurvivalCorrection, Sensor, ReferenceFrame +from imap_l3_processing.glows.descriptors import ( + GLOWS_L3E_ULTRA_HF_DESCRIPTOR, + GLOWS_L3E_ULTRA_SF_DESCRIPTOR, +) +from imap_l3_processing.maps.map_descriptors import ( + MapDescriptorParts, + SurvivalCorrection, + Sensor, + ReferenceFrame, +) from imap_l3_processing.maps.map_initializer import PossibleMapToProduce from imap_l3_processing.maps.sp_map_initializer import SPMapInitializer from imap_l3_processing.utils import furnish_spice_metakernel, SpiceKernelTypes @@ -23,14 +31,12 @@ "u45-ena-h-sf-sp-full-hae-4deg-3mo", "u45-ena-h-hf-sp-full-hae-6deg-3mo", "u45-ena-h-sf-sp-full-hae-6deg-3mo", - "u45-ena-h-hf-sp-full-hae-2deg-6mo", "u45-ena-h-sf-sp-full-hae-2deg-6mo", "u45-ena-h-hf-sp-full-hae-4deg-6mo", "u45-ena-h-sf-sp-full-hae-4deg-6mo", "u45-ena-h-hf-sp-full-hae-6deg-6mo", "u45-ena-h-sf-sp-full-hae-6deg-6mo", - "u45-ena-h-hf-sp-full-hae-2deg-1yr", "u45-ena-h-sf-sp-full-hae-2deg-1yr", "u45-ena-h-hf-sp-full-hae-4deg-1yr", @@ -46,14 +52,12 @@ "u90-ena-h-sf-sp-full-hae-4deg-3mo", "u90-ena-h-hf-sp-full-hae-6deg-3mo", "u90-ena-h-sf-sp-full-hae-6deg-3mo", - "u90-ena-h-hf-sp-full-hae-2deg-6mo", "u90-ena-h-sf-sp-full-hae-2deg-6mo", "u90-ena-h-hf-sp-full-hae-4deg-6mo", "u90-ena-h-sf-sp-full-hae-4deg-6mo", "u90-ena-h-hf-sp-full-hae-6deg-6mo", "u90-ena-h-sf-sp-full-hae-6deg-6mo", - "u90-ena-h-hf-sp-full-hae-2deg-1yr", "u90-ena-h-sf-sp-full-hae-2deg-1yr", "u90-ena-h-hf-sp-full-hae-4deg-1yr", @@ -66,11 +70,9 @@ "ulc-ena-h-hf-sp-full-hae-2deg-3mo", "ulc-ena-h-hf-sp-full-hae-4deg-3mo", "ulc-ena-h-hf-sp-full-hae-6deg-3mo", - "ulc-ena-h-hf-sp-full-hae-2deg-6mo", "ulc-ena-h-hf-sp-full-hae-4deg-6mo", "ulc-ena-h-hf-sp-full-hae-6deg-6mo", - "ulc-ena-h-hf-sp-full-hae-2deg-1yr", "ulc-ena-h-hf-sp-full-hae-4deg-1yr", "ulc-ena-h-hf-sp-full-hae-6deg-1yr", @@ -79,21 +81,32 @@ class UltraSPInitializer(SPMapInitializer): def __init__(self): - sf_sp_query_result = imap_data_access.query(instrument='glows', data_level='l3e', - descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, version="latest") - self.sf_glows_psets_by_repointing = {int(r["repointing"]): Path(r["file_path"]).name for r in - sf_sp_query_result} - hf_sp_query_result = imap_data_access.query(instrument='glows', data_level='l3e', - descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, version="latest") - self.hf_glows_psets_by_repointing = {int(r["repointing"]): Path(r["file_path"]).name for r in - hf_sp_query_result} + sf_sp_query_result = imap_data_access.query( + instrument="glows", + data_level="l3e", + descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + version="latest", + ) + self.sf_glows_psets_by_repointing = { + int(r["repointing"]): Path(r["file_path"]).name for r in sf_sp_query_result + } + hf_sp_query_result = imap_data_access.query( + instrument="glows", + data_level="l3e", + descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, + version="latest", + ) + self.hf_glows_psets_by_repointing = { + int(r["repointing"]): Path(r["file_path"]).name for r in hf_sp_query_result + } l2_query_result = imap_data_access.query(instrument="ultra", data_level="l2") self._energy_bin_group_sizes_files = imap_data_access.query( instrument="ultra", table="ancillary", descriptor="l2-energy-bin-group-sizes", - version="latest") + version="latest", + ) super().__init__("ultra", l2_query_result) @@ -101,25 +114,43 @@ def furnish_spice_dependencies(self, map_to_produce: PossibleMapToProduce): furnish_spice_metakernel( start_date=map_to_produce.input_metadata.start_date, end_date=map_to_produce.input_metadata.end_date, - kernel_types=ULTRA_SP_SPICE_KERNELS + kernel_types=ULTRA_SP_SPICE_KERNELS, ) - def _collect_glows_psets_by_repoint(self, descriptor: MapDescriptorParts) -> dict[int, str]: + def _collect_glows_psets_by_repoint( + self, descriptor: MapDescriptorParts + ) -> dict[int, str]: if descriptor.reference_frame == ReferenceFrame.Heliospheric: return self.hf_glows_psets_by_repointing elif descriptor.reference_frame == ReferenceFrame.Spacecraft: return self.sf_glows_psets_by_repointing else: - raise NotImplementedError("Reference frame should be either Spacecraft or Heliospheric") + raise NotImplementedError( + "Reference frame should be either Spacecraft or Heliospheric" + ) - def _get_l2_dependencies(self, descriptor: MapDescriptorParts) -> list[MapDescriptorParts]: + def _get_l2_dependencies( + self, descriptor: MapDescriptorParts + ) -> list[MapDescriptorParts]: if descriptor.sensor == Sensor.UltraCombined: - return [dataclasses.replace(descriptor, survival_correction=SurvivalCorrection.NotSurvivalCorrected, - sensor=Sensor.Ultra45), - dataclasses.replace(descriptor, survival_correction=SurvivalCorrection.NotSurvivalCorrected, - sensor=Sensor.Ultra90)] - - return [dataclasses.replace(descriptor, survival_correction=SurvivalCorrection.NotSurvivalCorrected)] + return [ + dataclasses.replace( + descriptor, + survival_correction=SurvivalCorrection.NotSurvivalCorrected, + sensor=Sensor.Ultra45, + ), + dataclasses.replace( + descriptor, + survival_correction=SurvivalCorrection.NotSurvivalCorrected, + sensor=Sensor.Ultra90, + ), + ] + + return [ + dataclasses.replace( + descriptor, survival_correction=SurvivalCorrection.NotSurvivalCorrected + ) + ] def _get_ancillary_files(self) -> list[str]: return [Path(f["file_path"]).name for f in self._energy_bin_group_sizes_files] diff --git a/imap_l3_processing/utils.py b/imap_l3_processing/utils.py index 4a6e1bb56..a97e8fdf3 100644 --- a/imap_l3_processing/utils.py +++ b/imap_l3_processing/utils.py @@ -37,7 +37,8 @@ from imap_l3_processing.version import VERSION logger = logging.getLogger(__name__) -_cache_directory: TemporaryDirectory|None = None +_cache_directory: TemporaryDirectory | None = None + def get_temp_cache_dir() -> Path: global _cache_directory @@ -45,12 +46,14 @@ def get_temp_cache_dir() -> Path: _cache_directory = TemporaryDirectory(ignore_cleanup_errors=True) return Path(_cache_directory.name) + def clear_temp_cache(): global _cache_directory if _cache_directory is not None: _cache_directory.cleanup() _cache_directory = None + class SpiceKernelTypes(enum.Enum): Leapseconds = "leapseconds" IMAPFrames = "imap_frames" @@ -64,8 +67,12 @@ class SpiceKernelTypes(enum.Enum): PlanetaryConstants = "planetary_constants" -def save_data(data: DataProduct, delete_if_present: bool = False, folder_path: Path = None, - cr_number=None) -> Path: +def save_data( + data: DataProduct, + delete_if_present: bool = False, + folder_path: Path = None, + cr_number=None, +) -> Path: science_file_path = data.input_metadata.to_science_file_path(cr_number=cr_number) file_path = science_file_path.construct_path() if folder_path is not None: @@ -79,11 +86,18 @@ def save_data(data: DataProduct, delete_if_present: bool = False, folder_path: P file_path.unlink(missing_ok=True) attribute_manager = ImapAttributeManager() - version = str(data.input_metadata.version.lookup(data.input_metadata.descriptor)).replace('v','') + version = str( + data.input_metadata.version.lookup(data.input_metadata.descriptor) + ).replace("v", "") attribute_manager.add_global_attribute("Data_version", version) - attribute_manager.add_instrument_attrs(data.input_metadata.instrument, data.input_metadata.data_level, - data.input_metadata.descriptor) - attribute_manager.add_global_attribute("Generation_date", date.today().strftime("%Y%m%d")) + attribute_manager.add_instrument_attrs( + data.input_metadata.instrument, + data.input_metadata.data_level, + data.input_metadata.descriptor, + ) + attribute_manager.add_global_attribute( + "Generation_date", date.today().strftime("%Y%m%d") + ) attribute_manager.add_global_attribute("Logical_source", logical_source) attribute_manager.add_global_attribute("Logical_file_id", logical_file_id) attribute_manager.add_global_attribute("ground_software_version", VERSION) @@ -103,12 +117,22 @@ def save_data(data: DataProduct, delete_if_present: bool = False, folder_path: P attribute_manager.add_global_attribute(key, value) if attribute_manager.try_load_global_metadata(logical_source) is None: - logical_source_global_attrs = generate_global_metadata_for_undefined_logical_source(data.input_metadata) - attribute_manager.add_global_attribute(logical_source, logical_source_global_attrs) - - attribute_manager.add_global_attribute("Spice_reference_frame", data.spice_frame_name.name) + logical_source_global_attrs = ( + generate_global_metadata_for_undefined_logical_source( + data.input_metadata + ) + ) + attribute_manager.add_global_attribute( + logical_source, logical_source_global_attrs + ) + + attribute_manager.add_global_attribute( + "Spice_reference_frame", data.spice_frame_name.name + ) else: - raise AssertionError(f"Found an unsupported map data product of type: {type(data)}") + raise AssertionError( + f"Found an unsupported map data product of type: {type(data)}" + ) elif data.parent_file_names: attribute_manager.add_global_attribute("Parents", data.parent_file_names) @@ -126,10 +150,10 @@ def generate_map_global_metadata(data_product: MapDataProduct) -> dict: match data_product.data: case ( - RectangularSpectralIndexMapData(spectral_index_map_data=map_data) | - RectangularIntensityMapData(intensity_map_data=map_data) | - HealPixIntensityMapData(intensity_map_data=map_data) | - ISNBackgroundSubtractedMapData(isn_rate_map_data=map_data) + RectangularSpectralIndexMapData(spectral_index_map_data=map_data) + | RectangularIntensityMapData(intensity_map_data=map_data) + | HealPixIntensityMapData(intensity_map_data=map_data) + | ISNBackgroundSubtractedMapData(isn_rate_map_data=map_data) ): [start_date] = map_data.epoch [epoch_delta] = map_data.epoch_delta @@ -139,7 +163,9 @@ def generate_map_global_metadata(data_product: MapDataProduct) -> dict: end_date = start_date + timedelta(seconds=epoch_delta / 1e9) case _: - raise ValueError(f"Found an unsupported map data product of type: {type(data_product.data)}") + raise ValueError( + f"Found an unsupported map data product of type: {type(data_product.data)}" + ) attrs["start_date"] = start_date.isoformat() attrs["end_date"] = end_date.isoformat() @@ -160,8 +186,10 @@ def generate_map_global_metadata(data_product: MapDataProduct) -> dict: return attrs -def generate_global_metadata_for_undefined_logical_source(input_metadata: InputMetadata) -> dict: - level = input_metadata.data_level.replace('l', '') +def generate_global_metadata_for_undefined_logical_source( + input_metadata: InputMetadata, +) -> dict: + level = input_metadata.data_level.replace("l", "") data_type_string = f"Level-{level}" if "spx" in input_metadata.descriptor: data_type_string += " Spectral Fit Index Map" @@ -184,7 +212,9 @@ def format_time(t: Optional[datetime]) -> Optional[str]: return None -def download_external_dependency(dependency_url: str, file_path: Path) -> Optional[Path]: +def download_external_dependency( + dependency_url: str, file_path: Path +) -> Optional[Path]: try: response = requests.get(dependency_url) if response.status_code == 200: @@ -192,7 +222,9 @@ def download_external_dependency(dependency_url: str, file_path: Path) -> Option file.write(response.content) return Path(file_path) else: - logger.error(f"Failed to download {dependency_url} with status code {response.status_code}") + logger.error( + f"Failed to download {dependency_url} with status code {response.status_code}" + ) except RequestException: logger.exception(f"Failed to download {dependency_url}") return None @@ -201,8 +233,9 @@ def download_external_dependency(dependency_url: str, file_path: Path) -> Option def read_mag_data(cdf_path: Union[str, Path]) -> MagData: with CDF(str(cdf_path)) as cdf: return MagData( - epoch=cdf['epoch'][...], - mag_data=read_numeric_variable(cdf["b_dsrf"])[:, :3]) + epoch=cdf["epoch"][...], + mag_data=read_numeric_variable(cdf["b_dsrf"])[:, :3], + ) def select_mag_path( @@ -212,18 +245,23 @@ def select_mag_path( science_files = dependencies.processing_input for level in ("l2", "l1d"): match = next( - (d.imap_file_paths[0] for d in science_files - if d.source == "mag" - and d.descriptor == descriptor - and d.data_type == level), + ( + d.imap_file_paths[0] + for d in science_files + if d.source == "mag" + and d.descriptor == descriptor + and d.data_type == level + ), None, ) if match is not None: return download(match.construct_path()), level return None, None -def get_dependency_paths_by_descriptor(deps: ProcessingInputCollection, descriptors: list[str]) -> dict[ - str, list[Path]]: + +def get_dependency_paths_by_descriptor( + deps: ProcessingInputCollection, descriptors: list[str] +) -> dict[str, list[Path]]: descriptor_to_paths = {key: [] for key in descriptors} for input_file in deps.get_science_inputs(): for descriptor in descriptors: @@ -237,10 +275,10 @@ def get_dependency_paths_by_descriptor(deps: ProcessingInputCollection, descript def furnish_local_spice(): kernels = Path(imap_l3_processing.__file__).parent.parent.joinpath("spice_kernels") - total_kernels = spiceypy.ktotal('ALL') + total_kernels = spiceypy.ktotal("ALL") current_kernels = [] for i in range(0, total_kernels): - current_kernels.append(Path(spiceypy.kdata(i, 'ALL')[0]).name) + current_kernels.append(Path(spiceypy.kdata(i, "ALL")[0]).name) for file in kernels.iterdir(): if file.name not in current_kernels: @@ -248,8 +286,8 @@ def furnish_local_spice(): def get_spice_parent_file_names() -> list[str]: - count = spiceypy.ktotal('ALL') - return [Path(spiceypy.kdata(i, 'ALL')[0]).name for i in range(0, count)] + count = spiceypy.ktotal("ALL") + return [Path(spiceypy.kdata(i, "ALL")[0]).name for i in range(0, count)] @dataclass @@ -258,36 +296,56 @@ class FurnishMetakernelOutput: spice_kernel_paths: list[Path] -def get_spice_kernels_file_names(start_date: datetime, end_date: datetime, kernel_types: list[SpiceKernelTypes]) -> \ - list[str]: - metakernel_url = urlparse(imap_data_access.config['DATA_ACCESS_URL'])._replace(path="metakernel").geturl() +def get_spice_kernels_file_names( + start_date: datetime, end_date: datetime, kernel_types: list[SpiceKernelTypes] +) -> list[str]: + metakernel_url = ( + urlparse(imap_data_access.config["DATA_ACCESS_URL"]) + ._replace(path="metakernel") + .geturl() + ) parameters: dict = { - 'file_types': [kernel_type.value for kernel_type in kernel_types], - 'start_time': f"{int((start_date - datetime(2000, 1, 1, 12)).total_seconds())}", - 'end_time': f"{int((end_date - datetime(2000, 1, 1, 12)).total_seconds())}", + "file_types": [kernel_type.value for kernel_type in kernel_types], + "start_time": f"{int((start_date - datetime(2000, 1, 1, 12)).total_seconds())}", + "end_time": f"{int((end_date - datetime(2000, 1, 1, 12)).total_seconds())}", } - kernels_res = requests.get(metakernel_url, params={**parameters, 'list_files': 'true'}) + kernels_res = requests.get( + metakernel_url, params={**parameters, "list_files": "true"} + ) kernels = json.loads(kernels_res.text) return kernels -def furnish_spice_metakernel(start_date: datetime, end_date: datetime, kernel_types: list[SpiceKernelTypes], metakernel_file_name: str="metakernel.txt") -> FurnishMetakernelOutput: - metakernel_path = imap_data_access.config.get("DATA_DIR") / "metakernel" / metakernel_file_name +def furnish_spice_metakernel( + start_date: datetime, + end_date: datetime, + kernel_types: list[SpiceKernelTypes], + metakernel_file_name: str = "metakernel.txt", +) -> FurnishMetakernelOutput: + metakernel_path = ( + imap_data_access.config.get("DATA_DIR") / "metakernel" / metakernel_file_name + ) kernel_path = imap_data_access.config.get("DATA_DIR") / "imap" / "spice" parameters: dict = { - 'spice_path': kernel_path, - 'file_types': [kernel_type.value for kernel_type in kernel_types], - 'start_time': str(int((start_date - datetime(2000, 1, 1, 12)).total_seconds())), - 'end_time': str(int((end_date - datetime(2000, 1, 1, 12)).total_seconds())), + "spice_path": kernel_path, + "file_types": [kernel_type.value for kernel_type in kernel_types], + "start_time": str(int((start_date - datetime(2000, 1, 1, 12)).total_seconds())), + "end_time": str(int((end_date - datetime(2000, 1, 1, 12)).total_seconds())), } - metakernel_url = urlparse(imap_data_access.config['DATA_ACCESS_URL'])._replace(path="metakernel").geturl() + metakernel_url = ( + urlparse(imap_data_access.config["DATA_ACCESS_URL"]) + ._replace(path="metakernel") + .geturl() + ) - logger.info(f"Getting SPICE Metakernel from: {metakernel_url}, with params: {parameters}") + logger.info( + f"Getting SPICE Metakernel from: {metakernel_url}, with params: {parameters}" + ) metakernel_res = requests.get(metakernel_url, params=parameters) @@ -301,7 +359,10 @@ def furnish_spice_metakernel(start_date: datetime, end_date: datetime, kernel_ty spiceypy.furnsh(str(metakernel_path)) - return FurnishMetakernelOutput(metakernel_path=metakernel_path, spice_kernel_paths=downloaded_paths) + return FurnishMetakernelOutput( + metakernel_path=metakernel_path, spice_kernel_paths=downloaded_paths + ) + @contextmanager def furnished_metakernel(start_date, end_date, kernel_types): @@ -310,6 +371,7 @@ def furnished_metakernel(start_date, end_date, kernel_types): finally: spiceypy.kclear() + def read_cdf_parents(server_file_name: str) -> set[str]: downloaded_path = imap_data_access.download(server_file_name) @@ -317,6 +379,7 @@ def read_cdf_parents(server_file_name: str) -> set[str]: parents = set(cdf.attrs["Parents"]) return parents + def get_version_from_query_result(science_file_query_result): if "major_version" in science_file_query_result: return Version( @@ -325,5 +388,3 @@ def get_version_from_query_result(science_file_query_result): ) else: return Version.from_version(science_file_query_result["version"]) - - diff --git a/imap_l3_processing/version.py b/imap_l3_processing/version.py index f5c1c29c3..3d3bf67dd 100644 --- a/imap_l3_processing/version.py +++ b/imap_l3_processing/version.py @@ -1 +1 @@ -VERSION = 'This will be replaced during docker build' +VERSION = "This will be replaced during docker build" diff --git a/run_local.py b/run_local.py index d68f57874..2c70197cc 100644 --- a/run_local.py +++ b/run_local.py @@ -15,55 +15,93 @@ import numpy as np import spiceypy import xarray as xr -from imap_data_access.processing_input import AncillaryInput, ProcessingInputCollection, ScienceInput, \ - RepointInput +from imap_data_access.processing_input import ( + AncillaryInput, + ProcessingInputCollection, + ScienceInput, + RepointInput, +) from imap_processing.spice.geometry import SpiceFrame from spacepy.pycdf import CDF from imap_l3_processing.codice.l3.hi.codice_hi_processor import CodiceHiProcessor -from imap_l3_processing.codice.l3.hi.direct_event.codice_hi_l3a_direct_events_dependencies import \ - CodiceHiL3aDirectEventsDependencies -from imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies import CodicePitchAngleDependencies -from imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies import \ - CodiceLoL3b3dDistributionsDependencies -from imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies import CodiceLoL3aDirectEventsDependencies -from imap_l3_processing.codice.l3.lo.codice_lo_l3a_partial_densities_dependencies import \ - CodiceLoL3aPartialDensitiesDependencies -from imap_l3_processing.codice.l3.lo.codice_lo_l3a_ratios_dependencies import CodiceLoL3aRatiosDependencies +from imap_l3_processing.codice.l3.hi.direct_event.codice_hi_l3a_direct_events_dependencies import ( + CodiceHiL3aDirectEventsDependencies, +) +from imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies import ( + CodicePitchAngleDependencies, +) +from imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies import ( + CodiceLoL3b3dDistributionsDependencies, +) +from imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies import ( + CodiceLoL3aDirectEventsDependencies, +) +from imap_l3_processing.codice.l3.lo.codice_lo_l3a_partial_densities_dependencies import ( + CodiceLoL3aPartialDensitiesDependencies, +) +from imap_l3_processing.codice.l3.lo.codice_lo_l3a_ratios_dependencies import ( + CodiceLoL3aRatiosDependencies, +) from imap_l3_processing.codice.l3.lo.codice_lo_processor import CodiceLoProcessor from imap_l3_processing.codice.l3.lo.models import CodiceLoL2SWSpeciesData -from imap_l3_processing.codice.l3.lo.sectored_intensities.science.mass_per_charge_lookup import MassPerChargeLookup +from imap_l3_processing.codice.l3.lo.sectored_intensities.science.mass_per_charge_lookup import ( + MassPerChargeLookup, +) from imap_l3_processing.constants import ONE_AU_IN_KM, TEMP_CDF_FOLDER_PATH from imap_l3_processing.glows.glows_processor import GlowsProcessor from imap_l3_processing.glows.l3a.glows_l3a_dependencies import GlowsL3ADependencies -from imap_l3_processing.glows.l3a.utils import read_l2_glows_data, create_glows_l3a_dictionary_from_cdf +from imap_l3_processing.glows.l3a.utils import ( + read_l2_glows_data, + create_glows_l3a_dictionary_from_cdf, +) from imap_l3_processing.glows.l3bc.glows_l3bc_dependencies import GlowsL3BCDependencies from imap_l3_processing.glows.l3d.glows_l3d_dependencies import GlowsL3DDependencies from imap_l3_processing.glows.l3e.glows_l3e_dependencies import GlowsL3EDependencies from imap_l3_processing.hi.hi_processor import HiProcessor -from imap_l3_processing.hi.hi_spectral_fit_dependencies import HiSpectralIndexDependencies -from imap_l3_processing.hit.l3.hit_l3_sectored_dependencies import HITL3SectoredDependencies +from imap_l3_processing.hi.hi_spectral_fit_dependencies import ( + HiSpectralIndexDependencies, +) +from imap_l3_processing.hit.l3.hit_l3_sectored_dependencies import ( + HITL3SectoredDependencies, +) from imap_l3_processing.hit.l3.hit_processor import HitProcessor from imap_l3_processing.hit.l3.models import HitL1Data from imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies import HitL3PhaDependencies -from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import CosineCorrectionLookupTable +from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import ( + CosineCorrectionLookupTable, +) from imap_l3_processing.hit.l3.pha.science.gain_lookup_table import GainLookupTable -from imap_l3_processing.hit.l3.pha.science.hit_event_type_lookup import HitEventTypeLookup +from imap_l3_processing.hit.l3.pha.science.hit_event_type_lookup import ( + HitEventTypeLookup, +) from imap_l3_processing.hit.l3.pha.science.range_fit_lookup import RangeFitLookup from imap_l3_processing.hit.l3.utils import read_l2_hit_data -from imap_l3_processing.lo.l3.lo_l3_spectral_fit_dependencies import LoL3SpectralFitDependencies +from imap_l3_processing.lo.l3.lo_l3_spectral_fit_dependencies import ( + LoL3SpectralFitDependencies, +) from imap_l3_processing.lo.lo_processor import perform_spectral_fit, LoProcessor -from imap_l3_processing.maps.hilo_l3_survival_dependencies import HiLoL3SurvivalDependencies, \ - HiL3SingleSensorFullSpinDependencies -from imap_l3_processing.maps.map_models import RectangularSpectralIndexDataProduct, RectangularIntensityDataProduct, \ - RectangularIntensityMapData +from imap_l3_processing.maps.hilo_l3_survival_dependencies import ( + HiLoL3SurvivalDependencies, + HiL3SingleSensorFullSpinDependencies, +) +from imap_l3_processing.maps.map_models import ( + RectangularSpectralIndexDataProduct, + RectangularIntensityDataProduct, + RectangularIntensityMapData, +) from imap_l3_processing.models import InputMetadata -from imap_l3_processing.swapi.l3a.science.pickup_ion.density_of_neutral_helium_lookup_table import \ - DensityOfNeutralHeliumLookupTable +from imap_l3_processing.swapi.l3a.science.pickup_ion.density_of_neutral_helium_lookup_table import ( + DensityOfNeutralHeliumLookupTable, +) from imap_l3_processing.swapi.l3a.swapi_l3a_dependencies import SwapiL3ADependencies from imap_l3_processing.swapi.l3a.utils import read_l2_swapi_data -from imap_l3_processing.swapi.response.efficiency_calibration_table import EfficiencyCalibrationTable -from imap_l3_processing.swapi.l3b.science.geometric_factor_calibration_table import GeometricFactorCalibrationTable +from imap_l3_processing.swapi.response.efficiency_calibration_table import ( + EfficiencyCalibrationTable, +) +from imap_l3_processing.swapi.l3b.science.geometric_factor_calibration_table import ( + GeometricFactorCalibrationTable, +) from imap_l3_processing.swapi.l3b.swapi_l3b_dependencies import SwapiL3BDependencies from imap_l3_processing.swapi.swapi_processor import SwapiProcessor from imap_l3_processing.swe.l3.swe_l3_dependencies import SweL3Dependencies @@ -79,30 +117,45 @@ ) from imap_l3_processing.utils import save_data, read_mag_data, furnish_local_spice from scripts.codice.create_fake_efficiency_ancillary import create_efficiency_lookup -from scripts.codice.create_more_accurate_l3a_direct_event import create_more_accurate_l3a_direct_events_cdf -from scripts.codice.create_more_accurate_l3a_direct_event_input import modify_l1a_priority_counts, \ - modify_l2_direct_events +from scripts.codice.create_more_accurate_l3a_direct_event import ( + create_more_accurate_l3a_direct_events_cdf, +) +from scripts.codice.create_more_accurate_l3a_direct_event_input import ( + modify_l1a_priority_counts, + modify_l2_direct_events, +) from scripts.hi.create_hi_full_spin_deps import create_hi_full_spin_deps -from scripts.ultra.create_l1c_l2_and_glows_with_matching_date_range import create_l1c_and_glows_with_matching_date_range -from tests.test_helpers import get_test_data_path, get_test_instrument_team_data_path, try_get_many_run_local_paths, \ - get_imap_data_dir_path +from scripts.ultra.create_l1c_l2_and_glows_with_matching_date_range import ( + create_l1c_and_glows_with_matching_date_range, +) +from tests.test_helpers import ( + get_test_data_path, + get_test_instrument_team_data_path, + try_get_many_run_local_paths, + get_imap_data_dir_path, +) def create_codice_lo_l3a_partial_densities_cdf(): codice_lo_l2_data = CodiceLoL2SWSpeciesData.read_from_cdf( - get_test_instrument_team_data_path('codice/lo/imap_codice_l2_lo-sw-species_20241110_v002.cdf')) + get_test_instrument_team_data_path( + "codice/lo/imap_codice_l2_lo-sw-species_20241110_v002.cdf" + ) + ) mpc_lookup = MassPerChargeLookup.read_from_file( - get_test_data_path('codice/imap_codice_mass-per-charge_20241110_v002.csv')) - deps = CodiceLoL3aPartialDensitiesDependencies(codice_l2_lo_data=codice_lo_l2_data, - mass_per_charge_lookup=mpc_lookup) + get_test_data_path("codice/imap_codice_mass-per-charge_20241110_v002.csv") + ) + deps = CodiceLoL3aPartialDensitiesDependencies( + codice_l2_lo_data=codice_lo_l2_data, mass_per_charge_lookup=mpc_lookup + ) input_metadata = InputMetadata( - instrument='codice', - data_level='l3a', + instrument="codice", + data_level="l3a", start_date=datetime(2024, 11, 10), end_date=datetime(2025, 1, 2), - version='v000', - descriptor='lo-partial-densities' + version="v000", + descriptor="lo-partial-densities", ) codice_lo_processor = CodiceLoProcessor(ProcessingInputCollection(), input_metadata) @@ -111,34 +164,51 @@ def create_codice_lo_l3a_partial_densities_cdf(): return cdf_path -def create_codice_lo_l3a_direct_events_cdf(l1a_paths: Optional[tuple[Path, Path]] = None) -> Path: +def create_codice_lo_l3a_direct_events_cdf( + l1a_paths: Optional[tuple[Path, Path]] = None, +) -> Path: codice_lo_l2_direct_event_cdf_path = modify_l2_direct_events( - get_test_instrument_team_data_path('codice/lo/imap_codice_l2_lo-direct-events_20241110_v002.cdf')) + get_test_instrument_team_data_path( + "codice/lo/imap_codice_l2_lo-direct-events_20241110_v002.cdf" + ) + ) if l1a_paths is None: - codice_lo_l1a_nsw_priority_path, codice_lo_l1a_sw_priority_path = modify_l1a_priority_counts( - get_test_instrument_team_data_path('codice/lo/imap_codice_l1a_lo-nsw-priority_20241110_v002.cdf'), - get_test_instrument_team_data_path('codice/lo/imap_codice_l1a_lo-sw-priority_20241110_v002.cdf')) + codice_lo_l1a_nsw_priority_path, codice_lo_l1a_sw_priority_path = ( + modify_l1a_priority_counts( + get_test_instrument_team_data_path( + "codice/lo/imap_codice_l1a_lo-nsw-priority_20241110_v002.cdf" + ), + get_test_instrument_team_data_path( + "codice/lo/imap_codice_l1a_lo-sw-priority_20241110_v002.cdf" + ), + ) + ) else: codice_lo_l1a_nsw_priority_path, codice_lo_l1a_sw_priority_path = l1a_paths - energy_lookup_path = get_test_data_path('codice/imap_codice_lo-energy-per-charge_20241110_v002.csv') - mass_coefficient_path = get_test_data_path('codice/imap_codice_mass-coefficient-lookup_20241110_v002.csv') + energy_lookup_path = get_test_data_path( + "codice/imap_codice_lo-energy-per-charge_20241110_v002.csv" + ) + mass_coefficient_path = get_test_data_path( + "codice/imap_codice_mass-coefficient-lookup_20241110_v002.csv" + ) deps = CodiceLoL3aDirectEventsDependencies.from_file_paths( sw_priority_rates_cdf=codice_lo_l1a_sw_priority_path, nsw_priority_rates_cdf=codice_lo_l1a_nsw_priority_path, direct_event_path=codice_lo_l2_direct_event_cdf_path, mass_coefficients_file_path=mass_coefficient_path, - esa_to_energy_per_charge_file_path=energy_lookup_path, ) + esa_to_energy_per_charge_file_path=energy_lookup_path, + ) input_metadata = InputMetadata( - instrument='codice', - data_level='l3a', + instrument="codice", + data_level="l3a", start_date=datetime(2024, 11, 10), end_date=datetime(2025, 1, 2), - version='v000', - descriptor='lo-direct-events' + version="v000", + descriptor="lo-direct-events", ) codice_lo_processor = CodiceLoProcessor(Mock(), input_metadata) @@ -152,12 +222,12 @@ def create_codice_lo_l3a_ratios_cdf(): deps = CodiceLoL3aRatiosDependencies.from_file_paths(partial_densities_file) input_metadata = InputMetadata( - instrument='codice', - data_level='l3a', + instrument="codice", + data_level="l3a", start_date=datetime(2024, 11, 10), end_date=datetime(2025, 1, 2), - version='v000', - descriptor='lo-sw-ratios' + version="v000", + descriptor="lo-sw-ratios", ) codice_lo_processor = CodiceLoProcessor(Mock(), input_metadata) @@ -171,12 +241,12 @@ def create_codice_lo_l3a_abundances_cdf(): deps = CodiceLoL3aRatiosDependencies.from_file_paths(partial_densities_file) input_metadata = InputMetadata( - instrument='codice', - data_level='l3a', + instrument="codice", + data_level="l3a", start_date=datetime(2024, 11, 10), end_date=datetime(2025, 1, 2), - version='v000', - descriptor='lo-sw-charge-state-distributions' + version="v000", + descriptor="lo-sw-charge-state-distributions", ) codice_lo_processor = CodiceLoProcessor(Mock(), input_metadata) @@ -186,12 +256,20 @@ def create_codice_lo_l3a_abundances_cdf(): def create_codice_lo_l3b_3d_distributions_cdf(species: str): l1a_paths = modify_l1a_priority_counts( - get_test_instrument_team_data_path('codice/lo/imap_codice_l1a_lo-nsw-priority_20241110_v002.cdf'), - get_test_instrument_team_data_path('codice/lo/imap_codice_l1a_lo-sw-priority_20241110_v002.cdf')) + get_test_instrument_team_data_path( + "codice/lo/imap_codice_l1a_lo-nsw-priority_20241110_v002.cdf" + ), + get_test_instrument_team_data_path( + "codice/lo/imap_codice_l1a_lo-sw-priority_20241110_v002.cdf" + ), + ) - codice_lo_l3a_direct_event_path = Path(create_codice_lo_l3a_direct_events_cdf(l1a_paths)) - accurate_codice_lo_l3a_direct_event_path = create_more_accurate_l3a_direct_events_cdf( - codice_lo_l3a_direct_event_path) + codice_lo_l3a_direct_event_path = Path( + create_codice_lo_l3a_direct_events_cdf(l1a_paths) + ) + accurate_codice_lo_l3a_direct_event_path = ( + create_more_accurate_l3a_direct_events_cdf(codice_lo_l3a_direct_event_path) + ) codice_lo_l1a_nsw_priority_path, codice_lo_l1a_sw_priority_path = l1a_paths @@ -199,20 +277,30 @@ def create_codice_lo_l3b_3d_distributions_cdf(species: str): l3a_file_path=accurate_codice_lo_l3a_direct_event_path, l1a_sw_file_path=codice_lo_l1a_sw_priority_path, l1a_nsw_file_path=codice_lo_l1a_nsw_priority_path, - mass_species_bin_lut=(get_test_data_path('codice/imap_codice_lo-mass-species-bin-lookup_20241110_v001.csv')), - geometric_factors_lut=(get_test_data_path('codice/imap_codice_lo-geometric-factors_20241110_v001.csv')), + mass_species_bin_lut=( + get_test_data_path( + "codice/imap_codice_lo-mass-species-bin-lookup_20241110_v001.csv" + ) + ), + geometric_factors_lut=( + get_test_data_path( + "codice/imap_codice_lo-geometric-factors_20241110_v001.csv" + ) + ), efficiency_factors_lut=(create_efficiency_lookup(species)), - energy_per_charge_lut=get_test_data_path("codice/imap_codice_lo-energy-per-charge_20241110_v002.csv"), - species=species + energy_per_charge_lut=get_test_data_path( + "codice/imap_codice_lo-energy-per-charge_20241110_v002.csv" + ), + species=species, ) input_metadata = InputMetadata( - instrument='codice', - data_level='l3b', + instrument="codice", + data_level="l3b", start_date=datetime(2024, 11, 10), end_date=datetime(2025, 1, 2), - version='v000', - descriptor=f'lo-{species}-3d-distribution' + version="v000", + descriptor=f"lo-{species}-3d-distribution", ) codice_lo_processor = CodiceLoProcessor(Mock(), input_metadata) @@ -220,20 +308,27 @@ def create_codice_lo_l3b_3d_distributions_cdf(species: str): return save_data(l3b_3d_distributions, delete_if_present=True) -def create_swapi_l3b_cdf(geometric_calibration_file, efficiency_calibration_file, cdf_file): - geometric_calibration = GeometricFactorCalibrationTable.from_file(geometric_calibration_file) +def create_swapi_l3b_cdf( + geometric_calibration_file, efficiency_calibration_file, cdf_file +): + geometric_calibration = GeometricFactorCalibrationTable.from_file( + geometric_calibration_file + ) efficiency_calibration = EfficiencyCalibrationTable(efficiency_calibration_file) swapi_cdf_data = CDF(cdf_file) swapi_data = read_l2_swapi_data(swapi_cdf_data) - swapi_l3_dependencies = SwapiL3BDependencies(swapi_data, geometric_calibration, efficiency_calibration) + swapi_l3_dependencies = SwapiL3BDependencies( + swapi_data, geometric_calibration, efficiency_calibration + ) input_metadata = InputMetadata( - instrument='swapi', - data_level='l3b', + instrument="swapi", + data_level="l3b", start_date=datetime(2010, 1, 1), end_date=datetime(2010, 1, 2), - version='v000') + version="v000", + ) processor = SwapiProcessor(Mock(), input_metadata) l3b_combined_vdf = processor.process_l3b(swapi_data, swapi_l3_dependencies) @@ -242,33 +337,46 @@ def create_swapi_l3b_cdf(geometric_calibration_file, efficiency_calibration_file @patch("imap_l3_processing.swapi.l3a.science.calculate_pickup_ion.spiceypy") -def create_swapi_l3a_cdf(geometric_factor_calibration_file, - density_of_neutral_helium_calibration_file, - imap_swapi_efficiency_lut_file, cdf_file, mock_spice): +def create_swapi_l3a_cdf( + geometric_factor_calibration_file, + density_of_neutral_helium_calibration_file, + imap_swapi_efficiency_lut_file, + cdf_file, + mock_spice, +): ephemeris_time_for_epoch = int(100000 * 1e9) mock_spice.unitim.return_value = ephemeris_time_for_epoch mock_light_time = 122.0 mock_spice.spkezr.return_value = (np.array([0, 0, 0, 0, 0, 0]), mock_light_time) mock_spice.latrec.return_value = np.array([0, 2, 0]) - mock_spice.reclat.return_value = np.array([0.99 * ONE_AU_IN_KM, np.deg2rad(255.7), 0.6]) + mock_spice.reclat.return_value = np.array( + [0.99 * ONE_AU_IN_KM, np.deg2rad(255.7), 0.6] + ) def mock_sxform(from_frame, to_frame, et): if from_frame == "IMAP_SWAPI": return np.eye(6) - return np.array([ - [0, 1, 0, 0, 0, 0], - [0, 0, 1, 0, 0, 0], - [1, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 1, 0], - [0, 0, 0, 0, 0, 1], - [0, 0, 0, 1, 0, 0], - ]) + return np.array( + [ + [0, 1, 0, 0, 0, 0], + [0, 0, 1, 0, 0, 0], + [1, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 1, 0], + [0, 0, 0, 0, 0, 1], + [0, 0, 0, 1, 0, 0], + ] + ) mock_spice.sxform.side_effect = mock_sxform - efficiency_calibration_table = EfficiencyCalibrationTable(imap_swapi_efficiency_lut_file) - density_of_neutral_helium_calibration_table = DensityOfNeutralHeliumLookupTable.from_file( - density_of_neutral_helium_calibration_file) + efficiency_calibration_table = EfficiencyCalibrationTable( + imap_swapi_efficiency_lut_file + ) + density_of_neutral_helium_calibration_table = ( + DensityOfNeutralHeliumLookupTable.from_file( + density_of_neutral_helium_calibration_file + ) + ) swapi_cdf_data = CDF(cdf_file) swapi_data = read_l2_swapi_data(swapi_cdf_data) swapi_l3_dependencies = SwapiL3ADependencies( @@ -281,11 +389,12 @@ def mock_sxform(from_frame, to_frame, et): ) input_metadata = InputMetadata( - instrument='swapi', - data_level='l3a', + instrument="swapi", + data_level="l3a", start_date=datetime(2025, 10, 23), end_date=datetime(2025, 10, 24), - version='v000') + version="v000", + ) processor = SwapiProcessor(Mock(), input_metadata) l3a_proton_sw = processor.process_l3a_proton(swapi_data, swapi_l3_dependencies) @@ -299,11 +408,12 @@ def mock_sxform(from_frame, to_frame, et): def create_swe_product(dependencies: SweL3Dependencies) -> Path: input_metadata = InputMetadata( - instrument='swe', - data_level='l3', + instrument="swe", + data_level="l3", start_date=datetime(2025, 6, 29), end_date=datetime(2025, 7, 1), - version='v000') + version="v000", + ) processor = SweProcessor(Mock(), input_metadata) output_data = processor.calculate_products(dependencies) cdf_path = save_data(output_data, delete_if_present=True) @@ -311,20 +421,26 @@ def create_swe_product(dependencies: SweL3Dependencies) -> Path: @patch("imap_l3_processing.swe.l3.science.moment_calculations.spiceypy.pxform") -def create_swe_product_with_fake_spice(dependencies: SweL3Dependencies, mock_spice_pxform) -> Path: +def create_swe_product_with_fake_spice( + dependencies: SweL3Dependencies, mock_spice_pxform +) -> Path: data = np.loadtxt(get_test_data_path("swe/ace_attitude.dat"), skiprows=1) def time_to_float(t): return (t - datetime(1999, 1, 1, tzinfo=timezone.utc)).total_seconds() - times = [datetime(1998, 12, 31, tzinfo=timezone.utc) + timedelta(days=row[1], seconds=row[2]) for row in data] + times = [ + datetime(1998, 12, 31, tzinfo=timezone.utc) + + timedelta(days=row[1], seconds=row[2]) + for row in data + ] times_as_floats = [time_to_float(t) for t in times] def fake_pxform(from_frame, to_frame, et): assert from_frame == "IMAP_DPS" assert to_frame == "IMAP_RTN" time = spiceypy.et2datetime(et) - correction_factor = (datetime(2025, 6, 30) - datetime(1999, 6, 8)) + correction_factor = datetime(2025, 6, 30) - datetime(1999, 6, 8) time_in_1999 = time - correction_factor r = np.interp(time_to_float(time_in_1999), times_as_floats, data[:, 4]) t = np.interp(time_to_float(time_in_1999), times_as_floats, data[:, 5]) @@ -338,77 +454,98 @@ def fake_pxform(from_frame, to_frame, et): c2 = np.sqrt(t2 + n2) c1 = np.sqrt((t2 + n2) * (t2 + n2) + r2 * t2 + r2 * n2) - mat = np.array([[(-n2 - t2) / c1, 0, r / c3], [r * t / c1, -n / c2, t / c3], [r * n / c1, t / c2, n / c3]]) + mat = np.array( + [ + [(-n2 - t2) / c1, 0, r / c3], + [r * t / c1, -n / c2, t / c3], + [r * n / c1, t / c2, n / c3], + ] + ) return mat mock_spice_pxform.side_effect = fake_pxform input_metadata = InputMetadata( - instrument='swe', - data_level='l3', + instrument="swe", + data_level="l3", start_date=datetime(2025, 6, 29), end_date=datetime(2025, 7, 1), - version='v000') + version="v000", + ) processor = SweProcessor(Mock(), input_metadata) output_data = processor.calculate_products(dependencies) cdf_path = save_data(output_data, delete_if_present=True) return cdf_path -def create_survival_corrected_full_spin_cdf(dependencies: HiL3SingleSensorFullSpinDependencies) -> Path: - input_metadata = InputMetadata(instrument="hi", - data_level="l3", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="v000", - descriptor="h90-ena-h-sf-sp-full-hae-4deg-1yr", - ) +def create_survival_corrected_full_spin_cdf( + dependencies: HiL3SingleSensorFullSpinDependencies, +) -> Path: + input_metadata = InputMetadata( + instrument="hi", + data_level="l3", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="v000", + descriptor="h90-ena-h-sf-sp-full-hae-4deg-1yr", + ) processor = HiProcessor(Mock(), input_metadata) - output_data = processor.process_full_spin_single_sensor(dependencies, SpiceFrame.ECLIPJ2000) + output_data = processor.process_full_spin_single_sensor( + dependencies, SpiceFrame.ECLIPJ2000 + ) - data_product = RectangularIntensityDataProduct(data=output_data, input_metadata=input_metadata) + data_product = RectangularIntensityDataProduct( + data=output_data, input_metadata=input_metadata + ) cdf_path = save_data(data_product, delete_if_present=True) return cdf_path def create_hi_spectral_index_cdf(dependencies: HiSpectralIndexDependencies) -> Path: - input_metadata = InputMetadata(instrument="hi", - data_level="l3", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="v000", - descriptor="h90-spx-h-hf-sp-full-hae-4deg-6mo", - ) + input_metadata = InputMetadata( + instrument="hi", + data_level="l3", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="v000", + descriptor="h90-spx-h-hf-sp-full-hae-4deg-6mo", + ) processor = HiProcessor(Mock(), input_metadata) output_data = processor.process_spectral_fit_index(dependencies) - data_product = RectangularSpectralIndexDataProduct(data=output_data, input_metadata=input_metadata) + data_product = RectangularSpectralIndexDataProduct( + data=output_data, input_metadata=input_metadata + ) cdf_path = save_data(data_product, delete_if_present=True) return cdf_path def create_lo_spectral_index_cdf(dependencies: LoL3SpectralFitDependencies) -> Path: - input_metadata = InputMetadata(instrument="lo", - data_level="l3", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="v000", - descriptor="l090-spx-h-hf-sp-ram-hae-6deg-1yr", - ) + input_metadata = InputMetadata( + instrument="lo", + data_level="l3", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="v000", + descriptor="l090-spx-h-hf-sp-ram-hae-6deg-1yr", + ) output_data = perform_spectral_fit(dependencies.map_data) - data_product = RectangularSpectralIndexDataProduct(data=output_data, input_metadata=input_metadata) + data_product = RectangularSpectralIndexDataProduct( + data=output_data, input_metadata=input_metadata + ) cdf_path = save_data(data_product, delete_if_present=True) return cdf_path def create_hit_sectored_cdf(dependencies: HITL3SectoredDependencies) -> Path: input_metadata = InputMetadata( - instrument='hit', - data_level='l3', - descriptor='macropixel', + instrument="hit", + data_level="l3", + descriptor="macropixel", start_date=datetime(2025, 1, 1), end_date=datetime(2025, 1, 2), - version='v000') + version="v000", + ) processor = HitProcessor(Mock(), input_metadata) output_data = processor.process_pitch_angle_product(dependencies) cdf_path = save_data(output_data, delete_if_present=True) @@ -417,42 +554,75 @@ def create_hit_sectored_cdf(dependencies: HITL3SectoredDependencies) -> Path: def create_hit_direct_event_cdf(): cosine_table = CosineCorrectionLookupTable( - get_test_data_path("hit/pha_events/imap_hit_range-2A-cosine-lookup_20250203_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-2B-cosine-lookup_20250203_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-3A-cosine-lookup_20250203_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-3B-cosine-lookup_20250203_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-4A-cosine-lookup_20250203_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-4B-cosine-lookup_20250203_v000.csv"), + get_test_data_path( + "hit/pha_events/imap_hit_range-2A-cosine-lookup_20250203_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-2B-cosine-lookup_20250203_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-3A-cosine-lookup_20250203_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-3B-cosine-lookup_20250203_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-4A-cosine-lookup_20250203_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-4B-cosine-lookup_20250203_v000.csv" + ), ) gain_table = GainLookupTable.from_file( get_test_data_path("hit/pha_events/imap_hit_hi-gain-lookup_20250203_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_lo-gain-lookup_20250203_v000.csv")) + get_test_data_path("hit/pha_events/imap_hit_lo-gain-lookup_20250203_v000.csv"), + ) range_fit_lookup = RangeFitLookup.from_files( - get_test_data_path("hit/pha_events/imap_hit_range-2A-charge-fit-lookup_20250319_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-3A-charge-fit-lookup_20250319_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-4A-charge-fit-lookup_20250319_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-2B-charge-fit-lookup_20250319_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-3B-charge-fit-lookup_20250319_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-4B-charge-fit-lookup_20250319_v000.csv"), + get_test_data_path( + "hit/pha_events/imap_hit_range-2A-charge-fit-lookup_20250319_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-3A-charge-fit-lookup_20250319_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-4A-charge-fit-lookup_20250319_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-2B-charge-fit-lookup_20250319_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-3B-charge-fit-lookup_20250319_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-4B-charge-fit-lookup_20250319_v000.csv" + ), ) event_type_look = HitEventTypeLookup.from_csv( - get_test_data_path("hit/pha_events/imap_hit_hit-event-type-lookup_20250228_v000.csv")) - - hit_l1_data = HitL1Data.read_from_cdf("data/imap/hit/l1a/2010/01/imap_hit_l1a_direct-events_20100105_v001.cdf") + get_test_data_path( + "hit/pha_events/imap_hit_hit-event-type-lookup_20250228_v000.csv" + ) + ) - direct_event_dependencies = HitL3PhaDependencies(hit_l1_data=hit_l1_data, cosine_correction_lookup=cosine_table, + hit_l1_data = HitL1Data.read_from_cdf( + "data/imap/hit/l1a/2010/01/imap_hit_l1a_direct-events_20100105_v001.cdf" + ) - gain_lookup=gain_table, range_fit_lookup=range_fit_lookup, - event_type_lookup=event_type_look) + direct_event_dependencies = HitL3PhaDependencies( + hit_l1_data=hit_l1_data, + cosine_correction_lookup=cosine_table, + gain_lookup=gain_table, + range_fit_lookup=range_fit_lookup, + event_type_lookup=event_type_look, + ) input_metadata = InputMetadata( instrument="hit", data_level="l3", start_date=datetime.now(), end_date=datetime.now() + timedelta(days=1), version="v001", - descriptor="direct-events" + descriptor="direct-events", ) processor = HitProcessor(Mock(), input_metadata) @@ -461,93 +631,144 @@ def create_hit_direct_event_cdf(): return file_path -@patch('imap_l3_processing.glows.glows_initializer.query') -@patch('imap_l3_processing.glows.l3bc.glows_l3bc_dependencies.download_dependency_from_path') -@patch('imap_l3_processing.glows.glows_processor.imap_data_access') -@patch('imap_l3_processing.glows.l3bc.glows_initializer_ancillary_dependencies.download') -@patch('imap_l3_processing.glows.l3bc.glows_initializer_ancillary_dependencies.download_external_dependency') -def run_l3b_initializer(mock_download_external, mock_download, mock_imap_data_access, - mock_download_dependency_from_path, mock_query): +@patch("imap_l3_processing.glows.glows_initializer.query") +@patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_dependencies.download_dependency_from_path" +) +@patch("imap_l3_processing.glows.glows_processor.imap_data_access") +@patch( + "imap_l3_processing.glows.l3bc.glows_initializer_ancillary_dependencies.download" +) +@patch( + "imap_l3_processing.glows.l3bc.glows_initializer_ancillary_dependencies.download_external_dependency" +) +def run_l3b_initializer( + mock_download_external, + mock_download, + mock_imap_data_access, + mock_download_dependency_from_path, + mock_query, +): local_cdfs: list[str] = os.listdir(get_test_data_path("glows/pipeline/l3a")) - l3a_dicts = [{'file_path': f"imap/glows/l3a/{file_path}", - 'start_date': file_path.split('_')[4].split('-')[0], - 'repointing': int(file_path.split('_')[4].split('-repoint')[1]) - } for file_path in local_cdfs] - - mock_query.side_effect = [ - l3a_dicts, [] + l3a_dicts = [ + { + "file_path": f"imap/glows/l3a/{file_path}", + "start_date": file_path.split("_")[4].split("-")[0], + "repointing": int(file_path.split("_")[4].split("-repoint")[1]), + } + for file_path in local_cdfs ] + mock_query.side_effect = [l3a_dicts, []] + mock_download.side_effect = [ - get_test_data_path('glows/imap_glows_pipeline-settings-l3bcde_20250423_v001.json'), - get_test_data_path('glows/imap_2026_105_01.repoint.csv'), + get_test_data_path( + "glows/imap_glows_pipeline-settings-l3bcde_20250423_v001.json" + ), + get_test_data_path("glows/imap_2026_105_01.repoint.csv"), ] mock_download_external.side_effect = [ - TEMP_CDF_FOLDER_PATH / 'f107_fluxtable.txt', - TEMP_CDF_FOLDER_PATH / 'lyman_alpha_composite.nc', + TEMP_CDF_FOLDER_PATH / "f107_fluxtable.txt", + TEMP_CDF_FOLDER_PATH / "lyman_alpha_composite.nc", Path( - r'C:\Users\Harrison\Downloads\glows_L3bc_pipeline_2025_04_11\glows_L3bc_pipeline_2025_04_11\external_dependencies\omni2_all_years.dat') + r"C:\Users\Harrison\Downloads\glows_L3bc_pipeline_2025_04_11\glows_L3bc_pipeline_2025_04_11\external_dependencies\omni2_all_years.dat" + ), ] mock_download_dependency_from_path.side_effect = [ - get_test_data_path('glows/imap_glows_uv-anisotropy-1CR_20100101_v001.json'), - get_test_data_path('glows/imap_glows_WawHelioIonMP_20100101_v002.json'), - get_test_data_path('glows/imap_glows_bad-days-list_20100101_v004.dat'), - get_test_data_path('glows/imap_glows_pipeline-settings-l3bcde_20250423_v001.json'), - get_test_data_path('glows/pipeline/l3a/imap_glows_l3a_hist_20250625-repoint01071_v001.cdf'), - get_test_data_path('glows/pipeline/l3a/imap_glows_l3a_hist_20250626-repoint01072_v001.cdf'), - get_test_data_path('glows/imap_glows_uv-anisotropy-1CR_20100101_v001.json'), - get_test_data_path('glows/imap_glows_WawHelioIonMP_20100101_v002.json'), - get_test_data_path('glows/imap_glows_bad-days-list_20100101_v004.dat'), - get_test_data_path('glows/imap_glows_pipeline-settings-l3bcde_20250423_v001.json'), - get_test_data_path('glows/pipeline/l3a/imap_glows_l3a_hist_20100521-repoint00289_v001.cdf'), - get_test_data_path('glows/pipeline/l3a/imap_glows_l3a_hist_20100522-repoint00290_v001.cdf'), + get_test_data_path("glows/imap_glows_uv-anisotropy-1CR_20100101_v001.json"), + get_test_data_path("glows/imap_glows_WawHelioIonMP_20100101_v002.json"), + get_test_data_path("glows/imap_glows_bad-days-list_20100101_v004.dat"), + get_test_data_path( + "glows/imap_glows_pipeline-settings-l3bcde_20250423_v001.json" + ), + get_test_data_path( + "glows/pipeline/l3a/imap_glows_l3a_hist_20250625-repoint01071_v001.cdf" + ), + get_test_data_path( + "glows/pipeline/l3a/imap_glows_l3a_hist_20250626-repoint01072_v001.cdf" + ), + get_test_data_path("glows/imap_glows_uv-anisotropy-1CR_20100101_v001.json"), + get_test_data_path("glows/imap_glows_WawHelioIonMP_20100101_v002.json"), + get_test_data_path("glows/imap_glows_bad-days-list_20100101_v004.dat"), + get_test_data_path( + "glows/imap_glows_pipeline-settings-l3bcde_20250423_v001.json" + ), + get_test_data_path( + "glows/pipeline/l3a/imap_glows_l3a_hist_20100521-repoint00289_v001.cdf" + ), + get_test_data_path( + "glows/pipeline/l3a/imap_glows_l3a_hist_20100522-repoint00290_v001.cdf" + ), ] - bad_days_list = AncillaryInput('imap_glows_bad-days-list_20100101_v004.dat') - waw_helio_ion = AncillaryInput('imap_glows_WawHelioIonMP_20100101_v002.json') - uv_anisotropy = AncillaryInput('imap_glows_uv-anisotropy-1CR_20100101_v001.json') - pipeline_settings = AncillaryInput('imap_glows_pipeline-settings-l3bcde_20250423_v001.json') + bad_days_list = AncillaryInput("imap_glows_bad-days-list_20100101_v004.dat") + waw_helio_ion = AncillaryInput("imap_glows_WawHelioIonMP_20100101_v002.json") + uv_anisotropy = AncillaryInput("imap_glows_uv-anisotropy-1CR_20100101_v001.json") + pipeline_settings = AncillaryInput( + "imap_glows_pipeline-settings-l3bcde_20250423_v001.json" + ) - repointings = RepointInput('imap_2026_105_01.repoint.csv') + repointings = RepointInput("imap_2026_105_01.repoint.csv") - input_collection = ProcessingInputCollection(bad_days_list, waw_helio_ion, uv_anisotropy, pipeline_settings, - repointings) - input_metadata = InputMetadata(instrument='glows', data_level='l3b', - start_date=datetime.fromisoformat('2010-01-05T00:00:00'), - end_date=datetime.fromisoformat('2025-01-06T00:00:00'), - version='v001') + input_collection = ProcessingInputCollection( + bad_days_list, waw_helio_ion, uv_anisotropy, pipeline_settings, repointings + ) + input_metadata = InputMetadata( + instrument="glows", + data_level="l3b", + start_date=datetime.fromisoformat("2010-01-05T00:00:00"), + end_date=datetime.fromisoformat("2025-01-06T00:00:00"), + version="v001", + ) processor = GlowsProcessor(input_collection, input_metadata) print(processor.process()) -@patch('imap_l3_processing.glows.glows_initializer.query') +@patch("imap_l3_processing.glows.glows_initializer.query") def run_glows_l3bc_processor_and_initializer(mock_query): input_metadata = InputMetadata( - instrument='glows', - data_level='l3b', + instrument="glows", + data_level="l3b", start_date=datetime(2013, 9, 8), end_date=datetime(2013, 9, 8), - version='v011') + version="v011", + ) - l3a_files = imap_data_access.query(instrument='glows', version=input_metadata.version, data_level='l3a', - start_date='20100422', end_date='20100625') + l3a_files = imap_data_access.query( + instrument="glows", + version=input_metadata.version, + data_level="l3a", + start_date="20100422", + end_date="20100625", + ) - l3a_files_2 = imap_data_access.query(instrument='glows', version=input_metadata.version, data_level='l3a', - start_date='20100922', end_date='20101123') + l3a_files_2 = imap_data_access.query( + instrument="glows", + version=input_metadata.version, + data_level="l3a", + start_date="20100922", + end_date="20101123", + ) mock_query.side_effect = [l3a_files + l3a_files_2, []] - bad_days_list = AncillaryInput('imap_glows_bad-days-list_20100101_v001.dat') - waw_helio_ion = AncillaryInput('imap_glows_WawHelioIonMP_20100101_v002.json') - uv_anisotropy = AncillaryInput('imap_glows_uv-anisotropy-1CR_20300101_v003.json') - pipeline_settings = AncillaryInput('imap_glows_pipeline-settings-l3bcde_20100101_v006.json') - input_collection = ProcessingInputCollection(bad_days_list, waw_helio_ion, uv_anisotropy, pipeline_settings) + bad_days_list = AncillaryInput("imap_glows_bad-days-list_20100101_v001.dat") + waw_helio_ion = AncillaryInput("imap_glows_WawHelioIonMP_20100101_v002.json") + uv_anisotropy = AncillaryInput("imap_glows_uv-anisotropy-1CR_20300101_v003.json") + pipeline_settings = AncillaryInput( + "imap_glows_pipeline-settings-l3bcde_20100101_v006.json" + ) + input_collection = ProcessingInputCollection( + bad_days_list, waw_helio_ion, uv_anisotropy, pipeline_settings + ) - processor = GlowsProcessor(dependencies=input_collection, input_metadata=input_metadata) + processor = GlowsProcessor( + dependencies=input_collection, input_metadata=input_metadata + ) print(processor.process()) @@ -555,7 +776,9 @@ def run_glows_l3bc_processor_and_initializer(mock_query): @patch("imap_l3_processing.glows.glows_processor.Path") @patch("imap_l3_processing.glows.glows_processor.run") @patch("imap_l3_processing.glows.glows_processor.get_pointing_date_range") -def run_glows_l3e_lo_with_mocks(mock_get_repoint_date_range, _, mock_path, mock_l3e_dependencies_class): +def run_glows_l3e_lo_with_mocks( + mock_get_repoint_date_range, _, mock_path, mock_l3e_dependencies_class +): mock_processing_input_collection = Mock() mock_processing_input_collection.get_file_paths.return_value = [Path("one path")] @@ -565,56 +788,90 @@ def run_glows_l3e_lo_with_mocks(mock_get_repoint_date_range, _, mock_path, mock_ Path("instrument_team_data/glows/GLOWS_L3d_to_L3e_processing/EnGridUltra.dat"), Path("instrument_team_data/glows/GLOWS_L3d_to_L3e_processing/tessXYZ8.dat"), Path("instrument_team_data/glows/GLOWS_L3d_to_L3e_processing/tessAng16.dat"), - Path("instrument_team_data/glows/GLOWS_L3d_to_L3e_processing/lyaSeriesV4_2021b.dat"), Path( - "instrument_team_data/glows/GLOWS_L3d_to_L3e_processing/solar_uv_anisotropy_NP.1.0_SP.1.0.dat"), - Path("instrument_team_data/glows/GLOWS_L3d_to_L3e_processing/speed3D.v01.Legendre.2021b.dat"), - Path("instrument_team_data/glows/GLOWS_L3d_to_L3e_processing/density3D.v01.Legendre.2021b.dat"), - Path("instrument_team_data/glows/GLOWS_L3d_to_L3e_processing/phion_Hydrogen_T12F107_2021b.dat"), - Path("instrument_team_data/glows/GLOWS_L3d_to_L3e_processing/swEqtrElectrons5_2021b.dat"), - Path("instrument_team_data/glows/GLOWS_L3d_to_L3e_processing/ionization.files.dat"), - pipeline_settings={"executable_dependency_paths": { - "energy-grid-lo": "EnGridLo.dat", - "energy-grid-hi": "EnGridHi.dat", - "energy-grid-ultra": "EnGridUltra.dat", - "tess-xyz-8": "tessXYZ8.dat", - "tess-ang-16": "tessAng16.dat", - "lya-series": "lyaSeriesV4_2021b.dat", - "solar-uv-anistropy": "solar_uv_anisotropy_NP.1.0_SP.1.0.dat", - "speed-3d": "speed3D.v01.Legendre.2021b.dat", - "density-3d": "density3D.v01.Legendre.2021b.dat", - "phion-hydrogen": "phion_Hydrogen_T12F107_2021b.dat", - "sw-eqtr-electrons": "swEqtrElectrons5_2021b.dat", - "ionization-files": "ionization.files.dat", + "instrument_team_data/glows/GLOWS_L3d_to_L3e_processing/lyaSeriesV4_2021b.dat" + ), + Path( + "instrument_team_data/glows/GLOWS_L3d_to_L3e_processing/solar_uv_anisotropy_NP.1.0_SP.1.0.dat" + ), + Path( + "instrument_team_data/glows/GLOWS_L3d_to_L3e_processing/speed3D.v01.Legendre.2021b.dat" + ), + Path( + "instrument_team_data/glows/GLOWS_L3d_to_L3e_processing/density3D.v01.Legendre.2021b.dat" + ), + Path( + "instrument_team_data/glows/GLOWS_L3d_to_L3e_processing/phion_Hydrogen_T12F107_2021b.dat" + ), + Path( + "instrument_team_data/glows/GLOWS_L3d_to_L3e_processing/swEqtrElectrons5_2021b.dat" + ), + Path( + "instrument_team_data/glows/GLOWS_L3d_to_L3e_processing/ionization.files.dat" + ), + pipeline_settings={ + "executable_dependency_paths": { + "energy-grid-lo": "EnGridLo.dat", + "energy-grid-hi": "EnGridHi.dat", + "energy-grid-ultra": "EnGridUltra.dat", + "tess-xyz-8": "tessXYZ8.dat", + "tess-ang-16": "tessAng16.dat", + "lya-series": "lyaSeriesV4_2021b.dat", + "solar-uv-anistropy": "solar_uv_anisotropy_NP.1.0_SP.1.0.dat", + "speed-3d": "speed3D.v01.Legendre.2021b.dat", + "density-3d": "density3D.v01.Legendre.2021b.dat", + "phion-hydrogen": "phion_Hydrogen_T12F107_2021b.dat", + "sw-eqtr-electrons": "swEqtrElectrons5_2021b.dat", + "ionization-files": "ionization.files.dat", + }, + "start_cr": 2092, }, - "start_cr": 2092}, elongation=defaultdict(lambda: 90), repointing_file=get_test_data_path("fake_1_day_repointing_file.csv"), ) mock_l3e_dependencies.copy_dependencies = Mock() - mock_l3e_dependencies_class.fetch_dependencies.return_value = (mock_l3e_dependencies, 2094) + mock_l3e_dependencies_class.fetch_dependencies.return_value = ( + mock_l3e_dependencies, + 2094, + ) mock_path.side_effect = [ # Path(get_test_instrument_team_data_path("glows/probSur.Imap.Lo_20090101_010101_2009.000_60.00.txt")), - Path(get_test_instrument_team_data_path("glows/probSur.Imap.Hi_2009.000_135.0.txt")), - Path(get_test_instrument_team_data_path("glows/probSur.Imap.Hi_2009.000_90.00.txt")), - Path(get_test_instrument_team_data_path("glows/probSur.Imap.Ul_20090101_010101_2009.000.txt")), + Path( + get_test_instrument_team_data_path( + "glows/probSur.Imap.Hi_2009.000_135.0.txt" + ) + ), + Path( + get_test_instrument_team_data_path( + "glows/probSur.Imap.Hi_2009.000_90.00.txt" + ) + ), + Path( + get_test_instrument_team_data_path( + "glows/probSur.Imap.Ul_20090101_010101_2009.000.txt" + ) + ), ] input_metadata = InputMetadata( - instrument='glows', - data_level='l3e', + instrument="glows", + data_level="l3e", start_date=datetime(2015, 4, 10), end_date=datetime(2015, 4, 11), - version='v001', - descriptor='survival-probability-lo') + version="v001", + descriptor="survival-probability-lo", + ) mock_get_repoint_date_range.return_value = ( np.datetime64(datetime.fromisoformat("2025-04-20T00:00:00")), - np.datetime64(datetime.fromisoformat("2025-04-21T00:00:00"))) + np.datetime64(datetime.fromisoformat("2025-04-21T00:00:00")), + ) - glows_processor: GlowsProcessor = GlowsProcessor(mock_processing_input_collection, input_metadata) + glows_processor: GlowsProcessor = GlowsProcessor( + mock_processing_input_collection, input_metadata + ) glows_processor.process() @@ -624,7 +881,9 @@ def run_glows_l3e_with_less_mocks(mock_spiceypy): mock_spiceypy.spkezr = spiceypy.spkezr mock_spiceypy.reclat = spiceypy.reclat mock_spiceypy.pxform = spiceypy.pxform - mock_spiceypy.datetime2et = lambda date: spiceypy.datetime2et(date + timedelta(days=365 * 16 + 4, hours=2)) + mock_spiceypy.datetime2et = lambda date: spiceypy.datetime2et( + date + timedelta(days=365 * 16 + 4, hours=2) + ) l3d_file = "imap_glows_l3d_solar-hist_20100101-repoint02092_v002.cdf" lo_processing_input_collection = ProcessingInputCollection( @@ -684,36 +943,64 @@ def run_glows_l3e_with_less_mocks(mock_spiceypy): RepointInput("imap_2001_052_001.repoint.csv"), ) - version = 'v007' + version = "v007" start_date = datetime(2010, 1, 1) end_date = datetime(2010, 1, 2) - lo_input_metadata = InputMetadata(instrument='glows', data_level='l3e', start_date=start_date, - end_date=end_date, version=version, - descriptor='survival-probability-lo') + lo_input_metadata = InputMetadata( + instrument="glows", + data_level="l3e", + start_date=start_date, + end_date=end_date, + version=version, + descriptor="survival-probability-lo", + ) - glows_processor: GlowsProcessor = GlowsProcessor(lo_processing_input_collection, lo_input_metadata) + glows_processor: GlowsProcessor = GlowsProcessor( + lo_processing_input_collection, lo_input_metadata + ) glows_processor.process() - hi_45_input_metadata = InputMetadata(instrument='glows', data_level='l3e', start_date=start_date, - end_date=end_date, version=version, - descriptor='survival-probability-hi-45') + hi_45_input_metadata = InputMetadata( + instrument="glows", + data_level="l3e", + start_date=start_date, + end_date=end_date, + version=version, + descriptor="survival-probability-hi-45", + ) - glows_processor: GlowsProcessor = GlowsProcessor(hi_45_processing_input_collection, hi_45_input_metadata) + glows_processor: GlowsProcessor = GlowsProcessor( + hi_45_processing_input_collection, hi_45_input_metadata + ) glows_processor.process() - hi_90_input_metadata = InputMetadata(instrument='glows', data_level='l3e', start_date=start_date, - end_date=end_date, version=version, - descriptor='survival-probability-hi-90') + hi_90_input_metadata = InputMetadata( + instrument="glows", + data_level="l3e", + start_date=start_date, + end_date=end_date, + version=version, + descriptor="survival-probability-hi-90", + ) - glows_processor: GlowsProcessor = GlowsProcessor(hi_90_processing_input_collection, hi_90_input_metadata) + glows_processor: GlowsProcessor = GlowsProcessor( + hi_90_processing_input_collection, hi_90_input_metadata + ) glows_processor.process() - ul_input_metadata = InputMetadata(instrument='glows', data_level='l3e', start_date=start_date, - end_date=end_date, version=version, - descriptor='survival-probability-ul') + ul_input_metadata = InputMetadata( + instrument="glows", + data_level="l3e", + start_date=start_date, + end_date=end_date, + version=version, + descriptor="survival-probability-ul", + ) - glows_processor: GlowsProcessor = GlowsProcessor(ul_processing_input_collection, ul_input_metadata) + glows_processor: GlowsProcessor = GlowsProcessor( + ul_processing_input_collection, ul_input_metadata + ) glows_processor.process() @@ -724,73 +1011,106 @@ def make_glows_l3a_files(): def run_glows_l3a(file_path): - date_in_path = re.search(r"imap_glows_l2_hist_(\d{8}).*.cdf", file_path.name).group(1) + date_in_path = re.search(r"imap_glows_l2_hist_(\d{8}).*.cdf", file_path.name).group( + 1 + ) start_date = datetime.strptime(date_in_path, "%Y%m%d") end_date = start_date + timedelta(days=1) input_metadata = InputMetadata( - instrument='glows', - data_level='l3a', + instrument="glows", + data_level="l3a", start_date=start_date, end_date=end_date, - version='v001' + version="v001", ) with CDF(str(file_path)) as cdf_data: l2_glows_data = read_l2_glows_data(cdf_data) - repoint_number = int(file_path.name.split('_')[4][-5:]) + repoint_number = int(file_path.name.split("_")[4][-5:]) input_metadata.repointing = repoint_number - dependencies = GlowsL3ADependencies(l2_glows_data, { - "calibration_data": get_test_instrument_team_data_path( - "glows/imap_glows_calibration-data_20100101_v002.dat"), - "settings": get_test_instrument_team_data_path("glows/imap_glows_pipeline-settings_20100101_v001.json"), - "time_dependent_bckgrd": get_test_instrument_team_data_path( - "glows/imap_glows_time-dep-bckgrd_20100101_v001.dat"), - "extra_heliospheric_bckgrd": get_test_instrument_team_data_path( - "glows/imap_glows_map-of-extra-helio-bckgrd_20100101_v002.dat"), - }) + dependencies = GlowsL3ADependencies( + l2_glows_data, + { + "calibration_data": get_test_instrument_team_data_path( + "glows/imap_glows_calibration-data_20100101_v002.dat" + ), + "settings": get_test_instrument_team_data_path( + "glows/imap_glows_pipeline-settings_20100101_v001.json" + ), + "time_dependent_bckgrd": get_test_instrument_team_data_path( + "glows/imap_glows_time-dep-bckgrd_20100101_v001.dat" + ), + "extra_heliospheric_bckgrd": get_test_instrument_team_data_path( + "glows/imap_glows_map-of-extra-helio-bckgrd_20100101_v002.dat" + ), + }, + ) processor = GlowsProcessor(ProcessingInputCollection(), input_metadata) l3a_data = processor.process_l3a(dependencies) glows_l3a_temp_path = TEMP_CDF_FOLDER_PATH / "glows" / "l3a" - print(save_data(l3a_data, folder_path=glows_l3a_temp_path, delete_if_present=True)) + print( + save_data(l3a_data, folder_path=glows_l3a_temp_path, delete_if_present=True) + ) def run_glows_l3bc(): input_metadata = InputMetadata( - instrument='glows', - data_level='l3b', + instrument="glows", + data_level="l3b", start_date=datetime(2013, 9, 8), end_date=datetime(2013, 9, 8), - version='v001') + version="v001", + ) cr = 2091 external_files = { - 'f107_raw_data': get_test_instrument_team_data_path('glows/f107_fluxtable.txt'), - 'omni_raw_data': get_test_instrument_team_data_path('glows/omni2_all_years.dat') + "f107_raw_data": get_test_instrument_team_data_path("glows/f107_fluxtable.txt"), + "omni_raw_data": get_test_instrument_team_data_path( + "glows/omni2_all_years.dat" + ), } ancillary_files = { - 'uv_anisotropy': get_test_data_path('glows/imap_glows_uv-anisotropy-1CR_20100101_v001.json'), - 'WawHelioIonMP_parameters': get_test_data_path('glows/imap_glows_WawHelioIonMP_20100101_v002.json'), - 'bad_days_list': get_test_data_path('glows/imap_glows_bad-days-list_v001.dat'), - 'pipeline_settings': get_test_instrument_team_data_path( - 'glows/imap_glows_pipeline-settings-L3bc_20250707_v002.json') + "uv_anisotropy": get_test_data_path( + "glows/imap_glows_uv-anisotropy-1CR_20100101_v001.json" + ), + "WawHelioIonMP_parameters": get_test_data_path( + "glows/imap_glows_WawHelioIonMP_20100101_v002.json" + ), + "bad_days_list": get_test_data_path("glows/imap_glows_bad-days-list_v001.dat"), + "pipeline_settings": get_test_instrument_team_data_path( + "glows/imap_glows_pipeline-settings-L3bc_20250707_v002.json" + ), } - l3a_data_folder_path = get_test_data_path('glows/l3a_products') + l3a_data_folder_path = get_test_data_path("glows/l3a_products") l3a_data = [] - l3a_file_names = [f"imap_glows_l3a_hist_201004{x}-repoint{x + 90:05d}_v012.cdf" for x in range(22, 31)] - l3a_file_names += [f"imap_glows_l3a_hist_201005{x:02d}-repoint{x + 120:05d}_v012.cdf" for x in range(1, 19)] + l3a_file_names = [ + f"imap_glows_l3a_hist_201004{x}-repoint{x + 90:05d}_v012.cdf" + for x in range(22, 31) + ] + l3a_file_names += [ + f"imap_glows_l3a_hist_201005{x:02d}-repoint{x + 120:05d}_v012.cdf" + for x in range(1, 19) + ] for name in l3a_file_names: - l3a_data.append(create_glows_l3a_dictionary_from_cdf(l3a_data_folder_path / name)) + l3a_data.append( + create_glows_l3a_dictionary_from_cdf(l3a_data_folder_path / name) + ) - dependencies = GlowsL3BCDependencies(l3a_data=l3a_data, external_files=external_files, - ancillary_files=ancillary_files, carrington_rotation_number=cr, - start_date=datetime(2009, 12, 20), end_date=datetime(2009, 12, 21), - zip_file_path=Path("fake/path/to/file.zip")) + dependencies = GlowsL3BCDependencies( + l3a_data=l3a_data, + external_files=external_files, + ancillary_files=ancillary_files, + carrington_rotation_number=cr, + start_date=datetime(2009, 12, 20), + end_date=datetime(2009, 12, 21), + zip_file_path=Path("fake/path/to/file.zip"), + ) processor = GlowsProcessor(Mock(), input_metadata) @@ -803,104 +1123,136 @@ def run_glows_l3bc(): print(save_data(l3c_data_product, delete_if_present=True)) -@patch('imap_l3_processing.glows.glows_processor.shutil') +@patch("imap_l3_processing.glows.glows_processor.shutil") def run_glows_l3d(mock_shutil): mock_shutil.move = shutil.copy input_metadata = InputMetadata( - instrument='glows', - data_level='l3d', + instrument="glows", + data_level="l3d", start_date=datetime(2013, 9, 8), end_date=datetime(2013, 9, 8), - version='v001', - descriptor='solar-hist' + version="v001", + descriptor="solar-hist", ) external_files = { - 'lya_raw_data': get_test_data_path('glows/lyman_alpha_composite.nc'), + "lya_raw_data": get_test_data_path("glows/lyman_alpha_composite.nc"), } ancillary_files = { - 'WawHelioIon': { - 'speed': get_test_instrument_team_data_path('glows/imap_glows_plasma-speed-2010a_v003.dat'), - 'p-dens': get_test_instrument_team_data_path('glows/imap_glows_proton-density-2010a_v003.dat'), - 'uv-anis': get_test_instrument_team_data_path('glows/imap_glows_uv-anisotropy-2010a_v003.dat'), - 'phion': get_test_instrument_team_data_path('glows/imap_glows_photoion-2010a_v003.dat'), - 'lya': get_test_instrument_team_data_path('glows/imap_glows_lya-2010a_v003.dat'), - 'e-dens': get_test_instrument_team_data_path('glows/imap_glows_electron-density-2010a_v003.dat'), + "WawHelioIon": { + "speed": get_test_instrument_team_data_path( + "glows/imap_glows_plasma-speed-2010a_v003.dat" + ), + "p-dens": get_test_instrument_team_data_path( + "glows/imap_glows_proton-density-2010a_v003.dat" + ), + "uv-anis": get_test_instrument_team_data_path( + "glows/imap_glows_uv-anisotropy-2010a_v003.dat" + ), + "phion": get_test_instrument_team_data_path( + "glows/imap_glows_photoion-2010a_v003.dat" + ), + "lya": get_test_instrument_team_data_path( + "glows/imap_glows_lya-2010a_v003.dat" + ), + "e-dens": get_test_instrument_team_data_path( + "glows/imap_glows_electron-density-2010a_v003.dat" + ), }, - 'pipeline_settings': get_test_data_path( - 'glows/l3d_drift_test/imap_glows_pipeline-settings-l3bcde_20100101_v006.json') + "pipeline_settings": get_test_data_path( + "glows/l3d_drift_test/imap_glows_pipeline-settings-l3bcde_20100101_v006.json" + ), } l3b_file_paths = [ - get_test_data_path('glows/pipeline/l3b/imap_glows_l3b_ion-rate-profile_20100103_v001.cdf'), - get_test_data_path('glows/pipeline/l3b/imap_glows_l3b_ion-rate-profile_20100519_v001.cdf') + get_test_data_path( + "glows/pipeline/l3b/imap_glows_l3b_ion-rate-profile_20100103_v001.cdf" + ), + get_test_data_path( + "glows/pipeline/l3b/imap_glows_l3b_ion-rate-profile_20100519_v001.cdf" + ), ] l3c_file_paths = [ - get_test_data_path('glows/pipeline/l3c/imap_glows_l3c_sw-profile_20100103_v001.cdf'), - get_test_data_path('glows/pipeline/l3c/imap_glows_l3c_sw-profile_20100519_v001.cdf'), + get_test_data_path( + "glows/pipeline/l3c/imap_glows_l3c_sw-profile_20100103_v001.cdf" + ), + get_test_data_path( + "glows/pipeline/l3c/imap_glows_l3c_sw-profile_20100519_v001.cdf" + ), ] - l3d_dependencies: GlowsL3DDependencies = GlowsL3DDependencies(external_files=external_files, - ancillary_files=ancillary_files, - l3b_file_paths=l3b_file_paths, - l3c_file_paths=l3c_file_paths) + l3d_dependencies: GlowsL3DDependencies = GlowsL3DDependencies( + external_files=external_files, + ancillary_files=ancillary_files, + l3b_file_paths=l3b_file_paths, + l3c_file_paths=l3c_file_paths, + ) processor = GlowsProcessor(ProcessingInputCollection(), input_metadata) - data_product, l3d_txt_paths, last_processed_cr = processor.process_l3d(l3d_dependencies) + data_product, l3d_txt_paths, last_processed_cr = processor.process_l3d( + l3d_dependencies + ) print("l3d_txts:") [print(txt_path) for txt_path in l3d_txt_paths] print(save_data(data_product, cr_number=last_processed_cr)) -def create_empty_hi_l1c_dataset(epoch: datetime, exposures: Optional[np.ndarray] = None, - spin_angles: Optional[np.ndarray] = None, - energies: Optional[np.ndarray] = None): +def create_empty_hi_l1c_dataset( + epoch: datetime, + exposures: Optional[np.ndarray] = None, + spin_angles: Optional[np.ndarray] = None, + energies: Optional[np.ndarray] = None, +): energies = energies if energies is not None else np.geomspace(1, 10000, 9) - spin_angles = spin_angles if spin_angles is not None else np.arange(0, 360, 0.1) + 0.05 - exposures = exposures if exposures is not None else np.ones(shape=(1, len(energies), len(spin_angles))) + spin_angles = ( + spin_angles if spin_angles is not None else np.arange(0, 360, 0.1) + 0.05 + ) + exposures = ( + exposures + if exposures is not None + else np.ones(shape=(1, len(energies), len(spin_angles))) + ) - return xr.Dataset({ - "exposure_times": ( - [ - "epoch", - "esa_energy_step", - "hi_pset_spin_angle_bin" - ], - exposures - ), - }, + return xr.Dataset( + { + "exposure_times": ( + ["epoch", "esa_energy_step", "hi_pset_spin_angle_bin"], + exposures, + ), + }, coords={ "epoch": [epoch], "esa_energy_step": energies, "hi_pset_spin_angle_bin": spin_angles, - } + }, ) -def create_empty_glows_l3e_dataset(epoch: datetime, survival_probabilities: np.ndarray, - spin_angles: Optional[np.ndarray] = None, - energies: Optional[np.ndarray] = None): +def create_empty_glows_l3e_dataset( + epoch: datetime, + survival_probabilities: np.ndarray, + spin_angles: Optional[np.ndarray] = None, + energies: Optional[np.ndarray] = None, +): energies = energies or np.geomspace(1, 10000, 16) spin_angles = spin_angles or np.arange(0, 360, 1) + 0.5 - return xr.Dataset({ - "probability_of_survival": ( - [ - "epoch", - "energy", - "spin_angle_bin" - ], - survival_probabilities - ) - }, + return xr.Dataset( + { + "probability_of_survival": ( + ["epoch", "energy", "spin_angle_bin"], + survival_probabilities, + ) + }, coords={ "epoch": [epoch], "energy": energies, "spin_angle_bin": spin_angles, - }) + }, + ) EPOCH = TypeVar("EPOCH") @@ -910,20 +1262,34 @@ def create_empty_glows_l3e_dataset(epoch: datetime, survival_probabilities: np.n def read_glows_survival_probability_data_from_cdf() -> tuple[np.ndarray, np.ndarray]: - l3e = CDF(str(get_test_data_path("glows/imap_glows_l3e_survival-probabilities-hi_20250324_v001.cdf"))) - return l3e["probability_of_survival"][...][:, 0], l3e["probability_of_survival"][...][:, 1] + l3e = CDF( + str( + get_test_data_path( + "glows/imap_glows_l3e_survival-probabilities-hi_20250324_v001.cdf" + ) + ) + ) + return l3e["probability_of_survival"][...][:, 0], l3e["probability_of_survival"][ + ... + ][:, 1] -@patch('imap_l3_processing.hi.hi_processor.HiLoL3SurvivalDependencies.fetch_dependencies') -def create_hi_l3_survival_corrected_cdf(survival_dependencies: HiLoL3SurvivalDependencies, spacing_degree: int, - mock_fetch_dependencies) -> Path: - input_metadata = InputMetadata(instrument="hi", - data_level="l3", - start_date=datetime(2025, 4, 9), - end_date=datetime(2025, 4, 10), - version="v001", - descriptor="h90-ena-h-sf-sp-ram-hae-4deg-6mo", - ) +@patch( + "imap_l3_processing.hi.hi_processor.HiLoL3SurvivalDependencies.fetch_dependencies" +) +def create_hi_l3_survival_corrected_cdf( + survival_dependencies: HiLoL3SurvivalDependencies, + spacing_degree: int, + mock_fetch_dependencies, +) -> Path: + input_metadata = InputMetadata( + instrument="hi", + data_level="l3", + start_date=datetime(2025, 4, 9), + end_date=datetime(2025, 4, 10), + version="v001", + descriptor="h90-ena-h-sf-sp-ram-hae-4deg-6mo", + ) mock_fetch_dependencies.return_value = survival_dependencies @@ -933,16 +1299,22 @@ def create_hi_l3_survival_corrected_cdf(survival_dependencies: HiLoL3SurvivalDep print(processor.process()) -@patch('imap_l3_processing.lo.lo_processor.HiLoL3SurvivalDependencies.fetch_dependencies') -def create_lo_l3_survival_corrected_cdf(survival_dependencies: HiLoL3SurvivalDependencies, spacing_degree: int, - mock_fetch_dependencies): - input_metadata = InputMetadata(instrument="lo", - data_level="l3", - start_date=datetime(2025, 4, 9), - end_date=datetime(2025, 4, 10), - version="v001", - descriptor=f"l090-ena-h-sf-sp-ram-hae-{spacing_degree}deg-6mo", - ) +@patch( + "imap_l3_processing.lo.lo_processor.HiLoL3SurvivalDependencies.fetch_dependencies" +) +def create_lo_l3_survival_corrected_cdf( + survival_dependencies: HiLoL3SurvivalDependencies, + spacing_degree: int, + mock_fetch_dependencies, +): + input_metadata = InputMetadata( + instrument="lo", + data_level="l3", + start_date=datetime(2025, 4, 9), + end_date=datetime(2025, 4, 10), + version="v001", + descriptor=f"l090-ena-h-sf-sp-ram-hae-{spacing_degree}deg-6mo", + ) mock_fetch_dependencies.return_value = survival_dependencies @@ -954,43 +1326,54 @@ def create_lo_l3_survival_corrected_cdf(survival_dependencies: HiLoL3SurvivalDep def create_codice_hi_l3a_direct_events_cdf(): codice_hi_de_dependencies = CodiceHiL3aDirectEventsDependencies.from_file_paths( - tof_lookup_path=get_test_instrument_team_data_path("codice/hi/imap_codice_tof-lookup_20241110_v002.csv"), + tof_lookup_path=get_test_instrument_team_data_path( + "codice/hi/imap_codice_tof-lookup_20241110_v002.csv" + ), codice_l2_hi_cdf=get_test_instrument_team_data_path( - "codice/hi/imap_codice_l2_hi-direct-events_20241110_v002.cdf")) + "codice/hi/imap_codice_l2_hi-direct-events_20241110_v002.cdf" + ), + ) input_metadata = InputMetadata( - instrument='codice', - data_level='l3a', + instrument="codice", + data_level="l3a", start_date=datetime(2024, 11, 10), end_date=datetime(2025, 1, 2), - version='v000', - descriptor='hi-direct-events' + version="v000", + descriptor="hi-direct-events", ) codice_lo_processor = CodiceHiProcessor(ProcessingInputCollection(), input_metadata) - direct_event_data = codice_lo_processor.process_l3a_direct_event(codice_hi_de_dependencies) + direct_event_data = codice_lo_processor.process_l3a_direct_event( + codice_hi_de_dependencies + ) cdf_path = save_data(direct_event_data, delete_if_present=True) return cdf_path def create_codice_hi_l3b_pitch_angles_cdf(): codice_hi_pitch_angle_dependencies = CodicePitchAngleDependencies.from_file_paths( - mag_file_path=get_test_data_path("mag/imap_mag_l1d_norm-mago_20250814_v003.cdf"), + mag_file_path=get_test_data_path( + "mag/imap_mag_l1d_norm-mago_20250814_v003.cdf" + ), codice_l2_sectored_intensities_path=get_test_instrument_team_data_path( - "codice/hi/imap_codice_l2_hi-sectored_20250814_v002.cdf") + "codice/hi/imap_codice_l2_hi-sectored_20250814_v002.cdf" + ), ) input_metadata = InputMetadata( - instrument='codice', - data_level='l3b', + instrument="codice", + data_level="l3b", start_date=datetime(2024, 11, 10), end_date=datetime(2025, 1, 2), - version='v000', - descriptor='hi-pitch-angle' + version="v000", + descriptor="hi-pitch-angle", ) codice_hi_processor = CodiceHiProcessor(ProcessingInputCollection(), input_metadata) - pitch_angle_data = codice_hi_processor.process_l3b(codice_hi_pitch_angle_dependencies) + pitch_angle_data = codice_hi_processor.process_l3b( + codice_hi_pitch_angle_dependencies + ) cdf_path = save_data(pitch_angle_data, delete_if_present=True) return cdf_path @@ -1020,18 +1403,38 @@ def create_codice_hi_l3b_pitch_angles_cdf(): if "swapi" in sys.argv: if "l3a" in sys.argv: paths = create_swapi_l3a_cdf( - str(get_test_data_path("swapi/imap_swapi_energy-gf-pui-lut_20100101_v001.csv")), - str(get_test_data_path( - "swapi/imap_swapi_l2_density-of-neutral-helium-lut-text-not-cdf_20241023_v002.cdf")), - str(get_test_data_path("swapi/imap_swapi_efficiency-lut_20241020_v002.dat")), - str(get_test_data_path("swapi/imap_swapi_l2_50-sweeps_20250606_v003.cdf")), + str( + get_test_data_path( + "swapi/imap_swapi_energy-gf-pui-lut_20100101_v001.csv" + ) + ), + str( + get_test_data_path( + "swapi/imap_swapi_l2_density-of-neutral-helium-lut-text-not-cdf_20241023_v002.cdf" + ) + ), + str( + get_test_data_path( + "swapi/imap_swapi_efficiency-lut_20241020_v002.dat" + ) + ), + str( + get_test_data_path( + "swapi/imap_swapi_l2_50-sweeps_20250606_v003.cdf" + ) + ), ) print(paths) if "l3b" in sys.argv: path = create_swapi_l3b_cdf( "tests/test_data/swapi/imap_swapi_energy-gf-sw-lut_20100101_v001.csv", "tests/test_data/swapi/imap_swapi_efficiency-lut_20241020_v002.dat", - str(get_test_data_path("swapi/imap_swapi_l2_50-sweeps_20250606_v003.cdf"))) + str( + get_test_data_path( + "swapi/imap_swapi_l2_50-sweeps_20250606_v003.cdf" + ) + ), + ) print(path) if "glows" in sys.argv: if "pre-b" in sys.argv: @@ -1058,11 +1461,15 @@ def create_codice_hi_l3b_pitch_angles_cdf(): print(f"hit direct event data product: {path}") else: mag_data = read_mag_data( - r"C:\Users\Petty\Downloads\HIT Validation-20250922T131655Z-1-001\HIT Validation\imap_mag_l1d_norm-dsrf_20100106_v001.cdf") + r"C:\Users\Petty\Downloads\HIT Validation-20250922T131655Z-1-001\HIT Validation\imap_mag_l1d_norm-dsrf_20100106_v001.cdf" + ) hit_data = read_l2_hit_data( - r"C:\Users\Petty\Downloads\HIT Validation-20250922T131655Z-1-001\HIT Validation\imap_hit_l2_macropixel-intensity_20100106_v001.cdf") + r"C:\Users\Petty\Downloads\HIT Validation-20250922T131655Z-1-001\HIT Validation\imap_hit_l2_macropixel-intensity_20100106_v001.cdf" + ) dependencies = HITL3SectoredDependencies(mag_data=mag_data, data=hit_data) - print(f"hit macropixel data product: {create_hit_sectored_cdf(dependencies)}") + print( + f"hit macropixel data product: {create_hit_sectored_cdf(dependencies)}" + ) if "swe" in sys.argv: dependencies = SweL3Dependencies.from_file_paths( @@ -1085,21 +1492,30 @@ def create_codice_hi_l3b_pitch_angles_cdf(): print(create_swe_product_with_fake_spice(dependencies)) if "hi" in sys.argv: - hi_targets = ["survival-probability", "spectral-index", "full-spin", "combined-sensors"] + hi_targets = [ + "survival-probability", + "spectral-index", + "full-spin", + "combined-sensors", + ] do_all = not np.any([t in sys.argv for t in hi_targets]) glows_l3e_folder = get_test_data_path("hi/fake_l3e_survival_probabilities/90") glows_l3_paths = list(glows_l3e_folder.iterdir()) - glows_l3_paths = glows_l3_paths[:len(glows_l3_paths) // 4] + glows_l3_paths = glows_l3_paths[: len(glows_l3_paths) // 4] l2_antiram_90_map_path = get_test_data_path( - "hi/validation/l2_maps/imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-1yr_20250415_v001.cdf") + "hi/validation/l2_maps/imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-1yr_20250415_v001.cdf" + ) l2_ram_90_map_path = get_test_data_path( - "hi/validation/l2_maps/imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-1yr_20250415_v001.cdf") + "hi/validation/l2_maps/imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-1yr_20250415_v001.cdf" + ) l2_antiram_45_map_path = get_test_data_path( - "hi/validation/l2_maps/imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-1yr_20250415_v001.cdf") + "hi/validation/l2_maps/imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-1yr_20250415_v001.cdf" + ) l2_ram_45_map_path = get_test_data_path( - "hi/validation/l2_maps/imap_hi_l2_h45-ena-h-sf-nsp-ram-hae-4deg-1yr_20250415_v001.cdf") + "hi/validation/l2_maps/imap_hi_l2_h45-ena-h-sf-nsp-ram-hae-4deg-1yr_20250415_v001.cdf" + ) l1c_paths = list((get_imap_data_dir_path() / "hi" / "l1c").rglob("*.cdf")) @@ -1108,7 +1524,8 @@ def create_codice_hi_l3b_pitch_angles_cdf(): map_file_path=l2_ram_90_map_path, l1c_paths=l1c_paths, glows_l3e_paths=glows_l3_paths, - l2_descriptor="h90-ena-h-sf-nsp-ram-hae-4deg-1yr") + l2_descriptor="h90-ena-h-sf-nsp-ram-hae-4deg-1yr", + ) create_hi_l3_survival_corrected_cdf(survival_dependencies, 4) if do_all or "spectral-index" in sys.argv: @@ -1122,42 +1539,50 @@ def create_codice_hi_l3b_pitch_angles_cdf(): map_file_path=l2_ram_90_map_path, l1c_paths=l1c_paths, glows_l3e_paths=glows_l3_paths, - l2_descriptor="h90-ena-h-sf-nsp-ram-hae-4deg-6mo") + l2_descriptor="h90-ena-h-sf-nsp-ram-hae-4deg-6mo", + ) antiram_survival_dependencies = HiLoL3SurvivalDependencies.from_file_paths( map_file_path=l2_antiram_90_map_path, l1c_paths=l1c_paths, glows_l3e_paths=glows_l3_paths, - l2_descriptor="h90-ena-h-sf-nsp-anti-hae-4deg-6mo") + l2_descriptor="h90-ena-h-sf-nsp-anti-hae-4deg-6mo", + ) full_spin_dependencies = HiL3SingleSensorFullSpinDependencies( ram_dependencies=ram_survival_dependencies, - antiram_dependencies=antiram_survival_dependencies + antiram_dependencies=antiram_survival_dependencies, ) print(create_survival_corrected_full_spin_cdf(full_spin_dependencies)) if "lo" in sys.argv: - lo_targets = ['spectral-index', 'survival_probabilities'] + lo_targets = ["spectral-index", "survival_probabilities"] do_all = not np.any([t in sys.argv for t in lo_targets]) if do_all or "spectral-index" in sys.argv: dependencies = LoL3SpectralFitDependencies( - RectangularIntensityMapData.read_from_path(Path( - '/data/imap/lo/l3/2025/04/imap_lo_l3_l090-ena-h-sf-sp-ram-hae-4deg-6mo_20250415_v002.cdf')) + RectangularIntensityMapData.read_from_path( + Path( + "/data/imap/lo/l3/2025/04/imap_lo_l3_l090-ena-h-sf-sp-ram-hae-4deg-6mo_20250415_v002.cdf" + ) + ) ) print(create_lo_spectral_index_cdf(dependencies)) if do_all or "survival_probabilities" in sys.argv: - - glows_l3e_folder = get_test_data_path("hi/fake_l3e_survival_probabilities/90") + glows_l3e_folder = get_test_data_path( + "hi/fake_l3e_survival_probabilities/90" + ) glows_l3_paths = list(glows_l3e_folder.iterdir()) - missing_paths, run_local_paths = try_get_many_run_local_paths([ - "hi/full_spin_deps/l1c", - "hi/full_spin_deps/imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20250415_v001.cdf", - ]) + missing_paths, run_local_paths = try_get_many_run_local_paths( + [ + "hi/full_spin_deps/l1c", + "hi/full_spin_deps/imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20250415_v001.cdf", + ] + ) if missing_paths: create_hi_full_spin_deps(sensor="90") @@ -1170,7 +1595,8 @@ def create_codice_hi_l3b_pitch_angles_cdf(): map_file_path=l2_ram_90_map_path, l1c_paths=l1c_paths, glows_l3e_paths=glows_l3_paths, - l2_descriptor="l090-ena-h-sf-nsp-ram-hae-4deg-6mo") + l2_descriptor="l090-ena-h-sf-nsp-ram-hae-4deg-6mo", + ) create_lo_l3_survival_corrected_cdf(survival_dependencies, 4) if "ultra" in sys.argv: @@ -1182,54 +1608,77 @@ def create_codice_hi_l3b_pitch_angles_cdf(): end_date=datetime(year=2025, month=9, day=1), data_level="l3", version="v001", - descriptor=f"u90-ena-h-sf-sp-full-hae-{spacing_degree}deg-6mo" + descriptor=f"u90-ena-h-sf-sp-full-hae-{spacing_degree}deg-6mo", + ) + processor = UltraProcessor( + input_metadata=processor_input_metadata, dependencies=Mock() ) - processor = UltraProcessor(input_metadata=processor_input_metadata, dependencies=Mock()) # @formatter:off - missing_map_and_pset_paths, [l2_map_path, *l1c_dependency_paths] = try_get_many_run_local_paths([ - "ultra/20250415-20250419/imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-6mo_20250415_v001.cdf", - "ultra/20250415-20250419/ultra_l1c/imap_ultra_l1c_45sensor-spacecraftpset_20250415-repoint00001_v001.cdf", - "ultra/20250415-20250419/ultra_l1c/imap_ultra_l1c_45sensor-spacecraftpset_20250416-repoint00002_v010.cdf", - "ultra/20250415-20250419/ultra_l1c/imap_ultra_l1c_45sensor-spacecraftpset_20250417-repoint00003_v010.cdf", - "ultra/20250415-20250419/ultra_l1c/imap_ultra_l1c_45sensor-spacecraftpset_20250418-repoint00004_v010.cdf", - ]) + missing_map_and_pset_paths, [l2_map_path, *l1c_dependency_paths] = ( + try_get_many_run_local_paths( + [ + "ultra/20250415-20250419/imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-6mo_20250415_v001.cdf", + "ultra/20250415-20250419/ultra_l1c/imap_ultra_l1c_45sensor-spacecraftpset_20250415-repoint00001_v001.cdf", + "ultra/20250415-20250419/ultra_l1c/imap_ultra_l1c_45sensor-spacecraftpset_20250416-repoint00002_v010.cdf", + "ultra/20250415-20250419/ultra_l1c/imap_ultra_l1c_45sensor-spacecraftpset_20250417-repoint00003_v010.cdf", + "ultra/20250415-20250419/ultra_l1c/imap_ultra_l1c_45sensor-spacecraftpset_20250418-repoint00004_v010.cdf", + ] + ) + ) # @formatter:on - missing_glows_paths, [*l3e_glows_paths] = try_get_many_run_local_paths([ - "ultra/20250415-20250419/glows_l3e/imap_glows_l3e_survival-probability-ultra_20250415_v010.cdf", - "ultra/20250415-20250419/glows_l3e/imap_glows_l3e_survival-probability-ultra_20250416_v010.cdf", - "ultra/20250415-20250419/glows_l3e/imap_glows_l3e_survival-probability-ultra_20250417_v010.cdf", - "ultra/20250415-20250419/glows_l3e/imap_glows_l3e_survival-probability-ultra_20250418_v010.cdf", - ]) + missing_glows_paths, [*l3e_glows_paths] = try_get_many_run_local_paths( + [ + "ultra/20250415-20250419/glows_l3e/imap_glows_l3e_survival-probability-ultra_20250415_v010.cdf", + "ultra/20250415-20250419/glows_l3e/imap_glows_l3e_survival-probability-ultra_20250416_v010.cdf", + "ultra/20250415-20250419/glows_l3e/imap_glows_l3e_survival-probability-ultra_20250417_v010.cdf", + "ultra/20250415-20250419/glows_l3e/imap_glows_l3e_survival-probability-ultra_20250418_v010.cdf", + ] + ) if missing_glows_paths or missing_map_and_pset_paths: - create_l1c_and_glows_with_matching_date_range(datetime(2025, 4, 15, 12), datetime(2025, 4, 19, 12)) - - l1c_dependency = [UltraL1CPSet.read_from_path(l1c_dependency_path) for l1c_dependency_path in - l1c_dependency_paths] - l3e_dependencies = [UltraGlowsL3eData.read_from_path(path) for path in l3e_glows_paths] + create_l1c_and_glows_with_matching_date_range( + datetime(2025, 4, 15, 12), datetime(2025, 4, 19, 12) + ) + + l1c_dependency = [ + UltraL1CPSet.read_from_path(l1c_dependency_path) + for l1c_dependency_path in l1c_dependency_paths + ] + l3e_dependencies = [ + UltraGlowsL3eData.read_from_path(path) for path in l3e_glows_paths + ] dependencies = UltraL3Dependencies.from_file_paths( - l2_map_path, - l1c_dependency_paths, - l3e_glows_paths + l2_map_path, l1c_dependency_paths, l3e_glows_paths ) - healpix_sp_corrected_data = correct_healpix_data_for_survival_probability(deps=dependencies, - spice_frame_name=SpiceFrame.IMAP_HAE) - rectangular_sp_data_product = processor._process_healpix_intensity_to_rectangular(healpix_sp_corrected_data, - spacing_degree, - spice_frame_name=SpiceFrame.IMAP_HAE) + healpix_sp_corrected_data = correct_healpix_data_for_survival_probability( + deps=dependencies, spice_frame_name=SpiceFrame.IMAP_HAE + ) + rectangular_sp_data_product = ( + processor._process_healpix_intensity_to_rectangular( + healpix_sp_corrected_data, + spacing_degree, + spice_frame_name=SpiceFrame.IMAP_HAE, + ) + ) - rectangular_sp_corrected_path = save_data(rectangular_sp_data_product, delete_if_present=True) + rectangular_sp_corrected_path = save_data( + rectangular_sp_data_product, delete_if_present=True + ) print(rectangular_sp_corrected_path) if "spectral-index" in sys.argv: spacing_degree = 2 - ultra_l3_path = get_test_data_path('ultra/fake_ultra_map_data.cdf') - fit_energy_ranges_path = get_test_data_path('ultra/imap_ultra_ulc-spx-energy-ranges_20250407_v000.dat') - dependencies = UltraL3SpectralIndexDependencies.from_file_paths(ultra_l3_path, fit_energy_ranges_path) + ultra_l3_path = get_test_data_path("ultra/fake_ultra_map_data.cdf") + fit_energy_ranges_path = get_test_data_path( + "ultra/imap_ultra_ulc-spx-energy-ranges_20250407_v000.dat" + ) + dependencies = UltraL3SpectralIndexDependencies.from_file_paths( + ultra_l3_path, fit_energy_ranges_path + ) processor_input_metadata = InputMetadata( instrument="ultra", @@ -1237,12 +1686,16 @@ def create_codice_hi_l3b_pitch_angles_cdf(): end_date=datetime(year=2025, month=9, day=1), data_level="l3", version="v001", - descriptor=f"u90-spx-h-sf-sp-full-hae-{spacing_degree}deg-6mo" + descriptor=f"u90-spx-h-sf-sp-full-hae-{spacing_degree}deg-6mo", ) - processor = UltraProcessor(input_metadata=processor_input_metadata, dependencies=None) + processor = UltraProcessor( + input_metadata=processor_input_metadata, dependencies=None + ) spectral_index_map_data = processor._process_spectral_index(dependencies) - output = processor._process_healpix_spectral_index_to_rectangular(spectral_index_map_data, spacing_degree) + output = processor._process_healpix_spectral_index_to_rectangular( + spectral_index_map_data, spacing_degree + ) print(save_data(output, True)) diff --git a/run_local_with_upload_and_download.py b/run_local_with_upload_and_download.py index 4327fec09..6bbbb8301 100644 --- a/run_local_with_upload_and_download.py +++ b/run_local_with_upload_and_download.py @@ -11,243 +11,545 @@ match target, subtarget: case "hi", "combine-l2": version = "v004" - subprocess.run([sys.executable, - 'imap_l3_data_processor.py', '--instrument', 'hi', '--data-level', 'l3', '--start-date', - '20250415', '--version', version, '--descriptor', 'hic-ena-h-hf-nsp-full-hae-4deg-1yr', - '--dependency', - f"""[{{"type": "science", "files": ["imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-4deg-1yr_20250415_{version}.cdf"]}}, + subprocess.run( + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "hi", + "--data-level", + "l3", + "--start-date", + "20250415", + "--version", + version, + "--descriptor", + "hic-ena-h-hf-nsp-full-hae-4deg-1yr", + "--dependency", + f"""[{{"type": "science", "files": ["imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-4deg-1yr_20250415_{version}.cdf"]}}, {{"type": "science", "files": ["imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-4deg-1yr_20250415_{version}.cdf"]}}, {{"type": "science", "files": ["imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-4deg-1yr_20250415_{version}.cdf"]}}, - {{"type": "science", "files": ["imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-4deg-1yr_20250415_{version}.cdf"]}}]"""]) + {{"type": "science", "files": ["imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-4deg-1yr_20250415_{version}.cdf"]}}]""", + ] + ) case "lo", "spx": version = "v902" - subprocess.run([sys.executable, - 'imap_l3_data_processor.py', '--instrument', 'lo', '--data-level', 'l3', '--start-date', - '20260101', '--version', version, '--descriptor', 'l090-spx-h-sf-sp-ram-hae-6deg-1yr', - '--dependency', - '[{"type": "science", "files": ["imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-1yr_20260101_v001.cdf"]}]']) - subprocess.run([sys.executable, - 'imap_l3_data_processor.py', '--instrument', 'lo', '--data-level', 'l3', '--start-date', - '20260101', '--version', version, '--descriptor', 'l090-spx-h-sf-nsp-ram-hae-6deg-1yr', - '--dependency', - '[{"type": "science", "files": ["imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-1yr_20260101_v902.cdf"]}]']) - subprocess.run([sys.executable, - 'imap_l3_data_processor.py', '--instrument', 'lo', '--data-level', 'l3', '--start-date', - '20260101', '--version', version, '--descriptor', 'l090-spxnbs-h-sf-nsp-ram-hae-6deg-1yr', - '--dependency', - '[{"type": "science", "files": ["imap_lo_l2_l090-enanbs-h-sf-nsp-ram-hae-6deg-1yr_20260101_v900.cdf"]}]']) + subprocess.run( + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "lo", + "--data-level", + "l3", + "--start-date", + "20260101", + "--version", + version, + "--descriptor", + "l090-spx-h-sf-sp-ram-hae-6deg-1yr", + "--dependency", + '[{"type": "science", "files": ["imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-1yr_20260101_v001.cdf"]}]', + ] + ) + subprocess.run( + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "lo", + "--data-level", + "l3", + "--start-date", + "20260101", + "--version", + version, + "--descriptor", + "l090-spx-h-sf-nsp-ram-hae-6deg-1yr", + "--dependency", + '[{"type": "science", "files": ["imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-1yr_20260101_v902.cdf"]}]', + ] + ) + subprocess.run( + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "lo", + "--data-level", + "l3", + "--start-date", + "20260101", + "--version", + version, + "--descriptor", + "l090-spxnbs-h-sf-nsp-ram-hae-6deg-1yr", + "--dependency", + '[{"type": "science", "files": ["imap_lo_l2_l090-enanbs-h-sf-nsp-ram-hae-6deg-1yr_20260101_v900.cdf"]}]', + ] + ) case "hi", "spx": - subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'hi', '--data-level', 'l3', - '--start-date', '20250415', '--version', 'v002', '--descriptor', - 'h90-spx-h-hf-sp-full-hae-4deg-6mo', '--dependency', - '[{"type": "science", "files": ["imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20250415_v002.cdf"]}]']) + subprocess.run( + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "hi", + "--data-level", + "l3", + "--start-date", + "20250415", + "--version", + "v002", + "--descriptor", + "h90-spx-h-hf-sp-full-hae-4deg-6mo", + "--dependency", + '[{"type": "science", "files": ["imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20250415_v002.cdf"]}]', + ] + ) case "hi", "survival": - subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'hi', '--data-level', 'l3', - '--start-date', '20250415', '--version', 'v002', '--descriptor', - 'h90-ena-h-sf-sp-full-hae-4deg-6mo', '--dependency', - '[{"type": "science", "files": ["imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20250415_v001.cdf"]},\n{"type": "science", "files": ["imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20250415_v001.cdf"]}]']) + subprocess.run( + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "hi", + "--data-level", + "l3", + "--start-date", + "20250415", + "--version", + "v002", + "--descriptor", + "h90-ena-h-sf-sp-full-hae-4deg-6mo", + "--dependency", + '[{"type": "science", "files": ["imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20250415_v001.cdf"]},\n{"type": "science", "files": ["imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20250415_v001.cdf"]}]', + ] + ) case "hit", "direct": - subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'hit', '--data-level', 'l3', - '--start-date', '20100106', '--version', 'v001', '--descriptor', - 'direct-events', '--dependency', - '[{"type": "science", "files": ["imap_hit_l1a_direct-events_20100106_v001.cdf"]},' + - '{"type": "ancillary", "files": ["imap_hit_range-2A-cosine-lookup_20250203_v000.csv"]},' + - '{"type": "ancillary", "files": ["imap_hit_range-2B-cosine-lookup_20250203_v000.csv"]},' + - '{"type": "ancillary", "files": ["imap_hit_range-3A-cosine-lookup_20250203_v000.csv"]},' + - '{"type": "ancillary", "files": ["imap_hit_range-3B-cosine-lookup_20250203_v000.csv"]},' + - '{"type": "ancillary", "files": ["imap_hit_range-4A-cosine-lookup_20250203_v000.csv"]},' + - '{"type": "ancillary", "files": ["imap_hit_range-4B-cosine-lookup_20250203_v000.csv"]},' + - '{"type": "ancillary", "files": ["imap_hit_hi-gain-lookup_20250203_v000.csv"]},' + - '{"type": "ancillary", "files": ["imap_hit_lo-gain-lookup_20250203_v000.csv"]},' + - '{"type": "ancillary", "files": ["imap_hit_hit-event-type-lookup_20250228_v000.csv"]},' + - '{"type": "ancillary", "files": ["imap_hit_range-2A-charge-fit-lookup_20250319_v000.csv"]},' + - '{"type": "ancillary", "files": ["imap_hit_range-3A-charge-fit-lookup_20250319_v000.csv"]},' + - '{"type": "ancillary", "files": ["imap_hit_range-4A-charge-fit-lookup_20250319_v000.csv"]},' + - '{"type": "ancillary", "files": ["imap_hit_range-2B-charge-fit-lookup_20250319_v000.csv"]},' + - '{"type": "ancillary", "files": ["imap_hit_range-3B-charge-fit-lookup_20250319_v000.csv"]},' + - '{"type": "ancillary", "files": ["imap_hit_range-4B-charge-fit-lookup_20250319_v000.csv"]}]' - ]) + subprocess.run( + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "hit", + "--data-level", + "l3", + "--start-date", + "20100106", + "--version", + "v001", + "--descriptor", + "direct-events", + "--dependency", + '[{"type": "science", "files": ["imap_hit_l1a_direct-events_20100106_v001.cdf"]},' + + '{"type": "ancillary", "files": ["imap_hit_range-2A-cosine-lookup_20250203_v000.csv"]},' + + '{"type": "ancillary", "files": ["imap_hit_range-2B-cosine-lookup_20250203_v000.csv"]},' + + '{"type": "ancillary", "files": ["imap_hit_range-3A-cosine-lookup_20250203_v000.csv"]},' + + '{"type": "ancillary", "files": ["imap_hit_range-3B-cosine-lookup_20250203_v000.csv"]},' + + '{"type": "ancillary", "files": ["imap_hit_range-4A-cosine-lookup_20250203_v000.csv"]},' + + '{"type": "ancillary", "files": ["imap_hit_range-4B-cosine-lookup_20250203_v000.csv"]},' + + '{"type": "ancillary", "files": ["imap_hit_hi-gain-lookup_20250203_v000.csv"]},' + + '{"type": "ancillary", "files": ["imap_hit_lo-gain-lookup_20250203_v000.csv"]},' + + '{"type": "ancillary", "files": ["imap_hit_hit-event-type-lookup_20250228_v000.csv"]},' + + '{"type": "ancillary", "files": ["imap_hit_range-2A-charge-fit-lookup_20250319_v000.csv"]},' + + '{"type": "ancillary", "files": ["imap_hit_range-3A-charge-fit-lookup_20250319_v000.csv"]},' + + '{"type": "ancillary", "files": ["imap_hit_range-4A-charge-fit-lookup_20250319_v000.csv"]},' + + '{"type": "ancillary", "files": ["imap_hit_range-2B-charge-fit-lookup_20250319_v000.csv"]},' + + '{"type": "ancillary", "files": ["imap_hit_range-3B-charge-fit-lookup_20250319_v000.csv"]},' + + '{"type": "ancillary", "files": ["imap_hit_range-4B-charge-fit-lookup_20250319_v000.csv"]}]', + ] + ) case "hit", "macropixel": - subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'hit', '--data-level', 'l3', - '--start-date', '20100106', '--version', 'v001', '--descriptor', - 'macropixel', '--dependency', - '[{"type": "science", "files": ["imap_hit_l2_macropixel-intensity_20100106_v001.cdf"]},' + - '{"type": "science", "files": ["imap_mag_l1d_norm-dsrf_20100106_v001.cdf"]}]' - ]) + subprocess.run( + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "hit", + "--data-level", + "l3", + "--start-date", + "20100106", + "--version", + "v001", + "--descriptor", + "macropixel", + "--dependency", + '[{"type": "science", "files": ["imap_hit_l2_macropixel-intensity_20100106_v001.cdf"]},' + + '{"type": "science", "files": ["imap_mag_l1d_norm-dsrf_20100106_v001.cdf"]}]', + ] + ) case "ultra", "survival": - subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'ultra', '--data-level', 'l3', - '--start-date', '20250415', '--version', 'v010', '--descriptor', - 'u90-ena-h-sf-sp-full-hae-4deg-3mo', '--dependency', - '[{"type": "science", "files": ["imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20250415_v010.cdf"]},' + - '{"type": "spice", "files": ["imap_sclk_0000.tsc"]},' + - '{"type": "spice", "files": ["naif0012.tls"]},' + - '{"type": "spice", "files": ["imap_science_0001.tf"]},' + - '{"type": "spice", "files": ["imap_dps_2025_105_2026_105_01.ah.bc"]}]' - ]) + subprocess.run( + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "ultra", + "--data-level", + "l3", + "--start-date", + "20250415", + "--version", + "v010", + "--descriptor", + "u90-ena-h-sf-sp-full-hae-4deg-3mo", + "--dependency", + '[{"type": "science", "files": ["imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20250415_v010.cdf"]},' + + '{"type": "spice", "files": ["imap_sclk_0000.tsc"]},' + + '{"type": "spice", "files": ["naif0012.tls"]},' + + '{"type": "spice", "files": ["imap_science_0001.tf"]},' + + '{"type": "spice", "files": ["imap_dps_2025_105_2026_105_01.ah.bc"]}]', + ] + ) case "ultra", "spx": - subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'ultra', '--data-level', 'l3', - '--start-date', '20250415', '--version', 'v010', '--descriptor', - 'u90-spx-h-sf-sp-full-hae-4deg-3mo', '--dependency', - '[' - '{"type": "science", "files": ["imap_ultra_l3_u90-ena-h-sf-sp-full-hae-4deg-3mo_20250415_v010.cdf"]},' - '{"type": "ancillary", "files": ["imap_ultra_ulc-spx-energy-ranges_20250407_v000.dat"]}' - ']' - ]) + subprocess.run( + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "ultra", + "--data-level", + "l3", + "--start-date", + "20250415", + "--version", + "v010", + "--descriptor", + "u90-spx-h-sf-sp-full-hae-4deg-3mo", + "--dependency", + "[" + '{"type": "science", "files": ["imap_ultra_l3_u90-ena-h-sf-sp-full-hae-4deg-3mo_20250415_v010.cdf"]},' + '{"type": "ancillary", "files": ["imap_ultra_ulc-spx-energy-ranges_20250407_v000.dat"]}' + "]", + ] + ) case "glows", "l3b": - subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'glows', '--data-level', 'l3b', - '--start-date', '20250415', '--version', 'v025', '--dependency', - '[{"type": "science", "files": ["imap_glows_l3a_hist_20250415-repoint01000_v026.cdf"]},' - '{"type": "ancillary", "files":["imap_glows_bad-days-list_20100101_v001.dat"]},' - '{"type": "ancillary", "files":["imap_glows_WawHelioIonMP_20100101_v002.json"]},' - '{"type": "ancillary", "files":["imap_glows_uv-anisotropy-1CR_20100101_v001.json"]},' - '{"type": "ancillary", "files":["imap_glows_pipeline-settings-l3bcde_19470303_v008.json"]},' - '{"type": "repoint", "files":["imap_2026_105_01.repoint.csv"]}]']) + subprocess.run( + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "glows", + "--data-level", + "l3b", + "--start-date", + "20250415", + "--version", + "v025", + "--dependency", + '[{"type": "science", "files": ["imap_glows_l3a_hist_20250415-repoint01000_v026.cdf"]},' + '{"type": "ancillary", "files":["imap_glows_bad-days-list_20100101_v001.dat"]},' + '{"type": "ancillary", "files":["imap_glows_WawHelioIonMP_20100101_v002.json"]},' + '{"type": "ancillary", "files":["imap_glows_uv-anisotropy-1CR_20100101_v001.json"]},' + '{"type": "ancillary", "files":["imap_glows_pipeline-settings-l3bcde_19470303_v008.json"]},' + '{"type": "repoint", "files":["imap_2026_105_01.repoint.csv"]}]', + ] + ) case "glows", "l3d": - if os.path.exists(PATH_TO_L3D_TOOLKIT / 'data_l3b'): shutil.rmtree(PATH_TO_L3D_TOOLKIT / 'data_l3b') - if os.path.exists(PATH_TO_L3D_TOOLKIT / 'data_l3c'): shutil.rmtree(PATH_TO_L3D_TOOLKIT / 'data_l3c') - if os.path.exists(PATH_TO_L3D_TOOLKIT / 'data_l3d'): shutil.rmtree(PATH_TO_L3D_TOOLKIT / 'data_l3d') - if os.path.exists(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt'): shutil.rmtree(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt') + if os.path.exists(PATH_TO_L3D_TOOLKIT / "data_l3b"): + shutil.rmtree(PATH_TO_L3D_TOOLKIT / "data_l3b") + if os.path.exists(PATH_TO_L3D_TOOLKIT / "data_l3c"): + shutil.rmtree(PATH_TO_L3D_TOOLKIT / "data_l3c") + if os.path.exists(PATH_TO_L3D_TOOLKIT / "data_l3d"): + shutil.rmtree(PATH_TO_L3D_TOOLKIT / "data_l3d") + if os.path.exists(PATH_TO_L3D_TOOLKIT / "data_l3d_txt"): + shutil.rmtree(PATH_TO_L3D_TOOLKIT / "data_l3d_txt") - subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'glows', '--data-level', 'l3d', - '--start-date', '20100101', '--version', 'v010', '--descriptor', - 'solar-hist', '--dependency', - '[{"type": "science", "files": ["imap_glows_l3b_ion-rate-profile_20100422_v013.cdf"]},' - '{"type": "science", "files": ["imap_glows_l3c_sw-profile_20100422_v012.cdf"]},' - '{"type": "ancillary", "files": ["imap_glows_plasma-speed-2010a_20100101_v003.dat"]},' - '{"type": "ancillary", "files": ["imap_glows_proton-density-2010a_20100101_v003.dat"]},' - '{"type": "ancillary", "files": ["imap_glows_uv-anisotropy-2010a_20100101_v003.dat"]},' - '{"type": "ancillary", "files": ["imap_glows_photoion-2010a_20100101_v003.dat"]},' - '{"type": "ancillary", "files": ["imap_glows_electron-density-2010a_20100101_v003.dat"]},' - '{"type": "ancillary", "files": ["imap_glows_lya-2010a_20100101_v003.dat"]},' - '{"type": "ancillary", "files": ["imap_glows_l3b-archive_20100422_v011.zip"]},' - '{"type": "ancillary", "files": ["imap_glows_pipeline-settings-l3bcde_20100101_v006.json"]}]' - ]) + subprocess.run( + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "glows", + "--data-level", + "l3d", + "--start-date", + "20100101", + "--version", + "v010", + "--descriptor", + "solar-hist", + "--dependency", + '[{"type": "science", "files": ["imap_glows_l3b_ion-rate-profile_20100422_v013.cdf"]},' + '{"type": "science", "files": ["imap_glows_l3c_sw-profile_20100422_v012.cdf"]},' + '{"type": "ancillary", "files": ["imap_glows_plasma-speed-2010a_20100101_v003.dat"]},' + '{"type": "ancillary", "files": ["imap_glows_proton-density-2010a_20100101_v003.dat"]},' + '{"type": "ancillary", "files": ["imap_glows_uv-anisotropy-2010a_20100101_v003.dat"]},' + '{"type": "ancillary", "files": ["imap_glows_photoion-2010a_20100101_v003.dat"]},' + '{"type": "ancillary", "files": ["imap_glows_electron-density-2010a_20100101_v003.dat"]},' + '{"type": "ancillary", "files": ["imap_glows_lya-2010a_20100101_v003.dat"]},' + '{"type": "ancillary", "files": ["imap_glows_l3b-archive_20100422_v011.zip"]},' + '{"type": "ancillary", "files": ["imap_glows_pipeline-settings-l3bcde_20100101_v006.json"]}]', + ] + ) case "swapi", "proton": - subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'swapi', '--data-level', 'l3a', - '--descriptor', 'proton-sw', '--start-date', '20250606', '--version', 'v000', - '--dependency', - '[{"type":"science","files":["imap_swapi_l2_sci_20251208_v006.cdf"]},' - '{"type":"ancillary","files":["imap_swapi_efficiency-lut_20241020_v002.dat"]},' - '{"type":"ancillary","files":["imap_swapi_energy-gf-pui-lut_20100101_v001.csv"]},' - '{"type":"ancillary","files":["imap_swapi_density-of-neutral-helium-lut_20241023_v000.dat"]},' - '{"type":"ancillary","files":["imap_swapi_azimuthal-transmission_20260425_v001.csv"]},' - '{"type":"ancillary","files":["imap_swapi_central-effective-area_20260425_v001.csv"]},' - '{"type":"ancillary","files":["imap_swapi_passband-fit-coefficients_20260425_v001.csv"]},' - '{"type": "spice", "files": ["imap_sclk_0000.tsc"]},' + - '{"type": "spice", "files": ["naif0012.tls"]},' + - '{"type": "spice", "files": ["imap_science_108.tf"]},' + - '{"type": "spice", "files": ["imap_001.tf"]},' + - '{"type": "spice", "files": ["imap_2025_105_2026_105_01.ah.bc"]},' + - '{"type": "spice", "files": ["imap_dps_2025_105_2026_105_01.ah.bc"]}' - ']']) + subprocess.run( + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "swapi", + "--data-level", + "l3a", + "--descriptor", + "proton-sw", + "--start-date", + "20250606", + "--version", + "v000", + "--dependency", + '[{"type":"science","files":["imap_swapi_l2_sci_20251208_v006.cdf"]},' + '{"type":"ancillary","files":["imap_swapi_efficiency-lut_20241020_v002.dat"]},' + '{"type":"ancillary","files":["imap_swapi_energy-gf-pui-lut_20100101_v001.csv"]},' + '{"type":"ancillary","files":["imap_swapi_density-of-neutral-helium-lut_20241023_v000.dat"]},' + '{"type":"ancillary","files":["imap_swapi_azimuthal-transmission_20260425_v001.csv"]},' + '{"type":"ancillary","files":["imap_swapi_central-effective-area_20260425_v001.csv"]},' + '{"type":"ancillary","files":["imap_swapi_passband-fit-coefficients_20260425_v001.csv"]},' + '{"type": "spice", "files": ["imap_sclk_0000.tsc"]},' + + '{"type": "spice", "files": ["naif0012.tls"]},' + + '{"type": "spice", "files": ["imap_science_108.tf"]},' + + '{"type": "spice", "files": ["imap_001.tf"]},' + + '{"type": "spice", "files": ["imap_2025_105_2026_105_01.ah.bc"]},' + + '{"type": "spice", "files": ["imap_dps_2025_105_2026_105_01.ah.bc"]}' + "]", + ] + ) case "swapi", "alpha": - subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'swapi', '--data-level', 'l3a', - '--descriptor', 'alpha-sw', '--start-date', '20250606', '--version', 'v000', - '--dependency', - '[{"type":"science","files":["imap_swapi_l2_sci_20250606_v001.cdf"]},' - '{"type":"ancillary","files":["imap_swapi_efficiency-lut_20241020_v002.dat"]},' - '{"type":"ancillary","files":["imap_swapi_energy-gf-pui-lut_20100101_v001.csv"]},' - '{"type":"ancillary","files":["imap_swapi_density-of-neutral-helium-lut_20241023_v000.dat"]}' - ']']) + subprocess.run( + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "swapi", + "--data-level", + "l3a", + "--descriptor", + "alpha-sw", + "--start-date", + "20250606", + "--version", + "v000", + "--dependency", + '[{"type":"science","files":["imap_swapi_l2_sci_20250606_v001.cdf"]},' + '{"type":"ancillary","files":["imap_swapi_efficiency-lut_20241020_v002.dat"]},' + '{"type":"ancillary","files":["imap_swapi_energy-gf-pui-lut_20100101_v001.csv"]},' + '{"type":"ancillary","files":["imap_swapi_density-of-neutral-helium-lut_20241023_v000.dat"]}' + "]", + ] + ) case "swapi", "pui": - subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'swapi', '--data-level', 'l3a', - '--descriptor', 'pui-he', '--start-date', '20250606', '--version', 'v000', - '--dependency', - '[{"type":"science","files":["imap_swapi_l2_sci_20250606_v001.cdf"]},' - '{"type":"ancillary","files":["imap_swapi_efficiency-lut_20241020_v002.dat"]},' - '{"type":"ancillary","files":["imap_swapi_energy-gf-pui-lut_20100101_v001.csv"]},' - '{"type":"ancillary","files":["imap_swapi_density-of-neutral-helium-lut_20241023_v000.dat"]},' - '{"type": "spice", "files": ["imap_sclk_0000.tsc"]},' + - '{"type": "spice", "files": ["naif0012.tls"]},' + - '{"type": "spice", "files": ["imap_science_0001.tf"]},' + - '{"type": "spice", "files": ["imap_001.tf"]},' + - '{"type": "spice", "files": ["imap_2025_105_2026_105_01.ah.bc"]},' + - '{"type": "spice", "files": ["de440.bsp"]},' + - '{"type": "spice", "files": ["imap_recon_20250415_20260415_v01.bsp"]},' + - '{"type": "spice", "files": ["imap_dps_2025_105_2026_105_01.ah.bc"]}' - ']']) + subprocess.run( + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "swapi", + "--data-level", + "l3a", + "--descriptor", + "pui-he", + "--start-date", + "20250606", + "--version", + "v000", + "--dependency", + '[{"type":"science","files":["imap_swapi_l2_sci_20250606_v001.cdf"]},' + '{"type":"ancillary","files":["imap_swapi_efficiency-lut_20241020_v002.dat"]},' + '{"type":"ancillary","files":["imap_swapi_energy-gf-pui-lut_20100101_v001.csv"]},' + '{"type":"ancillary","files":["imap_swapi_density-of-neutral-helium-lut_20241023_v000.dat"]},' + '{"type": "spice", "files": ["imap_sclk_0000.tsc"]},' + + '{"type": "spice", "files": ["naif0012.tls"]},' + + '{"type": "spice", "files": ["imap_science_0001.tf"]},' + + '{"type": "spice", "files": ["imap_001.tf"]},' + + '{"type": "spice", "files": ["imap_2025_105_2026_105_01.ah.bc"]},' + + '{"type": "spice", "files": ["de440.bsp"]},' + + '{"type": "spice", "files": ["imap_recon_20250415_20260415_v01.bsp"]},' + + '{"type": "spice", "files": ["imap_dps_2025_105_2026_105_01.ah.bc"]}' + "]", + ] + ) case "swapi", "l3b": - subprocess.run([sys.executable, 'imap_l3_data_processor.py', '--instrument', 'swapi', '--data-level', 'l3b', - '--start-date', '20250606', '--version', 'v003', '--dependency', - '[{"type": "science", "files": ["imap_swapi_l2_sci_20250606_v001.cdf"]},' - '{"type":"ancillary","files":["imap_swapi_energy-gf-sw-lut_20100101_v001.csv"]},' - '{"type":"ancillary","files":["imap_swapi_efficiency-lut_20241020_v002.dat"]}]']) + subprocess.run( + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "swapi", + "--data-level", + "l3b", + "--start-date", + "20250606", + "--version", + "v003", + "--dependency", + '[{"type": "science", "files": ["imap_swapi_l2_sci_20250606_v001.cdf"]},' + '{"type":"ancillary","files":["imap_swapi_energy-gf-sw-lut_20100101_v001.csv"]},' + '{"type":"ancillary","files":["imap_swapi_efficiency-lut_20241020_v002.dat"]}]', + ] + ) case "codice", "lo-direct-events": subprocess.run( - [sys.executable, 'imap_l3_data_processor.py', '--instrument', 'codice', '--data-level', 'l3a', - '--descriptor', 'lo-direct-events', '--start-date', '20241110', '--version', 'v000', - '--dependency', - '[' - '{"type": "science", "files": ["imap_codice_l1a_lo-sw-priority_20241110_v002.cdf"]},' - '{"type": "science", "files": ["imap_codice_l1a_lo-nsw-priority_20241110_v003.cdf"]},' - '{"type": "science", "files": ["imap_codice_l2_lo-direct-events_20241110_v006.cdf"]},' - '{"type": "ancillary", "files": ["imap_codice_mass-coefficient-lookup_20241110_v002.csv"]},' - '{"type": "ancillary", "files": ["imap_codice_lo-energy-per-charge_20241110_v002.csv"]}]' - ]) + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "codice", + "--data-level", + "l3a", + "--descriptor", + "lo-direct-events", + "--start-date", + "20241110", + "--version", + "v000", + "--dependency", + "[" + '{"type": "science", "files": ["imap_codice_l1a_lo-sw-priority_20241110_v002.cdf"]},' + '{"type": "science", "files": ["imap_codice_l1a_lo-nsw-priority_20241110_v003.cdf"]},' + '{"type": "science", "files": ["imap_codice_l2_lo-direct-events_20241110_v006.cdf"]},' + '{"type": "ancillary", "files": ["imap_codice_mass-coefficient-lookup_20241110_v002.csv"]},' + '{"type": "ancillary", "files": ["imap_codice_lo-energy-per-charge_20241110_v002.csv"]}]', + ] + ) case "codice", ("lo-partial-densities" | "partial-densities"): subprocess.run( - [sys.executable, 'imap_l3_data_processor.py', '--instrument', 'codice', '--data-level', 'l3a', - '--descriptor', 'lo-partial-densities', '--start-date', '20241110', '--version', 'v000', - '--dependency', - '[' - '{"type": "science", "files": ["imap_codice_l2_lo-sw-species_20241110_v002.cdf"]},' - '{"type": "ancillary", "files": ["imap_codice_mass-per-charge_20241110_v002.csv"]}]' - ]) + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "codice", + "--data-level", + "l3a", + "--descriptor", + "lo-partial-densities", + "--start-date", + "20241110", + "--version", + "v000", + "--dependency", + "[" + '{"type": "science", "files": ["imap_codice_l2_lo-sw-species_20241110_v002.cdf"]},' + '{"type": "ancillary", "files": ["imap_codice_mass-per-charge_20241110_v002.csv"]}]', + ] + ) case "codice", ("lo-sw-ratios" | "ratios"): subprocess.run( - [sys.executable, 'imap_l3_data_processor.py', '--instrument', 'codice', '--data-level', 'l3a', - '--descriptor', 'lo-sw-ratios', '--start-date', '20241110', '--version', 'v000', - '--dependency', - '[' - '{"type": "science", "files": ["imap_codice_l3a_lo-partial-densities_20241110_v000.cdf"]}' - ']' - ]) + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "codice", + "--data-level", + "l3a", + "--descriptor", + "lo-sw-ratios", + "--start-date", + "20241110", + "--version", + "v000", + "--dependency", + "[" + '{"type": "science", "files": ["imap_codice_l3a_lo-partial-densities_20241110_v000.cdf"]}' + "]", + ] + ) case "codice", ("lo-sw-charge-state-distributions"): subprocess.run( - [sys.executable, 'imap_l3_data_processor.py', '--instrument', 'codice', '--data-level', 'l3a', - '--descriptor', 'lo-sw-charge-state-distributions', '--start-date', '20241110', '--version', 'v000', - '--dependency', - '[' - '{"type": "science", "files": ["imap_codice_l3a_lo-partial-densities_20241110_v000.cdf"]}' - ']' - ]) + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "codice", + "--data-level", + "l3a", + "--descriptor", + "lo-sw-charge-state-distributions", + "--start-date", + "20241110", + "--version", + "v000", + "--dependency", + "[" + '{"type": "science", "files": ["imap_codice_l3a_lo-partial-densities_20241110_v000.cdf"]}' + "]", + ] + ) case "codice", "hi-direct-events": subprocess.run( - [sys.executable, 'imap_l3_data_processor.py', '--instrument', 'codice', '--data-level', 'l3a', - '--descriptor', 'hi-direct-events', '--start-date', '20241110', '--version', 'v000', - '--dependency', - '[' - '{"type": "science", "files": ["imap_codice_l2_hi-direct-events_20241110_v002.cdf"]},' - '{"type": "ancillary", "files": ["imap_codice_tof-lookup_20241110_v002.csv"]}]' - ]) + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "codice", + "--data-level", + "l3a", + "--descriptor", + "hi-direct-events", + "--start-date", + "20241110", + "--version", + "v000", + "--dependency", + "[" + '{"type": "science", "files": ["imap_codice_l2_hi-direct-events_20241110_v002.cdf"]},' + '{"type": "ancillary", "files": ["imap_codice_tof-lookup_20241110_v002.csv"]}]', + ] + ) case "codice", "hi-pitch-angle": subprocess.run( - [sys.executable, 'imap_l3_data_processor.py', '--instrument', 'codice', '--data-level', 'l3b', - '--descriptor', 'hi-pitch-angle', '--start-date', '20241110', '--version', 'v000', - '--dependency', - '[' - '{"type": "science", "files": ["imap_codice_l2_hi-sectored_20241110_v002.cdf"]},' - '{"type": "science", "files": ["imap_mag_l1d_norm-dsrf_20250630_v001.cdf"]}' - ']' - ]) + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "codice", + "--data-level", + "l3b", + "--descriptor", + "hi-pitch-angle", + "--start-date", + "20241110", + "--version", + "v000", + "--dependency", + "[" + '{"type": "science", "files": ["imap_codice_l2_hi-sectored_20241110_v002.cdf"]},' + '{"type": "science", "files": ["imap_mag_l1d_norm-dsrf_20250630_v001.cdf"]}' + "]", + ] + ) case "codice", descriptor if "3d-distribution" in descriptor: species = descriptor.split("-")[1] - assert species in ["hplus", "heplus", "heplus2", "oplus6"], NotImplementedError( - "Target instrument or product not implemented") + assert species in ["hplus", "heplus", "heplus2", "oplus6"], ( + NotImplementedError("Target instrument or product not implemented") + ) subprocess.run( - [sys.executable, 'imap_l3_data_processor.py', '--instrument', 'codice', '--data-level', 'l3a', - '--descriptor', f'lo-{species}-3d-distribution', '--start-date', '20241110', '--version', 'v000', - '--dependency', - '[' - '{"type": "science", "files": ["imap_codice_l1a_lo-sw-priority_20241110_v002.cdf"]},' - '{"type": "science", "files": ["imap_codice_l1a_lo-nsw-priority_20241110_v003.cdf"]},' - '{"type": "science", "files": ["imap_codice_l3a_lo-direct-events_20241110_v005.cdf"]},' - '{"type": "ancillary", "files": ["imap_codice_lo-mass-species-bin-lookup_20241110_v001.csv"]},' - '{"type": "ancillary", "files": ["imap_codice_lo-geometric-factors_20241110_v001.csv"]},' - '{"type": "ancillary", "files": ["imap_codice_lo-energy-per-charge_20241110_v002.csv"]},' - f'{{"type": "ancillary", "files": ["imap_codice_lo-{species}-efficiency_20241110_v001.csv"]}}' - ']' - ]) + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "codice", + "--data-level", + "l3a", + "--descriptor", + f"lo-{species}-3d-distribution", + "--start-date", + "20241110", + "--version", + "v000", + "--dependency", + "[" + '{"type": "science", "files": ["imap_codice_l1a_lo-sw-priority_20241110_v002.cdf"]},' + '{"type": "science", "files": ["imap_codice_l1a_lo-nsw-priority_20241110_v003.cdf"]},' + '{"type": "science", "files": ["imap_codice_l3a_lo-direct-events_20241110_v005.cdf"]},' + '{"type": "ancillary", "files": ["imap_codice_lo-mass-species-bin-lookup_20241110_v001.csv"]},' + '{"type": "ancillary", "files": ["imap_codice_lo-geometric-factors_20241110_v001.csv"]},' + '{"type": "ancillary", "files": ["imap_codice_lo-energy-per-charge_20241110_v002.csv"]},' + f'{{"type": "ancillary", "files": ["imap_codice_lo-{species}-efficiency_20241110_v001.csv"]}}' + "]", + ] + ) case _: raise NotImplementedError("Target instrument or product not implemented") diff --git a/scripts/batch_download_parents.py b/scripts/batch_download_parents.py index 2a4d8cbc0..d182b8e62 100644 --- a/scripts/batch_download_parents.py +++ b/scripts/batch_download_parents.py @@ -9,6 +9,7 @@ def download_parents(path: Path, output_dir: Path): import imap_data_access from spacepy.pycdf import CDF + retries = 10 imap_data_access.config["DATA_DIR"] = output_dir @@ -19,13 +20,12 @@ def download_parents(path: Path, output_dir: Path): for file in path.rglob("*.cdf"): print(f"opening {path / file}") with CDF(str(file)) as cdf: - for parent in cdf.attrs['Parents']: + for parent in cdf.attrs["Parents"]: files_to_download.add(parent) print(f"Downloading {len(files_to_download)} files") for i, file in enumerate(files_to_download): - print(f"[{i + 1}/{len(files_to_download)}] downloading {file}") failures = 0 while failures < retries: @@ -35,18 +35,20 @@ def download_parents(path: Path, output_dir: Path): except Exception as e: failures += 1 - print(f"failed to download file: {file} after {failures} attempts, retrying...") + print( + f"failed to download file: {file} after {failures} attempts, retrying..." + ) if failures == retries: raise Exception( - f"failed to download file: {file} after {retries} attempts, exiting").with_traceback( - e.__traceback__) - time.sleep(failures ** 2 * 0.5) + f"failed to download file: {file} after {retries} attempts, exiting" + ).with_traceback(e.__traceback__) + time.sleep(failures**2 * 0.5) except Exception as e: print(e) -if __name__ == '__main__': +if __name__ == "__main__": dotenv.load_dotenv(".env") output_dir = Path(os.getcwd()) / "hi_hf_validation_data" diff --git a/scripts/clean_metadata.py b/scripts/clean_metadata.py index 29170224b..de3ec785c 100644 --- a/scripts/clean_metadata.py +++ b/scripts/clean_metadata.py @@ -15,10 +15,10 @@ def fix_fieldnam(): with open(variable_yaml, mode="r") as file: yaml_data = yaml.safe_load(file) for key, item in yaml_data.items(): - if 'NAME' not in item.keys(): + if "NAME" not in item.keys(): continue - if "_" in yaml_data[key]['FIELDNAM']: - yaml_data[key]['FIELDNAM'] = key.replace("_", " ").title() + if "_" in yaml_data[key]["FIELDNAM"]: + yaml_data[key]["FIELDNAM"] = key.replace("_", " ").title() if len(key) > 30: print("key over 30", variable_yaml, key) @@ -29,8 +29,11 @@ def fix_fieldnam(): def get_variable_yaml_paths() -> list[Path]: yaml_path = Path(cdf.__file__).parent / "config" - return [yaml_path / str(filename) for filename in os.listdir(yaml_path) if - "variable_attrs" in str(filename)] + return [ + yaml_path / str(filename) + for filename in os.listdir(yaml_path) + if "variable_attrs" in str(filename) + ] if __name__ == "__main__": diff --git a/scripts/codice/create_fake_efficiency_ancillary.py b/scripts/codice/create_fake_efficiency_ancillary.py index f57941520..e8173bde0 100644 --- a/scripts/codice/create_fake_efficiency_ancillary.py +++ b/scripts/codice/create_fake_efficiency_ancillary.py @@ -2,27 +2,36 @@ import numpy as np -from imap_l3_processing.codice.l3.lo.constants import CODICE_LO_NUM_AZIMUTH_BINS, CODICE_LO_NUM_ESA_STEPS -from imap_l3_processing.codice.l3.lo.direct_events.science.mass_species_bin_lookup import MassSpeciesBinLookup +from imap_l3_processing.codice.l3.lo.constants import ( + CODICE_LO_NUM_AZIMUTH_BINS, + CODICE_LO_NUM_ESA_STEPS, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.mass_species_bin_lookup import ( + MassSpeciesBinLookup, +) from tests.test_helpers import get_run_local_data_path, get_test_data_path HEADER_TEXT = """# Efficiency Factors for the CoDICE-Lo Instrument for each energy (128 ESA steps) and position (24 values), energy is assumed to be in ESA step order""" -def create_efficiency_lookup(species: str, output_dir: Path = get_run_local_data_path("codice")) -> Path: +def create_efficiency_lookup( + species: str, output_dir: Path = get_run_local_data_path("codice") +) -> Path: output_dir.mkdir(parents=True, exist_ok=True) output_path = output_dir / f"imap_codice_lo-{species}-efficiency_20241110_v001.csv" open(output_path, "a").close() - efficiency_data = np.ones((CODICE_LO_NUM_ESA_STEPS, CODICE_LO_NUM_AZIMUTH_BINS), dtype=np.float64) + efficiency_data = np.ones( + (CODICE_LO_NUM_ESA_STEPS, CODICE_LO_NUM_AZIMUTH_BINS), dtype=np.float64 + ) np.savetxt(output_path, efficiency_data, delimiter=",", header=HEADER_TEXT) return output_path if __name__ == "__main__": - mass_species_bin_path = get_test_data_path( - 'codice/imap_codice_lo-mass-species-bin-lookup_20241110_v001.csv') + "codice/imap_codice_lo-mass-species-bin-lookup_20241110_v001.csv" + ) for species in MassSpeciesBinLookup.read_from_csv(mass_species_bin_path).species: print(create_efficiency_lookup(species)) diff --git a/scripts/codice/create_more_accurate_l3a_direct_event.py b/scripts/codice/create_more_accurate_l3a_direct_event.py index 9e2c40bf0..8606db4bc 100644 --- a/scripts/codice/create_more_accurate_l3a_direct_event.py +++ b/scripts/codice/create_more_accurate_l3a_direct_event.py @@ -3,18 +3,27 @@ import numpy as np from spacepy.pycdf import CDF -from imap_l3_processing.codice.l3.lo.direct_events.science.angle_lookup import SpinAngleLookup -from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import EnergyLookup -from imap_l3_processing.codice.l3.lo.direct_events.science.mass_species_bin_lookup import MassSpeciesBinLookup +from imap_l3_processing.codice.l3.lo.direct_events.science.angle_lookup import ( + SpinAngleLookup, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import ( + EnergyLookup, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.mass_species_bin_lookup import ( + MassSpeciesBinLookup, +) from tests.test_helpers import get_test_data_path def create_more_accurate_l3a_direct_events_cdf(template_cdf: Path): - mass_species_path = get_test_data_path("codice/imap_codice_lo-mass-species-bin-lookup_20241110_v001.csv") + mass_species_path = get_test_data_path( + "codice/imap_codice_lo-mass-species-bin-lookup_20241110_v001.csv" + ) mass_species_bin_lookup = MassSpeciesBinLookup.read_from_csv(mass_species_path) energy_lookup = EnergyLookup.read_from_csv( - get_test_data_path('codice/imap_codice_lo-energy-per-charge_20241110_v002.csv')) + get_test_data_path("codice/imap_codice_lo-energy-per-charge_20241110_v002.csv") + ) energies = energy_lookup.bin_centers spin_angle_lut = SpinAngleLookup() @@ -27,14 +36,21 @@ def create_more_accurate_l3a_direct_events_cdf(template_cdf: Path): position = rng.integers(1, 25, size=event_buffer_shape) cdf["position"] = position - cdf["spin_angle"] = rng.choice(spin_angle_lut.bin_centers, size=event_buffer_shape) + cdf["spin_angle"] = rng.choice( + spin_angle_lut.bin_centers, size=event_buffer_shape + ) - species = rng.integers(0, mass_species_bin_lookup.get_num_species(), size=event_buffer_shape) + species = rng.integers( + 0, mass_species_bin_lookup.get_num_species(), size=event_buffer_shape + ) mass_ranges = np.array(mass_species_bin_lookup.mass_ranges) lower_masses = mass_ranges[species, 0] upper_masses = mass_ranges[species, -1] - masses = rng.random(event_buffer_shape) * (upper_masses - lower_masses) + lower_masses + masses = ( + rng.random(event_buffer_shape) * (upper_masses - lower_masses) + + lower_masses + ) mass_per_charge_ranges = np.array(mass_species_bin_lookup.mass_per_charge) lower_mpc = mass_per_charge_ranges[species, 0] @@ -51,4 +67,5 @@ def create_more_accurate_l3a_direct_events_cdf(template_cdf: Path): if __name__ == "__main__": create_more_accurate_l3a_direct_events_cdf( - get_test_data_path("codice/imap_codice_l3a_lo-direct-events_20241110_v000.cdf")) + get_test_data_path("codice/imap_codice_l3a_lo-direct-events_20241110_v000.cdf") + ) diff --git a/scripts/codice/create_more_accurate_l3a_direct_event_input.py b/scripts/codice/create_more_accurate_l3a_direct_event_input.py index 53196a765..e0f90e9d9 100644 --- a/scripts/codice/create_more_accurate_l3a_direct_event_input.py +++ b/scripts/codice/create_more_accurate_l3a_direct_event_input.py @@ -4,14 +4,21 @@ import numpy as np from spacepy.pycdf import CDF -from imap_l3_processing.codice.l3.lo.direct_events.science.angle_lookup import PositionToElevationLookup +from imap_l3_processing.codice.l3.lo.direct_events.science.angle_lookup import ( + PositionToElevationLookup, +) from imap_l3_processing.codice.l3.lo.models import CODICE_LO_L2_NUM_PRIORITIES from imap_l3_processing.constants import ONE_SECOND_IN_NANOSECONDS from scripts.time_shift_cdf import convert_epoch_time -from tests.test_helpers import get_run_local_data_path, get_test_instrument_team_data_path +from tests.test_helpers import ( + get_run_local_data_path, + get_test_instrument_team_data_path, +) -def extend_priority_counts_to_24_spin_angles(template_cdf_path, priority_count_variables: list[str], output_dir): +def extend_priority_counts_to_24_spin_angles( + template_cdf_path, priority_count_variables: list[str], output_dir +): rng = np.random.default_rng() output_path = output_dir / template_cdf_path.name @@ -29,8 +36,12 @@ def extend_priority_counts_to_24_spin_angles(template_cdf_path, priority_count_v cdf[var] = template_cdf[var] cdf[var].attrs = template_cdf[var].attrs - rgfo_half_spin = rng.integers(0, 33, size=template_cdf["rgfo_half_spin"].shape) - randomly_fill_value = rng.choice([False, True], size=template_cdf["rgfo_half_spin"].shape) + rgfo_half_spin = rng.integers( + 0, 33, size=template_cdf["rgfo_half_spin"].shape + ) + randomly_fill_value = rng.choice( + [False, True], size=template_cdf["rgfo_half_spin"].shape + ) cdf["rgfo_half_spin"] = rgfo_half_spin # cdf["rgfo_half_spin"] = np.where(randomly_fill_value, cdf["rgfo_half_spin"].attrs["FILLVAL"], @@ -38,8 +49,9 @@ def extend_priority_counts_to_24_spin_angles(template_cdf_path, priority_count_v return output_path -def modify_l1a_priority_counts(instrument_team_l1a_nsw_path: Path, instrument_team_l1a_sw_path: Path) -> tuple[ - Path, Path]: +def modify_l1a_priority_counts( + instrument_team_l1a_nsw_path: Path, instrument_team_l1a_sw_path: Path +) -> tuple[Path, Path]: output_dir = get_run_local_data_path("codice/lo") output_dir.mkdir(parents=True, exist_ok=True) @@ -48,8 +60,9 @@ def modify_l1a_priority_counts(instrument_team_l1a_nsw_path: Path, instrument_te "p6_hplus_heplusplus", ] - modified_l1a_nsw_path = extend_priority_counts_to_24_spin_angles(instrument_team_l1a_nsw_path, - l1a_nsw_priority_counts_vars, output_dir) + modified_l1a_nsw_path = extend_priority_counts_to_24_spin_angles( + instrument_team_l1a_nsw_path, l1a_nsw_priority_counts_vars, output_dir + ) l1a_sw_priority_counts_vars = [ "p0_tcrs", @@ -59,12 +72,11 @@ def modify_l1a_priority_counts(instrument_team_l1a_nsw_path: Path, instrument_te "p4_dcrs", ] - modified_l1a_sw_path = extend_priority_counts_to_24_spin_angles(instrument_team_l1a_sw_path, - l1a_sw_priority_counts_vars, - output_dir) + modified_l1a_sw_path = extend_priority_counts_to_24_spin_angles( + instrument_team_l1a_sw_path, l1a_sw_priority_counts_vars, output_dir + ) - return (modified_l1a_nsw_path, - modified_l1a_sw_path) + return (modified_l1a_nsw_path, modified_l1a_sw_path) def modify_l2_direct_events(instrument_team_l2_path: Path) -> Path: @@ -81,7 +93,9 @@ def modify_l2_direct_events(instrument_team_l2_path: Path) -> Path: with CDF(str(instrument_team_l2_path)) as template_cdf: with CDF(str(output_path), masterpath="") as cdf: - epoch_delta = timedelta(minutes=2).total_seconds() * ONE_SECOND_IN_NANOSECONDS + epoch_delta = ( + timedelta(minutes=2).total_seconds() * ONE_SECOND_IN_NANOSECONDS + ) cdf["epoch_delta_plus"] = np.full(template_cdf["epoch"].shape, epoch_delta) cdf["epoch_delta_minus"] = np.full(template_cdf["epoch"].shape, epoch_delta) @@ -89,14 +103,20 @@ def modify_l2_direct_events(instrument_team_l2_path: Path) -> Path: elevation_var = f"p{priority_i}_elevation" position_var = f"p{priority_i}_position" - cdf[position_var] = rng.integers(1, 25, size=template_cdf[position_var].shape) + cdf[position_var] = rng.integers( + 1, 25, size=template_cdf[position_var].shape + ) cdf[position_var].attrs["FILLVAL"] = int_fillval - cdf[elevation_var] = rng.choice(elevation_lookup.bin_centers, size=template_cdf[position_var].shape) + cdf[elevation_var] = rng.choice( + elevation_lookup.bin_centers, size=template_cdf[position_var].shape + ) cdf[elevation_var].attrs["FILLVAL"] = float_fillval for epoch_i in range(cdf[elevation_var].shape[0]): - number_of_events = template_cdf[f"p{priority_i}_num_events"][epoch_i] + number_of_events = template_cdf[f"p{priority_i}_num_events"][ + epoch_i + ] filled_position = cdf[position_var][...] filled_position[epoch_i, number_of_events:] = int_fillval @@ -118,20 +138,31 @@ def modify_l2_direct_events(instrument_team_l2_path: Path) -> Path: target_datetime = datetime(year=2024, month=11, day=11) template_l2_de_path = get_test_instrument_team_data_path( - "codice/lo/imap_codice_l2_lo-direct-events_20241110_v002.cdf") + "codice/lo/imap_codice_l2_lo-direct-events_20241110_v002.cdf" + ) template_l1a_nsw_path = get_test_instrument_team_data_path( - 'codice/lo/imap_codice_l1a_lo-nsw-priority_20241110_v002.cdf') + "codice/lo/imap_codice_l1a_lo-nsw-priority_20241110_v002.cdf" + ) template_l1a_sw_path = get_test_instrument_team_data_path( - 'codice/lo/imap_codice_l1a_lo-sw-priority_20241110_v002.cdf') + "codice/lo/imap_codice_l1a_lo-sw-priority_20241110_v002.cdf" + ) modified_l2_path = modify_l2_direct_events(template_l2_de_path) - l1a_nsw_path, l1a_sw_path = modify_l1a_priority_counts(template_l1a_nsw_path, template_l1a_sw_path) + l1a_nsw_path, l1a_sw_path = modify_l1a_priority_counts( + template_l1a_nsw_path, template_l1a_sw_path + ) convert_epoch_time(modified_l2_path, target_datetime) - modified_l2_path.rename(modified_l2_path.parent / "imap_codice_l2_lo-direct-events_20241111_v000.cdf") + modified_l2_path.rename( + modified_l2_path.parent / "imap_codice_l2_lo-direct-events_20241111_v000.cdf" + ) convert_epoch_time(l1a_nsw_path, target_datetime) - l1a_nsw_path.rename(l1a_nsw_path.parent / "imap_codice_l1a_lo-nsw-priority_20241111_v000.cdf") + l1a_nsw_path.rename( + l1a_nsw_path.parent / "imap_codice_l1a_lo-nsw-priority_20241111_v000.cdf" + ) convert_epoch_time(l1a_sw_path, target_datetime) - l1a_sw_path.rename(l1a_sw_path.parent / "imap_codice_l1a_lo-sw-priority_20241111_v000.cdf") + l1a_sw_path.rename( + l1a_sw_path.parent / "imap_codice_l1a_lo-sw-priority_20241111_v000.cdf" + ) diff --git a/scripts/compare_l2_map_metadata.py b/scripts/compare_l2_map_metadata.py index 2b93601ad..3d7019316 100644 --- a/scripts/compare_l2_map_metadata.py +++ b/scripts/compare_l2_map_metadata.py @@ -60,5 +60,3 @@ not_included_keys = [key for key in l3_attrs.keys() if key not in l2_attrs.keys()] print(var, not_included_keys) - - diff --git a/scripts/confirm_reprocessing.py b/scripts/confirm_reprocessing.py index 9034f8a85..66bb94813 100644 --- a/scripts/confirm_reprocessing.py +++ b/scripts/confirm_reprocessing.py @@ -7,8 +7,11 @@ def report_oldest_file(**query_args): records = imap_data_access.query(**query_args, version="latest") - oldest = min(records, key=lambda x:x["ingestion_date"]) - print(f"Oldest file:\n{oldest["ingestion_date"]} {Path(oldest["file_path"]).name}") + oldest = min(records, key=lambda x: x["ingestion_date"]) + print( + f"Oldest file:\n{oldest['ingestion_date']} {Path(oldest['file_path']).name}" + ) + def get_unprocessed_files(reference_time_str, **query_args): reference_time = datetime.fromisoformat(reference_time_str) @@ -19,6 +22,7 @@ def get_unprocessed_files(reference_time_str, **query_args): if datetime.fromisoformat(r["ingestion_date"]) < reference_time ] + def analyze(reference_time_str, **query_args): files = get_unprocessed_files(reference_time_str, **query_args) if len(files) == 0: @@ -27,6 +31,7 @@ def analyze(reference_time_str, **query_args): for date, processed_at in files: print(f"File {date} was last processed at {processed_at}") + parser = argparse.ArgumentParser() parser.add_argument( "reprocessing_time", @@ -38,11 +43,10 @@ def analyze(reference_time_str, **query_args): parser.add_argument("--data-level") parser.add_argument("--descriptor") args = parser.parse_args() -query_args = dict(instrument=args.instrument, - descriptor=args.descriptor, - data_level=args.data_level) +query_args = dict( + instrument=args.instrument, descriptor=args.descriptor, data_level=args.data_level +) if args.reprocessing_time is not None: - analyze(args.reprocessing_time, - **query_args) + analyze(args.reprocessing_time, **query_args) else: - report_oldest_file(**query_args) \ No newline at end of file + report_oldest_file(**query_args) diff --git a/scripts/convert_product_definition_to_yaml.py b/scripts/convert_product_definition_to_yaml.py index 2ac3548e2..c01b728ee 100644 --- a/scripts/convert_product_definition_to_yaml.py +++ b/scripts/convert_product_definition_to_yaml.py @@ -6,30 +6,81 @@ def convert_csv_to_yaml(file_path): - required_for_support_data = ["NAME", "DATA_TYPE", "CATDESC", "VAR_TYPE", "DEPEND_0", "FIELDNAM", "FORMAT", "UNITS", - "VALIDMIN", "VALIDMAX", "FILLVAL", "RECORD_VARYING"] + required_for_support_data = [ + "NAME", + "DATA_TYPE", + "CATDESC", + "VAR_TYPE", + "DEPEND_0", + "FIELDNAM", + "FORMAT", + "UNITS", + "VALIDMIN", + "VALIDMAX", + "FILLVAL", + "RECORD_VARYING", + ] required_for_data = required_for_support_data + ["DISPLAY_TYPE"] - required_for_metadata = ["NAME", "DATA_TYPE", "CATDESC", "VAR_TYPE", "VAR_TYPE", "FIELDNAM", "FORMAT", ] - - non_required_columns = ["SCALE_TYP", "SCAL_PTR", "VAR_NOTES", "TIME_BASE", "TIME_SCALE", "LEAP_SECONDS_INCLUDED", - "ABSOLUTE_ERROR", "AVG_TYPE", "BIN_LOCATION", "DELTA_PLUS_VAR", "DELTA_MINUS_VAR", - "DERIVN", "DICT_KEY", "MONOTON", "SCALEMIN", "SCALEMAX", "REFERENCE_POSITION", - "RELATIVE_ERROR", "RESOLUTION", "SI_CONVERSION", "DEPEND_1", "DEPEND_2", "DEPEND_3", - "VARIABLE_PURPOSE", "UNIT_PTR", "LABL_PTR_1", "LABL_PTR_2", "LABL_PTR_3", "LABLAXIS"] + required_for_metadata = [ + "NAME", + "DATA_TYPE", + "CATDESC", + "VAR_TYPE", + "VAR_TYPE", + "FIELDNAM", + "FORMAT", + ] + + non_required_columns = [ + "SCALE_TYP", + "SCAL_PTR", + "VAR_NOTES", + "TIME_BASE", + "TIME_SCALE", + "LEAP_SECONDS_INCLUDED", + "ABSOLUTE_ERROR", + "AVG_TYPE", + "BIN_LOCATION", + "DELTA_PLUS_VAR", + "DELTA_MINUS_VAR", + "DERIVN", + "DICT_KEY", + "MONOTON", + "SCALEMIN", + "SCALEMAX", + "REFERENCE_POSITION", + "RELATIVE_ERROR", + "RESOLUTION", + "SI_CONVERSION", + "DEPEND_1", + "DEPEND_2", + "DEPEND_3", + "VARIABLE_PURPOSE", + "UNIT_PTR", + "LABL_PTR_1", + "LABL_PTR_2", + "LABL_PTR_3", + "LABLAXIS", + ] yaml_text = "" try: with open(file_path, "r") as f: csvreader = csv.reader(f) row_headers = next(csvreader) - header_to_index = {header: index for index, header in enumerate(row_headers)} + header_to_index = { + header: index for index, header in enumerate(row_headers) + } for row in csvreader: yaml_text += f"{row[0]}:\n" if row[header_to_index["NAME"]].lower() == "epoch": - required_variables = [variable for variable in required_for_support_data if - variable != "DEPEND_0"] + required_variables = [ + variable + for variable in required_for_support_data + if variable != "DEPEND_0" + ] elif row[header_to_index["VAR_TYPE"]] == "support_data": required_variables = required_for_support_data elif row[header_to_index["VAR_TYPE"]] == "metadata": @@ -37,10 +88,15 @@ def convert_csv_to_yaml(file_path): else: required_variables = required_for_data - if (("RECORD_VARYING" in required_variables or "RECORD_VARYING" in header_to_index) - and row[header_to_index["RECORD_VARYING"]].lower() == "nrv"): - required_variables = [variable for variable in required_variables if - variable != "DEPEND_0"] + if ( + "RECORD_VARYING" in required_variables + or "RECORD_VARYING" in header_to_index + ) and row[header_to_index["RECORD_VARYING"]].lower() == "nrv": + required_variables = [ + variable + for variable in required_variables + if variable != "DEPEND_0" + ] for index, rowitem in enumerate(row): if rowitem == "": @@ -48,10 +104,16 @@ def convert_csv_to_yaml(file_path): else: display_rowitem = rowitem - column_is_empty_and_not_required = (rowitem == "" and row_headers[index] not in required_variables) + column_is_empty_and_not_required = ( + rowitem == "" and row_headers[index] not in required_variables + ) column_not_listed_in_specified_list = row_headers[index] not in ( - required_variables + non_required_columns) - if column_is_empty_and_not_required or column_not_listed_in_specified_list: + required_variables + non_required_columns + ) + if ( + column_is_empty_and_not_required + or column_not_listed_in_specified_list + ): continue yaml_text += f" {row_headers[index]}: {display_rowitem}\n" @@ -85,7 +147,7 @@ def process_csv(): "glows-l3e-hi-45": "imap_l3_processing/cdf/data_product_definition_csv/imap_glows_l3e_survival-probability-hi-45_variable_attrs.csv", "glows-l3e-hi-90": "imap_l3_processing/cdf/data_product_definition_csv/imap_glows_l3e_survival-probability-hi-90_variable_attrs.csv", "glows-l3e-ul": "imap_l3_processing/cdf/data_product_definition_csv/imap_glows_l3e_survival-probability-ul_variable_attrs.csv", - "ultra-l3-survival-probability": "imap_l3_processing/cdf/data_product_definition_csv/imap_ultra_l3_variable_attrs.csv" + "ultra-l3-survival-probability": "imap_l3_processing/cdf/data_product_definition_csv/imap_ultra_l3_variable_attrs.csv", } file_path = instrument_to_file_path[args.instrument] @@ -95,8 +157,12 @@ def process_csv(): print(filename_without_extension) yaml_file_name = f"imap_l3_processing/cdf/config/{filename_without_extension}.yaml" if os.path.isfile(yaml_file_name): - if input( - "The Yaml already exists in the config folder! Are you sure you want to overwrite the other pairs work? [Y/n]") != "n": + if ( + input( + "The Yaml already exists in the config folder! Are you sure you want to overwrite the other pairs work? [Y/n]" + ) + != "n" + ): os.remove(yaml_file_name) with open(yaml_file_name, "w") as yaml_file: diff --git a/scripts/convert_variable_yaml_to_csv.py b/scripts/convert_variable_yaml_to_csv.py index 6a0ee3e5b..e453f4b9a 100644 --- a/scripts/convert_variable_yaml_to_csv.py +++ b/scripts/convert_variable_yaml_to_csv.py @@ -11,9 +11,17 @@ with open(yaml_path) as yaml_file: yaml_data = yaml.safe_load(yaml_file) - all_headers = {key: '' for key in - set(chain.from_iterable([yaml_data[variable].keys() for variable in yaml_data.keys()]))} - valid_variables = [yaml_data[var] for var in yaml_data.keys() if 'NAME' in yaml_data[var]] + all_headers = { + key: "" + for key in set( + chain.from_iterable( + [yaml_data[variable].keys() for variable in yaml_data.keys()] + ) + ) + } + valid_variables = [ + yaml_data[var] for var in yaml_data.keys() if "NAME" in yaml_data[var] + ] with open(outpath, "w") as out_file: writer = csv.DictWriter(out_file, fieldnames=all_headers.keys()) writer.writeheader() diff --git a/scripts/fetch_latest_data.py b/scripts/fetch_latest_data.py index 11bde547a..b3176416c 100644 --- a/scripts/fetch_latest_data.py +++ b/scripts/fetch_latest_data.py @@ -10,8 +10,8 @@ args = parser.parse_args() data = imap_data_access.query(instrument=args.instrument, data_level=args.level) -sorted_data = sorted(data, key=lambda r: r['ingestion_date']) -paths = [f['file_path'] for f in sorted_data[-int(args.count):]] +sorted_data = sorted(data, key=lambda r: r["ingestion_date"]) +paths = [f["file_path"] for f in sorted_data[-int(args.count) :]] for path in paths: imap_data_access.download(path) print(path) diff --git a/scripts/generate_hi_yaml.py b/scripts/generate_hi_yaml.py index 301fe69c5..c9c3856cf 100644 --- a/scripts/generate_hi_yaml.py +++ b/scripts/generate_hi_yaml.py @@ -9,7 +9,7 @@ "hnu": "Heliospheric Nose Upfield (HNU)", "gcs": "Galactic (GCS)", "rc": "Ribbon-centered (RC)", - "ccs": "custom (CCS)" + "ccs": "custom (CCS)", } logical_source_description_parts = { @@ -18,14 +18,14 @@ "hic": "Hi Combined", "ena": "ENA Intensity", "spx": "Spectral Index", - 'h': "Hydrogen", - 'sf': "Spacecraft Frame", - 'hf': "Heliospheric Frame", - 'sp': "Survival Probability-Corrected", - 'nsp': "Non-Survival Probability-Corrected", - 'full': "Full Spin", - 'ram': "Ram", - 'anti': "Anti-Ram", + "h": "Hydrogen", + "sf": "Spacecraft Frame", + "hf": "Heliospheric Frame", + "sp": "Survival Probability-Corrected", + "nsp": "Non-Survival Probability-Corrected", + "full": "Full Spin", + "ram": "Ram", + "anti": "Anti-Ram", "hae": "HAE", "2deg": "Rectangular 2 degree", "4deg": "Rectangular 4 degree", @@ -81,15 +81,28 @@ ] for descriptor in descriptors: - descriptor_parts = descriptor.split('-') + descriptor_parts = descriptor.split("-") sensor = descriptor_parts[0][1:] - [quantity, species, frame, sp_corrected, spin_range, coordinate_system, pixelation, time_range] = descriptor_parts[ - 1:] + [ + quantity, + species, + frame, + sp_corrected, + spin_range, + coordinate_system, + pixelation, + time_range, + ] = descriptor_parts[1:] products = { f"imap_{instrument}_l3_{descriptor}": { - "Logical_source_description": "IMAP Hi Instrument Level 3 " + ', '.join( - [logical_source_description_parts[part] for part in descriptor.split("-")]), + "Logical_source_description": "IMAP Hi Instrument Level 3 " + + ", ".join( + [ + logical_source_description_parts[part] + for part in descriptor.split("-") + ] + ), "Data_level": "3", "Data_type": f"L3_{time_range}>Level-3 {time_range}", "Map_descriptor": descriptor, @@ -101,7 +114,9 @@ "Spin_range": logical_source_description_parts[spin_range], "Survival_corrected": "True" if sp_corrected == "sp" else "False", "Species": species.capitalize(), - "Principal_data_quantity": "ENA Intensity" if quantity == "ena" else "ENA Spectral Index", + "Principal_data_quantity": "ENA Intensity" + if quantity == "ena" + else "ENA Spectral Index", } } diff --git a/scripts/generate_lo_yaml.py b/scripts/generate_lo_yaml.py index da4d3c349..b3ccae87e 100644 --- a/scripts/generate_lo_yaml.py +++ b/scripts/generate_lo_yaml.py @@ -9,7 +9,7 @@ "hnu": "Heliospheric Nose Upfield (HNU)", "gcs": "Galactic (GCS)", "rc": "Ribbon-centered (RC)", - "ccs": "custom (CCS)" + "ccs": "custom (CCS)", } logical_source_description_parts = { @@ -17,15 +17,15 @@ "lc": "Lo Combined", "ena": "ENA Intensity", "spx": "Spectral Index", - 'h': "Hydrogen", - 'sf': "Spacecraft Frame", - 'hf': "Heliospheric Frame", - 'hk': "Heliospheric Kinematic Frame", - 'sp': "Survival Probability-Corrected", - 'nsp': "Non-Survival Probability-Corrected", - 'full': "Full Spin", - 'ram': "Ram", - 'anti': "Anti-Ram", + "h": "Hydrogen", + "sf": "Spacecraft Frame", + "hf": "Heliospheric Frame", + "hk": "Heliospheric Kinematic Frame", + "sp": "Survival Probability-Corrected", + "nsp": "Non-Survival Probability-Corrected", + "full": "Full Spin", + "ram": "Ram", + "anti": "Anti-Ram", "hae": "HAE", "2deg": "Rectangular 2 degree", "4deg": "Rectangular 4 degree", @@ -45,28 +45,44 @@ ] for descriptor in descriptors: - descriptor_parts = descriptor.split('-') + descriptor_parts = descriptor.split("-") sensor = descriptor_parts[0][1:] - [quantity, species, frame, sp_corrected, spin_range, coordinate_system, pixelation, time_range] = descriptor_parts[ - 1:] + [ + quantity, + species, + frame, + sp_corrected, + spin_range, + coordinate_system, + pixelation, + time_range, + ] = descriptor_parts[1:] products = { f"imap_{instrument}_l3_{descriptor}": { - "Logical_source_description": "IMAP Lo Instrument Level 3 " + ', '.join( - [logical_source_description_parts[part] for part in descriptor.split("-")]), + "Logical_source_description": "IMAP Lo Instrument Level 3 " + + ", ".join( + [ + logical_source_description_parts[part] + for part in descriptor.split("-") + ] + ), "Data_level": "3", "Data_type": f"L3_{time_range}>Level-3 {time_range}", "Map_descriptor": descriptor, "Map_duration": time_range, "Instrument": f"Lo {sensor}" if sensor in ["45", "90"] else "Lo", - "Reference_frame": "Heliospheric Kinematic (HK)" if frame == "hk" else ( - "Spacecraft" if frame == "sf" else "Heliospheric (CG)"), + "Reference_frame": "Heliospheric Kinematic (HK)" + if frame == "hk" + else ("Spacecraft" if frame == "sf" else "Heliospheric (CG)"), "Coordinate_system": coordinate_systems[coordinate_system], "Tessellation_type": "lon, lat" if "deg" in pixelation else "pix index", "Spin_range": "Ram", "Survival_corrected": "True" if sp_corrected == "sp" else "False", "Species": species.capitalize(), - "Principal_data_quantity": "ENA Intensity" if quantity == "ena" else "ENA Spectral Index", + "Principal_data_quantity": "ENA Intensity" + if quantity == "ena" + else "ENA Spectral Index", } } diff --git a/scripts/generate_repointing_lines.py b/scripts/generate_repointing_lines.py index 3b6371a7b..d84a86e25 100644 --- a/scripts/generate_repointing_lines.py +++ b/scripts/generate_repointing_lines.py @@ -14,8 +14,12 @@ def add_repointings(): first_repointing_inclusive = 1000 - 104 for repointing in range(first_repointing_inclusive, last_repointing_exclusive): - start_datetime = repoint1000minus104start + (repointing - (first_repointing_inclusive)) * timedelta(days=1) - end_datetime = repoint1000minus104end + (repointing - (first_repointing_inclusive)) * timedelta(days=1) + start_datetime = repoint1000minus104start + ( + repointing - (first_repointing_inclusive) + ) * timedelta(days=1) + end_datetime = repoint1000minus104end + ( + repointing - (first_repointing_inclusive) + ) * timedelta(days=1) start_sclk = int((start_datetime - spacecraft_clock_start).total_seconds()) repoint_start_subsec_sclk = 20 @@ -23,8 +27,8 @@ def add_repointings(): end_sclk = int((end_datetime - spacecraft_clock_start).total_seconds()) repoint_end_subsec_sclk = 999980 - formatted_start = start_datetime.strftime('%Y-%m-%d %H:%M:%S.%f') - formatted_end = end_datetime.strftime('%Y-%m-%d %H:%M:%S.%f') + formatted_start = start_datetime.strftime("%Y-%m-%d %H:%M:%S.%f") + formatted_end = end_datetime.strftime("%Y-%m-%d %H:%M:%S.%f") csv_line = f"{start_sclk},{repoint_start_subsec_sclk},{end_sclk},{repoint_end_subsec_sclk},{formatted_start},{formatted_end},{repointing}" print(csv_line) diff --git a/scripts/generate_ultra_yaml.py b/scripts/generate_ultra_yaml.py index 4b763e96c..773d04440 100644 --- a/scripts/generate_ultra_yaml.py +++ b/scripts/generate_ultra_yaml.py @@ -9,7 +9,7 @@ "hnu": "Heliospheric Nose Upfield (HNU)", "gcs": "Galactic (GCS)", "rc": "Ribbon-centered (RC)", - "ccs": "custom (CCS)" + "ccs": "custom (CCS)", } logical_source_description_parts = { @@ -18,12 +18,12 @@ "ulc": "Ultra Combined", "ena": "ENA Intensity", "spx": "Spectral Index", - 'h': "Hydrogen", - 'sf': "Spacecraft Frame", - 'hf': "Heliospheric Frame", - 'sp': "Survival Probability-Corrected", - 'nsp': "Non-Survival Probability-Corrected", - 'full': "Full Spin", + "h": "Hydrogen", + "sf": "Spacecraft Frame", + "hf": "Heliospheric Frame", + "sp": "Survival Probability-Corrected", + "nsp": "Non-Survival Probability-Corrected", + "full": "Full Spin", "hae": "HAE", "2deg": "Rectangular 2 degree", "4deg": "Rectangular 4 degree", @@ -136,15 +136,28 @@ ] for descriptor in descriptors: - descriptor_parts = descriptor.split('-') + descriptor_parts = descriptor.split("-") sensor = descriptor_parts[0][1:] - [quantity, species, frame, sp_corrected, spin_range, coordinate_system, pixelation, time_range] = descriptor_parts[ - 1:] + [ + quantity, + species, + frame, + sp_corrected, + spin_range, + coordinate_system, + pixelation, + time_range, + ] = descriptor_parts[1:] products = { f"imap_{instrument}_l3_{descriptor}": { - "Logical_source_description": "IMAP Ultra Instrument Level 3 " + ', '.join( - [logical_source_description_parts[part] for part in descriptor.split("-")]), + "Logical_source_description": "IMAP Ultra Instrument Level 3 " + + ", ".join( + [ + logical_source_description_parts[part] + for part in descriptor.split("-") + ] + ), "Data_level": "3", "Data_type": f"L3_{time_range}>Level-3 {time_range}", "Map_descriptor": descriptor, @@ -156,7 +169,9 @@ "Spin_range": "Full Spin", "Survival_corrected": "True" if sp_corrected == "sp" else "False", "Species": species.capitalize(), - "Principal_data_quantity": "ENA Intensity" if quantity == "ena" else "ENA Spectral Index", + "Principal_data_quantity": "ENA Intensity" + if quantity == "ena" + else "ENA Spectral Index", } } diff --git a/scripts/get_batch_jobs_status.py b/scripts/get_batch_jobs_status.py index 991f33811..030ae4af5 100644 --- a/scripts/get_batch_jobs_status.py +++ b/scripts/get_batch_jobs_status.py @@ -30,35 +30,56 @@ class Command(enum.Enum): Query = "query" Logs = "logs" + arg_parser = ArgumentParser() subparsers = arg_parser.add_subparsers() -query_parser = subparsers.add_parser('query') -queryable_fields = ["status", "instrument", "data_level", "descriptor", "start_date", "version", "repointing", - "job_definition", "job_log_stream_id", "container_image", "container_command", - "started_at", "stopped_at"] +query_parser = subparsers.add_parser("query") +queryable_fields = [ + "status", + "instrument", + "data_level", + "descriptor", + "start_date", + "version", + "repointing", + "job_definition", + "job_log_stream_id", + "container_image", + "container_command", + "started_at", + "stopped_at", +] for field in queryable_fields: query_parser.add_argument(f"--{field}", type=str) -log_parser = subparsers.add_parser('logs') -log_parser.add_argument('log_id', type=str) +log_parser = subparsers.add_parser("logs") +log_parser.add_argument("log_id", type=str) args = arg_parser.parse_args() print(args) imap_dev_url = os.getenv("IMAP_DATA_ACCESS_URL") -headers = {'x-api-key': os.getenv("IMAP_API_KEY")} +headers = {"x-api-key": os.getenv("IMAP_API_KEY")} if "log_id" not in vars(args): - query_params = { key: val for key, val in vars(args).items() if val is not None and key not in ['command']} - response = requests.get(url=f"{imap_dev_url}/processing-jobs", headers=headers, params=query_params) + query_params = { + key: val + for key, val in vars(args).items() + if val is not None and key not in ["command"] + } + response = requests.get( + url=f"{imap_dev_url}/processing-jobs", headers=headers, params=query_params + ) processing_jobs = response.json() records = [] for processing_job in processing_jobs: if processing_job["started_at"] is not None: - processing_job["started_at"] = datetime.fromisoformat(processing_job["started_at"]).strftime("%Y-%m-%d %H:%M:%S") + processing_job["started_at"] = datetime.fromisoformat( + processing_job["started_at"] + ).strftime("%Y-%m-%d %H:%M:%S") relevant_fields = [ "status", @@ -68,18 +89,24 @@ class Command(enum.Enum): "start_date", "version", "started_at", - "job_log_stream_id" + "job_log_stream_id", ] - records.append({key: val for key, val in processing_job.items() if key in relevant_fields}) + records.append( + {key: val for key, val in processing_job.items() if key in relevant_fields} + ) - pd.set_option('display.max_colwidth', None) + pd.set_option("display.max_colwidth", None) if len(records) > 0: - print(pd.DataFrame.from_records(records).sort_values('started_at', na_position='last').tail(50)) + print( + pd.DataFrame.from_records(records) + .sort_values("started_at", na_position="last") + .tail(50) + ) else: print("Found no jobs that matched query params!") else: - logs = requests.get(url=f"{imap_dev_url}/processing-logs/{args.log_id}", headers=headers) + logs = requests.get( + url=f"{imap_dev_url}/processing-logs/{args.log_id}", headers=headers + ) print(logs.text) - - diff --git a/scripts/glows/create_example_glows_l3e_survival_probabilities_cdf.py b/scripts/glows/create_example_glows_l3e_survival_probabilities_cdf.py index 5f3852cf8..6f203465c 100644 --- a/scripts/glows/create_example_glows_l3e_survival_probabilities_cdf.py +++ b/scripts/glows/create_example_glows_l3e_survival_probabilities_cdf.py @@ -8,34 +8,50 @@ from spacepy import pycdf from spacepy.pycdf import CDF -from imap_l3_processing.glows.l3e.glows_l3e_hi_model import PROBABILITY_OF_SURVIVAL_VAR_NAME +from imap_l3_processing.glows.l3e.glows_l3e_hi_model import ( + PROBABILITY_OF_SURVIVAL_VAR_NAME, +) from tests.test_helpers import get_test_data_path -def create_survival_probabilities_file(glows_file_path: Path, date_for_file: datetime, sensor: str): +def create_survival_probabilities_file( + glows_file_path: Path, date_for_file: datetime, sensor: str +): filename = f"imap_glows_l3e_survival-probabilities-hi-{sensor}_{date_for_file.strftime('%Y%m%d')}_v001.cdf" - cdf_file_path = get_test_data_path(f"hi/fake_l3e_survival_probabilities/{sensor}/{filename}") + cdf_file_path = get_test_data_path( + f"hi/fake_l3e_survival_probabilities/{sensor}/{filename}" + ) cdf_file_path.unlink(missing_ok=True) with open(glows_file_path) as input_data: - energy_line = [line for line in input_data.readlines() if line.startswith("#energy_grid")] + energy_line = [ + line for line in input_data.readlines() if line.startswith("#energy_grid") + ] assert len(energy_line) == 1 - energies = np.array([float("".join(list(t))) for t in itertools.batched(energy_line[0][19:-1], 10)]) - energy_units = re.search(r'\[(.+)]', energy_line[0]).group(1) + energies = np.array( + [ + float("".join(list(t))) + for t in itertools.batched(energy_line[0][19:-1], 10) + ] + ) + energy_units = re.search(r"\[(.+)]", energy_line[0]).group(1) spin_angle_and_survival_probabilities = np.loadtxt(glows_file_path, skiprows=200) spin_angles = spin_angle_and_survival_probabilities[:, 0] survival_probabilities = spin_angle_and_survival_probabilities[:, 1:].T - with CDF(str(cdf_file_path), '') as c: + with CDF(str(cdf_file_path), "") as c: c.new("epoch", [date_for_file], pycdf.const.CDF_TIME_TT2000) c.new("epoch_delta", [12 * 60 * 60 * 1e9], pycdf.const.CDF_INT8) c.new("energy_grid", energies, pycdf.const.CDF_FLOAT, recVary=False) - c['energy_grid'].attrs["UNITS"] = energy_units + c["energy_grid"].attrs["UNITS"] = energy_units c.new("spin_angle", spin_angles, pycdf.const.CDF_FLOAT, recVary=False) - c.new(PROBABILITY_OF_SURVIVAL_VAR_NAME, np.array(survival_probabilities)[np.newaxis, ...], - pycdf.const.CDF_FLOAT, - compress=pycdf.const.GZIP_COMPRESSION) + c.new( + PROBABILITY_OF_SURVIVAL_VAR_NAME, + np.array(survival_probabilities)[np.newaxis, ...], + pycdf.const.CDF_FLOAT, + compress=pycdf.const.GZIP_COMPRESSION, + ) c["energy_grid"].attrs["FILLVAL"] = -1e31 c["spin_angle"].attrs["FILLVAL"] = -1e31 diff --git a/scripts/glows/create_l2_glows_cdf_from_json.py b/scripts/glows/create_l2_glows_cdf_from_json.py index ad09e91d9..f048c1989 100644 --- a/scripts/glows/create_l2_glows_cdf_from_json.py +++ b/scripts/glows/create_l2_glows_cdf_from_json.py @@ -12,8 +12,9 @@ from tests.test_helpers import get_test_data_path - -def fill_official_l2_cdf_with_json_values(output_folder: Path, input_path: Path) -> None: +def fill_official_l2_cdf_with_json_values( + output_folder: Path, input_path: Path +) -> None: official_l2_path = get_test_data_path( "glows/imap_glows_l2_hist_20260323-repoint00194_v005.cdf" ) @@ -27,7 +28,7 @@ def fill_official_l2_cdf_with_json_values(output_folder: Path, input_path: Path) epoch_window = end_of_epoch_window - start_of_epoch_window epoch = start_of_epoch_window + epoch_window / 2 - repoint_id = int(re.search(r'repoint(\d{5})', input_path.name)[1]) + repoint_id = int(re.search(r"repoint(\d{5})", input_path.name)[1]) new_name = ScienceFilePath.generate_from_inputs( instrument="glows", data_level="l2", @@ -109,8 +110,12 @@ def fill_official_l2_cdf_with_json_values(output_folder: Path, input_path: Path) "number_of_all_l1b_files" ] cdf.attrs["pkts_file_name"] = instrument_data["header"]["pkts_file_name"] - cdf.attrs["ancillary_data_files"] = instrument_data["header"]["ancillary_data_files"] - cdf.attrs["flight_software_version"] = instrument_data["header"]["flight_software_version"] + cdf.attrs["ancillary_data_files"] = instrument_data["header"][ + "ancillary_data_files" + ] + cdf.attrs["flight_software_version"] = instrument_data["header"][ + "flight_software_version" + ] if __name__ == "__main__": diff --git a/scripts/glows/create_l3a_glows_cdf_from_json.py b/scripts/glows/create_l3a_glows_cdf_from_json.py index ca0457268..c9ae59901 100644 --- a/scripts/glows/create_l3a_glows_cdf_from_json.py +++ b/scripts/glows/create_l3a_glows_cdf_from_json.py @@ -16,31 +16,34 @@ for file in files_to_convert: with open(Path(input_file_directory) / file, "r") as l3a_json: data = json.load(l3a_json) - date = file.split('_')[3][:8] + date = file.split("_")[3][:8] year = int(date[:4]) month = int(date[4:6]) day = int(date[6:8]) start_date = datetime(year, month, day) input_metadata = InputMetadata( - instrument='glows', - data_level='l3a', + instrument="glows", + data_level="l3a", start_date=start_date, end_date=start_date + timedelta(days=1), - version='v001', - descriptor='hist' + version="v001", + descriptor="hist", ) files = { - "settings": - Path(scripts.__file__).parent.parent / - "instrument_team_data/glows/imap_glows_pipeline-settings_20100101_v001.json" + "settings": Path(scripts.__file__).parent.parent + / "instrument_team_data/glows/imap_glows_pipeline-settings_20100101_v001.json" } data_with_spin_angle = GlowsProcessor.add_spin_angle_delta(data, files) - glows_l3a_lightcurve = create_glows_l3a_from_dictionary(data_with_spin_angle, - input_metadata) + glows_l3a_lightcurve = create_glows_l3a_from_dictionary( + data_with_spin_angle, input_metadata + ) - cdf_path = save_data(glows_l3a_lightcurve, delete_if_present=True, - folder_path=get_test_data_path("glows/l3a_products")) + cdf_path = save_data( + glows_l3a_lightcurve, + delete_if_present=True, + folder_path=get_test_data_path("glows/l3a_products"), + ) print("cdf written to", cdf_path) diff --git a/scripts/glows/download_l2_prod_data.py b/scripts/glows/download_l2_prod_data.py index 4b05b4b31..590de2e8d 100644 --- a/scripts/glows/download_l2_prod_data.py +++ b/scripts/glows/download_l2_prod_data.py @@ -1,4 +1,3 @@ - from datetime import datetime from pathlib import Path @@ -13,9 +12,13 @@ from tests.test_helpers import get_run_local_data_path GLOWS_L3E_REQUIRED_SPICE_KERNELS: list[SpiceKernelTypes] = [ - SpiceKernelTypes.ScienceFrames, SpiceKernelTypes.EphemerisReconstructed, SpiceKernelTypes.AttitudeHistory, - SpiceKernelTypes.PointingAttitude, SpiceKernelTypes.PlanetaryEphemeris, SpiceKernelTypes.Leapseconds, - SpiceKernelTypes.SpacecraftClock + SpiceKernelTypes.ScienceFrames, + SpiceKernelTypes.EphemerisReconstructed, + SpiceKernelTypes.AttitudeHistory, + SpiceKernelTypes.PointingAttitude, + SpiceKernelTypes.PlanetaryEphemeris, + SpiceKernelTypes.Leapseconds, + SpiceKernelTypes.SpacecraftClock, ] @@ -26,7 +29,9 @@ def datetime_to_j2000_seconds(date: datetime) -> int: glows_prod_data_path = get_run_local_data_path("glows_prod_data") imap_data_access.config["DATA_DIR"] = glows_prod_data_path -l2_query_response = imap_data_access.query(instrument="glows", data_level="l2", version="latest") +l2_query_response = imap_data_access.query( + instrument="glows", data_level="l2", version="latest" +) for file in l2_query_response: print(f"downloading: {Path(file['file_path']).name}") @@ -39,7 +44,7 @@ def datetime_to_j2000_seconds(date: datetime) -> int: "file_types": [t.value for t in GLOWS_L3E_REQUIRED_SPICE_KERNELS], "start_time": datetime_to_j2000_seconds(start_of_l2), "end_time": datetime_to_j2000_seconds(end_of_l2), - "list_files": "true" + "list_files": "true", } resp = requests.get("https://api.imap-mission.com/metakernel", params=metakernel_params) for spice_file in resp.json(): diff --git a/scripts/glows/download_validation_data.py b/scripts/glows/download_validation_data.py index f2cc0ff73..53effeb5e 100644 --- a/scripts/glows/download_validation_data.py +++ b/scripts/glows/download_validation_data.py @@ -5,7 +5,9 @@ from dotenv import load_dotenv load_dotenv() -os.environ["IMAP_DATA_DIR"] = str(Path(__file__).parent.parent.parent / "glows_l3_validation_data") +os.environ["IMAP_DATA_DIR"] = str( + Path(__file__).parent.parent.parent / "glows_l3_validation_data" +) import imap_data_access @@ -14,19 +16,16 @@ ("map-of-extra-helio-bckgrd", "20100101"), ("pipeline-settings", "20100101"), ("time-dep-bckgrd", "20100101"), - ("uv-anisotropy-1CR", "20100101"), ("WawHelioIonMP", "20100101"), ("bad-days-list", "20100101"), ("pipeline-settings-l3bcde", "20100101"), - ("plasma-speed-2010a", "20100101"), ("proton-density-2010a", "20100101"), ("uv-anisotropy-2010a", "20100101"), ("photoion-2010a", "20100101"), ("lya-2010a", "20100101"), ("electron-density-2010a", "20100101"), - ("lya", "19470303"), ("e-dens", "19470303"), ("p-dens", "19470303"), @@ -41,8 +40,10 @@ instrument="glows", descriptor=descriptor, start_date=start_date, - end_date=(datetime.strptime(start_date, "%Y%m%d") + timedelta(days=1)).strftime("%Y%m%d"), - version="latest" + end_date=(datetime.strptime(start_date, "%Y%m%d") + timedelta(days=1)).strftime( + "%Y%m%d" + ), + version="latest", ) print(imap_data_access.download(result["file_path"])) @@ -52,13 +53,20 @@ ("l3a", "hist"), ("l3b", "ion-rate-profile"), ("l3c", "sw-profile"), - ("l3d", "solar-hist") + ("l3d", "solar-hist"), ] for data_level, descriptor in data_files: - for qr in imap_data_access.query(instrument="glows", data_level=data_level, descriptor=descriptor, version="latest"): + for qr in imap_data_access.query( + instrument="glows", + data_level=data_level, + descriptor=descriptor, + version="latest", + ): print(imap_data_access.download(qr["file_path"])) # download l3bc archive -for qr in imap_data_access.query(table="ancillary", instrument="glows", descriptor="l3b-archive", version="latest"): +for qr in imap_data_access.query( + table="ancillary", instrument="glows", descriptor="l3b-archive", version="latest" +): print(imap_data_access.download(qr["file_path"])) diff --git a/scripts/glows/upload_l3a_ancillary_files.py b/scripts/glows/upload_l3a_ancillary_files.py index 910ae7b7a..6cb241c34 100644 --- a/scripts/glows/upload_l3a_ancillary_files.py +++ b/scripts/glows/upload_l3a_ancillary_files.py @@ -2,9 +2,9 @@ from tests.test_helpers import get_test_instrument_team_data_path -glows_file_dir = get_test_instrument_team_data_path('glows') +glows_file_dir = get_test_instrument_team_data_path("glows") -upload(glows_file_dir / 'imap_glows_calibration-data_20100101_v002.dat') -upload(glows_file_dir / 'imap_glows_map-of-extra-helio-bckgrd_20100101_v002.dat') -upload(glows_file_dir / 'imap_glows_pipeline-settings_20100101_v001.json') -upload(glows_file_dir / 'imap_glows_time-dep-bckgrd_20100101_v001.dat') +upload(glows_file_dir / "imap_glows_calibration-data_20100101_v002.dat") +upload(glows_file_dir / "imap_glows_map-of-extra-helio-bckgrd_20100101_v002.dat") +upload(glows_file_dir / "imap_glows_pipeline-settings_20100101_v001.json") +upload(glows_file_dir / "imap_glows_time-dep-bckgrd_20100101_v001.dat") diff --git a/scripts/hi/create_all_maps_from_inputs.py b/scripts/hi/create_all_maps_from_inputs.py index 4b781653d..f24ed7579 100644 --- a/scripts/hi/create_all_maps_from_inputs.py +++ b/scripts/hi/create_all_maps_from_inputs.py @@ -10,8 +10,13 @@ from tests.integration.integration_test_helpers import mock_imap_data_access -@patch('imap_l3_data_processor._parse_cli_arguments') -def run_all_maps(mock_parse_cli_arguments, *, output_dir, input_dir,): +@patch("imap_l3_data_processor._parse_cli_arguments") +def run_all_maps( + mock_parse_cli_arguments, + *, + output_dir, + input_dir, +): intermediate_output = output_dir.parent / f"{output_dir.name}_intermediate" mock_arguments = Mock() mock_arguments.instrument = "hi" @@ -23,26 +28,32 @@ def run_all_maps(mock_parse_cli_arguments, *, output_dir, input_dir,): mock_arguments.dependency = "[]" mock_arguments.upload_to_sdc = False mock_parse_cli_arguments.return_value = mock_arguments - with mock_imap_data_access(Path(intermediate_output), list(input_dir.rglob('*.*'))): + with mock_imap_data_access(Path(intermediate_output), list(input_dir.rglob("*.*"))): mock_arguments.descriptor = "sp-maps" - logging.basicConfig(force=True, level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') - + logging.basicConfig( + force=True, + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) imap_l3_data_processor.imap_l3_processor() spiceypy.kclear() - with mock_imap_data_access(output_dir, list(Path(intermediate_output).rglob('*.*'))): + with mock_imap_data_access( + output_dir, list(Path(intermediate_output).rglob("*.*")) + ): mock_arguments.descriptor = "hic-maps" imap_l3_data_processor.imap_l3_processor() spiceypy.kclear() if __name__ == "__main__": - output_dir = Path(f'{datetime.now().strftime('%Y%m%d')}_hi_validation_output') + output_dir = Path(f"{datetime.now().strftime('%Y%m%d')}_hi_validation_output") try: input_dir = Path(sys.argv[1]) except: - input_dir = Path('hi_validation_input') - print(f"Could not open input file, using default input path of {input_dir} instead") + input_dir = Path("hi_validation_input") + print( + f"Could not open input file, using default input path of {input_dir} instead" + ) run_all_maps(output_dir=output_dir, input_dir=input_dir) diff --git a/scripts/hi/create_cdf_from_ibex.py b/scripts/hi/create_cdf_from_ibex.py index 3102c2018..8f586f083 100644 --- a/scripts/hi/create_cdf_from_ibex.py +++ b/scripts/hi/create_cdf_from_ibex.py @@ -16,10 +16,10 @@ def extract_data(lines: list[str]): - start_row, dimension_string = lines[0][2:10].split(':') - data_lines = lines[int(start_row):] - data = np.fromstring('\t'.join(data_lines), dtype=float, sep='\t') - return data.reshape(tuple(int(x) for x in dimension_string.split('x'))).T + start_row, dimension_string = lines[0][2:10].split(":") + data_lines = lines[int(start_row) :] + data = np.fromstring("\t".join(data_lines), dtype=float, sep="\t") + return data.reshape(tuple(int(x) for x in dimension_string.split("x"))).T fluxes = [] @@ -28,17 +28,21 @@ def extract_data(lines: list[str]): for file in folder.iterdir(): if file.parts[-1].endswith("flux.txt") and "mono" not in file.parts[-1]: - group_file_name_prefix = '-'.join(file.parts[-1].split('-')[:-1]) + group_file_name_prefix = "-".join(file.parts[-1].split("-")[:-1]) - with open(file.parent / (group_file_name_prefix + "-flux.txt"), "r") as flux_file: + with open( + file.parent / (group_file_name_prefix + "-flux.txt"), "r" + ) as flux_file: fluxes.append(extract_data(flux_file.readlines())) - with open(file.parent / (group_file_name_prefix + "-fvar.txt"), "r") as variance_file: + with open( + file.parent / (group_file_name_prefix + "-fvar.txt"), "r" + ) as variance_file: variances.append(extract_data(variance_file.readlines())) rng = np.random.default_rng() pathname = f"C://Users//Harrison//Development//imap_L3_processing//tests//test_data//{folder.name}.cdf" -with CDF(pathname, '') as cdf: +with CDF(pathname, "") as cdf: cdf.col_major(True) flux_data = np.stack(fluxes, axis=-1) diff --git a/scripts/hi/create_cdf_from_instrument_team_data.py b/scripts/hi/create_cdf_from_instrument_team_data.py index 92ca6df5c..d16c44c13 100644 --- a/scripts/hi/create_cdf_from_instrument_team_data.py +++ b/scripts/hi/create_cdf_from_instrument_team_data.py @@ -9,11 +9,20 @@ from spacepy import pycdf from spacepy.pycdf import CDF -from imap_l3_processing.constants import FIVE_MINUTES_IN_NANOSECONDS, ONE_SECOND_IN_NANOSECONDS, SECONDS_PER_DAY -from tests.test_helpers import get_run_local_data_path, get_test_instrument_team_data_path - - -def create_l2_map_from_instrument_team(folder: Path, output_dir: Path, pixel_size: float = 4) -> Path: +from imap_l3_processing.constants import ( + FIVE_MINUTES_IN_NANOSECONDS, + ONE_SECOND_IN_NANOSECONDS, + SECONDS_PER_DAY, +) +from tests.test_helpers import ( + get_run_local_data_path, + get_test_instrument_team_data_path, +) + + +def create_l2_map_from_instrument_team( + folder: Path, output_dir: Path, pixel_size: float = 4 +) -> Path: epoch = np.array([datetime(2025, 5, 1)]) epoch_delta = np.array([FIVE_MINUTES_IN_NANOSECONDS]) @@ -21,10 +30,10 @@ def create_l2_map_from_instrument_team(folder: Path, output_dir: Path, pixel_siz for sub_folder in folder.iterdir(): for file in sub_folder.iterdir(): energy_string = re.search(r"_(\d*p\d*)keV", file.name).group(1) - energy = float(energy_string.replace('p', '.')) + energy = float(energy_string.replace("p", ".")) data[sub_folder.name][energy] = np.loadtxt(file, delimiter=",").T - assert (data['flux'].keys() == data['sigma'].keys()) - energy = np.array([key for key in sorted(data['flux'].keys())]) + assert data["flux"].keys() == data["sigma"].keys() + energy = np.array([key for key in sorted(data["flux"].keys())]) energy_delta_plus = np.full(energy.shape, 0.25) energy_delta_minus = np.full(energy.shape, 0.25) energy_label = energy.astype(str) @@ -38,27 +47,36 @@ def create_l2_map_from_instrument_team(folder: Path, output_dir: Path, pixel_siz lon_label = [f"{x} deg" for x in lon] lat_label = [f"{x} deg" for x in lat] - intensity = np.array([data['flux'][key] for key in sorted(data['flux'].keys())]) + intensity = np.array([data["flux"][key] for key in sorted(data["flux"].keys())]) ena_intensity = intensity.reshape(1, *intensity.shape) - ena_intensity_stat_uncert = np.array([data['sigma'][key] for key in sorted(data['sigma'].keys())])[np.newaxis, ...] + ena_intensity_stat_uncert = np.array( + [data["sigma"][key] for key in sorted(data["sigma"].keys())] + )[np.newaxis, ...] # interpolate the original grid onto a new spacing if not np.isclose(pixel_size, 4): coords = np.stack(np.meshgrid(lon, lat), axis=-1) - new_ena_intensity = np.full((1, ena_intensity.shape[1], len(lon), len(lat)), -1e31) - new_ena_intensity_stat_unc = np.full((1, ena_intensity.shape[1], len(lon), len(lat)), -1e31) + new_ena_intensity = np.full( + (1, ena_intensity.shape[1], len(lon), len(lat)), -1e31 + ) + new_ena_intensity_stat_unc = np.full( + (1, ena_intensity.shape[1], len(lon), len(lat)), -1e31 + ) lon_4_deg = np.arange(0.0, 360.0, 4.0) lat_4_deg = np.arange(-88.0, 92.0, 4.0) for energy_i in range(ena_intensity.shape[1]): - interp = scipy.interpolate.RegularGridInterpolator((lon_4_deg, lat_4_deg), ena_intensity[0, energy_i]) + interp = scipy.interpolate.RegularGridInterpolator( + (lon_4_deg, lat_4_deg), ena_intensity[0, energy_i] + ) new_ena_intensity[0, energy_i] = interp(coords).T - interp = scipy.interpolate.RegularGridInterpolator((lon_4_deg, lat_4_deg), - ena_intensity_stat_uncert[0, energy_i]) + interp = scipy.interpolate.RegularGridInterpolator( + (lon_4_deg, lat_4_deg), ena_intensity_stat_uncert[0, energy_i] + ) new_ena_intensity_stat_unc[0, energy_i] = interp(coords).T ena_intensity = new_ena_intensity @@ -69,17 +87,19 @@ def create_l2_map_from_instrument_team(folder: Path, output_dir: Path, pixel_siz solid_angle = build_solid_angle_map(pixel_size) output_dir.mkdir(parents=True, exist_ok=True) - pathname = output_dir / f'{folder.name}.cdf' + pathname = output_dir / f"{folder.name}.cdf" obs_date = np.full(ena_intensity.shape, datetime.now()) - obs_date_range = np.full(ena_intensity.shape, ONE_SECOND_IN_NANOSECONDS * SECONDS_PER_DAY * 2) + obs_date_range = np.full( + ena_intensity.shape, ONE_SECOND_IN_NANOSECONDS * SECONDS_PER_DAY * 2 + ) exposure_mask = ena_intensity == 0 exposure_factor = np.full(ena_intensity.shape, 1.0) exposure_factor[exposure_mask] = 0 Path(pathname).unlink(missing_ok=True) - with CDF(str(pathname), '') as cdf: + with CDF(str(pathname), "") as cdf: cdf.col_major(True) cdf.new("epoch", epoch, type=pycdf.const.CDF_TIME_TT2000) @@ -90,16 +110,42 @@ def create_l2_map_from_instrument_team(folder: Path, output_dir: Path, pixel_siz cdf.new("longitude", lon, recVary=False, type=pycdf.const.CDF_FLOAT) cdf.new("longitude_delta", lon_delta, recVary=False, type=pycdf.const.CDF_FLOAT) cdf.new("longitude_label", lon_label, recVary=False, type=pycdf.const.CDF_CHAR) - cdf.new("ena_intensity", ena_intensity, recVary=True, type=pycdf.const.CDF_FLOAT) - cdf.new("ena_intensity_stat_uncert", ena_intensity_stat_uncert, recVary=True, type=pycdf.const.CDF_FLOAT) - cdf.new("ena_intensity_sys_err", ena_intensity_sys_err, recVary=True, type=pycdf.const.CDF_FLOAT) - cdf.new("exposure_factor", exposure_factor, recVary=True, type=pycdf.const.CDF_FLOAT) + cdf.new( + "ena_intensity", ena_intensity, recVary=True, type=pycdf.const.CDF_FLOAT + ) + cdf.new( + "ena_intensity_stat_uncert", + ena_intensity_stat_uncert, + recVary=True, + type=pycdf.const.CDF_FLOAT, + ) + cdf.new( + "ena_intensity_sys_err", + ena_intensity_sys_err, + recVary=True, + type=pycdf.const.CDF_FLOAT, + ) + cdf.new( + "exposure_factor", exposure_factor, recVary=True, type=pycdf.const.CDF_FLOAT + ) cdf.new("obs_date", obs_date, recVary=True, type=pycdf.const.CDF_TIME_TT2000) - cdf.new("obs_date_range", obs_date_range, recVary=True, type=pycdf.const.CDF_INT8) + cdf.new( + "obs_date_range", obs_date_range, recVary=True, type=pycdf.const.CDF_INT8 + ) cdf.new("solid_angle", solid_angle, recVary=True, type=pycdf.const.CDF_FLOAT) cdf.new("epoch_delta", epoch_delta, recVary=True, type=pycdf.const.CDF_INT8) - cdf.new("energy_delta_plus", energy_delta_plus, recVary=False, type=pycdf.const.CDF_FLOAT) - cdf.new("energy_delta_minus", energy_delta_minus, recVary=False, type=pycdf.const.CDF_FLOAT) + cdf.new( + "energy_delta_plus", + energy_delta_plus, + recVary=False, + type=pycdf.const.CDF_FLOAT, + ) + cdf.new( + "energy_delta_minus", + energy_delta_minus, + recVary=False, + type=pycdf.const.CDF_FLOAT, + ) cdf.new("energy_label", energy_label, recVary=False, type=pycdf.const.CDF_CHAR) for metadata_var_names in ["latitude_label", "longitude_label", "energy_label"]: @@ -107,16 +153,22 @@ def create_l2_map_from_instrument_team(folder: Path, output_dir: Path, pixel_siz for var in cdf: if cdf[var].type() == pycdf.const.CDF_TIME_TT2000.value: - cdf[var].attrs['FILLVAL'] = datetime.fromisoformat("9999-12-31T23:59:59.999999999") + cdf[var].attrs["FILLVAL"] = datetime.fromisoformat( + "9999-12-31T23:59:59.999999999" + ) elif cdf[var].type() == pycdf.const.CDF_INT8.value: - cdf[var].attrs['FILLVAL'] = -9223372036854775808 + cdf[var].attrs["FILLVAL"] = -9223372036854775808 elif cdf[var].type() == pycdf.const.CDF_FLOAT.value: - cdf[var].attrs['FILLVAL'] = -1e31 + cdf[var].attrs["FILLVAL"] = -1e31 return pathname -if __name__ == '__main__': +if __name__ == "__main__": parent = get_test_instrument_team_data_path("hi") for folder in parent.iterdir(): - print(create_l2_map_from_instrument_team(folder, output_dir=get_run_local_data_path("lo"), pixel_size=6)) + print( + create_l2_map_from_instrument_team( + folder, output_dir=get_run_local_data_path("lo"), pixel_size=6 + ) + ) diff --git a/scripts/hi/create_hi_full_spin_deps.py b/scripts/hi/create_hi_full_spin_deps.py index f2a9e3713..b4122cf56 100644 --- a/scripts/hi/create_hi_full_spin_deps.py +++ b/scripts/hi/create_hi_full_spin_deps.py @@ -7,21 +7,30 @@ from spacepy.pycdf import CDF from imap_l3_processing.maps import survival_probability_processing -from imap_l3_processing.maps.hilo_l3_survival_dependencies import HiLoL3SurvivalDependencies +from imap_l3_processing.maps.hilo_l3_survival_dependencies import ( + HiLoL3SurvivalDependencies, +) from imap_l3_processing.maps.map_models import RectangularIntensityDataProduct from imap_l3_processing.models import InputMetadata from imap_l3_processing.utils import save_data, furnish_local_spice -from scripts.hi.create_cdf_from_instrument_team_data import create_l2_map_from_instrument_team +from scripts.hi.create_cdf_from_instrument_team_data import ( + create_l2_map_from_instrument_team, +) from scripts.hi.generate_fake_l1c_cdfs import generate_fake_l1c -from tests.test_helpers import get_test_instrument_team_data_path, get_test_data_path, get_run_local_data_path +from tests.test_helpers import ( + get_test_instrument_team_data_path, + get_test_data_path, + get_run_local_data_path, +) def create_hi_full_spin_deps( - sensor="90", - output_dir: Path = get_run_local_data_path("hi/full_spin_deps"), - start_date: datetime = datetime(year=2025, month=4, day=16, hour=12), - l2_map_dir: Path = None, - glows_dir: Path = None): + sensor="90", + output_dir: Path = get_run_local_data_path("hi/full_spin_deps"), + start_date: datetime = datetime(year=2025, month=4, day=16, hour=12), + l2_map_dir: Path = None, + glows_dir: Path = None, +): if l2_map_dir is None: l2_map_dir = get_test_instrument_team_data_path(f"hi/hi{sensor}-6months") if glows_dir is None: @@ -29,8 +38,12 @@ def create_hi_full_spin_deps( l1c_folder = output_dir / "l1c" l1c_folder.mkdir(parents=True, exist_ok=True) - l1c_files = [Path(s) for s in generate_fake_l1c(start_date, 2, output_dir=l1c_folder)] - original_full_spin_map_path = create_l2_map_from_instrument_team(l2_map_dir, output_dir=output_dir) + l1c_files = [ + Path(s) for s in generate_fake_l1c(start_date, 2, output_dir=l1c_folder) + ] + original_full_spin_map_path = create_l2_map_from_instrument_team( + l2_map_dir, output_dir=output_dir + ) parents = [f.name for f in l1c_files] glows_files = list(glows_dir.iterdir()) @@ -38,56 +51,89 @@ def create_hi_full_spin_deps( original_intensity = cdf["ena_intensity"][...] ramified_map_deps = HiLoL3SurvivalDependencies.from_file_paths( - original_full_spin_map_path, l1c_files, glows_files, - f"h{sensor}-ena-h-sf-nsp-ram-hae-4deg-6mo") + original_full_spin_map_path, + l1c_files, + glows_files, + f"h{sensor}-ena-h-sf-nsp-ram-hae-4deg-6mo", + ) antiramified_map_deps = HiLoL3SurvivalDependencies.from_file_paths( - original_full_spin_map_path, l1c_files, + original_full_spin_map_path, + l1c_files, glows_files, - f"h{sensor}-ena-h-sf-nsp-anti-hae-4deg-6mo") - - input_metadata = InputMetadata("hi", "l3", datetime(year=2025, month=4, day=15), None, "v001", - "h90-ena-h-sf-sp-full-hae-4deg-6mo") - - ram_data = survival_probability_processing.process_survival_probabilities(ramified_map_deps, SpiceFrame.ECLIPJ2000) - ram_exposure_is_zero = (np.isnan(ram_data.intensity_map_data.ena_intensity) - | (ram_data.intensity_map_data.ena_intensity == 0)) + f"h{sensor}-ena-h-sf-nsp-anti-hae-4deg-6mo", + ) + + input_metadata = InputMetadata( + "hi", + "l3", + datetime(year=2025, month=4, day=15), + None, + "v001", + "h90-ena-h-sf-sp-full-hae-4deg-6mo", + ) + + ram_data = survival_probability_processing.process_survival_probabilities( + ramified_map_deps, SpiceFrame.ECLIPJ2000 + ) + ram_exposure_is_zero = np.isnan(ram_data.intensity_map_data.ena_intensity) | ( + ram_data.intensity_map_data.ena_intensity == 0 + ) ram_cdf_path = save_data( RectangularIntensityDataProduct(input_metadata, ram_data), - delete_if_present=True, folder_path=output_dir) - ram_logical_source = f"imap_hi_l2_h{sensor}-ena-h-sf-nsp-ram-hae-4deg-6mo_20250415_v001" + delete_if_present=True, + folder_path=output_dir, + ) + ram_logical_source = ( + f"imap_hi_l2_h{sensor}-ena-h-sf-nsp-ram-hae-4deg-6mo_20250415_v001" + ) with CDF(str(ram_cdf_path), readonly=False) as ram: - ram["exposure_factor"] = np.where(ram_exposure_is_zero, 0, ram_data.intensity_map_data.exposure_factor) - ram["ena_intensity"] = np.where(ram_exposure_is_zero, ram["ena_intensity"].attrs["FILLVAL"], - original_intensity) + ram["exposure_factor"] = np.where( + ram_exposure_is_zero, 0, ram_data.intensity_map_data.exposure_factor + ) + ram["ena_intensity"] = np.where( + ram_exposure_is_zero, + ram["ena_intensity"].attrs["FILLVAL"], + original_intensity, + ) ram.attrs["Parents"] = parents ram.attrs["Logical_source"] = ram_logical_source ram["ena_intensity"].attrs["CATDESC"] = ram_logical_source shutil.move(ram_cdf_path, output_dir / f"{ram_logical_source}.cdf") - antiram_data = survival_probability_processing.process_survival_probabilities(antiramified_map_deps, - SpiceFrame.ECLIPJ2000) - antiram_exposure_is_zero = (np.isnan(antiram_data.intensity_map_data.ena_intensity) - | (antiram_data.intensity_map_data.ena_intensity == 0)) + antiram_data = survival_probability_processing.process_survival_probabilities( + antiramified_map_deps, SpiceFrame.ECLIPJ2000 + ) + antiram_exposure_is_zero = np.isnan( + antiram_data.intensity_map_data.ena_intensity + ) | (antiram_data.intensity_map_data.ena_intensity == 0) antiram_cdf_path = save_data( RectangularIntensityDataProduct(input_metadata, antiram_data), - delete_if_present=True, folder_path=output_dir) - antiram_logical_source = f"imap_hi_l2_h{sensor}-ena-h-sf-nsp-anti-hae-4deg-6mo_20250415_v001" + delete_if_present=True, + folder_path=output_dir, + ) + antiram_logical_source = ( + f"imap_hi_l2_h{sensor}-ena-h-sf-nsp-anti-hae-4deg-6mo_20250415_v001" + ) with CDF(str(antiram_cdf_path), readonly=False) as antiram: - antiram["exposure_factor"] = np.where(antiram_exposure_is_zero, 0, - antiram_data.intensity_map_data.exposure_factor) - antiram["ena_intensity"] = np.where(antiram_exposure_is_zero, antiram["ena_intensity"].attrs["FILLVAL"], - original_intensity) + antiram["exposure_factor"] = np.where( + antiram_exposure_is_zero, 0, antiram_data.intensity_map_data.exposure_factor + ) + antiram["ena_intensity"] = np.where( + antiram_exposure_is_zero, + antiram["ena_intensity"].attrs["FILLVAL"], + original_intensity, + ) antiram.attrs["Parents"] = parents antiram.attrs["Logical_source"] = antiram_logical_source antiram["ena_intensity"].attrs["CATDESC"] = antiram_logical_source shutil.move(antiram_cdf_path, output_dir / f"{antiram_logical_source}.cdf") -if __name__ == '__main__': +if __name__ == "__main__": furnish_local_spice() create_hi_full_spin_deps() diff --git a/scripts/hi/create_l2_from_pointing_sets.py b/scripts/hi/create_l2_from_pointing_sets.py index 8a9f7a6ec..fbeb1f1f7 100644 --- a/scripts/hi/create_l2_from_pointing_sets.py +++ b/scripts/hi/create_l2_from_pointing_sets.py @@ -10,6 +10,7 @@ output_dir = r"C:\Users\Petty\Development\imap_L3_processing\generate_hi_l2" + def create_input_file(sensor: str, start: datetime, end: datetime) -> str: static_files = [ "naif0012.tls", @@ -19,7 +20,7 @@ def create_input_file(sensor: str, start: datetime, end: datetime) -> str: "de440.bsp", f"imap_hi_{sensor}sensor-cal-prod_20240101_v002.csv", f"imap_hi_{sensor}sensor-esa-energies_20240101_v002.csv", - f"imap_hi_{sensor}sensor-esa-eta-fit-factors_20240101_v001.csv" + f"imap_hi_{sensor}sensor-esa-eta-fit-factors_20240101_v001.csv", ] inputs = ProcessingInputCollection() @@ -31,7 +32,11 @@ def create_input_file(sensor: str, start: datetime, end: datetime) -> str: date = start while date < end: repointing = int((date - start_repointings) / timedelta(days=1)) + 1000 - inputs.add(generate_imap_input(f"imap_hi_l1c_{sensor}sensor-pset_{date.strftime('%Y%m%d')}-repoint{repointing:05}_v971.cdf")) + inputs.add( + generate_imap_input( + f"imap_hi_l1c_{sensor}sensor-pset_{date.strftime('%Y%m%d')}-repoint{repointing:05}_v971.cdf" + ) + ) date += timedelta(days=1) dependency_file = f"imap_hi_l2_{sensor}sensor-de-{start.strftime('%Y%m%d')}-{end.strftime('%Y%m%d')}_{start.strftime('%Y%m%d')}_v001.json" dependency_path = generate_imap_file_path(dependency_file).construct_path() @@ -39,8 +44,9 @@ def create_input_file(sensor: str, start: datetime, end: datetime) -> str: dependency_path.write_text(inputs.serialize()) return dependency_file + def generate_map(descriptor: str, start: datetime, end: datetime): - sensor = descriptor.split('-')[0][1:] + sensor = descriptor.split("-")[0][1:] dependency_file = create_input_file(sensor, start, end) env = { @@ -50,7 +56,7 @@ def generate_map(descriptor: str, start: datetime, end: datetime): } version = "v000" - formatted_start_date = start_date.strftime('%Y%m%d') + formatted_start_date = start_date.strftime("%Y%m%d") output_filename = f"imap_hi_l2_{descriptor}_{formatted_start_date}_{version}.cdf" if not generate_imap_file_path(output_filename).construct_path().exists(): command = f"imap_cli --instrument hi --data-level l2 --descriptor {descriptor} --start-date {formatted_start_date} --version {version} --dependency {dependency_file}" @@ -58,25 +64,30 @@ def generate_map(descriptor: str, start: datetime, end: datetime): else: print(f"Skipping generation of {output_filename}, it already exists") + if __name__ == "__main__": imap_data_access.config["DATA_DIR"] = Path(output_dir) - duration = '1yr' + duration = "1yr" start_date = datetime(2025, 4, 15) end_date = datetime(2026, 4, 15) for sensor in [45, 90]: - for side in ['ram', 'anti', 'full']: + for side in ["ram", "anti", "full"]: for pixelation in [4, 6]: - descriptor = f"h{sensor}-ena-h-hf-nsp-{side}-hae-{pixelation}deg-{duration}" + descriptor = ( + f"h{sensor}-ena-h-hf-nsp-{side}-hae-{pixelation}deg-{duration}" + ) generate_map(descriptor, start_date, end_date) - duration = '6mo' + duration = "6mo" for sensor in [45, 90]: - for side in ['ram', 'anti', 'full']: + for side in ["ram", "anti", "full"]: for pixelation in [4, 6]: for start_date, end_date in [ (datetime(2025, 4, 15), datetime(2025, 10, 15)), - (datetime(2025, 10, 15), datetime(2026, 4, 15)) + (datetime(2025, 10, 15), datetime(2026, 4, 15)), ]: - descriptor = f"h{sensor}-ena-h-hf-nsp-{side}-hae-{pixelation}deg-{duration}" + descriptor = ( + f"h{sensor}-ena-h-hf-nsp-{side}-hae-{pixelation}deg-{duration}" + ) generate_map(descriptor, start_date, end_date) diff --git a/scripts/hi/generate_fake_l1c_cdfs.py b/scripts/hi/generate_fake_l1c_cdfs.py index 00f229c73..20db0ef08 100644 --- a/scripts/hi/generate_fake_l1c_cdfs.py +++ b/scripts/hi/generate_fake_l1c_cdfs.py @@ -7,22 +7,33 @@ from tests.test_helpers import get_test_data_path -def generate_fake_l1c(start_date: datetime, num_days: int, output_dir: Path) -> list[str]: +def generate_fake_l1c( + start_date: datetime, num_days: int, output_dir: Path +) -> list[str]: output_files = [] for i in range(num_days): date_to_set = start_date + timedelta(days=i) - logical_source = f"imap_hi_l1c_90sensor-pset_{date_to_set.strftime('%Y%m%d')}_v001" + logical_source = ( + f"imap_hi_l1c_90sensor-pset_{date_to_set.strftime('%Y%m%d')}_v001" + ) output_path = output_dir / f"{logical_source}.cdf" output_path.unlink(missing_ok=True) - with CDF(str(output_path), - masterpath=str(get_test_data_path("hi/imap_hi_l1c_90sensor-pset_20250415_v001.cdf"))) as cdf: - del cdf['epoch'] + with CDF( + str(output_path), + masterpath=str( + get_test_data_path("hi/imap_hi_l1c_90sensor-pset_20250415_v001.cdf") + ), + ) as cdf: + del cdf["epoch"] cdf.new("epoch", [date_to_set], type=pycdf.const.CDF_TIME_TT2000) output_files.append(str(output_path)) return output_files if __name__ == "__main__": - generate_fake_l1c(datetime(month=4, year=2025, day=16, hour=12), num_days=90, - output_dir=get_test_data_path("hi/fake_l1c/90")) + generate_fake_l1c( + datetime(month=4, year=2025, day=16, hour=12), + num_days=90, + output_dir=get_test_data_path("hi/fake_l1c/90"), + ) diff --git a/scripts/hi/stack_arrays_to_match_number_of_energies.py b/scripts/hi/stack_arrays_to_match_number_of_energies.py index 6083aab5c..b6138e0ca 100644 --- a/scripts/hi/stack_arrays_to_match_number_of_energies.py +++ b/scripts/hi/stack_arrays_to_match_number_of_energies.py @@ -3,23 +3,28 @@ from spacepy.pycdf import CDF import numpy as np -filename = f"imap_hi_l3_spectral-fit-index_{date.today().strftime("%Y%m%d")}_" +filename = f"imap_hi_l3_spectral-fit-index_{date.today().strftime('%Y%m%d')}_" temp_cdf = CDF( - rf"C:\Users\Harrison\Development\imap_L3_processing\temp_cdf_data\{filename}.cdf") + rf"C:\Users\Harrison\Development\imap_L3_processing\temp_cdf_data\{filename}.cdf" +) map_cdf = CDF( - rf"C:\Users\Harrison\Development\cava\test\test_data\map_files\map_cdfs\{filename}.cdf") + rf"C:\Users\Harrison\Development\cava\test\test_data\map_files\map_cdfs\{filename}.cdf" +) map_cdf.readonly(False) -new_spectral_index_values = np.repeat(map_cdf['spectral_fit_index'][...][..., np.newaxis], len(map_cdf["bin"][...]), - axis=-1) +new_spectral_index_values = np.repeat( + map_cdf["spectral_fit_index"][...][..., np.newaxis], + len(map_cdf["bin"][...]), + axis=-1, +) # new_spectral_index_values = np.stack([map_cdf['spectral_fit_index'], map_cdf['spectral_fit_index'], # map_cdf['spectral_fit_index'], map_cdf['spectral_fit_index'], # map_cdf['spectral_fit_index']], axis=-1) -temp_cdf_attrs = temp_cdf['spectral_fit_index'].attrs -del map_cdf['spectral_fit_index'] -map_cdf['spectral_fit_index'] = new_spectral_index_values -map_cdf['spectral_fit_index'].attrs = temp_cdf_attrs +temp_cdf_attrs = temp_cdf["spectral_fit_index"].attrs +del map_cdf["spectral_fit_index"] +map_cdf["spectral_fit_index"] = new_spectral_index_values +map_cdf["spectral_fit_index"].attrs = temp_cdf_attrs map_cdf.close() temp_cdf.close() diff --git a/scripts/hit/create_l1a_direct-event_data.py b/scripts/hit/create_l1a_direct-event_data.py index e402e03a4..49e598fe8 100644 --- a/scripts/hit/create_l1a_direct-event_data.py +++ b/scripts/hit/create_l1a_direct-event_data.py @@ -7,7 +7,14 @@ from spacepy.pycdf import CDF path = Path(__file__) -binary_data_path = path.parent.parent.parent / "tests" / "test_data" / "hit" / "pha_events" / "pha_binary" +binary_data_path = ( + path.parent.parent.parent + / "tests" + / "test_data" + / "hit" + / "pha_events" + / "pha_binary" +) start_time = datetime(year=2010, month=1, day=6) time_delta = timedelta(minutes=10) @@ -23,7 +30,9 @@ print(len(binary)) pha_raw.append(binary) -with CDF("tests/test_data/hit/pha_events/fake-menlo-imap_hit_l1a_pulse-height-events_20100106_v004", - masterpath='') as cdf: +with CDF( + "tests/test_data/hit/pha_events/fake-menlo-imap_hit_l1a_pulse-height-events_20100106_v004", + masterpath="", +) as cdf: cdf.new("epoch", np.array(epoch)) cdf.new("pha_raw", pha_raw) diff --git a/scripts/hit/create_l2_hit_sectored_rates_cdf_from_csv.py b/scripts/hit/create_l2_hit_sectored_rates_cdf_from_csv.py index 8233eecf4..31ebf1974 100644 --- a/scripts/hit/create_l2_hit_sectored_rates_cdf_from_csv.py +++ b/scripts/hit/create_l2_hit_sectored_rates_cdf_from_csv.py @@ -9,10 +9,11 @@ def create_l2_hit_sectored_rates_cdf_from_csv(csv_file_path: str, output_filename: str): - with CDF(output_filename, '') as cdf: + with CDF(output_filename, "") as cdf: cdf.col_major(True) - raw_data = np.loadtxt(csv_file_path, delimiter=",", skiprows=1, - usecols=range(1, 121)) + raw_data = np.loadtxt( + csv_file_path, delimiter=",", skiprows=1, usecols=range(1, 121) + ) flux_data = raw_data.reshape((-1, 8, 15)) epoch_count = 1 start_time = datetime(2025, 1, 1, 0, 5) @@ -78,8 +79,19 @@ def create_l2_hit_sectored_rates_cdf_from_csv(csv_file_path: str, output_filenam if __name__ == "__main__": path = Path(__file__) - input_path = path.parent.parent.parent / "instrument_team_data" / "hit" / "hit_l2_sectored_sample1.csv" - cdf_file_path = path.parent.parent.parent / "tests" / "test_data" / "hit" / "imap_hit_l2_macropixel-intensity_20250101_v002.cdf" + input_path = ( + path.parent.parent.parent + / "instrument_team_data" + / "hit" + / "hit_l2_sectored_sample1.csv" + ) + cdf_file_path = ( + path.parent.parent.parent + / "tests" + / "test_data" + / "hit" + / "imap_hit_l2_macropixel-intensity_20250101_v002.cdf" + ) try: create_l2_hit_sectored_rates_cdf_from_csv(str(input_path), str(cdf_file_path)) diff --git a/scripts/hit/create_mag_from_csv.py b/scripts/hit/create_mag_from_csv.py index 51d0e39a8..fdcd60cf0 100644 --- a/scripts/hit/create_mag_from_csv.py +++ b/scripts/hit/create_mag_from_csv.py @@ -11,20 +11,22 @@ def create_mag(csv_path, l2_path, output_path, offset): epoch_len = len(epoch) csv_data = np.loadtxt(csv_path, delimiter=",", skiprows=1) - mag_data = csv_data[offset:offset+epoch_len, -3:] - mag_sclk = csv_data[offset:offset+epoch_len, 6] - mag_epochs = [spiceypy.et2datetime(spiceypy.scs2e(-43, str(sclk))) for sclk in mag_sclk] + mag_data = csv_data[offset : offset + epoch_len, -3:] + mag_sclk = csv_data[offset : offset + epoch_len, 6] + mag_epochs = [ + spiceypy.et2datetime(spiceypy.scs2e(-43, str(sclk))) for sclk in mag_sclk + ] print(mag_sclk[0]) print(mag_epochs[:5]) print(epoch[:5]) with CDF(output_path, create=True) as output_cdf: - output_cdf['epoch'] = epoch - output_cdf['vectors'] = mag_data - output_cdf['vectors'].attrs["FILLVAL"] = -1e31 + output_cdf["epoch"] = epoch + output_cdf["vectors"] = mag_data + output_cdf["vectors"].attrs["FILLVAL"] = -1e31 -if __name__ == '__main__': +if __name__ == "__main__": spiceypy.furnsh("spice_kernels/imap_sclk_0000.tsc") spiceypy.furnsh("spice_kernels/naif0012.tls") # USAGE: python create_mag_from_csv.py @@ -32,4 +34,4 @@ def create_mag(csv_path, l2_path, output_path, offset): create_mag(csv_path, l2_path, output_path, int(offset)) # python scripts/hit/create_mag_from_csv.py instrument_team_data/hit/hit_l1a_sample2_fakeMAG.csv tests/test_data/hit/imap_hit_l2_macropixel-intensity_20100106_v001.cdf tests/test_data/hit/mag_data_for_hit_20100106 900 - # python scripts/hit/create_mag_from_csv.py instrument_team_data/hit/hit_l1a_sample2_fakeMAG.csv tests/test_data/hit/imap_hit_l2_macropixel-intensity_20100107_v001.cdf tests/test_data/hit/mag_data_for_hit_20100107 2340 \ No newline at end of file + # python scripts/hit/create_mag_from_csv.py instrument_team_data/hit/hit_l1a_sample2_fakeMAG.csv tests/test_data/hit/imap_hit_l2_macropixel-intensity_20100107_v001.cdf tests/test_data/hit/mag_data_for_hit_20100107 2340 diff --git a/scripts/hit/hit_plots.py b/scripts/hit/hit_plots.py index 136cc32ca..c372dc665 100644 --- a/scripts/hit/hit_plots.py +++ b/scripts/hit/hit_plots.py @@ -8,11 +8,10 @@ xs = [] ys = [] -for i in range(0, len(de['energy'][...])): - - if de['range'][i] == 3 and de['side'][i] == 1 and de['stim_tag'][i] != 1: - ys.append(de['delta_e'][i]) - xs.append(de['e_prime'][i]) +for i in range(0, len(de["energy"][...])): + if de["range"][i] == 3 and de["side"][i] == 1 and de["stim_tag"][i] != 1: + ys.append(de["delta_e"][i]) + xs.append(de["e_prime"][i]) # # for delta_e, e_prime, d_range in zip(de["delta_e"], de["e_prime"], de["range"], s): @@ -21,6 +20,6 @@ # ys.append(delta_e) plt.scatter(xs, ys) -plt.xscale('log') -plt.yscale('log') +plt.xscale("log") +plt.yscale("log") plt.show() diff --git a/scripts/lo/combine_existing_maps_to_fix_frozen_data.py b/scripts/lo/combine_existing_maps_to_fix_frozen_data.py index eb20d9889..7471194ef 100644 --- a/scripts/lo/combine_existing_maps_to_fix_frozen_data.py +++ b/scripts/lo/combine_existing_maps_to_fix_frozen_data.py @@ -13,9 +13,7 @@ from tests.test_helpers import get_run_local_data_path -def copy_to_output_directory( - release_directory: Path, output_maps: list[Path] -): +def copy_to_output_directory(release_directory: Path, output_maps: list[Path]): release_directory.mkdir(parents=True, exist_ok=True) for generated_path in output_maps: @@ -23,8 +21,11 @@ def copy_to_output_directory( if __name__ == "__main__": - - logging.basicConfig(force=True, level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + logging.basicConfig( + force=True, + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) output_data_path = get_run_local_data_path("lo_txt_pipeline") shutil.rmtree(output_data_path / "imap" / "lo" / "l2", ignore_errors=True) @@ -92,22 +93,21 @@ def copy_to_output_directory( input_files[sfp.descriptor] = file.name output_maps = [] for combined_descriptor, combined_dependencies in maps_needed_for_combination: - [combined_sp_map] = LoProcessor( - dependencies=ProcessingInputCollection(*[ScienceInput(input_files[desc]) for desc in combined_dependencies]), + dependencies=ProcessingInputCollection( + *[ScienceInput(input_files[desc]) for desc in combined_dependencies] + ), input_metadata=InputMetadata( instrument="lo", data_level="l3", - start_date=datetime(2025,11,25), + start_date=datetime(2025, 11, 25), end_date=None, version=VersionMap({}, Version(None, 2)), - descriptor=combined_descriptor - ) + descriptor=combined_descriptor, + ), ).process() - output_maps.extend([ - combined_sp_map - ]) + output_maps.extend([combined_sp_map]) release_directory = get_run_local_data_path("IMAP-Lo July 31st 2026 Maps") copy_to_output_directory(release_directory, output_maps) diff --git a/scripts/lo/create_l2_from_pointing_sets.py b/scripts/lo/create_l2_from_pointing_sets.py index 80d2c8c42..0b633d35d 100644 --- a/scripts/lo/create_l2_from_pointing_sets.py +++ b/scripts/lo/create_l2_from_pointing_sets.py @@ -28,7 +28,7 @@ def create_input_file() -> str: "imap_lo_l1c_pset_20250420-repoint01005_v002.cdf", "imap_lo_l1c_pset_20250424-repoint01009_v002.cdf", "imap_lo_l1c_pset_20250429-repoint01014_v002.cdf", - "imap_lo_l1c_pset_20250504-repoint01019_v002.cdf" + "imap_lo_l1c_pset_20250504-repoint01019_v002.cdf", ] inputs = ProcessingInputCollection() @@ -52,7 +52,7 @@ def generate_map(descriptor: str, start: datetime): } version = "v000" - formatted_start_date = start.strftime('%Y%m%d') + formatted_start_date = start.strftime("%Y%m%d") output_filename = f"imap_lo_l2_{descriptor}_{formatted_start_date}_{version}.cdf" command = f"imap_cli --instrument lo --data-level l2 --descriptor {descriptor} --start-date {formatted_start_date} --version {version} --dependency {dependency_file}" print(command) @@ -78,4 +78,4 @@ def generate_map(descriptor: str, start: datetime): "l090-enanbs-h-sf-nsp-ram-hae-6deg-1yr", ] for descriptor in descriptors: - generate_map(descriptor, start_date) \ No newline at end of file + generate_map(descriptor, start_date) diff --git a/scripts/make_cdf_all_fill.py b/scripts/make_cdf_all_fill.py index c35144514..5f2f8cfdb 100644 --- a/scripts/make_cdf_all_fill.py +++ b/scripts/make_cdf_all_fill.py @@ -13,8 +13,11 @@ shutil.copy(args.template_cdf, output_file_name) with CDF(output_file_name, readonly=False) as cdf: for var in cdf: - if var not in ["Epoch", "epoch"] and cdf[var].attrs.get("VAR_TYPE") != "metadata": + if ( + var not in ["Epoch", "epoch"] + and cdf[var].attrs.get("VAR_TYPE") != "metadata" + ): if "FILLVAL" in cdf[var].attrs: cdf[var] = np.full_like(cdf[var], cdf[var].attrs["FILLVAL"]) else: - print(f'The var {var} has no FILLVAL attribute.') + print(f"The var {var} has no FILLVAL attribute.") diff --git a/scripts/poll_batch_jobs_status.py b/scripts/poll_batch_jobs_status.py index 9162c8d45..24b3cbee1 100644 --- a/scripts/poll_batch_jobs_status.py +++ b/scripts/poll_batch_jobs_status.py @@ -7,17 +7,24 @@ def format_job(job) -> str: - dependencies_passed = job["container_command"].split("--dependency ")[1].split(" --upload-to-sdc")[0] + dependencies_passed = ( + job["container_command"].split("--dependency ")[1].split(" --upload-to-sdc")[0] + ) input_collection = ProcessingInputCollection() input_collection.deserialize(dependencies_passed) - processing_input = "\n".join([f"\t\t{type(pi)}: {pi.filename_list}" for pi in input_collection.processing_input]) + processing_input = "\n".join( + [ + f"\t\t{type(pi)}: {pi.filename_list}" + for pi in input_collection.processing_input + ] + ) return f""" descriptor: {job["descriptor"]} start_date: {job["start_date"]} - version: {job['version']} + version: {job["version"]} status: {job["status"]} processing_input: {processing_input} @@ -29,15 +36,20 @@ def format_job(job) -> str: AFTER_DATE = datetime.now(tz=timezone.utc) -url = 'https://api.dev.imap-mission.com/batch-job' +url = "https://api.dev.imap-mission.com/batch-job" while True: - response = requests.get(url=url, params={'instrument': INSTRUMENT, 'data_level': DATA_LEVEL}) + response = requests.get( + url=url, params={"instrument": INSTRUMENT, "data_level": DATA_LEVEL} + ) logs = json.loads(json.dumps(response.json())) - relevant_logs = [log for log in logs if - log['started_at'] is not None and - datetime.fromisoformat(log['started_at']).replace(tzinfo=timezone.utc) > AFTER_DATE - ] + relevant_logs = [ + log + for log in logs + if log["started_at"] is not None + and datetime.fromisoformat(log["started_at"]).replace(tzinfo=timezone.utc) + > AFTER_DATE + ] print("\n".join([format_job(job) for job in relevant_logs])) diff --git a/scripts/reprocess_all_in_situ.py b/scripts/reprocess_all_in_situ.py index 77a55294a..e0dbf33ca 100644 --- a/scripts/reprocess_all_in_situ.py +++ b/scripts/reprocess_all_in_situ.py @@ -4,38 +4,33 @@ LAUNCH_DATE = "20250924" -GLOWS_PRODUCTS= "glows", { - "l3a": [ - "hist" - ] -} -SWAPI_PRODUCTS= "swapi",{ - "l3a": [ - "proton-sw", - "alpha-sw", - "pui-he", - ], - "l3b":[ - "combined" - ] -} -HIT_PRODUCTS = "hit", { - "l3": [ - "direct-events", - "macropixel" - ] -} +GLOWS_PRODUCTS = "glows", {"l3a": ["hist"]} +SWAPI_PRODUCTS = ( + "swapi", + { + "l3a": [ + "proton-sw", + "alpha-sw", + "pui-he", + ], + "l3b": ["combined"], + }, +) +HIT_PRODUCTS = "hit", {"l3": ["direct-events", "macropixel"]} -CODICE_PRODUCTS= "codice",{ - "l3a": [ - # "lo-partial-densities", - # "lo-direct-events", - "hi-direct-events", - ], - # "l3b": [ - # "hi-pitch-angle" - # ] -} +CODICE_PRODUCTS = ( + "codice", + { + "l3a": [ + # "lo-partial-densities", + # "lo-direct-events", + "hi-direct-events", + ], + # "l3b": [ + # "hi-pitch-angle" + # ] + }, +) ALL_PRODUCTS = [GLOWS_PRODUCTS, SWAPI_PRODUCTS, HIT_PRODUCTS, CODICE_PRODUCTS] @@ -46,9 +41,9 @@ for data_level, descriptors in descriptors_by_level.items(): for descriptor in descriptors: imap_data_access.reprocess( - instrument = instrument, - data_level = data_level, - descriptor = descriptor, - start_date = LAUNCH_DATE, - end_date = TODAY - ) \ No newline at end of file + instrument=instrument, + data_level=data_level, + descriptor=descriptor, + start_date=LAUNCH_DATE, + end_date=TODAY, + ) diff --git a/scripts/run_spectral_index.py b/scripts/run_spectral_index.py index 02fb0128b..f90659f15 100644 --- a/scripts/run_spectral_index.py +++ b/scripts/run_spectral_index.py @@ -29,12 +29,16 @@ def run_spectral_index(input: Path): parsed_descriptor = parse_map_descriptor(parsed.descriptor) output_descriptor = map_descriptor_parts_to_string( - dataclasses.replace(parsed_descriptor, quantity=MapQuantity.SpectralIndex, quantity_suffix="") + dataclasses.replace( + parsed_descriptor, quantity=MapQuantity.SpectralIndex, quantity_suffix="" + ) ) dependencies = ProcessingInputCollection(ScienceInput(input.name)) - if parsed.instrument == 'ultra': - dependencies.add(AncillaryInput("imap_ultra_ulc-spx-energy-ranges_20250407_v000.dat")) + if parsed.instrument == "ultra": + dependencies.add( + AncillaryInput("imap_ultra_ulc-spx-energy-ranges_20250407_v000.dat") + ) data_path = parsed.construct_path() data_path.parent.mkdir(parents=True, exist_ok=True) @@ -56,6 +60,7 @@ def run_spectral_index(input: Path): if __name__ == "__main__": outputs = [] for file in Path( - '/Users/harrison/Development/imap_L3_processing/run_local_input_data/lo/integration_data/imap/lo/l3/2025/04').iterdir(): + "/Users/harrison/Development/imap_L3_processing/run_local_input_data/lo/integration_data/imap/lo/l3/2025/04" + ).iterdir(): outputs.extend(run_spectral_index(file)) print(outputs) diff --git a/scripts/swapi/determine_swapi_files_with_outdated_spice.py b/scripts/swapi/determine_swapi_files_with_outdated_spice.py index da8537ee0..b890df0ba 100644 --- a/scripts/swapi/determine_swapi_files_with_outdated_spice.py +++ b/scripts/swapi/determine_swapi_files_with_outdated_spice.py @@ -5,7 +5,12 @@ descriptors = ["pui-he"] for descriptor in descriptors: - query_results = imap_data_access.query(instrument="swapi", data_level=data_level, descriptor=descriptor, version="latest") + query_results = imap_data_access.query( + instrument="swapi", + data_level=data_level, + descriptor=descriptor, + version="latest", + ) dates_that_used_predict = [] for result in sorted(query_results, key=lambda r: r["start_date"]): diff --git a/scripts/swapi/fake_alpha_sw_temperature_density_lut_generator.py b/scripts/swapi/fake_alpha_sw_temperature_density_lut_generator.py index 71a6c5f2e..89a7eb3a3 100644 --- a/scripts/swapi/fake_alpha_sw_temperature_density_lut_generator.py +++ b/scripts/swapi/fake_alpha_sw_temperature_density_lut_generator.py @@ -9,7 +9,9 @@ densities = np.arange(1, 6.5, 0.5) temps = np.arange(1e4, 1.6e5, 1e4) -with open("imap_swapi_l2_alpha-density-temperature-lut-text-not-cdf_20240905_v002.cdf", 'w') as f: +with open( + "imap_swapi_l2_alpha-density-temperature-lut-text-not-cdf_20240905_v002.cdf", "w" +) as f: f.write("#\n") f.write(f"# Created on: {today()}\n") f.write("#\n") @@ -19,4 +21,5 @@ for u_sw, density, temp in itertools.product(u_sw_values, densities, temps): f.write( - f"{u_sw} {density} {density * 1.021:.4f} {temp:.4e} {temp / 1.025:0.4e}\n") + f"{u_sw} {density} {density * 1.021:.4f} {temp:.4e} {temp / 1.025:0.4e}\n" + ) diff --git a/scripts/swapi/fake_data_generator.py b/scripts/swapi/fake_data_generator.py index 5af6de3e0..3aafb9655 100644 --- a/scripts/swapi/fake_data_generator.py +++ b/scripts/swapi/fake_data_generator.py @@ -20,17 +20,19 @@ def generate_sweep_data(center): def generate_sweeps(n=5): time_base = 315576112184000000 # Jan 1, 2010 00:00:46 - start_times_in_seconds = time_base + np.arange(n)*12 + start_times_in_seconds = time_base + np.arange(n) * 12 seed = randint(0, 359) - angles = [seed - 72*i for i in range(5)] - center_points = 1050 + 20*np.sin(np.deg2rad(angles)) + angles = [seed - 72 * i for i in range(5)] + center_points = 1050 + 20 * np.sin(np.deg2rad(angles)) sweeps = [generate_sweep_data(c) for c in center_points] return list(zip(start_times_in_seconds, sweeps)) def generate_sweep_energies(): coarse_sweep_energies = np.geomspace(100, 19000, 62) - return np.concatenate((np.array([np.nan]), coarse_sweep_energies, np.array([np.nan] * 9))) + return np.concatenate( + (np.array([np.nan]), coarse_sweep_energies, np.array([np.nan] * 9)) + ) # K Factor may come from a lookup table in the future @@ -42,11 +44,12 @@ def get_k_factor(): def get_spin_phase_using_spice(time): arbitrary_offset = 0.4 rotation_time = 15 - rotations = arbitrary_offset + time/rotation_time + rotations = arbitrary_offset + time / rotation_time fractional, integral = np.modf(rotations) - return fractional*360 + return fractional * 360 def generate_peak(energies, height, center, narrowness): - return np.exp(np.log(height) - narrowness * np.square(np.log(energies) - np.log(center))) - + return np.exp( + np.log(height) - narrowness * np.square(np.log(energies) - np.log(center)) + ) diff --git a/scripts/swapi/fake_uncertainties_for_cdf.py b/scripts/swapi/fake_uncertainties_for_cdf.py index 999bfb877..98959d057 100644 --- a/scripts/swapi/fake_uncertainties_for_cdf.py +++ b/scripts/swapi/fake_uncertainties_for_cdf.py @@ -1,6 +1,7 @@ import numpy as np from spacepy import pycdf + def fake_uncertainties(cdf_path): original_cdf = pycdf.CDF(cdf_path, readonly=False) @@ -8,4 +9,4 @@ def fake_uncertainties(cdf_path): uncertainties = np.sqrt(6 * coincidence_count_rates) original_cdf["swp_coin_unc"] = uncertainties - original_cdf.save() \ No newline at end of file + original_cdf.save() diff --git a/scripts/swapi/fit_and_plot_alpha.py b/scripts/swapi/fit_and_plot_alpha.py index 43ab988fd..43cff380c 100644 --- a/scripts/swapi/fit_and_plot_alpha.py +++ b/scripts/swapi/fit_and_plot_alpha.py @@ -27,7 +27,11 @@ import imap_data_access import matplotlib.pyplot as plt import numpy as np -from imap_data_access import DependencyFilePath, ProcessingInputCollection, ScienceFilePath +from imap_data_access import ( + DependencyFilePath, + ProcessingInputCollection, + ScienceFilePath, +) from imap_data_access.processing_input import generate_imap_input from spacepy.pycdf import CDF @@ -65,9 +69,18 @@ def fetch_wind_swe_2min_alphas(compact_date_string, cache_directory): print(f"Downloading WIND SWE 2-min {year_month_string}...", end=" ", flush=True) try: subprocess.run( - ["curl", "--ssl-reqd", "-fsS", - "--connect-timeout", "15", "--max-time", "300", - "-o", str(cache_file_path), url], + [ + "curl", + "--ssl-reqd", + "-fsS", + "--connect-timeout", + "15", + "--max-time", + "300", + "-o", + str(cache_file_path), + url, + ], check=True, ) print("done") @@ -93,13 +106,17 @@ def fetch_wind_swe_2min_alphas(compact_date_string, cache_directory): for y in np.unique(year_column) } year_start_nanoseconds_column = np.array( - [year_starts_nanoseconds[int(y)] for y in year_column], dtype="int64") + [year_starts_nanoseconds[int(y)] for y in year_column], dtype="int64" + ) epoch_nanoseconds = year_start_nanoseconds_column + ( - (fractional_day_of_year_column - 1.0) * 86400.0 * 1e9).astype("int64") + (fractional_day_of_year_column - 1.0) * 86400.0 * 1e9 + ).astype("int64") epoch = epoch_nanoseconds.view("datetime64[ns]") day_start = np.datetime64( - f"{compact_date_string[:4]}-{compact_date_string[4:6]}-{compact_date_string[6:8]}", "ns") + f"{compact_date_string[:4]}-{compact_date_string[4:6]}-{compact_date_string[6:8]}", + "ns", + ) day_end = day_start + np.timedelta64(1, "D") day_mask = (epoch >= day_start) & (epoch < day_end) if not np.any(day_mask): @@ -119,10 +136,15 @@ def fetch_wind_swe_2min_alphas(compact_date_string, cache_directory): alpha_density_sigma = row[:, 38] # T = c * W^2 -> sigma_T = 2 * c * W * sigma_W (linear propagation, W >= 0). - alpha_temperature = WIND_ALPHA_TEMPERATURE_FACTOR_K_PER_KM2_PER_S2 * alpha_thermal_speed**2 + alpha_temperature = ( + WIND_ALPHA_TEMPERATURE_FACTOR_K_PER_KM2_PER_S2 * alpha_thermal_speed**2 + ) alpha_temperature_sigma = ( - 2.0 * WIND_ALPHA_TEMPERATURE_FACTOR_K_PER_KM2_PER_S2 - * np.abs(alpha_thermal_speed) * alpha_thermal_speed_sigma) + 2.0 + * WIND_ALPHA_TEMPERATURE_FACTOR_K_PER_KM2_PER_S2 + * np.abs(alpha_thermal_speed) + * alpha_thermal_speed_sigma + ) # GSE -> RTN at L1, Sun-Earth-line approximation. return { @@ -147,8 +169,13 @@ def build_dependency_collection(target_date, compact_date): them with the static SWAPI ancillary template into a ProcessingInputCollection. """ science_query_results = imap_data_access.query( - instrument="swapi", data_level="l2", descriptor="sci", - start_date=compact_date, end_date=compact_date, version="latest") + instrument="swapi", + data_level="l2", + descriptor="sci", + start_date=compact_date, + end_date=compact_date, + version="latest", + ) if not science_query_results: sys.exit(f"No SWAPI L2 sci file at the SDC for {compact_date}") @@ -158,8 +185,13 @@ def build_dependency_collection(target_date, compact_date): os.path.basename(row["file_path"]) for level in ("l2", "l1d") for row in imap_data_access.query( - instrument="mag", data_level=level, descriptor="norm-rtn", - start_date=compact_date, end_date=compact_date, version="latest") + instrument="mag", + data_level=level, + descriptor="norm-rtn", + start_date=compact_date, + end_date=compact_date, + version="latest", + ) ] if not magnetic_field_filenames: sys.exit(f"No MAG norm-rtn file (L2 or L1D) at the SDC for {compact_date}") @@ -168,22 +200,30 @@ def build_dependency_collection(target_date, compact_date): # window around the target date. Spacecraft ephemeris kernels need a wide # lookback because the reconstructed kernel ends weeks before the target # date and the predicted kernel covers the gap onward. - ephemeris_kernel_types = [SpiceKernelTypes.EphemerisReconstructed, - SpiceKernelTypes.EphemerisPredicted] - non_ephemeris_kernel_types = [t for t in SpiceKernelTypes - if t not in ephemeris_kernel_types] - spice_kernel_filenames = list(dict.fromkeys( - os.path.basename(path) for path in ( - get_spice_kernels_file_names( - target_date - timedelta(days=1), - target_date + timedelta(days=2), - non_ephemeris_kernel_types) - + get_spice_kernels_file_names( - target_date - timedelta(days=90), - target_date + timedelta(days=2), - ephemeris_kernel_types) + ephemeris_kernel_types = [ + SpiceKernelTypes.EphemerisReconstructed, + SpiceKernelTypes.EphemerisPredicted, + ] + non_ephemeris_kernel_types = [ + t for t in SpiceKernelTypes if t not in ephemeris_kernel_types + ] + spice_kernel_filenames = list( + dict.fromkeys( + os.path.basename(path) + for path in ( + get_spice_kernels_file_names( + target_date - timedelta(days=1), + target_date + timedelta(days=2), + non_ephemeris_kernel_types, + ) + + get_spice_kernels_file_names( + target_date - timedelta(days=90), + target_date + timedelta(days=2), + ephemeris_kernel_types, + ) + ) ) - )) + ) dynamic_filenames = [ os.path.basename(science_query_results[0]["file_path"]), @@ -191,18 +231,24 @@ def build_dependency_collection(target_date, compact_date): *spice_kernel_filenames, ] ancillary_template_path = ( - Path(__file__).parent / "imap_swapi_l3a_proton-sw_dependency_template.json") + Path(__file__).parent / "imap_swapi_l3a_proton-sw_dependency_template.json" + ) processing_input_collection = ProcessingInputCollection( - *[generate_imap_input(filename) for filename in dynamic_filenames]) + *[generate_imap_input(filename) for filename in dynamic_filenames] + ) processing_input_collection.deserialize(ancillary_template_path.read_text()) return processing_input_collection argument_parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter +) argument_parser.add_argument("date", help="UTC date in YYYY-MM-DD format") -argument_parser.add_argument("--use-cache", action="store_true", - help="Skip download/processing if the alpha-sw CDF already exists in IMAP_DATA_DIR.") +argument_parser.add_argument( + "--use-cache", + action="store_true", + help="Skip download/processing if the alpha-sw CDF already exists in IMAP_DATA_DIR.", +) arguments = argument_parser.parse_args() if "IMAP_API_KEY" not in os.environ: @@ -218,27 +264,38 @@ def build_dependency_collection(target_date, compact_date): imap_data_access.config["DATA_DIR"] = DATA_DIR output_cdf_path = ScienceFilePath( - f"imap_swapi_l3a_alpha-sw_{compact_date}_{PROCESSOR_OUTPUT_VERSION}.cdf").construct_path() + f"imap_swapi_l3a_alpha-sw_{compact_date}_{PROCESSOR_OUTPUT_VERSION}.cdf" +).construct_path() if not arguments.use_cache or not output_cdf_path.exists(): processing_input_collection = build_dependency_collection(target_date, compact_date) processing_input_collection.download_all_files() dependency_filename = ( - f"imap_swapi_l3a_alpha-sw_{compact_date}_{PROCESSOR_OUTPUT_VERSION}.json") + f"imap_swapi_l3a_alpha-sw_{compact_date}_{PROCESSOR_OUTPUT_VERSION}.json" + ) dependency_path = DependencyFilePath(dependency_filename).construct_path() dependency_path.parent.mkdir(parents=True, exist_ok=True) dependency_path.write_text(processing_input_collection.serialize()) output_cdf_path.unlink(missing_ok=True) result = subprocess.run( - [sys.executable, "imap_l3_data_processor.py", - "--instrument", "swapi", - "--data-level", "l3a", - "--descriptor", "alpha-sw", - "--start-date", compact_date, - "--version", PROCESSOR_OUTPUT_VERSION, - "--dependency", dependency_filename], + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "swapi", + "--data-level", + "l3a", + "--descriptor", + "alpha-sw", + "--start-date", + compact_date, + "--version", + PROCESSOR_OUTPUT_VERSION, + "--dependency", + dependency_filename, + ], cwd=L3_PROCESSING_ROOT, ) if result.returncode != 0: @@ -250,14 +307,19 @@ def build_dependency_collection(target_date, compact_date): timestamps = alpha_cdf["epoch"][...] quality_flags = alpha_cdf["swp_flags"][...] swapi_density = read_numeric_variable(alpha_cdf["alpha_sw_density"]) - swapi_density_uncertainty = read_numeric_variable(alpha_cdf["alpha_sw_density_uncert"]) + swapi_density_uncertainty = read_numeric_variable( + alpha_cdf["alpha_sw_density_uncert"] + ) swapi_temperature = read_numeric_variable(alpha_cdf["alpha_sw_temperature"]) - swapi_temperature_uncertainty = read_numeric_variable(alpha_cdf["alpha_sw_temperature_uncert"]) + swapi_temperature_uncertainty = read_numeric_variable( + alpha_cdf["alpha_sw_temperature_uncert"] + ) swapi_speed = read_numeric_variable(alpha_cdf["alpha_sw_speed"]) swapi_speed_uncertainty = read_numeric_variable(alpha_cdf["alpha_sw_speed_uncert"]) swapi_velocity_rtn = read_numeric_variable(alpha_cdf["alpha_sw_velocity_rtn_sun"]) swapi_velocity_rtn_covariance = read_numeric_variable( - alpha_cdf["alpha_sw_velocity_rtn_covariance"]) + alpha_cdf["alpha_sw_velocity_rtn_covariance"] + ) # Drop flagged records so fill values don't drag the y-axes. good_record_mask = quality_flags == 0 @@ -272,42 +334,93 @@ def build_dependency_collection(target_date, compact_date): swapi_velocity_rtn_covariance = swapi_velocity_rtn_covariance[good_record_mask] # Diagonal of the RTN velocity covariance gives variances on (vR, vT, vN). -swapi_velocity_rtn_sigma = np.sqrt(np.diagonal( - swapi_velocity_rtn_covariance, axis1=1, axis2=2)) +swapi_velocity_rtn_sigma = np.sqrt( + np.diagonal(swapi_velocity_rtn_covariance, axis1=1, axis2=2) +) wind_alphas = fetch_wind_swe_2min_alphas(compact_date, DATA_DIR / "wind") or {} panels = [ - ("Density [cm$^{-3}$]", "log", - swapi_density, swapi_density_uncertainty, - wind_alphas.get("density"), wind_alphas.get("density_sigma")), - ("Temperature [K]", "log", - swapi_temperature, swapi_temperature_uncertainty, - wind_alphas.get("temperature"), wind_alphas.get("temperature_sigma")), - ("Bulk speed [km/s]", "linear", - swapi_speed, swapi_speed_uncertainty, - wind_alphas.get("speed"), wind_alphas.get("speed_sigma")), - ("$v_R$ [km/s]", "linear", - swapi_velocity_rtn[:, 0], swapi_velocity_rtn_sigma[:, 0], - wind_alphas.get("velocity_r"), wind_alphas.get("velocity_r_sigma")), - ("$v_T$ [km/s]", "linear", - swapi_velocity_rtn[:, 1], swapi_velocity_rtn_sigma[:, 1], - wind_alphas.get("velocity_t"), wind_alphas.get("velocity_t_sigma")), - ("$v_N$ [km/s]", "linear", - swapi_velocity_rtn[:, 2], swapi_velocity_rtn_sigma[:, 2], - wind_alphas.get("velocity_n"), wind_alphas.get("velocity_n_sigma")), + ( + "Density [cm$^{-3}$]", + "log", + swapi_density, + swapi_density_uncertainty, + wind_alphas.get("density"), + wind_alphas.get("density_sigma"), + ), + ( + "Temperature [K]", + "log", + swapi_temperature, + swapi_temperature_uncertainty, + wind_alphas.get("temperature"), + wind_alphas.get("temperature_sigma"), + ), + ( + "Bulk speed [km/s]", + "linear", + swapi_speed, + swapi_speed_uncertainty, + wind_alphas.get("speed"), + wind_alphas.get("speed_sigma"), + ), + ( + "$v_R$ [km/s]", + "linear", + swapi_velocity_rtn[:, 0], + swapi_velocity_rtn_sigma[:, 0], + wind_alphas.get("velocity_r"), + wind_alphas.get("velocity_r_sigma"), + ), + ( + "$v_T$ [km/s]", + "linear", + swapi_velocity_rtn[:, 1], + swapi_velocity_rtn_sigma[:, 1], + wind_alphas.get("velocity_t"), + wind_alphas.get("velocity_t_sigma"), + ), + ( + "$v_N$ [km/s]", + "linear", + swapi_velocity_rtn[:, 2], + swapi_velocity_rtn_sigma[:, 2], + wind_alphas.get("velocity_n"), + wind_alphas.get("velocity_n_sigma"), + ), ] wind_epoch = wind_alphas.get("epoch") figure, axes = plt.subplots(len(panels), 1, sharex=True, figsize=(10, 14)) -for axis, (ylabel, yscale, swapi_values, swapi_uncertainties, - wind_values, wind_uncertainties) in zip(axes, panels): - axis.errorbar(timestamps, swapi_values, yerr=swapi_uncertainties, - fmt=".", capsize=2, color="tab:blue", label="SWAPI L3a") +for axis, ( + ylabel, + yscale, + swapi_values, + swapi_uncertainties, + wind_values, + wind_uncertainties, +) in zip(axes, panels): + axis.errorbar( + timestamps, + swapi_values, + yerr=swapi_uncertainties, + fmt=".", + capsize=2, + color="tab:blue", + label="SWAPI L3a", + ) if wind_values is not None and wind_epoch is not None: - axis.errorbar(wind_epoch, wind_values, yerr=wind_uncertainties, - fmt=".", capsize=2, color="tab:orange", alpha=0.6, - label="WIND SWE 2-min") + axis.errorbar( + wind_epoch, + wind_values, + yerr=wind_uncertainties, + fmt=".", + capsize=2, + color="tab:orange", + alpha=0.6, + label="WIND SWE 2-min", + ) axis.set_ylabel(ylabel) axis.set_yscale(yscale) axes[0].legend(loc="best") diff --git a/scripts/swapi/fit_and_plot_proton.py b/scripts/swapi/fit_and_plot_proton.py index 7a6602c59..543b00260 100644 --- a/scripts/swapi/fit_and_plot_proton.py +++ b/scripts/swapi/fit_and_plot_proton.py @@ -27,7 +27,11 @@ import imap_data_access import matplotlib.pyplot as plt import numpy as np -from imap_data_access import DependencyFilePath, ProcessingInputCollection, ScienceFilePath +from imap_data_access import ( + DependencyFilePath, + ProcessingInputCollection, + ScienceFilePath, +) from imap_data_access.processing_input import generate_imap_input from spacepy.pycdf import CDF @@ -65,9 +69,18 @@ def fetch_wind_swe_2min_protons(compact_date_string, cache_directory): print(f"Downloading WIND SWE 2-min {year_month_string}...", end=" ", flush=True) try: subprocess.run( - ["curl", "--ssl-reqd", "-fsS", - "--connect-timeout", "15", "--max-time", "300", - "-o", str(cache_file_path), url], + [ + "curl", + "--ssl-reqd", + "-fsS", + "--connect-timeout", + "15", + "--max-time", + "300", + "-o", + str(cache_file_path), + url, + ], check=True, ) print("done") @@ -92,13 +105,17 @@ def fetch_wind_swe_2min_protons(compact_date_string, cache_directory): for y in np.unique(year_column) } year_start_nanoseconds_column = np.array( - [year_starts_nanoseconds[int(y)] for y in year_column], dtype="int64") + [year_starts_nanoseconds[int(y)] for y in year_column], dtype="int64" + ) epoch_nanoseconds = year_start_nanoseconds_column + ( - (fractional_day_of_year_column - 1.0) * 86400.0 * 1e9).astype("int64") + (fractional_day_of_year_column - 1.0) * 86400.0 * 1e9 + ).astype("int64") epoch = epoch_nanoseconds.view("datetime64[ns]") day_start = np.datetime64( - f"{compact_date_string[:4]}-{compact_date_string[4:6]}-{compact_date_string[6:8]}", "ns") + f"{compact_date_string[:4]}-{compact_date_string[4:6]}-{compact_date_string[6:8]}", + "ns", + ) day_end = day_start + np.timedelta64(1, "D") day_mask = (epoch >= day_start) & (epoch < day_end) if not np.any(day_mask): @@ -118,10 +135,15 @@ def fetch_wind_swe_2min_protons(compact_date_string, cache_directory): proton_thermal_speed, proton_thermal_speed_sigma = row[:, 11], row[:, 12] proton_density, proton_density_sigma = row[:, 21], row[:, 22] - proton_temperature = WIND_PROTON_TEMPERATURE_FACTOR_K_PER_KM2_PER_S2 * proton_thermal_speed**2 + proton_temperature = ( + WIND_PROTON_TEMPERATURE_FACTOR_K_PER_KM2_PER_S2 * proton_thermal_speed**2 + ) proton_temperature_sigma = ( - 2.0 * WIND_PROTON_TEMPERATURE_FACTOR_K_PER_KM2_PER_S2 - * np.abs(proton_thermal_speed) * proton_thermal_speed_sigma) + 2.0 + * WIND_PROTON_TEMPERATURE_FACTOR_K_PER_KM2_PER_S2 + * np.abs(proton_thermal_speed) + * proton_thermal_speed_sigma + ) return { "epoch": epoch_day, @@ -145,8 +167,13 @@ def build_dependency_collection(target_date, compact_date): them with the static SWAPI ancillary template into a ProcessingInputCollection. """ science_query_results = imap_data_access.query( - instrument="swapi", data_level="l2", descriptor="sci", - start_date=compact_date, end_date=compact_date, version="latest") + instrument="swapi", + data_level="l2", + descriptor="sci", + start_date=compact_date, + end_date=compact_date, + version="latest", + ) if not science_query_results: sys.exit(f"No SWAPI L2 sci file at the SDC for {compact_date}") @@ -154,28 +181,41 @@ def build_dependency_collection(target_date, compact_date): os.path.basename(row["file_path"]) for level in ("l2", "l1d") for row in imap_data_access.query( - instrument="mag", data_level=level, descriptor="norm-rtn", - start_date=compact_date, end_date=compact_date, version="latest") + instrument="mag", + data_level=level, + descriptor="norm-rtn", + start_date=compact_date, + end_date=compact_date, + version="latest", + ) ] if not magnetic_field_filenames: sys.exit(f"No MAG norm-rtn file (L2 or L1D) at the SDC for {compact_date}") - ephemeris_kernel_types = [SpiceKernelTypes.EphemerisReconstructed, - SpiceKernelTypes.EphemerisPredicted] - non_ephemeris_kernel_types = [t for t in SpiceKernelTypes - if t not in ephemeris_kernel_types] - spice_kernel_filenames = list(dict.fromkeys( - os.path.basename(path) for path in ( - get_spice_kernels_file_names( - target_date - timedelta(days=1), - target_date + timedelta(days=2), - non_ephemeris_kernel_types) - + get_spice_kernels_file_names( - target_date - timedelta(days=90), - target_date + timedelta(days=2), - ephemeris_kernel_types) + ephemeris_kernel_types = [ + SpiceKernelTypes.EphemerisReconstructed, + SpiceKernelTypes.EphemerisPredicted, + ] + non_ephemeris_kernel_types = [ + t for t in SpiceKernelTypes if t not in ephemeris_kernel_types + ] + spice_kernel_filenames = list( + dict.fromkeys( + os.path.basename(path) + for path in ( + get_spice_kernels_file_names( + target_date - timedelta(days=1), + target_date + timedelta(days=2), + non_ephemeris_kernel_types, + ) + + get_spice_kernels_file_names( + target_date - timedelta(days=90), + target_date + timedelta(days=2), + ephemeris_kernel_types, + ) + ) ) - )) + ) dynamic_filenames = [ os.path.basename(science_query_results[0]["file_path"]), @@ -183,18 +223,24 @@ def build_dependency_collection(target_date, compact_date): *spice_kernel_filenames, ] ancillary_template_path = ( - Path(__file__).parent / "imap_swapi_l3a_proton-sw_dependency_template.json") + Path(__file__).parent / "imap_swapi_l3a_proton-sw_dependency_template.json" + ) processing_input_collection = ProcessingInputCollection( - *[generate_imap_input(filename) for filename in dynamic_filenames]) + *[generate_imap_input(filename) for filename in dynamic_filenames] + ) processing_input_collection.deserialize(ancillary_template_path.read_text()) return processing_input_collection argument_parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter +) argument_parser.add_argument("date", help="UTC date in YYYY-MM-DD format") -argument_parser.add_argument("--use-cache", action="store_true", - help="Skip download/processing if the proton-sw CDF already exists in IMAP_DATA_DIR.") +argument_parser.add_argument( + "--use-cache", + action="store_true", + help="Skip download/processing if the proton-sw CDF already exists in IMAP_DATA_DIR.", +) arguments = argument_parser.parse_args() if "IMAP_API_KEY" not in os.environ: @@ -208,27 +254,38 @@ def build_dependency_collection(target_date, compact_date): imap_data_access.config["DATA_DIR"] = DATA_DIR output_cdf_path = ScienceFilePath( - f"imap_swapi_l3a_proton-sw_{compact_date}_{PROCESSOR_OUTPUT_VERSION}.cdf").construct_path() + f"imap_swapi_l3a_proton-sw_{compact_date}_{PROCESSOR_OUTPUT_VERSION}.cdf" +).construct_path() if not arguments.use_cache or not output_cdf_path.exists(): processing_input_collection = build_dependency_collection(target_date, compact_date) processing_input_collection.download_all_files() dependency_filename = ( - f"imap_swapi_l3a_proton-sw_{compact_date}_{PROCESSOR_OUTPUT_VERSION}.json") + f"imap_swapi_l3a_proton-sw_{compact_date}_{PROCESSOR_OUTPUT_VERSION}.json" + ) dependency_path = DependencyFilePath(dependency_filename).construct_path() dependency_path.parent.mkdir(parents=True, exist_ok=True) dependency_path.write_text(processing_input_collection.serialize()) output_cdf_path.unlink(missing_ok=True) result = subprocess.run( - [sys.executable, "imap_l3_data_processor.py", - "--instrument", "swapi", - "--data-level", "l3a", - "--descriptor", "proton-sw", - "--start-date", compact_date, - "--version", PROCESSOR_OUTPUT_VERSION, - "--dependency", dependency_filename], + [ + sys.executable, + "imap_l3_data_processor.py", + "--instrument", + "swapi", + "--data-level", + "l3a", + "--descriptor", + "proton-sw", + "--start-date", + compact_date, + "--version", + PROCESSOR_OUTPUT_VERSION, + "--dependency", + dependency_filename, + ], cwd=L3_PROCESSING_ROOT, ) if result.returncode != 0: @@ -240,14 +297,21 @@ def build_dependency_collection(target_date, compact_date): timestamps = proton_cdf["epoch"][...] quality_flags = proton_cdf["swp_flags"][...] swapi_density = read_numeric_variable(proton_cdf["proton_sw_density"]) - swapi_density_uncertainty = read_numeric_variable(proton_cdf["proton_sw_density_uncert"]) + swapi_density_uncertainty = read_numeric_variable( + proton_cdf["proton_sw_density_uncert"] + ) swapi_temperature = read_numeric_variable(proton_cdf["proton_sw_temperature"]) - swapi_temperature_uncertainty = read_numeric_variable(proton_cdf["proton_sw_temperature_uncert"]) + swapi_temperature_uncertainty = read_numeric_variable( + proton_cdf["proton_sw_temperature_uncert"] + ) swapi_speed = read_numeric_variable(proton_cdf["proton_sw_speed"]) - swapi_speed_uncertainty = read_numeric_variable(proton_cdf["proton_sw_speed_uncert"]) + swapi_speed_uncertainty = read_numeric_variable( + proton_cdf["proton_sw_speed_uncert"] + ) swapi_velocity_rtn = read_numeric_variable(proton_cdf["proton_sw_velocity_rtn_sun"]) swapi_velocity_rtn_covariance = read_numeric_variable( - proton_cdf["proton_sw_velocity_rtn_covariance"]) + proton_cdf["proton_sw_velocity_rtn_covariance"] + ) good_record_mask = quality_flags == 0 timestamps = timestamps[good_record_mask] @@ -260,42 +324,93 @@ def build_dependency_collection(target_date, compact_date): swapi_velocity_rtn = swapi_velocity_rtn[good_record_mask] swapi_velocity_rtn_covariance = swapi_velocity_rtn_covariance[good_record_mask] -swapi_velocity_rtn_sigma = np.sqrt(np.diagonal( - swapi_velocity_rtn_covariance, axis1=1, axis2=2)) +swapi_velocity_rtn_sigma = np.sqrt( + np.diagonal(swapi_velocity_rtn_covariance, axis1=1, axis2=2) +) wind_protons = fetch_wind_swe_2min_protons(compact_date, DATA_DIR / "wind") or {} panels = [ - ("Density [cm$^{-3}$]", "log", - swapi_density, swapi_density_uncertainty, - wind_protons.get("density"), wind_protons.get("density_sigma")), - ("Temperature [K]", "log", - swapi_temperature, swapi_temperature_uncertainty, - wind_protons.get("temperature"), wind_protons.get("temperature_sigma")), - ("Bulk speed [km/s]", "linear", - swapi_speed, swapi_speed_uncertainty, - wind_protons.get("speed"), wind_protons.get("speed_sigma")), - ("$v_R$ [km/s]", "linear", - swapi_velocity_rtn[:, 0], swapi_velocity_rtn_sigma[:, 0], - wind_protons.get("velocity_r"), wind_protons.get("velocity_r_sigma")), - ("$v_T$ [km/s]", "linear", - swapi_velocity_rtn[:, 1], swapi_velocity_rtn_sigma[:, 1], - wind_protons.get("velocity_t"), wind_protons.get("velocity_t_sigma")), - ("$v_N$ [km/s]", "linear", - swapi_velocity_rtn[:, 2], swapi_velocity_rtn_sigma[:, 2], - wind_protons.get("velocity_n"), wind_protons.get("velocity_n_sigma")), + ( + "Density [cm$^{-3}$]", + "log", + swapi_density, + swapi_density_uncertainty, + wind_protons.get("density"), + wind_protons.get("density_sigma"), + ), + ( + "Temperature [K]", + "log", + swapi_temperature, + swapi_temperature_uncertainty, + wind_protons.get("temperature"), + wind_protons.get("temperature_sigma"), + ), + ( + "Bulk speed [km/s]", + "linear", + swapi_speed, + swapi_speed_uncertainty, + wind_protons.get("speed"), + wind_protons.get("speed_sigma"), + ), + ( + "$v_R$ [km/s]", + "linear", + swapi_velocity_rtn[:, 0], + swapi_velocity_rtn_sigma[:, 0], + wind_protons.get("velocity_r"), + wind_protons.get("velocity_r_sigma"), + ), + ( + "$v_T$ [km/s]", + "linear", + swapi_velocity_rtn[:, 1], + swapi_velocity_rtn_sigma[:, 1], + wind_protons.get("velocity_t"), + wind_protons.get("velocity_t_sigma"), + ), + ( + "$v_N$ [km/s]", + "linear", + swapi_velocity_rtn[:, 2], + swapi_velocity_rtn_sigma[:, 2], + wind_protons.get("velocity_n"), + wind_protons.get("velocity_n_sigma"), + ), ] wind_epoch = wind_protons.get("epoch") figure, axes = plt.subplots(len(panels), 1, sharex=True, figsize=(10, 14)) -for axis, (ylabel, yscale, swapi_values, swapi_uncertainties, - wind_values, wind_uncertainties) in zip(axes, panels): - axis.errorbar(timestamps, swapi_values, yerr=swapi_uncertainties, - fmt=".", capsize=2, color="tab:blue", label="SWAPI L3a") +for axis, ( + ylabel, + yscale, + swapi_values, + swapi_uncertainties, + wind_values, + wind_uncertainties, +) in zip(axes, panels): + axis.errorbar( + timestamps, + swapi_values, + yerr=swapi_uncertainties, + fmt=".", + capsize=2, + color="tab:blue", + label="SWAPI L3a", + ) if wind_values is not None and wind_epoch is not None: - axis.errorbar(wind_epoch, wind_values, yerr=wind_uncertainties, - fmt=".", capsize=2, color="tab:orange", alpha=0.6, - label="WIND SWE 2-min") + axis.errorbar( + wind_epoch, + wind_values, + yerr=wind_uncertainties, + fmt=".", + capsize=2, + color="tab:orange", + alpha=0.6, + label="WIND SWE 2-min", + ) axis.set_ylabel(ylabel) axis.set_yscale(yscale) axes[0].legend(loc="best") diff --git a/scripts/swapi/fit_and_plot_pui.py b/scripts/swapi/fit_and_plot_pui.py index f187be398..859b4d75d 100644 --- a/scripts/swapi/fit_and_plot_pui.py +++ b/scripts/swapi/fit_and_plot_pui.py @@ -38,7 +38,10 @@ ONE_SECOND_IN_NANOSECONDS, ) from imap_l3_processing.models import InputMetadata -from imap_l3_processing.swapi.constants import SWAPI_COARSE_SWEEP_BINS, SWAPI_L2_K_FACTOR +from imap_l3_processing.swapi.constants import ( + SWAPI_COARSE_SWEEP_BINS, + SWAPI_L2_K_FACTOR, +) from imap_l3_processing.swapi.l3a.science.pickup_ion.calculate_coincidence_rate import ( calculate_coincidence_rate, ) @@ -70,24 +73,30 @@ _COARSE_SWEEP_LEN = 62 -def replay_chunk_spectrum(dependencies, data_chunk, center_of_epoch, - sw_velocity_rtn_kms, fit_params): +def replay_chunk_spectrum( + dependencies, data_chunk, center_of_epoch, sw_velocity_rtn_kms, fit_params +): """Re-evaluate the forward model on the fit window for one 50-sweep chunk. Mirrors the geometry path in calculate_pickup_ion_values so the energies and modeled rate match production exactly. """ chunk_ephemeris_time = spiceypy.unitim( - center_of_epoch / ONE_SECOND_IN_NANOSECONDS, "TT", "ET") + center_of_epoch / ONE_SECOND_IN_NANOSECONDS, "TT", "ET" + ) sw_velocity_kms = float(np.linalg.norm(sw_velocity_rtn_kms)) - voltages_flat = (data_chunk.energy[:, SWAPI_COARSE_SWEEP_BINS] - / SWAPI_L2_K_FACTOR).flatten() + voltages_flat = ( + data_chunk.energy[:, SWAPI_COARSE_SWEEP_BINS] / SWAPI_L2_K_FACTOR + ).flatten() count_rates_flat = data_chunk.coincidence_count_rate[ - :, SWAPI_COARSE_SWEEP_BINS].flatten() + :, SWAPI_COARSE_SWEEP_BINS + ].flatten() central_effective_area_scale = ( - dependencies.efficiency_calibration_table - .central_effective_area_scale_for(center_of_epoch, "helium")) + dependencies.efficiency_calibration_table.central_effective_area_scale_for( + center_of_epoch, "helium" + ) + ) voltages_2d = voltages_flat.reshape(-1, _COARSE_SWEEP_LEN) count_rates_2d = count_rates_flat.reshape(-1, _COARSE_SWEEP_LEN) @@ -96,11 +105,13 @@ def replay_chunk_spectrum(dependencies, data_chunk, center_of_epoch, ) lower_energy_cutoff, upper_energy_cutoff = calculate_pickup_ion_fit_energy_range( - sw_velocity_kms) + sw_velocity_kms + ) voltages_per_step = np.mean(voltages_2d, axis=0) energies_per_step = np.abs(voltages_per_step) * SWAPI_L2_K_FACTOR - bin_mask = ((energies_per_step > lower_energy_cutoff) - & (energies_per_step < upper_energy_cutoff)) + bin_mask = (energies_per_step > lower_energy_cutoff) & ( + energies_per_step < upper_energy_cutoff + ) extracted_voltages = voltages_per_step[bin_mask] extracted_rates = count_rates_2d[:, bin_mask] @@ -113,7 +124,8 @@ def replay_chunk_spectrum(dependencies, data_chunk, center_of_epoch, 0.0, ) vasyliunas_siscoe_distribution = build_vasyliunas_siscoe_distribution( - chunk_ephemeris_time, sw_velocity_rtn_kms, + chunk_ephemeris_time, + sw_velocity_rtn_kms, dependencies.density_of_neutral_helium_calibration_table, dependencies.helium_inflow_vector, ) @@ -126,7 +138,8 @@ def replay_chunk_spectrum(dependencies, data_chunk, center_of_epoch, central_effective_area_scale=central_effective_area_scale, ) modeled_per_sweep = calculate_coincidence_rate( - chunk_response, vasyliunas_siscoe_distribution, fit_params_no_uncertainty) + chunk_response, vasyliunas_siscoe_distribution, fit_params_no_uncertainty + ) return { "energies_ev": extracted_voltages * SWAPI_L2_K_FACTOR, "observed_rate_per_sweep": extracted_rates, @@ -137,12 +150,20 @@ def replay_chunk_spectrum(dependencies, data_chunk, center_of_epoch, if __name__ == "__main__": argument_parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) argument_parser.add_argument("date", help="UTC date in YYYY-MM-DD format") - argument_parser.add_argument("--use-cache", action="store_true", - help="Skip download and refit; load the cached fit from /tmp.") - argument_parser.add_argument("--output-dir", type=Path, default=None, - help="If set, save the figure as pui_fit_.png in this directory and skip plt.show().") + argument_parser.add_argument( + "--use-cache", + action="store_true", + help="Skip download and refit; load the cached fit from /tmp.", + ) + argument_parser.add_argument( + "--output-dir", + type=Path, + default=None, + help="If set, save the figure as pui_fit_.png in this directory and skip plt.show().", + ) arguments = argument_parser.parse_args() if "IMAP_API_KEY" not in os.environ: @@ -155,49 +176,70 @@ def replay_chunk_spectrum(dependencies, data_chunk, center_of_epoch, imap_data_access.config["DATA_DIR"] = Path("/tmp/swapi_fit_and_plot_data") imap_data_access.config["DATA_DIR"].mkdir(parents=True, exist_ok=True) - ancillary_template_path = (Path(__file__).parent - / "imap_swapi_l3a_proton-sw_dependency_template.json") + ancillary_template_path = ( + Path(__file__).parent / "imap_swapi_l3a_proton-sw_dependency_template.json" + ) science_query_results = imap_data_access.query( - instrument="swapi", data_level="l2", descriptor="sci", - start_date=compact_date, end_date=compact_date, version="latest") + instrument="swapi", + data_level="l2", + descriptor="sci", + start_date=compact_date, + end_date=compact_date, + version="latest", + ) if not science_query_results: sys.exit(f"No SWAPI L2 sci file at the SDC for {arguments.date}") published_l3a_query_results = imap_data_access.query( - instrument="swapi", data_level="l3a", descriptor="pui-he", - start_date=compact_date, end_date=compact_date, version="latest") + instrument="swapi", + data_level="l3a", + descriptor="pui-he", + start_date=compact_date, + end_date=compact_date, + version="latest", + ) if published_l3a_query_results: published_l3a_cdf_path = imap_data_access.download( - published_l3a_query_results[0]["file_path"]) + published_l3a_query_results[0]["file_path"] + ) print(f"Downloaded published L3A pui-he from SDC: {published_l3a_cdf_path}") else: published_l3a_cdf_path = None print(f"No published SWAPI L3A pui-he file at the SDC for {arguments.date}") - ephemeris_kernel_types = [SpiceKernelTypes.EphemerisReconstructed, - SpiceKernelTypes.EphemerisPredicted] - non_ephemeris_kernel_types = [t for t in SpiceKernelTypes - if t not in ephemeris_kernel_types] - spice_kernel_filenames = list(dict.fromkeys( - os.path.basename(path) for path in ( - get_spice_kernels_file_names( - target_date - timedelta(days=1), - target_date + timedelta(days=2), - non_ephemeris_kernel_types) - + get_spice_kernels_file_names( - target_date - timedelta(days=90), - target_date + timedelta(days=2), - ephemeris_kernel_types) + ephemeris_kernel_types = [ + SpiceKernelTypes.EphemerisReconstructed, + SpiceKernelTypes.EphemerisPredicted, + ] + non_ephemeris_kernel_types = [ + t for t in SpiceKernelTypes if t not in ephemeris_kernel_types + ] + spice_kernel_filenames = list( + dict.fromkeys( + os.path.basename(path) + for path in ( + get_spice_kernels_file_names( + target_date - timedelta(days=1), + target_date + timedelta(days=2), + non_ephemeris_kernel_types, + ) + + get_spice_kernels_file_names( + target_date - timedelta(days=90), + target_date + timedelta(days=2), + ephemeris_kernel_types, + ) + ) ) - )) + ) dynamic_filenames = [ os.path.basename(science_query_results[0]["file_path"]), *spice_kernel_filenames, ] processing_input_collection = ProcessingInputCollection( - *[generate_imap_input(filename) for filename in dynamic_filenames]) + *[generate_imap_input(filename) for filename in dynamic_filenames] + ) processing_input_collection.deserialize(ancillary_template_path.read_text()) processing_input_collection.download_all_files() @@ -207,18 +249,25 @@ def replay_chunk_spectrum(dependencies, data_chunk, center_of_epoch, dependencies = SwapiL3ADependencies.fetch_dependencies(processing_input_collection) input_metadata = InputMetadata( - instrument="swapi", data_level="l3a", - start_date=target_date, end_date=target_date, - version="v001", descriptor="pui-he", + instrument="swapi", + data_level="l3a", + start_date=target_date, + end_date=target_date, + version="v001", + descriptor="pui-he", ) if not arguments.use_cache or not cache_path.exists(): - dependencies.swapi_response.warm_cache(dependencies.data.energy / SWAPI_L2_K_FACTOR) + dependencies.swapi_response.warm_cache( + dependencies.data.energy / SWAPI_L2_K_FACTOR + ) runner = ParallelChunkRunner( - dependencies.swapi_response, dependencies.efficiency_calibration_table) + dependencies.swapi_response, dependencies.efficiency_calibration_table + ) pui_chunks = list(chunk_l2_data(dependencies.data, 50)) proton_results = runner.run( - list(chunk_l2_data(dependencies.data, 5)), ProtonChunkFitter()) + list(chunk_l2_data(dependencies.data, 5)), ProtonChunkFitter() + ) pui_fitter = PuiChunkFitter( density_of_neutral_helium_lookup_table=dependencies.density_of_neutral_helium_calibration_table, hydrogen_inflow_vector=dependencies.hydrogen_inflow_vector, @@ -239,14 +288,22 @@ def replay_chunk_spectrum(dependencies, data_chunk, center_of_epoch, pickup_ion_data, sw_velocity_rtn_by_chunk_epoch = pickle.load(cache_file) epochs_tt2000 = np.asarray(pickup_ion_data.epoch, dtype=np.int64) - timestamps = np.array([cdf_library.tt2000_to_datetime(int(t)) for t in epochs_tt2000]) + timestamps = np.array( + [cdf_library.tt2000_to_datetime(int(t)) for t in epochs_tt2000] + ) - spectrogram_cache_path = Path(f"/tmp/swapi_pui_spectrograms_per_sweep_{compact_date}.pkl") + spectrogram_cache_path = Path( + f"/tmp/swapi_pui_spectrograms_per_sweep_{compact_date}.pkl" + ) all_chunks = list(chunk_l2_data(dependencies.data, 50)) if not arguments.use_cache or not spectrogram_cache_path.exists(): - print("Building per-sweep observed and modeled spectrograms (one replay per chunk)...") - dependencies.swapi_response.warm_cache(dependencies.data.energy / SWAPI_L2_K_FACTOR) + print( + "Building per-sweep observed and modeled spectrograms (one replay per chunk)..." + ) + dependencies.swapi_response.warm_cache( + dependencies.data.energy / SWAPI_L2_K_FACTOR + ) chunk_count = min(len(all_chunks), len(epochs_tt2000)) per_sweep_voltages = [] @@ -256,24 +313,31 @@ def replay_chunk_spectrum(dependencies, data_chunk, center_of_epoch, for chunk_index in range(chunk_count): data_chunk = all_chunks[chunk_index] - voltages_2d = (data_chunk.energy[:, SWAPI_COARSE_SWEEP_BINS] - / SWAPI_L2_K_FACTOR) + voltages_2d = ( + data_chunk.energy[:, SWAPI_COARSE_SWEEP_BINS] / SWAPI_L2_K_FACTOR + ) rates_2d = data_chunk.coincidence_count_rate[:, SWAPI_COARSE_SWEEP_BINS] per_sweep_voltages.append(voltages_2d) per_sweep_observed.append(rates_2d) per_sweep_epoch_tt2000.append( - np.asarray(data_chunk.sci_start_time, dtype=np.int64)) + np.asarray(data_chunk.sci_start_time, dtype=np.int64) + ) model_rates_for_chunk = np.full_like(rates_2d, np.nan, dtype=float) chunk_epoch = int(epochs_tt2000[chunk_index]) sw_velocity_rtn = sw_velocity_rtn_by_chunk_epoch.get(chunk_epoch) - if (sw_velocity_rtn is not None - and not np.any(np.isnan(sw_velocity_rtn)) - and np.isfinite(pickup_ion_data.cooling_index[chunk_index].n)): + if ( + sw_velocity_rtn is not None + and not np.any(np.isnan(sw_velocity_rtn)) + and np.isfinite(pickup_ion_data.cooling_index[chunk_index].n) + ): try: replay = replay_chunk_spectrum( - dependencies, data_chunk, chunk_epoch, sw_velocity_rtn, + dependencies, + data_chunk, + chunk_epoch, + sw_velocity_rtn, FittingParameters( pickup_ion_data.cooling_index[chunk_index], pickup_ion_data.ionization_rate[chunk_index], @@ -284,24 +348,37 @@ def replay_chunk_spectrum(dependencies, data_chunk, center_of_epoch, except Exception as replay_error: print(f"chunk {chunk_index} replay failed: {replay_error}") else: - model_rates_for_chunk[:, replay["bin_mask"]] = replay["modeled_rate_per_sweep"] + model_rates_for_chunk[:, replay["bin_mask"]] = replay[ + "modeled_rate_per_sweep" + ] per_sweep_model.append(model_rates_for_chunk) - energies_per_sweep_ev = np.concatenate(per_sweep_voltages, axis=0) * SWAPI_L2_K_FACTOR + energies_per_sweep_ev = ( + np.concatenate(per_sweep_voltages, axis=0) * SWAPI_L2_K_FACTOR + ) observed_spectrogram = np.concatenate(per_sweep_observed, axis=0) model_spectrogram = np.concatenate(per_sweep_model, axis=0) sweep_epochs_tt2000 = np.concatenate(per_sweep_epoch_tt2000, axis=0) with spectrogram_cache_path.open("wb") as spectrogram_file: pickle.dump( - (energies_per_sweep_ev, observed_spectrogram, model_spectrogram, - sweep_epochs_tt2000), - spectrogram_file) + ( + energies_per_sweep_ev, + observed_spectrogram, + model_spectrogram, + sweep_epochs_tt2000, + ), + spectrogram_file, + ) print(f"Cached spectrograms to {spectrogram_cache_path}") with spectrogram_cache_path.open("rb") as spectrogram_file: - (energies_per_sweep_ev, observed_spectrogram, model_spectrogram, - sweep_epochs_tt2000) = pickle.load(spectrogram_file) + ( + energies_per_sweep_ev, + observed_spectrogram, + model_spectrogram, + sweep_epochs_tt2000, + ) = pickle.load(spectrogram_file) import matplotlib.pyplot as plt from matplotlib.colors import LogNorm @@ -312,27 +389,32 @@ def replay_chunk_spectrum(dependencies, data_chunk, center_of_epoch, observed_sorted = observed_spectrogram[:, energy_sort_order] model_sorted = model_spectrogram[:, energy_sort_order] - positive_rate_values = np.concatenate([ - array[np.isfinite(array) & (array > 0)] - for array in (observed_sorted, model_sorted) - ]) + positive_rate_values = np.concatenate( + [ + array[np.isfinite(array) & (array > 0)] + for array in (observed_sorted, model_sorted) + ] + ) if positive_rate_values.size: rate_vmin = max(float(np.percentile(positive_rate_values, 1)), 1e-3) rate_vmax = float(positive_rate_values.max()) else: rate_vmin, rate_vmax = 1e-3, 1.0 - sweep_timestamps = np.array([ - cdf_library.tt2000_to_datetime(int(t)) for t in sweep_epochs_tt2000]) + sweep_timestamps = np.array( + [cdf_library.tt2000_to_datetime(int(t)) for t in sweep_epochs_tt2000] + ) figure = plt.figure(figsize=(11, 16), layout="constrained") grid_spec = figure.add_gridspec(8, 1, height_ratios=[1.6, 1.6, 1, 1, 1, 1, 1, 1]) observed_axis = figure.add_subplot(grid_spec[0]) observed_mesh = observed_axis.pcolormesh( - sweep_timestamps, mean_energies_ev_sorted, + sweep_timestamps, + mean_energies_ev_sorted, np.ma.masked_invalid(observed_sorted).T, - shading="nearest", cmap="viridis", + shading="nearest", + cmap="viridis", norm=LogNorm(vmin=rate_vmin, vmax=rate_vmax), ) observed_axis.set_yscale("log") @@ -341,31 +423,55 @@ def replay_chunk_spectrum(dependencies, data_chunk, center_of_epoch, figure.colorbar(observed_mesh, ax=observed_axis, label="rate / s$^{-1}$") plt.setp(observed_axis.get_xticklabels(), visible=False) - model_axis = figure.add_subplot(grid_spec[1], sharex=observed_axis, sharey=observed_axis) + model_axis = figure.add_subplot( + grid_spec[1], sharex=observed_axis, sharey=observed_axis + ) model_mesh = model_axis.pcolormesh( - sweep_timestamps, mean_energies_ev_sorted, + sweep_timestamps, + mean_energies_ev_sorted, np.ma.masked_invalid(model_sorted).T, - shading="nearest", cmap="viridis", + shading="nearest", + cmap="viridis", norm=LogNorm(vmin=rate_vmin, vmax=rate_vmax), ) model_axis.set_yscale("log") model_axis.set_ylabel("Energy / eV") - model_axis.set_title("Fitted helium PUI model coincidence count rate (per 12 s sweep)") + model_axis.set_title( + "Fitted helium PUI model coincidence count rate (per 12 s sweep)" + ) figure.colorbar(model_mesh, ax=model_axis, label="rate / s$^{-1}$") plt.setp(model_axis.get_xticklabels(), visible=False) - sw_speed_kms = np.array([ - np.linalg.norm(sw_velocity_rtn_by_chunk_epoch[int(epoch)]) - if int(epoch) in sw_velocity_rtn_by_chunk_epoch else np.nan - for epoch in epochs_tt2000 - ]) + sw_speed_kms = np.array( + [ + np.linalg.norm(sw_velocity_rtn_by_chunk_epoch[int(epoch)]) + if int(epoch) in sw_velocity_rtn_by_chunk_epoch + else np.nan + for epoch in epochs_tt2000 + ] + ) CUTOFF_SPEED_PANEL_INDEX = 2 timeseries_panels = [ ("Cooling index", "linear", pickup_ion_data.cooling_index, "pui_cooling_index"), - ("Ionization rate [s$^{-1}$]", "log", pickup_ion_data.ionization_rate, "pui_ionization_rate"), - ("Cutoff speed [km/s]", "linear", pickup_ion_data.cutoff_speed, "pui_cutoff_speed"), - ("Background rate [s$^{-1}$]", "linear", pickup_ion_data.background_rate, "pui_background_count_rate"), + ( + "Ionization rate [s$^{-1}$]", + "log", + pickup_ion_data.ionization_rate, + "pui_ionization_rate", + ), + ( + "Cutoff speed [km/s]", + "linear", + pickup_ion_data.cutoff_speed, + "pui_cutoff_speed", + ), + ( + "Background rate [s$^{-1}$]", + "linear", + pickup_ion_data.background_rate, + "pui_background_count_rate", + ), ("Density [cm$^{-3}$]", "log", pickup_ion_data.density, "pui_density"), ("Temperature [K]", "log", pickup_ion_data.temperature, "pui_temperature"), ] @@ -375,52 +481,91 @@ def replay_chunk_spectrum(dependencies, data_chunk, center_of_epoch, if published_l3a_cdf_path is not None: with CDF(str(published_l3a_cdf_path)) as published_cdf: published_epoch_tt2000 = np.asarray( - published_cdf.raw_var("epoch")[...], dtype=np.int64) - published_l3a_timestamps = np.array([ - cdf_library.tt2000_to_datetime(int(t)) for t in published_epoch_tt2000 - ]) + published_cdf.raw_var("epoch")[...], dtype=np.int64 + ) + published_l3a_timestamps = np.array( + [cdf_library.tt2000_to_datetime(int(t)) for t in published_epoch_tt2000] + ) for _, _, _, cdf_var_name in timeseries_panels: published_l3a_values_by_var[cdf_var_name] = ( np.asarray(published_cdf[cdf_var_name][...], dtype=float), - np.asarray(published_cdf[f"{cdf_var_name}_uncert"][...], dtype=float), + np.asarray( + published_cdf[f"{cdf_var_name}_uncert"][...], dtype=float + ), ) local_flags = np.asarray(pickup_ion_data.quality_flags, dtype=np.int64) local_flagged = local_flags != 0 - for panel_index, (ylabel, yscale, values, cdf_var_name) in enumerate(timeseries_panels): + for panel_index, (ylabel, yscale, values, cdf_var_name) in enumerate( + timeseries_panels + ): axis = figure.add_subplot(grid_spec[panel_index + 2], sharex=observed_axis) nominal_values_array = unumpy.nominal_values(values) std_devs_array = unumpy.std_devs(values) - show_legend = (panel_index == CUTOFF_SPEED_PANEL_INDEX - or published_l3a_timestamps is not None) - axis.errorbar(timestamps, - nominal_values_array, - yerr=std_devs_array, - fmt=".", capsize=2, color="tab:blue", - label="This run" if show_legend else None) + show_legend = ( + panel_index == CUTOFF_SPEED_PANEL_INDEX + or published_l3a_timestamps is not None + ) + axis.errorbar( + timestamps, + nominal_values_array, + yerr=std_devs_array, + fmt=".", + capsize=2, + color="tab:blue", + label="This run" if show_legend else None, + ) if np.any(local_flagged): - axis.scatter(timestamps[local_flagged], - nominal_values_array[local_flagged], - s=60, facecolors="none", edgecolors="red", - linewidths=1.2, zorder=3, - label="flag != 0" if show_legend else None) + axis.scatter( + timestamps[local_flagged], + nominal_values_array[local_flagged], + s=60, + facecolors="none", + edgecolors="red", + linewidths=1.2, + zorder=3, + label="flag != 0" if show_legend else None, + ) if published_l3a_timestamps is not None: - published_nominal, published_uncert = published_l3a_values_by_var[cdf_var_name] - valid = np.isfinite(published_nominal) & (published_uncert >= 0) & np.isfinite(published_uncert) - axis.errorbar(published_l3a_timestamps[valid], - published_nominal[valid], - yerr=published_uncert[valid], - fmt="x", capsize=2, color="red", - label="SDC L3A") + published_nominal, published_uncert = published_l3a_values_by_var[ + cdf_var_name + ] + valid = ( + np.isfinite(published_nominal) + & (published_uncert >= 0) + & np.isfinite(published_uncert) + ) + axis.errorbar( + published_l3a_timestamps[valid], + published_nominal[valid], + yerr=published_uncert[valid], + fmt="x", + capsize=2, + color="red", + label="SDC L3A", + ) if panel_index == CUTOFF_SPEED_PANEL_INDEX: - axis.plot(timestamps, sw_speed_kms, - color="tab:orange", linewidth=1.0, label="Proton SW speed") - axis.fill_between(timestamps, 0.8 * sw_speed_kms, 1.2 * sw_speed_kms, - color="tab:orange", alpha=0.15, linewidth=0, - label=r"$\pm 20\%$ SW speed") - if (panel_index == CUTOFF_SPEED_PANEL_INDEX - or published_l3a_timestamps is not None): + axis.plot( + timestamps, + sw_speed_kms, + color="tab:orange", + linewidth=1.0, + label="Proton SW speed", + ) + axis.fill_between( + timestamps, + 0.8 * sw_speed_kms, + 1.2 * sw_speed_kms, + color="tab:orange", + alpha=0.15, + linewidth=0, + label=r"$\pm 20\%$ SW speed", + ) + if ( + panel_index == CUTOFF_SPEED_PANEL_INDEX + or published_l3a_timestamps is not None + ): axis.legend(loc="best", fontsize="small") axis.set_ylabel(ylabel) axis.set_yscale(yscale) diff --git a/scripts/swapi/generate_cdf_from_dat.py b/scripts/swapi/generate_cdf_from_dat.py index bd0fd6369..81337a627 100644 --- a/scripts/swapi/generate_cdf_from_dat.py +++ b/scripts/swapi/generate_cdf_from_dat.py @@ -5,14 +5,21 @@ from spacepy.pycdf.const import CDF_TIME_TT2000 import imap_l3_processing -from scripts.swapi.swapi_alpha_sw_speed_temperature_density_demo import read_l2_data_from_dat +from scripts.swapi.swapi_alpha_sw_speed_temperature_density_demo import ( + read_l2_data_from_dat, +) def generate_cdf_from_dat(dat_file_path: str, output_filename: str): dat_data = read_l2_data_from_dat(dat_file_path) - cdf_file_path = Path(imap_l3_processing.__file__).parent.parent / 'swapi' / 'test_data' / output_filename - - with CDF(str(cdf_file_path), '') as cdf: + cdf_file_path = ( + Path(imap_l3_processing.__file__).parent.parent + / "swapi" + / "test_data" + / output_filename + ) + + with CDF(str(cdf_file_path), "") as cdf: cdf.col_major(True) cdf.new("energy", dat_data.energy, recVary=False) @@ -23,5 +30,9 @@ def generate_cdf_from_dat(dat_file_path: str, output_filename: str): if __name__ == "__main__": - generate_cdf_from_dat(os.path.abspath("../../instrument_team_data/swapi/swapi_test_data_50_sweeps.dat"), - "imap_swapi_l2_sci_20100101_v001") + generate_cdf_from_dat( + os.path.abspath( + "../../instrument_team_data/swapi/swapi_test_data_50_sweeps.dat" + ), + "imap_swapi_l2_sci_20100101_v001", + ) diff --git a/scripts/swapi/generate_collapsed_response_grid_reference.py b/scripts/swapi/generate_collapsed_response_grid_reference.py index a850b2645..0222fb928 100644 --- a/scripts/swapi/generate_collapsed_response_grid_reference.py +++ b/scripts/swapi/generate_collapsed_response_grid_reference.py @@ -38,7 +38,11 @@ _REPO_ROOT = Path(__file__).resolve().parents[2] _OUTPUT_PATH = ( - _REPO_ROOT / "tests" / "test_data" / "swapi" / "collapsed_response_grid_reference.csv" + _REPO_ROOT + / "tests" + / "test_data" + / "swapi" + / "collapsed_response_grid_reference.csv" ) _N_POINTS = 256 @@ -67,17 +71,20 @@ def shell_integral_h( central_speed = float(response_grid.central_speed) # SWAPI convention: hat{v}(θ,φ) = (-cosθ sinφ, -cosθ cosφ, -sinθ). - bulk_direction = np.array([ - -math.cos(bulk_elevation) * math.sin(bulk_azimuth), - -math.cos(bulk_elevation) * math.cos(bulk_azimuth), - -math.sin(bulk_elevation), - ]) + bulk_direction = np.array( + [ + -math.cos(bulk_elevation) * math.sin(bulk_azimuth), + -math.cos(bulk_elevation) * math.cos(bulk_azimuth), + -math.sin(bulk_elevation), + ] + ) velocity = bulk_speed * bulk_direction # cos-α band from |v|² = v_sw² + v'² + 2 v_sw v' cos α (law of cosines). cos_alpha_bounds = ( (np.array([_SPEED_RATIO_MIN, _SPEED_RATIO_MAX]) * central_speed) ** 2 - - bulk_speed**2 - v_prime**2 + - bulk_speed**2 + - v_prime**2 ) / (2.0 * bulk_speed * v_prime) if cos_alpha_bounds[0] >= 1.0 or cos_alpha_bounds[1] <= -1.0: return 0.0 @@ -85,16 +92,29 @@ def shell_integral_h( # Right-handed orthonormal frame with bulk_direction as polar axis. e_in_xy = np.array([-math.cos(bulk_azimuth), math.sin(bulk_azimuth), 0.0]) - rotation = np.column_stack([e_in_xy, np.cross(bulk_direction, e_in_xy), bulk_direction]) + rotation = np.column_stack( + [e_in_xy, np.cross(bulk_direction, e_in_xy), bulk_direction] + ) def integrate(passband, azimuth_min_deg, azimuth_max_deg): return dblquad( lambda alpha, beta: _shell_integrand( - alpha, beta, velocity, rotation, v_prime, response_grid, - passband, azimuth_min_deg, azimuth_max_deg, + alpha, + beta, + velocity, + rotation, + v_prime, + response_grid, + passband, + azimuth_min_deg, + azimuth_max_deg, ), - 0.0, 2.0 * math.pi, alpha_min, alpha_max, - epsabs=1e-13, epsrel=1e-3, + 0.0, + 2.0 * math.pi, + alpha_min, + alpha_max, + epsabs=1e-13, + epsrel=1e-3, )[0] return ( @@ -106,8 +126,15 @@ def integrate(passband, azimuth_min_deg, azimuth_max_deg): @numba.njit def _shell_integrand( - alpha, beta, velocity, rotation, v_prime, response_grid, - passband, azimuth_min_deg, azimuth_max_deg, + alpha, + beta, + velocity, + rotation, + v_prime, + response_grid, + passband, + azimuth_min_deg, + azimuth_max_deg, ): # `alpha`: polar angle on the shell from bulk-velocity direction; # caller restricts to the α-band where the shell crosses @@ -119,21 +146,33 @@ def _shell_integrand( local_x = sin_alpha * math.cos(beta) local_y = sin_alpha * math.sin(beta) local_z = math.cos(alpha) - shell_x = rotation[0, 0] * local_x + rotation[0, 1] * local_y + rotation[0, 2] * local_z - shell_y = rotation[1, 0] * local_x + rotation[1, 1] * local_y + rotation[1, 2] * local_z - shell_z = rotation[2, 0] * local_x + rotation[2, 1] * local_y + rotation[2, 2] * local_z + shell_x = ( + rotation[0, 0] * local_x + rotation[0, 1] * local_y + rotation[0, 2] * local_z + ) + shell_y = ( + rotation[1, 0] * local_x + rotation[1, 1] * local_y + rotation[1, 2] * local_z + ) + shell_z = ( + rotation[2, 0] * local_x + rotation[2, 1] * local_y + rotation[2, 2] * local_z + ) vx = velocity[0] + v_prime * shell_x vy = velocity[1] + v_prime * shell_y vz = velocity[2] + v_prime * shell_z speed = math.sqrt(vx * vx + vy * vy + vz * vz) - azimuth_deg, elevation_deg = velocity_components_to_angles_in_instrument_frame(vx, vy, vz) + azimuth_deg, elevation_deg = velocity_components_to_angles_in_instrument_frame( + vx, vy, vz + ) if not (azimuth_min_deg <= azimuth_deg <= azimuth_max_deg): return 0.0 effective_area = ( response_grid.central_effective_area * _EFFECTIVE_AREA_CM2_TO_KM2 - * interpolate_passband(passband, elevation_deg, speed / response_grid.central_speed) - * interpolate_azimuthal_transmission(response_grid.azimuthal_transmission, azimuth_deg) + * interpolate_passband( + passband, elevation_deg, speed / response_grid.central_speed + ) + * interpolate_azimuthal_transmission( + response_grid.azimuthal_transmission, azimuth_deg + ) ) # H is an angular integral (docs convention); dα dβ comes from dblquad, so # only the sin α solid-angle factor is multiplied in here. @@ -160,10 +199,10 @@ def _worker_compute(v_prime): def main(): - print(f"Loading SWAPI response and warming cache at V={_ESA_VOLTAGE}...", flush=True) - swapi_response = load_swapi_response( - warm_cache_voltages=np.array([_ESA_VOLTAGE]) + print( + f"Loading SWAPI response and warming cache at V={_ESA_VOLTAGE}...", flush=True ) + swapi_response = load_swapi_response(warm_cache_voltages=np.array([_ESA_VOLTAGE])) response_grid = swapi_response.get_response_grid( esa_voltage=_ESA_VOLTAGE, mass_per_charge_m_p_per_e=_MASS_PER_CHARGE, diff --git a/scripts/swapi/generate_reference_integrals.py b/scripts/swapi/generate_reference_integrals.py index 18b0ba21b..33c89fffe 100644 --- a/scripts/swapi/generate_reference_integrals.py +++ b/scripts/swapi/generate_reference_integrals.py @@ -49,9 +49,7 @@ _WIND_SAMPLES_PATH = ( _REPO_ROOT / "docs" / "swapi" / "figure_src" / "wind_solar_wind_samples_2025.csv" ) -_OUTPUT_PATH = ( - _REPO_ROOT / "tests" / "test_data" / "swapi" / "reference_integrals.csv" -) +_OUTPUT_PATH = _REPO_ROOT / "tests" / "test_data" / "swapi" / "reference_integrals.csv" def _peak_voltage(bulk_speed_km_s: float) -> float: @@ -90,9 +88,7 @@ def main(): ) for i in range(n_samples) ] - rotation_matrices = np.broadcast_to( - ANCHOR_ROTATION_SWAPI_TO_RTN, (n_samples, 3, 3) - ) + rotation_matrices = np.broadcast_to(ANCHOR_ROTATION_SWAPI_TO_RTN, (n_samples, 3, 3)) voltages = [_peak_voltage(float(v)) for v in bulk_speed] swapi_response.warm_cache(voltages) response_grids = [ diff --git a/scripts/swapi/generate_with_predicted_ephemeris.py b/scripts/swapi/generate_with_predicted_ephemeris.py index 818a301e4..fa6c010b8 100644 --- a/scripts/swapi/generate_with_predicted_ephemeris.py +++ b/scripts/swapi/generate_with_predicted_ephemeris.py @@ -18,29 +18,37 @@ SpiceKernelTypes.PointingAttitude, SpiceKernelTypes.SpacecraftClock, SpiceKernelTypes.EphemerisPredicted, - SpiceKernelTypes.PlanetaryEphemeris + SpiceKernelTypes.PlanetaryEphemeris, ] def generate_swapi_for_given_day(descriptor: str, day: datetime): - spice_kernel_files = get_spice_kernels_file_names(day, day + timedelta(days=1), SPICE_KERNELS) - day_as_string = day.strftime('%Y%m%d') - l2_file_path = imap_data_access.query(instrument='swapi', data_level='l2', start_date=day_as_string, - end_date=day_as_string, version='latest')[0]['file_path'] - - template_json = Path('scripts/swapi/imap_swapi_l3a_proton-sw_dependency_template.json') - - input_files = [ - *spice_kernel_files, - Path(l2_file_path).name - ] - - input_collection = ProcessingInputCollection(*[generate_imap_input(i) for i in input_files]) + spice_kernel_files = get_spice_kernels_file_names( + day, day + timedelta(days=1), SPICE_KERNELS + ) + day_as_string = day.strftime("%Y%m%d") + l2_file_path = imap_data_access.query( + instrument="swapi", + data_level="l2", + start_date=day_as_string, + end_date=day_as_string, + version="latest", + )[0]["file_path"] + + template_json = Path( + "scripts/swapi/imap_swapi_l3a_proton-sw_dependency_template.json" + ) + + input_files = [*spice_kernel_files, Path(l2_file_path).name] + + input_collection = ProcessingInputCollection( + *[generate_imap_input(i) for i in input_files] + ) input_collection.deserialize(template_json.read_text()) input_collection.download_all_files() - dependency_file_name = f'imap_swapi_l3a_{descriptor}_{day_as_string}_v002.json' + dependency_file_name = f"imap_swapi_l3a_{descriptor}_{day_as_string}_v002.json" output_filepath = DependencyFilePath(dependency_file_name) fullpath = output_filepath.construct_path() @@ -48,23 +56,27 @@ def generate_swapi_for_given_day(descriptor: str, day: datetime): fullpath.write_text(input_collection.serialize()) - args_string = (f"imap_l3_data_processor.py " - f"--instrument swapi --data-level l3a --descriptor {descriptor} " - f"--start-date {day_as_string} --version v003 " - f"--dependency {dependency_file_name}") + args_string = ( + f"imap_l3_data_processor.py " + f"--instrument swapi --data-level l3a --descriptor {descriptor} " + f"--start-date {day_as_string} --version v003 " + f"--dependency {dependency_file_name}" + ) return subprocess.Popen([sys.executable, *args_string.split(" ")]) -if __name__ == '__main__': +if __name__ == "__main__": if len(sys.argv) > 1: dates = [datetime.strptime(arg, "%Y%m%d") for arg in sys.argv[1:]] else: - dates = [datetime(2025, 12, 15), - datetime(2025, 12, 16), - datetime(2025, 12, 17), - datetime(2025, 12, 18)] - pids = [generate_swapi_for_given_day('alpha-sw', date) for date in dates] + dates = [ + datetime(2025, 12, 15), + datetime(2025, 12, 16), + datetime(2025, 12, 17), + datetime(2025, 12, 18), + ] + pids = [generate_swapi_for_given_day("alpha-sw", date) for date in dates] for pid in pids: pid.wait() diff --git a/scripts/swapi/pui_xarray_reference.py b/scripts/swapi/pui_xarray_reference.py index 35189bb8a..c3f650c9c 100644 --- a/scripts/swapi/pui_xarray_reference.py +++ b/scripts/swapi/pui_xarray_reference.py @@ -63,6 +63,7 @@ \end{pmatrix} $$ """ + import matplotlib.pyplot as plt import numpy as np import pandas as pd @@ -82,23 +83,37 @@ K_FACTOR = Q(1.89, "eV/V/e") # E_beam = K · |V|; eV carries the elementary charge ONE_AU = Q(1.0, "au") -HE_PLUS_MASS_PER_CHARGE = Q(HE_PUI_PARTICLE_MASS_PER_CHARGE_M_P_PER_E, "m_p/e") # He+ nucleus, q = +1 e +HE_PLUS_MASS_PER_CHARGE = Q( + HE_PUI_PARTICLE_MASS_PER_CHARGE_M_P_PER_E, "m_p/e" +) # He+ nucleus, q = +1 e HELIUM_EFFICIENCY_RATIO = 1.05 """## Load response files""" central_effective_area = HELIUM_EFFICIENCY_RATIO * ( - pd.read_csv(get_test_instrument_team_data_path("swapi/imap_swapi_central-effective-area_20260425_v001.csv")) - .set_index("esa_voltage").effective_area.to_xarray().rename(esa_voltage="V") + pd.read_csv( + get_test_instrument_team_data_path( + "swapi/imap_swapi_central-effective-area_20260425_v001.csv" + ) + ) + .set_index("esa_voltage") + .effective_area.to_xarray() + .rename(esa_voltage="V") .pint.quantify("cm**2") ) _voltages_mag = central_effective_area["V"].values voltages = xr.DataArray( - Q(_voltages_mag, "V"), dims="V", coords={"V": _voltages_mag}, + Q(_voltages_mag, "V"), + dims="V", + coords={"V": _voltages_mag}, ) passband_coefficients = ( - pd.read_csv(get_test_instrument_team_data_path("swapi/imap_swapi_passband-fit-coefficients_20260425_v001.csv")) + pd.read_csv( + get_test_instrument_team_data_path( + "swapi/imap_swapi_passband-fit-coefficients_20260425_v001.csv" + ) + ) .set_index(["region", "energy_ratio", "elevation"])[["0", "1", "2"]] .rename_axis(columns="degree") .to_xarray() @@ -107,20 +122,38 @@ .pipe(lambda da: da.assign_coords(degree=da["degree"].astype(int))) ) source_speed_ratio = np.sqrt( - passband_coefficients["energy_ratio"].values / K_FACTOR.to('eV/V/e').magnitude + passband_coefficients["energy_ratio"].values / K_FACTOR.to("eV/V/e").magnitude ) passband_coefficients = ( - passband_coefficients.assign_coords(speed_ratio=("energy_ratio", source_speed_ratio)) - .swap_dims({"energy_ratio": "speed_ratio"}).drop_vars("energy_ratio") + passband_coefficients.assign_coords( + speed_ratio=("energy_ratio", source_speed_ratio) + ) + .swap_dims({"energy_ratio": "speed_ratio"}) + .drop_vars("energy_ratio") ) az_transmission_native = ( - pd.read_csv(get_test_instrument_team_data_path("swapi/imap_swapi_azimuthal-transmission_20260425_v001.csv")) - .fillna(0).set_index("abs_azimuth").transmission.to_xarray() -).coarsen(abs_azimuth=10, boundary='trim').mean() -print("azimuth spacing:", az_transmission_native.abs_azimuth.to_series().diff().dropna().round(4).unique()) + ( + pd.read_csv( + get_test_instrument_team_data_path( + "swapi/imap_swapi_azimuthal-transmission_20260425_v001.csv" + ) + ) + .fillna(0) + .set_index("abs_azimuth") + .transmission.to_xarray() + ) + .coarsen(abs_azimuth=10, boundary="trim") + .mean() +) +print( + "azimuth spacing:", + az_transmission_native.abs_azimuth.to_series().diff().dropna().round(4).unique(), +) -density_data = np.loadtxt(get_test_instrument_team_data_path("swapi/density-of-neutral-helium-lut.dat")) +density_data = np.loadtxt( + get_test_instrument_team_data_path("swapi/density-of-neutral-helium-lut.dat") +) psi_axis, r_axis = np.unique(density_data[:, 0]), np.unique(density_data[:, 1]) density_values = density_data[:, 2].reshape(len(psi_axis), len(r_axis)) @@ -129,33 +162,42 @@ density_table = xr.DataArray( density_values, - dims=("psi", "r"), coords={"psi": psi_axis, "r": r_axis}, + dims=("psi", "r"), + coords={"psi": psi_axis, "r": r_axis}, ).pint.quantify("1/cm**3") """## Integration grid (native to the calibration files)""" elevation_deg = passband_coefficients["elevation_deg"] -speed_ratio = passband_coefficients["speed_ratio"] +speed_ratio = passband_coefficients["speed_ratio"] # Mirror az_transmission_native (abs_azimuth 0..180) into signed [-180, 180]. _abs_az = az_transmission_native["abs_azimuth"].values -_trans = az_transmission_native.values +_trans = az_transmission_native.values _az_signed = np.concatenate([-_abs_az[:0:-1], _abs_az]) -azimuth_deg = xr.DataArray(_az_signed, dims="azimuth_deg", coords={"azimuth_deg": _az_signed}) +azimuth_deg = xr.DataArray( + _az_signed, dims="azimuth_deg", coords={"azimuth_deg": _az_signed} +) azimuthal_transmission = xr.DataArray( np.concatenate([_trans[:0:-1], _trans]), - dims="azimuth_deg", coords={"azimuth_deg": _az_signed}, + dims="azimuth_deg", + coords={"azimuth_deg": _az_signed}, ) azimuth_rad = np.deg2rad(azimuth_deg) elevation_rad = np.deg2rad(elevation_deg) -direction = xr.concat([ - -np.cos(elevation_rad) * np.sin(azimuth_rad), - -np.cos(elevation_rad) * np.cos(azimuth_rad), - -np.sin(elevation_rad) * xr.ones_like(azimuth_rad), -], dim="cartesian").assign_coords(cartesian=["x", "y", "z"]) - -central_instrument_speed = np.sqrt(2 * K_FACTOR * voltages / HE_PLUS_MASS_PER_CHARGE).pint.to("km/s") +direction = xr.concat( + [ + -np.cos(elevation_rad) * np.sin(azimuth_rad), + -np.cos(elevation_rad) * np.cos(azimuth_rad), + -np.sin(elevation_rad) * xr.ones_like(azimuth_rad), + ], + dim="cartesian", +).assign_coords(cartesian=["x", "y", "z"]) + +central_instrument_speed = np.sqrt( + 2 * K_FACTOR * voltages / HE_PLUS_MASS_PER_CHARGE +).pint.to("km/s") """## Evaluate passband at each ESA voltage""" @@ -166,22 +208,24 @@ # A_0 and P share the SIMION normalization at (θ=0, k*=1.89 eV/V/e); enforce # P(elevation_deg=0, speed_ratio=1) = 1 to match the production passband grid. passband_per_region = passband_per_region / passband_per_region.interp( - elevation_deg=0.0, speed_ratio=1.0, + elevation_deg=0.0, + speed_ratio=1.0, ) az_is_sg = np.abs(azimuth_deg) <= 20 -passband_full = xr.where(az_is_sg, passband_per_region.sel(region="SG"), - passband_per_region.sel(region="OA")) +passband_full = xr.where( + az_is_sg, passband_per_region.sel(region="SG"), passband_per_region.sel(region="OA") +) """## Parameters""" SW_SPEED_KMS = 450.0 SW_AZ_DEG = 0.0 SW_EL_DEG = -10.0 -COOLING_INDEX = 2.0 # α +COOLING_INDEX = 2.0 # α CUTOFF_SPEED_KMS = 450.0 # v_b -IONIZATION_RATE_HZ = 2e-7 # β_E at 1 AU -HELIO_DIST_AU = 1.0 # r -INFLOW_PSI_DEG = 75.0 # ψ — IMAP-to-helium-inflow longitude offset +IONIZATION_RATE_HZ = 2e-7 # β_E at 1 AU +HELIO_DIST_AU = 1.0 # r +INFLOW_PSI_DEG = 75.0 # ψ — IMAP-to-helium-inflow longitude offset SW_SPEED_INERTIAL_KMS = 450.0 # |v_sw| in ECLIPJ2000 (≈ SW_SPEED + 25) """## Bulk-SW vector in SWAPI coords @@ -190,24 +234,26 @@ sw_az_rad = np.deg2rad(SW_AZ_DEG) sw_el_rad = np.deg2rad(SW_EL_DEG) -v_sw_vec = Q(SW_SPEED_KMS, "km/s") * np.array([ - -np.cos(sw_el_rad) * np.sin(sw_az_rad), - -np.cos(sw_el_rad) * np.cos(sw_az_rad), - -np.sin(sw_el_rad), -]) +v_sw_vec = Q(SW_SPEED_KMS, "km/s") * np.array( + [ + -np.cos(sw_el_rad) * np.sin(sw_az_rad), + -np.cos(sw_el_rad) * np.cos(sw_az_rad), + -np.sin(sw_el_rad), + ] +) v_sw = xr.DataArray(v_sw_vec, dims="cartesian", coords={"cartesian": ["x", "y", "z"]}) """## Per-V speed grid and SW-frame speed""" -speed_grid = ( - speed_ratio * central_instrument_speed -).transpose('V', 'speed_ratio') -v_sw_speed = np.sqrt((v_sw**2).sum("cartesian")) # scalar -v_dot_vsw = xr.dot(direction, v_sw, dim="cartesian") # (az, el) -speed_sw = np.sqrt(np.maximum( - speed_grid**2 + v_sw_speed**2 - 2 * speed_grid * v_dot_vsw, - Q(0, "km**2/s**2"), -)) +speed_grid = (speed_ratio * central_instrument_speed).transpose("V", "speed_ratio") +v_sw_speed = np.sqrt((v_sw**2).sum("cartesian")) # scalar +v_dot_vsw = xr.dot(direction, v_sw, dim="cartesian") # (az, el) +speed_sw = np.sqrt( + np.maximum( + speed_grid**2 + v_sw_speed**2 - 2 * speed_grid * v_dot_vsw, + Q(0, "km**2/s**2"), + ) +) r"""## V-S PUI VDF @@ -244,10 +290,14 @@ term3 = w ** (cooling_index - 3) # Term 4 `r·w^α` -term4 = density_table.pint.interp( - psi=(psi % Q(360, "deg")).magnitude, - r=(r * w**cooling_index).pint.to("au").pint.dequantify(), -).drop_vars(["psi", "r"]).fillna(0) +term4 = ( + density_table.pint.interp( + psi=(psi % Q(360, "deg")).magnitude, + r=(r * w**cooling_index).pint.to("au").pint.dequantify(), + ) + .drop_vars(["psi", "r"]) + .fillna(0) +) # Term 5 Θ(1 − w) term5 = xr.where((w < 1), 1, 0) @@ -256,20 +306,22 @@ """## Coincidence count rate per ESA voltage""" -effective_area = (central_effective_area * passband_full * azimuthal_transmission).clip(min=Q(0, "cm**2")) +effective_area = (central_effective_area * passband_full * azimuthal_transmission).clip( + min=Q(0, "cm**2") +) flux = f_PUI * speed_grid deg2_to_sr = (np.pi / 180.0) ** 2 cos_elevation = np.cos(np.deg2rad(elevation_deg)) -d3v = speed_grid ** 2 * deg2_to_sr * cos_elevation +d3v = speed_grid**2 * deg2_to_sr * cos_elevation integrand = effective_area * flux * d3v integral = ( integrand.integrate("azimuth_deg") - .integrate("elevation_deg") - .integrate("speed_ratio") + .integrate("elevation_deg") + .integrate("speed_ratio") ) count_rate = (integral * central_instrument_speed).pint.to("counts/s") @@ -288,10 +340,7 @@ ax0.set_xlabel("ESA voltage [V]") ax0.set_ylabel("Coincidence Rate [Hz]") ax0.set_ylim(1e-1, 1e2) -ax0.set_title( - f"|v_sw|={SW_SPEED_KMS:g} km/s " - f"az={SW_AZ_DEG:g}° el={SW_EL_DEG:g}°" -) +ax0.set_title(f"|v_sw|={SW_SPEED_KMS:g} km/s az={SW_AZ_DEG:g}° el={SW_EL_DEG:g}°") ax0.grid(True, alpha=0.3) ax1.semilogy(v_flat_order / cutoff_speed, f_flat_order) diff --git a/scripts/swapi/pui_xarray_reference_50sweep.py b/scripts/swapi/pui_xarray_reference_50sweep.py index d201bb9cb..90c90b828 100644 --- a/scripts/swapi/pui_xarray_reference_50sweep.py +++ b/scripts/swapi/pui_xarray_reference_50sweep.py @@ -6,6 +6,7 @@ - `evaluate_pui_sweep_xarray(...)` evaluates the V-S PUI count rate for one sweep at fitted (or truth) parameters. """ + import os import sys from dataclasses import dataclass @@ -80,8 +81,7 @@ def build_pui_xarray_context( .pipe(lambda da: da.assign_coords(degree=da["degree"].astype(int))) ) source_speed_ratio = np.sqrt( - passband_coefficients["energy_ratio"].values - / K_FACTOR.to("eV/V/e").magnitude + passband_coefficients["energy_ratio"].values / K_FACTOR.to("eV/V/e").magnitude ) passband_coefficients = ( passband_coefficients.assign_coords( @@ -92,11 +92,15 @@ def build_pui_xarray_context( ) az_transmission_native = ( - pd.read_csv(azimuthal_transmission_path) - .fillna(0) - .set_index("abs_azimuth") - .transmission.to_xarray() - ).coarsen(abs_azimuth=10, boundary="trim").mean() + ( + pd.read_csv(azimuthal_transmission_path) + .fillna(0) + .set_index("abs_azimuth") + .transmission.to_xarray() + ) + .coarsen(abs_azimuth=10, boundary="trim") + .mean() + ) density_data = np.loadtxt(density_of_neutral_helium_lut_path) psi_axis, r_axis = np.unique(density_data[:, 0]), np.unique(density_data[:, 1]) @@ -116,9 +120,7 @@ def build_pui_xarray_context( abs_azimuth_axis = az_transmission_native["abs_azimuth"].values transmission_values = az_transmission_native.values - signed_azimuth_axis = np.concatenate( - [-abs_azimuth_axis[:0:-1], abs_azimuth_axis] - ) + signed_azimuth_axis = np.concatenate([-abs_azimuth_axis[:0:-1], abs_azimuth_axis]) azimuth_deg = xr.DataArray( signed_azimuth_axis, dims="azimuth_deg", @@ -152,7 +154,8 @@ def build_pui_xarray_context( xr.polyval(log_beam_energy, passband_coefficients, degree_dim="degree") ).fillna(0) passband_per_region = passband_per_region / passband_per_region.interp( - elevation_deg=0.0, speed_ratio=1.0, + elevation_deg=0.0, + speed_ratio=1.0, ) azimuth_is_sg = np.abs(azimuth_deg) <= 20 passband_full = xr.where( @@ -219,9 +222,7 @@ def evaluate_pui_sweep_xarray( v_dot_vsw = xr.dot(context.direction, v_sw, dim="cartesian") speed_sw = np.sqrt( np.maximum( - context.speed_grid**2 - + v_sw_speed**2 - - 2 * context.speed_grid * v_dot_vsw, + context.speed_grid**2 + v_sw_speed**2 - 2 * context.speed_grid * v_dot_vsw, Q(0, "km**2/s**2"), ) ) @@ -231,7 +232,9 @@ def evaluate_pui_sweep_xarray( term4 = ( context.density_table.pint.interp( psi=(psi % Q(360, "deg")).magnitude, - r=(heliocentric_distance * w**cooling_index).pint.to("au").pint.dequantify(), + r=(heliocentric_distance * w**cooling_index) + .pint.to("au") + .pint.dequantify(), ) .drop_vars(["psi", "r"]) .fillna(0) @@ -273,7 +276,10 @@ def evaluate_pui_sweep_xarray( from imap_l3_processing.swapi.response.deadtime import deadtime_factor from imap_l3_processing.utils import SpiceKernelTypes, furnish_spice_metakernel from tests.swapi._helpers import load_swapi_response - from tests.test_helpers import get_test_data_path, get_test_instrument_team_data_path + from tests.test_helpers import ( + get_test_data_path, + get_test_instrument_team_data_path, + ) OUTPUT_PATH = get_test_data_path("swapi/pui_count_rate_reference_50sweep.h5") @@ -291,9 +297,7 @@ def evaluate_pui_sweep_xarray( # that window so the spin phase advances across the steps. TOTAL_COARSE_DURATION_S = SWEEP_DURATION_S * N_ESA_STEPS_PER_SWEEP / 72 STEP_DURATION_S = TOTAL_COARSE_DURATION_S / N_ESA_STEPS_PER_SWEEP - END_TIME_UTC = START_TIME_UTC + timedelta( - seconds=SWEEP_DURATION_S * N_SWEEPS + 60 - ) + END_TIME_UTC = START_TIME_UTC + timedelta(seconds=SWEEP_DURATION_S * N_SWEEPS + 60) # Truth values are chosen well inside every fit bound so frozen-value # integration tests are not sensitive to LM termination next to a wall: @@ -397,7 +401,9 @@ def evaluate_pui_sweep_xarray( solar_wind_speed_inertial_kms=SW_SPEED_INERTIAL_KMS, ) - print("Computing proton + alpha Maxwellian shoulder via production forward model...") + print( + "Computing proton + alpha Maxwellian shoulder via production forward model..." + ) voltage_repeated = np.broadcast_to( pui_context.voltages_v, (N_SWEEPS, N_ESA_STEPS_PER_SWEEP) ).ravel() diff --git a/scripts/swapi/report_out_of_range_uncertainties.py b/scripts/swapi/report_out_of_range_uncertainties.py index 6e269790e..fb34edd18 100644 --- a/scripts/swapi/report_out_of_range_uncertainties.py +++ b/scripts/swapi/report_out_of_range_uncertainties.py @@ -51,4 +51,4 @@ def get_all_swapi_l3a(): # report_out_of_range_uncertainties( # r"C:\Users\Petty\Development\imap_L3_processing\data\imap\swapi\l3a\2026\04\imap_swapi_l3a_pui-he_20260419_v001.0033.cdf" -# ) \ No newline at end of file +# ) diff --git a/scripts/swapi/swapi_alpha_sw_speed_temperature_density_demo.py b/scripts/swapi/swapi_alpha_sw_speed_temperature_density_demo.py index 3cf62aa1c..6189f3587 100644 --- a/scripts/swapi/swapi_alpha_sw_speed_temperature_density_demo.py +++ b/scripts/swapi/swapi_alpha_sw_speed_temperature_density_demo.py @@ -10,17 +10,24 @@ from uncertainties.unumpy import uarray, nominal_values, std_devs from imap_l3_processing.swapi.l3a.models import SwapiL2Data -from imap_l3_processing.swapi.l3a.science.calculate_alpha_solar_wind_speed import calculate_alpha_solar_wind_speed, \ - calculate_combined_sweeps, get_alpha_peak_indices -from imap_l3_processing.swapi.l3a.science.calculate_alpha_solar_wind_temperature_and_density import \ - calculate_alpha_solar_wind_temperature_and_density_for_combined_sweeps, AlphaTemperatureDensityCalibrationTable, \ - alpha_count_rate_model +from imap_l3_processing.swapi.l3a.science.calculate_alpha_solar_wind_speed import ( + calculate_alpha_solar_wind_speed, + calculate_combined_sweeps, + get_alpha_peak_indices, +) +from imap_l3_processing.swapi.l3a.science.calculate_alpha_solar_wind_temperature_and_density import ( + calculate_alpha_solar_wind_temperature_and_density_for_combined_sweeps, + AlphaTemperatureDensityCalibrationTable, + alpha_count_rate_model, +) def read_l2_data_from_dat(file_path: str) -> SwapiL2Data: data = np.loadtxt(file_path) data = data.reshape((-1, 72, 8)) - start_time = int(np.datetime64("2010-01-01", "ns") - np.datetime64("2000-01-01T12:00", "ns")) + start_time = int( + np.datetime64("2010-01-01", "ns") - np.datetime64("2000-01-01T12:00", "ns") + ) twelve_seconds_in_nanoseconds = 12_000_000_000 epochs = start_time + twelve_seconds_in_nanoseconds * np.arange(len(data)) @@ -32,20 +39,24 @@ def read_l2_data_from_dat(file_path: str) -> SwapiL2Data: coincident_count_rates = data[..., 7] fake_coincident_count_rate_uncertainties = np.sqrt(6 * coincident_count_rates) - return SwapiL2Data(epochs, - energy_for_first_sweep, - coincident_count_rates, - spin_angles, - fake_coincident_count_rate_uncertainties) + return SwapiL2Data( + epochs, + energy_for_first_sweep, + coincident_count_rates, + spin_angles, + fake_coincident_count_rate_uncertainties, + ) def read_l2_data(cdf_path: str) -> SwapiL2Data: cdf = CDF(cdf_path) - return SwapiL2Data(cdf.raw_var("epoch")[...], - cdf["energy"][...], - cdf["swp_coin_rate"][...], - cdf["spin_angles"][...], - cdf["swp_coin_unc"][...]) + return SwapiL2Data( + cdf.raw_var("epoch")[...], + cdf["energy"][...], + cdf["swp_coin_rate"][...], + cdf["spin_angles"][...], + cdf["swp_coin_unc"][...], + ) fig = plt.figure() @@ -54,28 +65,35 @@ def read_l2_data(cdf_path: str) -> SwapiL2Data: def plot_sweeps(data): for i in range(len(data.sci_start_time)): axes = fig.add_subplot(len(data.sci_start_time) + 1, 1, i + 1) - axes.loglog(data.energy, data.coincidence_count_rate[i, :], marker='.', linestyle="None") + axes.loglog( + data.energy, data.coincidence_count_rate[i, :], marker=".", linestyle="None" + ) axes.set(xlabel="Energy", ylabel="Count Rate") def run_example_dat_files(): data = read_l2_data_from_dat("swapi/test_data/swapi_test_data_v5.dat") - coincident_count_rate = uarray(data.coincidence_count_rate, data.coincidence_count_rate_uncertainty) + coincident_count_rate = uarray( + data.coincidence_count_rate, data.coincidence_count_rate_uncertainty + ) energy = data.energy alpha_speed = calculate_alpha_solar_wind_speed(coincident_count_rate, energy) temperature_density_lut = AlphaTemperatureDensityCalibrationTable.from_file( - r"swapi/test_data/imap_swapi_alpha-density-temperature-lut_20240920_v000.dat") - alpha_temperature, alpha_density = calculate_alpha_solar_wind_temperature_and_density_for_combined_sweeps( - temperature_density_lut, alpha_speed, - coincident_count_rate, energy) + r"swapi/test_data/imap_swapi_alpha-density-temperature-lut_20240920_v000.dat" + ) + alpha_temperature, alpha_density = ( + calculate_alpha_solar_wind_temperature_and_density_for_combined_sweeps( + temperature_density_lut, alpha_speed, coincident_count_rate, energy + ) + ) def plot_and_calculate_alpha_solar_wind_temperature_and_density_for_combined_sweeps( - table: AlphaTemperatureDensityCalibrationTable, - alpha_sw_speed: ufloat, - count_rates: uarray, - energies: ndarray, + table: AlphaTemperatureDensityCalibrationTable, + alpha_sw_speed: ufloat, + count_rates: uarray, + energies: ndarray, ): average_count_rates, energies = calculate_combined_sweeps(count_rates, energies) @@ -85,15 +103,22 @@ def plot_and_calculate_alpha_solar_wind_temperature_and_density_for_combined_swe peak_average_alpha_count_rates = average_count_rates[alpha_particle_peak_slice] initial_parameter_guess = [0.15, 3.6e5, nominal_values(alpha_sw_speed)] - values, covariance = scipy.optimize.curve_fit(alpha_count_rate_model, - peak_energies, - nominal_values(peak_average_alpha_count_rates), - sigma=std_devs(peak_average_alpha_count_rates), - absolute_sigma=True, - bounds=[[0, 0, 0], [np.inf, np.inf, np.inf]], - p0=initial_parameter_guess) - residual = abs(alpha_count_rate_model(peak_energies, *values) - nominal_values(peak_average_alpha_count_rates)) - reduced_chisq = np.sum(np.square(residual / std_devs(peak_average_alpha_count_rates))) / (len(peak_energies) - 3) + values, covariance = scipy.optimize.curve_fit( + alpha_count_rate_model, + peak_energies, + nominal_values(peak_average_alpha_count_rates), + sigma=std_devs(peak_average_alpha_count_rates), + absolute_sigma=True, + bounds=[[0, 0, 0], [np.inf, np.inf, np.inf]], + p0=initial_parameter_guess, + ) + residual = abs( + alpha_count_rate_model(peak_energies, *values) + - nominal_values(peak_average_alpha_count_rates) + ) + reduced_chisq = np.sum( + np.square(residual / std_devs(peak_average_alpha_count_rates)) + ) / (len(peak_energies) - 3) if reduced_chisq > 10: raise ValueError("Failed to fit - chi-squared too large", reduced_chisq) density, temperature, speed = correlated_values(values, covariance) @@ -120,25 +145,33 @@ def main(file_path): else: raise Exception("The demo can only load data from .dat or .cdf files!") - coincident_count_rate = uarray(data.coincidence_count_rate, data.coincidence_count_rate_uncertainty) + coincident_count_rate = uarray( + data.coincidence_count_rate, data.coincidence_count_rate_uncertainty + ) energy = data.energy alpha_speed = calculate_alpha_solar_wind_speed(coincident_count_rate, energy) temperature_density_lut = AlphaTemperatureDensityCalibrationTable.from_file( - r"../../tests/test_data/swapi/imap_swapi_alpha-density-temperature-lut_20240920_v000.dat") - alpha_temperature, alpha_density = plot_and_calculate_alpha_solar_wind_temperature_and_density_for_combined_sweeps( - temperature_density_lut, alpha_speed, - coincident_count_rate, energy) + r"../../tests/test_data/swapi/imap_swapi_alpha-density-temperature-lut_20240920_v000.dat" + ) + alpha_temperature, alpha_density = ( + plot_and_calculate_alpha_solar_wind_temperature_and_density_for_combined_sweeps( + temperature_density_lut, alpha_speed, coincident_count_rate, energy + ) + ) print(f"SW He++ speed: {alpha_speed.n} +/- {alpha_speed.s}") print(f"SW He++ temperature: {alpha_temperature.n} +/- {alpha_temperature.s}") print(f"SW He++ density: {alpha_density.n} +/- {alpha_density.s}") if __name__ == "__main__": - fig.legend([f"SW He++ speed: {alpha_speed :.3f}", - f"SW He++ temperature: {alpha_temperature :.3f}", - f"SW He++ density: {alpha_density :.3f}"], - ) + fig.legend( + [ + f"SW He++ speed: {alpha_speed:.3f}", + f"SW He++ temperature: {alpha_temperature:.3f}", + f"SW He++ density: {alpha_density:.3f}", + ], + ) fig.set_figheight(10) plt.show() diff --git a/scripts/swapi/upload_swapi_l3_dependencies.py b/scripts/swapi/upload_swapi_l3_dependencies.py index 4a7fa0111..3d96b6667 100644 --- a/scripts/swapi/upload_swapi_l3_dependencies.py +++ b/scripts/swapi/upload_swapi_l3_dependencies.py @@ -5,15 +5,21 @@ from tests.test_helpers import get_test_data_path, get_test_instrument_team_data_path -_swapi = get_test_data_path('swapi') +_swapi = get_test_data_path("swapi") files_to_upload = [ _swapi / "imap_swapi_density-of-neutral-helium-lut_20241023_v000.dat", _swapi / "imap_swapi_efficiency-lut_20241020_v000.dat", _swapi / "imap_swapi_energy-gf-pui-lut_20100101_v001.csv", _swapi / "imap_swapi_energy-gf-sw-lut_20100101_v001.csv", - get_test_instrument_team_data_path("swapi/imap_swapi_azimuthal-transmission_20260425_v001.csv"), - get_test_instrument_team_data_path("swapi/imap_swapi_central-effective-area_20260425_v001.csv"), - get_test_instrument_team_data_path("swapi/imap_swapi_passband-fit-coefficients_20260425_v001.csv"), + get_test_instrument_team_data_path( + "swapi/imap_swapi_azimuthal-transmission_20260425_v001.csv" + ), + get_test_instrument_team_data_path( + "swapi/imap_swapi_central-effective-area_20260425_v001.csv" + ), + get_test_instrument_team_data_path( + "swapi/imap_swapi_passband-fit-coefficients_20260425_v001.csv" + ), ] upload_to_local_dir = False for file in files_to_upload: diff --git a/scripts/swapi/view_one_pui_spectrum.py b/scripts/swapi/view_one_pui_spectrum.py index 20c1a4d37..ed12b2e83 100644 --- a/scripts/swapi/view_one_pui_spectrum.py +++ b/scripts/swapi/view_one_pui_spectrum.py @@ -16,6 +16,7 @@ caches. Pass --offline to resolve the L2 file and SPICE kernels from the local DATA_DIR cache (populated by a prior online run) instead of contacting the SDC. """ + import argparse import os import pickle @@ -35,7 +36,10 @@ from imap_processing.swapi.l2 import swapi_l2 from imap_l3_processing.constants import ONE_AU_IN_KM, ONE_SECOND_IN_NANOSECONDS -from imap_l3_processing.swapi.constants import SWAPI_COARSE_SWEEP_BINS, SWAPI_L2_K_FACTOR +from imap_l3_processing.swapi.constants import ( + SWAPI_COARSE_SWEEP_BINS, + SWAPI_L2_K_FACTOR, +) from imap_l3_processing.swapi.descriptors import ( AZIMUTHAL_TRANSMISSION_DESCRIPTOR, CENTRAL_EFFECTIVE_AREA_DESCRIPTOR, @@ -58,17 +62,27 @@ argument_parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter +) argument_parser.add_argument("date", help="UTC date in YYYY-MM-DD format") -argument_parser.add_argument("time", - help="UTC time in HH:MM[:SS] format; picks the 50-sweep " - "chunk whose central epoch is nearest.") -argument_parser.add_argument("--output-path", type=Path, default=None, - help="If set, save the figure to this path and skip plt.show().") -argument_parser.add_argument("--offline", action="store_true", - help="Resolve the L2 file and SPICE kernels from the local " - "DATA_DIR cache instead of querying/downloading from the " - "SDC. Requires a prior online run to have populated the cache.") +argument_parser.add_argument( + "time", + help="UTC time in HH:MM[:SS] format; picks the 50-sweep " + "chunk whose central epoch is nearest.", +) +argument_parser.add_argument( + "--output-path", + type=Path, + default=None, + help="If set, save the figure to this path and skip plt.show().", +) +argument_parser.add_argument( + "--offline", + action="store_true", + help="Resolve the L2 file and SPICE kernels from the local " + "DATA_DIR cache instead of querying/downloading from the " + "SDC. Requires a prior online run to have populated the cache.", +) arguments = argument_parser.parse_args() @@ -80,47 +94,67 @@ def _parse_hh_mm_ss(time_string: str): continue raise SystemExit(f"Could not parse time {time_string!r}; expected HH:MM[:SS].") + if not arguments.offline and "IMAP_API_KEY" not in os.environ: sys.exit("IMAP_API_KEY environment variable is required.") target_date = datetime.strptime(arguments.date, "%Y-%m-%d") -target_datetime_utc = datetime.combine(target_date.date(), _parse_hh_mm_ss(arguments.time)) +target_datetime_utc = datetime.combine( + target_date.date(), _parse_hh_mm_ss(arguments.time) +) compact_date = target_date.strftime("%Y%m%d") fit_cache_path = Path(f"/tmp/swapi_pui_fit_{compact_date}.pkl") spectrogram_cache_path = Path( - f"/tmp/swapi_pui_spectrograms_per_sweep_{compact_date}.pkl") + f"/tmp/swapi_pui_spectrograms_per_sweep_{compact_date}.pkl" +) if not fit_cache_path.exists() or not spectrogram_cache_path.exists(): - sys.exit(f"Missing pickle caches for {arguments.date}. " - f"Run scripts/swapi/fit_and_plot_pui.py {arguments.date} first.") + sys.exit( + f"Missing pickle caches for {arguments.date}. " + f"Run scripts/swapi/fit_and_plot_pui.py {arguments.date} first." + ) with fit_cache_path.open("rb") as cache_file: pickup_ion_data, sw_velocity_rtn_by_chunk_epoch = pickle.load(cache_file) with spectrogram_cache_path.open("rb") as spectrogram_file: - (energies_per_sweep_ev, observed_spectrogram, model_spectrogram, - _sweep_epochs_tt2000) = pickle.load(spectrogram_file) + ( + energies_per_sweep_ev, + observed_spectrogram, + model_spectrogram, + _sweep_epochs_tt2000, + ) = pickle.load(spectrogram_file) imap_data_access.config["DATA_DIR"] = Path("/tmp/swapi_fit_and_plot_data") imap_data_access.config["DATA_DIR"].mkdir(parents=True, exist_ok=True) -ancillary_template_path = (Path(__file__).parent - / "imap_swapi_l3a_proton-sw_dependency_template.json") +ancillary_template_path = ( + Path(__file__).parent / "imap_swapi_l3a_proton-sw_dependency_template.json" +) data_dir = imap_data_access.config["DATA_DIR"] def resolve_l2_filename_from_cache() -> str: - matches = sorted((data_dir / "imap" / "swapi" / "l2").rglob( - f"imap_swapi_l2_sci_{compact_date}_v*.cdf")) + matches = sorted( + (data_dir / "imap" / "swapi" / "l2").rglob( + f"imap_swapi_l2_sci_{compact_date}_v*.cdf" + ) + ) if not matches: - sys.exit(f"No cached SWAPI L2 sci file for {arguments.date} under {data_dir}. " - f"Run once without --offline to populate the cache.") + sys.exit( + f"No cached SWAPI L2 sci file for {arguments.date} under {data_dir}. " + f"Run once without --offline to populate the cache." + ) return matches[-1].name def resolve_spice_filenames_from_cache() -> list[str]: - names = sorted(p.name for p in (data_dir / "imap" / "spice").rglob("*") if p.is_file()) + names = sorted( + p.name for p in (data_dir / "imap" / "spice").rglob("*") if p.is_file() + ) if not names: - sys.exit(f"No cached SPICE kernels under {data_dir}. " - f"Run once without --offline to populate the cache.") + sys.exit( + f"No cached SPICE kernels under {data_dir}. " + f"Run once without --offline to populate the cache." + ) return names @@ -129,32 +163,46 @@ def resolve_spice_filenames_from_cache() -> list[str]: spice_kernel_filenames = resolve_spice_filenames_from_cache() else: science_query_results = imap_data_access.query( - instrument="swapi", data_level="l2", descriptor="sci", - start_date=compact_date, end_date=compact_date, version="latest") + instrument="swapi", + data_level="l2", + descriptor="sci", + start_date=compact_date, + end_date=compact_date, + version="latest", + ) if not science_query_results: sys.exit(f"No SWAPI L2 sci file at the SDC for {arguments.date}") l2_science_filename = os.path.basename(science_query_results[0]["file_path"]) - ephemeris_kernel_types = [SpiceKernelTypes.EphemerisReconstructed, - SpiceKernelTypes.EphemerisPredicted] - non_ephemeris_kernel_types = [t for t in SpiceKernelTypes - if t not in ephemeris_kernel_types] - spice_kernel_filenames = list(dict.fromkeys( - os.path.basename(path) for path in ( - get_spice_kernels_file_names( - target_date - timedelta(days=1), - target_date + timedelta(days=2), - non_ephemeris_kernel_types) - + get_spice_kernels_file_names( - target_date - timedelta(days=90), - target_date + timedelta(days=2), - ephemeris_kernel_types) + ephemeris_kernel_types = [ + SpiceKernelTypes.EphemerisReconstructed, + SpiceKernelTypes.EphemerisPredicted, + ] + non_ephemeris_kernel_types = [ + t for t in SpiceKernelTypes if t not in ephemeris_kernel_types + ] + spice_kernel_filenames = list( + dict.fromkeys( + os.path.basename(path) + for path in ( + get_spice_kernels_file_names( + target_date - timedelta(days=1), + target_date + timedelta(days=2), + non_ephemeris_kernel_types, + ) + + get_spice_kernels_file_names( + target_date - timedelta(days=90), + target_date + timedelta(days=2), + ephemeris_kernel_types, + ) + ) ) - )) + ) dynamic_filenames = [l2_science_filename, *spice_kernel_filenames] processing_input_collection = ProcessingInputCollection( - *[generate_imap_input(filename) for filename in dynamic_filenames]) + *[generate_imap_input(filename) for filename in dynamic_filenames] +) processing_input_collection.deserialize(ancillary_template_path.read_text()) processing_input_collection.download_all_files() @@ -166,31 +214,44 @@ def resolve_spice_filenames_from_cache() -> list[str]: def _swapi_ancillary_path(descriptor: str) -> Path: paths = processing_input_collection.get_file_paths( - source="swapi", descriptor=descriptor) + source="swapi", descriptor=descriptor + ) return imap_data_access.download(paths[0]) pui_xarray_context = build_pui_xarray_context( - azimuthal_transmission_path=_swapi_ancillary_path(AZIMUTHAL_TRANSMISSION_DESCRIPTOR), - central_effective_area_path=_swapi_ancillary_path(CENTRAL_EFFECTIVE_AREA_DESCRIPTOR), - passband_fit_coefficients_path=_swapi_ancillary_path(PASSBAND_FIT_COEFFICIENTS_DESCRIPTOR), - density_of_neutral_helium_lut_path=_swapi_ancillary_path(DENSITY_OF_NEUTRAL_HELIUM_DESCRIPTOR), + azimuthal_transmission_path=_swapi_ancillary_path( + AZIMUTHAL_TRANSMISSION_DESCRIPTOR + ), + central_effective_area_path=_swapi_ancillary_path( + CENTRAL_EFFECTIVE_AREA_DESCRIPTOR + ), + passband_fit_coefficients_path=_swapi_ancillary_path( + PASSBAND_FIT_COEFFICIENTS_DESCRIPTOR + ), + density_of_neutral_helium_lut_path=_swapi_ancillary_path( + DENSITY_OF_NEUTRAL_HELIUM_DESCRIPTOR + ), ) chunks = list(chunk_l2_data(dependencies.data, 50)) -chunk_central_datetimes = np.array([ - cdf_library.tt2000_to_datetime(int(t)) for t in pickup_ion_data.epoch -]) -seconds_from_target = np.array([ - abs((central - target_datetime_utc).total_seconds()) - for central in chunk_central_datetimes -]) +chunk_central_datetimes = np.array( + [cdf_library.tt2000_to_datetime(int(t)) for t in pickup_ion_data.epoch] +) +seconds_from_target = np.array( + [ + abs((central - target_datetime_utc).total_seconds()) + for central in chunk_central_datetimes + ] +) chunk_index = int(np.argmin(seconds_from_target)) data_chunk = chunks[chunk_index] -print(f"Selected chunk {chunk_index} (central epoch " - f"{chunk_central_datetimes[chunk_index].isoformat()}, " - f"{seconds_from_target[chunk_index]:.1f} s from requested " - f"{target_datetime_utc.isoformat()})") +print( + f"Selected chunk {chunk_index} (central epoch " + f"{chunk_central_datetimes[chunk_index].isoformat()}, " + f"{seconds_from_target[chunk_index]:.1f} s from requested " + f"{target_datetime_utc.isoformat()})" +) chunk_epoch_tt2000 = int(pickup_ion_data.epoch[chunk_index]) sw_velocity_rtn_kms = sw_velocity_rtn_by_chunk_epoch.get(chunk_epoch_tt2000) @@ -200,14 +261,17 @@ def _swapi_ancillary_path(descriptor: str) -> Path: sys.exit(f"Chunk {chunk_index} has no successful PUI fit.") chunk_ephemeris_time = spiceypy.unitim( - chunk_epoch_tt2000 / ONE_SECOND_IN_NANOSECONDS, "TT", "ET") + chunk_epoch_tt2000 / ONE_SECOND_IN_NANOSECONDS, "TT", "ET" +) vasyliunas_siscoe = build_vasyliunas_siscoe_distribution( - chunk_ephemeris_time, sw_velocity_rtn_kms, + chunk_ephemeris_time, + sw_velocity_rtn_kms, dependencies.density_of_neutral_helium_calibration_table, dependencies.helium_inflow_vector, ) bulk_sw_per_bin_swapi_kms = rotate_rtn_velocity_to_swapi_per_bin( - data_chunk, sw_velocity_rtn_kms) + data_chunk, sw_velocity_rtn_kms +) # Production averages per-sweep rates computed with the per-bin bulk SW # vector. At high E the f_PUI azimuthal window is narrow (~tens of degrees), @@ -225,15 +289,17 @@ def _swapi_ancillary_path(descriptor: str) -> Path: fit_background_rate_hz = float(pickup_ion_data.background_rate[chunk_index].n) # A rejected background is stored as NaN (see _set_background_to_fill_if_too_high); # adding it would wipe out the model lines, so fall back to no offset. -background_offset_hz = fit_background_rate_hz if np.isfinite(fit_background_rate_hz) else 0.0 +background_offset_hz = ( + fit_background_rate_hz if np.isfinite(fit_background_rate_hz) else 0.0 +) xarray_rate_per_sweep = np.empty(bulk_sw_per_bin_swapi_kms.shape[:2]) n_sweeps_in_chunk = bulk_sw_per_bin_swapi_kms.shape[0] print(f"Evaluating xarray reference per-sweep ({n_sweeps_in_chunk} sweeps)...") for sweep_index_in_chunk in range(n_sweeps_in_chunk): - sweep_bulk_sw_swapi_kms = ( - bulk_sw_per_bin_swapi_kms[sweep_index_in_chunk][ascending_voltage_order] - ) + sweep_bulk_sw_swapi_kms = bulk_sw_per_bin_swapi_kms[sweep_index_in_chunk][ + ascending_voltage_order + ] xarray_rate_per_sweep[sweep_index_in_chunk] = evaluate_pui_sweep_xarray( pui_xarray_context, sweep_bulk_sw_swapi_kms, @@ -253,21 +319,25 @@ def _swapi_ancillary_path(descriptor: str) -> Path: chunk_observed_mean = np.nanmean(chunk_observed_per_sweep, axis=0) chunk_model_mean = np.nanmean(chunk_model_per_sweep, axis=0) -chunk_energies_mean_ev = np.nanmean(chunk_voltages_per_sweep, axis=0) * SWAPI_L2_K_FACTOR +chunk_energies_mean_ev = ( + np.nanmean(chunk_voltages_per_sweep, axis=0) * SWAPI_L2_K_FACTOR +) # Poisson uncertainty on the chunk-mean rate at each bin: the mean of N # independent counting measurements over a livetime each, so total counts # = N · mean_rate · livetime and σ(mean_rate) = sqrt(mean_rate / (N · livetime)). valid_sweeps_per_bin = np.sum(np.isfinite(chunk_observed_per_sweep), axis=0) chunk_observed_uncertainty = np.sqrt( - chunk_observed_mean / (valid_sweeps_per_bin * swapi_l2.SWAPI_LIVETIME)) + chunk_observed_mean / (valid_sweeps_per_bin * swapi_l2.SWAPI_LIVETIME) +) pickup_ion_window_bin_mask = ~np.all(np.isnan(chunk_model_per_sweep), axis=0) xarray_energies_ev = pui_xarray_context.voltages_v * SWAPI_L2_K_FACTOR mean_energies_ev_per_step = np.nanmean( - chunk_voltages_per_sweep * SWAPI_L2_K_FACTOR, axis=0) + chunk_voltages_per_sweep * SWAPI_L2_K_FACTOR, axis=0 +) pickup_ion_bin_indices = np.where(pickup_ion_window_bin_mask)[0] fit_window_energies_ev = mean_energies_ev_per_step[pickup_ion_bin_indices] energy_sort_order_within_window = np.argsort(fit_window_energies_ev) @@ -276,13 +346,14 @@ def _swapi_ancillary_path(descriptor: str) -> Path: observed_spectrogram_sorted = chunk_observed_per_sweep[:, sorted_pickup_ion_bin_indices] model_spectrogram_sorted = chunk_model_per_sweep[:, sorted_pickup_ion_bin_indices] -positive_spectrogram_values = np.concatenate([ - array[np.isfinite(array) & (array > 0)] - for array in (observed_spectrogram_sorted, model_spectrogram_sorted) -]) +positive_spectrogram_values = np.concatenate( + [ + array[np.isfinite(array) & (array > 0)] + for array in (observed_spectrogram_sorted, model_spectrogram_sorted) + ] +) if positive_spectrogram_values.size: - spectrogram_vmin = max( - float(np.percentile(positive_spectrogram_values, 1)), 1e-3) + spectrogram_vmin = max(float(np.percentile(positive_spectrogram_values, 1)), 1e-3) spectrogram_vmax = float(positive_spectrogram_values.max()) else: spectrogram_vmin, spectrogram_vmax = 1e-3, 1.0 @@ -293,17 +364,33 @@ def _swapi_ancillary_path(descriptor: str) -> Path: line_axis = figure.add_subplot(grid_spec[0]) observed_spectrogram_axis = figure.add_subplot(grid_spec[1]) model_spectrogram_axis = figure.add_subplot( - grid_spec[2], sharex=observed_spectrogram_axis, - sharey=observed_spectrogram_axis) - -line_axis.errorbar(chunk_energies_mean_ev, chunk_observed_mean, - yerr=chunk_observed_uncertainty, - fmt=".", color="tab:blue", elinewidth=0.8, capsize=2, - label="Observed (chunk mean)") -line_axis.plot(chunk_energies_mean_ev, chunk_model_mean + background_offset_hz, - "x-", color="tab:orange", label="Model (Optimized) + background") -line_axis.plot(xarray_energies_ev, xarray_rate_per_step + background_offset_hz, - "+-", color="tab:green", label="Model (Reference) + background") + grid_spec[2], sharex=observed_spectrogram_axis, sharey=observed_spectrogram_axis +) + +line_axis.errorbar( + chunk_energies_mean_ev, + chunk_observed_mean, + yerr=chunk_observed_uncertainty, + fmt=".", + color="tab:blue", + elinewidth=0.8, + capsize=2, + label="Observed (chunk mean)", +) +line_axis.plot( + chunk_energies_mean_ev, + chunk_model_mean + background_offset_hz, + "x-", + color="tab:orange", + label="Model (Optimized) + background", +) +line_axis.plot( + xarray_energies_ev, + xarray_rate_per_step + background_offset_hz, + "+-", + color="tab:green", + label="Model (Reference) + background", +) line_axis.set_xscale("log") line_axis.set_yscale("log") line_axis.set_xlabel("Energy / eV") @@ -314,36 +401,52 @@ def _swapi_ancillary_path(descriptor: str) -> Path: f"α={fit_cooling_index:.2f} " f"v_b={fit_cutoff_speed_kms:.0f} km/s " f"β_E={fit_ionization_rate_hz:.2e} 1/s " - + (f"bg={fit_background_rate_hz:.3f} Hz" - if np.isfinite(fit_background_rate_hz) else "bg=nan (rejected)")) + + ( + f"bg={fit_background_rate_hz:.3f} Hz" + if np.isfinite(fit_background_rate_hz) + else "bg=nan (rejected)" + ) +) line_axis.grid(True, which="both", alpha=0.3) line_axis.legend() sweep_indices_in_chunk = np.arange(n_sweeps_in_chunk) for axis_for_spectrogram, spectrogram_values, label in ( - (observed_spectrogram_axis, observed_spectrogram_sorted, - "Observed (50 sweeps × PUI fit window)"), + ( + observed_spectrogram_axis, + observed_spectrogram_sorted, + "Observed (50 sweeps × PUI fit window)", + ), (model_spectrogram_axis, model_spectrogram_sorted, "Model"), ): mesh = axis_for_spectrogram.pcolormesh( - sweep_indices_in_chunk, mean_energies_ev_sorted, + sweep_indices_in_chunk, + mean_energies_ev_sorted, np.ma.masked_invalid(spectrogram_values).T, - shading="nearest", cmap="viridis", norm=spectrogram_norm, + shading="nearest", + cmap="viridis", + norm=spectrogram_norm, rasterized=True, ) axis_for_spectrogram.set_yscale("log") axis_for_spectrogram.set_ylabel("Energy / eV") axis_for_spectrogram.text( - 0.01, 0.95, label, + 0.01, + 0.95, + label, transform=axis_for_spectrogram.transAxes, - ha="left", va="top", fontsize=10, + ha="left", + va="top", + fontsize=10, bbox=dict(boxstyle="round,pad=0.3", fc="white", alpha=0.85, ec="none"), ) plt.setp(observed_spectrogram_axis.get_xticklabels(), visible=False) model_spectrogram_axis.set_xlabel("Sweep index within chunk") figure.colorbar( - mesh, ax=[observed_spectrogram_axis, model_spectrogram_axis], - label="Coincidence rate [Hz]") + mesh, + ax=[observed_spectrogram_axis, model_spectrogram_axis], + label="Coincidence rate [Hz]", +) if arguments.output_path is not None: arguments.output_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/scripts/swe/calculate_rebin_by_pitch_angle.py b/scripts/swe/calculate_rebin_by_pitch_angle.py index 1898e0dc8..e8c84a81a 100644 --- a/scripts/swe/calculate_rebin_by_pitch_angle.py +++ b/scripts/swe/calculate_rebin_by_pitch_angle.py @@ -5,16 +5,27 @@ import spiceypy from imap_data_access import SPICEFilePath -from imap_l3_processing.swe.l3.science.pitch_calculations import calculate_velocity_in_dsp_frame_km_s, \ - swe_rebin_intensity_by_pitch_angle_and_gyrophase, correct_and_rebin +from imap_l3_processing.swe.l3.science.pitch_calculations import ( + calculate_velocity_in_dsp_frame_km_s, + swe_rebin_intensity_by_pitch_angle_and_gyrophase, + correct_and_rebin, +) from imap_l3_processing.swe.l3.swe_l3_dependencies import SweL3Dependencies from imap_l3_processing.swe.swe_processor import SweProcessor -swe_l2_file_path = Path(r'data/imap/swe/l2/2025/12/imap_swe_l2_sci_20251217_v002.cdf') -swe_l1b_file_path = Path(r'data/imap/swe/l1b/2025/12/imap_swe_l1b_sci_20251217_v004.cdf') -mag_file_path = Path(r'data/imap/mag/l1d/2025/12/imap_mag_l1d_norm-dsrf_20251217_v001.cdf') -swapi_file_path = Path(r'data/imap/swapi/l3a/2025/12/imap_swapi_l3a_proton-sw_20251217_v002.cdf') -configuration_file_path = Path(r'data/imap/ancillary/swe/imap_swe_config_20251119_v002.json') +swe_l2_file_path = Path(r"data/imap/swe/l2/2025/12/imap_swe_l2_sci_20251217_v002.cdf") +swe_l1b_file_path = Path( + r"data/imap/swe/l1b/2025/12/imap_swe_l1b_sci_20251217_v004.cdf" +) +mag_file_path = Path( + r"data/imap/mag/l1d/2025/12/imap_mag_l1d_norm-dsrf_20251217_v001.cdf" +) +swapi_file_path = Path( + r"data/imap/swapi/l3a/2025/12/imap_swapi_l3a_proton-sw_20251217_v002.cdf" +) +configuration_file_path = Path( + r"data/imap/ancillary/swe/imap_swe_config_20251119_v002.json" +) spice_files = [ "naif0012.tls", @@ -26,44 +37,82 @@ "imap_pred_od013_20251201_20260112_v01.bsp", "imap_dps_2025_350_2025_351_001.ah.bc", "imap_dps_2025_350_2025_352_001.ah.bc", - "imap_dps_2025_351_2025_353_001.ah.bc" + "imap_dps_2025_351_2025_353_001.ah.bc", ] for spice in spice_files: spiceypy.furnsh(str(SPICEFilePath(spice).construct_path())) -swe_l3_dependencies = SweL3Dependencies.from_file_paths(swe_l2_file_path, swe_l1b_file_path, mag_file_path, - swapi_file_path, configuration_file_path) +swe_l3_dependencies = SweL3Dependencies.from_file_paths( + swe_l2_file_path, + swe_l1b_file_path, + mag_file_path, + swapi_file_path, + configuration_file_path, +) swe_processor = SweProcessor(sentinel.pic, sentinel.input_metadata) -corrected_energy_bins = np.array([[-3.2314079545247543, -2.1864079545247543, -0.7614079545247536, - 1.281092045475246, 4.083592045475245, 7.978592045475245, 13.393592045475241, - 20.898592045475244, 31.396092045475246, 45.978592045475246, - 66.26109204547525, 94.47609204547524, 133.71109204547523, 188.28859204547524, - 264.2410920454752, 369.8335920454752, 516.7510920454753, 721.0960920454753, - 1005.3360920454752, 1400.7735920454752, 1950.7760920454753, - 2715.811092045475, 3780.001092045475, 5260.243592045475]]) +corrected_energy_bins = np.array( + [ + [ + -3.2314079545247543, + -2.1864079545247543, + -0.7614079545247536, + 1.281092045475246, + 4.083592045475245, + 7.978592045475245, + 13.393592045475241, + 20.898592045475244, + 31.396092045475246, + 45.978592045475246, + 66.26109204547525, + 94.47609204547524, + 133.71109204547523, + 188.28859204547524, + 264.2410920454752, + 369.8335920454752, + 516.7510920454753, + 721.0960920454753, + 1005.3360920454752, + 1400.7735920454752, + 1950.7760920454753, + 2715.811092045475, + 3780.001092045475, + 5260.243592045475, + ] + ] +) # swe_processor.calculate_pitch_angle_products(swe_l3_dependencies, corrected_energy_bins) swe_l2_data = swe_l3_dependencies.swe_l2_data -counts = swe_l3_dependencies.swe_l1b_data.count_rates * (swe_l2_data.acquisition_duration[..., np.newaxis] / 1e6) -rebinned_mag_data = np.array([-0.91321456, 0.96044633, 6.76249351])[np.newaxis, np.newaxis, np.newaxis, :] +counts = swe_l3_dependencies.swe_l1b_data.count_rates * ( + swe_l2_data.acquisition_duration[..., np.newaxis] / 1e6 +) +rebinned_mag_data = np.array([-0.91321456, 0.96044633, 6.76249351])[ + np.newaxis, np.newaxis, np.newaxis, : +] def calculate(i): - dsp_velocities = calculate_velocity_in_dsp_frame_km_s(corrected_energy_bins[i], swe_l2_data.inst_el, - swe_l2_data.inst_az_spin_sector[i]) + dsp_velocities = calculate_velocity_in_dsp_frame_km_s( + corrected_energy_bins[i], + swe_l2_data.inst_el, + swe_l2_data.inst_az_spin_sector[i], + ) - rebinned_psd, rebinned_psd_by_pa_and_gyro = correct_and_rebin(swe_l2_data.phase_space_density[i], - np.array([0, 0, -400]), - dsp_velocities, - rebinned_mag_data[i], - swe_l3_dependencies.configuration) + rebinned_psd, rebinned_psd_by_pa_and_gyro = correct_and_rebin( + swe_l2_data.phase_space_density[i], + np.array([0, 0, -400]), + dsp_velocities, + rebinned_mag_data[i], + swe_l3_dependencies.configuration, + ) swe_rebin_intensity_by_pitch_angle_and_gyrophase( swe_l2_data.flux[i], counts[i], dsp_velocities, rebinned_mag_data, - swe_l3_dependencies.configuration) + swe_l3_dependencies.configuration, + ) calculate(0) diff --git a/scripts/swe/compare_outputs.py b/scripts/swe/compare_outputs.py index 1979a4e85..b1251f365 100644 --- a/scripts/swe/compare_outputs.py +++ b/scripts/swe/compare_outputs.py @@ -22,11 +22,13 @@ def canonicalize(thetas, phis): def read_numeric_variable(var: pycdf.Var) -> np.ndarray: assert np.issubdtype(var.dtype, np.number) - return np.where(var[...] == var.attrs['FILLVAL'], np.nan, var[...]) + return np.where(var[...] == var.attrs["FILLVAL"], np.nan, var[...]) -OUTPUT_PATH = Path(__file__).parent.parent.parent / 'comparisons' / 'interpolated_calibration' -HERITAGE_PATH = r'instrument_team_data/swe/swepam-nswe-1999-159.v1-02.hdf' +OUTPUT_PATH = ( + Path(__file__).parent.parent.parent / "comparisons" / "interpolated_calibration" +) +HERITAGE_PATH = r"instrument_team_data/swe/swepam-nswe-1999-159.v1-02.hdf" variable_mapping = { "spacecraft_potential": "potential", @@ -102,21 +104,25 @@ def get_heritage_variable(path, variable_spec): l3_swe_e = l3_vs.attach("swepam_e") match variable_spec: case (heritage_name, index): - heritage_data = np.array([x[l3_swe_e.field(heritage_name)._index][index] for x in l3_swe_e[:]]) + heritage_data = np.array( + [x[l3_swe_e.field(heritage_name)._index][index] for x in l3_swe_e[:]] + ) case heritage_name: - heritage_data = np.array([x[l3_swe_e.field(heritage_name)._index] for x in l3_swe_e[:]]) + heritage_data = np.array( + [x[l3_swe_e.field(heritage_name)._index] for x in l3_swe_e[:]] + ) return heritage_data def plot_heritage_variable(name): heritage_data = get_heritage_variable(HERITAGE_PATH, variable_mapping[name]) - plt.plot(heritage_data, label=f'Heritage: {name}') + plt.plot(heritage_data, label=f"Heritage: {name}") def plot_modern_variable(name): - l3_output_cdf = CDF('data/imap/swe/l3/2025/06/imap_swe_l3_sci_20250629_v000.cdf') + l3_output_cdf = CDF("data/imap/swe/l3/2025/06/imap_swe_l3_sci_20250629_v000.cdf") modern_data = read_numeric_variable(l3_output_cdf[name]) - plt.plot(modern_data, label=f'Modern: {name}') + plt.plot(modern_data, label=f"Modern: {name}") def compare_parallel_perp(): @@ -136,22 +142,25 @@ def compare_parallel_perp(): compare_parallel_perp() for modern_name, heritage_info in variable_mapping.items(): - - l3_hdf = HDF(r'instrument_team_data/swe/swepam-nswe-1999-159.v1-02.hdf') + l3_hdf = HDF(r"instrument_team_data/swe/swepam-nswe-1999-159.v1-02.hdf") l3_vs = l3_hdf.vstart() l3_swe_e = l3_vs.attach("swepam_e") match heritage_info: case (heritage_name, index): - heritage_data = np.array([x[l3_swe_e.field(heritage_name)._index][index] for x in l3_swe_e[:]]) + heritage_data = np.array( + [x[l3_swe_e.field(heritage_name)._index][index] for x in l3_swe_e[:]] + ) case heritage_name: - heritage_data = np.array([x[l3_swe_e.field(heritage_info)._index] for x in l3_swe_e[:]]) - l3_output_cdf = CDF('data/imap/swe/l3/2025/06/imap_swe_l3_sci_20250629_v000.cdf') + heritage_data = np.array( + [x[l3_swe_e.field(heritage_info)._index] for x in l3_swe_e[:]] + ) + l3_output_cdf = CDF("data/imap/swe/l3/2025/06/imap_swe_l3_sci_20250629_v000.cdf") modern_data = read_numeric_variable(l3_output_cdf[modern_name]) if "tensor" in modern_name: for i in range(6): - plt.plot(heritage_data[:, i], label=f'Heritage: {heritage_name} {i}') - plt.plot(modern_data[:, i], label=f'Modern: {modern_name} {i}') + plt.plot(heritage_data[:, i], label=f"Heritage: {heritage_name} {i}") + plt.plot(modern_data[:, i], label=f"Modern: {modern_name} {i}") plt.legend() plt.savefig(OUTPUT_PATH / f"{modern_name}_{i}.png") @@ -164,8 +173,8 @@ def compare_parallel_perp(): continue heritage_data = np.convolve(heritage_data, window_size, mode="same") modern_data = np.convolve(modern_data, window_size, mode="same") - plt.plot(heritage_data, label=f'Heritage: {heritage_name}') - plt.plot(modern_data, label=f'Modern: {modern_name}') + plt.plot(heritage_data, label=f"Heritage: {heritage_name}") + plt.plot(modern_data, label=f"Modern: {modern_name}") plt.legend() plt.savefig(OUTPUT_PATH / f"{modern_name}.png") @@ -180,11 +189,11 @@ def compare_parallel_perp(): def compute_canonicalized_data(population: str): - l3_hdf = HDF(r'instrument_team_data/swe/swepam-nswe-1999-159.v1-02.hdf') + l3_hdf = HDF(r"instrument_team_data/swe/swepam-nswe-1999-159.v1-02.hdf") l3_vs = l3_hdf.vstart() l3_swe_e = l3_vs.attach("swepam_e") - l3_output_cdf = CDF('data/imap/swe/l3/2025/06/imap_swe_l3_sci_20250629_v000.cdf') + l3_output_cdf = CDF("data/imap/swe/l3/2025/06/imap_swe_l3_sci_20250629_v000.cdf") if population == "core": theta_rtn_integrated = "core_temperature_theta_rtn_integrated" @@ -206,26 +215,40 @@ def compute_canonicalized_data(population: str): temp_phi = read_numeric_variable(l3_output_cdf[phi_rtn_integrated]) heritage_theta_rtn_integrated_data = np.array( - [x[l3_swe_e.field(heritage_theta_rtn_integrated)._index] for x in l3_swe_e[:]]) + [x[l3_swe_e.field(heritage_theta_rtn_integrated)._index] for x in l3_swe_e[:]] + ) heritage_phi_rtn_integrated_data = np.array( - [x[l3_swe_e.field(heritage_phi_rtn_integrated)._index] for x in l3_swe_e[:]]) + [x[l3_swe_e.field(heritage_phi_rtn_integrated)._index] for x in l3_swe_e[:]] + ) canonical_theta, canonical_phi = canonicalize(temp_theta, temp_phi) - heritage_canonical_theta, heritage_canonical_phi = canonicalize(heritage_theta_rtn_integrated_data, - heritage_phi_rtn_integrated_data) - - plt.plot(heritage_canonical_theta, '.', label=f'Heritage: {heritage_theta_rtn_integrated}') - plt.plot(canonical_theta, '.', label=f'Modern {theta_rtn_integrated}') + heritage_canonical_theta, heritage_canonical_phi = canonicalize( + heritage_theta_rtn_integrated_data, heritage_phi_rtn_integrated_data + ) + + plt.plot( + heritage_canonical_theta, + ".", + label=f"Heritage: {heritage_theta_rtn_integrated}", + ) + plt.plot(canonical_theta, ".", label=f"Modern {theta_rtn_integrated}") plt.legend() plt.savefig(OUTPUT_PATH / f"{population}_temperature_rtn_theta.png") plt.clf() - plt.plot(heritage_canonical_phi, '.', label=f'Heritage: {heritage_phi_rtn_integrated}') - plt.plot(canonical_phi, '.', label=f'Modern {phi_rtn_integrated}') + plt.plot( + heritage_canonical_phi, ".", label=f"Heritage: {heritage_phi_rtn_integrated}" + ) + plt.plot(canonical_phi, ".", label=f"Modern {phi_rtn_integrated}") plt.legend() plt.savefig(OUTPUT_PATH / f"{population}_temperature_rtn_phi.png") plt.clf() - return canonical_theta, canonical_phi, heritage_theta_rtn_integrated_data, heritage_phi_rtn_integrated_data + return ( + canonical_theta, + canonical_phi, + heritage_theta_rtn_integrated_data, + heritage_phi_rtn_integrated_data, + ) def compare_eigenvector_directions_with_canonicalization(): diff --git a/scripts/swe/create_fake_l2_cdf.py b/scripts/swe/create_fake_l2_cdf.py index c3fa5480e..3bdaf4c66 100644 --- a/scripts/swe/create_fake_l2_cdf.py +++ b/scripts/swe/create_fake_l2_cdf.py @@ -11,68 +11,76 @@ from spacepy.pycdf import CDF -def create_fake_swe_l1b_and_l2_cdf(l1_hdf_path: str, l2_hdf_path: str, output_l1b_swe_cdf_file_path: str, - output_l2_swe_cdf_file_path: str): +def create_fake_swe_l1b_and_l2_cdf( + l1_hdf_path: str, + l2_hdf_path: str, + output_l1b_swe_cdf_file_path: str, + output_l2_swe_cdf_file_path: str, +): l1_hdf = pyhdf.SD.SD(l1_hdf_path) counts = l1_hdf.select("DNSWE_COUNT")[:] total_number_of_energies = 20 spin_spectors = 30 apertures = 7 - counts_in_swe_shape = np.moveaxis(counts, 3, 2).reshape((-1, total_number_of_energies, spin_spectors, apertures)) + counts_in_swe_shape = np.moveaxis(counts, 3, 2).reshape( + (-1, total_number_of_energies, spin_spectors, apertures) + ) l2_swepam_electron_file = HDF(l2_hdf_path) vs_electron = l2_swepam_electron_file.vstart() vd_electron = vs_electron.attach("swepam_e") epochs = get_epochs_from_output_file(vd_electron) - fields = ["chisq_c", - "chisq_h", - "n_fc", - "n_fh", - "n_ic", - "n_ih", - "n_i", - "q_flux_ic", - "q_flux_ih", - "q_flux_i", - "q_flux_phi_ic", - "q_flux_phi_ih", - "q_flux_phi_i", - "q_flux_theta_ic", - "q_flux_theta_ih", - "q_flux_theta_i", - "t_para_fc", - "t_para_fh", - "t_para_ic", - "t_para_ih", - "t_para_i", - "t_perp_fc", - "t_perp_fh", - "t_perp_ic", - "t_perp_ih", - "t_perp_i", - "t_mat_ic", - "t_mat_ih", - "t_mat_i", - "t_phi_fc", - "t_phi_fh", - "t_phi_ic", - "t_phi_ih", - "t_phi_i", - "t_theta_fc", - "t_theta_fh", - "t_theta_ic", - "t_theta_ih", - "t_theta_i", - "v_fc", - "v_fh", - "v_ic", - "v_ih", - "v_i", - "v_rtn_fc", - "v_rtn_fh", - "v_rtn_ic", - "v_rtn_ih", - "v_rtn_i"] + fields = [ + "chisq_c", + "chisq_h", + "n_fc", + "n_fh", + "n_ic", + "n_ih", + "n_i", + "q_flux_ic", + "q_flux_ih", + "q_flux_i", + "q_flux_phi_ic", + "q_flux_phi_ih", + "q_flux_phi_i", + "q_flux_theta_ic", + "q_flux_theta_ih", + "q_flux_theta_i", + "t_para_fc", + "t_para_fh", + "t_para_ic", + "t_para_ih", + "t_para_i", + "t_perp_fc", + "t_perp_fh", + "t_perp_ic", + "t_perp_ih", + "t_perp_i", + "t_mat_ic", + "t_mat_ih", + "t_mat_i", + "t_phi_fc", + "t_phi_fh", + "t_phi_ic", + "t_phi_ih", + "t_phi_i", + "t_theta_fc", + "t_theta_fh", + "t_theta_ic", + "t_theta_ih", + "t_theta_i", + "v_fc", + "v_fh", + "v_ic", + "v_ih", + "v_i", + "v_rtn_fc", + "v_rtn_fh", + "v_rtn_ic", + "v_rtn_ih", + "v_rtn_i", + ] create_expected_cdf(vd_electron, fields, "swe_moments") with CDF(output_l2_swe_cdf_file_path, create=True) as cdf: @@ -92,13 +100,17 @@ def create_fake_swe_l1b_and_l2_cdf(l1_hdf_path: str, l2_hdf_path: str, output_l1 cdf.new("inst_el", [-63, -42, -21, 0, 21, 42, 63], recVary=False) shape_without_aperture_axis = psd.shape[:-1] - placeholder_insta_az_spin_sector = np.random.random(shape_without_aperture_axis) * 360 + placeholder_insta_az_spin_sector = ( + np.random.random(shape_without_aperture_axis) * 360 + ) create_variable(cdf, "inst_az_spin_sector", placeholder_insta_az_spin_sector) minutes = len(counts_in_swe_shape) measurement_count = placeholder_insta_az_spin_sector.size measurement_times = np.linspace(0, minutes * 60, measurement_count) - placeholder_acquisition_time = measurement_times.reshape(shape_without_aperture_axis) + placeholder_acquisition_time = measurement_times.reshape( + shape_without_aperture_axis + ) create_variable(cdf, "acquisition_time", placeholder_acquisition_time) acquisition_duration = np.full((minutes, 20, 30), 80000) @@ -109,14 +121,14 @@ def create_fake_swe_l1b_and_l2_cdf(l1_hdf_path: str, l2_hdf_path: str, output_l1 cdf["settle_duration"] = np.full((minutes, 4), 133333.3333333334) -def create_fake_swapi_l3a_cdf(l2_swepam_ion_file_path: str, - output_swapi_file_path: str): +def create_fake_swapi_l3a_cdf( + l2_swepam_ion_file_path: str, output_swapi_file_path: str +): l2_swepam_ion_file = HDF(l2_swepam_ion_file_path) vs_ion = l2_swepam_ion_file.vstart() vd_ion = vs_ion.attach("swepam_i") - sw_velocity_index = vd_ion.field( - "vel_p_rtn")._index + sw_velocity_index = vd_ion.field("vel_p_rtn")._index sw_velocity_data = np.array([x[sw_velocity_index] for x in vd_ion[:]]) sw_speed = np.linalg.norm(sw_velocity_data, axis=1) @@ -126,10 +138,10 @@ def create_fake_swapi_l3a_cdf(l2_swepam_ion_file_path: str, with CDF(output_swapi_file_path, readonly=False, create=True) as cdf: cdf.compress(pycdf.const.GZIP_COMPRESSION) - cdf['epoch'] = epochs - cdf['epoch_delta'] = epoch_deltas - cdf['proton_sw_speed'] = sw_speed - cdf['proton_sw_speed'].attrs["FILLVAL"] = -1e31 + cdf["epoch"] = epochs + cdf["epoch_delta"] = epoch_deltas + cdf["proton_sw_speed"] = sw_speed + cdf["proton_sw_speed"].attrs["FILLVAL"] = -1e31 ATTITUDE_DATA = """Year DOY Secofday Flag RTN_r RTN_t RTN_n J2GCI_x J2GCI_y J2GCI_z GSE_x GSE_y GSE_z @@ -162,15 +174,18 @@ def create_fake_swapi_l3a_cdf(l2_swepam_ion_file_path: str, 1999 160 3600 0 -0.98089 -0.17395 -0.08711 0.02904 0.94059 0.33831 0.98116 0.18238 -0.06375""" -def create_fake_mag_l1d_cdf(l2_swepam_electron_file_path: str, - output_mag_file_path: str): +def create_fake_mag_l1d_cdf( + l2_swepam_electron_file_path: str, output_mag_file_path: str +): buffer = io.StringIO(ATTITUDE_DATA) data = np.loadtxt(buffer, skiprows=1) def time_to_float(ts): return [(t - datetime(1999, 1, 1)).total_seconds() for t in ts] - times = [datetime(1998, 12, 31) + timedelta(days=row[1], seconds=row[2]) for row in data] + times = [ + datetime(1998, 12, 31) + timedelta(days=row[1], seconds=row[2]) for row in data + ] l2_swepam_electron_file_path = HDF(l2_swepam_electron_file_path) vs_electron = l2_swepam_electron_file_path.vstart() @@ -178,11 +193,17 @@ def time_to_float(ts): mag_epochs = get_epochs_from_output_file(vd_electron) - correction_factor = (datetime(2025, 6, 30) - datetime(1999, 6, 8)) + correction_factor = datetime(2025, 6, 30) - datetime(1999, 6, 8) mag_epochs_uncorrected = mag_epochs - correction_factor - r_att = np.interp(time_to_float(mag_epochs_uncorrected), time_to_float(times), data[:, 4]) - t_att = np.interp(time_to_float(mag_epochs_uncorrected), time_to_float(times), data[:, 5]) - n_att = np.interp(time_to_float(mag_epochs_uncorrected), time_to_float(times), data[:, 6]) + r_att = np.interp( + time_to_float(mag_epochs_uncorrected), time_to_float(times), data[:, 4] + ) + t_att = np.interp( + time_to_float(mag_epochs_uncorrected), time_to_float(times), data[:, 5] + ) + n_att = np.interp( + time_to_float(mag_epochs_uncorrected), time_to_float(times), data[:, 6] + ) rtn_att = np.stack((r_att, t_att, n_att), axis=-1) @@ -212,7 +233,13 @@ def convert_rtn_to_instrument(input_vector, att_vector): c2 = np.sqrt(t2 + n2) c1 = np.sqrt((t2 + n2) * (t2 + n2) + r2 * t2 + r2 * n2) - mat = np.array([[(-n2 - t2) / c1, 0, r / c3], [r * t / c1, -n / c2, t / c3], [r * n / c1, t / c2, n / c3]]) + mat = np.array( + [ + [(-n2 - t2) / c1, 0, r / c3], + [r * t / c1, -n / c2, t / c3], + [r * n / c1, t / c2, n / c3], + ] + ) return np.linalg.inv(mat) @ input_vector @@ -230,14 +257,29 @@ def get_epochs_from_output_file(dataset: VD) -> np.array: min_index = dataset.field("min")._index sec_index = dataset.field("sec")._index - correction_factor = (datetime(2025, 6, 30) - datetime(1999, 6, 8)) + correction_factor = datetime(2025, 6, 30) - datetime(1999, 6, 8) - return np.array([datetime(year=x[years_index], month=x[month_index], day=x[day_index], hour=x[hour_index], - minute=x[min_index], second=x[sec_index]) + correction_factor for x in dataset[:]]) + return np.array( + [ + datetime( + year=x[years_index], + month=x[month_index], + day=x[day_index], + hour=x[hour_index], + minute=x[min_index], + second=x[sec_index], + ) + + correction_factor + for x in dataset[:] + ] + ) def create_expected_cdf(dataset: VD, fields: list[str], file_name): - with pycdf.CDF(str(path.parent.parent.parent / f'temp_cdf_data/expected_{file_name}.cdf'), masterpath="") as file: + with pycdf.CDF( + str(path.parent.parent.parent / f"temp_cdf_data/expected_{file_name}.cdf"), + masterpath="", + ) as file: for field in fields: index = dataset.field(field)._index data = np.array([x[index] for x in dataset[:]]) @@ -246,14 +288,47 @@ def create_expected_cdf(dataset: VD, fields: list[str], file_name): if __name__ == "__main__": path = Path(__file__) - l1_hdf_path = path.parent.parent.parent / "instrument_team_data/swe/ACE_LV1_1999-159.swepam.hdf" - l2_electron_hdf_path = path.parent.parent.parent / "instrument_team_data/swe/swepam-nswe-1999-159.v1-02.hdf" - l2_ion_hdf_path = path.parent.parent.parent / "instrument_team_data/swe/swepam-swi-1999-159.v2-01.hdf" - - l2_swe_cdf_file_path = path.parent.parent.parent / "tests" / "test_data" / "swe" / "imap_swe_l2_sci-with-ace-data_20250630_v002.cdf" - l1b_swe_cdf_file_path = path.parent.parent.parent / "tests" / "test_data" / "swe" / "imap_swe_l1b_sci-with-ace-data_20250630_v002.cdf" - mag_file_path = path.parent.parent.parent / "tests" / "test_data" / "swe" / "imap_mag_l1d_norm-dsrf_20250630_v001.cdf" - swapi_file_path = path.parent.parent.parent / "tests" / "test_data" / "swe" / "imap_swapi_l3a_proton-sw_20250630_v001.cdf" + l1_hdf_path = ( + path.parent.parent.parent + / "instrument_team_data/swe/ACE_LV1_1999-159.swepam.hdf" + ) + l2_electron_hdf_path = ( + path.parent.parent.parent + / "instrument_team_data/swe/swepam-nswe-1999-159.v1-02.hdf" + ) + l2_ion_hdf_path = ( + path.parent.parent.parent + / "instrument_team_data/swe/swepam-swi-1999-159.v2-01.hdf" + ) + + l2_swe_cdf_file_path = ( + path.parent.parent.parent + / "tests" + / "test_data" + / "swe" + / "imap_swe_l2_sci-with-ace-data_20250630_v002.cdf" + ) + l1b_swe_cdf_file_path = ( + path.parent.parent.parent + / "tests" + / "test_data" + / "swe" + / "imap_swe_l1b_sci-with-ace-data_20250630_v002.cdf" + ) + mag_file_path = ( + path.parent.parent.parent + / "tests" + / "test_data" + / "swe" + / "imap_mag_l1d_norm-dsrf_20250630_v001.cdf" + ) + swapi_file_path = ( + path.parent.parent.parent + / "tests" + / "test_data" + / "swe" + / "imap_swapi_l3a_proton-sw_20250630_v001.cdf" + ) mag_file_path.unlink(missing_ok=True) swapi_file_path.unlink(missing_ok=True) diff --git a/scripts/swe/filter_good_times_from_l1b_sdc_cdf.py b/scripts/swe/filter_good_times_from_l1b_sdc_cdf.py index b1a1e5a8f..be6e53c62 100644 --- a/scripts/swe/filter_good_times_from_l1b_sdc_cdf.py +++ b/scripts/swe/filter_good_times_from_l1b_sdc_cdf.py @@ -5,52 +5,90 @@ l1b_cdf = CDF(sys.argv[1]) -with CDF("imap_swe_l1b_sci-3-11-good-times_20100101_v002", masterpath="", readonly=False) as cdf: - cdf["acq_duration"] = np.array([l1b_cdf["acq_duration"][2], l1b_cdf["acq_duration"][4]]) +with CDF( + "imap_swe_l1b_sci-3-11-good-times_20100101_v002", masterpath="", readonly=False +) as cdf: + cdf["acq_duration"] = np.array( + [l1b_cdf["acq_duration"][2], l1b_cdf["acq_duration"][4]] + ) cdf["acq_duration"].attrs = l1b_cdf["acq_duration"].attrs - cdf["acq_start_coarse"] = np.array([l1b_cdf["acq_start_coarse"][2], l1b_cdf["acq_start_coarse"][4]]) + cdf["acq_start_coarse"] = np.array( + [l1b_cdf["acq_start_coarse"][2], l1b_cdf["acq_start_coarse"][4]] + ) cdf["acq_start_coarse"].attrs = l1b_cdf["acq_start_coarse"].attrs - cdf["acq_start_fine"] = np.array([l1b_cdf["acq_start_fine"][2], l1b_cdf["acq_start_fine"][4]]) + cdf["acq_start_fine"] = np.array( + [l1b_cdf["acq_start_fine"][2], l1b_cdf["acq_start_fine"][4]] + ) cdf["acq_start_fine"].attrs = l1b_cdf["acq_start_fine"].attrs - cdf["acquisition_time"] = np.array([l1b_cdf["acquisition_time"][2], l1b_cdf["acquisition_time"][4]]) + cdf["acquisition_time"] = np.array( + [l1b_cdf["acquisition_time"][2], l1b_cdf["acquisition_time"][4]] + ) cdf["acquisition_time"].attrs = l1b_cdf["acquisition_time"].attrs - cdf["cem_nominal_only"] = np.array([l1b_cdf["cem_nominal_only"][2], l1b_cdf["cem_nominal_only"][4]]) + cdf["cem_nominal_only"] = np.array( + [l1b_cdf["cem_nominal_only"][2], l1b_cdf["cem_nominal_only"][4]] + ) cdf["cem_nominal_only"].attrs = l1b_cdf["cem_nominal_only"].attrs cdf["cksum"] = np.array([l1b_cdf["cksum"][2], l1b_cdf["cksum"][4]]) cdf["cksum"].attrs = l1b_cdf["cksum"].attrs cdf["epoch"] = np.array([l1b_cdf["epoch"][2], l1b_cdf["epoch"][4]]) cdf["epoch"].attrs = l1b_cdf["epoch"].attrs - cdf["esa_acq_cfg"] = np.array([l1b_cdf["esa_acq_cfg"][2], l1b_cdf["esa_acq_cfg"][4]]) + cdf["esa_acq_cfg"] = np.array( + [l1b_cdf["esa_acq_cfg"][2], l1b_cdf["esa_acq_cfg"][4]] + ) cdf["esa_acq_cfg"].attrs = l1b_cdf["esa_acq_cfg"].attrs - cdf["esa_table_num"] = np.array([l1b_cdf["esa_table_num"][2], l1b_cdf["esa_table_num"][4]]) + cdf["esa_table_num"] = np.array( + [l1b_cdf["esa_table_num"][2], l1b_cdf["esa_table_num"][4]] + ) cdf["esa_table_num"].attrs = l1b_cdf["esa_table_num"].attrs cdf["high_count"] = np.array([l1b_cdf["high_count"][2], l1b_cdf["high_count"][4]]) cdf["high_count"].attrs = l1b_cdf["high_count"].attrs - cdf["quarter_cycle"] = np.array([l1b_cdf["quarter_cycle"][2], l1b_cdf["quarter_cycle"][4]]) + cdf["quarter_cycle"] = np.array( + [l1b_cdf["quarter_cycle"][2], l1b_cdf["quarter_cycle"][4]] + ) cdf["quarter_cycle"].attrs = l1b_cdf["quarter_cycle"].attrs - cdf["repoint_warning"] = np.array([l1b_cdf["repoint_warning"][2], l1b_cdf["repoint_warning"][4]]) + cdf["repoint_warning"] = np.array( + [l1b_cdf["repoint_warning"][2], l1b_cdf["repoint_warning"][4]] + ) cdf["repoint_warning"].attrs = l1b_cdf["repoint_warning"].attrs - cdf["science_data"] = np.array([l1b_cdf["science_data"][2], l1b_cdf["science_data"][4]]) + cdf["science_data"] = np.array( + [l1b_cdf["science_data"][2], l1b_cdf["science_data"][4]] + ) cdf["science_data"].attrs = l1b_cdf["science_data"].attrs - cdf["settle_duration"] = np.array([l1b_cdf["settle_duration"][2], l1b_cdf["settle_duration"][4]]) + cdf["settle_duration"] = np.array( + [l1b_cdf["settle_duration"][2], l1b_cdf["settle_duration"][4]] + ) cdf["settle_duration"].attrs = l1b_cdf["settle_duration"].attrs cdf["shcoarse"] = np.array([l1b_cdf["shcoarse"][2], l1b_cdf["shcoarse"][4]]) cdf["shcoarse"].attrs = l1b_cdf["shcoarse"].attrs - cdf["spin_period"] = np.array([l1b_cdf["spin_period"][2], l1b_cdf["spin_period"][4]]) + cdf["spin_period"] = np.array( + [l1b_cdf["spin_period"][2], l1b_cdf["spin_period"][4]] + ) cdf["spin_period"].attrs = l1b_cdf["spin_period"].attrs - cdf["spin_period_source"] = np.array([l1b_cdf["spin_period_source"][2], l1b_cdf["spin_period_source"][4]]) + cdf["spin_period_source"] = np.array( + [l1b_cdf["spin_period_source"][2], l1b_cdf["spin_period_source"][4]] + ) cdf["spin_period_source"].attrs = l1b_cdf["spin_period_source"].attrs - cdf["spin_period_validity"] = np.array([l1b_cdf["spin_period_validity"][2], l1b_cdf["spin_period_validity"][4]]) + cdf["spin_period_validity"] = np.array( + [l1b_cdf["spin_period_validity"][2], l1b_cdf["spin_period_validity"][4]] + ) cdf["spin_period_validity"].attrs = l1b_cdf["spin_period_validity"].attrs cdf["spin_phase"] = np.array([l1b_cdf["spin_phase"][2], l1b_cdf["spin_phase"][4]]) cdf["spin_phase"].attrs = l1b_cdf["spin_phase"].attrs - cdf["spin_phase_validity"] = np.array([l1b_cdf["spin_phase_validity"][2], l1b_cdf["spin_phase_validity"][4]]) + cdf["spin_phase_validity"] = np.array( + [l1b_cdf["spin_phase_validity"][2], l1b_cdf["spin_phase_validity"][4]] + ) cdf["spin_phase_validity"].attrs = l1b_cdf["spin_phase_validity"].attrs - cdf["stim_cfg_reg"] = np.array([l1b_cdf["stim_cfg_reg"][2], l1b_cdf["stim_cfg_reg"][4]]) + cdf["stim_cfg_reg"] = np.array( + [l1b_cdf["stim_cfg_reg"][2], l1b_cdf["stim_cfg_reg"][4]] + ) cdf["stim_cfg_reg"].attrs = l1b_cdf["stim_cfg_reg"].attrs - cdf["stim_enabled"] = np.array([l1b_cdf["stim_enabled"][2], l1b_cdf["stim_enabled"][4]]) + cdf["stim_enabled"] = np.array( + [l1b_cdf["stim_enabled"][2], l1b_cdf["stim_enabled"][4]] + ) cdf["stim_enabled"].attrs = l1b_cdf["stim_enabled"].attrs - cdf["threshold_dac"] = np.array([l1b_cdf["threshold_dac"][2], l1b_cdf["threshold_dac"][4]]) + cdf["threshold_dac"] = np.array( + [l1b_cdf["threshold_dac"][2], l1b_cdf["threshold_dac"][4]] + ) cdf["threshold_dac"].attrs = l1b_cdf["threshold_dac"].attrs cdf["cycle"] = l1b_cdf["cycle"] diff --git a/scripts/swe/filter_good_times_from_l2_sdc_cdf.py b/scripts/swe/filter_good_times_from_l2_sdc_cdf.py index 3763ec44b..d3dab9488 100644 --- a/scripts/swe/filter_good_times_from_l2_sdc_cdf.py +++ b/scripts/swe/filter_good_times_from_l2_sdc_cdf.py @@ -8,21 +8,35 @@ with CDF("imap_swe_l2_sci-fake_20240510_v002.cdf", masterpath="", readonly=False) as w: w["epoch"] = np.array([bad_time_cdf["epoch"][2], bad_time_cdf["epoch"][4]]) w["epoch"].attrs = bad_time_cdf["epoch"].attrs - w["acq_duration"] = np.array([bad_time_cdf["acq_duration"][2], bad_time_cdf["acq_duration"][4]]) + w["acq_duration"] = np.array( + [bad_time_cdf["acq_duration"][2], bad_time_cdf["acq_duration"][4]] + ) w["acq_duration"].attrs = bad_time_cdf["acq_duration"].attrs - w["acquisition_time"] = np.array([bad_time_cdf["acquisition_time"][2], bad_time_cdf["acquisition_time"][4]]) + w["acquisition_time"] = np.array( + [bad_time_cdf["acquisition_time"][2], bad_time_cdf["acquisition_time"][4]] + ) w["acquisition_time"].attrs = bad_time_cdf["acquisition_time"].attrs - w["flux_spin_sector"] = np.array([bad_time_cdf["flux_spin_sector"][2], bad_time_cdf["flux_spin_sector"][4]]) + w["flux_spin_sector"] = np.array( + [bad_time_cdf["flux_spin_sector"][2], bad_time_cdf["flux_spin_sector"][4]] + ) w["flux_spin_sector"].attrs = bad_time_cdf["flux_spin_sector"].attrs w["inst_az_spin_sector"] = np.array( - [bad_time_cdf["inst_az_spin_sector"][2], bad_time_cdf["inst_az_spin_sector"][4]]) + [bad_time_cdf["inst_az_spin_sector"][2], bad_time_cdf["inst_az_spin_sector"][4]] + ) w["inst_az_spin_sector"].attrs = bad_time_cdf["inst_az_spin_sector"].attrs w["phase_space_density"] = np.array( - [bad_time_cdf["phase_space_density"][2], bad_time_cdf["phase_space_density"][4]]) + [bad_time_cdf["phase_space_density"][2], bad_time_cdf["phase_space_density"][4]] + ) w["phase_space_density"].attrs = bad_time_cdf["phase_space_density"].attrs w["phase_space_density_spin_sector"] = np.array( - [bad_time_cdf["phase_space_density_spin_sector"][2], bad_time_cdf["phase_space_density_spin_sector"][4]]) - w["phase_space_density_spin_sector"].attrs = bad_time_cdf["phase_space_density_spin_sector"].attrs + [ + bad_time_cdf["phase_space_density_spin_sector"][2], + bad_time_cdf["phase_space_density_spin_sector"][4], + ] + ) + w["phase_space_density_spin_sector"].attrs = bad_time_cdf[ + "phase_space_density_spin_sector" + ].attrs w["flux"] = np.array([bad_time_cdf["flux"][2], bad_time_cdf["flux"][4]]) w["flux"].attrs = bad_time_cdf["flux"].attrs diff --git a/scripts/swe/generate_sci_using_swapi_with_predicted_ephemeris.py b/scripts/swe/generate_sci_using_swapi_with_predicted_ephemeris.py index 8c38e17db..c36812087 100644 --- a/scripts/swe/generate_sci_using_swapi_with_predicted_ephemeris.py +++ b/scripts/swe/generate_sci_using_swapi_with_predicted_ephemeris.py @@ -22,20 +22,37 @@ def generate_swe_for_given_day(day: datetime): - spice_kernel_files = get_spice_kernels_file_names(day, day + timedelta(days=1), SPICE_KERNELS) - day_as_string = day.strftime('%Y%m%d') - l2_file_name = imap_data_access.query(instrument='swe', data_level='l2', start_date=day_as_string, - end_date=day_as_string, version='latest')[0]['file_path'] - - mag_l1d_file_name = imap_data_access.query(instrument='mag', data_level='l1d', start_date=day_as_string, - end_date=day_as_string, version='latest', descriptor='norm-dsrf')[0][ - 'file_path'] - - swapi_l3a_file_name = f'imap_swapi_l3a_proton-sw_{day_as_string}_v003.cdf' - - swe_l1b_file_name = imap_data_access.query(instrument='swe', data_level='l1b', start_date=day_as_string, - end_date=day_as_string, version='latest', descriptor='sci')[0][ - 'file_path'] + spice_kernel_files = get_spice_kernels_file_names( + day, day + timedelta(days=1), SPICE_KERNELS + ) + day_as_string = day.strftime("%Y%m%d") + l2_file_name = imap_data_access.query( + instrument="swe", + data_level="l2", + start_date=day_as_string, + end_date=day_as_string, + version="latest", + )[0]["file_path"] + + mag_l1d_file_name = imap_data_access.query( + instrument="mag", + data_level="l1d", + start_date=day_as_string, + end_date=day_as_string, + version="latest", + descriptor="norm-dsrf", + )[0]["file_path"] + + swapi_l3a_file_name = f"imap_swapi_l3a_proton-sw_{day_as_string}_v003.cdf" + + swe_l1b_file_name = imap_data_access.query( + instrument="swe", + data_level="l1b", + start_date=day_as_string, + end_date=day_as_string, + version="latest", + descriptor="sci", + )[0]["file_path"] swe_ancillary_file_name = "imap_swe_config_20251119_v002.json" @@ -45,13 +62,15 @@ def generate_swe_for_given_day(day: datetime): swe_l1b_file_name, mag_l1d_file_name, swapi_l3a_file_name, - swe_ancillary_file_name + swe_ancillary_file_name, ] - input_collection = ProcessingInputCollection(*[generate_imap_input(i) for i in input_files]) + input_collection = ProcessingInputCollection( + *[generate_imap_input(i) for i in input_files] + ) input_collection.download_all_files() - dependency_file_name = f'imap_swe_l3_sci_{day_as_string}_v002.json' + dependency_file_name = f"imap_swe_l3_sci_{day_as_string}_v002.json" output_filepath = DependencyFilePath(dependency_file_name) fullpath = output_filepath.construct_path() @@ -59,15 +78,17 @@ def generate_swe_for_given_day(day: datetime): fullpath.write_text(input_collection.serialize()) - args_string = (f"imap_l3_data_processor.py " - f"--instrument swe --data-level l3 --descriptor sci " - f"--start-date {day_as_string} --version v004 " - f"--dependency {dependency_file_name}") + args_string = ( + f"imap_l3_data_processor.py " + f"--instrument swe --data-level l3 --descriptor sci " + f"--start-date {day_as_string} --version v004 " + f"--dependency {dependency_file_name}" + ) return subprocess.Popen([sys.executable, *args_string.split(" ")]) -if __name__ == '__main__': +if __name__ == "__main__": if len(sys.argv) > 1: dates = [datetime.strptime(arg, "%Y%m%d") for arg in sys.argv[1:]] else: diff --git a/scripts/swe/plot_swe.py b/scripts/swe/plot_swe.py index c8e977022..37aba7760 100644 --- a/scripts/swe/plot_swe.py +++ b/scripts/swe/plot_swe.py @@ -6,15 +6,9 @@ raw = cdf["phase_space_density"][...] rebinned = np.where(raw == -1e31, np.nan, raw) -geometric_fractions = np.array([ - 0.0781351, - 0.151448, - 0.204686, - 0.181759, - 0.175125, - 0.138312, - 0.0697327 -]) +geometric_fractions = np.array( + [0.0781351, 0.151448, 0.204686, 0.181759, 0.175125, 0.138312, 0.0697327] +) rebinned_ma = np.ma.masked_invalid(rebinned) @@ -24,5 +18,7 @@ (fig, (ax1, ax2)) = plt.subplots(2, 1) ax1.pcolormesh(dist_1d.filled(np.nan).T, norm="log") -ax2.pcolormesh(np.ma.mean(rebinned_by_phi[:, :, 0:4], axis=-1).filled(np.nan).T, norm="log") +ax2.pcolormesh( + np.ma.mean(rebinned_by_phi[:, :, 0:4], axis=-1).filled(np.nan).T, norm="log" +) plt.show() diff --git a/scripts/swe/read_ace.py b/scripts/swe/read_ace.py index 56518d3b1..ced28fe04 100644 --- a/scripts/swe/read_ace.py +++ b/scripts/swe/read_ace.py @@ -5,6 +5,7 @@ @author: hafijulislam """ + import shutil import sys from datetime import datetime @@ -15,6 +16,7 @@ from matplotlib.colors import LogNorm from pyhdf.HDF import * from pyhdf.SD import SD, SDC + # import pyhdf.SD from pyhdf.VS import * from spacepy.pycdf import CDF @@ -69,9 +71,9 @@ def decompressed_counts(cem_count: int) -> int: # the quotient return ( - decompress_table[index]["base"] - + (multi * decompress_table[index]["step_size"]) - + ((decompress_table[index]["step_size"] - 1) // 2) + decompress_table[index]["base"] + + (multi * decompress_table[index]["step_size"]) + + ((decompress_table[index]["step_size"] - 1) // 2) ) @@ -140,13 +142,19 @@ def calculate_phase(phase, spin_period): sss = i // N_ENERGIES_LEVEL1 f1 = N_PHI_LEVEL1 * sss + k + 1 f2 = 120 * sss + N_ENERGIES_LEVEL1 * k + rrr + 1 - offset = (0.0547645 + f1 * 0.0575027 + (f2 - 0.5) * K_FOR_THIS_MODE) * 12.0 / spin_period + offset = ( + (0.0547645 + f1 * 0.0575027 + (f2 - 0.5) * K_FOR_THIS_MODE) + * 12.0 + / spin_period + ) phi[i][k] = phase + offset CONVERT_PHI_TO_IMAP_AZIMUTH = 270 return (phi + CONVERT_PHI_TO_IMAP_AZIMUTH - SWE_MOUNTING_ANGLE) % 360 -def calculate_acquisition_time(phase: float, spin_period: float, epoch: datetime) -> np.ndarray: +def calculate_acquisition_time( + phase: float, spin_period: float, epoch: datetime +) -> np.ndarray: met = get_met_from_epoch(epoch) met_rounded = (met // 15) * 15 @@ -164,22 +172,26 @@ def plot_dnswe_data_from_hdf(file_path): # Open the HDF4 file for reading hdf = SD(file_path, SDC.READ) # Select the DNSWE_COUNT dataset - data = hdf.select('DNSWE_COUNT')[:] + data = hdf.select("DNSWE_COUNT")[:] # Print the shape and data for reference print(f"Shape of DNSWE_COUNT dataset: {data.shape}") print(f"Data:\n{data}") # For simplicity, let's plot the first 2D slice (you can change this depending on your needs) - data_slice = data[:, 0, 0, 0, :] # Select a slice (e.g., first 2D slice from the 5D array) + data_slice = data[ + :, 0, 0, 0, : + ] # Select a slice (e.g., first 2D slice from the 5D array) # Plot the data plt.figure(figsize=(10, 6)) - plt.imshow(data_slice, norm=LogNorm(), aspect='auto', cmap='viridis', origin='lower') - plt.colorbar(label='DNSWE Values') - plt.title('Plot of DNSWE_COUNT dataset slice') - plt.xlabel('Dimension 1') - plt.ylabel('Dimension 2') + plt.imshow( + data_slice, norm=LogNorm(), aspect="auto", cmap="viridis", origin="lower" + ) + plt.colorbar(label="DNSWE Values") + plt.title("Plot of DNSWE_COUNT dataset slice") + plt.xlabel("Dimension 1") + plt.ylabel("Dimension 2") plt.show() @@ -207,11 +219,22 @@ def get_epochs_from_output_file(filepath: str) -> tuple[np.array, np.array]: sc_data = np.array([x[sct_index] for x in dataset[:]]) - correction_factor = (datetime(2025, 6, 30) - datetime(1999, 6, 8)) + correction_factor = datetime(2025, 6, 30) - datetime(1999, 6, 8) - return np.array([datetime(year=x[years_index], month=x[month_index], day=x[day_index], hour=x[hour_index], - minute=x[min_index], second=x[sec_index]) + correction_factor for x in - dataset[:]]), sc_data + return np.array( + [ + datetime( + year=x[years_index], + month=x[month_index], + day=x[day_index], + hour=x[hour_index], + minute=x[min_index], + second=x[sec_index], + ) + + correction_factor + for x in dataset[:] + ] + ), sc_data initial_epoch = datetime.fromisoformat("2025-06-30T12:00:00") @@ -219,24 +242,30 @@ def get_epochs_from_output_file(filepath: str) -> tuple[np.array, np.array]: # File path to the HDF4 file file_path = "instrument_team_data/swe/ACE_LV1_1999-159.swepam.hdf" xarray_data = hdf4_to_xarray(file_path) -new_ds = xarray_data['DNSWE_COUNT'].transpose('fakeDim32', 'fakeDim33', 'fakeDim35', 'fakeDim34', 'fakeDim36') +new_ds = xarray_data["DNSWE_COUNT"].transpose( + "fakeDim32", "fakeDim33", "fakeDim35", "fakeDim34", "fakeDim36" +) new_ds = new_ds.stack(merged_dim=("fakeDim33", "fakeDim35")) new_ds = new_ds.assign_coords({"merged_dim": range(20)}) -new_ds = new_ds.transpose('fakeDim32', 'merged_dim', 'fakeDim34', 'fakeDim36') +new_ds = new_ds.transpose("fakeDim32", "merged_dim", "fakeDim34", "fakeDim36") -epochs, sc_times = get_epochs_from_output_file("instrument_team_data/swe/swepam-nswe-1999-159.v1-02.hdf") +epochs, sc_times = get_epochs_from_output_file( + "instrument_team_data/swe/swepam-nswe-1999-159.v1-02.hdf" +) ds_expanded = xr.DataArray( new_ds.values, dims=("dim1", "dim2", "dim3", "dim4"), coords={ - "dim1": 'epoch', # Keep original coordinates - "dim2": 'energy', # Update dim2 to be 0-23 - "dim3": 'spin', - "dim4": 'cem' - } + "dim1": "epoch", # Keep original coordinates + "dim2": "energy", # Update dim2 to be 0-23 + "dim3": "spin", + "dim4": "cem", + }, +) +decompress_table = decompression_table = np.array( + [decompressed_counts(i) for i in range(256)] ) -decompress_table = decompression_table = np.array([decompressed_counts(i) for i in range(256)]) counts = decompress_table[ds_expanded.values.astype(int)] sample_time = 0.10031 sample_time_microseconds = int(sample_time * 1e6) @@ -245,32 +274,22 @@ def get_epochs_from_output_file(filepath: str) -> tuple[np.array, np.array]: t0 = 55990787 t1 = 56586115 -cal_t0, cal_t1 = np.array([ - [ - 1.359617, - 1.856468, - 1.981586, - 2.068891, - 1.965609, - 1.889478, - 1.105694 - ], +cal_t0, cal_t1 = np.array( [ - 1.290341, - 1.800907, - 1.931721, - 2.060242, - 1.992930, - 1.981554, - 1.117602 + [1.359617, 1.856468, 1.981586, 2.068891, 1.965609, 1.889478, 1.105694], + [1.290341, 1.800907, 1.931721, 2.060242, 1.992930, 1.981554, 1.117602], ] -]) +) -calibration = cal_t0 + ((sc_times - t0) / (t1 - t0))[:, np.newaxis, np.newaxis, np.newaxis] * (cal_t1 - cal_t0) +calibration = cal_t0 + ((sc_times - t0) / (t1 - t0))[ + :, np.newaxis, np.newaxis, np.newaxis +] * (cal_t1 - cal_t0) rates *= calibration -phase, spin = get_phase_and_spin('instrument_team_data/swe/swepam-nswe-1999-159.v1-02.hdf') +phase, spin = get_phase_and_spin( + "instrument_team_data/swe/swepam-nswe-1999-159.v1-02.hdf" +) acquisition_time = [] @@ -283,15 +302,36 @@ def get_epochs_from_output_file(filepath: str) -> tuple[np.array, np.array]: truncate_to = int(sys.argv[1]) else: truncate_to = len(epochs) -settle_duration_needed_to_fill_time_between_points = time_between_data_points / (20 * 30) - sample_time_microseconds +settle_duration_needed_to_fill_time_between_points = ( + time_between_data_points / (20 * 30) - sample_time_microseconds +) output_path = "tests/test_data/swe/imap_swe_l1b_sci_20250630_v003.cdf" shutil.copy("tests/test_data/swe/imap_swe_l1b_sci_20240510_v002.cdf", output_path) -energy = np.array([2.55714286, 3.65142857, 5.16, 7.30571429, - 10.32857143, 14.34285714, 19.95714286, 27.42857143, - 38.37142857, 52.82857143, 73.32857143, 102.0, - 142.14285714, 196.57142857, 272., 372.71428571, - 519.0, 712.57142857, 987.14285714, 1370.0]) +energy = np.array( + [ + 2.55714286, + 3.65142857, + 5.16, + 7.30571429, + 10.32857143, + 14.34285714, + 19.95714286, + 27.42857143, + 38.37142857, + 52.82857143, + 73.32857143, + 102.0, + 142.14285714, + 196.57142857, + 272.0, + 372.71428571, + 519.0, + 712.57142857, + 987.14285714, + 1370.0, + ] +) esa_energy = np.full((truncate_to, 20, 30), np.nan) @@ -299,21 +339,23 @@ def get_epochs_from_output_file(filepath: str) -> tuple[np.array, np.array]: energy = energy[:, np.newaxis, np.newaxis] with CDF(output_path, readonly=False) as cdf: - del cdf['science_data'] - del cdf['acquisition_time'] - del cdf['acq_duration'] - del cdf['settle_duration'] - del cdf['esa_step'] - cdf['epoch'] = epochs[:truncate_to] - cdf['epoch'].attrs['VAR_TYPE'] = 'support_data' - del cdf['epoch'].attrs['DEPEND_0'] - cdf['science_data'] = rates[:truncate_to, :, :, ::-1] - cdf['acquisition_time'] = np.array(acquisition_time[:truncate_to]) - cdf['acq_duration'] = np.full((truncate_to, 20, 30), sample_time_microseconds) - cdf['settle_duration'] = np.full((truncate_to, 4), round(settle_duration_needed_to_fill_time_between_points)) - cdf['esa_table_num'] = np.full((truncate_to, 4), 0) - cdf.new('esa_energy', esa_energy) - cdf.new('esa_step', np.arange(20), recVary=False) - cdf['esa_energy'].attrs['VAR_TYPE'] = 'support_data' - cdf['esa_step'].attrs['VAR_TYPE'] = 'support_data' - cdf['counts_stat_uncert'] = np.full(rates.shape, 1) + del cdf["science_data"] + del cdf["acquisition_time"] + del cdf["acq_duration"] + del cdf["settle_duration"] + del cdf["esa_step"] + cdf["epoch"] = epochs[:truncate_to] + cdf["epoch"].attrs["VAR_TYPE"] = "support_data" + del cdf["epoch"].attrs["DEPEND_0"] + cdf["science_data"] = rates[:truncate_to, :, :, ::-1] + cdf["acquisition_time"] = np.array(acquisition_time[:truncate_to]) + cdf["acq_duration"] = np.full((truncate_to, 20, 30), sample_time_microseconds) + cdf["settle_duration"] = np.full( + (truncate_to, 4), round(settle_duration_needed_to_fill_time_between_points) + ) + cdf["esa_table_num"] = np.full((truncate_to, 4), 0) + cdf.new("esa_energy", esa_energy) + cdf.new("esa_step", np.arange(20), recVary=False) + cdf["esa_energy"].attrs["VAR_TYPE"] = "support_data" + cdf["esa_step"].attrs["VAR_TYPE"] = "support_data" + cdf["counts_stat_uncert"] = np.full(rates.shape, 1) diff --git a/scripts/swe/swe_l2.py b/scripts/swe/swe_l2.py index 1637982ff..ff5ebdf29 100644 --- a/scripts/swe/swe_l2.py +++ b/scripts/swe/swe_l2.py @@ -38,15 +38,16 @@ def fake_cal(): use_l0_data = False # Fake the spin data if use_l0_data: - data_start_time = 4.53051293e+08 + data_start_time = 4.53051293e08 else: data_start_time = 488937480 for spice_file in Path("spice_kernels").iterdir(): spiceypy.furnsh(str(spice_file)) data_end_time = data_start_time + 60 * 1500 -spin_df = imap_processing.tests.conftest.generate_spin_data.__wrapped__()(data_start_time, end_met=data_end_time, - spin_period=15.0) +spin_df = imap_processing.tests.conftest.generate_spin_data.__wrapped__()( + data_start_time, end_met=data_end_time, spin_period=15.0 +) spin_csv_file_path = "spin_data.spin.csv" spin_df.to_csv(spin_csv_file_path, index=False) os.environ["SPIN_DATA_FILEPATH"] = str(spin_csv_file_path) @@ -61,7 +62,8 @@ def fake_cal(): loaded_l1b_dataset = load_cdf(l1b_path) else: loaded_l1b_dataset = load_cdf( - "tests/test_data/swe/imap_swe_l1b_sci_20250630_v003.cdf") # .isel(epoch=slice(None, 20)) + "tests/test_data/swe/imap_swe_l1b_sci_20250630_v003.cdf" + ) # .isel(epoch=slice(None, 20)) l2_dataset = swe_l2(loaded_l1b_dataset) phase_space_density = l2_dataset["phase_space_density_spin_sector"].values diff --git a/scripts/swe/swe_l2_old.py b/scripts/swe/swe_l2_old.py index 326057921..6cb585083 100644 --- a/scripts/swe/swe_l2_old.py +++ b/scripts/swe/swe_l2_old.py @@ -9,6 +9,7 @@ from imap_processing import imap_module_directory from imap_processing.cdf.utils import load_cdf from imap_processing.cdf.utils import write_cdf + # from imap_processing.spice.spin import set_global_spin_table_paths from imap_processing.swe.l1a.swe_l1a import swe_l1a from imap_processing.swe.l1b.swe_l1b import swe_l1b @@ -38,15 +39,17 @@ def fake_cal(): use_l0_data = False # Fake the spin data if use_l0_data: - data_start_time = 4.53051293e+08 + data_start_time = 4.53051293e08 else: data_start_time = 488937480 for spice_file in Path("spice_kernels").iterdir(): spiceypy.furnsh(str(spice_file)) data_end_time = data_start_time + 60 * 1500 -spin_df = imap_processing.tests.conftest.generate_spin_data.__wrapped__()(data_start_time, end_met=data_end_time, - ) # spin_period=15.0) +spin_df = imap_processing.tests.conftest.generate_spin_data.__wrapped__()( + data_start_time, + end_met=data_end_time, +) # spin_period=15.0) spin_csv_file_path = "spin_data.spin.csv" spin_df.to_csv(spin_csv_file_path, index=False) os.environ["SPIN_DATA_FILEPATH"] = str(spin_csv_file_path) @@ -61,7 +64,8 @@ def fake_cal(): loaded_l1b_dataset = load_cdf(l1b_path) else: loaded_l1b_dataset = load_cdf( - "tests/test_data/swe/imap_swe_l1b_sci_20250630_v003.cdf") # .isel(epoch=slice(None, 20)) + "tests/test_data/swe/imap_swe_l1b_sci_20250630_v003.cdf" + ) # .isel(epoch=slice(None, 20)) l2_dataset = swe_l2(loaded_l1b_dataset, "v001") phase_space_density = l2_dataset["phase_space_density_spin_sector"].values diff --git a/scripts/time_shift_cdf.py b/scripts/time_shift_cdf.py index cd5d00981..f39c3825d 100644 --- a/scripts/time_shift_cdf.py +++ b/scripts/time_shift_cdf.py @@ -7,14 +7,17 @@ from spacepy.pycdf import CDF -def convert_epoch_time(filename, target=datetime(2026, 10, 20, 12), - new_science_file_parts: Optional[dict[str, Any]] = None) -> Optional[Path]: +def convert_epoch_time( + filename, + target=datetime(2026, 10, 20, 12), + new_science_file_parts: Optional[dict[str, Any]] = None, +) -> Optional[Path]: file_path = Path(filename) new_science_file_parts = new_science_file_parts or {} with CDF(str(file_path), readonly=False) as cdf: - delta = target - cdf['epoch'][0] - cdf['epoch'][...] += delta + delta = target - cdf["epoch"][0] + cdf["epoch"][...] += delta try: science_file_path = ScienceFilePath(file_path) @@ -27,20 +30,24 @@ def convert_epoch_time(filename, target=datetime(2026, 10, 20, 12), "version": science_file_path.version, "extension": science_file_path.extension, "repointing": science_file_path.repointing, - "cr": science_file_path.cr + "cr": science_file_path.cr, } - new_science_file_path = ScienceFilePath.generate_from_inputs(**{**science_file_args, **new_science_file_parts}) + new_science_file_path = ScienceFilePath.generate_from_inputs( + **{**science_file_args, **new_science_file_parts} + ) new_name = new_science_file_path.construct_path().name new_path = file_path.rename(file_path.parent / new_name) except ImapFilePath.InvalidImapFileError: - print("Did not rename the file because it could not be converted into a science file path!") + print( + "Did not rename the file because it could not be converted into a science file path!" + ) return None return new_path -if __name__ == '__main__': +if __name__ == "__main__": convert_epoch_time(sys.argv[1]) diff --git a/scripts/truncate_cdf.py b/scripts/truncate_cdf.py index 91e50ab18..910364a43 100644 --- a/scripts/truncate_cdf.py +++ b/scripts/truncate_cdf.py @@ -14,7 +14,9 @@ def truncate_cdf(cdf_path: Path, number_of_epochs: int = 2) -> Path: cdf.attrs[attr_name] = list(original_cdf.attrs[attr_name]) for var in original_cdf: - depends_on_epoch = var == "epoch" or original_cdf[var].attrs.get("DEPEND_0") == "epoch" + depends_on_epoch = ( + var == "epoch" or original_cdf[var].attrs.get("DEPEND_0") == "epoch" + ) if depends_on_epoch: cdf.clone(original_cdf[var], var, data=False) cdf[var][:] = original_cdf[var][...][:number_of_epochs] @@ -28,7 +30,9 @@ def truncate_cdf(cdf_path: Path, number_of_epochs: int = 2) -> Path: parser = argparse.ArgumentParser( description="Truncate one or more CDF files to a smaller number of epochs along the epoch axis." ) - parser.add_argument("cdf_paths", nargs="+", type=Path, help="One or more CDF files to truncate.") + parser.add_argument( + "cdf_paths", nargs="+", type=Path, help="One or more CDF files to truncate." + ) parser.add_argument( "-n", "--number-of-epochs", diff --git a/scripts/ultra/create_example_ultra_l1c_pset.py b/scripts/ultra/create_example_ultra_l1c_pset.py index d23d237f6..2ec11a78f 100644 --- a/scripts/ultra/create_example_ultra_l1c_pset.py +++ b/scripts/ultra/create_example_ultra_l1c_pset.py @@ -17,6 +17,7 @@ DEFAULT_RECT_SPACING_DEG_L1C = 0.5 DEFAULT_HEALPIX_NSIDE_L1C = 128 + # TODO: Add ability to mock with/without energy dim to exposure_factor # The Helio frame L1C will have the energy dimension, but the spacecraft frame will not. def create_example_ultra_l1c_pset( @@ -222,9 +223,12 @@ def create_example_ultra_l1c_pset( def _write_ultra_l1c_cdf_with_parents( - out_path: Path = get_run_local_data_path("ultra/fake_l1c_psets/test_pset.cdf"), - date: str = "2025-09-01T00:00:00"): - out_xarray = create_example_ultra_l1c_pset(nside=1, timestr=date, energy_dependent_exposure=True) + out_path: Path = get_run_local_data_path("ultra/fake_l1c_psets/test_pset.cdf"), + date: str = "2025-09-01T00:00:00", +): + out_xarray = create_example_ultra_l1c_pset( + nside=1, timestr=date, energy_dependent_exposure=True + ) out_path.parent.mkdir(parents=True, exist_ok=True) out_path.unlink(missing_ok=True) @@ -235,34 +239,83 @@ def _write_ultra_l1c_cdf_with_parents( cdf.new("observation_time", np.full_like(out_xarray["counts"], 1)) cdf.new("quality_flags", out_xarray["quality_flags"].values) cdf.new("sensitivity", out_xarray["sensitivity"].values) - cdf.new(CoordNames.ELEVATION_L1C.value, out_xarray[CoordNames.ELEVATION_L1C.value].values, recVary=False) - cdf.new(CoordNames.AZIMUTH_L1C.value, out_xarray[CoordNames.AZIMUTH_L1C.value].values, recVary=False) - cdf.new("epoch", out_xarray[CoordNames.TIME.value].values, type=pycdf.const.CDF_TIME_TT2000.value) - cdf.new(CoordNames.ENERGY_ULTRA_L1C.value, out_xarray[CoordNames.ENERGY_ULTRA_L1C.value].values, recVary=False) - cdf.new("energy_bin_delta", np.full_like(out_xarray[CoordNames.ENERGY_ULTRA_L1C.value].values, 1), - recVary=False) - cdf.new(CoordNames.HEALPIX_INDEX.value, out_xarray[CoordNames.HEALPIX_INDEX.value].values, recVary=False) + cdf.new( + CoordNames.ELEVATION_L1C.value, + out_xarray[CoordNames.ELEVATION_L1C.value].values, + recVary=False, + ) + cdf.new( + CoordNames.AZIMUTH_L1C.value, + out_xarray[CoordNames.AZIMUTH_L1C.value].values, + recVary=False, + ) + cdf.new( + "epoch", + out_xarray[CoordNames.TIME.value].values, + type=pycdf.const.CDF_TIME_TT2000.value, + ) + cdf.new( + CoordNames.ENERGY_ULTRA_L1C.value, + out_xarray[CoordNames.ENERGY_ULTRA_L1C.value].values, + recVary=False, + ) + cdf.new( + "energy_bin_delta", + np.full_like(out_xarray[CoordNames.ENERGY_ULTRA_L1C.value].values, 1), + recVary=False, + ) + cdf.new( + CoordNames.HEALPIX_INDEX.value, + out_xarray[CoordNames.HEALPIX_INDEX.value].values, + recVary=False, + ) cdf[CoordNames.ENERGY_ULTRA_L1C.value].attrs["VAR_TYPE"] = "support_data" cdf[CoordNames.HEALPIX_INDEX.value].attrs["VAR_TYPE"] = "support_data" - _add_depends(cdf["counts"], [CoordNames.ENERGY_ULTRA_L1C.value, CoordNames.HEALPIX_INDEX.value], "epoch") - _add_depends(cdf["exposure_factor"], [CoordNames.ENERGY_ULTRA_L1C.value, CoordNames.HEALPIX_INDEX.value], - "epoch") - _add_depends(cdf["sensitivity"], [CoordNames.ENERGY_ULTRA_L1C.value, CoordNames.HEALPIX_INDEX.value], "epoch") - _add_depends(cdf["background_rates"], [CoordNames.ENERGY_ULTRA_L1C.value, CoordNames.HEALPIX_INDEX.value], - "epoch") - _add_depends(cdf["observation_time"], [CoordNames.ENERGY_ULTRA_L1C.value, CoordNames.HEALPIX_INDEX.value], - "epoch") - _add_depends(cdf["quality_flags"], [CoordNames.ENERGY_ULTRA_L1C.value, CoordNames.HEALPIX_INDEX.value], "epoch") + _add_depends( + cdf["counts"], + [CoordNames.ENERGY_ULTRA_L1C.value, CoordNames.HEALPIX_INDEX.value], + "epoch", + ) + _add_depends( + cdf["exposure_factor"], + [CoordNames.ENERGY_ULTRA_L1C.value, CoordNames.HEALPIX_INDEX.value], + "epoch", + ) + _add_depends( + cdf["sensitivity"], + [CoordNames.ENERGY_ULTRA_L1C.value, CoordNames.HEALPIX_INDEX.value], + "epoch", + ) + _add_depends( + cdf["background_rates"], + [CoordNames.ENERGY_ULTRA_L1C.value, CoordNames.HEALPIX_INDEX.value], + "epoch", + ) + _add_depends( + cdf["observation_time"], + [CoordNames.ENERGY_ULTRA_L1C.value, CoordNames.HEALPIX_INDEX.value], + "epoch", + ) + _add_depends( + cdf["quality_flags"], + [CoordNames.ENERGY_ULTRA_L1C.value, CoordNames.HEALPIX_INDEX.value], + "epoch", + ) _add_depends(cdf["energy_bin_delta"], [CoordNames.ENERGY_ULTRA_L1C.value]) for var in cdf: if cdf[var].type() == pycdf.const.CDF_TIME_TT2000.value: - cdf[var].attrs['FILLVAL'] = datetime.datetime.fromisoformat("9999-12-31T23:59:59.999999999") + cdf[var].attrs["FILLVAL"] = datetime.datetime.fromisoformat( + "9999-12-31T23:59:59.999999999" + ) elif cdf[var].type() == pycdf.const.CDF_INT8.value: - cdf[var].attrs['FILLVAL'] = -9223372036854775808 - elif cdf[var].type() == pycdf.const.CDF_FLOAT.value or pycdf.const.CDF_DOUBLE.value: - cdf[var].attrs['FILLVAL'] = -1e31 + cdf[var].attrs["FILLVAL"] = -9223372036854775808 + elif ( + cdf[var].type() == pycdf.const.CDF_FLOAT.value + or pycdf.const.CDF_DOUBLE.value + ): + cdf[var].attrs["FILLVAL"] = -1e31 def _add_depends(var: Var, depends: list[str], depend_0: str = None): diff --git a/scripts/ultra/create_example_ultra_l2_map.py b/scripts/ultra/create_example_ultra_l2_map.py index bccf05208..9c256bee1 100644 --- a/scripts/ultra/create_example_ultra_l2_map.py +++ b/scripts/ultra/create_example_ultra_l2_map.py @@ -17,13 +17,13 @@ def create_example_ultra_l2_map( # noqa: PLR0913 - nside: int = DEFAULT_HEALPIX_NSIDE_L1C, - stripe_center_lat: int = 0, - width_scale: float = 180.0, - counts_scaling_params: tuple[int, float] = (100, 0.01), - peak_exposure: float = 1000.0, - timestr: str = "2025-09-01T00:00:00", - head: str = "45", + nside: int = DEFAULT_HEALPIX_NSIDE_L1C, + stripe_center_lat: int = 0, + width_scale: float = 180.0, + counts_scaling_params: tuple[int, float] = (100, 0.01), + peak_exposure: float = 1000.0, + timestr: str = "2025-09-01T00:00:00", + head: str = "45", ) -> xr.Dataset: """ Mock the L1C PSET product with recognizable but unrealistic counts. @@ -94,7 +94,7 @@ def create_example_ultra_l2_map( # noqa: PLR0913 # Calculate probability based on distance from target latitude lat_diff = np.abs(lat_pix - stripe_center_lat) prob_scaling_factor = counts_scaling_params[1] * np.exp( - -(lat_diff ** 2) / (2 * width_scale ** 2) + -(lat_diff**2) / (2 * width_scale**2) ) # Generate counts using binomial distribution rng = np.random.default_rng(seed=42) @@ -134,7 +134,7 @@ def create_example_ultra_l2_map( # noqa: PLR0913 [ CoordNames.TIME.value, CoordNames.ENERGY_ULTRA_L1C.value, - CoordNames.HEALPIX_INDEX.value + CoordNames.HEALPIX_INDEX.value, ], exposure_time, ), @@ -156,9 +156,7 @@ def create_example_ultra_l2_map( # noqa: PLR0913 ), }, coords={ - CoordNames.TIME.value: [ - tt_j2000ns - ], + CoordNames.TIME.value: [tt_j2000ns], CoordNames.ENERGY_ULTRA_L1C.value: energy_bin_midpoints, CoordNames.HEALPIX_INDEX.value: pix_indices, }, @@ -168,7 +166,7 @@ def create_example_ultra_l2_map( # noqa: PLR0913 f"{timestr[5:7]}{timestr[8:10]}-repointNNNNN_vNNN" ), "Logical_source": f"imap_ultra_l2_{head}sensor-map", - "Data_version": "v001" + "Data_version": "v001", }, ) @@ -179,13 +177,19 @@ def create_example_ultra_l2_map( # noqa: PLR0913 pset_product[CoordNames.ELEVATION_L1C.value].attrs["VAR_TYPE"] = "data" pset_product.coords[CoordNames.TIME.value].attrs["VAR_TYPE"] = "support_data" - pset_product.coords[CoordNames.ENERGY_ULTRA_L1C.value].attrs["VAR_TYPE"] = "support_data" - pset_product.coords[CoordNames.HEALPIX_INDEX.value].attrs["VAR_TYPE"] = "support_data" + pset_product.coords[CoordNames.ENERGY_ULTRA_L1C.value].attrs["VAR_TYPE"] = ( + "support_data" + ) + pset_product.coords[CoordNames.HEALPIX_INDEX.value].attrs["VAR_TYPE"] = ( + "support_data" + ) return pset_product -def _write_ultra_l2_cdf_with_parents(out_path=get_run_local_data_path("ultra/fake_l2_maps/test_l2_map.cdf")): +def _write_ultra_l2_cdf_with_parents( + out_path=get_run_local_data_path("ultra/fake_l2_maps/test_l2_map.cdf"), +): out_xarray = create_example_ultra_l2_map(nside=2) out_path.parent.mkdir(parents=True, exist_ok=True) out_path.unlink(missing_ok=True) @@ -194,32 +198,75 @@ def _write_ultra_l2_cdf_with_parents(out_path=get_run_local_data_path("ultra/fak cdf.new("ena_intensity", out_xarray["counts"].values) cdf.new("exposure_factor", out_xarray["exposure_time"].values) cdf.new("sensitivity", out_xarray["sensitivity"].values) - cdf.new("latitude", out_xarray[CoordNames.ELEVATION_L1C.value].values, recVary=False) - cdf.new("longitude", out_xarray[CoordNames.AZIMUTH_L1C.value].values, recVary=False) - cdf.new("epoch", out_xarray[CoordNames.TIME.value].values, recVary=False, - type=pycdf.const.CDF_TIME_TT2000.value) - cdf.new("energy", out_xarray[CoordNames.ENERGY_ULTRA_L1C.value].values, recVary=False) - cdf.new("pixel_index", out_xarray[CoordNames.HEALPIX_INDEX.value].values, recVary=False) + cdf.new( + "latitude", out_xarray[CoordNames.ELEVATION_L1C.value].values, recVary=False + ) + cdf.new( + "longitude", out_xarray[CoordNames.AZIMUTH_L1C.value].values, recVary=False + ) + cdf.new( + "epoch", + out_xarray[CoordNames.TIME.value].values, + recVary=False, + type=pycdf.const.CDF_TIME_TT2000.value, + ) + cdf.new( + "energy", + out_xarray[CoordNames.ENERGY_ULTRA_L1C.value].values, + recVary=False, + ) + cdf.new( + "pixel_index", + out_xarray[CoordNames.HEALPIX_INDEX.value].values, + recVary=False, + ) cdf.new("epoch_delta", np.array([0])) - cdf.new("energy_delta_plus", np.full_like(out_xarray[CoordNames.ENERGY_ULTRA_L1C.value].values, 1)) - cdf.new("energy_delta_minus", np.full_like(out_xarray[CoordNames.ENERGY_ULTRA_L1C.value].values, 1)) - cdf.new("energy_label", [str(val) for val in out_xarray[CoordNames.ENERGY_ULTRA_L1C.value].values]) - cdf.new("obs_date", np.full(out_xarray["counts"].shape, - spiceypy.unitim(datetime.datetime.now().timestamp(), "ET", "TT") * 1e9), - type=pycdf.const.CDF_TIME_TT2000.value) + cdf.new( + "energy_delta_plus", + np.full_like(out_xarray[CoordNames.ENERGY_ULTRA_L1C.value].values, 1), + ) + cdf.new( + "energy_delta_minus", + np.full_like(out_xarray[CoordNames.ENERGY_ULTRA_L1C.value].values, 1), + ) + cdf.new( + "energy_label", + [str(val) for val in out_xarray[CoordNames.ENERGY_ULTRA_L1C.value].values], + ) + cdf.new( + "obs_date", + np.full( + out_xarray["counts"].shape, + spiceypy.unitim(datetime.datetime.now().timestamp(), "ET", "TT") * 1e9, + ), + type=pycdf.const.CDF_TIME_TT2000.value, + ) cdf.new("obs_date_range", np.full_like(out_xarray["counts"].values, 1)) - cdf.new("solid_angle", np.full_like(out_xarray[CoordNames.HEALPIX_INDEX.value].values, 1)) - cdf.new("ena_intensity_stat_uncert", np.full_like(out_xarray["counts"].values, 1)) + cdf.new( + "solid_angle", + np.full_like(out_xarray[CoordNames.HEALPIX_INDEX.value].values, 1), + ) + cdf.new( + "ena_intensity_stat_uncert", np.full_like(out_xarray["counts"].values, 1) + ) cdf.new("ena_intensity_sys_err", np.full_like(out_xarray["counts"].values, 1)) - cdf.new("pixel_index_label", [str(val) for val in out_xarray[CoordNames.HEALPIX_INDEX.value].values]) + cdf.new( + "pixel_index_label", + [str(val) for val in out_xarray[CoordNames.HEALPIX_INDEX.value].values], + ) for var in cdf: if cdf[var].type() == pycdf.const.CDF_TIME_TT2000.value: - cdf[var].attrs['FILLVAL'] = datetime.datetime.fromisoformat("9999-12-31T23:59:59.999999999") + cdf[var].attrs["FILLVAL"] = datetime.datetime.fromisoformat( + "9999-12-31T23:59:59.999999999" + ) elif cdf[var].type() == pycdf.const.CDF_INT8.value: - cdf[var].attrs['FILLVAL'] = -9223372036854775808 - elif cdf[var].type() == pycdf.const.CDF_FLOAT.value or pycdf.const.CDF_DOUBLE.value: - cdf[var].attrs['FILLVAL'] = -1e31 + cdf[var].attrs["FILLVAL"] = -9223372036854775808 + elif ( + cdf[var].type() == pycdf.const.CDF_FLOAT.value + or pycdf.const.CDF_DOUBLE.value + ): + cdf[var].attrs["FILLVAL"] = -1e31 if __name__ == "__main__": diff --git a/scripts/ultra/create_example_ultra_l2_map_with_power_law.py b/scripts/ultra/create_example_ultra_l2_map_with_power_law.py index 30512098c..48040e639 100644 --- a/scripts/ultra/create_example_ultra_l2_map_with_power_law.py +++ b/scripts/ultra/create_example_ultra_l2_map_with_power_law.py @@ -9,7 +9,9 @@ from tests.test_helpers import get_test_data_path -def _create_example_ultra_l2_map_with_power_law(out_path: Path, parents: list[str] = None): +def _create_example_ultra_l2_map_with_power_law( + out_path: Path, parents: list[str] = None +): out_path.parent.mkdir(parents=True, exist_ok=True) out_path.unlink(missing_ok=True) nside = 2 @@ -27,9 +29,15 @@ def _create_example_ultra_l2_map_with_power_law(out_path: Path, parents: list[st energy_breakpoint = 15 # 15 keV for i in range(num_lon): for j in range(num_lat): - breakpoint_index = np.searchsorted(energies, energy_breakpoint, side="right") - ena_intensities[0, :breakpoint_index, i, j] = power_law_1(energies[:breakpoint_index]) - ena_intensities[0, breakpoint_index:, i, j] = power_law_2(energies[breakpoint_index:]) + breakpoint_index = np.searchsorted( + energies, energy_breakpoint, side="right" + ) + ena_intensities[0, :breakpoint_index, i, j] = power_law_1( + energies[:breakpoint_index] + ) + ena_intensities[0, breakpoint_index:, i, j] = power_law_2( + energies[breakpoint_index:] + ) ena_intensities_delta = np.full_like(ena_intensities, 0.0001) @@ -41,15 +49,21 @@ def _create_example_ultra_l2_map_with_power_law(out_path: Path, parents: list[st cdf.new("latitude_delta", np.full_like(lat, 2), recVary=False) cdf.new("longitude", lon, recVary=False) cdf.new("longitude_delta", np.full_like(lon, 2), recVary=False) - cdf.new("epoch", np.array([datetime.now()]), type=pycdf.const.CDF_TIME_TT2000.value) + cdf.new( + "epoch", np.array([datetime.now()]), type=pycdf.const.CDF_TIME_TT2000.value + ) cdf.new("energy", energies, recVary=False) cdf.new("epoch_delta", np.array([1])) cdf.new("energy_delta_plus", np.full_like(energies, 0.1)) cdf.new("energy_delta_minus", np.full_like(energies, 0.1)) - cdf.new("energy_label", [str(val) for val in energies]), - cdf.new("obs_date", - np.full_like(ena_intensities, (datetime.now() - TT2000_EPOCH).total_seconds() * 1e9), - type=pycdf.const.CDF_TIME_TT2000.value) + (cdf.new("energy_label", [str(val) for val in energies]),) + cdf.new( + "obs_date", + np.full_like( + ena_intensities, (datetime.now() - TT2000_EPOCH).total_seconds() * 1e9 + ), + type=pycdf.const.CDF_TIME_TT2000.value, + ) cdf.new("obs_date_range", np.full_like(ena_intensities, 1)) cdf.new("solid_angle", np.full_like(ena_intensities, 1), recVary=False) cdf.new("ena_intensity_stat_uncert", ena_intensities_delta) @@ -58,16 +72,23 @@ def _create_example_ultra_l2_map_with_power_law(out_path: Path, parents: list[st cdf.new("longitude_label", lon.astype(str), recVary=False) if parents is not None: - cdf.attrs['Parents'] = parents + cdf.attrs["Parents"] = parents for var in cdf: if cdf[var].type() == pycdf.const.CDF_TIME_TT2000.value: - cdf[var].attrs['FILLVAL'] = datetime.fromisoformat("9999-12-31T23:59:59.999999999") + cdf[var].attrs["FILLVAL"] = datetime.fromisoformat( + "9999-12-31T23:59:59.999999999" + ) elif cdf[var].type() == pycdf.const.CDF_INT8.value: - cdf[var].attrs['FILLVAL'] = -9223372036854775808 - elif cdf[var].type() == pycdf.const.CDF_FLOAT.value or pycdf.const.CDF_DOUBLE.value: - cdf[var].attrs['FILLVAL'] = -1e31 + cdf[var].attrs["FILLVAL"] = -9223372036854775808 + elif ( + cdf[var].type() == pycdf.const.CDF_FLOAT.value + or pycdf.const.CDF_DOUBLE.value + ): + cdf[var].attrs["FILLVAL"] = -1e31 if __name__ == "__main__": - _create_example_ultra_l2_map_with_power_law(get_test_data_path('ultra') / 'fake_ultra_map_data.cdf') + _create_example_ultra_l2_map_with_power_law( + get_test_data_path("ultra") / "fake_ultra_map_data.cdf" + ) diff --git a/scripts/ultra/create_example_ultra_l3_survival_probabilities_cdf.py b/scripts/ultra/create_example_ultra_l3_survival_probabilities_cdf.py index 795b005c1..3b468663b 100644 --- a/scripts/ultra/create_example_ultra_l3_survival_probabilities_cdf.py +++ b/scripts/ultra/create_example_ultra_l3_survival_probabilities_cdf.py @@ -7,16 +7,23 @@ from spacepy import pycdf from spacepy.pycdf import CDF -from imap_l3_processing.glows.l3e.glows_l3e_ultra_model import HEALPIX_INDEX_VAR_NAME, PROBABILITY_OF_SURVIVAL_VAR_NAME +from imap_l3_processing.glows.l3e.glows_l3e_ultra_model import ( + HEALPIX_INDEX_VAR_NAME, + PROBABILITY_OF_SURVIVAL_VAR_NAME, +) from tests.test_helpers import get_test_data_path -def create_survival_probabilities_file(glows_file_path: Path, date_for_file: datetime, cdf_file_path: Path): +def create_survival_probabilities_file( + glows_file_path: Path, date_for_file: datetime, cdf_file_path: Path +): with open(glows_file_path) as input_data: - energy_line = [line for line in input_data.readlines() if line.startswith("#energy_grid")] + energy_line = [ + line for line in input_data.readlines() if line.startswith("#energy_grid") + ] assert len(energy_line) == 1 match = [float(x) for x in re.findall(r"\s+([0-9.]+)", energy_line[0])] - energy_units = re.search(r'\[(.+)]', energy_line[0]).group(1) + energy_units = re.search(r"\[(.+)]", energy_line[0]).group(1) energies = np.array(match) @@ -32,7 +39,9 @@ def create_survival_probabilities_file(glows_file_path: Path, date_for_file: dat skips = 0 for i in range(0, 3072): if i in healpix_index: - output_survival_probabilities[:, i] = spin_angle_and_survival_probabilities[i - skips, 3:-1].T + output_survival_probabilities[:, i] = spin_angle_and_survival_probabilities[ + i - skips, 3:-1 + ].T else: skips += 1 output_survival_probabilities[:, i] = 0 @@ -40,16 +49,24 @@ def create_survival_probabilities_file(glows_file_path: Path, date_for_file: dat # Removing last column as it appears to be extra survival_probabilities = spin_angle_and_survival_probabilities[:, 3:-1].T - with CDF(str(cdf_file_path), '') as c: + with CDF(str(cdf_file_path), "") as c: c.new("epoch", [date_for_file], pycdf.const.CDF_TIME_TT2000) c.new("epoch_delta", [12 * 60 * 60 * 1e9], pycdf.const.CDF_INT8) c.new("energy_grid", energies, pycdf.const.CDF_FLOAT, recVary=False) - c['energy_grid'].attrs["UNITS"] = energy_units + c["energy_grid"].attrs["UNITS"] = energy_units c.new("latitude", latitude, pycdf.const.CDF_FLOAT, recVary=False) c.new("longitude", longitude, pycdf.const.CDF_FLOAT, recVary=False) - c.new(HEALPIX_INDEX_VAR_NAME, output_healpix_index, pycdf.const.CDF_INT2, recVary=False) - c.new(PROBABILITY_OF_SURVIVAL_VAR_NAME, np.array(output_survival_probabilities)[np.newaxis, ...], - pycdf.const.CDF_FLOAT) + c.new( + HEALPIX_INDEX_VAR_NAME, + output_healpix_index, + pycdf.const.CDF_INT2, + recVary=False, + ) + c.new( + PROBABILITY_OF_SURVIVAL_VAR_NAME, + np.array(output_survival_probabilities)[np.newaxis, ...], + pycdf.const.CDF_FLOAT, + ) c["energy_grid"].attrs["FILLVAL"] = -1e31 c["latitude"].attrs["FILLVAL"] = -1e31 @@ -62,7 +79,12 @@ def create_survival_probabilities_file(glows_file_path: Path, date_for_file: dat if __name__ == "__main__": path = Path(__file__) - input_file_path = path.parent.parent.parent / "instrument_team_data" / "glows" / "probSur.Imap.Ul.V0_2009.000.dat" + input_file_path = ( + path.parent.parent.parent + / "instrument_team_data" + / "glows" + / "probSur.Imap.Ul.V0_2009.000.dat" + ) start_date = datetime(year=2025, month=4, day=15, hour=12) num_psets_to_generate = 4 @@ -70,7 +92,9 @@ def create_survival_probabilities_file(glows_file_path: Path, date_for_file: dat for i in range(num_psets_to_generate): date_to_set = start_date + timedelta(days=i) filename = f"imap_glows_l3e_survival-probability-ul_{date_to_set.strftime('%Y%m%d')}_v015.cdf" - cdf_file_path = get_test_data_path(f"ultra/fake_l3e_survival_probabilities/{filename}") + cdf_file_path = get_test_data_path( + f"ultra/fake_l3e_survival_probabilities/{filename}" + ) cdf_file_path.unlink(missing_ok=True) create_survival_probabilities_file(input_file_path, date_to_set, cdf_file_path) diff --git a/scripts/ultra/create_glows_l3e_cdf_from_dat.py b/scripts/ultra/create_glows_l3e_cdf_from_dat.py index 0caec3fa2..860d1d254 100644 --- a/scripts/ultra/create_glows_l3e_cdf_from_dat.py +++ b/scripts/ultra/create_glows_l3e_cdf_from_dat.py @@ -11,18 +11,20 @@ version = 0 -glows_dat_dir = Path(r'C:\Users\Harrison\Downloads\timeshifted_glows_data\fake_l3_validation') -glows_dat_files = list(glows_dat_dir.glob('*')) -output_dir = Path(r'C:\Users\Harrison\Downloads\timeshifted_glows_data\cdf') +glows_dat_dir = Path( + r"C:\Users\Harrison\Downloads\timeshifted_glows_data\fake_l3_validation" +) +glows_dat_files = list(glows_dat_dir.glob("*")) +output_dir = Path(r"C:\Users\Harrison\Downloads\timeshifted_glows_data\cdf") output_dir.mkdir(exist_ok=True) for glows_dat in glows_dat_files: - full_date_str = glows_dat.name.split('_')[1] - year_str, decimal_fractional_date_str = full_date_str.split('.') - fractional_year = float(f'0.{decimal_fractional_date_str}') + full_date_str = glows_dat.name.split("_")[1] + year_str, decimal_fractional_date_str = full_date_str.split(".") + fractional_year = float(f"0.{decimal_fractional_date_str}") start_date = datetime(int(year_str), 1, 1) + 365 * timedelta(fractional_year) - repointing = int(glows_dat.name.split('_')[2][:-4]) + repointing = int(glows_dat.name.split("_")[2][:-4]) input_metadata = InputMetadata( instrument="glows", @@ -35,8 +37,8 @@ ) call_args_object = GlowsL3eCallArguments( - formatted_date='', - decimal_date='', + formatted_date="", + decimal_date="", spacecraft_radius=np.float32(0.0), spacecraft_longitude=np.float32(0.0), spacecraft_latitude=np.float32(0.0), @@ -48,10 +50,11 @@ elongation=0, ) - glows_l3e = GlowsL3EUltraData.convert_dat_to_glows_l3e_ul_product(input_metadata, + glows_l3e = GlowsL3EUltraData.convert_dat_to_glows_l3e_ul_product( + input_metadata, glows_dat, start_date, call_args_object, ) - ul_cdf = save_data(glows_l3e, folder_path=output_dir) \ No newline at end of file + ul_cdf = save_data(glows_l3e, folder_path=output_dir) diff --git a/scripts/ultra/create_l1c_l2_and_glows_with_matching_date_range.py b/scripts/ultra/create_l1c_l2_and_glows_with_matching_date_range.py index 32becda3e..77b305a7b 100644 --- a/scripts/ultra/create_l1c_l2_and_glows_with_matching_date_range.py +++ b/scripts/ultra/create_l1c_l2_and_glows_with_matching_date_range.py @@ -3,19 +3,30 @@ import imap_data_access -from scripts.ultra.create_example_ultra_l1c_pset import _write_ultra_l1c_cdf_with_parents -from scripts.ultra.create_example_ultra_l2_map_with_power_law import _create_example_ultra_l2_map_with_power_law -from scripts.ultra.create_example_ultra_l3_survival_probabilities_cdf import create_survival_probabilities_file -from tests.test_helpers import get_test_instrument_team_data_path, get_run_local_data_path - - -def create_l1c_and_glows_with_matching_date_range(start_date: datetime, end_date: datetime): +from scripts.ultra.create_example_ultra_l1c_pset import ( + _write_ultra_l1c_cdf_with_parents, +) +from scripts.ultra.create_example_ultra_l2_map_with_power_law import ( + _create_example_ultra_l2_map_with_power_law, +) +from scripts.ultra.create_example_ultra_l3_survival_probabilities_cdf import ( + create_survival_probabilities_file, +) +from tests.test_helpers import ( + get_test_instrument_team_data_path, + get_run_local_data_path, +) + + +def create_l1c_and_glows_with_matching_date_range( + start_date: datetime, end_date: datetime +): process_date = start_date repoint_counter = 0 l1c_filepaths = [] all_paths = [] - base_path = f'ultra/{start_date:%Y%m%d}-{end_date:%Y%m%d}' + base_path = f"ultra/{start_date:%Y%m%d}-{end_date:%Y%m%d}" shutil.rmtree(get_run_local_data_path(base_path), ignore_errors=True) version_number = 10 @@ -24,26 +35,33 @@ def create_l1c_and_glows_with_matching_date_range(start_date: datetime, end_date repoint_counter += 1 l1c_file_path = get_run_local_data_path( - f'{base_path}/ultra_l1c/imap_ultra_l1c_45sensor-spacecraftpset_{process_date:%Y%m%d}-repoint{repoint_counter:05}_v{version_number:03}.cdf' + f"{base_path}/ultra_l1c/imap_ultra_l1c_45sensor-spacecraftpset_{process_date:%Y%m%d}-repoint{repoint_counter:05}_v{version_number:03}.cdf" ) l1c_file_path.parent.mkdir(parents=True, exist_ok=True) - _write_ultra_l1c_cdf_with_parents(date=process_date.isoformat(), out_path=l1c_file_path) + _write_ultra_l1c_cdf_with_parents( + date=process_date.isoformat(), out_path=l1c_file_path + ) l1c_filepaths.append(str(l1c_file_path)) all_paths.append(str(l1c_file_path)) glows_l3e_file_path = get_run_local_data_path( - f'{base_path}/glows_l3e/imap_glows_l3e_survival-probability-ultra_{process_date:%Y%m%d}_v{version_number:03}.cdf' + f"{base_path}/glows_l3e/imap_glows_l3e_survival-probability-ultra_{process_date:%Y%m%d}_v{version_number:03}.cdf" ) glows_l3e_file_path.parent.mkdir(parents=True, exist_ok=True) - glows_input_path = get_test_instrument_team_data_path('glows/probSur.Imap.Ul.V0_2009.000.dat') - create_survival_probabilities_file(glows_file_path=glows_input_path, date_for_file=process_date, - cdf_file_path=get_run_local_data_path(glows_l3e_file_path)) + glows_input_path = get_test_instrument_team_data_path( + "glows/probSur.Imap.Ul.V0_2009.000.dat" + ) + create_survival_probabilities_file( + glows_file_path=glows_input_path, + date_for_file=process_date, + cdf_file_path=get_run_local_data_path(glows_l3e_file_path), + ) all_paths.append(str(glows_l3e_file_path)) process_date = process_date + timedelta(days=1) l2_out_path = get_run_local_data_path( - f'{base_path}/imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-6mo_{start_date:%Y%m%d}_v{version_number:03}.cdf' + f"{base_path}/imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-6mo_{start_date:%Y%m%d}_v{version_number:03}.cdf" ) l2_out_path.parent.mkdir(parents=True, exist_ok=True) @@ -58,6 +76,8 @@ def create_example_ultra_l2_with_parents(out_path, parents: list[str]): if __name__ == "__main__": - paths = create_l1c_and_glows_with_matching_date_range(datetime(2025, 4, 15, 12), datetime(2025, 4, 19, 12)) + paths = create_l1c_and_glows_with_matching_date_range( + datetime(2025, 4, 15, 12), datetime(2025, 4, 19, 12) + ) for path in paths: imap_data_access.upload(path) diff --git a/scripts/ultra/create_l2_cdf_from_instrument_team_data.py b/scripts/ultra/create_l2_cdf_from_instrument_team_data.py index 3017afd86..bac5cf44a 100644 --- a/scripts/ultra/create_l2_cdf_from_instrument_team_data.py +++ b/scripts/ultra/create_l2_cdf_from_instrument_team_data.py @@ -9,7 +9,11 @@ from spacepy import pycdf from spacepy.pycdf import CDF -from imap_l3_processing.constants import FIVE_MINUTES_IN_NANOSECONDS, ONE_SECOND_IN_NANOSECONDS, SECONDS_PER_DAY +from imap_l3_processing.constants import ( + FIVE_MINUTES_IN_NANOSECONDS, + ONE_SECOND_IN_NANOSECONDS, + SECONDS_PER_DAY, +) parser = argparse.ArgumentParser() parser.add_argument("map_folder") @@ -23,16 +27,18 @@ for sub_folder in folder.iterdir(): for file in sub_folder.iterdir(): energy_string = re.search(r"_(\d*p\d*)keV", file.name).group(1) - energy = float(energy_string.replace('p', '.')) + energy = float(energy_string.replace("p", ".")) data[sub_folder.name][energy] = np.loadtxt(file, delimiter=",").T - assert (data['flux'].keys() == data['sigma'].keys()) - energy = np.array([key for key in sorted(data['flux'].keys())]) - intensity = np.array([data['flux'][key] for key in sorted(data['flux'].keys())]) + assert data["flux"].keys() == data["sigma"].keys() + energy = np.array([key for key in sorted(data["flux"].keys())]) + intensity = np.array([data["flux"][key] for key in sorted(data["flux"].keys())]) ena_intensity = intensity.reshape(1, *intensity.shape) energy_delta_plus = np.full(energy.shape, 1.0) energy_delta_minus = np.full(energy.shape, 1.0) energy_label = energy.astype(str) - ena_intensity_stat_uncert = np.array([data['sigma'][key] for key in sorted(data['sigma'].keys())])[np.newaxis, ...] + ena_intensity_stat_uncert = np.array( + [data["sigma"][key] for key in sorted(data["sigma"].keys())] + )[np.newaxis, ...] ena_intensity_sys_err = ena_intensity_stat_uncert / 2 epoch = np.array([datetime.now()]) @@ -44,40 +50,75 @@ pixel_index_label = [f"Pixel index {x}" for x in pixel_index] obs_date = np.full(ena_intensity.shape, datetime.now()) - obs_date_range = np.full(ena_intensity.shape, ONE_SECOND_IN_NANOSECONDS * SECONDS_PER_DAY * 2) + obs_date_range = np.full( + ena_intensity.shape, ONE_SECOND_IN_NANOSECONDS * SECONDS_PER_DAY * 2 + ) solid_angle = build_solid_angle_map(4) pathname = f"/Users/harrison/Development/imap_L3_processing/tests/test_data/ultra/fake_l2_maps/{folder.name}.cdf" Path(pathname).unlink(missing_ok=True) - with CDF(pathname, '') as cdf: + with CDF(pathname, "") as cdf: cdf.col_major(True) cdf.new("epoch", epoch, type=pycdf.const.CDF_TIME_TT2000) cdf.new("energy", energy, recVary=False, type=pycdf.const.CDF_FLOAT) cdf.new("latitude", lat, recVary=False, type=pycdf.const.CDF_FLOAT) cdf.new("longitude", lon, recVary=False, type=pycdf.const.CDF_FLOAT) - cdf.new("ena_intensity", ena_intensity, recVary=True, type=pycdf.const.CDF_FLOAT) - cdf.new("ena_intensity_stat_uncert", ena_intensity_stat_uncert, recVary=True, type=pycdf.const.CDF_FLOAT) - cdf.new("ena_intensity_sys_err", ena_intensity_sys_err, recVary=True, type=pycdf.const.CDF_FLOAT) - cdf.new("exposure_factor", exposure_factor, recVary=True, type=pycdf.const.CDF_FLOAT) + cdf.new( + "ena_intensity", ena_intensity, recVary=True, type=pycdf.const.CDF_FLOAT + ) + cdf.new( + "ena_intensity_stat_uncert", + ena_intensity_stat_uncert, + recVary=True, + type=pycdf.const.CDF_FLOAT, + ) + cdf.new( + "ena_intensity_sys_err", + ena_intensity_sys_err, + recVary=True, + type=pycdf.const.CDF_FLOAT, + ) + cdf.new( + "exposure_factor", exposure_factor, recVary=True, type=pycdf.const.CDF_FLOAT + ) cdf.new("obs_date", obs_date, recVary=True, type=pycdf.const.CDF_TIME_TT2000) - cdf.new("obs_date_range", obs_date_range, recVary=True, type=pycdf.const.CDF_INT8) + cdf.new( + "obs_date_range", obs_date_range, recVary=True, type=pycdf.const.CDF_INT8 + ) cdf.new("solid_angle", solid_angle, recVary=True, type=pycdf.const.CDF_FLOAT) cdf.new("epoch_delta", epoch_delta, recVary=True, type=pycdf.const.CDF_INT8) - cdf.new("energy_delta_plus", energy_delta_plus, recVary=False, type=pycdf.const.CDF_FLOAT) - cdf.new("energy_delta_minus", energy_delta_minus, recVary=False, type=pycdf.const.CDF_FLOAT) + cdf.new( + "energy_delta_plus", + energy_delta_plus, + recVary=False, + type=pycdf.const.CDF_FLOAT, + ) + cdf.new( + "energy_delta_minus", + energy_delta_minus, + recVary=False, + type=pycdf.const.CDF_FLOAT, + ) cdf.new("energy_label", energy_label, recVary=False, type=pycdf.const.CDF_CHAR) cdf.new("pixel_index", pixel_index, recVary=False, type=pycdf.const.CDF_INT8) - cdf.new("pixel_index_label", pixel_index_label, recVary=False, type=pycdf.const.CDF_CHAR) + cdf.new( + "pixel_index_label", + pixel_index_label, + recVary=False, + type=pycdf.const.CDF_CHAR, + ) for metadata_var_names in ["pixel_index_label", "energy_label"]: cdf[metadata_var_names].attrs["VAR_TYPE"] = "metadata" for var in cdf: if cdf[var].type() == pycdf.const.CDF_TIME_TT2000.value: - cdf[var].attrs['FILLVAL'] = datetime.fromisoformat("9999-12-31T23:59:59.999999999") + cdf[var].attrs["FILLVAL"] = datetime.fromisoformat( + "9999-12-31T23:59:59.999999999" + ) elif cdf[var].type() == pycdf.const.CDF_INT8.value: - cdf[var].attrs['FILLVAL'] = -9223372036854775808 + cdf[var].attrs["FILLVAL"] = -9223372036854775808 elif cdf[var].type() == pycdf.const.CDF_FLOAT.value: - cdf[var].attrs['FILLVAL'] = -1e31 + cdf[var].attrs["FILLVAL"] = -1e31 diff --git a/scripts/ultra/create_l2_dependency_files.py b/scripts/ultra/create_l2_dependency_files.py index afcec421d..413667ba0 100644 --- a/scripts/ultra/create_l2_dependency_files.py +++ b/scripts/ultra/create_l2_dependency_files.py @@ -9,59 +9,69 @@ from imap_l3_processing.utils import SpiceKernelTypes -REFERENCE_FRAMES = ['sf', 'hf'] -POINTING_SET_DESCRIPTORS_FROM_REF_FRAME = { - 'sf':'spacecraftpset', - 'hf':'heliopset' -} -SENSORS = ['u90', 'u45'] +REFERENCE_FRAMES = ["sf", "hf"] +POINTING_SET_DESCRIPTORS_FROM_REF_FRAME = {"sf": "spacecraftpset", "hf": "heliopset"} +SENSORS = ["u90", "u45"] START_DATE = datetime(2025, 9, 29) -END_DATE = datetime(2025,12,29) +END_DATE = datetime(2025, 12, 29) -SENSOR_TO_SENSORNAME = { - 'u90':'90sensor', - 'u45':'45sensor' -} +SENSOR_TO_SENSORNAME = {"u90": "90sensor", "u45": "45sensor"} SPICE_KERNELS = [ SpiceKernelTypes.SpacecraftClock, SpiceKernelTypes.Leapseconds, SpiceKernelTypes.IMAPFrames, SpiceKernelTypes.PointingAttitude, - SpiceKernelTypes.ScienceFrames + SpiceKernelTypes.ScienceFrames, ] + def create_l2_dependency_files(sensor, reference_frame): inputs = ProcessingInputCollection() - pointing_set_descriptor = f'{SENSOR_TO_SENSORNAME[sensor]}-{POINTING_SET_DESCRIPTORS_FROM_REF_FRAME[reference_frame]}' - dependency_file_descriptor = f'{sensor}-{reference_frame}' + pointing_set_descriptor = f"{SENSOR_TO_SENSORNAME[sensor]}-{POINTING_SET_DESCRIPTORS_FROM_REF_FRAME[reference_frame]}" + dependency_file_descriptor = f"{sensor}-{reference_frame}" spice_files = [] for kernel_type in SPICE_KERNELS: response = requests.get( - imap_data_access.config["DATA_ACCESS_URL"] + f"/spice-query?type={kernel_type.value}&start_time=0", + imap_data_access.config["DATA_ACCESS_URL"] + + f"/spice-query?type={kernel_type.value}&start_time=0", ) response.raise_for_status() file_json = response.json() for spice_file in file_json: - spice_start_date = datetime.strptime(spice_file["min_date_datetime"], "%Y-%m-%d, %H:%M:%S") - spice_end_date = datetime.strptime(spice_file["max_date_datetime"], "%Y-%m-%d, %H:%M:%S") + spice_start_date = datetime.strptime( + spice_file["min_date_datetime"], "%Y-%m-%d, %H:%M:%S" + ) + spice_end_date = datetime.strptime( + spice_file["max_date_datetime"], "%Y-%m-%d, %H:%M:%S" + ) if spice_start_date <= END_DATE and START_DATE < spice_end_date: spice_files.append(Path(spice_file["file_name"]).name) query_results = imap_data_access.query( descriptor=pointing_set_descriptor, - instrument='ultra', - data_level='l1c', - start_date=START_DATE.strftime('%Y%m%d'), - end_date=END_DATE.strftime('%Y%m%d'), - version='latest' + instrument="ultra", + data_level="l1c", + start_date=START_DATE.strftime("%Y%m%d"), + end_date=END_DATE.strftime("%Y%m%d"), + version="latest", ) - desired_repointings = [*range(21,28+1), 31, 32, *range(51,57+1), *range(62, 66+1)] + desired_repointings = [ + *range(21, 28 + 1), + 31, + 32, + *range(51, 57 + 1), + *range(62, 66 + 1), + ] - pset_files = [Path(file['file_path']).name for file in query_results if file['repointing'] in desired_repointings] + pset_files = [ + Path(file["file_path"]).name + for file in query_results + if file["repointing"] in desired_repointings + ] - inputs.add([generate_imap_input(f) for f in spice_files+pset_files]) + inputs.add([generate_imap_input(f) for f in spice_files + pset_files]) dependency_file = f"imap_ultra_l2_{dependency_file_descriptor}_{START_DATE.strftime('%Y%m%d')}_v001.json" dependency_path = generate_imap_file_path(dependency_file).construct_path() @@ -71,7 +81,7 @@ def create_l2_dependency_files(sensor, reference_frame): return dependency_file -if __name__ == '__main__': +if __name__ == "__main__": for sensor in SENSORS: for reference_frame in REFERENCE_FRAMES: create_l2_dependency_files(sensor, reference_frame) diff --git a/scripts/ultra/create_l2_from_pointing_sets.py b/scripts/ultra/create_l2_from_pointing_sets.py index b37f8abc0..69ddb3254 100644 --- a/scripts/ultra/create_l2_from_pointing_sets.py +++ b/scripts/ultra/create_l2_from_pointing_sets.py @@ -32,8 +32,11 @@ def create_input_file(sensor: str, start: datetime, end: datetime) -> str: date = start while date < end: repointing = int((date - start_repointings) / timedelta(days=1)) - inputs.add(generate_imap_input( - f"imap_ultra_l1c_{sensor}sensor-heliopset_{date.strftime('%Y%m%d')}-repoint{repointing:05}_v000.cdf")) + inputs.add( + generate_imap_input( + f"imap_ultra_l1c_{sensor}sensor-heliopset_{date.strftime('%Y%m%d')}-repoint{repointing:05}_v000.cdf" + ) + ) date += timedelta(days=1) dependency_file = f"imap_ultra_l2_{sensor}sensor-de-{start.strftime('%Y%m%d')}-{end.strftime('%Y%m%d')}_{start.strftime('%Y%m%d')}_v001.json" dependency_path = generate_imap_file_path(dependency_file).construct_path() @@ -43,7 +46,7 @@ def create_input_file(sensor: str, start: datetime, end: datetime) -> str: def generate_map(descriptor: str, start: datetime, end: datetime): - sensor = descriptor.split('-')[0][1:] + sensor = descriptor.split("-")[0][1:] dependency_file = create_input_file(sensor, start, end) env = { @@ -53,7 +56,7 @@ def generate_map(descriptor: str, start: datetime, end: datetime): } version = "v000" - formatted_start_date = start_date.strftime('%Y%m%d') + formatted_start_date = start_date.strftime("%Y%m%d") output_filename = f"imap_ultra_l2_{descriptor}_{formatted_start_date}_{version}.cdf" command = f"imap_cli --instrument ultra --data-level l2 --descriptor {descriptor} --start-date {formatted_start_date} --version {version} --dependency {dependency_file}" print(command) @@ -69,7 +72,7 @@ def generate_map(descriptor: str, start: datetime, end: datetime): start_date = datetime(2025, 4, 16) end_date = datetime(2025, 10, 15) - duration = '6mo' + duration = "6mo" descriptor = "u90-ena-h-hf-nsp-full-hae-2deg-6mo" generate_map(descriptor, start_date, end_date) # for sensor in [45, 90]: diff --git a/scripts/ultra/plot_l2_uncertainties.py b/scripts/ultra/plot_l2_uncertainties.py index 3f27cf2ec..2a8631652 100644 --- a/scripts/ultra/plot_l2_uncertainties.py +++ b/scripts/ultra/plot_l2_uncertainties.py @@ -12,12 +12,12 @@ ultra_imap_data_dir = get_run_local_data_path("ultra/integration_data") ultra_path_90 = Path( - r"C:\Users\Harrison\Downloads\ultra_combined_hf_validation\ULTRA-90sensor_helio-psets_20251201") + r"C:\Users\Harrison\Downloads\ultra_combined_hf_validation\ULTRA-90sensor_helio-psets_20251201" +) ultra_path_45 = Path( r"C:\Users\Harrison\Downloads\ultra_combined_hf_validation\ULTRA-45sensor_helio-psets_20251201" ) -glows_path = Path( - r"C:\Users\Harrison\Downloads\ultra_combined_hf_validation\glows") +glows_path = Path(r"C:\Users\Harrison\Downloads\ultra_combined_hf_validation\glows") ancil_files = [ INTEGRATION_TEST_DATA_PATH / "spice" / "naif020.tls", @@ -30,24 +30,38 @@ u90_paths = [Path(f) for f in list(ultra_path_90.glob("*.cdf"))] glows_paths = [Path(f) for f in list(glows_path.glob("*.cdf"))] l2_u90 = Path( - r"C:\Users\Harrison\Downloads\ultra_combined_hf_validation\imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-2deg-6mo_20250416_v000.cdf") + r"C:\Users\Harrison\Downloads\ultra_combined_hf_validation\imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-2deg-6mo_20250416_v000.cdf" +) l2_u45 = Path( - r"C:\Users\Harrison\Downloads\ultra_combined_hf_validation\imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-2deg-6mo_20250416_v000.cdf") + r"C:\Users\Harrison\Downloads\ultra_combined_hf_validation\imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-2deg-6mo_20250416_v000.cdf" +) start_date = datetime(2025, 4, 16) descriptor = "ulc-ena-h-hf-nsp-full-hae-2deg-6mo" for f in ancil_files: spiceypy.furnsh(str(f)) -ultra_nsp_combined_dep = UltraL3CombinedDependencies.from_file_paths(u45_pset_paths=u45_paths, u90_pset_paths=u90_paths, - glows_l3e_pset_paths=glows_paths, - u45_map_path=l2_u45, u90_map_path=l2_u90, - energy_bin_group_sizes_path=get_test_data_path( - "ultra/imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv")) -print(ultra_nsp_combined_dep.u45_l2_map.intensity_map_data.ena_intensity_stat_uncert.shape) +ultra_nsp_combined_dep = UltraL3CombinedDependencies.from_file_paths( + u45_pset_paths=u45_paths, + u90_pset_paths=u90_paths, + glows_l3e_pset_paths=glows_paths, + u45_map_path=l2_u45, + u90_map_path=l2_u90, + energy_bin_group_sizes_path=get_test_data_path( + "ultra/imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv" + ), +) +print( + ultra_nsp_combined_dep.u45_l2_map.intensity_map_data.ena_intensity_stat_uncert.shape +) hp32 = HEALPix(32) lon, lat = hp32.healpix_to_lonlat(np.arange(hp32.npix)) -plt.scatter(lon, lat, - c=ultra_nsp_combined_dep.u45_l2_map.intensity_map_data.ena_intensity_stat_uncert[0].reshape(12, -1)[0]) +plt.scatter( + lon, + lat, + c=ultra_nsp_combined_dep.u45_l2_map.intensity_map_data.ena_intensity_stat_uncert[ + 0 + ].reshape(12, -1)[0], +) plt.show() diff --git a/scripts/ultra/validation/create_L2_validation_maps.py b/scripts/ultra/validation/create_L2_validation_maps.py index bb5415152..edea0e1a0 100644 --- a/scripts/ultra/validation/create_L2_validation_maps.py +++ b/scripts/ultra/validation/create_L2_validation_maps.py @@ -6,20 +6,21 @@ from imap_processing.cli import Ultra L2_DESCRIPTORS = [ - 'u90-ena-h-hf-nsp-full-hae-2deg-1mo', - 'u90-ena-h-sf-nsp-full-hae-2deg-1mo', - 'u45-ena-h-hf-nsp-full-hae-2deg-1mo', - 'u45-ena-h-sf-nsp-full-hae-2deg-1mo', - 'u90-ena-h-hf-nsp-full-hae-4deg-1mo', - 'u90-ena-h-sf-nsp-full-hae-4deg-1mo', - 'u45-ena-h-hf-nsp-full-hae-4deg-1mo', - 'u45-ena-h-sf-nsp-full-hae-4deg-1mo', - 'u90-ena-h-hf-nsp-full-hae-6deg-1mo', - 'u90-ena-h-sf-nsp-full-hae-6deg-1mo', - 'u45-ena-h-hf-nsp-full-hae-6deg-1mo', - 'u45-ena-h-sf-nsp-full-hae-6deg-1mo', + "u90-ena-h-hf-nsp-full-hae-2deg-1mo", + "u90-ena-h-sf-nsp-full-hae-2deg-1mo", + "u45-ena-h-hf-nsp-full-hae-2deg-1mo", + "u45-ena-h-sf-nsp-full-hae-2deg-1mo", + "u90-ena-h-hf-nsp-full-hae-4deg-1mo", + "u90-ena-h-sf-nsp-full-hae-4deg-1mo", + "u45-ena-h-hf-nsp-full-hae-4deg-1mo", + "u45-ena-h-sf-nsp-full-hae-4deg-1mo", + "u90-ena-h-hf-nsp-full-hae-6deg-1mo", + "u90-ena-h-sf-nsp-full-hae-6deg-1mo", + "u45-ena-h-hf-nsp-full-hae-6deg-1mo", + "u45-ena-h-sf-nsp-full-hae-6deg-1mo", ] + def generate_l2_files(): groups = defaultdict(list) for descriptor in L2_DESCRIPTORS: @@ -28,20 +29,26 @@ def generate_l2_files(): for group, descriptors in groups.items(): pool.submit(make_maps, descriptors) + def make_maps(descriptors): print("making", descriptors) for descriptor in descriptors: - sensor, _, _, frame, *_ = descriptor.split('-') + sensor, _, _, frame, *_ = descriptor.split("-") dependency_filename = f"imap_ultra_l2_{sensor}-{frame}_20250929_v001.json" - dependency = (Path(__file__).parent / "l2_dependency_files" / dependency_filename).read_text() + dependency = ( + Path(__file__).parent / "l2_dependency_files" / dependency_filename + ).read_text() version = "v000" - start_date = '20251018' + start_date = "20251018" output_filename = f"imap_ultra_l2_{descriptor}_{start_date}_{version}.cdf" if not generate_imap_file_path(output_filename).construct_path().exists(): - Ultra("l2", descriptor, dependency, start_date, None, "v000", False).process() + Ultra( + "l2", descriptor, dependency, start_date, None, "v000", False + ).process() else: print(f"Skipping generation of {output_filename}, it already exists") -if __name__ == '__main__': - generate_l2_files() \ No newline at end of file + +if __name__ == "__main__": + generate_l2_files() diff --git a/scripts/ultra/validation/create_l3_validation_maps.py b/scripts/ultra/validation/create_l3_validation_maps.py index 537ee63d0..5fd7666da 100644 --- a/scripts/ultra/validation/create_l3_validation_maps.py +++ b/scripts/ultra/validation/create_l3_validation_maps.py @@ -10,8 +10,13 @@ from tests.integration.integration_test_helpers import mock_imap_data_access -@patch('imap_l3_data_processor._parse_cli_arguments') -def run_all_maps(mock_parse_cli_arguments, *, output_dir, input_dir,): +@patch("imap_l3_data_processor._parse_cli_arguments") +def run_all_maps( + mock_parse_cli_arguments, + *, + output_dir, + input_dir, +): mock_arguments = Mock() mock_arguments.instrument = "ultra" mock_arguments.data_level = "l3" @@ -22,9 +27,12 @@ def run_all_maps(mock_parse_cli_arguments, *, output_dir, input_dir,): mock_arguments.dependency = "[]" mock_arguments.upload_to_sdc = False mock_parse_cli_arguments.return_value = mock_arguments - logging.basicConfig(force=True, level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') - with mock_imap_data_access(output_dir, list(input_dir.rglob('*.*'))): + logging.basicConfig( + force=True, + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) + with mock_imap_data_access(output_dir, list(input_dir.rglob("*.*"))): mock_arguments.descriptor = "u45-maps" imap_l3_data_processor.imap_l3_processor() spiceypy.kclear() @@ -39,14 +47,14 @@ def run_all_maps(mock_parse_cli_arguments, *, output_dir, input_dir,): spiceypy.kclear() - - if __name__ == "__main__": - output_dir = Path(f'{datetime.now().strftime('%Y%m%d')}_ultra_validation_output') + output_dir = Path(f"{datetime.now().strftime('%Y%m%d')}_ultra_validation_output") try: input_dir = Path(sys.argv[1]) except: - input_dir = Path(r'C:\Users\Harrison\AppData\Local\cava\cava\data') - print(f"Could not open input file, using default input path of {input_dir} instead") + input_dir = Path(r"C:\Users\Harrison\AppData\Local\cava\cava\data") + print( + f"Could not open input file, using default input path of {input_dir} instead" + ) - run_all_maps(output_dir=output_dir, input_dir=input_dir) \ No newline at end of file + run_all_maps(output_dir=output_dir, input_dir=input_dir) diff --git a/scripts/ultra/validation/prepare_glows_for_ultra_validation.py b/scripts/ultra/validation/prepare_glows_for_ultra_validation.py index 8e8bd6522..b8d4e3784 100644 --- a/scripts/ultra/validation/prepare_glows_for_ultra_validation.py +++ b/scripts/ultra/validation/prepare_glows_for_ultra_validation.py @@ -6,12 +6,13 @@ output_dir = Path(r"C:\Users\Harrison\Downloads\glows_l3e_for_ultra_validation") output_dir.mkdir(exist_ok=True) input_dir = Path(r"C:\Users\Harrison\Downloads\glows_l3e_from_validation\l3e") -for i, file in enumerate(input_dir.rglob('imap_glows_l3e_survival-probability-ul*.cdf')): +for i, file in enumerate( + input_dir.rglob("imap_glows_l3e_survival-probability-ul*.cdf") +): if i > 184: break - new_repoint = re.sub(r'repoint\d{5}', f'repoint{i:05}', file.name) - new_epoch = datetime(2025,4,16)+timedelta(days=i) - new_name = re.sub(r'\d{8}', new_epoch.strftime('%Y%m%d'), new_repoint) - print(f'renaming {file.name} to {new_name}') - shutil.copy(file, output_dir/new_name) - + new_repoint = re.sub(r"repoint\d{5}", f"repoint{i:05}", file.name) + new_epoch = datetime(2025, 4, 16) + timedelta(days=i) + new_name = re.sub(r"\d{8}", new_epoch.strftime("%Y%m%d"), new_repoint) + print(f"renaming {file.name} to {new_name}") + shutil.copy(file, output_dir / new_name) diff --git a/scripts/ultra/validation/prepare_ul_repoint_files_for_ultra_validation.py b/scripts/ultra/validation/prepare_ul_repoint_files_for_ultra_validation.py index 286aea19d..bc1cec2d8 100644 --- a/scripts/ultra/validation/prepare_ul_repoint_files_for_ultra_validation.py +++ b/scripts/ultra/validation/prepare_ul_repoint_files_for_ultra_validation.py @@ -6,11 +6,13 @@ output_dir = Path(r"C:\Users\Harrison\Downloads\glows_l3e_for_ultra_validation") output_dir.mkdir(exist_ok=True) input_dir = Path(r"C:\Users\Harrison\Downloads\\") -for i, file in enumerate(input_dir.rglob('imap_glows_l3e_survival-probability-ul*.cdf')): +for i, file in enumerate( + input_dir.rglob("imap_glows_l3e_survival-probability-ul*.cdf") +): if i > 184: break - new_repoint = re.sub(r'repoint\d{5}', f'repoint{i:05}', file.name) - new_epoch = datetime(2025,4,16)+timedelta(days=i) - new_name = re.sub(r'\d{8}', new_epoch.strftime('%Y%m%d'), new_repoint) - print(f'renaming {file.name} to {new_name}') - shutil.copy(file, output_dir/new_name) \ No newline at end of file + new_repoint = re.sub(r"repoint\d{5}", f"repoint{i:05}", file.name) + new_epoch = datetime(2025, 4, 16) + timedelta(days=i) + new_name = re.sub(r"\d{8}", new_epoch.strftime("%Y%m%d"), new_repoint) + print(f"renaming {file.name} to {new_name}") + shutil.copy(file, output_dir / new_name) diff --git a/scripts/ultra/validation/upload_files.py b/scripts/ultra/validation/upload_files.py index 18cfb9cf3..00b78e201 100644 --- a/scripts/ultra/validation/upload_files.py +++ b/scripts/ultra/validation/upload_files.py @@ -5,23 +5,26 @@ import imap_data_access -imap_data_access.config['DATA_ACCESS_URL']='https://api.dev.imap-mission.com/api-key' -imap_data_access.config['API_KEY']='e6378327cc4fe2d2eb35373a9b39de3f85a89f896bec3010e0476709868d1590' +imap_data_access.config["DATA_ACCESS_URL"] = "https://api.dev.imap-mission.com/api-key" +imap_data_access.config["API_KEY"] = ( + "e6378327cc4fe2d2eb35373a9b39de3f85a89f896bec3010e0476709868d1590" +) l1_45_folder = Path( - r"C:\Users\Harrison\Downloads\ULTRA_45sensor_spacecraft-psets_20251113-20251125T164844Z-1-001\ULTRA_45sensor_spacecraft-psets_20251113") + r"C:\Users\Harrison\Downloads\ULTRA_45sensor_spacecraft-psets_20251113-20251125T164844Z-1-001\ULTRA_45sensor_spacecraft-psets_20251113" +) l1_90_folder = Path( r"C:\Users\Harrison\Downloads\ULTRA_90sensor_spacecraft-psets_20251113" ) -glows_folder = Path( - r"C:\Users\Harrison\Downloads\ultra_combined_hf_validation\glows" -) +glows_folder = Path(r"C:\Users\Harrison\Downloads\ultra_combined_hf_validation\glows") l2_45_folder = Path(r"C:\Users\Harrison\Downloads\u45_l2") l2_90_folder = Path(r"C:\Users\Harrison\Downloads\u90_l2") retries = 10 uploaded = [] failed = [] + + def upload_file(f): failures = 0 while failures < retries: @@ -38,7 +41,8 @@ def upload_file(f): print(f"failed to upload file: {f} after {retries} attempts") failed.append(f) return - time.sleep(failures ** 2 * 0.5) + time.sleep(failures**2 * 0.5) + with ThreadPoolExecutor() as executor: for f in itertools.chain(glows_folder.rglob("*.cdf")): @@ -47,5 +51,3 @@ def upload_file(f): print("uploaded", uploaded) print("failed", failed) - - diff --git a/tests/cdf/test_cdf_utils.py b/tests/cdf/test_cdf_utils.py index ae8ed73ed..01b64d52b 100644 --- a/tests/cdf/test_cdf_utils.py +++ b/tests/cdf/test_cdf_utils.py @@ -6,9 +6,18 @@ from imap_data_access.file_validation import Version from spacepy import pycdf -from imap_l3_processing.cdf.cdf_utils import write_cdf, read_variable_and_mask_fill_values, read_numeric_variable +from imap_l3_processing.cdf.cdf_utils import ( + write_cdf, + read_variable_and_mask_fill_values, + read_numeric_variable, +) from imap_l3_processing.cdf.imap_attribute_manager import ImapAttributeManager -from imap_l3_processing.models import DataProduct, DataProductVariable, InputMetadata, VersionMap +from imap_l3_processing.models import ( + DataProduct, + DataProductVariable, + InputMetadata, + VersionMap, +) from tests.temp_file_test_case import TempFileTestCase @@ -16,84 +25,185 @@ class TestCdfUtils(TempFileTestCase): def test_write_cdf(self): path = str(self.temp_directory / "write_cdf.cdf") data = TestDataProduct() - var_without_explicit_type, int_var, float_var, time_var, non_rec_varying_var = data.to_data_product_variables() + var_without_explicit_type, int_var, float_var, time_var, non_rec_varying_var = ( + data.to_data_product_variables() + ) attribute_manager = Mock(spec=ImapAttributeManager) - attribute_manager.get_global_attributes.return_value = {"global1": "global_val1", "global2": "global_val2"} + attribute_manager.get_global_attributes.return_value = { + "global1": "global_val1", + "global2": "global_val2", + } attribute_manager.get_variable_attributes.side_effect = [ - {"variable_attr1": "var_val1", "DATA_TYPE": "CDF_REAL4", "RECORD_VARYING": "rV"}, - {"variable_attr1": "var_val1", "FILLVAL": -9223372036854775808, "DATA_TYPE": "CDF_INT8", - "RECORD_VARYING": "RV"}, - {"variable_attr1": "var_val1", "FILLVAL": -1e31, "DATA_TYPE": "CDF_REAL4", "RECORD_VARYING": "RV"}, - {"variable_attr3": "var_val3", "FILLVAL": -9223372036854775808, "DATA_TYPE": "CDF_TIME_TT2000", - "RECORD_VARYING": "NRV"}, - {"variable_attr5": "var_val5", "variable_attr6": "var_val6", "DATA_TY" - "PE": "CDF_INT4", - "RECORD_VARYING": "NRV"}, + { + "variable_attr1": "var_val1", + "DATA_TYPE": "CDF_REAL4", + "RECORD_VARYING": "rV", + }, + { + "variable_attr1": "var_val1", + "FILLVAL": -9223372036854775808, + "DATA_TYPE": "CDF_INT8", + "RECORD_VARYING": "RV", + }, + { + "variable_attr1": "var_val1", + "FILLVAL": -1e31, + "DATA_TYPE": "CDF_REAL4", + "RECORD_VARYING": "RV", + }, + { + "variable_attr3": "var_val3", + "FILLVAL": -9223372036854775808, + "DATA_TYPE": "CDF_TIME_TT2000", + "RECORD_VARYING": "NRV", + }, + { + "variable_attr5": "var_val5", + "variable_attr6": "var_val6", + "DATA_TYPE": "CDF_INT4", + "RECORD_VARYING": "NRV", + }, ] write_cdf(path, data, attribute_manager) expected_data_product_logical_source = "imap_glows_l3_descriptor" - attribute_manager.get_global_attributes.assert_called_once_with(expected_data_product_logical_source) + attribute_manager.get_global_attributes.assert_called_once_with( + expected_data_product_logical_source + ) attribute_manager.get_variable_attributes.assert_has_calls( [call(var.name) for var in data.to_data_product_variables()] ) with pycdf.CDF(path) as actual_cdf: self.assertTrue(actual_cdf.col_major()) - self.assertEqual('global_val1', actual_cdf.attrs['global1'][...][0]) - self.assertEqual('global_val2', actual_cdf.attrs['global2'][...][0]) - - np.testing.assert_array_equal(var_without_explicit_type.value, - actual_cdf[var_without_explicit_type.name][...]) - self.assertEqual('var_val1', actual_cdf[var_without_explicit_type.name].attrs['variable_attr1']) - self.assertEqual(pycdf.const.CDF_REAL4.value, actual_cdf[var_without_explicit_type.name].type()) + self.assertEqual("global_val1", actual_cdf.attrs["global1"][...][0]) + self.assertEqual("global_val2", actual_cdf.attrs["global2"][...][0]) + + np.testing.assert_array_equal( + var_without_explicit_type.value, + actual_cdf[var_without_explicit_type.name][...], + ) + self.assertEqual( + "var_val1", + actual_cdf[var_without_explicit_type.name].attrs["variable_attr1"], + ) + self.assertEqual( + pycdf.const.CDF_REAL4.value, + actual_cdf[var_without_explicit_type.name].type(), + ) self.assertTrue(actual_cdf[var_without_explicit_type.name].rv()) - np.testing.assert_array_equal(int_var.value, actual_cdf.raw_var(int_var.name)) - self.assertEqual('var_val1', actual_cdf[int_var.name].attrs['variable_attr1']) - self.assertEqual(-9223372036854775808, actual_cdf[int_var.name].attrs['FILLVAL']) - self.assertEqual(pycdf.const.CDF_INT8.value, actual_cdf[int_var.name].attrs.type("FILLVAL")) - self.assertEqual(pycdf.const.CDF_INT8.value, actual_cdf[int_var.name].type()) + np.testing.assert_array_equal( + int_var.value, actual_cdf.raw_var(int_var.name) + ) + self.assertEqual( + "var_val1", actual_cdf[int_var.name].attrs["variable_attr1"] + ) + self.assertEqual( + -9223372036854775808, actual_cdf[int_var.name].attrs["FILLVAL"] + ) + self.assertEqual( + pycdf.const.CDF_INT8.value, + actual_cdf[int_var.name].attrs.type("FILLVAL"), + ) + self.assertEqual( + pycdf.const.CDF_INT8.value, actual_cdf[int_var.name].type() + ) self.assertTrue(actual_cdf[int_var.name].rv()) - np.testing.assert_array_equal(float_var.value, actual_cdf.raw_var(float_var.name)) - self.assertEqual('var_val1', actual_cdf[float_var.name].attrs['variable_attr1']) - self.assertEqual(-1e31, actual_cdf[float_var.name].attrs['FILLVAL']) - self.assertEqual(pycdf.const.CDF_REAL4.value, actual_cdf[float_var.name].attrs.type("FILLVAL")) - self.assertEqual(pycdf.const.CDF_REAL4.value, actual_cdf[float_var.name].type()) + np.testing.assert_array_equal( + float_var.value, actual_cdf.raw_var(float_var.name) + ) + self.assertEqual( + "var_val1", actual_cdf[float_var.name].attrs["variable_attr1"] + ) + self.assertEqual(-1e31, actual_cdf[float_var.name].attrs["FILLVAL"]) + self.assertEqual( + pycdf.const.CDF_REAL4.value, + actual_cdf[float_var.name].attrs.type("FILLVAL"), + ) + self.assertEqual( + pycdf.const.CDF_REAL4.value, actual_cdf[float_var.name].type() + ) self.assertTrue(actual_cdf[float_var.name].rv()) - np.testing.assert_array_equal(time_var.value, actual_cdf.raw_var(time_var.name)) - self.assertEqual('var_val3', actual_cdf[time_var.name].attrs['variable_attr3']) - self.assertEqual(datetime(9999, 12, 31, 23, 59, 59, 999999), actual_cdf[time_var.name].attrs['FILLVAL']) - self.assertEqual(pycdf.const.CDF_TIME_TT2000.value, actual_cdf[time_var.name].attrs.type("FILLVAL")) - self.assertEqual(pycdf.const.CDF_TIME_TT2000.value, actual_cdf[time_var.name].type()) + np.testing.assert_array_equal( + time_var.value, actual_cdf.raw_var(time_var.name) + ) + self.assertEqual( + "var_val3", actual_cdf[time_var.name].attrs["variable_attr3"] + ) + self.assertEqual( + datetime(9999, 12, 31, 23, 59, 59, 999999), + actual_cdf[time_var.name].attrs["FILLVAL"], + ) + self.assertEqual( + pycdf.const.CDF_TIME_TT2000.value, + actual_cdf[time_var.name].attrs.type("FILLVAL"), + ) + self.assertEqual( + pycdf.const.CDF_TIME_TT2000.value, actual_cdf[time_var.name].type() + ) self.assertFalse(actual_cdf[time_var.name].rv()) - self.assertEqual('var_val5', actual_cdf[non_rec_varying_var.name].attrs['variable_attr5']) - self.assertEqual('var_val6', actual_cdf[non_rec_varying_var.name].attrs['variable_attr6']) - self.assertEqual(non_rec_varying_var.value, actual_cdf[non_rec_varying_var.name][...]) - self.assertEqual(pycdf.const.CDF_INT4.value, actual_cdf[non_rec_varying_var.name].type()) + self.assertEqual( + "var_val5", actual_cdf[non_rec_varying_var.name].attrs["variable_attr5"] + ) + self.assertEqual( + "var_val6", actual_cdf[non_rec_varying_var.name].attrs["variable_attr6"] + ) + self.assertEqual( + non_rec_varying_var.value, actual_cdf[non_rec_varying_var.name][...] + ) + self.assertEqual( + pycdf.const.CDF_INT4.value, actual_cdf[non_rec_varying_var.name].type() + ) self.assertFalse(actual_cdf[non_rec_varying_var.name].rv()) - actual_var_attributes = [attribute for k, v in actual_cdf.items() for attribute in v.attrs] + actual_var_attributes = [ + attribute for k, v in actual_cdf.items() for attribute in v.attrs + ] self.assertNotIn("DATA_TYPE", actual_var_attributes) self.assertNotIn("RECORD_VARYING", actual_var_attributes) @patch("imap_l3_processing.cdf.cdf_utils.CDF") def test_write_cdf_compresses_variables(self, mock_new_cdf): attribute_manager = Mock(spec=ImapAttributeManager) - attribute_manager.get_global_attributes.return_value = {"global1": "global_val1", "global2": "global_val2"} + attribute_manager.get_global_attributes.return_value = { + "global1": "global_val1", + "global2": "global_val2", + } attribute_manager.get_variable_attributes.side_effect = [ - {"variable_attr1": "var_val1", "DATA_TYPE": "CDF_REAL4", "RECORD_VARYING": "rV"}, - {"variable_attr1": "var_val1", "FILLVAL": -9223372036854775808, "DATA_TYPE": "CDF_INT8", - "RECORD_VARYING": "RV"}, - {"variable_attr1": "var_val1", "FILLVAL": -1e31, "DATA_TYPE": "CDF_REAL4", "RECORD_VARYING": "RV"}, - {"variable_attr3": "var_val3", "FILLVAL": -9223372036854775808, "DATA_TYPE": "CDF_TIME_TT2000", - "RECORD_VARYING": "NRV"}, - {"variable_attr5": "var_val5", "variable_attr6": "var_val6", "DATA_TYPE": "CDF_INT4", - "RECORD_VARYING": "NRV"}, + { + "variable_attr1": "var_val1", + "DATA_TYPE": "CDF_REAL4", + "RECORD_VARYING": "rV", + }, + { + "variable_attr1": "var_val1", + "FILLVAL": -9223372036854775808, + "DATA_TYPE": "CDF_INT8", + "RECORD_VARYING": "RV", + }, + { + "variable_attr1": "var_val1", + "FILLVAL": -1e31, + "DATA_TYPE": "CDF_REAL4", + "RECORD_VARYING": "RV", + }, + { + "variable_attr3": "var_val3", + "FILLVAL": -9223372036854775808, + "DATA_TYPE": "CDF_TIME_TT2000", + "RECORD_VARYING": "NRV", + }, + { + "variable_attr5": "var_val5", + "variable_attr6": "var_val6", + "DATA_TYPE": "CDF_INT4", + "RECORD_VARYING": "NRV", + }, ] expected_data_product = TestDataProduct() @@ -102,43 +212,91 @@ def test_write_cdf_compresses_variables(self, mock_new_cdf): mock_new_cdf.return_value.__enter__.return_value.new.assert_has_calls( [ - call('var_without_explicit_type', mock.ANY, recVary=True, type=pycdf.const.CDF_REAL4, dims=mock.ANY, - compress=pycdf.const.GZIP_COMPRESSION, compress_param=7), - call('int_var', mock.ANY, recVary=True, type=pycdf.const.CDF_INT8, dims=mock.ANY, - compress=pycdf.const.GZIP_COMPRESSION, compress_param=7), - call('float_var', mock.ANY, recVary=True, type=pycdf.const.CDF_REAL4, dims=mock.ANY, - compress=pycdf.const.GZIP_COMPRESSION, compress_param=7), - call('time_var', mock.ANY, recVary=False, type=pycdf.const.CDF_TIME_TT2000, dims=mock.ANY, - compress=None, compress_param=None), - call('non_record_varying', mock.ANY, recVary=False, type=pycdf.const.CDF_INT4, dims=mock.ANY, - compress=None, compress_param=None), + call( + "var_without_explicit_type", + mock.ANY, + recVary=True, + type=pycdf.const.CDF_REAL4, + dims=mock.ANY, + compress=pycdf.const.GZIP_COMPRESSION, + compress_param=7, + ), + call( + "int_var", + mock.ANY, + recVary=True, + type=pycdf.const.CDF_INT8, + dims=mock.ANY, + compress=pycdf.const.GZIP_COMPRESSION, + compress_param=7, + ), + call( + "float_var", + mock.ANY, + recVary=True, + type=pycdf.const.CDF_REAL4, + dims=mock.ANY, + compress=pycdf.const.GZIP_COMPRESSION, + compress_param=7, + ), + call( + "time_var", + mock.ANY, + recVary=False, + type=pycdf.const.CDF_TIME_TT2000, + dims=mock.ANY, + compress=None, + compress_param=None, + ), + call( + "non_record_varying", + mock.ANY, + recVary=False, + type=pycdf.const.CDF_INT4, + dims=mock.ANY, + compress=None, + compress_param=None, + ), ] ) - def test_write_cdf_trims_numbers_in_logical_source_when_fetching_global_metadata(self): + def test_write_cdf_trims_numbers_in_logical_source_when_fetching_global_metadata( + self, + ): path = str(self.temp_directory / "write_cdf.cdf") data = TestDataProduct() attribute_manager = Mock(spec=ImapAttributeManager) - attribute_manager.get_global_attributes.side_effect = [KeyError("Logical Source Not Found"), - {"global1": "global_val1", "global2": "global_val2"}] - attribute_manager.get_variable_attributes.return_value = {"DATA_TYPE": "CDF_REAL4", "RECORD_VARYING": "NRV"} + attribute_manager.get_global_attributes.side_effect = [ + KeyError("Logical Source Not Found"), + {"global1": "global_val1", "global2": "global_val2"}, + ] + attribute_manager.get_variable_attributes.return_value = { + "DATA_TYPE": "CDF_REAL4", + "RECORD_VARYING": "NRV", + } expected_data_product_logical_source = "imap_glows_l3_descriptor-10100" data.input_metadata.descriptor = "descriptor-10100" write_cdf(path, data, attribute_manager) attribute_manager.get_global_attributes.assert_has_calls( - [call(expected_data_product_logical_source), call("imap_glows_l3_descriptor-")]) + [ + call(expected_data_product_logical_source), + call("imap_glows_l3_descriptor-"), + ] + ) with pycdf.CDF(path) as actual_cdf: self.assertTrue(actual_cdf.col_major()) - self.assertEqual('global_val1', str(actual_cdf.attrs['global1'])) - self.assertEqual('global_val2', str(actual_cdf.attrs['global2'])) + self.assertEqual("global_val1", str(actual_cdf.attrs["global1"])) + self.assertEqual("global_val2", str(actual_cdf.attrs["global2"])) def test_write_cdf_replaces_nan_inf_and_overflowing_floats_with_fill_value(self): epoch_fillval = datetime.fromisoformat("9999-12-31T23:59:59.999999") - epoch_data = np.ma.masked_array([datetime(2025, 3, 7, 17, 0), None], mask=[False, True]) + epoch_data = np.ma.masked_array( + [datetime(2025, 3, 7, 17, 0), None], mask=[False, True] + ) expected_cdf_epoch_data = np.array([datetime(2025, 3, 7, 17, 0), epoch_fillval]) float_var_input = np.array( @@ -156,9 +314,12 @@ def __init__(self): def to_data_product_variables(self) -> list[DataProductVariable]: return [ - DataProductVariable("epoch", epoch_data, pycdf.const.CDF_TIME_TT2000), - DataProductVariable("float_var", float_var_input, - pycdf.const.CDF_REAL4), + DataProductVariable( + "epoch", epoch_data, pycdf.const.CDF_TIME_TT2000 + ), + DataProductVariable( + "float_var", float_var_input, pycdf.const.CDF_REAL4 + ), ] path = str(self.temp_directory / "write_cdf.cdf") @@ -166,9 +327,18 @@ def to_data_product_variables(self) -> list[DataProductVariable]: attribute_manager = Mock(spec=ImapAttributeManager) attribute_manager.get_global_attributes.return_value = {} attribute_manager.get_variable_attributes.side_effect = [ - {"VAR_NAME": "epoch", "FILLVAL": epoch_fillval, - "DATA_TYPE": "CDF_TIME_TT2000", "RECORD_VARYING": "NRV"}, - {"VAR_NAME": "float_var", "FILLVAL": -1e31, "DATA_TYPE": "CDF_REAL4", "RECORD_VARYING": "NRV"}, + { + "VAR_NAME": "epoch", + "FILLVAL": epoch_fillval, + "DATA_TYPE": "CDF_TIME_TT2000", + "RECORD_VARYING": "NRV", + }, + { + "VAR_NAME": "float_var", + "FILLVAL": -1e31, + "DATA_TYPE": "CDF_REAL4", + "RECORD_VARYING": "NRV", + }, ] write_cdf(path, data, attribute_manager) @@ -176,9 +346,12 @@ def to_data_product_variables(self) -> list[DataProductVariable]: actual_float_var = actual_cdf["float_var"][...] self.assertFalse(np.any(np.isnan(actual_float_var))) self.assertFalse(np.any(np.isinf(actual_float_var))) - np.testing.assert_array_equal(expected_float_var, actual_float_var, strict=True) - np.testing.assert_array_equal(actual_cdf["epoch"][...], expected_cdf_epoch_data, strict=True) - + np.testing.assert_array_equal( + expected_float_var, actual_float_var, strict=True + ) + np.testing.assert_array_equal( + actual_cdf["epoch"][...], expected_cdf_epoch_data, strict=True + ) def test_can_write_empty_epoch_variable(self): epoch_data = [] @@ -190,7 +363,9 @@ def __init__(self): def to_data_product_variables(self) -> list[DataProductVariable]: return [ - DataProductVariable("epoch", epoch_data, pycdf.const.CDF_TIME_TT2000), + DataProductVariable( + "epoch", epoch_data, pycdf.const.CDF_TIME_TT2000 + ), ] path = str(self.temp_directory / "write_cdf.cdf") @@ -198,33 +373,57 @@ def to_data_product_variables(self) -> list[DataProductVariable]: attribute_manager = Mock(spec=ImapAttributeManager) attribute_manager.get_global_attributes.return_value = {} attribute_manager.get_variable_attributes.side_effect = [ - {"VAR_NAME": "epoch", "FILLVAL": epoch_fillval, - "DATA_TYPE": "CDF_TIME_TT2000", "RECORD_VARYING": "RV"}, - {"VAR_NAME": "float_var", "FILLVAL": -1e31, "DATA_TYPE": "CDF_REAL4", "RECORD_VARYING": "NRV"}, + { + "VAR_NAME": "epoch", + "FILLVAL": epoch_fillval, + "DATA_TYPE": "CDF_TIME_TT2000", + "RECORD_VARYING": "RV", + }, + { + "VAR_NAME": "float_var", + "FILLVAL": -1e31, + "DATA_TYPE": "CDF_REAL4", + "RECORD_VARYING": "NRV", + }, ] write_cdf(path, data, attribute_manager) with pycdf.CDF(path) as actual_cdf: - np.testing.assert_array_equal(actual_cdf["epoch"][...], np.array([], dtype=object), strict=True) - - + np.testing.assert_array_equal( + actual_cdf["epoch"][...], np.array([], dtype=object), strict=True + ) def test_can_write_empty_variables_of_various_shapes(self): - cases=[ - (np.zeros((0,5), dtype=object), "CDF_TIME_TT2000", "RV", datetime.fromisoformat("9999-12-31T23:59:59.999999")), - (np.zeros((0), dtype=object), "CDF_TIME_TT2000", "RV", datetime.fromisoformat("9999-12-31T23:59:59.999999")), - (np.zeros((0,5), dtype=float), "CDF_REAL8", "RV", -1e31), + cases = [ + ( + np.zeros((0, 5), dtype=object), + "CDF_TIME_TT2000", + "RV", + datetime.fromisoformat("9999-12-31T23:59:59.999999"), + ), + ( + np.zeros((0), dtype=object), + "CDF_TIME_TT2000", + "RV", + datetime.fromisoformat("9999-12-31T23:59:59.999999"), + ), + (np.zeros((0, 5), dtype=float), "CDF_REAL8", "RV", -1e31), (np.zeros((0), dtype=float), "CDF_REAL8", "RV", -1e31), ] for input_data, data_type, record_varying, fill_val in cases: - with self.subTest(input_data=input_data, data_type=data_type, rv=record_varying): + with self.subTest( + input_data=input_data, data_type=data_type, rv=record_varying + ): + class DataProductWithEmptyEpoch(DataProduct): def __init__(self): self.input_metadata = Mock() def to_data_product_variables(self) -> list[DataProductVariable]: return [ - DataProductVariable("var", input_data, pycdf.const.CDF_REAL8), + DataProductVariable( + "var", input_data, pycdf.const.CDF_REAL8 + ), ] path = self.temp_directory / "write_cdf.cdf" @@ -232,22 +431,33 @@ def to_data_product_variables(self) -> list[DataProductVariable]: attribute_manager = Mock(spec=ImapAttributeManager) attribute_manager.get_global_attributes.return_value = {} attribute_manager.get_variable_attributes.side_effect = [ - {"VAR_NAME": "var", "FILLVAL": fill_val, "DATA_TYPE": data_type, "RECORD_VARYING": record_varying}, + { + "VAR_NAME": "var", + "FILLVAL": fill_val, + "DATA_TYPE": data_type, + "RECORD_VARYING": record_varying, + }, ] write_cdf(str(path), data, attribute_manager) with pycdf.CDF(str(path)) as actual_cdf: - np.testing.assert_array_equal(actual_cdf["var"][...], input_data, strict=True) + np.testing.assert_array_equal( + actual_cdf["var"][...], input_data, strict=True + ) path.unlink() - def test_does_not_write_variable_attribute_if_value_is_empty_string(self): path = str(self.temp_directory / "write_cdf.cdf") data = TestDataProduct() - _, _, regular_var, time_var, non_rec_varying_var = data.to_data_product_variables() + _, _, regular_var, time_var, non_rec_varying_var = ( + data.to_data_product_variables() + ) attribute_manager = Mock(spec=ImapAttributeManager) - attribute_manager.get_global_attributes.return_value = {"global1": "global_val1", "global2": "global_val2"} + attribute_manager.get_global_attributes.return_value = { + "global1": "global_val1", + "global2": "global_val2", + } attribute_manager.get_variable_attributes.side_effect = [ {"DATA_TYPE": "CDF_REAL4", "RECORD_VARYING": "NRV"}, {"DATA_TYPE": "CDF_REAL4", "RECORD_VARYING": "NRV"}, @@ -259,41 +469,55 @@ def test_does_not_write_variable_attribute_if_value_is_empty_string(self): write_cdf(path, data, attribute_manager) with pycdf.CDF(path) as actual_cdf: - np.testing.assert_array_equal(regular_var.value, actual_cdf[regular_var.name][...]) - self.assertEqual('epoch', actual_cdf[regular_var.name].attrs['DEPEND_0']) - self.assertFalse('CATDESC' in actual_cdf[time_var.name].attrs) - self.assertFalse('DEPEND_0' in actual_cdf[non_rec_varying_var.name].attrs) + np.testing.assert_array_equal( + regular_var.value, actual_cdf[regular_var.name][...] + ) + self.assertEqual("epoch", actual_cdf[regular_var.name].attrs["DEPEND_0"]) + self.assertFalse("CATDESC" in actual_cdf[time_var.name].attrs) + self.assertFalse("DEPEND_0" in actual_cdf[non_rec_varying_var.name].attrs) def test_read_variable_and_mask_fill_values(self): path = str(self.temp_directory / "cdf.cdf") with pycdf.CDF(path, create=True) as actual_cdf: datetime_fill = datetime(9999, 12, 31, 23, 59, 59, 999999) - actual_cdf['date_var'] = [datetime(2025, 4, 16), datetime_fill] - actual_cdf['date_var'].attrs['FILLVAL'] = datetime_fill + actual_cdf["date_var"] = [datetime(2025, 4, 16), datetime_fill] + actual_cdf["date_var"].attrs["FILLVAL"] = datetime_fill int_fillval = -9223372036854775808 - actual_cdf['int_var'] = np.array([-1, 2, int_fillval, int_fillval + 1], dtype=np.int64) - actual_cdf['int_var'].attrs['FILLVAL'] = int_fillval - - masked_datetime_data: np.ma.masked_array = read_variable_and_mask_fill_values(actual_cdf['date_var']) - masked_int_data: np.ma.masked_array = read_variable_and_mask_fill_values(actual_cdf['int_var']) - np.testing.assert_array_equal(masked_datetime_data.data, - [datetime(2025, 4, 16), datetime_fill]) + actual_cdf["int_var"] = np.array( + [-1, 2, int_fillval, int_fillval + 1], dtype=np.int64 + ) + actual_cdf["int_var"].attrs["FILLVAL"] = int_fillval + + masked_datetime_data: np.ma.masked_array = ( + read_variable_and_mask_fill_values(actual_cdf["date_var"]) + ) + masked_int_data: np.ma.masked_array = read_variable_and_mask_fill_values( + actual_cdf["int_var"] + ) + np.testing.assert_array_equal( + masked_datetime_data.data, [datetime(2025, 4, 16), datetime_fill] + ) np.testing.assert_equal(masked_datetime_data.mask, [False, True]) - np.testing.assert_equal(masked_int_data.data, np.array([-1, 2, int_fillval, int_fillval + 1], dtype=np.int64)) - np.testing.assert_equal(masked_int_data.mask, np.array([False, False, True, False])) + np.testing.assert_equal( + masked_int_data.data, + np.array([-1, 2, int_fillval, int_fillval + 1], dtype=np.int64), + ) + np.testing.assert_equal( + masked_int_data.mask, np.array([False, False, True, False]) + ) def test_read_variable_replaces_fill_values_with_nan(self): path = str(self.temp_directory / "cdf.cdf") with pycdf.CDF(path, create=True) as actual_cdf: - actual_cdf['var'] = np.array([1, 2, -1e31, 4, 5], dtype=np.float64) - actual_cdf['var'].attrs['FILLVAL'] = -1e31 - actual_cdf['int_var'] = np.array([1, 2, 3, 4, 5], dtype=np.int32) - actual_cdf['int_var'].attrs['FILLVAL'] = 5 + actual_cdf["var"] = np.array([1, 2, -1e31, 4, 5], dtype=np.float64) + actual_cdf["var"].attrs["FILLVAL"] = -1e31 + actual_cdf["int_var"] = np.array([1, 2, 3, 4, 5], dtype=np.int32) + actual_cdf["int_var"].attrs["FILLVAL"] = 5 - data = read_numeric_variable(actual_cdf['var']) - int_data = read_numeric_variable(actual_cdf['int_var']) + data = read_numeric_variable(actual_cdf["var"]) + int_data = read_numeric_variable(actual_cdf["int_var"]) np.testing.assert_equal(data, np.array([1, 2, np.nan, 4, 5])) np.testing.assert_equal(int_data, np.array([1, 2, 3, 4, np.nan])) @@ -301,9 +525,14 @@ def test_read_variable_replaces_fill_values_with_nan(self): def test_write_map_cdf(self): class TestMapDataProduct(DataProduct): def __init__(self): - self.input_metadata = InputMetadata("hi", "l3", datetime(year=2025, month=1, day=1), - datetime(year=2025, month=5, day=31), "v001", - "h45-ena-h-sf-sp-ram-hae-6deg-6mo") + self.input_metadata = InputMetadata( + "hi", + "l3", + datetime(year=2025, month=1, day=1), + datetime(year=2025, month=5, day=31), + "v001", + "h45-ena-h-sf-sp-ram-hae-6deg-6mo", + ) self.parent_file_names = [] def to_data_product_variables(self) -> list[DataProductVariable]: @@ -311,35 +540,60 @@ def to_data_product_variables(self) -> list[DataProductVariable]: DataProductVariable("ena_intensity", np.arange(0, 10)), DataProductVariable("ena_intensity_stat_uncert", np.arange(0, 10)), ] + path = str(self.temp_directory / "write_cdf.cdf") data = TestMapDataProduct() attribute_manager = Mock(spec=ImapAttributeManager) - attribute_manager.get_global_attributes.return_value = {"global1": "global_val1", "global2": "global_val2"} - - variable_metadata = {"CATDESC": "This should be ignored", "DATA_TYPE": "CDF_REAL4", "FILLVAL": -1e31, "RECORD_VARYING": "rV"} - attribute_manager.get_variable_attributes.side_effect = lambda _: variable_metadata.copy() + attribute_manager.get_global_attributes.return_value = { + "global1": "global_val1", + "global2": "global_val2", + } + + variable_metadata = { + "CATDESC": "This should be ignored", + "DATA_TYPE": "CDF_REAL4", + "FILLVAL": -1e31, + "RECORD_VARYING": "rV", + } + attribute_manager.get_variable_attributes.side_effect = lambda _: ( + variable_metadata.copy() + ) write_cdf(path, data, attribute_manager) - expected_data_product_logical_source = "imap_hi_l3_h45-ena-h-sf-sp-ram-hae-6deg-6mo" - attribute_manager.get_global_attributes.assert_called_once_with(expected_data_product_logical_source) + expected_data_product_logical_source = ( + "imap_hi_l3_h45-ena-h-sf-sp-ram-hae-6deg-6mo" + ) + attribute_manager.get_global_attributes.assert_called_once_with( + expected_data_product_logical_source + ) attribute_manager.get_variable_attributes.assert_has_calls( [call(var.name) for var in data.to_data_product_variables()] ) with pycdf.CDF(path) as actual_cdf: - self.assertEqual("IMAP Hi45 H Inten, HAE SC Frame, Surv Corr, Ram, 6 deg, 6 Mon", - actual_cdf['ena_intensity'].attrs['CATDESC']) - self.assertEqual("IMAP Hi45 H Inten Stat. Unc., HAE SC Frame, Surv Corr, Ram, 6 deg, 6 Mon", - actual_cdf['ena_intensity_stat_uncert'].attrs['CATDESC']) + self.assertEqual( + "IMAP Hi45 H Inten, HAE SC Frame, Surv Corr, Ram, 6 deg, 6 Mon", + actual_cdf["ena_intensity"].attrs["CATDESC"], + ) + self.assertEqual( + "IMAP Hi45 H Inten Stat. Unc., HAE SC Frame, Surv Corr, Ram, 6 deg, 6 Mon", + actual_cdf["ena_intensity_stat_uncert"].attrs["CATDESC"], + ) class TestDataProduct(DataProduct): def __init__(self): super().__init__( - input_metadata=InputMetadata("glows", "l3", datetime(year=2025, month=5, day=10), - datetime(year=2025, month=5, day=12), VersionMap({'descriptor': Version(None, 3)}), "descriptor"), - parent_file_names=[] + input_metadata=InputMetadata( + "glows", + "l3", + datetime(year=2025, month=5, day=10), + datetime(year=2025, month=5, day=12), + VersionMap({"descriptor": Version(None, 3)}), + "descriptor", + ), + parent_file_names=[], ) def to_data_product_variables(self) -> list[DataProductVariable]: @@ -348,5 +602,5 @@ def to_data_product_variables(self) -> list[DataProductVariable]: DataProductVariable("int_var", np.arange(0, 10)), DataProductVariable("float_var", np.arange(0, 10)), DataProductVariable("time_var", np.arange(10, 20)), - DataProductVariable("non_record_varying", 100) + DataProductVariable("non_record_varying", 100), ] diff --git a/tests/cdf/test_imap_attribute_manager.py b/tests/cdf/test_imap_attribute_manager.py index f43ad1025..9f0dc62da 100644 --- a/tests/cdf/test_imap_attribute_manager.py +++ b/tests/cdf/test_imap_attribute_manager.py @@ -9,93 +9,178 @@ class TestImapCdfManager(TestCase): def setUp(self): - self.config_folder_path = Path(imap_l3_processing.__file__).parent.resolve() / 'cdf/config' + self.config_folder_path = ( + Path(imap_l3_processing.__file__).parent.resolve() / "cdf/config" + ) self.base_manager = CdfAttributeManager( - variable_schema_layers=[self.config_folder_path / 'imap_l3_variable_cdf_attrs_schema.yaml', - self.config_folder_path / 'default_variable_cdf_attrs_schema.yaml'], - global_schema_layers=[self.config_folder_path / 'default_global_cdf_attrs_schema.yaml'], - use_defaults=True) - self.base_manager.load_global_attributes(self.config_folder_path / 'imap_default_global_cdf_attrs.yaml') + variable_schema_layers=[ + self.config_folder_path / "imap_l3_variable_cdf_attrs_schema.yaml", + self.config_folder_path / "default_variable_cdf_attrs_schema.yaml", + ], + global_schema_layers=[ + self.config_folder_path / "default_global_cdf_attrs_schema.yaml" + ], + use_defaults=True, + ) + self.base_manager.load_global_attributes( + self.config_folder_path / "imap_default_global_cdf_attrs.yaml" + ) def test_constructor(self): manager = ImapAttributeManager() self.assertIsInstance(manager, CdfAttributeManager) - self.assertEqual(self.base_manager.get_global_attributes(), manager.get_global_attributes()) + self.assertEqual( + self.base_manager.get_global_attributes(), manager.get_global_attributes() + ) def test_load_instrument_and_variable_attributes_with_level(self): manager = ImapAttributeManager() - manager.add_instrument_attrs('swapi', 'l3a', "descriptor") - manager.add_global_attribute('ground_software_version', 'test version') - - self.base_manager.load_global_attributes(self.config_folder_path / 'imap_swapi_global_cdf_attrs.yaml') - self.base_manager.load_global_attributes(self.config_folder_path / 'imap_swapi_l3a_global_cdf_attrs.yaml') - self.base_manager.load_variable_attributes(self.config_folder_path / 'imap_swapi_l3a_variable_attrs.yaml') - self.base_manager.add_global_attribute('ground_software_version', 'test version') - - self.assertEqual(self.base_manager.get_global_attributes(), manager.get_global_attributes()) - self.assertEqual(self.base_manager.get_variable_attributes('epoch'), manager.get_variable_attributes('epoch')) - self.assertEqual(self.base_manager.get_variable_attributes('proton_sw_speed_uncert'), - manager.get_variable_attributes('proton_sw_speed_uncert')) - self.assertEqual(self.base_manager.get_variable_attributes('proton_sw_speed'), - manager.get_variable_attributes('proton_sw_speed')) - - self.assertEqual(self.base_manager.get_variable_attributes('alpha_sw_speed'), - manager.get_variable_attributes('alpha_sw_speed')) - self.assertEqual(self.base_manager.get_variable_attributes('alpha_sw_speed_uncert'), - manager.get_variable_attributes('alpha_sw_speed_uncert')) - - self.assertEqual(self.base_manager.get_variable_attributes('alpha_sw_density'), - manager.get_variable_attributes('alpha_sw_density')) - self.assertEqual(self.base_manager.get_variable_attributes('alpha_sw_density_uncert'), - manager.get_variable_attributes('alpha_sw_density_uncert')) - - self.assertEqual(self.base_manager.get_variable_attributes('alpha_sw_temperature'), - manager.get_variable_attributes('alpha_sw_temperature')) - self.assertEqual(self.base_manager.get_variable_attributes('alpha_sw_temperature_uncert'), - manager.get_variable_attributes('alpha_sw_temperature_uncert')) + manager.add_instrument_attrs("swapi", "l3a", "descriptor") + manager.add_global_attribute("ground_software_version", "test version") + + self.base_manager.load_global_attributes( + self.config_folder_path / "imap_swapi_global_cdf_attrs.yaml" + ) + self.base_manager.load_global_attributes( + self.config_folder_path / "imap_swapi_l3a_global_cdf_attrs.yaml" + ) + self.base_manager.load_variable_attributes( + self.config_folder_path / "imap_swapi_l3a_variable_attrs.yaml" + ) + self.base_manager.add_global_attribute( + "ground_software_version", "test version" + ) + + self.assertEqual( + self.base_manager.get_global_attributes(), manager.get_global_attributes() + ) + self.assertEqual( + self.base_manager.get_variable_attributes("epoch"), + manager.get_variable_attributes("epoch"), + ) + self.assertEqual( + self.base_manager.get_variable_attributes("proton_sw_speed_uncert"), + manager.get_variable_attributes("proton_sw_speed_uncert"), + ) + self.assertEqual( + self.base_manager.get_variable_attributes("proton_sw_speed"), + manager.get_variable_attributes("proton_sw_speed"), + ) + + self.assertEqual( + self.base_manager.get_variable_attributes("alpha_sw_speed"), + manager.get_variable_attributes("alpha_sw_speed"), + ) + self.assertEqual( + self.base_manager.get_variable_attributes("alpha_sw_speed_uncert"), + manager.get_variable_attributes("alpha_sw_speed_uncert"), + ) + + self.assertEqual( + self.base_manager.get_variable_attributes("alpha_sw_density"), + manager.get_variable_attributes("alpha_sw_density"), + ) + self.assertEqual( + self.base_manager.get_variable_attributes("alpha_sw_density_uncert"), + manager.get_variable_attributes("alpha_sw_density_uncert"), + ) + + self.assertEqual( + self.base_manager.get_variable_attributes("alpha_sw_temperature"), + manager.get_variable_attributes("alpha_sw_temperature"), + ) + self.assertEqual( + self.base_manager.get_variable_attributes("alpha_sw_temperature_uncert"), + manager.get_variable_attributes("alpha_sw_temperature_uncert"), + ) def test_l3b_metadata_configuration(self): manager = ImapAttributeManager() - manager.add_instrument_attrs('swapi', 'l3b', "descriptor") - - self.base_manager.load_global_attributes(self.config_folder_path / 'imap_swapi_global_cdf_attrs.yaml') - self.base_manager.load_global_attributes(self.config_folder_path / 'imap_swapi_l3b_global_cdf_attrs.yaml') - self.base_manager.load_variable_attributes(self.config_folder_path / 'imap_swapi_l3b_variable_attrs.yaml') - self.assertEqual(self.base_manager.get_global_attributes(), manager.get_global_attributes()) - self.assertEqual(self.base_manager.get_variable_attributes('epoch'), manager.get_variable_attributes('epoch')) - - self.assertEqual(self.base_manager.get_variable_attributes('combined_energy'), - manager.get_variable_attributes('combined_energy')) - self.assertEqual(self.base_manager.get_variable_attributes('combined_energy_stat_uncert_minus'), - manager.get_variable_attributes('combined_energy_stat_uncert_minus')) - self.assertEqual(self.base_manager.get_variable_attributes('combined_energy_stat_uncert_plus'), - manager.get_variable_attributes('combined_energy_stat_uncert_plus')) - - self.assertEqual(self.base_manager.get_variable_attributes('combined_differential_flux'), - manager.get_variable_attributes('combined_differential_flux')) - self.assertEqual(self.base_manager.get_variable_attributes('combined_differential_flux_stat_uncert'), - manager.get_variable_attributes('combined_differential_flux_stat_uncert')) + manager.add_instrument_attrs("swapi", "l3b", "descriptor") + + self.base_manager.load_global_attributes( + self.config_folder_path / "imap_swapi_global_cdf_attrs.yaml" + ) + self.base_manager.load_global_attributes( + self.config_folder_path / "imap_swapi_l3b_global_cdf_attrs.yaml" + ) + self.base_manager.load_variable_attributes( + self.config_folder_path / "imap_swapi_l3b_variable_attrs.yaml" + ) + self.assertEqual( + self.base_manager.get_global_attributes(), manager.get_global_attributes() + ) + self.assertEqual( + self.base_manager.get_variable_attributes("epoch"), + manager.get_variable_attributes("epoch"), + ) + + self.assertEqual( + self.base_manager.get_variable_attributes("combined_energy"), + manager.get_variable_attributes("combined_energy"), + ) + self.assertEqual( + self.base_manager.get_variable_attributes( + "combined_energy_stat_uncert_minus" + ), + manager.get_variable_attributes("combined_energy_stat_uncert_minus"), + ) + self.assertEqual( + self.base_manager.get_variable_attributes( + "combined_energy_stat_uncert_plus" + ), + manager.get_variable_attributes("combined_energy_stat_uncert_plus"), + ) + + self.assertEqual( + self.base_manager.get_variable_attributes("combined_differential_flux"), + manager.get_variable_attributes("combined_differential_flux"), + ) + self.assertEqual( + self.base_manager.get_variable_attributes( + "combined_differential_flux_stat_uncert" + ), + manager.get_variable_attributes("combined_differential_flux_stat_uncert"), + ) def test_load_instrument_and_variable_attributes_with_level_and_descriptor(self): manager = ImapAttributeManager() - manager.add_instrument_attrs('hit', 'l3', 'macropixel') + manager.add_instrument_attrs("hit", "l3", "macropixel") - self.base_manager.load_global_attributes(self.config_folder_path / 'imap_hit_global_cdf_attrs.yaml') self.base_manager.load_global_attributes( - self.config_folder_path / 'imap_hit_l3_macropixel_global_cdf_attrs.yaml') + self.config_folder_path / "imap_hit_global_cdf_attrs.yaml" + ) + self.base_manager.load_global_attributes( + self.config_folder_path / "imap_hit_l3_macropixel_global_cdf_attrs.yaml" + ) self.base_manager.load_variable_attributes( - self.config_folder_path / 'imap_hit_l3_macropixel_variable_attrs.yaml') - - self.assertEqual(self.base_manager.get_global_attributes(), manager.get_global_attributes()) - self.assertEqual(self.base_manager.get_variable_attributes('epoch'), manager.get_variable_attributes('epoch')) - - def test_gets_global_attrs_from_global_file_when_descriptor_specific_variable_file_exists_and_global_does_not(self): + self.config_folder_path / "imap_hit_l3_macropixel_variable_attrs.yaml" + ) + + self.assertEqual( + self.base_manager.get_global_attributes(), manager.get_global_attributes() + ) + self.assertEqual( + self.base_manager.get_variable_attributes("epoch"), + manager.get_variable_attributes("epoch"), + ) + + def test_gets_global_attrs_from_global_file_when_descriptor_specific_variable_file_exists_and_global_does_not( + self, + ): manager = ImapAttributeManager() - manager.add_instrument_attrs('hi', 'l3', '45sensor-spacecraft-survival-full-4deg-map') + manager.add_instrument_attrs( + "hi", "l3", "45sensor-spacecraft-survival-full-4deg-map" + ) - self.base_manager.load_global_attributes(self.config_folder_path / 'imap_hi_l3_global_cdf_attrs.yaml') self.base_manager.load_global_attributes( - self.config_folder_path / 'imap_hi_global_cdf_attrs.yaml') + self.config_folder_path / "imap_hi_l3_global_cdf_attrs.yaml" + ) + self.base_manager.load_global_attributes( + self.config_folder_path / "imap_hi_global_cdf_attrs.yaml" + ) - self.assertEqual(self.base_manager.get_global_attributes(), manager.get_global_attributes()) + self.assertEqual( + self.base_manager.get_global_attributes(), manager.get_global_attributes() + ) diff --git a/tests/codice/l3/hi/direct_event/test_codice_hi_mass_correction_lookup_table.py b/tests/codice/l3/hi/direct_event/test_codice_hi_mass_correction_lookup_table.py index cabb8b3c1..cda7f56e4 100644 --- a/tests/codice/l3/hi/direct_event/test_codice_hi_mass_correction_lookup_table.py +++ b/tests/codice/l3/hi/direct_event/test_codice_hi_mass_correction_lookup_table.py @@ -1,28 +1,58 @@ import unittest -from imap_l3_processing.codice.l3.hi.direct_event.codice_hi_mass_correction_lookup_table import \ - CodiceHiMassCorrectionLookupTable +from imap_l3_processing.codice.l3.hi.direct_event.codice_hi_mass_correction_lookup_table import ( + CodiceHiMassCorrectionLookupTable, +) from tests.test_helpers import get_test_data_path class TestCodiceHiMassCorrectionLookupTable(unittest.TestCase): def test_from_file(self): - file_path = get_test_data_path("codice/imap_codice_l3-hi-mass-correction-lut-truncated_20251008_v001.xlsx") + file_path = get_test_data_path( + "codice/imap_codice_l3-hi-mass-correction-lut-truncated_20251008_v001.xlsx" + ) mass_correction_lookup = CodiceHiMassCorrectionLookupTable.from_file(file_path) self.assertIsInstance(mass_correction_lookup, CodiceHiMassCorrectionLookupTable) def test_lookup_correction_factor(self): - file_path = get_test_data_path("codice/imap_codice_l3-hi-mass-correction-lut-truncated_20251008_v001.xlsx") + file_path = get_test_data_path( + "codice/imap_codice_l3-hi-mass-correction-lut-truncated_20251008_v001.xlsx" + ) mass_correction_lookup = CodiceHiMassCorrectionLookupTable.from_file(file_path) - self.assertEqual(0.753777, mass_correction_lookup.lookup_correction_factor(ssd_id=7, energy_channel=40, tof=6)) - - self.assertEqual(7.47142, mass_correction_lookup.lookup_correction_factor(ssd_id=3, energy_channel=24, tof=0)) - self.assertEqual(1, mass_correction_lookup.lookup_correction_factor(ssd_id=3, energy_channel=24, tof=1)) - self.assertEqual(12.8534, mass_correction_lookup.lookup_correction_factor(ssd_id=3, energy_channel=24, tof=6)) + self.assertEqual( + 0.753777, + mass_correction_lookup.lookup_correction_factor( + ssd_id=7, energy_channel=40, tof=6 + ), + ) + + self.assertEqual( + 7.47142, + mass_correction_lookup.lookup_correction_factor( + ssd_id=3, energy_channel=24, tof=0 + ), + ) + self.assertEqual( + 1, + mass_correction_lookup.lookup_correction_factor( + ssd_id=3, energy_channel=24, tof=1 + ), + ) + self.assertEqual( + 12.8534, + mass_correction_lookup.lookup_correction_factor( + ssd_id=3, energy_channel=24, tof=6 + ), + ) missing_ssds = [2, 6, 8, 9, 10, 12, 13, 14, 15] for ssd_id in missing_ssds: - self.assertEqual(1.0, mass_correction_lookup.lookup_correction_factor(ssd_id, energy_channel=24, tof=0)) + self.assertEqual( + 1.0, + mass_correction_lookup.lookup_correction_factor( + ssd_id, energy_channel=24, tof=0 + ), + ) diff --git a/tests/codice/l3/hi/direct_event/test_codice_l3a_direct_events_dependencies.py b/tests/codice/l3/hi/direct_event/test_codice_l3a_direct_events_dependencies.py index e8415ccfe..5afa8df10 100644 --- a/tests/codice/l3/hi/direct_event/test_codice_l3a_direct_events_dependencies.py +++ b/tests/codice/l3/hi/direct_event/test_codice_l3a_direct_events_dependencies.py @@ -3,55 +3,103 @@ from unittest.mock import call, patch import imap_data_access -from imap_data_access.processing_input import ProcessingInputCollection, ScienceInput, AncillaryInput +from imap_data_access.processing_input import ( + ProcessingInputCollection, + ScienceInput, + AncillaryInput, +) -from imap_l3_processing.codice.l3.hi.direct_event.codice_hi_l3a_direct_events_dependencies import \ - CodiceHiL3aDirectEventsDependencies +from imap_l3_processing.codice.l3.hi.direct_event.codice_hi_l3a_direct_events_dependencies import ( + CodiceHiL3aDirectEventsDependencies, +) class TestCodiceHiL3aDirectEventDependencies(unittest.TestCase): - - @patch("imap_l3_processing.codice.l3.hi.direct_event.codice_hi_l3a_direct_events_dependencies.download") @patch( - "imap_l3_processing.codice.l3.hi.direct_event.codice_hi_l3a_direct_events_dependencies.CodiceHiL3aDirectEventsDependencies.from_file_paths") + "imap_l3_processing.codice.l3.hi.direct_event.codice_hi_l3a_direct_events_dependencies.download" + ) + @patch( + "imap_l3_processing.codice.l3.hi.direct_event.codice_hi_l3a_direct_events_dependencies.CodiceHiL3aDirectEventsDependencies.from_file_paths" + ) def test_fetch_dependencies(self, mock_from_file_paths, mock_download): input_collection = ProcessingInputCollection() - expected_codice_l2_direct_event_science_file_download_path = "imap/codice/l2/2010/01/imap_codice_l2_hi-direct-events_20100105_v010.cdf" - codice_l2_direct_event_input_file_name = "imap_codice_l2_hi-direct-events_20100105_v010.cdf" + expected_codice_l2_direct_event_science_file_download_path = ( + "imap/codice/l2/2010/01/imap_codice_l2_hi-direct-events_20100105_v010.cdf" + ) + codice_l2_direct_event_input_file_name = ( + "imap_codice_l2_hi-direct-events_20100105_v010.cdf" + ) - expected_codice_l1a_direct_event_science_file_download_path = "imap/codice/l1a/2010/10/imap_codice_l1a_hi-direct-events_20101005_v010.cdf" - codice_l1a_direct_event_input_file_name = "imap_codice_l1a_hi-direct-events_20101005_v010.cdf" + expected_codice_l1a_direct_event_science_file_download_path = ( + "imap/codice/l1a/2010/10/imap_codice_l1a_hi-direct-events_20101005_v010.cdf" + ) + codice_l1a_direct_event_input_file_name = ( + "imap_codice_l1a_hi-direct-events_20101005_v010.cdf" + ) - expected_codice_hi_mass_correction_lut_ancillary_file_download_path = "codice/imap_codice_l3-hi-mass-correction-lut_20100105_v010.xlsx" - codice_hi_mass_correction_lut_input_file_name = "imap_codice_l3-hi-mass-correction-lut_20100105_v010.xlsx" + expected_codice_hi_mass_correction_lut_ancillary_file_download_path = ( + "codice/imap_codice_l3-hi-mass-correction-lut_20100105_v010.xlsx" + ) + codice_hi_mass_correction_lut_input_file_name = ( + "imap_codice_l3-hi-mass-correction-lut_20100105_v010.xlsx" + ) - codice_l2_direct_event_science_input = ScienceInput(codice_l2_direct_event_input_file_name) - codice_l1a_direct_event_science_input = ScienceInput(codice_l1a_direct_event_input_file_name) - codice_hi_mass_correction_lut_ancillary_input = AncillaryInput(codice_hi_mass_correction_lut_input_file_name) + codice_l2_direct_event_science_input = ScienceInput( + codice_l2_direct_event_input_file_name + ) + codice_l1a_direct_event_science_input = ScienceInput( + codice_l1a_direct_event_input_file_name + ) + codice_hi_mass_correction_lut_ancillary_input = AncillaryInput( + codice_hi_mass_correction_lut_input_file_name + ) non_codice_ancillary_input = AncillaryInput( - "imap/ancillary/hit/imap_codice_range-2A-cosine-lookup_20250203_v001.csv") - non_codice_science_input = ScienceInput("imap/hit/l3/2010/01/imap_hit_l3_hi-direct-events_20100105_v010.cdf") - non_l2_codice_science_input = ScienceInput("imap_codice_l3_hi-direct-events_20100105_v010.cdf") + "imap/ancillary/hit/imap_codice_range-2A-cosine-lookup_20250203_v001.csv" + ) + non_codice_science_input = ScienceInput( + "imap/hit/l3/2010/01/imap_hit_l3_hi-direct-events_20100105_v010.cdf" + ) + non_l2_codice_science_input = ScienceInput( + "imap_codice_l3_hi-direct-events_20100105_v010.cdf" + ) - input_collection.add([codice_l1a_direct_event_science_input, codice_l2_direct_event_science_input, codice_hi_mass_correction_lut_ancillary_input, non_codice_ancillary_input, non_codice_science_input, - non_l2_codice_science_input]) + input_collection.add( + [ + codice_l1a_direct_event_science_input, + codice_l2_direct_event_science_input, + codice_hi_mass_correction_lut_ancillary_input, + non_codice_ancillary_input, + non_codice_science_input, + non_l2_codice_science_input, + ] + ) - codice_l3_dependencies = CodiceHiL3aDirectEventsDependencies.fetch_dependencies(input_collection) + codice_l3_dependencies = CodiceHiL3aDirectEventsDependencies.fetch_dependencies( + input_collection + ) data_dir = imap_data_access.config["DATA_DIR"] - expected_download_codice_l2_direct_event_science_path = data_dir / expected_codice_l2_direct_event_science_file_download_path - expected_download_codice_l1a_direct_event_science_path = data_dir / expected_codice_l1a_direct_event_science_file_download_path - expected_download_codice_hi_mass_correction_lut_ancillary_path = data_dir / "imap/ancillary" / expected_codice_hi_mass_correction_lut_ancillary_file_download_path - - - mock_download.assert_has_calls([ - call(expected_download_codice_l2_direct_event_science_path), - call(expected_download_codice_l1a_direct_event_science_path), - call(expected_download_codice_hi_mass_correction_lut_ancillary_path), - ]) + expected_download_codice_l2_direct_event_science_path = ( + data_dir / expected_codice_l2_direct_event_science_file_download_path + ) + expected_download_codice_l1a_direct_event_science_path = ( + data_dir / expected_codice_l1a_direct_event_science_file_download_path + ) + expected_download_codice_hi_mass_correction_lut_ancillary_path = ( + data_dir + / "imap/ancillary" + / expected_codice_hi_mass_correction_lut_ancillary_file_download_path + ) + mock_download.assert_has_calls( + [ + call(expected_download_codice_l2_direct_event_science_path), + call(expected_download_codice_l1a_direct_event_science_path), + call(expected_download_codice_hi_mass_correction_lut_ancillary_path), + ] + ) mock_from_file_paths.assert_called_with( expected_download_codice_l2_direct_event_science_path, @@ -61,20 +109,47 @@ def test_fetch_dependencies(self, mock_from_file_paths, mock_download): self.assertEqual(mock_from_file_paths.return_value, codice_l3_dependencies) - @patch("imap_l3_processing.codice.l3.hi.models.CodiceL2HiDirectEventData.read_from_cdf") - @patch("imap_l3_processing.codice.l3.hi.models.CodiceL1aHiDirectEvents.read_from_cdf") - @patch("imap_l3_processing.codice.l3.hi.direct_event.codice_hi_mass_correction_lookup_table.CodiceHiMassCorrectionLookupTable.from_file") - def test_can_load_from_files(self, mock_from_file_mass_correction_lut_ancillary, mock_l1a_read_from_cdf, mock_l2_read_from_cdf): + @patch( + "imap_l3_processing.codice.l3.hi.models.CodiceL2HiDirectEventData.read_from_cdf" + ) + @patch( + "imap_l3_processing.codice.l3.hi.models.CodiceL1aHiDirectEvents.read_from_cdf" + ) + @patch( + "imap_l3_processing.codice.l3.hi.direct_event.codice_hi_mass_correction_lookup_table.CodiceHiMassCorrectionLookupTable.from_file" + ) + def test_can_load_from_files( + self, + mock_from_file_mass_correction_lut_ancillary, + mock_l1a_read_from_cdf, + mock_l2_read_from_cdf, + ): codice_l2_cdf_file = Path("CodiceL2CDF") codice_l1a_cdf_file = Path("CodiceL1aCDF") - codice_mass_correction_lut_ancillary_file = Path("CodiceMassCorrectionLutAncillary") + codice_mass_correction_lut_ancillary_file = Path( + "CodiceMassCorrectionLutAncillary" + ) - codice_l3_dependencies = CodiceHiL3aDirectEventsDependencies.from_file_paths(codice_l2_cdf_file, codice_l1a_cdf_file, codice_mass_correction_lut_ancillary_file) + codice_l3_dependencies = CodiceHiL3aDirectEventsDependencies.from_file_paths( + codice_l2_cdf_file, + codice_l1a_cdf_file, + codice_mass_correction_lut_ancillary_file, + ) mock_l2_read_from_cdf.assert_called_with(codice_l2_cdf_file) mock_l1a_read_from_cdf.assert_called_with(codice_l1a_cdf_file) - mock_from_file_mass_correction_lut_ancillary.assert_called_with(codice_mass_correction_lut_ancillary_file) + mock_from_file_mass_correction_lut_ancillary.assert_called_with( + codice_mass_correction_lut_ancillary_file + ) - self.assertEqual(mock_l2_read_from_cdf.return_value, codice_l3_dependencies.codice_l2_hi_data) - self.assertEqual(mock_l1a_read_from_cdf.return_value, codice_l3_dependencies.codice_l1a_hi_data) - self.assertEqual(mock_from_file_mass_correction_lut_ancillary.return_value, codice_l3_dependencies.codice_hi_mass_correction_lut_ancillary) + self.assertEqual( + mock_l2_read_from_cdf.return_value, codice_l3_dependencies.codice_l2_hi_data + ) + self.assertEqual( + mock_l1a_read_from_cdf.return_value, + codice_l3_dependencies.codice_l1a_hi_data, + ) + self.assertEqual( + mock_from_file_mass_correction_lut_ancillary.return_value, + codice_l3_dependencies.codice_hi_mass_correction_lut_ancillary, + ) diff --git a/tests/codice/l3/hi/pitch_angle/test_codice_pitch_angle_dependencies.py b/tests/codice/l3/hi/pitch_angle/test_codice_pitch_angle_dependencies.py index 626cc1a90..bfebcf226 100644 --- a/tests/codice/l3/hi/pitch_angle/test_codice_pitch_angle_dependencies.py +++ b/tests/codice/l3/hi/pitch_angle/test_codice_pitch_angle_dependencies.py @@ -4,49 +4,79 @@ import imap_data_access from imap_data_access.processing_input import ProcessingInputCollection, ScienceInput -from imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies import CodicePitchAngleDependencies +from imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies import ( + CodicePitchAngleDependencies, +) class TestCodicePitchAngleDependencies(unittest.TestCase): - @patch( - "imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies.CodicePitchAngleDependencies.from_file_paths") - @patch("imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies.imap_data_access.download") + "imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies.CodicePitchAngleDependencies.from_file_paths" + ) + @patch( + "imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies.imap_data_access.download" + ) def test_fetch_dependencies(self, mock_download, mock_from_files): - expected_codice_science_file_download_path = "imap/codice/l2/2010/01/imap_codice_l2_hi-sectored_20100105_v010.cdf" - expected_mag_download_path = "imap/mag/l1d/2010/01/imap_mag_l1d_norm-dsrf_20100105_v010.cdf" + expected_codice_science_file_download_path = ( + "imap/codice/l2/2010/01/imap_codice_l2_hi-sectored_20100105_v010.cdf" + ) + expected_mag_download_path = ( + "imap/mag/l1d/2010/01/imap_mag_l1d_norm-dsrf_20100105_v010.cdf" + ) - science_input_codice_l2 = ScienceInput("imap_codice_l2_hi-sectored_20100105_v010.cdf") + science_input_codice_l2 = ScienceInput( + "imap_codice_l2_hi-sectored_20100105_v010.cdf" + ) science_input_mag_l1d = ScienceInput("imap_mag_l1d_norm-dsrf_20100105_v010.cdf") process_input_collection = ProcessingInputCollection() process_input_collection.add(science_input_codice_l2) process_input_collection.add(science_input_mag_l1d) - dependencies = CodicePitchAngleDependencies.fetch_dependencies(process_input_collection) + dependencies = CodicePitchAngleDependencies.fetch_dependencies( + process_input_collection + ) data_dir = imap_data_access.config["DATA_DIR"] - expected_codice_science_file_path = data_dir / expected_codice_science_file_download_path + expected_codice_science_file_path = ( + data_dir / expected_codice_science_file_download_path + ) expected_mag_science_file_path = data_dir / expected_mag_download_path - mock_download.assert_has_calls([ - call(expected_codice_science_file_path), - call(expected_mag_science_file_path) - ]) + mock_download.assert_has_calls( + [ + call(expected_codice_science_file_path), + call(expected_mag_science_file_path), + ] + ) - mock_from_files.assert_called_with(expected_mag_science_file_path, expected_codice_science_file_path, - mag_is_preliminary=True) + mock_from_files.assert_called_with( + expected_mag_science_file_path, + expected_codice_science_file_path, + mag_is_preliminary=True, + ) self.assertEqual(mock_from_files.return_value, dependencies) @patch( - "imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies.CodicePitchAngleDependencies.from_file_paths") - @patch("imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies.imap_data_access.download") - def test_fetch_dependencies_uses_mag_l2_if_available(self, mock_download, mock_from_files): - expected_codice_science_file_download_path = "imap/codice/l2/2010/01/imap_codice_l2_hi-sectored_20100105_v010.cdf" - expected_mag_download_path = "imap/mag/l2/2010/01/imap_mag_l2_norm-dsrf_20100105_v010.cdf" + "imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies.CodicePitchAngleDependencies.from_file_paths" + ) + @patch( + "imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies.imap_data_access.download" + ) + def test_fetch_dependencies_uses_mag_l2_if_available( + self, mock_download, mock_from_files + ): + expected_codice_science_file_download_path = ( + "imap/codice/l2/2010/01/imap_codice_l2_hi-sectored_20100105_v010.cdf" + ) + expected_mag_download_path = ( + "imap/mag/l2/2010/01/imap_mag_l2_norm-dsrf_20100105_v010.cdf" + ) - science_input_codice_l2 = ScienceInput("imap_codice_l2_hi-sectored_20100105_v010.cdf") + science_input_codice_l2 = ScienceInput( + "imap_codice_l2_hi-sectored_20100105_v010.cdf" + ) science_input_mag_l2 = ScienceInput("imap_mag_l2_norm-dsrf_20100105_v010.cdf") science_input_mag_l1d = ScienceInput("imap_mag_l1d_norm-dsrf_20100105_v010.cdf") @@ -56,34 +86,51 @@ def test_fetch_dependencies_uses_mag_l2_if_available(self, mock_download, mock_f science_input_mag_l2, ) - dependencies = CodicePitchAngleDependencies.fetch_dependencies(process_input_collection) + dependencies = CodicePitchAngleDependencies.fetch_dependencies( + process_input_collection + ) data_dir = imap_data_access.config["DATA_DIR"] - expected_codice_science_file_path = data_dir / expected_codice_science_file_download_path + expected_codice_science_file_path = ( + data_dir / expected_codice_science_file_download_path + ) expected_mag_science_file_path = data_dir / expected_mag_download_path - mock_download.assert_has_calls([ - call(expected_codice_science_file_path), - call(expected_mag_science_file_path) - ]) + mock_download.assert_has_calls( + [ + call(expected_codice_science_file_path), + call(expected_mag_science_file_path), + ] + ) - mock_from_files.assert_called_with(expected_mag_science_file_path, expected_codice_science_file_path, - mag_is_preliminary=False) + mock_from_files.assert_called_with( + expected_mag_science_file_path, + expected_codice_science_file_path, + mag_is_preliminary=False, + ) self.assertEqual(mock_from_files.return_value, dependencies) - @patch('imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies.read_mag_data') @patch( - 'imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies.CodiceHiL2SectoredIntensitiesData.read_from_cdf') + "imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies.read_mag_data" + ) + @patch( + "imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies.CodiceHiL2SectoredIntensitiesData.read_from_cdf" + ) def test_from_file_paths(self, mock_codice_l2_data, mock_read_mag_data): codice_file_path = sentinel.codice_file_path mag_file_path = sentinel.mag_file_path - dependencies = CodicePitchAngleDependencies.from_file_paths(mag_file_path, codice_file_path) + dependencies = CodicePitchAngleDependencies.from_file_paths( + mag_file_path, codice_file_path + ) mock_codice_l2_data.assert_called_once_with(codice_file_path) mock_read_mag_data.assert_called_once_with(mag_file_path) - self.assertEqual(mock_codice_l2_data.return_value, dependencies.codice_sectored_intensities_data) + self.assertEqual( + mock_codice_l2_data.return_value, + dependencies.codice_sectored_intensities_data, + ) self.assertEqual(mock_read_mag_data.return_value, dependencies.mag_data) self.assertFalse(dependencies.mag_is_preliminary) diff --git a/tests/codice/l3/hi/test_codice_hi_processor.py b/tests/codice/l3/hi/test_codice_hi_processor.py index e1a143e84..4fa7e523b 100644 --- a/tests/codice/l3/hi/test_codice_hi_processor.py +++ b/tests/codice/l3/hi/test_codice_hi_processor.py @@ -7,30 +7,45 @@ import numpy as np from imap_l3_processing.codice.l3.hi.codice_hi_processor import CodiceHiProcessor -from imap_l3_processing.codice.l3.hi.direct_event.codice_hi_l3a_direct_events_dependencies import \ - CodiceHiL3aDirectEventsDependencies -from imap_l3_processing.codice.l3.hi.models import CodiceL1aHiDirectEvents, CodiceL2HiDirectEventData, CodiceHiL2SectoredIntensitiesData, \ - CodiceL3HiDirectEvents -from imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies import CodicePitchAngleDependencies -from imap_l3_processing.codice.l3.lo.constants import CODICE_INSTRUMENT_TO_SC_FRAME_SPIN_ANGLE_OFFSET +from imap_l3_processing.codice.l3.hi.direct_event.codice_hi_l3a_direct_events_dependencies import ( + CodiceHiL3aDirectEventsDependencies, +) +from imap_l3_processing.codice.l3.hi.models import ( + CodiceL1aHiDirectEvents, + CodiceL2HiDirectEventData, + CodiceHiL2SectoredIntensitiesData, + CodiceL3HiDirectEvents, +) +from imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies import ( + CodicePitchAngleDependencies, +) +from imap_l3_processing.codice.l3.lo.constants import ( + CODICE_INSTRUMENT_TO_SC_FRAME_SPIN_ANGLE_OFFSET, +) from imap_l3_processing.codice.quality_flags import CodiceL3Flags from imap_l3_processing.models import InputMetadata from tests.test_helpers import NumpyArrayMatcher, get_test_data_path class TestCodiceHiProcessor(unittest.TestCase): - @patch("imap_l3_processing.codice.l3.hi.codice_hi_processor.CodiceHiL3aDirectEventsDependencies.fetch_dependencies") - @patch("imap_l3_processing.codice.l3.hi.codice_hi_processor.CodiceHiProcessor.process_l3a_direct_event") + @patch( + "imap_l3_processing.codice.l3.hi.codice_hi_processor.CodiceHiL3aDirectEventsDependencies.fetch_dependencies" + ) + @patch( + "imap_l3_processing.codice.l3.hi.codice_hi_processor.CodiceHiProcessor.process_l3a_direct_event" + ) @patch("imap_l3_processing.codice.l3.hi.codice_hi_processor.save_data") - def test_process_l3a(self, mock_save_data, mock_process_l3a, mock_fetch_dependencies): + def test_process_l3a( + self, mock_save_data, mock_process_l3a, mock_fetch_dependencies + ): start_date = datetime(2024, 10, 7, 10, 00, 00) end_date = datetime(2024, 10, 8, 10, 00, 00) - input_metadata = InputMetadata('codice', "l3a", start_date, end_date, 'v02') + input_metadata = InputMetadata("codice", "l3a", start_date, end_date, "v02") mock_processed_direct_events = Mock() mock_process_l3a.return_value = mock_processed_direct_events input_collection = MagicMock() - input_collection.get_file_paths.return_value = [Path('path/to/parent_file_1')] + input_collection.get_file_paths.return_value = [Path("path/to/parent_file_1")] processor = CodiceHiProcessor(input_collection, input_metadata) product = processor.process() @@ -39,13 +54,33 @@ def test_process_l3a(self, mock_save_data, mock_process_l3a, mock_fetch_dependen mock_process_l3a.assert_called_with(mock_fetch_dependencies.return_value) mock_save_data.assert_called_with(mock_processed_direct_events) self.assertEqual([mock_save_data.return_value], product) - self.assertEqual(['parent_file_1'], mock_processed_direct_events.parent_file_names) + self.assertEqual( + ["parent_file_1"], mock_processed_direct_events.parent_file_names + ) def test_process_l3a_with_small_dataset(self): high_gain_events = { - (0, 1, 0): {"ssd_id": 11, "energy_channel": 1, "tof": 21, "l2_energy": 5, "factor": 0.5}, - (0, 1, 1): {"ssd_id": 12, "energy_channel": 2, "tof": 22, "l2_energy": 6, "factor": 1.0}, - (1, 0, 2): {"ssd_id": 13, "energy_channel": 3, "tof": 23, "l2_energy": 3, "factor": 2.0}, + (0, 1, 0): { + "ssd_id": 11, + "energy_channel": 1, + "tof": 21, + "l2_energy": 5, + "factor": 0.5, + }, + (0, 1, 1): { + "ssd_id": 12, + "energy_channel": 2, + "tof": 22, + "l2_energy": 6, + "factor": 1.0, + }, + (1, 0, 2): { + "ssd_id": 13, + "energy_channel": 3, + "tof": 23, + "l2_energy": 3, + "factor": 2.0, + }, } expected_factors = { (event["ssd_id"], event["energy_channel"], event["tof"]): event["factor"] @@ -61,8 +96,9 @@ def lookup_factor(ssd_id, energy_channel, tof): mock_codice_hi_mass_correction_lut_ancillary = Mock() mock_codice_hi_mass_correction_lut_ancillary.lookup_correction_factor.side_effect = lookup_factor - codice_hi_processor = CodiceHiProcessor(sentinel.processing_input_collection, - input_metadata=sentinel.input_metadata) + codice_hi_processor = CodiceHiProcessor( + sentinel.processing_input_collection, input_metadata=sentinel.input_metadata + ) number_of_events = np.array([[0, 2], [4, 3]]) shape = (2, 2, 4) @@ -118,7 +154,7 @@ def lookup_factor(ssd_id, energy_channel, tof): dependencies = CodiceHiL3aDirectEventsDependencies( codice_l2_hi_data=l2_data, codice_l1a_hi_data=l1a_data, - codice_hi_mass_correction_lut_ancillary=mock_codice_hi_mass_correction_lut_ancillary + codice_hi_mass_correction_lut_ancillary=mock_codice_hi_mass_correction_lut_ancillary, ) expected_incident_ion_energy = np.full(shape, np.nan) @@ -150,41 +186,41 @@ def lookup_factor(ssd_id, energy_channel, tof): spin_sector=sentinel.l2_spin_sector, ) - actual_output = codice_hi_processor.process_l3a_direct_event(dependencies=dependencies) + actual_output = codice_hi_processor.process_l3a_direct_event( + dependencies=dependencies + ) for field in dataclasses.fields(CodiceL3HiDirectEvents): - np.testing.assert_equal(getattr(actual_output, field.name), getattr(expected_output, field.name)) + np.testing.assert_equal( + getattr(actual_output, field.name), getattr(expected_output, field.name) + ) self.assertEqual(0, len(expected_factors)) def test_process_l3a_only_calculates_events_within_num_events(self): - codice_hi_processor = CodiceHiProcessor(sentinel.processing_input_collection, - input_metadata=sentinel.input_metadata) + codice_hi_processor = CodiceHiProcessor( + sentinel.processing_input_collection, input_metadata=sentinel.input_metadata + ) - expected_number_of_events = np.array([ - [0, 2], [4, 3] - ]) + expected_number_of_events = np.array([[0, 2], [4, 3]]) - expected_ssd_energy = np.ma.masked_equal(np.array([ - [ - [255, 255, 255, 255], - [5, 6, 255, 255] - ], - [ - [1, 2, 3, 4], - [5, 6, 2, 255] - ] - ], dtype=float), 255) + expected_ssd_energy = np.ma.masked_equal( + np.array( + [ + [[255, 255, 255, 255], [5, 6, 255, 255]], + [[1, 2, 3, 4], [5, 6, 2, 255]], + ], + dtype=float, + ), + 255, + ) - expected_energy_per_nuc = np.array([ + expected_energy_per_nuc = np.array( [ - [255, 255, 255, 255], - [15, 16, 255, 255] + [[255, 255, 255, 255], [15, 16, 255, 255]], + [[11, 12, 13, 14], [15, 16, 12, 255]], ], - [ - [11, 12, 13, 14], - [15, 16, 12, 255] - ] - ], dtype=float) + dtype=float, + ) l2_gain = np.ones_like(expected_ssd_energy) l1a_data = CodiceL1aHiDirectEvents( @@ -231,40 +267,51 @@ def test_process_l3a_only_calculates_events_within_num_events(self): tof=sentinel.expected_time_of_flight, type=sentinel.expected_type, energy_per_nuc=expected_energy_per_nuc, - estimated_mass=np.array([ - [ - [np.nan, np.nan, np.nan, np.nan], - [5 / 15, 6 / 16, np.nan, np.nan] - ], + estimated_mass=np.array( [ - [1 / 11, 2 / 12, 3 / 13, 4 / 14], - [5 / 15, 6 / 16, 2 / 12, np.nan] + [ + [np.nan, np.nan, np.nan, np.nan], + [5 / 15, 6 / 16, np.nan, np.nan], + ], + [ + [1 / 11, 2 / 12, 3 / 13, 4 / 14], + [5 / 15, 6 / 16, 2 / 12, np.nan], + ], ] - ]), + ), elevation_angle=sentinel.l2_elevation_angle, gain=l2_gain, spin_sector=sentinel.l2_spin_sector, ) - actual_output = codice_hi_processor.process_l3a_direct_event(dependencies=dependencies) + actual_output = codice_hi_processor.process_l3a_direct_event( + dependencies=dependencies + ) mock_lut.lookup_correction_factor.assert_not_called() for field in dataclasses.fields(CodiceL3HiDirectEvents): - np.testing.assert_equal(getattr(actual_output, field.name), getattr(expected_output, field.name)) + np.testing.assert_equal( + getattr(actual_output, field.name), getattr(expected_output, field.name) + ) - @patch("imap_l3_processing.codice.l3.hi.codice_hi_processor.CodicePitchAngleDependencies.fetch_dependencies") - @patch("imap_l3_processing.codice.l3.hi.codice_hi_processor.CodiceHiProcessor.process_l3b") + @patch( + "imap_l3_processing.codice.l3.hi.codice_hi_processor.CodicePitchAngleDependencies.fetch_dependencies" + ) + @patch( + "imap_l3_processing.codice.l3.hi.codice_hi_processor.CodiceHiProcessor.process_l3b" + ) @patch("imap_l3_processing.codice.l3.hi.codice_hi_processor.save_data") - def test_process_l3b_saves(self, mock_save_data, mock_process_l3b, - mock_fetch_dependencies): + def test_process_l3b_saves( + self, mock_save_data, mock_process_l3b, mock_fetch_dependencies + ): start_date = datetime(2024, 10, 7, 10, 00, 00) end_date = datetime(2024, 10, 8, 10, 00, 00) - input_metadata = InputMetadata('codice', "l3b", start_date, end_date, 'v02') + input_metadata = InputMetadata("codice", "l3b", start_date, end_date, "v02") mock_processed_pitch_angles = Mock() mock_process_l3b.return_value = mock_processed_pitch_angles input_collection = MagicMock() - input_collection.get_file_paths.return_value = [Path('path/to/parent_file_2')] + input_collection.get_file_paths.return_value = [Path("path/to/parent_file_2")] processor = CodiceHiProcessor(input_collection, input_metadata) product = processor.process() @@ -273,10 +320,12 @@ def test_process_l3b_saves(self, mock_save_data, mock_process_l3b, mock_process_l3b.assert_called_with(mock_fetch_dependencies.return_value) mock_save_data.assert_called_with(mock_processed_pitch_angles) self.assertEqual([mock_save_data.return_value], product) - self.assertEqual(['parent_file_2'], mock_processed_pitch_angles.parent_file_names) + self.assertEqual( + ["parent_file_2"], mock_processed_pitch_angles.parent_file_names + ) def test_raises_exception_on_non_l3_input_metadata(self): - input_metadata = InputMetadata('codice', "L2a", Mock(), Mock(), 'v02') + input_metadata = InputMetadata("codice", "L2a", Mock(), Mock(), "v02") processor = CodiceHiProcessor(Mock(), input_metadata) with self.assertRaises(NotImplementedError) as context: @@ -284,32 +333,64 @@ def test_raises_exception_on_non_l3_input_metadata(self): self.assertEqual("Unknown data level for CoDICE: L2a", str(context.exception)) @patch("imap_l3_processing.codice.l3.hi.codice_hi_processor.calculate_unit_vector") - @patch("imap_l3_processing.codice.l3.hi.codice_hi_processor.get_sector_unit_vectors") + @patch( + "imap_l3_processing.codice.l3.hi.codice_hi_processor.get_sector_unit_vectors" + ) @patch("imap_l3_processing.codice.l3.hi.codice_hi_processor.calculate_pitch_angle") @patch("imap_l3_processing.codice.l3.hi.codice_hi_processor.calculate_gyrophase") - @patch("imap_l3_processing.codice.l3.hi.codice_hi_processor.rebin_by_pitch_angle_and_gyrophase") - def test_process_l3b_with_mocks(self, mock_rebin_by_pitch_angle_and_gyrophase, mock_calculate_gyrophases, - mock_calculate_pitch_angles, mock_get_sector_unit_vectors, - mock_calculate_unit_vector): + @patch( + "imap_l3_processing.codice.l3.hi.codice_hi_processor.rebin_by_pitch_angle_and_gyrophase" + ) + def test_process_l3b_with_mocks( + self, + mock_rebin_by_pitch_angle_and_gyrophase, + mock_calculate_gyrophases, + mock_calculate_pitch_angles, + mock_get_sector_unit_vectors, + mock_calculate_unit_vector, + ): rng = np.random.default_rng() epoch_1 = datetime(2025, 2, 5) epoch_2 = datetime(2025, 2, 6) epoch = np.array([epoch_1, epoch_2]) - epoch_delta = np.full(epoch.shape, timedelta(days=.5)) + epoch_delta = np.full(epoch.shape, timedelta(days=0.5)) rebinned_mag_data = rng.random((len(epoch), 3)) mag_l1d_data = Mock() mag_l1d_data.rebin_to.return_value = rebinned_mag_data - h_intensity = rng.random((len(epoch), len(np.array([1.11, 1.17]) * 1.4), len(np.array([270, 15])), len( - np.array([15, 45, 75, 105, 135, 165])))) - he3he4_intensity = rng.random((len(epoch), len(np.array([1.11, 1.17]) * 1.4), len(np.array([270, 15])), len( - np.array([15, 45, 75, 105, 135, 165])))) - cno_intensity = rng.random((len(epoch), len(np.array([1.11, 1.17, 1.25])), len(np.array([270, 15])), len( - np.array([15, 45, 75, 105, 135, 165])))) + h_intensity = rng.random( + ( + len(epoch), + len(np.array([1.11, 1.17]) * 1.4), + len(np.array([270, 15])), + len(np.array([15, 45, 75, 105, 135, 165])), + ) + ) + he3he4_intensity = rng.random( + ( + len(epoch), + len(np.array([1.11, 1.17]) * 1.4), + len(np.array([270, 15])), + len(np.array([15, 45, 75, 105, 135, 165])), + ) + ) + cno_intensity = rng.random( + ( + len(epoch), + len(np.array([1.11, 1.17, 1.25])), + len(np.array([270, 15])), + len(np.array([15, 45, 75, 105, 135, 165])), + ) + ) fe_intensity = rng.random( - (len(epoch), len(np.array([1.11, 1.17]) * 1.3), len(np.array([270, 15])), - len(np.array([15, 45, 75, 105, 135, 165])))) + ( + len(epoch), + len(np.array([1.11, 1.17]) * 1.3), + len(np.array([270, 15])), + len(np.array([15, 45, 75, 105, 135, 165])), + ) + ) codice_l2_data = CodiceHiL2SectoredIntensitiesData( epoch=epoch, @@ -335,16 +416,18 @@ def test_process_l3b_with_mocks(self, mock_rebin_by_pitch_angle_and_gyrophase, m energy_he3he4_minus=(np.repeat(1.6, len(np.array([1.11, 1.17])))), ) - dependencies = CodicePitchAngleDependencies(mag_data=mag_l1d_data, - codice_sectored_intensities_data=codice_l2_data, - mag_is_preliminary=True) + dependencies = CodicePitchAngleDependencies( + mag_data=mag_l1d_data, + codice_sectored_intensities_data=codice_l2_data, + mag_is_preliminary=True, + ) expected_pitch_angles = np.linspace(15, 165, 6) expected_gyrophase = np.linspace(15, 345, 12) expected_pitch_angle_delta = np.repeat(15, 6) expected_gyrophase_delta = np.repeat(15, 12) - sector_vectors = rng.random((2,6,3)) + sector_vectors = rng.random((2, 6, 3)) sector_unit_vectors = rng.random((6, 2, 3)) mag_unit_vectors = rng.random((2, 3)) mock_calculate_unit_vector.side_effect = [mag_unit_vectors, sector_unit_vectors] @@ -373,36 +456,126 @@ def test_process_l3b_with_mocks(self, mock_rebin_by_pitch_angle_and_gyrophase, m expected_parents = [] mock_rebin_by_pitch_angle_and_gyrophase.side_effect = [ - (expected_h_intensity_binned_by_pa_and_gyro[0], 0, 0, expected_h_intensity_binned_by_pa[0], 0, 0), - (expected_he4_intensity_binned_by_pa_and_gyro[0], 0, 0, expected_he4_intensity_binned_by_pa[0], 0, 0), - (expected_cno_intensity_binned_by_pa_and_gyro[0], 0, 0, expected_cno_intensity_binned_by_pa[0], 0, 0), - (expected_fe_intensity_binned_by_pa_and_gyro[0], 0, 0, expected_fe_intensity_binned_by_pa[0], 0, 0), - (expected_h_intensity_binned_by_pa_and_gyro[1], 0, 0, expected_h_intensity_binned_by_pa[1], 0, 0), - (expected_he4_intensity_binned_by_pa_and_gyro[1], 0, 0, expected_he4_intensity_binned_by_pa[1], 0, 0), - (expected_cno_intensity_binned_by_pa_and_gyro[1], 0, 0, expected_cno_intensity_binned_by_pa[1], 0, 0), - (expected_fe_intensity_binned_by_pa_and_gyro[1], 0, 0, expected_fe_intensity_binned_by_pa[1], 0, 0), + ( + expected_h_intensity_binned_by_pa_and_gyro[0], + 0, + 0, + expected_h_intensity_binned_by_pa[0], + 0, + 0, + ), + ( + expected_he4_intensity_binned_by_pa_and_gyro[0], + 0, + 0, + expected_he4_intensity_binned_by_pa[0], + 0, + 0, + ), + ( + expected_cno_intensity_binned_by_pa_and_gyro[0], + 0, + 0, + expected_cno_intensity_binned_by_pa[0], + 0, + 0, + ), + ( + expected_fe_intensity_binned_by_pa_and_gyro[0], + 0, + 0, + expected_fe_intensity_binned_by_pa[0], + 0, + 0, + ), + ( + expected_h_intensity_binned_by_pa_and_gyro[1], + 0, + 0, + expected_h_intensity_binned_by_pa[1], + 0, + 0, + ), + ( + expected_he4_intensity_binned_by_pa_and_gyro[1], + 0, + 0, + expected_he4_intensity_binned_by_pa[1], + 0, + 0, + ), + ( + expected_cno_intensity_binned_by_pa_and_gyro[1], + 0, + 0, + expected_cno_intensity_binned_by_pa[1], + 0, + 0, + ), + ( + expected_fe_intensity_binned_by_pa_and_gyro[1], + 0, + 0, + expected_fe_intensity_binned_by_pa[1], + 0, + 0, + ), ] - codice_processor = CodiceHiProcessor(dependencies=Mock(), input_metadata=sentinel.input_metadata) + codice_processor = CodiceHiProcessor( + dependencies=Mock(), input_metadata=sentinel.input_metadata + ) codice_hi_data_product = codice_processor.process_l3b(dependencies=dependencies) mock_get_sector_unit_vectors.assert_called_once_with( codice_l2_data.elevation_angle, - NumpyArrayMatcher((codice_l2_data.spin_angles.T + CODICE_INSTRUMENT_TO_SC_FRAME_SPIN_ANGLE_OFFSET) % 360)) + NumpyArrayMatcher( + ( + codice_l2_data.spin_angles.T + + CODICE_INSTRUMENT_TO_SC_FRAME_SPIN_ANGLE_OFFSET + ) + % 360 + ), + ) mock_calculate_unit_vector.assert_has_calls( - [call(NumpyArrayMatcher(rebinned_mag_data)), call(NumpyArrayMatcher(sector_vectors.transpose(1,0,2)))]) + [ + call(NumpyArrayMatcher(rebinned_mag_data)), + call(NumpyArrayMatcher(sector_vectors.transpose(1, 0, 2))), + ] + ) - mock_calculate_pitch_angles.assert_has_calls([ - call(NumpyArrayMatcher(-1 * sector_unit_vectors), NumpyArrayMatcher(mag_unit_vectors[0])), - call(NumpyArrayMatcher(-1 * sector_unit_vectors), NumpyArrayMatcher(mag_unit_vectors[1])), - ]) + mock_calculate_pitch_angles.assert_has_calls( + [ + call( + NumpyArrayMatcher(-1 * sector_unit_vectors), + NumpyArrayMatcher(mag_unit_vectors[0]), + ), + call( + NumpyArrayMatcher(-1 * sector_unit_vectors), + NumpyArrayMatcher(mag_unit_vectors[1]), + ), + ] + ) - mock_calculate_gyrophases.assert_has_calls([ - call(NumpyArrayMatcher(-1 * sector_unit_vectors), NumpyArrayMatcher(mag_unit_vectors[0])), - call(NumpyArrayMatcher(-1 * sector_unit_vectors), NumpyArrayMatcher(mag_unit_vectors[1])), - ]) + mock_calculate_gyrophases.assert_has_calls( + [ + call( + NumpyArrayMatcher(-1 * sector_unit_vectors), + NumpyArrayMatcher(mag_unit_vectors[0]), + ), + call( + NumpyArrayMatcher(-1 * sector_unit_vectors), + NumpyArrayMatcher(mag_unit_vectors[1]), + ), + ] + ) - species_intensities = [h_intensity, he3he4_intensity, cno_intensity, fe_intensity] + species_intensities = [ + h_intensity, + he3he4_intensity, + cno_intensity, + fe_intensity, + ] species_uncertainties = [ np.zeros_like(h_intensity), @@ -412,27 +585,34 @@ def test_process_l3b_with_mocks(self, mock_rebin_by_pitch_angle_and_gyrophase, m ] expected_rebin_calls = [] - for species_intensity, species_uncertainty in zip(species_intensities, species_uncertainties): + for species_intensity, species_uncertainty in zip( + species_intensities, species_uncertainties + ): expected_rebin_calls.append( - call(intensity_data=NumpyArrayMatcher(species_intensity[0]), - intensity_delta_plus=NumpyArrayMatcher(species_uncertainty[0]), - intensity_delta_minus=NumpyArrayMatcher(species_uncertainty[0]), - pitch_angles=sentinel.epoch1_pitch_angle, - gyrophases=sentinel.epoch1_gyrophase, - number_of_pitch_angle_bins=6, number_of_gyrophase_bins=12 - ) - + call( + intensity_data=NumpyArrayMatcher(species_intensity[0]), + intensity_delta_plus=NumpyArrayMatcher(species_uncertainty[0]), + intensity_delta_minus=NumpyArrayMatcher(species_uncertainty[0]), + pitch_angles=sentinel.epoch1_pitch_angle, + gyrophases=sentinel.epoch1_gyrophase, + number_of_pitch_angle_bins=6, + number_of_gyrophase_bins=12, + ) ) - for species_intensity, species_uncertainty in zip(species_intensities, species_uncertainties): + for species_intensity, species_uncertainty in zip( + species_intensities, species_uncertainties + ): expected_rebin_calls.append( - call(intensity_data=NumpyArrayMatcher(species_intensity[1]), - intensity_delta_plus=NumpyArrayMatcher(species_uncertainty[1]), - intensity_delta_minus=NumpyArrayMatcher(species_uncertainty[1]), - pitch_angles=sentinel.epoch2_pitch_angle, - gyrophases=sentinel.epoch2_gyrophase, - number_of_pitch_angle_bins=6, number_of_gyrophase_bins=12 - ) + call( + intensity_data=NumpyArrayMatcher(species_intensity[1]), + intensity_delta_plus=NumpyArrayMatcher(species_uncertainty[1]), + intensity_delta_minus=NumpyArrayMatcher(species_uncertainty[1]), + pitch_angles=sentinel.epoch2_pitch_angle, + gyrophases=sentinel.epoch2_gyrophase, + number_of_pitch_angle_bins=6, + number_of_gyrophase_bins=12, + ) ) mock_rebin_by_pitch_angle_and_gyrophase.assert_has_calls(expected_rebin_calls) @@ -441,63 +621,129 @@ def test_process_l3b_with_mocks(self, mock_rebin_by_pitch_angle_and_gyrophase, m np.testing.assert_array_equal(epoch, codice_hi_data_product.epoch) np.testing.assert_array_equal(epoch_delta, codice_hi_data_product.epoch_delta) - np.testing.assert_array_equal(codice_hi_data_product.energy_h, codice_l2_data.energy_h) - np.testing.assert_array_equal(codice_hi_data_product.energy_h_plus, codice_l2_data.energy_h_plus) - np.testing.assert_array_equal(codice_hi_data_product.energy_h_minus, codice_l2_data.energy_h_minus) - np.testing.assert_array_equal(codice_hi_data_product.energy_cno, codice_l2_data.energy_cno) - np.testing.assert_array_equal(codice_hi_data_product.energy_cno_plus, codice_l2_data.energy_cno_plus) - np.testing.assert_array_equal(codice_hi_data_product.energy_cno_minus, codice_l2_data.energy_cno_minus) - np.testing.assert_array_equal(codice_hi_data_product.energy_fe, codice_l2_data.energy_fe) - np.testing.assert_array_equal(codice_hi_data_product.energy_fe_plus, codice_l2_data.energy_fe_plus) - np.testing.assert_array_equal(codice_hi_data_product.energy_fe_minus, codice_l2_data.energy_fe_minus) - np.testing.assert_array_equal(codice_hi_data_product.energy_he3he4, codice_l2_data.energy_he3he4) - np.testing.assert_array_equal(codice_hi_data_product.energy_he3he4_plus, codice_l2_data.energy_he3he4_plus) - np.testing.assert_array_equal(codice_hi_data_product.energy_he3he4_minus, codice_l2_data.energy_he3he4_minus) - - np.testing.assert_array_almost_equal(expected_pitch_angles, codice_hi_data_product.pitch_angle) - np.testing.assert_array_equal(expected_pitch_angle_delta, codice_hi_data_product.pitch_angle_delta) - np.testing.assert_array_almost_equal(expected_gyrophase, codice_hi_data_product.gyrophase) - np.testing.assert_array_equal(expected_gyrophase_delta, codice_hi_data_product.gyrophase_delta) - - np.testing.assert_allclose(codice_hi_data_product.h_intensity_by_pitch_angle, expected_h_intensity_binned_by_pa) - - np.testing.assert_array_equal(codice_hi_data_product.he3he4_intensity_by_pitch_angle, - expected_he4_intensity_binned_by_pa) - np.testing.assert_array_equal(codice_hi_data_product.cno_intensity_by_pitch_angle, - expected_cno_intensity_binned_by_pa) - np.testing.assert_array_equal(codice_hi_data_product.fe_intensity_by_pitch_angle, - expected_fe_intensity_binned_by_pa) - - np.testing.assert_array_equal(codice_hi_data_product.h_intensity_by_pitch_angle_and_gyrophase, - expected_h_intensity_binned_by_pa_and_gyro) - np.testing.assert_array_equal(codice_hi_data_product.he3he4_intensity_by_pitch_angle_and_gyrophase, - expected_he4_intensity_binned_by_pa_and_gyro) - np.testing.assert_array_equal(codice_hi_data_product.cno_intensity_by_pitch_angle_and_gyrophase, - expected_cno_intensity_binned_by_pa_and_gyro) - np.testing.assert_array_equal(codice_hi_data_product.fe_intensity_by_pitch_angle_and_gyrophase, - expected_fe_intensity_binned_by_pa_and_gyro) + np.testing.assert_array_equal( + codice_hi_data_product.energy_h, codice_l2_data.energy_h + ) + np.testing.assert_array_equal( + codice_hi_data_product.energy_h_plus, codice_l2_data.energy_h_plus + ) + np.testing.assert_array_equal( + codice_hi_data_product.energy_h_minus, codice_l2_data.energy_h_minus + ) + np.testing.assert_array_equal( + codice_hi_data_product.energy_cno, codice_l2_data.energy_cno + ) + np.testing.assert_array_equal( + codice_hi_data_product.energy_cno_plus, codice_l2_data.energy_cno_plus + ) + np.testing.assert_array_equal( + codice_hi_data_product.energy_cno_minus, codice_l2_data.energy_cno_minus + ) + np.testing.assert_array_equal( + codice_hi_data_product.energy_fe, codice_l2_data.energy_fe + ) + np.testing.assert_array_equal( + codice_hi_data_product.energy_fe_plus, codice_l2_data.energy_fe_plus + ) + np.testing.assert_array_equal( + codice_hi_data_product.energy_fe_minus, codice_l2_data.energy_fe_minus + ) + np.testing.assert_array_equal( + codice_hi_data_product.energy_he3he4, codice_l2_data.energy_he3he4 + ) + np.testing.assert_array_equal( + codice_hi_data_product.energy_he3he4_plus, codice_l2_data.energy_he3he4_plus + ) + np.testing.assert_array_equal( + codice_hi_data_product.energy_he3he4_minus, + codice_l2_data.energy_he3he4_minus, + ) + + np.testing.assert_array_almost_equal( + expected_pitch_angles, codice_hi_data_product.pitch_angle + ) + np.testing.assert_array_equal( + expected_pitch_angle_delta, codice_hi_data_product.pitch_angle_delta + ) + np.testing.assert_array_almost_equal( + expected_gyrophase, codice_hi_data_product.gyrophase + ) + np.testing.assert_array_equal( + expected_gyrophase_delta, codice_hi_data_product.gyrophase_delta + ) + + np.testing.assert_allclose( + codice_hi_data_product.h_intensity_by_pitch_angle, + expected_h_intensity_binned_by_pa, + ) + + np.testing.assert_array_equal( + codice_hi_data_product.he3he4_intensity_by_pitch_angle, + expected_he4_intensity_binned_by_pa, + ) + np.testing.assert_array_equal( + codice_hi_data_product.cno_intensity_by_pitch_angle, + expected_cno_intensity_binned_by_pa, + ) + np.testing.assert_array_equal( + codice_hi_data_product.fe_intensity_by_pitch_angle, + expected_fe_intensity_binned_by_pa, + ) + + np.testing.assert_array_equal( + codice_hi_data_product.h_intensity_by_pitch_angle_and_gyrophase, + expected_h_intensity_binned_by_pa_and_gyro, + ) + np.testing.assert_array_equal( + codice_hi_data_product.he3he4_intensity_by_pitch_angle_and_gyrophase, + expected_he4_intensity_binned_by_pa_and_gyro, + ) + np.testing.assert_array_equal( + codice_hi_data_product.cno_intensity_by_pitch_angle_and_gyrophase, + expected_cno_intensity_binned_by_pa_and_gyro, + ) + np.testing.assert_array_equal( + codice_hi_data_product.fe_intensity_by_pitch_angle_and_gyrophase, + expected_fe_intensity_binned_by_pa_and_gyro, + ) np.testing.assert_array_equal( codice_hi_data_product.codice_flags, np.array([CodiceL3Flags.PRELIMINARY_MAG, CodiceL3Flags.PRELIMINARY_MAG]), ) - np.testing.assert_array_equal(codice_hi_data_product.parent_file_names, expected_parents) + np.testing.assert_array_equal( + codice_hi_data_product.parent_file_names, expected_parents + ) def test_integration_test(self): - l2_direct_event_sci_path = get_test_data_path("codice/imap_codice_l2_hi-direct-events_20260831_v001.0003.cdf") - l1a_direct_event_sci_path = get_test_data_path("codice/imap_codice_l1a_hi-direct-events_20260831_v001.0003.cdf") - mass_correction_lut_ancillary_path = get_test_data_path("codice/imap_codice_l3-hi-mass-correction-lut-truncated_20251008_v001.xlsx") - - codice_hi_dependencies = CodiceHiL3aDirectEventsDependencies.from_file_paths(l2_direct_event_sci_path, l1a_direct_event_sci_path, mass_correction_lut_ancillary_path) - - input_metadata = InputMetadata(instrument='codice', - data_level="l3a", - start_date=Mock(spec=datetime), - end_date=Mock(spec=datetime), - version='v02', - descriptor='hi-direct-events') - processor = CodiceHiProcessor(dependencies=Mock(), input_metadata=input_metadata) + l2_direct_event_sci_path = get_test_data_path( + "codice/imap_codice_l2_hi-direct-events_20260831_v001.0003.cdf" + ) + l1a_direct_event_sci_path = get_test_data_path( + "codice/imap_codice_l1a_hi-direct-events_20260831_v001.0003.cdf" + ) + mass_correction_lut_ancillary_path = get_test_data_path( + "codice/imap_codice_l3-hi-mass-correction-lut-truncated_20251008_v001.xlsx" + ) + + codice_hi_dependencies = CodiceHiL3aDirectEventsDependencies.from_file_paths( + l2_direct_event_sci_path, + l1a_direct_event_sci_path, + mass_correction_lut_ancillary_path, + ) + + input_metadata = InputMetadata( + instrument="codice", + data_level="l3a", + start_date=Mock(spec=datetime), + end_date=Mock(spec=datetime), + version="v02", + descriptor="hi-direct-events", + ) + processor = CodiceHiProcessor( + dependencies=Mock(), input_metadata=input_metadata + ) try: processor.process_l3a_direct_event(codice_hi_dependencies) diff --git a/tests/codice/l3/hi/test_models.py b/tests/codice/l3/hi/test_models.py index 02a285222..5243635f2 100644 --- a/tests/codice/l3/hi/test_models.py +++ b/tests/codice/l3/hi/test_models.py @@ -8,24 +8,37 @@ import numpy as np from spacepy.pycdf import CDF -from imap_l3_processing.codice.l3.hi.models import CodiceL2HiDirectEventData, CodiceL3HiDirectEvents, \ - CodiceHiL2SectoredIntensitiesData, CodiceHiL3PitchAngleDataProduct, CodiceL1aHiDirectEvents -from tests.test_helpers import get_test_instrument_team_data_path, get_test_data_path, with_tempdir +from imap_l3_processing.codice.l3.hi.models import ( + CodiceL2HiDirectEventData, + CodiceL3HiDirectEvents, + CodiceHiL2SectoredIntensitiesData, + CodiceHiL3PitchAngleDataProduct, + CodiceL1aHiDirectEvents, +) +from tests.test_helpers import ( + get_test_instrument_team_data_path, + get_test_data_path, + with_tempdir, +) class TestModels(unittest.TestCase): def setUp(self): - if os.path.exists('test_cdf.cdf'): - os.remove('test_cdf.cdf') + if os.path.exists("test_cdf.cdf"): + os.remove("test_cdf.cdf") def tearDown(self): - if os.path.exists('test_cdf.cdf'): - os.remove('test_cdf.cdf') + if os.path.exists("test_cdf.cdf"): + os.remove("test_cdf.cdf") @with_tempdir def test_codice_hi_l2_data_read_from_instrument_team_cdf(self, tempdir): - l2_path = get_test_data_path("codice/imap_codice_l2_hi-direct-events_20250814_v001.cdf") - expanded_l2_path = TestModels._create_expanded_l2_hi_de_dataset(l2_path, tempdir) + l2_path = get_test_data_path( + "codice/imap_codice_l2_hi-direct-events_20250814_v001.cdf" + ) + expanded_l2_path = TestModels._create_expanded_l2_hi_de_dataset( + l2_path, tempdir + ) test_cases = [ ("6 priorities", l2_path), @@ -34,46 +47,80 @@ def test_codice_hi_l2_data_read_from_instrument_team_cdf(self, tempdir): for name, input_path in test_cases: with self.subTest(name): - l2_direct_event_data = CodiceL2HiDirectEventData.read_from_cdf(input_path) + l2_direct_event_data = CodiceL2HiDirectEventData.read_from_cdf( + input_path + ) with CDF(str(l2_path)) as cdf: # @formatter:off - np.testing.assert_array_equal(l2_direct_event_data.epoch, cdf["epoch"]) - np.testing.assert_array_equal(l2_direct_event_data.epoch_delta_plus, cdf["epoch_delta_plus"]) - np.testing.assert_array_equal(l2_direct_event_data.ssd_energy, cdf["ssd_energy"]) + np.testing.assert_array_equal( + l2_direct_event_data.epoch, cdf["epoch"] + ) + np.testing.assert_array_equal( + l2_direct_event_data.epoch_delta_plus, cdf["epoch_delta_plus"] + ) + np.testing.assert_array_equal( + l2_direct_event_data.ssd_energy, cdf["ssd_energy"] + ) expected_ssd_id = cdf["ssd_id"][...] - np.testing.assert_array_equal(l2_direct_event_data.ssd_id, expected_ssd_id) + np.testing.assert_array_equal( + l2_direct_event_data.ssd_id, expected_ssd_id + ) expected_energy_per_nuc = cdf["energy_per_nuc"][...] - expected_energy_per_nuc[expected_energy_per_nuc==-1e31] = np.nan - np.testing.assert_array_equal(l2_direct_event_data.energy_per_nuc, expected_energy_per_nuc) + expected_energy_per_nuc[expected_energy_per_nuc == -1e31] = np.nan + np.testing.assert_array_equal( + l2_direct_event_data.energy_per_nuc, expected_energy_per_nuc + ) expected_spin_angle = cdf["spin_angle"][...] - expected_spin_angle[expected_spin_angle==-1e31] = np.nan - np.testing.assert_array_equal(l2_direct_event_data.spin_angle, expected_spin_angle) + expected_spin_angle[expected_spin_angle == -1e31] = np.nan + np.testing.assert_array_equal( + l2_direct_event_data.spin_angle, expected_spin_angle + ) expected_spin_number = cdf["spin_number"][...] - np.testing.assert_array_equal(l2_direct_event_data.spin_number, expected_spin_number) + np.testing.assert_array_equal( + l2_direct_event_data.spin_number, expected_spin_number + ) expected_time_of_flight = cdf["tof"][...] - expected_time_of_flight[expected_time_of_flight==-1e31] = np.nan - np.testing.assert_array_equal(l2_direct_event_data.time_of_flight, expected_time_of_flight) - - np.testing.assert_array_equal(l2_direct_event_data.number_of_events, cdf["num_events"]) - np.testing.assert_array_equal(l2_direct_event_data.data_quality, cdf["data_quality"]) - np.testing.assert_array_equal(l2_direct_event_data.multi_flag, cdf["multi_flag"]) - - np.testing.assert_array_equal(l2_direct_event_data.type, cdf["type"]) + expected_time_of_flight[expected_time_of_flight == -1e31] = np.nan + np.testing.assert_array_equal( + l2_direct_event_data.time_of_flight, expected_time_of_flight + ) + + np.testing.assert_array_equal( + l2_direct_event_data.number_of_events, cdf["num_events"] + ) + np.testing.assert_array_equal( + l2_direct_event_data.data_quality, cdf["data_quality"] + ) + np.testing.assert_array_equal( + l2_direct_event_data.multi_flag, cdf["multi_flag"] + ) + + np.testing.assert_array_equal( + l2_direct_event_data.type, cdf["type"] + ) expected_elevation_angle = cdf["elevation_angle"][...] expected_elevation_angle[expected_elevation_angle == -1e31] = np.nan - np.testing.assert_array_equal(l2_direct_event_data.elevation_angle, expected_elevation_angle) + np.testing.assert_array_equal( + l2_direct_event_data.elevation_angle, expected_elevation_angle + ) self.assertIsInstance(l2_direct_event_data.gain, np.ma.masked_array) - np.testing.assert_array_equal(l2_direct_event_data.gain.data, cdf["gain"]) - - self.assertIsInstance(l2_direct_event_data.spin_sector, np.ma.masked_array) - np.testing.assert_array_equal(l2_direct_event_data.spin_sector.data, cdf["spin_sector"]) + np.testing.assert_array_equal( + l2_direct_event_data.gain.data, cdf["gain"] + ) + + self.assertIsInstance( + l2_direct_event_data.spin_sector, np.ma.masked_array + ) + np.testing.assert_array_equal( + l2_direct_event_data.spin_sector.data, cdf["spin_sector"] + ) # @formatter:on def test_codice_l3_hi_direct_event_data_products(self): @@ -84,12 +131,20 @@ def test_codice_l3_hi_direct_event_data_products(self): number_of_priority_events = 6 event_buffer_size = 3 - event_data_shape = (len(expected_epoch), number_of_priority_events, event_buffer_size) + event_data_shape = ( + len(expected_epoch), + number_of_priority_events, + event_buffer_size, + ) expected_priority_index = np.arange(number_of_priority_events) expected_event_index = np.arange(event_buffer_size) - expected_priority_index_label = np.array([str(i) for i in range(number_of_priority_events)]) - expected_event_index_label = np.array([str(i) for i in range(event_buffer_size)]) + expected_priority_index_label = np.array( + [str(i) for i in range(number_of_priority_events)] + ) + expected_event_index_label = np.array( + [str(i) for i in range(event_buffer_size)] + ) l3_data_product = CodiceL3HiDirectEvents( Mock(), @@ -113,19 +168,30 @@ def test_codice_l3_hi_direct_event_data_products(self): ) data_product_variables = l3_data_product.to_data_product_variables() - non_parent_fields = [f for f in fields(CodiceL3HiDirectEvents) if - f.name in CodiceL3HiDirectEvents.__annotations__] + non_parent_fields = [ + f + for f in fields(CodiceL3HiDirectEvents) + if f.name in CodiceL3HiDirectEvents.__annotations__ + ] self.assertEqual(len(data_product_variables), len(non_parent_fields)) - np.testing.assert_array_equal(l3_data_product.priority_index, expected_priority_index) + np.testing.assert_array_equal( + l3_data_product.priority_index, expected_priority_index + ) np.testing.assert_array_equal(l3_data_product.event_index, expected_event_index) - np.testing.assert_array_equal(l3_data_product.priority_index_label, expected_priority_index_label) - np.testing.assert_array_equal(l3_data_product.event_index_label, expected_event_index_label) + np.testing.assert_array_equal( + l3_data_product.priority_index_label, expected_priority_index_label + ) + np.testing.assert_array_equal( + l3_data_product.event_index_label, expected_event_index_label + ) for data_product_variable in data_product_variables: - np.testing.assert_array_equal(data_product_variable.value, - getattr(l3_data_product, data_product_variable.name)) + np.testing.assert_array_equal( + data_product_variable.value, + getattr(l3_data_product, data_product_variable.name), + ) def test_codice_hi_l3_pitch_angle_to_data_product(self): epoch_data = np.array([datetime.now()]) @@ -133,7 +199,9 @@ def test_codice_hi_l3_pitch_angle_to_data_product(self): pitch_angle = np.array([100, 200]) gyrophase = np.array([100, 200]) pitch_angle_size = len(epoch_data) * len(energy_data) * len(pitch_angle) - pitch_angle_and_gyrophase_size = len(epoch_data) * len(energy_data) * len(pitch_angle) * len(gyrophase) + pitch_angle_and_gyrophase_size = ( + len(epoch_data) * len(energy_data) * len(pitch_angle) * len(gyrophase) + ) energy_h_data = np.array([101, 103]) energy_cno_data = np.array([101, 103]) + 2 @@ -158,109 +226,205 @@ def test_codice_hi_l3_pitch_angle_to_data_product(self): "pitch_angle_delta": np.array([100, 200]), "gyrophase": gyrophase, "gyrophase_delta": np.array([100, 200]), - "h_intensity_by_pitch_angle": np.arange(pitch_angle_size).reshape(len(epoch_data), len(energy_data), - len(pitch_angle)) + 1, - "h_intensity_by_pitch_angle_and_gyrophase": np.arange(pitch_angle_and_gyrophase_size).reshape( - len(epoch_data), len(energy_data), len(pitch_angle), len(gyrophase)) + 2, - "he3he4_intensity_by_pitch_angle": np.arange(pitch_angle_size).reshape(len(epoch_data), len(energy_data), - len(pitch_angle)), - "he3he4_intensity_by_pitch_angle_and_gyrophase": np.arange(pitch_angle_and_gyrophase_size).reshape( - len(epoch_data), len(energy_data), len(pitch_angle), len(gyrophase)) + 3, - "cno_intensity_by_pitch_angle": np.arange(pitch_angle_size).reshape(len(epoch_data), len(energy_data), - len(pitch_angle)) + 4, - "cno_intensity_by_pitch_angle_and_gyrophase": np.arange(pitch_angle_and_gyrophase_size).reshape( - len(epoch_data), len(energy_data), len(pitch_angle), len(gyrophase)) + 5, - "fe_intensity_by_pitch_angle": np.arange(pitch_angle_size).reshape(len(epoch_data), len(energy_data), - len(pitch_angle)) + 6, - "fe_intensity_by_pitch_angle_and_gyrophase": np.arange(pitch_angle_and_gyrophase_size).reshape( - len(epoch_data), len(energy_data), len(pitch_angle), len(gyrophase)) + 7, + "h_intensity_by_pitch_angle": np.arange(pitch_angle_size).reshape( + len(epoch_data), len(energy_data), len(pitch_angle) + ) + + 1, + "h_intensity_by_pitch_angle_and_gyrophase": np.arange( + pitch_angle_and_gyrophase_size + ).reshape( + len(epoch_data), len(energy_data), len(pitch_angle), len(gyrophase) + ) + + 2, + "he3he4_intensity_by_pitch_angle": np.arange(pitch_angle_size).reshape( + len(epoch_data), len(energy_data), len(pitch_angle) + ), + "he3he4_intensity_by_pitch_angle_and_gyrophase": np.arange( + pitch_angle_and_gyrophase_size + ).reshape( + len(epoch_data), len(energy_data), len(pitch_angle), len(gyrophase) + ) + + 3, + "cno_intensity_by_pitch_angle": np.arange(pitch_angle_size).reshape( + len(epoch_data), len(energy_data), len(pitch_angle) + ) + + 4, + "cno_intensity_by_pitch_angle_and_gyrophase": np.arange( + pitch_angle_and_gyrophase_size + ).reshape( + len(epoch_data), len(energy_data), len(pitch_angle), len(gyrophase) + ) + + 5, + "fe_intensity_by_pitch_angle": np.arange(pitch_angle_size).reshape( + len(epoch_data), len(energy_data), len(pitch_angle) + ) + + 6, + "fe_intensity_by_pitch_angle_and_gyrophase": np.arange( + pitch_angle_and_gyrophase_size + ).reshape( + len(epoch_data), len(energy_data), len(pitch_angle), len(gyrophase) + ) + + 7, "codice_flags": np.zeros(len(epoch_data), dtype=np.uint16), } data_product = CodiceHiL3PitchAngleDataProduct( - input_metadata=Mock(), - **inputted_data_product_kwargs + input_metadata=Mock(), **inputted_data_product_kwargs ) - np.testing.assert_array_equal(data_product.energy_h_label, np.array([str(v) for v in energy_h_data])) - np.testing.assert_array_equal(data_product.energy_cno_label, np.array([str(v) for v in energy_cno_data])) - np.testing.assert_array_equal(data_product.energy_fe_label, np.array([str(v) for v in energy_fe_data])) - np.testing.assert_array_equal(data_product.energy_he3he4_label, np.array([str(v) for v in energy_he3he4_data])) - np.testing.assert_array_equal(data_product.pitch_angle_label, np.array([str(v) for v in pitch_angle])) - np.testing.assert_array_equal(data_product.gyrophase_label, np.array([str(v) for v in gyrophase])) + np.testing.assert_array_equal( + data_product.energy_h_label, np.array([str(v) for v in energy_h_data]) + ) + np.testing.assert_array_equal( + data_product.energy_cno_label, np.array([str(v) for v in energy_cno_data]) + ) + np.testing.assert_array_equal( + data_product.energy_fe_label, np.array([str(v) for v in energy_fe_data]) + ) + np.testing.assert_array_equal( + data_product.energy_he3he4_label, + np.array([str(v) for v in energy_he3he4_data]), + ) + np.testing.assert_array_equal( + data_product.pitch_angle_label, np.array([str(v) for v in pitch_angle]) + ) + np.testing.assert_array_equal( + data_product.gyrophase_label, np.array([str(v) for v in gyrophase]) + ) actual_data_product_variables = data_product.to_data_product_variables() - non_parent_fields = [f for f in fields(CodiceHiL3PitchAngleDataProduct) - if f.name in CodiceHiL3PitchAngleDataProduct.__annotations__] + non_parent_fields = [ + f + for f in fields(CodiceHiL3PitchAngleDataProduct) + if f.name in CodiceHiL3PitchAngleDataProduct.__annotations__ + ] data_product_fields = [f.name for f in non_parent_fields] self.assertEqual(len(actual_data_product_variables), len(non_parent_fields)) for data_product_var in actual_data_product_variables: self.assertIn(data_product_var.name, data_product_fields) - if data_product_var.name != 'epoch_delta': - np.testing.assert_array_equal(data_product_var.value, getattr(data_product, data_product_var.name)) + if data_product_var.name != "epoch_delta": + np.testing.assert_array_equal( + data_product_var.value, getattr(data_product, data_product_var.name) + ) else: - np.testing.assert_array_equal(data_product_var.value, np.array([10 * 1e9])) + np.testing.assert_array_equal( + data_product_var.value, np.array([10 * 1e9]) + ) def test_codice_hi_l1a_direct_event_read_from_sdc_cdf(self): - l1a_path = get_test_data_path("codice/imap_codice_l1a_hi-direct-events_20260831_v001.0003.cdf") + l1a_path = get_test_data_path( + "codice/imap_codice_l1a_hi-direct-events_20260831_v001.0003.cdf" + ) l1a_hi_direct_events = CodiceL1aHiDirectEvents.read_from_cdf(l1a_path) with CDF(str(l1a_path)) as cdf: - np.testing.assert_array_equal(l1a_hi_direct_events.epoch, cdf['epoch'][...]) - np.testing.assert_array_equal(l1a_hi_direct_events.ssd_energy, cdf['ssd_energy'][...]) - np.testing.assert_array_equal(l1a_hi_direct_events.tof, cdf['tof'][...]) + np.testing.assert_array_equal(l1a_hi_direct_events.epoch, cdf["epoch"][...]) + np.testing.assert_array_equal( + l1a_hi_direct_events.ssd_energy, cdf["ssd_energy"][...] + ) + np.testing.assert_array_equal(l1a_hi_direct_events.tof, cdf["tof"][...]) def test_l2_sectored_intensities_read_from_sdc_cdf(self): - l2_path = get_test_instrument_team_data_path("codice/hi/imap_codice_l2_hi-sectored_20251206_v002.cdf") + l2_path = get_test_instrument_team_data_path( + "codice/hi/imap_codice_l2_hi-sectored_20251206_v002.cdf" + ) l2_sectored_data = CodiceHiL2SectoredIntensitiesData.read_from_cdf(l2_path) with CDF(str(l2_path)) as cdf: np.testing.assert_array_equal(l2_sectored_data.epoch, cdf["epoch"]) - expected_epoch_delta = np.array([timedelta(seconds=ns / 1e9) for ns in cdf["epoch_delta_plus"][...]]) - np.testing.assert_array_equal(l2_sectored_data.epoch_delta_plus, expected_epoch_delta) - np.testing.assert_array_equal(l2_sectored_data.spin_angles, cdf['spin_angle'][...]) - np.testing.assert_array_equal(l2_sectored_data.elevation_angle, cdf['elevation_angle']) - - np.testing.assert_array_equal(l2_sectored_data.data_quality, cdf['data_quality']) - - np.testing.assert_array_equal(l2_sectored_data.h_intensities, cdf['h']) - np.testing.assert_array_equal(l2_sectored_data.energy_h, cdf['energy_h']) - np.testing.assert_array_equal(l2_sectored_data.energy_h_plus, cdf['energy_h_plus']) - np.testing.assert_array_equal(l2_sectored_data.energy_h_minus, cdf['energy_h_minus']) - - np.testing.assert_array_equal(l2_sectored_data.cno_intensities, cdf['cno']) - np.testing.assert_array_equal(l2_sectored_data.energy_cno, cdf['energy_cno']) - np.testing.assert_array_equal(l2_sectored_data.energy_cno_plus, cdf['energy_cno_plus']) - np.testing.assert_array_equal(l2_sectored_data.energy_cno_minus, cdf['energy_cno_minus']) - - np.testing.assert_array_equal(l2_sectored_data.fe_intensities, cdf['fe']) - np.testing.assert_array_equal(l2_sectored_data.energy_fe, cdf['energy_fe']) - np.testing.assert_array_equal(l2_sectored_data.energy_fe_plus, cdf['energy_fe_plus']) - np.testing.assert_array_equal(l2_sectored_data.energy_fe_minus, cdf['energy_fe_minus']) - - np.testing.assert_array_equal(l2_sectored_data.he3he4_intensities, cdf['he3he4']) - np.testing.assert_array_equal(l2_sectored_data.energy_he3he4, cdf['energy_he3he4']) - np.testing.assert_array_equal(l2_sectored_data.energy_he3he4_plus, cdf['energy_he3he4_plus']) - np.testing.assert_array_equal(l2_sectored_data.energy_he3he4_minus, cdf['energy_he3he4_minus']) - - def test_l2_sectored_intensities_read_from_instrument_team_cdf_handles_fill_values(self): - all_fill_l2_sectored_intensities_path = \ - get_test_data_path('codice/imap_codice_l2_hi-sectored_20250814_v002-all-fill.cdf') - l2_sectored_intensities = CodiceHiL2SectoredIntensitiesData.read_from_cdf(all_fill_l2_sectored_intensities_path) + expected_epoch_delta = np.array( + [timedelta(seconds=ns / 1e9) for ns in cdf["epoch_delta_plus"][...]] + ) + np.testing.assert_array_equal( + l2_sectored_data.epoch_delta_plus, expected_epoch_delta + ) + np.testing.assert_array_equal( + l2_sectored_data.spin_angles, cdf["spin_angle"][...] + ) + np.testing.assert_array_equal( + l2_sectored_data.elevation_angle, cdf["elevation_angle"] + ) + + np.testing.assert_array_equal( + l2_sectored_data.data_quality, cdf["data_quality"] + ) + + np.testing.assert_array_equal(l2_sectored_data.h_intensities, cdf["h"]) + np.testing.assert_array_equal(l2_sectored_data.energy_h, cdf["energy_h"]) + np.testing.assert_array_equal( + l2_sectored_data.energy_h_plus, cdf["energy_h_plus"] + ) + np.testing.assert_array_equal( + l2_sectored_data.energy_h_minus, cdf["energy_h_minus"] + ) + + np.testing.assert_array_equal(l2_sectored_data.cno_intensities, cdf["cno"]) + np.testing.assert_array_equal( + l2_sectored_data.energy_cno, cdf["energy_cno"] + ) + np.testing.assert_array_equal( + l2_sectored_data.energy_cno_plus, cdf["energy_cno_plus"] + ) + np.testing.assert_array_equal( + l2_sectored_data.energy_cno_minus, cdf["energy_cno_minus"] + ) + + np.testing.assert_array_equal(l2_sectored_data.fe_intensities, cdf["fe"]) + np.testing.assert_array_equal(l2_sectored_data.energy_fe, cdf["energy_fe"]) + np.testing.assert_array_equal( + l2_sectored_data.energy_fe_plus, cdf["energy_fe_plus"] + ) + np.testing.assert_array_equal( + l2_sectored_data.energy_fe_minus, cdf["energy_fe_minus"] + ) + + np.testing.assert_array_equal( + l2_sectored_data.he3he4_intensities, cdf["he3he4"] + ) + np.testing.assert_array_equal( + l2_sectored_data.energy_he3he4, cdf["energy_he3he4"] + ) + np.testing.assert_array_equal( + l2_sectored_data.energy_he3he4_plus, cdf["energy_he3he4_plus"] + ) + np.testing.assert_array_equal( + l2_sectored_data.energy_he3he4_minus, cdf["energy_he3he4_minus"] + ) + + def test_l2_sectored_intensities_read_from_instrument_team_cdf_handles_fill_values( + self, + ): + all_fill_l2_sectored_intensities_path = get_test_data_path( + "codice/imap_codice_l2_hi-sectored_20250814_v002-all-fill.cdf" + ) + l2_sectored_intensities = CodiceHiL2SectoredIntensitiesData.read_from_cdf( + all_fill_l2_sectored_intensities_path + ) with CDF(str(all_fill_l2_sectored_intensities_path)) as cdf: - np.testing.assert_array_equal(l2_sectored_intensities.cno_intensities, np.full_like(cdf['cno'], np.nan)) - np.testing.assert_array_equal(l2_sectored_intensities.fe_intensities, np.full_like(cdf['fe'], np.nan)) - np.testing.assert_array_equal(l2_sectored_intensities.h_intensities, np.full_like(cdf['h'], np.nan)) - np.testing.assert_array_equal(l2_sectored_intensities.he3he4_intensities, - np.full_like(cdf['he3he4'], np.nan)) + np.testing.assert_array_equal( + l2_sectored_intensities.cno_intensities, + np.full_like(cdf["cno"], np.nan), + ) + np.testing.assert_array_equal( + l2_sectored_intensities.fe_intensities, np.full_like(cdf["fe"], np.nan) + ) + np.testing.assert_array_equal( + l2_sectored_intensities.h_intensities, np.full_like(cdf["h"], np.nan) + ) + np.testing.assert_array_equal( + l2_sectored_intensities.he3he4_intensities, + np.full_like(cdf["he3he4"], np.nan), + ) @staticmethod def _create_expanded_l2_hi_de_dataset(unexpanded_cdf: Path, tempdir: Path) -> Path: expanded_cdf_path = tempdir / "expanded_hi_l2_direct_events.cdf" - with CDF(str(expanded_cdf_path), masterpath=str(unexpanded_cdf), readonly=False) as expanded_cdf: + with CDF( + str(expanded_cdf_path), masterpath=str(unexpanded_cdf), readonly=False + ) as expanded_cdf: old_priority_attrs = expanded_cdf["priority"].attrs del expanded_cdf["priority"] expanded_cdf["priority"] = np.arange(8) @@ -284,7 +448,7 @@ def _create_expanded_l2_hi_de_dataset(unexpanded_cdf: Path, tempdir: Path) -> Pa "ssd_energy", "ssd_id", "tof", - "type" + "type", ] for var in event_based_vars: diff --git a/tests/codice/l3/lo/direct_events/science/test_angle_lookups.py b/tests/codice/l3/lo/direct_events/science/test_angle_lookups.py index c5271affe..028c94f5e 100644 --- a/tests/codice/l3/lo/direct_events/science/test_angle_lookups.py +++ b/tests/codice/l3/lo/direct_events/science/test_angle_lookups.py @@ -2,8 +2,10 @@ import numpy as np -from imap_l3_processing.codice.l3.lo.direct_events.science.angle_lookup import SpinAngleLookup, \ - PositionToElevationLookup +from imap_l3_processing.codice.l3.lo.direct_events.science.angle_lookup import ( + SpinAngleLookup, + PositionToElevationLookup, +) class TestCoDICEAngleLookup(unittest.TestCase): @@ -45,7 +47,32 @@ def test_apd_to_elevation_index(self): actual_indices = [] for apd_id in range(1, 25): actual_indices.append(elevation_angle_lut.apd_to_elevation_index(apd_id)) - expected_indices = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1] + expected_indices = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 11, + 10, + 9, + 8, + 7, + 6, + 5, + 4, + 3, + 2, + 1, + ] np.testing.assert_array_equal(actual_indices, expected_indices) def test_apd_to_elevation(self): @@ -53,6 +80,30 @@ def test_apd_to_elevation(self): actual_elevations = [] for apd_id in range(1, 25): actual_elevations.append(elevation_angle_lut.apd_to_elevation(apd_id)) - expected_elevations = [0, 15, 30, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 165, 150, 135, 120, 105, 90, 75, - 60, 45, 30, 15] + expected_elevations = [ + 0, + 15, + 30, + 45, + 60, + 75, + 90, + 105, + 120, + 135, + 150, + 165, + 180, + 165, + 150, + 135, + 120, + 105, + 90, + 75, + 60, + 45, + 30, + 15, + ] np.testing.assert_array_equal(actual_elevations, expected_elevations) diff --git a/tests/codice/l3/lo/direct_events/science/test_efficiency_lookup.py b/tests/codice/l3/lo/direct_events/science/test_efficiency_lookup.py index 7210e943c..1288791d7 100644 --- a/tests/codice/l3/lo/direct_events/science/test_efficiency_lookup.py +++ b/tests/codice/l3/lo/direct_events/science/test_efficiency_lookup.py @@ -5,7 +5,9 @@ import numpy as np -from imap_l3_processing.codice.l3.lo.direct_events.science.efficiency_lookup import EfficiencyLookup +from imap_l3_processing.codice.l3.lo.direct_events.science.efficiency_lookup import ( + EfficiencyLookup, +) class TestEfficiencyLookup(unittest.TestCase): @@ -22,22 +24,28 @@ def test_read_from_csv(self): with open(output_csv, "w") as csvfile: csv_writer = csv.writer(csvfile) - csv_writer.writerow(["species","product","esa_step"]+[f"position_{i}" for i in range(num_positions)]) + csv_writer.writerow( + ["species", "product", "esa_step"] + + [f"position_{i}" for i in range(num_positions)] + ) for i in range(num_energies): - csv_writer.writerow([ - "hplus","sw",i, *expected_efficiency_data[i, :] - ]) + csv_writer.writerow( + ["hplus", "sw", i, *expected_efficiency_data[i, :]] + ) for i in range(num_energies): - csv_writer.writerow([ - "cplus6","sw",i, *expected_efficiency_data[i, :]*6 - ]) + csv_writer.writerow( + ["cplus6", "sw", i, *expected_efficiency_data[i, :] * 6] + ) for i in range(num_energies): - csv_writer.writerow([ - "hplus","nsw",i, *expected_efficiency_data[i, :]*42 - ]) - + csv_writer.writerow( + ["hplus", "nsw", i, *expected_efficiency_data[i, :] * 42] + ) efficiency_lookup = EfficiencyLookup.read_from_csv(output_csv, "hplus") - np.testing.assert_almost_equal(efficiency_lookup.efficiency_data, expected_efficiency_data) - self.assertEqual(efficiency_lookup.efficiency_data.shape, (num_energies, num_positions)) + np.testing.assert_almost_equal( + efficiency_lookup.efficiency_data, expected_efficiency_data + ) + self.assertEqual( + efficiency_lookup.efficiency_data.shape, (num_energies, num_positions) + ) diff --git a/tests/codice/l3/lo/direct_events/science/test_energy_lookup.py b/tests/codice/l3/lo/direct_events/science/test_energy_lookup.py index dbdee21de..117df203e 100644 --- a/tests/codice/l3/lo/direct_events/science/test_energy_lookup.py +++ b/tests/codice/l3/lo/direct_events/science/test_energy_lookup.py @@ -2,18 +2,33 @@ import numpy as np -from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import EnergyLookup +from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import ( + EnergyLookup, +) from tests.test_helpers import get_test_data_path class TestEnergyLookup(unittest.TestCase): def test_read_from_csv(self): - input_file = get_test_data_path("codice") / "imap_codice_lo-energy-per-charge_20241110_v002.csv" + input_file = ( + get_test_data_path("codice") + / "imap_codice_lo-energy-per-charge_20241110_v002.csv" + ) actual_energy_lookup = EnergyLookup.read_from_csv(input_file) np.testing.assert_array_equal(actual_energy_lookup.bin_centers[0], 81.216) - np.testing.assert_array_equal(actual_energy_lookup.bin_centers[127], 0.507357072) - np.testing.assert_array_equal(actual_energy_lookup.delta_minus[0], 81.216 - 79.60339125) - np.testing.assert_array_equal(actual_energy_lookup.delta_minus[127], 0.507357072 - 0.497319345) - np.testing.assert_array_equal(actual_energy_lookup.delta_plus[0], 82.855305 - 81.216) - np.testing.assert_array_equal(actual_energy_lookup.delta_plus[127], 0.517446956 - 0.507357072) + np.testing.assert_array_equal( + actual_energy_lookup.bin_centers[127], 0.507357072 + ) + np.testing.assert_array_equal( + actual_energy_lookup.delta_minus[0], 81.216 - 79.60339125 + ) + np.testing.assert_array_equal( + actual_energy_lookup.delta_minus[127], 0.507357072 - 0.497319345 + ) + np.testing.assert_array_equal( + actual_energy_lookup.delta_plus[0], 82.855305 - 81.216 + ) + np.testing.assert_array_equal( + actual_energy_lookup.delta_plus[127], 0.517446956 - 0.507357072 + ) diff --git a/tests/codice/l3/lo/direct_events/science/test_geometric_factor_lookup.py b/tests/codice/l3/lo/direct_events/science/test_geometric_factor_lookup.py index d21698bc6..6d510fa2c 100644 --- a/tests/codice/l3/lo/direct_events/science/test_geometric_factor_lookup.py +++ b/tests/codice/l3/lo/direct_events/science/test_geometric_factor_lookup.py @@ -4,48 +4,115 @@ import numpy as np -from imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup import GeometricFactorLookup +from imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup import ( + GeometricFactorLookup, +) from tests.test_helpers import get_test_data_folder class TestGeometricFactorLookup(TestCase): - def test_default_esa_step_end_index(self): geometric_factor_lookup = GeometricFactorLookup(2, 0.5) - expected_esa_step_end_index = \ - np.array( - [0, 1, 2, 3, 5, 7, 9, 11, 14, 17, 20, 23, 27, 31, 35, 39, 44, 49, 54, 59, 64, 69, 74, 79, 85, 91, 97, - 103, 109, 115, 121, 127]) - np.testing.assert_array_equal(geometric_factor_lookup._esa_step_end_index, expected_esa_step_end_index) + expected_esa_step_end_index = np.array( + [ + 0, + 1, + 2, + 3, + 5, + 7, + 9, + 11, + 14, + 17, + 20, + 23, + 27, + 31, + 35, + 39, + 44, + 49, + 54, + 59, + 64, + 69, + 74, + 79, + 85, + 91, + 97, + 103, + 109, + 115, + 121, + 127, + ] + ) + np.testing.assert_array_equal( + geometric_factor_lookup._esa_step_end_index, expected_esa_step_end_index + ) def test_read_from_csv(self): expected_num_esa_steps = 128 expected_num_positions = 24 - path = get_test_data_folder() / 'codice' / 'imap_codice_l2-lo-gfactor_20251212_v003.csv' + path = ( + get_test_data_folder() + / "codice" + / "imap_codice_l2-lo-gfactor_20251212_v003.csv" + ) lookup = GeometricFactorLookup.read_from_csv(path) self.assertIsInstance(lookup, GeometricFactorLookup) - self.assertEqual((expected_num_esa_steps, expected_num_positions), lookup._full_factor.shape) - self.assertEqual((expected_num_esa_steps, expected_num_positions), lookup._reduced_factor.shape) + self.assertEqual( + (expected_num_esa_steps, expected_num_positions), lookup._full_factor.shape + ) + self.assertEqual( + (expected_num_esa_steps, expected_num_positions), + lookup._reduced_factor.shape, + ) self.assertEqual(8.71567e-05, lookup._full_factor[0, 0]) self.assertEqual(9.7615504e-06, lookup._reduced_factor[0, 0]) - @patch("imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_ESA_STEPS", 3) - @patch("imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_SPIN_SECTORS", - 3) - @patch("imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_AZIMUTH_BINS", - 2) - def test_get_geometric_factors_gets_reduced_where_half_spin_greater_than_rgfo_half_spin_first_era(self): + @patch( + "imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_ESA_STEPS", + 3, + ) + @patch( + "imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_SPIN_SECTORS", + 3, + ) + @patch( + "imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_AZIMUTH_BINS", + 2, + ) + def test_get_geometric_factors_gets_reduced_where_half_spin_greater_than_rgfo_half_spin_first_era( + self, + ): rgfo_half_spin_threshold = 5 below_half_spin_threshold = 3 above_half_spin_threshold = 7 - rgfo_half_spin = np.ma.masked_array(data=np.array([rgfo_half_spin_threshold, rgfo_half_spin_threshold])) - half_spin = np.ma.masked_array(data=np.array([ - [below_half_spin_threshold, above_half_spin_threshold, rgfo_half_spin_threshold], - [above_half_spin_threshold, below_half_spin_threshold, above_half_spin_threshold], - ])) + rgfo_half_spin = np.ma.masked_array( + data=np.array([rgfo_half_spin_threshold, rgfo_half_spin_threshold]) + ) + half_spin = np.ma.masked_array( + data=np.array( + [ + [ + below_half_spin_threshold, + above_half_spin_threshold, + rgfo_half_spin_threshold, + ], + [ + above_half_spin_threshold, + below_half_spin_threshold, + above_half_spin_threshold, + ], + ] + ) + ) full_factors = np.array([[10, 20], [30, 40], [50, 60]]) reduced_factors = np.array([[1, 2], [3, 4], [5, 6]]) geometric_factor_lookup = GeometricFactorLookup( @@ -57,7 +124,7 @@ def test_get_geometric_factors_gets_reduced_where_half_spin_greater_than_rgfo_ha rgfo_spin_sector=np.zeros(2), rgfo_esa_step=np.zeros(2), half_spin=half_spin, - epoch=date(2025,11, 23), + epoch=date(2025, 11, 23), ) all_spin_sectors_and_positions = (3, 2) @@ -69,27 +136,54 @@ def test_get_geometric_factors_gets_reduced_where_half_spin_greater_than_rgfo_ha (1, 1, full_factors), (1, 2, reduced_factors), ] - for epoch, esa_step, expected_factors in epoch_esa_step_index_pairs_and_expected_factors: + for ( + epoch, + esa_step, + expected_factors, + ) in epoch_esa_step_index_pairs_and_expected_factors: np.testing.assert_array_equal( actual[epoch, esa_step, :, :], - np.broadcast_to(expected_factors[esa_step], all_spin_sectors_and_positions), - f"Mismatch at epoch:{epoch}, esa:{esa_step}" + np.broadcast_to( + expected_factors[esa_step], all_spin_sectors_and_positions + ), + f"Mismatch at epoch:{epoch}, esa:{esa_step}", ) - @patch("imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_ESA_STEPS", 3) - @patch("imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_SPIN_SECTORS", - 3) - @patch("imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_AZIMUTH_BINS", - 2) + @patch( + "imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_ESA_STEPS", + 3, + ) + @patch( + "imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_SPIN_SECTORS", + 3, + ) + @patch( + "imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_AZIMUTH_BINS", + 2, + ) def test_get_geometric_factors_gets_all_full_geometric_factors_for_second_era(self): rgfo_half_spin_threshold = 5 below_half_spin_threshold = 3 above_half_spin_threshold = 7 - rgfo_half_spin = np.ma.masked_array(data=np.array([rgfo_half_spin_threshold, rgfo_half_spin_threshold])) - half_spin = np.ma.masked_array(data=np.array([ - [below_half_spin_threshold, above_half_spin_threshold, rgfo_half_spin_threshold], - [above_half_spin_threshold, below_half_spin_threshold, above_half_spin_threshold], - ])) + rgfo_half_spin = np.ma.masked_array( + data=np.array([rgfo_half_spin_threshold, rgfo_half_spin_threshold]) + ) + half_spin = np.ma.masked_array( + data=np.array( + [ + [ + below_half_spin_threshold, + above_half_spin_threshold, + rgfo_half_spin_threshold, + ], + [ + above_half_spin_threshold, + below_half_spin_threshold, + above_half_spin_threshold, + ], + ] + ) + ) full_factors = np.array([[10, 20], [30, 40], [50, 60]]) reduced_factors = np.array([[1, 2], [3, 4], [5, 6]]) @@ -102,33 +196,56 @@ def test_get_geometric_factors_gets_all_full_geometric_factors_for_second_era(se rgfo_spin_sector=np.zeros(2), rgfo_esa_step=np.zeros(2), half_spin=half_spin, - epoch=date(2025,11, 24), + epoch=date(2025, 11, 24), ) actual_end_second_era = geometric_factor_lookup.get_geometric_factors( rgfo_half_spin=rgfo_half_spin, rgfo_spin_sector=np.zeros(2), rgfo_esa_step=np.zeros(2), half_spin=half_spin, - epoch=date(2026,1, 28), + epoch=date(2026, 1, 28), ) - expected = np.broadcast_to(full_factors[None, :, None, :], (2,3,3,2)) + expected = np.broadcast_to(full_factors[None, :, None, :], (2, 3, 3, 2)) np.testing.assert_array_equal(actual_start_second_era, expected) np.testing.assert_array_equal(actual_end_second_era, expected) - @patch("imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_ESA_STEPS", 3) - @patch("imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_SPIN_SECTORS", - 3) - @patch("imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_AZIMUTH_BINS", - 2) - def test_get_geometric_factors_gets_reduced_where_half_spin_greater_than_rgfo_half_spin_third_era(self): + @patch( + "imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_ESA_STEPS", + 3, + ) + @patch( + "imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_SPIN_SECTORS", + 3, + ) + @patch( + "imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_AZIMUTH_BINS", + 2, + ) + def test_get_geometric_factors_gets_reduced_where_half_spin_greater_than_rgfo_half_spin_third_era( + self, + ): rgfo_half_spin_threshold = 5 below_half_spin_threshold = 3 above_half_spin_threshold = 7 - rgfo_half_spin = np.ma.masked_array(data=np.array([rgfo_half_spin_threshold, rgfo_half_spin_threshold])) - half_spin = np.ma.masked_array(data=np.array([ - [below_half_spin_threshold, above_half_spin_threshold, above_half_spin_threshold], - [above_half_spin_threshold, below_half_spin_threshold, above_half_spin_threshold], - ])) + rgfo_half_spin = np.ma.masked_array( + data=np.array([rgfo_half_spin_threshold, rgfo_half_spin_threshold]) + ) + half_spin = np.ma.masked_array( + data=np.array( + [ + [ + below_half_spin_threshold, + above_half_spin_threshold, + above_half_spin_threshold, + ], + [ + above_half_spin_threshold, + below_half_spin_threshold, + above_half_spin_threshold, + ], + ] + ) + ) full_factors = np.array([[10, 20], [30, 40], [50, 60]]) reduced_factors = np.array([[1, 2], [3, 4], [5, 6]]) geometric_factor_lookup = GeometricFactorLookup( @@ -140,7 +257,7 @@ def test_get_geometric_factors_gets_reduced_where_half_spin_greater_than_rgfo_ha rgfo_spin_sector=np.zeros(2), rgfo_esa_step=np.zeros(2), half_spin=half_spin, - epoch=date(2026,1,29), + epoch=date(2026, 1, 29), ) all_spin_sectors_and_positions = (3, 2) @@ -152,22 +269,42 @@ def test_get_geometric_factors_gets_reduced_where_half_spin_greater_than_rgfo_ha (1, 1, full_factors), (1, 2, reduced_factors), ] - for epoch, esa_step, expected_factors in epoch_esa_step_index_pairs_and_expected_factors: + for ( + epoch, + esa_step, + expected_factors, + ) in epoch_esa_step_index_pairs_and_expected_factors: np.testing.assert_array_equal( actual[epoch, esa_step, :, :], - np.broadcast_to(expected_factors[esa_step], all_spin_sectors_and_positions)) - - @patch("imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_ESA_STEPS", 3) - @patch("imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_SPIN_SECTORS", - 24) - @patch("imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_AZIMUTH_BINS", - 2) - def test_get_geometric_factors_gets_reduced_where_equal_half_spin_greater_spin_sector_third_era(self): + np.broadcast_to( + expected_factors[esa_step], all_spin_sectors_and_positions + ), + ) + + @patch( + "imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_ESA_STEPS", + 3, + ) + @patch( + "imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_SPIN_SECTORS", + 24, + ) + @patch( + "imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_AZIMUTH_BINS", + 2, + ) + def test_get_geometric_factors_gets_reduced_where_equal_half_spin_greater_spin_sector_third_era( + self, + ): rgfo_half_spin = np.ma.masked_array(data=np.array([2, 2])) - half_spin = np.ma.masked_array(data=np.array([ - [2, 2, 2], - [2, 2, 2], - ])) + half_spin = np.ma.masked_array( + data=np.array( + [ + [2, 2, 2], + [2, 2, 2], + ] + ) + ) rgfo_spin_sector = np.ma.masked_array(data=np.array([6, 8])) rgfo_esa_step = np.ma.masked_array(data=np.array([3, 3])) @@ -182,7 +319,7 @@ def test_get_geometric_factors_gets_reduced_where_equal_half_spin_greater_spin_s rgfo_spin_sector=rgfo_spin_sector, rgfo_esa_step=rgfo_esa_step, half_spin=half_spin, - epoch=date(2026,1, 29), + epoch=date(2026, 1, 29), ) sector_mod_12 = np.arange(24) % 12 @@ -195,18 +332,31 @@ def test_get_geometric_factors_gets_reduced_where_equal_half_spin_greater_spin_s ) np.testing.assert_array_equal(actual, expected) - @patch("imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_ESA_STEPS", 3) - @patch("imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_SPIN_SECTORS", - 3) - @patch("imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_AZIMUTH_BINS", - 2) - def test_get_geometric_factors_gets_reduced_where_equal_half_spin_equal_spin_sector_greater_esa_step_third_era(self): + @patch( + "imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_ESA_STEPS", + 3, + ) + @patch( + "imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_SPIN_SECTORS", + 3, + ) + @patch( + "imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup.CODICE_LO_NUM_AZIMUTH_BINS", + 2, + ) + def test_get_geometric_factors_gets_reduced_where_equal_half_spin_equal_spin_sector_greater_esa_step_third_era( + self, + ): rgfo_half_spin = np.ma.masked_array(data=np.array([2, 2])) - half_spin = np.ma.masked_array(data=np.array([ - [2, 2, 2], - [2, 2, 2], - ])) + half_spin = np.ma.masked_array( + data=np.array( + [ + [2, 2, 2], + [2, 2, 2], + ] + ) + ) rgfo_spin_sector = np.ma.masked_array(data=np.array([1, 1])) rgfo_esa_step = np.ma.masked_array(data=np.array([0, 1])) @@ -221,7 +371,7 @@ def test_get_geometric_factors_gets_reduced_where_equal_half_spin_equal_spin_sec rgfo_spin_sector=rgfo_spin_sector, rgfo_esa_step=rgfo_esa_step, half_spin=half_spin, - epoch=date(2026,1, 29), + epoch=date(2026, 1, 29), ) epoch_esa_step_index_and_expected_factors = [ @@ -232,8 +382,11 @@ def test_get_geometric_factors_gets_reduced_where_equal_half_spin_equal_spin_sec (1, 1, full_factors), (1, 2, reduced_factors), ] - for epoch, esa_step, expected_factors in epoch_esa_step_index_and_expected_factors: + for ( + epoch, + esa_step, + expected_factors, + ) in epoch_esa_step_index_and_expected_factors: np.testing.assert_array_equal( - actual[epoch, esa_step, 1, :], - expected_factors[esa_step] + actual[epoch, esa_step, 1, :], expected_factors[esa_step] ) diff --git a/tests/codice/l3/lo/direct_events/science/test_mass_coefficient_lookup.py b/tests/codice/l3/lo/direct_events/science/test_mass_coefficient_lookup.py index 24d5bcec9..698c450d4 100644 --- a/tests/codice/l3/lo/direct_events/science/test_mass_coefficient_lookup.py +++ b/tests/codice/l3/lo/direct_events/science/test_mass_coefficient_lookup.py @@ -3,13 +3,22 @@ import unittest from pathlib import Path -from imap_l3_processing.codice.l3.lo.direct_events.science.mass_coefficient_lookup import MassCoefficientLookup +from imap_l3_processing.codice.l3.lo.direct_events.science.mass_coefficient_lookup import ( + MassCoefficientLookup, +) class TestMassCoefficientLookup(unittest.TestCase): def test_mass_coefficient_lookup(self): mass_coefficient_lookup_csv_path = "mass_coefficient_lookup.csv" - coefficients = [[5.29633], [-1.5053], [-2.86483], [0.473693], [0.0900633], [0.0783456]] + coefficients = [ + [5.29633], + [-1.5053], + [-2.86483], + [0.473693], + [0.0900633], + [0.0783456], + ] with tempfile.TemporaryDirectory() as tmpdir: temp_csv_file = Path(tmpdir) / mass_coefficient_lookup_csv_path diff --git a/tests/codice/l3/lo/direct_events/science/test_mass_species_bin_lookup.py b/tests/codice/l3/lo/direct_events/science/test_mass_species_bin_lookup.py index f79576482..7eab888f1 100644 --- a/tests/codice/l3/lo/direct_events/science/test_mass_species_bin_lookup.py +++ b/tests/codice/l3/lo/direct_events/science/test_mass_species_bin_lookup.py @@ -2,13 +2,17 @@ import numpy as np -from imap_l3_processing.codice.l3.lo.direct_events.science.mass_species_bin_lookup import MassSpeciesBinLookup +from imap_l3_processing.codice.l3.lo.direct_events.science.mass_species_bin_lookup import ( + MassSpeciesBinLookup, +) from tests.test_helpers import get_test_data_path class TestMassSpeciesBinLookup(unittest.TestCase): def test_read_csv(self): - csv_path = get_test_data_path("codice/imap_codice_lo-mass-species-bin-lookup_20250309_v003.csv") + csv_path = get_test_data_path( + "codice/imap_codice_lo-mass-species-bin-lookup_20250309_v003.csv" + ) mass_species_bin_lookup = MassSpeciesBinLookup.read_from_csv(csv_path) expected_species = ["hplus", "heplusplus", "oplus6", "heplus"] @@ -16,8 +20,12 @@ def test_read_csv(self): expected_mass_range = [(0.5, 1.5), (2.0, 7.5), (14.0, 18.0), (1.0, 7.5)] np.testing.assert_array_equal(mass_species_bin_lookup.species, expected_species) - np.testing.assert_array_equal(mass_species_bin_lookup.mass_per_charge, expected_mass_per_charge) - np.testing.assert_array_equal(mass_species_bin_lookup.mass_ranges, expected_mass_range) + np.testing.assert_array_equal( + mass_species_bin_lookup.mass_per_charge, expected_mass_per_charge + ) + np.testing.assert_array_equal( + mass_species_bin_lookup.mass_ranges, expected_mass_range + ) def test_get_species(self): lookup = MassSpeciesBinLookup( @@ -36,15 +44,14 @@ def test_get_species(self): self.assertEqual(4, lookup.get_num_species()) def test_get_species_index(self): - csv_path = get_test_data_path("codice/imap_codice_lo-mass-species-bin-lookup_20250309_v003.csv") + csv_path = get_test_data_path( + "codice/imap_codice_lo-mass-species-bin-lookup_20250309_v003.csv" + ) mass_species_bin_lookup = MassSpeciesBinLookup.read_from_csv(csv_path) - test_cases = [ - ("hplus", 0), - ("heplusplus", 1), - ("oplus6", 2), - ("heplus", 3) - ] + test_cases = [("hplus", 0), ("heplusplus", 1), ("oplus6", 2), ("heplus", 3)] for species, index in test_cases: with self.subTest(species): - self.assertEqual(index, mass_species_bin_lookup.get_species_index(species)) + self.assertEqual( + index, mass_species_bin_lookup.get_species_index(species) + ) diff --git a/tests/codice/l3/lo/science/test_codice_lo_calculations.py b/tests/codice/l3/lo/science/test_codice_lo_calculations.py index 28d463f13..7edfe1177 100644 --- a/tests/codice/l3/lo/science/test_codice_lo_calculations.py +++ b/tests/codice/l3/lo/science/test_codice_lo_calculations.py @@ -4,53 +4,84 @@ import numpy as np -from imap_l3_processing.codice.l3.lo.direct_events.science.angle_lookup import PositionToElevationLookup -from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import EnergyLookup -from imap_l3_processing.codice.l3.lo.direct_events.science.mass_coefficient_lookup import MassCoefficientLookup -from imap_l3_processing.codice.l3.lo.direct_events.science.mass_species_bin_lookup import MassSpeciesBinLookup +from imap_l3_processing.codice.l3.lo.direct_events.science.angle_lookup import ( + PositionToElevationLookup, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import ( + EnergyLookup, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.mass_coefficient_lookup import ( + MassCoefficientLookup, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.mass_species_bin_lookup import ( + MassSpeciesBinLookup, +) from imap_l3_processing.codice.l3.lo.models import CodiceLoDirectEventData -from imap_l3_processing.codice.l3.lo.science.codice_lo_calculations import calculate_partial_densities, \ - calculate_total_number_of_events, calculate_mass, calculate_mass_per_charge, \ - rebin_to_counts_by_species_elevation_and_spin_sector, rebin_direct_events_by_energy_and_spin_sector, \ - CODICE_LO_NUM_AZIMUTH_BINS, combine_priorities_for_species_and_calculate_count_uncertainty, \ - rebin_3d_distribution_azimuth_to_elevation, convert_counts_to_intensity, rebin_direct_events_for_normalization, \ - calculate_normalization_factor, lookup_normalization_per_event +from imap_l3_processing.codice.l3.lo.science.codice_lo_calculations import ( + calculate_partial_densities, + calculate_total_number_of_events, + calculate_mass, + calculate_mass_per_charge, + rebin_to_counts_by_species_elevation_and_spin_sector, + rebin_direct_events_by_energy_and_spin_sector, + CODICE_LO_NUM_AZIMUTH_BINS, + combine_priorities_for_species_and_calculate_count_uncertainty, + rebin_3d_distribution_azimuth_to_elevation, + convert_counts_to_intensity, + rebin_direct_events_for_normalization, + calculate_normalization_factor, + lookup_normalization_per_event, +) class TestCodiceLoCalculations(unittest.TestCase): - def test_calculate_total_number_of_events(self): - priority_0_tcrs = np.array([ - [[1, 2, 3], - [4, 5, 6], - [7, 8, 9], - [10, 11, 12]], - [[2, 2, 3], - [4, 5, 6], - [7, 8, 9], - [10, 11, 12]]]) + priority_0_tcrs = np.array( + [ + [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]], + [[2, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]], + ] + ) acquisition_time = np.array([1, 2, 3, 4]) * 1_000_000 expected_total_number_of_events = [6 + 30 + 72 + 132, 7 + 30 + 72 + 132] - actual_total_number_of_events = calculate_total_number_of_events(priority_0_tcrs, acquisition_time) - np.testing.assert_array_equal(actual_total_number_of_events, expected_total_number_of_events) + actual_total_number_of_events = calculate_total_number_of_events( + priority_0_tcrs, acquisition_time + ) + np.testing.assert_array_equal( + actual_total_number_of_events, expected_total_number_of_events + ) def test_calculate_mass(self): apd_energy = np.array([[np.exp(1)], [np.exp(2)]]) tof = np.array([[np.exp(50)], [np.exp(60)]]) - lookup = MassCoefficientLookup(np.array([10e-1, 10e-2, 10e-3, 10e-4, 10e-5, 10e-6])) + lookup = MassCoefficientLookup( + np.array([10e-1, 10e-2, 10e-3, 10e-4, 10e-5, 10e-6]) + ) actual_mass = calculate_mass(apd_energy, tof, lookup) - expected_mass_1 = lookup[0] + lookup[1] * 1 + lookup[2] * 50 + lookup[3] * 1 * 50 + lookup[4] * 1 + lookup[ - 5] * np.power(50, 3) - expected_mass_2 = lookup[0] + lookup[1] * 2 + lookup[2] * 60 + lookup[3] * 2 * 60 + lookup[4] * np.power(2, 2) + \ - lookup[5] * np.power(60, 3) + expected_mass_1 = ( + lookup[0] + + lookup[1] * 1 + + lookup[2] * 50 + + lookup[3] * 1 * 50 + + lookup[4] * 1 + + lookup[5] * np.power(50, 3) + ) + expected_mass_2 = ( + lookup[0] + + lookup[1] * 2 + + lookup[2] * 60 + + lookup[3] * 2 * 60 + + lookup[4] * np.power(2, 2) + + lookup[5] * np.power(60, 3) + ) - expected_mass_calculation = np.array([ - [np.e ** expected_mass_1], - [np.e ** expected_mass_2]]) + expected_mass_calculation = np.array( + [[np.e**expected_mass_1], [np.e**expected_mass_2]] + ) np.testing.assert_array_equal(actual_mass, expected_mass_calculation) @@ -62,17 +93,27 @@ def test_calculate_mass_per_charge(self): ENERGY_LOST_IN_CARBON_FOIL = 0 CONVERSION_CONSTANT_K = 1.692e-5 - mass_per_charge_1 = (1 + POST_ACCELERATION_VOLTAGE_IN_KV - ENERGY_LOST_IN_CARBON_FOIL) * ( - 5 ** 2) * CONVERSION_CONSTANT_K + mass_per_charge_1 = ( + (1 + POST_ACCELERATION_VOLTAGE_IN_KV - ENERGY_LOST_IN_CARBON_FOIL) + * (5**2) + * CONVERSION_CONSTANT_K + ) - mass_per_charge_2 = (2 + POST_ACCELERATION_VOLTAGE_IN_KV - ENERGY_LOST_IN_CARBON_FOIL) * ( - 6 ** 2) * CONVERSION_CONSTANT_K + mass_per_charge_2 = ( + (2 + POST_ACCELERATION_VOLTAGE_IN_KV - ENERGY_LOST_IN_CARBON_FOIL) + * (6**2) + * CONVERSION_CONSTANT_K + ) actual_mass_per_charge = calculate_mass_per_charge(energy_per_charge, tof) - np.testing.assert_array_equal(actual_mass_per_charge, np.array([[mass_per_charge_1], [mass_per_charge_2]])) + np.testing.assert_array_equal( + actual_mass_per_charge, np.array([[mass_per_charge_1], [mass_per_charge_2]]) + ) def test_calculate_mass_per_charge_handles_fill_value(self): - tof = np.ma.masked_array([[432, 234], [434, 347]], mask=[[False, True], [True, False]]) + tof = np.ma.masked_array( + [[432, 234], [434, 347]], mask=[[False, True], [True, False]] + ) apd_energy = np.array([[np.nan, 2342.2], [4324.8, np.nan]]) actual_mass_per_charge = calculate_mass_per_charge(apd_energy, tof) @@ -86,7 +127,9 @@ def test_calculate_mass_handles_fill_value(self): rng = np.random.default_rng() mass_coefficients = MassCoefficientLookup(coefficients=rng.random(size=6)) - tof = np.ma.masked_array([[432, 234], [434, 347]], mask=[[False, True], [True, False]]) + tof = np.ma.masked_array( + [[432, 234], [434, 347]], mask=[[False, True], [True, False]] + ) apd_energy = np.array([[np.nan, 2342.2], [4324.8, np.nan]]) actual_mass = calculate_mass(apd_energy, tof, mass_coefficients) @@ -103,28 +146,40 @@ def test_calculate_partial_densities(self): intensities = rng.random((len(epochs), len(energy_steps), 1)) mass_per_charge = 10 - partial_densities = calculate_partial_densities(intensities, energy_steps, mass_per_charge) + partial_densities = calculate_partial_densities( + intensities, energy_steps, mass_per_charge + ) expected_partial_densities = np.sum( - 2.283e-8 * np.deg2rad(15) * np.deg2rad(15) * .4 * intensities * np.sqrt( - energy_steps[np.newaxis, :, np.newaxis]) * np.sqrt(mass_per_charge), axis=(1, 2)) + 2.283e-8 + * np.deg2rad(15) + * np.deg2rad(15) + * 0.4 + * intensities + * np.sqrt(energy_steps[np.newaxis, :, np.newaxis]) + * np.sqrt(mass_per_charge), + axis=(1, 2), + ) np.testing.assert_array_equal(expected_partial_densities, partial_densities) def test_calculate_partial_densities_ignores_nan_intensities(self): energy_steps = np.array([1, 100, 10000]) - intensities = np.array([ - [[1], [2], [3]], - [[4], [5], [np.nan]] - ]) + intensities = np.array([[[1], [2], [3]], [[4], [5], [np.nan]]]) mass_per_charge = 10 - partial_densities = calculate_partial_densities(intensities, energy_steps, mass_per_charge) + partial_densities = calculate_partial_densities( + intensities, energy_steps, mass_per_charge + ) - expected_partial_densities = 2.283e-8 * np.deg2rad(15) * np.deg2rad(15) * .4 * np.array([ - 1 + 2 * 10 + 3 * 100, - 4 + 5 * 10 - ]) * np.sqrt(mass_per_charge) + expected_partial_densities = ( + 2.283e-8 + * np.deg2rad(15) + * np.deg2rad(15) + * 0.4 + * np.array([1 + 2 * 10 + 3 * 100, 4 + 5 * 10]) + * np.sqrt(mass_per_charge) + ) np.testing.assert_allclose(expected_partial_densities, partial_densities) @@ -137,59 +192,83 @@ def test_rebin_to_counts_by_species_elevation_and_spin_sector(self): # [[He+, Fe, He+, no call], [O+5, no call, no call, no call]], # [[Mg, no call, None, no call], [no call, no call, no call, no call]] - expected_species_returned = [ - "He+", "Fe", "He+", "O+5", - "Mg", None - ] + expected_species_returned = ["He+", "Fe", "He+", "O+5", "Mg", None] mock_species_mass_range_lookup = Mock(spec=MassSpeciesBinLookup) - mock_species_mass_range_lookup.get_species.side_effect = expected_species_returned + mock_species_mass_range_lookup.get_species.side_effect = ( + expected_species_returned + ) mock_species_mass_range_lookup.get_num_species.return_value = num_species def mock_get_species_index(species): - return_values = {"He+": 0, "Fe": 1, "Mg": 2, "O+5": 3, } + return_values = { + "He+": 0, + "Fe": 1, + "Mg": 2, + "O+5": 3, + } return return_values[species] mock_species_mass_range_lookup.get_species_index = mock_get_species_index - num_events = np.array([[4, 2], - [3, 1]]) - num_events = np.ma.masked_array(num_events, mask=[[False, False], [False, True]]) - - normalized_per_event = np.array([ - [[100.78, 30.9, 200.1, np.nan], [50.458, np.nan, np.nan, np.nan]], - [[60.22, np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan, np.nan]] - ]) - - spin_sector_indices = np.array([ - [[2, 1, 2, 0], [0, 0, 0, 0]], - [[2, 2, 2, 0], [0, 0, 0, 0]] - ]) - - apd_id = np.ma.masked_array(data=np.array([ - [[2, 24, 2, 0], [1, 25, 255, 255]], - [[9, 9, 9, 255], [1, 255, 255, 255]] - ]), mask=np.array([ - [[False, False, False, False], [False, False, True, True]], - [[False, True, False, True], [False, True, True, True]], - ])) - - energy_step = np.ma.masked_array(data=np.array([ - [[0, 100, 0, 0], [0, 100, 255, 255]], - [[127, 255, 127, 255], [0, 255, 255, 255]] - ]), mask=np.array([ - [[False, False, False, False], [False, False, True, True]], - [[False, True, False, True], [False, True, True, True]], - ])) - - mass = np.array([ - [[6, 5, 4, np.nan], [7, 5, np.nan, np.nan]], - [[3, 3, 3, np.nan], [7, np.nan, np.nan, np.nan]], - ]) - - mass_per_charge = np.array([ - [[1, 2, 3, np.nan], [8, 2, np.nan, np.nan]], - [[4, 4, 4, np.nan], [9, np.nan, np.nan, np.nan]], - ]) + num_events = np.array([[4, 2], [3, 1]]) + num_events = np.ma.masked_array( + num_events, mask=[[False, False], [False, True]] + ) + + normalized_per_event = np.array( + [ + [[100.78, 30.9, 200.1, np.nan], [50.458, np.nan, np.nan, np.nan]], + [[60.22, np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan, np.nan]], + ] + ) + + spin_sector_indices = np.array( + [[[2, 1, 2, 0], [0, 0, 0, 0]], [[2, 2, 2, 0], [0, 0, 0, 0]]] + ) + + apd_id = np.ma.masked_array( + data=np.array( + [ + [[2, 24, 2, 0], [1, 25, 255, 255]], + [[9, 9, 9, 255], [1, 255, 255, 255]], + ] + ), + mask=np.array( + [ + [[False, False, False, False], [False, False, True, True]], + [[False, True, False, True], [False, True, True, True]], + ] + ), + ) + + energy_step = np.ma.masked_array( + data=np.array( + [ + [[0, 100, 0, 0], [0, 100, 255, 255]], + [[127, 255, 127, 255], [0, 255, 255, 255]], + ] + ), + mask=np.array( + [ + [[False, False, False, False], [False, False, True, True]], + [[False, True, False, True], [False, True, True, True]], + ] + ), + ) + + mass = np.array( + [ + [[6, 5, 4, np.nan], [7, 5, np.nan, np.nan]], + [[3, 3, 3, np.nan], [7, np.nan, np.nan, np.nan]], + ] + ) + + mass_per_charge = np.array( + [ + [[1, 2, 3, np.nan], [8, 2, np.nan, np.nan]], + [[4, 4, 4, np.nan], [9, np.nan, np.nan, np.nan]], + ] + ) direct_event_data = Mock(spec=CodiceLoDirectEventData) direct_event_data.mass = mass @@ -205,29 +284,35 @@ def mock_get_species_index(species): mock_species_mass_range_lookup, ) - mock_species_mass_range_lookup.get_species.assert_has_calls([ - call(mass[0, 0, 0], mass_per_charge[0, 0, 0]), - call(mass[0, 0, 1], mass_per_charge[0, 0, 1]), - call(mass[0, 0, 2], mass_per_charge[0, 0, 2]), - - call(mass[0, 1, 0], mass_per_charge[0, 1, 0]), - - call(mass[1, 0, 0], mass_per_charge[1, 0, 0]), - call(mass[1, 0, 2], mass_per_charge[1, 0, 2]), - ]) + mock_species_mass_range_lookup.get_species.assert_has_calls( + [ + call(mass[0, 0, 0], mass_per_charge[0, 0, 0]), + call(mass[0, 0, 1], mass_per_charge[0, 0, 1]), + call(mass[0, 0, 2], mass_per_charge[0, 0, 2]), + call(mass[0, 1, 0], mass_per_charge[0, 1, 0]), + call(mass[1, 0, 0], mass_per_charge[1, 0, 0]), + call(mass[1, 0, 2], mass_per_charge[1, 0, 2]), + ] + ) self.assertIsInstance(actual_counts_3d_data, np.ndarray) - output_shape = [num_species, num_epochs, num_priorities, num_esa_steps, num_spin_sectors, - CODICE_LO_NUM_AZIMUTH_BINS] - self.assertEqual( - tuple(output_shape), - actual_counts_3d_data.shape) + output_shape = [ + num_species, + num_epochs, + num_priorities, + num_esa_steps, + num_spin_sectors, + CODICE_LO_NUM_AZIMUTH_BINS, + ] + self.assertEqual(tuple(output_shape), actual_counts_3d_data.shape) rebinned_shape = tuple(output_shape[1:]) expected_he_plus_counts = np.zeros(rebinned_shape) expected_he_plus_counts[0, 0, 0, 2, 1] = 300.88 - np.testing.assert_array_equal(actual_counts_3d_data[0, ...], expected_he_plus_counts) + np.testing.assert_array_equal( + actual_counts_3d_data[0, ...], expected_he_plus_counts + ) expected_fe_counts = np.zeros(rebinned_shape) expected_fe_counts[0, 0, 100, 1, 23] = 30.9 @@ -247,19 +332,28 @@ def test_rebin_direct_events_by_energy_and_spin_sector(self): num_priorities = 3 num_epochs = 1 event_buffer_len = 15 - num_events = np.ma.masked_array(np.array([[1, 2, 4]]), mask=[[False, True, False]]) - spin_sector = np.zeros((num_epochs, num_priorities, event_buffer_len), dtype=np.uint8) + num_events = np.ma.masked_array( + np.array([[1, 2, 4]]), mask=[[False, True, False]] + ) + spin_sector = np.zeros( + (num_epochs, num_priorities, event_buffer_len), dtype=np.uint8 + ) spin_sector[0, 0, :1] = [3] spin_sector[0, 2, :4] = [5, 6, 5, 7] - energy_step = np.zeros((num_epochs, num_priorities, event_buffer_len), dtype=np.uint8) + energy_step = np.zeros( + (num_epochs, num_priorities, event_buffer_len), dtype=np.uint8 + ) energy_step[0, 0, :1] = [7] energy_step[0, 2, :4] = [1, 3, 1, 4] - result = rebin_direct_events_by_energy_and_spin_sector(num_events, spin_sector, energy_step, - num_spin_angle_bins, num_energy_bins) + result = rebin_direct_events_by_energy_and_spin_sector( + num_events, spin_sector, energy_step, num_spin_angle_bins, num_energy_bins + ) - expected_rebinned_counts = np.zeros((num_epochs, num_priorities, num_energy_bins, num_spin_angle_bins)) + expected_rebinned_counts = np.zeros( + (num_epochs, num_priorities, num_energy_bins, num_spin_angle_bins) + ) expected_rebinned_counts[0, 0, 7, 3] = 1 expected_rebinned_counts[0, 2, 1, 5] = 2 expected_rebinned_counts[0, 2, 3, 6] = 1 @@ -267,28 +361,36 @@ def test_rebin_direct_events_by_energy_and_spin_sector(self): np.testing.assert_array_equal(result, expected_rebinned_counts) - def test_rebin_direct_events_by_energy_and_spin_sector_ignores_masked_events(self): num_energy_bins = 30 num_spin_angle_bins = 20 num_priorities = 3 num_epochs = 1 event_buffer_len = 15 - num_events = np.ma.masked_array(np.array([[1, 2, 4]]), mask=[[False, True, False]]) - spin_sector = np.ma.zeros((num_epochs, num_priorities, event_buffer_len), dtype=np.uint8) + num_events = np.ma.masked_array( + np.array([[1, 2, 4]]), mask=[[False, True, False]] + ) + spin_sector = np.ma.zeros( + (num_epochs, num_priorities, event_buffer_len), dtype=np.uint8 + ) spin_sector[0, 0, :1] = [3] spin_sector[0, 2, :4] = [5, 6, 255, 7] spin_sector[0, 2, 2] = np.ma.masked - energy_step = np.ma.zeros((num_epochs, num_priorities, event_buffer_len), dtype=np.uint8) + energy_step = np.ma.zeros( + (num_epochs, num_priorities, event_buffer_len), dtype=np.uint8 + ) energy_step[0, 0, :1] = [7] energy_step[0, 2, :4] = [1, 255, 1, 4] energy_step[0, 2, 1] = np.ma.masked - result = rebin_direct_events_by_energy_and_spin_sector(num_events, spin_sector, energy_step, - num_spin_angle_bins, num_energy_bins) + result = rebin_direct_events_by_energy_and_spin_sector( + num_events, spin_sector, energy_step, num_spin_angle_bins, num_energy_bins + ) - expected_rebinned_counts = np.zeros((num_epochs, num_priorities, num_energy_bins, num_spin_angle_bins)) + expected_rebinned_counts = np.zeros( + (num_epochs, num_priorities, num_energy_bins, num_spin_angle_bins) + ) expected_rebinned_counts[0, 0, 7, 3] = 1 expected_rebinned_counts[0, 2, 1, 5] = 1 expected_rebinned_counts[0, 2, 3, 6] = 0 @@ -302,19 +404,28 @@ def test_rebin_direct_events_for_normalization(self): num_priorities = 3 num_epochs = 1 event_buffer_len = 15 - num_events = np.ma.masked_array(np.array([[2, 2, 4]]), mask=[[False, True, False]]) - spin_sector = np.zeros((num_epochs, num_priorities, event_buffer_len), dtype=np.uint8) + num_events = np.ma.masked_array( + np.array([[2, 2, 4]]), mask=[[False, True, False]] + ) + spin_sector = np.zeros( + (num_epochs, num_priorities, event_buffer_len), dtype=np.uint8 + ) spin_sector[0, 0, :2] = [3, 3 + 12] spin_sector[0, 2, :4] = [5, 6, 5, 7 + 12] - energy_step = np.zeros((num_epochs, num_priorities, event_buffer_len), dtype=np.uint8) + energy_step = np.zeros( + (num_epochs, num_priorities, event_buffer_len), dtype=np.uint8 + ) energy_step[0, 0, :2] = [7, 7] energy_step[0, 2, :4] = [1, 3, 1, 4] - result = rebin_direct_events_for_normalization(num_events, spin_sector, energy_step, num_spin_sectors, - num_energy_bins) + result = rebin_direct_events_for_normalization( + num_events, spin_sector, energy_step, num_spin_sectors, num_energy_bins + ) - expected_rebinned_counts = np.zeros((num_epochs, num_priorities, num_energy_bins, num_spin_sectors)) + expected_rebinned_counts = np.zeros( + (num_epochs, num_priorities, num_energy_bins, num_spin_sectors) + ) expected_rebinned_counts[0, 0, 7, 3] = 2 expected_rebinned_counts[0, 2, 1, 5] = 2 expected_rebinned_counts[0, 2, 3, 6] = 1 @@ -333,23 +444,35 @@ def test_calculate_normalization_factor(self): num_esa_steps = 128 num_l1_spin_sectors = 12 num_l2_spin_sectors = 2 * num_l1_spin_sectors - priority_counts = np.zeros((num_epochs, num_priorities, num_esa_steps, num_l1_spin_sectors)) + priority_counts = np.zeros( + (num_epochs, num_priorities, num_esa_steps, num_l1_spin_sectors) + ) priority_counts[0, 0, 0, 0] = 4 priority_counts[0, 0, 0, 2] = 20 event_buffer_len = 15 num_events = np.zeros((num_epochs, num_priorities), dtype=int) - spin_sectors = np.zeros((num_epochs, num_priorities, event_buffer_len), dtype=int) - energy_steps = np.zeros((num_epochs, num_priorities, event_buffer_len), dtype=int) - + spin_sectors = np.zeros( + (num_epochs, num_priorities, event_buffer_len), dtype=int + ) + energy_steps = np.zeros( + (num_epochs, num_priorities, event_buffer_len), dtype=int + ) num_events[0, 0] = 4 spin_sectors[0, 0, :4] = [0, 2, 12, 0] energy_steps[0, 0, :4] = [0, 0, 0, 0] - result = calculate_normalization_factor(priority_counts, num_events, energy_steps, spin_sectors) + result = calculate_normalization_factor( + priority_counts, num_events, energy_steps, spin_sectors + ) - expected_shape = (num_epochs, num_priorities, num_esa_steps, num_l2_spin_sectors) + expected_shape = ( + num_epochs, + num_priorities, + num_esa_steps, + num_l2_spin_sectors, + ) expected = np.full(expected_shape, 0.0) expected[0, 0, 0, 0] = 4 / 3 expected[0, 0, 0, 0 + 12] = 4 / 3 @@ -364,22 +487,35 @@ def test_calculate_normalization_factor_floor_is_one(self): num_esa_steps = 128 num_l1_spin_sectors = 12 num_l2_spin_sectors = 2 * num_l1_spin_sectors - priority_counts = np.zeros((num_epochs, num_priorities, num_esa_steps, num_l1_spin_sectors)) - priority_counts[0, 0, 0, 0] = 1. - priority_counts[0, 0, 0, 2] = 0. + priority_counts = np.zeros( + (num_epochs, num_priorities, num_esa_steps, num_l1_spin_sectors) + ) + priority_counts[0, 0, 0, 0] = 1.0 + priority_counts[0, 0, 0, 2] = 0.0 event_buffer_len = 15 num_events = np.zeros((num_epochs, num_priorities), dtype=int) - spin_sectors = np.zeros((num_epochs, num_priorities, event_buffer_len), dtype=int) - energy_steps = np.zeros((num_epochs, num_priorities, event_buffer_len), dtype=int) + spin_sectors = np.zeros( + (num_epochs, num_priorities, event_buffer_len), dtype=int + ) + energy_steps = np.zeros( + (num_epochs, num_priorities, event_buffer_len), dtype=int + ) num_events[0, 0] = 3 spin_sectors[0, 0, :3] = [0, 12, 2] energy_steps[0, 0, :3] = [0, 0, 0] - result = calculate_normalization_factor(priority_counts, num_events, energy_steps, spin_sectors) + result = calculate_normalization_factor( + priority_counts, num_events, energy_steps, spin_sectors + ) - expected_shape = (num_epochs, num_priorities, num_esa_steps, num_l2_spin_sectors) + expected_shape = ( + num_epochs, + num_priorities, + num_esa_steps, + num_l2_spin_sectors, + ) expected = np.full(expected_shape, 0.0) self.assertEqual(1.0, result[0, 0, 0, 0]) self.assertEqual(1.0, result[0, 0, 0, 0 + 12]) @@ -395,18 +531,31 @@ def test_normalization_factor_is_fill_when_priority_nonzero_and_direct_zero(self num_esa_steps = 128 num_l1_spin_sectors = 12 num_l2_spin_sectors = 2 * num_l1_spin_sectors - priority_counts = np.zeros((num_epochs, num_priorities, num_esa_steps, num_l1_spin_sectors)) - priority_counts[0, 0, 0, 0] = 1. - priority_counts[0, 0, 0, 2] = 50. + priority_counts = np.zeros( + (num_epochs, num_priorities, num_esa_steps, num_l1_spin_sectors) + ) + priority_counts[0, 0, 0, 0] = 1.0 + priority_counts[0, 0, 0, 2] = 50.0 event_buffer_len = 15 num_events = np.zeros((num_epochs, num_priorities), dtype=int) - spin_sectors = np.zeros((num_epochs, num_priorities, event_buffer_len), dtype=int) - energy_steps = np.zeros((num_epochs, num_priorities, event_buffer_len), dtype=int) + spin_sectors = np.zeros( + (num_epochs, num_priorities, event_buffer_len), dtype=int + ) + energy_steps = np.zeros( + (num_epochs, num_priorities, event_buffer_len), dtype=int + ) - result = calculate_normalization_factor(priority_counts, num_events, energy_steps, spin_sectors) + result = calculate_normalization_factor( + priority_counts, num_events, energy_steps, spin_sectors + ) - expected_shape = (num_epochs, num_priorities, num_esa_steps, num_l2_spin_sectors) + expected_shape = ( + num_epochs, + num_priorities, + num_esa_steps, + num_l2_spin_sectors, + ) expected = np.full(expected_shape, 0.0) np.testing.assert_equal(result[0, 0, 0, 0], np.nan) np.testing.assert_equal(result[0, 0, 0, 0 + 12], np.nan) @@ -423,28 +572,44 @@ def test_lookup_normalization_per_event(self): num_l1_spin_sectors = 12 num_l2_spin_sectors = 2 * num_l1_spin_sectors - normalization = np.zeros((num_epochs, num_priorities, num_esa_steps, num_l2_spin_sectors)) - normalization[0, 0, 0, 0] = 123. - normalization[0, 0, 0, 12] = 456. - normalization[0, 0, 1, 2] = 789. + normalization = np.zeros( + (num_epochs, num_priorities, num_esa_steps, num_l2_spin_sectors) + ) + normalization[0, 0, 0, 0] = 123.0 + normalization[0, 0, 0, 12] = 456.0 + normalization[0, 0, 1, 2] = 789.0 event_buffer_len = 15 num_events = np.zeros((num_epochs, num_priorities), dtype=int) - spin_sectors = np.ma.array(np.full((num_epochs, num_priorities, event_buffer_len), 255, dtype=int), mask=True) - energy_steps = np.ma.array(np.full((num_epochs, num_priorities, event_buffer_len), 255, dtype=int), mask=True) + spin_sectors = np.ma.array( + np.full((num_epochs, num_priorities, event_buffer_len), 255, dtype=int), + mask=True, + ) + energy_steps = np.ma.array( + np.full((num_epochs, num_priorities, event_buffer_len), 255, dtype=int), + mask=True, + ) num_events[0, 0] = 3 spin_sectors[0, 0, :3] = [0, 12, 2] energy_steps[0, 0, :3] = [0, 0, 1] - normalization_per_event = lookup_normalization_per_event(normalization, num_events, energy_steps, spin_sectors) + normalization_per_event = lookup_normalization_per_event( + normalization, num_events, energy_steps, spin_sectors + ) - np.testing.assert_array_equal(normalization_per_event[0, 0, :3], [123, 456, 789]) + np.testing.assert_array_equal( + normalization_per_event[0, 0, :3], [123, 456, 789] + ) - expected_normalization_per_event = np.full((num_epochs, num_priorities, event_buffer_len), np.nan) + expected_normalization_per_event = np.full( + (num_epochs, num_priorities, event_buffer_len), np.nan + ) expected_normalization_per_event[0, 0, :3] = [123, 456, 789] - np.testing.assert_array_equal(normalization_per_event, expected_normalization_per_event) + np.testing.assert_array_equal( + normalization_per_event, expected_normalization_per_event + ) def test_lookup_normalization_per_event_skips_masked_num_events(self): num_epochs = 2 @@ -453,20 +618,36 @@ def test_lookup_normalization_per_event_skips_masked_num_events(self): num_l1_spin_sectors = 12 num_l2_spin_sectors = 2 * num_l1_spin_sectors - normalization = np.zeros((num_epochs, num_priorities, num_esa_steps, num_l2_spin_sectors)) - normalization[0, 0, 0, 0] = 123. - normalization[0, 0, 0, 12] = 456. - normalization[0, 0, 1, 2] = 789. + normalization = np.zeros( + (num_epochs, num_priorities, num_esa_steps, num_l2_spin_sectors) + ) + normalization[0, 0, 0, 0] = 123.0 + normalization[0, 0, 0, 12] = 456.0 + normalization[0, 0, 1, 2] = 789.0 event_buffer_len = 15 - num_events = np.ma.array(np.full((num_epochs, num_priorities), 65535), mask=True) - spin_sectors = np.ma.array(np.full((num_epochs, num_priorities, event_buffer_len), 255, dtype=int), mask=True) - energy_steps = np.ma.array(np.full((num_epochs, num_priorities, event_buffer_len), 255, dtype=int), mask=True) + num_events = np.ma.array( + np.full((num_epochs, num_priorities), 65535), mask=True + ) + spin_sectors = np.ma.array( + np.full((num_epochs, num_priorities, event_buffer_len), 255, dtype=int), + mask=True, + ) + energy_steps = np.ma.array( + np.full((num_epochs, num_priorities, event_buffer_len), 255, dtype=int), + mask=True, + ) - normalization_per_event = lookup_normalization_per_event(normalization, num_events, energy_steps, spin_sectors) + normalization_per_event = lookup_normalization_per_event( + normalization, num_events, energy_steps, spin_sectors + ) - expected_normalization_per_event = np.full((num_epochs, num_priorities, event_buffer_len), np.nan) - np.testing.assert_array_equal(normalization_per_event, expected_normalization_per_event) + expected_normalization_per_event = np.full( + (num_epochs, num_priorities, event_buffer_len), np.nan + ) + np.testing.assert_array_equal( + normalization_per_event, expected_normalization_per_event + ) def test_lookup_normalization_per_event_ignores_masked_events(self): num_epochs = 2 @@ -475,15 +656,23 @@ def test_lookup_normalization_per_event_ignores_masked_events(self): num_l1_spin_sectors = 12 num_l2_spin_sectors = 2 * num_l1_spin_sectors - normalization = np.zeros((num_epochs, num_priorities, num_esa_steps, num_l2_spin_sectors)) - normalization[0, 0, 0, 0] = 123. - normalization[0, 0, 0, 12] = 456. - normalization[0, 0, 1, 2] = 789. + normalization = np.zeros( + (num_epochs, num_priorities, num_esa_steps, num_l2_spin_sectors) + ) + normalization[0, 0, 0, 0] = 123.0 + normalization[0, 0, 0, 12] = 456.0 + normalization[0, 0, 1, 2] = 789.0 event_buffer_len = 15 num_events = np.zeros((num_epochs, num_priorities), dtype=int) - spin_sectors = np.ma.array(np.full((num_epochs, num_priorities, event_buffer_len), 255, dtype=int), mask=True) - energy_steps = np.ma.array(np.full((num_epochs, num_priorities, event_buffer_len), 255, dtype=int), mask=True) + spin_sectors = np.ma.array( + np.full((num_epochs, num_priorities, event_buffer_len), 255, dtype=int), + mask=True, + ) + energy_steps = np.ma.array( + np.full((num_epochs, num_priorities, event_buffer_len), 255, dtype=int), + mask=True, + ) num_events[0, 0] = 4 energy_steps[0, 0, 1] = 0 @@ -492,14 +681,22 @@ def test_lookup_normalization_per_event_ignores_masked_events(self): spin_sectors[0, 0, :4] = np.ma.masked_equal([255, 255, 0, 12], 255) energy_steps[0, 0, :4] = np.ma.masked_equal([255, 0, 255, 0], 255) - normalization_per_event = lookup_normalization_per_event(normalization, num_events, energy_steps, spin_sectors) + normalization_per_event = lookup_normalization_per_event( + normalization, num_events, energy_steps, spin_sectors + ) - np.testing.assert_array_equal(normalization_per_event[0, 0, :4], [np.nan, np.nan, np.nan, 456]) + np.testing.assert_array_equal( + normalization_per_event[0, 0, :4], [np.nan, np.nan, np.nan, 456] + ) - expected_normalization_per_event = np.full((num_epochs, num_priorities, event_buffer_len), np.nan) + expected_normalization_per_event = np.full( + (num_epochs, num_priorities, event_buffer_len), np.nan + ) expected_normalization_per_event[0, 0, :4] = [np.nan, np.nan, np.nan, 456] - np.testing.assert_array_equal(normalization_per_event, expected_normalization_per_event) + np.testing.assert_array_equal( + normalization_per_event, expected_normalization_per_event + ) def test_combine_priorities_for_species_and_calculate_count_uncertainty(self): num_epochs = 2 @@ -508,20 +705,32 @@ def test_combine_priorities_for_species_and_calculate_count_uncertainty(self): num_azimuth_bins = 5 rng = np.random.default_rng() - priority_1 = rng.random((num_epochs, num_energies, num_spin_sectors, num_azimuth_bins)) - priority_2 = rng.random((num_epochs, num_energies, num_spin_sectors, num_azimuth_bins)) - priority_3 = rng.random((num_epochs, num_energies, num_spin_sectors, num_azimuth_bins)) + priority_1 = rng.random( + (num_epochs, num_energies, num_spin_sectors, num_azimuth_bins) + ) + priority_2 = rng.random( + (num_epochs, num_energies, num_spin_sectors, num_azimuth_bins) + ) + priority_3 = rng.random( + (num_epochs, num_energies, num_spin_sectors, num_azimuth_bins) + ) counts = np.stack((priority_1, priority_2, priority_3), axis=1) - actual_counts, actual_count_uncertainty = combine_priorities_for_species_and_calculate_count_uncertainty(counts) + actual_counts, actual_count_uncertainty = ( + combine_priorities_for_species_and_calculate_count_uncertainty(counts) + ) expected_summed_counts = priority_1 + priority_2 + priority_3 - self.assertEqual((num_epochs, num_energies, num_spin_sectors, num_azimuth_bins), - actual_counts.shape) + self.assertEqual( + (num_epochs, num_energies, num_spin_sectors, num_azimuth_bins), + actual_counts.shape, + ) np.testing.assert_array_equal(actual_counts, expected_summed_counts) - np.testing.assert_array_equal(actual_count_uncertainty, np.sqrt(expected_summed_counts)) + np.testing.assert_array_equal( + actual_count_uncertainty, np.sqrt(expected_summed_counts) + ) def test_convert_count_rate_to_intensity(self): num_epochs = 3 @@ -530,26 +739,47 @@ def test_convert_count_rate_to_intensity(self): num_energies = 6 rng = np.random.default_rng() - counts = rng.random((num_epochs, num_energies, num_spin_angles, num_position_bins)) - acquisition_times = rng.random((num_epochs, num_energies,)) - energy_per_charge = EnergyLookup(bin_centers=rng.random(num_energies), - delta_plus=rng.random(num_energies), - delta_minus=rng.random(num_energies)) - geometric_factor = rng.random((num_epochs, num_energies, num_spin_angles, num_position_bins)) + counts = rng.random( + (num_epochs, num_energies, num_spin_angles, num_position_bins) + ) + acquisition_times = rng.random( + ( + num_epochs, + num_energies, + ) + ) + energy_per_charge = EnergyLookup( + bin_centers=rng.random(num_energies), + delta_plus=rng.random(num_energies), + delta_minus=rng.random(num_energies), + ) + geometric_factor = rng.random( + (num_epochs, num_energies, num_spin_angles, num_position_bins) + ) efficiency = rng.random((num_energies, num_position_bins)) mock_efficiency_lookup = Mock() mock_efficiency_lookup.efficiency_data = efficiency - intensity_data = convert_counts_to_intensity(counts, acquisition_times, energy_per_charge, mock_efficiency_lookup, - geometric_factor) + intensity_data = convert_counts_to_intensity( + counts, + acquisition_times, + energy_per_charge, + mock_efficiency_lookup, + geometric_factor, + ) - expected_denominator = (energy_per_charge.bin_centers[np.newaxis, :, np.newaxis, np.newaxis] - * geometric_factor - * efficiency[np.newaxis, :, np.newaxis, :]) + expected_denominator = ( + energy_per_charge.bin_centers[np.newaxis, :, np.newaxis, np.newaxis] + * geometric_factor + * efficiency[np.newaxis, :, np.newaxis, :] + ) - np.testing.assert_array_almost_equal(intensity_data * expected_denominator, counts/acquisition_times[:,:,np.newaxis, np.newaxis]) + np.testing.assert_array_almost_equal( + intensity_data * expected_denominator, + counts / acquisition_times[:, :, np.newaxis, np.newaxis], + ) def test_rebin_azimuth_to_elevation(self): num_epochs = 3 @@ -557,7 +787,9 @@ def test_rebin_azimuth_to_elevation(self): num_spin_angles = 24 num_azimuth_bins = 24 - intensity_data = np.zeros((num_epochs, num_energies, num_spin_angles, num_azimuth_bins)) + intensity_data = np.zeros( + (num_epochs, num_energies, num_spin_angles, num_azimuth_bins) + ) azimuth_1_intensity = 1 azimuth_2_intensity = 2 azimuth_3_intensity = 3 @@ -574,19 +806,32 @@ def test_rebin_azimuth_to_elevation(self): mock_position_to_elevation_lookup = Mock(spec=PositionToElevationLookup) mock_position_to_elevation_lookup.num_bins = num_elevation_bins mock_position_to_elevation_lookup.bin_centers = np.arange(num_elevation_bins) - mock_position_to_elevation_lookup.apd_to_elevation_index.return_value = np.array([1, 0, 2, 2]) + mock_position_to_elevation_lookup.apd_to_elevation_index.return_value = ( + np.array([1, 0, 2, 2]) + ) half_spin_per_esa = np.zeros((num_epochs, num_energies)) - actual_rebinned = rebin_3d_distribution_azimuth_to_elevation(intensity_data, - azimuths, - mock_position_to_elevation_lookup, - half_spin_per_esa) + actual_rebinned = rebin_3d_distribution_azimuth_to_elevation( + intensity_data, + azimuths, + mock_position_to_elevation_lookup, + half_spin_per_esa, + ) - mock_position_to_elevation_lookup.apd_to_elevation_index.assert_called_once_with(azimuths) + mock_position_to_elevation_lookup.apd_to_elevation_index.assert_called_once_with( + azimuths + ) - self.assertEqual((num_epochs, num_energies, num_spin_angles, num_elevation_bins), - actual_rebinned.shape) + self.assertEqual( + (num_epochs, num_energies, num_spin_angles, num_elevation_bins), + actual_rebinned.shape, + ) self.assertTrue(np.all(actual_rebinned[:, :, :, 0] == azimuth_2_intensity)) self.assertTrue(np.all(actual_rebinned[:, :, :, 1] == azimuth_1_intensity)) - self.assertTrue(np.all(actual_rebinned[:, :, :, 2] == (azimuth_3_intensity + azimuth_4_intensity))) + self.assertTrue( + np.all( + actual_rebinned[:, :, :, 2] + == (azimuth_3_intensity + azimuth_4_intensity) + ) + ) diff --git a/tests/codice/l3/lo/sectored_intensities/test_mass_per_charge_lookup.py b/tests/codice/l3/lo/sectored_intensities/test_mass_per_charge_lookup.py index 0be392c9f..ca56e1d18 100644 --- a/tests/codice/l3/lo/sectored_intensities/test_mass_per_charge_lookup.py +++ b/tests/codice/l3/lo/sectored_intensities/test_mass_per_charge_lookup.py @@ -1,14 +1,20 @@ import unittest -from imap_l3_processing.codice.l3.lo.sectored_intensities.science.mass_per_charge_lookup import MassPerChargeLookup +from imap_l3_processing.codice.l3.lo.sectored_intensities.science.mass_per_charge_lookup import ( + MassPerChargeLookup, +) from tests.test_helpers import get_test_data_path class TestMassPerChargeLookup(unittest.TestCase): def test_read_from_file(self): - mass_per_charge_csv_path = get_test_data_path("codice/imap_codice_mass-per-charge_20241110_v002.csv") + mass_per_charge_csv_path = get_test_data_path( + "codice/imap_codice_mass-per-charge_20241110_v002.csv" + ) - mass_per_charge_lookup = MassPerChargeLookup.read_from_file(mass_per_charge_csv_path) + mass_per_charge_lookup = MassPerChargeLookup.read_from_file( + mass_per_charge_csv_path + ) self.assertEqual(1, mass_per_charge_lookup.hplus) self.assertEqual(2, mass_per_charge_lookup.heplusplus) @@ -26,5 +32,5 @@ def test_read_from_file(self): self.assertEqual(7.25, mass_per_charge_lookup.fe_hiq) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/codice/l3/lo/test_codice_lo_l3a_direct_events_dependencies.py b/tests/codice/l3/lo/test_codice_lo_l3a_direct_events_dependencies.py index c1b7966a1..14c14ff13 100644 --- a/tests/codice/l3/lo/test_codice_lo_l3a_direct_events_dependencies.py +++ b/tests/codice/l3/lo/test_codice_lo_l3a_direct_events_dependencies.py @@ -2,25 +2,44 @@ from pathlib import Path from unittest.mock import patch, call, sentinel -from imap_data_access.processing_input import ProcessingInputCollection, ScienceInput, AncillaryInput - -from imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies import \ - CodiceLoL3aDirectEventsDependencies, DIRECT_EVENTS_DESCRIPTOR, SW_PRIORITY_DESCRIPTOR, NSW_PRIORITY_DESCRIPTOR, \ - MASS_COEFFICIENT_DESCRIPTOR -from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import \ - ESA_TO_ENERGY_PER_CHARGE_LOOKUP_DESCRIPTOR +from imap_data_access.processing_input import ( + ProcessingInputCollection, + ScienceInput, + AncillaryInput, +) + +from imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies import ( + CodiceLoL3aDirectEventsDependencies, + DIRECT_EVENTS_DESCRIPTOR, + SW_PRIORITY_DESCRIPTOR, + NSW_PRIORITY_DESCRIPTOR, + MASS_COEFFICIENT_DESCRIPTOR, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import ( + ESA_TO_ENERGY_PER_CHARGE_LOOKUP_DESCRIPTOR, +) class TestCodiceLoL3aDirectEventsDependencies(unittest.TestCase): - - @patch("imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies.download") @patch( - "imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies.CodiceLoL3aDirectEventsDependencies.from_file_paths") + "imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies.download" + ) + @patch( + "imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies.CodiceLoL3aDirectEventsDependencies.from_file_paths" + ) def test_fetch_dependencies(self, mock_from_file_paths, mock_download): - direct_events_science_file_name = f"imap_codice_l2_{DIRECT_EVENTS_DESCRIPTOR}_20100105_v010.cdf" - lo_sw_priority_science_file_name = f"imap_codice_l1a_{SW_PRIORITY_DESCRIPTOR}_20100105_v010.cdf" - lo_nsw_priority_science_file_name = f"imap_codice_l1a_{NSW_PRIORITY_DESCRIPTOR}_20100105_v010.cdf" - mass_coefficient_file_name = f"imap_codice_{MASS_COEFFICIENT_DESCRIPTOR}_20100105_v001.csv" + direct_events_science_file_name = ( + f"imap_codice_l2_{DIRECT_EVENTS_DESCRIPTOR}_20100105_v010.cdf" + ) + lo_sw_priority_science_file_name = ( + f"imap_codice_l1a_{SW_PRIORITY_DESCRIPTOR}_20100105_v010.cdf" + ) + lo_nsw_priority_science_file_name = ( + f"imap_codice_l1a_{NSW_PRIORITY_DESCRIPTOR}_20100105_v010.cdf" + ) + mass_coefficient_file_name = ( + f"imap_codice_{MASS_COEFFICIENT_DESCRIPTOR}_20100105_v001.csv" + ) energy_lookup_file_name = f"imap_codice_{ESA_TO_ENERGY_PER_CHARGE_LOOKUP_DESCRIPTOR}_20100105_v001.csv" mock_download.side_effect = [ @@ -28,7 +47,7 @@ def test_fetch_dependencies(self, mock_from_file_paths, mock_download): Path("some_local_folder") / lo_nsw_priority_science_file_name, Path("some_local_folder") / direct_events_science_file_name, Path("some_local_folder") / mass_coefficient_file_name, - Path("some_local_folder") / energy_lookup_file_name + Path("some_local_folder") / energy_lookup_file_name, ] processing_input_collection = ProcessingInputCollection( @@ -37,67 +56,107 @@ def test_fetch_dependencies(self, mock_from_file_paths, mock_download): ScienceInput(lo_nsw_priority_science_file_name), ScienceInput(lo_sw_priority_science_file_name), ScienceInput(direct_events_science_file_name), - - AncillaryInput("imap/ancillary/hit/imap_codice_range-2A-cosine-lookup_20250203_v001.csv"), - ScienceInput("imap/hit/l3/2010/01/imap_hit_l3_direct-events_20100105_v010.cdf"), - ScienceInput("imap_codice_l1_direct-events_20100105_v010.cdf") + AncillaryInput( + "imap/ancillary/hit/imap_codice_range-2A-cosine-lookup_20250203_v001.csv" + ), + ScienceInput( + "imap/hit/l3/2010/01/imap_hit_l3_direct-events_20100105_v010.cdf" + ), + ScienceInput("imap_codice_l1_direct-events_20100105_v010.cdf"), ) - codice_l3_dependencies = CodiceLoL3aDirectEventsDependencies.fetch_dependencies(processing_input_collection) + codice_l3_dependencies = CodiceLoL3aDirectEventsDependencies.fetch_dependencies( + processing_input_collection + ) - mock_download.assert_has_calls([ - call(lo_sw_priority_science_file_name), - call(lo_nsw_priority_science_file_name), - call(direct_events_science_file_name), - call(mass_coefficient_file_name), - call(energy_lookup_file_name) - ], any_order=True) + mock_download.assert_has_calls( + [ + call(lo_sw_priority_science_file_name), + call(lo_nsw_priority_science_file_name), + call(direct_events_science_file_name), + call(mass_coefficient_file_name), + call(energy_lookup_file_name), + ], + any_order=True, + ) mock_from_file_paths.assert_called_with( Path("some_local_folder") / lo_sw_priority_science_file_name, Path("some_local_folder") / lo_nsw_priority_science_file_name, Path("some_local_folder") / direct_events_science_file_name, Path("some_local_folder") / mass_coefficient_file_name, - Path("some_local_folder") / energy_lookup_file_name + Path("some_local_folder") / energy_lookup_file_name, ) self.assertEqual(mock_from_file_paths.return_value, codice_l3_dependencies) @patch( - 'imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies.EnergyLookup.read_from_csv') + "imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies.EnergyLookup.read_from_csv" + ) @patch( - 'imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies.CodiceLoL2DirectEventData.read_from_cdf') + "imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies.CodiceLoL2DirectEventData.read_from_cdf" + ) @patch( - 'imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies.MassCoefficientLookup.read_from_csv') + "imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies.MassCoefficientLookup.read_from_csv" + ) @patch( - 'imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies.CodiceLoL1aSWPriorityRates.read_from_cdf') + "imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies.CodiceLoL1aSWPriorityRates.read_from_cdf" + ) @patch( - 'imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies.CodiceLoL1aNSWPriorityRates.read_from_cdf') - def test_from_file_paths(self, mock_codice_lo_l1a_nsw_priority_rates, - mock_codice_lo_l1a_sw_priority_rates, - mock_mass_coefficient_lookup, - mock_direct_event_read_from_cdf, mock_energy_lookup_read_from_csv): + "imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies.CodiceLoL1aNSWPriorityRates.read_from_cdf" + ) + def test_from_file_paths( + self, + mock_codice_lo_l1a_nsw_priority_rates, + mock_codice_lo_l1a_sw_priority_rates, + mock_mass_coefficient_lookup, + mock_direct_event_read_from_cdf, + mock_energy_lookup_read_from_csv, + ): actual_dependencies = CodiceLoL3aDirectEventsDependencies.from_file_paths( sentinel.sw_priority, sentinel.nsw_priority, sentinel.direct_events_cdf_path, sentinel.mass_coefficient_lookup_path, - sentinel.energy_lookup_path + sentinel.energy_lookup_path, ) - mock_direct_event_read_from_cdf.assert_called_once_with(sentinel.direct_events_cdf_path) - mock_mass_coefficient_lookup.assert_called_once_with(sentinel.mass_coefficient_lookup_path) - mock_codice_lo_l1a_sw_priority_rates.assert_called_once_with(sentinel.sw_priority) - mock_codice_lo_l1a_nsw_priority_rates.assert_called_once_with(sentinel.nsw_priority) - mock_energy_lookup_read_from_csv.assert_called_once_with(sentinel.energy_lookup_path) + mock_direct_event_read_from_cdf.assert_called_once_with( + sentinel.direct_events_cdf_path + ) + mock_mass_coefficient_lookup.assert_called_once_with( + sentinel.mass_coefficient_lookup_path + ) + mock_codice_lo_l1a_sw_priority_rates.assert_called_once_with( + sentinel.sw_priority + ) + mock_codice_lo_l1a_nsw_priority_rates.assert_called_once_with( + sentinel.nsw_priority + ) + mock_energy_lookup_read_from_csv.assert_called_once_with( + sentinel.energy_lookup_path + ) - self.assertEqual(mock_direct_event_read_from_cdf.return_value, actual_dependencies.codice_l2_direct_events) - self.assertEqual(mock_mass_coefficient_lookup.return_value, actual_dependencies.mass_coefficient_lookup) - self.assertEqual(mock_codice_lo_l1a_sw_priority_rates.return_value, - actual_dependencies.codice_lo_l1a_sw_priority_rates) - self.assertEqual(mock_codice_lo_l1a_nsw_priority_rates.return_value, - actual_dependencies.codice_lo_l1a_nsw_priority_rates) - self.assertEqual(mock_energy_lookup_read_from_csv.return_value, actual_dependencies.energy_lookup) + self.assertEqual( + mock_direct_event_read_from_cdf.return_value, + actual_dependencies.codice_l2_direct_events, + ) + self.assertEqual( + mock_mass_coefficient_lookup.return_value, + actual_dependencies.mass_coefficient_lookup, + ) + self.assertEqual( + mock_codice_lo_l1a_sw_priority_rates.return_value, + actual_dependencies.codice_lo_l1a_sw_priority_rates, + ) + self.assertEqual( + mock_codice_lo_l1a_nsw_priority_rates.return_value, + actual_dependencies.codice_lo_l1a_nsw_priority_rates, + ) + self.assertEqual( + mock_energy_lookup_read_from_csv.return_value, + actual_dependencies.energy_lookup, + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/codice/l3/lo/test_codice_lo_l3a_partial_densities_dependencies.py b/tests/codice/l3/lo/test_codice_lo_l3a_partial_densities_dependencies.py index f5be265e4..79ddb70d2 100644 --- a/tests/codice/l3/lo/test_codice_lo_l3a_partial_densities_dependencies.py +++ b/tests/codice/l3/lo/test_codice_lo_l3a_partial_densities_dependencies.py @@ -3,72 +3,119 @@ from unittest.mock import patch, call, sentinel import imap_data_access -from imap_data_access.processing_input import ProcessingInputCollection, ScienceInput, AncillaryInput +from imap_data_access.processing_input import ( + ProcessingInputCollection, + ScienceInput, + AncillaryInput, +) -from imap_l3_processing.codice.l3.lo.codice_lo_l3a_partial_densities_dependencies import \ - CodiceLoL3aPartialDensitiesDependencies, SW_SPECIES_DESCRIPTOR, MASS_PER_CHARGE_DESCRIPTOR +from imap_l3_processing.codice.l3.lo.codice_lo_l3a_partial_densities_dependencies import ( + CodiceLoL3aPartialDensitiesDependencies, + SW_SPECIES_DESCRIPTOR, + MASS_PER_CHARGE_DESCRIPTOR, +) class TestCodiceLoL3aPartialDensitiesDependencies(unittest.TestCase): - - @patch("imap_l3_processing.codice.l3.lo.codice_lo_l3a_partial_densities_dependencies.imap_data_access.download") @patch( - "imap_l3_processing.codice.l3.lo.codice_lo_l3a_partial_densities_dependencies.CodiceLoL3aPartialDensitiesDependencies.from_file_paths") - def test_fetch_dependencies(self, mock_from_file_paths, mock_imap_data_access_download): + "imap_l3_processing.codice.l3.lo.codice_lo_l3a_partial_densities_dependencies.imap_data_access.download" + ) + @patch( + "imap_l3_processing.codice.l3.lo.codice_lo_l3a_partial_densities_dependencies.CodiceLoL3aPartialDensitiesDependencies.from_file_paths" + ) + def test_fetch_dependencies( + self, mock_from_file_paths, mock_imap_data_access_download + ): expected_codice_science_file_download_path = f"imap/codice/l2/2010/01/imap_codice_l2_{SW_SPECIES_DESCRIPTOR}_20100105_v010.cdf" expected_mass_per_charge_file_download_path = f"imap/ancillary/codice/imap_codice_{MASS_PER_CHARGE_DESCRIPTOR}_20100105_v001.csv" - sectored_science_input = ScienceInput(f"imap_codice_l2_{SW_SPECIES_DESCRIPTOR}_20100105_v010.cdf") - ancillary_input = AncillaryInput(f"imap_codice_{MASS_PER_CHARGE_DESCRIPTOR}_20100105_v001.csv") - - non_codice_ancillary_input = AncillaryInput("imap_codice_range-2A-cosine-lookup_20250203_v001.csv") - non_codice_science_input = ScienceInput("imap/hit/l3/2010/01/imap_hit_l3_direct-events_20100105_v010.cdf") - non_l2_codice_science_input = ScienceInput("imap_codice_l1_direct-events_20100105_v010.cdf") + sectored_science_input = ScienceInput( + f"imap_codice_l2_{SW_SPECIES_DESCRIPTOR}_20100105_v010.cdf" + ) + ancillary_input = AncillaryInput( + f"imap_codice_{MASS_PER_CHARGE_DESCRIPTOR}_20100105_v001.csv" + ) + + non_codice_ancillary_input = AncillaryInput( + "imap_codice_range-2A-cosine-lookup_20250203_v001.csv" + ) + non_codice_science_input = ScienceInput( + "imap/hit/l3/2010/01/imap_hit_l3_direct-events_20100105_v010.cdf" + ) + non_l2_codice_science_input = ScienceInput( + "imap_codice_l1_direct-events_20100105_v010.cdf" + ) processing_input_collection = ProcessingInputCollection( - sectored_science_input, ancillary_input, - non_codice_ancillary_input, non_codice_science_input, - non_l2_codice_science_input) - - expected_local_codice_science_file_path = Path("some_local_folder") / expected_codice_science_file_download_path - expected_local_mass_per_charge_file_download_path = Path( - "some_local_folder") / expected_mass_per_charge_file_download_path + sectored_science_input, + ancillary_input, + non_codice_ancillary_input, + non_codice_science_input, + non_l2_codice_science_input, + ) + + expected_local_codice_science_file_path = ( + Path("some_local_folder") / expected_codice_science_file_download_path + ) + expected_local_mass_per_charge_file_download_path = ( + Path("some_local_folder") / expected_mass_per_charge_file_download_path + ) mock_imap_data_access_download.side_effect = [ expected_local_codice_science_file_path, - expected_local_mass_per_charge_file_download_path + expected_local_mass_per_charge_file_download_path, ] - codice_l3_dependencies = CodiceLoL3aPartialDensitiesDependencies.fetch_dependencies(processing_input_collection) + codice_l3_dependencies = ( + CodiceLoL3aPartialDensitiesDependencies.fetch_dependencies( + processing_input_collection + ) + ) data_dir = imap_data_access.config["DATA_DIR"] - expected_download_sectored_science_path = data_dir / expected_codice_science_file_download_path - expected_download_ancillary_path = data_dir / expected_mass_per_charge_file_download_path - - mock_imap_data_access_download.assert_has_calls([ - call(expected_download_sectored_science_path.name), - call(expected_download_ancillary_path.name), - - ], any_order=True) - - mock_from_file_paths.assert_called_with(expected_local_codice_science_file_path, - expected_local_mass_per_charge_file_download_path) + expected_download_sectored_science_path = ( + data_dir / expected_codice_science_file_download_path + ) + expected_download_ancillary_path = ( + data_dir / expected_mass_per_charge_file_download_path + ) + + mock_imap_data_access_download.assert_has_calls( + [ + call(expected_download_sectored_science_path.name), + call(expected_download_ancillary_path.name), + ], + any_order=True, + ) + + mock_from_file_paths.assert_called_with( + expected_local_codice_science_file_path, + expected_local_mass_per_charge_file_download_path, + ) self.assertEqual(mock_from_file_paths.return_value, codice_l3_dependencies) @patch( - 'imap_l3_processing.codice.l3.lo.codice_lo_l3a_partial_densities_dependencies.CodiceLoL2SWSpeciesData.read_from_cdf') + "imap_l3_processing.codice.l3.lo.codice_lo_l3a_partial_densities_dependencies.CodiceLoL2SWSpeciesData.read_from_cdf" + ) @patch( - 'imap_l3_processing.codice.l3.lo.codice_lo_l3a_partial_densities_dependencies.MassPerChargeLookup.read_from_file') + "imap_l3_processing.codice.l3.lo.codice_lo_l3a_partial_densities_dependencies.MassPerChargeLookup.read_from_file" + ) def test_from_file_paths(self, mock_mpc_read_from_file, mock_read_from_cdf): actual_dependencies = CodiceLoL3aPartialDensitiesDependencies.from_file_paths( - sentinel.cdf_path, sentinel.mpc_lookup_path) + sentinel.cdf_path, sentinel.mpc_lookup_path + ) mock_mpc_read_from_file.assert_called_once_with(sentinel.mpc_lookup_path) mock_read_from_cdf.assert_called_once_with(sentinel.cdf_path) - self.assertEqual(mock_read_from_cdf.return_value, actual_dependencies.codice_l2_lo_data) - self.assertEqual(mock_mpc_read_from_file.return_value, actual_dependencies.mass_per_charge_lookup) + self.assertEqual( + mock_read_from_cdf.return_value, actual_dependencies.codice_l2_lo_data + ) + self.assertEqual( + mock_mpc_read_from_file.return_value, + actual_dependencies.mass_per_charge_lookup, + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/codice/l3/lo/test_codice_lo_l3a_ratios_dependencies.py b/tests/codice/l3/lo/test_codice_lo_l3a_ratios_dependencies.py index 4767993b6..bc119f6df 100644 --- a/tests/codice/l3/lo/test_codice_lo_l3a_ratios_dependencies.py +++ b/tests/codice/l3/lo/test_codice_lo_l3a_ratios_dependencies.py @@ -3,30 +3,42 @@ from imap_data_access.processing_input import ProcessingInputCollection, ScienceInput -from imap_l3_processing.codice.l3.lo.codice_lo_l3a_ratios_dependencies import CodiceLoL3aRatiosDependencies +from imap_l3_processing.codice.l3.lo.codice_lo_l3a_ratios_dependencies import ( + CodiceLoL3aRatiosDependencies, +) class TestCodiceLoL3aRatiosDependencies(unittest.TestCase): - @patch( - 'imap_l3_processing.codice.l3.lo.codice_lo_l3a_ratios_dependencies.CodiceLoPartialDensityData.read_from_cdf') + "imap_l3_processing.codice.l3.lo.codice_lo_l3a_ratios_dependencies.CodiceLoPartialDensityData.read_from_cdf" + ) def test_from_file_paths(self, mock_read_from_cdf): - actual_dependencies = CodiceLoL3aRatiosDependencies.from_file_paths(sentinel.cdf_path) + actual_dependencies = CodiceLoL3aRatiosDependencies.from_file_paths( + sentinel.cdf_path + ) mock_read_from_cdf.assert_called_once_with(sentinel.cdf_path) - self.assertEqual(mock_read_from_cdf.return_value, actual_dependencies.partial_density_data) + self.assertEqual( + mock_read_from_cdf.return_value, actual_dependencies.partial_density_data + ) @patch( - "imap_l3_processing.codice.l3.lo.codice_lo_l3a_ratios_dependencies.CodiceLoPartialDensityData.read_from_cdf") + "imap_l3_processing.codice.l3.lo.codice_lo_l3a_ratios_dependencies.CodiceLoPartialDensityData.read_from_cdf" + ) @patch( - "imap_l3_processing.codice.l3.lo.codice_lo_l3a_ratios_dependencies.imap_data_access.download") + "imap_l3_processing.codice.l3.lo.codice_lo_l3a_ratios_dependencies.imap_data_access.download" + ) def test_fetch_dependencies(self, mock_download, mock_read_from_cdf): expected_file_name = "imap_codice_l3a_lo-partial-densities_20200507_v002.cdf" science_input = ScienceInput(expected_file_name) processing_inputs = ProcessingInputCollection(science_input) - dependencies = CodiceLoL3aRatiosDependencies.fetch_dependencies(processing_inputs) + dependencies = CodiceLoL3aRatiosDependencies.fetch_dependencies( + processing_inputs + ) mock_download.assert_called_once_with(expected_file_name) - self.assertEqual(mock_read_from_cdf.return_value, dependencies.partial_density_data) + self.assertEqual( + mock_read_from_cdf.return_value, dependencies.partial_density_data + ) diff --git a/tests/codice/l3/lo/test_codice_lo_l3b_3d_distributions_dependencies.py b/tests/codice/l3/lo/test_codice_lo_l3b_3d_distributions_dependencies.py index 77fabb35e..2857b554c 100644 --- a/tests/codice/l3/lo/test_codice_lo_l3b_3d_distributions_dependencies.py +++ b/tests/codice/l3/lo/test_codice_lo_l3b_3d_distributions_dependencies.py @@ -1,13 +1,21 @@ import unittest from unittest.mock import patch, sentinel, call -from imap_data_access.processing_input import ProcessingInputCollection, ScienceInput, AncillaryInput +from imap_data_access.processing_input import ( + ProcessingInputCollection, + ScienceInput, + AncillaryInput, +) -from imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies import \ - CodiceLoL3b3dDistributionsDependencies, MASS_SPECIES_BIN_LOOKUP_DESCRIPTOR, GEOMETRIC_FACTOR_LOOKUP_DESCRIPTOR, \ - EFFICIENCY_FACTOR_LOOKUP_DESCRIPTOR -from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import \ - ESA_TO_ENERGY_PER_CHARGE_LOOKUP_DESCRIPTOR +from imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies import ( + CodiceLoL3b3dDistributionsDependencies, + MASS_SPECIES_BIN_LOOKUP_DESCRIPTOR, + GEOMETRIC_FACTOR_LOOKUP_DESCRIPTOR, + EFFICIENCY_FACTOR_LOOKUP_DESCRIPTOR, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import ( + ESA_TO_ENERGY_PER_CHARGE_LOOKUP_DESCRIPTOR, +) class TestCodiceLoL3b3dDistributions(unittest.TestCase): @@ -17,25 +25,41 @@ def test_fetch_dependencies_multiple_species(self): self._test_fetch_dependencies(species) @patch( - "imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies.imap_data_access.download") + "imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies.imap_data_access.download" + ) @patch( - "imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies.CodiceLoL3b3dDistributionsDependencies.from_file_paths") - def _test_fetch_dependencies(self, species: str, mock_3d_distribution_deps_from_file_paths, - mock_data_access_download): + "imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies.CodiceLoL3b3dDistributionsDependencies.from_file_paths" + ) + def _test_fetch_dependencies( + self, + species: str, + mock_3d_distribution_deps_from_file_paths, + mock_data_access_download, + ): l3a_direct_event_name = "imap_codice_l3a_lo-direct-events_20100105_v010.cdf" - mass_species_lut_name = f"imap_codice_{MASS_SPECIES_BIN_LOOKUP_DESCRIPTOR}_20100105_v010.csv" - geometric_factors_lut_name = f"imap_codice_{GEOMETRIC_FACTOR_LOOKUP_DESCRIPTOR}_20100105_v010.csv" - efficiency_factors_lut_name = f"imap_codice_{EFFICIENCY_FACTOR_LOOKUP_DESCRIPTOR}_20100105_v010.csv" + mass_species_lut_name = ( + f"imap_codice_{MASS_SPECIES_BIN_LOOKUP_DESCRIPTOR}_20100105_v010.csv" + ) + geometric_factors_lut_name = ( + f"imap_codice_{GEOMETRIC_FACTOR_LOOKUP_DESCRIPTOR}_20100105_v010.csv" + ) + efficiency_factors_lut_name = ( + f"imap_codice_{EFFICIENCY_FACTOR_LOOKUP_DESCRIPTOR}_20100105_v010.csv" + ) energy_per_charge_lut_name = f"imap_codice_{ESA_TO_ENERGY_PER_CHARGE_LOOKUP_DESCRIPTOR}_20100105_v010.csv" - unused_science_input = ScienceInput("imap_codice_l2_lo-direct-events_20100105_v010.cdf") + unused_science_input = ScienceInput( + "imap_codice_l2_lo-direct-events_20100105_v010.cdf" + ) - processing_input_collection = ProcessingInputCollection(AncillaryInput(mass_species_lut_name), - AncillaryInput(geometric_factors_lut_name), - AncillaryInput(efficiency_factors_lut_name), - AncillaryInput(energy_per_charge_lut_name), - ScienceInput(l3a_direct_event_name), - unused_science_input) + processing_input_collection = ProcessingInputCollection( + AncillaryInput(mass_species_lut_name), + AncillaryInput(geometric_factors_lut_name), + AncillaryInput(efficiency_factors_lut_name), + AncillaryInput(energy_per_charge_lut_name), + ScienceInput(l3a_direct_event_name), + unused_science_input, + ) mock_data_access_download.side_effect = [ sentinel.l3a_direct_event_downloaded_path, @@ -45,15 +69,19 @@ def _test_fetch_dependencies(self, species: str, mock_3d_distribution_deps_from_ sentinel.energy_per_charge_downloaded_path, ] - CodiceLoL3b3dDistributionsDependencies.fetch_dependencies(processing_input_collection, species) + CodiceLoL3b3dDistributionsDependencies.fetch_dependencies( + processing_input_collection, species + ) - mock_data_access_download.assert_has_calls([ - call(l3a_direct_event_name), - call(mass_species_lut_name), - call(geometric_factors_lut_name), - call(efficiency_factors_lut_name), - call(energy_per_charge_lut_name) - ]) + mock_data_access_download.assert_has_calls( + [ + call(l3a_direct_event_name), + call(mass_species_lut_name), + call(geometric_factors_lut_name), + call(efficiency_factors_lut_name), + call(energy_per_charge_lut_name), + ] + ) mock_3d_distribution_deps_from_file_paths.assert_called_once_with( l3a_file_path=sentinel.l3a_direct_event_downloaded_path, @@ -64,30 +92,48 @@ def _test_fetch_dependencies(self, species: str, mock_3d_distribution_deps_from_ species=species, ) - @patch('imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies.EfficiencyLookup.read_from_csv') @patch( - 'imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies.GeometricFactorLookup.read_from_csv') + "imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies.EfficiencyLookup.read_from_csv" + ) @patch( - 'imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies.MassSpeciesBinLookup.read_from_csv') + "imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies.GeometricFactorLookup.read_from_csv" + ) @patch( - 'imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies.CodiceLoDirectEventData.read_from_cdf') - @patch('imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies.EnergyLookup.read_from_csv') - def test_from_file_paths(self, mock_energy_lookup_from_csv, mock_l3a_direct_event_read_from_cdf, - mock_mass_species_from_csv, mock_geometric_factor_from_csv, - mock_efficiency_lut_read_from_csv): + "imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies.MassSpeciesBinLookup.read_from_csv" + ) + @patch( + "imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies.CodiceLoDirectEventData.read_from_cdf" + ) + @patch( + "imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies.EnergyLookup.read_from_csv" + ) + def test_from_file_paths( + self, + mock_energy_lookup_from_csv, + mock_l3a_direct_event_read_from_cdf, + mock_mass_species_from_csv, + mock_geometric_factor_from_csv, + mock_efficiency_lut_read_from_csv, + ): actual = CodiceLoL3b3dDistributionsDependencies.from_file_paths( l3a_file_path=sentinel.l3a_path, mass_species_bin_lut=sentinel.mass_species_path, geometric_factors_lut=sentinel.geometric_factor_path, efficiency_factors_lut=sentinel.efficiency_factor_path, energy_per_charge_lut=sentinel.energy_per_charge_path, - species="some-species" + species="some-species", ) mock_l3a_direct_event_read_from_cdf.assert_called_once_with(sentinel.l3a_path) mock_mass_species_from_csv.assert_called_once_with(sentinel.mass_species_path) - mock_geometric_factor_from_csv.assert_called_once_with(sentinel.geometric_factor_path) - mock_efficiency_lut_read_from_csv.assert_called_once_with(sentinel.efficiency_factor_path, "some-species") - mock_energy_lookup_from_csv.assert_called_once_with(sentinel.energy_per_charge_path) + mock_geometric_factor_from_csv.assert_called_once_with( + sentinel.geometric_factor_path + ) + mock_efficiency_lut_read_from_csv.assert_called_once_with( + sentinel.efficiency_factor_path, "some-species" + ) + mock_energy_lookup_from_csv.assert_called_once_with( + sentinel.energy_per_charge_path + ) expected_dependencies = CodiceLoL3b3dDistributionsDependencies( l3a_direct_event_data=mock_l3a_direct_event_read_from_cdf.return_value, @@ -95,5 +141,6 @@ def test_from_file_paths(self, mock_energy_lookup_from_csv, mock_l3a_direct_even geometric_factors_lookup=mock_geometric_factor_from_csv.return_value, efficiency_factors_lut=mock_efficiency_lut_read_from_csv.return_value, energy_per_charge_lut=mock_energy_lookup_from_csv.return_value, - species="some-species") + species="some-species", + ) self.assertEqual(expected_dependencies, actual) diff --git a/tests/codice/l3/lo/test_codice_lo_processor.py b/tests/codice/l3/lo/test_codice_lo_processor.py index 5fe7a4529..11b0db787 100644 --- a/tests/codice/l3/lo/test_codice_lo_processor.py +++ b/tests/codice/l3/lo/test_codice_lo_processor.py @@ -8,25 +8,54 @@ import numpy as np from imap_data_access.processing_input import ProcessingInputCollection -from imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies import \ - CodiceLoL3b3dDistributionsDependencies -from imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies import CodiceLoL3aDirectEventsDependencies -from imap_l3_processing.codice.l3.lo.codice_lo_l3a_partial_densities_dependencies import \ - CodiceLoL3aPartialDensitiesDependencies -from imap_l3_processing.codice.l3.lo.codice_lo_l3a_ratios_dependencies import CodiceLoL3aRatiosDependencies +from imap_l3_processing.codice.l3.lo.codice_lo_l3b_3d_distributions_dependencies import ( + CodiceLoL3b3dDistributionsDependencies, +) +from imap_l3_processing.codice.l3.lo.codice_lo_l3a_direct_events_dependencies import ( + CodiceLoL3aDirectEventsDependencies, +) +from imap_l3_processing.codice.l3.lo.codice_lo_l3a_partial_densities_dependencies import ( + CodiceLoL3aPartialDensitiesDependencies, +) +from imap_l3_processing.codice.l3.lo.codice_lo_l3a_ratios_dependencies import ( + CodiceLoL3aRatiosDependencies, +) from imap_l3_processing.codice.l3.lo.codice_lo_processor import CodiceLoProcessor -from imap_l3_processing.codice.l3.lo.direct_events.science.angle_lookup import SpinAngleLookup -from imap_l3_processing.codice.l3.lo.direct_events.science.efficiency_lookup import EfficiencyLookup -from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import EnergyLookup -from imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup import GeometricFactorLookup -from imap_l3_processing.codice.l3.lo.models import CodiceLoL3aPartialDensityDataProduct, CodiceLoL2DirectEventData, \ - CodiceLoL3aDirectEventDataProduct, CodiceLoL2SWSpeciesData, \ - CodiceLoL1aSWPriorityRates, CodiceLoL1aNSWPriorityRates, CodiceLoPartialDensityData, CodiceLoL3aRatiosDataProduct, \ - CODICE_LO_L2_NUM_PRIORITIES, CodiceLoL3ChargeStateDistributionsDataProduct, CodiceLoL3b3dDistributionDataProduct -from imap_l3_processing.codice.l3.lo.sectored_intensities.science.mass_per_charge_lookup import MassPerChargeLookup +from imap_l3_processing.codice.l3.lo.direct_events.science.angle_lookup import ( + SpinAngleLookup, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.efficiency_lookup import ( + EfficiencyLookup, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.energy_lookup import ( + EnergyLookup, +) +from imap_l3_processing.codice.l3.lo.direct_events.science.geometric_factor_lookup import ( + GeometricFactorLookup, +) +from imap_l3_processing.codice.l3.lo.models import ( + CodiceLoL3aPartialDensityDataProduct, + CodiceLoL2DirectEventData, + CodiceLoL3aDirectEventDataProduct, + CodiceLoL2SWSpeciesData, + CodiceLoL1aSWPriorityRates, + CodiceLoL1aNSWPriorityRates, + CodiceLoPartialDensityData, + CodiceLoL3aRatiosDataProduct, + CODICE_LO_L2_NUM_PRIORITIES, + CodiceLoL3ChargeStateDistributionsDataProduct, + CodiceLoL3b3dDistributionDataProduct, +) +from imap_l3_processing.codice.l3.lo.sectored_intensities.science.mass_per_charge_lookup import ( + MassPerChargeLookup, +) from imap_l3_processing.models import InputMetadata from imap_l3_processing.processor import Processor -from tests.test_helpers import create_dataclass_mock, get_test_data_path, NumpyArrayMatcher +from tests.test_helpers import ( + create_dataclass_mock, + get_test_data_path, + NumpyArrayMatcher, +) MODULE = "imap_l3_processing.codice.l3.lo.codice_lo_processor" @@ -36,97 +65,154 @@ def test_implements_processor(self): processor = CodiceLoProcessor(Mock(), Mock()) self.assertIsInstance(processor, Processor) - @patch(f'{MODULE}.CodiceLoL3aPartialDensitiesDependencies.fetch_dependencies') - @patch(f'{MODULE}.CodiceLoProcessor.process_l3a_partial_densities') - @patch(f'{MODULE}.save_data') - @patch('imap_l3_processing.processor.spiceypy') - def test_process_partial_densities(self, mock_spiceypy, mock_save_data, mock_process_l3a_partial_densities, - mock_fetch_dependencies): + @patch(f"{MODULE}.CodiceLoL3aPartialDensitiesDependencies.fetch_dependencies") + @patch(f"{MODULE}.CodiceLoProcessor.process_l3a_partial_densities") + @patch(f"{MODULE}.save_data") + @patch("imap_l3_processing.processor.spiceypy") + def test_process_partial_densities( + self, + mock_spiceypy, + mock_save_data, + mock_process_l3a_partial_densities, + mock_fetch_dependencies, + ): input_collection = MagicMock() - input_collection.get_file_paths.return_value = [Path('path/to/parent_file_1'), Path('path/to/parent_file_2')] - input_metadata = InputMetadata(instrument='codice', - data_level="l3a", - start_date=Mock(spec=datetime), - end_date=Mock(spec=datetime), - version='v02', - descriptor='lo-partial-densities') + input_collection.get_file_paths.return_value = [ + Path("path/to/parent_file_1"), + Path("path/to/parent_file_2"), + ] + input_metadata = InputMetadata( + instrument="codice", + data_level="l3a", + start_date=Mock(spec=datetime), + end_date=Mock(spec=datetime), + version="v02", + descriptor="lo-partial-densities", + ) mock_spiceypy.ktotal.return_value = 0 - processor = CodiceLoProcessor(dependencies=input_collection, input_metadata=input_metadata) + processor = CodiceLoProcessor( + dependencies=input_collection, input_metadata=input_metadata + ) product = processor.process() mock_fetch_dependencies.assert_called_once_with(processor.dependencies) - mock_process_l3a_partial_densities.assert_called_once_with(mock_fetch_dependencies.return_value) + mock_process_l3a_partial_densities.assert_called_once_with( + mock_fetch_dependencies.return_value + ) - mock_save_data.assert_called_once_with(mock_process_l3a_partial_densities.return_value) + mock_save_data.assert_called_once_with( + mock_process_l3a_partial_densities.return_value + ) - self.assertEqual(['parent_file_1', 'parent_file_2'], - mock_process_l3a_partial_densities.return_value.parent_file_names) + self.assertEqual( + ["parent_file_1", "parent_file_2"], + mock_process_l3a_partial_densities.return_value.parent_file_names, + ) self.assertEqual([mock_save_data.return_value], product) - @patch(f'{MODULE}.CodiceLoL3aRatiosDependencies.fetch_dependencies') - @patch(f'{MODULE}.CodiceLoProcessor.process_l3a_ratios') - @patch(f'{MODULE}.save_data') - @patch('imap_l3_processing.processor.spiceypy') - def test_process_ratios(self, mock_spiceypy, mock_save_data, mock_process_l3a_ratios, - mock_fetch_dependencies): + @patch(f"{MODULE}.CodiceLoL3aRatiosDependencies.fetch_dependencies") + @patch(f"{MODULE}.CodiceLoProcessor.process_l3a_ratios") + @patch(f"{MODULE}.save_data") + @patch("imap_l3_processing.processor.spiceypy") + def test_process_ratios( + self, + mock_spiceypy, + mock_save_data, + mock_process_l3a_ratios, + mock_fetch_dependencies, + ): input_collection = MagicMock() - input_collection.get_file_paths.return_value = [Path('path/to/parent_file_1')] - input_metadata = InputMetadata(instrument='codice', - data_level="l3a", - start_date=Mock(spec=datetime), - end_date=Mock(spec=datetime), - version='v02', - descriptor='lo-sw-ratios') + input_collection.get_file_paths.return_value = [Path("path/to/parent_file_1")] + input_metadata = InputMetadata( + instrument="codice", + data_level="l3a", + start_date=Mock(spec=datetime), + end_date=Mock(spec=datetime), + version="v02", + descriptor="lo-sw-ratios", + ) mock_spiceypy.ktotal.return_value = 0 - processor = CodiceLoProcessor(dependencies=input_collection, input_metadata=input_metadata) + processor = CodiceLoProcessor( + dependencies=input_collection, input_metadata=input_metadata + ) product = processor.process() mock_fetch_dependencies.assert_called_once_with(processor.dependencies) - mock_process_l3a_ratios.assert_called_once_with(mock_fetch_dependencies.return_value) + mock_process_l3a_ratios.assert_called_once_with( + mock_fetch_dependencies.return_value + ) mock_save_data.assert_called_once_with(mock_process_l3a_ratios.return_value) - self.assertEqual(['parent_file_1'], - mock_process_l3a_ratios.return_value.parent_file_names) + self.assertEqual( + ["parent_file_1"], mock_process_l3a_ratios.return_value.parent_file_names + ) self.assertEqual([mock_save_data.return_value], product) - @patch(f'{MODULE}.CodiceLoL3aRatiosDependencies.fetch_dependencies') - @patch(f'{MODULE}.CodiceLoProcessor.process_l3a_charge_state_distributions') - @patch(f'{MODULE}.save_data') - @patch('imap_l3_processing.processor.spiceypy') - def test_process_abundances(self, mock_spiceypy, mock_save_data, mock_process_l3a_abundances, - mock_fetch_dependencies): + @patch(f"{MODULE}.CodiceLoL3aRatiosDependencies.fetch_dependencies") + @patch(f"{MODULE}.CodiceLoProcessor.process_l3a_charge_state_distributions") + @patch(f"{MODULE}.save_data") + @patch("imap_l3_processing.processor.spiceypy") + def test_process_abundances( + self, + mock_spiceypy, + mock_save_data, + mock_process_l3a_abundances, + mock_fetch_dependencies, + ): input_collection = MagicMock() - input_collection.get_file_paths.return_value = [Path('path/to/parent_file_1')] - input_metadata = InputMetadata(instrument='codice', - data_level="l3a", - start_date=Mock(spec=datetime), - end_date=Mock(spec=datetime), - version='v02', - descriptor='lo-sw-charge-state-distributions') + input_collection.get_file_paths.return_value = [Path("path/to/parent_file_1")] + input_metadata = InputMetadata( + instrument="codice", + data_level="l3a", + start_date=Mock(spec=datetime), + end_date=Mock(spec=datetime), + version="v02", + descriptor="lo-sw-charge-state-distributions", + ) mock_spiceypy.ktotal.return_value = 0 - processor = CodiceLoProcessor(dependencies=input_collection, input_metadata=input_metadata) + processor = CodiceLoProcessor( + dependencies=input_collection, input_metadata=input_metadata + ) product = processor.process() mock_fetch_dependencies.assert_called_once_with(processor.dependencies) - mock_process_l3a_abundances.assert_called_once_with(mock_fetch_dependencies.return_value) + mock_process_l3a_abundances.assert_called_once_with( + mock_fetch_dependencies.return_value + ) mock_save_data.assert_called_once_with(mock_process_l3a_abundances.return_value) - self.assertEqual(['parent_file_1'], - mock_process_l3a_abundances.return_value.parent_file_names) + self.assertEqual( + ["parent_file_1"], + mock_process_l3a_abundances.return_value.parent_file_names, + ) self.assertEqual([mock_save_data.return_value], product) def test_process_ratios_calculate_abundance_ratios(self): now = datetime.now() data = CodiceLoPartialDensityData( epoch=np.array( - [now, now + timedelta(minutes=4), now + timedelta(minutes=8), - now + timedelta(minutes=12), now + timedelta(minutes=16)]), - epoch_delta=np.array([120_000_000_000, 120_000_000_000, 120_000_000_000, 120_000_000_000, 120_000_000_000]), + [ + now, + now + timedelta(minutes=4), + now + timedelta(minutes=8), + now + timedelta(minutes=12), + now + timedelta(minutes=16), + ] + ), + epoch_delta=np.array( + [ + 120_000_000_000, + 120_000_000_000, + 120_000_000_000, + 120_000_000_000, + 120_000_000_000, + ] + ), fe_hiq_partial_density=np.array([1, 2, 3, 10, 12]), fe_loq_partial_density=np.array([4, 5, 6, 11, 13]), oplus5_partial_density=np.array([1, 2, 4, 12, 14]), @@ -145,32 +231,59 @@ def test_process_ratios_calculate_abundance_ratios(self): dependency = CodiceLoL3aRatiosDependencies(data) input_metadata = Mock() - processor = CodiceLoProcessor(dependencies=Mock(), input_metadata=input_metadata) + processor = CodiceLoProcessor( + dependencies=Mock(), input_metadata=input_metadata + ) - ratios_data_product: CodiceLoL3aRatiosDataProduct = processor.process_l3a_ratios(dependency) + ratios_data_product: CodiceLoL3aRatiosDataProduct = ( + processor.process_l3a_ratios(dependency) + ) self.assertEqual(input_metadata, ratios_data_product.input_metadata) - np.testing.assert_array_equal(ratios_data_product.epoch, - [now + timedelta(minutes=4), now + timedelta(minutes=14)]) - np.testing.assert_array_equal(ratios_data_product.epoch_delta, [360_000_000_000, 240_000_000_000]) - np.testing.assert_array_almost_equal(ratios_data_product.fe_to_o_ratio, - np.array( - [(2 + 5) / (7 / 3 + 6 + 9 + 12), (11 + 12) / (13 + 14 + 15 + 16)])) - np.testing.assert_array_almost_equal(ratios_data_product.mg_to_o_ratio, - np.array([15 / (7 / 3 + 6 + 9 + 12), 17 / (13 + 14 + 15 + 16)])) - np.testing.assert_array_almost_equal(ratios_data_product.c_to_o_ratio, np.array( - [(18 + 21 + 24) / (7 / 3 + 6 + 9 + 12), (18 + 19 + 20) / (13 + 14 + 15 + 16)])) - - np.testing.assert_array_almost_equal(ratios_data_product.c6_to_c4_ratio, np.array([24 / 18, 20 / 18])) - np.testing.assert_array_almost_equal(ratios_data_product.c6_to_c5_ratio, np.array([24 / 21, 20 / 19])) - - np.testing.assert_array_almost_equal(ratios_data_product.o7_to_o6_ratio, np.array([9 / 6, 15 / 14])) - np.testing.assert_array_almost_equal(ratios_data_product.felo_to_fehi_ratio, np.array([5 / 2, 12 / 11])) + np.testing.assert_array_equal( + ratios_data_product.epoch, + [now + timedelta(minutes=4), now + timedelta(minutes=14)], + ) + np.testing.assert_array_equal( + ratios_data_product.epoch_delta, [360_000_000_000, 240_000_000_000] + ) + np.testing.assert_array_almost_equal( + ratios_data_product.fe_to_o_ratio, + np.array([(2 + 5) / (7 / 3 + 6 + 9 + 12), (11 + 12) / (13 + 14 + 15 + 16)]), + ) + np.testing.assert_array_almost_equal( + ratios_data_product.mg_to_o_ratio, + np.array([15 / (7 / 3 + 6 + 9 + 12), 17 / (13 + 14 + 15 + 16)]), + ) + np.testing.assert_array_almost_equal( + ratios_data_product.c_to_o_ratio, + np.array( + [ + (18 + 21 + 24) / (7 / 3 + 6 + 9 + 12), + (18 + 19 + 20) / (13 + 14 + 15 + 16), + ] + ), + ) + + np.testing.assert_array_almost_equal( + ratios_data_product.c6_to_c4_ratio, np.array([24 / 18, 20 / 18]) + ) + np.testing.assert_array_almost_equal( + ratios_data_product.c6_to_c5_ratio, np.array([24 / 21, 20 / 19]) + ) + + np.testing.assert_array_almost_equal( + ratios_data_product.o7_to_o6_ratio, np.array([9 / 6, 15 / 14]) + ) + np.testing.assert_array_almost_equal( + ratios_data_product.felo_to_fehi_ratio, np.array([5 / 2, 12 / 11]) + ) def test_process_ratios_calculate_abundance_ratios_using_safe_divide(self): now = datetime.now() data = CodiceLoPartialDensityData( epoch=np.array( - [now, now + timedelta(minutes=4), now + timedelta(minutes=8)]), + [now, now + timedelta(minutes=4), now + timedelta(minutes=8)] + ), epoch_delta=np.array([120_000_000_000, 120_000_000_000, 120_000_000_000]), fe_hiq_partial_density=np.array([0, 0, 0]), fe_loq_partial_density=np.array([0, 0, 0]), @@ -190,37 +303,78 @@ def test_process_ratios_calculate_abundance_ratios_using_safe_divide(self): dependency = CodiceLoL3aRatiosDependencies(data) input_metadata = Mock() - processor = CodiceLoProcessor(dependencies=Mock(), input_metadata=input_metadata) + processor = CodiceLoProcessor( + dependencies=Mock(), input_metadata=input_metadata + ) with warnings.catch_warnings(record=True) as w: ratios_data_product = processor.process_l3a_ratios(dependency) self.assertEqual(0, len(w)) self.assertEqual(input_metadata, ratios_data_product.input_metadata) - np.testing.assert_array_equal(ratios_data_product.epoch, - [now + timedelta(minutes=4)]) - np.testing.assert_array_equal(ratios_data_product.epoch_delta, [360_000_000_000]) - np.testing.assert_array_almost_equal(ratios_data_product.fe_to_o_ratio, np.array([np.nan])) - np.testing.assert_array_almost_equal(ratios_data_product.mg_to_o_ratio, np.array([np.nan])) - np.testing.assert_array_almost_equal(ratios_data_product.c_to_o_ratio, np.array([np.nan])) - np.testing.assert_array_almost_equal(ratios_data_product.c6_to_c4_ratio, np.array([np.nan])) - np.testing.assert_array_almost_equal(ratios_data_product.c6_to_c5_ratio, np.array([np.nan])) - np.testing.assert_array_almost_equal(ratios_data_product.o7_to_o6_ratio, np.array([np.nan])) - np.testing.assert_array_almost_equal(ratios_data_product.felo_to_fehi_ratio, np.array([np.nan])) + np.testing.assert_array_equal( + ratios_data_product.epoch, [now + timedelta(minutes=4)] + ) + np.testing.assert_array_equal( + ratios_data_product.epoch_delta, [360_000_000_000] + ) + np.testing.assert_array_almost_equal( + ratios_data_product.fe_to_o_ratio, np.array([np.nan]) + ) + np.testing.assert_array_almost_equal( + ratios_data_product.mg_to_o_ratio, np.array([np.nan]) + ) + np.testing.assert_array_almost_equal( + ratios_data_product.c_to_o_ratio, np.array([np.nan]) + ) + np.testing.assert_array_almost_equal( + ratios_data_product.c6_to_c4_ratio, np.array([np.nan]) + ) + np.testing.assert_array_almost_equal( + ratios_data_product.c6_to_c5_ratio, np.array([np.nan]) + ) + np.testing.assert_array_almost_equal( + ratios_data_product.o7_to_o6_ratio, np.array([np.nan]) + ) + np.testing.assert_array_almost_equal( + ratios_data_product.felo_to_fehi_ratio, np.array([np.nan]) + ) def test_process_abundances_calculates_abundance_ratios(self): now = datetime.now() data = CodiceLoPartialDensityData( epoch=np.array( - [now, now + timedelta(minutes=4), now + timedelta(minutes=8), - now + timedelta(minutes=12), now + timedelta(minutes=16)]), - epoch_delta=np.array([120_000_000_000, 120_000_000_000, 120_000_000_000, 120_000_000_000, 120_000_000_000]), + [ + now, + now + timedelta(minutes=4), + now + timedelta(minutes=8), + now + timedelta(minutes=12), + now + timedelta(minutes=16), + ] + ), + epoch_delta=np.array( + [ + 120_000_000_000, + 120_000_000_000, + 120_000_000_000, + 120_000_000_000, + 120_000_000_000, + ] + ), fe_hiq_partial_density=np.arange(5), fe_loq_partial_density=np.arange(5), oplus5_partial_density=np.array([0, 1, 2, 2, 2]), oplus6_partial_density=np.array([5, 5, 5, 6, 6]), oplus7_partial_density=np.array([8, 8, 8, 9, 9]), - oplus8_partial_density=np.array([11, 11, 11, 12, 12, ]), + oplus8_partial_density=np.array( + [ + 11, + 11, + 11, + 12, + 12, + ] + ), mg_partial_density=np.arange(5), cplus4_partial_density=np.array([16, 17, 18, 18, 18]), cplus5_partial_density=np.array([20, 20, 20, 21, 21]), @@ -233,38 +387,59 @@ def test_process_abundances_calculates_abundance_ratios(self): dependency = CodiceLoL3aRatiosDependencies(data) input_metadata = Mock() - processor = CodiceLoProcessor(dependencies=Mock(), input_metadata=input_metadata) + processor = CodiceLoProcessor( + dependencies=Mock(), input_metadata=input_metadata + ) - abundances_data_product = processor.process_l3a_charge_state_distributions(dependency) - self.assertIsInstance(abundances_data_product, CodiceLoL3ChargeStateDistributionsDataProduct) + abundances_data_product = processor.process_l3a_charge_state_distributions( + dependency + ) + self.assertIsInstance( + abundances_data_product, CodiceLoL3ChargeStateDistributionsDataProduct + ) self.assertEqual(input_metadata, abundances_data_product.input_metadata) - np.testing.assert_array_equal(abundances_data_product.epoch, - [now + timedelta(minutes=4), now + timedelta(minutes=14)]) - np.testing.assert_array_equal(abundances_data_product.epoch_delta, [360_000_000_000, 240_000_000_000]) - - np.testing.assert_array_almost_equal(abundances_data_product.oxygen_charge_state_distribution, - np.array( - [ - [1 / (1 + 5 + 8 + 11), 5 / (1 + 5 + 8 + 11), 8 / (1 + 5 + 8 + 11), - 11 / (1 + 5 + 8 + 11), ], - [2 / (2 + 6 + 9 + 12), 6 / (2 + 6 + 9 + 12), 9 / (2 + 6 + 9 + 12), - 12 / (2 + 6 + 9 + 12), ], - ]) - ) - np.testing.assert_array_almost_equal(abundances_data_product.carbon_charge_state_distribution, - np.array( - [ - [17 / (17 + 20 + 23), 20 / (17 + 20 + 23), 23 / (17 + 20 + 23)], - [18 / (18 + 21 + 24), 21 / (18 + 21 + 24), 24 / (18 + 21 + 24)], - ] - )) + np.testing.assert_array_equal( + abundances_data_product.epoch, + [now + timedelta(minutes=4), now + timedelta(minutes=14)], + ) + np.testing.assert_array_equal( + abundances_data_product.epoch_delta, [360_000_000_000, 240_000_000_000] + ) + + np.testing.assert_array_almost_equal( + abundances_data_product.oxygen_charge_state_distribution, + np.array( + [ + [ + 1 / (1 + 5 + 8 + 11), + 5 / (1 + 5 + 8 + 11), + 8 / (1 + 5 + 8 + 11), + 11 / (1 + 5 + 8 + 11), + ], + [ + 2 / (2 + 6 + 9 + 12), + 6 / (2 + 6 + 9 + 12), + 9 / (2 + 6 + 9 + 12), + 12 / (2 + 6 + 9 + 12), + ], + ] + ), + ) + np.testing.assert_array_almost_equal( + abundances_data_product.carbon_charge_state_distribution, + np.array( + [ + [17 / (17 + 20 + 23), 20 / (17 + 20 + 23), 23 / (17 + 20 + 23)], + [18 / (18 + 21 + 24), 21 / (18 + 21 + 24), 24 / (18 + 21 + 24)], + ] + ), + ) def test_process_abundances_uses_safe_divide(self): now = datetime.now() data = CodiceLoPartialDensityData( - epoch=np.array( - [now]), + epoch=np.array([now]), epoch_delta=np.array([120_000_000_000]), fe_hiq_partial_density=np.array([1]), fe_loq_partial_density=np.array([1]), @@ -283,81 +458,114 @@ def test_process_abundances_uses_safe_divide(self): ) dependency = CodiceLoL3aRatiosDependencies(data) input_metadata = Mock() - processor = CodiceLoProcessor(dependencies=Mock(), input_metadata=input_metadata) + processor = CodiceLoProcessor( + dependencies=Mock(), input_metadata=input_metadata + ) with warnings.catch_warnings(record=True) as w: - abundances_data_product = processor.process_l3a_charge_state_distributions(dependency) + abundances_data_product = processor.process_l3a_charge_state_distributions( + dependency + ) self.assertEqual(0, len(w)) - self.assertIsInstance(abundances_data_product, CodiceLoL3ChargeStateDistributionsDataProduct) + self.assertIsInstance( + abundances_data_product, CodiceLoL3ChargeStateDistributionsDataProduct + ) self.assertEqual(input_metadata, abundances_data_product.input_metadata) - np.testing.assert_array_equal(abundances_data_product.epoch, - [now]) - np.testing.assert_array_equal(abundances_data_product.epoch_delta, - [120_000_000_000]) - - np.testing.assert_array_almost_equal(abundances_data_product.oxygen_charge_state_distribution, - np.array( - [ - [np.nan, np.nan, np.nan, np.nan] - ]) - ) - np.testing.assert_array_almost_equal(abundances_data_product.carbon_charge_state_distribution, - np.array( - [ - [np.nan, np.nan, np.nan] - ] - )) - - @patch(f'{MODULE}.CodiceLoL3aDirectEventsDependencies.fetch_dependencies') - @patch(f'{MODULE}.CodiceLoProcessor.process_l3a_direct_event_data_product') - @patch(f'{MODULE}.save_data') - @patch('imap_l3_processing.processor.spiceypy') - def test_process_direct_events(self, mock_spiceypy, mock_save_data, mock_process_direct_event, - mock_fetch_dependencies): + np.testing.assert_array_equal(abundances_data_product.epoch, [now]) + np.testing.assert_array_equal( + abundances_data_product.epoch_delta, [120_000_000_000] + ) + + np.testing.assert_array_almost_equal( + abundances_data_product.oxygen_charge_state_distribution, + np.array([[np.nan, np.nan, np.nan, np.nan]]), + ) + np.testing.assert_array_almost_equal( + abundances_data_product.carbon_charge_state_distribution, + np.array([[np.nan, np.nan, np.nan]]), + ) + + @patch(f"{MODULE}.CodiceLoL3aDirectEventsDependencies.fetch_dependencies") + @patch(f"{MODULE}.CodiceLoProcessor.process_l3a_direct_event_data_product") + @patch(f"{MODULE}.save_data") + @patch("imap_l3_processing.processor.spiceypy") + def test_process_direct_events( + self, + mock_spiceypy, + mock_save_data, + mock_process_direct_event, + mock_fetch_dependencies, + ): input_collection = MagicMock() - input_collection.get_file_paths.return_value = [Path('path/to/parent_file_1'), Path('path/to/parent_file_2')] - input_metadata = InputMetadata(instrument='codice', - data_level="l3a", - start_date=Mock(spec=datetime), - end_date=Mock(spec=datetime), - version='v02', - descriptor='lo-direct-events') + input_collection.get_file_paths.return_value = [ + Path("path/to/parent_file_1"), + Path("path/to/parent_file_2"), + ] + input_metadata = InputMetadata( + instrument="codice", + data_level="l3a", + start_date=Mock(spec=datetime), + end_date=Mock(spec=datetime), + version="v02", + descriptor="lo-direct-events", + ) mock_spiceypy.ktotal.return_value = 0 - processor = CodiceLoProcessor(dependencies=input_collection, input_metadata=input_metadata) + processor = CodiceLoProcessor( + dependencies=input_collection, input_metadata=input_metadata + ) product = processor.process() mock_fetch_dependencies.assert_called_once_with(processor.dependencies) - mock_process_direct_event.assert_called_once_with(mock_fetch_dependencies.return_value) + mock_process_direct_event.assert_called_once_with( + mock_fetch_dependencies.return_value + ) mock_save_data.assert_called_once_with(mock_process_direct_event.return_value) self.assertEqual([mock_save_data.return_value], product) def test_raises_exception_on_non_l3_input_metadata(self): - input_metadata = InputMetadata('codice', "L2a", Mock(), Mock(), 'v02', "bad-descriptor") + input_metadata = InputMetadata( + "codice", "L2a", Mock(), Mock(), "v02", "bad-descriptor" + ) processor = CodiceLoProcessor(Mock(), input_metadata) with self.assertRaises(NotImplementedError) as context: processor.process() - self.assertEqual("Unknown data level and descriptor for CoDICE: L2a, bad-descriptor", str(context.exception)) + self.assertEqual( + "Unknown data level and descriptor for CoDICE: L2a, bad-descriptor", + str(context.exception), + ) - @patch(f'{MODULE}.calculate_partial_densities') + @patch(f"{MODULE}.calculate_partial_densities") def test_process_l3a_partial_densities(self, mock_calculate_partial_densities): input_collection = ProcessingInputCollection() - input_metadata = InputMetadata('codice', "l3a", Mock(spec=datetime), Mock(spec=datetime), 'v02', - descriptor='lo-partial-densities') - processor = CodiceLoProcessor(dependencies=input_collection, input_metadata=input_metadata) + input_metadata = InputMetadata( + "codice", + "l3a", + Mock(spec=datetime), + Mock(spec=datetime), + "v02", + descriptor="lo-partial-densities", + ) + processor = CodiceLoProcessor( + dependencies=input_collection, input_metadata=input_metadata + ) - epochs = np.array([datetime(2025, 1, 1), datetime(2025, 1, 2), datetime(2025, 1, 3)]) + epochs = np.array( + [datetime(2025, 1, 1), datetime(2025, 1, 2), datetime(2025, 1, 3)] + ) num_species = 14 codice_lo_l2_data = CodiceLoL2SWSpeciesData( *[Mock() for _ in range(len(fields(CodiceLoL2SWSpeciesData)))], ) - mass_per_charge_lookup = MassPerChargeLookup(*[i for i in range(len(fields(MassPerChargeLookup)))]) + mass_per_charge_lookup = MassPerChargeLookup( + *[i for i in range(len(fields(MassPerChargeLookup)))] + ) codice_lo_l2_data.epoch = epochs @@ -479,31 +687,46 @@ def test_process_l3a_partial_densities(self, mock_calculate_partial_densities): self.assertIsInstance(result_data, CodiceLoPartialDensityData) np.testing.assert_array_equal(result_data.epoch, codice_lo_l2_data.epoch) - np.testing.assert_array_equal(result_data.epoch_delta, codice_lo_l2_data.epoch_delta_plus) - self.assertEqual(sentinel.hplus_partial_density, result_data.hplus_partial_density), - self.assertEqual(sentinel.heplusplus_partial_density, result_data.heplusplus_partial_density), - self.assertEqual(cplus4_partial_density, result_data.cplus4_partial_density), - self.assertEqual(cplus5_partial_density, result_data.cplus5_partial_density), - self.assertEqual(cplus6_partial_density, result_data.cplus6_partial_density), - self.assertEqual(oplus5_partial_density, result_data.oplus5_partial_density), - self.assertEqual(oplus6_partial_density, result_data.oplus6_partial_density), - self.assertEqual(oplus7_partial_density, result_data.oplus7_partial_density), - self.assertEqual(oplus8_partial_density, result_data.oplus8_partial_density), - self.assertEqual(sentinel.ne_partial_density, result_data.ne_partial_density), - self.assertEqual(mg_partial_density, result_data.mg_partial_density), - self.assertEqual(sentinel.si_partial_density, result_data.si_partial_density), - self.assertEqual(fe_loq_partial_density, result_data.fe_loq_partial_density), - self.assertEqual(fe_hiq_partial_density, result_data.fe_hiq_partial_density), - - @patch(f'{MODULE}.SpinAngleLookup') - @patch(f'{MODULE}.lookup_normalization_per_event', autospec=True) - @patch(f'{MODULE}.calculate_normalization_factor', autospec=True) - @patch(f'{MODULE}.calculate_mass_per_charge', autospec=True) - @patch(f'{MODULE}.calculate_mass', autospec=True) - def test_process_l3a_direct_events(self, mock_calculate_mass, mock_calculate_mass_per_charge, - mock_calculate_normalization_factor, - mock_lookup_normalization_per_event, - mock_spin_angle_lookup_class): + np.testing.assert_array_equal( + result_data.epoch_delta, codice_lo_l2_data.epoch_delta_plus + ) + ( + self.assertEqual( + sentinel.hplus_partial_density, result_data.hplus_partial_density + ), + ) + ( + self.assertEqual( + sentinel.heplusplus_partial_density, + result_data.heplusplus_partial_density, + ), + ) + (self.assertEqual(cplus4_partial_density, result_data.cplus4_partial_density),) + (self.assertEqual(cplus5_partial_density, result_data.cplus5_partial_density),) + (self.assertEqual(cplus6_partial_density, result_data.cplus6_partial_density),) + (self.assertEqual(oplus5_partial_density, result_data.oplus5_partial_density),) + (self.assertEqual(oplus6_partial_density, result_data.oplus6_partial_density),) + (self.assertEqual(oplus7_partial_density, result_data.oplus7_partial_density),) + (self.assertEqual(oplus8_partial_density, result_data.oplus8_partial_density),) + (self.assertEqual(sentinel.ne_partial_density, result_data.ne_partial_density),) + (self.assertEqual(mg_partial_density, result_data.mg_partial_density),) + (self.assertEqual(sentinel.si_partial_density, result_data.si_partial_density),) + (self.assertEqual(fe_loq_partial_density, result_data.fe_loq_partial_density),) + (self.assertEqual(fe_hiq_partial_density, result_data.fe_hiq_partial_density),) + + @patch(f"{MODULE}.SpinAngleLookup") + @patch(f"{MODULE}.lookup_normalization_per_event", autospec=True) + @patch(f"{MODULE}.calculate_normalization_factor", autospec=True) + @patch(f"{MODULE}.calculate_mass_per_charge", autospec=True) + @patch(f"{MODULE}.calculate_mass", autospec=True) + def test_process_l3a_direct_events( + self, + mock_calculate_mass, + mock_calculate_mass_per_charge, + mock_calculate_normalization_factor, + mock_lookup_normalization_per_event, + mock_spin_angle_lookup_class, + ): rng = np.random.default_rng() num_spin_angle_bins = 24 @@ -518,7 +741,11 @@ def test_process_l3a_direct_events(self, mock_calculate_mass, mock_calculate_mas epochs = np.array([datetime.now(), datetime.now() + timedelta(hours=1)]) num_epochs = len(epochs) - priority_counts_variable_shape = (num_epochs, num_energy_bins, num_spin_angle_bins // 2) + priority_counts_variable_shape = ( + num_epochs, + num_energy_bins, + num_spin_angle_bins // 2, + ) sw_priority_rates = create_dataclass_mock(CodiceLoL1aSWPriorityRates) sw_priority_rates.epoch = epochs sw_priority_rates.p0_tcrs = rng.random(priority_counts_variable_shape) @@ -526,7 +753,9 @@ def test_process_l3a_direct_events(self, mock_calculate_mass, mock_calculate_mas sw_priority_rates.p2_heplusplus = rng.random(priority_counts_variable_shape) sw_priority_rates.p3_heavies = rng.random(priority_counts_variable_shape) sw_priority_rates.p4_dcrs = rng.random(priority_counts_variable_shape) - sw_priority_rates.half_spin_per_esa_step = rng.random((num_epochs, num_energy_bins)) + sw_priority_rates.half_spin_per_esa_step = rng.random( + (num_epochs, num_energy_bins) + ) sw_priority_rates.rgfo_spin_sector = rng.random(num_epochs) sw_priority_rates.rgfo_esa_step = rng.random(num_epochs) sw_priority_rates.nso_spin_sector = rng.random(num_epochs) @@ -534,20 +763,30 @@ def test_process_l3a_direct_events(self, mock_calculate_mass, mock_calculate_mas sw_priority_rates.esa_step = np.arange(128) sw_priority_rates.rgfo_half_spin = rng.random(num_epochs) sw_priority_rates.nso_half_spin = rng.random(num_epochs) - sw_priority_rates.acquisition_time_per_esa_step = rng.random((num_epochs, num_energy_bins)) + sw_priority_rates.acquisition_time_per_esa_step = rng.random( + (num_epochs, num_energy_bins) + ) nsw_priority_rates = create_dataclass_mock(CodiceLoL1aNSWPriorityRates) nsw_priority_rates.epoch = epochs nsw_priority_rates.p5_heavies = rng.random(priority_counts_variable_shape) - nsw_priority_rates.p6_hplus_heplusplus = rng.random(priority_counts_variable_shape) + nsw_priority_rates.p6_hplus_heplusplus = rng.random( + priority_counts_variable_shape + ) - mass_per_charge = rng.random((num_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size)) + mass_per_charge = rng.random( + (num_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size) + ) mock_calculate_mass_per_charge.return_value = mass_per_charge mass = rng.random((num_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size)) mock_calculate_mass.return_value = mass - codice_l2_variables = {f.name: rng.random((num_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size)) for f - in fields(CodiceLoL2DirectEventData)} + codice_l2_variables = { + f.name: rng.random( + (num_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size) + ) + for f in fields(CodiceLoL2DirectEventData) + } codice_l2_variables["epoch"] = epochs codice_l2_variables["spin_angle"] *= 360 @@ -579,109 +818,232 @@ def test_process_l3a_direct_events(self, mock_calculate_mass, mock_calculate_mas mock_energy_lookup.bin_centers = np.geomspace(14100, 88.082825, 128) mock_energy_lookup.num_bins = num_energy_bins - expected_normalization = rng.random((num_epochs, num_priorities, num_energy_bins, num_spin_angle_bins)) + expected_normalization = rng.random( + (num_epochs, num_priorities, num_energy_bins, num_spin_angle_bins) + ) mock_calculate_normalization_factor.return_value = expected_normalization - dependencies = CodiceLoL3aDirectEventsDependencies(sw_priority_rates, nsw_priority_rates, direct_events, Mock(), - mock_energy_lookup) + dependencies = CodiceLoL3aDirectEventsDependencies( + sw_priority_rates, + nsw_priority_rates, + direct_events, + Mock(), + mock_energy_lookup, + ) input_collection = ProcessingInputCollection() - input_metadata = InputMetadata('codice', "l3a", Mock(spec=datetime), Mock(spec=datetime), 'v02') - processor = CodiceLoProcessor(dependencies=input_collection, input_metadata=input_metadata) - l3a_direct_event_data_product = processor.process_l3a_direct_event_data_product(dependencies) + input_metadata = InputMetadata( + "codice", "l3a", Mock(spec=datetime), Mock(spec=datetime), "v02" + ) + processor = CodiceLoProcessor( + dependencies=input_collection, input_metadata=input_metadata + ) + l3a_direct_event_data_product = processor.process_l3a_direct_event_data_product( + dependencies + ) self.assertEqual(1, mock_calculate_mass.call_count) - np.testing.assert_equal(mock_calculate_mass.call_args.args[0], expected_apd_energy) + np.testing.assert_equal( + mock_calculate_mass.call_args.args[0], expected_apd_energy + ) np.testing.assert_equal(mock_calculate_mass.call_args.args[1], expected_tof) - np.testing.assert_equal(mock_calculate_mass.call_args.args[2], dependencies.mass_coefficient_lookup) + np.testing.assert_equal( + mock_calculate_mass.call_args.args[2], dependencies.mass_coefficient_lookup + ) self.assertEqual(1, mock_calculate_mass_per_charge.call_count) - np.testing.assert_equal(mock_calculate_mass_per_charge.call_args.args[0], expected_energy_per_charge) - np.testing.assert_equal(mock_calculate_mass_per_charge.call_args.args[1], expected_tof) + np.testing.assert_equal( + mock_calculate_mass_per_charge.call_args.args[0], expected_energy_per_charge + ) + np.testing.assert_equal( + mock_calculate_mass_per_charge.call_args.args[1], expected_tof + ) self.assertEqual(1, mock_calculate_normalization_factor.call_count) - actual_stacked_priorities = mock_calculate_normalization_factor.call_args.args[0] + actual_stacked_priorities = mock_calculate_normalization_factor.call_args.args[ + 0 + ] - np.testing.assert_equal(actual_stacked_priorities[:, 0], sw_priority_rates.p0_tcrs) - np.testing.assert_equal(actual_stacked_priorities[:, 1], sw_priority_rates.p1_hplus) - np.testing.assert_equal(actual_stacked_priorities[:, 2], sw_priority_rates.p2_heplusplus) - np.testing.assert_equal(actual_stacked_priorities[:, 3], sw_priority_rates.p3_heavies) - np.testing.assert_equal(actual_stacked_priorities[:, 4], sw_priority_rates.p4_dcrs) - np.testing.assert_equal(actual_stacked_priorities[:, 5], nsw_priority_rates.p5_heavies) - np.testing.assert_equal(actual_stacked_priorities[:, 6], nsw_priority_rates.p6_hplus_heplusplus) + np.testing.assert_equal( + actual_stacked_priorities[:, 0], sw_priority_rates.p0_tcrs + ) + np.testing.assert_equal( + actual_stacked_priorities[:, 1], sw_priority_rates.p1_hplus + ) + np.testing.assert_equal( + actual_stacked_priorities[:, 2], sw_priority_rates.p2_heplusplus + ) + np.testing.assert_equal( + actual_stacked_priorities[:, 3], sw_priority_rates.p3_heavies + ) + np.testing.assert_equal( + actual_stacked_priorities[:, 4], sw_priority_rates.p4_dcrs + ) + np.testing.assert_equal( + actual_stacked_priorities[:, 5], nsw_priority_rates.p5_heavies + ) + np.testing.assert_equal( + actual_stacked_priorities[:, 6], nsw_priority_rates.p6_hplus_heplusplus + ) - np.testing.assert_equal(mock_calculate_normalization_factor.call_args.args[1], expected_num_events) - np.testing.assert_equal(mock_calculate_normalization_factor.call_args.args[2], expected_energy_step) - np.testing.assert_equal(mock_calculate_normalization_factor.call_args.args[3], expected_spin_sector) + np.testing.assert_equal( + mock_calculate_normalization_factor.call_args.args[1], expected_num_events + ) + np.testing.assert_equal( + mock_calculate_normalization_factor.call_args.args[2], expected_energy_step + ) + np.testing.assert_equal( + mock_calculate_normalization_factor.call_args.args[3], expected_spin_sector + ) self.assertEqual(1, mock_lookup_normalization_per_event.call_count) - np.testing.assert_equal(mock_lookup_normalization_per_event.call_args.args[0], expected_normalization) - np.testing.assert_equal(mock_lookup_normalization_per_event.call_args.args[1], expected_num_events) - np.testing.assert_equal(mock_lookup_normalization_per_event.call_args.args[2], expected_energy_step) - np.testing.assert_equal(mock_lookup_normalization_per_event.call_args.args[3], expected_spin_sector) + np.testing.assert_equal( + mock_lookup_normalization_per_event.call_args.args[0], + expected_normalization, + ) + np.testing.assert_equal( + mock_lookup_normalization_per_event.call_args.args[1], expected_num_events + ) + np.testing.assert_equal( + mock_lookup_normalization_per_event.call_args.args[2], expected_energy_step + ) + np.testing.assert_equal( + mock_lookup_normalization_per_event.call_args.args[3], expected_spin_sector + ) - self.assertIsInstance(l3a_direct_event_data_product, CodiceLoL3aDirectEventDataProduct) + self.assertIsInstance( + l3a_direct_event_data_product, CodiceLoL3aDirectEventDataProduct + ) self.assertEqual(input_metadata, l3a_direct_event_data_product.input_metadata) np.testing.assert_array_equal(l3a_direct_event_data_product.epoch, epochs) - np.testing.assert_array_equal(l3a_direct_event_data_product.epoch_delta, direct_events.epoch_delta_plus) + np.testing.assert_array_equal( + l3a_direct_event_data_product.epoch_delta, direct_events.epoch_delta_plus + ) np.testing.assert_array_equal( - np.arange(CODICE_LO_L2_NUM_PRIORITIES), l3a_direct_event_data_product.priority_index + np.arange(CODICE_LO_L2_NUM_PRIORITIES), + l3a_direct_event_data_product.priority_index, + ) + np.testing.assert_array_equal( + expected_mass_per_charge, l3a_direct_event_data_product.mass_per_charge ) - np.testing.assert_array_equal(expected_mass_per_charge, l3a_direct_event_data_product.mass_per_charge) np.testing.assert_array_equal(expected_mass, l3a_direct_event_data_product.mass) np.testing.assert_array_equal( - l3a_direct_event_data_product.normalization, np.flip(expected_normalization, axis=2) - ) - self.assertEqual(l3a_direct_event_data_product.normalization_per_event, - mock_lookup_normalization_per_event.return_value) - - np.testing.assert_array_equal(expected_apd_energy, l3a_direct_event_data_product.apd_energy) - np.testing.assert_array_equal(expected_apd_gain, l3a_direct_event_data_product.gain) - np.testing.assert_array_equal(expected_apd_id, l3a_direct_event_data_product.apd_id) - np.testing.assert_array_equal(expected_data_quality, l3a_direct_event_data_product.data_quality) - np.testing.assert_array_equal(expected_elevation, l3a_direct_event_data_product.elevation) - np.testing.assert_array_equal(expected_energy_per_charge, l3a_direct_event_data_product.energy_per_charge) - np.testing.assert_array_equal(expected_energy_step, l3a_direct_event_data_product.energy_step) - np.testing.assert_array_equal(expected_multi_flag, l3a_direct_event_data_product.multi_flag) - np.testing.assert_array_equal(expected_num_events, l3a_direct_event_data_product.num_events) - np.testing.assert_array_equal(expected_position, l3a_direct_event_data_product.position) - np.testing.assert_array_equal(expected_spin_angle, l3a_direct_event_data_product.spin_angle) - np.testing.assert_array_equal(expected_spin_sector, l3a_direct_event_data_product.spin_sector) + l3a_direct_event_data_product.normalization, + np.flip(expected_normalization, axis=2), + ) + self.assertEqual( + l3a_direct_event_data_product.normalization_per_event, + mock_lookup_normalization_per_event.return_value, + ) + + np.testing.assert_array_equal( + expected_apd_energy, l3a_direct_event_data_product.apd_energy + ) + np.testing.assert_array_equal( + expected_apd_gain, l3a_direct_event_data_product.gain + ) + np.testing.assert_array_equal( + expected_apd_id, l3a_direct_event_data_product.apd_id + ) + np.testing.assert_array_equal( + expected_data_quality, l3a_direct_event_data_product.data_quality + ) + np.testing.assert_array_equal( + expected_elevation, l3a_direct_event_data_product.elevation + ) + np.testing.assert_array_equal( + expected_energy_per_charge, l3a_direct_event_data_product.energy_per_charge + ) + np.testing.assert_array_equal( + expected_energy_step, l3a_direct_event_data_product.energy_step + ) + np.testing.assert_array_equal( + expected_multi_flag, l3a_direct_event_data_product.multi_flag + ) + np.testing.assert_array_equal( + expected_num_events, l3a_direct_event_data_product.num_events + ) + np.testing.assert_array_equal( + expected_position, l3a_direct_event_data_product.position + ) + np.testing.assert_array_equal( + expected_spin_angle, l3a_direct_event_data_product.spin_angle + ) + np.testing.assert_array_equal( + expected_spin_sector, l3a_direct_event_data_product.spin_sector + ) np.testing.assert_array_equal(expected_tof, l3a_direct_event_data_product.tof) np.testing.assert_array_equal(expected_type, l3a_direct_event_data_product.type) - np.testing.assert_array_equal(np.flip(mock_energy_lookup.bin_centers), l3a_direct_event_data_product.energy_bin) - np.testing.assert_array_equal(np.flip(mock_energy_lookup.delta_plus), - l3a_direct_event_data_product.energy_bin_delta_plus) - np.testing.assert_array_equal(np.flip(mock_energy_lookup.delta_minus), - l3a_direct_event_data_product.energy_bin_delta_minus) - self.assertEqual(mock_spin_angle_lookup.bin_centers, l3a_direct_event_data_product.spin_angle_bin) - self.assertEqual(mock_spin_angle_lookup.bin_deltas, l3a_direct_event_data_product.spin_angle_bin_delta) - np.testing.assert_array_equal(sw_priority_rates.half_spin_per_esa_step, - l3a_direct_event_data_product.half_spin_per_esa_step) - np.testing.assert_array_equal(sw_priority_rates.rgfo_spin_sector, - l3a_direct_event_data_product.rgfo_spin_sector) - np.testing.assert_array_equal(sw_priority_rates.rgfo_esa_step, l3a_direct_event_data_product.rgfo_esa_step) - np.testing.assert_array_equal(sw_priority_rates.nso_spin_sector, l3a_direct_event_data_product.nso_spin_sector) - np.testing.assert_array_equal(sw_priority_rates.nso_esa_step, l3a_direct_event_data_product.nso_esa_step) - np.testing.assert_array_equal(sw_priority_rates.esa_step, l3a_direct_event_data_product.esa_step) - np.testing.assert_array_equal(sw_priority_rates.rgfo_half_spin, l3a_direct_event_data_product.rgfo_half_spin) - np.testing.assert_array_equal(sw_priority_rates.nso_half_spin, l3a_direct_event_data_product.nso_half_spin) - np.testing.assert_array_equal(sw_priority_rates.acquisition_time_per_esa_step, - l3a_direct_event_data_product.acquisition_time_per_esa_step) - - @patch(f'{MODULE}.SpinAngleLookup') - @patch(f'{MODULE}.lookup_normalization_per_event', autospec=True) - @patch(f'{MODULE}.calculate_normalization_factor', autospec=True) - @patch(f'{MODULE}.calculate_mass_per_charge', autospec=True) - @patch(f'{MODULE}.calculate_mass', autospec=True) + np.testing.assert_array_equal( + np.flip(mock_energy_lookup.bin_centers), + l3a_direct_event_data_product.energy_bin, + ) + np.testing.assert_array_equal( + np.flip(mock_energy_lookup.delta_plus), + l3a_direct_event_data_product.energy_bin_delta_plus, + ) + np.testing.assert_array_equal( + np.flip(mock_energy_lookup.delta_minus), + l3a_direct_event_data_product.energy_bin_delta_minus, + ) + self.assertEqual( + mock_spin_angle_lookup.bin_centers, + l3a_direct_event_data_product.spin_angle_bin, + ) + self.assertEqual( + mock_spin_angle_lookup.bin_deltas, + l3a_direct_event_data_product.spin_angle_bin_delta, + ) + np.testing.assert_array_equal( + sw_priority_rates.half_spin_per_esa_step, + l3a_direct_event_data_product.half_spin_per_esa_step, + ) + np.testing.assert_array_equal( + sw_priority_rates.rgfo_spin_sector, + l3a_direct_event_data_product.rgfo_spin_sector, + ) + np.testing.assert_array_equal( + sw_priority_rates.rgfo_esa_step, l3a_direct_event_data_product.rgfo_esa_step + ) + np.testing.assert_array_equal( + sw_priority_rates.nso_spin_sector, + l3a_direct_event_data_product.nso_spin_sector, + ) + np.testing.assert_array_equal( + sw_priority_rates.nso_esa_step, l3a_direct_event_data_product.nso_esa_step + ) + np.testing.assert_array_equal( + sw_priority_rates.esa_step, l3a_direct_event_data_product.esa_step + ) + np.testing.assert_array_equal( + sw_priority_rates.rgfo_half_spin, + l3a_direct_event_data_product.rgfo_half_spin, + ) + np.testing.assert_array_equal( + sw_priority_rates.nso_half_spin, l3a_direct_event_data_product.nso_half_spin + ) + np.testing.assert_array_equal( + sw_priority_rates.acquisition_time_per_esa_step, + l3a_direct_event_data_product.acquisition_time_per_esa_step, + ) + + @patch(f"{MODULE}.SpinAngleLookup") + @patch(f"{MODULE}.lookup_normalization_per_event", autospec=True) + @patch(f"{MODULE}.calculate_normalization_factor", autospec=True) + @patch(f"{MODULE}.calculate_mass_per_charge", autospec=True) + @patch(f"{MODULE}.calculate_mass", autospec=True) def test_process_l3a_direct_events_aligns_priority_to_direct_event_epochs_when_missing( - self, mock_calculate_mass, mock_calculate_mass_per_charge, - mock_calculate_normalization_factor, mock_lookup_normalization_per_event, - mock_spin_angle_lookup_class): + self, + mock_calculate_mass, + mock_calculate_mass_per_charge, + mock_calculate_normalization_factor, + mock_lookup_normalization_per_event, + mock_spin_angle_lookup_class, + ): rng = np.random.default_rng(seed=42) num_spin_angle_bins = 4 @@ -707,8 +1069,16 @@ def test_process_l3a_direct_events_aligns_priority_to_direct_event_epochs_when_m num_sw_epochs = len(sw_priority_epochs) num_nsw_epochs = len(nsw_priority_epochs) - sw_priority_counts_shape = (num_sw_epochs, num_energy_bins, priority_l1_spin_sectors) - nsw_priority_counts_shape = (num_nsw_epochs, num_energy_bins, priority_l1_spin_sectors) + sw_priority_counts_shape = ( + num_sw_epochs, + num_energy_bins, + priority_l1_spin_sectors, + ) + nsw_priority_counts_shape = ( + num_nsw_epochs, + num_energy_bins, + priority_l1_spin_sectors, + ) sw_priority_rates = create_dataclass_mock(CodiceLoL1aSWPriorityRates) sw_priority_rates.epoch = sw_priority_epochs @@ -717,7 +1087,9 @@ def test_process_l3a_direct_events_aligns_priority_to_direct_event_epochs_when_m sw_priority_rates.p2_heplusplus = rng.random(sw_priority_counts_shape) sw_priority_rates.p3_heavies = rng.random(sw_priority_counts_shape) sw_priority_rates.p4_dcrs = rng.random(sw_priority_counts_shape) - sw_priority_rates.half_spin_per_esa_step = rng.random((num_sw_epochs, num_energy_bins)) + sw_priority_rates.half_spin_per_esa_step = rng.random( + (num_sw_epochs, num_energy_bins) + ) sw_priority_rates.rgfo_spin_sector = rng.random(num_sw_epochs) sw_priority_rates.rgfo_esa_step = rng.random(num_sw_epochs) sw_priority_rates.nso_spin_sector = rng.random(num_sw_epochs) @@ -725,20 +1097,32 @@ def test_process_l3a_direct_events_aligns_priority_to_direct_event_epochs_when_m sw_priority_rates.esa_step = np.arange(num_energy_bins) sw_priority_rates.rgfo_half_spin = rng.random(num_sw_epochs) sw_priority_rates.nso_half_spin = rng.random(num_sw_epochs) - sw_priority_rates.acquisition_time_per_esa_step = rng.random((num_sw_epochs, num_energy_bins)) + sw_priority_rates.acquisition_time_per_esa_step = rng.random( + (num_sw_epochs, num_energy_bins) + ) nsw_priority_rates = create_dataclass_mock(CodiceLoL1aNSWPriorityRates) nsw_priority_rates.epoch = nsw_priority_epochs nsw_priority_rates.p5_heavies = rng.random(nsw_priority_counts_shape) nsw_priority_rates.p6_hplus_heplusplus = rng.random(nsw_priority_counts_shape) - mass_per_charge = rng.random((num_direct_event_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size)) + mass_per_charge = rng.random( + (num_direct_event_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size) + ) mock_calculate_mass_per_charge.return_value = mass_per_charge - mass = rng.random((num_direct_event_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size)) + mass = rng.random( + (num_direct_event_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size) + ) mock_calculate_mass.return_value = mass codice_l2_variables = { - f.name: rng.random((num_direct_event_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size)) + f.name: rng.random( + ( + num_direct_event_epochs, + CODICE_LO_L2_NUM_PRIORITIES, + event_buffer_size, + ) + ) for f in fields(CodiceLoL2DirectEventData) } codice_l2_variables["epoch"] = direct_event_epochs @@ -755,7 +1139,12 @@ def test_process_l3a_direct_events_aligns_priority_to_direct_event_epochs_when_m mock_energy_lookup.num_bins = num_energy_bins expected_normalization = rng.random( - (num_direct_event_epochs, num_priorities, num_energy_bins, num_spin_angle_bins) + ( + num_direct_event_epochs, + num_priorities, + num_energy_bins, + num_spin_angle_bins, + ) ) mock_calculate_normalization_factor.return_value = expected_normalization @@ -766,52 +1155,120 @@ def test_process_l3a_direct_events_aligns_priority_to_direct_event_epochs_when_m ) dependencies = CodiceLoL3aDirectEventsDependencies( - sw_priority_rates, nsw_priority_rates, direct_events, Mock(), mock_energy_lookup + sw_priority_rates, + nsw_priority_rates, + direct_events, + Mock(), + mock_energy_lookup, ) input_collection = ProcessingInputCollection() - input_metadata = InputMetadata('codice', "l3a", Mock(spec=datetime), Mock(spec=datetime), 'v02') - processor = CodiceLoProcessor(dependencies=input_collection, input_metadata=input_metadata) + input_metadata = InputMetadata( + "codice", "l3a", Mock(spec=datetime), Mock(spec=datetime), "v02" + ) + processor = CodiceLoProcessor( + dependencies=input_collection, input_metadata=input_metadata + ) result = processor.process_l3a_direct_event_data_product(dependencies) self.assertEqual(1, mock_calculate_normalization_factor.call_count) - actual_stacked_priorities = mock_calculate_normalization_factor.call_args.args[0] + actual_stacked_priorities = mock_calculate_normalization_factor.call_args.args[ + 0 + ] self.assertEqual( - (num_direct_event_epochs, num_priorities, num_energy_bins, priority_l1_spin_sectors), + ( + num_direct_event_epochs, + num_priorities, + num_energy_bins, + priority_l1_spin_sectors, + ), actual_stacked_priorities.shape, ) - np.testing.assert_equal(actual_stacked_priorities[0, 0], sw_priority_rates.p0_tcrs[0]) - np.testing.assert_equal(actual_stacked_priorities[0, 1], sw_priority_rates.p1_hplus[0]) - np.testing.assert_equal(actual_stacked_priorities[0, 2], sw_priority_rates.p2_heplusplus[0]) - np.testing.assert_equal(actual_stacked_priorities[0, 3], sw_priority_rates.p3_heavies[0]) - np.testing.assert_equal(actual_stacked_priorities[0, 4], sw_priority_rates.p4_dcrs[0]) - np.testing.assert_equal(actual_stacked_priorities[0, 5], nsw_priority_rates.p5_heavies[0]) - np.testing.assert_equal(actual_stacked_priorities[0, 6], nsw_priority_rates.p6_hplus_heplusplus[0]) - - expected_sw_nan_block = np.full((5, num_energy_bins, priority_l1_spin_sectors), np.nan) - np.testing.assert_equal(actual_stacked_priorities[1, 0:5], expected_sw_nan_block) - np.testing.assert_equal(actual_stacked_priorities[1, 5], nsw_priority_rates.p5_heavies[1]) - np.testing.assert_equal(actual_stacked_priorities[1, 6], nsw_priority_rates.p6_hplus_heplusplus[1]) - - np.testing.assert_equal(actual_stacked_priorities[2, 0], sw_priority_rates.p0_tcrs[1]) - np.testing.assert_equal(actual_stacked_priorities[2, 1], sw_priority_rates.p1_hplus[1]) - np.testing.assert_equal(actual_stacked_priorities[2, 2], sw_priority_rates.p2_heplusplus[1]) - np.testing.assert_equal(actual_stacked_priorities[2, 3], sw_priority_rates.p3_heavies[1]) - np.testing.assert_equal(actual_stacked_priorities[2, 4], sw_priority_rates.p4_dcrs[1]) - expected_nsw_nan_block = np.full((2, num_energy_bins, priority_l1_spin_sectors), np.nan) - np.testing.assert_equal(actual_stacked_priorities[2, 5:7], expected_nsw_nan_block) - - np.testing.assert_equal(actual_stacked_priorities[3, 0], sw_priority_rates.p0_tcrs[2]) - np.testing.assert_equal(actual_stacked_priorities[3, 1], sw_priority_rates.p1_hplus[2]) - np.testing.assert_equal(actual_stacked_priorities[3, 2], sw_priority_rates.p2_heplusplus[2]) - np.testing.assert_equal(actual_stacked_priorities[3, 3], sw_priority_rates.p3_heavies[2]) - np.testing.assert_equal(actual_stacked_priorities[3, 4], sw_priority_rates.p4_dcrs[2]) - np.testing.assert_equal(actual_stacked_priorities[3, 5], nsw_priority_rates.p5_heavies[2]) - np.testing.assert_equal(actual_stacked_priorities[3, 6], nsw_priority_rates.p6_hplus_heplusplus[2]) + np.testing.assert_equal( + actual_stacked_priorities[0, 0], sw_priority_rates.p0_tcrs[0] + ) + np.testing.assert_equal( + actual_stacked_priorities[0, 1], sw_priority_rates.p1_hplus[0] + ) + np.testing.assert_equal( + actual_stacked_priorities[0, 2], sw_priority_rates.p2_heplusplus[0] + ) + np.testing.assert_equal( + actual_stacked_priorities[0, 3], sw_priority_rates.p3_heavies[0] + ) + np.testing.assert_equal( + actual_stacked_priorities[0, 4], sw_priority_rates.p4_dcrs[0] + ) + np.testing.assert_equal( + actual_stacked_priorities[0, 5], nsw_priority_rates.p5_heavies[0] + ) + np.testing.assert_equal( + actual_stacked_priorities[0, 6], nsw_priority_rates.p6_hplus_heplusplus[0] + ) + + expected_sw_nan_block = np.full( + (5, num_energy_bins, priority_l1_spin_sectors), np.nan + ) + np.testing.assert_equal( + actual_stacked_priorities[1, 0:5], expected_sw_nan_block + ) + np.testing.assert_equal( + actual_stacked_priorities[1, 5], nsw_priority_rates.p5_heavies[1] + ) + np.testing.assert_equal( + actual_stacked_priorities[1, 6], nsw_priority_rates.p6_hplus_heplusplus[1] + ) + + np.testing.assert_equal( + actual_stacked_priorities[2, 0], sw_priority_rates.p0_tcrs[1] + ) + np.testing.assert_equal( + actual_stacked_priorities[2, 1], sw_priority_rates.p1_hplus[1] + ) + np.testing.assert_equal( + actual_stacked_priorities[2, 2], sw_priority_rates.p2_heplusplus[1] + ) + np.testing.assert_equal( + actual_stacked_priorities[2, 3], sw_priority_rates.p3_heavies[1] + ) + np.testing.assert_equal( + actual_stacked_priorities[2, 4], sw_priority_rates.p4_dcrs[1] + ) + expected_nsw_nan_block = np.full( + (2, num_energy_bins, priority_l1_spin_sectors), np.nan + ) + np.testing.assert_equal( + actual_stacked_priorities[2, 5:7], expected_nsw_nan_block + ) + + np.testing.assert_equal( + actual_stacked_priorities[3, 0], sw_priority_rates.p0_tcrs[2] + ) + np.testing.assert_equal( + actual_stacked_priorities[3, 1], sw_priority_rates.p1_hplus[2] + ) + np.testing.assert_equal( + actual_stacked_priorities[3, 2], sw_priority_rates.p2_heplusplus[2] + ) + np.testing.assert_equal( + actual_stacked_priorities[3, 3], sw_priority_rates.p3_heavies[2] + ) + np.testing.assert_equal( + actual_stacked_priorities[3, 4], sw_priority_rates.p4_dcrs[2] + ) + np.testing.assert_equal( + actual_stacked_priorities[3, 5], nsw_priority_rates.p5_heavies[2] + ) + np.testing.assert_equal( + actual_stacked_priorities[3, 6], nsw_priority_rates.p6_hplus_heplusplus[2] + ) self.assertEqual(1, mock_lookup_normalization_per_event.call_count) - np.testing.assert_equal(mock_lookup_normalization_per_event.call_args.args[0], expected_normalization) + np.testing.assert_equal( + mock_lookup_normalization_per_event.call_args.args[0], + expected_normalization, + ) actual_normalization_per_event = result.normalization_per_event @@ -822,34 +1279,62 @@ def test_process_l3a_direct_events_aligns_priority_to_direct_event_epochs_when_m actual_normalization_per_event[2, 5:7, :], np.zeros((2, event_buffer_size)) ) - sentinel_row = np.full(event_buffer_size, sentinel_normalization_per_event_value) + sentinel_row = np.full( + event_buffer_size, sentinel_normalization_per_event_value + ) np.testing.assert_array_equal( actual_normalization_per_event[0], - np.full((num_priorities, event_buffer_size), sentinel_normalization_per_event_value), + np.full( + (num_priorities, event_buffer_size), + sentinel_normalization_per_event_value, + ), ) np.testing.assert_array_equal( actual_normalization_per_event[3], - np.full((num_priorities, event_buffer_size), sentinel_normalization_per_event_value), - ) - np.testing.assert_array_equal(actual_normalization_per_event[1, 5], sentinel_row) - np.testing.assert_array_equal(actual_normalization_per_event[1, 6], sentinel_row) - np.testing.assert_array_equal(actual_normalization_per_event[2, 0], sentinel_row) - np.testing.assert_array_equal(actual_normalization_per_event[2, 1], sentinel_row) - np.testing.assert_array_equal(actual_normalization_per_event[2, 2], sentinel_row) - np.testing.assert_array_equal(actual_normalization_per_event[2, 3], sentinel_row) - np.testing.assert_array_equal(actual_normalization_per_event[2, 4], sentinel_row) - - np.testing.assert_array_equal(result.normalization, np.flip(expected_normalization, axis=2)) - - @patch(f'{MODULE}.SpinAngleLookup') - @patch(f'{MODULE}.lookup_normalization_per_event', autospec=True) - @patch(f'{MODULE}.calculate_normalization_factor', autospec=True) - @patch(f'{MODULE}.calculate_mass_per_charge', autospec=True) - @patch(f'{MODULE}.calculate_mass', autospec=True) + np.full( + (num_priorities, event_buffer_size), + sentinel_normalization_per_event_value, + ), + ) + np.testing.assert_array_equal( + actual_normalization_per_event[1, 5], sentinel_row + ) + np.testing.assert_array_equal( + actual_normalization_per_event[1, 6], sentinel_row + ) + np.testing.assert_array_equal( + actual_normalization_per_event[2, 0], sentinel_row + ) + np.testing.assert_array_equal( + actual_normalization_per_event[2, 1], sentinel_row + ) + np.testing.assert_array_equal( + actual_normalization_per_event[2, 2], sentinel_row + ) + np.testing.assert_array_equal( + actual_normalization_per_event[2, 3], sentinel_row + ) + np.testing.assert_array_equal( + actual_normalization_per_event[2, 4], sentinel_row + ) + + np.testing.assert_array_equal( + result.normalization, np.flip(expected_normalization, axis=2) + ) + + @patch(f"{MODULE}.SpinAngleLookup") + @patch(f"{MODULE}.lookup_normalization_per_event", autospec=True) + @patch(f"{MODULE}.calculate_normalization_factor", autospec=True) + @patch(f"{MODULE}.calculate_mass_per_charge", autospec=True) + @patch(f"{MODULE}.calculate_mass", autospec=True) def test_process_l3a_direct_events_handles_masked_priority_counts( - self, mock_calculate_mass, mock_calculate_mass_per_charge, - mock_calculate_normalization_factor, mock_lookup_normalization_per_event, - mock_spin_angle_lookup_class): + self, + mock_calculate_mass, + mock_calculate_mass_per_charge, + mock_calculate_normalization_factor, + mock_lookup_normalization_per_event, + mock_spin_angle_lookup_class, + ): rng = np.random.default_rng(seed=42) num_spin_angle_bins = 4 @@ -875,18 +1360,38 @@ def test_process_l3a_direct_events_handles_masked_priority_counts( num_sw_epochs = len(sw_priority_epochs) num_nsw_epochs = len(nsw_priority_epochs) - sw_priority_counts_shape = (num_sw_epochs, num_energy_bins, priority_l1_spin_sectors) - nsw_priority_counts_shape = (num_nsw_epochs, num_energy_bins, priority_l1_spin_sectors) + sw_priority_counts_shape = ( + num_sw_epochs, + num_energy_bins, + priority_l1_spin_sectors, + ) + nsw_priority_counts_shape = ( + num_nsw_epochs, + num_energy_bins, + priority_l1_spin_sectors, + ) sw_mask = np.full(sw_priority_counts_shape, False) sw_mask[0, 0, 0] = True sw_priority_rates = create_dataclass_mock(CodiceLoL1aSWPriorityRates) sw_priority_rates.epoch = sw_priority_epochs - sw_priority_rates.p0_tcrs = np.ma.array(rng.random(sw_priority_counts_shape), mask=sw_mask) - sw_priority_rates.p1_hplus = np.ma.array(rng.random(sw_priority_counts_shape), mask=sw_mask) - sw_priority_rates.p2_heplusplus = np.ma.array(rng.random(sw_priority_counts_shape), mask=sw_mask) - sw_priority_rates.p3_heavies = np.ma.array(rng.random(sw_priority_counts_shape), mask=sw_mask) - sw_priority_rates.p4_dcrs = np.ma.array(rng.random(sw_priority_counts_shape), mask=sw_mask) - sw_priority_rates.half_spin_per_esa_step = rng.random((num_sw_epochs, num_energy_bins)) + sw_priority_rates.p0_tcrs = np.ma.array( + rng.random(sw_priority_counts_shape), mask=sw_mask + ) + sw_priority_rates.p1_hplus = np.ma.array( + rng.random(sw_priority_counts_shape), mask=sw_mask + ) + sw_priority_rates.p2_heplusplus = np.ma.array( + rng.random(sw_priority_counts_shape), mask=sw_mask + ) + sw_priority_rates.p3_heavies = np.ma.array( + rng.random(sw_priority_counts_shape), mask=sw_mask + ) + sw_priority_rates.p4_dcrs = np.ma.array( + rng.random(sw_priority_counts_shape), mask=sw_mask + ) + sw_priority_rates.half_spin_per_esa_step = rng.random( + (num_sw_epochs, num_energy_bins) + ) sw_priority_rates.rgfo_spin_sector = rng.random(num_sw_epochs) sw_priority_rates.rgfo_esa_step = rng.random(num_sw_epochs) sw_priority_rates.nso_spin_sector = rng.random(num_sw_epochs) @@ -894,22 +1399,38 @@ def test_process_l3a_direct_events_handles_masked_priority_counts( sw_priority_rates.esa_step = np.arange(num_energy_bins) sw_priority_rates.rgfo_half_spin = rng.integers(0, 12, num_sw_epochs, np.uint8) sw_priority_rates.nso_half_spin = rng.integers(0, 12, num_sw_epochs, np.uint8) - sw_priority_rates.acquisition_time_per_esa_step = rng.random((num_sw_epochs, num_energy_bins)) + sw_priority_rates.acquisition_time_per_esa_step = rng.random( + (num_sw_epochs, num_energy_bins) + ) nsw_mask = np.full(nsw_priority_counts_shape, False) nsw_mask[0, 0, 0] = True nsw_priority_rates = create_dataclass_mock(CodiceLoL1aNSWPriorityRates) nsw_priority_rates.epoch = nsw_priority_epochs - nsw_priority_rates.p5_heavies = np.ma.array(rng.random(nsw_priority_counts_shape), mask=nsw_mask) - nsw_priority_rates.p6_hplus_heplusplus = np.ma.array(rng.random(nsw_priority_counts_shape), mask=nsw_mask) + nsw_priority_rates.p5_heavies = np.ma.array( + rng.random(nsw_priority_counts_shape), mask=nsw_mask + ) + nsw_priority_rates.p6_hplus_heplusplus = np.ma.array( + rng.random(nsw_priority_counts_shape), mask=nsw_mask + ) - mass_per_charge = rng.random((num_direct_event_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size)) + mass_per_charge = rng.random( + (num_direct_event_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size) + ) mock_calculate_mass_per_charge.return_value = mass_per_charge - mass = rng.random((num_direct_event_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size)) + mass = rng.random( + (num_direct_event_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size) + ) mock_calculate_mass.return_value = mass codice_l2_variables = { - f.name: rng.random((num_direct_event_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size)) + f.name: rng.random( + ( + num_direct_event_epochs, + CODICE_LO_L2_NUM_PRIORITIES, + event_buffer_size, + ) + ) for f in fields(CodiceLoL2DirectEventData) } codice_l2_variables["epoch"] = direct_event_epochs @@ -926,7 +1447,12 @@ def test_process_l3a_direct_events_handles_masked_priority_counts( mock_energy_lookup.num_bins = num_energy_bins expected_normalization = rng.random( - (num_direct_event_epochs, num_priorities, num_energy_bins, num_spin_angle_bins) + ( + num_direct_event_epochs, + num_priorities, + num_energy_bins, + num_spin_angle_bins, + ) ) mock_calculate_normalization_factor.return_value = expected_normalization @@ -937,32 +1463,53 @@ def test_process_l3a_direct_events_handles_masked_priority_counts( ) dependencies = CodiceLoL3aDirectEventsDependencies( - sw_priority_rates, nsw_priority_rates, direct_events, Mock(), mock_energy_lookup + sw_priority_rates, + nsw_priority_rates, + direct_events, + Mock(), + mock_energy_lookup, ) input_collection = ProcessingInputCollection() - input_metadata = InputMetadata('codice', "l3a", Mock(spec=datetime), Mock(spec=datetime), 'v02') - processor = CodiceLoProcessor(dependencies=input_collection, input_metadata=input_metadata) + input_metadata = InputMetadata( + "codice", "l3a", Mock(spec=datetime), Mock(spec=datetime), "v02" + ) + processor = CodiceLoProcessor( + dependencies=input_collection, input_metadata=input_metadata + ) result = processor.process_l3a_direct_event_data_product(dependencies) self.assertEqual(1, mock_calculate_normalization_factor.call_count) - actual_stacked_priorities = mock_calculate_normalization_factor.call_args.args[0] + actual_stacked_priorities = mock_calculate_normalization_factor.call_args.args[ + 0 + ] self.assertEqual( - (num_direct_event_epochs, num_priorities, num_energy_bins, priority_l1_spin_sectors), + ( + num_direct_event_epochs, + num_priorities, + num_energy_bins, + priority_l1_spin_sectors, + ), actual_stacked_priorities.shape, ) - np.testing.assert_equal(actual_stacked_priorities[0, :, 0, 0], np.full(7, np.nan)) + np.testing.assert_equal( + actual_stacked_priorities[0, :, 0, 0], np.full(7, np.nan) + ) - @patch(f'{MODULE}.SpinAngleLookup') - @patch(f'{MODULE}.lookup_normalization_per_event', autospec=True) - @patch(f'{MODULE}.calculate_normalization_factor', autospec=True) - @patch(f'{MODULE}.calculate_mass_per_charge', autospec=True) - @patch(f'{MODULE}.calculate_mass', autospec=True) + @patch(f"{MODULE}.SpinAngleLookup") + @patch(f"{MODULE}.lookup_normalization_per_event", autospec=True) + @patch(f"{MODULE}.calculate_normalization_factor", autospec=True) + @patch(f"{MODULE}.calculate_mass_per_charge", autospec=True) + @patch(f"{MODULE}.calculate_mass", autospec=True) def test_process_l3a_direct_events_aligns_passthrough_vars_to_direct_event_epochs_when_missing( - self, mock_calculate_mass, mock_calculate_mass_per_charge, - mock_calculate_normalization_factor, mock_lookup_normalization_per_event, - mock_spin_angle_lookup_class): + self, + mock_calculate_mass, + mock_calculate_mass_per_charge, + mock_calculate_normalization_factor, + mock_lookup_normalization_per_event, + mock_spin_angle_lookup_class, + ): rng = np.random.default_rng(seed=42) num_spin_angle_bins = 4 @@ -988,8 +1535,16 @@ def test_process_l3a_direct_events_aligns_passthrough_vars_to_direct_event_epoch num_sw_epochs = len(sw_priority_epochs) num_nsw_epochs = len(nsw_priority_epochs) - sw_priority_counts_shape = (num_sw_epochs, num_energy_bins, priority_l1_spin_sectors) - nsw_priority_counts_shape = (num_nsw_epochs, num_energy_bins, priority_l1_spin_sectors) + sw_priority_counts_shape = ( + num_sw_epochs, + num_energy_bins, + priority_l1_spin_sectors, + ) + nsw_priority_counts_shape = ( + num_nsw_epochs, + num_energy_bins, + priority_l1_spin_sectors, + ) sw_priority_rates = create_dataclass_mock(CodiceLoL1aSWPriorityRates) sw_priority_rates.epoch = sw_priority_epochs @@ -998,7 +1553,9 @@ def test_process_l3a_direct_events_aligns_passthrough_vars_to_direct_event_epoch sw_priority_rates.p2_heplusplus = rng.random(sw_priority_counts_shape) sw_priority_rates.p3_heavies = rng.random(sw_priority_counts_shape) sw_priority_rates.p4_dcrs = rng.random(sw_priority_counts_shape) - sw_priority_rates.half_spin_per_esa_step = rng.random((num_sw_epochs, num_energy_bins)) + sw_priority_rates.half_spin_per_esa_step = rng.random( + (num_sw_epochs, num_energy_bins) + ) sw_priority_rates.rgfo_spin_sector = rng.random(num_sw_epochs) sw_priority_rates.rgfo_esa_step = rng.random(num_sw_epochs) sw_priority_rates.nso_spin_sector = rng.random(num_sw_epochs) @@ -1006,20 +1563,32 @@ def test_process_l3a_direct_events_aligns_passthrough_vars_to_direct_event_epoch sw_priority_rates.esa_step = np.arange(num_energy_bins) sw_priority_rates.rgfo_half_spin = rng.integers(0, 12, num_sw_epochs, np.uint8) sw_priority_rates.nso_half_spin = rng.integers(0, 12, num_sw_epochs, np.uint8) - sw_priority_rates.acquisition_time_per_esa_step = rng.random((num_sw_epochs, num_energy_bins)) + sw_priority_rates.acquisition_time_per_esa_step = rng.random( + (num_sw_epochs, num_energy_bins) + ) nsw_priority_rates = create_dataclass_mock(CodiceLoL1aNSWPriorityRates) nsw_priority_rates.epoch = nsw_priority_epochs nsw_priority_rates.p5_heavies = rng.random(nsw_priority_counts_shape) nsw_priority_rates.p6_hplus_heplusplus = rng.random(nsw_priority_counts_shape) - mass_per_charge = rng.random((num_direct_event_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size)) + mass_per_charge = rng.random( + (num_direct_event_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size) + ) mock_calculate_mass_per_charge.return_value = mass_per_charge - mass = rng.random((num_direct_event_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size)) + mass = rng.random( + (num_direct_event_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size) + ) mock_calculate_mass.return_value = mass codice_l2_variables = { - f.name: rng.random((num_direct_event_epochs, CODICE_LO_L2_NUM_PRIORITIES, event_buffer_size)) + f.name: rng.random( + ( + num_direct_event_epochs, + CODICE_LO_L2_NUM_PRIORITIES, + event_buffer_size, + ) + ) for f in fields(CodiceLoL2DirectEventData) } codice_l2_variables["epoch"] = direct_event_epochs @@ -1036,7 +1605,12 @@ def test_process_l3a_direct_events_aligns_passthrough_vars_to_direct_event_epoch mock_energy_lookup.num_bins = num_energy_bins expected_normalization = rng.random( - (num_direct_event_epochs, num_priorities, num_energy_bins, num_spin_angle_bins) + ( + num_direct_event_epochs, + num_priorities, + num_energy_bins, + num_spin_angle_bins, + ) ) mock_calculate_normalization_factor.return_value = expected_normalization @@ -1047,16 +1621,27 @@ def test_process_l3a_direct_events_aligns_passthrough_vars_to_direct_event_epoch ) dependencies = CodiceLoL3aDirectEventsDependencies( - sw_priority_rates, nsw_priority_rates, direct_events, Mock(), mock_energy_lookup + sw_priority_rates, + nsw_priority_rates, + direct_events, + Mock(), + mock_energy_lookup, ) input_collection = ProcessingInputCollection() - input_metadata = InputMetadata('codice', "l3a", Mock(spec=datetime), Mock(spec=datetime), 'v02') - processor = CodiceLoProcessor(dependencies=input_collection, input_metadata=input_metadata) + input_metadata = InputMetadata( + "codice", "l3a", Mock(spec=datetime), Mock(spec=datetime), "v02" + ) + processor = CodiceLoProcessor( + dependencies=input_collection, input_metadata=input_metadata + ) result = processor.process_l3a_direct_event_data_product(dependencies) variables_to_compare = [ - (sw_priority_rates.acquisition_time_per_esa_step, result.acquisition_time_per_esa_step), + ( + sw_priority_rates.acquisition_time_per_esa_step, + result.acquisition_time_per_esa_step, + ), (sw_priority_rates.rgfo_half_spin, result.rgfo_half_spin), (sw_priority_rates.rgfo_spin_sector, result.rgfo_spin_sector), (sw_priority_rates.rgfo_esa_step, result.rgfo_esa_step), @@ -1084,18 +1669,24 @@ def test_process_l3a_direct_events_aligns_passthrough_vars_to_direct_event_epoch self.assertTrue(np.all(result_var.mask[2] == False)) self.assertTrue(np.all(result_var.mask[3] == False)) - @patch(f'{MODULE}.rebin_3d_distribution_azimuth_to_elevation') - @patch(f'{MODULE}.combine_priorities_for_species_and_calculate_count_uncertainty') - @patch(f'{MODULE}.PositionToElevationLookup') - @patch(f'{MODULE}.rebin_to_counts_by_species_elevation_and_spin_sector') - @patch(f'{MODULE}.convert_counts_to_intensity') - def test_process_l3b_3d_distributions(self, mock_convert_counts_to_intensity, mock_rebin, - mock_elevation_angle_lookup_class, - mock_combine_priorities_for_species_and_calculate_uncertainty, - mock_rebin_3d_distribution_azimuth_to_elevation): + @patch(f"{MODULE}.rebin_3d_distribution_azimuth_to_elevation") + @patch(f"{MODULE}.combine_priorities_for_species_and_calculate_count_uncertainty") + @patch(f"{MODULE}.PositionToElevationLookup") + @patch(f"{MODULE}.rebin_to_counts_by_species_elevation_and_spin_sector") + @patch(f"{MODULE}.convert_counts_to_intensity") + def test_process_l3b_3d_distributions( + self, + mock_convert_counts_to_intensity, + mock_rebin, + mock_elevation_angle_lookup_class, + mock_combine_priorities_for_species_and_calculate_uncertainty, + mock_rebin_3d_distribution_azimuth_to_elevation, + ): mock_elevation_lookup = mock_elevation_angle_lookup_class.return_value - input_metadata = InputMetadata('codice', "l3a", datetime(2026, 5, 13), Mock(spec=datetime), 'v02') + input_metadata = InputMetadata( + "codice", "l3a", datetime(2026, 5, 13), Mock(spec=datetime), "v02" + ) l3a_de_half_spin_per_esa_step = np.array([[3, 2, 1]]) mock_l3a_direct_event_data = Mock( @@ -1122,32 +1713,51 @@ def test_process_l3b_3d_distributions(self, mock_convert_counts_to_intensity, mo geometric_factors_lookup=mock_geometric_factor_lut, efficiency_factors_lut=mock_efficiency_lut, energy_per_charge_lut=mock_energy_lookup, - species=sentinel.species + species=sentinel.species, ) counts = np.array([]) count_uncerts = np.array([]) - mock_combine_priorities_for_species_and_calculate_uncertainty.return_value = (counts, count_uncerts) + mock_combine_priorities_for_species_and_calculate_uncertainty.return_value = ( + counts, + count_uncerts, + ) mock_mass_bin_lookup.get_species_index.return_value = sentinel.species_index - mock_convert_counts_to_intensity.side_effect = [sentinel.intensity, sentinel.intensity_uncert] + mock_convert_counts_to_intensity.side_effect = [ + sentinel.intensity, + sentinel.intensity_uncert, + ] mock_rebinned_intensity = np.arange(25).reshape(5, 5) mock_rebinned_uncert = np.arange(25, 50).reshape(5, 5) - mock_rebin_3d_distribution_azimuth_to_elevation.side_effect = [mock_rebinned_intensity, mock_rebinned_uncert] + mock_rebin_3d_distribution_azimuth_to_elevation.side_effect = [ + mock_rebinned_intensity, + mock_rebinned_uncert, + ] counts_3d_distribution = mock_rebin.return_value - counts_3d_distribution.__getitem__.return_value = sentinel.counts_for_species_by_priority + counts_3d_distribution.__getitem__.return_value = ( + sentinel.counts_for_species_by_priority + ) - processor = CodiceLoProcessor(dependencies=Mock(), input_metadata=input_metadata) - l3b_3d_distribution_data_product = processor.process_l3b_3d_distribution_product(dependencies) + processor = CodiceLoProcessor( + dependencies=Mock(), input_metadata=input_metadata + ) + l3b_3d_distribution_data_product = ( + processor.process_l3b_3d_distribution_product(dependencies) + ) mock_elevation_angle_lookup_class.assert_called_once() - counts_3d_distribution.__getitem__.assert_called_once_with(sentinel.species_index) + counts_3d_distribution.__getitem__.assert_called_once_with( + sentinel.species_index + ) mock_rebin.assert_called_once_with( direct_event_data=mock_l3a_direct_event_data, mass_species_bin_lookup=dependencies.mass_species_bin_lookup, ) - mock_combine_priorities_for_species_and_calculate_uncertainty.assert_called_once_with(sentinel.counts_for_species_by_priority) + mock_combine_priorities_for_species_and_calculate_uncertainty.assert_called_once_with( + sentinel.counts_for_species_by_priority + ) mock_compute_geometric_factors = mock_geometric_factor_lut.get_geometric_factors mock_compute_geometric_factors.assert_called_once_with( @@ -1155,50 +1765,121 @@ def test_process_l3b_3d_distributions(self, mock_convert_counts_to_intensity, mo sentinel.rgfo_spin_sector, sentinel.rgfo_esa_step, NumpyArrayMatcher(l3a_de_half_spin_per_esa_step), - date(2026, 5, 13) + date(2026, 5, 13), + ) + mock_convert_counts_to_intensity.assert_has_calls( + [ + call( + counts, + sentinel.acquisition_time, + mock_energy_lookup, + mock_efficiency_lut, + mock_compute_geometric_factors.return_value, + ), + call( + count_uncerts, + sentinel.acquisition_time, + mock_energy_lookup, + mock_efficiency_lut, + mock_compute_geometric_factors.return_value, + ), + ] ) - mock_convert_counts_to_intensity.assert_has_calls([ - call(counts, sentinel.acquisition_time, mock_energy_lookup, mock_efficiency_lut, mock_compute_geometric_factors.return_value), - call(count_uncerts, sentinel.acquisition_time, mock_energy_lookup, mock_efficiency_lut, mock_compute_geometric_factors.return_value), - ]) - mock_rebin_3d_distribution_azimuth_to_elevation.assert_has_calls([ - call(sentinel.intensity, NumpyArrayMatcher(np.arange(1, 25)), mock_elevation_lookup, NumpyArrayMatcher(l3a_de_half_spin_per_esa_step)), - call(sentinel.intensity_uncert, NumpyArrayMatcher(np.arange(1, 25)), mock_elevation_lookup, NumpyArrayMatcher(l3a_de_half_spin_per_esa_step))] + mock_rebin_3d_distribution_azimuth_to_elevation.assert_has_calls( + [ + call( + sentinel.intensity, + NumpyArrayMatcher(np.arange(1, 25)), + mock_elevation_lookup, + NumpyArrayMatcher(l3a_de_half_spin_per_esa_step), + ), + call( + sentinel.intensity_uncert, + NumpyArrayMatcher(np.arange(1, 25)), + mock_elevation_lookup, + NumpyArrayMatcher(l3a_de_half_spin_per_esa_step), + ), + ] ) expected_offset = 45.9086 - expected_spin_angles = np.array([ - 30 + expected_offset, - 90 + expected_offset, - 170 + expected_offset, - 350 + expected_offset - 360 - ]) - - self.assertIsInstance(l3b_3d_distribution_data_product, CodiceLoL3b3dDistributionDataProduct) - self.assertEqual(processor.input_metadata, l3b_3d_distribution_data_product.input_metadata) - self.assertEqual(mock_l3a_direct_event_data.epoch, l3b_3d_distribution_data_product.epoch) - self.assertEqual(mock_l3a_direct_event_data.epoch_delta, l3b_3d_distribution_data_product.epoch_delta) - self.assertEqual(mock_l3a_direct_event_data.rgfo_esa_step, l3b_3d_distribution_data_product.rgfo_esa_step) - self.assertEqual(mock_l3a_direct_event_data.rgfo_spin_sector, l3b_3d_distribution_data_product.rgfo_spin_sector) - self.assertEqual(mock_l3a_direct_event_data.rgfo_half_spin, l3b_3d_distribution_data_product.rgfo_half_spin) - np.testing.assert_array_equal(l3b_3d_distribution_data_product.half_spin_per_esa_step, - np.flip(mock_l3a_direct_event_data.half_spin_per_esa_step, axis=1)) - - self.assertEqual(mock_elevation_lookup.bin_centers, l3b_3d_distribution_data_product.elevation) - self.assertEqual(mock_elevation_lookup.bin_deltas, l3b_3d_distribution_data_product.elevation_delta) - np.testing.assert_array_equal(l3b_3d_distribution_data_product.spin_angle, expected_spin_angles) - self.assertEqual(sentinel.spin_angle_bin_delta, l3b_3d_distribution_data_product.spin_angle_delta) - - np.testing.assert_array_equal(np.flip(mock_energy_lookup.bin_centers), l3b_3d_distribution_data_product.energy) - np.testing.assert_array_equal(np.flip(mock_energy_lookup.delta_plus), - l3b_3d_distribution_data_product.energy_delta_plus) - np.testing.assert_array_equal(np.flip(mock_energy_lookup.delta_minus), - l3b_3d_distribution_data_product.energy_delta_minus) - - np.testing.assert_array_equal(np.flip(mock_rebinned_intensity, axis=1), - l3b_3d_distribution_data_product.species_data) - np.testing.assert_array_equal(np.flip(mock_rebinned_uncert, axis=1), l3b_3d_distribution_data_product.species_data_stat_uncert) + expected_spin_angles = np.array( + [ + 30 + expected_offset, + 90 + expected_offset, + 170 + expected_offset, + 350 + expected_offset - 360, + ] + ) + + self.assertIsInstance( + l3b_3d_distribution_data_product, CodiceLoL3b3dDistributionDataProduct + ) + self.assertEqual( + processor.input_metadata, l3b_3d_distribution_data_product.input_metadata + ) + self.assertEqual( + mock_l3a_direct_event_data.epoch, l3b_3d_distribution_data_product.epoch + ) + self.assertEqual( + mock_l3a_direct_event_data.epoch_delta, + l3b_3d_distribution_data_product.epoch_delta, + ) + self.assertEqual( + mock_l3a_direct_event_data.rgfo_esa_step, + l3b_3d_distribution_data_product.rgfo_esa_step, + ) + self.assertEqual( + mock_l3a_direct_event_data.rgfo_spin_sector, + l3b_3d_distribution_data_product.rgfo_spin_sector, + ) + self.assertEqual( + mock_l3a_direct_event_data.rgfo_half_spin, + l3b_3d_distribution_data_product.rgfo_half_spin, + ) + np.testing.assert_array_equal( + l3b_3d_distribution_data_product.half_spin_per_esa_step, + np.flip(mock_l3a_direct_event_data.half_spin_per_esa_step, axis=1), + ) + + self.assertEqual( + mock_elevation_lookup.bin_centers, + l3b_3d_distribution_data_product.elevation, + ) + self.assertEqual( + mock_elevation_lookup.bin_deltas, + l3b_3d_distribution_data_product.elevation_delta, + ) + np.testing.assert_array_equal( + l3b_3d_distribution_data_product.spin_angle, expected_spin_angles + ) + self.assertEqual( + sentinel.spin_angle_bin_delta, + l3b_3d_distribution_data_product.spin_angle_delta, + ) + + np.testing.assert_array_equal( + np.flip(mock_energy_lookup.bin_centers), + l3b_3d_distribution_data_product.energy, + ) + np.testing.assert_array_equal( + np.flip(mock_energy_lookup.delta_plus), + l3b_3d_distribution_data_product.energy_delta_plus, + ) + np.testing.assert_array_equal( + np.flip(mock_energy_lookup.delta_minus), + l3b_3d_distribution_data_product.energy_delta_minus, + ) + + np.testing.assert_array_equal( + np.flip(mock_rebinned_intensity, axis=1), + l3b_3d_distribution_data_product.species_data, + ) + np.testing.assert_array_equal( + np.flip(mock_rebinned_uncert, axis=1), + l3b_3d_distribution_data_product.species_data_stat_uncert, + ) self.assertEqual(sentinel.species, l3b_3d_distribution_data_product.species) def test_process_3d_distributions_save_for_each_species(self): @@ -1207,59 +1888,89 @@ def test_process_3d_distributions_save_for_each_species(self): with self.subTest(species=species): self._test_process_3d_distributions_save(species) - @patch(f'{MODULE}.CodiceLoL3b3dDistributionsDependencies.fetch_dependencies') - @patch(f'{MODULE}.CodiceLoProcessor.process_l3b_3d_distribution_product') - @patch(f'{MODULE}.save_data') - @patch('imap_l3_processing.processor.spiceypy') - def _test_process_3d_distributions_save(self, species, mock_spiceypy, mock_save_data, - mock_process_l3b_3d_distribution_product, - mock_fetch_dependencies): + @patch(f"{MODULE}.CodiceLoL3b3dDistributionsDependencies.fetch_dependencies") + @patch(f"{MODULE}.CodiceLoProcessor.process_l3b_3d_distribution_product") + @patch(f"{MODULE}.save_data") + @patch("imap_l3_processing.processor.spiceypy") + def _test_process_3d_distributions_save( + self, + species, + mock_spiceypy, + mock_save_data, + mock_process_l3b_3d_distribution_product, + mock_fetch_dependencies, + ): input_collection = MagicMock() - input_collection.get_file_paths.return_value = [Path('path/to/parent_file_1'), Path('path/to/parent_file_2')] - input_metadata = InputMetadata(instrument='codice', - data_level="l3a", - start_date=Mock(spec=datetime), - end_date=Mock(spec=datetime), - version='v02', - descriptor=f'lo-{species}-3d-distribution') + input_collection.get_file_paths.return_value = [ + Path("path/to/parent_file_1"), + Path("path/to/parent_file_2"), + ] + input_metadata = InputMetadata( + instrument="codice", + data_level="l3a", + start_date=Mock(spec=datetime), + end_date=Mock(spec=datetime), + version="v02", + descriptor=f"lo-{species}-3d-distribution", + ) mock_spiceypy.ktotal.return_value = 0 - processor = CodiceLoProcessor(dependencies=input_collection, input_metadata=input_metadata) + processor = CodiceLoProcessor( + dependencies=input_collection, input_metadata=input_metadata + ) product = processor.process() mock_fetch_dependencies.assert_called_once_with(processor.dependencies, species) - mock_process_l3b_3d_distribution_product.assert_called_once_with(mock_fetch_dependencies.return_value) + mock_process_l3b_3d_distribution_product.assert_called_once_with( + mock_fetch_dependencies.return_value + ) - mock_save_data.assert_called_once_with(mock_process_l3b_3d_distribution_product.return_value) + mock_save_data.assert_called_once_with( + mock_process_l3b_3d_distribution_product.return_value + ) - self.assertEqual(['parent_file_1', 'parent_file_2'], - mock_process_l3b_3d_distribution_product.return_value.parent_file_names) + self.assertEqual( + ["parent_file_1", "parent_file_2"], + mock_process_l3b_3d_distribution_product.return_value.parent_file_names, + ) self.assertEqual([mock_save_data.return_value], product) def test_process_l3a_direct_events_all_fill_integration(self): input_collection = MagicMock() - input_collection.get_file_paths.return_value = [Path('path/to/parent_file_1'), Path('path/to/parent_file_2')] - input_metadata = InputMetadata(instrument='codice', - data_level="l3a", - start_date=Mock(spec=datetime), - end_date=Mock(spec=datetime), - version='v02', - descriptor='lo-3d-instrument-frame') + input_collection.get_file_paths.return_value = [ + Path("path/to/parent_file_1"), + Path("path/to/parent_file_2"), + ] + input_metadata = InputMetadata( + instrument="codice", + data_level="l3a", + start_date=Mock(spec=datetime), + end_date=Mock(spec=datetime), + version="v02", + descriptor="lo-3d-instrument-frame", + ) dependencies = CodiceLoL3aDirectEventsDependencies.from_file_paths( sw_priority_rates_cdf=get_test_data_path( - "codice/imap_codice_l1a_lo-sw-priority_20260307_v003-all-fill.cdf"), + "codice/imap_codice_l1a_lo-sw-priority_20260307_v003-all-fill.cdf" + ), nsw_priority_rates_cdf=get_test_data_path( - "codice/imap_codice_l1a_lo-nsw-priority_20260307_v003-all-fill.cdf"), - direct_event_path=get_test_data_path("codice/imap_codice_l2_lo-direct-events_20260307_v003-all-fill.cdf"), + "codice/imap_codice_l1a_lo-nsw-priority_20260307_v003-all-fill.cdf" + ), + direct_event_path=get_test_data_path( + "codice/imap_codice_l2_lo-direct-events_20260307_v003-all-fill.cdf" + ), mass_coefficients_file_path=get_test_data_path( - "codice/imap_codice_mass-coefficient-lookup_20241110_v003.csv"), + "codice/imap_codice_mass-coefficient-lookup_20241110_v003.csv" + ), esa_to_energy_per_charge_file_path=get_test_data_path( "codice/imap_codice_lo-energy-per-charge_20241110_v002.csv" - ) + ), ) - processor = CodiceLoProcessor(dependencies=input_collection, input_metadata=input_metadata) + processor = CodiceLoProcessor( + dependencies=input_collection, input_metadata=input_metadata + ) try: processor.process_l3a_direct_event_data_product(dependencies) @@ -1267,5 +1978,5 @@ def test_process_l3a_direct_events_all_fill_integration(self): self.fail(e) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/codice/l3/lo/test_models.py b/tests/codice/l3/lo/test_models.py index a53a8cfae..0118d1b6f 100644 --- a/tests/codice/l3/lo/test_models.py +++ b/tests/codice/l3/lo/test_models.py @@ -29,8 +29,14 @@ ENERGY_DELTA_PLUS_VAR_NAME, ENERGY_DELTA_MINUS_VAR_NAME, ELEVATION_ANGLE_LABEL_VAR_NAME, - ENERGY_LABEL_VAR_NAME, SPIN_ANGLE_DELTA_VAR_NAME, SPIN_ANGLE_VAR_NAME, SPIN_ANGLE_LABEL_VAR_NAME, - RGFO_ESA_STEP_VAR_NAME, RGFO_SPIN_SECTOR_VAR_NAME, RGFO_HALF_SPIN_VAR_NAME, HALF_SPIN_PER_ESA_STEP_VAR_NAME, + ENERGY_LABEL_VAR_NAME, + SPIN_ANGLE_DELTA_VAR_NAME, + SPIN_ANGLE_VAR_NAME, + SPIN_ANGLE_LABEL_VAR_NAME, + RGFO_ESA_STEP_VAR_NAME, + RGFO_SPIN_SECTOR_VAR_NAME, + RGFO_HALF_SPIN_VAR_NAME, + HALF_SPIN_PER_ESA_STEP_VAR_NAME, ) from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.utils import save_data @@ -38,22 +44,22 @@ from tests.test_helpers import get_test_data_path LO_L2_SW_SPECIES_DATA_VARS = [ - 'hplus', - 'heplusplus', - 'heplus', - 'ne', - 'cplus4', - 'cplus5', - 'cplus6', - 'oplus5', - 'oplus6', - 'oplus7', - 'oplus8', - 'cnoplus', - 'mg', - 'si', - 'fe_loq', - 'fe_hiq', + "hplus", + "heplusplus", + "heplus", + "ne", + "cplus4", + "cplus5", + "cplus6", + "oplus5", + "oplus6", + "oplus7", + "oplus8", + "cnoplus", + "mg", + "si", + "fe_loq", + "fe_hiq", ] @@ -68,51 +74,77 @@ def test_lo_l2_sw_species_read_from_cdf(self): epoch_delta_plus = rng.random(len(epoch)) energy_per_charge = np.geomspace(2, 1000) spin_sector = np.linspace(0, 360, 24) - hplus = rng.random((len(epoch), len(energy_per_charge), len(spin_sector))) - heplus = rng.random((len(epoch), len(energy_per_charge), len(spin_sector))) - heplusplus = rng.random((len(epoch), len(energy_per_charge), len(spin_sector))) + hplus = rng.random( + (len(epoch), len(energy_per_charge), len(spin_sector)) + ) + heplus = rng.random( + (len(epoch), len(energy_per_charge), len(spin_sector)) + ) + heplusplus = rng.random( + (len(epoch), len(energy_per_charge), len(spin_sector)) + ) ne = rng.random((len(epoch), len(energy_per_charge), len(spin_sector))) - cplus4 = rng.random((len(epoch), len(energy_per_charge), len(spin_sector))) - cplus5 = rng.random((len(epoch), len(energy_per_charge), len(spin_sector))) - cplus6 = rng.random((len(epoch), len(energy_per_charge), len(spin_sector))) - oplus5 = rng.random((len(epoch), len(energy_per_charge), len(spin_sector))) - oplus6 = rng.random((len(epoch), len(energy_per_charge), len(spin_sector))) - oplus7 = rng.random((len(epoch), len(energy_per_charge), len(spin_sector))) - oplus8 = rng.random((len(epoch), len(energy_per_charge), len(spin_sector))) + cplus4 = rng.random( + (len(epoch), len(energy_per_charge), len(spin_sector)) + ) + cplus5 = rng.random( + (len(epoch), len(energy_per_charge), len(spin_sector)) + ) + cplus6 = rng.random( + (len(epoch), len(energy_per_charge), len(spin_sector)) + ) + oplus5 = rng.random( + (len(epoch), len(energy_per_charge), len(spin_sector)) + ) + oplus6 = rng.random( + (len(epoch), len(energy_per_charge), len(spin_sector)) + ) + oplus7 = rng.random( + (len(epoch), len(energy_per_charge), len(spin_sector)) + ) + oplus8 = rng.random( + (len(epoch), len(energy_per_charge), len(spin_sector)) + ) mg = rng.random((len(epoch), len(energy_per_charge), len(spin_sector))) si = rng.random((len(epoch), len(energy_per_charge), len(spin_sector))) cnoplus = rng.random((len(epoch), len(energy_per_charge))) - fe_loq = rng.random((len(epoch), len(energy_per_charge), len(spin_sector))) - fe_hiq = rng.random((len(epoch), len(energy_per_charge), len(spin_sector))) + fe_loq = rng.random( + (len(epoch), len(energy_per_charge), len(spin_sector)) + ) + fe_hiq = rng.random( + (len(epoch), len(energy_per_charge), len(spin_sector)) + ) data_quality = rng.random(len(epoch)) - cdf_file['epoch'] = epoch - cdf_file['epoch_delta_minus'] = epoch_delta_minus - cdf_file['epoch_delta_plus'] = epoch_delta_plus - cdf_file['energy_per_charge'] = energy_per_charge - cdf_file['spin_sector'] = spin_sector - cdf_file['hplus'] = hplus - cdf_file['heplusplus'] = heplusplus - cdf_file['heplus'] = heplus - cdf_file['ne'] = ne - cdf_file['cplus4'] = cplus4 - cdf_file['cplus5'] = cplus5 - cdf_file['cplus6'] = cplus6 - cdf_file['oplus5'] = oplus5 - cdf_file['oplus6'] = oplus6 - cdf_file['oplus7'] = oplus7 - cdf_file['oplus8'] = oplus8 - cdf_file['cnoplus'] = cnoplus - cdf_file['mg'] = mg - cdf_file['si'] = si - cdf_file['fe_loq'] = fe_loq - cdf_file['fe_hiq'] = fe_hiq - cdf_file['data_quality'] = data_quality + cdf_file["epoch"] = epoch + cdf_file["epoch_delta_minus"] = epoch_delta_minus + cdf_file["epoch_delta_plus"] = epoch_delta_plus + cdf_file["energy_per_charge"] = energy_per_charge + cdf_file["spin_sector"] = spin_sector + cdf_file["hplus"] = hplus + cdf_file["heplusplus"] = heplusplus + cdf_file["heplus"] = heplus + cdf_file["ne"] = ne + cdf_file["cplus4"] = cplus4 + cdf_file["cplus5"] = cplus5 + cdf_file["cplus6"] = cplus6 + cdf_file["oplus5"] = oplus5 + cdf_file["oplus6"] = oplus6 + cdf_file["oplus7"] = oplus7 + cdf_file["oplus8"] = oplus8 + cdf_file["cnoplus"] = cnoplus + cdf_file["mg"] = mg + cdf_file["si"] = si + cdf_file["fe_loq"] = fe_loq + cdf_file["fe_hiq"] = fe_hiq + cdf_file["data_quality"] = data_quality for var in LO_L2_SW_SPECIES_DATA_VARS: cdf_file[var].attrs["FILLVAL"] = -1e31 - result: CodiceLoL2SWSpeciesData = CodiceLoL2SWSpeciesData.read_from_cdf(cdf_file_path) + result: CodiceLoL2SWSpeciesData = CodiceLoL2SWSpeciesData.read_from_cdf( + cdf_file_path + ) np.testing.assert_array_equal(result.epoch, epoch) np.testing.assert_array_equal(result.epoch_delta_minus, epoch_delta_minus) np.testing.assert_array_equal(result.epoch_delta_plus, epoch_delta_plus) @@ -145,24 +177,30 @@ def test_lo_l2_sw_species_read_from_cdf_handles_fillval(self): epoch_delta_plus = rng.random(len(epoch)) energy_per_charge = np.geomspace(2, 1000) spin_sector = np.linspace(0, 360, 24) - fills = np.full((len(epoch), len(energy_per_charge), len(spin_sector)), -1e31) + fills = np.full( + (len(epoch), len(energy_per_charge), len(spin_sector)), -1e31 + ) data_quality = rng.random(len(epoch)) - cdf_file['epoch'] = epoch - cdf_file['epoch_delta_minus'] = epoch_delta_minus - cdf_file['epoch_delta_plus'] = epoch_delta_plus - cdf_file['energy_per_charge'] = energy_per_charge - cdf_file['spin_sector'] = spin_sector - cdf_file['data_quality'] = data_quality + cdf_file["epoch"] = epoch + cdf_file["epoch_delta_minus"] = epoch_delta_minus + cdf_file["epoch_delta_plus"] = epoch_delta_plus + cdf_file["energy_per_charge"] = energy_per_charge + cdf_file["spin_sector"] = spin_sector + cdf_file["data_quality"] = data_quality for var in LO_L2_SW_SPECIES_DATA_VARS: cdf_file[var] = fills cdf_file[var].attrs["FILLVAL"] = -1e31 - nans = np.full((len(epoch), len(energy_per_charge), len(spin_sector)), np.nan) + nans = np.full( + (len(epoch), len(energy_per_charge), len(spin_sector)), np.nan + ) - result: CodiceLoL2SWSpeciesData = CodiceLoL2SWSpeciesData.read_from_cdf(cdf_file_path) + result: CodiceLoL2SWSpeciesData = CodiceLoL2SWSpeciesData.read_from_cdf( + cdf_file_path + ) np.testing.assert_array_equal(result.epoch, epoch) np.testing.assert_array_equal(result.epoch_delta_minus, epoch_delta_minus) np.testing.assert_array_equal(result.epoch_delta_plus, epoch_delta_plus) @@ -214,11 +252,15 @@ def test_codice_lo_l3a_partial_density_to_data_product(self): self.assertEqual(16, len(actual_data_product_variables)) - for input_variable, actual_data_product_variable in zip(input_data_product_kwargs.items(), - actual_data_product_variables): + for input_variable, actual_data_product_variable in zip( + input_data_product_kwargs.items(), actual_data_product_variables + ): input_name, expected_value = input_variable - np.testing.assert_array_equal(actual_data_product_variable.value, getattr(data_product.data, input_name)) + np.testing.assert_array_equal( + actual_data_product_variable.value, + getattr(data_product.data, input_name), + ) self.assertEqual(input_name, actual_data_product_variable.name) def test_codice_lo_l3a_ratios_to_data_product(self): @@ -236,52 +278,69 @@ def test_codice_lo_l3a_ratios_to_data_product(self): "felo_to_fehi_ratio": np.array([8]), } - data_product = CodiceLoL3aRatiosDataProduct( - Mock(), **input_data_product_kwargs - ) + data_product = CodiceLoL3aRatiosDataProduct(Mock(), **input_data_product_kwargs) actual_data_product_variables = data_product.to_data_product_variables() self.assertEqual(9, len(actual_data_product_variables)) - for input_variable, actual_data_product_variable in zip(input_data_product_kwargs.items(), - actual_data_product_variables): + for input_variable, actual_data_product_variable in zip( + input_data_product_kwargs.items(), actual_data_product_variables + ): input_name, expected_value = input_variable - np.testing.assert_array_equal(actual_data_product_variable.value, getattr(data_product, input_name)) + np.testing.assert_array_equal( + actual_data_product_variable.value, getattr(data_product, input_name) + ) self.assertEqual(input_name, actual_data_product_variable.name) def test_codice_lo_l3a_partial_density_read_from_cdf(self): rng = np.random.default_rng() epoch = [datetime.now(), datetime.now() + timedelta(days=1)] - density_data = CodiceLoPartialDensityData(epoch=np.array(epoch), epoch_delta=np.array([2, 2]), - hplus_partial_density=rng.random(len(epoch), dtype='float32'), - heplusplus_partial_density=rng.random(len(epoch), dtype='float32'), - cplus4_partial_density=rng.random(len(epoch), dtype='float32'), - cplus5_partial_density=rng.random(len(epoch), dtype='float32'), - cplus6_partial_density=rng.random(len(epoch), dtype='float32'), - oplus5_partial_density=rng.random(len(epoch), dtype='float32'), - oplus6_partial_density=rng.random(len(epoch), dtype='float32'), - oplus7_partial_density=rng.random(len(epoch), dtype='float32'), - oplus8_partial_density=rng.random(len(epoch), dtype='float32'), - ne_partial_density=rng.random(len(epoch), dtype='float32'), - mg_partial_density=rng.random(len(epoch), dtype='float32'), - si_partial_density=rng.random(len(epoch), dtype='float32'), - fe_loq_partial_density=rng.random(len(epoch), dtype='float32'), - fe_hiq_partial_density=rng.random(len(epoch), dtype='float32'), ) - - input_metadata = InputMetadata("codice", "l3a", datetime.now(), datetime.now(), VersionMap({"lo-partial-densities": Version(None, 1)}), "lo-partial-densities") - - codice_lo_l3a_partial_densities = CodiceLoL3aPartialDensityDataProduct(input_metadata, density_data) + density_data = CodiceLoPartialDensityData( + epoch=np.array(epoch), + epoch_delta=np.array([2, 2]), + hplus_partial_density=rng.random(len(epoch), dtype="float32"), + heplusplus_partial_density=rng.random(len(epoch), dtype="float32"), + cplus4_partial_density=rng.random(len(epoch), dtype="float32"), + cplus5_partial_density=rng.random(len(epoch), dtype="float32"), + cplus6_partial_density=rng.random(len(epoch), dtype="float32"), + oplus5_partial_density=rng.random(len(epoch), dtype="float32"), + oplus6_partial_density=rng.random(len(epoch), dtype="float32"), + oplus7_partial_density=rng.random(len(epoch), dtype="float32"), + oplus8_partial_density=rng.random(len(epoch), dtype="float32"), + ne_partial_density=rng.random(len(epoch), dtype="float32"), + mg_partial_density=rng.random(len(epoch), dtype="float32"), + si_partial_density=rng.random(len(epoch), dtype="float32"), + fe_loq_partial_density=rng.random(len(epoch), dtype="float32"), + fe_hiq_partial_density=rng.random(len(epoch), dtype="float32"), + ) + + input_metadata = InputMetadata( + "codice", + "l3a", + datetime.now(), + datetime.now(), + VersionMap({"lo-partial-densities": Version(None, 1)}), + "lo-partial-densities", + ) + + codice_lo_l3a_partial_densities = CodiceLoL3aPartialDensityDataProduct( + input_metadata, density_data + ) with tempfile.TemporaryDirectory() as tmpdir: - cdf_path = save_data(codice_lo_l3a_partial_densities, folder_path=Path(tmpdir)) + cdf_path = save_data( + codice_lo_l3a_partial_densities, folder_path=Path(tmpdir) + ) read_in_density_data = CodiceLoPartialDensityData.read_from_cdf(cdf_path) for field in fields(CodiceLoPartialDensityData): - np.testing.assert_array_equal(getattr(read_in_density_data, field.name), - getattr(density_data, field.name), - strict=True) + np.testing.assert_array_equal( + getattr(read_in_density_data, field.name), + getattr(density_data, field.name), + strict=True, + ) def test_codice_lo_l3a_charge_state_distributions_to_data_product(self): rng = np.random.default_rng() @@ -293,32 +352,46 @@ def test_codice_lo_l3a_charge_state_distributions_to_data_product(self): "carbon_charge_state_distribution": rng.random((len(epoch_data), 3)), } - data_product = CodiceLoL3ChargeStateDistributionsDataProduct(Mock(), **input_data_product_kwargs) + data_product = CodiceLoL3ChargeStateDistributionsDataProduct( + Mock(), **input_data_product_kwargs + ) actual_data_product_variables = data_product.to_data_product_variables() self.assertEqual(6, len(actual_data_product_variables)) - for input_variable, actual_data_product_variable in zip(input_data_product_kwargs.items(), - actual_data_product_variables[:4]): + for input_variable, actual_data_product_variable in zip( + input_data_product_kwargs.items(), actual_data_product_variables[:4] + ): input_name, expected_value = input_variable - np.testing.assert_array_equal(actual_data_product_variable.value, getattr(data_product, input_name)) + np.testing.assert_array_equal( + actual_data_product_variable.value, getattr(data_product, input_name) + ) self.assertEqual(input_name, actual_data_product_variable.name) - oxygen_charge_states, carbon_charge_states, = actual_data_product_variables[4:] + ( + oxygen_charge_states, + carbon_charge_states, + ) = actual_data_product_variables[4:] np.testing.assert_array_equal(oxygen_charge_states.value, [5, 6, 7, 8]) self.assertEqual("oxygen_charge_state", oxygen_charge_states.name) np.testing.assert_array_equal(carbon_charge_states.value, [4, 5, 6]) self.assertEqual("carbon_charge_state", carbon_charge_states.name) - def test_codice_lo_l3a_direct_event_read_from_cdf_reads_where_no_variables_have_fill_value(self): - no_fill_l3a_cdf_path = get_test_data_path("codice/imap_codice_l3a_lo-direct-events_20260504_v003.cdf") + def test_codice_lo_l3a_direct_event_read_from_cdf_reads_where_no_variables_have_fill_value( + self, + ): + no_fill_l3a_cdf_path = get_test_data_path( + "codice/imap_codice_l3a_lo-direct-events_20260504_v003.cdf" + ) rng = np.random.default_rng() with tempfile.TemporaryDirectory() as tmpdir: tmpdir = Path(tmpdir) test_l3a_direct_event = tmpdir / "test_l3a_cdf.cdf" - with CDF(str(test_l3a_direct_event), masterpath=str(no_fill_l3a_cdf_path)) as cdf: + with CDF( + str(test_l3a_direct_event), masterpath=str(no_fill_l3a_cdf_path) + ) as cdf: cdf["apd_energy"] = rng.random(cdf["apd_energy"].shape) cdf["elevation"] = rng.random(cdf["elevation"].shape) cdf["energy_per_charge"] = rng.random(cdf["energy_per_charge"].shape) @@ -328,61 +401,146 @@ def test_codice_lo_l3a_direct_event_read_from_cdf_reads_where_no_variables_have_ cdf["normalization"] = rng.random(cdf["normalization"].shape) cdf["spin_angle"] = rng.random(cdf["spin_angle"].shape) cdf["spin_angle_bin"] = rng.random(cdf["spin_angle_bin"].shape) - cdf["spin_angle_bin_delta"] = rng.random(cdf["spin_angle_bin_delta"].shape) + cdf["spin_angle_bin_delta"] = rng.random( + cdf["spin_angle_bin_delta"].shape + ) cdf["type"] = rng.integers(0, 5, cdf["type"].shape) - cdf["normalization_per_event"] = rng.random(cdf["normalization_per_event"].shape) + cdf["normalization_per_event"] = rng.random( + cdf["normalization_per_event"].shape + ) cdf["position"] = rng.random(cdf["position"].shape) cdf["spin_sector"] = rng.random(cdf["spin_sector"].shape) - cdf["acquisition_time_per_esa_step"] = rng.random(cdf["acquisition_time_per_esa_step"].shape) + cdf["acquisition_time_per_esa_step"] = rng.random( + cdf["acquisition_time_per_esa_step"].shape + ) - cdf["rgfo_spin_sector"] = rng.integers(0, 24, cdf["rgfo_spin_sector"].shape) + cdf["rgfo_spin_sector"] = rng.integers( + 0, 24, cdf["rgfo_spin_sector"].shape + ) cdf["rgfo_half_spin"] = rng.integers(0, 32, cdf["rgfo_half_spin"].shape) cdf["rgfo_esa_step"] = rng.integers(0, 128, cdf["rgfo_esa_step"].shape) - cdf["half_spin_per_esa_step"] = rng.integers(0, 32, cdf["half_spin_per_esa_step"].shape) + cdf["half_spin_per_esa_step"] = rng.integers( + 0, 32, cdf["half_spin_per_esa_step"].shape + ) - actual_event_data = CodiceLoDirectEventData.read_from_cdf(test_l3a_direct_event) + actual_event_data = CodiceLoDirectEventData.read_from_cdf( + test_l3a_direct_event + ) with CDF(str(test_l3a_direct_event)) as cdf: - np.testing.assert_array_equal(actual_event_data.normalization_per_event, cdf["normalization_per_event"]) - np.testing.assert_array_equal(actual_event_data.position, cdf["position"]) - np.testing.assert_array_equal(actual_event_data.energy_step, cdf["energy_step"]) - np.testing.assert_array_equal(actual_event_data.spin_sector, cdf["spin_sector"]) - np.testing.assert_array_equal(actual_event_data.acquisition_time_per_esa_step, - cdf["acquisition_time_per_esa_step"]) - np.testing.assert_array_equal(actual_event_data.apd_energy, cdf["apd_energy"]) - np.testing.assert_array_equal(actual_event_data.elevation, cdf["elevation"]) - np.testing.assert_array_equal(actual_event_data.energy_per_charge, cdf["energy_per_charge"]) - np.testing.assert_array_equal(actual_event_data.energy_step, cdf["energy_step"]) + np.testing.assert_array_equal( + actual_event_data.normalization_per_event, + cdf["normalization_per_event"], + ) + np.testing.assert_array_equal( + actual_event_data.position, cdf["position"] + ) + np.testing.assert_array_equal( + actual_event_data.energy_step, cdf["energy_step"] + ) + np.testing.assert_array_equal( + actual_event_data.spin_sector, cdf["spin_sector"] + ) + np.testing.assert_array_equal( + actual_event_data.acquisition_time_per_esa_step, + cdf["acquisition_time_per_esa_step"], + ) + np.testing.assert_array_equal( + actual_event_data.apd_energy, cdf["apd_energy"] + ) + np.testing.assert_array_equal( + actual_event_data.elevation, cdf["elevation"] + ) + np.testing.assert_array_equal( + actual_event_data.energy_per_charge, cdf["energy_per_charge"] + ) + np.testing.assert_array_equal( + actual_event_data.energy_step, cdf["energy_step"] + ) np.testing.assert_array_equal(actual_event_data.mass, cdf["mass"]) - np.testing.assert_array_equal(actual_event_data.mass_per_charge, cdf["mass_per_charge"]) - np.testing.assert_array_equal(actual_event_data.normalization, cdf["normalization"]) - np.testing.assert_array_equal(actual_event_data.spin_angle, cdf["spin_angle"]) - np.testing.assert_array_equal(actual_event_data.spin_angle_bin, cdf["spin_angle_bin"]) - np.testing.assert_array_equal(actual_event_data.spin_angle_bin_delta, cdf["spin_angle_bin_delta"]) + np.testing.assert_array_equal( + actual_event_data.mass_per_charge, cdf["mass_per_charge"] + ) + np.testing.assert_array_equal( + actual_event_data.normalization, cdf["normalization"] + ) + np.testing.assert_array_equal( + actual_event_data.spin_angle, cdf["spin_angle"] + ) + np.testing.assert_array_equal( + actual_event_data.spin_angle_bin, cdf["spin_angle_bin"] + ) + np.testing.assert_array_equal( + actual_event_data.spin_angle_bin_delta, cdf["spin_angle_bin_delta"] + ) np.testing.assert_array_equal(actual_event_data.type, cdf["type"]) - np.testing.assert_array_equal(actual_event_data.rgfo_spin_sector, cdf["rgfo_spin_sector"]) - np.testing.assert_array_equal(actual_event_data.rgfo_half_spin, cdf["rgfo_half_spin"]) - np.testing.assert_array_equal(actual_event_data.rgfo_esa_step, cdf["rgfo_esa_step"]) - np.testing.assert_array_equal(actual_event_data.half_spin_per_esa_step, cdf["half_spin_per_esa_step"]) + np.testing.assert_array_equal( + actual_event_data.rgfo_spin_sector, cdf["rgfo_spin_sector"] + ) + np.testing.assert_array_equal( + actual_event_data.rgfo_half_spin, cdf["rgfo_half_spin"] + ) + np.testing.assert_array_equal( + actual_event_data.rgfo_esa_step, cdf["rgfo_esa_step"] + ) + np.testing.assert_array_equal( + actual_event_data.half_spin_per_esa_step, + cdf["half_spin_per_esa_step"], + ) def test_codice_lo_l3a_direct_event_read_from_cdf_handles_fill_value(self): - all_fill_l3a_cdf_path = get_test_data_path("codice/imap_codice_l3a_lo-direct-events_20260504_v003-all-fill.cdf") + all_fill_l3a_cdf_path = get_test_data_path( + "codice/imap_codice_l3a_lo-direct-events_20260504_v003-all-fill.cdf" + ) actual_event_data = CodiceLoDirectEventData.read_from_cdf(all_fill_l3a_cdf_path) with CDF(str(all_fill_l3a_cdf_path)) as cdf: # @formatter:off - np.testing.assert_array_equal(actual_event_data.apd_energy, np.full_like(cdf["apd_energy"][...], np.nan)) - np.testing.assert_array_equal(actual_event_data.data_quality, cdf["data_quality"]) - np.testing.assert_array_equal(actual_event_data.elevation, np.full_like(cdf["elevation"][...], np.nan)) - np.testing.assert_array_equal(actual_event_data.energy_per_charge, np.full_like(cdf["energy_per_charge"][...], np.nan)) - np.testing.assert_array_equal(actual_event_data.mass, np.full_like(cdf["mass"][...], np.nan)) - np.testing.assert_array_equal(actual_event_data.mass_per_charge, np.full_like(cdf["mass_per_charge"][...], np.nan)) - np.testing.assert_array_equal(actual_event_data.normalization, np.full_like(cdf["normalization"][...], np.nan)) - np.testing.assert_array_equal(actual_event_data.normalization_per_event, np.full_like(cdf["normalization_per_event"][...], np.nan)) - np.testing.assert_array_equal(actual_event_data.acquisition_time_per_esa_step, np.full_like(cdf["acquisition_time_per_esa_step"][...], np.nan)) - np.testing.assert_array_equal(actual_event_data.spin_angle, np.full_like(cdf["spin_angle"][...], np.nan)) - np.testing.assert_array_equal(actual_event_data.spin_angle_bin_delta, np.full_like(cdf["spin_angle_bin_delta"][...], np.nan)) - np.testing.assert_array_equal(actual_event_data.rgfo_spin_sector, np.full_like(cdf["rgfo_spin_sector"][...], np.nan)) + np.testing.assert_array_equal( + actual_event_data.apd_energy, + np.full_like(cdf["apd_energy"][...], np.nan), + ) + np.testing.assert_array_equal( + actual_event_data.data_quality, cdf["data_quality"] + ) + np.testing.assert_array_equal( + actual_event_data.elevation, np.full_like(cdf["elevation"][...], np.nan) + ) + np.testing.assert_array_equal( + actual_event_data.energy_per_charge, + np.full_like(cdf["energy_per_charge"][...], np.nan), + ) + np.testing.assert_array_equal( + actual_event_data.mass, np.full_like(cdf["mass"][...], np.nan) + ) + np.testing.assert_array_equal( + actual_event_data.mass_per_charge, + np.full_like(cdf["mass_per_charge"][...], np.nan), + ) + np.testing.assert_array_equal( + actual_event_data.normalization, + np.full_like(cdf["normalization"][...], np.nan), + ) + np.testing.assert_array_equal( + actual_event_data.normalization_per_event, + np.full_like(cdf["normalization_per_event"][...], np.nan), + ) + np.testing.assert_array_equal( + actual_event_data.acquisition_time_per_esa_step, + np.full_like(cdf["acquisition_time_per_esa_step"][...], np.nan), + ) + np.testing.assert_array_equal( + actual_event_data.spin_angle, + np.full_like(cdf["spin_angle"][...], np.nan), + ) + np.testing.assert_array_equal( + actual_event_data.spin_angle_bin_delta, + np.full_like(cdf["spin_angle_bin_delta"][...], np.nan), + ) + np.testing.assert_array_equal( + actual_event_data.rgfo_spin_sector, + np.full_like(cdf["rgfo_spin_sector"][...], np.nan), + ) # @formatter:on masked_variables = [ @@ -424,7 +582,9 @@ def test_codice_lo_l3a_direct_event_to_data_product(self): acquisition_time_per_esa_step=rng.random((len(epoch), len(energy_step))), epoch=epoch, epoch_delta=epoch_delta, - normalization=rng.random((len(epoch), len(priority), len(energy_step), len(spin_angle))), + normalization=rng.random( + (len(epoch), len(priority), len(energy_step), len(spin_angle)) + ), mass_per_charge=rng.random((len(epoch), len(priority), len(event_num))), mass=rng.random((len(epoch), len(priority), len(event_num))), apd_energy=rng.random((len(epoch), len(priority), len(event_num))), @@ -451,84 +611,187 @@ def test_codice_lo_l3a_direct_event_to_data_product(self): nso_half_spin=rng.random(len(epoch)), nso_spin_sector=rng.random(len(epoch)), nso_esa_step=rng.random(len(epoch)), - normalization_per_event=rng.random((len(epoch), len(priority), len(energy_step), len(spin_angle))), + normalization_per_event=rng.random( + (len(epoch), len(priority), len(energy_step), len(spin_angle)) + ), esa_step=esa_step, energy_per_charge=rng.random((len(epoch), len(priority), len(event_num))), type=rng.integers(0, 5, (len(epoch), len(priority), len(event_num))), ) - np.testing.assert_array_equal(direct_event.event_index, np.arange(len(event_num))) - np.testing.assert_array_equal(direct_event.priority_index_label, np.array(["0", "1", "2", "3", "4", "5", "6"])) - np.testing.assert_array_equal(direct_event.event_index_label, np.array([str(i) for i in range(len(event_num))])) - np.testing.assert_array_equal(direct_event.energy_bin_label, - np.array([str(e) for e in direct_event.energy_bin])) - np.testing.assert_array_equal(direct_event.spin_angle_bin_label, - np.array([str(spin_angle) for spin_angle in direct_event.spin_angle_bin])) + np.testing.assert_array_equal( + direct_event.event_index, np.arange(len(event_num)) + ) + np.testing.assert_array_equal( + direct_event.priority_index_label, + np.array(["0", "1", "2", "3", "4", "5", "6"]), + ) + np.testing.assert_array_equal( + direct_event.event_index_label, + np.array([str(i) for i in range(len(event_num))]), + ) + np.testing.assert_array_equal( + direct_event.energy_bin_label, + np.array([str(e) for e in direct_event.energy_bin]), + ) + np.testing.assert_array_equal( + direct_event.spin_angle_bin_label, + np.array([str(spin_angle) for spin_angle in direct_event.spin_angle_bin]), + ) data_products = direct_event.to_data_product_variables() - non_parent_fields = [f for f in fields(CodiceLoL3aDirectEventDataProduct) - if f.name in CodiceLoL3aDirectEventDataProduct.__annotations__ or - f.name in CodiceLoDirectEventData.__annotations__] + non_parent_fields = [ + f + for f in fields(CodiceLoL3aDirectEventDataProduct) + if f.name in CodiceLoL3aDirectEventDataProduct.__annotations__ + or f.name in CodiceLoDirectEventData.__annotations__ + ] self.assertEqual(len(non_parent_fields), len(data_products)) for data_product in data_products: - np.testing.assert_array_equal(getattr(direct_event, data_product.name), data_product.value) + np.testing.assert_array_equal( + getattr(direct_event, data_product.name), data_product.value + ) def test_codice_lo_l1a_sw_priority_read_from_cdf(self): - cdf_path = get_test_data_path("codice/imap_codice_l1a_lo-sw-priority_20260307_v004.cdf") - actual_l1a_sw_priority_rates = CodiceLoL1aSWPriorityRates.read_from_cdf(cdf_path) + cdf_path = get_test_data_path( + "codice/imap_codice_l1a_lo-sw-priority_20260307_v004.cdf" + ) + actual_l1a_sw_priority_rates = CodiceLoL1aSWPriorityRates.read_from_cdf( + cdf_path + ) with CDF(str(cdf_path)) as cdf: - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.epoch, cdf["epoch"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.epoch_delta_plus, cdf["epoch_delta_plus"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.epoch_delta_minus, cdf["epoch_delta_minus"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.acquisition_time_per_esa_step, - cdf["acquisition_time_per_esa_step"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.spin_sector_index, cdf["spin_sector"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.rgfo_half_spin, cdf["rgfo_half_spin"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.nso_half_spin, cdf["nso_half_spin"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.sw_bias_gain_mode, cdf["sw_bias_gain_mode"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.st_bias_gain_mode, cdf["st_bias_gain_mode"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.data_quality, cdf["data_quality"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.spin_period, cdf["spin_period"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.p0_tcrs, cdf["p0_tcrs"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.p1_hplus, cdf["p1_hplus"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.p2_heplusplus, cdf["p2_heplusplus"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.p3_heavies, cdf["p3_heavies"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.p4_dcrs, cdf["p4_dcrs"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.half_spin_per_esa_step, - cdf["half_spin_per_esa_step"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.rgfo_spin_sector, cdf["rgfo_spin_sector"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.rgfo_esa_step, cdf["rgfo_esa_step"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.nso_spin_sector, cdf["nso_spin_sector"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.nso_esa_step, cdf["nso_esa_step"][...]) - np.testing.assert_array_equal(actual_l1a_sw_priority_rates.esa_step, cdf["esa_step"][...]) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.epoch, cdf["epoch"][...] + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.epoch_delta_plus, + cdf["epoch_delta_plus"][...], + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.epoch_delta_minus, + cdf["epoch_delta_minus"][...], + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.acquisition_time_per_esa_step, + cdf["acquisition_time_per_esa_step"][...], + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.spin_sector_index, cdf["spin_sector"][...] + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.rgfo_half_spin, cdf["rgfo_half_spin"][...] + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.nso_half_spin, cdf["nso_half_spin"][...] + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.sw_bias_gain_mode, + cdf["sw_bias_gain_mode"][...], + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.st_bias_gain_mode, + cdf["st_bias_gain_mode"][...], + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.data_quality, cdf["data_quality"][...] + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.spin_period, cdf["spin_period"][...] + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.p0_tcrs, cdf["p0_tcrs"][...] + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.p1_hplus, cdf["p1_hplus"][...] + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.p2_heplusplus, cdf["p2_heplusplus"][...] + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.p3_heavies, cdf["p3_heavies"][...] + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.p4_dcrs, cdf["p4_dcrs"][...] + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.half_spin_per_esa_step, + cdf["half_spin_per_esa_step"][...], + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.rgfo_spin_sector, + cdf["rgfo_spin_sector"][...], + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.rgfo_esa_step, cdf["rgfo_esa_step"][...] + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.nso_spin_sector, + cdf["nso_spin_sector"][...], + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.nso_esa_step, cdf["nso_esa_step"][...] + ) + np.testing.assert_array_equal( + actual_l1a_sw_priority_rates.esa_step, cdf["esa_step"][...] + ) def test_codice_lo_l1a_nsw_priority_read_from_cdf(self): - cdf_path = get_test_data_path("codice/imap_codice_l1a_lo-nsw-priority_20260307_v004.cdf") - actual_l1a_nsw_priority_rates = CodiceLoL1aNSWPriorityRates.read_from_cdf(cdf_path) + cdf_path = get_test_data_path( + "codice/imap_codice_l1a_lo-nsw-priority_20260307_v004.cdf" + ) + actual_l1a_nsw_priority_rates = CodiceLoL1aNSWPriorityRates.read_from_cdf( + cdf_path + ) with CDF(str(cdf_path)) as cdf: - np.testing.assert_array_equal(actual_l1a_nsw_priority_rates.epoch, cdf["epoch"][...]) - np.testing.assert_array_equal(actual_l1a_nsw_priority_rates.epoch_delta_plus, cdf["epoch_delta_plus"][...]) - np.testing.assert_array_equal(actual_l1a_nsw_priority_rates.epoch_delta_minus, - cdf["epoch_delta_minus"][...]) - np.testing.assert_array_equal(actual_l1a_nsw_priority_rates.acquisition_time_per_esa_step, - cdf["acquisition_time_per_esa_step"][...]) - np.testing.assert_array_equal(actual_l1a_nsw_priority_rates.spin_sector_index, - cdf["spin_sector"][...]) - np.testing.assert_array_equal(actual_l1a_nsw_priority_rates.rgfo_half_spin, cdf["rgfo_half_spin"][...]) - np.testing.assert_array_equal(actual_l1a_nsw_priority_rates.data_quality, cdf["data_quality"][...]) - np.testing.assert_array_equal(actual_l1a_nsw_priority_rates.p5_heavies, cdf["p5_heavies"][...]) - np.testing.assert_array_equal(actual_l1a_nsw_priority_rates.p6_hplus_heplusplus, - cdf["p6_hplus_heplusplus"][...]) - np.testing.assert_array_equal(actual_l1a_nsw_priority_rates.nso_half_spin, cdf["nso_half_spin"][...]) - np.testing.assert_array_equal(actual_l1a_nsw_priority_rates.sw_bias_gain_mode, - cdf["sw_bias_gain_mode"][...]) - np.testing.assert_array_equal(actual_l1a_nsw_priority_rates.st_bias_gain_mode, - cdf["st_bias_gain_mode"][...]) - np.testing.assert_array_equal(actual_l1a_nsw_priority_rates.spin_period, cdf["spin_period"][...]) + np.testing.assert_array_equal( + actual_l1a_nsw_priority_rates.epoch, cdf["epoch"][...] + ) + np.testing.assert_array_equal( + actual_l1a_nsw_priority_rates.epoch_delta_plus, + cdf["epoch_delta_plus"][...], + ) + np.testing.assert_array_equal( + actual_l1a_nsw_priority_rates.epoch_delta_minus, + cdf["epoch_delta_minus"][...], + ) + np.testing.assert_array_equal( + actual_l1a_nsw_priority_rates.acquisition_time_per_esa_step, + cdf["acquisition_time_per_esa_step"][...], + ) + np.testing.assert_array_equal( + actual_l1a_nsw_priority_rates.spin_sector_index, cdf["spin_sector"][...] + ) + np.testing.assert_array_equal( + actual_l1a_nsw_priority_rates.rgfo_half_spin, cdf["rgfo_half_spin"][...] + ) + np.testing.assert_array_equal( + actual_l1a_nsw_priority_rates.data_quality, cdf["data_quality"][...] + ) + np.testing.assert_array_equal( + actual_l1a_nsw_priority_rates.p5_heavies, cdf["p5_heavies"][...] + ) + np.testing.assert_array_equal( + actual_l1a_nsw_priority_rates.p6_hplus_heplusplus, + cdf["p6_hplus_heplusplus"][...], + ) + np.testing.assert_array_equal( + actual_l1a_nsw_priority_rates.nso_half_spin, cdf["nso_half_spin"][...] + ) + np.testing.assert_array_equal( + actual_l1a_nsw_priority_rates.sw_bias_gain_mode, + cdf["sw_bias_gain_mode"][...], + ) + np.testing.assert_array_equal( + actual_l1a_nsw_priority_rates.st_bias_gain_mode, + cdf["st_bias_gain_mode"][...], + ) + np.testing.assert_array_equal( + actual_l1a_nsw_priority_rates.spin_period, cdf["spin_period"][...] + ) def test_codice_lo_3d_distributions_data_product(self): species = "hplus" @@ -562,29 +825,71 @@ def test_codice_lo_3d_distributions_data_product(self): self.assertEqual(18, len(actual_data_product_variables)) actual_variables = iter(actual_data_product_variables) - self.assert_variable_attributes(next(actual_variables), sentinel.epoch, EPOCH_VAR_NAME) - self.assert_variable_attributes(next(actual_variables), sentinel.epoch_delta, EPOCH_DELTA_VAR_NAME) - self.assert_variable_attributes(next(actual_variables), elevation, ELEVATION_VAR_NAME) - self.assert_variable_attributes(next(actual_variables), sentinel.elevation_delta, ELEVATION_DELTA_VAR_NAME) - self.assert_variable_attributes(next(actual_variables), spin_angle, SPIN_ANGLE_VAR_NAME) - self.assert_variable_attributes(next(actual_variables), sentinel.spin_angle_delta, SPIN_ANGLE_DELTA_VAR_NAME) + self.assert_variable_attributes( + next(actual_variables), sentinel.epoch, EPOCH_VAR_NAME + ) + self.assert_variable_attributes( + next(actual_variables), sentinel.epoch_delta, EPOCH_DELTA_VAR_NAME + ) + self.assert_variable_attributes( + next(actual_variables), elevation, ELEVATION_VAR_NAME + ) + self.assert_variable_attributes( + next(actual_variables), sentinel.elevation_delta, ELEVATION_DELTA_VAR_NAME + ) + self.assert_variable_attributes( + next(actual_variables), spin_angle, SPIN_ANGLE_VAR_NAME + ) + self.assert_variable_attributes( + next(actual_variables), sentinel.spin_angle_delta, SPIN_ANGLE_DELTA_VAR_NAME + ) self.assert_variable_attributes(next(actual_variables), energy, ENERGY_VAR_NAME) - self.assert_variable_attributes(next(actual_variables), sentinel.energy_delta_plus, ENERGY_DELTA_PLUS_VAR_NAME) - self.assert_variable_attributes(next(actual_variables), sentinel.energy_delta_minus, - ENERGY_DELTA_MINUS_VAR_NAME) - self.assert_variable_attributes(next(actual_variables), sentinel.species_data, - species) - self.assert_variable_attributes(next(actual_variables), energy.astype(str), ENERGY_LABEL_VAR_NAME) - self.assert_variable_attributes(next(actual_variables), spin_angle.astype(str), SPIN_ANGLE_LABEL_VAR_NAME) - self.assert_variable_attributes(next(actual_variables), elevation.astype(str), ELEVATION_ANGLE_LABEL_VAR_NAME) - self.assert_variable_attributes(next(actual_variables), sentinel.uncertainties, species_uncertainty) - self.assert_variable_attributes(next(actual_variables), sentinel.rgfo_esa_step, RGFO_ESA_STEP_VAR_NAME) - self.assert_variable_attributes(next(actual_variables), sentinel.rgfo_spin_sector, RGFO_SPIN_SECTOR_VAR_NAME) - self.assert_variable_attributes(next(actual_variables), sentinel.rgfo_half_spin, RGFO_HALF_SPIN_VAR_NAME) - self.assert_variable_attributes(next(actual_variables), sentinel.half_spin_per_esa_step, HALF_SPIN_PER_ESA_STEP_VAR_NAME) + self.assert_variable_attributes( + next(actual_variables), + sentinel.energy_delta_plus, + ENERGY_DELTA_PLUS_VAR_NAME, + ) + self.assert_variable_attributes( + next(actual_variables), + sentinel.energy_delta_minus, + ENERGY_DELTA_MINUS_VAR_NAME, + ) + self.assert_variable_attributes( + next(actual_variables), sentinel.species_data, species + ) + self.assert_variable_attributes( + next(actual_variables), energy.astype(str), ENERGY_LABEL_VAR_NAME + ) + self.assert_variable_attributes( + next(actual_variables), spin_angle.astype(str), SPIN_ANGLE_LABEL_VAR_NAME + ) + self.assert_variable_attributes( + next(actual_variables), + elevation.astype(str), + ELEVATION_ANGLE_LABEL_VAR_NAME, + ) + self.assert_variable_attributes( + next(actual_variables), sentinel.uncertainties, species_uncertainty + ) + self.assert_variable_attributes( + next(actual_variables), sentinel.rgfo_esa_step, RGFO_ESA_STEP_VAR_NAME + ) + self.assert_variable_attributes( + next(actual_variables), sentinel.rgfo_spin_sector, RGFO_SPIN_SECTOR_VAR_NAME + ) + self.assert_variable_attributes( + next(actual_variables), sentinel.rgfo_half_spin, RGFO_HALF_SPIN_VAR_NAME + ) + self.assert_variable_attributes( + next(actual_variables), + sentinel.half_spin_per_esa_step, + HALF_SPIN_PER_ESA_STEP_VAR_NAME, + ) def test_codice_lo_l2_direct_events_reads_from_correct_float_data(self): - all_fill_l2_cdf_path = get_test_data_path('codice/imap_codice_l2_lo-direct-events_20260307_v003-all-fill.cdf') + all_fill_l2_cdf_path = get_test_data_path( + "codice/imap_codice_l2_lo-direct-events_20260307_v003-all-fill.cdf" + ) rng = np.random.default_rng() with tempfile.TemporaryDirectory() as tmpdir: @@ -604,16 +909,33 @@ def test_codice_lo_l2_direct_events_reads_from_correct_float_data(self): l2_direct_event = CodiceLoL2DirectEventData.read_from_cdf(test_cdf_path) with CDF(str(test_cdf_path)) as cdf: - np.testing.assert_array_equal(l2_direct_event.apd_energy, cdf["apd_energy"][:, :7, ...]) - np.testing.assert_array_equal(l2_direct_event.elevation_angle, cdf["elevation_angle"][:, :7, ...]) - np.testing.assert_array_equal(l2_direct_event.energy_step, cdf["energy_step"][:, :7, ...]) - np.testing.assert_array_equal(l2_direct_event.energy_per_charge, cdf["energy_per_charge"][:, :7, ...]) - np.testing.assert_array_equal(l2_direct_event.spin_sector, cdf["spin_sector"][:, :7, ...]) - np.testing.assert_array_equal(l2_direct_event.spin_angle, cdf["spin_angle"][:, :7, ...]) - np.testing.assert_array_equal(l2_direct_event.type, cdf["type"][:, :7, ...]) + np.testing.assert_array_equal( + l2_direct_event.apd_energy, cdf["apd_energy"][:, :7, ...] + ) + np.testing.assert_array_equal( + l2_direct_event.elevation_angle, cdf["elevation_angle"][:, :7, ...] + ) + np.testing.assert_array_equal( + l2_direct_event.energy_step, cdf["energy_step"][:, :7, ...] + ) + np.testing.assert_array_equal( + l2_direct_event.energy_per_charge, + cdf["energy_per_charge"][:, :7, ...], + ) + np.testing.assert_array_equal( + l2_direct_event.spin_sector, cdf["spin_sector"][:, :7, ...] + ) + np.testing.assert_array_equal( + l2_direct_event.spin_angle, cdf["spin_angle"][:, :7, ...] + ) + np.testing.assert_array_equal( + l2_direct_event.type, cdf["type"][:, :7, ...] + ) def test_workaround_to_convert_float_spin_sector_to_int(self): - all_fill_l2_cdf_path = get_test_data_path('codice/imap_codice_l2_lo-direct-events_20260307_v003-all-fill.cdf') + all_fill_l2_cdf_path = get_test_data_path( + "codice/imap_codice_l2_lo-direct-events_20260307_v003-all-fill.cdf" + ) rng = np.random.default_rng() with tempfile.TemporaryDirectory() as tmpdir: @@ -621,59 +943,99 @@ def test_workaround_to_convert_float_spin_sector_to_int(self): test_cdf_path = tmpdir / "test_cdf.cdf" with CDF(str(test_cdf_path), masterpath=str(all_fill_l2_cdf_path)) as cdf: - expected_spin_sector = rng.integers(0, 24, cdf["spin_sector"].shape, dtype=int) + expected_spin_sector = rng.integers( + 0, 24, cdf["spin_sector"].shape, dtype=int + ) cdf["spin_sector"] = expected_spin_sector.astype(float) l2_direct_event = CodiceLoL2DirectEventData.read_from_cdf(test_cdf_path) - np.testing.assert_array_equal(l2_direct_event.spin_sector, expected_spin_sector[:, :7, ...], strict=True) + np.testing.assert_array_equal( + l2_direct_event.spin_sector, + expected_spin_sector[:, :7, ...], + strict=True, + ) def test_codice_lo_l2_direct_events_read_from_cdf_handles_fill_value(self): - all_fill_l2_cdf_path = get_test_data_path('codice/imap_codice_l2_lo-direct-events_20260307_v003-all-fill.cdf') + all_fill_l2_cdf_path = get_test_data_path( + "codice/imap_codice_l2_lo-direct-events_20260307_v003-all-fill.cdf" + ) l2_direct_event = CodiceLoL2DirectEventData.read_from_cdf(all_fill_l2_cdf_path) with CDF(str(all_fill_l2_cdf_path)) as cdf: np.testing.assert_array_equal(cdf["epoch"], l2_direct_event.epoch) - np.testing.assert_array_equal(cdf["epoch_delta_plus"], l2_direct_event.epoch_delta_plus) - np.testing.assert_array_equal(cdf["epoch_delta_minus"], l2_direct_event.epoch_delta_minus) - - np.testing.assert_array_equal(l2_direct_event.apd_energy, np.full((2, 7, 10000), np.nan)) - np.testing.assert_array_equal(l2_direct_event.energy_step, np.full((2, 7, 10000), np.nan)) - np.testing.assert_array_equal(l2_direct_event.energy_per_charge, np.full((2, 7, 10000), np.nan)) - np.testing.assert_array_equal(l2_direct_event.spin_angle, np.full((2, 7, 10000), np.nan)) - np.testing.assert_array_equal(l2_direct_event.spin_sector, np.full((2, 7, 10000), np.nan)) - np.testing.assert_array_equal(l2_direct_event.elevation_angle, np.full((2, 7, 10000), np.nan)) + np.testing.assert_array_equal( + cdf["epoch_delta_plus"], l2_direct_event.epoch_delta_plus + ) + np.testing.assert_array_equal( + cdf["epoch_delta_minus"], l2_direct_event.epoch_delta_minus + ) + + np.testing.assert_array_equal( + l2_direct_event.apd_energy, np.full((2, 7, 10000), np.nan) + ) + np.testing.assert_array_equal( + l2_direct_event.energy_step, np.full((2, 7, 10000), np.nan) + ) + np.testing.assert_array_equal( + l2_direct_event.energy_per_charge, np.full((2, 7, 10000), np.nan) + ) + np.testing.assert_array_equal( + l2_direct_event.spin_angle, np.full((2, 7, 10000), np.nan) + ) + np.testing.assert_array_equal( + l2_direct_event.spin_sector, np.full((2, 7, 10000), np.nan) + ) + np.testing.assert_array_equal( + l2_direct_event.elevation_angle, np.full((2, 7, 10000), np.nan) + ) self.assertIsInstance(l2_direct_event.position, np.ma.masked_array) - np.testing.assert_array_equal(l2_direct_event.position.data, cdf["position"][:, :7, ...]) + np.testing.assert_array_equal( + l2_direct_event.position.data, cdf["position"][:, :7, ...] + ) self.assertTrue(np.all(l2_direct_event.position.mask)) self.assertIsInstance(l2_direct_event.gain, np.ma.masked_array) - np.testing.assert_array_equal(l2_direct_event.gain.data, cdf["gain"][:, :7, ...]) + np.testing.assert_array_equal( + l2_direct_event.gain.data, cdf["gain"][:, :7, ...] + ) self.assertTrue(np.all(l2_direct_event.gain.mask)) self.assertIsInstance(l2_direct_event.apd_id, np.ma.masked_array) - np.testing.assert_array_equal(l2_direct_event.apd_id.data, cdf["apd_id"][:, :7, ...]) + np.testing.assert_array_equal( + l2_direct_event.apd_id.data, cdf["apd_id"][:, :7, ...] + ) self.assertTrue(np.all(l2_direct_event.apd_id.mask)) self.assertIsInstance(l2_direct_event.data_quality, np.ma.masked_array) - np.testing.assert_array_equal(l2_direct_event.data_quality.data, cdf["data_quality"][:, :7, ...]) + np.testing.assert_array_equal( + l2_direct_event.data_quality.data, cdf["data_quality"][:, :7, ...] + ) self.assertTrue(np.all(l2_direct_event.data_quality.mask)) self.assertIsInstance(l2_direct_event.multi_flag, np.ma.masked_array) - np.testing.assert_array_equal(l2_direct_event.multi_flag.data, cdf["multi_flag"][:, :7, ...]) + np.testing.assert_array_equal( + l2_direct_event.multi_flag.data, cdf["multi_flag"][:, :7, ...] + ) self.assertTrue(np.all(l2_direct_event.multi_flag.mask)) self.assertIsInstance(l2_direct_event.num_events, np.ma.masked_array) - np.testing.assert_array_equal(l2_direct_event.num_events.data, cdf["num_events"][:, :7, ...]) + np.testing.assert_array_equal( + l2_direct_event.num_events.data, cdf["num_events"][:, :7, ...] + ) self.assertTrue(np.all(l2_direct_event.num_events.mask)) self.assertIsInstance(l2_direct_event.tof, np.ma.masked_array) - np.testing.assert_array_equal(l2_direct_event.tof.data, cdf["tof"][:, :7, ...]) + np.testing.assert_array_equal( + l2_direct_event.tof.data, cdf["tof"][:, :7, ...] + ) self.assertTrue(np.all(l2_direct_event.tof.mask)) def test_codice_lo_l2_direct_events_read_from_cdf_masks_negative_tof_values(self): - all_fill_l2_cdf_path = get_test_data_path('codice/imap_codice_l2_lo-direct-events_20260307_v003-all-fill.cdf') + all_fill_l2_cdf_path = get_test_data_path( + "codice/imap_codice_l2_lo-direct-events_20260307_v003-all-fill.cdf" + ) rng = np.random.default_rng() with tempfile.TemporaryDirectory() as tmpdir: @@ -690,15 +1052,21 @@ def test_codice_lo_l2_direct_events_read_from_cdf_masks_negative_tof_values(self expected_truncated_tof = tof[:, :CODICE_LO_L2_NUM_PRIORITIES, ...] expected_mask = expected_truncated_tof < 0 - np.testing.assert_array_equal(l2_direct_event.tof.data, expected_truncated_tof) + np.testing.assert_array_equal( + l2_direct_event.tof.data, expected_truncated_tof + ) np.testing.assert_array_equal(l2_direct_event.tof.mask, expected_mask) def test_codice_lo_l1a_sw_priority_read_from_cdf_handles_fill_value(self): - l1a_sw_all_fill_path = get_test_data_path("codice/imap_codice_l1a_lo-sw-priority_20260307_v003-all-fill.cdf") + l1a_sw_all_fill_path = get_test_data_path( + "codice/imap_codice_l1a_lo-sw-priority_20260307_v003-all-fill.cdf" + ) l1a_sw = CodiceLoL1aSWPriorityRates.read_from_cdf(l1a_sw_all_fill_path) with CDF(str(l1a_sw_all_fill_path)) as cdf: - np.testing.assert_array_equal(l1a_sw.spin_period, np.full_like(cdf['spin_period'], np.nan)) + np.testing.assert_array_equal( + l1a_sw.spin_period, np.full_like(cdf["spin_period"], np.nan) + ) integer_variable_names = [ "nso_half_spin", @@ -724,37 +1092,55 @@ def test_codice_lo_l1a_sw_priority_read_from_cdf_handles_fill_value(self): np.testing.assert_equal(var_data.mask, np.full_like(cdf[var], True)) def test_codice_lo_l1a_nsw_priority_read_from_cdf_handles_fill_value(self): - l1a_nsw_all_fill_path = get_test_data_path("codice/imap_codice_l1a_lo-nsw-priority_20260307_v003-all-fill.cdf") + l1a_nsw_all_fill_path = get_test_data_path( + "codice/imap_codice_l1a_lo-nsw-priority_20260307_v003-all-fill.cdf" + ) l1a_nsw_model = CodiceLoL1aNSWPriorityRates.read_from_cdf(l1a_nsw_all_fill_path) with CDF(str(l1a_nsw_all_fill_path)) as cdf: - np.testing.assert_array_equal(l1a_nsw_model.spin_period, np.full_like(cdf['spin_period'], np.nan)) + np.testing.assert_array_equal( + l1a_nsw_model.spin_period, np.full_like(cdf["spin_period"], np.nan) + ) self.assertIsInstance(l1a_nsw_model.rgfo_half_spin, np.ma.masked_array) - np.testing.assert_array_equal(l1a_nsw_model.rgfo_half_spin.data, cdf["rgfo_half_spin"]) + np.testing.assert_array_equal( + l1a_nsw_model.rgfo_half_spin.data, cdf["rgfo_half_spin"] + ) self.assertTrue(np.all(l1a_nsw_model.rgfo_half_spin.mask)) self.assertIsInstance(l1a_nsw_model.data_quality, np.ma.masked_array) - np.testing.assert_array_equal(l1a_nsw_model.data_quality.data, cdf["data_quality"]) + np.testing.assert_array_equal( + l1a_nsw_model.data_quality.data, cdf["data_quality"] + ) self.assertTrue(np.all(l1a_nsw_model.data_quality.mask)) self.assertIsInstance(l1a_nsw_model.p5_heavies, np.ma.masked_array) - np.testing.assert_array_equal(l1a_nsw_model.p5_heavies.data, cdf["p5_heavies"]) + np.testing.assert_array_equal( + l1a_nsw_model.p5_heavies.data, cdf["p5_heavies"] + ) self.assertTrue(np.all(l1a_nsw_model.p5_heavies.mask)) self.assertIsInstance(l1a_nsw_model.p6_hplus_heplusplus, np.ma.masked_array) - np.testing.assert_array_equal(l1a_nsw_model.p6_hplus_heplusplus.data, cdf["p6_hplus_heplusplus"]) + np.testing.assert_array_equal( + l1a_nsw_model.p6_hplus_heplusplus.data, cdf["p6_hplus_heplusplus"] + ) self.assertTrue(np.all(l1a_nsw_model.p6_hplus_heplusplus.mask)) self.assertIsInstance(l1a_nsw_model.nso_half_spin, np.ma.masked_array) - np.testing.assert_array_equal(l1a_nsw_model.nso_half_spin.data, cdf["nso_half_spin"]) + np.testing.assert_array_equal( + l1a_nsw_model.nso_half_spin.data, cdf["nso_half_spin"] + ) self.assertTrue(np.all(l1a_nsw_model.nso_half_spin.mask)) self.assertIsInstance(l1a_nsw_model.sw_bias_gain_mode, np.ma.masked_array) - np.testing.assert_array_equal(l1a_nsw_model.sw_bias_gain_mode.data, cdf["sw_bias_gain_mode"]) + np.testing.assert_array_equal( + l1a_nsw_model.sw_bias_gain_mode.data, cdf["sw_bias_gain_mode"] + ) self.assertTrue(np.all(l1a_nsw_model.sw_bias_gain_mode.mask)) self.assertIsInstance(l1a_nsw_model.st_bias_gain_mode, np.ma.masked_array) - np.testing.assert_array_equal(l1a_nsw_model.st_bias_gain_mode.data, cdf["st_bias_gain_mode"]) + np.testing.assert_array_equal( + l1a_nsw_model.st_bias_gain_mode.data, cdf["st_bias_gain_mode"] + ) self.assertTrue(np.all(l1a_nsw_model.st_bias_gain_mode.mask)) diff --git a/tests/debug_utils/profile_wrappers.py b/tests/debug_utils/profile_wrappers.py index 64ee9015a..b5a9591b1 100644 --- a/tests/debug_utils/profile_wrappers.py +++ b/tests/debug_utils/profile_wrappers.py @@ -9,7 +9,7 @@ def wrapper_cprofile(*args, **kwargs): with cProfile.Profile() as p: result = function_to_be_decorated(*args, **kwargs) print(f"========\n{function_to_be_decorated}") - p.dump_stats(f"profile_{datetime.now().strftime("%Y%m%dT%H%M%S")}.prof") + p.dump_stats(f"profile_{datetime.now().strftime('%Y%m%dT%H%M%S')}.prof") return result return wrapper_cprofile diff --git a/tests/glows/l3a/science/test_bad_angle_flag_configuration.py b/tests/glows/l3a/science/test_bad_angle_flag_configuration.py index b23113ae8..5ca4d6a14 100644 --- a/tests/glows/l3a/science/test_bad_angle_flag_configuration.py +++ b/tests/glows/l3a/science/test_bad_angle_flag_configuration.py @@ -6,7 +6,9 @@ import numpy as np -from imap_l3_processing.glows.l3a.science.bad_angle_flag_configuration import BadAngleFlagConfiguration +from imap_l3_processing.glows.l3a.science.bad_angle_flag_configuration import ( + BadAngleFlagConfiguration, +) from tests.test_helpers import get_test_data_path @@ -16,21 +18,27 @@ def test_masks_data_when_flags_match_configuration(self): excluded_region = [True, False, True, False, False] excluded_by_instr_team = [False, False, False, True, False] suspected_transient = [False, True, True, False, False] - test_data = np.array([ - close_to_uv_source, - excluded_region, - excluded_by_instr_team, - suspected_transient, - ]) - - test_cases = [([True, True, False, True], [True, True, True, True, False]), - ([True, False, False, False], close_to_uv_source), - ([False, True, False, False], excluded_region), - ([False, False, True, False], excluded_by_instr_team), - ([False, False, False, True], suspected_transient), - ([False, True, False, True], [a or b for a, b in zip(excluded_region, suspected_transient)]), - ([False, False, False, False], [False, False, False, False, False]), - ] + test_data = np.array( + [ + close_to_uv_source, + excluded_region, + excluded_by_instr_team, + suspected_transient, + ] + ) + + test_cases = [ + ([True, True, False, True], [True, True, True, True, False]), + ([True, False, False, False], close_to_uv_source), + ([False, True, False, False], excluded_region), + ([False, False, True, False], excluded_by_instr_team), + ([False, False, False, True], suspected_transient), + ( + [False, True, False, True], + [a or b for a, b in zip(excluded_region, suspected_transient)], + ), + ([False, False, False, False], [False, False, False, False, False]), + ] for flag_configuration, expected_mask in test_cases: with self.subTest(f"Flag configuration: {flag_configuration}"): @@ -40,7 +48,8 @@ def test_masks_data_when_flags_match_configuration(self): def test_load_from_file(self): path = get_test_data_path( - "glows/imap_glows_l2_histogram-bad-angle-flags-configuration-json-not-cdf_20250701_v001.cdf") + "glows/imap_glows_l2_histogram-bad-angle-flags-configuration-json-not-cdf_20250701_v001.cdf" + ) expected = BadAngleFlagConfiguration( mask_close_to_uv_source=True, mask_inside_excluded_region=True, @@ -58,6 +67,6 @@ def test_load_from_file_v2(self): mask_excluded_by_instr_team=True, mask_suspected_transient=False, ) - with open(temp_file, 'w') as f: + with open(temp_file, "w") as f: json.dump(dataclasses.asdict(expected), f) self.assertEqual(expected, BadAngleFlagConfiguration.from_file(temp_file)) diff --git a/tests/glows/l3a/science/test_calculate_daily_lightcurve.py b/tests/glows/l3a/science/test_calculate_daily_lightcurve.py index 5948da19d..83d9078c0 100644 --- a/tests/glows/l3a/science/test_calculate_daily_lightcurve.py +++ b/tests/glows/l3a/science/test_calculate_daily_lightcurve.py @@ -4,20 +4,30 @@ import numpy as np from uncertainties.unumpy import uarray, nominal_values, std_devs -from imap_l3_processing.glows.l3a.science.bad_angle_flag_configuration import BadAngleFlagConfiguration -from imap_l3_processing.glows.l3a.science.calculate_daily_lightcurve import rebin_lightcurve, calculate_spin_angles -from imap_l3_processing.glows.l3a.science.time_independent_background_lookup_table import \ - TimeIndependentBackgroundLookupTable +from imap_l3_processing.glows.l3a.science.bad_angle_flag_configuration import ( + BadAngleFlagConfiguration, +) +from imap_l3_processing.glows.l3a.science.calculate_daily_lightcurve import ( + rebin_lightcurve, + calculate_spin_angles, +) +from imap_l3_processing.glows.l3a.science.time_independent_background_lookup_table import ( + TimeIndependentBackgroundLookupTable, +) class TestCalculateDailyLightcurve(unittest.TestCase): def setUp(self): - self.all_zero_time_independent_background = TimeIndependentBackgroundLookupTable( - latitudes=np.array([-90, 90]), - longitudes=np.array([0]), - background_values=np.zeros( - shape=(2, 1))) - self.enable_all_flags_configuration = BadAngleFlagConfiguration(True, True, True, True) + self.all_zero_time_independent_background = ( + TimeIndependentBackgroundLookupTable( + latitudes=np.array([-90, 90]), + longitudes=np.array([0]), + background_values=np.zeros(shape=(2, 1)), + ) + ) + self.enable_all_flags_configuration = BadAngleFlagConfiguration( + True, True, True, True + ) def test_rebin_with_no_flags_and_equal_exposure_times(self): photon_flux = np.full(3600, 1.0) @@ -30,10 +40,17 @@ def test_rebin_with_no_flags_and_equal_exposure_times(self): exposure_times = np.full(3600, 24.0) output_size = 90 background = np.zeros(90) - result, rebinned_exposure = rebin_lightcurve(self.all_zero_time_independent_background, - self.enable_all_flags_configuration, - photon_flux, latitudes, longitudes, flags, exposure_times, - output_size, background) + result, rebinned_exposure = rebin_lightcurve( + self.all_zero_time_independent_background, + self.enable_all_flags_configuration, + photon_flux, + latitudes, + longitudes, + flags, + exposure_times, + output_size, + background, + ) expected_output = np.full(90, 1.0) expected_output[45:] = 5.0 @@ -51,10 +68,17 @@ def test_rebin_with_different_output_size(self): latitudes = np.linspace(start=0, stop=0, num=2400) longitudes = np.linspace(start=0, stop=360, num=2400) - result, exposure = rebin_lightcurve(self.all_zero_time_independent_background, - self.enable_all_flags_configuration, photon_flux, latitudes, - longitudes, flags, - exposure_times, 80, background) + result, exposure = rebin_lightcurve( + self.all_zero_time_independent_background, + self.enable_all_flags_configuration, + photon_flux, + latitudes, + longitudes, + flags, + exposure_times, + 80, + background, + ) expected_output = np.full(80, 1.0) expected_exposure = np.full(80, 300.0) np.testing.assert_equal(result, expected_output, strict=True) @@ -77,10 +101,17 @@ def test_rebin_masking_flagged_bins(self): latitudes = np.linspace(start=0, stop=0, num=3600) longitudes = np.linspace(start=0, stop=360, num=3600) - result, exposure = rebin_lightcurve(self.all_zero_time_independent_background, - self.enable_all_flags_configuration, photon_flux, latitudes, - longitudes, flags, - exposure_times, output_size, background) + result, exposure = rebin_lightcurve( + self.all_zero_time_independent_background, + self.enable_all_flags_configuration, + photon_flux, + latitudes, + longitudes, + flags, + exposure_times, + output_size, + background, + ) expected_output = np.full(90, 1.0) expected_output[45:] = 5.0 @@ -113,14 +144,23 @@ def test_uses_configuration_to_flag_bins(self): latitudes = np.linspace(start=0, stop=0, num=3600) longitudes = np.linspace(start=0, stop=360, num=3600) - flag_configuration = BadAngleFlagConfiguration(mask_close_to_uv_source=True, - mask_inside_excluded_region=True, - mask_excluded_by_instr_team=False, - mask_suspected_transient=True, ) - result, exposure = rebin_lightcurve(self.all_zero_time_independent_background, flag_configuration, photon_flux, - latitudes, - longitudes, flags, - exposure_times, output_size, background) + flag_configuration = BadAngleFlagConfiguration( + mask_close_to_uv_source=True, + mask_inside_excluded_region=True, + mask_excluded_by_instr_team=False, + mask_suspected_transient=True, + ) + result, exposure = rebin_lightcurve( + self.all_zero_time_independent_background, + flag_configuration, + photon_flux, + latitudes, + longitudes, + flags, + exposure_times, + output_size, + background, + ) expected_output = np.full(90, 1.0) expected_output[45:] = 5.0 @@ -136,110 +176,156 @@ def test_uses_configuration_to_flag_bins(self): self.assertFalse(np.ma.isMaskedArray(exposure)) def test_rebin_uses_exposure_times(self): - photon_flux = np.array([10., 20., 30., 40.]) + photon_flux = np.array([10.0, 20.0, 30.0, 40.0]) flags = np.zeros((4, 4)) - exposure_times = np.array([2., 1., 0., 1.]) + exposure_times = np.array([2.0, 1.0, 0.0, 1.0]) background = np.zeros(1) latitudes = np.linspace(start=0, stop=0, num=4) longitudes = np.linspace(start=0, stop=360, num=4) - result, exposure = rebin_lightcurve(self.all_zero_time_independent_background, - self.enable_all_flags_configuration, photon_flux, latitudes, - longitudes, flags, - exposure_times, 1, background) + result, exposure = rebin_lightcurve( + self.all_zero_time_independent_background, + self.enable_all_flags_configuration, + photon_flux, + latitudes, + longitudes, + flags, + exposure_times, + 1, + background, + ) np.testing.assert_equal(result, [20.0], strict=True) np.testing.assert_equal(exposure, [4.0], strict=True) def test_rebin_uses_exposure_times_all_zero_bin(self): - photon_flux = np.array([10., 20., 30., 40.]) + photon_flux = np.array([10.0, 20.0, 30.0, 40.0]) flags = np.zeros((4, 4)) - exposure_times = np.array([0., 0., 0., 0.]) + exposure_times = np.array([0.0, 0.0, 0.0, 0.0]) background = np.zeros(1) latitudes = np.linspace(start=0, stop=0, num=4) longitudes = np.linspace(start=0, stop=360, num=4) - result, exposure = rebin_lightcurve(self.all_zero_time_independent_background, - self.enable_all_flags_configuration, photon_flux, latitudes, - longitudes, flags, - exposure_times, 1, background) + result, exposure = rebin_lightcurve( + self.all_zero_time_independent_background, + self.enable_all_flags_configuration, + photon_flux, + latitudes, + longitudes, + flags, + exposure_times, + 1, + background, + ) np.testing.assert_equal(result, [np.nan], strict=True) np.testing.assert_equal(exposure, [0.0], strict=True) def test_rebin_uses_exposure_times_and_background(self): - photon_flux = np.array([5., 20., 30., 40.]) + photon_flux = np.array([5.0, 20.0, 30.0, 40.0]) flags = np.zeros((4, 4)) - exposure_times = np.array([2., 1., 0., 1.]) + exposure_times = np.array([2.0, 1.0, 0.0, 1.0]) background = np.array([5, 4]) latitudes = np.linspace(start=0, stop=0, num=4) longitudes = np.linspace(start=0, stop=360, num=4) - result, exposure = rebin_lightcurve(self.all_zero_time_independent_background, - self.enable_all_flags_configuration, photon_flux, latitudes, - longitudes, flags, - exposure_times, 2, background) + result, exposure = rebin_lightcurve( + self.all_zero_time_independent_background, + self.enable_all_flags_configuration, + photon_flux, + latitudes, + longitudes, + flags, + exposure_times, + 2, + background, + ) np.testing.assert_equal(result, [5.0, 36.0], strict=True) np.testing.assert_equal(exposure, [3.0, 1.0], strict=True) def test_rebin_uses_uncertain_photon_flux_and_background(self): photon_flux = np.array([5, 20, 30, 40]) - photon_flux_uncertainty = np.array([0.1, 2.0, 3.0, .5]) + photon_flux_uncertainty = np.array([0.1, 2.0, 3.0, 0.5]) photon_flux = uarray(photon_flux, photon_flux_uncertainty) latitudes = np.linspace(start=0, stop=0, num=4) longitudes = np.linspace(start=0, stop=360, num=4) flags = np.zeros((4, 4)) - exposure_times = np.array([2., 1., 0., 1.]) + exposure_times = np.array([2.0, 1.0, 0.0, 1.0]) background = np.array([5, 4]) background_uncertainty = np.array([0.02, 0.08]) background = uarray(background, background_uncertainty) - result, exposure = rebin_lightcurve(self.all_zero_time_independent_background, - self.enable_all_flags_configuration, photon_flux, latitudes, - longitudes, flags, - exposure_times, 2, background) + result, exposure = rebin_lightcurve( + self.all_zero_time_independent_background, + self.enable_all_flags_configuration, + photon_flux, + latitudes, + longitudes, + flags, + exposure_times, + 2, + background, + ) np.testing.assert_equal(nominal_values(result), [5.0, 36.0], strict=True) - np.testing.assert_equal(std_devs(result), [0.6702901527613909, 0.5063595560468865], strict=True) + np.testing.assert_equal( + std_devs(result), [0.6702901527613909, 0.5063595560468865], strict=True + ) np.testing.assert_equal(exposure, [3.0, 1.0], strict=True) def test_rebin_subtracts_time_independent_background(self): time_independent_background = np.array([20, 2, 1, 0.2]) photon_flux = np.array([5, 20, 30, 40]) + time_independent_background - photon_flux_uncertainty = np.array([0.1, 2.0, 3.0, .5]) + photon_flux_uncertainty = np.array([0.1, 2.0, 3.0, 0.5]) photon_flux = uarray(photon_flux, photon_flux_uncertainty) latitudes = np.linspace(start=0, stop=0, num=4) longitudes = np.linspace(start=0, stop=360, num=4) flags = np.zeros((4, 4)) - exposure_times = np.array([2., 1., 0., 1.]) + exposure_times = np.array([2.0, 1.0, 0.0, 1.0]) background = np.array([5, 4]) background_uncertainty = np.array([0.02, 0.08]) background = uarray(background, background_uncertainty) mock_time_independent_background_lookup_table = Mock() - mock_time_independent_background_lookup_table.lookup.return_value = time_independent_background - - result, exposure = rebin_lightcurve(mock_time_independent_background_lookup_table, - self.enable_all_flags_configuration, photon_flux, latitudes, - longitudes, flags, - exposure_times, 2, background) - - mock_time_independent_background_lookup_table.lookup.assert_called_once_with(lat=latitudes, - lon=longitudes) + mock_time_independent_background_lookup_table.lookup.return_value = ( + time_independent_background + ) + + result, exposure = rebin_lightcurve( + mock_time_independent_background_lookup_table, + self.enable_all_flags_configuration, + photon_flux, + latitudes, + longitudes, + flags, + exposure_times, + 2, + background, + ) + + mock_time_independent_background_lookup_table.lookup.assert_called_once_with( + lat=latitudes, lon=longitudes + ) np.testing.assert_equal(nominal_values(result), [5.0, 36.0], strict=True) - np.testing.assert_equal(std_devs(result), [0.6702901527613909, 0.5063595560468865], strict=True) + np.testing.assert_equal( + std_devs(result), [0.6702901527613909, 0.5063595560468865], strict=True + ) np.testing.assert_equal(exposure, [3.0, 1.0], strict=True) def test_calculate_spin_angles(self): spin_angles = np.array([70, 130, 190, 250, 310, 10, 70, 130, 190, 350, 10, 30]) - np.testing.assert_array_equal([130, 310, 130, 10], calculate_spin_angles(4, spin_angles)) + np.testing.assert_array_equal( + [130, 310, 130, 10], calculate_spin_angles(4, spin_angles) + ) realistic_angles = np.linspace(0, 360, 3600, endpoint=False) + 0.05 expected_4_degree_bins = np.linspace(0, 360, 90, endpoint=False) + 2 - np.testing.assert_allclose(expected_4_degree_bins, calculate_spin_angles(90, realistic_angles)) + np.testing.assert_allclose( + expected_4_degree_bins, calculate_spin_angles(90, realistic_angles) + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/glows/l3a/science/test_time_independent_background_lookup_table.py b/tests/glows/l3a/science/test_time_independent_background_lookup_table.py index 008bc9d0b..fcd650acb 100644 --- a/tests/glows/l3a/science/test_time_independent_background_lookup_table.py +++ b/tests/glows/l3a/science/test_time_independent_background_lookup_table.py @@ -2,8 +2,9 @@ import numpy as np -from imap_l3_processing.glows.l3a.science.time_independent_background_lookup_table import \ - TimeIndependentBackgroundLookupTable +from imap_l3_processing.glows.l3a.science.time_independent_background_lookup_table import ( + TimeIndependentBackgroundLookupTable, +) from tests.test_helpers import get_test_data_path @@ -11,11 +12,13 @@ class TestTimeIndependentBackgroundLookupTable(unittest.TestCase): def test_interpolates_to_find_background(self): latitudes = np.array([90, 0, -90]) longitudes = np.array([0, 90, 180, 270]) - bg_map = np.array([ - [4, 4, 4, 4], - [5, 6, 7, 8], - [9, 9, 9, 9], - ]) + bg_map = np.array( + [ + [4, 4, 4, 4], + [5, 6, 7, 8], + [9, 9, 9, 9], + ] + ) table = TimeIndependentBackgroundLookupTable(longitudes, latitudes, bg_map) self.assertEqual(5, table.lookup(lat=0, lon=0)) @@ -28,16 +31,23 @@ def test_interpolates_to_find_background(self): self.assertEqual(4.75, table.lookup(lat=45, lon=45)) - np.testing.assert_equal([5, 5.5, 4.75], table.lookup(lat=np.array([0, 0, 45]), lon=np.array([0, 45, 45]))) + np.testing.assert_equal( + [5, 5.5, 4.75], + table.lookup(lat=np.array([0, 0, 45]), lon=np.array([0, 45, 45])), + ) def test_load_from_file(self): table = TimeIndependentBackgroundLookupTable.from_file( get_test_data_path( - "glows/imap_glows_l2_histogram-time-independent-background-map-text-not-cdf_20250701_v001.cdf")) + "glows/imap_glows_l2_histogram-time-independent-background-map-text-not-cdf_20250701_v001.cdf" + ) + ) expected_latitudes = np.array([90, 0, -90]) expected_longitudes = np.array([0, 90, 180, 270]) np.testing.assert_equal(expected_latitudes, table.latitudes) np.testing.assert_equal(expected_longitudes, table.longitudes) - np.testing.assert_array_equal([[0, 4, 3, 1], [1, 1, 3, 2], [5, 3, 2, 0]], table.background_values) + np.testing.assert_array_equal( + [[0, 4, 3, 1], [1, 1, 3, 2], [5, 3, 2, 0]], table.background_values + ) diff --git a/tests/glows/l3a/test_glows_l3a_dependencies.py b/tests/glows/l3a/test_glows_l3a_dependencies.py index ca6a198ce..238cbf30d 100644 --- a/tests/glows/l3a/test_glows_l3a_dependencies.py +++ b/tests/glows/l3a/test_glows_l3a_dependencies.py @@ -3,20 +3,29 @@ from unittest.mock import patch, call import imap_data_access -from imap_data_access.processing_input import ScienceInput, AncillaryInput, ProcessingInputCollection +from imap_data_access.processing_input import ( + ScienceInput, + AncillaryInput, + ProcessingInputCollection, +) -from imap_l3_processing.glows.descriptors import GLOWS_L2_DESCRIPTOR, GLOWS_CALIBRATION_DATA_DESCRIPTOR, \ - GLOWS_PIPELINE_SETTINGS_DESCRIPTOR, GLOWS_EXTRA_HELIOSPHERIC_BACKGROUND_DESCRIPTOR, \ - GLOWS_TIME_DEPENDENT_BACKGROUND_DESCRIPTOR +from imap_l3_processing.glows.descriptors import ( + GLOWS_L2_DESCRIPTOR, + GLOWS_CALIBRATION_DATA_DESCRIPTOR, + GLOWS_PIPELINE_SETTINGS_DESCRIPTOR, + GLOWS_EXTRA_HELIOSPHERIC_BACKGROUND_DESCRIPTOR, + GLOWS_TIME_DEPENDENT_BACKGROUND_DESCRIPTOR, +) from imap_l3_processing.glows.l3a.glows_l3a_dependencies import GlowsL3ADependencies class TestGlowsL3aDependencies(unittest.TestCase): - - @patch('imap_l3_processing.glows.l3a.glows_l3a_dependencies.CDF') - @patch('imap_l3_processing.glows.l3a.glows_l3a_dependencies.download') - @patch('imap_l3_processing.glows.l3a.glows_l3a_dependencies.read_l2_glows_data') - def test_fetch_dependencies(self, mock_read_l2_glows_data, mock_download, mock_cdf_constructor): + @patch("imap_l3_processing.glows.l3a.glows_l3a_dependencies.CDF") + @patch("imap_l3_processing.glows.l3a.glows_l3a_dependencies.download") + @patch("imap_l3_processing.glows.l3a.glows_l3a_dependencies.read_l2_glows_data") + def test_fetch_dependencies( + self, mock_read_l2_glows_data, mock_download, mock_cdf_constructor + ): mission = "imap" instrument = "glows" level = "l2a" @@ -24,12 +33,14 @@ def test_fetch_dependencies(self, mock_read_l2_glows_data, mock_download, mock_c version = "v001" repointing = "repoint00002" - ignored_dependency_filename = f'{mission}_{instrument}_{level}_ignored-data_{start_date}_{version}.cdf' - cdf_dependency_filename = f'{mission}_{instrument}_{level}_{GLOWS_L2_DESCRIPTOR}_{start_date}-{repointing}_{version}.cdf' - calibration_data_dependency_filename = f'{mission}_{instrument}_{GLOWS_CALIBRATION_DATA_DESCRIPTOR}_{start_date}_{version}.dat' - settings_dependency_filename = f'{mission}_{instrument}_{GLOWS_PIPELINE_SETTINGS_DESCRIPTOR}_{start_date}_{version}.json' - extra_heliospheric_background_dependency_filename = f'{mission}_{instrument}_{GLOWS_EXTRA_HELIOSPHERIC_BACKGROUND_DESCRIPTOR}_{start_date}_{version}.dat' - time_dependent_background_dependency_filename = f'{mission}_{instrument}_{GLOWS_TIME_DEPENDENT_BACKGROUND_DESCRIPTOR}_{start_date}_{version}.dat' + ignored_dependency_filename = ( + f"{mission}_{instrument}_{level}_ignored-data_{start_date}_{version}.cdf" + ) + cdf_dependency_filename = f"{mission}_{instrument}_{level}_{GLOWS_L2_DESCRIPTOR}_{start_date}-{repointing}_{version}.cdf" + calibration_data_dependency_filename = f"{mission}_{instrument}_{GLOWS_CALIBRATION_DATA_DESCRIPTOR}_{start_date}_{version}.dat" + settings_dependency_filename = f"{mission}_{instrument}_{GLOWS_PIPELINE_SETTINGS_DESCRIPTOR}_{start_date}_{version}.json" + extra_heliospheric_background_dependency_filename = f"{mission}_{instrument}_{GLOWS_EXTRA_HELIOSPHERIC_BACKGROUND_DESCRIPTOR}_{start_date}_{version}.dat" + time_dependent_background_dependency_filename = f"{mission}_{instrument}_{GLOWS_TIME_DEPENDENT_BACKGROUND_DESCRIPTOR}_{start_date}_{version}.dat" processing_input_collection = ProcessingInputCollection( ScienceInput(ignored_dependency_filename), @@ -39,8 +50,17 @@ def test_fetch_dependencies(self, mock_read_l2_glows_data, mock_download, mock_c AncillaryInput(extra_heliospheric_background_dependency_filename), AncillaryInput(time_dependent_background_dependency_filename), ) - science_data_dir = imap_data_access.config['DATA_DIR'] / 'imap' / 'glows' / 'l2a' / '2023' / '01' - ancillary_data_dir = imap_data_access.config['DATA_DIR'] / 'imap' / 'ancillary' / 'glows' + science_data_dir = ( + imap_data_access.config["DATA_DIR"] + / "imap" + / "glows" + / "l2a" + / "2023" + / "01" + ) + ancillary_data_dir = ( + imap_data_access.config["DATA_DIR"] / "imap" / "ancillary" / "glows" + ) cdf_path_str = "some_cdf.cdf" cdf_path_name = Path(cdf_path_str) @@ -66,15 +86,27 @@ def test_fetch_dependencies(self, mock_read_l2_glows_data, mock_download, mock_c self.assertEqual(calibration_data, result.ancillary_files["calibration_data"]) self.assertEqual(settings, result.ancillary_files["settings"]) - self.assertEqual(time_dependent_background_path, - result.ancillary_files["time_dependent_bckgrd"]) - self.assertEqual(extra_heliospheric_background, - result.ancillary_files["extra_heliospheric_bckgrd"]) + self.assertEqual( + time_dependent_background_path, + result.ancillary_files["time_dependent_bckgrd"], + ) + self.assertEqual( + extra_heliospheric_background, + result.ancillary_files["extra_heliospheric_bckgrd"], + ) - self.assertEqual([ - call(science_data_dir / cdf_dependency_filename), - call(ancillary_data_dir / calibration_data_dependency_filename), - call(ancillary_data_dir / extra_heliospheric_background_dependency_filename), - call(ancillary_data_dir / time_dependent_background_dependency_filename), - call(ancillary_data_dir / settings_dependency_filename), - ], mock_download.call_args_list) + self.assertEqual( + [ + call(science_data_dir / cdf_dependency_filename), + call(ancillary_data_dir / calibration_data_dependency_filename), + call( + ancillary_data_dir + / extra_heliospheric_background_dependency_filename + ), + call( + ancillary_data_dir / time_dependent_background_dependency_filename + ), + call(ancillary_data_dir / settings_dependency_filename), + ], + mock_download.call_args_list, + ) diff --git a/tests/glows/l3a/test_models.py b/tests/glows/l3a/test_models.py index 0538e6c3a..b5a684e8f 100644 --- a/tests/glows/l3a/test_models.py +++ b/tests/glows/l3a/test_models.py @@ -5,27 +5,50 @@ import numpy as np from numpy import ndarray -from imap_l3_processing.glows.l3a.models import GlowsL3LightCurve, PHOTON_FLUX_CDF_VAR_NAME, \ - EXPOSURE_TIMES_CDF_VAR_NAME, \ - NUM_OF_BINS_CDF_VAR_NAME, EPOCH_CDF_VAR_NAME, EPOCH_DELTA_CDF_VAR_NAME, \ - PHOTON_FLUX_UNCERTAINTY_CDF_VAR_NAME, SPIN_ANGLE_CDF_VAR_NAME, LATITUDE_CDF_VAR_NAME, LONGITUDE_CDF_VAR_NAME, \ - EXTRA_HELIOSPHERIC_BACKGROUND_CDF_VAR_NAME, TIME_DEPENDENT_BACKGROUND_CDF_VAR_NAME, \ - FILTER_TEMPERATURE_AVERAGE_CDF_VAR_NAME, FILTER_TEMPERATURE_STD_DEV_CDF_VAR_NAME, HV_VOLTAGE_AVERAGE_CDF_VAR_NAME, \ - HV_VOLTAGE_STD_DEV_CDF_VAR_NAME, SPIN_PERIOD_AVERAGE_CDF_VAR_NAME, SPIN_PERIOD_STD_DEV_CDF_VAR_NAME, \ - SPIN_PERIOD_GROUND_AVERAGE_CDF_VAR_NAME, SPIN_PERIOD_GROUND_STD_DEV_CDF_VAR_NAME, PULSE_LENGTH_AVERAGE_CDF_VAR_NAME, \ - PULSE_LENGTH_STD_DEV_CDF_VAR_NAME, POSITION_ANGLE_OFFSET_AVERAGE_CDF_VAR_NAME, \ - POSITION_ANGLE_OFFSET_STD_DEV_CDF_VAR_NAME, SPIN_AXIS_ORIENTATION_AVERAGE_CDF_VAR_NAME, \ - SPIN_AXIS_ORIENTATION_STD_DEV_CDF_VAR_NAME, SPACECRAFT_LOCATION_AVERAGE_CDF_VAR_NAME, \ - SPACECRAFT_LOCATION_STD_DEV_CDF_VAR_NAME, SPACECRAFT_VELOCITY_AVERAGE_CDF_VAR_NAME, \ - SPACECRAFT_VELOCITY_STD_DEV_CDF_VAR_NAME, RAW_HISTOGRAM_CDF_VAR_NAME, SPIN_ANGLE_DELTA_CDF_VAR_NAME, \ - START_TIME_CDF_VAR_NAME, END_TIME_CDF_VAR_NAME, REPOINTING_CDF_VAR_NAME +from imap_l3_processing.glows.l3a.models import ( + GlowsL3LightCurve, + PHOTON_FLUX_CDF_VAR_NAME, + EXPOSURE_TIMES_CDF_VAR_NAME, + NUM_OF_BINS_CDF_VAR_NAME, + EPOCH_CDF_VAR_NAME, + EPOCH_DELTA_CDF_VAR_NAME, + PHOTON_FLUX_UNCERTAINTY_CDF_VAR_NAME, + SPIN_ANGLE_CDF_VAR_NAME, + LATITUDE_CDF_VAR_NAME, + LONGITUDE_CDF_VAR_NAME, + EXTRA_HELIOSPHERIC_BACKGROUND_CDF_VAR_NAME, + TIME_DEPENDENT_BACKGROUND_CDF_VAR_NAME, + FILTER_TEMPERATURE_AVERAGE_CDF_VAR_NAME, + FILTER_TEMPERATURE_STD_DEV_CDF_VAR_NAME, + HV_VOLTAGE_AVERAGE_CDF_VAR_NAME, + HV_VOLTAGE_STD_DEV_CDF_VAR_NAME, + SPIN_PERIOD_AVERAGE_CDF_VAR_NAME, + SPIN_PERIOD_STD_DEV_CDF_VAR_NAME, + SPIN_PERIOD_GROUND_AVERAGE_CDF_VAR_NAME, + SPIN_PERIOD_GROUND_STD_DEV_CDF_VAR_NAME, + PULSE_LENGTH_AVERAGE_CDF_VAR_NAME, + PULSE_LENGTH_STD_DEV_CDF_VAR_NAME, + POSITION_ANGLE_OFFSET_AVERAGE_CDF_VAR_NAME, + POSITION_ANGLE_OFFSET_STD_DEV_CDF_VAR_NAME, + SPIN_AXIS_ORIENTATION_AVERAGE_CDF_VAR_NAME, + SPIN_AXIS_ORIENTATION_STD_DEV_CDF_VAR_NAME, + SPACECRAFT_LOCATION_AVERAGE_CDF_VAR_NAME, + SPACECRAFT_LOCATION_STD_DEV_CDF_VAR_NAME, + SPACECRAFT_VELOCITY_AVERAGE_CDF_VAR_NAME, + SPACECRAFT_VELOCITY_STD_DEV_CDF_VAR_NAME, + RAW_HISTOGRAM_CDF_VAR_NAME, + SPIN_ANGLE_DELTA_CDF_VAR_NAME, + START_TIME_CDF_VAR_NAME, + END_TIME_CDF_VAR_NAME, + REPOINTING_CDF_VAR_NAME, +) from tests.swapi.cdf_model_test_case import CdfModelTestCase class TestModels(CdfModelTestCase): def test_data_to_product_variables(self): photon_flux: ndarray = (np.arange(360) + 50).reshape(1, -1) - photon_flux_uncertainty = photon_flux / 100. + photon_flux_uncertainty = photon_flux / 100.0 raw_histogram = Mock() exposure_times: ndarray = (np.arange(360) + 100).reshape(1, -1) epoch: ndarray = np.array(datetime(2024, 11, 18, 12)) @@ -61,100 +84,190 @@ def test_data_to_product_variables(self): spacecraft_velocity_std_dev = Mock() repointing = 11 - data = GlowsL3LightCurve(input_metadata=Mock(), - identifier=repointing, - parent_file_names=Mock(), - exposure_times=exposure_times, - raw_histogram=raw_histogram, - number_of_bins=number_of_bins, - photon_flux=photon_flux, - epoch=epoch, epoch_delta=epoch_delta, - start_time=start_time, end_time=end_time, - spin_angle=spin_angle, spin_angle_delta=spin_angle_delta, - photon_flux_uncertainty=photon_flux_uncertainty, - latitude=latitudes, - longitude=longitudes, - extra_heliospheric_background=extra_heliospheric_background, - time_dependent_background=time_dependent_background, - filter_temperature_average=filter_temperature_average, - filter_temperature_std_dev=filter_temperature_std_dev, - hv_voltage_average=hv_voltage_average, - hv_voltage_std_dev=hv_voltage_std_dev, - spin_period_average=spin_period_average, - spin_period_std_dev=spin_period_std_dev, - spin_period_ground_average=spin_period_ground_average, - spin_period_ground_std_dev=spin_period_ground_std_dev, - pulse_length_average=pulse_length_average, - pulse_length_std_dev=pulse_length_std_dev, - position_angle_offset_average=position_angle_offset_average, - position_angle_offset_std_dev=position_angle_offset_std_dev, - spin_axis_orientation_average=spin_axis_orientation_average, - spin_axis_orientation_std_dev=spin_axis_orientation_std_dev, - spacecraft_location_average=spacecraft_location_average, - spacecraft_location_std_dev=spacecraft_location_std_dev, - spacecraft_velocity_average=spacecraft_velocity_average, - spacecraft_velocity_std_dev=spacecraft_velocity_std_dev, - ) + data = GlowsL3LightCurve( + input_metadata=Mock(), + identifier=repointing, + parent_file_names=Mock(), + exposure_times=exposure_times, + raw_histogram=raw_histogram, + number_of_bins=number_of_bins, + photon_flux=photon_flux, + epoch=epoch, + epoch_delta=epoch_delta, + start_time=start_time, + end_time=end_time, + spin_angle=spin_angle, + spin_angle_delta=spin_angle_delta, + photon_flux_uncertainty=photon_flux_uncertainty, + latitude=latitudes, + longitude=longitudes, + extra_heliospheric_background=extra_heliospheric_background, + time_dependent_background=time_dependent_background, + filter_temperature_average=filter_temperature_average, + filter_temperature_std_dev=filter_temperature_std_dev, + hv_voltage_average=hv_voltage_average, + hv_voltage_std_dev=hv_voltage_std_dev, + spin_period_average=spin_period_average, + spin_period_std_dev=spin_period_std_dev, + spin_period_ground_average=spin_period_ground_average, + spin_period_ground_std_dev=spin_period_ground_std_dev, + pulse_length_average=pulse_length_average, + pulse_length_std_dev=pulse_length_std_dev, + position_angle_offset_average=position_angle_offset_average, + position_angle_offset_std_dev=position_angle_offset_std_dev, + spin_axis_orientation_average=spin_axis_orientation_average, + spin_axis_orientation_std_dev=spin_axis_orientation_std_dev, + spacecraft_location_average=spacecraft_location_average, + spacecraft_location_std_dev=spacecraft_location_std_dev, + spacecraft_velocity_average=spacecraft_velocity_average, + spacecraft_velocity_std_dev=spacecraft_velocity_std_dev, + ) variables = data.to_data_product_variables() self.assertEqual(38, len(variables)) variables = iter(variables) # @formatter:off - self.assert_variable_attributes(next(variables), [repointing], REPOINTING_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), photon_flux, PHOTON_FLUX_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), photon_flux_uncertainty, PHOTON_FLUX_UNCERTAINTY_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), raw_histogram, RAW_HISTOGRAM_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), exposure_times, EXPOSURE_TIMES_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), number_of_bins, NUM_OF_BINS_CDF_VAR_NAME) + self.assert_variable_attributes( + next(variables), [repointing], REPOINTING_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), photon_flux, PHOTON_FLUX_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), + photon_flux_uncertainty, + PHOTON_FLUX_UNCERTAINTY_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), raw_histogram, RAW_HISTOGRAM_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), exposure_times, EXPOSURE_TIMES_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), number_of_bins, NUM_OF_BINS_CDF_VAR_NAME + ) self.assert_variable_attributes(next(variables), epoch, EPOCH_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), epoch_delta, EPOCH_DELTA_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), [start_time], START_TIME_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), [end_time], END_TIME_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), spin_angle, SPIN_ANGLE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), spin_angle_delta, SPIN_ANGLE_DELTA_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), latitudes, LATITUDE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), longitudes, LONGITUDE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), extra_heliospheric_background, - EXTRA_HELIOSPHERIC_BACKGROUND_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), time_dependent_background, - TIME_DEPENDENT_BACKGROUND_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), filter_temperature_average, - FILTER_TEMPERATURE_AVERAGE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), filter_temperature_std_dev, - FILTER_TEMPERATURE_STD_DEV_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), hv_voltage_average, HV_VOLTAGE_AVERAGE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), hv_voltage_std_dev, HV_VOLTAGE_STD_DEV_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), spin_period_average, SPIN_PERIOD_AVERAGE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), spin_period_std_dev, SPIN_PERIOD_STD_DEV_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), spin_period_ground_average, - SPIN_PERIOD_GROUND_AVERAGE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), spin_period_ground_std_dev, - SPIN_PERIOD_GROUND_STD_DEV_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), pulse_length_average, PULSE_LENGTH_AVERAGE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), pulse_length_std_dev, PULSE_LENGTH_STD_DEV_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), position_angle_offset_average, - POSITION_ANGLE_OFFSET_AVERAGE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), position_angle_offset_std_dev, - POSITION_ANGLE_OFFSET_STD_DEV_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), spin_axis_orientation_average, - SPIN_AXIS_ORIENTATION_AVERAGE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), spin_axis_orientation_std_dev, - SPIN_AXIS_ORIENTATION_STD_DEV_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), spacecraft_location_average, - SPACECRAFT_LOCATION_AVERAGE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), spacecraft_location_std_dev, - SPACECRAFT_LOCATION_STD_DEV_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), spacecraft_velocity_average, - SPACECRAFT_VELOCITY_AVERAGE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), spacecraft_velocity_std_dev, - SPACECRAFT_VELOCITY_STD_DEV_CDF_VAR_NAME) + self.assert_variable_attributes( + next(variables), epoch_delta, EPOCH_DELTA_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), [start_time], START_TIME_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), [end_time], END_TIME_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), spin_angle, SPIN_ANGLE_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), spin_angle_delta, SPIN_ANGLE_DELTA_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), latitudes, LATITUDE_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), longitudes, LONGITUDE_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), + extra_heliospheric_background, + EXTRA_HELIOSPHERIC_BACKGROUND_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + time_dependent_background, + TIME_DEPENDENT_BACKGROUND_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + filter_temperature_average, + FILTER_TEMPERATURE_AVERAGE_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + filter_temperature_std_dev, + FILTER_TEMPERATURE_STD_DEV_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), hv_voltage_average, HV_VOLTAGE_AVERAGE_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), hv_voltage_std_dev, HV_VOLTAGE_STD_DEV_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), spin_period_average, SPIN_PERIOD_AVERAGE_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), spin_period_std_dev, SPIN_PERIOD_STD_DEV_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), + spin_period_ground_average, + SPIN_PERIOD_GROUND_AVERAGE_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + spin_period_ground_std_dev, + SPIN_PERIOD_GROUND_STD_DEV_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), pulse_length_average, PULSE_LENGTH_AVERAGE_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), pulse_length_std_dev, PULSE_LENGTH_STD_DEV_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), + position_angle_offset_average, + POSITION_ANGLE_OFFSET_AVERAGE_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + position_angle_offset_std_dev, + POSITION_ANGLE_OFFSET_STD_DEV_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + spin_axis_orientation_average, + SPIN_AXIS_ORIENTATION_AVERAGE_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + spin_axis_orientation_std_dev, + SPIN_AXIS_ORIENTATION_STD_DEV_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + spacecraft_location_average, + SPACECRAFT_LOCATION_AVERAGE_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + spacecraft_location_std_dev, + SPACECRAFT_LOCATION_STD_DEV_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + spacecraft_velocity_average, + SPACECRAFT_VELOCITY_AVERAGE_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + spacecraft_velocity_std_dev, + SPACECRAFT_VELOCITY_STD_DEV_CDF_VAR_NAME, + ) self.assert_variable_attributes(next(variables), [0, 1], "lon_lat") - self.assert_variable_attributes(next(variables), ["lon", "lat"], "lon_lat_labels") + self.assert_variable_attributes( + next(variables), ["lon", "lat"], "lon_lat_labels" + ) self.assert_variable_attributes(next(variables), [0, 1, 2], "x_y_z") - self.assert_variable_attributes(next(variables), ["X", "Y", "Z"], "x_y_z_labels") + self.assert_variable_attributes( + next(variables), ["X", "Y", "Z"], "x_y_z_labels" + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/glows/l3a/test_utils.py b/tests/glows/l3a/test_utils.py index 5828a5e7e..5447384e8 100644 --- a/tests/glows/l3a/test_utils.py +++ b/tests/glows/l3a/test_utils.py @@ -7,65 +7,140 @@ from spacepy.pycdf import CDF from imap_l3_processing.glows.l3a.models import GlowsL2Data, GlowsL2Header -from imap_l3_processing.glows.l3a.utils import read_l2_glows_data, create_glows_l3a_from_dictionary, \ - create_glows_l3a_dictionary_from_cdf +from imap_l3_processing.glows.l3a.utils import ( + read_l2_glows_data, + create_glows_l3a_from_dictionary, + create_glows_l3a_dictionary_from_cdf, +) from imap_l3_processing.models import InputMetadata from tests.test_helpers import get_test_data_path class TestUtils(unittest.TestCase): def test_reading_l2_glows_data_into_models(self): - cdf = CDF(str(get_test_data_path('glows/imap_glows_l2_hist_20251224-repoint00088_v001.cdf'))) + cdf = CDF( + str( + get_test_data_path( + "glows/imap_glows_l2_hist_20251224-repoint00088_v001.cdf" + ) + ) + ) glows_l2_data: GlowsL2Data = read_l2_glows_data(cdf) self.assertEqual("2025-12-24T11:36:21.588935256", glows_l2_data["start_time"]) self.assertEqual("2025-12-25T09:39:47.969074488", glows_l2_data["end_time"]) - self.assertEqual((3600,), glows_l2_data["daily_lightcurve"]["photon_flux"].shape) - self.assertEqual(887.6062785692615, glows_l2_data["daily_lightcurve"]["photon_flux"][0]) - self.assertEqual(872.260576384695, glows_l2_data["daily_lightcurve"]["photon_flux"][-1]) + self.assertEqual( + (3600,), glows_l2_data["daily_lightcurve"]["photon_flux"].shape + ) + self.assertEqual( + 887.6062785692615, glows_l2_data["daily_lightcurve"]["photon_flux"][0] + ) + self.assertEqual( + 872.260576384695, glows_l2_data["daily_lightcurve"]["photon_flux"][-1] + ) - self.assertEqual((3600,), glows_l2_data["daily_lightcurve"]["flux_uncertainties"].shape) - self.assertEqual(6.338751356172933, glows_l2_data["daily_lightcurve"]["flux_uncertainties"][0]) + self.assertEqual( + (3600,), glows_l2_data["daily_lightcurve"]["flux_uncertainties"].shape + ) + self.assertEqual( + 6.338751356172933, + glows_l2_data["daily_lightcurve"]["flux_uncertainties"][0], + ) - self.assertEqual((3600,), glows_l2_data["daily_lightcurve"]["exposure_times"].shape) - self.assertEqual(22.090875733333327, glows_l2_data["daily_lightcurve"]["exposure_times"][0]) + self.assertEqual( + (3600,), glows_l2_data["daily_lightcurve"]["exposure_times"].shape + ) + self.assertEqual( + 22.090875733333327, glows_l2_data["daily_lightcurve"]["exposure_times"][0] + ) self.assertEqual((3600,), glows_l2_data["daily_lightcurve"]["spin_angle"].shape) - self.assertEqual(0.08840215463658296, glows_l2_data["daily_lightcurve"]["spin_angle"][0]) + self.assertEqual( + 0.08840215463658296, glows_l2_data["daily_lightcurve"]["spin_angle"][0] + ) - self.assertEqual((3600,), glows_l2_data["daily_lightcurve"]["histogram_flag_array"].shape) - self.assertEqual(np.int64, glows_l2_data["daily_lightcurve"]["histogram_flag_array"].dtype) - self.assertEqual(np.uint8(0), glows_l2_data["daily_lightcurve"]["histogram_flag_array"][84]) + self.assertEqual( + (3600,), glows_l2_data["daily_lightcurve"]["histogram_flag_array"].shape + ) + self.assertEqual( + np.int64, glows_l2_data["daily_lightcurve"]["histogram_flag_array"].dtype + ) + self.assertEqual( + np.uint8(0), glows_l2_data["daily_lightcurve"]["histogram_flag_array"][84] + ) - self.assertEqual((3600,), glows_l2_data["daily_lightcurve"]["ecliptic_lat"].shape) - self.assertEqual((3600,), glows_l2_data["daily_lightcurve"]["ecliptic_lon"].shape) - self.assertEqual(75.04420784333895, glows_l2_data["daily_lightcurve"]["ecliptic_lat"][0]) - self.assertEqual(268.7416390634532, glows_l2_data["daily_lightcurve"]["ecliptic_lon"][0]) + self.assertEqual( + (3600,), glows_l2_data["daily_lightcurve"]["ecliptic_lat"].shape + ) + self.assertEqual( + (3600,), glows_l2_data["daily_lightcurve"]["ecliptic_lon"].shape + ) + self.assertEqual( + 75.04420784333895, glows_l2_data["daily_lightcurve"]["ecliptic_lat"][0] + ) + self.assertEqual( + 268.7416390634532, glows_l2_data["daily_lightcurve"]["ecliptic_lon"][0] + ) - self.assertEqual((3600,), glows_l2_data["daily_lightcurve"]["raw_histogram"].shape) + self.assertEqual( + (3600,), glows_l2_data["daily_lightcurve"]["raw_histogram"].shape + ) self.assertEqual(19608, glows_l2_data["daily_lightcurve"]["raw_histogram"][0]) - self.assertEqual({ - "lon": -2.9634262485046428, - "lat": 0.06861815065029926 - }, glows_l2_data["spin_axis_orientation_average"]) - - self.assertEqual(cdf['identifier'][0], glows_l2_data['identifier']) - self.assertEqual(cdf['filter_temperature_average'][0], glows_l2_data['filter_temperature_average']) - self.assertEqual(cdf['filter_temperature_std_dev'][0], glows_l2_data['filter_temperature_std_dev']) - self.assertEqual(cdf['hv_voltage_average'][0], glows_l2_data['hv_voltage_average']) - self.assertEqual(cdf['hv_voltage_std_dev'][0], glows_l2_data['hv_voltage_std_dev']) - self.assertEqual(cdf['spin_period_average'][0], glows_l2_data['spin_period_average']) - self.assertEqual(cdf['spin_period_std_dev'][0], glows_l2_data['spin_period_std_dev']) - self.assertEqual(cdf['spin_period_ground_average'][0], glows_l2_data['spin_period_ground_average']) - self.assertEqual(cdf['spin_period_ground_std_dev'][0], glows_l2_data['spin_period_ground_std_dev']) - self.assertEqual(cdf['pulse_length_average'][0], glows_l2_data['pulse_length_average']) - self.assertEqual(cdf['pulse_length_std_dev'][0], glows_l2_data['pulse_length_std_dev']) - self.assertEqual(cdf['position_angle_offset_average'][0], glows_l2_data['position_angle_offset_average']) - self.assertEqual(cdf['position_angle_offset_std_dev'][0], glows_l2_data['position_angle_offset_std_dev']) - self.assertEqual("imap_glows_l2_hist_20251224-repoint00088_v001.cdf", glows_l2_data['l2_file_name']) + self.assertEqual( + {"lon": -2.9634262485046428, "lat": 0.06861815065029926}, + glows_l2_data["spin_axis_orientation_average"], + ) + + self.assertEqual(cdf["identifier"][0], glows_l2_data["identifier"]) + self.assertEqual( + cdf["filter_temperature_average"][0], + glows_l2_data["filter_temperature_average"], + ) + self.assertEqual( + cdf["filter_temperature_std_dev"][0], + glows_l2_data["filter_temperature_std_dev"], + ) + self.assertEqual( + cdf["hv_voltage_average"][0], glows_l2_data["hv_voltage_average"] + ) + self.assertEqual( + cdf["hv_voltage_std_dev"][0], glows_l2_data["hv_voltage_std_dev"] + ) + self.assertEqual( + cdf["spin_period_average"][0], glows_l2_data["spin_period_average"] + ) + self.assertEqual( + cdf["spin_period_std_dev"][0], glows_l2_data["spin_period_std_dev"] + ) + self.assertEqual( + cdf["spin_period_ground_average"][0], + glows_l2_data["spin_period_ground_average"], + ) + self.assertEqual( + cdf["spin_period_ground_std_dev"][0], + glows_l2_data["spin_period_ground_std_dev"], + ) + self.assertEqual( + cdf["pulse_length_average"][0], glows_l2_data["pulse_length_average"] + ) + self.assertEqual( + cdf["pulse_length_std_dev"][0], glows_l2_data["pulse_length_std_dev"] + ) + self.assertEqual( + cdf["position_angle_offset_average"][0], + glows_l2_data["position_angle_offset_average"], + ) + self.assertEqual( + cdf["position_angle_offset_std_dev"][0], + glows_l2_data["position_angle_offset_std_dev"], + ) + self.assertEqual( + "imap_glows_l2_hist_20251224-repoint00088_v001.cdf", + glows_l2_data["l2_file_name"], + ) self.assert_equal_xyz(cdf, "spacecraft_location_average", glows_l2_data) self.assert_equal_xyz(cdf, "spacecraft_location_std_dev", glows_l2_data) @@ -73,39 +148,54 @@ def test_reading_l2_glows_data_into_models(self): self.assert_equal_xyz(cdf, "spacecraft_velocity_std_dev", glows_l2_data) expected_header = GlowsL2Header( - flight_software_version='123456', + flight_software_version="123456", pkts_file_name="data_l0/repointings/imap_glows_l0_raw_20251224-repoint00088_v000.pkts", ancillary_data_files=[ "imap_glows_pipeline-settings_20251112_v001.json", - "imap_glows_l2-calibration_20251112_v002.dat" + "imap_glows_l2-calibration_20251112_v002.dat", ], ) self.assertEqual(expected_header, glows_l2_data["header"]) def assert_equal_xyz(self, cdf, variable_name, actual): expected_spacecraft_location_average = { - 'x': cdf[variable_name][0, 0], - 'y': cdf[variable_name][0, 1], - 'z': cdf[variable_name][0, 2], + "x": cdf[variable_name][0, 0], + "y": cdf[variable_name][0, 1], + "z": cdf[variable_name][0, 2], } self.assertEqual(expected_spacecraft_location_average, actual[variable_name]) def test_fails_if_cdf_has_more_than_one_histogram(self): - cdf = CDF(str(get_test_data_path('glows/glows_l2_with_too_many_histograms.cdf'))) + cdf = CDF( + str(get_test_data_path("glows/glows_l2_with_too_many_histograms.cdf")) + ) with self.assertRaises(AssertionError) as cm: glows_l2_data: GlowsL2Data = read_l2_glows_data(cdf) - self.assertEqual(("Level 2 file should have only one histogram",), cm.exception.args) + self.assertEqual( + ("Level 2 file should have only one histogram",), cm.exception.args + ) def test_create_glows_l3a_from_dictionary(self): - with open(get_test_data_path("glows/imap_glows_l3a_20130908085214_orbX_modX_p_v00.json")) as f: + with open( + get_test_data_path( + "glows/imap_glows_l3a_20130908085214_orbX_modX_p_v00.json" + ) + ) as f: data = json.load(f) expected_spin_delta = 3 - data["daily_lightcurve"]['spin_angle_delta'] = np.full_like(data['daily_lightcurve']['spin_angle'], - expected_spin_delta) - input_metadata = InputMetadata(instrument='glows', data_level='l3a', - descriptor='hist', start_date=datetime(2013, 9, 8), - end_date=datetime(2013, 9, 9), version="v003", repointing=sentinel.repoint) + data["daily_lightcurve"]["spin_angle_delta"] = np.full_like( + data["daily_lightcurve"]["spin_angle"], expected_spin_delta + ) + input_metadata = InputMetadata( + instrument="glows", + data_level="l3a", + descriptor="hist", + start_date=datetime(2013, 9, 8), + end_date=datetime(2013, 9, 9), + version="v003", + repointing=sentinel.repoint, + ) result = create_glows_l3a_from_dictionary(data, input_metadata) @@ -130,7 +220,7 @@ def test_create_glows_l3a_from_dictionary(self): self.assertEqual(498484, result.raw_histogram[0, 0]) self.assertEqual((1, 90), result.exposure_times.shape) - self.assertEqual(8.028e+02, result.exposure_times[0, 0]) + self.assertEqual(8.028e02, result.exposure_times[0, 0]) self.assertEqual(1, len(result.epoch_delta)) self.assertEqual(72360 / 2 * 1e9, result.epoch_delta[0]) @@ -188,11 +278,11 @@ def test_create_glows_l3a_from_dictionary(self): self.assertEqual(result.epoch.shape, result.number_of_bins.shape) self.assertEqual(-27.84, result.filter_temperature_average[0]) - self.assertEqual(0.000e+00, result.filter_temperature_std_dev[0]) + self.assertEqual(0.000e00, result.filter_temperature_std_dev[0]) self.assertEqual(1527.1, result.hv_voltage_average[0]) - self.assertEqual(8.795e+01, result.hv_voltage_std_dev[0]) + self.assertEqual(8.795e01, result.hv_voltage_std_dev[0]) self.assertEqual(15.0000000, result.spin_period_average[0]) - self.assertEqual(0.000e+00, result.spin_period_std_dev[0]) + self.assertEqual(0.000e00, result.spin_period_std_dev[0]) self.assertEqual(15.2366823, result.spin_period_ground_average[0]) self.assertEqual(1.498e-03, result.spin_period_ground_std_dev[0]) self.assertEqual(2.990e-01, result.pulse_length_average[0]) @@ -201,77 +291,177 @@ def test_create_glows_l3a_from_dictionary(self): self.assertEqual(9.991e-03, result.position_angle_offset_std_dev[0]) np.testing.assert_array_equal(65, result.number_of_bins[0]) - np.testing.assert_array_equal([162.092, 0.000], result.spin_axis_orientation_average[0]) - np.testing.assert_array_equal([2.345e-01, 0.000e+00], result.spin_axis_orientation_std_dev[0]) - np.testing.assert_array_equal([146231106.1, -36212054.4, 1049.3], result.spacecraft_location_average[0]) - np.testing.assert_array_equal([1.421e+05, 6.001e+05, 4.082e+01], result.spacecraft_location_std_dev[0]) - np.testing.assert_array_equal([6.669, 28.805, -0.002], result.spacecraft_velocity_average[0]) - np.testing.assert_array_equal([1.188e-01, 2.961e-02, 4.337e-19], result.spacecraft_velocity_std_dev[0]) + np.testing.assert_array_equal( + [162.092, 0.000], result.spin_axis_orientation_average[0] + ) + np.testing.assert_array_equal( + [2.345e-01, 0.000e00], result.spin_axis_orientation_std_dev[0] + ) + np.testing.assert_array_equal( + [146231106.1, -36212054.4, 1049.3], + result.spacecraft_location_average[0], + ) + np.testing.assert_array_equal( + [1.421e05, 6.001e05, 4.082e01], result.spacecraft_location_std_dev[0] + ) + np.testing.assert_array_equal( + [6.669, 28.805, -0.002], result.spacecraft_velocity_average[0] + ) + np.testing.assert_array_equal( + [1.188e-01, 2.961e-02, 4.337e-19], result.spacecraft_velocity_std_dev[0] + ) def test_create_glows_l3a_dictionary_from_cdf(self): cdf_path = get_test_data_path("glows") / "imap_glows_l3a_hist_20100101_v001.cdf" actual_dictionary = create_glows_l3a_dictionary_from_cdf(cdf_path) - self.assertEqual("imap_glows_l3a_hist_20100101_v001.cdf", actual_dictionary["filename"]) - self.assertEqual(datetime(2013, 9, 8, 8, 52, 14).strftime("%Y-%m-%d %H:%M:%S"), - actual_dictionary['start_time']) - self.assertEqual(datetime(2013, 9, 9, 4, 58, 14).strftime("%Y-%m-%d %H:%M:%S"), actual_dictionary['end_time']) - - self.assertEqual(74.87028793594668, actual_dictionary['daily_lightcurve']['ecliptic_lat'][0]) - self.assertEqual(74.87028793594668, actual_dictionary['daily_lightcurve']['ecliptic_lat'][-1]) - self.assertEqual(154.67118387992855, actual_dictionary['daily_lightcurve']['ecliptic_lon'][0]) - self.assertEqual(169.51280659858702, actual_dictionary['daily_lightcurve']['ecliptic_lon'][-1]) - self.assertEqual(802.8, actual_dictionary['daily_lightcurve']['exposure_times'][0]) - self.assertEqual(80.28, actual_dictionary['daily_lightcurve']['exposure_times'][2]) - self.assertAlmostEqual(620.93173891, actual_dictionary['daily_lightcurve']['photon_flux'][0]) - self.assertAlmostEqual(635.46586946, actual_dictionary['daily_lightcurve']['photon_flux'][-1]) - self.assertAlmostEqual(0.87946437, actual_dictionary['daily_lightcurve']['flux_uncertainties'][0]) - self.assertAlmostEqual(0.88969763, actual_dictionary['daily_lightcurve']['flux_uncertainties'][-1]) - self.assertAlmostEqual(7.20014416e-04, actual_dictionary['daily_lightcurve']['extra_heliospheric_bckgrd'][0]) - self.assertAlmostEqual(3.66514655e-04, actual_dictionary['daily_lightcurve']['extra_heliospheric_bckgrd'][-1]) - self.assertEqual(0, actual_dictionary['daily_lightcurve']['time_dependent_bckgrd'][0]) - self.assertEqual(1.0000e-02, actual_dictionary['daily_lightcurve']['time_dependent_bckgrd'][10]) - self.assertEqual(2, actual_dictionary['daily_lightcurve']['spin_angle'][0]) - self.assertEqual(358, actual_dictionary['daily_lightcurve']['spin_angle'][-1]) - self.assertEqual(498484, actual_dictionary['daily_lightcurve']['raw_histogram'][0]) - self.assertEqual(510152, actual_dictionary['daily_lightcurve']['raw_histogram'][-1]) - self.assertEqual(65, actual_dictionary['daily_lightcurve']['number_of_bins']) - - self.assertEqual(-27.84000015258789, actual_dictionary['filter_temperature_average']) - self.assertEqual(0.0, actual_dictionary['filter_temperature_std_dev']) - self.assertEqual(1527.0999755859375, actual_dictionary['hv_voltage_average']) - self.assertEqual(87.94999694824219, actual_dictionary['hv_voltage_std_dev']) - self.assertEqual(15.0, actual_dictionary['spin_period_average']) - self.assertEqual(0.0, actual_dictionary['spin_period_std_dev']) - self.assertEqual(15.236681938171387, actual_dictionary['spin_period_ground_average']) - self.assertEqual(0.0014979999978095293, actual_dictionary['spin_period_ground_std_dev']) - self.assertEqual(0.29899999499320984, actual_dictionary['pulse_length_average']) - self.assertEqual(0.017260000109672546, actual_dictionary['pulse_length_std_dev']) - - self.assertEqual(91.5780029296875, actual_dictionary['position_angle_offset_average']) - self.assertEqual(0.009991000406444073, actual_dictionary['position_angle_offset_std_dev']) - - self.assertAlmostEqual(162.09199524, actual_dictionary['spin_axis_orientation_average']['lon']) - self.assertAlmostEqual(0.0, actual_dictionary['spin_axis_orientation_average']['lat']) - self.assertAlmostEqual(0.23450001, actual_dictionary['spin_axis_orientation_std_dev']['lon']) - self.assertAlmostEqual(0.0, actual_dictionary['spin_axis_orientation_std_dev']['lat']) - - self.assertAlmostEqual(1.46231104e+08, actual_dictionary['spacecraft_location_average']['x']) - self.assertAlmostEqual(-3.62120560e+07, actual_dictionary['spacecraft_location_average']['y']) - self.assertAlmostEqual(1049.300048828125, actual_dictionary['spacecraft_location_average']['z']) - - self.assertAlmostEqual(1.42100000e+05, actual_dictionary['spacecraft_location_std_dev']['x']) - self.assertAlmostEqual(6.00100000e+05, actual_dictionary['spacecraft_location_std_dev']['y']) - self.assertAlmostEqual(4.08199997e+01, actual_dictionary['spacecraft_location_std_dev']['z']) - - self.assertAlmostEqual(6.66900015e+00, actual_dictionary['spacecraft_velocity_average']['x']) - self.assertAlmostEqual(2.88050003e+01, actual_dictionary['spacecraft_velocity_average']['y']) - self.assertAlmostEqual(-2.00000009e-03, actual_dictionary['spacecraft_velocity_average']['z']) - - self.assertAlmostEqual(1.18799999e-01, actual_dictionary['spacecraft_velocity_std_dev']['x']) - self.assertAlmostEqual(2.96100006e-02, actual_dictionary['spacecraft_velocity_std_dev']['y']) - self.assertAlmostEqual(4.33699998e-19, actual_dictionary['spacecraft_velocity_std_dev']['z']) - - -if __name__ == '__main__': + self.assertEqual( + "imap_glows_l3a_hist_20100101_v001.cdf", actual_dictionary["filename"] + ) + self.assertEqual( + datetime(2013, 9, 8, 8, 52, 14).strftime("%Y-%m-%d %H:%M:%S"), + actual_dictionary["start_time"], + ) + self.assertEqual( + datetime(2013, 9, 9, 4, 58, 14).strftime("%Y-%m-%d %H:%M:%S"), + actual_dictionary["end_time"], + ) + + self.assertEqual( + 74.87028793594668, actual_dictionary["daily_lightcurve"]["ecliptic_lat"][0] + ) + self.assertEqual( + 74.87028793594668, actual_dictionary["daily_lightcurve"]["ecliptic_lat"][-1] + ) + self.assertEqual( + 154.67118387992855, actual_dictionary["daily_lightcurve"]["ecliptic_lon"][0] + ) + self.assertEqual( + 169.51280659858702, + actual_dictionary["daily_lightcurve"]["ecliptic_lon"][-1], + ) + self.assertEqual( + 802.8, actual_dictionary["daily_lightcurve"]["exposure_times"][0] + ) + self.assertEqual( + 80.28, actual_dictionary["daily_lightcurve"]["exposure_times"][2] + ) + self.assertAlmostEqual( + 620.93173891, actual_dictionary["daily_lightcurve"]["photon_flux"][0] + ) + self.assertAlmostEqual( + 635.46586946, actual_dictionary["daily_lightcurve"]["photon_flux"][-1] + ) + self.assertAlmostEqual( + 0.87946437, actual_dictionary["daily_lightcurve"]["flux_uncertainties"][0] + ) + self.assertAlmostEqual( + 0.88969763, actual_dictionary["daily_lightcurve"]["flux_uncertainties"][-1] + ) + self.assertAlmostEqual( + 7.20014416e-04, + actual_dictionary["daily_lightcurve"]["extra_heliospheric_bckgrd"][0], + ) + self.assertAlmostEqual( + 3.66514655e-04, + actual_dictionary["daily_lightcurve"]["extra_heliospheric_bckgrd"][-1], + ) + self.assertEqual( + 0, actual_dictionary["daily_lightcurve"]["time_dependent_bckgrd"][0] + ) + self.assertEqual( + 1.0000e-02, + actual_dictionary["daily_lightcurve"]["time_dependent_bckgrd"][10], + ) + self.assertEqual(2, actual_dictionary["daily_lightcurve"]["spin_angle"][0]) + self.assertEqual(358, actual_dictionary["daily_lightcurve"]["spin_angle"][-1]) + self.assertEqual( + 498484, actual_dictionary["daily_lightcurve"]["raw_histogram"][0] + ) + self.assertEqual( + 510152, actual_dictionary["daily_lightcurve"]["raw_histogram"][-1] + ) + self.assertEqual(65, actual_dictionary["daily_lightcurve"]["number_of_bins"]) + + self.assertEqual( + -27.84000015258789, actual_dictionary["filter_temperature_average"] + ) + self.assertEqual(0.0, actual_dictionary["filter_temperature_std_dev"]) + self.assertEqual(1527.0999755859375, actual_dictionary["hv_voltage_average"]) + self.assertEqual(87.94999694824219, actual_dictionary["hv_voltage_std_dev"]) + self.assertEqual(15.0, actual_dictionary["spin_period_average"]) + self.assertEqual(0.0, actual_dictionary["spin_period_std_dev"]) + self.assertEqual( + 15.236681938171387, actual_dictionary["spin_period_ground_average"] + ) + self.assertEqual( + 0.0014979999978095293, actual_dictionary["spin_period_ground_std_dev"] + ) + self.assertEqual(0.29899999499320984, actual_dictionary["pulse_length_average"]) + self.assertEqual( + 0.017260000109672546, actual_dictionary["pulse_length_std_dev"] + ) + + self.assertEqual( + 91.5780029296875, actual_dictionary["position_angle_offset_average"] + ) + self.assertEqual( + 0.009991000406444073, actual_dictionary["position_angle_offset_std_dev"] + ) + + self.assertAlmostEqual( + 162.09199524, actual_dictionary["spin_axis_orientation_average"]["lon"] + ) + self.assertAlmostEqual( + 0.0, actual_dictionary["spin_axis_orientation_average"]["lat"] + ) + self.assertAlmostEqual( + 0.23450001, actual_dictionary["spin_axis_orientation_std_dev"]["lon"] + ) + self.assertAlmostEqual( + 0.0, actual_dictionary["spin_axis_orientation_std_dev"]["lat"] + ) + + self.assertAlmostEqual( + 1.46231104e08, actual_dictionary["spacecraft_location_average"]["x"] + ) + self.assertAlmostEqual( + -3.62120560e07, actual_dictionary["spacecraft_location_average"]["y"] + ) + self.assertAlmostEqual( + 1049.300048828125, actual_dictionary["spacecraft_location_average"]["z"] + ) + + self.assertAlmostEqual( + 1.42100000e05, actual_dictionary["spacecraft_location_std_dev"]["x"] + ) + self.assertAlmostEqual( + 6.00100000e05, actual_dictionary["spacecraft_location_std_dev"]["y"] + ) + self.assertAlmostEqual( + 4.08199997e01, actual_dictionary["spacecraft_location_std_dev"]["z"] + ) + + self.assertAlmostEqual( + 6.66900015e00, actual_dictionary["spacecraft_velocity_average"]["x"] + ) + self.assertAlmostEqual( + 2.88050003e01, actual_dictionary["spacecraft_velocity_average"]["y"] + ) + self.assertAlmostEqual( + -2.00000009e-03, actual_dictionary["spacecraft_velocity_average"]["z"] + ) + + self.assertAlmostEqual( + 1.18799999e-01, actual_dictionary["spacecraft_velocity_std_dev"]["x"] + ) + self.assertAlmostEqual( + 2.96100006e-02, actual_dictionary["spacecraft_velocity_std_dev"]["y"] + ) + self.assertAlmostEqual( + 4.33699998e-19, actual_dictionary["spacecraft_velocity_std_dev"]["z"] + ) + + +if __name__ == "__main__": unittest.main() diff --git a/tests/glows/l3bc/l3bc_toolkit/test_funcs.py b/tests/glows/l3bc/l3bc_toolkit/test_funcs.py index 77229ac19..31726af8f 100644 --- a/tests/glows/l3bc/l3bc_toolkit/test_funcs.py +++ b/tests/glows/l3bc/l3bc_toolkit/test_funcs.py @@ -10,7 +10,7 @@ "column_numbers": (0, 1, 2, 6, 9), "gap_marker": 9.999, "scale": False, - "const_if_empty": True + "const_if_empty": True, } @@ -38,7 +38,9 @@ def test_returns_zero_substitution_for_all_valid_alpha(self): np.testing.assert_array_equal(used_nominal_per_cr, np.array([False, False])) np.testing.assert_array_almost_equal(averaged, np.array([0.05, 0.06])) - def test_filters_invalid_samples_and_does_not_flag_cr_when_some_samples_remain(self): + def test_filters_invalid_samples_and_does_not_flag_cr_when_some_samples_remain( + self, + ): omni_raw = np.array( [ _row(2000, 1, 0, 0.04), @@ -79,7 +81,12 @@ def test_fills_trailing_missing_crs_with_nominal_value(self): np.testing.assert_array_almost_equal( averaged, np.array( - [0.05, 0.06, (NOMINAL_ALPHA_PROTON_RATIO_VALUE + 0.06) / 2, NOMINAL_ALPHA_PROTON_RATIO_VALUE] + [ + 0.05, + 0.06, + (NOMINAL_ALPHA_PROTON_RATIO_VALUE + 0.06) / 2, + NOMINAL_ALPHA_PROTON_RATIO_VALUE, + ] ), ) @@ -118,7 +125,9 @@ def test_smooths_interior_cr_gap_via_interpolation_and_does_not_flag(self): omni_raw, cr_grid, ALPHA_PARAM_SETTINGS ) - np.testing.assert_array_equal(used_nominal_per_cr, np.array([False, False, False])) + np.testing.assert_array_equal( + used_nominal_per_cr, np.array([False, False, False]) + ) np.testing.assert_array_almost_equal(averaged, np.array([0.05, 0.06, 0.07])) diff --git a/tests/glows/l3bc/science/test_filter_out_bad_days.py b/tests/glows/l3bc/science/test_filter_out_bad_days.py index 2f012866d..e588aa477 100644 --- a/tests/glows/l3bc/science/test_filter_out_bad_days.py +++ b/tests/glows/l3bc/science/test_filter_out_bad_days.py @@ -16,7 +16,7 @@ def test_filter_l3a_files(self): create_l3a_dict("2010-01-19 00:00:00", "2010-01-20 00:00:00"), create_l3a_dict("2010-01-30 00:00:00", "2010-01-30 07:28:00"), create_l3a_dict("2010-01-30 12:00:00", "2010-01-30 21:00:00"), - create_l3a_dict("2010-01-30 19:00:00", "2010-01-30 23:59:00") + create_l3a_dict("2010-01-30 19:00:00", "2010-01-30 23:59:00"), ] expected_filtered_list = [ @@ -28,13 +28,17 @@ def test_filter_l3a_files(self): create_l3a_dict("2010-01-30 12:00:00", "2010-01-30 21:00:00"), ] - filtered_list = filter_l3a_files(l3a_data=l3a_data, bad_day_list_path=get_test_data_path( - "glows") / "imap_glows_bad-days-list_v001.dat", cr=2092) + filtered_list = filter_l3a_files( + l3a_data=l3a_data, + bad_day_list_path=get_test_data_path("glows") + / "imap_glows_bad-days-list_v001.dat", + cr=2092, + ) self.assertEqual(expected_filtered_list, filtered_list) def create_l3a_dict(start_date: str, end_date: str) -> dict: return { - 'start_time': start_date, - 'end_time': end_date, + "start_time": start_date, + "end_time": end_date, } diff --git a/tests/glows/l3bc/science/test_generate_l3bc.py b/tests/glows/l3bc/science/test_generate_l3bc.py index e77d3f18a..8b6ba01fa 100644 --- a/tests/glows/l3bc/science/test_generate_l3bc.py +++ b/tests/glows/l3bc/science/test_generate_l3bc.py @@ -6,32 +6,53 @@ from imap_data_access.file_validation import Version from imap_l3_processing.glows.l3a.utils import create_glows_l3a_dictionary_from_cdf -from imap_l3_processing.glows.l3bc.cannot_process_carrington_rotation_error import CannotProcessCarringtonRotationError +from imap_l3_processing.glows.l3bc.cannot_process_carrington_rotation_error import ( + CannotProcessCarringtonRotationError, +) from imap_l3_processing.glows.l3bc.glows_l3bc_dependencies import GlowsL3BCDependencies from imap_l3_processing.glows.l3bc.science.generate_l3bc import generate_l3bc from imap_l3_processing.glows.quality_flags import GlowsL3Flags -from tests.test_helpers import get_test_data_path, get_test_instrument_team_data_path, assert_dict_close +from tests.test_helpers import ( + get_test_data_path, + get_test_instrument_team_data_path, + assert_dict_close, +) class TestGenerateL3BC(TestCase): def test_generate_l3bc_integration(self): cr = 2091 external_files = { - 'f107_raw_data': get_test_instrument_team_data_path('glows/f107_fluxtable.txt'), - 'omni_raw_data': get_test_instrument_team_data_path('glows/omni2_all_years.dat') + "f107_raw_data": get_test_instrument_team_data_path( + "glows/f107_fluxtable.txt" + ), + "omni_raw_data": get_test_instrument_team_data_path( + "glows/omni2_all_years.dat" + ), } ancillary_files = { - 'uv_anisotropy': get_test_data_path('glows/imap_glows_uv-anisotropy-1CR_20250514_v002.json'), - 'WawHelioIonMP_parameters': get_test_data_path('glows/imap_glows_WawHelioIonMP_v002.json'), - 'bad_days_list': get_test_data_path('glows/imap_glows_bad-days-list_v001.dat'), - 'pipeline_settings': get_test_data_path( - "glows/l3d_drift_test/imap_glows_pipeline-settings-l3bcde_20100101_v006.json") + "uv_anisotropy": get_test_data_path( + "glows/imap_glows_uv-anisotropy-1CR_20250514_v002.json" + ), + "WawHelioIonMP_parameters": get_test_data_path( + "glows/imap_glows_WawHelioIonMP_v002.json" + ), + "bad_days_list": get_test_data_path( + "glows/imap_glows_bad-days-list_v001.dat" + ), + "pipeline_settings": get_test_data_path( + "glows/l3d_drift_test/imap_glows_pipeline-settings-l3bcde_20100101_v006.json" + ), } - l3a_data_folder_path = get_test_data_path('glows/l3a_products') + l3a_data_folder_path = get_test_data_path("glows/l3a_products") l3a_data = [] - l3a_file_names = [f"imap_glows_l3a_hist_2010010{x}-repoint0000{x}_v001.cdf" for x in (1, 2, 3)] + l3a_file_names = [ + f"imap_glows_l3a_hist_2010010{x}-repoint0000{x}_v001.cdf" for x in (1, 2, 3) + ] for name in l3a_file_names: - l3a_data.append(create_glows_l3a_dictionary_from_cdf(l3a_data_folder_path / name)) + l3a_data.append( + create_glows_l3a_dictionary_from_cdf(l3a_data_folder_path / name) + ) dependencies = GlowsL3BCDependencies( l3a_data=l3a_data, @@ -40,16 +61,20 @@ def test_generate_l3bc_integration(self): carrington_rotation_number=cr, start_date=datetime.now(), end_date=datetime.now() + timedelta(days=1), - version=Version(1,1), - repointing_file_path=sentinel.repointing_file_path + version=Version(1, 1), + repointing_file_path=sentinel.repointing_file_path, ) actual_l3b, actual_l3c = generate_l3bc(dependencies) - with open(get_test_instrument_team_data_path("glows/imap_glows_l3b_cr_2091_v00.json")) as f: + with open( + get_test_instrument_team_data_path("glows/imap_glows_l3b_cr_2091_v00.json") + ) as f: expected_l3b = json.load(f) - with open(get_test_instrument_team_data_path("glows/imap_glows_l3c_cr_2091_v00.json")) as f: + with open( + get_test_instrument_team_data_path("glows/imap_glows_l3c_cr_2091_v00.json") + ) as f: expected_l3c = json.load(f) self.assertEqual( @@ -71,20 +96,34 @@ def test_generate_l3bc_integration(self): def test_generate_l3bc_integration_all_data_in_bad_season(self): cr = 2093 external_files = { - 'f107_raw_data': get_test_instrument_team_data_path('glows/f107_fluxtable.txt'), - 'omni_raw_data': get_test_instrument_team_data_path('glows/omni2_all_years.dat') + "f107_raw_data": get_test_instrument_team_data_path( + "glows/f107_fluxtable.txt" + ), + "omni_raw_data": get_test_instrument_team_data_path( + "glows/omni2_all_years.dat" + ), } ancillary_files = { - 'uv_anisotropy': get_test_data_path('glows/imap_glows_uv-anisotropy-1CR_20100101_v001.json'), - 'WawHelioIonMP_parameters': get_test_data_path('glows/imap_glows_WawHelioIonMP_20100101_v002.json'), - 'bad_days_list': get_test_data_path('glows/imap_glows_bad-days-list_v001.dat'), - 'pipeline_settings': get_test_instrument_team_data_path( - 'glows/imap_glows_pipeline-settings-L3bc_20250707_v002.json') + "uv_anisotropy": get_test_data_path( + "glows/imap_glows_uv-anisotropy-1CR_20100101_v001.json" + ), + "WawHelioIonMP_parameters": get_test_data_path( + "glows/imap_glows_WawHelioIonMP_20100101_v002.json" + ), + "bad_days_list": get_test_data_path( + "glows/imap_glows_bad-days-list_v001.dat" + ), + "pipeline_settings": get_test_instrument_team_data_path( + "glows/imap_glows_pipeline-settings-L3bc_20250707_v002.json" + ), } - l3a_data_folder_path = get_test_data_path('glows/l3a_products') + l3a_data_folder_path = get_test_data_path("glows/l3a_products") l3a_data = [ create_glows_l3a_dictionary_from_cdf( - l3a_data_folder_path / 'imap_glows_l3a_hist_20100201-repoint00032_v001.cdf')] + l3a_data_folder_path + / "imap_glows_l3a_hist_20100201-repoint00032_v001.cdf" + ) + ] dependencies = GlowsL3BCDependencies( l3a_data=l3a_data, @@ -93,25 +132,40 @@ def test_generate_l3bc_integration_all_data_in_bad_season(self): carrington_rotation_number=cr, start_date=datetime.now(), end_date=datetime.now() + timedelta(days=1), - version=Version(1,1), - repointing_file_path=sentinel.repointing_file_path + version=Version(1, 1), + repointing_file_path=sentinel.repointing_file_path, ) with self.assertRaises(CannotProcessCarringtonRotationError) as context: generate_l3bc(dependencies) - self.assertTrue("All days for Carrington Rotation are in a bad season." in str(context.exception)) + self.assertTrue( + "All days for Carrington Rotation are in a bad season." + in str(context.exception) + ) def test_generate_l3bc_appends_used_l3a_files_to_header(self): cr = 2096 external_files = { - 'f107_raw_data': get_test_instrument_team_data_path('glows/f107_fluxtable.txt'), - 'omni_raw_data': get_test_instrument_team_data_path('glows/omni2_all_years.dat') + "f107_raw_data": get_test_instrument_team_data_path( + "glows/f107_fluxtable.txt" + ), + "omni_raw_data": get_test_instrument_team_data_path( + "glows/omni2_all_years.dat" + ), } ancillary_files = { - 'uv_anisotropy': get_test_data_path('glows/imap_glows_uv-anisotropy-1CR_20100101_v001.json'), - 'WawHelioIonMP_parameters': get_test_data_path('glows/imap_glows_WawHelioIonMP_20100101_v002.json'), - 'bad_days_list': get_test_data_path('glows/imap_glows_bad-days-list_v001.dat'), - 'pipeline_settings': get_test_data_path('glows/imap_glows_pipeline-settings-l3bcde_20250423_v001.json') + "uv_anisotropy": get_test_data_path( + "glows/imap_glows_uv-anisotropy-1CR_20100101_v001.json" + ), + "WawHelioIonMP_parameters": get_test_data_path( + "glows/imap_glows_WawHelioIonMP_20100101_v002.json" + ), + "bad_days_list": get_test_data_path( + "glows/imap_glows_bad-days-list_v001.dat" + ), + "pipeline_settings": get_test_data_path( + "glows/imap_glows_pipeline-settings-l3bcde_20250423_v001.json" + ), } l3a_files = [ "imap_glows_l3a_hist_20100422-repoint00112_v012.cdf", @@ -143,8 +197,11 @@ def test_generate_l3bc_appends_used_l3a_files_to_header(self): "imap_glows_l3a_hist_20100518-repoint00138_v012.cdf", ] - l3a_folder_path = get_test_data_path('glows/l3a_products') - l3a_data = [create_glows_l3a_dictionary_from_cdf(l3a_folder_path / file) for file in l3a_files] + l3a_folder_path = get_test_data_path("glows/l3a_products") + l3a_data = [ + create_glows_l3a_dictionary_from_cdf(l3a_folder_path / file) + for file in l3a_files + ] dependencies = GlowsL3BCDependencies( l3a_data=l3a_data, @@ -153,8 +210,8 @@ def test_generate_l3bc_appends_used_l3a_files_to_header(self): carrington_rotation_number=cr, start_date=datetime.now(), end_date=datetime.now() + timedelta(days=1), - version=Version(1,1), - repointing_file_path=sentinel.repointing_file_path + version=Version(1, 1), + repointing_file_path=sentinel.repointing_file_path, ) l3b, l3c = generate_l3bc(dependencies) @@ -169,4 +226,6 @@ def test_generate_l3bc_appends_used_l3a_files_to_header(self): "imap_glows_l3a_hist_20100518-repoint00138_v012.cdf", ] - self.assertEqual(expected_l3a_parent_files, l3b['header']['l3a_input_files_name']) + self.assertEqual( + expected_l3a_parent_files, l3b["header"]["l3a_input_files_name"] + ) diff --git a/tests/glows/l3bc/science/test_time_parsing.py b/tests/glows/l3bc/science/test_time_parsing.py index 0a7d72226..8050ba64b 100644 --- a/tests/glows/l3bc/science/test_time_parsing.py +++ b/tests/glows/l3bc/science/test_time_parsing.py @@ -3,36 +3,54 @@ import numpy as np from astropy.time import Time, conf -from imap_l3_processing.glows.l3bc.l3bc_toolkit.funcs import time_from_yday as time_from_yday_l3bc -from imap_l3_processing.glows.l3d.science.toolkit.funcs import time_from_yday as time_from_yday_l3d +from imap_l3_processing.glows.l3bc.l3bc_toolkit.funcs import ( + time_from_yday as time_from_yday_l3bc, +) +from imap_l3_processing.glows.l3d.science.toolkit.funcs import ( + time_from_yday as time_from_yday_l3d, +) class TestTimeParsing(unittest.TestCase): def test_works_with_fast_parser(self): - time_rows = np.array([ - [2000.0, 5.0,1.0], - [2000.0, 51.0, 12.0], - [2000.0, 365.0, 23.0], - ]) + time_rows = np.array( + [ + [2000.0, 5.0, 1.0], + [2000.0, 51.0, 12.0], + [2000.0, 365.0, 23.0], + ] + ) with conf.set_temp("use_fast_parser", "force"): try: result_l3bc = time_from_yday_l3bc(time_rows) result_l3d = time_from_yday_l3d(time_rows) except ValueError: - self.fail('Fast parser should not crash, check formatting of date string.') - - np.testing.assert_array_equal(result_l3bc, Time([ - "2000-01-05T01:00", - "2000-02-20T12:00", - "2000-12-30T23:00", - ])) - - np.testing.assert_array_equal(result_l3d, Time([ - "2000-01-05T01:00", - "2000-02-20T12:00", - "2000-12-30T23:00", - ])) - - -if __name__ == '__main__': + self.fail( + "Fast parser should not crash, check formatting of date string." + ) + + np.testing.assert_array_equal( + result_l3bc, + Time( + [ + "2000-01-05T01:00", + "2000-02-20T12:00", + "2000-12-30T23:00", + ] + ), + ) + + np.testing.assert_array_equal( + result_l3d, + Time( + [ + "2000-01-05T01:00", + "2000-02-20T12:00", + "2000-12-30T23:00", + ] + ), + ) + + +if __name__ == "__main__": unittest.main() diff --git a/tests/glows/l3bc/test_dependency_validator.py b/tests/glows/l3bc/test_dependency_validator.py index 8d7a2fe39..66439f27d 100644 --- a/tests/glows/l3bc/test_dependency_validator.py +++ b/tests/glows/l3bc/test_dependency_validator.py @@ -5,13 +5,19 @@ from unittest.mock import patch, Mock -from imap_l3_processing.glows.l3bc.dependency_validator import validate_omni2_dependency, \ - validate_dependencies, validate_f107_fluxtable_dependency, validate_lyman_alpha_dependency +from imap_l3_processing.glows.l3bc.dependency_validator import ( + validate_omni2_dependency, + validate_dependencies, + validate_f107_fluxtable_dependency, + validate_lyman_alpha_dependency, +) from tests.test_helpers import get_test_data_path class TestDependencyValidator(unittest.TestCase): - def test_validate_omni2_dependency_returns_true_when_alpha_fill_but_density_and_speed_valid(self): + def test_validate_omni2_dependency_returns_true_when_alpha_fill_but_density_and_speed_valid( + self, + ): with TemporaryDirectory() as tmp: omni_path = Path(tmp) / "omni2_slice.txt" row = ["0"] * 35 @@ -51,22 +57,33 @@ def test_validate_omni2_dependency(self): for name, cr_start_date, cr_end_date_exclusive, expected in test_cases: with self.subTest(name): - actual = validate_omni2_dependency(file_path=file_path, - cr_start_date=cr_start_date, - cr_end_date_exclusive=cr_end_date_exclusive) + actual = validate_omni2_dependency( + file_path=file_path, + cr_start_date=cr_start_date, + cr_end_date_exclusive=cr_end_date_exclusive, + ) self.assertEqual(expected, actual) - @patch("imap_l3_processing.glows.l3bc.dependency_validator.validate_lyman_alpha_dependency") - @patch("imap_l3_processing.glows.l3bc.dependency_validator.validate_f107_fluxtable_dependency") - @patch("imap_l3_processing.glows.l3bc.dependency_validator.validate_omni2_dependency") - def test_validate_dependencies(self, mock_validate_omni2_dependency: Mock, - mock_validate_f107_fluxtable_dependency: Mock, - mock_validate_lyman_alpha_dependency: Mock): + @patch( + "imap_l3_processing.glows.l3bc.dependency_validator.validate_lyman_alpha_dependency" + ) + @patch( + "imap_l3_processing.glows.l3bc.dependency_validator.validate_f107_fluxtable_dependency" + ) + @patch( + "imap_l3_processing.glows.l3bc.dependency_validator.validate_omni2_dependency" + ) + def test_validate_dependencies( + self, + mock_validate_omni2_dependency: Mock, + mock_validate_f107_fluxtable_dependency: Mock, + mock_validate_lyman_alpha_dependency: Mock, + ): test_cases = [ ("happy case", True, True, True, True), ("omni validation fails", False, True, True, False), ("flux validation fails", True, False, True, False), - ("lyman validation fails", True, True, False, False) + ("lyman validation fails", True, True, False, False), ] cr_start_date = datetime(1994, 7, 10) cr_end_date = datetime(1994, 7, 17) @@ -75,24 +92,41 @@ def test_validate_dependencies(self, mock_validate_omni2_dependency: Mock, fluxtable_file_path = Path("fluxtable path") lyman_alpha_file_path = Path("fluxtable path") - for case_name, omni2_validation, flux_validation, lyman_alpha_validation, expected in test_cases: + for ( + case_name, + omni2_validation, + flux_validation, + lyman_alpha_validation, + expected, + ) in test_cases: with self.subTest(case_name): mock_validate_omni2_dependency.return_value = omni2_validation mock_validate_f107_fluxtable_dependency.return_value = flux_validation - mock_validate_lyman_alpha_dependency.return_value = lyman_alpha_validation + mock_validate_lyman_alpha_dependency.return_value = ( + lyman_alpha_validation + ) - actual = validate_dependencies(cr_start_date, cr_end_date, buffer, omni_file_path, - fluxtable_file_path, lyman_alpha_file_path) + actual = validate_dependencies( + cr_start_date, + cr_end_date, + buffer, + omni_file_path, + fluxtable_file_path, + lyman_alpha_file_path, + ) self.assertEqual(expected, actual) - mock_validate_omni2_dependency.assert_called_once_with(cr_start_date, cr_end_date, - omni_file_path) + mock_validate_omni2_dependency.assert_called_once_with( + cr_start_date, cr_end_date, omni_file_path + ) - mock_validate_f107_fluxtable_dependency.assert_called_once_with(cr_end_date, buffer, - fluxtable_file_path) + mock_validate_f107_fluxtable_dependency.assert_called_once_with( + cr_end_date, buffer, fluxtable_file_path + ) - mock_validate_lyman_alpha_dependency.assert_called_once_with(cr_end_date, buffer, - lyman_alpha_file_path) + mock_validate_lyman_alpha_dependency.assert_called_once_with( + cr_end_date, buffer, lyman_alpha_file_path + ) mock_validate_omni2_dependency.reset_mock() mock_validate_f107_fluxtable_dependency.reset_mock() mock_validate_lyman_alpha_dependency.reset_mock() @@ -105,19 +139,21 @@ def test_validate_f107_dependency(self): ] for name, end_date, buffer, expected in test_cases: with self.subTest(name): - is_validated = validate_f107_fluxtable_dependency(file_path=file_path, - end_date=end_date, - buffer=buffer) + is_validated = validate_f107_fluxtable_dependency( + file_path=file_path, end_date=end_date, buffer=buffer + ) self.assertEqual(expected, is_validated) def test_validate_lyman_alpha_dependency(self): file_path = get_test_data_path("glows/lyman_alpha_composite.nc") test_cases = [ - ("happy case", datetime(2025,3,16), timedelta(4), True), - ("not enough data end", datetime(2025,3,18), timedelta(4), False) + ("happy case", datetime(2025, 3, 16), timedelta(4), True), + ("not enough data end", datetime(2025, 3, 18), timedelta(4), False), ] for name, end_date, buffer, expected in test_cases: with self.subTest(name): - is_validated = validate_lyman_alpha_dependency(end_date=end_date, buffer=buffer, file_path=file_path) + is_validated = validate_lyman_alpha_dependency( + end_date=end_date, buffer=buffer, file_path=file_path + ) self.assertEqual(expected, is_validated) diff --git a/tests/glows/l3bc/test_glows_l3bc_dependencies.py b/tests/glows/l3bc/test_glows_l3bc_dependencies.py index 533557bb6..2281853a0 100644 --- a/tests/glows/l3bc/test_glows_l3bc_dependencies.py +++ b/tests/glows/l3bc/test_glows_l3bc_dependencies.py @@ -9,10 +9,15 @@ class TestGlowsL3BCDependencies(unittest.TestCase): - - @patch('imap_l3_processing.glows.l3bc.glows_l3bc_dependencies.create_glows_l3a_dictionary_from_cdf') - @patch('imap_l3_processing.glows.l3bc.glows_l3bc_dependencies.imap_data_access.download') - def test_download_from_cr_to_process(self, mock_download, mock_create_dictionary_from_cdf): + @patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_dependencies.create_glows_l3a_dictionary_from_cdf" + ) + @patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_dependencies.imap_data_access.download" + ) + def test_download_from_cr_to_process( + self, mock_download, mock_create_dictionary_from_cdf + ): mock_download.side_effect = [ sentinel.uv_anisotropy_downloaded_path, sentinel.waw_downloaded_path, @@ -40,35 +45,64 @@ def test_download_from_cr_to_process(self, mock_download, mock_create_dictionary ) external_dependencies = ExternalDependencies( - f107_index_file_path=TEMP_CDF_FOLDER_PATH / 'f107_fluxtable.txt', - omni2_data_path=TEMP_CDF_FOLDER_PATH / 'omni2_all_years.dat', + f107_index_file_path=TEMP_CDF_FOLDER_PATH / "f107_fluxtable.txt", + omni2_data_path=TEMP_CDF_FOLDER_PATH / "omni2_all_years.dat", lyman_alpha_path=Path("not used"), ) - dependency: GlowsL3BCDependencies = GlowsL3BCDependencies.download_from_cr_to_process(cr_to_process, - sentinel.version, - external_dependencies, - repointing_file_path) + dependency: GlowsL3BCDependencies = ( + GlowsL3BCDependencies.download_from_cr_to_process( + cr_to_process, + sentinel.version, + external_dependencies, + repointing_file_path, + ) + ) - self.assertEqual([call(sentinel.l3a_downloaded_path_1), call(sentinel.l3a_downloaded_path_2)], - mock_create_dictionary_from_cdf.call_args_list) + self.assertEqual( + [ + call(sentinel.l3a_downloaded_path_1), + call(sentinel.l3a_downloaded_path_2), + ], + mock_create_dictionary_from_cdf.call_args_list, + ) - mock_download.assert_has_calls([ - call("uv_anisotropy_path"), - call("waw_path"), - call("bad_days_list_path"), - call("pipeline_settings_path"), - call("l3a_path_1"), - call("l3a_path_2") - ]) + mock_download.assert_has_calls( + [ + call("uv_anisotropy_path"), + call("waw_path"), + call("bad_days_list_path"), + call("pipeline_settings_path"), + call("l3a_path_1"), + call("l3a_path_2"), + ] + ) - self.assertEqual(TEMP_CDF_FOLDER_PATH / 'f107_fluxtable.txt', dependency.external_files['f107_raw_data']) - self.assertEqual(TEMP_CDF_FOLDER_PATH / 'omni2_all_years.dat', dependency.external_files['omni_raw_data']) + self.assertEqual( + TEMP_CDF_FOLDER_PATH / "f107_fluxtable.txt", + dependency.external_files["f107_raw_data"], + ) + self.assertEqual( + TEMP_CDF_FOLDER_PATH / "omni2_all_years.dat", + dependency.external_files["omni_raw_data"], + ) - self.assertEqual(sentinel.uv_anisotropy_downloaded_path, dependency.ancillary_files['uv_anisotropy']) - self.assertEqual(sentinel.waw_downloaded_path, dependency.ancillary_files['WawHelioIonMP_parameters']) - self.assertEqual(sentinel.bad_day_list_downloaded_path, dependency.ancillary_files['bad_days_list']) - self.assertEqual(sentinel.settings_downloaded_path, dependency.ancillary_files['pipeline_settings']) + self.assertEqual( + sentinel.uv_anisotropy_downloaded_path, + dependency.ancillary_files["uv_anisotropy"], + ) + self.assertEqual( + sentinel.waw_downloaded_path, + dependency.ancillary_files["WawHelioIonMP_parameters"], + ) + self.assertEqual( + sentinel.bad_day_list_downloaded_path, + dependency.ancillary_files["bad_days_list"], + ) + self.assertEqual( + sentinel.settings_downloaded_path, + dependency.ancillary_files["pipeline_settings"], + ) self.assertEqual(sentinel.l3a_dictionary_1, dependency.l3a_data[0]) self.assertEqual(sentinel.l3a_dictionary_2, dependency.l3a_data[1]) diff --git a/tests/glows/l3bc/test_glows_l3bc_initializer.py b/tests/glows/l3bc/test_glows_l3bc_initializer.py index a132de81c..152a66b77 100644 --- a/tests/glows/l3bc/test_glows_l3bc_initializer.py +++ b/tests/glows/l3bc/test_glows_l3bc_initializer.py @@ -7,7 +7,10 @@ from imap_data_access import ProcessingInputCollection, RepointInput from imap_data_access.file_validation import Version -from imap_l3_processing.glows.l3bc.glows_l3bc_initializer import GlowsL3BCInitializer, GlowsL3BCInitializerData +from imap_l3_processing.glows.l3bc.glows_l3bc_initializer import ( + GlowsL3BCInitializer, + GlowsL3BCInitializerData, +) from imap_l3_processing.glows.l3bc.models import CRToProcess, ExternalDependencies from tests.test_helpers import create_mock_query_results, get_test_data_path @@ -15,15 +18,20 @@ class TestGlowsL3BCInitializer(unittest.TestCase): def setUp(self): self.external_dependencies_fetch_patcher = patch( - "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.ExternalDependencies.fetch_dependencies") - self.mock_external_dependencies_fetch = self.external_dependencies_fetch_patcher.start() + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.ExternalDependencies.fetch_dependencies" + ) + self.mock_external_dependencies_fetch = ( + self.external_dependencies_fetch_patcher.start() + ) self.download_patcher = patch( - "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.imap_data_access.download") + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.imap_data_access.download" + ) self.download_patcher.start() self.set_global_repoint_table_paths_patcher = patch( - 'imap_l3_processing.glows.l3bc.glows_l3bc_initializer.set_global_repoint_table_paths') + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.set_global_repoint_table_paths" + ) self.set_global_repoint_table_paths_patcher.start() def tearDown(self): @@ -31,41 +39,70 @@ def tearDown(self): self.download_patcher.stop() self.set_global_repoint_table_paths_patcher.stop() - @patch('imap_l3_processing.glows.l3bc.glows_l3bc_initializer.imap_data_access.download') - @patch('imap_l3_processing.glows.l3bc.glows_l3bc_initializer.set_global_repoint_table_paths') - @patch('imap_l3_processing.glows.l3bc.glows_l3bc_initializer.GlowsL3BCDependencies.download_from_cr_to_process') - @patch("imap_l3_processing.glows.l3bc.glows_l3bc_initializer.ExternalDependencies.fetch_dependencies") + @patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.imap_data_access.download" + ) + @patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.set_global_repoint_table_paths" + ) + @patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.GlowsL3BCDependencies.download_from_cr_to_process" + ) + @patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.ExternalDependencies.fetch_dependencies" + ) @patch("imap_l3_processing.glows.l3bc.glows_l3bc_initializer.get_date_range_of_cr") @patch("imap_l3_processing.glows.l3bc.glows_l3bc_initializer.get_best_ancillary") - @patch("imap_l3_processing.glows.l3bc.glows_l3bc_initializer.imap_data_access.query") - @patch("imap_l3_processing.glows.l3bc.glows_l3bc_initializer.GlowsL3BCInitializer.group_l3a_by_cr") - @patch("imap_l3_processing.glows.l3bc.glows_l3bc_initializer.GlowsL3BCInitializer.should_process_cr_candidate") - def test_get_crs_to_process(self, mock_should_process_cr_candidate, mock_group_l3a_by_cr, mock_query, - mock_get_best_ancillary, mock_get_date_range_of_cr, mock_fetch_external_deps, - mock_l3bc_deps_from_cr, mock_set_global_repoint_table_paths, mock_download): + @patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.imap_data_access.query" + ) + @patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.GlowsL3BCInitializer.group_l3a_by_cr" + ) + @patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.GlowsL3BCInitializer.should_process_cr_candidate" + ) + def test_get_crs_to_process( + self, + mock_should_process_cr_candidate, + mock_group_l3a_by_cr, + mock_query, + mock_get_best_ancillary, + mock_get_date_range_of_cr, + mock_fetch_external_deps, + mock_l3bc_deps_from_cr, + mock_set_global_repoint_table_paths, + mock_download, + ): mock_query.side_effect = [ - create_mock_query_results([ - "imap_glows_l3a_hist_20100101_v001.cdf", - "imap_glows_l3a_hist_20100201_v001.cdf" - ]), - create_mock_query_results([ - "imap_glows_l3b_ion-rate-profile_20100101-cr02091_v001.cdf", - "imap_glows_l3b_ion-rate-profile_20100201-cr02092_v001.cdf" - ]), - create_mock_query_results([ - "imap_glows_l3c_sw-profile_20100101-cr02091_v001.cdf", - "imap_glows_l3c_sw-profile_20100201-cr02092_v001.cdf" - ]), + create_mock_query_results( + [ + "imap_glows_l3a_hist_20100101_v001.cdf", + "imap_glows_l3a_hist_20100201_v001.cdf", + ] + ), + create_mock_query_results( + [ + "imap_glows_l3b_ion-rate-profile_20100101-cr02091_v001.cdf", + "imap_glows_l3b_ion-rate-profile_20100201-cr02092_v001.cdf", + ] + ), + create_mock_query_results( + [ + "imap_glows_l3c_sw-profile_20100101-cr02091_v001.cdf", + "imap_glows_l3c_sw-profile_20100201-cr02092_v001.cdf", + ] + ), sentinel.uv_anisotropy_query_result, sentinel.waw_helio_ion_query_result, sentinel.bad_days_list_query_result, - sentinel.pipeline_settings_query_result + sentinel.pipeline_settings_query_result, ] mock_group_l3a_by_cr.return_value = { 2091: {"imap_glows_l3a_hist_20100101_v001.cdf"}, - 2092: {"imap_glows_l3a_hist_20100201_v001.cdf"} + 2092: {"imap_glows_l3a_hist_20100201_v001.cdf"}, } mock_get_date_range_of_cr.side_effect = [ @@ -88,51 +125,114 @@ def test_get_crs_to_process(self, mock_should_process_cr_candidate, mock_group_l expected_l3bs_by_cr = { 2091: "imap_glows_l3b_ion-rate-profile_20100101-cr02091_v001.cdf", - 2092: "imap_glows_l3b_ion-rate-profile_20100201-cr02092_v001.cdf" + 2092: "imap_glows_l3b_ion-rate-profile_20100201-cr02092_v001.cdf", } expected_l3cs_by_cr = { 2091: "imap_glows_l3c_sw-profile_20100101-cr02091_v001.cdf", - 2092: "imap_glows_l3c_sw-profile_20100201-cr02092_v001.cdf" + 2092: "imap_glows_l3c_sw-profile_20100201-cr02092_v001.cdf", } major_version_to_process = 3 l3bc_initializer_data = GlowsL3BCInitializer.get_crs_to_process( ProcessingInputCollection(RepointInput("imap_2026_269_05.repoint.csv")), - major_version_to_process + major_version_to_process, ) mock_download.assert_called_once() - mock_set_global_repoint_table_paths.assert_called_with([mock_download.return_value]) - - mock_query.assert_has_calls([ - call(instrument="glows", data_level="l3a", descriptor="hist", version="latest"), - call(instrument="glows", data_level="l3b", descriptor='ion-rate-profile', version="latest"), - call(instrument="glows", data_level="l3c", descriptor='sw-profile', version="latest"), - call(table="ancillary", instrument="glows", descriptor="uv-anisotropy-1CR"), - call(table="ancillary", instrument="glows", descriptor="WawHelioIonMP"), - call(table="ancillary", instrument="glows", descriptor="bad-days-list"), - call(table="ancillary", instrument="glows", descriptor="pipeline-settings-l3bcde"), - ]) - - mock_group_l3a_by_cr.assert_called_once_with([ - "imap_glows_l3a_hist_20100101_v001.cdf", - "imap_glows_l3a_hist_20100201_v001.cdf", - ]) + mock_set_global_repoint_table_paths.assert_called_with( + [mock_download.return_value] + ) + + mock_query.assert_has_calls( + [ + call( + instrument="glows", + data_level="l3a", + descriptor="hist", + version="latest", + ), + call( + instrument="glows", + data_level="l3b", + descriptor="ion-rate-profile", + version="latest", + ), + call( + instrument="glows", + data_level="l3c", + descriptor="sw-profile", + version="latest", + ), + call( + table="ancillary", + instrument="glows", + descriptor="uv-anisotropy-1CR", + ), + call(table="ancillary", instrument="glows", descriptor="WawHelioIonMP"), + call(table="ancillary", instrument="glows", descriptor="bad-days-list"), + call( + table="ancillary", + instrument="glows", + descriptor="pipeline-settings-l3bcde", + ), + ] + ) + + mock_group_l3a_by_cr.assert_called_once_with( + [ + "imap_glows_l3a_hist_20100101_v001.cdf", + "imap_glows_l3a_hist_20100201_v001.cdf", + ] + ) mock_fetch_external_deps.assert_called_once() - mock_get_best_ancillary.assert_has_calls([ - call(datetime(2010, 1, 1), datetime(2010, 1, 31), sentinel.uv_anisotropy_query_result), - call(datetime(2010, 1, 1), datetime(2010, 1, 31), sentinel.waw_helio_ion_query_result), - call(datetime(2010, 1, 1), datetime(2010, 1, 31), sentinel.bad_days_list_query_result), - call(datetime(2010, 1, 1), datetime(2010, 1, 31), sentinel.pipeline_settings_query_result), - call(datetime(2010, 2, 1), datetime(2010, 2, 28), sentinel.uv_anisotropy_query_result), - call(datetime(2010, 2, 1), datetime(2010, 2, 28), sentinel.waw_helio_ion_query_result), - call(datetime(2010, 2, 1), datetime(2010, 2, 28), sentinel.bad_days_list_query_result), - call(datetime(2010, 2, 1), datetime(2010, 2, 28), sentinel.pipeline_settings_query_result), - ]) + mock_get_best_ancillary.assert_has_calls( + [ + call( + datetime(2010, 1, 1), + datetime(2010, 1, 31), + sentinel.uv_anisotropy_query_result, + ), + call( + datetime(2010, 1, 1), + datetime(2010, 1, 31), + sentinel.waw_helio_ion_query_result, + ), + call( + datetime(2010, 1, 1), + datetime(2010, 1, 31), + sentinel.bad_days_list_query_result, + ), + call( + datetime(2010, 1, 1), + datetime(2010, 1, 31), + sentinel.pipeline_settings_query_result, + ), + call( + datetime(2010, 2, 1), + datetime(2010, 2, 28), + sentinel.uv_anisotropy_query_result, + ), + call( + datetime(2010, 2, 1), + datetime(2010, 2, 28), + sentinel.waw_helio_ion_query_result, + ), + call( + datetime(2010, 2, 1), + datetime(2010, 2, 28), + sentinel.bad_days_list_query_result, + ), + call( + datetime(2010, 2, 1), + datetime(2010, 2, 28), + sentinel.pipeline_settings_query_result, + ), + ] + ) expected_cr_candidate_1 = CRToProcess( l3a_file_names={"imap_glows_l3a_hist_20100101_v001.cdf"}, @@ -140,7 +240,8 @@ def test_get_crs_to_process(self, mock_should_process_cr_candidate, mock_group_l waw_helio_ion_mp_file_name="waw_helio_ion_best_ancillary_1", bad_days_list_file_name="bad_days_list_best_ancillary_1", pipeline_settings_file_name="pipeline_settings_best_ancillary_1", - cr_start_date=datetime(2010, 1, 1), cr_end_date=datetime(2010, 1, 31), + cr_start_date=datetime(2010, 1, 1), + cr_end_date=datetime(2010, 1, 31), cr_rotation_number=2091, ) @@ -150,18 +251,35 @@ def test_get_crs_to_process(self, mock_should_process_cr_candidate, mock_group_l waw_helio_ion_mp_file_name="waw_helio_ion_best_ancillary_2", bad_days_list_file_name="bad_days_list_best_ancillary_2", pipeline_settings_file_name="pipeline_settings_best_ancillary_2", - cr_start_date=datetime(2010, 2, 1), cr_end_date=datetime(2010, 2, 28), + cr_start_date=datetime(2010, 2, 1), + cr_end_date=datetime(2010, 2, 28), cr_rotation_number=2092, ) - mock_should_process_cr_candidate.assert_has_calls([ - call(expected_cr_candidate_1, expected_l3bs_by_cr, mock_fetch_external_deps.return_value, major_version_to_process), - call(expected_cr_candidate_2, expected_l3bs_by_cr, mock_fetch_external_deps.return_value, major_version_to_process) - ], any_order=False) + mock_should_process_cr_candidate.assert_has_calls( + [ + call( + expected_cr_candidate_1, + expected_l3bs_by_cr, + mock_fetch_external_deps.return_value, + major_version_to_process, + ), + call( + expected_cr_candidate_2, + expected_l3bs_by_cr, + mock_fetch_external_deps.return_value, + major_version_to_process, + ), + ], + any_order=False, + ) - mock_l3bc_deps_from_cr.assert_called_once_with(expected_cr_candidate_1, 2, - mock_fetch_external_deps.return_value, - mock_download.return_value) + mock_l3bc_deps_from_cr.assert_called_once_with( + expected_cr_candidate_1, + 2, + mock_fetch_external_deps.return_value, + mock_download.return_value, + ) expected_initializer_data = GlowsL3BCInitializerData( external_dependencies=mock_fetch_external_deps.return_value, @@ -173,25 +291,42 @@ def test_get_crs_to_process(self, mock_should_process_cr_candidate, mock_group_l self.assertEqual(expected_initializer_data, l3bc_initializer_data) - @patch('imap_l3_processing.glows.l3bc.glows_l3bc_initializer.imap_data_access.download') - @patch("imap_l3_processing.glows.l3bc.glows_l3bc_initializer.GlowsL3BCInitializer.group_l3a_by_cr") - @patch("imap_l3_processing.glows.l3bc.glows_l3bc_initializer.imap_data_access.query") - def test_get_crs_to_process_return_no_crs_when_missing_ancillaries(self, mock_query, mock_group_l3a_by_cr, - mock_download): + @patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.imap_data_access.download" + ) + @patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.GlowsL3BCInitializer.group_l3a_by_cr" + ) + @patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.imap_data_access.query" + ) + def test_get_crs_to_process_return_no_crs_when_missing_ancillaries( + self, mock_query, mock_group_l3a_by_cr, mock_download + ): test_cases = [ "uv-anisotropy-1CR", "WawHelioIonMP", "bad-days-list", - "pipeline-settings-l3bcde" + "pipeline-settings-l3bcde", ] for missing_ancillary in test_cases: with self.subTest(missing_ancillary): + def query_that_returns_empty_list_for_missing_ancillary(**kwargs): - if kwargs.get("table") == "ancillary" and kwargs["descriptor"] != missing_ancillary: - return [{"start_date": "20100101", "end_date": None, "ingestion_date": "20100101 00:00:00", - "file_path": f"some/ancillary/{missing_ancillary}"}] + if ( + kwargs.get("table") == "ancillary" + and kwargs["descriptor"] != missing_ancillary + ): + return [ + { + "start_date": "20100101", + "end_date": None, + "ingestion_date": "20100101 00:00:00", + "file_path": f"some/ancillary/{missing_ancillary}", + } + ] else: return [] @@ -199,42 +334,96 @@ def query_that_returns_empty_list_for_missing_ancillary(**kwargs): 2091: {"glows_l3a_hist_20100101_v001.cdf"} } - mock_query.side_effect = query_that_returns_empty_list_for_missing_ancillary + mock_query.side_effect = ( + query_that_returns_empty_list_for_missing_ancillary + ) actual_crs_to_process = GlowsL3BCInitializer.get_crs_to_process( - ProcessingInputCollection(RepointInput("imap_2026_269_05.repoint.csv")), sentinel.major_version) - - mock_query.assert_has_calls([ - call(instrument="glows", data_level="l3a", descriptor="hist", version="latest"), - call(instrument="glows", data_level="l3b", descriptor="ion-rate-profile", version="latest"), - call(instrument="glows", data_level="l3c", descriptor="sw-profile", version="latest"), - call(table="ancillary", instrument="glows", descriptor="uv-anisotropy-1CR"), - call(table="ancillary", instrument="glows", descriptor="WawHelioIonMP"), - call(table="ancillary", instrument="glows", descriptor="bad-days-list"), - call(table="ancillary", instrument="glows", descriptor="pipeline-settings-l3bcde") - ]) + ProcessingInputCollection( + RepointInput("imap_2026_269_05.repoint.csv") + ), + sentinel.major_version, + ) + + mock_query.assert_has_calls( + [ + call( + instrument="glows", + data_level="l3a", + descriptor="hist", + version="latest", + ), + call( + instrument="glows", + data_level="l3b", + descriptor="ion-rate-profile", + version="latest", + ), + call( + instrument="glows", + data_level="l3c", + descriptor="sw-profile", + version="latest", + ), + call( + table="ancillary", + instrument="glows", + descriptor="uv-anisotropy-1CR", + ), + call( + table="ancillary", + instrument="glows", + descriptor="WawHelioIonMP", + ), + call( + table="ancillary", + instrument="glows", + descriptor="bad-days-list", + ), + call( + table="ancillary", + instrument="glows", + descriptor="pipeline-settings-l3bcde", + ), + ] + ) expected_initializer_result = GlowsL3BCInitializerData( external_dependencies=self.mock_external_dependencies_fetch.return_value, l3bc_dependencies=[], l3bs_by_cr={}, l3cs_by_cr={}, - repoint_file_path=mock_download.return_value + repoint_file_path=mock_download.return_value, ) self.assertEqual(expected_initializer_result, actual_crs_to_process) - @patch("imap_l3_processing.glows.l3bc.glows_l3bc_initializer.GlowsL3BCInitializer.should_process_cr_candidate", - return_value=True) + @patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.GlowsL3BCInitializer.should_process_cr_candidate", + return_value=True, + ) @patch("imap_l3_processing.glows.l3bc.glows_l3bc_initializer.get_best_ancillary") - @patch('imap_l3_processing.glows.l3bc.glows_l3bc_initializer.imap_data_access.download') - @patch("imap_l3_processing.glows.l3bc.glows_l3bc_initializer.imap_data_access.query") - @patch("imap_l3_processing.glows.l3bc.glows_l3bc_initializer.ExternalDependencies.fetch_dependencies") - @patch("imap_l3_processing.glows.l3bc.glows_l3bc_initializer.GlowsL3BCInitializer.group_l3a_by_cr") - def test_get_crs_to_process_returns_no_crs_when_missing_external_deps(self, mock_group_l3a_by_cr, - mock_fetch_ext_dependencies, mock_query, - mock_download, ___, - ____): + @patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.imap_data_access.download" + ) + @patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.imap_data_access.query" + ) + @patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.ExternalDependencies.fetch_dependencies" + ) + @patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.GlowsL3BCInitializer.group_l3a_by_cr" + ) + def test_get_crs_to_process_returns_no_crs_when_missing_external_deps( + self, + mock_group_l3a_by_cr, + mock_fetch_ext_dependencies, + mock_query, + mock_download, + ___, + ____, + ): test_cases = [ ExternalDependencies(None, Path("some_lyman"), Path("some_omni")), ExternalDependencies(Path("some_f107"), None, Path("some_omni")), @@ -246,37 +435,94 @@ def test_get_crs_to_process_returns_no_crs_when_missing_external_deps(self, mock mock_query.reset_mock() mock_fetch_ext_dependencies.return_value = external_deps - mock_group_l3a_by_cr.return_value = {2091: {"glows_l3a_hist_20100101_v001.cdf"}} + mock_group_l3a_by_cr.return_value = { + 2091: {"glows_l3a_hist_20100101_v001.cdf"} + } mock_query.side_effect = [ - create_mock_query_results(["imap_glows_l3a_hist_20100101_v001.cdf"]), - create_mock_query_results(["imap_glows_l3b_ion-rate-profile_20100101-cr00001_v001.cdf"]), - create_mock_query_results(["imap_glows_l3c_sw-profile_20100101-cr00001_v001.cdf"]), - create_mock_query_results(["imap_glows_uv-anisotropy-1CR_20100101_v001.dat"]), - create_mock_query_results(["imap_glows_WawHelioIonMP_20100101_v001.dat"]), - create_mock_query_results(["imap_glows_bad-days-list_20100101_v001.dat"]), - create_mock_query_results(["imap_glows_pipeline-settings-l3bcde_20100101_v003.json"]) + create_mock_query_results( + ["imap_glows_l3a_hist_20100101_v001.cdf"] + ), + create_mock_query_results( + ["imap_glows_l3b_ion-rate-profile_20100101-cr00001_v001.cdf"] + ), + create_mock_query_results( + ["imap_glows_l3c_sw-profile_20100101-cr00001_v001.cdf"] + ), + create_mock_query_results( + ["imap_glows_uv-anisotropy-1CR_20100101_v001.dat"] + ), + create_mock_query_results( + ["imap_glows_WawHelioIonMP_20100101_v001.dat"] + ), + create_mock_query_results( + ["imap_glows_bad-days-list_20100101_v001.dat"] + ), + create_mock_query_results( + ["imap_glows_pipeline-settings-l3bcde_20100101_v003.json"] + ), ] actual_crs_to_process = GlowsL3BCInitializer.get_crs_to_process( - ProcessingInputCollection(RepointInput("imap_2026_269_05.repoint.csv")), sentinel.major_version) - - mock_query.assert_has_calls([ - call(instrument="glows", data_level="l3a", descriptor="hist", version="latest"), - call(instrument="glows", data_level="l3b", descriptor="ion-rate-profile", version="latest"), - call(instrument="glows", data_level="l3c", descriptor="sw-profile", version="latest"), - call(table="ancillary", instrument="glows", descriptor="uv-anisotropy-1CR"), - call(table="ancillary", instrument="glows", descriptor="WawHelioIonMP"), - call(table="ancillary", instrument="glows", descriptor="bad-days-list"), - call(table="ancillary", instrument="glows", descriptor="pipeline-settings-l3bcde"), - ]) + ProcessingInputCollection( + RepointInput("imap_2026_269_05.repoint.csv") + ), + sentinel.major_version, + ) + + mock_query.assert_has_calls( + [ + call( + instrument="glows", + data_level="l3a", + descriptor="hist", + version="latest", + ), + call( + instrument="glows", + data_level="l3b", + descriptor="ion-rate-profile", + version="latest", + ), + call( + instrument="glows", + data_level="l3c", + descriptor="sw-profile", + version="latest", + ), + call( + table="ancillary", + instrument="glows", + descriptor="uv-anisotropy-1CR", + ), + call( + table="ancillary", + instrument="glows", + descriptor="WawHelioIonMP", + ), + call( + table="ancillary", + instrument="glows", + descriptor="bad-days-list", + ), + call( + table="ancillary", + instrument="glows", + descriptor="pipeline-settings-l3bcde", + ), + ] + ) expected_initializer_result = GlowsL3BCInitializerData( external_dependencies=external_deps, l3bc_dependencies=[], - l3bs_by_cr={1: "imap_glows_l3b_ion-rate-profile_20100101-cr00001_v001.cdf"}, - l3cs_by_cr={1: "imap_glows_l3c_sw-profile_20100101-cr00001_v001.cdf"}, - repoint_file_path=mock_download.return_value + l3bs_by_cr={ + 1: "imap_glows_l3b_ion-rate-profile_20100101-cr00001_v001.cdf" + }, + l3cs_by_cr={ + 1: "imap_glows_l3c_sw-profile_20100101-cr00001_v001.cdf" + }, + repoint_file_path=mock_download.return_value, ) self.assertEqual(expected_initializer_result, actual_crs_to_process) @@ -291,7 +537,6 @@ def test_should_process_cr_excludes_crs_within_buffer_period(self): cr_start_date=datetime(2012, 2, 6, 3, 36, 31, 680000), cr_end_date=datetime(2012, 3, 4, 10, 12, 57, 600000), cr_rotation_number=2120, - ) cr_candidate.buffer_time_has_elapsed_since_cr = Mock(return_value=False) cr_candidate.has_valid_external_dependencies = Mock(return_value=True) @@ -302,7 +547,11 @@ def test_should_process_cr_excludes_crs_within_buffer_period(self): omni2_data_path=Path("omni2_data_path"), ) - self.assertIsNone(GlowsL3BCInitializer.should_process_cr_candidate(cr_candidate, {}, external_dependencies, 1)) + self.assertIsNone( + GlowsL3BCInitializer.should_process_cr_candidate( + cr_candidate, {}, external_dependencies, 1 + ) + ) def test_should_process_cr_no_existing_l3b(self): @@ -314,7 +563,7 @@ def test_should_process_cr_no_existing_l3b(self): pipeline_settings_file_name="pipeline_settings.json", cr_start_date=datetime(2010, 1, 1), cr_end_date=datetime(2010, 2, 1), - cr_rotation_number=2091 + cr_rotation_number=2091, ) cr_candidate.buffer_time_has_elapsed_since_cr = Mock(return_value=True) cr_candidate.has_valid_external_dependencies = Mock(return_value=True) @@ -325,7 +574,9 @@ def test_should_process_cr_no_existing_l3b(self): omni2_data_path=Path("omni2_data_path"), ) - actual_version = GlowsL3BCInitializer.should_process_cr_candidate(cr_candidate, {}, external_dependencies, major_version=2) + actual_version = GlowsL3BCInitializer.should_process_cr_candidate( + cr_candidate, {}, external_dependencies, major_version=2 + ) self.assertEqual(Version(2, 1), actual_version) @@ -335,7 +586,7 @@ def test_should_process_cr_existing_cr_should_process(self, mock_read_cdf_parent new_l3a_version = { "l3a_file_names": { "imap_glows_l3a_hist_20100101-repoint000001_v001.cdf", - "imap_glows_l3a_hist_20100102-repoint000002_v002.cdf" + "imap_glows_l3a_hist_20100102-repoint000002_v002.cdf", } } @@ -343,19 +594,17 @@ def test_should_process_cr_existing_cr_should_process(self, mock_read_cdf_parent "l3a_file_names": { "imap_glows_l3a_hist_20100101-repoint000001_v001.cdf", "imap_glows_l3a_hist_20100102-repoint000002_v001.cdf" - "imap_glows_l3a_hist_20100103-repoint000002_v001.cdf" + "imap_glows_l3a_hist_20100103-repoint000002_v001.cdf", } } - new_ancillary = { - "uv_anisotropy_file_name": "new_uv_anisotropy.dat" - } + new_ancillary = {"uv_anisotropy_file_name": "new_uv_anisotropy.dat"} test_cases = [ ("new l3a file version", new_l3a_version, 2), ("new l3a files arrived", new_l3a_file, 2), ("new ancillary file arrived", new_ancillary, 2), - ("new major version", {}, 3) + ("new major version", {}, 3), ] for name, new_server_data, new_major_version in test_cases: @@ -368,7 +617,7 @@ def test_should_process_cr_existing_cr_should_process(self, mock_read_cdf_parent already_processed_cr = CRToProcess( l3a_file_names={ "imap_glows_l3a_hist_20100101-repoint000001_v001.cdf", - "imap_glows_l3a_hist_20100102-repoint000002_v001.cdf" + "imap_glows_l3a_hist_20100102-repoint000002_v001.cdf", }, uv_anisotropy_file_name="uv_anisotropy.dat", waw_helio_ion_mp_file_name="waw_helio_ion.dat", @@ -385,7 +634,9 @@ def test_should_process_cr_existing_cr_should_process(self, mock_read_cdf_parent omni2_data_path=Path("omni2_data_path"), ) - cr_candidate = dataclasses.replace(already_processed_cr, **new_server_data) + cr_candidate = dataclasses.replace( + already_processed_cr, **new_server_data + ) cr_candidate.buffer_time_has_elapsed_since_cr = Mock(return_value=True) cr_candidate.has_valid_external_dependencies = Mock(return_value=True) @@ -396,26 +647,36 @@ def test_should_process_cr_existing_cr_should_process(self, mock_read_cdf_parent "waw_helio_ion.dat", "bad_days_list.dat", "pipeline_settings.json", - "some_zip_file.zip" + "some_zip_file.zip", } - existing_file_version = Version(existing_file_major_version, existing_file_minor_version) - l3bs = {2091: f"imap_glows_l3b_ion-rate-profile_20100101-cr02091_{existing_file_version}.cdf"} - actual_version = GlowsL3BCInitializer.should_process_cr_candidate(cr_candidate, l3bs, - external_dependencies, new_major_version) - expected_version = Version(new_major_version, existing_file_minor_version + 1) + existing_file_version = Version( + existing_file_major_version, existing_file_minor_version + ) + l3bs = { + 2091: f"imap_glows_l3b_ion-rate-profile_20100101-cr02091_{existing_file_version}.cdf" + } + actual_version = GlowsL3BCInitializer.should_process_cr_candidate( + cr_candidate, l3bs, external_dependencies, new_major_version + ) + expected_version = Version( + new_major_version, existing_file_minor_version + 1 + ) mock_read_cdf_parents.assert_called_once_with( - f"imap_glows_l3b_ion-rate-profile_20100101-cr02091_{existing_file_version}.cdf") + f"imap_glows_l3b_ion-rate-profile_20100101-cr02091_{existing_file_version}.cdf" + ) self.assertEqual(expected_version, actual_version) @patch("imap_l3_processing.glows.l3bc.glows_l3bc_initializer.read_cdf_parents") - def test_should_process_cr_existing_cr_should_not_reprocess(self, mock_read_cdf_parents): + def test_should_process_cr_existing_cr_should_not_reprocess( + self, mock_read_cdf_parents + ): cr_candidate = CRToProcess( l3a_file_names={ "imap_glows_l3a_hist_20100101-repoint000001_v001.cdf", - "imap_glows_l3a_hist_20100102-repoint000002_v001.cdf" + "imap_glows_l3a_hist_20100102-repoint000002_v001.cdf", }, uv_anisotropy_file_name="uv_anisotropy.dat", waw_helio_ion_mp_file_name="waw_helio_ion.dat", @@ -428,7 +689,9 @@ def test_should_process_cr_existing_cr_should_not_reprocess(self, mock_read_cdf_ cr_candidate.buffer_time_has_elapsed_since_cr = Mock(return_value=True) cr_candidate.has_valid_external_dependencies = Mock(return_value=True) existing_version = Version(2, 1) - l3bs = {2091: f"imap_glows_l3b_ion-rate-profile_20100101-cr02091_{existing_version}.cdf"} + l3bs = { + 2091: f"imap_glows_l3b_ion-rate-profile_20100101-cr02091_{existing_version}.cdf" + } external_dependencies = ExternalDependencies( f107_index_file_path=Path("f107_index_file_path"), @@ -443,12 +706,18 @@ def test_should_process_cr_existing_cr_should_not_reprocess(self, mock_read_cdf_ "waw_helio_ion.dat", "bad_days_list.dat", "pipeline_settings.json", - "some_zip_file.zip" + "some_zip_file.zip", } - self.assertIsNone(GlowsL3BCInitializer.should_process_cr_candidate(cr_candidate, l3bs, external_dependencies, existing_version.major)) + self.assertIsNone( + GlowsL3BCInitializer.should_process_cr_candidate( + cr_candidate, l3bs, external_dependencies, existing_version.major + ) + ) - mock_read_cdf_parents.assert_called_once_with(f"imap_glows_l3b_ion-rate-profile_20100101-cr02091_{existing_version}.cdf") + mock_read_cdf_parents.assert_called_once_with( + f"imap_glows_l3b_ion-rate-profile_20100101-cr02091_{existing_version}.cdf" + ) def test_should_process_cr_with_invalid_dependencies(self): cr_candidate = CRToProcess( @@ -471,35 +740,42 @@ def test_should_process_cr_with_invalid_dependencies(self): omni2_data_path=Path("omni2_data_path"), ) - self.assertIsNone(GlowsL3BCInitializer.should_process_cr_candidate(cr_candidate, {}, external_dependencies, 1)) + self.assertIsNone( + GlowsL3BCInitializer.should_process_cr_candidate( + cr_candidate, {}, external_dependencies, 1 + ) + ) - cr_candidate.has_valid_external_dependencies.assert_called_once_with(external_dependencies) + cr_candidate.has_valid_external_dependencies.assert_called_once_with( + external_dependencies + ) - @patch('imap_l3_processing.glows.l3bc.glows_l3bc_initializer.imap_data_access.download') + @patch( + "imap_l3_processing.glows.l3bc.glows_l3bc_initializer.imap_data_access.download" + ) def test_group_l3a_by_cr(self, mock_download): - l3a_1 = 'imap_glows_l3a_hist_20100101-repoint00001_v001.cdf' - l3a_2 = 'imap_glows_l3a_hist_20100102-repoint00002_v001.cdf' - l3a_3 = 'imap_glows_l3a_hist_20100103-repoint00003_v001.cdf' - l3a_4 = 'imap_glows_l3a_hist_20100104-repoint00004_v001.cdf' + l3a_1 = "imap_glows_l3a_hist_20100101-repoint00001_v001.cdf" + l3a_2 = "imap_glows_l3a_hist_20100102-repoint00002_v001.cdf" + l3a_3 = "imap_glows_l3a_hist_20100103-repoint00003_v001.cdf" + l3a_4 = "imap_glows_l3a_hist_20100104-repoint00004_v001.cdf" glows_l3a_files = [ - get_test_data_path('glows/l3a_products') / l3a_1, - get_test_data_path('glows/l3a_products') / l3a_2, - get_test_data_path('glows/l3a_products') / l3a_3, - get_test_data_path('glows/l3a_products') / l3a_4, + get_test_data_path("glows/l3a_products") / l3a_1, + get_test_data_path("glows/l3a_products") / l3a_2, + get_test_data_path("glows/l3a_products") / l3a_3, + get_test_data_path("glows/l3a_products") / l3a_4, ] mock_download.side_effect = glows_l3a_files - expected = { - 2091: {l3a_1, l3a_2, l3a_3}, - 2092: {l3a_4} - } + expected = {2091: {l3a_1, l3a_2, l3a_3}, 2092: {l3a_4}} - grouped_l3a_files = GlowsL3BCInitializer.group_l3a_by_cr([l3a_1, l3a_2, l3a_3, l3a_4]) + grouped_l3a_files = GlowsL3BCInitializer.group_l3a_by_cr( + [l3a_1, l3a_2, l3a_3, l3a_4] + ) - mock_download.assert_has_calls([ - call(l3a_1), call(l3a_2), call(l3a_3), call(l3a_4) - ]) + mock_download.assert_has_calls( + [call(l3a_1), call(l3a_2), call(l3a_3), call(l3a_4)] + ) self.assertEqual(expected, grouped_l3a_files) diff --git a/tests/glows/l3bc/test_models.py b/tests/glows/l3bc/test_models.py index 4d408d576..4aab5333f 100644 --- a/tests/glows/l3bc/test_models.py +++ b/tests/glows/l3bc/test_models.py @@ -9,18 +9,29 @@ import numpy as np from spacepy import pycdf -from imap_l3_processing.constants import TEMP_CDF_FOLDER_PATH, CARRINGTON_ROTATION_IN_NANOSECONDS -from imap_l3_processing.glows.l3bc.models import GlowsL3BIonizationRate, GlowsL3CSolarWind, CRToProcess, \ - ExternalDependencies, F107_FLUX_TABLE_URL, LYMAN_ALPHA_COMPOSITE_INDEX_URL, OMNI2_URL +from imap_l3_processing.constants import ( + TEMP_CDF_FOLDER_PATH, + CARRINGTON_ROTATION_IN_NANOSECONDS, +) +from imap_l3_processing.glows.l3bc.models import ( + GlowsL3BIonizationRate, + GlowsL3CSolarWind, + CRToProcess, + ExternalDependencies, + F107_FLUX_TABLE_URL, + LYMAN_ALPHA_COMPOSITE_INDEX_URL, + OMNI2_URL, +) from tests.swapi.cdf_model_test_case import CdfModelTestCase from tests.test_helpers import get_test_instrument_team_data_path class TestModels(CdfModelTestCase): - @patch("imap_l3_processing.glows.l3bc.models.ExternalDependencies._comment_headers") @patch("imap_l3_processing.glows.l3bc.models.download_external_dependency") - def test_external_dependencies_fetch_dependencies(self, mock_download_external_dependencies, mock_comment_header): + def test_external_dependencies_fetch_dependencies( + self, mock_download_external_dependencies, mock_comment_header + ): mock_download_external_dependencies.side_effect = [ Path("external/file/f107_path"), Path("external/file/lya_path"), @@ -29,11 +40,16 @@ def test_external_dependencies_fetch_dependencies(self, mock_download_external_d external_dependencies = ExternalDependencies.fetch_dependencies() - mock_download_external_dependencies.assert_has_calls([ - call(F107_FLUX_TABLE_URL, TEMP_CDF_FOLDER_PATH / 'f107_fluxtable.txt'), - call(LYMAN_ALPHA_COMPOSITE_INDEX_URL, TEMP_CDF_FOLDER_PATH / 'lyman_alpha.txt'), - call(OMNI2_URL, TEMP_CDF_FOLDER_PATH / 'omni2.txt') - ]) + mock_download_external_dependencies.assert_has_calls( + [ + call(F107_FLUX_TABLE_URL, TEMP_CDF_FOLDER_PATH / "f107_fluxtable.txt"), + call( + LYMAN_ALPHA_COMPOSITE_INDEX_URL, + TEMP_CDF_FOLDER_PATH / "lyman_alpha.txt", + ), + call(OMNI2_URL, TEMP_CDF_FOLDER_PATH / "omni2.txt"), + ] + ) mock_comment_header.assert_called_once_with(Path("external/file/f107_path")) @@ -45,7 +61,9 @@ def test_external_dependencies_fetch_dependencies(self, mock_download_external_d self.assertEqual(expected_external_deps, external_dependencies) @patch("imap_l3_processing.glows.l3bc.models.download_external_dependency") - def test_external_dependencies_files_not_found(self, mock_download_external_dependencies): + def test_external_dependencies_files_not_found( + self, mock_download_external_dependencies + ): mock_download_external_dependencies.return_value = None external_dependencies = ExternalDependencies.fetch_dependencies() @@ -53,21 +71,22 @@ def test_external_dependencies_files_not_found(self, mock_download_external_depe self.assertEqual(ExternalDependencies(None, None, None), external_dependencies) @patch("imap_l3_processing.glows.l3bc.models.download_external_dependency") - def test_external_dependencies_comment_headers(self, mock_download_external_dependencies): + def test_external_dependencies_comment_headers( + self, mock_download_external_dependencies + ): with tempfile.TemporaryDirectory() as tmpdir: file_name = Path(tmpdir) / "flux_table.txt" with open(file_name, "w") as fp: - fp.writelines([ - "fluxdate fluxtime fluxjulian fluxcarrington fluxobsflux fluxadjflux fluxursi\n", - "---------- ---------- ------------ -------------- ----------- ----------- ----------\n", - "20041028 170000 02453307.229 002022.605 000132.7 000130.9 000117.8\n", - "20041028 200000 02453307.354 002022.610 000135.8 000134.0 000120.6\n"]) - - mock_download_external_dependencies.side_effect = [ - file_name, - None, - None - ] + fp.writelines( + [ + "fluxdate fluxtime fluxjulian fluxcarrington fluxobsflux fluxadjflux fluxursi\n", + "---------- ---------- ------------ -------------- ----------- ----------- ----------\n", + "20041028 170000 02453307.229 002022.605 000132.7 000130.9 000117.8\n", + "20041028 200000 02453307.354 002022.610 000135.8 000134.0 000120.6\n", + ] + ) + + mock_download_external_dependencies.side_effect = [file_name, None, None] ExternalDependencies.fetch_dependencies() @@ -78,7 +97,9 @@ def test_external_dependencies_comment_headers(self, mock_download_external_depe @patch("imap_l3_processing.glows.l3bc.models.datetime") @patch("imap_l3_processing.glows.l3bc.models.imap_data_access.download") - def test_cr_to_process_buffer_time_has_elapsed_since_cr(self, mock_download, mock_datetime): + def test_cr_to_process_buffer_time_has_elapsed_since_cr( + self, mock_download, mock_datetime + ): with TemporaryDirectory() as temp_dir: temp_dir = Path(temp_dir) fake_pipeline_settings = temp_dir / "fake_pipeline_settings.json" @@ -99,10 +120,7 @@ def test_cr_to_process_buffer_time_has_elapsed_since_cr(self, mock_download, moc cr_rotation_number=2091, ) - test_cases = [ - (datetime(2010, 2, 28), False), - (datetime(2010, 3, 10), True) - ] + test_cases = [(datetime(2010, 2, 28), False), (datetime(2010, 3, 10), True)] for current_time, expected_buffer_time_has_elapsed_since_cr in test_cases: with self.subTest(current_time): @@ -118,7 +136,9 @@ def test_cr_to_process_buffer_time_has_elapsed_since_cr(self, mock_download, moc @patch("imap_l3_processing.glows.l3bc.models.validate_dependencies") @patch("imap_l3_processing.glows.l3bc.models.imap_data_access.download") - def test_cr_to_process_has_valid_external_dependencies(self, mock_download, mock_validate_dependencies): + def test_cr_to_process_has_valid_external_dependencies( + self, mock_download, mock_validate_dependencies + ): with TemporaryDirectory() as temp_dir: temp_dir = Path(temp_dir) fake_pipeline_settings = temp_dir / "fake_pipeline_settings.json" @@ -136,16 +156,18 @@ def test_cr_to_process_has_valid_external_dependencies(self, mock_download, mock pipeline_settings_file_name="pipeline_settings.json", cr_start_date=datetime(2010, 1, 1), cr_end_date=datetime(2010, 2, 1), - cr_rotation_number=2091 + cr_rotation_number=2091, ) external_dependencies = ExternalDependencies( f107_index_file_path=Path("f107_index_file_path"), lyman_alpha_path=Path("lyman_alpha_path"), - omni2_data_path=Path("omni2_data_path") + omni2_data_path=Path("omni2_data_path"), ) - actual_has_valid_ext_deps = cr_to_process.has_valid_external_dependencies(external_dependencies) + actual_has_valid_ext_deps = cr_to_process.has_valid_external_dependencies( + external_dependencies + ) mock_validate_dependencies.assert_called_once_with( datetime(2010, 1, 1), @@ -155,191 +177,372 @@ def test_cr_to_process_has_valid_external_dependencies(self, mock_download, mock Path("f107_index_file_path"), Path("lyman_alpha_path"), ) - self.assertEqual(mock_validate_dependencies.return_value, actual_has_valid_ext_deps) + self.assertEqual( + mock_validate_dependencies.return_value, actual_has_valid_ext_deps + ) def test_l3b_to_data_product_variables(self): - data = GlowsL3BIonizationRate(input_metadata=sentinel.input_metadata, - epoch=sentinel.epoch, - epoch_delta=sentinel.epoch_delta, - mean_time=sentinel.mean_time, - cr=sentinel.cr, - uv_anisotropy_factor=sentinel.uv_anisotropy_factor, - lat_grid=sentinel.lat_grid, - sum_rate=sentinel.sum_rate, - ph_rate=sentinel.ph_rate, - cx_rate=sentinel.cx_rate, - sum_uncert=sentinel.sum_uncert, - ph_uncert=sentinel.ph_uncert, - cx_uncert=sentinel.cx_uncert, - lat_grid_label=sentinel.lat_grid_label, - uv_anisotropy_flag=sentinel.uv_anisotropy_flag, - used_l3a=sentinel.used_l3a, - glows_flags=sentinel.glows_flags, - ) + data = GlowsL3BIonizationRate( + input_metadata=sentinel.input_metadata, + epoch=sentinel.epoch, + epoch_delta=sentinel.epoch_delta, + mean_time=sentinel.mean_time, + cr=sentinel.cr, + uv_anisotropy_factor=sentinel.uv_anisotropy_factor, + lat_grid=sentinel.lat_grid, + sum_rate=sentinel.sum_rate, + ph_rate=sentinel.ph_rate, + cx_rate=sentinel.cx_rate, + sum_uncert=sentinel.sum_uncert, + ph_uncert=sentinel.ph_uncert, + cx_uncert=sentinel.cx_uncert, + lat_grid_label=sentinel.lat_grid_label, + uv_anisotropy_flag=sentinel.uv_anisotropy_flag, + used_l3a=sentinel.used_l3a, + glows_flags=sentinel.glows_flags, + ) variables = data.to_data_product_variables() variables = iter(variables) self.assert_variable_attributes(next(variables), sentinel.epoch, "epoch") - self.assert_variable_attributes(next(variables), sentinel.epoch_delta, "epoch_delta") - self.assert_variable_attributes(next(variables), sentinel.mean_time, "mean_time") + self.assert_variable_attributes( + next(variables), sentinel.epoch_delta, "epoch_delta" + ) + self.assert_variable_attributes( + next(variables), sentinel.mean_time, "mean_time" + ) self.assert_variable_attributes(next(variables), sentinel.cr, "cr") - self.assert_variable_attributes(next(variables), sentinel.uv_anisotropy_factor, "uv_anisotropy_factor") + self.assert_variable_attributes( + next(variables), sentinel.uv_anisotropy_factor, "uv_anisotropy_factor" + ) self.assert_variable_attributes(next(variables), sentinel.lat_grid, "lat_grid") self.assert_variable_attributes(next(variables), sentinel.sum_rate, "sum_rate") self.assert_variable_attributes(next(variables), sentinel.ph_rate, "ph_rate") self.assert_variable_attributes(next(variables), sentinel.cx_rate, "cx_rate") - self.assert_variable_attributes(next(variables), sentinel.sum_uncert, "sum_uncert") - self.assert_variable_attributes(next(variables), sentinel.ph_uncert, "ph_uncert") - self.assert_variable_attributes(next(variables), sentinel.cx_uncert, "cx_uncert") - self.assert_variable_attributes(next(variables), sentinel.lat_grid_label, "lat_grid_label") - self.assert_variable_attributes(next(variables), sentinel.uv_anisotropy_flag, "uv_anisotropy_flag") + self.assert_variable_attributes( + next(variables), sentinel.sum_uncert, "sum_uncert" + ) + self.assert_variable_attributes( + next(variables), sentinel.ph_uncert, "ph_uncert" + ) + self.assert_variable_attributes( + next(variables), sentinel.cx_uncert, "cx_uncert" + ) + self.assert_variable_attributes( + next(variables), sentinel.lat_grid_label, "lat_grid_label" + ) + self.assert_variable_attributes( + next(variables), sentinel.uv_anisotropy_flag, "uv_anisotropy_flag" + ) self.assert_variable_attributes(next(variables), sentinel.used_l3a, "used_l3a") - self.assert_variable_attributes(next(variables), sentinel.glows_flags, "glows_flags") + self.assert_variable_attributes( + next(variables), sentinel.glows_flags, "glows_flags" + ) self.assertIsNone(next(variables, None)) def test_l3b_from_instrument_team_dictionary(self): - glows_instrument_team_data_path = get_test_instrument_team_data_path('glows') - with open(glows_instrument_team_data_path / 'imap_glows_l3b_cr_2091_v00.json') as f: + glows_instrument_team_data_path = get_test_instrument_team_data_path("glows") + with open( + glows_instrument_team_data_path / "imap_glows_l3b_cr_2091_v00.json" + ) as f: instrument_team_l3b_dict = json.load(f) - instrument_team_l3b_dict['header']['ancillary_data_files'][ - 'pipeline_settings'] = glows_instrument_team_data_path / 'imap_glows_pipeline-settings-l3bcde_20250514_v004.json' + instrument_team_l3b_dict["header"]["ancillary_data_files"][ + "pipeline_settings" + ] = ( + glows_instrument_team_data_path + / "imap_glows_pipeline-settings-l3bcde_20250514_v004.json" + ) - result = GlowsL3BIonizationRate.from_instrument_team_dictionary(instrument_team_l3b_dict, - sentinel.input_metadata) + result = GlowsL3BIonizationRate.from_instrument_team_dictionary( + instrument_team_l3b_dict, sentinel.input_metadata + ) - latitude_grid = np.array(instrument_team_l3b_dict["ion_rate_profile"]["lat_grid"]) + latitude_grid = np.array( + instrument_team_l3b_dict["ion_rate_profile"]["lat_grid"] + ) self.assertIsInstance(result, GlowsL3BIonizationRate) self.assertEqual(sentinel.input_metadata, result.input_metadata) - np.testing.assert_array_equal([datetime.fromisoformat("2009-12-20 20:14:51.360")], result.epoch) - np.testing.assert_equal([CARRINGTON_ROTATION_IN_NANOSECONDS / 2], result.epoch_delta) - np.testing.assert_equal([datetime.fromisoformat("2010-01-02 03:43:28.667")], result.mean_time) + np.testing.assert_array_equal( + [datetime.fromisoformat("2009-12-20 20:14:51.360")], result.epoch + ) + np.testing.assert_equal( + [CARRINGTON_ROTATION_IN_NANOSECONDS / 2], result.epoch_delta + ) + np.testing.assert_equal( + [datetime.fromisoformat("2010-01-02 03:43:28.667")], result.mean_time + ) self.assertEqual([10000], result.uv_anisotropy_flag) self.assertEqual([2091], result.cr) - np.testing.assert_equal([instrument_team_l3b_dict["uv_anisotropy_factor"]], result.uv_anisotropy_factor) + np.testing.assert_equal( + [instrument_team_l3b_dict["uv_anisotropy_factor"]], + result.uv_anisotropy_factor, + ) np.testing.assert_equal(latitude_grid, result.lat_grid) - np.testing.assert_equal([instrument_team_l3b_dict["ion_rate_profile"]["sum_rate"]], result.sum_rate) - np.testing.assert_equal([instrument_team_l3b_dict["ion_rate_profile"]["ph_rate"]], result.ph_rate) - np.testing.assert_equal([instrument_team_l3b_dict["ion_rate_profile"]["cx_rate"]], result.cx_rate) - np.testing.assert_equal([instrument_team_l3b_dict["ion_rate_profile"]["sum_uncert"]], result.sum_uncert) - np.testing.assert_equal([instrument_team_l3b_dict["ion_rate_profile"]["ph_uncert"]], result.ph_uncert) - np.testing.assert_equal([instrument_team_l3b_dict["ion_rate_profile"]["cx_uncert"]], result.cx_uncert) np.testing.assert_equal( - ['-90°', '-80°', '-70°', '-60°', '-50°', '-40°', '-30°', '-20°', '-10°', '0°', '10°', '20°', '30°', '40°', - '50°', '60°', '70°', '80°', '90°'], result.lat_grid_label) - self.assertEqual([ - 'imap_glows_WawHelioIonMP_v002.json', - 'imap_glows_bad-days-list_v001.dat', - 'imap_glows_pipeline-settings-l3bcde_20250514_v004.json', - 'imap_glows_uv-anisotropy-1CR_v002.json', - 'imap_glows_plasma-speed-2010a_v003.dat', - 'imap_glows_proton-density-2010a_v003.dat', - 'imap_glows_uv-anisotropy-2010a_v003.dat', - 'imap_glows_photoion-2010a_v003.dat', - 'imap_glows_lya-2010a_v003.dat', - 'imap_glows_electron-density-2010a_v003.dat', - 'f107_fluxtable.txt' - ], result.parent_file_names) - np.testing.assert_equal(result.used_l3a[0, :3], np.array([ - "imap_glows_l3a_20100101000000_orbX_modX_p_v00.json", - "imap_glows_l3a_20100102000000_orbX_modX_p_v00.json", - "imap_glows_l3a_20100103000000_orbX_modX_p_v00.json"])) + [instrument_team_l3b_dict["ion_rate_profile"]["sum_rate"]], result.sum_rate + ) + np.testing.assert_equal( + [instrument_team_l3b_dict["ion_rate_profile"]["ph_rate"]], result.ph_rate + ) + np.testing.assert_equal( + [instrument_team_l3b_dict["ion_rate_profile"]["cx_rate"]], result.cx_rate + ) + np.testing.assert_equal( + [instrument_team_l3b_dict["ion_rate_profile"]["sum_uncert"]], + result.sum_uncert, + ) + np.testing.assert_equal( + [instrument_team_l3b_dict["ion_rate_profile"]["ph_uncert"]], + result.ph_uncert, + ) + np.testing.assert_equal( + [instrument_team_l3b_dict["ion_rate_profile"]["cx_uncert"]], + result.cx_uncert, + ) + np.testing.assert_equal( + [ + "-90°", + "-80°", + "-70°", + "-60°", + "-50°", + "-40°", + "-30°", + "-20°", + "-10°", + "0°", + "10°", + "20°", + "30°", + "40°", + "50°", + "60°", + "70°", + "80°", + "90°", + ], + result.lat_grid_label, + ) + self.assertEqual( + [ + "imap_glows_WawHelioIonMP_v002.json", + "imap_glows_bad-days-list_v001.dat", + "imap_glows_pipeline-settings-l3bcde_20250514_v004.json", + "imap_glows_uv-anisotropy-1CR_v002.json", + "imap_glows_plasma-speed-2010a_v003.dat", + "imap_glows_proton-density-2010a_v003.dat", + "imap_glows_uv-anisotropy-2010a_v003.dat", + "imap_glows_photoion-2010a_v003.dat", + "imap_glows_lya-2010a_v003.dat", + "imap_glows_electron-density-2010a_v003.dat", + "f107_fluxtable.txt", + ], + result.parent_file_names, + ) + np.testing.assert_equal( + result.used_l3a[0, :3], + np.array( + [ + "imap_glows_l3a_20100101000000_orbX_modX_p_v00.json", + "imap_glows_l3a_20100102000000_orbX_modX_p_v00.json", + "imap_glows_l3a_20100103000000_orbX_modX_p_v00.json", + ] + ), + ) self.assertEqual((1, 50), result.used_l3a.shape) np.testing.assert_array_equal(result.used_l3a[0, 3:], "") - np.testing.assert_equal([instrument_team_l3b_dict["glows_flags"]], result.glows_flags) + np.testing.assert_equal( + [instrument_team_l3b_dict["glows_flags"]], result.glows_flags + ) def test_l3c_to_data_product_variables(self): - data = GlowsL3CSolarWind(input_metadata=sentinel.input_metadata, - epoch=sentinel.epoch, - epoch_delta=sentinel.epoch_delta, - mean_time=sentinel.mean_time, - cr=sentinel.cr, - lat_grid=sentinel.lat_grid, - lat_grid_label=sentinel.lat_grid_label, - plasma_speed_ecliptic=sentinel.plasma_speed_ecliptic, - proton_density_ecliptic=sentinel.proton_density_ecliptic, - alpha_abundance_ecliptic=sentinel.alpha_abundance_ecliptic, - plasma_speed_profile=sentinel.plasma_speed_profile, - proton_density_profile=sentinel.proton_density_profile, - glows_flags=sentinel.glows_flags, - ) + data = GlowsL3CSolarWind( + input_metadata=sentinel.input_metadata, + epoch=sentinel.epoch, + epoch_delta=sentinel.epoch_delta, + mean_time=sentinel.mean_time, + cr=sentinel.cr, + lat_grid=sentinel.lat_grid, + lat_grid_label=sentinel.lat_grid_label, + plasma_speed_ecliptic=sentinel.plasma_speed_ecliptic, + proton_density_ecliptic=sentinel.proton_density_ecliptic, + alpha_abundance_ecliptic=sentinel.alpha_abundance_ecliptic, + plasma_speed_profile=sentinel.plasma_speed_profile, + proton_density_profile=sentinel.proton_density_profile, + glows_flags=sentinel.glows_flags, + ) variables = data.to_data_product_variables() variables = iter(variables) - self.assert_variable_attributes(next(variables), sentinel.epoch, "epoch", - expected_data_type=pycdf.const.CDF_TIME_TT2000) - self.assert_variable_attributes(next(variables), sentinel.epoch_delta, "epoch_delta", - expected_data_type=pycdf.const.CDF_INT8) - self.assert_variable_attributes(next(variables), sentinel.mean_time, "mean_time", - expected_data_type=pycdf.const.CDF_TIME_TT2000) - self.assert_variable_attributes(next(variables), sentinel.cr, "cr", expected_data_type=pycdf.const.CDF_INT2) - self.assert_variable_attributes(next(variables), sentinel.lat_grid, "lat_grid", - expected_data_type=pycdf.const.CDF_FLOAT, expected_record_varying=False) - self.assert_variable_attributes(next(variables), sentinel.lat_grid_label, "lat_grid_label", - expected_data_type=pycdf.const.CDF_CHAR, expected_record_varying=False) - self.assert_variable_attributes(next(variables), sentinel.plasma_speed_ecliptic, "plasma_speed_ecliptic", - expected_data_type=pycdf.const.CDF_FLOAT) - self.assert_variable_attributes(next(variables), sentinel.proton_density_ecliptic, "proton_density_ecliptic", - expected_data_type=pycdf.const.CDF_FLOAT) - self.assert_variable_attributes(next(variables), sentinel.alpha_abundance_ecliptic, "alpha_abundance_ecliptic", - expected_data_type=pycdf.const.CDF_FLOAT) - self.assert_variable_attributes(next(variables), sentinel.plasma_speed_profile, "plasma_speed_profile", - expected_data_type=pycdf.const.CDF_FLOAT) - self.assert_variable_attributes(next(variables), sentinel.proton_density_profile, "proton_density_profile", - expected_data_type=pycdf.const.CDF_FLOAT) - self.assert_variable_attributes(next(variables), sentinel.glows_flags, "glows_flags") + self.assert_variable_attributes( + next(variables), + sentinel.epoch, + "epoch", + expected_data_type=pycdf.const.CDF_TIME_TT2000, + ) + self.assert_variable_attributes( + next(variables), + sentinel.epoch_delta, + "epoch_delta", + expected_data_type=pycdf.const.CDF_INT8, + ) + self.assert_variable_attributes( + next(variables), + sentinel.mean_time, + "mean_time", + expected_data_type=pycdf.const.CDF_TIME_TT2000, + ) + self.assert_variable_attributes( + next(variables), sentinel.cr, "cr", expected_data_type=pycdf.const.CDF_INT2 + ) + self.assert_variable_attributes( + next(variables), + sentinel.lat_grid, + "lat_grid", + expected_data_type=pycdf.const.CDF_FLOAT, + expected_record_varying=False, + ) + self.assert_variable_attributes( + next(variables), + sentinel.lat_grid_label, + "lat_grid_label", + expected_data_type=pycdf.const.CDF_CHAR, + expected_record_varying=False, + ) + self.assert_variable_attributes( + next(variables), + sentinel.plasma_speed_ecliptic, + "plasma_speed_ecliptic", + expected_data_type=pycdf.const.CDF_FLOAT, + ) + self.assert_variable_attributes( + next(variables), + sentinel.proton_density_ecliptic, + "proton_density_ecliptic", + expected_data_type=pycdf.const.CDF_FLOAT, + ) + self.assert_variable_attributes( + next(variables), + sentinel.alpha_abundance_ecliptic, + "alpha_abundance_ecliptic", + expected_data_type=pycdf.const.CDF_FLOAT, + ) + self.assert_variable_attributes( + next(variables), + sentinel.plasma_speed_profile, + "plasma_speed_profile", + expected_data_type=pycdf.const.CDF_FLOAT, + ) + self.assert_variable_attributes( + next(variables), + sentinel.proton_density_profile, + "proton_density_profile", + expected_data_type=pycdf.const.CDF_FLOAT, + ) + self.assert_variable_attributes( + next(variables), sentinel.glows_flags, "glows_flags" + ) self.assertIsNone(next(variables, None)) def test_l3c_from_instrument_team_dictionary(self): - glows_instrument_team_data_path = get_test_instrument_team_data_path('glows') - with open(glows_instrument_team_data_path / 'imap_glows_l3c_cr_2091_v00.json') as f: + glows_instrument_team_data_path = get_test_instrument_team_data_path("glows") + with open( + glows_instrument_team_data_path / "imap_glows_l3c_cr_2091_v00.json" + ) as f: model = json.load(f) - model['header']['ancillary_data_files'][ - 'pipeline_settings'] = glows_instrument_team_data_path / 'imap_glows_pipeline-settings-L3bc_20250707_v002.json' + model["header"]["ancillary_data_files"]["pipeline_settings"] = ( + glows_instrument_team_data_path + / "imap_glows_pipeline-settings-L3bc_20250707_v002.json" + ) latitude_grid = model["solar_wind_profile"]["lat_grid"] result = GlowsL3CSolarWind.from_instrument_team_dictionary( - model, sentinel.input_metadata) + model, sentinel.input_metadata + ) self.assertIsInstance(result, GlowsL3CSolarWind) self.assertEqual(sentinel.input_metadata, result.input_metadata) - np.testing.assert_array_equal([datetime.fromisoformat("2009-12-20 20:14:51.360")], result.epoch) - np.testing.assert_equal([CARRINGTON_ROTATION_IN_NANOSECONDS / 2], result.epoch_delta) - np.testing.assert_array_equal([datetime.fromisoformat("2010-01-02 03:43:28.667")], result.mean_time) + np.testing.assert_array_equal( + [datetime.fromisoformat("2009-12-20 20:14:51.360")], result.epoch + ) + np.testing.assert_equal( + [CARRINGTON_ROTATION_IN_NANOSECONDS / 2], result.epoch_delta + ) + np.testing.assert_array_equal( + [datetime.fromisoformat("2010-01-02 03:43:28.667")], result.mean_time + ) self.assertEqual([2091], result.cr) np.testing.assert_equal(latitude_grid, result.lat_grid) self.assertEqual( - ['-90°', '-80°', '-70°', '-60°', '-50°', '-40°', '-30°', '-20°', '-10°', '0°', '10°', '20°', '30°', '40°', - '50°', '60°', '70°', '80°', '90°'], result.lat_grid_label) - np.testing.assert_equal([model["solar_wind_profile"]["plasma_speed"]], result.plasma_speed_profile) - np.testing.assert_equal([model["solar_wind_profile"]["proton_density"]], result.proton_density_profile) - - self.assertEqual([model["solar_wind_ecliptic"]["plasma_speed"]], result.plasma_speed_ecliptic) - self.assertEqual([model["solar_wind_ecliptic"]["proton_density"]], result.proton_density_ecliptic) - self.assertEqual([model["solar_wind_ecliptic"]["alpha_abundance"]], result.alpha_abundance_ecliptic) + [ + "-90°", + "-80°", + "-70°", + "-60°", + "-50°", + "-40°", + "-30°", + "-20°", + "-10°", + "0°", + "10°", + "20°", + "30°", + "40°", + "50°", + "60°", + "70°", + "80°", + "90°", + ], + result.lat_grid_label, + ) + np.testing.assert_equal( + [model["solar_wind_profile"]["plasma_speed"]], result.plasma_speed_profile + ) + np.testing.assert_equal( + [model["solar_wind_profile"]["proton_density"]], + result.proton_density_profile, + ) + + self.assertEqual( + [model["solar_wind_ecliptic"]["plasma_speed"]], result.plasma_speed_ecliptic + ) + self.assertEqual( + [model["solar_wind_ecliptic"]["proton_density"]], + result.proton_density_ecliptic, + ) + self.assertEqual( + [model["solar_wind_ecliptic"]["alpha_abundance"]], + result.alpha_abundance_ecliptic, + ) np.testing.assert_equal([model["glows_flags"]], result.glows_flags) - self.assertEqual([ - "imap_glows_WawHelioIonMP_v002.json", - "imap_glows_bad-days-list_v001.dat", - "imap_glows_pipeline-settings-L3bc_20250707_v002.json", - "imap_glows_uv-anisotropy-1CR_v002.json", - "imap_glows_plasma-speed-2010a_v003.dat", - "imap_glows_proton-density-2010a_v003.dat", - "imap_glows_uv-anisotropy-2010a_v003.dat", - "imap_glows_photoion-2010a_v003.dat", - "imap_glows_lya-2010a_v003.dat", - "imap_glows_electron-density-2010a_v003.dat", - "omni2_all_years.dat" - ], result.parent_file_names) - - -if __name__ == '__main__': + self.assertEqual( + [ + "imap_glows_WawHelioIonMP_v002.json", + "imap_glows_bad-days-list_v001.dat", + "imap_glows_pipeline-settings-L3bc_20250707_v002.json", + "imap_glows_uv-anisotropy-1CR_v002.json", + "imap_glows_plasma-speed-2010a_v003.dat", + "imap_glows_proton-density-2010a_v003.dat", + "imap_glows_uv-anisotropy-2010a_v003.dat", + "imap_glows_photoion-2010a_v003.dat", + "imap_glows_lya-2010a_v003.dat", + "imap_glows_electron-density-2010a_v003.dat", + "omni2_all_years.dat", + ], + result.parent_file_names, + ) + + +if __name__ == "__main__": unittest.main() diff --git a/tests/glows/l3bc/test_utils.py b/tests/glows/l3bc/test_utils.py index 332da0030..10c3b47a5 100644 --- a/tests/glows/l3bc/test_utils.py +++ b/tests/glows/l3bc/test_utils.py @@ -5,7 +5,10 @@ import numpy as np from imap_processing.spice.repoint import set_global_repoint_table_paths -from imap_l3_processing.glows.l3bc.utils import get_pointing_date_range, get_best_ancillary +from imap_l3_processing.glows.l3bc.utils import ( + get_pointing_date_range, + get_best_ancillary, +) from tests.test_helpers import get_test_data_path @@ -20,45 +23,103 @@ def test_determine_crs_to_process_based_on_ancillary_files(self): end_date = datetime(2010, 1, 2) case_1 = [ - {"file_path": "some/server/path/" + "older_ancillary.dat", "start_date": "20100101", "end_date": None, - "ingestion_date": "20100101 00:00:00"}, - {"file_path": "some/server/path/" + "newer_ancillary.dat", "start_date": "20100101", "end_date": None, - "ingestion_date": "20100102 00:00:00"} + { + "file_path": "some/server/path/" + "older_ancillary.dat", + "start_date": "20100101", + "end_date": None, + "ingestion_date": "20100101 00:00:00", + }, + { + "file_path": "some/server/path/" + "newer_ancillary.dat", + "start_date": "20100101", + "end_date": None, + "ingestion_date": "20100102 00:00:00", + }, ] case_2 = [ - {"file_path": "some/server/path/" + "older_ancillary.dat", "start_date": "20100101", "end_date": None, - "ingestion_date": "20100101 00:00:00"}, - {"file_path": "some/server/path/" + "newer_ancillary.dat", "start_date": "20100104", "end_date": None, - "ingestion_date": "20100102 00:00:00"} + { + "file_path": "some/server/path/" + "older_ancillary.dat", + "start_date": "20100101", + "end_date": None, + "ingestion_date": "20100101 00:00:00", + }, + { + "file_path": "some/server/path/" + "newer_ancillary.dat", + "start_date": "20100104", + "end_date": None, + "ingestion_date": "20100102 00:00:00", + }, ] case_3 = [ - {"file_path": "some/server/path/" + "older_ancillary.dat", "start_date": "20100101", "end_date": None, - "ingestion_date": "20100101 00:00:00"}, - {"file_path": "some/server/path/" + "newer_ancillary.dat", "start_date": "20091205", "end_date": "20091206", - "ingestion_date": "20100102 00:00:00"} + { + "file_path": "some/server/path/" + "older_ancillary.dat", + "start_date": "20100101", + "end_date": None, + "ingestion_date": "20100101 00:00:00", + }, + { + "file_path": "some/server/path/" + "newer_ancillary.dat", + "start_date": "20091205", + "end_date": "20091206", + "ingestion_date": "20100102 00:00:00", + }, ] test_cases = [ ("picks latest ingestion date", case_1, "newer_ancillary.dat"), - ("ignores ancillary with start date after cr", case_2, "older_ancillary.dat"), - ("ignores ancillary with end date before cr", case_3, "older_ancillary.dat") + ( + "ignores ancillary with start date after cr", + case_2, + "older_ancillary.dat", + ), + ( + "ignores ancillary with end date before cr", + case_3, + "older_ancillary.dat", + ), ] - for name, available_ancillaries, expected_best_ancillary_file_name in test_cases: + for ( + name, + available_ancillaries, + expected_best_ancillary_file_name, + ) in test_cases: with self.subTest(name): - actual_ancillary_name = get_best_ancillary(start_date, end_date, available_ancillaries) + actual_ancillary_name = get_best_ancillary( + start_date, end_date, available_ancillaries + ) - self.assertEqual(expected_best_ancillary_file_name, actual_ancillary_name) + self.assertEqual( + expected_best_ancillary_file_name, actual_ancillary_name + ) def test_get_pointing_date_range(self): repointing_number = 13 actual_start, actual_end = get_pointing_date_range(repointing_number) expected_start = np.datetime64( - datetime(year=2000, month=1, day=14, hour=12, minute=13, second=55, microsecond=816000).isoformat()) + datetime( + year=2000, + month=1, + day=14, + hour=12, + minute=13, + second=55, + microsecond=816000, + ).isoformat() + ) expected_end = np.datetime64( - datetime(year=2000, month=1, day=15, hour=11, minute=58, second=55, microsecond=816000).isoformat()) + datetime( + year=2000, + month=1, + day=15, + hour=11, + minute=58, + second=55, + microsecond=816000, + ).isoformat() + ) np.testing.assert_array_equal(actual_start, expected_start) np.testing.assert_array_equal(actual_end, expected_end) @@ -70,13 +131,26 @@ def test_get_repoint_date_range_handles_no_pointing(self): self.assertEqual(str(err.exception), "No pointing found for pointing: 5998") -def create_imap_data_access_json(file_path: str, data_level: str, start_date: str, - descriptor: str = "hist", version: str = "v001", - repointing: Optional[int] = None) -> dict: - return {'file_path': file_path, 'instrument': 'glows', 'data_level': data_level, 'descriptor': descriptor, - 'start_date': start_date, 'repointing': repointing, 'version': version, 'extension': 'pkts', - 'ingestion_date': '2024-10-11 15:28:32'} +def create_imap_data_access_json( + file_path: str, + data_level: str, + start_date: str, + descriptor: str = "hist", + version: str = "v001", + repointing: Optional[int] = None, +) -> dict: + return { + "file_path": file_path, + "instrument": "glows", + "data_level": data_level, + "descriptor": descriptor, + "start_date": start_date, + "repointing": repointing, + "version": version, + "extension": "pkts", + "ingestion_date": "2024-10-11 15:28:32", + } def create_l3a_path_by_date(file_date: str, repointing: int) -> str: - return f'imap/glows/l3a/2000/01/imap_glows_l3a_hist_{file_date}-repoint{str(repointing).zfill(5)}_v001.pkts' + return f"imap/glows/l3a/2000/01/imap_glows_l3a_hist_{file_date}-repoint{str(repointing).zfill(5)}_v001.pkts" diff --git a/tests/glows/l3d/test_glows_l3d_dependencies.py b/tests/glows/l3d/test_glows_l3d_dependencies.py index 49164a85a..2e35b6aee 100644 --- a/tests/glows/l3d/test_glows_l3d_dependencies.py +++ b/tests/glows/l3d/test_glows_l3d_dependencies.py @@ -11,15 +11,31 @@ class TestGlowsL3DDependencies(unittest.TestCase): @patch("imap_l3_processing.glows.l3d.glows_l3d_dependencies.ScienceFilePath") - @patch('imap_l3_processing.glows.l3d.glows_l3d_dependencies.imap_data_access.download') + @patch( + "imap_l3_processing.glows.l3d.glows_l3d_dependencies.imap_data_access.download" + ) def test_fetch_dependencies(self, mock_download, mock_science_file_path): - waw_helio_ion_mp_speed = get_test_data_path("glows/imap_glows_plasma-speed-2026d_20251113_v003.dat") - waw_helio_ion_mp_p_dens = get_test_data_path("glows/imap_glows_proton-density-2026d_20251113_v003.dat") - waw_helio_ion_mp_uv_anis = get_test_data_path("glows/imap_glows_uv-anisotropy-2026d_20251113_v003.dat") - waw_helio_ion_mp_phion = get_test_data_path("glows/imap_glows_photoion-2026d_20251113_v003.dat") - waw_helio_ion_mp_lya = get_test_data_path("glows/imap_glows_lya-2026d_20251113_v003.dat") - waw_helio_ion_mp_e_dens = get_test_data_path("glows/imap_glows_electron-density-2026d_20251113_v003.dat") - pipeline_settings = get_test_data_path("glows/imap_glows_pipeline-settings-l3bcde_20251113_v005.json") + waw_helio_ion_mp_speed = get_test_data_path( + "glows/imap_glows_plasma-speed-2026d_20251113_v003.dat" + ) + waw_helio_ion_mp_p_dens = get_test_data_path( + "glows/imap_glows_proton-density-2026d_20251113_v003.dat" + ) + waw_helio_ion_mp_uv_anis = get_test_data_path( + "glows/imap_glows_uv-anisotropy-2026d_20251113_v003.dat" + ) + waw_helio_ion_mp_phion = get_test_data_path( + "glows/imap_glows_photoion-2026d_20251113_v003.dat" + ) + waw_helio_ion_mp_lya = get_test_data_path( + "glows/imap_glows_lya-2026d_20251113_v003.dat" + ) + waw_helio_ion_mp_e_dens = get_test_data_path( + "glows/imap_glows_electron-density-2026d_20251113_v003.dat" + ) + pipeline_settings = get_test_data_path( + "glows/imap_glows_pipeline-settings-l3bcde_20251113_v005.json" + ) mock_processing_input_collection = Mock(spec=ProcessingInputCollection) @@ -55,66 +71,82 @@ def test_fetch_dependencies(self, mock_download, mock_science_file_path): ] mock_science_file_path.side_effect = [ - Mock(cr = 2000.5), - Mock(cr = 4000.5), - Mock(cr = 2000.5), - Mock(cr = 3000.5), + Mock(cr=2000.5), + Mock(cr=4000.5), + Mock(cr=2000.5), + Mock(cr=3000.5), ] external_dependencies = ExternalDependencies( lyman_alpha_path=sentinel.lyman_alpha_path, f107_index_file_path=None, - omni2_data_path=None + omni2_data_path=None, ) - actual_dependencies: GlowsL3DDependencies = GlowsL3DDependencies.fetch_dependencies( - mock_processing_input_collection, - external_dependencies + actual_dependencies: GlowsL3DDependencies = ( + GlowsL3DDependencies.fetch_dependencies( + mock_processing_input_collection, external_dependencies + ) ) - mock_processing_input_collection.get_file_paths.assert_has_calls([ - call(source='glows', descriptor="plasma-speed-2026d"), - call(source='glows', descriptor="proton-density-2026d"), - call(source='glows', descriptor="uv-anisotropy-2026d"), - call(source='glows', descriptor="photoion-2026d"), - call(source='glows', descriptor="lya-2026d"), - call(source='glows', descriptor="electron-density-2026d"), - call(source='glows', descriptor="pipeline-settings-l3bcde"), - call(source='glows', descriptor="ion-rate-profile"), - call(source='glows', descriptor="sw-profile"), - ]) - - mock_download.assert_has_calls([ - call(str(sentinel.waw_helio_ion_mp_speed)), - call(str(sentinel.waw_helio_ion_mp_p_dens)), - call(str(sentinel.waw_helio_ion_mp_uv_anis)), - call(str(sentinel.waw_helio_ion_mp_phion)), - call(str(sentinel.waw_helio_ion_mp_lya)), - call(str(sentinel.waw_helio_ion_mp_e_dens)), - call(str(sentinel.pipeline_settings)), - call(l3b_file_1), - call(l3b_file_2), - call(l3c_file_1), - call(l3c_file_2), - ]) - - self.assertEqual({ - 'pipeline_settings': pipeline_settings, - 'WawHelioIon': { - 'speed': waw_helio_ion_mp_speed, - 'p-dens': waw_helio_ion_mp_p_dens, - 'uv-anis': waw_helio_ion_mp_uv_anis, - 'phion': waw_helio_ion_mp_phion, - 'lya': waw_helio_ion_mp_lya, - 'e-dens': waw_helio_ion_mp_e_dens - }}, actual_dependencies.ancillary_files) - - self.assertEqual([sentinel.l3b_downloaded_path_1, sentinel.l3b_downloaded_path_2], - actual_dependencies.l3b_file_paths) - - self.assertEqual([sentinel.l3c_downloaded_path_1, sentinel.l3c_downloaded_path_2], - actual_dependencies.l3c_file_paths) + mock_processing_input_collection.get_file_paths.assert_has_calls( + [ + call(source="glows", descriptor="plasma-speed-2026d"), + call(source="glows", descriptor="proton-density-2026d"), + call(source="glows", descriptor="uv-anisotropy-2026d"), + call(source="glows", descriptor="photoion-2026d"), + call(source="glows", descriptor="lya-2026d"), + call(source="glows", descriptor="electron-density-2026d"), + call(source="glows", descriptor="pipeline-settings-l3bcde"), + call(source="glows", descriptor="ion-rate-profile"), + call(source="glows", descriptor="sw-profile"), + ] + ) + + mock_download.assert_has_calls( + [ + call(str(sentinel.waw_helio_ion_mp_speed)), + call(str(sentinel.waw_helio_ion_mp_p_dens)), + call(str(sentinel.waw_helio_ion_mp_uv_anis)), + call(str(sentinel.waw_helio_ion_mp_phion)), + call(str(sentinel.waw_helio_ion_mp_lya)), + call(str(sentinel.waw_helio_ion_mp_e_dens)), + call(str(sentinel.pipeline_settings)), + call(l3b_file_1), + call(l3b_file_2), + call(l3c_file_1), + call(l3c_file_2), + ] + ) + + self.assertEqual( + { + "pipeline_settings": pipeline_settings, + "WawHelioIon": { + "speed": waw_helio_ion_mp_speed, + "p-dens": waw_helio_ion_mp_p_dens, + "uv-anis": waw_helio_ion_mp_uv_anis, + "phion": waw_helio_ion_mp_phion, + "lya": waw_helio_ion_mp_lya, + "e-dens": waw_helio_ion_mp_e_dens, + }, + }, + actual_dependencies.ancillary_files, + ) + + self.assertEqual( + [sentinel.l3b_downloaded_path_1, sentinel.l3b_downloaded_path_2], + actual_dependencies.l3b_file_paths, + ) + + self.assertEqual( + [sentinel.l3c_downloaded_path_1, sentinel.l3c_downloaded_path_2], + actual_dependencies.l3c_file_paths, + ) self.assertEqual(actual_dependencies.end_cr, 3001.5) - self.assertEqual({'lya_raw_data': sentinel.lyman_alpha_path}, actual_dependencies.external_files) + self.assertEqual( + {"lya_raw_data": sentinel.lyman_alpha_path}, + actual_dependencies.external_files, + ) diff --git a/tests/glows/l3d/test_glows_l3d_initializer.py b/tests/glows/l3d/test_glows_l3d_initializer.py index c7bd77b72..eda08f4fb 100644 --- a/tests/glows/l3d/test_glows_l3d_initializer.py +++ b/tests/glows/l3d/test_glows_l3d_initializer.py @@ -3,252 +3,377 @@ from imap_data_access.file_validation import Version -from imap_l3_processing.glows.descriptors import GLOWS_L3E_ULTRA_HF_DESCRIPTOR, GLOWS_L3E_HI_45_DESCRIPTOR, \ - GLOWS_L3E_HI_90_DESCRIPTOR, GLOWS_L3E_LO_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, GLOWS_L3B_DESCRIPTOR +from imap_l3_processing.glows.descriptors import ( + GLOWS_L3E_ULTRA_HF_DESCRIPTOR, + GLOWS_L3E_HI_45_DESCRIPTOR, + GLOWS_L3E_HI_90_DESCRIPTOR, + GLOWS_L3E_LO_DESCRIPTOR, + GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + GLOWS_L3B_DESCRIPTOR, +) from imap_l3_processing.glows.l3d.glows_l3d_initializer import GlowsL3DInitializer from imap_l3_processing.glows.l3e.reprocess_info import ReprocessInfo, ReprocessTargets from tests.test_helpers import create_mock_query_results class TestGlowsL3DInitializer(unittest.TestCase): - - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.read_cdf_parents') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.read_pipeline_settings') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.query') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.download') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.GlowsL3DDependencies.fetch_dependencies') - def test_l3d_initializer(self, mock_fetch_l3d_dependencies, mock_download, mock_query, mock_read_pipeline_settings, - mock_read_cdf_parents): + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.read_cdf_parents") + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.read_pipeline_settings") + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.query") + @patch( + "imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.download" + ) + @patch( + "imap_l3_processing.glows.l3d.glows_l3d_initializer.GlowsL3DDependencies.fetch_dependencies" + ) + def test_l3d_initializer( + self, + mock_fetch_l3d_dependencies, + mock_download, + mock_query, + mock_read_pipeline_settings, + mock_read_cdf_parents, + ): l3bs = [ - 'imap_glows_l3b_ion-rate-profile_20200101-cr00000_v000.cdf', - 'imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3b_ion-rate-profile_20200201-cr00002_v001.cdf', - 'imap_glows_l3b_ion-rate-profile_20200301-cr00003_v000.cdf', + "imap_glows_l3b_ion-rate-profile_20200101-cr00000_v000.cdf", + "imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf", + "imap_glows_l3b_ion-rate-profile_20200201-cr00002_v001.cdf", + "imap_glows_l3b_ion-rate-profile_20200301-cr00003_v000.cdf", ] l3cs = [ - 'imap_glows_l3c_sw-profile_20200101-cr00000_v000.cdf', - 'imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3c_sw-profile_20200201-cr00002_v001.cdf', - 'imap_glows_l3c_sw-profile_20200301-cr00003_v000.cdf', + "imap_glows_l3c_sw-profile_20200101-cr00000_v000.cdf", + "imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf", + "imap_glows_l3c_sw-profile_20200201-cr00002_v001.cdf", + "imap_glows_l3c_sw-profile_20200301-cr00003_v000.cdf", ] external_dependencies = Mock() mock_read_cdf_parents.return_value = { - 'imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3b_ion-rate-profile_20200201-cr00002_v000.cdf', - 'imap_glows_l3b_ion-rate-profile_20200301-cr00003_v000.cdf', - 'imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3c_sw-profile_20200201-cr00002_v000.cdf', - 'imap_glows_l3c_sw-profile_20200301-cr00003_v000.cdf', - 'imap_glows_plasma-speed-2026d_19470301_v001.dat', - 'imap_glows_proton-density-2026d_19470301_v000.dat', - 'imap_glows_lya-2026d_19470301_v000.dat', - 'imap_glows_uv-anisotropy-2026d_19470301_v000.dat', - 'imap_glows_photoion-2026d_19470301_v000.dat', - 'imap_glows_electron-density-2026d_19470301_v000.dat' - 'lyman-alpha-composite.nc' + "imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf", + "imap_glows_l3b_ion-rate-profile_20200201-cr00002_v000.cdf", + "imap_glows_l3b_ion-rate-profile_20200301-cr00003_v000.cdf", + "imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf", + "imap_glows_l3c_sw-profile_20200201-cr00002_v000.cdf", + "imap_glows_l3c_sw-profile_20200301-cr00003_v000.cdf", + "imap_glows_plasma-speed-2026d_19470301_v001.dat", + "imap_glows_proton-density-2026d_19470301_v000.dat", + "imap_glows_lya-2026d_19470301_v000.dat", + "imap_glows_uv-anisotropy-2026d_19470301_v000.dat", + "imap_glows_photoion-2026d_19470301_v000.dat", + "imap_glows_electron-density-2026d_19470301_v000.dat" + "lyman-alpha-composite.nc", } mock_read_pipeline_settings.return_value = {"start_cr": 1} test_cases = { - 'matching versions with legacy': (None, None), - 'matching versions with new version format': (1, 1), - 'upgraded major version': (1, 2), + "matching versions with legacy": (None, None), + "matching versions with new version format": (1, 1), + "upgraded major version": (1, 2), } - for name, (existing_major_version, requested_major_version) in test_cases.items(): + for name, ( + existing_major_version, + requested_major_version, + ) in test_cases.items(): with self.subTest(name=name): - most_recent_l3d = f'imap_glows_l3d_solar-hist_19470301-cr00002_{Version(existing_major_version, 1)}.cdf' - mock_query.side_effect = self._create_ancillary_query_results([ - 'imap_glows_l3d_solar-hist_19470301-cr00001_v000.cdf', - 'imap_glows_l3d_solar-hist_19470301-cr00001_v001.cdf', - 'imap_glows_l3d_solar-hist_19470301-cr00001_v002.cdf', - 'imap_glows_l3d_solar-hist_19470301-cr00002_v000.cdf', - most_recent_l3d, - ]) + most_recent_l3d = f"imap_glows_l3d_solar-hist_19470301-cr00002_{Version(existing_major_version, 1)}.cdf" + mock_query.side_effect = self._create_ancillary_query_results( + [ + "imap_glows_l3d_solar-hist_19470301-cr00001_v000.cdf", + "imap_glows_l3d_solar-hist_19470301-cr00001_v001.cdf", + "imap_glows_l3d_solar-hist_19470301-cr00001_v002.cdf", + "imap_glows_l3d_solar-hist_19470301-cr00002_v000.cdf", + most_recent_l3d, + ] + ) mock_read_cdf_parents.reset_mock() mock_download.reset_mock() mock_read_pipeline_settings.reset_mock() mock_fetch_l3d_dependencies.reset_mock() - actual_version, actual_l3d_deps, actual_old_l3d = GlowsL3DInitializer.should_process_l3d(external_dependencies, - l3bs, l3cs, ReprocessInfo([]), requested_major_version) + actual_version, actual_l3d_deps, actual_old_l3d = ( + GlowsL3DInitializer.should_process_l3d( + external_dependencies, + l3bs, + l3cs, + ReprocessInfo([]), + requested_major_version, + ) + ) mock_read_cdf_parents.assert_called_once_with(most_recent_l3d) - mock_query.assert_has_calls([ - call(instrument='glows', data_level="l3d", descriptor="solar-hist"), - call(table='ancillary', instrument='glows', descriptor='plasma-speed-2026d', version='latest'), - call(table='ancillary', instrument='glows', descriptor='proton-density-2026d', version='latest'), - call(table='ancillary', instrument='glows', descriptor='uv-anisotropy-2026d', version='latest'), - call(table='ancillary', instrument='glows', descriptor='photoion-2026d', version='latest'), - call(table='ancillary', instrument='glows', descriptor='lya-2026d', version='latest'), - call(table='ancillary', instrument='glows', descriptor='electron-density-2026d', version='latest'), - call(table='ancillary', instrument='glows', descriptor='pipeline-settings-l3bcde', version='latest'), - ]) - - mock_download.assert_called_once_with("imap_glows_pipeline-settings-l3bcde_19470301_v000.json") - mock_read_pipeline_settings.assert_called_once_with(mock_download.return_value) + mock_query.assert_has_calls( + [ + call( + instrument="glows", + data_level="l3d", + descriptor="solar-hist", + ), + call( + table="ancillary", + instrument="glows", + descriptor="plasma-speed-2026d", + version="latest", + ), + call( + table="ancillary", + instrument="glows", + descriptor="proton-density-2026d", + version="latest", + ), + call( + table="ancillary", + instrument="glows", + descriptor="uv-anisotropy-2026d", + version="latest", + ), + call( + table="ancillary", + instrument="glows", + descriptor="photoion-2026d", + version="latest", + ), + call( + table="ancillary", + instrument="glows", + descriptor="lya-2026d", + version="latest", + ), + call( + table="ancillary", + instrument="glows", + descriptor="electron-density-2026d", + version="latest", + ), + call( + table="ancillary", + instrument="glows", + descriptor="pipeline-settings-l3bcde", + version="latest", + ), + ] + ) + + mock_download.assert_called_once_with( + "imap_glows_pipeline-settings-l3bcde_19470301_v000.json" + ) + mock_read_pipeline_settings.assert_called_once_with( + mock_download.return_value + ) [fetch_dependencies_call] = mock_fetch_l3d_dependencies.call_args_list [actual_l3d_inputs, actual_external_deps] = fetch_dependencies_call.args - pipeline_l3d_input_paths = actual_l3d_inputs.get_file_paths(source="glows") - pipeline_l3d_input_filenames = [p.name for p in pipeline_l3d_input_paths] - self.assertEqual([ - 'imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3b_ion-rate-profile_20200201-cr00002_v001.cdf', - 'imap_glows_l3b_ion-rate-profile_20200301-cr00003_v000.cdf', - 'imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3c_sw-profile_20200201-cr00002_v001.cdf', - 'imap_glows_l3c_sw-profile_20200301-cr00003_v000.cdf', - 'imap_glows_plasma-speed-2026d_19470301_v001.dat', - 'imap_glows_proton-density-2026d_19470301_v000.dat', - 'imap_glows_uv-anisotropy-2026d_19470301_v000.dat', - 'imap_glows_photoion-2026d_19470301_v000.dat', - 'imap_glows_lya-2026d_19470301_v000.dat', - 'imap_glows_electron-density-2026d_19470301_v000.dat', - 'imap_glows_pipeline-settings-l3bcde_19470301_v000.json' - ], pipeline_l3d_input_filenames) + pipeline_l3d_input_paths = actual_l3d_inputs.get_file_paths( + source="glows" + ) + pipeline_l3d_input_filenames = [ + p.name for p in pipeline_l3d_input_paths + ] + self.assertEqual( + [ + "imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf", + "imap_glows_l3b_ion-rate-profile_20200201-cr00002_v001.cdf", + "imap_glows_l3b_ion-rate-profile_20200301-cr00003_v000.cdf", + "imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf", + "imap_glows_l3c_sw-profile_20200201-cr00002_v001.cdf", + "imap_glows_l3c_sw-profile_20200301-cr00003_v000.cdf", + "imap_glows_plasma-speed-2026d_19470301_v001.dat", + "imap_glows_proton-density-2026d_19470301_v000.dat", + "imap_glows_uv-anisotropy-2026d_19470301_v000.dat", + "imap_glows_photoion-2026d_19470301_v000.dat", + "imap_glows_lya-2026d_19470301_v000.dat", + "imap_glows_electron-density-2026d_19470301_v000.dat", + "imap_glows_pipeline-settings-l3bcde_19470301_v000.json", + ], + pipeline_l3d_input_filenames, + ) self.assertEqual(external_dependencies, actual_external_deps) self.assertEqual(Version(requested_major_version, 2), actual_version) - self.assertEqual(mock_fetch_l3d_dependencies.return_value, actual_l3d_deps) + self.assertEqual( + mock_fetch_l3d_dependencies.return_value, actual_l3d_deps + ) self.assertEqual(most_recent_l3d, actual_old_l3d) - - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.read_cdf_parents') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.read_pipeline_settings') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.download') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.query') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.GlowsL3DDependencies.fetch_dependencies') - def test_l3d_initializer_should_process_l3d_when_major_version_changes(self, mock_fetch_l3d_dependencies, mock_query, _, mock_read_pipeline_settings, mock_read_cdf_parents): - most_recent_l3d = 'imap_glows_l3d_solar-hist_19470301-cr00002_v001.cdf' + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.read_cdf_parents") + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.read_pipeline_settings") + @patch( + "imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.download" + ) + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.query") + @patch( + "imap_l3_processing.glows.l3d.glows_l3d_initializer.GlowsL3DDependencies.fetch_dependencies" + ) + def test_l3d_initializer_should_process_l3d_when_major_version_changes( + self, + mock_fetch_l3d_dependencies, + mock_query, + _, + mock_read_pipeline_settings, + mock_read_cdf_parents, + ): + most_recent_l3d = "imap_glows_l3d_solar-hist_19470301-cr00002_v001.cdf" external_dependencies = Mock() l3bs = [ - 'imap_glows_l3b_ion-rate-profile_20200101-cr00000_v000.cdf', - 'imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3b_ion-rate-profile_20200201-cr00002_v000.cdf', - 'imap_glows_l3b_ion-rate-profile_20200301-cr00003_v000.cdf', + "imap_glows_l3b_ion-rate-profile_20200101-cr00000_v000.cdf", + "imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf", + "imap_glows_l3b_ion-rate-profile_20200201-cr00002_v000.cdf", + "imap_glows_l3b_ion-rate-profile_20200301-cr00003_v000.cdf", ] l3cs = [ - 'imap_glows_l3c_sw-profile_20200101-cr00000_v000.cdf', - 'imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3c_sw-profile_20200201-cr00002_v000.cdf', - 'imap_glows_l3c_sw-profile_20200301-cr00003_v000.cdf', + "imap_glows_l3c_sw-profile_20200101-cr00000_v000.cdf", + "imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf", + "imap_glows_l3c_sw-profile_20200201-cr00002_v000.cdf", + "imap_glows_l3c_sw-profile_20200301-cr00003_v000.cdf", ] mock_read_cdf_parents.return_value = { - 'imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3b_ion-rate-profile_20200201-cr00002_v000.cdf', - 'imap_glows_l3b_ion-rate-profile_20200301-cr00003_v000.cdf', - 'imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3c_sw-profile_20200201-cr00002_v000.cdf', - 'imap_glows_l3c_sw-profile_20200301-cr00003_v000.cdf', - 'lyman-alpha-composite.nc', - 'imap_glows_plasma-speed-2026d_19470301_v001.dat', - 'imap_glows_proton-density-2026d_19470301_v000.dat', - 'imap_glows_lya-2026d_19470301_v000.dat', - 'imap_glows_uv-anisotropy-2026d_19470301_v000.dat', - 'imap_glows_photoion-2026d_19470301_v000.dat', - 'imap_glows_electron-density-2026d_19470301_v000.dat' - + "imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf", + "imap_glows_l3b_ion-rate-profile_20200201-cr00002_v000.cdf", + "imap_glows_l3b_ion-rate-profile_20200301-cr00003_v000.cdf", + "imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf", + "imap_glows_l3c_sw-profile_20200201-cr00002_v000.cdf", + "imap_glows_l3c_sw-profile_20200301-cr00003_v000.cdf", + "lyman-alpha-composite.nc", + "imap_glows_plasma-speed-2026d_19470301_v001.dat", + "imap_glows_proton-density-2026d_19470301_v000.dat", + "imap_glows_lya-2026d_19470301_v000.dat", + "imap_glows_uv-anisotropy-2026d_19470301_v000.dat", + "imap_glows_photoion-2026d_19470301_v000.dat", + "imap_glows_electron-density-2026d_19470301_v000.dat", } mock_read_pipeline_settings.return_value = {"start_cr": 1} - mock_query.side_effect = self._create_ancillary_query_results([ - most_recent_l3d - ]) + mock_query.side_effect = self._create_ancillary_query_results([most_recent_l3d]) new_major_version = 1 - actual_version, actual_l3d_deps, actual_old_l3d = GlowsL3DInitializer.should_process_l3d(external_dependencies, - l3bs, l3cs, - ReprocessInfo([]), - new_major_version) + actual_version, actual_l3d_deps, actual_old_l3d = ( + GlowsL3DInitializer.should_process_l3d( + external_dependencies, l3bs, l3cs, ReprocessInfo([]), new_major_version + ) + ) self.assertEqual(Version(new_major_version, 2), actual_version) self.assertEqual(mock_fetch_l3d_dependencies.return_value, actual_l3d_deps) self.assertEqual(most_recent_l3d, actual_old_l3d) - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.download') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.read_pipeline_settings', return_value={"start_cr": 0}) - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.read_cdf_parents') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.query') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.GlowsL3DDependencies.fetch_dependencies') - def test_l3d_initializer_returns_no_old_cdf_if_none_found(self, mock_fetch_l3d_deps, mock_query, - mock_read_cdf_parents, _, __): - l3bs = ['imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf'] - l3cs = ['imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf'] + @patch( + "imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.download" + ) + @patch( + "imap_l3_processing.glows.l3d.glows_l3d_initializer.read_pipeline_settings", + return_value={"start_cr": 0}, + ) + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.read_cdf_parents") + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.query") + @patch( + "imap_l3_processing.glows.l3d.glows_l3d_initializer.GlowsL3DDependencies.fetch_dependencies" + ) + def test_l3d_initializer_returns_no_old_cdf_if_none_found( + self, mock_fetch_l3d_deps, mock_query, mock_read_cdf_parents, _, __ + ): + l3bs = ["imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf"] + l3cs = ["imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf"] external_deps = Mock() mock_query.side_effect = self._create_ancillary_query_results([]) - _, __, old_l3d = GlowsL3DInitializer.should_process_l3d(external_deps, l3bs, l3cs, ReprocessInfo([]),None) + _, __, old_l3d = GlowsL3DInitializer.should_process_l3d( + external_deps, l3bs, l3cs, ReprocessInfo([]), None + ) mock_read_cdf_parents.assert_not_called() mock_fetch_l3d_deps.assert_called_once() self.assertIsNone(old_l3d) - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.download') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.read_pipeline_settings') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.read_cdf_parents') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.query') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.GlowsL3DDependencies.fetch_dependencies') - def test_l3d_initializer_should_not_process_if_same_major_version(self, mock_fetch_l3d_deps, mock_query, - mock_read_cdf_parents, mock_read_pipeline_settings, _): + @patch( + "imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.download" + ) + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.read_pipeline_settings") + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.read_cdf_parents") + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.query") + @patch( + "imap_l3_processing.glows.l3d.glows_l3d_initializer.GlowsL3DDependencies.fetch_dependencies" + ) + def test_l3d_initializer_should_not_process_if_same_major_version( + self, + mock_fetch_l3d_deps, + mock_query, + mock_read_cdf_parents, + mock_read_pipeline_settings, + _, + ): l3bs = [ - 'imap_glows_l3b_ion-rate-profile_20200101-cr00000_v000.cdf', - 'imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf' + "imap_glows_l3b_ion-rate-profile_20200101-cr00000_v000.cdf", + "imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf", ] l3cs = [ - 'imap_glows_l3c_sw-profile_20200101-cr00000_v000.cdf', - 'imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf' + "imap_glows_l3c_sw-profile_20200101-cr00000_v000.cdf", + "imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf", ] external_dependencies = Mock() mock_query.side_effect = self._create_ancillary_query_results( - ['imap_glows_l3d_solar-hist_19470301-cr00002_v001.cdf']) + ["imap_glows_l3d_solar-hist_19470301-cr00002_v001.cdf"] + ) mock_read_cdf_parents.return_value = { - 'imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf', - 'imap_glows_plasma-speed-2026d_19470301_v001.dat', - 'imap_glows_proton-density-2026d_19470301_v000.dat', - 'imap_glows_uv-anisotropy-2026d_19470301_v000.dat', - 'imap_glows_photoion-2026d_19470301_v000.dat', - 'imap_glows_lya-2026d_19470301_v000.dat', - 'imap_glows_electron-density-2026d_19470301_v000.dat', - 'lyman-alpha-composite.nc' + "imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf", + "imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf", + "imap_glows_plasma-speed-2026d_19470301_v001.dat", + "imap_glows_proton-density-2026d_19470301_v000.dat", + "imap_glows_uv-anisotropy-2026d_19470301_v000.dat", + "imap_glows_photoion-2026d_19470301_v000.dat", + "imap_glows_lya-2026d_19470301_v000.dat", + "imap_glows_electron-density-2026d_19470301_v000.dat", + "lyman-alpha-composite.nc", } mock_read_pipeline_settings.return_value = {"start_cr": 1} - actual_l3d_deps = GlowsL3DInitializer.should_process_l3d(external_dependencies, l3bs, l3cs, ReprocessInfo({}),None) + actual_l3d_deps = GlowsL3DInitializer.should_process_l3d( + external_dependencies, l3bs, l3cs, ReprocessInfo({}), None + ) mock_fetch_l3d_deps.assert_not_called() self.assertIsNone(actual_l3d_deps) - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.GlowsL3DDependencies.fetch_dependencies') + @patch( + "imap_l3_processing.glows.l3d.glows_l3d_initializer.GlowsL3DDependencies.fetch_dependencies" + ) def test_l3d_initializer_should_not_process_when_no_l3bs(self, mock_fetch_l3d_deps): l3bs = [] l3cs = [] external_dependencies = Mock() - actual_l3d_deps = GlowsL3DInitializer.should_process_l3d(external_dependencies, l3bs, l3cs, ReprocessInfo([]),None) + actual_l3d_deps = GlowsL3DInitializer.should_process_l3d( + external_dependencies, l3bs, l3cs, ReprocessInfo([]), None + ) mock_fetch_l3d_deps.assert_not_called() self.assertIsNone(actual_l3d_deps) - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.download') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.query') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.read_pipeline_settings') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.GlowsL3DDependencies.fetch_dependencies') - def test_l3d_initializer_does_not_process_when_no_l3bs_after_start_cr(self, mock_fetch_l3d_deps, - mock_read_pipeline_settings, mock_query, __): - l3bs = ['imap_glows_l3b_ion-rate-profile_20200101-cr00000_v000.cdf', ] - l3cs = ['imap_glows_l3c_sw-profile_20200101-cr00000_v000.cdf', ] + @patch( + "imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.download" + ) + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.query") + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.read_pipeline_settings") + @patch( + "imap_l3_processing.glows.l3d.glows_l3d_initializer.GlowsL3DDependencies.fetch_dependencies" + ) + def test_l3d_initializer_does_not_process_when_no_l3bs_after_start_cr( + self, mock_fetch_l3d_deps, mock_read_pipeline_settings, mock_query, __ + ): + l3bs = [ + "imap_glows_l3b_ion-rate-profile_20200101-cr00000_v000.cdf", + ] + l3cs = [ + "imap_glows_l3c_sw-profile_20200101-cr00000_v000.cdf", + ] external_dependencies = Mock() @@ -256,56 +381,86 @@ def test_l3d_initializer_does_not_process_when_no_l3bs_after_start_cr(self, mock mock_read_pipeline_settings.return_value = {"start_cr": 1} - actual_l3d_deps = GlowsL3DInitializer.should_process_l3d(external_dependencies, l3bs, l3cs, ReprocessInfo([]),None) + actual_l3d_deps = GlowsL3DInitializer.should_process_l3d( + external_dependencies, l3bs, l3cs, ReprocessInfo([]), None + ) mock_fetch_l3d_deps.assert_not_called() self.assertIsNone(actual_l3d_deps) - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.download') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.read_cdf_parents') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.query') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.read_pipeline_settings') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.GlowsL3DDependencies.fetch_dependencies') - def test_l3d_initializer_forces_reprocess_if_reprocess_ancillary_specifies_l3e(self, mock_fetch_dependencies, mock_read_pipeline_settings, mock_query, mock_read_cdf_parents, _): - most_recent_l3d = 'imap_glows_l3d_solar-hist_19470301-cr00002_v001.0002.cdf' + @patch( + "imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.download" + ) + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.read_cdf_parents") + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.query") + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.read_pipeline_settings") + @patch( + "imap_l3_processing.glows.l3d.glows_l3d_initializer.GlowsL3DDependencies.fetch_dependencies" + ) + def test_l3d_initializer_forces_reprocess_if_reprocess_ancillary_specifies_l3e( + self, + mock_fetch_dependencies, + mock_read_pipeline_settings, + mock_query, + mock_read_cdf_parents, + _, + ): + most_recent_l3d = "imap_glows_l3d_solar-hist_19470301-cr00002_v001.0002.cdf" mock_fetch_dependencies.return_value = sentinel.l3d_deps - mock_query.side_effect = self._create_ancillary_query_results([most_recent_l3d]) * 6 + mock_query.side_effect = ( + self._create_ancillary_query_results([most_recent_l3d]) * 6 + ) mock_read_pipeline_settings.return_value = {"start_cr": 1} l3bs = [ - 'imap_glows_l3b_ion-rate-profile_20200101-cr00000_v000.cdf', - 'imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3b_ion-rate-profile_20200101-cr00002_v000.cdf', + "imap_glows_l3b_ion-rate-profile_20200101-cr00000_v000.cdf", + "imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf", + "imap_glows_l3b_ion-rate-profile_20200101-cr00002_v000.cdf", ] l3cs = [ - 'imap_glows_l3c_sw-profile_20200101-cr00000_v000.cdf', - 'imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3c_sw-profile_20200101-cr00002_v000.cdf', + "imap_glows_l3c_sw-profile_20200101-cr00000_v000.cdf", + "imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf", + "imap_glows_l3c_sw-profile_20200101-cr00002_v000.cdf", ] external_dependencies = Mock() mock_read_cdf_parents.return_value = { - 'imap_glows_l3b_ion-rate-profile_20200101-cr00000_v000.cdf', - 'imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3b_ion-rate-profile_20200101-cr00002_v000.cdf', - 'imap_glows_l3c_sw-profile_20200101-cr00000_v000.cdf', - 'imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3c_sw-profile_20200101-cr00002_v000.cdf', - 'imap_glows_plasma-speed-2026d_19470301_v001.dat', - 'imap_glows_proton-density-2026d_19470301_v000.dat', - 'imap_glows_uv-anisotropy-2026d_19470301_v000.dat', - 'imap_glows_photoion-2026d_19470301_v000.dat', - 'imap_glows_lya-2026d_19470301_v000.dat', - 'imap_glows_electron-density-2026d_19470301_v000.dat' + "imap_glows_l3b_ion-rate-profile_20200101-cr00000_v000.cdf", + "imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf", + "imap_glows_l3b_ion-rate-profile_20200101-cr00002_v000.cdf", + "imap_glows_l3c_sw-profile_20200101-cr00000_v000.cdf", + "imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf", + "imap_glows_l3c_sw-profile_20200101-cr00002_v000.cdf", + "imap_glows_plasma-speed-2026d_19470301_v001.dat", + "imap_glows_proton-density-2026d_19470301_v000.dat", + "imap_glows_uv-anisotropy-2026d_19470301_v000.dat", + "imap_glows_photoion-2026d_19470301_v000.dat", + "imap_glows_lya-2026d_19470301_v000.dat", + "imap_glows_electron-density-2026d_19470301_v000.dat", } cases = [ - (GLOWS_L3E_HI_45_DESCRIPTOR, {GLOWS_L3E_HI_45_DESCRIPTOR: ReprocessTargets([], [])}), - (GLOWS_L3E_HI_90_DESCRIPTOR, {GLOWS_L3E_HI_90_DESCRIPTOR: ReprocessTargets([], [])}), - (GLOWS_L3E_LO_DESCRIPTOR, {GLOWS_L3E_LO_DESCRIPTOR: ReprocessTargets([], [])}), - (GLOWS_L3E_ULTRA_SF_DESCRIPTOR, {GLOWS_L3E_ULTRA_SF_DESCRIPTOR: ReprocessTargets([], [])}), - (GLOWS_L3E_ULTRA_HF_DESCRIPTOR, {GLOWS_L3E_ULTRA_HF_DESCRIPTOR: ReprocessTargets([], [])}), + ( + GLOWS_L3E_HI_45_DESCRIPTOR, + {GLOWS_L3E_HI_45_DESCRIPTOR: ReprocessTargets([], [])}, + ), + ( + GLOWS_L3E_HI_90_DESCRIPTOR, + {GLOWS_L3E_HI_90_DESCRIPTOR: ReprocessTargets([], [])}, + ), + ( + GLOWS_L3E_LO_DESCRIPTOR, + {GLOWS_L3E_LO_DESCRIPTOR: ReprocessTargets([], [])}, + ), + ( + GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + {GLOWS_L3E_ULTRA_SF_DESCRIPTOR: ReprocessTargets([], [])}, + ), + ( + GLOWS_L3E_ULTRA_HF_DESCRIPTOR, + {GLOWS_L3E_ULTRA_HF_DESCRIPTOR: ReprocessTargets([], [])}, + ), ] for case, products_to_reprocess in cases: @@ -314,7 +469,9 @@ def test_l3d_initializer_forces_reprocess_if_reprocess_ancillary_specifies_l3e(s reprocess_info = ReprocessInfo(products_to_reprocess) - version, l3d_deps, old_l3d = GlowsL3DInitializer.should_process_l3d(external_dependencies, l3bs, l3cs, reprocess_info, 1) + version, l3d_deps, old_l3d = GlowsL3DInitializer.should_process_l3d( + external_dependencies, l3bs, l3cs, reprocess_info, 1 + ) self.assertEqual(Version(1, 3), version) self.assertIs(sentinel.l3d_deps, l3d_deps) @@ -324,73 +481,109 @@ def test_l3d_initializer_forces_reprocess_if_reprocess_ancillary_specifies_l3e(s [actual_l3d_inputs, actual_external_deps] = fetch_dependencies_call.args - pipeline_l3d_input_paths = actual_l3d_inputs.get_file_paths(source="glows") - pipeline_l3d_input_filenames = [p.name for p in pipeline_l3d_input_paths] - self.assertEqual([ - 'imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3b_ion-rate-profile_20200101-cr00002_v000.cdf', - 'imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3c_sw-profile_20200101-cr00002_v000.cdf', - 'imap_glows_plasma-speed-2026d_19470301_v001.dat', - 'imap_glows_proton-density-2026d_19470301_v000.dat', - 'imap_glows_uv-anisotropy-2026d_19470301_v000.dat', - 'imap_glows_photoion-2026d_19470301_v000.dat', - 'imap_glows_lya-2026d_19470301_v000.dat', - 'imap_glows_electron-density-2026d_19470301_v000.dat', - 'imap_glows_pipeline-settings-l3bcde_19470301_v000.json' - ], pipeline_l3d_input_filenames) + pipeline_l3d_input_paths = actual_l3d_inputs.get_file_paths( + source="glows" + ) + pipeline_l3d_input_filenames = [ + p.name for p in pipeline_l3d_input_paths + ] + self.assertEqual( + [ + "imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf", + "imap_glows_l3b_ion-rate-profile_20200101-cr00002_v000.cdf", + "imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf", + "imap_glows_l3c_sw-profile_20200101-cr00002_v000.cdf", + "imap_glows_plasma-speed-2026d_19470301_v001.dat", + "imap_glows_proton-density-2026d_19470301_v000.dat", + "imap_glows_uv-anisotropy-2026d_19470301_v000.dat", + "imap_glows_photoion-2026d_19470301_v000.dat", + "imap_glows_lya-2026d_19470301_v000.dat", + "imap_glows_electron-density-2026d_19470301_v000.dat", + "imap_glows_pipeline-settings-l3bcde_19470301_v000.json", + ], + pipeline_l3d_input_filenames, + ) self.assertEqual(external_dependencies, actual_external_deps) - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.download') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.read_cdf_parents') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.query') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.read_pipeline_settings') - @patch('imap_l3_processing.glows.l3d.glows_l3d_initializer.GlowsL3DDependencies.fetch_dependencies') - def test_l3d_initializer_does_not_force_reprocess_if_no_l3e_products(self, mock_fetch_dependencies, mock_read_pipeline_settings, mock_query, mock_read_cdf_parents, _): - most_recent_l3d = 'imap_glows_l3d_solar-hist_19470301-cr00002_v001.0002.cdf' + @patch( + "imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.download" + ) + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.read_cdf_parents") + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.imap_data_access.query") + @patch("imap_l3_processing.glows.l3d.glows_l3d_initializer.read_pipeline_settings") + @patch( + "imap_l3_processing.glows.l3d.glows_l3d_initializer.GlowsL3DDependencies.fetch_dependencies" + ) + def test_l3d_initializer_does_not_force_reprocess_if_no_l3e_products( + self, + mock_fetch_dependencies, + mock_read_pipeline_settings, + mock_query, + mock_read_cdf_parents, + _, + ): + most_recent_l3d = "imap_glows_l3d_solar-hist_19470301-cr00002_v001.0002.cdf" mock_fetch_dependencies.return_value = sentinel.l3d_deps - mock_query.side_effect = self._create_ancillary_query_results([most_recent_l3d]) * 2 + mock_query.side_effect = ( + self._create_ancillary_query_results([most_recent_l3d]) * 2 + ) mock_read_pipeline_settings.return_value = {"start_cr": 1} - l3bs = ['imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf'] - l3cs = ['imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf',] + l3bs = ["imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf"] + l3cs = [ + "imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf", + ] external_dependencies = Mock() mock_read_cdf_parents.return_value = { - 'imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf', - 'imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf', - 'imap_glows_plasma-speed-2026d_19470301_v001.dat', - 'imap_glows_proton-density-2026d_19470301_v000.dat', - 'imap_glows_uv-anisotropy-2026d_19470301_v000.dat', - 'imap_glows_photoion-2026d_19470301_v000.dat', - 'imap_glows_lya-2026d_19470301_v000.dat', - 'imap_glows_electron-density-2026d_19470301_v000.dat' + "imap_glows_l3b_ion-rate-profile_20200101-cr00001_v000.cdf", + "imap_glows_l3c_sw-profile_20200101-cr00001_v000.cdf", + "imap_glows_plasma-speed-2026d_19470301_v001.dat", + "imap_glows_proton-density-2026d_19470301_v000.dat", + "imap_glows_uv-anisotropy-2026d_19470301_v000.dat", + "imap_glows_photoion-2026d_19470301_v000.dat", + "imap_glows_lya-2026d_19470301_v000.dat", + "imap_glows_electron-density-2026d_19470301_v000.dat", } cases = [ - ("no l3e products to reprocess", {GLOWS_L3B_DESCRIPTOR: ReprocessTargets([], [])}), - ("no products", {}) + ( + "no l3e products to reprocess", + {GLOWS_L3B_DESCRIPTOR: ReprocessTargets([], [])}, + ), + ("no products", {}), ] for case, products_to_reprocess in cases: reprocess_info = ReprocessInfo(products_to_reprocess) - self.assertIsNone(GlowsL3DInitializer.should_process_l3d(external_dependencies, l3bs, l3cs, - reprocess_info, 1)) - - + self.assertIsNone( + GlowsL3DInitializer.should_process_l3d( + external_dependencies, l3bs, l3cs, reprocess_info, 1 + ) + ) def _create_ancillary_query_results(self, l3d_query_result): return [ create_mock_query_results(l3d_query_result), - create_mock_query_results(['imap_glows_plasma-speed-2026d_19470301_v001.dat']), - create_mock_query_results(['imap_glows_proton-density-2026d_19470301_v000.dat']), - create_mock_query_results(['imap_glows_uv-anisotropy-2026d_19470301_v000.dat']), - create_mock_query_results(['imap_glows_photoion-2026d_19470301_v000.dat']), - create_mock_query_results(['imap_glows_lya-2026d_19470301_v000.dat']), - create_mock_query_results(['imap_glows_electron-density-2026d_19470301_v000.dat']), - create_mock_query_results(['imap_glows_pipeline-settings-l3bcde_19470301_v000.json']), + create_mock_query_results( + ["imap_glows_plasma-speed-2026d_19470301_v001.dat"] + ), + create_mock_query_results( + ["imap_glows_proton-density-2026d_19470301_v000.dat"] + ), + create_mock_query_results( + ["imap_glows_uv-anisotropy-2026d_19470301_v000.dat"] + ), + create_mock_query_results(["imap_glows_photoion-2026d_19470301_v000.dat"]), + create_mock_query_results(["imap_glows_lya-2026d_19470301_v000.dat"]), + create_mock_query_results( + ["imap_glows_electron-density-2026d_19470301_v000.dat"] + ), + create_mock_query_results( + ["imap_glows_pipeline-settings-l3bcde_19470301_v000.json"] + ), ] diff --git a/tests/glows/l3d/test_l3d_utils.py b/tests/glows/l3d/test_l3d_utils.py index 059fe3c06..4b1c412b4 100644 --- a/tests/glows/l3d/test_l3d_utils.py +++ b/tests/glows/l3d/test_l3d_utils.py @@ -8,35 +8,79 @@ import imap_l3_processing from imap_l3_processing.glows.l3d.models import GlowsL3DSolarParamsHistory -from imap_l3_processing.glows.l3d.utils import create_glows_l3b_json_file_from_cdf, convert_json_to_l3d_data_product, \ - create_glows_l3c_json_file_from_cdf, get_parent_file_names_from_l3d_json, rename_l3d_text_outputs, \ - get_most_recently_uploaded_ancillary +from imap_l3_processing.glows.l3d.utils import ( + create_glows_l3b_json_file_from_cdf, + convert_json_to_l3d_data_product, + create_glows_l3c_json_file_from_cdf, + get_parent_file_names_from_l3d_json, + rename_l3d_text_outputs, + get_most_recently_uploaded_ancillary, +) from imap_l3_processing.models import InputMetadata from tests.test_helpers import get_test_data_path, create_mock_query_results class TestL3dUtils(unittest.TestCase): - - @patch('imap_l3_processing.glows.l3d.utils.os') - @patch('imap_l3_processing.glows.l3d.utils.json') - @patch('builtins.open', new_callable=mock_open, create=False) - def test_create_glows_l3c_json_file_from_cdf(self, mock_open_file, mock_json, mock_os): - l3c_path = get_test_data_path('glows/imap_glows_l3c_sw-profile_20101030_v008.cdf') + @patch("imap_l3_processing.glows.l3d.utils.os") + @patch("imap_l3_processing.glows.l3d.utils.json") + @patch("builtins.open", new_callable=mock_open, create=False) + def test_create_glows_l3c_json_file_from_cdf( + self, mock_open_file, mock_json, mock_os + ): + l3c_path = get_test_data_path( + "glows/imap_glows_l3c_sw-profile_20101030_v008.cdf" + ) expected: dict = { - 'solar_wind_profile': { - 'proton_density': [2.3197076, 2.2874057, 2.1938286, 2.5905547, 3.4460852, 4.4701824, - 5.5983787, 6.1791005, 7.7572236, 9.174307, 7.96888, 6.582614, - 5.348592, 4.2680264, 3.3404691, 2.4287271, 2.0275042, 1.9867367, - 1.9867367], - 'plasma_speed': [522., 526., 538., 491., 415., 351., 300., 279., 234., 204., 229., - 266., 310., 362., 423., 509., 561., 567., 567.] + "solar_wind_profile": { + "proton_density": [ + 2.3197076, + 2.2874057, + 2.1938286, + 2.5905547, + 3.4460852, + 4.4701824, + 5.5983787, + 6.1791005, + 7.7572236, + 9.174307, + 7.96888, + 6.582614, + 5.348592, + 4.2680264, + 3.3404691, + 2.4287271, + 2.0275042, + 1.9867367, + 1.9867367, + ], + "plasma_speed": [ + 522.0, + 526.0, + 538.0, + 491.0, + 415.0, + 351.0, + 300.0, + 279.0, + 234.0, + 204.0, + 229.0, + 266.0, + 310.0, + 362.0, + 423.0, + 509.0, + 561.0, + 567.0, + 567.0, + ], }, - 'solar_wind_ecliptic': { - 'proton_density': 6.015008449554443, - 'alpha_abundance': 0.02850634977221489, + "solar_wind_ecliptic": { + "proton_density": 6.015008449554443, + "alpha_abundance": 0.02850634977221489, }, - 'CR': 2103 + "CR": 2103, } json_file = MagicMock() @@ -44,57 +88,134 @@ def test_create_glows_l3c_json_file_from_cdf(self, mock_open_file, mock_json, mo create_glows_l3c_json_file_from_cdf(l3c_path) - data_path = Path(imap_l3_processing.__file__).parent / 'glows' / 'l3d' / 'science' / 'data_l3c' + data_path = ( + Path(imap_l3_processing.__file__).parent + / "glows" + / "l3d" + / "science" + / "data_l3c" + ) mock_os.makedirs.assert_called_once_with(data_path, exist_ok=True) - mock_open_file.assert_called_once_with(data_path / 'imap_glows_l3c_cr_2103_v008.json', 'w') + mock_open_file.assert_called_once_with( + data_path / "imap_glows_l3c_cr_2103_v008.json", "w" + ) mock_json.dump.assert_called_once() actual = mock_json.dump.call_args.args[0] self.assertEqual(json_file, mock_json.dump.call_args.args[1]) - self.assertEqual('imap_glows_l3c_sw-profile_20101030_v008.cdf', actual['header']['filename']) - - self.assertIsInstance(actual['solar_wind_profile']['proton_density'], list) - np.testing.assert_allclose(actual['solar_wind_profile']['proton_density'], - expected['solar_wind_profile']['proton_density']) + self.assertEqual( + "imap_glows_l3c_sw-profile_20101030_v008.cdf", actual["header"]["filename"] + ) - self.assertIsInstance(actual['solar_wind_profile']['plasma_speed'], list) - np.testing.assert_array_equal(actual['solar_wind_profile']['plasma_speed'], - expected['solar_wind_profile']['plasma_speed']) + self.assertIsInstance(actual["solar_wind_profile"]["proton_density"], list) + np.testing.assert_allclose( + actual["solar_wind_profile"]["proton_density"], + expected["solar_wind_profile"]["proton_density"], + ) - self.assertIsInstance(actual['solar_wind_ecliptic']['proton_density'], float) - self.assertEqual(expected['solar_wind_ecliptic']['proton_density'], - actual['solar_wind_ecliptic']['proton_density']) + self.assertIsInstance(actual["solar_wind_profile"]["plasma_speed"], list) + np.testing.assert_array_equal( + actual["solar_wind_profile"]["plasma_speed"], + expected["solar_wind_profile"]["plasma_speed"], + ) - self.assertIsInstance(actual['solar_wind_ecliptic']['alpha_abundance'], float) - self.assertEqual(expected['solar_wind_ecliptic']['alpha_abundance'], - actual['solar_wind_ecliptic']['alpha_abundance']) + self.assertIsInstance(actual["solar_wind_ecliptic"]["proton_density"], float) + self.assertEqual( + expected["solar_wind_ecliptic"]["proton_density"], + actual["solar_wind_ecliptic"]["proton_density"], + ) - self.assertIsInstance(actual['CR'], float) - self.assertEqual(expected['CR'], actual['CR']) + self.assertIsInstance(actual["solar_wind_ecliptic"]["alpha_abundance"], float) + self.assertEqual( + expected["solar_wind_ecliptic"]["alpha_abundance"], + actual["solar_wind_ecliptic"]["alpha_abundance"], + ) - @patch('imap_l3_processing.glows.l3d.utils.os') - @patch('imap_l3_processing.glows.l3d.utils.json') - @patch('builtins.open', new_callable=mock_open, create=False) - def test_create_glows_l3b_json_file_from_cdf(self, mock_open_file, mock_json, mock_os): - l3b_path = get_test_data_path("glows/imap_glows_l3b_ion-rate-profile_20100422_v011.cdf") + self.assertIsInstance(actual["CR"], float) + self.assertEqual(expected["CR"], actual["CR"]) + + @patch("imap_l3_processing.glows.l3d.utils.os") + @patch("imap_l3_processing.glows.l3d.utils.json") + @patch("builtins.open", new_callable=mock_open, create=False) + def test_create_glows_l3b_json_file_from_cdf( + self, mock_open_file, mock_json, mock_os + ): + l3b_path = get_test_data_path( + "glows/imap_glows_l3b_ion-rate-profile_20100422_v011.cdf" + ) expected: dict = { - 'header': { - 'filename': 'imap_glows_l3b_ion-rate-profile_20100422_v011.cdf', + "header": { + "filename": "imap_glows_l3b_ion-rate-profile_20100422_v011.cdf", }, - 'CR': 2096, - 'date': '2010-05-06T05:17:00.959996', - 'uv_anisotropy_factor': [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.], - 'ion_rate_profile': { - 'lat_grid': [-90., -80., -70., -60., -50., -40., -30., -20., -10., 0., 10., 20., 30., 40., 50., 60., - 70., 80., 90.], - 'ph_rate': [1.0475524e-07, 1.0475524e-07, 1.0475524e-07, 1.0475524e-07, - 1.0475524e-07, 1.0475524e-07, 1.0475524e-07, 1.0475524e-07, - 1.0475524e-07, 1.0475524e-07, 1.0475524e-07, 1.0475524e-07, - 1.0475524e-07, 1.0475524e-07, 1.0475524e-07, 1.0475524e-07, - 1.0475524e-07, 1.0475524e-07, 1.0475524e-07] + "CR": 2096, + "date": "2010-05-06T05:17:00.959996", + "uv_anisotropy_factor": [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + ], + "ion_rate_profile": { + "lat_grid": [ + -90.0, + -80.0, + -70.0, + -60.0, + -50.0, + -40.0, + -30.0, + -20.0, + -10.0, + 0.0, + 10.0, + 20.0, + 30.0, + 40.0, + 50.0, + 60.0, + 70.0, + 80.0, + 90.0, + ], + "ph_rate": [ + 1.0475524e-07, + 1.0475524e-07, + 1.0475524e-07, + 1.0475524e-07, + 1.0475524e-07, + 1.0475524e-07, + 1.0475524e-07, + 1.0475524e-07, + 1.0475524e-07, + 1.0475524e-07, + 1.0475524e-07, + 1.0475524e-07, + 1.0475524e-07, + 1.0475524e-07, + 1.0475524e-07, + 1.0475524e-07, + 1.0475524e-07, + 1.0475524e-07, + 1.0475524e-07, + ], }, "uv_anisotropy_flag": 1000, "glows_flags": 4, @@ -105,7 +226,13 @@ def test_create_glows_l3b_json_file_from_cdf(self, mock_open_file, mock_json, mo create_glows_l3b_json_file_from_cdf(l3b_path) - data_path = Path(imap_l3_processing.__file__).parent / 'glows' / 'l3d' / 'science' / 'data_l3b' + data_path = ( + Path(imap_l3_processing.__file__).parent + / "glows" + / "l3d" + / "science" + / "data_l3b" + ) mock_os.makedirs.assert_called_once_with(data_path, exist_ok=True) @@ -113,27 +240,35 @@ def test_create_glows_l3b_json_file_from_cdf(self, mock_open_file, mock_json, mo actual = mock_json.dump.call_args.args[0] self.assertEqual(json_file, mock_json.dump.call_args.args[1]) - mock_open_file.assert_called_once_with(data_path / 'imap_glows_l3b_cr_2096_v011.json', 'w') + mock_open_file.assert_called_once_with( + data_path / "imap_glows_l3b_cr_2096_v011.json", "w" + ) - self.assertIsInstance(actual['CR'], int) - self.assertEqual(expected['CR'], actual['CR']) + self.assertIsInstance(actual["CR"], int) + self.assertEqual(expected["CR"], actual["CR"]) - self.assertIsInstance(actual['date'], str) - self.assertEqual(actual['date'], expected['date']) + self.assertIsInstance(actual["date"], str) + self.assertEqual(actual["date"], expected["date"]) - np.testing.assert_array_equal(expected["header"]['filename'], - actual["header"]['filename']) + np.testing.assert_array_equal( + expected["header"]["filename"], actual["header"]["filename"] + ) - self.assertIsInstance(actual['uv_anisotropy_factor'], list) - np.testing.assert_array_equal(expected["uv_anisotropy_factor"], - actual["uv_anisotropy_factor"]) + self.assertIsInstance(actual["uv_anisotropy_factor"], list) + np.testing.assert_array_equal( + expected["uv_anisotropy_factor"], actual["uv_anisotropy_factor"] + ) - np.testing.assert_array_equal(expected["ion_rate_profile"]['lat_grid'], - actual["ion_rate_profile"]['lat_grid']) + np.testing.assert_array_equal( + expected["ion_rate_profile"]["lat_grid"], + actual["ion_rate_profile"]["lat_grid"], + ) self.assertIsInstance(actual["ion_rate_profile"]["lat_grid"], list) - np.testing.assert_array_almost_equal(expected["ion_rate_profile"]['ph_rate'], - actual["ion_rate_profile"]['ph_rate']) + np.testing.assert_array_almost_equal( + expected["ion_rate_profile"]["ph_rate"], + actual["ion_rate_profile"]["ph_rate"], + ) self.assertIsInstance(actual["ion_rate_profile"]["ph_rate"], list) self.assertIsInstance(actual["uv_anisotropy_flag"], int) @@ -146,34 +281,88 @@ def test_convert_json_l3d_to_data_product(self): input_metadata = Mock(spec=InputMetadata) input_metadata.start_date = datetime.fromisoformat("2025-05-12") l3d_data_product: GlowsL3DSolarParamsHistory = convert_json_to_l3d_data_product( - get_test_data_path('glows/imap_glows_l3d_solar-params-history_19470303-cr02095_v00.json'), + get_test_data_path( + "glows/imap_glows_l3d_solar-params-history_19470303-cr02095_v00.json" + ), input_metadata, sentinel.parent_file_names, ) self.assertEqual(input_metadata, l3d_data_product.input_metadata) - self.assertEqual(input_metadata.start_date, datetime.fromisoformat("1947-03-03").date()) + self.assertEqual( + input_metadata.start_date, datetime.fromisoformat("1947-03-03").date() + ) self.assertEqual(sentinel.parent_file_names, l3d_data_product.parent_file_names) np.testing.assert_array_equal( - [-90, -80, -70, -60, -50, -40, -30, -20, -10, 0, 10, 20, 30, 40, 50, 60, 70, 80, 90], - l3d_data_product.latitude) + [ + -90, + -80, + -70, + -60, + -50, + -40, + -30, + -20, + -10, + 0, + 10, + 20, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + ], + l3d_data_product.latitude, + ) self.assertEqual(846, len(l3d_data_product.cr)) self.assertEqual(1250.5, l3d_data_product.cr[0]) self.assertEqual(2095.5, l3d_data_product.cr[-1]) self.assertEqual(846, len(l3d_data_product.epoch)) - self.assertEqual(datetime.fromisoformat("1947-03-03 07:35:32.640"), l3d_data_product.epoch[0]) - self.assertEqual(datetime.fromisoformat("2010-04-08 22:40:35.040"), l3d_data_product.epoch[-1]) + self.assertEqual( + datetime.fromisoformat("1947-03-03 07:35:32.640"), l3d_data_product.epoch[0] + ) + self.assertEqual( + datetime.fromisoformat("2010-04-08 22:40:35.040"), + l3d_data_product.epoch[-1], + ) - np.testing.assert_array_equal(np.arange(-90, 100, step=10), l3d_data_product.latitude) + np.testing.assert_array_equal( + np.arange(-90, 100, step=10), l3d_data_product.latitude + ) self.assertEqual(846, len(l3d_data_product.plasma_speed)) np.testing.assert_allclose( - [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, - -1.0, -1.0, -1.0], l3d_data_product.plasma_speed[0], rtol=1e-6) + [ + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + ], + l3d_data_product.plasma_speed[0], + rtol=1e-6, + ) np.testing.assert_allclose( [ 468.1725019310069, @@ -194,15 +383,38 @@ def test_convert_json_l3d_to_data_product(self): 460.79164287941717, 503.2472821431676, 508.93503849297326, - 509.10575708733097 + 509.10575708733097, ], l3d_data_product.plasma_speed[-1], - rtol=1e-6) + rtol=1e-6, + ) self.assertEqual(846, len(l3d_data_product.proton_density)) np.testing.assert_allclose( - [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, - -1.0, -1.0, -1.0], l3d_data_product.proton_density[0], rtol=1e-6) + [ + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + ], + l3d_data_product.proton_density[0], + rtol=1e-6, + ) np.testing.assert_allclose( [ 3.134819994443553, @@ -223,40 +435,103 @@ def test_convert_json_l3d_to_data_product(self): 3.1651607050304924, 2.68129704271522, 2.6272758631787716, - 2.625943370301117 - ], l3d_data_product.proton_density[-1], rtol=1e-6) + 2.625943370301117, + ], + l3d_data_product.proton_density[-1], + rtol=1e-6, + ) self.assertEqual(846, len(l3d_data_product.ultraviolet_anisotropy)) np.testing.assert_allclose( - [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, - -1.0, -1.0, -1.0], l3d_data_product.ultraviolet_anisotropy[0], rtol=1e-6) + [ + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + ], + l3d_data_product.ultraviolet_anisotropy[0], + rtol=1e-6, + ) np.testing.assert_allclose( - [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 - ], l3d_data_product.ultraviolet_anisotropy[-1], rtol=1e-6) + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + ], + l3d_data_product.ultraviolet_anisotropy[-1], + rtol=1e-6, + ) self.assertEqual(846, len(l3d_data_product.phion)) - np.testing.assert_allclose(1.830438692701064e-07, l3d_data_product.phion[0], rtol=1e-6) - np.testing.assert_allclose(1.0538921234648196e-07, l3d_data_product.phion[-1], rtol=1e-6) + np.testing.assert_allclose( + 1.830438692701064e-07, l3d_data_product.phion[0], rtol=1e-6 + ) + np.testing.assert_allclose( + 1.0538921234648196e-07, l3d_data_product.phion[-1], rtol=1e-6 + ) self.assertEqual(846, len(l3d_data_product.lyman_alpha)) - np.testing.assert_allclose(619975763789.9209, l3d_data_product.lyman_alpha[0], rtol=1e-6) - np.testing.assert_allclose(396312787467.8571, l3d_data_product.lyman_alpha[-1], rtol=1e-6) + np.testing.assert_allclose( + 619975763789.9209, l3d_data_product.lyman_alpha[0], rtol=1e-6 + ) + np.testing.assert_allclose( + 396312787467.8571, l3d_data_product.lyman_alpha[-1], rtol=1e-6 + ) self.assertEqual(846, len(l3d_data_product.electron_density)) self.assertEqual(-1.0, l3d_data_product.electron_density[0]) - np.testing.assert_allclose(5.978193763000784, l3d_data_product.electron_density[-1], rtol=1e-6) + np.testing.assert_allclose( + 5.978193763000784, l3d_data_product.electron_density[-1], rtol=1e-6 + ) self.assertEqual(846, len(l3d_data_product.plasma_speed_flag)) self.assertEqual(-1.0, l3d_data_product.plasma_speed_flag[0]) - np.testing.assert_allclose(20001.0, l3d_data_product.plasma_speed_flag[-1], rtol=1e-6) + np.testing.assert_allclose( + 20001.0, l3d_data_product.plasma_speed_flag[-1], rtol=1e-6 + ) self.assertEqual(846, len(l3d_data_product.uv_anisotropy_flag)) self.assertEqual(-1.0, l3d_data_product.uv_anisotropy_flag[0]) - np.testing.assert_allclose(30001.0, l3d_data_product.uv_anisotropy_flag[-1], rtol=1e-6) + np.testing.assert_allclose( + 30001.0, l3d_data_product.uv_anisotropy_flag[-1], rtol=1e-6 + ) self.assertEqual(846, len(l3d_data_product.proton_density_flag)) self.assertEqual(-1.0, l3d_data_product.proton_density_flag[0]) - np.testing.assert_allclose(10001.0, l3d_data_product.proton_density_flag[-1], rtol=1e-6) + np.testing.assert_allclose( + 10001.0, l3d_data_product.proton_density_flag[-1], rtol=1e-6 + ) self.assertEqual(846, len(l3d_data_product.glows_flags)) self.assertEqual(0, int(l3d_data_product.glows_flags[0])) @@ -278,7 +553,7 @@ def test_get_parent_file_names_from_l3d_json(self): "imap_glows_l3b_ion-rate-profile_20100422_v011.cdf", "imap_glows_l3b_ion-rate-profile_20100519_v011.cdf", "imap_glows_l3c_sw-profile_20100422_v011.cdf", - "imap_glows_l3c_sw-profile_20100519_v011.cdf" + "imap_glows_l3c_sw-profile_20100519_v011.cdf", ] self.assertCountEqual(expected, l3bc_filenames) @@ -288,7 +563,9 @@ def test_rename_l3d_text_outputs(self, mock_get_midpoint_of_cr): with tempfile.TemporaryDirectory() as tmpdir: tmpdir = Path(tmpdir) original_lya_path = tmpdir / "imap_glows_l3d_lya_19470303-cr02092_v00.dat" - original_p_dens_path = tmpdir / "imap_glows_l3d_p-dens_19470303-cr02092_v00.dat" + original_p_dens_path = ( + tmpdir / "imap_glows_l3d_p-dens_19470303-cr02092_v00.dat" + ) original_lya_path.write_text("lya") original_p_dens_path.write_text("p_dens") @@ -296,14 +573,20 @@ def test_rename_l3d_text_outputs(self, mock_get_midpoint_of_cr): mock_get_midpoint_of_cr.return_value = datetime(2010, 2, 1) version = "v012" - actual_new_paths = rename_l3d_text_outputs([original_lya_path, original_p_dens_path], version) + actual_new_paths = rename_l3d_text_outputs( + [original_lya_path, original_p_dens_path], version + ) mock_get_midpoint_of_cr.assert_has_calls([call(2092), call(2092)]) expected_lya_output = tmpdir / "imap_glows_lya_19470303_20100201_v012.dat" - expected_p_dens_output = tmpdir / "imap_glows_p-dens_19470303_20100201_v012.dat" + expected_p_dens_output = ( + tmpdir / "imap_glows_p-dens_19470303_20100201_v012.dat" + ) - self.assertEqual([expected_lya_output, expected_p_dens_output], actual_new_paths) + self.assertEqual( + [expected_lya_output, expected_p_dens_output], actual_new_paths + ) self.assertEqual("lya", expected_lya_output.read_text()) self.assertEqual("p_dens", expected_p_dens_output.read_text()) @@ -312,15 +595,23 @@ def test_rename_l3d_text_outputs(self, mock_get_midpoint_of_cr): self.assertFalse(original_p_dens_path.exists()) def test_get_most_recently_uploaded_ancillary(self): - query_result = create_mock_query_results([ - "imap_glows_l3d_solar-hist_20100101-cr02091_v001.cdf", - "imap_glows_l3d_solar-hist_20100201-cr02092_v002.cdf", - "imap_glows_l3d_solar-hist_20100301-cr02093_v001.cdf" - - ], ingestion_dates=[datetime(2010, 1, 2), datetime(2010, 5, 2), datetime(2010, 3, 2)]) + query_result = create_mock_query_results( + [ + "imap_glows_l3d_solar-hist_20100101-cr02091_v001.cdf", + "imap_glows_l3d_solar-hist_20100201-cr02092_v002.cdf", + "imap_glows_l3d_solar-hist_20100301-cr02093_v001.cdf", + ], + ingestion_dates=[ + datetime(2010, 1, 2), + datetime(2010, 5, 2), + datetime(2010, 3, 2), + ], + ) - [expected] = create_mock_query_results(["imap_glows_l3d_solar-hist_20100201-cr02092_v002.cdf"], - ingestion_dates=[datetime(2010, 5, 2)]) + [expected] = create_mock_query_results( + ["imap_glows_l3d_solar-hist_20100201-cr02092_v002.cdf"], + ingestion_dates=[datetime(2010, 5, 2)], + ) self.assertEqual(expected, get_most_recently_uploaded_ancillary(query_result)) diff --git a/tests/glows/l3d/test_models.py b/tests/glows/l3d/test_models.py index 47bfecfef..89930104f 100644 --- a/tests/glows/l3d/test_models.py +++ b/tests/glows/l3d/test_models.py @@ -3,11 +3,24 @@ import numpy as np from imap_l3_processing.constants import CARRINGTON_ROTATION_IN_NANOSECONDS -from imap_l3_processing.glows.l3d.models import GlowsL3DSolarParamsHistory, SPEED_CDF_VAR_NAME, PHION_CDF_VAR_NAME, \ - EPOCH_CDF_VAR_NAME, EPOCH_DELTA_CDF_VAR_NAME, LATITUDE_CDF_CDF_VAR_NAME, \ - LATITUDE_LABEL_CDF_VAR_NAME, CR_CDF_VAR_NAME, PROTON_DENSITY_CDF_VAR_NAME, UV_ANISOTROPY_CDF_VAR_NAME, \ - LYMAN_ALPHA_CDF_VAR_NAME, ELECTRON_DENSITY_CDF_VAR_NAME, PLASMA_SPEED_FLAG_CDF_VAR_NAME, \ - UV_ANISOTROPY_FLAG_CDF_VAR_NAME, PROTON_DENSITY_FLAG_CDF_VAR_NAME, GLOWS_FLAGS_CDF_VAR_NAME +from imap_l3_processing.glows.l3d.models import ( + GlowsL3DSolarParamsHistory, + SPEED_CDF_VAR_NAME, + PHION_CDF_VAR_NAME, + EPOCH_CDF_VAR_NAME, + EPOCH_DELTA_CDF_VAR_NAME, + LATITUDE_CDF_CDF_VAR_NAME, + LATITUDE_LABEL_CDF_VAR_NAME, + CR_CDF_VAR_NAME, + PROTON_DENSITY_CDF_VAR_NAME, + UV_ANISOTROPY_CDF_VAR_NAME, + LYMAN_ALPHA_CDF_VAR_NAME, + ELECTRON_DENSITY_CDF_VAR_NAME, + PLASMA_SPEED_FLAG_CDF_VAR_NAME, + UV_ANISOTROPY_FLAG_CDF_VAR_NAME, + PROTON_DENSITY_FLAG_CDF_VAR_NAME, + GLOWS_FLAGS_CDF_VAR_NAME, +) from tests.swapi.cdf_model_test_case import CdfModelTestCase @@ -49,23 +62,49 @@ def test_glows_l3d_solar_params_history_to_data_product(self): variables = data_product.to_data_product_variables() - expected_epoch_delta = np.full_like(epoch, CARRINGTON_ROTATION_IN_NANOSECONDS / 2) - expected_latitude_label = [f'{deg:.1f} degrees' for deg in latitude] + expected_epoch_delta = np.full_like( + epoch, CARRINGTON_ROTATION_IN_NANOSECONDS / 2 + ) + expected_latitude_label = [f"{deg:.1f} degrees" for deg in latitude] self.assertEqual(15, len(variables)) variables = iter(variables) self.assert_variable_attributes(next(variables), epoch, EPOCH_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), expected_epoch_delta, EPOCH_DELTA_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), latitude, LATITUDE_CDF_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), expected_latitude_label, LATITUDE_LABEL_CDF_VAR_NAME) + self.assert_variable_attributes( + next(variables), expected_epoch_delta, EPOCH_DELTA_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), latitude, LATITUDE_CDF_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), expected_latitude_label, LATITUDE_LABEL_CDF_VAR_NAME + ) self.assert_variable_attributes(next(variables), cr, CR_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), plasma_speed, SPEED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), proton_density, PROTON_DENSITY_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), ultraviolet_anisotropy, UV_ANISOTROPY_CDF_VAR_NAME) + self.assert_variable_attributes( + next(variables), plasma_speed, SPEED_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), proton_density, PROTON_DENSITY_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), ultraviolet_anisotropy, UV_ANISOTROPY_CDF_VAR_NAME + ) self.assert_variable_attributes(next(variables), phion, PHION_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), lyman_alpha, LYMAN_ALPHA_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), electron_density, ELECTRON_DENSITY_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), plasma_speed_flag, PLASMA_SPEED_FLAG_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), uv_anisotropy_flag, UV_ANISOTROPY_FLAG_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), proton_density_flag, PROTON_DENSITY_FLAG_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), glows_flags, GLOWS_FLAGS_CDF_VAR_NAME) \ No newline at end of file + self.assert_variable_attributes( + next(variables), lyman_alpha, LYMAN_ALPHA_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), electron_density, ELECTRON_DENSITY_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), plasma_speed_flag, PLASMA_SPEED_FLAG_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), uv_anisotropy_flag, UV_ANISOTROPY_FLAG_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), proton_density_flag, PROTON_DENSITY_FLAG_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), glows_flags, GLOWS_FLAGS_CDF_VAR_NAME + ) diff --git a/tests/glows/l3e/test_glows_l3e_call_arguments.py b/tests/glows/l3e/test_glows_l3e_call_arguments.py index edcaa3f50..22a4c65e8 100644 --- a/tests/glows/l3e/test_glows_l3e_call_arguments.py +++ b/tests/glows/l3e/test_glows_l3e_call_arguments.py @@ -2,11 +2,13 @@ import numpy as np -from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import GlowsL3eCallArguments, GlowsL3eSpacecraftInfo +from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import ( + GlowsL3eCallArguments, + GlowsL3eSpacecraftInfo, +) class TestGlowsL3eCallArguments(unittest.TestCase): - def test_to_argument_list(self): glows_l3e_call_arguments = GlowsL3eCallArguments( elongation=80.0001, @@ -14,14 +16,14 @@ def test_to_argument_list(self): formatted_date="20240101", spacecraft_info=GlowsL3eSpacecraftInfo( spacecraft_radius=np.float32(987), - spin_axis_latitude=np.float32(.01234), + spin_axis_latitude=np.float32(0.01234), spin_axis_longitude=np.float32(87.11213), spacecraft_latitude=np.float32(1.134141), spacecraft_longitude=np.float32(84.12314), spacecraft_velocity_x=np.float32(1.23143), spacecraft_velocity_y=np.float32(12.34141), spacecraft_velocity_z=np.float32(123), - ) + ), ) call_args_list = glows_l3e_call_arguments.to_argument_list() diff --git a/tests/glows/l3e/test_glows_l3e_dependencies.py b/tests/glows/l3e/test_glows_l3e_dependencies.py index 534d1e2e3..5b5fdddde 100644 --- a/tests/glows/l3e/test_glows_l3e_dependencies.py +++ b/tests/glows/l3e/test_glows_l3e_dependencies.py @@ -10,23 +10,25 @@ class TestGlowsL3EDependencies(TestCase): - @patch('imap_l3_processing.glows.l3e.glows_l3e_dependencies.imap_data_access.download') + @patch( + "imap_l3_processing.glows.l3e.glows_l3e_dependencies.imap_data_access.download" + ) def test_fetch_dependencies(self, mock_download): mock_processing_input_collection = Mock() - mock_energy_grid_lo = Path('energy_grid_lo_path') - mock_energy_grid_hi = Path('energy_grid_hi_sdc_path') - mock_energy_grid_ultra = Path('energy_grid_ultra_sdc_path') - mock_tess_xyz_8 = Path('tess_xyz_8_sdc_path') - mock_lya_series = Path('lya_series_sdc_path') - mock_solar_uv_anisotropy = Path('solar_uv_anisotropy_sdc_path') - mock_speed_3d_sw = Path('speed_3d_sw_sdc_path') - mock_density_3d_sw = Path('density_3d_sw_sdc_path') - mock_phion_hydrogen = Path('phion_hydrogen_sdc_path') - mock_sw_eqtr_electrons = Path('sw_eqtr_electrons_sdc_path') - mock_pipeline_settings = Path('l3bcde_pipeline_settings.json') - mock_tess_ang_16 = Path('tess_ang_16_path') - mock_repoint_file = Path('repoint.csv') + mock_energy_grid_lo = Path("energy_grid_lo_path") + mock_energy_grid_hi = Path("energy_grid_hi_sdc_path") + mock_energy_grid_ultra = Path("energy_grid_ultra_sdc_path") + mock_tess_xyz_8 = Path("tess_xyz_8_sdc_path") + mock_lya_series = Path("lya_series_sdc_path") + mock_solar_uv_anisotropy = Path("solar_uv_anisotropy_sdc_path") + mock_speed_3d_sw = Path("speed_3d_sw_sdc_path") + mock_density_3d_sw = Path("density_3d_sw_sdc_path") + mock_phion_hydrogen = Path("phion_hydrogen_sdc_path") + mock_sw_eqtr_electrons = Path("sw_eqtr_electrons_sdc_path") + mock_pipeline_settings = Path("l3bcde_pipeline_settings.json") + mock_tess_ang_16 = Path("tess_ang_16_path") + mock_repoint_file = Path("repoint.csv") mock_processing_input_collection.get_file_paths.side_effect = [ [mock_lya_series], @@ -41,7 +43,7 @@ def test_fetch_dependencies(self, mock_download): [mock_energy_grid_lo], [mock_energy_grid_hi], [mock_energy_grid_ultra], - [mock_repoint_file] + [mock_repoint_file], ] mock_lya_series_path = Mock() @@ -50,7 +52,9 @@ def test_fetch_dependencies(self, mock_download): mock_density_3d_sw_path = Mock() mock_phion_hydrogen_path = Mock() mock_sw_eqtr_electrons_path = Mock() - pipeline_settings_path = get_test_data_path("glows/imap_glows_pipeline-settings-l3bcde_20251113_v004.json") + pipeline_settings_path = get_test_data_path( + "glows/imap_glows_pipeline-settings-l3bcde_20251113_v004.json" + ) mock_energy_grid_lo_path = Mock() mock_tess_xyz_8_path = Mock() mock_energy_grid_hi_path = Mock() @@ -74,62 +78,79 @@ def test_fetch_dependencies(self, mock_download): mock_downloaded_repoint_file, ] - actual_dependencies = GlowsL3EDependencies.fetch_dependencies(mock_processing_input_collection) + actual_dependencies = GlowsL3EDependencies.fetch_dependencies( + mock_processing_input_collection + ) self.assertEqual(13, mock_processing_input_collection.get_file_paths.call_count) - mock_processing_input_collection.get_file_paths.assert_has_calls([ - call(source="glows", descriptor="lya"), - call(source="glows", descriptor="uv-anis"), - call(source="glows", descriptor="speed"), - call(source="glows", descriptor="p-dens"), - call(source="glows", descriptor="phion"), - call(source="glows", descriptor="e-dens"), - call(source="glows", descriptor="pipeline-settings-l3bcde"), - call(source="glows", descriptor="tess-xyz-8"), - call(source="glows", descriptor="tess-ang-16"), - call(source="glows", descriptor="energy-grid-lo"), - call(source="glows", descriptor="energy-grid-hi"), - call(source="glows", descriptor="energy-grid-ultra"), - call(data_type=RepointInput.data_type), - ]) - - mock_download.assert_has_calls([ - call(mock_lya_series), - call(mock_solar_uv_anisotropy), - call(mock_speed_3d_sw), - call(mock_density_3d_sw), - call(mock_phion_hydrogen), - call(mock_sw_eqtr_electrons), - call(mock_pipeline_settings), - call(mock_tess_xyz_8), - call(mock_tess_ang_16), - call(mock_energy_grid_lo), - call(mock_energy_grid_hi), - call(mock_energy_grid_ultra), - call(mock_repoint_file), - ], any_order=False) - - expected_pipeline_settings = {"executable_dependency_paths": { - "energy-grid-lo": "EnGridLo.dat", - "energy-grid-hi": "EnGridHi.dat", - "energy-grid-ultra": "EnGridUltra.dat", - "tess-xyz-8": "tessXYZ8.dat", - "tess-ang-16": "tessAng16.dat", - "ionization-files": "ionization.files.dat", - }} + mock_processing_input_collection.get_file_paths.assert_has_calls( + [ + call(source="glows", descriptor="lya"), + call(source="glows", descriptor="uv-anis"), + call(source="glows", descriptor="speed"), + call(source="glows", descriptor="p-dens"), + call(source="glows", descriptor="phion"), + call(source="glows", descriptor="e-dens"), + call(source="glows", descriptor="pipeline-settings-l3bcde"), + call(source="glows", descriptor="tess-xyz-8"), + call(source="glows", descriptor="tess-ang-16"), + call(source="glows", descriptor="energy-grid-lo"), + call(source="glows", descriptor="energy-grid-hi"), + call(source="glows", descriptor="energy-grid-ultra"), + call(data_type=RepointInput.data_type), + ] + ) + + mock_download.assert_has_calls( + [ + call(mock_lya_series), + call(mock_solar_uv_anisotropy), + call(mock_speed_3d_sw), + call(mock_density_3d_sw), + call(mock_phion_hydrogen), + call(mock_sw_eqtr_electrons), + call(mock_pipeline_settings), + call(mock_tess_xyz_8), + call(mock_tess_ang_16), + call(mock_energy_grid_lo), + call(mock_energy_grid_hi), + call(mock_energy_grid_ultra), + call(mock_repoint_file), + ], + any_order=False, + ) + + expected_pipeline_settings = { + "executable_dependency_paths": { + "energy-grid-lo": "EnGridLo.dat", + "energy-grid-hi": "EnGridHi.dat", + "energy-grid-ultra": "EnGridUltra.dat", + "tess-xyz-8": "tessXYZ8.dat", + "tess-ang-16": "tessAng16.dat", + "ionization-files": "ionization.files.dat", + } + } self.assertEqual(mock_lya_series_path, actual_dependencies.lya_series) - self.assertEqual(mock_solar_uv_anisotropy_path, actual_dependencies.solar_uv_anisotropy) + self.assertEqual( + mock_solar_uv_anisotropy_path, actual_dependencies.solar_uv_anisotropy + ) self.assertEqual(mock_speed_3d_sw_path, actual_dependencies.speed_3d_sw) self.assertEqual(mock_density_3d_sw_path, actual_dependencies.density_3d_sw) self.assertEqual(mock_phion_hydrogen_path, actual_dependencies.phion_hydrogen) - self.assertEqual(mock_sw_eqtr_electrons_path, actual_dependencies.sw_eqtr_electrons) - self.assertEqual(expected_pipeline_settings['executable_dependency_paths'], - actual_dependencies.pipeline_settings['executable_dependency_paths']) + self.assertEqual( + mock_sw_eqtr_electrons_path, actual_dependencies.sw_eqtr_electrons + ) + self.assertEqual( + expected_pipeline_settings["executable_dependency_paths"], + actual_dependencies.pipeline_settings["executable_dependency_paths"], + ) self.assertEqual(mock_energy_grid_lo_path, actual_dependencies.energy_grid_lo) self.assertEqual(mock_energy_grid_hi_path, actual_dependencies.energy_grid_hi) - self.assertEqual(mock_energy_grid_ultra_path, actual_dependencies.energy_grid_ultra) + self.assertEqual( + mock_energy_grid_ultra_path, actual_dependencies.energy_grid_ultra + ) self.assertEqual(mock_tess_xyz_8_path, actual_dependencies.tess_xyz_8) self.assertEqual(mock_tess_ang_16_path, actual_dependencies.tess_ang16) self.assertEqual( @@ -140,7 +161,9 @@ def test_fetch_dependencies(self, mock_download): mock_downloaded_repoint_file, actual_dependencies.repointing_file ) - @patch("imap_l3_processing.glows.l3e.glows_l3e_dependencies.furnish_spice_metakernel") + @patch( + "imap_l3_processing.glows.l3e.glows_l3e_dependencies.furnish_spice_metakernel" + ) def test_collect_spice_dependencies(self, mock_furnish_spice_metakernel): expected_kernel_types_with_predicted = [ SpiceKernelTypes.ScienceFrames, @@ -150,47 +173,65 @@ def test_collect_spice_dependencies(self, mock_furnish_spice_metakernel): SpiceKernelTypes.PointingAttitude, SpiceKernelTypes.PlanetaryEphemeris, SpiceKernelTypes.Leapseconds, - SpiceKernelTypes.SpacecraftClock + SpiceKernelTypes.SpacecraftClock, ] expected_kernel_types_without_predicted = [ - x for x in expected_kernel_types_with_predicted if x != SpiceKernelTypes.EphemerisPredicted + x + for x in expected_kernel_types_with_predicted + if x != SpiceKernelTypes.EphemerisPredicted ] metakernel_with_predict_path = Path("metakernel_with_predict_ephem.txt") - kernels_with_predict = [Path("some_kernel_with_imap_data"), Path("predicted_ephem"), Path("some_kernel_with_planet_data")] + kernels_with_predict = [ + Path("some_kernel_with_imap_data"), + Path("predicted_ephem"), + Path("some_kernel_with_planet_data"), + ] metakernel_without_predict_path = Path("metakernel_without_predict_ephem.txt") - kernels_without_predict = [Path("some_kernel_with_imap_data"), Path("some_kernel_with_planet_data")] - metakernel_output_with_predict = FurnishMetakernelOutput(metakernel_path=metakernel_with_predict_path, - spice_kernel_paths=kernels_with_predict, ) - metakernel_output_without_predict = FurnishMetakernelOutput(metakernel_path=metakernel_without_predict_path, - spice_kernel_paths=kernels_without_predict) + kernels_without_predict = [ + Path("some_kernel_with_imap_data"), + Path("some_kernel_with_planet_data"), + ] + metakernel_output_with_predict = FurnishMetakernelOutput( + metakernel_path=metakernel_with_predict_path, + spice_kernel_paths=kernels_with_predict, + ) + metakernel_output_without_predict = FurnishMetakernelOutput( + metakernel_path=metakernel_without_predict_path, + spice_kernel_paths=kernels_without_predict, + ) mock_furnish_spice_metakernel.side_effect = [ metakernel_output_with_predict, - metakernel_output_without_predict + metakernel_output_without_predict, ] - actual_metakernel_with_predict, actual_metakernel_without_predict = GlowsL3EDependencies.collect_spice_dependencies(sentinel.start_date, sentinel.end_date) - - mock_furnish_spice_metakernel.assert_has_calls([ - call( - start_date=sentinel.start_date, - end_date=sentinel.end_date, - kernel_types=expected_kernel_types_with_predicted, - metakernel_file_name="metakernel_with_predict_ephem.txt", - ), - call( - start_date=sentinel.start_date, - end_date=sentinel.end_date, - kernel_types=expected_kernel_types_without_predicted, - metakernel_file_name="metakernel_without_predict_ephem.txt", + actual_metakernel_with_predict, actual_metakernel_without_predict = ( + GlowsL3EDependencies.collect_spice_dependencies( + sentinel.start_date, sentinel.end_date + ) + ) - ), - ]) + mock_furnish_spice_metakernel.assert_has_calls( + [ + call( + start_date=sentinel.start_date, + end_date=sentinel.end_date, + kernel_types=expected_kernel_types_with_predicted, + metakernel_file_name="metakernel_with_predict_ephem.txt", + ), + call( + start_date=sentinel.start_date, + end_date=sentinel.end_date, + kernel_types=expected_kernel_types_without_predicted, + metakernel_file_name="metakernel_without_predict_ephem.txt", + ), + ] + ) self.assertEqual(metakernel_output_with_predict, actual_metakernel_with_predict) - self.assertEqual(metakernel_output_without_predict, actual_metakernel_without_predict) - - + self.assertEqual( + metakernel_output_without_predict, actual_metakernel_without_predict + ) def test_get_hi_parents(self): dependencies = self._create_l3e_dependencies() @@ -245,21 +286,21 @@ def test_get_ul_parents(self): self.assertEqual(expected_parent_file_names, dependencies.get_ul_parents()) - @patch('builtins.open', new_callable=mock_open) - @patch('imap_l3_processing.glows.l3e.glows_l3e_dependencies.shutil') + @patch("builtins.open", new_callable=mock_open) + @patch("imap_l3_processing.glows.l3e.glows_l3e_dependencies.shutil") def test_copy_dependencies(self, mock_shutil, mock_file): glows_l3e_dependencies = GlowsL3EDependencies( - Path('2025/05/03/imap_glows_energy_grid_lo'), - Path('2025/05/03/imap_glows_energy_grid_hi'), - Path('2025/05/03/imap_glows_energy_grid_ultra'), - Path('2025/05/03/imap_glows_tess_xyz_8'), - Path('2025/05/03/imap_glows_tess_ang16'), - Path('2025/05/03/imap_glows_lya_series'), - Path('2025/05/03/imap_glows_solar_uv_anisotropy'), - Path('2025/05/03/imap_glows_speed_3d_sw'), - Path('2025/05/03/imap_glows_density_3d_sw'), - Path('2025/05/03/imap_glows_phion_hydrogen'), - Path('2025/05/03/imap_glows_sw_eqtr_electrons'), + Path("2025/05/03/imap_glows_energy_grid_lo"), + Path("2025/05/03/imap_glows_energy_grid_hi"), + Path("2025/05/03/imap_glows_energy_grid_ultra"), + Path("2025/05/03/imap_glows_tess_xyz_8"), + Path("2025/05/03/imap_glows_tess_ang16"), + Path("2025/05/03/imap_glows_lya_series"), + Path("2025/05/03/imap_glows_solar_uv_anisotropy"), + Path("2025/05/03/imap_glows_speed_3d_sw"), + Path("2025/05/03/imap_glows_density_3d_sw"), + Path("2025/05/03/imap_glows_phion_hydrogen"), + Path("2025/05/03/imap_glows_sw_eqtr_electrons"), { "executable_dependency_paths": { "energy-grid-lo": "EnGridLo.dat", @@ -271,40 +312,64 @@ def test_copy_dependencies(self, mock_shutil, mock_file): } }, Path("path/to/some/pipeline_settings_file.csv"), - Path("repoint.csv") + Path("repoint.csv"), ) - expected_energy_grid_lo = 'EnGridLo.dat' - expected_energy_grid_hi = 'EnGridHi.dat' - expected_energy_grid_ultra = 'EnGridUltra.dat' - expected_tess_xyz_8 = 'tessXYZ8.dat' - expected_tess_ang16 = 'tessAng16.dat' + expected_energy_grid_lo = "EnGridLo.dat" + expected_energy_grid_hi = "EnGridHi.dat" + expected_energy_grid_ultra = "EnGridUltra.dat" + expected_tess_xyz_8 = "tessXYZ8.dat" + expected_tess_ang16 = "tessAng16.dat" glows_l3e_dependencies.copy_dependencies() - mock_shutil.copy.assert_has_calls([ - call(glows_l3e_dependencies.energy_grid_lo, expected_energy_grid_lo), - call(glows_l3e_dependencies.energy_grid_hi, expected_energy_grid_hi), - call(glows_l3e_dependencies.energy_grid_ultra, expected_energy_grid_ultra), - call(glows_l3e_dependencies.tess_xyz_8, expected_tess_xyz_8), - call(glows_l3e_dependencies.tess_ang16, expected_tess_ang16), - call(glows_l3e_dependencies.lya_series, glows_l3e_dependencies.lya_series.name), - call(glows_l3e_dependencies.solar_uv_anisotropy, glows_l3e_dependencies.solar_uv_anisotropy.name), - call(glows_l3e_dependencies.speed_3d_sw, glows_l3e_dependencies.speed_3d_sw.name), - call(glows_l3e_dependencies.density_3d_sw, glows_l3e_dependencies.density_3d_sw.name), - call(glows_l3e_dependencies.phion_hydrogen, glows_l3e_dependencies.phion_hydrogen.name), - call(glows_l3e_dependencies.sw_eqtr_electrons, glows_l3e_dependencies.sw_eqtr_electrons.name), - ]) + mock_shutil.copy.assert_has_calls( + [ + call(glows_l3e_dependencies.energy_grid_lo, expected_energy_grid_lo), + call(glows_l3e_dependencies.energy_grid_hi, expected_energy_grid_hi), + call( + glows_l3e_dependencies.energy_grid_ultra, expected_energy_grid_ultra + ), + call(glows_l3e_dependencies.tess_xyz_8, expected_tess_xyz_8), + call(glows_l3e_dependencies.tess_ang16, expected_tess_ang16), + call( + glows_l3e_dependencies.lya_series, + glows_l3e_dependencies.lya_series.name, + ), + call( + glows_l3e_dependencies.solar_uv_anisotropy, + glows_l3e_dependencies.solar_uv_anisotropy.name, + ), + call( + glows_l3e_dependencies.speed_3d_sw, + glows_l3e_dependencies.speed_3d_sw.name, + ), + call( + glows_l3e_dependencies.density_3d_sw, + glows_l3e_dependencies.density_3d_sw.name, + ), + call( + glows_l3e_dependencies.phion_hydrogen, + glows_l3e_dependencies.phion_hydrogen.name, + ), + call( + glows_l3e_dependencies.sw_eqtr_electrons, + glows_l3e_dependencies.sw_eqtr_electrons.name, + ), + ] + ) mock_file.assert_called_once_with("ionization.files.dat", "w") - mock_file.return_value.write.assert_called_once_with(( - f"{glows_l3e_dependencies.lya_series.name}\n" - f"{glows_l3e_dependencies.solar_uv_anisotropy.name}\n" - f"{glows_l3e_dependencies.speed_3d_sw.name}\n" - f"{glows_l3e_dependencies.density_3d_sw.name}\n" - f"{glows_l3e_dependencies.phion_hydrogen.name}\n" - f"{glows_l3e_dependencies.sw_eqtr_electrons.name}\n" - )) + mock_file.return_value.write.assert_called_once_with( + ( + f"{glows_l3e_dependencies.lya_series.name}\n" + f"{glows_l3e_dependencies.solar_uv_anisotropy.name}\n" + f"{glows_l3e_dependencies.speed_3d_sw.name}\n" + f"{glows_l3e_dependencies.density_3d_sw.name}\n" + f"{glows_l3e_dependencies.phion_hydrogen.name}\n" + f"{glows_l3e_dependencies.sw_eqtr_electrons.name}\n" + ) + ) def _create_l3e_dependencies(self) -> GlowsL3EDependencies: return GlowsL3EDependencies( diff --git a/tests/glows/l3e/test_glows_l3e_hi_model.py b/tests/glows/l3e/test_glows_l3e_hi_model.py index 06a04b3f1..9212d8f92 100644 --- a/tests/glows/l3e/test_glows_l3e_hi_model.py +++ b/tests/glows/l3e/test_glows_l3e_hi_model.py @@ -4,7 +4,10 @@ import numpy as np -from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import GlowsL3eCallArguments, GlowsL3eSpacecraftInfo +from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import ( + GlowsL3eCallArguments, + GlowsL3eSpacecraftInfo, +) from imap_l3_processing.glows.l3e.glows_l3e_hi_model import GlowsL3EHiData from imap_l3_processing.models import DataProductVariable from tests.test_helpers import get_test_instrument_team_data_path @@ -14,21 +17,26 @@ class TestL3eHiModel(unittest.TestCase): def test_l3e_hi_model_to_data_product_variables(self): test_cases = { - 'case 1':( - [1.2234,2,4.51,66.7666], - [1,41650233.0,4.22,9.5], - ['1.22','2.00','4.51','66.77'], - ['1','41650233','4','10'] + "case 1": ( + [1.2234, 2, 4.51, 66.7666], + [1, 41650233.0, 4.22, 9.5], + ["1.22", "2.00", "4.51", "66.77"], + ["1", "41650233", "4", "10"], + ), + "case 2": ( + [4.536, 48.193, 4253.1], + [13, 14.0, 34.2, 19.5], + ["4.54", "48.19", "4253.10"], + ["13", "14", "34", "20"], ), - 'case 2':( - [4.536,48.193,4253.1], - [13,14.0,34.2,19.5], - ['4.54','48.19','4253.10'], - ['13','14','34','20'] - ) } - for name, (energy_array, spin_angle_array, expected_energy_labels, expected_spin_angle_labels) in test_cases.items(): + for name, ( + energy_array, + spin_angle_array, + expected_energy_labels, + expected_spin_angle_labels, + ) in test_cases.items(): with self.subTest(name): l3e_hi: GlowsL3EHiData = GlowsL3EHiData( Mock(), @@ -58,134 +66,806 @@ def test_l3e_hi_model_to_data_product_variables(self): DataProductVariable("epoch", sentinel.epoch), DataProductVariable("epoch_delta", sentinel.epoch_delta), DataProductVariable("energy_grid", energy_array), - DataProductVariable("energy_delta_plus", sentinel.energy_delta_plus), - DataProductVariable("energy_delta_minus", sentinel.energy_delta_minus), + DataProductVariable( + "energy_delta_plus", sentinel.energy_delta_plus + ), + DataProductVariable( + "energy_delta_minus", sentinel.energy_delta_minus + ), DataProductVariable("spin_angle", spin_angle_array), DataProductVariable("surv_prob", sentinel.probability_of_survival), DataProductVariable("energy_label", expected_energy_labels), DataProductVariable("spin_angle_label", expected_spin_angle_labels), - DataProductVariable("spin_axis_latitude", sentinel.spin_axis_latitude), - DataProductVariable("spin_axis_longitude", sentinel.spin_axis_longitude), + DataProductVariable( + "spin_axis_latitude", sentinel.spin_axis_latitude + ), + DataProductVariable( + "spin_axis_longitude", sentinel.spin_axis_longitude + ), DataProductVariable("program_version", sentinel.program_version), - DataProductVariable("spacecraft_radius", sentinel.spacecraft_radius), - DataProductVariable("spacecraft_latitude", sentinel.spacecraft_latitude), - DataProductVariable("spacecraft_longitude", sentinel.spacecraft_longitude), - DataProductVariable("spacecraft_velocity_x", sentinel.spacecraft_velocity_x), - DataProductVariable("spacecraft_velocity_y", sentinel.spacecraft_velocity_y), - DataProductVariable("spacecraft_velocity_z", sentinel.spacecraft_velocity_z), + DataProductVariable( + "spacecraft_radius", sentinel.spacecraft_radius + ), + DataProductVariable( + "spacecraft_latitude", sentinel.spacecraft_latitude + ), + DataProductVariable( + "spacecraft_longitude", sentinel.spacecraft_longitude + ), + DataProductVariable( + "spacecraft_velocity_x", sentinel.spacecraft_velocity_x + ), + DataProductVariable( + "spacecraft_velocity_y", sentinel.spacecraft_velocity_y + ), + DataProductVariable( + "spacecraft_velocity_z", sentinel.spacecraft_velocity_z + ), DataProductVariable("elongation", sentinel.elongation), DataProductVariable("glows_flags", sentinel.glows_flags), ] self.assertEqual(expected_data_products, data_products) - @patch('imap_l3_processing.glows.l3e.glows_l3e_hi_model.calculate_energy_deltas') + @patch("imap_l3_processing.glows.l3e.glows_l3e_hi_model.calculate_energy_deltas") def test_convert_dat_to_glows_l3e_hi_product(self, mock_calculate_energy_deltas): - hi_file_path = get_test_instrument_team_data_path("glows/probSur.Imap.Hi_2009.000_90.00.txt") + hi_file_path = get_test_instrument_team_data_path( + "glows/probSur.Imap.Hi_2009.000_90.00.txt" + ) expected_epoch = datetime(year=2009, month=8, day=16) epoch_delta = timedelta(hours=8) - expected_epoch_delta_in_nanoseconds = 8*3600*1e9 - expected_energy = [0.3749896542976582, 0.4881381387720609, 0.6354277772546533, 0.8271602401781013, - 1.076745599456691, 1.401640250140305, 1.824567838312671, 2.375108588863466, - 3.091768193234094, 4.024670958420553, 5.239065580337116, 6.819888740878518, - 8.877705713882047, 11.55644347537280, 15.04345718406192, 19.58263409767632] + expected_epoch_delta_in_nanoseconds = 8 * 3600 * 1e9 + expected_energy = [ + 0.3749896542976582, + 0.4881381387720609, + 0.6354277772546533, + 0.8271602401781013, + 1.076745599456691, + 1.401640250140305, + 1.824567838312671, + 2.375108588863466, + 3.091768193234094, + 4.024670958420553, + 5.239065580337116, + 6.819888740878518, + 8.877705713882047, + 11.55644347537280, + 15.04345718406192, + 19.58263409767632, + ] - mock_calculate_energy_deltas.return_value = (sentinel.expected_energy_delta_plus, sentinel.expected_energy_delta_minus) + mock_calculate_energy_deltas.return_value = ( + sentinel.expected_energy_delta_plus, + sentinel.expected_energy_delta_minus, + ) expected_spin_angle = np.array( - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, - 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, - 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, - 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, - 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, - 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, - 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, - 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, - 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, - 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, - 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, - 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, - 254, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, - 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, - 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, - 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, - 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, - 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360], dtype=np.float64) + [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 359, + 360, + ], + dtype=np.float64, + ) expected_survival_probability_index_8 = [ - 0.8589063329371907, 0.8587603216362375, 0.8585838125331493, 0.8583938671767950, 0.8581975494622297, - 0.8579940886919561, 0.8577658336186133, 0.8575214109067385, 0.8572599641959735, 0.8569812266143973, - 0.8566932795409066, 0.8563886260499640, 0.8560556586637402, 0.8557098877131397, 0.8553438616744717, - 0.8549578606822365, 0.8545510333975665, 0.8541212480707067, 0.8536681161158036, 0.8531976189241352, - 0.8526922669152210, 0.8521695594671225, 0.8516090453648694, 0.8510264168109199, 0.8504115338966615, - 0.8497787446956853, 0.8491284768090441, 0.8484307529868772, 0.8477197767662112, 0.8469592190599952, - 0.8461609726558839, 0.8453310589628533, 0.8444869719908985, 0.8436110574283868, 0.8426901005437170, - 0.8417444019589297, 0.8407741696303948, 0.8397702695719742, 0.8387397217249644, 0.8376368056061494, - 0.8365059181016049, 0.8353506712749569, 0.8341197401096629, 0.8328807386203723, 0.8316151236695234, - 0.8302970047784992, 0.8289519503384518, 0.8275561150432003, 0.8261200965047648, 0.8246070354411250, - 0.8230668102146183, 0.8214856128640838, 0.8198277050221140, 0.8181845942358368, 0.8164978928432362, - 0.8148309289431280, 0.8131205906925642, 0.8113809258453120, 0.8096509045157287, 0.8079093484999044, - 0.8061612273184158, 0.8044016098272827, 0.8026764914872416, 0.8009798800464115, 0.7992804912990893, - 0.7975942906910145, 0.7959642742484756, 0.7942911491765003, 0.7926979440533350, 0.7911359974877107, - 0.7895681692802368, 0.7880552693505788, 0.7865820633788240, 0.7851433182389695, 0.7838179439870212, - 0.7825923824434343, 0.7815656413769618, 0.7808147672716648, 0.7804437518138578, 0.7803130526774916, - 0.7805042213127741, 0.7809074064230767, 0.7813841009090794, 0.7819760778000449, 0.7827500216229394, - 0.7838103093482217, 0.7852133370825548, 0.7868617457922777, 0.7887631522950826, 0.7908012747902750, - 0.7928705592892216, 0.7949576084461400, 0.7970472829205981, 0.7991794478975991, 0.8013141332087443, - 0.8034770881143130, 0.8056414449314634, 0.8078210292202376, 0.8099514597952638, 0.8120972717505870, - 0.8141922936155517, 0.8162799495859382, 0.8183661997086748, 0.8204529303103028, 0.8225442185915449, - 0.8246255008053819, 0.8266809204723378, 0.8286882267316884, 0.8305683903957821, 0.8324033176411660, - 0.8341171708051789, 0.8358134217032417, 0.8374520711185758, 0.8390655387793522, 0.8406882235809617, - 0.8422533842982793, 0.8437133249342935, 0.8451236245850865, 0.8464408939197524, 0.8476841636780257, - 0.8488764128329585, 0.8500031695041402, 0.8511294318653039, 0.8522065671879705, 0.8532635205533413, - 0.8542850871415247, 0.8552602056521883, 0.8562005567721789, 0.8570951569076538, 0.8579399483931489, - 0.8587430520841162, 0.8595486843278016, 0.8603074584511685, 0.8610408869701144, 0.8617638553578701, - 0.8624663050361265, 0.8631193864034373, 0.8637317777523140, 0.8643221475784872, 0.8648832636500384, - 0.8654137915804599, 0.8659313206452271, 0.8664381867080074, 0.8669254461854356, 0.8673860980091474, - 0.8678333557993616, 0.8682417934320824, 0.8686624857971838, 0.8690300296220324, 0.8693965768813886, - 0.8697486392127265, 0.8700837888261288, 0.8704044462925420, 0.8707110890085822, 0.8710108193918249, - 0.8712891065199926, 0.8715586093104385, 0.8718059308003806, 0.8720371638380639, 0.8722689042395361, - 0.8724713677077062, 0.8726762347681811, 0.8728679808089833, 0.8730318743361172, 0.8731905734470720, - 0.8733435122176896, 0.8734819340852058, 0.8735987818461556, 0.8737025069718742, 0.8738009111413396, - 0.8738940497203151, 0.8739674128793775, 0.8740283659839079, 0.8740767944974310, 0.8741198280035212, - 0.8741419468495819, 0.8741503018130508, 0.8741524268548572, 0.8741406243503601, 0.8741196786892612, - 0.8740871319460586, 0.8740299021694693, 0.8739595970117827, 0.8738911754598945, 0.8738085706855024, - 0.8737121351399008, 0.8735851165165418, 0.8734681027336583, 0.8733205433096817, 0.8731680229742780, - 0.8730005251022243, 0.8728193547081913, 0.8726222591204129, 0.8724091878053493, 0.8721881990724981, - 0.8719404959958943, 0.8716726564365842, 0.8713956164937880, 0.8710970955594298, 0.8707809341848330, - 0.8704483245506168, 0.8700869953326894, 0.8697240184707551, 0.8693228177240478, 0.8689058188918802, - 0.8684725481212315, 0.8679864074740811, 0.8674846479170634, 0.8669565386658245, 0.8663987931741293, - 0.8658292976477142, 0.8652400888199558, 0.8646159344170304, 0.8639645675796087, 0.8632894095541491, - 0.8625775450692091, 0.8618508940390988, 0.8610662587207843, 0.8602420947979763, 0.8593876584994568, - 0.8585135191949332, 0.8576052938566121, 0.8566647318199154, 0.8557040506174665, 0.8546950723577439, - 0.8536288094574640, 0.8525140899403636, 0.8513132917711719, 0.8500547007246933, 0.8487031312551817, - 0.8473557396340265, 0.8459512894425343, 0.8445566831823717, 0.8431045795291137, 0.8416159404817778, - 0.8400524806016000, 0.8384689895980637, 0.8367684445784317, 0.8350331558210311, 0.8332845367962914, - 0.8315062693986432, 0.8297374174385194, 0.8279583012690772, 0.8261865084237222, 0.8243942252989166, - 0.8225734619575098, 0.8207549333753910, 0.8189216746886322, 0.8170647071344279, 0.8152032410605699, - 0.8133635963914283, 0.8115323792261186, 0.8097108939706253, 0.8079113822965517, 0.8061213846923494, - 0.8043408988724510, 0.8025426088683201, 0.8007814521258226, 0.7990987049136656, 0.7976307399916195, - 0.7964515215938459, 0.7955921535640210, 0.7949442384447885, 0.7944722659231427, 0.7940835322381860, - 0.7937574117930607, 0.7935319301733967, 0.7935410410847828, 0.7938430647730597, 0.7944549685472722, - 0.7953201271372370, 0.7963353399742404, 0.7974666717653967, 0.7986284092599631, 0.7998758861147037, - 0.8011160184065348, 0.8023966457837027, 0.8037516415367600, 0.8050684075928743, 0.8064302119915984, - 0.8078016416648872, 0.8091911716432389, 0.8105695706967822, 0.8120156466367995, 0.8134179619097438, - 0.8148718795368090, 0.8163047327580164, 0.8177597708329662, 0.8192293148111823, 0.8207074925156855, - 0.8222059167731761, 0.8236269008644003, 0.8250637062532796, 0.8264471756096386, 0.8278214795110156, - 0.8291718933569523, 0.8305216689363951, 0.8318736481229983, 0.8331525651945888, 0.8344452634309963, - 0.8356818249509860, 0.8368599136784342, 0.8379673523433857, 0.8390344990656603, 0.8400886238827425, - 0.8411180665764300, 0.8421142442355391, 0.8431022785867203, 0.8440249657050137, 0.8449432095397274, - 0.8458194063508257, 0.8466739192575265, 0.8474814859179764, 0.8482437338193161, 0.8489939765878994, - 0.8497227079261302, 0.8503990404728226, 0.8510604321333799, 0.8516904214587106, 0.8522976961031688, - 0.8528845012461620, 0.8534308899160006, 0.8539713965861790, 0.8544590509167357, 0.8549124471041260, - 0.8553675524645131, 0.8557728466303356, 0.8561569562069312, 0.8565541426261458, 0.8568977169319288, - 0.8572347416962208, 0.8575440542473425, 0.8578267049055189, 0.8580766194405365, 0.8583205882562980, - 0.8585270608521915, 0.8587229325896764, 0.8589085011920993, 0.8590676557001768, 0.8592175194277411, - 0.8593425048092776, 0.8594494587601790, 0.8595467856580804, 0.8596372067196015, 0.8596791152243492, - 0.8597382969327882, 0.8597823055545957, 0.8597949778619707, 0.8598094662503897, 0.8598097898645332, - 0.8597880908859125, 0.8597611602425049, 0.8597126790735709, 0.8596589797562888, 0.8595919584381491, - 0.8595034812547143, 0.8594095601838275, 0.8593104306912213, 0.8591896577901775, 0.8590583978048111 + 0.8589063329371907, + 0.8587603216362375, + 0.8585838125331493, + 0.8583938671767950, + 0.8581975494622297, + 0.8579940886919561, + 0.8577658336186133, + 0.8575214109067385, + 0.8572599641959735, + 0.8569812266143973, + 0.8566932795409066, + 0.8563886260499640, + 0.8560556586637402, + 0.8557098877131397, + 0.8553438616744717, + 0.8549578606822365, + 0.8545510333975665, + 0.8541212480707067, + 0.8536681161158036, + 0.8531976189241352, + 0.8526922669152210, + 0.8521695594671225, + 0.8516090453648694, + 0.8510264168109199, + 0.8504115338966615, + 0.8497787446956853, + 0.8491284768090441, + 0.8484307529868772, + 0.8477197767662112, + 0.8469592190599952, + 0.8461609726558839, + 0.8453310589628533, + 0.8444869719908985, + 0.8436110574283868, + 0.8426901005437170, + 0.8417444019589297, + 0.8407741696303948, + 0.8397702695719742, + 0.8387397217249644, + 0.8376368056061494, + 0.8365059181016049, + 0.8353506712749569, + 0.8341197401096629, + 0.8328807386203723, + 0.8316151236695234, + 0.8302970047784992, + 0.8289519503384518, + 0.8275561150432003, + 0.8261200965047648, + 0.8246070354411250, + 0.8230668102146183, + 0.8214856128640838, + 0.8198277050221140, + 0.8181845942358368, + 0.8164978928432362, + 0.8148309289431280, + 0.8131205906925642, + 0.8113809258453120, + 0.8096509045157287, + 0.8079093484999044, + 0.8061612273184158, + 0.8044016098272827, + 0.8026764914872416, + 0.8009798800464115, + 0.7992804912990893, + 0.7975942906910145, + 0.7959642742484756, + 0.7942911491765003, + 0.7926979440533350, + 0.7911359974877107, + 0.7895681692802368, + 0.7880552693505788, + 0.7865820633788240, + 0.7851433182389695, + 0.7838179439870212, + 0.7825923824434343, + 0.7815656413769618, + 0.7808147672716648, + 0.7804437518138578, + 0.7803130526774916, + 0.7805042213127741, + 0.7809074064230767, + 0.7813841009090794, + 0.7819760778000449, + 0.7827500216229394, + 0.7838103093482217, + 0.7852133370825548, + 0.7868617457922777, + 0.7887631522950826, + 0.7908012747902750, + 0.7928705592892216, + 0.7949576084461400, + 0.7970472829205981, + 0.7991794478975991, + 0.8013141332087443, + 0.8034770881143130, + 0.8056414449314634, + 0.8078210292202376, + 0.8099514597952638, + 0.8120972717505870, + 0.8141922936155517, + 0.8162799495859382, + 0.8183661997086748, + 0.8204529303103028, + 0.8225442185915449, + 0.8246255008053819, + 0.8266809204723378, + 0.8286882267316884, + 0.8305683903957821, + 0.8324033176411660, + 0.8341171708051789, + 0.8358134217032417, + 0.8374520711185758, + 0.8390655387793522, + 0.8406882235809617, + 0.8422533842982793, + 0.8437133249342935, + 0.8451236245850865, + 0.8464408939197524, + 0.8476841636780257, + 0.8488764128329585, + 0.8500031695041402, + 0.8511294318653039, + 0.8522065671879705, + 0.8532635205533413, + 0.8542850871415247, + 0.8552602056521883, + 0.8562005567721789, + 0.8570951569076538, + 0.8579399483931489, + 0.8587430520841162, + 0.8595486843278016, + 0.8603074584511685, + 0.8610408869701144, + 0.8617638553578701, + 0.8624663050361265, + 0.8631193864034373, + 0.8637317777523140, + 0.8643221475784872, + 0.8648832636500384, + 0.8654137915804599, + 0.8659313206452271, + 0.8664381867080074, + 0.8669254461854356, + 0.8673860980091474, + 0.8678333557993616, + 0.8682417934320824, + 0.8686624857971838, + 0.8690300296220324, + 0.8693965768813886, + 0.8697486392127265, + 0.8700837888261288, + 0.8704044462925420, + 0.8707110890085822, + 0.8710108193918249, + 0.8712891065199926, + 0.8715586093104385, + 0.8718059308003806, + 0.8720371638380639, + 0.8722689042395361, + 0.8724713677077062, + 0.8726762347681811, + 0.8728679808089833, + 0.8730318743361172, + 0.8731905734470720, + 0.8733435122176896, + 0.8734819340852058, + 0.8735987818461556, + 0.8737025069718742, + 0.8738009111413396, + 0.8738940497203151, + 0.8739674128793775, + 0.8740283659839079, + 0.8740767944974310, + 0.8741198280035212, + 0.8741419468495819, + 0.8741503018130508, + 0.8741524268548572, + 0.8741406243503601, + 0.8741196786892612, + 0.8740871319460586, + 0.8740299021694693, + 0.8739595970117827, + 0.8738911754598945, + 0.8738085706855024, + 0.8737121351399008, + 0.8735851165165418, + 0.8734681027336583, + 0.8733205433096817, + 0.8731680229742780, + 0.8730005251022243, + 0.8728193547081913, + 0.8726222591204129, + 0.8724091878053493, + 0.8721881990724981, + 0.8719404959958943, + 0.8716726564365842, + 0.8713956164937880, + 0.8710970955594298, + 0.8707809341848330, + 0.8704483245506168, + 0.8700869953326894, + 0.8697240184707551, + 0.8693228177240478, + 0.8689058188918802, + 0.8684725481212315, + 0.8679864074740811, + 0.8674846479170634, + 0.8669565386658245, + 0.8663987931741293, + 0.8658292976477142, + 0.8652400888199558, + 0.8646159344170304, + 0.8639645675796087, + 0.8632894095541491, + 0.8625775450692091, + 0.8618508940390988, + 0.8610662587207843, + 0.8602420947979763, + 0.8593876584994568, + 0.8585135191949332, + 0.8576052938566121, + 0.8566647318199154, + 0.8557040506174665, + 0.8546950723577439, + 0.8536288094574640, + 0.8525140899403636, + 0.8513132917711719, + 0.8500547007246933, + 0.8487031312551817, + 0.8473557396340265, + 0.8459512894425343, + 0.8445566831823717, + 0.8431045795291137, + 0.8416159404817778, + 0.8400524806016000, + 0.8384689895980637, + 0.8367684445784317, + 0.8350331558210311, + 0.8332845367962914, + 0.8315062693986432, + 0.8297374174385194, + 0.8279583012690772, + 0.8261865084237222, + 0.8243942252989166, + 0.8225734619575098, + 0.8207549333753910, + 0.8189216746886322, + 0.8170647071344279, + 0.8152032410605699, + 0.8133635963914283, + 0.8115323792261186, + 0.8097108939706253, + 0.8079113822965517, + 0.8061213846923494, + 0.8043408988724510, + 0.8025426088683201, + 0.8007814521258226, + 0.7990987049136656, + 0.7976307399916195, + 0.7964515215938459, + 0.7955921535640210, + 0.7949442384447885, + 0.7944722659231427, + 0.7940835322381860, + 0.7937574117930607, + 0.7935319301733967, + 0.7935410410847828, + 0.7938430647730597, + 0.7944549685472722, + 0.7953201271372370, + 0.7963353399742404, + 0.7974666717653967, + 0.7986284092599631, + 0.7998758861147037, + 0.8011160184065348, + 0.8023966457837027, + 0.8037516415367600, + 0.8050684075928743, + 0.8064302119915984, + 0.8078016416648872, + 0.8091911716432389, + 0.8105695706967822, + 0.8120156466367995, + 0.8134179619097438, + 0.8148718795368090, + 0.8163047327580164, + 0.8177597708329662, + 0.8192293148111823, + 0.8207074925156855, + 0.8222059167731761, + 0.8236269008644003, + 0.8250637062532796, + 0.8264471756096386, + 0.8278214795110156, + 0.8291718933569523, + 0.8305216689363951, + 0.8318736481229983, + 0.8331525651945888, + 0.8344452634309963, + 0.8356818249509860, + 0.8368599136784342, + 0.8379673523433857, + 0.8390344990656603, + 0.8400886238827425, + 0.8411180665764300, + 0.8421142442355391, + 0.8431022785867203, + 0.8440249657050137, + 0.8449432095397274, + 0.8458194063508257, + 0.8466739192575265, + 0.8474814859179764, + 0.8482437338193161, + 0.8489939765878994, + 0.8497227079261302, + 0.8503990404728226, + 0.8510604321333799, + 0.8516904214587106, + 0.8522976961031688, + 0.8528845012461620, + 0.8534308899160006, + 0.8539713965861790, + 0.8544590509167357, + 0.8549124471041260, + 0.8553675524645131, + 0.8557728466303356, + 0.8561569562069312, + 0.8565541426261458, + 0.8568977169319288, + 0.8572347416962208, + 0.8575440542473425, + 0.8578267049055189, + 0.8580766194405365, + 0.8583205882562980, + 0.8585270608521915, + 0.8587229325896764, + 0.8589085011920993, + 0.8590676557001768, + 0.8592175194277411, + 0.8593425048092776, + 0.8594494587601790, + 0.8595467856580804, + 0.8596372067196015, + 0.8596791152243492, + 0.8597382969327882, + 0.8597823055545957, + 0.8597949778619707, + 0.8598094662503897, + 0.8598097898645332, + 0.8597880908859125, + 0.8597611602425049, + 0.8597126790735709, + 0.8596589797562888, + 0.8595919584381491, + 0.8595034812547143, + 0.8594095601838275, + 0.8593104306912213, + 0.8591896577901775, + 0.8590583978048111, ] expected_survival_probability_shape = (1, 16, 360) @@ -199,7 +879,7 @@ def test_convert_dat_to_glows_l3e_hi_product(self, mock_calculate_energy_deltas) spacecraft_info = MagicMock(spec=GlowsL3eSpacecraftInfo) spacecraft_info.spin_axis_latitude = spin_axis_lat spacecraft_info.spin_axis_longitude = spin_axis_lon - spacecraft_info.spacecraft_radius = .5 + spacecraft_info.spacecraft_radius = 0.5 spacecraft_info.spacecraft_longitude = 85.4 spacecraft_info.spacecraft_latitude = 45.1 @@ -211,36 +891,75 @@ def test_convert_dat_to_glows_l3e_hi_product(self, mock_calculate_energy_deltas) args.elongation = 135.0 - expected_program_version = 'Hi.v00.01' + expected_program_version = "Hi.v00.01" - l3e_hi_product: GlowsL3EHiData = GlowsL3EHiData.convert_dat_to_glows_l3e_hi_product(mock_metadata, hi_file_path, - expected_epoch, - epoch_delta, - args) + l3e_hi_product: GlowsL3EHiData = ( + GlowsL3EHiData.convert_dat_to_glows_l3e_hi_product( + mock_metadata, hi_file_path, expected_epoch, epoch_delta, args + ) + ) - np.testing.assert_equal(np.array([expected_epoch]), l3e_hi_product.epoch, strict=True) - np.testing.assert_equal(np.array([expected_epoch_delta_in_nanoseconds]), l3e_hi_product.epoch_delta, strict=True) + np.testing.assert_equal( + np.array([expected_epoch]), l3e_hi_product.epoch, strict=True + ) + np.testing.assert_equal( + np.array([expected_epoch_delta_in_nanoseconds]), + l3e_hi_product.epoch_delta, + strict=True, + ) np.testing.assert_equal(l3e_hi_product.energy, expected_energy, strict=True) - np.testing.assert_equal(l3e_hi_product.spin_angle, expected_spin_angle, strict=True) - np.testing.assert_equal(l3e_hi_product.probability_of_survival.shape, expected_survival_probability_shape, - strict=True) - np.testing.assert_equal(l3e_hi_product.probability_of_survival[0][7], - expected_survival_probability_index_8, strict=True) + np.testing.assert_equal( + l3e_hi_product.spin_angle, expected_spin_angle, strict=True + ) + np.testing.assert_equal( + l3e_hi_product.probability_of_survival.shape, + expected_survival_probability_shape, + strict=True, + ) + np.testing.assert_equal( + l3e_hi_product.probability_of_survival[0][7], + expected_survival_probability_index_8, + strict=True, + ) - np.testing.assert_equal(np.array([spin_axis_lat]), l3e_hi_product.spin_axis_lat, strict=True) - np.testing.assert_equal(np.array([spin_axis_lon]), l3e_hi_product.spin_axis_lon, strict=True) - np.testing.assert_equal(np.array([expected_program_version]), l3e_hi_product.program_version, strict=True) + np.testing.assert_equal( + np.array([spin_axis_lat]), l3e_hi_product.spin_axis_lat, strict=True + ) + np.testing.assert_equal( + np.array([spin_axis_lon]), l3e_hi_product.spin_axis_lon, strict=True + ) + np.testing.assert_equal( + np.array([expected_program_version]), + l3e_hi_product.program_version, + strict=True, + ) - np.testing.assert_equal(np.array([.5]), l3e_hi_product.spacecraft_radius, strict=True) - np.testing.assert_equal(np.array([85.4]), l3e_hi_product.spacecraft_longitude, strict=True) - np.testing.assert_equal(np.array([45.1]), l3e_hi_product.spacecraft_latitude, strict=True) + np.testing.assert_equal( + np.array([0.5]), l3e_hi_product.spacecraft_radius, strict=True + ) + np.testing.assert_equal( + np.array([85.4]), l3e_hi_product.spacecraft_longitude, strict=True + ) + np.testing.assert_equal( + np.array([45.1]), l3e_hi_product.spacecraft_latitude, strict=True + ) - np.testing.assert_equal(np.array([2.1]), l3e_hi_product.spacecraft_velocity_x, strict=True) - np.testing.assert_equal(np.array([2.2]), l3e_hi_product.spacecraft_velocity_y, strict=True) - np.testing.assert_equal(np.array([2.3]), l3e_hi_product.spacecraft_velocity_z, strict=True) + np.testing.assert_equal( + np.array([2.1]), l3e_hi_product.spacecraft_velocity_x, strict=True + ) + np.testing.assert_equal( + np.array([2.2]), l3e_hi_product.spacecraft_velocity_y, strict=True + ) + np.testing.assert_equal( + np.array([2.3]), l3e_hi_product.spacecraft_velocity_z, strict=True + ) np.testing.assert_equal([135.0], l3e_hi_product.elongation) - np.testing.assert_equal(sentinel.expected_energy_delta_plus, l3e_hi_product.energy_delta_plus) - np.testing.assert_equal(sentinel.expected_energy_delta_minus, l3e_hi_product.energy_delta_minus) + np.testing.assert_equal( + sentinel.expected_energy_delta_plus, l3e_hi_product.energy_delta_plus + ) + np.testing.assert_equal( + sentinel.expected_energy_delta_minus, l3e_hi_product.energy_delta_minus + ) mock_calculate_energy_deltas.assert_called_once_with(l3e_hi_product.energy) diff --git a/tests/glows/l3e/test_glows_l3e_initializer.py b/tests/glows/l3e/test_glows_l3e_initializer.py index b6cd0aac4..4b2e190e9 100644 --- a/tests/glows/l3e/test_glows_l3e_initializer.py +++ b/tests/glows/l3e/test_glows_l3e_initializer.py @@ -7,62 +7,101 @@ from imap_data_access import RepointInput from imap_data_access.file_validation import Version -from imap_l3_processing.glows.descriptors import GLOWS_L3BCDE_DESCRIPTORS, GLOWS_L3E_HI_45_DESCRIPTOR, \ - GLOWS_L3E_HI_90_DESCRIPTOR, GLOWS_L3E_LO_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, GLOWS_L3E_ULTRA_HF_DESCRIPTOR, \ - GLOWS_L3E_DESCRIPTORS +from imap_l3_processing.glows.descriptors import ( + GLOWS_L3BCDE_DESCRIPTORS, + GLOWS_L3E_HI_45_DESCRIPTOR, + GLOWS_L3E_HI_90_DESCRIPTOR, + GLOWS_L3E_LO_DESCRIPTOR, + GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + GLOWS_L3E_ULTRA_HF_DESCRIPTOR, + GLOWS_L3E_DESCRIPTORS, +) from imap_l3_processing.glows.l3d.models import GlowsL3DProcessorOutput -from imap_l3_processing.glows.l3e.glows_l3e_initializer import GlowsL3EInitializer, GlowsL3EInitializerOutput, \ - find_first_updated_cr, identify_versions_for_l3e_output_files +from imap_l3_processing.glows.l3e.glows_l3e_initializer import ( + GlowsL3EInitializer, + GlowsL3EInitializerOutput, + find_first_updated_cr, + identify_versions_for_l3e_output_files, +) from imap_l3_processing.glows.l3e.glows_l3e_utils import GlowsL3eVersionsForRepointings from imap_l3_processing.glows.l3e.reprocess_info import ReprocessInfo from imap_l3_processing.models import VersionMap from tests.test_helpers import create_mock_query_results, get_test_data_path -MODULE = 'imap_l3_processing.glows.l3e.glows_l3e_initializer' +MODULE = "imap_l3_processing.glows.l3e.glows_l3e_initializer" + class TestGlowsL3EInitializer(unittest.TestCase): - @patch(f'{MODULE}.get_pointing_date_range') - @patch(f'{MODULE}.GlowsL3EDependencies.fetch_dependencies') - @patch(f'{MODULE}.identify_versions_for_l3e_output_files') - @patch(f'{MODULE}.find_first_updated_cr') - @patch(f'{MODULE}.get_most_recently_uploaded_ancillary') - @patch(f'{MODULE}.imap_data_access.query') - @patch(f'{MODULE}.GlowsL3EDependencies.collect_spice_dependencies') - def test_get_repointings_to_process(self, mock_collect_spice_dependencies, mock_query, mock_get_most_recently_uploaded_ancillary, - mock_find_first_updated_cr, mock_identify_versions_for_l3e_output_files, - mock_fetch_dependencies, mock_get_pointing_date_range): - mock_query.side_effect = create_mock_query_results([ - 'imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf', - 'imap_glows_energy-grid-lo_20200101_v000.cdf', - 'imap_glows_tess-xyz-8_20200101_v000.cdf', - 'imap_glows_energy-grid-hi_20200101_v000.cdf', - 'imap_glows_energy-grid-ultra_20200101_v000.cdf', - 'imap_glows_tess-ang-16_20200101_v000.cdf', - ]) + @patch(f"{MODULE}.get_pointing_date_range") + @patch(f"{MODULE}.GlowsL3EDependencies.fetch_dependencies") + @patch(f"{MODULE}.identify_versions_for_l3e_output_files") + @patch(f"{MODULE}.find_first_updated_cr") + @patch(f"{MODULE}.get_most_recently_uploaded_ancillary") + @patch(f"{MODULE}.imap_data_access.query") + @patch(f"{MODULE}.GlowsL3EDependencies.collect_spice_dependencies") + def test_get_repointings_to_process( + self, + mock_collect_spice_dependencies, + mock_query, + mock_get_most_recently_uploaded_ancillary, + mock_find_first_updated_cr, + mock_identify_versions_for_l3e_output_files, + mock_fetch_dependencies, + mock_get_pointing_date_range, + ): + mock_query.side_effect = create_mock_query_results( + [ + "imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf", + "imap_glows_energy-grid-lo_20200101_v000.cdf", + "imap_glows_tess-xyz-8_20200101_v000.cdf", + "imap_glows_energy-grid-hi_20200101_v000.cdf", + "imap_glows_energy-grid-ultra_20200101_v000.cdf", + "imap_glows_tess-ang-16_20200101_v000.cdf", + ] + ) mock_get_most_recently_uploaded_ancillary.side_effect = [ - create_mock_query_results(['imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_energy-grid-lo_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_tess-xyz-8_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_energy-grid-hi_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_energy-grid-ultra_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_tess-ang-16_20200101_v000.cdf'])[0], + create_mock_query_results( + ["imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf"] + )[0], + create_mock_query_results(["imap_glows_energy-grid-lo_20200101_v000.cdf"])[ + 0 + ], + create_mock_query_results(["imap_glows_tess-xyz-8_20200101_v000.cdf"])[0], + create_mock_query_results(["imap_glows_energy-grid-hi_20200101_v000.cdf"])[ + 0 + ], + create_mock_query_results( + ["imap_glows_energy-grid-ultra_20200101_v000.cdf"] + )[0], + create_mock_query_results(["imap_glows_tess-ang-16_20200101_v000.cdf"])[0], ] mock_spice_with_predict, mock_spice_without_predict = Mock(), Mock() - mock_collect_spice_dependencies.return_value = (mock_spice_with_predict, mock_spice_without_predict) + mock_collect_spice_dependencies.return_value = ( + mock_spice_with_predict, + mock_spice_without_predict, + ) - updated_l3d = Path('path/to/imap_glows_l3d_solar-hist_19470303-cr02091_v000.cdf') - updated_l3d_text_file_path = Path("imap_glows_e-dens_19470303_20100101_v000.dat") - glows_l3d_processor_output = GlowsL3DProcessorOutput(updated_l3d, [updated_l3d_text_file_path], sentinel.last_processed_cr) - previous_l3d = 'imap_glows_l3d_solar-hist_19470303-cr02090_v000.cdf' + updated_l3d = Path( + "path/to/imap_glows_l3d_solar-hist_19470303-cr02091_v000.cdf" + ) + updated_l3d_text_file_path = Path( + "imap_glows_e-dens_19470303_20100101_v000.dat" + ) + glows_l3d_processor_output = GlowsL3DProcessorOutput( + updated_l3d, [updated_l3d_text_file_path], sentinel.last_processed_cr + ) + previous_l3d = "imap_glows_l3d_solar-hist_19470303-cr02090_v000.cdf" mock_find_first_updated_cr.return_value = 2091 mock_l3e_dependencies = mock_fetch_dependencies.return_value - mock_l3e_dependencies.pipeline_settings = {"start_cr": sentinel.start_of_mission_cr} - mock_l3e_dependencies.repointing_file = Path('path/to/repointing_file') + mock_l3e_dependencies.pipeline_settings = { + "start_cr": sentinel.start_of_mission_cr + } + mock_l3e_dependencies.repointing_file = Path("path/to/repointing_file") expected_hi_45 = {1234: Version(None, 1), 2468: Version(None, 1)} expected_hi_90 = {1234: Version(None, 2), 2468: Version(None, 2)} @@ -108,17 +147,37 @@ def test_get_repointings_to_process(self, mock_collect_spice_dependencies, mock_ mock_find_first_updated_cr.assert_called_once_with(updated_l3d, previous_l3d) - mock_identify_versions_for_l3e_output_files.assert_called_once_with(sentinel.start_of_mission_cr, sentinel.last_processed_cr, 2090, repointing_file_path, - input_major_version, sentinel.reprocess_info) + mock_identify_versions_for_l3e_output_files.assert_called_once_with( + sentinel.start_of_mission_cr, + sentinel.last_processed_cr, + 2090, + repointing_file_path, + input_major_version, + sentinel.reprocess_info, + ) - mock_query.assert_has_calls([ - call(table="ancillary", instrument='glows', descriptor='pipeline-settings-l3bcde'), - call(table="ancillary", instrument='glows', descriptor='energy-grid-lo'), - call(table="ancillary", instrument='glows', descriptor='tess-xyz-8'), - call(table="ancillary", instrument='glows', descriptor='energy-grid-hi'), - call(table="ancillary", instrument='glows', descriptor='energy-grid-ultra'), - call(table="ancillary", instrument='glows', descriptor='tess-ang-16'), - ]) + mock_query.assert_has_calls( + [ + call( + table="ancillary", + instrument="glows", + descriptor="pipeline-settings-l3bcde", + ), + call( + table="ancillary", instrument="glows", descriptor="energy-grid-lo" + ), + call(table="ancillary", instrument="glows", descriptor="tess-xyz-8"), + call( + table="ancillary", instrument="glows", descriptor="energy-grid-hi" + ), + call( + table="ancillary", + instrument="glows", + descriptor="energy-grid-ultra", + ), + call(table="ancillary", instrument="glows", descriptor="tess-ang-16"), + ] + ) mock_l3e_dependencies.copy_dependencies.assert_called_once() @@ -129,37 +188,39 @@ def test_get_repointings_to_process(self, mock_collect_spice_dependencies, mock_ pipeline_l3e_input_paths = actual_l3e_inputs.get_file_paths(source="glows") pipeline_l3e_input_filenames = [p.name for p in pipeline_l3e_input_paths] - self.assertEqual([ - updated_l3d.name, - "imap_glows_e-dens_19470303_20100101_v000.dat", - 'imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf', - 'imap_glows_energy-grid-lo_20200101_v000.cdf', - 'imap_glows_tess-xyz-8_20200101_v000.cdf', - 'imap_glows_energy-grid-hi_20200101_v000.cdf', - 'imap_glows_energy-grid-ultra_20200101_v000.cdf', - 'imap_glows_tess-ang-16_20200101_v000.cdf', - ], pipeline_l3e_input_filenames) - - [repoint_input] = actual_l3e_inputs.get_file_paths(data_type=RepointInput.data_type) + self.assertEqual( + [ + updated_l3d.name, + "imap_glows_e-dens_19470303_20100101_v000.dat", + "imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf", + "imap_glows_energy-grid-lo_20200101_v000.cdf", + "imap_glows_tess-xyz-8_20200101_v000.cdf", + "imap_glows_energy-grid-hi_20200101_v000.cdf", + "imap_glows_energy-grid-ultra_20200101_v000.cdf", + "imap_glows_tess-ang-16_20200101_v000.cdf", + ], + pipeline_l3e_input_filenames, + ) + + [repoint_input] = actual_l3e_inputs.get_file_paths( + data_type=RepointInput.data_type + ) self.assertEqual("imap_2026_105_01.repoint.csv", repoint_input.name) self.assertEqual(actual_initializer_output, expected_initializer_data) - mock_get_pointing_date_range.assert_has_calls([ - call(1234), - call(2468) - ]) + mock_get_pointing_date_range.assert_has_calls([call(1234), call(2468)]) mock_collect_spice_dependencies.assert_called_once_with( start_date=datetime(2010, 1, 1), end_date=datetime(2011, 2, 2), ) - @patch(f'{MODULE}.get_most_recently_uploaded_ancillary') - @patch(f'{MODULE}.imap_data_access') - @patch(f'{MODULE}.GlowsL3EDependencies.fetch_dependencies') - @patch(f'{MODULE}.identify_versions_for_l3e_output_files') - @patch(f'{MODULE}.find_first_updated_cr') + @patch(f"{MODULE}.get_most_recently_uploaded_ancillary") + @patch(f"{MODULE}.imap_data_access") + @patch(f"{MODULE}.GlowsL3EDependencies.fetch_dependencies") + @patch(f"{MODULE}.identify_versions_for_l3e_output_files") + @patch(f"{MODULE}.find_first_updated_cr") def test_get_repointings_to_process_identical_l3d_files( self, mock_find_first_updated_cr, @@ -185,21 +246,31 @@ def test_get_repointings_to_process_identical_l3d_files( "start_cr": sentinel.start_of_mission_cr } mock_find_first_updated_cr.return_value = None - mock_identify_versions_for_l3e_output_files.return_value = GlowsL3eVersionsForRepointings( - repointing_numbers=[], - hi_90_repointings={}, - hi_45_repointings={}, - lo_repointings={}, - ultra_sf_repointings={}, - ultra_hf_repointings={}, + mock_identify_versions_for_l3e_output_files.return_value = ( + GlowsL3eVersionsForRepointings( + repointing_numbers=[], + hi_90_repointings={}, + hi_45_repointings={}, + lo_repointings={}, + ultra_sf_repointings={}, + ultra_hf_repointings={}, + ) ) mock_get_most_recently_uploaded_ancillary.side_effect = [ - create_mock_query_results(['imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_energy-grid-lo_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_tess-xyz-8_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_energy-grid-hi_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_energy-grid-ultra_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_tess-ang-16_20200101_v000.cdf'])[0], + create_mock_query_results( + ["imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf"] + )[0], + create_mock_query_results(["imap_glows_energy-grid-lo_20200101_v000.cdf"])[ + 0 + ], + create_mock_query_results(["imap_glows_tess-xyz-8_20200101_v000.cdf"])[0], + create_mock_query_results(["imap_glows_energy-grid-hi_20200101_v000.cdf"])[ + 0 + ], + create_mock_query_results( + ["imap_glows_energy-grid-ultra_20200101_v000.cdf"] + )[0], + create_mock_query_results(["imap_glows_tess-ang-16_20200101_v000.cdf"])[0], ] repointing_file_path = Path("imap_2026_105_01.repoint.csv") @@ -210,7 +281,9 @@ def test_get_repointings_to_process_identical_l3d_files( sentinel.version_map, sentinel.reprocess_info, ) - mock_find_first_updated_cr.assert_called_once_with(glows_l3d_processor_output.l3d_cdf_file_path, previous_l3d) + mock_find_first_updated_cr.assert_called_once_with( + glows_l3d_processor_output.l3d_cdf_file_path, previous_l3d + ) mock_identify_versions_for_l3e_output_files.assert_called_once_with( sentinel.start_of_mission_cr, expected_last_cr, @@ -221,43 +294,64 @@ def test_get_repointings_to_process_identical_l3d_files( ) self.assertIsNone(actual_initializer_output) - @patch(f'{MODULE}.imap_data_access.query') - @patch(f'{MODULE}.get_most_recently_uploaded_ancillary') - @patch(f'{MODULE}.GlowsL3EDependencies.fetch_dependencies') - @patch(f'{MODULE}.identify_versions_for_l3e_output_files') - @patch(f'{MODULE}.find_first_updated_cr') - def test_get_repointings_to_process_uses_mission_start_cr_when_no_previous_l3d(self, mock_find_first_updated_cr, - mock_identify_versions_for_l3e_output_files, - mock_fetch_dependencies, - mock_get_most_recently_uploaded_ancillary, - _, ): + @patch(f"{MODULE}.imap_data_access.query") + @patch(f"{MODULE}.get_most_recently_uploaded_ancillary") + @patch(f"{MODULE}.GlowsL3EDependencies.fetch_dependencies") + @patch(f"{MODULE}.identify_versions_for_l3e_output_files") + @patch(f"{MODULE}.find_first_updated_cr") + def test_get_repointings_to_process_uses_mission_start_cr_when_no_previous_l3d( + self, + mock_find_first_updated_cr, + mock_identify_versions_for_l3e_output_files, + mock_fetch_dependencies, + mock_get_most_recently_uploaded_ancillary, + _, + ): mock_get_most_recently_uploaded_ancillary.side_effect = [ - create_mock_query_results(['imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_energy-grid-lo_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_tess-xyz-8_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_energy-grid-hi_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_energy-grid-ultra_20200101_v000.cdf'])[0], - create_mock_query_results(['imap_glows_tess-ang-16_20200101_v000.cdf'])[0], + create_mock_query_results( + ["imap_glows_pipeline-settings-l3bcde_20200101_v000.cdf"] + )[0], + create_mock_query_results(["imap_glows_energy-grid-lo_20200101_v000.cdf"])[ + 0 + ], + create_mock_query_results(["imap_glows_tess-xyz-8_20200101_v000.cdf"])[0], + create_mock_query_results(["imap_glows_energy-grid-hi_20200101_v000.cdf"])[ + 0 + ], + create_mock_query_results( + ["imap_glows_energy-grid-ultra_20200101_v000.cdf"] + )[0], + create_mock_query_results(["imap_glows_tess-ang-16_20200101_v000.cdf"])[0], ] - mock_identify_versions_for_l3e_output_files.return_value = GlowsL3eVersionsForRepointings( - repointing_numbers=[], - ultra_sf_repointings={}, - ultra_hf_repointings={}, - lo_repointings={}, - hi_45_repointings={}, - hi_90_repointings={} + mock_identify_versions_for_l3e_output_files.return_value = ( + GlowsL3eVersionsForRepointings( + repointing_numbers=[], + ultra_sf_repointings={}, + ultra_hf_repointings={}, + lo_repointings={}, + hi_45_repointings={}, + hi_90_repointings={}, + ) ) - updated_l3d = Path('path/to/imap_glows_l3d_solar-hist_19470303-cr02091_v012.0001.cdf') - glows_l3d_processor_output = GlowsL3DProcessorOutput(updated_l3d, [], sentinel.last_processed_cr) + updated_l3d = Path( + "path/to/imap_glows_l3d_solar-hist_19470303-cr02091_v012.0001.cdf" + ) + glows_l3d_processor_output = GlowsL3DProcessorOutput( + updated_l3d, [], sentinel.last_processed_cr + ) previous_l3d = None - mock_fetch_dependencies.return_value.pipeline_settings = {"start_cr": sentinel.start_of_mission_cr} + mock_fetch_dependencies.return_value.pipeline_settings = { + "start_cr": sentinel.start_of_mission_cr + } repointing_file_path = Path("imap_2026_105_01.repoint.csv") _ = GlowsL3EInitializer.get_repointings_to_process( - glows_l3d_processor_output, previous_l3d, - repointing_file_path, sentinel.version_map, + glows_l3d_processor_output, + previous_l3d, + repointing_file_path, + sentinel.version_map, sentinel.reprocess_info, ) @@ -271,23 +365,25 @@ def test_get_repointings_to_process_uses_mission_start_cr_when_no_previous_l3d(s sentinel.reprocess_info, ) - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access.download') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.CDF') + @patch( + "imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access.download" + ) + @patch("imap_l3_processing.glows.l3e.glows_l3e_initializer.CDF") def test_find_first_updated_cr(self, mock_CDF, mock_download): num_crs = 10 mock_download.return_value = sentinel.download_old_l3d old_l3d = { - 'cr_grid': np.arange(num_crs) + 0.5, - 'lyman_alpha': np.arange(num_crs), - 'phion': np.arange(num_crs), - 'plasma_speed': np.arange(0, 20).reshape((num_crs, 2)), - 'plasma_speed_flag': np.arange(num_crs), - 'proton_density': np.arange(0, 20).reshape((num_crs, 2)), - 'proton_density_flag': np.arange(num_crs), - 'uv_anisotropy': np.arange(0, 20).reshape((num_crs, 2)), - 'uv_anisotropy_flag': np.arange(num_crs), - 'glows_flags': np.arange(num_crs), + "cr_grid": np.arange(num_crs) + 0.5, + "lyman_alpha": np.arange(num_crs), + "phion": np.arange(num_crs), + "plasma_speed": np.arange(0, 20).reshape((num_crs, 2)), + "plasma_speed_flag": np.arange(num_crs), + "proton_density": np.arange(0, 20).reshape((num_crs, 2)), + "proton_density_flag": np.arange(num_crs), + "uv_anisotropy": np.arange(0, 20).reshape((num_crs, 2)), + "uv_anisotropy_flag": np.arange(num_crs), + "glows_flags": np.arange(num_crs), } new_lyman_alpha = np.arange(num_crs) @@ -318,18 +414,17 @@ def test_find_first_updated_cr(self, mock_CDF, mock_download): new_glows_flags[9] = 10 cases = [ - ('cr_grid', np.append(old_l3d['cr_grid'], 10.5), 10), - ('lyman_alpha', new_lyman_alpha, 1), - ('phion', new_phion, 2), - ('plasma_speed_flag', new_plasma_speed_flag, 3), - ('proton_density_flag', new_proton_density_flag, 4), - ('uv_anisotropy_flag', new_uv_anisotropy_flag, 5), - - ('plasma_speed', new_plasma_speed, 6), - ('proton_density', new_proton_density, 7), - ('uv_anisotropy', new_uv_anisotropy, 8), - ('glows_flags', new_glows_flags, 9), - ('no_change', None, None) + ("cr_grid", np.append(old_l3d["cr_grid"], 10.5), 10), + ("lyman_alpha", new_lyman_alpha, 1), + ("phion", new_phion, 2), + ("plasma_speed_flag", new_plasma_speed_flag, 3), + ("proton_density_flag", new_proton_density_flag, 4), + ("uv_anisotropy_flag", new_uv_anisotropy_flag, 5), + ("plasma_speed", new_plasma_speed, 6), + ("proton_density", new_proton_density, 7), + ("uv_anisotropy", new_uv_anisotropy, 8), + ("glows_flags", new_glows_flags, 9), + ("no_change", None, None), ] for case, change, expected in cases: @@ -357,16 +452,20 @@ def test_find_first_updated_cr(self, mock_CDF, mock_download): self.assertEqual(actual_cr, expected) - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_numbers_within_cr_window') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access') - def test_identify_versions_for_l3e_output_files_gives_minor_version_1_for_non_existing_l3e(self, - mock_imap_data_access, - mock_get_repoint_numbers_within_cr_window): + @patch( + "imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_numbers_within_cr_window" + ) + @patch("imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access") + def test_identify_versions_for_l3e_output_files_gives_minor_version_1_for_non_existing_l3e( + self, mock_imap_data_access, mock_get_repoint_numbers_within_cr_window + ): start_cr_of_mission = 2093 end_cr_of_mission = 2094 first_cr_updated_in_l3d = None repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") - version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) + version_map = VersionMap( + {desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)} + ) mock_imap_data_access.query.side_effect = [ create_mock_query_results([]), @@ -380,51 +479,99 @@ def test_identify_versions_for_l3e_output_files_gives_minor_version_1_for_non_ex updated_repointing_numbers = set() mock_get_repoint_numbers_within_cr_window.side_effect = [ all_repointing_numbers, - updated_repointing_numbers + updated_repointing_numbers, ] reprocess_info = ReprocessInfo({}) - result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, first_cr_updated_in_l3d, - repointing_path, version_map, reprocess_info) - - mock_imap_data_access.query.assert_has_calls([ - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) - ]) - - expected_versions_for_hi45_repoint_number = {repoint_number: Version(3, 1) for repoint_number in - all_repointing_numbers} - expected_versions_for_hi90_repoint_number = {repoint_number: Version(4, 1) for repoint_number in - all_repointing_numbers} - expected_versions_for_lo_repoint_number = {repoint_number: Version(5, 1) for repoint_number in - all_repointing_numbers} - expected_versions_for_ultra_sf_repoint_number = {repoint_number: Version(6, 1) for repoint_number in - all_repointing_numbers} - expected_versions_for_ultra_hf_repoint_number = {repoint_number: Version(7, 1) for repoint_number in - all_repointing_numbers} + result = identify_versions_for_l3e_output_files( + start_cr_of_mission, + end_cr_of_mission, + first_cr_updated_in_l3d, + repointing_path, + version_map, + reprocess_info, + ) + + mock_imap_data_access.query.assert_has_calls( + [ + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_HI_45_DESCRIPTOR, + ), + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_HI_90_DESCRIPTOR, + ), + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_LO_DESCRIPTOR, + ), + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + ), + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, + ), + ] + ) + + expected_versions_for_hi45_repoint_number = { + repoint_number: Version(3, 1) for repoint_number in all_repointing_numbers + } + expected_versions_for_hi90_repoint_number = { + repoint_number: Version(4, 1) for repoint_number in all_repointing_numbers + } + expected_versions_for_lo_repoint_number = { + repoint_number: Version(5, 1) for repoint_number in all_repointing_numbers + } + expected_versions_for_ultra_sf_repoint_number = { + repoint_number: Version(6, 1) for repoint_number in all_repointing_numbers + } + expected_versions_for_ultra_hf_repoint_number = { + repoint_number: Version(7, 1) for repoint_number in all_repointing_numbers + } self.assertCountEqual(all_repointing_numbers, result.repointing_numbers) - self.assertEqual(expected_versions_for_hi90_repoint_number, result.hi_90_repointings) - self.assertEqual(expected_versions_for_hi45_repoint_number, result.hi_45_repointings) + self.assertEqual( + expected_versions_for_hi90_repoint_number, result.hi_90_repointings + ) + self.assertEqual( + expected_versions_for_hi45_repoint_number, result.hi_45_repointings + ) self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) self.assertEqual( expected_versions_for_ultra_sf_repoint_number, result.ultra_sf_repointings ) - self.assertEqual(expected_versions_for_ultra_hf_repoint_number, result.ultra_hf_repointings) + self.assertEqual( + expected_versions_for_ultra_hf_repoint_number, result.ultra_hf_repointings + ) - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_numbers_within_cr_window') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access') - def test_identify_versions_for_l3e_output_files_increments_major_and_minor_when_given_higher_major_version(self, - mock_imap_data_access, - mock_get_repoint_numbers_within_cr_window): + @patch( + "imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_numbers_within_cr_window" + ) + @patch("imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access") + def test_identify_versions_for_l3e_output_files_increments_major_and_minor_when_given_higher_major_version( + self, mock_imap_data_access, mock_get_repoint_numbers_within_cr_window + ): start_cr_of_mission = 2093 end_cr_of_mission = 2094 first_cr_updated_in_l3d = None repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") - version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) + version_map = VersionMap( + {desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)} + ) all_repointing_numbers = set(range(3682, 3736)) updated_repointing_numbers = set() @@ -434,48 +581,90 @@ def test_identify_versions_for_l3e_output_files_increments_major_and_minor_when_ mock_get_repoint_numbers_within_cr_window.side_effect = [ all_repointing_numbers, - updated_repointing_numbers + updated_repointing_numbers, ] mock_imap_data_access.query.side_effect = [ - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03682_{Version(old_major_version, 1)}.cdf', - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03683_{Version(3, 1)}.cdf', - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint03735_{Version(3, 1)}.cdf' - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03683_{Version(old_major_version, 2)}.cdf', - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03684_{Version(4, 2)}.cdf', - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint03735_{Version(4, 2)}.cdf' - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-lo_20250101-repoint03684_{Version(old_major_version, 3)}.cdf', - f'imap_glows_l3e_survival-probability-lo_20250101-repoint03685_{Version(5, 3)}.cdf', - f'imap_glows_l3e_survival-probability-lo_20250101-repoint03735_{Version(5, 3)}.cdf' - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03685_{Version(old_major_version, 4)}.cdf', - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03686_{Version(6, 4)}.cdf', - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03735_{Version(6, 4)}.cdf' - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03686_{Version(old_major_version, 5)}.cdf', - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03687_{Version(7, 5)}.cdf', - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03735_{Version(7, 5)}.cdf' - ]) + create_mock_query_results( + [ + f"imap_glows_l3e_survival-probability-hi-90_20250101-repoint03682_{Version(old_major_version, 1)}.cdf", + f"imap_glows_l3e_survival-probability-hi-90_20250101-repoint03683_{Version(3, 1)}.cdf", + f"imap_glows_l3e_survival-probability-hi-90_20250101-repoint03735_{Version(3, 1)}.cdf", + ] + ), + create_mock_query_results( + [ + f"imap_glows_l3e_survival-probability-hi-45_20250101-repoint03683_{Version(old_major_version, 2)}.cdf", + f"imap_glows_l3e_survival-probability-hi-45_20250101-repoint03684_{Version(4, 2)}.cdf", + f"imap_glows_l3e_survival-probability-hi-45_20250101-repoint03735_{Version(4, 2)}.cdf", + ] + ), + create_mock_query_results( + [ + f"imap_glows_l3e_survival-probability-lo_20250101-repoint03684_{Version(old_major_version, 3)}.cdf", + f"imap_glows_l3e_survival-probability-lo_20250101-repoint03685_{Version(5, 3)}.cdf", + f"imap_glows_l3e_survival-probability-lo_20250101-repoint03735_{Version(5, 3)}.cdf", + ] + ), + create_mock_query_results( + [ + f"imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03685_{Version(old_major_version, 4)}.cdf", + f"imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03686_{Version(6, 4)}.cdf", + f"imap_glows_l3e_survival-probability-ul-sf_20250101-repoint03735_{Version(6, 4)}.cdf", + ] + ), + create_mock_query_results( + [ + f"imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03686_{Version(old_major_version, 5)}.cdf", + f"imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03687_{Version(7, 5)}.cdf", + f"imap_glows_l3e_survival-probability-ul-hf_20250101-repoint03735_{Version(7, 5)}.cdf", + ] + ), ] - result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, - first_cr_updated_in_l3d, repointing_path, - version_map, ReprocessInfo({})) + result = identify_versions_for_l3e_output_files( + start_cr_of_mission, + end_cr_of_mission, + first_cr_updated_in_l3d, + repointing_path, + version_map, + ReprocessInfo({}), + ) - mock_imap_data_access.query.assert_has_calls([ - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) - ]) + mock_imap_data_access.query.assert_has_calls( + [ + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_HI_45_DESCRIPTOR, + ), + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_HI_90_DESCRIPTOR, + ), + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_LO_DESCRIPTOR, + ), + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + ), + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, + ), + ] + ) self.assertCountEqual(list(range(3682, 3735)), result.repointing_numbers) @@ -491,93 +680,168 @@ def test_identify_versions_for_l3e_output_files_increments_major_and_minor_when_ self.assertEqual(Version(6, 5), result.ultra_sf_repointings[3685]) self.assertEqual(Version(7, 6), result.ultra_hf_repointings[3686]) - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_numbers_within_cr_window') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access') + @patch( + "imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_numbers_within_cr_window" + ) + @patch("imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access") def test_identify_versions_for_l3e_output_files_increments_minor_when_same_major_and_updated_l3d_covers_pointing( - self, mock_imap_data_access, mock_get_repoint_numbers_within_cr_window + self, mock_imap_data_access, mock_get_repoint_numbers_within_cr_window ): start_cr_of_mission = 2093 end_cr_of_mission = 2095 first_cr_updated_in_l3d = None repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") - version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) + version_map = VersionMap( + {desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)} + ) all_repointing_numbers = set(range(3682, 3763)) updated_repointing_numbers = set(range(3709, 3763)) mock_get_repoint_numbers_within_cr_window.side_effect = [ all_repointing_numbers, - updated_repointing_numbers + updated_repointing_numbers, ] mock_imap_data_access.query.side_effect = [ - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint{repoint:05d}_{Version(3, 1)}.cdf' for - repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint{repoint:05d}_{Version(4, 2)}.cdf' for - repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-lo_20250101-repoint{repoint:05d}_{Version(5, 3)}.cdf' for repoint - in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint{repoint:05d}_{Version(6, 4)}.cdf' for - repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint{repoint:05d}_{Version(7, 5)}.cdf' for - repoint in all_repointing_numbers - ]) + create_mock_query_results( + [ + f"imap_glows_l3e_survival-probability-hi-90_20250101-repoint{repoint:05d}_{Version(3, 1)}.cdf" + for repoint in all_repointing_numbers + ] + ), + create_mock_query_results( + [ + f"imap_glows_l3e_survival-probability-hi-45_20250101-repoint{repoint:05d}_{Version(4, 2)}.cdf" + for repoint in all_repointing_numbers + ] + ), + create_mock_query_results( + [ + f"imap_glows_l3e_survival-probability-lo_20250101-repoint{repoint:05d}_{Version(5, 3)}.cdf" + for repoint in all_repointing_numbers + ] + ), + create_mock_query_results( + [ + f"imap_glows_l3e_survival-probability-ul-sf_20250101-repoint{repoint:05d}_{Version(6, 4)}.cdf" + for repoint in all_repointing_numbers + ] + ), + create_mock_query_results( + [ + f"imap_glows_l3e_survival-probability-ul-hf_20250101-repoint{repoint:05d}_{Version(7, 5)}.cdf" + for repoint in all_repointing_numbers + ] + ), ] - result = identify_versions_for_l3e_output_files(start_cr_of_mission, end_cr_of_mission, - first_cr_updated_in_l3d, repointing_path, - version_map, ReprocessInfo({})) - - mock_imap_data_access.query.assert_has_calls([ - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) - ]) - - expected_versions_for_hi45_repoint_number = {repoint_number: Version(3, 2) for repoint_number in - updated_repointing_numbers} - expected_versions_for_hi90_repoint_number = {repoint_number: Version(4, 3) for repoint_number in - updated_repointing_numbers} - expected_versions_for_lo_repoint_number = {repoint_number: Version(5, 4) for repoint_number in - updated_repointing_numbers} - expected_versions_for_ultra_sf_repoint_number = {repoint_number: Version(6, 5) for repoint_number in - updated_repointing_numbers} - expected_versions_for_ultra_hf_repoint_number = {repoint_number: Version(7, 6) for repoint_number in - updated_repointing_numbers} + result = identify_versions_for_l3e_output_files( + start_cr_of_mission, + end_cr_of_mission, + first_cr_updated_in_l3d, + repointing_path, + version_map, + ReprocessInfo({}), + ) + + mock_imap_data_access.query.assert_has_calls( + [ + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_HI_45_DESCRIPTOR, + ), + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_HI_90_DESCRIPTOR, + ), + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_LO_DESCRIPTOR, + ), + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + ), + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, + ), + ] + ) + + expected_versions_for_hi45_repoint_number = { + repoint_number: Version(3, 2) + for repoint_number in updated_repointing_numbers + } + expected_versions_for_hi90_repoint_number = { + repoint_number: Version(4, 3) + for repoint_number in updated_repointing_numbers + } + expected_versions_for_lo_repoint_number = { + repoint_number: Version(5, 4) + for repoint_number in updated_repointing_numbers + } + expected_versions_for_ultra_sf_repoint_number = { + repoint_number: Version(6, 5) + for repoint_number in updated_repointing_numbers + } + expected_versions_for_ultra_hf_repoint_number = { + repoint_number: Version(7, 6) + for repoint_number in updated_repointing_numbers + } self.assertCountEqual(updated_repointing_numbers, result.repointing_numbers) - self.assertEqual(expected_versions_for_hi90_repoint_number, result.hi_90_repointings) - self.assertEqual(expected_versions_for_hi45_repoint_number, result.hi_45_repointings) + self.assertEqual( + expected_versions_for_hi90_repoint_number, result.hi_90_repointings + ) + self.assertEqual( + expected_versions_for_hi45_repoint_number, result.hi_45_repointings + ) self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) - self.assertEqual(expected_versions_for_ultra_sf_repoint_number, result.ultra_sf_repointings) - self.assertEqual(expected_versions_for_ultra_hf_repoint_number, result.ultra_hf_repointings) + self.assertEqual( + expected_versions_for_ultra_sf_repoint_number, result.ultra_sf_repointings + ) + self.assertEqual( + expected_versions_for_ultra_hf_repoint_number, result.ultra_hf_repointings + ) - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_data') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_numbers_within_cr_window') - @patch('imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access') + @patch("imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_data") + @patch( + "imap_l3_processing.glows.l3e.glows_l3e_initializer.get_repoint_numbers_within_cr_window" + ) + @patch("imap_l3_processing.glows.l3e.glows_l3e_initializer.imap_data_access") def test_identify_versions_for_l3e_output_files_uses_reprocess_info( - self, mock_imap_data_access, mock_get_repoint_numbers_within_cr_window, mock_get_repoint_data + self, + mock_imap_data_access, + mock_get_repoint_numbers_within_cr_window, + mock_get_repoint_data, ): start_cr_of_mission = 2093 end_cr_of_mission = 2095 first_cr_updated_in_l3d = None repointing_path = get_test_data_path("fake_1_day_repointing_file.csv") - version_map = VersionMap({desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)}) + version_map = VersionMap( + {desc: Version(3 + i, 5) for i, desc in enumerate(GLOWS_L3E_DESCRIPTORS)} + ) repoint_number = 3700 mock_reprocess_info = create_autospec(ReprocessInfo, instance=True) mock_reprocess_info.get_repoints_for_descriptor.side_effect = [ - set(), set(), {repoint_number}, set(), set() + set(), + set(), + {repoint_number}, + set(), + set(), ] all_repointing_numbers = set(range(3682, 3763)) @@ -585,30 +849,40 @@ def test_identify_versions_for_l3e_output_files_uses_reprocess_info( mock_get_repoint_numbers_within_cr_window.side_effect = [ all_repointing_numbers, - updated_repointing_numbers + updated_repointing_numbers, ] mock_imap_data_access.query.side_effect = [ - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-90_20250101-repoint{repoint:05d}_{Version(3, 1)}.cdf' for - repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-hi-45_20250101-repoint{repoint:05d}_{Version(4, 2)}.cdf' for - repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-lo_20250101-repoint{repoint:05d}_{Version(5, 3)}.cdf' for repoint - in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-sf_20250101-repoint{repoint:05d}_{Version(6, 4)}.cdf' for - repoint in all_repointing_numbers - ]), - create_mock_query_results([ - f'imap_glows_l3e_survival-probability-ul-hf_20250101-repoint{repoint:05d}_{Version(7, 5)}.cdf' for - repoint in all_repointing_numbers - ]) + create_mock_query_results( + [ + f"imap_glows_l3e_survival-probability-hi-90_20250101-repoint{repoint:05d}_{Version(3, 1)}.cdf" + for repoint in all_repointing_numbers + ] + ), + create_mock_query_results( + [ + f"imap_glows_l3e_survival-probability-hi-45_20250101-repoint{repoint:05d}_{Version(4, 2)}.cdf" + for repoint in all_repointing_numbers + ] + ), + create_mock_query_results( + [ + f"imap_glows_l3e_survival-probability-lo_20250101-repoint{repoint:05d}_{Version(5, 3)}.cdf" + for repoint in all_repointing_numbers + ] + ), + create_mock_query_results( + [ + f"imap_glows_l3e_survival-probability-ul-sf_20250101-repoint{repoint:05d}_{Version(6, 4)}.cdf" + for repoint in all_repointing_numbers + ] + ), + create_mock_query_results( + [ + f"imap_glows_l3e_survival-probability-ul-hf_20250101-repoint{repoint:05d}_{Version(7, 5)}.cdf" + for repoint in all_repointing_numbers + ] + ), ] result = identify_versions_for_l3e_output_files( @@ -620,22 +894,51 @@ def test_identify_versions_for_l3e_output_files_uses_reprocess_info( mock_reprocess_info, ) - mock_imap_data_access.query.assert_has_calls([ - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_45_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_HI_90_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_LO_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR), - call(instrument='glows', data_level='l3e', version="latest", descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR) - ]) - mock_reprocess_info.get_repoints_for_descriptor.assert_has_calls([ - call(GLOWS_L3E_HI_45_DESCRIPTOR, mock_get_repoint_data.return_value), - call(GLOWS_L3E_HI_90_DESCRIPTOR, mock_get_repoint_data.return_value), - call(GLOWS_L3E_LO_DESCRIPTOR, mock_get_repoint_data.return_value), - call(GLOWS_L3E_ULTRA_SF_DESCRIPTOR, mock_get_repoint_data.return_value), - call(GLOWS_L3E_ULTRA_HF_DESCRIPTOR, mock_get_repoint_data.return_value), - ]) + mock_imap_data_access.query.assert_has_calls( + [ + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_HI_45_DESCRIPTOR, + ), + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_HI_90_DESCRIPTOR, + ), + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_LO_DESCRIPTOR, + ), + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + ), + call( + instrument="glows", + data_level="l3e", + version="latest", + descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, + ), + ] + ) + mock_reprocess_info.get_repoints_for_descriptor.assert_has_calls( + [ + call(GLOWS_L3E_HI_45_DESCRIPTOR, mock_get_repoint_data.return_value), + call(GLOWS_L3E_HI_90_DESCRIPTOR, mock_get_repoint_data.return_value), + call(GLOWS_L3E_LO_DESCRIPTOR, mock_get_repoint_data.return_value), + call(GLOWS_L3E_ULTRA_SF_DESCRIPTOR, mock_get_repoint_data.return_value), + call(GLOWS_L3E_ULTRA_HF_DESCRIPTOR, mock_get_repoint_data.return_value), + ] + ) expected_versions_for_lo_repoint_number = {repoint_number: Version(5, 4)} self.assertEqual([repoint_number], result.repointing_numbers) - self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) \ No newline at end of file + self.assertEqual(expected_versions_for_lo_repoint_number, result.lo_repointings) diff --git a/tests/glows/l3e/test_glows_l3e_lo_model.py b/tests/glows/l3e/test_glows_l3e_lo_model.py index e73c086fa..5c55f8b4a 100644 --- a/tests/glows/l3e/test_glows_l3e_lo_model.py +++ b/tests/glows/l3e/test_glows_l3e_lo_model.py @@ -4,13 +4,33 @@ import numpy as np -from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import GlowsL3eCallArguments, GlowsL3eSpacecraftInfo -from imap_l3_processing.glows.l3e.glows_l3e_lo_model import GlowsL3ELoData, SPIN_AXIS_LONGITUDE_VAR_NAME, \ - SPIN_AXIS_LATITUDE_VAR_NAME, EPOCH_CDF_VAR_NAME, ENERGY_VAR_NAME, SPIN_ANGLE_VAR_NAME, \ - PROBABILITY_OF_SURVIVAL_VAR_NAME, ENERGY_LABEL_VAR_NAME, SPIN_ANGLE_LABEL_VAR_NAME, ELONGATION_VAR_NAME, \ - PROGRAM_VERSION_VAR_NAME, SPACECRAFT_RADIUS_VAR_NAME, SPACECRAFT_LONGITUDE_VAR_NAME, SPACECRAFT_LATITUDE_VAR_NAME, \ - SPACECRAFT_VELOCITY_X_VAR_NAME, SPACECRAFT_VELOCITY_Y_VAR_NAME, SPACECRAFT_VELOCITY_Z_VAR_NAME, \ - GLOWS_FLAGS_VAR_NAME, ENERGY_DELTA_PLUS_VAR_NAME, ENERGY_DELTA_MINUS_VAR_NAME, EPOCH_DELTA_CDF_VAR_NAME +from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import ( + GlowsL3eCallArguments, + GlowsL3eSpacecraftInfo, +) +from imap_l3_processing.glows.l3e.glows_l3e_lo_model import ( + GlowsL3ELoData, + SPIN_AXIS_LONGITUDE_VAR_NAME, + SPIN_AXIS_LATITUDE_VAR_NAME, + EPOCH_CDF_VAR_NAME, + ENERGY_VAR_NAME, + SPIN_ANGLE_VAR_NAME, + PROBABILITY_OF_SURVIVAL_VAR_NAME, + ENERGY_LABEL_VAR_NAME, + SPIN_ANGLE_LABEL_VAR_NAME, + ELONGATION_VAR_NAME, + PROGRAM_VERSION_VAR_NAME, + SPACECRAFT_RADIUS_VAR_NAME, + SPACECRAFT_LONGITUDE_VAR_NAME, + SPACECRAFT_LATITUDE_VAR_NAME, + SPACECRAFT_VELOCITY_X_VAR_NAME, + SPACECRAFT_VELOCITY_Y_VAR_NAME, + SPACECRAFT_VELOCITY_Z_VAR_NAME, + GLOWS_FLAGS_VAR_NAME, + ENERGY_DELTA_PLUS_VAR_NAME, + ENERGY_DELTA_MINUS_VAR_NAME, + EPOCH_DELTA_CDF_VAR_NAME, +) from imap_l3_processing.models import DataProductVariable from tests.test_helpers import get_test_instrument_team_data_path, NumpyArrayMatcher @@ -18,142 +38,499 @@ class TestL3eLoModel(unittest.TestCase): def test_l3e_lo_model_to_data_product_variables(self): test_cases = { - 'case 1':( - [1.2234,2,4.51,66.7666], - [1,41650233.0,4.22,9.5], - ['1.22','2.00','4.51','66.77'], - ['1','41650233','4','10'] + "case 1": ( + [1.2234, 2, 4.51, 66.7666], + [1, 41650233.0, 4.22, 9.5], + ["1.22", "2.00", "4.51", "66.77"], + ["1", "41650233", "4", "10"], + ), + "case 2": ( + [4.536, 48.193, 4253.1], + [13, 14.0, 34.2, 19.5], + ["4.54", "48.19", "4253.10"], + ["13", "14", "34", "20"], ), - 'case 2':( - [4.536,48.193,4253.1], - [13,14.0,34.2,19.5], - ['4.54','48.19','4253.10'], - ['13','14','34','20'] - ) } - for name, (energy_array, spin_angle_array, expected_energy_labels, expected_spin_angle_labels) in test_cases.items(): + for name, ( + energy_array, + spin_angle_array, + expected_energy_labels, + expected_spin_angle_labels, + ) in test_cases.items(): with self.subTest(name): l3e_lo: GlowsL3ELoData = GlowsL3ELoData( - Mock(), - sentinel.epoch, - sentinel.epoch_delta, - energy_array, - sentinel.energy_delta_plus, - sentinel.energy_delta_minus, - spin_angle_array, - sentinel.probability_of_survival, - sentinel.elongation, - sentinel.spin_axis_latitude, - sentinel.spin_axis_longitude, - sentinel.program_version, - sentinel.spacecraft_radius, - sentinel.spacecraft_latitude, - sentinel.spacecraft_longitude, - sentinel.spacecraft_velocity_x, - sentinel.spacecraft_velocity_y, - sentinel.spacecraft_velocity_z, - sentinel.glows_flags, - ) + Mock(), + sentinel.epoch, + sentinel.epoch_delta, + energy_array, + sentinel.energy_delta_plus, + sentinel.energy_delta_minus, + spin_angle_array, + sentinel.probability_of_survival, + sentinel.elongation, + sentinel.spin_axis_latitude, + sentinel.spin_axis_longitude, + sentinel.program_version, + sentinel.spacecraft_radius, + sentinel.spacecraft_latitude, + sentinel.spacecraft_longitude, + sentinel.spacecraft_velocity_x, + sentinel.spacecraft_velocity_y, + sentinel.spacecraft_velocity_z, + sentinel.glows_flags, + ) data_products = l3e_lo.to_data_product_variables() - expected_data_products = [ - DataProductVariable(EPOCH_CDF_VAR_NAME, sentinel.epoch), - DataProductVariable(EPOCH_DELTA_CDF_VAR_NAME, sentinel.epoch_delta), - DataProductVariable(ENERGY_VAR_NAME, energy_array), - DataProductVariable(ENERGY_DELTA_PLUS_VAR_NAME, sentinel.energy_delta_plus), - DataProductVariable(ENERGY_DELTA_MINUS_VAR_NAME, sentinel.energy_delta_minus), - DataProductVariable(SPIN_ANGLE_VAR_NAME, spin_angle_array), - DataProductVariable(PROBABILITY_OF_SURVIVAL_VAR_NAME, sentinel.probability_of_survival), - DataProductVariable(ENERGY_LABEL_VAR_NAME, expected_energy_labels), - DataProductVariable(SPIN_ANGLE_LABEL_VAR_NAME, expected_spin_angle_labels), - DataProductVariable(ELONGATION_VAR_NAME, sentinel.elongation), - DataProductVariable(SPIN_AXIS_LATITUDE_VAR_NAME, np.array([sentinel.spin_axis_latitude])), - DataProductVariable(SPIN_AXIS_LONGITUDE_VAR_NAME, np.array([sentinel.spin_axis_longitude])), - DataProductVariable(PROGRAM_VERSION_VAR_NAME, np.array([sentinel.program_version])), - DataProductVariable(SPACECRAFT_RADIUS_VAR_NAME, np.array([sentinel.spacecraft_radius])), - DataProductVariable(SPACECRAFT_LATITUDE_VAR_NAME, np.array([sentinel.spacecraft_latitude])), - DataProductVariable(SPACECRAFT_LONGITUDE_VAR_NAME, np.array([sentinel.spacecraft_longitude])), - DataProductVariable(SPACECRAFT_VELOCITY_X_VAR_NAME, np.array([sentinel.spacecraft_velocity_x])), - DataProductVariable(SPACECRAFT_VELOCITY_Y_VAR_NAME, np.array([sentinel.spacecraft_velocity_y])), - DataProductVariable(SPACECRAFT_VELOCITY_Z_VAR_NAME, np.array([sentinel.spacecraft_velocity_z])), - DataProductVariable(GLOWS_FLAGS_VAR_NAME, sentinel.glows_flags), + DataProductVariable(EPOCH_CDF_VAR_NAME, sentinel.epoch), + DataProductVariable(EPOCH_DELTA_CDF_VAR_NAME, sentinel.epoch_delta), + DataProductVariable(ENERGY_VAR_NAME, energy_array), + DataProductVariable( + ENERGY_DELTA_PLUS_VAR_NAME, sentinel.energy_delta_plus + ), + DataProductVariable( + ENERGY_DELTA_MINUS_VAR_NAME, sentinel.energy_delta_minus + ), + DataProductVariable(SPIN_ANGLE_VAR_NAME, spin_angle_array), + DataProductVariable( + PROBABILITY_OF_SURVIVAL_VAR_NAME, + sentinel.probability_of_survival, + ), + DataProductVariable(ENERGY_LABEL_VAR_NAME, expected_energy_labels), + DataProductVariable( + SPIN_ANGLE_LABEL_VAR_NAME, expected_spin_angle_labels + ), + DataProductVariable(ELONGATION_VAR_NAME, sentinel.elongation), + DataProductVariable( + SPIN_AXIS_LATITUDE_VAR_NAME, + np.array([sentinel.spin_axis_latitude]), + ), + DataProductVariable( + SPIN_AXIS_LONGITUDE_VAR_NAME, + np.array([sentinel.spin_axis_longitude]), + ), + DataProductVariable( + PROGRAM_VERSION_VAR_NAME, np.array([sentinel.program_version]) + ), + DataProductVariable( + SPACECRAFT_RADIUS_VAR_NAME, + np.array([sentinel.spacecraft_radius]), + ), + DataProductVariable( + SPACECRAFT_LATITUDE_VAR_NAME, + np.array([sentinel.spacecraft_latitude]), + ), + DataProductVariable( + SPACECRAFT_LONGITUDE_VAR_NAME, + np.array([sentinel.spacecraft_longitude]), + ), + DataProductVariable( + SPACECRAFT_VELOCITY_X_VAR_NAME, + np.array([sentinel.spacecraft_velocity_x]), + ), + DataProductVariable( + SPACECRAFT_VELOCITY_Y_VAR_NAME, + np.array([sentinel.spacecraft_velocity_y]), + ), + DataProductVariable( + SPACECRAFT_VELOCITY_Z_VAR_NAME, + np.array([sentinel.spacecraft_velocity_z]), + ), + DataProductVariable(GLOWS_FLAGS_VAR_NAME, sentinel.glows_flags), ] self.assertEqual(expected_data_products, data_products) @patch("imap_l3_processing.glows.l3e.glows_l3e_lo_model.calculate_energy_deltas") def test_convert_dat_to_glows_l3e_lo_product(self, mock_calculate_energy_deltas): - lo_file_path = get_test_instrument_team_data_path("glows/probSur.Imap.Lo_20090101_010101_2009.000_60.00.txt") + lo_file_path = get_test_instrument_team_data_path( + "glows/probSur.Imap.Lo_20090101_010101_2009.000_60.00.txt" + ) epoch = datetime(year=2009, month=1, day=1) epoch_delta = timedelta(hours=3) - expected_epoch_delta_in_nanoseconds = 3*3600*1e9 - expected_energy = [0.1700000, 0.2212954, 0.2880685, 0.3749897, 0.4881381, 0.6354278, 0.8271602, 1.0767456, - 1.4016403, 1.8245678, 2.3751086, 3.0917682, 4.0246710] + expected_epoch_delta_in_nanoseconds = 3 * 3600 * 1e9 + expected_energy = [ + 0.1700000, + 0.2212954, + 0.2880685, + 0.3749897, + 0.4881381, + 0.6354278, + 0.8271602, + 1.0767456, + 1.4016403, + 1.8245678, + 2.3751086, + 3.0917682, + 4.0246710, + ] - mock_calculate_energy_deltas.return_value = sentinel.energy_delta_plus, sentinel.energy_delta_minus + mock_calculate_energy_deltas.return_value = ( + sentinel.energy_delta_plus, + sentinel.energy_delta_minus, + ) expected_prob_of_survival_first_col_1 = [ - 0.48543928E+00, 0.48415770E+00, 0.48286930E+00, 0.48155365E+00, 0.48022886E+00, 0.47885535E+00, - 0.47745419E+00, 0.47602527E+00, 0.47456880E+00, 0.47308723E+00, 0.47161934E+00, 0.47007995E+00, - 0.46855039E+00, 0.46696817E+00, 0.46535270E+00, 0.46368022E+00, 0.46201160E+00, 0.46026144E+00, - 0.45852743E+00, 0.45670696E+00, 0.45486823E+00, 0.45294031E+00, 0.45103086E+00, 0.44905075E+00, - 0.44699322E+00, 0.44493658E+00, 0.44274056E+00, 0.44049151E+00, 0.43813752E+00, 0.43578356E+00, - 0.43334287E+00, 0.43079981E+00, 0.42816069E+00, 0.42550222E+00, 0.42269926E+00, 0.41984889E+00, - 0.41688223E+00, 0.41387416E+00, 0.41075224E+00, 0.40761023E+00, 0.40432881E+00, 0.40096472E+00, - 0.39747450E+00, 0.39382089E+00, 0.39011010E+00, 0.38626253E+00, 0.38238974E+00, 0.37847099E+00, - 0.37447377E+00, 0.37034872E+00, 0.36611061E+00, 0.36168805E+00, 0.35708535E+00, 0.35234758E+00, - 0.34755946E+00, 0.34276461E+00, 0.33796280E+00, 0.33305954E+00, 0.32807347E+00, 0.32294017E+00, - 0.31786557E+00, 0.31276043E+00, 0.30773144E+00, 0.30281685E+00, 0.29797800E+00, 0.29321404E+00, - 0.28846079E+00, 0.28371704E+00, 0.27912920E+00, 0.27468323E+00, 0.27045568E+00, 0.26630037E+00, - 0.26237361E+00, 0.25859967E+00, 0.25499865E+00, 0.25157148E+00, 0.24843419E+00, 0.24562329E+00, - 0.24332850E+00, 0.24163423E+00, 0.24043665E+00, 0.23972507E+00, 0.23958627E+00, 0.23984571E+00, - 0.24038628E+00, 0.24120584E+00, 0.24238245E+00, 0.24409800E+00, 0.24644643E+00, 0.24933851E+00, - 0.25273734E+00, 0.25673467E+00, 0.26113737E+00, 0.26578085E+00, 0.27063217E+00, 0.27563385E+00, - 0.28065683E+00, 0.28568105E+00, 0.29082592E+00, 0.29609185E+00, 0.30147878E+00, 0.30707066E+00, - 0.31284106E+00, 0.31876025E+00, 0.32458878E+00, 0.32999671E+00, 0.33517199E+00, 0.34027166E+00, - 0.34522320E+00, 0.35023574E+00, 0.35525591E+00, 0.36033213E+00, 0.36532967E+00, 0.37024320E+00, - 0.37475598E+00, 0.37896325E+00, 0.38299882E+00, 0.38699896E+00, 0.39085028E+00, 0.39468049E+00, - 0.39842597E+00, 0.40207629E+00, 0.40552207E+00, 0.40883026E+00, 0.41207903E+00, 0.41518970E+00, - 0.41827498E+00, 0.42130951E+00, 0.42430284E+00, 0.42722657E+00, 0.43005367E+00, 0.43275769E+00, - 0.43536314E+00, 0.43791211E+00, 0.44040868E+00, 0.44286096E+00, 0.44523738E+00, 0.44760685E+00, - 0.44986863E+00, 0.45215006E+00, 0.45433928E+00, 0.45643657E+00, 0.45854771E+00, 0.46059383E+00, - 0.46257694E+00, 0.46460808E+00, 0.46651408E+00, 0.46842937E+00, 0.47031930E+00, 0.47214870E+00, - 0.47394193E+00, 0.47575376E+00, 0.47747515E+00, 0.47919249E+00, 0.48087054E+00, 0.48248460E+00, - 0.48411980E+00, 0.48572180E+00, 0.48729084E+00, 0.48882613E+00, 0.49029764E+00, 0.49173477E+00, - 0.49316337E+00, 0.49455725E+00, 0.49591591E+00, 0.49723950E+00, 0.49855476E+00, 0.49983531E+00, - 0.50105632E+00, 0.50224193E+00, 0.50338975E+00, 0.50452839E+00, 0.50568552E+00, 0.50673081E+00, - 0.50774070E+00, 0.50873875E+00, 0.50974595E+00, 0.51063701E+00, 0.51151521E+00, 0.51238110E+00, - 0.51321059E+00, 0.51397275E+00, 0.51472495E+00, 0.51540664E+00, 0.51610368E+00, 0.51667264E+00, - 0.51730154E+00, 0.51776070E+00, 0.51827571E+00, 0.51868642E+00, 0.51905088E+00, 0.51938042E+00, - 0.51966850E+00, 0.51988677E+00, 0.52010423E+00, 0.52020368E+00, 0.52028969E+00, 0.52027278E+00, - 0.52016485E+00, 0.52001351E+00, 0.51981105E+00, 0.51959358E+00, 0.51926552E+00, 0.51884032E+00, - 0.51847414E+00, 0.51792884E+00, 0.51731622E+00, 0.51669638E+00, 0.51593604E+00, 0.51513940E+00, - 0.51427166E+00, 0.51335442E+00, 0.51238909E+00, 0.51129305E+00, 0.51014602E+00, 0.50890185E+00, - 0.50758409E+00, 0.50615044E+00, 0.50464683E+00, 0.50296036E+00, 0.50118289E+00, 0.49930168E+00, - 0.49730277E+00, 0.49530416E+00, 0.49314260E+00, 0.49083236E+00, 0.48844397E+00, 0.48583154E+00, - 0.48323435E+00, 0.48058649E+00, 0.47780153E+00, 0.47506716E+00, 0.47209482E+00, 0.46907585E+00, - 0.46597907E+00, 0.46291516E+00, 0.45981653E+00, 0.45665363E+00, 0.45351121E+00, 0.45044243E+00, - 0.44730392E+00, 0.44413627E+00, 0.44106236E+00, 0.43794686E+00, 0.43485646E+00, 0.43175941E+00, - 0.42864794E+00, 0.42564675E+00, 0.42266427E+00, 0.41976102E+00, 0.41687658E+00, 0.41406969E+00, - 0.41130938E+00, 0.40859441E+00, 0.40588626E+00, 0.40343902E+00, 0.40125003E+00, 0.39935862E+00, - 0.39784753E+00, 0.39670361E+00, 0.39585956E+00, 0.39523582E+00, 0.39476103E+00, 0.39439905E+00, - 0.39423968E+00, 0.39435100E+00, 0.39473203E+00, 0.39554310E+00, 0.39666286E+00, 0.39808096E+00, - 0.39972118E+00, 0.40153794E+00, 0.40352131E+00, 0.40548763E+00, 0.40751478E+00, 0.40963578E+00, - 0.41174409E+00, 0.41388333E+00, 0.41612846E+00, 0.41834220E+00, 0.42056149E+00, 0.42288866E+00, - 0.42519516E+00, 0.42763343E+00, 0.43005765E+00, 0.43251298E+00, 0.43502684E+00, 0.43754215E+00, - 0.44011421E+00, 0.44269314E+00, 0.44537654E+00, 0.44802922E+00, 0.45079502E+00, 0.45340240E+00, - 0.45589495E+00, 0.45836848E+00, 0.46074029E+00, 0.46320577E+00, 0.46557171E+00, 0.46804433E+00, - 0.47046801E+00, 0.47291016E+00, 0.47527772E+00, 0.47750468E+00, 0.47952019E+00, 0.48141970E+00, - 0.48317952E+00, 0.48487642E+00, 0.48650926E+00, 0.48807554E+00, 0.48963920E+00, 0.49111387E+00, - 0.49259059E+00, 0.49402593E+00, 0.49539224E+00, 0.49660220E+00, 0.49767488E+00, 0.49868284E+00, - 0.49962355E+00, 0.50044291E+00, 0.50121762E+00, 0.50196273E+00, 0.50261243E+00, 0.50326871E+00, - 0.50387423E+00, 0.50434609E+00, 0.50480509E+00, 0.50515270E+00, 0.50543081E+00, 0.50566424E+00, - 0.50580062E+00, 0.50590890E+00, 0.50589201E+00, 0.50588645E+00, 0.50578122E+00, 0.50561852E+00, - 0.50539304E+00, 0.50510737E+00, 0.50474719E+00, 0.50435977E+00, 0.50384726E+00, 0.50336489E+00, - 0.50279180E+00, 0.50216504E+00, 0.50148265E+00, 0.50078978E+00, 0.50000804E+00, 0.49918459E+00, - 0.49836367E+00, 0.49744147E+00, 0.49651714E+00, 0.49557195E+00, 0.49456744E+00, 0.49353034E+00, - 0.49245761E+00, 0.49135232E+00, 0.49023631E+00, 0.48909069E+00, 0.48791635E+00, 0.48669295E+00 + 0.48543928e00, + 0.48415770e00, + 0.48286930e00, + 0.48155365e00, + 0.48022886e00, + 0.47885535e00, + 0.47745419e00, + 0.47602527e00, + 0.47456880e00, + 0.47308723e00, + 0.47161934e00, + 0.47007995e00, + 0.46855039e00, + 0.46696817e00, + 0.46535270e00, + 0.46368022e00, + 0.46201160e00, + 0.46026144e00, + 0.45852743e00, + 0.45670696e00, + 0.45486823e00, + 0.45294031e00, + 0.45103086e00, + 0.44905075e00, + 0.44699322e00, + 0.44493658e00, + 0.44274056e00, + 0.44049151e00, + 0.43813752e00, + 0.43578356e00, + 0.43334287e00, + 0.43079981e00, + 0.42816069e00, + 0.42550222e00, + 0.42269926e00, + 0.41984889e00, + 0.41688223e00, + 0.41387416e00, + 0.41075224e00, + 0.40761023e00, + 0.40432881e00, + 0.40096472e00, + 0.39747450e00, + 0.39382089e00, + 0.39011010e00, + 0.38626253e00, + 0.38238974e00, + 0.37847099e00, + 0.37447377e00, + 0.37034872e00, + 0.36611061e00, + 0.36168805e00, + 0.35708535e00, + 0.35234758e00, + 0.34755946e00, + 0.34276461e00, + 0.33796280e00, + 0.33305954e00, + 0.32807347e00, + 0.32294017e00, + 0.31786557e00, + 0.31276043e00, + 0.30773144e00, + 0.30281685e00, + 0.29797800e00, + 0.29321404e00, + 0.28846079e00, + 0.28371704e00, + 0.27912920e00, + 0.27468323e00, + 0.27045568e00, + 0.26630037e00, + 0.26237361e00, + 0.25859967e00, + 0.25499865e00, + 0.25157148e00, + 0.24843419e00, + 0.24562329e00, + 0.24332850e00, + 0.24163423e00, + 0.24043665e00, + 0.23972507e00, + 0.23958627e00, + 0.23984571e00, + 0.24038628e00, + 0.24120584e00, + 0.24238245e00, + 0.24409800e00, + 0.24644643e00, + 0.24933851e00, + 0.25273734e00, + 0.25673467e00, + 0.26113737e00, + 0.26578085e00, + 0.27063217e00, + 0.27563385e00, + 0.28065683e00, + 0.28568105e00, + 0.29082592e00, + 0.29609185e00, + 0.30147878e00, + 0.30707066e00, + 0.31284106e00, + 0.31876025e00, + 0.32458878e00, + 0.32999671e00, + 0.33517199e00, + 0.34027166e00, + 0.34522320e00, + 0.35023574e00, + 0.35525591e00, + 0.36033213e00, + 0.36532967e00, + 0.37024320e00, + 0.37475598e00, + 0.37896325e00, + 0.38299882e00, + 0.38699896e00, + 0.39085028e00, + 0.39468049e00, + 0.39842597e00, + 0.40207629e00, + 0.40552207e00, + 0.40883026e00, + 0.41207903e00, + 0.41518970e00, + 0.41827498e00, + 0.42130951e00, + 0.42430284e00, + 0.42722657e00, + 0.43005367e00, + 0.43275769e00, + 0.43536314e00, + 0.43791211e00, + 0.44040868e00, + 0.44286096e00, + 0.44523738e00, + 0.44760685e00, + 0.44986863e00, + 0.45215006e00, + 0.45433928e00, + 0.45643657e00, + 0.45854771e00, + 0.46059383e00, + 0.46257694e00, + 0.46460808e00, + 0.46651408e00, + 0.46842937e00, + 0.47031930e00, + 0.47214870e00, + 0.47394193e00, + 0.47575376e00, + 0.47747515e00, + 0.47919249e00, + 0.48087054e00, + 0.48248460e00, + 0.48411980e00, + 0.48572180e00, + 0.48729084e00, + 0.48882613e00, + 0.49029764e00, + 0.49173477e00, + 0.49316337e00, + 0.49455725e00, + 0.49591591e00, + 0.49723950e00, + 0.49855476e00, + 0.49983531e00, + 0.50105632e00, + 0.50224193e00, + 0.50338975e00, + 0.50452839e00, + 0.50568552e00, + 0.50673081e00, + 0.50774070e00, + 0.50873875e00, + 0.50974595e00, + 0.51063701e00, + 0.51151521e00, + 0.51238110e00, + 0.51321059e00, + 0.51397275e00, + 0.51472495e00, + 0.51540664e00, + 0.51610368e00, + 0.51667264e00, + 0.51730154e00, + 0.51776070e00, + 0.51827571e00, + 0.51868642e00, + 0.51905088e00, + 0.51938042e00, + 0.51966850e00, + 0.51988677e00, + 0.52010423e00, + 0.52020368e00, + 0.52028969e00, + 0.52027278e00, + 0.52016485e00, + 0.52001351e00, + 0.51981105e00, + 0.51959358e00, + 0.51926552e00, + 0.51884032e00, + 0.51847414e00, + 0.51792884e00, + 0.51731622e00, + 0.51669638e00, + 0.51593604e00, + 0.51513940e00, + 0.51427166e00, + 0.51335442e00, + 0.51238909e00, + 0.51129305e00, + 0.51014602e00, + 0.50890185e00, + 0.50758409e00, + 0.50615044e00, + 0.50464683e00, + 0.50296036e00, + 0.50118289e00, + 0.49930168e00, + 0.49730277e00, + 0.49530416e00, + 0.49314260e00, + 0.49083236e00, + 0.48844397e00, + 0.48583154e00, + 0.48323435e00, + 0.48058649e00, + 0.47780153e00, + 0.47506716e00, + 0.47209482e00, + 0.46907585e00, + 0.46597907e00, + 0.46291516e00, + 0.45981653e00, + 0.45665363e00, + 0.45351121e00, + 0.45044243e00, + 0.44730392e00, + 0.44413627e00, + 0.44106236e00, + 0.43794686e00, + 0.43485646e00, + 0.43175941e00, + 0.42864794e00, + 0.42564675e00, + 0.42266427e00, + 0.41976102e00, + 0.41687658e00, + 0.41406969e00, + 0.41130938e00, + 0.40859441e00, + 0.40588626e00, + 0.40343902e00, + 0.40125003e00, + 0.39935862e00, + 0.39784753e00, + 0.39670361e00, + 0.39585956e00, + 0.39523582e00, + 0.39476103e00, + 0.39439905e00, + 0.39423968e00, + 0.39435100e00, + 0.39473203e00, + 0.39554310e00, + 0.39666286e00, + 0.39808096e00, + 0.39972118e00, + 0.40153794e00, + 0.40352131e00, + 0.40548763e00, + 0.40751478e00, + 0.40963578e00, + 0.41174409e00, + 0.41388333e00, + 0.41612846e00, + 0.41834220e00, + 0.42056149e00, + 0.42288866e00, + 0.42519516e00, + 0.42763343e00, + 0.43005765e00, + 0.43251298e00, + 0.43502684e00, + 0.43754215e00, + 0.44011421e00, + 0.44269314e00, + 0.44537654e00, + 0.44802922e00, + 0.45079502e00, + 0.45340240e00, + 0.45589495e00, + 0.45836848e00, + 0.46074029e00, + 0.46320577e00, + 0.46557171e00, + 0.46804433e00, + 0.47046801e00, + 0.47291016e00, + 0.47527772e00, + 0.47750468e00, + 0.47952019e00, + 0.48141970e00, + 0.48317952e00, + 0.48487642e00, + 0.48650926e00, + 0.48807554e00, + 0.48963920e00, + 0.49111387e00, + 0.49259059e00, + 0.49402593e00, + 0.49539224e00, + 0.49660220e00, + 0.49767488e00, + 0.49868284e00, + 0.49962355e00, + 0.50044291e00, + 0.50121762e00, + 0.50196273e00, + 0.50261243e00, + 0.50326871e00, + 0.50387423e00, + 0.50434609e00, + 0.50480509e00, + 0.50515270e00, + 0.50543081e00, + 0.50566424e00, + 0.50580062e00, + 0.50590890e00, + 0.50589201e00, + 0.50588645e00, + 0.50578122e00, + 0.50561852e00, + 0.50539304e00, + 0.50510737e00, + 0.50474719e00, + 0.50435977e00, + 0.50384726e00, + 0.50336489e00, + 0.50279180e00, + 0.50216504e00, + 0.50148265e00, + 0.50078978e00, + 0.50000804e00, + 0.49918459e00, + 0.49836367e00, + 0.49744147e00, + 0.49651714e00, + 0.49557195e00, + 0.49456744e00, + 0.49353034e00, + 0.49245761e00, + 0.49135232e00, + 0.49023631e00, + 0.48909069e00, + 0.48791635e00, + 0.48669295e00, ] expected_spin_angle = np.arange(1, 361, 1, dtype=np.float64) @@ -166,12 +543,12 @@ def test_convert_dat_to_glows_l3e_lo_product(self, mock_calculate_energy_deltas) spin_axis_lon = 90.0 args = MagicMock(spec=GlowsL3eCallArguments) - expected_program_version = 'Lo.v00.01' + expected_program_version = "Lo.v00.01" spacecraft_info = MagicMock(spec=GlowsL3eSpacecraftInfo) spacecraft_info.spin_axis_latitude = spin_axis_lat spacecraft_info.spin_axis_longitude = spin_axis_lon - spacecraft_info.spacecraft_radius = .5 + spacecraft_info.spacecraft_radius = 0.5 spacecraft_info.spacecraft_longitude = 85.4 spacecraft_info.spacecraft_latitude = 45.1 @@ -181,37 +558,75 @@ def test_convert_dat_to_glows_l3e_lo_product(self, mock_calculate_energy_deltas) args.spacecraft_info = spacecraft_info - l3e_lo_product: GlowsL3ELoData = GlowsL3ELoData.convert_dat_to_glows_l3e_lo_product(mock_metadata, lo_file_path, - epoch, - epoch_delta, - elongation_value, - args) + l3e_lo_product: GlowsL3ELoData = ( + GlowsL3ELoData.convert_dat_to_glows_l3e_lo_product( + mock_metadata, lo_file_path, epoch, epoch_delta, elongation_value, args + ) + ) - mock_calculate_energy_deltas.assert_called_once_with(NumpyArrayMatcher(l3e_lo_product.energy)) + mock_calculate_energy_deltas.assert_called_once_with( + NumpyArrayMatcher(l3e_lo_product.energy) + ) np.testing.assert_equal([epoch], l3e_lo_product.epoch, strict=True) - np.testing.assert_equal([expected_epoch_delta_in_nanoseconds], l3e_lo_product.epoch_delta, strict=True) + np.testing.assert_equal( + [expected_epoch_delta_in_nanoseconds], + l3e_lo_product.epoch_delta, + strict=True, + ) np.testing.assert_equal(l3e_lo_product.energy, expected_energy, strict=True) - np.testing.assert_equal(l3e_lo_product.energy_delta_plus, sentinel.energy_delta_plus) - np.testing.assert_equal(l3e_lo_product.energy_delta_minus, sentinel.energy_delta_minus,) - - np.testing.assert_equal(l3e_lo_product.spin_angle, expected_spin_angle, strict=True) - np.testing.assert_equal(l3e_lo_product.probability_of_survival.shape, expected_survival_probability_shape, - strict=True) - np.testing.assert_equal(l3e_lo_product.probability_of_survival[0][0], - expected_prob_of_survival_first_col_1, strict=True) - np.testing.assert_equal(l3e_lo_product.elongation, np.array([elongation_value]), strict=True) + np.testing.assert_equal( + l3e_lo_product.energy_delta_plus, sentinel.energy_delta_plus + ) + np.testing.assert_equal( + l3e_lo_product.energy_delta_minus, + sentinel.energy_delta_minus, + ) - np.testing.assert_equal(np.array([spin_axis_lat]), l3e_lo_product.spin_axis_lat, strict=True) - np.testing.assert_equal(np.array([spin_axis_lon]), l3e_lo_product.spin_axis_lon, strict=True) + np.testing.assert_equal( + l3e_lo_product.spin_angle, expected_spin_angle, strict=True + ) + np.testing.assert_equal( + l3e_lo_product.probability_of_survival.shape, + expected_survival_probability_shape, + strict=True, + ) + np.testing.assert_equal( + l3e_lo_product.probability_of_survival[0][0], + expected_prob_of_survival_first_col_1, + strict=True, + ) + np.testing.assert_equal( + l3e_lo_product.elongation, np.array([elongation_value]), strict=True + ) - np.testing.assert_equal([expected_program_version], l3e_lo_product.program_version, strict=True) + np.testing.assert_equal( + np.array([spin_axis_lat]), l3e_lo_product.spin_axis_lat, strict=True + ) + np.testing.assert_equal( + np.array([spin_axis_lon]), l3e_lo_product.spin_axis_lon, strict=True + ) - np.testing.assert_equal(l3e_lo_product.spacecraft_radius, np.array([.5]), strict=True) - np.testing.assert_equal(l3e_lo_product.spacecraft_longitude, np.array([85.4]), strict=True) - np.testing.assert_equal(l3e_lo_product.spacecraft_latitude, np.array([45.1]), strict=True) + np.testing.assert_equal( + [expected_program_version], l3e_lo_product.program_version, strict=True + ) - np.testing.assert_equal(l3e_lo_product.spacecraft_velocity_x, np.array([2.1]), strict=True) - np.testing.assert_equal(l3e_lo_product.spacecraft_velocity_y, np.array([2.2]), strict=True) - np.testing.assert_equal(l3e_lo_product.spacecraft_velocity_z, np.array([2.3]), strict=True) + np.testing.assert_equal( + l3e_lo_product.spacecraft_radius, np.array([0.5]), strict=True + ) + np.testing.assert_equal( + l3e_lo_product.spacecraft_longitude, np.array([85.4]), strict=True + ) + np.testing.assert_equal( + l3e_lo_product.spacecraft_latitude, np.array([45.1]), strict=True + ) + np.testing.assert_equal( + l3e_lo_product.spacecraft_velocity_x, np.array([2.1]), strict=True + ) + np.testing.assert_equal( + l3e_lo_product.spacecraft_velocity_y, np.array([2.2]), strict=True + ) + np.testing.assert_equal( + l3e_lo_product.spacecraft_velocity_z, np.array([2.3]), strict=True + ) diff --git a/tests/glows/l3e/test_glows_l3e_ultra_model.py b/tests/glows/l3e/test_glows_l3e_ultra_model.py index 82a18e779..dff3996c1 100644 --- a/tests/glows/l3e/test_glows_l3e_ultra_model.py +++ b/tests/glows/l3e/test_glows_l3e_ultra_model.py @@ -4,14 +4,35 @@ import numpy as np -from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import GlowsL3eCallArguments, GlowsL3eSpacecraftInfo -from imap_l3_processing.glows.l3e.glows_l3e_ultra_model import GlowsL3EUltraData, EPOCH_CDF_VAR_NAME, EPOCH_DELTA_CDF_VAR_NAME, ENERGY_VAR_NAME, \ - PROBABILITY_OF_SURVIVAL_VAR_NAME, HEALPIX_INDEX_VAR_NAME, ENERGY_LABEL_VAR_NAME, PIXEL_INDEX_LABEL_VAR_NAME, \ - SPIN_AXIS_LATITUDE_VAR_NAME, SPIN_AXIS_LONGITUDE_VAR_NAME, PROGRAM_VERSION_VAR_NAME, SPACECRAFT_RADIUS_VAR_NAME, \ - SPACECRAFT_LONGITUDE_VAR_NAME, SPACECRAFT_LATITUDE_VAR_NAME, SPACECRAFT_VELOCITY_X_VAR_NAME, \ - SPACECRAFT_VELOCITY_Y_VAR_NAME, SPACECRAFT_VELOCITY_Z_VAR_NAME, ELONGATION_EXCLUDED_VAR_NAME, \ - PIXEL_LATITUDE_VAR_NAME, PIXEL_LONGITUDE_VAR_NAME, GLOWS_FLAGS_VAR_NAME, ENERGY_DELTA_PLUS_VAR_NAME, \ - ENERGY_DELTA_MINUS_VAR_NAME +from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import ( + GlowsL3eCallArguments, + GlowsL3eSpacecraftInfo, +) +from imap_l3_processing.glows.l3e.glows_l3e_ultra_model import ( + GlowsL3EUltraData, + EPOCH_CDF_VAR_NAME, + EPOCH_DELTA_CDF_VAR_NAME, + ENERGY_VAR_NAME, + PROBABILITY_OF_SURVIVAL_VAR_NAME, + HEALPIX_INDEX_VAR_NAME, + ENERGY_LABEL_VAR_NAME, + PIXEL_INDEX_LABEL_VAR_NAME, + SPIN_AXIS_LATITUDE_VAR_NAME, + SPIN_AXIS_LONGITUDE_VAR_NAME, + PROGRAM_VERSION_VAR_NAME, + SPACECRAFT_RADIUS_VAR_NAME, + SPACECRAFT_LONGITUDE_VAR_NAME, + SPACECRAFT_LATITUDE_VAR_NAME, + SPACECRAFT_VELOCITY_X_VAR_NAME, + SPACECRAFT_VELOCITY_Y_VAR_NAME, + SPACECRAFT_VELOCITY_Z_VAR_NAME, + ELONGATION_EXCLUDED_VAR_NAME, + PIXEL_LATITUDE_VAR_NAME, + PIXEL_LONGITUDE_VAR_NAME, + GLOWS_FLAGS_VAR_NAME, + ENERGY_DELTA_PLUS_VAR_NAME, + ENERGY_DELTA_MINUS_VAR_NAME, +) from imap_l3_processing.models import DataProductVariable from tests.test_helpers import get_test_instrument_team_data_path, NumpyArrayMatcher @@ -19,22 +40,26 @@ class TestL3eUltraModel(unittest.TestCase): def test_l3e_ultra_model_to_data_product_variables(self): test_cases = { - 'case 1': ( + "case 1": ( [1.2234, 2, 4.51, 66.7666], [1, 41650233.0, 4.22, 9.5], - ['1.22', '2.00', '4.51', '66.77'], - ['1', '41650233', '4', '10'] + ["1.22", "2.00", "4.51", "66.77"], + ["1", "41650233", "4", "10"], ), - 'case 2': ( + "case 2": ( [4.536, 48.193, 4253.1], [13, 14.0, 34.2, 19.5], - ['4.54', '48.19', '4253.10'], - ['13', '14', '34', '20'] - ) + ["4.54", "48.19", "4253.10"], + ["13", "14", "34", "20"], + ), } - for name, (energy_array, healpix_index_array, expected_energy_labels, - expected_healpix_labels) in test_cases.items(): + for name, ( + energy_array, + healpix_index_array, + expected_energy_labels, + expected_healpix_labels, + ) in test_cases.items(): with self.subTest(name): l3e_ultra: GlowsL3EUltraData = GlowsL3EUltraData( Mock(), @@ -65,24 +90,57 @@ def test_l3e_ultra_model_to_data_product_variables(self): DataProductVariable(EPOCH_CDF_VAR_NAME, sentinel.epoch), DataProductVariable(EPOCH_DELTA_CDF_VAR_NAME, sentinel.epoch_delta), DataProductVariable(ENERGY_VAR_NAME, energy_array), - DataProductVariable(ENERGY_DELTA_PLUS_VAR_NAME, sentinel.energy_delta_plus), - DataProductVariable(ENERGY_DELTA_MINUS_VAR_NAME, sentinel.energy_delta_minus), + DataProductVariable( + ENERGY_DELTA_PLUS_VAR_NAME, sentinel.energy_delta_plus + ), + DataProductVariable( + ENERGY_DELTA_MINUS_VAR_NAME, sentinel.energy_delta_minus + ), DataProductVariable(HEALPIX_INDEX_VAR_NAME, healpix_index_array), - DataProductVariable(PROBABILITY_OF_SURVIVAL_VAR_NAME, sentinel.probability_of_survival), + DataProductVariable( + PROBABILITY_OF_SURVIVAL_VAR_NAME, + sentinel.probability_of_survival, + ), DataProductVariable(ENERGY_LABEL_VAR_NAME, expected_energy_labels), - DataProductVariable(PIXEL_INDEX_LABEL_VAR_NAME, expected_healpix_labels), - DataProductVariable(SPIN_AXIS_LATITUDE_VAR_NAME, sentinel.spin_axis_latitude), - DataProductVariable(SPIN_AXIS_LONGITUDE_VAR_NAME, sentinel.spin_axis_longitude), - DataProductVariable(PROGRAM_VERSION_VAR_NAME, sentinel.program_version), - DataProductVariable(SPACECRAFT_RADIUS_VAR_NAME, sentinel.spacecraft_radius), - DataProductVariable(SPACECRAFT_LATITUDE_VAR_NAME, sentinel.spacecraft_latitude), - DataProductVariable(SPACECRAFT_LONGITUDE_VAR_NAME, sentinel.spacecraft_longitude), - DataProductVariable(SPACECRAFT_VELOCITY_X_VAR_NAME, sentinel.spacecraft_velocity_x), - DataProductVariable(SPACECRAFT_VELOCITY_Y_VAR_NAME, sentinel.spacecraft_velocity_y), - DataProductVariable(SPACECRAFT_VELOCITY_Z_VAR_NAME, sentinel.spacecraft_velocity_z), - DataProductVariable(ELONGATION_EXCLUDED_VAR_NAME, sentinel.elongation_excluded), - DataProductVariable(PIXEL_LATITUDE_VAR_NAME, sentinel.pixel_latitude), - DataProductVariable(PIXEL_LONGITUDE_VAR_NAME, sentinel.pixel_longitude), + DataProductVariable( + PIXEL_INDEX_LABEL_VAR_NAME, expected_healpix_labels + ), + DataProductVariable( + SPIN_AXIS_LATITUDE_VAR_NAME, sentinel.spin_axis_latitude + ), + DataProductVariable( + SPIN_AXIS_LONGITUDE_VAR_NAME, sentinel.spin_axis_longitude + ), + DataProductVariable( + PROGRAM_VERSION_VAR_NAME, sentinel.program_version + ), + DataProductVariable( + SPACECRAFT_RADIUS_VAR_NAME, sentinel.spacecraft_radius + ), + DataProductVariable( + SPACECRAFT_LATITUDE_VAR_NAME, sentinel.spacecraft_latitude + ), + DataProductVariable( + SPACECRAFT_LONGITUDE_VAR_NAME, sentinel.spacecraft_longitude + ), + DataProductVariable( + SPACECRAFT_VELOCITY_X_VAR_NAME, sentinel.spacecraft_velocity_x + ), + DataProductVariable( + SPACECRAFT_VELOCITY_Y_VAR_NAME, sentinel.spacecraft_velocity_y + ), + DataProductVariable( + SPACECRAFT_VELOCITY_Z_VAR_NAME, sentinel.spacecraft_velocity_z + ), + DataProductVariable( + ELONGATION_EXCLUDED_VAR_NAME, sentinel.elongation_excluded + ), + DataProductVariable( + PIXEL_LATITUDE_VAR_NAME, sentinel.pixel_latitude + ), + DataProductVariable( + PIXEL_LONGITUDE_VAR_NAME, sentinel.pixel_longitude + ), DataProductVariable(GLOWS_FLAGS_VAR_NAME, sentinel.glows_flags), ] @@ -90,33 +148,116 @@ def test_l3e_ultra_model_to_data_product_variables(self): @patch("imap_l3_processing.glows.l3e.glows_l3e_ultra_model.calculate_energy_deltas") def test_convert_dat_to_glows_l3e_ul_product(self, mock_calculate_energy_deltas): - ul_file_path = get_test_instrument_team_data_path("glows/probSur.Imap.Ul_20250420_000000_2025.300.txt") + ul_file_path = get_test_instrument_team_data_path( + "glows/probSur.Imap.Ul_20250420_000000_2025.300.txt" + ) expected_epoch = datetime(year=2009, month=1, day=1) epoch_delta = timedelta(hours=10) - expected_epoch_delta_in_nanoseconds = 10*3600*1e9 + expected_epoch_delta_in_nanoseconds = 10 * 3600 * 1e9 - mock_calculate_energy_deltas.return_value = sentinel.energy_delta_plus, sentinel.energy_delta_minus + mock_calculate_energy_deltas.return_value = ( + sentinel.energy_delta_plus, + sentinel.energy_delta_minus, + ) expected_energy = np.array( - [2.3751086, 3.0917682, 4.0246710, 5.2390656, 6.8198887, 8.8777057, 11.5564435, 15.0434572, 19.5826341, - 25.4914514, 33.1831812, 43.1957952, 56.2295915, 73.1961745, 95.2822137, 124.0324417, 161.4576950, - 210.1755551, 273.5934261, 356.1468544]) + [ + 2.3751086, + 3.0917682, + 4.0246710, + 5.2390656, + 6.8198887, + 8.8777057, + 11.5564435, + 15.0434572, + 19.5826341, + 25.4914514, + 33.1831812, + 43.1957952, + 56.2295915, + 73.1961745, + 95.2822137, + 124.0324417, + 161.4576950, + 210.1755551, + 273.5934261, + 356.1468544, + ] + ) row_0_probability_of_survival = np.array( - [0.84827693E+00, 0.86517360E+00, 0.88086645E+00, 0.89551288E+00, 0.90925039E+00, 0.92211605E+00, - 0.93428862E+00, 0.94576000E+00, 0.95660359E+00, 0.96660933E+00, 0.97533752E+00, 0.98226689E+00, - 0.98714107E+00, 0.99025210E+00, 0.99218849E+00, 0.99346478E+00, 0.99439230E+00, 0.99512632E+00, - 0.99574163E+00, 0.99627086E+00]) - - row_804_probability_of_survival = np.array([np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, - np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, - np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, - np.nan, np.nan]) + [ + 0.84827693e00, + 0.86517360e00, + 0.88086645e00, + 0.89551288e00, + 0.90925039e00, + 0.92211605e00, + 0.93428862e00, + 0.94576000e00, + 0.95660359e00, + 0.96660933e00, + 0.97533752e00, + 0.98226689e00, + 0.98714107e00, + 0.99025210e00, + 0.99218849e00, + 0.99346478e00, + 0.99439230e00, + 0.99512632e00, + 0.99574163e00, + 0.99627086e00, + ] + ) + + row_804_probability_of_survival = np.array( + [ + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + np.nan, + ] + ) row_3071_expected_probability_of_survival = np.array( - [0.84498413E+00, 0.86231633E+00, 0.87840630E+00, 0.89344499E+00, 0.90753390E+00, 0.92079246E+00, - 0.93326641E+00, 0.94509283E+00, 0.95623858E+00, 0.96649540E+00, 0.97543873E+00, 0.98248159E+00, - 0.98738469E+00, 0.99048237E+00, 0.99239223E+00, 0.99364490E+00, 0.99455000E+00, 0.99526291E+00, - 0.99585178E+00, 0.99635733E+00]) + [ + 0.84498413e00, + 0.86231633e00, + 0.87840630e00, + 0.89344499e00, + 0.90753390e00, + 0.92079246e00, + 0.93326641e00, + 0.94509283e00, + 0.95623858e00, + 0.96649540e00, + 0.97543873e00, + 0.98248159e00, + 0.98738469e00, + 0.99048237e00, + 0.99239223e00, + 0.99364490e00, + 0.99455000e00, + 0.99526291e00, + 0.99585178e00, + 0.99635733e00, + ] + ) expected_survival_probability_shape = (1, 20, 3072) @@ -127,12 +268,12 @@ def test_convert_dat_to_glows_l3e_ul_product(self, mock_calculate_energy_deltas) spin_axis_lon = 90.0 args = MagicMock(spec=GlowsL3eCallArguments) - expected_program_version = 'Ultra.v00.01' + expected_program_version = "Ultra.v00.01" spacecraft_info = MagicMock(spec=GlowsL3eSpacecraftInfo) spacecraft_info.spin_axis_latitude = spin_axis_lat spacecraft_info.spin_axis_longitude = spin_axis_lon - spacecraft_info.spacecraft_radius = .5 + spacecraft_info.spacecraft_radius = 0.5 spacecraft_info.spacecraft_longitude = 85.4 spacecraft_info.spacecraft_latitude = 45.1 @@ -144,51 +285,100 @@ def test_convert_dat_to_glows_l3e_ul_product(self, mock_calculate_energy_deltas) args.elongation = 30.0 - l3e_ul_product: GlowsL3EUltraData = GlowsL3EUltraData.convert_dat_to_glows_l3e_ul_product(mock_metadata, - ul_file_path, - expected_epoch, - epoch_delta, - args) + l3e_ul_product: GlowsL3EUltraData = ( + GlowsL3EUltraData.convert_dat_to_glows_l3e_ul_product( + mock_metadata, ul_file_path, expected_epoch, epoch_delta, args + ) + ) - mock_calculate_energy_deltas.assert_called_once_with(NumpyArrayMatcher(l3e_ul_product.energy),) + mock_calculate_energy_deltas.assert_called_once_with( + NumpyArrayMatcher(l3e_ul_product.energy), + ) np.testing.assert_equal([expected_epoch], l3e_ul_product.epoch, strict=True) - np.testing.assert_equal([expected_epoch_delta_in_nanoseconds], l3e_ul_product.epoch_delta, strict=True) + np.testing.assert_equal( + [expected_epoch_delta_in_nanoseconds], + l3e_ul_product.epoch_delta, + strict=True, + ) np.testing.assert_equal(l3e_ul_product.energy, expected_energy, strict=True) - np.testing.assert_equal(l3e_ul_product.energy_delta_plus, sentinel.energy_delta_plus, strict=True) - np.testing.assert_equal(l3e_ul_product.energy_delta_minus, sentinel.energy_delta_minus, strict=True) - - np.testing.assert_equal(expected_heal_pix, l3e_ul_product.healpix_index, strict=True) - np.testing.assert_equal(expected_survival_probability_shape, l3e_ul_product.probability_of_survival.shape, - strict=True) - np.testing.assert_equal(l3e_ul_product.probability_of_survival[0].T[0, :], row_0_probability_of_survival, - strict=True) - np.testing.assert_equal(l3e_ul_product.probability_of_survival[0].T[804, :], - row_804_probability_of_survival, strict=True) - np.testing.assert_equal(l3e_ul_product.probability_of_survival[0].T[3071, :], - row_3071_expected_probability_of_survival, strict=True) - - np.testing.assert_equal(np.array([spin_axis_lat]), l3e_ul_product.spin_axis_lat, strict=True) - np.testing.assert_equal(np.array([spin_axis_lon]), l3e_ul_product.spin_axis_lon, strict=True) - - np.testing.assert_equal(np.array([expected_program_version]), l3e_ul_product.program_version, strict=True) - - np.testing.assert_equal(np.array([.5]), l3e_ul_product.spacecraft_radius, strict=True) - np.testing.assert_equal(np.array([85.4]), l3e_ul_product.spacecraft_longitude, strict=True) - np.testing.assert_equal(np.array([45.1]), l3e_ul_product.spacecraft_latitude, strict=True) - - np.testing.assert_equal(np.array([2.1]), l3e_ul_product.spacecraft_velocity_x, strict=True) - np.testing.assert_equal(np.array([2.2]), l3e_ul_product.spacecraft_velocity_y, strict=True) - np.testing.assert_equal(np.array([2.3]), l3e_ul_product.spacecraft_velocity_z, strict=True) - - np.testing.assert_equal(np.array([30.0]), l3e_ul_product.elongation_excluded, strict=True) - - np.testing.assert_equal((1, 3072), l3e_ul_product.pixel_latitude.shape, strict=True) - np.testing.assert_equal((1, 3072), l3e_ul_product.pixel_longitude.shape, strict=True) + np.testing.assert_equal( + l3e_ul_product.energy_delta_plus, sentinel.energy_delta_plus, strict=True + ) + np.testing.assert_equal( + l3e_ul_product.energy_delta_minus, sentinel.energy_delta_minus, strict=True + ) + + np.testing.assert_equal( + expected_heal_pix, l3e_ul_product.healpix_index, strict=True + ) + np.testing.assert_equal( + expected_survival_probability_shape, + l3e_ul_product.probability_of_survival.shape, + strict=True, + ) + np.testing.assert_equal( + l3e_ul_product.probability_of_survival[0].T[0, :], + row_0_probability_of_survival, + strict=True, + ) + np.testing.assert_equal( + l3e_ul_product.probability_of_survival[0].T[804, :], + row_804_probability_of_survival, + strict=True, + ) + np.testing.assert_equal( + l3e_ul_product.probability_of_survival[0].T[3071, :], + row_3071_expected_probability_of_survival, + strict=True, + ) + + np.testing.assert_equal( + np.array([spin_axis_lat]), l3e_ul_product.spin_axis_lat, strict=True + ) + np.testing.assert_equal( + np.array([spin_axis_lon]), l3e_ul_product.spin_axis_lon, strict=True + ) + + np.testing.assert_equal( + np.array([expected_program_version]), + l3e_ul_product.program_version, + strict=True, + ) + + np.testing.assert_equal( + np.array([0.5]), l3e_ul_product.spacecraft_radius, strict=True + ) + np.testing.assert_equal( + np.array([85.4]), l3e_ul_product.spacecraft_longitude, strict=True + ) + np.testing.assert_equal( + np.array([45.1]), l3e_ul_product.spacecraft_latitude, strict=True + ) + + np.testing.assert_equal( + np.array([2.1]), l3e_ul_product.spacecraft_velocity_x, strict=True + ) + np.testing.assert_equal( + np.array([2.2]), l3e_ul_product.spacecraft_velocity_y, strict=True + ) + np.testing.assert_equal( + np.array([2.3]), l3e_ul_product.spacecraft_velocity_z, strict=True + ) + + np.testing.assert_equal( + np.array([30.0]), l3e_ul_product.elongation_excluded, strict=True + ) + + np.testing.assert_equal( + (1, 3072), l3e_ul_product.pixel_latitude.shape, strict=True + ) + np.testing.assert_equal( + (1, 3072), l3e_ul_product.pixel_longitude.shape, strict=True + ) np.testing.assert_equal(87.07582, l3e_ul_product.pixel_latitude[0, 0]) np.testing.assert_equal(45.00000, l3e_ul_product.pixel_longitude[0, 0]) np.testing.assert_equal(np.nan, l3e_ul_product.pixel_latitude[0, 804]) np.testing.assert_equal(np.nan, l3e_ul_product.pixel_longitude[0, 804]) - diff --git a/tests/glows/l3e/test_glows_l3e_utils.py b/tests/glows/l3e/test_glows_l3e_utils.py index a213544fe..66e47959c 100644 --- a/tests/glows/l3e/test_glows_l3e_utils.py +++ b/tests/glows/l3e/test_glows_l3e_utils.py @@ -7,7 +7,10 @@ import imap_data_access import numpy as np from imap_data_access import SPICEFilePath -from imap_processing.spice.repoint import get_repoint_data, set_global_repoint_table_paths +from imap_processing.spice.repoint import ( + get_repoint_data, + set_global_repoint_table_paths, +) from spacepy.pycdf import CDF, const from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import ( @@ -22,29 +25,38 @@ compute_glows_flags_for_repoint, get_repoint_numbers_within_cr_window, determine_spacecraft_info_for_l3e_executable, - determine_spacecraft_info_using_predict_if_needed, calculate_energy_deltas, + determine_spacecraft_info_using_predict_if_needed, + calculate_energy_deltas, ) from imap_l3_processing.glows.quality_flags import GlowsL3Flags from imap_l3_processing.utils import FurnishMetakernelOutput -from tests.integration.integration_test_helpers import mock_imap_data_access, create_metakernel +from tests.integration.integration_test_helpers import ( + mock_imap_data_access, + create_metakernel, +) from tests.test_helpers import get_test_data_path, get_integration_test_data_path class TestGlowsL3EUtils(unittest.TestCase): - @patch("imap_l3_processing.glows.l3e.glows_l3e_utils.spiceypy.datetime2et") @patch("imap_l3_processing.glows.l3e.glows_l3e_utils.spiceypy.spkezr") @patch("imap_l3_processing.glows.l3e.glows_l3e_utils.spiceypy.reclat") @patch("imap_l3_processing.glows.l3e.glows_l3e_utils.spiceypy.pxform") - def test_determine_call_args_for_l3e_executable(self, mock_pxform: Mock, mock_reclat: Mock, mock_spkezr: Mock, - mock_date_time_2et: Mock): + def test_determine_call_args_for_l3e_executable( + self, + mock_pxform: Mock, + mock_reclat: Mock, + mock_spkezr: Mock, + mock_date_time_2et: Mock, + ): start_time = datetime.fromisoformat("2025-05-01 00:00:00") repointing_midpoint = datetime.fromisoformat("2025-05-01 12:00:00") elongation = 90 spacecraft_info = Mock() - call_args: GlowsL3eCallArguments = determine_call_args_for_l3e_executable(start_time, repointing_midpoint, - elongation, spacecraft_info) + call_args: GlowsL3eCallArguments = determine_call_args_for_l3e_executable( + start_time, repointing_midpoint, elongation, spacecraft_info + ) self.assertEqual("20250501_000000", call_args.formatted_date) self.assertEqual("2025.33014", call_args.decimal_date) @@ -55,34 +67,52 @@ def test_determine_call_args_for_l3e_executable(self, mock_pxform: Mock, mock_re @patch("imap_l3_processing.glows.l3e.glows_l3e_utils.spiceypy.spkezr") @patch("imap_l3_processing.glows.l3e.glows_l3e_utils.spiceypy.reclat") @patch("imap_l3_processing.glows.l3e.glows_l3e_utils.spiceypy.pxform") - def test_determine_spacecraft_info_for_l3e_executable(self, mock_pxform: Mock, mock_reclat: Mock, mock_spkezr: Mock, - mock_date_time_2et: Mock): + def test_determine_spacecraft_info_for_l3e_executable( + self, + mock_pxform: Mock, + mock_reclat: Mock, + mock_spkezr: Mock, + mock_date_time_2et: Mock, + ): repointing_midpoint = datetime.fromisoformat("2025-05-01 12:00:00") x, y, z, vx, vy, vz = 1.0, 2.0, 3.0, 4.0, 5.0, 6.0 position_data = [x, y, z, vx, vy, vz] mock_spkezr.return_value = (position_data, Mock()) - radius, longitude, latitude = 70000000, -8.0, -.9 + radius, longitude, latitude = 70000000, -8.0, -0.9 - rotation_matrix = np.array([[10.0, 11.0, 12.0], [13.0, 14.0, 15.0], [16.0, 17.0, 18.0]]) + rotation_matrix = np.array( + [[10.0, 11.0, 12.0], [13.0, 14.0, 15.0], [16.0, 17.0, 18.0]] + ) mock_pxform.return_value = rotation_matrix spin_axis_long, spin_axis_lat = -1.4, 0.2 - mock_reclat.side_effect = [(radius, longitude, latitude), (Mock(), spin_axis_long, spin_axis_lat)] + mock_reclat.side_effect = [ + (radius, longitude, latitude), + (Mock(), spin_axis_long, spin_axis_lat), + ] - call_args: GlowsL3eSpacecraftInfo = determine_spacecraft_info_for_l3e_executable(repointing_midpoint) + call_args: GlowsL3eSpacecraftInfo = ( + determine_spacecraft_info_for_l3e_executable(repointing_midpoint) + ) self.assertIsInstance(call_args, GlowsL3eSpacecraftInfo) mock_date_time_2et.assert_called_once_with(repointing_midpoint) - mock_spkezr.assert_called_once_with("IMAP", mock_date_time_2et.return_value, "ECLIPJ2000", "NONE", "SUN") + mock_spkezr.assert_called_once_with( + "IMAP", mock_date_time_2et.return_value, "ECLIPJ2000", "NONE", "SUN" + ) np.testing.assert_array_equal([x, y, z], mock_reclat.call_args_list[0][0][0]) - mock_pxform.assert_called_once_with("IMAP_DPS", "ECLIPJ2000", mock_date_time_2et.return_value) - np.testing.assert_array_equal([12.0, 15.0, 18.0], mock_reclat.call_args_list[1][0][0]) + mock_pxform.assert_called_once_with( + "IMAP_DPS", "ECLIPJ2000", mock_date_time_2et.return_value + ) + np.testing.assert_array_equal( + [12.0, 15.0, 18.0], mock_reclat.call_args_list[1][0][0] + ) self.assertEqual(0.4679210985587912, call_args.spacecraft_radius) self.assertEqual(261.6337638953414, call_args.spacecraft_longitude) @@ -90,17 +120,24 @@ def test_determine_spacecraft_info_for_l3e_executable(self, mock_pxform: Mock, m self.assertEqual(vx, call_args.spacecraft_velocity_x) self.assertEqual(vy, call_args.spacecraft_velocity_y) self.assertEqual(vz, call_args.spacecraft_velocity_z) - self.assertEqual(np.rad2deg(spin_axis_long) % 360, call_args.spin_axis_longitude) + self.assertEqual( + np.rad2deg(spin_axis_long) % 360, call_args.spin_axis_longitude + ) self.assertEqual(np.rad2deg(spin_axis_lat), call_args.spin_axis_latitude) def test_determine_spacecraft_info_using_predict_if_needed(self): - with (tempfile.TemporaryDirectory() as tmp_dir): - + with tempfile.TemporaryDirectory() as tmp_dir: input_files = [ get_integration_test_data_path("spice/de440.bsp"), - get_integration_test_data_path("spice/imap_dps_2025_359_2026_058_002.ah.bc"), - get_integration_test_data_path("spice/imap_dps_2025_359_2026_131_002.ah.bc"), - get_integration_test_data_path("spice/imap_recon_20250925_20260520_v01.bsp"), + get_integration_test_data_path( + "spice/imap_dps_2025_359_2026_058_002.ah.bc" + ), + get_integration_test_data_path( + "spice/imap_dps_2025_359_2026_131_002.ah.bc" + ), + get_integration_test_data_path( + "spice/imap_recon_20250925_20260520_v01.bsp" + ), get_integration_test_data_path("spice/imap_science_120.tf"), get_integration_test_data_path("spice/imap_sclk_0171.tsc"), get_integration_test_data_path("spice/naif0012.tls"), @@ -108,49 +145,91 @@ def test_determine_spacecraft_info_using_predict_if_needed(self): with mock_imap_data_access(Path(tmp_dir), input_files): spice_paths_with_predict = [SPICEFilePath(p.name) for p in input_files] - spice_paths_without_predict = [p for p in spice_paths_with_predict if "imap_dps_2025_359_2026_131" not in p.filename.name] + spice_paths_without_predict = [ + p + for p in spice_paths_with_predict + if "imap_dps_2025_359_2026_131" not in p.filename.name + ] spice_dir = imap_data_access.config["DATA_DIR"] / "imap/spice" metakernel_with_predict = Path(tmp_dir, "metakernel_with_predict") - metakernel_with_predict.write_text(create_metakernel(spice_dir, spice_paths_with_predict)) + metakernel_with_predict.write_text( + create_metakernel(spice_dir, spice_paths_with_predict) + ) metakernel_without_predict = Path(tmp_dir, "metakernel_without_predict") - metakernel_without_predict.write_text(create_metakernel(spice_dir, spice_paths_without_predict)) - - input_files_with_predict = [Path("parent/kernel 1"), Path("kernel 2"), Path("kernel 3"), Path("kernel 4")] - input_files_without_predict = [Path("kernel 5"), Path("kernel 6"), Path("kernel 7"), Path("kernel 8")] - - spice_with_predict = FurnishMetakernelOutput(metakernel_with_predict, input_files_with_predict) - spice_without_predict = FurnishMetakernelOutput(metakernel_without_predict, input_files_without_predict) + metakernel_without_predict.write_text( + create_metakernel(spice_dir, spice_paths_without_predict) + ) + + input_files_with_predict = [ + Path("parent/kernel 1"), + Path("kernel 2"), + Path("kernel 3"), + Path("kernel 4"), + ] + input_files_without_predict = [ + Path("kernel 5"), + Path("kernel 6"), + Path("kernel 7"), + Path("kernel 8"), + ] + + spice_with_predict = FurnishMetakernelOutput( + metakernel_with_predict, input_files_with_predict + ) + spice_without_predict = FurnishMetakernelOutput( + metakernel_without_predict, input_files_without_predict + ) date_with_predict = datetime(2026, 3, 25, 12) date_without_predict = datetime(2026, 1, 15, 12) - result_without_predict = determine_spacecraft_info_using_predict_if_needed(date_without_predict, spice_with_predict, spice_without_predict) + result_without_predict = ( + determine_spacecraft_info_using_predict_if_needed( + date_without_predict, spice_with_predict, spice_without_predict + ) + ) spacecraft_info, glows_flags, kernel_names = result_without_predict self.assertEqual(GlowsL3Flags.NONE, glows_flags) - self.assertEqual(["kernel 5", "kernel 6", "kernel 7", "kernel 8"], kernel_names) - self.assertEqual(GlowsL3eSpacecraftInfo(spacecraft_radius=0.974926812207828, - spacecraft_longitude=np.float64(114.98987223111648), - spacecraft_latitude=np.float64(-0.02732735672494243), - spacecraft_velocity_x=np.float64(-27.105121226323945), - spacecraft_velocity_y=np.float64(-12.572196725157356), - spacecraft_velocity_z=np.float64(0.04038850296808949), - spin_axis_longitude=np.float64(291.46869723545007), - spin_axis_latitude=np.float64(0.08089846053031882)), spacecraft_info) - - result_with_predict = determine_spacecraft_info_using_predict_if_needed(date_with_predict, spice_with_predict, spice_without_predict) + self.assertEqual( + ["kernel 5", "kernel 6", "kernel 7", "kernel 8"], kernel_names + ) + self.assertEqual( + GlowsL3eSpacecraftInfo( + spacecraft_radius=0.974926812207828, + spacecraft_longitude=np.float64(114.98987223111648), + spacecraft_latitude=np.float64(-0.02732735672494243), + spacecraft_velocity_x=np.float64(-27.105121226323945), + spacecraft_velocity_y=np.float64(-12.572196725157356), + spacecraft_velocity_z=np.float64(0.04038850296808949), + spin_axis_longitude=np.float64(291.46869723545007), + spin_axis_latitude=np.float64(0.08089846053031882), + ), + spacecraft_info, + ) + + result_with_predict = determine_spacecraft_info_using_predict_if_needed( + date_with_predict, spice_with_predict, spice_without_predict + ) spacecraft_info, glows_flags, kernel_names = result_with_predict self.assertEqual(GlowsL3Flags.PREDICTIVE_EPHEMERIS, glows_flags) - self.assertEqual(["kernel 1", "kernel 2", "kernel 3", "kernel 4"], kernel_names) - self.assertEqual(GlowsL3eSpacecraftInfo(spacecraft_radius=0.9869148557889134, - spacecraft_longitude=np.float64(184.44380384879602), - spacecraft_latitude=np.float64(0.04367233154056944), - spacecraft_velocity_x=np.float64(1.8304001531697336), - spacecraft_velocity_y=np.float64(-29.587138738875474), - spacecraft_velocity_z=np.float64(-0.009923833866874787), - spin_axis_longitude=np.float64(0.9116206658792297), - spin_axis_latitude=np.float64(0.0758987849137639)) ,spacecraft_info) + self.assertEqual( + ["kernel 1", "kernel 2", "kernel 3", "kernel 4"], kernel_names + ) + self.assertEqual( + GlowsL3eSpacecraftInfo( + spacecraft_radius=0.9869148557889134, + spacecraft_longitude=np.float64(184.44380384879602), + spacecraft_latitude=np.float64(0.04367233154056944), + spacecraft_velocity_x=np.float64(1.8304001531697336), + spacecraft_velocity_y=np.float64(-29.587138738875474), + spacecraft_velocity_z=np.float64(-0.009923833866874787), + spin_axis_longitude=np.float64(0.9116206658792297), + spin_axis_latitude=np.float64(0.0758987849137639), + ), + spacecraft_info, + ) def test_get_lo_pivot_angle_from_l1b_file_real_cdf(self): l1b_file = get_test_data_path( @@ -175,16 +254,38 @@ def test_get_lo_pivot_angle_from_l1b_file_scenarios(self): datetime(2026, 3, 20, 23, 0), datetime(2026, 3, 20, 23, 30), ] - epochs = first_thirty_minutes + thirty_minutes_to_22_hours_thirty_minutes + after_22_hours_thirty_minutes + epochs = ( + first_thirty_minutes + + thirty_minutes_to_22_hours_thirty_minutes + + after_22_hours_thirty_minutes + ) shifted_epochs = [e + timedelta(hours=10) for e in epochs] cases = [ - ("realistic", epochs, [89.1, 89.9, 89.9, 89.9, 89.9, 89.9, 89.9, 89.9, 89.9], 90), + ( + "realistic", + epochs, + [89.1, 89.9, 89.9, 89.9, 89.9, 89.9, 89.9, 89.9, 89.9], + 90, + ), ("basic", epochs, [10, 20, 30, 40, 50, 60, 70, 80, 90], 50), - ("only uses data within 3-15 hours from first point", shifted_epochs, - [999, 999, 30, 40, 50, 60, 70, 999, 999], 50), - ("uses median and rounds", epochs, [999, 999, 120.2, 34.4, 86.8, 50.9, 77.7, 999, 999], 78), - ("fallback to 90 if no points in interval", first_thirty_minutes + after_22_hours_thirty_minutes, - [10, 10, 10, 10], 90), + ( + "only uses data within 3-15 hours from first point", + shifted_epochs, + [999, 999, 30, 40, 50, 60, 70, 999, 999], + 50, + ), + ( + "uses median and rounds", + epochs, + [999, 999, 120.2, 34.4, 86.8, 50.9, 77.7, 999, 999], + 78, + ), + ( + "fallback to 90 if no points in interval", + first_thirty_minutes + after_22_hours_thirty_minutes, + [10, 10, 10, 10], + 90, + ), ("fallback to 90 if no points at all", [], [], 90), ] for name, epochs, pivot_angles, expected in cases: @@ -214,12 +315,34 @@ def test_compute_glows_flags_for_repoint(self): ] cases = [ - ("between crs 1 and 2", datetime(2025, 5, 10), GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO), + ( + "between crs 1 and 2", + datetime(2025, 5, 10), + GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO, + ), ("between crs 2 and 3", datetime(2025, 5, 25, 23), GlowsL3Flags.NONE), - ("between crs 3 and 4", datetime(2025, 5, 26, 1), GlowsL3Flags.PREDICTIVE_EPHEMERIS), - ("exactly on cr 4", datetime(2025, 6, 5), GlowsL3Flags.PREDICTIVE_EPHEMERIS), - ("between cr 4 and 5", datetime(2025, 6, 7), GlowsL3Flags.PREDICTIVE_EPHEMERIS | GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO), - ("after last data point", datetime(2025, 6, 15, 1), GlowsL3Flags.PREDICTIVE_EPHEMERIS | GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO) + ( + "between crs 3 and 4", + datetime(2025, 5, 26, 1), + GlowsL3Flags.PREDICTIVE_EPHEMERIS, + ), + ( + "exactly on cr 4", + datetime(2025, 6, 5), + GlowsL3Flags.PREDICTIVE_EPHEMERIS, + ), + ( + "between cr 4 and 5", + datetime(2025, 6, 7), + GlowsL3Flags.PREDICTIVE_EPHEMERIS + | GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO, + ), + ( + "after last data point", + datetime(2025, 6, 15, 1), + GlowsL3Flags.PREDICTIVE_EPHEMERIS + | GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO, + ), ] for name, repoint_midpoint, expected in cases: @@ -228,22 +351,33 @@ def test_compute_glows_flags_for_repoint(self): cdf_path = Path(tmp_dir, "l3d.cdf") with CDF(str(cdf_path), create=True) as cdf: cdf["epoch"] = epochs - cdf.new("glows_flags", data=flags, type=const.CDF_UINT2, recVary=True) + cdf.new( + "glows_flags", + data=flags, + type=const.CDF_UINT2, + recVary=True, + ) actual = compute_glows_flags_for_repoint(cdf_path, repoint_midpoint) self.assertIsInstance(actual, int) self.assertEqual(expected, actual) - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.get_lo_pivot_angle_from_l1b_file') - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access') - def test_get_lo_pivot_angles(self, mock_imap_data_access, mock_get_pivot_angle_from_file): + @patch( + "imap_l3_processing.glows.l3e.glows_l3e_utils.get_lo_pivot_angle_from_l1b_file" + ) + @patch("imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access") + def test_get_lo_pivot_angles( + self, mock_imap_data_access, mock_get_pivot_angle_from_file + ): available_repointings = [1, 2, 3, 4, 5, 6] mock_imap_data_access.query.return_value = [ - {'file_path': f'file{i}.cdf', 'repointing': i} + {"file_path": f"file{i}.cdf", "repointing": i} for i in available_repointings ] - mock_imap_data_access.download.side_effect = lambda name: Path("local/path/to", name) + mock_imap_data_access.download.side_effect = lambda name: Path( + "local/path/to", name + ) pivot_angles_by_file_path = { Path("local/path/to/file1.cdf"): 25, Path("local/path/to/file2.cdf"): 75, @@ -266,17 +400,22 @@ def mock_read_from_cdf(path: Path): descriptor="nhk", version="latest", ) - mock_imap_data_access.download.assert_has_calls([ - call("file3.cdf"), - call("file4.cdf"), - call("file6.cdf"), - ]) - self.assertEqual({ - 3: LoPivotAngle(parent_filename="file3.cdf", pivot_angle=105), - 4: LoPivotAngle(parent_filename="file4.cdf", pivot_angle=90), - 6: LoPivotAngle(parent_filename="file6.cdf", pivot_angle=84), - 10: LoPivotAngle(parent_filename=None, pivot_angle=90), - }, result) + mock_imap_data_access.download.assert_has_calls( + [ + call("file3.cdf"), + call("file4.cdf"), + call("file6.cdf"), + ] + ) + self.assertEqual( + { + 3: LoPivotAngle(parent_filename="file3.cdf", pivot_angle=105), + 4: LoPivotAngle(parent_filename="file4.cdf", pivot_angle=90), + 6: LoPivotAngle(parent_filename="file6.cdf", pivot_angle=84), + 10: LoPivotAngle(parent_filename=None, pivot_angle=90), + }, + result, + ) def test_get_repoint_numbers_within_cr_window(self): start_cr = 2093 @@ -288,7 +427,9 @@ def test_get_repoint_numbers_within_cr_window(self): set_global_repoint_table_paths([repointing_path]) repointing_data = get_repoint_data() - actual_repoint_numbers = get_repoint_numbers_within_cr_window(start_cr, end_cr, repointing_data) + actual_repoint_numbers = get_repoint_numbers_within_cr_window( + start_cr, end_cr, repointing_data + ) np.testing.assert_array_equal(actual_repoint_numbers, expected_repoint_numbers) @@ -301,17 +442,24 @@ def test_get_repoint_numbers_within_cr_window_returns_empty_for_none_start(self) set_global_repoint_table_paths([repointing_path]) repointing_data = get_repoint_data() - actual_repoint_numbers = get_repoint_numbers_within_cr_window(start_cr, end_cr, repointing_data) + actual_repoint_numbers = get_repoint_numbers_within_cr_window( + start_cr, end_cr, repointing_data + ) np.testing.assert_array_equal(actual_repoint_numbers, []) - def test_calculate_energy_deltas(self): centers = np.array([10, 1000, 100000]) expected_energy_delta_minus = np.array([9, 900, 90000]) expected_energy_delta_plus = np.array([90, 9000, 900000]) - actual_energy_delta_plus, actual_energy_delta_minus = calculate_energy_deltas(centers) + actual_energy_delta_plus, actual_energy_delta_minus = calculate_energy_deltas( + centers + ) - np.testing.assert_array_equal(actual_energy_delta_plus, expected_energy_delta_plus) - np.testing.assert_array_equal(actual_energy_delta_minus, expected_energy_delta_minus) \ No newline at end of file + np.testing.assert_array_equal( + actual_energy_delta_plus, expected_energy_delta_plus + ) + np.testing.assert_array_equal( + actual_energy_delta_minus, expected_energy_delta_minus + ) diff --git a/tests/glows/l3e/test_reprocess_info.py b/tests/glows/l3e/test_reprocess_info.py index 9e3287825..6e880071d 100644 --- a/tests/glows/l3e/test_reprocess_info.py +++ b/tests/glows/l3e/test_reprocess_info.py @@ -4,12 +4,27 @@ from unittest.mock import patch from imap_data_access import ProcessingInputCollection, AncillaryInput, RepointInput -from imap_processing.spice.repoint import set_global_repoint_table_paths, get_repoint_data - -from imap_l3_processing.glows.descriptors import GLOWS_L3E_HI_45_DESCRIPTOR, GLOWS_L3E_HI_90_DESCRIPTOR, \ - GLOWS_L3E_LO_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, GLOWS_L3E_ULTRA_HF_DESCRIPTOR, GLOWS_L3B_DESCRIPTOR, \ - GLOWS_L3C_DESCRIPTOR, GLOWS_L3D_DESCRIPTOR, GLOWS_REPROCESSING_DESCRIPTOR -from imap_l3_processing.glows.l3e.reprocess_info import ReprocessInfo, ReprocessTargets, fetch_reprocess_info +from imap_processing.spice.repoint import ( + set_global_repoint_table_paths, + get_repoint_data, +) + +from imap_l3_processing.glows.descriptors import ( + GLOWS_L3E_HI_45_DESCRIPTOR, + GLOWS_L3E_HI_90_DESCRIPTOR, + GLOWS_L3E_LO_DESCRIPTOR, + GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + GLOWS_L3E_ULTRA_HF_DESCRIPTOR, + GLOWS_L3B_DESCRIPTOR, + GLOWS_L3C_DESCRIPTOR, + GLOWS_L3D_DESCRIPTOR, + GLOWS_REPROCESSING_DESCRIPTOR, +) +from imap_l3_processing.glows.l3e.reprocess_info import ( + ReprocessInfo, + ReprocessTargets, + fetch_reprocess_info, +) from tests.test_helpers import get_test_data_folder, get_test_data_path @@ -27,7 +42,9 @@ def test_parse_from_ancillary(self, mock_datetime): test_file: Path = ( get_test_data_folder() / "glows" / "glows_reprocessing_ancillary_file.dat" ) - mock_datetime.now.return_value = datetime(2026, 8, 11, 11, 40, tzinfo=timezone.utc) + mock_datetime.now.return_value = datetime( + 2026, 8, 11, 11, 40, tzinfo=timezone.utc + ) cases = [ (datetime(2026, 8, 11, 11, 41, tzinfo=timezone.utc), products), (datetime(2026, 8, 11, 11, 41), products), @@ -40,25 +57,69 @@ def test_parse_from_ancillary(self, mock_datetime): reprocess_info = ReprocessInfo.parse_from_ancillary(test_file) - self.assertEqual(expected_products, reprocess_info.products_to_reprocess) + self.assertEqual( + expected_products, reprocess_info.products_to_reprocess + ) mock_datetime.fromisoformat.assert_called_with("2226-08-11 11:40") mock_datetime.now.assert_called_with(timezone.utc) def test_should_reprocess_l3d_if_l3d_or_l3e_products_are_specified(self): cases = [ - ("should reprocess l3e hi45", - {GLOWS_L3E_HI_45_DESCRIPTOR: ReprocessTargets([], []), GLOWS_L3B_DESCRIPTOR: ReprocessTargets([], [])}, True), - ("should reprocess l3e hi90", - {GLOWS_L3E_HI_90_DESCRIPTOR: ReprocessTargets([], []), GLOWS_L3B_DESCRIPTOR: ReprocessTargets([], [])}, True), - ("should reprocess l3e lo", - {GLOWS_L3E_LO_DESCRIPTOR: ReprocessTargets([], []), GLOWS_L3B_DESCRIPTOR: ReprocessTargets([], [])}, True), - ("should reprocess l3e ultra sf", - {GLOWS_L3E_ULTRA_SF_DESCRIPTOR: ReprocessTargets([], []), GLOWS_L3B_DESCRIPTOR: ReprocessTargets([], [])}, True), - ("should reprocess l3e ultra hf", - {GLOWS_L3E_ULTRA_HF_DESCRIPTOR: ReprocessTargets([], []), GLOWS_L3B_DESCRIPTOR: ReprocessTargets([], [])}, True), - ("should not reprocess l3b alone", {GLOWS_L3B_DESCRIPTOR: ReprocessTargets([], [])}, False), - ("should not reprocess l3c alone", {GLOWS_L3C_DESCRIPTOR: ReprocessTargets([], [])}, False), - ("should reprocess l3d alone", {GLOWS_L3D_DESCRIPTOR: ReprocessTargets([], [])}, True), + ( + "should reprocess l3e hi45", + { + GLOWS_L3E_HI_45_DESCRIPTOR: ReprocessTargets([], []), + GLOWS_L3B_DESCRIPTOR: ReprocessTargets([], []), + }, + True, + ), + ( + "should reprocess l3e hi90", + { + GLOWS_L3E_HI_90_DESCRIPTOR: ReprocessTargets([], []), + GLOWS_L3B_DESCRIPTOR: ReprocessTargets([], []), + }, + True, + ), + ( + "should reprocess l3e lo", + { + GLOWS_L3E_LO_DESCRIPTOR: ReprocessTargets([], []), + GLOWS_L3B_DESCRIPTOR: ReprocessTargets([], []), + }, + True, + ), + ( + "should reprocess l3e ultra sf", + { + GLOWS_L3E_ULTRA_SF_DESCRIPTOR: ReprocessTargets([], []), + GLOWS_L3B_DESCRIPTOR: ReprocessTargets([], []), + }, + True, + ), + ( + "should reprocess l3e ultra hf", + { + GLOWS_L3E_ULTRA_HF_DESCRIPTOR: ReprocessTargets([], []), + GLOWS_L3B_DESCRIPTOR: ReprocessTargets([], []), + }, + True, + ), + ( + "should not reprocess l3b alone", + {GLOWS_L3B_DESCRIPTOR: ReprocessTargets([], [])}, + False, + ), + ( + "should not reprocess l3c alone", + {GLOWS_L3C_DESCRIPTOR: ReprocessTargets([], [])}, + False, + ), + ( + "should reprocess l3d alone", + {GLOWS_L3D_DESCRIPTOR: ReprocessTargets([], [])}, + True, + ), ] for case, products_to_reprocess, expected_result in cases: @@ -94,25 +155,35 @@ def test_get_repoints(self): repoints_for_cr2093 = list(range(3682, 3696)) repoints_for_cr2094 = list(range(3710, 3724)) - expected_repoints_lo = set([2045, 2046] + repoints_for_cr2093 + [3700] + repoints_for_cr2094) + expected_repoints_lo = set( + [2045, 2046] + repoints_for_cr2093 + [3700] + repoints_for_cr2094 + ) expected_repoints_hi = {2050} expected_repoints_ultra = set() self.assertEqual(expected_repoints_lo, repoints_for_lo) self.assertEqual(expected_repoints_hi, repoints_for_hi) self.assertEqual(expected_repoints_ultra, repoints_for_ultra) - @patch('imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access.download') - def test_fetch_reprocess_info_downloads_and_parses_reprocess_info(self, mock_download): + @patch("imap_l3_processing.glows.l3e.glows_l3e_utils.imap_data_access.download") + def test_fetch_reprocess_info_downloads_and_parses_reprocess_info( + self, mock_download + ): processing_input = ProcessingInputCollection( - AncillaryInput(f"imap_glows_{GLOWS_REPROCESSING_DESCRIPTOR}_20250101_v000.dat"), + AncillaryInput( + f"imap_glows_{GLOWS_REPROCESSING_DESCRIPTOR}_20250101_v000.dat" + ), RepointInput("imap_2026_269_12.repoint.csv"), ) - path_to_downloaded_ancillary = get_test_data_folder() / 'glows' / 'glows_reprocessing_ancillary_file.dat' + path_to_downloaded_ancillary = ( + get_test_data_folder() / "glows" / "glows_reprocessing_ancillary_file.dat" + ) mock_download.return_value = path_to_downloaded_ancillary reprocess_info = fetch_reprocess_info(processing_input) - mock_download.assert_called_with(f'imap_glows_{GLOWS_REPROCESSING_DESCRIPTOR}_20250101_v000.dat') + mock_download.assert_called_with( + f"imap_glows_{GLOWS_REPROCESSING_DESCRIPTOR}_20250101_v000.dat" + ) expected_products_to_reprocess = { GLOWS_L3E_LO_DESCRIPTOR: ReprocessTargets([245, 246], [2310]), @@ -122,4 +193,6 @@ def test_fetch_reprocess_info_downloads_and_parses_reprocess_info(self, mock_dow GLOWS_L3E_ULTRA_HF_DESCRIPTOR: ReprocessTargets([243], []), GLOWS_L3D_DESCRIPTOR: ReprocessTargets([], []), } - self.assertEqual(expected_products_to_reprocess, reprocess_info.products_to_reprocess) + self.assertEqual( + expected_products_to_reprocess, reprocess_info.products_to_reprocess + ) diff --git a/tests/glows/test_glows_processor.py b/tests/glows/test_glows_processor.py index 2035c5016..f1c442350 100644 --- a/tests/glows/test_glows_processor.py +++ b/tests/glows/test_glows_processor.py @@ -21,38 +21,73 @@ from imap_l3_processing.constants import TEMP_CDF_FOLDER_PATH from imap_l3_processing.glows import l3d -from imap_l3_processing.glows.descriptors import GLOWS_L3A_DESCRIPTOR, GLOWS_L3E_ULTRA_SF_DESCRIPTOR, \ - GLOWS_L3E_ULTRA_HF_DESCRIPTOR, GLOWS_L3B_DESCRIPTOR, GLOWS_L3C_DESCRIPTOR, GLOWS_L3D_DESCRIPTOR, \ - GLOWS_L3E_HI_45_DESCRIPTOR, GLOWS_L3E_HI_90_DESCRIPTOR, GLOWS_L3E_LO_DESCRIPTOR -from imap_l3_processing.glows.glows_processor import GlowsProcessor, process_l3d, process_l3e, process_l3bc, \ - process_l3e_ul_sf, process_l3e_hi, process_l3e_lo, process_l3e_ul_hf -from imap_l3_processing.glows.l3a.utils import create_glows_l3a_dictionary_from_cdf, create_glows_l3a_from_dictionary -from imap_l3_processing.glows.l3bc.cannot_process_carrington_rotation_error import CannotProcessCarringtonRotationError +from imap_l3_processing.glows.descriptors import ( + GLOWS_L3A_DESCRIPTOR, + GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + GLOWS_L3E_ULTRA_HF_DESCRIPTOR, + GLOWS_L3B_DESCRIPTOR, + GLOWS_L3C_DESCRIPTOR, + GLOWS_L3D_DESCRIPTOR, + GLOWS_L3E_HI_45_DESCRIPTOR, + GLOWS_L3E_HI_90_DESCRIPTOR, + GLOWS_L3E_LO_DESCRIPTOR, +) +from imap_l3_processing.glows.glows_processor import ( + GlowsProcessor, + process_l3d, + process_l3e, + process_l3bc, + process_l3e_ul_sf, + process_l3e_hi, + process_l3e_lo, + process_l3e_ul_hf, +) +from imap_l3_processing.glows.l3a.utils import ( + create_glows_l3a_dictionary_from_cdf, + create_glows_l3a_from_dictionary, +) +from imap_l3_processing.glows.l3bc.cannot_process_carrington_rotation_error import ( + CannotProcessCarringtonRotationError, +) from imap_l3_processing.glows.l3bc.glows_l3bc_dependencies import GlowsL3BCDependencies -from imap_l3_processing.glows.l3bc.glows_l3bc_initializer import GlowsL3BCInitializerData +from imap_l3_processing.glows.l3bc.glows_l3bc_initializer import ( + GlowsL3BCInitializerData, +) from imap_l3_processing.glows.l3bc.models import ExternalDependencies from imap_l3_processing.glows.l3d.glows_l3d_dependencies import GlowsL3DDependencies from imap_l3_processing.glows.l3d.models import GlowsL3DProcessorOutput from imap_l3_processing.glows.l3d.utils import PATH_TO_L3D_TOOLKIT -from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import GlowsL3eCallArguments, GlowsL3eSpacecraftInfo +from imap_l3_processing.glows.l3e.glows_l3e_call_arguments import ( + GlowsL3eCallArguments, + GlowsL3eSpacecraftInfo, +) from imap_l3_processing.glows.l3e.glows_l3e_dependencies import GlowsL3EDependencies from imap_l3_processing.glows.l3e.glows_l3e_hi_model import GlowsL3EHiData from imap_l3_processing.glows.l3e.glows_l3e_initializer import GlowsL3EInitializerOutput from imap_l3_processing.glows.l3e.glows_l3e_lo_model import GlowsL3ELoData from imap_l3_processing.glows.l3e.glows_l3e_ultra_model import GlowsL3EUltraData -from imap_l3_processing.glows.l3e.glows_l3e_utils import GlowsL3eVersionsForRepointings, LoPivotAngle +from imap_l3_processing.glows.l3e.glows_l3e_utils import ( + GlowsL3eVersionsForRepointings, + LoPivotAngle, +) from imap_l3_processing.glows.quality_flags import GlowsL3Flags from imap_l3_processing.models import InputMetadata, VersionMap from imap_l3_processing.utils import save_data -from tests.test_helpers import get_test_instrument_team_data_path, get_test_data_path, get_test_data_folder, \ - assert_dataclass_fields +from tests.test_helpers import ( + get_test_instrument_team_data_path, + get_test_data_path, + get_test_data_folder, + assert_dataclass_fields, +) -MODULE = 'imap_l3_processing.glows.glows_processor' +MODULE = "imap_l3_processing.glows.glows_processor" -class TestGlowsProcessor(unittest.TestCase): +class TestGlowsProcessor(unittest.TestCase): def setUp(self): - self.l3bc_initializer_patcher = patch(f"{MODULE}.GlowsL3BCInitializer.get_crs_to_process") + self.l3bc_initializer_patcher = patch( + f"{MODULE}.GlowsL3BCInitializer.get_crs_to_process" + ) self.mock_l3bc_initializer = self.l3bc_initializer_patcher.start() self.mock_external_deps = Mock() @@ -70,19 +105,21 @@ def setUp(self): self.mock_l3e_initializer_patcher = patch(f"{MODULE}.GlowsL3EInitializer") self.mock_l3e_initializer = self.mock_l3e_initializer_patcher.start() - self.mock_l3e_initializer.get_repointings_to_process.return_value = GlowsL3EInitializerOutput( - dependencies=Mock(), - repointings=GlowsL3eVersionsForRepointings( - repointing_numbers=[], - hi_90_repointings={}, - hi_45_repointings={}, - lo_repointings={}, - ultra_sf_repointings={}, - ultra_hf_repointings={}, - ), - l3d_cdf_path=Path("path/to/l3d.cdf"), - metakernel_with_predict_ephem=Mock(), - metakernel_without_predict_ephem=Mock(), + self.mock_l3e_initializer.get_repointings_to_process.return_value = ( + GlowsL3EInitializerOutput( + dependencies=Mock(), + repointings=GlowsL3eVersionsForRepointings( + repointing_numbers=[], + hi_90_repointings={}, + hi_45_repointings={}, + lo_repointings={}, + ultra_sf_repointings={}, + ultra_hf_repointings={}, + ), + l3d_cdf_path=Path("path/to/l3d.cdf"), + metakernel_with_predict_ephem=Mock(), + metakernel_without_predict_ephem=Mock(), + ) ) self.fetch_reprocess_info_patcher = patch(f"{MODULE}.fetch_reprocess_info") @@ -95,39 +132,64 @@ def tearDown(self): self.mock_l3e_initializer_patcher.stop() self.fetch_reprocess_info_patcher.stop() - if os.path.exists(PATH_TO_L3D_TOOLKIT / 'data_l3b'): shutil.rmtree(PATH_TO_L3D_TOOLKIT / 'data_l3b') - if os.path.exists(PATH_TO_L3D_TOOLKIT / 'data_l3c'): shutil.rmtree(PATH_TO_L3D_TOOLKIT / 'data_l3c') - if os.path.exists(PATH_TO_L3D_TOOLKIT / 'data_l3d'): shutil.rmtree(PATH_TO_L3D_TOOLKIT / 'data_l3d') - if os.path.exists(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt'): shutil.rmtree(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt') + if os.path.exists(PATH_TO_L3D_TOOLKIT / "data_l3b"): + shutil.rmtree(PATH_TO_L3D_TOOLKIT / "data_l3b") + if os.path.exists(PATH_TO_L3D_TOOLKIT / "data_l3c"): + shutil.rmtree(PATH_TO_L3D_TOOLKIT / "data_l3c") + if os.path.exists(PATH_TO_L3D_TOOLKIT / "data_l3d"): + shutil.rmtree(PATH_TO_L3D_TOOLKIT / "data_l3d") + if os.path.exists(PATH_TO_L3D_TOOLKIT / "data_l3d_txt"): + shutil.rmtree(PATH_TO_L3D_TOOLKIT / "data_l3d_txt") @patch("imap_l3_processing.glows.glows_processor.GlowsL3BCInitializer") - @patch('imap_l3_processing.glows.glows_processor.GlowsL3ADependencies') - @patch('imap_l3_processing.glows.glows_processor.L3aData') - @patch('imap_l3_processing.glows.glows_processor.save_data') - @patch('imap_l3_processing.utils.spiceypy') - def test_processor_handles_l3a(self, mock_spiceypy, mock_save_data, mock_l3a_data, - mock_glows_dependencies_class, mock_glows_initializer): + @patch("imap_l3_processing.glows.glows_processor.GlowsL3ADependencies") + @patch("imap_l3_processing.glows.glows_processor.L3aData") + @patch("imap_l3_processing.glows.glows_processor.save_data") + @patch("imap_l3_processing.utils.spiceypy") + def test_processor_handles_l3a( + self, + mock_spiceypy, + mock_save_data, + mock_l3a_data, + mock_glows_dependencies_class, + mock_glows_initializer, + ): mock_spiceypy.ktotal.return_value = 0 - instrument = 'glows' + instrument = "glows" start_date = datetime(2024, 10, 7, 10, 00, 00) end_date = datetime(2024, 10, 8, 10, 00, 00) outgoing_data_level = "l3a" - outgoing_version = 'v002' + outgoing_version = "v002" - mock_fetched_dependencies = mock_glows_dependencies_class.fetch_dependencies.return_value - mock_fetched_dependencies.ancillary_files = {"settings": get_test_instrument_team_data_path( - "glows/imap_glows_pipeline-settings_20100101_v001.json")} - l3a_json_path = get_test_data_folder() / "glows" / "imap_glows_l3a_20130908085214_orbX_modX_p_v00.json" + mock_fetched_dependencies = ( + mock_glows_dependencies_class.fetch_dependencies.return_value + ) + mock_fetched_dependencies.ancillary_files = { + "settings": get_test_instrument_team_data_path( + "glows/imap_glows_pipeline-settings_20100101_v001.json" + ) + } + l3a_json_path = ( + get_test_data_folder() + / "glows" + / "imap_glows_l3a_20130908085214_orbX_modX_p_v00.json" + ) with open(l3a_json_path) as f: mock_l3a_data.return_value.data = json.load(f) mock_cdf_path = mock_save_data.return_value - input_metadata = InputMetadata(instrument, outgoing_data_level, start_date, end_date, - outgoing_version, repointing=5) + input_metadata = InputMetadata( + instrument, + outgoing_data_level, + start_date, + end_date, + outgoing_version, + repointing=5, + ) mock_processing_input_collection = Mock() parent_file_path = Path("test/parent_path") - expected_parent_file_names = ['parent_path'] + expected_parent_file_names = ["parent_path"] expected_global_metadata_attrs = { "flight_software_version": 131848, "ground_software_version": "0.3", @@ -142,52 +204,76 @@ def test_processor_handles_l3a(self, mock_spiceypy, mock_save_data, mock_l3a_dat "data_ancillary/imap_glows_suspected_transients_v002.dat", "data_ancillary/imap_glows_map_of_extra_helio_bckgrd_v001.dat", "data_ancillary/imap_glows_time_dep_bckgrd_v001.dat", - "data_ancillary/imap_l1_anc_sc_Merged_2010-2030_mockByGlowsTeam001.csv" + "data_ancillary/imap_l1_anc_sc_Merged_2010-2030_mockByGlowsTeam001.csv", ], - "l2_input_file_name": "data_l2_histograms/imap_glows_l2_20130908085214_orbX_modX_p_v00.json" + "l2_input_file_name": "data_l2_histograms/imap_glows_l2_20130908085214_orbX_modX_p_v00.json", } - mock_processing_input_collection.get_file_paths.return_value = [parent_file_path] + mock_processing_input_collection.get_file_paths.return_value = [ + parent_file_path + ] - processor = GlowsProcessor(dependencies=mock_processing_input_collection, input_metadata=input_metadata) + processor = GlowsProcessor( + dependencies=mock_processing_input_collection, input_metadata=input_metadata + ) products = processor.process() mock_glows_initializer.assert_not_called() - mock_glows_dependencies_class.fetch_dependencies.assert_called_with(mock_processing_input_collection) + mock_glows_dependencies_class.fetch_dependencies.assert_called_with( + mock_processing_input_collection + ) expected_data_to_save = create_glows_l3a_from_dictionary( - mock_l3a_data.return_value.data, replace(input_metadata, descriptor=GLOWS_L3A_DESCRIPTOR)) + mock_l3a_data.return_value.data, + replace(input_metadata, descriptor=GLOWS_L3A_DESCRIPTOR), + ) expected_data_to_save.parent_file_names = expected_parent_file_names actual_data = mock_save_data.call_args.args[0] self.assertEqual(expected_parent_file_names, actual_data.parent_file_names) - self.assertEqual(expected_global_metadata_attrs, actual_data.global_metadata_attrs) - assert_dataclass_fields(expected_data_to_save, actual_data, omit=["global_metadata_attrs"]) + self.assertEqual( + expected_global_metadata_attrs, actual_data.global_metadata_attrs + ) + assert_dataclass_fields( + expected_data_to_save, actual_data, omit=["global_metadata_attrs"] + ) self.assertEqual([mock_cdf_path], products) - @patch('imap_l3_processing.glows.glows_processor.GlowsL3ADependencies') - @patch('imap_l3_processing.glows.glows_processor.save_data') + @patch("imap_l3_processing.glows.glows_processor.GlowsL3ADependencies") + @patch("imap_l3_processing.glows.glows_processor.save_data") def test_does_not_save_cdf_if_process_l3a_returns_none(self, mock_save_data, _): - instrument = 'glows' + instrument = "glows" start_date = datetime(2024, 10, 7, 10, 00, 00) end_date = datetime(2024, 10, 8, 10, 00, 00) outgoing_data_level = "l3a" - outgoing_version = 'v002' - input_metadata = InputMetadata(instrument, outgoing_data_level, start_date, end_date, - outgoing_version, repointing=5) + outgoing_version = "v002" + input_metadata = InputMetadata( + instrument, + outgoing_data_level, + start_date, + end_date, + outgoing_version, + repointing=5, + ) mock_processing_input_collection = Mock() - processor = GlowsProcessor(dependencies=mock_processing_input_collection, input_metadata=input_metadata) + processor = GlowsProcessor( + dependencies=mock_processing_input_collection, input_metadata=input_metadata + ) processor.process_l3a = Mock(return_value=None) products = processor.process() mock_save_data.assert_not_called() self.assertEqual([], products) - @patch('imap_l3_processing.glows.glows_processor.create_glows_l3a_from_dictionary') - @patch('imap_l3_processing.glows.glows_processor.L3aData') + @patch("imap_l3_processing.glows.glows_processor.create_glows_l3a_from_dictionary") + @patch("imap_l3_processing.glows.glows_processor.L3aData") def test_process_l3a(self, l3a_data_constructor, create_glows_l3a_from_dictionary): - input_metadata = InputMetadata('glows', "l3a", datetime(2024, 10, 7, 10, 00, 00), - datetime(2024, 10, 8, 10, 00, 00), - 'v02') + input_metadata = InputMetadata( + "glows", + "l3a", + datetime(2024, 10, 7, 10, 00, 00), + datetime(2024, 10, 8, 10, 00, 00), + "v02", + ) processor = GlowsProcessor(dependencies=Mock(), input_metadata=input_metadata) @@ -195,40 +281,44 @@ def test_process_l3a(self, l3a_data_constructor, create_glows_l3a_from_dictionar fetched_dependencies = Mock() mock_l3a_data = l3a_data_constructor.return_value - mock_l3a_data.data = { - "daily_lightcurve": { - "exposure_times": np.array([1]) - } - } + mock_l3a_data.data = {"daily_lightcurve": {"exposure_times": np.array([1])}} result = processor.process_l3a(fetched_dependencies) self.assertIs(create_glows_l3a_from_dictionary.return_value, result) - l3a_data_constructor.assert_called_once_with(fetched_dependencies.ancillary_files) - mock_l3a_data.process_l2_data_file.assert_called_once_with(fetched_dependencies.data) + l3a_data_constructor.assert_called_once_with( + fetched_dependencies.ancillary_files + ) + mock_l3a_data.process_l2_data_file.assert_called_once_with( + fetched_dependencies.data + ) mock_l3a_data.generate_l3a_data.assert_called_once_with( - fetched_dependencies.ancillary_files) - processor.add_spin_angle_delta.assert_called_with(mock_l3a_data.data, - fetched_dependencies.ancillary_files) - create_glows_l3a_from_dictionary.assert_called_once_with(processor.add_spin_angle_delta.return_value, - replace(input_metadata, - descriptor=GLOWS_L3A_DESCRIPTOR)) - - @patch('imap_l3_processing.glows.glows_processor.create_glows_l3a_from_dictionary') - @patch('imap_l3_processing.glows.glows_processor.L3aData') - def test_process_l3a_returns_none_when_no_input_histrogram_bins_are_valid(self, l3a_data_constructor, - create_glows_l3a_from_dictionary): - input_metadata = InputMetadata('glows', "l3a", datetime(2024, 10, 7, 10, 00, 00), - datetime(2024, 10, 8, 10, 00, 00), - 'v02') + fetched_dependencies.ancillary_files + ) + processor.add_spin_angle_delta.assert_called_with( + mock_l3a_data.data, fetched_dependencies.ancillary_files + ) + create_glows_l3a_from_dictionary.assert_called_once_with( + processor.add_spin_angle_delta.return_value, + replace(input_metadata, descriptor=GLOWS_L3A_DESCRIPTOR), + ) + + @patch("imap_l3_processing.glows.glows_processor.create_glows_l3a_from_dictionary") + @patch("imap_l3_processing.glows.glows_processor.L3aData") + def test_process_l3a_returns_none_when_no_input_histrogram_bins_are_valid( + self, l3a_data_constructor, create_glows_l3a_from_dictionary + ): + input_metadata = InputMetadata( + "glows", + "l3a", + datetime(2024, 10, 7, 10, 00, 00), + datetime(2024, 10, 8, 10, 00, 00), + "v02", + ) processor = GlowsProcessor(dependencies=Mock(), input_metadata=input_metadata) mock_l3a_data = l3a_data_constructor.return_value - mock_l3a_data.data = { - "daily_lightcurve": { - "exposure_times": np.array([]) - } - } + mock_l3a_data.data = {"daily_lightcurve": {"exposure_times": np.array([])}} processor.add_spin_angle_delta = Mock() fetched_dependencies = Mock() @@ -236,39 +326,46 @@ def test_process_l3a_returns_none_when_no_input_histrogram_bins_are_valid(self, self.assertIsNone(result) - mock_l3a_data.process_l2_data_file.assert_called_once_with(fetched_dependencies.data) + mock_l3a_data.process_l2_data_file.assert_called_once_with( + fetched_dependencies.data + ) mock_l3a_data.generate_l3a_data.assert_called_once_with( - fetched_dependencies.ancillary_files) + fetched_dependencies.ancillary_files + ) processor.add_spin_angle_delta.assert_not_called() create_glows_l3a_from_dictionary.assert_not_called() def test_add_spin_angle_delta(self): - cases = [ - (60, 3), - (90, 2) - ] + cases = [(60, 3), (90, 2)] for bins, expected_delta in cases: with self.subTest(bin=bins, expected_delta=expected_delta): ancillary_files = {} - with open(get_test_data_path("glows/imap_glows_l3a_20130908085214_orbX_modX_p_v00.json")) as f: + with open( + get_test_data_path( + "glows/imap_glows_l3a_20130908085214_orbX_modX_p_v00.json" + ) + ) as f: example_data = json.load(f) with tempfile.TemporaryDirectory() as tempdir: temp_file_path = Path(tempdir) / "settings.json" example_settings = get_test_instrument_team_data_path( - "glows/imap_glows_pipeline-settings_20100101_v001.json") + "glows/imap_glows_pipeline-settings_20100101_v001.json" + ) with open(example_settings) as file: loaded_file = json.load(file) - loaded_file['l3a_nominal_number_of_bins'] = bins + loaded_file["l3a_nominal_number_of_bins"] = bins - with open(temp_file_path, 'w') as file: + with open(temp_file_path, "w") as file: json.dump(loaded_file, file) - ancillary_files['settings'] = temp_file_path + ancillary_files["settings"] = temp_file_path - result = GlowsProcessor.add_spin_angle_delta(deepcopy(example_data), ancillary_files) + result = GlowsProcessor.add_spin_angle_delta( + deepcopy(example_data), ancillary_files + ) for k, v in example_data.items(): if k != "daily_lightcurve": self.assertEqual(v, result[k]) @@ -276,73 +373,104 @@ def test_add_spin_angle_delta(self): for k2, v2 in example_data["daily_lightcurve"].items(): self.assertEqual(v2, result["daily_lightcurve"][k2]) spin_angle_delta = result["daily_lightcurve"]["spin_angle_delta"] - self.assertEqual(len(example_data["daily_lightcurve"]["spin_angle"]), len(spin_angle_delta)) + self.assertEqual( + len(example_data["daily_lightcurve"]["spin_angle"]), + len(spin_angle_delta), + ) self.assertTrue(np.all(spin_angle_delta == expected_delta)) @patch("imap_l3_processing.utils.spiceypy") @patch("imap_l3_processing.glows.glows_processor.save_data") - @patch('imap_l3_processing.glows.glows_processor.GlowsL3BIonizationRate') - @patch('imap_l3_processing.glows.glows_processor.GlowsL3CSolarWind') - @patch('imap_l3_processing.glows.glows_processor.filter_l3a_files') - @patch('imap_l3_processing.glows.glows_processor.generate_l3bc') - @patch('imap_l3_processing.glows.glows_processor.GlowsProcessor.archive_dependencies') - @patch('imap_l3_processing.glows.glows_processor.GlowsL3BCInitializer') - def test_process_l3bc(self, mock_glows_l3bc_initializer, mock_archive_dependencies, mock_generate_l3bc, - mock_filter_bad_days, - mock_l3c_model_class, mock_l3b_model_class, mock_save_data, - mock_spiceypy): + @patch("imap_l3_processing.glows.glows_processor.GlowsL3BIonizationRate") + @patch("imap_l3_processing.glows.glows_processor.GlowsL3CSolarWind") + @patch("imap_l3_processing.glows.glows_processor.filter_l3a_files") + @patch("imap_l3_processing.glows.glows_processor.generate_l3bc") + @patch( + "imap_l3_processing.glows.glows_processor.GlowsProcessor.archive_dependencies" + ) + @patch("imap_l3_processing.glows.glows_processor.GlowsL3BCInitializer") + def test_process_l3bc( + self, + mock_glows_l3bc_initializer, + mock_archive_dependencies, + mock_generate_l3bc, + mock_filter_bad_days, + mock_l3c_model_class, + mock_l3b_model_class, + mock_save_data, + mock_spiceypy, + ): mock_spiceypy.ktotal.return_value = 1 - mock_spiceypy.kdata.return_value = ['kernel_1', 'type', 'source', 'handle'] + mock_spiceypy.kdata.return_value = ["kernel_1", "type", "source", "handle"] first_cr_number_to_process = 2091 second_cr_number_to_process = 2092 - external_deps = ExternalDependencies(f107_index_file_path=sentinel.f107_index_file_path, - lyman_alpha_path=sentinel.lyman_alpha_path, - omni2_data_path=sentinel.omni2_data_path) + external_deps = ExternalDependencies( + f107_index_file_path=sentinel.f107_index_file_path, + lyman_alpha_path=sentinel.lyman_alpha_path, + omni2_data_path=sentinel.omni2_data_path, + ) mock_archive_dependencies.side_effect = [Path("path1.zip"), Path("path2.zip")] l3a_data_1 = {"filename": "l3a_file_1"} first_dependency_version = Version(1, 1) - first_dependency = GlowsL3BCDependencies(l3a_data=[l3a_data_1], - external_files=sentinel.external_files_1, - ancillary_files={ - 'bad_days_list': sentinel.bad_days_list_1, - }, - carrington_rotation_number=first_cr_number_to_process, - version=first_dependency_version, - start_date=Mock(), - end_date=Mock(), - repointing_file_path=sentinel.repointing_file_path) + first_dependency = GlowsL3BCDependencies( + l3a_data=[l3a_data_1], + external_files=sentinel.external_files_1, + ancillary_files={ + "bad_days_list": sentinel.bad_days_list_1, + }, + carrington_rotation_number=first_cr_number_to_process, + version=first_dependency_version, + start_date=Mock(), + end_date=Mock(), + repointing_file_path=sentinel.repointing_file_path, + ) l3a_data_2 = {"filename": "l3a_file_2"} second_dependency_version = Version(1, 2) - second_dependency = GlowsL3BCDependencies(l3a_data=[l3a_data_2], - external_files=sentinel.external_files_2, - ancillary_files={ - 'bad_days_list': sentinel.bad_days_list_2, - }, - carrington_rotation_number=second_cr_number_to_process, - version=second_dependency_version, - start_date=Mock(), - end_date=Mock(), - repointing_file_path=sentinel.repointing_file_path) - - mock_generate_l3bc.side_effect = [(sentinel.l3b_data_1, sentinel.l3c_data_1), - (sentinel.l3b_data_2, sentinel.l3c_data_2)] - mock_filter_bad_days.side_effect = [sentinel.filtered_days_1, sentinel.filtered_days_2] + second_dependency = GlowsL3BCDependencies( + l3a_data=[l3a_data_2], + external_files=sentinel.external_files_2, + ancillary_files={ + "bad_days_list": sentinel.bad_days_list_2, + }, + carrington_rotation_number=second_cr_number_to_process, + version=second_dependency_version, + start_date=Mock(), + end_date=Mock(), + repointing_file_path=sentinel.repointing_file_path, + ) + + mock_generate_l3bc.side_effect = [ + (sentinel.l3b_data_1, sentinel.l3c_data_1), + (sentinel.l3b_data_2, sentinel.l3c_data_2), + ] + mock_filter_bad_days.side_effect = [ + sentinel.filtered_days_1, + sentinel.filtered_days_2, + ] l3b_model_1 = Mock(parent_file_names=["file1"]) l3b_model_2 = Mock(parent_file_names=["file2"]) l3c_model_1 = Mock(parent_file_names=["file3"]) l3c_model_2 = Mock(parent_file_names=["file4"]) - mock_l3b_model_class.from_instrument_team_dictionary.side_effect = [l3b_model_1, l3b_model_2] - mock_l3c_model_class.from_instrument_team_dictionary.side_effect = [l3c_model_1, l3c_model_2] - mock_save_data.side_effect = [Path("path/to/l3b_file_1.cdf"), - Path("path/to/l3c_file_1.cdf"), - Path("path/to/l3b_file_2.cdf"), - Path("path/to/l3c_file_2.cdf")] + mock_l3b_model_class.from_instrument_team_dictionary.side_effect = [ + l3b_model_1, + l3b_model_2, + ] + mock_l3c_model_class.from_instrument_team_dictionary.side_effect = [ + l3c_model_1, + l3c_model_2, + ] + mock_save_data.side_effect = [ + Path("path/to/l3b_file_1.cdf"), + Path("path/to/l3c_file_1.cdf"), + Path("path/to/l3b_file_2.cdf"), + Path("path/to/l3c_file_2.cdf"), + ] input_major_version = 2 input_version = VersionMap( { @@ -356,119 +484,211 @@ def test_process_l3bc(self, mock_glows_l3bc_initializer, mock_archive_dependenci GLOWS_L3E_ULTRA_HF_DESCRIPTOR: Version(5, 1), } ) - input_metadata = InputMetadata('glows', "l3b", datetime(2024, 10, 7, 10, 00, 00), - datetime(2024, 10, 8, 10, 00, 00), - version=input_version) + input_metadata = InputMetadata( + "glows", + "l3b", + datetime(2024, 10, 7, 10, 00, 00), + datetime(2024, 10, 8, 10, 00, 00), + version=input_version, + ) - mock_glows_l3bc_initializer.get_crs_to_process.return_value = GlowsL3BCInitializerData( - external_dependencies=external_deps, - l3bc_dependencies=[first_dependency, second_dependency], - l3bs_by_cr={}, - l3cs_by_cr={}, - repoint_file_path=sentinel.repoint_file_path + mock_glows_l3bc_initializer.get_crs_to_process.return_value = ( + GlowsL3BCInitializerData( + external_dependencies=external_deps, + l3bc_dependencies=[first_dependency, second_dependency], + l3bs_by_cr={}, + l3cs_by_cr={}, + repoint_file_path=sentinel.repoint_file_path, + ) ) - processor = GlowsProcessor(dependencies=sentinel.dependencies, input_metadata=input_metadata) + processor = GlowsProcessor( + dependencies=sentinel.dependencies, input_metadata=input_metadata + ) products = processor.process() - self.assertEqual([Path("path/to/l3b_file_1.cdf"), - Path("path/to/l3c_file_1.cdf"), - Path("path1.zip"), - Path("path/to/l3b_file_2.cdf"), - Path("path/to/l3c_file_2.cdf"), - Path("path2.zip") - ], products) + self.assertEqual( + [ + Path("path/to/l3b_file_1.cdf"), + Path("path/to/l3c_file_1.cdf"), + Path("path1.zip"), + Path("path/to/l3b_file_2.cdf"), + Path("path/to/l3c_file_2.cdf"), + Path("path2.zip"), + ], + products, + ) - dependencies_with_filtered_list_1 = replace(first_dependency, l3a_data=sentinel.filtered_days_1) - dependencies_with_filtered_list_2 = replace(second_dependency, l3a_data=sentinel.filtered_days_2) + dependencies_with_filtered_list_1 = replace( + first_dependency, l3a_data=sentinel.filtered_days_1 + ) + dependencies_with_filtered_list_2 = replace( + second_dependency, l3a_data=sentinel.filtered_days_2 + ) mock_filter_bad_days.assert_has_calls( - [call([l3a_data_1], sentinel.bad_days_list_1, first_cr_number_to_process), - call([l3a_data_2], sentinel.bad_days_list_2, second_cr_number_to_process)]) + [ + call( + [l3a_data_1], sentinel.bad_days_list_1, first_cr_number_to_process + ), + call( + [l3a_data_2], sentinel.bad_days_list_2, second_cr_number_to_process + ), + ] + ) mock_generate_l3bc.assert_has_calls( - [call(dependencies_with_filtered_list_1), call(dependencies_with_filtered_list_2)]) - - mock_glows_l3bc_initializer.get_crs_to_process.assert_called_once_with(sentinel.dependencies, - input_major_version) - - expected_l3b_metadata_1 = InputMetadata("glows", "l3b", first_dependency.start_date, - first_dependency.end_date, - VersionMap({GLOWS_L3B_DESCRIPTOR: first_dependency_version}), - "ion-rate-profile") - expected_l3b_metadata_2 = InputMetadata("glows", "l3b", second_dependency.start_date, - second_dependency.end_date, - VersionMap({GLOWS_L3B_DESCRIPTOR: second_dependency_version}), - "ion-rate-profile") - expected_l3c_metadata_1 = InputMetadata("glows", "l3c", first_dependency.start_date, - first_dependency.end_date, - VersionMap({GLOWS_L3C_DESCRIPTOR: first_dependency_version}), - "sw-profile") - expected_l3c_metadata_2 = InputMetadata("glows", "l3c", second_dependency.start_date, - second_dependency.end_date, - VersionMap({GLOWS_L3C_DESCRIPTOR: second_dependency_version}), - "sw-profile") + [ + call(dependencies_with_filtered_list_1), + call(dependencies_with_filtered_list_2), + ] + ) + + mock_glows_l3bc_initializer.get_crs_to_process.assert_called_once_with( + sentinel.dependencies, input_major_version + ) + + expected_l3b_metadata_1 = InputMetadata( + "glows", + "l3b", + first_dependency.start_date, + first_dependency.end_date, + VersionMap({GLOWS_L3B_DESCRIPTOR: first_dependency_version}), + "ion-rate-profile", + ) + expected_l3b_metadata_2 = InputMetadata( + "glows", + "l3b", + second_dependency.start_date, + second_dependency.end_date, + VersionMap({GLOWS_L3B_DESCRIPTOR: second_dependency_version}), + "ion-rate-profile", + ) + expected_l3c_metadata_1 = InputMetadata( + "glows", + "l3c", + first_dependency.start_date, + first_dependency.end_date, + VersionMap({GLOWS_L3C_DESCRIPTOR: first_dependency_version}), + "sw-profile", + ) + expected_l3c_metadata_2 = InputMetadata( + "glows", + "l3c", + second_dependency.start_date, + second_dependency.end_date, + VersionMap({GLOWS_L3C_DESCRIPTOR: second_dependency_version}), + "sw-profile", + ) mock_l3b_model_class.from_instrument_team_dictionary.assert_has_calls( - [call(sentinel.l3b_data_1, expected_l3b_metadata_1), - call(sentinel.l3b_data_2, expected_l3b_metadata_2)]) + [ + call(sentinel.l3b_data_1, expected_l3b_metadata_1), + call(sentinel.l3b_data_2, expected_l3b_metadata_2), + ] + ) mock_l3c_model_class.from_instrument_team_dictionary.assert_has_calls( - [call(sentinel.l3c_data_1, expected_l3c_metadata_1), - call(sentinel.l3c_data_2, expected_l3c_metadata_2)]) - - mock_save_data.assert_has_calls([ - call(l3b_model_1, cr_number=first_cr_number_to_process), - call(l3c_model_1, cr_number=first_cr_number_to_process), - call(l3b_model_2, cr_number=second_cr_number_to_process), - call(l3c_model_2, cr_number=second_cr_number_to_process) - ]) - - mock_archive_dependencies.assert_has_calls([ - call(l3bc_deps=first_dependency, external_dependencies=external_deps), - call(l3bc_deps=second_dependency, external_dependencies=external_deps), - ]) - - self.assertEqual(["file1", "l3a_file_1", "path1.zip", "kernel_1"], l3b_model_1.parent_file_names) - self.assertEqual(["file2", "l3a_file_2", "path2.zip", "kernel_1"], l3b_model_2.parent_file_names) - self.assertEqual(["file3", "path1.zip", "l3b_file_1.cdf", "kernel_1", ], l3c_model_1.parent_file_names) - self.assertEqual(["file4", "path2.zip", "l3b_file_2.cdf", "kernel_1", ], l3c_model_2.parent_file_names) - - @patch('imap_l3_processing.glows.glows_processor.filter_l3a_files') - @patch('imap_l3_processing.glows.glows_processor.GlowsL3BIonizationRate.from_instrument_team_dictionary') - @patch('imap_l3_processing.glows.glows_processor.GlowsL3CSolarWind.from_instrument_team_dictionary') - @patch('imap_l3_processing.glows.glows_processor.generate_l3bc') - @patch('imap_l3_processing.glows.glows_processor.GlowsProcessor.archive_dependencies') - @patch('imap_l3_processing.glows.glows_processor.save_data') + [ + call(sentinel.l3c_data_1, expected_l3c_metadata_1), + call(sentinel.l3c_data_2, expected_l3c_metadata_2), + ] + ) + + mock_save_data.assert_has_calls( + [ + call(l3b_model_1, cr_number=first_cr_number_to_process), + call(l3c_model_1, cr_number=first_cr_number_to_process), + call(l3b_model_2, cr_number=second_cr_number_to_process), + call(l3c_model_2, cr_number=second_cr_number_to_process), + ] + ) + + mock_archive_dependencies.assert_has_calls( + [ + call(l3bc_deps=first_dependency, external_dependencies=external_deps), + call(l3bc_deps=second_dependency, external_dependencies=external_deps), + ] + ) + + self.assertEqual( + ["file1", "l3a_file_1", "path1.zip", "kernel_1"], + l3b_model_1.parent_file_names, + ) + self.assertEqual( + ["file2", "l3a_file_2", "path2.zip", "kernel_1"], + l3b_model_2.parent_file_names, + ) + self.assertEqual( + [ + "file3", + "path1.zip", + "l3b_file_1.cdf", + "kernel_1", + ], + l3c_model_1.parent_file_names, + ) + self.assertEqual( + [ + "file4", + "path2.zip", + "l3b_file_2.cdf", + "kernel_1", + ], + l3c_model_2.parent_file_names, + ) + + @patch("imap_l3_processing.glows.glows_processor.filter_l3a_files") + @patch( + "imap_l3_processing.glows.glows_processor.GlowsL3BIonizationRate.from_instrument_team_dictionary" + ) + @patch( + "imap_l3_processing.glows.glows_processor.GlowsL3CSolarWind.from_instrument_team_dictionary" + ) + @patch("imap_l3_processing.glows.glows_processor.generate_l3bc") + @patch( + "imap_l3_processing.glows.glows_processor.GlowsProcessor.archive_dependencies" + ) + @patch("imap_l3_processing.glows.glows_processor.save_data") @patch("imap_l3_processing.glows.glows_processor.GlowsL3BCInitializer") - def test_process_l3bc_catches_no_data_error_and_continues(self, mock_glows_initializer_class, mock_save_data, - mock_archive_dependencies, mock_generate_l3bc, - mock_l3c_from_instrument_team_dictionary, - mock_l3b_from_instrument_team_dictionary, _): - bc_dependencies_1 = GlowsL3BCDependencies(version=Version(1, 1), - carrington_rotation_number=2096, - start_date=datetime(year=2021, month=1, day=1), - end_date=datetime(year=2021, month=1, day=1), - l3a_data=[], - external_files=defaultdict(Mock), - ancillary_files=defaultdict(Mock), - repointing_file_path=sentinel.repointing_file_path - ) - bc_dependencies_2 = GlowsL3BCDependencies(version=Version(1, 1), - carrington_rotation_number=2096, - start_date=datetime(year=2021, month=1, day=1), - end_date=datetime(year=2021, month=1, day=1), - l3a_data=[], - external_files=defaultdict(Mock), - ancillary_files=defaultdict(Mock), - repointing_file_path=sentinel.repointing_file_path - ) + def test_process_l3bc_catches_no_data_error_and_continues( + self, + mock_glows_initializer_class, + mock_save_data, + mock_archive_dependencies, + mock_generate_l3bc, + mock_l3c_from_instrument_team_dictionary, + mock_l3b_from_instrument_team_dictionary, + _, + ): + bc_dependencies_1 = GlowsL3BCDependencies( + version=Version(1, 1), + carrington_rotation_number=2096, + start_date=datetime(year=2021, month=1, day=1), + end_date=datetime(year=2021, month=1, day=1), + l3a_data=[], + external_files=defaultdict(Mock), + ancillary_files=defaultdict(Mock), + repointing_file_path=sentinel.repointing_file_path, + ) + bc_dependencies_2 = GlowsL3BCDependencies( + version=Version(1, 1), + carrington_rotation_number=2096, + start_date=datetime(year=2021, month=1, day=1), + end_date=datetime(year=2021, month=1, day=1), + l3a_data=[], + external_files=defaultdict(Mock), + ancillary_files=defaultdict(Mock), + repointing_file_path=sentinel.repointing_file_path, + ) external_dependencies = ExternalDependencies(None, None, None) - mock_glows_initializer_class.get_crs_to_process.return_value = GlowsL3BCInitializerData( - external_dependencies=external_dependencies, - l3bc_dependencies=[bc_dependencies_1, bc_dependencies_2], - l3bs_by_cr={}, - l3cs_by_cr={}, - repoint_file_path=sentinel.repoint_file_path + mock_glows_initializer_class.get_crs_to_process.return_value = ( + GlowsL3BCInitializerData( + external_dependencies=external_dependencies, + l3bc_dependencies=[bc_dependencies_1, bc_dependencies_2], + l3bs_by_cr={}, + l3cs_by_cr={}, + repoint_file_path=sentinel.repoint_file_path, + ) ) zip_file_1 = "imap_glows_archive_20210101_v001.zip" @@ -480,26 +700,34 @@ def test_process_l3bc_catches_no_data_error_and_continues(self, mock_glows_initi l3c_path = Path("path_to/l3c") mock_save_data.side_effect = [l3b_path, l3c_path] - version_map = VersionMap({ - GLOWS_L3B_DESCRIPTOR: Version(1, 1), - GLOWS_L3C_DESCRIPTOR: Version(1, 1), - GLOWS_L3D_DESCRIPTOR: Version(5, 1), - GLOWS_L3E_HI_45_DESCRIPTOR: Version(5, 1), - GLOWS_L3E_HI_90_DESCRIPTOR: Version(5, 1), - GLOWS_L3E_LO_DESCRIPTOR: Version(5, 1), - GLOWS_L3E_ULTRA_SF_DESCRIPTOR: Version(5, 1), - GLOWS_L3E_ULTRA_HF_DESCRIPTOR: Version(5, 1), - }) - - input_metadata = InputMetadata('glows', "l3b", datetime(2024, 10, 7, 10, 00, 00), - datetime(2024, 10, 8, 10, 00, 00), - version_map) + version_map = VersionMap( + { + GLOWS_L3B_DESCRIPTOR: Version(1, 1), + GLOWS_L3C_DESCRIPTOR: Version(1, 1), + GLOWS_L3D_DESCRIPTOR: Version(5, 1), + GLOWS_L3E_HI_45_DESCRIPTOR: Version(5, 1), + GLOWS_L3E_HI_90_DESCRIPTOR: Version(5, 1), + GLOWS_L3E_LO_DESCRIPTOR: Version(5, 1), + GLOWS_L3E_ULTRA_SF_DESCRIPTOR: Version(5, 1), + GLOWS_L3E_ULTRA_HF_DESCRIPTOR: Version(5, 1), + } + ) + + input_metadata = InputMetadata( + "glows", + "l3b", + datetime(2024, 10, 7, 10, 00, 00), + datetime(2024, 10, 8, 10, 00, 00), + version_map, + ) processor = GlowsProcessor(Mock(), input_metadata) mock_generate_l3bc.side_effect = [ - CannotProcessCarringtonRotationError("All days for Carrington Rotation are in a bad season."), - ({}, {}) + CannotProcessCarringtonRotationError( + "All days for Carrington Rotation are in a bad season." + ), + ({}, {}), ] products = processor.process() @@ -507,54 +735,88 @@ def test_process_l3bc_catches_no_data_error_and_continues(self, mock_glows_initi mock_l3b_from_instrument_team_dictionary.assert_called_once() mock_l3c_from_instrument_team_dictionary.assert_called_once() - mock_archive_dependencies.assert_has_calls([ - call(l3bc_deps=bc_dependencies_1, external_dependencies=external_dependencies), - call(l3bc_deps=bc_dependencies_2, external_dependencies=external_dependencies) - ]) + mock_archive_dependencies.assert_has_calls( + [ + call( + l3bc_deps=bc_dependencies_1, + external_dependencies=external_dependencies, + ), + call( + l3bc_deps=bc_dependencies_2, + external_dependencies=external_dependencies, + ), + ] + ) self.assertEqual(2, mock_save_data.call_count) - mock_save_data.assert_has_calls([ - call(mock_l3b_from_instrument_team_dictionary.return_value, cr_number=2096), - call(mock_l3c_from_instrument_team_dictionary.return_value, cr_number=2096), - ]) + mock_save_data.assert_has_calls( + [ + call( + mock_l3b_from_instrument_team_dictionary.return_value, + cr_number=2096, + ), + call( + mock_l3c_from_instrument_team_dictionary.return_value, + cr_number=2096, + ), + ] + ) self.assertEqual(3, len(products)) - self.assertEqual([ - l3b_path, - l3c_path, - zip_file_2 - ], products) - - @patch('imap_l3_processing.glows.glows_processor.GlowsProcessor.archive_dependencies') - @patch('imap_l3_processing.glows.glows_processor.generate_l3bc') - def test_process_l3bc_catches_exceptions_from_science_code_and_continues(self, mock_generate_l3bc, _): - l3a_data_folder_path = get_test_data_path('glows/l3a_products') + self.assertEqual([l3b_path, l3c_path, zip_file_2], products) + + @patch( + "imap_l3_processing.glows.glows_processor.GlowsProcessor.archive_dependencies" + ) + @patch("imap_l3_processing.glows.glows_processor.generate_l3bc") + def test_process_l3bc_catches_exceptions_from_science_code_and_continues( + self, mock_generate_l3bc, _ + ): + l3a_data_folder_path = get_test_data_path("glows/l3a_products") l3a_data = [ create_glows_l3a_dictionary_from_cdf( - l3a_data_folder_path / 'imap_glows_l3a_hist_20100201-repoint00032_v001.cdf')] + l3a_data_folder_path + / "imap_glows_l3a_hist_20100201-repoint00032_v001.cdf" + ) + ] external_dependencies = ExternalDependencies( - f107_index_file_path=get_test_instrument_team_data_path('glows/f107_fluxtable.txt'), - omni2_data_path=get_test_instrument_team_data_path('glows/omni_2010.dat'), - lyman_alpha_path=None + f107_index_file_path=get_test_instrument_team_data_path( + "glows/f107_fluxtable.txt" + ), + omni2_data_path=get_test_instrument_team_data_path("glows/omni_2010.dat"), + lyman_alpha_path=None, ) l3bc_deps = GlowsL3BCDependencies( l3a_data=l3a_data, ancillary_files={ - 'uv_anisotropy': get_test_data_path('glows/imap_glows_uv-anisotropy-1CR_20100101_v001.json'), - 'WawHelioIonMP_parameters': get_test_data_path('glows/imap_glows_WawHelioIonMP_20100101_v002.json'), - 'bad_days_list': get_test_data_path('glows/imap_glows_bad-days-list_v001.dat'), - 'pipeline_settings': get_test_instrument_team_data_path( - 'glows/imap_glows_pipeline-settings-L3bc_20250707_v002.json') + "uv_anisotropy": get_test_data_path( + "glows/imap_glows_uv-anisotropy-1CR_20100101_v001.json" + ), + "WawHelioIonMP_parameters": get_test_data_path( + "glows/imap_glows_WawHelioIonMP_20100101_v002.json" + ), + "bad_days_list": get_test_data_path( + "glows/imap_glows_bad-days-list_v001.dat" + ), + "pipeline_settings": get_test_instrument_team_data_path( + "glows/imap_glows_pipeline-settings-L3bc_20250707_v002.json" + ), }, external_files={ - 'f107_raw_data': get_test_instrument_team_data_path('glows/f107_fluxtable.txt'), - 'omni_raw_data': get_test_instrument_team_data_path('glows/omni_2010.dat'), + "f107_raw_data": get_test_instrument_team_data_path( + "glows/f107_fluxtable.txt" + ), + "omni_raw_data": get_test_instrument_team_data_path( + "glows/omni_2010.dat" + ), }, carrington_rotation_number=1, - start_date=Mock(), end_date=Mock(), version=Version(1, 1), - repointing_file_path=sentinel.repointing_file_path + start_date=Mock(), + end_date=Mock(), + version=Version(1, 1), + repointing_file_path=sentinel.repointing_file_path, ) initializer_data = GlowsL3BCInitializerData( @@ -562,43 +824,66 @@ def test_process_l3bc_catches_exceptions_from_science_code_and_continues(self, m l3bc_dependencies=[l3bc_deps], l3bs_by_cr={}, l3cs_by_cr={}, - repoint_file_path=sentinel.repoint_file_path + repoint_file_path=sentinel.repoint_file_path, ) mock_generate_l3bc.side_effect = Exception( - 'L3c not generated: observed cx rate is smaller than min of the cx_grid') + "L3c not generated: observed cx rate is smaller than min of the cx_grid" + ) processor = GlowsProcessor(dependencies=Mock(), input_metadata=Mock()) processor_output = process_l3bc(processor, initializer_data) self.assertEqual([], processor_output.data_products) - @patch('imap_l3_processing.glows.glows_processor.GlowsProcessor.archive_dependencies') + @patch( + "imap_l3_processing.glows.glows_processor.GlowsProcessor.archive_dependencies" + ) @patch("imap_l3_processing.glows.glows_processor.save_data") - def test_l3bc_uses_all_l3a_file_names_for_l3b_parents(self, mock_save_data, mock_archive_deps): + def test_l3bc_uses_all_l3a_file_names_for_l3b_parents( + self, mock_save_data, mock_archive_deps + ): mock_archive_deps.return_value = Path("some/path/to/archive.zip") - l3a_data_folder_path = get_test_data_path('glows/l3a_products') + l3a_data_folder_path = get_test_data_path("glows/l3a_products") - l3a_filenames = ['imap_glows_l3a_hist_20100104-repoint00004_v001.cdf', - 'imap_glows_l3a_hist_20100119-repoint00019_v001.cdf'] - l3a_data = [create_glows_l3a_dictionary_from_cdf(l3a_data_folder_path / file) for file in l3a_filenames] + l3a_filenames = [ + "imap_glows_l3a_hist_20100104-repoint00004_v001.cdf", + "imap_glows_l3a_hist_20100119-repoint00019_v001.cdf", + ] + l3a_data = [ + create_glows_l3a_dictionary_from_cdf(l3a_data_folder_path / file) + for file in l3a_filenames + ] l3bc_deps = GlowsL3BCDependencies( l3a_data=l3a_data, ancillary_files={ - 'uv_anisotropy': get_test_data_path('glows/imap_glows_uv-anisotropy-1CR_20100101_v001.json'), - 'WawHelioIonMP_parameters': get_test_data_path('glows/imap_glows_WawHelioIonMP_20100101_v002.json'), - 'bad_days_list': get_test_data_path('glows/imap_glows_bad-days-list_v001.dat'), - 'pipeline_settings': get_test_instrument_team_data_path( - 'glows/imap_glows_pipeline-settings-L3bc_20250707_v002.json') + "uv_anisotropy": get_test_data_path( + "glows/imap_glows_uv-anisotropy-1CR_20100101_v001.json" + ), + "WawHelioIonMP_parameters": get_test_data_path( + "glows/imap_glows_WawHelioIonMP_20100101_v002.json" + ), + "bad_days_list": get_test_data_path( + "glows/imap_glows_bad-days-list_v001.dat" + ), + "pipeline_settings": get_test_instrument_team_data_path( + "glows/imap_glows_pipeline-settings-L3bc_20250707_v002.json" + ), }, external_files={ - 'f107_raw_data': get_test_instrument_team_data_path('glows/f107_fluxtable.txt'), - 'omni_raw_data': get_test_instrument_team_data_path('glows/omni2_all_years.dat'), + "f107_raw_data": get_test_instrument_team_data_path( + "glows/f107_fluxtable.txt" + ), + "omni_raw_data": get_test_instrument_team_data_path( + "glows/omni2_all_years.dat" + ), }, carrington_rotation_number=2092, - start_date=Mock(), end_date=Mock(), version=Version(1, 1), + start_date=Mock(), + end_date=Mock(), + version=Version(1, 1), repointing_file_path=sentinel.repointing_file_path, ) @@ -607,7 +892,7 @@ def test_l3bc_uses_all_l3a_file_names_for_l3b_parents(self, mock_save_data, mock l3bc_dependencies=[l3bc_deps], l3bs_by_cr={}, l3cs_by_cr={}, - repoint_file_path=sentinel.repoint_file_path + repoint_file_path=sentinel.repoint_file_path, ) processor = GlowsProcessor(dependencies=Mock(), input_metadata=Mock()) @@ -618,35 +903,56 @@ def test_l3bc_uses_all_l3a_file_names_for_l3b_parents(self, mock_save_data, mock expected_parents = [ *l3a_filenames, - 'archive.zip', - 'f107_fluxtable.txt', - 'imap_glows_uv-anisotropy-1CR_20100101_v001.json', - 'imap_glows_WawHelioIonMP_20100101_v002.json', - 'imap_glows_bad-days-list_v001.dat', - 'imap_glows_pipeline-settings-L3bc_20250707_v002.json', + "archive.zip", + "f107_fluxtable.txt", + "imap_glows_uv-anisotropy-1CR_20100101_v001.json", + "imap_glows_WawHelioIonMP_20100101_v002.json", + "imap_glows_bad-days-list_v001.dat", + "imap_glows_pipeline-settings-L3bc_20250707_v002.json", ] self.assertEqual(set(expected_parents), set(l3b_data_product.parent_file_names)) - @patch("imap_l3_processing.glows.glows_processor.GlowsL3EInitializer.get_repointings_to_process") - def test_process_l3d_uploads_even_if_no_l3e_needs_to_be_produced(self, mock_l3e_initializer_get_repoints): + @patch( + "imap_l3_processing.glows.glows_processor.GlowsL3EInitializer.get_repointings_to_process" + ) + def test_process_l3d_uploads_even_if_no_l3e_needs_to_be_produced( + self, mock_l3e_initializer_get_repoints + ): mock_l3e_initializer_get_repoints.return_value = None self.test_process_l3d() @patch("imap_l3_processing.glows.glows_processor.process_l3e") - @patch("imap_l3_processing.glows.glows_processor.create_glows_l3b_json_file_from_cdf") - @patch("imap_l3_processing.glows.glows_processor.create_glows_l3c_json_file_from_cdf") - @patch('imap_l3_processing.glows.glows_processor.save_data') - @patch('imap_l3_processing.glows.glows_processor.rename_l3d_text_outputs') - @patch('imap_l3_processing.glows.glows_processor.get_parent_file_names_from_l3d_json') - @patch('imap_l3_processing.glows.glows_processor.convert_json_to_l3d_data_product') - @patch('imap_l3_processing.glows.glows_processor.run') - @patch('imap_l3_processing.glows.glows_processor.shutil') + @patch( + "imap_l3_processing.glows.glows_processor.create_glows_l3b_json_file_from_cdf" + ) + @patch( + "imap_l3_processing.glows.glows_processor.create_glows_l3c_json_file_from_cdf" + ) + @patch("imap_l3_processing.glows.glows_processor.save_data") + @patch("imap_l3_processing.glows.glows_processor.rename_l3d_text_outputs") + @patch( + "imap_l3_processing.glows.glows_processor.get_parent_file_names_from_l3d_json" + ) + @patch("imap_l3_processing.glows.glows_processor.convert_json_to_l3d_data_product") + @patch("imap_l3_processing.glows.glows_processor.run") + @patch("imap_l3_processing.glows.glows_processor.shutil") @patch("imap_l3_processing.glows.glows_processor.os") @patch("imap_l3_processing.glows.glows_processor.GlowsL3DInitializer") - def test_process_l3d(self, mock_glows_l3d_initializer, mock_os, mock_shutil, mock_run, - mock_convert_json_to_l3d_data_product, mock_get_parent_file_names_from_l3d_json, - mock_rename_l3d, mock_save_data, mock_convert_l3c_to_json, mock_convert_l3b_to_json, _): + def test_process_l3d( + self, + mock_glows_l3d_initializer, + mock_os, + mock_shutil, + mock_run, + mock_convert_json_to_l3d_data_product, + mock_get_parent_file_names_from_l3d_json, + mock_rename_l3d, + mock_save_data, + mock_convert_l3c_to_json, + mock_convert_l3b_to_json, + _, + ): cr_number = 2092 @@ -673,11 +979,11 @@ def test_process_l3d(self, mock_glows_l3d_initializer, mock_os, mock_shutil, moc end_cr=expected_end_cr, ) - old_l3d = Path('imap_glows_l3d_solar-hist_19470303-cr02090_v001.cdf') + old_l3d = Path("imap_glows_l3d_solar-hist_19470303-cr02090_v001.cdf") major_version = 12 l3d_output_version = Version(major_version, 5) - expected_l3d_cdf_filename = f'imap_glows_l3d_{GLOWS_L3D_DESCRIPTOR}_19470303-cr0{expected_end_cr}_{l3d_output_version}.cdf' + expected_l3d_cdf_filename = f"imap_glows_l3d_{GLOWS_L3D_DESCRIPTOR}_19470303-cr0{expected_end_cr}_{l3d_output_version}.cdf" mock_glows_l3d_initializer.should_process_l3d.return_value = ( l3d_output_version, @@ -685,13 +991,15 @@ def test_process_l3d(self, mock_glows_l3d_initializer, mock_os, mock_shutil, moc old_l3d, ) - mock_run.return_value = CompletedProcess(args=[], returncode=0, stdout=f'Processed CR= {expected_end_cr}') + mock_run.return_value = CompletedProcess( + args=[], returncode=0, stdout=f"Processed CR= {expected_end_cr}" + ) mock_os.listdir.return_value = [ - f'{cr_number}_txt_file_1', - f'{cr_number}_txt_file_2', - f'{cr_number + 1}_txt_file_1', - f'{cr_number + 1}_txt_file_2', + f"{cr_number}_txt_file_1", + f"{cr_number}_txt_file_2", + f"{cr_number + 1}_txt_file_1", + f"{cr_number + 1}_txt_file_2", ] mock_convert_json_to_l3d_data_product.return_value = sentinel.l3d_data_product @@ -715,136 +1023,214 @@ def test_process_l3d(self, mock_glows_l3d_initializer, mock_os, mock_shutil, moc GLOWS_L3E_ULTRA_HF_DESCRIPTOR: Version(2, 1), } ) - input_metadata = InputMetadata('glows', "l3b", datetime(2024, 10, 7), None, version=input_version_map) + input_metadata = InputMetadata( + "glows", "l3b", datetime(2024, 10, 7), None, version=input_version_map + ) processing_input_collection = Mock() processor = GlowsProcessor(processing_input_collection, input_metadata) products = processor.process() - mock_convert_l3b_to_json.assert_has_calls([call(sentinel.l3b_file_1), call(sentinel.l3b_file_2)]) - mock_convert_l3c_to_json.assert_has_calls([call(sentinel.l3c_file_1), call(sentinel.l3c_file_2)]) + mock_convert_l3b_to_json.assert_has_calls( + [call(sentinel.l3b_file_1), call(sentinel.l3b_file_2)] + ) + mock_convert_l3c_to_json.assert_has_calls( + [call(sentinel.l3c_file_1), call(sentinel.l3c_file_2)] + ) mock_glows_l3d_initializer.should_process_l3d.assert_called_with( - self.mock_external_deps, [], [], self.mock_reprocess_info, major_version) + self.mock_external_deps, [], [], self.mock_reprocess_info, major_version + ) self.mock_fetch_reprocess_info.assert_called_with(processing_input_collection) - self.assertEqual([ - Path("imap_glows_e-dens_19470303_20100101_v000.dat"), - Path("imap_glows_lya_19470303_20100101_v000.dat"), - Path("l3d_cdf.cdf") - ], products) + self.assertEqual( + [ + Path("imap_glows_e-dens_19470303_20100101_v000.dat"), + Path("imap_glows_lya_19470303_20100101_v000.dat"), + Path("l3d_cdf.cdf"), + ], + products, + ) self.assertEqual(2, mock_shutil.copy.call_count) - mock_shutil.copy.assert_has_calls([ - call(Path("imap_glows_e-dens_19470303_20100101_v000.dat"), - imap_data_access.config[ - "DATA_DIR"] / "imap/ancillary/glows/imap_glows_e-dens_19470303_20100101_v000.dat"), - call(Path("imap_glows_lya_19470303_20100101_v000.dat"), - imap_data_access.config["DATA_DIR"] / "imap/ancillary/glows/imap_glows_lya_19470303_20100101_v000.dat") - ]) + mock_shutil.copy.assert_has_calls( + [ + call( + Path("imap_glows_e-dens_19470303_20100101_v000.dat"), + imap_data_access.config["DATA_DIR"] + / "imap/ancillary/glows/imap_glows_e-dens_19470303_20100101_v000.dat", + ), + call( + Path("imap_glows_lya_19470303_20100101_v000.dat"), + imap_data_access.config["DATA_DIR"] + / "imap/ancillary/glows/imap_glows_lya_19470303_20100101_v000.dat", + ), + ] + ) self.assertEqual(2, mock_os.makedirs.call_count) - mock_os.makedirs.assert_has_calls([ - call(PATH_TO_L3D_TOOLKIT / 'data_l3d', exist_ok=True), - call(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt', exist_ok=True), - ]) - - expected_dependencies = json.dumps({ - 'external_files': { - 'lya_raw_data': str(Path('path/to/lya')), - }, - 'ancillary_files': { - 'pipeline_settings': - str(Path('glows/imap_glows_pipeline-settings-l3bcde_20250514_v004.json')), - 'WawHelioIon': { - 'speed': str(Path('path/to/speed')), - 'p-dens': str(Path('path/to/p-dens')), - 'uv-anis': str(Path('path/to/uv-anis')), - 'phion': str(Path('path/to/phion')), - 'lya': str(Path('path/to/lya')), - 'e-dens': str(Path('path/to/e-dens')) + mock_os.makedirs.assert_has_calls( + [ + call(PATH_TO_L3D_TOOLKIT / "data_l3d", exist_ok=True), + call(PATH_TO_L3D_TOOLKIT / "data_l3d_txt", exist_ok=True), + ] + ) + + expected_dependencies = json.dumps( + { + "external_files": { + "lya_raw_data": str(Path("path/to/lya")), }, - }, - "l3d_cdf_filename": expected_l3d_cdf_filename - }) + "ancillary_files": { + "pipeline_settings": str( + Path( + "glows/imap_glows_pipeline-settings-l3bcde_20250514_v004.json" + ) + ), + "WawHelioIon": { + "speed": str(Path("path/to/speed")), + "p-dens": str(Path("path/to/p-dens")), + "uv-anis": str(Path("path/to/uv-anis")), + "phion": str(Path("path/to/phion")), + "lya": str(Path("path/to/lya")), + "e-dens": str(Path("path/to/e-dens")), + }, + }, + "l3d_cdf_filename": expected_l3d_cdf_filename, + } + ) - expected_working_directory = Path(l3d.__file__).parent / 'science' + expected_working_directory = Path(l3d.__file__).parent / "science" self.assertEqual(1, mock_run.call_count) - mock_run.assert_has_calls([ - call([sys.executable, './generate_l3d.py', f'{expected_end_cr}', expected_dependencies], - cwd=str(expected_working_directory), - check=True, - capture_output=True, text=True), - ]) + mock_run.assert_has_calls( + [ + call( + [ + sys.executable, + "./generate_l3d.py", + f"{expected_end_cr}", + expected_dependencies, + ], + cwd=str(expected_working_directory), + check=True, + capture_output=True, + text=True, + ), + ] + ) - mock_os.listdir.assert_called_once_with(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt') + mock_os.listdir.assert_called_once_with(PATH_TO_L3D_TOOLKIT / "data_l3d_txt") expected_l3d_txt_paths = [ - Path(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt' / f'{expected_end_cr}_txt_file_1', ), - Path(PATH_TO_L3D_TOOLKIT / 'data_l3d_txt' / f'{expected_end_cr}_txt_file_2', ), + Path( + PATH_TO_L3D_TOOLKIT / "data_l3d_txt" / f"{expected_end_cr}_txt_file_1", + ), + Path( + PATH_TO_L3D_TOOLKIT / "data_l3d_txt" / f"{expected_end_cr}_txt_file_2", + ), ] - mock_rename_l3d.assert_has_calls([call(expected_l3d_txt_paths, str(Version(None, l3d_output_version.minor)))]) + mock_rename_l3d.assert_has_calls( + [call(expected_l3d_txt_paths, str(Version(None, l3d_output_version.minor)))] + ) - mock_get_parent_file_names_from_l3d_json.assert_called_once_with(expected_working_directory / 'data_l3d') + mock_get_parent_file_names_from_l3d_json.assert_called_once_with( + expected_working_directory / "data_l3d" + ) - expected_data_product_metadata = InputMetadata(instrument="glows", data_level="l3d", descriptor="solar-hist", - start_date=datetime(1947, 3, 3), - end_date=datetime(1947, 3, 3), - version=VersionMap({GLOWS_L3D_DESCRIPTOR: l3d_output_version})) + expected_data_product_metadata = InputMetadata( + instrument="glows", + data_level="l3d", + descriptor="solar-hist", + start_date=datetime(1947, 3, 3), + end_date=datetime(1947, 3, 3), + version=VersionMap({GLOWS_L3D_DESCRIPTOR: l3d_output_version}), + ) mock_convert_json_to_l3d_data_product.assert_called_once_with( - expected_working_directory / 'data_l3d' / f'imap_glows_l3d_solar-params-history_19470303-cr0{expected_end_cr}_v00.json', + expected_working_directory + / "data_l3d" + / f"imap_glows_l3d_solar-params-history_19470303-cr0{expected_end_cr}_v00.json", expected_data_product_metadata, - mock_get_parent_file_names_from_l3d_json.return_value) + mock_get_parent_file_names_from_l3d_json.return_value, + ) - mock_save_data.assert_called_once_with(sentinel.l3d_data_product, cr_number=expected_end_cr) + mock_save_data.assert_called_once_with( + sentinel.l3d_data_product, cr_number=expected_end_cr + ) - @patch('imap_l3_processing.glows.glows_processor.save_data') - @patch('imap_l3_processing.glows.glows_processor.rename_l3d_text_outputs') - @patch('imap_l3_processing.glows.glows_processor.PATH_TO_L3D_TOOLKIT', get_test_data_path('glows/science')) - @patch('imap_l3_processing.glows.glows_processor.run') - @patch('imap_l3_processing.processor.spiceypy') - def test_process_l3d_adds_parent_file_names_to_output(self, mock_spicepy, mock_run, mock_rename_l3d_text_output, - mock_save_data): + @patch("imap_l3_processing.glows.glows_processor.save_data") + @patch("imap_l3_processing.glows.glows_processor.rename_l3d_text_outputs") + @patch( + "imap_l3_processing.glows.glows_processor.PATH_TO_L3D_TOOLKIT", + get_test_data_path("glows/science"), + ) + @patch("imap_l3_processing.glows.glows_processor.run") + @patch("imap_l3_processing.processor.spiceypy") + def test_process_l3d_adds_parent_file_names_to_output( + self, mock_spicepy, mock_run, mock_rename_l3d_text_output, mock_save_data + ): mock_spicepy.ktotal.return_value = 0 - l3b_path_1 = get_test_data_path('glows/imap_glows_l3b_ion-rate-profile_20100422_v011.cdf') - l3b_path_2 = get_test_data_path('glows/imap_glows_l3b_ion-rate-profile_20100519_v011.cdf') - l3c_path_1 = get_test_data_path('glows/imap_glows_l3c_sw-profile_20100422_v011.cdf') - l3c_path_2 = get_test_data_path('glows/imap_glows_l3c_sw-profile_20100519_v011.cdf') + l3b_path_1 = get_test_data_path( + "glows/imap_glows_l3b_ion-rate-profile_20100422_v011.cdf" + ) + l3b_path_2 = get_test_data_path( + "glows/imap_glows_l3b_ion-rate-profile_20100519_v011.cdf" + ) + l3c_path_1 = get_test_data_path( + "glows/imap_glows_l3c_sw-profile_20100422_v011.cdf" + ) + l3c_path_2 = get_test_data_path( + "glows/imap_glows_l3c_sw-profile_20100519_v011.cdf" + ) pipeline_settings_path = get_test_data_path( - 'glows/l3d_drift_test/imap_glows_pipeline-settings-l3bcde_20100101_v006.json') - speed_path = get_test_data_path('glows/imap_glows_plasma-speed-Legendre-2010a_v001.dat') - p_dens_path = get_test_data_path('glows/imap_glows_proton-density-Legendre-2010a_v001.dat') - uv_anis_path = get_test_data_path('glows/imap_glows_uv-anisotropy-2010a_v001.dat') - phion_path = get_test_data_path('glows/imap_glows_photoion-2010a_v001.dat') - lya_path = get_test_data_path('glows/imap_glows_lya-2010a_v001.dat') - e_dens_path = get_test_data_path('glows/imap_glows_electron-density-2010a_v001.dat') + "glows/l3d_drift_test/imap_glows_pipeline-settings-l3bcde_20100101_v006.json" + ) + speed_path = get_test_data_path( + "glows/imap_glows_plasma-speed-Legendre-2010a_v001.dat" + ) + p_dens_path = get_test_data_path( + "glows/imap_glows_proton-density-Legendre-2010a_v001.dat" + ) + uv_anis_path = get_test_data_path( + "glows/imap_glows_uv-anisotropy-2010a_v001.dat" + ) + phion_path = get_test_data_path("glows/imap_glows_photoion-2010a_v001.dat") + lya_path = get_test_data_path("glows/imap_glows_lya-2010a_v001.dat") + e_dens_path = get_test_data_path( + "glows/imap_glows_electron-density-2010a_v001.dat" + ) - lyman_alpha_composite_path = get_test_data_path('glows/lyman_alpha_composite.nc') + lyman_alpha_composite_path = get_test_data_path( + "glows/lyman_alpha_composite.nc" + ) l3b_file_paths = [l3b_path_1, l3b_path_2] l3c_file_paths = [l3c_path_1, l3c_path_2] ancillary_inputs = { - 'pipeline_settings': pipeline_settings_path, - 'WawHelioIon': { - 'speed': speed_path, - 'p-dens': p_dens_path, - 'uv-anis': uv_anis_path, - 'phion': phion_path, - 'lya': lya_path, - 'e-dens': e_dens_path - } - } - external_inputs = { - 'lya_raw_data': lyman_alpha_composite_path + "pipeline_settings": pipeline_settings_path, + "WawHelioIon": { + "speed": speed_path, + "p-dens": p_dens_path, + "uv-anis": uv_anis_path, + "phion": phion_path, + "lya": lya_path, + "e-dens": e_dens_path, + }, } + external_inputs = {"lya_raw_data": lyman_alpha_composite_path} cr_number = 2095 - mock_run.return_value = CompletedProcess(args=[], returncode=0, stdout=f'Processed CR= {cr_number}') + mock_run.return_value = CompletedProcess( + args=[], returncode=0, stdout=f"Processed CR= {cr_number}" + ) - l3d_dependencies = GlowsL3DDependencies(l3b_file_paths=l3b_file_paths, l3c_file_paths=l3c_file_paths, - ancillary_files=ancillary_inputs, external_files=external_inputs, - end_cr=cr_number) + l3d_dependencies = GlowsL3DDependencies( + l3b_file_paths=l3b_file_paths, + l3c_file_paths=l3c_file_paths, + ancillary_files=ancillary_inputs, + external_files=external_inputs, + end_cr=cr_number, + ) _ = process_l3d(l3d_dependencies, Version(1, 1)) @@ -862,7 +1248,7 @@ def test_process_l3d_adds_parent_file_names_to_output(self, mock_spicepy, mock_r "imap_glows_l3b_ion-rate-profile_20100422_v011.cdf", "imap_glows_l3b_ion-rate-profile_20100519_v011.cdf", "imap_glows_l3c_sw-profile_20100422_v011.cdf", - "imap_glows_l3c_sw-profile_20100519_v011.cdf" + "imap_glows_l3c_sw-profile_20100519_v011.cdf", ] expected_l3d_txt_file_paths = [ @@ -894,45 +1280,51 @@ def test_process_l3d_adds_parent_file_names_to_output(self, mock_spicepy, mock_r ) self.assertEqual("v001", mock_rename_l3d_text_output.call_args[0][1]) - @patch('imap_l3_processing.glows.glows_processor.rename_l3d_text_outputs') - @patch('imap_l3_processing.glows.glows_processor.get_parent_file_names_from_l3d_json') - @patch('imap_l3_processing.glows.glows_processor.os') - @patch('imap_l3_processing.glows.glows_processor.run') - @patch('imap_l3_processing.glows.glows_processor.convert_json_to_l3d_data_product') - def test_process_l3d_handles_unexpected_exception_from_science(self, - mock_convert_json_to_l3d, - mock_run, mock_os, _, __): + @patch("imap_l3_processing.glows.glows_processor.rename_l3d_text_outputs") + @patch( + "imap_l3_processing.glows.glows_processor.get_parent_file_names_from_l3d_json" + ) + @patch("imap_l3_processing.glows.glows_processor.os") + @patch("imap_l3_processing.glows.glows_processor.run") + @patch("imap_l3_processing.glows.glows_processor.convert_json_to_l3d_data_product") + def test_process_l3d_handles_unexpected_exception_from_science( + self, mock_convert_json_to_l3d, mock_run, mock_os, _, __ + ): ancillary_files = { - 'pipeline_settings': get_test_data_path( - "glows/l3d_drift_test/imap_glows_pipeline-settings-l3bcde_20100101_v006.json"), - 'WawHelioIon': { - 'speed': Path('path/to/speed'), - 'p-dens': Path('path/to/p-dens'), - 'uv-anis': Path('path/to/uv-anis'), - 'phion': Path('path/to/phion'), - 'lya': Path('path/to/lya'), - 'e-dens': Path('path/to/e-dens') - } + "pipeline_settings": get_test_data_path( + "glows/l3d_drift_test/imap_glows_pipeline-settings-l3bcde_20100101_v006.json" + ), + "WawHelioIon": { + "speed": Path("path/to/speed"), + "p-dens": Path("path/to/p-dens"), + "uv-anis": Path("path/to/uv-anis"), + "phion": Path("path/to/phion"), + "lya": Path("path/to/lya"), + "e-dens": Path("path/to/e-dens"), + }, } external_files = { - 'lya_raw_data': Path('path/to/lya'), + "lya_raw_data": Path("path/to/lya"), } l3b_file_paths = [] l3c_file_paths = [] expected_cr = 2096 - l3d_dependencies = GlowsL3DDependencies(ancillary_files=ancillary_files, - external_files=external_files, - l3b_file_paths=l3b_file_paths, - l3c_file_paths=l3c_file_paths, - end_cr=expected_cr) - - mock_os.listdir.return_value = ['2096_txt_file_1', - '2096_txt_file_2', - '2096_txt_file_3', - '2096_txt_file_4', - '2096_txt_file_5', - '2096_txt_file_6' - ] + l3d_dependencies = GlowsL3DDependencies( + ancillary_files=ancillary_files, + external_files=external_files, + l3b_file_paths=l3b_file_paths, + l3c_file_paths=l3c_file_paths, + end_cr=expected_cr, + ) + + mock_os.listdir.return_value = [ + "2096_txt_file_1", + "2096_txt_file_2", + "2096_txt_file_3", + "2096_txt_file_4", + "2096_txt_file_5", + "2096_txt_file_6", + ] unexpected_exception = r"""Traceback (most recent call last): File "...\imap-L3-processing\imap_l3_processing\glows\l3d\science\generate_l3d.py", line 46, in @@ -950,7 +1342,9 @@ def test_process_l3d_handles_unexpected_exception_from_science(self, ^^^^^^^^^^^^^ FileNotFoundError: [Errno 2] No such file or directory: 'imap_glows_l3a_hist_20100511-repoint00131_v011.cdf'""" - mock_run.side_effect = [CalledProcessError(cmd="", returncode=1, stderr=unexpected_exception)] + mock_run.side_effect = [ + CalledProcessError(cmd="", returncode=1, stderr=unexpected_exception) + ] with self.assertRaises(Exception) as context: process_l3d(l3d_dependencies, Version(1, 1)) @@ -958,25 +1352,40 @@ def test_process_l3d_handles_unexpected_exception_from_science(self, mock_convert_json_to_l3d.assert_not_called() - @patch('imap_l3_processing.glows.glows_processor.shutil') - @patch('imap_l3_processing.glows.glows_processor.save_data') + @patch("imap_l3_processing.glows.glows_processor.shutil") + @patch("imap_l3_processing.glows.glows_processor.save_data") def test_process_glows_l3d_drift(self, mock_save_data, _): l3d_drift_test_data = get_test_data_path("glows/l3d_drift_test") lyman_alpha_composite = l3d_drift_test_data / "lyman_alpha_composite.nc" expected_cr = 2098 - l3b_1 = l3d_drift_test_data / "imap_glows_l3b_ion-rate-profile_20100422_v013.cdf" - l3b_2 = l3d_drift_test_data / "imap_glows_l3b_ion-rate-profile_20100519_v013.cdf" + l3b_1 = ( + l3d_drift_test_data / "imap_glows_l3b_ion-rate-profile_20100422_v013.cdf" + ) + l3b_2 = ( + l3d_drift_test_data / "imap_glows_l3b_ion-rate-profile_20100519_v013.cdf" + ) l3c_1 = l3d_drift_test_data / "imap_glows_l3c_sw-profile_20100422_v012.cdf" l3c_2 = l3d_drift_test_data / "imap_glows_l3c_sw-profile_20100519_v012.cdf" - plasma_speed_legendre = l3d_drift_test_data / "imap_glows_plasma-speed-2010a_20100101_v003.dat" - proton_density_legendre = l3d_drift_test_data / "imap_glows_proton-density-2010a_20100101_v003.dat" - uv_anisotropy = l3d_drift_test_data / "imap_glows_uv-anisotropy-2010a_20100101_v003.dat" + plasma_speed_legendre = ( + l3d_drift_test_data / "imap_glows_plasma-speed-2010a_20100101_v003.dat" + ) + proton_density_legendre = ( + l3d_drift_test_data / "imap_glows_proton-density-2010a_20100101_v003.dat" + ) + uv_anisotropy = ( + l3d_drift_test_data / "imap_glows_uv-anisotropy-2010a_20100101_v003.dat" + ) photoion = l3d_drift_test_data / "imap_glows_photoion-2010a_20100101_v003.dat" lya_2010a = l3d_drift_test_data / "imap_glows_lya-2010a_20100101_v003.dat" - electron_density = l3d_drift_test_data / "imap_glows_electron-density-2010a_20100101_v003.dat" - pipeline_settings = l3d_drift_test_data / "imap_glows_pipeline-settings-l3bcde_20100101_v006.json" + electron_density = ( + l3d_drift_test_data / "imap_glows_electron-density-2010a_20100101_v003.dat" + ) + pipeline_settings = ( + l3d_drift_test_data + / "imap_glows_pipeline-settings-l3bcde_20100101_v006.json" + ) l3d_dependencies = GlowsL3DDependencies( external_files={ @@ -1020,14 +1429,19 @@ def test_process_glows_l3d_drift(self, mock_save_data, _): self.assertEqual(expected_parents, set(data_product.parent_file_names)) - expected_txt_filenames = ["imap_glows_e-dens_19470303_20100629_v004.dat", - "imap_glows_lya_19470303_20100629_v004.dat", - "imap_glows_p-dens_19470303_20100629_v004.dat", - "imap_glows_phion_19470303_20100629_v004.dat", - "imap_glows_speed_19470303_20100629_v004.dat", - "imap_glows_uv-anis_19470303_20100629_v004.dat"] + expected_txt_filenames = [ + "imap_glows_e-dens_19470303_20100629_v004.dat", + "imap_glows_lya_19470303_20100629_v004.dat", + "imap_glows_p-dens_19470303_20100629_v004.dat", + "imap_glows_phion_19470303_20100629_v004.dat", + "imap_glows_speed_19470303_20100629_v004.dat", + "imap_glows_uv-anis_19470303_20100629_v004.dat", + ] for file in expected_txt_filenames: - self.assertIn(PATH_TO_L3D_TOOLKIT / "data_l3d_txt" / file, glows_l3d_output.l3d_text_file_paths) + self.assertIn( + PATH_TO_L3D_TOOLKIT / "data_l3d_txt" / file, + glows_l3d_output.l3d_text_file_paths, + ) test_cases = [ ( @@ -1041,20 +1455,65 @@ def test_process_glows_l3d_drift(self, mock_save_data, _): expected_txt_filenames[1], "lyman_alpha", 849, - [1947.167990000000, 6.199757637899209e+11, 1250.5], - [2010.49252, 391688271318.51849, 2098.50000, ], + [1947.167990000000, 6.199757637899209e11, 1250.5], + [ + 2010.49252, + 391688271318.51849, + 2098.50000, + ], ), ( expected_txt_filenames[2], "proton_density", 849, - [1.94716799e+03, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 1.25050000e+03, -1.00000000e+00], - [2010.49252260274, 3.533487319946289, 3.50542426109314, 3.409455299377441, 3.878353357315063, - 4.971815586090088, 6.224776268005371, 7.651707649230957, 8.510316848754883, 8.865212440490723, - 8.813090324401855, 9.078702926635742, 9.078702926635742, 7.368694305419922, 5.78929615020752, - 4.63243579864502, 3.519419431686401, 3.034561634063721, 3.000047445297241, 3.000047445297241, - 2098.5, 88888.0], + [ + 1.94716799e03, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + 1.25050000e03, + -1.00000000e00, + ], + [ + 2010.49252260274, + 3.533487319946289, + 3.50542426109314, + 3.409455299377441, + 3.878353357315063, + 4.971815586090088, + 6.224776268005371, + 7.651707649230957, + 8.510316848754883, + 8.865212440490723, + 8.813090324401855, + 9.078702926635742, + 9.078702926635742, + 7.368694305419922, + 5.78929615020752, + 4.63243579864502, + 3.519419431686401, + 3.034561634063721, + 3.000047445297241, + 3.000047445297241, + 2098.5, + 88888.0, + ], ), ( expected_txt_filenames[3], @@ -1067,77 +1526,199 @@ def test_process_glows_l3d_drift(self, mock_save_data, _): expected_txt_filenames[4], "plasma_speed", 849, - [1.94716799e+03, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 1.25050000e+03, -1], - [2010.49252260274, 397.0, 399.0, 406.0, 374.0, 316.0, 268.0, 228.0, 209.0, 202.0, 203.0, 198.0, - 198.0, 235.0, 283.0, 332.0, 398.0, 436.0, 439.0, 439.0, 2098.5, 88888.0], + [ + 1.94716799e03, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + 1.25050000e03, + -1, + ], + [ + 2010.49252260274, + 397.0, + 399.0, + 406.0, + 374.0, + 316.0, + 268.0, + 228.0, + 209.0, + 202.0, + 203.0, + 198.0, + 198.0, + 235.0, + 283.0, + 332.0, + 398.0, + 436.0, + 439.0, + 439.0, + 2098.5, + 88888.0, + ], ), ( expected_txt_filenames[5], "uv_anisotropy", 849, - [1.94716799e+03, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 1.25050000e+03, -1], - [2010.49252260274, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2098.5, 88888.0], + [ + 1.94716799e03, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + 1.25050000e03, + -1, + ], + [ + 2010.49252260274, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 2098.5, + 88888.0, + ], ), ] [save_data_call_args] = mock_save_data.call_args_list l3d_data_product = save_data_call_args.args[0] - expected_cdf_filename = "imap_glows_l3d_solar-hist_19470303-cr02098_v001.0004.cdf" + expected_cdf_filename = ( + "imap_glows_l3d_solar-hist_19470303-cr02098_v001.0004.cdf" + ) with tempfile.TemporaryDirectory() as tmp_dir: tmp_dir = Path(tmp_dir) save_data(l3d_data_product, cr_number=expected_cr, folder_path=tmp_dir) with CDF(str(tmp_dir / expected_cdf_filename)) as actual_cdf: - for filename, cdf_var_name, length_of_data, first_line, last_line in test_cases: + for ( + filename, + cdf_var_name, + length_of_data, + first_line, + last_line, + ) in test_cases: with self.subTest(msg=filename): l3d_text_path = PATH_TO_L3D_TOOLKIT / "data_l3d_txt" / filename actual = np.loadtxt(l3d_text_path) - actual_input_file_line = next((l for l in l3d_text_path.read_text().split("\n") if l.startswith("# input files:")), None) + actual_input_file_line = next( + ( + l + for l in l3d_text_path.read_text().split("\n") + if l.startswith("# input files:") + ), + None, + ) self.assertIsNotNone(actual_input_file_line) - expected_input_file_line = f"# input files: {expected_cdf_filename}" - self.assertEqual(expected_input_file_line, actual_input_file_line) + expected_input_file_line = ( + f"# input files: {expected_cdf_filename}" + ) + self.assertEqual( + expected_input_file_line, actual_input_file_line + ) self.assertEqual(length_of_data, len(actual)) - self.assertEqual(length_of_data, len(actual_cdf[cdf_var_name][...])) + self.assertEqual( + length_of_data, len(actual_cdf[cdf_var_name][...]) + ) np.testing.assert_allclose(actual[0], first_line) np.testing.assert_allclose(actual[-1], last_line) - @patch('imap_l3_processing.glows.glows_processor.compute_glows_flags_for_repoint') - @patch('imap_l3_processing.glows.glows_processor.get_lo_pivot_angles') - @patch('imap_l3_processing.glows.glows_processor.get_pointing_date_range') - @patch('imap_l3_processing.glows.glows_processor.determine_spacecraft_info_using_predict_if_needed') - @patch('imap_l3_processing.glows.glows_processor.process_l3e_hi') - @patch('imap_l3_processing.glows.glows_processor.process_l3e_lo') - @patch('imap_l3_processing.glows.glows_processor.process_l3e_ul_hf') - @patch('imap_l3_processing.glows.glows_processor.process_l3e_ul_sf') - def test_process_l3e(self, mock_process_ultra, mock_process_ultra_hf, mock_process_lo, mock_process_hi, - mock_determine_spacecraft_info, - mock_get_pointing_date_range, - mock_get_lo_pivot_angles, - mock_compute_glows_flags_for_repoint, - ): + @patch("imap_l3_processing.glows.glows_processor.compute_glows_flags_for_repoint") + @patch("imap_l3_processing.glows.glows_processor.get_lo_pivot_angles") + @patch("imap_l3_processing.glows.glows_processor.get_pointing_date_range") + @patch( + "imap_l3_processing.glows.glows_processor.determine_spacecraft_info_using_predict_if_needed" + ) + @patch("imap_l3_processing.glows.glows_processor.process_l3e_hi") + @patch("imap_l3_processing.glows.glows_processor.process_l3e_lo") + @patch("imap_l3_processing.glows.glows_processor.process_l3e_ul_hf") + @patch("imap_l3_processing.glows.glows_processor.process_l3e_ul_sf") + def test_process_l3e( + self, + mock_process_ultra, + mock_process_ultra_hf, + mock_process_lo, + mock_process_hi, + mock_determine_spacecraft_info, + mock_get_pointing_date_range, + mock_get_lo_pivot_angles, + mock_compute_glows_flags_for_repoint, + ): mock_process_hi.side_effect = [ - [Path('path/to/first_hi_l3e')], - [Path('path/to/second_hi_l3e')], + [Path("path/to/first_hi_l3e")], + [Path("path/to/second_hi_l3e")], ] - mock_process_lo.return_value = [Path('path/to/lo_l3e')] - mock_process_ultra.return_value = [Path('path/to/ultra_l3e')] - mock_process_ultra_hf.return_value = [Path('path/to/ultra_l3e_hf')] + mock_process_lo.return_value = [Path("path/to/lo_l3e")] + mock_process_ultra.return_value = [Path("path/to/ultra_l3e")] + mock_process_ultra_hf.return_value = [Path("path/to/ultra_l3e_hf")] mock_get_lo_pivot_angles.return_value = {25: LoPivotAngle("l1b_nhk.cdf", 75)} mock_compute_glows_flags_for_repoint.return_value = 4 - mock_determine_spacecraft_info.return_value = sentinel.spacecraft_info, GlowsL3Flags.PREDICTIVE_EPHEMERIS, ["spice kernel"] + mock_determine_spacecraft_info.return_value = ( + sentinel.spacecraft_info, + GlowsL3Flags.PREDICTIVE_EPHEMERIS, + ["spice kernel"], + ) expected_l3e_products = [ - Path('path/to/lo_l3e'), - Path('path/to/first_hi_l3e'), Path('path/to/second_hi_l3e'), - Path('path/to/ultra_l3e'), - Path('path/to/ultra_l3e_hf') + Path("path/to/lo_l3e"), + Path("path/to/first_hi_l3e"), + Path("path/to/second_hi_l3e"), + Path("path/to/ultra_l3e"), + Path("path/to/ultra_l3e_hf"), ] start_epoch = datetime(2020, 1, 1) @@ -1150,7 +1731,9 @@ def test_process_l3e(self, mock_process_ultra, mock_process_ultra_hf, mock_proce actual_l3e_products = process_l3e(initializer_data) mock_get_pointing_date_range.assert_called_once_with(25) - mock_compute_glows_flags_for_repoint.assert_called_once_with(initializer_data.l3d_cdf_path, repointing_midpoint) + mock_compute_glows_flags_for_repoint.assert_called_once_with( + initializer_data.l3d_cdf_path, repointing_midpoint + ) mock_determine_spacecraft_info.assert_called_once_with( datetime(2020, 1, 1, 12), initializer_data.metakernel_with_predict_ephem, @@ -1181,36 +1764,82 @@ def test_process_l3e(self, mock_process_ultra, mock_process_ultra_hf, mock_proce ), ] ) - mock_process_lo.assert_called_once_with(["lo_ancillary.dat", "spice kernel", "l1b_nhk.cdf"], 25, start_epoch, epoch_delta, 75, - Version(None, 3), expected_flags, sentinel.spacecraft_info) - mock_process_ultra.assert_called_once_with(["ul_ancillary.dat", "spice kernel"], 25, start_epoch, epoch_delta, Version(None, 4), expected_flags, sentinel.spacecraft_info) - mock_process_ultra_hf.assert_called_once_with(["ul_ancillary.dat", "spice kernel"], 25, start_epoch, epoch_delta, Version(None, 4), expected_flags, sentinel.spacecraft_info) + mock_process_lo.assert_called_once_with( + ["lo_ancillary.dat", "spice kernel", "l1b_nhk.cdf"], + 25, + start_epoch, + epoch_delta, + 75, + Version(None, 3), + expected_flags, + sentinel.spacecraft_info, + ) + mock_process_ultra.assert_called_once_with( + ["ul_ancillary.dat", "spice kernel"], + 25, + start_epoch, + epoch_delta, + Version(None, 4), + expected_flags, + sentinel.spacecraft_info, + ) + mock_process_ultra_hf.assert_called_once_with( + ["ul_ancillary.dat", "spice kernel"], + 25, + start_epoch, + epoch_delta, + Version(None, 4), + expected_flags, + sentinel.spacecraft_info, + ) self.assertEqual(expected_l3e_products, actual_l3e_products) mock_get_lo_pivot_angles.assert_called_once_with([25]) - @patch('imap_l3_processing.glows.glows_processor.determine_spacecraft_info_using_predict_if_needed') - @patch('imap_l3_processing.glows.glows_processor.get_lo_pivot_angles') - @patch('imap_l3_processing.glows.glows_processor.process_l3e_ul_hf') - @patch('imap_l3_processing.glows.glows_processor.process_l3e_ul_sf') - @patch('imap_l3_processing.glows.glows_processor.process_l3e_hi') - @patch('imap_l3_processing.glows.glows_processor.process_l3e_lo') - @patch('imap_l3_processing.glows.glows_processor.compute_glows_flags_for_repoint') - @patch('imap_l3_processing.glows.glows_processor.get_pointing_date_range') - @patch('imap_l3_processing.glows.glows_processor.process_l3d') - def test_process_l3e_invoked_from_top_level_process(self, mock_process_l3d, mock_get_pointing_date_range, mock_compute_flags, - mock_process_lo, mock_process_hi, mock_process_ul_sf, mock_process_ul_hf, - mock_get_lo_pivot_angles, mock_determine_spacecraft_info): + @patch( + "imap_l3_processing.glows.glows_processor.determine_spacecraft_info_using_predict_if_needed" + ) + @patch("imap_l3_processing.glows.glows_processor.get_lo_pivot_angles") + @patch("imap_l3_processing.glows.glows_processor.process_l3e_ul_hf") + @patch("imap_l3_processing.glows.glows_processor.process_l3e_ul_sf") + @patch("imap_l3_processing.glows.glows_processor.process_l3e_hi") + @patch("imap_l3_processing.glows.glows_processor.process_l3e_lo") + @patch("imap_l3_processing.glows.glows_processor.compute_glows_flags_for_repoint") + @patch("imap_l3_processing.glows.glows_processor.get_pointing_date_range") + @patch("imap_l3_processing.glows.glows_processor.process_l3d") + def test_process_l3e_invoked_from_top_level_process( + self, + mock_process_l3d, + mock_get_pointing_date_range, + mock_compute_flags, + mock_process_lo, + mock_process_hi, + mock_process_ul_sf, + mock_process_ul_hf, + mock_get_lo_pivot_angles, + mock_determine_spacecraft_info, + ): mock_get_lo_pivot_angles.return_value = { 2902: LoPivotAngle("l1b_2902", 90), 2905: LoPivotAngle("l1b_2905", 90), } mock_compute_flags.return_value = 0 - self.mock_l3d_initializer.should_process_l3d.return_value = (sentinel.l3d_version, sentinel.glows_l3d_dependency, sentinel.old_l3d) - process_l3d_result = GlowsL3DProcessorOutput(sentinel.l3d_cdf_file_path, [sentinel.l3d_text_file_paths], - sentinel.last_processed_cr) + self.mock_l3d_initializer.should_process_l3d.return_value = ( + sentinel.l3d_version, + sentinel.glows_l3d_dependency, + sentinel.old_l3d, + ) + process_l3d_result = GlowsL3DProcessorOutput( + sentinel.l3d_cdf_file_path, + [sentinel.l3d_text_file_paths], + sentinel.last_processed_cr, + ) mock_process_l3d.return_value = process_l3d_result - mock_determine_spacecraft_info.return_value = sentinel.spacecraft_info, GlowsL3Flags.PREDICTIVE_EPHEMERIS, ["spice kernel"] + mock_determine_spacecraft_info.return_value = ( + sentinel.spacecraft_info, + GlowsL3Flags.PREDICTIVE_EPHEMERIS, + ["spice kernel"], + ) input_major_version = 5 @@ -1226,17 +1855,22 @@ def test_process_l3e_invoked_from_top_level_process(self, mock_process_l3d, mock GLOWS_L3E_ULTRA_HF_DESCRIPTOR: Version(input_major_version, 1), } ) - input_metadata = InputMetadata('glows', "l3b", datetime(2024, 10, 7), None, version=input_version_map) + input_metadata = InputMetadata( + "glows", "l3b", datetime(2024, 10, 7), None, version=input_version_map + ) - mock_get_pointing_date_range.return_value = datetime(2025, 1, 1), datetime(2025, 1, 2) + mock_get_pointing_date_range.return_value = ( + datetime(2025, 1, 1), + datetime(2025, 1, 2), + ) l3e_initialzer_repointings = GlowsL3eVersionsForRepointings( [2902, 2903, 2904, 2905, 2906, 2907], - {2902: Version(None, 1), 2903: Version(1,11)}, - {2902: Version(None, 1), 2904: Version(2,12)}, - {2902: Version(None, 1), 2905: Version(3,13)}, - {2902: Version(None, 1), 2906: Version(4,14)}, - {2902: Version(None, 1), 2907: Version(5,15)}, + {2902: Version(None, 1), 2903: Version(1, 11)}, + {2902: Version(None, 1), 2904: Version(2, 12)}, + {2902: Version(None, 1), 2905: Version(3, 13)}, + {2902: Version(None, 1), 2906: Version(4, 14)}, + {2902: Version(None, 1), 2907: Version(5, 15)}, ) l3e_dependencies = GlowsL3EDependencies( @@ -1264,17 +1898,25 @@ def test_process_l3e_invoked_from_top_level_process(self, mock_process_l3d, mock Path("path/to/some/pipeline_settings_file.csv"), Path("repoint.csv"), ) - self.mock_l3e_initializer.get_repointings_to_process.return_value = GlowsL3EInitializerOutput( - l3e_dependencies, - l3e_initialzer_repointings, - sentinel.l3d_cdf_file_path, - metakernel_with_predict_ephem=Mock(), - metakernel_without_predict_ephem=Mock(), + self.mock_l3e_initializer.get_repointings_to_process.return_value = ( + GlowsL3EInitializerOutput( + l3e_dependencies, + l3e_initialzer_repointings, + sentinel.l3d_cdf_file_path, + metakernel_with_predict_ephem=Mock(), + metakernel_without_predict_ephem=Mock(), + ) ) expected_lo_files = [ - ['imap_glows_l3e_survival-probability-lo_20250101-repoint02902_v001.cdf', sentinel.lo_dat_1], - ['imap_glows_l3e_survival-probability-lo_20250101-repoint02905_v003.0013.cdf', sentinel.lo_dat_2], + [ + "imap_glows_l3e_survival-probability-lo_20250101-repoint02902_v001.cdf", + sentinel.lo_dat_1, + ], + [ + "imap_glows_l3e_survival-probability-lo_20250101-repoint02905_v003.0013.cdf", + sentinel.lo_dat_2, + ], ] mock_process_lo.side_effect = expected_lo_files @@ -1358,7 +2000,6 @@ def test_process_l3e_invoked_from_top_level_process(self, mock_process_l3d, mock processor = GlowsProcessor(processing_input_collection, input_metadata) products = processor.process() - self.mock_l3e_initializer.get_repointings_to_process.assert_called_once_with( process_l3d_result, sentinel.old_l3d, @@ -1367,18 +2008,35 @@ def test_process_l3e_invoked_from_top_level_process(self, mock_process_l3d, mock self.mock_reprocess_info, ) self.assertEqual(expected_products, products) - self.mock_fetch_reprocess_info.assert_called_once_with(processing_input_collection) + self.mock_fetch_reprocess_info.assert_called_once_with( + processing_input_collection + ) - @patch('imap_l3_processing.glows.glows_processor.Processor.get_parent_file_names') - @patch('imap_l3_processing.glows.glows_processor.save_data') - @patch("imap_l3_processing.glows.glows_processor.GlowsL3EUltraData.convert_dat_to_glows_l3e_ul_product") + @patch("imap_l3_processing.glows.glows_processor.Processor.get_parent_file_names") + @patch("imap_l3_processing.glows.glows_processor.save_data") + @patch( + "imap_l3_processing.glows.glows_processor.GlowsL3EUltraData.convert_dat_to_glows_l3e_ul_product" + ) @patch("imap_l3_processing.glows.glows_processor.run") - @patch("imap_l3_processing.glows.glows_processor.determine_call_args_for_l3e_executable") + @patch( + "imap_l3_processing.glows.glows_processor.determine_call_args_for_l3e_executable" + ) @patch("imap_l3_processing.glows.glows_processor.shutil") - def test_process_l3e_ultra_sf(self, mock_shutil, mock_determine_call_args, mock_run, - mock_convert_dat_to_glows_l3e_ul_product, - mock_save_data, mock_get_parent_file_names): - mock_get_parent_file_names.return_value = ["l3d_file", "ancillary_1", "ancillary_2", "ancillary_3"] + def test_process_l3e_ultra_sf( + self, + mock_shutil, + mock_determine_call_args, + mock_run, + mock_convert_dat_to_glows_l3e_ul_product, + mock_save_data, + mock_get_parent_file_names, + ): + mock_get_parent_file_names.return_value = [ + "l3d_file", + "ancillary_1", + "ancillary_2", + "ancillary_3", + ] epoch_start_date = datetime(year=2024, month=10, day=7) epoch_end_date = datetime(year=2024, month=10, day=7, hour=23) @@ -1386,9 +2044,15 @@ def test_process_l3e_ultra_sf(self, mock_shutil, mock_determine_call_args, mock_ repointing = 20 version = Version(1, 12) - expected_input_metadata = InputMetadata('glows', "l3e", start_date=epoch_start_date, end_date=epoch_end_date, - version=VersionMap({GLOWS_L3E_ULTRA_SF_DESCRIPTOR:version}), descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, - repointing=repointing) + expected_input_metadata = InputMetadata( + "glows", + "l3e", + start_date=epoch_start_date, + end_date=epoch_end_date, + version=VersionMap({GLOWS_L3E_ULTRA_SF_DESCRIPTOR: version}), + descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + repointing=repointing, + ) ultra_args = ["20241007_000000", "date.001", "vx", "vy", "vz", "30.000"] @@ -1403,42 +2067,89 @@ def test_process_l3e_ultra_sf(self, mock_shutil, mock_determine_call_args, mock_ parent_file_names = ["l3d_file", "ancillary_1", "ancillary_2", "ancillary_3"] glows_flags = 4 spacecraft_info = Mock() - products = process_l3e_ul_sf(parent_file_names, repointing, epoch_start_date, epoch_delta, version, glows_flags, spacecraft_info) + products = process_l3e_ul_sf( + parent_file_names, + repointing, + epoch_start_date, + epoch_delta, + version, + glows_flags, + spacecraft_info, + ) expected_repointing_midpoint = epoch_start_date + epoch_delta - mock_determine_call_args.assert_called_once_with(epoch_start_date, expected_repointing_midpoint, 30, spacecraft_info=spacecraft_info) + mock_determine_call_args.assert_called_once_with( + epoch_start_date, + expected_repointing_midpoint, + 30, + spacecraft_info=spacecraft_info, + ) mock_run.assert_called_once_with(["./survProbUltra"] + ultra_args) output_data_path = Path("probSur.Imap.Ul_20241007_000000_date.001.dat") mock_convert_dat_to_glows_l3e_ul_product.assert_called_once_with( - expected_input_metadata, output_data_path, expected_repointing_midpoint, epoch_delta, call_args_object) + expected_input_metadata, + output_data_path, + expected_repointing_midpoint, + epoch_delta, + call_args_object, + ) expected_first_data_path = AncillaryFilePath( - "imap_glows_survival-probability-ul-sf-raw_20241007-repoint00020_v012.dat").construct_path() + "imap_glows_survival-probability-ul-sf-raw_20241007-repoint00020_v012.dat" + ).construct_path() - mock_shutil.move.assert_called_once_with(output_data_path, expected_first_data_path) + mock_shutil.move.assert_called_once_with( + output_data_path, expected_first_data_path + ) mock_save_data.assert_called_once_with(sentinel.ultra_data_1) - survival_data_product: GlowsL3EUltraData = mock_save_data.call_args_list[0].args[0] - self.assertEqual(["l3d_file", "ancillary_1", "ancillary_2", "ancillary_3"], - survival_data_product.parent_file_names) - np.testing.assert_array_equal(survival_data_product.glows_flags, np.array([glows_flags], dtype=np.uint16)) + survival_data_product: GlowsL3EUltraData = mock_save_data.call_args_list[ + 0 + ].args[0] + self.assertEqual( + ["l3d_file", "ancillary_1", "ancillary_2", "ancillary_3"], + survival_data_product.parent_file_names, + ) + np.testing.assert_array_equal( + survival_data_product.glows_flags, np.array([glows_flags], dtype=np.uint16) + ) - self.assertEqual(products, ["imap_glows_l3e_survival-probability-ul-sf_20241007-repoint00020_v001.0012.cdf", - expected_first_data_path]) + self.assertEqual( + products, + [ + "imap_glows_l3e_survival-probability-ul-sf_20241007-repoint00020_v001.0012.cdf", + expected_first_data_path, + ], + ) - @patch('imap_l3_processing.glows.glows_processor.Processor.get_parent_file_names') - @patch('imap_l3_processing.glows.glows_processor.save_data') - @patch("imap_l3_processing.glows.glows_processor.GlowsL3EUltraData.convert_dat_to_glows_l3e_ul_product") + @patch("imap_l3_processing.glows.glows_processor.Processor.get_parent_file_names") + @patch("imap_l3_processing.glows.glows_processor.save_data") + @patch( + "imap_l3_processing.glows.glows_processor.GlowsL3EUltraData.convert_dat_to_glows_l3e_ul_product" + ) @patch("imap_l3_processing.glows.glows_processor.run") - @patch("imap_l3_processing.glows.glows_processor.determine_call_args_for_l3e_executable") + @patch( + "imap_l3_processing.glows.glows_processor.determine_call_args_for_l3e_executable" + ) @patch("imap_l3_processing.glows.glows_processor.shutil") - def test_process_l3e_ultra_hf(self, mock_shutil, mock_determine_call_args, mock_run, - mock_convert_dat_to_glows_l3e_ul_product, - mock_save_data, mock_get_parent_file_names): - mock_get_parent_file_names.return_value = ["l3d_file", "ancillary_1", "ancillary_2", "ancillary_3"] + def test_process_l3e_ultra_hf( + self, + mock_shutil, + mock_determine_call_args, + mock_run, + mock_convert_dat_to_glows_l3e_ul_product, + mock_save_data, + mock_get_parent_file_names, + ): + mock_get_parent_file_names.return_value = [ + "l3d_file", + "ancillary_1", + "ancillary_2", + "ancillary_3", + ] epoch_start_date = datetime(year=2024, month=10, day=7) epoch_end_date = datetime(year=2024, month=10, day=7, hour=23) @@ -1446,9 +2157,15 @@ def test_process_l3e_ultra_hf(self, mock_shutil, mock_determine_call_args, mock_ repointing = 20 version = Version(1, 12) - input_metadata = InputMetadata('glows', "l3e", start_date=epoch_start_date, end_date=epoch_end_date, - version=VersionMap({GLOWS_L3E_ULTRA_HF_DESCRIPTOR: version}), descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, - repointing=repointing) + input_metadata = InputMetadata( + "glows", + "l3e", + start_date=epoch_start_date, + end_date=epoch_end_date, + version=VersionMap({GLOWS_L3E_ULTRA_HF_DESCRIPTOR: version}), + descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, + repointing=repointing, + ) spacecraft_frame_spacecraft_info = GlowsL3eSpacecraftInfo( spacecraft_radius=500, @@ -1465,7 +2182,7 @@ def test_process_l3e_ultra_hf(self, mock_shutil, mock_determine_call_args, mock_ formatted_date="20241007_000000", decimal_date="date.001", spacecraft_info=spacecraft_frame_spacecraft_info, - elongation=30 + elongation=30, ) expected_rest_frame_args = GlowsL3eCallArguments( @@ -1481,7 +2198,7 @@ def test_process_l3e_ultra_hf(self, mock_shutil, mock_determine_call_args, mock_ spin_axis_longitude=240, spin_axis_latitude=3, ), - elongation=30 + elongation=30, ) mock_convert_dat_to_glows_l3e_ul_product.return_value = sentinel.ultra_data_hf @@ -1490,47 +2207,91 @@ def test_process_l3e_ultra_hf(self, mock_shutil, mock_determine_call_args, mock_ parent_file_names = ["l3d_file", "ancillary_1", "ancillary_2", "ancillary_3"] glows_flags = 8 - products = process_l3e_ul_hf(parent_file_names, repointing, epoch_start_date, epoch_delta, version, glows_flags, spacecraft_info=spacecraft_frame_spacecraft_info) + products = process_l3e_ul_hf( + parent_file_names, + repointing, + epoch_start_date, + epoch_delta, + version, + glows_flags, + spacecraft_info=spacecraft_frame_spacecraft_info, + ) expected_repointing_midpoint = epoch_start_date + epoch_delta - mock_determine_call_args.assert_called_once_with(epoch_start_date, expected_repointing_midpoint, 30, spacecraft_info=spacecraft_frame_spacecraft_info) + mock_determine_call_args.assert_called_once_with( + epoch_start_date, + expected_repointing_midpoint, + 30, + spacecraft_info=spacecraft_frame_spacecraft_info, + ) - mock_run.assert_called_once_with(["./survProbUltra"] + expected_rest_frame_args.to_argument_list()) + mock_run.assert_called_once_with( + ["./survProbUltra"] + expected_rest_frame_args.to_argument_list() + ) output_data_path = Path("probSur.Imap.Ul.V0_20241007_000000_date.001.dat") mock_convert_dat_to_glows_l3e_ul_product.assert_called_once_with( - input_metadata, output_data_path, expected_repointing_midpoint, epoch_delta, expected_rest_frame_args) + input_metadata, + output_data_path, + expected_repointing_midpoint, + epoch_delta, + expected_rest_frame_args, + ) expected_first_data_path = AncillaryFilePath( - "imap_glows_survival-probability-ul-hf-raw_20241007-repoint00020_v012.dat").construct_path() + "imap_glows_survival-probability-ul-hf-raw_20241007-repoint00020_v012.dat" + ).construct_path() - mock_shutil.move.assert_called_once_with(output_data_path, expected_first_data_path) + mock_shutil.move.assert_called_once_with( + output_data_path, expected_first_data_path + ) mock_save_data.assert_called_once_with(sentinel.ultra_data_hf) - survival_data_product: GlowsL3EUltraData = mock_save_data.call_args_list[0].args[0] - self.assertEqual(["l3d_file", "ancillary_1", "ancillary_2", "ancillary_3"], - survival_data_product.parent_file_names) - np.testing.assert_array_equal(survival_data_product.glows_flags, np.array([glows_flags], dtype=np.uint16)) + survival_data_product: GlowsL3EUltraData = mock_save_data.call_args_list[ + 0 + ].args[0] + self.assertEqual( + ["l3d_file", "ancillary_1", "ancillary_2", "ancillary_3"], + survival_data_product.parent_file_names, + ) + np.testing.assert_array_equal( + survival_data_product.glows_flags, np.array([glows_flags], dtype=np.uint16) + ) - self.assertEqual(products, ["imap_glows_l3e_survival-probability-ul-hf_20241007-repoint00020_v001.0012.cdf", - expected_first_data_path]) + self.assertEqual( + products, + [ + "imap_glows_l3e_survival-probability-ul-hf_20241007-repoint00020_v001.0012.cdf", + expected_first_data_path, + ], + ) - @patch('imap_l3_processing.glows.glows_processor.save_data') - @patch("imap_l3_processing.glows.glows_processor.GlowsL3EHiData.convert_dat_to_glows_l3e_hi_product") + @patch("imap_l3_processing.glows.glows_processor.save_data") + @patch( + "imap_l3_processing.glows.glows_processor.GlowsL3EHiData.convert_dat_to_glows_l3e_hi_product" + ) @patch("imap_l3_processing.glows.glows_processor.run") - @patch("imap_l3_processing.glows.glows_processor.determine_call_args_for_l3e_executable") + @patch( + "imap_l3_processing.glows.glows_processor.determine_call_args_for_l3e_executable" + ) @patch("imap_l3_processing.glows.glows_processor.shutil") - def test_process_l3e_hi(self, mock_shutil, mock_determine_call_args, - mock_run, mock_convert_dat_to_glows_l3e_hi_product, mock_save_data): + def test_process_l3e_hi( + self, + mock_shutil, + mock_determine_call_args, + mock_run, + mock_convert_dat_to_glows_l3e_hi_product, + mock_save_data, + ): test_cases = [ ("90", 90, GLOWS_L3E_HI_90_DESCRIPTOR), - ("45", 135, GLOWS_L3E_HI_45_DESCRIPTOR) + ("45", 135, GLOWS_L3E_HI_45_DESCRIPTOR), ] for descriptor_elongation, elongation, descriptor in test_cases: - with (self.subTest(elongation=elongation)): + with self.subTest(elongation=elongation): mock_determine_call_args.reset_mock() mock_run.reset_mock() mock_convert_dat_to_glows_l3e_hi_product.reset_mock() @@ -1544,80 +2305,127 @@ def test_process_l3e_hi(self, mock_shutil, mock_determine_call_args, version = Version(1, 12) expected_input_metadata = InputMetadata( - instrument='glows', data_level="l3e", start_date=epoch_start_date, end_date=epoch_end_date, - version=VersionMap({descriptor: version}), descriptor=f'survival-probability-hi-{descriptor_elongation}', - repointing=repointing) + instrument="glows", + data_level="l3e", + start_date=epoch_start_date, + end_date=epoch_end_date, + version=VersionMap({descriptor: version}), + descriptor=f"survival-probability-hi-{descriptor_elongation}", + repointing=repointing, + ) l3e_dependencies = MagicMock(spec=GlowsL3EDependencies) - l3e_dependencies.pipeline_settings = {'start_cr': 2090} - l3e_dependencies.repointing_file = get_test_data_path("fake_2_day_repointing_on_may18_file.csv") + l3e_dependencies.pipeline_settings = {"start_cr": 2090} + l3e_dependencies.repointing_file = get_test_data_path( + "fake_2_day_repointing_on_may18_file.csv" + ) - hi_args = ["20241007_000000", "date.001", "vx", "vy", "vz", descriptor_elongation] + hi_args = [ + "20241007_000000", + "date.001", + "vx", + "vy", + "vz", + descriptor_elongation, + ] mock_call_args_object = MagicMock(spec=GlowsL3eCallArguments) mock_call_args_object.to_argument_list.return_value = hi_args mock_determine_call_args.return_value = mock_call_args_object - mock_convert_dat_to_glows_l3e_hi_product.return_value = sentinel.hi_data_1 + mock_convert_dat_to_glows_l3e_hi_product.return_value = ( + sentinel.hi_data_1 + ) saved_cdf_path = Path( - f"imap_glows_l3e_survival-probability-hi-{descriptor_elongation}_20241007-repoint00020_v001.0012.cdf") + f"imap_glows_l3e_survival-probability-hi-{descriptor_elongation}_20241007-repoint00020_v001.0012.cdf" + ) mock_save_data.return_value = saved_cdf_path - parent_file_names = ["some_l3e_hi_parent.dat", "some_repointing_file.repoint.csv"] + parent_file_names = [ + "some_l3e_hi_parent.dat", + "some_repointing_file.repoint.csv", + ] glows_flags = 16 spacecraft_info = Mock() - products = process_l3e_hi(parent_file_names, repointing, epoch_start_date, epoch_delta, elongation, - version, glows_flags, spacecraft_info=spacecraft_info) + products = process_l3e_hi( + parent_file_names, + repointing, + epoch_start_date, + epoch_delta, + elongation, + version, + glows_flags, + spacecraft_info=spacecraft_info, + ) expected_repointing_midpoint = epoch_start_date + epoch_delta - mock_determine_call_args.assert_called_once_with(epoch_start_date, expected_repointing_midpoint, - float(elongation), spacecraft_info=spacecraft_info) + mock_determine_call_args.assert_called_once_with( + epoch_start_date, + expected_repointing_midpoint, + float(elongation), + spacecraft_info=spacecraft_info, + ) mock_run.assert_called_once_with(["./survProbHi"] + hi_args) first_output_data_path = Path( - f"probSur.Imap.Hi_20241007_000000_date.001_{descriptor_elongation[:5]}.dat") + f"probSur.Imap.Hi_20241007_000000_date.001_{descriptor_elongation[:5]}.dat" + ) mock_convert_dat_to_glows_l3e_hi_product.assert_called_once_with( expected_input_metadata, first_output_data_path, expected_repointing_midpoint, epoch_delta, - mock_call_args_object + mock_call_args_object, ) mock_save_data.assert_called_once_with(sentinel.hi_data_1) - survival_data_product: GlowsL3EHiData = mock_save_data.call_args_list[0].args[0] - self.assertEqual(parent_file_names, survival_data_product.parent_file_names) - np.testing.assert_array_equal(survival_data_product.glows_flags, - np.array([glows_flags], dtype=np.uint16)) + survival_data_product: GlowsL3EHiData = mock_save_data.call_args_list[ + 0 + ].args[0] + self.assertEqual( + parent_file_names, survival_data_product.parent_file_names + ) + np.testing.assert_array_equal( + survival_data_product.glows_flags, + np.array([glows_flags], dtype=np.uint16), + ) expected_output_data_path = AncillaryFilePath( f"imap_glows_survival-probability-hi-{descriptor_elongation}-raw_20241007-repoint00020_v012.dat" ).construct_path() - mock_shutil.move.assert_called_once_with(first_output_data_path, expected_output_data_path) + mock_shutil.move.assert_called_once_with( + first_output_data_path, expected_output_data_path + ) - expected_products = [ - saved_cdf_path, - expected_output_data_path - ] + expected_products = [saved_cdf_path, expected_output_data_path] self.assertEqual(expected_products, products) - @patch('imap_l3_processing.glows.glows_processor.save_data') - @patch("imap_l3_processing.glows.glows_processor.GlowsL3ELoData.convert_dat_to_glows_l3e_lo_product") + @patch("imap_l3_processing.glows.glows_processor.save_data") + @patch( + "imap_l3_processing.glows.glows_processor.GlowsL3ELoData.convert_dat_to_glows_l3e_lo_product" + ) @patch("imap_l3_processing.glows.glows_processor.run") - @patch("imap_l3_processing.glows.glows_processor.determine_call_args_for_l3e_executable") + @patch( + "imap_l3_processing.glows.glows_processor.determine_call_args_for_l3e_executable" + ) @patch("imap_l3_processing.glows.glows_processor.shutil") - def test_process_l3e_lo(self, mock_shutil, - mock_determine_call_args, - mock_run, mock_convert_dat_to_glows_l3e_lo_product, mock_save_data): + def test_process_l3e_lo( + self, + mock_shutil, + mock_determine_call_args, + mock_run, + mock_convert_dat_to_glows_l3e_lo_product, + mock_save_data, + ): test_cases = [(75, "75.00"), (105, "105.0")] for elongation, elongation_filename in test_cases: - with (self.subTest(elongation)): + with self.subTest(elongation): epoch_start_date = datetime(year=2024, month=10, day=7) epoch_end_date = datetime(year=2024, month=10, day=7, hour=23) epoch_delta = (epoch_end_date - epoch_start_date) / 2 @@ -1625,10 +2433,23 @@ def test_process_l3e_lo(self, mock_shutil, version = Version(1, 12) expected_input_metadata = InputMetadata( - instrument='glows', data_level="l3e", start_date=epoch_start_date, end_date=epoch_end_date, - version=VersionMap({GLOWS_L3E_LO_DESCRIPTOR: version}), descriptor=GLOWS_L3E_LO_DESCRIPTOR, repointing=repointing) + instrument="glows", + data_level="l3e", + start_date=epoch_start_date, + end_date=epoch_end_date, + version=VersionMap({GLOWS_L3E_LO_DESCRIPTOR: version}), + descriptor=GLOWS_L3E_LO_DESCRIPTOR, + repointing=repointing, + ) - lo_call_args = ["20241007_000000", "date.100", "vx", "vy", "vz", f"{elongation:.3f}"] + lo_call_args = [ + "20241007_000000", + "date.100", + "vx", + "vy", + "vz", + f"{elongation:.3f}", + ] spacecraft_info = GlowsL3eSpacecraftInfo( spacecraft_radius=np.float32(100.0), @@ -1653,45 +2474,77 @@ def test_process_l3e_lo(self, mock_shutil, lo_data_1 = Mock() mock_convert_dat_to_glows_l3e_lo_product.return_value = lo_data_1 - output_cdf_path = Path("imap_glows_l3e_survival-probability-lo_20241007-repoint00020_v001.0012.cdf") + output_cdf_path = Path( + "imap_glows_l3e_survival-probability-lo_20241007-repoint00020_v001.0012.cdf" + ) mock_save_data.return_value = output_cdf_path - parent_file_names = ["l3d_file", "ancillary_1", "ancillary_2", "ancillary_3"] + parent_file_names = [ + "l3d_file", + "ancillary_1", + "ancillary_2", + "ancillary_3", + ] glows_flags = 32 - products = process_l3e_lo(parent_file_names, repointing, epoch_start_date, epoch_delta, elongation, - version, glows_flags, spacecraft_info=spacecraft_info) + products = process_l3e_lo( + parent_file_names, + repointing, + epoch_start_date, + epoch_delta, + elongation, + version, + glows_flags, + spacecraft_info=spacecraft_info, + ) expected_repointing_midpoint = epoch_start_date + epoch_delta - mock_determine_call_args.assert_called_once_with(epoch_start_date, expected_repointing_midpoint, - elongation, spacecraft_info=spacecraft_info) + mock_determine_call_args.assert_called_once_with( + epoch_start_date, + expected_repointing_midpoint, + elongation, + spacecraft_info=spacecraft_info, + ) mock_run.assert_called_once_with(["./survProbLo"] + lo_call_args) - first_output_file_path = Path(f"probSur.Imap.Lo_20241007_000000_date.100_{elongation_filename}.dat") + first_output_file_path = Path( + f"probSur.Imap.Lo_20241007_000000_date.100_{elongation_filename}.dat" + ) - mock_convert_dat_to_glows_l3e_lo_product.assert_called_once_with(expected_input_metadata, - first_output_file_path, - expected_repointing_midpoint, - epoch_delta, - elongation, l3e_args) + mock_convert_dat_to_glows_l3e_lo_product.assert_called_once_with( + expected_input_metadata, + first_output_file_path, + expected_repointing_midpoint, + epoch_delta, + elongation, + l3e_args, + ) expected_first_output_file_path = AncillaryFilePath( "imap_glows_survival-probability-lo-raw_20241007-repoint00020_v012.dat" ).construct_path() - mock_shutil.move.assert_called_once_with(first_output_file_path, expected_first_output_file_path) + mock_shutil.move.assert_called_once_with( + first_output_file_path, expected_first_output_file_path + ) mock_save_data.assert_called_once_with(lo_data_1) - survival_data_product: GlowsL3ELoData = mock_save_data.call_args_list[0].args[0] - self.assertEqual(parent_file_names, - survival_data_product.parent_file_names) - np.testing.assert_array_equal(survival_data_product.glows_flags, - np.array([glows_flags], dtype=np.uint16)) + survival_data_product: GlowsL3ELoData = mock_save_data.call_args_list[ + 0 + ].args[0] + self.assertEqual( + parent_file_names, survival_data_product.parent_file_names + ) + np.testing.assert_array_equal( + survival_data_product.glows_flags, + np.array([glows_flags], dtype=np.uint16), + ) - self.assertEqual([output_cdf_path, - expected_first_output_file_path], products) + self.assertEqual( + [output_cdf_path, expected_first_output_file_path], products + ) mock_shutil.reset_mock() mock_determine_call_args.reset_mock() @@ -1699,47 +2552,59 @@ def test_process_l3e_lo(self, mock_shutil, mock_convert_dat_to_glows_l3e_lo_product.reset_mock() mock_save_data.reset_mock() - @patch('imap_l3_processing.glows.glows_processor.compute_glows_flags_for_repoint') - @patch('imap_l3_processing.glows.glows_processor.get_lo_pivot_angles') - @patch('imap_l3_processing.glows.glows_processor.get_pointing_date_range') - @patch('imap_l3_processing.glows.glows_processor.process_l3e_hi') - @patch('imap_l3_processing.glows.glows_processor.process_l3e_lo') - @patch('imap_l3_processing.glows.glows_processor.process_l3e_ul_hf') - @patch('imap_l3_processing.glows.glows_processor.process_l3e_ul_sf') - @patch('imap_l3_processing.glows.glows_processor.determine_spacecraft_info_using_predict_if_needed') - def test_process_l3e_skips_repointing_on_exception(self, mock_determine_spacecraft_info, mock_process_ultra_sf, mock_process_ultra_hf, - mock_process_lo, - mock_process_hi, mock_get_pointing_date_range, - mock_get_lo_pivot_angles, - mock_compute_glows_flags_for_repoint): + @patch("imap_l3_processing.glows.glows_processor.compute_glows_flags_for_repoint") + @patch("imap_l3_processing.glows.glows_processor.get_lo_pivot_angles") + @patch("imap_l3_processing.glows.glows_processor.get_pointing_date_range") + @patch("imap_l3_processing.glows.glows_processor.process_l3e_hi") + @patch("imap_l3_processing.glows.glows_processor.process_l3e_lo") + @patch("imap_l3_processing.glows.glows_processor.process_l3e_ul_hf") + @patch("imap_l3_processing.glows.glows_processor.process_l3e_ul_sf") + @patch( + "imap_l3_processing.glows.glows_processor.determine_spacecraft_info_using_predict_if_needed" + ) + def test_process_l3e_skips_repointing_on_exception( + self, + mock_determine_spacecraft_info, + mock_process_ultra_sf, + mock_process_ultra_hf, + mock_process_lo, + mock_process_hi, + mock_get_pointing_date_range, + mock_get_lo_pivot_angles, + mock_compute_glows_flags_for_repoint, + ): mock_compute_glows_flags_for_repoint.return_value = 0 mock_process_hi.side_effect = [ - ValueError("Failed to generate hi"), [Path('path/to/first_hi-45_l3e')], - [Path('path/to/second_hi-90_l3e')], [Path('path/to/second_hi-45_l3e')], - [Path('path/to/third_hi-90_l3e')], [Path('path/to/third_hi-45_l3e')], - [Path('path/to/fourth_hi-90_l3e')], [Path('path/to/fourth_hi-45_l3e')], + ValueError("Failed to generate hi"), + [Path("path/to/first_hi-45_l3e")], + [Path("path/to/second_hi-90_l3e")], + [Path("path/to/second_hi-45_l3e")], + [Path("path/to/third_hi-90_l3e")], + [Path("path/to/third_hi-45_l3e")], + [Path("path/to/fourth_hi-90_l3e")], + [Path("path/to/fourth_hi-45_l3e")], ] mock_process_lo.side_effect = [ - [Path('path/to/first_lo_l3e')], + [Path("path/to/first_lo_l3e")], ValueError("Failed to generate lo"), - [Path('path/to/third_lo_l3e')], - [Path('path/to/fourth_lo_l3e')], + [Path("path/to/third_lo_l3e")], + [Path("path/to/fourth_lo_l3e")], ] mock_process_ultra_sf.side_effect = [ - [Path('path/to/first_ultra_l3e')], - [Path('path/to/second_ultra_l3e')], + [Path("path/to/first_ultra_l3e")], + [Path("path/to/second_ultra_l3e")], ValueError("Failed to generate ultra!"), - [Path('path/to/fourth_ultra_l3e')] + [Path("path/to/fourth_ultra_l3e")], ] mock_process_ultra_hf.side_effect = [ - [Path('path/to/first_ultra_l3e_hf')], - [Path('path/to/second_ultra_l3e_hf')], - [Path('path/to/third_ultra_l3e_hf')], - ValueError("Failed to generate ultra!") + [Path("path/to/first_ultra_l3e_hf")], + [Path("path/to/second_ultra_l3e_hf")], + [Path("path/to/third_ultra_l3e_hf")], + ValueError("Failed to generate ultra!"), ] mock_get_lo_pivot_angles.return_value = { 24: LoPivotAngle(parent_filename="l1b_nhk_24", pivot_angle=124), @@ -1749,22 +2614,22 @@ def test_process_l3e_skips_repointing_on_exception(self, mock_determine_spacecra } expected_l3e_products = [ - Path('path/to/first_lo_l3e'), - Path('path/to/first_hi-45_l3e'), - Path('path/to/first_ultra_l3e'), - Path('path/to/first_ultra_l3e_hf'), - Path('path/to/second_hi-90_l3e'), - Path('path/to/second_hi-45_l3e'), - Path('path/to/second_ultra_l3e'), - Path('path/to/second_ultra_l3e_hf'), - Path('path/to/third_lo_l3e'), - Path('path/to/third_hi-90_l3e'), - Path('path/to/third_hi-45_l3e'), - Path('path/to/third_ultra_l3e_hf'), - Path('path/to/fourth_lo_l3e'), - Path('path/to/fourth_hi-90_l3e'), - Path('path/to/fourth_hi-45_l3e'), - Path('path/to/fourth_ultra_l3e'), + Path("path/to/first_lo_l3e"), + Path("path/to/first_hi-45_l3e"), + Path("path/to/first_ultra_l3e"), + Path("path/to/first_ultra_l3e_hf"), + Path("path/to/second_hi-90_l3e"), + Path("path/to/second_hi-45_l3e"), + Path("path/to/second_ultra_l3e"), + Path("path/to/second_ultra_l3e_hf"), + Path("path/to/third_lo_l3e"), + Path("path/to/third_hi-90_l3e"), + Path("path/to/third_hi-45_l3e"), + Path("path/to/third_ultra_l3e_hf"), + Path("path/to/fourth_lo_l3e"), + Path("path/to/fourth_hi-90_l3e"), + Path("path/to/fourth_hi-45_l3e"), + Path("path/to/fourth_ultra_l3e"), ] start_epoch_1 = datetime(2020, 1, 1) @@ -1794,7 +2659,11 @@ def test_process_l3e_skips_repointing_on_exception(self, mock_determine_spacecra (sentinel.spacecraft_info24, GlowsL3Flags.NONE, []), (sentinel.spacecraft_info25, GlowsL3Flags.NONE, []), (sentinel.spacecraft_info26, GlowsL3Flags.NONE, []), - (sentinel.spacecraft_info27, GlowsL3Flags.PREDICTIVE_EPHEMERIS, ["predict"]), + ( + sentinel.spacecraft_info27, + GlowsL3Flags.PREDICTIVE_EPHEMERIS, + ["predict"], + ), ] mock_dependencies = Mock() @@ -1812,11 +2681,36 @@ def test_process_l3e_skips_repointing_on_exception(self, mock_determine_spacecra dependencies=mock_dependencies, repointings=GlowsL3eVersionsForRepointings( repointing_numbers=[24, 25, 26, 27], - hi_90_repointings={24: Version(None, 1), 25: Version(None, 1), 26: Version(None, 1), 27: Version(None, 1)}, - hi_45_repointings={24: Version(None, 2), 25: Version(None, 2), 26: Version(None, 2), 27: Version(None, 2)}, - lo_repointings={24: Version(None, 3), 25: Version(None, 3), 26: Version(None, 3), 27: Version(None, 3)}, - ultra_hf_repointings={24: Version(None, 4), 25: Version(None, 4), 26: Version(None, 4), 27: Version(None, 4)}, - ultra_sf_repointings={24: Version(None, 4), 25: Version(None, 4), 26: Version(None, 4), 27: Version(None, 4)}, + hi_90_repointings={ + 24: Version(None, 1), + 25: Version(None, 1), + 26: Version(None, 1), + 27: Version(None, 1), + }, + hi_45_repointings={ + 24: Version(None, 2), + 25: Version(None, 2), + 26: Version(None, 2), + 27: Version(None, 2), + }, + lo_repointings={ + 24: Version(None, 3), + 25: Version(None, 3), + 26: Version(None, 3), + 27: Version(None, 3), + }, + ultra_hf_repointings={ + 24: Version(None, 4), + 25: Version(None, 4), + 26: Version(None, 4), + 27: Version(None, 4), + }, + ultra_sf_repointings={ + 24: Version(None, 4), + 25: Version(None, 4), + 26: Version(None, 4), + 27: Version(None, 4), + }, ), l3d_cdf_path=Path("path/to/l3d.cdf"), metakernel_with_predict_ephem=metakernel_with_predict_ephem, @@ -1825,38 +2719,220 @@ def test_process_l3e_skips_repointing_on_exception(self, mock_determine_spacecra actual_l3e_products = process_l3e(initializer_data) - mock_get_pointing_date_range.assert_has_calls([call(24), call(25), call(26), call(27)]) - - mock_process_hi.assert_has_calls([ - call(hi_parents, 24, start_epoch_1, epoch_delta_1, 90, Version(None, 1), 0, sentinel.spacecraft_info24), - call(hi_parents, 24, start_epoch_1, epoch_delta_1, 135, Version(None, 2), 0, sentinel.spacecraft_info24), - call(hi_parents, 25, start_epoch_2, epoch_delta_2, 90, Version(None, 1), 0, sentinel.spacecraft_info25), - call(hi_parents, 25, start_epoch_2, epoch_delta_2, 135, Version(None, 2), 0, sentinel.spacecraft_info25), - call(hi_parents, 26, start_epoch_3, epoch_delta_3, 90, Version(None, 1), 0, sentinel.spacecraft_info26), - call(hi_parents, 26, start_epoch_3, epoch_delta_3, 135, Version(None, 2), 0, sentinel.spacecraft_info26), - call(hi_parents + ["predict"], 27, start_epoch_4, epoch_delta_4, 90, Version(None, 1), GlowsL3Flags.PREDICTIVE_EPHEMERIS, sentinel.spacecraft_info27), - call(hi_parents + ["predict"], 27, start_epoch_4, epoch_delta_4, 135, Version(None, 2), GlowsL3Flags.PREDICTIVE_EPHEMERIS, sentinel.spacecraft_info27), - ]) - mock_process_lo.assert_has_calls([ - call(lo_parents + ["l1b_nhk_24"], 24, start_epoch_1, epoch_delta_1, 124, Version(None, 3), 0, sentinel.spacecraft_info24), - call(lo_parents + ["l1b_nhk_25"], 25, start_epoch_2, epoch_delta_2, 125, Version(None, 3), 0, sentinel.spacecraft_info25), - call(lo_parents + ["l1b_nhk_26"], 26, start_epoch_3, epoch_delta_3, 126, Version(None, 3), 0, sentinel.spacecraft_info26), - call(lo_parents + ["predict"], 27, start_epoch_4, epoch_delta_4, 90, Version(None, 3), GlowsL3Flags.PREDICTIVE_EPHEMERIS, sentinel.spacecraft_info27), - ]) - - mock_process_ultra_sf.assert_has_calls([ - call(ultra_parents, 24, start_epoch_1, epoch_delta_1, Version(None, 4), 0, sentinel.spacecraft_info24), - call(ultra_parents, 25, start_epoch_2, epoch_delta_2, Version(None, 4), 0, sentinel.spacecraft_info25), - call(ultra_parents, 26, start_epoch_3, epoch_delta_3, Version(None, 4), 0, sentinel.spacecraft_info26), - call(ultra_parents + ["predict"], 27, start_epoch_4, epoch_delta_4, Version(None, 4), GlowsL3Flags.PREDICTIVE_EPHEMERIS, sentinel.spacecraft_info27), - ]) - - mock_process_ultra_hf.assert_has_calls([ - call(ultra_parents, 24, start_epoch_1, epoch_delta_1, Version(None, 4), 0, sentinel.spacecraft_info24), - call(ultra_parents, 25, start_epoch_2, epoch_delta_2, Version(None, 4), 0, sentinel.spacecraft_info25), - call(ultra_parents, 26, start_epoch_3, epoch_delta_3, Version(None, 4), 0, sentinel.spacecraft_info26), - call(ultra_parents + ["predict"], 27, start_epoch_4, epoch_delta_4, Version(None, 4), GlowsL3Flags.PREDICTIVE_EPHEMERIS, sentinel.spacecraft_info27), - ]) + mock_get_pointing_date_range.assert_has_calls( + [call(24), call(25), call(26), call(27)] + ) + + mock_process_hi.assert_has_calls( + [ + call( + hi_parents, + 24, + start_epoch_1, + epoch_delta_1, + 90, + Version(None, 1), + 0, + sentinel.spacecraft_info24, + ), + call( + hi_parents, + 24, + start_epoch_1, + epoch_delta_1, + 135, + Version(None, 2), + 0, + sentinel.spacecraft_info24, + ), + call( + hi_parents, + 25, + start_epoch_2, + epoch_delta_2, + 90, + Version(None, 1), + 0, + sentinel.spacecraft_info25, + ), + call( + hi_parents, + 25, + start_epoch_2, + epoch_delta_2, + 135, + Version(None, 2), + 0, + sentinel.spacecraft_info25, + ), + call( + hi_parents, + 26, + start_epoch_3, + epoch_delta_3, + 90, + Version(None, 1), + 0, + sentinel.spacecraft_info26, + ), + call( + hi_parents, + 26, + start_epoch_3, + epoch_delta_3, + 135, + Version(None, 2), + 0, + sentinel.spacecraft_info26, + ), + call( + hi_parents + ["predict"], + 27, + start_epoch_4, + epoch_delta_4, + 90, + Version(None, 1), + GlowsL3Flags.PREDICTIVE_EPHEMERIS, + sentinel.spacecraft_info27, + ), + call( + hi_parents + ["predict"], + 27, + start_epoch_4, + epoch_delta_4, + 135, + Version(None, 2), + GlowsL3Flags.PREDICTIVE_EPHEMERIS, + sentinel.spacecraft_info27, + ), + ] + ) + mock_process_lo.assert_has_calls( + [ + call( + lo_parents + ["l1b_nhk_24"], + 24, + start_epoch_1, + epoch_delta_1, + 124, + Version(None, 3), + 0, + sentinel.spacecraft_info24, + ), + call( + lo_parents + ["l1b_nhk_25"], + 25, + start_epoch_2, + epoch_delta_2, + 125, + Version(None, 3), + 0, + sentinel.spacecraft_info25, + ), + call( + lo_parents + ["l1b_nhk_26"], + 26, + start_epoch_3, + epoch_delta_3, + 126, + Version(None, 3), + 0, + sentinel.spacecraft_info26, + ), + call( + lo_parents + ["predict"], + 27, + start_epoch_4, + epoch_delta_4, + 90, + Version(None, 3), + GlowsL3Flags.PREDICTIVE_EPHEMERIS, + sentinel.spacecraft_info27, + ), + ] + ) + + mock_process_ultra_sf.assert_has_calls( + [ + call( + ultra_parents, + 24, + start_epoch_1, + epoch_delta_1, + Version(None, 4), + 0, + sentinel.spacecraft_info24, + ), + call( + ultra_parents, + 25, + start_epoch_2, + epoch_delta_2, + Version(None, 4), + 0, + sentinel.spacecraft_info25, + ), + call( + ultra_parents, + 26, + start_epoch_3, + epoch_delta_3, + Version(None, 4), + 0, + sentinel.spacecraft_info26, + ), + call( + ultra_parents + ["predict"], + 27, + start_epoch_4, + epoch_delta_4, + Version(None, 4), + GlowsL3Flags.PREDICTIVE_EPHEMERIS, + sentinel.spacecraft_info27, + ), + ] + ) + + mock_process_ultra_hf.assert_has_calls( + [ + call( + ultra_parents, + 24, + start_epoch_1, + epoch_delta_1, + Version(None, 4), + 0, + sentinel.spacecraft_info24, + ), + call( + ultra_parents, + 25, + start_epoch_2, + epoch_delta_2, + Version(None, 4), + 0, + sentinel.spacecraft_info25, + ), + call( + ultra_parents, + 26, + start_epoch_3, + epoch_delta_3, + Version(None, 4), + 0, + sentinel.spacecraft_info26, + ), + call( + ultra_parents + ["predict"], + 27, + start_epoch_4, + epoch_delta_4, + Version(None, 4), + GlowsL3Flags.PREDICTIVE_EPHEMERIS, + sentinel.spacecraft_info27, + ), + ] + ) self.assertEqual(expected_l3e_products, actual_l3e_products) mock_get_lo_pivot_angles.assert_called_once_with([24, 25, 26, 27]) @@ -1865,7 +2941,9 @@ def test_process_l3e_skips_repointing_on_exception(self, mock_determine_spacecra @patch("imap_l3_processing.glows.glows_processor.ZipFile") def test_archive_dependencies(self, mock_zip, mock_json): version_number = Version(1, 1) - expected_filepath = TEMP_CDF_FOLDER_PATH / "imap_glows_l3b-archive_20250314_v001.zip" + expected_filepath = ( + TEMP_CDF_FOLDER_PATH / "imap_glows_l3b-archive_20250314_v001.zip" + ) expected_json_filename = "cr_to_process.json" l3bc_dependencies = GlowsL3BCDependencies( @@ -1873,16 +2951,18 @@ def test_archive_dependencies(self, mock_zip, mock_json): carrington_rotation_number=2095, start_date=datetime.fromisoformat("2025-03-14 12:34:56.789"), end_date=datetime.fromisoformat("2025-03-24 12:34:56.789"), - l3a_data=[{"filename": "imap_glows_l3a_hist_20250314-repoint000001_v001.cdf"}, - {"filename": "imap_glows_l3a_hist_20100102-repoint000002_v001.cdf"}], + l3a_data=[ + {"filename": "imap_glows_l3a_hist_20250314-repoint000001_v001.cdf"}, + {"filename": "imap_glows_l3a_hist_20100102-repoint000002_v001.cdf"}, + ], external_files={}, ancillary_files={ - 'uv_anisotropy': Path("uv_anisotropy.dat"), - 'WawHelioIonMP_parameters': Path("waw_helio_ion.dat"), - 'bad_days_list': Path("bad_days_list.dat"), - 'pipeline_settings': Path("pipeline_settings.json"), + "uv_anisotropy": Path("uv_anisotropy.dat"), + "WawHelioIonMP_parameters": Path("waw_helio_ion.dat"), + "bad_days_list": Path("bad_days_list.dat"), + "pipeline_settings": Path("pipeline_settings.json"), }, - repointing_file_path=Path("repointing.csv") + repointing_file_path=Path("repointing.csv"), ) external_dependencies = ExternalDependencies( @@ -1891,22 +2971,27 @@ def test_archive_dependencies(self, mock_zip, mock_json): omni2_data_path=Path("omni2_data_path"), ) - expected_json_to_serialize = {"cr_rotation_number": 2095, - "l3a_paths": ["imap_glows_l3a_hist_20250314-repoint000001_v001.cdf", - "imap_glows_l3a_hist_20100102-repoint000002_v001.cdf"], - "cr_start_date": "2025-03-14 12:34:56.789000", - "cr_end_date": "2025-03-24 12:34:56.789000", - "bad_days_list": "bad_days_list.dat", - "pipeline_settings": "pipeline_settings.json", - "waw_helioion_mp": "waw_helio_ion.dat", - "uv_anisotropy": "uv_anisotropy.dat", - "repointing_file": "repointing.csv", - } + expected_json_to_serialize = { + "cr_rotation_number": 2095, + "l3a_paths": [ + "imap_glows_l3a_hist_20250314-repoint000001_v001.cdf", + "imap_glows_l3a_hist_20100102-repoint000002_v001.cdf", + ], + "cr_start_date": "2025-03-14 12:34:56.789000", + "cr_end_date": "2025-03-24 12:34:56.789000", + "bad_days_list": "bad_days_list.dat", + "pipeline_settings": "pipeline_settings.json", + "waw_helioion_mp": "waw_helio_ion.dat", + "uv_anisotropy": "uv_anisotropy.dat", + "repointing_file": "repointing.csv", + } mock_zip_file = MagicMock() mock_zip.return_value.__enter__.return_value = mock_zip_file - actual_zip_file_name = GlowsProcessor.archive_dependencies(l3bc_dependencies, external_dependencies) + actual_zip_file_name = GlowsProcessor.archive_dependencies( + l3bc_dependencies, external_dependencies + ) self.assertEqual(expected_filepath, actual_zip_file_name) @@ -1914,12 +2999,16 @@ def test_archive_dependencies(self, mock_zip, mock_json): mock_json.dumps.assert_called_once_with(expected_json_to_serialize) - mock_zip_file.write.assert_has_calls([ - call(Path("lyman_alpha_path"), "lyman_alpha_composite.nc"), - call(Path("omni2_data_path"), "omni2_all_years.dat"), - call(Path("f107_index_file_path"), "f107_fluxtable.txt"), - ]) - mock_zip_file.writestr.assert_called_once_with(expected_json_filename, mock_json.dumps.return_value) + mock_zip_file.write.assert_has_calls( + [ + call(Path("lyman_alpha_path"), "lyman_alpha_composite.nc"), + call(Path("omni2_data_path"), "omni2_all_years.dat"), + call(Path("f107_index_file_path"), "f107_fluxtable.txt"), + ] + ) + mock_zip_file.writestr.assert_called_once_with( + expected_json_filename, mock_json.dumps.return_value + ) def _build_l3e_initializer_output(self) -> GlowsL3EInitializerOutput: mock_dependencies = Mock() @@ -1945,5 +3034,5 @@ def _build_l3e_initializer_output(self) -> GlowsL3EInitializerOutput: ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/hi/test_hi_combined_dependencies.py b/tests/hi/test_hi_combined_dependencies.py index f701d0d6e..634f342b0 100644 --- a/tests/hi/test_hi_combined_dependencies.py +++ b/tests/hi/test_hi_combined_dependencies.py @@ -4,11 +4,15 @@ from imap_data_access.processing_input import ProcessingInputCollection, ScienceInput -from imap_l3_processing.hi.hi_combined_sensor_dependencies import HiL3CombinedMapDependencies +from imap_l3_processing.hi.hi_combined_sensor_dependencies import ( + HiL3CombinedMapDependencies, +) class TestHiCombinedDependencies(unittest.TestCase): - @patch("imap_l3_processing.hi.hi_combined_sensor_dependencies.RectangularIntensityMapData.read_from_path") + @patch( + "imap_l3_processing.hi.hi_combined_sensor_dependencies.RectangularIntensityMapData.read_from_path" + ) def test_from_file_paths(self, read_cdf): hi_map_paths = [ Path("imap_hi_l3_h45-ena-h-hf-sp-ram-hae-4deg-6mo_20251022_v001.cdf"), @@ -21,15 +25,19 @@ def test_from_file_paths(self, read_cdf): Path("imap_hi_l3_h90-ena-h-hf-sp-anti-hae-4deg-6mo_20250422_v001.cdf"), ] - expected_45_return_maps = [sentinel.read_data45_1, - sentinel.read_data45_2, - sentinel.read_data45_3, - sentinel.read_data45_4] + expected_45_return_maps = [ + sentinel.read_data45_1, + sentinel.read_data45_2, + sentinel.read_data45_3, + sentinel.read_data45_4, + ] - expected_90_return_maps = [sentinel.read_data90_1, - sentinel.read_data90_2, - sentinel.read_data90_3, - sentinel.read_data90_4] + expected_90_return_maps = [ + sentinel.read_data90_1, + sentinel.read_data90_2, + sentinel.read_data90_3, + sentinel.read_data90_4, + ] read_cdf.side_effect = expected_45_return_maps + expected_90_return_maps @@ -41,8 +49,12 @@ def test_from_file_paths(self, read_cdf): self.assertEqual(expected_45_return_maps, result.h45_maps) self.assertEqual(expected_90_return_maps, result.h90_maps) - @patch("imap_l3_processing.hi.hi_combined_sensor_dependencies.imap_data_access.download") - @patch("imap_l3_processing.hi.hi_combined_sensor_dependencies.RectangularIntensityMapData.read_from_path") + @patch( + "imap_l3_processing.hi.hi_combined_sensor_dependencies.imap_data_access.download" + ) + @patch( + "imap_l3_processing.hi.hi_combined_sensor_dependencies.RectangularIntensityMapData.read_from_path" + ) def test_fetch_dependencies(self, read_cdf, mock_download): file_name1 = "imap_hi_l3_h45-ena-h-hf-sp-ram-hae-4deg-6mo_20251022_v001.cdf" file_name2 = "imap_hi_l3_h45-ena-h-hf-sp-anti-hae-4deg-6mo_20251022_v001.cdf" @@ -53,7 +65,16 @@ def test_fetch_dependencies(self, read_cdf, mock_download): file_name7 = "imap_hi_l3_h90-ena-h-hf-sp-ram-hae-4deg-6mo_20250422_v001.cdf" file_name8 = "imap_hi_l3_h90-ena-h-hf-sp-anti-hae-4deg-6mo_20250422_v001.cdf" - file_names = [file_name1, file_name2, file_name3, file_name4, file_name5, file_name6, file_name7, file_name8] + file_names = [ + file_name1, + file_name2, + file_name3, + file_name4, + file_name5, + file_name6, + file_name7, + file_name8, + ] processing_input = ProcessingInputCollection( ScienceInput(file_name1), ScienceInput(file_name2), @@ -97,7 +118,21 @@ def test_fetch_dependencies(self, read_cdf, mock_download): self.assertEqual(read_cdf.call_count, 8) self.assertEqual(mock_download.call_count, 8) - self.assertEqual([sentinel.read_data1, sentinel.read_data2, sentinel.read_data3, sentinel.read_data4], - result.h45_maps) - self.assertEqual([sentinel.read_data5, sentinel.read_data6, sentinel.read_data7, sentinel.read_data8], - result.h90_maps) + self.assertEqual( + [ + sentinel.read_data1, + sentinel.read_data2, + sentinel.read_data3, + sentinel.read_data4, + ], + result.h45_maps, + ) + self.assertEqual( + [ + sentinel.read_data5, + sentinel.read_data6, + sentinel.read_data7, + sentinel.read_data8, + ], + result.h90_maps, + ) diff --git a/tests/hi/test_hi_combined_initializer.py b/tests/hi/test_hi_combined_initializer.py index e7108d5ab..fca3253e7 100644 --- a/tests/hi/test_hi_combined_initializer.py +++ b/tests/hi/test_hi_combined_initializer.py @@ -10,392 +10,451 @@ from tests.test_helpers import create_mock_query_results l3_sp_input_maps = [ - 'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', - 'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', - 'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', - 'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', + "imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf", ] l2_nsp_input_maps = [ - 'imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-6deg-1yr_20250101_v001.cdf', - 'imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-6deg-1yr_20250101_v001.cdf', - 'imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-6deg-1yr_20250101_v001.cdf', - 'imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-6deg-1yr_20250101_v001.cdf', + "imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-6deg-1yr_20250101_v001.cdf", ] class TestHiCombinedInitializer(unittest.TestCase): def setUp(self): self.hi_combined_initializer_query_patcher = patch( - 'imap_l3_processing.hi.hi_combined_initializer.imap_data_access.query') - self.hi_combined_initializer_mock_query = self.hi_combined_initializer_query_patcher.start() + "imap_l3_processing.hi.hi_combined_initializer.imap_data_access.query" + ) + self.hi_combined_initializer_mock_query = ( + self.hi_combined_initializer_query_patcher.start() + ) def teardown(self): self.hi_combined_initializer_mock_query.stop() def test_get_maps_that_should_be_produced_no_existing_combined_sp(self): for deg in ["6deg", "4deg"]: - descriptor = f'hic-ena-h-hf-sp-full-hae-{deg}-1yr' + descriptor = f"hic-ena-h-hf-sp-full-hae-{deg}-1yr" for input_major_version in [None, 2]: - expected_version_map = VersionMap({descriptor: Version(input_major_version, 1)}) + expected_version_map = VersionMap( + {descriptor: Version(input_major_version, 1)} + ) with self.subTest(deg=deg, major_version=input_major_version): self.hi_combined_initializer_mock_query.side_effect = [ - create_mock_query_results([ - f'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-{deg}-1yr_20250101_v001.cdf', - f'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-{deg}-1yr_20250101_v001.cdf', - - f'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-{deg}-1yr_20250101_v001.cdf', - f'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-{deg}-1yr_20250101_v001.cdf', - ]), - create_mock_query_results([]) + create_mock_query_results( + [ + f"imap_hi_l3_h45-ena-h-hf-sp-ram-hae-{deg}-1yr_20250101_v001.cdf", + f"imap_hi_l3_h45-ena-h-hf-sp-anti-hae-{deg}-1yr_20250101_v001.cdf", + f"imap_hi_l3_h90-ena-h-hf-sp-ram-hae-{deg}-1yr_20250101_v001.cdf", + f"imap_hi_l3_h90-ena-h-hf-sp-anti-hae-{deg}-1yr_20250101_v001.cdf", + ] + ), + create_mock_query_results([]), ] expected_maps_to_produce = [ PossibleMapToProduce( input_files={ - f'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-{deg}-1yr_20250101_v001.cdf', - f'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-{deg}-1yr_20250101_v001.cdf', - - f'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-{deg}-1yr_20250101_v001.cdf', - f'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-{deg}-1yr_20250101_v001.cdf', + f"imap_hi_l3_h45-ena-h-hf-sp-ram-hae-{deg}-1yr_20250101_v001.cdf", + f"imap_hi_l3_h45-ena-h-hf-sp-anti-hae-{deg}-1yr_20250101_v001.cdf", + f"imap_hi_l3_h90-ena-h-hf-sp-ram-hae-{deg}-1yr_20250101_v001.cdf", + f"imap_hi_l3_h90-ena-h-hf-sp-anti-hae-{deg}-1yr_20250101_v001.cdf", }, input_metadata=InputMetadata( - instrument='hi', - data_level='l3', + instrument="hi", + data_level="l3", start_date=datetime(2025, 1, 1), end_date=datetime(2026, 1, 1), version=expected_version_map, - descriptor=descriptor - ) + descriptor=descriptor, + ), ) ] initializer = HiCombinedInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced( - descriptor, input_major_version) + actual_maps_to_produce = ( + initializer.get_maps_that_should_be_produced( + descriptor, input_major_version + ) + ) - self.hi_combined_initializer_mock_query.assert_has_calls([ - call(instrument='hi', data_level='l3'), - call(instrument='hi', data_level='l2') - ]) + self.hi_combined_initializer_mock_query.assert_has_calls( + [ + call(instrument="hi", data_level="l3"), + call(instrument="hi", data_level="l2"), + ] + ) self.assertEqual(expected_maps_to_produce, actual_maps_to_produce) - @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') - def test_get_maps_that_should_be_produced_new_major_version(self, mock_read_cdf_parents: Mock): - mock_read_cdf_parents.return_value = { - *l3_sp_input_maps - } - - existing_file_without_major_number = 'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.cdf' - existing_file_with_lower_major_number = 'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.0001.cdf' - for existing_file in [existing_file_without_major_number, existing_file_with_lower_major_number]: + @patch("imap_l3_processing.maps.map_initializer.read_cdf_parents") + def test_get_maps_that_should_be_produced_new_major_version( + self, mock_read_cdf_parents: Mock + ): + mock_read_cdf_parents.return_value = {*l3_sp_input_maps} + + existing_file_without_major_number = ( + "imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.cdf" + ) + existing_file_with_lower_major_number = ( + "imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.0001.cdf" + ) + for existing_file in [ + existing_file_without_major_number, + existing_file_with_lower_major_number, + ]: with self.subTest(existing_file=existing_file): mock_read_cdf_parents.reset_mock() self.hi_combined_initializer_mock_query.reset_mock() self.hi_combined_initializer_mock_query.side_effect = [ - create_mock_query_results([ - 'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', - 'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', - 'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', - 'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', - existing_file, - ]), - create_mock_query_results([]) + create_mock_query_results( + [ + "imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf", + existing_file, + ] + ), + create_mock_query_results([]), ] - descriptor = 'hic-ena-h-hf-sp-full-hae-6deg-1yr' + descriptor = "hic-ena-h-hf-sp-full-hae-6deg-1yr" expected_maps_to_produce = [ PossibleMapToProduce( input_files={ - 'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', - 'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', - - 'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', - 'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', + "imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf", }, input_metadata=InputMetadata( - instrument='hi', - data_level='l3', + instrument="hi", + data_level="l3", start_date=datetime(2025, 1, 1), end_date=datetime(2026, 1, 1), version=VersionMap({descriptor: Version(2, 2)}), - descriptor=descriptor - ) + descriptor=descriptor, + ), ) ] initializer = HiCombinedInitializer() actual_maps_to_produce = initializer.get_maps_that_should_be_produced( - 'hic-ena-h-hf-sp-full-hae-6deg-1yr', 2) + "hic-ena-h-hf-sp-full-hae-6deg-1yr", 2 + ) mock_read_cdf_parents.assert_called_once_with(existing_file) - self.hi_combined_initializer_mock_query.assert_has_calls([ - call(instrument='hi', data_level='l3'), - call(instrument='hi', data_level='l2') - ]) + self.hi_combined_initializer_mock_query.assert_has_calls( + [ + call(instrument="hi", data_level="l3"), + call(instrument="hi", data_level="l2"), + ] + ) self.assertEqual(expected_maps_to_produce, actual_maps_to_produce) def test_get_maps_that_should_be_produced_no_existing_combined_nsp(self): for deg in ["6deg", "4deg"]: - descriptor = f'hic-ena-h-hf-nsp-full-hae-{deg}-1yr' + descriptor = f"hic-ena-h-hf-nsp-full-hae-{deg}-1yr" with self.subTest(msg=descriptor): self.hi_combined_initializer_mock_query.side_effect = [ create_mock_query_results([]), - create_mock_query_results([ - f'imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-{deg}-1yr_20250101_v001.cdf', - f'imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-{deg}-1yr_20250101_v001.cdf', - - f'imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-{deg}-1yr_20250101_v001.cdf', - f'imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-{deg}-1yr_20250101_v001.cdf', - ]) + create_mock_query_results( + [ + f"imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-{deg}-1yr_20250101_v001.cdf", + f"imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-{deg}-1yr_20250101_v001.cdf", + f"imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-{deg}-1yr_20250101_v001.cdf", + f"imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-{deg}-1yr_20250101_v001.cdf", + ] + ), ] expected_maps_to_produce = [ PossibleMapToProduce( input_files={ - f'imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-{deg}-1yr_20250101_v001.cdf', - f'imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-{deg}-1yr_20250101_v001.cdf', - - f'imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-{deg}-1yr_20250101_v001.cdf', - f'imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-{deg}-1yr_20250101_v001.cdf', + f"imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-{deg}-1yr_20250101_v001.cdf", + f"imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-{deg}-1yr_20250101_v001.cdf", + f"imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-{deg}-1yr_20250101_v001.cdf", + f"imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-{deg}-1yr_20250101_v001.cdf", }, input_metadata=InputMetadata( - instrument='hi', - data_level='l3', + instrument="hi", + data_level="l3", start_date=datetime(2025, 1, 1), end_date=datetime(2026, 1, 1), version=VersionMap({descriptor: Version(None, 1)}), - descriptor=descriptor - ) + descriptor=descriptor, + ), ) ] initializer = HiCombinedInitializer() actual_maps_to_produce = initializer.get_maps_that_should_be_produced( - descriptor, None) + descriptor, None + ) - self.hi_combined_initializer_mock_query.assert_has_calls([ - call(instrument='hi', data_level='l3'), - call(instrument='hi', data_level='l2') - ]) + self.hi_combined_initializer_mock_query.assert_has_calls( + [ + call(instrument="hi", data_level="l3"), + call(instrument="hi", data_level="l2"), + ] + ) self.assertEqual(expected_maps_to_produce, actual_maps_to_produce) - @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') - def test_get_maps_that_should_be_produced_waits_for_full_set_of_new_inputs(self, - mock_read_cdf_parents: Mock): + @patch("imap_l3_processing.maps.map_initializer.read_cdf_parents") + def test_get_maps_that_should_be_produced_waits_for_full_set_of_new_inputs( + self, mock_read_cdf_parents: Mock + ): self.hi_combined_initializer_mock_query.side_effect = [ - create_mock_query_results([ - *l3_sp_input_maps, - 'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.cdf', - - 'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20260101_v002.cdf', - 'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20260101_v002.cdf', - 'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20260101_v002.cdf', - ]), - create_mock_query_results([]) + create_mock_query_results( + [ + *l3_sp_input_maps, + "imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20260101_v002.cdf", + "imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20260101_v002.cdf", + "imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20260101_v002.cdf", + ] + ), + create_mock_query_results([]), ] - mock_read_cdf_parents.return_value = { - *l3_sp_input_maps - } + mock_read_cdf_parents.return_value = {*l3_sp_input_maps} expected_maps_to_produce = [] initializer = HiCombinedInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced('hic-ena-h-hf-sp-full-hae-6deg-1yr', None) + actual_maps_to_produce = initializer.get_maps_that_should_be_produced( + "hic-ena-h-hf-sp-full-hae-6deg-1yr", None + ) - self.hi_combined_initializer_mock_query.assert_has_calls([ - call(instrument='hi', data_level='l3'), - call(instrument='hi', data_level='l2') - ]) + self.hi_combined_initializer_mock_query.assert_has_calls( + [ + call(instrument="hi", data_level="l3"), + call(instrument="hi", data_level="l2"), + ] + ) - mock_read_cdf_parents.assert_called_once_with('imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.cdf') + mock_read_cdf_parents.assert_called_once_with( + "imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_v001.cdf" + ) self.assertEqual(expected_maps_to_produce, actual_maps_to_produce) def test_get_maps_that_should_be_produced_filters_based_on_input_descriptor(self): self.hi_combined_initializer_mock_query.side_effect = [ - create_mock_query_results([ - *l3_sp_input_maps, - ]), - create_mock_query_results([]) + create_mock_query_results( + [ + *l3_sp_input_maps, + ] + ), + create_mock_query_results([]), ] expected_maps_to_produce = [] initializer = HiCombinedInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced('hic-ena-h-hf-nsp-full-hae-6deg-1yr', - None) + actual_maps_to_produce = initializer.get_maps_that_should_be_produced( + "hic-ena-h-hf-nsp-full-hae-6deg-1yr", None + ) - self.hi_combined_initializer_mock_query.assert_has_calls([ - call(instrument='hi', data_level='l3'), - call(instrument='hi', data_level='l2') - ]) + self.hi_combined_initializer_mock_query.assert_has_calls( + [ + call(instrument="hi", data_level="l3"), + call(instrument="hi", data_level="l2"), + ] + ) self.assertEqual(expected_maps_to_produce, actual_maps_to_produce) - @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') - def test_get_maps_that_should_be_produced_increments_versions_correctly(self, mock_read_cdf_parents): + @patch("imap_l3_processing.maps.map_initializer.read_cdf_parents") + def test_get_maps_that_should_be_produced_increments_versions_correctly( + self, mock_read_cdf_parents + ): cases = [(None, "v001", 2), (2, "v002.0001", 2), (2, "v002.0002", 3)] for input_major_version, existing_file_version, expected_minor_version in cases: - with self.subTest(input_major_version=input_major_version, existing_file_version=existing_file_version, expected_minor_version=expected_minor_version): + with self.subTest( + input_major_version=input_major_version, + existing_file_version=existing_file_version, + expected_minor_version=expected_minor_version, + ): self.hi_combined_initializer_mock_query.reset_mock() mock_read_cdf_parents.reset_mock() self.hi_combined_initializer_mock_query.reset_mock() mock_read_cdf_parents.reset_mock() self.hi_combined_initializer_mock_query.side_effect = [ - create_mock_query_results([ - f'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_{existing_file_version}.cdf', - - 'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v002.cdf', - *l3_sp_input_maps, - ]), - create_mock_query_results([]) + create_mock_query_results( + [ + f"imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_{existing_file_version}.cdf", + "imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v002.cdf", + *l3_sp_input_maps, + ] + ), + create_mock_query_results([]), ] - descriptor = 'hic-ena-h-hf-sp-full-hae-6deg-1yr' + descriptor = "hic-ena-h-hf-sp-full-hae-6deg-1yr" expected_maps_to_produce = [ PossibleMapToProduce( input_files={ - 'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', - 'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf', - 'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf', - - 'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v002.cdf', - + "imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250101_v002.cdf", }, input_metadata=InputMetadata( - instrument='hi', - data_level='l3', + instrument="hi", + data_level="l3", start_date=datetime(2025, 1, 1), end_date=datetime(2026, 1, 1), - version=VersionMap({descriptor: Version(input_major_version, expected_minor_version)}), - descriptor=descriptor - ) + version=VersionMap( + { + descriptor: Version( + input_major_version, expected_minor_version + ) + } + ), + descriptor=descriptor, + ), ) ] - mock_read_cdf_parents.return_value = [ - *l3_sp_input_maps - ] + mock_read_cdf_parents.return_value = [*l3_sp_input_maps] initializer = HiCombinedInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced(descriptor, input_major_version) - mock_read_cdf_parents.assert_called_once_with(f'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_{existing_file_version}.cdf') + actual_maps_to_produce = initializer.get_maps_that_should_be_produced( + descriptor, input_major_version + ) + mock_read_cdf_parents.assert_called_once_with( + f"imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250101_{existing_file_version}.cdf" + ) - self.hi_combined_initializer_mock_query.assert_has_calls([ - call(instrument='hi', data_level='l3'), - call(instrument='hi', data_level='l2') - ]) + self.hi_combined_initializer_mock_query.assert_has_calls( + [ + call(instrument="hi", data_level="l3"), + call(instrument="hi", data_level="l2"), + ] + ) self.assertEqual(expected_maps_to_produce, actual_maps_to_produce) - @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') - def test_get_maps_that_should_be_produced_can_produce_multiple_maps(self, mock_read_cdf_parents): + @patch("imap_l3_processing.maps.map_initializer.read_cdf_parents") + def test_get_maps_that_should_be_produced_can_produce_multiple_maps( + self, mock_read_cdf_parents + ): self.hi_combined_initializer_mock_query.side_effect = [ - create_mock_query_results([ - ]), - create_mock_query_results([ - *l2_nsp_input_maps, - 'imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf', - 'imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf', - 'imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf', - 'imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf', - ]), + create_mock_query_results([]), + create_mock_query_results( + [ + *l2_nsp_input_maps, + "imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf", + "imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf", + "imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf", + "imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf", + ] + ), ] - descriptor = 'hic-ena-h-hf-nsp-full-hae-6deg-1yr' + descriptor = "hic-ena-h-hf-nsp-full-hae-6deg-1yr" expected_maps_to_produce = [ PossibleMapToProduce( input_files={ - 'imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-6deg-1yr_20250101_v001.cdf', - 'imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-6deg-1yr_20250101_v001.cdf', - - 'imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-6deg-1yr_20250101_v001.cdf', - 'imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-6deg-1yr_20250101_v001.cdf', + "imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-6deg-1yr_20250101_v001.cdf", + "imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-6deg-1yr_20250101_v001.cdf", }, input_metadata=InputMetadata( - instrument='hi', - data_level='l3', + instrument="hi", + data_level="l3", start_date=datetime(2025, 1, 1), end_date=datetime(2026, 1, 1), version=VersionMap({descriptor: Version(None, 1)}), - descriptor=descriptor - ) + descriptor=descriptor, + ), ), PossibleMapToProduce( input_files={ - 'imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf', - 'imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf', - - 'imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf', - 'imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf', + "imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf", + "imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf", + "imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf", + "imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf", }, input_metadata=InputMetadata( - instrument='hi', - data_level='l3', + instrument="hi", + data_level="l3", start_date=datetime(2026, 1, 1), end_date=datetime(2027, 1, 1), version=VersionMap({descriptor: Version(None, 1)}), - descriptor=descriptor - ) + descriptor=descriptor, + ), ), ] initializer = HiCombinedInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced('hic-ena-h-hf-nsp-full-hae-6deg-1yr', - None) + actual_maps_to_produce = initializer.get_maps_that_should_be_produced( + "hic-ena-h-hf-nsp-full-hae-6deg-1yr", None + ) mock_read_cdf_parents.assert_not_called() - self.hi_combined_initializer_mock_query.assert_has_calls([ - call(instrument='hi', data_level='l3'), - call(instrument='hi', data_level='l2') - ]) + self.hi_combined_initializer_mock_query.assert_has_calls( + [ + call(instrument="hi", data_level="l3"), + call(instrument="hi", data_level="l2"), + ] + ) self.assertEqual(expected_maps_to_produce, actual_maps_to_produce) - @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') + @patch("imap_l3_processing.maps.map_initializer.read_cdf_parents") def test_get_maps_that_can_be_produced(self, mock_read_cdf_parents): query_results_for_existing_l3 = create_mock_query_results([]) - query_results_for_available_inputs = create_mock_query_results(['imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf', - 'imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf', - 'imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf', - 'imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf', ]) + query_results_for_available_inputs = create_mock_query_results( + [ + "imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf", + "imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf", + "imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf", + "imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf", + ] + ) self.hi_combined_initializer_mock_query.side_effect = [ - query_results_for_existing_l3, query_results_for_available_inputs, + query_results_for_existing_l3, + query_results_for_available_inputs, ] initializer = HiCombinedInitializer() - descriptor = 'hic-ena-h-hf-nsp-anti-hae-6deg-1yr' + descriptor = "hic-ena-h-hf-nsp-anti-hae-6deg-1yr" actual_possible_maps = initializer.get_maps_that_can_be_produced(descriptor) expected_possible_maps = [ PossibleMapToProduce( input_files={ - 'imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf', - 'imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf', - 'imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf', - 'imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf', + "imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf", + "imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf", + "imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-6deg-1yr_20260101_v001.cdf", + "imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-6deg-1yr_20260101_v001.cdf", }, input_metadata=InputMetadata( - instrument='hi', - data_level='l3', - start_date=datetime(2026,1,1), - end_date=datetime(2027,1,1), + instrument="hi", + data_level="l3", + start_date=datetime(2026, 1, 1), + end_date=datetime(2027, 1, 1), version=VersionMap({}, Version(None, 1)), descriptor=descriptor, - ) + ), ) ] - self.assertEqual(expected_possible_maps, actual_possible_maps) \ No newline at end of file + self.assertEqual(expected_possible_maps, actual_possible_maps) diff --git a/tests/hi/test_hi_processor.py b/tests/hi/test_hi_processor.py index 341da49c4..84d3d87e8 100644 --- a/tests/hi/test_hi_processor.py +++ b/tests/hi/test_hi_processor.py @@ -8,23 +8,36 @@ from imap_processing.spice.geometry import SpiceFrame from imap_l3_processing.hi.hi_processor import HiProcessor -from imap_l3_processing.hi.hi_spectral_fit_dependencies import HiSpectralIndexDependencies -from imap_l3_processing.maps.hilo_l3_survival_dependencies import HiL3SingleSensorFullSpinDependencies, \ - HiLoL3SurvivalDependencies -from imap_l3_processing.maps.map_models import RectangularSpectralIndexDataProduct, RectangularIntensityDataProduct +from imap_l3_processing.hi.hi_spectral_fit_dependencies import ( + HiSpectralIndexDependencies, +) +from imap_l3_processing.maps.hilo_l3_survival_dependencies import ( + HiL3SingleSensorFullSpinDependencies, + HiLoL3SurvivalDependencies, +) +from imap_l3_processing.maps.map_models import ( + RectangularSpectralIndexDataProduct, + RectangularIntensityDataProduct, +) from imap_l3_processing.models import InputMetadata, Instrument from tests.maps.test_builders import create_rectangular_intensity_map_data from tests.test_helpers import get_test_data_path, run_periodically class TestHiProcessor(unittest.TestCase): - @patch('imap_l3_processing.hi.hi_processor.MapProcessor.get_parent_file_names') - @patch('imap_l3_processing.hi.hi_processor.HiSpectralIndexDependencies.fetch_dependencies') - @patch('imap_l3_processing.maps.spectral_fit.fit_arrays_to_power_law') - @patch('imap_l3_processing.hi.hi_processor.save_data') - def test_process_spectral_fit(self, mock_save_data, mock_spectral_fit, - mock_fetch_dependencies, - mock_get_parent_file_names): + @patch("imap_l3_processing.hi.hi_processor.MapProcessor.get_parent_file_names") + @patch( + "imap_l3_processing.hi.hi_processor.HiSpectralIndexDependencies.fetch_dependencies" + ) + @patch("imap_l3_processing.maps.spectral_fit.fit_arrays_to_power_law") + @patch("imap_l3_processing.hi.hi_processor.save_data") + def test_process_spectral_fit( + self, + mock_save_data, + mock_spectral_fit, + mock_fetch_dependencies, + mock_get_parent_file_names, + ): mock_get_parent_file_names.return_value = ["l2_or_l3_map"] lat = np.array([0, 45]) long = np.array([0, 45, 90]) @@ -34,9 +47,15 @@ def test_process_spectral_fit(self, mock_save_data, mock_spectral_fit, flux = np.full((1, 2, 3, 2), 1) intensity_stat_unc = 5 - hi_l3_data = create_rectangular_intensity_map_data(lat=lat, lon=long, energy=energy, epoch=epoch, flux=flux, - intensity_stat_uncert=intensity_stat_unc, - energy_delta=energy_delta) + hi_l3_data = create_rectangular_intensity_map_data( + lat=lat, + lon=long, + energy=energy, + epoch=epoch, + flux=flux, + intensity_stat_uncert=intensity_stat_unc, + energy_delta=energy_delta, + ) intensity_data = hi_l3_data.intensity_map_data intensity_data.exposure_factor = np.full_like(flux, 1) dependencies = HiSpectralIndexDependencies(map_data=hi_l3_data) @@ -45,31 +64,42 @@ def test_process_spectral_fit(self, mock_save_data, mock_spectral_fit, mock_fetch_dependencies.return_value = dependencies - input_metadata = InputMetadata(instrument="hi", - data_level="", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="", - descriptor="h90-spx-h-hf-sp-full-hae-4deg-6mo", - ) + input_metadata = InputMetadata( + instrument="hi", + data_level="", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="", + descriptor="h90-spx-h-hf-sp-full-hae-4deg-6mo", + ) expected_index_unc = np.full((1, 1, 3, 2), 1.0) expected_index = np.full_like(expected_index_unc, 2.0) expected_scalar = np.full_like(expected_index_unc, 3.0) expected_scalar_unc = np.full_like(expected_index_unc, 4.0) expected_chisq = np.full_like(expected_index_unc, 5.0) - mock_spectral_fit.return_value = expected_scalar, expected_scalar_unc, expected_index, expected_index_unc, expected_chisq + mock_spectral_fit.return_value = ( + expected_scalar, + expected_scalar_unc, + expected_index, + expected_index_unc, + expected_chisq, + ) processor = HiProcessor(upstream_dependencies, input_metadata) product = processor.process() mock_fetch_dependencies.assert_called_with(upstream_dependencies) - mock_spectral_fit.assert_called_once_with(intensity_data.ena_intensity, - intensity_data.ena_intensity_stat_uncert, - intensity_data.energy) + mock_spectral_fit.assert_called_once_with( + intensity_data.ena_intensity, + intensity_data.ena_intensity_stat_uncert, + intensity_data.energy, + ) mock_save_data.assert_called_once() actual_hi_data_product = mock_save_data.call_args_list[0].args[0] - self.assertIsInstance(actual_hi_data_product, RectangularSpectralIndexDataProduct) + self.assertIsInstance( + actual_hi_data_product, RectangularSpectralIndexDataProduct + ) self.assertEqual(input_metadata, actual_hi_data_product.input_metadata) actual_data = actual_hi_data_product.data spectral_index_data = actual_data.spectral_index_map_data @@ -78,18 +108,39 @@ def test_process_spectral_fit(self, mock_save_data, mock_spectral_fit, expected_energy_delta_plus = np.array([12]) expected_energy = np.array([4]) expected_exposure = np.full((1, 1, 3, 2), 2) - np.testing.assert_array_equal( spectral_index_data.energy_delta_minus,expected_energy_delta_minus) - np.testing.assert_array_equal(spectral_index_data.energy_delta_plus, expected_energy_delta_plus) - np.testing.assert_array_equal(spectral_index_data.ena_spectral_index,expected_index) - np.testing.assert_array_equal(spectral_index_data.ena_spectral_index_stat_uncert, expected_index_unc) - np.testing.assert_array_equal(spectral_index_data.ena_spectral_index_scalar_coefficient, expected_scalar) - np.testing.assert_array_equal(spectral_index_data.ena_spectral_index_scalar_coefficient_stat_uncert, expected_scalar_unc) - np.testing.assert_array_equal(spectral_index_data.ena_spectral_index_chisq, expected_chisq) + np.testing.assert_array_equal( + spectral_index_data.energy_delta_minus, expected_energy_delta_minus + ) + np.testing.assert_array_equal( + spectral_index_data.energy_delta_plus, expected_energy_delta_plus + ) + np.testing.assert_array_equal( + spectral_index_data.ena_spectral_index, expected_index + ) + np.testing.assert_array_equal( + spectral_index_data.ena_spectral_index_stat_uncert, expected_index_unc + ) + np.testing.assert_array_equal( + spectral_index_data.ena_spectral_index_scalar_coefficient, expected_scalar + ) + np.testing.assert_array_equal( + spectral_index_data.ena_spectral_index_scalar_coefficient_stat_uncert, + expected_scalar_unc, + ) + np.testing.assert_array_equal( + spectral_index_data.ena_spectral_index_chisq, expected_chisq + ) np.testing.assert_array_equal(spectral_index_data.energy, expected_energy) - np.testing.assert_array_equal(spectral_index_data.latitude, intensity_data.latitude) - np.testing.assert_array_equal(spectral_index_data.longitude, intensity_data.longitude) + np.testing.assert_array_equal( + spectral_index_data.latitude, intensity_data.latitude + ) + np.testing.assert_array_equal( + spectral_index_data.longitude, intensity_data.longitude + ) np.testing.assert_array_equal(spectral_index_data.epoch, intensity_data.epoch) - np.testing.assert_array_equal(spectral_index_data.exposure_factor, expected_exposure) + np.testing.assert_array_equal( + spectral_index_data.exposure_factor, expected_exposure + ) self.assertEqual(["l2_or_l3_map"], actual_hi_data_product.parent_file_names) self.assertEqual(hi_l3_data.coords, spectral_index_coords) self.assertEqual(SpiceFrame.ECLIPJ2000, actual_hi_data_product.spice_frame_name) @@ -99,51 +150,80 @@ def test_process_spectral_fit(self, mock_save_data, mock_spectral_fit, @run_periodically(timedelta(days=3)) def test_spectral_fit_against_validation_data(self): test_cases = [ - ("hi45", "hi/fake_l2_maps/hi45-6months.cdf", - "hi/validation/spectral_index/IMAP-Hi45_6months_4.0x4.0_fit_gam.csv", - "hi/validation/spectral_index/IMAP-Hi45_6months_4.0x4.0_fit_gam_sig.csv"), - ("hi90", "hi/fake_l2_maps/hi90-6months.cdf", - "hi/validation/spectral_index/IMAP-Hi90_6months_4.0x4.0_fit_gam.csv", - "hi/validation/spectral_index/IMAP-Hi90_6months_4.0x4.0_fit_gam_sig.csv"), - ("hi45-zirnstein-mondel", "hi/fake_l2_maps/hi45-zirnstein-mondel-6months.cdf", - "hi/validation/spectral_index/IMAP-Hi45_gdf_Zirnstein_model_6months_4.0x4.0_fit_gam.csv", - "hi/validation/spectral_index/IMAP-Hi45_gdf_Zirnstein_model_6months_4.0x4.0_fit_gam_sig.csv"), - ("hi90-zirnstein-mondel", "hi/fake_l2_maps/hi90-zirnstein-mondel-6months.cdf", - "hi/validation/spectral_index/IMAP-Hi90_gdf_Zirnstein_model_6months_4.0x4.0_fit_gam.csv", - "hi/validation/spectral_index/IMAP-Hi90_gdf_Zirnstein_model_6months_4.0x4.0_fit_gam_sig.csv"), + ( + "hi45", + "hi/fake_l2_maps/hi45-6months.cdf", + "hi/validation/spectral_index/IMAP-Hi45_6months_4.0x4.0_fit_gam.csv", + "hi/validation/spectral_index/IMAP-Hi45_6months_4.0x4.0_fit_gam_sig.csv", + ), + ( + "hi90", + "hi/fake_l2_maps/hi90-6months.cdf", + "hi/validation/spectral_index/IMAP-Hi90_6months_4.0x4.0_fit_gam.csv", + "hi/validation/spectral_index/IMAP-Hi90_6months_4.0x4.0_fit_gam_sig.csv", + ), + ( + "hi45-zirnstein-mondel", + "hi/fake_l2_maps/hi45-zirnstein-mondel-6months.cdf", + "hi/validation/spectral_index/IMAP-Hi45_gdf_Zirnstein_model_6months_4.0x4.0_fit_gam.csv", + "hi/validation/spectral_index/IMAP-Hi45_gdf_Zirnstein_model_6months_4.0x4.0_fit_gam_sig.csv", + ), + ( + "hi90-zirnstein-mondel", + "hi/fake_l2_maps/hi90-zirnstein-mondel-6months.cdf", + "hi/validation/spectral_index/IMAP-Hi90_gdf_Zirnstein_model_6months_4.0x4.0_fit_gam.csv", + "hi/validation/spectral_index/IMAP-Hi90_gdf_Zirnstein_model_6months_4.0x4.0_fit_gam_sig.csv", + ), ] - for name, input_file_path, expected_gamma_path, expected_sigma_path in test_cases: + for ( + name, + input_file_path, + expected_gamma_path, + expected_sigma_path, + ) in test_cases: with self.subTest(name): dependencies = HiSpectralIndexDependencies.from_file_paths( get_test_data_path(input_file_path) ) - expected_gamma = np.loadtxt(get_test_data_path(expected_gamma_path), delimiter=",", dtype=str).T + expected_gamma = np.loadtxt( + get_test_data_path(expected_gamma_path), delimiter=",", dtype=str + ).T expected_gamma[expected_gamma == "NaN"] = "-1" expected_gamma = expected_gamma.astype(np.float64) expected_gamma[expected_gamma == -1] = np.nan - expected_gamma_sigma = np.loadtxt(get_test_data_path(expected_sigma_path), delimiter=",", - dtype=str).T + expected_gamma_sigma = np.loadtxt( + get_test_data_path(expected_sigma_path), delimiter=",", dtype=str + ).T expected_gamma_sigma[expected_gamma_sigma == "NaN"] = "-1" expected_gamma_sigma = expected_gamma_sigma.astype(np.float64) expected_gamma_sigma[expected_gamma_sigma == -1] = np.nan - input_metadata = InputMetadata(instrument="hi", - data_level="l3", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="", - descriptor="spectral-fit-index", - ) + input_metadata = InputMetadata( + instrument="hi", + data_level="l3", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="", + descriptor="spectral-fit-index", + ) processor = HiProcessor(None, input_metadata) output_data = processor.process_spectral_fit_index(dependencies) - np.testing.assert_allclose(output_data.spectral_index_map_data.ena_spectral_index[0, 0], - expected_gamma, atol=1e-3) - np.testing.assert_allclose(output_data.spectral_index_map_data.ena_spectral_index_stat_uncert[0, 0], - expected_gamma_sigma, atol=1e-3) + np.testing.assert_allclose( + output_data.spectral_index_map_data.ena_spectral_index[0, 0], + expected_gamma, + atol=1e-3, + ) + np.testing.assert_allclose( + output_data.spectral_index_map_data.ena_spectral_index_stat_uncert[ + 0, 0 + ], + expected_gamma_sigma, + atol=1e-3, + ) def test_spectral_fit_fields_other_than_fit_fields(self): input_energies = np.array([1, 10, 99]) + 0.5 @@ -151,8 +231,9 @@ def test_spectral_fit_fields_other_than_fit_fields(self): lat = np.arange(-90, 90, 45) lon = np.arange(0, 360, 45) - input_map = create_rectangular_intensity_map_data(energy=input_energies, energy_delta=input_deltas, lat=lat, - lon=lon) + input_map = create_rectangular_intensity_map_data( + energy=input_energies, energy_delta=input_deltas, lat=lat, lon=lon + ) intensity_data = input_map.intensity_map_data intensity_data.obs_date[0, 0] = datetime(2025, 1, 1) intensity_data.obs_date[0, 1] = datetime(2025, 1, 1) @@ -174,16 +255,28 @@ def test_spectral_fit_fields_other_than_fit_fields(self): self.assertEqual(output.spectral_index_map_data.energy.shape[0], 1) self.assertEqual(output.spectral_index_map_data.energy[0], 10) - np.testing.assert_allclose(output.spectral_index_map_data.energy_delta_minus, np.array([9])) - np.testing.assert_allclose(output.spectral_index_map_data.energy_delta_plus, np.array([90])) + np.testing.assert_allclose( + output.spectral_index_map_data.energy_delta_minus, np.array([9]) + ) + np.testing.assert_allclose( + output.spectral_index_map_data.energy_delta_plus, np.array([90]) + ) self.assertEqual(output.spectral_index_map_data.energy_label.shape[0], 1) self.assertEqual("1.0 - 100.0", output.spectral_index_map_data.energy_label[0]) expected_ena_shape = np.array([1, 1, len(lon), len(lat)]) - np.testing.assert_array_equal(output.spectral_index_map_data.obs_date, - np.full(expected_ena_shape, datetime(2026, 1, 1))) - np.testing.assert_array_equal(output.spectral_index_map_data.obs_date_range, np.full(expected_ena_shape, 2)) - np.testing.assert_array_equal(output.spectral_index_map_data.exposure_factor, np.full(expected_ena_shape, 6)) + np.testing.assert_array_equal( + output.spectral_index_map_data.obs_date, + np.full(expected_ena_shape, datetime(2026, 1, 1)), + ) + np.testing.assert_array_equal( + output.spectral_index_map_data.obs_date_range, + np.full(expected_ena_shape, 2), + ) + np.testing.assert_array_equal( + output.spectral_index_map_data.exposure_factor, + np.full(expected_ena_shape, 6), + ) def test_spectral_fit_other_fields_with_one_zero_exposure(self): input_energies = np.array([1, 10, 99]) + 0.5 @@ -191,8 +284,9 @@ def test_spectral_fit_other_fields_with_one_zero_exposure(self): lat = np.arange(-90, 90, 45) lon = np.arange(0, 360, 45) - input_map = create_rectangular_intensity_map_data(energy=input_energies, energy_delta=input_deltas, lat=lat, - lon=lon) + input_map = create_rectangular_intensity_map_data( + energy=input_energies, energy_delta=input_deltas, lat=lat, lon=lon + ) intensity_data = input_map.intensity_map_data intensity_data.obs_date[0, 0] = datetime(2025, 1, 1) intensity_data.obs_date[0, 1] = datetime(2025, 1, 1) @@ -207,7 +301,9 @@ def test_spectral_fit_other_fields_with_one_zero_exposure(self): intensity_data.obs_date_range[0, 0] = 1 intensity_data.obs_date_range[0, 1] = 1 intensity_data.obs_date_range[0, 2] = 3 - intensity_data.obs_date_range.mask = np.ma.getmaskarray(intensity_data.obs_date_range) + intensity_data.obs_date_range.mask = np.ma.getmaskarray( + intensity_data.obs_date_range + ) intensity_data.obs_date_range.mask[0, 1] = True processor = HiProcessor(sentinel.input_metadata, sentinel.dependencies) @@ -217,10 +313,16 @@ def test_spectral_fit_other_fields_with_one_zero_exposure(self): output = processor.process_spectral_fit_index(dependencies) expected_ena_shape = np.array([1, 1, len(lon), len(lat)]) - np.testing.assert_array_equal(output.spectral_index_map_data.obs_date, - np.full(expected_ena_shape, datetime(2026, 1, 1)), strict=True) - np.testing.assert_array_equal(output.spectral_index_map_data.obs_date_range, np.full(expected_ena_shape, 2.0), - strict=True) + np.testing.assert_array_equal( + output.spectral_index_map_data.obs_date, + np.full(expected_ena_shape, datetime(2026, 1, 1)), + strict=True, + ) + np.testing.assert_array_equal( + output.spectral_index_map_data.obs_date_range, + np.full(expected_ena_shape, 2.0), + strict=True, + ) def test_spectral_fit_other_fields_with_all_zero_exposure(self): input_energies = np.array([1, 10, 99]) + 0.5 @@ -228,15 +330,18 @@ def test_spectral_fit_other_fields_with_all_zero_exposure(self): lat = np.arange(-90, 90, 45) lon = np.arange(0, 360, 45) - input_map = create_rectangular_intensity_map_data(energy=input_energies, energy_delta=input_deltas, lat=lat, - lon=lon) + input_map = create_rectangular_intensity_map_data( + energy=input_energies, energy_delta=input_deltas, lat=lat, lon=lon + ) intensity_data = input_map.intensity_map_data intensity_data.obs_date.mask = np.ma.getmaskarray(intensity_data.obs_date) intensity_data.obs_date.mask[:] = True intensity_data.exposure_factor[:] = 0.0 - intensity_data.obs_date_range.mask = np.ma.getmaskarray(intensity_data.obs_date_range) + intensity_data.obs_date_range.mask = np.ma.getmaskarray( + intensity_data.obs_date_range + ) intensity_data.obs_date_range.mask[:] = True processor = HiProcessor(sentinel.input_metadata, sentinel.dependencies) @@ -247,25 +352,37 @@ def test_spectral_fit_other_fields_with_all_zero_exposure(self): expected_ena_shape = np.array([1, 1, len(lon), len(lat)]) - np.testing.assert_array_equal(output.spectral_index_map_data.obs_date.mask, np.full(expected_ena_shape, True), - strict=True) - np.testing.assert_array_equal(output.spectral_index_map_data.obs_date_range.mask, - np.full(expected_ena_shape, True), strict=True) - - @patch('imap_l3_processing.hi.hi_processor.MapProcessor.get_parent_file_names') - @patch("imap_l3_processing.hi.hi_processor.HiLoL3SurvivalDependencies.fetch_dependencies") + np.testing.assert_array_equal( + output.spectral_index_map_data.obs_date.mask, + np.full(expected_ena_shape, True), + strict=True, + ) + np.testing.assert_array_equal( + output.spectral_index_map_data.obs_date_range.mask, + np.full(expected_ena_shape, True), + strict=True, + ) + + @patch("imap_l3_processing.hi.hi_processor.MapProcessor.get_parent_file_names") + @patch( + "imap_l3_processing.hi.hi_processor.HiLoL3SurvivalDependencies.fetch_dependencies" + ) @patch("imap_l3_processing.hi.hi_processor.process_survival_probabilities") - @patch('imap_l3_processing.hi.hi_processor.save_data') - def test_process_ram_or_antiram_spin(self, mock_save_data, - mock_process_survival_prob, - mock_fetch_survival_dependencies, mock_get_parent_file_names): + @patch("imap_l3_processing.hi.hi_processor.save_data") + def test_process_ram_or_antiram_spin( + self, + mock_save_data, + mock_process_survival_prob, + mock_fetch_survival_dependencies, + mock_get_parent_file_names, + ): mock_get_parent_file_names.return_value = ["somewhere"] cases = [ ("ram", "h90-ena-h-sf-sp-ram-hae-4deg-6mo", SpiceFrame.ECLIPJ2000), ("anti", "h90-ena-h-sf-sp-anti-hae-4deg-6mo", SpiceFrame.IMAP_GCS), - ("anti", "h90-ena-h-hf-sp-anti-hae-4deg-6mo", SpiceFrame.IMAP_HNU) + ("anti", "h90-ena-h-hf-sp-anti-hae-4deg-6mo", SpiceFrame.IMAP_HNU), ] for case, descriptor, spice_frame_name in cases: @@ -274,60 +391,82 @@ def test_process_ram_or_antiram_spin(self, mock_save_data, mock_fetch_survival_dependencies.reset_mock() mock_process_survival_prob.reset_mock() - input_metadata = InputMetadata(instrument="hi", - data_level="l3", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="", - descriptor=descriptor, - ) + input_metadata = InputMetadata( + instrument="hi", + data_level="l3", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="", + descriptor=descriptor, + ) - dependencies: HiLoL3SurvivalDependencies = mock_fetch_survival_dependencies.return_value + dependencies: HiLoL3SurvivalDependencies = ( + mock_fetch_survival_dependencies.return_value + ) dependencies.dependency_file_paths = [ - Path("folder/map"), Path("folder/l1c")] + Path("folder/map"), + Path("folder/l1c"), + ] - mock_process_survival_prob.return_value = sentinel.survival_probabilities + mock_process_survival_prob.return_value = ( + sentinel.survival_probabilities + ) processor = HiProcessor(sentinel.input_dependencies, input_metadata) product = processor.process(spice_frame_name=spice_frame_name) - mock_fetch_survival_dependencies.assert_called_once_with(sentinel.input_dependencies, - Instrument.IMAP_HI) + mock_fetch_survival_dependencies.assert_called_once_with( + sentinel.input_dependencies, Instrument.IMAP_HI + ) - mock_process_survival_prob.assert_called_once_with(dependencies, spice_frame_name) + mock_process_survival_prob.assert_called_once_with( + dependencies, spice_frame_name + ) - mock_save_data.assert_called_once_with(RectangularIntensityDataProduct( - input_metadata=input_metadata, - parent_file_names=["l1c", "map", "somewhere"], - data=sentinel.survival_probabilities, - spice_frame_name=spice_frame_name, - )) + mock_save_data.assert_called_once_with( + RectangularIntensityDataProduct( + input_metadata=input_metadata, + parent_file_names=["l1c", "map", "somewhere"], + data=sentinel.survival_probabilities, + spice_frame_name=spice_frame_name, + ) + ) self.assertEqual([mock_save_data.return_value], product) - @patch('imap_l3_processing.hi.hi_processor.MapProcessor.get_parent_file_names') - @patch("imap_l3_processing.hi.hi_processor.HiL3SingleSensorFullSpinDependencies.fetch_dependencies") + @patch("imap_l3_processing.hi.hi_processor.MapProcessor.get_parent_file_names") + @patch( + "imap_l3_processing.hi.hi_processor.HiL3SingleSensorFullSpinDependencies.fetch_dependencies" + ) @patch("imap_l3_processing.hi.hi_processor.process_survival_probabilities") - @patch('imap_l3_processing.hi.hi_processor.save_data') - @patch('imap_l3_processing.hi.hi_processor.UnweightedCombination') - def test_process_full_spin_single_sensor_map(self, mock_unweighted_combination_class, mock_save_data, - mock_process_survival_prob, - mock_fetch_full_spin_single_sensor_dependencies, - mock_get_parent_file_names): + @patch("imap_l3_processing.hi.hi_processor.save_data") + @patch("imap_l3_processing.hi.hi_processor.UnweightedCombination") + def test_process_full_spin_single_sensor_map( + self, + mock_unweighted_combination_class, + mock_save_data, + mock_process_survival_prob, + mock_fetch_full_spin_single_sensor_dependencies, + mock_get_parent_file_names, + ): mock_unweighted_combination = Mock() mock_unweighted_combination_class.return_value = mock_unweighted_combination mock_get_parent_file_names.return_value = ["ram_map", "antiram_map"] - input_metadata = InputMetadata(instrument="hi", - data_level="l3", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="", - descriptor="h90-ena-h-sf-sp-full-hae-4deg-6mo", - ) - - full_spin_dependencies: HiL3SingleSensorFullSpinDependencies = mock_fetch_full_spin_single_sensor_dependencies.return_value + input_metadata = InputMetadata( + instrument="hi", + data_level="l3", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="", + descriptor="h90-ena-h-sf-sp-full-hae-4deg-6mo", + ) + + full_spin_dependencies: HiL3SingleSensorFullSpinDependencies = ( + mock_fetch_full_spin_single_sensor_dependencies.return_value + ) full_spin_dependencies.dependency_file_paths = [ - Path("folder/ram_map", Path("folder/antiram_map"), Path("folder/l1c"))] + Path("folder/ram_map", Path("folder/antiram_map"), Path("folder/l1c")) + ] mock_process_survival_prob.side_effect = [ sentinel.survival_corrected_ram, @@ -337,33 +476,46 @@ def test_process_full_spin_single_sensor_map(self, mock_unweighted_combination_c processor = HiProcessor(sentinel.dependencies, input_metadata) product = processor.process(spice_frame_name=sentinel.spice_frame) - mock_fetch_full_spin_single_sensor_dependencies.assert_called_once_with(sentinel.dependencies) - - mock_process_survival_prob.assert_has_calls([ - call(full_spin_dependencies.ram_dependencies, sentinel.spice_frame), - call(full_spin_dependencies.antiram_dependencies, sentinel.spice_frame) - ]) - - mock_unweighted_combination.combine_rectangular_intensity_map_data.assert_called_once_with([ - sentinel.survival_corrected_ram, sentinel.survival_corrected_antiram]) - - mock_save_data.assert_called_once_with(RectangularIntensityDataProduct( - input_metadata=input_metadata, - parent_file_names=["antiram_map", "l1c", "ram_map"], - data=mock_unweighted_combination.combine_rectangular_intensity_map_data.return_value, - spice_frame_name=sentinel.spice_frame, - )) + mock_fetch_full_spin_single_sensor_dependencies.assert_called_once_with( + sentinel.dependencies + ) + + mock_process_survival_prob.assert_has_calls( + [ + call(full_spin_dependencies.ram_dependencies, sentinel.spice_frame), + call(full_spin_dependencies.antiram_dependencies, sentinel.spice_frame), + ] + ) + + mock_unweighted_combination.combine_rectangular_intensity_map_data.assert_called_once_with( + [sentinel.survival_corrected_ram, sentinel.survival_corrected_antiram] + ) + + mock_save_data.assert_called_once_with( + RectangularIntensityDataProduct( + input_metadata=input_metadata, + parent_file_names=["antiram_map", "l1c", "ram_map"], + data=mock_unweighted_combination.combine_rectangular_intensity_map_data.return_value, + spice_frame_name=sentinel.spice_frame, + ) + ) self.assertEqual([mock_save_data.return_value], product) - @patch('imap_l3_processing.hi.hi_processor.MapProcessor.get_parent_file_names') - @patch('imap_l3_processing.hi.hi_processor.save_data') + @patch("imap_l3_processing.hi.hi_processor.MapProcessor.get_parent_file_names") + @patch("imap_l3_processing.hi.hi_processor.save_data") @patch("imap_l3_processing.hi.hi_processor.process_survival_probabilities") - @patch("imap_l3_processing.hi.hi_processor.HiL3SingleSensorFullSpinDependencies.fetch_dependencies") + @patch( + "imap_l3_processing.hi.hi_processor.HiL3SingleSensorFullSpinDependencies.fetch_dependencies" + ) @patch("imap_l3_processing.hi.hi_processor.UncertaintyWeightedCombination") - def test_process_cg_corrected_full_spin_map(self, mock_uncertainty_weighted_combination, - mock_fetch_dependencies, - mock_process_survival_probabilities, mock_save_data, - mock_get_parent_file_names): + def test_process_cg_corrected_full_spin_map( + self, + mock_uncertainty_weighted_combination, + mock_fetch_dependencies, + mock_process_survival_probabilities, + mock_save_data, + mock_get_parent_file_names, + ): sensors = ["h45", "h90"] @@ -374,14 +526,21 @@ def test_process_cg_corrected_full_spin_map(self, mock_uncertainty_weighted_comb mock_process_survival_probabilities.reset_mock() mock_save_data.reset_mock() mock_get_parent_file_names.reset_mock() - mock_get_parent_file_names.return_value = ["ram_map", "antiram_map", "l1c"] + mock_get_parent_file_names.return_value = [ + "ram_map", + "antiram_map", + "l1c", + ] mock_combination = Mock() mock_uncertainty_weighted_combination.return_value = mock_combination full_spin_dependencies: HiL3SingleSensorFullSpinDependencies = Mock() full_spin_dependencies.dependency_file_paths = [ - Path("folder/ram_map", Path("folder/antiram_map"), Path("folder/l1c"))] + Path( + "folder/ram_map", Path("folder/antiram_map"), Path("folder/l1c") + ) + ] mock_fetch_dependencies.return_value = full_spin_dependencies mock_process_survival_probabilities.side_effect = [ @@ -389,51 +548,80 @@ def test_process_cg_corrected_full_spin_map(self, mock_uncertainty_weighted_comb sentinel.survival_corrected_antiram, ] - input_metadata = InputMetadata(instrument="hi", - data_level="l3", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="", - descriptor=f"{sensor}-ena-h-hf-sp-full-hae-4deg-6mo") + input_metadata = InputMetadata( + instrument="hi", + data_level="l3", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="", + descriptor=f"{sensor}-ena-h-hf-sp-full-hae-4deg-6mo", + ) processor = HiProcessor(sentinel.dependencies, input_metadata) product = processor.process(spice_frame_name=sentinel.spice_frame) mock_fetch_dependencies.assert_called_once_with(sentinel.dependencies) - mock_process_survival_probabilities.assert_has_calls([ - call(full_spin_dependencies.ram_dependencies, sentinel.spice_frame), - call(full_spin_dependencies.antiram_dependencies, sentinel.spice_frame) - ]) + mock_process_survival_probabilities.assert_has_calls( + [ + call( + full_spin_dependencies.ram_dependencies, + sentinel.spice_frame, + ), + call( + full_spin_dependencies.antiram_dependencies, + sentinel.spice_frame, + ), + ] + ) - mock_combination.combine_rectangular_intensity_map_data.assert_called_once_with([ - sentinel.survival_corrected_ram, sentinel.survival_corrected_antiram]) + mock_combination.combine_rectangular_intensity_map_data.assert_called_once_with( + [ + sentinel.survival_corrected_ram, + sentinel.survival_corrected_antiram, + ] + ) - mock_save_data.assert_called_once_with(RectangularIntensityDataProduct( - input_metadata=input_metadata, - parent_file_names=["antiram_map", "l1c", "ram_map"], - data=mock_combination.combine_rectangular_intensity_map_data.return_value, - spice_frame_name=sentinel.spice_frame, - )) + mock_save_data.assert_called_once_with( + RectangularIntensityDataProduct( + input_metadata=input_metadata, + parent_file_names=["antiram_map", "l1c", "ram_map"], + data=mock_combination.combine_rectangular_intensity_map_data.return_value, + spice_frame_name=sentinel.spice_frame, + ) + ) self.assertEqual([mock_save_data.return_value], product) - @patch('imap_l3_processing.hi.hi_processor.MapProcessor.get_parent_file_names') - @patch('imap_l3_processing.hi.hi_processor.save_data') - @patch("imap_l3_processing.hi.hi_processor.HiL3CombinedMapDependencies.fetch_dependencies") + @patch("imap_l3_processing.hi.hi_processor.MapProcessor.get_parent_file_names") + @patch("imap_l3_processing.hi.hi_processor.save_data") + @patch( + "imap_l3_processing.hi.hi_processor.HiL3CombinedMapDependencies.fetch_dependencies" + ) @patch("imap_l3_processing.hi.hi_processor.ExposureWeightedCombination") @patch("imap_l3_processing.hi.hi_processor.UncertaintyWeightedCombination") - def test_process_combined_sensor_map(self, mock_uncertainty_weighted_combination_class, - mock_exposure_weighted_combination_class, mock_fetch_dependencies, - mock_save_data, mock_get_parent_file_names): + def test_process_combined_sensor_map( + self, + mock_uncertainty_weighted_combination_class, + mock_exposure_weighted_combination_class, + mock_fetch_dependencies, + mock_save_data, + mock_get_parent_file_names, + ): mock_exposure_combination = Mock() - mock_exposure_weighted_combination_class.return_value = mock_exposure_combination + mock_exposure_weighted_combination_class.return_value = ( + mock_exposure_combination + ) mock_uncertainty_weighted_combination = Mock() - mock_uncertainty_weighted_combination_class.return_value = mock_uncertainty_weighted_combination + mock_uncertainty_weighted_combination_class.return_value = ( + mock_uncertainty_weighted_combination + ) mock_uncertainty_weighted_combination.combine_rectangular_intensity_map_data.side_effect = [ - sentinel.h45_full_spin, sentinel.h90_full_spin] + sentinel.h45_full_spin, + sentinel.h90_full_spin, + ] parent_map_names = [ "imap_hi_l2_h45-ena-h-hf-sp-full-hae-4deg-6mo_20250101_v000.cdf", @@ -444,33 +632,40 @@ def test_process_combined_sensor_map(self, mock_uncertainty_weighted_combination mock_get_parent_file_names.return_value = parent_map_names - input_metadata = InputMetadata(instrument="hi", - data_level="l3", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="", - descriptor="hic-ena-h-hf-sp-full-hae-4deg-6mo") - dependencies = ProcessingInputCollection(*[ScienceInput(map_name) for map_name in parent_map_names]) + input_metadata = InputMetadata( + instrument="hi", + data_level="l3", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="", + descriptor="hic-ena-h-hf-sp-full-hae-4deg-6mo", + ) + dependencies = ProcessingInputCollection( + *[ScienceInput(map_name) for map_name in parent_map_names] + ) data_product = HiProcessor(dependencies, input_metadata).process() mock_fetch_dependencies.assert_called_once_with(dependencies) - mock_uncertainty_weighted_combination.combine_rectangular_intensity_map_data.assert_has_calls([ - call(mock_fetch_dependencies.return_value.h45_maps), - call(mock_fetch_dependencies.return_value.h90_maps) - ]) - - mock_exposure_combination.combine_rectangular_intensity_map_data.assert_called_once_with([ - sentinel.h45_full_spin, - sentinel.h90_full_spin - ]) - - mock_save_data.assert_called_once_with(RectangularIntensityDataProduct( - input_metadata=input_metadata, - parent_file_names=parent_map_names, - data=mock_exposure_combination.combine_rectangular_intensity_map_data.return_value, - spice_frame_name=SpiceFrame.ECLIPJ2000, - )) + mock_uncertainty_weighted_combination.combine_rectangular_intensity_map_data.assert_has_calls( + [ + call(mock_fetch_dependencies.return_value.h45_maps), + call(mock_fetch_dependencies.return_value.h90_maps), + ] + ) + + mock_exposure_combination.combine_rectangular_intensity_map_data.assert_called_once_with( + [sentinel.h45_full_spin, sentinel.h90_full_spin] + ) + + mock_save_data.assert_called_once_with( + RectangularIntensityDataProduct( + input_metadata=input_metadata, + parent_file_names=parent_map_names, + data=mock_exposure_combination.combine_rectangular_intensity_map_data.return_value, + spice_frame_name=SpiceFrame.ECLIPJ2000, + ) + ) self.assertEqual([mock_save_data.return_value], data_product) @@ -480,37 +675,50 @@ def test_raises_error_for_currently_unimplemented_maps(self): ] for descriptor in cases: with self.subTest(descriptor): - input_metadata = InputMetadata(instrument="hi", - data_level="l3", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="", - descriptor=descriptor, - ) + input_metadata = InputMetadata( + instrument="hi", + data_level="l3", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="", + descriptor=descriptor, + ) processor = HiProcessor(ProcessingInputCollection(), input_metadata) with self.assertRaises(NotImplementedError): product = processor.process() - @patch('imap_l3_processing.hi.hi_processor.HiSpectralIndexDependencies.fetch_dependencies') - @patch('imap_l3_processing.hi.hi_processor.fit_spectral_index_map') - @patch('imap_l3_processing.hi.hi_processor.slice_energy_range_by_bin') - @patch('imap_l3_processing.hi.hi_processor.save_data') - def test_spectral_fit_with_energy_range(self, _, mock_slice_energy_range_by_bin, - mock_fit_spectral_index_map, mock_fetch_dependencies): + @patch( + "imap_l3_processing.hi.hi_processor.HiSpectralIndexDependencies.fetch_dependencies" + ) + @patch("imap_l3_processing.hi.hi_processor.fit_spectral_index_map") + @patch("imap_l3_processing.hi.hi_processor.slice_energy_range_by_bin") + @patch("imap_l3_processing.hi.hi_processor.save_data") + def test_spectral_fit_with_energy_range( + self, + _, + mock_slice_energy_range_by_bin, + mock_fit_spectral_index_map, + mock_fetch_dependencies, + ): upstream_dependencies = ProcessingInputCollection() dependencies = mock_fetch_dependencies.return_value - input_metadata = InputMetadata(instrument="hi", - data_level="", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="", - descriptor="h90-spx1020-h-hf-sp-full-hae-4deg-6mo", - ) + input_metadata = InputMetadata( + instrument="hi", + data_level="", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="", + descriptor="h90-spx1020-h-hf-sp-full-hae-4deg-6mo", + ) processor = HiProcessor(upstream_dependencies, input_metadata) processor.process() - mock_slice_energy_range_by_bin.assert_called_with(dependencies.map_data.intensity_map_data, 10, 20) - mock_fit_spectral_index_map.assert_called_with(mock_slice_energy_range_by_bin.return_value) + mock_slice_energy_range_by_bin.assert_called_with( + dependencies.map_data.intensity_map_data, 10, 20 + ) + mock_fit_spectral_index_map.assert_called_with( + mock_slice_energy_range_by_bin.return_value + ) diff --git a/tests/hi/test_hi_sp_initializer.py b/tests/hi/test_hi_sp_initializer.py index c9b98a140..91fc8bdf9 100644 --- a/tests/hi/test_hi_sp_initializer.py +++ b/tests/hi/test_hi_sp_initializer.py @@ -6,7 +6,10 @@ from imap_data_access.file_validation import Version from imap_l3_processing.hi.hi_sp_initializer import HiSPInitializer, HI_SP_SPICE_KERNELS -from imap_l3_processing.maps.map_descriptors import parse_map_descriptor, map_descriptor_parts_to_string +from imap_l3_processing.maps.map_descriptors import ( + parse_map_descriptor, + map_descriptor_parts_to_string, +) from imap_l3_processing.maps.map_initializer import PossibleMapToProduce from imap_l3_processing.maps.sp_map_initializer import logger from imap_l3_processing.models import InputMetadata, VersionMap @@ -15,56 +18,63 @@ class TestHiSPInitializer(unittest.TestCase): def setUp(self): - self.query_patcher = patch('imap_l3_processing.hi.hi_sp_initializer.imap_data_access.query') + self.query_patcher = patch( + "imap_l3_processing.hi.hi_sp_initializer.imap_data_access.query" + ) self.mock_query = self.query_patcher.start() def tearDown(self): self.query_patcher.stop() - @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') - def test_get_maps_that_can_be_produced_always_returns_possible_maps_with_default_version_map(self, mock_read_cdf_parents): - descriptor = 'h45-ena-h-sf-sp-anti-hae-4deg-3mo' + @patch("imap_l3_processing.maps.map_initializer.read_cdf_parents") + def test_get_maps_that_can_be_produced_always_returns_possible_maps_with_default_version_map( + self, mock_read_cdf_parents + ): + descriptor = "h45-ena-h-sf-sp-anti-hae-4deg-3mo" cases = [ (None, VersionMap({}, Version(None, 1))), - (4, VersionMap({}, Version(None, 1))), + (4, VersionMap({}, Version(None, 1))), ] for major_version, expected_version in cases: with self.subTest(major_version=major_version): - self.mock_query.side_effect = [ - create_mock_query_results([ - 'imap_glows_l3e_survival-probability-hi-45_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100103-repoint00003_v001.cdf', - - 'imap_glows_l3e_survival-probability-hi-45_20100401-repoint00101_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100402-repoint00102_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100403-repoint00103_v001.cdf', - - 'imap_glows_l3e_survival-probability-hi-45_20100702-repoint00202_v001.cdf', - ]), + create_mock_query_results( + [ + "imap_glows_l3e_survival-probability-hi-45_20100101-repoint00001_v001.cdf", + "imap_glows_l3e_survival-probability-hi-45_20100102-repoint00002_v001.cdf", + "imap_glows_l3e_survival-probability-hi-45_20100103-repoint00003_v001.cdf", + "imap_glows_l3e_survival-probability-hi-45_20100401-repoint00101_v001.cdf", + "imap_glows_l3e_survival-probability-hi-45_20100402-repoint00102_v001.cdf", + "imap_glows_l3e_survival-probability-hi-45_20100403-repoint00103_v001.cdf", + "imap_glows_l3e_survival-probability-hi-45_20100702-repoint00202_v001.cdf", + ] + ), + create_mock_query_results([]), + create_mock_query_results( + [ + "imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf", + "imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf", + "imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf", + "imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20101001_v001.cdf", + ] + ), create_mock_query_results([]), - create_mock_query_results([ - 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf', - 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf', - 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf', - 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20101001_v001.cdf' - ]), - create_mock_query_results([]) ] - mock_read_cdf_parents.side_effect = self.create_fake_read_cdf_parents("45") + mock_read_cdf_parents.side_effect = self.create_fake_read_cdf_parents( + "45" + ) expected_possible_maps = [ PossibleMapToProduce( input_files={ - 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100103-repoint00003_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20100101-repoint00001_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20100102-repoint00002_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20100103-repoint00003_v001.cdf', + "imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf", + "imap_glows_l3e_survival-probability-hi-45_20100101-repoint00001_v001.cdf", + "imap_glows_l3e_survival-probability-hi-45_20100102-repoint00002_v001.cdf", + "imap_glows_l3e_survival-probability-hi-45_20100103-repoint00003_v001.cdf", + "imap_hi_l1c_45sensor-pset_20100101-repoint00001_v001.cdf", + "imap_hi_l1c_45sensor-pset_20100102-repoint00002_v001.cdf", + "imap_hi_l1c_45sensor-pset_20100103-repoint00003_v001.cdf", }, input_metadata=InputMetadata( instrument="hi", @@ -73,17 +83,17 @@ def test_get_maps_that_can_be_produced_always_returns_possible_maps_with_default end_date=datetime(2010, 4, 2, 7, 30), version=expected_version, descriptor=descriptor, - ) + ), ), PossibleMapToProduce( input_files={ - 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100401-repoint00101_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100402-repoint00102_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100403-repoint00103_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20100401-repoint00101_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20100402-repoint00102_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20100403-repoint00103_v001.cdf', + "imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf", + "imap_glows_l3e_survival-probability-hi-45_20100401-repoint00101_v001.cdf", + "imap_glows_l3e_survival-probability-hi-45_20100402-repoint00102_v001.cdf", + "imap_glows_l3e_survival-probability-hi-45_20100403-repoint00103_v001.cdf", + "imap_hi_l1c_45sensor-pset_20100401-repoint00101_v001.cdf", + "imap_hi_l1c_45sensor-pset_20100402-repoint00102_v001.cdf", + "imap_hi_l1c_45sensor-pset_20100403-repoint00103_v001.cdf", }, input_metadata=InputMetadata( instrument="hi", @@ -92,15 +102,15 @@ def test_get_maps_that_can_be_produced_always_returns_possible_maps_with_default end_date=datetime(2010, 7, 1, 7, 30), version=expected_version, descriptor=descriptor, - ) + ), ), PossibleMapToProduce( input_files={ - 'imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20100702-repoint00202_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20100401-repoint00201_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20100402-repoint00202_v001.cdf', - 'imap_hi_l1c_45sensor-pset_20100403-repoint00203_v001.cdf', + "imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf", + "imap_glows_l3e_survival-probability-hi-45_20100702-repoint00202_v001.cdf", + "imap_hi_l1c_45sensor-pset_20100401-repoint00201_v001.cdf", + "imap_hi_l1c_45sensor-pset_20100402-repoint00202_v001.cdf", + "imap_hi_l1c_45sensor-pset_20100403-repoint00203_v001.cdf", }, input_metadata=InputMetadata( instrument="hi", @@ -109,333 +119,462 @@ def test_get_maps_that_can_be_produced_always_returns_possible_maps_with_default end_date=datetime(2010, 9, 30, 7, 30), version=expected_version, descriptor=descriptor, - ) + ), ), ] initializer = HiSPInitializer() - self.mock_query.assert_has_calls([ - call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-45', version="latest"), - call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-90', version="latest"), - call(instrument='hi', data_level='l2'), - call(instrument='hi', data_level='l3'), - ]) + self.mock_query.assert_has_calls( + [ + call( + instrument="glows", + data_level="l3e", + descriptor="survival-probability-hi-45", + version="latest", + ), + call( + instrument="glows", + data_level="l3e", + descriptor="survival-probability-hi-90", + version="latest", + ), + call(instrument="hi", data_level="l2"), + call(instrument="hi", data_level="l3"), + ] + ) - actual_possible_maps = initializer.get_maps_that_can_be_produced('h45-ena-h-sf-sp-anti-hae-4deg-3mo') + actual_possible_maps = initializer.get_maps_that_can_be_produced( + "h45-ena-h-sf-sp-anti-hae-4deg-3mo" + ) - mock_read_cdf_parents.assert_has_calls([ - call('imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf'), - call('imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf'), - call('imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf'), - call('imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20101001_v001.cdf'), - ], any_order=True) + mock_read_cdf_parents.assert_has_calls( + [ + call( + "imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf" + ), + call( + "imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf" + ), + call( + "imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf" + ), + call( + "imap_hi_l2_h45-ena-h-sf-nsp-anti-hae-4deg-3mo_20101001_v001.cdf" + ), + ], + any_order=True, + ) self.assertEqual(expected_possible_maps, actual_possible_maps) - - @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') - def test_get_maps_that_can_be_produced_full_spin_descriptor(self, mock_read_cdf_parents): + @patch("imap_l3_processing.maps.map_initializer.read_cdf_parents") + def test_get_maps_that_can_be_produced_full_spin_descriptor( + self, mock_read_cdf_parents + ): self.mock_query.side_effect = [ create_mock_query_results([]), - create_mock_query_results([ - 'imap_glows_l3e_survival-probability-hi-90_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100103-repoint00003_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100104-repoint00004_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100703-repoint00203_v001.cdf', - - ]), - create_mock_query_results([ - 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20090101_v001.cdf', - 'imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20090701_v001.cdf', - 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20100101_v001.cdf', - 'imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20100101_v001.cdf', - ]), - create_mock_query_results([]) + create_mock_query_results( + [ + "imap_glows_l3e_survival-probability-hi-90_20100101-repoint00001_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100102-repoint00002_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100103-repoint00003_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100104-repoint00004_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100703-repoint00203_v001.cdf", + ] + ), + create_mock_query_results( + [ + "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20090101_v001.cdf", + "imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20090701_v001.cdf", + "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20100101_v001.cdf", + "imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20100101_v001.cdf", + ] + ), + create_mock_query_results([]), ] mock_read_cdf_parents.side_effect = [ [ - 'imap_hi_l1c_90sensor-pset_20100101-repoint00001_v001.cdf', - 'imap_hi_l1c_90sensor-pset_20100102-repoint00002_v001.cdf', + "imap_hi_l1c_90sensor-pset_20100101-repoint00001_v001.cdf", + "imap_hi_l1c_90sensor-pset_20100102-repoint00002_v001.cdf", ], [ - 'imap_hi_l1c_90sensor-pset_20100101-repoint00001_v001.cdf', - 'imap_hi_l1c_90sensor-pset_20100102-repoint00002_v001.cdf', - ] + "imap_hi_l1c_90sensor-pset_20100101-repoint00001_v001.cdf", + "imap_hi_l1c_90sensor-pset_20100102-repoint00002_v001.cdf", + ], ] initializer = HiSPInitializer() - descriptor = 'h90-ena-h-sf-sp-full-hae-4deg-6mo' + descriptor = "h90-ena-h-sf-sp-full-hae-4deg-6mo" actual_possible_maps = initializer.get_maps_that_can_be_produced(descriptor) - self.mock_query.assert_has_calls([ - call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-45', version="latest"), - call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-90', version="latest"), - call(instrument='hi', data_level='l2'), - call(instrument='hi', data_level='l3'), - ]) + self.mock_query.assert_has_calls( + [ + call( + instrument="glows", + data_level="l3e", + descriptor="survival-probability-hi-45", + version="latest", + ), + call( + instrument="glows", + data_level="l3e", + descriptor="survival-probability-hi-90", + version="latest", + ), + call(instrument="hi", data_level="l2"), + call(instrument="hi", data_level="l3"), + ] + ) - mock_read_cdf_parents.assert_has_calls([ - call('imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20100101_v001.cdf'), - call('imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20100101_v001.cdf'), - ]) + mock_read_cdf_parents.assert_has_calls( + [ + call("imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20100101_v001.cdf"), + call("imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20100101_v001.cdf"), + ] + ) expected_possible_map_to_produce = PossibleMapToProduce( input_files={ - 'imap_glows_l3e_survival-probability-hi-90_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100102-repoint00002_v001.cdf', - 'imap_hi_l1c_90sensor-pset_20100101-repoint00001_v001.cdf', - 'imap_hi_l1c_90sensor-pset_20100102-repoint00002_v001.cdf', - 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20100101_v001.cdf', - 'imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20100101_v001.cdf', + "imap_glows_l3e_survival-probability-hi-90_20100101-repoint00001_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100102-repoint00002_v001.cdf", + "imap_hi_l1c_90sensor-pset_20100101-repoint00001_v001.cdf", + "imap_hi_l1c_90sensor-pset_20100102-repoint00002_v001.cdf", + "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20100101_v001.cdf", + "imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20100101_v001.cdf", }, input_metadata=InputMetadata( instrument="hi", data_level="l3", start_date=datetime(2010, 1, 1), end_date=datetime(2010, 7, 2, 15), - version=VersionMap({},Version(None,1)), + version=VersionMap({}, Version(None, 1)), descriptor=descriptor, - ) + ), ) self.assertEqual([expected_possible_map_to_produce], actual_possible_maps) - @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') - def test_get_maps_that_can_be_produced_full_spin_with_mismatched_parents(self, mock_read_cdf_parents): + @patch("imap_l3_processing.maps.map_initializer.read_cdf_parents") + def test_get_maps_that_can_be_produced_full_spin_with_mismatched_parents( + self, mock_read_cdf_parents + ): self.mock_query.side_effect = [ create_mock_query_results([]), - create_mock_query_results([ - 'imap_glows_l3e_survival-probability-hi-90_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100103-repoint00003_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100104-repoint00004_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100703-repoint00203_v001.cdf', - ]), - create_mock_query_results([ - 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20100101_v001.cdf', - 'imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20100101_v001.cdf', - ]), - create_mock_query_results([]) + create_mock_query_results( + [ + "imap_glows_l3e_survival-probability-hi-90_20100101-repoint00001_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100102-repoint00002_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100103-repoint00003_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100104-repoint00004_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100703-repoint00203_v001.cdf", + ] + ), + create_mock_query_results( + [ + "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20100101_v001.cdf", + "imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20100101_v001.cdf", + ] + ), + create_mock_query_results([]), ] mock_read_cdf_parents.side_effect = [ [ - 'imap_hi_l1c_90sensor-pset_20100101-repoint00001_v001.cdf', - 'imap_hi_l1c_90sensor-pset_20100102-repoint00002_v001.cdf', + "imap_hi_l1c_90sensor-pset_20100101-repoint00001_v001.cdf", + "imap_hi_l1c_90sensor-pset_20100102-repoint00002_v001.cdf", ], [ - 'imap_hi_l1c_90sensor-pset_20100101-repoint00001_v001.cdf', - ] + "imap_hi_l1c_90sensor-pset_20100101-repoint00001_v001.cdf", + ], ] initializer = HiSPInitializer() - with self.assertLogs(logger=logger, level='WARNING') as log_context: - actual_possible_maps = initializer.get_maps_that_can_be_produced('h90-ena-h-sf-sp-full-hae-4deg-6mo') - expected_message = ('Expected all input maps to be created from the same pointing sets! l2_file_paths: ' - 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20100101_v001.cdf, ' - 'imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20100101_v001.cdf') + with self.assertLogs(logger=logger, level="WARNING") as log_context: + actual_possible_maps = initializer.get_maps_that_can_be_produced( + "h90-ena-h-sf-sp-full-hae-4deg-6mo" + ) + expected_message = ( + "Expected all input maps to be created from the same pointing sets! l2_file_paths: " + "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20100101_v001.cdf, " + "imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20100101_v001.cdf" + ) self.assertIn(expected_message, log_context.output[0]) + self.mock_query.assert_has_calls( + [ + call( + instrument="glows", + data_level="l3e", + descriptor="survival-probability-hi-45", + version="latest", + ), + call( + instrument="glows", + data_level="l3e", + descriptor="survival-probability-hi-90", + version="latest", + ), + call(instrument="hi", data_level="l2"), + call(instrument="hi", data_level="l3"), + ] + ) - self.mock_query.assert_has_calls([ - call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-45', version="latest"), - call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-90', version="latest"), - call(instrument='hi', data_level='l2'), - call(instrument='hi', data_level='l3'), - ]) - - mock_read_cdf_parents.assert_has_calls([ - call('imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20100101_v001.cdf'), - call('imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20100101_v001.cdf'), - ]) + mock_read_cdf_parents.assert_has_calls( + [ + call("imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-6mo_20100101_v001.cdf"), + call("imap_hi_l2_h90-ena-h-sf-nsp-ram-hae-4deg-6mo_20100101_v001.cdf"), + ] + ) self.assertEqual([], actual_possible_maps) - @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') + @patch("imap_l3_processing.maps.map_initializer.read_cdf_parents") def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): self.mock_query.side_effect = [ create_mock_query_results([]), - create_mock_query_results([ - 'imap_glows_l3e_survival-probability-hi-90_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100103-repoint00003_v001.cdf', - - 'imap_glows_l3e_survival-probability-hi-90_20100401-repoint00101_v002.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100402-repoint00102_v002.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100403-repoint00103_v002.cdf', - - 'imap_glows_l3e_survival-probability-hi-90_20100702-repoint00201_v001.cdf', - ]), - create_mock_query_results([ - 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf', - 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v000.cdf', - 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf', - 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf', - 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20101001_v001.cdf' - - ]), - create_mock_query_results([ - 'imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf', - 'imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v000.cdf', - 'imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100401_v001.cdf', - ]), + create_mock_query_results( + [ + "imap_glows_l3e_survival-probability-hi-90_20100101-repoint00001_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100102-repoint00002_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100103-repoint00003_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100401-repoint00101_v002.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100402-repoint00102_v002.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100403-repoint00103_v002.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100702-repoint00201_v001.cdf", + ] + ), + create_mock_query_results( + [ + "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf", + "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v000.cdf", + "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf", + "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf", + "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20101001_v001.cdf", + ] + ), + create_mock_query_results( + [ + "imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf", + "imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v000.cdf", + "imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100401_v001.cdf", + ] + ), ] mock_read_cdf_parents.side_effect = self.create_fake_read_cdf_parents("90") - descriptor = 'h90-ena-h-sf-sp-anti-hae-4deg-3mo' + descriptor = "h90-ena-h-sf-sp-anti-hae-4deg-3mo" expected_possible_maps = [ PossibleMapToProduce( input_files={ - 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100401-repoint00101_v002.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100402-repoint00102_v002.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100403-repoint00103_v002.cdf', - 'imap_hi_l1c_90sensor-pset_20100401-repoint00101_v001.cdf', - 'imap_hi_l1c_90sensor-pset_20100402-repoint00102_v001.cdf', - 'imap_hi_l1c_90sensor-pset_20100403-repoint00103_v001.cdf', + "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100401-repoint00101_v002.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100402-repoint00102_v002.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100403-repoint00103_v002.cdf", + "imap_hi_l1c_90sensor-pset_20100401-repoint00101_v001.cdf", + "imap_hi_l1c_90sensor-pset_20100402-repoint00102_v001.cdf", + "imap_hi_l1c_90sensor-pset_20100403-repoint00103_v001.cdf", }, input_metadata=InputMetadata( instrument="hi", data_level="l3", start_date=datetime(2010, 4, 1), end_date=datetime(2010, 7, 1, 7, 30), - version=VersionMap({descriptor:Version(None,2)}), + version=VersionMap({descriptor: Version(None, 2)}), descriptor=descriptor, - ) + ), ), PossibleMapToProduce( input_files={ - 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100702-repoint00201_v001.cdf', - 'imap_hi_l1c_90sensor-pset_20100401-repoint00201_v001.cdf', - 'imap_hi_l1c_90sensor-pset_20100402-repoint00202_v001.cdf', - 'imap_hi_l1c_90sensor-pset_20100403-repoint00203_v001.cdf', + "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100702-repoint00201_v001.cdf", + "imap_hi_l1c_90sensor-pset_20100401-repoint00201_v001.cdf", + "imap_hi_l1c_90sensor-pset_20100402-repoint00202_v001.cdf", + "imap_hi_l1c_90sensor-pset_20100403-repoint00203_v001.cdf", }, input_metadata=InputMetadata( instrument="hi", data_level="l3", start_date=datetime(2010, 7, 1), end_date=datetime(2010, 9, 30, 7, 30), - version=VersionMap({descriptor:Version(None,1)}), + version=VersionMap({descriptor: Version(None, 1)}), descriptor=descriptor, - ) + ), ), ] initializer = HiSPInitializer() - self.mock_query.assert_has_calls([ - call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-45', version="latest"), - call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-90', version="latest"), - call(instrument='hi', data_level='l2'), - call(instrument='hi', data_level='l3'), - ]) - - actual_possible_maps = initializer.get_maps_that_should_be_produced(descriptor, None) + self.mock_query.assert_has_calls( + [ + call( + instrument="glows", + data_level="l3e", + descriptor="survival-probability-hi-45", + version="latest", + ), + call( + instrument="glows", + data_level="l3e", + descriptor="survival-probability-hi-90", + version="latest", + ), + call(instrument="hi", data_level="l2"), + call(instrument="hi", data_level="l3"), + ] + ) - mock_read_cdf_parents.assert_has_calls([ - call('imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf'), - call('imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf'), - call('imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf'), - call('imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20101001_v001.cdf'), + actual_possible_maps = initializer.get_maps_that_should_be_produced( + descriptor, None + ) - call('imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf'), - call('imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100401_v001.cdf'), - ]) + mock_read_cdf_parents.assert_has_calls( + [ + call("imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf"), + call("imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf"), + call("imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf"), + call("imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20101001_v001.cdf"), + call("imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf"), + call("imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100401_v001.cdf"), + ] + ) self.assertEqual(expected_possible_maps, actual_possible_maps) - - - @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') - def test_get_maps_that_should_be_produced_remakes_when_major_version_changes(self, mock_read_cdf_parents): - for major_version in [2,3]: + @patch("imap_l3_processing.maps.map_initializer.read_cdf_parents") + def test_get_maps_that_should_be_produced_remakes_when_major_version_changes( + self, mock_read_cdf_parents + ): + for major_version in [2, 3]: with self.subTest(major_version=major_version): self.mock_query.reset_mock() mock_read_cdf_parents.reset_mock() self.mock_query.side_effect = [ create_mock_query_results([]), - create_mock_query_results([ - 'imap_glows_l3e_survival-probability-hi-90_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100103-repoint00003_v001.cdf', - ]), - create_mock_query_results([ - 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf', - 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v000.cdf', - ]), - create_mock_query_results([ - 'imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf', - 'imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v000.cdf', - ]), + create_mock_query_results( + [ + "imap_glows_l3e_survival-probability-hi-90_20100101-repoint00001_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100102-repoint00002_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100103-repoint00003_v001.cdf", + ] + ), + create_mock_query_results( + [ + "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf", + "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v000.cdf", + ] + ), + create_mock_query_results( + [ + "imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf", + "imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v000.cdf", + ] + ), ] - mock_read_cdf_parents.side_effect = self.create_fake_read_cdf_parents("90") + mock_read_cdf_parents.side_effect = self.create_fake_read_cdf_parents( + "90" + ) - descriptor = 'h90-ena-h-sf-sp-anti-hae-4deg-3mo' + descriptor = "h90-ena-h-sf-sp-anti-hae-4deg-3mo" expected_possible_maps = [ PossibleMapToProduce( input_files={ - 'imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20100103-repoint00003_v001.cdf', - 'imap_hi_l1c_90sensor-pset_20100101-repoint00001_v001.cdf', - 'imap_hi_l1c_90sensor-pset_20100102-repoint00002_v001.cdf', - 'imap_hi_l1c_90sensor-pset_20100103-repoint00003_v001.cdf', + "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100101-repoint00001_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100102-repoint00002_v001.cdf", + "imap_glows_l3e_survival-probability-hi-90_20100103-repoint00003_v001.cdf", + "imap_hi_l1c_90sensor-pset_20100101-repoint00001_v001.cdf", + "imap_hi_l1c_90sensor-pset_20100102-repoint00002_v001.cdf", + "imap_hi_l1c_90sensor-pset_20100103-repoint00003_v001.cdf", }, input_metadata=InputMetadata( instrument="hi", data_level="l3", start_date=datetime(2010, 1, 1), end_date=datetime(2010, 4, 2, 7, 30), - version=VersionMap({descriptor:Version(major_version,2)}), + version=VersionMap({descriptor: Version(major_version, 2)}), descriptor=descriptor, - ) + ), ), ] initializer = HiSPInitializer() - self.mock_query.assert_has_calls([ - call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-45', version="latest"), - call(instrument='glows', data_level='l3e', descriptor='survival-probability-hi-90', version="latest"), - call(instrument='hi', data_level='l2'), - call(instrument='hi', data_level='l3'), - ]) - - actual_possible_maps = initializer.get_maps_that_should_be_produced(descriptor, major_version) + self.mock_query.assert_has_calls( + [ + call( + instrument="glows", + data_level="l3e", + descriptor="survival-probability-hi-45", + version="latest", + ), + call( + instrument="glows", + data_level="l3e", + descriptor="survival-probability-hi-90", + version="latest", + ), + call(instrument="hi", data_level="l2"), + call(instrument="hi", data_level="l3"), + ] + ) - mock_read_cdf_parents.assert_has_calls([ - call('imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf'), + actual_possible_maps = initializer.get_maps_that_should_be_produced( + descriptor, major_version + ) - call('imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf'), - ]) + mock_read_cdf_parents.assert_has_calls( + [ + call( + "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf" + ), + call( + "imap_hi_l3_h90-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf" + ), + ] + ) self.assertEqual(expected_possible_maps, actual_possible_maps) def test_get_dependencies(self): cases = [ - ("h90-ena-h-sf-sp-anti-hae-4deg-3mo", ["h90-ena-h-sf-nsp-anti-hae-4deg-3mo"]), + ( + "h90-ena-h-sf-sp-anti-hae-4deg-3mo", + ["h90-ena-h-sf-nsp-anti-hae-4deg-3mo"], + ), ("h45-ena-h-sf-sp-ram-hae-4deg-3mo", ["h45-ena-h-sf-nsp-ram-hae-4deg-3mo"]), - ("h45-ena-h-sf-sp-full-hae-4deg-3mo", - ["h45-ena-h-sf-nsp-anti-hae-4deg-3mo", "h45-ena-h-sf-nsp-ram-hae-4deg-3mo"]), - ("h90-ena-h-sf-sp-full-hae-4deg-3mo", - ["h90-ena-h-sf-nsp-anti-hae-4deg-3mo", "h90-ena-h-sf-nsp-ram-hae-4deg-3mo"]), + ( + "h45-ena-h-sf-sp-full-hae-4deg-3mo", + [ + "h45-ena-h-sf-nsp-anti-hae-4deg-3mo", + "h45-ena-h-sf-nsp-ram-hae-4deg-3mo", + ], + ), + ( + "h90-ena-h-sf-sp-full-hae-4deg-3mo", + [ + "h90-ena-h-sf-nsp-anti-hae-4deg-3mo", + "h90-ena-h-sf-nsp-ram-hae-4deg-3mo", + ], + ), ] for descriptor, dependencies in cases: with self.subTest(descriptor): initializer = HiSPInitializer() - actual_dependencies = initializer._get_l2_dependencies(parse_map_descriptor(descriptor)) - self.assertEqual(dependencies, [map_descriptor_parts_to_string(d) for d in actual_dependencies]) + actual_dependencies = initializer._get_l2_dependencies( + parse_map_descriptor(descriptor) + ) + self.assertEqual( + dependencies, + [map_descriptor_parts_to_string(d) for d in actual_dependencies], + ) - @patch('imap_l3_processing.hi.hi_sp_initializer.furnish_spice_metakernel') + @patch("imap_l3_processing.hi.hi_sp_initializer.furnish_spice_metakernel") def test_furnish_spice_dependencies(self, mock_furnish_metakernel): start_date = datetime(2025, 4, 15) end_date = datetime(2025, 7, 15) @@ -453,13 +592,15 @@ def test_furnish_spice_dependencies(self, mock_furnish_metakernel): hi_initializer = HiSPInitializer() hi_initializer.furnish_spice_dependencies(map_to_produce) - mock_furnish_metakernel.assert_called_once_with(start_date=start_date, end_date=end_date, - kernel_types=HI_SP_SPICE_KERNELS) + mock_furnish_metakernel.assert_called_once_with( + start_date=start_date, end_date=end_date, kernel_types=HI_SP_SPICE_KERNELS + ) @patch("imap_l3_processing.maps.map_initializer.generate_imap_input") @patch("imap_l3_processing.maps.map_initializer.ProcessingInputCollection") - def test_possible_maps_to_produce_constructs_processing_input_collection(self, mock_collection, - mock_generate_imap_input): + def test_possible_maps_to_produce_constructs_processing_input_collection( + self, mock_collection, mock_generate_imap_input + ): path1 = "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v000.cdf" path2 = "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100102_v000.cdf" path3 = "imap_hi_l2_h90-ena-h-sf-nsp-anti-hae-4deg-3mo_20100103_v000.cdf" @@ -470,59 +611,67 @@ def test_possible_maps_to_produce_constructs_processing_input_collection(self, m } def mock_generate_input(path): - return {path1: sentinel.file1, path2: sentinel.file2, path3: sentinel.file3}[path] + return { + path1: sentinel.file1, + path2: sentinel.file2, + path3: sentinel.file3, + }[path] mock_generate_imap_input.side_effect = mock_generate_input - processing_input = PossibleMapToProduce(input_files=files, input_metadata=Mock()).processing_input_collection + processing_input = PossibleMapToProduce( + input_files=files, input_metadata=Mock() + ).processing_input_collection - mock_collection.assert_called_once_with(sentinel.file1, sentinel.file2, sentinel.file3) + mock_collection.assert_called_once_with( + sentinel.file1, sentinel.file2, sentinel.file3 + ) self.assertEqual(mock_collection.return_value, processing_input) def create_fake_read_cdf_parents(self, sensor: str) -> Callable[[str], set[str]]: def fake_read_cdf_parents(file_name: str) -> set[str]: mapping = { - f'imap_hi_l2_h{sensor}-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf': { - f'imap_hi_l1c_{sensor}sensor-pset_20100101-repoint00001_v001.cdf', - f'imap_hi_l1c_{sensor}sensor-pset_20100102-repoint00002_v001.cdf', - f'imap_hi_l1c_{sensor}sensor-pset_20100103-repoint00003_v001.cdf', - 'imap_science_0001.tf', + f"imap_hi_l2_h{sensor}-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf": { + f"imap_hi_l1c_{sensor}sensor-pset_20100101-repoint00001_v001.cdf", + f"imap_hi_l1c_{sensor}sensor-pset_20100102-repoint00002_v001.cdf", + f"imap_hi_l1c_{sensor}sensor-pset_20100103-repoint00003_v001.cdf", + "imap_science_0001.tf", }, - f'imap_hi_l2_h{sensor}-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf': { - f'imap_hi_l1c_{sensor}sensor-pset_20100401-repoint00101_v001.cdf', - f'imap_hi_l1c_{sensor}sensor-pset_20100402-repoint00102_v001.cdf', - f'imap_hi_l1c_{sensor}sensor-pset_20100403-repoint00103_v001.cdf', + f"imap_hi_l2_h{sensor}-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf": { + f"imap_hi_l1c_{sensor}sensor-pset_20100401-repoint00101_v001.cdf", + f"imap_hi_l1c_{sensor}sensor-pset_20100402-repoint00102_v001.cdf", + f"imap_hi_l1c_{sensor}sensor-pset_20100403-repoint00103_v001.cdf", }, - f'imap_hi_l2_h{sensor}-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf': { - f'imap_hi_l1c_{sensor}sensor-pset_20100401-repoint00201_v001.cdf', - f'imap_hi_l1c_{sensor}sensor-pset_20100402-repoint00202_v001.cdf', - f'imap_hi_l1c_{sensor}sensor-pset_20100403-repoint00203_v001.cdf', + f"imap_hi_l2_h{sensor}-ena-h-sf-nsp-anti-hae-4deg-3mo_20100701_v001.cdf": { + f"imap_hi_l1c_{sensor}sensor-pset_20100401-repoint00201_v001.cdf", + f"imap_hi_l1c_{sensor}sensor-pset_20100402-repoint00202_v001.cdf", + f"imap_hi_l1c_{sensor}sensor-pset_20100403-repoint00203_v001.cdf", }, - f'imap_hi_l2_h{sensor}-ena-h-sf-nsp-anti-hae-4deg-3mo_20101001_v001.cdf': { - f'imap_hi_l1c_{sensor}sensor-pset_20101001-repoint00301_v001.cdf', - f'imap_hi_l1c_{sensor}sensor-pset_20101002-repoint00302_v001.cdf', - f'imap_hi_l1c_{sensor}sensor-pset_20101003-repoint00303_v001.cdf', + f"imap_hi_l2_h{sensor}-ena-h-sf-nsp-anti-hae-4deg-3mo_20101001_v001.cdf": { + f"imap_hi_l1c_{sensor}sensor-pset_20101001-repoint00301_v001.cdf", + f"imap_hi_l1c_{sensor}sensor-pset_20101002-repoint00302_v001.cdf", + f"imap_hi_l1c_{sensor}sensor-pset_20101003-repoint00303_v001.cdf", }, - f'imap_hi_l3_h{sensor}-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf': { - f'imap_hi_l2_h{sensor}-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf', - f'imap_hi_l1c_{sensor}sensor-pset_20100101-repoint00001_v001.cdf', - f'imap_hi_l1c_{sensor}sensor-pset_20100102-repoint00002_v001.cdf', - f'imap_hi_l1c_{sensor}sensor-pset_20100103-repoint00003_v001.cdf', - f'imap_glows_l3e_survival-probability-hi-{sensor}_20100101-repoint00001_v001.cdf', - f'imap_glows_l3e_survival-probability-hi-{sensor}_20100102-repoint00002_v001.cdf', - f'imap_glows_l3e_survival-probability-hi-{sensor}_20100103-repoint00003_v001.cdf', - 'imap_science_0001.tf', + f"imap_hi_l3_h{sensor}-ena-h-sf-sp-anti-hae-4deg-3mo_20100101_v001.cdf": { + f"imap_hi_l2_h{sensor}-ena-h-sf-nsp-anti-hae-4deg-3mo_20100101_v001.cdf", + f"imap_hi_l1c_{sensor}sensor-pset_20100101-repoint00001_v001.cdf", + f"imap_hi_l1c_{sensor}sensor-pset_20100102-repoint00002_v001.cdf", + f"imap_hi_l1c_{sensor}sensor-pset_20100103-repoint00003_v001.cdf", + f"imap_glows_l3e_survival-probability-hi-{sensor}_20100101-repoint00001_v001.cdf", + f"imap_glows_l3e_survival-probability-hi-{sensor}_20100102-repoint00002_v001.cdf", + f"imap_glows_l3e_survival-probability-hi-{sensor}_20100103-repoint00003_v001.cdf", + "imap_science_0001.tf", }, - f'imap_hi_l3_h{sensor}-ena-h-sf-sp-anti-hae-4deg-3mo_20100401_v001.cdf': { - f'imap_hi_l2_h{sensor}-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf', - f'imap_hi_l1c_{sensor}sensor-pset_20100401-repoint00101_v001.cdf', - f'imap_hi_l1c_{sensor}sensor-pset_20100402-repoint00102_v001.cdf', - f'imap_hi_l1c_{sensor}sensor-pset_20100403-repoint00103_v001.cdf', - f'imap_glows_l3e_survival-probability-hi-{sensor}_20100401-repoint00101_v001.cdf', - f'imap_glows_l3e_survival-probability-hi-{sensor}_20100402-repoint00102_v001.cdf', - f'imap_glows_l3e_survival-probability-hi-{sensor}_20100403-repoint00103_v001.cdf', - 'imap_science_0001.tf', + f"imap_hi_l3_h{sensor}-ena-h-sf-sp-anti-hae-4deg-3mo_20100401_v001.cdf": { + f"imap_hi_l2_h{sensor}-ena-h-sf-nsp-anti-hae-4deg-3mo_20100401_v001.cdf", + f"imap_hi_l1c_{sensor}sensor-pset_20100401-repoint00101_v001.cdf", + f"imap_hi_l1c_{sensor}sensor-pset_20100402-repoint00102_v001.cdf", + f"imap_hi_l1c_{sensor}sensor-pset_20100403-repoint00103_v001.cdf", + f"imap_glows_l3e_survival-probability-hi-{sensor}_20100401-repoint00101_v001.cdf", + f"imap_glows_l3e_survival-probability-hi-{sensor}_20100402-repoint00102_v001.cdf", + f"imap_glows_l3e_survival-probability-hi-{sensor}_20100403-repoint00103_v001.cdf", + "imap_science_0001.tf", }, } return mapping[file_name] diff --git a/tests/hi/test_hi_spectral_fit_dependencies.py b/tests/hi/test_hi_spectral_fit_dependencies.py index fb3f12ede..38cb519fb 100644 --- a/tests/hi/test_hi_spectral_fit_dependencies.py +++ b/tests/hi/test_hi_spectral_fit_dependencies.py @@ -3,17 +3,26 @@ from unittest.mock import patch, Mock import numpy as np -from imap_data_access.processing_input import ScienceInput, ProcessingInputCollection, AncillaryInput - -from imap_l3_processing.hi.hi_spectral_fit_dependencies import HiSpectralIndexDependencies -from imap_l3_processing.maps.map_models import RectangularIntensityMapData, IntensityMapData +from imap_data_access.processing_input import ( + ScienceInput, + ProcessingInputCollection, + AncillaryInput, +) + +from imap_l3_processing.hi.hi_spectral_fit_dependencies import ( + HiSpectralIndexDependencies, +) +from imap_l3_processing.maps.map_models import ( + RectangularIntensityMapData, + IntensityMapData, +) from imap_l3_processing.maps.quality_flags import MapL3Flags class TestHiSpectralFitDependencies(unittest.TestCase): - @patch( - "imap_l3_processing.hi.hi_spectral_fit_dependencies.RectangularIntensityMapData.read_from_path") + "imap_l3_processing.hi.hi_spectral_fit_dependencies.RectangularIntensityMapData.read_from_path" + ) def test_from_file_paths(self, mock_read_from_path): hi_l3 = Path("test_hi_l3_cdf.cdf") @@ -24,15 +33,17 @@ def test_from_file_paths(self, mock_read_from_path): self.assertEqual(mock_read_from_path.call_count, 1) self.assertEqual(mock_read_from_path.return_value, result.map_data) - @patch("imap_l3_processing.hi.hi_spectral_fit_dependencies.imap_data_access.download") @patch( - "imap_l3_processing.hi.hi_spectral_fit_dependencies.RectangularIntensityMapData.read_from_path") + "imap_l3_processing.hi.hi_spectral_fit_dependencies.imap_data_access.download" + ) + @patch( + "imap_l3_processing.hi.hi_spectral_fit_dependencies.RectangularIntensityMapData.read_from_path" + ) def test_fetch_dependencies(self, mock_read_from_path, mock_download): file_name = "imap_hi_l3_h90-spx-h-hf-sp-full-hae-4deg-6mo_20250422_v001.cdf" ancillary_file = "imap_hi_ancillary_20250422_v001.dat" processing_input = ProcessingInputCollection( - ScienceInput(file_name), - AncillaryInput(ancillary_file) + ScienceInput(file_name), AncillaryInput(ancillary_file) ) dependency = HiSpectralIndexDependencies.fetch_dependencies(processing_input) @@ -43,8 +54,7 @@ def test_fetch_dependencies(self, mock_read_from_path, mock_download): def test_raises_value_error_if_instrument_doesnt_match(self): file_name = "imap_lo_l3_h90-spx-h-hf-sp-full-hae-4deg-6mo_20250422_v001.cdf" - processing_input = ProcessingInputCollection( - ScienceInput(file_name)) + processing_input = ProcessingInputCollection(ScienceInput(file_name)) with self.assertRaises(ValueError) as error: _ = HiSpectralIndexDependencies.fetch_dependencies(processing_input) @@ -56,8 +66,11 @@ def test_fit_energy_ranges_returns_full_energy_range(self): energy_delta_minus = np.full((1,), 0.1) energy_delta_plus = np.full((1,), 0.2) - map_data = _create_intensity_map_data(energy=energy, energy_delta_plus=energy_delta_plus, - energy_delta_minus=energy_delta_minus) + map_data = _create_intensity_map_data( + energy=energy, + energy_delta_plus=energy_delta_plus, + energy_delta_minus=energy_delta_minus, + ) data = RectangularIntensityMapData(map_data, Mock()) deps = HiSpectralIndexDependencies(data) @@ -69,14 +82,20 @@ def _create_intensity_map_data( ): full_shape = (1, 1, 8, 15) return IntensityMapData( - ena_intensity=ena_intensity if ena_intensity is not None else np.full(full_shape, 1), + ena_intensity=ena_intensity + if ena_intensity is not None + else np.full(full_shape, 1), ena_intensity_stat_uncert=None, ena_intensity_sys_err=None, epoch=None, epoch_delta=None, energy=energy if energy is not None else np.full((1,), 1), - energy_delta_plus=energy_delta_plus if energy_delta_plus is not None else np.full((1,), 0.1), - energy_delta_minus=energy_delta_minus if energy_delta_minus is not None else np.full((1,), 0.1), + energy_delta_plus=energy_delta_plus + if energy_delta_plus is not None + else np.full((1,), 0.1), + energy_delta_minus=energy_delta_minus + if energy_delta_minus is not None + else np.full((1,), 0.1), energy_label=None, latitude=None, longitude=None, diff --git a/tests/hi/test_utils.py b/tests/hi/test_utils.py index e0dac7ab0..ae4f8569c 100644 --- a/tests/hi/test_utils.py +++ b/tests/hi/test_utils.py @@ -15,27 +15,30 @@ EPOCH_CDF_VAR_NAME, GLOWS_FLAGS_VAR_NAME, ) -from imap_l3_processing.hi.utils import read_l1c_rectangular_pointing_set_data, read_glows_l3e_data +from imap_l3_processing.hi.utils import ( + read_l1c_rectangular_pointing_set_data, + read_glows_l3e_data, +) from tests.test_helpers import get_test_data_folder, with_tempdir class TestUtils(unittest.TestCase): def setUp(self): - if os.path.exists('imap_hi_l1c_pointing-set_20250101-repoint01111_v000.cdf'): - os.remove('imap_hi_l1c_pointing-set_20250101-repoint01111_v000.cdf') + if os.path.exists("imap_hi_l1c_pointing-set_20250101-repoint01111_v000.cdf"): + os.remove("imap_hi_l1c_pointing-set_20250101-repoint01111_v000.cdf") def tearDown(self): - if os.path.exists('imap_hi_l1c_pointing-set_20250101-repoint01111_v000.cdf'): - os.remove('imap_hi_l1c_pointing-set_20250101-repoint01111_v000.cdf') + if os.path.exists("imap_hi_l1c_pointing-set_20250101-repoint01111_v000.cdf"): + os.remove("imap_hi_l1c_pointing-set_20250101-repoint01111_v000.cdf") def test_read_l1c_data(self): rng = np.random.default_rng() pathname = "imap_hi_l1c_pointing-set_20250101-repoint01111_v000.cdf" - cases = ['exposure_times', 'exposure_time'] + cases = ["exposure_times", "exposure_time"] for exposure_time_var_name in cases: - with CDF(pathname, '') as cdf: + with CDF(pathname, "") as cdf: epoch = np.array([datetime(2000, 1, 2)]) epoch_delta = np.array([1_000_000_000]) repointing = 1111 @@ -48,17 +51,21 @@ def test_read_l1c_data(self): cdf["epoch_delta"] = epoch_delta cdf[exposure_time_var_name] = exposure_times cdf["esa_energy_step"] = energy_step - cdf['hae_longitude'] = hae_longitude - cdf['hae_latitude'] = hae_latitude + cdf["hae_longitude"] = hae_longitude + cdf["hae_latitude"] = hae_latitude for var in cdf: - cdf[var].attrs['FILLVAL'] = 1000000 + cdf[var].attrs["FILLVAL"] = 1000000 for path in [pathname, Path(pathname)]: - with self.subTest(path=path, exposure_time_var_name=exposure_time_var_name): + with self.subTest( + path=path, exposure_time_var_name=exposure_time_var_name + ): result = read_l1c_rectangular_pointing_set_data(path) self.assertEqual(epoch[0], result.epoch) - np.testing.assert_array_equal(result.epoch_delta, epoch_delta, strict=True) + np.testing.assert_array_equal( + result.epoch_delta, epoch_delta, strict=True + ) self.assertEqual(repointing, result.repointing) self.assertEqual([43264184000000], result.epoch_j2000) np.testing.assert_array_equal(exposure_times, result.exposure_times) @@ -69,53 +76,76 @@ def test_read_l1c_data(self): self.tearDown() def test_read_hi_l1c_fill_values_create_nan_data(self): - path = get_test_data_folder() / 'hi' / 'imap_hi_l1c_pset-with-fill-values_20250101-repoint00001_v000.cdf' + path = ( + get_test_data_folder() + / "hi" + / "imap_hi_l1c_pset-with-fill-values_20250101-repoint00001_v000.cdf" + ) result = read_l1c_rectangular_pointing_set_data(path) with CDF(str(path)) as cdf: - self.assertEqual(cdf['epoch'][0], result.epoch) - np.testing.assert_array_equal(result.epoch_delta, cdf['epoch_delta'], strict=True) + self.assertEqual(cdf["epoch"][0], result.epoch) + np.testing.assert_array_equal( + result.epoch_delta, cdf["epoch_delta"], strict=True + ) self.assertEqual(1, result.repointing) - np.testing.assert_array_equal(result.esa_energy_step, cdf['esa_energy_step']) - np.testing.assert_array_equal(result.exposure_times, np.full_like(cdf['exposure_times'], 0)) + np.testing.assert_array_equal( + result.esa_energy_step, cdf["esa_energy_step"] + ) + np.testing.assert_array_equal( + result.exposure_times, np.full_like(cdf["exposure_times"], 0) + ) @patch("imap_l3_processing.hi.utils.met_to_ttj2000ns") def test_read_lo_l1c(self, mock_met_to_ttj2000ns): - path = get_test_data_folder() / 'lo' / 'imap_lo_l1c_pset_20260101-repoint01261_v001.cdf' + path = ( + get_test_data_folder() + / "lo" + / "imap_lo_l1c_pset_20260101-repoint01261_v001.cdf" + ) - mock_met_to_ttj2000ns.side_effect = [ - np.array([3]), - np.array([2]) - ] + mock_met_to_ttj2000ns.side_effect = [np.array([3]), np.array([2])] result = read_l1c_rectangular_pointing_set_data(path) with CDF(str(path)) as cdf: - mock_met_to_ttj2000ns.assert_has_calls([ - call(cdf['pointing_end_met']), - call(cdf['pointing_start_met']) - ]) + mock_met_to_ttj2000ns.assert_has_calls( + [call(cdf["pointing_end_met"]), call(cdf["pointing_start_met"])] + ) - self.assertEqual(cdf['epoch'][0], result.epoch) + self.assertEqual(cdf["epoch"][0], result.epoch) self.assertEqual((1,), result.epoch_delta.shape) np.testing.assert_array_almost_equal(result.epoch_delta, [1]) self.assertEqual(1261, result.repointing) - np.testing.assert_array_equal(result.esa_energy_step, cdf['esa_energy_step']) - np.testing.assert_array_equal(result.exposure_times, cdf['exposure_time'][...]) - self.assertEqual(cdf['pointing_start_met'][0], result.pointing_start_met) - self.assertEqual(cdf['pointing_end_met'][0], result.pointing_end_met) + np.testing.assert_array_equal( + result.esa_energy_step, cdf["esa_energy_step"] + ) + np.testing.assert_array_equal( + result.exposure_times, cdf["exposure_time"][...] + ) + self.assertEqual(cdf["pointing_start_met"][0], result.pointing_start_met) + self.assertEqual(cdf["pointing_end_met"][0], result.pointing_end_met) @with_tempdir def test_read_glows_l3e_data(self, temp_dir: Path): rng = np.random.default_rng() - pathname = str(temp_dir / "imap_glows_l3e_survival-probability-hi_20250415-repoint01000_v001.cdf") + pathname = str( + temp_dir + / "imap_glows_l3e_survival-probability-hi_20250415-repoint01000_v001.cdf" + ) - with CDF(pathname, '') as cdf: + with CDF(pathname, "") as cdf: epoch = np.array([datetime(2000, 1, 2)]) energy = rng.random((16,)) spin_angle = rng.random(360) - probability_of_survival = rng.random((1, 16, 360,)) + probability_of_survival = rng.random( + ( + 1, + 16, + 360, + ) + ) flags = np.array([2**15 | 2**2], dtype=np.uint16) cdf[EPOCH_CDF_VAR_NAME] = epoch @@ -124,7 +154,7 @@ def test_read_glows_l3e_data(self, temp_dir: Path): cdf[PROBABILITY_OF_SURVIVAL_VAR_NAME] = probability_of_survival cdf[GLOWS_FLAGS_VAR_NAME] = flags for var in cdf: - cdf[var].attrs['FILLVAL'] = 1000000 + cdf[var].attrs["FILLVAL"] = 1000000 for path in [pathname, Path(pathname)]: with self.subTest(path=path): @@ -136,11 +166,17 @@ def test_read_glows_l3e_data(self, temp_dir: Path): self.assertEqual(1000, result.repointing) np.testing.assert_array_equal(energy, result.energy) np.testing.assert_array_equal(spin_angle, result.spin_angle) - np.testing.assert_array_equal(probability_of_survival, result.probability_of_survival) + np.testing.assert_array_equal( + probability_of_survival, result.probability_of_survival + ) np.testing.assert_array_equal(flags, result.flags) def test_read_glows_l3e_fill_values_create_nan_data(self): - path = get_test_data_folder() / 'hi' / 'imap_glows_l3e_survival-probabilities-with-fill-values_20250101-repoint01111_v000.cdf' + path = ( + get_test_data_folder() + / "hi" + / "imap_glows_l3e_survival-probabilities-with-fill-values_20250101-repoint01111_v000.cdf" + ) result = read_glows_l3e_data(path) with CDF(str(path)) as cdf: @@ -148,8 +184,14 @@ def test_read_glows_l3e_fill_values_create_nan_data(self): self.assertEqual(cdf.raw_var(EPOCH_CDF_VAR_NAME)[...], result.epoch_j2000) self.assertEqual(1111, result.repointing) - np.testing.assert_array_equal(result.energy, np.full_like(cdf[ENERGY_VAR_NAME], np.nan)) - np.testing.assert_array_equal(result.spin_angle, np.full_like(cdf[SPIN_ANGLE_VAR_NAME], np.nan)) - np.testing.assert_array_equal(result.probability_of_survival, - np.full_like(cdf[PROBABILITY_OF_SURVIVAL_VAR_NAME], np.nan)) - np.testing.assert_array_equal(result.flags, [0]) \ No newline at end of file + np.testing.assert_array_equal( + result.energy, np.full_like(cdf[ENERGY_VAR_NAME], np.nan) + ) + np.testing.assert_array_equal( + result.spin_angle, np.full_like(cdf[SPIN_ANGLE_VAR_NAME], np.nan) + ) + np.testing.assert_array_equal( + result.probability_of_survival, + np.full_like(cdf[PROBABILITY_OF_SURVIVAL_VAR_NAME], np.nan), + ) + np.testing.assert_array_equal(result.flags, [0]) diff --git a/tests/hit/l3/hit_test_builders.py b/tests/hit/l3/hit_test_builders.py index a530a2ce7..dc85ad0a2 100644 --- a/tests/hit/l3/hit_test_builders.py +++ b/tests/hit/l3/hit_test_builders.py @@ -1,19 +1,51 @@ -from imap_l3_processing.hit.l3.pha.pha_event_reader import RawPHAEvent, PHAWord, Detector +from imap_l3_processing.hit.l3.pha.pha_event_reader import ( + RawPHAEvent, + PHAWord, + Detector, +) -def create_raw_pha_event(particle_id=0, priority_buffer_num=0, stim_tag=False, haz_tag=False, time_tag=False, - a_b_side=False, has_unread=False, long_event_flag=False, culling_flag=False, spare=False, - pha_words=None) -> RawPHAEvent: +def create_raw_pha_event( + particle_id=0, + priority_buffer_num=0, + stim_tag=False, + haz_tag=False, + time_tag=False, + a_b_side=False, + has_unread=False, + long_event_flag=False, + culling_flag=False, + spare=False, + pha_words=None, +) -> RawPHAEvent: if pha_words is None: pha_words = [ - PHAWord(adc_value=0, adc_overflow=False, detector=Detector.from_address(1), is_low_gain=False, - is_last_pha=False), - PHAWord(adc_value=10, adc_overflow=False, detector=Detector.from_address(8), is_low_gain=True, - is_last_pha=True) + PHAWord( + adc_value=0, + adc_overflow=False, + detector=Detector.from_address(1), + is_low_gain=False, + is_last_pha=False, + ), + PHAWord( + adc_value=10, + adc_overflow=False, + detector=Detector.from_address(8), + is_low_gain=True, + is_last_pha=True, + ), ] - return RawPHAEvent(particle_id=particle_id, priority_buffer_num=priority_buffer_num, haz_tag=haz_tag, - stim_tag=stim_tag, - time_tag=time_tag, long_event_flag=long_event_flag, a_b_side_flag=a_b_side, - has_unread_adcs=has_unread, - culling_flag=culling_flag, spare=spare, pha_words=pha_words) + return RawPHAEvent( + particle_id=particle_id, + priority_buffer_num=priority_buffer_num, + haz_tag=haz_tag, + stim_tag=stim_tag, + time_tag=time_tag, + long_event_flag=long_event_flag, + a_b_side_flag=a_b_side, + has_unread_adcs=has_unread, + culling_flag=culling_flag, + spare=spare, + pha_words=pha_words, + ) diff --git a/tests/hit/l3/pha/science/test_calculate_pha.py b/tests/hit/l3/pha/science/test_calculate_pha.py index ec7b3105a..18d85ecae 100644 --- a/tests/hit/l3/pha/science/test_calculate_pha.py +++ b/tests/hit/l3/pha/science/test_calculate_pha.py @@ -5,10 +5,19 @@ import numpy as np from imap_l3_processing.hit.l3.pha.pha_event_reader import PHAWord, Detector -from imap_l3_processing.hit.l3.pha.science.calculate_pha import EventAnalysis, analyze_event, calculate_mev, \ - process_pha_event, EventOutput, compute_charge -from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import DetectedRange, DetectorRange, \ - DetectorSide +from imap_l3_processing.hit.l3.pha.science.calculate_pha import ( + EventAnalysis, + analyze_event, + calculate_mev, + process_pha_event, + EventOutput, + compute_charge, +) +from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import ( + DetectedRange, + DetectorRange, + DetectorSide, +) from imap_l3_processing.hit.l3.pha.science.gain_lookup_table import DetectorGain, Gain from imap_l3_processing.hit.l3.pha.science.hit_event_type_lookup import Rule from imap_l3_processing.hit.l3.pha.science.range_fit_lookup import RangeFitLookup @@ -30,35 +39,60 @@ def test_calculate_mev(self): DetectorGain.LOW: {10: Gain(a=low_gain_a_value, b=low_gain_b_value)}, DetectorGain.HIGH: {22: Gain(a=high_gain_a_value, b=high_gain_b_value)}, } - low_gain_word = PHAWord(adc_value=low_gain_adc_value, is_low_gain=True, detector=Detector.from_address(10), - is_last_pha=False, adc_overflow=False) + low_gain_word = PHAWord( + adc_value=low_gain_adc_value, + is_low_gain=True, + detector=Detector.from_address(10), + is_last_pha=False, + adc_overflow=False, + ) mev = calculate_mev(low_gain_word, gain_lookup_table) self.assertEqual(low_gain_a_value * low_gain_adc_value + low_gain_b_value, mev) - low_gain_word = PHAWord(adc_value=high_gain_adc_value, is_low_gain=False, detector=Detector.from_address(22), - is_last_pha=False, adc_overflow=False) + low_gain_word = PHAWord( + adc_value=high_gain_adc_value, + is_low_gain=False, + detector=Detector.from_address(22), + is_last_pha=False, + adc_overflow=False, + ) mev = calculate_mev(low_gain_word, gain_lookup_table) - self.assertEqual(high_gain_a_value * high_gain_adc_value + high_gain_b_value, mev) + self.assertEqual( + high_gain_a_value * high_gain_adc_value + high_gain_b_value, mev + ) def test_analyze_range(self): - detector_1_in_group_1 = Detector(layer=1, side="A", segment="b", address=10, group="L1A14") - detector_2_in_group_1 = Detector(layer=1, side="A", segment="c", address=11, group="L1A14") - detector_1_in_group_2 = Detector(layer=2, side="A", segment="c", address=15, group="L2A") - detector_1_in_group_3 = Detector(layer=3, side="A", segment="a", address=20, group="L3A") + detector_1_in_group_1 = Detector( + layer=1, side="A", segment="b", address=10, group="L1A14" + ) + detector_2_in_group_1 = Detector( + layer=1, side="A", segment="c", address=11, group="L1A14" + ) + detector_1_in_group_2 = Detector( + layer=2, side="A", segment="c", address=15, group="L2A" + ) + detector_1_in_group_3 = Detector( + layer=3, side="A", segment="a", address=20, group="L3A" + ) - detector_to_pha_value = [(detector_1_in_group_1, 10), - (detector_2_in_group_1, 20), - (detector_1_in_group_2, 20), - (detector_1_in_group_3, 10)] + detector_to_pha_value = [ + (detector_1_in_group_1, 10), + (detector_2_in_group_1, 20), + (detector_1_in_group_2, 20), + (detector_1_in_group_3, 10), + ] - detector_to_word_tuples = self._create_event_from_detector_to_pha_value_dict(detector_to_pha_value) + detector_to_word_tuples = self._create_event_from_detector_to_pha_value_dict( + detector_to_pha_value + ) words = [word for _, word in detector_to_word_tuples.items()] raw_pha_event = create_raw_pha_event(pha_words=words) - rule_stub = Rule(range=DetectedRange(DetectorRange.R2, DetectorSide.A), - included_detector_groups=["L1A14", "L2A"], - excluded_detector_groups=["L2B"] - ) + rule_stub = Rule( + range=DetectedRange(DetectorRange.R2, DetectorSide.A), + included_detector_groups=["L1A14", "L2A"], + excluded_detector_groups=["L2B"], + ) range_lookup_table = Mock() range_lookup_table.lookup_range.return_value = rule_stub @@ -66,36 +100,61 @@ def test_analyze_range(self): range_lookup_table.lookup_range.assert_called_with({"L1A14", "L2A", "L3A"}) - expected_highest_words = [detector_to_word_tuples[str(detector_2_in_group_1)], - detector_to_word_tuples[str(detector_1_in_group_2)]] - expected_event_analysis = EventAnalysis(range=rule_stub.range, - l1_detector=detector_2_in_group_1, - l2_detector=detector_1_in_group_2, - e_delta_word=detector_to_word_tuples[str(detector_2_in_group_1)], - e_prime_word=detector_to_word_tuples[str(detector_1_in_group_2)], - words_with_highest_energy=expected_highest_words) + expected_highest_words = [ + detector_to_word_tuples[str(detector_2_in_group_1)], + detector_to_word_tuples[str(detector_1_in_group_2)], + ] + expected_event_analysis = EventAnalysis( + range=rule_stub.range, + l1_detector=detector_2_in_group_1, + l2_detector=detector_1_in_group_2, + e_delta_word=detector_to_word_tuples[str(detector_2_in_group_1)], + e_prime_word=detector_to_word_tuples[str(detector_1_in_group_2)], + words_with_highest_energy=expected_highest_words, + ) self.assertEqual(expected_event_analysis, event_analysis) def test_analyze_range_mixed_sides(self): - detector_2_in_L2A_group = Detector(layer=2, side="A", segment="c", address=11, group="L2A") - detector_1_in_L2A_group = Detector(layer=2, side="A", segment="b", address=10, group="L2A") - detector_1_in_L2B_group = Detector(layer=2, side="B", segment="c", address=15, group="L2B") - detector_1_in_L1B14_group = Detector(layer=1, side="B", segment="a", address=20, group="L1B14") + detector_2_in_L2A_group = Detector( + layer=2, side="A", segment="c", address=11, group="L2A" + ) + detector_1_in_L2A_group = Detector( + layer=2, side="A", segment="b", address=10, group="L2A" + ) + detector_1_in_L2B_group = Detector( + layer=2, side="B", segment="c", address=15, group="L2B" + ) + detector_1_in_L1B14_group = Detector( + layer=1, side="B", segment="a", address=20, group="L1B14" + ) - detector_to_pha_value = [(detector_1_in_L2A_group, 10), - (detector_2_in_L2A_group, 20), - (detector_1_in_L2B_group, 20), - (detector_1_in_L1B14_group, 10)] + detector_to_pha_value = [ + (detector_1_in_L2A_group, 10), + (detector_2_in_L2A_group, 20), + (detector_1_in_L2B_group, 20), + (detector_1_in_L1B14_group, 10), + ] - detector_to_word_tuples = self._create_event_from_detector_to_pha_value_dict(detector_to_pha_value) + detector_to_word_tuples = self._create_event_from_detector_to_pha_value_dict( + detector_to_pha_value + ) words = [word for _, word in detector_to_word_tuples.items()] raw_pha_event = create_raw_pha_event(pha_words=words) - rule_stub = Rule(range=DetectedRange(DetectorRange.R2, DetectorSide.B), - included_detector_groups=["L2B", "L2A", "L1B14"], - excluded_detector_groups=["L1A14", "L1A0", "L3B", "L1B0", "L4iA", "L4iB", "L4oB"] - ) + rule_stub = Rule( + range=DetectedRange(DetectorRange.R2, DetectorSide.B), + included_detector_groups=["L2B", "L2A", "L1B14"], + excluded_detector_groups=[ + "L1A14", + "L1A0", + "L3B", + "L1B0", + "L4iA", + "L4iB", + "L4oB", + ], + ) range_lookup_table = Mock() range_lookup_table.lookup_range.return_value = rule_stub @@ -109,132 +168,264 @@ def test_analyze_range_mixed_sides(self): detector_to_word_tuples[str(detector_1_in_L1B14_group)], ] - expected_event_analysis = EventAnalysis(range=rule_stub.range, - l1_detector=detector_1_in_L1B14_group, - l2_detector=detector_1_in_L2B_group, - e_delta_word=detector_to_word_tuples[str(detector_1_in_L1B14_group)], - e_prime_word=detector_to_word_tuples[str(detector_1_in_L2B_group)], - words_with_highest_energy=expected_highest_words) + expected_event_analysis = EventAnalysis( + range=rule_stub.range, + l1_detector=detector_1_in_L1B14_group, + l2_detector=detector_1_in_L2B_group, + e_delta_word=detector_to_word_tuples[str(detector_1_in_L1B14_group)], + e_prime_word=detector_to_word_tuples[str(detector_1_in_L2B_group)], + words_with_highest_energy=expected_highest_words, + ) self.assertEqual(expected_event_analysis, event_analysis) def test_analyze_range_with_range_4(self): - detector_1_in_group_1 = Detector(layer=1, side="A", segment="b", address=10, group="L1A14") - detector_2_in_group_1 = Detector(layer=1, side="A", segment="c", address=11, group="L1A14") - detector_1_in_group_2 = Detector(layer=2, side="A", segment="c", address=15, group="L2A") - detector_1_in_group_3 = Detector(layer=3, side="A", segment="a", address=20, group="L3A") - detector_1_in_group_4 = Detector(layer=3, side="B", segment="a", address=21, group="L3B") - detector_1_in_group_5 = Detector(layer=2, side="B", segment="a", address=22, group="L2B") - - detector_to_pha_value = [(detector_1_in_group_1, 10), - (detector_2_in_group_1, 20), - (detector_1_in_group_2, 20), - (detector_1_in_group_3, 11), - (detector_1_in_group_4, 12), - (detector_1_in_group_5, 13), - ] - - detector_to_word_tuples = self._create_event_from_detector_to_pha_value_dict(detector_to_pha_value) + detector_1_in_group_1 = Detector( + layer=1, side="A", segment="b", address=10, group="L1A14" + ) + detector_2_in_group_1 = Detector( + layer=1, side="A", segment="c", address=11, group="L1A14" + ) + detector_1_in_group_2 = Detector( + layer=2, side="A", segment="c", address=15, group="L2A" + ) + detector_1_in_group_3 = Detector( + layer=3, side="A", segment="a", address=20, group="L3A" + ) + detector_1_in_group_4 = Detector( + layer=3, side="B", segment="a", address=21, group="L3B" + ) + detector_1_in_group_5 = Detector( + layer=2, side="B", segment="a", address=22, group="L2B" + ) + + detector_to_pha_value = [ + (detector_1_in_group_1, 10), + (detector_2_in_group_1, 20), + (detector_1_in_group_2, 20), + (detector_1_in_group_3, 11), + (detector_1_in_group_4, 12), + (detector_1_in_group_5, 13), + ] + + detector_to_word_tuples = self._create_event_from_detector_to_pha_value_dict( + detector_to_pha_value + ) words = [word for _, word in detector_to_word_tuples.items()] raw_pha_event = create_raw_pha_event(pha_words=words) - rule_stub = Rule(range=DetectedRange(DetectorRange.R4, DetectorSide.A), - included_detector_groups=["L1A14", "L2A", "L3A", "L3B", "L2B"], - excluded_detector_groups=["L1B"] - ) + rule_stub = Rule( + range=DetectedRange(DetectorRange.R4, DetectorSide.A), + included_detector_groups=["L1A14", "L2A", "L3A", "L3B", "L2B"], + excluded_detector_groups=["L1B"], + ) range_lookup_table = Mock() range_lookup_table.lookup_range.return_value = rule_stub event_analysis = analyze_event(raw_pha_event, range_lookup_table) - range_lookup_table.lookup_range.assert_called_with({"L1A14", "L2A", "L3A", "L3B", "L2B"}) - - expected_highest_words = [detector_to_word_tuples[str(detector_2_in_group_1)], - detector_to_word_tuples[str(detector_1_in_group_2)], - detector_to_word_tuples[str(detector_1_in_group_3)], - detector_to_word_tuples[str(detector_1_in_group_4)], - detector_to_word_tuples[str(detector_1_in_group_5)], - ] - expected_event_analysis = EventAnalysis(range=rule_stub.range, - l1_detector=detector_2_in_group_1, - l2_detector=detector_1_in_group_2, - e_delta_word=detector_to_word_tuples[str(detector_1_in_group_3)], - e_prime_word=detector_to_word_tuples[str(detector_1_in_group_4)], - words_with_highest_energy=expected_highest_words) + range_lookup_table.lookup_range.assert_called_with( + {"L1A14", "L2A", "L3A", "L3B", "L2B"} + ) + + expected_highest_words = [ + detector_to_word_tuples[str(detector_2_in_group_1)], + detector_to_word_tuples[str(detector_1_in_group_2)], + detector_to_word_tuples[str(detector_1_in_group_3)], + detector_to_word_tuples[str(detector_1_in_group_4)], + detector_to_word_tuples[str(detector_1_in_group_5)], + ] + expected_event_analysis = EventAnalysis( + range=rule_stub.range, + l1_detector=detector_2_in_group_1, + l2_detector=detector_1_in_group_2, + e_delta_word=detector_to_word_tuples[str(detector_1_in_group_3)], + e_prime_word=detector_to_word_tuples[str(detector_1_in_group_4)], + words_with_highest_energy=expected_highest_words, + ) self.assertEqual(expected_event_analysis, event_analysis) def test_analyze_event_uses_detectors_with_highest_adc_values(self): - detector_l1ab = Detector(layer=1, side="A", segment="b", address=10, group="L1A14") - detector_l1ac = Detector(layer=1, side="A", segment="c", address=11, group="L1A14") - detector_l2ac = Detector(layer=2, side="A", segment="c", address=15, group="L2A") + detector_l1ab = Detector( + layer=1, side="A", segment="b", address=10, group="L1A14" + ) + detector_l1ac = Detector( + layer=1, side="A", segment="c", address=11, group="L1A14" + ) + detector_l2ac = Detector( + layer=2, side="A", segment="c", address=15, group="L2A" + ) - word_l1ab = PHAWord(adc_value=1, detector=detector_l1ab, is_low_gain=False, adc_overflow=False, - is_last_pha=False) - word_l1ac = PHAWord(adc_value=2, detector=detector_l1ac, is_low_gain=False, adc_overflow=False, - is_last_pha=False) - word_l2ac = PHAWord(adc_value=3, detector=detector_l2ac, is_low_gain=False, adc_overflow=False, - is_last_pha=False) + word_l1ab = PHAWord( + adc_value=1, + detector=detector_l1ab, + is_low_gain=False, + adc_overflow=False, + is_last_pha=False, + ) + word_l1ac = PHAWord( + adc_value=2, + detector=detector_l1ac, + is_low_gain=False, + adc_overflow=False, + is_last_pha=False, + ) + word_l2ac = PHAWord( + adc_value=3, + detector=detector_l2ac, + is_low_gain=False, + adc_overflow=False, + is_last_pha=False, + ) test_cases = [ - ("Higher ADC value is used both high gain", 1, False, 2, False, detector_l1ac, word_l1ac), - ("Higher ADC value is used both low gain", 2, True, 1, True, detector_l1ab, word_l1ab), - ("L1ab Low gain ADC * 20 is greater than L1ac ADC", 1, True, 19, False, detector_l1ab, word_l1ab), - ("L1ab Low gain ADC * 20 is less than L1ac ADC", 1, True, 21, False, detector_l1ac, word_l1ac), - ("Filters out saturated high gain words", 2047, False, 1, False, detector_l1ac, word_l1ac), + ( + "Higher ADC value is used both high gain", + 1, + False, + 2, + False, + detector_l1ac, + word_l1ac, + ), + ( + "Higher ADC value is used both low gain", + 2, + True, + 1, + True, + detector_l1ab, + word_l1ab, + ), + ( + "L1ab Low gain ADC * 20 is greater than L1ac ADC", + 1, + True, + 19, + False, + detector_l1ab, + word_l1ab, + ), + ( + "L1ab Low gain ADC * 20 is less than L1ac ADC", + 1, + True, + 21, + False, + detector_l1ac, + word_l1ac, + ), + ( + "Filters out saturated high gain words", + 2047, + False, + 1, + False, + detector_l1ac, + word_l1ac, + ), ] - for name, l1ab_adc, l1ab_is_low_gain, l1ac_adc, l1ac_is_low_gain, expected_l1_detector, expected_l1_word in test_cases: + for ( + name, + l1ab_adc, + l1ab_is_low_gain, + l1ac_adc, + l1ac_is_low_gain, + expected_l1_detector, + expected_l1_word, + ) in test_cases: with self.subTest(name): word_l1ab.adc_value = l1ab_adc word_l1ab.is_low_gain = l1ab_is_low_gain word_l1ac.adc_value = l1ac_adc word_l1ac.is_low_gain = l1ac_is_low_gain - raw_pha_event = create_raw_pha_event(pha_words=[word_l1ab, word_l1ac, word_l2ac]) + raw_pha_event = create_raw_pha_event( + pha_words=[word_l1ab, word_l1ac, word_l2ac] + ) - rule_stub = Rule(range=DetectedRange(DetectorRange.R2, DetectorSide.A), - included_detector_groups=["L1A14", "L2A"], - excluded_detector_groups=["L1B"] - ) + rule_stub = Rule( + range=DetectedRange(DetectorRange.R2, DetectorSide.A), + included_detector_groups=["L1A14", "L2A"], + excluded_detector_groups=["L1B"], + ) range_lookup_table = Mock() range_lookup_table.lookup_range.return_value = rule_stub event_analysis = analyze_event(raw_pha_event, range_lookup_table) - expected_event_analysis = EventAnalysis(range=rule_stub.range, - l1_detector=expected_l1_detector, - l2_detector=detector_l2ac, - e_delta_word=expected_l1_word, - e_prime_word=word_l2ac, - words_with_highest_energy=[expected_l1_word, word_l2ac]) + expected_event_analysis = EventAnalysis( + range=rule_stub.range, + l1_detector=expected_l1_detector, + l2_detector=detector_l2ac, + e_delta_word=expected_l1_word, + e_prime_word=word_l2ac, + words_with_highest_energy=[expected_l1_word, word_l2ac], + ) self.assertEqual(expected_event_analysis.range, event_analysis.range) - self.assertEqual(expected_event_analysis.l1_detector, event_analysis.l1_detector) - self.assertEqual(expected_event_analysis.l2_detector, event_analysis.l2_detector) - self.assertEqual(expected_event_analysis.e_delta_word, event_analysis.e_delta_word) - self.assertEqual(expected_event_analysis.e_prime_word, event_analysis.e_prime_word) - self.assertEqual(expected_event_analysis.words_with_highest_energy, event_analysis.words_with_highest_energy) + self.assertEqual( + expected_event_analysis.l1_detector, event_analysis.l1_detector + ) + self.assertEqual( + expected_event_analysis.l2_detector, event_analysis.l2_detector + ) + self.assertEqual( + expected_event_analysis.e_delta_word, event_analysis.e_delta_word + ) + self.assertEqual( + expected_event_analysis.e_prime_word, event_analysis.e_prime_word + ) + self.assertEqual( + expected_event_analysis.words_with_highest_energy, + event_analysis.words_with_highest_energy, + ) def test_analyze_event_filters_out_events_with_all_saturated_data(self): - detector_l1ab = Detector(layer=1, side="A", segment="b", address=10, group="L1A14") - detector_l1ac = Detector(layer=1, side="A", segment="c", address=11, group="L1A14") - detector_l2ac = Detector(layer=2, side="A", segment="c", address=15, group="L2A") - - word_l1ab = PHAWord(adc_value=2047, detector=detector_l1ab, is_low_gain=False, adc_overflow=False, - is_last_pha=False) - word_l1ac = PHAWord(adc_value=2047, detector=detector_l1ac, is_low_gain=False, adc_overflow=False, - is_last_pha=False) - word_l2ac = PHAWord(adc_value=1, detector=detector_l2ac, is_low_gain=False, adc_overflow=False, - is_last_pha=False) - - raw_pha_event = create_raw_pha_event(pha_words=[word_l1ab, word_l1ac, word_l2ac]) - - rule_stub = Rule(range=DetectedRange(DetectorRange.R2, DetectorSide.A), - included_detector_groups=["L1A14", "L2A"], - excluded_detector_groups=["L1B"] - ) + detector_l1ab = Detector( + layer=1, side="A", segment="b", address=10, group="L1A14" + ) + detector_l1ac = Detector( + layer=1, side="A", segment="c", address=11, group="L1A14" + ) + detector_l2ac = Detector( + layer=2, side="A", segment="c", address=15, group="L2A" + ) + + word_l1ab = PHAWord( + adc_value=2047, + detector=detector_l1ab, + is_low_gain=False, + adc_overflow=False, + is_last_pha=False, + ) + word_l1ac = PHAWord( + adc_value=2047, + detector=detector_l1ac, + is_low_gain=False, + adc_overflow=False, + is_last_pha=False, + ) + word_l2ac = PHAWord( + adc_value=1, + detector=detector_l2ac, + is_low_gain=False, + adc_overflow=False, + is_last_pha=False, + ) + + raw_pha_event = create_raw_pha_event( + pha_words=[word_l1ab, word_l1ac, word_l2ac] + ) + + rule_stub = Rule( + range=DetectedRange(DetectorRange.R2, DetectorSide.A), + included_detector_groups=["L1A14", "L2A"], + excluded_detector_groups=["L1B"], + ) range_lookup_table = Mock() range_lookup_table.lookup_range.return_value = rule_stub @@ -243,17 +434,29 @@ def test_analyze_event_filters_out_events_with_all_saturated_data(self): self.assertIsNone(event_analysis) def test_analyze_event_handles_no_calc_rules(self): - detector_1_in_group_1 = Detector(layer=1, side="A", segment="b", address=10, group="L1A14") - detector_2_in_group_1 = Detector(layer=1, side="A", segment="c", address=11, group="L1A14") - detector_1_in_group_2 = Detector(layer=2, side="A", segment="c", address=15, group="L2A") - detector_1_in_group_3 = Detector(layer=3, side="A", segment="a", address=20, group="L3A") + detector_1_in_group_1 = Detector( + layer=1, side="A", segment="b", address=10, group="L1A14" + ) + detector_2_in_group_1 = Detector( + layer=1, side="A", segment="c", address=11, group="L1A14" + ) + detector_1_in_group_2 = Detector( + layer=2, side="A", segment="c", address=15, group="L2A" + ) + detector_1_in_group_3 = Detector( + layer=3, side="A", segment="a", address=20, group="L3A" + ) - detector_to_pha_value = [(detector_1_in_group_1, 10), - (detector_2_in_group_1, 20), - (detector_1_in_group_2, 20), - (detector_1_in_group_3, 10)] + detector_to_pha_value = [ + (detector_1_in_group_1, 10), + (detector_2_in_group_1, 20), + (detector_1_in_group_2, 20), + (detector_1_in_group_3, 10), + ] - detector_to_word_tuples = self._create_event_from_detector_to_pha_value_dict(detector_to_pha_value) + detector_to_word_tuples = self._create_event_from_detector_to_pha_value_dict( + detector_to_pha_value + ) words = [word for _, word in detector_to_word_tuples.items()] raw_pha_event = create_raw_pha_event(pha_words=words) @@ -268,13 +471,27 @@ def test_analyze_event_handles_no_calc_rules(self): @patch("imap_l3_processing.hit.l3.pha.science.calculate_pha.analyze_event") @patch("imap_l3_processing.hit.l3.pha.science.calculate_pha.calculate_mev") - def test_process_event_for_event_with_invalid_detector_sequences(self, mock_calculate_mev, mock_analyze_event): - raw_pha_event = create_raw_pha_event(pha_words=[ - PHAWord(detector=Detector.from_address(12), adc_value=10, adc_overflow=False, - is_low_gain=True, is_last_pha=False), - PHAWord(detector=Detector.from_address(28), adc_value=20, + def test_process_event_for_event_with_invalid_detector_sequences( + self, mock_calculate_mev, mock_analyze_event + ): + raw_pha_event = create_raw_pha_event( + pha_words=[ + PHAWord( + detector=Detector.from_address(12), + adc_value=10, + adc_overflow=False, + is_low_gain=True, + is_last_pha=False, + ), + PHAWord( + detector=Detector.from_address(28), + adc_value=20, adc_overflow=False, - is_low_gain=False, is_last_pha=False)]) + is_low_gain=False, + is_last_pha=False, + ), + ] + ) word1_mev = 101.3 word2_mev = 66.8 @@ -284,69 +501,119 @@ def test_process_event_for_event_with_invalid_detector_sequences(self, mock_calc event_output = process_pha_event(raw_pha_event, Mock(), Mock(), Mock(), Mock()) - expected_event_output = EventOutput(original_event=raw_pha_event, energies=[word1_mev, word2_mev], charge=None, - total_energy=None, detected_range=None, e_delta=None, e_prime=None, ) + expected_event_output = EventOutput( + original_event=raw_pha_event, + energies=[word1_mev, word2_mev], + charge=None, + total_energy=None, + detected_range=None, + e_delta=None, + e_prime=None, + ) self.assertEqual(expected_event_output, event_output) @patch("imap_l3_processing.hit.l3.pha.science.calculate_pha.compute_charge") - @patch("imap_l3_processing.hit.l3.pha.science.calculate_pha.analyze_event", spec_set=True) + @patch( + "imap_l3_processing.hit.l3.pha.science.calculate_pha.analyze_event", + spec_set=True, + ) def test_process_pha_event(self, mock_analyze_event, mock_compute_charge): pha_word1_adc_value = 53 pha_word2_adc_value = 92 pha_word3_adc_value = 48 - l1_word = PHAWord(detector=Detector.from_address(12), adc_value=pha_word1_adc_value, adc_overflow=False, - is_low_gain=True, is_last_pha=False) - l2_high_energy_word = PHAWord(detector=Detector.from_address(28), adc_value=pha_word2_adc_value, - adc_overflow=False, - is_low_gain=False, is_last_pha=False) - l2_low_energy_word = PHAWord(detector=Detector.from_address(29), adc_value=pha_word3_adc_value, - adc_overflow=False, - is_low_gain=False, is_last_pha=False) + l1_word = PHAWord( + detector=Detector.from_address(12), + adc_value=pha_word1_adc_value, + adc_overflow=False, + is_low_gain=True, + is_last_pha=False, + ) + l2_high_energy_word = PHAWord( + detector=Detector.from_address(28), + adc_value=pha_word2_adc_value, + adc_overflow=False, + is_low_gain=False, + is_last_pha=False, + ) + l2_low_energy_word = PHAWord( + detector=Detector.from_address(29), + adc_value=pha_word3_adc_value, + adc_overflow=False, + is_low_gain=False, + is_last_pha=False, + ) - raw_pha_event = create_raw_pha_event(pha_words=[l1_word, l2_high_energy_word, l2_low_energy_word]) + raw_pha_event = create_raw_pha_event( + pha_words=[l1_word, l2_high_energy_word, l2_low_energy_word] + ) word1_gain = Gain(a=10.76, b=14.3) word2_gain = Gain(a=25.89, b=11.2) word3_gain = Gain(a=31.3, b=5.9) - mock_analyze_event.return_value = EventAnalysis(range=sentinel.detected_range, l1_detector=sentinel.l1_detector, - l2_detector=sentinel.l2_detector, - e_delta_word=l1_word, - e_prime_word=l2_high_energy_word, - words_with_highest_energy=[l1_word, l2_high_energy_word]) + mock_analyze_event.return_value = EventAnalysis( + range=sentinel.detected_range, + l1_detector=sentinel.l1_detector, + l2_detector=sentinel.l2_detector, + e_delta_word=l1_word, + e_prime_word=l2_high_energy_word, + words_with_highest_energy=[l1_word, l2_high_energy_word], + ) gain_lookup_table = { DetectorGain.HIGH: {28: word2_gain, 29: word3_gain}, - DetectorGain.LOW: {12: word1_gain} + DetectorGain.LOW: {12: word1_gain}, } cosine_correction = 12.2 mock_cosine_correction_table = Mock() - mock_cosine_correction_table.get_cosine_correction.return_value = cosine_correction + mock_cosine_correction_table.get_cosine_correction.return_value = ( + cosine_correction + ) - event_output = process_pha_event(raw_pha_event, mock_cosine_correction_table, gain_lookup_table, - sentinel.range_fit_lookup, sentinel.rule_lookup) + event_output = process_pha_event( + raw_pha_event, + mock_cosine_correction_table, + gain_lookup_table, + sentinel.range_fit_lookup, + sentinel.rule_lookup, + ) mock_analyze_event.assert_called_once_with(raw_pha_event, sentinel.rule_lookup) - mock_cosine_correction_table.get_cosine_correction.assert_called_once_with(sentinel.detected_range, - sentinel.l1_detector, - sentinel.l2_detector) + mock_cosine_correction_table.get_cosine_correction.assert_called_once_with( + sentinel.detected_range, sentinel.l1_detector, sentinel.l2_detector + ) self.assertIsInstance(event_output, EventOutput) - expected_l1_energy = cosine_correction * (word1_gain.a * pha_word1_adc_value + word1_gain.b) - expected_l2_higher_energy = cosine_correction * (word2_gain.a * pha_word2_adc_value + word2_gain.b) - expected_energy3 = cosine_correction * (word3_gain.a * pha_word3_adc_value + word3_gain.b) - self.assertEqual([expected_l1_energy, expected_l2_higher_energy, expected_energy3], event_output.energies) + expected_l1_energy = cosine_correction * ( + word1_gain.a * pha_word1_adc_value + word1_gain.b + ) + expected_l2_higher_energy = cosine_correction * ( + word2_gain.a * pha_word2_adc_value + word2_gain.b + ) + expected_energy3 = cosine_correction * ( + word3_gain.a * pha_word3_adc_value + word3_gain.b + ) + self.assertEqual( + [expected_l1_energy, expected_l2_higher_energy, expected_energy3], + event_output.energies, + ) - self.assertEqual(expected_l1_energy + expected_l2_higher_energy, event_output.total_energy) + self.assertEqual( + expected_l1_energy + expected_l2_higher_energy, event_output.total_energy + ) self.assertEqual(raw_pha_event, event_output.original_event) - mock_compute_charge.assert_called_once_with(sentinel.detected_range, expected_l1_energy, - expected_l2_higher_energy, sentinel.range_fit_lookup) + mock_compute_charge.assert_called_once_with( + sentinel.detected_range, + expected_l1_energy, + expected_l2_higher_energy, + sentinel.range_fit_lookup, + ) self.assertEqual(mock_compute_charge.return_value, event_output.charge) self.assertEqual(sentinel.detected_range, event_output.detected_range) self.assertEqual(expected_l1_energy, event_output.e_delta) @@ -366,33 +633,55 @@ def test_compute_charge(self): for delta_e, index_1, index_2 in cases: with self.subTest(delta_e): mock_double_power_law_fit_params = Mock() - mock_double_power_law_fit_params.evaluate_e_prime.return_value = (charges, delta_e_losses) - - detected_range = DetectedRange(range=DetectorRange.R2, side=DetectorSide.A) + mock_double_power_law_fit_params.evaluate_e_prime.return_value = ( + charges, + delta_e_losses, + ) + + detected_range = DetectedRange( + range=DetectorRange.R2, side=DetectorSide.A + ) e_prime = 200 - charge = compute_charge(detected_range, delta_e, e_prime, - mock_double_power_law_fit_params) + charge = compute_charge( + detected_range, delta_e, e_prime, mock_double_power_law_fit_params + ) - mock_double_power_law_fit_params.evaluate_e_prime.assert_called_once_with(detected_range, - e_prime) + mock_double_power_law_fit_params.evaluate_e_prime.assert_called_once_with( + detected_range, e_prime + ) B = math.log(charges[index_2] / charges[index_1]) / math.log( - delta_e_losses[index_2] / delta_e_losses[index_1]) + delta_e_losses[index_2] / delta_e_losses[index_1] + ) A = charges[index_1] / (delta_e_losses[index_1] ** B) - self.assertAlmostEqual(charge, A * delta_e ** B) + self.assertAlmostEqual(charge, A * delta_e**B) - def test_compute_charge_returns_nan_if_e_prime_or_delta_e_are_outside_of_range(self): + def test_compute_charge_returns_nan_if_e_prime_or_delta_e_are_outside_of_range( + self, + ): detected_range_1 = DetectedRange(range=DetectorRange.R2, side=DetectorSide.A) detected_range_2 = DetectedRange(range=DetectorRange.R3, side=DetectorSide.B) detected_range_3 = DetectedRange(range=DetectorRange.R4, side=DetectorSide.A) range_fit_lookup_table = RangeFitLookup.from_files( - get_test_data_path("hit/pha_events/imap_hit_range-2A-charge-fit-lookup_20250319_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-3A-charge-fit-lookup_20250319_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-4A-charge-fit-lookup_20250319_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-2B-charge-fit-lookup_20250319_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-3B-charge-fit-lookup_20250319_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-4B-charge-fit-lookup_20250319_v000.csv"), + get_test_data_path( + "hit/pha_events/imap_hit_range-2A-charge-fit-lookup_20250319_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-3A-charge-fit-lookup_20250319_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-4A-charge-fit-lookup_20250319_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-2B-charge-fit-lookup_20250319_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-3B-charge-fit-lookup_20250319_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-4B-charge-fit-lookup_20250319_v000.csv" + ), ) test_cases = [ ("detector 1 - delta_e over max", detected_range_1, 430.1, 860, True), @@ -422,35 +711,65 @@ def test_compute_charge_returns_nan_if_e_prime_or_delta_e_are_outside_of_range(s ] for test_name, detected_range, delta_e, e_prime, expected_nan in test_cases: with self.subTest(test_name): - result = compute_charge(detected_range, delta_e=delta_e, e_prime=e_prime, - double_power_law_lookup=range_fit_lookup_table) + result = compute_charge( + detected_range, + delta_e=delta_e, + e_prime=e_prime, + double_power_law_lookup=range_fit_lookup_table, + ) self.assertEqual(expected_nan, np.isnan(result)) - def _create_detector_from_string(self, detector_string: str, detector_group: str = None): + def _create_detector_from_string( + self, detector_string: str, detector_group: str = None + ): layer = int(detector_string[1]) group = detector_group if detector_group is not None else "L1A4c" - return Detector(layer=layer, - side=detector_string[2], - segment=detector_string[3:], - address=0, - group=group) + return Detector( + layer=layer, + side=detector_string[2], + segment=detector_string[3:], + address=0, + group=group, + ) - def _create_event_from_detectors_and_groups(self, detectors: list[str], groups: list[str] = None): + def _create_event_from_detectors_and_groups( + self, detectors: list[str], groups: list[str] = None + ): if groups and len(detectors) != len(groups): - self.assertTrue(False, msg="Detectors and groups must have the same length in setup") + self.assertTrue( + False, msg="Detectors and groups must have the same length in setup" + ) - detector_objects = [self._create_detector_from_string(d, g) for d, g in zip(detectors, groups)] - pha_words = [PHAWord(adc_value=0, adc_overflow=False, detector=d, is_low_gain=False, is_last_pha=False) for d in - detector_objects] + detector_objects = [ + self._create_detector_from_string(d, g) for d, g in zip(detectors, groups) + ] + pha_words = [ + PHAWord( + adc_value=0, + adc_overflow=False, + detector=d, + is_low_gain=False, + is_last_pha=False, + ) + for d in detector_objects + ] raw_pha_event = create_raw_pha_event(pha_words=pha_words) return raw_pha_event - def _create_event_from_detector_to_pha_value_dict(self, detector_to_pha_tuples: list[tuple[Detector, int]]): + def _create_event_from_detector_to_pha_value_dict( + self, detector_to_pha_tuples: list[tuple[Detector, int]] + ): pha_words = [] detector_to_word_tuples = {} for detector, value in detector_to_pha_tuples: - word = PHAWord(adc_value=value, adc_overflow=False, detector=detector, is_low_gain=False, is_last_pha=False) + word = PHAWord( + adc_value=value, + adc_overflow=False, + detector=detector, + is_low_gain=False, + is_last_pha=False, + ) pha_words.append(detector) detector_to_word_tuples[str(detector)] = word diff --git a/tests/hit/l3/pha/science/test_cosine_correction_lookup_table.py b/tests/hit/l3/pha/science/test_cosine_correction_lookup_table.py index c4329b1da..a5be36acf 100644 --- a/tests/hit/l3/pha/science/test_cosine_correction_lookup_table.py +++ b/tests/hit/l3/pha/science/test_cosine_correction_lookup_table.py @@ -1,19 +1,36 @@ import unittest -from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import CosineCorrectionLookupTable, Detector, \ - DetectedRange, DetectorSide, DetectorRange +from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import ( + CosineCorrectionLookupTable, + Detector, + DetectedRange, + DetectorSide, + DetectorRange, +) from tests.test_helpers import get_test_data_path class TestCosineCorrectionLookupTable(unittest.TestCase): def test_can_create_from_file(self): - lookup_table = CosineCorrectionLookupTable(get_test_data_path( - "hit/pha_events/imap_hit_range-2A-cosine-lookup_20250203_v000.csv"), get_test_data_path( - "hit/pha_events/imap_hit_range-3A-cosine-lookup_20250203_v000.csv"), get_test_data_path( - "hit/pha_events/imap_hit_range-4A-cosine-lookup_20250203_v000.csv"), get_test_data_path( - "hit/pha_events/imap_hit_range-2B-cosine-lookup_20250203_v000.csv"), get_test_data_path( - "hit/pha_events/imap_hit_range-3B-cosine-lookup_20250203_v000.csv"), get_test_data_path( - "hit/pha_events/imap_hit_range-4B-cosine-lookup_20250203_v000.csv") + lookup_table = CosineCorrectionLookupTable( + get_test_data_path( + "hit/pha_events/imap_hit_range-2A-cosine-lookup_20250203_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-3A-cosine-lookup_20250203_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-4A-cosine-lookup_20250203_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-2B-cosine-lookup_20250203_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-3B-cosine-lookup_20250203_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-4B-cosine-lookup_20250203_v000.csv" + ), ) self.assertEqual(150, len(lookup_table._range2A_corrections)) @@ -24,23 +41,67 @@ def test_can_create_from_file(self): self.assertEqual(150, len(lookup_table._range3B_corrections)) self.assertEqual(150, len(lookup_table._range4B_corrections)) - L1A0b_detector = Detector(layer=1, side="A", segment="0b", address=1234, group="L1A") - L2A0_detector = Detector(layer=2, side="A", segment="0", address=5678, group="L2A") + L1A0b_detector = Detector( + layer=1, side="A", segment="0b", address=1234, group="L1A" + ) + L2A0_detector = Detector( + layer=2, side="A", segment="0", address=5678, group="L2A" + ) - L1B0b_detector = Detector(layer=1, side="B", segment="0b", address=1234, group="L1A") - L2B0_detector = Detector(layer=2, side="B", segment="0", address=5678, group="L2A") + L1B0b_detector = Detector( + layer=1, side="B", segment="0b", address=1234, group="L1A" + ) + L2B0_detector = Detector( + layer=2, side="B", segment="0", address=5678, group="L2A" + ) - self.assertEqual(0.978643, lookup_table.get_cosine_correction(DetectedRange(DetectorRange.R2, DetectorSide.A), - L1A0b_detector, L2A0_detector)) - self.assertEqual(0.978643, lookup_table.get_cosine_correction(DetectedRange(DetectorRange.R2, DetectorSide.B), - L1B0b_detector, L2B0_detector)) + self.assertEqual( + 0.978643, + lookup_table.get_cosine_correction( + DetectedRange(DetectorRange.R2, DetectorSide.A), + L1A0b_detector, + L2A0_detector, + ), + ) + self.assertEqual( + 0.978643, + lookup_table.get_cosine_correction( + DetectedRange(DetectorRange.R2, DetectorSide.B), + L1B0b_detector, + L2B0_detector, + ), + ) - self.assertEqual(0.674600, lookup_table.get_cosine_correction(DetectedRange(DetectorRange.R3, DetectorSide.A), - L1A0b_detector, L2A0_detector)) - self.assertEqual(0.674600, lookup_table.get_cosine_correction(DetectedRange(DetectorRange.R3, DetectorSide.B), - L1B0b_detector, L2B0_detector)) + self.assertEqual( + 0.674600, + lookup_table.get_cosine_correction( + DetectedRange(DetectorRange.R3, DetectorSide.A), + L1A0b_detector, + L2A0_detector, + ), + ) + self.assertEqual( + 0.674600, + lookup_table.get_cosine_correction( + DetectedRange(DetectorRange.R3, DetectorSide.B), + L1B0b_detector, + L2B0_detector, + ), + ) - self.assertEqual(0.705122, lookup_table.get_cosine_correction(DetectedRange(DetectorRange.R4, DetectorSide.A), - L1A0b_detector, L2A0_detector)) - self.assertEqual(0.705122, lookup_table.get_cosine_correction(DetectedRange(DetectorRange.R4, DetectorSide.B), - L1B0b_detector, L2B0_detector)) + self.assertEqual( + 0.705122, + lookup_table.get_cosine_correction( + DetectedRange(DetectorRange.R4, DetectorSide.A), + L1A0b_detector, + L2A0_detector, + ), + ) + self.assertEqual( + 0.705122, + lookup_table.get_cosine_correction( + DetectedRange(DetectorRange.R4, DetectorSide.B), + L1B0b_detector, + L2B0_detector, + ), + ) diff --git a/tests/hit/l3/pha/science/test_gain_lookup_table.py b/tests/hit/l3/pha/science/test_gain_lookup_table.py index 5ac4eb3eb..bc5ca2898 100644 --- a/tests/hit/l3/pha/science/test_gain_lookup_table.py +++ b/tests/hit/l3/pha/science/test_gain_lookup_table.py @@ -1,6 +1,9 @@ import unittest -from imap_l3_processing.hit.l3.pha.science.gain_lookup_table import GainLookupTable, DetectorGain +from imap_l3_processing.hit.l3.pha.science.gain_lookup_table import ( + GainLookupTable, + DetectorGain, +) from tests.test_helpers import get_test_data_path @@ -8,9 +11,12 @@ class TestGainLookupTable(unittest.TestCase): def test_loads_from_file(self): lookup = GainLookupTable.from_file( high_gain_file_path=get_test_data_path( - 'hit/pha_events/imap_hit_hi-gain-lookup_20250203_v000.csv'), + "hit/pha_events/imap_hit_hi-gain-lookup_20250203_v000.csv" + ), low_gain_file_path=get_test_data_path( - 'hit/pha_events/imap_hit_lo-gain-lookup_20250203_v000.csv')) + "hit/pha_events/imap_hit_lo-gain-lookup_20250203_v000.csv" + ), + ) self.assertEqual(2, len(lookup)) diff --git a/tests/hit/l3/pha/science/test_hit_event_type_lookup.py b/tests/hit/l3/pha/science/test_hit_event_type_lookup.py index 893621bf6..abade9518 100644 --- a/tests/hit/l3/pha/science/test_hit_event_type_lookup.py +++ b/tests/hit/l3/pha/science/test_hit_event_type_lookup.py @@ -1,8 +1,15 @@ import os import unittest -from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import DetectedRange, DetectorRange, DetectorSide -from imap_l3_processing.hit.l3.pha.science.hit_event_type_lookup import HitEventTypeLookup, Rule +from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import ( + DetectedRange, + DetectorRange, + DetectorSide, +) +from imap_l3_processing.hit.l3.pha.science.hit_event_type_lookup import ( + HitEventTypeLookup, + Rule, +) class TestHitEventTypeLookup(unittest.TestCase): @@ -15,12 +22,18 @@ def tearDown(self): def test_hit_event_type_lookup_from_csv(self): with open(self.fake_lookup_path, "w") as csvfile: - csvfile.write("\n".join(["L1A4,L1B14,L2A,L3A,L2B,L40B,L4B14,Range", - "1,0,1,0,0,0,0,2A", - "1,0,1,1,0,*,0,3A", - "1,0,1,1,1,*,0,3A", - "0,1,0,*,1,*,1,2B", - "*,0,0,1,1,*,0,NOCALC"])) + csvfile.write( + "\n".join( + [ + "L1A4,L1B14,L2A,L3A,L2B,L40B,L4B14,Range", + "1,0,1,0,0,0,0,2A", + "1,0,1,1,0,*,0,3A", + "1,0,1,1,1,*,0,3A", + "0,1,0,*,1,*,1,2B", + "*,0,0,1,1,*,0,NOCALC", + ] + ) + ) lookup = HitEventTypeLookup.from_csv(self.fake_lookup_path) self.assertEqual(4, len(lookup._rules)) @@ -30,12 +43,25 @@ def test_hit_event_type_lookup_from_csv(self): Range2B = DetectedRange(DetectorRange.R2, DetectorSide.B) test_cases = [ - (Rule(Range2A, ["L1A4", "L2A"], ["L1B14", "L3A", "L2B", "L40B", "L4B14"]), {"L1A4", "L2A"}), - (Rule(Range3A, ["L1A4", "L2A", "L3A"], ["L1B14", "L2B", "L4B14"]), {"L1A4", "L2A", "L3A"}), - (Rule(Range3A, ["L1A4", "L2A", "L3A", "L2B"], ["L1B14", "L4B14"]), - {"L1A4", "L2A", "L3A", "L2B", "L40B"}), - (Rule(Range2B, ["L1B14", "L2B", "L4B14"], ["L1A4", "L2A"]), {"L1B14", "L2B", "L4B14"}), - (None, {"L3A", "L2B", "L40B"}) + ( + Rule( + Range2A, ["L1A4", "L2A"], ["L1B14", "L3A", "L2B", "L40B", "L4B14"] + ), + {"L1A4", "L2A"}, + ), + ( + Rule(Range3A, ["L1A4", "L2A", "L3A"], ["L1B14", "L2B", "L4B14"]), + {"L1A4", "L2A", "L3A"}, + ), + ( + Rule(Range3A, ["L1A4", "L2A", "L3A", "L2B"], ["L1B14", "L4B14"]), + {"L1A4", "L2A", "L3A", "L2B", "L40B"}, + ), + ( + Rule(Range2B, ["L1B14", "L2B", "L4B14"], ["L1A4", "L2A"]), + {"L1B14", "L2B", "L4B14"}, + ), + (None, {"L3A", "L2B", "L40B"}), ] for expected, detector_groups in test_cases: with self.subTest(detector_groups): diff --git a/tests/hit/l3/pha/science/test_range_fit_lookup.py b/tests/hit/l3/pha/science/test_range_fit_lookup.py index c81dd4043..35d2d7866 100644 --- a/tests/hit/l3/pha/science/test_range_fit_lookup.py +++ b/tests/hit/l3/pha/science/test_range_fit_lookup.py @@ -5,24 +5,32 @@ import numpy as np -from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import DetectedRange, DetectorSide, \ - DetectorRange -from imap_l3_processing.hit.l3.pha.science.range_fit_lookup import double_power_law, RangeFitLookup +from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import ( + DetectedRange, + DetectorSide, + DetectorRange, +) +from imap_l3_processing.hit.l3.pha.science.range_fit_lookup import ( + double_power_law, + RangeFitLookup, +) from tests.test_helpers import get_test_data_path class TestRangeFitLookup(unittest.TestCase): - def test_double_power_law(self): - a1 = 1.182819e+01 + a1 = 1.182819e01 b1 = 1.865991e-02 - a2 = 6.508012e+01 + a2 = 6.508012e01 b2 = 8.420725e-01 - gamma = 1.508828e+00 + gamma = 1.508828e00 e_prime = 5 result = double_power_law(e_prime, a1, b1, a2, b2, gamma) - self.assertAlmostEqual(result, ((a1 * e_prime ** b1) ** gamma + (a2 * e_prime ** b2) ** gamma) ** (1 / gamma)) + self.assertAlmostEqual( + result, + ((a1 * e_prime**b1) ** gamma + (a2 * e_prime**b2) ** gamma) ** (1 / gamma), + ) def test_loads_csv_into_array(self): test_cases = [ @@ -47,13 +55,20 @@ def test_loads_csv_into_array(self): charge1_parameters = [1, 2, 3, 4, 5] charge2_parameters = [4, 5, 6, 7, 8] - for path in [range2A_path, range3A_path, range4A_path, range2B_path, range3B_path, range4B_path]: - with open(path, 'w') as f: + for path in [ + range2A_path, + range3A_path, + range4A_path, + range2B_path, + range3B_path, + range4B_path, + ]: + with open(path, "w") as f: csv_writer = csv.writer(f) csv_writer.writerow([0, 0, 0, 0, 0, 0]) path_to_populate = os.path.join(tempdir, file_to_populate) - with open(path_to_populate, 'w') as range_file: + with open(path_to_populate, "w") as range_file: range_file.write("""# # Format: ion_charge, a1, b1, a2, b2, gamma # @@ -68,25 +83,44 @@ def test_loads_csv_into_array(self): range4A_file=range4A_path, range2B_file=range2B_path, range3B_file=range3B_path, - range4B_file=range4B_path + range4B_file=range4B_path, ) e_prime = 11.2 - charges, delta_e_loss = lookup_table.evaluate_e_prime(range, e_prime) + charges, delta_e_loss = lookup_table.evaluate_e_prime( + range, e_prime + ) np.testing.assert_equal(charges, expected_charges) - np.testing.assert_allclose([double_power_law(e_prime, *charge1_parameters), - double_power_law(e_prime, *charge2_parameters)], delta_e_loss) + np.testing.assert_allclose( + [ + double_power_law(e_prime, *charge1_parameters), + double_power_law(e_prime, *charge2_parameters), + ], + delta_e_loss, + ) def test_load_from_real_example_files(self): range_fit_lookup = RangeFitLookup.from_files( - get_test_data_path("hit/pha_events/imap_hit_range-2A-charge-fit-lookup_20250319_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-3A-charge-fit-lookup_20250319_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-4A-charge-fit-lookup_20250319_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-2B-charge-fit-lookup_20250319_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-3B-charge-fit-lookup_20250319_v000.csv"), - get_test_data_path("hit/pha_events/imap_hit_range-4B-charge-fit-lookup_20250319_v000.csv"), + get_test_data_path( + "hit/pha_events/imap_hit_range-2A-charge-fit-lookup_20250319_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-3A-charge-fit-lookup_20250319_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-4A-charge-fit-lookup_20250319_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-2B-charge-fit-lookup_20250319_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-3B-charge-fit-lookup_20250319_v000.csv" + ), + get_test_data_path( + "hit/pha_events/imap_hit_range-4B-charge-fit-lookup_20250319_v000.csv" + ), ) self.assertIsInstance(range_fit_lookup, RangeFitLookup) diff --git a/tests/hit/l3/pha/test_l3_pha_dependencies.py b/tests/hit/l3/pha/test_l3_pha_dependencies.py index 27655f684..6e3a02c34 100644 --- a/tests/hit/l3/pha/test_l3_pha_dependencies.py +++ b/tests/hit/l3/pha/test_l3_pha_dependencies.py @@ -2,30 +2,59 @@ from unittest.mock import patch, call, sentinel import imap_data_access -from imap_data_access.processing_input import ProcessingInputCollection, ScienceInput, AncillaryInput - -from imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies import HitL3PhaDependencies, HIT_L1A_EVENT_DESCRIPTOR, \ - HIT_L3_LO_GAIN_LOOKUP_DESCRIPTOR, HIT_L3_HI_GAIN_LOOKUP_DESCRIPTOR, \ - HIT_L3_EVENT_TYPE_LOOKUP_DESCRIPTOR, \ - HIT_L3_RANGE_2A_COSINE_LOOKUP_DESCRIPTOR, HIT_L3_RANGE_3A_COSINE_LOOKUP_DESCRIPTOR, \ - HIT_L3_RANGE_4A_COSINE_LOOKUP_DESCRIPTOR, HIT_L3_RANGE_2B_COSINE_LOOKUP_DESCRIPTOR, \ - HIT_L3_RANGE_3B_COSINE_LOOKUP_DESCRIPTOR, HIT_L3_RANGE_4B_COSINE_LOOKUP_DESCRIPTOR, \ - HIT_L3_RANGE_2A_CHARGE_FIT_LOOKUP_DESCRIPTOR, HIT_L3_RANGE_3A_CHARGE_FIT_LOOKUP_DESCRIPTOR, \ - HIT_L3_RANGE_4A_CHARGE_FIT_LOOKUP_DESCRIPTOR, HIT_L3_RANGE_2B_CHARGE_FIT_LOOKUP_DESCRIPTOR, \ - HIT_L3_RANGE_3B_CHARGE_FIT_LOOKUP_DESCRIPTOR, HIT_L3_RANGE_4B_CHARGE_FIT_LOOKUP_DESCRIPTOR +from imap_data_access.processing_input import ( + ProcessingInputCollection, + ScienceInput, + AncillaryInput, +) + +from imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies import ( + HitL3PhaDependencies, + HIT_L1A_EVENT_DESCRIPTOR, + HIT_L3_LO_GAIN_LOOKUP_DESCRIPTOR, + HIT_L3_HI_GAIN_LOOKUP_DESCRIPTOR, + HIT_L3_EVENT_TYPE_LOOKUP_DESCRIPTOR, + HIT_L3_RANGE_2A_COSINE_LOOKUP_DESCRIPTOR, + HIT_L3_RANGE_3A_COSINE_LOOKUP_DESCRIPTOR, + HIT_L3_RANGE_4A_COSINE_LOOKUP_DESCRIPTOR, + HIT_L3_RANGE_2B_COSINE_LOOKUP_DESCRIPTOR, + HIT_L3_RANGE_3B_COSINE_LOOKUP_DESCRIPTOR, + HIT_L3_RANGE_4B_COSINE_LOOKUP_DESCRIPTOR, + HIT_L3_RANGE_2A_CHARGE_FIT_LOOKUP_DESCRIPTOR, + HIT_L3_RANGE_3A_CHARGE_FIT_LOOKUP_DESCRIPTOR, + HIT_L3_RANGE_4A_CHARGE_FIT_LOOKUP_DESCRIPTOR, + HIT_L3_RANGE_2B_CHARGE_FIT_LOOKUP_DESCRIPTOR, + HIT_L3_RANGE_3B_CHARGE_FIT_LOOKUP_DESCRIPTOR, + HIT_L3_RANGE_4B_CHARGE_FIT_LOOKUP_DESCRIPTOR, +) class TestHitL3PhaDependencies(unittest.TestCase): - @patch("imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies.download") - @patch('imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies.HitL1Data.read_from_cdf') - @patch('imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies.CosineCorrectionLookupTable') - @patch('imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies.GainLookupTable.from_file') - @patch('imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies.RangeFitLookup.from_files') - @patch('imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies.HitEventTypeLookup.from_csv') - def test_fetch_dependencies(self, mock_hit_event_type_from_lookup, mock_range_fit_from_lookup, - mock_gain_lookup_from_file, - mock_cosine_correction_lookup_from_file, mock_read_from_cdf, mock_download): + @patch( + "imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies.HitL1Data.read_from_cdf" + ) + @patch( + "imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies.CosineCorrectionLookupTable" + ) + @patch( + "imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies.GainLookupTable.from_file" + ) + @patch( + "imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies.RangeFitLookup.from_files" + ) + @patch( + "imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies.HitEventTypeLookup.from_csv" + ) + def test_fetch_dependencies( + self, + mock_hit_event_type_from_lookup, + mock_range_fit_from_lookup, + mock_gain_lookup_from_file, + mock_cosine_correction_lookup_from_file, + mock_read_from_cdf, + mock_download, + ): mock_download.side_effect = [ sentinel.l1_data_cdf_path, sentinel.range_2A_cosine_lookup_cdf_path, @@ -47,76 +76,109 @@ def test_fetch_dependencies(self, mock_hit_event_type_from_lookup, mock_range_fi input_collection = ProcessingInputCollection() - start_date = '20100105' - mission = 'imap' - instrument = 'hit' - data_level = 'l2' - version = 'v001' + start_date = "20100105" + mission = "imap" + instrument = "hit" + data_level = "l2" + version = "v001" science_l1a_event_path = f"{mission}_{instrument}_{data_level}_{HIT_L1A_EVENT_DESCRIPTOR}_{start_date}_{version}.cdf" - file_paths = [f"{mission}_{instrument}_{HIT_L3_RANGE_2A_COSINE_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", - f"{mission}_{instrument}_{HIT_L3_RANGE_3A_COSINE_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", - f"{mission}_{instrument}_{HIT_L3_RANGE_4A_COSINE_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", - f"{mission}_{instrument}_{HIT_L3_RANGE_2B_COSINE_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", - f"{mission}_{instrument}_{HIT_L3_RANGE_3B_COSINE_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", - f"{mission}_{instrument}_{HIT_L3_RANGE_4B_COSINE_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", - f"{mission}_{instrument}_{HIT_L3_LO_GAIN_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", - f"{mission}_{instrument}_{HIT_L3_HI_GAIN_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", - f"{mission}_{instrument}_{HIT_L3_RANGE_2A_CHARGE_FIT_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", - f"{mission}_{instrument}_{HIT_L3_RANGE_3A_CHARGE_FIT_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", - f"{mission}_{instrument}_{HIT_L3_RANGE_4A_CHARGE_FIT_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", - f"{mission}_{instrument}_{HIT_L3_RANGE_2B_CHARGE_FIT_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", - f"{mission}_{instrument}_{HIT_L3_RANGE_3B_CHARGE_FIT_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", - f"{mission}_{instrument}_{HIT_L3_RANGE_4B_CHARGE_FIT_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", - f"{mission}_{instrument}_{HIT_L3_EVENT_TYPE_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", - ] + file_paths = [ + f"{mission}_{instrument}_{HIT_L3_RANGE_2A_COSINE_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", + f"{mission}_{instrument}_{HIT_L3_RANGE_3A_COSINE_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", + f"{mission}_{instrument}_{HIT_L3_RANGE_4A_COSINE_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", + f"{mission}_{instrument}_{HIT_L3_RANGE_2B_COSINE_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", + f"{mission}_{instrument}_{HIT_L3_RANGE_3B_COSINE_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", + f"{mission}_{instrument}_{HIT_L3_RANGE_4B_COSINE_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", + f"{mission}_{instrument}_{HIT_L3_LO_GAIN_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", + f"{mission}_{instrument}_{HIT_L3_HI_GAIN_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", + f"{mission}_{instrument}_{HIT_L3_RANGE_2A_CHARGE_FIT_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", + f"{mission}_{instrument}_{HIT_L3_RANGE_3A_CHARGE_FIT_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", + f"{mission}_{instrument}_{HIT_L3_RANGE_4A_CHARGE_FIT_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", + f"{mission}_{instrument}_{HIT_L3_RANGE_2B_CHARGE_FIT_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", + f"{mission}_{instrument}_{HIT_L3_RANGE_3B_CHARGE_FIT_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", + f"{mission}_{instrument}_{HIT_L3_RANGE_4B_CHARGE_FIT_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", + f"{mission}_{instrument}_{HIT_L3_EVENT_TYPE_LOOKUP_DESCRIPTOR}_{start_date}_{version}.csv", + ] ancillary_input_file_paths = [AncillaryInput(path) for path in file_paths] science_input = ScienceInput(science_l1a_event_path) input_collection.add([science_input, *ancillary_input_file_paths]) - hit_l3_pha_dependencies = HitL3PhaDependencies.fetch_dependencies(input_collection) + hit_l3_pha_dependencies = HitL3PhaDependencies.fetch_dependencies( + input_collection + ) - science_dir = imap_data_access.config["DATA_DIR"] / 'imap' / 'hit' / 'l2' / '2010' / '01' - ancillary_dir = imap_data_access.config["DATA_DIR"] / 'imap' / 'ancillary' / 'hit' + science_dir = ( + imap_data_access.config["DATA_DIR"] / "imap" / "hit" / "l2" / "2010" / "01" + ) + ancillary_dir = ( + imap_data_access.config["DATA_DIR"] / "imap" / "ancillary" / "hit" + ) expected_file_paths = [ancillary_dir / path for path in file_paths] - mock_download.assert_has_calls([ - call(science_dir / science_l1a_event_path), - *[call(file_path) for file_path in expected_file_paths], - ]) + mock_download.assert_has_calls( + [ + call(science_dir / science_l1a_event_path), + *[call(file_path) for file_path in expected_file_paths], + ] + ) mock_read_from_cdf.assert_called_with(sentinel.l1_data_cdf_path) - mock_cosine_correction_lookup_from_file.assert_called_once_with(sentinel.range_2A_cosine_lookup_cdf_path, - sentinel.range_3A_cosine_lookup_cdf_path, - sentinel.range_4A_cosine_lookup_cdf_path, - sentinel.range_2B_cosine_lookup_cdf_path, - sentinel.range_3B_cosine_lookup_cdf_path, - sentinel.range_4B_cosine_lookup_cdf_path) - mock_gain_lookup_from_file.assert_called_once_with(sentinel.hi_gain_lookup_cdf_path, - sentinel.lo_gain_lookup_cdf_path, ) - mock_range_fit_from_lookup.assert_called_once_with(sentinel.range_2A_charge_fit_lookup_cdf_path, - sentinel.range_3A_charge_fit_lookup_cdf_path, - sentinel.range_4A_charge_fit_lookup_cdf_path, - sentinel.range_2B_charge_fit_lookup_cdf_path, - sentinel.range_3B_charge_fit_lookup_cdf_path, - sentinel.range_4B_charge_fit_lookup_cdf_path, - ) - - mock_hit_event_type_from_lookup.assert_called_once_with(sentinel.event_type_lookup_path) - - self.assertEqual(mock_read_from_cdf.return_value, hit_l3_pha_dependencies.hit_l1_data) - self.assertEqual(mock_cosine_correction_lookup_from_file.return_value, - hit_l3_pha_dependencies.cosine_correction_lookup) - self.assertEqual(mock_gain_lookup_from_file.return_value, hit_l3_pha_dependencies.gain_lookup) - self.assertEqual(mock_range_fit_from_lookup.return_value, hit_l3_pha_dependencies.range_fit_lookup) - self.assertEqual(mock_hit_event_type_from_lookup.return_value, hit_l3_pha_dependencies.event_type_lookup) - - @patch('imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies.download') - @patch('imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies.HitL1Data.read_from_cdf') - def test_fetch_dependencies_throws_when_no_data_dependency_provided(self, mock_read_from_cdf, - mock_download_dependency): + mock_cosine_correction_lookup_from_file.assert_called_once_with( + sentinel.range_2A_cosine_lookup_cdf_path, + sentinel.range_3A_cosine_lookup_cdf_path, + sentinel.range_4A_cosine_lookup_cdf_path, + sentinel.range_2B_cosine_lookup_cdf_path, + sentinel.range_3B_cosine_lookup_cdf_path, + sentinel.range_4B_cosine_lookup_cdf_path, + ) + mock_gain_lookup_from_file.assert_called_once_with( + sentinel.hi_gain_lookup_cdf_path, + sentinel.lo_gain_lookup_cdf_path, + ) + mock_range_fit_from_lookup.assert_called_once_with( + sentinel.range_2A_charge_fit_lookup_cdf_path, + sentinel.range_3A_charge_fit_lookup_cdf_path, + sentinel.range_4A_charge_fit_lookup_cdf_path, + sentinel.range_2B_charge_fit_lookup_cdf_path, + sentinel.range_3B_charge_fit_lookup_cdf_path, + sentinel.range_4B_charge_fit_lookup_cdf_path, + ) + + mock_hit_event_type_from_lookup.assert_called_once_with( + sentinel.event_type_lookup_path + ) + + self.assertEqual( + mock_read_from_cdf.return_value, hit_l3_pha_dependencies.hit_l1_data + ) + self.assertEqual( + mock_cosine_correction_lookup_from_file.return_value, + hit_l3_pha_dependencies.cosine_correction_lookup, + ) + self.assertEqual( + mock_gain_lookup_from_file.return_value, hit_l3_pha_dependencies.gain_lookup + ) + self.assertEqual( + mock_range_fit_from_lookup.return_value, + hit_l3_pha_dependencies.range_fit_lookup, + ) + self.assertEqual( + mock_hit_event_type_from_lookup.return_value, + hit_l3_pha_dependencies.event_type_lookup, + ) + + @patch("imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies.download") + @patch( + "imap_l3_processing.hit.l3.pha.hit_l3_pha_dependencies.HitL1Data.read_from_cdf" + ) + def test_fetch_dependencies_throws_when_no_data_dependency_provided( + self, mock_read_from_cdf, mock_download_dependency + ): with self.assertRaises(ValueError) as e: _ = HitL3PhaDependencies.fetch_dependencies(ProcessingInputCollection()) - self.assertEqual(str(e.exception), f"Missing {HIT_L1A_EVENT_DESCRIPTOR} dependency.") + self.assertEqual( + str(e.exception), f"Missing {HIT_L1A_EVENT_DESCRIPTOR} dependency." + ) diff --git a/tests/hit/l3/pha/test_pha_event_reader.py b/tests/hit/l3/pha/test_pha_event_reader.py index 370bfbdd6..491b774ae 100644 --- a/tests/hit/l3/pha/test_pha_event_reader.py +++ b/tests/hit/l3/pha/test_pha_event_reader.py @@ -4,22 +4,48 @@ from bitstring import BitArray, BitStream from spacepy.pycdf import CDF -from imap_l3_processing.hit.l3.pha.pha_event_reader import PHAEventReader, RawPHAEvent, PHAWord, Detector +from imap_l3_processing.hit.l3.pha.pha_event_reader import ( + PHAEventReader, + RawPHAEvent, + PHAWord, + Detector, +) from tests.hit.l3.hit_test_builders import create_raw_pha_event from tests.test_helpers import get_test_data_path class TestPHAEventReader(unittest.TestCase): - def test_can_create_detector_from_address(self): address_to_expected_detector = [ - (35, "L1B1a", Detector(layer=1, side="B", segment="1a", address=35, group="L1B14")), - (6, "L2A3", Detector(layer=2, side="A", segment="3", address=6, group="L2A")), - (31, "L4Ao", Detector(layer=4, side="A", segment="o", address=31, group="L4oA")), - (10, "L4AUnknown_10", Detector(layer=4, side="A", segment="UNKNOWN_10", address=10, group="Unknown")), + ( + 35, + "L1B1a", + Detector(layer=1, side="B", segment="1a", address=35, group="L1B14"), + ), + ( + 6, + "L2A3", + Detector(layer=2, side="A", segment="3", address=6, group="L2A"), + ), + ( + 31, + "L4Ao", + Detector(layer=4, side="A", segment="o", address=31, group="L4oA"), + ), + ( + 10, + "L4AUnknown_10", + Detector( + layer=4, side="A", segment="UNKNOWN_10", address=10, group="Unknown" + ), + ), ] - for address, expected_detector_name, expected_detector in address_to_expected_detector: + for ( + address, + expected_detector_name, + expected_detector, + ) in address_to_expected_detector: with self.subTest(f"{address} should become {expected_detector_name}"): detector = Detector.from_address(address) self.assertEqual(expected_detector, detector) @@ -68,18 +94,38 @@ def test_read_l1_pha_data(self): pha2_is_last_event = True pha2_is_last_pha_event = "1" - event_record_header_parts = [particle_id_binary, priority_number_binary, stim_tag_binary, haz_tag_binary, - time_tag_binary, a_b_side_binary, has_unread_adcs_binary, long_event_flag_binary, - culling_flag_binary, spare_binary] - pha1_data_parts = [pha1_adc_value_binary, pha1_adc_overflow_binary, pha1_detector_address_binary, - pha1_is_low_gain_binary, - pha1_is_last_pha_event] - pha2_data_parts = [pha2_adc_value_binary, pha2_adc_overflow_binary, pha2_detector_address_binary, - pha2_is_low_gain_binary, - pha2_is_last_pha_event] + event_record_header_parts = [ + particle_id_binary, + priority_number_binary, + stim_tag_binary, + haz_tag_binary, + time_tag_binary, + a_b_side_binary, + has_unread_adcs_binary, + long_event_flag_binary, + culling_flag_binary, + spare_binary, + ] + pha1_data_parts = [ + pha1_adc_value_binary, + pha1_adc_overflow_binary, + pha1_detector_address_binary, + pha1_is_low_gain_binary, + pha1_is_last_pha_event, + ] + pha2_data_parts = [ + pha2_adc_value_binary, + pha2_adc_overflow_binary, + pha2_detector_address_binary, + pha2_is_low_gain_binary, + pha2_is_last_pha_event, + ] - binary = "".join(event_record_header_parts[::-1]) + "".join(pha1_data_parts[::-1]) + "".join( - pha2_data_parts[::-1]) + binary = ( + "".join(event_record_header_parts[::-1]) + + "".join(pha1_data_parts[::-1]) + + "".join(pha2_data_parts[::-1]) + ) bitstream = BitStream("0b" + binary) event_data = PHAEventReader.read_pha_event(bitstream) @@ -99,13 +145,17 @@ def test_read_l1_pha_data(self): self.assertEqual(2, len(event_data.pha_words)) self.assertEqual(pha1_adc_value, event_data.pha_words[0].adc_value) self.assertEqual(pha1_adc_overflow, event_data.pha_words[0].adc_overflow) - self.assertEqual(pha1_adc_detector_address, event_data.pha_words[0].detector.address) + self.assertEqual( + pha1_adc_detector_address, event_data.pha_words[0].detector.address + ) self.assertEqual(pha1_is_low_gain, event_data.pha_words[0].is_low_gain) self.assertEqual(pha1_is_last_event, event_data.pha_words[0].is_last_pha) self.assertEqual(pha2_adc_value, event_data.pha_words[1].adc_value) self.assertEqual(pha2_adc_overflow, event_data.pha_words[1].adc_overflow) - self.assertEqual(pha2_adc_detector_address, event_data.pha_words[1].detector.address) + self.assertEqual( + pha2_adc_detector_address, event_data.pha_words[1].detector.address + ) self.assertEqual(pha2_is_low_gain, event_data.pha_words[1].is_low_gain) self.assertEqual(pha2_is_last_event, event_data.pha_words[1].is_last_pha) @@ -115,21 +165,31 @@ def test_reads_extended_header_section_when_long_event_flag_set(self): expected_delta_e_index = 108 expected_e_prime_index = 105 - extended_header_binary = "".join([ - BitArray(uint=expected_delta_e_index, length=9).bin, - BitArray(uint=expected_e_prime_index, length=7).bin, - BitArray(uint=expected_detector_flags, length=8).bin, - ]) + extended_header_binary = "".join( + [ + BitArray(uint=expected_delta_e_index, length=9).bin, + BitArray(uint=expected_e_prime_index, length=7).bin, + BitArray(uint=expected_detector_flags, length=8).bin, + ] + ) pha_word1 = self._create_pha_data(is_last_event=False) pha_word2 = self._create_pha_data(is_last_event=True) - bitstream = BitStream("0b" + header_binary + extended_header_binary + pha_word1 + pha_word2) + bitstream = BitStream( + "0b" + header_binary + extended_header_binary + pha_word1 + pha_word2 + ) event_data = PHAEventReader.read_pha_event(bitstream) self.assertIsNone(event_data.extended_stim_header) - self.assertEqual(expected_detector_flags, event_data.extended_header.detector_flags) - self.assertEqual(expected_delta_e_index, event_data.extended_header.delta_e_index) - self.assertEqual(expected_e_prime_index, event_data.extended_header.e_prime_index) + self.assertEqual( + expected_detector_flags, event_data.extended_header.detector_flags + ) + self.assertEqual( + expected_delta_e_index, event_data.extended_header.delta_e_index + ) + self.assertEqual( + expected_e_prime_index, event_data.extended_header.e_prime_index + ) def test_reads_extended_stim_header_section_when_long_event_and_stim_flag_set(self): header_binary = self._create_pha_header(stim_tag=True, long_event_flag=True) @@ -139,22 +199,32 @@ def test_reads_extended_stim_header_section_when_long_event_and_stim_flag_set(se expected_stim_step = 12 expected_stim_gain = False expected_a_l_stim = 0 - stim_block_binary = "".join([ - BitArray(uint=expected_a_l_stim, length=1).bin, - BitArray(uint=0, length=2).bin, - BitArray(uint=expected_stim_gain, length=1).bin, - BitArray(uint=expected_stim_step, length=4).bin, - ]) - - extended_stim_header_binary = "".join([ - BitArray(uint=expected_tbd, length=12).bin, - BitArray(uint=expected_dac_value, length=12).bin, - ]) + stim_block_binary = "".join( + [ + BitArray(uint=expected_a_l_stim, length=1).bin, + BitArray(uint=0, length=2).bin, + BitArray(uint=expected_stim_gain, length=1).bin, + BitArray(uint=expected_stim_step, length=4).bin, + ] + ) + + extended_stim_header_binary = "".join( + [ + BitArray(uint=expected_tbd, length=12).bin, + BitArray(uint=expected_dac_value, length=12).bin, + ] + ) pha_word1 = self._create_pha_data(is_last_event=False) pha_word2 = self._create_pha_data(is_last_event=True) bitstream = BitStream( - "0b" + header_binary + extended_stim_header_binary + stim_block_binary + pha_word1 + pha_word2) + "0b" + + header_binary + + extended_stim_header_binary + + stim_block_binary + + pha_word1 + + pha_word2 + ) event_data = PHAEventReader.read_pha_event(bitstream) self.assertIsNone(event_data.extended_header) @@ -171,16 +241,20 @@ def test_reads_stim_block_when_stim_flag_set(self): expected_stim_step = 11 expected_stim_gain = True expected_a_l_stim = True - stim_block_binary = "".join([ - BitArray(uint=expected_a_l_stim, length=1).bin, - BitArray(uint=0, length=2).bin, - BitArray(uint=expected_stim_gain, length=1).bin, - BitArray(uint=expected_stim_step, length=4).bin, - ]) + stim_block_binary = "".join( + [ + BitArray(uint=expected_a_l_stim, length=1).bin, + BitArray(uint=0, length=2).bin, + BitArray(uint=expected_stim_gain, length=1).bin, + BitArray(uint=expected_stim_step, length=4).bin, + ] + ) pha_word1 = self._create_pha_data(is_last_event=False) pha_word2 = self._create_pha_data(is_last_event=True) - bitstream = BitStream("0b" + header_binary + stim_block_binary + pha_word1 + pha_word2) + bitstream = BitStream( + "0b" + header_binary + stim_block_binary + pha_word1 + pha_word2 + ) event_data = PHAEventReader.read_pha_event(bitstream) self.assertIsNone(event_data.extended_header) @@ -209,9 +283,17 @@ def test_read_all_pha_events_stops_reading_events_when_it_finds_all_zeros(self): self.assertEqual(event3, events[2]) def test_read_events_adds_extra_bits_for_odd_length_pha_words(self): - event1 = create_raw_pha_event(pha_words=[ - PHAWord(detector=Detector.from_address(14), adc_value=1000, adc_overflow=True, is_low_gain=True, - is_last_pha=True)]) + event1 = create_raw_pha_event( + pha_words=[ + PHAWord( + detector=Detector.from_address(14), + adc_value=1000, + adc_overflow=True, + is_low_gain=True, + is_last_pha=True, + ) + ] + ) event2 = create_raw_pha_event() @@ -225,7 +307,13 @@ def test_read_events_adds_extra_bits_for_odd_length_pha_words(self): self.assertEqual(event2, events[1]) def test_reads_pha_events_from_cdf(self): - cdf = CDF(str(get_test_data_path("hit/pha_events/imap_hit_l1a_direct-events_20100105_v009.cdf"))) + cdf = CDF( + str( + get_test_data_path( + "hit/pha_events/imap_hit_l1a_direct-events_20100105_v009.cdf" + ) + ) + ) event_total = 0 for event_buffer in cdf["pha_raw"][...]: @@ -235,11 +323,15 @@ def test_reads_pha_events_from_cdf(self): self.assertEqual(6967, event_total) def test_verify_against_HIT_script_output(self): - bitstream = BitStream(filename=get_test_data_path("hit/pha_events/full_event_record_buffer.bin")) + bitstream = BitStream( + filename=get_test_data_path("hit/pha_events/full_event_record_buffer.bin") + ) events = PHAEventReader.read_all_pha_events(bitstream.bin) - with open(get_test_data_path("hit/pha_events/expected_pha_data_for_all_events.csv")) as expected_file: + with open( + get_test_data_path("hit/pha_events/expected_pha_data_for_all_events.csv") + ) as expected_file: reader = csv.reader(expected_file) for expected_row, event in zip(reader, events): row = [" " for _ in range(128)] @@ -257,57 +349,80 @@ def test_verify_against_HIT_script_output(self): def _create_pha_event_binary(self, pha_event: RawPHAEvent) -> str: binary_string = "" - binary_string += self._create_pha_header(particle_id=pha_event.particle_id, - buf_priority_num=pha_event.priority_buffer_num, - stim_tag=pha_event.stim_tag, haz_tag=pha_event.haz_tag, - time_tag=pha_event.time_tag, - a_b_side=pha_event.a_b_side_flag, - has_unread_adcs=pha_event.has_unread_adcs, - long_event_flag=pha_event.long_event_flag, - culling_flag=pha_event.culling_flag, - spare=pha_event.spare) + binary_string += self._create_pha_header( + particle_id=pha_event.particle_id, + buf_priority_num=pha_event.priority_buffer_num, + stim_tag=pha_event.stim_tag, + haz_tag=pha_event.haz_tag, + time_tag=pha_event.time_tag, + a_b_side=pha_event.a_b_side_flag, + has_unread_adcs=pha_event.has_unread_adcs, + long_event_flag=pha_event.long_event_flag, + culling_flag=pha_event.culling_flag, + spare=pha_event.spare, + ) if pha_event.extended_header is not None: - binary_string += "".join([ - BitArray(uint=pha_event.extended_header.e_prime_index, length=8).bin, - BitArray(uint=pha_event.extended_header.delta_e_index, length=8).bin, - BitArray(uint=pha_event.extended_header.detector_flags, length=8).bin, - ]) + binary_string += "".join( + [ + BitArray( + uint=pha_event.extended_header.e_prime_index, length=8 + ).bin, + BitArray( + uint=pha_event.extended_header.delta_e_index, length=8 + ).bin, + BitArray( + uint=pha_event.extended_header.detector_flags, length=8 + ).bin, + ] + ) if pha_event.stim_block is not None: - binary_string += "".join([ - BitArray(uint=pha_event.stim_block.a_l_stim, length=1).bin, - BitArray(uint=0, length=2).bin, - BitArray(uint=pha_event.stim_block.stim_gain, length=1).bin, - BitArray(uint=pha_event.stim_block.stim_step, length=4).bin, - ]) + binary_string += "".join( + [ + BitArray(uint=pha_event.stim_block.a_l_stim, length=1).bin, + BitArray(uint=0, length=2).bin, + BitArray(uint=pha_event.stim_block.stim_gain, length=1).bin, + BitArray(uint=pha_event.stim_block.stim_step, length=4).bin, + ] + ) if pha_event.extended_stim_header is not None: - binary_string += "".join([ - BitArray(uint=pha_event.extended_stim_header.tbd, length=12).bin, - BitArray(uint=pha_event.extended_stim_header.dac_value, length=12).bin, - ]) + binary_string += "".join( + [ + BitArray(uint=pha_event.extended_stim_header.tbd, length=12).bin, + BitArray( + uint=pha_event.extended_stim_header.dac_value, length=12 + ).bin, + ] + ) for word in pha_event.pha_words: - binary_string += self._create_pha_data(adc_value=word.adc_value, adc_overflow=word.adc_overflow, - adc_detector_address=word.detector.address, - is_low_gain=word.is_low_gain, is_last_event=word.is_last_pha) + binary_string += self._create_pha_data( + adc_value=word.adc_value, + adc_overflow=word.adc_overflow, + adc_detector_address=word.detector.address, + is_low_gain=word.is_low_gain, + is_last_event=word.is_last_pha, + ) if len(pha_event.pha_words) % 2 == 1: binary_string += "0000" return binary_string - def _create_pha_header(self, - particle_id: int = 0, - buf_priority_num: int = 0, - stim_tag: bool = False, - haz_tag: bool = False, - time_tag: int = 0, - a_b_side: bool = False, - has_unread_adcs: bool = False, - long_event_flag: bool = False, - culling_flag: bool = False, - spare: bool = False): + def _create_pha_header( + self, + particle_id: int = 0, + buf_priority_num: int = 0, + stim_tag: bool = False, + haz_tag: bool = False, + time_tag: int = 0, + a_b_side: bool = False, + has_unread_adcs: bool = False, + long_event_flag: bool = False, + culling_flag: bool = False, + spare: bool = False, + ): bit_arrays = [ BitArray(uint=int(spare), length=1), BitArray(uint=int(culling_flag), length=1), @@ -322,13 +437,14 @@ def _create_pha_header(self, ] return "".join([bit_array.bin for bit_array in bit_arrays]) - def _create_pha_data(self, - adc_value=0, - adc_overflow=False, - adc_detector_address=0, - is_low_gain=True, - is_last_event=True, - ): + def _create_pha_data( + self, + adc_value=0, + adc_overflow=False, + adc_detector_address=0, + is_low_gain=True, + is_last_event=True, + ): bit_arrays = [ BitArray(uint=int(is_last_event), length=1), BitArray(uint=int(is_low_gain), length=1), diff --git a/tests/hit/l3/sectored_products/science/test_sectored_products_algorithms.py b/tests/hit/l3/sectored_products/science/test_sectored_products_algorithms.py index 85dde9c03..484a78418 100644 --- a/tests/hit/l3/sectored_products/science/test_sectored_products_algorithms.py +++ b/tests/hit/l3/sectored_products/science/test_sectored_products_algorithms.py @@ -2,39 +2,42 @@ import numpy as np -from imap_l3_processing.hit.l3.sectored_products.science.sectored_products_algorithms import get_sector_unit_vectors, \ - get_hit_bin_polar_coordinates +from imap_l3_processing.hit.l3.sectored_products.science.sectored_products_algorithms import ( + get_sector_unit_vectors, + get_hit_bin_polar_coordinates, +) class TestSectoredProductsAlgorithms(TestCase): - def test_get_sector_unit_vectors(self): test_cases = [ - ([0], [[0], [270]], [[[0, 0, 1]], [[0, 0, 1]]]), ([0], [[90], [180]], [[[0, 0, 1]], [[0, 0, 1]]]), ([0], [[180], [90]], [[[0, 0, 1]], [[0, 0, 1]]]), ([0], [[270], [0]], [[[0, 0, 1]], [[0, 0, 1]]]), # ([0], [360], [[[0, 0, 1]]]), - ([90], [[0], [270]], [[[1, 0, 0]], [[0, -1, 0]]]), ([90], [[90], [180]], [[[0, 1, 0]], [[-1, 0, 0]]]), ([90], [[180], [90]], [[[-1, 0, 0]], [[0, 1, 0]]]), ([90], [[270], [0]], [[[0, -1, 0]], [[1, 0, 0]]]), - ([180], [[0], [90]], [[[0, 0, -1]], [[0, 0, -1]]]), ([180], [[90], [0]], [[[0, 0, -1]], [[0, 0, -1]]]), - ] for elevation, azimuth, expected in test_cases: with self.subTest(elevation=elevation, azimuth=azimuth, expected=expected): - np.testing.assert_array_almost_equal(get_sector_unit_vectors(elevation, azimuth), expected) + np.testing.assert_array_almost_equal( + get_sector_unit_vectors(elevation, azimuth), expected + ) def test_get_hit_bin_polar_coordinates(self): - declinations, azimuths, declination_delta, azimuth_delta = get_hit_bin_polar_coordinates() + declinations, azimuths, declination_delta, azimuth_delta = ( + get_hit_bin_polar_coordinates() + ) np.testing.assert_array_equal(declination_delta, [11.25] * 8, strict=True) np.testing.assert_array_equal(azimuth_delta, [12.0] * 15, strict=True) - np.testing.assert_array_almost_equal(declinations, 11.25 + np.arange(0, 8) * 22.5) + np.testing.assert_array_almost_equal( + declinations, 11.25 + np.arange(0, 8) * 22.5 + ) np.testing.assert_array_almost_equal(azimuths, 12 + np.arange(15) * 24) diff --git a/tests/hit/l3/sectored_products/test_models.py b/tests/hit/l3/sectored_products/test_models.py index 078b6af0d..dea37aac1 100644 --- a/tests/hit/l3/sectored_products/test_models.py +++ b/tests/hit/l3/sectored_products/test_models.py @@ -24,64 +24,65 @@ def test_to_data_product_variables(self): mock_measurement_pitch_angle = Mock() mock_measurement_pitch_angle.shape = (1, 8, 15) - data = HitPitchAngleDataProduct(sentinel.input_meta_data, - sentinel.epoch, - epoch_deltas, - pitch_angles, - sentinel.pitch_angles_deltas, - gyrophases, - sentinel.gyrophase_deltas, - sentinel.h_macropixel_intensity, - sentinel.h_macropixel_intensity_delta_plus, - sentinel.h_macropixel_intensity_delta_minus, - sentinel.h_pa_intensity, - sentinel.h_pa_intensity_delta_plus, - sentinel.h_pa_intensity_delta_minus, - h_energy, - sentinel.h_energy_delta_plus, - sentinel.h_energy_delta_minus, - sentinel.he4_macropixel_intensity, - sentinel.he4_macropixel_intensity_delta_plus, - sentinel.he4_macropixel_intensity_delta_minus, - sentinel.he4_pa_intensity, - sentinel.he4_pa_intensity_delta_plus, - sentinel.he4_pa_intensity_delta_minus, - he_energy, - sentinel.he4_energy_delta_plus, - sentinel.he4_energy_delta_minus, - sentinel.cno_macropixel_intensity, - sentinel.cno_macropixel_intensity_delta_plus, - sentinel.cno_macropixel_intensity_delta_minus, - sentinel.cno_pa_intensity, - sentinel.cno_pa_intensity_delta_plus, - sentinel.cno_pa_intensity_delta_minus, - cno_energy, - sentinel.cno_energy_delta_plus, - sentinel.cno_energy_delta_minus, - sentinel.ne_mg_si_macropixel_intensity, - sentinel.ne_mg_si_macropixel_intensity_delta_plus, - sentinel.ne_mg_si_macropixel_intensity_delta_minus, - sentinel.ne_mg_si_pa_intensity, - sentinel.ne_mg_si_pa_intensity_delta_plus, - sentinel.ne_mg_si_pa_intensity_delta_minus, - nemgsi_energy, - sentinel.ne_mg_si_energy_delta_plus, - sentinel.ne_mg_si_energy_delta_minus, - sentinel.iron_macropixel_intensity, - sentinel.iron_macropixel_intensity_delta_plus, - sentinel.iron_macropixel_intensity_delta_minus, - sentinel.iron_pa_intensity, - sentinel.iron_pa_intensity_delta_plus, - sentinel.iron_pa_intensity_delta_minus, - fe_energy, - sentinel.iron_energy_delta_plus, - sentinel.iron_energy_delta_minus, - mock_measurement_pitch_angle, - sentinel.measurement_gyrophase, - np.array([10, 180, 350]), - np.array([10.25, 90.75, 170.5]), - sentinel.quality_flags, - ) + data = HitPitchAngleDataProduct( + sentinel.input_meta_data, + sentinel.epoch, + epoch_deltas, + pitch_angles, + sentinel.pitch_angles_deltas, + gyrophases, + sentinel.gyrophase_deltas, + sentinel.h_macropixel_intensity, + sentinel.h_macropixel_intensity_delta_plus, + sentinel.h_macropixel_intensity_delta_minus, + sentinel.h_pa_intensity, + sentinel.h_pa_intensity_delta_plus, + sentinel.h_pa_intensity_delta_minus, + h_energy, + sentinel.h_energy_delta_plus, + sentinel.h_energy_delta_minus, + sentinel.he4_macropixel_intensity, + sentinel.he4_macropixel_intensity_delta_plus, + sentinel.he4_macropixel_intensity_delta_minus, + sentinel.he4_pa_intensity, + sentinel.he4_pa_intensity_delta_plus, + sentinel.he4_pa_intensity_delta_minus, + he_energy, + sentinel.he4_energy_delta_plus, + sentinel.he4_energy_delta_minus, + sentinel.cno_macropixel_intensity, + sentinel.cno_macropixel_intensity_delta_plus, + sentinel.cno_macropixel_intensity_delta_minus, + sentinel.cno_pa_intensity, + sentinel.cno_pa_intensity_delta_plus, + sentinel.cno_pa_intensity_delta_minus, + cno_energy, + sentinel.cno_energy_delta_plus, + sentinel.cno_energy_delta_minus, + sentinel.ne_mg_si_macropixel_intensity, + sentinel.ne_mg_si_macropixel_intensity_delta_plus, + sentinel.ne_mg_si_macropixel_intensity_delta_minus, + sentinel.ne_mg_si_pa_intensity, + sentinel.ne_mg_si_pa_intensity_delta_plus, + sentinel.ne_mg_si_pa_intensity_delta_minus, + nemgsi_energy, + sentinel.ne_mg_si_energy_delta_plus, + sentinel.ne_mg_si_energy_delta_minus, + sentinel.iron_macropixel_intensity, + sentinel.iron_macropixel_intensity_delta_plus, + sentinel.iron_macropixel_intensity_delta_minus, + sentinel.iron_pa_intensity, + sentinel.iron_pa_intensity_delta_plus, + sentinel.iron_pa_intensity_delta_minus, + fe_energy, + sentinel.iron_energy_delta_plus, + sentinel.iron_energy_delta_minus, + mock_measurement_pitch_angle, + sentinel.measurement_gyrophase, + np.array([10, 180, 350]), + np.array([10.25, 90.75, 170.5]), + sentinel.quality_flags, + ) data_product_variables = data.to_data_product_variables() expected_epoch_deltas = np.array([5e9]) @@ -92,78 +93,222 @@ def test_to_data_product_variables(self): DataProductVariable("pitch_angle_delta", sentinel.pitch_angles_deltas), DataProductVariable("gyrophase", NumpyArrayMatcher(gyrophases)), DataProductVariable("gyrophase_delta", sentinel.gyrophase_deltas), - DataProductVariable("h_macropixel_intensity", sentinel.h_macropixel_intensity), - DataProductVariable("h_macropixel_intensity_delta_plus", sentinel.h_macropixel_intensity_delta_plus), - DataProductVariable("h_macropixel_intensity_delta_minus", sentinel.h_macropixel_intensity_delta_minus), + DataProductVariable( + "h_macropixel_intensity", sentinel.h_macropixel_intensity + ), + DataProductVariable( + "h_macropixel_intensity_delta_plus", + sentinel.h_macropixel_intensity_delta_plus, + ), + DataProductVariable( + "h_macropixel_intensity_delta_minus", + sentinel.h_macropixel_intensity_delta_minus, + ), DataProductVariable("h_macropixel_intensity_pa", sentinel.h_pa_intensity), - DataProductVariable("h_macropixel_intensity_pa_delta_plus", sentinel.h_pa_intensity_delta_plus), - DataProductVariable("h_macropixel_intensity_pa_delta_minus", sentinel.h_pa_intensity_delta_minus), + DataProductVariable( + "h_macropixel_intensity_pa_delta_plus", + sentinel.h_pa_intensity_delta_plus, + ), + DataProductVariable( + "h_macropixel_intensity_pa_delta_minus", + sentinel.h_pa_intensity_delta_minus, + ), DataProductVariable("h_energy", NumpyArrayMatcher(h_energy)), DataProductVariable("h_energy_delta_plus", sentinel.h_energy_delta_plus), DataProductVariable("h_energy_delta_minus", sentinel.h_energy_delta_minus), - DataProductVariable("he4_macropixel_intensity", sentinel.he4_macropixel_intensity), - DataProductVariable("he4_macropixel_intensity_delta_plus", sentinel.he4_macropixel_intensity_delta_plus), - DataProductVariable("he4_macropixel_intensity_delta_minus", sentinel.he4_macropixel_intensity_delta_minus), - DataProductVariable("he4_macropixel_intensity_pa", sentinel.he4_pa_intensity), - DataProductVariable("he4_macropixel_intensity_pa_delta_plus", sentinel.he4_pa_intensity_delta_plus), - DataProductVariable("he4_macropixel_intensity_pa_delta_minus", sentinel.he4_pa_intensity_delta_minus), + DataProductVariable( + "he4_macropixel_intensity", sentinel.he4_macropixel_intensity + ), + DataProductVariable( + "he4_macropixel_intensity_delta_plus", + sentinel.he4_macropixel_intensity_delta_plus, + ), + DataProductVariable( + "he4_macropixel_intensity_delta_minus", + sentinel.he4_macropixel_intensity_delta_minus, + ), + DataProductVariable( + "he4_macropixel_intensity_pa", sentinel.he4_pa_intensity + ), + DataProductVariable( + "he4_macropixel_intensity_pa_delta_plus", + sentinel.he4_pa_intensity_delta_plus, + ), + DataProductVariable( + "he4_macropixel_intensity_pa_delta_minus", + sentinel.he4_pa_intensity_delta_minus, + ), DataProductVariable("he4_energy", NumpyArrayMatcher(he_energy)), - DataProductVariable("he4_energy_delta_plus", sentinel.he4_energy_delta_plus), - DataProductVariable("he4_energy_delta_minus", sentinel.he4_energy_delta_minus), - DataProductVariable("cno_macropixel_intensity", sentinel.cno_macropixel_intensity), - DataProductVariable("cno_macropixel_intensity_delta_plus", sentinel.cno_macropixel_intensity_delta_plus), - DataProductVariable("cno_macropixel_intensity_delta_minus", sentinel.cno_macropixel_intensity_delta_minus), - DataProductVariable("cno_macropixel_intensity_pa", sentinel.cno_pa_intensity), - DataProductVariable("cno_macropixel_intensity_pa_delta_plus", sentinel.cno_pa_intensity_delta_plus), - DataProductVariable("cno_macropixel_intensity_pa_delta_minus", sentinel.cno_pa_intensity_delta_minus), + DataProductVariable( + "he4_energy_delta_plus", sentinel.he4_energy_delta_plus + ), + DataProductVariable( + "he4_energy_delta_minus", sentinel.he4_energy_delta_minus + ), + DataProductVariable( + "cno_macropixel_intensity", sentinel.cno_macropixel_intensity + ), + DataProductVariable( + "cno_macropixel_intensity_delta_plus", + sentinel.cno_macropixel_intensity_delta_plus, + ), + DataProductVariable( + "cno_macropixel_intensity_delta_minus", + sentinel.cno_macropixel_intensity_delta_minus, + ), + DataProductVariable( + "cno_macropixel_intensity_pa", sentinel.cno_pa_intensity + ), + DataProductVariable( + "cno_macropixel_intensity_pa_delta_plus", + sentinel.cno_pa_intensity_delta_plus, + ), + DataProductVariable( + "cno_macropixel_intensity_pa_delta_minus", + sentinel.cno_pa_intensity_delta_minus, + ), DataProductVariable("cno_energy", NumpyArrayMatcher(cno_energy)), - DataProductVariable("cno_energy_delta_plus", sentinel.cno_energy_delta_plus), - DataProductVariable("cno_energy_delta_minus", sentinel.cno_energy_delta_minus), - DataProductVariable("nemgsi_macropixel_intensity", sentinel.ne_mg_si_macropixel_intensity), - DataProductVariable("nemgsi_macropixel_intensity_delta_plus", - sentinel.ne_mg_si_macropixel_intensity_delta_plus), - DataProductVariable("nemgsi_macropixel_intensity_delta_minus", - sentinel.ne_mg_si_macropixel_intensity_delta_minus), - DataProductVariable("nemgsi_macropixel_intensity_pa", sentinel.ne_mg_si_pa_intensity), - DataProductVariable("nemgsi_macropixel_intensity_pa_delta_plus", sentinel.ne_mg_si_pa_intensity_delta_plus), - DataProductVariable("nemgsi_macropixel_intensity_pa_delta_minus", - sentinel.ne_mg_si_pa_intensity_delta_minus), + DataProductVariable( + "cno_energy_delta_plus", sentinel.cno_energy_delta_plus + ), + DataProductVariable( + "cno_energy_delta_minus", sentinel.cno_energy_delta_minus + ), + DataProductVariable( + "nemgsi_macropixel_intensity", sentinel.ne_mg_si_macropixel_intensity + ), + DataProductVariable( + "nemgsi_macropixel_intensity_delta_plus", + sentinel.ne_mg_si_macropixel_intensity_delta_plus, + ), + DataProductVariable( + "nemgsi_macropixel_intensity_delta_minus", + sentinel.ne_mg_si_macropixel_intensity_delta_minus, + ), + DataProductVariable( + "nemgsi_macropixel_intensity_pa", sentinel.ne_mg_si_pa_intensity + ), + DataProductVariable( + "nemgsi_macropixel_intensity_pa_delta_plus", + sentinel.ne_mg_si_pa_intensity_delta_plus, + ), + DataProductVariable( + "nemgsi_macropixel_intensity_pa_delta_minus", + sentinel.ne_mg_si_pa_intensity_delta_minus, + ), DataProductVariable("nemgsi_energy", NumpyArrayMatcher(nemgsi_energy)), - DataProductVariable("nemgsi_energy_delta_plus", sentinel.ne_mg_si_energy_delta_plus), - DataProductVariable("nemgsi_energy_delta_minus", sentinel.ne_mg_si_energy_delta_minus), - DataProductVariable("fe_macropixel_intensity", sentinel.iron_macropixel_intensity), - DataProductVariable("fe_macropixel_intensity_delta_plus", sentinel.iron_macropixel_intensity_delta_plus), - DataProductVariable("fe_macropixel_intensity_delta_minus", sentinel.iron_macropixel_intensity_delta_minus), - DataProductVariable("fe_macropixel_intensity_pa", sentinel.iron_pa_intensity), - DataProductVariable("fe_macropixel_intensity_pa_delta_plus", sentinel.iron_pa_intensity_delta_plus), - DataProductVariable("fe_macropixel_intensity_pa_delta_minus", sentinel.iron_pa_intensity_delta_minus), + DataProductVariable( + "nemgsi_energy_delta_plus", sentinel.ne_mg_si_energy_delta_plus + ), + DataProductVariable( + "nemgsi_energy_delta_minus", sentinel.ne_mg_si_energy_delta_minus + ), + DataProductVariable( + "fe_macropixel_intensity", sentinel.iron_macropixel_intensity + ), + DataProductVariable( + "fe_macropixel_intensity_delta_plus", + sentinel.iron_macropixel_intensity_delta_plus, + ), + DataProductVariable( + "fe_macropixel_intensity_delta_minus", + sentinel.iron_macropixel_intensity_delta_minus, + ), + DataProductVariable( + "fe_macropixel_intensity_pa", sentinel.iron_pa_intensity + ), + DataProductVariable( + "fe_macropixel_intensity_pa_delta_plus", + sentinel.iron_pa_intensity_delta_plus, + ), + DataProductVariable( + "fe_macropixel_intensity_pa_delta_minus", + sentinel.iron_pa_intensity_delta_minus, + ), DataProductVariable("fe_energy", NumpyArrayMatcher(fe_energy)), - DataProductVariable("fe_energy_delta_plus", sentinel.iron_energy_delta_plus), - DataProductVariable("fe_energy_delta_minus", sentinel.iron_energy_delta_minus), - DataProductVariable("measurement_pitch_angle", mock_measurement_pitch_angle), - DataProductVariable("measurement_gyrophase", sentinel.measurement_gyrophase), - DataProductVariable("pitch_angle_label", - ["Pitch Angle Label 1", "Pitch Angle Label 2", "Pitch Angle Label 3", - "Pitch Angle Label 4", "Pitch Angle Label 5", "Pitch Angle Label 6"]), - DataProductVariable("gyrophase_label", - ["Gyrophase Label 1", "Gyrophase Label 2", "Gyrophase Label 3", "Gyrophase Label 4"]), - DataProductVariable("h_energy_label", - ["H Energy Label 1", "H Energy Label 2", "H Energy Label 3", "H Energy Label 4"]), - DataProductVariable("he4_energy_label", ["He4 Energy Label 1", "He4 Energy Label 2", "He4 Energy Label 3", - "He4 Energy Label 4"]), - DataProductVariable("cno_energy_label", ["CNO Energy Label 1", "CNO Energy Label 2", "CNO Energy Label 3", - "CNO Energy Label 4"]), - DataProductVariable("nemgsi_energy_label", - ["NeMgSi Energy Label 1", "NeMgSi Energy Label 2", "NeMgSi Energy Label 3", - "NeMgSi Energy Label 4"]), - DataProductVariable("fe_energy_label", - ["Fe Energy Label 1", "Fe Energy Label 2", "Fe Energy Label 3", "Fe Energy Label 4"]), + DataProductVariable( + "fe_energy_delta_plus", sentinel.iron_energy_delta_plus + ), + DataProductVariable( + "fe_energy_delta_minus", sentinel.iron_energy_delta_minus + ), + DataProductVariable( + "measurement_pitch_angle", mock_measurement_pitch_angle + ), + DataProductVariable( + "measurement_gyrophase", sentinel.measurement_gyrophase + ), + DataProductVariable( + "pitch_angle_label", + [ + "Pitch Angle Label 1", + "Pitch Angle Label 2", + "Pitch Angle Label 3", + "Pitch Angle Label 4", + "Pitch Angle Label 5", + "Pitch Angle Label 6", + ], + ), + DataProductVariable( + "gyrophase_label", + [ + "Gyrophase Label 1", + "Gyrophase Label 2", + "Gyrophase Label 3", + "Gyrophase Label 4", + ], + ), + DataProductVariable( + "h_energy_label", + [ + "H Energy Label 1", + "H Energy Label 2", + "H Energy Label 3", + "H Energy Label 4", + ], + ), + DataProductVariable( + "he4_energy_label", + [ + "He4 Energy Label 1", + "He4 Energy Label 2", + "He4 Energy Label 3", + "He4 Energy Label 4", + ], + ), + DataProductVariable( + "cno_energy_label", + [ + "CNO Energy Label 1", + "CNO Energy Label 2", + "CNO Energy Label 3", + "CNO Energy Label 4", + ], + ), + DataProductVariable( + "nemgsi_energy_label", + [ + "NeMgSi Energy Label 1", + "NeMgSi Energy Label 2", + "NeMgSi Energy Label 3", + "NeMgSi Energy Label 4", + ], + ), + DataProductVariable( + "fe_energy_label", + [ + "Fe Energy Label 1", + "Fe Energy Label 2", + "Fe Energy Label 3", + "Fe Energy Label 4", + ], + ), DataProductVariable("azimuth", NumpyArrayMatcher(np.array([10, 180, 350]))), - DataProductVariable("zenith", NumpyArrayMatcher(np.array([10.25, 90.75, 170.5]))), - DataProductVariable("azimuth_label", - ["10.0", "180.0", "350.0"]), - DataProductVariable("zenith_label", - ["10.25", "90.75", "170.5"]), + DataProductVariable( + "zenith", NumpyArrayMatcher(np.array([10.25, 90.75, 170.5])) + ), + DataProductVariable("azimuth_label", ["10.0", "180.0", "350.0"]), + DataProductVariable("zenith_label", ["10.25", "90.75", "170.5"]), DataProductVariable("hit_flags", sentinel.quality_flags), ] diff --git a/tests/hit/l3/test_hit_processor.py b/tests/hit/l3/test_hit_processor.py index 7603f6180..3ae2e147d 100644 --- a/tests/hit/l3/test_hit_processor.py +++ b/tests/hit/l3/test_hit_processor.py @@ -7,15 +7,29 @@ import numpy as np -from imap_l3_processing.constants import UNSIGNED_INT2_FILL_VALUE, UNSIGNED_INT1_FILL_VALUE -from imap_l3_processing.hit.l3.hit_l3_sectored_dependencies import HITL3SectoredDependencies +from imap_l3_processing.constants import ( + UNSIGNED_INT2_FILL_VALUE, + UNSIGNED_INT1_FILL_VALUE, +) +from imap_l3_processing.hit.l3.hit_l3_sectored_dependencies import ( + HITL3SectoredDependencies, +) from imap_l3_processing.hit.l3.hit_processor import HitProcessor from imap_l3_processing.hit.l3.models import HitL2Data, HitL1Data -from imap_l3_processing.hit.l3.pha.pha_event_reader import PHAWord, Detector, RawPHAEvent, PHAExtendedHeader, StimBlock, \ - ExtendedStimHeader +from imap_l3_processing.hit.l3.pha.pha_event_reader import ( + PHAWord, + Detector, + RawPHAEvent, + PHAExtendedHeader, + StimBlock, + ExtendedStimHeader, +) from imap_l3_processing.hit.l3.pha.science.calculate_pha import EventOutput -from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import DetectedRange, DetectorSide, \ - DetectorRange +from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import ( + DetectedRange, + DetectorSide, + DetectorRange, +) from imap_l3_processing.hit.l3.sectored_products.models import HitPitchAngleDataProduct from imap_l3_processing.hit.quality_flags import HitL3Flags from imap_l3_processing.models import MagData, InputMetadata @@ -25,28 +39,35 @@ class TestHitProcessor(TestCase): def test_is_a_processor(self): - self.assertIsInstance( - HitProcessor(Mock(), Mock()), - Processor - ) + self.assertIsInstance(HitProcessor(Mock(), Mock()), Processor) @patch("imap_l3_processing.utils.spiceypy") - @patch('imap_l3_processing.hit.l3.hit_processor.save_data') - @patch('imap_l3_processing.hit.l3.hit_processor.HITL3SectoredDependencies.fetch_dependencies') - @patch('imap_l3_processing.hit.l3.hit_processor.calculate_unit_vector') - @patch('imap_l3_processing.hit.l3.hit_processor.get_hit_bin_polar_coordinates') - @patch('imap_l3_processing.hit.l3.hit_processor.get_sector_unit_vectors') - @patch('imap_l3_processing.hit.l3.hit_processor.calculate_pitch_angle') - @patch('imap_l3_processing.hit.l3.hit_processor.calculate_gyrophase') - @patch('imap_l3_processing.hit.l3.hit_processor.rebin_by_pitch_angle_and_gyrophase') - @patch('imap_l3_processing.hit.l3.hit_processor.rotate_particle_vectors_from_hit_despun_to_imap_despun') - def test_process_pitch_angle_product(self, mock_rotate_particle_vectors_from_hit_despun_to_imap_despun, - mock_hit_rebin, mock_calculate_gyrophase, - mock_calculate_pitch_angle, - mock_get_sector_unit_vectors, mock_get_hit_bin_polar_coordinates, - mock_calculate_unit_vector, - mock_fetch_dependencies, mock_save_data, - mock_spiceypy): + @patch("imap_l3_processing.hit.l3.hit_processor.save_data") + @patch( + "imap_l3_processing.hit.l3.hit_processor.HITL3SectoredDependencies.fetch_dependencies" + ) + @patch("imap_l3_processing.hit.l3.hit_processor.calculate_unit_vector") + @patch("imap_l3_processing.hit.l3.hit_processor.get_hit_bin_polar_coordinates") + @patch("imap_l3_processing.hit.l3.hit_processor.get_sector_unit_vectors") + @patch("imap_l3_processing.hit.l3.hit_processor.calculate_pitch_angle") + @patch("imap_l3_processing.hit.l3.hit_processor.calculate_gyrophase") + @patch("imap_l3_processing.hit.l3.hit_processor.rebin_by_pitch_angle_and_gyrophase") + @patch( + "imap_l3_processing.hit.l3.hit_processor.rotate_particle_vectors_from_hit_despun_to_imap_despun" + ) + def test_process_pitch_angle_product( + self, + mock_rotate_particle_vectors_from_hit_despun_to_imap_despun, + mock_hit_rebin, + mock_calculate_gyrophase, + mock_calculate_pitch_angle, + mock_get_sector_unit_vectors, + mock_get_hit_bin_polar_coordinates, + mock_calculate_unit_vector, + mock_fetch_dependencies, + mock_save_data, + mock_spiceypy, + ): mock_spiceypy.ktotal.return_value = 0 input_metadata = InputMetadata( @@ -55,11 +76,13 @@ def test_process_pitch_angle_product(self, mock_rotate_particle_vectors_from_hit start_date=None, end_date=None, version="", - descriptor="macropixel" + descriptor="macropixel", ) mock_processing_input_collection = Mock() - mock_processing_input_collection.get_file_paths.return_value = [Path("test/parent_path")] + mock_processing_input_collection.get_file_paths.return_value = [ + Path("test/parent_path") + ] epochs = np.array([123, 234]) epoch_deltas = np.array([12, 13]) @@ -107,35 +130,70 @@ def test_process_pitch_angle_product(self, mock_rotate_particle_vectors_from_hit delta_minus_NeMgSi_time2 = np.full(2, 102) mock_hit_data.cno = np.array([CNO_time1, CNO_time2]) - mock_hit_data.delta_plus_cno = np.array([delta_plus_CNO_time1, delta_plus_CNO_time2]) - mock_hit_data.delta_minus_cno = np.array([delta_minus_CNO_time1, delta_minus_CNO_time2]) + mock_hit_data.delta_plus_cno = np.array( + [delta_plus_CNO_time1, delta_plus_CNO_time2] + ) + mock_hit_data.delta_minus_cno = np.array( + [delta_minus_CNO_time1, delta_minus_CNO_time2] + ) mock_hit_data.he4 = np.array([helium4_time1, helium4_time2]) - mock_hit_data.delta_plus_he4 = np.array([delta_plus_helium4_time1, delta_plus_helium4_time2]) - mock_hit_data.delta_minus_he4 = np.array([delta_minus_helium4_time1, delta_minus_helium4_time2]) + mock_hit_data.delta_plus_he4 = np.array( + [delta_plus_helium4_time1, delta_plus_helium4_time2] + ) + mock_hit_data.delta_minus_he4 = np.array( + [delta_minus_helium4_time1, delta_minus_helium4_time2] + ) mock_hit_data.h = np.array([hydrogen_time1, hydrogen_time2]) - mock_hit_data.delta_plus_h = np.array([delta_plus_hydrogen_time1, delta_plus_hydrogen_time2]) - mock_hit_data.delta_minus_h = np.array([delta_minus_hydrogen_time1, delta_minus_hydrogen_time2]) + mock_hit_data.delta_plus_h = np.array( + [delta_plus_hydrogen_time1, delta_plus_hydrogen_time2] + ) + mock_hit_data.delta_minus_h = np.array( + [delta_minus_hydrogen_time1, delta_minus_hydrogen_time2] + ) mock_hit_data.fe = np.array([iron_time1, iron_time2]) - mock_hit_data.delta_plus_fe = np.array([delta_plus_iron_time1, delta_plus_iron_time2]) - mock_hit_data.delta_minus_fe = np.array([delta_minus_iron_time1, delta_minus_iron_time2]) + mock_hit_data.delta_plus_fe = np.array( + [delta_plus_iron_time1, delta_plus_iron_time2] + ) + mock_hit_data.delta_minus_fe = np.array( + [delta_minus_iron_time1, delta_minus_iron_time2] + ) mock_hit_data.nemgsi = np.array([NeMgSi_time1, NeMgSi_time2]) - mock_hit_data.delta_plus_nemgsi = np.array([delta_plus_NeMgSi_time1, delta_plus_NeMgSi_time2]) - mock_hit_data.delta_minus_nemgsi = np.array([delta_minus_NeMgSi_time1, delta_minus_NeMgSi_time2]) + mock_hit_data.delta_plus_nemgsi = np.array( + [delta_plus_NeMgSi_time1, delta_plus_NeMgSi_time2] + ) + mock_hit_data.delta_minus_nemgsi = np.array( + [delta_minus_NeMgSi_time1, delta_minus_NeMgSi_time2] + ) mock_dependencies.data = mock_hit_data mock_fetch_dependencies.return_value = mock_dependencies - mock_calculate_unit_vector.side_effect = [sentinel.mag_unit_vector1, sentinel.mag_unit_vector2] + mock_calculate_unit_vector.side_effect = [ + sentinel.mag_unit_vector1, + sentinel.mag_unit_vector2, + ] mock_get_hit_bin_polar_coordinates.return_value = ( - sentinel.pitch_angles, sentinel.gyrophases, sentinel.pitch_angle_deltas, sentinel.gyrophase_deltas) + sentinel.pitch_angles, + sentinel.gyrophases, + sentinel.pitch_angle_deltas, + sentinel.gyrophase_deltas, + ) sector_unit_vectors_zenith_first = np.full((8, 15, 3), np.sqrt(1 / 3)) mock_get_sector_unit_vectors.return_value = sector_unit_vectors_zenith_first expected_sector_unit_vectors = np.full((15, 8, 3), np.sqrt(1 / 3)) - mock_rotate_particle_vectors_from_hit_despun_to_imap_despun.return_value = sentinel.rotated_particle_vector - mock_calculate_pitch_angle.side_effect = [sentinel.pitch_angle1, sentinel.pitch_angle2] - mock_calculate_gyrophase.side_effect = [sentinel.gyrophase1, sentinel.gyrophase2] + mock_rotate_particle_vectors_from_hit_despun_to_imap_despun.return_value = ( + sentinel.rotated_particle_vector + ) + mock_calculate_pitch_angle.side_effect = [ + sentinel.pitch_angle1, + sentinel.pitch_angle2, + ] + mock_calculate_gyrophase.side_effect = [ + sentinel.gyrophase1, + sentinel.gyrophase2, + ] rebinned_pa_gyro_CNO_time1 = np.full((2, 8, 15), 1) rebinned_pa_gyro_CNO_delta_plus_time1 = np.full((2, 8, 15), 100) @@ -200,294 +258,622 @@ def test_process_pitch_angle_product(self, mock_rotate_particle_vectors_from_hit rebinned_pa_NeMgSi_delta_minus_time2 = np.full((2, 8), 2001) mock_hit_rebin.side_effect = [ - (rebinned_pa_gyro_hydrogen_time1, rebinned_pa_gyro_hydrogen_delta_plus_time1, - rebinned_pa_gyro_hydrogen_delta_minus_time1, - rebinned_pa_hydrogen_time1, rebinned_pa_hydrogen_delta_plus_time1, rebinned_pa_hydrogen_delta_minus_time1), - (rebinned_pa_gyro_helium4_time1, rebinned_pa_gyro_helium4_delta_plus_time1, - rebinned_pa_gyro_helium4_delta_minus_time1, - rebinned_pa_helium4_time1, rebinned_pa_helium4_delta_plus_time1, rebinned_pa_helium4_delta_minus_time1), - (rebinned_pa_gyro_CNO_time1, rebinned_pa_gyro_CNO_delta_plus_time1, rebinned_pa_gyro_CNO_delta_minus_time1, - rebinned_pa_CNO_time1, rebinned_pa_CNO_delta_plus_time1, rebinned_pa_CNO_delta_minus_time1), - (rebinned_pa_gyro_NeMgSi_time1, rebinned_pa_gyro_NeMgSi_delta_plus_time1, - rebinned_pa_gyro_NeMgSi_delta_minus_time1, - rebinned_pa_NeMgSi_time1, rebinned_pa_NeMgSi_delta_plus_time1, rebinned_pa_NeMgSi_delta_minus_time1), - (rebinned_pa_gyro_iron_time1, rebinned_pa_gyro_iron_delta_plus_time1, - rebinned_pa_gyro_iron_delta_minus_time1, - rebinned_pa_iron_time1, rebinned_pa_iron_delta_plus_time1, rebinned_pa_iron_delta_minus_time1), - - (rebinned_pa_gyro_hydrogen_time2, rebinned_pa_gyro_hydrogen_delta_plus_time2, - rebinned_pa_gyro_hydrogen_delta_minus_time2, - rebinned_pa_hydrogen_time2, rebinned_pa_hydrogen_delta_plus_time2, rebinned_pa_hydrogen_delta_minus_time2), - (rebinned_pa_gyro_helium4_time2, rebinned_pa_gyro_helium4_delta_plus_time2, - rebinned_pa_gyro_helium4_delta_minus_time2, - rebinned_pa_helium4_time2, rebinned_pa_helium4_delta_plus_time2, rebinned_pa_helium4_delta_minus_time2), - (rebinned_pa_gyro_CNO_time2, rebinned_pa_gyro_CNO_delta_plus_time2, rebinned_pa_gyro_CNO_delta_minus_time2, - rebinned_pa_CNO_time2, rebinned_pa_CNO_delta_plus_time2, rebinned_pa_CNO_delta_minus_time2), - (rebinned_pa_gyro_NeMgSi_time2, rebinned_pa_gyro_NeMgSi_delta_plus_time2, - rebinned_pa_gyro_NeMgSi_delta_minus_time2, - rebinned_pa_NeMgSi_time2, rebinned_pa_NeMgSi_delta_plus_time2, rebinned_pa_NeMgSi_delta_minus_time2), - (rebinned_pa_gyro_iron_time2, rebinned_pa_gyro_iron_delta_plus_time2, - rebinned_pa_gyro_iron_delta_minus_time2, - rebinned_pa_iron_time2, rebinned_pa_iron_delta_plus_time2, rebinned_pa_iron_delta_minus_time2), + ( + rebinned_pa_gyro_hydrogen_time1, + rebinned_pa_gyro_hydrogen_delta_plus_time1, + rebinned_pa_gyro_hydrogen_delta_minus_time1, + rebinned_pa_hydrogen_time1, + rebinned_pa_hydrogen_delta_plus_time1, + rebinned_pa_hydrogen_delta_minus_time1, + ), + ( + rebinned_pa_gyro_helium4_time1, + rebinned_pa_gyro_helium4_delta_plus_time1, + rebinned_pa_gyro_helium4_delta_minus_time1, + rebinned_pa_helium4_time1, + rebinned_pa_helium4_delta_plus_time1, + rebinned_pa_helium4_delta_minus_time1, + ), + ( + rebinned_pa_gyro_CNO_time1, + rebinned_pa_gyro_CNO_delta_plus_time1, + rebinned_pa_gyro_CNO_delta_minus_time1, + rebinned_pa_CNO_time1, + rebinned_pa_CNO_delta_plus_time1, + rebinned_pa_CNO_delta_minus_time1, + ), + ( + rebinned_pa_gyro_NeMgSi_time1, + rebinned_pa_gyro_NeMgSi_delta_plus_time1, + rebinned_pa_gyro_NeMgSi_delta_minus_time1, + rebinned_pa_NeMgSi_time1, + rebinned_pa_NeMgSi_delta_plus_time1, + rebinned_pa_NeMgSi_delta_minus_time1, + ), + ( + rebinned_pa_gyro_iron_time1, + rebinned_pa_gyro_iron_delta_plus_time1, + rebinned_pa_gyro_iron_delta_minus_time1, + rebinned_pa_iron_time1, + rebinned_pa_iron_delta_plus_time1, + rebinned_pa_iron_delta_minus_time1, + ), + ( + rebinned_pa_gyro_hydrogen_time2, + rebinned_pa_gyro_hydrogen_delta_plus_time2, + rebinned_pa_gyro_hydrogen_delta_minus_time2, + rebinned_pa_hydrogen_time2, + rebinned_pa_hydrogen_delta_plus_time2, + rebinned_pa_hydrogen_delta_minus_time2, + ), + ( + rebinned_pa_gyro_helium4_time2, + rebinned_pa_gyro_helium4_delta_plus_time2, + rebinned_pa_gyro_helium4_delta_minus_time2, + rebinned_pa_helium4_time2, + rebinned_pa_helium4_delta_plus_time2, + rebinned_pa_helium4_delta_minus_time2, + ), + ( + rebinned_pa_gyro_CNO_time2, + rebinned_pa_gyro_CNO_delta_plus_time2, + rebinned_pa_gyro_CNO_delta_minus_time2, + rebinned_pa_CNO_time2, + rebinned_pa_CNO_delta_plus_time2, + rebinned_pa_CNO_delta_minus_time2, + ), + ( + rebinned_pa_gyro_NeMgSi_time2, + rebinned_pa_gyro_NeMgSi_delta_plus_time2, + rebinned_pa_gyro_NeMgSi_delta_minus_time2, + rebinned_pa_NeMgSi_time2, + rebinned_pa_NeMgSi_delta_plus_time2, + rebinned_pa_NeMgSi_delta_minus_time2, + ), + ( + rebinned_pa_gyro_iron_time2, + rebinned_pa_gyro_iron_delta_plus_time2, + rebinned_pa_gyro_iron_delta_minus_time2, + rebinned_pa_iron_time2, + rebinned_pa_iron_delta_plus_time2, + rebinned_pa_iron_delta_minus_time2, + ), ] processor = HitProcessor(mock_processing_input_collection, input_metadata) product = processor.process() - mock_fetch_dependencies.assert_called_once_with(mock_processing_input_collection) + mock_fetch_dependencies.assert_called_once_with( + mock_processing_input_collection + ) - mock_mag_data.rebin_to.assert_called_once_with(mock_hit_data.epoch, mock_hit_data.epoch_delta) + mock_mag_data.rebin_to.assert_called_once_with( + mock_hit_data.epoch, mock_hit_data.epoch_delta + ) - mock_calculate_unit_vector.assert_has_calls([ - call(sentinel.mag_vector1), - call(sentinel.mag_vector2) - ]) - mock_get_sector_unit_vectors.assert_called_once_with(mock_hit_data.zenith, mock_hit_data.azimuth) + mock_calculate_unit_vector.assert_has_calls( + [call(sentinel.mag_vector1), call(sentinel.mag_vector2)] + ) + mock_get_sector_unit_vectors.assert_called_once_with( + mock_hit_data.zenith, mock_hit_data.azimuth + ) - self.assertEqual(1, mock_rotate_particle_vectors_from_hit_despun_to_imap_despun.call_count) + self.assertEqual( + 1, mock_rotate_particle_vectors_from_hit_despun_to_imap_despun.call_count + ) np.testing.assert_array_equal( - mock_rotate_particle_vectors_from_hit_despun_to_imap_despun.call_args_list[0].args[0], - -expected_sector_unit_vectors) + mock_rotate_particle_vectors_from_hit_despun_to_imap_despun.call_args_list[ + 0 + ].args[0], + -expected_sector_unit_vectors, + ) self.assertEqual(2, mock_calculate_pitch_angle.call_count) - np.testing.assert_array_equal(mock_calculate_pitch_angle.call_args_list[0].args[0], - sentinel.rotated_particle_vector) - self.assertEqual(mock_calculate_pitch_angle.call_args_list[0].args[1], sentinel.mag_unit_vector1) + np.testing.assert_array_equal( + mock_calculate_pitch_angle.call_args_list[0].args[0], + sentinel.rotated_particle_vector, + ) + self.assertEqual( + mock_calculate_pitch_angle.call_args_list[0].args[1], + sentinel.mag_unit_vector1, + ) - np.testing.assert_array_equal(mock_calculate_pitch_angle.call_args_list[1].args[0], - sentinel.rotated_particle_vector) - self.assertEqual(mock_calculate_pitch_angle.call_args_list[1].args[1], sentinel.mag_unit_vector2) + np.testing.assert_array_equal( + mock_calculate_pitch_angle.call_args_list[1].args[0], + sentinel.rotated_particle_vector, + ) + self.assertEqual( + mock_calculate_pitch_angle.call_args_list[1].args[1], + sentinel.mag_unit_vector2, + ) self.assertEqual(2, mock_calculate_gyrophase.call_count) - np.testing.assert_array_equal(mock_calculate_gyrophase.call_args_list[0].args[0], - sentinel.rotated_particle_vector) - self.assertEqual(mock_calculate_gyrophase.call_args_list[0].args[1], sentinel.mag_unit_vector1) + np.testing.assert_array_equal( + mock_calculate_gyrophase.call_args_list[0].args[0], + sentinel.rotated_particle_vector, + ) + self.assertEqual( + mock_calculate_gyrophase.call_args_list[0].args[1], + sentinel.mag_unit_vector1, + ) - np.testing.assert_array_equal(mock_calculate_gyrophase.call_args_list[1].args[0], - sentinel.rotated_particle_vector) - self.assertEqual(mock_calculate_gyrophase.call_args_list[1].args[1], sentinel.mag_unit_vector2) + np.testing.assert_array_equal( + mock_calculate_gyrophase.call_args_list[1].args[0], + sentinel.rotated_particle_vector, + ) + self.assertEqual( + mock_calculate_gyrophase.call_args_list[1].args[1], + sentinel.mag_unit_vector2, + ) number_of_pitch_angle_bins = 8 number_of_gyrophase_bins = 15 - mock_hit_rebin.assert_has_calls([ - call(NumpyArrayMatcher(hydrogen_time1), NumpyArrayMatcher(delta_plus_hydrogen_time1), - NumpyArrayMatcher(delta_minus_hydrogen_time1), sentinel.pitch_angle1, sentinel.gyrophase1, - number_of_pitch_angle_bins, - number_of_gyrophase_bins), - call(NumpyArrayMatcher(helium4_time1), NumpyArrayMatcher(delta_plus_helium4_time1), - NumpyArrayMatcher(delta_minus_helium4_time1), sentinel.pitch_angle1, sentinel.gyrophase1, - number_of_pitch_angle_bins, - number_of_gyrophase_bins), - call(NumpyArrayMatcher(CNO_time1), NumpyArrayMatcher(delta_plus_CNO_time1), - NumpyArrayMatcher(delta_minus_CNO_time1), sentinel.pitch_angle1, sentinel.gyrophase1, - number_of_pitch_angle_bins, - number_of_gyrophase_bins), - call(NumpyArrayMatcher(NeMgSi_time1), NumpyArrayMatcher(delta_plus_NeMgSi_time1), - NumpyArrayMatcher(delta_minus_NeMgSi_time1), sentinel.pitch_angle1, sentinel.gyrophase1, - number_of_pitch_angle_bins, - number_of_gyrophase_bins), - call(NumpyArrayMatcher(iron_time1), NumpyArrayMatcher(delta_plus_iron_time1), - NumpyArrayMatcher(delta_minus_iron_time1), sentinel.pitch_angle1, sentinel.gyrophase1, - number_of_pitch_angle_bins, - number_of_gyrophase_bins), - call(NumpyArrayMatcher(hydrogen_time2), NumpyArrayMatcher(delta_plus_hydrogen_time2), - NumpyArrayMatcher(delta_minus_hydrogen_time2), sentinel.pitch_angle2, sentinel.gyrophase2, - number_of_pitch_angle_bins, - number_of_gyrophase_bins), - call(NumpyArrayMatcher(helium4_time2), NumpyArrayMatcher(delta_plus_helium4_time2), - NumpyArrayMatcher(delta_minus_helium4_time2), sentinel.pitch_angle2, sentinel.gyrophase2, - number_of_pitch_angle_bins, - number_of_gyrophase_bins), - call(NumpyArrayMatcher(CNO_time2), NumpyArrayMatcher(delta_plus_CNO_time2), - NumpyArrayMatcher(delta_minus_CNO_time2), sentinel.pitch_angle2, sentinel.gyrophase2, - number_of_pitch_angle_bins, - number_of_gyrophase_bins), - call(NumpyArrayMatcher(NeMgSi_time2), NumpyArrayMatcher(delta_plus_NeMgSi_time2), - NumpyArrayMatcher(delta_minus_NeMgSi_time2), sentinel.pitch_angle2, sentinel.gyrophase2, - number_of_pitch_angle_bins, - number_of_gyrophase_bins), - call(NumpyArrayMatcher(iron_time2), NumpyArrayMatcher(delta_plus_iron_time2), - NumpyArrayMatcher(delta_minus_iron_time2), sentinel.pitch_angle2, sentinel.gyrophase2, - number_of_pitch_angle_bins, - number_of_gyrophase_bins), - ]) - - saved_data_product: HitPitchAngleDataProduct = mock_save_data.call_args_list[0].args[0] + mock_hit_rebin.assert_has_calls( + [ + call( + NumpyArrayMatcher(hydrogen_time1), + NumpyArrayMatcher(delta_plus_hydrogen_time1), + NumpyArrayMatcher(delta_minus_hydrogen_time1), + sentinel.pitch_angle1, + sentinel.gyrophase1, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ), + call( + NumpyArrayMatcher(helium4_time1), + NumpyArrayMatcher(delta_plus_helium4_time1), + NumpyArrayMatcher(delta_minus_helium4_time1), + sentinel.pitch_angle1, + sentinel.gyrophase1, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ), + call( + NumpyArrayMatcher(CNO_time1), + NumpyArrayMatcher(delta_plus_CNO_time1), + NumpyArrayMatcher(delta_minus_CNO_time1), + sentinel.pitch_angle1, + sentinel.gyrophase1, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ), + call( + NumpyArrayMatcher(NeMgSi_time1), + NumpyArrayMatcher(delta_plus_NeMgSi_time1), + NumpyArrayMatcher(delta_minus_NeMgSi_time1), + sentinel.pitch_angle1, + sentinel.gyrophase1, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ), + call( + NumpyArrayMatcher(iron_time1), + NumpyArrayMatcher(delta_plus_iron_time1), + NumpyArrayMatcher(delta_minus_iron_time1), + sentinel.pitch_angle1, + sentinel.gyrophase1, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ), + call( + NumpyArrayMatcher(hydrogen_time2), + NumpyArrayMatcher(delta_plus_hydrogen_time2), + NumpyArrayMatcher(delta_minus_hydrogen_time2), + sentinel.pitch_angle2, + sentinel.gyrophase2, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ), + call( + NumpyArrayMatcher(helium4_time2), + NumpyArrayMatcher(delta_plus_helium4_time2), + NumpyArrayMatcher(delta_minus_helium4_time2), + sentinel.pitch_angle2, + sentinel.gyrophase2, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ), + call( + NumpyArrayMatcher(CNO_time2), + NumpyArrayMatcher(delta_plus_CNO_time2), + NumpyArrayMatcher(delta_minus_CNO_time2), + sentinel.pitch_angle2, + sentinel.gyrophase2, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ), + call( + NumpyArrayMatcher(NeMgSi_time2), + NumpyArrayMatcher(delta_plus_NeMgSi_time2), + NumpyArrayMatcher(delta_minus_NeMgSi_time2), + sentinel.pitch_angle2, + sentinel.gyrophase2, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ), + call( + NumpyArrayMatcher(iron_time2), + NumpyArrayMatcher(delta_plus_iron_time2), + NumpyArrayMatcher(delta_minus_iron_time2), + sentinel.pitch_angle2, + sentinel.gyrophase2, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ), + ] + ) + + saved_data_product: HitPitchAngleDataProduct = mock_save_data.call_args_list[ + 0 + ].args[0] self.assertEqual(input_metadata, saved_data_product.input_metadata) self.assertEqual(["parent_path"], saved_data_product.parent_file_names) np.testing.assert_array_equal(saved_data_product.epochs, epochs) np.testing.assert_array_equal(saved_data_product.epoch_deltas, epoch_deltas) - np.testing.assert_array_equal(saved_data_product.pitch_angles, sentinel.pitch_angles) - np.testing.assert_array_equal(saved_data_product.pitch_angle_deltas, sentinel.pitch_angle_deltas) - np.testing.assert_array_equal(saved_data_product.gyrophases, sentinel.gyrophases) - np.testing.assert_array_equal(saved_data_product.gyrophase_deltas, sentinel.gyrophase_deltas) - - np.testing.assert_array_equal(saved_data_product.h_macropixel_intensity, - [rebinned_pa_gyro_hydrogen_time1, - rebinned_pa_gyro_hydrogen_time2]) - np.testing.assert_array_equal(saved_data_product.h_macropixel_intensity_delta_plus, - [rebinned_pa_gyro_hydrogen_delta_plus_time1, - rebinned_pa_gyro_hydrogen_delta_plus_time2]) - np.testing.assert_array_equal(saved_data_product.h_macropixel_intensity_delta_minus, - [rebinned_pa_gyro_hydrogen_delta_minus_time1, - rebinned_pa_gyro_hydrogen_delta_minus_time2]) - - np.testing.assert_array_equal(saved_data_product.he4_macropixel_intensity, - [rebinned_pa_gyro_helium4_time1, - rebinned_pa_gyro_helium4_time2]) - np.testing.assert_array_equal(saved_data_product.he4_macropixel_intensity_delta_plus, - [rebinned_pa_gyro_helium4_delta_plus_time1, - rebinned_pa_gyro_helium4_delta_plus_time2]) - np.testing.assert_array_equal(saved_data_product.he4_macropixel_intensity_delta_minus, - [rebinned_pa_gyro_helium4_delta_minus_time1, - rebinned_pa_gyro_helium4_delta_minus_time2]) - np.testing.assert_array_equal(saved_data_product.cno_macropixel_intensity, - [rebinned_pa_gyro_CNO_time1, - rebinned_pa_gyro_CNO_time2]) - np.testing.assert_array_equal(saved_data_product.cno_macropixel_intensity_delta_plus, - [rebinned_pa_gyro_CNO_delta_plus_time1, - rebinned_pa_gyro_CNO_delta_plus_time2]) - np.testing.assert_array_equal(saved_data_product.cno_macropixel_intensity_delta_minus, - [rebinned_pa_gyro_CNO_delta_minus_time1, - rebinned_pa_gyro_CNO_delta_minus_time2]) - - np.testing.assert_array_equal(saved_data_product.ne_mg_si_macropixel_intensity, - [rebinned_pa_gyro_NeMgSi_time1, - rebinned_pa_gyro_NeMgSi_time2]) - np.testing.assert_array_equal(saved_data_product.ne_mg_si_macropixel_intensity_delta_plus, - [rebinned_pa_gyro_NeMgSi_delta_plus_time1, - rebinned_pa_gyro_NeMgSi_delta_plus_time2]) - np.testing.assert_array_equal(saved_data_product.ne_mg_si_macropixel_intensity_delta_minus, - [rebinned_pa_gyro_NeMgSi_delta_minus_time1, - rebinned_pa_gyro_NeMgSi_delta_minus_time2]) - - np.testing.assert_array_equal(saved_data_product.iron_macropixel_intensity, - [rebinned_pa_gyro_iron_time1, - rebinned_pa_gyro_iron_time2]) - np.testing.assert_array_equal(saved_data_product.iron_macropixel_intensity_delta_plus, - [rebinned_pa_gyro_iron_delta_plus_time1, - rebinned_pa_gyro_iron_delta_plus_time2]) - np.testing.assert_array_equal(saved_data_product.iron_macropixel_intensity_delta_minus, - [rebinned_pa_gyro_iron_delta_minus_time1, - rebinned_pa_gyro_iron_delta_minus_time2]) - - np.testing.assert_array_equal(saved_data_product.h_macropixel_intensity_pa, - [rebinned_pa_hydrogen_time1, rebinned_pa_hydrogen_time2]) - np.testing.assert_array_equal(saved_data_product.h_macropixel_intensity_pa_delta_plus, - [rebinned_pa_hydrogen_delta_plus_time1, - rebinned_pa_hydrogen_delta_plus_time2]) - np.testing.assert_array_equal(saved_data_product.h_macropixel_intensity_pa_delta_minus, - [rebinned_pa_hydrogen_delta_minus_time1, - rebinned_pa_hydrogen_delta_minus_time2]) - - np.testing.assert_array_equal(saved_data_product.he4_macropixel_intensity_pa, - [rebinned_pa_helium4_time1, rebinned_pa_helium4_time2]) - np.testing.assert_array_equal(saved_data_product.he4_macropixel_intensity_pa_delta_plus, - [rebinned_pa_helium4_delta_plus_time1, rebinned_pa_helium4_delta_plus_time2]) - np.testing.assert_array_equal(saved_data_product.he4_macropixel_intensity_pa_delta_minus, - [rebinned_pa_helium4_delta_minus_time1, - rebinned_pa_helium4_delta_minus_time2]) - np.testing.assert_array_equal(saved_data_product.cno_macropixel_intensity_pa, - [rebinned_pa_CNO_time1, rebinned_pa_CNO_time2]) - np.testing.assert_array_equal(saved_data_product.cno_macropixel_intensity_pa_delta_plus, - [rebinned_pa_CNO_delta_plus_time1, rebinned_pa_CNO_delta_plus_time2]) - np.testing.assert_array_equal(saved_data_product.cno_macropixel_intensity_pa_delta_minus, - [rebinned_pa_CNO_delta_minus_time1, rebinned_pa_CNO_delta_minus_time2]) - - np.testing.assert_array_equal(saved_data_product.ne_mg_si_macropixel_intensity_pa, - [rebinned_pa_NeMgSi_time1, rebinned_pa_NeMgSi_time2]) - np.testing.assert_array_equal(saved_data_product.ne_mg_si_macropixel_intensity_pa_delta_plus, - [rebinned_pa_NeMgSi_delta_plus_time1, rebinned_pa_NeMgSi_delta_plus_time2]) - np.testing.assert_array_equal(saved_data_product.ne_mg_si_macropixel_intensity_pa_delta_minus, - [rebinned_pa_NeMgSi_delta_minus_time1, rebinned_pa_NeMgSi_delta_minus_time2]) - - np.testing.assert_array_equal(saved_data_product.iron_macropixel_intensity_pa, - [rebinned_pa_iron_time1, rebinned_pa_iron_time2]) - np.testing.assert_array_equal(saved_data_product.iron_macropixel_intensity_pa_delta_plus, - [rebinned_pa_iron_delta_plus_time1, rebinned_pa_iron_delta_plus_time2]) - np.testing.assert_array_equal(saved_data_product.iron_macropixel_intensity_pa_delta_minus, - [rebinned_pa_iron_delta_minus_time1, rebinned_pa_iron_delta_minus_time2]) + np.testing.assert_array_equal( + saved_data_product.pitch_angles, sentinel.pitch_angles + ) + np.testing.assert_array_equal( + saved_data_product.pitch_angle_deltas, sentinel.pitch_angle_deltas + ) + np.testing.assert_array_equal( + saved_data_product.gyrophases, sentinel.gyrophases + ) + np.testing.assert_array_equal( + saved_data_product.gyrophase_deltas, sentinel.gyrophase_deltas + ) + + np.testing.assert_array_equal( + saved_data_product.h_macropixel_intensity, + [rebinned_pa_gyro_hydrogen_time1, rebinned_pa_gyro_hydrogen_time2], + ) + np.testing.assert_array_equal( + saved_data_product.h_macropixel_intensity_delta_plus, + [ + rebinned_pa_gyro_hydrogen_delta_plus_time1, + rebinned_pa_gyro_hydrogen_delta_plus_time2, + ], + ) + np.testing.assert_array_equal( + saved_data_product.h_macropixel_intensity_delta_minus, + [ + rebinned_pa_gyro_hydrogen_delta_minus_time1, + rebinned_pa_gyro_hydrogen_delta_minus_time2, + ], + ) + + np.testing.assert_array_equal( + saved_data_product.he4_macropixel_intensity, + [rebinned_pa_gyro_helium4_time1, rebinned_pa_gyro_helium4_time2], + ) + np.testing.assert_array_equal( + saved_data_product.he4_macropixel_intensity_delta_plus, + [ + rebinned_pa_gyro_helium4_delta_plus_time1, + rebinned_pa_gyro_helium4_delta_plus_time2, + ], + ) + np.testing.assert_array_equal( + saved_data_product.he4_macropixel_intensity_delta_minus, + [ + rebinned_pa_gyro_helium4_delta_minus_time1, + rebinned_pa_gyro_helium4_delta_minus_time2, + ], + ) + np.testing.assert_array_equal( + saved_data_product.cno_macropixel_intensity, + [rebinned_pa_gyro_CNO_time1, rebinned_pa_gyro_CNO_time2], + ) + np.testing.assert_array_equal( + saved_data_product.cno_macropixel_intensity_delta_plus, + [ + rebinned_pa_gyro_CNO_delta_plus_time1, + rebinned_pa_gyro_CNO_delta_plus_time2, + ], + ) + np.testing.assert_array_equal( + saved_data_product.cno_macropixel_intensity_delta_minus, + [ + rebinned_pa_gyro_CNO_delta_minus_time1, + rebinned_pa_gyro_CNO_delta_minus_time2, + ], + ) + + np.testing.assert_array_equal( + saved_data_product.ne_mg_si_macropixel_intensity, + [rebinned_pa_gyro_NeMgSi_time1, rebinned_pa_gyro_NeMgSi_time2], + ) + np.testing.assert_array_equal( + saved_data_product.ne_mg_si_macropixel_intensity_delta_plus, + [ + rebinned_pa_gyro_NeMgSi_delta_plus_time1, + rebinned_pa_gyro_NeMgSi_delta_plus_time2, + ], + ) + np.testing.assert_array_equal( + saved_data_product.ne_mg_si_macropixel_intensity_delta_minus, + [ + rebinned_pa_gyro_NeMgSi_delta_minus_time1, + rebinned_pa_gyro_NeMgSi_delta_minus_time2, + ], + ) + + np.testing.assert_array_equal( + saved_data_product.iron_macropixel_intensity, + [rebinned_pa_gyro_iron_time1, rebinned_pa_gyro_iron_time2], + ) + np.testing.assert_array_equal( + saved_data_product.iron_macropixel_intensity_delta_plus, + [ + rebinned_pa_gyro_iron_delta_plus_time1, + rebinned_pa_gyro_iron_delta_plus_time2, + ], + ) + np.testing.assert_array_equal( + saved_data_product.iron_macropixel_intensity_delta_minus, + [ + rebinned_pa_gyro_iron_delta_minus_time1, + rebinned_pa_gyro_iron_delta_minus_time2, + ], + ) + + np.testing.assert_array_equal( + saved_data_product.h_macropixel_intensity_pa, + [rebinned_pa_hydrogen_time1, rebinned_pa_hydrogen_time2], + ) + np.testing.assert_array_equal( + saved_data_product.h_macropixel_intensity_pa_delta_plus, + [ + rebinned_pa_hydrogen_delta_plus_time1, + rebinned_pa_hydrogen_delta_plus_time2, + ], + ) + np.testing.assert_array_equal( + saved_data_product.h_macropixel_intensity_pa_delta_minus, + [ + rebinned_pa_hydrogen_delta_minus_time1, + rebinned_pa_hydrogen_delta_minus_time2, + ], + ) + + np.testing.assert_array_equal( + saved_data_product.he4_macropixel_intensity_pa, + [rebinned_pa_helium4_time1, rebinned_pa_helium4_time2], + ) + np.testing.assert_array_equal( + saved_data_product.he4_macropixel_intensity_pa_delta_plus, + [ + rebinned_pa_helium4_delta_plus_time1, + rebinned_pa_helium4_delta_plus_time2, + ], + ) + np.testing.assert_array_equal( + saved_data_product.he4_macropixel_intensity_pa_delta_minus, + [ + rebinned_pa_helium4_delta_minus_time1, + rebinned_pa_helium4_delta_minus_time2, + ], + ) + np.testing.assert_array_equal( + saved_data_product.cno_macropixel_intensity_pa, + [rebinned_pa_CNO_time1, rebinned_pa_CNO_time2], + ) + np.testing.assert_array_equal( + saved_data_product.cno_macropixel_intensity_pa_delta_plus, + [rebinned_pa_CNO_delta_plus_time1, rebinned_pa_CNO_delta_plus_time2], + ) + np.testing.assert_array_equal( + saved_data_product.cno_macropixel_intensity_pa_delta_minus, + [rebinned_pa_CNO_delta_minus_time1, rebinned_pa_CNO_delta_minus_time2], + ) + + np.testing.assert_array_equal( + saved_data_product.ne_mg_si_macropixel_intensity_pa, + [rebinned_pa_NeMgSi_time1, rebinned_pa_NeMgSi_time2], + ) + np.testing.assert_array_equal( + saved_data_product.ne_mg_si_macropixel_intensity_pa_delta_plus, + [rebinned_pa_NeMgSi_delta_plus_time1, rebinned_pa_NeMgSi_delta_plus_time2], + ) + np.testing.assert_array_equal( + saved_data_product.ne_mg_si_macropixel_intensity_pa_delta_minus, + [ + rebinned_pa_NeMgSi_delta_minus_time1, + rebinned_pa_NeMgSi_delta_minus_time2, + ], + ) + + np.testing.assert_array_equal( + saved_data_product.iron_macropixel_intensity_pa, + [rebinned_pa_iron_time1, rebinned_pa_iron_time2], + ) + np.testing.assert_array_equal( + saved_data_product.iron_macropixel_intensity_pa_delta_plus, + [rebinned_pa_iron_delta_plus_time1, rebinned_pa_iron_delta_plus_time2], + ) + np.testing.assert_array_equal( + saved_data_product.iron_macropixel_intensity_pa_delta_minus, + [rebinned_pa_iron_delta_minus_time1, rebinned_pa_iron_delta_minus_time2], + ) self.assertIs(mock_hit_data.h_energy, saved_data_product.h_energies) - self.assertIs(mock_hit_data.h_energy_delta_plus, saved_data_product.h_energy_delta_plus) - self.assertIs(mock_hit_data.h_energy_delta_minus, saved_data_product.h_energy_delta_minus) + self.assertIs( + mock_hit_data.h_energy_delta_plus, saved_data_product.h_energy_delta_plus + ) + self.assertIs( + mock_hit_data.h_energy_delta_minus, saved_data_product.h_energy_delta_minus + ) self.assertIs(mock_hit_data.he4_energy, saved_data_product.he4_energies) - self.assertIs(mock_hit_data.he4_energy_delta_plus, saved_data_product.he4_energy_delta_plus) - self.assertIs(mock_hit_data.he4_energy_delta_minus, saved_data_product.he4_energy_delta_minus) + self.assertIs( + mock_hit_data.he4_energy_delta_plus, + saved_data_product.he4_energy_delta_plus, + ) + self.assertIs( + mock_hit_data.he4_energy_delta_minus, + saved_data_product.he4_energy_delta_minus, + ) self.assertIs(mock_hit_data.cno_energy, saved_data_product.cno_energies) - self.assertIs(mock_hit_data.cno_energy_delta_plus, saved_data_product.cno_energy_delta_plus) - self.assertIs(mock_hit_data.cno_energy_delta_minus, saved_data_product.cno_energy_delta_minus) + self.assertIs( + mock_hit_data.cno_energy_delta_plus, + saved_data_product.cno_energy_delta_plus, + ) + self.assertIs( + mock_hit_data.cno_energy_delta_minus, + saved_data_product.cno_energy_delta_minus, + ) self.assertIs(mock_hit_data.nemgsi_energy, saved_data_product.ne_mg_si_energies) - self.assertIs(mock_hit_data.nemgsi_energy_delta_plus, saved_data_product.ne_mg_si_energy_delta_plus) - self.assertIs(mock_hit_data.nemgsi_energy_delta_minus, saved_data_product.ne_mg_si_energy_delta_minus) + self.assertIs( + mock_hit_data.nemgsi_energy_delta_plus, + saved_data_product.ne_mg_si_energy_delta_plus, + ) + self.assertIs( + mock_hit_data.nemgsi_energy_delta_minus, + saved_data_product.ne_mg_si_energy_delta_minus, + ) self.assertIs(mock_hit_data.fe_energy, saved_data_product.iron_energies) - self.assertIs(mock_hit_data.fe_energy_delta_plus, saved_data_product.iron_energy_delta_plus) - self.assertIs(mock_hit_data.fe_energy_delta_minus, saved_data_product.iron_energy_delta_minus) + self.assertIs( + mock_hit_data.fe_energy_delta_plus, + saved_data_product.iron_energy_delta_plus, + ) + self.assertIs( + mock_hit_data.fe_energy_delta_minus, + saved_data_product.iron_energy_delta_minus, + ) self.assertIs(mock_hit_data.azimuth, saved_data_product.azimuth) self.assertIs(mock_hit_data.zenith, saved_data_product.zenith) - np.testing.assert_array_equal(saved_data_product.measurement_pitch_angle, - np.array([sentinel.pitch_angle1, sentinel.pitch_angle2])) - np.testing.assert_array_equal(saved_data_product.measurement_gyrophase, - np.array([sentinel.gyrophase1, sentinel.gyrophase2])) + np.testing.assert_array_equal( + saved_data_product.measurement_pitch_angle, + np.array([sentinel.pitch_angle1, sentinel.pitch_angle2]), + ) + np.testing.assert_array_equal( + saved_data_product.measurement_gyrophase, + np.array([sentinel.gyrophase1, sentinel.gyrophase2]), + ) - np.testing.assert_array_equal(saved_data_product.hit_flags, - np.array([HitL3Flags.PRELIMINARY_MAG, HitL3Flags.PRELIMINARY_MAG])) + np.testing.assert_array_equal( + saved_data_product.hit_flags, + np.array([HitL3Flags.PRELIMINARY_MAG, HitL3Flags.PRELIMINARY_MAG]), + ) self.assertEqual([mock_save_data.return_value], product) @patch("imap_l3_processing.utils.spiceypy") @patch("imap_l3_processing.hit.l3.hit_processor.save_data") @patch("imap_l3_processing.hit.l3.hit_processor.process_pha_event", autospec=True) - @patch("imap_l3_processing.hit.l3.hit_processor.HitL3PhaDependencies.fetch_dependencies") + @patch( + "imap_l3_processing.hit.l3.hit_processor.HitL3PhaDependencies.fetch_dependencies" + ) @patch("imap_l3_processing.hit.l3.hit_processor.PHAEventReader.read_all_pha_events") - def test_process_direct_event_product(self, mock_read_all_events, mock_fetch_dependencies, mock_process_pha_event, - mock_save_data, mock_spiceypy): + def test_process_direct_event_product( + self, + mock_read_all_events, + mock_fetch_dependencies, + mock_process_pha_event, + mock_save_data, + mock_spiceypy, + ): mock_spiceypy.ktotal.return_value = 0 - pha_word_1 = PHAWord(adc_overflow=False, adc_value=11, - detector=Detector(layer=1, side="A", segment="1A", address=2, group="L1A4"), - is_last_pha=True, - is_low_gain=True) - - raw_pha_event_1 = RawPHAEvent(particle_id=1, priority_buffer_num=2, stim_tag=False, haz_tag=False, time_tag=20, - a_b_side_flag=False, has_unread_adcs=True, long_event_flag=False, - culling_flag=True, - spare=True, pha_words=[pha_word_1]) - - raw_pha_event_2 = RawPHAEvent(particle_id=1, priority_buffer_num=2, stim_tag=False, haz_tag=False, time_tag=20, - a_b_side_flag=False, has_unread_adcs=True, long_event_flag=False, - culling_flag=True, - spare=True, pha_words=[deepcopy(pha_word_1)]) - - pha_word_2 = PHAWord(adc_overflow=True, adc_value=12, - detector=Detector(layer=2, side="B", segment="1B", address=5, group="L2B"), - is_last_pha=False, - is_low_gain=False) - - pha_word_3 = PHAWord(adc_overflow=False, adc_value=11, - detector=Detector(layer=1, side="A", segment="2A", address=8, group="L1A1"), - is_last_pha=True, - is_low_gain=True) - - raw_pha_event_3 = RawPHAEvent(particle_id=2, priority_buffer_num=3, stim_tag=True, haz_tag=True, time_tag=30, - a_b_side_flag=True, has_unread_adcs=False, long_event_flag=True, - culling_flag=False, - spare=False, pha_words=[pha_word_2, pha_word_3], - extended_header=PHAExtendedHeader(detector_flags=2, delta_e_index=2, - e_prime_index=True), - stim_block=StimBlock(stim_step=1, stim_gain=2, unused=3, a_l_stim=True), - extended_stim_header=ExtendedStimHeader(dac_value=123, tbd=666)) - - event_output_1 = EventOutput(original_event=raw_pha_event_1, charge=9.0, energies=[1], total_energy=99, - detected_range=DetectedRange(DetectorRange.R2, DetectorSide.A), e_delta=103.7, - e_prime=63.27) - event_output_2 = EventOutput(original_event=raw_pha_event_2, charge=10.0, energies=[4], total_energy=100, - detected_range=None, e_delta=None, e_prime=None) - event_output_3 = EventOutput(original_event=raw_pha_event_3, charge=12.0, energies=[9, 8], total_energy=200, - detected_range=DetectedRange(DetectorRange.R3, DetectorSide.A), e_delta=106.7, - e_prime=69.27) + pha_word_1 = PHAWord( + adc_overflow=False, + adc_value=11, + detector=Detector(layer=1, side="A", segment="1A", address=2, group="L1A4"), + is_last_pha=True, + is_low_gain=True, + ) + + raw_pha_event_1 = RawPHAEvent( + particle_id=1, + priority_buffer_num=2, + stim_tag=False, + haz_tag=False, + time_tag=20, + a_b_side_flag=False, + has_unread_adcs=True, + long_event_flag=False, + culling_flag=True, + spare=True, + pha_words=[pha_word_1], + ) + + raw_pha_event_2 = RawPHAEvent( + particle_id=1, + priority_buffer_num=2, + stim_tag=False, + haz_tag=False, + time_tag=20, + a_b_side_flag=False, + has_unread_adcs=True, + long_event_flag=False, + culling_flag=True, + spare=True, + pha_words=[deepcopy(pha_word_1)], + ) + + pha_word_2 = PHAWord( + adc_overflow=True, + adc_value=12, + detector=Detector(layer=2, side="B", segment="1B", address=5, group="L2B"), + is_last_pha=False, + is_low_gain=False, + ) + + pha_word_3 = PHAWord( + adc_overflow=False, + adc_value=11, + detector=Detector(layer=1, side="A", segment="2A", address=8, group="L1A1"), + is_last_pha=True, + is_low_gain=True, + ) + + raw_pha_event_3 = RawPHAEvent( + particle_id=2, + priority_buffer_num=3, + stim_tag=True, + haz_tag=True, + time_tag=30, + a_b_side_flag=True, + has_unread_adcs=False, + long_event_flag=True, + culling_flag=False, + spare=False, + pha_words=[pha_word_2, pha_word_3], + extended_header=PHAExtendedHeader( + detector_flags=2, delta_e_index=2, e_prime_index=True + ), + stim_block=StimBlock(stim_step=1, stim_gain=2, unused=3, a_l_stim=True), + extended_stim_header=ExtendedStimHeader(dac_value=123, tbd=666), + ) + + event_output_1 = EventOutput( + original_event=raw_pha_event_1, + charge=9.0, + energies=[1], + total_energy=99, + detected_range=DetectedRange(DetectorRange.R2, DetectorSide.A), + e_delta=103.7, + e_prime=63.27, + ) + event_output_2 = EventOutput( + original_event=raw_pha_event_2, + charge=10.0, + energies=[4], + total_energy=100, + detected_range=None, + e_delta=None, + e_prime=None, + ) + event_output_3 = EventOutput( + original_event=raw_pha_event_3, + charge=12.0, + energies=[9, 8], + total_energy=200, + detected_range=DetectedRange(DetectorRange.R3, DetectorSide.A), + e_delta=106.7, + e_prime=69.27, + ) input_metadata = InputMetadata( instrument="hit", @@ -495,37 +881,70 @@ def test_process_direct_event_product(self, mock_read_all_events, mock_fetch_dep start_date=None, end_date=None, version="", - descriptor="direct-events" + descriptor="direct-events", ) mock_processing_input_collection = Mock() - mock_processing_input_collection.get_file_paths.return_value = [Path("test/parent_path")] + mock_processing_input_collection.get_file_paths.return_value = [ + Path("test/parent_path") + ] mock_hit_l3_pha_dependencies = mock_fetch_dependencies.return_value - mock_hit_l3_pha_dependencies.hit_l1_data.epoch = [datetime(year=2020, month=2, day=1), - datetime(year=2020, month=2, day=1, hour=1)] - mock_hit_l3_pha_dependencies.hit_l1_data.event_binary = [sentinel.binary_stream_1, sentinel.binary_stream_2] + mock_hit_l3_pha_dependencies.hit_l1_data.epoch = [ + datetime(year=2020, month=2, day=1), + datetime(year=2020, month=2, day=1, hour=1), + ] + mock_hit_l3_pha_dependencies.hit_l1_data.event_binary = [ + sentinel.binary_stream_1, + sentinel.binary_stream_2, + ] - mock_process_pha_event.side_effect = [event_output_1, event_output_2, event_output_3] - mock_read_all_events.side_effect = [[raw_pha_event_1, raw_pha_event_2], [raw_pha_event_3]] + mock_process_pha_event.side_effect = [ + event_output_1, + event_output_2, + event_output_3, + ] + mock_read_all_events.side_effect = [ + [raw_pha_event_1, raw_pha_event_2], + [raw_pha_event_3], + ] processor = HitProcessor(mock_processing_input_collection, input_metadata) product = processor.process() - mock_fetch_dependencies.assert_called_once_with(mock_processing_input_collection) - mock_read_all_events.assert_has_calls([call(sentinel.binary_stream_1), call(sentinel.binary_stream_2)]) + mock_fetch_dependencies.assert_called_once_with( + mock_processing_input_collection + ) + mock_read_all_events.assert_has_calls( + [call(sentinel.binary_stream_1), call(sentinel.binary_stream_2)] + ) mock_process_pha_event.assert_has_calls( - [call(raw_pha_event_1, mock_hit_l3_pha_dependencies.cosine_correction_lookup, - mock_hit_l3_pha_dependencies.gain_lookup, mock_hit_l3_pha_dependencies.range_fit_lookup, - mock_hit_l3_pha_dependencies.event_type_lookup), - call(raw_pha_event_2, mock_hit_l3_pha_dependencies.cosine_correction_lookup, - mock_hit_l3_pha_dependencies.gain_lookup, mock_hit_l3_pha_dependencies.range_fit_lookup, - mock_hit_l3_pha_dependencies.event_type_lookup), - call(raw_pha_event_3, mock_hit_l3_pha_dependencies.cosine_correction_lookup, - mock_hit_l3_pha_dependencies.gain_lookup, mock_hit_l3_pha_dependencies.range_fit_lookup, - mock_hit_l3_pha_dependencies.event_type_lookup)], - any_order=False) + [ + call( + raw_pha_event_1, + mock_hit_l3_pha_dependencies.cosine_correction_lookup, + mock_hit_l3_pha_dependencies.gain_lookup, + mock_hit_l3_pha_dependencies.range_fit_lookup, + mock_hit_l3_pha_dependencies.event_type_lookup, + ), + call( + raw_pha_event_2, + mock_hit_l3_pha_dependencies.cosine_correction_lookup, + mock_hit_l3_pha_dependencies.gain_lookup, + mock_hit_l3_pha_dependencies.range_fit_lookup, + mock_hit_l3_pha_dependencies.event_type_lookup, + ), + call( + raw_pha_event_3, + mock_hit_l3_pha_dependencies.cosine_correction_lookup, + mock_hit_l3_pha_dependencies.gain_lookup, + mock_hit_l3_pha_dependencies.range_fit_lookup, + mock_hit_l3_pha_dependencies.event_type_lookup, + ), + ], + any_order=False, + ) mock_save_data.assert_called_once() @@ -535,26 +954,61 @@ def test_process_direct_event_product(self, mock_read_all_events, mock_fetch_dep self.assertEqual(input_metadata, direct_event_product.input_metadata) self.assertEqual(["parent_path"], direct_event_product.parent_file_names) - np.testing.assert_array_equal(direct_event_product.epoch, np.array( - [datetime(year=2020, month=2, day=1), datetime(year=2020, month=2, day=1), - datetime(year=2020, month=2, day=1, hour=1)])) - np.testing.assert_array_equal(direct_event_product.charge, np.array([9.0, 10.0, 12.0])) - np.testing.assert_array_equal(direct_event_product.energy, np.array([99, 100, 200])) - np.testing.assert_array_equal(direct_event_product.particle_id, np.array([1, 1, 2])) + np.testing.assert_array_equal( + direct_event_product.epoch, + np.array( + [ + datetime(year=2020, month=2, day=1), + datetime(year=2020, month=2, day=1), + datetime(year=2020, month=2, day=1, hour=1), + ] + ), + ) + np.testing.assert_array_equal( + direct_event_product.charge, np.array([9.0, 10.0, 12.0]) + ) + np.testing.assert_array_equal( + direct_event_product.energy, np.array([99, 100, 200]) + ) + np.testing.assert_array_equal( + direct_event_product.particle_id, np.array([1, 1, 2]) + ) - np.testing.assert_array_equal(direct_event_product.e_delta, np.array([103.7, np.nan, 106.7])) - np.testing.assert_array_equal(direct_event_product.e_prime, np.array([63.27, np.nan, 69.27])) + np.testing.assert_array_equal( + direct_event_product.e_delta, np.array([103.7, np.nan, 106.7]) + ) + np.testing.assert_array_equal( + direct_event_product.e_prime, np.array([63.27, np.nan, 69.27]) + ) - np.testing.assert_array_equal(direct_event_product.detected_range, np.array([2, np.nan, 3])) + np.testing.assert_array_equal( + direct_event_product.detected_range, np.array([2, np.nan, 3]) + ) - np.testing.assert_array_equal(direct_event_product.priority_buffer_number, np.array([2, 2, 3])) - np.testing.assert_array_equal(direct_event_product.latency, np.array([20, 20, 30])) - np.testing.assert_array_equal(direct_event_product.stim_tag, np.array([False, False, True])) - np.testing.assert_array_equal(direct_event_product.long_event_flag, np.array([False, False, True])) - np.testing.assert_array_equal(direct_event_product.haz_tag, np.array([False, False, True])) - np.testing.assert_array_equal(direct_event_product.a_b_side, np.array([0, np.nan, 0])) - np.testing.assert_array_equal(direct_event_product.has_unread_adcs, np.array([True, True, False])) - np.testing.assert_array_equal(direct_event_product.culling_flag, np.array([True, True, False])) + np.testing.assert_array_equal( + direct_event_product.priority_buffer_number, np.array([2, 2, 3]) + ) + np.testing.assert_array_equal( + direct_event_product.latency, np.array([20, 20, 30]) + ) + np.testing.assert_array_equal( + direct_event_product.stim_tag, np.array([False, False, True]) + ) + np.testing.assert_array_equal( + direct_event_product.long_event_flag, np.array([False, False, True]) + ) + np.testing.assert_array_equal( + direct_event_product.haz_tag, np.array([False, False, True]) + ) + np.testing.assert_array_equal( + direct_event_product.a_b_side, np.array([0, np.nan, 0]) + ) + np.testing.assert_array_equal( + direct_event_product.has_unread_adcs, np.array([True, True, False]) + ) + np.testing.assert_array_equal( + direct_event_product.culling_flag, np.array([True, True, False]) + ) self.assertEqual((3, 64), direct_event_product.pha_value.shape) expected_values = { @@ -587,10 +1041,16 @@ def test_process_direct_event_product(self, mock_read_all_events, mock_fetch_dep adc_val, err_msg=f"Did not match at index {idx}", ) - np.testing.assert_array_equal(direct_event_product.energy_at_detector[idx], energy, - err_msg=f"Did not match at index {idx}") - np.testing.assert_array_equal(direct_event_product.is_low_gain[idx], is_low_gain, - err_msg=f"Did not match at index {idx}") + np.testing.assert_array_equal( + direct_event_product.energy_at_detector[idx], + energy, + err_msg=f"Did not match at index {idx}", + ) + np.testing.assert_array_equal( + direct_event_product.is_low_gain[idx], + is_low_gain, + err_msg=f"Did not match at index {idx}", + ) mask[idx] = False self.assertTrue( @@ -603,8 +1063,10 @@ def test_process_direct_event_product(self, mock_read_all_events, mock_fetch_dep direct_event_product.detector_flags, np.array([UNSIGNED_INT2_FILL_VALUE, UNSIGNED_INT2_FILL_VALUE, 2]), ) - np.testing.assert_array_equal(direct_event_product.deindex, - np.array([UNSIGNED_INT2_FILL_VALUE, UNSIGNED_INT2_FILL_VALUE, 2])) + np.testing.assert_array_equal( + direct_event_product.deindex, + np.array([UNSIGNED_INT2_FILL_VALUE, UNSIGNED_INT2_FILL_VALUE, 2]), + ) np.testing.assert_array_equal( direct_event_product.epindex, np.array([UNSIGNED_INT2_FILL_VALUE, UNSIGNED_INT2_FILL_VALUE, True]), @@ -612,19 +1074,37 @@ def test_process_direct_event_product(self, mock_read_all_events, mock_fetch_dep np.testing.assert_array_equal( direct_event_product.stim_gain, np.array([False, False, True]) ) - np.testing.assert_array_equal(direct_event_product.a_l_stim, np.array([False, False, True])) - np.testing.assert_array_equal(direct_event_product.stim_step, - np.array([UNSIGNED_INT1_FILL_VALUE, UNSIGNED_INT1_FILL_VALUE, 1])) - np.testing.assert_array_equal(direct_event_product.dac_value, - np.array([UNSIGNED_INT2_FILL_VALUE, UNSIGNED_INT2_FILL_VALUE, 123])) - np.testing.assert_array_equal(direct_event_product.hit_flags, np.zeros_like(direct_event_product.epoch, dtype=np.uint16), strict=True) + np.testing.assert_array_equal( + direct_event_product.a_l_stim, np.array([False, False, True]) + ) + np.testing.assert_array_equal( + direct_event_product.stim_step, + np.array([UNSIGNED_INT1_FILL_VALUE, UNSIGNED_INT1_FILL_VALUE, 1]), + ) + np.testing.assert_array_equal( + direct_event_product.dac_value, + np.array([UNSIGNED_INT2_FILL_VALUE, UNSIGNED_INT2_FILL_VALUE, 123]), + ) + np.testing.assert_array_equal( + direct_event_product.hit_flags, + np.zeros_like(direct_event_product.epoch, dtype=np.uint16), + strict=True, + ) @patch("imap_l3_processing.hit.l3.hit_processor.PHAEventReader.read_all_pha_events") @patch("imap_l3_processing.hit.l3.hit_processor.process_pha_event", autospec=True) - def test_direct_event_parsing_errors_result_in_skipped_event(self, mock_process_pha_event, mock_read_all_events): - hit_l1_data = HitL1Data(epoch=np.array([datetime(year=2020, month=2, day=1), - datetime(year=2020, month=2, day=1, hour=1)]), - event_binary=np.array([sentinel.binary_stream_1, sentinel.binary_stream_2])) + def test_direct_event_parsing_errors_result_in_skipped_event( + self, mock_process_pha_event, mock_read_all_events + ): + hit_l1_data = HitL1Data( + epoch=np.array( + [ + datetime(year=2020, month=2, day=1), + datetime(year=2020, month=2, day=1, hour=1), + ] + ), + event_binary=np.array([sentinel.binary_stream_1, sentinel.binary_stream_2]), + ) mock_direct_event_dependencies = Mock() mock_direct_event_dependencies.hit_l1_data = hit_l1_data @@ -639,7 +1119,7 @@ def test_direct_event_parsing_errors_result_in_skipped_event(self, mock_process_ has_unread_adcs=False, culling_flag=False, spare=False, - pha_words=[] + pha_words=[], ) mock_read_all_events.side_effect = [Exception("error parsing"), [event]] @@ -650,15 +1130,19 @@ def test_direct_event_parsing_errors_result_in_skipped_event(self, mock_process_ start_date=None, end_date=None, version="", - descriptor="direct-events" + descriptor="direct-events", ) processor = HitProcessor(Mock(), input_metadata) with self.assertLogs(): - actual_product = processor.process_direct_event_product(mock_direct_event_dependencies) + actual_product = processor.process_direct_event_product( + mock_direct_event_dependencies + ) self.assertEqual(1, len(actual_product.epoch)) - self.assertEqual(datetime(year=2020, month=2, day=1, hour=1), actual_product.epoch[0]) + self.assertEqual( + datetime(year=2020, month=2, day=1, hour=1), actual_product.epoch[0] + ) def test_raise_error_if_descriptor_doesnt_match(self): input_metadata = InputMetadata( @@ -667,14 +1151,16 @@ def test_raise_error_if_descriptor_doesnt_match(self): start_date=None, end_date=None, version="", - descriptor="spectral-index" + descriptor="spectral-index", ) processor = HitProcessor(Mock(), input_metadata) with self.assertRaises(ValueError) as e: processor.process() - self.assertEqual(e.exception.args[0], - "Don't know how to generate 'spectral-index' /n Known HIT l3 data products: 'macropixel', 'direct-events'.") + self.assertEqual( + e.exception.args[0], + "Don't know how to generate 'spectral-index' /n Known HIT l3 data products: 'macropixel', 'direct-events'.", + ) T = TypeVar("T") diff --git a/tests/hit/l3/test_l3_sectored_dependencies.py b/tests/hit/l3/test_l3_sectored_dependencies.py index 6790e2932..4e7c43d4c 100644 --- a/tests/hit/l3/test_l3_sectored_dependencies.py +++ b/tests/hit/l3/test_l3_sectored_dependencies.py @@ -5,37 +5,50 @@ import imap_data_access from imap_data_access.processing_input import ScienceInput, ProcessingInputCollection -from imap_l3_processing.hit.l3.hit_l3_sectored_dependencies import HITL3SectoredDependencies, HIT_L2_DESCRIPTOR +from imap_l3_processing.hit.l3.hit_l3_sectored_dependencies import ( + HITL3SectoredDependencies, + HIT_L2_DESCRIPTOR, +) class TestHITL3SectoredDependencies(TestCase): @patch("imap_l3_processing.hit.l3.hit_l3_sectored_dependencies.read_mag_data") @patch("imap_l3_processing.hit.l3.hit_l3_sectored_dependencies.read_l2_hit_data") - @patch('imap_l3_processing.hit.l3.hit_l3_sectored_dependencies.imap_data_access.download') - def test_fetch_dependencies(self, mock_download, mock_read_hit_data, mock_read_mag_data): - hit_l2_data_dependency = ScienceInput(f"imap_hit_l2_{HIT_L2_DESCRIPTOR}_20240908_v001.cdf") + @patch( + "imap_l3_processing.hit.l3.hit_l3_sectored_dependencies.imap_data_access.download" + ) + def test_fetch_dependencies( + self, mock_download, mock_read_hit_data, mock_read_mag_data + ): + hit_l2_data_dependency = ScienceInput( + f"imap_hit_l2_{HIT_L2_DESCRIPTOR}_20240908_v001.cdf" + ) mag_data_dependency = ScienceInput("imap_mag_l1d_norm-dsrf_20240906_v001.cdf") - hit_data_dir = imap_data_access.config["DATA_DIR"] / 'imap' / 'hit' / 'l2' / '2024' / '09' - mag_data_dir = imap_data_access.config["DATA_DIR"] / 'imap' / 'mag' / 'l1d' / '2024' / '09' + hit_data_dir = ( + imap_data_access.config["DATA_DIR"] / "imap" / "hit" / "l2" / "2024" / "09" + ) + mag_data_dir = ( + imap_data_access.config["DATA_DIR"] / "imap" / "mag" / "l1d" / "2024" / "09" + ) - expected_file_paths = [hit_data_dir / f"imap_hit_l2_{HIT_L2_DESCRIPTOR}_20240908_v001.cdf", - mag_data_dir / "imap_mag_l1d_norm-dsrf_20240906_v001.cdf"] - processing_input_collection = ProcessingInputCollection(hit_l2_data_dependency, mag_data_dependency) + expected_file_paths = [ + hit_data_dir / f"imap_hit_l2_{HIT_L2_DESCRIPTOR}_20240908_v001.cdf", + mag_data_dir / "imap_mag_l1d_norm-dsrf_20240906_v001.cdf", + ] + processing_input_collection = ProcessingInputCollection( + hit_l2_data_dependency, mag_data_dependency + ) hit_data_path = Path("hit") mag_data_path = Path("mag") - mock_download.side_effect = [ - hit_data_path, - mag_data_path - ] + mock_download.side_effect = [hit_data_path, mag_data_path] deps = HITL3SectoredDependencies.fetch_dependencies(processing_input_collection) - mock_download.assert_has_calls([ - call(expected_file_paths[0]), - call(expected_file_paths[1]) - ]) + mock_download.assert_has_calls( + [call(expected_file_paths[0]), call(expected_file_paths[1])] + ) mock_read_hit_data.assert_called_with(hit_data_path) mock_read_mag_data.assert_called_with(mag_data_path) @@ -46,33 +59,42 @@ def test_fetch_dependencies(self, mock_download, mock_read_hit_data, mock_read_m @patch("imap_l3_processing.hit.l3.hit_l3_sectored_dependencies.read_mag_data") @patch("imap_l3_processing.hit.l3.hit_l3_sectored_dependencies.read_l2_hit_data") - @patch('imap_l3_processing.hit.l3.hit_l3_sectored_dependencies.imap_data_access.download') - def test_fetch_dependencies_uses_mag_l2_if_available(self, mock_download, mock_read_hit_data, mock_read_mag_data): - hit_l2_data_dependency = ScienceInput(f"imap_hit_l2_{HIT_L2_DESCRIPTOR}_20240908_v001.cdf") + @patch( + "imap_l3_processing.hit.l3.hit_l3_sectored_dependencies.imap_data_access.download" + ) + def test_fetch_dependencies_uses_mag_l2_if_available( + self, mock_download, mock_read_hit_data, mock_read_mag_data + ): + hit_l2_data_dependency = ScienceInput( + f"imap_hit_l2_{HIT_L2_DESCRIPTOR}_20240908_v001.cdf" + ) mag_data_dependency = ScienceInput("imap_mag_l1d_norm-dsrf_20240906_v001.cdf") mag_l2_data_dependency = ScienceInput("imap_mag_l2_norm-dsrf_20240906_v001.cdf") - hit_data_dir = imap_data_access.config["DATA_DIR"] / 'imap' / 'hit' / 'l2' / '2024' / '09' - mag_data_dir = imap_data_access.config["DATA_DIR"] / 'imap' / 'mag' / 'l2' / '2024' / '09' + hit_data_dir = ( + imap_data_access.config["DATA_DIR"] / "imap" / "hit" / "l2" / "2024" / "09" + ) + mag_data_dir = ( + imap_data_access.config["DATA_DIR"] / "imap" / "mag" / "l2" / "2024" / "09" + ) - expected_file_paths = [hit_data_dir / f"imap_hit_l2_{HIT_L2_DESCRIPTOR}_20240908_v001.cdf", - mag_data_dir / "imap_mag_l2_norm-dsrf_20240906_v001.cdf"] - processing_input_collection = ProcessingInputCollection(hit_l2_data_dependency, mag_data_dependency, - mag_l2_data_dependency) + expected_file_paths = [ + hit_data_dir / f"imap_hit_l2_{HIT_L2_DESCRIPTOR}_20240908_v001.cdf", + mag_data_dir / "imap_mag_l2_norm-dsrf_20240906_v001.cdf", + ] + processing_input_collection = ProcessingInputCollection( + hit_l2_data_dependency, mag_data_dependency, mag_l2_data_dependency + ) hit_data_path = Path("hit") mag_data_path = Path("mag") - mock_download.side_effect = [ - hit_data_path, - mag_data_path - ] + mock_download.side_effect = [hit_data_path, mag_data_path] deps = HITL3SectoredDependencies.fetch_dependencies(processing_input_collection) - mock_download.assert_has_calls([ - call(expected_file_paths[0]), - call(expected_file_paths[1]) - ]) + mock_download.assert_has_calls( + [call(expected_file_paths[0]), call(expected_file_paths[1])] + ) mock_read_hit_data.assert_called_with(hit_data_path) mock_read_mag_data.assert_called_with(mag_data_path) diff --git a/tests/hit/l3/test_models.py b/tests/hit/l3/test_models.py index 9a1f54178..f226f3d28 100644 --- a/tests/hit/l3/test_models.py +++ b/tests/hit/l3/test_models.py @@ -13,12 +13,12 @@ class TestModels(CdfModelTestCase): def setUp(self) -> None: - if os.path.exists('test_cdf.cdf'): - os.remove('test_cdf.cdf') + if os.path.exists("test_cdf.cdf"): + os.remove("test_cdf.cdf") def tearDown(self) -> None: - if os.path.exists('test_cdf.cdf'): - os.remove('test_cdf.cdf') + if os.path.exists("test_cdf.cdf"): + os.remove("test_cdf.cdf") def test_direct_events_to_data_product_variables_with_multiple_events(self): input_metadata = InputMetadata( @@ -27,7 +27,7 @@ def test_direct_events_to_data_product_variables_with_multiple_events(self): start_date=datetime.min, end_date=datetime.max, version="1", - descriptor="direct-events" + descriptor="direct-events", ) epoch = [datetime.now(), datetime.now() + timedelta(hours=1)] @@ -55,35 +55,36 @@ def test_direct_events_to_data_product_variables_with_multiple_events(self): a_l_stim = np.array([None, True]) stim_step = np.array([None, 1]) dac_value = np.array([None, 123]) - hit_flag_values = np.array([0,0]) + hit_flag_values = np.array([0, 0]) - direct_event = HitDirectEventDataProduct(input_metadata=input_metadata, - epoch=epoch, - charge=charge, - energy=energy, - e_delta=e_delta, - e_prime=e_prime, - detected_range=detected_range, - particle_id=particle, - priority_buffer_number=priority_buffer_number, - latency=latency, - stim_tag=stim_tag, - long_event_flag=long_event_flag, - haz_tag=haz_tag, - a_b_side=a_b_side, - has_unread_adcs=has_unread_adcs, - culling_flag=culling_flag, - pha_value=pha_value, - energy_at_detector=energy_at_detector, - is_low_gain=is_low_gain, - detector_flags=detector_flags, - deindex=deindex, - epindex=epindex, - stim_gain=stim_gain, - a_l_stim=a_l_stim, - stim_step=stim_step, - dac_value=dac_value, - hit_flags=hit_flag_values, + direct_event = HitDirectEventDataProduct( + input_metadata=input_metadata, + epoch=epoch, + charge=charge, + energy=energy, + e_delta=e_delta, + e_prime=e_prime, + detected_range=detected_range, + particle_id=particle, + priority_buffer_number=priority_buffer_number, + latency=latency, + stim_tag=stim_tag, + long_event_flag=long_event_flag, + haz_tag=haz_tag, + a_b_side=a_b_side, + has_unread_adcs=has_unread_adcs, + culling_flag=culling_flag, + pha_value=pha_value, + energy_at_detector=energy_at_detector, + is_low_gain=is_low_gain, + detector_flags=detector_flags, + deindex=deindex, + epindex=epindex, + stim_gain=stim_gain, + a_l_stim=a_l_stim, + stim_step=stim_step, + dac_value=dac_value, + hit_flags=hit_flag_values, ) expected_variables = [ @@ -95,7 +96,9 @@ def test_direct_events_to_data_product_variables_with_multiple_events(self): DataProductVariable(models.E_PRIME_VAR_NAME, e_prime), DataProductVariable(models.DETECTED_RANGE_VAR_NAME, detected_range), DataProductVariable(models.PARTICLE_ID_VAR_NAME, particle), - DataProductVariable(models.PRIORITY_BUFFER_NUMBER_VAR_NAME, priority_buffer_number), + DataProductVariable( + models.PRIORITY_BUFFER_NUMBER_VAR_NAME, priority_buffer_number + ), DataProductVariable(models.LATENCY_VAR_NAME, latency), DataProductVariable(models.STIM_TAG_VAR_NAME, stim_tag), DataProductVariable(models.LONG_EVENT_FLAG_VAR_NAME, long_event_flag), @@ -114,26 +117,102 @@ def test_direct_events_to_data_product_variables_with_multiple_events(self): DataProductVariable(models.STIM_STEP_VAR_NAME, stim_step), DataProductVariable(models.DAC_VALUE_VAR_NAME, dac_value), DataProductVariable(models.HIT_FLAGS_CDF_VAR_NAME, hit_flag_values), - DataProductVariable(models.DETECTOR_ID_VAR_NAME, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, - 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, - 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, - 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, - 59, 60, 61, 62, 63]), + DataProductVariable( + models.DETECTOR_ID_VAR_NAME, + [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + ], + ), ] actual_variables = direct_event.to_data_product_variables() self.assertEqual(28, len(actual_variables)) - for expected_variable, actual_variable in zip(expected_variables, actual_variables): - self.assert_variable_attributes(actual_variable, expected_variable.value, expected_variable.name, - expected_variable.cdf_data_type, expected_variable.record_varying) + for expected_variable, actual_variable in zip( + expected_variables, actual_variables + ): + self.assert_variable_attributes( + actual_variable, + expected_variable.value, + expected_variable.name, + expected_variable.cdf_data_type, + expected_variable.record_varying, + ) def test_hit_l1_data_read_from_cdf(self): - expected_epochs = np.array([datetime(year=2020, month=1, day=1), datetime(year=2020, month=1, day=1, hour=2)]) + expected_epochs = np.array( + [ + datetime(year=2020, month=1, day=1), + datetime(year=2020, month=1, day=1, hour=2), + ] + ) expected_raw_binaries = ["10101011", "10101001010101"] - pathname = 'test_cdf' - with CDF(pathname, '') as cdf: + pathname = "test_cdf" + with CDF(pathname, "") as cdf: cdf["epoch"] = expected_epochs cdf["pha_raw"] = expected_raw_binaries @@ -142,4 +221,6 @@ def test_hit_l1_data_read_from_cdf(self): hit_l1_data = HitL1Data.read_from_cdf(file_path) np.testing.assert_array_equal(hit_l1_data.epoch, expected_epochs) - np.testing.assert_array_equal(expected_raw_binaries, hit_l1_data.event_binary) + np.testing.assert_array_equal( + expected_raw_binaries, hit_l1_data.event_binary + ) diff --git a/tests/hit/l3/test_utils.py b/tests/hit/l3/test_utils.py index 07c31e2a8..5705f92c9 100644 --- a/tests/hit/l3/test_utils.py +++ b/tests/hit/l3/test_utils.py @@ -17,14 +17,43 @@ def setUp(self) -> None: def test_read_l2_hit_data(self): rng = np.random.default_rng() - expected_azimuth_data = [12., 36., 60., 84., 108., 132., 156., 180., 204., 228., 252., - 276., 300., 324., 348.] - expected_zenith_data = [11.25, 33.75, 56.25, 78.75, 101.25, 123.75, 146.25, 168.75] + expected_azimuth_data = [ + 12.0, + 36.0, + 60.0, + 84.0, + 108.0, + 132.0, + 156.0, + 180.0, + 204.0, + 228.0, + 252.0, + 276.0, + 300.0, + 324.0, + 348.0, + ] + expected_zenith_data = [ + 11.25, + 33.75, + 56.25, + 78.75, + 101.25, + 123.75, + 146.25, + 168.75, + ] start_time = datetime(2010, 1, 1, 0, 5) epoch_data = np.array( - [start_time, start_time + timedelta(minutes=5), start_time + timedelta(minutes=10, microseconds=999)]) + [ + start_time, + start_time + timedelta(minutes=5), + start_time + timedelta(minutes=10, microseconds=999), + ] + ) hydrogen_data = rng.random((3, 3, 15, 8)) helium_data = rng.random((3, 2, 15, 8)) @@ -38,13 +67,19 @@ def test_read_l2_hit_data(self): nemgsi_delta = nemgsi_data * 0.1 iron_delta = iron_data * 0.1 - pathname = self.write_test_data_file(epoch_data, hydrogen_data, helium_data, cno_data, nemgsi_data, iron_data, - hydrogen_delta, - helium_delta, - cno_delta, - nemgsi_delta, - iron_delta, - ) + pathname = self.write_test_data_file( + epoch_data, + hydrogen_data, + helium_data, + cno_data, + nemgsi_data, + iron_data, + hydrogen_delta, + helium_delta, + cno_delta, + nemgsi_delta, + iron_delta, + ) for path in [pathname, Path(pathname)]: with self.subTest(path): @@ -57,8 +92,10 @@ def test_read_l2_hit_data(self): np.testing.assert_array_equal(iron_data, result.fe) np.testing.assert_array_equal(epoch_data, result.epoch) - np.testing.assert_array_equal([timedelta(minutes=5), timedelta(minutes=5), timedelta(minutes=5)], - result.epoch_delta) + np.testing.assert_array_equal( + [timedelta(minutes=5), timedelta(minutes=5), timedelta(minutes=5)], + result.epoch_delta, + ) np.testing.assert_array_equal([0, 1, 2], result.h_energy) np.testing.assert_array_equal([0, 1], result.he4_energy) @@ -104,42 +141,114 @@ def test_read_l2_hit_data_handles_fill_values(self): nemgsi_data = np.full((1, 2, 15, 8), fill_value=fill_val) iron_data = np.full((1, 1, 15, 8), fill_value=fill_val) - pathname = self.write_test_data_file(epoch_data, hydrogen_data, helium_data, cno_data, nemgsi_data, iron_data, - hydrogen_data, helium_data, cno_data, nemgsi_data, iron_data) + pathname = self.write_test_data_file( + epoch_data, + hydrogen_data, + helium_data, + cno_data, + nemgsi_data, + iron_data, + hydrogen_data, + helium_data, + cno_data, + nemgsi_data, + iron_data, + ) for path in [pathname, Path(pathname)]: with self.subTest(path): result = read_l2_hit_data(path) - np.testing.assert_array_equal(result.h, np.full_like(hydrogen_data, np.nan)) - np.testing.assert_array_equal(result.he4, np.full_like(helium_data, np.nan)) - np.testing.assert_array_equal(result.cno, np.full_like(cno_data, np.nan)) - np.testing.assert_array_equal(result.nemgsi, np.full_like(nemgsi_data, np.nan)) - np.testing.assert_array_equal(result.fe, np.full_like(iron_data, np.nan)) - - np.testing.assert_array_equal(result.delta_plus_h, np.full_like(hydrogen_data, np.nan)) - np.testing.assert_array_equal(result.delta_minus_h, np.full_like(hydrogen_data, np.nan)) - np.testing.assert_array_equal(result.delta_plus_he4, np.full_like(helium_data, np.nan)) - np.testing.assert_array_equal(result.delta_minus_he4, np.full_like(helium_data, np.nan)) - np.testing.assert_array_equal(result.delta_plus_cno, np.full_like(cno_data, np.nan)) - np.testing.assert_array_equal(result.delta_minus_cno, np.full_like(cno_data, np.nan)) - np.testing.assert_array_equal(result.delta_plus_nemgsi, np.full_like(nemgsi_data, np.nan)) - np.testing.assert_array_equal(result.delta_minus_nemgsi, np.full_like(nemgsi_data, np.nan)) - np.testing.assert_array_equal(result.delta_plus_fe, np.full_like(iron_data, np.nan)) - np.testing.assert_array_equal(result.delta_minus_fe, np.full_like(iron_data, np.nan)) - - def write_test_data_file(self, epoch_data, hydrogen_data, helium_data, cno_data, nemgsi_data, iron_data, - hydrogen_delta, - helium_delta, - cno_delta, - nemgsi_delta, - iron_delta, - ) -> str: - pathname = os.path.join(self.temp_dir.name, 'test_cdf.cdf') - expected_azimuth_data = [12., 36., 60., 84., 108., 132., 156., 180., 204., 228., 252., - 276., 300., 324., 348.] - expected_zenith_data = [11.25, 33.75, 56.25, 78.75, 101.25, 123.75, 146.25, 168.75] - with CDF(pathname, '') as cdf: + np.testing.assert_array_equal( + result.h, np.full_like(hydrogen_data, np.nan) + ) + np.testing.assert_array_equal( + result.he4, np.full_like(helium_data, np.nan) + ) + np.testing.assert_array_equal( + result.cno, np.full_like(cno_data, np.nan) + ) + np.testing.assert_array_equal( + result.nemgsi, np.full_like(nemgsi_data, np.nan) + ) + np.testing.assert_array_equal( + result.fe, np.full_like(iron_data, np.nan) + ) + + np.testing.assert_array_equal( + result.delta_plus_h, np.full_like(hydrogen_data, np.nan) + ) + np.testing.assert_array_equal( + result.delta_minus_h, np.full_like(hydrogen_data, np.nan) + ) + np.testing.assert_array_equal( + result.delta_plus_he4, np.full_like(helium_data, np.nan) + ) + np.testing.assert_array_equal( + result.delta_minus_he4, np.full_like(helium_data, np.nan) + ) + np.testing.assert_array_equal( + result.delta_plus_cno, np.full_like(cno_data, np.nan) + ) + np.testing.assert_array_equal( + result.delta_minus_cno, np.full_like(cno_data, np.nan) + ) + np.testing.assert_array_equal( + result.delta_plus_nemgsi, np.full_like(nemgsi_data, np.nan) + ) + np.testing.assert_array_equal( + result.delta_minus_nemgsi, np.full_like(nemgsi_data, np.nan) + ) + np.testing.assert_array_equal( + result.delta_plus_fe, np.full_like(iron_data, np.nan) + ) + np.testing.assert_array_equal( + result.delta_minus_fe, np.full_like(iron_data, np.nan) + ) + + def write_test_data_file( + self, + epoch_data, + hydrogen_data, + helium_data, + cno_data, + nemgsi_data, + iron_data, + hydrogen_delta, + helium_delta, + cno_delta, + nemgsi_delta, + iron_delta, + ) -> str: + pathname = os.path.join(self.temp_dir.name, "test_cdf.cdf") + expected_azimuth_data = [ + 12.0, + 36.0, + 60.0, + 84.0, + 108.0, + 132.0, + 156.0, + 180.0, + 204.0, + 228.0, + 252.0, + 276.0, + 300.0, + 324.0, + 348.0, + ] + expected_zenith_data = [ + 11.25, + 33.75, + 56.25, + 78.75, + 101.25, + 123.75, + 146.25, + 168.75, + ] + with CDF(pathname, "") as cdf: cdf.col_major(True) cdf["h_macropixel_intensity"] = hydrogen_data diff --git a/tests/integration/integration_test_helpers.py b/tests/integration/integration_test_helpers.py index 9312a9579..a69a974ef 100644 --- a/tests/integration/integration_test_helpers.py +++ b/tests/integration/integration_test_helpers.py @@ -25,7 +25,7 @@ class ImapQueryPatcher: def __init__(self, input_files: list[Path | str]): self.input_files = input_files - self.patcher = patch.object(imap_data_access, 'query', new=self) + self.patcher = patch.object(imap_data_access, "query", new=self) def start(self): self.patcher.start() @@ -57,9 +57,13 @@ def __call__(self, **kwargs): raise ValueError(f"Unexpected file type: {input_file}") query_results = create_mock_query_results(filtered_by_type) - filtered_query_results = [qr for qr in query_results if desired_attributes.issubset(set(qr.items()))] + filtered_query_results = [ + qr for qr in query_results if desired_attributes.issubset(set(qr.items())) + ] if use_latest: - filtered_query_results = self.filter_results_for_latest(filtered_query_results) + filtered_query_results = self.filter_results_for_latest( + filtered_query_results + ) return filtered_query_results @staticmethod @@ -67,7 +71,11 @@ def filter_results_for_latest(query_results: list[dict]): query_results_by_date = defaultdict(list) for qr in query_results: query_results_by_date[qr["start_date"]].append(qr) - return [max(qrs, key=get_version_from_query_result) for qrs in query_results_by_date.values()] + return [ + max(qrs, key=get_version_from_query_result) + for qrs in query_results_by_date.values() + ] + def fake_download(file: Path | str): filename = Path(file).name @@ -91,6 +99,7 @@ def stage_input_file(file_path: Path) -> Path: shutil.copy(file_path, destination) return destination + METAKERNEL_TEMPLATE = """ \\begindata @@ -100,20 +109,24 @@ def stage_input_file(file_path: Path) -> Path: """ -def create_metakernel(path_to_spice_dir: str, requested_spice_paths: list[SPICEFilePath]) -> str: +def create_metakernel( + path_to_spice_dir: str, requested_spice_paths: list[SPICEFilePath] +) -> str: all_spice_paths = [] for spice_file in requested_spice_paths: spice_subdir_name = spice_file.construct_path().parent.name spice_file_name = spice_file.construct_path().name spice_path = str(Path(path_to_spice_dir) / spice_subdir_name / spice_file_name) - chunked_spice_path = [spice_path[i:i + 79] for i in range(0, len(spice_path), 79)] + chunked_spice_path = [ + spice_path[i : i + 79] for i in range(0, len(spice_path), 79) + ] for i, chunk in enumerate(chunked_spice_path[:-1]): chunked_spice_path[i] = f"{chunk}+" all_spice_paths.extend(chunked_spice_path) - formatted_spice_file_names = ',\n'.join([f"'{spice}'" for spice in all_spice_paths]) + formatted_spice_file_names = ",\n".join([f"'{spice}'" for spice in all_spice_paths]) return METAKERNEL_TEMPLATE.format(kernels=formatted_spice_file_names) @@ -136,9 +149,9 @@ def __call__(self, url, **kwargs): parsed_url = urlparse(url) if "imap-mission" in url: if parsed_url.path == "/metakernel": - if 'params' in kwargs and kwargs['params'].get('list_files') == 'true': + if "params" in kwargs and kwargs["params"].get("list_files") == "true": response.text = json.dumps(self.spice_file_names) - elif 'params' in kwargs: + elif "params" in kwargs: prefix = kwargs["params"].get("spice_path") or "" spice_file_paths = [ @@ -164,7 +177,9 @@ def __call__(self, url, **kwargs): class mock_imap_data_access: - def __init__(self, data_dir: Path, input_files: list[Path], omni_file: Path | None = None): + def __init__( + self, data_dir: Path, input_files: list[Path], omni_file: Path | None = None + ): self.data_dir = data_dir valid_files = [] @@ -182,8 +197,12 @@ def __init__(self, data_dir: Path, input_files: list[Path], omni_file: Path | No self.input_files = valid_files self.env_patcher = patch.dict(os.environ, {"IMAP_DATA_DIR": str(self.data_dir)}) - self.data_dir_patcher = patch.dict(imap_data_access.config, {"DATA_DIR": self.data_dir}) - self.download_patcher = patch.object(imap_data_access, "download", new=fake_download) + self.data_dir_patcher = patch.dict( + imap_data_access.config, {"DATA_DIR": self.data_dir} + ) + self.download_patcher = patch.object( + imap_data_access, "download", new=fake_download + ) self.query_patcher = ImapQueryPatcher(self.input_files) self.requests_get_patcher = RequestsGetPatcher(spice_file_paths, omni_file) @@ -194,7 +213,8 @@ def __enter__(self): self.query_patcher.start() self.requests_get_patcher.start() - if self.data_dir.exists(): shutil.rmtree(self.data_dir) + if self.data_dir.exists(): + shutil.rmtree(self.data_dir) self.data_dir.mkdir(exist_ok=True, parents=True) for file_path in self.input_files: @@ -219,7 +239,14 @@ def wrapped(obj, *args, **kwargs): return wrapped + def run_istp_compliance_check(cdf_path: Path): - cdf_as_json = requests.post("https://skteditor.heliophysics.net/cgi-bin/cdf2json.cgi", files={"file": open(cdf_path, "rb")}).json() + cdf_as_json = requests.post( + "https://skteditor.heliophysics.net/cgi-bin/cdf2json.cgi", + files={"file": open(cdf_path, "rb")}, + ).json() cdf_json = list(cdf_as_json.values())[0] - return requests.post("https://skteditor.heliophysics.net/cgi-bin/validate.cgi", json={cdf_path.name: cdf_json}).text + return requests.post( + "https://skteditor.heliophysics.net/cgi-bin/validate.cgi", + json={cdf_path.name: cdf_json}, + ).text diff --git a/tests/integration/test_codice_processor_integration.py b/tests/integration/test_codice_processor_integration.py index 135493904..1933acd62 100644 --- a/tests/integration/test_codice_processor_integration.py +++ b/tests/integration/test_codice_processor_integration.py @@ -21,8 +21,13 @@ class CodiceProcessorIntegration(unittest.TestCase): @patch("imap_l3_data_processor._parse_cli_arguments") def test_codice_lo_direct_events(self, mock_parse_cli_arguments): - energy_per_charge_path = CODICE_TEST_DATA_DIR / "imap_codice_lo-energy-per-charge_20241110_v002.csv" - mass_coefficient_path = CODICE_TEST_DATA_DIR / "imap_codice_mass-coefficient-lookup_20241110_v003.csv" + energy_per_charge_path = ( + CODICE_TEST_DATA_DIR / "imap_codice_lo-energy-per-charge_20241110_v002.csv" + ) + mass_coefficient_path = ( + CODICE_TEST_DATA_DIR + / "imap_codice_mass-coefficient-lookup_20241110_v003.csv" + ) input_files = [ CODICE_TEST_DATA_DIR / "imap_codice_l2_lo-direct-events_20260307_v004.cdf", CODICE_TEST_DATA_DIR / "imap_codice_l1a_lo-nsw-priority_20260307_v004.cdf", @@ -35,10 +40,12 @@ def test_codice_lo_direct_events(self, mock_parse_cli_arguments): logging.basicConfig( force=True, level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", ) - processing_input_collection = ProcessingInputCollection(*(generate_imap_input(f.name) for f in input_files)) + processing_input_collection = ProcessingInputCollection( + *(generate_imap_input(f.name) for f in input_files) + ) mock_arguments = Mock() mock_arguments.instrument = "codice" @@ -56,8 +63,12 @@ def test_codice_lo_direct_events(self, mock_parse_cli_arguments): imap_l3_data_processor.imap_l3_processor() expected_output_path = ScienceFilePath( - "imap_codice_l3a_lo-direct-events_20260307_v001.cdf").construct_path() - self.assertTrue(expected_output_path.exists(), f"Expected file {expected_output_path.name} not found") + "imap_codice_l3a_lo-direct-events_20260307_v001.cdf" + ).construct_path() + self.assertTrue( + expected_output_path.exists(), + f"Expected file {expected_output_path.name} not found", + ) expected_parents = { "imap_codice_l2_lo-direct-events_20260307_v004.cdf", @@ -71,14 +82,19 @@ def test_codice_lo_direct_events(self, mock_parse_cli_arguments): self.assertEqual(expected_parents, set(cdf.attrs["Parents"])) @patch("imap_l3_data_processor._parse_cli_arguments") - def test_codice_lo_partial_densities_and_sw_products(self, mock_parse_cli_arguments): + def test_codice_lo_partial_densities_and_sw_products( + self, mock_parse_cli_arguments + ): input_files = [ INTEGRATION_DATA_DIR / "imap_codice_l2_lo-sw-species_20260301_v001.cdf", INTEGRATION_DATA_DIR / "imap_codice_mass-per-charge_20241110_v003.csv", - INTEGRATION_DATA_DIR / "imap_codice_l3a_lo-partial-densities-25ccf871_20260301_v001.json", - INTEGRATION_DATA_DIR / "imap_codice_l3a_lo-sw-ratios-25ccf871_20260301_v001.json", + INTEGRATION_DATA_DIR + / "imap_codice_l3a_lo-partial-densities-25ccf871_20260301_v001.json", + INTEGRATION_DATA_DIR + / "imap_codice_l3a_lo-sw-ratios-25ccf871_20260301_v001.json", ] with mock_imap_data_access(OUTPUT_DIR, input_files): + def run_processor(descriptor: str, dependency_file: str): mock_arguments = Mock() mock_arguments.instrument = "codice" @@ -127,10 +143,13 @@ def test_codice_lo_3d_distributions(self, mock_parse_cli_arguments): CODICE_TEST_DATA_DIR / "imap_codice_lo-energy-per-charge_20241110_v002.csv", CODICE_TEST_DATA_DIR / "imap_codice_l3-lo-efficiency_20251008_v003.csv", CODICE_TEST_DATA_DIR / "imap_codice_l2-lo-gfactor_20251212_v003.csv", - CODICE_TEST_DATA_DIR / "imap_codice_lo-mass-species-bin-lookup_20250309_v003.csv", + CODICE_TEST_DATA_DIR + / "imap_codice_lo-mass-species-bin-lookup_20250309_v003.csv", ] - processing_input = ProcessingInputCollection(*(generate_imap_input(f.name) for f in input_files)) + processing_input = ProcessingInputCollection( + *(generate_imap_input(f.name) for f in input_files) + ) dependency_json = processing_input.serialize() with mock_imap_data_access(OUTPUT_DIR, input_files): @@ -159,10 +178,14 @@ def test_codice_lo_3d_distributions(self, mock_parse_cli_arguments): @patch("imap_l3_data_processor._parse_cli_arguments") def test_codice_hi_direct_events(self, mock_parse_cli_arguments): input_files = [ - CODICE_TEST_DATA_DIR / "imap_codice_l1a_hi-direct-events_20260831_v001.0003.cdf", - CODICE_TEST_DATA_DIR / "imap_codice_l2_hi-direct-events_20260831_v001.0003.cdf", - CODICE_TEST_DATA_DIR / "imap_codice_l3-hi-mass-correction-lut-truncated_20251008_v001.xlsx", - INTEGRATION_DATA_DIR / "imap_codice_l3a_hi-direct-events-e968219e_20260831_v001.json", + CODICE_TEST_DATA_DIR + / "imap_codice_l1a_hi-direct-events_20260831_v001.0003.cdf", + CODICE_TEST_DATA_DIR + / "imap_codice_l2_hi-direct-events_20260831_v001.0003.cdf", + CODICE_TEST_DATA_DIR + / "imap_codice_l3-hi-mass-correction-lut-truncated_20251008_v001.xlsx", + INTEGRATION_DATA_DIR + / "imap_codice_l3a_hi-direct-events-e968219e_20260831_v001.json", ] with mock_imap_data_access(OUTPUT_DIR, input_files): mock_arguments = Mock() @@ -173,7 +196,9 @@ def test_codice_hi_direct_events(self, mock_parse_cli_arguments): mock_arguments.end_date = None mock_arguments.repointing = None mock_arguments.version = "v001" - mock_arguments.dependency = "imap_codice_l3a_hi-direct-events-e968219e_20260831_v001.json" + mock_arguments.dependency = ( + "imap_codice_l3a_hi-direct-events-e968219e_20260831_v001.json" + ) mock_arguments.upload_to_sdc = False mock_parse_cli_arguments.return_value = mock_arguments @@ -200,12 +225,16 @@ def test_codice_hi_pitch_angle(self, mock_parse_cli_arguments): mock_arguments.end_date = None mock_arguments.repointing = None mock_arguments.version = "v001" - processing_input = ProcessingInputCollection(*[generate_imap_input(f.name) for f in input_files]) + processing_input = ProcessingInputCollection( + *[generate_imap_input(f.name) for f in input_files] + ) mock_arguments.dependency = processing_input.serialize() mock_arguments.upload_to_sdc = False mock_parse_cli_arguments.return_value = mock_arguments imap_l3_data_processor.imap_l3_processor() - expected_output_path = ScienceFilePath("imap_codice_l3b_hi-pitch-angle_20260120_v001.cdf").construct_path() + expected_output_path = ScienceFilePath( + "imap_codice_l3b_hi-pitch-angle_20260120_v001.cdf" + ).construct_path() self.assertTrue(expected_output_path.exists()) diff --git a/tests/integration/test_glows_processor_integration.py b/tests/integration/test_glows_processor_integration.py index 5094b09af..8a621ea62 100644 --- a/tests/integration/test_glows_processor_integration.py +++ b/tests/integration/test_glows_processor_integration.py @@ -440,8 +440,12 @@ def test_l3bcde_first_time_processing(self): [unfiltered] = identified repointings_to_update = set(range(36, 116)) # Repoint 20 is not otherwise updated, so it is only there because the force reprocessing config lists it - self.assertEqual(repointings_to_update | {20}, set(unfiltered.repointing_numbers)) - self.assertEqual(repointings_to_update | {20}, set(unfiltered.ultra_sf_repointings)) + self.assertEqual( + repointings_to_update | {20}, set(unfiltered.repointing_numbers) + ) + self.assertEqual( + repointings_to_update | {20}, set(unfiltered.ultra_sf_repointings) + ) for versions in [ unfiltered.hi_90_repointings, unfiltered.hi_45_repointings, diff --git a/tests/integration/test_hit_processor_integration.py b/tests/integration/test_hit_processor_integration.py index 13cc99490..a6cf7948b 100644 --- a/tests/integration/test_hit_processor_integration.py +++ b/tests/integration/test_hit_processor_integration.py @@ -2,7 +2,11 @@ import unittest from unittest.mock import patch, Mock -from imap_data_access.processing_input import ProcessingInputCollection, generate_imap_input, ScienceFilePath +from imap_data_access.processing_input import ( + ProcessingInputCollection, + generate_imap_input, + ScienceFilePath, +) import imap_l3_data_processor from tests.integration.integration_test_helpers import mock_imap_data_access @@ -13,7 +17,9 @@ class HitProcessorIntegration(unittest.TestCase): @patch("imap_l3_data_processor._parse_cli_arguments") def test_hit_macropixel_product(self, mock_parse_cli_arguments): input_files = [ - get_test_data_path("hit/imap_hit_l2_macropixel-intensity_20260228_v001.0002.cdf"), + get_test_data_path( + "hit/imap_hit_l2_macropixel-intensity_20260228_v001.0002.cdf" + ), get_test_data_path("hit/imap_mag_l1d_norm-dsrf_20260228_v005.cdf"), get_test_data_path("hit/imap_mag_l2_norm-dsrf_20260228_v002.cdf"), ] @@ -24,10 +30,12 @@ def test_hit_macropixel_product(self, mock_parse_cli_arguments): logging.basicConfig( force=True, level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", ) - processing_input_collection = ProcessingInputCollection(*(generate_imap_input(f.name) for f in input_files)) + processing_input_collection = ProcessingInputCollection( + *(generate_imap_input(f.name) for f in input_files) + ) mock_arguments = Mock() mock_arguments.instrument = "hit" @@ -45,5 +53,9 @@ def test_hit_macropixel_product(self, mock_parse_cli_arguments): imap_l3_data_processor.imap_l3_processor() expected_output_path = ScienceFilePath( - "imap_hit_l3_macropixel_20260228_v001.cdf").construct_path() - self.assertTrue(expected_output_path.exists(), f"Expected file {expected_output_path.name} not found") + "imap_hit_l3_macropixel_20260228_v001.cdf" + ).construct_path() + self.assertTrue( + expected_output_path.exists(), + f"Expected file {expected_output_path.name} not found", + ) diff --git a/tests/integration/test_map_processor_integration.py b/tests/integration/test_map_processor_integration.py index 844382af3..983c93696 100644 --- a/tests/integration/test_map_processor_integration.py +++ b/tests/integration/test_map_processor_integration.py @@ -14,7 +14,11 @@ import imap_l3_data_processor from imap_l3_processing.hi.hi_combined_initializer import HI_COMBINED_DESCRIPTORS from tests.integration.integration_test_helpers import mock_imap_data_access -from tests.test_helpers import get_run_local_data_path, run_periodically, get_test_data_path +from tests.test_helpers import ( + get_run_local_data_path, + run_periodically, + get_test_data_path, +) INTEGRATION_TEST_DATA_PATH = Path(__file__).parent / "test_data" @@ -35,13 +39,21 @@ def test_lo_l3_multiple_arcs(self, mock_parse_cli_arguments): INTEGRATION_TEST_DATA_PATH / "spice" / "naif020.tls", INTEGRATION_TEST_DATA_PATH / "spice" / "imap_science_108.tf", INTEGRATION_TEST_DATA_PATH / "spice" / "imap_sclk_008.tsc", - INTEGRATION_TEST_DATA_PATH / "spice" / "imap_dps_2025_105_2026_105_009.ah.bc", + INTEGRATION_TEST_DATA_PATH + / "spice" + / "imap_dps_2025_105_2026_105_009.ah.bc", INTEGRATION_TEST_DATA_PATH / "spice" / "de440.bsp", - INTEGRATION_TEST_DATA_PATH / "spice" / "imap_recon_20250415_20260415_v01.bsp"] + INTEGRATION_TEST_DATA_PATH + / "spice" + / "imap_recon_20250415_20260415_v01.bsp", + ] with mock_imap_data_access(lo_imap_data_dir, input_files): - logging.basicConfig(force=True, level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + logging.basicConfig( + force=True, + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) mock_arguments = Mock() mock_arguments.instrument = "lo" @@ -58,12 +70,20 @@ def test_lo_l3_multiple_arcs(self, mock_parse_cli_arguments): imap_l3_data_processor.imap_l3_processor() expected_map_path = ScienceFilePath( - 'imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-1yr_20250415_v001.cdf').construct_path() - self.assertTrue(expected_map_path.exists(), f"Expected file {expected_map_path.name} not found") + "imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-1yr_20250415_v001.cdf" + ).construct_path() + self.assertTrue( + expected_map_path.exists(), + f"Expected file {expected_map_path.name} not found", + ) expected_map_path = ScienceFilePath( - 'imap_lo_l3_l090-ena-h-hf-sp-ram-hae-6deg-1yr_20250415_v001.cdf').construct_path() - self.assertTrue(expected_map_path.exists(), f"Expected file {expected_map_path.name} not found") + "imap_lo_l3_l090-ena-h-hf-sp-ram-hae-6deg-1yr_20250415_v001.cdf" + ).construct_path() + self.assertTrue( + expected_map_path.exists(), + f"Expected file {expected_map_path.name} not found", + ) @patch("imap_l3_data_processor._parse_cli_arguments") def test_hi_all_sp_maps_single_sensor(self, mock_parse_cli_arguments): @@ -71,38 +91,84 @@ def test_hi_all_sp_maps_single_sensor(self, mock_parse_cli_arguments): hi_imap_data_dir = get_run_local_data_path("hi/integration_data") input_files = [ - hi_test_data_dir / 'single-sensor' / "imap_hi_l2_h45-ena-h-sf-nsp-ram-hae-4deg-1yr_20250415_v001.cdf", - hi_test_data_dir / 'single-sensor' / "imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-6deg-1yr_20250415_v971.cdf", - hi_test_data_dir / 'single-sensor' / "imap_hi_l1c_90sensor-pset_20250415-repoint01000_v001.cdf", - hi_test_data_dir / 'single-sensor' / "imap_hi_l1c_45sensor-pset_20250415-repoint01000_v971.cdf", - hi_test_data_dir / 'single-sensor' / "imap_hi_l1c_45sensor-pset_20251015-repoint01183_v971.cdf", - hi_test_data_dir / 'single-sensor' / "imap_hi_l1c_90sensor-pset_20251015-repoint01183_v971.cdf", - hi_test_data_dir / 'single-sensor' / "imap_glows_l3e_survival-probability-hi-45_20250415-repoint01000_v001.cdf", - hi_test_data_dir / 'single-sensor' / "imap_glows_l3e_survival-probability-hi-45_20260418-repoint02000_v001.cdf", - hi_test_data_dir / 'single-sensor' / "imap_glows_l3e_survival-probability-hi-90_20250415-repoint01000_v001.cdf", - hi_test_data_dir / 'single-sensor' / "imap_glows_l3e_survival-probability-hi-90_20260418-repoint02000_v001.cdf", - hi_test_data_dir / 'single-sensor' / "imap_glows_l3e_survival-probability-hi-90_20251015-repoint01183_v001.cdf", - hi_test_data_dir / 'single-sensor' / "imap_glows_l3e_survival-probability-hi-45_20251015-repoint01183_v001.cdf", - hi_test_data_dir / 'single-sensor' / "imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-4deg-6mo_20250415_v000.cdf", - hi_test_data_dir / 'single-sensor' / "imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-4deg-6mo_20250415_v000.cdf", - hi_test_data_dir / 'single-sensor' / "imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-4deg-6mo_20251015_v000.cdf", - hi_test_data_dir / 'single-sensor' / "imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-4deg-6mo_20251015_v000.cdf", - hi_test_data_dir / 'single-sensor' / "imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-4deg-6mo_20250415_v000.cdf", - hi_test_data_dir / 'single-sensor' / "imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-4deg-6mo_20250415_v000.cdf", - hi_test_data_dir / 'single-sensor' / "imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-4deg-6mo_20251015_v000.cdf", - hi_test_data_dir / 'single-sensor' / "imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-4deg-6mo_20251015_v000.cdf", - + hi_test_data_dir + / "single-sensor" + / "imap_hi_l2_h45-ena-h-sf-nsp-ram-hae-4deg-1yr_20250415_v001.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-6deg-1yr_20250415_v971.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_hi_l1c_90sensor-pset_20250415-repoint01000_v001.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_hi_l1c_45sensor-pset_20250415-repoint01000_v971.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_hi_l1c_45sensor-pset_20251015-repoint01183_v971.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_hi_l1c_90sensor-pset_20251015-repoint01183_v971.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_glows_l3e_survival-probability-hi-45_20250415-repoint01000_v001.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_glows_l3e_survival-probability-hi-45_20260418-repoint02000_v001.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_glows_l3e_survival-probability-hi-90_20250415-repoint01000_v001.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_glows_l3e_survival-probability-hi-90_20260418-repoint02000_v001.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_glows_l3e_survival-probability-hi-90_20251015-repoint01183_v001.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_glows_l3e_survival-probability-hi-45_20251015-repoint01183_v001.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-4deg-6mo_20250415_v000.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-4deg-6mo_20250415_v000.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-4deg-6mo_20251015_v000.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-4deg-6mo_20251015_v000.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-4deg-6mo_20250415_v000.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-4deg-6mo_20250415_v000.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-4deg-6mo_20251015_v000.cdf", + hi_test_data_dir + / "single-sensor" + / "imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-4deg-6mo_20251015_v000.cdf", INTEGRATION_TEST_DATA_PATH / "spice" / "naif020.tls", INTEGRATION_TEST_DATA_PATH / "spice" / "imap_science_108.tf", INTEGRATION_TEST_DATA_PATH / "spice" / "imap_sclk_008.tsc", - INTEGRATION_TEST_DATA_PATH / "spice" / "imap_dps_2025_105_2026_105_009.ah.bc", + INTEGRATION_TEST_DATA_PATH + / "spice" + / "imap_dps_2025_105_2026_105_009.ah.bc", INTEGRATION_TEST_DATA_PATH / "spice" / "de440.bsp", - INTEGRATION_TEST_DATA_PATH / "spice" / "imap_recon_20250415_20260415_v01.bsp", + INTEGRATION_TEST_DATA_PATH + / "spice" + / "imap_recon_20250415_20260415_v01.bsp", ] with mock_imap_data_access(hi_imap_data_dir, input_files): - logging.basicConfig(force=True, level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + logging.basicConfig( + force=True, + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) mock_arguments = Mock() mock_arguments.instrument = "hi" @@ -120,8 +186,12 @@ def test_hi_all_sp_maps_single_sensor(self, mock_parse_cli_arguments): imap_l3_data_processor.imap_l3_processor() expected_map_path = ScienceFilePath( - 'imap_hi_l3_h45-ena-h-sf-sp-ram-hae-4deg-1yr_20250415_v001.cdf').construct_path() - self.assertTrue(expected_map_path.exists(), f"Expected file {expected_map_path.name} not found") + "imap_hi_l3_h45-ena-h-sf-sp-ram-hae-4deg-1yr_20250415_v001.cdf" + ).construct_path() + self.assertTrue( + expected_map_path.exists(), + f"Expected file {expected_map_path.name} not found", + ) expected_parents = { "imap_hi_l2_h45-ena-h-sf-nsp-ram-hae-4deg-1yr_20250415_v001.cdf", @@ -133,62 +203,86 @@ def test_hi_all_sp_maps_single_sensor(self, mock_parse_cli_arguments): self.assertEqual(expected_parents, set(cdf.attrs["Parents"])) expected_map_path = ScienceFilePath( - 'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250415_v001.cdf').construct_path() - self.assertTrue(expected_map_path.exists(), f"Expected file {expected_map_path.name} not found") + "imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250415_v001.cdf" + ).construct_path() + self.assertTrue( + expected_map_path.exists(), + f"Expected file {expected_map_path.name} not found", + ) expected_map_path = ScienceFilePath( - 'imap_hi_l3_h45-ena-h-hf-sp-full-hae-4deg-6mo_20250415_v001.cdf').construct_path() + "imap_hi_l3_h45-ena-h-hf-sp-full-hae-4deg-6mo_20250415_v001.cdf" + ).construct_path() - self.assertTrue(expected_map_path.exists(), f"Expected file {expected_map_path.name} not found") + self.assertTrue( + expected_map_path.exists(), + f"Expected file {expected_map_path.name} not found", + ) - expected_parents = {'imap_hi_l1c_45sensor-pset_20250415-repoint01000_v971.cdf', - 'imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-4deg-6mo_20250415_v000.cdf', - 'imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-4deg-6mo_20250415_v000.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20250415-repoint01000_v001.cdf', - } + expected_parents = { + "imap_hi_l1c_45sensor-pset_20250415-repoint01000_v971.cdf", + "imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-4deg-6mo_20250415_v000.cdf", + "imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-4deg-6mo_20250415_v000.cdf", + "imap_glows_l3e_survival-probability-hi-45_20250415-repoint01000_v001.cdf", + } with CDF(str(expected_map_path)) as cdf: self.assertEqual(expected_parents, set(cdf.attrs["Parents"])) expected_map_path = ScienceFilePath( - 'imap_hi_l3_h45-ena-h-hf-sp-full-hae-4deg-6mo_20251015_v001.cdf').construct_path() - self.assertTrue(expected_map_path.exists(), f"Expected file {expected_map_path.name} not found") + "imap_hi_l3_h45-ena-h-hf-sp-full-hae-4deg-6mo_20251015_v001.cdf" + ).construct_path() + self.assertTrue( + expected_map_path.exists(), + f"Expected file {expected_map_path.name} not found", + ) - expected_parents = {'imap_hi_l1c_45sensor-pset_20251015-repoint01183_v971.cdf', - 'imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-4deg-6mo_20251015_v000.cdf', - 'imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-4deg-6mo_20251015_v000.cdf', - 'imap_glows_l3e_survival-probability-hi-45_20251015-repoint01183_v001.cdf', - } + expected_parents = { + "imap_hi_l1c_45sensor-pset_20251015-repoint01183_v971.cdf", + "imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-4deg-6mo_20251015_v000.cdf", + "imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-4deg-6mo_20251015_v000.cdf", + "imap_glows_l3e_survival-probability-hi-45_20251015-repoint01183_v001.cdf", + } with CDF(str(expected_map_path)) as cdf: self.assertEqual(expected_parents, set(cdf.attrs["Parents"])) expected_map_path = ScienceFilePath( - 'imap_hi_l3_h90-ena-h-hf-sp-full-hae-4deg-6mo_20250415_v001.cdf').construct_path() - self.assertTrue(expected_map_path.exists(), f"Expected file {expected_map_path.name} not found") + "imap_hi_l3_h90-ena-h-hf-sp-full-hae-4deg-6mo_20250415_v001.cdf" + ).construct_path() + self.assertTrue( + expected_map_path.exists(), + f"Expected file {expected_map_path.name} not found", + ) - expected_parents = {'imap_hi_l1c_90sensor-pset_20250415-repoint01000_v001.cdf', - 'imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-4deg-6mo_20250415_v000.cdf', - 'imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-4deg-6mo_20250415_v000.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20250415-repoint01000_v001.cdf', - } + expected_parents = { + "imap_hi_l1c_90sensor-pset_20250415-repoint01000_v001.cdf", + "imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-4deg-6mo_20250415_v000.cdf", + "imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-4deg-6mo_20250415_v000.cdf", + "imap_glows_l3e_survival-probability-hi-90_20250415-repoint01000_v001.cdf", + } with CDF(str(expected_map_path)) as cdf: self.assertEqual(expected_parents, set(cdf.attrs["Parents"])) expected_map_path = ScienceFilePath( - 'imap_hi_l3_h90-ena-h-hf-sp-full-hae-4deg-6mo_20251015_v001.cdf').construct_path() - self.assertTrue(expected_map_path.exists(), f"Expected file {expected_map_path.name} not found") + "imap_hi_l3_h90-ena-h-hf-sp-full-hae-4deg-6mo_20251015_v001.cdf" + ).construct_path() + self.assertTrue( + expected_map_path.exists(), + f"Expected file {expected_map_path.name} not found", + ) - expected_parents = {'imap_hi_l1c_90sensor-pset_20251015-repoint01183_v971.cdf', - 'imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-4deg-6mo_20251015_v000.cdf', - 'imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-4deg-6mo_20251015_v000.cdf', - 'imap_glows_l3e_survival-probability-hi-90_20251015-repoint01183_v001.cdf', - } + expected_parents = { + "imap_hi_l1c_90sensor-pset_20251015-repoint01183_v971.cdf", + "imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-4deg-6mo_20251015_v000.cdf", + "imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-4deg-6mo_20251015_v000.cdf", + "imap_glows_l3e_survival-probability-hi-90_20251015-repoint01183_v001.cdf", + } with CDF(str(expected_map_path)) as cdf: self.assertEqual(expected_parents, set(cdf.attrs["Parents"])) - self.assertTrue(np.any(cdf['quality_flags'][...] > 0)) + self.assertTrue(np.any(cdf["quality_flags"][...] > 0)) @patch("imap_l3_data_processor._parse_cli_arguments") def test_hi_combined_sensor(self, mock_parse_cli_arguments): @@ -196,24 +290,29 @@ def test_hi_combined_sensor(self, mock_parse_cli_arguments): hi_imap_data_dir = get_run_local_data_path("hi/integration_data") sp_files = [ - 'imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250415_v001.cdf', - 'imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250415_v001.cdf', - 'imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250415_v001.cdf', - 'imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250415_v001.cdf', + "imap_hi_l3_h45-ena-h-hf-sp-ram-hae-6deg-1yr_20250415_v001.cdf", + "imap_hi_l3_h45-ena-h-hf-sp-anti-hae-6deg-1yr_20250415_v001.cdf", + "imap_hi_l3_h90-ena-h-hf-sp-ram-hae-6deg-1yr_20250415_v001.cdf", + "imap_hi_l3_h90-ena-h-hf-sp-anti-hae-6deg-1yr_20250415_v001.cdf", ] nsp_files = [ - 'imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-6deg-1yr_20250415_v000.cdf', - 'imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-6deg-1yr_20250415_v000.cdf', - 'imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-6deg-1yr_20250415_v000.cdf', - 'imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-6deg-1yr_20250415_v000.cdf', + "imap_hi_l2_h45-ena-h-hf-nsp-anti-hae-6deg-1yr_20250415_v000.cdf", + "imap_hi_l2_h45-ena-h-hf-nsp-ram-hae-6deg-1yr_20250415_v000.cdf", + "imap_hi_l2_h90-ena-h-hf-nsp-anti-hae-6deg-1yr_20250415_v000.cdf", + "imap_hi_l2_h90-ena-h-hf-nsp-ram-hae-6deg-1yr_20250415_v000.cdf", ] - input_files = [hi_test_data_dir / 'combined' / f for f in [*sp_files, *nsp_files]] + input_files = [ + hi_test_data_dir / "combined" / f for f in [*sp_files, *nsp_files] + ] with mock_imap_data_access(hi_imap_data_dir, input_files): - logging.basicConfig(force=True, level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + logging.basicConfig( + force=True, + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + ) mock_arguments = Mock() mock_arguments.instrument = "hi" @@ -224,10 +323,13 @@ def test_hi_combined_sensor(self, mock_parse_cli_arguments): mock_arguments.repointing = None mock_arguments.version = "v001" - version_descriptors = {descriptor: {"major_version": 4, "minor_version": 7} for descriptor in HI_COMBINED_DESCRIPTORS} + version_descriptors = { + descriptor: {"major_version": 4, "minor_version": 7} + for descriptor in HI_COMBINED_DESCRIPTORS + } dependency_with_version_info = { "version": version_descriptors, - "dependency": [] + "dependency": [], } mock_arguments.dependency = json.dumps(dependency_with_version_info) mock_arguments.upload_to_sdc = False @@ -237,15 +339,23 @@ def test_hi_combined_sensor(self, mock_parse_cli_arguments): imap_l3_data_processor.imap_l3_processor() expected_map_path = ScienceFilePath( - 'imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250415_v004.0001.cdf').construct_path() - self.assertTrue(expected_map_path.exists(), f"Expected file {expected_map_path.name} not found") + "imap_hi_l3_hic-ena-h-hf-sp-full-hae-6deg-1yr_20250415_v004.0001.cdf" + ).construct_path() + self.assertTrue( + expected_map_path.exists(), + f"Expected file {expected_map_path.name} not found", + ) with CDF(str(expected_map_path)) as cdf: self.assertEqual(set(sp_files), set(cdf.attrs["Parents"])) expected_map_path = ScienceFilePath( - 'imap_hi_l3_hic-ena-h-hf-nsp-full-hae-6deg-1yr_20250415_v004.0001.cdf').construct_path() - self.assertTrue(expected_map_path.exists(), f"Expected file {expected_map_path.name} not found") + "imap_hi_l3_hic-ena-h-hf-nsp-full-hae-6deg-1yr_20250415_v004.0001.cdf" + ).construct_path() + self.assertTrue( + expected_map_path.exists(), + f"Expected file {expected_map_path.name} not found", + ) with CDF(str(expected_map_path)) as cdf: self.assertEqual(set(nsp_files), set(cdf.attrs["Parents"])) @@ -281,7 +391,7 @@ def test_lo_all_sp_maps(self, mock_parse_cli_arguments): logging.basicConfig( force=True, level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", ) mock_arguments = Mock() @@ -300,11 +410,19 @@ def test_lo_all_sp_maps(self, mock_parse_cli_arguments): imap_l3_data_processor.imap_l3_processor() expected_ena_path = ScienceFilePath( - "imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-1yr_20260101_v001.cdf").construct_path() + "imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-1yr_20260101_v001.cdf" + ).construct_path() expected_hf_ena_path = ScienceFilePath( - "imap_lo_l3_l090-ena-h-hf-sp-ram-hae-6deg-1yr_20260101_v001.cdf").construct_path() - self.assertTrue(expected_ena_path.exists(), f"Expected file {expected_ena_path.name} not found") - self.assertTrue(expected_hf_ena_path.exists(), f"Expected file {expected_hf_ena_path.name} not found") + "imap_lo_l3_l090-ena-h-hf-sp-ram-hae-6deg-1yr_20260101_v001.cdf" + ).construct_path() + self.assertTrue( + expected_ena_path.exists(), + f"Expected file {expected_ena_path.name} not found", + ) + self.assertTrue( + expected_hf_ena_path.exists(), + f"Expected file {expected_hf_ena_path.name} not found", + ) expected_ena_parents = { "imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-1yr_20260101_v900.cdf", "imap_lo_l1c_pset_20260101-repoint01261_v001.cdf", @@ -329,17 +447,23 @@ def test_lo_isn_maps(self, mock_parse_cli_arguments): lo_test_data_dir = INTEGRATION_TEST_DATA_PATH / "lo" lo_imap_data_dir = get_run_local_data_path("lo/integration_data") - input_file = "imap_lo_l2_l090-enanbs-h-sf-nsp-ram-hae-6deg-1yr_20250415_v000.cdf" + input_file = ( + "imap_lo_l2_l090-enanbs-h-sf-nsp-ram-hae-6deg-1yr_20250415_v000.cdf" + ) descriptor = "l090-isn-h-sf-nsp-ram-hae-6deg-1yr" - expected_output_file = "imap_lo_l3_l090-isn-h-sf-nsp-ram-hae-6deg-1yr_20250415_v001.cdf" + expected_output_file = ( + "imap_lo_l3_l090-isn-h-sf-nsp-ram-hae-6deg-1yr_20250415_v001.cdf" + ) - processing_input_collection = ProcessingInputCollection(ScienceInput(input_file)) + processing_input_collection = ProcessingInputCollection( + ScienceInput(input_file) + ) - with (mock_imap_data_access(lo_imap_data_dir, [lo_test_data_dir / input_file])): + with mock_imap_data_access(lo_imap_data_dir, [lo_test_data_dir / input_file]): logging.basicConfig( force=True, level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", ) mock_arguments = Mock() @@ -357,8 +481,7 @@ def test_lo_isn_maps(self, mock_parse_cli_arguments): imap_l3_data_processor.imap_l3_processor() - expected_ena_path = ScienceFilePath( - expected_output_file).construct_path() + expected_ena_path = ScienceFilePath(expected_output_file).construct_path() self.assertTrue( expected_ena_path.exists(), @@ -380,24 +503,30 @@ def test_ultra_all_sp_maps(self, mock_parse_cli_arguments): ultra_imap_data_dir = get_run_local_data_path("ultra/integration_data") input_files = [ - ultra_test_data_dir / "imap_glows_l3e_survival-probability-ul-sf_20250415-repoint00000_v001.cdf", - ultra_test_data_dir / "imap_glows_l3e_survival-probability-ul-sf_20261020-repoint02000_v001.cdf", - ultra_test_data_dir / "imap_ultra_l1c_45sensor-spacecraftpset_20250416-repoint00000_v000.cdf", - ultra_test_data_dir / "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-6deg-3mo_20250416_v001.cdf", - + ultra_test_data_dir + / "imap_glows_l3e_survival-probability-ul-sf_20250415-repoint00000_v001.cdf", + ultra_test_data_dir + / "imap_glows_l3e_survival-probability-ul-sf_20261020-repoint02000_v001.cdf", + ultra_test_data_dir + / "imap_ultra_l1c_45sensor-spacecraftpset_20250416-repoint00000_v000.cdf", + ultra_test_data_dir + / "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-6deg-3mo_20250416_v001.cdf", INTEGRATION_TEST_DATA_PATH / "spice" / "naif020.tls", INTEGRATION_TEST_DATA_PATH / "spice" / "imap_science_108.tf", INTEGRATION_TEST_DATA_PATH / "spice" / "imap_sclk_008.tsc", - INTEGRATION_TEST_DATA_PATH / "spice" / "imap_dps_2025_105_2026_105_009.ah.bc", - - get_test_data_path("ultra/imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv"), + INTEGRATION_TEST_DATA_PATH + / "spice" + / "imap_dps_2025_105_2026_105_009.ah.bc", + get_test_data_path( + "ultra/imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv" + ), ] with mock_imap_data_access(ultra_imap_data_dir, input_files): logging.basicConfig( force=True, level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", ) mock_arguments = Mock() @@ -415,13 +544,17 @@ def test_ultra_all_sp_maps(self, mock_parse_cli_arguments): imap_l3_data_processor.imap_l3_processor() expected_map_path = ScienceFilePath( - "imap_ultra_l3_u45-ena-h-sf-sp-full-hae-6deg-3mo_20250416_v001.cdf").construct_path() - self.assertTrue(expected_map_path.exists(), f"Expected file {expected_map_path.name} not found") + "imap_ultra_l3_u45-ena-h-sf-sp-full-hae-6deg-3mo_20250416_v001.cdf" + ).construct_path() + self.assertTrue( + expected_map_path.exists(), + f"Expected file {expected_map_path.name} not found", + ) expected_parents = { "imap_glows_l3e_survival-probability-ul-sf_20250415-repoint00000_v001.cdf", "imap_ultra_l1c_45sensor-spacecraftpset_20250416-repoint00000_v000.cdf", - "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-6deg-3mo_20250416_v001.cdf" + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-6deg-3mo_20250416_v001.cdf", } with CDF(str(expected_map_path)) as cdf: @@ -438,26 +571,37 @@ def test_ultra_combined_sp_maps(self, mock_parse_cli_arguments): INTEGRATION_TEST_DATA_PATH / "spice" / "naif020.tls", INTEGRATION_TEST_DATA_PATH / "spice" / "imap_science_108.tf", INTEGRATION_TEST_DATA_PATH / "spice" / "imap_sclk_008.tsc", - INTEGRATION_TEST_DATA_PATH / "spice" / "imap_dps_2025_105_2026_105_009.ah.bc", - - get_test_data_path("ultra/imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv"), + INTEGRATION_TEST_DATA_PATH + / "spice" + / "imap_dps_2025_105_2026_105_009.ah.bc", + get_test_data_path( + "ultra/imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv" + ), ] input_files = [ - ultra_path / "imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-6deg-3mo_20250416_v001.cdf", - ultra_path / "imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-6deg-3mo_20250416_v001.cdf", - ultra_path / "imap_ultra_l1c_45sensor-heliopset_20250416-repoint00000_v000.cdf", - ultra_path / "imap_ultra_l1c_45sensor-heliopset_20251017-repoint00184_v000.cdf", - ultra_path / "imap_ultra_l1c_90sensor-heliopset_20250416-repoint00000_v000.cdf", - ultra_path / "imap_ultra_l1c_90sensor-heliopset_20251017-repoint00184_v000.cdf", - ultra_path / "imap_glows_l3e_survival-probability-ul-hf_20250415-repoint00000_v001.cdf", - ultra_path / "imap_glows_l3e_survival-probability-ul-hf_20261020-repoint02000_v001.cdf", + ultra_path + / "imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-6deg-3mo_20250416_v001.cdf", + ultra_path + / "imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-6deg-3mo_20250416_v001.cdf", + ultra_path + / "imap_ultra_l1c_45sensor-heliopset_20250416-repoint00000_v000.cdf", + ultra_path + / "imap_ultra_l1c_45sensor-heliopset_20251017-repoint00184_v000.cdf", + ultra_path + / "imap_ultra_l1c_90sensor-heliopset_20250416-repoint00000_v000.cdf", + ultra_path + / "imap_ultra_l1c_90sensor-heliopset_20251017-repoint00184_v000.cdf", + ultra_path + / "imap_glows_l3e_survival-probability-ul-hf_20250415-repoint00000_v001.cdf", + ultra_path + / "imap_glows_l3e_survival-probability-ul-hf_20261020-repoint02000_v001.cdf", ] with mock_imap_data_access(ultra_imap_data_dir, input_files + ancil_files): logging.basicConfig( force=True, level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", ) mock_arguments = Mock() @@ -475,8 +619,12 @@ def test_ultra_combined_sp_maps(self, mock_parse_cli_arguments): imap_l3_data_processor.imap_l3_processor() expected_sp_map_path = ScienceFilePath( - "imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-6deg-3mo_20250416_v001.cdf").construct_path() - self.assertTrue(expected_sp_map_path.exists(), f"Expected file {expected_sp_map_path.name} not found") + "imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-6deg-3mo_20250416_v001.cdf" + ).construct_path() + self.assertTrue( + expected_sp_map_path.exists(), + f"Expected file {expected_sp_map_path.name} not found", + ) expected_parents = { "imap_glows_l3e_survival-probability-ul-hf_20250415-repoint00000_v001.cdf", @@ -499,24 +647,33 @@ def test_ultra_combined_nsp_maps(self, mock_parse_cli_arguments): INTEGRATION_TEST_DATA_PATH / "spice" / "naif020.tls", INTEGRATION_TEST_DATA_PATH / "spice" / "imap_science_108.tf", INTEGRATION_TEST_DATA_PATH / "spice" / "imap_sclk_008.tsc", - INTEGRATION_TEST_DATA_PATH / "spice" / "imap_dps_2025_105_2026_105_009.ah.bc", - - get_test_data_path("ultra/imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv"), + INTEGRATION_TEST_DATA_PATH + / "spice" + / "imap_dps_2025_105_2026_105_009.ah.bc", + get_test_data_path( + "ultra/imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv" + ), ] input_files = [ - ultra_path / "imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-6deg-3mo_20250416_v001.cdf", - ultra_path / "imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-6deg-3mo_20250416_v001.cdf", - ultra_path / "imap_ultra_l1c_45sensor-heliopset_20250416-repoint00000_v000.cdf", - ultra_path / "imap_ultra_l1c_45sensor-heliopset_20251017-repoint00184_v000.cdf", - ultra_path / "imap_ultra_l1c_90sensor-heliopset_20250416-repoint00000_v000.cdf", - ultra_path / "imap_ultra_l1c_90sensor-heliopset_20251017-repoint00184_v000.cdf", + ultra_path + / "imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-6deg-3mo_20250416_v001.cdf", + ultra_path + / "imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-6deg-3mo_20250416_v001.cdf", + ultra_path + / "imap_ultra_l1c_45sensor-heliopset_20250416-repoint00000_v000.cdf", + ultra_path + / "imap_ultra_l1c_45sensor-heliopset_20251017-repoint00184_v000.cdf", + ultra_path + / "imap_ultra_l1c_90sensor-heliopset_20250416-repoint00000_v000.cdf", + ultra_path + / "imap_ultra_l1c_90sensor-heliopset_20251017-repoint00184_v000.cdf", ] with mock_imap_data_access(ultra_imap_data_dir, input_files + ancil_files): logging.basicConfig( force=True, level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", ) mock_arguments = Mock() @@ -534,8 +691,12 @@ def test_ultra_combined_nsp_maps(self, mock_parse_cli_arguments): imap_l3_data_processor.imap_l3_processor() expected_nsp_map_path = ScienceFilePath( - "imap_ultra_l3_ulc-ena-h-hf-nsp-full-hae-6deg-3mo_20250416_v001.cdf").construct_path() - self.assertTrue(expected_nsp_map_path.exists(), f"Expected file {expected_nsp_map_path.name} not found") + "imap_ultra_l3_ulc-ena-h-hf-nsp-full-hae-6deg-3mo_20250416_v001.cdf" + ).construct_path() + self.assertTrue( + expected_nsp_map_path.exists(), + f"Expected file {expected_nsp_map_path.name} not found", + ) expected_parents = { "imap_ultra_l1c_45sensor-heliopset_20250416-repoint00000_v000.cdf", diff --git a/tests/integration/test_swe_processor_integration.py b/tests/integration/test_swe_processor_integration.py index f6268160f..7f0f3f6f9 100644 --- a/tests/integration/test_swe_processor_integration.py +++ b/tests/integration/test_swe_processor_integration.py @@ -11,8 +11,12 @@ import imap_l3_processing from tests.integration.integration_test_helpers import mock_imap_data_access -from tests.test_helpers import get_run_local_data_path, get_test_data_path, get_integration_test_data_path, \ - run_periodically +from tests.test_helpers import ( + get_run_local_data_path, + get_test_data_path, + get_integration_test_data_path, + run_periodically, +) SWE_OUTPUT_DATA_DIR = get_run_local_data_path("swe_integration") @@ -39,7 +43,7 @@ def test_swe_processor_with_local_data(self): spice_test_data_dir / "imap_sclk_0171.tsc", spice_test_data_dir / "de440.bsp", spice_test_data_dir / "imap_recon_20250925_20260511_v01.bsp", - spice_test_data_dir / "imap_dps_2025_359_2026_131_002.ah.bc" + spice_test_data_dir / "imap_dps_2025_359_2026_131_002.ah.bc", ] with mock_imap_data_access(SWE_OUTPUT_DATA_DIR, input_files): @@ -62,8 +66,9 @@ def test_swe_processor_with_local_data(self): ], ) - expected_file_path = ScienceFilePath("imap_swe_l3_sci_20260120_v001.cdf").construct_path() + expected_file_path = ScienceFilePath( + "imap_swe_l3_sci_20260120_v001.cdf" + ).construct_path() self.assertEqual(0, result.returncode) self.assertTrue(expected_file_path.exists()) - diff --git a/tests/lo/l3/test_lo_combined_dependencies.py b/tests/lo/l3/test_lo_combined_dependencies.py index 7ffe25b7c..07c285137 100644 --- a/tests/lo/l3/test_lo_combined_dependencies.py +++ b/tests/lo/l3/test_lo_combined_dependencies.py @@ -7,29 +7,30 @@ class TestLoCombinedDependencies(unittest.TestCase): - - @patch("imap_l3_processing.lo.l3.lo_combined_dependencies.RectangularIntensityMapData.read_from_path") - @patch("imap_l3_processing.lo.l3.lo_combined_dependencies.imap_data_access.download") + @patch( + "imap_l3_processing.lo.l3.lo_combined_dependencies.RectangularIntensityMapData.read_from_path" + ) + @patch( + "imap_l3_processing.lo.l3.lo_combined_dependencies.imap_data_access.download" + ) def test_fetch_dependencies(self, mock_download, mock_read_from_path): - lo75_input = 'imap_lo_l3_l075-ena-h-hf-sp-ram-hae-6deg-6mo_20250422_v001.cdf' - lo90_input = 'imap_lo_l3_l090-ena-h-hf-sp-ram-hae-6deg-6mo_20250422_v001.cdf' - lo105_input = 'imap_lo_l3_l105-ena-h-hf-sp-ram-hae-6deg-6mo_20250422_v001.cdf' + lo75_input = "imap_lo_l3_l075-ena-h-hf-sp-ram-hae-6deg-6mo_20250422_v001.cdf" + lo90_input = "imap_lo_l3_l090-ena-h-hf-sp-ram-hae-6deg-6mo_20250422_v001.cdf" + lo105_input = "imap_lo_l3_l105-ena-h-hf-sp-ram-hae-6deg-6mo_20250422_v001.cdf" mock_lo_map_data = [sentinel.lo75, sentinel.lo90, sentinel.lo105] mock_read_from_path.side_effect = mock_lo_map_data - inputs = ProcessingInputCollection(ScienceInput(lo75_input), ScienceInput(lo90_input), ScienceInput(lo105_input)) + inputs = ProcessingInputCollection( + ScienceInput(lo75_input), + ScienceInput(lo90_input), + ScienceInput(lo105_input), + ) dependencies = LoCombinedDependencies.fetch_dependencies(inputs) mock_download.assert_has_calls( - [ - call(lo75_input), - call(lo90_input), - call(lo105_input) - ] + [call(lo75_input), call(lo90_input), call(lo105_input)] ) mock_read_from_path.assert_called_with(mock_download.return_value) self.assertEqual(mock_lo_map_data, dependencies.map_data) - - diff --git a/tests/lo/l3/test_lo_l3_isn_background_subtracted_dependencies.py b/tests/lo/l3/test_lo_l3_isn_background_subtracted_dependencies.py index 8cb40298c..ec6dd9b94 100644 --- a/tests/lo/l3/test_lo_l3_isn_background_subtracted_dependencies.py +++ b/tests/lo/l3/test_lo_l3_isn_background_subtracted_dependencies.py @@ -3,20 +3,28 @@ from imap_data_access.processing_input import ScienceInput, ProcessingInputCollection -from imap_l3_processing.lo.l3.lo_l3_isn_background_subtracted_dependencies import \ - LoL3ISNBackgroundSubtractedDependencies +from imap_l3_processing.lo.l3.lo_l3_isn_background_subtracted_dependencies import ( + LoL3ISNBackgroundSubtractedDependencies, +) class TestLoL3ISNBackgroundSubtractedDependencies(unittest.TestCase): - - @patch("imap_l3_processing.lo.l3.lo_l3_isn_background_subtracted_dependencies.ISNRateData.read_from_path") - @patch("imap_l3_processing.lo.l3.lo_l3_isn_background_subtracted_dependencies.download") + @patch( + "imap_l3_processing.lo.l3.lo_l3_isn_background_subtracted_dependencies.ISNRateData.read_from_path" + ) + @patch( + "imap_l3_processing.lo.l3.lo_l3_isn_background_subtracted_dependencies.download" + ) def test_fetch_dependencies(self, mock_download, mock_read_from_path): - file_name = "imap_lo_l2_l090-isnnbkgnd-h-sf-nsp-ram-hae-6deg-1yr_20250422_v001.cdf" + file_name = ( + "imap_lo_l2_l090-isnnbkgnd-h-sf-nsp-ram-hae-6deg-1yr_20250422_v001.cdf" + ) input = ScienceInput(file_name) processing_input_collection = ProcessingInputCollection(input) - dependencies = LoL3ISNBackgroundSubtractedDependencies.fetch_dependencies(processing_input_collection) + dependencies = LoL3ISNBackgroundSubtractedDependencies.fetch_dependencies( + processing_input_collection + ) mock_download.assert_called_once_with(file_name) diff --git a/tests/lo/l3/test_lo_l3_spectral_fit_dependencies.py b/tests/lo/l3/test_lo_l3_spectral_fit_dependencies.py index 4f2928b9f..bf1eea77a 100644 --- a/tests/lo/l3/test_lo_l3_spectral_fit_dependencies.py +++ b/tests/lo/l3/test_lo_l3_spectral_fit_dependencies.py @@ -3,12 +3,15 @@ from imap_data_access.processing_input import ScienceInput, ProcessingInputCollection -from imap_l3_processing.lo.l3.lo_l3_spectral_fit_dependencies import LoL3SpectralFitDependencies +from imap_l3_processing.lo.l3.lo_l3_spectral_fit_dependencies import ( + LoL3SpectralFitDependencies, +) class TestLoL3SpectralFitDependencies(unittest.TestCase): - - @patch("imap_l3_processing.lo.l3.lo_l3_spectral_fit_dependencies.RectangularIntensityMapData.read_from_path") + @patch( + "imap_l3_processing.lo.l3.lo_l3_spectral_fit_dependencies.RectangularIntensityMapData.read_from_path" + ) @patch("imap_l3_processing.lo.l3.lo_l3_spectral_fit_dependencies.download") def test_fetch_dependencies(self, mock_download, mock_read_from_path): for input_data_level in ("l2", "l3"): @@ -22,10 +25,14 @@ def test_fetch_dependencies(self, mock_download, mock_read_from_path): input_deps = ProcessingInputCollection(ena_input, *extra_inputs) - dependencies = LoL3SpectralFitDependencies.fetch_dependencies(input_deps) + dependencies = LoL3SpectralFitDependencies.fetch_dependencies( + input_deps + ) mock_download.assert_called_once_with(file_name) - self.assertEqual(dependencies.map_data, mock_read_from_path.return_value) + self.assertEqual( + dependencies.map_data, mock_read_from_path.return_value + ) mock_read_from_path.assert_called_with(mock_download.return_value) @@ -34,12 +41,25 @@ def test_raises_error_if_unexpected_number_of_files(self): l2_file_name = "imap_lo_l2_l090-ena-h-hf-nsp-ram-hae-4deg-6mo_20250422_v001.cdf" cases = [ ("0 files", ProcessingInputCollection()), - ("2 L3 files", ProcessingInputCollection( - ScienceInput(l3_file_name), - ScienceInput(l3_file_name), - )), - ("1 L2 and 1 L3 file", ProcessingInputCollection(ScienceInput(l2_file_name), ScienceInput(l3_file_name))), - ("2 L2 files", ProcessingInputCollection(ScienceInput(l2_file_name), ScienceInput(l2_file_name))) + ( + "2 L3 files", + ProcessingInputCollection( + ScienceInput(l3_file_name), + ScienceInput(l3_file_name), + ), + ), + ( + "1 L2 and 1 L3 file", + ProcessingInputCollection( + ScienceInput(l2_file_name), ScienceInput(l3_file_name) + ), + ), + ( + "2 L2 files", + ProcessingInputCollection( + ScienceInput(l2_file_name), ScienceInput(l2_file_name) + ), + ), ] for name, collection in cases: diff --git a/tests/lo/l3/test_lo_sp_initializer.py b/tests/lo/l3/test_lo_sp_initializer.py index 9a2668059..f2510cb92 100644 --- a/tests/lo/l3/test_lo_sp_initializer.py +++ b/tests/lo/l3/test_lo_sp_initializer.py @@ -4,7 +4,10 @@ from imap_data_access.file_validation import Version -from imap_l3_processing.lo.l3.lo_sp_initializer import LoSPInitializer, LO_SP_MAP_KERNELS +from imap_l3_processing.lo.l3.lo_sp_initializer import ( + LoSPInitializer, + LO_SP_MAP_KERNELS, +) from imap_l3_processing.maps.map_initializer import PossibleMapToProduce, MapInitializer from imap_l3_processing.models import InputMetadata, VersionMap from tests.integration.integration_test_helpers import ImapQueryPatcher @@ -12,7 +15,9 @@ class TestLoSPInitializer(unittest.TestCase): def setUp(self): - self.query_patcher = patch('imap_l3_processing.lo.l3.lo_sp_initializer.imap_data_access.query') + self.query_patcher = patch( + "imap_l3_processing.lo.l3.lo_sp_initializer.imap_data_access.query" + ) self.mock_query = self.query_patcher.start() def tearDown(self): @@ -22,91 +27,109 @@ def test_is_a_map_initializer(self): initializer = LoSPInitializer() self.assertIsInstance(initializer, MapInitializer) - @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') - def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): - self.mock_query.side_effect = ImapQueryPatcher([ - 'imap_glows_l3e_survival-probability-lo_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-lo_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-lo_20100103-repoint00003_v001.cdf', - - 'imap_glows_l3e_survival-probability-lo_20100401-repoint00101_v002.cdf', - 'imap_glows_l3e_survival-probability-lo_20100402-repoint00102_v002.cdf', - 'imap_glows_l3e_survival-probability-lo_20100403-repoint00103_v002.cdf', - - 'imap_glows_l3e_survival-probability-lo_20110410-repoint00201_v001.cdf', - - 'imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100101_v001.cdf', - 'imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100401_v001.cdf', - 'imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100401_v000.cdf', - - 'imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100701_v001.cdf', - 'imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20101001_v001.cdf', - - 'imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-12mo_20100101_v003.0001.cdf', - 'imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-12mo_20100401_v003.0001.cdf' - ]) + @patch("imap_l3_processing.maps.map_initializer.read_cdf_parents") + def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): + self.mock_query.side_effect = ImapQueryPatcher( + [ + "imap_glows_l3e_survival-probability-lo_20100101-repoint00001_v001.cdf", + "imap_glows_l3e_survival-probability-lo_20100102-repoint00002_v001.cdf", + "imap_glows_l3e_survival-probability-lo_20100103-repoint00003_v001.cdf", + "imap_glows_l3e_survival-probability-lo_20100401-repoint00101_v002.cdf", + "imap_glows_l3e_survival-probability-lo_20100402-repoint00102_v002.cdf", + "imap_glows_l3e_survival-probability-lo_20100403-repoint00103_v002.cdf", + "imap_glows_l3e_survival-probability-lo_20110410-repoint00201_v001.cdf", + "imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100101_v001.cdf", + "imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100401_v001.cdf", + "imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100401_v000.cdf", + "imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100701_v001.cdf", + "imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20101001_v001.cdf", + "imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-12mo_20100101_v003.0001.cdf", + "imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-12mo_20100401_v003.0001.cdf", + ] + ) mock_read_cdf_parents.side_effect = [ [ "imap_lo_l1c_pset_20100101-repoint00001_v001.cdf", "imap_lo_l1c_pset_20100102-repoint00002_v001.cdf", - "imap_lo_l1c_pset_20100103-repoint00003_v001.cdf" + "imap_lo_l1c_pset_20100103-repoint00003_v001.cdf", ], [ - 'imap_lo_l1c_pset_20100401-repoint00101_v001.cdf', - 'imap_lo_l1c_pset_20100402-repoint00102_v001.cdf', - 'imap_lo_l1c_pset_20100403-repoint00103_v001.cdf' + "imap_lo_l1c_pset_20100401-repoint00101_v001.cdf", + "imap_lo_l1c_pset_20100402-repoint00102_v001.cdf", + "imap_lo_l1c_pset_20100403-repoint00103_v001.cdf", ], [ - 'imap_lo_l1c_pset_20100701-repoint00201_v001.cdf', - 'imap_lo_l1c_pset_20100702-repoint00202_v001.cdf', - 'imap_lo_l1c_pset_20100703-repoint00203_v001.cdf' + "imap_lo_l1c_pset_20100701-repoint00201_v001.cdf", + "imap_lo_l1c_pset_20100702-repoint00202_v001.cdf", + "imap_lo_l1c_pset_20100703-repoint00203_v001.cdf", ], [ - 'imap_lo_l1c_pset_20101001-repoint00301_v001.cdf', - 'imap_lo_l1c_pset_20101002-repoint00302_v001.cdf', - 'imap_lo_l1c_pset_20101003-repoint00303_v001.cdf' + "imap_lo_l1c_pset_20101001-repoint00301_v001.cdf", + "imap_lo_l1c_pset_20101002-repoint00302_v001.cdf", + "imap_lo_l1c_pset_20101003-repoint00303_v001.cdf", ], [ - 'imap_glows_l3e_survival-probability-lo_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-lo_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-lo_20100103-repoint00003_v001.cdf', + "imap_glows_l3e_survival-probability-lo_20100101-repoint00001_v001.cdf", + "imap_glows_l3e_survival-probability-lo_20100102-repoint00002_v001.cdf", + "imap_glows_l3e_survival-probability-lo_20100103-repoint00003_v001.cdf", "imap_lo_l1c_pset_20100101-repoint00001_v001.cdf", "imap_lo_l1c_pset_20100102-repoint00002_v001.cdf", "imap_lo_l1c_pset_20100103-repoint00003_v001.cdf", - 'imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100101_v001.cdf' + "imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100101_v001.cdf", ], [ - 'imap_glows_l3e_survival-probability-lo_20100401-repoint00101_v001.cdf', - 'imap_glows_l3e_survival-probability-lo_20100402-repoint00102_v001.cdf', - 'imap_glows_l3e_survival-probability-lo_20100403-repoint00103_v001.cdf', - 'imap_lo_l1c_pset_20100401-repoint00101_v001.cdf', - 'imap_lo_l1c_pset_20100402-repoint00102_v001.cdf', - 'imap_lo_l1c_pset_20100403-repoint00103_v001.cdf', - 'imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100401_v001.cdf' - ] + "imap_glows_l3e_survival-probability-lo_20100401-repoint00101_v001.cdf", + "imap_glows_l3e_survival-probability-lo_20100402-repoint00102_v001.cdf", + "imap_glows_l3e_survival-probability-lo_20100403-repoint00103_v001.cdf", + "imap_lo_l1c_pset_20100401-repoint00101_v001.cdf", + "imap_lo_l1c_pset_20100402-repoint00102_v001.cdf", + "imap_lo_l1c_pset_20100403-repoint00103_v001.cdf", + "imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100401_v001.cdf", + ], ] initializer = LoSPInitializer() descriptor = "l090-ena-h-sf-sp-ram-hae-6deg-12mo" - actual_maps_to_produce = initializer.get_maps_that_should_be_produced(descriptor, - 3) - - self.mock_query.assert_has_calls([ - call(instrument='glows', data_level='l3e', descriptor='survival-probability-lo', version="latest"), - call(instrument='lo', data_level='l2'), - call(instrument='lo', data_level='l3'), - ]) + actual_maps_to_produce = initializer.get_maps_that_should_be_produced( + descriptor, 3 + ) - mock_read_cdf_parents.assert_has_calls([ - call('imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100101_v001.cdf'), - call('imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100401_v001.cdf'), - call('imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100701_v001.cdf'), - call('imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20101001_v001.cdf'), + self.mock_query.assert_has_calls( + [ + call( + instrument="glows", + data_level="l3e", + descriptor="survival-probability-lo", + version="latest", + ), + call(instrument="lo", data_level="l2"), + call(instrument="lo", data_level="l3"), + ] + ) - call('imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-12mo_20100101_v003.0001.cdf'), - call('imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-12mo_20100401_v003.0001.cdf') - ]) + mock_read_cdf_parents.assert_has_calls( + [ + call( + "imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100101_v001.cdf" + ), + call( + "imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100401_v001.cdf" + ), + call( + "imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100701_v001.cdf" + ), + call( + "imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20101001_v001.cdf" + ), + call( + "imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-12mo_20100101_v003.0001.cdf" + ), + call( + "imap_lo_l3_l090-ena-h-sf-sp-ram-hae-6deg-12mo_20100401_v003.0001.cdf" + ), + ] + ) expected_possible_map_to_produce = PossibleMapToProduce( input_metadata=InputMetadata( @@ -115,23 +138,23 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): start_date=datetime(2010, 4, 1), end_date=datetime(2011, 4, 1, 6, 0), version=VersionMap({descriptor: Version(3, 2)}), - descriptor=descriptor + descriptor=descriptor, ), input_files={ - 'imap_glows_l3e_survival-probability-lo_20100401-repoint00101_v002.cdf', - 'imap_glows_l3e_survival-probability-lo_20100402-repoint00102_v002.cdf', - 'imap_glows_l3e_survival-probability-lo_20100403-repoint00103_v002.cdf', - 'imap_lo_l1c_pset_20100401-repoint00101_v001.cdf', - 'imap_lo_l1c_pset_20100402-repoint00102_v001.cdf', - 'imap_lo_l1c_pset_20100403-repoint00103_v001.cdf', - 'imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100401_v001.cdf' - } + "imap_glows_l3e_survival-probability-lo_20100401-repoint00101_v002.cdf", + "imap_glows_l3e_survival-probability-lo_20100402-repoint00102_v002.cdf", + "imap_glows_l3e_survival-probability-lo_20100403-repoint00103_v002.cdf", + "imap_lo_l1c_pset_20100401-repoint00101_v001.cdf", + "imap_lo_l1c_pset_20100402-repoint00102_v001.cdf", + "imap_lo_l1c_pset_20100403-repoint00103_v001.cdf", + "imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-12mo_20100401_v001.cdf", + }, ) self.assertIn(expected_possible_map_to_produce, actual_maps_to_produce) self.assertEqual(2, len(actual_maps_to_produce)) - @patch('imap_l3_processing.lo.l3.lo_sp_initializer.furnish_spice_metakernel') + @patch("imap_l3_processing.lo.l3.lo_sp_initializer.furnish_spice_metakernel") def test_furnish_spice_dependencies(self, mock_furnish_metakernel): start_date = datetime(2025, 4, 15) end_date = datetime(2025, 7, 15) @@ -149,5 +172,6 @@ def test_furnish_spice_dependencies(self, mock_furnish_metakernel): lo_initializer = LoSPInitializer() lo_initializer.furnish_spice_dependencies(map_to_produce) - mock_furnish_metakernel.assert_called_once_with(start_date=start_date, end_date=end_date, - kernel_types=LO_SP_MAP_KERNELS) + mock_furnish_metakernel.assert_called_once_with( + start_date=start_date, end_date=end_date, kernel_types=LO_SP_MAP_KERNELS + ) diff --git a/tests/lo/test_lo_processor.py b/tests/lo/test_lo_processor.py index 73c00204f..29ffd10a3 100644 --- a/tests/lo/test_lo_processor.py +++ b/tests/lo/test_lo_processor.py @@ -9,64 +9,96 @@ from imap_l3_processing.lo.l3.lo_combined_dependencies import LoCombinedDependencies from imap_l3_processing.lo.lo_processor import LoProcessor, isn_background_subtraction -from imap_l3_processing.maps.map_models import RectangularSpectralIndexDataProduct, RectangularIntensityDataProduct, \ - InputRectangularPointingSet, ISNRateData, ISNBackgroundSubtractedData, ISNBackgroundSubtractedMapData +from imap_l3_processing.maps.map_models import ( + RectangularSpectralIndexDataProduct, + RectangularIntensityDataProduct, + InputRectangularPointingSet, + ISNRateData, + ISNBackgroundSubtractedData, + ISNBackgroundSubtractedMapData, +) from imap_l3_processing.models import InputMetadata, Instrument from tests.maps import test_builders class TestLoProcessor(unittest.TestCase): - - @patch('imap_l3_processing.hi.hi_processor.MapProcessor.get_parent_file_names') - @patch('imap_l3_processing.lo.lo_processor.slice_energy_range_by_bin') - @patch('imap_l3_processing.lo.lo_processor.LoL3SpectralFitDependencies.fetch_dependencies') - @patch('imap_l3_processing.lo.lo_processor.fit_spectral_index_map') - @patch('imap_l3_processing.lo.lo_processor.save_data') - def test_process_spectral_index(self, mock_save_data, mock_fit_spectral_index_map, mock_fetch_dependencies, - mock_slice_energy_range_by_bin, mock_get_parent_file_names): + @patch("imap_l3_processing.hi.hi_processor.MapProcessor.get_parent_file_names") + @patch("imap_l3_processing.lo.lo_processor.slice_energy_range_by_bin") + @patch( + "imap_l3_processing.lo.lo_processor.LoL3SpectralFitDependencies.fetch_dependencies" + ) + @patch("imap_l3_processing.lo.lo_processor.fit_spectral_index_map") + @patch("imap_l3_processing.lo.lo_processor.save_data") + def test_process_spectral_index( + self, + mock_save_data, + mock_fit_spectral_index_map, + mock_fetch_dependencies, + mock_slice_energy_range_by_bin, + mock_get_parent_file_names, + ): mock_get_parent_file_names.return_value = ["some_input_file_name"] input_collection = Mock() lo_l3_spectral_fit_dependency = Mock() - lo_l3_spectral_fit_dependency.map_data.intensity_map_data = test_builders.create_intensity_map_data() + lo_l3_spectral_fit_dependency.map_data.intensity_map_data = ( + test_builders.create_intensity_map_data() + ) lo_l3_spectral_fit_dependency.map_data.intensity_map_data.energy = np.array( - [1, 10, 100, 1000, 10000, 100000, 1000000, 10000000]) + [1, 10, 100, 1000, 10000, 100000, 1000000, 10000000] + ) mock_fetch_dependencies.return_value = lo_l3_spectral_fit_dependency - metadata = InputMetadata(instrument="lo", - data_level="l3", - version="v000", - start_date=datetime(2020, 1, 1, 1), - end_date=datetime(2020, 1, 1, 1), - descriptor="l090-spx-h-hf-sp-ram-hae-6deg-1yr") + metadata = InputMetadata( + instrument="lo", + data_level="l3", + version="v000", + start_date=datetime(2020, 1, 1, 1), + end_date=datetime(2020, 1, 1, 1), + descriptor="l090-spx-h-hf-sp-ram-hae-6deg-1yr", + ) processor = LoProcessor(input_collection, input_metadata=metadata) product = processor.process(spice_frame_name=SpiceFrame.ECLIPJ2000) mock_fetch_dependencies.assert_called_once_with(input_collection) mock_slice_energy_range_by_bin.assert_called_once_with( - lo_l3_spectral_fit_dependency.map_data.intensity_map_data, 4, 7) - mock_fit_spectral_index_map.assert_called_once_with(mock_slice_energy_range_by_bin.return_value) + lo_l3_spectral_fit_dependency.map_data.intensity_map_data, 4, 7 + ) + mock_fit_spectral_index_map.assert_called_once_with( + mock_slice_energy_range_by_bin.return_value + ) data_product = mock_save_data.call_args_list[0].args[0] self.assertIsInstance(data_product, RectangularSpectralIndexDataProduct) - self.assertEqual(data_product.data.spectral_index_map_data, - mock_fit_spectral_index_map.return_value) - self.assertEqual(data_product.data.coords, lo_l3_spectral_fit_dependency.map_data.coords) + self.assertEqual( + data_product.data.spectral_index_map_data, + mock_fit_spectral_index_map.return_value, + ) + self.assertEqual( + data_product.data.coords, lo_l3_spectral_fit_dependency.map_data.coords + ) self.assertEqual(data_product.input_metadata, processor.input_metadata) self.assertEqual(data_product.parent_file_names, ["some_input_file_name"]) self.assertEqual(data_product.spice_frame_name, SpiceFrame.ECLIPJ2000) self.assertEqual([mock_save_data.return_value], product) - @patch('imap_l3_processing.hi.hi_processor.MapProcessor.get_parent_file_names') - @patch('imap_l3_processing.lo.lo_processor.LoL3SpectralFitDependencies.fetch_dependencies') - @patch('imap_l3_processing.lo.lo_processor.fit_spectral_index_map') - @patch('imap_l3_processing.lo.lo_processor.save_data') - @patch('imap_l3_processing.lo.lo_processor.slice_energy_range_by_bin') - def test_process_spectral_index_with_range_specified(self, mock_slice_energy_range_by_bin, mock_save_data, - mock_fit_spectral_index_map, mock_fetch_dependencies, - mock_get_parent_file_names): + @patch("imap_l3_processing.hi.hi_processor.MapProcessor.get_parent_file_names") + @patch( + "imap_l3_processing.lo.lo_processor.LoL3SpectralFitDependencies.fetch_dependencies" + ) + @patch("imap_l3_processing.lo.lo_processor.fit_spectral_index_map") + @patch("imap_l3_processing.lo.lo_processor.save_data") + @patch("imap_l3_processing.lo.lo_processor.slice_energy_range_by_bin") + def test_process_spectral_index_with_range_specified( + self, + mock_slice_energy_range_by_bin, + mock_save_data, + mock_fit_spectral_index_map, + mock_fetch_dependencies, + mock_get_parent_file_names, + ): test_cases = [ ("l090-spx0104-h-hf-sp-ram-hae-6deg-1yr", (1, 4)), ("l090-spxnbs0105-h-hf-sp-ram-hae-6deg-1yr", (1, 5)), @@ -85,15 +117,18 @@ def test_process_spectral_index_with_range_specified(self, mock_slice_energy_ran input_collection = Mock() lo_l3_spectral_fit_dependency = mock_fetch_dependencies.return_value - lo_l3_spectral_fit_dependency.map_data.intensity_map_data.energy = np.array( - [1, 10, 1000, 10000, 100000, 1000000, 10000000]) + lo_l3_spectral_fit_dependency.map_data.intensity_map_data.energy = ( + np.array([1, 10, 1000, 10000, 100000, 1000000, 10000000]) + ) - metadata = InputMetadata(instrument="lo", - data_level="l3", - version="v000", - start_date=datetime(2020, 1, 1, 1), - end_date=datetime(2020, 1, 1, 1), - descriptor=descriptor) + metadata = InputMetadata( + instrument="lo", + data_level="l3", + version="v000", + start_date=datetime(2020, 1, 1, 1), + end_date=datetime(2020, 1, 1, 1), + descriptor=descriptor, + ) processor = LoProcessor(input_collection, input_metadata=metadata) product = processor.process() @@ -102,94 +137,125 @@ def test_process_spectral_index_with_range_specified(self, mock_slice_energy_ran mock_slice_energy_range_by_bin.assert_called_once_with( lo_l3_spectral_fit_dependency.map_data.intensity_map_data, expected_energy_start, - expected_energy_end + expected_energy_end, ) - mock_fit_spectral_index_map.assert_called_once_with(mock_slice_energy_range_by_bin.return_value) + mock_fit_spectral_index_map.assert_called_once_with( + mock_slice_energy_range_by_bin.return_value + ) data_product = mock_save_data.call_args_list[0].args[0] self.assertIsInstance(data_product, RectangularSpectralIndexDataProduct) - self.assertEqual(data_product.data.spectral_index_map_data, - mock_fit_spectral_index_map.return_value) - self.assertEqual(data_product.data.coords, lo_l3_spectral_fit_dependency.map_data.coords) + self.assertEqual( + data_product.data.spectral_index_map_data, + mock_fit_spectral_index_map.return_value, + ) + self.assertEqual( + data_product.data.coords, + lo_l3_spectral_fit_dependency.map_data.coords, + ) self.assertEqual(data_product.input_metadata, processor.input_metadata) self.assertEqual(data_product.spice_frame_name, SpiceFrame.ECLIPJ2000) - self.assertEqual(data_product.parent_file_names, ["some_input_file_name"]) + self.assertEqual( + data_product.parent_file_names, ["some_input_file_name"] + ) self.assertEqual([mock_save_data.return_value], product) - @patch('imap_l3_processing.lo.lo_processor.MapProcessor.get_parent_file_names') - @patch("imap_l3_processing.lo.lo_processor.HiLoL3SurvivalDependencies.fetch_dependencies") + @patch("imap_l3_processing.lo.lo_processor.MapProcessor.get_parent_file_names") + @patch( + "imap_l3_processing.lo.lo_processor.HiLoL3SurvivalDependencies.fetch_dependencies" + ) @patch("imap_l3_processing.lo.lo_processor.process_survival_probabilities") - @patch('imap_l3_processing.lo.lo_processor.save_data') - def test_process_survival_probabilities(self, mock_save_data, - mock_process_survival_prob, - mock_fetch_survival_dependencies, mock_get_parent_file_names): + @patch("imap_l3_processing.lo.lo_processor.save_data") + def test_process_survival_probabilities( + self, + mock_save_data, + mock_process_survival_prob, + mock_fetch_survival_dependencies, + mock_get_parent_file_names, + ): mock_get_parent_file_names.return_value = ["somewhere"] - cases = { - "spacecraft": "sf", - "heliospheric": "hf" - } + cases = {"spacecraft": "sf", "heliospheric": "hf"} for case, reference_frame in cases.items(): with self.subTest(case): - input_metadata = InputMetadata(instrument="lo", - data_level="l3", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="", - descriptor=f"l090-ena-h-{reference_frame}-sp-ram-hae-4deg-6mo", - ) + input_metadata = InputMetadata( + instrument="lo", + data_level="l3", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="", + descriptor=f"l090-ena-h-{reference_frame}-sp-ram-hae-4deg-6mo", + ) dependencies = Mock( - l1c_data=[InputRectangularPointingSet( - epoch=datetime(2025, 1, 1), - epoch_delta=None, - repointing=1, - epoch_j2000=np.array([10]), - exposure_times=np.full((1, 7, 3600, 40), 2), - esa_energy_step=np.arange(7), - pointing_start_met=np.array([43200_000_000_000]), - pointing_end_met=np.array([43200_000_100_000]), - hae_longitude=np.ones(3600).reshape((1, 3600)), - hae_latitude=np.ones(3600).reshape((1, 3600)) - )], - dependency_file_paths=[Path("folder/map"), Path("folder/l1c")] + l1c_data=[ + InputRectangularPointingSet( + epoch=datetime(2025, 1, 1), + epoch_delta=None, + repointing=1, + epoch_j2000=np.array([10]), + exposure_times=np.full((1, 7, 3600, 40), 2), + esa_energy_step=np.arange(7), + pointing_start_met=np.array([43200_000_000_000]), + pointing_end_met=np.array([43200_000_100_000]), + hae_longitude=np.ones(3600).reshape((1, 3600)), + hae_latitude=np.ones(3600).reshape((1, 3600)), + ) + ], + dependency_file_paths=[Path("folder/map"), Path("folder/l1c")], ) mock_fetch_survival_dependencies.return_value = dependencies - mock_process_survival_prob.return_value = sentinel.survival_probabilities + mock_process_survival_prob.return_value = ( + sentinel.survival_probabilities + ) processor = LoProcessor(sentinel.input_dependencies, input_metadata) product = processor.process(spice_frame_name=SpiceFrame.IMAP_DPS) - mock_fetch_survival_dependencies.assert_called_once_with(sentinel.input_dependencies, - Instrument.IMAP_LO) + mock_fetch_survival_dependencies.assert_called_once_with( + sentinel.input_dependencies, Instrument.IMAP_LO + ) - mock_process_survival_prob.assert_called_once_with(dependencies, SpiceFrame.IMAP_DPS, cg_corrected=False) + mock_process_survival_prob.assert_called_once_with( + dependencies, SpiceFrame.IMAP_DPS, cg_corrected=False + ) - np.testing.assert_array_equal(np.full((1, 7, 3600), 80), dependencies.l1c_data[0].exposure_times) + np.testing.assert_array_equal( + np.full((1, 7, 3600), 80), dependencies.l1c_data[0].exposure_times + ) - mock_save_data.assert_called_once_with(RectangularIntensityDataProduct( - input_metadata=input_metadata, - parent_file_names=["l1c", "map", "somewhere"], - data=sentinel.survival_probabilities, - spice_frame_name=SpiceFrame.IMAP_DPS, - )) + mock_save_data.assert_called_once_with( + RectangularIntensityDataProduct( + input_metadata=input_metadata, + parent_file_names=["l1c", "map", "somewhere"], + data=sentinel.survival_probabilities, + spice_frame_name=SpiceFrame.IMAP_DPS, + ) + ) self.assertEqual([mock_save_data.return_value], product) mock_fetch_survival_dependencies.reset_mock() mock_process_survival_prob.reset_mock() mock_save_data.reset_mock() - @patch("imap_l3_processing.lo.lo_processor.LoCombinedDependencies.fetch_dependencies") + @patch( + "imap_l3_processing.lo.lo_processor.LoCombinedDependencies.fetch_dependencies" + ) @patch("imap_l3_processing.lo.lo_processor.ExposureWeightedCombination") - @patch('imap_l3_processing.lo.lo_processor.save_data') - def test_process_lo_combined(self, mock_save_data, - mock_exposure_weighted_combination_class, - mock_fetch_combined_dependencies): - - mock_exposure_weighted_combination = mock_exposure_weighted_combination_class.return_value + @patch("imap_l3_processing.lo.lo_processor.save_data") + def test_process_lo_combined( + self, + mock_save_data, + mock_exposure_weighted_combination_class, + mock_fetch_combined_dependencies, + ): + + mock_exposure_weighted_combination = ( + mock_exposure_weighted_combination_class.return_value + ) input_metadata = InputMetadata( instrument="lo", @@ -205,27 +271,41 @@ def test_process_lo_combined(self, mock_save_data, ) processing_input = ProcessingInputCollection( - ScienceInput('imap_lo_l3_l075-ena-h-hf-sp-ram-hae-6deg-6mo_20250422_v001.cdf'), - ScienceInput('imap_lo_l3_l090-ena-h-hf-sp-ram-hae-6deg-6mo_20250422_v001.cdf'), - ScienceInput('imap_lo_l3_l105-ena-h-hf-sp-ram-hae-6deg-6mo_20250422_v001.cdf'), + ScienceInput( + "imap_lo_l3_l075-ena-h-hf-sp-ram-hae-6deg-6mo_20250422_v001.cdf" + ), + ScienceInput( + "imap_lo_l3_l090-ena-h-hf-sp-ram-hae-6deg-6mo_20250422_v001.cdf" + ), + ScienceInput( + "imap_lo_l3_l105-ena-h-hf-sp-ram-hae-6deg-6mo_20250422_v001.cdf" + ), ) lo_processor = LoProcessor(processing_input, input_metadata) - [combined_product] = lo_processor.process(spice_frame_name=SpiceFrame.ECLIPJ2000) + [combined_product] = lo_processor.process( + spice_frame_name=SpiceFrame.ECLIPJ2000 + ) mock_fetch_combined_dependencies.assert_called_once_with(processing_input) - mock_combine = mock_exposure_weighted_combination.combine_rectangular_intensity_map_data + mock_combine = ( + mock_exposure_weighted_combination.combine_rectangular_intensity_map_data + ) - mock_combine.assert_called_once_with([sentinel.l075, sentinel.l090, sentinel.l105]) + mock_combine.assert_called_once_with( + [sentinel.l075, sentinel.l090, sentinel.l105] + ) expected_parents = [ - 'imap_lo_l3_l075-ena-h-hf-sp-ram-hae-6deg-6mo_20250422_v001.cdf', - 'imap_lo_l3_l090-ena-h-hf-sp-ram-hae-6deg-6mo_20250422_v001.cdf', - 'imap_lo_l3_l105-ena-h-hf-sp-ram-hae-6deg-6mo_20250422_v001.cdf', + "imap_lo_l3_l075-ena-h-hf-sp-ram-hae-6deg-6mo_20250422_v001.cdf", + "imap_lo_l3_l090-ena-h-hf-sp-ram-hae-6deg-6mo_20250422_v001.cdf", + "imap_lo_l3_l105-ena-h-hf-sp-ram-hae-6deg-6mo_20250422_v001.cdf", ] - expected_data_product = RectangularIntensityDataProduct(input_metadata, mock_combine.return_value, SpiceFrame.ECLIPJ2000) + expected_data_product = RectangularIntensityDataProduct( + input_metadata, mock_combine.return_value, SpiceFrame.ECLIPJ2000 + ) expected_data_product.parent_file_names.extend(expected_parents) mock_save_data.assert_called_once_with(expected_data_product) @@ -235,17 +315,27 @@ def test_rejects_unimplemented_descriptors(self): input_collection = ProcessingInputCollection() cases = [ - ("not-a-valid-descriptor", ValueError, ("Could not parse descriptor not-a-valid-descriptor",)), - ("l090-ena-h-hf-nsp-ram-hae-6deg-1yr", NotImplementedError, ("l090-ena-h-hf-nsp-ram-hae-6deg-1yr",)), + ( + "not-a-valid-descriptor", + ValueError, + ("Could not parse descriptor not-a-valid-descriptor",), + ), + ( + "l090-ena-h-hf-nsp-ram-hae-6deg-1yr", + NotImplementedError, + ("l090-ena-h-hf-nsp-ram-hae-6deg-1yr",), + ), ] for descriptor, exception_class, exception_args in cases: with self.subTest(descriptor): - metadata = InputMetadata(instrument="lo", - data_level="l3", - version="v000", - start_date=datetime(2020, 1, 1, 1), - end_date=datetime(2020, 1, 1, 1), - descriptor=descriptor) + metadata = InputMetadata( + instrument="lo", + data_level="l3", + version="v000", + start_date=datetime(2020, 1, 1, 1), + end_date=datetime(2020, 1, 1, 1), + descriptor=descriptor, + ) processor = LoProcessor(input_collection, input_metadata=metadata) with self.assertRaises(exception_class) as cm: @@ -271,42 +361,86 @@ def test_isn_background_subtraction(self): obs_date=np.arange(1 * 7 * 60 * 30).reshape((1, 7, 60, 30)) * 5, obs_date_range=np.arange(1 * 7 * 60 * 30).reshape((1, 7, 60, 30)) * 6, ena_intensity=np.arange(1 * 7 * 60 * 30).reshape((1, 7, 60, 30)) * 7, - ena_intensity_sys_err=np.arange(1 * 7 * 60 * 30).reshape((1, 7, 60, 30)) * 8, - ena_intensity_stat_uncert=np.arange(1 * 7 * 60 * 30).reshape((1, 7, 60, 30)) * 9, + ena_intensity_sys_err=np.arange(1 * 7 * 60 * 30).reshape((1, 7, 60, 30)) + * 8, + ena_intensity_stat_uncert=np.arange(1 * 7 * 60 * 30).reshape((1, 7, 60, 30)) + * 9, bg_rate_sys_err=np.arange(1 * 7 * 60 * 30).reshape((1, 7, 60, 30)) * 11, ena_count_rate=np.ones((1, 7, 60, 30)) * 3, bg_rate=np.ones((1, 7, 60, 30)) * 2, bg_rate_stat_uncert=np.ones((1, 7, 60, 30)) * 3, ena_count_rate_stat_uncert=np.ones((1, 7, 60, 30)) * 2, ) - actual_map_data: ISNBackgroundSubtractedMapData = isn_background_subtraction(input_data) - actual_isn_rate_map_data: ISNBackgroundSubtractedData = actual_map_data.isn_rate_map_data + actual_map_data: ISNBackgroundSubtractedMapData = isn_background_subtraction( + input_data + ) + actual_isn_rate_map_data: ISNBackgroundSubtractedData = ( + actual_map_data.isn_rate_map_data + ) expected_isn_bg_rate_subtracted = np.ones((1, 7, 60, 30)) - expected_isn_bg_rate_subtracted_stat_uncert = np.ones((1, 7, 60, 30)) * np.sqrt(13) - - np.testing.assert_array_equal(actual_isn_rate_map_data.isn_bg_rate_subtracted, expected_isn_bg_rate_subtracted) - np.testing.assert_array_equal(actual_isn_rate_map_data.isn_bg_rate_subtracted_stat_uncert, expected_isn_bg_rate_subtracted_stat_uncert) - - np.testing.assert_array_equal(actual_isn_rate_map_data.energy, input_data.energy) - np.testing.assert_array_equal(actual_isn_rate_map_data.energy_delta_plus, input_data.energy_delta_plus) - np.testing.assert_array_equal(actual_isn_rate_map_data.energy_delta_minus, input_data.energy_delta_minus) - np.testing.assert_array_equal(actual_isn_rate_map_data.energy_label, input_data.energy_label) - np.testing.assert_array_equal(actual_isn_rate_map_data.exposure_factor, input_data.exposure_factor) - np.testing.assert_array_equal(actual_isn_rate_map_data.obs_date, input_data.obs_date) - np.testing.assert_array_equal(actual_isn_rate_map_data.obs_date_range, input_data.obs_date_range) - np.testing.assert_array_equal(actual_isn_rate_map_data.bg_rate, input_data.bg_rate) - np.testing.assert_array_equal(actual_isn_rate_map_data.bg_rate_stat_uncert, input_data.bg_rate_stat_uncert) - np.testing.assert_array_equal(actual_isn_rate_map_data.bg_rate_sys_err, input_data.bg_rate_sys_err) - np.testing.assert_array_equal(actual_isn_rate_map_data.bg_rate_sys_err, - actual_isn_rate_map_data.isn_bg_rate_subtracted_sys_err) + expected_isn_bg_rate_subtracted_stat_uncert = np.ones((1, 7, 60, 30)) * np.sqrt( + 13 + ) + + np.testing.assert_array_equal( + actual_isn_rate_map_data.isn_bg_rate_subtracted, + expected_isn_bg_rate_subtracted, + ) + np.testing.assert_array_equal( + actual_isn_rate_map_data.isn_bg_rate_subtracted_stat_uncert, + expected_isn_bg_rate_subtracted_stat_uncert, + ) + + np.testing.assert_array_equal( + actual_isn_rate_map_data.energy, input_data.energy + ) + np.testing.assert_array_equal( + actual_isn_rate_map_data.energy_delta_plus, input_data.energy_delta_plus + ) + np.testing.assert_array_equal( + actual_isn_rate_map_data.energy_delta_minus, input_data.energy_delta_minus + ) + np.testing.assert_array_equal( + actual_isn_rate_map_data.energy_label, input_data.energy_label + ) + np.testing.assert_array_equal( + actual_isn_rate_map_data.exposure_factor, input_data.exposure_factor + ) + np.testing.assert_array_equal( + actual_isn_rate_map_data.obs_date, input_data.obs_date + ) + np.testing.assert_array_equal( + actual_isn_rate_map_data.obs_date_range, input_data.obs_date_range + ) + np.testing.assert_array_equal( + actual_isn_rate_map_data.bg_rate, input_data.bg_rate + ) + np.testing.assert_array_equal( + actual_isn_rate_map_data.bg_rate_stat_uncert, input_data.bg_rate_stat_uncert + ) + np.testing.assert_array_equal( + actual_isn_rate_map_data.bg_rate_sys_err, input_data.bg_rate_sys_err + ) + np.testing.assert_array_equal( + actual_isn_rate_map_data.bg_rate_sys_err, + actual_isn_rate_map_data.isn_bg_rate_subtracted_sys_err, + ) self.assertEqual(actual_isn_rate_map_data.epoch, sentinel.epoch) self.assertEqual(actual_isn_rate_map_data.solid_angle, sentinel.solid_angle) self.assertEqual(actual_isn_rate_map_data.latitude, sentinel.latitude) - self.assertEqual(actual_isn_rate_map_data.latitude_delta, sentinel.latitude_delta) - self.assertEqual(actual_isn_rate_map_data.latitude_label, sentinel.latitude_label) + self.assertEqual( + actual_isn_rate_map_data.latitude_delta, sentinel.latitude_delta + ) + self.assertEqual( + actual_isn_rate_map_data.latitude_label, sentinel.latitude_label + ) self.assertEqual(actual_isn_rate_map_data.longitude, sentinel.longitude) - self.assertEqual(actual_isn_rate_map_data.longitude_delta, sentinel.longitude_delta) - self.assertEqual(actual_isn_rate_map_data.longitude_label, sentinel.longitude_label) + self.assertEqual( + actual_isn_rate_map_data.longitude_delta, sentinel.longitude_delta + ) + self.assertEqual( + actual_isn_rate_map_data.longitude_label, sentinel.longitude_label + ) self.assertEqual(actual_isn_rate_map_data.epoch_delta, sentinel.epoch_delta) diff --git a/tests/maps/test_builders.py b/tests/maps/test_builders.py index a76b7056e..92210515f 100644 --- a/tests/maps/test_builders.py +++ b/tests/maps/test_builders.py @@ -4,27 +4,48 @@ import numpy as np import spiceypy -from imap_l3_processing.maps.map_models import RectangularIntensityMapData, IntensityMapData, RectangularCoords, \ - InputRectangularPointingSet, GlowsL3eRectangularMapInputData, RectangularSpectralIndexMapData, SpectralIndexMapData +from imap_l3_processing.maps.map_models import ( + RectangularIntensityMapData, + IntensityMapData, + RectangularCoords, + InputRectangularPointingSet, + GlowsL3eRectangularMapInputData, + RectangularSpectralIndexMapData, + SpectralIndexMapData, +) from imap_l3_processing.maps.quality_flags import MapL3Flags -def create_intensity_map_data(epoch=None, epoch_delta=None, lon=None, lat=None, energy=None, - energy_delta=None, flux=None, - intensity_stat_uncert=None) -> IntensityMapData: +def create_intensity_map_data( + epoch=None, + epoch_delta=None, + lon=None, + lat=None, + energy=None, + energy_delta=None, + flux=None, + intensity_stat_uncert=None, +) -> IntensityMapData: lon = lon if lon is not None else np.array([1.0]) lat = lat if lat is not None else np.array([1.0]) energy = energy if energy is not None else np.array([1.0]) - energy_delta = energy_delta if energy_delta is not None else np.full((len(energy), 2), 1) + energy_delta = ( + energy_delta if energy_delta is not None else np.full((len(energy), 2), 1) + ) epoch = epoch if epoch is not None else np.ma.array([datetime.now()]) epoch_delta = epoch_delta if epoch_delta is not None else np.ma.array([86400 * 1e9]) - flux = flux if flux is not None else np.full((len(epoch), len(energy), len(lon), len(lat)), fill_value=1) - intensity_stat_uncert = intensity_stat_uncert if intensity_stat_uncert is not None else np.full( - (len(epoch), len(energy), len(lon), len(lat)), - fill_value=1) + flux = ( + flux + if flux is not None + else np.full((len(epoch), len(energy), len(lon), len(lat)), fill_value=1) + ) + intensity_stat_uncert = ( + intensity_stat_uncert + if intensity_stat_uncert is not None + else np.full((len(epoch), len(energy), len(lon), len(lat)), fill_value=1) + ) if isinstance(flux, np.ndarray): - more_real_flux = flux else: @@ -55,25 +76,44 @@ def create_intensity_map_data(epoch=None, epoch_delta=None, lon=None, lat=None, ) -def create_rectangular_intensity_map_data(epoch=None, epoch_delta=None, lon=None, lat=None, energy=None, - energy_delta=None, flux=None, - intensity_stat_uncert=None, - include_ena_intensity_sys_err_minus=False) -> RectangularIntensityMapData: +def create_rectangular_intensity_map_data( + epoch=None, + epoch_delta=None, + lon=None, + lat=None, + energy=None, + energy_delta=None, + flux=None, + intensity_stat_uncert=None, + include_ena_intensity_sys_err_minus=False, +) -> RectangularIntensityMapData: lon = lon if lon is not None else np.array([1.0]) lat = lat if lat is not None else np.array([1.0]) energy = energy if energy is not None else np.array([1.0]) - energy_delta = energy_delta if energy_delta is not None else np.full((len(energy), 2), 1) + energy_delta = ( + energy_delta if energy_delta is not None else np.full((len(energy), 2), 1) + ) epoch = epoch if epoch is not None else np.ma.array([datetime.now()]) epoch_delta = epoch_delta if epoch_delta is not None else np.ma.array([86400 * 1e9]) - flux = flux if flux is not None else np.full((len(epoch), len(energy), len(lon), len(lat)), fill_value=1) - intensity_stat_uncert = intensity_stat_uncert if intensity_stat_uncert is not None else np.full( - (len(epoch), len(energy), len(lon), len(lat)), - fill_value=1) + flux = ( + flux + if flux is not None + else np.full((len(epoch), len(energy), len(lon), len(lat)), fill_value=1) + ) + intensity_stat_uncert = ( + intensity_stat_uncert + if intensity_stat_uncert is not None + else np.full((len(epoch), len(energy), len(lon), len(lat)), fill_value=1) + ) ena_intensity_sys_err_minus = None ena_intensity_sys_err_plus = None if include_ena_intensity_sys_err_minus: - ena_intensity_sys_err_minus = np.full((len(epoch), len(energy), len(lon), len(lat)), fill_value=2) - ena_intensity_sys_err_plus = np.full((len(epoch), len(energy), len(lon), len(lat)), fill_value=3) + ena_intensity_sys_err_minus = np.full( + (len(epoch), len(energy), len(lon), len(lat)), fill_value=2 + ) + ena_intensity_sys_err_plus = np.full( + (len(epoch), len(energy), len(lon), len(lat)), fill_value=3 + ) if isinstance(flux, np.ndarray): more_real_flux = flux @@ -92,15 +132,17 @@ def create_rectangular_intensity_map_data(epoch=None, epoch_delta=None, lon=None latitude=lat, longitude=lon, exposure_factor=np.full_like(flux, 0), - obs_date=np.ma.array(np.full(more_real_flux.shape, datetime(year=2010, month=1, day=1))), + obs_date=np.ma.array( + np.full(more_real_flux.shape, datetime(year=2010, month=1, day=1)) + ), obs_date_range=np.ma.array(np.full_like(more_real_flux, 0)), solid_angle=np.full_like(more_real_flux, 0), ena_intensity=flux, ena_intensity_stat_uncert=intensity_stat_uncert, - ena_intensity_sys_err=flux * .001, - bg_intensity=flux * .01, - bg_intensity_stat_uncert=intensity_stat_uncert * .01, - bg_intensity_sys_err=flux * .01 * .001, + ena_intensity_sys_err=flux * 0.001, + bg_intensity=flux * 0.01, + bg_intensity_stat_uncert=intensity_stat_uncert * 0.01, + bg_intensity_sys_err=flux * 0.01 * 0.001, ena_intensity_sys_err_minus=ena_intensity_sys_err_minus, ena_intensity_sys_err_plus=ena_intensity_sys_err_plus, quality_flags=np.full_like(flux, MapL3Flags.NONE), @@ -110,13 +152,18 @@ def create_rectangular_intensity_map_data(epoch=None, epoch_delta=None, lon=None latitude_label=lat.astype(str), longitude_delta=np.full_like(lon, 0), longitude_label=lon.astype(str), - ) + ), ) -def create_rectangular_intensity_map(intensity_map_data: IntensityMapData = None, - coords: RectangularCoords = None) -> RectangularIntensityMapData: - intensity_map = intensity_map_data if intensity_map_data is not None else create_intensity_map_data() +def create_rectangular_intensity_map( + intensity_map_data: IntensityMapData = None, coords: RectangularCoords = None +) -> RectangularIntensityMapData: + intensity_map = ( + intensity_map_data + if intensity_map_data is not None + else create_intensity_map_data() + ) if coords is None: lon = np.array([1.0]) @@ -131,7 +178,9 @@ def create_rectangular_intensity_map(intensity_map_data: IntensityMapData = None return RectangularIntensityMapData(intensity_map_data=intensity_map, coords=coords) -def construct_intensity_data_with_all_zero_fields(num_pixels: int = 1) -> IntensityMapData: +def construct_intensity_data_with_all_zero_fields( + num_pixels: int = 1, +) -> IntensityMapData: return IntensityMapData( epoch=np.array([0]), epoch_delta=np.array([0]), @@ -156,20 +205,34 @@ def construct_intensity_data_with_all_zero_fields(num_pixels: int = 1) -> Intens ) -def create_rectangular_spectral_index_map_data(epoch=None, epoch_delta=None, lon=None, lat=None, energy=None, - energy_delta=None, spectral_index=None, - spectral_index_stat_unc=None): +def create_rectangular_spectral_index_map_data( + epoch=None, + epoch_delta=None, + lon=None, + lat=None, + energy=None, + energy_delta=None, + spectral_index=None, + spectral_index_stat_unc=None, +): lon = lon if lon is not None else np.array([1.0]) lat = lat if lat is not None else np.array([1.0]) energy = energy if energy is not None else np.array([1.0]) - energy_delta = energy_delta if energy_delta is not None else np.full((len(energy), 2), 1) + energy_delta = ( + energy_delta if energy_delta is not None else np.full((len(energy), 2), 1) + ) epoch = epoch if epoch is not None else np.ma.array([datetime.now()]) epoch_delta = epoch_delta if epoch_delta is not None else np.ma.array([86400 * 1e9]) - spectral_index = spectral_index if spectral_index is not None else np.full((len(epoch), len(lon), len(lat)), - fill_value=1) - spectral_index_stat_unc = spectral_index_stat_unc if spectral_index_stat_unc is not None else np.full( - (len(epoch), len(energy), len(lon), len(lat)), - fill_value=1) + spectral_index = ( + spectral_index + if spectral_index is not None + else np.full((len(epoch), len(lon), len(lat)), fill_value=1) + ) + spectral_index_stat_unc = ( + spectral_index_stat_unc + if spectral_index_stat_unc is not None + else np.full((len(epoch), len(energy), len(lon), len(lat)), fill_value=1) + ) ena_spectral_index_scalar_coefficient = np.full_like(spectral_index, 3) ena_spectral_index_scalar_coefficient_stat_unc = np.full_like(spectral_index, 4) ena_spectral_index_chisq = np.full_like(spectral_index, 5) @@ -212,18 +275,22 @@ def create_rectangular_spectral_index_map_data(epoch=None, epoch_delta=None, lon def create_l1c_pset( - epoch: datetime = datetime(2025, 4, 15, 12), - epoch_delta: np.ndarray = np.array([43200_000_000_000]), - repointing: int = 1, - energy_steps: np.ndarray = np.array([1]), - exposures: Optional[np.ndarray] = None, - pointing_start_met=None, - pointing_end_met=None, - hae_longitude=None, - hae_latitude=None, + epoch: datetime = datetime(2025, 4, 15, 12), + epoch_delta: np.ndarray = np.array([43200_000_000_000]), + repointing: int = 1, + energy_steps: np.ndarray = np.array([1]), + exposures: Optional[np.ndarray] = None, + pointing_start_met=None, + pointing_end_met=None, + hae_longitude=None, + hae_latitude=None, ) -> InputRectangularPointingSet: epoch_j2000 = np.array([spiceypy.datetime2et(epoch)]) * 1e9 - exposures = exposures if exposures is not None else np.full(shape=(1, energy_steps.shape[0], 3600), fill_value=1.) + exposures = ( + exposures + if exposures is not None + else np.full(shape=(1, energy_steps.shape[0], 3600), fill_value=1.0) + ) ram_latitudes = np.linspace(89.95, -89.95, 1800) anti_latitudes = np.linspace(-89.95, 89.95, 1800) full_rotation_latitudes = np.concat([ram_latitudes, anti_latitudes]) @@ -256,4 +323,6 @@ def create_l3e_pset( epoch_j2000 = np.array([spiceypy.datetime2et(epoch)]) * 1e9 sp = sp or np.full(shape=(1, len(energy_steps), len(spin_angle)), fill_value=0.5) flags = flags or np.full(1, 0, dtype=np.uint16) - return GlowsL3eRectangularMapInputData(epoch, epoch_j2000, repointing, energy_steps, spin_angle, sp, flags) + return GlowsL3eRectangularMapInputData( + epoch, epoch_j2000, repointing, energy_steps, spin_angle, sp, flags + ) diff --git a/tests/maps/test_hilo_l3_survival_dependencies.py b/tests/maps/test_hilo_l3_survival_dependencies.py index 220547c64..56fadbb94 100644 --- a/tests/maps/test_hilo_l3_survival_dependencies.py +++ b/tests/maps/test_hilo_l3_survival_dependencies.py @@ -4,85 +4,134 @@ from imap_data_access.processing_input import ProcessingInputCollection, ScienceInput -from imap_l3_processing.maps.hilo_l3_survival_dependencies import HiLoL3SurvivalDependencies, \ - HiL3SingleSensorFullSpinDependencies +from imap_l3_processing.maps.hilo_l3_survival_dependencies import ( + HiLoL3SurvivalDependencies, + HiL3SingleSensorFullSpinDependencies, +) from imap_l3_processing.models import Instrument class TestHiLoL3SurvivalDependencies(unittest.TestCase): - @patch("imap_l3_processing.maps.hilo_l3_survival_dependencies.parse_map_descriptor") - @patch("imap_l3_processing.maps.hilo_l3_survival_dependencies.imap_data_access.download") + @patch( + "imap_l3_processing.maps.hilo_l3_survival_dependencies.imap_data_access.download" + ) @patch("imap_l3_processing.maps.hilo_l3_survival_dependencies.read_glows_l3e_data") - @patch("imap_l3_processing.maps.hilo_l3_survival_dependencies.read_l1c_rectangular_pointing_set_data") - @patch("imap_l3_processing.maps.hilo_l3_survival_dependencies.RectangularIntensityMapData.read_from_path") - def test_fetch_dependencies(self, mock_read_from_path, mock_read_l1c, mock_read_glows_l3e, - mock_imap_data_access_download, mock_parse_map_descriptor): + @patch( + "imap_l3_processing.maps.hilo_l3_survival_dependencies.read_l1c_rectangular_pointing_set_data" + ) + @patch( + "imap_l3_processing.maps.hilo_l3_survival_dependencies.RectangularIntensityMapData.read_from_path" + ) + def test_fetch_dependencies( + self, + mock_read_from_path, + mock_read_l1c, + mock_read_glows_l3e, + mock_imap_data_access_download, + mock_parse_map_descriptor, + ): for instrument in [Instrument.IMAP_HI, Instrument.IMAP_LO]: mock_read_from_path.reset_mock() mock_parse_map_descriptor.reset_mock() - l1c_file_names = [f"imap_{instrument.value}_l1c_45sensor-pset_20201001_v001.cdf", - f"imap_{instrument.value}_l1c_45sensor-pset_20201002_v002.cdf", - f"imap_{instrument.value}_l1c_45sensor-pset_20201003_v001.cdf"] + l1c_file_names = [ + f"imap_{instrument.value}_l1c_45sensor-pset_20201001_v001.cdf", + f"imap_{instrument.value}_l1c_45sensor-pset_20201002_v002.cdf", + f"imap_{instrument.value}_l1c_45sensor-pset_20201003_v001.cdf", + ] glows_file_names = [ f"imap_glows_l3e_survival-probability-{instrument.value}_20201001_v001.cdf", f"imap_glows_l3e_survival-probability-{instrument.value}_20201002_v002.cdf", - f"imap_glows_l3e_survival-probability-{instrument.value}_20201003_v003.cdf"] + f"imap_glows_l3e_survival-probability-{instrument.value}_20201003_v003.cdf", + ] l2_map_descriptor = "ena-h-sf-nsp-ram-hae-6deg-1yr" - l2_filename = f"imap_{instrument.value}_l2_{l2_map_descriptor}_20250415_v001.cdf" + l2_filename = ( + f"imap_{instrument.value}_l2_{l2_map_descriptor}_20250415_v001.cdf" + ) dependencies = ProcessingInputCollection( ScienceInput(l2_filename), *[ScienceInput(l1c) for l1c in l1c_file_names], - *[ScienceInput(l3e) for l3e in glows_file_names] + *[ScienceInput(l3e) for l3e in glows_file_names], ) - downloaded_files = [sentinel.l2_file_path, sentinel.l1c_file_path_1, - sentinel.l1c_file_path_2, - sentinel.l1c_file_path_3, sentinel.glows_file_path_1, - sentinel.glows_file_path_2, sentinel.glows_file_path_3] + downloaded_files = [ + sentinel.l2_file_path, + sentinel.l1c_file_path_1, + sentinel.l1c_file_path_2, + sentinel.l1c_file_path_3, + sentinel.glows_file_path_1, + sentinel.glows_file_path_2, + sentinel.glows_file_path_3, + ] mock_imap_data_access_download.side_effect = downloaded_files - mock_read_l1c.side_effect = [sentinel.l1c_data_1, sentinel.l1c_data_2, sentinel.l1c_data_3] - - mock_read_glows_l3e.side_effect = [sentinel.glows_data_1, sentinel.glows_data_2, - sentinel.glows_data_3] - - actual = HiLoL3SurvivalDependencies.fetch_dependencies(dependencies, instrument) + mock_read_l1c.side_effect = [ + sentinel.l1c_data_1, + sentinel.l1c_data_2, + sentinel.l1c_data_3, + ] + + mock_read_glows_l3e.side_effect = [ + sentinel.glows_data_1, + sentinel.glows_data_2, + sentinel.glows_data_3, + ] + + actual = HiLoL3SurvivalDependencies.fetch_dependencies( + dependencies, instrument + ) - expected_imap_data_access_calls = [call(l2_filename)] + [call(path) for path in - l1c_file_names + glows_file_names] - mock_imap_data_access_download.assert_has_calls(expected_imap_data_access_calls) + expected_imap_data_access_calls = [call(l2_filename)] + [ + call(path) for path in l1c_file_names + glows_file_names + ] + mock_imap_data_access_download.assert_has_calls( + expected_imap_data_access_calls + ) mock_read_from_path.assert_called_once_with(sentinel.l2_file_path) - mock_read_l1c.assert_has_calls([ - call(sentinel.l1c_file_path_1), - call(sentinel.l1c_file_path_2), - call(sentinel.l1c_file_path_3) - ]) + mock_read_l1c.assert_has_calls( + [ + call(sentinel.l1c_file_path_1), + call(sentinel.l1c_file_path_2), + call(sentinel.l1c_file_path_3), + ] + ) - mock_read_glows_l3e.assert_has_calls([ - call(sentinel.glows_file_path_1), - call(sentinel.glows_file_path_2), - call(sentinel.glows_file_path_3) - ]) + mock_read_glows_l3e.assert_has_calls( + [ + call(sentinel.glows_file_path_1), + call(sentinel.glows_file_path_2), + call(sentinel.glows_file_path_3), + ] + ) mock_parse_map_descriptor.assert_called_once_with(l2_map_descriptor) self.assertEqual(actual.l2_data, mock_read_from_path.return_value) - self.assertEqual(actual.l1c_data, [sentinel.l1c_data_1, - sentinel.l1c_data_2, - sentinel.l1c_data_3]) - self.assertEqual(actual.glows_l3e_data, [sentinel.glows_data_1, - sentinel.glows_data_2, - sentinel.glows_data_3, ]) - self.assertEqual(actual.l2_map_descriptor_parts, mock_parse_map_descriptor.return_value) + self.assertEqual( + actual.l1c_data, + [sentinel.l1c_data_1, sentinel.l1c_data_2, sentinel.l1c_data_3], + ) + self.assertEqual( + actual.glows_l3e_data, + [ + sentinel.glows_data_1, + sentinel.glows_data_2, + sentinel.glows_data_3, + ], + ) + self.assertEqual( + actual.l2_map_descriptor_parts, mock_parse_map_descriptor.return_value + ) self.assertEqual(actual.dependency_file_paths, downloaded_files) - @patch("imap_l3_processing.maps.hilo_l3_survival_dependencies.HiLoL3SurvivalDependencies.fetch_dependencies") + @patch( + "imap_l3_processing.maps.hilo_l3_survival_dependencies.HiLoL3SurvivalDependencies.fetch_dependencies" + ) def test_fetch_single_sensor_full_spin_dependencies(self, mock_fetch_dependencies): ram_map_descriptor = "h90-ena-h-hf-nsp-ram-hae-6deg-1yr" ram_l2_filename = f"imap_hi_l2_{ram_map_descriptor}_20250415_v001.cdf" @@ -96,29 +145,55 @@ def test_fetch_single_sensor_full_spin_dependencies(self, mock_fetch_dependencie anti_input = ScienceInput(anti_l2_filename) glows_input = ScienceInput(glows_l3e_filename) l1c_input = ScienceInput(l1c_filename) - dependencies = ProcessingInputCollection(ram_input, anti_input, glows_input, l1c_input) + dependencies = ProcessingInputCollection( + ram_input, anti_input, glows_input, l1c_input + ) mock_ram_survival_dependencies = Mock(spec=HiLoL3SurvivalDependencies) - mock_ram_survival_dependencies.dependency_file_paths = [Path("ram_input1"), Path("ram_input2")] + mock_ram_survival_dependencies.dependency_file_paths = [ + Path("ram_input1"), + Path("ram_input2"), + ] mock_antiram_survival_dependencies = Mock(spec=HiLoL3SurvivalDependencies) - mock_antiram_survival_dependencies.dependency_file_paths = [Path("antiram_input1"), Path("antiram_input2")] + mock_antiram_survival_dependencies.dependency_file_paths = [ + Path("antiram_input1"), + Path("antiram_input2"), + ] - mock_fetch_dependencies.side_effect = [mock_ram_survival_dependencies, mock_antiram_survival_dependencies] + mock_fetch_dependencies.side_effect = [ + mock_ram_survival_dependencies, + mock_antiram_survival_dependencies, + ] result = HiL3SingleSensorFullSpinDependencies.fetch_dependencies(dependencies) self.assertIsInstance(result, HiL3SingleSensorFullSpinDependencies) self.assertEqual(result.ram_dependencies, mock_ram_survival_dependencies) - self.assertEqual(result.antiram_dependencies, mock_antiram_survival_dependencies) + self.assertEqual( + result.antiram_dependencies, mock_antiram_survival_dependencies + ) [ram_dependencies, instrument] = mock_fetch_dependencies.call_args_list[0].args self.assertIsInstance(ram_dependencies, ProcessingInputCollection) - self.assertEqual([ram_input, glows_input, l1c_input], ram_dependencies.processing_input) + self.assertEqual( + [ram_input, glows_input, l1c_input], ram_dependencies.processing_input + ) self.assertEqual(Instrument.IMAP_HI, instrument) - [antiram_dependencies, instrument] = mock_fetch_dependencies.call_args_list[1].args + [antiram_dependencies, instrument] = mock_fetch_dependencies.call_args_list[ + 1 + ].args self.assertIsInstance(antiram_dependencies, ProcessingInputCollection) - self.assertEqual([anti_input, glows_input, l1c_input], antiram_dependencies.processing_input) + self.assertEqual( + [anti_input, glows_input, l1c_input], antiram_dependencies.processing_input + ) self.assertEqual(Instrument.IMAP_HI, instrument) - self.assertEqual([Path("ram_input1"), Path("ram_input2"), Path("antiram_input1"), Path("antiram_input2")], - result.dependency_file_paths) + self.assertEqual( + [ + Path("ram_input1"), + Path("ram_input2"), + Path("antiram_input1"), + Path("antiram_input2"), + ], + result.dependency_file_paths, + ) diff --git a/tests/maps/test_map_combination.py b/tests/maps/test_map_combination.py index 1d5578f4b..ef06ad53b 100644 --- a/tests/maps/test_map_combination.py +++ b/tests/maps/test_map_combination.py @@ -4,46 +4,73 @@ import numpy as np -from imap_l3_processing.maps.map_combination import UnweightedCombination, CombinationStrategy, \ - ExposureWeightedCombination, UncertaintyWeightedCombination -from imap_l3_processing.maps.map_models import IntensityMapData, RectangularIntensityMapData, RectangularCoords, \ - HealPixIntensityMapData, HealPixCoords +from imap_l3_processing.maps.map_combination import ( + UnweightedCombination, + CombinationStrategy, + ExposureWeightedCombination, + UncertaintyWeightedCombination, +) +from imap_l3_processing.maps.map_models import ( + IntensityMapData, + RectangularIntensityMapData, + RectangularCoords, + HealPixIntensityMapData, + HealPixCoords, +) from imap_l3_processing.maps.quality_flags import MapL3Flags -from tests.maps.test_builders import construct_intensity_data_with_all_zero_fields, create_rectangular_intensity_map +from tests.maps.test_builders import ( + construct_intensity_data_with_all_zero_fields, + create_rectangular_intensity_map, +) class TestMapCombination(unittest.TestCase): def test_combine_maps_works_when_passed_all_nan_arrays(self): - for combination_strategy in [UnweightedCombination, ExposureWeightedCombination, - UncertaintyWeightedCombination]: + for combination_strategy in [ + UnweightedCombination, + ExposureWeightedCombination, + UncertaintyWeightedCombination, + ]: with self.subTest(strategy=combination_strategy): combination = combination_strategy() self._combine_nan_maps(combination) def test_combine_maps_does_nothing_when_passed_a_single_map(self): - for combination_strategy in [UnweightedCombination, ExposureWeightedCombination, - UncertaintyWeightedCombination]: + for combination_strategy in [ + UnweightedCombination, + ExposureWeightedCombination, + UncertaintyWeightedCombination, + ]: with self.subTest(strategy=combination_strategy): combination = combination_strategy() self._combine_single_map(combination) def test_combine_maps_works_for_maps_with_no_background_data(self): - for combination_strategy in [UnweightedCombination, ExposureWeightedCombination, - UncertaintyWeightedCombination]: + for combination_strategy in [ + UnweightedCombination, + ExposureWeightedCombination, + UncertaintyWeightedCombination, + ]: with self.subTest(strategy=combination_strategy): combination = combination_strategy() self._combine_maps_with_no_background_data(combination) def test_combine_maps_works_for_maps_with_no_intensity_sys_errors(self): - for combination_strategy in [UnweightedCombination, ExposureWeightedCombination, - UncertaintyWeightedCombination]: + for combination_strategy in [ + UnweightedCombination, + ExposureWeightedCombination, + UncertaintyWeightedCombination, + ]: with self.subTest(strategy=combination_strategy): combination = combination_strategy() self._combine_maps_with_no_intensity_sys_errors(combination) def test_check_maps_match(self): - for combination_strategy in [UnweightedCombination, ExposureWeightedCombination, - UncertaintyWeightedCombination]: + for combination_strategy in [ + UnweightedCombination, + ExposureWeightedCombination, + UncertaintyWeightedCombination, + ]: with self.subTest(strategy=combination_strategy): combination = combination_strategy() self._check_maps_match(combination) @@ -64,7 +91,9 @@ def _combine_single_map(self, combination: CombinationStrategy): map_1.ena_intensity_stat_uncert = np.array([1]) combine_one = combination._combine([map_1]) - np.testing.assert_equal(dataclasses.asdict(combine_one), dataclasses.asdict(map_1)) + np.testing.assert_equal( + dataclasses.asdict(combine_one), dataclasses.asdict(map_1) + ) def _combine_maps_with_no_background_data(self, combination: CombinationStrategy): map_1 = construct_intensity_data_with_all_zero_fields() @@ -74,20 +103,26 @@ def _combine_maps_with_no_background_data(self, combination: CombinationStrategy map_1.bg_intensity_stat_uncert = None combine_one = combination._combine([map_1]) - np.testing.assert_equal(dataclasses.asdict(combine_one), dataclasses.asdict(map_1)) + np.testing.assert_equal( + dataclasses.asdict(combine_one), dataclasses.asdict(map_1) + ) - def _combine_maps_with_no_intensity_sys_errors(self, combination: CombinationStrategy): + def _combine_maps_with_no_intensity_sys_errors( + self, combination: CombinationStrategy + ): map_1 = construct_intensity_data_with_all_zero_fields() map_1.ena_intensity_stat_uncert = np.array([1]) map_1.ena_intensity_sys_err_plus = None map_1.ena_intensity_sys_err_minus = None combine_one = combination._combine([map_1]) - np.testing.assert_equal(dataclasses.asdict(combine_one), dataclasses.asdict(map_1)) + np.testing.assert_equal( + dataclasses.asdict(combine_one), dataclasses.asdict(map_1) + ) def _check_maps_match(self, combination: CombinationStrategy): map_1 = construct_intensity_data_with_all_zero_fields() - map_1.ena_intensity_sys_err_plus = np.array([0] * 1), + map_1.ena_intensity_sys_err_plus = (np.array([0] * 1),) map_1.ena_intensity_sys_err_minus = np.array([0] * 2) fields_which_may_differ = { @@ -112,8 +147,12 @@ def _check_maps_match(self, combination: CombinationStrategy): generic_value = 10 for field in dataclasses.fields(map_1): - replacement_value = alternate_values_by_type.get(type(getattr(map_1, field.name)[0]), generic_value) - map_with_difference = dataclasses.replace(map_1, **{field.name: np.array([replacement_value])}) + replacement_value = alternate_values_by_type.get( + type(getattr(map_1, field.name)[0]), generic_value + ) + map_with_difference = dataclasses.replace( + map_1, **{field.name: np.array([replacement_value])} + ) if field.name not in fields_which_may_differ: with self.assertRaises(AssertionError, msg=field.name): combination._check_maps_match([map_1, map_with_difference]) @@ -121,7 +160,9 @@ def _check_maps_match(self, combination: CombinationStrategy): try: combination._check_maps_match([map_1, map_with_difference]) except: - self.fail(f"Differences in other fields should be alright: {field.name}") + self.fail( + f"Differences in other fields should be alright: {field.name}" + ) def test_combine_maps_does_a_time_weighted_average_of_intensity(self): map_1 = construct_intensity_data_with_all_zero_fields() @@ -166,12 +207,15 @@ def test_combine_maps_does_a_time_weighted_average_of_intensity(self): map_2.exposure_factor = np.array([3, 1, 5, 2, 0]) map_2.obs_date = np.ma.masked_equal( - [datetime(2025, 5, 9), - datetime(2025, 5, 10), - datetime(2025, 5, 11), - datetime(2025, 5, 12), - DATETIME_FILL], - DATETIME_FILL) + [ + datetime(2025, 5, 9), + datetime(2025, 5, 10), + datetime(2025, 5, 11), + datetime(2025, 5, 12), + DATETIME_FILL, + ], + DATETIME_FILL, + ) map_2.survival_probability = np.array([0, 0.5, np.nan, 1, 0]) expected_combined_bg_stat_unc = [ @@ -179,7 +223,7 @@ def test_combine_maps_does_a_time_weighted_average_of_intensity(self): np.sqrt((1 * 9) / 1), np.sqrt((25 * 100 + 25 * 4) / 100), np.sqrt((36 * 100 + 4 * 1) / 64), - np.nan + np.nan, ] expected_combined_exposure = [4, 1, 10, 8, 0] @@ -189,14 +233,14 @@ def test_combine_maps_does_a_time_weighted_average_of_intensity(self): (1 * 4) / 1, (5 * 10 + 5 * 2) / (5 + 5), (6 * 100 + 2 * 0) / (6 + 2), - np.nan + np.nan, ] expected_stat_unc = [ np.sqrt((1 * 100 + 9 * 1) / 16), 2, np.sqrt((25 * 100 + 25 * 9) / 100), np.sqrt((36 * 100 + 16 * 4) / 64), - np.nan + np.nan, ] expected_combined_bg_intensity = [7, 7, 4.5, 2, np.nan] @@ -213,7 +257,7 @@ def test_combine_maps_does_a_time_weighted_average_of_intensity(self): (1 * 6) / 1, (5 * 10 + 5 * 7) / (5 + 5), (6 * 10 + 2 * 8) / (6 + 2), - np.nan + np.nan, ] expected_combined_sys_err_plus = [ @@ -221,7 +265,7 @@ def test_combine_maps_does_a_time_weighted_average_of_intensity(self): (1 * 9) / 1, (5 * 11 + 5 * 10) / (5 + 5), (6 * 11 + 2 * 11) / (6 + 2), - np.nan + np.nan, ] expected_obs_date = np.ma.array( @@ -291,7 +335,7 @@ def test_combine_maps_does_a_time_weighted_average_of_intensity(self): ) def test_combine_handles_maps_with_different_patterns_of_missing_intensity_and_bg_intensity( - self, + self, ): map_1 = construct_intensity_data_with_all_zero_fields(num_pixels=5) map_1.ena_intensity = np.array([1, np.nan, np.nan, np.nan, np.nan]) @@ -353,8 +397,10 @@ def test_combine_handles_maps_with_different_patterns_of_missing_intensity_and_b np.testing.assert_array_equal( combined_map.intensity_map_data.bg_intensity, expected_combined_bg_intensity ) - np.testing.assert_array_equal(combined_map.intensity_map_data.bg_intensity_stat_uncert, - expected_combined_bg_stat_unc) + np.testing.assert_array_equal( + combined_map.intensity_map_data.bg_intensity_stat_uncert, + expected_combined_bg_stat_unc, + ) np.testing.assert_array_equal( combined_map.intensity_map_data.bg_intensity_sys_err, expected_combined_bg_intensity_sys_err, @@ -385,9 +431,12 @@ def test_combine_maps_handles_integer_obs_date(self): ) combine_two = exposure_weighted_strategy.combine_rectangular_intensity_map_data( - [rectangular_map_1, rectangular_map_2]) + [rectangular_map_1, rectangular_map_2] + ) - np.testing.assert_equal(combine_two.intensity_map_data.obs_date.mask, expected_obs_date.mask) + np.testing.assert_equal( + combine_two.intensity_map_data.obs_date.mask, expected_obs_date.mask + ) np.testing.assert_equal( combine_two.intensity_map_data.obs_date, expected_obs_date ) @@ -418,47 +467,57 @@ def test_combine_unweighted_combination_strategy(self): DATETIME_FILL, ) map_1.survival_probability = np.array([0.9, 1, 1, 1, np.nan, 1, 1, 1, 1]) - map_1.quality_flags = np.array([ - MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO, - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.NONE, - MapL3Flags.NONE, - MapL3Flags.NONE, - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.NONE, - MapL3Flags.NONE, - MapL3Flags.NONE - ]) + map_1.quality_flags = np.array( + [ + MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO, + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.NONE, + MapL3Flags.NONE, + MapL3Flags.NONE, + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.NONE, + MapL3Flags.NONE, + MapL3Flags.NONE, + ] + ) map_2 = construct_intensity_data_with_all_zero_fields() map_2.ena_intensity = np.array([5, 6, 7, 8, np.nan, 100, 9, 10, 11]) map_2.exposure_factor = np.array([10, 0, 10, 10, 10, 100, 100, np.nan, 30]) - map_2.ena_intensity_sys_err = np.array([4, 1, 4, 5, np.nan, 1, np.nan, np.nan, np.nan]) - map_2.ena_intensity_stat_uncert = np.array([8, 1, 8, 10, np.nan, 1, np.nan, np.nan, np.nan]) + map_2.ena_intensity_sys_err = np.array( + [4, 1, 4, 5, np.nan, 1, np.nan, np.nan, np.nan] + ) + map_2.ena_intensity_stat_uncert = np.array( + [8, 1, 8, 10, np.nan, 1, np.nan, np.nan, np.nan] + ) map_2.obs_date = np.ma.masked_equal( - [datetime(2025, 5, 9), - datetime(2025, 5, 10), - datetime(2025, 5, 11), - datetime(2025, 5, 12), - DATETIME_FILL, - DATETIME_FILL, - DATETIME_FILL, - DATETIME_FILL, - datetime(2025, 5, 13), - ], - DATETIME_FILL) + [ + datetime(2025, 5, 9), + datetime(2025, 5, 10), + datetime(2025, 5, 11), + datetime(2025, 5, 12), + DATETIME_FILL, + DATETIME_FILL, + DATETIME_FILL, + DATETIME_FILL, + datetime(2025, 5, 13), + ], + DATETIME_FILL, + ) map_2.survival_probability = np.array([0.1, 1, 1, 1, 0.2, 0.2, 1, 0.8, 0]) - map_2.quality_flags = np.array([ - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.NONE, - MapL3Flags.PERSISTED_LAST_POINT, - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO, - MapL3Flags.PERSISTED_LAST_POINT, - MapL3Flags.NONE, - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.NONE - ]) + map_2.quality_flags = np.array( + [ + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.NONE, + MapL3Flags.PERSISTED_LAST_POINT, + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO, + MapL3Flags.PERSISTED_LAST_POINT, + MapL3Flags.NONE, + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.NONE, + ] + ) expected_combined_intensity = [3, np.nan, 5, 6, np.nan, 100, 4.5, 5, 12] expected_sys_err = [2.5, np.nan, np.nan, 6.5, np.nan, 1, np.nan, np.nan, np.nan] @@ -474,28 +533,54 @@ def test_combine_unweighted_combination_strategy(self): MapL3Flags.PREDICTIVE_EPHEMERIS | MapL3Flags.PERSISTED_LAST_POINT, MapL3Flags.NONE, MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.NONE] + MapL3Flags.NONE, + ] exposure_unweighted_strategy = UnweightedCombination() - rectangular_map_1: RectangularIntensityMapData = create_rectangular_intensity_map(map_1) - rectangular_map_2: RectangularIntensityMapData = create_rectangular_intensity_map(map_2) + rectangular_map_1: RectangularIntensityMapData = ( + create_rectangular_intensity_map(map_1) + ) + rectangular_map_2: RectangularIntensityMapData = ( + create_rectangular_intensity_map(map_2) + ) - combine_two = exposure_unweighted_strategy.combine_rectangular_intensity_map_data( - [rectangular_map_1, rectangular_map_2]) + combine_two = ( + exposure_unweighted_strategy.combine_rectangular_intensity_map_data( + [rectangular_map_1, rectangular_map_2] + ) + ) - np.testing.assert_equal(combine_two.intensity_map_data.ena_intensity, expected_combined_intensity) - np.testing.assert_equal(combine_two.intensity_map_data.ena_intensity_sys_err, expected_sys_err) - np.testing.assert_equal(combine_two.intensity_map_data.ena_intensity_stat_uncert, expected_stat_unc) - np.testing.assert_equal(combine_two.intensity_map_data.obs_date[-1], expected_obs_date) - np.testing.assert_equal(combine_two.intensity_map_data.survival_probability, expected_survival_probability) - np.testing.assert_equal(combine_two.intensity_map_data.quality_flags, expected_quality_flags) + np.testing.assert_equal( + combine_two.intensity_map_data.ena_intensity, expected_combined_intensity + ) + np.testing.assert_equal( + combine_two.intensity_map_data.ena_intensity_sys_err, expected_sys_err + ) + np.testing.assert_equal( + combine_two.intensity_map_data.ena_intensity_stat_uncert, expected_stat_unc + ) + np.testing.assert_equal( + combine_two.intensity_map_data.obs_date[-1], expected_obs_date + ) + np.testing.assert_equal( + combine_two.intensity_map_data.survival_probability, + expected_survival_probability, + ) + np.testing.assert_equal( + combine_two.intensity_map_data.quality_flags, expected_quality_flags + ) def test_combine_rectangular_intensity_map_data_errors_if_coords_not_matching(self): delta_array = np.array([1]) label_array = np.array(["one"]) - def make_data(lat_delta=delta_array, lon_delta=delta_array, lat_label=label_array, lon_label=label_array): + def make_data( + lat_delta=delta_array, + lon_delta=delta_array, + lat_label=label_array, + lon_label=label_array, + ): return RectangularIntensityMapData( intensity_map_data=construct_intensity_data_with_all_zero_fields(), coords=RectangularCoords( @@ -503,7 +588,7 @@ def make_data(lat_delta=delta_array, lon_delta=delta_array, lat_label=label_arra longitude_delta=lon_delta, latitude_label=lat_label, longitude_label=lon_label, - ) + ), ) base_map = make_data() @@ -517,7 +602,9 @@ def make_data(lat_delta=delta_array, lon_delta=delta_array, lat_label=label_arra with self.subTest(name): with self.assertRaises(AssertionError): strategy = UnweightedCombination() - strategy.combine_rectangular_intensity_map_data([base_map, not_matching_map]) + strategy.combine_rectangular_intensity_map_data( + [base_map, not_matching_map] + ) def test_combine_healpix_intensity_map_data_errors_if_coords_not_matching(self): index_array = np.array([1]) @@ -529,19 +616,29 @@ def make_data(pixel_index=index_array, pixel_index_label=label_array): coords=HealPixCoords( pixel_index=pixel_index, pixel_index_label=pixel_index_label, - ) + ), ) base_map = make_data() cases = [ - ("pixel_index", make_data(pixel_index=np.array([2]), pixel_index_label=label_array)), - ("pixel_index_label", make_data(pixel_index=index_array, pixel_index_label=np.array(["diff label"]))), + ( + "pixel_index", + make_data(pixel_index=np.array([2]), pixel_index_label=label_array), + ), + ( + "pixel_index_label", + make_data( + pixel_index=index_array, pixel_index_label=np.array(["diff label"]) + ), + ), ] for name, not_matching_map in cases: with self.subTest(name): with self.assertRaises(AssertionError): strategy = UnweightedCombination() - strategy.combine_healpix_intensity_map_data([base_map, not_matching_map]) + strategy.combine_healpix_intensity_map_data( + [base_map, not_matching_map] + ) def test_calculate_weighted_sys_err(self): uncertainties_1 = np.array([10, 20, 30, 40]) @@ -551,8 +648,10 @@ def test_calculate_weighted_sys_err(self): exposures_2 = np.array([4, 3, 2, 1]) expected = np.array([15, 30, 45, 60]) - actual = CombinationStrategy.calculate_weighted_sys_err(np.array([uncertainties_1, uncertainties_2]), - np.array([exposures_1, exposures_2])) + actual = CombinationStrategy.calculate_weighted_sys_err( + np.array([uncertainties_1, uncertainties_2]), + np.array([exposures_1, exposures_2]), + ) np.testing.assert_array_equal(actual, expected) def test_uncertainty_weighted_combination(self): @@ -563,12 +662,15 @@ def test_uncertainty_weighted_combination(self): map_1.ena_intensity_stat_uncert = np.array([10, np.nan, 10, 10, np.nan]) DATETIME_FILL = datetime(9999, 12, 31, 23, 59, 59, 999999) map_1.obs_date = np.ma.masked_equal( - [datetime(2025, 5, 5), - DATETIME_FILL, - datetime(2025, 5, 7), - datetime(2025, 5, 8), - DATETIME_FILL], - DATETIME_FILL) + [ + datetime(2025, 5, 5), + DATETIME_FILL, + datetime(2025, 5, 7), + datetime(2025, 5, 8), + DATETIME_FILL, + ], + DATETIME_FILL, + ) map_1.survival_probability = np.array([1, 0.2, 0.8, 1, np.nan]) map_2 = construct_intensity_data_with_all_zero_fields() @@ -577,57 +679,72 @@ def test_uncertainty_weighted_combination(self): map_2.ena_intensity_sys_err = np.array([9, 4, np.inf, 0, np.nan]) map_2.ena_intensity_stat_uncert = np.array([1, 2, np.inf, 4, np.nan]) map_2.obs_date = np.ma.masked_equal( - [datetime(2025, 5, 9), - datetime(2025, 5, 10), - datetime(2025, 5, 11), - datetime(2025, 5, 12), - DATETIME_FILL], - DATETIME_FILL) + [ + datetime(2025, 5, 9), + datetime(2025, 5, 10), + datetime(2025, 5, 11), + datetime(2025, 5, 12), + DATETIME_FILL, + ], + DATETIME_FILL, + ) map_2.survival_probability = np.array([1, 0.5, 0.2, 0.25, np.nan]) expected_combined_exposure = [4, 1, 5, 8, 0] expected_combined_intensity = [ - (1 / (10 ** 2) + 5 / (1 ** 2)) / (1 / (10 ** 2) + 1 / (1 ** 2)), - (6 / (2 ** 2)) / (1 / (2 ** 2)), - (3 / (10 ** 2)) / (1 / (10 ** 2)), - (4 / (10 ** 2) + 8 / (4 ** 2)) / (1 / (10 ** 2) + 1 / (4 ** 2)), - np.nan + (1 / (10**2) + 5 / (1**2)) / (1 / (10**2) + 1 / (1**2)), + (6 / (2**2)) / (1 / (2**2)), + (3 / (10**2)) / (1 / (10**2)), + (4 / (10**2) + 8 / (4**2)) / (1 / (10**2) + 1 / (4**2)), + np.nan, ] expected_stat_unc = [ - np.sqrt(1 / (1 / (10 ** 2) + 1 / (1 ** 2))), - np.sqrt(1 / (1 / (2 ** 2))), - np.sqrt(1 / (1 / (10 ** 2))), - np.sqrt(1 / (1 / (10 ** 2) + 1 / (4 ** 2))), - np.nan + np.sqrt(1 / (1 / (10**2) + 1 / (1**2))), + np.sqrt(1 / (1 / (2**2))), + np.sqrt(1 / (1 / (10**2))), + np.sqrt(1 / (1 / (10**2) + 1 / (4**2))), + np.nan, ] expected_sys_err = [ (1 * 1 + 3 * 9) / (1 + 3), (1 * 4) / 1, (5 * 10) / (5), (6 * 100 + 2 * 0) / (6 + 2), - np.nan + np.nan, ] expected_obs_date = np.ma.array( - [datetime(2025, 5, 8), - datetime(2025, 5, 10), - datetime(2025, 5, 7), - datetime(2025, 5, 9), np.ma.masked]) + [ + datetime(2025, 5, 8), + datetime(2025, 5, 10), + datetime(2025, 5, 7), + datetime(2025, 5, 9), + np.ma.masked, + ] + ) expected_survival_probability = np.array( - [1, 0.5, 0.8, (1 / 10 ** 2 + 0.25 / 4 ** 2) / (1 / 10 ** 2 + 1 / 4 ** 2), np.nan]) + [1, 0.5, 0.8, (1 / 10**2 + 0.25 / 4**2) / (1 / 10**2 + 1 / 4**2), np.nan] + ) exposure_weighted_strategy = UncertaintyWeightedCombination() rectangular_map_1 = create_rectangular_intensity_map(map_1) rectangular_map_2 = create_rectangular_intensity_map(map_2) combine_two = exposure_weighted_strategy.combine_rectangular_intensity_map_data( - [rectangular_map_1, rectangular_map_2]) + [rectangular_map_1, rectangular_map_2] + ) - np.testing.assert_equal(combine_two.intensity_map_data.ena_intensity, expected_combined_intensity) - np.testing.assert_equal(combine_two.intensity_map_data.ena_intensity_sys_err, expected_sys_err) + np.testing.assert_equal( + combine_two.intensity_map_data.ena_intensity, expected_combined_intensity + ) + np.testing.assert_equal( + combine_two.intensity_map_data.ena_intensity_sys_err, expected_sys_err + ) np.testing.assert_equal( combine_two.intensity_map_data.ena_intensity_stat_uncert, expected_stat_unc ) - np.testing.assert_equal(combine_two.intensity_map_data.exposure_factor, expected_combined_exposure) + np.testing.assert_equal( + combine_two.intensity_map_data.exposure_factor, expected_combined_exposure + ) np.testing.assert_equal( combine_two.intensity_map_data.obs_date.mask, expected_obs_date.mask ) @@ -635,14 +752,15 @@ def test_uncertainty_weighted_combination(self): combine_two.intensity_map_data.obs_date, expected_obs_date ) np.testing.assert_equal( - combine_two.intensity_map_data.survival_probability, expected_survival_probability + combine_two.intensity_map_data.survival_probability, + expected_survival_probability, ) def test_combination_handles_missing_sp_data(self): test_cases = [ ExposureWeightedCombination, UncertaintyWeightedCombination, - UnweightedCombination + UnweightedCombination, ] for combination_strategy in test_cases: @@ -653,10 +771,14 @@ def test_combination_handles_missing_sp_data(self): map_2 = construct_intensity_data_with_all_zero_fields() map_2.survival_probability = None - combined_map = combination_strategy().combine_rectangular_intensity_map_data([ - create_rectangular_intensity_map(map_1), - create_rectangular_intensity_map(map_2) - ]) + combined_map = ( + combination_strategy().combine_rectangular_intensity_map_data( + [ + create_rectangular_intensity_map(map_1), + create_rectangular_intensity_map(map_2), + ] + ) + ) self.assertIsNone(combined_map.intensity_map_data.survival_probability) @@ -664,41 +786,53 @@ def test_combination_handles_quality_flag(self): test_cases = [ ExposureWeightedCombination, UncertaintyWeightedCombination, - UnweightedCombination + UnweightedCombination, ] for combination_strategy in test_cases: with self.subTest(combination_strategy.__name__): map_1 = construct_intensity_data_with_all_zero_fields() - map_1.quality_flags = np.array([ - MapL3Flags.NONE, - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.NONE, - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.NONE, - MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO, - ]) + map_1.quality_flags = np.array( + [ + MapL3Flags.NONE, + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.NONE, + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.NONE, + MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO, + ] + ) map_2 = construct_intensity_data_with_all_zero_fields() - map_2.quality_flags = np.array([ - MapL3Flags.NONE, - MapL3Flags.NONE, - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO, - MapL3Flags.PREDICTIVE_EPHEMERIS, - ]) - - combined_map = combination_strategy().combine_rectangular_intensity_map_data([ - create_rectangular_intensity_map(map_1), - create_rectangular_intensity_map(map_2) - ]) - - np.testing.assert_equal(combined_map.intensity_map_data.quality_flags, [ - MapL3Flags.NONE, - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO, - MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO | MapL3Flags.PREDICTIVE_EPHEMERIS, - ]) + map_2.quality_flags = np.array( + [ + MapL3Flags.NONE, + MapL3Flags.NONE, + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO, + MapL3Flags.PREDICTIVE_EPHEMERIS, + ] + ) + + combined_map = ( + combination_strategy().combine_rectangular_intensity_map_data( + [ + create_rectangular_intensity_map(map_1), + create_rectangular_intensity_map(map_2), + ] + ) + ) + + np.testing.assert_equal( + combined_map.intensity_map_data.quality_flags, + [ + MapL3Flags.NONE, + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO, + MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO + | MapL3Flags.PREDICTIVE_EPHEMERIS, + ], + ) diff --git a/tests/maps/test_map_descriptors.py b/tests/maps/test_map_descriptors.py index c7e4f4e10..c0b1ffe54 100644 --- a/tests/maps/test_map_descriptors.py +++ b/tests/maps/test_map_descriptors.py @@ -1,11 +1,19 @@ import unittest -from imap_l3_processing.maps.map_descriptors import ReferenceFrame, SurvivalCorrection, MapDescriptorParts, Sensor, \ - MapQuantity, SpinPhase, PixelSize, parse_map_descriptor, map_descriptor_parts_to_string +from imap_l3_processing.maps.map_descriptors import ( + ReferenceFrame, + SurvivalCorrection, + MapDescriptorParts, + Sensor, + MapQuantity, + SpinPhase, + PixelSize, + parse_map_descriptor, + map_descriptor_parts_to_string, +) class TestMapDescriptors(unittest.TestCase): - def test_parse_map_descriptor(self): cg = ReferenceFrame.Heliospheric no_cg = ReferenceFrame.Spacecraft @@ -13,138 +21,407 @@ def test_parse_map_descriptor(self): no_sp = SurvivalCorrection.NotSurvivalCorrected test_cases = [ - ("h45-ena-h-hf-sp-ram-hae-1deg-3mo", - MapDescriptorParts(sensor=Sensor.Hi45, quantity=MapQuantity.Intensity, quantity_suffix="", - survival_correction=sp, reference_frame=cg, - spin_phase=SpinPhase.RamOnly, coord="hae", - duration="3mo", - grid=PixelSize.OneDegree)), - ("h45-ena-h-sf-sp-anti-hae-4deg-3mo", - MapDescriptorParts(Sensor.Hi45, no_cg, sp, SpinPhase.AntiRamOnly, "hae", - PixelSize.FourDegrees, "3mo", - MapQuantity.Intensity, "")), - ("h90-ena-h-hf-nsp-ram-hae-6deg-1yr", MapDescriptorParts(Sensor.Hi90, cg, no_sp, SpinPhase.RamOnly, "hae", - PixelSize.SixDegrees, "1yr", - MapQuantity.Intensity, "")), - ("h90-ena-h-hf-nsp-full-hae-6deg-6mo", MapDescriptorParts(Sensor.Hi90, cg, no_sp, SpinPhase.FullSpin, "hae", - PixelSize.SixDegrees, "6mo", - MapQuantity.Intensity, "")), - ("h45-spx-h-hf-sp-full-hae-4deg-6mo", MapDescriptorParts(Sensor.Hi45, cg, sp, SpinPhase.FullSpin, "hae", - PixelSize.FourDegrees, "6mo", - MapQuantity.SpectralIndex, "")), - ("hic-ena-h-hf-nsp-full-hae-6deg-6mo", - MapDescriptorParts(Sensor.HiCombined, cg, no_sp, SpinPhase.FullSpin, "hae", - PixelSize.SixDegrees, "6mo", - MapQuantity.Intensity, "")), + ( + "h45-ena-h-hf-sp-ram-hae-1deg-3mo", + MapDescriptorParts( + sensor=Sensor.Hi45, + quantity=MapQuantity.Intensity, + quantity_suffix="", + survival_correction=sp, + reference_frame=cg, + spin_phase=SpinPhase.RamOnly, + coord="hae", + duration="3mo", + grid=PixelSize.OneDegree, + ), + ), + ( + "h45-ena-h-sf-sp-anti-hae-4deg-3mo", + MapDescriptorParts( + Sensor.Hi45, + no_cg, + sp, + SpinPhase.AntiRamOnly, + "hae", + PixelSize.FourDegrees, + "3mo", + MapQuantity.Intensity, + "", + ), + ), + ( + "h90-ena-h-hf-nsp-ram-hae-6deg-1yr", + MapDescriptorParts( + Sensor.Hi90, + cg, + no_sp, + SpinPhase.RamOnly, + "hae", + PixelSize.SixDegrees, + "1yr", + MapQuantity.Intensity, + "", + ), + ), + ( + "h90-ena-h-hf-nsp-full-hae-6deg-6mo", + MapDescriptorParts( + Sensor.Hi90, + cg, + no_sp, + SpinPhase.FullSpin, + "hae", + PixelSize.SixDegrees, + "6mo", + MapQuantity.Intensity, + "", + ), + ), + ( + "h45-spx-h-hf-sp-full-hae-4deg-6mo", + MapDescriptorParts( + Sensor.Hi45, + cg, + sp, + SpinPhase.FullSpin, + "hae", + PixelSize.FourDegrees, + "6mo", + MapQuantity.SpectralIndex, + "", + ), + ), + ( + "hic-ena-h-hf-nsp-full-hae-6deg-6mo", + MapDescriptorParts( + Sensor.HiCombined, + cg, + no_sp, + SpinPhase.FullSpin, + "hae", + PixelSize.SixDegrees, + "6mo", + MapQuantity.Intensity, + "", + ), + ), ( "l090-ena-h-hf-nsp-full-hae-6deg-6mo", - MapDescriptorParts(Sensor.Lo90, cg, no_sp, SpinPhase.FullSpin, "hae", - PixelSize.SixDegrees, "6mo", - MapQuantity.Intensity, "")), - ("l090-ena-h-hk-sp-ram-hae-4deg-1yr", MapDescriptorParts(Sensor.Lo90, ReferenceFrame.HeliosphericKinematic, - sp, SpinPhase.RamOnly, "hae", - PixelSize.FourDegrees, "1yr", - MapQuantity.Intensity, "")), - ("l075-ena-h-hk-sp-ram-hae-4deg-1yr", MapDescriptorParts(Sensor.Lo75, ReferenceFrame.HeliosphericKinematic, - sp, SpinPhase.RamOnly, "hae", - PixelSize.FourDegrees, "1yr", - MapQuantity.Intensity, "")), - ("l105-ena-h-hk-sp-ram-hae-4deg-1yr", MapDescriptorParts(Sensor.Lo105, ReferenceFrame.HeliosphericKinematic, - sp, SpinPhase.RamOnly, "hae", - PixelSize.FourDegrees, "1yr", - MapQuantity.Intensity, "")), - ("u90-spx-h-sf-sp-full-hae-nside8-6mo", MapDescriptorParts(Sensor.Ultra90, ReferenceFrame.Spacecraft, - sp, SpinPhase.FullSpin, "hae", PixelSize.Nside8, - "6mo", MapQuantity.SpectralIndex, "")), - ("u45-spx-h-sf-sp-full-hae-2deg-6mo", MapDescriptorParts(Sensor.Ultra45, ReferenceFrame.Spacecraft, - sp, SpinPhase.FullSpin, "hae", - PixelSize.TwoDegrees, - "6mo", MapQuantity.SpectralIndex, "")), - ("ulc-spx-h-sf-sp-full-hae-nside16-6mo", MapDescriptorParts(Sensor.UltraCombined, ReferenceFrame.Spacecraft, - sp, SpinPhase.FullSpin, "hae", - PixelSize.Nside16, - "6mo", MapQuantity.SpectralIndex, "")), - ("u45-spx-h-sf-sp-full-hae-nside16-6mo", MapDescriptorParts(Sensor.Ultra45, ReferenceFrame.Spacecraft, - sp, SpinPhase.FullSpin, "hae", - PixelSize.Nside16, - "6mo", MapQuantity.SpectralIndex, "")), - ("u45-spx-h-sf-sp-full-custom-nside16-6mo", MapDescriptorParts(Sensor.Ultra45, ReferenceFrame.Spacecraft, - sp, SpinPhase.FullSpin, "custom", - PixelSize.Nside16, - "6mo", MapQuantity.SpectralIndex, "")), - ("ilo-ena-h-sf-sp-ram-imaphae-2deg-3mo", MapDescriptorParts(Sensor.LoCombined, ReferenceFrame.Spacecraft, - sp, SpinPhase.RamOnly, "imaphae", - PixelSize.TwoDegrees, - "3mo", MapQuantity.Intensity, "")), + MapDescriptorParts( + Sensor.Lo90, + cg, + no_sp, + SpinPhase.FullSpin, + "hae", + PixelSize.SixDegrees, + "6mo", + MapQuantity.Intensity, + "", + ), + ), + ( + "l090-ena-h-hk-sp-ram-hae-4deg-1yr", + MapDescriptorParts( + Sensor.Lo90, + ReferenceFrame.HeliosphericKinematic, + sp, + SpinPhase.RamOnly, + "hae", + PixelSize.FourDegrees, + "1yr", + MapQuantity.Intensity, + "", + ), + ), + ( + "l075-ena-h-hk-sp-ram-hae-4deg-1yr", + MapDescriptorParts( + Sensor.Lo75, + ReferenceFrame.HeliosphericKinematic, + sp, + SpinPhase.RamOnly, + "hae", + PixelSize.FourDegrees, + "1yr", + MapQuantity.Intensity, + "", + ), + ), + ( + "l105-ena-h-hk-sp-ram-hae-4deg-1yr", + MapDescriptorParts( + Sensor.Lo105, + ReferenceFrame.HeliosphericKinematic, + sp, + SpinPhase.RamOnly, + "hae", + PixelSize.FourDegrees, + "1yr", + MapQuantity.Intensity, + "", + ), + ), + ( + "u90-spx-h-sf-sp-full-hae-nside8-6mo", + MapDescriptorParts( + Sensor.Ultra90, + ReferenceFrame.Spacecraft, + sp, + SpinPhase.FullSpin, + "hae", + PixelSize.Nside8, + "6mo", + MapQuantity.SpectralIndex, + "", + ), + ), + ( + "u45-spx-h-sf-sp-full-hae-2deg-6mo", + MapDescriptorParts( + Sensor.Ultra45, + ReferenceFrame.Spacecraft, + sp, + SpinPhase.FullSpin, + "hae", + PixelSize.TwoDegrees, + "6mo", + MapQuantity.SpectralIndex, + "", + ), + ), + ( + "ulc-spx-h-sf-sp-full-hae-nside16-6mo", + MapDescriptorParts( + Sensor.UltraCombined, + ReferenceFrame.Spacecraft, + sp, + SpinPhase.FullSpin, + "hae", + PixelSize.Nside16, + "6mo", + MapQuantity.SpectralIndex, + "", + ), + ), + ( + "u45-spx-h-sf-sp-full-hae-nside16-6mo", + MapDescriptorParts( + Sensor.Ultra45, + ReferenceFrame.Spacecraft, + sp, + SpinPhase.FullSpin, + "hae", + PixelSize.Nside16, + "6mo", + MapQuantity.SpectralIndex, + "", + ), + ), + ( + "u45-spx-h-sf-sp-full-custom-nside16-6mo", + MapDescriptorParts( + Sensor.Ultra45, + ReferenceFrame.Spacecraft, + sp, + SpinPhase.FullSpin, + "custom", + PixelSize.Nside16, + "6mo", + MapQuantity.SpectralIndex, + "", + ), + ), + ( + "ilo-ena-h-sf-sp-ram-imaphae-2deg-3mo", + MapDescriptorParts( + Sensor.LoCombined, + ReferenceFrame.Spacecraft, + sp, + SpinPhase.RamOnly, + "imaphae", + PixelSize.TwoDegrees, + "3mo", + MapQuantity.Intensity, + "", + ), + ), ("not-valid-at-all", None), ("invalid_prefix-hic-ena-h-hf-nsp-full-hae-6deg-6mo", None), ("hic-ena-h-hf-nsp-full-hae-6deg-6mo-invalid-suffix", None), ("hic-ena-h-hf-nsp-full-imap?-6deg-6mo-invalid-suffix", None), ("hic-ena-h-hf-nsp-full-imap_v1-6deg-6mo-invalid-suffix", None), - ("hic-spxwItHcUsToMsUfFiX-h-sf-sp-full-hae-6deg-6mo", MapDescriptorParts(Sensor.HiCombined, - ReferenceFrame.Spacecraft, sp, - SpinPhase.FullSpin, "hae", - PixelSize.SixDegrees, - "6mo", - MapQuantity.SpectralIndex, - "wItHcUsToMsUfFiX")), - ("hic-enaCUSTOM-h-sf-sp-full-CUSTOM1-6deg-6mo", MapDescriptorParts(Sensor.HiCombined, - ReferenceFrame.Spacecraft, sp, - SpinPhase.FullSpin, "CUSTOM1", - PixelSize.SixDegrees, - "6mo", - MapQuantity.Intensity, "CUSTOM")), - ("hic-enaCUSTOM-h-sf-sp-full-imapeclipj2000-6deg-6mo", MapDescriptorParts(Sensor.HiCombined, - ReferenceFrame.Spacecraft, sp, - SpinPhase.FullSpin, - "imapeclipj2000", - PixelSize.SixDegrees, - "6mo", - MapQuantity.Intensity, "CUSTOM")), - ("hic-enaCUSTOM-h-sf-sp-full-hae-6deg-0mo", MapDescriptorParts(Sensor.HiCombined, - ReferenceFrame.Spacecraft, sp, - SpinPhase.FullSpin, "hae", - PixelSize.SixDegrees, - "0mo", - MapQuantity.Intensity, "CUSTOM")), - ("hic-enaCUSTOM-h-sf-sp-full-hae-6deg-50yr", MapDescriptorParts(Sensor.HiCombined, - ReferenceFrame.Spacecraft, sp, - SpinPhase.FullSpin, "hae", - PixelSize.SixDegrees, - "50yr", - MapQuantity.Intensity, "CUSTOM")), - ("l090-isn-h-sf-nsp-ram-hae-6deg-1yr", MapDescriptorParts(Sensor.Lo90, ReferenceFrame.Spacecraft, - no_sp, SpinPhase.RamOnly, "hae", - PixelSize.SixDegrees, - "1yr", - MapQuantity.ISNBackgroundSubtracted, "")), - ("l090-spxnbs-h-sf-nsp-ram-hae-6deg-1yr", MapDescriptorParts(Sensor.Lo90, ReferenceFrame.Spacecraft, - no_sp, SpinPhase.RamOnly, "hae", - PixelSize.SixDegrees, - "1yr", - MapQuantity.SpectralIndexNBS, "")), - ("l090-spxnbs1111-h-sf-nsp-ram-hae-6deg-1yr", MapDescriptorParts(Sensor.Lo90, ReferenceFrame.Spacecraft, - no_sp, SpinPhase.RamOnly, "hae", - PixelSize.SixDegrees, - "1yr", - MapQuantity.SpectralIndexNBS, "", (11, 11))), - ("l090-spxnbs1111thisisdense-h-sf-nsp-ram-hae-6deg-1yr", MapDescriptorParts(Sensor.Lo90, ReferenceFrame.Spacecraft, - no_sp, SpinPhase.RamOnly, "hae", - PixelSize.SixDegrees, - "1yr", - MapQuantity.SpectralIndexNBS, "thisisdense", - (11, 11))), - ("l090-spxnbs-h-hk-nsp-ram-hae-6deg-1yr", MapDescriptorParts(Sensor.Lo90, ReferenceFrame.HeliosphericKinematic, - no_sp, SpinPhase.RamOnly, "hae", - PixelSize.SixDegrees, - "1yr", - MapQuantity.SpectralIndexNBS, "")), - ("h45-spx0607-h-hf-sp-full-hae-4deg-6mo", MapDescriptorParts(Sensor.Hi45, cg, sp, SpinPhase.FullSpin, "hae", - PixelSize.FourDegrees, "6mo", - MapQuantity.SpectralIndex, "", (6, 7))), - ("h45-spx1234customsuffix-h-hf-sp-full-hae-4deg-6mo", MapDescriptorParts(Sensor.Hi45, cg, sp, SpinPhase.FullSpin, "hae", - PixelSize.FourDegrees, "6mo", - MapQuantity.SpectralIndex, "customsuffix", (12, 34))), + ( + "hic-spxwItHcUsToMsUfFiX-h-sf-sp-full-hae-6deg-6mo", + MapDescriptorParts( + Sensor.HiCombined, + ReferenceFrame.Spacecraft, + sp, + SpinPhase.FullSpin, + "hae", + PixelSize.SixDegrees, + "6mo", + MapQuantity.SpectralIndex, + "wItHcUsToMsUfFiX", + ), + ), + ( + "hic-enaCUSTOM-h-sf-sp-full-CUSTOM1-6deg-6mo", + MapDescriptorParts( + Sensor.HiCombined, + ReferenceFrame.Spacecraft, + sp, + SpinPhase.FullSpin, + "CUSTOM1", + PixelSize.SixDegrees, + "6mo", + MapQuantity.Intensity, + "CUSTOM", + ), + ), + ( + "hic-enaCUSTOM-h-sf-sp-full-imapeclipj2000-6deg-6mo", + MapDescriptorParts( + Sensor.HiCombined, + ReferenceFrame.Spacecraft, + sp, + SpinPhase.FullSpin, + "imapeclipj2000", + PixelSize.SixDegrees, + "6mo", + MapQuantity.Intensity, + "CUSTOM", + ), + ), + ( + "hic-enaCUSTOM-h-sf-sp-full-hae-6deg-0mo", + MapDescriptorParts( + Sensor.HiCombined, + ReferenceFrame.Spacecraft, + sp, + SpinPhase.FullSpin, + "hae", + PixelSize.SixDegrees, + "0mo", + MapQuantity.Intensity, + "CUSTOM", + ), + ), + ( + "hic-enaCUSTOM-h-sf-sp-full-hae-6deg-50yr", + MapDescriptorParts( + Sensor.HiCombined, + ReferenceFrame.Spacecraft, + sp, + SpinPhase.FullSpin, + "hae", + PixelSize.SixDegrees, + "50yr", + MapQuantity.Intensity, + "CUSTOM", + ), + ), + ( + "l090-isn-h-sf-nsp-ram-hae-6deg-1yr", + MapDescriptorParts( + Sensor.Lo90, + ReferenceFrame.Spacecraft, + no_sp, + SpinPhase.RamOnly, + "hae", + PixelSize.SixDegrees, + "1yr", + MapQuantity.ISNBackgroundSubtracted, + "", + ), + ), + ( + "l090-spxnbs-h-sf-nsp-ram-hae-6deg-1yr", + MapDescriptorParts( + Sensor.Lo90, + ReferenceFrame.Spacecraft, + no_sp, + SpinPhase.RamOnly, + "hae", + PixelSize.SixDegrees, + "1yr", + MapQuantity.SpectralIndexNBS, + "", + ), + ), + ( + "l090-spxnbs1111-h-sf-nsp-ram-hae-6deg-1yr", + MapDescriptorParts( + Sensor.Lo90, + ReferenceFrame.Spacecraft, + no_sp, + SpinPhase.RamOnly, + "hae", + PixelSize.SixDegrees, + "1yr", + MapQuantity.SpectralIndexNBS, + "", + (11, 11), + ), + ), + ( + "l090-spxnbs1111thisisdense-h-sf-nsp-ram-hae-6deg-1yr", + MapDescriptorParts( + Sensor.Lo90, + ReferenceFrame.Spacecraft, + no_sp, + SpinPhase.RamOnly, + "hae", + PixelSize.SixDegrees, + "1yr", + MapQuantity.SpectralIndexNBS, + "thisisdense", + (11, 11), + ), + ), + ( + "l090-spxnbs-h-hk-nsp-ram-hae-6deg-1yr", + MapDescriptorParts( + Sensor.Lo90, + ReferenceFrame.HeliosphericKinematic, + no_sp, + SpinPhase.RamOnly, + "hae", + PixelSize.SixDegrees, + "1yr", + MapQuantity.SpectralIndexNBS, + "", + ), + ), + ( + "h45-spx0607-h-hf-sp-full-hae-4deg-6mo", + MapDescriptorParts( + Sensor.Hi45, + cg, + sp, + SpinPhase.FullSpin, + "hae", + PixelSize.FourDegrees, + "6mo", + MapQuantity.SpectralIndex, + "", + (6, 7), + ), + ), + ( + "h45-spx1234customsuffix-h-hf-sp-full-hae-4deg-6mo", + MapDescriptorParts( + Sensor.Hi45, + cg, + sp, + SpinPhase.FullSpin, + "hae", + PixelSize.FourDegrees, + "6mo", + MapQuantity.SpectralIndex, + "customsuffix", + (12, 34), + ), + ), ("h45-spx012-h-hf-sp-full-hae-4deg-6mo", None), ("h45-spx12345-h-hf-sp-full-hae-4deg-6mo", None), ] @@ -155,8 +432,10 @@ def test_parse_map_descriptor(self): self.assertEqual(expected, descriptor_parts) if descriptor_parts is not None: - self.assertEqual(descriptor, map_descriptor_parts_to_string(descriptor_parts)) + self.assertEqual( + descriptor, map_descriptor_parts_to_string(descriptor_parts) + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/maps/test_map_models.py b/tests/maps/test_map_models.py index 2298dc033..b802bef55 100644 --- a/tests/maps/test_map_models.py +++ b/tests/maps/test_map_models.py @@ -14,9 +14,16 @@ from spacepy.pycdf import CDF from xarray import Dataset -from imap_l3_processing.cdf.cdf_utils import read_variable_and_mask_fill_values, read_numeric_variable -from imap_l3_processing.constants import ONE_SECOND_IN_NANOSECONDS, SECONDS_PER_DAY, FIVE_MINUTES_IN_NANOSECONDS, \ - TT2000_EPOCH +from imap_l3_processing.cdf.cdf_utils import ( + read_variable_and_mask_fill_values, + read_numeric_variable, +) +from imap_l3_processing.constants import ( + ONE_SECOND_IN_NANOSECONDS, + SECONDS_PER_DAY, + FIVE_MINUTES_IN_NANOSECONDS, + TT2000_EPOCH, +) from imap_l3_processing.maps import map_models from imap_l3_processing.maps.map_models import ( RectangularCoords, @@ -46,7 +53,6 @@ class TestMapModels(unittest.TestCase): - def test_rectangular_spectral_index_to_data_product_variables(self): input_metadata = Mock() @@ -128,8 +134,7 @@ def test_rectangular_spectral_index_to_data_product_variables(self): sentinel.ena_spectral_index_chisq, ), DataProductVariable( - map_models.QUALITY_FLAGS_VAR_NAME, - sentinel.quality_flags + map_models.QUALITY_FLAGS_VAR_NAME, sentinel.quality_flags ), DataProductVariable( map_models.LATITUDE_DELTA_VAR_NAME, sentinel.latitude_delta @@ -203,9 +208,9 @@ def test_rectangular_intensity_to_data_product_variables(self): ] for ( - test_name, - additional_inputs, - expected_additional_data_products_vars, + test_name, + additional_inputs, + expected_additional_data_products_vars, ) in test_cases: with self.subTest(test_name): input_metadata = sentinel.input_metadata @@ -230,15 +235,14 @@ def test_rectangular_intensity_to_data_product_variables(self): ena_intensity_stat_uncert=sentinel.ena_intensity_stat_uncert, ena_intensity_sys_err=sentinel.ena_intensity_sys_err, quality_flags=sentinel.quality_flags, - **additional_inputs + **additional_inputs, ), coords=RectangularCoords( longitude_delta=sentinel.longitude_delta, longitude_label=sentinel.longitude_label, latitude_delta=sentinel.latitude_delta, latitude_label=sentinel.latitude_label, - - ) + ), ), spice_frame_name=SpiceFrame.ECLIPJ2000, ) @@ -402,7 +406,9 @@ def test_isn_background_subtracted_to_data_product_variables(self): self.assertEqual(expected_variables, actual_variables) def test_healpix_map_nside_property(self): - path_to_cdf = get_test_data_folder() / 'ultra' / 'fake_l2_maps' / 'test_l2_map.cdf' + path_to_cdf = ( + get_test_data_folder() / "ultra" / "fake_l2_maps" / "test_l2_map.cdf" + ) actual = HealPixIntensityMapData.read_from_path(path_to_cdf) expected_nside = 2 @@ -410,41 +416,88 @@ def test_healpix_map_nside_property(self): self.assertEqual(expected_nside, actual.coords.nside) def test_ultra_l2_map_read_from_file(self): - path_to_cdf = get_test_data_folder() / 'ultra' / 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-2deg-6mo_20250415_v102.cdf' + path_to_cdf = ( + get_test_data_folder() + / "ultra" + / "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-2deg-6mo_20250415_v102.cdf" + ) map_data = RectangularIntensityMapData.read_from_path(path_to_cdf) actual_intensity_data = map_data.intensity_map_data actual_coords = map_data.coords with CDF(str(path_to_cdf)) as expected: - date_range_var = read_variable_and_mask_fill_values(expected["obs_date_range"]) + date_range_var = read_variable_and_mask_fill_values( + expected["obs_date_range"] + ) obs_date = convert_tt2000_time_to_datetime( - read_variable_and_mask_fill_values(expected["obs_date"]).filled(0)) - - self.assertEqual(expected['epoch'][...], actual_intensity_data.epoch) - np.testing.assert_array_equal(expected["epoch_delta"][...], actual_intensity_data.epoch_delta) - np.testing.assert_array_equal(expected["energy"][...], actual_intensity_data.energy) - np.testing.assert_array_equal(expected["energy_delta_plus"][...], actual_intensity_data.energy_delta_plus) - np.testing.assert_array_equal(expected["energy_delta_minus"][...], actual_intensity_data.energy_delta_minus) - np.testing.assert_array_equal(expected["energy_label"][...], actual_intensity_data.energy_label) - np.testing.assert_array_equal(expected["latitude"][...], actual_intensity_data.latitude) - np.testing.assert_array_equal(expected["latitude_delta"][...], actual_coords.latitude_delta) - np.testing.assert_array_equal(expected["latitude_label"][...], actual_coords.latitude_label) - np.testing.assert_array_equal(expected["longitude"][...], actual_intensity_data.longitude) - np.testing.assert_array_equal(expected["longitude_delta"][...], actual_coords.longitude_delta) - np.testing.assert_array_equal(expected["longitude_label"][...], actual_coords.longitude_label) - np.testing.assert_array_equal(expected["exposure_factor"][...], actual_intensity_data.exposure_factor) + read_variable_and_mask_fill_values(expected["obs_date"]).filled(0) + ) + + self.assertEqual(expected["epoch"][...], actual_intensity_data.epoch) + np.testing.assert_array_equal( + expected["epoch_delta"][...], actual_intensity_data.epoch_delta + ) + np.testing.assert_array_equal( + expected["energy"][...], actual_intensity_data.energy + ) + np.testing.assert_array_equal( + expected["energy_delta_plus"][...], + actual_intensity_data.energy_delta_plus, + ) + np.testing.assert_array_equal( + expected["energy_delta_minus"][...], + actual_intensity_data.energy_delta_minus, + ) + np.testing.assert_array_equal( + expected["energy_label"][...], actual_intensity_data.energy_label + ) + np.testing.assert_array_equal( + expected["latitude"][...], actual_intensity_data.latitude + ) + np.testing.assert_array_equal( + expected["latitude_delta"][...], actual_coords.latitude_delta + ) + np.testing.assert_array_equal( + expected["latitude_label"][...], actual_coords.latitude_label + ) + np.testing.assert_array_equal( + expected["longitude"][...], actual_intensity_data.longitude + ) + np.testing.assert_array_equal( + expected["longitude_delta"][...], actual_coords.longitude_delta + ) + np.testing.assert_array_equal( + expected["longitude_label"][...], actual_coords.longitude_label + ) + np.testing.assert_array_equal( + expected["exposure_factor"][...], actual_intensity_data.exposure_factor + ) np.testing.assert_array_equal(obs_date, actual_intensity_data.obs_date) - np.testing.assert_array_equal(date_range_var, actual_intensity_data.obs_date_range) - np.testing.assert_array_equal(expected["solid_angle"][...], actual_intensity_data.solid_angle) - np.testing.assert_array_equal(expected["ena_intensity"][...], actual_intensity_data.ena_intensity) - np.testing.assert_array_equal(expected["ena_intensity_stat_uncert"][...], - actual_intensity_data.ena_intensity_stat_uncert) - np.testing.assert_array_equal(expected["ena_intensity_sys_err"][...], - actual_intensity_data.ena_intensity_sys_err) + np.testing.assert_array_equal( + date_range_var, actual_intensity_data.obs_date_range + ) + np.testing.assert_array_equal( + expected["solid_angle"][...], actual_intensity_data.solid_angle + ) + np.testing.assert_array_equal( + expected["ena_intensity"][...], actual_intensity_data.ena_intensity + ) + np.testing.assert_array_equal( + expected["ena_intensity_stat_uncert"][...], + actual_intensity_data.ena_intensity_stat_uncert, + ) + np.testing.assert_array_equal( + expected["ena_intensity_sys_err"][...], + actual_intensity_data.ena_intensity_sys_err, + ) def test_ultra_healpix_intensity_read_from_xarray(self): - full_shape = [CoordNames.TIME.value, CoordNames.ENERGY_L2.value, CoordNames.HEALPIX_INDEX.value] + full_shape = [ + CoordNames.TIME.value, + CoordNames.ENERGY_L2.value, + CoordNames.HEALPIX_INDEX.value, + ] input_xarray: Dataset = Dataset( data_vars={ "latitude": ([CoordNames.HEALPIX_INDEX.value], np.full((12,), 4)), @@ -524,7 +577,10 @@ def test_ultra_healpix_intensity_read_from_xarray(self): np.testing.assert_array_equal( input_xarray[CoordNames.ENERGY_L2.value], output.intensity_map_data.energy ) - np.testing.assert_array_equal(input_xarray["energy_delta_minus"], output.intensity_map_data.energy_delta_minus) + np.testing.assert_array_equal( + input_xarray["energy_delta_minus"], + output.intensity_map_data.energy_delta_minus, + ) np.testing.assert_array_equal( input_xarray["energy_delta_plus"], output.intensity_map_data.energy_delta_plus, @@ -532,7 +588,11 @@ def test_ultra_healpix_intensity_read_from_xarray(self): np.testing.assert_array_equal( input_xarray["energy_label"], output.intensity_map_data.energy_label ) - np.testing.assert_array_equal(output.intensity_map_data.quality_flags, np.full((2, 15, 12),MapL3Flags.NONE), strict=True) + np.testing.assert_array_equal( + output.intensity_map_data.quality_flags, + np.full((2, 15, 12), MapL3Flags.NONE), + strict=True, + ) def test_read_intensity_map_with_rectangular_cords_data_from_cdf(self): @@ -548,7 +608,7 @@ def test_read_intensity_map_with_rectangular_cords_data_from_cdf(self): obs_date_datetime, np.full(map_data_shape, False), False, - True + True, ), ( "obs date is int", @@ -556,7 +616,7 @@ def test_read_intensity_map_with_rectangular_cords_data_from_cdf(self): np.full(map_data_shape, TT2000_EPOCH) + timedelta(seconds=1), np.full(map_data_shape, False), True, - False + False, ), ( "obs date is all fill", @@ -564,14 +624,21 @@ def test_read_intensity_map_with_rectangular_cords_data_from_cdf(self): np.full(map_data_shape, TT2000_EPOCH), np.full(map_data_shape, True), True, - True + True, ), ] - for test_name, obs_date_in_cdf, expected_obs_date, expected_obs_date_mask, include_bg, include_ena_sys_err_minus in test_cases: + for ( + test_name, + obs_date_in_cdf, + expected_obs_date, + expected_obs_date_mask, + include_bg, + include_ena_sys_err_minus, + ) in test_cases: with tempfile.TemporaryDirectory() as temp_dir: pathname = os.path.join(temp_dir, "test_cdf") - with CDF(pathname, '') as cdf: + with CDF(pathname, "") as cdf: cdf.col_major(True) ena_intensity = rng.random(map_data_shape) @@ -602,7 +669,10 @@ def test_read_intensity_map_with_rectangular_cords_data_from_cdf(self): lon_label = [f"{x} deg" for x in lon] obs_date = obs_date_in_cdf - obs_date_range = np.full(ena_intensity.shape, ONE_SECOND_IN_NANOSECONDS * SECONDS_PER_DAY * 2) + obs_date_range = np.full( + ena_intensity.shape, + ONE_SECOND_IN_NANOSECONDS * SECONDS_PER_DAY * 2, + ) solid_angle = build_solid_angle_map(4) solid_angle = solid_angle[np.newaxis, ...] @@ -615,8 +685,14 @@ def test_read_intensity_map_with_rectangular_cords_data_from_cdf(self): cdf.new("longitude_delta", lon_delta, recVary=False) cdf.new("longitude_label", lon_label, recVary=False) cdf.new("ena_intensity", ena_intensity, recVary=True) - cdf.new("ena_intensity_stat_uncert", ena_intensity_stat_uncert, recVary=True) - cdf.new("ena_intensity_sys_err", ena_intensity_sys_err, recVary=True) + cdf.new( + "ena_intensity_stat_uncert", + ena_intensity_stat_uncert, + recVary=True, + ) + cdf.new( + "ena_intensity_sys_err", ena_intensity_sys_err, recVary=True + ) cdf.new("exposure_factor", exposure, recVary=True) cdf.new("obs_date", obs_date, recVary=True) cdf.new("obs_date_range", obs_date_range, recVary=True) @@ -628,15 +704,29 @@ def test_read_intensity_map_with_rectangular_cords_data_from_cdf(self): cdf.new("survival_probability", survival_probability, recVary=False) if include_bg: cdf.new("bg_intensity", bg_intensity, recVary=True) - cdf.new("bg_intensity_stat_uncert", bg_intensity_stat_unc, recVary=True) - cdf.new("bg_intensity_sys_err", bg_intensity_sys_err, recVary=True) + cdf.new( + "bg_intensity_stat_uncert", + bg_intensity_stat_unc, + recVary=True, + ) + cdf.new( + "bg_intensity_sys_err", bg_intensity_sys_err, recVary=True + ) if include_ena_sys_err_minus: - cdf.new("ena_intensity_sys_err_minus", ena_intensity_sys_err_minus, recVary=True) - cdf.new("ena_intensity_sys_err_plus", ena_intensity_sys_err_plus, recVary=True) + cdf.new( + "ena_intensity_sys_err_minus", + ena_intensity_sys_err_minus, + recVary=True, + ) + cdf.new( + "ena_intensity_sys_err_plus", + ena_intensity_sys_err_plus, + recVary=True, + ) for var in cdf: - cdf[var].attrs['FILLVAL'] = 1000000 + cdf[var].attrs["FILLVAL"] = 1000000 cdf["obs_date"].attrs["FILLVAL"] = obs_date_fillval @@ -651,7 +741,9 @@ def test_read_intensity_map_with_rectangular_cords_data_from_cdf(self): np.testing.assert_array_equal(map_data.epoch, epoch) np.testing.assert_array_equal(map_data.epoch_delta, epoch_delta) np.testing.assert_array_equal(map_data.energy, energy) - np.testing.assert_array_equal(map_data.energy_delta_plus, energy_delta_plus) + np.testing.assert_array_equal( + map_data.energy_delta_plus, energy_delta_plus + ) np.testing.assert_array_equal( map_data.energy_delta_minus, energy_delta_minus ) @@ -662,10 +754,16 @@ def test_read_intensity_map_with_rectangular_cords_data_from_cdf(self): np.testing.assert_array_equal( rectangular_coords.latitude_delta, lat_delta ) - np.testing.assert_array_equal(rectangular_coords.latitude_label, lat_label) + np.testing.assert_array_equal( + rectangular_coords.latitude_label, lat_label + ) np.testing.assert_array_equal(map_data.longitude, lon) - np.testing.assert_array_equal(rectangular_coords.longitude_delta, lon_delta) - np.testing.assert_array_equal(rectangular_coords.longitude_label, lon_label) + np.testing.assert_array_equal( + rectangular_coords.longitude_delta, lon_delta + ) + np.testing.assert_array_equal( + rectangular_coords.longitude_label, lon_label + ) np.testing.assert_array_equal( map_data.ena_intensity, ena_intensity ) @@ -673,11 +771,21 @@ def test_read_intensity_map_with_rectangular_cords_data_from_cdf(self): map_data.ena_intensity_stat_uncert, ena_intensity_stat_uncert, ) - np.testing.assert_array_equal(map_data.ena_intensity_sys_err, ena_intensity_sys_err) - np.testing.assert_array_equal(map_data.exposure_factor, exposure) - np.testing.assert_array_equal(map_data.obs_date.data, expected_obs_date) - np.testing.assert_array_equal(map_data.obs_date.mask, expected_obs_date_mask) - np.testing.assert_array_equal(map_data.obs_date_range, obs_date_range) + np.testing.assert_array_equal( + map_data.ena_intensity_sys_err, ena_intensity_sys_err + ) + np.testing.assert_array_equal( + map_data.exposure_factor, exposure + ) + np.testing.assert_array_equal( + map_data.obs_date.data, expected_obs_date + ) + np.testing.assert_array_equal( + map_data.obs_date.mask, expected_obs_date_mask + ) + np.testing.assert_array_equal( + map_data.obs_date_range, obs_date_range + ) np.testing.assert_array_equal(map_data.solid_angle, solid_angle) np.testing.assert_array_equal( map_data.survival_probability, survival_probability @@ -727,17 +835,21 @@ def test_read_intensity_map_with_rectangular_coords_data_from_cdf_missing_flags( input_quality_flags[0, :, 83, :] = MapL3Flags.PREDICTIVE_EPHEMERIS input_quality_flags[0, :, 84, :] = MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO input_quality_flags[0, :, 85, :] = MapL3Flags.PERSISTED_LAST_POINT - input_quality_flags[0, :, 86, :] = MapL3Flags.PREDICTIVE_EPHEMERIS | MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO | MapL3Flags.PERSISTED_LAST_POINT + input_quality_flags[0, :, 86, :] = ( + MapL3Flags.PREDICTIVE_EPHEMERIS + | MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO + | MapL3Flags.PERSISTED_LAST_POINT + ) cases = ( ("includes quality flags", input_quality_flags, input_quality_flags), - ("does not include quality flags", None, np.full(map_data_shape, MapL3Flags.NONE)), + ( + "does not include quality flags", + None, + np.full(map_data_shape, MapL3Flags.NONE), + ), ) - for ( - case_name, - input_quality_flags, - expected_quality_flags - ) in cases: + for case_name, input_quality_flags, expected_quality_flags in cases: with tempfile.TemporaryDirectory() as temp_dir, self.subTest(case_name): pathname = os.path.join(temp_dir, "test_cdf") with CDF(pathname, "") as cdf: @@ -820,7 +932,9 @@ def test_read_intensity_map_with_rectangular_coords_data_from_cdf_missing_flags( recVary=True, ) if input_quality_flags is not None: - cdf.new(QUALITY_FLAGS_VAR_NAME, input_quality_flags, recVary=True) + cdf.new( + QUALITY_FLAGS_VAR_NAME, input_quality_flags, recVary=True + ) for var in cdf: cdf[var].attrs["FILLVAL"] = 1000000 @@ -880,13 +994,22 @@ def test_read_intensity_map_with_rectangular_coords_data_from_cdf_missing_flags( map_data.survival_probability, survival_probability ) np.testing.assert_array_equal(map_data.bg_intensity, bg_intensity) - np.testing.assert_array_equal(map_data.bg_intensity_sys_err, bg_intensity_sys_err) - np.testing.assert_array_equal(map_data.bg_intensity_stat_uncert, bg_intensity_stat_unc) - np.testing.assert_array_equal(map_data.ena_intensity_sys_err_minus, - ena_intensity_sys_err_minus) - np.testing.assert_array_equal(map_data.ena_intensity_sys_err_plus, - ena_intensity_sys_err_plus) - np.testing.assert_array_equal(map_data.quality_flags, expected_quality_flags, strict=True) + np.testing.assert_array_equal( + map_data.bg_intensity_sys_err, bg_intensity_sys_err + ) + np.testing.assert_array_equal( + map_data.bg_intensity_stat_uncert, bg_intensity_stat_unc + ) + np.testing.assert_array_equal( + map_data.ena_intensity_sys_err_minus, + ena_intensity_sys_err_minus, + ) + np.testing.assert_array_equal( + map_data.ena_intensity_sys_err_plus, ena_intensity_sys_err_plus + ) + np.testing.assert_array_equal( + map_data.quality_flags, expected_quality_flags, strict=True + ) def test_fill_values_in_read_rectangular_intensity_map_data_from_cdf(self): path = get_test_data_folder() / "hi" / "map_with_fill_values.cdf" @@ -903,10 +1026,20 @@ def test_fill_values_in_read_rectangular_intensity_map_data_from_cdf(self): self.assertTrue(np.all(map_data.obs_date.mask)) self.assertTrue(np.all(map_data.obs_date_range.mask)) - np.testing.assert_array_equal(map_data.energy, np.full_like(cdf["energy"], np.nan)) - np.testing.assert_array_equal(map_data.energy_delta_plus, np.full_like(cdf["energy_delta_plus"], np.nan)) - np.testing.assert_array_equal(map_data.energy_delta_minus, np.full_like(cdf["energy_delta_minus"], np.nan)) - np.testing.assert_array_equal(map_data.latitude, np.full_like(cdf["latitude"], np.nan)) + np.testing.assert_array_equal( + map_data.energy, np.full_like(cdf["energy"], np.nan) + ) + np.testing.assert_array_equal( + map_data.energy_delta_plus, + np.full_like(cdf["energy_delta_plus"], np.nan), + ) + np.testing.assert_array_equal( + map_data.energy_delta_minus, + np.full_like(cdf["energy_delta_minus"], np.nan), + ) + np.testing.assert_array_equal( + map_data.latitude, np.full_like(cdf["latitude"], np.nan) + ) np.testing.assert_array_equal( map_data.longitude, np.full_like(cdf["longitude"], np.nan) ) @@ -917,13 +1050,23 @@ def test_fill_values_in_read_rectangular_intensity_map_data_from_cdf(self): map_data.ena_intensity_stat_uncert, np.full_like(cdf["ena_intensity_stat_uncert"], np.nan), ) - np.testing.assert_array_equal(map_data.ena_intensity_sys_err, - np.full_like(cdf["ena_intensity_sys_err"], np.nan)) - np.testing.assert_array_equal(map_data.exposure_factor, np.full_like(cdf["exposure_factor"], np.nan)) - np.testing.assert_array_equal(map_data.solid_angle, np.full_like(cdf["solid_angle"], np.nan)) - np.testing.assert_array_equal(map_data.bg_intensity, np.full_like(cdf["bg_intensity"], np.nan)) - np.testing.assert_array_equal(map_data.bg_intensity_sys_err, - np.full_like(cdf["bg_intensity_sys_err"], np.nan)) + np.testing.assert_array_equal( + map_data.ena_intensity_sys_err, + np.full_like(cdf["ena_intensity_sys_err"], np.nan), + ) + np.testing.assert_array_equal( + map_data.exposure_factor, np.full_like(cdf["exposure_factor"], np.nan) + ) + np.testing.assert_array_equal( + map_data.solid_angle, np.full_like(cdf["solid_angle"], np.nan) + ) + np.testing.assert_array_equal( + map_data.bg_intensity, np.full_like(cdf["bg_intensity"], np.nan) + ) + np.testing.assert_array_equal( + map_data.bg_intensity_sys_err, + np.full_like(cdf["bg_intensity_sys_err"], np.nan), + ) np.testing.assert_array_equal( map_data.bg_intensity_stat_uncert, np.full_like(cdf["bg_intensity_stat_uncert"], np.nan), @@ -933,10 +1076,12 @@ def test_fill_values_in_read_rectangular_intensity_map_data_from_cdf(self): np.full_like(cdf["survival_probability"], np.nan), ) np.testing.assert_array_equal( - map_data.ena_intensity_sys_err_plus, np.full_like(cdf["ena_intensity_sys_err_plus"], np.nan) + map_data.ena_intensity_sys_err_plus, + np.full_like(cdf["ena_intensity_sys_err_plus"], np.nan), ) np.testing.assert_array_equal( - map_data.ena_intensity_sys_err_minus, np.full_like(cdf["ena_intensity_sys_err_minus"], np.nan) + map_data.ena_intensity_sys_err_minus, + np.full_like(cdf["ena_intensity_sys_err_minus"], np.nan), ) def test_healpix_intensity_map_data_to_skymap(self): @@ -944,10 +1089,11 @@ def test_healpix_intensity_map_data_to_skymap(self): num_epochs = 1 num_energies = 5 - num_healpix_indices = 12 * (expected_nside ** 2) + num_healpix_indices = 12 * (expected_nside**2) - fake_data_per_energy_per_pixel = np.arange(num_epochs * num_energies * num_healpix_indices) \ - .reshape(num_epochs, num_energies, num_healpix_indices) + fake_data_per_energy_per_pixel = np.arange( + num_epochs * num_energies * num_healpix_indices + ).reshape(num_epochs, num_energies, num_healpix_indices) fake_data_per_pixel = np.arange(num_healpix_indices) @@ -957,11 +1103,13 @@ def test_healpix_intensity_map_data_to_skymap(self): quality_flags[0, :, 18] = MapL3Flags.PERSISTED_LAST_POINT expected_pred_ephem = quality_flags & MapL3Flags.PREDICTIVE_EPHEMERIS - expected_nominal_alpha_proton = quality_flags & MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO + expected_nominal_alpha_proton = ( + quality_flags & MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO + ) expected_persisted_last_point = quality_flags & MapL3Flags.PERSISTED_LAST_POINT intensity_map_data = IntensityMapData( - epoch=np.array([np.datetime64('1970-01-01T00:00:00')]), + epoch=np.array([np.datetime64("1970-01-01T00:00:00")]), epoch_delta=np.array([999]), energy=np.arange(num_energies) * 1.1, energy_delta_plus=np.arange(num_energies) * 1.2, @@ -970,7 +1118,8 @@ def test_healpix_intensity_map_data_to_skymap(self): latitude=fake_data_per_pixel * 2.2, longitude=fake_data_per_pixel * 2.3, exposure_factor=fake_data_per_energy_per_pixel * 2.2, - obs_date=np.datetime64('1970-01-01T00:00:00') + fake_data_per_energy_per_pixel * 10000, + obs_date=np.datetime64("1970-01-01T00:00:00") + + fake_data_per_energy_per_pixel * 10000, obs_date_range=fake_data_per_energy_per_pixel * 3.2, solid_angle=fake_data_per_pixel * 3.4, ena_intensity=fake_data_per_energy_per_pixel * 2.2, @@ -985,7 +1134,7 @@ def test_healpix_intensity_map_data_to_skymap(self): coords=HealPixCoords( pixel_index=np.arange(num_healpix_indices), pixel_index_label=np.arange(num_healpix_indices).astype(str), - ) + ), ) skymap = healpix_intensity_map_data.to_healpix_skymap() @@ -996,31 +1145,97 @@ def test_healpix_intensity_map_data_to_skymap(self): actual_dataset = skymap.data_1d # @formatter:off - np.testing.assert_array_equal(actual_dataset.coords[CoordNames.TIME.value].values, intensity_map_data.epoch) - np.testing.assert_array_equal(actual_dataset.coords[CoordNames.ENERGY_L2.value].values, intensity_map_data.energy) - np.testing.assert_array_equal(actual_dataset.coords[CoordNames.GENERIC_PIXEL.value].values, healpix_intensity_map_data.coords.pixel_index) - - np.testing.assert_array_equal(actual_dataset.data_vars["latitude"].values, intensity_map_data.latitude) - np.testing.assert_array_equal(actual_dataset.data_vars["longitude"].values, intensity_map_data.longitude) - np.testing.assert_array_equal(actual_dataset.data_vars["solid_angle"].values, intensity_map_data.solid_angle) - - np.testing.assert_array_equal(actual_dataset.data_vars["obs_date_range"].values, intensity_map_data.obs_date_range) - np.testing.assert_array_equal(actual_dataset.data_vars["obs_date"].values, intensity_map_data.obs_date.astype(np.float64)) - - np.testing.assert_array_equal(actual_dataset.data_vars["exposure_factor"].values, intensity_map_data.exposure_factor) - np.testing.assert_array_equal(actual_dataset.data_vars["ena_intensity"].values, intensity_map_data.ena_intensity) - np.testing.assert_array_equal(actual_dataset.data_vars["ena_intensity_stat_uncert"].values, intensity_map_data.ena_intensity_stat_uncert) - np.testing.assert_array_equal(actual_dataset.data_vars["ena_intensity_sys_err"].values, intensity_map_data.ena_intensity_sys_err) - - np.testing.assert_array_equal(actual_dataset.data_vars["survival_probability"].values, intensity_map_data.survival_probability) - np.testing.assert_array_equal(actual_dataset.data_vars["predicted_ephemeris_flag"].values, expected_pred_ephem) - np.testing.assert_array_equal(actual_dataset.data_vars["nominal_alpha_proton_ratio_flag"].values, expected_nominal_alpha_proton) - np.testing.assert_array_equal(actual_dataset.data_vars["persisted_last_point_flag"].values, expected_persisted_last_point) - - for key in [ "obs_date", "obs_date_range", "exposure_factor", "ena_intensity", "ena_intensity_stat_uncert", "ena_intensity_sys_err", "survival_probability", "predicted_ephemeris_flag", "nominal_alpha_proton_ratio_flag", "persisted_last_point_flag" ]: - self.assertEqual((CoordNames.TIME.value, CoordNames.ENERGY_L2.value, CoordNames.GENERIC_PIXEL.value), actual_dataset.data_vars[key].dims) - for key in [ "latitude", "longitude", "solid_angle" ]: - self.assertEqual((CoordNames.GENERIC_PIXEL.value,), actual_dataset.data_vars[key].dims) + np.testing.assert_array_equal( + actual_dataset.coords[CoordNames.TIME.value].values, + intensity_map_data.epoch, + ) + np.testing.assert_array_equal( + actual_dataset.coords[CoordNames.ENERGY_L2.value].values, + intensity_map_data.energy, + ) + np.testing.assert_array_equal( + actual_dataset.coords[CoordNames.GENERIC_PIXEL.value].values, + healpix_intensity_map_data.coords.pixel_index, + ) + + np.testing.assert_array_equal( + actual_dataset.data_vars["latitude"].values, intensity_map_data.latitude + ) + np.testing.assert_array_equal( + actual_dataset.data_vars["longitude"].values, intensity_map_data.longitude + ) + np.testing.assert_array_equal( + actual_dataset.data_vars["solid_angle"].values, + intensity_map_data.solid_angle, + ) + + np.testing.assert_array_equal( + actual_dataset.data_vars["obs_date_range"].values, + intensity_map_data.obs_date_range, + ) + np.testing.assert_array_equal( + actual_dataset.data_vars["obs_date"].values, + intensity_map_data.obs_date.astype(np.float64), + ) + + np.testing.assert_array_equal( + actual_dataset.data_vars["exposure_factor"].values, + intensity_map_data.exposure_factor, + ) + np.testing.assert_array_equal( + actual_dataset.data_vars["ena_intensity"].values, + intensity_map_data.ena_intensity, + ) + np.testing.assert_array_equal( + actual_dataset.data_vars["ena_intensity_stat_uncert"].values, + intensity_map_data.ena_intensity_stat_uncert, + ) + np.testing.assert_array_equal( + actual_dataset.data_vars["ena_intensity_sys_err"].values, + intensity_map_data.ena_intensity_sys_err, + ) + + np.testing.assert_array_equal( + actual_dataset.data_vars["survival_probability"].values, + intensity_map_data.survival_probability, + ) + np.testing.assert_array_equal( + actual_dataset.data_vars["predicted_ephemeris_flag"].values, + expected_pred_ephem, + ) + np.testing.assert_array_equal( + actual_dataset.data_vars["nominal_alpha_proton_ratio_flag"].values, + expected_nominal_alpha_proton, + ) + np.testing.assert_array_equal( + actual_dataset.data_vars["persisted_last_point_flag"].values, + expected_persisted_last_point, + ) + + for key in [ + "obs_date", + "obs_date_range", + "exposure_factor", + "ena_intensity", + "ena_intensity_stat_uncert", + "ena_intensity_sys_err", + "survival_probability", + "predicted_ephemeris_flag", + "nominal_alpha_proton_ratio_flag", + "persisted_last_point_flag", + ]: + self.assertEqual( + ( + CoordNames.TIME.value, + CoordNames.ENERGY_L2.value, + CoordNames.GENERIC_PIXEL.value, + ), + actual_dataset.data_vars[key].dims, + ) + for key in ["latitude", "longitude", "solid_angle"]: + self.assertEqual( + (CoordNames.GENERIC_PIXEL.value,), actual_dataset.data_vars[key].dims + ) # @formatter:on def test_healpix_intensity_map_data_to_skymap_with_no_sp(self): @@ -1084,16 +1299,14 @@ def test_read_intensity_data_handles_missing_obs_date_and_missing_sp(self): def test_calculate_datetime_weighted_average(self): - map_1_obs_date = np.ma.array(data=[ - datetime(2000, 1, 1), - datetime(2000, 1, 1), - datetime(9999, 12, 31)], - mask=[False, False, True]) - map_2_obs_date = np.ma.array(data=[ - datetime(9999, 12, 31), - datetime(2000, 1, 4), - datetime(9999, 12, 31)], - mask=[True, False, True]) + map_1_obs_date = np.ma.array( + data=[datetime(2000, 1, 1), datetime(2000, 1, 1), datetime(9999, 12, 31)], + mask=[False, False, True], + ) + map_2_obs_date = np.ma.array( + data=[datetime(9999, 12, 31), datetime(2000, 1, 4), datetime(9999, 12, 31)], + mask=[True, False, True], + ) map_1_weights = [1, 1, 1] map_2_weights = [1, 2, 1] @@ -1101,8 +1314,8 @@ def test_calculate_datetime_weighted_average(self): weights = np.array([map_1_weights, map_2_weights]) original_weights = weights.copy() average_obs_date: np.ma.masked_array = calculate_datetime_weighted_average( - np.ma.masked_array([map_1_obs_date, map_2_obs_date]), - weights, axis=0) + np.ma.masked_array([map_1_obs_date, map_2_obs_date]), weights, axis=0 + ) self.assertIsInstance(average_obs_date, np.ma.masked_array) @@ -1115,38 +1328,68 @@ def test_calculate_datetime_weighted_average(self): self.assertEqual(TT2000_EPOCH, average_obs_date.data[2]) self.assertTrue(average_obs_date.mask[2]) - np.testing.assert_equal(weights, original_weights, "should not mutate the input weights") - + np.testing.assert_equal( + weights, original_weights, "should not mutate the input weights" + ) def test_lo_isn_data_read_from_path(self): - path_to_cdf = get_test_data_folder() / 'lo' / 'imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-1yr_20250415_v000' + path_to_cdf = ( + get_test_data_folder() + / "lo" + / "imap_lo_l2_l090-ena-h-sf-nsp-ram-hae-6deg-1yr_20250415_v000" + ) actual = ISNRateData.read_from_path(path_to_cdf) with CDF(str(path_to_cdf)) as expected: - np.testing.assert_array_equal(actual.epoch, expected['epoch'][...]) - - np.testing.assert_array_equal(actual.bg_rate, read_numeric_variable(expected['bg_rate'])) - np.testing.assert_array_equal(actual.bg_rate_stat_uncert, - read_numeric_variable(expected['bg_rate_stat_uncert'])) - np.testing.assert_array_equal(actual.bg_rate_sys_err, read_numeric_variable(expected['bg_rate_sys_err'])) - np.testing.assert_array_equal(actual.ena_count_rate, read_numeric_variable(expected['ena_count_rate'])) - np.testing.assert_array_equal(actual.ena_count_rate_stat_uncert, - read_numeric_variable(expected['ena_count_rate_stat_uncert'])) - np.testing.assert_array_equal(actual.ena_intensity, read_numeric_variable(expected['ena_intensity'])) - np.testing.assert_array_equal(actual.ena_intensity_stat_uncert, - read_numeric_variable(expected['ena_intensity_stat_uncert'])) - np.testing.assert_array_equal(actual.ena_intensity_sys_err, - read_numeric_variable(expected['ena_intensity_sys_err'])) - np.testing.assert_array_equal(actual.energy, expected['energy'][...]) - np.testing.assert_array_equal(actual.exposure_factor, - read_numeric_variable(expected['exposure_factor'])) - np.testing.assert_array_equal(actual.latitude_delta, expected['latitude_delta'][...]) - np.testing.assert_array_equal(actual.latitude, expected['latitude'][...]) - np.testing.assert_array_equal(actual.longitude_delta, expected['longitude_delta'][...]) - np.testing.assert_array_equal(actual.longitude, expected['longitude'][...]) - np.testing.assert_array_equal(actual.solid_angle, read_numeric_variable(expected['solid_angle'])) - - -if __name__ == '__main__': + np.testing.assert_array_equal(actual.epoch, expected["epoch"][...]) + + np.testing.assert_array_equal( + actual.bg_rate, read_numeric_variable(expected["bg_rate"]) + ) + np.testing.assert_array_equal( + actual.bg_rate_stat_uncert, + read_numeric_variable(expected["bg_rate_stat_uncert"]), + ) + np.testing.assert_array_equal( + actual.bg_rate_sys_err, + read_numeric_variable(expected["bg_rate_sys_err"]), + ) + np.testing.assert_array_equal( + actual.ena_count_rate, read_numeric_variable(expected["ena_count_rate"]) + ) + np.testing.assert_array_equal( + actual.ena_count_rate_stat_uncert, + read_numeric_variable(expected["ena_count_rate_stat_uncert"]), + ) + np.testing.assert_array_equal( + actual.ena_intensity, read_numeric_variable(expected["ena_intensity"]) + ) + np.testing.assert_array_equal( + actual.ena_intensity_stat_uncert, + read_numeric_variable(expected["ena_intensity_stat_uncert"]), + ) + np.testing.assert_array_equal( + actual.ena_intensity_sys_err, + read_numeric_variable(expected["ena_intensity_sys_err"]), + ) + np.testing.assert_array_equal(actual.energy, expected["energy"][...]) + np.testing.assert_array_equal( + actual.exposure_factor, + read_numeric_variable(expected["exposure_factor"]), + ) + np.testing.assert_array_equal( + actual.latitude_delta, expected["latitude_delta"][...] + ) + np.testing.assert_array_equal(actual.latitude, expected["latitude"][...]) + np.testing.assert_array_equal( + actual.longitude_delta, expected["longitude_delta"][...] + ) + np.testing.assert_array_equal(actual.longitude, expected["longitude"][...]) + np.testing.assert_array_equal( + actual.solid_angle, read_numeric_variable(expected["solid_angle"]) + ) + + +if __name__ == "__main__": unittest.main() diff --git a/tests/maps/test_rectangular_survival_probability.py b/tests/maps/test_rectangular_survival_probability.py index 1238a4b1d..1bc43d87e 100644 --- a/tests/maps/test_rectangular_survival_probability.py +++ b/tests/maps/test_rectangular_survival_probability.py @@ -13,24 +13,33 @@ from imap_l3_processing.glows.quality_flags import GlowsL3Flags from imap_l3_processing.maps.map_descriptors import SpinPhase -from imap_l3_processing.maps.map_models import GlowsL3eRectangularMapInputData, InputRectangularPointingSet +from imap_l3_processing.maps.map_models import ( + GlowsL3eRectangularMapInputData, + InputRectangularPointingSet, +) from imap_l3_processing.maps.quality_flags import MapL3Flags -from imap_l3_processing.maps.rectangular_survival_probability import Sensor, \ - RectangularSurvivalProbabilitySkyMap, RectangularSurvivalProbabilityPointingSet, \ - interpolate_angular_data_to_nearest_neighbor +from imap_l3_processing.maps.rectangular_survival_probability import ( + Sensor, + RectangularSurvivalProbabilitySkyMap, + RectangularSurvivalProbabilityPointingSet, + interpolate_angular_data_to_nearest_neighbor, +) from tests.maps import test_builders from tests.maps.test_builders import create_l1c_pset from tests.spice_test_case import SpiceTestCase class TestRectangularSurvivalProbability(SpiceTestCase): - def setUp(self): self.num_energies = 2 self.l1c_epoch = datetime(2025, 1, 1, 0, 30) self.l1c_epoch_delta = np.array([86400_000_000_000]) - self.l1c_hae_longitude = (0.05 + np.linspace(0, 360, 3600, endpoint=False) + 90).reshape((1, -1)) % 360 - self.l1c_hae_latitude = np.linspace(-90, 90, 3600, endpoint=False).reshape((1, -1)) + self.l1c_hae_longitude = ( + 0.05 + np.linspace(0, 360, 3600, endpoint=False) + 90 + ).reshape((1, -1)) % 360 + self.l1c_hae_latitude = np.linspace(-90, 90, 3600, endpoint=False).reshape( + (1, -1) + ) self.hi_energies = np.geomspace(1, 10000, self.num_energies) @@ -38,13 +47,18 @@ def setUp(self): epoch=self.l1c_epoch, epoch_delta=self.l1c_epoch_delta, repointing=1, - epoch_j2000=np.array(np.array([spiceypy.datetime2et(self.l1c_epoch)]) * 1e9), - exposure_times=np.arange(self.num_energies * 3600).reshape((1, self.num_energies, 3600)) + 1.1, + epoch_j2000=np.array( + np.array([spiceypy.datetime2et(self.l1c_epoch)]) * 1e9 + ), + exposure_times=np.arange(self.num_energies * 3600).reshape( + (1, self.num_energies, 3600) + ) + + 1.1, esa_energy_step=np.arange(self.num_energies), pointing_start_met=None, pointing_end_met=None, hae_longitude=self.l1c_hae_longitude, - hae_latitude=self.l1c_hae_latitude + hae_latitude=self.l1c_hae_latitude, ) self.glows_data = GlowsL3eRectangularMapInputData( @@ -72,13 +86,17 @@ def _mock_calculate_ram_mask(dataset): return dataset self.mock_add_spacecraft_position_and_velocity_to_pset_patcher = patch( - 'imap_l3_processing.maps.rectangular_survival_probability.add_spacecraft_position_and_velocity_to_pset') + "imap_l3_processing.maps.rectangular_survival_probability.add_spacecraft_position_and_velocity_to_pset" + ) - self.mock_add_sc_velocity_to_pset = self.mock_add_spacecraft_position_and_velocity_to_pset_patcher.start() + self.mock_add_sc_velocity_to_pset = ( + self.mock_add_spacecraft_position_and_velocity_to_pset_patcher.start() + ) self.mock_add_sc_velocity_to_pset.side_effect = _mock_add_spacecraft_velocity self.mock_calculate_ram_mask_patcher = patch( - 'imap_l3_processing.maps.rectangular_survival_probability.calculate_ram_mask') + "imap_l3_processing.maps.rectangular_survival_probability.calculate_ram_mask" + ) self.mock_calculate_ram_mask = self.mock_calculate_ram_mask_patcher.start() self.mock_calculate_ram_mask.side_effect = _mock_calculate_ram_mask @@ -86,33 +104,54 @@ def tearDown(self): self.mock_add_spacecraft_position_and_velocity_to_pset_patcher.stop() self.mock_calculate_ram_mask_patcher.stop() - @patch('imap_l3_processing.maps.rectangular_survival_probability.PointingSet.__init__') - @patch('imap_l3_processing.maps.rectangular_survival_probability.add_spacecraft_position_and_velocity_to_pset') - @patch('imap_l3_processing.maps.rectangular_survival_probability.calculate_ram_mask') - def test_survival_probability_pointing_set_calls_parent_constructor(self, _, - __, - mock_rectangular_pointing_set_constructor): - pointing_set = RectangularSurvivalProbabilityPointingSet(self.l1c_hi_dataset, Sensor.Hi45, SpinPhase.RamOnly, - self.glows_data, - self.hi_energies) + @patch( + "imap_l3_processing.maps.rectangular_survival_probability.PointingSet.__init__" + ) + @patch( + "imap_l3_processing.maps.rectangular_survival_probability.add_spacecraft_position_and_velocity_to_pset" + ) + @patch( + "imap_l3_processing.maps.rectangular_survival_probability.calculate_ram_mask" + ) + def test_survival_probability_pointing_set_calls_parent_constructor( + self, _, __, mock_rectangular_pointing_set_constructor + ): + pointing_set = RectangularSurvivalProbabilityPointingSet( + self.l1c_hi_dataset, + Sensor.Hi45, + SpinPhase.RamOnly, + self.glows_data, + self.hi_energies, + ) self.assertIsInstance(pointing_set, PointingSet) mock_rectangular_pointing_set_constructor.assert_called_once() - @patch('imap_l3_processing.maps.rectangular_survival_probability.add_spacecraft_position_and_velocity_to_pset') - @patch('imap_l3_processing.maps.rectangular_survival_probability.calculate_ram_mask') - def test_survival_probability_pointing_set_no_cg(self, mock_calculate_ram_mask, mock_add_sc_velocity_to_pset): - mock_calculate_ram_mask.return_value = xr.Dataset({ - "ram_mask": [True] - }) + @patch( + "imap_l3_processing.maps.rectangular_survival_probability.add_spacecraft_position_and_velocity_to_pset" + ) + @patch( + "imap_l3_processing.maps.rectangular_survival_probability.calculate_ram_mask" + ) + def test_survival_probability_pointing_set_no_cg( + self, mock_calculate_ram_mask, mock_add_sc_velocity_to_pset + ): + mock_calculate_ram_mask.return_value = xr.Dataset({"ram_mask": [True]}) - pointing_set = RectangularSurvivalProbabilityPointingSet(self.l1c_hi_dataset, Sensor.Hi45, SpinPhase.RamOnly, - self.glows_data, - self.hi_energies, False) + pointing_set = RectangularSurvivalProbabilityPointingSet( + self.l1c_hi_dataset, + Sensor.Hi45, + SpinPhase.RamOnly, + self.glows_data, + self.hi_energies, + False, + ) - mock_calculate_ram_mask.assert_called_once_with(mock_add_sc_velocity_to_pset.return_value) + mock_calculate_ram_mask.assert_called_once_with( + mock_add_sc_velocity_to_pset.return_value + ) - self.assertEqual(pointing_set.data['directional_mask'], [True]) + self.assertEqual(pointing_set.data["directional_mask"], [True]) def test_survival_probability_pointing_set(self): test_cases = [ @@ -123,46 +162,68 @@ def test_survival_probability_pointing_set(self): (Sensor.Lo90, 0, SpinPhase.AntiRamOnly, ~self.ram_mask), ] - expected_repointing_midpoint = self.l1c_hi_dataset.epoch_j2000 + self.l1c_epoch_delta / 2 + expected_repointing_midpoint = ( + self.l1c_hi_dataset.epoch_j2000 + self.l1c_epoch_delta / 2 + ) for sensor, expected_sensor_angle, spin_phase, expected_mask in test_cases: with self.subTest(f"{sensor.value}"): - pointing_set = RectangularSurvivalProbabilityPointingSet(self.l1c_hi_dataset, sensor, spin_phase, - self.glows_data, - self.hi_energies) + pointing_set = RectangularSurvivalProbabilityPointingSet( + self.l1c_hi_dataset, + sensor, + spin_phase, + self.glows_data, + self.hi_energies, + ) self.assertIsInstance(pointing_set, PointingSet) - self.assertEqual(pointing_set.spice_reference_frame, geometry.SpiceFrame.IMAP_HAE) + self.assertEqual( + pointing_set.spice_reference_frame, geometry.SpiceFrame.IMAP_HAE + ) self.assertIn("exposure", pointing_set.data.data_vars) np.testing.assert_array_equal( pointing_set.data["exposure"].values, - self.l1c_hi_dataset.exposure_times) + self.l1c_hi_dataset.exposure_times, + ) np.testing.assert_array_equal( - pointing_set.data["directional_mask"].values, - expected_mask + pointing_set.data["directional_mask"].values, expected_mask ) self.assertIn(CoordNames.AZIMUTH_L1C.value, pointing_set.data.coords) np.testing.assert_array_almost_equal( np.arange(0, 360, 0.1) + 0.05, - pointing_set.data[CoordNames.AZIMUTH_L1C.value].values) + pointing_set.data[CoordNames.AZIMUTH_L1C.value].values, + ) - self.assertIn(CoordNames.ENERGY_ULTRA_L1C.value, pointing_set.data.coords) - np.testing.assert_array_equal(self.l1c_hi_dataset.esa_energy_step, - pointing_set.data[CoordNames.ENERGY_ULTRA_L1C.value].values) + self.assertIn( + CoordNames.ENERGY_ULTRA_L1C.value, pointing_set.data.coords + ) + np.testing.assert_array_equal( + self.l1c_hi_dataset.esa_energy_step, + pointing_set.data[CoordNames.ENERGY_ULTRA_L1C.value].values, + ) self.assertIn(CoordNames.TIME.value, pointing_set.data.coords) - np.testing.assert_array_equal(pointing_set.data[CoordNames.TIME.value].values, - expected_repointing_midpoint) + np.testing.assert_array_equal( + pointing_set.data[CoordNames.TIME.value].values, + expected_repointing_midpoint, + ) - np.testing.assert_array_equal(pointing_set.az_el_points[:, 0], - self.l1c_hae_longitude[0]) + np.testing.assert_array_equal( + pointing_set.az_el_points[:, 0], self.l1c_hae_longitude[0] + ) - np.testing.assert_array_equal(pointing_set.az_el_points[:, 1], self.l1c_hae_latitude[0]) + np.testing.assert_array_equal( + pointing_set.az_el_points[:, 1], self.l1c_hae_latitude[0] + ) - @patch("imap_l3_processing.maps.rectangular_survival_probability.apply_compton_getting_correction") - def test_hi_cg_corrected_survival_probability_pointing_set(self, mock_cg_correction): + @patch( + "imap_l3_processing.maps.rectangular_survival_probability.apply_compton_getting_correction" + ) + def test_hi_cg_corrected_survival_probability_pointing_set( + self, mock_cg_correction + ): self.mock_add_sc_velocity_to_pset.side_effect = None corrected_hae_longitude = np.full((1, 3, 3600), 2) corrected_hae_latitude = np.full((1, 3, 3600), 1) @@ -170,10 +231,14 @@ def test_hi_cg_corrected_survival_probability_pointing_set(self, mock_cg_correct l1c_dataset = test_builders.create_l1c_pset( epoch=self.l1c_hi_dataset.epoch, epoch_delta=self.l1c_hi_dataset.epoch_delta, - exposures=np.array([[np.full((3600,), 1), np.full((3600,), 2), np.full((3600,), 3)]]), + exposures=np.array( + [[np.full((3600,), 1), np.full((3600,), 2), np.full((3600,), 3)]] + ), ) - corrected_az_el_pairs = np.stack([corrected_hae_longitude[0], corrected_hae_latitude[0]], axis=2) + corrected_az_el_pairs = np.stack( + [corrected_hae_longitude[0], corrected_hae_latitude[0]], axis=2 + ) energy_sc = np.ones((1, 3, 3600)) energy_sc[0, 0, :1800] = 10 @@ -185,69 +250,129 @@ def test_hi_cg_corrected_survival_probability_pointing_set(self, mock_cg_correct sc_velocity = np.arange(3) + 1000 - hi_hf_energies = np.array([.001, .1, 1]) - expected_exposures = np.array([ - [np.concatenate((l1c_dataset.exposure_times[0, 0, :1800], l1c_dataset.exposure_times[0, 1, 1800:3600])), - np.concatenate((l1c_dataset.exposure_times[0, 1, :1800], l1c_dataset.exposure_times[0, 2, 1800:3600])), - l1c_dataset.exposure_times[0, 2, 0:3600]], - ]) + hi_hf_energies = np.array([0.001, 0.1, 1]) + expected_exposures = np.array( + [ + [ + np.concatenate( + ( + l1c_dataset.exposure_times[0, 0, :1800], + l1c_dataset.exposure_times[0, 1, 1800:3600], + ) + ), + np.concatenate( + ( + l1c_dataset.exposure_times[0, 1, :1800], + l1c_dataset.exposure_times[0, 2, 1800:3600], + ) + ), + l1c_dataset.exposure_times[0, 2, 0:3600], + ], + ] + ) - mock_cg_correction.return_value = xr.Dataset({ - "energy_sc": ( - [CoordNames.TIME.value, CoordNames.ENERGY_L2.value, CoordNames.AZIMUTH_L2.value], energy_sc - ), - "hae_latitude": ( - [CoordNames.TIME.value, CoordNames.ENERGY_L2.value, CoordNames.AZIMUTH_L2.value], corrected_hae_latitude - ), - "hae_longitude": ( - [CoordNames.TIME.value, CoordNames.ENERGY_L2.value, CoordNames.AZIMUTH_L2.value], - corrected_hae_longitude - ), - "sc_velocity": ( - [CoordNames.CARTESIAN_VECTOR.value], - sc_velocity - ) - }, + mock_cg_correction.return_value = xr.Dataset( + { + "energy_sc": ( + [ + CoordNames.TIME.value, + CoordNames.ENERGY_L2.value, + CoordNames.AZIMUTH_L2.value, + ], + energy_sc, + ), + "hae_latitude": ( + [ + CoordNames.TIME.value, + CoordNames.ENERGY_L2.value, + CoordNames.AZIMUTH_L2.value, + ], + corrected_hae_latitude, + ), + "hae_longitude": ( + [ + CoordNames.TIME.value, + CoordNames.ENERGY_L2.value, + CoordNames.AZIMUTH_L2.value, + ], + corrected_hae_longitude, + ), + "sc_velocity": ([CoordNames.CARTESIAN_VECTOR.value], sc_velocity), + }, coords={ CoordNames.TIME.value: [1_000_000_000], CoordNames.ENERGY_L2.value: hi_hf_energies, CoordNames.AZIMUTH_L1C.value: np.arange(3600), - CoordNames.CARTESIAN_VECTOR.value: np.arange(3) - } + CoordNames.CARTESIAN_VECTOR.value: np.arange(3), + }, ) - cg_pointing_set = RectangularSurvivalProbabilityPointingSet(l1c_dataset, Sensor.Hi90, SpinPhase.RamOnly, - self.glows_data, hi_hf_energies, - cg_corrected=True) + cg_pointing_set = RectangularSurvivalProbabilityPointingSet( + l1c_dataset, + Sensor.Hi90, + SpinPhase.RamOnly, + self.glows_data, + hi_hf_energies, + cg_corrected=True, + ) [actual_uncorrected_pset] = self.mock_add_sc_velocity_to_pset.call_args[0] - np.testing.assert_array_equal(actual_uncorrected_pset['hae_longitude'].values[0], l1c_dataset.hae_longitude[0]) - np.testing.assert_array_equal(actual_uncorrected_pset['hae_latitude'].values[0], l1c_dataset.hae_latitude[0]) + np.testing.assert_array_equal( + actual_uncorrected_pset["hae_longitude"].values[0], + l1c_dataset.hae_longitude[0], + ) + np.testing.assert_array_equal( + actual_uncorrected_pset["hae_latitude"].values[0], + l1c_dataset.hae_latitude[0], + ) - np.testing.assert_array_equal(actual_uncorrected_pset['epoch'].values, self.l1c_hi_dataset.epoch_j2000) - np.testing.assert_array_equal(actual_uncorrected_pset['epoch_delta'].values, self.l1c_hi_dataset.epoch_delta) + np.testing.assert_array_equal( + actual_uncorrected_pset["epoch"].values, self.l1c_hi_dataset.epoch_j2000 + ) + np.testing.assert_array_equal( + actual_uncorrected_pset["epoch_delta"].values, + self.l1c_hi_dataset.epoch_delta, + ) pset_with_sc_velocity, actual_hf_energies = mock_cg_correction.call_args[0] - self.assertEqual(pset_with_sc_velocity, self.mock_add_sc_velocity_to_pset.return_value) - self.mock_calculate_ram_mask.assert_called_once_with(mock_cg_correction.return_value) + self.assertEqual( + pset_with_sc_velocity, self.mock_add_sc_velocity_to_pset.return_value + ) + self.mock_calculate_ram_mask.assert_called_once_with( + mock_cg_correction.return_value + ) expected_energies_in_eV = hi_hf_energies * 1000 np.testing.assert_array_equal(actual_hf_energies, expected_energies_in_eV) self.assertEqual((CoordNames.ENERGY_ULTRA_L1C.value,), actual_hf_energies.dims) - np.testing.assert_array_equal(cg_pointing_set.data['epoch'], - self.l1c_hi_dataset.epoch_j2000 + (self.l1c_hi_dataset.epoch_delta / 2)) - np.testing.assert_array_equal(cg_pointing_set.az_el_points, corrected_az_el_pairs) + np.testing.assert_array_equal( + cg_pointing_set.data["epoch"], + self.l1c_hi_dataset.epoch_j2000 + (self.l1c_hi_dataset.epoch_delta / 2), + ) + np.testing.assert_array_equal( + cg_pointing_set.az_el_points, corrected_az_el_pairs + ) - np.testing.assert_array_equal(cg_pointing_set.data['exposure'], expected_exposures) + np.testing.assert_array_equal( + cg_pointing_set.data["exposure"], expected_exposures + ) - @patch("imap_l3_processing.maps.rectangular_survival_probability.add_spacecraft_position_and_velocity_to_pset") - @patch("imap_l3_processing.maps.rectangular_survival_probability.apply_compton_getting_correction") - def test_lo_cg_corrected_survival_probability_pointing_set(self, mock_cg_correction, mock_add_sc_velocity_to_pset): + @patch( + "imap_l3_processing.maps.rectangular_survival_probability.add_spacecraft_position_and_velocity_to_pset" + ) + @patch( + "imap_l3_processing.maps.rectangular_survival_probability.apply_compton_getting_correction" + ) + def test_lo_cg_corrected_survival_probability_pointing_set( + self, mock_cg_correction, mock_add_sc_velocity_to_pset + ): corrected_hae_longitude = np.full((1, 3, 3600), 2) corrected_hae_latitude = np.full((1, 3, 3600), 1) - corrected_az_el_pairs = np.stack([corrected_hae_longitude[0], corrected_hae_latitude[0]], axis=2) + corrected_az_el_pairs = np.stack( + [corrected_hae_longitude[0], corrected_hae_latitude[0]], axis=2 + ) uncorrected_hae_lon = np.ones((1, 3600)) uncorrected_hae_lat = np.full((1, 3600), 2) @@ -255,7 +380,9 @@ def test_lo_cg_corrected_survival_probability_pointing_set(self, mock_cg_correct l1c_dataset = test_builders.create_l1c_pset( epoch=self.l1c_hi_dataset.epoch, epoch_delta=None, - exposures=np.array([[np.full((3600,), 1), np.full((3600,), 2), np.full((3600,), 3)]]), + exposures=np.array( + [[np.full((3600,), 1), np.full((3600,), 2), np.full((3600,), 3)]] + ), pointing_start_met=np.array([800_000_000]), pointing_end_met=np.array([800_100_000]), ) @@ -273,71 +400,127 @@ def test_lo_cg_corrected_survival_probability_pointing_set(self, mock_cg_correct sc_velocity = np.arange(3) + 1000 - lo_hf_energies = np.array([.001, .1, 1]) - expected_exposures = np.array([ - [np.concatenate((l1c_dataset.exposure_times[0, 0, :1800], l1c_dataset.exposure_times[0, 1, 1800:3600])), - np.concatenate((l1c_dataset.exposure_times[0, 1, :1800], l1c_dataset.exposure_times[0, 2, 1800:3600])), - l1c_dataset.exposure_times[0, 2, 0:3600]], - ]) + lo_hf_energies = np.array([0.001, 0.1, 1]) + expected_exposures = np.array( + [ + [ + np.concatenate( + ( + l1c_dataset.exposure_times[0, 0, :1800], + l1c_dataset.exposure_times[0, 1, 1800:3600], + ) + ), + np.concatenate( + ( + l1c_dataset.exposure_times[0, 1, :1800], + l1c_dataset.exposure_times[0, 2, 1800:3600], + ) + ), + l1c_dataset.exposure_times[0, 2, 0:3600], + ], + ] + ) - mock_cg_correction.return_value = xr.Dataset({ - "energy_sc": ( - [CoordNames.TIME.value, CoordNames.ENERGY_L2.value, CoordNames.AZIMUTH_L2.value], energy_sc - ), - "hae_latitude": ( - [CoordNames.TIME.value, CoordNames.ENERGY_L2.value, CoordNames.AZIMUTH_L2.value], corrected_hae_latitude - ), - "hae_longitude": ( - [CoordNames.TIME.value, CoordNames.ENERGY_L2.value, CoordNames.AZIMUTH_L2.value], - corrected_hae_longitude - ), - "sc_velocity": ( - [CoordNames.CARTESIAN_VECTOR.value], - sc_velocity - ) - }, + mock_cg_correction.return_value = xr.Dataset( + { + "energy_sc": ( + [ + CoordNames.TIME.value, + CoordNames.ENERGY_L2.value, + CoordNames.AZIMUTH_L2.value, + ], + energy_sc, + ), + "hae_latitude": ( + [ + CoordNames.TIME.value, + CoordNames.ENERGY_L2.value, + CoordNames.AZIMUTH_L2.value, + ], + corrected_hae_latitude, + ), + "hae_longitude": ( + [ + CoordNames.TIME.value, + CoordNames.ENERGY_L2.value, + CoordNames.AZIMUTH_L2.value, + ], + corrected_hae_longitude, + ), + "sc_velocity": ([CoordNames.CARTESIAN_VECTOR.value], sc_velocity), + }, coords={ CoordNames.TIME.value: [1_000_000_000], CoordNames.ENERGY_L2.value: lo_hf_energies, CoordNames.AZIMUTH_L1C.value: np.arange(3600), - CoordNames.CARTESIAN_VECTOR.value: np.arange(3) - } + CoordNames.CARTESIAN_VECTOR.value: np.arange(3), + }, ) - cases = { - "lo90": Sensor.Lo90, - "lo75": Sensor.Lo75, - "lo105": Sensor.Lo105 - } + cases = {"lo90": Sensor.Lo90, "lo75": Sensor.Lo75, "lo105": Sensor.Lo105} for name, sensor in cases.items(): with self.subTest(name): - cg_pointing_set = RectangularSurvivalProbabilityPointingSet(l1c_dataset, sensor, SpinPhase.RamOnly, - self.glows_data, lo_hf_energies, - cg_corrected=True) + cg_pointing_set = RectangularSurvivalProbabilityPointingSet( + l1c_dataset, + sensor, + SpinPhase.RamOnly, + self.glows_data, + lo_hf_energies, + cg_corrected=True, + ) - expected_pointing_epoch_midpoint = self.l1c_hi_dataset.epoch_j2000 + (100_000 * 1e9 / 2) + expected_pointing_epoch_midpoint = self.l1c_hi_dataset.epoch_j2000 + ( + 100_000 * 1e9 / 2 + ) [actual_uncorrected_pset] = mock_add_sc_velocity_to_pset.call_args[0] - np.testing.assert_array_equal(actual_uncorrected_pset['hae_longitude'], uncorrected_hae_lon) - np.testing.assert_array_equal(actual_uncorrected_pset['hae_latitude'], uncorrected_hae_lat) + np.testing.assert_array_equal( + actual_uncorrected_pset["hae_longitude"], uncorrected_hae_lon + ) + np.testing.assert_array_equal( + actual_uncorrected_pset["hae_latitude"], uncorrected_hae_lat + ) - np.testing.assert_array_equal(actual_uncorrected_pset['epoch'].values, self.l1c_hi_dataset.epoch_j2000) - np.testing.assert_array_equal(actual_uncorrected_pset['epoch_delta'].values, None) - np.testing.assert_array_equal(actual_uncorrected_pset['pointing_start_met'].values, [800_000_000]) - np.testing.assert_array_equal(actual_uncorrected_pset['pointing_end_met'].values, [800_100_000]) - self.assertEqual(actual_uncorrected_pset.attrs['Logical_source'], "imap_lo") + np.testing.assert_array_equal( + actual_uncorrected_pset["epoch"].values, + self.l1c_hi_dataset.epoch_j2000, + ) + np.testing.assert_array_equal( + actual_uncorrected_pset["epoch_delta"].values, None + ) + np.testing.assert_array_equal( + actual_uncorrected_pset["pointing_start_met"].values, [800_000_000] + ) + np.testing.assert_array_equal( + actual_uncorrected_pset["pointing_end_met"].values, [800_100_000] + ) + self.assertEqual( + actual_uncorrected_pset.attrs["Logical_source"], "imap_lo" + ) - pset_with_sc_velocity, actual_hf_energies = mock_cg_correction.call_args[0] - self.assertEqual(pset_with_sc_velocity, mock_add_sc_velocity_to_pset.return_value) + pset_with_sc_velocity, actual_hf_energies = ( + mock_cg_correction.call_args[0] + ) + self.assertEqual( + pset_with_sc_velocity, mock_add_sc_velocity_to_pset.return_value + ) expected_energies_in_eV = lo_hf_energies * 1000 - np.testing.assert_array_equal(actual_hf_energies, expected_energies_in_eV) + np.testing.assert_array_equal( + actual_hf_energies, expected_energies_in_eV + ) - np.testing.assert_array_equal(cg_pointing_set.data['epoch'], expected_pointing_epoch_midpoint) - np.testing.assert_array_equal(cg_pointing_set.az_el_points, corrected_az_el_pairs) + np.testing.assert_array_equal( + cg_pointing_set.data["epoch"], expected_pointing_epoch_midpoint + ) + np.testing.assert_array_equal( + cg_pointing_set.az_el_points, corrected_az_el_pairs + ) - np.testing.assert_array_equal(cg_pointing_set.data['exposure'], expected_exposures) + np.testing.assert_array_equal( + cg_pointing_set.data["exposure"], expected_exposures + ) def test_exposure_weighting_with_interpolated_survival_probabilities(self): test_cases = [ @@ -346,44 +529,66 @@ def test_exposure_weighting_with_interpolated_survival_probabilities(self): ] for spin_phase, expected_mask in test_cases: - with (self.subTest(spin_phase)): + with self.subTest(spin_phase): self.hi_energies = np.array([10, 10_000]) self.glows_data.energy = np.array([1, 100, 100_000]) - self.glows_data.probability_of_survival = np.repeat([2, 4, 7], 360).reshape(1, 3, 360) + self.glows_data.probability_of_survival = np.repeat( + [2, 4, 7], 360 + ).reshape(1, 3, 360) - expected_interpolated_survival_probabilities = \ - np.repeat([3, 6], 3600).reshape(1, 2, 3600) * self.l1c_hi_dataset.exposure_times + expected_interpolated_survival_probabilities = ( + np.repeat([3, 6], 3600).reshape(1, 2, 3600) + * self.l1c_hi_dataset.exposure_times + ) sensor, expected_skygrid_elevation_index = Sensor.Hi90, 901 - pointing_set = RectangularSurvivalProbabilityPointingSet(self.l1c_hi_dataset, sensor, spin_phase, - self.glows_data, - self.hi_energies) + pointing_set = RectangularSurvivalProbabilityPointingSet( + self.l1c_hi_dataset, + sensor, + spin_phase, + self.glows_data, + self.hi_energies, + ) - self.assertIn("survival_probability_times_exposure", pointing_set.data.data_vars) - self.assertEqual((1, self.num_energies, 3600), - pointing_set.data["survival_probability_times_exposure"].values.shape) + self.assertIn( + "survival_probability_times_exposure", pointing_set.data.data_vars + ) + self.assertEqual( + (1, self.num_energies, 3600), + pointing_set.data[ + "survival_probability_times_exposure" + ].values.shape, + ) np.testing.assert_array_equal( expected_interpolated_survival_probabilities, - pointing_set.data["survival_probability_times_exposure"].values) + pointing_set.data["survival_probability_times_exposure"].values, + ) np.testing.assert_array_equal( - pointing_set.data["directional_mask"], - expected_mask + pointing_set.data["directional_mask"], expected_mask ) def test_exposure_weighted_survivals_are_repeated_to_match_l1c_shape(self): - pointing_set = RectangularSurvivalProbabilityPointingSet(self.l1c_hi_dataset, Sensor.Hi90, SpinPhase.RamOnly, - self.glows_data, - self.hi_energies) + pointing_set = RectangularSurvivalProbabilityPointingSet( + self.l1c_hi_dataset, + Sensor.Hi90, + SpinPhase.RamOnly, + self.glows_data, + self.hi_energies, + ) - survivals = pointing_set.data[ - "survival_probability_times_exposure"].values / self.l1c_hi_dataset.exposure_times + survivals = ( + pointing_set.data["survival_probability_times_exposure"].values + / self.l1c_hi_dataset.exposure_times + ) every_tenth_value = survivals[:, :, ::10, np.newaxis] groups_of_ten = survivals.reshape(1, self.num_energies, 360, 10) self.assertTrue(np.all(np.isclose(every_tenth_value, groups_of_ten))) - @patch("imap_l3_processing.maps.rectangular_survival_probability.interpolate_angular_data_to_nearest_neighbor") + @patch( + "imap_l3_processing.maps.rectangular_survival_probability.interpolate_angular_data_to_nearest_neighbor" + ) def test_survivals_matched_with_corresponding_exposures(self, mock_interpolate): self.hi_energies = np.array([1, 100]) self.glows_data.energy = np.array([1, 100, 100_000]) @@ -391,12 +596,20 @@ def test_survivals_matched_with_corresponding_exposures(self, mock_interpolate): first_energy_corresponding_glows_data = np.linspace(0, 1, 3600) second_energy_corresponding_glows_data = np.linspace(0, 1, 3600) + 100.2 - mock_interpolate.return_value = np.array([first_energy_corresponding_glows_data, - second_energy_corresponding_glows_data]) + mock_interpolate.return_value = np.array( + [ + first_energy_corresponding_glows_data, + second_energy_corresponding_glows_data, + ] + ) - pointing_set = RectangularSurvivalProbabilityPointingSet(self.l1c_hi_dataset, Sensor.Hi90, SpinPhase.RamOnly, - self.glows_data, - self.hi_energies) + pointing_set = RectangularSurvivalProbabilityPointingSet( + self.l1c_hi_dataset, + Sensor.Hi90, + SpinPhase.RamOnly, + self.glows_data, + self.hi_energies, + ) pset_spin_angles = np.linspace(0, 360, 3600, endpoint=False) + 0.05 pset_azimuths = np.mod(pset_spin_angles, 360) @@ -404,20 +617,38 @@ def test_survivals_matched_with_corresponding_exposures(self, mock_interpolate): mock_interpolate.assert_called_once() get_interpolated_glows_data_args = mock_interpolate.call_args_list[0].args - np.testing.assert_array_equal(pset_azimuths, get_interpolated_glows_data_args[0]) - np.testing.assert_array_equal(self.glows_data.spin_angle, get_interpolated_glows_data_args[1]) - np.testing.assert_array_equal(self.glows_data.probability_of_survival[0, :2], - get_interpolated_glows_data_args[2]) + np.testing.assert_array_equal( + pset_azimuths, get_interpolated_glows_data_args[0] + ) + np.testing.assert_array_equal( + self.glows_data.spin_angle, get_interpolated_glows_data_args[1] + ) + np.testing.assert_array_equal( + self.glows_data.probability_of_survival[0, :2], + get_interpolated_glows_data_args[2], + ) corresponding_glows_data = np.array( - [first_energy_corresponding_glows_data, second_energy_corresponding_glows_data])[np.newaxis, ...] - - np.testing.assert_array_almost_equal(pointing_set.data["survival_probability_times_exposure"].values, - corresponding_glows_data * self.l1c_hi_dataset.exposure_times) + [ + first_energy_corresponding_glows_data, + second_energy_corresponding_glows_data, + ] + )[np.newaxis, ...] + + np.testing.assert_array_almost_equal( + pointing_set.data["survival_probability_times_exposure"].values, + corresponding_glows_data * self.l1c_hi_dataset.exposure_times, + ) - @patch("imap_l3_processing.maps.rectangular_survival_probability.interpolate_angular_data_to_nearest_neighbor") - @patch("imap_l3_processing.maps.rectangular_survival_probability.apply_compton_getting_correction") - def test_survivals_matched_with_corresponding_exposures_cg_corrected(self, mock_cg_correction, mock_interpolate): + @patch( + "imap_l3_processing.maps.rectangular_survival_probability.interpolate_angular_data_to_nearest_neighbor" + ) + @patch( + "imap_l3_processing.maps.rectangular_survival_probability.apply_compton_getting_correction" + ) + def test_survivals_matched_with_corresponding_exposures_cg_corrected( + self, mock_cg_correction, mock_interpolate + ): test_cases = [ Sensor.Hi90, Sensor.Hi45, @@ -514,7 +745,11 @@ def test_survivals_matched_with_corresponding_exposures_cg_corrected(self, mock_ ) corresponding_glows_data = np.array( - [first_energy_corresponding_glows_data, second_energy_corresponding_glows_data])[np.newaxis, ...] + [ + first_energy_corresponding_glows_data, + second_energy_corresponding_glows_data, + ] + )[np.newaxis, ...] np.testing.assert_array_almost_equal( pointing_set.data["survival_probability_times_exposure"].values, @@ -529,41 +764,100 @@ def test_survival_probability_pointing_set_propagates_flags( (GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO, 0.0, 1.0, 0.0), (GlowsL3Flags.PREDICTIVE_EPHEMERIS, 1.0, 0.0, 0.0), (GlowsL3Flags.PERSISTED_LAST_POINT, 0.0, 0.0, 1.0), - (GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO | GlowsL3Flags.PREDICTIVE_EPHEMERIS, 1.0, 1.0, 0.0), - (GlowsL3Flags.PERSISTED_LAST_POINT | GlowsL3Flags.PREDICTIVE_EPHEMERIS, 1.0, 0.0, 1.0), - (GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO | GlowsL3Flags.PERSISTED_LAST_POINT, 0.0, 1.0, 1.0), - (GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO | GlowsL3Flags.PERSISTED_LAST_POINT | GlowsL3Flags.PREDICTIVE_EPHEMERIS, 1.0, 1.0, 1.0), + ( + GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO + | GlowsL3Flags.PREDICTIVE_EPHEMERIS, + 1.0, + 1.0, + 0.0, + ), + ( + GlowsL3Flags.PERSISTED_LAST_POINT | GlowsL3Flags.PREDICTIVE_EPHEMERIS, + 1.0, + 0.0, + 1.0, + ), + ( + GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO + | GlowsL3Flags.PERSISTED_LAST_POINT, + 0.0, + 1.0, + 1.0, + ), + ( + GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO + | GlowsL3Flags.PERSISTED_LAST_POINT + | GlowsL3Flags.PREDICTIVE_EPHEMERIS, + 1.0, + 1.0, + 1.0, + ), ] - for flag_value, expected_pred_ephem_value, expected_nominal_alpha_proton_value, expected_persisted_last_point_value in cases: + for ( + flag_value, + expected_pred_ephem_value, + expected_nominal_alpha_proton_value, + expected_persisted_last_point_value, + ) in cases: with self.subTest(flag_value): exposure_array = np.full(self.l1c_hi_dataset.exposure_times.shape, 1.0) exposure_array[:, :, 1000] = 0.0 self.glows_data.flags[0] = flag_value self.l1c_hi_dataset.exposure_times = exposure_array pointing_set = RectangularSurvivalProbabilityPointingSet( - self.l1c_hi_dataset, Sensor.Hi90, SpinPhase.RamOnly, - glows_dataset=self.glows_data, energies=self.hi_energies) + self.l1c_hi_dataset, + Sensor.Hi90, + SpinPhase.RamOnly, + glows_dataset=self.glows_data, + energies=self.hi_energies, + ) actual_pred_ephem = pointing_set.data["predicted_ephemeris_flag"].values - expected_pred_ephem = np.full((1, self.num_energies, 3600), expected_pred_ephem_value) + expected_pred_ephem = np.full( + (1, self.num_energies, 3600), expected_pred_ephem_value + ) expected_pred_ephem[:, :, 1000] = 0 - np.testing.assert_array_equal(actual_pred_ephem, expected_pred_ephem, strict=True) - self.assertEqual(pointing_set.data["predicted_ephemeris_flag"].dims, - pointing_set.data["survival_probability_times_exposure"].dims) + np.testing.assert_array_equal( + actual_pred_ephem, expected_pred_ephem, strict=True + ) + self.assertEqual( + pointing_set.data["predicted_ephemeris_flag"].dims, + pointing_set.data["survival_probability_times_exposure"].dims, + ) - actual_nominal_alpha_proton = pointing_set.data["nominal_alpha_proton_ratio_flag"].values - expected_nominal_alpha_proton = np.full((1, self.num_energies, 3600), expected_nominal_alpha_proton_value) + actual_nominal_alpha_proton = pointing_set.data[ + "nominal_alpha_proton_ratio_flag" + ].values + expected_nominal_alpha_proton = np.full( + (1, self.num_energies, 3600), expected_nominal_alpha_proton_value + ) expected_nominal_alpha_proton[:, :, 1000] = 0 - np.testing.assert_array_equal(actual_nominal_alpha_proton, expected_nominal_alpha_proton, strict=True) - self.assertEqual(pointing_set.data["nominal_alpha_proton_ratio_flag"].dims, - pointing_set.data["survival_probability_times_exposure"].dims) + np.testing.assert_array_equal( + actual_nominal_alpha_proton, + expected_nominal_alpha_proton, + strict=True, + ) + self.assertEqual( + pointing_set.data["nominal_alpha_proton_ratio_flag"].dims, + pointing_set.data["survival_probability_times_exposure"].dims, + ) - actual_persisted_last_point = pointing_set.data["persisted_last_point_flag"].values - expected_persisted_last_point = np.full((1, self.num_energies, 3600), expected_persisted_last_point_value) + actual_persisted_last_point = pointing_set.data[ + "persisted_last_point_flag" + ].values + expected_persisted_last_point = np.full( + (1, self.num_energies, 3600), expected_persisted_last_point_value + ) expected_persisted_last_point[:, :, 1000] = 0 - np.testing.assert_array_equal(actual_persisted_last_point, expected_persisted_last_point, strict=True) - self.assertEqual(pointing_set.data["persisted_last_point_flag"].dims, - pointing_set.data["survival_probability_times_exposure"].dims) + np.testing.assert_array_equal( + actual_persisted_last_point, + expected_persisted_last_point, + strict=True, + ) + self.assertEqual( + pointing_set.data["persisted_last_point_flag"].dims, + pointing_set.data["survival_probability_times_exposure"].dims, + ) def test_interpolate_angular_data_to_nearest_neighbor(self): input_cases = [ @@ -581,143 +875,206 @@ def test_interpolate_angular_data_to_nearest_neighbor(self): basic_glows_data = np.array([np.linspace(1, 360, 360, endpoint=True) + 1000]) glows_cases = [ ("basic", basic_glows_spin_angles, basic_glows_data), - ("0 instead of 360", np.mod(basic_glows_spin_angles, 360), basic_glows_data), - ("with 0 at start", np.roll(np.mod(basic_glows_spin_angles, 360), 1), np.roll(basic_glows_data, 1)), - ("rolled", np.roll(basic_glows_spin_angles, 90), np.roll(basic_glows_data, 90)), + ( + "0 instead of 360", + np.mod(basic_glows_spin_angles, 360), + basic_glows_data, + ), + ( + "with 0 at start", + np.roll(np.mod(basic_glows_spin_angles, 360), 1), + np.roll(basic_glows_data, 1), + ), + ( + "rolled", + np.roll(basic_glows_spin_angles, 90), + np.roll(basic_glows_data, 90), + ), ] for name, glows_spin_angles, glows_data in glows_cases: with self.subTest(name): input_angles, expected_output = np.array(input_cases).T - result = interpolate_angular_data_to_nearest_neighbor(input_angles, glows_spin_angles, glows_data) + result = interpolate_angular_data_to_nearest_neighbor( + input_angles, glows_spin_angles, glows_data + ) np.testing.assert_array_equal(expected_output + 1000, result[0]) - @patch('imap_l3_processing.maps.rectangular_survival_probability.RectangularSkyMap.project_pset_values_to_map') - @patch('imap_l3_processing.maps.rectangular_survival_probability.RectangularSkyMap.__init__') - def test_survival_probability_map_construction(self, mock_skymap_constructor, mock_project_pset): - class TestableRectangularSurvivalProbabilitySkyMap(RectangularSurvivalProbabilitySkyMap): + @patch( + "imap_l3_processing.maps.rectangular_survival_probability.RectangularSkyMap.project_pset_values_to_map" + ) + @patch( + "imap_l3_processing.maps.rectangular_survival_probability.RectangularSkyMap.__init__" + ) + def test_survival_probability_map_construction( + self, mock_skymap_constructor, mock_project_pset + ): + class TestableRectangularSurvivalProbabilitySkyMap( + RectangularSurvivalProbabilitySkyMap + ): def __init__(self, *args): self.data_1d = MagicMock() super().__init__(*args) pset_1 = MagicMock() - pset_1.data['directional_mask'] = sentinel.directional_mask_1 + pset_1.data["directional_mask"] = sentinel.directional_mask_1 pset_2 = MagicMock() - pset_2.data['directional_mask'] = sentinel.directional_mask_2 + pset_2.data["directional_mask"] = sentinel.directional_mask_2 - actual_sky_map = TestableRectangularSurvivalProbabilitySkyMap([pset_1, pset_2], - sentinel.spacing_deg, - sentinel.spice_frame) + actual_sky_map = TestableRectangularSurvivalProbabilitySkyMap( + [pset_1, pset_2], sentinel.spacing_deg, sentinel.spice_frame + ) self.assertIsInstance(actual_sky_map, RectangularSkyMap) - mock_skymap_constructor.assert_called_with(sentinel.spacing_deg, sentinel.spice_frame) - - expected_values_to_project = ["survival_probability_times_exposure", "exposure", "predicted_ephemeris_flag", - "nominal_alpha_proton_ratio_flag", "persisted_last_point_flag"] - mock_project_pset.assert_has_calls([ - call(pset_1, expected_values_to_project, pset_valid_mask=pset_1.data['directional_mask']), - call(pset_2, expected_values_to_project, pset_valid_mask=pset_2.data['directional_mask']), - ]) - - def test_survival_probability_sky_map_returns_exposure_weighted_survival_probabilities(self): - self.l1c_hi_dataset.hae_longitude = np.concat([ - np.full((1, 1800), 30.05), - np.full((1, 1800), 210.05) - ], axis=1) - - self.l1c_hi_dataset.hae_latitude = np.concat([ - np.linspace(90, -90, 1800, endpoint=False) - 0.05, - np.linspace(-90, 90, 1800, endpoint=False) + 0.05, - ]).reshape(1, -1) + mock_skymap_constructor.assert_called_with( + sentinel.spacing_deg, sentinel.spice_frame + ) + + expected_values_to_project = [ + "survival_probability_times_exposure", + "exposure", + "predicted_ephemeris_flag", + "nominal_alpha_proton_ratio_flag", + "persisted_last_point_flag", + ] + mock_project_pset.assert_has_calls( + [ + call( + pset_1, + expected_values_to_project, + pset_valid_mask=pset_1.data["directional_mask"], + ), + call( + pset_2, + expected_values_to_project, + pset_valid_mask=pset_2.data["directional_mask"], + ), + ] + ) + + def test_survival_probability_sky_map_returns_exposure_weighted_survival_probabilities( + self, + ): + self.l1c_hi_dataset.hae_longitude = np.concat( + [np.full((1, 1800), 30.05), np.full((1, 1800), 210.05)], axis=1 + ) + + self.l1c_hi_dataset.hae_latitude = np.concat( + [ + np.linspace(90, -90, 1800, endpoint=False) - 0.05, + np.linspace(-90, 90, 1800, endpoint=False) + 0.05, + ] + ).reshape(1, -1) self.ram_mask = np.concat((np.full(1800, True), np.full(1800, False))) pset1_glows = self.glows_data pset1_l1c = self.l1c_hi_dataset - pset2_glows = dataclasses.replace(self.glows_data, - probability_of_survival=4 * self.glows_data.probability_of_survival) - pset2_l1c = dataclasses.replace(self.l1c_hi_dataset, exposure_times=2 * self.l1c_hi_dataset.exposure_times) + pset2_glows = dataclasses.replace( + self.glows_data, + probability_of_survival=4 * self.glows_data.probability_of_survival, + ) + pset2_l1c = dataclasses.replace( + self.l1c_hi_dataset, exposure_times=2 * self.l1c_hi_dataset.exposure_times + ) - pset1 = RectangularSurvivalProbabilityPointingSet(pset1_l1c, Sensor.Hi90, SpinPhase.RamOnly, - pset1_glows, self.hi_energies) + pset1 = RectangularSurvivalProbabilityPointingSet( + pset1_l1c, Sensor.Hi90, SpinPhase.RamOnly, pset1_glows, self.hi_energies + ) - pset2 = RectangularSurvivalProbabilityPointingSet(pset2_l1c, Sensor.Hi90, SpinPhase.RamOnly, - pset2_glows, self.hi_energies) + pset2 = RectangularSurvivalProbabilityPointingSet( + pset2_l1c, Sensor.Hi90, SpinPhase.RamOnly, pset2_glows, self.hi_energies + ) spice_frame = SpiceFrame.IMAP_HAE - actual_skymap = RectangularSurvivalProbabilitySkyMap([pset1, pset2], - 0.1, spice_frame) + actual_skymap = RectangularSurvivalProbabilitySkyMap( + [pset1, pset2], 0.1, spice_frame + ) survival_probability_dataset = actual_skymap.to_dataset() - expected = 3 * np.repeat(self.glows_data.probability_of_survival[:, [0, -1], 0:180], 10, axis=2) + expected = 3 * np.repeat( + self.glows_data.probability_of_survival[:, [0, -1], 0:180], 10, axis=2 + ) expected = np.flip(expected, axis=2) survival_prob_in_skygrid_shape = np.full((1, 2, 3600, 1800), np.nan) survival_prob_in_skygrid_shape[:, :, 300, :] = expected - self.assertIn("exposure_weighted_survival_probabilities", survival_probability_dataset) - self.assertEqual((1, 2, 3600, 1800), - survival_probability_dataset["exposure_weighted_survival_probabilities"].values.shape) + self.assertIn( + "exposure_weighted_survival_probabilities", survival_probability_dataset + ) + self.assertEqual( + (1, 2, 3600, 1800), + survival_probability_dataset[ + "exposure_weighted_survival_probabilities" + ].values.shape, + ) - actual_sp_at_relevant_long = survival_probability_dataset["exposure_weighted_survival_probabilities"].values[:, - :, 300, :] + actual_sp_at_relevant_long = survival_probability_dataset[ + "exposure_weighted_survival_probabilities" + ].values[:, :, 300, :] np.testing.assert_array_almost_equal(expected, actual_sp_at_relevant_long) - np.testing.assert_array_almost_equal(survival_prob_in_skygrid_shape, - survival_probability_dataset[ - "exposure_weighted_survival_probabilities"].values) + np.testing.assert_array_almost_equal( + survival_prob_in_skygrid_shape, + survival_probability_dataset[ + "exposure_weighted_survival_probabilities" + ].values, + ) def test_survival_probability_sky_map_returns_flags(self): - self.l1c_hi_dataset.hae_longitude = np.concat([ - np.full((1, 1800), 30.05), - np.full((1, 1800), 210.05) - ], axis=1) - - self.l1c_hi_dataset.hae_latitude = np.concat([ - np.linspace(90, -90, 1800, endpoint=False) - 0.05, - np.linspace(-90, 90, 1800, endpoint=False) + 0.05, - ]).reshape(1, -1) + self.l1c_hi_dataset.hae_longitude = np.concat( + [np.full((1, 1800), 30.05), np.full((1, 1800), 210.05)], axis=1 + ) + + self.l1c_hi_dataset.hae_latitude = np.concat( + [ + np.linspace(90, -90, 1800, endpoint=False) - 0.05, + np.linspace(-90, 90, 1800, endpoint=False) + 0.05, + ] + ).reshape(1, -1) self.ram_mask = np.concat((np.full(1800, True), np.full(1800, False))) cases = { - ( - GlowsL3Flags.NONE, - GlowsL3Flags.NONE, - MapL3Flags.NONE - ), + (GlowsL3Flags.NONE, GlowsL3Flags.NONE, MapL3Flags.NONE), ( GlowsL3Flags.PREDICTIVE_EPHEMERIS, GlowsL3Flags.NONE, - MapL3Flags.PREDICTIVE_EPHEMERIS + MapL3Flags.PREDICTIVE_EPHEMERIS, ), ( GlowsL3Flags.PREDICTIVE_EPHEMERIS, GlowsL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.PREDICTIVE_EPHEMERIS + MapL3Flags.PREDICTIVE_EPHEMERIS, ), ( GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO, GlowsL3Flags.NONE, - GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO + GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO, ), ( - GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO | GlowsL3Flags.PREDICTIVE_EPHEMERIS, + GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO + | GlowsL3Flags.PREDICTIVE_EPHEMERIS, GlowsL3Flags.NONE, - MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO | MapL3Flags.PREDICTIVE_EPHEMERIS + MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO | MapL3Flags.PREDICTIVE_EPHEMERIS, ), ( GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO, GlowsL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO | MapL3Flags.PREDICTIVE_EPHEMERIS + MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO | MapL3Flags.PREDICTIVE_EPHEMERIS, ), ( GlowsL3Flags.PREDICTIVE_EPHEMERIS, GlowsL3Flags.PERSISTED_LAST_POINT, - MapL3Flags.PREDICTIVE_EPHEMERIS | MapL3Flags.PERSISTED_LAST_POINT - ) + MapL3Flags.PREDICTIVE_EPHEMERIS | MapL3Flags.PERSISTED_LAST_POINT, + ), } for pset1_flags, pset2_flags, expected_quality_flags in cases: - with self.subTest(pset1_flags=pset1_flags, pset2_flags=pset2_flags, expected=expected_quality_flags): + with self.subTest( + pset1_flags=pset1_flags, + pset2_flags=pset2_flags, + expected=expected_quality_flags, + ): pset1_glows = copy.deepcopy(self.glows_data) pset1_glows.flags[0] = pset1_flags pset1_l1c = pset2_l1c = self.l1c_hi_dataset @@ -725,21 +1082,40 @@ def test_survival_probability_sky_map_returns_flags(self): pset2_glows = copy.deepcopy(self.glows_data) pset2_glows.flags[0] = pset2_flags - pset1 = RectangularSurvivalProbabilityPointingSet(pset1_l1c, Sensor.Hi90, SpinPhase.RamOnly, - pset1_glows, self.hi_energies) + pset1 = RectangularSurvivalProbabilityPointingSet( + pset1_l1c, + Sensor.Hi90, + SpinPhase.RamOnly, + pset1_glows, + self.hi_energies, + ) - pset2 = RectangularSurvivalProbabilityPointingSet(pset2_l1c, Sensor.Hi90, SpinPhase.RamOnly, - pset2_glows, self.hi_energies) + pset2 = RectangularSurvivalProbabilityPointingSet( + pset2_l1c, + Sensor.Hi90, + SpinPhase.RamOnly, + pset2_glows, + self.hi_energies, + ) spice_frame = SpiceFrame.IMAP_HAE - actual_skymap = RectangularSurvivalProbabilitySkyMap([pset1, pset2], - 0.1, spice_frame) + actual_skymap = RectangularSurvivalProbabilitySkyMap( + [pset1, pset2], 0.1, spice_frame + ) survival_probability_dataset = actual_skymap.to_dataset() - expected_flags_in_full_shape = np.full((1, 2, 3600, 1800), MapL3Flags.NONE) + expected_flags_in_full_shape = np.full( + (1, 2, 3600, 1800), MapL3Flags.NONE + ) expected_flags_in_full_shape[:, :, 300, :] = expected_quality_flags self.assertIn("quality_flags", survival_probability_dataset) - self.assertEqual((1, 2, 3600, 1800), survival_probability_dataset["quality_flags"].values.shape) + self.assertEqual( + (1, 2, 3600, 1800), + survival_probability_dataset["quality_flags"].values.shape, + ) - np.testing.assert_array_equal(expected_flags_in_full_shape, survival_probability_dataset["quality_flags"].values) + np.testing.assert_array_equal( + expected_flags_in_full_shape, + survival_probability_dataset["quality_flags"].values, + ) diff --git a/tests/maps/test_spectral_fit.py b/tests/maps/test_spectral_fit.py index 97ad5b804..0494c9f23 100644 --- a/tests/maps/test_spectral_fit.py +++ b/tests/maps/test_spectral_fit.py @@ -6,11 +6,19 @@ import numpy as np from scipy.stats import linregress -from imap_l3_processing.maps.map_models import IntensityMapData, RectangularIntensityMapData +from imap_l3_processing.maps.map_models import ( + IntensityMapData, + RectangularIntensityMapData, +) from imap_l3_processing.maps.mpfit import mpfit from imap_l3_processing.maps.quality_flags import MapL3Flags -from imap_l3_processing.maps.spectral_fit import power_law, fit_arrays_to_power_law, fit_spectral_index_map, \ - calculate_spectral_index_for_multiple_ranges, slice_energy_range_by_bin +from imap_l3_processing.maps.spectral_fit import ( + power_law, + fit_arrays_to_power_law, + fit_spectral_index_map, + calculate_spectral_index_for_multiple_ranges, + slice_energy_range_by_bin, +) from tests.test_helpers import get_test_data_path @@ -20,7 +28,7 @@ def test_power_law_function(self): x = np.array([1, 2, 3]) y = np.array([4, 10, 22]) err = np.array([2, 2, 2]) - keywords = {'xval': x, 'yval': y, 'errval': err} + keywords = {"xval": x, "yval": y, "errval": err} expected_residual = np.array([1, 1, 2]) status, actual_residuals = power_law(params, **keywords) @@ -35,65 +43,102 @@ def test_finds_best_fit(self): errors = 0.2 * np.abs(flux_data) - cases = [ - ("rectangular", (1, 1)), - ("healpix", (1,)) - ] + cases = [("rectangular", (1, 1)), ("healpix", (1,))] for name, spacial_dimension_shape in cases: with self.subTest(name): - flux = np.array(flux_data).reshape(1, len(energies), *spacial_dimension_shape) - uncertainty = np.array(errors).reshape(1, len(energies), *spacial_dimension_shape) - - result_A, result_A_error, result_gamma, result_gamma_error, result_chi_square = fit_arrays_to_power_law(flux, uncertainty, energies) - np.testing.assert_array_equal(result_gamma, - np.array(true_gamma).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_array_almost_equal(result_gamma_error, - np.array([0.060068]).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_array_equal(result_A, np.array(true_A).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_array_almost_equal(result_A_error, - np.array([0.161515]).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_array_almost_equal( result_chi_square ,np.zeros((1,1, *spacial_dimension_shape))) + flux = np.array(flux_data).reshape( + 1, len(energies), *spacial_dimension_shape + ) + uncertainty = np.array(errors).reshape( + 1, len(energies), *spacial_dimension_shape + ) + + ( + result_A, + result_A_error, + result_gamma, + result_gamma_error, + result_chi_square, + ) = fit_arrays_to_power_law(flux, uncertainty, energies) + np.testing.assert_array_equal( + result_gamma, + np.array(true_gamma).reshape(1, 1, *spacial_dimension_shape), + ) + np.testing.assert_array_almost_equal( + result_gamma_error, + np.array([0.060068]).reshape(1, 1, *spacial_dimension_shape), + ) + np.testing.assert_array_equal( + result_A, np.array(true_A).reshape(1, 1, *spacial_dimension_shape) + ) + np.testing.assert_array_almost_equal( + result_A_error, + np.array([0.161515]).reshape(1, 1, *spacial_dimension_shape), + ) + np.testing.assert_array_almost_equal( + result_chi_square, np.zeros((1, 1, *spacial_dimension_shape)) + ) def test_finds_chi_squared_for_fit(self): energies = np.array([1, 10, 30, 100]) flux_data = np.array([99, 10, 3, 1]) - errors = np.array([1,1, 0.1, 1e-3]) + errors = np.array([1, 1, 0.1, 1e-3]) - cases = [ - ("rectangular", (1, 1)), - ("healpix", (1,)) - ] + cases = [("rectangular", (1, 1)), ("healpix", (1,))] for name, spacial_dimension_shape in cases: with self.subTest(name): - flux = np.array(flux_data).reshape(1, len(energies), *spacial_dimension_shape) - uncertainty = np.array(errors).reshape(1, len(energies), *spacial_dimension_shape) + flux = np.array(flux_data).reshape( + 1, len(energies), *spacial_dimension_shape + ) + uncertainty = np.array(errors).reshape( + 1, len(energies), *spacial_dimension_shape + ) - result_A, result_A_error, result_gamma, result_gamma_error, result_chi_square = fit_arrays_to_power_law(flux, uncertainty, energies) - fitted_flux = result_A * np.power(energies.reshape((-1,*spacial_dimension_shape)), -result_gamma) + ( + result_A, + result_A_error, + result_gamma, + result_gamma_error, + result_chi_square, + ) = fit_arrays_to_power_law(flux, uncertainty, energies) + fitted_flux = result_A * np.power( + energies.reshape((-1, *spacial_dimension_shape)), -result_gamma + ) residual = fitted_flux - flux - chisquared = np.sum(np.square(residual/uncertainty)) + chisquared = np.sum(np.square(residual / uncertainty)) reduced_chisquared = chisquared / (len(energies) - 2) - np.testing.assert_array_almost_equal(result_chi_square, reduced_chisquared) + np.testing.assert_array_almost_equal( + result_chi_square, reduced_chisquared + ) def test_handles_small_fluxes(self): - energies = np.array([5.65826829, 8.45453724, 12.70203527]) + energies = np.array([5.65826829, 8.45453724, 12.70203527]) flux_data = np.array([0.12815728, 0.00414464, 0.0383057]) errors = np.array([0.12532869, 0.00414464, 0.02755597]) - cases = [ - ("rectangular", (1, 1)), - ("healpix", (1,)) - ] + cases = [("rectangular", (1, 1)), ("healpix", (1,))] for name, spacial_dimension_shape in cases: with self.subTest(name): - flux = np.array(flux_data).reshape(1, len(energies), *spacial_dimension_shape) - uncertainty = np.array(errors).reshape(1, len(energies), *spacial_dimension_shape) + flux = np.array(flux_data).reshape( + 1, len(energies), *spacial_dimension_shape + ) + uncertainty = np.array(errors).reshape( + 1, len(energies), *spacial_dimension_shape + ) - result_a, result_a_error, result_gamma, result_gamma_error, result_chi_square = fit_arrays_to_power_law(flux, uncertainty, energies) - fitted_flux = result_a * np.power(energies.reshape((-1,*spacial_dimension_shape)), -result_gamma) + ( + result_a, + result_a_error, + result_gamma, + result_gamma_error, + result_chi_square, + ) = fit_arrays_to_power_law(flux, uncertainty, energies) + fitted_flux = result_a * np.power( + energies.reshape((-1, *spacial_dimension_shape)), -result_gamma + ) self.assertGreater(result_a, 0) def test_spectral_fit_map(self): @@ -133,23 +178,43 @@ def test_spectral_fit_map(self): expected_energy_minus = expected_energy_midpoint - 0.5 expected_energy_plus = 10.5 - expected_energy_midpoint expected_energy_label = "0.5 - 10.5" - np.testing.assert_array_equal(spectral_intensity_map.ena_spectral_index, - np.array(true_gamma).reshape(1, 1, 1, 1)) + np.testing.assert_array_equal( + spectral_intensity_map.ena_spectral_index, + np.array(true_gamma).reshape(1, 1, 1, 1), + ) - np.testing.assert_array_almost_equal(spectral_intensity_map.ena_spectral_index_stat_uncert, - np.array([0.060068]).reshape(1, 1, 1, 1)) + np.testing.assert_array_almost_equal( + spectral_intensity_map.ena_spectral_index_stat_uncert, + np.array([0.060068]).reshape(1, 1, 1, 1), + ) - np.testing.assert_array_almost_equal(spectral_intensity_map.ena_spectral_index_scalar_coefficient_stat_uncert, - np.array([0.161515]).reshape(1, 1, 1, 1)) + np.testing.assert_array_almost_equal( + spectral_intensity_map.ena_spectral_index_scalar_coefficient_stat_uncert, + np.array([0.161515]).reshape(1, 1, 1, 1), + ) - np.testing.assert_array_almost_equal(spectral_intensity_map.energy, [expected_energy_midpoint]) - np.testing.assert_array_almost_equal(spectral_intensity_map.energy_delta_minus, [expected_energy_minus]) - np.testing.assert_array_almost_equal(spectral_intensity_map.energy_delta_plus, [expected_energy_plus]) - np.testing.assert_equal(spectral_intensity_map.energy_label, [expected_energy_label]) + np.testing.assert_array_almost_equal( + spectral_intensity_map.energy, [expected_energy_midpoint] + ) + np.testing.assert_array_almost_equal( + spectral_intensity_map.energy_delta_minus, [expected_energy_minus] + ) + np.testing.assert_array_almost_equal( + spectral_intensity_map.energy_delta_plus, [expected_energy_plus] + ) + np.testing.assert_equal( + spectral_intensity_map.energy_label, [expected_energy_label] + ) - np.testing.assert_array_almost_equal(spectral_intensity_map.longitude, data.longitude) - np.testing.assert_array_almost_equal(spectral_intensity_map.latitude, data.latitude) - np.testing.assert_array_almost_equal(spectral_intensity_map.solid_angle, data.solid_angle) + np.testing.assert_array_almost_equal( + spectral_intensity_map.longitude, data.longitude + ) + np.testing.assert_array_almost_equal( + spectral_intensity_map.latitude, data.latitude + ) + np.testing.assert_array_almost_equal( + spectral_intensity_map.solid_angle, data.solid_angle + ) def test_spectral_fit_fields_other_than_fit_fields(self): input_energies = np.array([1, 10, 99]) + 0.5 @@ -182,7 +247,7 @@ def test_spectral_fit_fields_other_than_fit_fields(self): ena_intensity=np.full(input_shape, 1), ena_intensity_sys_err=np.full(input_shape, 1), ena_intensity_stat_uncert=np.full(input_shape, 1), - quality_flags=quality_flags + quality_flags=quality_flags, ) input_map.obs_date[0, 0] = datetime(2025, 1, 1) @@ -207,14 +272,21 @@ def test_spectral_fit_fields_other_than_fit_fields(self): self.assertEqual("1.0 - 100.0", output.energy_label[0]) expected_ena_shape = np.array([1, 1, len(longitude), len(latitude)]) - np.testing.assert_array_equal(output.obs_date, - np.full(expected_ena_shape, datetime(2026, 1, 1))) - np.testing.assert_array_equal(output.obs_date_range, np.full(expected_ena_shape, 2)) - np.testing.assert_array_equal(output.exposure_factor, np.full(expected_ena_shape, 6)) + np.testing.assert_array_equal( + output.obs_date, np.full(expected_ena_shape, datetime(2026, 1, 1)) + ) + np.testing.assert_array_equal( + output.obs_date_range, np.full(expected_ena_shape, 2) + ) + np.testing.assert_array_equal( + output.exposure_factor, np.full(expected_ena_shape, 6) + ) expected_quality_flags = np.full(expected_ena_shape, MapL3Flags.NONE) expected_quality_flags[0, 0, 0] = MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO - expected_quality_flags[0, 0, 1] = MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO | MapL3Flags.PREDICTIVE_EPHEMERIS + expected_quality_flags[0, 0, 1] = ( + MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO | MapL3Flags.PREDICTIVE_EPHEMERIS + ) expected_quality_flags[0, 0, 2:4] = MapL3Flags.PREDICTIVE_EPHEMERIS expected_quality_flags[0, 0, 4] = MapL3Flags.PERSISTED_LAST_POINT @@ -230,22 +302,34 @@ def test_finds_best_fit_with_nan_in_flux(self): errors = 0.2 * np.abs(flux_data) - cases = [ - ("rectangular", (1, 1)), - ("healpix", (1,)) - ] + cases = [("rectangular", (1, 1)), ("healpix", (1,))] for name, spacial_dimension_shape in cases: with self.subTest(name): - flux = np.array(flux_data).reshape(1, len(energies), *spacial_dimension_shape) - uncertainty = np.array(errors).reshape(1, len(energies), *spacial_dimension_shape) - - scalar_coefficients, scalar_coefficient_error, result, result_error, chisq = fit_arrays_to_power_law(flux, uncertainty, energies) - np.testing.assert_array_equal(result, np.array(true_gamma).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_allclose(scalar_coefficients, - np.array(true_A).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_array_equal(chisq, np.zeros((1, 1, *spacial_dimension_shape))) + flux = np.array(flux_data).reshape( + 1, len(energies), *spacial_dimension_shape + ) + uncertainty = np.array(errors).reshape( + 1, len(energies), *spacial_dimension_shape + ) + ( + scalar_coefficients, + scalar_coefficient_error, + result, + result_error, + chisq, + ) = fit_arrays_to_power_law(flux, uncertainty, energies) + np.testing.assert_array_equal( + result, np.array(true_gamma).reshape(1, 1, *spacial_dimension_shape) + ) + np.testing.assert_allclose( + scalar_coefficients, + np.array(true_A).reshape(1, 1, *spacial_dimension_shape), + ) + np.testing.assert_array_equal( + chisq, np.zeros((1, 1, *spacial_dimension_shape)) + ) def test_spectral_fit_map_negative_gammas(self): epoch = np.array([datetime.now()]) @@ -284,23 +368,45 @@ def test_spectral_fit_map_negative_gammas(self): expected_energy_minus = expected_energy_midpoint - 0.5 expected_energy_plus = 10.5 - expected_energy_midpoint expected_energy_label = "0.5 - 10.5" - np.testing.assert_array_equal(spectral_intensity_map.ena_spectral_index_stat_uncert, - np.full(shape=(1, 1, 1, 1), fill_value=np.nan)) - np.testing.assert_array_equal(spectral_intensity_map.ena_spectral_index_scalar_coefficient, - np.full(shape=(1, 1, 1, 1), fill_value=np.nan)) - np.testing.assert_array_equal(spectral_intensity_map.ena_spectral_index, - np.full(shape=(1, 1, 1, 1), fill_value=np.nan)) - np.testing.assert_array_equal(spectral_intensity_map.ena_spectral_index_chisq, - np.full(shape=(1, 1, 1, 1), fill_value=np.nan)) - - np.testing.assert_array_almost_equal(spectral_intensity_map.energy, [expected_energy_midpoint]) - np.testing.assert_array_almost_equal(spectral_intensity_map.energy_delta_minus, [expected_energy_minus]) - np.testing.assert_array_almost_equal(spectral_intensity_map.energy_delta_plus, [expected_energy_plus]) - np.testing.assert_equal(spectral_intensity_map.energy_label, [expected_energy_label]) - - np.testing.assert_array_almost_equal(spectral_intensity_map.longitude, data.longitude) - np.testing.assert_array_almost_equal(spectral_intensity_map.latitude, data.latitude) - np.testing.assert_array_almost_equal(spectral_intensity_map.solid_angle, data.solid_angle) + np.testing.assert_array_equal( + spectral_intensity_map.ena_spectral_index_stat_uncert, + np.full(shape=(1, 1, 1, 1), fill_value=np.nan), + ) + np.testing.assert_array_equal( + spectral_intensity_map.ena_spectral_index_scalar_coefficient, + np.full(shape=(1, 1, 1, 1), fill_value=np.nan), + ) + np.testing.assert_array_equal( + spectral_intensity_map.ena_spectral_index, + np.full(shape=(1, 1, 1, 1), fill_value=np.nan), + ) + np.testing.assert_array_equal( + spectral_intensity_map.ena_spectral_index_chisq, + np.full(shape=(1, 1, 1, 1), fill_value=np.nan), + ) + + np.testing.assert_array_almost_equal( + spectral_intensity_map.energy, [expected_energy_midpoint] + ) + np.testing.assert_array_almost_equal( + spectral_intensity_map.energy_delta_minus, [expected_energy_minus] + ) + np.testing.assert_array_almost_equal( + spectral_intensity_map.energy_delta_plus, [expected_energy_plus] + ) + np.testing.assert_equal( + spectral_intensity_map.energy_label, [expected_energy_label] + ) + + np.testing.assert_array_almost_equal( + spectral_intensity_map.longitude, data.longitude + ) + np.testing.assert_array_almost_equal( + spectral_intensity_map.latitude, data.latitude + ) + np.testing.assert_array_almost_equal( + spectral_intensity_map.solid_angle, data.solid_angle + ) def test_finds_best_fit_with_nan_in_uncertainty(self): energies = np.geomspace(1, 10, 23) @@ -312,16 +418,35 @@ def test_finds_best_fit_with_nan_in_uncertainty(self): errors[0] = np.nan errors[-1] = np.nan - for name, spacial_dimension_shape in [("rectangular", (1, 1)), ("healpix", (1,))]: + for name, spacial_dimension_shape in [ + ("rectangular", (1, 1)), + ("healpix", (1,)), + ]: with self.subTest(name): - flux = np.array(flux_data).reshape(1, len(energies), *spacial_dimension_shape) - uncertainty = np.array(errors).reshape(1, len(energies), *spacial_dimension_shape) + flux = np.array(flux_data).reshape( + 1, len(energies), *spacial_dimension_shape + ) + uncertainty = np.array(errors).reshape( + 1, len(energies), *spacial_dimension_shape + ) - scalar_coefficients, scalar_coefficient_error, result, result_error, chisq = fit_arrays_to_power_law(flux, uncertainty, energies) - np.testing.assert_array_equal(result, np.array(true_gamma).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_allclose(scalar_coefficients, - np.array(true_A).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_array_equal(chisq, np.zeros((1, 1, *spacial_dimension_shape))) + ( + scalar_coefficients, + scalar_coefficient_error, + result, + result_error, + chisq, + ) = fit_arrays_to_power_law(flux, uncertainty, energies) + np.testing.assert_array_equal( + result, np.array(true_gamma).reshape(1, 1, *spacial_dimension_shape) + ) + np.testing.assert_allclose( + scalar_coefficients, + np.array(true_A).reshape(1, 1, *spacial_dimension_shape), + ) + np.testing.assert_array_equal( + chisq, np.zeros((1, 1, *spacial_dimension_shape)) + ) def test_finds_best_fit_with_zero_in_flux_and_uncertainty(self): energies = np.geomspace(1, 10, 23) @@ -332,51 +457,84 @@ def test_finds_best_fit_with_zero_in_flux_and_uncertainty(self): flux_data[0:3] = 0 errors[0:3] = 0 - for name, spacial_dimension_shape in [("rectangular", (1, 1)), ("healpix", (1,))]: + for name, spacial_dimension_shape in [ + ("rectangular", (1, 1)), + ("healpix", (1,)), + ]: with self.subTest(name): - flux = np.array(flux_data).reshape(1, len(energies), *spacial_dimension_shape) - uncertainty = np.array(errors).reshape(1, len(energies), *spacial_dimension_shape) + flux = np.array(flux_data).reshape( + 1, len(energies), *spacial_dimension_shape + ) + uncertainty = np.array(errors).reshape( + 1, len(energies), *spacial_dimension_shape + ) - scalar_coefficients, scalar_coefficient_error, result, result_error, chisq = fit_arrays_to_power_law(flux, uncertainty, energies) - np.testing.assert_array_equal(result, np.array(true_gamma).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_array_equal(scalar_coefficients, - np.array(true_A).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_array_equal(chisq, np.zeros((1, 1, *spacial_dimension_shape))) + ( + scalar_coefficients, + scalar_coefficient_error, + result, + result_error, + chisq, + ) = fit_arrays_to_power_law(flux, uncertainty, energies) + np.testing.assert_array_equal( + result, np.array(true_gamma).reshape(1, 1, *spacial_dimension_shape) + ) + np.testing.assert_array_equal( + scalar_coefficients, + np.array(true_A).reshape(1, 1, *spacial_dimension_shape), + ) + np.testing.assert_array_equal( + chisq, np.zeros((1, 1, *spacial_dimension_shape)) + ) def test_finds_best_fit_with_ibex_data(self): energies = np.array([0.71, 1.11, 1.74, 2.73, 4.29]) - flux_data = np.array([[[[46.710853, 61.45169], - [60.682266, 60.523616]], - [[34.896639, 11.987692], - [29.323209, 19.239248]], - [[18.899919, 17.140896], - [16.12169, 14.099767]], - [[14.709325, 12.165362], - [13.114869, 13.726924]], - [[5.815468, 5.071331], - [6.060111, 6.499908]]]]) - errors = np.array([[[[4.17239236e+02, 5.87577556e+02], - [4.59947913e+02, 7.02472212e+02]], - [[6.38942760e+01, 3.62103490e+01], - [2.87564900e+01, 5.06517210e+01]], - [[1.58932960e+01, 1.41284380e+01], - [1.34992710e+01, 9.66067900e+00]], - [[2.01988700e+00, 2.75456600e+00], - [1.83946100e+00, 2.25438200e+00]], - [[3.00438000e-01, 3.76740000e-01], - [2.15943000e-01, 3.76271000e-01]]]]) - - scalar_coefficients, scalar_coefficient_errors, index, index_error, chisq = fit_arrays_to_power_law(flux_data, errors, energies) - np.testing.assert_array_almost_equal(index, np.array([[[[1.811566, 1.489658], - [1.480259, 1.317993]]]])) - np.testing.assert_array_almost_equal(index_error, np.array([[[[0.279224, 0.409522], - [0.260374, 0.318162]]]])) - np.testing.assert_array_almost_equal(scalar_coefficients, np.array([[[[81.92187 , 44.822599], - [52.494651, 44.647085]]]])) - np.testing.assert_array_almost_equal(scalar_coefficient_errors, np.array([[[[32.245376, 26.075033], - [19.584559, 20.080747]]]])) - np.testing.assert_array_almost_equal(chisq, np.array([[[[0.44621 , 0.391638], - [0.344915, 0.475525]]]])) + flux_data = np.array( + [ + [ + [[46.710853, 61.45169], [60.682266, 60.523616]], + [[34.896639, 11.987692], [29.323209, 19.239248]], + [[18.899919, 17.140896], [16.12169, 14.099767]], + [[14.709325, 12.165362], [13.114869, 13.726924]], + [[5.815468, 5.071331], [6.060111, 6.499908]], + ] + ] + ) + errors = np.array( + [ + [ + [[4.17239236e02, 5.87577556e02], [4.59947913e02, 7.02472212e02]], + [[6.38942760e01, 3.62103490e01], [2.87564900e01, 5.06517210e01]], + [[1.58932960e01, 1.41284380e01], [1.34992710e01, 9.66067900e00]], + [[2.01988700e00, 2.75456600e00], [1.83946100e00, 2.25438200e00]], + [ + [3.00438000e-01, 3.76740000e-01], + [2.15943000e-01, 3.76271000e-01], + ], + ] + ] + ) + + scalar_coefficients, scalar_coefficient_errors, index, index_error, chisq = ( + fit_arrays_to_power_law(flux_data, errors, energies) + ) + np.testing.assert_array_almost_equal( + index, np.array([[[[1.811566, 1.489658], [1.480259, 1.317993]]]]) + ) + np.testing.assert_array_almost_equal( + index_error, np.array([[[[0.279224, 0.409522], [0.260374, 0.318162]]]]) + ) + np.testing.assert_array_almost_equal( + scalar_coefficients, + np.array([[[[81.92187, 44.822599], [52.494651, 44.647085]]]]), + ) + np.testing.assert_array_almost_equal( + scalar_coefficient_errors, + np.array([[[[32.245376, 26.075033], [19.584559, 20.080747]]]]), + ) + np.testing.assert_array_almost_equal( + chisq, np.array([[[[0.44621, 0.391638], [0.344915, 0.475525]]]]) + ) def test_finds_best_fit_with_zeros_in_flux_and_not_uncertainty(self): energies = np.geomspace(1, 1e10, 23) @@ -387,17 +545,35 @@ def test_finds_best_fit_with_zeros_in_flux_and_not_uncertainty(self): flux_data[0:3] = 0 - for name, spacial_dimension_shape in [("rectangular", (1, 1)), ("healpix", (1,))]: + for name, spacial_dimension_shape in [ + ("rectangular", (1, 1)), + ("healpix", (1,)), + ]: with self.subTest(name): - flux = np.array(flux_data).reshape(1, len(energies), *spacial_dimension_shape) - uncertainty = np.array(errors).reshape(1, len(energies), *spacial_dimension_shape) + flux = np.array(flux_data).reshape( + 1, len(energies), *spacial_dimension_shape + ) + uncertainty = np.array(errors).reshape( + 1, len(energies), *spacial_dimension_shape + ) - scalar_coefficients, scalar_coefficient_error, result, result_error, chisq = fit_arrays_to_power_law(flux, uncertainty, energies) - np.testing.assert_array_almost_equal(result, - np.array([1.472697]).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_array_almost_equal(scalar_coefficients, - np.array([1.251819]).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_array_almost_equal(chisq, np.array([2.218472]).reshape(1, 1, *spacial_dimension_shape)) + ( + scalar_coefficients, + scalar_coefficient_error, + result, + result_error, + chisq, + ) = fit_arrays_to_power_law(flux, uncertainty, energies) + np.testing.assert_array_almost_equal( + result, np.array([1.472697]).reshape(1, 1, *spacial_dimension_shape) + ) + np.testing.assert_array_almost_equal( + scalar_coefficients, + np.array([1.251819]).reshape(1, 1, *spacial_dimension_shape), + ) + np.testing.assert_array_almost_equal( + chisq, np.array([2.218472]).reshape(1, 1, *spacial_dimension_shape) + ) def test_returns_nan_when_only_one_point_is_valid(self): energies = np.geomspace(1, 1e10, 5) @@ -409,21 +585,43 @@ def test_returns_nan_when_only_one_point_is_valid(self): flux_data[1:] = 0 errors[1:] = 0 - for name, spacial_dimension_shape in [("rectangular", (1, 1)), ("healpix", (1,))]: + for name, spacial_dimension_shape in [ + ("rectangular", (1, 1)), + ("healpix", (1,)), + ]: with self.subTest(name): - flux = np.array(flux_data).reshape(1, len(energies), *spacial_dimension_shape) - uncertainty = np.array(errors).reshape(1, len(energies), *spacial_dimension_shape) - scalar_coefficients, scalar_coefficient_error, result, result_error, chisq = fit_arrays_to_power_law(flux, uncertainty, energies) - - np.testing.assert_array_almost_equal(result, - np.array(np.nan).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_array_almost_equal(result_error, - np.array(np.nan).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_array_equal(scalar_coefficients, - np.array(np.nan).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_array_equal(scalar_coefficient_error, - np.array(np.nan).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_array_equal(chisq, np.array(np.nan).reshape(1, 1, *spacial_dimension_shape)) + flux = np.array(flux_data).reshape( + 1, len(energies), *spacial_dimension_shape + ) + uncertainty = np.array(errors).reshape( + 1, len(energies), *spacial_dimension_shape + ) + ( + scalar_coefficients, + scalar_coefficient_error, + result, + result_error, + chisq, + ) = fit_arrays_to_power_law(flux, uncertainty, energies) + + np.testing.assert_array_almost_equal( + result, np.array(np.nan).reshape(1, 1, *spacial_dimension_shape) + ) + np.testing.assert_array_almost_equal( + result_error, + np.array(np.nan).reshape(1, 1, *spacial_dimension_shape), + ) + np.testing.assert_array_equal( + scalar_coefficients, + np.array(np.nan).reshape(1, 1, *spacial_dimension_shape), + ) + np.testing.assert_array_equal( + scalar_coefficient_error, + np.array(np.nan).reshape(1, 1, *spacial_dimension_shape), + ) + np.testing.assert_array_equal( + chisq, np.array(np.nan).reshape(1, 1, *spacial_dimension_shape) + ) def test_returns_nan_when_there_are_less_than_2_positive_fluxes(self): energies = np.geomspace(1, 1e10, 5) @@ -431,21 +629,43 @@ def test_returns_nan_when_there_are_less_than_2_positive_fluxes(self): flux_data = np.full_like(energies, 0) errors = np.full_like(flux_data, 2) - for name, spacial_dimension_shape in [("rectangular", (1, 1)), ("healpix", (1,))]: + for name, spacial_dimension_shape in [ + ("rectangular", (1, 1)), + ("healpix", (1,)), + ]: with self.subTest(name): - flux = np.array(flux_data).reshape(1, len(energies), *spacial_dimension_shape) - uncertainty = np.array(errors).reshape(1, len(energies), *spacial_dimension_shape) - - scalar_coefficients, scalar_coefficient_error, result, result_error, chisq = fit_arrays_to_power_law(flux, uncertainty, energies) - np.testing.assert_array_almost_equal(result, - np.array(np.nan).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_array_almost_equal(result_error, - np.array(np.nan).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_array_almost_equal(scalar_coefficients, - np.array(np.nan).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_array_almost_equal(scalar_coefficient_error, - np.array(np.nan).reshape(1, 1, *spacial_dimension_shape)) - np.testing.assert_array_equal(chisq, np.array(np.nan).reshape(1, 1, *spacial_dimension_shape)) + flux = np.array(flux_data).reshape( + 1, len(energies), *spacial_dimension_shape + ) + uncertainty = np.array(errors).reshape( + 1, len(energies), *spacial_dimension_shape + ) + + ( + scalar_coefficients, + scalar_coefficient_error, + result, + result_error, + chisq, + ) = fit_arrays_to_power_law(flux, uncertainty, energies) + np.testing.assert_array_almost_equal( + result, np.array(np.nan).reshape(1, 1, *spacial_dimension_shape) + ) + np.testing.assert_array_almost_equal( + result_error, + np.array(np.nan).reshape(1, 1, *spacial_dimension_shape), + ) + np.testing.assert_array_almost_equal( + scalar_coefficients, + np.array(np.nan).reshape(1, 1, *spacial_dimension_shape), + ) + np.testing.assert_array_almost_equal( + scalar_coefficient_error, + np.array(np.nan).reshape(1, 1, *spacial_dimension_shape), + ) + np.testing.assert_array_equal( + chisq, np.array(np.nan).reshape(1, 1, *spacial_dimension_shape) + ) def test_spectral_fit_can_fit_multiple_energy_ranges(self): epoch = np.array([datetime.now()]) @@ -453,8 +673,12 @@ def test_spectral_fit_can_fit_multiple_energy_ranges(self): input_energy_range_2 = np.geomspace(101, 10000, 12) true_A_range_1, true_gamma_range_1 = 2.0, 1.5 true_A_range_2, true_gamma_range_2 = 0.1, 3.5 - flux_data_range_1 = true_A_range_1 * np.power(input_energy_range_1, -true_gamma_range_1) - flux_data_range_2 = true_A_range_2 * np.power(input_energy_range_2, -true_gamma_range_2) + flux_data_range_1 = true_A_range_1 * np.power( + input_energy_range_1, -true_gamma_range_1 + ) + flux_data_range_2 = true_A_range_2 * np.power( + input_energy_range_2, -true_gamma_range_2 + ) longitude = np.array([45]) latitude = np.array([90]) errors_range_1 = 0.1 * np.abs(flux_data_range_1) @@ -470,12 +694,16 @@ def test_spectral_fit_can_fit_multiple_energy_ranges(self): quality_flags_range_2 = np.full((12,), MapL3Flags.NONE) quality_flags_range_2[1] = MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO quality_flags_range_2[2] = MapL3Flags.PREDICTIVE_EPHEMERIS - quality_flags = np.concat((quality_flags_range_1, quality_flags_range_2)).reshape(full_shape) + quality_flags = np.concat( + (quality_flags_range_1, quality_flags_range_2) + ).reshape(full_shape) - expected_quality_flags = np.array([ - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO | MapL3Flags.PREDICTIVE_EPHEMERIS - ]).reshape((1, 2, 1, 1)) + expected_quality_flags = np.array( + [ + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO | MapL3Flags.PREDICTIVE_EPHEMERIS, + ] + ).reshape((1, 2, 1, 1)) data = IntensityMapData( epoch=epoch, @@ -499,52 +727,84 @@ def test_spectral_fit_can_fit_multiple_energy_ranges(self): ) output_energies = np.array([[1, 100.5], [100.5, 10000.5]]) - spectral_index_map_data = calculate_spectral_index_for_multiple_ranges(data, output_energies) - np.testing.assert_array_equal(spectral_index_map_data.ena_spectral_index[0, 0, 0, 0], true_gamma_range_1) - np.testing.assert_array_equal(spectral_index_map_data.ena_spectral_index[0, 1, 0, 0], true_gamma_range_2) - np.testing.assert_almost_equal(spectral_index_map_data.ena_spectral_index_scalar_coefficient[0, 0, 0, 0], - true_A_range_1) - np.testing.assert_almost_equal(spectral_index_map_data.ena_spectral_index_scalar_coefficient[0, 1, 0, 0], - true_A_range_2) - np.testing.assert_array_almost_equal(spectral_index_map_data.ena_spectral_index_stat_uncert[0, 0, 0, 0], - 0.020704) - np.testing.assert_array_almost_equal(spectral_index_map_data.ena_spectral_index_stat_uncert[0, 1, 0, 0], - 2.00179e-05) - - np.testing.assert_almost_equal(spectral_index_map_data.energy, - np.array([7.088723439378913, 1002.5219448969683])) - np.testing.assert_almost_equal(spectral_index_map_data.energy_delta_plus, np.array([93.4112766, 8997.9780551])) - np.testing.assert_almost_equal(spectral_index_map_data.energy_delta_minus, np.array([6.5887234, 902.0219449])) + spectral_index_map_data = calculate_spectral_index_for_multiple_ranges( + data, output_energies + ) + np.testing.assert_array_equal( + spectral_index_map_data.ena_spectral_index[0, 0, 0, 0], true_gamma_range_1 + ) + np.testing.assert_array_equal( + spectral_index_map_data.ena_spectral_index[0, 1, 0, 0], true_gamma_range_2 + ) + np.testing.assert_almost_equal( + spectral_index_map_data.ena_spectral_index_scalar_coefficient[0, 0, 0, 0], + true_A_range_1, + ) + np.testing.assert_almost_equal( + spectral_index_map_data.ena_spectral_index_scalar_coefficient[0, 1, 0, 0], + true_A_range_2, + ) + np.testing.assert_array_almost_equal( + spectral_index_map_data.ena_spectral_index_stat_uncert[0, 0, 0, 0], 0.020704 + ) + np.testing.assert_array_almost_equal( + spectral_index_map_data.ena_spectral_index_stat_uncert[0, 1, 0, 0], + 2.00179e-05, + ) + + np.testing.assert_almost_equal( + spectral_index_map_data.energy, + np.array([7.088723439378913, 1002.5219448969683]), + ) + np.testing.assert_almost_equal( + spectral_index_map_data.energy_delta_plus, + np.array([93.4112766, 8997.9780551]), + ) + np.testing.assert_almost_equal( + spectral_index_map_data.energy_delta_minus, + np.array([6.5887234, 902.0219449]), + ) np.testing.assert_array_equal(spectral_index_map_data.epoch, epoch) - np.testing.assert_array_equal(spectral_index_map_data.epoch_delta, data.epoch_delta) + np.testing.assert_array_equal( + spectral_index_map_data.epoch_delta, data.epoch_delta + ) np.testing.assert_array_equal(spectral_index_map_data.latitude, latitude) np.testing.assert_array_equal(spectral_index_map_data.longitude, longitude) - np.testing.assert_array_equal(spectral_index_map_data.solid_angle, data.solid_angle) + np.testing.assert_array_equal( + spectral_index_map_data.solid_angle, data.solid_angle + ) np.testing.assert_array_equal( spectral_index_map_data.energy_label, ["0.5 - 100.5", "100.5 - 10000.5"] ) - np.testing.assert_array_equal(spectral_index_map_data.exposure_factor, np.reshape([11, 12], (1, 2, 1, 1))) - np.testing.assert_array_equal(spectral_index_map_data.obs_date, - np.full((1, 2, 1, 1), datetime(year=2010, month=1, day=1))) - np.testing.assert_array_equal(spectral_index_map_data.obs_date_range, np.full((1, 2, 1, 1), 100000)) + np.testing.assert_array_equal( + spectral_index_map_data.exposure_factor, np.reshape([11, 12], (1, 2, 1, 1)) + ) + np.testing.assert_array_equal( + spectral_index_map_data.obs_date, + np.full((1, 2, 1, 1), datetime(year=2010, month=1, day=1)), + ) + np.testing.assert_array_equal( + spectral_index_map_data.obs_date_range, np.full((1, 2, 1, 1), 100000) + ) self.assertEqual((1, 2, 1, 1), spectral_index_map_data.quality_flags.shape) - np.testing.assert_array_equal(spectral_index_map_data.quality_flags, expected_quality_flags) + np.testing.assert_array_equal( + spectral_index_map_data.quality_flags, expected_quality_flags + ) - @patch('imap_l3_processing.maps.spectral_fit.mpfit') - def test_spectral_fit_returns_nan_if_fit_status_is_not_positive_or_equal_to_five(self, mock_mpfit): + @patch("imap_l3_processing.maps.spectral_fit.mpfit") + def test_spectral_fit_returns_nan_if_fit_status_is_not_positive_or_equal_to_five( + self, mock_mpfit + ): energies = np.geomspace(1, 10, 23) true_A, true_gamma = 2.0, 1.5 flux_data = true_A * np.power(energies, -true_gamma) errors = 0.2 * np.abs(flux_data) - cases = [ - ("rectangular", (1, 1)), - ("healpix", (1,)) - ] + cases = [("rectangular", (1, 1)), ("healpix", (1,))] invalid_status_cases = [-1, 0, 5] @@ -552,20 +812,32 @@ def test_spectral_fit_returns_nan_if_fit_status_is_not_positive_or_equal_to_five for status_code in invalid_status_cases: with self.subTest(f"Pixelization: {name}, status_code: {status_code}"): mock_mpfit.return_value = Mock(status=status_code, params=(0, 0)) - flux = np.array(flux_data).reshape(1, len(energies), *spacial_dimension_shape) - uncertainty = np.array(errors).reshape(1, len(energies), *spacial_dimension_shape) - - scalar, scalar_error, result, result_error, chisq = fit_arrays_to_power_law(flux, uncertainty, energies) - expected_nan_result = np.full((1, 1, *spacial_dimension_shape), np.nan) + flux = np.array(flux_data).reshape( + 1, len(energies), *spacial_dimension_shape + ) + uncertainty = np.array(errors).reshape( + 1, len(energies), *spacial_dimension_shape + ) + + scalar, scalar_error, result, result_error, chisq = ( + fit_arrays_to_power_law(flux, uncertainty, energies) + ) + expected_nan_result = np.full( + (1, 1, *spacial_dimension_shape), np.nan + ) np.testing.assert_array_equal(result, expected_nan_result) np.testing.assert_array_equal(result_error, expected_nan_result) np.testing.assert_array_equal(scalar, expected_nan_result) np.testing.assert_array_equal(scalar_error, expected_nan_result) np.testing.assert_array_equal(chisq, expected_nan_result) - @patch('imap_l3_processing.maps.spectral_fit.mpfit', wraps=mpfit) - @patch('imap_l3_processing.maps.spectral_fit.scipy.stats.linregress', wraps=linregress) - def test_passes_initial_guess_to_mpfit_based_on_linear_fit_in_log_space(self, mock_linregress, mock_mpfit): + @patch("imap_l3_processing.maps.spectral_fit.mpfit", wraps=mpfit) + @patch( + "imap_l3_processing.maps.spectral_fit.scipy.stats.linregress", wraps=linregress + ) + def test_passes_initial_guess_to_mpfit_based_on_linear_fit_in_log_space( + self, mock_linregress, mock_mpfit + ): energies = np.geomspace(1e1, 1e4, 6) flux_data = np.array([1e7, 1e5, 1e6, 1e4, 1e5, 1e3]) @@ -612,7 +884,9 @@ def test_spectral_fit_against_validation_data(self): delimiter=",", dtype=np.float64, ).T - expected_a = np.loadtxt(get_test_data_path(expected_a_path), delimiter=",", dtype=np.float64).T + expected_a = np.loadtxt( + get_test_data_path(expected_a_path), delimiter=",", dtype=np.float64 + ).T expected_gamma_sigma = np.loadtxt( get_test_data_path(expected_sigma_path), delimiter=",", @@ -623,31 +897,44 @@ def test_spectral_fit_against_validation_data(self): delimiter=",", dtype=np.float64, ).T - expected_chisq = np.loadtxt(get_test_data_path(expected_chisq_path), delimiter=",", dtype=np.float64).T + expected_chisq = np.loadtxt( + get_test_data_path(expected_chisq_path), + delimiter=",", + dtype=np.float64, + ).T - output_data = calculate_spectral_index_for_multiple_ranges(input_data, [[0, np.inf]]) + output_data = calculate_spectral_index_for_multiple_ranges( + input_data, [[0, np.inf]] + ) - np.testing.assert_allclose(output_data.ena_spectral_index[0, 0], - expected_gamma, atol=1e-3) + np.testing.assert_allclose( + output_data.ena_spectral_index[0, 0], expected_gamma, atol=1e-3 + ) np.testing.assert_allclose( output_data.ena_spectral_index_stat_uncert[0, 0], expected_gamma_sigma, atol=1e-3, ) - np.testing.assert_allclose(output_data.ena_spectral_index_scalar_coefficient[0, 0], - expected_a, atol=1e-3) - np.testing.assert_allclose(output_data.ena_spectral_index_scalar_coefficient_stat_uncert[0, 0], - expected_a_sigma, atol=1e-3) - np.testing.assert_allclose(output_data.ena_spectral_index_chisq[0, 0], expected_chisq) - np.testing.assert_allclose(output_data.quality_flags, np.full((1,1,90,45), MapL3Flags.NONE)) + np.testing.assert_allclose( + output_data.ena_spectral_index_scalar_coefficient[0, 0], + expected_a, + atol=1e-3, + ) + np.testing.assert_allclose( + output_data.ena_spectral_index_scalar_coefficient_stat_uncert[0, 0], + expected_a_sigma, + atol=1e-3, + ) + np.testing.assert_allclose( + output_data.ena_spectral_index_chisq[0, 0], expected_chisq + ) + np.testing.assert_allclose( + output_data.quality_flags, np.full((1, 1, 90, 45), MapL3Flags.NONE) + ) def test_slice_energy_range_by_bin(self): def build_array(*values): - return np.array([ - [ - [[a]] for a in values - ] - ]) + return np.array([[[[a]] for a in values]]) input_data = IntensityMapData( epoch=sentinel.epoch, @@ -659,12 +946,13 @@ def build_array(*values): latitude=sentinel.latitude, longitude=sentinel.longitude, exposure_factor=build_array(1, 2, 3, 4, 5), - obs_date=build_array(datetime(2026, 1, 1), - datetime(2026, 1, 2), - datetime(2026, 1, 3), - datetime(2026, 1, 4), - datetime(2026, 1, 5), - ), + obs_date=build_array( + datetime(2026, 1, 1), + datetime(2026, 1, 2), + datetime(2026, 1, 3), + datetime(2026, 1, 4), + datetime(2026, 1, 5), + ), obs_date_range=build_array(1000, 2000, 3000, 4000, 5000), solid_angle=sentinel.solid_angle, ena_intensity=build_array(100, 200, 300, 400, 500), @@ -675,7 +963,7 @@ def build_array(*values): MapL3Flags.NONE, MapL3Flags.PREDICTIVE_EPHEMERIS, MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.NONE + MapL3Flags.NONE, ), ) @@ -689,31 +977,32 @@ def build_array(*values): latitude=sentinel.latitude, longitude=sentinel.longitude, exposure_factor=build_array(1, 2, 3), - obs_date=build_array(datetime(2026, 1, 1), - datetime(2026, 1, 2), - datetime(2026, 1, 3), - ), + obs_date=build_array( + datetime(2026, 1, 1), + datetime(2026, 1, 2), + datetime(2026, 1, 3), + ), obs_date_range=build_array(1000, 2000, 3000), solid_angle=sentinel.solid_angle, ena_intensity=build_array(100, 200, 300), ena_intensity_stat_uncert=build_array(11, 12, 13), ena_intensity_sys_err=build_array(21, 22, 23), - quality_flags=build_array(MapL3Flags.NONE, MapL3Flags.NONE, MapL3Flags.PREDICTIVE_EPHEMERIS), + quality_flags=build_array( + MapL3Flags.NONE, MapL3Flags.NONE, MapL3Flags.PREDICTIVE_EPHEMERIS + ), ) actual = slice_energy_range_by_bin(input_data, 1, 3) self.assertIsInstance(actual, IntensityMapData) for field in dataclasses.fields(expected_data): - np.testing.assert_equal(getattr(actual, field.name), getattr(expected_data, field.name)) + np.testing.assert_equal( + getattr(actual, field.name), getattr(expected_data, field.name) + ) def test_slice_energy_range_by_bin_raises_error(self): def build_array(*values): - return np.array([ - [ - [[a]] for a in values - ] - ]) + return np.array([[[[a]] for a in values]]) input_data = IntensityMapData( epoch=sentinel.epoch, @@ -725,12 +1014,13 @@ def build_array(*values): latitude=sentinel.latitude, longitude=sentinel.longitude, exposure_factor=build_array(1, 2, 3, 4, 5), - obs_date=build_array(datetime(2026, 1, 1), - datetime(2026, 1, 2), - datetime(2026, 1, 3), - datetime(2026, 1, 4), - datetime(2026, 1, 5), - ), + obs_date=build_array( + datetime(2026, 1, 1), + datetime(2026, 1, 2), + datetime(2026, 1, 3), + datetime(2026, 1, 4), + datetime(2026, 1, 5), + ), obs_date_range=build_array(1000, 2000, 3000, 4000, 5000), solid_angle=sentinel.solid_angle, ena_intensity=build_array(100, 200, 300, 400, 500), @@ -741,19 +1031,16 @@ def build_array(*values): MapL3Flags.NONE, MapL3Flags.PREDICTIVE_EPHEMERIS, MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.NONE + MapL3Flags.NONE, ), ) - cases = [ - (1, 10), - (0, 3), - (3, 0), - (1, 1), - (10, 3) - ] + cases = [(1, 10), (0, 3), (3, 0), (1, 1), (10, 3)] for start_bin, end_bin in cases: with self.subTest(f"{start_bin}-{end_bin}"): with self.assertRaises(ValueError) as cm: slice_energy_range_by_bin(input_data, start_bin, end_bin) - self.assertEqual(str(cm.exception), f"Error slicing energy bins {start_bin},{end_bin}") + self.assertEqual( + str(cm.exception), + f"Error slicing energy bins {start_bin},{end_bin}", + ) diff --git a/tests/maps/test_survival_probability_processing.py b/tests/maps/test_survival_probability_processing.py index 7c569d353..60cb9fbd5 100644 --- a/tests/maps/test_survival_probability_processing.py +++ b/tests/maps/test_survival_probability_processing.py @@ -6,55 +6,99 @@ from imap_processing.ena_maps.utils.coordinates import CoordNames from imap_processing.spice.geometry import SpiceFrame -from imap_l3_processing.maps.hilo_l3_survival_dependencies import HiLoL3SurvivalDependencies -from imap_l3_processing.maps.map_descriptors import PixelSize, parse_map_descriptor, ReferenceFrame -from imap_l3_processing.maps.map_models import GlowsL3eRectangularMapInputData, InputRectangularPointingSet +from imap_l3_processing.maps.hilo_l3_survival_dependencies import ( + HiLoL3SurvivalDependencies, +) +from imap_l3_processing.maps.map_descriptors import ( + PixelSize, + parse_map_descriptor, + ReferenceFrame, +) +from imap_l3_processing.maps.map_models import ( + GlowsL3eRectangularMapInputData, + InputRectangularPointingSet, +) from imap_l3_processing.maps.quality_flags import MapL3Flags -from imap_l3_processing.maps.survival_probability_processing import process_survival_probabilities, \ - combine_glows_l3e_with_l1c_pointing, filter_bad_days -from tests.maps.test_builders import create_rectangular_intensity_map_data, create_l1c_pset, create_l3e_pset +from imap_l3_processing.maps.survival_probability_processing import ( + process_survival_probabilities, + combine_glows_l3e_with_l1c_pointing, + filter_bad_days, +) +from tests.maps.test_builders import ( + create_rectangular_intensity_map_data, + create_l1c_pset, + create_l3e_pset, +) from tests.spice_test_case import SpiceTestCase from tests.test_helpers import create_dataclass_mock class TestSurvivalProbabilityProcessing(SpiceTestCase): - @patch('imap_l3_processing.maps.survival_probability_processing.RectangularSurvivalProbabilitySkyMap') - @patch('imap_l3_processing.maps.survival_probability_processing.RectangularSurvivalProbabilityPointingSet') - @patch('imap_l3_processing.maps.survival_probability_processing.combine_glows_l3e_with_l1c_pointing') - @patch('imap_l3_processing.maps.survival_probability_processing.filter_bad_days') - def test_process_survival_probability(self, mock_filter_bad_days, mock_combine_glows_l3e_with_l1c_pointing, - mock_survival_probability_pointing_set, mock_survival_skymap): + @patch( + "imap_l3_processing.maps.survival_probability_processing.RectangularSurvivalProbabilitySkyMap" + ) + @patch( + "imap_l3_processing.maps.survival_probability_processing.RectangularSurvivalProbabilityPointingSet" + ) + @patch( + "imap_l3_processing.maps.survival_probability_processing.combine_glows_l3e_with_l1c_pointing" + ) + @patch("imap_l3_processing.maps.survival_probability_processing.filter_bad_days") + def test_process_survival_probability( + self, + mock_filter_bad_days, + mock_combine_glows_l3e_with_l1c_pointing, + mock_survival_probability_pointing_set, + mock_survival_skymap, + ): rng = np.random.default_rng() input_map_flux = rng.random((1, 9, 90, 45)) epoch = datetime.now() - input_map = create_rectangular_intensity_map_data(epoch=[epoch], flux=input_map_flux, - include_ena_intensity_sys_err_minus=True) + input_map = create_rectangular_intensity_map_data( + epoch=[epoch], flux=input_map_flux, include_ena_intensity_sys_err_minus=True + ) intensity_map_data = input_map.intensity_map_data input_map.intensity_map_data.energy = sentinel.hi_l2_energies l2_grid = PixelSize.FourDegrees - l2_descriptor_parts = Mock(sensor=sentinel.l2_sensor, spin_phase=sentinel.l2_spin, grid=l2_grid, - reference_frame=ReferenceFrame.Heliospheric) - dependencies = HiLoL3SurvivalDependencies(l2_data=input_map, - l1c_data=sentinel.l1c_data, - glows_l3e_data=sentinel.glows_l3e_data, - l2_map_descriptor_parts=l2_descriptor_parts, - dependency_file_paths=[]) + l2_descriptor_parts = Mock( + sensor=sentinel.l2_sensor, + spin_phase=sentinel.l2_spin, + grid=l2_grid, + reference_frame=ReferenceFrame.Heliospheric, + ) + dependencies = HiLoL3SurvivalDependencies( + l2_data=input_map, + l1c_data=sentinel.l1c_data, + glows_l3e_data=sentinel.glows_l3e_data, + l2_map_descriptor_parts=l2_descriptor_parts, + dependency_file_paths=[], + ) mock_filter_bad_days.return_value = sentinel.filtered_l1c - mock_combine_glows_l3e_with_l1c_pointing.return_value = [(sentinel.hi_l1c_1, sentinel.glows_l3e_1), - (sentinel.hi_l1c_2, sentinel.glows_l3e_2), - (sentinel.hi_l1c_3, sentinel.glows_l3e_3)] + mock_combine_glows_l3e_with_l1c_pointing.return_value = [ + (sentinel.hi_l1c_1, sentinel.glows_l3e_1), + (sentinel.hi_l1c_2, sentinel.glows_l3e_2), + (sentinel.hi_l1c_3, sentinel.glows_l3e_3), + ] - mock_survival_probability_pointing_set.side_effect = [sentinel.pset_1, sentinel.pset_2, sentinel.pset_3] + mock_survival_probability_pointing_set.side_effect = [ + sentinel.pset_1, + sentinel.pset_2, + sentinel.pset_3, + ] computed_survival_probabilities = rng.random((1, 9, 90, 45)) quality_flags = np.full((1, 9, 90, 45), MapL3Flags.NONE) - quality_flags[rng.random((1, 9, 90, 45)) > 0.8] = MapL3Flags.PREDICTIVE_EPHEMERIS - quality_flags[rng.random((1, 9, 90, 45)) > 0.7] = MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO + quality_flags[rng.random((1, 9, 90, 45)) > 0.8] = ( + MapL3Flags.PREDICTIVE_EPHEMERIS + ) + quality_flags[rng.random((1, 9, 90, 45)) > 0.7] = ( + MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO + ) mock_survival_skymap.return_value.to_dataset.return_value = xr.Dataset( data_vars={ @@ -65,7 +109,7 @@ def test_process_survival_probability(self, mock_filter_bad_days, mock_combine_g CoordNames.AZIMUTH_L2.value, CoordNames.ELEVATION_L2.value, ], - computed_survival_probabilities + computed_survival_probabilities, ), "quality_flags": ( [ @@ -74,7 +118,7 @@ def test_process_survival_probability(self, mock_filter_bad_days, mock_combine_g CoordNames.AZIMUTH_L2.value, CoordNames.ELEVATION_L2.value, ], - quality_flags + quality_flags, ), }, coords={ @@ -82,39 +126,74 @@ def test_process_survival_probability(self, mock_filter_bad_days, mock_combine_g CoordNames.ENERGY_ULTRA_L1C.value: rng.random((9,)), CoordNames.AZIMUTH_L2.value: rng.random((90,)), CoordNames.ELEVATION_L2.value: rng.random((45,)), - } + }, ) - survival_data = process_survival_probabilities(dependencies, SpiceFrame.IMAP_DPS) + survival_data = process_survival_probabilities( + dependencies, SpiceFrame.IMAP_DPS + ) mock_filter_bad_days.assert_called_once_with(sentinel.l1c_data) - mock_combine_glows_l3e_with_l1c_pointing.assert_called_once_with(sentinel.glows_l3e_data, sentinel.filtered_l1c) + mock_combine_glows_l3e_with_l1c_pointing.assert_called_once_with( + sentinel.glows_l3e_data, sentinel.filtered_l1c + ) - mock_survival_probability_pointing_set.assert_has_calls([ - call(sentinel.hi_l1c_1, sentinel.l2_sensor, sentinel.l2_spin, sentinel.glows_l3e_1, - sentinel.hi_l2_energies, cg_corrected=True), - call(sentinel.hi_l1c_2, sentinel.l2_sensor, sentinel.l2_spin, sentinel.glows_l3e_2, - sentinel.hi_l2_energies, cg_corrected=True), - call(sentinel.hi_l1c_3, sentinel.l2_sensor, sentinel.l2_spin, sentinel.glows_l3e_3, - sentinel.hi_l2_energies, cg_corrected=True) - ]) + mock_survival_probability_pointing_set.assert_has_calls( + [ + call( + sentinel.hi_l1c_1, + sentinel.l2_sensor, + sentinel.l2_spin, + sentinel.glows_l3e_1, + sentinel.hi_l2_energies, + cg_corrected=True, + ), + call( + sentinel.hi_l1c_2, + sentinel.l2_sensor, + sentinel.l2_spin, + sentinel.glows_l3e_2, + sentinel.hi_l2_energies, + cg_corrected=True, + ), + call( + sentinel.hi_l1c_3, + sentinel.l2_sensor, + sentinel.l2_spin, + sentinel.glows_l3e_3, + sentinel.hi_l2_energies, + cg_corrected=True, + ), + ] + ) - mock_survival_skymap.assert_called_once_with([sentinel.pset_1, sentinel.pset_2, sentinel.pset_3], - l2_grid, - SpiceFrame.IMAP_DPS) + mock_survival_skymap.assert_called_once_with( + [sentinel.pset_1, sentinel.pset_2, sentinel.pset_3], + l2_grid, + SpiceFrame.IMAP_DPS, + ) mock_survival_skymap.return_value.to_dataset.assert_called_once() - np.testing.assert_array_equal(survival_data.intensity_map_data.ena_intensity, - intensity_map_data.ena_intensity / computed_survival_probabilities) - np.testing.assert_array_equal(survival_data.intensity_map_data.ena_intensity_stat_uncert, - intensity_map_data.ena_intensity_stat_uncert / computed_survival_probabilities) - np.testing.assert_array_equal(survival_data.intensity_map_data.ena_intensity_sys_err, - intensity_map_data.ena_intensity_sys_err / computed_survival_probabilities) + np.testing.assert_array_equal( + survival_data.intensity_map_data.ena_intensity, + intensity_map_data.ena_intensity / computed_survival_probabilities, + ) + np.testing.assert_array_equal( + survival_data.intensity_map_data.ena_intensity_stat_uncert, + intensity_map_data.ena_intensity_stat_uncert + / computed_survival_probabilities, + ) + np.testing.assert_array_equal( + survival_data.intensity_map_data.ena_intensity_sys_err, + intensity_map_data.ena_intensity_sys_err / computed_survival_probabilities, + ) - np.testing.assert_array_equal(survival_data.intensity_map_data.bg_intensity, - intensity_map_data.bg_intensity / computed_survival_probabilities) + np.testing.assert_array_equal( + survival_data.intensity_map_data.bg_intensity, + intensity_map_data.bg_intensity / computed_survival_probabilities, + ) np.testing.assert_array_equal( survival_data.intensity_map_data.bg_intensity_sys_err, intensity_map_data.bg_intensity_sys_err / computed_survival_probabilities, @@ -130,108 +209,197 @@ def test_process_survival_probability(self, mock_filter_bad_days, mock_combine_g computed_survival_probabilities, ) - np.testing.assert_array_equal(survival_data.intensity_map_data.ena_intensity_sys_err_minus, - intensity_map_data.ena_intensity_sys_err_minus / computed_survival_probabilities) - - np.testing.assert_array_equal(survival_data.intensity_map_data.ena_intensity_sys_err_plus, - intensity_map_data.ena_intensity_sys_err_plus / computed_survival_probabilities) - - np.testing.assert_array_equal(survival_data.intensity_map_data.epoch, intensity_map_data.epoch) - np.testing.assert_array_equal(survival_data.intensity_map_data.epoch_delta, intensity_map_data.epoch_delta) - np.testing.assert_array_equal(survival_data.intensity_map_data.energy, intensity_map_data.energy) - np.testing.assert_array_equal(survival_data.intensity_map_data.energy_delta_plus, - intensity_map_data.energy_delta_plus) - np.testing.assert_array_equal(survival_data.intensity_map_data.energy_delta_minus, - intensity_map_data.energy_delta_minus) - np.testing.assert_array_equal(survival_data.intensity_map_data.energy_label, intensity_map_data.energy_label) - np.testing.assert_array_equal(survival_data.intensity_map_data.latitude, intensity_map_data.latitude) - np.testing.assert_array_equal(survival_data.coords.latitude_delta, input_map.coords.latitude_delta) - np.testing.assert_array_equal(survival_data.coords.latitude_label, input_map.coords.latitude_label) - np.testing.assert_array_equal(survival_data.intensity_map_data.longitude, intensity_map_data.longitude) - np.testing.assert_array_equal(survival_data.coords.longitude_delta, input_map.coords.longitude_delta) - np.testing.assert_array_equal(survival_data.coords.longitude_label, input_map.coords.longitude_label) - np.testing.assert_array_equal(survival_data.intensity_map_data.exposure_factor, - intensity_map_data.exposure_factor) - np.testing.assert_array_equal(survival_data.intensity_map_data.obs_date, intensity_map_data.obs_date) - np.testing.assert_array_equal(survival_data.intensity_map_data.obs_date_range, - intensity_map_data.obs_date_range) - np.testing.assert_array_equal(survival_data.intensity_map_data.solid_angle, intensity_map_data.solid_angle) - np.testing.assert_array_equal(survival_data.intensity_map_data.quality_flags, quality_flags) - - @patch('imap_l3_processing.maps.survival_probability_processing.RectangularSurvivalProbabilitySkyMap') - @patch('imap_l3_processing.maps.survival_probability_processing.RectangularSurvivalProbabilityPointingSet') - @patch('imap_l3_processing.maps.survival_probability_processing.combine_glows_l3e_with_l1c_pointing') - @patch('imap_l3_processing.maps.survival_probability_processing.filter_bad_days') - def test_process_survival_probability_cg_correction_argument(self, _mock_filter_bad_days, - mock_combine_glows_l3e_with_l1c_pointing, - mock_survival_probability_pointing_set, - mock_survival_skymap): + np.testing.assert_array_equal( + survival_data.intensity_map_data.ena_intensity_sys_err_minus, + intensity_map_data.ena_intensity_sys_err_minus + / computed_survival_probabilities, + ) + + np.testing.assert_array_equal( + survival_data.intensity_map_data.ena_intensity_sys_err_plus, + intensity_map_data.ena_intensity_sys_err_plus + / computed_survival_probabilities, + ) + + np.testing.assert_array_equal( + survival_data.intensity_map_data.epoch, intensity_map_data.epoch + ) + np.testing.assert_array_equal( + survival_data.intensity_map_data.epoch_delta, intensity_map_data.epoch_delta + ) + np.testing.assert_array_equal( + survival_data.intensity_map_data.energy, intensity_map_data.energy + ) + np.testing.assert_array_equal( + survival_data.intensity_map_data.energy_delta_plus, + intensity_map_data.energy_delta_plus, + ) + np.testing.assert_array_equal( + survival_data.intensity_map_data.energy_delta_minus, + intensity_map_data.energy_delta_minus, + ) + np.testing.assert_array_equal( + survival_data.intensity_map_data.energy_label, + intensity_map_data.energy_label, + ) + np.testing.assert_array_equal( + survival_data.intensity_map_data.latitude, intensity_map_data.latitude + ) + np.testing.assert_array_equal( + survival_data.coords.latitude_delta, input_map.coords.latitude_delta + ) + np.testing.assert_array_equal( + survival_data.coords.latitude_label, input_map.coords.latitude_label + ) + np.testing.assert_array_equal( + survival_data.intensity_map_data.longitude, intensity_map_data.longitude + ) + np.testing.assert_array_equal( + survival_data.coords.longitude_delta, input_map.coords.longitude_delta + ) + np.testing.assert_array_equal( + survival_data.coords.longitude_label, input_map.coords.longitude_label + ) + np.testing.assert_array_equal( + survival_data.intensity_map_data.exposure_factor, + intensity_map_data.exposure_factor, + ) + np.testing.assert_array_equal( + survival_data.intensity_map_data.obs_date, intensity_map_data.obs_date + ) + np.testing.assert_array_equal( + survival_data.intensity_map_data.obs_date_range, + intensity_map_data.obs_date_range, + ) + np.testing.assert_array_equal( + survival_data.intensity_map_data.solid_angle, intensity_map_data.solid_angle + ) + np.testing.assert_array_equal( + survival_data.intensity_map_data.quality_flags, quality_flags + ) + + @patch( + "imap_l3_processing.maps.survival_probability_processing.RectangularSurvivalProbabilitySkyMap" + ) + @patch( + "imap_l3_processing.maps.survival_probability_processing.RectangularSurvivalProbabilityPointingSet" + ) + @patch( + "imap_l3_processing.maps.survival_probability_processing.combine_glows_l3e_with_l1c_pointing" + ) + @patch("imap_l3_processing.maps.survival_probability_processing.filter_bad_days") + def test_process_survival_probability_cg_correction_argument( + self, + _mock_filter_bad_days, + mock_combine_glows_l3e_with_l1c_pointing, + mock_survival_probability_pointing_set, + mock_survival_skymap, + ): test_cases = [True, False] for cg_correction_value in test_cases: rng = np.random.default_rng() input_map_flux = rng.random((1, 9, 90, 45)) epoch = datetime.now() - input_map = create_rectangular_intensity_map_data(epoch=[epoch], flux=input_map_flux) + input_map = create_rectangular_intensity_map_data( + epoch=[epoch], flux=input_map_flux + ) input_map.intensity_map_data.energy = sentinel.hi_l2_energies l2_grid = PixelSize.FourDegrees - l2_descriptor_parts = Mock(sensor=sentinel.l2_sensor, spin_phase=sentinel.l2_spin, grid=l2_grid, - reference_frame=ReferenceFrame.Heliospheric) - dependencies = HiLoL3SurvivalDependencies(l2_data=input_map, - l1c_data=sentinel.l1c_data, - glows_l3e_data=sentinel.glows_l3e_data, - l2_map_descriptor_parts=l2_descriptor_parts, - dependency_file_paths=[]) - - mock_combine_glows_l3e_with_l1c_pointing.return_value = [(sentinel.hi_l1c_1, sentinel.glows_l3e_1), - (sentinel.hi_l1c_2, sentinel.glows_l3e_2), - (sentinel.hi_l1c_3, sentinel.glows_l3e_3)] - - mock_survival_probability_pointing_set.side_effect = [sentinel.pset_1, sentinel.pset_2, sentinel.pset_3] + l2_descriptor_parts = Mock( + sensor=sentinel.l2_sensor, + spin_phase=sentinel.l2_spin, + grid=l2_grid, + reference_frame=ReferenceFrame.Heliospheric, + ) + dependencies = HiLoL3SurvivalDependencies( + l2_data=input_map, + l1c_data=sentinel.l1c_data, + glows_l3e_data=sentinel.glows_l3e_data, + l2_map_descriptor_parts=l2_descriptor_parts, + dependency_file_paths=[], + ) + + mock_combine_glows_l3e_with_l1c_pointing.return_value = [ + (sentinel.hi_l1c_1, sentinel.glows_l3e_1), + (sentinel.hi_l1c_2, sentinel.glows_l3e_2), + (sentinel.hi_l1c_3, sentinel.glows_l3e_3), + ] + + mock_survival_probability_pointing_set.side_effect = [ + sentinel.pset_1, + sentinel.pset_2, + sentinel.pset_3, + ] computed_survival_probabilities = rng.random((1, 9, 90, 45)) - quality_flags = np.full_like(computed_survival_probabilities, MapL3Flags.NONE) - - mock_survival_skymap.return_value.to_dataset.return_value = xr.Dataset({ - "exposure_weighted_survival_probabilities": ( - [ - CoordNames.TIME.value, - CoordNames.ENERGY_ULTRA_L1C.value, - CoordNames.AZIMUTH_L2.value, - CoordNames.ELEVATION_L2.value, - ], - computed_survival_probabilities - ), - "quality_flags": ( - [ - CoordNames.TIME.value, - CoordNames.ENERGY_ULTRA_L1C.value, - CoordNames.AZIMUTH_L2.value, - CoordNames.ELEVATION_L2.value, - ], - quality_flags - ), - }, + quality_flags = np.full_like( + computed_survival_probabilities, MapL3Flags.NONE + ) + + mock_survival_skymap.return_value.to_dataset.return_value = xr.Dataset( + { + "exposure_weighted_survival_probabilities": ( + [ + CoordNames.TIME.value, + CoordNames.ENERGY_ULTRA_L1C.value, + CoordNames.AZIMUTH_L2.value, + CoordNames.ELEVATION_L2.value, + ], + computed_survival_probabilities, + ), + "quality_flags": ( + [ + CoordNames.TIME.value, + CoordNames.ENERGY_ULTRA_L1C.value, + CoordNames.AZIMUTH_L2.value, + CoordNames.ELEVATION_L2.value, + ], + quality_flags, + ), + }, coords={ CoordNames.TIME.value: [epoch], CoordNames.ENERGY_ULTRA_L1C.value: rng.random((9,)), CoordNames.AZIMUTH_L2.value: rng.random((90,)), CoordNames.ELEVATION_L2.value: rng.random((45,)), - }) - - process_survival_probabilities(dependencies, SpiceFrame.IMAP_DPS, - cg_corrected=cg_correction_value) - - mock_survival_probability_pointing_set.assert_has_calls([ - call(sentinel.hi_l1c_1, sentinel.l2_sensor, sentinel.l2_spin, sentinel.glows_l3e_1, - sentinel.hi_l2_energies, cg_corrected=cg_correction_value), - call(sentinel.hi_l1c_2, sentinel.l2_sensor, sentinel.l2_spin, sentinel.glows_l3e_2, - sentinel.hi_l2_energies, cg_corrected=cg_correction_value), - call(sentinel.hi_l1c_3, sentinel.l2_sensor, sentinel.l2_spin, sentinel.glows_l3e_3, - sentinel.hi_l2_energies, cg_corrected=cg_correction_value) - ]) + }, + ) + + process_survival_probabilities( + dependencies, SpiceFrame.IMAP_DPS, cg_corrected=cg_correction_value + ) + + mock_survival_probability_pointing_set.assert_has_calls( + [ + call( + sentinel.hi_l1c_1, + sentinel.l2_sensor, + sentinel.l2_spin, + sentinel.glows_l3e_1, + sentinel.hi_l2_energies, + cg_corrected=cg_correction_value, + ), + call( + sentinel.hi_l1c_2, + sentinel.l2_sensor, + sentinel.l2_spin, + sentinel.glows_l3e_2, + sentinel.hi_l2_energies, + cg_corrected=cg_correction_value, + ), + call( + sentinel.hi_l1c_3, + sentinel.l2_sensor, + sentinel.l2_spin, + sentinel.glows_l3e_3, + sentinel.hi_l2_energies, + cg_corrected=cg_correction_value, + ), + ] + ) def test_integration_uses_fill_values_for_missing_l3e_data(self): t1 = datetime(2025, 4, 29, 12) @@ -240,38 +408,66 @@ def test_integration_uses_fill_values_for_missing_l3e_data(self): t4 = datetime(2025, 5, 15, 12) l1c_psets = [ - create_l1c_pset(epoch=t1, repointing=1, - hae_longitude=np.concat([np.full((1800), 304 + 2), np.full((1800), 124 + 2)])[np.newaxis, - :]), - create_l1c_pset(epoch=t2, repointing=2, - hae_longitude=np.concat([np.full((1800), 312 + 2), np.full((1800), 132 + 2)])[np.newaxis, - :]), - create_l1c_pset(epoch=t3, repointing=3, - hae_longitude=np.concat([np.full((1800), 316 + 2), np.full((1800), 136 + 2)])[np.newaxis, - :]), - create_l1c_pset(epoch=t4, repointing=4, - hae_longitude=np.concat([np.full((1800), 320 + 2), np.full((1800), 140 + 2)])[np.newaxis, - :]) + create_l1c_pset( + epoch=t1, + repointing=1, + hae_longitude=np.concat( + [np.full((1800), 304 + 2), np.full((1800), 124 + 2)] + )[np.newaxis, :], + ), + create_l1c_pset( + epoch=t2, + repointing=2, + hae_longitude=np.concat( + [np.full((1800), 312 + 2), np.full((1800), 132 + 2)] + )[np.newaxis, :], + ), + create_l1c_pset( + epoch=t3, + repointing=3, + hae_longitude=np.concat( + [np.full((1800), 316 + 2), np.full((1800), 136 + 2)] + )[np.newaxis, :], + ), + create_l1c_pset( + epoch=t4, + repointing=4, + hae_longitude=np.concat( + [np.full((1800), 320 + 2), np.full((1800), 140 + 2)] + )[np.newaxis, :], + ), ] l3e_psets = [ create_l3e_pset(epoch=t1, repointing=1), create_l3e_pset(epoch=t2, repointing=2), - create_l3e_pset(epoch=t4, repointing=4) + create_l3e_pset(epoch=t4, repointing=4), ] l2_intensity_map = create_rectangular_intensity_map_data() descriptor = parse_map_descriptor("h90-ena-h-sf-nsp-ram-hae-4deg-3mo") - survival_dependencies = HiLoL3SurvivalDependencies(l2_intensity_map, l1c_psets, l3e_psets, descriptor, - Mock()) + survival_dependencies = HiLoL3SurvivalDependencies( + l2_intensity_map, l1c_psets, l3e_psets, descriptor, Mock() + ) expected_sky_strip = np.full(45, 2.0) default_sp_sky_strip = np.full(45, np.nan) - output_map = process_survival_probabilities(survival_dependencies, SpiceFrame.IMAP_HAE) - np.testing.assert_equal(output_map.intensity_map_data.ena_intensity[0, 0, 76, :], expected_sky_strip) - np.testing.assert_equal(output_map.intensity_map_data.ena_intensity[0, 0, 78, :], expected_sky_strip) - np.testing.assert_equal(output_map.intensity_map_data.ena_intensity[0, 0, 79, :], default_sp_sky_strip) - np.testing.assert_equal(output_map.intensity_map_data.ena_intensity[0, 0, 80, :], expected_sky_strip) + output_map = process_survival_probabilities( + survival_dependencies, SpiceFrame.IMAP_HAE + ) + np.testing.assert_equal( + output_map.intensity_map_data.ena_intensity[0, 0, 76, :], expected_sky_strip + ) + np.testing.assert_equal( + output_map.intensity_map_data.ena_intensity[0, 0, 78, :], expected_sky_strip + ) + np.testing.assert_equal( + output_map.intensity_map_data.ena_intensity[0, 0, 79, :], + default_sp_sky_strip, + ) + np.testing.assert_equal( + output_map.intensity_map_data.ena_intensity[0, 0, 80, :], expected_sky_strip + ) def test_combine_glows_l3e_with_l1c_pointing(self): glows_l3e_data = [ @@ -285,7 +481,7 @@ def test_combine_glows_l3e_with_l1c_pointing(self): create_dataclass_mock(InputRectangularPointingSet, repointing=1), create_dataclass_mock(InputRectangularPointingSet, repointing=2), create_dataclass_mock(InputRectangularPointingSet, repointing=3), - create_dataclass_mock(InputRectangularPointingSet, repointing=4) + create_dataclass_mock(InputRectangularPointingSet, repointing=4), ] expected = [ @@ -302,10 +498,18 @@ def test_filter_bad_days(self): bad_day_exposures = np.zeros((1, 7, 3600, 40)) hi_l1c_data = [ - create_dataclass_mock(InputRectangularPointingSet, exposure_times=good_day_exposures.copy()), - create_dataclass_mock(InputRectangularPointingSet, exposure_times=bad_day_exposures.copy()), - create_dataclass_mock(InputRectangularPointingSet, exposure_times=good_day_exposures.copy()), - create_dataclass_mock(InputRectangularPointingSet, exposure_times=bad_day_exposures.copy()) + create_dataclass_mock( + InputRectangularPointingSet, exposure_times=good_day_exposures.copy() + ), + create_dataclass_mock( + InputRectangularPointingSet, exposure_times=bad_day_exposures.copy() + ), + create_dataclass_mock( + InputRectangularPointingSet, exposure_times=good_day_exposures.copy() + ), + create_dataclass_mock( + InputRectangularPointingSet, exposure_times=bad_day_exposures.copy() + ), ] filtered_psets = filter_bad_days(hi_l1c_data) diff --git a/tests/scripts/test_convert_product_definition_to_yaml.py b/tests/scripts/test_convert_product_definition_to_yaml.py index 1efca334b..d5c8c03e4 100644 --- a/tests/scripts/test_convert_product_definition_to_yaml.py +++ b/tests/scripts/test_convert_product_definition_to_yaml.py @@ -5,13 +5,19 @@ class TestConvertProductDefinitionToYamlTest(unittest.TestCase): - def test_parse_required_for_data_with_multiple_depend(self): row_columns = { - "NAME": ["intensity_by_pitch_angle_and_gyrophase", "energy_label", "pitch_angle_label"], + "NAME": [ + "intensity_by_pitch_angle_and_gyrophase", + "energy_label", + "pitch_angle_label", + ], "DATA_TYPE": ["float32", "str", "str"], - "CATDESC": ["Intensity organized by pitch angle and gyrophase", - "energy label", "pitch angle label"], + "CATDESC": [ + "Intensity organized by pitch angle and gyrophase", + "energy label", + "pitch angle label", + ], "VAR_TYPE": ["data", "metadata", "metadata"], "DEPEND_0": ["epoch", "", ""], "DEPEND_1": ["energy", "", ""], @@ -26,7 +32,7 @@ def test_parse_required_for_data_with_multiple_depend(self): "LABL_PTR_1": ["energy_label", "", ""], "LABL_PTR_2": ["pitch_angle_label", "", ""], "VARIABLE_PURPOSE": ["primary var, summary", "", ""], - "RECORD_VARYING": ["rv", "nrv", "nrv"] + "RECORD_VARYING": ["rv", "nrv", "nrv"], } rows = [ @@ -37,7 +43,7 @@ def test_parse_required_for_data_with_multiple_depend(self): ] filePath = "csv_test_file.csv" - with open(filePath, "w", newline='') as csvfile: + with open(filePath, "w", newline="") as csvfile: test_csv_writer = csv.writer(csvfile) test_csv_writer.writerows(rows) @@ -79,8 +85,22 @@ def test_parse_required_for_data_with_multiple_depend(self): self.assertEqual(expected_yaml, actual_yaml) def test_parse_required_for_data_only_depend_0(self): - required_for_data = ["NAME", "DATA_TYPE", "CATDESC", "VAR_TYPE", "RECORD_VARYING", "DEPEND_0", "DISPLAY_TYPE", - "FIELDNAM", "FORMAT", "LABLAXIS", "UNITS", "VALIDMIN", "VALIDMAX", "FILLVAL"] + required_for_data = [ + "NAME", + "DATA_TYPE", + "CATDESC", + "VAR_TYPE", + "RECORD_VARYING", + "DEPEND_0", + "DISPLAY_TYPE", + "FIELDNAM", + "FORMAT", + "LABLAXIS", + "UNITS", + "VALIDMIN", + "VALIDMAX", + "FILLVAL", + ] test_csv_values = [ "core_fit_num_points", @@ -96,10 +116,10 @@ def test_parse_required_for_data_only_depend_0(self): "", "", "", - "-1.00E+31" + "-1.00E+31", ] filePath = "csv_test_file.csv" - with open(filePath, "w", newline='') as csvfile: + with open(filePath, "w", newline="") as csvfile: test_csv_writer = csv.writer(csvfile) test_csv_writer.writerow(required_for_data) test_csv_writer.writerow(test_csv_values) @@ -121,8 +141,21 @@ def test_parse_required_for_data_only_depend_0(self): self.assertEqual(expected_yaml, actual_yaml) def test_parse_required_for_support_data_non_rv(self): - required_for_data = ["NAME", "DATA_TYPE", "CATDESC", "VAR_TYPE", "RECORD_VARYING", "DISPLAY_TYPE", - "FIELDNAM", "FORMAT", "LABLAXIS", "UNITS", "VALIDMIN", "VALIDMAX", "FILLVAL"] + required_for_data = [ + "NAME", + "DATA_TYPE", + "CATDESC", + "VAR_TYPE", + "RECORD_VARYING", + "DISPLAY_TYPE", + "FIELDNAM", + "FORMAT", + "LABLAXIS", + "UNITS", + "VALIDMIN", + "VALIDMAX", + "FILLVAL", + ] test_csv_values = [ "energy", @@ -137,10 +170,10 @@ def test_parse_required_for_support_data_non_rv(self): "", "", "", - "-1.00E+31" + "-1.00E+31", ] filePath = "csv_test_file.csv" - with open(filePath, "w", newline='') as csvfile: + with open(filePath, "w", newline="") as csvfile: test_csv_writer = csv.writer(csvfile) test_csv_writer.writerow(required_for_data) test_csv_writer.writerow(test_csv_values) @@ -161,9 +194,22 @@ def test_parse_required_for_support_data_non_rv(self): self.assertEqual(expected_yaml, actual_yaml) def test_parse_required_for_data_epoch(self): - required_for_data = ["NAME", "DATA_TYPE", "CATDESC", "VAR_TYPE", "DEPEND_0", "DISPLAY_TYPE", - "FIELDNAM", "FORMAT", "LABLAXIS", "UNITS", "VALIDMIN", "VALIDMAX", "FILLVAL", - "RECORD_VARYING"] + required_for_data = [ + "NAME", + "DATA_TYPE", + "CATDESC", + "VAR_TYPE", + "DEPEND_0", + "DISPLAY_TYPE", + "FIELDNAM", + "FORMAT", + "LABLAXIS", + "UNITS", + "VALIDMIN", + "VALIDMAX", + "FILLVAL", + "RECORD_VARYING", + ] test_csv_values = [ "epoch", @@ -179,10 +225,10 @@ def test_parse_required_for_data_epoch(self): "", "", "-1.00E+31", - "rv" + "rv", ] filePath = "csv_test_file.csv" - with open(filePath, "w", newline='') as csvfile: + with open(filePath, "w", newline="") as csvfile: test_csv_writer = csv.writer(csvfile) test_csv_writer.writerow(required_for_data) test_csv_writer.writerow(test_csv_values) @@ -203,47 +249,48 @@ def test_parse_required_for_data_epoch(self): self.assertEqual(expected_yaml, actual_yaml) def test_parse_does_not_include_columns_if_they_are_blank(self): - all_headers = ["NAME", - "DATA_TYPE", - "Data shape", - "CATDESC", - "VAR_TYPE", - "RECORD_VARYING", - "DEPEND_0", - "DEPEND_1", - "DEPEND_2", - "DEPEND_3", - "DISPLAY_TYPE", - "FIELDNAM", - "FORMAT", - "LABLAXIS", - "UNITS", - "VALIDMIN", - "VALIDMAX", - "FILLVAL", - "LABL_PTRS", - "UNIT_PTR", - "SCALE_TYP", - "SCAL_PTR", - "VAR_NOTES", - "TIME_BASE", - "TIME_SCALE", - "LEAP_SECONDS_INCLUDED", - "ABSOLUTE_ERROR", - "AVG_TYPE", - "BIN_LOCATION", - "DELTA_PLUS_VAR", - "DELTA_MINUS_VAR", - "DERIVN", - "DICT_KEY", - "MONOTON", - "SCALEMIN", - "SCALEMAX", - "REFERENCE_POSITION", - "RELATIVE_ERROR", - "RESOLUTION", - "SI_CONVERSION" - ] + all_headers = [ + "NAME", + "DATA_TYPE", + "Data shape", + "CATDESC", + "VAR_TYPE", + "RECORD_VARYING", + "DEPEND_0", + "DEPEND_1", + "DEPEND_2", + "DEPEND_3", + "DISPLAY_TYPE", + "FIELDNAM", + "FORMAT", + "LABLAXIS", + "UNITS", + "VALIDMIN", + "VALIDMAX", + "FILLVAL", + "LABL_PTRS", + "UNIT_PTR", + "SCALE_TYP", + "SCAL_PTR", + "VAR_NOTES", + "TIME_BASE", + "TIME_SCALE", + "LEAP_SECONDS_INCLUDED", + "ABSOLUTE_ERROR", + "AVG_TYPE", + "BIN_LOCATION", + "DELTA_PLUS_VAR", + "DELTA_MINUS_VAR", + "DERIVN", + "DICT_KEY", + "MONOTON", + "SCALEMIN", + "SCALEMAX", + "REFERENCE_POSITION", + "RELATIVE_ERROR", + "RESOLUTION", + "SI_CONVERSION", + ] row_contents = [ "stim_tag", @@ -285,11 +332,11 @@ def test_parse_does_not_include_columns_if_they_are_blank(self): "", "", "", - "" + "", ] filePath = "csv_test_file.csv" - with open(filePath, "w", newline='') as csvfile: + with open(filePath, "w", newline="") as csvfile: test_csv_writer = csv.writer(csvfile) test_csv_writer.writerow(all_headers) test_csv_writer.writerow(row_contents) diff --git a/tests/swapi/_helpers.py b/tests/swapi/_helpers.py index 1ede531ae..e0e90591e 100644 --- a/tests/swapi/_helpers.py +++ b/tests/swapi/_helpers.py @@ -108,4 +108,5 @@ def synthesize_count_rates(ctx, sw_params: SolarWindParams) -> np.ndarray: ideal, _ = model_solar_wind_ideal_coincidence_rates(sw_params, ctx) return ideal * deadtime_factor(ideal) + NOMINAL_TEST_EPOCH_TT2000 = pycdf.lib.datetime_to_tt2000(datetime(2026, 1, 1)) diff --git a/tests/swapi/cdf_model_test_case.py b/tests/swapi/cdf_model_test_case.py index 207bf8abb..70e349053 100644 --- a/tests/swapi/cdf_model_test_case.py +++ b/tests/swapi/cdf_model_test_case.py @@ -6,12 +6,28 @@ class CdfModelTestCase(unittest.TestCase): - def assert_variable_attributes(self, variable: DataProductVariable, - expected_data, expected_name, - expected_data_type=None, - expected_record_varying=None): + def assert_variable_attributes( + self, + variable: DataProductVariable, + expected_data, + expected_name, + expected_data_type=None, + expected_record_varying=None, + ): self.assertEqual(expected_name, variable.name) - np.testing.assert_array_equal(variable.value, expected_data, f"mismatch for var name: {variable.name}", - strict=True) - self.assertEqual(expected_data_type, variable.cdf_data_type, f"mismatch for var name: {variable.name}") - self.assertEqual(expected_record_varying, variable.record_varying, f"mismatch for var name: {variable.name}") + np.testing.assert_array_equal( + variable.value, + expected_data, + f"mismatch for var name: {variable.name}", + strict=True, + ) + self.assertEqual( + expected_data_type, + variable.cdf_data_type, + f"mismatch for var name: {variable.name}", + ) + self.assertEqual( + expected_record_varying, + variable.record_varying, + f"mismatch for var name: {variable.name}", + ) diff --git a/tests/swapi/l3a/science/pickup_ion/test_build_collapsed_response_grid.py b/tests/swapi/l3a/science/pickup_ion/test_build_collapsed_response_grid.py index 27a967417..3b0fd3dca 100644 --- a/tests/swapi/l3a/science/pickup_ion/test_build_collapsed_response_grid.py +++ b/tests/swapi/l3a/science/pickup_ion/test_build_collapsed_response_grid.py @@ -83,7 +83,9 @@ def test_grid_matches_precomputed_shell_integral_reference(self): trusted = h_truth > 0.05 * peak self.assertGreater(trusted.sum(), 16, "trusted region too small") - rel_err = np.abs(collapsed.values[trusted] - h_truth[trusted]) / h_truth[trusted] + rel_err = ( + np.abs(collapsed.values[trusted] - h_truth[trusted]) / h_truth[trusted] + ) worst = int(np.argmax(rel_err)) worst_index = int(np.flatnonzero(trusted)[worst]) self.assertLess( @@ -136,10 +138,11 @@ def test_matches_shell_integral_when_grid_cell_aligned_with_singularity(self): cos_elevation = np.cos(np.deg2rad(elevation_deg)) cos_delta_azimuth_target = 1.0 - 5e-7 cos_angle_target = cos_delta_azimuth_target * cos_elevation - v_singular_kms = float(np.sqrt( - speed ** 2 + bulk_speed ** 2 - - 2 * cos_angle_target * speed * bulk_speed - )) + v_singular_kms = float( + np.sqrt( + speed**2 + bulk_speed**2 - 2 * cos_angle_target * speed * bulk_speed + ) + ) collapsed = build_collapsed_response_grid( response_grid, diff --git a/tests/swapi/l3a/science/pickup_ion/test_calculate_coincidence_rate.py b/tests/swapi/l3a/science/pickup_ion/test_calculate_coincidence_rate.py index 162385312..fd4a59292 100644 --- a/tests/swapi/l3a/science/pickup_ion/test_calculate_coincidence_rate.py +++ b/tests/swapi/l3a/science/pickup_ion/test_calculate_coincidence_rate.py @@ -11,6 +11,7 @@ bulk-aligned geometry near its 1/sin(delta_azimuth) Jacobian singularity, and at very low rates (covered by `atol`). """ + import unittest import numpy as np @@ -59,14 +60,14 @@ def test_matches_xarray_reference_per_voltage(self): # (−cos θ sin φ, −cos θ cos φ, −sin θ). sw_az_rad = np.radians(_SW_AZIMUTH_DEG) sw_el_rad = np.radians(_SW_ELEVATION_DEG) - bulk_vec = _SW_SPEED_KMS * np.array([ - -np.cos(sw_el_rad) * np.sin(sw_az_rad), - -np.cos(sw_el_rad) * np.cos(sw_az_rad), - -np.sin(sw_el_rad), - ]) - bulk_sw_per_bin = np.broadcast_to( - bulk_vec, (1, voltage_v.size, 3) - ).copy() + bulk_vec = _SW_SPEED_KMS * np.array( + [ + -np.cos(sw_el_rad) * np.sin(sw_az_rad), + -np.cos(sw_el_rad) * np.cos(sw_az_rad), + -np.sin(sw_el_rad), + ] + ) + bulk_sw_per_bin = np.broadcast_to(bulk_vec, (1, voltage_v.size, 3)).copy() swapi_response = load_swapi_response( warm_cache_voltages=voltage_v.astype(float) diff --git a/tests/swapi/l3a/science/pickup_ion/test_density_of_neutral_helium_lookup_table.py b/tests/swapi/l3a/science/pickup_ion/test_density_of_neutral_helium_lookup_table.py index 3e108c91b..60a367900 100644 --- a/tests/swapi/l3a/science/pickup_ion/test_density_of_neutral_helium_lookup_table.py +++ b/tests/swapi/l3a/science/pickup_ion/test_density_of_neutral_helium_lookup_table.py @@ -2,8 +2,9 @@ import numpy as np -from imap_l3_processing.swapi.l3a.science.pickup_ion.density_of_neutral_helium_lookup_table import \ - DensityOfNeutralHeliumLookupTable +from imap_l3_processing.swapi.l3a.science.pickup_ion.density_of_neutral_helium_lookup_table import ( + DensityOfNeutralHeliumLookupTable, +) def _table_with_360_anchor(): diff --git a/tests/swapi/l3a/science/pickup_ion/test_goodness_of_fit.py b/tests/swapi/l3a/science/pickup_ion/test_goodness_of_fit.py index 5deadf7db..81cdd1754 100644 --- a/tests/swapi/l3a/science/pickup_ion/test_goodness_of_fit.py +++ b/tests/swapi/l3a/science/pickup_ion/test_goodness_of_fit.py @@ -32,7 +32,7 @@ def _make_pui_model( # custom-shaped cutoff, with only one point below CUTOFF_DROP_RATIO = 0.4 rolloff_start = int(np.argmax(~below_cutoff)) - shape[rolloff_start:rolloff_start + 3] = np.array([0.9, 0.5, 0.2]) + shape[rolloff_start : rolloff_start + 3] = np.array([0.9, 0.5, 0.2]) # scale by peak rate model_rates = rate_at_cutoff * shape @@ -58,12 +58,19 @@ def test_fit_acceptance_criteria(self): chunk_mean_model_rates = model_rates.mean(axis=0) peak_model_rate = chunk_mean_model_rates.max() - cutoff_mask_energy = esa_energies[chunk_mean_model_rates >= peak_model_rate * CUTOFF_DROP_RATIO].max() + cutoff_mask_energy = esa_energies[ + chunk_mean_model_rates >= peak_model_rate * CUTOFF_DROP_RATIO + ].max() past_cutoff_mask = esa_energies > cutoff_mask_energy - assert past_cutoff_mask.sum() == 4 # 4 points by construction: three zero, one nonzero + assert ( + past_cutoff_mask.sum() == 4 + ) # 4 points by construction: three zero, one nonzero def extra_counts_past_cutoff(fraction: float) -> dict: - return dict(observed_rates=model_rates + past_cutoff_mask * peak_model_rate * fraction) + return dict( + observed_rates=model_rates + + past_cutoff_mask * peak_model_rate * fraction + ) def poisson_sampled_observed_rates() -> dict: livetime = 0.145 @@ -73,7 +80,10 @@ def poisson_sampled_observed_rates() -> dict: cases = { "all good": (True, {}), - "cutoff just below maximum": (True, dict(cutoff_speed_kms=max_cutoff - 1.0)), + "cutoff just below maximum": ( + True, + dict(cutoff_speed_kms=max_cutoff - 1.0), + ), "cutoff at maximum": (True, dict(cutoff_speed_kms=max_cutoff)), "cutoff above maximum": (False, dict(cutoff_speed_kms=max_cutoff + 1.0)), "model underpredicts a little": (True, dict(model_rates=model_rates * 0.9)), @@ -84,7 +94,9 @@ def poisson_sampled_observed_rates() -> dict: } for case_name, (expected, overrides) in cases.items(): with self.subTest(case_name): - self.assertEqual(expected, is_good_fit(**{**defaults, **overrides}), msg=case_name) + self.assertEqual( + expected, is_good_fit(**{**defaults, **overrides}), msg=case_name + ) def test_fitted_parameter_range_criteria(self): """A fit whose cutoff speed leaves the allowed multiple of the bulk @@ -108,30 +120,56 @@ def test_fitted_parameter_range_criteria(self): cases = { "all good": (True, {}), "cutoff speed below the lowest allowed multiple": ( - False, dict(cutoff_speed_kms=min_cutoff_speed - 1.0)), + False, + dict(cutoff_speed_kms=min_cutoff_speed - 1.0), + ), "cutoff speed at the lowest allowed multiple of the bulk speed": ( - True, dict(cutoff_speed_kms=min_cutoff_speed)), + True, + dict(cutoff_speed_kms=min_cutoff_speed), + ), "cutoff speed above the lowest allowed multiple": ( - True, dict(cutoff_speed_kms=min_cutoff_speed + 1.0)), + True, + dict(cutoff_speed_kms=min_cutoff_speed + 1.0), + ), "cutoff speed below the highest allowed multiple": ( - True, dict(cutoff_speed_kms=max_cutoff_speed - 1.0)), + True, + dict(cutoff_speed_kms=max_cutoff_speed - 1.0), + ), "cutoff speed at the highest allowed multiple of the bulk speed": ( - True, dict(cutoff_speed_kms=max_cutoff_speed)), + True, + dict(cutoff_speed_kms=max_cutoff_speed), + ), "cutoff speed above the highest allowed multiple": ( - False, dict(cutoff_speed_kms=max_cutoff_speed + 1.0)), + False, + dict(cutoff_speed_kms=max_cutoff_speed + 1.0), + ), "ionization rate below the lower limit": ( - False, dict(ionization_rate=MIN_IONIZATION_RATE * 0.99)), + False, + dict(ionization_rate=MIN_IONIZATION_RATE * 0.99), + ), "ionization rate at the lower limit": ( - True, dict(ionization_rate=MIN_IONIZATION_RATE)), + True, + dict(ionization_rate=MIN_IONIZATION_RATE), + ), "ionization rate above the lower limit": ( - True, dict(ionization_rate=MIN_IONIZATION_RATE * 1.01)), + True, + dict(ionization_rate=MIN_IONIZATION_RATE * 1.01), + ), "ionization rate below the upper limit": ( - True, dict(ionization_rate=MAX_IONIZATION_RATE * 0.99)), + True, + dict(ionization_rate=MAX_IONIZATION_RATE * 0.99), + ), "ionization rate at the upper limit": ( - True, dict(ionization_rate=MAX_IONIZATION_RATE)), + True, + dict(ionization_rate=MAX_IONIZATION_RATE), + ), "ionization rate above the upper limit": ( - False, dict(ionization_rate=MAX_IONIZATION_RATE * 1.01)), + False, + dict(ionization_rate=MAX_IONIZATION_RATE * 1.01), + ), } for case_name, (expected, overrides) in cases.items(): with self.subTest(case_name): - self.assertEqual(expected, is_good_fit(**{**defaults, **overrides}), msg=case_name) + self.assertEqual( + expected, is_good_fit(**{**defaults, **overrides}), msg=case_name + ) diff --git a/tests/swapi/l3a/science/pickup_ion/test_utils.py b/tests/swapi/l3a/science/pickup_ion/test_utils.py index e45960a8b..1ce8b3a62 100644 --- a/tests/swapi/l3a/science/pickup_ion/test_utils.py +++ b/tests/swapi/l3a/science/pickup_ion/test_utils.py @@ -15,9 +15,30 @@ [-0.803319036, -0.595067395, -0.023944118, 0.0, 0.0, 0.0], [0.594803234, -0.803675802, 0.017728995, 0.0, 0.0, 0.0], [-0.029793255, 0.0, 0.999556082, 0.0, 0.0, 0.0], - [-1.16314295e-06, 1.56750981e-06, 6.68593934e-08, -0.803319036, -0.595067395, -0.023944118], - [-1.56457525e-06, -1.16063465e-06, -1.21809529e-07, 0.594803234, -0.803675802, 0.017728995], - [1.26218156e-07, 5.29395592e-23, 3.76211978e-09, -0.029793255, 0.0, 0.999556082], + [ + -1.16314295e-06, + 1.56750981e-06, + 6.68593934e-08, + -0.803319036, + -0.595067395, + -0.023944118, + ], + [ + -1.56457525e-06, + -1.16063465e-06, + -1.21809529e-07, + 0.594803234, + -0.803675802, + 0.017728995, + ], + [ + 1.26218156e-07, + 5.29395592e-23, + 3.76211978e-09, + -0.029793255, + 0.0, + 0.999556082, + ], ] ) @@ -42,7 +63,9 @@ def test_adds_imap_inertial_velocity_to_rotated_input(self, mock_spice): ] ) np.testing.assert_array_almost_equal(output_velocity, expected) - mock_spice.sxform.assert_called_with("INPUT_FRAME", "OUTPUT_FRAME", ephemeris_time) + mock_spice.sxform.assert_called_with( + "INPUT_FRAME", "OUTPUT_FRAME", ephemeris_time + ) mock_spice.spkezr.assert_called_with( "IMAP", ephemeris_time, "OUTPUT_FRAME", "NONE", "SUN" ) diff --git a/tests/swapi/l3a/science/pickup_ion/test_vasyliunas_siscoe_distribution.py b/tests/swapi/l3a/science/pickup_ion/test_vasyliunas_siscoe_distribution.py index f17bc0837..c8fd14e25 100644 --- a/tests/swapi/l3a/science/pickup_ion/test_vasyliunas_siscoe_distribution.py +++ b/tests/swapi/l3a/science/pickup_ion/test_vasyliunas_siscoe_distribution.py @@ -13,11 +13,14 @@ from imap_l3_processing.swapi.l3a.science.pickup_ion.uniform_speed_grid import ( UniformSpeedGrid, ) -from imap_l3_processing.swapi.l3a.science.pickup_ion.vasyliunas_siscoe_distribution import \ - vasyliunas_siscoe_vdf +from imap_l3_processing.swapi.l3a.science.pickup_ion.vasyliunas_siscoe_distribution import ( + vasyliunas_siscoe_vdf, +) from tests.test_helpers import NumpyArrayMatcher -_VASYLIUNAS_SISCOE_MODULE = "imap_l3_processing.swapi.l3a.science.pickup_ion.vasyliunas_siscoe_distribution" +_VASYLIUNAS_SISCOE_MODULE = ( + "imap_l3_processing.swapi.l3a.science.pickup_ion.vasyliunas_siscoe_distribution" +) _IONIZATION_RATE_HZ = 0.47 _SOLAR_WIND_SPEED_INERTIAL_KMS = 456.0 @@ -56,7 +59,8 @@ def _expected_uncut_filled_shell(cutoff_speed: float) -> np.ndarray: # closed form of the filled shell, with the mocked unit neutral # helium density converted from cm^-3 to km^-3. return ( - SWAPI_PUI_COOLING_INDEX / (4 * np.pi) + SWAPI_PUI_COOLING_INDEX + / (4 * np.pi) * (_IONIZATION_RATE_HZ * ONE_AU_IN_KM**2) / (_DISTANCE_KM * _SOLAR_WIND_SPEED_INERTIAL_KMS * cutoff_speed**3) * (_SPEED_GRID.centers / cutoff_speed) ** (SWAPI_PUI_COOLING_INDEX - 3) @@ -79,7 +83,7 @@ def test_scales_the_cutoff_cell_by_the_fraction_below_it_and_zeros_the_cells_abo expected = self._expected_uncut_filled_shell(cutoff_speed) expected[cutoff_index] *= fraction_below_cutoff - expected[cutoff_index + 1:] = 0.0 + expected[cutoff_index + 1 :] = 0.0 np.testing.assert_allclose(result, expected, rtol=1e-12) mock_density.assert_called_with( diff --git a/tests/swapi/l3a/science/solar_wind/alpha/test_fit_solar_wind_alpha_model.py b/tests/swapi/l3a/science/solar_wind/alpha/test_fit_solar_wind_alpha_model.py index a8ff4bf46..48b09e13e 100644 --- a/tests/swapi/l3a/science/solar_wind/alpha/test_fit_solar_wind_alpha_model.py +++ b/tests/swapi/l3a/science/solar_wind/alpha/test_fit_solar_wind_alpha_model.py @@ -49,10 +49,10 @@ # physical voltages SWAPI sweeps in flight. _N_BINS_PER_SWEEP = 62 _N_SWEEPS = 5 -_ONE_SWEEP_VOLTAGE = np.logspace( - np.log10(3500.0), np.log10(140.0), _N_BINS_PER_SWEEP -) -_FIVE_SWEEP_VOLTAGE = np.broadcast_to(_ONE_SWEEP_VOLTAGE, (_N_SWEEPS, _N_BINS_PER_SWEEP)).copy() +_ONE_SWEEP_VOLTAGE = np.logspace(np.log10(3500.0), np.log10(140.0), _N_BINS_PER_SWEEP) +_FIVE_SWEEP_VOLTAGE = np.broadcast_to( + _ONE_SWEEP_VOLTAGE, (_N_SWEEPS, _N_BINS_PER_SWEEP) +).copy() _N_MEAS = _FIVE_SWEEP_VOLTAGE.size # Slow-wind ground-truth moments. RTN +R points sunward, so a sunward solar @@ -230,7 +230,9 @@ def _synthesize_proton_plus_alpha_count_rate( species=Species.ALPHA, rotation_matrices=rotation_matrices, ) - alpha_true, _ = model_solar_wind_ideal_coincidence_rates(alpha_params, alpha_ctx) + alpha_true, _ = model_solar_wind_ideal_coincidence_rates( + alpha_params, alpha_ctx + ) total_true = proton_true + alpha_true else: total_true = proton_true @@ -384,8 +386,7 @@ def test_alpha_velocity_equals_proton_velocity_plus_delta_v_along_bhat(self): """The post-fit `velocity_rtn` satisfies the algebraic identity v_α = v_p* + Δv·B̂ exactly (not approximately) — the dataclass stores the constraint, not a free vector.""" v_alpha = self.result.velocity_rtn_nominal() expected = ( - _TRUE_PROTON_VELOCITY_RTN - + self.result.delta_v.nominal_value * _B_HAT_RTN + _TRUE_PROTON_VELOCITY_RTN + self.result.delta_v.nominal_value * _B_HAT_RTN ) np.testing.assert_allclose(v_alpha, expected, atol=1e-9) @@ -398,9 +399,7 @@ def test_alpha_velocity_recovers_truth(self): ) -class TestAlphaPeakEnergyRatioGuard( - _SyntheticAlphaSpectrumFixture, unittest.TestCase -): +class TestAlphaPeakEnergyRatioGuard(_SyntheticAlphaSpectrumFixture, unittest.TestCase): """Only fitted alpha speeds below the tolerable ratio to the proton speed are rejected.""" def test_alpha_speed_ratio_lower_bound(self): @@ -470,9 +469,7 @@ def setUpClass(cls): def test_alpha_velocity_minus_proton_velocity_is_parallel_to_bhat(self): """For a tilted B̂, the recovered (v_α − v_p) lies along ±B̂ — equivalently, the cross product (v_α − v_p) × B̂ is zero up to numerical noise.""" - delta = ( - self.result.velocity_rtn_nominal() - self.proton_velocity_rtn - ) + delta = self.result.velocity_rtn_nominal() - self.proton_velocity_rtn np.testing.assert_allclose(np.cross(delta, self.b_hat), 0.0, atol=1e-9) def test_recovered_delta_v_matches_dot_product_of_velocity_offset(self): @@ -580,18 +577,22 @@ def test_fit_error_flag_set_when_least_squares_does_not_converge(self): count_rate=np.full(_FIVE_SWEEP_VOLTAGE.shape, 100.0), ) - with patch.object( - alpha_module, - "calculate_initial_guess", - return_value=(0.2, 4.0e5, 0.0, np.array([10, 11, 12])), - ), patch.object( - alpha_module._AlphaEvaluator, - "residuals", - return_value=np.full(_N_MEAS, 1.0), - ), patch.object( - alpha_module.scipy.optimize, - "least_squares", - return_value=non_converged, + with ( + patch.object( + alpha_module, + "calculate_initial_guess", + return_value=(0.2, 4.0e5, 0.0, np.array([10, 11, 12])), + ), + patch.object( + alpha_module._AlphaEvaluator, + "residuals", + return_value=np.full(_N_MEAS, 1.0), + ), + patch.object( + alpha_module.scipy.optimize, + "least_squares", + return_value=non_converged, + ), ): result = fit_solar_wind_alpha_model( proton_ctx=proton_ctx, @@ -755,13 +756,16 @@ def setUpClass(cls): def test_jacobian_shape_is_n_residuals_by_three(self): """The analytic Jacobian has shape (N_residuals, 3) — one row per measurement bin, one column per fit parameter (log n_α, log T_α, Δv).""" - self.assertEqual(self.analytic_jacobian.shape, (self.alpha_ctx.count_rate.size, 3)) + self.assertEqual( + self.analytic_jacobian.shape, (self.alpha_ctx.count_rate.size, 3) + ) def test_log_density_column_equals_alpha_only_rate_times_deadtime_squared(self): """Because the rate is linear in n_α, the analytic ∂(observable)/∂(log n_α) column equals 𝒟²(R_total) · R_α exactly with no quadrature slack — an identity check independent of finite-difference noise.""" from imap_l3_processing.swapi.l3a.science.solar_wind.forward_model import ( model_solar_wind_ideal_coincidence_rates, ) + alpha_only_params = SolarWindParams( density=_TRUE_ALPHA_DENSITY_CM3, velocity_rtn=_TRUE_PROTON_VELOCITY_RTN + _TRUE_DELTA_V_KM_S * _B_HAT_RTN, @@ -891,24 +895,28 @@ def test_least_squares_receives_jac_callable_and_no_diff_step(self): rotation_matrices=self.rotation_matrices, ) - with patch.object( - alpha_module, - "calculate_initial_guess", - return_value=( - _TRUE_ALPHA_DENSITY_CM3, - _TRUE_ALPHA_TEMPERATURE_K, - 0.0, - peak_bin_idx, + with ( + patch.object( + alpha_module, + "calculate_initial_guess", + return_value=( + _TRUE_ALPHA_DENSITY_CM3, + _TRUE_ALPHA_TEMPERATURE_K, + 0.0, + peak_bin_idx, + ), ), - ), patch.object( - alpha_module._AlphaEvaluator, - "residuals", - return_value=np.zeros(n_peak_residuals), - ), patch.object( - alpha_module.scipy.optimize, - "least_squares", - return_value=mock_result, - ) as mock_lm: + patch.object( + alpha_module._AlphaEvaluator, + "residuals", + return_value=np.zeros(n_peak_residuals), + ), + patch.object( + alpha_module.scipy.optimize, + "least_squares", + return_value=mock_result, + ) as mock_lm, + ): fit_solar_wind_alpha_model( proton_ctx=proton_ctx, alpha_ctx=alpha_ctx, diff --git a/tests/swapi/l3a/science/solar_wind/proton/test_calculate_initial_guess.py b/tests/swapi/l3a/science/solar_wind/proton/test_calculate_initial_guess.py index 99ee495ed..a9d1890a1 100644 --- a/tests/swapi/l3a/science/solar_wind/proton/test_calculate_initial_guess.py +++ b/tests/swapi/l3a/science/solar_wind/proton/test_calculate_initial_guess.py @@ -30,9 +30,7 @@ # RTN → SWAPI rotation. Body +Y (the SWAPI boresight / spin axis) in RTN is # column 1 of the transpose, i.e. -R̂_RTN. The solar wind direction (anti- # parallel to the spin axis) is therefore +R̂. -_R_BASE_RTN_TO_SWAPI = np.array( - [[0.0, 1.0, 0.0], [-1.0, 0.0, 0.0], [0.0, 0.0, 1.0]] -) +_R_BASE_RTN_TO_SWAPI = np.array([[0.0, 1.0, 0.0], [-1.0, 0.0, 0.0], [0.0, 0.0, 1.0]]) def _esa_voltage_for_proton_speed(speed_km_s: float) -> float: @@ -71,17 +69,14 @@ def _build_proton_ctx(count_rate: np.ndarray, esa_voltage: np.ndarray): return ctx -def _make_synthetic_ctx_at_known_truth(truth: SolarWindParams, - n_bins: int = 71): +def _make_synthetic_ctx_at_known_truth(truth: SolarWindParams, n_bins: int = 71): """Build a context whose `count_rate` array is the noiseless ideal forward model evaluated at `truth`.""" bulk_speed = float(np.linalg.norm(truth.velocity_rtn)) # Wide enough to bracket ±5σ at T=1e5 K, narrow enough to keep all bins # on-instrument. speed_grid = np.linspace(0.4 * bulk_speed, 1.6 * bulk_speed, n_bins) - voltages = np.array( - [_esa_voltage_for_proton_speed(s) for s in speed_grid] - ) + voltages = np.array([_esa_voltage_for_proton_speed(s) for s in speed_grid]) # Build a placeholder ctx, evaluate the forward model at `truth` to get # ideal rates, then build the production ctx with those rates as @@ -98,11 +93,8 @@ def _ctx_with_peak_at(self, peak_speed_kms: float): """Build a context whose count-rate spectrum has its maximum at a bin whose ESA voltage corresponds to `peak_speed_kms`. The shape of the spectrum doesn't matter — we patch out the refiner.""" - speed_grid = np.linspace(0.4 * peak_speed_kms, 1.6 * peak_speed_kms, - 31) - voltages = np.array( - [_esa_voltage_for_proton_speed(s) for s in speed_grid] - ) + speed_grid = np.linspace(0.4 * peak_speed_kms, 1.6 * peak_speed_kms, 31) + voltages = np.array([_esa_voltage_for_proton_speed(s) for s in speed_grid]) peak_idx = len(speed_grid) // 2 # middle bin voltages[peak_idx] = _esa_voltage_for_proton_speed(peak_speed_kms) count_rate = np.linspace(0.1, 0.5, len(voltages)) @@ -138,8 +130,9 @@ def test_temperature_seed_uses_documented_speed_squared_formula(self): temperature_seed_arg = patched_refine.call_args.args[3] expected_temperature = 60_000.0 * (peak_speed / 400.0) ** 2 - np.testing.assert_allclose(temperature_seed_arg, expected_temperature, - rtol=1e-12) + np.testing.assert_allclose( + temperature_seed_arg, expected_temperature, rtol=1e-12 + ) def test_temperature_seed_floors_at_one_ev(self): """At a low peak speed where 60_000·(v/400)² is below 1 eV, the temperature seed handed to the refiner is clamped to `INITIAL_TEMPERATURE_FLOOR_K`.""" @@ -162,9 +155,7 @@ class TestCalculateInitialGuessDirection(unittest.TestCase): def test_initial_velocity_is_anti_parallel_to_spin_axis(self): """With every rotation matrix aligning body +Y to -R̂, the returned bulk velocity points along +R̂ — the negation of the chunk-mean spin axis.""" speed_grid = np.linspace(300.0, 600.0, 31) - voltages = np.array( - [_esa_voltage_for_proton_speed(s) for s in speed_grid] - ) + voltages = np.array([_esa_voltage_for_proton_speed(s) for s in speed_grid]) count_rate = np.linspace(0.1, 0.5, len(voltages)) peak_idx = len(speed_grid) // 2 # middle bin count_rate[peak_idx] = 100.0 @@ -175,9 +166,7 @@ def test_initial_velocity_is_anti_parallel_to_spin_axis(self): expected_axis = np.array([-1.0, 0.0, 0.0]) guess = calculate_initial_guess(ctx) - v_unit = guess.velocity_rtn / np.linalg.norm( - guess.velocity_rtn - ) + v_unit = guess.velocity_rtn / np.linalg.norm(guess.velocity_rtn) np.testing.assert_allclose(v_unit, -expected_axis, atol=1e-12) def test_velocity_magnitude_matches_truth_bulk_speed(self): @@ -238,9 +227,7 @@ def test_wraps_runtime_error_on_pathological_edge_spike_spectrum(self): """A count-rate spectrum with an extreme isolated spike at the lowest-speed bin (1e10 surrounded by 1e-6) drives `curve_fit` past its `maxfev` budget; the refiner re-raises the resulting `RuntimeError` with a wrapped message that names the peak-bin seed, and the scipy error is preserved on `__cause__`.""" peak_speed = 250.0 speed_grid = np.linspace(peak_speed, peak_speed + 600.0, 71) - voltages = np.array( - [_esa_voltage_for_proton_speed(s) for s in speed_grid] - ) + voltages = np.array([_esa_voltage_for_proton_speed(s) for s in speed_grid]) count_rate = np.full(len(voltages), 1e-6) count_rate[0] = 1.0e10 ctx = _build_proton_ctx(count_rate, voltages) diff --git a/tests/swapi/l3a/science/solar_wind/proton/test_escape_local_minimum.py b/tests/swapi/l3a/science/solar_wind/proton/test_escape_local_minimum.py index f1f2ed82e..3fa41fc28 100644 --- a/tests/swapi/l3a/science/solar_wind/proton/test_escape_local_minimum.py +++ b/tests/swapi/l3a/science/solar_wind/proton/test_escape_local_minimum.py @@ -78,13 +78,17 @@ def test_returns_lm1_unchanged_whenflipped_seed_far_worse(self): flipped_mse_far_above_gate = lm1.mse * self._GATE_FACTOR * 10.0 flipped_params = _proton_params(velocity_rtn=(450.0, 0.0, 0.0)) - with patch.object( - escape_local_minimum_module, - "flipped_seed", - return_value=(flipped_mse_far_above_gate, flipped_params), - ) as mock_flipped, patch.object( - escape_local_minimum_module, "_restart_from_flipped_params", - ) as mock_opt: + with ( + patch.object( + escape_local_minimum_module, + "flipped_seed", + return_value=(flipped_mse_far_above_gate, flipped_params), + ) as mock_flipped, + patch.object( + escape_local_minimum_module, + "_restart_from_flipped_params", + ) as mock_opt, + ): out = escape_local_minimum(lm1, ctx) self.assertIs(out, lm1) @@ -99,13 +103,16 @@ def test_returns_lm1_unchanged_whenflipped_seed_exactly_at_gate(self): lm1 = _result(_proton_params(), mse=1.0) flipped_mse_at_gate = lm1.mse * self._GATE_FACTOR - with patch.object( - escape_local_minimum_module, - "flipped_seed", - return_value=(flipped_mse_at_gate, _proton_params()), - ), patch.object( - escape_local_minimum_module, "_restart_from_flipped_params" - ) as mock_opt: + with ( + patch.object( + escape_local_minimum_module, + "flipped_seed", + return_value=(flipped_mse_at_gate, _proton_params()), + ), + patch.object( + escape_local_minimum_module, "_restart_from_flipped_params" + ) as mock_opt, + ): out = escape_local_minimum(lm1, ctx) self.assertIs(out, lm1) @@ -126,18 +133,21 @@ def test_returns_lm2_when_lm2_mse_is_lower(self): flipped_seed_mse_iter1 = lm1.mse * (self._GATE_FACTOR / 2.0) flipped_seed_mse_iter2 = better_lm2.mse * (self._GATE_FACTOR * 2.0) - with patch.object( - escape_local_minimum_module, - "flipped_seed", - side_effect=[ - (flipped_seed_mse_iter1, better_params), - (flipped_seed_mse_iter2, better_params), - ], - ) as mock_flipped, patch.object( - escape_local_minimum_module, - "_restart_from_flipped_params", - return_value=better_lm2, - ) as mock_opt: + with ( + patch.object( + escape_local_minimum_module, + "flipped_seed", + side_effect=[ + (flipped_seed_mse_iter1, better_params), + (flipped_seed_mse_iter2, better_params), + ], + ) as mock_flipped, + patch.object( + escape_local_minimum_module, + "_restart_from_flipped_params", + return_value=better_lm2, + ) as mock_opt, + ): out = escape_local_minimum(lm1, ctx) self.assertIs(out, better_lm2) @@ -157,15 +167,18 @@ def test_returns_lm1_when_lm2_mse_is_higher(self): # Flipped seed clears the cheap gate (within threshold²×current). flipped_seed_mse = lm1.mse * (self._GATE_FACTOR / 2.0) - with patch.object( - escape_local_minimum_module, - "flipped_seed", - return_value=(flipped_seed_mse, worse_params), - ), patch.object( - escape_local_minimum_module, - "_restart_from_flipped_params", - return_value=worse_lm2, - ) as mock_opt: + with ( + patch.object( + escape_local_minimum_module, + "flipped_seed", + return_value=(flipped_seed_mse, worse_params), + ), + patch.object( + escape_local_minimum_module, + "_restart_from_flipped_params", + return_value=worse_lm2, + ) as mock_opt, + ): out = escape_local_minimum(lm1, ctx) self.assertIs(out, lm1) @@ -189,19 +202,22 @@ def test_runs_at_most_max_basin_refine_iters_lm2_calls(self): ] lm1 = results[0] gate_factor = self._GATE_FACTOR - with patch.object( - escape_local_minimum_module, - "flipped_seed", - # Always within gate (factor / 2 < gate_factor), always with a fresh seed. - side_effect=[ - (results[i].mse * (gate_factor / 2.0), results[i].sw_params) - for i in range(chain_length - 1) - ], - ), patch.object( - escape_local_minimum_module, - "_restart_from_flipped_params", - side_effect=results[1:], - ) as mock_opt: + with ( + patch.object( + escape_local_minimum_module, + "flipped_seed", + # Always within gate (factor / 2 < gate_factor), always with a fresh seed. + side_effect=[ + (results[i].mse * (gate_factor / 2.0), results[i].sw_params) + for i in range(chain_length - 1) + ], + ), + patch.object( + escape_local_minimum_module, + "_restart_from_flipped_params", + side_effect=results[1:], + ) as mock_opt, + ): out = escape_local_minimum(lm1, ctx) # Pin the cap from the source constant — never hardcode the number. diff --git a/tests/swapi/l3a/science/solar_wind/proton/test_fit_solar_wind_proton_model.py b/tests/swapi/l3a/science/solar_wind/proton/test_fit_solar_wind_proton_model.py index a294a40ba..f11e6f1b3 100644 --- a/tests/swapi/l3a/science/solar_wind/proton/test_fit_solar_wind_proton_model.py +++ b/tests/swapi/l3a/science/solar_wind/proton/test_fit_solar_wind_proton_model.py @@ -21,21 +21,79 @@ ) from imap_l3_processing.swapi.quality_flags import SwapiL3Flags from imap_l3_processing.swapi.species import Species -from tests.swapi._helpers import NOMINAL_TEST_EPOCH_TT2000, load_swapi_response, synthesize_count_rates +from tests.swapi._helpers import ( + NOMINAL_TEST_EPOCH_TT2000, + load_swapi_response, + synthesize_count_rates, +) # Mean SWAPI L2 coarse-sweep voltages (V), descending — a 62-bin sweep that # covers the proton speed range densely. Identical to the set used by # `docs/swapi/figure_src/plot_fit_accuracy.py`. _VOLTAGES_PER_SWEEP = np.array( [ - 9895.52, 9088.69, 8348.80, 7667.55, 7042.16, 6469.31, 5941.77, 5457.31, - 5013.22, 4603.65, 4230.77, 3886.92, 3569.16, 3278.72, 3011.13, 2766.25, - 2539.54, 2333.83, 2144.24, 1969.31, 1808.74, 1660.86, 1525.75, 1401.82, - 1287.58, 1182.24, 1085.15, 995.55, 914.31, 839.94, 771.70, 709.46, - 651.59, 598.47, 549.91, 505.12, 463.89, 425.92, 391.18, 359.35, 329.94, - 303.02, 278.25, 255.55, 234.77, 215.61, 197.95, 181.82, 167.04, 153.46, - 140.91, 129.50, 118.91, 109.20, 100.30, 92.11, 84.61, 77.73, 71.40, - 65.59, 60.23, 55.34, + 9895.52, + 9088.69, + 8348.80, + 7667.55, + 7042.16, + 6469.31, + 5941.77, + 5457.31, + 5013.22, + 4603.65, + 4230.77, + 3886.92, + 3569.16, + 3278.72, + 3011.13, + 2766.25, + 2539.54, + 2333.83, + 2144.24, + 1969.31, + 1808.74, + 1660.86, + 1525.75, + 1401.82, + 1287.58, + 1182.24, + 1085.15, + 995.55, + 914.31, + 839.94, + 771.70, + 709.46, + 651.59, + 598.47, + 549.91, + 505.12, + 463.89, + 425.92, + 391.18, + 359.35, + 329.94, + 303.02, + 278.25, + 255.55, + 234.77, + 215.61, + 197.95, + 181.82, + 167.04, + 153.46, + 140.91, + 129.50, + 118.91, + 109.20, + 100.30, + 92.11, + 84.61, + 77.73, + 71.40, + 65.59, + 60.23, + 55.34, ] ) _N_BINS_PER_SWEEP = len(_VOLTAGES_PER_SWEEP) @@ -66,9 +124,9 @@ # The +Y column of `_ANCHOR_ROTATION_MATRIX` is the SWAPI spin axis expressed # in RTN. For a proper rotation matrix it must be unit-norm; assert that here # so the bulk-velocity construction below can use it directly. -assert np.isclose( - np.linalg.norm(_ANCHOR_ROTATION_MATRIX[:, 1]), 1.0, atol=1e-3 -), "expected +Y column of anchor rotation to be unit-norm" +assert np.isclose(np.linalg.norm(_ANCHOR_ROTATION_MATRIX[:, 1]), 1.0, atol=1e-3), ( + "expected +Y column of anchor rotation to be unit-norm" +) # Ground-truth solar-wind parameters used for the parameter-recovery test. # Moderate-speed slow-stream proton population — well inside the SWAPI energy @@ -83,23 +141,16 @@ _TRUE_DENSITY_CM3 = 5.0 _TRUE_TEMPERATURE_K = 1.0e5 _TRUE_BULK_SPEED_KM_S = 450.0 -_TRUE_VELOCITY_RTN_KM_S = ( - -_TRUE_BULK_SPEED_KM_S * _ANCHOR_ROTATION_MATRIX[:, 1] -) +_TRUE_VELOCITY_RTN_KM_S = -_TRUE_BULK_SPEED_KM_S * _ANCHOR_ROTATION_MATRIX[:, 1] def _per_bin_rotation_matrices() -> np.ndarray: """Synthesize plausible per-bin SWAPI→RTN matrices for `_N_SWEEPS` sweeps; details don't affect what's being tested.""" - sweep_index = np.repeat( - np.arange(_N_SWEEPS), _N_BINS_PER_SWEEP - ) - bin_index_in_sweep = np.tile( - np.arange(1, _N_BINS_PER_SWEEP + 1), _N_SWEEPS - ) + sweep_index = np.repeat(np.arange(_N_SWEEPS), _N_BINS_PER_SWEEP) + bin_index_in_sweep = np.tile(np.arange(1, _N_BINS_PER_SWEEP + 1), _N_SWEEPS) sample_times_s = ( - sweep_index * _SWEEP_DURATION_S - + bin_index_in_sweep * _SAMPLE_TIME_PER_BIN_S + sweep_index * _SWEEP_DURATION_S + bin_index_in_sweep * _SAMPLE_TIME_PER_BIN_S ) spin_axis = _ANCHOR_ROTATION_MATRIX[:, 1] / np.linalg.norm( @@ -131,9 +182,7 @@ def _build_synthetic_fit_context(truth_params: SolarWindParams): `SolarWindFitContext` whose count rates are forward-modelled from `truth_params`. Returns `(swapi_response, rotation_matrices, fit_ctx)`.""" all_voltages_2d = np.tile(_VOLTAGES_PER_SWEEP, (_N_SWEEPS, 1)) - swapi_response = load_swapi_response( - warm_cache_voltages=all_voltages_2d.ravel() - ) + swapi_response = load_swapi_response(warm_cache_voltages=all_voltages_2d.ravel()) rotation_matrices = _per_bin_rotation_matrices() # Build a context with placeholder rates first, then forward-model the @@ -203,9 +252,7 @@ def test_recovers_temperature(self): def test_recovers_velocity_components(self): """Fitting noise-free synthesized rates recovers all three RTN bulk-velocity components within 1 km/s.""" nominal = self.result.velocity_rtn_nominal() - np.testing.assert_allclose( - nominal, _TRUE_VELOCITY_RTN_KM_S, atol=1.0 - ) + np.testing.assert_allclose(nominal, _TRUE_VELOCITY_RTN_KM_S, atol=1.0) def test_quality_flag_is_none_on_successful_convergence(self): """A clean LM convergence on noise-free data leaves the bad-fit flag cleared (`SwapiL3Flags.NONE`).""" @@ -244,12 +291,8 @@ def test_velocity_rtn_nominal_matches_per_component_nominal_values( self, ): """The nominal-velocity vector matches the per-component `.nominal_value` of the stored UFloat triple.""" - per_component = np.array( - [v.nominal_value for v in self.result.velocity_rtn] - ) - np.testing.assert_array_equal( - self.result.velocity_rtn_nominal(), per_component - ) + per_component = np.array([v.nominal_value for v in self.result.velocity_rtn]) + np.testing.assert_array_equal(self.result.velocity_rtn_nominal(), per_component) def test_velocity_rtn_covariance_is_three_by_three(self): """The covariance accessor returns a 3x3 matrix matching the RTN component count.""" @@ -271,9 +314,7 @@ def test_covariance_diagonal_matches_per_component_variance(self): """Each diagonal entry of the velocity covariance equals the square of the corresponding UFloat `std_dev`.""" covariance = self.result.velocity_rtn_covariance() for i, ufloat_value in enumerate(self.result.velocity_rtn): - self.assertAlmostEqual( - covariance[i, i], ufloat_value.std_dev ** 2, places=10 - ) + self.assertAlmostEqual(covariance[i, i], ufloat_value.std_dev**2, places=10) class TestQualityFlagBranches(unittest.TestCase): @@ -281,9 +322,7 @@ class TestQualityFlagBranches(unittest.TestCase): @classmethod def setUpClass(cls): - _, _, cls.fit_ctx = _build_synthetic_fit_context( - truth_params=_truth_params() - ) + _, _, cls.fit_ctx = _build_synthetic_fit_context(truth_params=_truth_params()) def _patch_optimizer_with_result(self, optimize_result): return patch( @@ -301,9 +340,7 @@ def test_fit_error_flag_when_optimizer_reports_failure(self): mass=PROTON_MASS_KG, ), residuals=np.zeros(_N_BINS_PER_SWEEP * _N_SWEEPS), - jacobian=np.zeros( - (_N_BINS_PER_SWEEP * _N_SWEEPS, N_STATE) - ), + jacobian=np.zeros((_N_BINS_PER_SWEEP * _N_SWEEPS, N_STATE)), success=False, ) with self._patch_optimizer_with_result(failed_result): @@ -323,9 +360,7 @@ def test_quality_flag_when_temperature_above_threshold(self): mass=PROTON_MASS_KG, ), residuals=np.zeros(_N_BINS_PER_SWEEP * _N_SWEEPS), - jacobian=np.zeros( - (_N_BINS_PER_SWEEP * _N_SWEEPS, N_STATE) - ), + jacobian=np.zeros((_N_BINS_PER_SWEEP * _N_SWEEPS, N_STATE)), success=True, ) with patch( @@ -345,9 +380,7 @@ class TestPipelineOrder(unittest.TestCase): @classmethod def setUpClass(cls): - _, _, cls.fit_ctx = _build_synthetic_fit_context( - truth_params=_truth_params() - ) + _, _, cls.fit_ctx = _build_synthetic_fit_context(truth_params=_truth_params()) def test_construct_fit_result_uses_post_escape_local_minimum_result(self): """When `escape_local_minimum` returns a result distinct from LM-1, the final fit result carries the post-escape parameters.""" @@ -366,9 +399,7 @@ def test_construct_fit_result_uses_post_escape_local_minimum_result(self): mass=PROTON_MASS_KG, ), residuals=np.zeros(_N_BINS_PER_SWEEP * _N_SWEEPS), - jacobian=np.zeros( - (_N_BINS_PER_SWEEP * _N_SWEEPS, N_STATE) - ), + jacobian=np.zeros((_N_BINS_PER_SWEEP * _N_SWEEPS, N_STATE)), success=True, ) @@ -380,9 +411,7 @@ def test_construct_fit_result_uses_post_escape_local_minimum_result(self): # density alone is decisive: it, temperature, and velocity_rtn # propagate together from the same OptimizeSolarWindProtonParamsResult. - self.assertAlmostEqual( - result.density.nominal_value, post_escape_density - ) + self.assertAlmostEqual(result.density.nominal_value, post_escape_density) if __name__ == "__main__": diff --git a/tests/swapi/l3a/science/solar_wind/proton/test_optimize_solar_wind_proton_params.py b/tests/swapi/l3a/science/solar_wind/proton/test_optimize_solar_wind_proton_params.py index 482d28a66..99bfde20a 100644 --- a/tests/swapi/l3a/science/solar_wind/proton/test_optimize_solar_wind_proton_params.py +++ b/tests/swapi/l3a/science/solar_wind/proton/test_optimize_solar_wind_proton_params.py @@ -190,7 +190,5 @@ def test_uses_lm_with_xtol_from_doc_spec(self): self.assertEqual(kwargs["xtol"], 1e-4) - - if __name__ == "__main__": unittest.main() diff --git a/tests/swapi/l3a/science/solar_wind/test_azimuthal_regions.py b/tests/swapi/l3a/science/solar_wind/test_azimuthal_regions.py index 6b1aecd67..f474e6d5b 100644 --- a/tests/swapi/l3a/science/solar_wind/test_azimuthal_regions.py +++ b/tests/swapi/l3a/science/solar_wind/test_azimuthal_regions.py @@ -32,7 +32,8 @@ def test_open_aperture_positive_wing_flags_and_sign(self): def test_region_namedtuple_field_order_is_stable(self): """The AzimuthalRegion NamedTuple field order is locked so positional unpacking in numba forward-model loops keeps working.""" self.assertEqual( - AzimuthalRegion._fields, ("is_sunglasses", "is_open_aperture", "azimuth_sign") + AzimuthalRegion._fields, + ("is_sunglasses", "is_open_aperture", "azimuth_sign"), ) diff --git a/tests/swapi/l3a/science/solar_wind/test_fit_context.py b/tests/swapi/l3a/science/solar_wind/test_fit_context.py index 60abf894c..c03ebd228 100644 --- a/tests/swapi/l3a/science/solar_wind/test_fit_context.py +++ b/tests/swapi/l3a/science/solar_wind/test_fit_context.py @@ -54,12 +54,14 @@ def setUp(self): count_rate=np.array([[10.0, 20.0, 30.0], [40.0, 50.0, 60.0]]), esa_voltage=np.array([[100.0, 200.0, 300.0], [400.0, 500.0, 600.0]]), response_grids=[ - "s0b0", "s0b1", "s0b2", - "s1b0", "s1b1", "s1b2", + "s0b0", + "s0b1", + "s0b2", + "s1b0", + "s1b1", + "s1b2", ], - rotation_matrices=np.stack( - [np.eye(3) * (i + 1) for i in range(6)] - ), + rotation_matrices=np.stack([np.eye(3) * (i + 1) for i in range(6)]), mass_kg=PROTON_MASS_KG, ) @@ -78,16 +80,12 @@ def test_subset_with_all_bin_indices_returns_equivalent_context(self): def test_subset_picks_bin_axis_at_the_given_indices_preserving_sweeps(self): """Selecting bin indices [0, 2] yields a context with those two bin columns from each sweep; count_rate and esa_voltage stay 2D as (n_sweeps, n_kept_bins).""" kept = self.full_ctx.subset(np.array([0, 2])) - np.testing.assert_array_equal( - kept.count_rate, [[10.0, 30.0], [40.0, 60.0]] - ) + np.testing.assert_array_equal(kept.count_rate, [[10.0, 30.0], [40.0, 60.0]]) np.testing.assert_array_equal( kept.esa_voltage, [[100.0, 300.0], [400.0, 600.0]] ) # Flat sweep-major bin-minor order: s0b0, s0b2, s1b0, s1b2 - self.assertEqual( - list(kept.response_grids), ["s0b0", "s0b2", "s1b0", "s1b2"] - ) + self.assertEqual(list(kept.response_grids), ["s0b0", "s0b2", "s1b0", "s1b2"]) np.testing.assert_array_equal( kept.rotation_matrices, self.full_ctx.rotation_matrices[[0, 2, 3, 5]], diff --git a/tests/swapi/l3a/science/solar_wind/test_forward_model.py b/tests/swapi/l3a/science/solar_wind/test_forward_model.py index ab3f80217..515594ebe 100644 --- a/tests/swapi/l3a/science/solar_wind/test_forward_model.py +++ b/tests/swapi/l3a/science/solar_wind/test_forward_model.py @@ -82,8 +82,6 @@ def _esa_voltage_at_speed(speed_km_s: float) -> float: _JACOBIAN_ATOL = 1e-3 - - def _build_fit_context_for_voltages( response: SwapiResponse, esa_voltages: np.ndarray, @@ -142,9 +140,7 @@ def test_returns_one_rate_and_one_jacobian_row_per_sweep(self): ] ), ) - rates, jacobian = model_solar_wind_ideal_coincidence_rates( - proton_params(), ctx - ) + rates, jacobian = model_solar_wind_ideal_coincidence_rates(proton_params(), ctx) self.assertEqual(rates.shape, (3,)) self.assertEqual(jacobian.shape, (3, N_STATE)) diff --git a/tests/swapi/l3a/science/solar_wind/test_integration_limits.py b/tests/swapi/l3a/science/solar_wind/test_integration_limits.py index c7f8f4145..d19813879 100644 --- a/tests/swapi/l3a/science/solar_wind/test_integration_limits.py +++ b/tests/swapi/l3a/science/solar_wind/test_integration_limits.py @@ -204,9 +204,7 @@ def setUpClass(cls): cls.identity = np.eye(3) # Bulk pointed at az=+90°: identity rotation makes v_inst = (-v, 0, 0), # giving azimuth = atan2(v, 0) = 90° and elevation = 0°. - cls.sw_bulk_at_90 = proton_params( - velocity_rtn=(-_BULK_SPEED_KM_S, 0.0, 0.0) - ) + cls.sw_bulk_at_90 = proton_params(velocity_rtn=(-_BULK_SPEED_KM_S, 0.0, 0.0)) def test_oa_positive_region_is_active_for_bulk_at_90_deg_azimuth(self): """A bulk at az ≈ +90° lies inside the OA+ band, so the region is active and a quadrature is returned.""" @@ -229,12 +227,8 @@ def test_oa_positive_azimuth_window_is_inside_20_to_150_degrees(self): self.identity, 0.0, ) - self.assertGreaterEqual( - quadrature.azimuth_points.min(), _OA_AZIMUTH_INNER_DEG - ) - self.assertLessEqual( - quadrature.azimuth_points.max(), _OA_AZIMUTH_OUTER_DEG - ) + self.assertGreaterEqual(quadrature.azimuth_points.min(), _OA_AZIMUTH_INNER_DEG) + self.assertLessEqual(quadrature.azimuth_points.max(), _OA_AZIMUTH_OUTER_DEG) def test_oa_negative_region_is_skipped_for_bulk_at_plus_90_deg_azimuth(self): """A bulk at az ≈ +90° is too far from the OA- band for a narrow Maxwellian to reach, so the OA- region is skipped.""" @@ -285,12 +279,8 @@ def test_oa_negative_azimuth_window_is_inside_minus_150_to_minus_20(self): self.identity, 0.0, ) - self.assertGreaterEqual( - quadrature.azimuth_points.min(), -_OA_AZIMUTH_OUTER_DEG - ) - self.assertLessEqual( - quadrature.azimuth_points.max(), -_OA_AZIMUTH_INNER_DEG - ) + self.assertGreaterEqual(quadrature.azimuth_points.min(), -_OA_AZIMUTH_OUTER_DEG) + self.assertLessEqual(quadrature.azimuth_points.max(), -_OA_AZIMUTH_INNER_DEG) def test_oa_positive_region_is_skipped_for_bulk_at_minus_90_deg_azimuth(self): """A bulk at az ≈ -90° is too far from the OA+ band for a narrow Maxwellian to reach, so the OA+ region is skipped.""" @@ -373,9 +363,7 @@ def test_speed_quadrature_arrays_are_all_15_long(self): def test_returns_skip_when_bulk_speed_window_does_not_overlap_passband(self): """A bulk so far above the passband that `bulk - k·σ` exceeds the upper edge yields skip=True and a `None` quadrature.""" - sw = proton_params( - velocity_rtn=(0.0, -1500.0, 0.0), temperature=10_000.0 - ) + sw = proton_params(velocity_rtn=(0.0, -1500.0, 0.0), temperature=10_000.0) skip, speed_quadrature = get_speed_quadrature( sw, self.response_grid, REGION_SUNGLASSES, 0.0 ) @@ -407,15 +395,15 @@ def test_uses_region_specific_passband(self): # silently caught here. self.assertNotEqual(sg_quad.points.min(), oa_quad.points.min()) - def test_speed_cubed_times_passband_is_v3_times_normalized_passband_at_each_node(self): + def test_speed_cubed_times_passband_is_v3_times_normalized_passband_at_each_node( + self, + ): """The `speed_cubed_times_passband` array equals `v³ · P(θ, v/v0) / P(0, 1)` reconstructed from `interpolate_passband` at every GL node.""" sw = proton_params() _, speed_quadrature = get_speed_quadrature( sw, self.response_grid, REGION_SUNGLASSES, 0.0 ) - on_axis_peak = interpolate_passband( - self.response_grid.sg_passband, 0.0, 1.0 - ) + on_axis_peak = interpolate_passband(self.response_grid.sg_passband, 0.0, 1.0) expected = np.array( [ v**3 diff --git a/tests/swapi/l3a/science/solar_wind/test_params.py b/tests/swapi/l3a/science/solar_wind/test_params.py index 63f3bf8b0..e1c834e3c 100644 --- a/tests/swapi/l3a/science/solar_wind/test_params.py +++ b/tests/swapi/l3a/science/solar_wind/test_params.py @@ -57,9 +57,7 @@ def test_from_vector_inverts_to_vector(self): ) self.assertAlmostEqual(round_tripped.density, original.density) self.assertAlmostEqual(round_tripped.temperature, original.temperature) - np.testing.assert_array_equal( - round_tripped.velocity_rtn, original.velocity_rtn - ) + np.testing.assert_array_equal(round_tripped.velocity_rtn, original.velocity_rtn) self.assertEqual(round_tripped.mass, original.mass) def test_from_vector_carries_mass_through_unchanged(self): diff --git a/tests/swapi/l3a/science/solar_wind/test_trim_open_aperture.py b/tests/swapi/l3a/science/solar_wind/test_trim_open_aperture.py index 6a26fc72f..0588609d1 100644 --- a/tests/swapi/l3a/science/solar_wind/test_trim_open_aperture.py +++ b/tests/swapi/l3a/science/solar_wind/test_trim_open_aperture.py @@ -56,9 +56,7 @@ def _proton_params_at_elevation( ) -> SolarWindParams: v_x_swapi = -speed * math.cos(math.radians(bulk_el_deg)) v_z_swapi = -speed * math.sin(math.radians(bulk_el_deg)) - v_rtn = NOMINAL_SWAPI_TO_RTN_ROTATION @ np.array( - [v_x_swapi, 0.0, v_z_swapi] - ) + v_rtn = NOMINAL_SWAPI_TO_RTN_ROTATION @ np.array([v_x_swapi, 0.0, v_z_swapi]) return proton_params( velocity_rtn=tuple(v_rtn), density=density, temperature=temperature ) @@ -193,7 +191,9 @@ def test_endpoints_expand_one_node_past_threshold_crossings(self): # interior. The integrand reduces to the Maxwellian centered at the # bulk azimuth, which we recompute here. rg = _make_response_grid() - sw = proton_params(velocity_rtn=(0.0, -450.0, 0.0)) # bulk_az = +90°, bulk_el = 0° + sw = proton_params( + velocity_rtn=(0.0, -450.0, 0.0) + ) # bulk_az = +90°, bulk_el = 0° lo, hi = _trim(rg, sw, azimuth_lo=20.0, azimuth_hi=150.0, sg_rate=0.0) @@ -202,7 +202,8 @@ def test_endpoints_expand_one_node_past_threshold_crossings(self): speed = bulk_speed(sw) central = rg.central_speed delta_v_sq = ( - central**2 + speed**2 + central**2 + + speed**2 - 2.0 * central * speed * np.cos(np.radians(scan - 90.0)) ) integrand = np.exp(-delta_v_sq / (2.0 * sigma**2)) @@ -227,9 +228,13 @@ def test_mirror_symmetric_bulk_yields_mirror_symmetric_windows(self): sw_pos = proton_params(velocity_rtn=(0.0, -450.0, 0.0)) sw_neg = proton_params(velocity_rtn=(0.0, +450.0, 0.0)) - lo_pos, hi_pos = _trim(rg, sw_pos, azimuth_lo=20.0, azimuth_hi=150.0, sg_rate=0.0) + lo_pos, hi_pos = _trim( + rg, sw_pos, azimuth_lo=20.0, azimuth_hi=150.0, sg_rate=0.0 + ) # OA-: same magnitudes, mirrored around 0. - lo_neg, hi_neg = _trim(rg, sw_neg, azimuth_lo=-150.0, azimuth_hi=-20.0, sg_rate=0.0) + lo_neg, hi_neg = _trim( + rg, sw_neg, azimuth_lo=-150.0, azimuth_hi=-20.0, sg_rate=0.0 + ) # The implementation picks scan-grid nodes by index, and # `linspace(20, 150, N)` mirrors `linspace(-150, -20, N)` up to a @@ -246,9 +251,7 @@ def test_high_sg_rate_forces_skip_for_otherwise_aligned_bulk(self): rg = _make_response_grid() sw = proton_params(velocity_rtn=(0.0, -450.0, 0.0)) # az_inst = +90° - lo_open, hi_open = _trim( - rg, sw, azimuth_lo=20.0, azimuth_hi=150.0, sg_rate=0.0 - ) + lo_open, hi_open = _trim(rg, sw, azimuth_lo=20.0, azimuth_hi=150.0, sg_rate=0.0) self.assertGreater(hi_open, lo_open) lo, hi = _trim(rg, sw, azimuth_lo=20.0, azimuth_hi=150.0, sg_rate=1.0e15) diff --git a/tests/swapi/l3a/science/solar_wind/test_uncertainty_calibration.py b/tests/swapi/l3a/science/solar_wind/test_uncertainty_calibration.py index 7bf33b6e6..2dd22a745 100644 --- a/tests/swapi/l3a/science/solar_wind/test_uncertainty_calibration.py +++ b/tests/swapi/l3a/science/solar_wind/test_uncertainty_calibration.py @@ -76,8 +76,7 @@ def _per_bin_rotation_matrices() -> np.ndarray: sweep_index = np.repeat(np.arange(_N_SWEEPS), _N_BINS_PER_SWEEP) bin_index_in_sweep = np.tile(np.arange(1, _N_BINS_PER_SWEEP + 1), _N_SWEEPS) sample_times_s = ( - sweep_index * _SWEEP_DURATION_S - + bin_index_in_sweep * _SAMPLE_TIME_PER_BIN_S + sweep_index * _SWEEP_DURATION_S + bin_index_in_sweep * _SAMPLE_TIME_PER_BIN_S ) spin_axis = NOMINAL_SWAPI_TO_RTN_ROTATION[:, 1] spin_axis = spin_axis / np.linalg.norm(spin_axis) @@ -102,9 +101,7 @@ def _per_bin_rotation_matrices() -> np.ndarray: _TRUE_BULK_SPEED_KM_S = 450.0 _TRUE_PROTON_DENSITY_CM3 = 5.0 _TRUE_PROTON_TEMPERATURE_K = 1.0e5 -_TRUE_PROTON_VELOCITY_RTN = ( - -_TRUE_BULK_SPEED_KM_S * NOMINAL_SWAPI_TO_RTN_ROTATION[:, 1] -) +_TRUE_PROTON_VELOCITY_RTN = -_TRUE_BULK_SPEED_KM_S * NOMINAL_SWAPI_TO_RTN_ROTATION[:, 1] _TRUE_ALPHA_DENSITY_CM3 = 0.2 _TRUE_ALPHA_TEMPERATURE_K = 4.0e5 _TRUE_DELTA_V_KM_S = 30.0 @@ -144,7 +141,9 @@ def _build_truth_count_rates(response, *, with_alpha: bool) -> np.ndarray: species=Species.PROTON, rotation_matrices=_ROTATION_MATRICES, ) - proton_rates, _ = model_solar_wind_ideal_coincidence_rates(proton_params, proton_ctx) + proton_rates, _ = model_solar_wind_ideal_coincidence_rates( + proton_params, proton_ctx + ) total_rates = proton_rates if with_alpha: alpha_velocity = _TRUE_PROTON_VELOCITY_RTN + _TRUE_DELTA_V_KM_S * _B_HAT_RTN @@ -162,7 +161,9 @@ def _build_truth_count_rates(response, *, with_alpha: bool) -> np.ndarray: species=Species.ALPHA, rotation_matrices=_ROTATION_MATRICES, ) - alpha_rates, _ = model_solar_wind_ideal_coincidence_rates(alpha_params, alpha_ctx) + alpha_rates, _ = model_solar_wind_ideal_coincidence_rates( + alpha_params, alpha_ctx + ) total_rates = proton_rates + alpha_rates deadtime_applied = total_rates * deadtime_factor(total_rates) return deadtime_applied.reshape(_VOLTAGE_AXIS.shape) @@ -349,7 +350,9 @@ def _run_trial(trial_seed: int) -> dict | None: b_hat_rtn=_B_HAT_RTN.copy(), quality_flag=int(alpha_moments.quality_flag), ) - return _alpha_moments_from_fit(chunk_result, epoch=0, sc_velocity_rtn=_SC_VELOCITY_RTN) + return _alpha_moments_from_fit( + chunk_result, epoch=0, sc_velocity_rtn=_SC_VELOCITY_RTN + ) def _run_mc_in_parallel(n_trials: int) -> tuple[dict, int]: diff --git a/tests/swapi/l3a/science/solar_wind/test_utils.py b/tests/swapi/l3a/science/solar_wind/test_utils.py index 16f041cd6..bfa99f8ff 100644 --- a/tests/swapi/l3a/science/solar_wind/test_utils.py +++ b/tests/swapi/l3a/science/solar_wind/test_utils.py @@ -31,9 +31,7 @@ def test_single_rotation_returns_its_y_axis(self): def test_constant_axis_across_sweeps_returns_that_axis(self): """Five identical identity rotations average to the same +Y_RTN unit vector, confirming the mean is over the sweep axis.""" rotations = np.stack([np.eye(3)] * 5) - np.testing.assert_array_equal( - average_spin_axis_rtn(rotations), [0.0, 1.0, 0.0] - ) + np.testing.assert_array_equal(average_spin_axis_rtn(rotations), [0.0, 1.0, 0.0]) def test_averages_two_axes_at_pm_45_deg_to_a_unit_vector(self): """Two body-+Y directions tilted ±45° in the RT-plane average to +Y_RTN after renormalization, cancelling the R components.""" @@ -73,9 +71,7 @@ def _proton_params( mass=PROTON_MASS_KG, ) - def _response_grid( - self, central_effective_area: float = 0.5 - ) -> ResponseGrid: + def _response_grid(self, central_effective_area: float = 0.5) -> ResponseGrid: return ResponseGrid( sg_passband=None, oa_passband=None, diff --git a/tests/swapi/l3a/science/test_inflow_vector.py b/tests/swapi/l3a/science/test_inflow_vector.py index 393e813b1..238695834 100644 --- a/tests/swapi/l3a/science/test_inflow_vector.py +++ b/tests/swapi/l3a/science/test_inflow_vector.py @@ -6,7 +6,11 @@ class TestInflowVector(unittest.TestCase): def test_from_file(self): - vector = InflowVector.from_file(get_test_data_path("swapi/imap_swapi_hydrogen-inflow-vector_20100101_v001.dat")) + vector = InflowVector.from_file( + get_test_data_path( + "swapi/imap_swapi_hydrogen-inflow-vector_20100101_v001.dat" + ) + ) self.assertIsInstance(vector, InflowVector) self.assertEqual(22, vector.speed_km_per_s) self.assertEqual(252.2, vector.longitude_deg_eclipj2000) @@ -15,12 +19,16 @@ def test_from_file(self): def test_raises_parse_error(self): with self.assertRaises(AssertionError) as exc_ctx: InflowVector.from_file( - get_test_data_path("swapi/imap_swapi_density-of-neutral-helium-lut_20241023_v000.dat")) + get_test_data_path( + "swapi/imap_swapi_density-of-neutral-helium-lut_20241023_v000.dat" + ) + ) self.assertEqual( "Failed to parse Inflow Vector from imap_swapi_density-of-neutral-helium-lut_20241023_v000.dat", - str(exc_ctx.exception)) + str(exc_ctx.exception), + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/swapi/l3a/test_swapi_l3a_dependencies.py b/tests/swapi/l3a/test_swapi_l3a_dependencies.py index a91d7031d..0ff33b08a 100644 --- a/tests/swapi/l3a/test_swapi_l3a_dependencies.py +++ b/tests/swapi/l3a/test_swapi_l3a_dependencies.py @@ -3,29 +3,42 @@ from unittest.mock import patch, call, sentinel import imap_data_access -from imap_data_access.processing_input import ScienceInput, ProcessingInputCollection, AncillaryInput +from imap_data_access.processing_input import ( + ScienceInput, + ProcessingInputCollection, + AncillaryInput, +) -from imap_l3_processing.swapi.descriptors import SWAPI_L2_DESCRIPTOR, DENSITY_OF_NEUTRAL_HELIUM_DESCRIPTOR, \ - EFFICIENCY_LOOKUP_TABLE_DESCRIPTOR, \ - HYDROGEN_INFLOW_VECTOR_DESCRIPTOR, HELIUM_INFLOW_VECTOR_DESCRIPTOR, \ - AZIMUTHAL_TRANSMISSION_DESCRIPTOR, CENTRAL_EFFECTIVE_AREA_DESCRIPTOR, PASSBAND_FIT_COEFFICIENTS_DESCRIPTOR, \ - MAG_RTN_DESCRIPTOR +from imap_l3_processing.swapi.descriptors import ( + SWAPI_L2_DESCRIPTOR, + DENSITY_OF_NEUTRAL_HELIUM_DESCRIPTOR, + EFFICIENCY_LOOKUP_TABLE_DESCRIPTOR, + HYDROGEN_INFLOW_VECTOR_DESCRIPTOR, + HELIUM_INFLOW_VECTOR_DESCRIPTOR, + AZIMUTHAL_TRANSMISSION_DESCRIPTOR, + CENTRAL_EFFECTIVE_AREA_DESCRIPTOR, + PASSBAND_FIT_COEFFICIENTS_DESCRIPTOR, + MAG_RTN_DESCRIPTOR, +) from imap_l3_processing.swapi.l3a.swapi_l3a_dependencies import SwapiL3ADependencies class TestSwapiL3ADependencies(unittest.TestCase): - @patch("imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.select_mag_path") - @patch("imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.SwapiL3ADependencies.from_file_paths") + @patch( + "imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.SwapiL3ADependencies.from_file_paths" + ) @patch("imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.download") - def test_fetch_dependencies(self, mock_download, mock_from_file_paths, mock_select_mag_path): + def test_fetch_dependencies( + self, mock_download, mock_from_file_paths, mock_select_mag_path + ): input_collection = ProcessingInputCollection() - start_date = '20100105' - mission = 'imap' - instrument = 'swapi' - data_level = 'l2' - version = 'v010' + start_date = "20100105" + mission = "imap" + instrument = "swapi" + data_level = "l2" + version = "v010" mock_download.side_effect = [ sentinel.swapi_l2_data, @@ -50,40 +63,66 @@ def test_fetch_dependencies(self, mock_download, mock_from_file_paths, mock_sele science_input = ScienceInput(swapi_science_file_download_path) efficiency_ancillary = AncillaryInput(swapi_efficiency_file_name) - density_of_neutral_helium_ancillary = AncillaryInput(swapi_density_of_neutral_helium_lookup) + density_of_neutral_helium_ancillary = AncillaryInput( + swapi_density_of_neutral_helium_lookup + ) hydrogen_inflow_ancillary = AncillaryInput(swapi_hydrogen_inflow_filename) helium_inflow_ancillary = AncillaryInput(swapi_helium_inflow_filename) - azimuthal_transmission_ancillary = AncillaryInput(swapi_azimuthal_transmission_filename) - central_effective_area_ancillary = AncillaryInput(swapi_central_effective_area_filename) - passband_fit_coefficients_ancillary = AncillaryInput(swapi_passband_fit_coefficients_filename) + azimuthal_transmission_ancillary = AncillaryInput( + swapi_azimuthal_transmission_filename + ) + central_effective_area_ancillary = AncillaryInput( + swapi_central_effective_area_filename + ) + passband_fit_coefficients_ancillary = AncillaryInput( + swapi_passband_fit_coefficients_filename + ) input_collection.add( - [science_input, efficiency_ancillary, - density_of_neutral_helium_ancillary, - hydrogen_inflow_ancillary, - helium_inflow_ancillary, - azimuthal_transmission_ancillary, - central_effective_area_ancillary, - passband_fit_coefficients_ancillary, - ]) - - actual_swapi_l3_dependencies = SwapiL3ADependencies.fetch_dependencies(input_collection) - - science_data_dir = imap_data_access.config["DATA_DIR"] / 'imap' / 'swapi' / 'l2' / '2010' / '01' - ancillary_data_dir = imap_data_access.config["DATA_DIR"] / 'imap' / 'ancillary' / 'swapi' - - mock_download.assert_has_calls([ - call(science_data_dir / swapi_science_file_download_path), - call(ancillary_data_dir / swapi_efficiency_file_name), - call(ancillary_data_dir / swapi_density_of_neutral_helium_lookup), - call(ancillary_data_dir / swapi_hydrogen_inflow_filename), - call(ancillary_data_dir / swapi_helium_inflow_filename), - call(ancillary_data_dir / swapi_azimuthal_transmission_filename), - call(ancillary_data_dir / swapi_central_effective_area_filename), - call(ancillary_data_dir / swapi_passband_fit_coefficients_filename), - ]) - - mock_select_mag_path.assert_called_once_with(input_collection, MAG_RTN_DESCRIPTOR) + [ + science_input, + efficiency_ancillary, + density_of_neutral_helium_ancillary, + hydrogen_inflow_ancillary, + helium_inflow_ancillary, + azimuthal_transmission_ancillary, + central_effective_area_ancillary, + passband_fit_coefficients_ancillary, + ] + ) + + actual_swapi_l3_dependencies = SwapiL3ADependencies.fetch_dependencies( + input_collection + ) + + science_data_dir = ( + imap_data_access.config["DATA_DIR"] + / "imap" + / "swapi" + / "l2" + / "2010" + / "01" + ) + ancillary_data_dir = ( + imap_data_access.config["DATA_DIR"] / "imap" / "ancillary" / "swapi" + ) + + mock_download.assert_has_calls( + [ + call(science_data_dir / swapi_science_file_download_path), + call(ancillary_data_dir / swapi_efficiency_file_name), + call(ancillary_data_dir / swapi_density_of_neutral_helium_lookup), + call(ancillary_data_dir / swapi_hydrogen_inflow_filename), + call(ancillary_data_dir / swapi_helium_inflow_filename), + call(ancillary_data_dir / swapi_azimuthal_transmission_filename), + call(ancillary_data_dir / swapi_central_effective_area_filename), + call(ancillary_data_dir / swapi_passband_fit_coefficients_filename), + ] + ) + + mock_select_mag_path.assert_called_once_with( + input_collection, MAG_RTN_DESCRIPTOR + ) mock_from_file_paths.assert_called_with( sentinel.swapi_l2_data, @@ -98,12 +137,18 @@ def test_fetch_dependencies(self, mock_download, mock_from_file_paths, mock_sele False, ) - self.assertEqual(mock_from_file_paths.return_value, actual_swapi_l3_dependencies) + self.assertEqual( + mock_from_file_paths.return_value, actual_swapi_l3_dependencies + ) @patch("imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.select_mag_path") - @patch("imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.SwapiL3ADependencies.from_file_paths") + @patch( + "imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.SwapiL3ADependencies.from_file_paths" + ) @patch("imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.download") - def test_fetch_dependencies_with_l1d_mag(self, mock_download, mock_from_file_paths, mock_select_mag_path): + def test_fetch_dependencies_with_l1d_mag( + self, mock_download, mock_from_file_paths, mock_select_mag_path + ): input_collection = ProcessingInputCollection() mock_download.side_effect = [ @@ -118,8 +163,8 @@ def test_fetch_dependencies_with_l1d_mag(self, mock_download, mock_from_file_pat ] mock_select_mag_path.return_value = (sentinel.mag_path, "l1d") - start_date = '20100105' - version = 'v010' + start_date = "20100105" + version = "v010" descriptors = [ ("imap_swapi_l2", SWAPI_L2_DESCRIPTOR, ScienceInput), ("imap_swapi", EFFICIENCY_LOOKUP_TABLE_DESCRIPTOR, AncillaryInput), @@ -140,31 +185,52 @@ def test_fetch_dependencies_with_l1d_mag(self, mock_download, mock_from_file_pat self.assertEqual(sentinel.mag_path, args[8]) self.assertEqual(True, args[9]) - @patch('imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.read_mag_rtn_data') - @patch('imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.SwapiResponse.from_files') - @patch('imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.CDF') - @patch('imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.InflowVector.from_file') - @patch('imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.EfficiencyCalibrationTable') - @patch('imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.DensityOfNeutralHeliumLookupTable.from_file') - @patch('imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.read_l2_swapi_data') - def test_from_file_paths(self, mock_read_l2_swapi, mock_neutral_helium_from_file, - mock_efficiency_lookup_class, - mock_inflow_vector_from_file, mock_CDF, - mock_swapi_response_from_files, mock_read_mag_rtn_data): + @patch("imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.read_mag_rtn_data") + @patch( + "imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.SwapiResponse.from_files" + ) + @patch("imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.CDF") + @patch("imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.InflowVector.from_file") + @patch( + "imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.EfficiencyCalibrationTable" + ) + @patch( + "imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.DensityOfNeutralHeliumLookupTable.from_file" + ) + @patch("imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.read_l2_swapi_data") + def test_from_file_paths( + self, + mock_read_l2_swapi, + mock_neutral_helium_from_file, + mock_efficiency_lookup_class, + mock_inflow_vector_from_file, + mock_CDF, + mock_swapi_response_from_files, + mock_read_mag_rtn_data, + ): science_path = Path("imap_swapi_l2_sci_20100105_v010.cdf") efficiency_path = Path("imap_swapi_efficiency_20100105_v010.cdf") neutral_helium_path = Path("imap_swapi_neutral_he_20100105_v010.cdf") hydrogen_vector_path = Path("imap_swapi_h_inflow_20100105_v010.dat") helium_vector_path = Path("imap_swapi_he_inflow_20100105_v010.dat") - azimuthal_transmission_path = Path("imap_swapi_azimuthal_transmission_20100105_v010.cdf") - central_effective_area_path = Path("imap_swapi_central_effective_area_20100105_v010.cdf") - passband_fit_coefficients_path = Path("imap_swapi_passband_fit_coefficients_20100105_v010.cdf") + azimuthal_transmission_path = Path( + "imap_swapi_azimuthal_transmission_20100105_v010.cdf" + ) + central_effective_area_path = Path( + "imap_swapi_central_effective_area_20100105_v010.cdf" + ) + passband_fit_coefficients_path = Path( + "imap_swapi_passband_fit_coefficients_20100105_v010.cdf" + ) mag_path = Path("imap_mag_l2_norm-rtn_20100105_v010.cdf") mock_read_l2_swapi.return_value = sentinel.swapi_l2_data mock_efficiency_lookup_class.return_value = sentinel.efficiency_lookup mock_neutral_helium_from_file.return_value = sentinel.neutral_helium_data - mock_inflow_vector_from_file.side_effect = [sentinel.hydrogen_vector, sentinel.helium_vector] + mock_inflow_vector_from_file.side_effect = [ + sentinel.hydrogen_vector, + sentinel.helium_vector, + ] mock_swapi_response_from_files.return_value = sentinel.swapi_response mock_read_mag_rtn_data.return_value = sentinel.mag_data @@ -197,27 +263,45 @@ def test_from_file_paths(self, mock_read_l2_swapi, mock_neutral_helium_from_file mock_efficiency_lookup_class.assert_called_once_with(efficiency_path) mock_neutral_helium_from_file.assert_called_once_with(neutral_helium_path) mock_inflow_vector_from_file.assert_has_calls( - [call(hydrogen_vector_path), call(helium_vector_path)]) + [call(hydrogen_vector_path), call(helium_vector_path)] + ) mock_swapi_response_from_files.assert_called_once_with( azimuthal_transmission_path=azimuthal_transmission_path, central_effective_area_path=central_effective_area_path, passband_fit_coefficients_path=passband_fit_coefficients_path, - efficiency_table_path=efficiency_path) + efficiency_table_path=efficiency_path, + ) mock_read_mag_rtn_data.assert_called_once_with(mag_path) self.assertEqual(expected_dependencies, actual_dependencies) - @patch('imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.read_mag_rtn_data') - @patch('imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.SwapiResponse.from_files') - @patch('imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.CDF') - @patch('imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.InflowVector.from_file') - @patch('imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.EfficiencyCalibrationTable') - @patch('imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.DensityOfNeutralHeliumLookupTable.from_file') - @patch('imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.read_l2_swapi_data') - def test_from_file_paths_without_mag(self, mock_read_l2_swapi, mock_neutral_helium_from_file, - mock_efficiency_lookup_class, mock_inflow_vector_from_file, mock_CDF, - mock_swapi_response_from_files, mock_read_mag_rtn_data): - mock_inflow_vector_from_file.side_effect = [sentinel.hydrogen_vector, sentinel.helium_vector] + @patch("imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.read_mag_rtn_data") + @patch( + "imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.SwapiResponse.from_files" + ) + @patch("imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.CDF") + @patch("imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.InflowVector.from_file") + @patch( + "imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.EfficiencyCalibrationTable" + ) + @patch( + "imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.DensityOfNeutralHeliumLookupTable.from_file" + ) + @patch("imap_l3_processing.swapi.l3a.swapi_l3a_dependencies.read_l2_swapi_data") + def test_from_file_paths_without_mag( + self, + mock_read_l2_swapi, + mock_neutral_helium_from_file, + mock_efficiency_lookup_class, + mock_inflow_vector_from_file, + mock_CDF, + mock_swapi_response_from_files, + mock_read_mag_rtn_data, + ): + mock_inflow_vector_from_file.side_effect = [ + sentinel.hydrogen_vector, + sentinel.helium_vector, + ] actual = SwapiL3ADependencies.from_file_paths( Path("science.cdf"), diff --git a/tests/swapi/l3b/science/test_calculate_solar_wind_differential_flux.py b/tests/swapi/l3b/science/test_calculate_solar_wind_differential_flux.py index 2f9464f05..1b03ae14a 100644 --- a/tests/swapi/l3b/science/test_calculate_solar_wind_differential_flux.py +++ b/tests/swapi/l3b/science/test_calculate_solar_wind_differential_flux.py @@ -3,9 +3,12 @@ import numpy as np -from imap_l3_processing.swapi.l3b.science.calculate_solar_wind_differential_flux import \ - calculate_combined_solar_wind_differential_flux -from imap_l3_processing.swapi.l3b.science.geometric_factor_calibration_table import GeometricFactorCalibrationTable +from imap_l3_processing.swapi.l3b.science.calculate_solar_wind_differential_flux import ( + calculate_combined_solar_wind_differential_flux, +) +from imap_l3_processing.swapi.l3b.science.geometric_factor_calibration_table import ( + GeometricFactorCalibrationTable, +) class TestCalculateSolarWindDifferentialFlux(TestCase): @@ -14,10 +17,15 @@ def test_calculate_combined_differential_flux(self): count_rates = np.array([10, 20, 30]) efficiency = 0.0882 mock_geometric_factor_table = create_autospec(GeometricFactorCalibrationTable) - mock_geometric_factor_table.lookup_geometric_factor.return_value = np.array([1e-12, 1e-13, 1e-14]) + mock_geometric_factor_table.lookup_geometric_factor.return_value = np.array( + [1e-12, 1e-13, 1e-14] + ) - differential_flux = calculate_combined_solar_wind_differential_flux(energies, count_rates, efficiency, - mock_geometric_factor_table) - expected_flux = np.array([11.337868480725623, 302.3431594860166, 6802.721088435374]) + differential_flux = calculate_combined_solar_wind_differential_flux( + energies, count_rates, efficiency, mock_geometric_factor_table + ) + expected_flux = np.array( + [11.337868480725623, 302.3431594860166, 6802.721088435374] + ) np.testing.assert_array_almost_equal(expected_flux, differential_flux) mock_geometric_factor_table.lookup_geometric_factor.assert_called_with(energies) diff --git a/tests/swapi/l3b/science/test_calculate_solar_wind_vdf.py b/tests/swapi/l3b/science/test_calculate_solar_wind_vdf.py index d8efb146e..0d5d8cabf 100644 --- a/tests/swapi/l3b/science/test_calculate_solar_wind_vdf.py +++ b/tests/swapi/l3b/science/test_calculate_solar_wind_vdf.py @@ -3,9 +3,13 @@ import numpy as np -from imap_l3_processing.swapi.l3b.science.calculate_solar_wind_vdf import calculate_proton_solar_wind_vdf, \ - GeometricFactorCalibrationTable, calculate_alpha_solar_wind_vdf, calculate_pui_solar_wind_vdf, \ - calculate_delta_minus_plus +from imap_l3_processing.swapi.l3b.science.calculate_solar_wind_vdf import ( + calculate_proton_solar_wind_vdf, + GeometricFactorCalibrationTable, + calculate_alpha_solar_wind_vdf, + calculate_pui_solar_wind_vdf, + calculate_delta_minus_plus, +) class TestCalculateSolarWindVDF(TestCase): @@ -14,13 +18,16 @@ def test_calculate_proton_solar_wind_vdf(self): count_rates = np.array([10, 20, 30]) efficiency = 0.0882 mock_geometric_factor_table = create_autospec(GeometricFactorCalibrationTable) - mock_geometric_factor_table.lookup_geometric_factor.return_value = np.array([1e-12, 1e-13, 1e-14]) + mock_geometric_factor_table.lookup_geometric_factor.return_value = np.array( + [1e-12, 1e-13, 1e-14] + ) expected_velocities = [437.6947142244463, 379.05474162054054, 309.496900517614] expected_probabilities = [0.001487, 0.039664, 0.892442] - velocities, probabilities = calculate_proton_solar_wind_vdf(energies, count_rates, efficiency, - mock_geometric_factor_table) + velocities, probabilities = calculate_proton_solar_wind_vdf( + energies, count_rates, efficiency, mock_geometric_factor_table + ) np.testing.assert_array_equal(velocities, expected_velocities) np.testing.assert_array_almost_equal(probabilities, expected_probabilities) @@ -49,13 +56,16 @@ def test_calculate_alpha_solar_wind_vdf(self): count_rates = np.array([10, 20, 30]) efficiency = 0.0882 mock_geometric_factor_table = create_autospec(GeometricFactorCalibrationTable) - mock_geometric_factor_table.lookup_geometric_factor.return_value = np.array([1e-12, 1e-13, 1e-14]) + mock_geometric_factor_table.lookup_geometric_factor.return_value = np.array( + [1e-12, 1e-13, 1e-14] + ) expected_velocities = [310.5624166704235, 268.95494229727456, 219.6007908093385] expected_probabilities = [0.002954, 0.078785, 1.772657] - velocities, probabilities = calculate_alpha_solar_wind_vdf(energies, count_rates, efficiency, - mock_geometric_factor_table) + velocities, probabilities = calculate_alpha_solar_wind_vdf( + energies, count_rates, efficiency, mock_geometric_factor_table + ) np.testing.assert_array_equal(velocities, expected_velocities) np.testing.assert_array_almost_equal(probabilities, expected_probabilities) @@ -66,13 +76,20 @@ def test_calculate_pickup_ion_solar_wind_vdf(self): count_rates = np.array([10, 20, 30]) efficiency = 0.0882 mock_geometric_factor_table = create_autospec(GeometricFactorCalibrationTable) - mock_geometric_factor_table.lookup_geometric_factor.return_value = np.array([1e-12, 1e-13, 1e-14]) - - expected_velocities = [219.58573945228636, 190.16682867447082, 155.27056541857408] + mock_geometric_factor_table.lookup_geometric_factor.return_value = np.array( + [1e-12, 1e-13, 1e-14] + ) + + expected_velocities = [ + 219.58573945228636, + 190.16682867447082, + 155.27056541857408, + ] expected_probabilities = [0.00591, 0.157591, 3.5458] - velocities, probabilities = calculate_pui_solar_wind_vdf(energies, count_rates, efficiency, - mock_geometric_factor_table) + velocities, probabilities = calculate_pui_solar_wind_vdf( + energies, count_rates, efficiency, mock_geometric_factor_table + ) np.testing.assert_array_equal(velocities, expected_velocities) np.testing.assert_array_almost_equal(probabilities, expected_probabilities) diff --git a/tests/swapi/l3b/science/test_geometric_factor_calibration_table.py b/tests/swapi/l3b/science/test_geometric_factor_calibration_table.py index 40a9ab242..d83605c18 100644 --- a/tests/swapi/l3b/science/test_geometric_factor_calibration_table.py +++ b/tests/swapi/l3b/science/test_geometric_factor_calibration_table.py @@ -1,12 +1,16 @@ import unittest -from imap_l3_processing.swapi.l3b.science.geometric_factor_calibration_table import GeometricFactorCalibrationTable +from imap_l3_processing.swapi.l3b.science.geometric_factor_calibration_table import ( + GeometricFactorCalibrationTable, +) from tests.test_helpers import get_test_data_path class TestGeometricFactorTable(unittest.TestCase): def test_geometric_factor_table_from_file(self): - file_path = get_test_data_path("swapi/imap_swapi_energy-gf-sw-lut_20100101_v001.csv") + file_path = get_test_data_path( + "swapi/imap_swapi_energy-gf-sw-lut_20100101_v001.csv" + ) calibration_table = GeometricFactorCalibrationTable.from_file(file_path) @@ -15,5 +19,11 @@ def test_geometric_factor_table_from_file(self): known_energy = 8165.393844536367 energy_to_interpolate = 14194.87288073211 - self.assertEqual(6.419796603112413e-13, calibration_table.lookup_geometric_factor(known_energy)) - self.assertAlmostEqual(5.711128783363629e-13, calibration_table.lookup_geometric_factor(energy_to_interpolate)) + self.assertEqual( + 6.419796603112413e-13, + calibration_table.lookup_geometric_factor(known_energy), + ) + self.assertAlmostEqual( + 5.711128783363629e-13, + calibration_table.lookup_geometric_factor(energy_to_interpolate), + ) diff --git a/tests/swapi/l3b/test_models.py b/tests/swapi/l3b/test_models.py index 4b86fd8cf..e51718f2d 100644 --- a/tests/swapi/l3b/test_models.py +++ b/tests/swapi/l3b/test_models.py @@ -24,10 +24,9 @@ class TestModels(CdfModelTestCase): def test_combined_vdf_data_products(self): - input_metadata = InputMetadata("swapi", "l3b", - datetime(2024, 9, 8), - datetime(2024, 9, 9), - "v001", "") + input_metadata = InputMetadata( + "swapi", "l3b", datetime(2024, 9, 8), datetime(2024, 9, 9), "v001", "" + ) epoch = np.array([1, 2, 3]) epoch_delta = np.full_like(epoch, FIVE_MINUTES_IN_NANOSECONDS) proton_velocities = np.array([4, 5, 6]) @@ -41,16 +40,28 @@ def test_combined_vdf_data_products(self): alpha_velocities_delta_plus = np.array([0.11, 0.12, 0.13]) alpha_velocities_delta_minus = 1 + np.array([0.11, 0.12, 0.13]) alpha_vdf = np.array([[14, 15, 16, 17], [17, 18, 19, 20], [20, 21, 22, 23]]) - alpha_vdf_uncertainties = np.array([[0.7, 0.8, 0.9, 1.0], [0.10, 0.11, 0.12, 0.13], [0.13, 0.14, 0.15, 0.16]]) + alpha_vdf_uncertainties = np.array( + [[0.7, 0.8, 0.9, 1.0], [0.10, 0.11, 0.12, 0.13], [0.13, 0.14, 0.15, 0.16]] + ) pui_velocities = np.array([23, 24, 25]) pui_velocities_delta_plus = np.array([0.23, 0.24, 0.25]) pui_velocities_delta_minus = 1 + np.array([0.23, 0.24, 0.25]) pui_vdf = np.array([[26, 27, 28, 29], [29, 30, 31, 32], [32, 33, 34, 35]]) - pui_vdf_uncertainties = np.array([[0.72, 0.8, 0.9, 1.0], [0.10, 0.121, 0.12, 0.13], [0.13, 0.142, 0.15, 0.16]]) - combined_energies = np.array([[100, 300, 1000, 3000], [100, 300, 1000, 3000],[100, 300, 1000, 3000]]) + pui_vdf_uncertainties = np.array( + [ + [0.72, 0.8, 0.9, 1.0], + [0.10, 0.121, 0.12, 0.13], + [0.13, 0.142, 0.15, 0.16], + ] + ) + combined_energies = np.array( + [[100, 300, 1000, 3000], [100, 300, 1000, 3000], [100, 300, 1000, 3000]] + ) combined_energies_delta_plus = np.array([44, 55, 66]) combined_energies_delta_minus = 1 + np.array([44, 55, 66]) - combined_differential_flux = np.array([[26, 27.2, 28, 29], [29.2, 30, 31, 32.1], [32, 33.5, 34, 36]]) + combined_differential_flux = np.array( + [[26, 27.2, 28, 29], [29.2, 30, 31, 32.1], [32, 33.5, 34, 36]] + ) combined_differential_flux_uncertainties = np.array( [ [0.725, 0.8, 0.9, 1.0], @@ -59,7 +70,9 @@ def test_combined_vdf_data_products(self): ] ) - expected_combined_energy_labels = np.array(["ESA 01", "ESA 02", "ESA 03", "ESA 04"]) + expected_combined_energy_labels = np.array( + ["ESA 01", "ESA 02", "ESA 03", "ESA 04"] + ) vdf = SwapiL3BCombinedVDF( input_metadata=input_metadata, @@ -79,21 +92,46 @@ def test_combined_vdf_data_products(self): combined_energy=combined_energies, combined_energy_delta_minus=combined_energies_delta_minus, combined_energy_delta_plus=combined_energies_delta_plus, - combined_differential_flux=uarray(combined_differential_flux, combined_differential_flux_uncertainties)) + combined_differential_flux=uarray( + combined_differential_flux, combined_differential_flux_uncertainties + ), + ) variables = vdf.to_data_product_variables() self.assertEqual(8, len(variables)) - self.assert_variable_attributes(variables[0], epoch, EPOCH_CDF_VAR_NAME, pycdf.const.CDF_TIME_TT2000) - self.assert_variable_attributes(variables[1], epoch_delta, EPOCH_DELTA_CDF_VAR_NAME) + self.assert_variable_attributes( + variables[0], epoch, EPOCH_CDF_VAR_NAME, pycdf.const.CDF_TIME_TT2000 + ) + self.assert_variable_attributes( + variables[1], epoch_delta, EPOCH_DELTA_CDF_VAR_NAME + ) - self.assert_variable_attributes(variables[2], combined_energies, SOLAR_WIND_ENERGY_CDF_VAR_NAME) - self.assert_variable_attributes(variables[3], combined_energies_delta_minus, - SOLAR_WIND_COMBINED_ENERGY_DELTA_MINUS_CDF_VAR_NAME) - self.assert_variable_attributes(variables[4], combined_energies_delta_plus, - SOLAR_WIND_COMBINED_ENERGY_DELTA_PLUS_CDF_VAR_NAME) - self.assert_variable_attributes(variables[5], expected_combined_energy_labels, SOLAR_WIND_ENERGY_LABEL_CDF_VAR_NAME) - self.assert_variable_attributes(variables[6], combined_differential_flux, - COMBINED_SOLAR_WIND_DIFFERENTIAL_FLUX_CDF_VAR_NAME) - self.assert_variable_attributes(variables[7], combined_differential_flux_uncertainties, - COMBINED_SOLAR_WIND_DIFFERENTIAL_FLUX_DELTA_CDF_VAR_NAME) + self.assert_variable_attributes( + variables[2], combined_energies, SOLAR_WIND_ENERGY_CDF_VAR_NAME + ) + self.assert_variable_attributes( + variables[3], + combined_energies_delta_minus, + SOLAR_WIND_COMBINED_ENERGY_DELTA_MINUS_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + variables[4], + combined_energies_delta_plus, + SOLAR_WIND_COMBINED_ENERGY_DELTA_PLUS_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + variables[5], + expected_combined_energy_labels, + SOLAR_WIND_ENERGY_LABEL_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + variables[6], + combined_differential_flux, + COMBINED_SOLAR_WIND_DIFFERENTIAL_FLUX_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + variables[7], + combined_differential_flux_uncertainties, + COMBINED_SOLAR_WIND_DIFFERENTIAL_FLUX_DELTA_CDF_VAR_NAME, + ) diff --git a/tests/swapi/l3b/test_swapi_l3b_dependencies.py b/tests/swapi/l3b/test_swapi_l3b_dependencies.py index bd3236c5b..3fd283f71 100644 --- a/tests/swapi/l3b/test_swapi_l3b_dependencies.py +++ b/tests/swapi/l3b/test_swapi_l3b_dependencies.py @@ -4,53 +4,77 @@ from unittest.mock import patch, call, sentinel import imap_data_access -from imap_data_access.processing_input import ScienceInput, ProcessingInputCollection, AncillaryInput - -from imap_l3_processing.swapi.descriptors import SWAPI_L2_DESCRIPTOR, GEOMETRIC_FACTOR_SW_LOOKUP_TABLE_DESCRIPTOR, \ - EFFICIENCY_LOOKUP_TABLE_DESCRIPTOR +from imap_data_access.processing_input import ( + ScienceInput, + ProcessingInputCollection, + AncillaryInput, +) + +from imap_l3_processing.swapi.descriptors import ( + SWAPI_L2_DESCRIPTOR, + GEOMETRIC_FACTOR_SW_LOOKUP_TABLE_DESCRIPTOR, + EFFICIENCY_LOOKUP_TABLE_DESCRIPTOR, +) from imap_l3_processing.swapi.l3b.swapi_l3b_dependencies import SwapiL3BDependencies class TestSwapiL3BDependencies(unittest.TestCase): def setUp(self) -> None: - self.mock_imap_patcher = patch('imap_l3_processing.utils.imap_data_access') + self.mock_imap_patcher = patch("imap_l3_processing.utils.imap_data_access") self.mock_imap_api = self.mock_imap_patcher.start() def tearDown(self) -> None: self.mock_imap_patcher.stop() @patch( - "imap_l3_processing.swapi.l3b.swapi_l3b_dependencies.SwapiL3BDependencies.from_file_paths") - @patch( - "imap_l3_processing.swapi.l3b.swapi_l3b_dependencies.download") + "imap_l3_processing.swapi.l3b.swapi_l3b_dependencies.SwapiL3BDependencies.from_file_paths" + ) + @patch("imap_l3_processing.swapi.l3b.swapi_l3b_dependencies.download") def test_fetch_dependencies(self, mock_download, mock_from_file_paths): - incoming_data_level = 'l2' - version = 'v002' + incoming_data_level = "l2" + version = "v002" start_date = datetime(2025, 1, 1).strftime("%Y%m%d") - science_file_path = f'imap_swapi_{incoming_data_level}_{SWAPI_L2_DESCRIPTOR}_{start_date}_{version}.cdf' - geometric_calibration_path = f'imap_swapi_{GEOMETRIC_FACTOR_SW_LOOKUP_TABLE_DESCRIPTOR}_{start_date}_{version}.cdf' - efficiency_table_path = f'imap_swapi_{EFFICIENCY_LOOKUP_TABLE_DESCRIPTOR}_{start_date}_{version}.cdf' + science_file_path = f"imap_swapi_{incoming_data_level}_{SWAPI_L2_DESCRIPTOR}_{start_date}_{version}.cdf" + geometric_calibration_path = f"imap_swapi_{GEOMETRIC_FACTOR_SW_LOOKUP_TABLE_DESCRIPTOR}_{start_date}_{version}.cdf" + efficiency_table_path = f"imap_swapi_{EFFICIENCY_LOOKUP_TABLE_DESCRIPTOR}_{start_date}_{version}.cdf" science_input = ScienceInput(science_file_path) geometric_calibration_input = AncillaryInput(geometric_calibration_path) efficiency_table_input = AncillaryInput(efficiency_table_path) - dependencies = ProcessingInputCollection(science_input, geometric_calibration_input, efficiency_table_input) + dependencies = ProcessingInputCollection( + science_input, geometric_calibration_input, efficiency_table_input + ) - actual_swapi_l3b_dependencies = SwapiL3BDependencies.fetch_dependencies(dependencies) + actual_swapi_l3b_dependencies = SwapiL3BDependencies.fetch_dependencies( + dependencies + ) - sci_data_dir = imap_data_access.config["DATA_DIR"] / 'imap' / 'swapi' / 'l2' / '2025' / '01' - ancillary_data_dir = imap_data_access.config["DATA_DIR"] / 'imap' / 'ancillary' / 'swapi' + sci_data_dir = ( + imap_data_access.config["DATA_DIR"] + / "imap" + / "swapi" + / "l2" + / "2025" + / "01" + ) + ancillary_data_dir = ( + imap_data_access.config["DATA_DIR"] / "imap" / "ancillary" / "swapi" + ) expected_download_science_path = sci_data_dir / science_file_path - expected_geometric_calibration_path = ancillary_data_dir / geometric_calibration_path + expected_geometric_calibration_path = ( + ancillary_data_dir / geometric_calibration_path + ) expected_efficiency_table_path = ancillary_data_dir / efficiency_table_path - mock_download.assert_has_calls([ - call(expected_download_science_path), - call(expected_geometric_calibration_path), - call(expected_efficiency_table_path), - ]) + mock_download.assert_has_calls( + [ + call(expected_download_science_path), + call(expected_geometric_calibration_path), + call(expected_efficiency_table_path), + ] + ) mock_from_file_paths.assert_called_with( expected_download_science_path, @@ -58,42 +82,64 @@ def test_fetch_dependencies(self, mock_download, mock_from_file_paths): expected_efficiency_table_path, ) - self.assertEqual(mock_from_file_paths.return_value, actual_swapi_l3b_dependencies) - - @patch('imap_l3_processing.swapi.l3b.swapi_l3b_dependencies.CDF') - @patch('imap_l3_processing.swapi.l3b.swapi_l3b_dependencies.EfficiencyCalibrationTable') - @patch('imap_l3_processing.swapi.l3b.swapi_l3b_dependencies.GeometricFactorCalibrationTable.from_file') - @patch('imap_l3_processing.swapi.l3b.swapi_l3b_dependencies.read_l2_swapi_data') - def test_from_file_paths(self, mock_read_l2_swapi, mock_read_geometric_factor, mock_efficiency_calibration, - mock_cdf): - start_date = '20100105' - mission = 'imap' - instrument = 'swapi' - data_level = 'l2' - version = 'v010' + self.assertEqual( + mock_from_file_paths.return_value, actual_swapi_l3b_dependencies + ) - swapi_science_file_download_path = Path(f"{mission}_{instrument}_{data_level}_{SWAPI_L2_DESCRIPTOR}_{start_date}_{version}.cdf") - swapi_geometric_factor_calibration_file_path = Path(f"{mission}_{instrument}_{GEOMETRIC_FACTOR_SW_LOOKUP_TABLE_DESCRIPTOR}_{start_date}_{version}.cdf") - swapi_efficiency_calibration_file_path = Path(f"{mission}_{instrument}_{EFFICIENCY_LOOKUP_TABLE_DESCRIPTOR}_{start_date}_{version}.cdf") + @patch("imap_l3_processing.swapi.l3b.swapi_l3b_dependencies.CDF") + @patch( + "imap_l3_processing.swapi.l3b.swapi_l3b_dependencies.EfficiencyCalibrationTable" + ) + @patch( + "imap_l3_processing.swapi.l3b.swapi_l3b_dependencies.GeometricFactorCalibrationTable.from_file" + ) + @patch("imap_l3_processing.swapi.l3b.swapi_l3b_dependencies.read_l2_swapi_data") + def test_from_file_paths( + self, + mock_read_l2_swapi, + mock_read_geometric_factor, + mock_efficiency_calibration, + mock_cdf, + ): + start_date = "20100105" + mission = "imap" + instrument = "swapi" + data_level = "l2" + version = "v010" + + swapi_science_file_download_path = Path( + f"{mission}_{instrument}_{data_level}_{SWAPI_L2_DESCRIPTOR}_{start_date}_{version}.cdf" + ) + swapi_geometric_factor_calibration_file_path = Path( + f"{mission}_{instrument}_{GEOMETRIC_FACTOR_SW_LOOKUP_TABLE_DESCRIPTOR}_{start_date}_{version}.cdf" + ) + swapi_efficiency_calibration_file_path = Path( + f"{mission}_{instrument}_{EFFICIENCY_LOOKUP_TABLE_DESCRIPTOR}_{start_date}_{version}.cdf" + ) mock_read_l2_swapi.return_value = sentinel.swapi_l2_data mock_read_geometric_factor.return_value = sentinel.geometric_factor_data mock_efficiency_calibration.return_value = sentinel.efficiency_calibration_data - expected_dependencies = SwapiL3BDependencies(sentinel.swapi_l2_data, - sentinel.geometric_factor_data, - sentinel.efficiency_calibration_data, - ) + expected_dependencies = SwapiL3BDependencies( + sentinel.swapi_l2_data, + sentinel.geometric_factor_data, + sentinel.efficiency_calibration_data, + ) actual_dependencies = SwapiL3BDependencies.from_file_paths( swapi_science_file_download_path, swapi_geometric_factor_calibration_file_path, - swapi_efficiency_calibration_file_path + swapi_efficiency_calibration_file_path, ) mock_read_l2_swapi.assert_called_once_with(mock_cdf.return_value) - mock_read_geometric_factor.assert_called_once_with(swapi_geometric_factor_calibration_file_path) - mock_efficiency_calibration.assert_called_once_with(swapi_efficiency_calibration_file_path) + mock_read_geometric_factor.assert_called_once_with( + swapi_geometric_factor_calibration_file_path + ) + mock_efficiency_calibration.assert_called_once_with( + swapi_efficiency_calibration_file_path + ) mock_cdf.assert_called_once_with(str(swapi_science_file_download_path)) diff --git a/tests/swapi/response/test_azimuthal_transmission.py b/tests/swapi/response/test_azimuthal_transmission.py index 456d37b45..20c6e6a98 100644 --- a/tests/swapi/response/test_azimuthal_transmission.py +++ b/tests/swapi/response/test_azimuthal_transmission.py @@ -79,7 +79,9 @@ def test_linear_interpolation_off_center_between_grid_points(self): for fraction in [0.25, 0.7, 0.9]: with self.subTest(fraction=fraction): az = az_lower + fraction * _SPACING_DEG - expected = (1 - fraction) * self.values[idx] + fraction * self.values[idx + 1] + expected = (1 - fraction) * self.values[idx] + fraction * self.values[ + idx + 1 + ] got = interpolate_azimuthal_transmission(self.grid, az) self.assertAlmostEqual(got, float(expected)) @@ -170,7 +172,9 @@ def test_csv_layout_matches_production_spacing(self): self.assertAlmostEqual(float(grid.values[0]), 1e-3) -def _build_grid_satisfying_invariants(spacing: float = 0.1, n: int = 1801) -> np.ndarray: +def _build_grid_satisfying_invariants( + spacing: float = 0.1, n: int = 1801 +) -> np.ndarray: """Build a synthetic values array that respects the SG-floor and OA-plateau invariants the interpolator's short-circuit branches assume. Cells outside those two ranges are set to a neutral 0.5 so a test can corrupt a single @@ -178,7 +182,9 @@ def _build_grid_satisfying_invariants(spacing: float = 0.1, n: int = 1801) -> np azimuths = np.arange(n) * spacing values = np.full(n, 0.5) values[azimuths <= SG_PLATEAU_AZIMUTH_MAX_DEG] = SG_PLATEAU_TRANSMISSION - in_plateau = (azimuths >= OA_PLATEAU_AZIMUTH_MIN_DEG) & (azimuths <= OA_PLATEAU_AZIMUTH_MAX_DEG) + in_plateau = (azimuths >= OA_PLATEAU_AZIMUTH_MIN_DEG) & ( + azimuths <= OA_PLATEAU_AZIMUTH_MAX_DEG + ) values[in_plateau] = OA_PLATEAU_TRANSMISSION return values @@ -239,9 +245,11 @@ class TestSwapiResponseRejectsInvalidTransmissionCsv(unittest.TestCase): silently driving the short-circuit branches in the interpolator to lie.""" def test_from_files_raises_when_csv_violates_invariants(self): - df = pd.read_csv(get_test_instrument_team_data_path( - "swapi/imap_swapi_azimuthal-transmission_20260425_v001.csv" - )) + df = pd.read_csv( + get_test_instrument_team_data_path( + "swapi/imap_swapi_azimuthal-transmission_20260425_v001.csv" + ) + ) # Corrupt a single cell inside the OA plateau range. df.loc[500, "transmission"] = 0.5 diff --git a/tests/swapi/response/test_efficiency_calibration_table.py b/tests/swapi/response/test_efficiency_calibration_table.py index 536803c05..9ce85a64c 100644 --- a/tests/swapi/response/test_efficiency_calibration_table.py +++ b/tests/swapi/response/test_efficiency_calibration_table.py @@ -3,7 +3,9 @@ from spacepy import pycdf -from imap_l3_processing.swapi.response.efficiency_calibration_table import EfficiencyCalibrationTable +from imap_l3_processing.swapi.response.efficiency_calibration_table import ( + EfficiencyCalibrationTable, +) from imap_l3_processing.swapi.species import Species from tests.test_helpers import get_test_data_path @@ -11,72 +13,139 @@ class TestEfficiencyCalibrationTable(unittest.TestCase): def test_relative_efficiency_for_protons(self): """Each queried time returns the proton efficiency from the most recent row that precedes it, unscaled.""" - calibration_table_path = get_test_data_path("swapi/imap_swapi_efficiency-lut-test_20241020_v001.dat") + calibration_table_path = get_test_data_path( + "swapi/imap_swapi_efficiency-lut-test_20241020_v001.dat" + ) efficiency_table = EfficiencyCalibrationTable(calibration_table_path) self.assertEqual( - efficiency_table.relative_efficiency(pycdf.lib.datetime_to_tt2000(datetime(year=2001, month=2, day=1)), Species.PROTON), 0.1) + efficiency_table.relative_efficiency( + pycdf.lib.datetime_to_tt2000(datetime(year=2001, month=2, day=1)), + Species.PROTON, + ), + 0.1, + ) self.assertEqual( - efficiency_table.relative_efficiency(pycdf.lib.datetime_to_tt2000(datetime(year=2013, month=10, day=1)), Species.PROTON), - 0.1) + efficiency_table.relative_efficiency( + pycdf.lib.datetime_to_tt2000(datetime(year=2013, month=10, day=1)), + Species.PROTON, + ), + 0.1, + ) self.assertEqual( - efficiency_table.relative_efficiency(pycdf.lib.datetime_to_tt2000(datetime(year=2014, month=10, day=3)), Species.PROTON), - 0.09) + efficiency_table.relative_efficiency( + pycdf.lib.datetime_to_tt2000(datetime(year=2014, month=10, day=3)), + Species.PROTON, + ), + 0.09, + ) self.assertEqual( - efficiency_table.relative_efficiency(pycdf.lib.datetime_to_tt2000(datetime(year=2024, month=10, day=1)), Species.PROTON), - 0.09) + efficiency_table.relative_efficiency( + pycdf.lib.datetime_to_tt2000(datetime(year=2024, month=10, day=1)), + Species.PROTON, + ), + 0.09, + ) self.assertEqual( - efficiency_table.relative_efficiency(pycdf.lib.datetime_to_tt2000(datetime(year=2024, month=10, day=3)), Species.PROTON), - 0.0882) + efficiency_table.relative_efficiency( + pycdf.lib.datetime_to_tt2000(datetime(year=2024, month=10, day=3)), + Species.PROTON, + ), + 0.0882, + ) def test_relative_efficiency_for_helium(self): """A helium species reads the helium column by the same most-recent-preceding-row rule as protons.""" - calibration_table_path = get_test_data_path("swapi/imap_swapi_efficiency-lut-test_20241020_v001.dat") + calibration_table_path = get_test_data_path( + "swapi/imap_swapi_efficiency-lut-test_20241020_v001.dat" + ) efficiency_table = EfficiencyCalibrationTable(calibration_table_path) self.assertEqual( - efficiency_table.relative_efficiency(pycdf.lib.datetime_to_tt2000(datetime(year=2001, month=2, day=1)), Species.ALPHA), 0.9) + efficiency_table.relative_efficiency( + pycdf.lib.datetime_to_tt2000(datetime(year=2001, month=2, day=1)), + Species.ALPHA, + ), + 0.9, + ) self.assertEqual( - efficiency_table.relative_efficiency(pycdf.lib.datetime_to_tt2000(datetime(year=2013, month=10, day=1)), Species.ALPHA), - 0.9) + efficiency_table.relative_efficiency( + pycdf.lib.datetime_to_tt2000(datetime(year=2013, month=10, day=1)), + Species.ALPHA, + ), + 0.9, + ) self.assertEqual( - efficiency_table.relative_efficiency(pycdf.lib.datetime_to_tt2000(datetime(year=2014, month=10, day=3)), Species.ALPHA), - 0.95) + efficiency_table.relative_efficiency( + pycdf.lib.datetime_to_tt2000(datetime(year=2014, month=10, day=3)), + Species.ALPHA, + ), + 0.95, + ) self.assertEqual( - efficiency_table.relative_efficiency(pycdf.lib.datetime_to_tt2000(datetime(year=2024, month=10, day=1)), Species.ALPHA), - 0.95) + efficiency_table.relative_efficiency( + pycdf.lib.datetime_to_tt2000(datetime(year=2024, month=10, day=1)), + Species.ALPHA, + ), + 0.95, + ) self.assertEqual( - efficiency_table.relative_efficiency(pycdf.lib.datetime_to_tt2000(datetime(year=2024, month=10, day=3)), Species.ALPHA), - 0.99) + efficiency_table.relative_efficiency( + pycdf.lib.datetime_to_tt2000(datetime(year=2024, month=10, day=3)), + Species.ALPHA, + ), + 0.99, + ) def test_relative_efficiency_handles_float_input(self): """A TT2000 passed as a float rather than an int resolves to the same row.""" - calibration_table_path = get_test_data_path("swapi/imap_swapi_efficiency-lut-test_20241020_v001.dat") + calibration_table_path = get_test_data_path( + "swapi/imap_swapi_efficiency-lut-test_20241020_v001.dat" + ) efficiency_table = EfficiencyCalibrationTable(calibration_table_path) - time_as_float = float(pycdf.lib.datetime_to_tt2000(datetime(year=2001, month=2, day=1))) - self.assertEqual(efficiency_table.relative_efficiency(time_as_float, Species.ALPHA), 0.9) - self.assertEqual(efficiency_table.relative_efficiency(time_as_float, Species.PROTON), 0.1) + time_as_float = float( + pycdf.lib.datetime_to_tt2000(datetime(year=2001, month=2, day=1)) + ) + self.assertEqual( + efficiency_table.relative_efficiency(time_as_float, Species.ALPHA), 0.9 + ) + self.assertEqual( + efficiency_table.relative_efficiency(time_as_float, Species.PROTON), 0.1 + ) def test_raises_exception_if_ask_for_time_before_the_table_starts(self): """A time earlier than every row raises rather than falling back to the first row.""" - calibration_table_path = get_test_data_path("swapi/imap_swapi_efficiency-lut-test_20241020_v001.dat") + calibration_table_path = get_test_data_path( + "swapi/imap_swapi_efficiency-lut-test_20241020_v001.dat" + ) efficiency_table = EfficiencyCalibrationTable(calibration_table_path) with self.assertRaises(ValueError) as content_manager: time = datetime(year=1999, month=1, day=4) - efficiency_table.relative_efficiency(pycdf.lib.datetime_to_tt2000(time), Species.PROTON) + efficiency_table.relative_efficiency( + pycdf.lib.datetime_to_tt2000(time), Species.PROTON + ) - self.assertEqual((f"No efficiency data for {time}",), content_manager.exception.args) + self.assertEqual( + (f"No efficiency data for {time}",), content_manager.exception.args + ) def test_loads_calibration_table_with_single_row(self): """A one-row table still parses and serves both species for any later time.""" calibration_table_path = get_test_data_path( - "swapi/imap_swapi_efficiency-lut-single-row-test_20241020_v001.dat") + "swapi/imap_swapi_efficiency-lut-single-row-test_20241020_v001.dat" + ) efficiency_table = EfficiencyCalibrationTable(calibration_table_path) # file has one row, with 0.1 for H, 0.9 for He. - time_after_row = pycdf.lib.datetime_to_tt2000(datetime(year=2001, month=2, day=1)) - self.assertEqual(efficiency_table.relative_efficiency(time_after_row, Species.PROTON), 0.1) - self.assertEqual(efficiency_table.relative_efficiency(time_after_row, Species.ALPHA), 0.9) + time_after_row = pycdf.lib.datetime_to_tt2000( + datetime(year=2001, month=2, day=1) + ) + self.assertEqual( + efficiency_table.relative_efficiency(time_after_row, Species.PROTON), 0.1 + ) + self.assertEqual( + efficiency_table.relative_efficiency(time_after_row, Species.ALPHA), 0.9 + ) diff --git a/tests/swapi/response/test_passband_grid.py b/tests/swapi/response/test_passband_grid.py index 32a773757..bd6c85688 100644 --- a/tests/swapi/response/test_passband_grid.py +++ b/tests/swapi/response/test_passband_grid.py @@ -43,8 +43,7 @@ def _gaussian_values_df( # round so that there are true zeros like the real SIMION-derived passband np.exp( -((elev - peak_elevation) ** 2) / (2.0 * sigma_el**2) - - ((SWAPI_K_FACTOR * sr**2 - peak_er) ** 2) - / (2.0 * sigma_er**2) + - ((SWAPI_K_FACTOR * sr**2 - peak_er) ** 2) / (2.0 * sigma_er**2) ) ), } @@ -96,11 +95,15 @@ def test_boundary_marks_cutoff_crossing(self): continue with self.subTest(row=i): np.testing.assert_allclose( - np.interp(grid.min_boundary[i], _TARGET_SPEED_RATIOS, grid.values[i]), + np.interp( + grid.min_boundary[i], _TARGET_SPEED_RATIOS, grid.values[i] + ), cutoff, ) np.testing.assert_allclose( - np.interp(grid.max_boundary[i], _TARGET_SPEED_RATIOS, grid.values[i]), + np.interp( + grid.max_boundary[i], _TARGET_SPEED_RATIOS, grid.values[i] + ), cutoff, ) @@ -159,9 +162,7 @@ def test_speed_ratio_indexing_via_k_factor(self): must produce a grid peaking at speed_ratio = 1.0.""" grid = self.default_grid - row_at_peak_elevation = grid.values[ - _row_index_for_elevation(grid, 0.0) - ] + row_at_peak_elevation = grid.values[_row_index_for_elevation(grid, 0.0)] peak_speed_ratio = _TARGET_SPEED_RATIOS[int(np.argmax(row_at_peak_elevation))] self.assertAlmostEqual(peak_speed_ratio, 1.0, places=2) @@ -176,7 +177,7 @@ def setUp(self): def test_matches_corner_weighted_sum(self): """`interpolate_passband` returns the bilinear-weighted sum of the four - enclosing corner values.""" + enclosing corner values.""" cases = [ # (elevation, speed_ratio, label) (0.0, 1.0, "on-node"), @@ -194,9 +195,7 @@ def test_matches_corner_weighted_sum(self): i_lo, j_lo = int(i_float), int(j_float) i_weight, j_weight = i_float - i_lo, j_float - j_lo - corners = self.grid.values[ - i_lo : i_lo + 2, j_lo : j_lo + 2 - ] + corners = self.grid.values[i_lo : i_lo + 2, j_lo : j_lo + 2] weights = np.array( [ [(1 - i_weight) * (1 - j_weight), (1 - i_weight) * j_weight], diff --git a/tests/swapi/response/test_swapi_response.py b/tests/swapi/response/test_swapi_response.py index 677092fca..fa5b398f1 100644 --- a/tests/swapi/response/test_swapi_response.py +++ b/tests/swapi/response/test_swapi_response.py @@ -25,7 +25,7 @@ def _load_response() -> SwapiResponse: ), efficiency_table_path=get_test_data_path( "swapi/imap_swapi_efficiency-lut-test_20241020_v001.dat" - ) + ), ) @@ -49,7 +49,9 @@ def test_populates_cache_with_unique_finite_voltages(self): """Duplicates collapse, NaN/inf are skipped.""" voltages = np.array([100.0, 100.0, 200.0, 300.0, np.nan, np.inf]) self.response.warm_cache(voltages) - self.assertEqual(set(self.response._passband_grid_cache.keys()), {100.0, 200.0, 300.0}) + self.assertEqual( + set(self.response._passband_grid_cache.keys()), {100.0, 200.0, 300.0} + ) self.assertEqual(len(self.response._passband_grid_cache), 3) def test_warming_twice_at_same_voltage_is_a_noop(self): @@ -57,19 +59,25 @@ def test_warming_twice_at_same_voltage_is_a_noop(self): PassbandGrid objects — it does not rebuild them.""" self.response.warm_cache([750.0]) first_grids = { - region: self.response._passband_grid_cache[750.0][region] for region in ("SG", "OA") + region: self.response._passband_grid_cache[750.0][region] + for region in ("SG", "OA") } self.response.warm_cache([750.0]) for region in ("SG", "OA"): with self.subTest(region=region): - self.assertIs(self.response._passband_grid_cache[750.0][region], first_grids[region]) + self.assertIs( + self.response._passband_grid_cache[750.0][region], + first_grids[region], + ) def test_accepts_2d_voltage_array(self): """Multidimensional voltage inputs are flattened.""" voltages = np.array([[100.0, 200.0], [200.0, 300.0]]) self.response.warm_cache(voltages) - self.assertEqual(set(self.response._passband_grid_cache.keys()), {100.0, 200.0, 300.0}) + self.assertEqual( + set(self.response._passband_grid_cache.keys()), {100.0, 200.0, 300.0} + ) class TestPassbandInterpolation(_RealResponseFixture): @@ -99,18 +107,18 @@ def test_values_match_manual_polyval(self): log_beam_energy = np.log(SWAPI_K_FACTOR * voltage) n_degrees = coeffs.shape[1] degrees = np.arange(n_degrees - 1, -1, -1) - expected_exponent = (coeffs.values * log_beam_energy ** degrees).sum( + expected_exponent = (coeffs.values * log_beam_energy**degrees).sum( axis=1 ) expected = np.exp(expected_exponent) - npt.assert_allclose( - returned["value"].to_numpy(), expected, rtol=1e-12 - ) + npt.assert_allclose(returned["value"].to_numpy(), expected, rtol=1e-12) def _assert_grid_value_bounds(self, esa_voltage): self.response.warm_cache([esa_voltage]) - cached = self.response._passband_grid_cache[self.response._cache_key(esa_voltage)] + cached = self.response._passband_grid_cache[ + self.response._cache_key(esa_voltage) + ] for region in ("SG", "OA"): grid = cached[region] self.assertGreaterEqual( @@ -141,32 +149,65 @@ def test_get_response_grid(self): response = _load_response() response.warm_cache(esa_voltage) - response_grid = response.get_response_grid(time_as_tt2000, esa_voltage, Species.PROTON) - self.assertAlmostEqual(response_grid.central_effective_area, 0.37553722631276887) - self.assertIs(response_grid.azimuthal_transmission, response._azimuthal_transmission_grid) - self.assertIs(response_grid.oa_passband, response._passband_grid_cache[round(esa_voltage, 3)]["OA"]) - self.assertIs(response_grid.sg_passband, response._passband_grid_cache[round(esa_voltage, 3)]["SG"]) - expected_central_speed_km_per_s = np.sqrt( - 2 * 1.89 * 552.1339894 * constants.PROTON_CHARGE_COULOMBS / constants.PROTON_MASS_KG - ) / constants.METERS_PER_KILOMETER - self.assertAlmostEqual(response_grid.central_speed, expected_central_speed_km_per_s) - - response_grid2 = response.get_response_grid(time_as_tt2000, esa_voltage, Species.PROTON) + response_grid = response.get_response_grid( + time_as_tt2000, esa_voltage, Species.PROTON + ) + self.assertAlmostEqual( + response_grid.central_effective_area, 0.37553722631276887 + ) + self.assertIs( + response_grid.azimuthal_transmission, response._azimuthal_transmission_grid + ) + self.assertIs( + response_grid.oa_passband, + response._passband_grid_cache[round(esa_voltage, 3)]["OA"], + ) + self.assertIs( + response_grid.sg_passband, + response._passband_grid_cache[round(esa_voltage, 3)]["SG"], + ) + expected_central_speed_km_per_s = ( + np.sqrt( + 2 + * 1.89 + * 552.1339894 + * constants.PROTON_CHARGE_COULOMBS + / constants.PROTON_MASS_KG + ) + / constants.METERS_PER_KILOMETER + ) + self.assertAlmostEqual( + response_grid.central_speed, expected_central_speed_km_per_s + ) + + response_grid2 = response.get_response_grid( + time_as_tt2000, esa_voltage, Species.PROTON + ) self.assertIs(response_grid, response_grid2, "cache miss") for species, mass_per_charge in [ (Species.ALPHA, constants.ALPHA_MASS_PER_CHARGE_M_P_PER_E), (Species.HELIUM_PLUS, constants.HE_PUI_PARTICLE_MASS_PER_CHARGE_M_P_PER_E), ]: - response_grid_species = response.get_response_grid(time_as_tt2000, esa_voltage, species) - self.assertAlmostEqual(response_grid_species.central_effective_area / response_grid.central_effective_area, 1.05) + response_grid_species = response.get_response_grid( + time_as_tt2000, esa_voltage, species + ) + self.assertAlmostEqual( + response_grid_species.central_effective_area + / response_grid.central_effective_area, + 1.05, + ) self.assertAlmostEqual( response_grid_species.central_speed / response_grid.central_speed, np.sqrt(constants.PROTON_MASS_PER_CHARGE_M_P_PER_E / mass_per_charge), - msg="central speed should be lower by the sqrt mass per charge ratio" + msg="central speed should be lower by the sqrt mass per charge ratio", ) for attr in ["sg_passband", "oa_passband", "azimuthal_transmission"]: - self.assertIs(getattr(response_grid, attr), getattr(response_grid_species, attr), msg="shared attribute mismatch") + self.assertIs( + getattr(response_grid, attr), + getattr(response_grid_species, attr), + msg="shared attribute mismatch", + ) if __name__ == "__main__": diff --git a/tests/swe/l3/science/test_models.py b/tests/swe/l3/science/test_models.py index 3cdb5fea7..a086ae03d 100644 --- a/tests/swe/l3/science/test_models.py +++ b/tests/swe/l3/science/test_models.py @@ -3,50 +3,106 @@ import numpy as np -from imap_l3_processing.swe.l3.models import SweL3Data, EPOCH_CDF_VAR_NAME, EPOCH_DELTA_CDF_VAR_NAME, \ - ENERGY_CDF_VAR_NAME, \ - ENERGY_DELTA_PLUS_CDF_VAR_NAME, ENERGY_DELTA_MINUS_CDF_VAR_NAME, PITCH_ANGLE_CDF_VAR_NAME, \ - PITCH_ANGLE_DELTA_CDF_VAR_NAME, PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_CDF_VAR_NAME, \ - PHASE_SPACE_DENSITY_1D_CDF_VAR_NAME, PHASE_SPACE_DENSITY_OUTWARD_CDF_VAR_NAME, \ - PHASE_SPACE_DENSITY_INWARD_CDF_VAR_NAME, \ - SPACECRAFT_POTENTIAL_CDF_VAR_NAME, CORE_HALO_BREAKPOINT_CDF_VAR_NAME, CORE_FIT_NUM_POINTS_CDF_VAR_NAME, \ - CHISQ_C_CDF_VAR_NAME, CHISQ_H_CDF_VAR_NAME, CORE_DENSITY_FIT_CDF_VAR_NAME, HALO_DENSITY_FIT_CDF_VAR_NAME, \ - CORE_T_PARALLEL_FIT_CDF_VAR_NAME, HALO_T_PARALLEL_FIT_CDF_VAR_NAME, CORE_T_PERPENDICULAR_FIT_CDF_VAR_NAME, \ - HALO_T_PERPENDICULAR_FIT_CDF_VAR_NAME, CORE_TEMPERATURE_PHI_RTN_FIT_CDF_VAR_NAME, \ - HALO_TEMPERATURE_PHI_RTN_FIT_CDF_VAR_NAME, CORE_TEMPERATURE_THETA_RTN_FIT_CDF_VAR_NAME, \ - HALO_TEMPERATURE_THETA_RTN_FIT_CDF_VAR_NAME, CORE_SPEED_FIT_CDF_VAR_NAME, HALO_SPEED_FIT_CDF_VAR_NAME, \ - CORE_VELOCITY_VECTOR_RTN_FIT_CDF_VAR_NAME, HALO_VELOCITY_VECTOR_RTN_FIT_CDF_VAR_NAME, \ - INTENSITY_BY_PITCH_ANGLE_CDF_VAR_NAME, INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME, \ - CORE_DENSITY_INTEGRATED_CDF_VAR_NAME, HALO_DENSITY_INTEGRATED_CDF_VAR_NAME, TOTAL_DENSITY_INTEGRATED_CDF_VAR_NAME, \ - CORE_SPEED_INTEGRATED_CDF_VAR_NAME, HALO_SPEED_INTEGRATED_CDF_VAR_NAME, TOTAL_SPEED_INTEGRATED_CDF_VAR_NAME, \ - CORE_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME, HALO_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME, \ - TOTAL_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME, CORE_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME, \ - CORE_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME, CORE_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME, \ - HALO_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME, HALO_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME, \ - HALO_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME, TOTAL_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME, \ - TOTAL_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME, TOTAL_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME, \ - CORE_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME, \ - CORE_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME, HALO_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME, \ - HALO_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME, TOTAL_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME, \ - TOTAL_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME, CORE_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME, \ - CORE_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, HALO_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME, \ - HALO_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, TOTAL_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME, \ - TOTAL_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, SweL3MomentData, CORE_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME, \ - CORE_T_PARALLEL_INTEGRATED_CDF_VAR_NAME, HALO_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME, \ - HALO_T_PARALLEL_INTEGRATED_CDF_VAR_NAME, TOTAL_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME, \ - TOTAL_T_PARALLEL_INTEGRATED_CDF_VAR_NAME, CORE_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME, \ - HALO_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME, TOTAL_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME, \ - PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME, GYROPHASE_DELTA_CDF_VAR_NAME, \ - GYROPHASE_BINS_CDF_VAR_NAME, INTENSITY_UNCERTAINTY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME, \ - INTENSITY_UNCERTAINTY_BY_PITCH_ANGLE_CDF_VAR_NAME, ENERGY_LABEL, PITCH_ANGLE_LABEL, GYROPHASE_LABEL, RTN_LABEL, \ - CORE_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME, \ - HALO_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME, TOTAL_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME, \ - CORE_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, HALO_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, \ - TOTAL_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, TENSOR_ID, SWE_FLAGS_VAR_NAME, \ - CORE_VELOCITY_VECTOR_RTN_FIT_LABEL, HALO_VELOCITY_VECTOR_RTN_FIT_LABEL, CORE_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL, \ - HALO_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL, TOTAL_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL, \ - CORE_TEMPERATURE_TENSOR_INTEGRATED_LABEL, HALO_TEMPERATURE_TENSOR_INTEGRATED_LABEL, \ - TOTAL_TEMPERATURE_TENSOR_INTEGRATED_LABEL +from imap_l3_processing.swe.l3.models import ( + SweL3Data, + EPOCH_CDF_VAR_NAME, + EPOCH_DELTA_CDF_VAR_NAME, + ENERGY_CDF_VAR_NAME, + ENERGY_DELTA_PLUS_CDF_VAR_NAME, + ENERGY_DELTA_MINUS_CDF_VAR_NAME, + PITCH_ANGLE_CDF_VAR_NAME, + PITCH_ANGLE_DELTA_CDF_VAR_NAME, + PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_CDF_VAR_NAME, + PHASE_SPACE_DENSITY_1D_CDF_VAR_NAME, + PHASE_SPACE_DENSITY_OUTWARD_CDF_VAR_NAME, + PHASE_SPACE_DENSITY_INWARD_CDF_VAR_NAME, + SPACECRAFT_POTENTIAL_CDF_VAR_NAME, + CORE_HALO_BREAKPOINT_CDF_VAR_NAME, + CORE_FIT_NUM_POINTS_CDF_VAR_NAME, + CHISQ_C_CDF_VAR_NAME, + CHISQ_H_CDF_VAR_NAME, + CORE_DENSITY_FIT_CDF_VAR_NAME, + HALO_DENSITY_FIT_CDF_VAR_NAME, + CORE_T_PARALLEL_FIT_CDF_VAR_NAME, + HALO_T_PARALLEL_FIT_CDF_VAR_NAME, + CORE_T_PERPENDICULAR_FIT_CDF_VAR_NAME, + HALO_T_PERPENDICULAR_FIT_CDF_VAR_NAME, + CORE_TEMPERATURE_PHI_RTN_FIT_CDF_VAR_NAME, + HALO_TEMPERATURE_PHI_RTN_FIT_CDF_VAR_NAME, + CORE_TEMPERATURE_THETA_RTN_FIT_CDF_VAR_NAME, + HALO_TEMPERATURE_THETA_RTN_FIT_CDF_VAR_NAME, + CORE_SPEED_FIT_CDF_VAR_NAME, + HALO_SPEED_FIT_CDF_VAR_NAME, + CORE_VELOCITY_VECTOR_RTN_FIT_CDF_VAR_NAME, + HALO_VELOCITY_VECTOR_RTN_FIT_CDF_VAR_NAME, + INTENSITY_BY_PITCH_ANGLE_CDF_VAR_NAME, + INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME, + CORE_DENSITY_INTEGRATED_CDF_VAR_NAME, + HALO_DENSITY_INTEGRATED_CDF_VAR_NAME, + TOTAL_DENSITY_INTEGRATED_CDF_VAR_NAME, + CORE_SPEED_INTEGRATED_CDF_VAR_NAME, + HALO_SPEED_INTEGRATED_CDF_VAR_NAME, + TOTAL_SPEED_INTEGRATED_CDF_VAR_NAME, + CORE_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME, + HALO_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME, + TOTAL_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME, + CORE_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME, + CORE_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME, + CORE_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME, + HALO_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME, + HALO_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME, + HALO_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME, + TOTAL_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME, + TOTAL_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME, + TOTAL_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME, + CORE_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME, + CORE_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME, + HALO_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME, + HALO_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME, + TOTAL_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME, + TOTAL_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME, + CORE_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME, + CORE_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, + HALO_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME, + HALO_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, + TOTAL_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME, + TOTAL_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, + SweL3MomentData, + CORE_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME, + CORE_T_PARALLEL_INTEGRATED_CDF_VAR_NAME, + HALO_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME, + HALO_T_PARALLEL_INTEGRATED_CDF_VAR_NAME, + TOTAL_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME, + TOTAL_T_PARALLEL_INTEGRATED_CDF_VAR_NAME, + CORE_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME, + HALO_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME, + TOTAL_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME, + PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME, + GYROPHASE_DELTA_CDF_VAR_NAME, + GYROPHASE_BINS_CDF_VAR_NAME, + INTENSITY_UNCERTAINTY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME, + INTENSITY_UNCERTAINTY_BY_PITCH_ANGLE_CDF_VAR_NAME, + ENERGY_LABEL, + PITCH_ANGLE_LABEL, + GYROPHASE_LABEL, + RTN_LABEL, + CORE_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME, + HALO_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME, + TOTAL_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME, + CORE_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, + HALO_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, + TOTAL_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, + TENSOR_ID, + SWE_FLAGS_VAR_NAME, + CORE_VELOCITY_VECTOR_RTN_FIT_LABEL, + HALO_VELOCITY_VECTOR_RTN_FIT_LABEL, + CORE_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL, + HALO_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL, + TOTAL_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL, + CORE_TEMPERATURE_TENSOR_INTEGRATED_LABEL, + HALO_TEMPERATURE_TENSOR_INTEGRATED_LABEL, + TOTAL_TEMPERATURE_TENSOR_INTEGRATED_LABEL, +) from tests.swapi.cdf_model_test_case import CdfModelTestCase @@ -157,112 +213,477 @@ def test_data_to_product_variables(self): variables = iter(variables) # @formatter:off - self.assert_variable_attributes(next(variables), sentinel.epoch, EPOCH_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.epoch_delta, EPOCH_DELTA_CDF_VAR_NAME) + self.assert_variable_attributes( + next(variables), sentinel.epoch, EPOCH_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), sentinel.epoch_delta, EPOCH_DELTA_CDF_VAR_NAME + ) self.assert_variable_attributes(next(variables), energy, ENERGY_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.energy_delta_plus, ENERGY_DELTA_PLUS_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.energy_delta_minus, ENERGY_DELTA_MINUS_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), pitch_angle, PITCH_ANGLE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.pitch_angle_delta, PITCH_ANGLE_DELTA_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), gyrophase_bins, GYROPHASE_BINS_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.gyrophase_delta, GYROPHASE_DELTA_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.psd_by_pitch_angle, PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.psd_by_pitch_angle_and_gyrophase, PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.energy_spectrum, PHASE_SPACE_DENSITY_1D_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.energy_spectrum_inbound, PHASE_SPACE_DENSITY_INWARD_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.energy_spectrum_outbound, PHASE_SPACE_DENSITY_OUTWARD_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.intensity_by_pitch_angle, INTENSITY_BY_PITCH_ANGLE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.intensity_by_pitch_angle_and_gyrophase, INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.intensity_uncertainty_by_pitch_angle, INTENSITY_UNCERTAINTY_BY_PITCH_ANGLE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.intensity_uncertainty_by_pitch_angle_and_gyrophase, INTENSITY_UNCERTAINTY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.spacecraft_potential, SPACECRAFT_POTENTIAL_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_halo_breakpoint, CORE_HALO_BREAKPOINT_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_fit_num_points, CORE_FIT_NUM_POINTS_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.chisq_c, CHISQ_C_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.chisq_h, CHISQ_H_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_density_fit, CORE_DENSITY_FIT_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.halo_density_fit, HALO_DENSITY_FIT_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_t_parallel_fit, CORE_T_PARALLEL_FIT_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.halo_t_parallel_fit, HALO_T_PARALLEL_FIT_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_t_perpendicular_fit, CORE_T_PERPENDICULAR_FIT_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.halo_t_perpendicular_fit, HALO_T_PERPENDICULAR_FIT_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_temperature_phi_rtn_fit, CORE_TEMPERATURE_PHI_RTN_FIT_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.halo_temperature_phi_rtn_fit, HALO_TEMPERATURE_PHI_RTN_FIT_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_temperature_theta_rtn_fit, CORE_TEMPERATURE_THETA_RTN_FIT_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.halo_temperature_theta_rtn_fit, HALO_TEMPERATURE_THETA_RTN_FIT_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_speed_fit, CORE_SPEED_FIT_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.halo_speed_fit, HALO_SPEED_FIT_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_velocity_vector_rtn_fit, CORE_VELOCITY_VECTOR_RTN_FIT_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.halo_velocity_vector_rtn_fit, HALO_VELOCITY_VECTOR_RTN_FIT_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_density_integrated, CORE_DENSITY_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.halo_density_integrated, HALO_DENSITY_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.total_density_integrated, TOTAL_DENSITY_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_speed_integrated, CORE_SPEED_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.halo_speed_integrated, HALO_SPEED_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.total_speed_integrated, TOTAL_SPEED_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_velocity_vector_rtn_integrated, CORE_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.halo_velocity_vector_rtn_integrated, HALO_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.total_velocity_vector_rtn_integrated, TOTAL_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_heat_flux_magnitude_integrated, CORE_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_heat_flux_theta_integrated, CORE_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_heat_flux_phi_integrated, CORE_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.halo_heat_flux_magnitude_integrated, HALO_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.halo_heat_flux_theta_integrated, HALO_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.halo_heat_flux_phi_integrated, HALO_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.total_heat_flux_magnitude_integrated, TOTAL_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.total_heat_flux_theta_integrated, TOTAL_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.total_heat_flux_phi_integrated, TOTAL_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_t_parallel_integrated, CORE_T_PARALLEL_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), [1,3], CORE_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), [2,4], CORE_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.halo_t_parallel_integrated, HALO_T_PARALLEL_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), [5,7], HALO_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), [6,8], HALO_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.total_t_parallel_integrated, TOTAL_T_PARALLEL_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), [9,11], TOTAL_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), [10,12], TOTAL_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_temperature_theta_rtn_integrated, CORE_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_temperature_phi_rtn_integrated, CORE_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.halo_temperature_theta_rtn_integrated, HALO_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.halo_temperature_phi_rtn_integrated, HALO_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.total_temperature_theta_rtn_integrated, TOTAL_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.total_temperature_phi_rtn_integrated, TOTAL_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_temperature_parallel_to_mag, CORE_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), [11,13], CORE_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), [12,14], CORE_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.halo_temperature_parallel_to_mag, HALO_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), [15,17], HALO_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), [16,18], HALO_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.total_temperature_parallel_to_mag, TOTAL_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), [19,21], TOTAL_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), [20,22], TOTAL_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.core_temperature_tensor_integrated, CORE_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.halo_temperature_tensor_integrated, HALO_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.total_temperature_tensor_integrated, TOTAL_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.swp_flags, SWE_FLAGS_VAR_NAME) - self.assert_variable_attributes(next(variables), sentinel.raw_1d_psd_rebinned, 'raw_1d_psd_rebinned') - self.assert_variable_attributes(next(variables), sentinel.raw_psd_by_theta_rebinned, 'raw_psd_by_theta_rebinned') - self.assert_variable_attributes(next(variables), sentinel.raw_psd_by_phi_rebinned, 'raw_psd_by_phi_rebinned') - self.assert_variable_attributes(next(variables), sentinel.inst_az, 'inst_az') - self.assert_variable_attributes(next(variables), sentinel.inst_el, 'inst_el') - self.assert_variable_attributes(next(variables), ["10.0 eV", "20.0 eV", "30.0 eV"], ENERGY_LABEL) - self.assert_variable_attributes(next(variables), ["PA 11", "PA 12", "PA 13", "PA 14", "PA 15"], PITCH_ANGLE_LABEL) - self.assert_variable_attributes(next(variables), ["Gyrophase 20", "Gyrophase 40", "Gyrophase 60", "Gyrophase 80"], GYROPHASE_LABEL) + self.assert_variable_attributes( + next(variables), sentinel.energy_delta_plus, ENERGY_DELTA_PLUS_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), + sentinel.energy_delta_minus, + ENERGY_DELTA_MINUS_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), pitch_angle, PITCH_ANGLE_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), sentinel.pitch_angle_delta, PITCH_ANGLE_DELTA_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), gyrophase_bins, GYROPHASE_BINS_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), sentinel.gyrophase_delta, GYROPHASE_DELTA_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), + sentinel.psd_by_pitch_angle, + PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.psd_by_pitch_angle_and_gyrophase, + PHASE_SPACE_DENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.energy_spectrum, + PHASE_SPACE_DENSITY_1D_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.energy_spectrum_inbound, + PHASE_SPACE_DENSITY_INWARD_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.energy_spectrum_outbound, + PHASE_SPACE_DENSITY_OUTWARD_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.intensity_by_pitch_angle, + INTENSITY_BY_PITCH_ANGLE_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.intensity_by_pitch_angle_and_gyrophase, + INTENSITY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.intensity_uncertainty_by_pitch_angle, + INTENSITY_UNCERTAINTY_BY_PITCH_ANGLE_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.intensity_uncertainty_by_pitch_angle_and_gyrophase, + INTENSITY_UNCERTAINTY_BY_PITCH_ANGLE_AND_GYROPHASE_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.spacecraft_potential, + SPACECRAFT_POTENTIAL_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.core_halo_breakpoint, + CORE_HALO_BREAKPOINT_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.core_fit_num_points, + CORE_FIT_NUM_POINTS_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), sentinel.chisq_c, CHISQ_C_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), sentinel.chisq_h, CHISQ_H_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), sentinel.core_density_fit, CORE_DENSITY_FIT_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), sentinel.halo_density_fit, HALO_DENSITY_FIT_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), + sentinel.core_t_parallel_fit, + CORE_T_PARALLEL_FIT_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.halo_t_parallel_fit, + HALO_T_PARALLEL_FIT_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.core_t_perpendicular_fit, + CORE_T_PERPENDICULAR_FIT_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.halo_t_perpendicular_fit, + HALO_T_PERPENDICULAR_FIT_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.core_temperature_phi_rtn_fit, + CORE_TEMPERATURE_PHI_RTN_FIT_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.halo_temperature_phi_rtn_fit, + HALO_TEMPERATURE_PHI_RTN_FIT_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.core_temperature_theta_rtn_fit, + CORE_TEMPERATURE_THETA_RTN_FIT_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.halo_temperature_theta_rtn_fit, + HALO_TEMPERATURE_THETA_RTN_FIT_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), sentinel.core_speed_fit, CORE_SPEED_FIT_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), sentinel.halo_speed_fit, HALO_SPEED_FIT_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), + sentinel.core_velocity_vector_rtn_fit, + CORE_VELOCITY_VECTOR_RTN_FIT_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.halo_velocity_vector_rtn_fit, + HALO_VELOCITY_VECTOR_RTN_FIT_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.core_density_integrated, + CORE_DENSITY_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.halo_density_integrated, + HALO_DENSITY_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.total_density_integrated, + TOTAL_DENSITY_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.core_speed_integrated, + CORE_SPEED_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.halo_speed_integrated, + HALO_SPEED_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.total_speed_integrated, + TOTAL_SPEED_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.core_velocity_vector_rtn_integrated, + CORE_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.halo_velocity_vector_rtn_integrated, + HALO_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.total_velocity_vector_rtn_integrated, + TOTAL_VELOCITY_VECTOR_RTN_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.core_heat_flux_magnitude_integrated, + CORE_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.core_heat_flux_theta_integrated, + CORE_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.core_heat_flux_phi_integrated, + CORE_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.halo_heat_flux_magnitude_integrated, + HALO_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.halo_heat_flux_theta_integrated, + HALO_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.halo_heat_flux_phi_integrated, + HALO_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.total_heat_flux_magnitude_integrated, + TOTAL_HEAT_FLUX_MAGNITUDE_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.total_heat_flux_theta_integrated, + TOTAL_HEAT_FLUX_THETA_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.total_heat_flux_phi_integrated, + TOTAL_HEAT_FLUX_PHI_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.core_t_parallel_integrated, + CORE_T_PARALLEL_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), [1, 3], CORE_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), [2, 4], CORE_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), + sentinel.halo_t_parallel_integrated, + HALO_T_PARALLEL_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), [5, 7], HALO_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), [6, 8], HALO_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), + sentinel.total_t_parallel_integrated, + TOTAL_T_PARALLEL_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), [9, 11], TOTAL_T_PERPENDICULAR_INTEGRATED_CDF_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), + [10, 12], + TOTAL_T_PERPENDICULAR_RATIO_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.core_temperature_theta_rtn_integrated, + CORE_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.core_temperature_phi_rtn_integrated, + CORE_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.halo_temperature_theta_rtn_integrated, + HALO_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.halo_temperature_phi_rtn_integrated, + HALO_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.total_temperature_theta_rtn_integrated, + TOTAL_TEMPERATURE_THETA_RTN_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.total_temperature_phi_rtn_integrated, + TOTAL_TEMPERATURE_PHI_RTN_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.core_temperature_parallel_to_mag, + CORE_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + [11, 13], + CORE_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + [12, 14], + CORE_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.halo_temperature_parallel_to_mag, + HALO_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + [15, 17], + HALO_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + [16, 18], + HALO_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.total_temperature_parallel_to_mag, + TOTAL_TEMPERATURE_PARALLEL_TO_MAG_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + [19, 21], + TOTAL_TEMPERATURE_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + [20, 22], + TOTAL_TEMPERATURE_RATIO_PERPENDICULAR_TO_MAG_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.core_temperature_tensor_integrated, + CORE_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.halo_temperature_tensor_integrated, + HALO_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), + sentinel.total_temperature_tensor_integrated, + TOTAL_TEMPERATURE_TENSOR_INTEGRATED_CDF_VAR_NAME, + ) + self.assert_variable_attributes( + next(variables), sentinel.swp_flags, SWE_FLAGS_VAR_NAME + ) + self.assert_variable_attributes( + next(variables), sentinel.raw_1d_psd_rebinned, "raw_1d_psd_rebinned" + ) + self.assert_variable_attributes( + next(variables), + sentinel.raw_psd_by_theta_rebinned, + "raw_psd_by_theta_rebinned", + ) + self.assert_variable_attributes( + next(variables), sentinel.raw_psd_by_phi_rebinned, "raw_psd_by_phi_rebinned" + ) + self.assert_variable_attributes(next(variables), sentinel.inst_az, "inst_az") + self.assert_variable_attributes(next(variables), sentinel.inst_el, "inst_el") + self.assert_variable_attributes( + next(variables), ["10.0 eV", "20.0 eV", "30.0 eV"], ENERGY_LABEL + ) + self.assert_variable_attributes( + next(variables), + ["PA 11", "PA 12", "PA 13", "PA 14", "PA 15"], + PITCH_ANGLE_LABEL, + ) + self.assert_variable_attributes( + next(variables), + ["Gyrophase 20", "Gyrophase 40", "Gyrophase 60", "Gyrophase 80"], + GYROPHASE_LABEL, + ) self.assert_variable_attributes(next(variables), ["R", "T", "N"], RTN_LABEL) - self.assert_variable_attributes(next(variables), ["Core v fit R", "Core v fit T", "Core v fit N"], CORE_VELOCITY_VECTOR_RTN_FIT_LABEL) - self.assert_variable_attributes(next(variables), ["Halo v fit R", "Halo v fit T", "Halo v fit N"], HALO_VELOCITY_VECTOR_RTN_FIT_LABEL) - self.assert_variable_attributes(next(variables), ["Core v int. R", "Core v int. T", "Core v int. N"], CORE_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL) - self.assert_variable_attributes(next(variables), ["Halo v int. R", "Halo v int. T", "Halo v int. N"], HALO_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL) - self.assert_variable_attributes(next(variables), ["Total v int. R", "Total v int. T", "Total v int. N"], TOTAL_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL) - self.assert_variable_attributes(next(variables), ["Core T_XX", "Core T_XY", "Core T_YY", "Core T_XZ", "Core T_YZ", "Core T_ZZ"], CORE_TEMPERATURE_TENSOR_INTEGRATED_LABEL) - self.assert_variable_attributes(next(variables), ["Halo T_XX", "Halo T_XY", "Halo T_YY", "Halo T_XZ", "Halo T_YZ", "Halo T_ZZ"], HALO_TEMPERATURE_TENSOR_INTEGRATED_LABEL) - self.assert_variable_attributes(next(variables), ["Total T_XX", "Total T_XY", "Total T_YY", "Total T_XZ", "Total T_YZ", "Total T_ZZ"], TOTAL_TEMPERATURE_TENSOR_INTEGRATED_LABEL) - self.assert_variable_attributes(next(variables), sentinel.inst_az_label, 'inst_az_label') - self.assert_variable_attributes(next(variables), sentinel.inst_el_label, 'inst_el_label') - self.assert_variable_attributes(next(variables),[1,2,3,4,5, 6], TENSOR_ID) + self.assert_variable_attributes( + next(variables), + ["Core v fit R", "Core v fit T", "Core v fit N"], + CORE_VELOCITY_VECTOR_RTN_FIT_LABEL, + ) + self.assert_variable_attributes( + next(variables), + ["Halo v fit R", "Halo v fit T", "Halo v fit N"], + HALO_VELOCITY_VECTOR_RTN_FIT_LABEL, + ) + self.assert_variable_attributes( + next(variables), + ["Core v int. R", "Core v int. T", "Core v int. N"], + CORE_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL, + ) + self.assert_variable_attributes( + next(variables), + ["Halo v int. R", "Halo v int. T", "Halo v int. N"], + HALO_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL, + ) + self.assert_variable_attributes( + next(variables), + ["Total v int. R", "Total v int. T", "Total v int. N"], + TOTAL_VELOCITY_VECTOR_RTN_INTEGRATED_LABEL, + ) + self.assert_variable_attributes( + next(variables), + [ + "Core T_XX", + "Core T_XY", + "Core T_YY", + "Core T_XZ", + "Core T_YZ", + "Core T_ZZ", + ], + CORE_TEMPERATURE_TENSOR_INTEGRATED_LABEL, + ) + self.assert_variable_attributes( + next(variables), + [ + "Halo T_XX", + "Halo T_XY", + "Halo T_YY", + "Halo T_XZ", + "Halo T_YZ", + "Halo T_ZZ", + ], + HALO_TEMPERATURE_TENSOR_INTEGRATED_LABEL, + ) + self.assert_variable_attributes( + next(variables), + [ + "Total T_XX", + "Total T_XY", + "Total T_YY", + "Total T_XZ", + "Total T_YZ", + "Total T_ZZ", + ], + TOTAL_TEMPERATURE_TENSOR_INTEGRATED_LABEL, + ) + self.assert_variable_attributes( + next(variables), sentinel.inst_az_label, "inst_az_label" + ) + self.assert_variable_attributes( + next(variables), sentinel.inst_el_label, "inst_el_label" + ) + self.assert_variable_attributes(next(variables), [1, 2, 3, 4, 5, 6], TENSOR_ID) self.assertEqual([], list(variables), "unexpected variable found") -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/swe/l3/science/test_moments_calculation.py b/tests/swe/l3/science/test_moments_calculation.py index a7dc65d4b..8cb08ab13 100644 --- a/tests/swe/l3/science/test_moments_calculation.py +++ b/tests/swe/l3/science/test_moments_calculation.py @@ -10,176 +10,324 @@ from imap_l3_processing.predicted_ephemeris_tracker import PredictedEphemerisTracker from imap_l3_processing.swe.l3.science import moment_calculations -from imap_l3_processing.swe.l3.science.moment_calculations import compute_maxwellian_weight_factors, \ - filter_and_flatten_regress_parameters, regress, calculate_fit_temperature_density_velocity, rotate_temperature, \ - apply_rotation_matrix, Moments, halotrunc, compute_density_scale, core_fit_moments_retrying_on_failure, \ - halo_fit_moments_retrying_on_failure, scale_halo_density, rotate_vector_to_rtn_spherical_coordinates, \ - calculate_primary_eigenvector, rotation_matrix_builder, rotate_temperature_tensor_to_mag, MomentFitResults, \ - rotate_rtn_vectors_to_dps +from imap_l3_processing.swe.l3.science.moment_calculations import ( + compute_maxwellian_weight_factors, + filter_and_flatten_regress_parameters, + regress, + calculate_fit_temperature_density_velocity, + rotate_temperature, + apply_rotation_matrix, + Moments, + halotrunc, + compute_density_scale, + core_fit_moments_retrying_on_failure, + halo_fit_moments_retrying_on_failure, + scale_halo_density, + rotate_vector_to_rtn_spherical_coordinates, + calculate_primary_eigenvector, + rotation_matrix_builder, + rotate_temperature_tensor_to_mag, + MomentFitResults, + rotate_rtn_vectors_to_dps, +) from tests.test_helpers import create_dataclass_mock from tests.test_helpers import get_test_data_path from imap_l3_processing.swe.swe_processor import check_temperature_outlier_flag from imap_l3_processing.swe.quality_flags import SweL3Flags -MODULE = 'imap_l3_processing.swe.l3.science.moment_calculations' +MODULE = "imap_l3_processing.swe.l3.science.moment_calculations" -class TestMomentsCalculation(unittest.TestCase): - @patch(f'{MODULE}.halotrunc') - @patch(f'{MODULE}.regress') - @patch(f'{MODULE}.calculate_fit_temperature_density_velocity') - @patch(f'{MODULE}.filter_and_flatten_regress_parameters') - def test_halo_fit_moments_retrying_on_failure(self, mock_filter_and_flatten_regress_parameters, - mock_calculate_fit_temp_dens_velocity, mock_regress, mock_halotrunc): +class TestMomentsCalculation(unittest.TestCase): + @patch(f"{MODULE}.halotrunc") + @patch(f"{MODULE}.regress") + @patch(f"{MODULE}.calculate_fit_temperature_density_velocity") + @patch(f"{MODULE}.filter_and_flatten_regress_parameters") + def test_halo_fit_moments_retrying_on_failure( + self, + mock_filter_and_flatten_regress_parameters, + mock_calculate_fit_temp_dens_velocity, + mock_regress, + mock_halotrunc, + ): density_history = np.array([100, 89, 72]) negative_density_value = -1 - density_greater_than_rolling_average_of_history = np.average(density_history) * 1.35 + 1 + density_greater_than_rolling_average_of_history = ( + np.average(density_history) * 1.35 + 1 + ) valid_density = 13 moments_1 = create_dataclass_mock(Moments, density=sentinel.density1) moments_2 = create_dataclass_mock(Moments, density=sentinel.density2) - moments_3_with_valid_density = create_dataclass_mock(Moments, density=sentinel.density3) + moments_3_with_valid_density = create_dataclass_mock( + Moments, density=sentinel.density3 + ) mock_calculate_fit_temp_dens_velocity.side_effect = [ moments_1, moments_2, - moments_3_with_valid_density + moments_3_with_valid_density, ] mock_filter_and_flatten_regress_parameters.side_effect = [ - (sentinel.filtered_velocity_vectors, sentinel.filtered_weights, sentinel.filtered_yreg), - (sentinel.filtered_velocity_vectors_1, sentinel.filtered_weights_1, sentinel.filtered_yreg_1), - (sentinel.filtered_velocity_vectors_1, sentinel.filtered_weights_1, sentinel.filtered_yreg_1) + ( + sentinel.filtered_velocity_vectors, + sentinel.filtered_weights, + sentinel.filtered_yreg, + ), + ( + sentinel.filtered_velocity_vectors_1, + sentinel.filtered_weights_1, + sentinel.filtered_yreg_1, + ), + ( + sentinel.filtered_velocity_vectors_1, + sentinel.filtered_weights_1, + sentinel.filtered_yreg_1, + ), ] mock_regress.side_effect = [ (sentinel.fit_function_1, sentinel.chisq_1), (sentinel.fit_function_2, sentinel.chisq_2), - (sentinel.fit_function_3, sentinel.chisq_3) + (sentinel.fit_function_3, sentinel.chisq_3), ] mock_halotrunc.side_effect = [ negative_density_value, density_greater_than_rolling_average_of_history, - valid_density + valid_density, ] - moment_fit_result = halo_fit_moments_retrying_on_failure(sentinel.corrected_energy_bins, - sentinel.velocity_vectors, - sentinel.phase_space_density, - sentinel.weights, - 0, - 8, - density_history, - sentinel.spacecraft_potential, - sentinel.halo_core_breakpoint) - - mock_filter_and_flatten_regress_parameters.assert_has_calls([ - call(sentinel.corrected_energy_bins, sentinel.velocity_vectors, sentinel.phase_space_density, - sentinel.weights, 0, 8), - call(sentinel.corrected_energy_bins, sentinel.velocity_vectors, sentinel.phase_space_density, - sentinel.weights, 0, 7), - call(sentinel.corrected_energy_bins, sentinel.velocity_vectors, sentinel.phase_space_density, - sentinel.weights, 0, 6) - ]) - - mock_regress.assert_has_calls([ - call(sentinel.filtered_velocity_vectors, sentinel.filtered_weights, sentinel.filtered_yreg), - call(sentinel.filtered_velocity_vectors_1, sentinel.filtered_weights_1, sentinel.filtered_yreg_1), - call(sentinel.filtered_velocity_vectors_1, sentinel.filtered_weights_1, sentinel.filtered_yreg_1) - ]) - - mock_calculate_fit_temp_dens_velocity.assert_has_calls([ - call(sentinel.fit_function_1), - call(sentinel.fit_function_2), - call(sentinel.fit_function_3), - ]) - - mock_halotrunc.assert_has_calls([ - call(moments_1, sentinel.halo_core_breakpoint, sentinel.spacecraft_potential, ), - call(moments_2, sentinel.halo_core_breakpoint, sentinel.spacecraft_potential, ), - call(moments_3_with_valid_density, sentinel.halo_core_breakpoint, sentinel.spacecraft_potential, ) - ]) + moment_fit_result = halo_fit_moments_retrying_on_failure( + sentinel.corrected_energy_bins, + sentinel.velocity_vectors, + sentinel.phase_space_density, + sentinel.weights, + 0, + 8, + density_history, + sentinel.spacecraft_potential, + sentinel.halo_core_breakpoint, + ) + + mock_filter_and_flatten_regress_parameters.assert_has_calls( + [ + call( + sentinel.corrected_energy_bins, + sentinel.velocity_vectors, + sentinel.phase_space_density, + sentinel.weights, + 0, + 8, + ), + call( + sentinel.corrected_energy_bins, + sentinel.velocity_vectors, + sentinel.phase_space_density, + sentinel.weights, + 0, + 7, + ), + call( + sentinel.corrected_energy_bins, + sentinel.velocity_vectors, + sentinel.phase_space_density, + sentinel.weights, + 0, + 6, + ), + ] + ) + + mock_regress.assert_has_calls( + [ + call( + sentinel.filtered_velocity_vectors, + sentinel.filtered_weights, + sentinel.filtered_yreg, + ), + call( + sentinel.filtered_velocity_vectors_1, + sentinel.filtered_weights_1, + sentinel.filtered_yreg_1, + ), + call( + sentinel.filtered_velocity_vectors_1, + sentinel.filtered_weights_1, + sentinel.filtered_yreg_1, + ), + ] + ) + + mock_calculate_fit_temp_dens_velocity.assert_has_calls( + [ + call(sentinel.fit_function_1), + call(sentinel.fit_function_2), + call(sentinel.fit_function_3), + ] + ) + + mock_halotrunc.assert_has_calls( + [ + call( + moments_1, + sentinel.halo_core_breakpoint, + sentinel.spacecraft_potential, + ), + call( + moments_2, + sentinel.halo_core_breakpoint, + sentinel.spacecraft_potential, + ), + call( + moments_3_with_valid_density, + sentinel.halo_core_breakpoint, + sentinel.spacecraft_potential, + ), + ] + ) self.assertIs(moments_3_with_valid_density, moment_fit_result.moments) self.assertIs(sentinel.fit_function_3, moment_fit_result.regress_result) self.assertEqual(sentinel.chisq_3, moment_fit_result.chisq) self.assertEqual(6, moment_fit_result.number_of_points) - @patch(f'{MODULE}.regress') - @patch(f'{MODULE}.calculate_fit_temperature_density_velocity') - @patch(f'{MODULE}.filter_and_flatten_regress_parameters') - def test_fit_moments_retrying_on_failure(self, mock_filter_and_flatten_regress_parameters, - mock_calculate_fit_temp_dens_velocity, mock_regress): + @patch(f"{MODULE}.regress") + @patch(f"{MODULE}.calculate_fit_temperature_density_velocity") + @patch(f"{MODULE}.filter_and_flatten_regress_parameters") + def test_fit_moments_retrying_on_failure( + self, + mock_filter_and_flatten_regress_parameters, + mock_calculate_fit_temp_dens_velocity, + mock_regress, + ): density_history = np.array([100, 89, 72]) negative_density_value = -1 - density_greater_than_rolling_average_of_history = np.average(density_history) * 1.85 + 1 + density_greater_than_rolling_average_of_history = ( + np.average(density_history) * 1.85 + 1 + ) valid_density = 13 - moments_with_valid_density = create_dataclass_mock(Moments, density=valid_density) + moments_with_valid_density = create_dataclass_mock( + Moments, density=valid_density + ) mock_calculate_fit_temp_dens_velocity.side_effect = [ create_dataclass_mock(Moments, density=negative_density_value), - create_dataclass_mock(Moments, density=density_greater_than_rolling_average_of_history), - moments_with_valid_density + create_dataclass_mock( + Moments, density=density_greater_than_rolling_average_of_history + ), + moments_with_valid_density, ] mock_filter_and_flatten_regress_parameters.side_effect = [ - (sentinel.filtered_velocity_vectors, sentinel.filtered_weights, sentinel.filtered_yreg), - (sentinel.filtered_velocity_vectors_1, sentinel.filtered_weights_1, sentinel.filtered_yreg_1), - (sentinel.filtered_velocity_vectors_1, sentinel.filtered_weights_1, sentinel.filtered_yreg_1) + ( + sentinel.filtered_velocity_vectors, + sentinel.filtered_weights, + sentinel.filtered_yreg, + ), + ( + sentinel.filtered_velocity_vectors_1, + sentinel.filtered_weights_1, + sentinel.filtered_yreg_1, + ), + ( + sentinel.filtered_velocity_vectors_1, + sentinel.filtered_weights_1, + sentinel.filtered_yreg_1, + ), ] mock_regress.side_effect = [ (sentinel.fit_function_1, sentinel.chisq_1), (sentinel.fit_function_2, sentinel.chisq_2), - (sentinel.fit_function_3, sentinel.chisq_3) + (sentinel.fit_function_3, sentinel.chisq_3), ] - moment_fit_result = core_fit_moments_retrying_on_failure(sentinel.corrected_energy_bins, - sentinel.velocity_vectors, - sentinel.phase_space_density, - sentinel.weights, - 0, - 8, - density_history) - - mock_filter_and_flatten_regress_parameters.assert_has_calls([ - call(sentinel.corrected_energy_bins, sentinel.velocity_vectors, sentinel.phase_space_density, - sentinel.weights, 0, 8), - call(sentinel.corrected_energy_bins, sentinel.velocity_vectors, sentinel.phase_space_density, - sentinel.weights, 0, 7), - call(sentinel.corrected_energy_bins, sentinel.velocity_vectors, sentinel.phase_space_density, - sentinel.weights, 0, 6) - ]) - - mock_regress.assert_has_calls([ - call(sentinel.filtered_velocity_vectors, sentinel.filtered_weights, sentinel.filtered_yreg), - call(sentinel.filtered_velocity_vectors_1, sentinel.filtered_weights_1, sentinel.filtered_yreg_1), - call(sentinel.filtered_velocity_vectors_1, sentinel.filtered_weights_1, sentinel.filtered_yreg_1) - ]) - - mock_calculate_fit_temp_dens_velocity.assert_has_calls([ - call(sentinel.fit_function_1), - call(sentinel.fit_function_2), - call(sentinel.fit_function_3), - ]) + moment_fit_result = core_fit_moments_retrying_on_failure( + sentinel.corrected_energy_bins, + sentinel.velocity_vectors, + sentinel.phase_space_density, + sentinel.weights, + 0, + 8, + density_history, + ) + + mock_filter_and_flatten_regress_parameters.assert_has_calls( + [ + call( + sentinel.corrected_energy_bins, + sentinel.velocity_vectors, + sentinel.phase_space_density, + sentinel.weights, + 0, + 8, + ), + call( + sentinel.corrected_energy_bins, + sentinel.velocity_vectors, + sentinel.phase_space_density, + sentinel.weights, + 0, + 7, + ), + call( + sentinel.corrected_energy_bins, + sentinel.velocity_vectors, + sentinel.phase_space_density, + sentinel.weights, + 0, + 6, + ), + ] + ) + + mock_regress.assert_has_calls( + [ + call( + sentinel.filtered_velocity_vectors, + sentinel.filtered_weights, + sentinel.filtered_yreg, + ), + call( + sentinel.filtered_velocity_vectors_1, + sentinel.filtered_weights_1, + sentinel.filtered_yreg_1, + ), + call( + sentinel.filtered_velocity_vectors_1, + sentinel.filtered_weights_1, + sentinel.filtered_yreg_1, + ), + ] + ) + + mock_calculate_fit_temp_dens_velocity.assert_has_calls( + [ + call(sentinel.fit_function_1), + call(sentinel.fit_function_2), + call(sentinel.fit_function_3), + ] + ) self.assertIs(moments_with_valid_density, moment_fit_result.moments) self.assertIs(sentinel.fit_function_3, moment_fit_result.regress_result) self.assertEqual(sentinel.chisq_3, moment_fit_result.chisq) self.assertEqual(6, moment_fit_result.number_of_points) - @patch(f'{MODULE}.regress') - @patch(f'{MODULE}.calculate_fit_temperature_density_velocity') - @patch(f'{MODULE}.filter_and_flatten_regress_parameters') + @patch(f"{MODULE}.regress") + @patch(f"{MODULE}.calculate_fit_temperature_density_velocity") + @patch(f"{MODULE}.filter_and_flatten_regress_parameters") def test_fit_moments_retrying_on_failure_should_stop_retrying_with_few_energies_and_return_none_if_negative_density( - self, - mock_filter_and_flatten_regress_parameters, - mock_calculate_fit_temp_dens_velocity, - mock_regress): + self, + mock_filter_and_flatten_regress_parameters, + mock_calculate_fit_temp_dens_velocity, + mock_regress, + ): density_history = np.array([100, 89, 72]) negative_density_value = -1 @@ -189,42 +337,69 @@ def test_fit_moments_retrying_on_failure_should_stop_retrying_with_few_energies_ ] mock_filter_and_flatten_regress_parameters.side_effect = [ - (sentinel.filtered_velocity_vectors, sentinel.filtered_weights, sentinel.filtered_yreg) + ( + sentinel.filtered_velocity_vectors, + sentinel.filtered_weights, + sentinel.filtered_yreg, + ) ] mock_regress.side_effect = [ (sentinel.fit_function_1, sentinel.chisq_1), ] - self.assertIsNone(core_fit_moments_retrying_on_failure(sentinel.corrected_energy_bins, - sentinel.velocity_vectors, - sentinel.phase_space_density, - sentinel.weights, - 0, - 3, - density_history)) - - mock_filter_and_flatten_regress_parameters.assert_has_calls([( - call(sentinel.corrected_energy_bins, sentinel.velocity_vectors, sentinel.phase_space_density, - sentinel.weights, 0, 3, ) - )]) - - mock_regress.assert_has_calls([ - call(sentinel.filtered_velocity_vectors, sentinel.filtered_weights, sentinel.filtered_yreg), - ]) - - mock_calculate_fit_temp_dens_velocity.assert_has_calls([ - call(sentinel.fit_function_1), - ]) - - @patch(f'{MODULE}.regress') - @patch(f'{MODULE}.calculate_fit_temperature_density_velocity') - @patch(f'{MODULE}.filter_and_flatten_regress_parameters') + self.assertIsNone( + core_fit_moments_retrying_on_failure( + sentinel.corrected_energy_bins, + sentinel.velocity_vectors, + sentinel.phase_space_density, + sentinel.weights, + 0, + 3, + density_history, + ) + ) + + mock_filter_and_flatten_regress_parameters.assert_has_calls( + [ + ( + call( + sentinel.corrected_energy_bins, + sentinel.velocity_vectors, + sentinel.phase_space_density, + sentinel.weights, + 0, + 3, + ) + ) + ] + ) + + mock_regress.assert_has_calls( + [ + call( + sentinel.filtered_velocity_vectors, + sentinel.filtered_weights, + sentinel.filtered_yreg, + ), + ] + ) + + mock_calculate_fit_temp_dens_velocity.assert_has_calls( + [ + call(sentinel.fit_function_1), + ] + ) + + @patch(f"{MODULE}.regress") + @patch(f"{MODULE}.calculate_fit_temperature_density_velocity") + @patch(f"{MODULE}.filter_and_flatten_regress_parameters") def test_fit_moments_retrying_on_failure_should_stop_retrying_with_few_energies_but_return_results_if_high_density( - self, - mock_filter_and_flatten_regress_parameters, - mock_calculate_fit_temp_dens_velocity, - mock_regress): + self, + mock_filter_and_flatten_regress_parameters, + mock_calculate_fit_temp_dens_velocity, + mock_regress, + ): density_history = np.array([100, 89, 72]) high_density_value = 200 @@ -235,44 +410,69 @@ def test_fit_moments_retrying_on_failure_should_stop_retrying_with_few_energies_ ] mock_filter_and_flatten_regress_parameters.side_effect = [ - (sentinel.filtered_velocity_vectors, sentinel.filtered_weights, sentinel.filtered_yreg) + ( + sentinel.filtered_velocity_vectors, + sentinel.filtered_weights, + sentinel.filtered_yreg, + ) ] mock_regress.side_effect = [ (sentinel.fit_function_1, sentinel.chisq_1), ] - result = core_fit_moments_retrying_on_failure(sentinel.corrected_energy_bins, - sentinel.velocity_vectors, - sentinel.phase_space_density, - sentinel.weights, - 0, - 3, - density_history) + result = core_fit_moments_retrying_on_failure( + sentinel.corrected_energy_bins, + sentinel.velocity_vectors, + sentinel.phase_space_density, + sentinel.weights, + 0, + 3, + density_history, + ) self.assertIsInstance(result, MomentFitResults) self.assertIs(calculated_moments, result.moments) - mock_filter_and_flatten_regress_parameters.assert_has_calls([( - call(sentinel.corrected_energy_bins, sentinel.velocity_vectors, sentinel.phase_space_density, - sentinel.weights, 0, 3, ) - )]) + mock_filter_and_flatten_regress_parameters.assert_has_calls( + [ + ( + call( + sentinel.corrected_energy_bins, + sentinel.velocity_vectors, + sentinel.phase_space_density, + sentinel.weights, + 0, + 3, + ) + ) + ] + ) - mock_regress.assert_has_calls([ - call(sentinel.filtered_velocity_vectors, sentinel.filtered_weights, sentinel.filtered_yreg), - ]) + mock_regress.assert_has_calls( + [ + call( + sentinel.filtered_velocity_vectors, + sentinel.filtered_weights, + sentinel.filtered_yreg, + ), + ] + ) - mock_calculate_fit_temp_dens_velocity.assert_has_calls([ - call(sentinel.fit_function_1), - ]) + mock_calculate_fit_temp_dens_velocity.assert_has_calls( + [ + call(sentinel.fit_function_1), + ] + ) - @patch(f'{MODULE}.regress') - @patch(f'{MODULE}.calculate_fit_temperature_density_velocity') - @patch(f'{MODULE}.filter_and_flatten_regress_parameters') + @patch(f"{MODULE}.regress") + @patch(f"{MODULE}.calculate_fit_temperature_density_velocity") + @patch(f"{MODULE}.filter_and_flatten_regress_parameters") def test_fit_moments_retrying_on_failure_should_stop_retrying_with_few_energies_and_return_none_if_no_density( - self, - mock_filter_and_flatten_regress_parameters, - mock_calculate_fit_temp_dens_velocity, - mock_regress): + self, + mock_filter_and_flatten_regress_parameters, + mock_calculate_fit_temp_dens_velocity, + mock_regress, + ): density_history = np.array([100, 89, 72]) high_density_value = 200 @@ -283,71 +483,107 @@ def test_fit_moments_retrying_on_failure_should_stop_retrying_with_few_energies_ ] mock_filter_and_flatten_regress_parameters.side_effect = [ - (sentinel.filtered_velocity_vectors, sentinel.filtered_weights, sentinel.filtered_yreg) + ( + sentinel.filtered_velocity_vectors, + sentinel.filtered_weights, + sentinel.filtered_yreg, + ) ] mock_regress.side_effect = [ (sentinel.fit_function_1, sentinel.chisq_1), ] - result = core_fit_moments_retrying_on_failure(sentinel.corrected_energy_bins, - sentinel.velocity_vectors, - sentinel.phase_space_density, - sentinel.weights, - 0, - 3, - density_history) + result = core_fit_moments_retrying_on_failure( + sentinel.corrected_energy_bins, + sentinel.velocity_vectors, + sentinel.phase_space_density, + sentinel.weights, + 0, + 3, + density_history, + ) self.assertIsNone(result) def test_compute_maxwellian_weight_factors_reproduces_heritage_results(self): counts = np.array([[[536.0, 20000, 536.0], [1.2, 3072.0000001359296, 1.2]]]) - acquisition_duration = np.array([[80000., 40000.]]) + acquisition_duration = np.array([[80000.0, 40000.0]]) count_rates = counts / acquisition_duration[:, :, np.newaxis] - weight_factors = compute_maxwellian_weight_factors(count_rates, acquisition_duration) + weight_factors = compute_maxwellian_weight_factors( + count_rates, acquisition_duration + ) first_weight = np.sqrt(21.25 + 536) / 536 second_weight = np.sqrt(87381.25 + 20000.0) / 20000 third_weight = moment_calculations.MINIMUM_WEIGHT fourth_weight = np.sqrt(341.25 + 3072) / 3072 - np.testing.assert_array_almost_equal(weight_factors, - np.array([[[first_weight, second_weight, first_weight], - [third_weight, fourth_weight, third_weight]]])) + np.testing.assert_array_almost_equal( + weight_factors, + np.array( + [ + [ + [first_weight, second_weight, first_weight], + [third_weight, fourth_weight, third_weight], + ] + ] + ), + ) def test_regress_reproduces_heritage_results_given_all_test_data(self): - velocity_vectors = np.loadtxt(get_test_data_path("swe/fake_velocity_vectors.csv"), delimiter=",", - dtype=np.float64) - weights = np.loadtxt(get_test_data_path("swe/fake_weights.csv"), delimiter=",", dtype=np.float64) - yreg = np.loadtxt(get_test_data_path("swe/fake_yreg.csv"), delimiter=",", dtype=np.float64) + velocity_vectors = np.loadtxt( + get_test_data_path("swe/fake_velocity_vectors.csv"), + delimiter=",", + dtype=np.float64, + ) + weights = np.loadtxt( + get_test_data_path("swe/fake_weights.csv"), delimiter=",", dtype=np.float64 + ) + yreg = np.loadtxt( + get_test_data_path("swe/fake_yreg.csv"), delimiter=",", dtype=np.float64 + ) regression_values, chisq = regress(velocity_vectors, weights, yreg) - np.testing.assert_array_almost_equal(regression_values, - [59277.743670, - 57999.454287, - 58812.363965, - 1073.651710, - -185.816917, - 455.279162, - -1409352.589002, - 2485621.191362, - -7552247.685865, - -0.281777]) + np.testing.assert_array_almost_equal( + regression_values, + [ + 59277.743670, + 57999.454287, + 58812.363965, + 1073.651710, + -185.816917, + 455.279162, + -1409352.589002, + 2485621.191362, + -7552247.685865, + -0.281777, + ], + ) self.assertAlmostEqual(75398.120454, chisq, places=6) - def test_calculate_fit_temperature_density_velocity_is_consistent_with_heritage_on_full_data(self): - regress_output_of_full_fake_data = np.array([59277.743670, - 57999.454287, - 58812.363965, - 1073.651710, - -185.816917, - 455.279162, - -1409352.589002, - 2485621.191362, - -7552247.685865, - -0.281777], dtype=np.float64) - moments = calculate_fit_temperature_density_velocity(regress_output_of_full_fake_data) + def test_calculate_fit_temperature_density_velocity_is_consistent_with_heritage_on_full_data( + self, + ): + regress_output_of_full_fake_data = np.array( + [ + 59277.743670, + 57999.454287, + 58812.363965, + 1073.651710, + -185.816917, + 455.279162, + -1409352.589002, + 2485621.191362, + -7552247.685865, + -0.281777, + ], + dtype=np.float64, + ) + moments = calculate_fit_temperature_density_velocity( + regress_output_of_full_fake_data + ) self.assertAlmostEqual(1.958299, moments.alpha, places=5) self.assertAlmostEqual(1.729684, moments.beta, places=5) @@ -361,40 +597,54 @@ def test_calculate_fit_temperature_density_velocity_is_consistent_with_heritage_ def test_filter_and_flatten_regress_parameters(self): corrected_energy_bins = np.array([-1, 0, 3, 4.5, 5]) - phase_space_density = np.array([ - [[1, 2], [2, 3]], - [[5, 6], [6, 7]], - [[3, 1e-36], [0, 0]], - [[10, 11], [0, 12]], - [[21, 22], [23, 24]], - ]) - - weights = np.array([ - [[1, 2], [2, 3]], - [[5, 6], [6, 7]], - [[3, 1e-36], [0, 0]], - [[10, 11], [0, 12]], - [[21, 22], [23, 24]], - ]) - - velocity_vectors = np.array([ - [[[1, 0, 0], [1, 0, 0]], [[2, 0, 0], [2, 0, 0]]], - [[[5, 0, 0], [5, 0, 0]], [[6, 0, 0], [6, 0, 0]]], - [[[3, 0, 0], [4, 0, 0]], [[0, 0, 0], [0, 0, 0]]], - [[[10, 0, 0], [10, 0, 0]], [[0, 0, 0], [0, 0, 0]]], - [[[20, 0, 0], [8, 0, 0]], [[12, 0, 0], [23, 0, 0]]], - ]) + phase_space_density = np.array( + [ + [[1, 2], [2, 3]], + [[5, 6], [6, 7]], + [[3, 1e-36], [0, 0]], + [[10, 11], [0, 12]], + [[21, 22], [23, 24]], + ] + ) + + weights = np.array( + [ + [[1, 2], [2, 3]], + [[5, 6], [6, 7]], + [[3, 1e-36], [0, 0]], + [[10, 11], [0, 12]], + [[21, 22], [23, 24]], + ] + ) + + velocity_vectors = np.array( + [ + [[[1, 0, 0], [1, 0, 0]], [[2, 0, 0], [2, 0, 0]]], + [[[5, 0, 0], [5, 0, 0]], [[6, 0, 0], [6, 0, 0]]], + [[[3, 0, 0], [4, 0, 0]], [[0, 0, 0], [0, 0, 0]]], + [[[10, 0, 0], [10, 0, 0]], [[0, 0, 0], [0, 0, 0]]], + [[[20, 0, 0], [8, 0, 0]], [[12, 0, 0], [23, 0, 0]]], + ] + ) core_breakpoint_index = 1 core_halo_breakpoint_index = 4 - vectors, actual_weights, yreg = filter_and_flatten_regress_parameters(corrected_energy_bins, velocity_vectors, - phase_space_density, weights, - core_breakpoint_index, - core_halo_breakpoint_index) + vectors, actual_weights, yreg = filter_and_flatten_regress_parameters( + corrected_energy_bins, + velocity_vectors, + phase_space_density, + weights, + core_breakpoint_index, + core_halo_breakpoint_index, + ) - np.testing.assert_array_equal(vectors, [[3, 0, 0], [4, 0, 0], [10, 0, 0], [10, 0, 0], [0, 0, 0]]) + np.testing.assert_array_equal( + vectors, [[3, 0, 0], [4, 0, 0], [10, 0, 0], [10, 0, 0], [0, 0, 0]] + ) np.testing.assert_array_equal(actual_weights, [3, 1e-36, 10, 11, 12]) - np.testing.assert_array_equal(yreg, [np.log(3), -80.6, np.log(10), np.log(11), np.log(12)]) + np.testing.assert_array_equal( + yreg, [np.log(3), -80.6, np.log(10), np.log(11), np.log(12)] + ) def test_rotate_dps_vector_to_rtn(self): dsp_vector = np.array([0, 1, 0]) @@ -410,29 +660,41 @@ def test_rotate_dps_vector_to_rtn(self): rtn_vector = apply_rotation_matrix(input_rotation_matrix, dsp_vector) np.testing.assert_array_equal(rtn_vector, expected_rtn_vector) - @patch(f'{MODULE}.spiceypy.pxform') - @patch(f'{MODULE}.spiceypy.datetime2et') - def test_get_dps_to_rtn_rotation_matrix_returns_pxform_result(self, mock_datetime2et, mock_pxform): + @patch(f"{MODULE}.spiceypy.pxform") + @patch(f"{MODULE}.spiceypy.datetime2et") + def test_get_dps_to_rtn_rotation_matrix_returns_pxform_result( + self, mock_datetime2et, mock_pxform + ): epoch = datetime(year=2020, month=3, day=10) rotation_matrix = np.array([[0, 0, 1], [1, 0, 0], [0, 1, 0]]) mock_pxform.return_value = rotation_matrix - actual_rotation_matrix = moment_calculations.get_dps_to_rtn_rotation_matrix(epoch) + actual_rotation_matrix = moment_calculations.get_dps_to_rtn_rotation_matrix( + epoch + ) mock_datetime2et.assert_called_once_with(epoch) - mock_pxform.assert_called_once_with("IMAP_DPS", "IMAP_RTN", mock_datetime2et.return_value) + mock_pxform.assert_called_once_with( + "IMAP_DPS", "IMAP_RTN", mock_datetime2et.return_value + ) np.testing.assert_array_equal(actual_rotation_matrix, rotation_matrix) - @patch(f'{MODULE}.spiceypy.pxform') - @patch(f'{MODULE}.spiceypy.datetime2et') - def test_get_dps_to_rtn_rotation_matrix_propagates_spice_errors(self, mock_datetime2et, mock_pxform): + @patch(f"{MODULE}.spiceypy.pxform") + @patch(f"{MODULE}.spiceypy.datetime2et") + def test_get_dps_to_rtn_rotation_matrix_propagates_spice_errors( + self, mock_datetime2et, mock_pxform + ): epoch = datetime(year=2020, month=3, day=10) - mock_pxform.side_effect = spiceypy.utils.exceptions.SpiceyError("Missing coverage for IMAP_DPS") + mock_pxform.side_effect = spiceypy.utils.exceptions.SpiceyError( + "Missing coverage for IMAP_DPS" + ) with self.assertRaises(spiceypy.utils.exceptions.SpiceyError): moment_calculations.get_dps_to_rtn_rotation_matrix(epoch) mock_datetime2et.assert_called_once_with(epoch) - mock_pxform.assert_called_once_with("IMAP_DPS", "IMAP_RTN", mock_datetime2et.return_value) + mock_pxform.assert_called_once_with( + "IMAP_DPS", "IMAP_RTN", mock_datetime2et.return_value + ) def test_rotate_temperature(self): temperature_alpha = math.pi / 4 @@ -451,17 +713,23 @@ def test_rotate_temperature(self): cases = [ ( rotation_matrix, - np.array([ - np.asin(expected_rtn_temperature[2]), - np.atan2(expected_rtn_temperature[1], expected_rtn_temperature[0]), - ]), + np.array( + [ + np.asin(expected_rtn_temperature[2]), + np.atan2( + expected_rtn_temperature[1], expected_rtn_temperature[0] + ), + ] + ), ), (np.full((3, 3), np.nan), np.full(2, np.nan)), ] for input_rotation_matrix, expected_angles in cases: with self.subTest(input_rotation_matrix): - theta, phi = rotate_temperature(input_rotation_matrix, temperature_alpha, temperature_beta) + theta, phi = rotate_temperature( + input_rotation_matrix, temperature_alpha, temperature_beta + ) np.testing.assert_array_equal([theta, phi], expected_angles) def test_momscale(self): @@ -471,15 +739,26 @@ def test_momscale(self): test_cases = [ ("normal values", 1.92424e4, 1.74908e5, 68.3, 3497.7912, 1e-4), ("the sun froze", 12301, 12301, 1500, 7.0965e62, 1e57), - ("the sun froze", 12301, 12301, 400.3, 1.99655e+45, 1e40) + ("the sun froze", 12301, 12301, 400.3, 1.99655e45, 1e40), ] - for test_name, t_parallel, t_perpendicular, speed, expected_density_scale, allowable_difference in test_cases: + for ( + test_name, + t_parallel, + t_perpendicular, + speed, + expected_density_scale, + allowable_difference, + ) in test_cases: with self.subTest(test_name): temperature = (t_parallel + 2 * t_perpendicular) / 3 - dscale = compute_density_scale(core_halo_break - spacecraft_potential, speed, temperature) + dscale = compute_density_scale( + core_halo_break - spacecraft_potential, speed, temperature + ) - self.assertAlmostEqual(expected_density_scale, dscale, delta=allowable_difference) + self.assertAlmostEqual( + expected_density_scale, dscale, delta=allowable_difference + ) def test_halotrunc(self): density = 130 @@ -500,20 +779,72 @@ def test_halotrunc(self): ) test_cases = [ - ("t parallel less then 1e4, density should not change", 1e3, 1e7, 15, moments.density), - ("t perpendicular less then 1e4, density should not change", 1e7, 1e2, 15, moments.density), - ("t perpendicular greater then 1e8, density should be thrown out", 1e9, 1e1, 15, None), - ("t parallel greater then 1e8, density should be thrown out", 1e2, 1e9, 15, None), - ("core energy range is greater than 5 and temp is greater than 1e7", 1e7, 1.5e7, 15, moments.density), - ("core energy range is less than 5 and temperatures greater than 1e4", 1e5, 1.5e6, 10, moments.density), - ("core energy range is greater than 5 and temperatures greater than 1e4", 1e5, 1.5e6, 15, 126.487), + ( + "t parallel less then 1e4, density should not change", + 1e3, + 1e7, + 15, + moments.density, + ), + ( + "t perpendicular less then 1e4, density should not change", + 1e7, + 1e2, + 15, + moments.density, + ), + ( + "t perpendicular greater then 1e8, density should be thrown out", + 1e9, + 1e1, + 15, + None, + ), + ( + "t parallel greater then 1e8, density should be thrown out", + 1e2, + 1e9, + 15, + None, + ), + ( + "core energy range is greater than 5 and temp is greater than 1e7", + 1e7, + 1.5e7, + 15, + moments.density, + ), + ( + "core energy range is less than 5 and temperatures greater than 1e4", + 1e5, + 1.5e6, + 10, + moments.density, + ), + ( + "core energy range is greater than 5 and temperatures greater than 1e4", + 1e5, + 1.5e6, + 15, + 126.487, + ), ] - for test_name, t_parallel, t_perpendicular, core_halo_breakpoint, expected_density in test_cases: + for ( + test_name, + t_parallel, + t_perpendicular, + core_halo_breakpoint, + expected_density, + ) in test_cases: with self.subTest(test_name): - moments = dataclasses.replace(moments, t_parallel=t_parallel, t_perpendicular=t_perpendicular) + moments = dataclasses.replace( + moments, t_parallel=t_parallel, t_perpendicular=t_perpendicular + ) - scaled_density = halotrunc(moments, core_halo_breakpoint, spacecraft_potential) + scaled_density = halotrunc( + moments, core_halo_breakpoint, spacecraft_potential + ) self.assertAlmostEqual(expected_density, scaled_density, places=3) @@ -521,71 +852,134 @@ def test_integrate(self): istart = 1 iend = 19 - inst_az_data = np.loadtxt(get_test_data_path("swe/instrument_azimuth.csv"), delimiter=",").reshape(20, 7, 30) + inst_az_data = np.loadtxt( + get_test_data_path("swe/instrument_azimuth.csv"), delimiter="," + ).reshape(20, 7, 30) inst_az_data_reshaped = inst_az_data[:, 0, :] - phase_space_density = np.loadtxt(get_test_data_path("swe/phase_space_density.csv"), delimiter=",").reshape(20, - 7, - 30) + phase_space_density = np.loadtxt( + get_test_data_path("swe/phase_space_density.csv"), delimiter="," + ).reshape(20, 7, 30) phase_space_density_expected_shape = np.moveaxis(phase_space_density, 2, 1) - energy = np.loadtxt(get_test_data_path("swe/energies.csv"), delimiter=",").reshape(20, 7, 30)[:, 0, 0] - - sintheta = np.array([-0.1673557, 0.91652155, -0.83665564, 0., 0.83665564, -0.91652155, 0.1673557]) + energy = np.loadtxt( + get_test_data_path("swe/energies.csv"), delimiter="," + ).reshape(20, 7, 30)[:, 0, 0] + + sintheta = np.array( + [ + -0.1673557, + 0.91652155, + -0.83665564, + 0.0, + 0.83665564, + -0.91652155, + 0.1673557, + ] + ) - costheta = np.array([0.9858965815825497, -0.39998531498835127, -0.5477292602242684, 1.0, -0.5477292602242684, - -0.39998531498835127, 0.9858965815825497]) + costheta = np.array( + [ + 0.9858965815825497, + -0.39998531498835127, + -0.5477292602242684, + 1.0, + -0.5477292602242684, + -0.39998531498835127, + 0.9858965815825497, + ] + ) deltheta = np.array([0.6178, 0.3770, 0.3857, 0.3805, 0.3805, 0.3805, 0.6196]) cdelnv = np.array([0, 0, 0, 0]) cdelt = np.array([0, 0, 0, 0, 0, 0]) spacecraft_potential = 12 - integrate_outputs = moment_calculations.integrate(istart, iend, energy - spacecraft_potential, sintheta, - costheta, - deltheta, phase_space_density_expected_shape, - inst_az_data_reshaped, spacecraft_potential, cdelnv, cdelt) + integrate_outputs = moment_calculations.integrate( + istart, + iend, + energy - spacecraft_potential, + sintheta, + costheta, + deltheta, + phase_space_density_expected_shape, + inst_az_data_reshaped, + spacecraft_potential, + cdelnv, + cdelt, + ) - np.testing.assert_allclose(6.71828e+23, integrate_outputs.density, rtol=1e-5) + np.testing.assert_allclose(6.71828e23, integrate_outputs.density, rtol=1e-5) - np.testing.assert_allclose(np.array([-1053.53, 327.55, 1919.34]), integrate_outputs.velocity, - rtol=1e-4) - np.testing.assert_allclose(np.array(-10.7911), integrate_outputs.base_energy, - rtol=1e-4) + np.testing.assert_allclose( + np.array([-1053.53, 327.55, 1919.34]), integrate_outputs.velocity, rtol=1e-4 + ) + np.testing.assert_allclose( + np.array(-10.7911), integrate_outputs.base_energy, rtol=1e-4 + ) np.testing.assert_allclose( np.array([48.1855, -1.06466, 62.4149, -1.06081, -37.8901, -8.58049]), - integrate_outputs.temperature, rtol=2e-4) + integrate_outputs.temperature, + rtol=2e-4, + ) np.testing.assert_allclose( - np.array([9.92303e+24, -2.88985e+25, 9.45357e+24]), - integrate_outputs.heat_flux, rtol=1e-4) + np.array([9.92303e24, -2.88985e25, 9.45357e24]), + integrate_outputs.heat_flux, + rtol=1e-4, + ) def test_integrate_returns_early_when_density_is_negative(self): istart = 1 iend = 19 - inst_az_data = np.loadtxt(get_test_data_path("swe/instrument_azimuth.csv"), delimiter=",").reshape(20, 7, 30) + inst_az_data = np.loadtxt( + get_test_data_path("swe/instrument_azimuth.csv"), delimiter="," + ).reshape(20, 7, 30) inst_az_data_reshaped = inst_az_data[:, 0, :] - phase_space_density = np.loadtxt(get_test_data_path("swe/phase_space_density.csv"), delimiter=",").reshape(20, - 7, - 30) + phase_space_density = np.loadtxt( + get_test_data_path("swe/phase_space_density.csv"), delimiter="," + ).reshape(20, 7, 30) phase_space_density_expected_shape = np.moveaxis(phase_space_density, 2, 1) - energy = np.loadtxt(get_test_data_path("swe/energies.csv"), delimiter=",").reshape(20, 7, 30)[:, 0, 0] - - sintheta = np.array([-0.89100652, -0.66913061, -0.35836795, 0., 0.35836795, 0.66913061, 0.89100652]) + energy = np.loadtxt( + get_test_data_path("swe/energies.csv"), delimiter="," + ).reshape(20, 7, 30)[:, 0, 0] + + sintheta = np.array( + [ + -0.89100652, + -0.66913061, + -0.35836795, + 0.0, + 0.35836795, + 0.66913061, + 0.89100652, + ] + ) - costheta = np.array([0.4539905, 0.74314483, 0.93358043, 1., 0.93358043, 0.74314483, 0.4539905]) + costheta = np.array( + [0.4539905, 0.74314483, 0.93358043, 1.0, 0.93358043, 0.74314483, 0.4539905] + ) deltheta = np.array([0.6178, 0.3770, 0.3857, 0.3805, 0.3805, 0.3805, 0.6196]) spacecraft_potential = 12 cdelnv = np.array([0, 0, 0, 0]) cdelt = np.array([0, 0, 0, 0, 0, 0]) - integrate_outputs = moment_calculations.integrate(istart, iend, energy - spacecraft_potential, sintheta, - costheta, - deltheta, phase_space_density_expected_shape, - inst_az_data_reshaped, spacecraft_potential, cdelnv, cdelt) + integrate_outputs = moment_calculations.integrate( + istart, + iend, + energy - spacecraft_potential, + sintheta, + costheta, + deltheta, + phase_space_density_expected_shape, + inst_az_data_reshaped, + spacecraft_potential, + cdelnv, + cdelt, + ) self.assertIsNone(integrate_outputs) @@ -593,45 +987,78 @@ def test_integrate_with_nonzero_cdelnv_and_cdelt(self): istart = 1 iend = 19 - inst_az_data = np.loadtxt(get_test_data_path("swe/instrument_azimuth.csv"), delimiter=",").reshape(20, 7, 30) + inst_az_data = np.loadtxt( + get_test_data_path("swe/instrument_azimuth.csv"), delimiter="," + ).reshape(20, 7, 30) - phase_space_density = np.loadtxt(get_test_data_path("swe/phase_space_density.csv"), delimiter=",").reshape(20, - 7, - 30) + phase_space_density = np.loadtxt( + get_test_data_path("swe/phase_space_density.csv"), delimiter="," + ).reshape(20, 7, 30) inst_az_data_energy_spin = inst_az_data[:, 0, :] phase_space_density_energy_spin_cem = np.moveaxis(phase_space_density, 2, 1) - energy = np.loadtxt(get_test_data_path("swe/energies.csv"), delimiter=",").reshape(20, 7, 30)[:, 0, 0] - - artificial_all_positive_sintheta = np.array([0.89100652, 0.66913061, 0.35836795, 0., 0.35836795, 0.66913061, - 0.89100652]) + energy = np.loadtxt( + get_test_data_path("swe/energies.csv"), delimiter="," + ).reshape(20, 7, 30)[:, 0, 0] + + artificial_all_positive_sintheta = np.array( + [ + 0.89100652, + 0.66913061, + 0.35836795, + 0.0, + 0.35836795, + 0.66913061, + 0.89100652, + ] + ) - costheta = np.array([0.4539905, 0.74314483, 0.93358043, 1., 0.93358043, 0.74314483, 0.4539905]) + costheta = np.array( + [0.4539905, 0.74314483, 0.93358043, 1.0, 0.93358043, 0.74314483, 0.4539905] + ) deltheta = np.array([0.6178, 0.3770, 0.3857, 0.3805, 0.3805, 0.3805, 0.6196]) spacecraft_potential = 12 cdelnv = np.array([1e23, 2e23, 3e23, 4e23]) cdelt = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6]) * 1e40 - integrate_outputs = moment_calculations.integrate(istart, iend, energy - spacecraft_potential, - artificial_all_positive_sintheta, - costheta, - deltheta, phase_space_density_energy_spin_cem, - inst_az_data_energy_spin, spacecraft_potential, cdelnv, cdelt) + integrate_outputs = moment_calculations.integrate( + istart, + iend, + energy - spacecraft_potential, + artificial_all_positive_sintheta, + costheta, + deltheta, + phase_space_density_energy_spin_cem, + inst_az_data_energy_spin, + spacecraft_potential, + cdelnv, + cdelt, + ) - np.testing.assert_allclose(1.94762e+25, integrate_outputs.density, rtol=1e-5) + np.testing.assert_allclose(1.94762e25, integrate_outputs.density, rtol=1e-5) - np.testing.assert_allclose(np.array([53.8532, 10.9898, -2355.75]), integrate_outputs.velocity, - rtol=1e-4) + np.testing.assert_allclose( + np.array([53.8532, 10.9898, -2355.75]), + integrate_outputs.velocity, + rtol=1e-4, + ) - np.testing.assert_allclose(np.array(-10.7911), integrate_outputs.base_energy, - rtol=1e-4) np.testing.assert_allclose( - np.array([5.13448e+13, 1.0269e+14, 1.54034e+14, 2.05379e+14, 2.56724e+14, 3.08069e+14]), - integrate_outputs.temperature, rtol=2e-4) + np.array(-10.7911), integrate_outputs.base_energy, rtol=1e-4 + ) np.testing.assert_allclose( - np.array([-5.50653e+24, -1.61473e+24, -3.40728e+26]), - integrate_outputs.heat_flux, rtol=2e-4) + np.array( + [5.13448e13, 1.0269e14, 1.54034e14, 2.05379e14, 2.56724e14, 3.08069e14] + ), + integrate_outputs.temperature, + rtol=2e-4, + ) + np.testing.assert_allclose( + np.array([-5.50653e24, -1.61473e24, -3.40728e26]), + integrate_outputs.heat_flux, + rtol=2e-4, + ) def test_scale_core_density(self): core_velocity = np.array([300, 400, 500], dtype=float) @@ -647,50 +1074,135 @@ def test_scale_core_density(self): velocity_z=7, density=8, aoo=9e5, - ao=10 + ao=10, ) ifit = 5 spacecraft_potential = 12 cosin_p = np.array([0.9034, 0.6947, 0.3730, 0.0, -0.3714, -0.6896, -0.8996]) - aperture_field_of_view = [0.6178, 0.3770, 0.3857, 0.3805, 0.3805, 0.3805, 0.6196] + aperture_field_of_view = [ + 0.6178, + 0.3770, + 0.3857, + 0.3805, + 0.3805, + 0.3805, + 0.6196, + ] - regress_outputs = np.array([-1e-9, -9e-10, -8e-10, -7e-10, -6e-10, -5e-10, -4e-10, -3e-10, -2e-10, -1e-10]) + regress_outputs = np.array( + [ + -1e-9, + -9e-10, + -8e-10, + -7e-10, + -6e-10, + -5e-10, + -4e-10, + -3e-10, + -2e-10, + -1e-10, + ] + ) core_density = 1.23456789 base_energy = 100 - swepam_energies = np.array([2.55714286, 3.65142857, 5.16, 7.30571429, - 10.32857143, 14.34285714, 19.95714286, 27.42857143, - 38.37142857, 52.82857143, 73.32857143, 102.0, - 142.14285714, 196.57142857, 272., 372.71428571, - 519.0, 712.57142857, 987.14285714, 1370.0]) + swepam_energies = np.array( + [ + 2.55714286, + 3.65142857, + 5.16, + 7.30571429, + 10.32857143, + 14.34285714, + 19.95714286, + 27.42857143, + 38.37142857, + 52.82857143, + 73.32857143, + 102.0, + 142.14285714, + 196.57142857, + 272.0, + 372.71428571, + 519.0, + 712.57142857, + 987.14285714, + 1370.0, + ] + ) phi = np.broadcast_to((np.arange(0, 30) * 360 / 30)[np.newaxis, :], (20, 30)) - core_density_output = \ - moment_calculations.scale_core_density( - core_density, - core_velocity, core_temp, - core_moment_fit, ifit, swepam_energies - spacecraft_potential, - spacecraft_potential, cosin_p, - aperture_field_of_view, - phi, - regress_outputs, - base_energy) + core_density_output = moment_calculations.scale_core_density( + core_density, + core_velocity, + core_temp, + core_moment_fit, + ifit, + swepam_energies - spacecraft_potential, + spacecraft_potential, + cosin_p, + aperture_field_of_view, + phi, + regress_outputs, + base_energy, + ) np.testing.assert_allclose( - np.array([5718524.18228547, -263791830.41031304, 85711056.45722340, -8111566.20922477, ]), - core_density_output.cdelnv, rtol=2e-5) + np.array( + [ + 5718524.18228547, + -263791830.41031304, + 85711056.45722340, + -8111566.20922477, + ] + ), + core_density_output.cdelnv, + rtol=2e-5, + ) + np.testing.assert_allclose( + np.array( + [ + 9087717.95919351, + 395602.46759361, + 10176602.48232761, + 3589.91254779, + -1123.68989106, + 11120511.58411137, + ] + ), + core_density_output.cdelt, + rtol=5e-5, + ) np.testing.assert_allclose( - np.array([9087717.95919351, 395602.46759361, 10176602.48232761, 3589.91254779, -1123.68989106, - 11120511.58411137, ]), - core_density_output.cdelt, rtol=5e-5) - np.testing.assert_allclose(5718525.41685336, core_density_output.density, rtol=1e-5) + 5718525.41685336, core_density_output.density, rtol=1e-5 + ) np.testing.assert_allclose( - np.array([1.58917372, 0.06918342, 1.77959155, 0.00063640, -0.00018571, 1.94465965, ]), - core_density_output.temperature, rtol=5e-5) - np.testing.assert_allclose(np.array([-46.12927998, 14.98840069, -1.41836371, ]), core_density_output.velocity, - rtol=1e-5) + np.array( + [ + 1.58917372, + 0.06918342, + 1.77959155, + 0.00063640, + -0.00018571, + 1.94465965, + ] + ), + core_density_output.temperature, + rtol=5e-5, + ) + np.testing.assert_allclose( + np.array( + [ + -46.12927998, + 14.98840069, + -1.41836371, + ] + ), + core_density_output.velocity, + rtol=1e-5, + ) def test_scale_core_density_with_base_less_than_zero(self): core_velocity = np.array([300, 400, 500], dtype=float) @@ -706,49 +1218,126 @@ def test_scale_core_density_with_base_less_than_zero(self): velocity_z=7, density=8, aoo=9, - ao=10 + ao=10, ) ifit = 5 spacecraft_potential = 14 cosin_p = np.array([0.9034, 0.6947, 0.3730, 0.0, -0.3714, -0.6896, -0.8996]) - aperture_field_of_view = [0.6178, 0.3770, 0.3857, 0.3805, 0.3805, 0.3805, 0.6196] + aperture_field_of_view = [ + 0.6178, + 0.3770, + 0.3857, + 0.3805, + 0.3805, + 0.3805, + 0.6196, + ] - regress_outputs = np.array([-1e-9, -9e-10, -8e-10, -7e-10, -6e-10, -5e-10, -4e-10, -3e-10, -2e-10, -1e-10]) + regress_outputs = np.array( + [ + -1e-9, + -9e-10, + -8e-10, + -7e-10, + -6e-10, + -5e-10, + -4e-10, + -3e-10, + -2e-10, + -1e-10, + ] + ) core_density = 1.23456789 base_energy = 100 - swepam_energies = np.array([2.55714286, 3.65142857, 5.16, 7.30571429, - 10.32857143, 14.34285714, 19.95714286, 27.42857143, - 38.37142857, 52.82857143, 73.32857143, 102.0, - 142.14285714, 196.57142857, 272., 372.71428571, - 519.0, 712.57142857, 987.14285714, 1370.0]) + swepam_energies = np.array( + [ + 2.55714286, + 3.65142857, + 5.16, + 7.30571429, + 10.32857143, + 14.34285714, + 19.95714286, + 27.42857143, + 38.37142857, + 52.82857143, + 73.32857143, + 102.0, + 142.14285714, + 196.57142857, + 272.0, + 372.71428571, + 519.0, + 712.57142857, + 987.14285714, + 1370.0, + ] + ) phi = np.broadcast_to((np.arange(0, 30) * 360 / 30)[np.newaxis, :], (20, 30)) core_density_output = moment_calculations.scale_core_density( core_density, - core_velocity, core_temp, - core_moment_fit, ifit, + core_velocity, + core_temp, + core_moment_fit, + ifit, swepam_energies - spacecraft_potential, - spacecraft_potential, cosin_p, + spacecraft_potential, + cosin_p, aperture_field_of_view, phi, regress_outputs, - base_energy) + base_energy, + ) np.testing.assert_allclose( - np.array([817041.86654199, -14699338.23262659, 4776103.65373425, -451959.71106777, ]), - core_density_output.cdelnv, 2e-5) + np.array( + [ + 817041.86654199, + -14699338.23262659, + 4776103.65373425, + -451959.71106777, + ] + ), + core_density_output.cdelnv, + 2e-5, + ) np.testing.assert_allclose( - np.array([194444.92879664, 8463.82584106, 217743.48011274, 76.19579312, -24.62356212, 237940.02226200, ]), - core_density_output.cdelt, 5e-5) + np.array( + [ + 194444.92879664, + 8463.82584106, + 217743.48011274, + 76.19579312, + -24.62356212, + 237940.02226200, + ] + ), + core_density_output.cdelt, + 5e-5, + ) np.testing.assert_allclose(817043, core_density_output.density, rtol=1e-5) np.testing.assert_allclose( - np.array([0.23800124, 0.01038931, 0.26654716, 0.00015370, 0.00004541, 0.29131156, ]), + np.array( + [ + 0.23800124, + 0.01038931, + 0.26654716, + 0.00015370, + 0.00004541, + 0.29131156, + ] + ), core_density_output.temperature, - rtol=5e-5) - np.testing.assert_allclose(np.array([-17.99044364, 5.84620013, -0.55240957]), core_density_output.velocity, - rtol=1e-5) + rtol=5e-5, + ) + np.testing.assert_allclose( + np.array([-17.99044364, 5.84620013, -0.55240957]), + core_density_output.velocity, + rtol=1e-5, + ) def test_scale_halo_density(self): halo_velocities = np.array([300, 400, 500], dtype=float) @@ -764,14 +1353,35 @@ def test_scale_halo_density(self): velocity_z=7, density=8, aoo=9, - ao=10 + ao=10, ) space_craft_potential = 12 cosin_p = np.array([0.9034, 0.6947, 0.3730, 0.0, -0.3714, -0.6896, -0.8996]) - aperture_field_of_view = [0.6178, 0.3770, 0.3857, 0.3805, 0.3805, 0.3805, 0.6196] + aperture_field_of_view = [ + 0.6178, + 0.3770, + 0.3857, + 0.3805, + 0.3805, + 0.3805, + 0.6196, + ] - regress_outputs = np.array([-1e-9, -9e-10, -8e-10, -7e-10, -6e-10, -5e-10, -4e-10, -3e-10, -2e-10, -1e-10]) + regress_outputs = np.array( + [ + -1e-9, + -9e-10, + -8e-10, + -7e-10, + -6e-10, + -5e-10, + -4e-10, + -3e-10, + -2e-10, + -1e-10, + ] + ) base_energy = 100 phi = np.broadcast_to((np.arange(0, 30) * 360 / 30)[np.newaxis, :], (20, 30)) @@ -780,20 +1390,47 @@ def test_scale_halo_density(self): core_halo_break = 80 scaled_density = scale_halo_density( - halo_density, halo_velocities, halo_temps, halo_moment_fits, - space_craft_potential, core_halo_break, + halo_density, + halo_velocities, + halo_temps, + halo_moment_fits, + space_craft_potential, + core_halo_break, cosin_p, aperture_field_of_view, phi, regress_outputs, - base_energy) + base_energy, + ) - np.testing.assert_allclose(1.8148638e+09, scaled_density.density, rtol=1e-5) - np.testing.assert_allclose(np.array([-281.57623, 91.485629, -8.6923434, ]), scaled_density.velocity) - np.testing.assert_allclose(np.array([58.311792, 2.5465379, 65.29518, 0.030580338, -9.4024419e-05, 71.348932, ]), - scaled_density.temperature) + np.testing.assert_allclose(1.8148638e09, scaled_density.density, rtol=1e-5) + np.testing.assert_allclose( + np.array( + [ + -281.57623, + 91.485629, + -8.6923434, + ] + ), + scaled_density.velocity, + ) + np.testing.assert_allclose( + np.array( + [ + 58.311792, + 2.5465379, + 65.29518, + 0.030580338, + -9.4024419e-05, + 71.348932, + ] + ), + scaled_density.temperature, + ) - def test_scale_halo_density_difference_in_breakpoints_is_greater_than_base_energy(self): + def test_scale_halo_density_difference_in_breakpoints_is_greater_than_base_energy( + self, + ): halo_velocities = np.array([300, 400, 500], dtype=float) halo_temps = np.array([10, 20, 30, 40, 50, 60], dtype=float) @@ -807,14 +1444,35 @@ def test_scale_halo_density_difference_in_breakpoints_is_greater_than_base_energ velocity_z=7, density=8, aoo=9, - ao=10 + ao=10, ) space_craft_potential = 12 cosin_p = np.array([0.9034, 0.6947, 0.3730, 0.0, -0.3714, -0.6896, -0.8996]) - aperture_field_of_view = [0.6178, 0.3770, 0.3857, 0.3805, 0.3805, 0.3805, 0.6196] + aperture_field_of_view = [ + 0.6178, + 0.3770, + 0.3857, + 0.3805, + 0.3805, + 0.3805, + 0.6196, + ] - regress_outputs = np.array([-1e-9, -9e-10, -8e-10, -7e-10, -6e-10, -5e-10, -4e-10, -3e-10, -2e-10, -1e-10]) + regress_outputs = np.array( + [ + -1e-9, + -9e-10, + -8e-10, + -7e-10, + -6e-10, + -5e-10, + -4e-10, + -3e-10, + -2e-10, + -1e-10, + ] + ) base_energy = 60 phi = np.broadcast_to((np.arange(0, 30) * 360 / 30)[np.newaxis, :], (20, 30)) @@ -823,18 +1481,32 @@ def test_scale_halo_density_difference_in_breakpoints_is_greater_than_base_energ core_halo_break = 80 scaled_density = scale_halo_density( - halo_density, halo_velocities, halo_temps, halo_moment_fits, - space_craft_potential, core_halo_break, + halo_density, + halo_velocities, + halo_temps, + halo_moment_fits, + space_craft_potential, + core_halo_break, cosin_p, aperture_field_of_view, phi, regress_outputs, - base_energy) + base_energy, + ) - np.testing.assert_allclose(3.0876812e+08, scaled_density.density, rtol=1e-5) - np.testing.assert_allclose(np.array([1521.8375, 1116.6945, 1636.3078]), scaled_density.velocity, rtol=1e-5) - np.testing.assert_allclose(np.array([-67.071986, 60.433456, -14.211044, 129.49805, 161.93695, 72.622535]), - scaled_density.temperature, rtol=3e-5) + np.testing.assert_allclose(3.0876812e08, scaled_density.density, rtol=1e-5) + np.testing.assert_allclose( + np.array([1521.8375, 1116.6945, 1636.3078]), + scaled_density.velocity, + rtol=1e-5, + ) + np.testing.assert_allclose( + np.array( + [-67.071986, 60.433456, -14.211044, 129.49805, 161.93695, 72.622535] + ), + scaled_density.temperature, + rtol=3e-5, + ) def test_rotate_heat_flux(self): rotation_matrix = np.array([[0, 0, 1], [1, 0, 0], [0, 1, 0]]) @@ -846,13 +1518,24 @@ def test_rotate_heat_flux(self): (rotation_matrix, [0, 0, -1], 1, 0, math.pi), (nan_rotation_matrix, [2, 0, 0], 2, np.nan, np.nan), ] - for input_rotation_matrix, input, expected_mag, expected_theta, expected_phi in cases: + for ( + input_rotation_matrix, + input, + expected_mag, + expected_theta, + expected_phi, + ) in cases: with self.subTest(input): dsp_vector = np.array(input) - magnitude, theta, phi = rotate_vector_to_rtn_spherical_coordinates(input_rotation_matrix, dsp_vector) + magnitude, theta, phi = rotate_vector_to_rtn_spherical_coordinates( + input_rotation_matrix, dsp_vector + ) - np.testing.assert_array_equal([magnitude, theta, phi], [expected_mag, expected_theta, expected_phi]) + np.testing.assert_array_equal( + [magnitude, theta, phi], + [expected_mag, expected_theta, expected_phi], + ) def test_calculate_primary_eigenvector(self): cases = [ @@ -882,23 +1565,46 @@ def test_calculate_primary_eigenvector_error_cases(self): np.testing.assert_equal(temps, [np.nan, np.nan, np.nan]) def test_rotation_matrix_builder(self): - cases = [([25, 50, 75], [[0.26726124, 0.53452247, 0.80178374], [0.00000000, 0.83205032, -0.55470020], - [-0.96362412, 0.14824985, 0.22237480]]), - ([500.123, 999.99, 750.23], - [[0.37143612, 0.74268216, 0.55718797], [0.00000000, 0.60012156, -0.79990882], - [-0.92845851, 0.29711503, 0.22290681, ]]), - ([0, 0, 1], [[0, 0, 1], [0, 1, 0], [-1, 0, 0]])] + cases = [ + ( + [25, 50, 75], + [ + [0.26726124, 0.53452247, 0.80178374], + [0.00000000, 0.83205032, -0.55470020], + [-0.96362412, 0.14824985, 0.22237480], + ], + ), + ( + [500.123, 999.99, 750.23], + [ + [0.37143612, 0.74268216, 0.55718797], + [0.00000000, 0.60012156, -0.79990882], + [ + -0.92845851, + 0.29711503, + 0.22290681, + ], + ], + ), + ([0, 0, 1], [[0, 0, 1], [0, 1, 0], [-1, 0, 0]]), + ] for b_vector, expected_matrix in cases: with self.subTest(b_vector): - np.testing.assert_allclose(rotation_matrix_builder(b_vector), expected_matrix, rtol=2e-7) + np.testing.assert_allclose( + rotation_matrix_builder(b_vector), expected_matrix, rtol=2e-7 + ) def test_rotate_temperature_tensor_to_mag(self): expected_t_par = 9e4 expected_t_perp = 3e4 expected_ratio = 5 - t_parallel, t_perp_average, t_perp_ratio, = rotate_temperature_tensor_to_mag([1, 2, 5, 3, 6, 9], [0, 0, 1]) + ( + t_parallel, + t_perp_average, + t_perp_ratio, + ) = rotate_temperature_tensor_to_mag([1, 2, 5, 3, 6, 9], [0, 0, 1]) self.assertEqual(expected_t_par, t_parallel) self.assertEqual(expected_t_perp, t_perp_average) self.assertEqual(expected_ratio, t_perp_ratio) @@ -908,7 +1614,11 @@ def test_rotate_temperature_tensor_to_mag_with_ratio_less_than_one(self): expected_t_perp = 6e4 expected_ratio = 3 - t_parallel, t_perp_average, t_perp_ratio, = rotate_temperature_tensor_to_mag([9, 6, 3, 5, 2, 1], [0, 0, 1]) + ( + t_parallel, + t_perp_average, + t_perp_ratio, + ) = rotate_temperature_tensor_to_mag([9, 6, 3, 5, 2, 1], [0, 0, 1]) self.assertEqual(expected_t_par, t_parallel) self.assertEqual(expected_t_perp, t_perp_average) self.assertEqual(expected_ratio, t_perp_ratio) @@ -918,40 +1628,225 @@ def test_rotate_temperature_sensor_to_mag_with_negative_diagonal(self): expected_t_perp = 2e4 expected_ratio = 0 - t_parallel, t_perp_average, t_perp_ratio, = rotate_temperature_tensor_to_mag([9, 1, 1, 8, 1, -5], [0, 1, 0]) + ( + t_parallel, + t_perp_average, + t_perp_ratio, + ) = rotate_temperature_tensor_to_mag([9, 1, 1, 8, 1, -5], [0, 1, 0]) self.assertEqual(expected_t_par, t_parallel) self.assertEqual(expected_t_perp, t_perp_average) self.assertEqual(expected_ratio, t_perp_ratio) - def test_check_temperature_outlier_flag(self): # Make sure to replace temperatures with an array of the actual temperatures!! - temperatures = np.asarray([115994.66 , 116026.59 , 118674.664, 81440.03 , 64882.246, - 81693.66 , 97943.58 , 106707.7 , 110098.68 , 118224.14 , - 114332.87 , 72112.38 , 112917.73 , 114900.66 , 111031.34 , - 101084.9 , 104621.27 , 105482.9 , 111774.54 , 117124.625, - 115963.13 , 116497.22 , 117109.44 , 116067.71 , 112034.125, - 115428.125, 112301.64 , 114308.836, 112778.016, 112792.6 , - 112106.82 , 113119.766, 113032.69 , 113007.12 , 112749.11 , - 114590.734, 155481.47 , 117895.46 , 117952.94 , 115017.12 , - 110511.68 , 112724.29 , 111804.49 , 111790.33 , 110265.12 , - 111288.8 , 113818.85 , 113016.04 , 112490.38 , 113095.38 , - 112651.984, 113292.79 , 114452.2 , 113125.41 , 107491.9 , - 116490.92 , 114905.55 , 119953.88 , 123044.64 , 121589.14 , - 127217.89 , 124885.76 , 126984.54 , 126696.9 , 132537.52 , - 130044.89 , 203212.89 , 126546.88 , 129790.25 , 130226.16 , - 128036.91 , 123267.95 , 132878.33 , 133391.45 , 132277.88 , - 130638.78 , 131021.56 , 130341.88 , 132855.5 , 133775.25 , - 130266.41 , 126470.85 , 124299.48 , 126457.62 , 125728.02 , - 132413.53 , 125549.89 , 127563.04 , 133693. , 125295.82 , - 130699.61 , 125014.195, 126216.7 , 131265.03 , 133390.77 , - 130349.43 , 130807.32 , 129782.42 , 129749.54 , 128326.375]) - expected_values = np.asarray([0., 0., 0., 1., 1., 1., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., - 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., - 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., - 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., - 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., - 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]) + temperatures = np.asarray( + [ + 115994.66, + 116026.59, + 118674.664, + 81440.03, + 64882.246, + 81693.66, + 97943.58, + 106707.7, + 110098.68, + 118224.14, + 114332.87, + 72112.38, + 112917.73, + 114900.66, + 111031.34, + 101084.9, + 104621.27, + 105482.9, + 111774.54, + 117124.625, + 115963.13, + 116497.22, + 117109.44, + 116067.71, + 112034.125, + 115428.125, + 112301.64, + 114308.836, + 112778.016, + 112792.6, + 112106.82, + 113119.766, + 113032.69, + 113007.12, + 112749.11, + 114590.734, + 155481.47, + 117895.46, + 117952.94, + 115017.12, + 110511.68, + 112724.29, + 111804.49, + 111790.33, + 110265.12, + 111288.8, + 113818.85, + 113016.04, + 112490.38, + 113095.38, + 112651.984, + 113292.79, + 114452.2, + 113125.41, + 107491.9, + 116490.92, + 114905.55, + 119953.88, + 123044.64, + 121589.14, + 127217.89, + 124885.76, + 126984.54, + 126696.9, + 132537.52, + 130044.89, + 203212.89, + 126546.88, + 129790.25, + 130226.16, + 128036.91, + 123267.95, + 132878.33, + 133391.45, + 132277.88, + 130638.78, + 131021.56, + 130341.88, + 132855.5, + 133775.25, + 130266.41, + 126470.85, + 124299.48, + 126457.62, + 125728.02, + 132413.53, + 125549.89, + 127563.04, + 133693.0, + 125295.82, + 130699.61, + 125014.195, + 126216.7, + 131265.03, + 133390.77, + 130349.43, + 130807.32, + 129782.42, + 129749.54, + 128326.375, + ] + ) + expected_values = np.asarray( + [ + 0.0, + 0.0, + 0.0, + 1.0, + 1.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 1.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + ] + ) expected_values = expected_values.astype(int).astype(SweL3Flags) expected_values[expected_values == 0] = SweL3Flags.NONE expected_values[expected_values == 1] = SweL3Flags.TEMPERATURE_OUTLIER @@ -963,7 +1858,9 @@ def test_check_temperature_outlier_flag(self): @patch("spiceypy.datetime2et") @patch("spiceypy.pxform") @patch(f"{MODULE}.PredictedEphemerisTracker") - def test_rotate_rtn_vectors_to_dps(self, mock_predicted_ephemeris_tracker_class, mock_pxform, mock_datetime2et): + def test_rotate_rtn_vectors_to_dps( + self, mock_predicted_ephemeris_tracker_class, mock_pxform, mock_datetime2et + ): pred_tracker_1 = create_autospec(PredictedEphemerisTracker, used_predict=False) pred_tracker_1.run.return_value = np.eye(3) * 2 @@ -979,36 +1876,33 @@ def test_rotate_rtn_vectors_to_dps(self, mock_predicted_ephemeris_tracker_class, pred_tracker_3, ] - epochs = np.array([ - datetime(2020, 1, 1), - datetime(2020, 1, 2), - datetime(2020, 1, 3) - ]) - rtn_vectors = np.array([ - [1, 2, 3], - [4, 5, 6], - [7, 8, 9] - ]) + epochs = np.array( + [datetime(2020, 1, 1), datetime(2020, 1, 2), datetime(2020, 1, 3)] + ) + rtn_vectors = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) mock_datetime2et.side_effect = (sentinel.et_1, sentinel.et_2, sentinel.et_3) dps_vectors, used_predict = rotate_rtn_vectors_to_dps(epochs, rtn_vectors) self.assertEqual(3, mock_predicted_ephemeris_tracker_class.call_count) - self.assertEqual(epochs[0], mock_datetime2et.call_args_list[0].args[0]) self.assertEqual(epochs[1], mock_datetime2et.call_args_list[1].args[0]) self.assertEqual(epochs[2], mock_datetime2et.call_args_list[2].args[0]) - pred_tracker_1.run.assert_called_once_with(mock_pxform, "IMAP_RTN", "IMAP_DPS", sentinel.et_1) - pred_tracker_2.run.assert_called_once_with(mock_pxform, "IMAP_RTN", "IMAP_DPS", sentinel.et_2) - pred_tracker_3.run.assert_called_once_with(mock_pxform, "IMAP_RTN", "IMAP_DPS", sentinel.et_3) + pred_tracker_1.run.assert_called_once_with( + mock_pxform, "IMAP_RTN", "IMAP_DPS", sentinel.et_1 + ) + pred_tracker_2.run.assert_called_once_with( + mock_pxform, "IMAP_RTN", "IMAP_DPS", sentinel.et_2 + ) + pred_tracker_3.run.assert_called_once_with( + mock_pxform, "IMAP_RTN", "IMAP_DPS", sentinel.et_3 + ) - expected_dps_vectors = np.array([ - [2, 4, 6], - [12, 15, 18], - [np.nan, np.nan, np.nan] - ]) + expected_dps_vectors = np.array( + [[2, 4, 6], [12, 15, 18], [np.nan, np.nan, np.nan]] + ) np.testing.assert_array_equal(dps_vectors, expected_dps_vectors) - np.testing.assert_array_equal(used_predict, [False, True, False]) \ No newline at end of file + np.testing.assert_array_equal(used_predict, [False, True, False]) diff --git a/tests/swe/l3/science/test_pitch_calculations.py b/tests/swe/l3/science/test_pitch_calculations.py index 6769cb01f..b22ce6c69 100644 --- a/tests/swe/l3/science/test_pitch_calculations.py +++ b/tests/swe/l3/science/test_pitch_calculations.py @@ -3,112 +3,121 @@ import numpy as np -from imap_l3_processing.swe.l3.science.pitch_calculations import mec_breakpoint_finder, \ - average_over_look_directions, calculate_velocity_in_dsp_frame_km_s, calculate_look_directions, rebin_by_pitch_angle, \ - correct_and_rebin, calculate_energy_in_ev_from_velocity_in_km_per_second, integrate_distribution_to_get_1d_spectrum, \ - integrate_distribution_to_get_inbound_and_outbound_1d_spectrum, \ - rebin_by_pitch_angle_and_gyrophase, swe_rebin_intensity_by_pitch_angle_and_gyrophase +from imap_l3_processing.swe.l3.science.pitch_calculations import ( + mec_breakpoint_finder, + average_over_look_directions, + calculate_velocity_in_dsp_frame_km_s, + calculate_look_directions, + rebin_by_pitch_angle, + correct_and_rebin, + calculate_energy_in_ev_from_velocity_in_km_per_second, + integrate_distribution_to_get_1d_spectrum, + integrate_distribution_to_get_inbound_and_outbound_1d_spectrum, + rebin_by_pitch_angle_and_gyrophase, + swe_rebin_intensity_by_pitch_angle_and_gyrophase, +) from tests.test_helpers import build_swe_configuration, NumpyArrayMatcher from imap_l3_processing.swe.quality_flags import SweL3Flags + class TestPitchCalculations(unittest.TestCase): def test_average_flux(self): - flux_data = np.array([ - [ - [1, 2, 3, 4], - [5, 6, 7, 8], - [9, 10, 11, 12], - ], + flux_data = np.array( [ - [13, 14, 15, 16], - [17, 18, 19, 20], - [21, 22, 23, 24], - ], - ]) + [ + [1, 2, 3, 4], + [5, 6, 7, 8], + [9, 10, 11, 12], + ], + [ + [13, 14, 15, 16], + [17, 18, 19, 20], + [21, 22, 23, 24], + ], + ] + ) geometric_weights = [0.5, 0.25, 0.25, 0] result = average_over_look_directions(flux_data, geometric_weights, 1e-32) expected_result = [ - ((1 * 0.5 + 2 * 0.25 + 3 * 0.25) + (5 * 0.5 + 6 * 0.25 + 7 * 0.25) + (9 * 0.5 + 10 * 0.25 + 11 * 0.25)) / 3, - ((13 * 0.5 + 14 * 0.25 + 15 * 0.25) + (17 * 0.5 + 18 * 0.25 + 19 * 0.25) + ( - 21 * 0.5 + 22 * 0.25 + 23 * 0.25)) / 3, - + ( + (1 * 0.5 + 2 * 0.25 + 3 * 0.25) + + (5 * 0.5 + 6 * 0.25 + 7 * 0.25) + + (9 * 0.5 + 10 * 0.25 + 11 * 0.25) + ) + / 3, + ( + (13 * 0.5 + 14 * 0.25 + 15 * 0.25) + + (17 * 0.5 + 18 * 0.25 + 19 * 0.25) + + (21 * 0.5 + 22 * 0.25 + 23 * 0.25) + ) + / 3, ] np.testing.assert_almost_equal(result, expected_result) def test_average_over_look_directions_with_zeroes(self): - flux_data = np.array([ + flux_data = np.array( [ - [1, 2, 3, 4], - ], - [ - [0, 0, 0, 1e-36], - ], - ]) + [ + [1, 2, 3, 4], + ], + [ + [0, 0, 0, 1e-36], + ], + ] + ) geometric_weights = [0.25, 0.25, 0.25, 0.25] result = average_over_look_directions(flux_data, geometric_weights, 1e-34) - expected_result = [ - (1 * 0.25 + 2 * 0.25 + 3 * 0.25 + 4 * 0.25), 1e-34 - ] + expected_result = [(1 * 0.25 + 2 * 0.25 + 3 * 0.25 + 4 * 0.25), 1e-34] np.testing.assert_allclose(result, expected_result) def test_look_direction(self): inst_az = np.array([[0, 90], [180, 270]]) inst_el = np.array([-90, 0, 90]) shape = (2, 2, 3, 3) - expected_look_direction = np.array([ + expected_look_direction = np.array( [ [ - [0, 0, -1], - [1, 0, 0], - [0, 0, 1] + [[0, 0, -1], [1, 0, 0], [0, 0, 1]], + [[0, 0, -1], [0, 1, 0], [0, 0, 1]], ], [ - [0, 0, -1], - [0, 1, 0], - [0, 0, 1] - ] - ], - [ - [ - [0, 0, -1], - [-1, 0, 0], - [0, 0, 1] + [[0, 0, -1], [-1, 0, 0], [0, 0, 1]], + [[0, 0, -1], [0, -1, 0], [0, 0, 1]], ], - [ - [0, 0, -1], - [0, -1, 0], - [0, 0, 1] - ] ] - ]) + ) actual_look_direction = calculate_look_directions(inst_el, inst_az) - np.testing.assert_array_almost_equal(actual_look_direction, expected_look_direction) + np.testing.assert_array_almost_equal( + actual_look_direction, expected_look_direction + ) def test_compute_velocity(self): energy = np.array([1, 2]) # 2 -- energy inst_el = np.array([0, 90]) inst_az = np.array([[0], [90]]) # 2, 1 -- energy, spin sector - scalar_speeds = np.sqrt(energy * 1.602176634e-19 * 2 / - 9.109_383_7139e-31) / 1000 - expected_velocity = np.array([ - [ # energy 1 - [ # spin sector 1 - [-scalar_speeds[0], 0, 0], # CEM 1 - [0, 0, -scalar_speeds[0]], + scalar_speeds = ( + np.sqrt(energy * 1.602176634e-19 * 2 / 9.109_383_7139e-31) / 1000 + ) + expected_velocity = np.array( + [ + [ # energy 1 + [ # spin sector 1 + [-scalar_speeds[0], 0, 0], # CEM 1 + [0, 0, -scalar_speeds[0]], + ], ], - ], - [ # energy 2 - [ - [0, -scalar_speeds[1], 0], - [0, 0, -scalar_speeds[1]], + [ # energy 2 + [ + [0, -scalar_speeds[1], 0], + [0, 0, -scalar_speeds[1]], + ], ], - ], - ] + ] ) velocity = calculate_velocity_in_dsp_frame_km_s(energy, inst_el, inst_az) @@ -121,26 +130,71 @@ def test_compute_velocity_and_confirm_energy_calculation(self): inst_az = rng.random((24, 30)) * 360 velocity = calculate_velocity_in_dsp_frame_km_s(energy, inst_el, inst_az) scalar_velocity = np.linalg.norm(velocity, axis=-1) - calculated_energy = 0.5 * 9.109_383_7139e-31 * np.square(scalar_velocity * 1000) / 1.602176634e-19 + calculated_energy = ( + 0.5 + * 9.109_383_7139e-31 + * np.square(scalar_velocity * 1000) + / 1.602176634e-19 + ) np.testing.assert_almost_equal(calculated_energy[0], energy[0]) np.testing.assert_almost_equal(calculated_energy[1], energy[1]) self.assertEqual((24, 30, 7, 3), velocity.shape) def test_mec_breakpoint_finder(self): - average_psd = np.asarray([3.86611617e-05, 1.75927552e-05, 6.06703495e-06, 1.47614412e-06, - 2.46270751e-07, 4.97864654e-08, 3.90323281e-08, 4.21037028e-08, - 4.07355835e-08, 3.45166807e-08, 2.52075419e-08, 1.54987172e-08, - 7.84932795e-09, 3.32903783e-09, 1.40908421e-09, 8.53206391e-10, - 7.21564216e-10, 6.31903627e-10, 5.07238502e-10, 3.58827457e-10]) - energies = np.asarray([ 2.66 , 3.37852249, 4.29113316, 5.45025936, - 6.92249015, 8.79240175, 11.16741618, 14.18397245, - 18.01536462, 22.88169719, 29.06252952, 36.91293593, - 46.88390382, 59.54824189, 75.63348661, 96.06369752, - 122.01254227, 154.97072104, 196.83160382, 250. ]) - - sc_pot_to_test, ch_break_to_test, quality_flag_to_test = mec_breakpoint_finder(energies, average_psd) - expected_values = [np.float64(8.79240175),np.float64(75.63348660999998)] + average_psd = np.asarray( + [ + 3.86611617e-05, + 1.75927552e-05, + 6.06703495e-06, + 1.47614412e-06, + 2.46270751e-07, + 4.97864654e-08, + 3.90323281e-08, + 4.21037028e-08, + 4.07355835e-08, + 3.45166807e-08, + 2.52075419e-08, + 1.54987172e-08, + 7.84932795e-09, + 3.32903783e-09, + 1.40908421e-09, + 8.53206391e-10, + 7.21564216e-10, + 6.31903627e-10, + 5.07238502e-10, + 3.58827457e-10, + ] + ) + energies = np.asarray( + [ + 2.66, + 3.37852249, + 4.29113316, + 5.45025936, + 6.92249015, + 8.79240175, + 11.16741618, + 14.18397245, + 18.01536462, + 22.88169719, + 29.06252952, + 36.91293593, + 46.88390382, + 59.54824189, + 75.63348661, + 96.06369752, + 122.01254227, + 154.97072104, + 196.83160382, + 250.0, + ] + ) + + sc_pot_to_test, ch_break_to_test, quality_flag_to_test = mec_breakpoint_finder( + energies, average_psd + ) + expected_values = [np.float64(8.79240175), np.float64(75.63348660999998)] self.assertAlmostEqual(expected_values[0], sc_pot_to_test) self.assertAlmostEqual(expected_values[1], ch_break_to_test) self.assertEqual(SweL3Flags.NONE, quality_flag_to_test) @@ -151,51 +205,49 @@ def test_rebin_by_pitch_angle(self): energy = np.array([10 * 0.8, 10 / 0.8, 10 * 0.9 * 0.9, 10 / 0.9]) config = build_swe_configuration( - pitch_angle_bins=[45, 135], - pitch_angle_deltas=[45, 45], - energy_bins=[10] + pitch_angle_bins=[45, 135], pitch_angle_deltas=[45, 45], energy_bins=[10] ) result = rebin_by_pitch_angle(flux, pitch_angle, energy, config) - expected_result = np.array( - [ - [100, 128] - ] - ) + expected_result = np.array([[100, 128]]) np.testing.assert_almost_equal(result, expected_result) def test_rebin_by_pitch_angle_and_gyrophase(self): - psd = np.array([1000, 10, - 32, 256, - 3, 9, 81, - 5, 25, 125]) + psd = np.array([1000, 10, 32, 256, 3, 9, 81, 5, 25, 125]) pitch_angle = np.array([25, 60, 25, 60, 120, 170, 165, 120, 170, 165]) gyrophase = np.array([10, 120, 210, 350, 10, 120, 95, 210, 350, 260]) central_energy_value = 10 # bin range is really 60 - 140 - energy = np.array([central_energy_value * 0.8, central_energy_value / 0.8, - central_energy_value * 0.8 ** 2, central_energy_value / 0.8, - central_energy_value * 0.8 ** 2, central_energy_value * 0.8, central_energy_value / 0.8, - central_energy_value * 0.8, central_energy_value, central_energy_value / 0.8]) + energy = np.array( + [ + central_energy_value * 0.8, + central_energy_value / 0.8, + central_energy_value * 0.8**2, + central_energy_value / 0.8, + central_energy_value * 0.8**2, + central_energy_value * 0.8, + central_energy_value / 0.8, + central_energy_value * 0.8, + central_energy_value, + central_energy_value / 0.8, + ] + ) config = build_swe_configuration( pitch_angle_bins=[45, 135], pitch_angle_deltas=[45, 45], energy_bins=[central_energy_value], gyrophase_bins=[90, 270], - gyrophase_deltas=[90, 90] + gyrophase_deltas=[90, 90], ) - rebinned_by_gyro = rebin_by_pitch_angle_and_gyrophase(psd, pitch_angle, gyrophase, energy, config) + rebinned_by_gyro = rebin_by_pitch_angle_and_gyrophase( + psd, pitch_angle, gyrophase, energy, config + ) - expected_gyro = np.array([ - [ - [100, 128], - [27, 25] - ] - ]) + expected_gyro = np.array([[[100, 128], [27, 25]]]) np.testing.assert_almost_equal(expected_gyro, rebinned_by_gyro) @@ -205,19 +257,17 @@ def test_rebin_by_pitch_angle_ignores_zero_measurements(self): energy = np.array([10 * 0.8, 10 / 0.8, 10 * 0.9 * 0.9, 10 / 0.9, 10 / 0.9]) config = build_swe_configuration( - pitch_angle_bins=[45, 135], - pitch_angle_deltas=[45, 45], - energy_bins=[10] + pitch_angle_bins=[45, 135], pitch_angle_deltas=[45, 45], energy_bins=[10] ) result = rebin_by_pitch_angle(flux, pitch_angle, energy, config) - expected_result = np.array([ - [100, 128] - ]) + expected_result = np.array([[100, 128]]) np.testing.assert_almost_equal(result, expected_result) - def test_rebin_by_pitch_angle_uses_energy_within_configured_percent_of_nominal(self): + def test_rebin_by_pitch_angle_uses_energy_within_configured_percent_of_nominal( + self, + ): flux = np.array([9999, 50, 50, 9999]) pitch_angle = np.array([25, 60, 120, 170]) energy = np.array([10 * 0.69, 10 * 0.71, 10 * 1.49, 10 * 1.51]) @@ -231,9 +281,7 @@ def test_rebin_by_pitch_angle_uses_energy_within_configured_percent_of_nominal(s ) result = rebin_by_pitch_angle(flux, pitch_angle, energy, config) - expected_result = np.array([ - [50] - ]) + expected_result = np.array([[50]]) np.testing.assert_almost_equal(result, expected_result) def test_rebin_by_pitch_angle_skips_bins_with_less_than_two_measurements(self): @@ -251,26 +299,57 @@ def test_rebin_by_pitch_angle_skips_bins_with_less_than_two_measurements(self): result = rebin_by_pitch_angle(flux, pitch_angle, energy, config) - expected_result = np.array( - [ - [np.nan, 50], - [np.nan, np.nan] - ] - ) + expected_result = np.array([[np.nan, 50], [np.nan, np.nan]]) np.testing.assert_almost_equal(result, expected_result) def test_rebin_by_pitch_angle_skips_bins_with_invalid_measurements(self): test_cases = [ - ('no energy is close enough', [5.9, 6, 8.2, 11.8, 12.2], np.nan), - ('overall max value is in range and an energy is close enough below', [8, 8.21, 11.8, 12, 12.1], 50), - ('overall max value is in range and an energy is close enough above', [8, 8.20, 11.79, 12, 12.1], 50), - ('min and max are outside window and points on both side of nominal', [5.8, 5.9, 14.1, 9, 11], 50), - ('two lowest mins and max are outside window and points only below', [5.8, 5.9, 14.1, 8, 9], np.nan), - ('two lowest mins and max are outside window and points only above', [5.8, 5.9, 14.1, 11, 12], np.nan), - ('second_lowest in window and an energy close enough', [5.8, 6.1, 14.1, 11, 12], 50), - ('second_lowest in window and points on both sides', [5.8, 6.1, 14.1, 8, 12], 50), - ('second_lowest in window but all above and not close', [5.8, 11.3, 14.1, 11.3, 12], np.nan), - ('second_lowest in window but all below and not close', [5.8, 6.1, 14.1, 8, 8.7], np.nan), + ("no energy is close enough", [5.9, 6, 8.2, 11.8, 12.2], np.nan), + ( + "overall max value is in range and an energy is close enough below", + [8, 8.21, 11.8, 12, 12.1], + 50, + ), + ( + "overall max value is in range and an energy is close enough above", + [8, 8.20, 11.79, 12, 12.1], + 50, + ), + ( + "min and max are outside window and points on both side of nominal", + [5.8, 5.9, 14.1, 9, 11], + 50, + ), + ( + "two lowest mins and max are outside window and points only below", + [5.8, 5.9, 14.1, 8, 9], + np.nan, + ), + ( + "two lowest mins and max are outside window and points only above", + [5.8, 5.9, 14.1, 11, 12], + np.nan, + ), + ( + "second_lowest in window and an energy close enough", + [5.8, 6.1, 14.1, 11, 12], + 50, + ), + ( + "second_lowest in window and points on both sides", + [5.8, 6.1, 14.1, 8, 12], + 50, + ), + ( + "second_lowest in window but all above and not close", + [5.8, 11.3, 14.1, 11.3, 12], + np.nan, + ), + ( + "second_lowest in window but all below and not close", + [5.8, 6.1, 14.1, 8, 8.7], + np.nan, + ), ] for case, energy, expected_output in test_cases: with self.subTest(case): @@ -281,7 +360,7 @@ def test_rebin_by_pitch_angle_skips_bins_with_invalid_measurements(self): energy_bin_low_multiplier=0.6, energy_bin_high_multiplier=1.4, high_energy_proximity_threshold=0.18, - low_energy_proximity_threshold=0.12 + low_energy_proximity_threshold=0.12, ) pitch_angle = np.array([30, 35, 40, 45, 50]) @@ -300,41 +379,59 @@ def test_rebin_full_size_data(self): pitch_angle_bins = np.linspace(0, 180, 20, endpoint=False) + 4.5 pitch_angle_deltas = np.repeat(4.5, 20) energy_bins = np.geomspace(2, 5000, 24) - config = build_swe_configuration(pitch_angle_bins=pitch_angle_bins, - pitch_angle_deltas=pitch_angle_deltas, - energy_bins=energy_bins) + config = build_swe_configuration( + pitch_angle_bins=pitch_angle_bins, + pitch_angle_deltas=pitch_angle_deltas, + energy_bins=energy_bins, + ) result = rebin_by_pitch_angle(flux, pitch_angle, energy, config) self.assertEqual((24, 20), result.shape) def test_calculate_energy(self): - velocities = np.array([ - [400, 0, 0], - [0, -25000, 0], - ]) + velocities = np.array( + [ + [400, 0, 0], + [0, -25000, 0], + ] + ) expected_energies = [0.45485041, 1776.7594] result = calculate_energy_in_ev_from_velocity_in_km_per_second(velocities) np.testing.assert_allclose(result, expected_energies, rtol=1e-7) - @patch('imap_l3_processing.swe.l3.science.pitch_calculations.rebin_by_pitch_angle_and_gyrophase') - @patch('imap_l3_processing.swe.l3.science.pitch_calculations.calculate_gyrophase') - @patch('imap_l3_processing.swe.l3.science.pitch_calculations.calculate_energy_in_ev_from_velocity_in_km_per_second') - @patch('imap_l3_processing.swe.l3.science.pitch_calculations.rebin_by_pitch_angle') - @patch('imap_l3_processing.swe.l3.science.pitch_calculations.calculate_pitch_angle') - @patch('imap_l3_processing.swe.l3.science.pitch_calculations.calculate_velocity_in_sw_frame') - def test_correct_and_rebin(self, mock_calculate_velocity_in_sw_frame, - mock_calculate_pitch_angle, mock_rebin_by_pitch_angle, mock_calculate_energy, - mock_calculate_gyrophase, mock_rebin_by_pitch_angle_and_gyrophase): - - mag_vectors = np.array([ - [ - [1, 0, 0], - [0, 1, 0], - ], + @patch( + "imap_l3_processing.swe.l3.science.pitch_calculations.rebin_by_pitch_angle_and_gyrophase" + ) + @patch("imap_l3_processing.swe.l3.science.pitch_calculations.calculate_gyrophase") + @patch( + "imap_l3_processing.swe.l3.science.pitch_calculations.calculate_energy_in_ev_from_velocity_in_km_per_second" + ) + @patch("imap_l3_processing.swe.l3.science.pitch_calculations.rebin_by_pitch_angle") + @patch("imap_l3_processing.swe.l3.science.pitch_calculations.calculate_pitch_angle") + @patch( + "imap_l3_processing.swe.l3.science.pitch_calculations.calculate_velocity_in_sw_frame" + ) + def test_correct_and_rebin( + self, + mock_calculate_velocity_in_sw_frame, + mock_calculate_pitch_angle, + mock_rebin_by_pitch_angle, + mock_calculate_energy, + mock_calculate_gyrophase, + mock_rebin_by_pitch_angle_and_gyrophase, + ): + + mag_vectors = np.array( [ - [0, 0, 1], - [1, 1, 1], - ], - ]) + [ + [1, 0, 0], + [0, 1, 0], + ], + [ + [0, 0, 1], + [1, 1, 1], + ], + ] + ) flux_data = Mock() solar_wind_vector = Mock() @@ -347,27 +444,38 @@ def test_correct_and_rebin(self, mock_calculate_velocity_in_sw_frame, config=configuration, ) - expected_mag_vectors_with_cem_axis = np.array([ + expected_mag_vectors_with_cem_axis = np.array( [ - [[1, 0, 0]], - [[0, 1, 0]], - ], - [ - [[0, 0, 1]], - [[1, 1, 1]], - ], - ]) + [ + [[1, 0, 0]], + [[0, 1, 0]], + ], + [ + [[0, 0, 1]], + [[1, 1, 1]], + ], + ] + ) - mock_calculate_velocity_in_sw_frame.assert_called_once_with(sentinel.dsp_velocities, solar_wind_vector) + mock_calculate_velocity_in_sw_frame.assert_called_once_with( + sentinel.dsp_velocities, solar_wind_vector + ) - mock_calculate_pitch_angle.assert_called_once_with(mock_calculate_velocity_in_sw_frame.return_value, - NumpyArrayMatcher(expected_mag_vectors_with_cem_axis)) - mock_calculate_gyrophase.assert_called_once_with(mock_calculate_velocity_in_sw_frame.return_value, - NumpyArrayMatcher(expected_mag_vectors_with_cem_axis)) + mock_calculate_pitch_angle.assert_called_once_with( + mock_calculate_velocity_in_sw_frame.return_value, + NumpyArrayMatcher(expected_mag_vectors_with_cem_axis), + ) + mock_calculate_gyrophase.assert_called_once_with( + mock_calculate_velocity_in_sw_frame.return_value, + NumpyArrayMatcher(expected_mag_vectors_with_cem_axis), + ) - mock_calculate_energy.assert_called_with(mock_calculate_velocity_in_sw_frame.return_value) + mock_calculate_energy.assert_called_with( + mock_calculate_velocity_in_sw_frame.return_value + ) mock_rebin_by_pitch_angle.assert_called_with( - flux_data, mock_calculate_pitch_angle.return_value, + flux_data, + mock_calculate_pitch_angle.return_value, mock_calculate_energy.return_value, configuration, ) @@ -378,75 +486,96 @@ def test_correct_and_rebin(self, mock_calculate_velocity_in_sw_frame, mock_calculate_energy.return_value, configuration, ) - self.assertEqual((mock_rebin_by_pitch_angle.return_value, mock_rebin_by_pitch_angle_and_gyrophase.return_value), - result) + self.assertEqual( + ( + mock_rebin_by_pitch_angle.return_value, + mock_rebin_by_pitch_angle_and_gyrophase.return_value, + ), + result, + ) def test_integrate_distribution_to_get_1d_spectrum(self): pitch_angle_bins = [30, 90] pitch_angle_deltas = [15, 15] - configuration = build_swe_configuration(pitch_angle_bins=pitch_angle_bins, - pitch_angle_deltas=pitch_angle_deltas, - in_vs_out_energy_index=1, - ) + configuration = build_swe_configuration( + pitch_angle_bins=pitch_angle_bins, + pitch_angle_deltas=pitch_angle_deltas, + in_vs_out_energy_index=1, + ) - psd_by_pitch_angles = np.array([ - [10, 20], - [5, 10], - [7, 15], - ]) + psd_by_pitch_angles = np.array( + [ + [10, 20], + [5, 10], + [7, 15], + ] + ) - actual_integrated_spectrum = integrate_distribution_to_get_1d_spectrum(psd_by_pitch_angles, configuration) + actual_integrated_spectrum = integrate_distribution_to_get_1d_spectrum( + psd_by_pitch_angles, configuration + ) - bin1_factor = ((0.5 * np.deg2rad(30)) / 2) - bin2_factor = ((1 * np.deg2rad(30)) / 2) + bin1_factor = (0.5 * np.deg2rad(30)) / 2 + bin2_factor = (1 * np.deg2rad(30)) / 2 expected_integrated_spectrum = [ 10 * bin1_factor + 20 * bin2_factor, 5 * bin1_factor + 10 * bin2_factor, - 7 * bin1_factor + 15 * bin2_factor + 7 * bin1_factor + 15 * bin2_factor, ] - np.testing.assert_allclose(actual_integrated_spectrum, expected_integrated_spectrum) + np.testing.assert_allclose( + actual_integrated_spectrum, expected_integrated_spectrum + ) def test_integrate_distribution_to_get_1d_spectrum_ignores_nan_values(self): pitch_angle_bins = [30, 90] pitch_angle_deltas = [15, 15] - configuration = build_swe_configuration(pitch_angle_bins=pitch_angle_bins, - pitch_angle_deltas=pitch_angle_deltas, - in_vs_out_energy_index=1, - ) - - psd_by_energy_and_pitch_angles = np.array([ - [10, np.nan], - [5, 10], - [np.nan, 15], - ]) - - actual_integrated_spectrum = integrate_distribution_to_get_1d_spectrum(psd_by_energy_and_pitch_angles, - configuration) - - bin1_factor = ((0.5 * np.deg2rad(30)) / 2) - bin2_factor = ((1 * np.deg2rad(30)) / 2) + configuration = build_swe_configuration( + pitch_angle_bins=pitch_angle_bins, + pitch_angle_deltas=pitch_angle_deltas, + in_vs_out_energy_index=1, + ) + + psd_by_energy_and_pitch_angles = np.array( + [ + [10, np.nan], + [5, 10], + [np.nan, 15], + ] + ) + + actual_integrated_spectrum = integrate_distribution_to_get_1d_spectrum( + psd_by_energy_and_pitch_angles, configuration + ) + + bin1_factor = (0.5 * np.deg2rad(30)) / 2 + bin2_factor = (1 * np.deg2rad(30)) / 2 expected_integrated_spectrum = [ 10 * bin1_factor, 5 * bin1_factor + 10 * bin2_factor, - 15 * bin2_factor + 15 * bin2_factor, ] - np.testing.assert_allclose(actual_integrated_spectrum, expected_integrated_spectrum) + np.testing.assert_allclose( + actual_integrated_spectrum, expected_integrated_spectrum + ) def test_integrate_distribution_decides_inbound_and_outbound_based_on_config_energy_index( - self): + self, + ): pitch_angle_bins = [10, 85.5, 94.5, 100] pitch_angle_deltas = [4.5, 4.5, 4.5, 4.5] - psd_by_pitch_angles = np.array([ - [10, 20, 30, 40], - [15, 20, 5, 10], - [1, 2, 3, 4], - ]) + psd_by_pitch_angles = np.array( + [ + [10, 20, 30, 40], + [15, 20, 5, 10], + [1, 2, 3, 4], + ] + ) - bin1_factor = (np.sin(np.deg2rad(10)) * np.deg2rad(4.5 * 2)) - bin2_factor = (np.sin(np.deg2rad(85.5)) * np.deg2rad(4.5 * 2)) - bin3_factor = (np.sin(np.deg2rad(94.5)) * np.deg2rad(4.5 * 2)) - bin4_factor = (np.sin(np.deg2rad(100)) * np.deg2rad(4.5 * 2)) + bin1_factor = np.sin(np.deg2rad(10)) * np.deg2rad(4.5 * 2) + bin2_factor = np.sin(np.deg2rad(85.5)) * np.deg2rad(4.5 * 2) + bin3_factor = np.sin(np.deg2rad(94.5)) * np.deg2rad(4.5 * 2) + bin4_factor = np.sin(np.deg2rad(100)) * np.deg2rad(4.5 * 2) expected_A_spectrum = [ 10 * bin1_factor + 20 * bin2_factor, @@ -465,32 +594,37 @@ def test_integrate_distribution_decides_inbound_and_outbound_based_on_config_ene ] for index, expected_in, expected_out in cases: with self.subTest(index): - configuration = build_swe_configuration(pitch_angle_bins=pitch_angle_bins, - pitch_angle_deltas=pitch_angle_deltas, - in_vs_out_energy_index=index) + configuration = build_swe_configuration( + pitch_angle_bins=pitch_angle_bins, + pitch_angle_deltas=pitch_angle_deltas, + in_vs_out_energy_index=index, + ) - in_spectrum, out_spectrum = integrate_distribution_to_get_inbound_and_outbound_1d_spectrum( - psd_by_pitch_angles, - configuration) + in_spectrum, out_spectrum = ( + integrate_distribution_to_get_inbound_and_outbound_1d_spectrum( + psd_by_pitch_angles, configuration + ) + ) np.testing.assert_allclose(in_spectrum, expected_in) np.testing.assert_allclose(out_spectrum, expected_out) - def test_integrate_distribution_to_get_inbound_and_outbound_ignores_fill( - self): + def test_integrate_distribution_to_get_inbound_and_outbound_ignores_fill(self): pitch_angle_bins = [10, 85.5, 94.5, 100] pitch_angle_deltas = [4.5, 4.5, 4.5, 4.5] - psd_by_pitch_angles = np.array([ - [10, 20, 30, 40], - [np.nan, 20, 5, 10], - [1, 2, 3, np.nan], - ]) + psd_by_pitch_angles = np.array( + [ + [10, 20, 30, 40], + [np.nan, 20, 5, 10], + [1, 2, 3, np.nan], + ] + ) - bin1_factor = (np.sin(np.deg2rad(10)) * np.deg2rad(4.5 * 2)) - bin2_factor = (np.sin(np.deg2rad(85.5)) * np.deg2rad(4.5 * 2)) - bin3_factor = (np.sin(np.deg2rad(94.5)) * np.deg2rad(4.5 * 2)) - bin4_factor = (np.sin(np.deg2rad(100)) * np.deg2rad(4.5 * 2)) + bin1_factor = np.sin(np.deg2rad(10)) * np.deg2rad(4.5 * 2) + bin2_factor = np.sin(np.deg2rad(85.5)) * np.deg2rad(4.5 * 2) + bin3_factor = np.sin(np.deg2rad(94.5)) * np.deg2rad(4.5 * 2) + bin4_factor = np.sin(np.deg2rad(100)) * np.deg2rad(4.5 * 2) expected_A_spectrum = [ 10 * bin1_factor + 20 * bin2_factor, @@ -503,69 +637,85 @@ def test_integrate_distribution_to_get_inbound_and_outbound_ignores_fill( 3 * bin3_factor + 0, ] - configuration = build_swe_configuration(pitch_angle_bins=pitch_angle_bins, - pitch_angle_deltas=pitch_angle_deltas, - in_vs_out_energy_index=0) + configuration = build_swe_configuration( + pitch_angle_bins=pitch_angle_bins, + pitch_angle_deltas=pitch_angle_deltas, + in_vs_out_energy_index=0, + ) - in_spectrum, out_spectrum = integrate_distribution_to_get_inbound_and_outbound_1d_spectrum( - psd_by_pitch_angles, - configuration) + in_spectrum, out_spectrum = ( + integrate_distribution_to_get_inbound_and_outbound_1d_spectrum( + psd_by_pitch_angles, configuration + ) + ) np.testing.assert_allclose(in_spectrum, expected_A_spectrum) np.testing.assert_allclose(out_spectrum, expected_B_spectrum) - @patch('imap_l3_processing.swe.l3.science.pitch_calculations.calculate_unit_vector') - @patch('imap_l3_processing.swe.l3.science.pitch_calculations.calculate_pitch_angle') - @patch('imap_l3_processing.swe.l3.science.pitch_calculations.calculate_gyrophase') - def test_rebin_swe_intensity(self, mock_calculate_gyrophases, mock_calculate_pitch_angles, - mock_calculate_unit_vector): + @patch("imap_l3_processing.swe.l3.science.pitch_calculations.calculate_unit_vector") + @patch("imap_l3_processing.swe.l3.science.pitch_calculations.calculate_pitch_angle") + @patch("imap_l3_processing.swe.l3.science.pitch_calculations.calculate_gyrophase") + def test_rebin_swe_intensity( + self, + mock_calculate_gyrophases, + mock_calculate_pitch_angles, + mock_calculate_unit_vector, + ): intensity_data_for_energy_1 = [[0, 1, 2, 3], [4, 5, 6, 7]] intensity_data_for_energy_2 = [[8, 9, 10, 11], [12, 13, 14, 15]] - intensity = np.array([ - intensity_data_for_energy_1, - intensity_data_for_energy_2 - ]) + intensity = np.array([intensity_data_for_energy_1, intensity_data_for_energy_2]) pitch_angles = np.array( [ [[45, 45, 135, 135], [45, 45, 135, 135]], [[45, 45, 135, 135], [45, 45, 135, 135]], - ]) + ] + ) mock_calculate_pitch_angles.return_value = pitch_angles gyrophases = np.array( [ [[135, 225, 225, 135], [45, 315, 315, 45]], - [[135, 225, 225, 135], [45, 315, 315, 45]] - ]) + [[135, 225, 225, 135], [45, 315, 315, 45]], + ] + ) mock_calculate_gyrophases.return_value = gyrophases - counts = np.array([ - [[4, 9, 49, 36], [0, 16, 64, 25]], - [[4 * 4, 4 * 9, 4 * 49, 4 * 36], [4 * 1, 4 * 16, 4 * 64, 4 * 25]], - ]) - - normalized_mag_vectors = np.array([ - [ - [1, 0, 0], - [0, 1, 0], - ], + counts = np.array( [ - [0, 0, 1], - [1, 1, 1], - ], - ]) - expected_mag_vectors_with_cem_axis = np.array([ + [[4, 9, 49, 36], [0, 16, 64, 25]], + [[4 * 4, 4 * 9, 4 * 49, 4 * 36], [4 * 1, 4 * 16, 4 * 64, 4 * 25]], + ] + ) + + normalized_mag_vectors = np.array( [ - [[1, 0, 0]], - [[0, 1, 0]], - ], + [ + [1, 0, 0], + [0, 1, 0], + ], + [ + [0, 0, 1], + [1, 1, 1], + ], + ] + ) + expected_mag_vectors_with_cem_axis = np.array( [ - [[0, 0, 1]], - [[1, 1, 1]], - ], - ]) - mock_calculate_unit_vector.side_effect = [sentinel.normalized_dsp_velocities, normalized_mag_vectors] + [ + [[1, 0, 0]], + [[0, 1, 0]], + ], + [ + [[0, 0, 1]], + [[1, 1, 1]], + ], + ] + ) + mock_calculate_unit_vector.side_effect = [ + sentinel.normalized_dsp_velocities, + normalized_mag_vectors, + ] config = build_swe_configuration( pitch_angle_bins=np.array([45, 135]), @@ -573,37 +723,58 @@ def test_rebin_swe_intensity(self, mock_calculate_gyrophases, mock_calculate_pit gyrophase_bins=np.array([45, 135, 225, 315]), gyrophase_deltas=np.array([45, 45, 45, 45]), ) - expected_intensity_by_pa_and_gyro = np.array([ - [[4, 0, 1, 5], [7, 3, 2, 6]], - [[12, 8, 9, 13], [15, 11, 10, 14]] - ]) + expected_intensity_by_pa_and_gyro = np.array( + [[[4, 0, 1, 5], [7, 3, 2, 6]], [[12, 8, 9, 13], [15, 11, 10, 14]]] + ) expected_intensity_by_pa = np.array([[2.5, 4.5], [10.5, 12.5]]) - expected_uncertainty_by_pa_gyro = expected_intensity_by_pa_and_gyro * np.array([ - [[np.nan, 1 / 2, 1 / 3, 1 / 4], [1 / 5, 1 / 6, 1 / 7, 1 / 8]], - [[1 / 2, 1 / 4, 1 / 6, 1 / 8], [1 / 10, 1 / 12, 1 / 14, 1 / 16]] - ]) + expected_uncertainty_by_pa_gyro = expected_intensity_by_pa_and_gyro * np.array( + [ + [[np.nan, 1 / 2, 1 / 3, 1 / 4], [1 / 5, 1 / 6, 1 / 7, 1 / 8]], + [[1 / 2, 1 / 4, 1 / 6, 1 / 8], [1 / 10, 1 / 12, 1 / 14, 1 / 16]], + ] + ) expected_uncertainty_by_pa = expected_intensity_by_pa * np.array( - [[1 / np.sqrt(4 + 9 + 0 + 16), 1 / np.sqrt(49 + 36 + 64 + 25)], - [1 / (2 * np.sqrt(4 + 9 + 1 + 16)), - 1 / (2 * np.sqrt(49 + 36 + 64 + 25))]]) - - rebinned_data = swe_rebin_intensity_by_pitch_angle_and_gyrophase(intensity, - counts, - sentinel.dsp_velocities, - sentinel.mag_vectors, config) - mock_calculate_unit_vector.assert_has_calls([call(sentinel.dsp_velocities), call(sentinel.mag_vectors)]) - mock_calculate_pitch_angles.assert_called_once_with(sentinel.normalized_dsp_velocities, - NumpyArrayMatcher(expected_mag_vectors_with_cem_axis)) - mock_calculate_gyrophases.assert_called_once_with(sentinel.normalized_dsp_velocities, - NumpyArrayMatcher(expected_mag_vectors_with_cem_axis)) - actual_rebinned_by_pa_and_gyro, actual_rebinned_by_pa, actual_intensity_uncertainty_by_pa_and_gyro, actual_intensity_uncertainty_by_pa = rebinned_data - np.testing.assert_equal(actual_rebinned_by_pa_and_gyro, expected_intensity_by_pa_and_gyro) + [ + [1 / np.sqrt(4 + 9 + 0 + 16), 1 / np.sqrt(49 + 36 + 64 + 25)], + [ + 1 / (2 * np.sqrt(4 + 9 + 1 + 16)), + 1 / (2 * np.sqrt(49 + 36 + 64 + 25)), + ], + ] + ) + + rebinned_data = swe_rebin_intensity_by_pitch_angle_and_gyrophase( + intensity, counts, sentinel.dsp_velocities, sentinel.mag_vectors, config + ) + mock_calculate_unit_vector.assert_has_calls( + [call(sentinel.dsp_velocities), call(sentinel.mag_vectors)] + ) + mock_calculate_pitch_angles.assert_called_once_with( + sentinel.normalized_dsp_velocities, + NumpyArrayMatcher(expected_mag_vectors_with_cem_axis), + ) + mock_calculate_gyrophases.assert_called_once_with( + sentinel.normalized_dsp_velocities, + NumpyArrayMatcher(expected_mag_vectors_with_cem_axis), + ) + ( + actual_rebinned_by_pa_and_gyro, + actual_rebinned_by_pa, + actual_intensity_uncertainty_by_pa_and_gyro, + actual_intensity_uncertainty_by_pa, + ) = rebinned_data + np.testing.assert_equal( + actual_rebinned_by_pa_and_gyro, expected_intensity_by_pa_and_gyro + ) np.testing.assert_equal(actual_rebinned_by_pa, expected_intensity_by_pa) - np.testing.assert_array_almost_equal(actual_intensity_uncertainty_by_pa_and_gyro, - expected_uncertainty_by_pa_gyro) - np.testing.assert_array_almost_equal(actual_intensity_uncertainty_by_pa, expected_uncertainty_by_pa) + np.testing.assert_array_almost_equal( + actual_intensity_uncertainty_by_pa_and_gyro, expected_uncertainty_by_pa_gyro + ) + np.testing.assert_array_almost_equal( + actual_intensity_uncertainty_by_pa, expected_uncertainty_by_pa + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/swe/l3/test_swe_l3_dependencies.py b/tests/swe/l3/test_swe_l3_dependencies.py index 18aa9f3a3..268276b30 100644 --- a/tests/swe/l3/test_swe_l3_dependencies.py +++ b/tests/swe/l3/test_swe_l3_dependencies.py @@ -2,7 +2,11 @@ from pathlib import Path from unittest.mock import patch, Mock, call -from imap_data_access.processing_input import ScienceInput, AncillaryInput, ProcessingInputCollection +from imap_data_access.processing_input import ( + ScienceInput, + AncillaryInput, + ProcessingInputCollection, +) from imap_l3_processing.swe.l3.swe_l3_dependencies import SweL3Dependencies @@ -15,15 +19,23 @@ class TestSweL3Dependencies(unittest.TestCase): @patch(f"{MODULE}.read_l2_swe_data") @patch(f"{MODULE}.read_l3a_swapi_proton_data") @patch(f"{MODULE}.read_swe_config") - def test_from_file_paths(self, mock_read_swe_config, mock_read_swapi_data, mock_read_swe_data, mock_read_mag_data, - mock_read_l1b_swe_data): + def test_from_file_paths( + self, + mock_read_swe_config, + mock_read_swapi_data, + mock_read_swe_data, + mock_read_mag_data, + mock_read_l1b_swe_data, + ): swe_path = Path("test_swe_cdf.cdf") swe_l1b_path = Path("test_swe_l1b_cdf.cdf") mag_path = Path("test_mag_cdf.cdf") swapi_path = Path("test_swapi_cdf.cdf") config_path = Path("test_config.json") - result = SweL3Dependencies.from_file_paths(swe_path, swe_l1b_path, mag_path, swapi_path, config_path) + result = SweL3Dependencies.from_file_paths( + swe_path, swe_l1b_path, mag_path, swapi_path, config_path + ) mock_read_l1b_swe_data.assert_called_once_with(swe_l1b_path) mock_read_swe_data.assert_called_once_with(swe_path) @@ -34,52 +46,78 @@ def test_from_file_paths(self, mock_read_swe_config, mock_read_swapi_data, mock_ self.assertEqual(mock_read_l1b_swe_data.return_value, result.swe_l1b_data) self.assertEqual(mock_read_swe_data.return_value, result.swe_l2_data) self.assertEqual(mock_read_mag_data.return_value, result.mag_data) - self.assertEqual(mock_read_swapi_data.return_value, result.swapi_l3a_proton_data) + self.assertEqual( + mock_read_swapi_data.return_value, result.swapi_l3a_proton_data + ) self.assertEqual(mock_read_swe_config.return_value, result.configuration) self.assertFalse(result.mag_is_preliminary) result_preliminary = SweL3Dependencies.from_file_paths( - swe_path, swe_l1b_path, mag_path, swapi_path, config_path, mag_is_preliminary=True + swe_path, + swe_l1b_path, + mag_path, + swapi_path, + config_path, + mag_is_preliminary=True, ) self.assertTrue(result_preliminary.mag_is_preliminary) @patch("imap_l3_processing.utils.download") @patch(f"{MODULE}.download") @patch(f"{MODULE}.SweL3Dependencies.from_file_paths") - def test_fetch_dependencies(self, mock_from_file_paths, mock_download_dependency, mock_utils_download): + def test_fetch_dependencies( + self, mock_from_file_paths, mock_download_dependency, mock_utils_download + ): swe_l2_dependency = ScienceInput("imap_swe_l2_sci_20200101_v000.cdf") swe_l1b_dependency = ScienceInput("imap_swe_l1b_sci_20200101_v000.cdf") mag_l1d_dependency = ScienceInput("imap_mag_l1d_norm-dsrf_20200101_v000.cdf") swapi_l3a_dependency = ScienceInput("imap_swapi_l3_proton-sw_20200101_v000.cdf") config_dependency = AncillaryInput("imap_swe_config_20250101_v000.json") - processing_input_collection = ProcessingInputCollection(swe_l2_dependency, swe_l1b_dependency, - mag_l1d_dependency, swapi_l3a_dependency, - config_dependency) + processing_input_collection = ProcessingInputCollection( + swe_l2_dependency, + swe_l1b_dependency, + mag_l1d_dependency, + swapi_l3a_dependency, + config_dependency, + ) expected_swe_path = Mock() expected_swe_l1b_path = Mock() expected_mag_path = Mock() expected_swapi_path = Mock() expected_config_path = Mock() - mock_download_dependency.side_effect = [expected_swe_path, expected_swe_l1b_path, - expected_swapi_path, - expected_config_path] + mock_download_dependency.side_effect = [ + expected_swe_path, + expected_swe_l1b_path, + expected_swapi_path, + expected_config_path, + ] mock_utils_download.return_value = expected_mag_path result = SweL3Dependencies.fetch_dependencies(processing_input_collection) - mock_download_dependency.assert_has_calls([call(swe_l2_dependency.imap_file_paths[0].construct_path()), - call(swe_l1b_dependency.imap_file_paths[0].construct_path()), - call(swapi_l3a_dependency.imap_file_paths[0].construct_path()), - call(config_dependency.imap_file_paths[0].construct_path())], - any_order=False) - mock_utils_download.assert_called_once_with(mag_l1d_dependency.imap_file_paths[0].construct_path()) - - mock_from_file_paths.assert_called_with(expected_swe_path, expected_swe_l1b_path, expected_mag_path, - expected_swapi_path, - expected_config_path, - mag_is_preliminary=True) + mock_download_dependency.assert_has_calls( + [ + call(swe_l2_dependency.imap_file_paths[0].construct_path()), + call(swe_l1b_dependency.imap_file_paths[0].construct_path()), + call(swapi_l3a_dependency.imap_file_paths[0].construct_path()), + call(config_dependency.imap_file_paths[0].construct_path()), + ], + any_order=False, + ) + mock_utils_download.assert_called_once_with( + mag_l1d_dependency.imap_file_paths[0].construct_path() + ) + + mock_from_file_paths.assert_called_with( + expected_swe_path, + expected_swe_l1b_path, + expected_mag_path, + expected_swapi_path, + expected_config_path, + mag_is_preliminary=True, + ) self.assertEqual(mock_from_file_paths.return_value, result) @patch(f"{MODULE}.select_mag_path") @@ -90,7 +128,10 @@ def test_fetch_dependencies_raises_if_no_mag(self, _, mock_select_mag_path): swapi_l3a_dependency = ScienceInput("imap_swapi_l3_proton-sw_20200101_v000.cdf") config_dependency = AncillaryInput("imap_swe_config_20250101_v000.json") processing_input_collection = ProcessingInputCollection( - swe_l2_dependency, swe_l1b_dependency, swapi_l3a_dependency, config_dependency, + swe_l2_dependency, + swe_l1b_dependency, + swapi_l3a_dependency, + config_dependency, ) mock_select_mag_path.return_value = (None, None) @@ -102,7 +143,9 @@ def test_fetch_dependencies_raises_if_no_mag(self, _, mock_select_mag_path): @patch("imap_l3_processing.utils.download") @patch(f"{MODULE}.download") @patch(f"{MODULE}.SweL3Dependencies.from_file_paths") - def test_fetch_dependencies_prioritizes_mag_l2_data(self, mock_from_file_paths, mock_download, mock_utils_download): + def test_fetch_dependencies_prioritizes_mag_l2_data( + self, mock_from_file_paths, mock_download, mock_utils_download + ): swe_l2_dependency = ScienceInput("imap_swe_l2_sci_20200101_v000.cdf") swe_l1b_dependency = ScienceInput("imap_swe_l1b_sci_20200101_v000.cdf") mag_l1d_dependency = ScienceInput("imap_mag_l1d_norm-dsrf_20200101_v000.cdf") @@ -134,12 +177,18 @@ def test_fetch_dependencies_prioritizes_mag_l2_data(self, mock_from_file_paths, result = SweL3Dependencies.fetch_dependencies(processing_input_collection) - mock_download.assert_has_calls([call(swe_l2_dependency.imap_file_paths[0].construct_path()), - call(swe_l1b_dependency.imap_file_paths[0].construct_path()), - call(swapi_l3a_dependency.imap_file_paths[0].construct_path()), - call(config_dependency.imap_file_paths[0].construct_path())], - any_order=False) - mock_utils_download.assert_called_once_with(mag_l2_dependency.imap_file_paths[0].construct_path()) + mock_download.assert_has_calls( + [ + call(swe_l2_dependency.imap_file_paths[0].construct_path()), + call(swe_l1b_dependency.imap_file_paths[0].construct_path()), + call(swapi_l3a_dependency.imap_file_paths[0].construct_path()), + call(config_dependency.imap_file_paths[0].construct_path()), + ], + any_order=False, + ) + mock_utils_download.assert_called_once_with( + mag_l2_dependency.imap_file_paths[0].construct_path() + ) mock_from_file_paths.assert_called_with( expected_swe_path, diff --git a/tests/swe/l3/test_utils.py b/tests/swe/l3/test_utils.py index 33bc43db4..40687faa4 100644 --- a/tests/swe/l3/test_utils.py +++ b/tests/swe/l3/test_utils.py @@ -10,8 +10,13 @@ from spacepy.pycdf import CDF from imap_l3_processing.swe.l3.models import SweL2Data, SwapiL3aProtonData, SweL1bData -from imap_l3_processing.swe.l3.utils import read_swe_config, read_l2_swe_data, read_l3a_swapi_proton_data, \ - read_l1b_swe_data, compute_epoch_delta_in_ns +from imap_l3_processing.swe.l3.utils import ( + read_swe_config, + read_l2_swe_data, + read_l3a_swapi_proton_data, + read_l1b_swe_data, + compute_epoch_delta_in_ns, +) from tests.spice_test_case import SpiceTestCase from tests.test_helpers import get_test_data_path @@ -23,23 +28,25 @@ def _copy_with_rtn_velocity(source: Path, dest: Path) -> Path: copy in a tempdir rather than mutating the checked-in CDF.""" shutil.copyfile(source, dest) with CDF(str(dest), readonly=False) as cdf: - if 'proton_sw_velocity_rtn' in cdf: + if "proton_sw_velocity_rtn" in cdf: return dest - speed = cdf['proton_sw_speed'][:].astype(float) + speed = cdf["proton_sw_speed"][:].astype(float) velocity_rtn = np.stack( [-speed, np.zeros_like(speed), np.zeros_like(speed)], axis=-1 ).astype(np.float32) - cdf.new('proton_sw_velocity_rtn', data=velocity_rtn) - cdf['proton_sw_velocity_rtn'].attrs['FILLVAL'] = np.float32(-1e31) - cdf['proton_sw_velocity_rtn'].attrs['UNITS'] = 'km/s' + cdf.new("proton_sw_velocity_rtn", data=velocity_rtn) + cdf["proton_sw_velocity_rtn"].attrs["FILLVAL"] = np.float32(-1e31) + cdf["proton_sw_velocity_rtn"].attrs["UNITS"] = "km/s" return dest class TestUtils(SpiceTestCase): def test_read_swe_config(self): - result = read_swe_config(get_test_data_path('swe/example_swe_config.json')) - self.assertEqual([0.0781351, 0.151448, 0.204686, 0.181759, 0.175125, 0.138312, 0.0697327], - result["geometric_fractions"]) + result = read_swe_config(get_test_data_path("swe/example_swe_config.json")) + self.assertEqual( + [0.0781351, 0.151448, 0.204686, 0.181759, 0.175125, 0.138312, 0.0697327], + result["geometric_fractions"], + ) self.assertEqual(20, len(result["pitch_angle_bins"])) self.assertEqual(13.5, result["pitch_angle_bins"][1]) self.assertEqual(20, len(result["pitch_angle_deltas"])) @@ -61,7 +68,8 @@ def test_read_swe_config(self): def test_read_l1b_swe_data(self): result: SweL1bData = read_l1b_swe_data( - get_test_data_path('swe/imap_swe_l1b_sci_20240510_v002.cdf')) + get_test_data_path("swe/imap_swe_l1b_sci_20240510_v002.cdf") + ) self.assertEqual((6,), result.epoch.shape) self.assertEqual(datetime(2010, 1, 1, 0, 0), result.epoch[0]) @@ -75,7 +83,9 @@ def test_read_l1b_swe_data(self): self.assertEqual(3333, result.settle_duration[0, 0]) def test_read_l2_swe_data(self): - result: SweL2Data = read_l2_swe_data(get_test_data_path('swe/imap_swe_l2_sci_20260512_v001-trimmed.cdf')) + result: SweL2Data = read_l2_swe_data( + get_test_data_path("swe/imap_swe_l2_sci_20260512_v001-trimmed.cdf") + ) self.assertEqual(result.epoch[0], datetime(2026, 5, 11, 23, 59, 45, 714250)) self.assertEqual(len(result.epoch), 6) @@ -105,11 +115,16 @@ def test_read_l2_swe_data(self): self.assertTrue(np.all(np.isnan(result.inst_az_spin_sector[1]))) spiceypy.sct2e(-43, 0.1) self.assertEqual(result.phase_space_density.shape, (6, 24, 30, 7)) - self.assertEqual(result.phase_space_density[3][11][8][4], 3.1664245168848865e-28) + self.assertEqual( + result.phase_space_density[3][11][8][4], 3.1664245168848865e-28 + ) self.assertTrue(np.all(np.isnan(result.phase_space_density[1]))) self.assertEqual(result.acquisition_time.shape, (6, 24, 30)) - self.assertEqual(result.acquisition_time[0][0][0], np.datetime64('2026-05-11T23:59:28.899551988')) + self.assertEqual( + result.acquisition_time[0][0][0], + np.datetime64("2026-05-11T23:59:28.899551988"), + ) self.assertTrue(np.all(np.isnat(result.acquisition_time[1]))) self.assertEqual(result.acquisition_duration.shape, (6, 24, 30)) @@ -117,7 +132,9 @@ def test_read_l2_swe_data(self): self.assertTrue(np.all(result.acquisition_duration[1] == 0)) self.assertEqual(result.phase_space_density_rebinned.shape, (6, 24, 30, 7)) - self.assertEqual(result.phase_space_density_rebinned[3][11][8][4], 1.2744421815220178e-27) + self.assertEqual( + result.phase_space_density_rebinned[3][11][8][4], 1.2744421815220178e-27 + ) self.assertTrue(np.isnan(result.phase_space_density_rebinned[0][0][0][0])) self.assertTrue(np.all(np.isnan(result.phase_space_density_rebinned[1]))) @@ -127,8 +144,8 @@ def test_read_l2_swe_data(self): def test_read_l3a_swapi_proton_data(self): with tempfile.TemporaryDirectory() as tempdir: cdf_path = _copy_with_rtn_velocity( - get_test_data_path('swe/imap_swapi_l3a_proton-sw_20250101_v001.cdf'), - Path(tempdir, 'swapi_l3a_with_rtn.cdf'), + get_test_data_path("swe/imap_swapi_l3a_proton-sw_20250101_v001.cdf"), + Path(tempdir, "swapi_l3a_with_rtn.cdf"), ) result = read_l3a_swapi_proton_data(cdf_path) self.assertIsInstance(result, SwapiL3aProtonData) @@ -139,18 +156,20 @@ def test_read_l3a_swapi_proton_data(self): self.assertEqual(datetime(2025, 1, 1), result.epoch[0]) self.assertEqual(timedelta(seconds=30), result.epoch_delta[0]) np.testing.assert_array_equal( - np.array([1, 0, 0, 0, 0, 0, 0, 0, 0, 0], dtype=np.uint16), result.swp_flags, strict=True + np.array([1, 0, 0, 0, 0, 0, 0, 0, 0, 0], dtype=np.uint16), + result.swp_flags, + strict=True, ) def test_read_l3a_swapi_proton_data_with_fill_values(self): with tempfile.TemporaryDirectory() as tempdir: cdf_with_fill_path = _copy_with_rtn_velocity( - get_test_data_path('swe/imap_swapi_l3a_proton-sw_20250101_v001.cdf'), - Path(tempdir, 'swe_file_with_fill.cdf'), + get_test_data_path("swe/imap_swapi_l3a_proton-sw_20250101_v001.cdf"), + Path(tempdir, "swe_file_with_fill.cdf"), ) with CDF(str(cdf_with_fill_path), readonly=False) as cdf: - velocity_fill_value = cdf['proton_sw_velocity_rtn'].attrs['FILLVAL'] - cdf['proton_sw_velocity_rtn'][0] = [velocity_fill_value] * 3 + velocity_fill_value = cdf["proton_sw_velocity_rtn"].attrs["FILLVAL"] + cdf["proton_sw_velocity_rtn"][0] = [velocity_fill_value] * 3 swapi_l3a_data = read_l3a_swapi_proton_data(cdf_with_fill_path) self.assertTrue(np.all(np.isnan(swapi_l3a_data.proton_sw_velocity_rtn[0]))) @@ -158,10 +177,12 @@ def test_read_l3a_swapi_proton_data_with_fill_values(self): def test_compute_epoch_delta_in_ns(self): acq_duration_microseconds = np.full((4, 24, 30), 80_000) settle_duration_microseconds = np.full((4, 4), 10000 / 3) - result = compute_epoch_delta_in_ns(acq_duration_microseconds, settle_duration_microseconds) + result = compute_epoch_delta_in_ns( + acq_duration_microseconds, settle_duration_microseconds + ) expected = np.full(4, 30 * 1e9) np.testing.assert_array_equal(expected, result) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/swe/test_swe_processor.py b/tests/swe/test_swe_processor.py index d936f5ea2..ba0965dc4 100644 --- a/tests/swe/test_swe_processor.py +++ b/tests/swe/test_swe_processor.py @@ -4,7 +4,11 @@ from unittest.mock import patch, call, Mock, sentinel, create_autospec import numpy as np -from imap_data_access.processing_input import ScienceInput, ProcessingInputCollection, AncillaryInput +from imap_data_access.processing_input import ( + ScienceInput, + ProcessingInputCollection, + AncillaryInput, +) from imap_processing.quality_flags import SweL1bFlags from spiceypy import SpiceyError @@ -13,34 +17,56 @@ from imap_l3_processing.predicted_ephemeris_tracker import PredictedEphemerisTracker from imap_l3_processing.swe.l3.models import SweL2Data, SwapiL3aProtonData, SweL1bData from imap_l3_processing.swe.l3.models import SweL3MomentData -from imap_l3_processing.swe.l3.science.moment_calculations import IntegrateOutputs, MomentFitResults, ScaleDensityOutput +from imap_l3_processing.swe.l3.science.moment_calculations import ( + IntegrateOutputs, + MomentFitResults, + ScaleDensityOutput, +) from imap_l3_processing.swe.l3.science.moment_calculations import Moments from imap_l3_processing.swe.l3.swe_l3_dependencies import SweL3Dependencies from imap_l3_processing.swe.quality_flags import SweL3Flags -from imap_l3_processing.swe.swe_processor import SweProcessor, check_and_mask_negative_moments, logger -from tests.test_helpers import NumpyArrayMatcher, build_swe_configuration, create_dataclass_mock, build_moments, \ - build_moment_fit_results, build_swe_moment_data +from imap_l3_processing.swe.swe_processor import ( + SweProcessor, + check_and_mask_negative_moments, + logger, +) +from tests.test_helpers import ( + NumpyArrayMatcher, + build_swe_configuration, + create_dataclass_mock, + build_moments, + build_moment_fit_results, + build_swe_moment_data, +) MODULE = "imap_l3_processing.swe.swe_processor" class TestSweProcessor(unittest.TestCase): - @patch('imap_l3_processing.processor.spiceypy') - @patch('imap_l3_processing.swe.swe_processor.save_data') - @patch('imap_l3_processing.swe.swe_processor.SweL3Dependencies.fetch_dependencies') - @patch('imap_l3_processing.swe.swe_processor.SweProcessor.calculate_products') - def test_process(self, mock_calculate_products, mock_fetch_dependencies, mock_save_data, mock_spiceypy): + @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.swe.swe_processor.save_data") + @patch("imap_l3_processing.swe.swe_processor.SweL3Dependencies.fetch_dependencies") + @patch("imap_l3_processing.swe.swe_processor.SweProcessor.calculate_products") + def test_process( + self, + mock_calculate_products, + mock_fetch_dependencies, + mock_save_data, + mock_spiceypy, + ): input_file_names = [ "imap_swe_l2_sci_20200101_v000.cdf", "imap_swe_l1b_sci_20200101_v000.cdf", "imap_mag_l1d_norm-dsrf_20200101_v000.cdf", "imap_swapi_l3_proton-sw_20200101_v000.cdf", - "imap_swe_config-json-not-cdf_20200101_v000.cdf" + "imap_swe_config-json-not-cdf_20200101_v000.cdf", ] mock_spiceypy.ktotal.return_value = 0 - science_inputs = [ScienceInput(file_name) for file_name in input_file_names[0:4]] + science_inputs = [ + ScienceInput(file_name) for file_name in input_file_names[0:4] + ] config_input = AncillaryInput(input_file_names[4]) mock_dependencies = ProcessingInputCollection(*science_inputs, config_input) @@ -53,24 +79,45 @@ def test_process(self, mock_calculate_products, mock_fetch_dependencies, mock_sa self.assertEqual(calculate_products_return.parent_file_names, input_file_names) mock_fetch_dependencies.assert_called_once_with(mock_dependencies) - mock_calculate_products.assert_called_once_with(mock_fetch_dependencies.return_value) + mock_calculate_products.assert_called_once_with( + mock_fetch_dependencies.return_value + ) mock_save_data.assert_called_once_with(mock_calculate_products.return_value) self.assertEqual([mock_save_data.return_value], product) - @patch('imap_l3_processing.swe.swe_processor.check_and_mask_negative_moments') + @patch("imap_l3_processing.swe.swe_processor.check_and_mask_negative_moments") @patch("imap_l3_processing.swe.swe_processor.compute_epoch_delta_in_ns") - @patch('imap_l3_processing.swe.swe_processor.average_over_look_directions') - @patch('imap_l3_processing.swe.swe_processor.mec_breakpoint_finder') - @patch('imap_l3_processing.swe.swe_processor.SweProcessor.calculate_pitch_angle_products') - @patch('imap_l3_processing.swe.swe_processor.SweProcessor.calculate_moment_products') - def test_calculate_products(self, mock_calculate_moment_products, mock_calculate_pitch_angle_products, - mock_breakpoint_finder, mock_average_over_look_directions, - mock_compute_epoch_delta_in_ns, mock_check_and_mask_negative_moments): + @patch("imap_l3_processing.swe.swe_processor.average_over_look_directions") + @patch("imap_l3_processing.swe.swe_processor.mec_breakpoint_finder") + @patch( + "imap_l3_processing.swe.swe_processor.SweProcessor.calculate_pitch_angle_products" + ) + @patch( + "imap_l3_processing.swe.swe_processor.SweProcessor.calculate_moment_products" + ) + def test_calculate_products( + self, + mock_calculate_moment_products, + mock_calculate_pitch_angle_products, + mock_breakpoint_finder, + mock_average_over_look_directions, + mock_compute_epoch_delta_in_ns, + mock_check_and_mask_negative_moments, + ): mock_compute_epoch_delta_in_ns.return_value = [30e9, 40e9] epochs = datetime.now() + np.arange(7) * timedelta(minutes=1) - swapi_epochs = datetime.now() - timedelta(seconds=15) + np.arange(2) * timedelta(minutes=.5) + swapi_epochs = ( + datetime.now() + - timedelta(seconds=15) + + np.arange(2) * timedelta(minutes=0.5) + ) mock_breakpoint_finder.side_effect = [ - (1, 2, SweL3Flags.FALLBACK_POTENTIAL_ESTIMATE | SweL3Flags.POTENTIAL_FIT_UNCONVERGED), + ( + 1, + 2, + SweL3Flags.FALLBACK_POTENTIAL_ESTIMATE + | SweL3Flags.POTENTIAL_FIT_UNCONVERGED, + ), (3, 4, SweL3Flags.NONE), (5, 6, SweL3Flags.NONE), (7, 8, SweL3Flags.FALLBACK_POTENTIAL_ESTIMATE), @@ -118,15 +165,17 @@ def test_calculate_products(self, mock_calculate_moment_products, mock_calculate data_quality=l2_data_quality, ) - expected_corrected_energy_bins = np.array([ - [2 - 1, 4 - 1, 6 - 1], - [2 - 3, 4 - 3, 6 - 3], - [2 - 5, 4 - 5, 6 - 5], - [2 - 7, 4 - 7, 6 - 7], - [2 - 9, 4 - 9, 6 - 9], - [2 - 11, 4 - 11, 6 - 11], - [2 - 13, 4 - 13, 6 - 13], - ]) + expected_corrected_energy_bins = np.array( + [ + [2 - 1, 4 - 1, 6 - 1], + [2 - 3, 4 - 3, 6 - 3], + [2 - 5, 4 - 5, 6 - 5], + [2 - 7, 4 - 7, 6 - 7], + [2 - 9, 4 - 9, 6 - 9], + [2 - 11, 4 - 11, 6 - 11], + [2 - 13, 4 - 13, 6 - 13], + ] + ) mag_l1d_data = Mock() @@ -150,34 +199,47 @@ def test_calculate_products(self, mock_calculate_moment_products, mock_calculate spacecraft_potential_initial_guess=15, core_halo_breakpoint_initial_guess=90, gyrophase_bins=[90, 180, 270], - gyrophase_delta=[90, 90, 90] + gyrophase_delta=[90, 90, 90], ) mock_moment_data = build_swe_moment_data(len(epochs)) - mock_moment_data.quality_flags = np.array([ - SweL3Flags.NONE, - SweL3Flags.NONE, - SweL3Flags.PREDICTIVE_EPHEMERIS, - SweL3Flags.NONE, - SweL3Flags.PREDICTIVE_EPHEMERIS, - SweL3Flags.NONE, - SweL3Flags.NONE, - ], dtype=np.uint16) + mock_moment_data.quality_flags = np.array( + [ + SweL3Flags.NONE, + SweL3Flags.NONE, + SweL3Flags.PREDICTIVE_EPHEMERIS, + SweL3Flags.NONE, + SweL3Flags.PREDICTIVE_EPHEMERIS, + SweL3Flags.NONE, + SweL3Flags.NONE, + ], + dtype=np.uint16, + ) mock_calculate_moment_products.return_value = mock_moment_data - negative_moment_flags_return = np.array([SweL3Flags.NONE] * len(epochs), dtype=np.uint16) + negative_moment_flags_return = np.array( + [SweL3Flags.NONE] * len(epochs), dtype=np.uint16 + ) negative_moment_flags_return[4] = SweL3Flags.NEGATIVE_MOMENT mock_check_and_mask_negative_moments.return_value = negative_moment_flags_return - calculate_pitch_angle_flags = np.array([SweL3Flags.NONE] * len(epochs), dtype=np.uint16) + calculate_pitch_angle_flags = np.array( + [SweL3Flags.NONE] * len(epochs), dtype=np.uint16 + ) calculate_pitch_angle_flags[0] = SweL3Flags.FALLBACK_SWAPI_SPEED calculate_pitch_angle_flags[2] = SweL3Flags.FALLBACK_SWAPI_SPEED - expected_phase_space_density_by_pitch_angle = np.full((7, 3, 3), physical_psd_value) + expected_phase_space_density_by_pitch_angle = np.full( + (7, 3, 3), physical_psd_value + ) expected_phase_space_density_by_pitch_angle[1, 0, 0] = unphysical_psd_value - expected_phase_space_density_by_pitch_angle_and_gyrophase = np.full((7, 3, 3, 3), physical_psd_value) - expected_phase_space_density_by_pitch_angle_and_gyrophase[2, 0, 0, 0] = unphysical_psd_value + expected_phase_space_density_by_pitch_angle_and_gyrophase = np.full( + (7, 3, 3, 3), physical_psd_value + ) + expected_phase_space_density_by_pitch_angle_and_gyrophase[2, 0, 0, 0] = ( + unphysical_psd_value + ) expected_energy_spectrum = np.full((7, 3), physical_psd_value) expected_energy_spectrum[3, 0] = unphysical_psd_value expected_energy_spectrum_inbound = np.full((7, 3), physical_psd_value) @@ -191,92 +253,169 @@ def test_calculate_products(self, mock_calculate_moment_products, mock_calculate expected_energy_spectrum, expected_energy_spectrum_inbound, expected_energy_spectrum_outbound, - sentinel.intensity_by_pitch_angle_and_gyrophase, sentinel.intensity_by_pitch_angle, - sentinel.uncertainty_by_pitch_angle_and_gyrophase, sentinel.uncertainty_by_pitch_angle, - calculate_pitch_angle_flags + sentinel.intensity_by_pitch_angle_and_gyrophase, + sentinel.intensity_by_pitch_angle, + sentinel.uncertainty_by_pitch_angle_and_gyrophase, + sentinel.uncertainty_by_pitch_angle, + calculate_pitch_angle_flags, ) - input_metadata = InputMetadata("swe", "l3", datetime(2025, 2, 21), - datetime(2025, 2, 22), "v001") + input_metadata = InputMetadata( + "swe", "l3", datetime(2025, 2, 21), datetime(2025, 2, 22), "v001" + ) swe_l1b_data = Mock() - swe_l3_dependency = SweL3Dependencies(swe_l2_data, swe_l1b_data, mag_l1d_data, swapi_l3a_proton_data, - swe_config, - mag_is_preliminary=True) + swe_l3_dependency = SweL3Dependencies( + swe_l2_data, + swe_l1b_data, + mag_l1d_data, + swapi_l3a_proton_data, + swe_config, + mag_is_preliminary=True, + ) swe_processor = SweProcessor(swe_l3_dependency, input_metadata) swe_l3_data = swe_processor.calculate_products(swe_l3_dependency) - mock_compute_epoch_delta_in_ns.assert_called_once_with(swe_l2_data.acquisition_duration, - swe_l1b_data.settle_duration) + mock_compute_epoch_delta_in_ns.assert_called_once_with( + swe_l2_data.acquisition_duration, swe_l1b_data.settle_duration + ) self.assertEqual(7, mock_average_over_look_directions.call_count) - first_average_over_look_directions_call_args = mock_average_over_look_directions.call_args_list[0].args - np.testing.assert_array_equal(first_average_over_look_directions_call_args[0], - swe_l2_data.phase_space_density[0]) - np.testing.assert_array_equal(first_average_over_look_directions_call_args[1], - swe_config["geometric_fractions"]) - np.testing.assert_array_equal(first_average_over_look_directions_call_args[2], - swe_config["minimum_phase_space_density_value"]) - - middle_average_over_look_directions_call_args = mock_average_over_look_directions.call_args_list[3].args - np.testing.assert_array_equal(middle_average_over_look_directions_call_args[0], - np.mean([ - swe_l2_data.phase_space_density[0], - swe_l2_data.phase_space_density[1], - swe_l2_data.phase_space_density[2], - swe_l2_data.phase_space_density[3], - swe_l2_data.phase_space_density[4], - swe_l2_data.phase_space_density[5], - swe_l2_data.phase_space_density[6], - ], axis=0)) - np.testing.assert_array_equal(middle_average_over_look_directions_call_args[1], - swe_config["geometric_fractions"]) - np.testing.assert_array_equal(middle_average_over_look_directions_call_args[2], - swe_config["minimum_phase_space_density_value"]) - - last_average_over_look_directions_call_args = mock_average_over_look_directions.call_args_list[6].args - np.testing.assert_array_equal(last_average_over_look_directions_call_args[0], - np.mean([ - swe_l2_data.phase_space_density[3], - swe_l2_data.phase_space_density[4], - swe_l2_data.phase_space_density[5], - swe_l2_data.phase_space_density[6], - ], axis=0)) - np.testing.assert_array_equal(last_average_over_look_directions_call_args[1], - swe_config["geometric_fractions"]) - np.testing.assert_array_equal(last_average_over_look_directions_call_args[2], - swe_config["minimum_phase_space_density_value"]) - - mock_breakpoint_finder.assert_has_calls([ - call(swe_l2_data.energy, mock_average_over_look_directions.return_value), - call(swe_l2_data.energy, mock_average_over_look_directions.return_value), - call(swe_l2_data.energy, mock_average_over_look_directions.return_value), - call(swe_l2_data.energy, mock_average_over_look_directions.return_value), - call(swe_l2_data.energy, mock_average_over_look_directions.return_value), - call(swe_l2_data.energy, mock_average_over_look_directions.return_value), - call(swe_l2_data.energy, mock_average_over_look_directions.return_value), - ]) - - mag_l1d_data.rebin_to.assert_called_with(epochs, [timedelta(seconds=30), timedelta(seconds=40)]) + first_average_over_look_directions_call_args = ( + mock_average_over_look_directions.call_args_list[0].args + ) + np.testing.assert_array_equal( + first_average_over_look_directions_call_args[0], + swe_l2_data.phase_space_density[0], + ) + np.testing.assert_array_equal( + first_average_over_look_directions_call_args[1], + swe_config["geometric_fractions"], + ) + np.testing.assert_array_equal( + first_average_over_look_directions_call_args[2], + swe_config["minimum_phase_space_density_value"], + ) + + middle_average_over_look_directions_call_args = ( + mock_average_over_look_directions.call_args_list[3].args + ) + np.testing.assert_array_equal( + middle_average_over_look_directions_call_args[0], + np.mean( + [ + swe_l2_data.phase_space_density[0], + swe_l2_data.phase_space_density[1], + swe_l2_data.phase_space_density[2], + swe_l2_data.phase_space_density[3], + swe_l2_data.phase_space_density[4], + swe_l2_data.phase_space_density[5], + swe_l2_data.phase_space_density[6], + ], + axis=0, + ), + ) + np.testing.assert_array_equal( + middle_average_over_look_directions_call_args[1], + swe_config["geometric_fractions"], + ) + np.testing.assert_array_equal( + middle_average_over_look_directions_call_args[2], + swe_config["minimum_phase_space_density_value"], + ) + + last_average_over_look_directions_call_args = ( + mock_average_over_look_directions.call_args_list[6].args + ) + np.testing.assert_array_equal( + last_average_over_look_directions_call_args[0], + np.mean( + [ + swe_l2_data.phase_space_density[3], + swe_l2_data.phase_space_density[4], + swe_l2_data.phase_space_density[5], + swe_l2_data.phase_space_density[6], + ], + axis=0, + ), + ) + np.testing.assert_array_equal( + last_average_over_look_directions_call_args[1], + swe_config["geometric_fractions"], + ) + np.testing.assert_array_equal( + last_average_over_look_directions_call_args[2], + swe_config["minimum_phase_space_density_value"], + ) + + mock_breakpoint_finder.assert_has_calls( + [ + call( + swe_l2_data.energy, mock_average_over_look_directions.return_value + ), + call( + swe_l2_data.energy, mock_average_over_look_directions.return_value + ), + call( + swe_l2_data.energy, mock_average_over_look_directions.return_value + ), + call( + swe_l2_data.energy, mock_average_over_look_directions.return_value + ), + call( + swe_l2_data.energy, mock_average_over_look_directions.return_value + ), + call( + swe_l2_data.energy, mock_average_over_look_directions.return_value + ), + call( + swe_l2_data.energy, mock_average_over_look_directions.return_value + ), + ] + ) + + mag_l1d_data.rebin_to.assert_called_with( + epochs, [timedelta(seconds=30), timedelta(seconds=40)] + ) mock_calculate_moment_products.assert_called_once() self.assertEqual(swe_l2_data, mock_calculate_moment_products.call_args[0][0]) self.assertEqual(swe_l1b_data, mock_calculate_moment_products.call_args[0][1]) - self.assertEqual(mag_l1d_data.rebin_to.return_value, mock_calculate_moment_products.call_args[0][2]) - np.testing.assert_array_equal(mock_calculate_moment_products.call_args[0][3], expected_spacecraft_potential) - np.testing.assert_array_equal(mock_calculate_moment_products.call_args[0][4], expected_core_halo_breakpoint) - np.testing.assert_array_equal(mock_calculate_moment_products.call_args[0][5], expected_corrected_energy_bins) + self.assertEqual( + mag_l1d_data.rebin_to.return_value, + mock_calculate_moment_products.call_args[0][2], + ) + np.testing.assert_array_equal( + mock_calculate_moment_products.call_args[0][3], + expected_spacecraft_potential, + ) + np.testing.assert_array_equal( + mock_calculate_moment_products.call_args[0][4], + expected_core_halo_breakpoint, + ) + np.testing.assert_array_equal( + mock_calculate_moment_products.call_args[0][5], + expected_corrected_energy_bins, + ) self.assertEqual(swe_config, mock_calculate_moment_products.call_args[0][6]) mock_calculate_pitch_angle_products.assert_called_once() - self.assertEqual(swe_l3_dependency, mock_calculate_pitch_angle_products.call_args[0][0]) - np.testing.assert_array_equal(mock_calculate_pitch_angle_products.call_args[0][1], - expected_corrected_energy_bins) + self.assertEqual( + swe_l3_dependency, mock_calculate_pitch_angle_products.call_args[0][0] + ) + np.testing.assert_array_equal( + mock_calculate_pitch_angle_products.call_args[0][1], + expected_corrected_energy_bins, + ) # @formatter:off - self.assertEqual(swe_l3_data.input_metadata, replace(input_metadata, descriptor="sci")) + self.assertEqual( + swe_l3_data.input_metadata, replace(input_metadata, descriptor="sci") + ) # pass through from l2 np.testing.assert_array_equal(swe_l3_data.epoch, swe_l2_data.epoch) - np.testing.assert_array_equal(swe_l3_data.epoch_delta, mock_compute_epoch_delta_in_ns.return_value) + np.testing.assert_array_equal( + swe_l3_data.epoch_delta, mock_compute_epoch_delta_in_ns.return_value + ) np.testing.assert_array_equal(swe_l3_data.inst_el, swe_l2_data.inst_el) self.assertEqual(swe_l3_data.inst_el_label, swe_l2_data.inst_el_label) self.assertEqual(swe_l3_data.inst_az, swe_l2_data.inst_az) @@ -284,33 +423,67 @@ def test_calculate_products(self, mock_calculate_moment_products, mock_calculate # coming from the config np.testing.assert_array_equal(swe_l3_data.energy, swe_config["energy_bins"]) - np.testing.assert_array_equal(swe_l3_data.energy_delta_plus, swe_config["energy_delta_plus"]) - np.testing.assert_array_equal(swe_l3_data.energy_delta_minus, swe_config["energy_delta_minus"]) - np.testing.assert_array_equal(swe_l3_data.pitch_angle, swe_config["pitch_angle_bins"]) - np.testing.assert_array_equal(swe_l3_data.pitch_angle_delta, swe_config["pitch_angle_deltas"]) - np.testing.assert_array_equal(swe_l3_data.gyrophase_bins, swe_config["gyrophase_bins"]) - np.testing.assert_array_equal(swe_l3_data.gyrophase_delta, swe_config["gyrophase_deltas"]) - np.testing.assert_array_equal(swe_l3_data.intensity_by_pitch_angle_and_gyrophase, - sentinel.intensity_by_pitch_angle_and_gyrophase) - np.testing.assert_array_equal(swe_l3_data.intensity_by_pitch_angle, sentinel.intensity_by_pitch_angle) - np.testing.assert_array_equal(swe_l3_data.intensity_uncertainty_by_pitch_angle_and_gyrophase, - sentinel.uncertainty_by_pitch_angle_and_gyrophase) - np.testing.assert_array_equal(swe_l3_data.intensity_uncertainty_by_pitch_angle, - sentinel.uncertainty_by_pitch_angle) + np.testing.assert_array_equal( + swe_l3_data.energy_delta_plus, swe_config["energy_delta_plus"] + ) + np.testing.assert_array_equal( + swe_l3_data.energy_delta_minus, swe_config["energy_delta_minus"] + ) + np.testing.assert_array_equal( + swe_l3_data.pitch_angle, swe_config["pitch_angle_bins"] + ) + np.testing.assert_array_equal( + swe_l3_data.pitch_angle_delta, swe_config["pitch_angle_deltas"] + ) + np.testing.assert_array_equal( + swe_l3_data.gyrophase_bins, swe_config["gyrophase_bins"] + ) + np.testing.assert_array_equal( + swe_l3_data.gyrophase_delta, swe_config["gyrophase_deltas"] + ) + np.testing.assert_array_equal( + swe_l3_data.intensity_by_pitch_angle_and_gyrophase, + sentinel.intensity_by_pitch_angle_and_gyrophase, + ) + np.testing.assert_array_equal( + swe_l3_data.intensity_by_pitch_angle, sentinel.intensity_by_pitch_angle + ) + np.testing.assert_array_equal( + swe_l3_data.intensity_uncertainty_by_pitch_angle_and_gyrophase, + sentinel.uncertainty_by_pitch_angle_and_gyrophase, + ) + np.testing.assert_array_equal( + swe_l3_data.intensity_uncertainty_by_pitch_angle, + sentinel.uncertainty_by_pitch_angle, + ) # need for both moments and pitch angle - np.testing.assert_array_equal(swe_l3_data.spacecraft_potential, expected_spacecraft_potential) - np.testing.assert_array_equal(swe_l3_data.core_halo_breakpoint, expected_core_halo_breakpoint) + np.testing.assert_array_equal( + swe_l3_data.spacecraft_potential, expected_spacecraft_potential + ) + np.testing.assert_array_equal( + swe_l3_data.core_halo_breakpoint, expected_core_halo_breakpoint + ) # pitch angle specific - np.testing.assert_array_equal(swe_l3_data.phase_space_density_by_pitch_angle, - expected_phase_space_density_by_pitch_angle) - np.testing.assert_array_equal(swe_l3_data.phase_space_density_1d, expected_energy_spectrum) - np.testing.assert_array_equal(swe_l3_data.phase_space_density_inward, expected_energy_spectrum_inbound) - np.testing.assert_array_equal(swe_l3_data.phase_space_density_outward, expected_energy_spectrum_outbound) + np.testing.assert_array_equal( + swe_l3_data.phase_space_density_by_pitch_angle, + expected_phase_space_density_by_pitch_angle, + ) + np.testing.assert_array_equal( + swe_l3_data.phase_space_density_1d, expected_energy_spectrum + ) + np.testing.assert_array_equal( + swe_l3_data.phase_space_density_inward, expected_energy_spectrum_inbound + ) + np.testing.assert_array_equal( + swe_l3_data.phase_space_density_outward, expected_energy_spectrum_outbound + ) self.assertEqual(mock_moment_data, swe_l3_data.moment_data) - np.testing.assert_array_equal(swe_l3_data.phase_space_density_by_pitch_angle_and_gyrophase, - expected_phase_space_density_by_pitch_angle_and_gyrophase) + np.testing.assert_array_equal( + swe_l3_data.phase_space_density_by_pitch_angle_and_gyrophase, + expected_phase_space_density_by_pitch_angle_and_gyrophase, + ) expected_phi_avg_at_unphysical_cell = ( unphysical_psd_value * geometric_fractions[0] + physical_psd_value * (sum_geometric_fractions - geometric_fractions[0]) @@ -318,38 +491,69 @@ def test_calculate_products(self, mock_calculate_moment_products, mock_calculate expected_psd_by_phi = np.full((7, 3, 5), physical_psd_value) expected_psd_by_phi[6, 0, 0] = expected_phi_avg_at_unphysical_cell - np.testing.assert_allclose(swe_l3_data.raw_psd_by_phi_rebinned, expected_psd_by_phi) + np.testing.assert_allclose( + swe_l3_data.raw_psd_by_phi_rebinned, expected_psd_by_phi + ) expected_1d_psd = np.full((7, 3), physical_psd_value) - expected_1d_psd[6, 0] = (expected_phi_avg_at_unphysical_cell + 4 * physical_psd_value) / 5 + expected_1d_psd[6, 0] = ( + expected_phi_avg_at_unphysical_cell + 4 * physical_psd_value + ) / 5 np.testing.assert_allclose(swe_l3_data.raw_1d_psd_rebinned, expected_1d_psd) expected_psd_by_theta = np.full((7, 3, 7), physical_psd_value) - expected_psd_by_theta[6, 0, 0] = (unphysical_psd_value + 4 * physical_psd_value) / 5 - np.testing.assert_allclose(swe_l3_data.raw_psd_by_theta_rebinned, expected_psd_by_theta) - - expected_quality_flags = np.array([ - SweL3Flags.FALLBACK_SWAPI_SPEED | SweL3Flags.FALLBACK_POTENTIAL_ESTIMATE | SweL3Flags.POTENTIAL_FIT_UNCONVERGED | SweL3Flags.PRELIMINARY_MAG, - SweL3Flags.PRELIMINARY_MAG | SweL3Flags.FALLBACK_CALIBRATION_EXTRAPOLATED | SweL3Flags.UNPHYSICAL_PSD, - SweL3Flags.PRELIMINARY_MAG | SweL3Flags.FALLBACK_CALIBRATION_EXTRAPOLATED | SweL3Flags.UNPHYSICAL_PSD | SweL3Flags.PREDICTIVE_EPHEMERIS | SweL3Flags.FALLBACK_SWAPI_SPEED, - SweL3Flags.FALLBACK_POTENTIAL_ESTIMATE | SweL3Flags.PRELIMINARY_MAG | SweL3Flags.UNPHYSICAL_PSD, - SweL3Flags.PRELIMINARY_MAG | SweL3Flags.NEGATIVE_MOMENT | SweL3Flags.FALLBACK_CALIBRATION_EXTRAPOLATED | SweL3Flags.UNPHYSICAL_PSD | SweL3Flags.PREDICTIVE_EPHEMERIS, - SweL3Flags.PRELIMINARY_MAG | SweL3Flags.UNPHYSICAL_PSD, - SweL3Flags.PRELIMINARY_MAG | SweL3Flags.UNPHYSICAL_PSD, - ]) + expected_psd_by_theta[6, 0, 0] = ( + unphysical_psd_value + 4 * physical_psd_value + ) / 5 + np.testing.assert_allclose( + swe_l3_data.raw_psd_by_theta_rebinned, expected_psd_by_theta + ) + + expected_quality_flags = np.array( + [ + SweL3Flags.FALLBACK_SWAPI_SPEED + | SweL3Flags.FALLBACK_POTENTIAL_ESTIMATE + | SweL3Flags.POTENTIAL_FIT_UNCONVERGED + | SweL3Flags.PRELIMINARY_MAG, + SweL3Flags.PRELIMINARY_MAG + | SweL3Flags.FALLBACK_CALIBRATION_EXTRAPOLATED + | SweL3Flags.UNPHYSICAL_PSD, + SweL3Flags.PRELIMINARY_MAG + | SweL3Flags.FALLBACK_CALIBRATION_EXTRAPOLATED + | SweL3Flags.UNPHYSICAL_PSD + | SweL3Flags.PREDICTIVE_EPHEMERIS + | SweL3Flags.FALLBACK_SWAPI_SPEED, + SweL3Flags.FALLBACK_POTENTIAL_ESTIMATE + | SweL3Flags.PRELIMINARY_MAG + | SweL3Flags.UNPHYSICAL_PSD, + SweL3Flags.PRELIMINARY_MAG + | SweL3Flags.NEGATIVE_MOMENT + | SweL3Flags.FALLBACK_CALIBRATION_EXTRAPOLATED + | SweL3Flags.UNPHYSICAL_PSD + | SweL3Flags.PREDICTIVE_EPHEMERIS, + SweL3Flags.PRELIMINARY_MAG | SweL3Flags.UNPHYSICAL_PSD, + SweL3Flags.PRELIMINARY_MAG | SweL3Flags.UNPHYSICAL_PSD, + ] + ) np.testing.assert_array_equal(swe_l3_data.swe_flags, expected_quality_flags) mock_check_and_mask_negative_moments.assert_called_once() - self.assertIs(mock_moment_data, mock_check_and_mask_negative_moments.call_args[0][0]) + self.assertIs( + mock_moment_data, mock_check_and_mask_negative_moments.call_args[0][0] + ) - @patch('imap_l3_processing.swe.swe_processor.check_temperature_outlier_flag') - @patch('imap_l3_processing.swe.swe_processor.check_and_mask_negative_moments') + @patch("imap_l3_processing.swe.swe_processor.check_temperature_outlier_flag") + @patch("imap_l3_processing.swe.swe_processor.check_and_mask_negative_moments") @patch("imap_l3_processing.swe.swe_processor.compute_epoch_delta_in_ns") - @patch('imap_l3_processing.swe.swe_processor.average_over_look_directions') - @patch('imap_l3_processing.swe.swe_processor.mec_breakpoint_finder') - @patch('imap_l3_processing.swe.swe_processor.SweProcessor.calculate_pitch_angle_products') - @patch('imap_l3_processing.swe.swe_processor.SweProcessor.calculate_moment_products') + @patch("imap_l3_processing.swe.swe_processor.average_over_look_directions") + @patch("imap_l3_processing.swe.swe_processor.mec_breakpoint_finder") + @patch( + "imap_l3_processing.swe.swe_processor.SweProcessor.calculate_pitch_angle_products" + ) + @patch( + "imap_l3_processing.swe.swe_processor.SweProcessor.calculate_moment_products" + ) def test_negative_moment_mask_runs_before_temperature_outlier_checks( self, mock_calculate_moment_products, @@ -400,8 +604,12 @@ def test_negative_moment_mask_runs_before_temperature_outlier_checks( moment_data = build_swe_moment_data(num_epochs) mock_calculate_moment_products.return_value = moment_data - mock_check_and_mask_negative_moments.return_value = np.zeros(num_epochs, dtype=np.uint16) - mock_check_temperature_outlier_flag.return_value = np.zeros(num_epochs, dtype=np.uint16) + mock_check_and_mask_negative_moments.return_value = np.zeros( + num_epochs, dtype=np.uint16 + ) + mock_check_temperature_outlier_flag.return_value = np.zeros( + num_epochs, dtype=np.uint16 + ) mock_calculate_pitch_angle_products.return_value = ( np.zeros((num_epochs, 3, 3)), np.zeros((num_epochs, 3, 3, 3)), @@ -416,45 +624,71 @@ def test_negative_moment_mask_runs_before_temperature_outlier_checks( ) manager = Mock() - manager.attach_mock(mock_check_and_mask_negative_moments, 'mask') - manager.attach_mock(mock_check_temperature_outlier_flag, 'outlier') + manager.attach_mock(mock_check_and_mask_negative_moments, "mask") + manager.attach_mock(mock_check_temperature_outlier_flag, "outlier") - input_metadata = InputMetadata("swe", "l3", datetime(2025, 2, 21), datetime(2025, 2, 22), "v001") + input_metadata = InputMetadata( + "swe", "l3", datetime(2025, 2, 21), datetime(2025, 2, 22), "v001" + ) swe_l3_dependency = SweL3Dependencies( - swe_l2_data, Mock(), Mock(), swapi_l3a_proton_data, swe_config, + swe_l2_data, + Mock(), + Mock(), + swapi_l3a_proton_data, + swe_config, mag_is_preliminary=False, ) swe_processor = SweProcessor(swe_l3_dependency, input_metadata) swe_processor.calculate_products(swe_l3_dependency) method_call_order = [c[0] for c in manager.mock_calls] - self.assertEqual('mask', method_call_order[0]) - self.assertEqual(['outlier'] * 8, method_call_order[1:]) - - @patch('imap_l3_processing.swe.swe_processor.swe_rebin_intensity_by_pitch_angle_and_gyrophase') - @patch('imap_l3_processing.swe.swe_processor.calculate_velocity_in_dsp_frame_km_s') - @patch('imap_l3_processing.swe.swe_processor.average_over_look_directions') - @patch('imap_l3_processing.swe.swe_processor.mec_breakpoint_finder') - @patch('imap_l3_processing.swe.swe_processor.rotate_rtn_vectors_to_dps') - @patch('imap_l3_processing.swe.swe_processor.correct_and_rebin') - @patch('imap_l3_processing.swe.swe_processor.integrate_distribution_to_get_1d_spectrum') - @patch('imap_l3_processing.swe.swe_processor.integrate_distribution_to_get_inbound_and_outbound_1d_spectrum') - @patch('imap_l3_processing.swe.swe_processor.NearestInterpolator') - def test_calculate_pitch_angle_and_gyrophase_products(self, mock_nearest_interpolator, - mock_integrate_distribution_to_get_inbound_and_outbound_1d_spectrum, - mock_integrate_distribution_to_get_1d_spectrum, - mock_correct_and_rebin, - mock_rotate_rtn_vectors_to_dps, - _, - mock_average_over_look_directions, - mock_calculate_velocities, mock_swe_rebin_intensity): + self.assertEqual("mask", method_call_order[0]) + self.assertEqual(["outlier"] * 8, method_call_order[1:]) + + @patch( + "imap_l3_processing.swe.swe_processor.swe_rebin_intensity_by_pitch_angle_and_gyrophase" + ) + @patch("imap_l3_processing.swe.swe_processor.calculate_velocity_in_dsp_frame_km_s") + @patch("imap_l3_processing.swe.swe_processor.average_over_look_directions") + @patch("imap_l3_processing.swe.swe_processor.mec_breakpoint_finder") + @patch("imap_l3_processing.swe.swe_processor.rotate_rtn_vectors_to_dps") + @patch("imap_l3_processing.swe.swe_processor.correct_and_rebin") + @patch( + "imap_l3_processing.swe.swe_processor.integrate_distribution_to_get_1d_spectrum" + ) + @patch( + "imap_l3_processing.swe.swe_processor.integrate_distribution_to_get_inbound_and_outbound_1d_spectrum" + ) + @patch("imap_l3_processing.swe.swe_processor.NearestInterpolator") + def test_calculate_pitch_angle_and_gyrophase_products( + self, + mock_nearest_interpolator, + mock_integrate_distribution_to_get_inbound_and_outbound_1d_spectrum, + mock_integrate_distribution_to_get_1d_spectrum, + mock_correct_and_rebin, + mock_rotate_rtn_vectors_to_dps, + _, + mock_average_over_look_directions, + mock_calculate_velocities, + mock_swe_rebin_intensity, + ): epochs = np.arange(3) * timedelta(minutes=1) + datetime.now() - mag_epochs = np.arange(10) * timedelta(minutes=.5) + datetime.now() - timedelta(seconds=15) - swapi_epochs = np.arange(10) * timedelta(minutes=.5) + datetime.now() - timedelta(seconds=15) + mag_epochs = ( + np.arange(10) * timedelta(minutes=0.5) + + datetime.now() + - timedelta(seconds=15) + ) + swapi_epochs = ( + np.arange(10) * timedelta(minutes=0.5) + + datetime.now() + - timedelta(seconds=15) + ) spacecraft_potential = np.array([12, 16, 19]) energies = np.array([2, 4, 6]) - corrected_energy_bins = energies.reshape(1, -1) - spacecraft_potential.reshape(-1, 1) + corrected_energy_bins = energies.reshape(1, -1) - spacecraft_potential.reshape( + -1, 1 + ) pitch_angle_bins = [0, 90, 180] gyrophase_bins = [0, 360] @@ -478,12 +712,11 @@ def test_calculate_pitch_angle_and_gyrophase_products(self, mock_nearest_interpo swe_l1b_data = SweL1bData( epoch=epochs, count_rates=np.arange(27).reshape(3, 3, 3) + 10, - settle_duration=np.array([]) + settle_duration=np.array([]), ) mag_l1d_data = MagData( - epoch=mag_epochs, - mag_data=np.arange(7, 22).reshape(5, 3).repeat(2, axis=0) + epoch=mag_epochs, mag_data=np.arange(7, 22).reshape(5, 3).repeat(2, axis=0) ) swapi_l3a_proton_data = SwapiL3aProtonData( @@ -497,33 +730,58 @@ def test_calculate_pitch_angle_and_gyrophase_products(self, mock_nearest_interpo rotated_dps_vectors[3, :] = np.nan rotated_dps_vectors[4, :] = np.nan - mock_rotate_rtn_vectors_to_dps.return_value = rotated_dps_vectors, sentinel.solar_wind_uses_pred_ephem_flags - counts = swe_l1b_data.count_rates * swe_l2_data.acquisition_duration[:, :, np.newaxis] / 1e6 + mock_rotate_rtn_vectors_to_dps.return_value = ( + rotated_dps_vectors, + sentinel.solar_wind_uses_pred_ephem_flags, + ) + counts = ( + swe_l1b_data.count_rates + * swe_l2_data.acquisition_duration[:, :, np.newaxis] + / 1e6 + ) mock_average_over_look_directions.return_value = np.array([5, 10, 15]) closest_mag_data = np.arange(9).reshape(3, 3) closest_swapi_data = np.arange(8, 17).reshape(3, 3) - mock_interpolator_mag = create_autospec(NearestInterpolator, best_indices=sentinel.mag_best_indices) - mock_interpolator_swapi = create_autospec(NearestInterpolator, best_indices=np.array([3,4,5])) + mock_interpolator_mag = create_autospec( + NearestInterpolator, best_indices=sentinel.mag_best_indices + ) + mock_interpolator_swapi = create_autospec( + NearestInterpolator, best_indices=np.array([3, 4, 5]) + ) mock_interpolator_mag.interpolate_data.return_value = closest_mag_data mock_interpolator_swapi.interpolate_data.return_value = closest_swapi_data - mock_nearest_interpolator.side_effect = [mock_interpolator_mag, mock_interpolator_swapi] + mock_nearest_interpolator.side_effect = [ + mock_interpolator_mag, + mock_interpolator_swapi, + ] rebinned_by_pitch_list = [ - i + np.arange(len(swe_l2_data.energy) * len(pitch_angle_bins)).reshape(len(swe_l2_data.energy), - len(pitch_angle_bins)) for i in - range(len(epochs))] + i + + np.arange(len(swe_l2_data.energy) * len(pitch_angle_bins)).reshape( + len(swe_l2_data.energy), len(pitch_angle_bins) + ) + for i in range(len(epochs)) + ] rebinned_by_pitch_and_gyrophase_list = [ - i + np.arange(len(swe_l2_data.energy) * len(pitch_angle_bins) * len(gyrophase_bins)).reshape( - len(swe_l2_data.energy), - len(pitch_angle_bins), len(gyrophase_bins)) for i in - range(len(epochs))] + i + + np.arange( + len(swe_l2_data.energy) * len(pitch_angle_bins) * len(gyrophase_bins) + ).reshape( + len(swe_l2_data.energy), len(pitch_angle_bins), len(gyrophase_bins) + ) + for i in range(len(epochs)) + ] - rebinned_by_pitch_and_gyrophase = [(pa, pa_and_gyro) for pa, pa_and_gyro in - zip(rebinned_by_pitch_list, rebinned_by_pitch_and_gyrophase_list)] + rebinned_by_pitch_and_gyrophase = [ + (pa, pa_and_gyro) + for pa, pa_and_gyro in zip( + rebinned_by_pitch_list, rebinned_by_pitch_and_gyrophase_list + ) + ] mock_correct_and_rebin.side_effect = rebinned_by_pitch_and_gyrophase integrated_spectrum = np.arange(9).reshape(3, 3) + 11 @@ -532,12 +790,25 @@ def test_calculate_pitch_angle_and_gyrophase_products(self, mock_nearest_interpo expected_uncertainty_by_pa_and_gyro = np.arange(27).reshape(3, 3, 3) + 27 expected_uncertainty_by_pa = np.arange(27).reshape(3, 3, 3) + 28 mock_swe_rebin_intensity.side_effect = [ - (expected_intensity_by_pa_and_gyro[0], expected_intensity_by_pa[0], expected_uncertainty_by_pa_and_gyro[0], - expected_uncertainty_by_pa[0]), - (expected_intensity_by_pa_and_gyro[1], expected_intensity_by_pa[1], expected_uncertainty_by_pa_and_gyro[1], - expected_uncertainty_by_pa[1]), - (expected_intensity_by_pa_and_gyro[2], expected_intensity_by_pa[2], expected_uncertainty_by_pa_and_gyro[2], - expected_uncertainty_by_pa[2])] + ( + expected_intensity_by_pa_and_gyro[0], + expected_intensity_by_pa[0], + expected_uncertainty_by_pa_and_gyro[0], + expected_uncertainty_by_pa[0], + ), + ( + expected_intensity_by_pa_and_gyro[1], + expected_intensity_by_pa[1], + expected_uncertainty_by_pa_and_gyro[1], + expected_uncertainty_by_pa[1], + ), + ( + expected_intensity_by_pa_and_gyro[2], + expected_intensity_by_pa[2], + expected_uncertainty_by_pa_and_gyro[2], + expected_uncertainty_by_pa[2], + ), + ] mock_integrate_distribution_to_get_1d_spectrum.side_effect = integrated_spectrum @@ -546,10 +817,18 @@ def test_calculate_pitch_angle_and_gyrophase_products(self, mock_nearest_interpo mock_integrate_distribution_to_get_inbound_and_outbound_1d_spectrum.side_effect = [ (expected_inbound_spectrum[0], expected_outbound_spectrum[0]), (expected_inbound_spectrum[1], expected_outbound_spectrum[1]), - (expected_inbound_spectrum[2], expected_outbound_spectrum[2])] + (expected_inbound_spectrum[2], expected_outbound_spectrum[2]), + ] - geometric_fractions = [0.0697327, 0.138312, 0.175125, 0.181759, - 0.204686, 0.151448, 0.0781351] + geometric_fractions = [ + 0.0697327, + 0.138312, + 0.175125, + 0.181759, + 0.204686, + 0.151448, + 0.0781351, + ] swe_config = build_swe_configuration( geometric_fractions=geometric_fractions, pitch_angle_bins=pitch_angle_bins, @@ -564,9 +843,12 @@ def test_calculate_pitch_angle_and_gyrophase_products(self, mock_nearest_interpo core_halo_breakpoint_initial_guess=90, ) - input_metadata = InputMetadata("swe", "l3", datetime(2025, 2, 21), - datetime(2025, 2, 22), "v001") - swel3_dependency = SweL3Dependencies(swe_l2_data, swe_l1b_data, mag_l1d_data, swapi_l3a_proton_data, swe_config) + input_metadata = InputMetadata( + "swe", "l3", datetime(2025, 2, 21), datetime(2025, 2, 22), "v001" + ) + swel3_dependency = SweL3Dependencies( + swe_l2_data, swe_l1b_data, mag_l1d_data, swapi_l3a_proton_data, swe_config + ) swe_processor = SweProcessor(dependencies=[], input_metadata=input_metadata) mock_interpolator_swapi.interpolate_flags.side_effect = [ @@ -574,126 +856,233 @@ def test_calculate_pitch_angle_and_gyrophase_products(self, mock_nearest_interpo np.array([False, True, False]), ] - actual_phase_space_density_by_pitch_angle, actual_phase_space_density_by_pa_and_gyrophase, actual_energy_spectrum, actual_energy_spectrum_inbound, actual_energy_spectrum_outbound, \ - actual_intensity_by_pa_and_gyro, actual_intensity_by_pa, actual_uncertainty_by_pa_and_gyro, actual_uncertainty_by_pa, actual_swe_flags \ - = swe_processor.calculate_pitch_angle_products(swel3_dependency, corrected_energy_bins) + ( + actual_phase_space_density_by_pitch_angle, + actual_phase_space_density_by_pa_and_gyrophase, + actual_energy_spectrum, + actual_energy_spectrum_inbound, + actual_energy_spectrum_outbound, + actual_intensity_by_pa_and_gyro, + actual_intensity_by_pa, + actual_uncertainty_by_pa_and_gyro, + actual_uncertainty_by_pa, + actual_swe_flags, + ) = swe_processor.calculate_pitch_angle_products( + swel3_dependency, corrected_energy_bins + ) self.assertEqual(3, mock_correct_and_rebin.call_count) self.assertEqual(3, mock_integrate_distribution_to_get_1d_spectrum.call_count) mock_rotate_rtn_vectors_to_dps.assert_called_once() rotate_args = mock_rotate_rtn_vectors_to_dps.call_args[0] np.testing.assert_array_equal(rotate_args[0], swapi_epochs) - np.testing.assert_array_equal(rotate_args[1], swapi_l3a_proton_data.proton_sw_velocity_rtn) + np.testing.assert_array_equal( + rotate_args[1], swapi_l3a_proton_data.proton_sw_velocity_rtn + ) expected_dps_vectors_after_speed_fallback = rotated_dps_vectors.copy() expected_dps_vectors_after_speed_fallback[3] = [0.0, 0.0, -400.0] expected_dps_vectors_after_speed_fallback[4] = [0.0, 0.0, -400.0] - mock_nearest_interpolator.assert_has_calls([ - call( - from_epoch=mag_epochs, - from_data=mag_l1d_data.mag_data, - to_epoch=swe_l2_data.acquisition_time, - maximum_distance=np.timedelta64(1, 'm') - ), - call( - from_epoch=swapi_epochs, - from_data=NumpyArrayMatcher(expected_dps_vectors_after_speed_fallback), - to_epoch=epochs, - maximum_distance=np.timedelta64(5, 'm') - ) - ]) + mock_nearest_interpolator.assert_has_calls( + [ + call( + from_epoch=mag_epochs, + from_data=mag_l1d_data.mag_data, + to_epoch=swe_l2_data.acquisition_time, + maximum_distance=np.timedelta64(1, "m"), + ), + call( + from_epoch=swapi_epochs, + from_data=NumpyArrayMatcher( + expected_dps_vectors_after_speed_fallback + ), + to_epoch=epochs, + maximum_distance=np.timedelta64(5, "m"), + ), + ] + ) mock_interpolator_swapi.interpolate_flags.call_args_list[0][0] - expected_fallback_flags = [False, False, False, True, True, False, False, False, False, False,] + expected_fallback_flags = [ + False, + False, + False, + True, + True, + False, + False, + False, + False, + False, + ] self.assertEqual(2, mock_interpolator_swapi.interpolate_flags.call_count) - mock_interpolator_swapi.interpolate_flags.assert_has_calls([call(NumpyArrayMatcher(expected_fallback_flags)), call(sentinel.solar_wind_uses_pred_ephem_flags)], any_order=False) - + mock_interpolator_swapi.interpolate_flags.assert_has_calls( + [ + call(NumpyArrayMatcher(expected_fallback_flags)), + call(sentinel.solar_wind_uses_pred_ephem_flags), + ], + any_order=False, + ) - expected_flags = np.array([SweL3Flags.FALLBACK_SWAPI_SPEED, SweL3Flags.FALLBACK_SWAPI_SPEED | SweL3Flags.PREDICTIVE_EPHEMERIS, SweL3Flags.NONE]) + expected_flags = np.array( + [ + SweL3Flags.FALLBACK_SWAPI_SPEED, + SweL3Flags.FALLBACK_SWAPI_SPEED | SweL3Flags.PREDICTIVE_EPHEMERIS, + SweL3Flags.NONE, + ] + ) - np.testing.assert_array_equal(actual_phase_space_density_by_pitch_angle, rebinned_by_pitch_list) - np.testing.assert_array_equal(actual_phase_space_density_by_pa_and_gyrophase, - rebinned_by_pitch_and_gyrophase_list) + np.testing.assert_array_equal( + actual_phase_space_density_by_pitch_angle, rebinned_by_pitch_list + ) + np.testing.assert_array_equal( + actual_phase_space_density_by_pa_and_gyrophase, + rebinned_by_pitch_and_gyrophase_list, + ) np.testing.assert_array_equal(actual_energy_spectrum, integrated_spectrum) - np.testing.assert_array_equal(actual_energy_spectrum_inbound, expected_inbound_spectrum) - np.testing.assert_array_equal(actual_energy_spectrum_outbound, expected_outbound_spectrum) - np.testing.assert_array_equal(actual_energy_spectrum_outbound, expected_outbound_spectrum) - np.testing.assert_array_equal(actual_intensity_by_pa_and_gyro, expected_intensity_by_pa_and_gyro) + np.testing.assert_array_equal( + actual_energy_spectrum_inbound, expected_inbound_spectrum + ) + np.testing.assert_array_equal( + actual_energy_spectrum_outbound, expected_outbound_spectrum + ) + np.testing.assert_array_equal( + actual_energy_spectrum_outbound, expected_outbound_spectrum + ) + np.testing.assert_array_equal( + actual_intensity_by_pa_and_gyro, expected_intensity_by_pa_and_gyro + ) np.testing.assert_array_equal(actual_intensity_by_pa, expected_intensity_by_pa) - np.testing.assert_array_equal(actual_uncertainty_by_pa_and_gyro, expected_uncertainty_by_pa_and_gyro) - np.testing.assert_array_equal(actual_uncertainty_by_pa, expected_uncertainty_by_pa) + np.testing.assert_array_equal( + actual_uncertainty_by_pa_and_gyro, expected_uncertainty_by_pa_and_gyro + ) + np.testing.assert_array_equal( + actual_uncertainty_by_pa, expected_uncertainty_by_pa + ) np.testing.assert_array_equal(actual_swe_flags, expected_flags) - def call_with_array_matchers(*args): return call(*[NumpyArrayMatcher(x) for x in args]) actual_calc_velocity_calls = mock_calculate_velocities.call_args_list expected_calc_velocity_calls = [ - call_with_array_matchers(swe_l2_data.energy - 12, swe_l2_data.inst_el, - swe_l2_data.inst_az_spin_sector[0]), - call_with_array_matchers(swe_l2_data.energy - 16, swe_l2_data.inst_el, - swe_l2_data.inst_az_spin_sector[1]), - call_with_array_matchers(swe_l2_data.energy - 19, swe_l2_data.inst_el, - swe_l2_data.inst_az_spin_sector[2]) + call_with_array_matchers( + swe_l2_data.energy - 12, + swe_l2_data.inst_el, + swe_l2_data.inst_az_spin_sector[0], + ), + call_with_array_matchers( + swe_l2_data.energy - 16, + swe_l2_data.inst_el, + swe_l2_data.inst_az_spin_sector[1], + ), + call_with_array_matchers( + swe_l2_data.energy - 19, + swe_l2_data.inst_el, + swe_l2_data.inst_az_spin_sector[2], + ), ] self.assertEqual(expected_calc_velocity_calls, actual_calc_velocity_calls) correct_and_rebin_actual_calls = mock_correct_and_rebin.call_args_list correct_and_rebin_expected_calls = [ - call_with_array_matchers(swe_l2_data.phase_space_density[0], closest_swapi_data[0], - mock_calculate_velocities.return_value, closest_mag_data[0], - swe_config), - call_with_array_matchers(swe_l2_data.phase_space_density[1], closest_swapi_data[1], - mock_calculate_velocities.return_value, closest_mag_data[1], - swe_config), - call_with_array_matchers(swe_l2_data.phase_space_density[2], closest_swapi_data[2], - mock_calculate_velocities.return_value, closest_mag_data[2], - swe_config) + call_with_array_matchers( + swe_l2_data.phase_space_density[0], + closest_swapi_data[0], + mock_calculate_velocities.return_value, + closest_mag_data[0], + swe_config, + ), + call_with_array_matchers( + swe_l2_data.phase_space_density[1], + closest_swapi_data[1], + mock_calculate_velocities.return_value, + closest_mag_data[1], + swe_config, + ), + call_with_array_matchers( + swe_l2_data.phase_space_density[2], + closest_swapi_data[2], + mock_calculate_velocities.return_value, + closest_mag_data[2], + swe_config, + ), ] - self.assertEqual(correct_and_rebin_expected_calls, correct_and_rebin_actual_calls) + self.assertEqual( + correct_and_rebin_expected_calls, correct_and_rebin_actual_calls + ) mock_rebin_intensity_expected_calls = [ - call_with_array_matchers(swe_l2_data.flux[0], counts[0], mock_calculate_velocities.return_value, - closest_mag_data[0], swe_config), - call_with_array_matchers(swe_l2_data.flux[1], counts[1], mock_calculate_velocities.return_value, - closest_mag_data[1], swe_config), - call_with_array_matchers(swe_l2_data.flux[2], counts[2], mock_calculate_velocities.return_value, - closest_mag_data[2], swe_config) + call_with_array_matchers( + swe_l2_data.flux[0], + counts[0], + mock_calculate_velocities.return_value, + closest_mag_data[0], + swe_config, + ), + call_with_array_matchers( + swe_l2_data.flux[1], + counts[1], + mock_calculate_velocities.return_value, + closest_mag_data[1], + swe_config, + ), + call_with_array_matchers( + swe_l2_data.flux[2], + counts[2], + mock_calculate_velocities.return_value, + closest_mag_data[2], + swe_config, + ), ] - self.assertEqual(mock_rebin_intensity_expected_calls, mock_swe_rebin_intensity.call_args_list) - - mock_integrate_distribution_to_get_1d_spectrum.assert_has_calls([ - call(rebinned_by_pitch_list[0], swe_config), - call(rebinned_by_pitch_list[1], swe_config), - call(rebinned_by_pitch_list[2], swe_config) - ]) - mock_integrate_distribution_to_get_inbound_and_outbound_1d_spectrum.assert_has_calls([ - call(rebinned_by_pitch_list[0], swe_config), - call(rebinned_by_pitch_list[1], swe_config), - call(rebinned_by_pitch_list[2], swe_config) - ]) + self.assertEqual( + mock_rebin_intensity_expected_calls, mock_swe_rebin_intensity.call_args_list + ) + + mock_integrate_distribution_to_get_1d_spectrum.assert_has_calls( + [ + call(rebinned_by_pitch_list[0], swe_config), + call(rebinned_by_pitch_list[1], swe_config), + call(rebinned_by_pitch_list[2], swe_config), + ] + ) + mock_integrate_distribution_to_get_inbound_and_outbound_1d_spectrum.assert_has_calls( + [ + call(rebinned_by_pitch_list[0], swe_config), + call(rebinned_by_pitch_list[1], swe_config), + call(rebinned_by_pitch_list[2], swe_config), + ] + ) @patch("imap_l3_processing.swe.swe_processor.rotate_rtn_vectors_to_dps") - @patch("imap_l3_processing.swe.swe_processor.SweProcessor.calculate_moment_products") - def test_calculate_pitch_angle_products_makes_nan_if_no_mag_close_enough(self, mock_calculate_moments, mock_rotate_rtn_vectors_to_dps): + @patch( + "imap_l3_processing.swe.swe_processor.SweProcessor.calculate_moment_products" + ) + def test_calculate_pitch_angle_products_makes_nan_if_no_mag_close_enough( + self, mock_calculate_moments, mock_rotate_rtn_vectors_to_dps + ): epochs = np.array([datetime(2025, 3, 6)]) - mag_epochs = np.array([ - datetime(2025, 3, 6, 0, 1, 30), - datetime(2025, 3, 6, 0, 2, 30), - datetime(2025, 3, 6, 0, 3, 30), - datetime(2025, 3, 6, 0, 4, 30), - datetime(2025, 3, 6, 0, 5, 30), - datetime(2025, 3, 6, 0, 6, 30), - datetime(2025, 3, 6, 0, 7, 30), - datetime(2025, 3, 6, 0, 8, 30), - datetime(2025, 3, 6, 0, 9, 30), - datetime(2025, 3, 6, 0, 10, 30), - ]) + mag_epochs = np.array( + [ + datetime(2025, 3, 6, 0, 1, 30), + datetime(2025, 3, 6, 0, 2, 30), + datetime(2025, 3, 6, 0, 3, 30), + datetime(2025, 3, 6, 0, 4, 30), + datetime(2025, 3, 6, 0, 5, 30), + datetime(2025, 3, 6, 0, 6, 30), + datetime(2025, 3, 6, 0, 7, 30), + datetime(2025, 3, 6, 0, 8, 30), + datetime(2025, 3, 6, 0, 9, 30), + datetime(2025, 3, 6, 0, 10, 30), + ] + ) swapi_epochs = np.array([datetime(2025, 3, 6)]) - mock_rotate_rtn_vectors_to_dps.return_value = np.tile([0.0, 0.0, -400.0], (len(swapi_epochs), 1)), np.zeros(len(swapi_epochs), dtype=bool) + mock_rotate_rtn_vectors_to_dps.return_value = ( + np.tile([0.0, 0.0, -400.0], (len(swapi_epochs), 1)), + np.zeros(len(swapi_epochs), dtype=bool), + ) pitch_angle_bins = [70, 100, 130] @@ -705,17 +1094,26 @@ def test_calculate_pitch_angle_products_makes_nan_if_no_mag_close_enough(self, m swe_l2_data = SweL2Data( epoch=epochs, - phase_space_density=np.arange(num_epochs * num_energies * 5 * 7).reshape(num_epochs, num_energies, 5, - 7) + 100, - flux=np.arange(num_epochs * num_energies * 5 * 7).reshape(num_epochs, num_energies, 5, 7), + phase_space_density=np.arange(num_epochs * num_energies * 5 * 7).reshape( + num_epochs, num_energies, 5, 7 + ) + + 100, + flux=np.arange(num_epochs * num_energies * 5 * 7).reshape( + num_epochs, num_energies, 5, 7 + ), energy=np.arange(num_energies) + 20, inst_el=np.array([-30, -20, -10, 0, 10, 20, 30]), inst_el_label=sentinel.inst_el_label, inst_az=sentinel.inst_az, inst_az_label=sentinel.inst_az_label, - inst_az_spin_sector=np.arange(num_epochs * num_energies * 5).reshape(num_epochs, num_energies, 5), - acquisition_time=np.linspace(datetime(2025, 3, 6), datetime(2025, 3, 6, 0, 1), - num_epochs * num_energies * 5).reshape(num_epochs, num_energies, 5), + inst_az_spin_sector=np.arange(num_epochs * num_energies * 5).reshape( + num_epochs, num_energies, 5 + ), + acquisition_time=np.linspace( + datetime(2025, 3, 6), + datetime(2025, 3, 6, 0, 1), + num_epochs * num_energies * 5, + ).reshape(num_epochs, num_energies, 5), acquisition_duration=np.full((num_epochs, num_energies, 5), 80000), phase_space_density_rebinned=np.zeros((num_epochs, num_energies, 5, 7)), data_quality=np.zeros(num_epochs, dtype=np.uint8), @@ -724,12 +1122,11 @@ def test_calculate_pitch_angle_products_makes_nan_if_no_mag_close_enough(self, m swe_l1b_data = SweL1bData( epoch=epochs, count_rates=np.full((num_epochs, num_energies, 5, 7), 350), - settle_duration=np.full((num_epochs, 3), 333) + settle_duration=np.full((num_epochs, 3), 333), ) mag_l1d_data = MagData( - epoch=mag_epochs, - mag_data=np.arange(7, 22).reshape(5, 3).repeat(2, axis=0) + epoch=mag_epochs, mag_data=np.arange(7, 22).reshape(5, 3).repeat(2, axis=0) ) swapi_l3a_proton_data = SwapiL3aProtonData( @@ -737,10 +1134,17 @@ def test_calculate_pitch_angle_products_makes_nan_if_no_mag_close_enough(self, m epoch_delta=np.repeat(timedelta(seconds=30), 10), proton_sw_velocity_rtn=np.tile([-400.0, 0.0, 0.0], (len(swapi_epochs), 1)), proton_sw_speed=np.full(len(swapi_epochs), 400.0), - swp_flags=np.full(len(swapi_epochs), 0) + swp_flags=np.full(len(swapi_epochs), 0), ) - geometric_fractions = [0.0697327, 0.138312, 0.175125, 0.181759, - 0.204686, 0.151448, 0.0781351] + geometric_fractions = [ + 0.0697327, + 0.138312, + 0.175125, + 0.181759, + 0.204686, + 0.151448, + 0.0781351, + ] energy_bins = [8, 10, 13] swe_config = build_swe_configuration( geometric_fractions=geometric_fractions, @@ -755,67 +1159,126 @@ def test_calculate_pitch_angle_products_makes_nan_if_no_mag_close_enough(self, m max_mag_offset_in_minutes=1, spacecraft_potential_initial_guess=15, core_halo_breakpoint_initial_guess=90, - in_vs_out_energy_index=len(energy_bins) - 1 + in_vs_out_energy_index=len(energy_bins) - 1, ) - input_metadata = InputMetadata("swe", "l3", datetime(2025, 2, 21), - datetime(2025, 2, 22), "v001") - swel3_dependency = SweL3Dependencies(swe_l2_data, swe_l1b_data, mag_l1d_data, swapi_l3a_proton_data, swe_config) - swe_processor = SweProcessor(ProcessingInputCollection(), input_metadata=input_metadata) + input_metadata = InputMetadata( + "swe", "l3", datetime(2025, 2, 21), datetime(2025, 2, 22), "v001" + ) + swel3_dependency = SweL3Dependencies( + swe_l2_data, swe_l1b_data, mag_l1d_data, swapi_l3a_proton_data, swe_config + ) + swe_processor = SweProcessor( + ProcessingInputCollection(), input_metadata=input_metadata + ) swe_l3_data = swe_processor.calculate_products(swel3_dependency) - self.assertEqual(replace(input_metadata, descriptor="sci"), swe_l3_data.input_metadata) - self.assertEqual(swe_l3_data.pitch_angle, swel3_dependency.configuration["pitch_angle_bins"]) - self.assertEqual(swe_l3_data.pitch_angle_delta, swel3_dependency.configuration["pitch_angle_deltas"]) - self.assertEqual(swe_l3_data.energy, swel3_dependency.configuration["energy_bins"]) - self.assertEqual(swe_l3_data.energy_delta_plus, swel3_dependency.configuration["energy_delta_plus"]) - self.assertEqual(swe_l3_data.energy_delta_minus, swel3_dependency.configuration["energy_delta_minus"]) - np.testing.assert_array_equal(swe_l3_data.phase_space_density_by_pitch_angle, - np.full((len(epochs), len(energy_bins), len(pitch_angle_bins)), np.nan)) + self.assertEqual( + replace(input_metadata, descriptor="sci"), swe_l3_data.input_metadata + ) + self.assertEqual( + swe_l3_data.pitch_angle, swel3_dependency.configuration["pitch_angle_bins"] + ) + self.assertEqual( + swe_l3_data.pitch_angle_delta, + swel3_dependency.configuration["pitch_angle_deltas"], + ) + self.assertEqual( + swe_l3_data.energy, swel3_dependency.configuration["energy_bins"] + ) + self.assertEqual( + swe_l3_data.energy_delta_plus, + swel3_dependency.configuration["energy_delta_plus"], + ) + self.assertEqual( + swe_l3_data.energy_delta_minus, + swel3_dependency.configuration["energy_delta_minus"], + ) + np.testing.assert_array_equal( + swe_l3_data.phase_space_density_by_pitch_angle, + np.full((len(epochs), len(energy_bins), len(pitch_angle_bins)), np.nan), + ) np.testing.assert_array_equal(swe_l3_data.epoch, swe_l2_data.epoch) - np.testing.assert_array_equal(swe_l3_data.phase_space_density_1d, np.full((len(epochs), len(energy_bins)), np.nan)) - np.testing.assert_array_equal(swe_l3_data.phase_space_density_inward, - np.full((len(epochs), len(energy_bins)), np.nan)) - np.testing.assert_array_equal(swe_l3_data.phase_space_density_outward, - np.full((len(epochs), len(energy_bins)), np.nan)) - np.testing.assert_array_equal(swe_l3_data.intensity_by_pitch_angle, - np.full((len(epochs), num_energies, len(pitch_angle_bins)), np.nan)) - np.testing.assert_array_equal(swe_l3_data.intensity_by_pitch_angle_and_gyrophase, - np.full((len(epochs), num_energies, len(pitch_angle_bins), len(gyrophase_bins)), - np.nan)) - np.testing.assert_array_equal(swe_l3_data.intensity_uncertainty_by_pitch_angle, - np.full((len(epochs), num_energies, len(pitch_angle_bins)), np.nan)) - np.testing.assert_array_equal(swe_l3_data.intensity_uncertainty_by_pitch_angle_and_gyrophase, - np.full((len(epochs), num_energies, len(pitch_angle_bins), len(gyrophase_bins)), - np.nan)) + np.testing.assert_array_equal( + swe_l3_data.phase_space_density_1d, + np.full((len(epochs), len(energy_bins)), np.nan), + ) + np.testing.assert_array_equal( + swe_l3_data.phase_space_density_inward, + np.full((len(epochs), len(energy_bins)), np.nan), + ) + np.testing.assert_array_equal( + swe_l3_data.phase_space_density_outward, + np.full((len(epochs), len(energy_bins)), np.nan), + ) + np.testing.assert_array_equal( + swe_l3_data.intensity_by_pitch_angle, + np.full((len(epochs), num_energies, len(pitch_angle_bins)), np.nan), + ) + np.testing.assert_array_equal( + swe_l3_data.intensity_by_pitch_angle_and_gyrophase, + np.full( + (len(epochs), num_energies, len(pitch_angle_bins), len(gyrophase_bins)), + np.nan, + ), + ) + np.testing.assert_array_equal( + swe_l3_data.intensity_uncertainty_by_pitch_angle, + np.full((len(epochs), num_energies, len(pitch_angle_bins)), np.nan), + ) + np.testing.assert_array_equal( + swe_l3_data.intensity_uncertainty_by_pitch_angle_and_gyrophase, + np.full( + (len(epochs), num_energies, len(pitch_angle_bins), len(gyrophase_bins)), + np.nan, + ), + ) @patch("imap_l3_processing.swe.swe_processor.rotate_rtn_vectors_to_dps") - @patch("imap_l3_processing.swe.swe_processor.SweProcessor.calculate_moment_products") - def test_calculate_pitch_angle_products_without_mocks(self, mock_calculate_moments, mock_rotate_rtn_vectors_to_dps): + @patch( + "imap_l3_processing.swe.swe_processor.SweProcessor.calculate_moment_products" + ) + def test_calculate_pitch_angle_products_without_mocks( + self, mock_calculate_moments, mock_rotate_rtn_vectors_to_dps + ): epochs = np.array([datetime(2025, 3, 6)]) mag_start_time = datetime(2025, 3, 6, 0, 1, 0) - mag_epochs = np.array([mag_start_time + i * timedelta(seconds=1) for i in range(10)]) + mag_epochs = np.array( + [mag_start_time + i * timedelta(seconds=1) for i in range(10)] + ) swapi_epochs = np.array([datetime(2025, 3, 6), datetime(2025, 3, 10)]) swp_flags = np.zeros(2) mock_calculate_moments.return_value = build_swe_moment_data(len(epochs)) - mock_rotate_rtn_vectors_to_dps.return_value = np.full((len(swapi_epochs), 3), np.nan), np.zeros(len(swapi_epochs), dtype=bool) + mock_rotate_rtn_vectors_to_dps.return_value = ( + np.full((len(swapi_epochs), 3), np.nan), + np.zeros(len(swapi_epochs), dtype=bool), + ) pitch_angle_bins = [70, 100, 130] num_energies = 9 num_epochs = 1 swe_l2_data = SweL2Data( epoch=epochs, - phase_space_density=np.arange(num_epochs * num_energies * 5 * 7).reshape(num_epochs, num_energies, 5, - 7) + 100, - flux=np.arange(num_epochs * num_energies * 5 * 7).reshape(num_epochs, num_energies, 5, 7), + phase_space_density=np.arange(num_epochs * num_energies * 5 * 7).reshape( + num_epochs, num_energies, 5, 7 + ) + + 100, + flux=np.arange(num_epochs * num_energies * 5 * 7).reshape( + num_epochs, num_energies, 5, 7 + ), energy=np.arange(num_energies) + 5, inst_el=np.array([-30, -20, -10, 0, 10, 20, 30]), inst_el_label=sentinel.inst_el_label, inst_az=sentinel.inst_az, inst_az_label=sentinel.inst_az_label, - inst_az_spin_sector=np.arange(num_epochs * num_energies * 5).reshape(num_epochs, num_energies, 5), - acquisition_time=np.linspace(datetime(2025, 3, 6), datetime(2025, 3, 6, 0, 1), - num_epochs * num_energies * 5).reshape(num_epochs, num_energies, 5), + inst_az_spin_sector=np.arange(num_epochs * num_energies * 5).reshape( + num_epochs, num_energies, 5 + ), + acquisition_time=np.linspace( + datetime(2025, 3, 6), + datetime(2025, 3, 6, 0, 1), + num_epochs * num_energies * 5, + ).reshape(num_epochs, num_energies, 5), acquisition_duration=np.full((num_epochs, num_energies, 5), 80000), phase_space_density_rebinned=np.zeros((num_epochs, num_energies, 5, 7)), data_quality=np.zeros(num_epochs, dtype=np.uint8), @@ -823,15 +1286,18 @@ def test_calculate_pitch_angle_products_without_mocks(self, mock_calculate_momen swe_l1b_data = SweL1bData( epoch=epochs, - count_rates=(np.arange(num_epochs * num_energies * 5 * 7).reshape(num_epochs, num_energies, 5, - 7) + 350) * 1e6, - settle_duration=np.full((num_epochs, 3), 333) - + count_rates=( + np.arange(num_epochs * num_energies * 5 * 7).reshape( + num_epochs, num_energies, 5, 7 + ) + + 350 + ) + * 1e6, + settle_duration=np.full((num_epochs, 3), 333), ) mag_l1d_data = MagData( - epoch=mag_epochs, - mag_data=np.arange(7, 22).reshape(5, 3).repeat(2, axis=0) + epoch=mag_epochs, mag_data=np.arange(7, 22).reshape(5, 3).repeat(2, axis=0) ) swapi_l3a_proton_data = SwapiL3aProtonData( @@ -839,10 +1305,17 @@ def test_calculate_pitch_angle_products_without_mocks(self, mock_calculate_momen epoch_delta=np.repeat(timedelta(seconds=30), 10), proton_sw_velocity_rtn=np.full((len(swapi_epochs), 3), np.nan), proton_sw_speed=np.full(len(swapi_epochs), 400.0), - swp_flags=swp_flags + swp_flags=swp_flags, ) - geometric_fractions = [0.0697327, 0.138312, 0.175125, 0.181759, - 0.204686, 0.151448, 0.0781351] + geometric_fractions = [ + 0.0697327, + 0.138312, + 0.175125, + 0.181759, + 0.204686, + 0.151448, + 0.0781351, + ] energy_bins = [8, 10, 13] swe_config = build_swe_configuration( geometric_fractions=geometric_fractions, @@ -857,80 +1330,140 @@ def test_calculate_pitch_angle_products_without_mocks(self, mock_calculate_momen core_halo_breakpoint_initial_guess=90, in_vs_out_energy_index=len(energy_bins) - 1, gyrophase_bins=[90, 180, 270], - gyrophase_deltas=[45, 45, 45] + gyrophase_deltas=[45, 45, 45], ) - input_metadata = InputMetadata("swe", "l3", datetime(2025, 2, 21), - datetime(2025, 2, 22), "v001") + input_metadata = InputMetadata( + "swe", "l3", datetime(2025, 2, 21), datetime(2025, 2, 22), "v001" + ) - swel3_dependency = SweL3Dependencies(swe_l2_data, swe_l1b_data, mag_l1d_data, swapi_l3a_proton_data, swe_config) - swe_processor = SweProcessor(ProcessingInputCollection(), input_metadata=input_metadata) + swel3_dependency = SweL3Dependencies( + swe_l2_data, swe_l1b_data, mag_l1d_data, swapi_l3a_proton_data, swe_config + ) + swe_processor = SweProcessor( + ProcessingInputCollection(), input_metadata=input_metadata + ) swe_l3_data = swe_processor.calculate_products(swel3_dependency) - self.assertEqual(replace(input_metadata, descriptor="sci"), swe_l3_data.input_metadata) - self.assertEqual(swe_l3_data.pitch_angle, swel3_dependency.configuration["pitch_angle_bins"]) - self.assertEqual(swe_l3_data.pitch_angle_delta, swel3_dependency.configuration["pitch_angle_deltas"]) - self.assertEqual(swe_l3_data.energy, swel3_dependency.configuration["energy_bins"]) - self.assertEqual(swe_l3_data.energy_delta_plus, swel3_dependency.configuration["energy_delta_plus"]) - self.assertEqual(swe_l3_data.energy_delta_minus, swel3_dependency.configuration["energy_delta_minus"]) - self.assertEqual(swe_l3_data.gyrophase_bins, swel3_dependency.configuration["gyrophase_bins"]) - self.assertEqual(swe_l3_data.gyrophase_delta, swel3_dependency.configuration["gyrophase_deltas"]) - np.testing.assert_allclose(swe_l3_data.phase_space_density_by_pitch_angle, - np.array([[[np.nan, 249.548992, 310.137124], - [np.nan, 308.858857, 363.635632], - [np.nan, 394.907904, 437.401188]]])) + self.assertEqual( + replace(input_metadata, descriptor="sci"), swe_l3_data.input_metadata + ) + self.assertEqual( + swe_l3_data.pitch_angle, swel3_dependency.configuration["pitch_angle_bins"] + ) + self.assertEqual( + swe_l3_data.pitch_angle_delta, + swel3_dependency.configuration["pitch_angle_deltas"], + ) + self.assertEqual( + swe_l3_data.energy, swel3_dependency.configuration["energy_bins"] + ) + self.assertEqual( + swe_l3_data.energy_delta_plus, + swel3_dependency.configuration["energy_delta_plus"], + ) + self.assertEqual( + swe_l3_data.energy_delta_minus, + swel3_dependency.configuration["energy_delta_minus"], + ) + self.assertEqual( + swe_l3_data.gyrophase_bins, swel3_dependency.configuration["gyrophase_bins"] + ) + self.assertEqual( + swe_l3_data.gyrophase_delta, + swel3_dependency.configuration["gyrophase_deltas"], + ) + np.testing.assert_allclose( + swe_l3_data.phase_space_density_by_pitch_angle, + np.array( + [ + [ + [np.nan, 249.548992, 310.137124], + [np.nan, 308.858857, 363.635632], + [np.nan, 394.907904, 437.401188], + ] + ] + ), + ) np.testing.assert_allclose(swe_l3_data.epoch_delta, np.array([1807492500])) np.testing.assert_array_equal(swe_l3_data.epoch, swe_l2_data.epoch) - np.testing.assert_allclose(swe_l3_data.phase_space_density_1d, np.array([[126.537227, 152.557743, 189.536765]])) - np.testing.assert_allclose(swe_l3_data.phase_space_density_inward, np.array([[0., 0., 0.]])) - np.testing.assert_allclose(swe_l3_data.phase_space_density_outward, - np.array([[253.074453, 305.115485, 379.073531]])) - self.assertEqual((1, 9, 3, 3), swe_l3_data.intensity_by_pitch_angle_and_gyrophase.shape) + np.testing.assert_allclose( + swe_l3_data.phase_space_density_1d, + np.array([[126.537227, 152.557743, 189.536765]]), + ) + np.testing.assert_allclose( + swe_l3_data.phase_space_density_inward, np.array([[0.0, 0.0, 0.0]]) + ) + np.testing.assert_allclose( + swe_l3_data.phase_space_density_outward, + np.array([[253.074453, 305.115485, 379.073531]]), + ) + self.assertEqual( + (1, 9, 3, 3), swe_l3_data.intensity_by_pitch_angle_and_gyrophase.shape + ) self.assertEqual((1, 9, 3), swe_l3_data.intensity_by_pitch_angle.shape) - np.testing.assert_allclose(swe_l3_data.intensity_by_pitch_angle[0, 1, 1:3], - np.array([49.5, 53. ])) - np.testing.assert_allclose(swe_l3_data.intensity_by_pitch_angle_and_gyrophase[0, 0, 1, 2], - np.array([15.])) - - np.testing.assert_allclose(swe_l3_data.intensity_uncertainty_by_pitch_angle[0, 0, 1:3], - np.array([0.000717, 0.000762]), atol=1e-6) - np.testing.assert_allclose(swe_l3_data.intensity_uncertainty_by_pitch_angle_and_gyrophase[0, 0, 1:3, 2], - np.array([0.000717, 0.000762]), atol=1e-6) - expected_swe_flags = np.array([ - SweL3Flags.FALLBACK_POTENTIAL_ESTIMATE - |SweL3Flags.BACKUP_SPLINE_UNRESOLVED - |SweL3Flags.BREAKPOINT_FIT_UNCONVERGED - |SweL3Flags.FALLBACK_SWAPI_SPEED - |SweL3Flags.UNPHYSICAL_PSD - ]) + np.testing.assert_allclose( + swe_l3_data.intensity_by_pitch_angle[0, 1, 1:3], np.array([49.5, 53.0]) + ) + np.testing.assert_allclose( + swe_l3_data.intensity_by_pitch_angle_and_gyrophase[0, 0, 1, 2], + np.array([15.0]), + ) + + np.testing.assert_allclose( + swe_l3_data.intensity_uncertainty_by_pitch_angle[0, 0, 1:3], + np.array([0.000717, 0.000762]), + atol=1e-6, + ) + np.testing.assert_allclose( + swe_l3_data.intensity_uncertainty_by_pitch_angle_and_gyrophase[ + 0, 0, 1:3, 2 + ], + np.array([0.000717, 0.000762]), + atol=1e-6, + ) + expected_swe_flags = np.array( + [ + SweL3Flags.FALLBACK_POTENTIAL_ESTIMATE + | SweL3Flags.BACKUP_SPLINE_UNRESOLVED + | SweL3Flags.BREAKPOINT_FIT_UNCONVERGED + | SweL3Flags.FALLBACK_SWAPI_SPEED + | SweL3Flags.UNPHYSICAL_PSD + ] + ) np.testing.assert_array_equal(swe_l3_data.swe_flags, expected_swe_flags) @patch(f"{MODULE}.get_dps_to_rtn_rotation_matrix") - @patch('imap_l3_processing.swe.swe_processor.rotate_temperature_tensor_to_mag') - @patch('imap_l3_processing.swe.swe_processor.calculate_primary_eigenvector') - @patch('imap_l3_processing.swe.swe_processor.rotate_vector_to_rtn_spherical_coordinates') - @patch('imap_l3_processing.swe.swe_processor.scale_halo_density') - @patch('imap_l3_processing.swe.swe_processor.scale_core_density') - @patch('imap_l3_processing.swe.swe_processor.integrate') - @patch('imap_l3_processing.swe.swe_processor.halo_fit_moments_retrying_on_failure') - @patch('imap_l3_processing.swe.swe_processor.core_fit_moments_retrying_on_failure') - @patch('imap_l3_processing.swe.swe_processor.compute_maxwellian_weight_factors') - @patch('imap_l3_processing.swe.swe_processor.calculate_velocity_in_dsp_frame_km_s') - @patch('imap_l3_processing.swe.swe_processor.apply_rotation_matrix') - @patch('imap_l3_processing.swe.swe_processor.rotate_temperature') - def test_calculate_moment_products(self, mock_rotate_temperature, - mock_rotate_dps_vector_to_rtn, - mock_calculate_velocity_in_dsp_frame_km_s, - mock_compute_maxwellian_weight_factors, - mock_core_fit_moments_retrying_on_failure: Mock, - mock_halo_fit_moments_retrying_on_failure: Mock, - mock_integrate: Mock, - mock_scale_core_density: Mock, - mock_scale_halo_density: Mock, - mock_rotate_vector_to_rtn_spherical_coordinates, - mock_calculate_primary_eigenvector, - mock_rotate_temperature_tensor_to_mag, - mock_get_dps_to_rtn_rotation_matrix): + @patch("imap_l3_processing.swe.swe_processor.rotate_temperature_tensor_to_mag") + @patch("imap_l3_processing.swe.swe_processor.calculate_primary_eigenvector") + @patch( + "imap_l3_processing.swe.swe_processor.rotate_vector_to_rtn_spherical_coordinates" + ) + @patch("imap_l3_processing.swe.swe_processor.scale_halo_density") + @patch("imap_l3_processing.swe.swe_processor.scale_core_density") + @patch("imap_l3_processing.swe.swe_processor.integrate") + @patch("imap_l3_processing.swe.swe_processor.halo_fit_moments_retrying_on_failure") + @patch("imap_l3_processing.swe.swe_processor.core_fit_moments_retrying_on_failure") + @patch("imap_l3_processing.swe.swe_processor.compute_maxwellian_weight_factors") + @patch("imap_l3_processing.swe.swe_processor.calculate_velocity_in_dsp_frame_km_s") + @patch("imap_l3_processing.swe.swe_processor.apply_rotation_matrix") + @patch("imap_l3_processing.swe.swe_processor.rotate_temperature") + def test_calculate_moment_products( + self, + mock_rotate_temperature, + mock_rotate_dps_vector_to_rtn, + mock_calculate_velocity_in_dsp_frame_km_s, + mock_compute_maxwellian_weight_factors, + mock_core_fit_moments_retrying_on_failure: Mock, + mock_halo_fit_moments_retrying_on_failure: Mock, + mock_integrate: Mock, + mock_scale_core_density: Mock, + mock_scale_halo_density: Mock, + mock_rotate_vector_to_rtn_spherical_coordinates, + mock_calculate_primary_eigenvector, + mock_rotate_temperature_tensor_to_mag, + mock_get_dps_to_rtn_rotation_matrix, + ): epochs = datetime.now() + np.arange(3) * timedelta(minutes=1) dps_to_rtn_matrices = [ sentinel.dps_to_rtn_1, @@ -957,54 +1490,118 @@ def test_calculate_moment_products(self, mock_rotate_temperature, ) expected_sin_theta = np.sin(np.deg2rad(90 - instrument_elevation)) expected_cos_theta = np.cos(np.deg2rad(90 - instrument_elevation)) - np.testing.assert_allclose(expected_cos_theta, - [-0.9034, -0.6947, -0.3730, 0.0, 0.3714, 0.6896, 0.8996], atol=0.03) - swe_l1_data = SweL1bData(epoch=epochs, - count_rates=[sentinel.l1b_count_rates_1, sentinel.l1b_count_rates_2, - sentinel.l1b_count_rates_3], - settle_duration=Mock()) + np.testing.assert_allclose( + expected_cos_theta, + [-0.9034, -0.6947, -0.3730, 0.0, 0.3714, 0.6896, 0.8996], + atol=0.03, + ) + swe_l1_data = SweL1bData( + epoch=epochs, + count_rates=[ + sentinel.l1b_count_rates_1, + sentinel.l1b_count_rates_2, + sentinel.l1b_count_rates_3, + ], + settle_duration=Mock(), + ) spacecraft_potential = np.array([12, 14, 16]) core_halo_breakpoint = np.array([96, 54, 103]) - corrected_energy_bins = swe_l2_data.energy.reshape(1, -1) - spacecraft_potential.reshape(-1, 1) + corrected_energy_bins = swe_l2_data.energy.reshape( + 1, -1 + ) - spacecraft_potential.reshape(-1, 1) velocity_in_dsp_frame_km_s = [ np.full(shape=(24, 30, 7, 3), fill_value=1), np.full(shape=(24, 30, 7, 3), fill_value=2), - np.full(shape=(24, 30, 7, 3), fill_value=3) + np.full(shape=(24, 30, 7, 3), fill_value=3), ] - mock_calculate_velocity_in_dsp_frame_km_s.side_effect = velocity_in_dsp_frame_km_s + mock_calculate_velocity_in_dsp_frame_km_s.side_effect = ( + velocity_in_dsp_frame_km_s + ) - maxwellian_weight_factors = [np.full(shape=(24, 30, 7), fill_value=1), - np.full(shape=(24, 30, 7), fill_value=2), - np.full(shape=(24, 30, 7), fill_value=3)] + maxwellian_weight_factors = [ + np.full(shape=(24, 30, 7), fill_value=1), + np.full(shape=(24, 30, 7), fill_value=2), + np.full(shape=(24, 30, 7), fill_value=3), + ] mock_compute_maxwellian_weight_factors.side_effect = maxwellian_weight_factors - core_moments1 = create_dataclass_mock(Moments, density=15, velocity_x=5, velocity_y=6, velocity_z=7, - t_parallel=1e5, t_perpendicular=1e5) - halo_moments1 = create_dataclass_mock(Moments, density=22, velocity_x=5, velocity_y=6, - velocity_z=7, t_parallel=1e5, t_perpendicular=1e5) - core_moments2 = create_dataclass_mock(Moments, density=12, velocity_x=8, velocity_y=9, velocity_z=10, - t_parallel=1e5, t_perpendicular=1e5) - halo_moments2 = create_dataclass_mock(Moments, density=19, velocity_x=8, velocity_y=9, - velocity_z=10, t_parallel=1e5, t_perpendicular=1e5) - - core_moment_fit_results_1 = MomentFitResults(moments=core_moments1, chisq=4730912.0, number_of_points=10, - regress_result=sentinel.core_regress_result_1) - core_moment_fit_results_2 = MomentFitResults(moments=core_moments2, chisq=4705988.0, number_of_points=11, - regress_result=sentinel.core_regress_result_2) - halo_moment_fit_results_1 = MomentFitResults(moments=halo_moments1, chisq=3412.0, number_of_points=12, - regress_result=sentinel.halo_regress_result_1) - halo_moment_fit_results_2 = MomentFitResults(moments=halo_moments2, chisq=3214.0, number_of_points=13, - regress_result=sentinel.halo_regress_result_2) - - mock_core_fit_moments_retrying_on_failure.side_effect = [core_moment_fit_results_1, core_moment_fit_results_2, - None] - mock_halo_fit_moments_retrying_on_failure.side_effect = [halo_moment_fit_results_1, halo_moment_fit_results_2, - None] + core_moments1 = create_dataclass_mock( + Moments, + density=15, + velocity_x=5, + velocity_y=6, + velocity_z=7, + t_parallel=1e5, + t_perpendicular=1e5, + ) + halo_moments1 = create_dataclass_mock( + Moments, + density=22, + velocity_x=5, + velocity_y=6, + velocity_z=7, + t_parallel=1e5, + t_perpendicular=1e5, + ) + core_moments2 = create_dataclass_mock( + Moments, + density=12, + velocity_x=8, + velocity_y=9, + velocity_z=10, + t_parallel=1e5, + t_perpendicular=1e5, + ) + halo_moments2 = create_dataclass_mock( + Moments, + density=19, + velocity_x=8, + velocity_y=9, + velocity_z=10, + t_parallel=1e5, + t_perpendicular=1e5, + ) + + core_moment_fit_results_1 = MomentFitResults( + moments=core_moments1, + chisq=4730912.0, + number_of_points=10, + regress_result=sentinel.core_regress_result_1, + ) + core_moment_fit_results_2 = MomentFitResults( + moments=core_moments2, + chisq=4705988.0, + number_of_points=11, + regress_result=sentinel.core_regress_result_2, + ) + halo_moment_fit_results_1 = MomentFitResults( + moments=halo_moments1, + chisq=3412.0, + number_of_points=12, + regress_result=sentinel.halo_regress_result_1, + ) + halo_moment_fit_results_2 = MomentFitResults( + moments=halo_moments2, + chisq=3214.0, + number_of_points=13, + regress_result=sentinel.halo_regress_result_2, + ) + + mock_core_fit_moments_retrying_on_failure.side_effect = [ + core_moment_fit_results_1, + core_moment_fit_results_2, + None, + ] + mock_halo_fit_moments_retrying_on_failure.side_effect = [ + halo_moment_fit_results_1, + halo_moment_fit_results_2, + None, + ] core_rtn_theta_1, core_rtn_phi_1 = (62, 190) halo_rtn_theta_1, halo_rtn_phi_1 = (54, 222) @@ -1031,7 +1628,7 @@ def test_calculate_moment_products(self, mock_rotate_temperature, halo_fit_velocity_1, halo_integrated_velocity_rtn, core_fit_velocity_2, - halo_fit_velocity_2 + halo_fit_velocity_2, ] core_integrate_output = Mock() @@ -1041,25 +1638,39 @@ def test_calculate_moment_products(self, mock_rotate_temperature, total_integrate_output.temperature = total_temperature halo_integrate_output = Mock() - mock_integrate.side_effect = [core_integrate_output, total_integrate_output, halo_integrate_output, None, None] + mock_integrate.side_effect = [ + core_integrate_output, + total_integrate_output, + halo_integrate_output, + None, + None, + ] scaled_core_velocity = [900, 800, 700] scaled_core_temperature = np.array([90000, 80000, 70000, 80000, 90000, 100000]) core_cdelnv = Mock() core_cdelt = Mock() - scale_core_density_output = ScaleDensityOutput(density=400, velocity=scaled_core_velocity, - temperature=scaled_core_temperature, - cdelnv=core_cdelnv, - cdelt=core_cdelt) + scale_core_density_output = ScaleDensityOutput( + density=400, + velocity=scaled_core_velocity, + temperature=scaled_core_temperature, + cdelnv=core_cdelnv, + cdelt=core_cdelt, + ) mock_scale_core_density.side_effect = [scale_core_density_output] scaled_halo_velocity = [2000, 1800, 1700] - scaled_halo_temperature = np.array([180000, 190000, 200000, 201000, 202000, 203000]) - scale_halo_density_output = ScaleDensityOutput(density=500, velocity=scaled_halo_velocity, - temperature=scaled_halo_temperature, - cdelnv=None, - cdelt=None) + scaled_halo_temperature = np.array( + [180000, 190000, 200000, 201000, 202000, 203000] + ) + scale_halo_density_output = ScaleDensityOutput( + density=500, + velocity=scaled_halo_velocity, + temperature=scaled_halo_temperature, + cdelnv=None, + cdelt=None, + ) mock_scale_halo_density.side_effect = [scale_halo_density_output] @@ -1069,7 +1680,8 @@ def test_calculate_moment_products(self, mock_rotate_temperature, (101, np.deg2rad(102), np.deg2rad(103)), (104, np.deg2rad(105), np.deg2rad(106)), (13, np.deg2rad(14), np.deg2rad(15)), - (19, np.deg2rad(20), np.deg2rad(21))] + (19, np.deg2rad(20), np.deg2rad(21)), + ] core_primary_evec = Mock() total_primary_evec = Mock() @@ -1089,66 +1701,120 @@ def test_calculate_moment_products(self, mock_rotate_temperature, (halo_primary_evec, np.array([t_par_2, t_perp_2, gyro_2])), ] - mock_rotate_temperature_tensor_to_mag.side_effect = [(11.4, 12.4, 13.4), (14.4, 15.4, 16.4), (17.4, 18.4, 19.4)] + mock_rotate_temperature_tensor_to_mag.side_effect = [ + (11.4, 12.4, 13.4), + (14.4, 15.4, 16.4), + (17.4, 18.4, 19.4), + ] - input_metadata = InputMetadata("swe", "l3", datetime(2025, 2, 21), - datetime(2025, 2, 22), "v001") + input_metadata = InputMetadata( + "swe", "l3", datetime(2025, 2, 21), datetime(2025, 2, 22), "v001" + ) swe_processor = SweProcessor(dependencies=[], input_metadata=input_metadata) config = build_swe_configuration() - rebinned_mag_data = [sentinel.mag_data_1, sentinel.mag_data_2, sentinel.mag_data_3] + rebinned_mag_data = [ + sentinel.mag_data_1, + sentinel.mag_data_2, + sentinel.mag_data_3, + ] - swe_moment_data = swe_processor.calculate_moment_products(swe_l2_data, swe_l1_data, rebinned_mag_data, - spacecraft_potential, - core_halo_breakpoint, - corrected_energy_bins, - config) + swe_moment_data = swe_processor.calculate_moment_products( + swe_l2_data, + swe_l1_data, + rebinned_mag_data, + spacecraft_potential, + core_halo_breakpoint, + corrected_energy_bins, + config, + ) - calculate_velocity_call_1 = mock_calculate_velocity_in_dsp_frame_km_s.mock_calls[0] - np.testing.assert_array_equal(corrected_energy_bins[0], calculate_velocity_call_1.args[0]) - np.testing.assert_array_equal(swe_l2_data.inst_el, calculate_velocity_call_1.args[1]) - np.testing.assert_array_equal(swe_l2_data.inst_az_spin_sector[0], calculate_velocity_call_1.args[2]) + calculate_velocity_call_1 = ( + mock_calculate_velocity_in_dsp_frame_km_s.mock_calls[0] + ) + np.testing.assert_array_equal( + corrected_energy_bins[0], calculate_velocity_call_1.args[0] + ) + np.testing.assert_array_equal( + swe_l2_data.inst_el, calculate_velocity_call_1.args[1] + ) + np.testing.assert_array_equal( + swe_l2_data.inst_az_spin_sector[0], calculate_velocity_call_1.args[2] + ) - calculate_velocity_call_2 = mock_calculate_velocity_in_dsp_frame_km_s.mock_calls[1] - np.testing.assert_array_equal(corrected_energy_bins[1], calculate_velocity_call_2.args[0]) - np.testing.assert_array_equal(swe_l2_data.inst_el, calculate_velocity_call_2.args[1]) - np.testing.assert_array_equal(swe_l2_data.inst_az_spin_sector[1], calculate_velocity_call_2.args[2]) + calculate_velocity_call_2 = ( + mock_calculate_velocity_in_dsp_frame_km_s.mock_calls[1] + ) + np.testing.assert_array_equal( + corrected_energy_bins[1], calculate_velocity_call_2.args[0] + ) + np.testing.assert_array_equal( + swe_l2_data.inst_el, calculate_velocity_call_2.args[1] + ) + np.testing.assert_array_equal( + swe_l2_data.inst_az_spin_sector[1], calculate_velocity_call_2.args[2] + ) - calculate_velocity_call_3 = mock_calculate_velocity_in_dsp_frame_km_s.mock_calls[2] - np.testing.assert_array_equal(corrected_energy_bins[2], calculate_velocity_call_3.args[0]) - np.testing.assert_array_equal(swe_l2_data.inst_el, calculate_velocity_call_3.args[1]) - np.testing.assert_array_equal(swe_l2_data.inst_az_spin_sector[2], calculate_velocity_call_3.args[2]) + calculate_velocity_call_3 = ( + mock_calculate_velocity_in_dsp_frame_km_s.mock_calls[2] + ) + np.testing.assert_array_equal( + corrected_energy_bins[2], calculate_velocity_call_3.args[0] + ) + np.testing.assert_array_equal( + swe_l2_data.inst_el, calculate_velocity_call_3.args[1] + ) + np.testing.assert_array_equal( + swe_l2_data.inst_az_spin_sector[2], calculate_velocity_call_3.args[2] + ) self.assertEqual(3, mock_compute_maxwellian_weight_factors.call_count) mock_compute_maxwellian_weight_factors.assert_has_calls( - [call(sentinel.l1b_count_rates_1, 10.0), - call(sentinel.l1b_count_rates_2, 20.0), - call(sentinel.l1b_count_rates_3, 30.0), - ]) + [ + call(sentinel.l1b_count_rates_1, 10.0), + call(sentinel.l1b_count_rates_2, 20.0), + call(sentinel.l1b_count_rates_3, 30.0), + ] + ) self.assertEqual(3, mock_core_fit_moments_retrying_on_failure.call_count) self.assertEqual(3, mock_halo_fit_moments_retrying_on_failure.call_count) - core_fit_moments_call_1 = mock_core_fit_moments_retrying_on_failure.mock_calls[0] - np.testing.assert_array_equal(core_fit_moments_call_1.args[0], corrected_energy_bins[0]) - np.testing.assert_array_equal(velocity_in_dsp_frame_km_s[0] * 100000, - core_fit_moments_call_1.args[1]) - np.testing.assert_array_equal(swe_l2_data.phase_space_density[0], core_fit_moments_call_1.args[2]) - np.testing.assert_array_equal(maxwellian_weight_factors[0], - core_fit_moments_call_1.args[3]) + core_fit_moments_call_1 = mock_core_fit_moments_retrying_on_failure.mock_calls[ + 0 + ] + np.testing.assert_array_equal( + core_fit_moments_call_1.args[0], corrected_energy_bins[0] + ) + np.testing.assert_array_equal( + velocity_in_dsp_frame_km_s[0] * 100000, core_fit_moments_call_1.args[1] + ) + np.testing.assert_array_equal( + swe_l2_data.phase_space_density[0], core_fit_moments_call_1.args[2] + ) + np.testing.assert_array_equal( + maxwellian_weight_factors[0], core_fit_moments_call_1.args[3] + ) self.assertEqual(3, core_fit_moments_call_1.args[4]) self.assertEqual(7, core_fit_moments_call_1.args[5]) np.testing.assert_array_equal(core_fit_moments_call_1.args[6], [100, 100, 100]) - halo_fit_moments_call_1 = mock_halo_fit_moments_retrying_on_failure.mock_calls[0] - np.testing.assert_array_equal(corrected_energy_bins[0], - halo_fit_moments_call_1.args[0]) - np.testing.assert_array_equal(velocity_in_dsp_frame_km_s[0] * 100000, - halo_fit_moments_call_1.args[1]) - np.testing.assert_array_equal(swe_l2_data.phase_space_density[0], halo_fit_moments_call_1.args[2]) - np.testing.assert_array_equal(maxwellian_weight_factors[0], - halo_fit_moments_call_1.args[3]) + halo_fit_moments_call_1 = mock_halo_fit_moments_retrying_on_failure.mock_calls[ + 0 + ] + np.testing.assert_array_equal( + corrected_energy_bins[0], halo_fit_moments_call_1.args[0] + ) + np.testing.assert_array_equal( + velocity_in_dsp_frame_km_s[0] * 100000, halo_fit_moments_call_1.args[1] + ) + np.testing.assert_array_equal( + swe_l2_data.phase_space_density[0], halo_fit_moments_call_1.args[2] + ) + np.testing.assert_array_equal( + maxwellian_weight_factors[0], halo_fit_moments_call_1.args[3] + ) self.assertEqual(6, halo_fit_moments_call_1.args[4]) self.assertEqual(11, halo_fit_moments_call_1.args[5]) np.testing.assert_array_equal(halo_fit_moments_call_1.args[6], [25, 25, 25]) @@ -1156,54 +1822,84 @@ def test_calculate_moment_products(self, mock_rotate_temperature, self.assertEqual(core_halo_breakpoint[0], halo_fit_moments_call_1.args[8]) core_fit_moments_2 = mock_core_fit_moments_retrying_on_failure.mock_calls[1] - np.testing.assert_array_equal(corrected_energy_bins[1], - core_fit_moments_2.args[0]) - np.testing.assert_array_equal(velocity_in_dsp_frame_km_s[1] * 100000, - core_fit_moments_2.args[1]) - np.testing.assert_array_equal(swe_l2_data.phase_space_density[1], core_fit_moments_2.args[2]) - np.testing.assert_array_equal(maxwellian_weight_factors[1], - core_fit_moments_2.args[3]) + np.testing.assert_array_equal( + corrected_energy_bins[1], core_fit_moments_2.args[0] + ) + np.testing.assert_array_equal( + velocity_in_dsp_frame_km_s[1] * 100000, core_fit_moments_2.args[1] + ) + np.testing.assert_array_equal( + swe_l2_data.phase_space_density[1], core_fit_moments_2.args[2] + ) + np.testing.assert_array_equal( + maxwellian_weight_factors[1], core_fit_moments_2.args[3] + ) self.assertEqual(4, core_fit_moments_2.args[4]) self.assertEqual(6, core_fit_moments_2.args[5]) - np.testing.assert_array_equal(core_fit_moments_2.args[6], [100, 100, core_moments1.density]) + np.testing.assert_array_equal( + core_fit_moments_2.args[6], [100, 100, core_moments1.density] + ) halo_fit_moments_2 = mock_halo_fit_moments_retrying_on_failure.mock_calls[1] - np.testing.assert_array_equal(corrected_energy_bins[1], - halo_fit_moments_2.args[0]) - np.testing.assert_array_equal(velocity_in_dsp_frame_km_s[1] * 100000, - halo_fit_moments_2.args[1]) - np.testing.assert_array_equal(swe_l2_data.phase_space_density[1], halo_fit_moments_2.args[2]) - np.testing.assert_array_equal(maxwellian_weight_factors[1], - halo_fit_moments_2.args[3]) + np.testing.assert_array_equal( + corrected_energy_bins[1], halo_fit_moments_2.args[0] + ) + np.testing.assert_array_equal( + velocity_in_dsp_frame_km_s[1] * 100000, halo_fit_moments_2.args[1] + ) + np.testing.assert_array_equal( + swe_l2_data.phase_space_density[1], halo_fit_moments_2.args[2] + ) + np.testing.assert_array_equal( + maxwellian_weight_factors[1], halo_fit_moments_2.args[3] + ) self.assertEqual(5, halo_fit_moments_2.args[4]) self.assertEqual(10, halo_fit_moments_2.args[5]) - np.testing.assert_array_equal(halo_fit_moments_2.args[6], [25, 25, halo_moments1.density]) + np.testing.assert_array_equal( + halo_fit_moments_2.args[6], [25, 25, halo_moments1.density] + ) self.assertEqual(spacecraft_potential[1], halo_fit_moments_2.args[7]) self.assertEqual(core_halo_breakpoint[1], halo_fit_moments_2.args[8]) core_fit_moments_3 = mock_core_fit_moments_retrying_on_failure.mock_calls[2] - np.testing.assert_array_equal(corrected_energy_bins[2], - core_fit_moments_3.args[0]) - np.testing.assert_array_equal(velocity_in_dsp_frame_km_s[2] * 100000, - core_fit_moments_3.args[1]) - np.testing.assert_array_equal(swe_l2_data.phase_space_density[2], core_fit_moments_3.args[2]) - np.testing.assert_array_equal(maxwellian_weight_factors[2], - core_fit_moments_3.args[3]) + np.testing.assert_array_equal( + corrected_energy_bins[2], core_fit_moments_3.args[0] + ) + np.testing.assert_array_equal( + velocity_in_dsp_frame_km_s[2] * 100000, core_fit_moments_3.args[1] + ) + np.testing.assert_array_equal( + swe_l2_data.phase_space_density[2], core_fit_moments_3.args[2] + ) + np.testing.assert_array_equal( + maxwellian_weight_factors[2], core_fit_moments_3.args[3] + ) self.assertEqual(5, core_fit_moments_3.args[4]) self.assertEqual(9, core_fit_moments_3.args[5]) - np.testing.assert_array_equal(core_fit_moments_3.args[6], [100, core_moments1.density, core_moments2.density]) + np.testing.assert_array_equal( + core_fit_moments_3.args[6], + [100, core_moments1.density, core_moments2.density], + ) halo_fit_moments_3 = mock_halo_fit_moments_retrying_on_failure.mock_calls[2] - np.testing.assert_array_equal(corrected_energy_bins[2], - halo_fit_moments_3.args[0]) - np.testing.assert_array_equal(velocity_in_dsp_frame_km_s[2] * 100000, - halo_fit_moments_3.args[1]) - np.testing.assert_array_equal(swe_l2_data.phase_space_density[2], halo_fit_moments_3.args[2]) - np.testing.assert_array_equal(maxwellian_weight_factors[2], - halo_fit_moments_3.args[3]) + np.testing.assert_array_equal( + corrected_energy_bins[2], halo_fit_moments_3.args[0] + ) + np.testing.assert_array_equal( + velocity_in_dsp_frame_km_s[2] * 100000, halo_fit_moments_3.args[1] + ) + np.testing.assert_array_equal( + swe_l2_data.phase_space_density[2], halo_fit_moments_3.args[2] + ) + np.testing.assert_array_equal( + maxwellian_weight_factors[2], halo_fit_moments_3.args[3] + ) self.assertEqual(8, halo_fit_moments_3.args[4]) self.assertEqual(11, halo_fit_moments_3.args[5]) - np.testing.assert_array_equal(halo_fit_moments_3.args[6], [25, halo_moments1.density, halo_moments2.density]) + np.testing.assert_array_equal( + halo_fit_moments_3.args[6], + [25, halo_moments1.density, halo_moments2.density], + ) self.assertEqual(spacecraft_potential[2], halo_fit_moments_3.args[7]) self.assertEqual(core_halo_breakpoint[2], halo_fit_moments_3.args[8]) @@ -1213,285 +1909,554 @@ def test_calculate_moment_products(self, mock_rotate_temperature, ) self.assertEqual(7, mock_rotate_dps_vector_to_rtn.call_count) - self.assertEqual(dps_to_rtn_matrices[0], mock_rotate_dps_vector_to_rtn.call_args_list[0].args[0]) + self.assertEqual( + dps_to_rtn_matrices[0], + mock_rotate_dps_vector_to_rtn.call_args_list[0].args[0], + ) np.testing.assert_array_equal( - np.array([core_moments1.velocity_x, core_moments1.velocity_y, core_moments1.velocity_z]), - mock_rotate_dps_vector_to_rtn.call_args_list[0].args[1]) - self.assertEqual(dps_to_rtn_matrices[0], mock_rotate_dps_vector_to_rtn.call_args_list[1].args[0]) + np.array( + [ + core_moments1.velocity_x, + core_moments1.velocity_y, + core_moments1.velocity_z, + ] + ), + mock_rotate_dps_vector_to_rtn.call_args_list[0].args[1], + ) + self.assertEqual( + dps_to_rtn_matrices[0], + mock_rotate_dps_vector_to_rtn.call_args_list[1].args[0], + ) np.testing.assert_array_equal( scaled_core_velocity, - mock_rotate_dps_vector_to_rtn.call_args_list[1].args[1]) + mock_rotate_dps_vector_to_rtn.call_args_list[1].args[1], + ) - self.assertEqual(dps_to_rtn_matrices[0], mock_rotate_dps_vector_to_rtn.call_args_list[2].args[0]) + self.assertEqual( + dps_to_rtn_matrices[0], + mock_rotate_dps_vector_to_rtn.call_args_list[2].args[0], + ) np.testing.assert_array_equal( total_integrate_output.velocity, - mock_rotate_dps_vector_to_rtn.call_args_list[2].args[1]) + mock_rotate_dps_vector_to_rtn.call_args_list[2].args[1], + ) - self.assertEqual(dps_to_rtn_matrices[0], mock_rotate_dps_vector_to_rtn.call_args_list[3].args[0]) + self.assertEqual( + dps_to_rtn_matrices[0], + mock_rotate_dps_vector_to_rtn.call_args_list[3].args[0], + ) np.testing.assert_array_equal( - np.array([halo_moments1.velocity_x, halo_moments1.velocity_y, halo_moments1.velocity_z]), - mock_rotate_dps_vector_to_rtn.call_args_list[3].args[1]) + np.array( + [ + halo_moments1.velocity_x, + halo_moments1.velocity_y, + halo_moments1.velocity_z, + ] + ), + mock_rotate_dps_vector_to_rtn.call_args_list[3].args[1], + ) - self.assertEqual(dps_to_rtn_matrices[0], mock_rotate_dps_vector_to_rtn.call_args_list[4].args[0]) + self.assertEqual( + dps_to_rtn_matrices[0], + mock_rotate_dps_vector_to_rtn.call_args_list[4].args[0], + ) np.testing.assert_array_equal( scaled_halo_velocity, - mock_rotate_dps_vector_to_rtn.call_args_list[4].args[1]) + mock_rotate_dps_vector_to_rtn.call_args_list[4].args[1], + ) - self.assertEqual(dps_to_rtn_matrices[1], mock_rotate_dps_vector_to_rtn.call_args_list[5].args[0]) + self.assertEqual( + dps_to_rtn_matrices[1], + mock_rotate_dps_vector_to_rtn.call_args_list[5].args[0], + ) np.testing.assert_array_equal( - np.array([core_moments2.velocity_x, core_moments2.velocity_y, core_moments2.velocity_z]), - mock_rotate_dps_vector_to_rtn.call_args_list[5].args[1]) + np.array( + [ + core_moments2.velocity_x, + core_moments2.velocity_y, + core_moments2.velocity_z, + ] + ), + mock_rotate_dps_vector_to_rtn.call_args_list[5].args[1], + ) - self.assertEqual(dps_to_rtn_matrices[1], mock_rotate_dps_vector_to_rtn.call_args_list[6].args[0]) + self.assertEqual( + dps_to_rtn_matrices[1], + mock_rotate_dps_vector_to_rtn.call_args_list[6].args[0], + ) np.testing.assert_array_equal( - np.array([halo_moments2.velocity_x, halo_moments2.velocity_y, halo_moments2.velocity_z]), - mock_rotate_dps_vector_to_rtn.call_args_list[6].args[1]) + np.array( + [ + halo_moments2.velocity_x, + halo_moments2.velocity_y, + halo_moments2.velocity_z, + ] + ), + mock_rotate_dps_vector_to_rtn.call_args_list[6].args[1], + ) self.assertEqual(4, mock_rotate_temperature.call_count) mock_rotate_temperature.assert_has_calls( - [call(dps_to_rtn_matrices[0], core_moments1.alpha, core_moments1.beta), - call(dps_to_rtn_matrices[0], halo_moments1.alpha, halo_moments1.beta), - call(dps_to_rtn_matrices[1], core_moments2.alpha, core_moments2.beta), - call(dps_to_rtn_matrices[1], halo_moments2.alpha, halo_moments2.beta), ]) + [ + call(dps_to_rtn_matrices[0], core_moments1.alpha, core_moments1.beta), + call(dps_to_rtn_matrices[0], halo_moments1.alpha, halo_moments1.beta), + call(dps_to_rtn_matrices[1], core_moments2.alpha, core_moments2.beta), + call(dps_to_rtn_matrices[1], halo_moments2.alpha, halo_moments2.beta), + ] + ) def call_with_array_matchers(*args): return call(*[NumpyArrayMatcher(x) for x in args]) self.assertEqual(5, mock_integrate.call_count) mock_integrate.assert_has_calls( - [call_with_array_matchers(4, 5, - corrected_energy_bins[0], expected_sin_theta, expected_cos_theta, - config["aperture_field_of_view_radians"], - swe_l2_data.phase_space_density[0], - swe_l2_data.inst_az_spin_sector[0], - 12, np.array([0, 0, 0, 0]), np.array([0, 0, 0, 0, 0, 0])), - call_with_array_matchers(4, 5, - corrected_energy_bins[0], expected_sin_theta, expected_cos_theta, - config["aperture_field_of_view_radians"], - swe_l2_data.phase_space_density[0], - swe_l2_data.inst_az_spin_sector[0], - 12, core_cdelnv, core_cdelt), - call_with_array_matchers(6, 5, - corrected_energy_bins[0], expected_sin_theta, expected_cos_theta, - config["aperture_field_of_view_radians"], - swe_l2_data.phase_space_density[0], - swe_l2_data.inst_az_spin_sector[0], - 12, np.array([0, 0, 0, 0]), np.array([0, 0, 0, 0, 0, 0])), - call_with_array_matchers(5, 4, - corrected_energy_bins[1], expected_sin_theta, expected_cos_theta, - config["aperture_field_of_view_radians"], - swe_l2_data.phase_space_density[1], - swe_l2_data.inst_az_spin_sector[1], - 14, np.array([0, 0, 0, 0]), np.array([0, 0, 0, 0, 0, 0])), - call_with_array_matchers(5, 5, - corrected_energy_bins[1], expected_sin_theta, expected_cos_theta, - config["aperture_field_of_view_radians"], - swe_l2_data.phase_space_density[1], - swe_l2_data.inst_az_spin_sector[1], - 14, np.array([0, 0, 0, 0]), np.array([0, 0, 0, 0, 0, 0])), - - ] + [ + call_with_array_matchers( + 4, + 5, + corrected_energy_bins[0], + expected_sin_theta, + expected_cos_theta, + config["aperture_field_of_view_radians"], + swe_l2_data.phase_space_density[0], + swe_l2_data.inst_az_spin_sector[0], + 12, + np.array([0, 0, 0, 0]), + np.array([0, 0, 0, 0, 0, 0]), + ), + call_with_array_matchers( + 4, + 5, + corrected_energy_bins[0], + expected_sin_theta, + expected_cos_theta, + config["aperture_field_of_view_radians"], + swe_l2_data.phase_space_density[0], + swe_l2_data.inst_az_spin_sector[0], + 12, + core_cdelnv, + core_cdelt, + ), + call_with_array_matchers( + 6, + 5, + corrected_energy_bins[0], + expected_sin_theta, + expected_cos_theta, + config["aperture_field_of_view_radians"], + swe_l2_data.phase_space_density[0], + swe_l2_data.inst_az_spin_sector[0], + 12, + np.array([0, 0, 0, 0]), + np.array([0, 0, 0, 0, 0, 0]), + ), + call_with_array_matchers( + 5, + 4, + corrected_energy_bins[1], + expected_sin_theta, + expected_cos_theta, + config["aperture_field_of_view_radians"], + swe_l2_data.phase_space_density[1], + swe_l2_data.inst_az_spin_sector[1], + 14, + np.array([0, 0, 0, 0]), + np.array([0, 0, 0, 0, 0, 0]), + ), + call_with_array_matchers( + 5, + 5, + corrected_energy_bins[1], + expected_sin_theta, + expected_cos_theta, + config["aperture_field_of_view_radians"], + swe_l2_data.phase_space_density[1], + swe_l2_data.inst_az_spin_sector[1], + 14, + np.array([0, 0, 0, 0]), + np.array([0, 0, 0, 0, 0, 0]), + ), + ] ) self.assertEqual(1, mock_scale_core_density.call_count) - mock_scale_core_density.assert_has_calls([ - call_with_array_matchers(core_integrate_output.density, core_integrate_output.velocity, - core_integrate_output.temperature, core_moments1, 3, corrected_energy_bins[0], - spacecraft_potential[0], expected_cos_theta, - config["aperture_field_of_view_radians"], - swe_l2_data.inst_az_spin_sector[0], - sentinel.core_regress_result_1, - core_integrate_output.base_energy) - ]) + mock_scale_core_density.assert_has_calls( + [ + call_with_array_matchers( + core_integrate_output.density, + core_integrate_output.velocity, + core_integrate_output.temperature, + core_moments1, + 3, + corrected_energy_bins[0], + spacecraft_potential[0], + expected_cos_theta, + config["aperture_field_of_view_radians"], + swe_l2_data.inst_az_spin_sector[0], + sentinel.core_regress_result_1, + core_integrate_output.base_energy, + ) + ] + ) self.assertEqual(1, mock_scale_halo_density.call_count) - mock_scale_halo_density.assert_has_calls([ - call_with_array_matchers(halo_integrate_output.density, halo_integrate_output.velocity, - halo_integrate_output.temperature, halo_moments1, spacecraft_potential[0], - core_halo_breakpoint[0], expected_cos_theta, - config["aperture_field_of_view_radians"], - swe_l2_data.inst_az_spin_sector[0], sentinel.halo_regress_result_1, - halo_integrate_output.base_energy) - ]) + mock_scale_halo_density.assert_has_calls( + [ + call_with_array_matchers( + halo_integrate_output.density, + halo_integrate_output.velocity, + halo_integrate_output.temperature, + halo_moments1, + spacecraft_potential[0], + core_halo_breakpoint[0], + expected_cos_theta, + config["aperture_field_of_view_radians"], + swe_l2_data.inst_az_spin_sector[0], + sentinel.halo_regress_result_1, + halo_integrate_output.base_energy, + ) + ] + ) self.assertEqual(6, mock_rotate_vector_to_rtn_spherical_coordinates.call_count) - mock_rotate_vector_to_rtn_spherical_coordinates.assert_has_calls([ - call(dps_to_rtn_matrices[0], core_integrate_output.heat_flux), - call(dps_to_rtn_matrices[0], core_primary_evec), - call(dps_to_rtn_matrices[0], total_integrate_output.heat_flux), - call(dps_to_rtn_matrices[0], total_primary_evec), - call(dps_to_rtn_matrices[0], halo_integrate_output.heat_flux), - call(dps_to_rtn_matrices[0], halo_primary_evec), - ]) + mock_rotate_vector_to_rtn_spherical_coordinates.assert_has_calls( + [ + call(dps_to_rtn_matrices[0], core_integrate_output.heat_flux), + call(dps_to_rtn_matrices[0], core_primary_evec), + call(dps_to_rtn_matrices[0], total_integrate_output.heat_flux), + call(dps_to_rtn_matrices[0], total_primary_evec), + call(dps_to_rtn_matrices[0], halo_integrate_output.heat_flux), + call(dps_to_rtn_matrices[0], halo_primary_evec), + ] + ) self.assertEqual(3, mock_calculate_primary_eigenvector.call_count) - mock_calculate_primary_eigenvector.assert_has_calls([ - call(scaled_core_temperature), - call(total_integrate_output.temperature), - call(scaled_halo_temperature), - ]) + mock_calculate_primary_eigenvector.assert_has_calls( + [ + call(scaled_core_temperature), + call(total_integrate_output.temperature), + call(scaled_halo_temperature), + ] + ) self.assertEqual(3, mock_rotate_temperature_tensor_to_mag.call_count) - mock_rotate_temperature_tensor_to_mag.assert_has_calls([ - call(scale_core_density_output.temperature, sentinel.mag_data_1), - call(total_integrate_output.temperature, sentinel.mag_data_1), - call(scale_halo_density_output.temperature, sentinel.mag_data_1), - ]) + mock_rotate_temperature_tensor_to_mag.assert_has_calls( + [ + call(scale_core_density_output.temperature, sentinel.mag_data_1), + call(total_integrate_output.temperature, sentinel.mag_data_1), + call(scale_halo_density_output.temperature, sentinel.mag_data_1), + ] + ) # @formatter:off self.assertIsInstance(swe_moment_data, SweL3MomentData) - np.testing.assert_array_equal(swe_moment_data.core_fit_num_points, [core_moment_fit_results_1.number_of_points, - core_moment_fit_results_2.number_of_points, - np.nan]) - np.testing.assert_array_equal(swe_moment_data.core_chisq, - [core_moment_fit_results_1.chisq, core_moment_fit_results_2.chisq, np.nan]) - np.testing.assert_array_equal(swe_moment_data.halo_chisq, - [halo_moment_fit_results_1.chisq, halo_moment_fit_results_2.chisq, np.nan]) - np.testing.assert_array_equal(swe_moment_data.core_density_fit, - [core_moments1.density, core_moments2.density, np.nan]) - np.testing.assert_array_equal(swe_moment_data.halo_density_fit, - [halo_moments1.density, halo_moments2.density, np.nan]) - np.testing.assert_array_equal(swe_moment_data.core_t_parallel_fit, - [core_moments1.t_parallel, core_moments2.t_parallel, np.nan]) - np.testing.assert_array_equal(swe_moment_data.halo_t_parallel_fit, - [halo_moments1.t_parallel, halo_moments2.t_parallel, np.nan]) - np.testing.assert_array_equal(swe_moment_data.core_t_perpendicular_fit, - [core_moments1.t_perpendicular, core_moments2.t_perpendicular, np.nan]) - np.testing.assert_array_equal(swe_moment_data.halo_t_perpendicular_fit, - [halo_moments1.t_perpendicular, halo_moments2.t_perpendicular, np.nan]) - np.testing.assert_array_equal(swe_moment_data.core_temperature_phi_rtn_fit, - [core_rtn_phi_1, core_rtn_phi_2, np.nan]) - np.testing.assert_array_equal(swe_moment_data.halo_temperature_phi_rtn_fit, - [halo_rtn_phi_1, halo_rtn_phi_2, np.nan]) - np.testing.assert_array_equal(swe_moment_data.core_temperature_theta_rtn_fit, - [core_rtn_theta_1, core_rtn_theta_2, np.nan]) - np.testing.assert_array_equal(swe_moment_data.halo_temperature_theta_rtn_fit, - [halo_rtn_theta_1, halo_rtn_theta_2, np.nan]) - np.testing.assert_array_equal(swe_moment_data.core_speed_fit, - [np.linalg.norm(core_fit_velocity_1), np.linalg.norm(core_fit_velocity_2), - np.nan]) - np.testing.assert_array_equal(swe_moment_data.halo_speed_fit, - [np.linalg.norm(halo_fit_velocity_1), np.linalg.norm(halo_fit_velocity_2), - np.nan]) - np.testing.assert_array_equal(swe_moment_data.core_velocity_vector_rtn_fit, - [core_fit_velocity_1, core_fit_velocity_2, [np.nan, np.nan, np.nan]]) - np.testing.assert_array_equal(swe_moment_data.halo_velocity_vector_rtn_fit, - [halo_fit_velocity_1, halo_fit_velocity_2, [np.nan, np.nan, np.nan]]) - np.testing.assert_array_equal(swe_moment_data.core_density_integrated, - [scale_core_density_output.density, np.nan, np.nan]) - np.testing.assert_array_equal(swe_moment_data.halo_density_integrated, - [scale_halo_density_output.density, np.nan, np.nan]) - np.testing.assert_array_equal(swe_moment_data.total_density_integrated, - [total_integrate_output.density, np.nan, np.nan]) - np.testing.assert_array_equal(swe_moment_data.core_speed_integrated, - [np.linalg.norm(core_integrated_velocity_rtn), np.nan, np.nan]) - np.testing.assert_array_equal(swe_moment_data.halo_speed_integrated, - [np.linalg.norm(halo_integrated_velocity_rtn), np.nan, np.nan]) - np.testing.assert_array_equal(swe_moment_data.total_speed_integrated, - [np.linalg.norm(total_integrated_velocity_rtn), np.nan, np.nan]) - np.testing.assert_array_equal(swe_moment_data.core_velocity_vector_rtn_integrated, - [core_integrated_velocity_rtn, [np.nan, np.nan, np.nan], - [np.nan, np.nan, np.nan]]) - np.testing.assert_array_equal(swe_moment_data.halo_velocity_vector_rtn_integrated, - [halo_integrated_velocity_rtn, [np.nan, np.nan, np.nan], - [np.nan, np.nan, np.nan]]) - np.testing.assert_array_equal(swe_moment_data.total_velocity_vector_rtn_integrated, - [total_integrated_velocity_rtn, [np.nan, np.nan, np.nan], - [np.nan, np.nan, np.nan]]) - - np.testing.assert_allclose(swe_moment_data.core_heat_flux_magnitude_integrated, [10, np.nan, np.nan]) - np.testing.assert_allclose(swe_moment_data.core_heat_flux_theta_integrated, [11, np.nan, np.nan]) - np.testing.assert_allclose(swe_moment_data.core_heat_flux_phi_integrated, [12, np.nan, np.nan]) - np.testing.assert_allclose(swe_moment_data.halo_heat_flux_magnitude_integrated, [13, np.nan, np.nan]) - np.testing.assert_allclose(swe_moment_data.halo_heat_flux_theta_integrated, [14, np.nan, np.nan]) - np.testing.assert_allclose(swe_moment_data.halo_heat_flux_phi_integrated, [15, np.nan, np.nan]) - np.testing.assert_allclose(swe_moment_data.total_heat_flux_magnitude_integrated, [101, np.nan, np.nan]) - np.testing.assert_allclose(swe_moment_data.total_heat_flux_theta_integrated, [102, np.nan, np.nan]) - np.testing.assert_allclose(swe_moment_data.total_heat_flux_phi_integrated, [103, np.nan, np.nan]) - - np.testing.assert_array_equal(swe_moment_data.core_t_parallel_integrated, [t_par_1, np.nan, np.nan]) - np.testing.assert_array_equal(swe_moment_data.core_t_perpendicular_integrated, - [[t_perp_1, gyro_1], [np.nan, np.nan], [np.nan, np.nan]]) - - np.testing.assert_array_equal(swe_moment_data.halo_t_parallel_integrated, [t_par_2, np.nan, np.nan]) - np.testing.assert_array_equal(swe_moment_data.halo_t_perpendicular_integrated, - [[t_perp_2, gyro_2], [np.nan, np.nan], [np.nan, np.nan]]) - - np.testing.assert_array_equal(swe_moment_data.total_t_parallel_integrated, [t_par_3, np.nan, np.nan]) - np.testing.assert_array_equal(swe_moment_data.total_t_perpendicular_integrated, - [[t_perp_3, gyro_3], [np.nan, np.nan], [np.nan, np.nan]]) - np.testing.assert_allclose(swe_moment_data.core_temperature_theta_rtn_integrated, [17, np.nan, np.nan]) - np.testing.assert_allclose(swe_moment_data.core_temperature_phi_rtn_integrated, [18, np.nan, np.nan]) - - np.testing.assert_allclose(swe_moment_data.halo_temperature_theta_rtn_integrated, [20, np.nan, np.nan]) - np.testing.assert_allclose(swe_moment_data.halo_temperature_phi_rtn_integrated, [21, np.nan, np.nan]) - - np.testing.assert_allclose(swe_moment_data.total_temperature_theta_rtn_integrated, [105, np.nan, np.nan]) - np.testing.assert_allclose(swe_moment_data.total_temperature_phi_rtn_integrated, [106, np.nan, np.nan]) - - np.testing.assert_array_equal(swe_moment_data.core_temperature_parallel_to_mag, [11.4, np.nan, np.nan]) - np.testing.assert_array_equal(swe_moment_data.core_temperature_perpendicular_to_mag, - [[12.4, 13.4], [np.nan, np.nan], [np.nan, np.nan]]) - np.testing.assert_array_equal(swe_moment_data.halo_temperature_parallel_to_mag, [17.4, np.nan, np.nan]) - np.testing.assert_array_equal(swe_moment_data.halo_temperature_perpendicular_to_mag, - [[18.4, 19.4], [np.nan, np.nan], [np.nan, np.nan]]) - np.testing.assert_array_equal(swe_moment_data.total_temperature_parallel_to_mag, [14.4, np.nan, np.nan]) - np.testing.assert_array_equal(swe_moment_data.total_temperature_perpendicular_to_mag, - [[15.4, 16.4], [np.nan, np.nan], [np.nan, np.nan]]) - np.testing.assert_array_equal(swe_moment_data.total_temperature_perpendicular_to_mag, - [[15.4, 16.4], [np.nan, np.nan], [np.nan, np.nan]]) - - np.testing.assert_array_equal(swe_moment_data.core_temperature_tensor_integrated, - [scaled_core_temperature * 1e4, np.full(6, np.nan), np.full(6, np.nan)]) - np.testing.assert_array_equal(swe_moment_data.halo_temperature_tensor_integrated, - [scaled_halo_temperature * 1e4, np.full(6, np.nan), np.full(6, np.nan)]) - np.testing.assert_array_equal(swe_moment_data.total_temperature_tensor_integrated, - [total_temperature * 1e4, np.full(6, np.nan), np.full(6, np.nan)]) + np.testing.assert_array_equal( + swe_moment_data.core_fit_num_points, + [ + core_moment_fit_results_1.number_of_points, + core_moment_fit_results_2.number_of_points, + np.nan, + ], + ) + np.testing.assert_array_equal( + swe_moment_data.core_chisq, + [core_moment_fit_results_1.chisq, core_moment_fit_results_2.chisq, np.nan], + ) + np.testing.assert_array_equal( + swe_moment_data.halo_chisq, + [halo_moment_fit_results_1.chisq, halo_moment_fit_results_2.chisq, np.nan], + ) + np.testing.assert_array_equal( + swe_moment_data.core_density_fit, + [core_moments1.density, core_moments2.density, np.nan], + ) + np.testing.assert_array_equal( + swe_moment_data.halo_density_fit, + [halo_moments1.density, halo_moments2.density, np.nan], + ) + np.testing.assert_array_equal( + swe_moment_data.core_t_parallel_fit, + [core_moments1.t_parallel, core_moments2.t_parallel, np.nan], + ) + np.testing.assert_array_equal( + swe_moment_data.halo_t_parallel_fit, + [halo_moments1.t_parallel, halo_moments2.t_parallel, np.nan], + ) + np.testing.assert_array_equal( + swe_moment_data.core_t_perpendicular_fit, + [core_moments1.t_perpendicular, core_moments2.t_perpendicular, np.nan], + ) + np.testing.assert_array_equal( + swe_moment_data.halo_t_perpendicular_fit, + [halo_moments1.t_perpendicular, halo_moments2.t_perpendicular, np.nan], + ) + np.testing.assert_array_equal( + swe_moment_data.core_temperature_phi_rtn_fit, + [core_rtn_phi_1, core_rtn_phi_2, np.nan], + ) + np.testing.assert_array_equal( + swe_moment_data.halo_temperature_phi_rtn_fit, + [halo_rtn_phi_1, halo_rtn_phi_2, np.nan], + ) + np.testing.assert_array_equal( + swe_moment_data.core_temperature_theta_rtn_fit, + [core_rtn_theta_1, core_rtn_theta_2, np.nan], + ) + np.testing.assert_array_equal( + swe_moment_data.halo_temperature_theta_rtn_fit, + [halo_rtn_theta_1, halo_rtn_theta_2, np.nan], + ) + np.testing.assert_array_equal( + swe_moment_data.core_speed_fit, + [ + np.linalg.norm(core_fit_velocity_1), + np.linalg.norm(core_fit_velocity_2), + np.nan, + ], + ) + np.testing.assert_array_equal( + swe_moment_data.halo_speed_fit, + [ + np.linalg.norm(halo_fit_velocity_1), + np.linalg.norm(halo_fit_velocity_2), + np.nan, + ], + ) + np.testing.assert_array_equal( + swe_moment_data.core_velocity_vector_rtn_fit, + [core_fit_velocity_1, core_fit_velocity_2, [np.nan, np.nan, np.nan]], + ) + np.testing.assert_array_equal( + swe_moment_data.halo_velocity_vector_rtn_fit, + [halo_fit_velocity_1, halo_fit_velocity_2, [np.nan, np.nan, np.nan]], + ) + np.testing.assert_array_equal( + swe_moment_data.core_density_integrated, + [scale_core_density_output.density, np.nan, np.nan], + ) + np.testing.assert_array_equal( + swe_moment_data.halo_density_integrated, + [scale_halo_density_output.density, np.nan, np.nan], + ) + np.testing.assert_array_equal( + swe_moment_data.total_density_integrated, + [total_integrate_output.density, np.nan, np.nan], + ) + np.testing.assert_array_equal( + swe_moment_data.core_speed_integrated, + [np.linalg.norm(core_integrated_velocity_rtn), np.nan, np.nan], + ) + np.testing.assert_array_equal( + swe_moment_data.halo_speed_integrated, + [np.linalg.norm(halo_integrated_velocity_rtn), np.nan, np.nan], + ) + np.testing.assert_array_equal( + swe_moment_data.total_speed_integrated, + [np.linalg.norm(total_integrated_velocity_rtn), np.nan, np.nan], + ) + np.testing.assert_array_equal( + swe_moment_data.core_velocity_vector_rtn_integrated, + [ + core_integrated_velocity_rtn, + [np.nan, np.nan, np.nan], + [np.nan, np.nan, np.nan], + ], + ) + np.testing.assert_array_equal( + swe_moment_data.halo_velocity_vector_rtn_integrated, + [ + halo_integrated_velocity_rtn, + [np.nan, np.nan, np.nan], + [np.nan, np.nan, np.nan], + ], + ) + np.testing.assert_array_equal( + swe_moment_data.total_velocity_vector_rtn_integrated, + [ + total_integrated_velocity_rtn, + [np.nan, np.nan, np.nan], + [np.nan, np.nan, np.nan], + ], + ) + + np.testing.assert_allclose( + swe_moment_data.core_heat_flux_magnitude_integrated, [10, np.nan, np.nan] + ) + np.testing.assert_allclose( + swe_moment_data.core_heat_flux_theta_integrated, [11, np.nan, np.nan] + ) + np.testing.assert_allclose( + swe_moment_data.core_heat_flux_phi_integrated, [12, np.nan, np.nan] + ) + np.testing.assert_allclose( + swe_moment_data.halo_heat_flux_magnitude_integrated, [13, np.nan, np.nan] + ) + np.testing.assert_allclose( + swe_moment_data.halo_heat_flux_theta_integrated, [14, np.nan, np.nan] + ) + np.testing.assert_allclose( + swe_moment_data.halo_heat_flux_phi_integrated, [15, np.nan, np.nan] + ) + np.testing.assert_allclose( + swe_moment_data.total_heat_flux_magnitude_integrated, [101, np.nan, np.nan] + ) + np.testing.assert_allclose( + swe_moment_data.total_heat_flux_theta_integrated, [102, np.nan, np.nan] + ) + np.testing.assert_allclose( + swe_moment_data.total_heat_flux_phi_integrated, [103, np.nan, np.nan] + ) + + np.testing.assert_array_equal( + swe_moment_data.core_t_parallel_integrated, [t_par_1, np.nan, np.nan] + ) + np.testing.assert_array_equal( + swe_moment_data.core_t_perpendicular_integrated, + [[t_perp_1, gyro_1], [np.nan, np.nan], [np.nan, np.nan]], + ) + + np.testing.assert_array_equal( + swe_moment_data.halo_t_parallel_integrated, [t_par_2, np.nan, np.nan] + ) + np.testing.assert_array_equal( + swe_moment_data.halo_t_perpendicular_integrated, + [[t_perp_2, gyro_2], [np.nan, np.nan], [np.nan, np.nan]], + ) + + np.testing.assert_array_equal( + swe_moment_data.total_t_parallel_integrated, [t_par_3, np.nan, np.nan] + ) + np.testing.assert_array_equal( + swe_moment_data.total_t_perpendicular_integrated, + [[t_perp_3, gyro_3], [np.nan, np.nan], [np.nan, np.nan]], + ) + np.testing.assert_allclose( + swe_moment_data.core_temperature_theta_rtn_integrated, [17, np.nan, np.nan] + ) + np.testing.assert_allclose( + swe_moment_data.core_temperature_phi_rtn_integrated, [18, np.nan, np.nan] + ) + + np.testing.assert_allclose( + swe_moment_data.halo_temperature_theta_rtn_integrated, [20, np.nan, np.nan] + ) + np.testing.assert_allclose( + swe_moment_data.halo_temperature_phi_rtn_integrated, [21, np.nan, np.nan] + ) + + np.testing.assert_allclose( + swe_moment_data.total_temperature_theta_rtn_integrated, + [105, np.nan, np.nan], + ) + np.testing.assert_allclose( + swe_moment_data.total_temperature_phi_rtn_integrated, [106, np.nan, np.nan] + ) + + np.testing.assert_array_equal( + swe_moment_data.core_temperature_parallel_to_mag, [11.4, np.nan, np.nan] + ) + np.testing.assert_array_equal( + swe_moment_data.core_temperature_perpendicular_to_mag, + [[12.4, 13.4], [np.nan, np.nan], [np.nan, np.nan]], + ) + np.testing.assert_array_equal( + swe_moment_data.halo_temperature_parallel_to_mag, [17.4, np.nan, np.nan] + ) + np.testing.assert_array_equal( + swe_moment_data.halo_temperature_perpendicular_to_mag, + [[18.4, 19.4], [np.nan, np.nan], [np.nan, np.nan]], + ) + np.testing.assert_array_equal( + swe_moment_data.total_temperature_parallel_to_mag, [14.4, np.nan, np.nan] + ) + np.testing.assert_array_equal( + swe_moment_data.total_temperature_perpendicular_to_mag, + [[15.4, 16.4], [np.nan, np.nan], [np.nan, np.nan]], + ) + np.testing.assert_array_equal( + swe_moment_data.total_temperature_perpendicular_to_mag, + [[15.4, 16.4], [np.nan, np.nan], [np.nan, np.nan]], + ) + + np.testing.assert_array_equal( + swe_moment_data.core_temperature_tensor_integrated, + [scaled_core_temperature * 1e4, np.full(6, np.nan), np.full(6, np.nan)], + ) + np.testing.assert_array_equal( + swe_moment_data.halo_temperature_tensor_integrated, + [scaled_halo_temperature * 1e4, np.full(6, np.nan), np.full(6, np.nan)], + ) + np.testing.assert_array_equal( + swe_moment_data.total_temperature_tensor_integrated, + [total_temperature * 1e4, np.full(6, np.nan), np.full(6, np.nan)], + ) # @formatter:on @patch(f"{MODULE}.get_dps_to_rtn_rotation_matrix") - @patch('imap_l3_processing.swe.swe_processor.rotate_vector_to_rtn_spherical_coordinates') - @patch('imap_l3_processing.swe.swe_processor.scale_halo_density') - @patch('imap_l3_processing.swe.swe_processor.scale_core_density') - @patch('imap_l3_processing.swe.swe_processor.rotate_temperature') - @patch('imap_l3_processing.swe.swe_processor.apply_rotation_matrix') - @patch('imap_l3_processing.swe.swe_processor.compute_maxwellian_weight_factors') - @patch('imap_l3_processing.swe.swe_processor.integrate') - @patch('imap_l3_processing.swe.swe_processor.halo_fit_moments_retrying_on_failure') - @patch('imap_l3_processing.swe.swe_processor.core_fit_moments_retrying_on_failure') - def test_calculate_moment_products_only_integrates_temperatures_in_a_range(self, - mock_core_fit_moments_retrying_on_failure, - mock_halo_fit_moments_retrying_on_failure, - mock_integrate, - _, mock_rotate_dps_vector_to_rtn, - mock_rotate_temperature, - mock_scale_core_density, - mock_scale_halo_density, - mock_rotate_vector_to_rtn_spherical_coordinates, - mock_get_dps_to_rtn_rotation_matrix): + @patch( + "imap_l3_processing.swe.swe_processor.rotate_vector_to_rtn_spherical_coordinates" + ) + @patch("imap_l3_processing.swe.swe_processor.scale_halo_density") + @patch("imap_l3_processing.swe.swe_processor.scale_core_density") + @patch("imap_l3_processing.swe.swe_processor.rotate_temperature") + @patch("imap_l3_processing.swe.swe_processor.apply_rotation_matrix") + @patch("imap_l3_processing.swe.swe_processor.compute_maxwellian_weight_factors") + @patch("imap_l3_processing.swe.swe_processor.integrate") + @patch("imap_l3_processing.swe.swe_processor.halo_fit_moments_retrying_on_failure") + @patch("imap_l3_processing.swe.swe_processor.core_fit_moments_retrying_on_failure") + def test_calculate_moment_products_only_integrates_temperatures_in_a_range( + self, + mock_core_fit_moments_retrying_on_failure, + mock_halo_fit_moments_retrying_on_failure, + mock_integrate, + _, + mock_rotate_dps_vector_to_rtn, + mock_rotate_temperature, + mock_scale_core_density, + mock_scale_halo_density, + mock_rotate_vector_to_rtn_spherical_coordinates, + mock_get_dps_to_rtn_rotation_matrix, + ): mock_core_fit_moments_retrying_on_failure.side_effect = [ - build_moment_fit_results(moments=build_moments(t_parallel=1e3 - 1, t_perpendicular=1e3 - 1)), - build_moment_fit_results(moments=build_moments(t_parallel=1e3 + 1, t_perpendicular=1e3 + 1)), - build_moment_fit_results(moments=build_moments(t_parallel=1e7 - 1, t_perpendicular=1e7 - 1)), - build_moment_fit_results(moments=build_moments(t_parallel=1e7 + 1, t_perpendicular=1e7 + 1)), + build_moment_fit_results( + moments=build_moments(t_parallel=1e3 - 1, t_perpendicular=1e3 - 1) + ), + build_moment_fit_results( + moments=build_moments(t_parallel=1e3 + 1, t_perpendicular=1e3 + 1) + ), + build_moment_fit_results( + moments=build_moments(t_parallel=1e7 - 1, t_perpendicular=1e7 - 1) + ), + build_moment_fit_results( + moments=build_moments(t_parallel=1e7 + 1, t_perpendicular=1e7 + 1) + ), ] mock_halo_fit_moments_retrying_on_failure.side_effect = [ - build_moment_fit_results(moments=build_moments(t_parallel=1e4 - 1, t_perpendicular=1e4 - 1)), - build_moment_fit_results(moments=build_moments(t_parallel=1e4 + 1, t_perpendicular=1e4 + 1)), - build_moment_fit_results(moments=build_moments(t_parallel=1e8 - 1, t_perpendicular=1e8 - 1)), - build_moment_fit_results(moments=build_moments(t_parallel=1e8 + 1, t_perpendicular=1e8 + 1)), + build_moment_fit_results( + moments=build_moments(t_parallel=1e4 - 1, t_perpendicular=1e4 - 1) + ), + build_moment_fit_results( + moments=build_moments(t_parallel=1e4 + 1, t_perpendicular=1e4 + 1) + ), + build_moment_fit_results( + moments=build_moments(t_parallel=1e8 - 1, t_perpendicular=1e8 - 1) + ), + build_moment_fit_results( + moments=build_moments(t_parallel=1e8 + 1, t_perpendicular=1e8 + 1) + ), ] - mock_rotate_temperature.return_value = ((4, 5)) - mock_rotate_vector_to_rtn_spherical_coordinates.return_value = ((1, 2, 3)) + mock_rotate_temperature.return_value = (4, 5) + mock_rotate_vector_to_rtn_spherical_coordinates.return_value = (1, 2, 3) - input_metadata = InputMetadata("swe", "l3", datetime(2025, 2, 21), - datetime(2025, 2, 22), "v001") + input_metadata = InputMetadata( + "swe", "l3", datetime(2025, 2, 21), datetime(2025, 2, 22), "v001" + ) swe_processor = SweProcessor(dependencies=[], input_metadata=input_metadata) config = build_swe_configuration() @@ -1513,14 +2478,18 @@ def test_calculate_moment_products_only_integrates_temperatures_in_a_range(self, phase_space_density_rebinned=np.array([]), data_quality=np.array([]), ) - swe_l1_data = SweL1bData(epoch=epochs, - count_rates=[Mock(), Mock(), Mock(), Mock()], - settle_duration=Mock()) + swe_l1_data = SweL1bData( + epoch=epochs, + count_rates=[Mock(), Mock(), Mock(), Mock()], + settle_duration=Mock(), + ) spacecraft_potential = np.array([12, 14, 16, 18]) core_halo_breakpoint = np.array([96, 54, 103, 124]) - corrected_energy_bins = swe_l2_data.energy.reshape(1, -1) - spacecraft_potential.reshape(-1, 1) + corrected_energy_bins = swe_l2_data.energy.reshape( + 1, -1 + ) - spacecraft_potential.reshape(-1, 1) mock_integrate.return_value.temperature = np.full(6, 0) @@ -1532,22 +2501,36 @@ def test_calculate_moment_products_only_integrates_temperatures_in_a_range(self, mock_rotate_dps_vector_to_rtn.return_value = (2, 3, 4) rebinned_mag_data = np.full((3, 3), 1) - swe_moment_data = swe_processor.calculate_moment_products(swe_l2_data, swe_l1_data, rebinned_mag_data, - spacecraft_potential, - core_halo_breakpoint, - corrected_energy_bins, - config) + swe_moment_data = swe_processor.calculate_moment_products( + swe_l2_data, + swe_l1_data, + rebinned_mag_data, + spacecraft_potential, + core_halo_breakpoint, + corrected_energy_bins, + config, + ) self.assertEqual(6, mock_integrate.call_count) self.assertEqual(2, mock_scale_core_density.call_count) self.assertEqual(2, mock_scale_halo_density.call_count) - np.testing.assert_array_equal([np.nan, 1, 1, np.nan], swe_moment_data.core_density_integrated) + np.testing.assert_array_equal( + [np.nan, 1, 1, np.nan], swe_moment_data.core_density_integrated + ) self.assertEqual(4, len(swe_moment_data.core_velocity_vector_rtn_integrated)) - np.testing.assert_array_equal([np.nan, np.nan, np.nan], swe_moment_data.core_velocity_vector_rtn_integrated[0]) - np.testing.assert_array_equal([np.nan, np.nan, np.nan], swe_moment_data.core_velocity_vector_rtn_integrated[3]) + np.testing.assert_array_equal( + [np.nan, np.nan, np.nan], + swe_moment_data.core_velocity_vector_rtn_integrated[0], + ) + np.testing.assert_array_equal( + [np.nan, np.nan, np.nan], + swe_moment_data.core_velocity_vector_rtn_integrated[3], + ) - np.testing.assert_array_equal([np.nan, 1, 1, np.nan], swe_moment_data.halo_density_integrated) + np.testing.assert_array_equal( + [np.nan, 1, 1, np.nan], swe_moment_data.halo_density_integrated + ) self.assertEqual(4, len(swe_moment_data.halo_velocity_vector_rtn_integrated)) @patch(f"{MODULE}.PredictedEphemerisTracker") @@ -1560,34 +2543,55 @@ def test_calculate_moment_products_only_integrates_temperatures_in_a_range(self, @patch(f"{MODULE}.calculate_velocity_in_dsp_frame_km_s") @patch(f"{MODULE}.get_dps_to_rtn_rotation_matrix") def test_calculate_moments_products_flags_data_and_uses_fill_based_on_spice_call_result( - self, - mock_get_dps_to_rtn_rotation_matrix, - mock_calculate_velocity_in_dsp_frame_km_s, - mock_compute_maxwellian_weight_factors, - mock_core_fit_moments_retrying_on_failure: Mock, - mock_halo_fit_moments_retrying_on_failure: Mock, - mock_integrate: Mock, - mock_scale_core_density: Mock, - mock_scale_halo_density: Mock, - mock_predicted_ephemeris_tracker: Mock, ): - epochs = datetime(2025, 1, 1, tzinfo=timezone.utc) + np.arange(3) * timedelta(minutes=1) - - mock_predicted_ephemeris_tracker_epoch_1 = create_autospec(PredictedEphemerisTracker, used_predict=False) - mock_predicted_ephemeris_tracker_epoch_2 = create_autospec(PredictedEphemerisTracker, used_predict=False) - mock_predicted_ephemeris_tracker_epoch_3 = create_autospec(PredictedEphemerisTracker, used_predict=True) - - mock_predicted_ephemeris_tracker.side_effect = (mock_predicted_ephemeris_tracker_epoch_1, - mock_predicted_ephemeris_tracker_epoch_2, - mock_predicted_ephemeris_tracker_epoch_3) + self, + mock_get_dps_to_rtn_rotation_matrix, + mock_calculate_velocity_in_dsp_frame_km_s, + mock_compute_maxwellian_weight_factors, + mock_core_fit_moments_retrying_on_failure: Mock, + mock_halo_fit_moments_retrying_on_failure: Mock, + mock_integrate: Mock, + mock_scale_core_density: Mock, + mock_scale_halo_density: Mock, + mock_predicted_ephemeris_tracker: Mock, + ): + epochs = datetime(2025, 1, 1, tzinfo=timezone.utc) + np.arange(3) * timedelta( + minutes=1 + ) + + mock_predicted_ephemeris_tracker_epoch_1 = create_autospec( + PredictedEphemerisTracker, used_predict=False + ) + mock_predicted_ephemeris_tracker_epoch_2 = create_autospec( + PredictedEphemerisTracker, used_predict=False + ) + mock_predicted_ephemeris_tracker_epoch_3 = create_autospec( + PredictedEphemerisTracker, used_predict=True + ) + + mock_predicted_ephemeris_tracker.side_effect = ( + mock_predicted_ephemeris_tracker_epoch_1, + mock_predicted_ephemeris_tracker_epoch_2, + mock_predicted_ephemeris_tracker_epoch_3, + ) mock_predicted_ephemeris_tracker_epoch_1.run.return_value = np.eye(3) - mock_predicted_ephemeris_tracker_epoch_2.run.side_effect = SpiceyError("Missing coverage for IMAP_DPS") + mock_predicted_ephemeris_tracker_epoch_2.run.side_effect = SpiceyError( + "Missing coverage for IMAP_DPS" + ) mock_predicted_ephemeris_tracker_epoch_3.run.return_value = np.eye(3) - mock_calculate_velocity_in_dsp_frame_km_s.return_value = np.full(shape=(24, 30, 7, 3), fill_value=1) - mock_compute_maxwellian_weight_factors.return_value = np.full(shape=(24, 30, 7), fill_value=1) - mock_core_fit_moments_retrying_on_failure.return_value = build_moment_fit_results() - mock_halo_fit_moments_retrying_on_failure.return_value = build_moment_fit_results() + mock_calculate_velocity_in_dsp_frame_km_s.return_value = np.full( + shape=(24, 30, 7, 3), fill_value=1 + ) + mock_compute_maxwellian_weight_factors.return_value = np.full( + shape=(24, 30, 7), fill_value=1 + ) + mock_core_fit_moments_retrying_on_failure.return_value = ( + build_moment_fit_results() + ) + mock_halo_fit_moments_retrying_on_failure.return_value = ( + build_moment_fit_results() + ) mock_integrate.return_value = IntegrateOutputs( 100, np.array([1, 2, 3]), @@ -1628,12 +2632,18 @@ def test_calculate_moments_products_flags_data_and_uses_fill_based_on_spice_call ) swe_l1_data = SweL1bData( epoch=epochs, - count_rates=[sentinel.l1b_count_rates_1, sentinel.l1b_count_rates_2, sentinel.l1b_count_rates_3], + count_rates=[ + sentinel.l1b_count_rates_1, + sentinel.l1b_count_rates_2, + sentinel.l1b_count_rates_3, + ], settle_duration=Mock(), ) spacecraft_potential = np.array([12, 14, 16]) core_halo_breakpoint = np.array([96, 54, 103]) - corrected_energy_bins = swe_l2_data.energy.reshape(1, -1) - spacecraft_potential.reshape(-1, 1) + corrected_energy_bins = swe_l2_data.energy.reshape( + 1, -1 + ) - spacecraft_potential.reshape(-1, 1) input_metadata = InputMetadata( "swe", @@ -1656,9 +2666,15 @@ def test_calculate_moments_products_flags_data_and_uses_fill_based_on_spice_call config, ) - mock_predicted_ephemeris_tracker_epoch_1.run.assert_called_once_with(mock_get_dps_to_rtn_rotation_matrix, epochs[0]) - mock_predicted_ephemeris_tracker_epoch_2.run.assert_called_once_with(mock_get_dps_to_rtn_rotation_matrix, epochs[1]) - mock_predicted_ephemeris_tracker_epoch_3.run.assert_called_once_with(mock_get_dps_to_rtn_rotation_matrix, epochs[2]) + mock_predicted_ephemeris_tracker_epoch_1.run.assert_called_once_with( + mock_get_dps_to_rtn_rotation_matrix, epochs[0] + ) + mock_predicted_ephemeris_tracker_epoch_2.run.assert_called_once_with( + mock_get_dps_to_rtn_rotation_matrix, epochs[1] + ) + mock_predicted_ephemeris_tracker_epoch_3.run.assert_called_once_with( + mock_get_dps_to_rtn_rotation_matrix, epochs[2] + ) rotated_field_names = ( "core_temperature_phi_rtn_fit", @@ -1713,11 +2729,20 @@ def test_calculate_moments_products_flags_data_and_uses_fill_based_on_spice_call ) _assert_field_nan_at_index_and_finite_elsewhere( - self, swe_moment_data, rotated_field_names, nan_index=1, finite_indices=(0, 2) + self, + swe_moment_data, + rotated_field_names, + nan_index=1, + finite_indices=(0, 2), ) _assert_fields_finite_everywhere(self, swe_moment_data, finite_field_names) - np.testing.assert_array_equal(swe_moment_data.quality_flags, np.array([SweL3Flags.NONE, SweL3Flags.NONE, SweL3Flags.PREDICTIVE_EPHEMERIS])) + np.testing.assert_array_equal( + swe_moment_data.quality_flags, + np.array( + [SweL3Flags.NONE, SweL3Flags.NONE, SweL3Flags.PREDICTIVE_EPHEMERIS] + ), + ) self.assertEqual(np.uint16, swe_moment_data.quality_flags.dtype) def test_calculate_moment_products_handles_bad_fit_indices_and_continues(self): @@ -1739,64 +2764,83 @@ def test_calculate_moment_products_handles_bad_fit_indices_and_continues(self): phase_space_density_rebinned=np.array([]), data_quality=np.array([]), ) - swe_l1_data = SweL1bData(epoch=epochs, - count_rates=np.full((1, 11, 3, 7), 10.5), - settle_duration=Mock()) + swe_l1_data = SweL1bData( + epoch=epochs, + count_rates=np.full((1, 11, 3, 7), 10.5), + settle_duration=Mock(), + ) spacecraft_potential = np.array([12, 14]) core_halo_breakpoint = np.array([12, 54]) - corrected_energy_bins = swe_l2_data.energy.reshape(1, -1) - spacecraft_potential.reshape(-1, 1) + corrected_energy_bins = swe_l2_data.energy.reshape( + 1, -1 + ) - spacecraft_potential.reshape(-1, 1) - input_metadata = InputMetadata("swe", "l3", datetime(2025, 2, 21), - datetime(2025, 2, 22), "v001") + input_metadata = InputMetadata( + "swe", "l3", datetime(2025, 2, 21), datetime(2025, 2, 22), "v001" + ) - swe_processor = SweProcessor(ProcessingInputCollection(), input_metadata=input_metadata) + swe_processor = SweProcessor( + ProcessingInputCollection(), input_metadata=input_metadata + ) config = build_swe_configuration() rebinned_mag_data = [sentinel.mag_data_1] - with self.assertLogs(logger, level='INFO') as log_context: - swe_moment_data = swe_processor.calculate_moment_products(swe_l2_data, swe_l1_data, rebinned_mag_data, - spacecraft_potential, - core_halo_breakpoint, - corrected_energy_bins, - config) + with self.assertLogs(logger, level="INFO") as log_context: + swe_moment_data = swe_processor.calculate_moment_products( + swe_l2_data, + swe_l1_data, + rebinned_mag_data, + spacecraft_potential, + core_halo_breakpoint, + corrected_energy_bins, + config, + ) self.assertIsInstance(swe_moment_data, SweL3MomentData) - self.assertEqual(log_context.output, - [ - "INFO:imap_l3_processing.swe.swe_processor:Bad core-halo breakpoint value at index 0. Continuing."]) - - @patch('imap_l3_processing.swe.swe_processor.rotate_temperature_tensor_to_mag') - @patch('imap_l3_processing.swe.swe_processor.calculate_primary_eigenvector') - @patch('imap_l3_processing.swe.swe_processor.rotate_vector_to_rtn_spherical_coordinates') - @patch('imap_l3_processing.swe.swe_processor.scale_halo_density') - @patch('imap_l3_processing.swe.swe_processor.scale_core_density') - @patch('imap_l3_processing.swe.swe_processor.integrate') - @patch('imap_l3_processing.swe.swe_processor.halo_fit_moments_retrying_on_failure') - @patch('imap_l3_processing.swe.swe_processor.core_fit_moments_retrying_on_failure') - @patch('imap_l3_processing.swe.swe_processor.compute_maxwellian_weight_factors') - @patch('imap_l3_processing.swe.swe_processor.calculate_velocity_in_dsp_frame_km_s') - @patch('imap_l3_processing.swe.swe_processor.apply_rotation_matrix') - @patch('imap_l3_processing.swe.swe_processor.rotate_temperature') - def test_calculate_moment_products_handles_errors_with_fill(self, mock_rotate_temperature, - mock_rotate_dps_vector_to_rtn, - mock_calculate_velocity_in_dsp_frame_km_s, - mock_compute_maxwellian_weight_factors, - mock_core_fit_moments_retrying_on_failure: Mock, - mock_halo_fit_moments_retrying_on_failure: Mock, - mock_integrate: Mock, - mock_scale_core_density: Mock, - mock_scale_halo_density: Mock, - mock_rotate_vector_to_rtn_spherical_coordinates, - mock_calculate_primary_eigenvector, - mock_rotate_temperature_tensor_to_mag): + self.assertEqual( + log_context.output, + [ + "INFO:imap_l3_processing.swe.swe_processor:Bad core-halo breakpoint value at index 0. Continuing." + ], + ) + + @patch("imap_l3_processing.swe.swe_processor.rotate_temperature_tensor_to_mag") + @patch("imap_l3_processing.swe.swe_processor.calculate_primary_eigenvector") + @patch( + "imap_l3_processing.swe.swe_processor.rotate_vector_to_rtn_spherical_coordinates" + ) + @patch("imap_l3_processing.swe.swe_processor.scale_halo_density") + @patch("imap_l3_processing.swe.swe_processor.scale_core_density") + @patch("imap_l3_processing.swe.swe_processor.integrate") + @patch("imap_l3_processing.swe.swe_processor.halo_fit_moments_retrying_on_failure") + @patch("imap_l3_processing.swe.swe_processor.core_fit_moments_retrying_on_failure") + @patch("imap_l3_processing.swe.swe_processor.compute_maxwellian_weight_factors") + @patch("imap_l3_processing.swe.swe_processor.calculate_velocity_in_dsp_frame_km_s") + @patch("imap_l3_processing.swe.swe_processor.apply_rotation_matrix") + @patch("imap_l3_processing.swe.swe_processor.rotate_temperature") + def test_calculate_moment_products_handles_errors_with_fill( + self, + mock_rotate_temperature, + mock_rotate_dps_vector_to_rtn, + mock_calculate_velocity_in_dsp_frame_km_s, + mock_compute_maxwellian_weight_factors, + mock_core_fit_moments_retrying_on_failure: Mock, + mock_halo_fit_moments_retrying_on_failure: Mock, + mock_integrate: Mock, + mock_scale_core_density: Mock, + mock_scale_halo_density: Mock, + mock_rotate_vector_to_rtn_spherical_coordinates, + mock_calculate_primary_eigenvector, + mock_rotate_temperature_tensor_to_mag, + ): mock_rotate_dps_vector_to_rtn.return_value = np.full(3, np.nan) mock_rotate_temperature.return_value = 1, 2 mock_core_fit_moments_retrying_on_failure.return_value = MomentFitResults( moments=Moments(1, 2, 3, 4, 5, 6, 7, 8, 9, 10), chisq=2.5, number_of_points=100, - regress_result=np.array([1, 2, 3, 4]) + regress_result=np.array([1, 2, 3, 4]), ) cases = [ mock_rotate_temperature, @@ -1810,7 +2854,7 @@ def test_calculate_moment_products_handles_errors_with_fill(self, mock_rotate_te mock_scale_halo_density, mock_rotate_vector_to_rtn_spherical_coordinates, mock_calculate_primary_eigenvector, - mock_rotate_temperature_tensor_to_mag + mock_rotate_temperature_tensor_to_mag, ] for failing_function in cases: @@ -1835,31 +2879,48 @@ def test_calculate_moment_products_handles_errors_with_fill(self, mock_rotate_te phase_space_density_rebinned=np.array([]), data_quality=np.array([]), ) - swe_l1_data = SweL1bData(epoch=epochs, - count_rates=[sentinel.l1b_count_rates_1, sentinel.l1b_count_rates_2, - sentinel.l1b_count_rates_3], - settle_duration=Mock()) + swe_l1_data = SweL1bData( + epoch=epochs, + count_rates=[ + sentinel.l1b_count_rates_1, + sentinel.l1b_count_rates_2, + sentinel.l1b_count_rates_3, + ], + settle_duration=Mock(), + ) spacecraft_potential = np.array([12, 14, 16]) core_halo_breakpoint = np.array([96, 54, 103]) - corrected_energy_bins = swe_l2_data.energy.reshape(1, -1) - spacecraft_potential.reshape(-1, 1) + corrected_energy_bins = swe_l2_data.energy.reshape( + 1, -1 + ) - spacecraft_potential.reshape(-1, 1) - input_metadata = InputMetadata("swe", "l3", datetime(2025, 2, 21), - datetime(2025, 2, 22), "v001") + input_metadata = InputMetadata( + "swe", "l3", datetime(2025, 2, 21), datetime(2025, 2, 22), "v001" + ) - swe_processor = SweProcessor(ProcessingInputCollection(), input_metadata=input_metadata) + swe_processor = SweProcessor( + ProcessingInputCollection(), input_metadata=input_metadata + ) config = build_swe_configuration() - rebinned_mag_data = [sentinel.mag_data_1, sentinel.mag_data_2, sentinel.mag_data_3] + rebinned_mag_data = [ + sentinel.mag_data_1, + sentinel.mag_data_2, + sentinel.mag_data_3, + ] with self.assertLogs(): - swe_moment_data = swe_processor.calculate_moment_products(swe_l2_data, swe_l1_data, - rebinned_mag_data, - spacecraft_potential, - core_halo_breakpoint, - corrected_energy_bins, - config) + swe_moment_data = swe_processor.calculate_moment_products( + swe_l2_data, + swe_l1_data, + rebinned_mag_data, + spacecraft_potential, + core_halo_breakpoint, + corrected_energy_bins, + config, + ) self.assertIsInstance(swe_moment_data, SweL3MomentData) @@ -2001,24 +3062,35 @@ def _build_finite_moment_data(num_epochs: int) -> SweL3MomentData: ) -def _assert_field_nan_at_index_and_finite_elsewhere(test: unittest.TestCase, moment_data: SweL3MomentData, - field_names: tuple[str, ...], nan_index: int, - finite_indices: tuple[int, ...]) -> None: +def _assert_field_nan_at_index_and_finite_elsewhere( + test: unittest.TestCase, + moment_data: SweL3MomentData, + field_names: tuple[str, ...], + nan_index: int, + finite_indices: tuple[int, ...], +) -> None: for field_name in field_names: field = getattr(moment_data, field_name) - test.assertTrue(np.all(np.isnan(field[nan_index])), - f"{field_name}[{nan_index}] should be NaN, got {field[nan_index]!r}") + test.assertTrue( + np.all(np.isnan(field[nan_index])), + f"{field_name}[{nan_index}] should be NaN, got {field[nan_index]!r}", + ) for finite_index in finite_indices: - test.assertTrue(np.all(np.isfinite(field[finite_index])), - f"{field_name}[{finite_index}] should be finite, got {field[finite_index]!r}") + test.assertTrue( + np.all(np.isfinite(field[finite_index])), + f"{field_name}[{finite_index}] should be finite, got {field[finite_index]!r}", + ) -def _assert_fields_finite_everywhere(test: unittest.TestCase, moment_data: SweL3MomentData, - field_names: tuple[str, ...]) -> None: +def _assert_fields_finite_everywhere( + test: unittest.TestCase, moment_data: SweL3MomentData, field_names: tuple[str, ...] +) -> None: for field_name in field_names: field = getattr(moment_data, field_name) - test.assertTrue(np.all(np.isfinite(field)), - f"{field_name} should be finite at every index, got {field!r}") + test.assertTrue( + np.all(np.isfinite(field)), + f"{field_name} should be finite at every index, got {field!r}", + ) class TestCheckAndMaskNegativeMoments(unittest.TestCase): @@ -2029,18 +3101,29 @@ def test_negative_core_density_masks_core_and_total_and_flags(self): flags = check_and_mask_negative_moments(moment_data) expected_flags = np.array( - [SweL3Flags.NONE, SweL3Flags.NEGATIVE_MOMENT, SweL3Flags.NONE], dtype=np.uint16 + [SweL3Flags.NONE, SweL3Flags.NEGATIVE_MOMENT, SweL3Flags.NONE], + dtype=np.uint16, ) np.testing.assert_array_equal(flags, expected_flags) self.assertEqual(np.uint16, flags.dtype) _assert_field_nan_at_index_and_finite_elsewhere( - self, moment_data, _CORE_INTEGRATED_FIELD_NAMES, nan_index=1, finite_indices=(0, 2) + self, + moment_data, + _CORE_INTEGRATED_FIELD_NAMES, + nan_index=1, + finite_indices=(0, 2), ) _assert_field_nan_at_index_and_finite_elsewhere( - self, moment_data, _TOTAL_INTEGRATED_FIELD_NAMES, nan_index=1, finite_indices=(0, 2) + self, + moment_data, + _TOTAL_INTEGRATED_FIELD_NAMES, + nan_index=1, + finite_indices=(0, 2), + ) + _assert_fields_finite_everywhere( + self, moment_data, _HALO_INTEGRATED_FIELD_NAMES ) - _assert_fields_finite_everywhere(self, moment_data, _HALO_INTEGRATED_FIELD_NAMES) _assert_fields_finite_everywhere(self, moment_data, _FIT_FIELD_NAMES) def test_negative_core_tensor_diagonal_masks_core_and_total_and_flags(self): @@ -2057,18 +3140,29 @@ def test_negative_core_tensor_diagonal_masks_core_and_total_and_flags(self): flags = check_and_mask_negative_moments(moment_data) expected_flags = np.array( - [SweL3Flags.NONE, SweL3Flags.NEGATIVE_MOMENT, SweL3Flags.NONE], dtype=np.uint16 + [SweL3Flags.NONE, SweL3Flags.NEGATIVE_MOMENT, SweL3Flags.NONE], + dtype=np.uint16, ) np.testing.assert_array_equal(flags, expected_flags) self.assertEqual(np.uint16, flags.dtype) _assert_field_nan_at_index_and_finite_elsewhere( - self, moment_data, _CORE_INTEGRATED_FIELD_NAMES, nan_index=1, finite_indices=(0, 2) + self, + moment_data, + _CORE_INTEGRATED_FIELD_NAMES, + nan_index=1, + finite_indices=(0, 2), ) _assert_field_nan_at_index_and_finite_elsewhere( - self, moment_data, _TOTAL_INTEGRATED_FIELD_NAMES, nan_index=1, finite_indices=(0, 2) + self, + moment_data, + _TOTAL_INTEGRATED_FIELD_NAMES, + nan_index=1, + finite_indices=(0, 2), + ) + _assert_fields_finite_everywhere( + self, moment_data, _HALO_INTEGRATED_FIELD_NAMES ) - _assert_fields_finite_everywhere(self, moment_data, _HALO_INTEGRATED_FIELD_NAMES) _assert_fields_finite_everywhere(self, moment_data, _FIT_FIELD_NAMES) def test_negative_core_tensor_off_diagonal_does_not_flag_or_mask(self): @@ -2090,9 +3184,15 @@ def test_negative_core_tensor_off_diagonal_does_not_flag_or_mask(self): np.testing.assert_array_equal(flags, expected_flags) self.assertEqual(np.uint16, flags.dtype) - _assert_fields_finite_everywhere(self, moment_data, _CORE_INTEGRATED_FIELD_NAMES) - _assert_fields_finite_everywhere(self, moment_data, _HALO_INTEGRATED_FIELD_NAMES) - _assert_fields_finite_everywhere(self, moment_data, _TOTAL_INTEGRATED_FIELD_NAMES) + _assert_fields_finite_everywhere( + self, moment_data, _CORE_INTEGRATED_FIELD_NAMES + ) + _assert_fields_finite_everywhere( + self, moment_data, _HALO_INTEGRATED_FIELD_NAMES + ) + _assert_fields_finite_everywhere( + self, moment_data, _TOTAL_INTEGRATED_FIELD_NAMES + ) _assert_fields_finite_everywhere(self, moment_data, _FIT_FIELD_NAMES) def test_negative_halo_density_masks_halo_and_total_and_flags(self): @@ -2102,18 +3202,29 @@ def test_negative_halo_density_masks_halo_and_total_and_flags(self): flags = check_and_mask_negative_moments(moment_data) expected_flags = np.array( - [SweL3Flags.NONE, SweL3Flags.NEGATIVE_MOMENT, SweL3Flags.NONE], dtype=np.uint16 + [SweL3Flags.NONE, SweL3Flags.NEGATIVE_MOMENT, SweL3Flags.NONE], + dtype=np.uint16, ) np.testing.assert_array_equal(flags, expected_flags) self.assertEqual(np.uint16, flags.dtype) _assert_field_nan_at_index_and_finite_elsewhere( - self, moment_data, _HALO_INTEGRATED_FIELD_NAMES, nan_index=1, finite_indices=(0, 2) + self, + moment_data, + _HALO_INTEGRATED_FIELD_NAMES, + nan_index=1, + finite_indices=(0, 2), ) _assert_field_nan_at_index_and_finite_elsewhere( - self, moment_data, _TOTAL_INTEGRATED_FIELD_NAMES, nan_index=1, finite_indices=(0, 2) + self, + moment_data, + _TOTAL_INTEGRATED_FIELD_NAMES, + nan_index=1, + finite_indices=(0, 2), + ) + _assert_fields_finite_everywhere( + self, moment_data, _CORE_INTEGRATED_FIELD_NAMES ) - _assert_fields_finite_everywhere(self, moment_data, _CORE_INTEGRATED_FIELD_NAMES) _assert_fields_finite_everywhere(self, moment_data, _FIT_FIELD_NAMES) def test_negative_halo_tensor_diagonal_masks_halo_and_total_and_flags(self): @@ -2130,18 +3241,29 @@ def test_negative_halo_tensor_diagonal_masks_halo_and_total_and_flags(self): flags = check_and_mask_negative_moments(moment_data) expected_flags = np.array( - [SweL3Flags.NONE, SweL3Flags.NEGATIVE_MOMENT, SweL3Flags.NONE], dtype=np.uint16 + [SweL3Flags.NONE, SweL3Flags.NEGATIVE_MOMENT, SweL3Flags.NONE], + dtype=np.uint16, ) np.testing.assert_array_equal(flags, expected_flags) self.assertEqual(np.uint16, flags.dtype) _assert_field_nan_at_index_and_finite_elsewhere( - self, moment_data, _HALO_INTEGRATED_FIELD_NAMES, nan_index=1, finite_indices=(0, 2) + self, + moment_data, + _HALO_INTEGRATED_FIELD_NAMES, + nan_index=1, + finite_indices=(0, 2), ) _assert_field_nan_at_index_and_finite_elsewhere( - self, moment_data, _TOTAL_INTEGRATED_FIELD_NAMES, nan_index=1, finite_indices=(0, 2) + self, + moment_data, + _TOTAL_INTEGRATED_FIELD_NAMES, + nan_index=1, + finite_indices=(0, 2), + ) + _assert_fields_finite_everywhere( + self, moment_data, _CORE_INTEGRATED_FIELD_NAMES ) - _assert_fields_finite_everywhere(self, moment_data, _CORE_INTEGRATED_FIELD_NAMES) _assert_fields_finite_everywhere(self, moment_data, _FIT_FIELD_NAMES) def test_negative_halo_tensor_off_diagonal_does_not_flag_or_mask(self): @@ -2163,9 +3285,15 @@ def test_negative_halo_tensor_off_diagonal_does_not_flag_or_mask(self): np.testing.assert_array_equal(flags, expected_flags) self.assertEqual(np.uint16, flags.dtype) - _assert_fields_finite_everywhere(self, moment_data, _CORE_INTEGRATED_FIELD_NAMES) - _assert_fields_finite_everywhere(self, moment_data, _HALO_INTEGRATED_FIELD_NAMES) - _assert_fields_finite_everywhere(self, moment_data, _TOTAL_INTEGRATED_FIELD_NAMES) + _assert_fields_finite_everywhere( + self, moment_data, _CORE_INTEGRATED_FIELD_NAMES + ) + _assert_fields_finite_everywhere( + self, moment_data, _HALO_INTEGRATED_FIELD_NAMES + ) + _assert_fields_finite_everywhere( + self, moment_data, _TOTAL_INTEGRATED_FIELD_NAMES + ) _assert_fields_finite_everywhere(self, moment_data, _FIT_FIELD_NAMES) def test_negative_total_density_masks_only_total_and_flags(self): @@ -2175,16 +3303,25 @@ def test_negative_total_density_masks_only_total_and_flags(self): flags = check_and_mask_negative_moments(moment_data) expected_flags = np.array( - [SweL3Flags.NONE, SweL3Flags.NEGATIVE_MOMENT, SweL3Flags.NONE], dtype=np.uint16 + [SweL3Flags.NONE, SweL3Flags.NEGATIVE_MOMENT, SweL3Flags.NONE], + dtype=np.uint16, ) np.testing.assert_array_equal(flags, expected_flags) self.assertEqual(np.uint16, flags.dtype) _assert_field_nan_at_index_and_finite_elsewhere( - self, moment_data, _TOTAL_INTEGRATED_FIELD_NAMES, nan_index=1, finite_indices=(0, 2) + self, + moment_data, + _TOTAL_INTEGRATED_FIELD_NAMES, + nan_index=1, + finite_indices=(0, 2), + ) + _assert_fields_finite_everywhere( + self, moment_data, _CORE_INTEGRATED_FIELD_NAMES + ) + _assert_fields_finite_everywhere( + self, moment_data, _HALO_INTEGRATED_FIELD_NAMES ) - _assert_fields_finite_everywhere(self, moment_data, _CORE_INTEGRATED_FIELD_NAMES) - _assert_fields_finite_everywhere(self, moment_data, _HALO_INTEGRATED_FIELD_NAMES) _assert_fields_finite_everywhere(self, moment_data, _FIT_FIELD_NAMES) def test_negative_total_tensor_diagonal_masks_only_total_and_flags(self): @@ -2201,16 +3338,25 @@ def test_negative_total_tensor_diagonal_masks_only_total_and_flags(self): flags = check_and_mask_negative_moments(moment_data) expected_flags = np.array( - [SweL3Flags.NONE, SweL3Flags.NEGATIVE_MOMENT, SweL3Flags.NONE], dtype=np.uint16 + [SweL3Flags.NONE, SweL3Flags.NEGATIVE_MOMENT, SweL3Flags.NONE], + dtype=np.uint16, ) np.testing.assert_array_equal(flags, expected_flags) self.assertEqual(np.uint16, flags.dtype) _assert_field_nan_at_index_and_finite_elsewhere( - self, moment_data, _TOTAL_INTEGRATED_FIELD_NAMES, nan_index=1, finite_indices=(0, 2) + self, + moment_data, + _TOTAL_INTEGRATED_FIELD_NAMES, + nan_index=1, + finite_indices=(0, 2), + ) + _assert_fields_finite_everywhere( + self, moment_data, _CORE_INTEGRATED_FIELD_NAMES + ) + _assert_fields_finite_everywhere( + self, moment_data, _HALO_INTEGRATED_FIELD_NAMES ) - _assert_fields_finite_everywhere(self, moment_data, _CORE_INTEGRATED_FIELD_NAMES) - _assert_fields_finite_everywhere(self, moment_data, _HALO_INTEGRATED_FIELD_NAMES) _assert_fields_finite_everywhere(self, moment_data, _FIT_FIELD_NAMES) def test_negative_total_tensor_off_diagonal_does_not_flag_or_mask(self): @@ -2232,9 +3378,15 @@ def test_negative_total_tensor_off_diagonal_does_not_flag_or_mask(self): np.testing.assert_array_equal(flags, expected_flags) self.assertEqual(np.uint16, flags.dtype) - _assert_fields_finite_everywhere(self, moment_data, _CORE_INTEGRATED_FIELD_NAMES) - _assert_fields_finite_everywhere(self, moment_data, _HALO_INTEGRATED_FIELD_NAMES) - _assert_fields_finite_everywhere(self, moment_data, _TOTAL_INTEGRATED_FIELD_NAMES) + _assert_fields_finite_everywhere( + self, moment_data, _CORE_INTEGRATED_FIELD_NAMES + ) + _assert_fields_finite_everywhere( + self, moment_data, _HALO_INTEGRATED_FIELD_NAMES + ) + _assert_fields_finite_everywhere( + self, moment_data, _TOTAL_INTEGRATED_FIELD_NAMES + ) _assert_fields_finite_everywhere(self, moment_data, _FIT_FIELD_NAMES) def test_combines_negative_sources_correctly(self): @@ -2242,8 +3394,22 @@ def test_combines_negative_sources_correctly(self): moment_data.core_density_integrated[1] = -1.0 moment_data.halo_density_integrated[2] = -1.0 - moment_data.core_temperature_tensor_integrated[3] = [-1.0, 0.0, 1.0, 0.0, 0.0, 1.0] - moment_data.halo_temperature_tensor_integrated[3] = [-1.0, 0.0, 1.0, 0.0, 0.0, 1.0] + moment_data.core_temperature_tensor_integrated[3] = [ + -1.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + ] + moment_data.halo_temperature_tensor_integrated[3] = [ + -1.0, + 0.0, + 1.0, + 0.0, + 0.0, + 1.0, + ] moment_data.total_density_integrated[3] = -1.0 flags = check_and_mask_negative_moments(moment_data) @@ -2260,8 +3426,9 @@ def test_combines_negative_sources_correctly(self): np.testing.assert_array_equal(flags, expected_flags) self.assertEqual(np.uint16, flags.dtype) - def assert_population_at_epoch(epoch: int, field_names: tuple[str, ...], - should_be_masked: bool) -> None: + def assert_population_at_epoch( + epoch: int, field_names: tuple[str, ...], should_be_masked: bool + ) -> None: for field_name in field_names: slice_at_epoch = getattr(moment_data, field_name)[epoch] if should_be_masked: @@ -2283,8 +3450,14 @@ def assert_population_at_epoch(epoch: int, field_names: tuple[str, ...], ) for epoch, core_masked, halo_masked, total_masked in epoch_masking_matrix: with self.subTest(epoch=epoch): - assert_population_at_epoch(epoch, _CORE_INTEGRATED_FIELD_NAMES, core_masked) - assert_population_at_epoch(epoch, _HALO_INTEGRATED_FIELD_NAMES, halo_masked) - assert_population_at_epoch(epoch, _TOTAL_INTEGRATED_FIELD_NAMES, total_masked) + assert_population_at_epoch( + epoch, _CORE_INTEGRATED_FIELD_NAMES, core_masked + ) + assert_population_at_epoch( + epoch, _HALO_INTEGRATED_FIELD_NAMES, halo_masked + ) + assert_population_at_epoch( + epoch, _TOTAL_INTEGRATED_FIELD_NAMES, total_masked + ) _assert_fields_finite_everywhere(self, moment_data, _FIT_FIELD_NAMES) diff --git a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/archive/map_SCFrame.py b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/archive/map_SCFrame.py index 87ff94626..c8a8b8c0d 100755 --- a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/archive/map_SCFrame.py +++ b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/archive/map_SCFrame.py @@ -13,11 +13,11 @@ nesa = 7 nalpha = 40 nphi = 3600 -dang = 0.1 * PI /180.0 +dang = 0.1 * PI / 180.0 deg = 6.0 nra = 60 ncolat = 30 -nmap = nra*ncolat*nesa +nmap = nra * ncolat * nesa def argParsing(): @@ -29,28 +29,21 @@ def argParsing(): ) parser.add_argument( - '-de', '--defiledir', + "-de", + "--defiledir", required=True, - help='Path to the location of CSV files generated by flight_to_csv.py' + help="Path to the location of CSV files generated by flight_to_csv.py", ) - + parser.add_argument( - '-se', '--start_esa', - required=False, - type = int, - help='Start ESA Number' + "-se", "--start_esa", required=False, type=int, help="Start ESA Number" ) - + parser.add_argument( - '-ee', '--end_esa', - required=False, - type = int, - help='End ESA Number' + "-ee", "--end_esa", required=False, type=int, help="End ESA Number" ) - - - return parser.parse_args() + return parser.parse_args() eff_h = 1.0 @@ -63,7 +56,7 @@ def argParsing(): 5: 0.200, 6: 0.405, 7: 0.787, - 8: 1.821 + 8: 1.821, } gf = { @@ -74,7 +67,7 @@ def argParsing(): 5: 14.0e-5, 6: 17.7e-5, 7: 22.5e-5, - 8: 6.721e-5 + 8: 6.721e-5, } ox_gf = { @@ -97,7 +90,7 @@ def argParsing(): 4: 0.000197526, 5: 0.000258304, 6: 0.000113958, - 7: 0.000189929 + 7: 0.000189929, } # args = argParsing() @@ -110,126 +103,125 @@ def argParsing(): # start_esa = args.start_esa # end_esa = args.end_esa -for pp in [75,90,105]: - work_dir1 = f'./outdir/pivot_{pp}/daily' +for pp in [75, 90, 105]: + work_dir1 = f"./outdir/pivot_{pp}/daily" map_dir = f"./outdir/pivot_{pp}/maps/" - + os.makedirs(map_dir, exist_ok=True) - + data_dir = Path(work_dir1) - + print(f"Making Maps for Pivot {pp}") - - for esa in range(1,8): - + + for esa in range(1, 8): print(esa) - h_cnts_map = np.zeros((30,60)) - h_rate_map = np.zeros((30,60)) - h_flux_map = np.zeros((30,60)) - h_func_map = np.zeros((30,60)) - h_fvar_map = np.zeros((30,60)) - exposure = np.zeros((30,60)) - background = np.zeros((30,60)) - back_map = np.zeros((30,60)) - back_rate_map = np.zeros((30,60)) - stonoise = np.zeros((30,60)) - h_rate_unc = np.zeros((30,60)) - h_rate_var = np.zeros((30,60)) - - for filepath in data_dir.glob(f'*esa{esa}.csv'): - # for yd in range(start_day,end_day+1): - # sk = '' - # file = work_dir1 + f"/data_YD_{yd}_esa{esa}.csv" + h_cnts_map = np.zeros((30, 60)) + h_rate_map = np.zeros((30, 60)) + h_flux_map = np.zeros((30, 60)) + h_func_map = np.zeros((30, 60)) + h_fvar_map = np.zeros((30, 60)) + exposure = np.zeros((30, 60)) + background = np.zeros((30, 60)) + back_map = np.zeros((30, 60)) + back_rate_map = np.zeros((30, 60)) + stonoise = np.zeros((30, 60)) + h_rate_unc = np.zeros((30, 60)) + h_rate_var = np.zeros((30, 60)) + + for filepath in data_dir.glob(f"*esa{esa}.csv"): + # for yd in range(start_day,end_day+1): + # sk = '' + # file = work_dir1 + f"/data_YD_{yd}_esa{esa}.csv" file = str(filepath) # if not os.path.exists(file): # print("No file") # continue df = pd.read_csv(file) - ps_ra = df['ra'].values - ps_dec = df['dec'].values - counts = df['counts'].values - expo = df['expo'].values - + ps_ra = df["ra"].values + ps_dec = df["dec"].values + counts = df["counts"].values + expo = df["expo"].values + for ia in range(0, 60): - ra = ps_ra[ia] - dec = ps_dec[ia] - - theta = 90.0 + dec - - imap = int(ra/deg) - if (imap == 60): - imap = 0 - jmap = int(theta/deg) - if (jmap == 30): - # jmap = 29 - jmap = 0 - - h_cnts_map[jmap,imap] += counts[ia] - exposure[jmap,imap] += expo[ia] - + ra = ps_ra[ia] + dec = ps_dec[ia] + + theta = 90.0 + dec + + imap = int(ra / deg) + if imap == 60: + imap = 0 + jmap = int(theta / deg) + if jmap == 30: + # jmap = 29 + jmap = 0 + + h_cnts_map[jmap, imap] += counts[ia] + exposure[jmap, imap] += expo[ia] + for imap in range(0, nra): - for jmap in range(0,ncolat): - - expo = exposure[jmap,imap] + for jmap in range(0, ncolat): + expo = exposure[jmap, imap] energy = esa_energy[esa] # if el == 'Hy': # geo = gf[esa] # brate = bg_rate[esa] # else: # geo = ox_gf[esa] - + geo = gf[esa] brate = bg_rate[esa] - - if (expo > 0.0): - back_rate_map[jmap,imap] = background[jmap,imap] - h_rate_map[jmap,imap] = h_cnts_map[jmap,imap] / expo - h_flux_map[jmap,imap] = h_rate_map[jmap,imap] / (geo * energy) - back_map[jmap,imap] = background[jmap,imap] / (geo * energy) - h_rate_unc[jmap,imap] = np.sqrt( h_cnts_map[jmap,imap] )/expo - h_rate_var[jmap,imap] = h_rate_unc[jmap,imap]**2 - if (brate > 0.0): - stonoise[jmap,imap] = h_rate_map[jmap,imap] / ( brate ) - if (h_cnts_map[jmap,imap] > 1.0): - h_func_map[jmap,imap] = h_flux_map[jmap,imap] / np.sqrt( h_cnts_map[jmap,imap] ) - h_fvar_map[jmap,imap] = h_func_map[jmap,imap]*2 + + if expo > 0.0: + back_rate_map[jmap, imap] = background[jmap, imap] + h_rate_map[jmap, imap] = h_cnts_map[jmap, imap] / expo + h_flux_map[jmap, imap] = h_rate_map[jmap, imap] / (geo * energy) + back_map[jmap, imap] = background[jmap, imap] / (geo * energy) + h_rate_unc[jmap, imap] = np.sqrt(h_cnts_map[jmap, imap]) / expo + h_rate_var[jmap, imap] = h_rate_unc[jmap, imap] ** 2 + if brate > 0.0: + stonoise[jmap, imap] = h_rate_map[jmap, imap] / (brate) + if h_cnts_map[jmap, imap] > 1.0: + h_func_map[jmap, imap] = h_flux_map[jmap, imap] / np.sqrt( + h_cnts_map[jmap, imap] + ) + h_fvar_map[jmap, imap] = h_func_map[jmap, imap] * 2 else: - h_func_map[jmap,imap] = 0.0 - h_fvar_map[jmap,imap] = 0.0 - - + h_func_map[jmap, imap] = 0.0 + h_fvar_map[jmap, imap] = 0.0 + sbg_file = pd.DataFrame(stonoise) - sbg_file.to_csv(map_dir+f"/map_stbg_esa{esa}.csv", index=False) - + sbg_file.to_csv(map_dir + f"/map_stbg_esa{esa}.csv", index=False) + expo_file = pd.DataFrame(exposure) - expo_file.to_csv(map_dir+f"/map_expo_esa{esa}.csv", index=False) - + expo_file.to_csv(map_dir + f"/map_expo_esa{esa}.csv", index=False) + cnts_file = pd.DataFrame(h_cnts_map) - cnts_file.to_csv(map_dir+f"/map_cnts_esa{esa}.csv", index=False) - + cnts_file.to_csv(map_dir + f"/map_cnts_esa{esa}.csv", index=False) + rate_file = pd.DataFrame(h_rate_map) - rate_file.to_csv(map_dir+f"/map_rate_esa{esa}.csv", index=False) - + rate_file.to_csv(map_dir + f"/map_rate_esa{esa}.csv", index=False) + flux_file = pd.DataFrame(h_flux_map) - flux_file.to_csv(map_dir+f"/map_flux_esa{esa}.csv", index=False) - + flux_file.to_csv(map_dir + f"/map_flux_esa{esa}.csv", index=False) + func_file = pd.DataFrame(h_func_map) - func_file.to_csv(map_dir+f"/map_func_esa{esa}.csv", index=False) - + func_file.to_csv(map_dir + f"/map_func_esa{esa}.csv", index=False) + rate_unc_file = pd.DataFrame(h_rate_unc) - rate_unc_file.to_csv(map_dir+f"/map_runc_esa{esa}.csv", index=False) - + rate_unc_file.to_csv(map_dir + f"/map_runc_esa{esa}.csv", index=False) + h_rate_var_file = pd.DataFrame(h_rate_var) - h_rate_var_file.to_csv(map_dir+f"/map_rvar_esa{esa}.csv", index=False) - + h_rate_var_file.to_csv(map_dir + f"/map_rvar_esa{esa}.csv", index=False) + h_fvar_map_file = pd.DataFrame(h_fvar_map) - h_fvar_map_file.to_csv(map_dir+f"/map_fvar_esa{esa}.csv", index=False) - + h_fvar_map_file.to_csv(map_dir + f"/map_fvar_esa{esa}.csv", index=False) + # func_file = pd.DataFrame(h_func_map) # func_file.to_csv(f"2009_lo_maps/2009_lo_map_func_{esa}_{el}.csv", index=False) - + # back_file = pd.DataFrame(back_map) # back_file.to_csv(f"2009_lo_maps/2009_lo_map_back_{esa}_{el}.csv", index=False) - + # backrate_file = pd.DataFrame(back_rate_map) # backrate_file.to_csv(f"2009_lo_maps/2009_lo_map_backrate_{esa}_{el}.csv", index=False) diff --git a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/crap/new_patch_arc2.py b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/crap/new_patch_arc2.py index 720ff65a9..e49c2ca88 100755 --- a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/crap/new_patch_arc2.py +++ b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/crap/new_patch_arc2.py @@ -7,10 +7,11 @@ PI = np.pi -def somewhat_great_circle_centered(rot, lat0, lon0, angle, npts=400): + +def somewhat_great_circle_centered(rot, lat0, lon0, angle, npts=400): """ Somewhat great circle whose pole is (lat0, lon0). - angle is the angle from the center + angle is the angle from the center Returns coordinates in the ROTATED frame. """ @@ -28,25 +29,22 @@ def somewhat_great_circle_centered(rot, lat0, lon0, angle, npts=400): v = np.cross(p, u) - t = np.linspace(0, 2*np.pi, npts) + t = np.linspace(0, 2 * np.pi, npts) angle_r = np.radians(angle) - pts = ( - np.cos(angle_r) * p - + np.sin(angle_r) * ( - np.outer(np.cos(t), u) + - np.outer(np.sin(t), v) + pts = np.cos(angle_r) * p + np.sin(angle_r) * ( + np.outer(np.cos(t), u) + np.outer(np.sin(t), v) ) -) # rotate into map frame pts_rot = rot.apply(pts) - lat = np.degrees(np.arcsin(pts_rot[:,2])) - lon = np.degrees(np.arctan2(pts_rot[:,1], pts_rot[:,0])) + lat = np.degrees(np.arcsin(pts_rot[:, 2])) + lon = np.degrees(np.arctan2(pts_rot[:, 1], pts_rot[:, 0])) lon = (lon + 180) % 360 - 180 return lat, lon + def plot_wrapped_line(ax, lon, lat, **plot_kwargs): """ Plot a line while avoiding wraparound artifacts. @@ -56,14 +54,16 @@ def plot_wrapped_line(ax, lon, lat, **plot_kwargs): lat = np.asarray(lat) jumps = np.where(np.abs(np.diff(lon)) > 180)[0] - split_idx = np.append(jumps, len(lon)-1) + split_idx = np.append(jumps, len(lon) - 1) start = 0 for idx in split_idx: - ax.plot(lon[start:idx+1], - lat[start:idx+1], - transform=ccrs.PlateCarree(), - **plot_kwargs) + ax.plot( + lon[start : idx + 1], + lat[start : idx + 1], + transform=ccrs.PlateCarree(), + **plot_kwargs, + ) start = idx + 1 @@ -75,14 +75,16 @@ def sph_to_cart(lat, lon): z = np.sin(lat_rad) return np.array([x, y, z]) + def rotation_matrix_to_center(new_colat0, new_lon0): # Step 1: rotate around Z axis by -lon0 - r1 = R.from_euler('z', -new_lon0, degrees=True) + r1 = R.from_euler("z", -new_lon0, degrees=True) # Step 2: rotate around Y axis by 90-lat0 - r2 = R.from_euler('y', 90-new_colat0, degrees=True) + r2 = R.from_euler("y", 90 - new_colat0, degrees=True) # Combined rotation return r2 * r1 + def inverse_rotation_matrix_to_center(rot, xyz_rot): """ Apply inverse rotation to xyz_rot points. @@ -92,55 +94,101 @@ def inverse_rotation_matrix_to_center(rot, xyz_rot): """ return rot.inv().apply(xyz_rot) + def make_imap_lo_reference_points(rot, option=1): - if option==1: + if option == 1: npoints = 4 lons = np.array([259.200622, 255.499300, 289.800000, 79.200622]) - lats = np.array([5.116296, 34.999941, -35.600000, -5.116296]) - labels=[ 'Nose', 'V1','V2','Tail'] - if option==2: + lats = np.array([5.116296, 34.999941, -35.600000, -5.116296]) + labels = ["Nose", "V1", "V2", "Tail"] + if option == 2: npoints = 7 - lons = np.array([259.200622, 255.499300, 289.800000, 79.200622, 221,150.,0.0 ]) - lats = np.array([5.116296, 34.999941, -35.600000, -5.116296, 39, 0.0, 0.0]) - labels=[ 'Nose', 'V1','V2','Tail', 'Rcen','Stbd', 'Port'] - if option==3: + lons = np.array( + [259.200622, 255.499300, 289.800000, 79.200622, 221, 150.0, 0.0] + ) + lats = np.array([5.116296, 34.999941, -35.600000, -5.116296, 39, 0.0, 0.0]) + labels = ["Nose", "V1", "V2", "Tail", "Rcen", "Stbd", "Port"] + if option == 3: npoints = 10 - lons = np.array([259.200622, 255.499300, 289.800000, 79.200622, 221,150.,0.0, 41, 250.624523, 70.624523]) - lats = np.array([5.116296, 34.999941, -35.600000, -5.116296, 39, 0.0, 0.0, -39, -41.242979, 41.242979]) - labels=[ 'Nose', 'V1','V2','Tail', 'RibC','Stbd', 'Port','DwnB','ULSR', 'DLSR'] - + lons = np.array( + [ + 259.200622, + 255.499300, + 289.800000, + 79.200622, + 221, + 150.0, + 0.0, + 41, + 250.624523, + 70.624523, + ] + ) + lats = np.array( + [ + 5.116296, + 34.999941, + -35.600000, + -5.116296, + 39, + 0.0, + 0.0, + -39, + -41.242979, + 41.242979, + ] + ) + labels = [ + "Nose", + "V1", + "V2", + "Tail", + "RibC", + "Stbd", + "Port", + "DwnB", + "ULSR", + "DLSR", + ] # --- Reference point in ORIGINAL coordinates --- - lat_points = np.linspace(0.0,0.0,npoints) - lon_points = np.linspace(0.0,0.0,npoints) - for i in range(0,npoints): - ref_lat = lats[i] # degrees - ref_lon = lons[i] # degrees - - # Convert to xyz - xyz_ref = np.array([ - np.cos(np.radians(ref_lat)) * np.cos(np.radians(ref_lon)), - np.cos(np.radians(ref_lat)) * np.sin(np.radians(ref_lon)), - np.sin(np.radians(ref_lat)) - ]) + lat_points = np.linspace(0.0, 0.0, npoints) + lon_points = np.linspace(0.0, 0.0, npoints) + for i in range(0, npoints): + ref_lat = lats[i] # degrees + ref_lon = lons[i] # degrees + + # Convert to xyz + xyz_ref = np.array( + [ + np.cos(np.radians(ref_lat)) * np.cos(np.radians(ref_lon)), + np.cos(np.radians(ref_lat)) * np.sin(np.radians(ref_lon)), + np.sin(np.radians(ref_lat)), + ] + ) # Rotate to map frame xyz_ref_rot = rot.apply(xyz_ref) # Back to lat/lon in rotated frame lat_ref_rot = np.degrees(np.arcsin(xyz_ref_rot[2])) - lon_ref_rot = -np.degrees(np.arctan2(xyz_ref_rot[1], xyz_ref_rot[0])) # negative sign since we are looking from the inside - lon_ref_rot = (lon_ref_rot + 180) % 360 - 180 # wrap + lon_ref_rot = -np.degrees( + np.arctan2(xyz_ref_rot[1], xyz_ref_rot[0]) + ) # negative sign since we are looking from the inside + lon_ref_rot = (lon_ref_rot + 180) % 360 - 180 # wrap lat_points[i] = lat_ref_rot lon_points[i] = lon_ref_rot - + return labels, lon_points, lat_points # --- Load data --- -def make_imap_lo_map(file, lat_center, lon_center, vmin, vmax, output, opt_points=1, opt_ribbon=0): + +def make_imap_lo_map( + file, lat_center, lon_center, vmin, vmax, output, opt_points=1, opt_ribbon=0 +): data = pd.read_csv(file, header=0).values ny, nx = data.shape @@ -150,93 +198,110 @@ def make_imap_lo_map(file, lat_center, lon_center, vmin, vmax, output, opt_point lons = np.linspace(3, 357, nx) # Build corner grid for pcolormesh - lat_edges = np.linspace(-90, 90, ny+1) - lon_edges = np.linspace(0, 360, nx+1) + lat_edges = np.linspace(-90, 90, ny + 1) + lon_edges = np.linspace(0, 360, nx + 1) lon_grid, lat_grid = np.meshgrid(lon_edges, lat_edges) # Flatten grid for rotation - xyz = np.column_stack([ - np.cos(np.radians(lat_grid.ravel())) * np.cos(np.radians(lon_grid.ravel())), - np.cos(np.radians(lat_grid.ravel())) * np.sin(np.radians(lon_grid.ravel())), - np.sin(np.radians(lat_grid.ravel())) - ]) + xyz = np.column_stack( + [ + np.cos(np.radians(lat_grid.ravel())) * np.cos(np.radians(lon_grid.ravel())), + np.cos(np.radians(lat_grid.ravel())) * np.sin(np.radians(lon_grid.ravel())), + np.sin(np.radians(lat_grid.ravel())), + ] + ) # --- Rotation --- new_colat0, new_lon0 = 90.0 - lat_center, lon_center - rot = R.from_euler('y', 90-new_colat0, degrees=True) * R.from_euler('z', -new_lon0, degrees=True) + rot = R.from_euler("y", 90 - new_colat0, degrees=True) * R.from_euler( + "z", -new_lon0, degrees=True + ) xyz_rot = rot.apply(xyz) - lat_rot = np.degrees(np.arcsin(xyz_rot[:,2])).reshape(ny+1, nx+1) - lon_rot = np.degrees(np.arctan2(xyz_rot[:,1], xyz_rot[:,0])).reshape(ny+1, nx+1) + lat_rot = np.degrees(np.arcsin(xyz_rot[:, 2])).reshape(ny + 1, nx + 1) + lon_rot = np.degrees(np.arctan2(xyz_rot[:, 1], xyz_rot[:, 0])).reshape( + ny + 1, nx + 1 + ) lon_rot = (lon_rot + 180) % 360 - 180 # For the data lon_rot_inside = -lon_rot # flip longitude - #lon_edges = compute_edges(lon_rot) - #lat_edges = compute_edges(lat_rot) + # lon_edges = compute_edges(lon_rot) + # lat_edges = compute_edges(lat_rot) # --- Plot using pcolormesh --- - fig, ax = plt.subplots(figsize=(10,5), subplot_kw={'projection': ccrs.Mollweide()}) - ax.set_facecolor('darkblue') + fig, ax = plt.subplots(figsize=(10, 5), subplot_kw={"projection": ccrs.Mollweide()}) + ax.set_facecolor("darkblue") - vmin= vmin + vmin = vmin vmax = vmax data_clipped = np.clip(data, vmin, vmax) data_filled = np.copy(data_clipped) data_filled[np.isnan(data_clipped)] = vmin - pcm = ax.pcolormesh(lon_rot_inside, lat_rot, data_filled, # no interpolation! - transform=ccrs.PlateCarree(), - cmap='viridis', norm=LogNorm(vmin=vmin, vmax=vmax), shading='auto') - + pcm = ax.pcolormesh( + lon_rot_inside, + lat_rot, + data_filled, # no interpolation! + transform=ccrs.PlateCarree(), + cmap="viridis", + norm=LogNorm(vmin=vmin, vmax=vmax), + shading="auto", + ) lat_lines = np.linspace(-90, 90, 7) # every ~30 deg for lat0 in lat_lines: - lon_line = np.linspace(0, 360, nx+1) + lon_line = np.linspace(0, 360, nx + 1) lat_line = np.full_like(lon_line, lat0) # Convert to xyz and rotate - xyz_line = np.column_stack([ - np.cos(np.radians(lat_line)) * np.cos(np.radians(lon_line)), - np.cos(np.radians(lat_line)) * np.sin(np.radians(lon_line)), - np.sin(np.radians(lat_line)) - ]) + xyz_line = np.column_stack( + [ + np.cos(np.radians(lat_line)) * np.cos(np.radians(lon_line)), + np.cos(np.radians(lat_line)) * np.sin(np.radians(lon_line)), + np.sin(np.radians(lat_line)), + ] + ) xyz_line_rot = rot.apply(xyz_line) # Back to lat/lon - lat_rot_line = np.degrees(np.arcsin(xyz_line_rot[:,2])) - lon_rot_line = -np.degrees(np.arctan2(xyz_line_rot[:,1], xyz_line_rot[:,0])) + lat_rot_line = np.degrees(np.arcsin(xyz_line_rot[:, 2])) + lon_rot_line = -np.degrees(np.arctan2(xyz_line_rot[:, 1], xyz_line_rot[:, 0])) lon_rot_line = (lon_rot_line + 180) % 360 - 180 # wrap to [-180,180] # flip for inside view # --- Split line wherever jump > 180° jump_idx = np.where(np.abs(np.diff(lon_rot_line)) > 180)[0] - split_idx = np.append(jump_idx, len(lon_rot_line)-1) + split_idx = np.append(jump_idx, len(lon_rot_line) - 1) start = 0 for idx in split_idx: - seg_lon = lon_rot_line[start:idx+1] - seg_lat = lat_rot_line[start:idx+1] - ax.plot(seg_lon, seg_lat, transform=ccrs.PlateCarree(), - color='orange', lw=0.8, alpha=0.7) - start = idx+1 - - - # pick a tolerance in degrees - tol = 100.0 # widen if labels disappear + seg_lon = lon_rot_line[start : idx + 1] + seg_lat = lat_rot_line[start : idx + 1] + ax.plot( + seg_lon, + seg_lat, + transform=ccrs.PlateCarree(), + color="orange", + lw=0.8, + alpha=0.7, + ) + start = idx + 1 - lon_here = ((seg_lon[0] + 180) % 360) - 180 # ensure [-180,180] + # pick a tolerance in degrees + tol = 100.0 # widen if labels disappear - if abs(lon_here) < tol: # only near 0° rotated longitude + lon_here = ((seg_lon[0] + 180) % 360) - 180 # ensure [-180,180] + if abs(lon_here) < tol: # only near 0° rotated longitude theta = np.radians(90.0 - seg_lat[0]) - phi = -np.radians(seg_lon[0]) # inside view sign flip - phi = np.mod(phi, 2.0*np.pi) + phi = -np.radians(seg_lon[0]) # inside view sign flip + phi = np.mod(phi, 2.0 * np.pi) xyz = [ - np.sin(theta)*np.cos(phi), - np.sin(theta)*np.sin(phi), - np.cos(theta) + np.sin(theta) * np.cos(phi), + np.sin(theta) * np.sin(phi), + np.cos(theta), ] xyz_inv = rot.inv().apply(xyz) @@ -249,83 +314,92 @@ def make_imap_lo_map(file, lat_center, lon_center, vmin, vmax, output, opt_point seg_lat[0], f"{int(round(lat_orig))}°", transform=ccrs.PlateCarree(), - color='orange', + color="orange", fontsize=6, - verticalalignment='top', - horizontalalignment='right' + verticalalignment="top", + horizontalalignment="right", ) - #theta = np.radians(90.0-seg_lat[0]) - #phi = -np.radians(seg_lon[0]) # sign inversion looking from inside - #phi = np.mod(phi, 2.0*np.pi) # wrap to [0, 2π) - #if (phi > PI): + # theta = np.radians(90.0-seg_lat[0]) + # phi = -np.radians(seg_lon[0]) # sign inversion looking from inside + # phi = np.mod(phi, 2.0*np.pi) # wrap to [0, 2π) + # if (phi > PI): # xyz = [np.sin(theta)*np.cos(phi),np.sin(theta)*np.sin(phi), np.cos(theta) ] # xyz_inv = rot.inv().apply(xyz) # lat_orig = np.degrees(np.arcsin(xyz_inv[2])) # lon_orig = np.degrees(np.arctan2(xyz_inv[1], xyz_inv[0])) # if not np.isnan(lat_orig): # lat_lab_1 = round(lat_orig) - # else: + # else: # lat_lab_1 = '' # lon_lab = ((seg_lon[0]+180) % 360) - 180 # if lat_lab_1 != '': # ## lon_lab_inside = -lon_lab - ## transform=ccrs.PlateCarree(), - # color='orange', fontsize=6, - # verticalalignment='center', horizontalalignment='right') - + ## transform=ccrs.PlateCarree(), + # color='orange', fontsize=6, + # verticalalignment='center', horizontalalignment='right') lon_lines = np.linspace(0, 360, 13) # every 30 deg for lon0 in lon_lines: - lat_line = np.linspace(-90, 90, ny+1) + lat_line = np.linspace(-90, 90, ny + 1) lon_line = np.full_like(lat_line, lon0) - xyz_line = np.column_stack([ - np.cos(np.radians(lat_line)) * np.cos(np.radians(lon_line)), - np.cos(np.radians(lat_line)) * np.sin(np.radians(lon_line)), - np.sin(np.radians(lat_line)) - ]) + xyz_line = np.column_stack( + [ + np.cos(np.radians(lat_line)) * np.cos(np.radians(lon_line)), + np.cos(np.radians(lat_line)) * np.sin(np.radians(lon_line)), + np.sin(np.radians(lat_line)), + ] + ) xyz_line_rot = rot.apply(xyz_line) - lat_rot_line = np.degrees(np.arcsin(xyz_line_rot[:,2])) - lon_rot_line = -np.degrees(np.arctan2(xyz_line_rot[:,1], xyz_line_rot[:,0])) + lat_rot_line = np.degrees(np.arcsin(xyz_line_rot[:, 2])) + lon_rot_line = -np.degrees(np.arctan2(xyz_line_rot[:, 1], xyz_line_rot[:, 0])) lon_rot_line = (lon_rot_line + 180) % 360 - 180 # flip for inside view jump_idx = np.where(np.abs(np.diff(lon_rot_line)) > 180)[0] - split_idx = np.append(jump_idx, len(lon_rot_line)-1) + split_idx = np.append(jump_idx, len(lon_rot_line) - 1) start = 0 # Find the row closest to 0° rotated latitude - lat0_idx = np.argmin(np.abs(lat_rot_line)) # lat_rot_line = your rotated lat array for the line + lat0_idx = np.argmin( + np.abs(lat_rot_line) + ) # lat_rot_line = your rotated lat array for the line # Get the corresponding longitude and latitude along that row lon_line = lon_rot_line[lat0_idx] lat_line = lat_rot_line[lat0_idx] for idx in split_idx: - seg_lon = lon_rot_line[start:idx+1] - seg_lat = lat_rot_line[start:idx+1] - ax.plot(seg_lon, seg_lat, transform=ccrs.PlateCarree(), - color='orange', lw=0.8, alpha=0.7) - - mid_idx = len(seg_lon) // 2 + seg_lon = lon_rot_line[start : idx + 1] + seg_lat = lat_rot_line[start : idx + 1] + ax.plot( + seg_lon, + seg_lat, + transform=ccrs.PlateCarree(), + color="orange", + lw=0.8, + alpha=0.7, + ) + + mid_idx = len(seg_lon) // 2 lon_plot = seg_lon[mid_idx] lat_plot = seg_lat[mid_idx] - start = idx+1 + start = idx + 1 - xyz_rot = sph_to_cart(lat_plot, lon_plot) # returns (3,) or (1,3) + xyz_rot = sph_to_cart(lat_plot, lon_plot) # returns (3,) or (1,3) # print(xyz_rot) # ensure shape (1,3) for scipy Rotation xyz_rot = np.atleast_2d(xyz_rot) # print(xyz_rot) - # --- invert rotation --- + # --- invert rotation --- xyz_orig = rot.inv().apply(xyz_rot) - # --- back to spherical --- + # --- back to spherical --- lat_label = np.degrees(np.arcsin(xyz_orig[0, 2])) lon_label = np.degrees(np.arctan2(xyz_orig[0, 1], xyz_orig[0, 0])) @@ -333,34 +407,38 @@ def make_imap_lo_map(file, lat_center, lon_center, vmin, vmax, output, opt_point xyz_rot_label = rot.apply(xyz) lat_plot = np.degrees(np.arcsin(xyz_rot_label[2])) - print('lon_label = ',lon_label) - # lon_label = (-1.0*lon_label + 180) % 360 - 180 - # print('lon_label (2) = ',lon_label) + print("lon_label = ", lon_label) + # lon_label = (-1.0*lon_label + 180) % 360 - 180 + # print('lon_label (2) = ',lon_label) lon_plot = -lon_plot if np.abs(lat_label) < 30.0: - ax.text(lon_plot, lat_plot, - f"{lon_label:.0f}°", - transform=ccrs.PlateCarree(), - color='orange', fontsize=6, - verticalalignment='bottom', horizontalalignment='right') - - if (opt_points > 0): - + ax.text( + lon_plot, + lat_plot, + f"{lon_label:.0f}°", + transform=ccrs.PlateCarree(), + color="orange", + fontsize=6, + verticalalignment="bottom", + horizontalalignment="right", + ) + + if opt_points > 0: labels, lon_points, lat_points = make_imap_lo_reference_points(rot, opt_points) npoints = len(labels) - for i in range(0,npoints): + for i in range(0, npoints): lon_ref_rot = lon_points[i] lat_ref_rot = lat_points[i] ref_label = labels[i] - # --- Plot marker --- + # --- Plot marker --- ax.plot( lon_ref_rot, lat_ref_rot, - marker='o', + marker="o", markersize=1, - color='white', + color="white", transform=ccrs.PlateCarree(), - zorder=5 + zorder=5, ) # --- Add text --- @@ -369,50 +447,51 @@ def make_imap_lo_map(file, lat_center, lon_center, vmin, vmax, output, opt_point lat_ref_rot, f" {ref_label}", transform=ccrs.PlateCarree(), - color='white', + color="white", fontsize=6, - ha='left', - va='center', - zorder=5 + ha="left", + va="center", + zorder=5, ) -# now optionally add the great circle line - - if (opt_ribbon > 0): + # now optionally add the great circle line - lat_line, lon_line = somewhat_great_circle_centered( rot, - lat0=39, - lon0=221.0, - angle=70-15 + if opt_ribbon > 0: + lat_line, lon_line = somewhat_great_circle_centered( + rot, lat0=39, lon0=221.0, angle=70 - 15 ) plot_wrapped_line( - ax, - -lon_line, # inside view flip if you use it - lat_line, - color='white', - lw=0.5 + ax, + -lon_line, # inside view flip if you use it + lat_line, + color="white", + lw=0.5, ) - - lat_line, lon_line = somewhat_great_circle_centered( rot, - lat0=39, - lon0=221.0, - angle=70+15 + lat_line, lon_line = somewhat_great_circle_centered( + rot, lat0=39, lon0=221.0, angle=70 + 15 ) plot_wrapped_line( - ax, - -lon_line, # inside view flip if you use it - lat_line, - color='white', - lw=0.5 + ax, + -lon_line, # inside view flip if you use it + lat_line, + color="white", + lw=0.5, ) - - plt.colorbar(pcm, ax=ax, orientation='horizontal', pad=0.05, label='Flux') + plt.colorbar(pcm, ax=ax, orientation="horizontal", pad=0.05, label="Flux") ax.set_global() - # plt.show() - plt.savefig(output, dpi=200, facecolor='w', edgecolor='w', orientation='portrait', format=None, transparent=False, bbox_inches=None, pad_inches=0.1) - - + # plt.show() + plt.savefig( + output, + dpi=200, + facecolor="w", + edgecolor="w", + orientation="portrait", + format=None, + transparent=False, + bbox_inches=None, + pad_inches=0.1, + ) diff --git a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/csv2yaml.py b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/csv2yaml.py index a22cdf71c..7d9a653d3 100755 --- a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/csv2yaml.py +++ b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/csv2yaml.py @@ -10,15 +10,7 @@ import os from datetime import datetime, timedelta, timezone -esa_energy = { - 1: 16, - 2: 30, - 3: 56, - 4: 106, - 5: 200, - 6: 404, - 7: 787 -} +esa_energy = {1: 16, 2: 30, 3: 56, 4: 106, 5: 200, 6: 404, 7: 787} label_map = { "cnts": "Counts", @@ -31,7 +23,7 @@ "fvar": "Variance Intensity (#/cm4 s2 sr2 keV2)", "runc": "Unc Rate", "rvar": "Variance Rate (#/s2)", - "flux": "Intensity (#/cm2 s sr keV)" + "flux": "Intensity (#/cm2 s sr keV)", } @@ -43,13 +35,14 @@ def latest_monday_tag(): days_since_monday = today.weekday() last_monday = today - timedelta(days=days_since_monday) - year = last_monday.strftime('%Y') + year = last_monday.strftime("%Y") year_short = last_monday.strftime("%y") doy = last_monday.strftime("%j") - - end_day = last_monday.strftime('%Y%m%d') - return year,end_day,f"v{year_short}DOY{doy}" + end_day = last_monday.strftime("%Y%m%d") + + return year, end_day, f"v{year_short}DOY{doy}" + def csv_to_map_yaml( csv_file, @@ -61,22 +54,20 @@ def csv_to_map_yaml( uniqueness_index=0, label="Flux", pivot="90", - metadata="flux" + metadata="flux", ): data_rows = [] - with open(csv_file, newline="") as f: reader = csv.reader(f) header = next(reader) # 10–59, not used for row in reader: formatted_row = [ - f"{float(x):.3E}" if float(x) != 0 else "0.000E+00" - for x in row + f"{float(x):.3E}" if float(x) != 0 else "0.000E+00" for x in row ] data_rows.append(formatted_row) - - _,_,tag = latest_monday_tag() + + _, _, tag = latest_monday_tag() # ---------------------------- # Write YAML-like output @@ -107,76 +98,89 @@ def csv_to_map_yaml( for row in data_rows: f.write("- [" + ", ".join(row) + "]\n") -year,end_day,tag = latest_monday_tag() -start_day = '20251108' -sec_layr = f'{start_day}_{end_day}_lo_{year}' -for pp in [75,90,105]: +year, end_day, tag = latest_monday_tag() +start_day = "20251108" +sec_layr = f"{start_day}_{end_day}_lo_{year}" + +for pp in [75, 90, 105]: if pp == 75: - pp_str = 'p075' + pp_str = "p075" if pp == 90: - pp_str = 'p090' + pp_str = "p090" if pp == 105: - pp_str = 'p105' - work_dir1 = f'./outdir/pivot_{pp}/maps' + pp_str = "p105" + work_dir1 = f"./outdir/pivot_{pp}/maps" # yaml_dir_flux = f'./outdir/pivot_{pp}/yaml/cava_txt_lo_H_{tag}{pp_str}F/{sec_layr}_flux' # yaml_dir_rate = f'./outdir/pivot_{pp}/yaml/cava_txt_lo_H_{tag}{pp_str}R/{sec_layr}_rate' # yaml_dir_cnts = f'./outdir/pivot_{pp}/yaml/cava_txt_lo_H_{tag}{pp_str}C/{sec_layr}_cnts' - - yaml_dir_flux = f'./outdir/cava/cava_txt_lo_H_{tag}{pp_str}F/{sec_layr}_flux' - yaml_dir_rate = f'./outdir/cava/cava_txt_lo_H_{tag}{pp_str}R/{sec_layr}_rate' - yaml_dir_cnts = f'./outdir/cava/cava_txt_lo_H_{tag}{pp_str}C/{sec_layr}_cnts' - + + yaml_dir_flux = f"./outdir/cava/cava_txt_lo_H_{tag}{pp_str}F/{sec_layr}_flux" + yaml_dir_rate = f"./outdir/cava/cava_txt_lo_H_{tag}{pp_str}R/{sec_layr}_rate" + yaml_dir_cnts = f"./outdir/cava/cava_txt_lo_H_{tag}{pp_str}C/{sec_layr}_cnts" + os.makedirs(yaml_dir_flux, exist_ok=True) os.makedirs(yaml_dir_rate, exist_ok=True) os.makedirs(yaml_dir_cnts, exist_ok=True) - - for esa in range(1,8): - print(pp,esa) - for tt in ["expo","rate","flux","fvar","fser","rvar","cnts"]:#,"stbg","func","runc",,"rvar"]: - filename = os.path.join(work_dir1, f"map_{tt}_esa{esa}.csv") - energy = esa_energy[esa] - label = label_map[tt] - base_filename = os.path.splitext(os.path.basename(filename))[0] - - if tt=='expo': - tt='exposure' - outputs = [ os.path.join(yaml_dir_rate, f"IMAPLo-{energy}ev-{tt}.txt"), - os.path.join(yaml_dir_flux, f"IMAPLo-{energy}ev-{tt}.txt") ] - if tt=='cnts': - outputs = [ os.path.join(yaml_dir_cnts, f"IMAPLo-{energy}ev-flux.txt"), - os.path.join(yaml_dir_cnts, f"IMAPLo-{energy}ev-exposure.txt"), - os.path.join(yaml_dir_cnts, f"IMAPLo-{energy}ev-variance.txt")] - - if tt=="rvar": - tt='variance' - outputs = [ os.path.join(yaml_dir_rate, f"IMAPLo-{energy}ev-{tt}.txt") ] - - if tt=="fvar": - tt='variance' - outputs = [ os.path.join(yaml_dir_flux, f"IMAPLo-{energy}ev-{tt}.txt") ] - - if tt=="fser": - outputs = [ os.path.join(yaml_dir_flux, f"IMAPLo-{energy}ev-{tt}.txt") ] - - if tt=="flux": - tt='flux' - outputs = [ os.path.join(yaml_dir_flux, f"IMAPLo-{energy}ev-{tt}.txt") ] - - if tt =='rate': - tt='flux' - outputs = [ os.path.join(yaml_dir_rate, f"IMAPLo-{energy}ev-{tt}.txt") ] - - for output in outputs: - name = output - fst_str = name.split('/')[-2] - des = fst_str.split('_')[-1] - csv_to_map_yaml( - csv_file=filename, - output_file=output, - energy_value=energy, - map_year=year, - label=label, - pivot=pp, - metadata=des - ) + + for esa in range(1, 8): + print(pp, esa) + for tt in [ + "expo", + "rate", + "flux", + "fvar", + "fser", + "rvar", + "cnts", + ]: # ,"stbg","func","runc",,"rvar"]: + filename = os.path.join(work_dir1, f"map_{tt}_esa{esa}.csv") + energy = esa_energy[esa] + label = label_map[tt] + base_filename = os.path.splitext(os.path.basename(filename))[0] + + if tt == "expo": + tt = "exposure" + outputs = [ + os.path.join(yaml_dir_rate, f"IMAPLo-{energy}ev-{tt}.txt"), + os.path.join(yaml_dir_flux, f"IMAPLo-{energy}ev-{tt}.txt"), + ] + if tt == "cnts": + outputs = [ + os.path.join(yaml_dir_cnts, f"IMAPLo-{energy}ev-flux.txt"), + os.path.join(yaml_dir_cnts, f"IMAPLo-{energy}ev-exposure.txt"), + os.path.join(yaml_dir_cnts, f"IMAPLo-{energy}ev-variance.txt"), + ] + + if tt == "rvar": + tt = "variance" + outputs = [os.path.join(yaml_dir_rate, f"IMAPLo-{energy}ev-{tt}.txt")] + + if tt == "fvar": + tt = "variance" + outputs = [os.path.join(yaml_dir_flux, f"IMAPLo-{energy}ev-{tt}.txt")] + + if tt == "fser": + outputs = [os.path.join(yaml_dir_flux, f"IMAPLo-{energy}ev-{tt}.txt")] + + if tt == "flux": + tt = "flux" + outputs = [os.path.join(yaml_dir_flux, f"IMAPLo-{energy}ev-{tt}.txt")] + + if tt == "rate": + tt = "flux" + outputs = [os.path.join(yaml_dir_rate, f"IMAPLo-{energy}ev-{tt}.txt")] + + for output in outputs: + name = output + fst_str = name.split("/")[-2] + des = fst_str.split("_")[-1] + csv_to_map_yaml( + csv_file=filename, + output_file=output, + energy_value=energy, + map_year=year, + label=label, + pivot=pp, + metadata=des, + ) diff --git a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/csv_to_soc.py b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/csv_to_soc.py index 319521947..26e872587 100755 --- a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/csv_to_soc.py +++ b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/csv_to_soc.py @@ -16,6 +16,7 @@ # Configuration # ----------------------------- + # ----------------------------- # Helper: infer file type # ----------------------------- @@ -26,16 +27,14 @@ def infer_file_type(filename): return key return "unknown" + # ----------------------------- # Header builder # ----------------------------- def build_header(file_type, hmin, hmax): now = datetime.now().strftime("%a %b %d %H:%M:%S %Y") - meta = TYPE_MAP.get( - file_type, - {"h_title": "Unknown", "desc": "unknown"} - ) + meta = TYPE_MAP.get(file_type, {"h_title": "Unknown", "desc": "unknown"}) header_lines = [ "27:30x60:-5.0x-5.0:7x6:0:9", @@ -58,93 +57,77 @@ def build_header(file_type, hmin, hmax): " chat=0 smearspread='0/0/0' calc='0/0/0'", " frame_epoch=914561779.587 resp_class=lo_triple " "zaxis_ra_deg=+274.476346 zaxis_dec_deg=-22.782473", - " ram_ra_deg=+186.7725 ram_dec_deg=-3.2847 " - "mtype_list='40' energy_list='21,41'", - " check_mt_list='40' check_species='20' rate_factor='1000' " - "e_nominal='0.015'", + " ram_ra_deg=+186.7725 ram_dec_deg=-3.2847 mtype_list='40' energy_list='21,41'", + " check_mt_list='40' check_species='20' rate_factor='1000' e_nominal='0.015'", + "", "", "", "", "", "", - "" ] return ["# " + line for line in header_lines[:27]] -for pp in [75,90,105]: - - INPUT_DIR = f'./outdir/pivot_{pp}/maps' - OUTPUT_DIR = f'./outdir/pivot_{pp}/txt_maps/' + +for pp in [75, 90, 105]: + INPUT_DIR = f"./outdir/pivot_{pp}/maps" + OUTPUT_DIR = f"./outdir/pivot_{pp}/txt_maps/" os.makedirs(OUTPUT_DIR, exist_ok=True) - + # Mapping file type → header description TYPE_MAP = { - "cats": { - "h_title": "Total Counts", - "desc": "counts" - }, - "expo": { - "h_title": "Exposure Time", - "desc": "exposure (s)" - }, - "flux": { - "h_title": "Flux", - "desc": "flux (1 / cm^2 s sr keV)" - }, - "rate": { - "h_title": "Rate", - "desc": "rate ( 1/s)" - } + "cats": {"h_title": "Total Counts", "desc": "counts"}, + "expo": {"h_title": "Exposure Time", "desc": "exposure (s)"}, + "flux": {"h_title": "Flux", "desc": "flux (1 / cm^2 s sr keV)"}, + "rate": {"h_title": "Rate", "desc": "rate ( 1/s)"}, } - - + # ----------------------------- # Batch processing # ----------------------------- csv_files = Path(INPUT_DIR).glob("*.csv") - + input_dir = Path(INPUT_DIR) - + # if not csv_files: # raise RuntimeError(f"No CSV files found in {INPUT_DIR}") - - for csv_file in input_dir.glob("*.csv"): + for csv_file in input_dir.glob("*.csv"): if "manifest" in csv_file.name: continue fname = os.path.basename(csv_file) - + # Infer type from filename file_type = infer_file_type(fname) - + # Read CSV df = pd.read_csv(csv_file) data = df.to_numpy(dtype=float) - #data = df.iloc[:, :60].to_numpy(dtype=float) - + # data = df.iloc[:, :60].to_numpy(dtype=float) + if data.shape != (30, 60): raise Warning(f"{fname}: Expected 30x60, got {data.shape}") continue - + # Compute h_min / h_max h_min = np.nanmin(data) h_max = np.nanmax(data) - + # Build header header = build_header(file_type, h_min, h_max) - + # Output filename out_name = os.path.splitext(fname)[0] + ".txt" out_path = os.path.join(OUTPUT_DIR, out_name) - + # with open(f'/Users/hafijulislam/UNH/csv_to_isoc/converted/map_cnts_esa{esa}.txt', "w") as f: - with open(out_path, "w") as f: + with open(out_path, "w") as f: f.write("\n".join(header) + "\n") for row in data: f.write("\t".join(f"{val:.6e}" for val in row) + "\n") - + print(f"Converted Maps for Pivot {pp}") diff --git a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/l1b_to_spin.py b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/l1b_to_spin.py index 2060a937b..f14321923 100755 --- a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/l1b_to_spin.py +++ b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/l1b_to_spin.py @@ -7,6 +7,7 @@ This code produce spin angle distribution from the l1b histograms. """ + from spacepy.pycdf import CDF import numpy as np import os @@ -17,111 +18,140 @@ import re import shutil -def radec2cart(ra,th): + +def radec2cart(ra, th): x = np.cos(np.radians(ra)) * np.sin(np.radians(th)) y = np.sin(np.radians(ra)) * np.sin(np.radians(th)) z = np.cos(np.radians(th)) - - return np.array([x,y,z]) + + return np.array([x, y, z]) + def equatorial_to_ecliptic(alpha, delta): - + # Obliquity of ecliptic at J2000.0 epoch epsilon_0 = 23.43929111 # degrees - + alpha_rad = np.radians(alpha) delta_rad = np.radians(delta) epsilon_rad = np.radians(epsilon_0) - - numerator = np.sin(alpha_rad) * np.cos(epsilon_rad) + np.tan(delta_rad) * np.sin(epsilon_rad) + + numerator = np.sin(alpha_rad) * np.cos(epsilon_rad) + np.tan(delta_rad) * np.sin( + epsilon_rad + ) denominator = np.cos(alpha_rad) - + # Calculate ecliptic longitude lambda_ecl_rad = np.arctan2(numerator, denominator) - + # Calculate ecliptic latitude - beta_ecl_rad = np.arcsin(np.sin(delta_rad) * np.cos(epsilon_rad) - - np.cos(delta_rad) * np.sin(epsilon_rad) * np.sin(alpha_rad)) - + beta_ecl_rad = np.arcsin( + np.sin(delta_rad) * np.cos(epsilon_rad) + - np.cos(delta_rad) * np.sin(epsilon_rad) * np.sin(alpha_rad) + ) lambda_ecl = np.degrees(lambda_ecl_rad) beta_ecl = np.degrees(beta_ecl_rad) lambda_ecl = lambda_ecl % 360 - + return lambda_ecl, beta_ecl -def create_ra_dec(s_ra,s_dec,pivot_angle): - - spin_ra,spin_dec = s_ra,s_dec # Using inertial pointing file give spin axis information + +def create_ra_dec(s_ra, s_dec, pivot_angle): + + spin_ra, spin_dec = ( + s_ra, + s_dec, + ) # Using inertial pointing file give spin axis information spin_th = 90.0 - spin_dec - nep_ra,nep_dec = 270, 66.56 # Standard NEP direction in J2000 + nep_ra, nep_dec = 270, 66.56 # Standard NEP direction in J2000 nep_th = 90.0 - nep_dec a_spine_deg = pivot_angle - a_spine = np.radians(a_spine_deg) - + a_spine = np.radians(a_spine_deg) + def norm(v): return v / np.linalg.norm(v) - e_nep = norm(radec2cart(nep_ra, nep_th)) - e_av_spin = norm(radec2cart(spin_ra, spin_th)) - + e_nep = norm(radec2cart(nep_ra, nep_th)) + e_av_spin = norm(radec2cart(spin_ra, spin_th)) + e_perp_pole = e_nep - np.dot(e_nep, e_av_spin) * e_av_spin e_perp_pole = norm(e_perp_pole) - - e_perp_ram = np.cross(e_av_spin, e_perp_pole) - e_perp_ram = norm(e_perp_ram) - - bin_edges = np.arange(0,366,6) - bin_centers = 0.5 * (bin_edges[1:]+bin_edges[:-1]) - raf,decf = [],[] + + e_perp_ram = np.cross(e_av_spin, e_perp_pole) + e_perp_ram = norm(e_perp_ram) + + bin_edges = np.arange(0, 366, 6) + bin_centers = 0.5 * (bin_edges[1:] + bin_edges[:-1]) + raf, decf = [], [] for i in bin_centers: - - x,y,z = np.cos(a_spine)*e_av_spin+np.sin(a_spine)*np.cos(np.radians(i))*e_perp_pole + np.sin(a_spine)*np.sin(np.radians(i))*e_perp_ram + x, y, z = ( + np.cos(a_spine) * e_av_spin + + np.sin(a_spine) * np.cos(np.radians(i)) * e_perp_pole + + np.sin(a_spine) * np.sin(np.radians(i)) * e_perp_ram + ) - eq_ra = np.degrees(np.arctan2(y,x)) - eq_dec =np.degrees( np.arcsin(z)) - ra,dec = equatorial_to_ecliptic(eq_ra,eq_dec) + eq_ra = np.degrees(np.arctan2(y, x)) + eq_dec = np.degrees(np.arcsin(z)) + ra, dec = equatorial_to_ecliptic(eq_ra, eq_dec) raf.append(ra) decf.append(dec) - - return raf,decf - -def estimate_exposure_time(filename,YD, esa): - cols = ["YD","gd_start","gd_end","bin_start","bin_end","Instrument", "E-Step1", "E-Step2", "E-Step3", "E-Step4", "E-Step5", "E-Step6", "E-Step7", "Comment"] - - df1 = pd.read_csv(filename,names=cols) - - df = df1[df1['YD']==YD] + + return raf, decf + + +def estimate_exposure_time(filename, YD, esa): + cols = [ + "YD", + "gd_start", + "gd_end", + "bin_start", + "bin_end", + "Instrument", + "E-Step1", + "E-Step2", + "E-Step3", + "E-Step4", + "E-Step5", + "E-Step6", + "E-Step7", + "Comment", + ] + + df1 = pd.read_csv(filename, names=cols) + + df = df1[df1["YD"] == YD] if df.empty: raise ValueError(f"No matching rows found for {YD} in Goodtime file") - - result = np.zeros((7,60)) + + result = np.zeros((7, 60)) start_arr = [] end_arr = [] - + for _, row in df.iterrows(): # Ensure numerical types for subtraction; convert if needed - gd_end = float(row['gd_end']) - gd_start = float(row['gd_start']) + gd_end = float(row["gd_end"]) + gd_start = float(row["gd_start"]) start_arr.append(gd_start) end_arr.append(gd_end) dt = (gd_end - gd_start) / (7 * 60) for i in range(1, 7 + 1): - dist = dt * row[f'E-Step{i}'] + dist = dt * row[f"E-Step{i}"] arr = np.zeros(60) - arr[int(row['bin_start']):int(row['bin_end']) + 1] = dist - result[i-1] += arr - + arr[int(row["bin_start"]) : int(row["bin_end"]) + 1] = dist + result[i - 1] += arr + # Convert to DataFrame for clarity bin_indices = np.arange(60) - result_df = pd.DataFrame(result.T, columns=[f'E-Step{i}' for i in range(1, 7+1)]) - result_df['bin'] = bin_indices - - return start_arr,end_arr,result_df[f'E-Step{esa}'] + result_df = pd.DataFrame(result.T, columns=[f"E-Step{i}" for i in range(1, 7 + 1)]) + result_df["bin"] = bin_indices + + return start_arr, end_arr, result_df[f"E-Step{esa}"] + def route_by_pivot(data_dir, pivot_csv): @@ -133,140 +163,147 @@ def route_by_pivot(data_dir, pivot_csv): if g and int(g.group(1)) in m: shutil.move(str(f), Path(data_dir) / f"pivot_{m[int(g.group(1))]}" / f.name) + ## Input files # file_path = '/Users/hafijulislam/UNH/imap-data-access/l1b_histrates/imap_lo_l1b_histrates_20251214-repoint00078_v001.cdf' # file_path = '/Users/hafijulislam/Library/CloudStorage/GoogleDrive-hislam09@gmail.com/My Drive/UNH/imap-data-access/l1b_histrates/imap_lo_l1b_histrates_20260114-repoint00126_v001.cdf' -data_dir_path = '../input_l1b_histrates' +data_dir_path = "../input_l1b_histrates" data_dir = Path(data_dir_path) ## Make output directories -for x in [75,90,105]: - os.makedirs(f'./outdir/pivot_{x}/daily', exist_ok=True) +for x in [75, 90, 105]: + os.makedirs(f"./outdir/pivot_{x}/daily", exist_ok=True) for file in data_dir.glob("*.cdf"): try: file_path = str(file) - + # basename = os.path.basename(file_path) basename = file.name - - pointing_file = './config_files/pointing_file.csv' - goodtime_file = './config_files/imap_lo_goodtimes_2.csv' + + pointing_file = "./config_files/pointing_file.csv" + goodtime_file = "./config_files/imap_lo_goodtimes_2.csv" pivot_csv = "./config_files/share_pivot.csv" - + for f in [pointing_file, goodtime_file, pivot_csv]: if not os.path.exists(f): print(f"File not found: {f}") sys.exit(1) - - pointing_cols = ['YD', 'spin_ra','spin_dec'] + + pointing_cols = ["YD", "spin_ra", "spin_dec"] df_point = pd.read_csv(pointing_file, names=pointing_cols, skiprows=1) - + ## Convert yyyymmdd to YYYYDOY - - yymmdd = basename.split('_')[4].split('-')[0] + + yymmdd = basename.split("_")[4].split("-")[0] date = datetime.strptime(yymmdd, "%Y%m%d") YD = f"{date.year}{date.timetuple().tm_yday:03d}" int_YD = int(YD) - + print(f"Processing DOY: {YD}") - + ## Grab spin axis information from the pointing file - - df_p = df_point[df_point['YD']==int_YD] + + df_p = df_point[df_point["YD"] == int_YD] if df_p.empty: raise ValueError(f"No matching rows found for {YD} in the pointing file") - - s_ra = df_p['spin_ra'].astype(float).values[0] - s_dec = df_p['spin_dec'].astype(float).values[0] - + + s_ra = df_p["spin_ra"].astype(float).values[0] + s_dec = df_p["spin_dec"].astype(float).values[0] + ### ----------------- df_pivot = pd.read_csv(pivot_csv) - df_pp=df_pivot[df_pivot['DOY']==int_YD] - pivot = df_pp['Pivot'].astype(float).values[0] - + df_pp = df_pivot[df_pivot["DOY"] == int_YD] + pivot = df_pp["Pivot"].astype(float).values[0] + PIVOT_ANGLE = pivot - + pivot_str = f"pivot_{int(pivot)}" - - ra,dec = create_ra_dec(s_ra,s_dec,PIVOT_ANGLE) - + + ra, dec = create_ra_dec(s_ra, s_dec, PIVOT_ANGLE) + cdf = CDF(file_path) - - - for ESA in range(1,8): - + + for ESA in range(1, 8): nep_cnts = np.zeros((60)) nep_expo = np.zeros((60)) - + ## Filter through Goodtime, create masking - start_arr,end_arr,expo = estimate_exposure_time(goodtime_file ,int_YD, ESA) - + start_arr, end_arr, expo = estimate_exposure_time( + goodtime_file, int_YD, ESA + ) + met_epoch = datetime(2010, 1, 1, 0, 0, 0) - epoch_sec = cdf['epoch'][:] + epoch_sec = cdf["epoch"][:] met_sec = [] - + for x in range(len(epoch_sec)): - ss = (epoch_sec[x]-met_epoch).total_seconds() + ss = (epoch_sec[x] - met_epoch).total_seconds() met_sec.append(ss) - - met_sec = np.asarray(met_sec, dtype=float) - mask = np.zeros_like(epoch_sec,dtype=bool) - + + met_sec = np.asarray(met_sec, dtype=float) + mask = np.zeros_like(epoch_sec, dtype=bool) + for start, end in zip(start_arr, end_arr): mask |= (met_sec >= start) & (met_sec <= end) - + ### Extract values from cdf - esa = ESA-1 - hcnts = cdf['h_counts'][...][mask,esa,:] - exposure = np.sum(cdf['exposure_time_6deg'][...][mask,esa,:].T,axis=1) - + esa = ESA - 1 + hcnts = cdf["h_counts"][...][mask, esa, :] + exposure = np.sum(cdf["exposure_time_6deg"][...][mask, esa, :].T, axis=1) + ## If No filter is appled then uncomment below # hcnts = cdf['h_counts'][...][:,esa,:] # exposure = np.sum(cdf['exposure_time_6deg'][...][:,esa,:].T,axis=1) - + ## Sum over filtered time blocks - hcnts = np.sum(hcnts.T,axis=1) - + hcnts = np.sum(hcnts.T, axis=1) + ## Convert from Spin Angle to NEP angle nep_cnts[0:10] = hcnts[50:60] nep_cnts[10:30] = hcnts[0:20] nep_cnts[30:60] = hcnts[20:50] - + nep_expo[0:10] = exposure[50:60] nep_expo[10:30] = exposure[0:20] nep_expo[30:60] = exposure[20:50] - - nep_rates = nep_cnts/nep_expo - - nep_angles = np.linspace(0,360,61) + + nep_rates = nep_cnts / nep_expo + + nep_angles = np.linspace(0, 360, 61) bin_centers = 0.5 * (nep_angles[1:] + nep_angles[:-1]) - + ## Spin axis into ECLIPJ2000 - seq_ra,seq_dec = equatorial_to_ecliptic(s_ra,s_dec) - + seq_ra, seq_dec = equatorial_to_ecliptic(s_ra, s_dec) + df_new = pd.DataFrame() - - df_new['bins'] = bin_centers - df_new['counts'] = nep_cnts - df_new['ra'] = ra - df_new['dec'] = dec - df_new['expo'] = nep_expo - df_new['spin_ra'] = seq_ra - df_new['spin_dec'] = seq_dec + + df_new["bins"] = bin_centers + df_new["counts"] = nep_cnts + df_new["ra"] = ra + df_new["dec"] = dec + df_new["expo"] = nep_expo + df_new["spin_ra"] = seq_ra + df_new["spin_dec"] = seq_dec # Provenance columns for map manifest - df_new['date_yyyymmdd'] = yymmdd - df_new['yd'] = YD - df_new['repoint'] = basename.split("-repoint")[1].split("_")[0] if "-repoint" in basename else "" - df_new['pivot'] = int(pivot) - df_new['l1b_product'] = "histrates" - df_new['l1b_filename'] = basename - df_new['l1b_path'] = str(Path(file_path).resolve()) - - df_new.to_csv(f"./outdir/{pivot_str}/daily/data_YD_{YD}_esa{ESA}.csv", index=False) - + df_new["date_yyyymmdd"] = yymmdd + df_new["yd"] = YD + df_new["repoint"] = ( + basename.split("-repoint")[1].split("_")[0] + if "-repoint" in basename + else "" + ) + df_new["pivot"] = int(pivot) + df_new["l1b_product"] = "histrates" + df_new["l1b_filename"] = basename + df_new["l1b_path"] = str(Path(file_path).resolve()) + + df_new.to_csv( + f"./outdir/{pivot_str}/daily/data_YD_{YD}_esa{ESA}.csv", index=False + ) + except Exception as e: print(f"Skipping {file.name}: {e}") diff --git a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/map_SCFrame.py b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/map_SCFrame.py index 922a1983b..8c6197fdd 100755 --- a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/map_SCFrame.py +++ b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/map_SCFrame.py @@ -1,4 +1,3 @@ - #!/usr/bin/python """ Created on Wed Mar 18 2026 @@ -17,11 +16,11 @@ nesa = 7 nalpha = 40 nphi = 3600 -dang = 0.1 * PI /180.0 +dang = 0.1 * PI / 180.0 deg = 6.0 nra = 60 ncolat = 30 -nmap = nra*ncolat*nesa +nmap = nra * ncolat * nesa eff_h = 1.0 @@ -33,7 +32,7 @@ 5: 0.200, 6: 0.405, 7: 0.787, - 8: 1.821 + 8: 1.821, } gf = { @@ -44,7 +43,7 @@ 5: 14.0e-5, 6: 17.7e-5, 7: 22.5e-5, - 8: 6.721e-5 + 8: 6.721e-5, } dg = { @@ -55,7 +54,7 @@ 5: 4.5e-5, 6: 2.0e-5, 7: 1.4e-5, - 8: 6.721e-5 + 8: 6.721e-5, } ox_gf = { @@ -83,60 +82,80 @@ ## Get background Rate -def get_brate(YD,esa): - - backfile = './config_files/imap_lo_H_background.csv' - - df = pd.read_csv(backfile,sep=',', names=['YD','start','end','bin_start','bin_end','Lo','esa1','esa2','esa3','esa4','esa5','esa6','esa7','type']) - - brate = df[(df['YD']==int(YD)) & (df['type']=='rate')][f'esa{esa}'].values[0] - + +def get_brate(YD, esa): + + backfile = "./config_files/imap_lo_H_background.csv" + + df = pd.read_csv( + backfile, + sep=",", + names=[ + "YD", + "start", + "end", + "bin_start", + "bin_end", + "Lo", + "esa1", + "esa2", + "esa3", + "esa4", + "esa5", + "esa6", + "esa7", + "type", + ], + ) + + brate = df[(df["YD"] == int(YD)) & (df["type"] == "rate")][f"esa{esa}"].values[0] + return brate -for pp in [75,90,105]: + +for pp in [75, 90, 105]: pivot_deg = pp + 4.0 pivot = np.radians(pivot_deg) - work_dir1 = f'./outdir/pivot_{pp}/daily' + work_dir1 = f"./outdir/pivot_{pp}/daily" map_dir = f"./outdir/pivot_{pp}/maps/" - + os.makedirs(map_dir, exist_ok=True) - + data_dir = Path(work_dir1) - + print(f"Making Maps for Pivot {pp}") - - for esa in range(1,8): + + for esa in range(1, 8): print(esa) map_manifest_rows = [] - - h_cnts_map = np.zeros((30,60)) - exposure = np.zeros((30,60)) - - h_rate_map = np.zeros((30,60)) - h_rate_var = np.zeros((30,60)) - - h_flux_map = np.zeros((30,60)) - h_fvar_map = np.zeros((30,60)) - h_fser_map = np.zeros((30,60)) - h_fvto_map = np.zeros((30,60)) - - back_rate_map = np.zeros((30,60)) - back_rate_var = np.zeros((30,60)) - back_flux_map = np.zeros((30,60)) - back_flux_var = np.zeros((30,60)) - - stonoise_map = np.zeros((30,60)) - stonoise_var_map = np.zeros((30,60)) - - cosalpha_map = np.zeros((30,60)) - - for filepath in data_dir.glob(f'*esa{esa}.csv'): - - file = str(filepath) - filename = file.split('/')[-1] - YD = filename.split('_')[2] + + h_cnts_map = np.zeros((30, 60)) + exposure = np.zeros((30, 60)) + + h_rate_map = np.zeros((30, 60)) + h_rate_var = np.zeros((30, 60)) + + h_flux_map = np.zeros((30, 60)) + h_fvar_map = np.zeros((30, 60)) + h_fser_map = np.zeros((30, 60)) + h_fvto_map = np.zeros((30, 60)) + + back_rate_map = np.zeros((30, 60)) + back_rate_var = np.zeros((30, 60)) + back_flux_map = np.zeros((30, 60)) + back_flux_var = np.zeros((30, 60)) + + stonoise_map = np.zeros((30, 60)) + stonoise_var_map = np.zeros((30, 60)) + + cosalpha_map = np.zeros((30, 60)) + + for filepath in data_dir.glob(f"*esa{esa}.csv"): + file = str(filepath) + filename = file.split("/")[-1] + YD = filename.split("_")[2] df = pd.read_csv(file) @@ -169,78 +188,88 @@ def get_brate(YD,esa): prov["daily_csv_path"] = str(Path(file).resolve()) map_manifest_rows.append(prov) - ps_ra = df['ra'].values - ps_dec = df['dec'].values - counts = df['counts'].values - expo = df['expo'].values - + ps_ra = df["ra"].values + ps_dec = df["dec"].values + counts = df["counts"].values + expo = df["expo"].values + for ia in range(0, 60): - ra = ps_ra[ia] - dec = ps_dec[ia] - - theta = 90.0 + dec - - imap = int(ra/deg) - if (imap == 60): - imap = 0 - jmap = int(theta/deg) - if (jmap == 30): - jmap = 0 - - # Bin center angle in radians - alpha = np.radians((ia + 0.5) * 6.0) - - # coord system with x = NEP, y = RAM, z = Sun - # look_x = np.sin(pivot)*np.cos(alpha) - look_y = np.sin(pivot)*np.sin(alpha) - # look_z = np.cos(pivot) - cosalpha = look_y - - cosalpha_map[jmap, imap] = cosalpha - - h_cnts_map[jmap,imap] += counts[ia] - exposure[jmap,imap] += expo[ia] - + ra = ps_ra[ia] + dec = ps_dec[ia] + + theta = 90.0 + dec + + imap = int(ra / deg) + if imap == 60: + imap = 0 + jmap = int(theta / deg) + if jmap == 30: + jmap = 0 + + # Bin center angle in radians + alpha = np.radians((ia + 0.5) * 6.0) + + # coord system with x = NEP, y = RAM, z = Sun + # look_x = np.sin(pivot)*np.cos(alpha) + look_y = np.sin(pivot) * np.sin(alpha) + # look_z = np.cos(pivot) + cosalpha = look_y + + cosalpha_map[jmap, imap] = cosalpha + + h_cnts_map[jmap, imap] += counts[ia] + exposure[jmap, imap] += expo[ia] + for imap in range(0, nra): - for jmap in range(0,ncolat): - - expo = exposure[jmap,imap] + for jmap in range(0, ncolat): + expo = exposure[jmap, imap] energy = esa_energy[esa] geo = gf[esa] dge = dg[esa] - + # Only look up background rate if this bin has exposure, # so we don't depend on YD when there were no input files. - if (expo > 0.0): + if expo > 0.0: brate = get_brate(YD, esa) - back_rate_map[jmap,imap] = brate - back_rate_var[jmap,imap] = brate/expo - h_rate_map[jmap,imap] = h_cnts_map[jmap,imap] / expo - h_flux_map[jmap,imap] = h_rate_map[jmap,imap] / (geo * energy) - h_fser_map[jmap,imap] = h_rate_map[jmap,imap] * dge / (geo * geo * energy ) - back_flux_map[jmap,imap] = brate / (geo * energy) - back_flux_var[jmap,imap] = back_rate_var[jmap,imap] / (geo * energy)**2 + back_rate_map[jmap, imap] = brate + back_rate_var[jmap, imap] = brate / expo + h_rate_map[jmap, imap] = h_cnts_map[jmap, imap] / expo + h_flux_map[jmap, imap] = h_rate_map[jmap, imap] / (geo * energy) + h_fser_map[jmap, imap] = ( + h_rate_map[jmap, imap] * dge / (geo * geo * energy) + ) + back_flux_map[jmap, imap] = brate / (geo * energy) + back_flux_var[jmap, imap] = ( + back_rate_var[jmap, imap] / (geo * energy) ** 2 + ) # represent uncertainty in terms of variance (Poisson counts) - h_rate_var[jmap,imap] = h_rate_map[jmap,imap] / expo - if (brate > 0.0): - stonoise_map[jmap,imap] = h_rate_map[jmap,imap] / ( brate ) + h_rate_var[jmap, imap] = h_rate_map[jmap, imap] / expo + if brate > 0.0: + stonoise_map[jmap, imap] = h_rate_map[jmap, imap] / (brate) # Guard against division by zero in stonoise_var components - if (back_rate_map[jmap,imap] > 0.0) and (h_rate_map[jmap,imap] > 0.0): - stonoise_var_map[jmap,imap] = ( - h_rate_var[jmap,imap] / back_rate_map[jmap,imap]**2 - + back_rate_var[jmap,imap] / h_rate_map[jmap,imap]**2 + if (back_rate_map[jmap, imap] > 0.0) and ( + h_rate_map[jmap, imap] > 0.0 + ): + stonoise_var_map[jmap, imap] = ( + h_rate_var[jmap, imap] / back_rate_map[jmap, imap] ** 2 + + back_rate_var[jmap, imap] + / h_rate_map[jmap, imap] ** 2 ) else: - stonoise_var_map[jmap,imap] = 0.0 + stonoise_var_map[jmap, imap] = 0.0 # represent uncertainty in terms of variance (Poisson counts) - if (h_cnts_map[jmap,imap] > 0.0): - h_fvar_map[jmap,imap] = h_flux_map[jmap,imap]**2 / h_cnts_map[jmap,imap] - h_fvto_map[jmap,imap] = (h_flux_map[jmap,imap]**2 / h_cnts_map[jmap,imap]) + h_fser_map[jmap,imap]**2 + if h_cnts_map[jmap, imap] > 0.0: + h_fvar_map[jmap, imap] = ( + h_flux_map[jmap, imap] ** 2 / h_cnts_map[jmap, imap] + ) + h_fvto_map[jmap, imap] = ( + h_flux_map[jmap, imap] ** 2 / h_cnts_map[jmap, imap] + ) + h_fser_map[jmap, imap] ** 2 else: - h_fvar_map[jmap,imap] = 0.0 - h_fvto_map[jmap,imap] = h_fser_map[jmap,imap]**2 - + h_fvar_map[jmap, imap] = 0.0 + h_fvto_map[jmap, imap] = h_fser_map[jmap, imap] ** 2 + # ------------------------------------------------------------ # Final ram-side filter # Keep only pixels where cosalpha > 0 @@ -266,69 +295,60 @@ def get_brate(YD,esa): arr[~ram_mask] = 0.0 sbg_file = pd.DataFrame(stonoise_map) - sbg_file.to_csv(map_dir+f"/map_stbg_esa{esa}.csv", index=False) - + sbg_file.to_csv(map_dir + f"/map_stbg_esa{esa}.csv", index=False) + svar_file = pd.DataFrame(stonoise_var_map) - svar_file.to_csv(map_dir+f"/map_svar_esa{esa}.csv", index=False) - + svar_file.to_csv(map_dir + f"/map_svar_esa{esa}.csv", index=False) + expo_file = pd.DataFrame(exposure) - expo_file.to_csv(map_dir+f"/map_expo_esa{esa}.csv", index=False) - + expo_file.to_csv(map_dir + f"/map_expo_esa{esa}.csv", index=False) + cnts_file = pd.DataFrame(h_cnts_map) - cnts_file.to_csv(map_dir+f"/map_cnts_esa{esa}.csv", index=False) - + cnts_file.to_csv(map_dir + f"/map_cnts_esa{esa}.csv", index=False) + rate_file = pd.DataFrame(h_rate_map) - rate_file.to_csv(map_dir+f"/map_rate_esa{esa}.csv", index=False) - + rate_file.to_csv(map_dir + f"/map_rate_esa{esa}.csv", index=False) + flux_file = pd.DataFrame(h_flux_map) - flux_file.to_csv(map_dir+f"/map_flux_esa{esa}.csv", index=False) + flux_file.to_csv(map_dir + f"/map_flux_esa{esa}.csv", index=False) sflx_file = pd.DataFrame(h_fser_map) - sflx_file.to_csv(map_dir+f"/map_fser_esa{esa}.csv", index=False) - + sflx_file.to_csv(map_dir + f"/map_fser_esa{esa}.csv", index=False) + h_rate_var_file = pd.DataFrame(h_rate_var) - h_rate_var_file.to_csv(map_dir+f"/map_rvar_esa{esa}.csv", index=False) - + h_rate_var_file.to_csv(map_dir + f"/map_rvar_esa{esa}.csv", index=False) + h_fvar_map_file = pd.DataFrame(h_fvar_map) - h_fvar_map_file.to_csv(map_dir+f"/map_fvar_esa{esa}.csv", index=False) - + h_fvar_map_file.to_csv(map_dir + f"/map_fvar_esa{esa}.csv", index=False) + h_fvto_map_file = pd.DataFrame(h_fvto_map) - h_fvto_map_file.to_csv(map_dir+f"/map_fvto_esa{esa}.csv", index=False) + h_fvto_map_file.to_csv(map_dir + f"/map_fvto_esa{esa}.csv", index=False) # Background rate and flux products from the same brate backrate_file = pd.DataFrame(back_rate_map) - backrate_file.to_csv(map_dir+f"/map_brate_esa{esa}.csv", index=False) + backrate_file.to_csv(map_dir + f"/map_brate_esa{esa}.csv", index=False) backrate_var_file = pd.DataFrame(back_rate_var) - backrate_var_file.to_csv(map_dir+f"/map_bvar_esa{esa}.csv", index=False) + backrate_var_file.to_csv(map_dir + f"/map_bvar_esa{esa}.csv", index=False) bflux_file = pd.DataFrame(back_flux_map) - bflux_file.to_csv(map_dir+f"/map_bflux_esa{esa}.csv", index=False) + bflux_file.to_csv(map_dir + f"/map_bflux_esa{esa}.csv", index=False) bflux_var_file = pd.DataFrame(back_flux_var) - bflux_var_file.to_csv(map_dir+f"/map_bfvar_esa{esa}.csv", index=False) + bflux_var_file.to_csv(map_dir + f"/map_bfvar_esa{esa}.csv", index=False) cosalpha_file = pd.DataFrame(cosalpha_map) - cosalpha_file.to_csv(map_dir+f"/map_cosalpha_esa{esa}.csv", index=False) + cosalpha_file.to_csv(map_dir + f"/map_cosalpha_esa{esa}.csv", index=False) # Write map-level provenance manifest for this pivot/ESA if map_manifest_rows: - manifest_df = pd.DataFrame(map_manifest_rows) manifest_df["date_yyyymmdd"] = manifest_df["date_yyyymmdd"].astype(str) manifest_df["repoint"] = manifest_df["repoint"].astype(str) - manifest_df = ( - manifest_df - .drop_duplicates() - .sort_values( - by=["date_yyyymmdd", "repoint"], - ascending=[True, True] - ) + manifest_df = manifest_df.drop_duplicates().sort_values( + by=["date_yyyymmdd", "repoint"], ascending=[True, True] ) - manifest_df.to_csv( - map_dir + f"/map_l1b_manifest_esa{esa}.csv", - index=False - ) + manifest_df.to_csv(map_dir + f"/map_l1b_manifest_esa{esa}.csv", index=False) diff --git a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/new_patch_arc2.py b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/new_patch_arc2.py index 917aca684..778de8fd4 100755 --- a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/new_patch_arc2.py +++ b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/new_patch_arc2.py @@ -7,10 +7,11 @@ PI = np.pi -def somewhat_great_circle_centered(rot, lat0, lon0, angle, npts=400): + +def somewhat_great_circle_centered(rot, lat0, lon0, angle, npts=400): """ Somewhat great circle whose pole is (lat0, lon0). - angle is the angle from the center + angle is the angle from the center Returns coordinates in the ROTATED frame. """ @@ -28,25 +29,22 @@ def somewhat_great_circle_centered(rot, lat0, lon0, angle, npts=400): v = np.cross(p, u) - t = np.linspace(0, 2*np.pi, npts) + t = np.linspace(0, 2 * np.pi, npts) angle_r = np.radians(angle) - pts = ( - np.cos(angle_r) * p - + np.sin(angle_r) * ( - np.outer(np.cos(t), u) + - np.outer(np.sin(t), v) + pts = np.cos(angle_r) * p + np.sin(angle_r) * ( + np.outer(np.cos(t), u) + np.outer(np.sin(t), v) ) -) # rotate into map frame pts_rot = rot.apply(pts) - lat = np.degrees(np.arcsin(pts_rot[:,2])) - lon = np.degrees(np.arctan2(pts_rot[:,1], pts_rot[:,0])) + lat = np.degrees(np.arcsin(pts_rot[:, 2])) + lon = np.degrees(np.arctan2(pts_rot[:, 1], pts_rot[:, 0])) lon = (lon + 180) % 360 - 180 return lat, lon + def plot_wrapped_line(ax, lon, lat, **plot_kwargs): """ Plot a line while avoiding wraparound artifacts. @@ -56,14 +54,16 @@ def plot_wrapped_line(ax, lon, lat, **plot_kwargs): lat = np.asarray(lat) jumps = np.where(np.abs(np.diff(lon)) > 180)[0] - split_idx = np.append(jumps, len(lon)-1) + split_idx = np.append(jumps, len(lon) - 1) start = 0 for idx in split_idx: - ax.plot(lon[start:idx+1], - lat[start:idx+1], - transform=ccrs.PlateCarree(), - **plot_kwargs) + ax.plot( + lon[start : idx + 1], + lat[start : idx + 1], + transform=ccrs.PlateCarree(), + **plot_kwargs, + ) start = idx + 1 @@ -75,14 +75,16 @@ def sph_to_cart(lat, lon): z = np.sin(lat_rad) return np.array([x, y, z]) + def rotation_matrix_to_center(new_colat0, new_lon0): # Step 1: rotate around Z axis by -lon0 - r1 = R.from_euler('z', -new_lon0, degrees=True) + r1 = R.from_euler("z", -new_lon0, degrees=True) # Step 2: rotate around Y axis by 90-lat0 - r2 = R.from_euler('y', 90-new_colat0, degrees=True) + r2 = R.from_euler("y", 90 - new_colat0, degrees=True) # Combined rotation return r2 * r1 + def inverse_rotation_matrix_to_center(rot, xyz_rot): """ Apply inverse rotation to xyz_rot points. @@ -92,55 +94,110 @@ def inverse_rotation_matrix_to_center(rot, xyz_rot): """ return rot.inv().apply(xyz_rot) + def make_imap_lo_reference_points(rot, option=1): - if option==1: + if option == 1: npoints = 4 lons = np.array([259.200622, 255.499300, 289.800000, 79.200622]) - lats = np.array([5.116296, 34.999941, -35.600000, -5.116296]) - labels=[ 'Nose', 'V1','V2','Tail'] - if option==2: + lats = np.array([5.116296, 34.999941, -35.600000, -5.116296]) + labels = ["Nose", "V1", "V2", "Tail"] + if option == 2: npoints = 7 - lons = np.array([259.200622, 255.499300, 289.800000, 79.200622, 221,150.,0.0 ]) - lats = np.array([5.116296, 34.999941, -35.600000, -5.116296, 39, 0.0, 0.0]) - labels=[ 'Nose', 'V1','V2','Tail', 'Rcen','Stbd', 'Port'] - if option==3: + lons = np.array( + [259.200622, 255.499300, 289.800000, 79.200622, 221, 150.0, 0.0] + ) + lats = np.array([5.116296, 34.999941, -35.600000, -5.116296, 39, 0.0, 0.0]) + labels = ["Nose", "V1", "V2", "Tail", "Rcen", "Stbd", "Port"] + if option == 3: npoints = 10 - lons = np.array([259.200622, 255.499300, 289.800000, 79.200622, 221,150.,0.0, 41, 250.624523, 70.624523]) - lats = np.array([5.116296, 34.999941, -35.600000, -5.116296, 39, 0.0, 0.0, -39, -41.242979, 41.242979]) - labels=[ 'Nose', 'V1','V2','Tail', 'RibC','Stbd', 'Port','DwnB','ULSR', 'DLSR'] - + lons = np.array( + [ + 259.200622, + 255.499300, + 289.800000, + 79.200622, + 221, + 150.0, + 0.0, + 41, + 250.624523, + 70.624523, + ] + ) + lats = np.array( + [ + 5.116296, + 34.999941, + -35.600000, + -5.116296, + 39, + 0.0, + 0.0, + -39, + -41.242979, + 41.242979, + ] + ) + labels = [ + "Nose", + "V1", + "V2", + "Tail", + "RibC", + "Stbd", + "Port", + "DwnB", + "ULSR", + "DLSR", + ] # --- Reference point in ORIGINAL coordinates --- - lat_points = np.linspace(0.0,0.0,npoints) - lon_points = np.linspace(0.0,0.0,npoints) - for i in range(0,npoints): - ref_lat = lats[i] # degrees - ref_lon = lons[i] # degrees - - # Convert to xyz - xyz_ref = np.array([ - np.cos(np.radians(ref_lat)) * np.cos(np.radians(ref_lon)), - np.cos(np.radians(ref_lat)) * np.sin(np.radians(ref_lon)), - np.sin(np.radians(ref_lat)) - ]) + lat_points = np.linspace(0.0, 0.0, npoints) + lon_points = np.linspace(0.0, 0.0, npoints) + for i in range(0, npoints): + ref_lat = lats[i] # degrees + ref_lon = lons[i] # degrees + + # Convert to xyz + xyz_ref = np.array( + [ + np.cos(np.radians(ref_lat)) * np.cos(np.radians(ref_lon)), + np.cos(np.radians(ref_lat)) * np.sin(np.radians(ref_lon)), + np.sin(np.radians(ref_lat)), + ] + ) # Rotate to map frame xyz_ref_rot = rot.apply(xyz_ref) # Back to lat/lon in rotated frame lat_ref_rot = np.degrees(np.arcsin(xyz_ref_rot[2])) - lon_ref_rot = -np.degrees(np.arctan2(xyz_ref_rot[1], xyz_ref_rot[0])) # negative sign since we are looking from the inside - lon_ref_rot = (lon_ref_rot + 180) % 360 - 180 # wrap + lon_ref_rot = -np.degrees( + np.arctan2(xyz_ref_rot[1], xyz_ref_rot[0]) + ) # negative sign since we are looking from the inside + lon_ref_rot = (lon_ref_rot + 180) % 360 - 180 # wrap lat_points[i] = lat_ref_rot lon_points[i] = lon_ref_rot - + return labels, lon_points, lat_points # --- Load data --- -def make_imap_lo_map(file, lat_center, lon_center, vmin, vmax, output, label_colorbar='Flux', plot_title='PPM = 90', opt_points=1, opt_ribbon=0): + +def make_imap_lo_map( + file, + lat_center, + lon_center, + vmin, + vmax, + output, + label_colorbar="Flux", + plot_title="PPM = 90", + opt_points=1, + opt_ribbon=0, +): data = pd.read_csv(file, header=0).values ny, nx = data.shape @@ -150,93 +207,110 @@ def make_imap_lo_map(file, lat_center, lon_center, vmin, vmax, output, label_col lons = np.linspace(3, 357, nx) # Build corner grid for pcolormesh - lat_edges = np.linspace(-90, 90, ny+1) - lon_edges = np.linspace(0, 360, nx+1) + lat_edges = np.linspace(-90, 90, ny + 1) + lon_edges = np.linspace(0, 360, nx + 1) lon_grid, lat_grid = np.meshgrid(lon_edges, lat_edges) # Flatten grid for rotation - xyz = np.column_stack([ - np.cos(np.radians(lat_grid.ravel())) * np.cos(np.radians(lon_grid.ravel())), - np.cos(np.radians(lat_grid.ravel())) * np.sin(np.radians(lon_grid.ravel())), - np.sin(np.radians(lat_grid.ravel())) - ]) + xyz = np.column_stack( + [ + np.cos(np.radians(lat_grid.ravel())) * np.cos(np.radians(lon_grid.ravel())), + np.cos(np.radians(lat_grid.ravel())) * np.sin(np.radians(lon_grid.ravel())), + np.sin(np.radians(lat_grid.ravel())), + ] + ) # --- Rotation --- new_colat0, new_lon0 = 90.0 - lat_center, lon_center - rot = R.from_euler('y', 90-new_colat0, degrees=True) * R.from_euler('z', -new_lon0, degrees=True) + rot = R.from_euler("y", 90 - new_colat0, degrees=True) * R.from_euler( + "z", -new_lon0, degrees=True + ) xyz_rot = rot.apply(xyz) - lat_rot = np.degrees(np.arcsin(xyz_rot[:,2])).reshape(ny+1, nx+1) - lon_rot = np.degrees(np.arctan2(xyz_rot[:,1], xyz_rot[:,0])).reshape(ny+1, nx+1) + lat_rot = np.degrees(np.arcsin(xyz_rot[:, 2])).reshape(ny + 1, nx + 1) + lon_rot = np.degrees(np.arctan2(xyz_rot[:, 1], xyz_rot[:, 0])).reshape( + ny + 1, nx + 1 + ) lon_rot = (lon_rot + 180) % 360 - 180 # For the data lon_rot_inside = -lon_rot # flip longitude - #lon_edges = compute_edges(lon_rot) - #lat_edges = compute_edges(lat_rot) + # lon_edges = compute_edges(lon_rot) + # lat_edges = compute_edges(lat_rot) # --- Plot using pcolormesh --- - fig, ax = plt.subplots(figsize=(10,5), subplot_kw={'projection': ccrs.Mollweide()}) - ax.set_facecolor('darkblue') + fig, ax = plt.subplots(figsize=(10, 5), subplot_kw={"projection": ccrs.Mollweide()}) + ax.set_facecolor("darkblue") - vmin= vmin + vmin = vmin vmax = vmax data_clipped = np.clip(data, vmin, vmax) data_filled = np.copy(data_clipped) data_filled[np.isnan(data_clipped)] = vmin - pcm = ax.pcolormesh(lon_rot_inside, lat_rot, data_filled, # no interpolation! - transform=ccrs.PlateCarree(), - cmap='viridis', norm=LogNorm(vmin=vmin, vmax=vmax), shading='auto') - + pcm = ax.pcolormesh( + lon_rot_inside, + lat_rot, + data_filled, # no interpolation! + transform=ccrs.PlateCarree(), + cmap="viridis", + norm=LogNorm(vmin=vmin, vmax=vmax), + shading="auto", + ) lat_lines = np.linspace(-90, 90, 7) # every ~30 deg for lat0 in lat_lines: - lon_line = np.linspace(0, 360, nx+1) + lon_line = np.linspace(0, 360, nx + 1) lat_line = np.full_like(lon_line, lat0) # Convert to xyz and rotate - xyz_line = np.column_stack([ - np.cos(np.radians(lat_line)) * np.cos(np.radians(lon_line)), - np.cos(np.radians(lat_line)) * np.sin(np.radians(lon_line)), - np.sin(np.radians(lat_line)) - ]) + xyz_line = np.column_stack( + [ + np.cos(np.radians(lat_line)) * np.cos(np.radians(lon_line)), + np.cos(np.radians(lat_line)) * np.sin(np.radians(lon_line)), + np.sin(np.radians(lat_line)), + ] + ) xyz_line_rot = rot.apply(xyz_line) # Back to lat/lon - lat_rot_line = np.degrees(np.arcsin(xyz_line_rot[:,2])) - lon_rot_line = -np.degrees(np.arctan2(xyz_line_rot[:,1], xyz_line_rot[:,0])) + lat_rot_line = np.degrees(np.arcsin(xyz_line_rot[:, 2])) + lon_rot_line = -np.degrees(np.arctan2(xyz_line_rot[:, 1], xyz_line_rot[:, 0])) lon_rot_line = (lon_rot_line + 180) % 360 - 180 # wrap to [-180,180] # flip for inside view # --- Split line wherever jump > 180° jump_idx = np.where(np.abs(np.diff(lon_rot_line)) > 180)[0] - split_idx = np.append(jump_idx, len(lon_rot_line)-1) + split_idx = np.append(jump_idx, len(lon_rot_line) - 1) start = 0 for idx in split_idx: - seg_lon = lon_rot_line[start:idx+1] - seg_lat = lat_rot_line[start:idx+1] - ax.plot(seg_lon, seg_lat, transform=ccrs.PlateCarree(), - color='orange', lw=0.8, alpha=0.7) - start = idx+1 - - - # pick a tolerance in degrees - tol = 100.0 # widen if labels disappear + seg_lon = lon_rot_line[start : idx + 1] + seg_lat = lat_rot_line[start : idx + 1] + ax.plot( + seg_lon, + seg_lat, + transform=ccrs.PlateCarree(), + color="orange", + lw=0.8, + alpha=0.7, + ) + start = idx + 1 - lon_here = ((seg_lon[0] + 180) % 360) - 180 # ensure [-180,180] + # pick a tolerance in degrees + tol = 100.0 # widen if labels disappear - if abs(lon_here) < tol: # only near 0° rotated longitude + lon_here = ((seg_lon[0] + 180) % 360) - 180 # ensure [-180,180] + if abs(lon_here) < tol: # only near 0° rotated longitude theta = np.radians(90.0 - seg_lat[0]) - phi = -np.radians(seg_lon[0]) # inside view sign flip - phi = np.mod(phi, 2.0*np.pi) + phi = -np.radians(seg_lon[0]) # inside view sign flip + phi = np.mod(phi, 2.0 * np.pi) xyz = [ - np.sin(theta)*np.cos(phi), - np.sin(theta)*np.sin(phi), - np.cos(theta) + np.sin(theta) * np.cos(phi), + np.sin(theta) * np.sin(phi), + np.cos(theta), ] xyz_inv = rot.inv().apply(xyz) @@ -249,83 +323,92 @@ def make_imap_lo_map(file, lat_center, lon_center, vmin, vmax, output, label_col seg_lat[0], f"{int(round(lat_orig))}°", transform=ccrs.PlateCarree(), - color='orange', + color="orange", fontsize=6, - verticalalignment='top', - horizontalalignment='right' + verticalalignment="top", + horizontalalignment="right", ) - #theta = np.radians(90.0-seg_lat[0]) - #phi = -np.radians(seg_lon[0]) # sign inversion looking from inside - #phi = np.mod(phi, 2.0*np.pi) # wrap to [0, 2π) - #if (phi > PI): + # theta = np.radians(90.0-seg_lat[0]) + # phi = -np.radians(seg_lon[0]) # sign inversion looking from inside + # phi = np.mod(phi, 2.0*np.pi) # wrap to [0, 2π) + # if (phi > PI): # xyz = [np.sin(theta)*np.cos(phi),np.sin(theta)*np.sin(phi), np.cos(theta) ] # xyz_inv = rot.inv().apply(xyz) # lat_orig = np.degrees(np.arcsin(xyz_inv[2])) # lon_orig = np.degrees(np.arctan2(xyz_inv[1], xyz_inv[0])) # if not np.isnan(lat_orig): # lat_lab_1 = round(lat_orig) - # else: + # else: # lat_lab_1 = '' # lon_lab = ((seg_lon[0]+180) % 360) - 180 # if lat_lab_1 != '': # ## lon_lab_inside = -lon_lab - ## transform=ccrs.PlateCarree(), - # color='orange', fontsize=6, - # verticalalignment='center', horizontalalignment='right') - + ## transform=ccrs.PlateCarree(), + # color='orange', fontsize=6, + # verticalalignment='center', horizontalalignment='right') lon_lines = np.linspace(0, 360, 13) # every 30 deg for lon0 in lon_lines: - lat_line = np.linspace(-90, 90, ny+1) + lat_line = np.linspace(-90, 90, ny + 1) lon_line = np.full_like(lat_line, lon0) - xyz_line = np.column_stack([ - np.cos(np.radians(lat_line)) * np.cos(np.radians(lon_line)), - np.cos(np.radians(lat_line)) * np.sin(np.radians(lon_line)), - np.sin(np.radians(lat_line)) - ]) + xyz_line = np.column_stack( + [ + np.cos(np.radians(lat_line)) * np.cos(np.radians(lon_line)), + np.cos(np.radians(lat_line)) * np.sin(np.radians(lon_line)), + np.sin(np.radians(lat_line)), + ] + ) xyz_line_rot = rot.apply(xyz_line) - lat_rot_line = np.degrees(np.arcsin(xyz_line_rot[:,2])) - lon_rot_line = -np.degrees(np.arctan2(xyz_line_rot[:,1], xyz_line_rot[:,0])) + lat_rot_line = np.degrees(np.arcsin(xyz_line_rot[:, 2])) + lon_rot_line = -np.degrees(np.arctan2(xyz_line_rot[:, 1], xyz_line_rot[:, 0])) lon_rot_line = (lon_rot_line + 180) % 360 - 180 # flip for inside view jump_idx = np.where(np.abs(np.diff(lon_rot_line)) > 180)[0] - split_idx = np.append(jump_idx, len(lon_rot_line)-1) + split_idx = np.append(jump_idx, len(lon_rot_line) - 1) start = 0 # Find the row closest to 0° rotated latitude - lat0_idx = np.argmin(np.abs(lat_rot_line)) # lat_rot_line = your rotated lat array for the line + lat0_idx = np.argmin( + np.abs(lat_rot_line) + ) # lat_rot_line = your rotated lat array for the line # Get the corresponding longitude and latitude along that row lon_line = lon_rot_line[lat0_idx] lat_line = lat_rot_line[lat0_idx] for idx in split_idx: - seg_lon = lon_rot_line[start:idx+1] - seg_lat = lat_rot_line[start:idx+1] - ax.plot(seg_lon, seg_lat, transform=ccrs.PlateCarree(), - color='orange', lw=0.8, alpha=0.7) - - mid_idx = len(seg_lon) // 2 + seg_lon = lon_rot_line[start : idx + 1] + seg_lat = lat_rot_line[start : idx + 1] + ax.plot( + seg_lon, + seg_lat, + transform=ccrs.PlateCarree(), + color="orange", + lw=0.8, + alpha=0.7, + ) + + mid_idx = len(seg_lon) // 2 lon_plot = seg_lon[mid_idx] lat_plot = seg_lat[mid_idx] - start = idx+1 + start = idx + 1 - xyz_rot = sph_to_cart(lat_plot, lon_plot) # returns (3,) or (1,3) + xyz_rot = sph_to_cart(lat_plot, lon_plot) # returns (3,) or (1,3) # print(xyz_rot) # ensure shape (1,3) for scipy Rotation xyz_rot = np.atleast_2d(xyz_rot) # print(xyz_rot) - # --- invert rotation --- + # --- invert rotation --- xyz_orig = rot.inv().apply(xyz_rot) - # --- back to spherical --- + # --- back to spherical --- lat_label = np.degrees(np.arcsin(xyz_orig[0, 2])) lon_label = np.degrees(np.arctan2(xyz_orig[0, 1], xyz_orig[0, 0])) @@ -333,34 +416,38 @@ def make_imap_lo_map(file, lat_center, lon_center, vmin, vmax, output, label_col xyz_rot_label = rot.apply(xyz) lat_plot = np.degrees(np.arcsin(xyz_rot_label[2])) - # print('lon_label = ',lon_label) - # lon_label = (-1.0*lon_label + 180) % 360 - 180 - # print('lon_label (2) = ',lon_label) + # print('lon_label = ',lon_label) + # lon_label = (-1.0*lon_label + 180) % 360 - 180 + # print('lon_label (2) = ',lon_label) lon_plot = -lon_plot if np.abs(lat_label) < 30.0: - ax.text(lon_plot, lat_plot, - f"{lon_label:.0f}°", - transform=ccrs.PlateCarree(), - color='orange', fontsize=6, - verticalalignment='bottom', horizontalalignment='right') - - if (opt_points > 0): - + ax.text( + lon_plot, + lat_plot, + f"{lon_label:.0f}°", + transform=ccrs.PlateCarree(), + color="orange", + fontsize=6, + verticalalignment="bottom", + horizontalalignment="right", + ) + + if opt_points > 0: labels, lon_points, lat_points = make_imap_lo_reference_points(rot, opt_points) npoints = len(labels) - for i in range(0,npoints): + for i in range(0, npoints): lon_ref_rot = lon_points[i] lat_ref_rot = lat_points[i] ref_label = labels[i] - # --- Plot marker --- + # --- Plot marker --- ax.plot( lon_ref_rot, lat_ref_rot, - marker='o', + marker="o", markersize=1, - color='white', + color="white", transform=ccrs.PlateCarree(), - zorder=5 + zorder=5, ) # --- Add text --- @@ -369,57 +456,60 @@ def make_imap_lo_map(file, lat_center, lon_center, vmin, vmax, output, label_col lat_ref_rot, f" {ref_label}", transform=ccrs.PlateCarree(), - color='white', + color="white", fontsize=6, - ha='left', - va='center', - zorder=5 + ha="left", + va="center", + zorder=5, ) -# now optionally add the great circle line + # now optionally add the great circle line - if (opt_ribbon > 0): - - lat_line, lon_line = somewhat_great_circle_centered( rot, - lat0=39, - lon0=221.0, - angle=70-15 + if opt_ribbon > 0: + lat_line, lon_line = somewhat_great_circle_centered( + rot, lat0=39, lon0=221.0, angle=70 - 15 ) plot_wrapped_line( - ax, - -lon_line, # inside view flip if you use it - lat_line, - color='white', - lw=0.5 + ax, + -lon_line, # inside view flip if you use it + lat_line, + color="white", + lw=0.5, ) - - lat_line, lon_line = somewhat_great_circle_centered( rot, - lat0=39, - lon0=221.0, - angle=70+15 + lat_line, lon_line = somewhat_great_circle_centered( + rot, lat0=39, lon0=221.0, angle=70 + 15 ) plot_wrapped_line( - ax, - -lon_line, # inside view flip if you use it - lat_line, - color='white', - lw=0.5 + ax, + -lon_line, # inside view flip if you use it + lat_line, + color="white", + lw=0.5, ) -# label = label_map[dtype] + # label = label_map[dtype] # "Intensity (counts cm$^{-2}$ s$^{-1}$ sr$^{-1}$ keV$^{-1}$)") -# value = esa_energy[esa] * 1000.0 -# s = f"{value:.2f}" -# cbar.set_label(label) -# cbar.set_label(f"{label} [ECLIPJ2000] at ESA (eV) = {s}") + # value = esa_energy[esa] * 1000.0 + # s = f"{value:.2f}" + # cbar.set_label(label) + # cbar.set_label(f"{label} [ECLIPJ2000] at ESA (eV) = {s}") plt.suptitle(plot_title) - plt.colorbar(pcm, ax=ax, orientation='horizontal', pad=0.05, label=label_colorbar) + plt.colorbar(pcm, ax=ax, orientation="horizontal", pad=0.05, label=label_colorbar) ax.set_global() - # plt.show() - plt.savefig(output, dpi=200, facecolor='w', edgecolor='w', orientation='portrait', format=None, transparent=False, bbox_inches=None, pad_inches=0.1) + # plt.show() + plt.savefig( + output, + dpi=200, + facecolor="w", + edgecolor="w", + orientation="portrait", + format=None, + transparent=False, + bbox_inches=None, + pad_inches=0.1, + ) plt.close(fig) - diff --git a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/new_patch_arc2_linear.py b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/new_patch_arc2_linear.py index 24a93d562..c27edae7e 100755 --- a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/new_patch_arc2_linear.py +++ b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/new_patch_arc2_linear.py @@ -6,10 +6,11 @@ PI = np.pi -def somewhat_great_circle_centered(rot, lat0, lon0, angle, npts=400): + +def somewhat_great_circle_centered(rot, lat0, lon0, angle, npts=400): """ Somewhat great circle whose pole is (lat0, lon0). - angle is the angle from the center + angle is the angle from the center Returns coordinates in the ROTATED frame. """ @@ -27,25 +28,22 @@ def somewhat_great_circle_centered(rot, lat0, lon0, angle, npts=400): v = np.cross(p, u) - t = np.linspace(0, 2*np.pi, npts) + t = np.linspace(0, 2 * np.pi, npts) angle_r = np.radians(angle) - pts = ( - np.cos(angle_r) * p - + np.sin(angle_r) * ( - np.outer(np.cos(t), u) + - np.outer(np.sin(t), v) + pts = np.cos(angle_r) * p + np.sin(angle_r) * ( + np.outer(np.cos(t), u) + np.outer(np.sin(t), v) ) -) # rotate into map frame pts_rot = rot.apply(pts) - lat = np.degrees(np.arcsin(pts_rot[:,2])) - lon = np.degrees(np.arctan2(pts_rot[:,1], pts_rot[:,0])) + lat = np.degrees(np.arcsin(pts_rot[:, 2])) + lon = np.degrees(np.arctan2(pts_rot[:, 1], pts_rot[:, 0])) lon = (lon + 180) % 360 - 180 return lat, lon + def plot_wrapped_line(ax, lon, lat, **plot_kwargs): """ Plot a line while avoiding wraparound artifacts. @@ -55,14 +53,16 @@ def plot_wrapped_line(ax, lon, lat, **plot_kwargs): lat = np.asarray(lat) jumps = np.where(np.abs(np.diff(lon)) > 180)[0] - split_idx = np.append(jumps, len(lon)-1) + split_idx = np.append(jumps, len(lon) - 1) start = 0 for idx in split_idx: - ax.plot(lon[start:idx+1], - lat[start:idx+1], - transform=ccrs.PlateCarree(), - **plot_kwargs) + ax.plot( + lon[start : idx + 1], + lat[start : idx + 1], + transform=ccrs.PlateCarree(), + **plot_kwargs, + ) start = idx + 1 @@ -74,14 +74,16 @@ def sph_to_cart(lat, lon): z = np.sin(lat_rad) return np.array([x, y, z]) + def rotation_matrix_to_center(new_colat0, new_lon0): # Step 1: rotate around Z axis by -lon0 - r1 = R.from_euler('z', -new_lon0, degrees=True) + r1 = R.from_euler("z", -new_lon0, degrees=True) # Step 2: rotate around Y axis by 90-lat0 - r2 = R.from_euler('y', 90-new_colat0, degrees=True) + r2 = R.from_euler("y", 90 - new_colat0, degrees=True) # Combined rotation return r2 * r1 + def inverse_rotation_matrix_to_center(rot, xyz_rot): """ Apply inverse rotation to xyz_rot points. @@ -91,55 +93,110 @@ def inverse_rotation_matrix_to_center(rot, xyz_rot): """ return rot.inv().apply(xyz_rot) + def make_imap_lo_reference_points(rot, option=1): - if option==1: + if option == 1: npoints = 4 lons = np.array([259.200622, 255.499300, 289.800000, 79.200622]) - lats = np.array([5.116296, 34.999941, -35.600000, -5.116296]) - labels=[ 'Nose', 'V1','V2','Tail'] - if option==2: + lats = np.array([5.116296, 34.999941, -35.600000, -5.116296]) + labels = ["Nose", "V1", "V2", "Tail"] + if option == 2: npoints = 7 - lons = np.array([259.200622, 255.499300, 289.800000, 79.200622, 221,150.,0.0 ]) - lats = np.array([5.116296, 34.999941, -35.600000, -5.116296, 39, 0.0, 0.0]) - labels=[ 'Nose', 'V1','V2','Tail', 'Rcen','Stbd', 'Port'] - if option==3: + lons = np.array( + [259.200622, 255.499300, 289.800000, 79.200622, 221, 150.0, 0.0] + ) + lats = np.array([5.116296, 34.999941, -35.600000, -5.116296, 39, 0.0, 0.0]) + labels = ["Nose", "V1", "V2", "Tail", "Rcen", "Stbd", "Port"] + if option == 3: npoints = 10 - lons = np.array([259.200622, 255.499300, 289.800000, 79.200622, 221,150.,0.0, 41, 250.624523, 70.624523]) - lats = np.array([5.116296, 34.999941, -35.600000, -5.116296, 39, 0.0, 0.0, -39, -41.242979, 41.242979]) - labels=[ 'Nose', 'V1','V2','Tail', 'RibC','Stbd', 'Port','DwnB','ULSR', 'DLSR'] - + lons = np.array( + [ + 259.200622, + 255.499300, + 289.800000, + 79.200622, + 221, + 150.0, + 0.0, + 41, + 250.624523, + 70.624523, + ] + ) + lats = np.array( + [ + 5.116296, + 34.999941, + -35.600000, + -5.116296, + 39, + 0.0, + 0.0, + -39, + -41.242979, + 41.242979, + ] + ) + labels = [ + "Nose", + "V1", + "V2", + "Tail", + "RibC", + "Stbd", + "Port", + "DwnB", + "ULSR", + "DLSR", + ] # --- Reference point in ORIGINAL coordinates --- - lat_points = np.linspace(0.0,0.0,npoints) - lon_points = np.linspace(0.0,0.0,npoints) - for i in range(0,npoints): - ref_lat = lats[i] # degrees - ref_lon = lons[i] # degrees - - # Convert to xyz - xyz_ref = np.array([ - np.cos(np.radians(ref_lat)) * np.cos(np.radians(ref_lon)), - np.cos(np.radians(ref_lat)) * np.sin(np.radians(ref_lon)), - np.sin(np.radians(ref_lat)) - ]) + lat_points = np.linspace(0.0, 0.0, npoints) + lon_points = np.linspace(0.0, 0.0, npoints) + for i in range(0, npoints): + ref_lat = lats[i] # degrees + ref_lon = lons[i] # degrees + + # Convert to xyz + xyz_ref = np.array( + [ + np.cos(np.radians(ref_lat)) * np.cos(np.radians(ref_lon)), + np.cos(np.radians(ref_lat)) * np.sin(np.radians(ref_lon)), + np.sin(np.radians(ref_lat)), + ] + ) # Rotate to map frame xyz_ref_rot = rot.apply(xyz_ref) # Back to lat/lon in rotated frame lat_ref_rot = np.degrees(np.arcsin(xyz_ref_rot[2])) - lon_ref_rot = -np.degrees(np.arctan2(xyz_ref_rot[1], xyz_ref_rot[0])) # negative sign since we are looking from the inside - lon_ref_rot = (lon_ref_rot + 180) % 360 - 180 # wrap + lon_ref_rot = -np.degrees( + np.arctan2(xyz_ref_rot[1], xyz_ref_rot[0]) + ) # negative sign since we are looking from the inside + lon_ref_rot = (lon_ref_rot + 180) % 360 - 180 # wrap lat_points[i] = lat_ref_rot lon_points[i] = lon_ref_rot - + return labels, lon_points, lat_points # --- Load data --- -def make_imap_lo_map(file, lat_center, lon_center, vmin, vmax, output, label_colorbar='Flux', plot_title='PPM = 90', opt_points=1, opt_ribbon=0): + +def make_imap_lo_map( + file, + lat_center, + lon_center, + vmin, + vmax, + output, + label_colorbar="Flux", + plot_title="PPM = 90", + opt_points=1, + opt_ribbon=0, +): data = pd.read_csv(file, header=0).values ny, nx = data.shape @@ -149,37 +206,43 @@ def make_imap_lo_map(file, lat_center, lon_center, vmin, vmax, output, label_col lons = np.linspace(3, 357, nx) # Build corner grid for pcolormesh - lat_edges = np.linspace(-90, 90, ny+1) - lon_edges = np.linspace(0, 360, nx+1) + lat_edges = np.linspace(-90, 90, ny + 1) + lon_edges = np.linspace(0, 360, nx + 1) lon_grid, lat_grid = np.meshgrid(lon_edges, lat_edges) # Flatten grid for rotation - xyz = np.column_stack([ - np.cos(np.radians(lat_grid.ravel())) * np.cos(np.radians(lon_grid.ravel())), - np.cos(np.radians(lat_grid.ravel())) * np.sin(np.radians(lon_grid.ravel())), - np.sin(np.radians(lat_grid.ravel())) - ]) + xyz = np.column_stack( + [ + np.cos(np.radians(lat_grid.ravel())) * np.cos(np.radians(lon_grid.ravel())), + np.cos(np.radians(lat_grid.ravel())) * np.sin(np.radians(lon_grid.ravel())), + np.sin(np.radians(lat_grid.ravel())), + ] + ) # --- Rotation --- new_colat0, new_lon0 = 90.0 - lat_center, lon_center - rot = R.from_euler('y', 90-new_colat0, degrees=True) * R.from_euler('z', -new_lon0, degrees=True) + rot = R.from_euler("y", 90 - new_colat0, degrees=True) * R.from_euler( + "z", -new_lon0, degrees=True + ) xyz_rot = rot.apply(xyz) - lat_rot = np.degrees(np.arcsin(xyz_rot[:,2])).reshape(ny+1, nx+1) - lon_rot = np.degrees(np.arctan2(xyz_rot[:,1], xyz_rot[:,0])).reshape(ny+1, nx+1) + lat_rot = np.degrees(np.arcsin(xyz_rot[:, 2])).reshape(ny + 1, nx + 1) + lon_rot = np.degrees(np.arctan2(xyz_rot[:, 1], xyz_rot[:, 0])).reshape( + ny + 1, nx + 1 + ) lon_rot = (lon_rot + 180) % 360 - 180 # For the data lon_rot_inside = -lon_rot # flip longitude - #lon_edges = compute_edges(lon_rot) - #lat_edges = compute_edges(lat_rot) + # lon_edges = compute_edges(lon_rot) + # lat_edges = compute_edges(lat_rot) # --- Plot using pcolormesh --- - fig, ax = plt.subplots(figsize=(10,5), subplot_kw={'projection': ccrs.Mollweide()}) - ax.set_facecolor('darkblue') + fig, ax = plt.subplots(figsize=(10, 5), subplot_kw={"projection": ccrs.Mollweide()}) + ax.set_facecolor("darkblue") - vmin= vmin + vmin = vmin vmax = vmax data_clipped = np.clip(data, vmin, vmax) data_filled = np.copy(data_clipped) @@ -190,58 +253,64 @@ def make_imap_lo_map(file, lat_center, lon_center, vmin, vmax, output, label_col lat_rot, data_filled, transform=ccrs.PlateCarree(), - cmap='viridis', + cmap="viridis", vmin=vmin, vmax=vmax, - shading='auto' + shading="auto", ) lat_lines = np.linspace(-90, 90, 7) # every ~30 deg for lat0 in lat_lines: - lon_line = np.linspace(0, 360, nx+1) + lon_line = np.linspace(0, 360, nx + 1) lat_line = np.full_like(lon_line, lat0) # Convert to xyz and rotate - xyz_line = np.column_stack([ - np.cos(np.radians(lat_line)) * np.cos(np.radians(lon_line)), - np.cos(np.radians(lat_line)) * np.sin(np.radians(lon_line)), - np.sin(np.radians(lat_line)) - ]) + xyz_line = np.column_stack( + [ + np.cos(np.radians(lat_line)) * np.cos(np.radians(lon_line)), + np.cos(np.radians(lat_line)) * np.sin(np.radians(lon_line)), + np.sin(np.radians(lat_line)), + ] + ) xyz_line_rot = rot.apply(xyz_line) # Back to lat/lon - lat_rot_line = np.degrees(np.arcsin(xyz_line_rot[:,2])) - lon_rot_line = -np.degrees(np.arctan2(xyz_line_rot[:,1], xyz_line_rot[:,0])) + lat_rot_line = np.degrees(np.arcsin(xyz_line_rot[:, 2])) + lon_rot_line = -np.degrees(np.arctan2(xyz_line_rot[:, 1], xyz_line_rot[:, 0])) lon_rot_line = (lon_rot_line + 180) % 360 - 180 # wrap to [-180,180] # flip for inside view # --- Split line wherever jump > 180° jump_idx = np.where(np.abs(np.diff(lon_rot_line)) > 180)[0] - split_idx = np.append(jump_idx, len(lon_rot_line)-1) + split_idx = np.append(jump_idx, len(lon_rot_line) - 1) start = 0 for idx in split_idx: - seg_lon = lon_rot_line[start:idx+1] - seg_lat = lat_rot_line[start:idx+1] - ax.plot(seg_lon, seg_lat, transform=ccrs.PlateCarree(), - color='orange', lw=0.8, alpha=0.7) - start = idx+1 - - - # pick a tolerance in degrees - tol = 100.0 # widen if labels disappear + seg_lon = lon_rot_line[start : idx + 1] + seg_lat = lat_rot_line[start : idx + 1] + ax.plot( + seg_lon, + seg_lat, + transform=ccrs.PlateCarree(), + color="orange", + lw=0.8, + alpha=0.7, + ) + start = idx + 1 - lon_here = ((seg_lon[0] + 180) % 360) - 180 # ensure [-180,180] + # pick a tolerance in degrees + tol = 100.0 # widen if labels disappear - if abs(lon_here) < tol: # only near 0° rotated longitude + lon_here = ((seg_lon[0] + 180) % 360) - 180 # ensure [-180,180] + if abs(lon_here) < tol: # only near 0° rotated longitude theta = np.radians(90.0 - seg_lat[0]) - phi = -np.radians(seg_lon[0]) # inside view sign flip - phi = np.mod(phi, 2.0*np.pi) + phi = -np.radians(seg_lon[0]) # inside view sign flip + phi = np.mod(phi, 2.0 * np.pi) xyz = [ - np.sin(theta)*np.cos(phi), - np.sin(theta)*np.sin(phi), - np.cos(theta) + np.sin(theta) * np.cos(phi), + np.sin(theta) * np.sin(phi), + np.cos(theta), ] xyz_inv = rot.inv().apply(xyz) @@ -254,83 +323,92 @@ def make_imap_lo_map(file, lat_center, lon_center, vmin, vmax, output, label_col seg_lat[0], f"{int(round(lat_orig))}°", transform=ccrs.PlateCarree(), - color='orange', + color="orange", fontsize=6, - verticalalignment='top', - horizontalalignment='right' + verticalalignment="top", + horizontalalignment="right", ) - #theta = np.radians(90.0-seg_lat[0]) - #phi = -np.radians(seg_lon[0]) # sign inversion looking from inside - #phi = np.mod(phi, 2.0*np.pi) # wrap to [0, 2π) - #if (phi > PI): + # theta = np.radians(90.0-seg_lat[0]) + # phi = -np.radians(seg_lon[0]) # sign inversion looking from inside + # phi = np.mod(phi, 2.0*np.pi) # wrap to [0, 2π) + # if (phi > PI): # xyz = [np.sin(theta)*np.cos(phi),np.sin(theta)*np.sin(phi), np.cos(theta) ] # xyz_inv = rot.inv().apply(xyz) # lat_orig = np.degrees(np.arcsin(xyz_inv[2])) # lon_orig = np.degrees(np.arctan2(xyz_inv[1], xyz_inv[0])) # if not np.isnan(lat_orig): # lat_lab_1 = round(lat_orig) - # else: + # else: # lat_lab_1 = '' # lon_lab = ((seg_lon[0]+180) % 360) - 180 # if lat_lab_1 != '': # ## lon_lab_inside = -lon_lab - ## transform=ccrs.PlateCarree(), - # color='orange', fontsize=6, - # verticalalignment='center', horizontalalignment='right') - + ## transform=ccrs.PlateCarree(), + # color='orange', fontsize=6, + # verticalalignment='center', horizontalalignment='right') lon_lines = np.linspace(0, 360, 13) # every 30 deg for lon0 in lon_lines: - lat_line = np.linspace(-90, 90, ny+1) + lat_line = np.linspace(-90, 90, ny + 1) lon_line = np.full_like(lat_line, lon0) - xyz_line = np.column_stack([ - np.cos(np.radians(lat_line)) * np.cos(np.radians(lon_line)), - np.cos(np.radians(lat_line)) * np.sin(np.radians(lon_line)), - np.sin(np.radians(lat_line)) - ]) + xyz_line = np.column_stack( + [ + np.cos(np.radians(lat_line)) * np.cos(np.radians(lon_line)), + np.cos(np.radians(lat_line)) * np.sin(np.radians(lon_line)), + np.sin(np.radians(lat_line)), + ] + ) xyz_line_rot = rot.apply(xyz_line) - lat_rot_line = np.degrees(np.arcsin(xyz_line_rot[:,2])) - lon_rot_line = -np.degrees(np.arctan2(xyz_line_rot[:,1], xyz_line_rot[:,0])) + lat_rot_line = np.degrees(np.arcsin(xyz_line_rot[:, 2])) + lon_rot_line = -np.degrees(np.arctan2(xyz_line_rot[:, 1], xyz_line_rot[:, 0])) lon_rot_line = (lon_rot_line + 180) % 360 - 180 # flip for inside view jump_idx = np.where(np.abs(np.diff(lon_rot_line)) > 180)[0] - split_idx = np.append(jump_idx, len(lon_rot_line)-1) + split_idx = np.append(jump_idx, len(lon_rot_line) - 1) start = 0 # Find the row closest to 0° rotated latitude - lat0_idx = np.argmin(np.abs(lat_rot_line)) # lat_rot_line = your rotated lat array for the line + lat0_idx = np.argmin( + np.abs(lat_rot_line) + ) # lat_rot_line = your rotated lat array for the line # Get the corresponding longitude and latitude along that row lon_line = lon_rot_line[lat0_idx] lat_line = lat_rot_line[lat0_idx] for idx in split_idx: - seg_lon = lon_rot_line[start:idx+1] - seg_lat = lat_rot_line[start:idx+1] - ax.plot(seg_lon, seg_lat, transform=ccrs.PlateCarree(), - color='orange', lw=0.8, alpha=0.7) - - mid_idx = len(seg_lon) // 2 + seg_lon = lon_rot_line[start : idx + 1] + seg_lat = lat_rot_line[start : idx + 1] + ax.plot( + seg_lon, + seg_lat, + transform=ccrs.PlateCarree(), + color="orange", + lw=0.8, + alpha=0.7, + ) + + mid_idx = len(seg_lon) // 2 lon_plot = seg_lon[mid_idx] lat_plot = seg_lat[mid_idx] - start = idx+1 + start = idx + 1 - xyz_rot = sph_to_cart(lat_plot, lon_plot) # returns (3,) or (1,3) + xyz_rot = sph_to_cart(lat_plot, lon_plot) # returns (3,) or (1,3) # print(xyz_rot) # ensure shape (1,3) for scipy Rotation xyz_rot = np.atleast_2d(xyz_rot) # print(xyz_rot) - # --- invert rotation --- + # --- invert rotation --- xyz_orig = rot.inv().apply(xyz_rot) - # --- back to spherical --- + # --- back to spherical --- lat_label = np.degrees(np.arcsin(xyz_orig[0, 2])) lon_label = np.degrees(np.arctan2(xyz_orig[0, 1], xyz_orig[0, 0])) @@ -338,34 +416,38 @@ def make_imap_lo_map(file, lat_center, lon_center, vmin, vmax, output, label_col xyz_rot_label = rot.apply(xyz) lat_plot = np.degrees(np.arcsin(xyz_rot_label[2])) - # print('lon_label = ',lon_label) - # lon_label = (-1.0*lon_label + 180) % 360 - 180 - # print('lon_label (2) = ',lon_label) + # print('lon_label = ',lon_label) + # lon_label = (-1.0*lon_label + 180) % 360 - 180 + # print('lon_label (2) = ',lon_label) lon_plot = -lon_plot if np.abs(lat_label) < 30.0: - ax.text(lon_plot, lat_plot, - f"{lon_label:.0f}°", - transform=ccrs.PlateCarree(), - color='orange', fontsize=6, - verticalalignment='bottom', horizontalalignment='right') - - if (opt_points > 0): - + ax.text( + lon_plot, + lat_plot, + f"{lon_label:.0f}°", + transform=ccrs.PlateCarree(), + color="orange", + fontsize=6, + verticalalignment="bottom", + horizontalalignment="right", + ) + + if opt_points > 0: labels, lon_points, lat_points = make_imap_lo_reference_points(rot, opt_points) npoints = len(labels) - for i in range(0,npoints): + for i in range(0, npoints): lon_ref_rot = lon_points[i] lat_ref_rot = lat_points[i] ref_label = labels[i] - # --- Plot marker --- + # --- Plot marker --- ax.plot( lon_ref_rot, lat_ref_rot, - marker='o', + marker="o", markersize=1, - color='white', + color="white", transform=ccrs.PlateCarree(), - zorder=5 + zorder=5, ) # --- Add text --- @@ -374,57 +456,60 @@ def make_imap_lo_map(file, lat_center, lon_center, vmin, vmax, output, label_col lat_ref_rot, f" {ref_label}", transform=ccrs.PlateCarree(), - color='white', + color="white", fontsize=6, - ha='left', - va='center', - zorder=5 + ha="left", + va="center", + zorder=5, ) -# now optionally add the great circle line + # now optionally add the great circle line - if (opt_ribbon > 0): - - lat_line, lon_line = somewhat_great_circle_centered( rot, - lat0=39, - lon0=221.0, - angle=70-15 + if opt_ribbon > 0: + lat_line, lon_line = somewhat_great_circle_centered( + rot, lat0=39, lon0=221.0, angle=70 - 15 ) plot_wrapped_line( - ax, - -lon_line, # inside view flip if you use it - lat_line, - color='white', - lw=0.5 + ax, + -lon_line, # inside view flip if you use it + lat_line, + color="white", + lw=0.5, ) - - lat_line, lon_line = somewhat_great_circle_centered( rot, - lat0=39, - lon0=221.0, - angle=70+15 + lat_line, lon_line = somewhat_great_circle_centered( + rot, lat0=39, lon0=221.0, angle=70 + 15 ) plot_wrapped_line( - ax, - -lon_line, # inside view flip if you use it - lat_line, - color='white', - lw=0.5 + ax, + -lon_line, # inside view flip if you use it + lat_line, + color="white", + lw=0.5, ) -# label = label_map[dtype] + # label = label_map[dtype] # "Intensity (counts cm$^{-2}$ s$^{-1}$ sr$^{-1}$ keV$^{-1}$)") -# value = esa_energy[esa] * 1000.0 -# s = f"{value:.2f}" -# cbar.set_label(label) -# cbar.set_label(f"{label} [ECLIPJ2000] at ESA (eV) = {s}") + # value = esa_energy[esa] * 1000.0 + # s = f"{value:.2f}" + # cbar.set_label(label) + # cbar.set_label(f"{label} [ECLIPJ2000] at ESA (eV) = {s}") plt.suptitle(plot_title) - plt.colorbar(pcm, ax=ax, orientation='horizontal', pad=0.05, label=label_colorbar) + plt.colorbar(pcm, ax=ax, orientation="horizontal", pad=0.05, label=label_colorbar) ax.set_global() - # plt.show() - plt.savefig(output, dpi=200, facecolor='w', edgecolor='w', orientation='portrait', format=None, transparent=False, bbox_inches=None, pad_inches=0.1) + # plt.show() + plt.savefig( + output, + dpi=200, + facecolor="w", + edgecolor="w", + orientation="portrait", + format=None, + transparent=False, + bbox_inches=None, + pad_inches=0.1, + ) plt.close(fig) - diff --git a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/old/csv2yaml.py b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/old/csv2yaml.py index 91faa4d19..8d8bb6efa 100755 --- a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/old/csv2yaml.py +++ b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/old/csv2yaml.py @@ -10,15 +10,7 @@ import os from datetime import datetime, timedelta, timezone -esa_energy = { - 1: 16, - 2: 30, - 3: 56, - 4: 106, - 5: 200, - 6: 404, - 7: 787 -} +esa_energy = {1: 16, 2: 30, 3: 56, 4: 106, 5: 200, 6: 404, 7: 787} label_map = { "cnts": "Counts", @@ -30,7 +22,7 @@ "fvar": "VVariance Intensity (#/cm4 s2 sr2 keV2)", "runc": "Unc Rate", "rvar": "Variance Rate (#/s2)", - "flux": "Intensity (#/cm2 s sr keV)" + "flux": "Intensity (#/cm2 s sr keV)", } @@ -47,6 +39,7 @@ def latest_monday_tag(): return f"maps_v{year_short}DOY{doy}" + def csv_to_map_yaml( csv_file, output_file, @@ -56,21 +49,19 @@ def csv_to_map_yaml( map_year="2026", uniqueness_index=0, label="Flux", - pivot="90" + pivot="90", ): data_rows = [] - with open(csv_file, newline="") as f: reader = csv.reader(f) header = next(reader) # 10–59, not used for row in reader: formatted_row = [ - f"{float(x):.3E}" if float(x) != 0 else "0.000E+00" - for x in row + f"{float(x):.3E}" if float(x) != 0 else "0.000E+00" for x in row ] data_rows.append(formatted_row) - + tag = latest_monday_tag() # ---------------------------- @@ -102,56 +93,66 @@ def csv_to_map_yaml( for row in data_rows: f.write("- [" + ", ".join(row) + "]\n") -for pp in [75,90,105]: - - work_dir1 = f'./outdir/pivot_{pp}/maps' - yaml_dir_flux = f'./outdir/pivot_{pp}/yaml/flux' - yaml_dir_rate = f'./outdir/pivot_{pp}/yaml/rate' - yaml_dir_cnts = f'./outdir/pivot_{pp}/yaml/cnts' - + +for pp in [75, 90, 105]: + work_dir1 = f"./outdir/pivot_{pp}/maps" + yaml_dir_flux = f"./outdir/pivot_{pp}/yaml/flux" + yaml_dir_rate = f"./outdir/pivot_{pp}/yaml/rate" + yaml_dir_cnts = f"./outdir/pivot_{pp}/yaml/cnts" + os.makedirs(yaml_dir_flux, exist_ok=True) os.makedirs(yaml_dir_rate, exist_ok=True) os.makedirs(yaml_dir_cnts, exist_ok=True) - - for esa in range(1,8): - print(pp,esa) - for tt in ["expo","rate","flux","fvar","rvar","cnts"]:#,"stbg","func","runc",,"rvar"]: - filename = os.path.join(work_dir1, f"map_{tt}_esa{esa}.csv") - energy = esa_energy[esa] - label = label_map[tt] - base_filename = os.path.splitext(os.path.basename(filename))[0] - - if tt=='expo': - tt='exposure' - outputs = [ os.path.join(yaml_dir_rate, f"IMAPLo-{energy}ev-{tt}.txt"), - os.path.join(yaml_dir_flux, f"IMAPLo-{energy}ev-{tt}.txt") ] - if tt=='cnts': - - outputs = [ os.path.join(yaml_dir_cnts, f"IMAPLo-{energy}ev-flux.txt"), - os.path.join(yaml_dir_cnts, f"IMAPLo-{energy}ev-exposure.txt"), - os.path.join(yaml_dir_cnts, f"IMAPLo-{energy}ev-variance.txt")] - - if tt=="rvar": - tt='variance' - outputs = [ os.path.join(yaml_dir_rate, f"IMAPLo-{energy}ev-{tt}.txt") ] - - if tt=="fvar": - tt='variance' - outputs = [ os.path.join(yaml_dir_flux, f"IMAPLo-{energy}ev-{tt}.txt") ] - - if tt=="flux": - tt='flux' - outputs = [ os.path.join(yaml_dir_flux, f"IMAPLo-{energy}ev-{tt}.txt") ] - - if tt =='rate': - tt='flux' - outputs = [ os.path.join(yaml_dir_rate, f"IMAPLo-{energy}ev-{tt}.txt") ] - - for output in outputs: - csv_to_map_yaml( - csv_file=filename, - output_file=output, - energy_value=energy, - label=label, - pivot=pp - ) + + for esa in range(1, 8): + print(pp, esa) + for tt in [ + "expo", + "rate", + "flux", + "fvar", + "rvar", + "cnts", + ]: # ,"stbg","func","runc",,"rvar"]: + filename = os.path.join(work_dir1, f"map_{tt}_esa{esa}.csv") + energy = esa_energy[esa] + label = label_map[tt] + base_filename = os.path.splitext(os.path.basename(filename))[0] + + if tt == "expo": + tt = "exposure" + outputs = [ + os.path.join(yaml_dir_rate, f"IMAPLo-{energy}ev-{tt}.txt"), + os.path.join(yaml_dir_flux, f"IMAPLo-{energy}ev-{tt}.txt"), + ] + if tt == "cnts": + outputs = [ + os.path.join(yaml_dir_cnts, f"IMAPLo-{energy}ev-flux.txt"), + os.path.join(yaml_dir_cnts, f"IMAPLo-{energy}ev-exposure.txt"), + os.path.join(yaml_dir_cnts, f"IMAPLo-{energy}ev-variance.txt"), + ] + + if tt == "rvar": + tt = "variance" + outputs = [os.path.join(yaml_dir_rate, f"IMAPLo-{energy}ev-{tt}.txt")] + + if tt == "fvar": + tt = "variance" + outputs = [os.path.join(yaml_dir_flux, f"IMAPLo-{energy}ev-{tt}.txt")] + + if tt == "flux": + tt = "flux" + outputs = [os.path.join(yaml_dir_flux, f"IMAPLo-{energy}ev-{tt}.txt")] + + if tt == "rate": + tt = "flux" + outputs = [os.path.join(yaml_dir_rate, f"IMAPLo-{energy}ev-{tt}.txt")] + + for output in outputs: + csv_to_map_yaml( + csv_file=filename, + output_file=output, + energy_value=energy, + label=label, + pivot=pp, + ) diff --git a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/plot_map_DN.py b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/plot_map_DN.py index cb2a0f01a..fcae4b7a8 100755 --- a/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/plot_map_DN.py +++ b/tests/test_data/lo/lo_txt_pipeline/3S5_l1b_ram_maps/plot_map_DN.py @@ -14,6 +14,7 @@ PI = np.pi + def sph_to_cart(lat, lon): lat_rad = np.radians(lat) lon_rad = np.radians(lon) @@ -22,14 +23,16 @@ def sph_to_cart(lat, lon): z = np.sin(lat_rad) return np.array([x, y, z]) + def rotation_matrix_to_center(new_colat0, new_lon0): # Step 1: rotate around Z axis by -lon0 - r1 = R.from_euler('z', -new_lon0, degrees=True) + r1 = R.from_euler("z", -new_lon0, degrees=True) # Step 2: rotate around Y axis by 90-lat0 - r2 = R.from_euler('y', 90-new_colat0, degrees=True) + r2 = R.from_euler("y", 90 - new_colat0, degrees=True) # Combined rotation return r2 * r1 + def inverse_rotation_matrix_to_center(rot, xyz_rot): """ Apply inverse rotation to xyz_rot points. @@ -39,6 +42,7 @@ def inverse_rotation_matrix_to_center(rot, xyz_rot): """ return rot.inv().apply(xyz_rot) + esa_energy = { 1: 0.016, 2: 0.030, @@ -47,97 +51,120 @@ def inverse_rotation_matrix_to_center(rot, xyz_rot): 5: 0.200, 6: 0.404, 7: 0.787, - 8: 1.821 + 8: 1.821, } label_map = { - "cnts": "Counts", - "rate": "Rate (counts s$^{-1}$)", - "backrate": "Rate (counts s$^{-1}$)", - "expo": "Exposure Time (sec)", - "stbg": "Signal/Ubiq-Bkgd", - "func": "Unc Intensity", - "fser": "Flux Sys Uncertainty", - "fvar": "Variance Intensity", - "fvto": "Variance Total", - "runc": "Unc Rate", - "rvar": "Variance Rate", - "flux": "Intensity (counts cm$^{-2}$ s$^{-1}$ sr$^{-1}$ keV$^{-1}$)", - "cosalpha": "cos(alpha), alpha angle between Ram and Boresight" - } - -# MAIN - -for pp in [75,90,105]: - work_dir1 = f'./outdir/pivot_{pp}/maps' - plot_dir = f'./outdir/pivot_{pp}/plots' - os.makedirs(plot_dir, exist_ok=True) - - for esa in range(1,8): - print(esa) - for tt in ["expo","rate","flux","cnts","fser","func","runc","fvar","fvto","rvar","stbg","cosalpha"]: - filename = os.path.join(work_dir1, f"map_{tt}_esa{esa}.csv") - data = np.loadtxt(filename, delimiter=',', skiprows=1) + "cnts": "Counts", + "rate": "Rate (counts s$^{-1}$)", + "backrate": "Rate (counts s$^{-1}$)", + "expo": "Exposure Time (sec)", + "stbg": "Signal/Ubiq-Bkgd", + "func": "Unc Intensity", + "fser": "Flux Sys Uncertainty", + "fvar": "Variance Intensity", + "fvto": "Variance Total", + "runc": "Unc Rate", + "rvar": "Variance Rate", + "flux": "Intensity (counts cm$^{-2}$ s$^{-1}$ sr$^{-1}$ keV$^{-1}$)", + "cosalpha": "cos(alpha), alpha angle between Ram and Boresight", +} - base_filename = os.path.splitext(os.path.basename(filename))[0] - output = os.path.join(plot_dir, f"map_{tt}_esa{esa}.png") - print(tt) +# MAIN - if tt =="expo": - vmax = np.max(data) - vmin = 1e-1 +for pp in [75, 90, 105]: + work_dir1 = f"./outdir/pivot_{pp}/maps" + plot_dir = f"./outdir/pivot_{pp}/plots" + os.makedirs(plot_dir, exist_ok=True) - if tt =="cosalpha": + for esa in range(1, 8): + print(esa) + for tt in [ + "expo", + "rate", + "flux", + "cnts", + "fser", + "func", + "runc", + "fvar", + "fvto", + "rvar", + "stbg", + "cosalpha", + ]: + filename = os.path.join(work_dir1, f"map_{tt}_esa{esa}.csv") + data = np.loadtxt(filename, delimiter=",", skiprows=1) + + base_filename = os.path.splitext(os.path.basename(filename))[0] + output = os.path.join(plot_dir, f"map_{tt}_esa{esa}.png") + print(tt) + + if tt == "expo": + vmax = np.max(data) + vmin = 1e-1 + + if tt == "cosalpha": + vmax = np.max(data) + vmin = -1.0 * vmax + + if tt in [ + "cnts", + "rate", + "flux", + "fvar", + "fvto", + "fser", + "stbg", + "func", + "runc", + "rvar", + ]: + if esa == 5: + vmax = np.max(data) + vmin = vmax * 1e-2 + elif esa == 4 or esa == 3: + vmax = np.max(data) + vmin = vmax * 1e-4 + elif esa == 1 or esa == 2: + vmax = np.max(data) + vmin = vmax * 1e-3 + elif esa == 6: vmax = np.max(data) - vmin = -1.0*vmax - - if tt in ["cnts", "rate", "flux", "fvar", "fvto", "fser", "stbg", "func", "runc", "rvar"]: - if esa==5: - vmax = np.max(data) - vmin = vmax * 1e-2 - elif esa==4 or esa==3: - vmax = np.max(data) - vmin = vmax * 1e-4 - elif esa==1 or esa==2: - vmax = np.max(data) - vmin = vmax * 1e-3 - elif esa==6: - vmax = np.max(data) - vmin = vmax * 1e-2 - else: - vmax = np.max(data) - vmin = vmax * 1e-2 - - lat_center = 5.0 - lon_center = -105.0 - - label = label_map[tt] - # "Intensity (counts cm$^{-2}$ s$^{-1}$ sr$^{-1}$ keV$^{-1}$)") - value = esa_energy[esa] * 1000.0 - s = f"{value:.2f}" - label = label + f'[ECLIPJ2000] at ESA (eV) = {s}' -# print(label) - title = f'Pivot:{pp}' - if tt =="cosalpha": - pchlin.make_imap_lo_map(filename, lat_center, lon_center, vmin, vmax,output,label_colorbar=label,plot_title=title) + vmin = vmax * 1e-2 else: - patch.make_imap_lo_map(filename, lat_center, lon_center, vmin, vmax,output,label_colorbar=label,plot_title=title) - - - - - - - - - - - - - - - - - - - + vmax = np.max(data) + vmin = vmax * 1e-2 + + lat_center = 5.0 + lon_center = -105.0 + + label = label_map[tt] + # "Intensity (counts cm$^{-2}$ s$^{-1}$ sr$^{-1}$ keV$^{-1}$)") + value = esa_energy[esa] * 1000.0 + s = f"{value:.2f}" + label = label + f"[ECLIPJ2000] at ESA (eV) = {s}" + # print(label) + title = f"Pivot:{pp}" + if tt == "cosalpha": + pchlin.make_imap_lo_map( + filename, + lat_center, + lon_center, + vmin, + vmax, + output, + label_colorbar=label, + plot_title=title, + ) + else: + patch.make_imap_lo_map( + filename, + lat_center, + lon_center, + vmin, + vmax, + output, + label_colorbar=label, + plot_title=title, + ) diff --git a/tests/test_data_utils.py b/tests/test_data_utils.py index 87d2f5316..7da1fd0f2 100644 --- a/tests/test_data_utils.py +++ b/tests/test_data_utils.py @@ -9,151 +9,259 @@ class TestDataUtils(unittest.TestCase): def test_1d_rebin(self): - hit_data_epoch = np.array([datetime(2020, 4, 4, 0, 5), datetime(2020, 4, 4, 0, 15)]) + hit_data_epoch = np.array( + [datetime(2020, 4, 4, 0, 5), datetime(2020, 4, 4, 0, 15)] + ) hit_data_delta = np.array([timedelta(seconds=300), timedelta(seconds=300)]) extra_data_at_beginning = [0, 0.5] - extra_dates_at_beginning = [datetime(2020, 4, 3, 0, 23), datetime(2020, 4, 3, 0, 23, 30)] - extra_dates_at_end = [datetime(2020, 4, 4, 1, 0), datetime(2020, 4, 4, 1, 23, 30)] + extra_dates_at_beginning = [ + datetime(2020, 4, 3, 0, 23), + datetime(2020, 4, 3, 0, 23, 30), + ] + extra_dates_at_end = [ + datetime(2020, 4, 4, 1, 0), + datetime(2020, 4, 4, 1, 23, 30), + ] extra_data_at_end = [1, 1.5] - input_data = np.array(extra_data_at_beginning + - [0, 1, 0, 1, 0, 1, 0, 0] + extra_data_at_end) - mag_epoch = np.array(extra_dates_at_beginning + [datetime(2020, 4, 4, 0, 1), - datetime(2020, 4, 4, 0, 3), - datetime(2020, 4, 4, 0, 6), - datetime(2020, 4, 4, 0, 9, 55), - datetime(2020, 4, 4, 0, 12, 45), - datetime(2020, 4, 4, 0, 13), - datetime(2020, 4, 4, 0, 15), - datetime(2020, 4, 4, 0, 17)] + extra_dates_at_end) + input_data = np.array( + extra_data_at_beginning + [0, 1, 0, 1, 0, 1, 0, 0] + extra_data_at_end + ) + mag_epoch = np.array( + extra_dates_at_beginning + + [ + datetime(2020, 4, 4, 0, 1), + datetime(2020, 4, 4, 0, 3), + datetime(2020, 4, 4, 0, 6), + datetime(2020, 4, 4, 0, 9, 55), + datetime(2020, 4, 4, 0, 12, 45), + datetime(2020, 4, 4, 0, 13), + datetime(2020, 4, 4, 0, 15), + datetime(2020, 4, 4, 0, 17), + ] + + extra_dates_at_end + ) expected_average = np.array([1 / 2, 1 / 4]) - actual_average = rebin(from_epoch=mag_epoch, from_data=input_data, - to_epoch=hit_data_epoch, - to_epoch_delta=hit_data_delta) + actual_average = rebin( + from_epoch=mag_epoch, + from_data=input_data, + to_epoch=hit_data_epoch, + to_epoch_delta=hit_data_delta, + ) np.testing.assert_array_equal(actual_average, expected_average) def test_rebin_with_extra_data_at_beginning_and_end(self): - hit_data_epoch = np.array([datetime(2020, 4, 4, 0, 5), datetime(2020, 4, 4, 0, 15)]) + hit_data_epoch = np.array( + [datetime(2020, 4, 4, 0, 5), datetime(2020, 4, 4, 0, 15)] + ) hit_data_delta = np.array([timedelta(seconds=300), timedelta(seconds=300)]) extra_data_at_beginning = [[0.5, 0, 1], [0, 0.5, 1]] - extra_dates_at_beginning = [datetime(2020, 4, 3, 0, 23), datetime(2020, 4, 3, 0, 23, 30)] - extra_dates_at_end = [datetime(2020, 4, 4, 1, 0), datetime(2020, 4, 4, 1, 23, 30)] + extra_dates_at_beginning = [ + datetime(2020, 4, 3, 0, 23), + datetime(2020, 4, 3, 0, 23, 30), + ] + extra_dates_at_end = [ + datetime(2020, 4, 4, 1, 0), + datetime(2020, 4, 4, 1, 23, 30), + ] extra_data_at_end = [[0.5, 1, 1], [1, 0.5, 0.5]] - mag_data = np.array(extra_data_at_beginning + - [[0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], - [1, 0, 0]] + extra_data_at_end) - mag_epoch = np.array(extra_dates_at_beginning + [datetime(2020, 4, 4, 0, 1), - datetime(2020, 4, 4, 0, 3), datetime(2020, 4, 4, 0, 6), - datetime(2020, 4, 4, 0, 9, 55), - datetime(2020, 4, 4, 0, 12, 45), - datetime(2020, 4, 4, 0, 13), - datetime(2020, 4, 4, 0, 15), - datetime(2020, 4, 4, 0, 17)] + extra_dates_at_end) + mag_data = np.array( + extra_data_at_beginning + + [ + [0, 0, 1], + [0, 1, 0], + [1, 0, 0], + [0, 1, 0], + [0, 0, 1], + [0, 1, 0], + [1, 0, 0], + [1, 0, 0], + ] + + extra_data_at_end + ) + mag_epoch = np.array( + extra_dates_at_beginning + + [ + datetime(2020, 4, 4, 0, 1), + datetime(2020, 4, 4, 0, 3), + datetime(2020, 4, 4, 0, 6), + datetime(2020, 4, 4, 0, 9, 55), + datetime(2020, 4, 4, 0, 12, 45), + datetime(2020, 4, 4, 0, 13), + datetime(2020, 4, 4, 0, 15), + datetime(2020, 4, 4, 0, 17), + ] + + extra_dates_at_end + ) expected_average = np.array([[1 / 4, 1 / 2, 1 / 4], [1 / 2, 1 / 4, 1 / 4]]) - actual_average = rebin(from_epoch=mag_epoch, from_data=mag_data, - to_epoch=hit_data_epoch, - to_epoch_delta=hit_data_delta) + actual_average = rebin( + from_epoch=mag_epoch, + from_data=mag_data, + to_epoch=hit_data_epoch, + to_epoch_delta=hit_data_delta, + ) np.testing.assert_array_equal(actual_average, expected_average) def test_rebin_handles_missing_data(self): - hit_data_epoch = np.array([datetime(2020, 4, 4, 0, 5), - datetime(2020, 4, 4, 0, 15), - datetime(2020, 4, 4, 0, 25)]) - hit_data_delta = np.array([timedelta(seconds=300), timedelta(seconds=300), timedelta(seconds=300)]) + hit_data_epoch = np.array( + [ + datetime(2020, 4, 4, 0, 5), + datetime(2020, 4, 4, 0, 15), + datetime(2020, 4, 4, 0, 25), + ] + ) + hit_data_delta = np.array( + [timedelta(seconds=300), timedelta(seconds=300), timedelta(seconds=300)] + ) mag_data = np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0]]) - mag_epoch = [datetime(2020, 4, 4, 0, 1), - datetime(2020, 4, 4, 0, 3), - datetime(2020, 4, 4, 0, 22), - datetime(2020, 4, 4, 0, 27)] - expected_average = np.array([[0, 1 / 2, 1 / 2], [np.nan, np.nan, np.nan], [1 / 2, 1 / 2, 0]]) - - actual_average = rebin(from_epoch=mag_epoch, from_data=mag_data, - to_epoch=hit_data_epoch, - to_epoch_delta=hit_data_delta) + mag_epoch = [ + datetime(2020, 4, 4, 0, 1), + datetime(2020, 4, 4, 0, 3), + datetime(2020, 4, 4, 0, 22), + datetime(2020, 4, 4, 0, 27), + ] + expected_average = np.array( + [[0, 1 / 2, 1 / 2], [np.nan, np.nan, np.nan], [1 / 2, 1 / 2, 0]] + ) + + actual_average = rebin( + from_epoch=mag_epoch, + from_data=mag_data, + to_epoch=hit_data_epoch, + to_epoch_delta=hit_data_delta, + ) np.testing.assert_array_equal(actual_average, expected_average) def test_rebin_with_missing_data_fills_with_nan(self): - hit_data_epoch = np.array([datetime(2020, 4, 4, 0, 5), - datetime(2020, 4, 4, 0, 15), - datetime(2020, 4, 4, 0, 25)]) - hit_data_delta = np.array([timedelta(seconds=300), timedelta(seconds=300), timedelta(seconds=300)]) + hit_data_epoch = np.array( + [ + datetime(2020, 4, 4, 0, 5), + datetime(2020, 4, 4, 0, 15), + datetime(2020, 4, 4, 0, 25), + ] + ) + hit_data_delta = np.array( + [timedelta(seconds=300), timedelta(seconds=300), timedelta(seconds=300)] + ) mag_data = np.empty((0, 3)) mag_epoch = [] - actual_average = rebin(from_epoch=mag_epoch, from_data=mag_data, - to_epoch=hit_data_epoch, - to_epoch_delta=hit_data_delta) + actual_average = rebin( + from_epoch=mag_epoch, + from_data=mag_data, + to_epoch=hit_data_epoch, + to_epoch_delta=hit_data_delta, + ) expected_average = [ [np.nan, np.nan, np.nan], [np.nan, np.nan, np.nan], - [np.nan, np.nan, np.nan] + [np.nan, np.nan, np.nan], ] np.testing.assert_array_equal(expected_average, actual_average) def test_rebin_empty_bins_with_nan(self): - hit_data_epoch = np.array([datetime(2020, 4, 4, 0, 5), - datetime(2020, 4, 4, 0, 15), - datetime(2020, 4, 4, 0, 25)]) - hit_data_delta = np.array([timedelta(seconds=300), timedelta(seconds=300), timedelta(seconds=300)]) + hit_data_epoch = np.array( + [ + datetime(2020, 4, 4, 0, 5), + datetime(2020, 4, 4, 0, 15), + datetime(2020, 4, 4, 0, 25), + ] + ) + hit_data_delta = np.array( + [timedelta(seconds=300), timedelta(seconds=300), timedelta(seconds=300)] + ) mag_data = np.array([[0, 0, 1], [0, 1, 0]]) - mag_epoch = [datetime(2020, 4, 4, 0, 1), - datetime(2020, 4, 4, 0, 3)] - actual_average = rebin(from_epoch=mag_epoch, from_data=mag_data, - to_epoch=hit_data_epoch, - to_epoch_delta=hit_data_delta) + mag_epoch = [datetime(2020, 4, 4, 0, 1), datetime(2020, 4, 4, 0, 3)] + actual_average = rebin( + from_epoch=mag_epoch, + from_data=mag_data, + to_epoch=hit_data_epoch, + to_epoch_delta=hit_data_delta, + ) expected_average = [ [0, 0.5, 0.5], [np.nan, np.nan, np.nan], - [np.nan, np.nan, np.nan] + [np.nan, np.nan, np.nan], ] np.testing.assert_array_equal(actual_average, expected_average) def test_gap_in_to_epoch(self): - hit_data_epoch = np.array([datetime(2020, 4, 4, 0, 5), - datetime(2020, 4, 4, 0, 15), - datetime(2020, 4, 4, 0, 30)]) - hit_data_delta = np.array([timedelta(seconds=300), timedelta(seconds=300), timedelta(seconds=300)]) + hit_data_epoch = np.array( + [ + datetime(2020, 4, 4, 0, 5), + datetime(2020, 4, 4, 0, 15), + datetime(2020, 4, 4, 0, 30), + ] + ) + hit_data_delta = np.array( + [timedelta(seconds=300), timedelta(seconds=300), timedelta(seconds=300)] + ) mag_data = np.array([[0, 0, 1], [0, 1, 0], [0, 0, 1], [1, 0, 0]]) - mag_epoch = [datetime(2020, 4, 4, 0, 1), - datetime(2020, 4, 4, 0, 7), - datetime(2020, 4, 4, 0, 17), - datetime(2020, 4, 4, 0, 21) - ] - actual_average = rebin(from_epoch=mag_epoch, from_data=mag_data, - to_epoch=hit_data_epoch, - to_epoch_delta=hit_data_delta) - expected_average = [ - [0, 0.5, 0.5], - [0, 0, 1], - [np.nan, np.nan, np.nan] + mag_epoch = [ + datetime(2020, 4, 4, 0, 1), + datetime(2020, 4, 4, 0, 7), + datetime(2020, 4, 4, 0, 17), + datetime(2020, 4, 4, 0, 21), ] + actual_average = rebin( + from_epoch=mag_epoch, + from_data=mag_data, + to_epoch=hit_data_epoch, + to_epoch_delta=hit_data_delta, + ) + expected_average = [[0, 0.5, 0.5], [0, 0, 1], [np.nan, np.nan, np.nan]] np.testing.assert_array_equal(actual_average, expected_average) def test_nearest_interpolator_interpolate(self): test_cases = [ - ("matching cadence", [datetime(2020, 4, 4), - datetime(2020, 4, 5), - datetime(2020, 4, 6), - datetime(2020, 4, 7)], - [[0, 0, 1], [0, 2, 0], [0, 0, 3], [4, 0, 0]], [False, True, False, False]), - ("to slower cadence", [datetime(2020, 4, 4), - datetime(2020, 4, 6), - datetime(2020, 4, 8)], - [[0, 0, 1], [0, 0, 3], [4, 0, 0]], [False, False, False]), - ("to faster cadence", [datetime(2020, 4, 4, hour=8), - datetime(2020, 4, 5), - datetime(2020, 4, 5, hour=16)], - [[0, 0, 1], [0, 2, 0], [0, 0, 3]], [False, True, False]), - ("outside range", [datetime(2020, 4, 2, hour=23), - datetime(2020, 4, 5), - datetime(2020, 4, 8, hour=1)], - [[np.nan, np.nan, np.nan], [0, 2, 0], [np.nan, np.nan, np.nan]], [False, True, False]), - - ("ties round down", [datetime(2020, 4, 4, hour=12), - datetime(2020, 4, 5, hour=12)], - [[0, 0, 1], [0, 2, 0], ], [False, True]), + ( + "matching cadence", + [ + datetime(2020, 4, 4), + datetime(2020, 4, 5), + datetime(2020, 4, 6), + datetime(2020, 4, 7), + ], + [[0, 0, 1], [0, 2, 0], [0, 0, 3], [4, 0, 0]], + [False, True, False, False], + ), + ( + "to slower cadence", + [datetime(2020, 4, 4), datetime(2020, 4, 6), datetime(2020, 4, 8)], + [[0, 0, 1], [0, 0, 3], [4, 0, 0]], + [False, False, False], + ), + ( + "to faster cadence", + [ + datetime(2020, 4, 4, hour=8), + datetime(2020, 4, 5), + datetime(2020, 4, 5, hour=16), + ], + [[0, 0, 1], [0, 2, 0], [0, 0, 3]], + [False, True, False], + ), + ( + "outside range", + [ + datetime(2020, 4, 2, hour=23), + datetime(2020, 4, 5), + datetime(2020, 4, 8, hour=1), + ], + [[np.nan, np.nan, np.nan], [0, 2, 0], [np.nan, np.nan, np.nan]], + [False, True, False], + ), + ( + "ties round down", + [datetime(2020, 4, 4, hour=12), datetime(2020, 4, 5, hour=12)], + [ + [0, 0, 1], + [0, 2, 0], + ], + [False, True], + ), ] for case, to_dates, expected_values, expected_quality_flags in test_cases: @@ -161,43 +269,69 @@ def test_nearest_interpolator_interpolate(self): to_data_epoch = np.array(to_dates) from_data = np.array([[0, 0, 1], [0, 2, 0], [0, 0, 3], [4, 0, 0]]) quality_flags = np.array([False, True, False, False]) - from_date_epoch = np.array([datetime(2020, 4, 4), - datetime(2020, 4, 5), - datetime(2020, 4, 6), - datetime(2020, 4, 7) - ]) + from_date_epoch = np.array( + [ + datetime(2020, 4, 4), + datetime(2020, 4, 5), + datetime(2020, 4, 6), + datetime(2020, 4, 7), + ] + ) - interpolator = NearestInterpolator(from_date_epoch, from_data, to_data_epoch, timedelta(days=1)) + interpolator = NearestInterpolator( + from_date_epoch, from_data, to_data_epoch, timedelta(days=1) + ) - np.testing.assert_array_equal(interpolator.interpolate_data(), expected_values) - np.testing.assert_array_equal(interpolator.interpolate_flags(quality_flags), expected_quality_flags) + np.testing.assert_array_equal( + interpolator.interpolate_data(), expected_values + ) + np.testing.assert_array_equal( + interpolator.interpolate_flags(quality_flags), + expected_quality_flags, + ) def test_find_closest_neighbor_handles_large_dataset(self): to_epoch = np.array( [ [ [ - datetime(2020, 4, 4) + timedelta(minutes=i) + timedelta(seconds=j) + timedelta(seconds=k) for k - in range(30) + datetime(2020, 4, 4) + + timedelta(minutes=i) + + timedelta(seconds=j) + + timedelta(seconds=k) + for k in range(30) ] for j in range(24) ] - for i in range(1440)]) + for i in range(1440) + ] + ) from_data = np.repeat([[1, 0, 0]], 86400 * 2, axis=0) - from_epoch = np.array([datetime(2020, 4, 4) + timedelta(seconds=0.5) * i for i in range(86400 * 2)]) + from_epoch = np.array( + [ + datetime(2020, 4, 4) + timedelta(seconds=0.5) * i + for i in range(86400 * 2) + ] + ) t0 = time.perf_counter() - actual_neighbor_values = NearestInterpolator(from_epoch, from_data, to_epoch.astype(np.datetime64), timedelta(days=1)).interpolate_data() + actual_neighbor_values = NearestInterpolator( + from_epoch, from_data, to_epoch.astype(np.datetime64), timedelta(days=1) + ).interpolate_data() t1 = time.perf_counter() self.assertEqual((1440, 24, 30, 3), actual_neighbor_values.shape) self.assertLess(t1 - t0, 10) def test_nearest_interpolator_handles_nan(self): to_epoch = np.array([datetime(2020, 4, 4, minute=1)]) - from_epoch = np.array([datetime(2020, 4, 4, minute=1), datetime(2020, 4, 4, minute=2)]) + from_epoch = np.array( + [datetime(2020, 4, 4, minute=1), datetime(2020, 4, 4, minute=2)] + ) from_data = np.array([np.nan, 2]) - interpolator = NearestInterpolator(from_epoch, from_data, to_epoch, timedelta(minutes=1)) + interpolator = NearestInterpolator( + from_epoch, from_data, to_epoch, timedelta(minutes=1) + ) self.assertEqual(interpolator.interpolate_data(), [2]) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index d35118b6e..019babe3d 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -18,7 +18,10 @@ import tests from imap_l3_processing.models import VersionMap from imap_l3_processing.swe.l3.models import SweConfiguration, SweL3MomentData -from imap_l3_processing.swe.l3.science.moment_calculations import Moments, MomentFitResults +from imap_l3_processing.swe.l3.science.moment_calculations import ( + Moments, + MomentFitResults, +) run_local_data_path = Path(tests.__file__).parent.parent / "run_local_input_data" @@ -36,9 +39,11 @@ def try_get_many_run_local_paths(extensions: list[str]) -> tuple[bool, list[Path missing_path = True return missing_path, paths + def get_imap_data_dir_path() -> Path: return Path(imap_l3_processing.__file__).parent.parent / "data" / "imap" + def get_test_data_path(filename: str) -> Path: return Path(tests.__file__).parent / "test_data" / filename @@ -46,6 +51,7 @@ def get_test_data_path(filename: str) -> Path: def get_integration_test_data_path(filename: str) -> Path: return Path(tests.__file__).parent / "integration" / "test_data" / filename + def get_spice_data_path(filename: str) -> Path: return Path(tests.__file__).parent.parent / "spice_kernels" / filename @@ -72,21 +78,40 @@ def build_swe_configuration(**args) -> SweConfiguration: def build_moments(**args) -> Moments: - default_moments = dict(alpha=1, beta=2, t_parallel=3e5, t_perpendicular=4e5, velocity_x=500, velocity_y=600, - velocity_z=700, density=80, aoo=9, ao=10) + default_moments = dict( + alpha=1, + beta=2, + t_parallel=3e5, + t_perpendicular=4e5, + velocity_x=500, + velocity_y=600, + velocity_z=700, + density=80, + aoo=9, + ao=10, + ) default_moments.update(**args) return Moments(**default_moments) -def build_moment_fit_results(moments: Moments = None, chisq: float = 1, number_of_points: int = 10, - regress_result: np.ndarray = None) -> MomentFitResults: +def build_moment_fit_results( + moments: Moments = None, + chisq: float = 1, + number_of_points: int = 10, + regress_result: np.ndarray = None, +) -> MomentFitResults: if moments is None: moments = build_moments() if regress_result is None: regress_result = np.ndarray([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) - return MomentFitResults(moments=moments, chisq=chisq, number_of_points=number_of_points, - regress_result=regress_result) + return MomentFitResults( + moments=moments, + chisq=chisq, + number_of_points=number_of_points, + regress_result=regress_result, + ) + def build_swe_moment_data(epoch_len): integrated_field_shapes: dict[str, tuple[int, ...]] = { @@ -132,20 +157,28 @@ def build_swe_moment_data(epoch_len): } mock_moment_data = create_dataclass_mock(SweL3MomentData) for name, trailing_shape in integrated_field_shapes.items(): - setattr(mock_moment_data, name, np.ones((epoch_len, *trailing_shape), dtype=np.float64)) + setattr( + mock_moment_data, + name, + np.ones((epoch_len, *trailing_shape), dtype=np.float64), + ) mock_moment_data.core_t_parallel_integrated = np.arange(epoch_len, dtype=np.float64) mock_moment_data.core_t_parallel_fit = np.arange(epoch_len, dtype=np.float64) - mock_moment_data.core_t_perpendicular_integrated = np.arange(epoch_len * 2, dtype=np.float64).reshape((-1, 2)) + mock_moment_data.core_t_perpendicular_integrated = np.arange( + epoch_len * 2, dtype=np.float64 + ).reshape((-1, 2)) mock_moment_data.core_t_perpendicular_fit = np.arange(epoch_len, dtype=np.float64) mock_moment_data.halo_t_parallel_integrated = np.arange(epoch_len, dtype=np.float64) mock_moment_data.halo_t_parallel_fit = np.arange(epoch_len, dtype=np.float64) - mock_moment_data.halo_t_perpendicular_integrated = np.arange(epoch_len * 2, dtype=np.float64).reshape((-1, 2)) + mock_moment_data.halo_t_perpendicular_integrated = np.arange( + epoch_len * 2, dtype=np.float64 + ).reshape((-1, 2)) mock_moment_data.halo_t_perpendicular_fit = np.arange(epoch_len, dtype=np.float64) mock_moment_data.quality_flags = np.zeros(epoch_len, dtype=np.uint16) return mock_moment_data -T = TypeVar('T') +T = TypeVar("T") def create_dataclass_mock(obj: Type[T], **kwargs) -> T: @@ -176,12 +209,15 @@ def assert_dict_close(x, y, rtol=1e-7, path=None): path = [] path_str = " > ".join(path) if isinstance(x, dict) and isinstance(y, dict): - assert set(x.keys()) == set( - y.keys()), f"keys differ at {path_str}\n expected keys: {x.keys()}\n actual keys: {y.keys()}" + assert set(x.keys()) == set(y.keys()), ( + f"keys differ at {path_str}\n expected keys: {x.keys()}\n actual keys: {y.keys()}" + ) for k in x: assert_dict_close(x[k], y[k], rtol, path.copy() + [k]) elif isinstance(x, (list, np.ndarray, float)): - np.testing.assert_allclose(x, y, rtol=rtol, err_msg=f"path to failure: {path_str}") + np.testing.assert_allclose( + x, y, rtol=rtol, err_msg=f"path to failure: {path_str}" + ) else: assert x == y, f"{x} != {y} at path {path_str}" @@ -203,7 +239,11 @@ def environment_variables(env_vars: dict): def decorator(func): def wrapper(*args, **kwargs): - old_vars = {k: os.environ.get(v) for k, v in env_vars.items() if os.environ.get(str(v)) is not None} + old_vars = { + k: os.environ.get(v) + for k, v in env_vars.items() + if os.environ.get(str(v)) is not None + } for k, v in env_vars.items(): os.environ[k] = str(v) @@ -222,8 +262,9 @@ def wrapper(*args, **kwargs): return decorator -def create_mock_query_results(file_names: list[Path | str], ingestion_dates: Optional[list[datetime]] = None) -> list[ - dict]: +def create_mock_query_results( + file_names: list[Path | str], ingestion_dates: Optional[list[datetime]] = None +) -> list[dict]: file_paths = [] if ingestion_dates is None: @@ -239,37 +280,49 @@ def create_mock_query_results(file_names: list[Path | str], ingestion_dates: Opt match imap_file_path: case ScienceFilePath(): - version_object = Version(imap_file_path.major_version, imap_file_path.minor_version) - file_paths.append({ - "instrument": imap_file_path.instrument, - "data_level": imap_file_path.data_level, - "descriptor": imap_file_path.descriptor, - "start_date": imap_file_path.start_date, - "ingestion_date": ingestion_date.strftime("%Y%m%d %H:%M:%S"), - "major_version": version_object.major, - "minor_version": version_object.minor, - "cr": imap_file_path.cr, - "file_path": file_path, - "repointing": imap_file_path.repointing - }) + version_object = Version( + imap_file_path.major_version, imap_file_path.minor_version + ) + file_paths.append( + { + "instrument": imap_file_path.instrument, + "data_level": imap_file_path.data_level, + "descriptor": imap_file_path.descriptor, + "start_date": imap_file_path.start_date, + "ingestion_date": ingestion_date.strftime("%Y%m%d %H:%M:%S"), + "major_version": version_object.major, + "minor_version": version_object.minor, + "cr": imap_file_path.cr, + "file_path": file_path, + "repointing": imap_file_path.repointing, + } + ) case AncillaryFilePath(): - file_paths.append({ - "instrument": imap_file_path.instrument, - "descriptor": imap_file_path.descriptor, - "start_date": imap_file_path.start_date, - "end_date": imap_file_path.end_date, - "ingestion_date": ingestion_date.strftime("%Y%m%d %H:%M:%S"), - "version": imap_file_path.version, - "file_path": file_path, - }) + file_paths.append( + { + "instrument": imap_file_path.instrument, + "descriptor": imap_file_path.descriptor, + "start_date": imap_file_path.start_date, + "end_date": imap_file_path.end_date, + "ingestion_date": ingestion_date.strftime("%Y%m%d %H:%M:%S"), + "version": imap_file_path.version, + "file_path": file_path, + } + ) case SPICEFilePath(): continue case _: raise NotImplementedError(f"Unexpected file path type {imap_file_path}") return file_paths -def create_mock_version_map(descriptor: Optional[str] = 'descriptor', major_version: Optional[int] = None, minor_version: Optional[int] = 1): - return VersionMap({descriptor:Version(major_version, minor_version)}) + +def create_mock_version_map( + descriptor: Optional[str] = "descriptor", + major_version: Optional[int] = None, + minor_version: Optional[int] = 1, +): + return VersionMap({descriptor: Version(major_version, minor_version)}) + @dataclass class PeriodicallyRunTest: @@ -282,7 +335,9 @@ def run_periodically(frequency: timedelta): def run_periodically_decorator(test_item): @wraps(test_item) def test_thing(*args): - periodically_run_tests_path = Path(__file__).parent / "periodically_run_tests.json" + periodically_run_tests_path = ( + Path(__file__).parent / "periodically_run_tests.json" + ) periodically_run_tests = json.loads(periodically_run_tests_path.read_text()) last_run = periodically_run_tests.get(test_item.__name__) @@ -290,15 +345,21 @@ def test_thing(*args): if last_run is not None: last_run_time = datetime.fromisoformat(last_run) + frequency if datetime.now() < last_run_time: - raise SkipTest(f'Skipping expensive test, {test_item.__name__}, because it passed recently') + raise SkipTest( + f"Skipping expensive test, {test_item.__name__}, because it passed recently" + ) try: test_item(*args) periodically_run_tests[test_item.__name__] = datetime.now().isoformat() - periodically_run_tests_path.write_text(json.dumps(periodically_run_tests, indent=4)) + periodically_run_tests_path.write_text( + json.dumps(periodically_run_tests, indent=4) + ) except Exception as e: periodically_run_tests[test_item.__name__] = None - periodically_run_tests_path.write_text(json.dumps(periodically_run_tests, indent=4)) + periodically_run_tests_path.write_text( + json.dumps(periodically_run_tests, indent=4) + ) raise e return test_thing diff --git a/tests/test_imap_l3_data_processor.py b/tests/test_imap_l3_data_processor.py index 1e529e053..8793010f7 100644 --- a/tests/test_imap_l3_data_processor.py +++ b/tests/test_imap_l3_data_processor.py @@ -5,8 +5,12 @@ from unittest.mock import patch, call, Mock, sentinel from imap_data_access.file_validation import Version -from imap_data_access.processing_input import ScienceInput, ProcessingInputCollection, \ - AncillaryInput, SPICEInput +from imap_data_access.processing_input import ( + ScienceInput, + ProcessingInputCollection, + AncillaryInput, + SPICEInput, +) from imap_l3_data_processor import imap_l3_processor from imap_l3_processing.hi.hi_combined_initializer import HI_COMBINED_DESCRIPTORS @@ -14,28 +18,44 @@ from imap_l3_processing.lo.l3.lo_sp_initializer import LO_SP_MAP_DESCRIPTORS from imap_l3_processing.maps.map_initializer import PossibleMapToProduce from imap_l3_processing.models import InputMetadata, VersionMap -from imap_l3_processing.ultra.ultra_combined_nsp_initializer import ULTRA_COMBINED_NSP_DESCRIPTORS -from imap_l3_processing.ultra.ultra_sp_initializer import ULTRA_45_DESCRIPTORS, ULTRA_90_DESCRIPTORS, \ - ULTRA_COMBINED_SP_DESCRIPTORS +from imap_l3_processing.ultra.ultra_combined_nsp_initializer import ( + ULTRA_COMBINED_NSP_DESCRIPTORS, +) +from imap_l3_processing.ultra.ultra_sp_initializer import ( + ULTRA_45_DESCRIPTORS, + ULTRA_90_DESCRIPTORS, + ULTRA_COMBINED_SP_DESCRIPTORS, +) class TestImapL3DataProcessor(TestCase): - @patch('imap_l3_data_processor.ProcessingInputCollection') - @patch('imap_l3_data_processor.imap_data_access.upload') - @patch('imap_l3_data_processor.SwapiProcessor') - @patch('imap_l3_data_processor.SweProcessor') - @patch('imap_l3_data_processor.HitProcessor') - @patch('imap_l3_data_processor.HiProcessor') - @patch('imap_l3_data_processor.LoProcessor') - @patch('imap_l3_data_processor.UltraProcessor') - @patch('imap_l3_data_processor.GlowsProcessor') - @patch('imap_l3_data_processor.CodiceLoProcessor') - @patch('imap_l3_data_processor.CodiceHiProcessor') - @patch('imap_l3_data_processor.argparse') - def test_invokes_correct_processor(self, mock_argparse, mock_codice_hi, mock_codice_lo, mock_glows, - mock_ultra, mock_lo, mock_hi, mock_hit, mock_swe, - mock_swapi, mock_upload, - mock_processing_input): + @patch("imap_l3_data_processor.ProcessingInputCollection") + @patch("imap_l3_data_processor.imap_data_access.upload") + @patch("imap_l3_data_processor.SwapiProcessor") + @patch("imap_l3_data_processor.SweProcessor") + @patch("imap_l3_data_processor.HitProcessor") + @patch("imap_l3_data_processor.HiProcessor") + @patch("imap_l3_data_processor.LoProcessor") + @patch("imap_l3_data_processor.UltraProcessor") + @patch("imap_l3_data_processor.GlowsProcessor") + @patch("imap_l3_data_processor.CodiceLoProcessor") + @patch("imap_l3_data_processor.CodiceHiProcessor") + @patch("imap_l3_data_processor.argparse") + def test_invokes_correct_processor( + self, + mock_argparse, + mock_codice_hi, + mock_codice_lo, + mock_glows, + mock_ultra, + mock_lo, + mock_hi, + mock_hit, + mock_swe, + mock_swapi, + mock_upload, + mock_processing_input, + ): cases = [ ("swapi", "l3a", "proton", mock_swapi), ("swapi", "l3b", "combined", mock_swapi), @@ -56,7 +76,7 @@ def test_invokes_correct_processor(self, mock_argparse, mock_codice_hi, mock_cod expected_processor.reset_mock() mock_upload.reset_mock() mock_processing_input.reset_mock() - with self.subTest(f'{instrument}{data_level}'): + with self.subTest(f"{instrument}{data_level}"): mock_argument_parser = mock_argparse.ArgumentParser.return_value mock_argument_parser.parse_args.return_value.instrument = instrument mock_argument_parser.parse_args.return_value.data_level = data_level @@ -67,130 +87,242 @@ def test_invokes_correct_processor(self, mock_argparse, mock_codice_hi, mock_cod mock_argument_parser.parse_args.return_value.version = "v099" mock_argument_parser.parse_args.return_value.descriptor = descriptor - expected_input_metadata = InputMetadata(instrument, data_level, datetime(2025, 1, 1), - datetime(2025, 1, 1), - VersionMap({},Version(None, 99)), descriptor=descriptor, - repointing=22) + expected_input_metadata = InputMetadata( + instrument, + data_level, + datetime(2025, 1, 1), + datetime(2025, 1, 1), + VersionMap({}, Version(None, 99)), + descriptor=descriptor, + repointing=22, + ) expected_processor.return_value.process.return_value = [sentinel.cdf] imap_l3_processor() - mock_argument_parser.add_argument.assert_has_calls([ - call("--instrument"), - call("--data-level"), - call("--descriptor"), - call("--start-date"), - call("--end-date", required=False), - call("--repointing", required=False), - call("--version", required=False), - call("--dependency"), - call("--upload-to-sdc", action="store_true", required=False, - help="Upload completed output files to the IMAP SDC.") - ]) - - mock_processing_input.return_value.deserialize.assert_called_once_with("[]") - - expected_processor.assert_called_once_with(mock_processing_input.return_value, expected_input_metadata) + mock_argument_parser.add_argument.assert_has_calls( + [ + call("--instrument"), + call("--data-level"), + call("--descriptor"), + call("--start-date"), + call("--end-date", required=False), + call("--repointing", required=False), + call("--version", required=False), + call("--dependency"), + call( + "--upload-to-sdc", + action="store_true", + required=False, + help="Upload completed output files to the IMAP SDC.", + ), + ] + ) + + mock_processing_input.return_value.deserialize.assert_called_once_with( + "[]" + ) + + expected_processor.assert_called_once_with( + mock_processing_input.return_value, expected_input_metadata + ) mock_upload.assert_called_once_with(sentinel.cdf) - @patch('imap_l3_data_processor.HiCombinedInitializer') - @patch('imap_l3_data_processor.UltraCombinedNSPInitializer') - @patch('imap_l3_data_processor.UltraSPInitializer') - @patch('imap_l3_data_processor.UltraProcessor') - @patch('imap_l3_data_processor.HiSPInitializer') - @patch('imap_l3_data_processor.HiProcessor') - @patch('imap_l3_data_processor.LoSPInitializer') - @patch('imap_l3_data_processor.LoProcessor') - @patch('imap_l3_data_processor.imap_data_access.upload') - @patch('imap_l3_data_processor.argparse') - def test_scheduled_map_jobs_invoke_correct_initializer(self, mock_argparse, mock_upload, - mock_lo_processor_class, mock_lo_initializer_class, - mock_hi_processor_class, mock_hi_initializer_class, - mock_ultra_processor_class, mock_ultra_initializer_class, - mock_ultra_combined_nsp_initializer_class, - mock_hi_combined_initializer_class): + @patch("imap_l3_data_processor.HiCombinedInitializer") + @patch("imap_l3_data_processor.UltraCombinedNSPInitializer") + @patch("imap_l3_data_processor.UltraSPInitializer") + @patch("imap_l3_data_processor.UltraProcessor") + @patch("imap_l3_data_processor.HiSPInitializer") + @patch("imap_l3_data_processor.HiProcessor") + @patch("imap_l3_data_processor.LoSPInitializer") + @patch("imap_l3_data_processor.LoProcessor") + @patch("imap_l3_data_processor.imap_data_access.upload") + @patch("imap_l3_data_processor.argparse") + def test_scheduled_map_jobs_invoke_correct_initializer( + self, + mock_argparse, + mock_upload, + mock_lo_processor_class, + mock_lo_initializer_class, + mock_hi_processor_class, + mock_hi_initializer_class, + mock_ultra_processor_class, + mock_ultra_initializer_class, + mock_ultra_combined_nsp_initializer_class, + mock_hi_combined_initializer_class, + ): instrument_cases = [ - ("hi", "sp-maps", mock_hi_initializer_class, mock_hi_processor_class, HI_SP_MAP_DESCRIPTORS), - ("hi", "hic-maps", mock_hi_combined_initializer_class, mock_hi_processor_class, HI_COMBINED_DESCRIPTORS), - ("lo", "all-maps", mock_lo_initializer_class, mock_lo_processor_class, LO_SP_MAP_DESCRIPTORS), - ("ultra", "u45-maps", mock_ultra_initializer_class, mock_ultra_processor_class, ULTRA_45_DESCRIPTORS), - ("ultra", "u90-maps", mock_ultra_initializer_class, mock_ultra_processor_class, ULTRA_90_DESCRIPTORS), ( - "ultra", "ulc-sp-maps", mock_ultra_initializer_class, mock_ultra_processor_class, - ULTRA_COMBINED_SP_DESCRIPTORS), - ("ultra", "ulc-nsp-maps", mock_ultra_combined_nsp_initializer_class, mock_ultra_processor_class, - ULTRA_COMBINED_NSP_DESCRIPTORS), + "hi", + "sp-maps", + mock_hi_initializer_class, + mock_hi_processor_class, + HI_SP_MAP_DESCRIPTORS, + ), + ( + "hi", + "hic-maps", + mock_hi_combined_initializer_class, + mock_hi_processor_class, + HI_COMBINED_DESCRIPTORS, + ), + ( + "lo", + "all-maps", + mock_lo_initializer_class, + mock_lo_processor_class, + LO_SP_MAP_DESCRIPTORS, + ), + ( + "ultra", + "u45-maps", + mock_ultra_initializer_class, + mock_ultra_processor_class, + ULTRA_45_DESCRIPTORS, + ), + ( + "ultra", + "u90-maps", + mock_ultra_initializer_class, + mock_ultra_processor_class, + ULTRA_90_DESCRIPTORS, + ), + ( + "ultra", + "ulc-sp-maps", + mock_ultra_initializer_class, + mock_ultra_processor_class, + ULTRA_COMBINED_SP_DESCRIPTORS, + ), + ( + "ultra", + "ulc-nsp-maps", + mock_ultra_combined_nsp_initializer_class, + mock_ultra_processor_class, + ULTRA_COMBINED_NSP_DESCRIPTORS, + ), ] - for instrument, descriptor, mock_initializer_class, mock_processor_class, expected_descriptors in instrument_cases: - version_descriptors = {descriptor: { "major_version": 123,"minor_version": 0} for descriptor in expected_descriptors} + for ( + instrument, + descriptor, + mock_initializer_class, + mock_processor_class, + expected_descriptors, + ) in instrument_cases: + version_descriptors = { + descriptor: {"major_version": 123, "minor_version": 0} + for descriptor in expected_descriptors + } dependency_with_version_info = { "version": version_descriptors, - "dependency": [] + "dependency": [], } - dependency_with_version_info_as_string = json.dumps(dependency_with_version_info) + dependency_with_version_info_as_string = json.dumps( + dependency_with_version_info + ) dependency_cases = [ ("old style", "[]", None, VersionMap({descriptor: Version(None, 1)})), - ("new style", dependency_with_version_info_as_string, 123, VersionMap({descriptor: Version(123, 1)})), + ( + "new style", + dependency_with_version_info_as_string, + 123, + VersionMap({descriptor: Version(123, 1)}), + ), ] - for source_of_version_info, dependency_contents, expected_major_version, expected_version_map in dependency_cases: - + for ( + source_of_version_info, + dependency_contents, + expected_major_version, + expected_version_map, + ) in dependency_cases: mock_upload.reset_mock() mock_argparse.reset_mock() mock_initializer_class.reset_mock() mock_processor_class.reset_mock() - with self.subTest(instrument=instrument, descriptor=descriptor, dependency=source_of_version_info): + with self.subTest( + instrument=instrument, + descriptor=descriptor, + dependency=source_of_version_info, + ): data_level = "l3" mock_argument_parser = mock_argparse.ArgumentParser.return_value mock_argument_parser.parse_args.return_value.instrument = instrument mock_argument_parser.parse_args.return_value.data_level = data_level - mock_argument_parser.parse_args.return_value.dependency = dependency_contents + mock_argument_parser.parse_args.return_value.dependency = ( + dependency_contents + ) mock_argument_parser.parse_args.return_value.start_date = "20250101" mock_argument_parser.parse_args.return_value.end_date = None - mock_argument_parser.parse_args.return_value.repointing = "repoint00022" + mock_argument_parser.parse_args.return_value.repointing = ( + "repoint00022" + ) mock_argument_parser.parse_args.return_value.version = "v001" mock_argument_parser.parse_args.return_value.descriptor = descriptor - expected_input_metadata = InputMetadata(instrument, data_level, datetime(2025, 1, 1), - datetime(2025, 1, 1), - expected_version_map, descriptor=descriptor, - ) + expected_input_metadata = InputMetadata( + instrument, + data_level, + datetime(2025, 1, 1), + datetime(2025, 1, 1), + expected_version_map, + descriptor=descriptor, + ) mock_initializer = mock_initializer_class.return_value mock_processor = mock_processor_class.return_value - possible_map_to_produce = PossibleMapToProduce(set(), expected_input_metadata) - mock_initializer.get_maps_that_should_be_produced.return_value = [possible_map_to_produce] + possible_map_to_produce = PossibleMapToProduce( + set(), expected_input_metadata + ) + mock_initializer.get_maps_that_should_be_produced.return_value = [ + possible_map_to_produce + ] mock_processor.process.return_value = [sentinel.cdf] imap_l3_processor() - mock_initializer.get_maps_that_should_be_produced.assert_has_calls([ - call(descriptor, expected_major_version) for descriptor in expected_descriptors - ]) - - self.assertEqual(len(expected_descriptors), mock_processor_class.call_count) - - self.assertEqual(len(expected_descriptors), mock_initializer.furnish_spice_dependencies.call_count) - mock_initializer.furnish_spice_dependencies.assert_called_with(possible_map_to_produce) - - mock_processor_class.assert_called_with(possible_map_to_produce.processing_input_collection, - expected_input_metadata) - self.assertEqual(len(expected_descriptors), mock_processor.process.call_count) + mock_initializer.get_maps_that_should_be_produced.assert_has_calls( + [ + call(descriptor, expected_major_version) + for descriptor in expected_descriptors + ] + ) + + self.assertEqual( + len(expected_descriptors), mock_processor_class.call_count + ) + + self.assertEqual( + len(expected_descriptors), + mock_initializer.furnish_spice_dependencies.call_count, + ) + mock_initializer.furnish_spice_dependencies.assert_called_with( + possible_map_to_produce + ) + + mock_processor_class.assert_called_with( + possible_map_to_produce.processing_input_collection, + expected_input_metadata, + ) + self.assertEqual( + len(expected_descriptors), mock_processor.process.call_count + ) self.assertEqual(len(expected_descriptors), mock_upload.call_count) mock_upload.assert_called_with(sentinel.cdf) - @patch('imap_l3_data_processor.imap_data_access.upload') - @patch('imap_l3_data_processor.HiSPInitializer') - @patch('imap_l3_data_processor.HiProcessor') - @patch('imap_l3_data_processor.argparse') - def test_failing_to_produce_an_sp_map_continues(self, mock_argparse, mock_hi_processor, mock_hi_initializer_class, - mock_upload): + @patch("imap_l3_data_processor.imap_data_access.upload") + @patch("imap_l3_data_processor.HiSPInitializer") + @patch("imap_l3_data_processor.HiProcessor") + @patch("imap_l3_data_processor.argparse") + def test_failing_to_produce_an_sp_map_continues( + self, mock_argparse, mock_hi_processor, mock_hi_initializer_class, mock_upload + ): mock_hi_initializer = mock_hi_initializer_class.return_value instrument = "hi" @@ -207,13 +339,14 @@ def test_failing_to_produce_an_sp_map_continues(self, mock_argparse, mock_hi_pro mock_argument_parser.parse_args.return_value.descriptor = descriptor mock_hi_initializer.get_maps_that_should_be_produced.side_effect = [ - [PossibleMapToProduce(set(), Mock())] for _ in range(len(HI_SP_MAP_DESCRIPTORS)) + [PossibleMapToProduce(set(), Mock())] + for _ in range(len(HI_SP_MAP_DESCRIPTORS)) ] mock_hi_processor.return_value.process.side_effect = [ [Path("hi_sp_map_1.cdf")], ValueError("something went wrong!"), - [Path("hi_sp_map_2.cdf")] + [Path("hi_sp_map_2.cdf")], ] with self.assertLogs(): @@ -221,23 +354,34 @@ def test_failing_to_produce_an_sp_map_continues(self, mock_argparse, mock_hi_pro self.assertEqual(len(HI_SP_MAP_DESCRIPTORS), mock_hi_processor.call_count) - mock_upload.assert_has_calls([ - call(Path("hi_sp_map_1.cdf")), - call(Path("hi_sp_map_2.cdf")), - ]) - - @patch('imap_l3_data_processor.spiceypy') - @patch('imap_l3_data_processor.argparse') - @patch('imap_l3_data_processor.imap_data_access.download') - @patch('imap_l3_data_processor.ProcessingInputCollection') - @patch('imap_l3_data_processor.imap_data_access.upload') - @patch('imap_l3_data_processor.SwapiProcessor') - def test_get_spice_kernels_based_on_input_collection(self, _, __, mock_processing_input_collection, mock_download, - mock_arg_parser_class, mock_spicepy): + mock_upload.assert_has_calls( + [ + call(Path("hi_sp_map_1.cdf")), + call(Path("hi_sp_map_2.cdf")), + ] + ) + + @patch("imap_l3_data_processor.spiceypy") + @patch("imap_l3_data_processor.argparse") + @patch("imap_l3_data_processor.imap_data_access.download") + @patch("imap_l3_data_processor.ProcessingInputCollection") + @patch("imap_l3_data_processor.imap_data_access.upload") + @patch("imap_l3_data_processor.SwapiProcessor") + def test_get_spice_kernels_based_on_input_collection( + self, + _, + __, + mock_processing_input_collection, + mock_download, + mock_arg_parser_class, + mock_spicepy, + ): ancillary_input = AncillaryInput("imap_swe_ancillary_20250101_v112.cdf") spice_input_1 = SPICEInput("naif0012.tls") spice_input_2 = SPICEInput("imap_sclk_0012.tls") - imap_data_access_dependency = ProcessingInputCollection(spice_input_1, ancillary_input, spice_input_2) + imap_data_access_dependency = ProcessingInputCollection( + spice_input_1, ancillary_input, spice_input_2 + ) imap_data_access_dependency.deserialize = Mock() mock_processing_input_collection.return_value = imap_data_access_dependency @@ -252,30 +396,31 @@ def test_get_spice_kernels_based_on_input_collection(self, _, __, mock_processin mock_argument_parser.parse_args.return_value.dependency = "[]" mock_argument_parser.parse_args.return_value.repointing = None - mock_download.side_effect = [ - Path("naif0012.tls"), - Path("imap_sclk_0012.tls") - ] + mock_download.side_effect = [Path("naif0012.tls"), Path("imap_sclk_0012.tls")] imap_l3_processor() - expected_spice_paths = imap_data_access_dependency.get_file_paths(data_type='spice') + expected_spice_paths = imap_data_access_dependency.get_file_paths( + data_type="spice" + ) self.assertEqual(2, mock_download.call_count) - mock_download.assert_has_calls([ - call(expected_spice_paths[0]), - call(expected_spice_paths[1]) - ]) + mock_download.assert_has_calls( + [call(expected_spice_paths[0]), call(expected_spice_paths[1])] + ) mock_spicepy.furnsh.assert_has_calls( - [ - call("naif0012.tls"), - call("imap_sclk_0012.tls") - ]) - - @patch('imap_l3_data_processor.ProcessingInputCollection') - @patch('imap_l3_data_processor.imap_data_access.upload') - @patch('imap_l3_data_processor.SweProcessor') - @patch('imap_l3_data_processor.argparse') - def test_sets_end_date_to_start_date_if_not_specified(self, mock_argparse, mock_processor_class, mock_upload, - mock_processing_input_collection): + [call("naif0012.tls"), call("imap_sclk_0012.tls")] + ) + + @patch("imap_l3_data_processor.ProcessingInputCollection") + @patch("imap_l3_data_processor.imap_data_access.upload") + @patch("imap_l3_data_processor.SweProcessor") + @patch("imap_l3_data_processor.argparse") + def test_sets_end_date_to_start_date_if_not_specified( + self, + mock_argparse, + mock_processor_class, + mock_upload, + mock_processing_input_collection, + ): cases = [("20170630", datetime(2017, 6, 30)), (None, datetime(2016, 6, 30))] instrument_argument = "swe" @@ -283,10 +428,14 @@ def test_sets_end_date_to_start_date_if_not_specified(self, mock_argparse, mock_ start_date_argument = "20160630" version_argument = "v092" descriptor_argument = "pitch-angle" - science_input_1 = ScienceInput("imap_swe_l1_sci_20250101_v112.cdf", "imap_swe_l1_sci_20250102_v112.cdf") + science_input_1 = ScienceInput( + "imap_swe_l1_sci_20250101_v112.cdf", "imap_swe_l1_sci_20250102_v112.cdf" + ) science_input_2 = ScienceInput("imap_mag_l1d_norm-dsrf_20250101_v112.cdf") ancillary_input = AncillaryInput("imap_swe_ancillary_20250101_v112.cdf") - imap_data_access_dependency = ProcessingInputCollection(science_input_1, science_input_2, ancillary_input) + imap_data_access_dependency = ProcessingInputCollection( + science_input_1, science_input_2, ancillary_input + ) mock_processing_input_collection.return_value = imap_data_access_dependency mock_processing_input_collection.deserialize = Mock() @@ -296,7 +445,9 @@ def test_sets_end_date_to_start_date_if_not_specified(self, mock_argparse, mock_ mock_argument_parser.parse_args.return_value.instrument = instrument_argument mock_argument_parser.parse_args.return_value.data_level = data_level_argument - mock_argument_parser.parse_args.return_value.dependency = imap_data_access_dependency.serialize() + mock_argument_parser.parse_args.return_value.dependency = ( + imap_data_access_dependency.serialize() + ) mock_argument_parser.parse_args.return_value.start_date = start_date_argument mock_argument_parser.parse_args.return_value.version = version_argument mock_argument_parser.parse_args.return_value.descriptor = descriptor_argument @@ -315,33 +466,54 @@ def test_sets_end_date_to_start_date_if_not_specified(self, mock_argparse, mock_ imap_l3_processor() parser = mock_argparse.ArgumentParser() - parser.add_argument.assert_has_calls([ - call("--instrument"), - call("--data-level"), - call("--descriptor"), - call("--start-date"), - call("--end-date", required=False), - call("--repointing", required=False), - call("--version", required=False), - call("--dependency"), - call("--upload-to-sdc", action="store_true", required=False, - help="Upload completed output files to the IMAP SDC.") - ]) - - expected_input_metadata = InputMetadata("swe", "l3", datetime(year=2016, month=6, day=30), - expected_end_date, VersionMap({}, Version(None, 92)), "pitch-angle") - - mock_processor_class.assert_called_with(imap_data_access_dependency, expected_input_metadata) + parser.add_argument.assert_has_calls( + [ + call("--instrument"), + call("--data-level"), + call("--descriptor"), + call("--start-date"), + call("--end-date", required=False), + call("--repointing", required=False), + call("--version", required=False), + call("--dependency"), + call( + "--upload-to-sdc", + action="store_true", + required=False, + help="Upload completed output files to the IMAP SDC.", + ), + ] + ) + + expected_input_metadata = InputMetadata( + "swe", + "l3", + datetime(year=2016, month=6, day=30), + expected_end_date, + VersionMap({}, Version(None, 92)), + "pitch-angle", + ) + + mock_processor_class.assert_called_with( + imap_data_access_dependency, expected_input_metadata + ) mock_processor.process.assert_called() - mock_upload.assert_called_once_with(mock_processor_class.return_value.process.return_value[0]) - - @patch('imap_l3_data_processor.GlowsProcessor') - @patch('imap_l3_data_processor.argparse') - @patch('imap_l3_data_processor.ProcessingInputCollection') - @patch('imap_l3_data_processor.imap_data_access.upload') - def test_uploads_multiple_files(self, mock_upload, mock_processing_input_collection, mock_argparse, - mock_glows_processor): + mock_upload.assert_called_once_with( + mock_processor_class.return_value.process.return_value[0] + ) + + @patch("imap_l3_data_processor.GlowsProcessor") + @patch("imap_l3_data_processor.argparse") + @patch("imap_l3_data_processor.ProcessingInputCollection") + @patch("imap_l3_data_processor.imap_data_access.upload") + def test_uploads_multiple_files( + self, + mock_upload, + mock_processing_input_collection, + mock_argparse, + mock_glows_processor, + ): instrument_arg = "glows" start_date_arg = "20250101" version_arg = "v001" @@ -355,7 +527,9 @@ def test_uploads_multiple_files(self, mock_upload, mock_processing_input_collect mock_argument_parser = mock_argparse.ArgumentParser.return_value mock_argument_parser.parse_args.return_value.instrument = instrument_arg - mock_argument_parser.parse_args.return_value.dependency = processing_input_collection.serialize() + mock_argument_parser.parse_args.return_value.dependency = ( + processing_input_collection.serialize() + ) mock_argument_parser.parse_args.return_value.start_date = start_date_arg mock_argument_parser.parse_args.return_value.version = version_arg mock_argument_parser.parse_args.return_value.descriptor = descriptor_arg @@ -364,28 +538,37 @@ def test_uploads_multiple_files(self, mock_upload, mock_processing_input_collect mock_argument_parser.parse_args.return_value.data_level = "l3b" mock_argument_parser.parse_args.return_value.end_date = None - mock_glows_processor.return_value.process.return_value = [sentinel.one, sentinel.two, sentinel.three] + mock_glows_processor.return_value.process.return_value = [ + sentinel.one, + sentinel.two, + sentinel.three, + ] imap_l3_processor() - mock_upload.assert_has_calls([ - call(sentinel.one), - call(sentinel.two), - call(sentinel.three) - ]) - - @patch('imap_l3_data_processor.GlowsProcessor') - @patch('imap_l3_data_processor.argparse') - @patch('imap_l3_data_processor.ProcessingInputCollection') - @patch('imap_l3_data_processor.imap_data_access.upload') - def test_does_not_upload_files_if_upload_to_sdc_flag_is_set_to_false(self, mock_upload, - mock_processing_input_collection, - mock_argparse, - mock_glows_processor): - processing_input_collection = ProcessingInputCollection(ScienceInput("imap_glows_l3_science_20250101_v001.cdf")) + mock_upload.assert_has_calls( + [call(sentinel.one), call(sentinel.two), call(sentinel.three)] + ) + + @patch("imap_l3_data_processor.GlowsProcessor") + @patch("imap_l3_data_processor.argparse") + @patch("imap_l3_data_processor.ProcessingInputCollection") + @patch("imap_l3_data_processor.imap_data_access.upload") + def test_does_not_upload_files_if_upload_to_sdc_flag_is_set_to_false( + self, + mock_upload, + mock_processing_input_collection, + mock_argparse, + mock_glows_processor, + ): + processing_input_collection = ProcessingInputCollection( + ScienceInput("imap_glows_l3_science_20250101_v001.cdf") + ) mock_argument_parser = mock_argparse.ArgumentParser.return_value mock_argument_parser.parse_args.return_value.instrument = "glows" - mock_argument_parser.parse_args.return_value.dependency = processing_input_collection.serialize() + mock_argument_parser.parse_args.return_value.dependency = ( + processing_input_collection.serialize() + ) mock_argument_parser.parse_args.return_value.start_date = "20250101" mock_argument_parser.parse_args.return_value.version = "v001" mock_argument_parser.parse_args.return_value.descriptor = "desc" @@ -405,18 +588,25 @@ def test_does_not_upload_files_if_upload_to_sdc_flag_is_set_to_false(self, mock_ mock_glows_processor.return_value.process.assert_called() mock_upload.assert_not_called() - @patch('imap_l3_data_processor.GlowsProcessor') - @patch('imap_l3_data_processor.argparse') - @patch('imap_l3_data_processor.ProcessingInputCollection') - @patch('imap_l3_data_processor.imap_data_access.upload') - def test_upload_fails_tries_all_uploads_and_raises_exception(self, mock_upload, - mock_processing_input_collection, - mock_argparse, - mock_glows_processor): - processing_input_collection = ProcessingInputCollection(ScienceInput("imap_glows_l3_science_20250101_v001.cdf")) + @patch("imap_l3_data_processor.GlowsProcessor") + @patch("imap_l3_data_processor.argparse") + @patch("imap_l3_data_processor.ProcessingInputCollection") + @patch("imap_l3_data_processor.imap_data_access.upload") + def test_upload_fails_tries_all_uploads_and_raises_exception( + self, + mock_upload, + mock_processing_input_collection, + mock_argparse, + mock_glows_processor, + ): + processing_input_collection = ProcessingInputCollection( + ScienceInput("imap_glows_l3_science_20250101_v001.cdf") + ) mock_argument_parser = mock_argparse.ArgumentParser.return_value mock_argument_parser.parse_args.return_value.instrument = "glows" - mock_argument_parser.parse_args.return_value.dependency = processing_input_collection.serialize() + mock_argument_parser.parse_args.return_value.dependency = ( + processing_input_collection.serialize() + ) mock_argument_parser.parse_args.return_value.start_date = "20250101" mock_argument_parser.parse_args.return_value.version = "v001" mock_argument_parser.parse_args.return_value.descriptor = "desc" @@ -429,8 +619,12 @@ def test_upload_fails_tries_all_uploads_and_raises_exception(self, mock_upload, mock_processing_input_collection.deserialize = Mock() mock_processing_input_collection.get_science_inputs = Mock(return_value=[]) - mock_glows_processor.return_value.process.return_value = ["data_file_1.cdf", "data_file_2.cdf", - "data_file_3.cdf", "data_file_4.cdf"] + mock_glows_processor.return_value.process.return_value = [ + "data_file_1.cdf", + "data_file_2.cdf", + "data_file_3.cdf", + "data_file_4.cdf", + ] error1 = ValueError("Failure uploading!") error2 = ValueError("Failure uploading 2") @@ -439,20 +633,27 @@ def test_upload_fails_tries_all_uploads_and_raises_exception(self, mock_upload, with self.assertRaises(IOError) as exception_ctx: imap_l3_processor() - self.assertEqual((f"Failed to upload some files: {[error1, error2]}",), exception_ctx.exception.args) + self.assertEqual( + (f"Failed to upload some files: {[error1, error2]}",), + exception_ctx.exception.args, + ) mock_glows_processor.return_value.process.assert_called() - mock_upload.assert_has_calls([ - call("data_file_1.cdf"), - call("data_file_2.cdf"), - call("data_file_3.cdf"), - call("data_file_4.cdf"), - ]) + mock_upload.assert_has_calls( + [ + call("data_file_1.cdf"), + call("data_file_2.cdf"), + call("data_file_3.cdf"), + call("data_file_4.cdf"), + ] + ) - @patch('imap_l3_data_processor.argparse') - @patch('imap_l3_data_processor.imap_data_access.upload') - def test_throws_exception_for_unimplemented_instrument(self, mock_upload, mock_argparse): + @patch("imap_l3_data_processor.argparse") + @patch("imap_l3_data_processor.imap_data_access.upload") + def test_throws_exception_for_unimplemented_instrument( + self, mock_upload, mock_argparse + ): instrument_argument = "new_instrument" data_level_argument = "l3a" start_date_argument = "20160630" @@ -465,7 +666,9 @@ def test_throws_exception_for_unimplemented_instrument(self, mock_upload, mock_a mock_argument_parser.parse_args.return_value.instrument = instrument_argument mock_argument_parser.parse_args.return_value.data_level = data_level_argument - mock_argument_parser.parse_args.return_value.dependency = imap_data_access_dependency.serialize() + mock_argument_parser.parse_args.return_value.dependency = ( + imap_data_access_dependency.serialize() + ) mock_argument_parser.parse_args.return_value.start_date = start_date_argument mock_argument_parser.parse_args.return_value.end_date = end_date_argument mock_argument_parser.parse_args.return_value.version = version_argument @@ -473,13 +676,17 @@ def test_throws_exception_for_unimplemented_instrument(self, mock_upload, mock_a with self.assertRaises(NotImplementedError) as exception_manager: imap_l3_processor() - self.assertEqual(str(exception_manager.exception), - "Level l3a data processing has not yet been implemented for new_instrument") + self.assertEqual( + str(exception_manager.exception), + "Level l3a data processing has not yet been implemented for new_instrument", + ) mock_upload.assert_not_called() - @patch('imap_l3_data_processor.argparse') - @patch('imap_l3_data_processor.imap_data_access.upload') - def test_throws_exception_for_codice_descriptor_not_matching_hi_or_lo(self, mock_upload, mock_argparse): + @patch("imap_l3_data_processor.argparse") + @patch("imap_l3_data_processor.imap_data_access.upload") + def test_throws_exception_for_codice_descriptor_not_matching_hi_or_lo( + self, mock_upload, mock_argparse + ): instrument_argument = "codice" data_level_argument = "l3a" start_date_argument = "20160630" @@ -493,7 +700,9 @@ def test_throws_exception_for_codice_descriptor_not_matching_hi_or_lo(self, mock mock_argument_parser.parse_args.return_value.instrument = instrument_argument mock_argument_parser.parse_args.return_value.data_level = data_level_argument - mock_argument_parser.parse_args.return_value.dependency = imap_data_access_dependency.serialize() + mock_argument_parser.parse_args.return_value.dependency = ( + imap_data_access_dependency.serialize() + ) mock_argument_parser.parse_args.return_value.start_date = start_date_argument mock_argument_parser.parse_args.return_value.end_date = end_date_argument mock_argument_parser.parse_args.return_value.version = version_argument @@ -502,13 +711,17 @@ def test_throws_exception_for_codice_descriptor_not_matching_hi_or_lo(self, mock with self.assertRaises(NotImplementedError) as exception_manager: imap_l3_processor() - self.assertEqual(str(exception_manager.exception), - "Unknown descriptor 'bad' for codice instrument") + self.assertEqual( + str(exception_manager.exception), + "Unknown descriptor 'bad' for codice instrument", + ) mock_upload.assert_not_called() - @patch('imap_l3_data_processor.argparse') - @patch('imap_l3_data_processor.imap_data_access.upload') - def test_throws_exception_when_attempting_to_process_non_l3_data_levels(self, mock_upload, mock_argparse): + @patch("imap_l3_data_processor.argparse") + @patch("imap_l3_data_processor.imap_data_access.upload") + def test_throws_exception_when_attempting_to_process_non_l3_data_levels( + self, mock_upload, mock_argparse + ): instrument_argument = "swapi" data_level_argument = "l4" start_date_argument = "20160630" @@ -521,7 +734,9 @@ def test_throws_exception_when_attempting_to_process_non_l3_data_levels(self, mo mock_argument_parser.parse_args.return_value.instrument = instrument_argument mock_argument_parser.parse_args.return_value.data_level = data_level_argument - mock_argument_parser.parse_args.return_value.dependency = imap_data_access_dependency.serialize() + mock_argument_parser.parse_args.return_value.dependency = ( + imap_data_access_dependency.serialize() + ) mock_argument_parser.parse_args.return_value.start_date = start_date_argument mock_argument_parser.parse_args.return_value.end_date = end_date_argument mock_argument_parser.parse_args.return_value.version = version_argument @@ -529,32 +744,43 @@ def test_throws_exception_when_attempting_to_process_non_l3_data_levels(self, mo with self.assertRaises(NotImplementedError) as exception_manager: imap_l3_processor() - self.assertEqual(str(exception_manager.exception), - "Level l4 data processing has not yet been implemented for swapi") + self.assertEqual( + str(exception_manager.exception), + "Level l4 data processing has not yet been implemented for swapi", + ) mock_upload.assert_not_called() - @patch('imap_l3_data_processor.ProcessingInputCollection') - @patch('imap_l3_data_processor.imap_data_access.upload') - @patch('imap_l3_data_processor.SweProcessor') - @patch('imap_l3_data_processor.argparse') - def test_version_comes_from_dependency(self, mock_argparse, mock_processor_class, mock_upload, - mock_processing_input_collection): + @patch("imap_l3_data_processor.ProcessingInputCollection") + @patch("imap_l3_data_processor.imap_data_access.upload") + @patch("imap_l3_data_processor.SweProcessor") + @patch("imap_l3_data_processor.argparse") + def test_version_comes_from_dependency( + self, + mock_argparse, + mock_processor_class, + mock_upload, + mock_processing_input_collection, + ): instrument_argument = "swe" data_level_argument = "l3" start_date_argument = "20160630" version_argument = "v092" descriptor_argument = "pitch-angle" - science_input_1 = ScienceInput("imap_swe_l1_sci_20250101_v112.cdf", "imap_swe_l1_sci_20250102_v112.cdf") + science_input_1 = ScienceInput( + "imap_swe_l1_sci_20250101_v112.cdf", "imap_swe_l1_sci_20250102_v112.cdf" + ) science_input_2 = ScienceInput("imap_mag_l1d_norm-dsrf_20250101_v112.cdf") ancillary_input = AncillaryInput("imap_swe_ancillary_20250101_v112.cdf") - imap_data_access_dependency = ProcessingInputCollection(science_input_1, science_input_2, ancillary_input) + imap_data_access_dependency = ProcessingInputCollection( + science_input_1, science_input_2, ancillary_input + ) serialized_dependency_paths = imap_data_access_dependency.serialize() dependency_without_version_info = json.loads(serialized_dependency_paths) dependency_with_version_info = { "version": { "sci": {"major_version": 2, "minor_version": 1}, - "var": {"major_version": 3, "minor_version": 2} + "var": {"major_version": 3, "minor_version": 2}, }, "dependency": dependency_without_version_info, } @@ -573,12 +799,24 @@ def test_version_comes_from_dependency(self, mock_argparse, mock_processor_class mock_argument_parser.parse_args.return_value.repointing = None mock_processor = mock_processor_class.return_value - cases = [("version in dependency", dependency_with_version_info, VersionMap({'sci':Version(2,1), 'var':Version(3,2)})), - ("version on command line", dependency_without_version_info, VersionMap({}, Version(None,92)))] + cases = [ + ( + "version in dependency", + dependency_with_version_info, + VersionMap({"sci": Version(2, 1), "var": Version(3, 2)}), + ), + ( + "version on command line", + dependency_without_version_info, + VersionMap({}, Version(None, 92)), + ), + ] input_end_date, expected_end_date = "20170630", datetime(2017, 6, 30) for name, dependency, expected_version_map in cases: with self.subTest(name): - mock_argument_parser.parse_args.return_value.dependency = json.dumps(dependency) + mock_argument_parser.parse_args.return_value.dependency = json.dumps( + dependency + ) mock_upload.reset_mock() imap_data_access_dependency.deserialize.reset_mock() @@ -589,24 +827,42 @@ def test_version_comes_from_dependency(self, mock_argparse, mock_processor_class imap_l3_processor() parser = mock_argparse.ArgumentParser() - parser.add_argument.assert_has_calls([ - call("--instrument"), - call("--data-level"), - call("--descriptor"), - call("--start-date"), - call("--end-date", required=False), - call("--repointing", required=False), - call("--version", required=False), - call("--dependency"), - call("--upload-to-sdc", action="store_true", required=False, - help="Upload completed output files to the IMAP SDC.") - ]) - - expected_input_metadata = InputMetadata("swe", "l3", datetime(year=2016, month=6, day=30), - expected_end_date, expected_version_map, "pitch-angle") - - mock_processor_class.assert_called_with(imap_data_access_dependency, expected_input_metadata) - - imap_data_access_dependency.deserialize.assert_called_once_with(serialized_dependency_paths) + parser.add_argument.assert_has_calls( + [ + call("--instrument"), + call("--data-level"), + call("--descriptor"), + call("--start-date"), + call("--end-date", required=False), + call("--repointing", required=False), + call("--version", required=False), + call("--dependency"), + call( + "--upload-to-sdc", + action="store_true", + required=False, + help="Upload completed output files to the IMAP SDC.", + ), + ] + ) + + expected_input_metadata = InputMetadata( + "swe", + "l3", + datetime(year=2016, month=6, day=30), + expected_end_date, + expected_version_map, + "pitch-angle", + ) + + mock_processor_class.assert_called_with( + imap_data_access_dependency, expected_input_metadata + ) + + imap_data_access_dependency.deserialize.assert_called_once_with( + serialized_dependency_paths + ) mock_processor.process.assert_called() - mock_upload.assert_called_once_with(mock_processor_class.return_value.process.return_value[0]) + mock_upload.assert_called_once_with( + mock_processor_class.return_value.process.return_value[0] + ) diff --git a/tests/test_models.py b/tests/test_models.py index 216687320..1f91bf58c 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -8,24 +8,49 @@ class TestModels(unittest.TestCase): - def test_calculate_average_mag_data_with_extra_data_at_beginning_and_end(self): - hit_data_epoch = np.array([datetime(2020, 4, 4, 0, 5), datetime(2020, 4, 4, 0, 15)]) + hit_data_epoch = np.array( + [datetime(2020, 4, 4, 0, 5), datetime(2020, 4, 4, 0, 15)] + ) hit_data_delta = np.array([timedelta(seconds=300), timedelta(seconds=300)]) extra_data_at_beginning = [[0.5, 0, 1], [0, 0.5, 1]] - extra_dates_at_beginning = [datetime(2020, 4, 3, 0, 23), datetime(2020, 4, 3, 0, 23, 30)] - extra_dates_at_end = [datetime(2020, 4, 4, 1, 0), datetime(2020, 4, 4, 1, 23, 30)] + extra_dates_at_beginning = [ + datetime(2020, 4, 3, 0, 23), + datetime(2020, 4, 3, 0, 23, 30), + ] + extra_dates_at_end = [ + datetime(2020, 4, 4, 1, 0), + datetime(2020, 4, 4, 1, 23, 30), + ] extra_data_at_end = [[0.5, 1, 1], [1, 0.5, 0.5]] - mag_data = np.array(extra_data_at_beginning + - [[0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 1], [0, 1, 0], [1, 0, 0], - [1, 0, 0]] + extra_data_at_end) - mag_epoch = np.array(extra_dates_at_beginning + [datetime(2020, 4, 4, 0, 1), - datetime(2020, 4, 4, 0, 3), datetime(2020, 4, 4, 0, 6), - datetime(2020, 4, 4, 0, 9, 55), - datetime(2020, 4, 4, 0, 12, 45), - datetime(2020, 4, 4, 0, 13), - datetime(2020, 4, 4, 0, 15), - datetime(2020, 4, 4, 0, 17)] + extra_dates_at_end) + mag_data = np.array( + extra_data_at_beginning + + [ + [0, 0, 1], + [0, 1, 0], + [1, 0, 0], + [0, 1, 0], + [0, 0, 1], + [0, 1, 0], + [1, 0, 0], + [1, 0, 0], + ] + + extra_data_at_end + ) + mag_epoch = np.array( + extra_dates_at_beginning + + [ + datetime(2020, 4, 4, 0, 1), + datetime(2020, 4, 4, 0, 3), + datetime(2020, 4, 4, 0, 6), + datetime(2020, 4, 4, 0, 9, 55), + datetime(2020, 4, 4, 0, 12, 45), + datetime(2020, 4, 4, 0, 13), + datetime(2020, 4, 4, 0, 15), + datetime(2020, 4, 4, 0, 17), + ] + + extra_dates_at_end + ) expected_average = np.array([[1 / 4, 1 / 2, 1 / 4], [1 / 2, 1 / 4, 1 / 4]]) mag_data_model = MagData(mag_epoch, mag_data) @@ -34,20 +59,31 @@ def test_calculate_average_mag_data_with_extra_data_at_beginning_and_end(self): np.testing.assert_array_equal(actual_average, expected_average) def test_calculate_average_mag_data_handles_missing_mag_data(self): - hit_data_epoch = np.array([datetime(2020, 4, 4, 0, 5), - datetime(2020, 4, 4, 0, 15), - datetime(2020, 4, 4, 0, 25)]) - hit_data_delta = np.array([timedelta(seconds=300), timedelta(seconds=300), timedelta(seconds=300)]) + hit_data_epoch = np.array( + [ + datetime(2020, 4, 4, 0, 5), + datetime(2020, 4, 4, 0, 15), + datetime(2020, 4, 4, 0, 25), + ] + ) + hit_data_delta = np.array( + [timedelta(seconds=300), timedelta(seconds=300), timedelta(seconds=300)] + ) mag_data = np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 0]]) - mag_epoch = [datetime(2020, 4, 4, 0, 1), - datetime(2020, 4, 4, 0, 3), - datetime(2020, 4, 4, 0, 22), - datetime(2020, 4, 4, 0, 27)] - expected_average = np.array([[0, 1 / 2, 1 / 2], [np.nan, np.nan, np.nan], [1 / 2, 1 / 2, 0]]) + mag_epoch = [ + datetime(2020, 4, 4, 0, 1), + datetime(2020, 4, 4, 0, 3), + datetime(2020, 4, 4, 0, 22), + datetime(2020, 4, 4, 0, 27), + ] + expected_average = np.array( + [[0, 1 / 2, 1 / 2], [np.nan, np.nan, np.nan], [1 / 2, 1 / 2, 0]] + ) mag_data_model = MagData(mag_epoch, mag_data) - actual_average = mag_data_model.rebin_to(epoch=hit_data_epoch, - epoch_delta=hit_data_delta) + actual_average = mag_data_model.rebin_to( + epoch=hit_data_epoch, epoch_delta=hit_data_delta + ) np.testing.assert_array_equal(actual_average, expected_average) def test_version_map(self): @@ -70,15 +106,12 @@ def test_version_map(self): def test_version_map_throws_error_if_no_fallback(self): version_map = VersionMap( - { - "sci": Version(1, 5) - }, + {"sci": Version(1, 5)}, ) with self.assertRaises(KeyError): version_map.lookup("other") - def test_version_map_equality(self): version_map = VersionMap( { @@ -111,5 +144,6 @@ def test_version_map_equality(self): self.assertNotEqual(version_map, different_map) self.assertNotEqual(version_map, different_fallback) -if __name__ == '__main__': + +if __name__ == "__main__": unittest.main() diff --git a/tests/test_pitch_angles.py b/tests/test_pitch_angles.py index bd2ebe957..aefbc0166 100644 --- a/tests/test_pitch_angles.py +++ b/tests/test_pitch_angles.py @@ -2,13 +2,17 @@ import numpy as np -from imap_l3_processing.pitch_angles import calculate_pitch_angle, calculate_unit_vector, calculate_gyrophase, \ - rotate_from_imap_despun_to_hit_despun, rotate_particle_vectors_from_hit_despun_to_imap_despun, \ - rebin_by_pitch_angle_and_gyrophase +from imap_l3_processing.pitch_angles import ( + calculate_pitch_angle, + calculate_unit_vector, + calculate_gyrophase, + rotate_from_imap_despun_to_hit_despun, + rotate_particle_vectors_from_hit_despun_to_imap_despun, + rebin_by_pitch_angle_and_gyrophase, +) class TestPitchAngles(unittest.TestCase): - def test_calculate_pitch_angle(self): hit_unit_vector = np.array([[-0.09362045, 0.8466484, 0.5238528]]) mag_unit_vector = np.array([-0.42566603, 0.7890057, 0.44303328]) @@ -36,11 +40,15 @@ def test_calculate_unit_vector(self): unit_vector = calculate_unit_vector(vector) - np.testing.assert_array_almost_equal(np.array([0.38103832, 0.47982603, 0.7903017]), unit_vector) + np.testing.assert_array_almost_equal( + np.array([0.38103832, 0.47982603, 0.7903017]), unit_vector + ) many_vectors = np.array([[0.1, 0, 0], [30, 40, 0]]) unit_vectors = calculate_unit_vector(many_vectors) - np.testing.assert_array_almost_equal([[1, 0, 0], [3 / 5, 4 / 5, 0]], unit_vectors) + np.testing.assert_array_almost_equal( + [[1, 0, 0], [3 / 5, 4 / 5, 0]], unit_vectors + ) def test_rotate_from_imap_despun_to_hit_despun(self): vector = np.array([1.0, 1.0, 1.0]) @@ -52,35 +60,46 @@ def test_rotate_from_imap_despun_to_hit_despun(self): np.testing.assert_array_almost_equal(rotated_vector, expected_vector) def test_rotate_particle_vectors_from_hit_despun_to_imap_despun(self): - particle_vectors = np.array([ - [[0, 0, 1], [1, 0, 0]], - [[1, 1, 0], [0, 1, 1]], - [[0, 1, 0], [1, 0, 1]], - [[0, 0, 0], [1, 1, 1]] - ], dtype=float) + particle_vectors = np.array( + [ + [[0, 0, 1], [1, 0, 0]], + [[1, 1, 0], [0, 1, 1]], + [[0, 1, 0], [1, 0, 1]], + [[0, 0, 0], [1, 1, 1]], + ], + dtype=float, + ) - rotated_particle_vectors = rotate_particle_vectors_from_hit_despun_to_imap_despun(particle_vectors) + rotated_particle_vectors = ( + rotate_particle_vectors_from_hit_despun_to_imap_despun(particle_vectors) + ) - expected_particle_vectors = np.array([ - [[0, 0, 1], [0.866025, 0.5, 0.]], - [[0.366025, 1.366025, 0.], [-0.5, 0.866025, 1.]], - [[- 0.5, 0.866025, 0.], [0.866025, 0.5, 1.]], - [[0, 0, 0], [0.366025, 1.366025, 1.]] - ]) + expected_particle_vectors = np.array( + [ + [[0, 0, 1], [0.866025, 0.5, 0.0]], + [[0.366025, 1.366025, 0.0], [-0.5, 0.866025, 1.0]], + [[-0.5, 0.866025, 0.0], [0.866025, 0.5, 1.0]], + [[0, 0, 0], [0.366025, 1.366025, 1.0]], + ] + ) - np.testing.assert_array_almost_equal(rotated_particle_vectors, expected_particle_vectors) + np.testing.assert_array_almost_equal( + rotated_particle_vectors, expected_particle_vectors + ) def test_calculate_gyrophase(self): - particle_vector = np.array([ - [0, 1, 0], - [1, 0, 0], - [0, -1, 0], - [-1, 0, 0], - [.5, .5, 0], - [.5, -.5, 0], - [0, 0, -1], - [0, 0, 1], - ]) + particle_vector = np.array( + [ + [0, 1, 0], + [1, 0, 0], + [0, -1, 0], + [-1, 0, 0], + [0.5, 0.5, 0], + [0.5, -0.5, 0], + [0, 0, -1], + [0, 0, 1], + ] + ) mag_field_vector = np.array([0, 0, 1]) gyrophases = calculate_gyrophase(particle_vector, mag_field_vector) @@ -90,20 +109,17 @@ def test_calculate_gyrophase(self): np.testing.assert_array_equal(gyrophases, expected_gyrophases) def test_calculate_gyrophase_fany_input_arrays(self): - particle_vector = np.array([ + particle_vector = np.array( [ - [0, 1, 0], - [1, 0, 0], - [0, -1, 0], - [-1, 0, 0] - ], - [ - [.5, .5, 0], - [.5, -.5, 0], - [0, 0, -1], - [0, 0, 1], + [[0, 1, 0], [1, 0, 0], [0, -1, 0], [-1, 0, 0]], + [ + [0.5, 0.5, 0], + [0.5, -0.5, 0], + [0, 0, -1], + [0, 0, 1], + ], ] - ]) + ) mag_field_vector = np.array([[0, 0, 1], [0, 0, 1]])[..., np.newaxis, :] gyrophases = calculate_gyrophase(particle_vector, mag_field_vector) @@ -113,9 +129,11 @@ def test_calculate_gyrophase_fany_input_arrays(self): np.testing.assert_array_equal(gyrophases, expected_gyrophases) def test_gyrophase_degenerate_particle_and_magnetic_field_returns_zero(self): - particle_vector = np.array([ - [0, 0, 1], - ]) + particle_vector = np.array( + [ + [0, 0, 1], + ] + ) mag_field_vector = np.array([0, 0, 1]) gyrophases = calculate_gyrophase(particle_vector, mag_field_vector) @@ -124,11 +142,17 @@ def test_gyrophase_degenerate_particle_and_magnetic_field_returns_zero(self): np.testing.assert_array_equal(gyrophases, expected_gyrophases) - def test_gyrophase_degenerate_dps_frame_y_axis_and_magnetic_field_returns_all_nans(self): - particle_vector = np.array([[ - [1, 0, 0], - [-1, 0, 0], - ]]) + def test_gyrophase_degenerate_dps_frame_y_axis_and_magnetic_field_returns_all_nans( + self, + ): + particle_vector = np.array( + [ + [ + [1, 0, 0], + [-1, 0, 0], + ] + ] + ) for mag_field_vector in [np.array([0, 1, 0]), np.array([0, -1, 0])]: with self.subTest(mag_field_vector): @@ -142,93 +166,167 @@ def test_rebin_intensity_by_pitch_angle_and_gyrophase(self): default_gyrophases = np.array([[45, 135, 225, 315], [45, 135, 225, 315]]) intensity_data_for_energy_1 = [[0, 1, 2, 3], [4, 5, 6, 7]] intensity_data_for_energy_2 = [[8, 9, 10, 11], [12, 13, 14, 15]] - intensity = np.array([ - intensity_data_for_energy_1, - intensity_data_for_energy_2 - ]) + intensity = np.array([intensity_data_for_energy_1, intensity_data_for_energy_2]) intensity_delta_plus = intensity * 0.09 intensity_delta_minus = intensity * 0.11 test_cases = [ - ('One bin', default_pitch_angles, default_gyrophases, 1, 1, [[[3.5]], [[11.5]]], [[3.5], [11.5]]), - ('Rebinned to same bins', default_pitch_angles, default_gyrophases, 2, 4, intensity, - np.array([[1.5, 5.5], [9.5, 13.5]])), - ('Gyrophase rotated by 90', default_pitch_angles, (default_gyrophases + 90) % 360, 2, 4, - [ - [[3, 0, 1, 2], [7, 4, 5, 6]], - [[11, 8, 9, 10], [15, 12, 13, 14]] - ], - np.array([[1.5, 5.5], [9.5, 13.5]])), - ('Includes empty bins', default_pitch_angles, default_gyrophases, 2, 6, - [ - [[0, np.nan, 1, 2, np.nan, 3], [4, np.nan, 5, 6, np.nan, 7]], - [[8, np.nan, 9, 10, np.nan, 11], [12, np.nan, 13, 14, np.nan, 15]] - ], - np.array([[1.5, 5.5], [9.5, 13.5]])), - ('Varying Pitch Angles', np.array([[45, 45, 135, 135], [45, 45, 135, 135]]), - np.array([[135, 225, 225, 135], [45, 315, 315, 45]]), 2, 4, - [ - [[4, 0, 1, 5], [7, 3, 2, 6]], - [[12, 8, 9, 13], [15, 11, 10, 14]] - ], - [[2.5, 4.5], [10.5, 12.5]]), - ('Nan in pitch angles', np.array([[45, 45, 135, 135], [np.nan, 45, 135, 135]]), - np.array([[135, 225, 225, 135], [45, 315, 315, 45]]), 2, 4, - [ - [[np.nan, 0, 1, 5], [7, 3, 2, 6]], - [[np.nan, 8, 9, 13], [15, 11, 10, 14]] - ], - [[2, 4.5], [10, 12.5]]), ( - 'Nan for all pitch angles in a bin', np.array([[np.nan, np.nan, 135, 135], [np.nan, np.nan, 135, 135]]), - np.array([[135, 225, 225, 135], [45, 315, 315, 45]]), 2, 4, + "One bin", + default_pitch_angles, + default_gyrophases, + 1, + 1, + [[[3.5]], [[11.5]]], + [[3.5], [11.5]], + ), + ( + "Rebinned to same bins", + default_pitch_angles, + default_gyrophases, + 2, + 4, + intensity, + np.array([[1.5, 5.5], [9.5, 13.5]]), + ), + ( + "Gyrophase rotated by 90", + default_pitch_angles, + (default_gyrophases + 90) % 360, + 2, + 4, + [[[3, 0, 1, 2], [7, 4, 5, 6]], [[11, 8, 9, 10], [15, 12, 13, 14]]], + np.array([[1.5, 5.5], [9.5, 13.5]]), + ), + ( + "Includes empty bins", + default_pitch_angles, + default_gyrophases, + 2, + 6, + [ + [[0, np.nan, 1, 2, np.nan, 3], [4, np.nan, 5, 6, np.nan, 7]], + [[8, np.nan, 9, 10, np.nan, 11], [12, np.nan, 13, 14, np.nan, 15]], + ], + np.array([[1.5, 5.5], [9.5, 13.5]]), + ), + ( + "Varying Pitch Angles", + np.array([[45, 45, 135, 135], [45, 45, 135, 135]]), + np.array([[135, 225, 225, 135], [45, 315, 315, 45]]), + 2, + 4, + [[[4, 0, 1, 5], [7, 3, 2, 6]], [[12, 8, 9, 13], [15, 11, 10, 14]]], + [[2.5, 4.5], [10.5, 12.5]], + ), + ( + "Nan in pitch angles", + np.array([[45, 45, 135, 135], [np.nan, 45, 135, 135]]), + np.array([[135, 225, 225, 135], [45, 315, 315, 45]]), + 2, + 4, + [ + [[np.nan, 0, 1, 5], [7, 3, 2, 6]], + [[np.nan, 8, 9, 13], [15, 11, 10, 14]], + ], + [[2, 4.5], [10, 12.5]], + ), + ( + "Nan for all pitch angles in a bin", + np.array([[np.nan, np.nan, 135, 135], [np.nan, np.nan, 135, 135]]), + np.array([[135, 225, 225, 135], [45, 315, 315, 45]]), + 2, + 4, [ [[np.nan, np.nan, np.nan, np.nan], [7, 3, 2, 6]], - [[np.nan, np.nan, np.nan, np.nan], [15, 11, 10, 14]] + [[np.nan, np.nan, np.nan, np.nan], [15, 11, 10, 14]], + ], + [[np.nan, 4.5], [np.nan, 12.5]], + ), + ( + "Nan in gyrophase", + np.array([[45, 45, 135, 135], [45, 45, 135, 135]]), + np.array([[np.nan, 225, 225, 135], [45, 315, 315, 45]]), + 2, + 4, + [ + [[4, np.nan, 1, 5], [7, 3, 2, 6]], + [[12, np.nan, 9, 13], [15, 11, 10, 14]], ], - [[np.nan, 4.5], [np.nan, 12.5]]), - ('Nan in gyrophase', np.array([[45, 45, 135, 135], [45, 45, 135, 135]]), - np.array([[np.nan, 225, 225, 135], [45, 315, 315, 45]]), 2, 4, - [ - [[4, np.nan, 1, 5], [7, 3, 2, 6]], - [[12, np.nan, 9, 13], [15, 11, 10, 14]] - ], - [[2.5, 4.5], [10.5, 12.5]]), - - ('Gyrophase bin includes 360 bin', default_pitch_angles, - np.array([[360, 90, 180, 270], [360, 90, 180, 270]]), - 2, 4, intensity, - np.array([[1.5, 5.5], [9.5, 13.5]])) + [[2.5, 4.5], [10.5, 12.5]], + ), + ( + "Gyrophase bin includes 360 bin", + default_pitch_angles, + np.array([[360, 90, 180, 270], [360, 90, 180, 270]]), + 2, + 4, + intensity, + np.array([[1.5, 5.5], [9.5, 13.5]]), + ), ] - for case, pitch_angles, gyrophases, number_of_pitch_angle_bins, number_of_gyrophase_bins, \ - expected_intensity_by_pa_gyrophase, expected_intensity_by_pa in test_cases: - with (((self.subTest(case)))): - rebinned_data = rebin_by_pitch_angle_and_gyrophase(intensity, intensity_delta_plus, - intensity_delta_minus, pitch_angles, gyrophases, - number_of_pitch_angle_bins, number_of_gyrophase_bins) + for ( + case, + pitch_angles, + gyrophases, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + expected_intensity_by_pa_gyrophase, + expected_intensity_by_pa, + ) in test_cases: + with self.subTest(case): + rebinned_data = rebin_by_pitch_angle_and_gyrophase( + intensity, + intensity_delta_plus, + intensity_delta_minus, + pitch_angles, + gyrophases, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ) intensity_pa_and_gyro, _, _ = rebinned_data[0:3] intensity_pa_only, _, _ = rebinned_data[3:6] - np.testing.assert_array_equal(intensity_pa_and_gyro, expected_intensity_by_pa_gyrophase) - np.testing.assert_array_equal(intensity_pa_only, expected_intensity_by_pa) + np.testing.assert_array_equal( + intensity_pa_and_gyro, expected_intensity_by_pa_gyrophase + ) + np.testing.assert_array_equal( + intensity_pa_only, expected_intensity_by_pa + ) - def test_rebin_by_pitch_angle_and_gyrophase_pa_product_only_correctly_weights_when_averaging(self): + def test_rebin_by_pitch_angle_and_gyrophase_pa_product_only_correctly_weights_when_averaging( + self, + ): intensity = np.array([[[0, 1, 2], [3, 4, 5]]]) - rebinned_data = rebin_by_pitch_angle_and_gyrophase(intensity, intensity * 0.01, intensity * 0.01, - np.array([[45, 45, 45], [135, 135, 135]]), - np.array([[45, 135, 225], [45, 135, 225]]), 1, 2) + rebinned_data = rebin_by_pitch_angle_and_gyrophase( + intensity, + intensity * 0.01, + intensity * 0.01, + np.array([[45, 45, 45], [135, 135, 135]]), + np.array([[45, 135, 225], [45, 135, 225]]), + 1, + 2, + ) intensity_pa_and_gyro, _, _ = rebinned_data[0:3] intensity_pa_only, _, _ = rebinned_data[3:6] np.testing.assert_equal(intensity_pa_and_gyro, [[[2, 3.5]]]) np.testing.assert_equal(intensity_pa_only, [[2.5]]) - def test_rebin_by_pitch_angle_and_gyrophase_pa_product_only_handles_nan_intensity(self): + def test_rebin_by_pitch_angle_and_gyrophase_pa_product_only_handles_nan_intensity( + self, + ): intensity = np.array([[[0, np.nan, 2, 7], [3, 4, np.nan, 8]]]) - rebinned_data = rebin_by_pitch_angle_and_gyrophase(intensity, intensity * 0.01, intensity * 0.01, - np.array([[45, 45, 45, 45], [135, 135, 135, 135]]), - np.array([[45, 135, 225, 300], [45, 135, 225, 300]]), 2, 2) + rebinned_data = rebin_by_pitch_angle_and_gyrophase( + intensity, + intensity * 0.01, + intensity * 0.01, + np.array([[45, 45, 45, 45], [135, 135, 135, 135]]), + np.array([[45, 135, 225, 300], [45, 135, 225, 300]]), + 2, + 2, + ) intensity_pa_and_gyro, _, _ = rebinned_data[0:3] intensity_pa_only, _, _ = rebinned_data[3:6] @@ -240,48 +338,57 @@ def test_rebin_by_pitch_angle_and_gyrophase_includes_uncertainties(self): default_gyrophases = np.array([[45, 135, 225, 315], [45, 135, 225, 315]]) intensity_data_for_energy_1 = [[0, 1, 2, 3], [4, 5, 6, 7]] intensity_data_for_energy_2 = [[8, 9, 10, 11], [12, 13, 14, 15]] - intensity = np.array([ - intensity_data_for_energy_1, - intensity_data_for_energy_2 - ]) + intensity = np.array([intensity_data_for_energy_1, intensity_data_for_energy_2]) intensity_delta_plus = intensity * 0.09 intensity_delta_minus = intensity * 0.11 number_of_pitch_angle_bins = 2 number_of_gyrophase_bins = 4 - expected_delta_plus_pa_gyro = np.array([ - [[0, 0.09, 0.18, 0.27], [0.36, 0.45, 0.54, 0.63]], - [[0.72, 0.81, 0.9, 0.99], [1.08, 1.17, 1.26, 1.35]] - ]) - - expected_delta_minus_pa_gyro = np.array([ - [[0, 0.11, 0.22, 0.33], [0.44, 0.55, 0.66, 0.77]], - [[0.88, 0.99, 1.1, 1.21], [1.32, 1.43, 1.54, 1.65]] - ]) - - expected_delta_plus_pa_only = np.array([ - [0.084187, 0.252562], - [0.43045, 0.60958] - ]) - - expected_delta_minus_pa_only = np.array([ - [0.102896, 0.308687], - [0.526106, 0.745042] - ]) + expected_delta_plus_pa_gyro = np.array( + [ + [[0, 0.09, 0.18, 0.27], [0.36, 0.45, 0.54, 0.63]], + [[0.72, 0.81, 0.9, 0.99], [1.08, 1.17, 1.26, 1.35]], + ] + ) - rebinned_data = rebin_by_pitch_angle_and_gyrophase(intensity, intensity_delta_plus, intensity_delta_minus, - default_pitch_angles, default_gyrophases, - number_of_pitch_angle_bins, number_of_gyrophase_bins) + expected_delta_minus_pa_gyro = np.array( + [ + [[0, 0.11, 0.22, 0.33], [0.44, 0.55, 0.66, 0.77]], + [[0.88, 0.99, 1.1, 1.21], [1.32, 1.43, 1.54, 1.65]], + ] + ) + + expected_delta_plus_pa_only = np.array( + [[0.084187, 0.252562], [0.43045, 0.60958]] + ) + + expected_delta_minus_pa_only = np.array( + [[0.102896, 0.308687], [0.526106, 0.745042]] + ) + + rebinned_data = rebin_by_pitch_angle_and_gyrophase( + intensity, + intensity_delta_plus, + intensity_delta_minus, + default_pitch_angles, + default_gyrophases, + number_of_pitch_angle_bins, + number_of_gyrophase_bins, + ) _, delta_plus_pa_gyro, delta_minus_pa_gyro = rebinned_data[0:3] _, delta_plus_pa_only, delta_minus_pa_only = rebinned_data[3:6] np.testing.assert_allclose(delta_plus_pa_gyro, expected_delta_plus_pa_gyro) np.testing.assert_allclose(delta_minus_pa_gyro, expected_delta_minus_pa_gyro) - np.testing.assert_allclose(delta_plus_pa_only, expected_delta_plus_pa_only, rtol=1e-5) - np.testing.assert_allclose(delta_minus_pa_only, expected_delta_minus_pa_only, rtol=1e-5) + np.testing.assert_allclose( + delta_plus_pa_only, expected_delta_plus_pa_only, rtol=1e-5 + ) + np.testing.assert_allclose( + delta_minus_pa_only, expected_delta_minus_pa_only, rtol=1e-5 + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/test_predicted_ephemeris_tracker.py b/tests/test_predicted_ephemeris_tracker.py index e073c716c..7cbad336b 100644 --- a/tests/test_predicted_ephemeris_tracker.py +++ b/tests/test_predicted_ephemeris_tracker.py @@ -89,7 +89,6 @@ def requires_predict(): self.assertEqual(2, mock_spice_function_1.call_count) self.assertEqual(1, mock_spice_function_2.call_count) - def test_predicted_ephemeris_tracker_skips_retry_if_no_predict_kernel(self): other_kernel = get_integration_test_spice_data_path( "imap_2025_105_2026_105_01.ah.bc" @@ -106,4 +105,11 @@ def test_predicted_ephemeris_tracker_allows_unknown_kernel_names(self): with KernelPool(kernels): tracker = PredictedEphemerisTracker() - tracker.run(spiceypy.spkezr, "IMAP", spiceypy.datetime2et(datetime(2025,10, 10)), "ECLIPJ2000", "NONE", "SUN") \ No newline at end of file + tracker.run( + spiceypy.spkezr, + "IMAP", + spiceypy.datetime2et(datetime(2025, 10, 10)), + "ECLIPJ2000", + "NONE", + "SUN", + ) diff --git a/tests/test_processor.py b/tests/test_processor.py index 6db65ae7e..5d32ebefd 100644 --- a/tests/test_processor.py +++ b/tests/test_processor.py @@ -2,7 +2,11 @@ from pathlib import Path from unittest.mock import Mock, patch -from imap_data_access.processing_input import ProcessingInputCollection, ScienceInput, AncillaryInput +from imap_data_access.processing_input import ( + ProcessingInputCollection, + ScienceInput, + AncillaryInput, +) from imap_l3_processing.swapi.swapi_processor import SwapiProcessor @@ -10,9 +14,11 @@ class TestProcessor(unittest.TestCase): @patch("imap_l3_processing.processor.get_spice_parent_file_names") def test_get_parent_file_names(self, mock_get_spice_parent_file_names): - input_collection = ProcessingInputCollection(ScienceInput("imap_swapi_l2_science_20250417_v001.cdf"), - ScienceInput("imap_swapi_l2_science_20250418_v001.cdf"), - AncillaryInput("imap_swapi_lookup_20250417_v001.cdf")) + input_collection = ProcessingInputCollection( + ScienceInput("imap_swapi_l2_science_20250417_v001.cdf"), + ScienceInput("imap_swapi_l2_science_20250418_v001.cdf"), + AncillaryInput("imap_swapi_lookup_20250417_v001.cdf"), + ) mock_get_spice_parent_file_names.return_value = ["kernel_1", "kernel_2"] @@ -29,16 +35,16 @@ def test_get_parent_file_names(self, mock_get_spice_parent_file_names): self.assertEqual(expected_parent_file_names, actual_parent_file_names) - @patch('imap_l3_processing.processor.spiceypy') + @patch("imap_l3_processing.processor.spiceypy") def test_get_parent_files_takes_optional_parameter(self, mock_spiceypy): - file_paths = [Path('path')] + file_paths = [Path("path")] mock_spiceypy.ktotal.return_value = 0 processor = SwapiProcessor(Mock(), Mock()) actual_parent_file_names = processor.get_parent_file_names(file_paths) - self.assertEqual(['path'], actual_parent_file_names) + self.assertEqual(["path"], actual_parent_file_names) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/ultra/test_models.py b/tests/ultra/test_models.py index 715067635..df4360b77 100644 --- a/tests/ultra/test_models.py +++ b/tests/ultra/test_models.py @@ -23,10 +23,12 @@ class TestModels(SpiceTestCase): - @with_tempdir def test_glows_l3e_read_from_file(self, temp_dir): - path_to_cdf = temp_dir / 'imap_glows_l3e_survival-probability-ul_20250415-repoint01000_v001.cdf' + path_to_cdf = ( + temp_dir + / "imap_glows_l3e_survival-probability-ul_20250415-repoint01000_v001.cdf" + ) rng = np.random.RandomState(42) expected_epoch = datetime(2025, 4, 15, 12, 0, 0, 1) @@ -35,7 +37,7 @@ def test_glows_l3e_read_from_file(self, temp_dir): expected_probability_of_survival = rng.random((1, 16, 3072)) expected_flags = np.array([GlowsL3Flags.PREDICTIVE_EPHEMERIS], dtype=np.uint16) - with CDF(str(path_to_cdf), masterpath='') as cdf: + with CDF(str(path_to_cdf), masterpath="") as cdf: cdf[EPOCH_CDF_VAR_NAME] = [expected_epoch] cdf[EPOCH_CDF_VAR_NAME].attrs["FILLVAL"] = -38470932875435 @@ -50,7 +52,6 @@ def test_glows_l3e_read_from_file(self, temp_dir): cdf[GLOWS_FLAGS_VAR_NAME] = expected_flags - actual = UltraGlowsL3eData.read_from_path(path_to_cdf) self.assertEqual(expected_epoch, actual.epoch) @@ -65,16 +66,26 @@ def test_glows_l3e_read_from_file(self, temp_dir): def test_ultra_l1c_read_from_file_and_can_convert_to_xarray(self): expected_epoch = datetime(2026, 5, 18, 10, 3, 11, 602839) - run_local_path = get_test_data_folder() / 'ultra' / 'imap_ultra_l1c_90sensor-spacecraftpset_20260518-repoint00252_v002.cdf' + run_local_path = ( + get_test_data_folder() + / "ultra" + / "imap_ultra_l1c_90sensor-spacecraftpset_20260518-repoint00252_v002.cdf" + ) actual = UltraL1CPSet.read_from_path(run_local_path) with CDF(str(run_local_path)) as expected: self.assertEqual(expected_epoch, actual.epoch) self.assertEqual(252, actual.repointing) - np.testing.assert_array_equal(expected[CoordNames.ENERGY_ULTRA_L1C.value][...], actual.energy) - np.testing.assert_array_equal(expected["exposure_factor"][...], actual.exposure) - np.testing.assert_array_equal(expected["epoch_delta"][...], actual.epoch_delta) + np.testing.assert_array_equal( + expected[CoordNames.ENERGY_ULTRA_L1C.value][...], actual.energy + ) + np.testing.assert_array_equal( + expected["exposure_factor"][...], actual.exposure + ) + np.testing.assert_array_equal( + expected["epoch_delta"][...], actual.epoch_delta + ) np.testing.assert_array_equal( expected[CoordNames.HEALPIX_INDEX.value][...], actual.healpix_index ) @@ -89,27 +100,38 @@ def test_ultra_l1c_read_from_file_and_can_convert_to_xarray(self): ) def test_ultra_l1c_convert_to_xarray(self): - l1c_path = get_test_data_path("ultra/imap_ultra_l1c_90sensor-spacecraftpset_20260518-repoint00252_v002.cdf") + l1c_path = get_test_data_path( + "ultra/imap_ultra_l1c_90sensor-spacecraftpset_20260518-repoint00252_v002.cdf" + ) actual = UltraL1CPSet.read_from_path(l1c_path) dataset = actual.to_xarray() - self.assertEqual([ - "exposure_factor", - "sensitivity", - CoordNames.AZIMUTH_L1C.value, - CoordNames.ELEVATION_L1C.value, - ], list(dataset.data_vars.keys())) + self.assertEqual( + [ + "exposure_factor", + "sensitivity", + CoordNames.AZIMUTH_L1C.value, + CoordNames.ELEVATION_L1C.value, + ], + list(dataset.data_vars.keys()), + ) - self.assertEqual([ - CoordNames.TIME.value, - CoordNames.ENERGY_ULTRA_L1C.value, - CoordNames.HEALPIX_INDEX.value, - ], list(dataset.coords.keys())) + self.assertEqual( + [ + CoordNames.TIME.value, + CoordNames.ENERGY_ULTRA_L1C.value, + CoordNames.HEALPIX_INDEX.value, + ], + list(dataset.coords.keys()), + ) - def test_ultra_l1c_read_from_file_handles_longitude_latitude_with_time_dimension(self): + def test_ultra_l1c_read_from_file_handles_longitude_latitude_with_time_dimension( + self, + ): l1c_path = get_integration_test_data_path( - "ultra/imap_ultra_l1c_45sensor-spacecraftpset_20250416-repoint00000_v000.cdf") + "ultra/imap_ultra_l1c_45sensor-spacecraftpset_20250416-repoint00000_v000.cdf" + ) actual = UltraL1CPSet.read_from_path(l1c_path) self.assertEqual(1, actual.longitude.ndim) diff --git a/tests/ultra/test_ultra_combined_nsp_initializer.py b/tests/ultra/test_ultra_combined_nsp_initializer.py index b0adbfd6d..43bc6131b 100644 --- a/tests/ultra/test_ultra_combined_nsp_initializer.py +++ b/tests/ultra/test_ultra_combined_nsp_initializer.py @@ -6,14 +6,17 @@ from imap_l3_processing.maps.map_initializer import MapInitializer, PossibleMapToProduce from imap_l3_processing.models import InputMetadata, VersionMap -from imap_l3_processing.ultra.ultra_combined_nsp_initializer import UltraCombinedNSPInitializer +from imap_l3_processing.ultra.ultra_combined_nsp_initializer import ( + UltraCombinedNSPInitializer, +) from tests.integration.integration_test_helpers import ImapQueryPatcher class TestUltraCombinedNSPInitializer(unittest.TestCase): def setUp(self): self.mock_query_patcher = patch( - 'imap_l3_processing.ultra.ultra_combined_nsp_initializer.imap_data_access.query') + "imap_l3_processing.ultra.ultra_combined_nsp_initializer.imap_data_access.query" + ) self.mock_query = self.mock_query_patcher.start() def tearDown(self): @@ -22,26 +25,27 @@ def tearDown(self): def test_is_map_initializer(self): self.assertIsInstance(UltraCombinedNSPInitializer(), MapInitializer) - @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') + @patch("imap_l3_processing.maps.map_initializer.read_cdf_parents") def test_get_maps_that_can_be_produced(self, mock_read_cdf_parents): - self.mock_query.side_effect = ImapQueryPatcher([ - 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - ]) + self.mock_query.side_effect = ImapQueryPatcher( + [ + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf", + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf", + ] + ) initializer = UltraCombinedNSPInitializer() - descriptor = 'ulc-ena-h-sf-nsp-full-hae-4deg-3mo' + descriptor = "ulc-ena-h-sf-nsp-full-hae-4deg-3mo" mock_read_cdf_parents.side_effect = [ [ "imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", "imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", "imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf", - ], [ "imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", "imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", - "imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf" + "imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf", ], ] actual_possible_maps = initializer.get_maps_that_can_be_produced(descriptor) @@ -49,8 +53,8 @@ def test_get_maps_that_can_be_produced(self, mock_read_cdf_parents): expected_possible_maps = [ PossibleMapToProduce( input_files={ - 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf", + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf", "imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", "imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", "imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf", @@ -59,74 +63,72 @@ def test_get_maps_that_can_be_produced(self, mock_read_cdf_parents): "imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf", }, input_metadata=InputMetadata( - instrument='ultra', - data_level='l3', + instrument="ultra", + data_level="l3", start_date=datetime(2010, 1, 1), end_date=datetime(2010, 4, 2, 7, 30), - version=VersionMap({},Version(None,1)), + version=VersionMap({}, Version(None, 1)), descriptor=descriptor, - ) + ), ) ] self.assertEqual(expected_possible_maps, actual_possible_maps) - @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') + @patch("imap_l3_processing.maps.map_initializer.read_cdf_parents") def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): cases = [ - (None, Version(None,2)), - (4, Version(4,2)), + (None, Version(None, 2)), + (4, Version(4, 2)), ] for major_version, expected_version in cases: with self.subTest(major_version=major_version): - self.mock_query.side_effect = ImapQueryPatcher([ - 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - - 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v002.cdf', - - 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', - 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', - - 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf', - - f'imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_{Version(major_version, 1)}.cdf', - f'imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_{Version(major_version, 1)}.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' - ]) + self.mock_query.side_effect = ImapQueryPatcher( + [ + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf", + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf", + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v002.cdf", + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf", + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf", + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf", + f"imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_{Version(major_version, 1)}.cdf", + f"imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_{Version(major_version, 1)}.cdf", + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv", + ] + ) mock_read_cdf_parents.side_effect = [ [ "imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", "imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", "imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf", - ], [ "imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", "imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", - "imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf" + "imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf", + ], + [ + "imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf", ], [ - 'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf' - ], [ - 'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v002.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v002.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v002.cdf' + "imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v002.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v002.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v002.cdf", ], [ - 'imap_ultra_l1c_45sensor-spacecraftpset_20100701-repoint00201_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100702-repoint00202_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100703-repoint00203_v001.cdf' + "imap_ultra_l1c_45sensor-spacecraftpset_20100701-repoint00201_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100702-repoint00202_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100703-repoint00203_v001.cdf", ], [ - 'imap_ultra_l1c_90sensor-spacecraftpset_20100701-repoint00201_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100702-repoint00202_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100703-repoint00203_v001.cdf' + "imap_ultra_l1c_90sensor-spacecraftpset_20100701-repoint00201_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100702-repoint00202_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100703-repoint00203_v001.cdf", ], [ "imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", @@ -135,44 +137,68 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): "imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", "imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", "imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf", - 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv', + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf", + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf", + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv", ], [ - 'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv', - ] + "imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v001.cdf", + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv", + ], ] initializer = UltraCombinedNSPInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced("ulc-ena-h-sf-nsp-full-hae-4deg-3mo", - major_version) - - self.mock_query.assert_has_calls([ - call(instrument='ultra', data_level='l2'), - call(instrument="ultra", table='ancillary', descriptor='l2-energy-bin-group-sizes', - version='latest'), - call(instrument='ultra', data_level='l3') - ]) - - mock_read_cdf_parents.assert_has_calls([ - call('imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), - call('imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), - call('imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), - call('imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v002.cdf'), - call('imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), - call('imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), - - call(f'imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_{Version(major_version, 1)}.cdf'), - call(f'imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_{Version(major_version, 1)}.cdf'), - ]) + actual_maps_to_produce = initializer.get_maps_that_should_be_produced( + "ulc-ena-h-sf-nsp-full-hae-4deg-3mo", major_version + ) + + self.mock_query.assert_has_calls( + [ + call(instrument="ultra", data_level="l2"), + call( + instrument="ultra", + table="ancillary", + descriptor="l2-energy-bin-group-sizes", + version="latest", + ), + call(instrument="ultra", data_level="l3"), + ] + ) + + mock_read_cdf_parents.assert_has_calls( + [ + call( + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf" + ), + call( + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf" + ), + call( + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf" + ), + call( + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v002.cdf" + ), + call( + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf" + ), + call( + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf" + ), + call( + f"imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_{Version(major_version, 1)}.cdf" + ), + call( + f"imap_ultra_l3_ulc-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_{Version(major_version, 1)}.cdf" + ), + ] + ) descriptor = "ulc-ena-h-sf-nsp-full-hae-4deg-3mo" @@ -182,20 +208,20 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): data_level="l3", start_date=datetime(2010, 4, 1), end_date=datetime(2010, 7, 1, 7, 30), - version=VersionMap({descriptor:expected_version}), - descriptor=descriptor + version=VersionMap({descriptor: expected_version}), + descriptor=descriptor, ), input_files={ - 'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v002.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v002.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v002.cdf', - 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v002.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' - } + "imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v002.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v002.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v002.cdf", + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v002.cdf", + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv", + }, ) expected_possible_map_to_produce_with_no_previous_version = PossibleMapToProduce( input_metadata=InputMetadata( @@ -204,26 +230,28 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): start_date=datetime(2010, 7, 1), end_date=datetime(2010, 9, 30, 7, 30), version=VersionMap({descriptor: Version(major_version, 1)}), - descriptor=descriptor + descriptor=descriptor, ), input_files={ - 'imap_ultra_l1c_45sensor-spacecraftpset_20100701-repoint00201_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100702-repoint00202_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100703-repoint00203_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100701-repoint00201_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100702-repoint00202_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100703-repoint00203_v001.cdf', - 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', - 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' - } + "imap_ultra_l1c_45sensor-spacecraftpset_20100701-repoint00201_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100702-repoint00202_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100703-repoint00203_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100701-repoint00201_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100702-repoint00202_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100703-repoint00203_v001.cdf", + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf", + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf", + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv", + }, ) - expected_maps_to_produce = [expected_possible_map_to_produce_with_previous_version, - expected_possible_map_to_produce_with_no_previous_version] + expected_maps_to_produce = [ + expected_possible_map_to_produce_with_previous_version, + expected_possible_map_to_produce_with_no_previous_version, + ] self.assertEqual(expected_maps_to_produce, actual_maps_to_produce) def test_initializer_raises_when_given_invalid_map_descriptor(self): initializer = UltraCombinedNSPInitializer() with self.assertRaises(ValueError) as e: initializer.get_maps_that_should_be_produced("ulc-invalid", None) - self.assertEqual(e.exception.args, ("Invalid map descriptor: ulc-invalid",)) \ No newline at end of file + self.assertEqual(e.exception.args, ("Invalid map descriptor: ulc-invalid",)) diff --git a/tests/ultra/test_ultra_l3_dependencies.py b/tests/ultra/test_ultra_l3_dependencies.py index 48ff731ae..750921d90 100644 --- a/tests/ultra/test_ultra_l3_dependencies.py +++ b/tests/ultra/test_ultra_l3_dependencies.py @@ -16,112 +16,183 @@ from imap_l3_processing.utils import clear_temp_cache, get_temp_cache_dir from tests.test_helpers import get_test_data_path + class TestUltraL3Dependencies(unittest.TestCase): def setUp(self): clear_temp_cache() - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.UltraGlowsL3eData.read_from_path') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.UltraL1CPSet.read_from_path') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.RectangularIntensityMapData.read_from_path') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.HealPixIntensityMapData.read_from_xarray') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.imap_data_access.download') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.ScienceInput') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.ultra_l2') - def test_fetch_dependencies(self, mock_ultra_l2, mock_science_input_constructor, mock_download, mock_read_xarray_healpix, mock_read_from_path_rectangular, mock_read_ultra_l1c, - mock_read_glows): - l1c_input_paths = [Path("imap_ultra_l1c_45sensor-pset_20251010_v001.cdf"), - Path("imap_ultra_l1c_45sensor-pset_20251011_v001.cdf"), - Path("imap_ultra_l1c_45sensor-pset_20251012_v001.cdf")] + + @patch( + "imap_l3_processing.ultra.ultra_l3_dependencies.UltraGlowsL3eData.read_from_path" + ) + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.UltraL1CPSet.read_from_path") + @patch( + "imap_l3_processing.ultra.ultra_l3_dependencies.RectangularIntensityMapData.read_from_path" + ) + @patch( + "imap_l3_processing.ultra.ultra_l3_dependencies.HealPixIntensityMapData.read_from_xarray" + ) + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.imap_data_access.download") + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.ScienceInput") + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.ultra_l2") + def test_fetch_dependencies( + self, + mock_ultra_l2, + mock_science_input_constructor, + mock_download, + mock_read_xarray_healpix, + mock_read_from_path_rectangular, + mock_read_ultra_l1c, + mock_read_glows, + ): + l1c_input_paths = [ + Path("imap_ultra_l1c_45sensor-pset_20251010_v001.cdf"), + Path("imap_ultra_l1c_45sensor-pset_20251011_v001.cdf"), + Path("imap_ultra_l1c_45sensor-pset_20251012_v001.cdf"), + ] glows_file_paths = [ "imap_glows_l3e_survival-probability-ul-sf_20201001_v001.cdf", "imap_glows_l3e_survival-probability-ul-sf_20201002_v002.cdf", - "imap_glows_l3e_survival-probability-ul-sf_20201003_v003.cdf"] + "imap_glows_l3e_survival-probability-ul-sf_20201003_v003.cdf", + ] - l2_energy_bin_group_sizes = 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' + l2_energy_bin_group_sizes = ( + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv" + ) l2_energy_bin_group_sizes_path = get_test_data_path( - 'ultra/imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv') + "ultra/imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv" + ) input_collection = Mock() input_collection.get_file_paths.side_effect = [ [sentinel.l2_server_path], l1c_input_paths, glows_file_paths, - [l2_energy_bin_group_sizes] + [l2_energy_bin_group_sizes], ] - returned_download_paths = [sentinel.l2_map_path, l1c_input_paths[0], l1c_input_paths[1], l1c_input_paths[2], - sentinel.glows_path_1, sentinel.glows_path_2, sentinel.glows_path_3, - l2_energy_bin_group_sizes_path] + returned_download_paths = [ + sentinel.l2_map_path, + l1c_input_paths[0], + l1c_input_paths[1], + l1c_input_paths[2], + sentinel.glows_path_1, + sentinel.glows_path_2, + sentinel.glows_path_3, + l2_energy_bin_group_sizes_path, + ] mock_download.side_effect = returned_download_paths - mock_science_input_constructor.return_value = Mock(descriptor="ultra-nsp-descriptor-6deg") + mock_science_input_constructor.return_value = Mock( + descriptor="ultra-nsp-descriptor-6deg" + ) - l1c_data = [sentinel.ultra_l1c_data_1, sentinel.ultra_l1c_data_2, - sentinel.ultra_l1c_data_3] + l1c_data = [ + sentinel.ultra_l1c_data_1, + sentinel.ultra_l1c_data_2, + sentinel.ultra_l1c_data_3, + ] mock_read_ultra_l1c.side_effect = l1c_data - glows_l3e_data = [sentinel.glows_data_1, sentinel.glows_data_2, sentinel.glows_data_3] + glows_l3e_data = [ + sentinel.glows_data_1, + sentinel.glows_data_2, + sentinel.glows_data_3, + ] mock_read_glows.side_effect = glows_l3e_data mock_ultra_l2.return_value = [sentinel.ultra_l2_healpix_map_dataset] mock_read_xarray_healpix.return_value = sentinel.ultra_l2_healpix_data - mock_read_from_path_rectangular.return_value = sentinel.ultra_l2_rectangular_data + mock_read_from_path_rectangular.return_value = ( + sentinel.ultra_l2_rectangular_data + ) dependencies = UltraL3Dependencies.fetch_dependencies(input_collection) mock_science_input_constructor.assert_called_with(sentinel.l2_map_path.name) - input_collection.get_file_paths.assert_has_calls([ - call("ultra", data_type="l2"), - call("ultra", data_type="l1c"), - call("glows"), - call(data_type="ancillary", descriptor="l2-energy-bin-group-sizes") - ]) - expected_data_dictionary = {"imap_ultra_l1c_45sensor-pset_20251010_v001": l1c_input_paths[0], - "imap_ultra_l1c_45sensor-pset_20251011_v001": l1c_input_paths[1], - "imap_ultra_l1c_45sensor-pset_20251012_v001": l1c_input_paths[2]} - mock_ultra_l2.assert_has_calls([call(expected_data_dictionary, descriptor="ultra-nsp-descriptor-nside32")]) - + input_collection.get_file_paths.assert_has_calls( + [ + call("ultra", data_type="l2"), + call("ultra", data_type="l1c"), + call("glows"), + call(data_type="ancillary", descriptor="l2-energy-bin-group-sizes"), + ] + ) + expected_data_dictionary = { + "imap_ultra_l1c_45sensor-pset_20251010_v001": l1c_input_paths[0], + "imap_ultra_l1c_45sensor-pset_20251011_v001": l1c_input_paths[1], + "imap_ultra_l1c_45sensor-pset_20251012_v001": l1c_input_paths[2], + } + mock_ultra_l2.assert_has_calls( + [call(expected_data_dictionary, descriptor="ultra-nsp-descriptor-nside32")] + ) - expected_parent_file_paths = [sentinel.l2_server_path, *l1c_input_paths, *glows_file_paths, - l2_energy_bin_group_sizes] - mock_download.assert_has_calls([call(file_path) for file_path in - expected_parent_file_paths]) + expected_parent_file_paths = [ + sentinel.l2_server_path, + *l1c_input_paths, + *glows_file_paths, + l2_energy_bin_group_sizes, + ] + mock_download.assert_has_calls( + [call(file_path) for file_path in expected_parent_file_paths] + ) - mock_read_xarray_healpix.assert_called_once_with(sentinel.ultra_l2_healpix_map_dataset) + mock_read_xarray_healpix.assert_called_once_with( + sentinel.ultra_l2_healpix_map_dataset + ) mock_read_from_path_rectangular.assert_called_once_with(sentinel.l2_map_path) self.assertEqual(l1c_data, dependencies.ultra_l1c_pset) self.assertEqual(glows_l3e_data, dependencies.glows_l3e_sp) - self.assertEqual(sentinel.ultra_l2_healpix_data, dependencies.ultra_l2_healpix_map) - self.assertEqual(sentinel.ultra_l2_rectangular_data, dependencies.ultra_l2_rectangular_map) + self.assertEqual( + sentinel.ultra_l2_healpix_data, dependencies.ultra_l2_healpix_map + ) + self.assertEqual( + sentinel.ultra_l2_rectangular_data, dependencies.ultra_l2_rectangular_map + ) self.assertEqual(returned_download_paths, dependencies.dependency_file_paths) - np.testing.assert_array_equal(np.array([0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 46], dtype=np.uint8), - dependencies.energy_bin_group_sizes, strict=True) - - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.load_or_create_healpix_l2') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.UltraGlowsL3eData.read_from_path') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.UltraL1CPSet.read_from_path') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.RectangularIntensityMapData.read_from_path') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.imap_data_access.download') - def test_fetch_dependencies_without_energy_ancillary_file(self, mock_download, - mock_read_from_path, mock_read_ultra_l1c, - mock_read_glows, - mock_load_or_create_healpix_l2): - l1c_input_paths = ["imap_ultra_l1c_pset_20251010_v001.cdf", "imap_ultra_l1c_pset_20251011_v001.cdf", - "imap_ultra_l1c_pset_20251012_v001.cdf"] + np.testing.assert_array_equal( + np.array([0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 46], dtype=np.uint8), + dependencies.energy_bin_group_sizes, + strict=True, + ) + + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.load_or_create_healpix_l2") + @patch( + "imap_l3_processing.ultra.ultra_l3_dependencies.UltraGlowsL3eData.read_from_path" + ) + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.UltraL1CPSet.read_from_path") + @patch( + "imap_l3_processing.ultra.ultra_l3_dependencies.RectangularIntensityMapData.read_from_path" + ) + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.imap_data_access.download") + def test_fetch_dependencies_without_energy_ancillary_file( + self, + mock_download, + mock_read_from_path, + mock_read_ultra_l1c, + mock_read_glows, + mock_load_or_create_healpix_l2, + ): + l1c_input_paths = [ + "imap_ultra_l1c_pset_20251010_v001.cdf", + "imap_ultra_l1c_pset_20251011_v001.cdf", + "imap_ultra_l1c_pset_20251012_v001.cdf", + ] glows_file_paths = [ "imap_glows_l3e_survival-probability-ul-sf_20201001_v001.cdf", "imap_glows_l3e_survival-probability-ul-sf_20201002_v002.cdf", - "imap_glows_l3e_survival-probability-ul-sf_20201003_v003.cdf"] + "imap_glows_l3e_survival-probability-ul-sf_20201003_v003.cdf", + ] input_collection = Mock() input_collection.get_file_paths.side_effect = [ [sentinel.l2_server_path], l1c_input_paths, glows_file_paths, - [] + [], ] mock_download.return_value = Path("downloaded_file") dependencies = UltraL3Dependencies.fetch_dependencies(input_collection) @@ -130,17 +201,24 @@ def test_fetch_dependencies_without_energy_ancillary_file(self, mock_download, def test_raise_error_for_more_than_one_input_files_paths(self): input_collection = Mock() - input_collection.get_file_paths.return_value = [sentinel.imap_l2_map_path_1, sentinel.imap_l2_map_path_2] + input_collection.get_file_paths.return_value = [ + sentinel.imap_l2_map_path_1, + sentinel.imap_l2_map_path_2, + ] with self.assertRaises(AssertionError) as e: UltraL3Dependencies.fetch_dependencies(input_collection) self.assertEqual("Incorrect number of map dependencies: 2", str(e.exception)) - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.load_or_create_healpix_l2') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.RectangularIntensityMapData.read_from_path') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.UltraGlowsL3eData.read_from_path') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.UltraL1CPSet.read_from_path') + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.load_or_create_healpix_l2") + @patch( + "imap_l3_processing.ultra.ultra_l3_dependencies.RectangularIntensityMapData.read_from_path" + ) + @patch( + "imap_l3_processing.ultra.ultra_l3_dependencies.UltraGlowsL3eData.read_from_path" + ) + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.UltraL1CPSet.read_from_path") def test_from_file_paths( self, mock_read_l1c: Mock, @@ -149,31 +227,53 @@ def test_from_file_paths( mock_load_or_create_healpix_l2: Mock, ): ultra_l2_input_path = Path("um_path") - ultra_l1c_input_paths = [Path("imap/u_path_3"), Path("imap/u_path_1"), Path("imap/u_path_2")] + ultra_l1c_input_paths = [ + Path("imap/u_path_3"), + Path("imap/u_path_1"), + Path("imap/u_path_2"), + ] glows_input_paths = [Path("g_path_1"), Path("g_path_2")] - mock_read_l1c.side_effect = [sentinel.ultra_data3, sentinel.ultra_data1, sentinel.ultra_data2] + mock_read_l1c.side_effect = [ + sentinel.ultra_data3, + sentinel.ultra_data1, + sentinel.ultra_data2, + ] mock_read_glows.side_effect = [sentinel.glows_data1, sentinel.glows_data2] mock_read_l2_from_path.return_value = sentinel.ultra_l2_rectangular_data mock_load_or_create_healpix_l2.return_value = sentinel.ultra_l2_healpix_data - result = UltraL3Dependencies.from_file_paths(ultra_l2_input_path, ultra_l1c_input_paths, glows_input_paths, - None) + result = UltraL3Dependencies.from_file_paths( + ultra_l2_input_path, ultra_l1c_input_paths, glows_input_paths, None + ) mock_read_l2_from_path.assert_called_with(ultra_l2_input_path) mock_read_l1c.assert_has_calls( [call(file_path) for file_path in ultra_l1c_input_paths] ) - mock_read_glows.assert_has_calls([call(file_path) for file_path in glows_input_paths]) - mock_load_or_create_healpix_l2.assert_called_with(ultra_l2_input_path, ultra_l1c_input_paths) + mock_read_glows.assert_has_calls( + [call(file_path) for file_path in glows_input_paths] + ) + mock_load_or_create_healpix_l2.assert_called_with( + ultra_l2_input_path, ultra_l1c_input_paths + ) - self.assertEqual(result.ultra_l1c_pset, [sentinel.ultra_data3, sentinel.ultra_data1, sentinel.ultra_data2]) - self.assertEqual(result.glows_l3e_sp, [sentinel.glows_data1, sentinel.glows_data2]) + self.assertEqual( + result.ultra_l1c_pset, + [sentinel.ultra_data3, sentinel.ultra_data1, sentinel.ultra_data2], + ) + self.assertEqual( + result.glows_l3e_sp, [sentinel.glows_data1, sentinel.glows_data2] + ) self.assertEqual(result.ultra_l2_healpix_map, sentinel.ultra_l2_healpix_data) - self.assertEqual(result.ultra_l2_rectangular_map, sentinel.ultra_l2_rectangular_data) + self.assertEqual( + result.ultra_l2_rectangular_map, sentinel.ultra_l2_rectangular_data + ) - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.HealPixIntensityMapData.read_from_xarray') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.ultra_l2') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.ScienceInput') + @patch( + "imap_l3_processing.ultra.ultra_l3_dependencies.HealPixIntensityMapData.read_from_xarray" + ) + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.ultra_l2") + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.ScienceInput") def test_load_or_create_healpix_l2_saves_to_cache( self, mock_science_input_constructor, @@ -181,10 +281,16 @@ def test_load_or_create_healpix_l2_saves_to_cache( mock_read_l2_xarray: Mock, ): ultra_l2_input_path = Path("um_path") - ultra_l1c_input_paths = [Path("imap/u_path_3"), Path("imap/u_path_1"), Path("imap/u_path_2")] + ultra_l1c_input_paths = [ + Path("imap/u_path_3"), + Path("imap/u_path_1"), + Path("imap/u_path_2"), + ] mock_read_l2_xarray.return_value = sentinel.ultra_l2_healpix_data l2_ultra_descriptor = "u45-ena-h-hf-nsp-full-hae-6deg" - mock_science_input_constructor.return_value = Mock(descriptor=l2_ultra_descriptor) + mock_science_input_constructor.return_value = Mock( + descriptor=l2_ultra_descriptor + ) mock_ultra_l2.return_value = [sentinel.ultra_l2_healpix_map] result = load_or_create_healpix_l2(ultra_l2_input_path, ultra_l1c_input_paths) @@ -198,7 +304,10 @@ def test_load_or_create_healpix_l2_saves_to_cache( }, descriptor="u45-ena-h-hf-nsp-full-hae-nside32", ) - self.assertEqual(["u_path_1", "u_path_2", "u_path_3"], list(mock_ultra_l2.call_args.args[0].keys())) + self.assertEqual( + ["u_path_1", "u_path_2", "u_path_3"], + list(mock_ultra_l2.call_args.args[0].keys()), + ) self.assertEqual(result, sentinel.ultra_l2_healpix_data) @@ -211,9 +320,11 @@ def test_load_or_create_healpix_l2_saves_to_cache( cached_healpix = pickle.load(f) self.assertEqual(sentinel.ultra_l2_healpix_data, cached_healpix) - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.HealPixIntensityMapData.read_from_xarray') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.ultra_l2') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.ScienceInput') + @patch( + "imap_l3_processing.ultra.ultra_l3_dependencies.HealPixIntensityMapData.read_from_xarray" + ) + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.ultra_l2") + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.ScienceInput") def test_load_or_create_healpix_l2_loads_from_cache( self, mock_science_input_constructor, @@ -221,14 +332,21 @@ def test_load_or_create_healpix_l2_loads_from_cache( mock_read_l2_xarray: Mock, ): temp_cache_dir = get_temp_cache_dir() - raw_cache_key = "['l1c_path_1.cdf', 'l1c_path_2.cdf']u45-ena-h-hf-nsp-full-hae-nside32" + raw_cache_key = ( + "['l1c_path_1.cdf', 'l1c_path_2.cdf']u45-ena-h-hf-nsp-full-hae-nside32" + ) cached_healpix_filename = sha256(raw_cache_key.encode("utf-8")).hexdigest() with open(temp_cache_dir / cached_healpix_filename, "wb") as f: pickle.dump(sentinel.ultra_l2_healpix_data, f) ultra_l2_input_path = Path("um_path") - ultra_l1c_input_paths = [Path("imap/l1c_path_1.cdf"), Path("imap/l1c_path_2.cdf")] + ultra_l1c_input_paths = [ + Path("imap/l1c_path_1.cdf"), + Path("imap/l1c_path_2.cdf"), + ] l2_ultra_descriptor = "u45-ena-h-hf-nsp-full-hae-6deg" - mock_science_input_constructor.return_value = Mock(descriptor=l2_ultra_descriptor) + mock_science_input_constructor.return_value = Mock( + descriptor=l2_ultra_descriptor + ) result = load_or_create_healpix_l2(ultra_l2_input_path, ultra_l1c_input_paths) @@ -237,99 +355,164 @@ def test_load_or_create_healpix_l2_loads_from_cache( self.assertEqual(result, sentinel.ultra_l2_healpix_data) - - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.RectangularIntensityMapData.read_from_path') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.imap_data_access.download') - def test_spectral_index_fetch_dependencies(self, mock_download, mock_rectangular_read_from_path): + @patch( + "imap_l3_processing.ultra.ultra_l3_dependencies.RectangularIntensityMapData.read_from_path" + ) + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.imap_data_access.download") + def test_spectral_index_fetch_dependencies( + self, mock_download, mock_rectangular_read_from_path + ): for input_data_type in ["l2", "l3"]: mock_rectangular_read_from_path.reset_mock() with self.subTest(input_data_type): - map_file_name = f'imap_ultra_{input_data_type}_ultra-cool-descriptor_20250601_v000.cdf' - ancillary_file_name = 'imap_ultra_spx-energy-ranges_20250601_v000.cdf' - pointing_set_file_name = 'imap_ultra_l1c_u90spacecraftpset_20250601-repoint00001_v000.cdf' - glows_file_name = 'imap_glows_l3e_survival-probability-ul-hf_20250601-repoint00001_v000.cdf' + map_file_name = f"imap_ultra_{input_data_type}_ultra-cool-descriptor_20250601_v000.cdf" + ancillary_file_name = "imap_ultra_spx-energy-ranges_20250601_v000.cdf" + pointing_set_file_name = ( + "imap_ultra_l1c_u90spacecraftpset_20250601-repoint00001_v000.cdf" + ) + glows_file_name = "imap_glows_l3e_survival-probability-ul-hf_20250601-repoint00001_v000.cdf" expected_energy_ranges = [[5, 15], [15, 50]] map_input = ScienceInput(map_file_name) ancillary_input = AncillaryInput(ancillary_file_name) - extra_inputs_from_mapper = [ScienceInput(pointing_set_file_name), ScienceInput(glows_file_name)] - processing_input_collection = ProcessingInputCollection(map_input, ancillary_input, *extra_inputs_from_mapper) + extra_inputs_from_mapper = [ + ScienceInput(pointing_set_file_name), + ScienceInput(glows_file_name), + ] + processing_input_collection = ProcessingInputCollection( + map_input, ancillary_input, *extra_inputs_from_mapper + ) mock_download.side_effect = [ "map_file", - get_test_data_path('ultra/imap_ultra_ulc-spx-energy-ranges_20250407_v000.dat') + get_test_data_path( + "ultra/imap_ultra_ulc-spx-energy-ranges_20250407_v000.dat" + ), ] - ultra_l3_dependencies = UltraL3SpectralIndexDependencies.fetch_dependencies(processing_input_collection) + ultra_l3_dependencies = ( + UltraL3SpectralIndexDependencies.fetch_dependencies( + processing_input_collection + ) + ) - mock_download.assert_has_calls([ - call(map_file_name), - call(ancillary_file_name) - ]) + mock_download.assert_has_calls( + [call(map_file_name), call(ancillary_file_name)] + ) mock_rectangular_read_from_path.assert_called_once_with("map_file") - self.assertEqual(ultra_l3_dependencies.map_data, mock_rectangular_read_from_path.return_value) - np.testing.assert_array_equal(ultra_l3_dependencies.fit_energy_ranges, expected_energy_ranges) + self.assertEqual( + ultra_l3_dependencies.map_data, + mock_rectangular_read_from_path.return_value, + ) + np.testing.assert_array_equal( + ultra_l3_dependencies.fit_energy_ranges, expected_energy_ranges + ) def test_spectral_index_fetch_dependencies_raises_exception_on_missing_map(self): - ancillary_input = AncillaryInput('imap_ultra_spx-energy-ranges_20250601_v000.dat') + ancillary_input = AncillaryInput( + "imap_ultra_spx-energy-ranges_20250601_v000.dat" + ) with self.assertRaises(ValueError) as context: - UltraL3SpectralIndexDependencies.fetch_dependencies(ProcessingInputCollection(ancillary_input)) + UltraL3SpectralIndexDependencies.fetch_dependencies( + ProcessingInputCollection(ancillary_input) + ) self.assertEqual("Expected 1 input map, got 0: []", str(context.exception)) def test_spectral_index_fetch_dependencies_raises_exception_on_multiple_maps(self): - ancillary_input = AncillaryInput('imap_ultra_spx-energy-ranges_20250601_v000.dat') - science_input = ScienceInput('imap_ultra_l3_map_20250601_v000.cdf') + ancillary_input = AncillaryInput( + "imap_ultra_spx-energy-ranges_20250601_v000.dat" + ) + science_input = ScienceInput("imap_ultra_l3_map_20250601_v000.cdf") with self.assertRaises(ValueError) as context: - UltraL3SpectralIndexDependencies.fetch_dependencies(ProcessingInputCollection(ancillary_input, science_input, science_input)) + UltraL3SpectralIndexDependencies.fetch_dependencies( + ProcessingInputCollection(ancillary_input, science_input, science_input) + ) expected_msg = "Expected 1 input map, got 2: [imap_ultra_l3_map_20250601_v000.cdf, imap_ultra_l3_map_20250601_v000.cdf]" self.assertEqual(expected_msg, str(context.exception)) - def test_spectral_index_fetch_dependencies_raises_exception_on_missing_ancillary_file(self): - science_input = ScienceInput('imap_ultra_l3_ultra-cool-descriptor_20250601_v000.cdf') + def test_spectral_index_fetch_dependencies_raises_exception_on_missing_ancillary_file( + self, + ): + science_input = ScienceInput( + "imap_ultra_l3_ultra-cool-descriptor_20250601_v000.cdf" + ) with self.assertRaises(ValueError) as context: - UltraL3SpectralIndexDependencies.fetch_dependencies(ProcessingInputCollection(science_input)) - self.assertEqual("Missing fit energy ranges ancillary file", str(context.exception)) + UltraL3SpectralIndexDependencies.fetch_dependencies( + ProcessingInputCollection(science_input) + ) + self.assertEqual( + "Missing fit energy ranges ancillary file", str(context.exception) + ) - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.RectangularIntensityMapData.read_from_path') + @patch( + "imap_l3_processing.ultra.ultra_l3_dependencies.RectangularIntensityMapData.read_from_path" + ) def test_spectral_index_from_file_paths(self, mock_read_from_path): map_file_path = Mock() - ancillary_file_path = get_test_data_path('ultra') / 'imap_ultra_ulc-spx-energy-ranges_20250407_v000.dat' + ancillary_file_path = ( + get_test_data_path("ultra") + / "imap_ultra_ulc-spx-energy-ranges_20250407_v000.dat" + ) expected_energy_range_values = np.loadtxt(ancillary_file_path) - actual_dependencies = UltraL3SpectralIndexDependencies.from_file_paths(map_file_path, ancillary_file_path) + actual_dependencies = UltraL3SpectralIndexDependencies.from_file_paths( + map_file_path, ancillary_file_path + ) self.assertEqual(mock_read_from_path.return_value, actual_dependencies.map_data) - np.testing.assert_array_equal(actual_dependencies.fit_energy_ranges, expected_energy_range_values) + np.testing.assert_array_equal( + actual_dependencies.fit_energy_ranges, expected_energy_range_values + ) def test_spectral_index_get_fit_energy_ranges(self): expected_energy_range_values = np.array([[5, 10], [15, 20]]) - dependencies = UltraL3SpectralIndexDependencies(map_data=Mock(), - fit_energy_ranges=expected_energy_range_values) + dependencies = UltraL3SpectralIndexDependencies( + map_data=Mock(), fit_energy_ranges=expected_energy_range_values + ) actual_fit_energy_ranges = dependencies.get_fit_energy_ranges() - np.testing.assert_array_equal(actual_fit_energy_ranges, expected_energy_range_values) + np.testing.assert_array_equal( + actual_fit_energy_ranges, expected_energy_range_values + ) + class TestUltraL3CombinedDependencies(unittest.TestCase): def setUp(self): clear_temp_cache() - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.load_or_create_healpix_l2') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.UltraL1CPSet.read_from_path') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.RectangularIntensityMapData.read_from_path') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.imap_data_access.download') - def test_fetch_dependencies_no_survival_corrected(self, mock_download, - mock_l2_read_from_path, mock_l1c_read_from_path, - mock_load_or_create_healpix_l2, - ): - u45_map_file_name = 'imap_ultra_l2_u45-cool-descriptor_20250601_v000.cdf' - u90_map_file_name = 'imap_ultra_l2_u90-cool-descriptor_20250601_v000.cdf' - u45_pset_file_name = ["imap_ultra_l1c_45sensor-spacecraftpset_20251010_v001.cdf", "imap_ultra_l1c_45sensor-spacecraftpset_20251011_v001.cdf", - "imap_ultra_l1c_45sensor-spacecraftpset_20251012_v001.cdf"] - u90_pset_file_name = ["imap_ultra_l1c_90sensor-spacecraftpset_20251010_v001.cdf", "imap_ultra_l1c_90sensor-spacecraftpset_20251011_v001.cdf", - "imap_ultra_l1c_90sensor-spacecraftpset_20251012_v001.cdf"] - l2_energy_bin_group_sizes_file_name = 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' - l2_energy_bin_group_sizes_file_path = get_test_data_path("ultra/" + l2_energy_bin_group_sizes_file_name) + + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.load_or_create_healpix_l2") + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.UltraL1CPSet.read_from_path") + @patch( + "imap_l3_processing.ultra.ultra_l3_dependencies.RectangularIntensityMapData.read_from_path" + ) + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.imap_data_access.download") + def test_fetch_dependencies_no_survival_corrected( + self, + mock_download, + mock_l2_read_from_path, + mock_l1c_read_from_path, + mock_load_or_create_healpix_l2, + ): + u45_map_file_name = "imap_ultra_l2_u45-cool-descriptor_20250601_v000.cdf" + u90_map_file_name = "imap_ultra_l2_u90-cool-descriptor_20250601_v000.cdf" + u45_pset_file_name = [ + "imap_ultra_l1c_45sensor-spacecraftpset_20251010_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20251011_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20251012_v001.cdf", + ] + u90_pset_file_name = [ + "imap_ultra_l1c_90sensor-spacecraftpset_20251010_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20251011_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20251012_v001.cdf", + ] + l2_energy_bin_group_sizes_file_name = ( + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv" + ) + l2_energy_bin_group_sizes_file_path = get_test_data_path( + "ultra/" + l2_energy_bin_group_sizes_file_name + ) u45_map_input = ScienceInput(u45_map_file_name) u90_map_input = ScienceInput(u90_map_file_name) @@ -337,8 +520,13 @@ def test_fetch_dependencies_no_survival_corrected(self, mock_download, u90_pset_inputs = [ScienceInput(pset) for pset in u90_pset_file_name] ancillary_inputs = AncillaryInput(l2_energy_bin_group_sizes_file_name) - processing_input_collection = ProcessingInputCollection(u45_map_input, u90_map_input, *u45_pset_inputs, - *u90_pset_inputs, ancillary_inputs) + processing_input_collection = ProcessingInputCollection( + u45_map_input, + u90_map_input, + *u45_pset_inputs, + *u90_pset_inputs, + ancillary_inputs, + ) u45_l2_path = Path("imap_ultra_l2_u45-cool-descriptor_20250601_v000.cdf") u90_l2_path = Path("imap_ultra_l2_u90-cool-descriptor_20250601_v000.cdf") @@ -352,32 +540,43 @@ def test_fetch_dependencies_no_survival_corrected(self, mock_download, Path("imap_ultra_l1c_u90-pset_20251012_v001.cdf"), u45_l2_path, u90_l2_path, - l2_energy_bin_group_sizes_file_path + l2_energy_bin_group_sizes_file_path, ] mock_download.side_effect = expected_file_paths mock_l1c_read_from_path.side_effect = [ - sentinel.u45_l1c_1, sentinel.u45_l1c_2, sentinel.u45_l1c_3, - sentinel.u90_l1c_1, sentinel.u90_l1c_2, sentinel.u90_l1c_3, + sentinel.u45_l1c_1, + sentinel.u45_l1c_2, + sentinel.u45_l1c_3, + sentinel.u90_l1c_1, + sentinel.u90_l1c_2, + sentinel.u90_l1c_3, ] - mock_l2_read_from_path.side_effect = [sentinel.u45_rectangular_l2, sentinel.u90_rectangular_l2] + mock_l2_read_from_path.side_effect = [ + sentinel.u45_rectangular_l2, + sentinel.u90_rectangular_l2, + ] mock_load_or_create_healpix_l2.side_effect = [ sentinel.u45_healpix_dataset, - sentinel.u90_healpix_dataset + sentinel.u90_healpix_dataset, ] - combined_dependencies = UltraL3CombinedDependencies.fetch_dependencies(processing_input_collection) + combined_dependencies = UltraL3CombinedDependencies.fetch_dependencies( + processing_input_collection + ) - mock_download.assert_has_calls([ - *[call(filename) for filename in u45_pset_file_name], - *[call(filename) for filename in u90_pset_file_name], - call(u45_map_file_name), - call(u90_map_file_name), - call(l2_energy_bin_group_sizes_file_name) - ]) + mock_download.assert_has_calls( + [ + *[call(filename) for filename in u45_pset_file_name], + *[call(filename) for filename in u90_pset_file_name], + call(u45_map_file_name), + call(u90_map_file_name), + call(l2_energy_bin_group_sizes_file_name), + ] + ) mock_l1c_read_from_path.assert_has_calls( [ @@ -408,56 +607,79 @@ def test_fetch_dependencies_no_survival_corrected(self, mock_download, self.assertEqual(u90_deps.ultra_l2_healpix_map, sentinel.u90_healpix_dataset) self.assertEqual(u45_deps.ultra_l2_rectangular_map, sentinel.u45_rectangular_l2) self.assertEqual(u90_deps.ultra_l2_rectangular_map, sentinel.u90_rectangular_l2) - self.assertEqual(u45_deps.ultra_l1c_pset, [sentinel.u45_l1c_1, sentinel.u45_l1c_2, sentinel.u45_l1c_3]) - self.assertEqual(u90_deps.ultra_l1c_pset, [sentinel.u90_l1c_1, sentinel.u90_l1c_2, sentinel.u90_l1c_3]) + self.assertEqual( + u45_deps.ultra_l1c_pset, + [sentinel.u45_l1c_1, sentinel.u45_l1c_2, sentinel.u45_l1c_3], + ) + self.assertEqual( + u90_deps.ultra_l1c_pset, + [sentinel.u90_l1c_1, sentinel.u90_l1c_2, sentinel.u90_l1c_3], + ) self.assertEqual(u45_deps.glows_l3e_sp, []) self.assertEqual(u90_deps.glows_l3e_sp, []) - np.testing.assert_array_equal(np.array([0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 46], dtype=np.uint8), - u45_deps.energy_bin_group_sizes, strict=True) - np.testing.assert_array_equal(np.array([0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 46], dtype=np.uint8), - u90_deps.energy_bin_group_sizes, strict=True) + np.testing.assert_array_equal( + np.array([0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 46], dtype=np.uint8), + u45_deps.energy_bin_group_sizes, + strict=True, + ) + np.testing.assert_array_equal( + np.array([0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 46], dtype=np.uint8), + u90_deps.energy_bin_group_sizes, + strict=True, + ) expected_45_file_paths = [ u45_l2_path, Path("imap_ultra_l1c_u45-pset_20251010_v001.cdf"), Path("imap_ultra_l1c_u45-pset_20251011_v001.cdf"), Path("imap_ultra_l1c_u45-pset_20251012_v001.cdf"), - l2_energy_bin_group_sizes_file_path + l2_energy_bin_group_sizes_file_path, ] expected_90_file_paths = [ u90_l2_path, Path("imap_ultra_l1c_u90-pset_20251010_v001.cdf"), Path("imap_ultra_l1c_u90-pset_20251011_v001.cdf"), Path("imap_ultra_l1c_u90-pset_20251012_v001.cdf"), - l2_energy_bin_group_sizes_file_path + l2_energy_bin_group_sizes_file_path, ] self.assertEqual(u45_deps.dependency_file_paths, expected_45_file_paths) self.assertEqual(u90_deps.dependency_file_paths, expected_90_file_paths) - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.load_or_create_healpix_l2') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.RectangularIntensityMapData.read_from_path') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.UltraL1CPSet.read_from_path') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.imap_data_access.download') - def test_fetch_dependencies_without_ancillary_file(self, mock_download, - mock_l1c_read_from_path, mock_l2_read_from_path, - mock_load_or_create_healpix_l2): - u45_map_file_name = 'imap_ultra_l2_u45-cool-descriptor_20250601_v000.cdf' - u90_map_file_name = 'imap_ultra_l2_u90-cool-descriptor_20250601_v000.cdf' - u45_pset_file_name = ["imap_ultra_l1c_45sensor-spacecraftpset_20251010_v001.cdf", - "imap_ultra_l1c_45sensor-spacecraftpset_20251011_v001.cdf", - "imap_ultra_l1c_45sensor-spacecraftpset_20251012_v001.cdf"] - u90_pset_file_name = ["imap_ultra_l1c_90sensor-spacecraftpset_20251010_v001.cdf", - "imap_ultra_l1c_90sensor-spacecraftpset_20251011_v001.cdf", - "imap_ultra_l1c_90sensor-spacecraftpset_20251012_v001.cdf"] + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.load_or_create_healpix_l2") + @patch( + "imap_l3_processing.ultra.ultra_l3_dependencies.RectangularIntensityMapData.read_from_path" + ) + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.UltraL1CPSet.read_from_path") + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.imap_data_access.download") + def test_fetch_dependencies_without_ancillary_file( + self, + mock_download, + mock_l1c_read_from_path, + mock_l2_read_from_path, + mock_load_or_create_healpix_l2, + ): + u45_map_file_name = "imap_ultra_l2_u45-cool-descriptor_20250601_v000.cdf" + u90_map_file_name = "imap_ultra_l2_u90-cool-descriptor_20250601_v000.cdf" + u45_pset_file_name = [ + "imap_ultra_l1c_45sensor-spacecraftpset_20251010_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20251011_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20251012_v001.cdf", + ] + u90_pset_file_name = [ + "imap_ultra_l1c_90sensor-spacecraftpset_20251010_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20251011_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20251012_v001.cdf", + ] u45_map_input = ScienceInput(u45_map_file_name) u90_map_input = ScienceInput(u90_map_file_name) u45_pset_inputs = [ScienceInput(pset) for pset in u45_pset_file_name] u90_pset_inputs = [ScienceInput(pset) for pset in u90_pset_file_name] - processing_input_collection = ProcessingInputCollection(u45_map_input, u90_map_input, *u45_pset_inputs, - *u90_pset_inputs) + processing_input_collection = ProcessingInputCollection( + u45_map_input, u90_map_input, *u45_pset_inputs, *u90_pset_inputs + ) expected_file_paths = [ Path("imap_ultra_l1c_u45-pset_20251010_v001.cdf"), @@ -472,34 +694,54 @@ def test_fetch_dependencies_without_ancillary_file(self, mock_download, mock_download.side_effect = expected_file_paths - combined_dependencies = UltraL3CombinedDependencies.fetch_dependencies(processing_input_collection) + combined_dependencies = UltraL3CombinedDependencies.fetch_dependencies( + processing_input_collection + ) self.assertIsNone(combined_dependencies.u45_dependencies.energy_bin_group_sizes) self.assertIsNone(combined_dependencies.u90_dependencies.energy_bin_group_sizes) - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.load_or_create_healpix_l2') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.UltraGlowsL3eData.read_from_path') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.UltraL1CPSet.read_from_path') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.imap_data_access.download') - @patch('imap_l3_processing.ultra.ultra_l3_dependencies.RectangularIntensityMapData.read_from_path') - def test_fetch_dependencies_with_glows_files(self, mock_l2_read_from_paths, mock_download, - mock_l1c_read_from_path, mock_l3e_read_from_path, - mock_load_or_create_healpix_l2, - ): + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.load_or_create_healpix_l2") + @patch( + "imap_l3_processing.ultra.ultra_l3_dependencies.UltraGlowsL3eData.read_from_path" + ) + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.UltraL1CPSet.read_from_path") + @patch("imap_l3_processing.ultra.ultra_l3_dependencies.imap_data_access.download") + @patch( + "imap_l3_processing.ultra.ultra_l3_dependencies.RectangularIntensityMapData.read_from_path" + ) + def test_fetch_dependencies_with_glows_files( + self, + mock_l2_read_from_paths, + mock_download, + mock_l1c_read_from_path, + mock_l3e_read_from_path, + mock_load_or_create_healpix_l2, + ): cases = [["sf", "spacecraftpset"], ["hf", "heliopset"]] for frame, pset in cases: - with (self.subTest(frame)): - u45_map_file_name = 'imap_ultra_l2_u45-cool-descriptor_20250601_v000.cdf' - u90_map_file_name = 'imap_ultra_l2_u90-cool-descriptor_20250601_v000.cdf' - u45_pset_file_names = [f"imap_ultra_l1c_45sensor-{pset}_20251010_v001.cdf", - f"imap_ultra_l1c_45sensor-{pset}_20251011_v001.cdf", - f"imap_ultra_l1c_45sensor-{pset}_20251012_v001.cdf"] - u90_pset_file_names = [f"imap_ultra_l1c_90sensor-{pset}_20251010_v001.cdf", - f"imap_ultra_l1c_90sensor-{pset}_20251011_v001.cdf", - f"imap_ultra_l1c_90sensor-{pset}_20251012_v001.cdf"] - glows_file_names = [f"imap_glows_l3e_survival-probability-ul-{frame}_20251010_v001.cdf", - f"imap_glows_l3e_survival-probability-ul-{frame}_20251011_v001.cdf", - f"imap_glows_l3e_survival-probability-ul-{frame}_20251012_v001.cdf"] + with self.subTest(frame): + u45_map_file_name = ( + "imap_ultra_l2_u45-cool-descriptor_20250601_v000.cdf" + ) + u90_map_file_name = ( + "imap_ultra_l2_u90-cool-descriptor_20250601_v000.cdf" + ) + u45_pset_file_names = [ + f"imap_ultra_l1c_45sensor-{pset}_20251010_v001.cdf", + f"imap_ultra_l1c_45sensor-{pset}_20251011_v001.cdf", + f"imap_ultra_l1c_45sensor-{pset}_20251012_v001.cdf", + ] + u90_pset_file_names = [ + f"imap_ultra_l1c_90sensor-{pset}_20251010_v001.cdf", + f"imap_ultra_l1c_90sensor-{pset}_20251011_v001.cdf", + f"imap_ultra_l1c_90sensor-{pset}_20251012_v001.cdf", + ] + glows_file_names = [ + f"imap_glows_l3e_survival-probability-ul-{frame}_20251010_v001.cdf", + f"imap_glows_l3e_survival-probability-ul-{frame}_20251011_v001.cdf", + f"imap_glows_l3e_survival-probability-ul-{frame}_20251012_v001.cdf", + ] u45_map_input = ScienceInput(u45_map_file_name) u90_map_input = ScienceInput(u90_map_file_name) @@ -524,14 +766,24 @@ def test_fetch_dependencies_with_glows_files(self, mock_l2_read_from_paths, mock Path("imap_ultra_l1c_u90-pset_20251011_v001.cdf"), Path("imap_ultra_l1c_u90-pset_20251012_v001.cdf"), ] - u45_l2_path = Path("imap_ultra_l2_u45-cool-descriptor_20250601_v000.cdf") - u90_l2_path = Path("imap_ultra_l2_u90-cool-descriptor_20250601_v000.cdf") + u45_l2_path = Path( + "imap_ultra_l2_u45-cool-descriptor_20250601_v000.cdf" + ) + u90_l2_path = Path( + "imap_ultra_l2_u90-cool-descriptor_20250601_v000.cdf" + ) expected_file_paths = [ *u45_l1c_paths, *u90_l1c_paths, - Path(f"imap_glows_l3e_survival-probability-ul-{frame}_20251010_v001.cdf"), - Path(f"imap_glows_l3e_survival-probability-ul-{frame}_20251011_v001.cdf"), - Path(f"imap_glows_l3e_survival-probability-ul-{frame}_20251012_v001.cdf"), + Path( + f"imap_glows_l3e_survival-probability-ul-{frame}_20251010_v001.cdf" + ), + Path( + f"imap_glows_l3e_survival-probability-ul-{frame}_20251011_v001.cdf" + ), + Path( + f"imap_glows_l3e_survival-probability-ul-{frame}_20251012_v001.cdf" + ), u45_l2_path, u90_l2_path, ] @@ -539,86 +791,164 @@ def test_fetch_dependencies_with_glows_files(self, mock_l2_read_from_paths, mock mock_download.side_effect = expected_file_paths mock_l1c_read_from_path.side_effect = [ - sentinel.u45_l1c_1, sentinel.u45_l1c_2, sentinel.u45_l1c_3, - sentinel.u90_l1c_1, sentinel.u90_l1c_2, sentinel.u90_l1c_3, + sentinel.u45_l1c_1, + sentinel.u45_l1c_2, + sentinel.u45_l1c_3, + sentinel.u90_l1c_1, + sentinel.u90_l1c_2, + sentinel.u90_l1c_3, ] mock_load_or_create_healpix_l2.side_effect = [ sentinel.u45_healpix_dataset, - sentinel.u90_healpix_dataset + sentinel.u90_healpix_dataset, + ] + mock_l2_read_from_paths.side_effect = [ + sentinel.u45_rectangular_l2, + sentinel.u90_rectangular_l2, ] - mock_l2_read_from_paths.side_effect = [sentinel.u45_rectangular_l2, sentinel.u90_rectangular_l2] mock_l3e_read_from_path.side_effect = [ - sentinel.glows_1, sentinel.glows_2, sentinel.glows_3, - sentinel.glows_1, sentinel.glows_2, sentinel.glows_3, + sentinel.glows_1, + sentinel.glows_2, + sentinel.glows_3, + sentinel.glows_1, + sentinel.glows_2, + sentinel.glows_3, ] - combined_dependencies = UltraL3CombinedDependencies.fetch_dependencies(processing_input_collection) - - mock_download.assert_has_calls([ - *[call(filename) for filename in u45_pset_file_names], - *[call(filename) for filename in u90_pset_file_names], - *[call(filename) for filename in glows_file_names], - call(u45_map_file_name), - call(u90_map_file_name) - ]) - - mock_l1c_read_from_path.assert_has_calls([ - call(Path("imap_ultra_l1c_u45-pset_20251010_v001.cdf")), - call(Path("imap_ultra_l1c_u45-pset_20251011_v001.cdf")), - call(Path("imap_ultra_l1c_u45-pset_20251012_v001.cdf")), - call(Path("imap_ultra_l1c_u90-pset_20251010_v001.cdf")), - call(Path("imap_ultra_l1c_u90-pset_20251011_v001.cdf")), - call(Path("imap_ultra_l1c_u90-pset_20251012_v001.cdf")), - ]) - - mock_l3e_read_from_path.assert_has_calls([ - call(Path(f"imap_glows_l3e_survival-probability-ul-{frame}_20251010_v001.cdf")), - call(Path(f"imap_glows_l3e_survival-probability-ul-{frame}_20251011_v001.cdf")), - call(Path(f"imap_glows_l3e_survival-probability-ul-{frame}_20251012_v001.cdf")), - call(Path(f"imap_glows_l3e_survival-probability-ul-{frame}_20251010_v001.cdf")), - call(Path(f"imap_glows_l3e_survival-probability-ul-{frame}_20251011_v001.cdf")), - call(Path(f"imap_glows_l3e_survival-probability-ul-{frame}_20251012_v001.cdf")), - ]) - - mock_l2_read_from_paths.assert_has_calls([ - call(u45_l2_path), - call(u90_l2_path), - ]) - - mock_load_or_create_healpix_l2.assert_has_calls([ - call(u45_l2_path, u45_l1c_paths), - call(u90_l2_path, u90_l1c_paths), - ]) + combined_dependencies = UltraL3CombinedDependencies.fetch_dependencies( + processing_input_collection + ) + + mock_download.assert_has_calls( + [ + *[call(filename) for filename in u45_pset_file_names], + *[call(filename) for filename in u90_pset_file_names], + *[call(filename) for filename in glows_file_names], + call(u45_map_file_name), + call(u90_map_file_name), + ] + ) + + mock_l1c_read_from_path.assert_has_calls( + [ + call(Path("imap_ultra_l1c_u45-pset_20251010_v001.cdf")), + call(Path("imap_ultra_l1c_u45-pset_20251011_v001.cdf")), + call(Path("imap_ultra_l1c_u45-pset_20251012_v001.cdf")), + call(Path("imap_ultra_l1c_u90-pset_20251010_v001.cdf")), + call(Path("imap_ultra_l1c_u90-pset_20251011_v001.cdf")), + call(Path("imap_ultra_l1c_u90-pset_20251012_v001.cdf")), + ] + ) + + mock_l3e_read_from_path.assert_has_calls( + [ + call( + Path( + f"imap_glows_l3e_survival-probability-ul-{frame}_20251010_v001.cdf" + ) + ), + call( + Path( + f"imap_glows_l3e_survival-probability-ul-{frame}_20251011_v001.cdf" + ) + ), + call( + Path( + f"imap_glows_l3e_survival-probability-ul-{frame}_20251012_v001.cdf" + ) + ), + call( + Path( + f"imap_glows_l3e_survival-probability-ul-{frame}_20251010_v001.cdf" + ) + ), + call( + Path( + f"imap_glows_l3e_survival-probability-ul-{frame}_20251011_v001.cdf" + ) + ), + call( + Path( + f"imap_glows_l3e_survival-probability-ul-{frame}_20251012_v001.cdf" + ) + ), + ] + ) + + mock_l2_read_from_paths.assert_has_calls( + [ + call(u45_l2_path), + call(u90_l2_path), + ] + ) + + mock_load_or_create_healpix_l2.assert_has_calls( + [ + call(u45_l2_path, u45_l1c_paths), + call(u90_l2_path, u90_l1c_paths), + ] + ) u45_deps = combined_dependencies.u45_dependencies u90_deps = combined_dependencies.u90_dependencies - self.assertEqual(u45_deps.ultra_l2_healpix_map, sentinel.u45_healpix_dataset) - self.assertEqual(u90_deps.ultra_l2_healpix_map, sentinel.u90_healpix_dataset) - self.assertEqual(u45_deps.ultra_l2_rectangular_map, sentinel.u45_rectangular_l2) - self.assertEqual(u90_deps.ultra_l2_rectangular_map, sentinel.u90_rectangular_l2) - self.assertEqual(u45_deps.ultra_l1c_pset, - [sentinel.u45_l1c_1, sentinel.u45_l1c_2, sentinel.u45_l1c_3]) - self.assertEqual(u90_deps.ultra_l1c_pset, - [sentinel.u90_l1c_1, sentinel.u90_l1c_2, sentinel.u90_l1c_3]) - self.assertEqual(u45_deps.glows_l3e_sp, - [sentinel.glows_1, sentinel.glows_2, sentinel.glows_3]) - self.assertEqual(u90_deps.glows_l3e_sp, - [sentinel.glows_1, sentinel.glows_2, sentinel.glows_3]) + self.assertEqual( + u45_deps.ultra_l2_healpix_map, sentinel.u45_healpix_dataset + ) + self.assertEqual( + u90_deps.ultra_l2_healpix_map, sentinel.u90_healpix_dataset + ) + self.assertEqual( + u45_deps.ultra_l2_rectangular_map, sentinel.u45_rectangular_l2 + ) + self.assertEqual( + u90_deps.ultra_l2_rectangular_map, sentinel.u90_rectangular_l2 + ) + self.assertEqual( + u45_deps.ultra_l1c_pset, + [sentinel.u45_l1c_1, sentinel.u45_l1c_2, sentinel.u45_l1c_3], + ) + self.assertEqual( + u90_deps.ultra_l1c_pset, + [sentinel.u90_l1c_1, sentinel.u90_l1c_2, sentinel.u90_l1c_3], + ) + self.assertEqual( + u45_deps.glows_l3e_sp, + [sentinel.glows_1, sentinel.glows_2, sentinel.glows_3], + ) + self.assertEqual( + u90_deps.glows_l3e_sp, + [sentinel.glows_1, sentinel.glows_2, sentinel.glows_3], + ) expected_u45_file_paths = [ u45_l2_path, *u45_l1c_paths, - Path(f"imap_glows_l3e_survival-probability-ul-{frame}_20251010_v001.cdf"), - Path(f"imap_glows_l3e_survival-probability-ul-{frame}_20251011_v001.cdf"), - Path(f"imap_glows_l3e_survival-probability-ul-{frame}_20251012_v001.cdf"), + Path( + f"imap_glows_l3e_survival-probability-ul-{frame}_20251010_v001.cdf" + ), + Path( + f"imap_glows_l3e_survival-probability-ul-{frame}_20251011_v001.cdf" + ), + Path( + f"imap_glows_l3e_survival-probability-ul-{frame}_20251012_v001.cdf" + ), ] expected_u90_file_paths = [ u90_l2_path, *u90_l1c_paths, - Path(f"imap_glows_l3e_survival-probability-ul-{frame}_20251010_v001.cdf"), - Path(f"imap_glows_l3e_survival-probability-ul-{frame}_20251011_v001.cdf"), - Path(f"imap_glows_l3e_survival-probability-ul-{frame}_20251012_v001.cdf"), + Path( + f"imap_glows_l3e_survival-probability-ul-{frame}_20251010_v001.cdf" + ), + Path( + f"imap_glows_l3e_survival-probability-ul-{frame}_20251011_v001.cdf" + ), + Path( + f"imap_glows_l3e_survival-probability-ul-{frame}_20251012_v001.cdf" + ), ] - self.assertEqual(u45_deps.dependency_file_paths, expected_u45_file_paths) - self.assertEqual(u90_deps.dependency_file_paths, expected_u90_file_paths) - + self.assertEqual( + u45_deps.dependency_file_paths, expected_u45_file_paths + ) + self.assertEqual( + u90_deps.dependency_file_paths, expected_u90_file_paths + ) diff --git a/tests/ultra/test_ultra_l3_survival_probabillity.py b/tests/ultra/test_ultra_l3_survival_probabillity.py index 12e6dbead..ffb7514bb 100644 --- a/tests/ultra/test_ultra_l3_survival_probabillity.py +++ b/tests/ultra/test_ultra_l3_survival_probabillity.py @@ -15,16 +15,23 @@ ) from imap_l3_processing.glows.quality_flags import GlowsL3Flags from imap_l3_processing.ultra.models import UltraL1CPSet, UltraGlowsL3eData -from imap_l3_processing.ultra.science.ultra_survival_probability import UltraSurvivalProbability, \ - UltraSurvivalProbabilitySkyMap +from imap_l3_processing.ultra.science.ultra_survival_probability import ( + UltraSurvivalProbability, + UltraSurvivalProbabilitySkyMap, +) from tests.spice_test_case import SpiceTestCase class TestUltraSurvivalProbability(unittest.TestCase): - - @patch('imap_l3_processing.ultra.science.ultra_survival_probability.geometry.frame_transform_az_el') - @patch('imap_l3_processing.ultra.science.ultra_survival_probability.spiceypy.unitim') - def test_ultra_survival_probability_pset_calls_super(self, _, mock_frame_transform_az_el): + @patch( + "imap_l3_processing.ultra.science.ultra_survival_probability.geometry.frame_transform_az_el" + ) + @patch( + "imap_l3_processing.ultra.science.ultra_survival_probability.spiceypy.unitim" + ) + def test_ultra_survival_probability_pset_calls_super( + self, _, mock_frame_transform_az_el + ): input_l1c_pset = _create_ultra_l1c_pset(np.array([2]), np.full((1, 1, 12), 1)) glows = _build_glows_l3e_ultra() @@ -36,80 +43,178 @@ def test_ultra_survival_probability_pset_calls_super(self, _, mock_frame_transfo self.assertEqual(input_l1c_pset.to_xarray(), prod.data) self.assertEqual(geometry.SpiceFrame.IMAP_DPS, prod.spice_reference_frame) - @patch('imap_l3_processing.ultra.science.ultra_survival_probability.geometry.frame_transform_az_el') - @patch('imap_l3_processing.ultra.science.ultra_survival_probability.spiceypy.unitim') - @patch.object(UltraConstants, 'PSET_ENERGY_BIN_EDGES', new=[0.5, 1.5, 4]) - def test_ultra_survival_probability_rotates_to_glows_frame(self, mock_unitim, mock_frame_transform_az_el): + @patch( + "imap_l3_processing.ultra.science.ultra_survival_probability.geometry.frame_transform_az_el" + ) + @patch( + "imap_l3_processing.ultra.science.ultra_survival_probability.spiceypy.unitim" + ) + @patch.object(UltraConstants, "PSET_ENERGY_BIN_EDGES", new=[0.5, 1.5, 4]) + def test_ultra_survival_probability_rotates_to_glows_frame( + self, mock_unitim, mock_frame_transform_az_el + ): glows_energies = np.array([1, 2]) input_l1c_pset = _create_ultra_l1c_pset(glows_energies, np.full((1, 2, 12), 1)) glows_surv_prob = np.array([[[1] * 6 + [0] * 6, [2] * 6 + [3] * 6]]) - glows = _build_glows_l3e_ultra(nside=1, survival_probabilities=glows_surv_prob, energies=glows_energies) + glows = _build_glows_l3e_ultra( + nside=1, survival_probabilities=glows_surv_prob, energies=glows_energies + ) rotate_el_by_180 = np.array([[1, -1]] * 12) - mock_frame_transform_az_el.side_effect = lambda et, az_el, to, _from: az_el * rotate_el_by_180 + mock_frame_transform_az_el.side_effect = lambda et, az_el, to, _from: ( + az_el * rotate_el_by_180 + ) prod = UltraSurvivalProbability(input_l1c_pset, glows, bin_groups=[0, 1, 2]) expected_tt = (input_l1c_pset.epoch - TT2000_EPOCH).total_seconds() + 43200 mock_unitim.assert_called_once_with(expected_tt, "TT", "ET") - mock_frame_transform_az_el.assert_called_once_with(mock_unitim.return_value, prod.az_el_points, - geometry.SpiceFrame.IMAP_DPS, - geometry.SpiceFrame.ECLIPJ2000) - expected_survival_probabilities_values = np.array([[[0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1], - [3, 3, 3, 3, 2, 2, 3, 3, 2, 2, 2, 2]]]) - survival_probability_times_exposure = prod.data['survival_probability_times_exposure'] - np.testing.assert_array_equal(survival_probability_times_exposure.values, - expected_survival_probabilities_values) - - @patch('imap_l3_processing.ultra.science.ultra_survival_probability.spiceypy.unitim') - @patch('imap_l3_processing.ultra.science.ultra_survival_probability.geometry.frame_transform_az_el') - @patch.object(UltraConstants, 'PSET_ENERGY_BIN_EDGES', new=[.1, 10]) - def test_ultra_survival_probability_is_multiplied_by_exposure(self, mock_frame_transform_az_el, _): + mock_frame_transform_az_el.assert_called_once_with( + mock_unitim.return_value, + prod.az_el_points, + geometry.SpiceFrame.IMAP_DPS, + geometry.SpiceFrame.ECLIPJ2000, + ) + expected_survival_probabilities_values = np.array( + [ + [ + [0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1], + [3, 3, 3, 3, 2, 2, 3, 3, 2, 2, 2, 2], + ] + ] + ) + survival_probability_times_exposure = prod.data[ + "survival_probability_times_exposure" + ] + np.testing.assert_array_equal( + survival_probability_times_exposure.values, + expected_survival_probabilities_values, + ) + + @patch( + "imap_l3_processing.ultra.science.ultra_survival_probability.spiceypy.unitim" + ) + @patch( + "imap_l3_processing.ultra.science.ultra_survival_probability.geometry.frame_transform_az_el" + ) + @patch.object(UltraConstants, "PSET_ENERGY_BIN_EDGES", new=[0.1, 10]) + def test_ultra_survival_probability_is_multiplied_by_exposure( + self, mock_frame_transform_az_el, _ + ): glows_energies = np.array([1]) input_l1c_pset = _create_ultra_l1c_pset(glows_energies, np.full((1, 1, 12), 2)) glows_surv_prob = np.array([[[1] * 6 + [0] * 6]]) - glows = _build_glows_l3e_ultra(nside=1, survival_probabilities=glows_surv_prob, energies=glows_energies) + glows = _build_glows_l3e_ultra( + nside=1, survival_probabilities=glows_surv_prob, energies=glows_energies + ) mock_frame_transform_az_el.side_effect = lambda et, az_el, to, _from: az_el prod = UltraSurvivalProbability(input_l1c_pset, glows, bin_groups=[0, 1]) - expected_survival_probabilities_values = np.array([[[2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0]]]) - survival_probability_times_exposure = prod.data['survival_probability_times_exposure'] - np.testing.assert_array_equal(survival_probability_times_exposure.values, - expected_survival_probabilities_values) - + expected_survival_probabilities_values = np.array( + [[[2, 2, 2, 2, 2, 2, 0, 0, 0, 0, 0, 0]]] + ) + survival_probability_times_exposure = prod.data[ + "survival_probability_times_exposure" + ] + np.testing.assert_array_equal( + survival_probability_times_exposure.values, + expected_survival_probabilities_values, + ) - @patch('imap_l3_processing.ultra.science.ultra_survival_probability.spiceypy.unitim') - @patch('imap_l3_processing.ultra.science.ultra_survival_probability.geometry.frame_transform_az_el') - @patch.object(UltraConstants, 'PSET_ENERGY_BIN_EDGES', new=[.1, 10]) - def test_ultra_survival_probability_pset_includes_quality_flags(self, mock_frame_transform_az_el, _): + @patch( + "imap_l3_processing.ultra.science.ultra_survival_probability.spiceypy.unitim" + ) + @patch( + "imap_l3_processing.ultra.science.ultra_survival_probability.geometry.frame_transform_az_el" + ) + @patch.object(UltraConstants, "PSET_ENERGY_BIN_EDGES", new=[0.1, 10]) + def test_ultra_survival_probability_pset_includes_quality_flags( + self, mock_frame_transform_az_el, _ + ): exposure = np.array([[[2] * 6 + [0] * 6]]) expected_unflagged = np.array([[[0] * 12]]) expected_flagged = exposure.copy() cases = [ - (GlowsL3Flags.NONE, expected_unflagged, expected_unflagged, expected_unflagged), - (GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO, expected_unflagged, expected_flagged, expected_unflagged), - (GlowsL3Flags.PREDICTIVE_EPHEMERIS, expected_flagged, expected_unflagged, expected_unflagged), - (GlowsL3Flags.PERSISTED_LAST_POINT, expected_unflagged, expected_unflagged, expected_flagged), - (GlowsL3Flags.PREDICTIVE_EPHEMERIS | GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO, expected_flagged, expected_flagged, expected_unflagged), - (GlowsL3Flags.PREDICTIVE_EPHEMERIS | GlowsL3Flags.PERSISTED_LAST_POINT, expected_flagged, expected_unflagged, expected_flagged), - (GlowsL3Flags.PERSISTED_LAST_POINT | GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO, expected_unflagged, expected_flagged, expected_flagged), - (GlowsL3Flags.PERSISTED_LAST_POINT | GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO | GlowsL3Flags.PREDICTIVE_EPHEMERIS, expected_flagged, expected_flagged, expected_flagged), + ( + GlowsL3Flags.NONE, + expected_unflagged, + expected_unflagged, + expected_unflagged, + ), + ( + GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO, + expected_unflagged, + expected_flagged, + expected_unflagged, + ), + ( + GlowsL3Flags.PREDICTIVE_EPHEMERIS, + expected_flagged, + expected_unflagged, + expected_unflagged, + ), + ( + GlowsL3Flags.PERSISTED_LAST_POINT, + expected_unflagged, + expected_unflagged, + expected_flagged, + ), + ( + GlowsL3Flags.PREDICTIVE_EPHEMERIS + | GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO, + expected_flagged, + expected_flagged, + expected_unflagged, + ), + ( + GlowsL3Flags.PREDICTIVE_EPHEMERIS | GlowsL3Flags.PERSISTED_LAST_POINT, + expected_flagged, + expected_unflagged, + expected_flagged, + ), + ( + GlowsL3Flags.PERSISTED_LAST_POINT + | GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO, + expected_unflagged, + expected_flagged, + expected_flagged, + ), + ( + GlowsL3Flags.PERSISTED_LAST_POINT + | GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO + | GlowsL3Flags.PREDICTIVE_EPHEMERIS, + expected_flagged, + expected_flagged, + expected_flagged, + ), ] - for flag, expected_predicted_ephem_flag, expected_nominal_alpha_proton_ratio_flag, expected_persisted_last_point_flag in cases: + for ( + flag, + expected_predicted_ephem_flag, + expected_nominal_alpha_proton_ratio_flag, + expected_persisted_last_point_flag, + ) in cases: with self.subTest(flag): glows_energies = np.array([1]) input_l1c_pset = _create_ultra_l1c_pset(glows_energies, exposure) glows_surv_prob = np.full((1, 1, 12), 2) - glows = _build_glows_l3e_ultra(nside=1, survival_probabilities=glows_surv_prob, energies=glows_energies, flags=flag) + glows = _build_glows_l3e_ultra( + nside=1, + survival_probabilities=glows_surv_prob, + energies=glows_energies, + flags=flag, + ) - mock_frame_transform_az_el.side_effect = lambda et, az_el, to, _from: az_el + mock_frame_transform_az_el.side_effect = lambda et, az_el, to, _from: ( + az_el + ) prod = UltraSurvivalProbability( input_l1c_pset, glows, bin_groups=[0, 1] @@ -123,154 +228,296 @@ def test_ultra_survival_probability_pset_includes_quality_flags(self, mock_frame prod.data["survival_probability_times_exposure"].dims, ) np.testing.assert_array_equal( - prod.data["nominal_alpha_proton_ratio_flag"], expected_nominal_alpha_proton_ratio_flag + prod.data["nominal_alpha_proton_ratio_flag"], + expected_nominal_alpha_proton_ratio_flag, ) np.testing.assert_array_equal( - prod.data["persisted_last_point_flag"], expected_persisted_last_point_flag + prod.data["persisted_last_point_flag"], + expected_persisted_last_point_flag, ) self.assertEqual( prod.data["nominal_alpha_proton_ratio_flag"].dims, prod.data["survival_probability_times_exposure"].dims, ) - @patch('imap_l3_processing.ultra.science.ultra_survival_probability.spiceypy.unitim') - @patch('imap_l3_processing.ultra.science.ultra_survival_probability.geometry.frame_transform_az_el') - @patch.object(UltraConstants, 'PSET_ENERGY_BIN_EDGES', new=[1, 100, 10000]) - def test_ultra_survival_probability_interpolates_over_energy(self, mock_frame_transform_az_el, _): + @patch( + "imap_l3_processing.ultra.science.ultra_survival_probability.spiceypy.unitim" + ) + @patch( + "imap_l3_processing.ultra.science.ultra_survival_probability.geometry.frame_transform_az_el" + ) + @patch.object(UltraConstants, "PSET_ENERGY_BIN_EDGES", new=[1, 100, 10000]) + def test_ultra_survival_probability_interpolates_over_energy( + self, mock_frame_transform_az_el, _ + ): glows_energies = np.array([1, 100, 10000]) ultra_energies = np.array([10, 1000]) input_l1c_pset = _create_ultra_l1c_pset(ultra_energies, np.full((1, 2, 12), 1)) glows_surv_prob = np.array([[[1] * 12, [3] * 12, [5] * 12]]) - glows = _build_glows_l3e_ultra(nside=1, survival_probabilities=glows_surv_prob, energies=glows_energies) + glows = _build_glows_l3e_ultra( + nside=1, survival_probabilities=glows_surv_prob, energies=glows_energies + ) mock_frame_transform_az_el.side_effect = lambda et, az_el, to, _from: az_el prod = UltraSurvivalProbability(input_l1c_pset, glows, bin_groups=[0, 1, 2]) expected_survival_probabilities_values = np.array([[[2] * 12, [4] * 12]]) - survival_probability_times_exposure = prod.data['survival_probability_times_exposure'] - np.testing.assert_array_equal(survival_probability_times_exposure.values, - expected_survival_probabilities_values) - - @patch('imap_l3_processing.ultra.science.ultra_survival_probability.spiceypy.unitim') - @patch('imap_l3_processing.ultra.science.ultra_survival_probability.geometry.frame_transform_az_el') - @patch.object(UltraConstants, 'PSET_ENERGY_BIN_EDGES', new=[1, 1, 1, 1, 100, 100, 100, 100, 10000]) - def test_ultra_survival_probability_combines_fine_energy_bins(self, mock_frame_transform_az_el, _): + survival_probability_times_exposure = prod.data[ + "survival_probability_times_exposure" + ] + np.testing.assert_array_equal( + survival_probability_times_exposure.values, + expected_survival_probabilities_values, + ) + + @patch( + "imap_l3_processing.ultra.science.ultra_survival_probability.spiceypy.unitim" + ) + @patch( + "imap_l3_processing.ultra.science.ultra_survival_probability.geometry.frame_transform_az_el" + ) + @patch.object( + UltraConstants, + "PSET_ENERGY_BIN_EDGES", + new=[1, 1, 1, 1, 100, 100, 100, 100, 10000], + ) + def test_ultra_survival_probability_combines_fine_energy_bins( + self, mock_frame_transform_az_el, _ + ): glows_energies = np.array([1, 100, 10000]) ultra_energies = np.array([10, 10, 10, 10, 1000, 1000, 1000, 1000]) - input_l1c_pset = _create_ultra_l1c_pset(ultra_energies, np.full((1, 2 * 4, 12), 1)) + input_l1c_pset = _create_ultra_l1c_pset( + ultra_energies, np.full((1, 2 * 4, 12), 1) + ) glows_surv_prob = np.array([[[1] * 12, [3] * 12, [5] * 12]]) - glows = _build_glows_l3e_ultra(nside=1, survival_probabilities=glows_surv_prob, energies=glows_energies) + glows = _build_glows_l3e_ultra( + nside=1, survival_probabilities=glows_surv_prob, energies=glows_energies + ) mock_frame_transform_az_el.side_effect = lambda et, az_el, to, _from: az_el prod = UltraSurvivalProbability(input_l1c_pset, glows) expected_survival_probabilities_values = np.array([[[2] * 12, [4] * 12]]) - survival_probability_times_exposure = prod.data['survival_probability_times_exposure'] - np.testing.assert_array_equal(survival_probability_times_exposure.values, - expected_survival_probabilities_values) + survival_probability_times_exposure = prod.data[ + "survival_probability_times_exposure" + ] + np.testing.assert_array_equal( + survival_probability_times_exposure.values, + expected_survival_probabilities_values, + ) - @patch('imap_l3_processing.ultra.science.ultra_survival_probability.spiceypy.unitim') - @patch('imap_l3_processing.ultra.science.ultra_survival_probability.geometry.frame_transform_az_el') - def test_ultra_survival_probability_handles_different_resolutions(self, mock_frame_transform_az_el, _): + @patch( + "imap_l3_processing.ultra.science.ultra_survival_probability.spiceypy.unitim" + ) + @patch( + "imap_l3_processing.ultra.science.ultra_survival_probability.geometry.frame_transform_az_el" + ) + def test_ultra_survival_probability_handles_different_resolutions( + self, mock_frame_transform_az_el, _ + ): glows_energies = np.array([1]) ultra_energies = np.array([10]) input_l1c_pset = _create_ultra_l1c_pset(ultra_energies, np.full((1, 1, 48), 1)) glows_surv_prob = np.array([[np.arange(12)]]) - glows = _build_glows_l3e_ultra(nside=1, survival_probabilities=glows_surv_prob, energies=glows_energies) + glows = _build_glows_l3e_ultra( + nside=1, survival_probabilities=glows_surv_prob, energies=glows_energies + ) mock_frame_transform_az_el.side_effect = lambda et, az_el, to, _from: az_el prod = UltraSurvivalProbability(input_l1c_pset, glows, bin_groups=[0, 1]) - expected_survival_probabilities_values = np.array([[[0.7667581, 1.25558603, 1.74441397, 2.2332419, 0.75, 0.25, - 0.75, 1.25, 1.75, 2.25, 2.75, 2.25, - 2.83574061, 2.4043331, 2.9043331, 3.4043331, 3.9043331, - 4.4043331, - 4.9043331, 4.33574061, 4.25, 4.75, 5.25, 5.75, - 6.25, 6.75, 6.25, 4.75, 6.56137065, 6.12996314, - 6.62996314, 7.12996314, 7.62996314, 8.12996314, 8.62996314, - 8.06137065, - 8.75, 8.25, 8.75, 9.25, 9.75, 10.25, - 10.75, 10.25, 8.7667581, 9.25558603, 9.74441397, - 10.2332419]]]) - survival_probability_times_exposure = prod.data['survival_probability_times_exposure'] - np.testing.assert_array_almost_equal(survival_probability_times_exposure.values, - expected_survival_probabilities_values) + expected_survival_probabilities_values = np.array( + [ + [ + [ + 0.7667581, + 1.25558603, + 1.74441397, + 2.2332419, + 0.75, + 0.25, + 0.75, + 1.25, + 1.75, + 2.25, + 2.75, + 2.25, + 2.83574061, + 2.4043331, + 2.9043331, + 3.4043331, + 3.9043331, + 4.4043331, + 4.9043331, + 4.33574061, + 4.25, + 4.75, + 5.25, + 5.75, + 6.25, + 6.75, + 6.25, + 4.75, + 6.56137065, + 6.12996314, + 6.62996314, + 7.12996314, + 7.62996314, + 8.12996314, + 8.62996314, + 8.06137065, + 8.75, + 8.25, + 8.75, + 9.25, + 9.75, + 10.25, + 10.75, + 10.25, + 8.7667581, + 9.25558603, + 9.74441397, + 10.2332419, + ] + ] + ] + ) + survival_probability_times_exposure = prod.data[ + "survival_probability_times_exposure" + ] + np.testing.assert_array_almost_equal( + survival_probability_times_exposure.values, + expected_survival_probabilities_values, + ) class TestUltraSurvivalProbabilitySkyMap(SpiceTestCase): def test_ultra_survival_probability_skymap(self): pointing_set_nside = 2 - pointing_set_pixels = 12 * pointing_set_nside ** 2 + pointing_set_pixels = 12 * pointing_set_nside**2 l1c_exposure_1 = np.full((1, 1, pointing_set_pixels), 0.5) l1c_exposure_2 = np.full((1, 1, pointing_set_pixels), 1) l3e_sp_1 = np.full((1, 1, pointing_set_pixels), 0.5) l3e_sp_2 = np.full((1, 1, pointing_set_pixels), 0.25) glows_energies = np.array([1]) - l1c_1 = _create_ultra_l1c_pset(energy=np.array([1]), exposure_factor=l1c_exposure_1) - l3e_glows_1 = _build_glows_l3e_ultra(survival_probabilities=l3e_sp_1, energies=glows_energies, - nside=pointing_set_nside, flags=GlowsL3Flags.PREDICTIVE_EPHEMERIS) - l1c_2 = _create_ultra_l1c_pset(energy=np.array([1]), exposure_factor=l1c_exposure_2) - l3e_glows_2 = _build_glows_l3e_ultra(survival_probabilities=l3e_sp_2, energies=glows_energies, - nside=pointing_set_nside, flags=GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO | GlowsL3Flags.PERSISTED_LAST_POINT) + l1c_1 = _create_ultra_l1c_pset( + energy=np.array([1]), exposure_factor=l1c_exposure_1 + ) + l3e_glows_1 = _build_glows_l3e_ultra( + survival_probabilities=l3e_sp_1, + energies=glows_energies, + nside=pointing_set_nside, + flags=GlowsL3Flags.PREDICTIVE_EPHEMERIS, + ) + l1c_2 = _create_ultra_l1c_pset( + energy=np.array([1]), exposure_factor=l1c_exposure_2 + ) + l3e_glows_2 = _build_glows_l3e_ultra( + survival_probabilities=l3e_sp_2, + energies=glows_energies, + nside=pointing_set_nside, + flags=GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO + | GlowsL3Flags.PERSISTED_LAST_POINT, + ) pset_1 = UltraSurvivalProbability(l1c_1, l3e_glows_1, bin_groups=[0, 1]) pset_2 = UltraSurvivalProbability(l1c_2, l3e_glows_2, bin_groups=[0, 1]) output_nside = 1 - prod = UltraSurvivalProbabilitySkyMap([pset_1, pset_2], geometry.SpiceFrame.IMAP_DPS, output_nside) + prod = UltraSurvivalProbabilitySkyMap( + [pset_1, pset_2], geometry.SpiceFrame.IMAP_DPS, output_nside + ) self.assertIsInstance(prod, HealpixSkyMap) - exposure_weighted_survival_probabilities = prod.data_1d["exposure_weighted_survival_probabilities"].values - expected_output_pixels = 12 * output_nside ** 2 - expected_exposure_weighted_survival_probabilities = np.full((1, 1, expected_output_pixels), 1 / 3) - np.testing.assert_array_almost_equal(exposure_weighted_survival_probabilities, - expected_exposure_weighted_survival_probabilities) + exposure_weighted_survival_probabilities = prod.data_1d[ + "exposure_weighted_survival_probabilities" + ].values + expected_output_pixels = 12 * output_nside**2 + expected_exposure_weighted_survival_probabilities = np.full( + (1, 1, expected_output_pixels), 1 / 3 + ) + np.testing.assert_array_almost_equal( + exposure_weighted_survival_probabilities, + expected_exposure_weighted_survival_probabilities, + ) actual_quality_flags = prod.data_1d["quality_flags"].values - expected_quality_flag = GlowsL3Flags.PREDICTIVE_EPHEMERIS | GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO | GlowsL3Flags.PERSISTED_LAST_POINT - np.testing.assert_equal(actual_quality_flags, np.full((1, 1, expected_output_pixels), expected_quality_flag)) - + expected_quality_flag = ( + GlowsL3Flags.PREDICTIVE_EPHEMERIS + | GlowsL3Flags.NOMINAL_ALPHA_PROTON_RATIO + | GlowsL3Flags.PERSISTED_LAST_POINT + ) + np.testing.assert_equal( + actual_quality_flags, + np.full((1, 1, expected_output_pixels), expected_quality_flag), + ) def test_ultra_survival_probability_skymap_ignores_nan_survival_probabilities(self): pointing_set_nside = 2 - pointing_set_pixels = 12 * pointing_set_nside ** 2 + pointing_set_pixels = 12 * pointing_set_nside**2 l1c_exposure_1 = np.full((1, 1, pointing_set_pixels), 0.5) l1c_exposure_2 = np.full((1, 1, pointing_set_pixels), 1) l3e_sp_1 = np.full((1, 1, pointing_set_pixels), 0.5) l3e_sp_2 = np.full((1, 1, pointing_set_pixels), np.nan) glows_energies = np.array([1]) - l1c_1 = _create_ultra_l1c_pset(energy=np.array([1]), exposure_factor=l1c_exposure_1) - l3e_glows_1 = _build_glows_l3e_ultra(survival_probabilities=l3e_sp_1, energies=glows_energies, - nside=pointing_set_nside) - l1c_2 = _create_ultra_l1c_pset(energy=np.array([1]), exposure_factor=l1c_exposure_2) - l3e_glows_2 = _build_glows_l3e_ultra(survival_probabilities=l3e_sp_2, energies=glows_energies, - nside=pointing_set_nside) + l1c_1 = _create_ultra_l1c_pset( + energy=np.array([1]), exposure_factor=l1c_exposure_1 + ) + l3e_glows_1 = _build_glows_l3e_ultra( + survival_probabilities=l3e_sp_1, + energies=glows_energies, + nside=pointing_set_nside, + ) + l1c_2 = _create_ultra_l1c_pset( + energy=np.array([1]), exposure_factor=l1c_exposure_2 + ) + l3e_glows_2 = _build_glows_l3e_ultra( + survival_probabilities=l3e_sp_2, + energies=glows_energies, + nside=pointing_set_nside, + ) pset_1 = UltraSurvivalProbability(l1c_1, l3e_glows_1, bin_groups=[0, 1]) pset_2 = UltraSurvivalProbability(l1c_2, l3e_glows_2, bin_groups=[0, 1]) output_nside = 1 - prod = UltraSurvivalProbabilitySkyMap([pset_1, pset_2], geometry.SpiceFrame.IMAP_DPS, output_nside) + prod = UltraSurvivalProbabilitySkyMap( + [pset_1, pset_2], geometry.SpiceFrame.IMAP_DPS, output_nside + ) self.assertIsInstance(prod, HealpixSkyMap) - exposure_weighted_survival_probabilities = prod.data_1d["exposure_weighted_survival_probabilities"].values - expected_output_pixels = 12 * output_nside ** 2 - expected_exposure_weighted_survival_probabilities = np.full((1, 1, expected_output_pixels), 0.5) - np.testing.assert_array_almost_equal(exposure_weighted_survival_probabilities, - expected_exposure_weighted_survival_probabilities) - - -def _build_glows_l3e_ultra(survival_probabilities: np.ndarray = None, energies: np.ndarray = None, nside: int = 16, flags: GlowsL3Flags = GlowsL3Flags.NONE): + exposure_weighted_survival_probabilities = prod.data_1d[ + "exposure_weighted_survival_probabilities" + ].values + expected_output_pixels = 12 * output_nside**2 + expected_exposure_weighted_survival_probabilities = np.full( + (1, 1, expected_output_pixels), 0.5 + ) + np.testing.assert_array_almost_equal( + exposure_weighted_survival_probabilities, + expected_exposure_weighted_survival_probabilities, + ) + + +def _build_glows_l3e_ultra( + survival_probabilities: np.ndarray = None, + energies: np.ndarray = None, + nside: int = 16, + flags: GlowsL3Flags = GlowsL3Flags.NONE, +): energies = np.arange(1, 21) if energies is None else energies - num_pixels = 12 * (nside ** 2) - survival_probabilities = np.random.rand(1, len(energies), - num_pixels) if survival_probabilities is None else survival_probabilities + num_pixels = 12 * (nside**2) + survival_probabilities = ( + np.random.rand(1, len(energies), num_pixels) + if survival_probabilities is None + else survival_probabilities + ) healpix_pixels = np.arange(num_pixels) healpix_grid = HEALPix(nside) @@ -282,18 +529,20 @@ def _build_glows_l3e_ultra(survival_probabilities: np.ndarray = None, energies: energy=energies, healpix_index=healpix_pixels, survival_probability=survival_probabilities, - flags=np.array([flags], dtype=np.uint16) + flags=np.array([flags], dtype=np.uint16), ) -def _create_ultra_l1c_pset(energy: np.ndarray, - exposure_factor: np.ndarray, - sensitivity: np.ndarray = None, - counts: np.ndarray = None, - latitude: np.ndarray = None, - longitude: np.ndarray = None, - epoch: datetime = None, - epoch_delta: int = ONE_SECOND_IN_NANOSECONDS * SECONDS_PER_DAY): +def _create_ultra_l1c_pset( + energy: np.ndarray, + exposure_factor: np.ndarray, + sensitivity: np.ndarray = None, + counts: np.ndarray = None, + latitude: np.ndarray = None, + longitude: np.ndarray = None, + epoch: datetime = None, + epoch_delta: int = ONE_SECOND_IN_NANOSECONDS * SECONDS_PER_DAY, +): epoch = datetime(2025, 10, 1, 0) if epoch is None else epoch sensitivity = sensitivity or np.full_like(exposure_factor[0], 1) healpix_index = np.arange(exposure_factor.shape[-1]) @@ -313,5 +562,5 @@ def _create_ultra_l1c_pset(energy: np.ndarray, return input_l1c_pset -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/tests/ultra/test_ultra_l3_to_rectangular_dependencies.py b/tests/ultra/test_ultra_l3_to_rectangular_dependencies.py index 05110d260..ec2d1be08 100644 --- a/tests/ultra/test_ultra_l3_to_rectangular_dependencies.py +++ b/tests/ultra/test_ultra_l3_to_rectangular_dependencies.py @@ -3,15 +3,18 @@ from imap_data_access import ScienceInput, ProcessingInputCollection -from imap_l3_processing.ultra.ultra_l3_to_rectangular_dependencies import UltraL3ToRectangularDependencies +from imap_l3_processing.ultra.ultra_l3_to_rectangular_dependencies import ( + UltraL3ToRectangularDependencies, +) class TestUltraL3ToRectangularDependencies(unittest.TestCase): - - @patch('imap_l3_processing.ultra.ultra_l3_to_rectangular_dependencies.HealPixIntensityMapData.read_from_path') - @patch('imap_l3_processing.ultra.ultra_l3_to_rectangular_dependencies.download') + @patch( + "imap_l3_processing.ultra.ultra_l3_to_rectangular_dependencies.HealPixIntensityMapData.read_from_path" + ) + @patch("imap_l3_processing.ultra.ultra_l3_to_rectangular_dependencies.download") def test_fetch_dependencies(self, mock_download, mock_healpix_read_from_path): - map_file_name = 'imap_ultra_l3_ultra-cool-descriptor_20250601_v000.cdf' + map_file_name = "imap_ultra_l3_ultra-cool-descriptor_20250601_v000.cdf" map_input = ScienceInput(map_file_name) processing_input_collection = ProcessingInputCollection(map_input) @@ -19,14 +22,21 @@ def test_fetch_dependencies(self, mock_download, mock_healpix_read_from_path): "map_file", ] - ultra_l3_to_rectangular_dependencies = UltraL3ToRectangularDependencies.fetch_dependencies( - processing_input_collection) + ultra_l3_to_rectangular_dependencies = ( + UltraL3ToRectangularDependencies.fetch_dependencies( + processing_input_collection + ) + ) - mock_download.assert_has_calls([ - call(map_file_name), - ]) + mock_download.assert_has_calls( + [ + call(map_file_name), + ] + ) mock_healpix_read_from_path.assert_called_once_with("map_file") - self.assertEqual(ultra_l3_to_rectangular_dependencies.healpix_map_data, - mock_healpix_read_from_path.return_value) + self.assertEqual( + ultra_l3_to_rectangular_dependencies.healpix_map_data, + mock_healpix_read_from_path.return_value, + ) diff --git a/tests/ultra/test_ultra_processor.py b/tests/ultra/test_ultra_processor.py index b21bb1e51..f997d9f2e 100644 --- a/tests/ultra/test_ultra_processor.py +++ b/tests/ultra/test_ultra_processor.py @@ -7,7 +7,11 @@ import numpy as np import xarray as xr -from imap_data_access.processing_input import AncillaryInput, ScienceInput, ProcessingInputCollection +from imap_data_access.processing_input import ( + AncillaryInput, + ScienceInput, + ProcessingInputCollection, +) from imap_processing.ena_maps.ena_maps import RectangularSkyMap from imap_processing.ena_maps.utils.coordinates import CoordNames from imap_processing.spice.geometry import SpiceFrame @@ -78,7 +82,9 @@ def test_correct_healpix_data_for_survival_probability( input_map_flux = rng.random((1, 9, 12)) epoch = datetime.now() - input_l2_healpix_map = _create_ultra_l2_healpix_data(epoch=[epoch], flux=input_map_flux, healpix_indices=healpix_indices) + input_l2_healpix_map = _create_ultra_l2_healpix_data( + epoch=[epoch], flux=input_map_flux, healpix_indices=healpix_indices + ) input_l2_healpix_map.intensity_map_data.energy = sentinel.ultra_l2_energies input_l2_rectangular_map = create_rectangular_intensity_map_data() @@ -110,10 +116,18 @@ def test_correct_healpix_data_for_survival_probability( sentinel.pset_3, ] computed_survival_probabilities = rng.random((1, 9, healpix_indices.shape[0])) - expected_quality_flags = np.full((1, 9, healpix_indices.shape[0]), MapL3Flags.NONE) - expected_quality_flags[rng.random((1, 9, healpix_indices.shape[0])) > 0.8] = MapL3Flags.PREDICTIVE_EPHEMERIS - expected_quality_flags[rng.random((1, 9, healpix_indices.shape[0])) > 0.7] = MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO - expected_quality_flags[rng.random((1, 9, healpix_indices.shape[0])) > 0.75] = MapL3Flags.PERSISTED_LAST_POINT + expected_quality_flags = np.full( + (1, 9, healpix_indices.shape[0]), MapL3Flags.NONE + ) + expected_quality_flags[rng.random((1, 9, healpix_indices.shape[0])) > 0.8] = ( + MapL3Flags.PREDICTIVE_EPHEMERIS + ) + expected_quality_flags[rng.random((1, 9, healpix_indices.shape[0])) > 0.7] = ( + MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO + ) + expected_quality_flags[rng.random((1, 9, healpix_indices.shape[0])) > 0.75] = ( + MapL3Flags.PERSISTED_LAST_POINT + ) mock_survival_skymap.return_value.to_dataset.return_value = xr.Dataset( { @@ -141,16 +155,37 @@ def test_correct_healpix_data_for_survival_probability( }, ) - healpix_intensity_map_data = correct_healpix_data_for_survival_probability(dependencies, SpiceFrame.ECLIPJ2000) + healpix_intensity_map_data = correct_healpix_data_for_survival_probability( + dependencies, SpiceFrame.ECLIPJ2000 + ) - mock_combine_glows_l3e_with_l1c_pointing.assert_called_once_with(sentinel.glows_l3e_sp, sentinel.ultra_l1c_pset) - mock_survival_probability_pointing_set.assert_has_calls([ - call(sentinel.ultra_l1c_1, sentinel.glows_l3e_1, bin_groups=sentinel.bin_groups), - call(sentinel.ultra_l1c_2, sentinel.glows_l3e_2, bin_groups=sentinel.bin_groups), - call(sentinel.ultra_l1c_3, sentinel.glows_l3e_3, bin_groups=sentinel.bin_groups) - ]) - mock_survival_skymap.assert_called_once_with([sentinel.pset_1, sentinel.pset_2, sentinel.pset_3], - SpiceFrame.ECLIPJ2000, input_l2_healpix_map.coords.nside) + mock_combine_glows_l3e_with_l1c_pointing.assert_called_once_with( + sentinel.glows_l3e_sp, sentinel.ultra_l1c_pset + ) + mock_survival_probability_pointing_set.assert_has_calls( + [ + call( + sentinel.ultra_l1c_1, + sentinel.glows_l3e_1, + bin_groups=sentinel.bin_groups, + ), + call( + sentinel.ultra_l1c_2, + sentinel.glows_l3e_2, + bin_groups=sentinel.bin_groups, + ), + call( + sentinel.ultra_l1c_3, + sentinel.glows_l3e_3, + bin_groups=sentinel.bin_groups, + ), + ] + ) + mock_survival_skymap.assert_called_once_with( + [sentinel.pset_1, sentinel.pset_2, sentinel.pset_3], + SpiceFrame.ECLIPJ2000, + input_l2_healpix_map.coords.nside, + ) mock_survival_skymap.return_value.to_dataset.assert_called_once_with() actual_intensity_map_data = healpix_intensity_map_data.intensity_map_data @@ -174,20 +209,47 @@ def test_correct_healpix_data_for_survival_probability( computed_survival_probabilities, ) - np.testing.assert_array_equal(actual_intensity_map_data.epoch, intensity_data.epoch) - np.testing.assert_array_equal(actual_intensity_map_data.epoch_delta, intensity_data.epoch_delta) - np.testing.assert_array_equal(actual_intensity_map_data.energy, intensity_data.energy) - np.testing.assert_array_equal(actual_intensity_map_data.energy_delta_plus, intensity_data.energy_delta_plus) - np.testing.assert_array_equal(actual_intensity_map_data.energy_delta_minus, - intensity_data.energy_delta_minus) - np.testing.assert_array_equal(actual_intensity_map_data.energy_label, intensity_data.energy_label) - np.testing.assert_array_equal(actual_intensity_map_data.latitude, intensity_data.latitude) - np.testing.assert_array_equal(actual_intensity_map_data.longitude, intensity_data.longitude) - np.testing.assert_array_equal(actual_intensity_map_data.exposure_factor, intensity_data.exposure_factor) - np.testing.assert_array_equal(actual_intensity_map_data.obs_date, intensity_data.obs_date) - np.testing.assert_array_equal(actual_intensity_map_data.obs_date_range, intensity_data.obs_date_range) - np.testing.assert_array_equal(actual_intensity_map_data.solid_angle, intensity_data.solid_angle) - np.testing.assert_array_equal(actual_intensity_map_data.quality_flags, expected_quality_flags) + np.testing.assert_array_equal( + actual_intensity_map_data.epoch, intensity_data.epoch + ) + np.testing.assert_array_equal( + actual_intensity_map_data.epoch_delta, intensity_data.epoch_delta + ) + np.testing.assert_array_equal( + actual_intensity_map_data.energy, intensity_data.energy + ) + np.testing.assert_array_equal( + actual_intensity_map_data.energy_delta_plus, + intensity_data.energy_delta_plus, + ) + np.testing.assert_array_equal( + actual_intensity_map_data.energy_delta_minus, + intensity_data.energy_delta_minus, + ) + np.testing.assert_array_equal( + actual_intensity_map_data.energy_label, intensity_data.energy_label + ) + np.testing.assert_array_equal( + actual_intensity_map_data.latitude, intensity_data.latitude + ) + np.testing.assert_array_equal( + actual_intensity_map_data.longitude, intensity_data.longitude + ) + np.testing.assert_array_equal( + actual_intensity_map_data.exposure_factor, intensity_data.exposure_factor + ) + np.testing.assert_array_equal( + actual_intensity_map_data.obs_date, intensity_data.obs_date + ) + np.testing.assert_array_equal( + actual_intensity_map_data.obs_date_range, intensity_data.obs_date_range + ) + np.testing.assert_array_equal( + actual_intensity_map_data.solid_angle, intensity_data.solid_angle + ) + np.testing.assert_array_equal( + actual_intensity_map_data.quality_flags, expected_quality_flags + ) coords = healpix_intensity_map_data.coords np.testing.assert_array_equal( coords.pixel_index, input_l2_healpix_map.coords.pixel_index @@ -209,7 +271,9 @@ def test_correct_healpix_data_for_survival_probability_caches_results( mock_survival_skymap, mock_healpix_intensity_map_data, ): - mock_healpix_intensity_map_data.return_value = sentinel.healpix_intensity_map_data + mock_healpix_intensity_map_data.return_value = ( + sentinel.healpix_intensity_map_data + ) rng = np.random.default_rng() healpix_indices = np.arange(12) @@ -271,20 +335,25 @@ def test_correct_healpix_data_for_survival_probability_caches_results( }, ) - healpix_intensity_map_data = correct_healpix_data_for_survival_probability(dependencies, SpiceFrame.ECLIPJ2000) + healpix_intensity_map_data = correct_healpix_data_for_survival_probability( + dependencies, SpiceFrame.ECLIPJ2000 + ) cache_dir = get_temp_cache_dir() - expected_cache_key = (f"['{input_l1c_pset_name_1}', '{input_l1c_pset_name_2}']" - f"['{input_glows_l3e_name_1}', '{input_glows_l3e_name_2}']" - "a-map-descriptor-nside32") - expected_cache_path = cache_dir / sha256(expected_cache_key.encode("utf-8")).hexdigest() + expected_cache_key = ( + f"['{input_l1c_pset_name_1}', '{input_l1c_pset_name_2}']" + f"['{input_glows_l3e_name_1}', '{input_glows_l3e_name_2}']" + "a-map-descriptor-nside32" + ) + expected_cache_path = ( + cache_dir / sha256(expected_cache_key.encode("utf-8")).hexdigest() + ) with open(expected_cache_path, "rb") as f: cached_data = pickle.load(f) self.assertEqual(healpix_intensity_map_data, cached_data) - @patch("imap_l3_processing.ultra.ultra_processor.HealPixIntensityMapData") @patch("imap_l3_processing.ultra.ultra_processor.UltraSurvivalProbabilitySkyMap") @patch("imap_l3_processing.ultra.ultra_processor.UltraSurvivalProbability") @@ -305,10 +374,14 @@ def test_correct_healpix_data_for_survival_probability_uses_cached_result( input_glows_l3e_name_2 = "imap_glows_l3e_a-glows-descriptor_20250602_v000.cdf" cache_dir = get_temp_cache_dir() - expected_cache_key = (f"['{input_l1c_pset_name_1}', '{input_l1c_pset_name_2}']" - f"['{input_glows_l3e_name_1}', '{input_glows_l3e_name_2}']" - "a-map-descriptor-nside32") - expected_cache_path = cache_dir / sha256(expected_cache_key.encode("utf-8")).hexdigest() + expected_cache_key = ( + f"['{input_l1c_pset_name_1}', '{input_l1c_pset_name_2}']" + f"['{input_glows_l3e_name_1}', '{input_glows_l3e_name_2}']" + "a-map-descriptor-nside32" + ) + expected_cache_path = ( + cache_dir / sha256(expected_cache_key.encode("utf-8")).hexdigest() + ) cached_result = sentinel.healpix_intensity_map_data with open(expected_cache_path, "wb") as f: @@ -341,7 +414,9 @@ def test_correct_healpix_data_for_survival_probability_uses_cached_result( energy_bin_group_sizes=sentinel.bin_groups, ) - healpix_intensity_map_data = correct_healpix_data_for_survival_probability(dependencies, SpiceFrame.ECLIPJ2000) + healpix_intensity_map_data = correct_healpix_data_for_survival_probability( + dependencies, SpiceFrame.ECLIPJ2000 + ) self.assertEqual(cached_result, healpix_intensity_map_data) mock_combine_glows_l3e_with_l1c_pointing.assert_not_called() @@ -349,13 +424,22 @@ def test_correct_healpix_data_for_survival_probability_uses_cached_result( mock_survival_skymap.assert_not_called() mock_healpix_intensity_map_data.assert_not_called() - @patch('imap_l3_processing.ultra.ultra_processor.correct_healpix_data_for_survival_probability') - @patch('imap_l3_processing.utils.spiceypy') - @patch('imap_l3_processing.ultra.ultra_processor.save_data') - @patch('imap_l3_processing.ultra.ultra_processor.UltraL3Dependencies.fetch_dependencies') - def _test_process_survival_probability(self, degree_spacing, mock_fetch_dependencies, - mock_save_data, mock_spiceypy, - mock_correct_healpix_data_for_survival_probability): + @patch( + "imap_l3_processing.ultra.ultra_processor.correct_healpix_data_for_survival_probability" + ) + @patch("imap_l3_processing.utils.spiceypy") + @patch("imap_l3_processing.ultra.ultra_processor.save_data") + @patch( + "imap_l3_processing.ultra.ultra_processor.UltraL3Dependencies.fetch_dependencies" + ) + def _test_process_survival_probability( + self, + degree_spacing, + mock_fetch_dependencies, + mock_save_data, + mock_spiceypy, + mock_correct_healpix_data_for_survival_probability, + ): rng = np.random.default_rng() healpix_indices = np.arange(12) mock_spiceypy.ktotal.return_value = 1 @@ -364,7 +448,9 @@ def _test_process_survival_probability(self, degree_spacing, mock_fetch_dependen input_map_flux = rng.random((1, 9, 12)) epoch = datetime.now() - input_l2_healpix_map = _create_ultra_l2_healpix_data(epoch=[epoch], flux=input_map_flux, healpix_indices=healpix_indices) + input_l2_healpix_map = _create_ultra_l2_healpix_data( + epoch=[epoch], flux=input_map_flux, healpix_indices=healpix_indices + ) input_l2_healpix_map.intensity_map_data.energy = sentinel.ultra_l2_energies input_l2_rectangular_map = create_rectangular_intensity_map_data() @@ -372,12 +458,14 @@ def _test_process_survival_probability(self, degree_spacing, mock_fetch_dependen input_l1c_pset_name = "imap_ultra_l1c_a-pset-descriptor_20250601_v000.cdf" input_glows_l3e_name = "imap_glows_l3e_a-glows-descriptor_20250601_v000.cdf" input_deps = ProcessingInputCollection(ScienceInput(input_l2_map_name)) - input_metadata = InputMetadata(instrument="ultra", - data_level="l3", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="", - descriptor=f"u90-ena-h-sf-sp-full-hae-{degree_spacing}deg-6mo") + input_metadata = InputMetadata( + instrument="ultra", + data_level="l3", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="", + descriptor=f"u90-ena-h-sf-sp-full-hae-{degree_spacing}deg-6mo", + ) mock_fetch_dependencies.return_value = UltraL3Dependencies( ultra_l2_healpix_map=input_l2_healpix_map, @@ -393,19 +481,27 @@ def _test_process_survival_probability(self, degree_spacing, mock_fetch_dependen ) rectangular_predicted_ephemeris_data = np.array([0.73, 0.24, 0, 0.99, 1, 0]) - nominal_alpha_proton_ratio_data = np.full_like(rectangular_predicted_ephemeris_data, 0.0) - persisted_last_point_data = np.full_like(rectangular_predicted_ephemeris_data, 0.0) - - expected_quality_flags = np.array([ - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.NONE, - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.NONE - ]) - - mock_healpix_map_data = mock_correct_healpix_data_for_survival_probability.return_value + nominal_alpha_proton_ratio_data = np.full_like( + rectangular_predicted_ephemeris_data, 0.0 + ) + persisted_last_point_data = np.full_like( + rectangular_predicted_ephemeris_data, 0.0 + ) + + expected_quality_flags = np.array( + [ + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.NONE, + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.NONE, + ] + ) + + mock_healpix_map_data = ( + mock_correct_healpix_data_for_survival_probability.return_value + ) mock_healpix_skymap = Mock() mock_healpix_map_data.to_healpix_skymap = Mock(return_value=mock_healpix_skymap) @@ -423,13 +519,18 @@ def _test_process_survival_probability(self, degree_spacing, mock_fetch_dependen "predicted_ephemeris_flag": Mock( values=rectangular_predicted_ephemeris_data ), - "nominal_alpha_proton_ratio_flag": Mock(values=nominal_alpha_proton_ratio_data), - "persisted_last_point_flag": Mock(values=persisted_last_point_data) + "nominal_alpha_proton_ratio_flag": Mock( + values=nominal_alpha_proton_ratio_data + ), + "persisted_last_point_flag": Mock(values=persisted_last_point_data), } mock_rectangular_sky_map = Mock(spec=RectangularSkyMap) mock_rectangular_sky_map.to_dataset.return_value = mock_rectangular_map_dataset - mock_healpix_skymap.to_rectangular_skymap.return_value = mock_rectangular_sky_map, 0 + mock_healpix_skymap.to_rectangular_skymap.return_value = ( + mock_rectangular_sky_map, + 0, + ) processor = UltraProcessor(input_deps, input_metadata) product = processor.process(SpiceFrame.IMAP_GCS) @@ -439,69 +540,159 @@ def _test_process_survival_probability(self, degree_spacing, mock_fetch_dependen mock_fetch_dependencies.return_value, SpiceFrame.IMAP_GCS ) mock_healpix_map_data.to_healpix_skymap.assert_called_once() - mock_healpix_skymap.to_rectangular_skymap.assert_called_once_with(degree_spacing, [ - "ena_intensity", - "ena_intensity_stat_uncert", - "ena_intensity_sys_err", - "predicted_ephemeris_flag", - "nominal_alpha_proton_ratio_flag", - "persisted_last_point_flag", - "survival_probability", - ]) + mock_healpix_skymap.to_rectangular_skymap.assert_called_once_with( + degree_spacing, + [ + "ena_intensity", + "ena_intensity_stat_uncert", + "ena_intensity_sys_err", + "predicted_ephemeris_flag", + "nominal_alpha_proton_ratio_flag", + "persisted_last_point_flag", + "survival_probability", + ], + ) mock_save_data.assert_called_once() actual_rectangular_data_product = mock_save_data.call_args_list[0].args[0] - self.assertIsInstance(actual_rectangular_data_product, RectangularIntensityDataProduct) + self.assertIsInstance( + actual_rectangular_data_product, RectangularIntensityDataProduct + ) self.assertEqual(4, len(actual_rectangular_data_product.parent_file_names)) - self.assertEqual({input_l2_map_name, fake_spice.name, input_l1c_pset_name, input_glows_l3e_name}, - set(actual_rectangular_data_product.parent_file_names)) - self.assertEqual(SpiceFrame.IMAP_GCS, actual_rectangular_data_product.spice_frame_name) + self.assertEqual( + { + input_l2_map_name, + fake_spice.name, + input_l1c_pset_name, + input_glows_l3e_name, + }, + set(actual_rectangular_data_product.parent_file_names), + ) + self.assertEqual( + SpiceFrame.IMAP_GCS, actual_rectangular_data_product.spice_frame_name + ) actual_rectangular_data = actual_rectangular_data_product.data - self.assertIsInstance(actual_rectangular_data.intensity_map_data, IntensityMapData) + self.assertIsInstance( + actual_rectangular_data.intensity_map_data, IntensityMapData + ) # @formatter:off - self.assertEqual(sentinel.rectangular_ena_intensity, actual_rectangular_data.intensity_map_data.ena_intensity) - self.assertEqual(sentinel.rectangular_ena_intensity_stat_uncert, actual_rectangular_data.intensity_map_data.ena_intensity_stat_uncert) - self.assertEqual(sentinel.rectangular_ena_intensity_sys_err, actual_rectangular_data.intensity_map_data.ena_intensity_sys_err) - self.assertEqual(sentinel.rectangular_survival_probability, actual_rectangular_data.intensity_map_data.survival_probability) - np.testing.assert_array_equal(actual_rectangular_data.intensity_map_data.quality_flags, expected_quality_flags, strict=True) + self.assertEqual( + sentinel.rectangular_ena_intensity, + actual_rectangular_data.intensity_map_data.ena_intensity, + ) + self.assertEqual( + sentinel.rectangular_ena_intensity_stat_uncert, + actual_rectangular_data.intensity_map_data.ena_intensity_stat_uncert, + ) + self.assertEqual( + sentinel.rectangular_ena_intensity_sys_err, + actual_rectangular_data.intensity_map_data.ena_intensity_sys_err, + ) + self.assertEqual( + sentinel.rectangular_survival_probability, + actual_rectangular_data.intensity_map_data.survival_probability, + ) + np.testing.assert_array_equal( + actual_rectangular_data.intensity_map_data.quality_flags, + expected_quality_flags, + strict=True, + ) expected_passthrough = input_l2_rectangular_map.intensity_map_data - self.assertIs(expected_passthrough.epoch, actual_rectangular_data.intensity_map_data.epoch) - self.assertIs(expected_passthrough.epoch_delta, actual_rectangular_data.intensity_map_data.epoch_delta) - self.assertIs(expected_passthrough.energy, actual_rectangular_data.intensity_map_data.energy) - self.assertIs(expected_passthrough.energy_delta_plus, actual_rectangular_data.intensity_map_data.energy_delta_plus) - self.assertIs(expected_passthrough.energy_delta_minus, actual_rectangular_data.intensity_map_data.energy_delta_minus) - self.assertIs(expected_passthrough.energy_label, actual_rectangular_data.intensity_map_data.energy_label) - self.assertIs(expected_passthrough.latitude, actual_rectangular_data.intensity_map_data.latitude) - self.assertIs(expected_passthrough.longitude, actual_rectangular_data.intensity_map_data.longitude) - self.assertIs(expected_passthrough.exposure_factor, actual_rectangular_data.intensity_map_data.exposure_factor) - self.assertIs(expected_passthrough.obs_date, actual_rectangular_data.intensity_map_data.obs_date) - self.assertIs(expected_passthrough.obs_date_range, actual_rectangular_data.intensity_map_data.obs_date_range) - self.assertIs(expected_passthrough.solid_angle, actual_rectangular_data.intensity_map_data.solid_angle) + self.assertIs( + expected_passthrough.epoch, actual_rectangular_data.intensity_map_data.epoch + ) + self.assertIs( + expected_passthrough.epoch_delta, + actual_rectangular_data.intensity_map_data.epoch_delta, + ) + self.assertIs( + expected_passthrough.energy, + actual_rectangular_data.intensity_map_data.energy, + ) + self.assertIs( + expected_passthrough.energy_delta_plus, + actual_rectangular_data.intensity_map_data.energy_delta_plus, + ) + self.assertIs( + expected_passthrough.energy_delta_minus, + actual_rectangular_data.intensity_map_data.energy_delta_minus, + ) + self.assertIs( + expected_passthrough.energy_label, + actual_rectangular_data.intensity_map_data.energy_label, + ) + self.assertIs( + expected_passthrough.latitude, + actual_rectangular_data.intensity_map_data.latitude, + ) + self.assertIs( + expected_passthrough.longitude, + actual_rectangular_data.intensity_map_data.longitude, + ) + self.assertIs( + expected_passthrough.exposure_factor, + actual_rectangular_data.intensity_map_data.exposure_factor, + ) + self.assertIs( + expected_passthrough.obs_date, + actual_rectangular_data.intensity_map_data.obs_date, + ) + self.assertIs( + expected_passthrough.obs_date_range, + actual_rectangular_data.intensity_map_data.obs_date_range, + ) + self.assertIs( + expected_passthrough.solid_angle, + actual_rectangular_data.intensity_map_data.solid_angle, + ) expected_coords = input_l2_rectangular_map.coords - self.assertIs(expected_coords.latitude_delta, actual_rectangular_data.coords.latitude_delta) - self.assertIs(expected_coords.latitude_label, actual_rectangular_data.coords.latitude_label) - self.assertIs(expected_coords.longitude_delta, actual_rectangular_data.coords.longitude_delta) - self.assertIs(expected_coords.longitude_label, actual_rectangular_data.coords.longitude_label) + self.assertIs( + expected_coords.latitude_delta, + actual_rectangular_data.coords.latitude_delta, + ) + self.assertIs( + expected_coords.latitude_label, + actual_rectangular_data.coords.latitude_label, + ) + self.assertIs( + expected_coords.longitude_delta, + actual_rectangular_data.coords.longitude_delta, + ) + self.assertIs( + expected_coords.longitude_label, + actual_rectangular_data.coords.longitude_label, + ) # @formatter:on self.assertEqual([mock_save_data.return_value], product) - @patch('imap_l3_processing.ultra.ultra_processor.MapProcessor.get_parent_file_names') + @patch( + "imap_l3_processing.ultra.ultra_processor.MapProcessor.get_parent_file_names" + ) @patch("imap_l3_processing.ultra.ultra_processor.ExposureWeightedCombination") - @patch('imap_l3_processing.ultra.ultra_processor.save_data') - @patch('imap_l3_processing.ultra.ultra_processor.UltraL3CombinedDependencies.fetch_dependencies') - def _test_process_combined_sensor(self, degree_spacing, mock_fetch_dependencies, mock_save_data, - mock_exposure_weighted_combination, _): - input_metadata = InputMetadata(instrument="ultra", - data_level="l3", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="", - descriptor=f"ulc-ena-h-sf-nsp-full-hae-{degree_spacing}deg-6mo", - ) + @patch("imap_l3_processing.ultra.ultra_processor.save_data") + @patch( + "imap_l3_processing.ultra.ultra_processor.UltraL3CombinedDependencies.fetch_dependencies" + ) + def _test_process_combined_sensor( + self, + degree_spacing, + mock_fetch_dependencies, + mock_save_data, + mock_exposure_weighted_combination, + _, + ): + input_metadata = InputMetadata( + instrument="ultra", + data_level="l3", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="", + descriptor=f"ulc-ena-h-sf-nsp-full-hae-{degree_spacing}deg-6mo", + ) mock_fetch_dependencies.return_value = UltraL3CombinedDependencies( u45_dependencies=UltraL3Dependencies( @@ -540,24 +731,40 @@ def _test_process_combined_sensor(self, degree_spacing, mock_fetch_dependencies, ) predicted_ephemeris_rectangular_values = np.array([0.243, 0, 1, 0, 0.993, 0]) - nominal_alpha_proton_ratio_rectangular_values = np.array([0, 0.34, 0.8, 0, 0, 0]) + nominal_alpha_proton_ratio_rectangular_values = np.array( + [0, 0.34, 0.8, 0, 0, 0] + ) persisted_last_point_rectangular_values = np.array([0, 0, 0.4, 0, 0, 1]) - expected_quality_flags = np.array([ - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO, - MapL3Flags.PREDICTIVE_EPHEMERIS | MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO | MapL3Flags.PERSISTED_LAST_POINT, - MapL3Flags.NONE, - MapL3Flags.PREDICTIVE_EPHEMERIS, - MapL3Flags.PERSISTED_LAST_POINT - ]) + expected_quality_flags = np.array( + [ + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO, + MapL3Flags.PREDICTIVE_EPHEMERIS + | MapL3Flags.NOMINAL_ALPHA_PROTON_RATIO + | MapL3Flags.PERSISTED_LAST_POINT, + MapL3Flags.NONE, + MapL3Flags.PREDICTIVE_EPHEMERIS, + MapL3Flags.PERSISTED_LAST_POINT, + ] + ) converted_rectangular_skymap.to_dataset.return_value = { "ena_intensity": Mock(values=sentinel.rectangular_ena_intensity), - "ena_intensity_stat_uncert": Mock(values=sentinel.rectangular_ena_intensity_stat_uncert), - "ena_intensity_sys_err": Mock(values=sentinel.rectangular_ena_intensity_sys_err), - "predicted_ephemeris_flag": Mock(values=predicted_ephemeris_rectangular_values), - "nominal_alpha_proton_ratio_flag": Mock(values=nominal_alpha_proton_ratio_rectangular_values), - "persisted_last_point_flag": Mock(values=persisted_last_point_rectangular_values), + "ena_intensity_stat_uncert": Mock( + values=sentinel.rectangular_ena_intensity_stat_uncert + ), + "ena_intensity_sys_err": Mock( + values=sentinel.rectangular_ena_intensity_sys_err + ), + "predicted_ephemeris_flag": Mock( + values=predicted_ephemeris_rectangular_values + ), + "nominal_alpha_proton_ratio_flag": Mock( + values=nominal_alpha_proton_ratio_rectangular_values + ), + "persisted_last_point_flag": Mock( + values=persisted_last_point_rectangular_values + ), } processor = UltraProcessor(sentinel.dependencies, input_metadata) @@ -566,7 +773,8 @@ def _test_process_combined_sensor(self, degree_spacing, mock_fetch_dependencies, mock_fetch_dependencies.assert_called_once_with(sentinel.dependencies) mock_exposure_weighted_combination.return_value.combine_healpix_intensity_map_data.assert_called_once_with( - [sentinel.u45_l2_healpix_map, sentinel.u90_l2_healpix_map]) + [sentinel.u45_l2_healpix_map, sentinel.u90_l2_healpix_map] + ) mock_exposure_weighted_combination.return_value.combine_rectangular_intensity_map_data.assert_called_once_with( [sentinel.u45_l2_rectangular_map, sentinel.u90_l2_rectangular_map] @@ -583,13 +791,14 @@ def _test_process_combined_sensor(self, degree_spacing, mock_fetch_dependencies, "predicted_ephemeris_flag", "nominal_alpha_proton_ratio_flag", "persisted_last_point_flag", - ] + ], ) actual_data_product = mock_save_data.call_args_list[0].args[0] self.assertEqual( - sentinel.rectangular_ena_intensity, actual_data_product.data.intensity_map_data.ena_intensity + sentinel.rectangular_ena_intensity, + actual_data_product.data.intensity_map_data.ena_intensity, ) self.assertEqual( sentinel.rectangular_ena_intensity_stat_uncert, @@ -599,41 +808,88 @@ def _test_process_combined_sensor(self, degree_spacing, mock_fetch_dependencies, sentinel.rectangular_ena_intensity_sys_err, actual_data_product.data.intensity_map_data.ena_intensity_sys_err, ) - np.testing.assert_array_equal(actual_data_product.data.intensity_map_data.quality_flags, expected_quality_flags) - self.assertEqual(rectangular_combination_return_value.intensity_map_data.obs_date, actual_data_product.data.intensity_map_data.obs_date) - self.assertEqual(rectangular_combination_return_value.intensity_map_data.obs_date_range, actual_data_product.data.intensity_map_data.obs_date_range) - self.assertEqual(rectangular_combination_return_value.intensity_map_data.solid_angle, actual_data_product.data.intensity_map_data.solid_angle) - self.assertEqual(rectangular_combination_return_value.intensity_map_data.exposure_factor, actual_data_product.data.intensity_map_data.exposure_factor) - self.assertEqual(rectangular_combination_return_value.intensity_map_data.longitude, actual_data_product.data.intensity_map_data.longitude) - self.assertEqual(rectangular_combination_return_value.intensity_map_data.latitude, actual_data_product.data.intensity_map_data.latitude) + np.testing.assert_array_equal( + actual_data_product.data.intensity_map_data.quality_flags, + expected_quality_flags, + ) + self.assertEqual( + rectangular_combination_return_value.intensity_map_data.obs_date, + actual_data_product.data.intensity_map_data.obs_date, + ) + self.assertEqual( + rectangular_combination_return_value.intensity_map_data.obs_date_range, + actual_data_product.data.intensity_map_data.obs_date_range, + ) + self.assertEqual( + rectangular_combination_return_value.intensity_map_data.solid_angle, + actual_data_product.data.intensity_map_data.solid_angle, + ) + self.assertEqual( + rectangular_combination_return_value.intensity_map_data.exposure_factor, + actual_data_product.data.intensity_map_data.exposure_factor, + ) + self.assertEqual( + rectangular_combination_return_value.intensity_map_data.longitude, + actual_data_product.data.intensity_map_data.longitude, + ) + self.assertEqual( + rectangular_combination_return_value.intensity_map_data.latitude, + actual_data_product.data.intensity_map_data.latitude, + ) self.assertEqual([mock_save_data.return_value], product) - self.assertEqual(['u45_map', 'u45_l1c', 'u90_map', 'u90_l1c'], actual_data_product.parent_file_names) + self.assertEqual( + ["u45_map", "u45_l1c", "u90_map", "u90_l1c"], + actual_data_product.parent_file_names, + ) - @patch('imap_l3_processing.ultra.ultra_processor.correct_healpix_data_for_survival_probability') - @patch('imap_l3_processing.ultra.ultra_processor.UltraProcessor._process_healpix_intensity_to_rectangular') - @patch('imap_l3_processing.ultra.ultra_processor.MapProcessor.get_parent_file_names') + @patch( + "imap_l3_processing.ultra.ultra_processor.correct_healpix_data_for_survival_probability" + ) + @patch( + "imap_l3_processing.ultra.ultra_processor.UltraProcessor._process_healpix_intensity_to_rectangular" + ) + @patch( + "imap_l3_processing.ultra.ultra_processor.MapProcessor.get_parent_file_names" + ) @patch("imap_l3_processing.ultra.ultra_processor.UncertaintyWeightedCombination") - @patch('imap_l3_processing.ultra.ultra_processor.save_data') - @patch('imap_l3_processing.ultra.ultra_processor.UltraL3CombinedDependencies.fetch_dependencies') - def _test_process_combined_sensor_survival_probability(self, degree_spacing, mock_fetch_dependencies, - mock_save_data, mock_uncertainty_weighted_combination, - mock_get_parent_file_names, mock_healpix_to_rectangular, - mock_correct_healpix_data): + @patch("imap_l3_processing.ultra.ultra_processor.save_data") + @patch( + "imap_l3_processing.ultra.ultra_processor.UltraL3CombinedDependencies.fetch_dependencies" + ) + def _test_process_combined_sensor_survival_probability( + self, + degree_spacing, + mock_fetch_dependencies, + mock_save_data, + mock_uncertainty_weighted_combination, + mock_get_parent_file_names, + mock_healpix_to_rectangular, + mock_correct_healpix_data, + ): mock_get_parent_file_names.return_value = ["ram_map", "antiram_map"] - input_metadata = InputMetadata(instrument="ultra", - data_level="l3", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="", - descriptor=f"ulc-ena-h-sf-sp-full-hae-{degree_spacing}deg-6mo", - ) + input_metadata = InputMetadata( + instrument="ultra", + data_level="l3", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="", + descriptor=f"ulc-ena-h-sf-sp-full-hae-{degree_spacing}deg-6mo", + ) mock_fetch_dependencies.return_value = UltraL3CombinedDependencies( u45_dependencies=UltraL3Dependencies( ultra_l2_healpix_map=sentinel.u45_l2_healpix_map, ultra_l2_rectangular_map=sentinel.u45_l2_rectangular_map, - ultra_l1c_pset=[sentinel.u45_l1c_1, sentinel.u45_l1c_2, sentinel.u45_l1c_3], - glows_l3e_sp=[sentinel.glows_pset_1, sentinel.glows_pset_2, sentinel.glows_pset_3], + ultra_l1c_pset=[ + sentinel.u45_l1c_1, + sentinel.u45_l1c_2, + sentinel.u45_l1c_3, + ], + glows_l3e_sp=[ + sentinel.glows_pset_1, + sentinel.glows_pset_2, + sentinel.glows_pset_3, + ], energy_bin_group_sizes=sentinel.energy_bin_sizes, dependency_file_paths=[ Path("folder/u45_map"), @@ -643,8 +899,16 @@ def _test_process_combined_sensor_survival_probability(self, degree_spacing, moc u90_dependencies=UltraL3Dependencies( ultra_l2_healpix_map=sentinel.u90_l2_healpix_map, ultra_l2_rectangular_map=sentinel.u90_l2_rectangular_map, - ultra_l1c_pset=[sentinel.u90_l1c_1, sentinel.u90_l1c_2, sentinel.u90_l1c_3], - glows_l3e_sp=[sentinel.glows_pset_1, sentinel.glows_pset_2, sentinel.glows_pset_3], + ultra_l1c_pset=[ + sentinel.u90_l1c_1, + sentinel.u90_l1c_2, + sentinel.u90_l1c_3, + ], + glows_l3e_sp=[ + sentinel.glows_pset_1, + sentinel.glows_pset_2, + sentinel.glows_pset_3, + ], energy_bin_group_sizes=sentinel.energy_bin_sizes, dependency_file_paths=[ Path("folder/u90_map"), @@ -656,8 +920,10 @@ def _test_process_combined_sensor_survival_probability(self, degree_spacing, moc mock_combination_strategy = Mock() mock_uncertainty_weighted_combination.return_value = mock_combination_strategy - mock_correct_healpix_data.side_effect = [sentinel.u45_l2_survival_corrected_map, - sentinel.u90_l2_survival_corrected_map] + mock_correct_healpix_data.side_effect = [ + sentinel.u45_l2_survival_corrected_map, + sentinel.u90_l2_survival_corrected_map, + ] processor = UltraProcessor(sentinel.dependencies, input_metadata) product = processor.process(spice_frame_name=sentinel.spice_frame) @@ -665,41 +931,69 @@ def _test_process_combined_sensor_survival_probability(self, degree_spacing, moc mock_fetch_dependencies.assert_called_once_with(sentinel.dependencies) mock_combination_strategy.combine_healpix_intensity_map_data.assert_called_once_with( - [sentinel.u45_l2_survival_corrected_map, sentinel.u90_l2_survival_corrected_map]) + [ + sentinel.u45_l2_survival_corrected_map, + sentinel.u90_l2_survival_corrected_map, + ] + ) mock_combination_strategy.combine_rectangular_intensity_map_data.assert_called_once_with( - [sentinel.u45_l2_rectangular_map, sentinel.u90_l2_rectangular_map]) + [sentinel.u45_l2_rectangular_map, sentinel.u90_l2_rectangular_map] + ) mock_healpix_to_rectangular.assert_called_once_with( mock_combination_strategy.combine_healpix_intensity_map_data.return_value, mock_combination_strategy.combine_rectangular_intensity_map_data.return_value, degree_spacing, - spice_frame_name=sentinel.spice_frame) + spice_frame_name=sentinel.spice_frame, + ) mock_save_data.assert_called_once_with(mock_healpix_to_rectangular.return_value) self.assertEqual([mock_save_data.return_value], product) - mock_correct_healpix_data.assert_has_calls([ - call(mock_fetch_dependencies.return_value.u45_dependencies, sentinel.spice_frame), - call(mock_fetch_dependencies.return_value.u90_dependencies, sentinel.spice_frame) - ]) - mock_healpix_to_rectangular.return_value.add_paths_to_parents.assert_has_calls([ - call([ - Path("folder/u45_map"), - Path("folder/u45_l1c"), - ]), - call([ - Path("folder/u90_map"), - Path("folder/u90_l1c"), - ]), - ]) - - @patch('imap_l3_processing.ultra.ultra_processor.correct_healpix_data_for_survival_probability') - @patch('imap_l3_processing.processor.spiceypy') - @patch('imap_l3_processing.ultra.ultra_processor.save_data') - @patch('imap_l3_processing.ultra.ultra_processor.UltraL3Dependencies.fetch_dependencies') - def test_defaults_to_ECLIPJ2000_spice_frame(self, mock_fetch_dependencies, - mock_save_data, mock_spiceypy, - mock_correct_for_survival_probability): + mock_correct_healpix_data.assert_has_calls( + [ + call( + mock_fetch_dependencies.return_value.u45_dependencies, + sentinel.spice_frame, + ), + call( + mock_fetch_dependencies.return_value.u90_dependencies, + sentinel.spice_frame, + ), + ] + ) + mock_healpix_to_rectangular.return_value.add_paths_to_parents.assert_has_calls( + [ + call( + [ + Path("folder/u45_map"), + Path("folder/u45_l1c"), + ] + ), + call( + [ + Path("folder/u90_map"), + Path("folder/u90_l1c"), + ] + ), + ] + ) + + @patch( + "imap_l3_processing.ultra.ultra_processor.correct_healpix_data_for_survival_probability" + ) + @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.ultra.ultra_processor.save_data") + @patch( + "imap_l3_processing.ultra.ultra_processor.UltraL3Dependencies.fetch_dependencies" + ) + def test_defaults_to_ECLIPJ2000_spice_frame( + self, + mock_fetch_dependencies, + mock_save_data, + mock_spiceypy, + mock_correct_for_survival_probability, + ): rng = np.random.default_rng() healpix_indices = np.arange(12) input_map_flux = rng.random((1, 9, 12)) @@ -710,7 +1004,9 @@ def test_defaults_to_ECLIPJ2000_spice_frame(self, mock_fetch_dependencies, fake_spice = Path("path/to/fake/spice.tls") mock_spiceypy.kdata.return_value = [fake_spice] - input_l2_map = _create_ultra_l2_healpix_data(epoch=[epoch], flux=input_map_flux, healpix_indices=healpix_indices) + input_l2_map = _create_ultra_l2_healpix_data( + epoch=[epoch], flux=input_map_flux, healpix_indices=healpix_indices + ) input_l2_rectangular_map = create_rectangular_intensity_map_data() input_l2_map.intensity_map_data.energy = sentinel.ultra_l2_energies @@ -726,25 +1022,38 @@ def test_defaults_to_ECLIPJ2000_spice_frame(self, mock_fetch_dependencies, ultra_l2_rectangular_map=input_l2_rectangular_map, ultra_l1c_pset=sentinel.ultra_l1c_pset, glows_l3e_sp=sentinel.glows_l3e_sp, - dependency_file_paths=[Path(input_l2_map_name), Path(input_l1c_pset_name), Path(input_glows_l3e_name)], - energy_bin_group_sizes=None + dependency_file_paths=[ + Path(input_l2_map_name), + Path(input_l1c_pset_name), + Path(input_glows_l3e_name), + ], + energy_bin_group_sizes=None, + ) + input_metadata = InputMetadata( + instrument="ultra", + data_level="l3", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="", + descriptor="u90-ena-h-sf-sp-full-hae-2deg-6mo", ) - input_metadata = InputMetadata(instrument="ultra", - data_level="l3", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="", - descriptor="u90-ena-h-sf-sp-full-hae-2deg-6mo" - ) mock_healpix_skymap = Mock() - mock_correct_for_survival_probability.return_value.to_healpix_skymap = Mock(return_value=mock_healpix_skymap) + mock_correct_for_survival_probability.return_value.to_healpix_skymap = Mock( + return_value=mock_healpix_skymap + ) mock_rectangular_map_dataset = { "ena_intensity": Mock(values=sentinel.rectangular_ena_intensity), - "ena_intensity_stat_uncert": Mock(values=sentinel.rectangular_ena_intensity_stat_unc), - "ena_intensity_sys_err": Mock(values=sentinel.rectangular_ena_intensity_sys_err), - "survival_probability": Mock(values=sentinel.rectangular_survival_probability), + "ena_intensity_stat_uncert": Mock( + values=sentinel.rectangular_ena_intensity_stat_unc + ), + "ena_intensity_sys_err": Mock( + values=sentinel.rectangular_ena_intensity_sys_err + ), + "survival_probability": Mock( + values=sentinel.rectangular_survival_probability + ), "predicted_ephemeris_flag": Mock(values=np.array([])), "nominal_alpha_proton_ratio_flag": Mock(values=np.array([])), "persisted_last_point_flag": Mock(values=np.array([])), @@ -752,39 +1061,55 @@ def test_defaults_to_ECLIPJ2000_spice_frame(self, mock_fetch_dependencies, mock_rectangular_sky_map = Mock(spec=RectangularSkyMap) mock_rectangular_sky_map.to_dataset.return_value = mock_rectangular_map_dataset - mock_healpix_skymap.to_rectangular_skymap.return_value = mock_rectangular_sky_map, 0 + mock_healpix_skymap.to_rectangular_skymap.return_value = ( + mock_rectangular_sky_map, + 0, + ) processor = UltraProcessor(input_deps, input_metadata) processor.process() mock_correct_for_survival_probability.assert_called_once_with( - mock_fetch_dependencies.return_value, - SpiceFrame.ECLIPJ2000 + mock_fetch_dependencies.return_value, SpiceFrame.ECLIPJ2000 ) - @patch('imap_l3_processing.processor.spiceypy') - @patch('imap_l3_processing.ultra.ultra_processor.save_data') - @patch('imap_l3_processing.ultra.ultra_processor.calculate_spectral_index_for_multiple_ranges') - @patch('imap_l3_processing.ultra.ultra_processor.UltraL3SpectralIndexDependencies.fetch_dependencies') - def _test_process_spectral_index(self, degree_spacing, - mock_fetch_dependencies, - mock_calculate_spectral_index, mock_save_data, - mock_spiceypy): + @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.ultra.ultra_processor.save_data") + @patch( + "imap_l3_processing.ultra.ultra_processor.calculate_spectral_index_for_multiple_ranges" + ) + @patch( + "imap_l3_processing.ultra.ultra_processor.UltraL3SpectralIndexDependencies.fetch_dependencies" + ) + def _test_process_spectral_index( + self, + degree_spacing, + mock_fetch_dependencies, + mock_calculate_spectral_index, + mock_save_data, + mock_spiceypy, + ): mock_spiceypy.ktotal.return_value = 0 - map_file_name = 'imap_ultra_l3_ultra-cool-descriptor_20250601_v000.cdf' - energy_range_file_name = 'imap_ultra_energy-range-descriptor_20250601_v000.dat' - input_deps = ProcessingInputCollection(ScienceInput(map_file_name), AncillaryInput(energy_range_file_name)) + map_file_name = "imap_ultra_l3_ultra-cool-descriptor_20250601_v000.cdf" + energy_range_file_name = "imap_ultra_energy-range-descriptor_20250601_v000.dat" + input_deps = ProcessingInputCollection( + ScienceInput(map_file_name), AncillaryInput(energy_range_file_name) + ) - input_metadata = InputMetadata(instrument="ultra", - data_level="l3", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="v000", - descriptor=f"u90-spx-h-sf-sp-full-hae-{degree_spacing}deg-6mo") + input_metadata = InputMetadata( + instrument="ultra", + data_level="l3", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="v000", + descriptor=f"u90-spx-h-sf-sp-full-hae-{degree_spacing}deg-6mo", + ) input_map_data = RectangularIntensityMapData(Mock(), Mock()) - dependencies = UltraL3SpectralIndexDependencies(input_map_data, sentinel.energy_ranges) + dependencies = UltraL3SpectralIndexDependencies( + input_map_data, sentinel.energy_ranges + ) mock_fetch_dependencies.return_value = dependencies mock_spectral_index_map_data = Mock(spec=SpectralIndexMapData) @@ -797,112 +1122,188 @@ def _test_process_spectral_index(self, degree_spacing, mock_save_data.assert_called_once() actual_rectangular_data_product = mock_save_data.call_args_list[0].args[0] - self.assertIsInstance(actual_rectangular_data_product, RectangularSpectralIndexDataProduct) - self.assertEqual(expected_parent_file_names, actual_rectangular_data_product.parent_file_names) - self.assertEqual(processor.input_metadata, actual_rectangular_data_product.input_metadata) - self.assertEqual(SpiceFrame.ECLIPJ2000, actual_rectangular_data_product.spice_frame_name) + self.assertIsInstance( + actual_rectangular_data_product, RectangularSpectralIndexDataProduct + ) + self.assertEqual( + expected_parent_file_names, + actual_rectangular_data_product.parent_file_names, + ) + self.assertEqual( + processor.input_metadata, actual_rectangular_data_product.input_metadata + ) + self.assertEqual( + SpiceFrame.ECLIPJ2000, actual_rectangular_data_product.spice_frame_name + ) - self.assertIsInstance(actual_rectangular_data_product, RectangularSpectralIndexDataProduct) + self.assertIsInstance( + actual_rectangular_data_product, RectangularSpectralIndexDataProduct + ) - actual_rectangular_data: RectangularSpectralIndexMapData = actual_rectangular_data_product.data - self.assertIsInstance(actual_rectangular_data_product.data, RectangularSpectralIndexMapData) - self.assertIs(mock_spectral_index_map_data, actual_rectangular_data.spectral_index_map_data) + actual_rectangular_data: RectangularSpectralIndexMapData = ( + actual_rectangular_data_product.data + ) + self.assertIsInstance( + actual_rectangular_data_product.data, RectangularSpectralIndexMapData + ) + self.assertIs( + mock_spectral_index_map_data, + actual_rectangular_data.spectral_index_map_data, + ) self.assertIs(input_map_data.coords, actual_rectangular_data.coords) mock_fetch_dependencies.assert_called_once_with(input_deps) - mock_calculate_spectral_index.assert_called_once_with(dependencies.map_data.intensity_map_data, - sentinel.energy_ranges) + mock_calculate_spectral_index.assert_called_once_with( + dependencies.map_data.intensity_map_data, sentinel.energy_ranges + ) self.assertEqual([mock_save_data.return_value], product) - @patch('imap_l3_processing.ultra.ultra_processor.save_data') - @patch('imap_l3_processing.processor.spiceypy') - @patch('imap_l3_processing.ultra.ultra_processor.calculate_spectral_index_for_multiple_ranges') - @patch('imap_l3_processing.ultra.ultra_processor.fit_spectral_index_map') - @patch('imap_l3_processing.ultra.ultra_processor.slice_energy_range_by_bin') - @patch('imap_l3_processing.ultra.ultra_processor.UltraL3SpectralIndexDependencies.fetch_dependencies') - def test_process_spectral_index_with_custom_energy_bin_range(self, - mock_fetch_dependencies, mock_slice_energy_range_by_bin, mock_fit_spectral_index_map, - mock_calculate_spectral_index_for_multiple_ranges, mock_spiceypy, mock_save_data - ): + @patch("imap_l3_processing.ultra.ultra_processor.save_data") + @patch("imap_l3_processing.processor.spiceypy") + @patch( + "imap_l3_processing.ultra.ultra_processor.calculate_spectral_index_for_multiple_ranges" + ) + @patch("imap_l3_processing.ultra.ultra_processor.fit_spectral_index_map") + @patch("imap_l3_processing.ultra.ultra_processor.slice_energy_range_by_bin") + @patch( + "imap_l3_processing.ultra.ultra_processor.UltraL3SpectralIndexDependencies.fetch_dependencies" + ) + def test_process_spectral_index_with_custom_energy_bin_range( + self, + mock_fetch_dependencies, + mock_slice_energy_range_by_bin, + mock_fit_spectral_index_map, + mock_calculate_spectral_index_for_multiple_ranges, + mock_spiceypy, + mock_save_data, + ): mock_spiceypy.ktotal.return_value = 0 - map_file_name = 'imap_ultra_l3_ultra-cool-descriptor_20250601_v000.cdf' - energy_range_file_name = 'imap_ultra_energy-range-descriptor_20250601_v000.dat' - input_deps = ProcessingInputCollection(ScienceInput(map_file_name), AncillaryInput(energy_range_file_name)) + map_file_name = "imap_ultra_l3_ultra-cool-descriptor_20250601_v000.cdf" + energy_range_file_name = "imap_ultra_energy-range-descriptor_20250601_v000.dat" + input_deps = ProcessingInputCollection( + ScienceInput(map_file_name), AncillaryInput(energy_range_file_name) + ) - input_metadata = InputMetadata(instrument="ultra", - data_level="l3", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="v000", - descriptor="u90-spx0510-h-sf-sp-full-hae-4deg-6mo") + input_metadata = InputMetadata( + instrument="ultra", + data_level="l3", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="v000", + descriptor="u90-spx0510-h-sf-sp-full-hae-4deg-6mo", + ) dependencies = mock_fetch_dependencies.return_value processor = UltraProcessor(input_deps, input_metadata) result = processor.process() - mock_slice_energy_range_by_bin.assert_called_once_with(dependencies.map_data.intensity_map_data, 5, 10) - mock_fit_spectral_index_map.assert_called_once_with(mock_slice_energy_range_by_bin.return_value) + mock_slice_energy_range_by_bin.assert_called_once_with( + dependencies.map_data.intensity_map_data, 5, 10 + ) + mock_fit_spectral_index_map.assert_called_once_with( + mock_slice_energy_range_by_bin.return_value + ) mock_calculate_spectral_index_for_multiple_ranges.assert_not_called() mock_save_data.assert_called_once() [product] = mock_save_data.call_args.args self.assertIsInstance(product, RectangularSpectralIndexDataProduct) - self.assertEqual(mock_fit_spectral_index_map.return_value, product.data.spectral_index_map_data) + self.assertEqual( + mock_fit_spectral_index_map.return_value, + product.data.spectral_index_map_data, + ) def test_process_raises_exception_when_generating_a_healpix_map(self): - input_metadata = InputMetadata(instrument="ultra", - data_level="l3", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="v000", - descriptor="u90-spx-h-sf-sp-full-hae-nside8-6mo") + input_metadata = InputMetadata( + instrument="ultra", + data_level="l3", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="v000", + descriptor="u90-spx-h-sf-sp-full-hae-nside8-6mo", + ) processor = UltraProcessor(ProcessingInputCollection(), input_metadata) with self.assertRaises(NotImplementedError): processor.process() - @patch('imap_l3_processing.ultra.ultra_processor.save_data') - @patch('imap_l3_processing.ultra.ultra_processor.UltraL3SpectralIndexDependencies.fetch_dependencies') - def test_process_spectral_index_validating_output_values(self, mock_fetch_dependencies, mock_save_data): - input_metadata = InputMetadata(instrument="ultra", - data_level="l3", - start_date=datetime.now(), - end_date=datetime.now() + timedelta(days=1), - version="v000", - descriptor="u90-spx-h-sf-sp-full-hae-6deg-6mo") - input_map_path = get_test_data_path('ultra/fake_ultra_map_data_with_breakpoint_at_15keV.cdf') - fit_energy_ranges_path = get_test_data_path('ultra/imap_ultra_ulc-spx-energy-ranges_20250407_v000.dat') - dependencies = UltraL3SpectralIndexDependencies.from_file_paths(input_map_path, fit_energy_ranges_path) + @patch("imap_l3_processing.ultra.ultra_processor.save_data") + @patch( + "imap_l3_processing.ultra.ultra_processor.UltraL3SpectralIndexDependencies.fetch_dependencies" + ) + def test_process_spectral_index_validating_output_values( + self, mock_fetch_dependencies, mock_save_data + ): + input_metadata = InputMetadata( + instrument="ultra", + data_level="l3", + start_date=datetime.now(), + end_date=datetime.now() + timedelta(days=1), + version="v000", + descriptor="u90-spx-h-sf-sp-full-hae-6deg-6mo", + ) + input_map_path = get_test_data_path( + "ultra/fake_ultra_map_data_with_breakpoint_at_15keV.cdf" + ) + fit_energy_ranges_path = get_test_data_path( + "ultra/imap_ultra_ulc-spx-energy-ranges_20250407_v000.dat" + ) + dependencies = UltraL3SpectralIndexDependencies.from_file_paths( + input_map_path, fit_energy_ranges_path + ) mock_fetch_dependencies.return_value = dependencies - expected_ena_spectral_index = np.array([2] * (60 * 30) + [3.5] * (60 * 30)).reshape(1, 2, 60, 30) + expected_ena_spectral_index = np.array( + [2] * (60 * 30) + [3.5] * (60 * 30) + ).reshape(1, 2, 60, 30) processing_input_collection = ProcessingInputCollection() processor = UltraProcessor(processing_input_collection, input_metadata) product = processor.process() - actual_data_product: RectangularSpectralIndexDataProduct = mock_save_data.call_args[0][0] + actual_data_product: RectangularSpectralIndexDataProduct = ( + mock_save_data.call_args[0][0] + ) - np.testing.assert_array_almost_equal(actual_data_product.data.spectral_index_map_data.ena_spectral_index, - expected_ena_spectral_index) + np.testing.assert_array_almost_equal( + actual_data_product.data.spectral_index_map_data.ena_spectral_index, + expected_ena_spectral_index, + ) self.assertEqual([mock_save_data.return_value], product) -def _create_ultra_l2_healpix_data(epoch=None, lon=None, lat=None, energy=None, energy_delta=None, flux=None, - intensity_stat_unc=None, healpix_indices=None) -> HealPixIntensityMapData: + +def _create_ultra_l2_healpix_data( + epoch=None, + lon=None, + lat=None, + energy=None, + energy_delta=None, + flux=None, + intensity_stat_unc=None, + healpix_indices=None, +) -> HealPixIntensityMapData: epoch = epoch if epoch is not None else np.array([datetime.now()]) lon = lon if lon is not None else np.array([1.0]) lat = lat if lat is not None else np.array([1.0]) healpix_indices = healpix_indices if healpix_indices is not None else np.arange(12) energy = energy if energy is not None else np.array([1.0]) - energy_delta = energy_delta if energy_delta is not None else np.full((len(energy), 2), 1) - flux = flux if flux is not None else np.full((len(epoch), len(energy), len(healpix_indices)), fill_value=1) - intensity_stat_uncert = intensity_stat_unc if intensity_stat_unc is not None else np.full( - flux.shape, - fill_value=1) + energy_delta = ( + energy_delta if energy_delta is not None else np.full((len(energy), 2), 1) + ) + flux = ( + flux + if flux is not None + else np.full((len(epoch), len(energy), len(healpix_indices)), fill_value=1) + ) + intensity_stat_uncert = ( + intensity_stat_unc + if intensity_stat_unc is not None + else np.full(flux.shape, fill_value=1) + ) if isinstance(flux, np.ndarray): more_real_flux = flux @@ -926,10 +1327,10 @@ def _create_ultra_l2_healpix_data(epoch=None, lon=None, lat=None, energy=None, e ena_intensity=flux, ena_intensity_stat_uncert=intensity_stat_uncert, ena_intensity_sys_err=np.full_like(flux, 0), - quality_flags=np.full_like(flux, MapL3Flags.NONE) + quality_flags=np.full_like(flux, MapL3Flags.NONE), ), HealPixCoords( pixel_index=healpix_indices, - pixel_index_label=np.full(healpix_indices.shape, "healpix index label") - ) - ) \ No newline at end of file + pixel_index_label=np.full(healpix_indices.shape, "healpix index label"), + ), + ) diff --git a/tests/ultra/test_ultra_sp_initializer.py b/tests/ultra/test_ultra_sp_initializer.py index 106008cc3..0323217e1 100644 --- a/tests/ultra/test_ultra_sp_initializer.py +++ b/tests/ultra/test_ultra_sp_initializer.py @@ -4,11 +4,17 @@ from imap_data_access.file_validation import Version -from imap_l3_processing.glows.descriptors import GLOWS_L3E_ULTRA_SF_DESCRIPTOR, GLOWS_L3E_ULTRA_HF_DESCRIPTOR +from imap_l3_processing.glows.descriptors import ( + GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + GLOWS_L3E_ULTRA_HF_DESCRIPTOR, +) from imap_l3_processing.maps.map_initializer import PossibleMapToProduce from imap_l3_processing.maps.sp_map_initializer import SPMapInitializer from imap_l3_processing.models import InputMetadata, VersionMap -from imap_l3_processing.ultra.ultra_sp_initializer import UltraSPInitializer, ULTRA_SP_SPICE_KERNELS +from imap_l3_processing.ultra.ultra_sp_initializer import ( + UltraSPInitializer, + ULTRA_SP_SPICE_KERNELS, +) from tests.integration.integration_test_helpers import ImapQueryPatcher @@ -17,116 +23,146 @@ def test_is_map_initializer(self): self.assertIsInstance(UltraSPInitializer(), SPMapInitializer) def setUp(self): - self.mock_query_patcher = patch('imap_l3_processing.ultra.ultra_sp_initializer.imap_data_access.query') + self.mock_query_patcher = patch( + "imap_l3_processing.ultra.ultra_sp_initializer.imap_data_access.query" + ) self.mock_query = self.mock_query_patcher.start() def tearDown(self): self.mock_query_patcher.stop() - @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') + @patch("imap_l3_processing.maps.map_initializer.read_cdf_parents") def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): - sensors = [ - "90", - "45" - ] + sensors = ["90", "45"] for sensor in sensors: self.mock_query.reset_mock() with self.subTest(sensor=sensor): - self.mock_query.side_effect = ImapQueryPatcher([ - 'imap_glows_l3e_survival-probability-ul-sf_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100103-repoint00003_v001.cdf', - - 'imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v002.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100402-repoint00102_v002.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100403-repoint00103_v002.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v001.cdf', - - 'imap_glows_l3e_survival-probability-ul-sf_20100703-repoint00201_v001.cdf', - - f'imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - f'imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - f'imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v000.cdf', - f'imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', - f'imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf', - - f'imap_ultra_l3_u{sensor}-ena-h-sf-sp-full-hae-4deg-3mo_20100101_v001.cdf', - f'imap_ultra_l3_u{sensor}-ena-h-sf-sp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' - ]) + self.mock_query.side_effect = ImapQueryPatcher( + [ + "imap_glows_l3e_survival-probability-ul-sf_20100101-repoint00001_v001.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100102-repoint00002_v001.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100103-repoint00003_v001.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v002.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100402-repoint00102_v002.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100403-repoint00103_v002.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v001.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100703-repoint00201_v001.cdf", + f"imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf", + f"imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + f"imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v000.cdf", + f"imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf", + f"imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf", + f"imap_ultra_l3_u{sensor}-ena-h-sf-sp-full-hae-4deg-3mo_20100101_v001.cdf", + f"imap_ultra_l3_u{sensor}-ena-h-sf-sp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv", + ] + ) mock_read_cdf_parents.side_effect = [ [ f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100101-repoint00001_v001.cdf", f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100102-repoint00002_v001.cdf", - f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100103-repoint00003_v001.cdf" + f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100103-repoint00003_v001.cdf", ], [ - f'imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100403-repoint00103_v001.cdf' + f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100401-repoint00101_v001.cdf", + f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100402-repoint00102_v001.cdf", + f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100403-repoint00103_v001.cdf", ], [ f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100701-repoint00201_v001.cdf", f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100702-repoint00202_v001.cdf", - f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100703-repoint00203_v001.cdf" + f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100703-repoint00203_v001.cdf", ], [ f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20101001-repoint00301_v001.cdf", f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20101002-repoint00302_v001.cdf", - f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20101003-repoint00303_v001.cdf" + f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20101003-repoint00303_v001.cdf", ], [ - 'imap_glows_l3e_survival-probability-ul-sf_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100103-repoint00003_v001.cdf', + "imap_glows_l3e_survival-probability-ul-sf_20100101-repoint00001_v001.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100102-repoint00002_v001.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100103-repoint00003_v001.cdf", f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100101-repoint00001_v001.cdf", f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100102-repoint00002_v001.cdf", f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100103-repoint00003_v001.cdf", - f'imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv', + f"imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf", + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv", ], [ - 'imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v001.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100402-repoint00102_v001.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100403-repoint00103_v001.cdf', - f'imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - f'imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv', - ] + "imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v001.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100402-repoint00102_v001.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100403-repoint00103_v001.cdf", + f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100401-repoint00101_v001.cdf", + f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100402-repoint00102_v001.cdf", + f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100403-repoint00103_v001.cdf", + f"imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv", + ], ] initializer = UltraSPInitializer() actual_maps_to_produce = initializer.get_maps_that_should_be_produced( - f"u{sensor}-ena-h-sf-sp-full-hae-4deg-3mo", None) + f"u{sensor}-ena-h-sf-sp-full-hae-4deg-3mo", None + ) with self.assertRaises(NotImplementedError) as e: - initializer.get_maps_that_should_be_produced(f"u{sensor}-ena-h-hk-sp-full-hae-4deg-3mo", None) - self.assertEqual(e.exception.args, ("Reference frame should be either Spacecraft or Heliospheric",)) - - self.mock_query.assert_has_calls([ - call(instrument='glows', data_level='l3e', descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, - version="latest"), - call(instrument='glows', data_level='l3e', descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, - version="latest"), - call(instrument='ultra', data_level='l2'), - call(instrument="ultra", table='ancillary', descriptor='l2-energy-bin-group-sizes', - version='latest'), - call(instrument='ultra', data_level='l3') - ]) + initializer.get_maps_that_should_be_produced( + f"u{sensor}-ena-h-hk-sp-full-hae-4deg-3mo", None + ) + self.assertEqual( + e.exception.args, + ("Reference frame should be either Spacecraft or Heliospheric",), + ) - mock_read_cdf_parents.assert_has_calls([ - call(f'imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), - call(f'imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), - call(f'imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), - call(f'imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf'), + self.mock_query.assert_has_calls( + [ + call( + instrument="glows", + data_level="l3e", + descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + version="latest", + ), + call( + instrument="glows", + data_level="l3e", + descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, + version="latest", + ), + call(instrument="ultra", data_level="l2"), + call( + instrument="ultra", + table="ancillary", + descriptor="l2-energy-bin-group-sizes", + version="latest", + ), + call(instrument="ultra", data_level="l3"), + ] + ) - call(f'imap_ultra_l3_u{sensor}-ena-h-sf-sp-full-hae-4deg-3mo_20100101_v001.cdf'), - call(f'imap_ultra_l3_u{sensor}-ena-h-sf-sp-full-hae-4deg-3mo_20100401_v001.cdf'), - ]) + mock_read_cdf_parents.assert_has_calls( + [ + call( + f"imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf" + ), + call( + f"imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf" + ), + call( + f"imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf" + ), + call( + f"imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf" + ), + call( + f"imap_ultra_l3_u{sensor}-ena-h-sf-sp-full-hae-4deg-3mo_20100101_v001.cdf" + ), + call( + f"imap_ultra_l3_u{sensor}-ena-h-sf-sp-full-hae-4deg-3mo_20100401_v001.cdf" + ), + ] + ) descriptor = f"u{sensor}-ena-h-sf-sp-full-hae-4deg-3mo" expected_possible_map_to_produce = PossibleMapToProduce( input_metadata=InputMetadata( @@ -135,150 +171,187 @@ def test_get_maps_that_should_be_produced(self, mock_read_cdf_parents): start_date=datetime(2010, 4, 1), end_date=datetime(2010, 7, 1, 7, 30), version=VersionMap({descriptor: Version(None, 2)}), - descriptor=descriptor + descriptor=descriptor, ), input_files={ - 'imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v002.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100402-repoint00102_v002.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100403-repoint00103_v002.cdf', - f'imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - f'imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - f'imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - f'imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' - } + "imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v002.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100402-repoint00102_v002.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100403-repoint00103_v002.cdf", + f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100401-repoint00101_v001.cdf", + f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100402-repoint00102_v001.cdf", + f"imap_ultra_l1c_{sensor}sensor-spacecraftpset_20100403-repoint00103_v001.cdf", + f"imap_ultra_l2_u{sensor}-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv", + }, ) self.assertIn(expected_possible_map_to_produce, actual_maps_to_produce) self.assertEqual(2, len(actual_maps_to_produce)) - @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') - def test_get_maps_that_should_be_produced_lists_combined_maps(self, mock_read_cdf_parents): - self.mock_query.side_effect = ImapQueryPatcher([ - 'imap_glows_l3e_survival-probability-ul-sf_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100103-repoint00003_v001.cdf', - - 'imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v002.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100402-repoint00102_v002.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100403-repoint00103_v002.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v001.cdf', - - 'imap_glows_l3e_survival-probability-ul-sf_20100703-repoint00201_v001.cdf', - - 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v000.cdf', - 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', - 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf', - - 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v000.cdf', - 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', - 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf', - - 'imap_ultra_l3_ulc-ena-h-sf-sp-full-hae-4deg-3mo_20100101_v001.cdf', - 'imap_ultra_l3_ulc-ena-h-sf-sp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' - ]) + @patch("imap_l3_processing.maps.map_initializer.read_cdf_parents") + def test_get_maps_that_should_be_produced_lists_combined_maps( + self, mock_read_cdf_parents + ): + self.mock_query.side_effect = ImapQueryPatcher( + [ + "imap_glows_l3e_survival-probability-ul-sf_20100101-repoint00001_v001.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100102-repoint00002_v001.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100103-repoint00003_v001.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v002.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100402-repoint00102_v002.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100403-repoint00103_v002.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v001.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100703-repoint00201_v001.cdf", + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf", + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v000.cdf", + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf", + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf", + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf", + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v000.cdf", + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf", + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf", + "imap_ultra_l3_ulc-ena-h-sf-sp-full-hae-4deg-3mo_20100101_v001.cdf", + "imap_ultra_l3_ulc-ena-h-sf-sp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv", + ] + ) mock_read_cdf_parents.side_effect = [ [ "imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", "imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", - "imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf" + "imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf", ], [ "imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", "imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", - "imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf" + "imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf", ], [ - 'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf' + "imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf", ], [ - 'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v001.cdf' + "imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v001.cdf", ], [ - 'imap_ultra_l1c_45sensor-spacecraftpset_20100701-repoint00201_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100702-repoint00202_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100703-repoint00203_v001.cdf' + "imap_ultra_l1c_45sensor-spacecraftpset_20100701-repoint00201_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100702-repoint00202_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100703-repoint00203_v001.cdf", ], [ - 'imap_ultra_l1c_90sensor-spacecraftpset_20100701-repoint00201_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100702-repoint00202_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100703-repoint00203_v001.cdf' + "imap_ultra_l1c_90sensor-spacecraftpset_20100701-repoint00201_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100702-repoint00202_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100703-repoint00203_v001.cdf", ], [ - 'imap_ultra_l1c_45sensor-spacecraftpset_20101001-repoint00301_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20101002-repoint00302_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20101003-repoint00303_v001.cdf' + "imap_ultra_l1c_45sensor-spacecraftpset_20101001-repoint00301_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20101002-repoint00302_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20101003-repoint00303_v001.cdf", ], [ - 'imap_ultra_l1c_90sensor-spacecraftpset_20101001-repoint00301_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20101002-repoint00302_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20101003-repoint00303_v001.cdf' + "imap_ultra_l1c_90sensor-spacecraftpset_20101001-repoint00301_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20101002-repoint00302_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20101003-repoint00303_v001.cdf", ], [ - 'imap_glows_l3e_survival-probability-ul-sf_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100103-repoint00003_v001.cdf', + "imap_glows_l3e_survival-probability-ul-sf_20100101-repoint00001_v001.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100102-repoint00002_v001.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100103-repoint00003_v001.cdf", "imap_ultra_l1c_45sensor-spacecraftpset_20100101-repoint00001_v001.cdf", "imap_ultra_l1c_45sensor-spacecraftpset_20100102-repoint00002_v001.cdf", "imap_ultra_l1c_45sensor-spacecraftpset_20100103-repoint00003_v001.cdf", "imap_ultra_l1c_90sensor-spacecraftpset_20100101-repoint00001_v001.cdf", "imap_ultra_l1c_90sensor-spacecraftpset_20100102-repoint00002_v001.cdf", "imap_ultra_l1c_90sensor-spacecraftpset_20100103-repoint00003_v001.cdf", - 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv', + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf", + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf", + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv", ], [ - 'imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v001.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100402-repoint00102_v001.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100403-repoint00103_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv', - ] + "imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v001.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100402-repoint00102_v001.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100403-repoint00103_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v001.cdf", + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv", + ], ] initializer = UltraSPInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced("ulc-ena-h-sf-sp-full-hae-4deg-3mo", - None) - - self.mock_query.assert_has_calls([ - call(instrument='glows', data_level='l3e', descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, version="latest"), - call(instrument='glows', data_level='l3e', descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, version="latest"), - call(instrument='ultra', data_level='l2'), - call(instrument="ultra", table='ancillary', descriptor='l2-energy-bin-group-sizes', - version='latest'), - call(instrument='ultra', data_level='l3'), - ]) + actual_maps_to_produce = initializer.get_maps_that_should_be_produced( + "ulc-ena-h-sf-sp-full-hae-4deg-3mo", None + ) - mock_read_cdf_parents.assert_has_calls([ - call('imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), - call('imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), - call('imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), - call('imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), - call('imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), - call('imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), - call('imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf'), - call('imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf'), + self.mock_query.assert_has_calls( + [ + call( + instrument="glows", + data_level="l3e", + descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + version="latest", + ), + call( + instrument="glows", + data_level="l3e", + descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, + version="latest", + ), + call(instrument="ultra", data_level="l2"), + call( + instrument="ultra", + table="ancillary", + descriptor="l2-energy-bin-group-sizes", + version="latest", + ), + call(instrument="ultra", data_level="l3"), + ] + ) - call('imap_ultra_l3_ulc-ena-h-sf-sp-full-hae-4deg-3mo_20100101_v001.cdf'), - call('imap_ultra_l3_ulc-ena-h-sf-sp-full-hae-4deg-3mo_20100401_v001.cdf'), - ]) + mock_read_cdf_parents.assert_has_calls( + [ + call( + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf" + ), + call( + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100101_v001.cdf" + ), + call( + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf" + ), + call( + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf" + ), + call( + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf" + ), + call( + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100701_v001.cdf" + ), + call( + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf" + ), + call( + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20101001_v001.cdf" + ), + call( + "imap_ultra_l3_ulc-ena-h-sf-sp-full-hae-4deg-3mo_20100101_v001.cdf" + ), + call( + "imap_ultra_l3_ulc-ena-h-sf-sp-full-hae-4deg-3mo_20100401_v001.cdf" + ), + ] + ) descriptor = "ulc-ena-h-sf-sp-full-hae-4deg-3mo" expected_possible_map_to_produce = PossibleMapToProduce( @@ -288,154 +361,191 @@ def test_get_maps_that_should_be_produced_lists_combined_maps(self, mock_read_cd start_date=datetime(2010, 4, 1), end_date=datetime(2010, 7, 1, 7, 30), version=VersionMap({descriptor: Version(None, 2)}), - descriptor=descriptor + descriptor=descriptor, ), input_files={ - 'imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v002.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100402-repoint00102_v002.cdf', - 'imap_glows_l3e_survival-probability-ul-sf_20100403-repoint00103_v002.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - 'imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v001.cdf', - 'imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v001.cdf', - 'imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' - } + "imap_glows_l3e_survival-probability-ul-sf_20100401-repoint00101_v002.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100402-repoint00102_v002.cdf", + "imap_glows_l3e_survival-probability-ul-sf_20100403-repoint00103_v002.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100401-repoint00101_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100402-repoint00102_v001.cdf", + "imap_ultra_l1c_45sensor-spacecraftpset_20100403-repoint00103_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100401-repoint00101_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100402-repoint00102_v001.cdf", + "imap_ultra_l1c_90sensor-spacecraftpset_20100403-repoint00103_v001.cdf", + "imap_ultra_l2_u45-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv", + }, ) self.assertIn(expected_possible_map_to_produce, actual_maps_to_produce) self.assertEqual(2, len(actual_maps_to_produce)) - @patch('imap_l3_processing.maps.map_initializer.read_cdf_parents') - def test_get_maps_that_should_be_produced_lists_cg_corrected_maps(self, mock_read_cdf_parents): - self.mock_query.side_effect = ImapQueryPatcher([ - 'imap_glows_l3e_survival-probability-ul-hf_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20100103-repoint00003_v001.cdf', - - 'imap_glows_l3e_survival-probability-ul-hf_20100401-repoint00101_v002.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20100402-repoint00102_v002.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20100403-repoint00103_v002.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20100401-repoint00101_v001.cdf', - - 'imap_glows_l3e_survival-probability-ul-hf_20100703-repoint00201_v001.cdf', - - 'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - 'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v000.cdf', - 'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', - 'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20101001_v001.cdf', - - 'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - 'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v000.cdf', - 'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100701_v001.cdf', - 'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20101001_v001.cdf', - - 'imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-4deg-3mo_20100101_v001.cdf', - 'imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' - ]) + @patch("imap_l3_processing.maps.map_initializer.read_cdf_parents") + def test_get_maps_that_should_be_produced_lists_cg_corrected_maps( + self, mock_read_cdf_parents + ): + self.mock_query.side_effect = ImapQueryPatcher( + [ + "imap_glows_l3e_survival-probability-ul-hf_20100101-repoint00001_v001.cdf", + "imap_glows_l3e_survival-probability-ul-hf_20100102-repoint00002_v001.cdf", + "imap_glows_l3e_survival-probability-ul-hf_20100103-repoint00003_v001.cdf", + "imap_glows_l3e_survival-probability-ul-hf_20100401-repoint00101_v002.cdf", + "imap_glows_l3e_survival-probability-ul-hf_20100402-repoint00102_v002.cdf", + "imap_glows_l3e_survival-probability-ul-hf_20100403-repoint00103_v002.cdf", + "imap_glows_l3e_survival-probability-ul-hf_20100401-repoint00101_v001.cdf", + "imap_glows_l3e_survival-probability-ul-hf_20100703-repoint00201_v001.cdf", + "imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf", + "imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v000.cdf", + "imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100701_v001.cdf", + "imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20101001_v001.cdf", + "imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf", + "imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v000.cdf", + "imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100701_v001.cdf", + "imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20101001_v001.cdf", + "imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-4deg-3mo_20100101_v001.cdf", + "imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv", + ] + ) mock_read_cdf_parents.side_effect = [ [ "imap_ultra_l1c_45sensor-heliopset_20100101-repoint00001_v001.cdf", "imap_ultra_l1c_45sensor-heliopset_20100102-repoint00002_v001.cdf", - "imap_ultra_l1c_45sensor-heliopset_20100103-repoint00003_v001.cdf" + "imap_ultra_l1c_45sensor-heliopset_20100103-repoint00003_v001.cdf", ], [ "imap_ultra_l1c_90sensor-heliopset_20100101-repoint00001_v001.cdf", "imap_ultra_l1c_90sensor-heliopset_20100102-repoint00002_v001.cdf", - "imap_ultra_l1c_90sensor-heliopset_20100103-repoint00003_v001.cdf" + "imap_ultra_l1c_90sensor-heliopset_20100103-repoint00003_v001.cdf", ], [ - 'imap_ultra_l1c_45sensor-heliopset_20100401-repoint00101_v001.cdf', - 'imap_ultra_l1c_45sensor-heliopset_20100402-repoint00102_v001.cdf', - 'imap_ultra_l1c_45sensor-heliopset_20100403-repoint00103_v001.cdf' + "imap_ultra_l1c_45sensor-heliopset_20100401-repoint00101_v001.cdf", + "imap_ultra_l1c_45sensor-heliopset_20100402-repoint00102_v001.cdf", + "imap_ultra_l1c_45sensor-heliopset_20100403-repoint00103_v001.cdf", ], [ - 'imap_ultra_l1c_90sensor-heliopset_20100401-repoint00101_v001.cdf', - 'imap_ultra_l1c_90sensor-heliopset_20100402-repoint00102_v001.cdf', - 'imap_ultra_l1c_90sensor-heliopset_20100403-repoint00103_v001.cdf' + "imap_ultra_l1c_90sensor-heliopset_20100401-repoint00101_v001.cdf", + "imap_ultra_l1c_90sensor-heliopset_20100402-repoint00102_v001.cdf", + "imap_ultra_l1c_90sensor-heliopset_20100403-repoint00103_v001.cdf", ], [ - 'imap_ultra_l1c_45sensor-heliopset_20100701-repoint00201_v001.cdf', - 'imap_ultra_l1c_45sensor-heliopset_20100702-repoint00202_v001.cdf', - 'imap_ultra_l1c_45sensor-heliopset_20100703-repoint00203_v001.cdf' + "imap_ultra_l1c_45sensor-heliopset_20100701-repoint00201_v001.cdf", + "imap_ultra_l1c_45sensor-heliopset_20100702-repoint00202_v001.cdf", + "imap_ultra_l1c_45sensor-heliopset_20100703-repoint00203_v001.cdf", ], [ - 'imap_ultra_l1c_90sensor-heliopset_20100701-repoint00201_v001.cdf', - 'imap_ultra_l1c_90sensor-heliopset_20100702-repoint00202_v001.cdf', - 'imap_ultra_l1c_90sensor-heliopset_20100703-repoint00203_v001.cdf' + "imap_ultra_l1c_90sensor-heliopset_20100701-repoint00201_v001.cdf", + "imap_ultra_l1c_90sensor-heliopset_20100702-repoint00202_v001.cdf", + "imap_ultra_l1c_90sensor-heliopset_20100703-repoint00203_v001.cdf", ], [ - 'imap_ultra_l1c_45sensor-heliopset_20101001-repoint00301_v001.cdf', - 'imap_ultra_l1c_45sensor-heliopset_20101002-repoint00302_v001.cdf', - 'imap_ultra_l1c_45sensor-heliopset_20101003-repoint00303_v001.cdf' + "imap_ultra_l1c_45sensor-heliopset_20101001-repoint00301_v001.cdf", + "imap_ultra_l1c_45sensor-heliopset_20101002-repoint00302_v001.cdf", + "imap_ultra_l1c_45sensor-heliopset_20101003-repoint00303_v001.cdf", ], [ - 'imap_ultra_l1c_90sensor-heliopset_20101001-repoint00301_v001.cdf', - 'imap_ultra_l1c_90sensor-heliopset_20101002-repoint00302_v001.cdf', - 'imap_ultra_l1c_90sensor-heliopset_20101003-repoint00303_v001.cdf' + "imap_ultra_l1c_90sensor-heliopset_20101001-repoint00301_v001.cdf", + "imap_ultra_l1c_90sensor-heliopset_20101002-repoint00302_v001.cdf", + "imap_ultra_l1c_90sensor-heliopset_20101003-repoint00303_v001.cdf", ], [ - 'imap_glows_l3e_survival-probability-ul-hf_20100101-repoint00001_v001.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20100102-repoint00002_v001.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20100103-repoint00003_v001.cdf', + "imap_glows_l3e_survival-probability-ul-hf_20100101-repoint00001_v001.cdf", + "imap_glows_l3e_survival-probability-ul-hf_20100102-repoint00002_v001.cdf", + "imap_glows_l3e_survival-probability-ul-hf_20100103-repoint00003_v001.cdf", "imap_ultra_l1c_45sensor-heliopset_20100101-repoint00001_v001.cdf", "imap_ultra_l1c_45sensor-heliopset_20100102-repoint00002_v001.cdf", "imap_ultra_l1c_45sensor-heliopset_20100103-repoint00003_v001.cdf", "imap_ultra_l1c_90sensor-heliopset_20100101-repoint00001_v001.cdf", "imap_ultra_l1c_90sensor-heliopset_20100102-repoint00002_v001.cdf", "imap_ultra_l1c_90sensor-heliopset_20100103-repoint00003_v001.cdf", - 'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - 'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv', + "imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf", + "imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf", + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv", ], [ - 'imap_glows_l3e_survival-probability-ul-hf_20100401-repoint00101_v001.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20100402-repoint00102_v001.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20100403-repoint00103_v001.cdf', - 'imap_ultra_l1c_45sensor-heliopset_20100401-repoint00101_v001.cdf', - 'imap_ultra_l1c_45sensor-heliopset_20100402-repoint00102_v001.cdf', - 'imap_ultra_l1c_45sensor-heliopset_20100403-repoint00103_v001.cdf', - 'imap_ultra_l1c_90sensor-heliopset_20100401-repoint00101_v001.cdf', - 'imap_ultra_l1c_90sensor-heliopset_20100402-repoint00102_v001.cdf', - 'imap_ultra_l1c_90sensor-heliopset_20100403-repoint00103_v001.cdf', - 'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv', - ] + "imap_glows_l3e_survival-probability-ul-hf_20100401-repoint00101_v001.cdf", + "imap_glows_l3e_survival-probability-ul-hf_20100402-repoint00102_v001.cdf", + "imap_glows_l3e_survival-probability-ul-hf_20100403-repoint00103_v001.cdf", + "imap_ultra_l1c_45sensor-heliopset_20100401-repoint00101_v001.cdf", + "imap_ultra_l1c_45sensor-heliopset_20100402-repoint00102_v001.cdf", + "imap_ultra_l1c_45sensor-heliopset_20100403-repoint00103_v001.cdf", + "imap_ultra_l1c_90sensor-heliopset_20100401-repoint00101_v001.cdf", + "imap_ultra_l1c_90sensor-heliopset_20100402-repoint00102_v001.cdf", + "imap_ultra_l1c_90sensor-heliopset_20100403-repoint00103_v001.cdf", + "imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv", + ], ] initializer = UltraSPInitializer() - actual_maps_to_produce = initializer.get_maps_that_should_be_produced("ulc-ena-h-hf-sp-full-hae-4deg-3mo", - None) - - self.mock_query.assert_has_calls([ - call(instrument='glows', data_level='l3e', descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, version="latest"), - call(instrument='glows', data_level='l3e', descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, version="latest"), - call(instrument='ultra', data_level='l2'), - call(instrument="ultra", table='ancillary', descriptor='l2-energy-bin-group-sizes', - version='latest'), - call(instrument='ultra', data_level='l3'), - ]) + actual_maps_to_produce = initializer.get_maps_that_should_be_produced( + "ulc-ena-h-hf-sp-full-hae-4deg-3mo", None + ) - mock_read_cdf_parents.assert_has_calls([ - call('imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), - call('imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf'), - call('imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), - call('imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf'), - call('imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), - call('imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100701_v001.cdf'), - call('imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20101001_v001.cdf'), - call('imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20101001_v001.cdf'), + self.mock_query.assert_has_calls( + [ + call( + instrument="glows", + data_level="l3e", + descriptor=GLOWS_L3E_ULTRA_SF_DESCRIPTOR, + version="latest", + ), + call( + instrument="glows", + data_level="l3e", + descriptor=GLOWS_L3E_ULTRA_HF_DESCRIPTOR, + version="latest", + ), + call(instrument="ultra", data_level="l2"), + call( + instrument="ultra", + table="ancillary", + descriptor="l2-energy-bin-group-sizes", + version="latest", + ), + call(instrument="ultra", data_level="l3"), + ] + ) - call('imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-4deg-3mo_20100101_v001.cdf'), - call('imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-4deg-3mo_20100401_v001.cdf'), - ]) + mock_read_cdf_parents.assert_has_calls( + [ + call( + "imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf" + ), + call( + "imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100101_v001.cdf" + ), + call( + "imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf" + ), + call( + "imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf" + ), + call( + "imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100701_v001.cdf" + ), + call( + "imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100701_v001.cdf" + ), + call( + "imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20101001_v001.cdf" + ), + call( + "imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20101001_v001.cdf" + ), + call( + "imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-4deg-3mo_20100101_v001.cdf" + ), + call( + "imap_ultra_l3_ulc-ena-h-hf-sp-full-hae-4deg-3mo_20100401_v001.cdf" + ), + ] + ) descriptor = "ulc-ena-h-hf-sp-full-hae-4deg-3mo" expected_possible_map_to_produce = PossibleMapToProduce( @@ -448,24 +558,24 @@ def test_get_maps_that_should_be_produced_lists_cg_corrected_maps(self, mock_rea descriptor=descriptor, ), input_files={ - 'imap_glows_l3e_survival-probability-ul-hf_20100401-repoint00101_v002.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20100402-repoint00102_v002.cdf', - 'imap_glows_l3e_survival-probability-ul-hf_20100403-repoint00103_v002.cdf', - 'imap_ultra_l1c_45sensor-heliopset_20100401-repoint00101_v001.cdf', - 'imap_ultra_l1c_45sensor-heliopset_20100402-repoint00102_v001.cdf', - 'imap_ultra_l1c_45sensor-heliopset_20100403-repoint00103_v001.cdf', - 'imap_ultra_l1c_90sensor-heliopset_20100401-repoint00101_v001.cdf', - 'imap_ultra_l1c_90sensor-heliopset_20100402-repoint00102_v001.cdf', - 'imap_ultra_l1c_90sensor-heliopset_20100403-repoint00103_v001.cdf', - 'imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf', - 'imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv' - } + "imap_glows_l3e_survival-probability-ul-hf_20100401-repoint00101_v002.cdf", + "imap_glows_l3e_survival-probability-ul-hf_20100402-repoint00102_v002.cdf", + "imap_glows_l3e_survival-probability-ul-hf_20100403-repoint00103_v002.cdf", + "imap_ultra_l1c_45sensor-heliopset_20100401-repoint00101_v001.cdf", + "imap_ultra_l1c_45sensor-heliopset_20100402-repoint00102_v001.cdf", + "imap_ultra_l1c_45sensor-heliopset_20100403-repoint00103_v001.cdf", + "imap_ultra_l1c_90sensor-heliopset_20100401-repoint00101_v001.cdf", + "imap_ultra_l1c_90sensor-heliopset_20100402-repoint00102_v001.cdf", + "imap_ultra_l1c_90sensor-heliopset_20100403-repoint00103_v001.cdf", + "imap_ultra_l2_u45-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2_u90-ena-h-hf-nsp-full-hae-4deg-3mo_20100401_v001.cdf", + "imap_ultra_l2-energy-bin-group-sizes_20250101_v000.csv", + }, ) self.assertIn(expected_possible_map_to_produce, actual_maps_to_produce) self.assertEqual(2, len(actual_maps_to_produce)) - @patch('imap_l3_processing.ultra.ultra_sp_initializer.furnish_spice_metakernel') + @patch("imap_l3_processing.ultra.ultra_sp_initializer.furnish_spice_metakernel") def test_furnish_spice_dependencies(self, mock_furnish_metakernel): start_date = datetime(2025, 4, 15) end_date = datetime(2025, 7, 15) @@ -477,12 +587,15 @@ def test_furnish_spice_dependencies(self, mock_furnish_metakernel): start_date=start_date, end_date=end_date, version=VersionMap({descriptor: Version(None, 0)}), - descriptor=descriptor + descriptor=descriptor, ) map_to_produce = PossibleMapToProduce(set(), input_metadata) ultra_initializer = UltraSPInitializer() ultra_initializer.furnish_spice_dependencies(map_to_produce) - mock_furnish_metakernel.assert_called_once_with(start_date=start_date, end_date=end_date, - kernel_types=ULTRA_SP_SPICE_KERNELS) + mock_furnish_metakernel.assert_called_once_with( + start_date=start_date, + end_date=end_date, + kernel_types=ULTRA_SP_SPICE_KERNELS, + ) From 9ecd008f354e4acf0d453b7f114521bfb576435b Mon Sep 17 00:00:00 2001 From: Jonathan Niehof Date: Fri, 23 Oct 2026 11:30:55 -0600 Subject: [PATCH 3/4] Hand-fix formatting --- imap_l3_processing/maps/mpfit.py | 768 +++++++++--------- ...ate_example_ultra_l2_map_with_power_law.py | 2 +- .../codice/l3/lo/test_codice_lo_processor.py | 24 +- 3 files changed, 403 insertions(+), 391 deletions(-) diff --git a/imap_l3_processing/maps/mpfit.py b/imap_l3_processing/maps/mpfit.py index 1c4b34e07..73e58da64 100644 --- a/imap_l3_processing/maps/mpfit.py +++ b/imap_l3_processing/maps/mpfit.py @@ -412,187 +412,189 @@ def myfunct(p, fjac=None, x=None, y=None, err=None) import scipy.linalg.blas -# Original FORTRAN documentation -# ********** +# fmt: off +# Original FORTRAN documentation +# ********** # -# subroutine lmdif +# subroutine lmdif # -# the purpose of lmdif is to minimize the sum of the squares of -# m nonlinear functions in n variables by a modification of -# the levenberg-marquardt algorithm. the user must provide a -# subroutine which calculates the functions. the jacobian is -# then calculated by a forward-difference approximation. +# the purpose of lmdif is to minimize the sum of the squares of +# m nonlinear functions in n variables by a modification of +# the levenberg-marquardt algorithm. the user must provide a +# subroutine which calculates the functions. the jacobian is +# then calculated by a forward-difference approximation. # -# the subroutine statement is +# the subroutine statement is # -# subroutine lmdif(fcn,m,n,x,fvec,ftol,xtol,gtol,maxfev,epsfcn, -# diag,mode,factor,nprint,info,nfev,fjac, -# ldfjac,ipvt,qtf,wa1,wa2,wa3,wa4) +# subroutine lmdif(fcn,m,n,x,fvec,ftol,xtol,gtol,maxfev,epsfcn, +# diag,mode,factor,nprint,info,nfev,fjac, +# ldfjac,ipvt,qtf,wa1,wa2,wa3,wa4) # -# where +# where # -# fcn is the name of the user-supplied subroutine which -# calculates the functions. fcn must be declared -# in an external statement in the user calling -# program, and should be written as follows. +# fcn is the name of the user-supplied subroutine which +# calculates the functions. fcn must be declared +# in an external statement in the user calling +# program, and should be written as follows. # -# subroutine fcn(m,n,x,fvec,iflag) -# integer m,n,iflag -# double precision x(n),fvec(m) -# ---------- -# calculate the functions at x and -# return this vector in fvec. -# ---------- -# return -# end +# subroutine fcn(m,n,x,fvec,iflag) +# integer m,n,iflag +# double precision x(n),fvec(m) +# ---------- +# calculate the functions at x and +# return this vector in fvec. +# ---------- +# return +# end # -# the value of iflag should not be changed by fcn unless -# the user wants to terminate execution of lmdif. -# in this case set iflag to a negative integer. +# the value of iflag should not be changed by fcn unless +# the user wants to terminate execution of lmdif. +# in this case set iflag to a negative integer. # -# m is a positive integer input variable set to the number -# of functions. +# m is a positive integer input variable set to the number +# of functions. # -# n is a positive integer input variable set to the number -# of variables. n must not exceed m. +# n is a positive integer input variable set to the number +# of variables. n must not exceed m. # -# x is an array of length n. on input x must contain -# an initial estimate of the solution vector. on output x -# contains the final estimate of the solution vector. +# x is an array of length n. on input x must contain +# an initial estimate of the solution vector. on output x +# contains the final estimate of the solution vector. # -# fvec is an output array of length m which contains -# the functions evaluated at the output x. +# fvec is an output array of length m which contains +# the functions evaluated at the output x. # -# ftol is a nonnegative input variable. termination -# occurs when both the actual and predicted relative -# reductions in the sum of squares are at most ftol. -# therefore, ftol measures the relative error desired -# in the sum of squares. +# ftol is a nonnegative input variable. termination +# occurs when both the actual and predicted relative +# reductions in the sum of squares are at most ftol. +# therefore, ftol measures the relative error desired +# in the sum of squares. # -# xtol is a nonnegative input variable. termination -# occurs when the relative error between two consecutive -# iterates is at most xtol. therefore, xtol measures the -# relative error desired in the approximate solution. +# xtol is a nonnegative input variable. termination +# occurs when the relative error between two consecutive +# iterates is at most xtol. therefore, xtol measures the +# relative error desired in the approximate solution. # -# gtol is a nonnegative input variable. termination -# occurs when the cosine of the angle between fvec and -# any column of the jacobian is at most gtol in absolute -# value. therefore, gtol measures the orthogonality -# desired between the function vector and the columns -# of the jacobian. +# gtol is a nonnegative input variable. termination +# occurs when the cosine of the angle between fvec and +# any column of the jacobian is at most gtol in absolute +# value. therefore, gtol measures the orthogonality +# desired between the function vector and the columns +# of the jacobian. # -# maxfev is a positive integer input variable. termination -# occurs when the number of calls to fcn is at least -# maxfev by the end of an iteration. +# maxfev is a positive integer input variable. termination +# occurs when the number of calls to fcn is at least +# maxfev by the end of an iteration. # -# epsfcn is an input variable used in determining a suitable -# step length for the forward-difference approximation. this -# approximation assumes that the relative errors in the -# functions are of the order of epsfcn. if epsfcn is less -# than the machine precision, it is assumed that the relative -# errors in the functions are of the order of the machine -# precision. +# epsfcn is an input variable used in determining a suitable +# step length for the forward-difference approximation. this +# approximation assumes that the relative errors in the +# functions are of the order of epsfcn. if epsfcn is less +# than the machine precision, it is assumed that the relative +# errors in the functions are of the order of the machine +# precision. # -# diag is an array of length n. if mode = 1 (see -# below), diag is internally set. if mode = 2, diag -# must contain positive entries that serve as -# multiplicative scale factors for the variables. +# diag is an array of length n. if mode = 1 (see +# below), diag is internally set. if mode = 2, diag +# must contain positive entries that serve as +# multiplicative scale factors for the variables. # -# mode is an integer input variable. if mode = 1, the -# variables will be scaled internally. if mode = 2, -# the scaling is specified by the input diag. other -# values of mode are equivalent to mode = 1. +# mode is an integer input variable. if mode = 1, the +# variables will be scaled internally. if mode = 2, +# the scaling is specified by the input diag. other +# values of mode are equivalent to mode = 1. # -# factor is a positive input variable used in determining the -# initial step bound. this bound is set to the product of -# factor and the euclidean norm of diag*x if nonzero, or else -# to factor itself. in most cases factor should lie in the -# interval (.1,100.). 100. is a generally recommended value. +# factor is a positive input variable used in determining the +# initial step bound. this bound is set to the product of +# factor and the euclidean norm of diag*x if nonzero, or else +# to factor itself. in most cases factor should lie in the +# interval (.1,100.). 100. is a generally recommended value. # -# nprint is an integer input variable that enables controlled -# printing of iterates if it is positive. in this case, -# fcn is called with iflag = 0 at the beginning of the first -# iteration and every nprint iterations thereafter and -# immediately prior to return, with x and fvec available -# for printing. if nprint is not positive, no special calls -# of fcn with iflag = 0 are made. +# nprint is an integer input variable that enables controlled +# printing of iterates if it is positive. in this case, +# fcn is called with iflag = 0 at the beginning of the first +# iteration and every nprint iterations thereafter and +# immediately prior to return, with x and fvec available +# for printing. if nprint is not positive, no special calls +# of fcn with iflag = 0 are made. # -# info is an integer output variable. if the user has -# terminated execution, info is set to the (negative) -# value of iflag. see description of fcn. otherwise, -# info is set as follows. +# info is an integer output variable. if the user has +# terminated execution, info is set to the (negative) +# value of iflag. see description of fcn. otherwise, +# info is set as follows. # -# info = 0 improper input parameters. +# info = 0 improper input parameters. # -# info = 1 both actual and predicted relative reductions -# in the sum of squares are at most ftol. +# info = 1 both actual and predicted relative reductions +# in the sum of squares are at most ftol. # -# info = 2 relative error between two consecutive iterates -# is at most xtol. +# info = 2 relative error between two consecutive iterates +# is at most xtol. # -# info = 3 conditions for info = 1 and info = 2 both hold. +# info = 3 conditions for info = 1 and info = 2 both hold. # -# info = 4 the cosine of the angle between fvec and any -# column of the jacobian is at most gtol in -# absolute value. +# info = 4 the cosine of the angle between fvec and any +# column of the jacobian is at most gtol in +# absolute value. # -# info = 5 number of calls to fcn has reached or -# exceeded maxfev. +# info = 5 number of calls to fcn has reached or +# exceeded maxfev. # -# info = 6 ftol is too small. no further reduction in -# the sum of squares is possible. +# info = 6 ftol is too small. no further reduction in +# the sum of squares is possible. # -# info = 7 xtol is too small. no further improvement in -# the approximate solution x is possible. +# info = 7 xtol is too small. no further improvement in +# the approximate solution x is possible. # -# info = 8 gtol is too small. fvec is orthogonal to the -# columns of the jacobian to machine precision. +# info = 8 gtol is too small. fvec is orthogonal to the +# columns of the jacobian to machine precision. # -# nfev is an integer output variable set to the number of -# calls to fcn. +# nfev is an integer output variable set to the number of +# calls to fcn. # -# fjac is an output m by n array. the upper n by n submatrix -# of fjac contains an upper triangular matrix r with -# diagonal elements of nonincreasing magnitude such that +# fjac is an output m by n array. the upper n by n submatrix +# of fjac contains an upper triangular matrix r with +# diagonal elements of nonincreasing magnitude such that # -# t t t -# p *(jac *jac)*p = r *r, +# t t t +# p *(jac *jac)*p = r *r, # -# where p is a permutation matrix and jac is the final -# calculated jacobian. column j of p is column ipvt(j) -# (see below) of the identity matrix. the lower trapezoidal -# part of fjac contains information generated during -# the computation of r. +# where p is a permutation matrix and jac is the final +# calculated jacobian. column j of p is column ipvt(j) +# (see below) of the identity matrix. the lower trapezoidal +# part of fjac contains information generated during +# the computation of r. # -# ldfjac is a positive integer input variable not less than m -# which specifies the leading dimension of the array fjac. +# ldfjac is a positive integer input variable not less than m +# which specifies the leading dimension of the array fjac. # -# ipvt is an integer output array of length n. ipvt -# defines a permutation matrix p such that jac*p = q*r, -# where jac is the final calculated jacobian, q is -# orthogonal (not stored), and r is upper triangular -# with diagonal elements of nonincreasing magnitude. -# column j of p is column ipvt(j) of the identity matrix. +# ipvt is an integer output array of length n. ipvt +# defines a permutation matrix p such that jac*p = q*r, +# where jac is the final calculated jacobian, q is +# orthogonal (not stored), and r is upper triangular +# with diagonal elements of nonincreasing magnitude. +# column j of p is column ipvt(j) of the identity matrix. # -# qtf is an output array of length n which contains -# the first n elements of the vector (q transpose)*fvec. +# qtf is an output array of length n which contains +# the first n elements of the vector (q transpose)*fvec. # -# wa1, wa2, and wa3 are work arrays of length n. +# wa1, wa2, and wa3 are work arrays of length n. # -# wa4 is a work array of length m. +# wa4 is a work array of length m. # -# subprograms called +# subprograms called # -# user-supplied ...... fcn +# user-supplied ...... fcn # -# minpack-supplied ... dpmpar,enorm,fdjac2,,qrfac +# minpack-supplied ... dpmpar,enorm,fdjac2,,qrfac # -# fortran-supplied ... dabs,dmax1,dmin1,dsqrt,mod +# fortran-supplied ... dabs,dmax1,dmin1,dsqrt,mod # -# argonne national laboratory. minpack project. march 1980. -# burton s. garbow, kenneth e. hillstrom, jorge j. more +# argonne national laboratory. minpack project. march 1980. +# burton s. garbow, kenneth e. hillstrom, jorge j. more # -# ********** +# ********** +# fmt: on class mpfit: @@ -1521,21 +1523,23 @@ def defiter( print(p + (pformat % x[i]) + " ") return 0 + # fmt: off # DO_ITERSTOP: # if keyword_set(iterstop) then begin - # k = get_kbrd(0) - # if k EQ string(byte(7)) then begin - # message, 'WARNING: minimization not complete', /info - # print, 'Do you want to terminate this procedure? (y/n)', $ - # format='(A,$)' - # k = '' - # read, k - # if strupcase(strmid(k,0,1)) EQ 'Y' then begin - # message, 'WARNING: Procedure is terminating.', /info - # mperr = -1 - # endif - # endif + # k = get_kbrd(0) + # if k EQ string(byte(7)) then begin + # message, 'WARNING: minimization not complete', /info + # print, 'Do you want to terminate this procedure? (y/n)', $ + # format='(A,$)' + # k = '' + # read, k + # if strupcase(strmid(k,0,1)) EQ 'Y' then begin + # message, 'WARNING: Procedure is terminating.', /info + # mperr = -1 + # endif + # endif # endif + # fmt: on # Procedure to parse the parameter values in PARINFO, which is a list of dictionaries def parinfo(self, parinfo=None, key="a", default=None, n=0): @@ -1699,82 +1703,83 @@ def fdjac2( fjac[0:, j] = (fp - fm) / (2 * h[j]) return fjac - # Original FORTRAN documentation - # ********** + # fmt: off + # Original FORTRAN documentation + # ********** # - # subroutine qrfac + # subroutine qrfac # - # this subroutine uses householder transformations with column - # pivoting (optional) to compute a qr factorization of the - # m by n matrix a. that is, qrfac determines an orthogonal - # matrix q, a permutation matrix p, and an upper trapezoidal - # matrix r with diagonal elements of nonincreasing magnitude, - # such that a*p = q*r. the householder transformation for - # column k, k = 1,2,...,min(m,n), is of the form + # this subroutine uses householder transformations with column + # pivoting (optional) to compute a qr factorization of the + # m by n matrix a. that is, qrfac determines an orthogonal + # matrix q, a permutation matrix p, and an upper trapezoidal + # matrix r with diagonal elements of nonincreasing magnitude, + # such that a*p = q*r. the householder transformation for + # column k, k = 1,2,...,min(m,n), is of the form # - # t - # i - (1/u(k))*u*u + # t + # i - (1/u(k))*u*u # - # where u has zeros in the first k-1 positions. the form of - # this transformation and the method of pivoting first - # appeared in the corresponding linpack subroutine. + # where u has zeros in the first k-1 positions. the form of + # this transformation and the method of pivoting first + # appeared in the corresponding linpack subroutine. # - # the subroutine statement is + # the subroutine statement is # - # subroutine qrfac(m,n,a,lda,pivot,ipvt,lipvt,rdiag,acnorm,wa) + # subroutine qrfac(m,n,a,lda,pivot,ipvt,lipvt,rdiag,acnorm,wa) # - # where + # where # - # m is a positive integer input variable set to the number - # of rows of a. + # m is a positive integer input variable set to the number + # of rows of a. # - # n is a positive integer input variable set to the number - # of columns of a. + # n is a positive integer input variable set to the number + # of columns of a. # - # a is an m by n array. on input a contains the matrix for - # which the qr factorization is to be computed. on output - # the strict upper trapezoidal part of a contains the strict - # upper trapezoidal part of r, and the lower trapezoidal - # part of a contains a factored form of q (the non-trivial - # elements of the u vectors described above). + # a is an m by n array. on input a contains the matrix for + # which the qr factorization is to be computed. on output + # the strict upper trapezoidal part of a contains the strict + # upper trapezoidal part of r, and the lower trapezoidal + # part of a contains a factored form of q (the non-trivial + # elements of the u vectors described above). # - # lda is a positive integer input variable not less than m - # which specifies the leading dimension of the array a. + # lda is a positive integer input variable not less than m + # which specifies the leading dimension of the array a. # - # pivot is a logical input variable. if pivot is set true, - # then column pivoting is enforced. if pivot is set false, - # then no column pivoting is done. + # pivot is a logical input variable. if pivot is set true, + # then column pivoting is enforced. if pivot is set false, + # then no column pivoting is done. # - # ipvt is an integer output array of length lipvt. ipvt - # defines the permutation matrix p such that a*p = q*r. - # column j of p is column ipvt(j) of the identity matrix. - # if pivot is false, ipvt is not referenced. + # ipvt is an integer output array of length lipvt. ipvt + # defines the permutation matrix p such that a*p = q*r. + # column j of p is column ipvt(j) of the identity matrix. + # if pivot is false, ipvt is not referenced. # - # lipvt is a positive integer input variable. if pivot is false, - # then lipvt may be as small as 1. if pivot is true, then - # lipvt must be at least n. + # lipvt is a positive integer input variable. if pivot is false, + # then lipvt may be as small as 1. if pivot is true, then + # lipvt must be at least n. # - # rdiag is an output array of length n which contains the - # diagonal elements of r. + # rdiag is an output array of length n which contains the + # diagonal elements of r. # - # acnorm is an output array of length n which contains the - # norms of the corresponding columns of the input matrix a. - # if this information is not needed, then acnorm can coincide - # with rdiag. + # acnorm is an output array of length n which contains the + # norms of the corresponding columns of the input matrix a. + # if this information is not needed, then acnorm can coincide + # with rdiag. # - # wa is a work array of length n. if pivot is false, then wa - # can coincide with rdiag. + # wa is a work array of length n. if pivot is false, then wa + # can coincide with rdiag. # - # subprograms called + # subprograms called # - # minpack-supplied ... dpmpar,enorm + # minpack-supplied ... dpmpar,enorm # - # fortran-supplied ... dmax1,dsqrt,min0 + # fortran-supplied ... dmax1,dsqrt,min0 # - # argonne national laboratory. minpack project. march 1980. - # burton s. garbow, kenneth e. hillstrom, jorge j. more + # argonne national laboratory. minpack project. march 1980. + # burton s. garbow, kenneth e. hillstrom, jorge j. more # - # ********** + # ********** # # PIVOTING / PERMUTING: # @@ -1795,40 +1800,41 @@ def fdjac2( # output. # # EXAMPLE: decompose the matrix [[9.,2.,6.],[4.,8.,7.]] - # aa = [[9.,2.,6.],[4.,8.,7.]] - # mpfit_qrfac, aa, aapvt, rdiag, aanorm - # IDL> print, aa - # 1.81818* 0.181818* 0.545455* - # -8.54545+ 1.90160* 0.432573* - # IDL> print, rdiag - # -11.0000+ -7.48166+ + # aa = [[9.,2.,6.],[4.,8.,7.]] + # mpfit_qrfac, aa, aapvt, rdiag, aanorm + # IDL> print, aa + # 1.81818* 0.181818* 0.545455* + # -8.54545+ 1.90160* 0.432573* + # IDL> print, rdiag + # -11.0000+ -7.48166+ # # The components marked with a * are the components of the # reflectors, and those marked with a + are components of R. # # To reconstruct Q and R we proceed as follows. First R. - # r = fltarr(m, n) - # for i = 0, n-1 do r(0:i,i) = aa(0:i,i) # fill in lower diag - # r(lindgen(n)*(m+1)) = rdiag + # r = fltarr(m, n) + # for i = 0, n-1 do r(0:i,i) = aa(0:i,i) # fill in lower diag + # r(lindgen(n)*(m+1)) = rdiag # # Next, Q, which are composed from the reflectors. Each reflector v # is taken from the upper trapezoid of aa, and converted to a matrix # via (I - 2 vT . v / (v . vT)). # - # hh = ident # identity matrix + # hh = ident # identity matrix # for i = 0, n-1 do begin - # v = aa(*,i) & if i GT 0 then v(0:i-1) = 0 # extract reflector - # hh = hh # (ident - 2*(v # v)/total(v * v)) # generate matrix + # v = aa(*,i) & if i GT 0 then v(0:i-1) = 0 # extract reflector + # hh = hh # (ident - 2*(v # v)/total(v * v)) # generate matrix # endfor # # Test the result: - # IDL> print, hh # transpose(r) - # 9.00000 4.00000 - # 2.00000 8.00000 - # 6.00000 7.00000 + # IDL> print, hh # transpose(r) + # 9.00000 4.00000 + # 2.00000 8.00000 + # 6.00000 7.00000 # # Note that it is usually never necessary to form the Q matrix # explicitly, and MPFIT does not. + # fmt: on def qrfac(self, a, pivot=0): @@ -1911,83 +1917,85 @@ def qrfac(self, a, pivot=0): rdiag[j] = -ajnorm return [a, ipvt, rdiag, acnorm] - # Original FORTRAN documentation - # ********** + # fmt: off + # Original FORTRAN documentation + # ********** # - # subroutine qrsolv + # subroutine qrsolv # - # given an m by n matrix a, an n by n diagonal matrix d, - # and an m-vector b, the problem is to determine an x which - # solves the system + # given an m by n matrix a, an n by n diagonal matrix d, + # and an m-vector b, the problem is to determine an x which + # solves the system # - # a*x = b , d*x = 0 , + # a*x = b , d*x = 0 , # - # in the least squares sense. + # in the least squares sense. # - # this subroutine completes the solution of the problem - # if it is provided with the necessary information from the - # factorization, with column pivoting, of a. that is, if - # a*p = q*r, where p is a permutation matrix, q has orthogonal - # columns, and r is an upper triangular matrix with diagonal - # elements of nonincreasing magnitude, then qrsolv expects - # the full upper triangle of r, the permutation matrix p, - # and the first n components of (q transpose)*b. the system - # a*x = b, d*x = 0, is then equivalent to + # this subroutine completes the solution of the problem + # if it is provided with the necessary information from the + # factorization, with column pivoting, of a. that is, if + # a*p = q*r, where p is a permutation matrix, q has orthogonal + # columns, and r is an upper triangular matrix with diagonal + # elements of nonincreasing magnitude, then qrsolv expects + # the full upper triangle of r, the permutation matrix p, + # and the first n components of (q transpose)*b. the system + # a*x = b, d*x = 0, is then equivalent to # - # t t - # r*z = q *b , p *d*p*z = 0 , + # t t + # r*z = q *b , p *d*p*z = 0 , # - # where x = p*z. if this system does not have full rank, - # then a least squares solution is obtained. on output qrsolv - # also provides an upper triangular matrix s such that + # where x = p*z. if this system does not have full rank, + # then a least squares solution is obtained. on output qrsolv + # also provides an upper triangular matrix s such that # - # t t t - # p *(a *a + d*d)*p = s *s . + # t t t + # p *(a *a + d*d)*p = s *s . # - # s is computed within qrsolv and may be of separate interest. + # s is computed within qrsolv and may be of separate interest. # - # the subroutine statement is + # the subroutine statement is # - # subroutine qrsolv(n,r,ldr,ipvt,diag,qtb,x,sdiag,wa) + # subroutine qrsolv(n,r,ldr,ipvt,diag,qtb,x,sdiag,wa) # - # where + # where # - # n is a positive integer input variable set to the order of r. + # n is a positive integer input variable set to the order of r. # - # r is an n by n array. on input the full upper triangle - # must contain the full upper triangle of the matrix r. - # on output the full upper triangle is unaltered, and the - # strict lower triangle contains the strict upper triangle - # (transposed) of the upper triangular matrix s. + # r is an n by n array. on input the full upper triangle + # must contain the full upper triangle of the matrix r. + # on output the full upper triangle is unaltered, and the + # strict lower triangle contains the strict upper triangle + # (transposed) of the upper triangular matrix s. # - # ldr is a positive integer input variable not less than n - # which specifies the leading dimension of the array r. + # ldr is a positive integer input variable not less than n + # which specifies the leading dimension of the array r. # - # ipvt is an integer input array of length n which defines the - # permutation matrix p such that a*p = q*r. column j of p - # is column ipvt(j) of the identity matrix. + # ipvt is an integer input array of length n which defines the + # permutation matrix p such that a*p = q*r. column j of p + # is column ipvt(j) of the identity matrix. # - # diag is an input array of length n which must contain the - # diagonal elements of the matrix d. + # diag is an input array of length n which must contain the + # diagonal elements of the matrix d. # - # qtb is an input array of length n which must contain the first - # n elements of the vector (q transpose)*b. + # qtb is an input array of length n which must contain the first + # n elements of the vector (q transpose)*b. # - # x is an output array of length n which contains the least - # squares solution of the system a*x = b, d*x = 0. + # x is an output array of length n which contains the least + # squares solution of the system a*x = b, d*x = 0. # - # sdiag is an output array of length n which contains the - # diagonal elements of the upper triangular matrix s. + # sdiag is an output array of length n which contains the + # diagonal elements of the upper triangular matrix s. # - # wa is a work array of length n. + # wa is a work array of length n. # - # subprograms called + # subprograms called # - # fortran-supplied ... dabs,dsqrt + # fortran-supplied ... dabs,dsqrt # - # argonne national laboratory. minpack project. march 1980. - # burton s. garbow, kenneth e. hillstrom, jorge j. more + # argonne national laboratory. minpack project. march 1980. + # burton s. garbow, kenneth e. hillstrom, jorge j. more # + # fmt: on def qrsolv(self, r, ipvt, diag, qtb, sdiag): if self.debug: @@ -2065,99 +2073,101 @@ def qrsolv(self, r, ipvt, diag, qtb, sdiag): x[ipvt] = wa return (r, x, sdiag) - # Original FORTRAN documentation + # fmt: off + # Original FORTRAN documentation # - # subroutine lmpar + # subroutine lmpar # - # given an m by n matrix a, an n by n nonsingular diagonal - # matrix d, an m-vector b, and a positive number delta, - # the problem is to determine a value for the parameter - # par such that if x solves the system + # given an m by n matrix a, an n by n nonsingular diagonal + # matrix d, an m-vector b, and a positive number delta, + # the problem is to determine a value for the parameter + # par such that if x solves the system # - # a*x = b , sqrt(par)*d*x = 0 , + # a*x = b , sqrt(par)*d*x = 0 , # - # in the least squares sense, and dxnorm is the euclidean - # norm of d*x, then either par is zero and + # in the least squares sense, and dxnorm is the euclidean + # norm of d*x, then either par is zero and # - # (dxnorm-delta) .le. 0.1*delta , + # (dxnorm-delta) .le. 0.1*delta , # - # or par is positive and + # or par is positive and # - # abs(dxnorm-delta) .le. 0.1*delta . + # abs(dxnorm-delta) .le. 0.1*delta . # - # this subroutine completes the solution of the problem - # if it is provided with the necessary information from the - # qr factorization, with column pivoting, of a. that is, if - # a*p = q*r, where p is a permutation matrix, q has orthogonal - # columns, and r is an upper triangular matrix with diagonal - # elements of nonincreasing magnitude, then lmpar expects - # the full upper triangle of r, the permutation matrix p, - # and the first n components of (q transpose)*b. on output - # lmpar also provides an upper triangular matrix s such that + # this subroutine completes the solution of the problem + # if it is provided with the necessary information from the + # qr factorization, with column pivoting, of a. that is, if + # a*p = q*r, where p is a permutation matrix, q has orthogonal + # columns, and r is an upper triangular matrix with diagonal + # elements of nonincreasing magnitude, then lmpar expects + # the full upper triangle of r, the permutation matrix p, + # and the first n components of (q transpose)*b. on output + # lmpar also provides an upper triangular matrix s such that # - # t t t - # p *(a *a + par*d*d)*p = s *s . + # t t t + # p *(a *a + par*d*d)*p = s *s . # - # s is employed within lmpar and may be of separate interest. + # s is employed within lmpar and may be of separate interest. # - # only a few iterations are generally needed for convergence - # of the algorithm. if, however, the limit of 10 iterations - # is reached, then the output par will contain the best - # value obtained so far. + # only a few iterations are generally needed for convergence + # of the algorithm. if, however, the limit of 10 iterations + # is reached, then the output par will contain the best + # value obtained so far. # - # the subroutine statement is + # the subroutine statement is # - # subroutine lmpar(n,r,ldr,ipvt,diag,qtb,delta,par,x,sdiag, - # wa1,wa2) + # subroutine lmpar(n,r,ldr,ipvt,diag,qtb,delta,par,x,sdiag, + # wa1,wa2) # - # where + # where # - # n is a positive integer input variable set to the order of r. + # n is a positive integer input variable set to the order of r. # - # r is an n by n array. on input the full upper triangle - # must contain the full upper triangle of the matrix r. - # on output the full upper triangle is unaltered, and the - # strict lower triangle contains the strict upper triangle - # (transposed) of the upper triangular matrix s. + # r is an n by n array. on input the full upper triangle + # must contain the full upper triangle of the matrix r. + # on output the full upper triangle is unaltered, and the + # strict lower triangle contains the strict upper triangle + # (transposed) of the upper triangular matrix s. # - # ldr is a positive integer input variable not less than n - # which specifies the leading dimension of the array r. + # ldr is a positive integer input variable not less than n + # which specifies the leading dimension of the array r. # - # ipvt is an integer input array of length n which defines the - # permutation matrix p such that a*p = q*r. column j of p - # is column ipvt(j) of the identity matrix. + # ipvt is an integer input array of length n which defines the + # permutation matrix p such that a*p = q*r. column j of p + # is column ipvt(j) of the identity matrix. # - # diag is an input array of length n which must contain the - # diagonal elements of the matrix d. + # diag is an input array of length n which must contain the + # diagonal elements of the matrix d. # - # qtb is an input array of length n which must contain the first - # n elements of the vector (q transpose)*b. + # qtb is an input array of length n which must contain the first + # n elements of the vector (q transpose)*b. # - # delta is a positive input variable which specifies an upper - # bound on the euclidean norm of d*x. + # delta is a positive input variable which specifies an upper + # bound on the euclidean norm of d*x. # - # par is a nonnegative variable. on input par contains an - # initial estimate of the levenberg-marquardt parameter. - # on output par contains the final estimate. + # par is a nonnegative variable. on input par contains an + # initial estimate of the levenberg-marquardt parameter. + # on output par contains the final estimate. # - # x is an output array of length n which contains the least - # squares solution of the system a*x = b, sqrt(par)*d*x = 0, - # for the output par. + # x is an output array of length n which contains the least + # squares solution of the system a*x = b, sqrt(par)*d*x = 0, + # for the output par. # - # sdiag is an output array of length n which contains the - # diagonal elements of the upper triangular matrix s. + # sdiag is an output array of length n which contains the + # diagonal elements of the upper triangular matrix s. # - # wa1 and wa2 are work arrays of length n. + # wa1 and wa2 are work arrays of length n. # - # subprograms called + # subprograms called # - # minpack-supplied ... dpmpar,enorm,qrsolv + # minpack-supplied ... dpmpar,enorm,qrsolv # - # fortran-supplied ... dabs,dmax1,dmin1,dsqrt + # fortran-supplied ... dabs,dmax1,dmin1,dsqrt # - # argonne national laboratory. minpack project. march 1980. - # burton s. garbow, kenneth e. hillstrom, jorge j. more + # argonne national laboratory. minpack project. march 1980. + # burton s. garbow, kenneth e. hillstrom, jorge j. more # + # fmt: on def lmpar(self, r, ipvt, diag, qtb, delta, x, sdiag, par=None): @@ -2289,72 +2299,74 @@ def tie(self, p, ptied=None): exec(cmd) return p - # Original FORTRAN documentation - # ********** + # fmt: off + # Original FORTRAN documentation + # ********** # - # subroutine covar + # subroutine covar # - # given an m by n matrix a, the problem is to determine - # the covariance matrix corresponding to a, defined as + # given an m by n matrix a, the problem is to determine + # the covariance matrix corresponding to a, defined as # - # t - # inverse(a *a) . + # t + # inverse(a *a) . # - # this subroutine completes the solution of the problem - # if it is provided with the necessary information from the - # qr factorization, with column pivoting, of a. that is, if - # a*p = q*r, where p is a permutation matrix, q has orthogonal - # columns, and r is an upper triangular matrix with diagonal - # elements of nonincreasing magnitude, then covar expects - # the full upper triangle of r and the permutation matrix p. - # the covariance matrix is then computed as + # this subroutine completes the solution of the problem + # if it is provided with the necessary information from the + # qr factorization, with column pivoting, of a. that is, if + # a*p = q*r, where p is a permutation matrix, q has orthogonal + # columns, and r is an upper triangular matrix with diagonal + # elements of nonincreasing magnitude, then covar expects + # the full upper triangle of r and the permutation matrix p. + # the covariance matrix is then computed as # - # t t - # p*inverse(r *r)*p . + # t t + # p*inverse(r *r)*p . # - # if a is nearly rank deficient, it may be desirable to compute - # the covariance matrix corresponding to the linearly independent - # columns of a. to define the numerical rank of a, covar uses - # the tolerance tol. if l is the largest integer such that + # if a is nearly rank deficient, it may be desirable to compute + # the covariance matrix corresponding to the linearly independent + # columns of a. to define the numerical rank of a, covar uses + # the tolerance tol. if l is the largest integer such that # - # abs(r(l,l)) .gt. tol*abs(r(1,1)) , + # abs(r(l,l)) .gt. tol*abs(r(1,1)) , # - # then covar computes the covariance matrix corresponding to - # the first l columns of r. for k greater than l, column - # and row ipvt(k) of the covariance matrix are set to zero. + # then covar computes the covariance matrix corresponding to + # the first l columns of r. for k greater than l, column + # and row ipvt(k) of the covariance matrix are set to zero. # - # the subroutine statement is + # the subroutine statement is # - # subroutine covar(n,r,ldr,ipvt,tol,wa) + # subroutine covar(n,r,ldr,ipvt,tol,wa) # - # where + # where # - # n is a positive integer input variable set to the order of r. + # n is a positive integer input variable set to the order of r. # - # r is an n by n array. on input the full upper triangle must - # contain the full upper triangle of the matrix r. on output - # r contains the square symmetric covariance matrix. + # r is an n by n array. on input the full upper triangle must + # contain the full upper triangle of the matrix r. on output + # r contains the square symmetric covariance matrix. # - # ldr is a positive integer input variable not less than n - # which specifies the leading dimension of the array r. + # ldr is a positive integer input variable not less than n + # which specifies the leading dimension of the array r. # - # ipvt is an integer input array of length n which defines the - # permutation matrix p such that a*p = q*r. column j of p - # is column ipvt(j) of the identity matrix. + # ipvt is an integer input array of length n which defines the + # permutation matrix p such that a*p = q*r. column j of p + # is column ipvt(j) of the identity matrix. # - # tol is a nonnegative input variable used to define the - # numerical rank of a in the manner described above. + # tol is a nonnegative input variable used to define the + # numerical rank of a in the manner described above. # - # wa is a work array of length n. + # wa is a work array of length n. # - # subprograms called + # subprograms called # - # fortran-supplied ... dabs + # fortran-supplied ... dabs # - # argonne national laboratory. minpack project. august 1980. - # burton s. garbow, kenneth e. hillstrom, jorge j. more + # argonne national laboratory. minpack project. august 1980. + # burton s. garbow, kenneth e. hillstrom, jorge j. more # - # ********** + # ********** + # fmt: on def calc_covar(self, rr, ipvt=None, tol=1.0e-14): diff --git a/scripts/ultra/create_example_ultra_l2_map_with_power_law.py b/scripts/ultra/create_example_ultra_l2_map_with_power_law.py index 48040e639..07d8378a6 100644 --- a/scripts/ultra/create_example_ultra_l2_map_with_power_law.py +++ b/scripts/ultra/create_example_ultra_l2_map_with_power_law.py @@ -56,7 +56,7 @@ def _create_example_ultra_l2_map_with_power_law( cdf.new("epoch_delta", np.array([1])) cdf.new("energy_delta_plus", np.full_like(energies, 0.1)) cdf.new("energy_delta_minus", np.full_like(energies, 0.1)) - (cdf.new("energy_label", [str(val) for val in energies]),) + cdf.new("energy_label", [str(val) for val in energies]) cdf.new( "obs_date", np.full_like( diff --git a/tests/codice/l3/lo/test_codice_lo_processor.py b/tests/codice/l3/lo/test_codice_lo_processor.py index 11b0db787..477327b1c 100644 --- a/tests/codice/l3/lo/test_codice_lo_processor.py +++ b/tests/codice/l3/lo/test_codice_lo_processor.py @@ -701,18 +701,18 @@ def test_process_l3a_partial_densities(self, mock_calculate_partial_densities): result_data.heplusplus_partial_density, ), ) - (self.assertEqual(cplus4_partial_density, result_data.cplus4_partial_density),) - (self.assertEqual(cplus5_partial_density, result_data.cplus5_partial_density),) - (self.assertEqual(cplus6_partial_density, result_data.cplus6_partial_density),) - (self.assertEqual(oplus5_partial_density, result_data.oplus5_partial_density),) - (self.assertEqual(oplus6_partial_density, result_data.oplus6_partial_density),) - (self.assertEqual(oplus7_partial_density, result_data.oplus7_partial_density),) - (self.assertEqual(oplus8_partial_density, result_data.oplus8_partial_density),) - (self.assertEqual(sentinel.ne_partial_density, result_data.ne_partial_density),) - (self.assertEqual(mg_partial_density, result_data.mg_partial_density),) - (self.assertEqual(sentinel.si_partial_density, result_data.si_partial_density),) - (self.assertEqual(fe_loq_partial_density, result_data.fe_loq_partial_density),) - (self.assertEqual(fe_hiq_partial_density, result_data.fe_hiq_partial_density),) + self.assertEqual(cplus4_partial_density, result_data.cplus4_partial_density) + self.assertEqual(cplus5_partial_density, result_data.cplus5_partial_density) + self.assertEqual(cplus6_partial_density, result_data.cplus6_partial_density) + self.assertEqual(oplus5_partial_density, result_data.oplus5_partial_density) + self.assertEqual(oplus6_partial_density, result_data.oplus6_partial_density) + self.assertEqual(oplus7_partial_density, result_data.oplus7_partial_density) + self.assertEqual(oplus8_partial_density, result_data.oplus8_partial_density) + self.assertEqual(sentinel.ne_partial_density, result_data.ne_partial_density) + self.assertEqual(mg_partial_density, result_data.mg_partial_density) + self.assertEqual(sentinel.si_partial_density, result_data.si_partial_density) + self.assertEqual(fe_loq_partial_density, result_data.fe_loq_partial_density) + self.assertEqual(fe_hiq_partial_density, result_data.fe_hiq_partial_density) @patch(f"{MODULE}.SpinAngleLookup") @patch(f"{MODULE}.lookup_normalization_per_event", autospec=True) From 3d1551235345c0bb03c35955a134356a637a5e6e Mon Sep 17 00:00:00 2001 From: Jonathan Niehof Date: Fri, 23 Oct 2026 14:09:00 -0600 Subject: [PATCH 4/4] Fix tests that relied on unused imports --- tests/codice/l3/lo/test_codice_lo_processor.py | 10 +++++----- tests/glows/test_glows_processor.py | 2 +- tests/swapi/test_swapi_processor.py | 10 +++++----- tests/swe/test_swe_processor.py | 2 +- tests/test_processor.py | 2 +- tests/test_utils.py | 2 +- tests/ultra/test_ultra_processor.py | 6 +++--- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/codice/l3/lo/test_codice_lo_processor.py b/tests/codice/l3/lo/test_codice_lo_processor.py index 477327b1c..b35aee303 100644 --- a/tests/codice/l3/lo/test_codice_lo_processor.py +++ b/tests/codice/l3/lo/test_codice_lo_processor.py @@ -68,7 +68,7 @@ def test_implements_processor(self): @patch(f"{MODULE}.CodiceLoL3aPartialDensitiesDependencies.fetch_dependencies") @patch(f"{MODULE}.CodiceLoProcessor.process_l3a_partial_densities") @patch(f"{MODULE}.save_data") - @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.utils.spiceypy") def test_process_partial_densities( self, mock_spiceypy, @@ -114,7 +114,7 @@ def test_process_partial_densities( @patch(f"{MODULE}.CodiceLoL3aRatiosDependencies.fetch_dependencies") @patch(f"{MODULE}.CodiceLoProcessor.process_l3a_ratios") @patch(f"{MODULE}.save_data") - @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.utils.spiceypy") def test_process_ratios( self, mock_spiceypy, @@ -154,7 +154,7 @@ def test_process_ratios( @patch(f"{MODULE}.CodiceLoL3aRatiosDependencies.fetch_dependencies") @patch(f"{MODULE}.CodiceLoProcessor.process_l3a_charge_state_distributions") @patch(f"{MODULE}.save_data") - @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.utils.spiceypy") def test_process_abundances( self, mock_spiceypy, @@ -489,7 +489,7 @@ def test_process_abundances_uses_safe_divide(self): @patch(f"{MODULE}.CodiceLoL3aDirectEventsDependencies.fetch_dependencies") @patch(f"{MODULE}.CodiceLoProcessor.process_l3a_direct_event_data_product") @patch(f"{MODULE}.save_data") - @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.utils.spiceypy") def test_process_direct_events( self, mock_spiceypy, @@ -1891,7 +1891,7 @@ def test_process_3d_distributions_save_for_each_species(self): @patch(f"{MODULE}.CodiceLoL3b3dDistributionsDependencies.fetch_dependencies") @patch(f"{MODULE}.CodiceLoProcessor.process_l3b_3d_distribution_product") @patch(f"{MODULE}.save_data") - @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.utils.spiceypy") def _test_process_3d_distributions_save( self, species, diff --git a/tests/glows/test_glows_processor.py b/tests/glows/test_glows_processor.py index f1c442350..fc3de6c10 100644 --- a/tests/glows/test_glows_processor.py +++ b/tests/glows/test_glows_processor.py @@ -1164,7 +1164,7 @@ def test_process_l3d( get_test_data_path("glows/science"), ) @patch("imap_l3_processing.glows.glows_processor.run") - @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.utils.spiceypy") def test_process_l3d_adds_parent_file_names_to_output( self, mock_spicepy, mock_run, mock_rename_l3d_text_output, mock_save_data ): diff --git a/tests/swapi/test_swapi_processor.py b/tests/swapi/test_swapi_processor.py index b3cc8b754..ecda66634 100644 --- a/tests/swapi/test_swapi_processor.py +++ b/tests/swapi/test_swapi_processor.py @@ -51,7 +51,7 @@ class TestSwapiProcessor(TestCase): @patch("imap_l3_processing.swapi.swapi_processor.PuiChunkFitter") @patch("imap_l3_processing.swapi.swapi_processor.ParallelChunkRunner") @patch("imap_l3_processing.swapi.swapi_processor.SwapiL3ADependencies") - @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.utils.spiceypy") def test_process_l3a_pui( self, mock_spicepy, @@ -267,7 +267,7 @@ def test_process_l3a_pui( @patch("imap_l3_processing.swapi.swapi_processor.ParallelChunkRunner") @patch("imap_l3_processing.swapi.swapi_processor.chunk_l2_data") @patch("imap_l3_processing.swapi.swapi_processor.SwapiL3ADependencies") - @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.utils.spiceypy") def test_process_l3a_proton( self, mock_spicepy, @@ -448,7 +448,7 @@ def test_process_l3a_proton( @patch("imap_l3_processing.swapi.swapi_processor.ParallelChunkRunner") @patch("imap_l3_processing.swapi.swapi_processor.chunk_l2_data") @patch("imap_l3_processing.swapi.swapi_processor.SwapiL3ADependencies") - @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.utils.spiceypy") def test_process_l3a_alpha( self, mock_spicepy, @@ -619,7 +619,7 @@ def test_process_l3a_alpha( @patch( "imap_l3_processing.swapi.swapi_processor.calculate_combined_solar_wind_differential_flux" ) - @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.utils.spiceypy") def test_process_l3b( self, mock_spiceypy, @@ -1047,7 +1047,7 @@ def test_process_l3b( self.assertEqual([mock_save_data.return_value], product) @patch("imap_l3_processing.swapi.swapi_processor.SwapiL3ADependencies") - @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.utils.spiceypy") def test_process_l3a_raises_for_unknown_descriptor( self, mock_spicepy, mock_swapi_l3_dependencies_class ): diff --git a/tests/swe/test_swe_processor.py b/tests/swe/test_swe_processor.py index ba0965dc4..117233967 100644 --- a/tests/swe/test_swe_processor.py +++ b/tests/swe/test_swe_processor.py @@ -43,7 +43,7 @@ class TestSweProcessor(unittest.TestCase): - @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.utils.spiceypy") @patch("imap_l3_processing.swe.swe_processor.save_data") @patch("imap_l3_processing.swe.swe_processor.SweL3Dependencies.fetch_dependencies") @patch("imap_l3_processing.swe.swe_processor.SweProcessor.calculate_products") diff --git a/tests/test_processor.py b/tests/test_processor.py index 5d32ebefd..7b7b1e860 100644 --- a/tests/test_processor.py +++ b/tests/test_processor.py @@ -35,7 +35,7 @@ def test_get_parent_file_names(self, mock_get_spice_parent_file_names): self.assertEqual(expected_parent_file_names, actual_parent_file_names) - @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.utils.spiceypy") def test_get_parent_files_takes_optional_parameter(self, mock_spiceypy): file_paths = [Path("path")] mock_spiceypy.ktotal.return_value = 0 diff --git a/tests/test_utils.py b/tests/test_utils.py index 2ed7abc75..8dc3ff146 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -801,7 +801,7 @@ def test_get_dependency_paths_by_descriptor(self): + f", expected: {sorted(expected_output[output_key])} actual {sorted(actual_output[output_key])}", ) - @patch("imap_l3_processing.glows.l3bc.utils.imap_data_access.download") + @patch("imap_l3_processing.utils.imap_data_access.download") @with_tempdir def test_read_cdf_parents(self, temp_dir, mock_download): cdf_downloaded_path = temp_dir / "l3b.cdf" diff --git a/tests/ultra/test_ultra_processor.py b/tests/ultra/test_ultra_processor.py index f997d9f2e..0a8804735 100644 --- a/tests/ultra/test_ultra_processor.py +++ b/tests/ultra/test_ultra_processor.py @@ -982,7 +982,7 @@ def _test_process_combined_sensor_survival_probability( @patch( "imap_l3_processing.ultra.ultra_processor.correct_healpix_data_for_survival_probability" ) - @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.utils.spiceypy") @patch("imap_l3_processing.ultra.ultra_processor.save_data") @patch( "imap_l3_processing.ultra.ultra_processor.UltraL3Dependencies.fetch_dependencies" @@ -1073,7 +1073,7 @@ def test_defaults_to_ECLIPJ2000_spice_frame( mock_fetch_dependencies.return_value, SpiceFrame.ECLIPJ2000 ) - @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.utils.spiceypy") @patch("imap_l3_processing.ultra.ultra_processor.save_data") @patch( "imap_l3_processing.ultra.ultra_processor.calculate_spectral_index_for_multiple_ranges" @@ -1159,7 +1159,7 @@ def _test_process_spectral_index( self.assertEqual([mock_save_data.return_value], product) @patch("imap_l3_processing.ultra.ultra_processor.save_data") - @patch("imap_l3_processing.processor.spiceypy") + @patch("imap_l3_processing.utils.spiceypy") @patch( "imap_l3_processing.ultra.ultra_processor.calculate_spectral_index_for_multiple_ranges" )