From e1a507d29f58a7497bde6ff17d7e256f2e245bb0 Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Tue, 10 Nov 2026 16:13:20 -0600 Subject: [PATCH 1/2] FIX: preserve Bringi phase masks --- cmac/cmac_processing.py | 18 +++++++++++++----- cmac/cmac_radar.py | 13 ++++++++----- cmac/tests/test_phase_masks.py | 22 +++++++++++++++++++++- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/cmac/cmac_processing.py b/cmac/cmac_processing.py index eff270f..13d2f1c 100644 --- a/cmac/cmac_processing.py +++ b/cmac/cmac_processing.py @@ -405,11 +405,19 @@ def remove_sys_phase(phidp, sys_phase, wrap_min=-90.0): def fix_phase_fields(orig_kdp, orig_phidp, rrange, happy_kdp, max_kdp=15.0): - - orig_kdp['data'][happy_kdp.gate_excluded] = 0.0 - orig_kdp['data'][orig_kdp['data'] > max_kdp] = max_kdp - interg = integrate.cumulative_trapezoid(orig_kdp['data'], rrange, axis=1) - orig_phidp['data'][:, 0:-1] = interg/len(rrange) + kdp_data = np.ma.asanyarray(orig_kdp['data']) + phidp_data = np.ma.asanyarray(orig_phidp['data']) + mask = (np.ma.getmaskarray(kdp_data) | + np.ma.getmaskarray(phidp_data) | + happy_kdp.gate_excluded) + kdp_values = np.ma.filled(kdp_data, 0.0).copy() + kdp_values[happy_kdp.gate_excluded] = 0.0 + kdp_values[kdp_values > max_kdp] = max_kdp + interg = integrate.cumulative_trapezoid(kdp_values, rrange, axis=1) + phidp_values = np.ma.filled(phidp_data, 0.0).copy() + phidp_values[:, 0:-1] = interg/len(rrange) + orig_kdp['data'] = np.ma.array(kdp_values, mask=mask) + orig_phidp['data'] = np.ma.array(phidp_values, mask=mask) return orig_phidp, orig_kdp diff --git a/cmac/cmac_radar.py b/cmac/cmac_radar.py index f96ac47..09fb7f0 100644 --- a/cmac/cmac_radar.py +++ b/cmac/cmac_radar.py @@ -459,15 +459,18 @@ def cmac(radar, sonde, config, geotiff=None, flip_velocity=False, if verbose: print('## PhiDP system phase offset removed: ' '%.2f degrees' % sys_phase) - dp = dp.filled(fill_value) - dz = dz.filled(fill_value) + dp_mask = np.ma.getmaskarray(dp) + dz_mask = np.ma.getmaskarray(dz) + dp = np.ma.filled(dp, fill_value) + dz = np.ma.filled(dz, fill_value) rng = np.tile(radar.range['data'], (radar.nrays, 1)) / 1e3 kdp, phidp, _ = csu_kdp.calc_kdp_bringi(dp=dp, dz=dz, rng=rng, bad=fill_value) - kdp_data = np.ma.masked_where( - kdp_gates.gate_excluded, kdp).filled(-9999.) - phidp_data = np.ma.masked_where(kdp_gates.gate_excluded, phidp).filled(-9999.) + invalid = (kdp_gates.gate_excluded | dp_mask | dz_mask | + (kdp == fill_value) | (phidp == fill_value)) + kdp_data = np.ma.masked_where(invalid, kdp) + phidp_data = np.ma.masked_where(invalid, phidp) phidp = copy.deepcopy(radar.fields[field_config['input_phidp_field']]) kdp = pyart.config.get_metadata("corrected_specific_differential_phase") phidp["data"] = phidp_data diff --git a/cmac/tests/test_phase_masks.py b/cmac/tests/test_phase_masks.py index 4438cac..bbe4607 100644 --- a/cmac/tests/test_phase_masks.py +++ b/cmac/tests/test_phase_masks.py @@ -1,8 +1,10 @@ """Tests for the masks carried by paired phase fields.""" +from types import SimpleNamespace + import numpy as np -from cmac.cmac_processing import synchronize_phase_field_masks +from cmac.cmac_processing import fix_phase_fields, synchronize_phase_field_masks def test_synchronize_phase_field_masks_uses_union_of_masks(): @@ -15,3 +17,21 @@ def test_synchronize_phase_field_masks_uses_union_of_masks(): np.testing.assert_array_equal(np.ma.getmaskarray(phidp['data']), expected) np.testing.assert_array_equal(np.ma.getmaskarray(kdp['data']), expected) + +def test_fix_phase_fields_does_not_integrate_fill_values(): + kdp = {'data': np.ma.array([[1.0, -9999.0, 2.0]], mask=[[0, 1, 0]])} + phidp = {'data': np.ma.array([[0.0, 0.0, 0.0]], mask=[[0, 0, 0]])} + gates = SimpleNamespace(gate_excluded=np.array([[False, False, True]])) + + filtered_phase, filtered_kdp = fix_phase_fields( + kdp, phidp, np.array([0.0, 1.0, 2.0]), gates) + + assert np.ma.getdata(filtered_phase['data']).max() < 10.0 + assert np.ma.getdata(filtered_kdp['data']).max() < 10.0 + np.testing.assert_array_equal( + np.ma.getmaskarray(filtered_phase['data']), + [[False, True, True]]) + np.testing.assert_array_equal( + np.ma.getmaskarray(filtered_kdp['data']), + [[False, True, True]]) + From 81e0ad749f78a82caa990bae2d073d038300a506 Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Tue, 10 Nov 2026 16:28:47 -0600 Subject: [PATCH 2/2] FIX: derive Bringi fill value from field metadata --- cmac/cmac_radar.py | 23 ++++++++++++++++++++--- cmac/tests/test_phase_masks.py | 9 +++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/cmac/cmac_radar.py b/cmac/cmac_radar.py index 09fb7f0..fff5bc4 100644 --- a/cmac/cmac_radar.py +++ b/cmac/cmac_radar.py @@ -23,6 +23,21 @@ from . import csu_kdp +def _get_field_fill_value(field): + """Return a scalar missing-data value from a Py-ART field dictionary.""" + for key in ('missing_value', '_FillValue'): + value = field.get(key) + if value is not None: + value = np.asarray(value).reshape(-1) + if value.size == 1 and np.isfinite(value[0]): + return float(value[0]) + data = field.get('data') + if np.ma.isMaskedArray(data) and np.isfinite(data.fill_value): + return float(data.fill_value) + raise ValueError( + "Bringi requires a missing-data value, but the field defines none") + + def add_clutter_map(radar, clutter_radar, field_name='ground_clutter'): """Add a clutter map, aligning PPI sweeps by elevation. @@ -434,9 +449,11 @@ def cmac(radar, sonde, config, geotiff=None, flip_velocity=False, # Bringi's ray-wise retrieval is valid throughout the volume. Unlike # the LP retrieval, do not discard gates above the freezing level. kdp_gates = copy.deepcopy(cmac_gates) - fill_value = -9999. - dp = radar.fields[field_config['input_phidp_field']]['data'] - dz = radar.fields[field_config['reflectivity']]['data'] + dp_field = radar.fields[field_config['input_phidp_field']] + dz_field = radar.fields[field_config['reflectivity']] + fill_value = _get_field_fill_value(dp_field) + dp = dp_field['data'] + dz = dz_field['data'] # calc_kdp_bringi assumes PhiDP has the system phase offset removed # and is already unfolded. Feeding it the raw phase carries the # offset (~220 degrees at BNF) through to the Z-PHI attenuation, diff --git a/cmac/tests/test_phase_masks.py b/cmac/tests/test_phase_masks.py index bbe4607..2af1b0d 100644 --- a/cmac/tests/test_phase_masks.py +++ b/cmac/tests/test_phase_masks.py @@ -5,6 +5,7 @@ import numpy as np from cmac.cmac_processing import fix_phase_fields, synchronize_phase_field_masks +from cmac.cmac_radar import _get_field_fill_value def test_synchronize_phase_field_masks_uses_union_of_masks(): @@ -35,3 +36,11 @@ def test_fix_phase_fields_does_not_integrate_fill_values(): np.ma.getmaskarray(filtered_kdp['data']), [[False, True, True]]) + +def test_get_field_fill_value_prefers_missing_value(): + assert _get_field_fill_value({ + 'missing_value': -9999, + '_FillValue': -32767, + }) == -9999.0 + assert _get_field_fill_value({'_FillValue': -32767}) == -32767.0 +