From 95b6cecedfeec9d7c72117b039596ee6fca946d3 Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Tue, 10 Nov 2026 13:03:52 -0600 Subject: [PATCH 1/3] FIX: synchronize corrected phase field masks --- cmac/cmac_processing.py | 22 ++++++++++++++++++++++ cmac/cmac_radar.py | 8 ++++++-- cmac/tests/test_phase_masks.py | 17 +++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 cmac/tests/test_phase_masks.py diff --git a/cmac/cmac_processing.py b/cmac/cmac_processing.py index 5456961..eff270f 100644 --- a/cmac/cmac_processing.py +++ b/cmac/cmac_processing.py @@ -412,6 +412,28 @@ def fix_phase_fields(orig_kdp, orig_phidp, rrange, happy_kdp, orig_phidp['data'][:, 0:-1] = interg/len(rrange) return orig_phidp, orig_kdp + +def synchronize_phase_field_masks(phidp, kdp): + """Give a phase field and its KDP field the same gate mask. + + Differential phase and specific differential phase are generated by + separate Py-ART/CMAC paths. Those paths can preserve different masks, + even though the fields describe the same gates and are consumed as a + pair by downstream products. Preserve every masked gate from either + input so neither field exposes a value where its companion is masked. + + The field dictionaries are modified in place and returned for convenient + use by callers. + """ + phidp_data = np.ma.asanyarray(phidp['data']) + kdp_data = np.ma.asanyarray(kdp['data']) + mask = np.logical_or( + np.ma.getmaskarray(phidp_data), np.ma.getmaskarray(kdp_data)) + phidp['data'] = np.ma.array(phidp_data, mask=mask, copy=False) + kdp['data'] = np.ma.array(kdp_data, mask=mask, copy=False) + return phidp, kdp + + def return_csu_kdp(radar): dzN = _extract_unmasked_data(radar, 'reflectivity') dpN = _extract_unmasked_data(radar, 'differential_phase') diff --git a/cmac/cmac_radar.py b/cmac/cmac_radar.py index a1ef3bc..00ecde2 100644 --- a/cmac/cmac_radar.py +++ b/cmac/cmac_radar.py @@ -13,7 +13,8 @@ import netCDF4 from .cmac_processing import ( - do_my_fuzz, get_melt, get_texture, fix_phase_fields, gen_clutter_field_from_refl, beam_block, + do_my_fuzz, get_melt, get_texture, fix_phase_fields, + synchronize_phase_field_masks, gen_clutter_field_from_refl, beam_block, snow_rate, rain_rate, get_sys_phase, remove_sys_phase) from .config import (get_cmac_values, get_field_names, get_metadata, get_zs_relationships, get_default_metadata) @@ -417,6 +418,7 @@ def cmac(radar, sonde, config, geotiff=None, flip_velocity=False, kdp_gates = copy.deepcopy(cmac_gates) kdp_gates.exclude_above('height', fzl) + phidp, kdp = synchronize_phase_field_masks(phidp, kdp) radar.add_field('corrected_differential_phase', phidp, replace_existing=True) radar.add_field('corrected_specific_diff_phase', kdp, @@ -427,8 +429,10 @@ def cmac(radar, sonde, config, geotiff=None, flip_velocity=False, phidp_filt, kdp_filt = fix_phase_fields( copy.deepcopy(kdp), copy.deepcopy(phidp), radar.range['data'], kdp_gates, max_kdp=cmac_config.get('max_kdp', 15.0)) + phidp_filt, kdp_filt = synchronize_phase_field_masks( + phidp_filt, kdp_filt) - radar.add_field('filtered_corrected_differential_phase', phidp, + radar.add_field('filtered_corrected_differential_phase', phidp_filt, replace_existing=True) radar.fields[ 'filtered_corrected_differential_phase']['long_name'] = 'Filtered corrected differential propagation phase shift' diff --git a/cmac/tests/test_phase_masks.py b/cmac/tests/test_phase_masks.py new file mode 100644 index 0000000..4438cac --- /dev/null +++ b/cmac/tests/test_phase_masks.py @@ -0,0 +1,17 @@ +"""Tests for the masks carried by paired phase fields.""" + +import numpy as np + +from cmac.cmac_processing import synchronize_phase_field_masks + + +def test_synchronize_phase_field_masks_uses_union_of_masks(): + phidp = {'data': np.ma.array([[1.0, 2.0, 3.0]], mask=[[0, 1, 0]])} + kdp = {'data': np.ma.array([[4.0, 5.0, 6.0]], mask=[[0, 0, 1]])} + + synchronize_phase_field_masks(phidp, kdp) + + expected = np.array([[False, True, True]]) + np.testing.assert_array_equal(np.ma.getmaskarray(phidp['data']), expected) + np.testing.assert_array_equal(np.ma.getmaskarray(kdp['data']), expected) + From 4ed7255a7e5086b08213a454f76580d057f1a777 Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Tue, 10 Nov 2026 13:21:50 -0600 Subject: [PATCH 2/3] FIX: align PPI clutter maps by elevation --- cmac/__init__.py | 2 +- cmac/cmac_radar.py | 60 ++++++++++++++++++++++++++++++++++ cmac/tests/test_clutter_map.py | 40 +++++++++++++++++++++++ scripts/cmac | 21 ++---------- 4 files changed, 103 insertions(+), 20 deletions(-) create mode 100644 cmac/tests/test_clutter_map.py diff --git a/cmac/__init__.py b/cmac/__init__.py index 335e91e..512e57f 100644 --- a/cmac/__init__.py +++ b/cmac/__init__.py @@ -18,7 +18,7 @@ """ -from .cmac_radar import cmac, area_coverage +from .cmac_radar import add_clutter_map, cmac, area_coverage from .cmac_ppi_quicklooks import quicklooks_ppi from .cmac_rhi_quicklooks import quicklooks_rhi from .cmac_processing import snr_and_sounding, do_my_fuzz diff --git a/cmac/cmac_radar.py b/cmac/cmac_radar.py index 00ecde2..2a9b1c4 100644 --- a/cmac/cmac_radar.py +++ b/cmac/cmac_radar.py @@ -22,6 +22,66 @@ from .gate_id_backends import radar_palette_gate_id from . import csu_kdp + +def add_clutter_map(radar, clutter_radar, field_name='ground_clutter'): + """Add a clutter map, aligning PPI sweeps by elevation. + + Clutter maps and data volumes may have different sweep and ray counts. + For PPI data, fixed angle is the sweep elevation, so it is used to match + sweeps. Within a matched sweep, each radar ray receives the nearest + clutter-map azimuth. Unmatched radar sweeps remain uncluttered. + """ + if field_name not in clutter_radar.fields: + raise KeyError("clutter map has no %r field" % field_name) + + target_shape = (radar.nrays, radar.ngates) + clutter_data = np.ma.asanyarray(clutter_radar.fields[field_name]['data']) + if radar.scan_type != 'ppi': + if clutter_data.shape != target_shape: + raise ValueError( + "clutter field shape %s does not match radar shape %s" + % (clutter_data.shape, target_shape)) + field = copy.deepcopy(clutter_radar.fields[field_name]) + radar.add_field(field_name, field, replace_existing=True) + return + + matched = np.zeros(target_shape, dtype=clutter_data.dtype) + radar_angles = np.asarray(radar.fixed_angle['data'], dtype=float) + clutter_angles = np.asarray( + clutter_radar.fixed_angle['data'], dtype=float) + clutter_azimuth = np.asarray(clutter_radar.azimuth['data'], dtype=float) + clutter_values = np.ma.filled(clutter_data, 0) + + for radar_sweep, radar_angle in enumerate(radar_angles): + clutter_sweep = int(np.abs(clutter_angles - radar_angle).argmin()) + angle_error = abs(clutter_angles[clutter_sweep] - radar_angle) + if angle_error > 0.05: + warnings.warn( + 'No clutter-map sweep matches radar elevation %.4f; ' + 'leaving that sweep uncluttered.' % radar_angle, + UserWarning) + continue + + radar_start, radar_end = radar.get_start_end(radar_sweep) + clutter_start, clutter_end = clutter_radar.get_start_end( + clutter_sweep) + radar_indices = np.arange(int(radar_start), int(radar_end) + 1) + clutter_indices = np.arange( + int(clutter_start), int(clutter_end) + 1) + radar_azimuth = np.asarray(radar.azimuth['data'])[radar_indices] + map_azimuth = clutter_azimuth[clutter_indices] + azimuth_distance = np.abs( + (radar_azimuth[:, None] - map_azimuth[None, :] + 180.0) + % 360.0 - 180.0) + nearest = azimuth_distance.argmin(axis=1) + matched[radar_indices] = clutter_values[ + clutter_indices[nearest]] + + field = copy.deepcopy(clutter_radar.fields[field_name]) + field['data'] = matched + radar.add_field(field_name, field, replace_existing=True) + + def cmac(radar, sonde, config, geotiff=None, flip_velocity=False, meta_append=None, verbose=True, snow_density=None, snowfall=True, config_file=None): diff --git a/cmac/tests/test_clutter_map.py b/cmac/tests/test_clutter_map.py new file mode 100644 index 0000000..80ba086 --- /dev/null +++ b/cmac/tests/test_clutter_map.py @@ -0,0 +1,40 @@ +"""Tests for aligning PPI clutter maps with radar sweeps.""" + +import numpy as np + +from cmac import add_clutter_map + + +class _Radar: + def __init__(self, fixed_angle, azimuth, clutter=None): + self.scan_type = 'ppi' + self.nrays = len(azimuth) + self.ngates = 2 + self.nsweeps = len(fixed_angle) + self.fixed_angle = {'data': np.asarray(fixed_angle)} + self.azimuth = {'data': np.asarray(azimuth)} + self.sweep_start_ray_index = { + 'data': np.arange(self.nsweeps, dtype=int) * 2} + self.sweep_end_ray_index = { + 'data': np.arange(1, self.nrays, 2, dtype=int)} + self.fields = {} if clutter is None else { + 'ground_clutter': {'data': np.asarray(clutter)}} + + def get_start_end(self, sweep): + return (self.sweep_start_ray_index['data'][sweep], + self.sweep_end_ray_index['data'][sweep]) + + def add_field(self, name, field, replace_existing=False): + self.fields[name] = field + + +def test_ppi_clutter_map_matches_sweeps_by_elevation_and_azimuth(): + radar = _Radar([1.0, 2.0], [0.0, 180.0, 0.0, 180.0]) + clutter = _Radar([1.0], [1.0, 181.0], [[1, 0], [0, 1]]) + + add_clutter_map(radar, clutter) + + np.testing.assert_array_equal( + radar.fields['ground_clutter']['data'], + [[1, 0], [0, 1], [0, 0], [0, 0]]) + diff --git a/scripts/cmac b/scripts/cmac index 44cb0cb..e927556 100755 --- a/scripts/cmac +++ b/scripts/cmac @@ -5,11 +5,11 @@ import argparse from pathlib import Path import netCDF4 -import numpy as np import pyart import xarray as xr from cmac import ( + add_clutter_map, area_coverage, cmac, get_cmac_values, @@ -70,24 +70,7 @@ def main(): print('## Reading dictionary...') print('## Adding clutter field..') clutter_radar = pyart.io.read(str(args.clutter_file)) - clutter_field_dict = clutter_radar.fields['ground_clutter'].copy() - # For XSAPR data at NSA, sometimes only 3 of 6 sweeps are present; - # copy clutter information from the bottom sweeps only. - if clutter_radar.nsweeps > radar.nsweeps: - fname = next(iter(radar.fields)) - clutter_field_dict['data'] = np.zeros_like( - radar.fields[fname]['data']) - for i, azi_angle in enumerate(radar.fixed_angle['data']): - ind = np.argwhere( - clutter_radar.fixed_angle['data'] == azi_angle) - if len(ind) > 0: - clutter = clutter_radar.get_field( - int(ind), 'ground_clutter') - start, end = radar.get_start_end(i) - clutter_field_dict['data'][ - int(start):int(end + 1)] = clutter - radar.add_field( - 'ground_clutter', clutter_field_dict, replace_existing=True) + add_clutter_map(radar, clutter_radar) del clutter_radar cmac_radar = cmac( From 3a16ab762c21707780375750eb947a4089bc65a9 Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Tue, 10 Nov 2026 13:29:49 -0600 Subject: [PATCH 3/3] FIX: keep above-freezing gates for Bringi KDP --- cmac/cmac_radar.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cmac/cmac_radar.py b/cmac/cmac_radar.py index 2a9b1c4..f96ac47 100644 --- a/cmac/cmac_radar.py +++ b/cmac/cmac_radar.py @@ -422,15 +422,18 @@ def cmac(radar, sonde, config, geotiff=None, flip_velocity=False, radar.fields["unfolded_differential_phase"] = copy.deepcopy( radar.fields[field_config['input_phidp_field']]) - kdp_gates = copy.deepcopy(cmac_gates) - kdp_gates.exclude_above('height', fzl) if cmac_config['kdp_method'] == 'lp': + kdp_gates = copy.deepcopy(cmac_gates) + kdp_gates.exclude_above('height', fzl) phidp, kdp = pyart.correct.phase_proc_lp_gf( radar, gatefilter=kdp_gates, offset=ref_offset, debug=True, LP_solver='cylp', nowrap=phidp_nowrap, fzl=fzl, self_const=self_const, phidp_field=field_config['input_phidp_field'], refl_field=field_config['reflectivity']) elif cmac_config['kdp_method'] == 'bringi': + # Bringi's ray-wise retrieval is valid throughout the volume. Unlike + # the LP retrieval, do not discard gates above the freezing level. + kdp_gates = copy.deepcopy(cmac_gates) fill_value = -9999. dp = radar.fields[field_config['input_phidp_field']]['data'] dz = radar.fields[field_config['reflectivity']]['data'] @@ -474,9 +477,10 @@ def cmac(radar, sonde, config, geotiff=None, flip_velocity=False, print("Processed phase") - # We do not use KDP, phase above freezing level + # LP excludes phase above the freezing level; Bringi does not. kdp_gates = copy.deepcopy(cmac_gates) - kdp_gates.exclude_above('height', fzl) + if cmac_config['kdp_method'] == 'lp': + kdp_gates.exclude_above('height', fzl) phidp, kdp = synchronize_phase_field_masks(phidp, kdp) radar.add_field('corrected_differential_phase', phidp,