Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cmac/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

"""

from .cmac_radar import cmac, area_coverage
from .cmac_radar import add_clutter_map, cmac, area_coverage
from .cmac_ppi_quicklooks import quicklooks_ppi
from .cmac_rhi_quicklooks import quicklooks_rhi
from .cmac_processing import snr_and_sounding, do_my_fuzz
Expand Down
22 changes: 22 additions & 0 deletions cmac/cmac_processing.py
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,28 @@ def fix_phase_fields(orig_kdp, orig_phidp, rrange, happy_kdp,
orig_phidp['data'][:, 0:-1] = interg/len(rrange)
return orig_phidp, orig_kdp


def synchronize_phase_field_masks(phidp, kdp):
"""Give a phase field and its KDP field the same gate mask.

Differential phase and specific differential phase are generated by
separate Py-ART/CMAC paths. Those paths can preserve different masks,
even though the fields describe the same gates and are consumed as a
pair by downstream products. Preserve every masked gate from either
input so neither field exposes a value where its companion is masked.

The field dictionaries are modified in place and returned for convenient
use by callers.
"""
phidp_data = np.ma.asanyarray(phidp['data'])
kdp_data = np.ma.asanyarray(kdp['data'])
mask = np.logical_or(
np.ma.getmaskarray(phidp_data), np.ma.getmaskarray(kdp_data))
phidp['data'] = np.ma.array(phidp_data, mask=mask, copy=False)
kdp['data'] = np.ma.array(kdp_data, mask=mask, copy=False)
return phidp, kdp


def return_csu_kdp(radar):
dzN = _extract_unmasked_data(radar, 'reflectivity')
dpN = _extract_unmasked_data(radar, 'differential_phase')
Expand Down
80 changes: 74 additions & 6 deletions cmac/cmac_radar.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,75 @@
import netCDF4

from .cmac_processing import (
do_my_fuzz, get_melt, get_texture, fix_phase_fields, gen_clutter_field_from_refl, beam_block,
do_my_fuzz, get_melt, get_texture, fix_phase_fields,
synchronize_phase_field_masks, gen_clutter_field_from_refl, beam_block,
snow_rate, rain_rate, get_sys_phase, remove_sys_phase)
from .config import (get_cmac_values, get_field_names, get_metadata,
get_zs_relationships, get_default_metadata)
from .gate_id import append_gate_id_category, get_gate_id_categories
from .gate_id_backends import radar_palette_gate_id
from . import csu_kdp


def add_clutter_map(radar, clutter_radar, field_name='ground_clutter'):
"""Add a clutter map, aligning PPI sweeps by elevation.

Clutter maps and data volumes may have different sweep and ray counts.
For PPI data, fixed angle is the sweep elevation, so it is used to match
sweeps. Within a matched sweep, each radar ray receives the nearest
clutter-map azimuth. Unmatched radar sweeps remain uncluttered.
"""
if field_name not in clutter_radar.fields:
raise KeyError("clutter map has no %r field" % field_name)

target_shape = (radar.nrays, radar.ngates)
clutter_data = np.ma.asanyarray(clutter_radar.fields[field_name]['data'])
if radar.scan_type != 'ppi':
if clutter_data.shape != target_shape:
raise ValueError(
"clutter field shape %s does not match radar shape %s"
% (clutter_data.shape, target_shape))
field = copy.deepcopy(clutter_radar.fields[field_name])
radar.add_field(field_name, field, replace_existing=True)
return

matched = np.zeros(target_shape, dtype=clutter_data.dtype)
radar_angles = np.asarray(radar.fixed_angle['data'], dtype=float)
clutter_angles = np.asarray(
clutter_radar.fixed_angle['data'], dtype=float)
clutter_azimuth = np.asarray(clutter_radar.azimuth['data'], dtype=float)
clutter_values = np.ma.filled(clutter_data, 0)

for radar_sweep, radar_angle in enumerate(radar_angles):
clutter_sweep = int(np.abs(clutter_angles - radar_angle).argmin())
angle_error = abs(clutter_angles[clutter_sweep] - radar_angle)
if angle_error > 0.05:
warnings.warn(
'No clutter-map sweep matches radar elevation %.4f; '
'leaving that sweep uncluttered.' % radar_angle,
UserWarning)
continue

radar_start, radar_end = radar.get_start_end(radar_sweep)
clutter_start, clutter_end = clutter_radar.get_start_end(
clutter_sweep)
radar_indices = np.arange(int(radar_start), int(radar_end) + 1)
clutter_indices = np.arange(
int(clutter_start), int(clutter_end) + 1)
radar_azimuth = np.asarray(radar.azimuth['data'])[radar_indices]
map_azimuth = clutter_azimuth[clutter_indices]
azimuth_distance = np.abs(
(radar_azimuth[:, None] - map_azimuth[None, :] + 180.0)
% 360.0 - 180.0)
nearest = azimuth_distance.argmin(axis=1)
matched[radar_indices] = clutter_values[
clutter_indices[nearest]]

field = copy.deepcopy(clutter_radar.fields[field_name])
field['data'] = matched
radar.add_field(field_name, field, replace_existing=True)


def cmac(radar, sonde, config, geotiff=None, flip_velocity=False,
meta_append=None, verbose=True, snow_density=None, snowfall=True,
config_file=None):
Expand Down Expand Up @@ -361,15 +422,18 @@ def cmac(radar, sonde, config, geotiff=None, flip_velocity=False,
radar.fields["unfolded_differential_phase"] = copy.deepcopy(
radar.fields[field_config['input_phidp_field']])

kdp_gates = copy.deepcopy(cmac_gates)
kdp_gates.exclude_above('height', fzl)
if cmac_config['kdp_method'] == 'lp':
kdp_gates = copy.deepcopy(cmac_gates)
kdp_gates.exclude_above('height', fzl)
phidp, kdp = pyart.correct.phase_proc_lp_gf(
radar, gatefilter=kdp_gates, offset=ref_offset, debug=True,
LP_solver='cylp', nowrap=phidp_nowrap, fzl=fzl, self_const=self_const,
phidp_field=field_config['input_phidp_field'],
refl_field=field_config['reflectivity'])
elif cmac_config['kdp_method'] == 'bringi':
# Bringi's ray-wise retrieval is valid throughout the volume. Unlike
# the LP retrieval, do not discard gates above the freezing level.
kdp_gates = copy.deepcopy(cmac_gates)
fill_value = -9999.
dp = radar.fields[field_config['input_phidp_field']]['data']
dz = radar.fields[field_config['reflectivity']]['data']
Expand Down Expand Up @@ -413,10 +477,12 @@ def cmac(radar, sonde, config, geotiff=None, flip_velocity=False,


print("Processed phase")
# We do not use KDP, phase above freezing level
# LP excludes phase above the freezing level; Bringi does not.
kdp_gates = copy.deepcopy(cmac_gates)
kdp_gates.exclude_above('height', fzl)
if cmac_config['kdp_method'] == 'lp':
kdp_gates.exclude_above('height', fzl)

phidp, kdp = synchronize_phase_field_masks(phidp, kdp)
radar.add_field('corrected_differential_phase', phidp,
replace_existing=True)
radar.add_field('corrected_specific_diff_phase', kdp,
Expand All @@ -427,8 +493,10 @@ def cmac(radar, sonde, config, geotiff=None, flip_velocity=False,
phidp_filt, kdp_filt = fix_phase_fields(
copy.deepcopy(kdp), copy.deepcopy(phidp), radar.range['data'],
kdp_gates, max_kdp=cmac_config.get('max_kdp', 15.0))
phidp_filt, kdp_filt = synchronize_phase_field_masks(
phidp_filt, kdp_filt)

radar.add_field('filtered_corrected_differential_phase', phidp,
radar.add_field('filtered_corrected_differential_phase', phidp_filt,
replace_existing=True)
radar.fields[
'filtered_corrected_differential_phase']['long_name'] = 'Filtered corrected differential propagation phase shift'
Expand Down
40 changes: 40 additions & 0 deletions cmac/tests/test_clutter_map.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
"""Tests for aligning PPI clutter maps with radar sweeps."""

import numpy as np

from cmac import add_clutter_map


class _Radar:
def __init__(self, fixed_angle, azimuth, clutter=None):
self.scan_type = 'ppi'
self.nrays = len(azimuth)
self.ngates = 2
self.nsweeps = len(fixed_angle)
self.fixed_angle = {'data': np.asarray(fixed_angle)}
self.azimuth = {'data': np.asarray(azimuth)}
self.sweep_start_ray_index = {
'data': np.arange(self.nsweeps, dtype=int) * 2}
self.sweep_end_ray_index = {
'data': np.arange(1, self.nrays, 2, dtype=int)}
self.fields = {} if clutter is None else {
'ground_clutter': {'data': np.asarray(clutter)}}

def get_start_end(self, sweep):
return (self.sweep_start_ray_index['data'][sweep],
self.sweep_end_ray_index['data'][sweep])

def add_field(self, name, field, replace_existing=False):
self.fields[name] = field


def test_ppi_clutter_map_matches_sweeps_by_elevation_and_azimuth():
radar = _Radar([1.0, 2.0], [0.0, 180.0, 0.0, 180.0])
clutter = _Radar([1.0], [1.0, 181.0], [[1, 0], [0, 1]])

add_clutter_map(radar, clutter)

np.testing.assert_array_equal(
radar.fields['ground_clutter']['data'],
[[1, 0], [0, 1], [0, 0], [0, 0]])

17 changes: 17 additions & 0 deletions cmac/tests/test_phase_masks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
"""Tests for the masks carried by paired phase fields."""

import numpy as np

from cmac.cmac_processing import synchronize_phase_field_masks


def test_synchronize_phase_field_masks_uses_union_of_masks():
phidp = {'data': np.ma.array([[1.0, 2.0, 3.0]], mask=[[0, 1, 0]])}
kdp = {'data': np.ma.array([[4.0, 5.0, 6.0]], mask=[[0, 0, 1]])}

synchronize_phase_field_masks(phidp, kdp)

expected = np.array([[False, True, True]])
np.testing.assert_array_equal(np.ma.getmaskarray(phidp['data']), expected)
np.testing.assert_array_equal(np.ma.getmaskarray(kdp['data']), expected)

21 changes: 2 additions & 19 deletions scripts/cmac
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ import argparse
from pathlib import Path

import netCDF4
import numpy as np
import pyart
import xarray as xr

from cmac import (
add_clutter_map,
area_coverage,
cmac,
get_cmac_values,
Expand Down Expand Up @@ -70,24 +70,7 @@ def main():
print('## Reading dictionary...')
print('## Adding clutter field..')
clutter_radar = pyart.io.read(str(args.clutter_file))
clutter_field_dict = clutter_radar.fields['ground_clutter'].copy()
# For XSAPR data at NSA, sometimes only 3 of 6 sweeps are present;
# copy clutter information from the bottom sweeps only.
if clutter_radar.nsweeps > radar.nsweeps:
fname = next(iter(radar.fields))
clutter_field_dict['data'] = np.zeros_like(
radar.fields[fname]['data'])
for i, azi_angle in enumerate(radar.fixed_angle['data']):
ind = np.argwhere(
clutter_radar.fixed_angle['data'] == azi_angle)
if len(ind) > 0:
clutter = clutter_radar.get_field(
int(ind), 'ground_clutter')
start, end = radar.get_start_end(i)
clutter_field_dict['data'][
int(start):int(end + 1)] = clutter
radar.add_field(
'ground_clutter', clutter_field_dict, replace_existing=True)
add_clutter_map(radar, clutter_radar)
del clutter_radar

cmac_radar = cmac(
Expand Down
Loading