diff --git a/cmac/gate_id_backends.py b/cmac/gate_id_backends.py index 1582cf6..42308bc 100644 --- a/cmac/gate_id_backends.py +++ b/cmac/gate_id_backends.py @@ -161,33 +161,27 @@ def radar_palette_available(): - """Return True if ``radar_palette.gateid`` can be imported.""" + """Return True if CMAC's gate-ID classifier can be imported.""" try: - import radar_palette.gateid # noqa: F401 + from . import radar_palette # noqa: F401 except ImportError: return False return True def _import_radar_palette(): - """Import and return the ``radar_palette.gateid`` entry points. + """Import and return CMAC's radar-palette entry points. Raised as an ``ImportError`` with an actionable message rather than a bare ``ModuleNotFoundError`` from somewhere deep in the VAP, because selecting this backend is a configuration choice and the fix is an install. """ try: - from radar_palette.gateid import entry_points, single + from .radar_palette.gateid import entry_points, single except ImportError as err: raise ImportError( - "gate_id_method='radar_palette' requires radar_palette.gateid, " - "which CMAC does not depend on: it is imported only when a radar " - "configuration selects this backend. The gateid module is not in " - "radar-palette's released main branch yet -- install from the " - "gateid development branch (pip install " - "'radar-palette @ git+https://github.com/scollis/radar-palette" - "@gateid'), or set gate_id_method='cmac_fuzzy' in the radar " - "configuration to use CMAC's own fuzzy classifier." + "CMAC's radar-palette gate-ID implementation could not " + "be imported. Check the CMAC installation." ) from err return entry_points, single diff --git a/cmac/radar_palette/__init__.py b/cmac/radar_palette/__init__.py new file mode 100644 index 0000000..afc36a2 --- /dev/null +++ b/cmac/radar_palette/__init__.py @@ -0,0 +1,9 @@ +"""Radar-palette components used by CMAC's gate-ID backend. + +Only the gate-ID implementation is included here; the rest of radar-palette +is intentionally not part of CMAC's surface. +""" + +from . import gateid + +__all__ = ["gateid"] diff --git a/cmac/radar_palette/gateid/__init__.py b/cmac/radar_palette/gateid/__init__.py new file mode 100644 index 0000000..76914ce --- /dev/null +++ b/cmac/radar_palette/gateid/__init__.py @@ -0,0 +1,5 @@ +"""Radar-palette gate identification implementation used by CMAC.""" + +from . import single + +__all__ = ["single"] diff --git a/cmac/radar_palette/gateid/entry_points.py b/cmac/radar_palette/gateid/entry_points.py new file mode 100644 index 0000000..62675f5 --- /dev/null +++ b/cmac/radar_palette/gateid/entry_points.py @@ -0,0 +1,393 @@ +"""Public entry points: classify a radar volume of either object family. + +Following the package convention, these accept a Py-ART :class:`~pyart.core.Radar` +or an xradar :class:`~xarray.DataTree` and return the family the caller passed +in, unless ``output_flavor`` asks otherwise. Conversion is delegated to +:mod:`radar_palette.io.flavors`; nothing here reimplements the mapping between +the two data models. + +Computation happens on the Py-ART surface, as elsewhere in this package, because +that is where the gate-geometry helpers live. +""" + +import numpy as np + +from ..gateid import single +from .features import build_features, resolve_fields +from .multitrip import volume_echo_top +from ..io.flavors import ( + detect_radar_flavor, + resolve_output_flavor, + to_pyart_radar, + to_radar_flavor, +) + +__all__ = [ + "gate_id", + "meteorological_gatefilter", +] + +#: Added to the volume echo top to get the second-trip ceiling, so a storm +#: slightly taller than the one sampled is not vetoed. +TRIP_CEILING_MARGIN_M = 3000.0 + +#: Scan modes whose sweeps are a constant-elevation surface. +PPI_MODES = ("ppi", "sector", "azimuth_surveillance") +#: Scan modes whose sweeps are a vertical cross-section. +RHI_MODES = ("rhi", "elevation_surveillance", "manual_rhi") + + +def _decode_sweep_modes(radar): + """Return the decoded ``sweep_mode`` string for each sweep. + + The two object families store this differently and both are easy to get + wrong. Py-ART holds a masked array of single *characters*, one row per + sweep, so joining a row without dropping the mask turns ``sector`` into + byte junk. The :class:`pyart.xradar.Xradar` wrapper instead holds one + whole byte string per sweep, so splitting it per character yields empty + strings and every sweep looks like an unknown scan type. + + Both shapes are handled here, and the result is verified by a test that + runs the same volume through both families. + """ + raw = np.atleast_1d(radar.sweep_mode["data"]) + + def _clean(value): + if isinstance(value, (bytes, np.bytes_)): + value = value.decode(errors="replace") + return str(value).replace("\x00", "").strip().lower() + + # One whole string per sweep (the xradar wrapper). + if raw.ndim == 1 and raw.dtype.kind in "SU" and raw.dtype.itemsize > 1: + return [_clean(v) for v in raw] + if raw.ndim == 1 and raw.dtype.kind == "O": + return [_clean(v) for v in raw] + + # A row of characters per sweep (Py-ART's CfRadial1 reader). + modes = [] + for row in np.atleast_2d(raw): + filled = np.ma.filled(np.ma.asarray(row), b" ").ravel() + modes.append( + _clean( + b"".join( + c if isinstance(c, (bytes, np.bytes_)) else str(c).encode() + for c in filled + ) + ) + ) + return modes + + +def _sweep_geometry_ok(radar, sweep, max_ppi_span=5.0, min_rhi_span=20.0): + """Whether a sweep is a usable PPI or RHI. + + ``sweep_mode`` is checked *and* the elevation span is measured, because the + label is not always right: an RHI mislabelled as a PPI, processed as one, + mixes every elevation into a single surface and yields gate altitudes above + 100 km. + """ + modes = _decode_sweep_modes(radar) + mode = modes[sweep] if sweep < len(modes) else "" + elev = radar.elevation["data"][radar.get_slice(sweep)] + elev = np.asarray(elev, dtype="f8") + elev = elev[np.isfinite(elev)] + if elev.size == 0: + return False, mode, np.nan + span = float(elev.max() - elev.min()) + if mode.startswith(PPI_MODES): + return span <= max_ppi_span, mode, span + if mode.startswith(RHI_MODES): + return span >= min_rhi_span, mode, span + return False, mode, span + + +def _nyquist_velocity(radar, radar_like=None): + """Nyquist velocity in m/s, or NaN. + + Py-ART keeps this in ``instrument_parameters``; the xradar layout keeps a + ``nyquist_velocity`` variable on each sweep and the Py-ART wrapper does not + surface it, so the tree is consulted directly as a fallback. Losing it is + not harmless: without a Nyquist the folding-aware velocity texture cannot + be computed, and the phase-coherence test that identifies second trip and + receiver noise silently disappears. + """ + params = getattr(radar, "instrument_parameters", None) or {} + if "nyquist_velocity" in params: + vals = np.asarray(params["nyquist_velocity"]["data"], dtype="f8") + vals = vals[np.isfinite(vals)] + if vals.size: + return float(np.median(vals)) + + tree = getattr(radar, "ds", None) or radar_like + children = getattr(tree, "children", None) + if children: + found = [] + for key in children: + if not str(key).startswith("sweep"): + continue + node = tree[key] + data = node.to_dataset() if hasattr(node, "to_dataset") else node + if "nyquist_velocity" in data: + vals = np.asarray(data["nyquist_velocity"].values, dtype="f8") + vals = vals[np.isfinite(vals)] + if vals.size: + found.append(float(np.median(vals))) + if found: + return float(np.median(found)) + return np.nan + + +def gate_id( + radar_like, + freezing_level_m=None, + temperature_field=None, + field_name="gate_id", + texture_window=4, + add_margin=True, + snr_min=3.0, + vel_texture_max=None, + min_run=3, + despeckle_keep_dbz=30.0, + incoherent_frac=single.INCOHERENT_TEXTURE_FRAC, + output_flavor=None, + replace_existing=True, +): + """Classify the dominant scatterer at every gate. + + Parameters + ---------- + radar_like : pyart.core.Radar or xarray.DataTree + Volume carrying uncorrected polarimetric moments. + freezing_level_m : float, optional + Height of the 0 degC level, metres MSL. Enables the melting-layer hard + constraints and, absent ``temperature_field``, a lapse-rate temperature. + temperature_field : str, optional + Name of a field holding gate temperature in degrees Celsius. Strongly + preferred over the lapse-rate fallback; the choice is recorded in the + returned metadata as ``temp_source``. + field_name : str, optional + Output field name. + texture_window : int, optional + Footprint of the texture operator, in gates and rays. + add_margin : bool, optional + Also write ``_margin``, the winning minus runner-up score. + snr_min : float or None, optional + Gates below this SNR are forced to ``no_scatter`` before scoring. + vel_texture_max : float or None, optional + Absolute velocity-texture cut in m/s. Normally leave this ``None`` and + use ``incoherent_frac``, which is independent of PRF. + min_run : int, optional + Class runs shorter than this many gates along a ray are dissolved. + despeckle_keep_dbz : float or None, optional + Reflectivity at or above which a gate is never despeckled or gated out. + incoherent_frac : float or None, optional + Velocity texture, as a fraction of the uniform-random-phase limit, + above which a gate cannot be first-trip weather. + output_flavor : RadarFlavor or str or None, optional + Family to return. Defaults to mirroring the input. + replace_existing : bool, optional + Overwrite an existing field of the same name. + + Returns + ------- + radar : pyart.core.Radar or xarray.DataTree + The input volume with ``gate_id`` attached, in the requested family. + meta : dict + Resolved field names, missing fields, temperature provenance, per-class + gate counts, and the number of gates removed by each speckle control. + + Notes + ----- + Sweeps whose geometry is unusable are left unclassified and reported in + ``meta['skipped_sweeps']`` rather than silently producing wrong answers. + + Examples + -------- + Classify a Py-ART volume:: + + radar, meta = gate_id(radar, freezing_level_m=4670.0) + + Classify an xradar volume and get a ``DataTree`` back:: + + tree, meta = gate_id(tree, freezing_level_m=4670.0) + """ + input_flavor = detect_radar_flavor(radar_like) + output_flavor = resolve_output_flavor(output_flavor, input_flavor) + radar, _ = to_pyart_radar(radar_like) + + resolved, missing = resolve_fields(radar.fields) + if "z" not in resolved or "rhohv" not in resolved: + raise ValueError( + f"gate_id needs at least reflectivity and RhoHV; missing {sorted(missing)}" + ) + + shape = (radar.nrays, radar.ngates) + codes = np.zeros(shape, dtype="i2") + margin = np.full(shape, np.nan, dtype="f8") + counts = dict.fromkeys(single.CLASSES.values(), 0) + skipped, n_noise, n_speck = [], 0, 0 + + nyquist = _nyquist_velocity(radar, radar_like) + params = getattr(radar, "instrument_parameters", None) or {} + + # The second-trip ceiling is a property of the storm, so it is derived + # once over every usable sweep rather than per sweep. See + # radar_palette.gateid.multitrip.volume_echo_top for why this matters. + trip_ceiling_m, ceiling_source = None, "sweep_echo_top" + if radar.nsweeps > 1: + pairs = [] + for sweep in range(radar.nsweeps): + ok, _, _ = _sweep_geometry_ok(radar, sweep) + if not ok: + continue + sl = radar.get_slice(sweep) + pairs.append( + ( + np.ma.filled( + radar.fields[resolved["z"]]["data"][sl].astype("f8"), np.nan + ), + np.asarray(radar.gate_altitude["data"][sl], dtype="f8"), + ) + ) + top = volume_echo_top(pairs) if pairs else np.nan + if np.isfinite(top): + trip_ceiling_m = float(top) + TRIP_CEILING_MARGIN_M + ceiling_source = "volume_echo_top" + + temp_source = "none" + for sweep in range(radar.nsweeps): + ok, mode, span = _sweep_geometry_ok(radar, sweep) + if not ok: + skipped.append({"sweep": sweep, "sweep_mode": mode, "elevation_span": span}) + continue + sl = radar.get_slice(sweep) + moments = { + key: np.ma.filled(radar.fields[name]["data"][sl].astype("f8"), np.nan) + for key, name in resolved.items() + } + temperature = None + if temperature_field and temperature_field in radar.fields: + temperature = np.ma.filled( + radar.fields[temperature_field]["data"][sl].astype("f8"), np.nan + ) + + prt = None + if "prt" in params: + values = np.asarray(params["prt"]["data"], dtype="f8")[sl] + values = values[np.isfinite(values)] + if values.size: + prt = float(np.median(values)) + + feats, fmeta = build_features( + moments, + gate_altitude=radar.gate_altitude["data"][sl], + gate_range=radar.range["data"], + nyquist=nyquist, + temperature=temperature, + freezing_level_m=freezing_level_m, + texture_window=texture_window, + elevation=radar.elevation["data"][sl], + prt=prt, + trip_ceiling_m=trip_ceiling_m, + radar_altitude_m=float(np.asarray(radar.altitude["data"]).ravel()[0]), + ) + temp_source = fmeta["temp_source"] + + sweep_codes, info = single.classify( + feats, + shape=fmeta["shape"], + snr_min=snr_min, + vel_texture_max=vel_texture_max, + min_run=min_run, + despeckle_keep_dbz=despeckle_keep_dbz, + incoherent_frac=incoherent_frac, + ) + + codes[sl] = sweep_codes.reshape(fmeta["shape"]) + margin[sl] = info["margin"].reshape(fmeta["shape"]) + n_noise += info["n_noise_floor"] + n_speck += max(info["n_despeckled"], 0) + + for code, name in single.CLASSES.items(): + counts[name] = int((codes == code).sum()) + + labels = [single.CLASSES[c] for c in sorted(single.CLASSES)] + radar.add_field( + field_name, + { + "data": np.ma.masked_array(codes, mask=False), + "long_name": "Classification of dominant scatterer", + "standard_name": "gate_id", + "units": "", + "flag_values": sorted(single.CLASSES), + "flag_meanings": " ".join(labels), + "notes": ",".join( + f"{c}:{single.CLASSES[c]}" for c in sorted(single.CLASSES) + ), + "comment": ( + "Computed from uncorrected moments. Class scores are " + "normalised by attainable weight budget; gates with no " + "positive evidence are 0=unclassified rather than a " + f"silent argmax tie-break. temperature: {temp_source}." + ), + }, + replace_existing=replace_existing, + ) + + if add_margin: + radar.add_field( + f"{field_name}_margin", + { + "data": np.ma.masked_invalid(margin), + "long_name": "Winning minus runner-up normalised class score", + "units": "", + "comment": ( + "Per-gate confidence. Small values mark gates where " + "two classes are nearly tied." + ), + }, + replace_existing=replace_existing, + ) + + meta = { + "resolved": resolved, + "missing": missing, + "temp_source": temp_source, + "nyquist": nyquist, + "class_counts": counts, + "skipped_sweeps": skipped, + "n_noise_floor": n_noise, + "n_despeckled": n_speck, + "trip_ceiling_m": trip_ceiling_m, + "trip_ceiling_source": ceiling_source, + } + return to_radar_flavor(radar, output_flavor), meta + + +def meteorological_gatefilter(radar_like, field_name="gate_id", include=None): + """Build a :class:`pyart.filters.GateFilter` keeping hydrometeor classes. + + Parameters + ---------- + radar_like : pyart.core.Radar or xarray.DataTree + Volume already carrying ``field_name``. + field_name : str, optional + Name of the gate-ID field. + include : sequence of str, optional + Class names to keep. Defaults to the liquid and frozen classes plus + ``melting_wet``. + + Returns + ------- + pyart.filters.GateFilter + Filter suitable for handing to downstream correction steps. + """ + import pyart + + radar, _ = to_pyart_radar(radar_like) + include = include or (single.LIQUID + single.FROZEN + ("melting_wet",)) + gatefilter = pyart.filters.GateFilter(radar) + gatefilter.exclude_all() + for name in include: + gatefilter.include_equal(field_name, single.NAME_TO_CODE[name]) + return gatefilter diff --git a/cmac/radar_palette/gateid/features.py b/cmac/radar_palette/gateid/features.py new file mode 100644 index 0000000..071d024 --- /dev/null +++ b/cmac/radar_palette/gateid/features.py @@ -0,0 +1,256 @@ +"""Feature extraction from a radar volume, for either object family. + +Every feature here is derived from *uncorrected* moments, because the gate ID +runs before correction. Nothing in this module may require a ``corrected_*`` +field. + +Field-name resolution +--------------------- +Moment names are not stable across instruments or even across deployment years +of one instrument: BNF C-SAPR2 volumes from 2025 lack the ``uncorrected_`` +variants that 2026 volumes carry. Each logical feature is therefore resolved +through an ordered preference list and the winning name is returned in the +metadata, so a field that has silently vanished is an auditable fact rather +than an all-NaN column nobody notices. + +Ordering matters: the first match wins, so the shared portion of the Py-ART and +ODIM name lists must agree or the two object families would resolve different +fields from the same volume. A test enforces that. +""" + +import numpy as np + +from . import single + +__all__ = [ + "FIELD_CANDIDATES", + "build_features", + "resolve_fields", +] + +#: Logical feature name -> ordered candidate moment names. CfRadial/ARM names +#: come first, then the ODIM short names, so both families agree on the shared +#: block. +FIELD_CANDIDATES = { + "z": [ + "uncorrected_reflectivity_h", + "reflectivity_horizontal", + "reflectivity", + "DBZ", + "DBZH", + ], + "zdr": [ + "uncorrected_differential_reflectivity", + "differential_reflectivity", + "ZDR", + ], + "rhohv": [ + "uncorrected_copol_correlation_coeff", + "copol_correlation_coeff", + "cross_correlation_ratio", + "cross_correlation_ratio_hv", + "RHOHV", + ], + "ncp": [ + "uncorrected_normalized_coherent_power", + "normalized_coherent_power", + "NCP", + ], + "snr": [ + "uncorrected_signal_to_noise_ratio_copolar_h", + "signal_to_noise_ratio_copolar_h", + "signal_to_noise_ratio", + "SNR", + ], + "sw": ["uncorrected_spectral_width_h", "spectral_width", "WIDTH", "WRADH"], + "phidp": ["uncorrected_differential_phase", "differential_phase", "PHIDP"], + "vel": [ + "uncorrected_mean_doppler_velocity_h", + "mean_doppler_velocity", + "velocity", + "VEL", + "VRADH", + ], +} + +#: Moments whose along-ray standard deviation is computed. SD(rho_hv) is the +#: strongest non-meteorological discriminant measured on BNF data (Cohen's +#: d = 3.29 against precipitation); SD(PhiDP) is the classical one. +TEXTURE_FIELDS = ( + ("rhohv", "rhohv_texture"), + ("phidp", "phidp_texture"), + ("zdr", "zdr_texture"), + ("z", "z_texture"), +) + + +def resolve_fields(available, candidates=None): + """Map logical feature names onto the moments actually present. + + Parameters + ---------- + available : iterable of str + Field names present on the volume. + candidates : dict, optional + Override for :data:`FIELD_CANDIDATES`. + + Returns + ------- + resolved : dict + Logical name -> the moment name that will be used. + missing : list of str + Logical names with no candidate present. + """ + candidates = FIELD_CANDIDATES if candidates is None else candidates + have = set(available) + resolved, missing = {}, [] + for short, names in candidates.items(): + for name in names: + if name in have: + resolved[short] = name + break + else: + missing.append(short) + return resolved, missing + + +def build_features( + moments, + gate_altitude, + gate_range, + nyquist=None, + temperature=None, + freezing_level_m=None, + texture_window=4, + elevation=None, + prt=None, + radar_altitude_m=0.0, + trip_ceiling_m=None, +): + """Assemble the feature dictionary the classifier consumes. + + Parameters + ---------- + moments : dict of str to ndarray + Logical feature name -> 2-D array shaped (ray, gate). Keys follow + :data:`FIELD_CANDIDATES`; missing moments are simply absent. + gate_altitude : ndarray + Gate altitude in metres above mean sea level, shaped like the moments. + gate_range : ndarray + Slant range in metres, broadcastable to the moment shape. + nyquist : float, optional + Nyquist velocity in m/s. Required for folding-aware velocity texture; + without it the velocity features are omitted rather than computed + wrongly. + temperature : ndarray, optional + Gate temperature in degrees Celsius. Strongly preferred over + ``freezing_level_m`` alone. + freezing_level_m : float, optional + Height of the 0 degC level in metres MSL. Used for height-over-isotherm + and, if ``temperature`` is absent, for a 6.5 K/km lapse-rate estimate + that is flagged as such in the returned metadata. + texture_window : int, optional + Footprint, in gates and rays, of the texture operator. + elevation : array_like, optional + Elevation of each ray in degrees. Enables the geometric second-trip + veto; without it ``trip_possible`` is not produced and the multi-trip + class rests on moment evidence alone. + prt : float, optional + Pulse repetition time in seconds, for the unambiguous range. + radar_altitude_m : float, optional + Radar altitude above mean sea level. + trip_ceiling_m : float, optional + Height above which no scatterer is credible, for the second-trip veto. + Should be derived from the whole VOLUME: a low sweep's own echo top + reflects how high its beam reaches, not how tall the storm is, and + using it vetoes second trip precisely where it is most likely. When + omitted the ceiling falls back to this sweep's echo top, which is + correct only for a single-sweep input such as an RHI. + + Returns + ------- + features : dict of str to ndarray + Flattened 1-D feature arrays. + meta : dict + ``shape``, ``nyquist``, ``temp_source`` and the derived feature names. + """ + if "z" not in moments: + raise ValueError("reflectivity ('z') is required") + shape = np.asarray(moments["z"]).shape + + feats = {} + for key, arr in moments.items(): + feats[key] = np.asarray(arr, dtype="f8").ravel() + + height = np.broadcast_to(np.asarray(gate_altitude, dtype="f8"), shape) + rng = np.broadcast_to(np.asarray(gate_range, dtype="f8"), shape) + feats["height_km"] = (height / 1000.0).ravel() + feats["range_km"] = (rng / 1000.0).ravel() + + for short, key in TEXTURE_FIELDS: + if short in moments: + feats[key] = single.texture( + np.asarray(moments[short], dtype="f8"), texture_window + ).ravel() + + if "zdr" in moments and "rhohv" in moments: + feats["dr"] = single.depolarization_ratio( + np.asarray(moments["zdr"], dtype="f8"), + np.asarray(moments["rhohv"], dtype="f8"), + ).ravel() + + if ( + "vel" in moments + and nyquist is not None + and np.isfinite(nyquist) + and nyquist > 0 + ): + vel = np.asarray(moments["vel"], dtype="f8") + vtex = single.angular_texture(vel, nyquist, texture_window) + feats["vel_texture"] = vtex.ravel() + feats["vel_norm"] = (np.abs(vel) / nyquist).ravel() + # Normalised by the uniform-random-phase limit, so the membership + # functions are independent of PRF and transfer between radars. + feats["vel_texture_norm"] = (vtex / (nyquist / np.sqrt(3.0))).ravel() + + # Geometric plausibility of second trip. Without a PRT this is all-ones, + # so the moment evidence stands alone rather than being silently + # overridden by a veto that could not be evaluated. + if elevation is not None: + from .multitrip import second_trip_possible + + possible, trip_meta = second_trip_possible( + rng, + np.broadcast_to(np.asarray(elevation, dtype="f8").reshape(-1, 1), shape), + prt=prt, + ceiling_m=trip_ceiling_m, + radar_altitude_m=radar_altitude_m, + reflectivity=np.asarray(moments["z"], dtype="f8"), + gate_height_m=height, + ) + feats["trip_possible"] = possible.astype("f8").ravel() + else: + trip_meta = None + + temp_source = "none" + if temperature is not None: + feats["temp"] = np.broadcast_to( + np.asarray(temperature, dtype="f8"), shape + ).ravel() + temp_source = "provided" + elif freezing_level_m is not None: + feats["temp"] = ((freezing_level_m - height) * 6.5e-3).ravel() + temp_source = "lapse_rate_from_freezing_level" + + if freezing_level_m is not None: + feats["hoi0_km"] = ((height - freezing_level_m) / 1000.0).ravel() + + meta = { + "shape": shape, + "nyquist": nyquist, + "temp_source": temp_source, + "freezing_level_m": freezing_level_m, + "second_trip": trip_meta, + "derived": sorted(set(feats) - set(moments)), + } + return feats, meta diff --git a/cmac/radar_palette/gateid/multitrip.py b/cmac/radar_palette/gateid/multitrip.py new file mode 100644 index 0000000..3e64732 --- /dev/null +++ b/cmac/radar_palette/gateid/multitrip.py @@ -0,0 +1,303 @@ +"""Geometric plausibility of second-trip echo. + +Second trip is hard to identify from moments alone, because the signature it +presents -- incoherent phase, low SQI/NCP, noisy rho_hv -- is shared by +receiver noise and by genuinely turbulent weather. Reflectivity texture and +velocity texture both go high in a sheared updraught, so a purely +moment-based test will label real convection as second trip. + +Geometry breaks the tie, and does so without ambiguity. An echo appearing at +apparent range ``r`` in the *n*-th trip is truly at ``r + n * r_max``, where +``r_max = c * PRT / 2``. Push that true range through the beam-height equation +at the sweep's elevation and the answer is usually absurd: for a BNF C-SAPR2 +volume at PRF 1240 Hz, ``r_max`` is 120.9 km, so a second-trip echo seen at +40 km apparent range is really at 161 km, which at 5.5 deg elevation places it +17 km above the radar and at 12 deg places it 35 km up. Nothing scatters there. + +So the rule is: second trip is only *possible* where the second-trip beam is +still low enough to intersect the troposphere. That is a hard geometric veto, +not a tuned threshold, and it depends only on the PRT and the elevation angle +-- both of which the volume already carries. + +The echo-top ceiling can be taken from the data rather than assumed. If the +tallest real echo in this volume reaches 17 km, an apparent gate whose +second-trip height exceeds that by a margin cannot be second trip from this +storm. + +References +---------- +Doviak, R. J., and D. S. Zrnic, 1993: *Doppler Radar and Weather +Observations*, 2nd ed. Academic Press. Range folding and the unambiguous +range. + +Zrnic, D. S., and A. V. Ryzhkov, 1999: Polarimetry for weather surveillance +radars. *Bull. Amer. Meteor. Soc.*, 80, 389-406. Second-trip signatures in +polarimetric moments. +""" + +import numpy as np + +__all__ = [ + "SPEED_OF_LIGHT", + "beam_height", + "echo_top_height", + "volume_echo_top", + "max_elevation_for_second_trip", + "second_trip_possible", + "unambiguous_range", +] + +#: Speed of light in m/s. +SPEED_OF_LIGHT = 299792458.0 + +#: Effective-earth-radius multiplier for standard refraction. +EFFECTIVE_EARTH = 4.0 / 3.0 + +#: Mean earth radius in m. +EARTH_RADIUS = 6371000.0 + + +def unambiguous_range(prt=None, prf=None): + """Maximum unambiguous range in metres, ``c * PRT / 2``. + + Parameters + ---------- + prt : float, optional + Pulse repetition time in seconds. + prf : float, optional + Pulse repetition frequency in Hz. Used if ``prt`` is not given. + + Returns + ------- + float + Unambiguous range in metres, or NaN if neither argument is usable. + """ + if prt is None and prf: + prt = 1.0 / float(prf) + if not prt or not np.isfinite(prt) or prt <= 0: + return np.nan + return SPEED_OF_LIGHT * float(prt) / 2.0 + + +def beam_height(range_m, elevation_deg, radar_altitude_m=0.0): + """Beam-centre height above mean sea level under standard refraction. + + Uses the 4/3-earth approximation, which is what operational radar + processing assumes and is adequate for deciding whether an echo would sit + in the troposphere or the mesosphere. + + Parameters + ---------- + range_m : array_like + Slant range in metres. + elevation_deg : array_like + Elevation angle in degrees. + radar_altitude_m : float, optional + Radar altitude above mean sea level in metres. + + Returns + ------- + ndarray + Height in metres above mean sea level. + """ + r = np.asarray(range_m, dtype="f8") + el = np.deg2rad(np.asarray(elevation_deg, dtype="f8")) + ke_a = EFFECTIVE_EARTH * EARTH_RADIUS + return ( + np.sqrt(r**2 + ke_a**2 + 2.0 * r * ke_a * np.sin(el)) - ke_a + radar_altitude_m + ) + + +def echo_top_height(reflectivity, gate_height_m, threshold_dbz=10.0, percentile=99.9): + """Height of the tallest credible echo in a sweep, in metres. + + Taken as a high percentile rather than the maximum, so a single noisy gate + aloft cannot raise the ceiling and admit second trip everywhere. + + Parameters + ---------- + reflectivity : array_like + Reflectivity in dBZ. + gate_height_m : array_like + Gate height above mean sea level, same shape. + threshold_dbz : float, optional + Reflectivity above which a gate counts as echo. + percentile : float, optional + Percentile of the echo-height distribution to report. + + Returns + ------- + float + Height in metres, or NaN if there is no echo. + """ + z = np.asarray(reflectivity, dtype="f8") + h = np.asarray(gate_height_m, dtype="f8") + lit = np.isfinite(z) & np.isfinite(h) & (z >= threshold_dbz) + if not lit.any(): + return np.nan + return float(np.percentile(h[lit], percentile)) + + +def volume_echo_top(sweeps, threshold_dbz=10.0, percentile=99.9): + """Echo top over a whole volume, in metres. + + The ceiling for the second-trip veto must describe the *storm*, not the + beam. A 1.5 deg sweep cannot see above about 4 km at the ranges it + samples, so its own echo top is a property of the scan geometry rather + than of the weather -- on a BNF volume whose storm reached 17.0 km, sweep + 0 reported 3.85 km. Using that as the ceiling vetoes second trip beyond + ~60 km apparent range on exactly the low sweeps where second trip is most + likely, which is self-defeating. + + Parameters + ---------- + sweeps : iterable of (reflectivity, gate_height_m) + One pair per sweep, each a 2-D array in dBZ and metres MSL. + threshold_dbz : float, optional + Reflectivity above which a gate counts as echo. + percentile : float, optional + Percentile of the echo-height distribution, per sweep. + + Returns + ------- + float + The tallest per-sweep echo top, or NaN if no sweep has echo. + """ + tops = [echo_top_height(z, h, threshold_dbz, percentile) for z, h in sweeps] + tops = [t for t in tops if np.isfinite(t)] + return float(max(tops)) if tops else np.nan + + +def max_elevation_for_second_trip( + unambiguous_range_m, ceiling_m, max_range_m, radar_altitude_m=0.0, trip=2 +): + """Highest elevation at which trip-``n`` echo could still be tropospheric. + + Solves ``beam_height(r_apparent + (trip - 1) * r_max, el) <= ceiling`` for + the elevation, at the *shortest* apparent range in the sweep, which is the + most permissive case. Above the returned angle no gate in the sweep can be + second trip. + + Parameters + ---------- + unambiguous_range_m : float + ``c * PRT / 2``. + ceiling_m : float + Height above which no scatterer is credible -- typically the echo top + plus a margin. + max_range_m : float + Longest gate range in the sweep, used only as a sanity bound. + radar_altitude_m : float, optional + Radar altitude above mean sea level. + trip : int, optional + Trip number. 2 is second trip. + + Returns + ------- + float + Elevation in degrees, 0 if even a horizontal beam is too high, or NaN + if the inputs are unusable. + """ + if not np.isfinite(unambiguous_range_m) or not np.isfinite(ceiling_m): + return np.nan + folded = (trip - 1) * float(unambiguous_range_m) + if folded <= 0: + return np.nan + + # The nearest plausible apparent range gives the lowest second-trip + # height, so it bounds the elevation most generously. + r_true = folded + max(float(max_range_m) * 0.02, 1000.0) + candidates = np.linspace(0.0, 90.0, 1801) + heights = beam_height(r_true, candidates, radar_altitude_m) + ok = heights <= ceiling_m + if not ok.any(): + return 0.0 + return float(candidates[ok][-1]) + + +def second_trip_possible( + range_m, + elevation_deg, + prt=None, + prf=None, + ceiling_m=None, + radar_altitude_m=0.0, + reflectivity=None, + gate_height_m=None, + ceiling_margin_m=3000.0, + trip=2, +): + """Boolean mask: gates where trip-``n`` echo is geometrically credible. + + A gate is possible only if the beam, followed out to the *true* range the + echo would have to occupy, is still at or below ``ceiling_m``. Everything + else is impossible regardless of how incoherent it looks. + + Parameters + ---------- + range_m : array_like + Apparent slant range of each gate, in metres. + elevation_deg : array_like + Elevation of each gate, in degrees. Broadcast against ``range_m``. + prt, prf : float, optional + Pulse repetition time or frequency, for the unambiguous range. + ceiling_m : float, optional + Height above which no scatterer is credible. If not given it is taken + from the data via :func:`echo_top_height` plus ``ceiling_margin_m``, + and failing that defaults to 20 km. + radar_altitude_m : float, optional + Radar altitude above mean sea level. + reflectivity, gate_height_m : array_like, optional + Used to derive the ceiling when ``ceiling_m`` is not supplied. + ceiling_margin_m : float, optional + Added to the measured echo top, so a storm slightly taller than the + one sampled does not get vetoed. + trip : int, optional + Trip number to test. + + Returns + ------- + possible : ndarray of bool + True where trip-``n`` echo could exist. + info : dict + ``unambiguous_range_m``, ``ceiling_m``, ``max_elevation_deg`` and the + fraction of gates left possible, for reporting. + """ + r = np.asarray(range_m, dtype="f8") + el = np.broadcast_to(np.asarray(elevation_deg, dtype="f8"), r.shape) + + r_max = unambiguous_range(prt=prt, prf=prf) + if not np.isfinite(r_max): + # Without a PRT nothing can be said, so nothing is vetoed: the moment + # evidence stands alone rather than being silently overridden. + return np.ones(r.shape, dtype=bool), { + "unambiguous_range_m": np.nan, + "ceiling_m": np.nan, + "max_elevation_deg": np.nan, + "fraction_possible": 1.0, + "note": "no PRT or PRF available; geometric veto not applied", + } + + if ceiling_m is None: + top = np.nan + if reflectivity is not None and gate_height_m is not None: + top = echo_top_height(reflectivity, gate_height_m) + ceiling_m = (top + ceiling_margin_m) if np.isfinite(top) else 20000.0 + + true_range = r + (trip - 1) * r_max + height = beam_height(true_range, el, radar_altitude_m) + possible = height <= ceiling_m + + info = { + "unambiguous_range_m": float(r_max), + "ceiling_m": float(ceiling_m), + "max_elevation_deg": max_elevation_for_second_trip( + r_max, + ceiling_m, + float(np.nanmax(r)) if r.size else 0.0, + radar_altitude_m, + trip, + ), + "fraction_possible": float(possible.mean()) if possible.size else 0.0, + } + return possible, info diff --git a/cmac/radar_palette/gateid/single.py b/cmac/radar_palette/gateid/single.py new file mode 100644 index 0000000..6105a13 --- /dev/null +++ b/cmac/radar_palette/gateid/single.py @@ -0,0 +1,707 @@ +"""Single-volume gate identification. + +Per-gate classification of the dominant scatterer from polarimetric moments, +computed from a single radar volume with no reference to any other. This is the +memoryless baseline; :mod:`radar_palette.gateid.temporal` builds on it. + +The classifier is intended for processing chains that identify gates *before* +correction, where the gate ID is not a diagnostic product but the control flow: +it selects which gates reach phase processing, attenuation correction, velocity +dealiasing and rainfall estimation. A defect here is silent and propagates. + +Design decisions worth knowing +------------------------------ +**Scores are normalised by attainable weight budget.** In a plain additive fuzzy +scheme a class can only win if the evidence it *could* muster is competitive. +Profiling an operational configuration on 4.75 million gates showed ``rain`` +reaching at most 3.0 without a sounding while ``snow`` reached 4.0, so snow took +84% of gates above 40 dBZ in warm-season convection. No amount of +membership-function tuning fixes that; the budget itself is the bug. + +**A class must have its evidence present to be eligible.** Budget normalisation +alone lets a class score a perfect 1.0 on two of five terms and tie one that +satisfied all of its own. See :data:`MIN_EVIDENCE_COVERAGE`. + +**Gates with no positive evidence are ``unclassified``, never class zero.** +``argmax`` breaks ties toward index 0, silently. + +**Phase coherence, not an arbitrary threshold, identifies incoherent echo.** A +magnetron randomises transmit phase pulse to pulse, so second-trip echo and +receiver noise both have radial velocity uniformly distributed over the Nyquist +interval, giving a texture of ``v_nyq/sqrt(3)``. See +:data:`INCOHERENT_TEXTURE_FRAC`. + +References +---------- +Park, H. S., A. V. Ryzhkov, D. S. Zrnic, and K.-E. Kim, 2009: The hydrometeor +classification algorithm for the polarimetric WSR-88D. *Wea. Forecasting*, 24, +730-748, doi:10.1175/2008WAF2222205.1. Source of the melting-layer hard +constraints. + +Ryzhkov, A., S. Y. Matrosov, V. Melnikov, D. Zrnic, P. Zhang, Q. Cao, M. Knight, +C. Simmer, and S. Troemel, 2017: Estimation of depolarization ratio using +weather radars with simultaneous transmission/reception. *J. Appl. Meteor. +Climatol.*, 56, 1797-1816, doi:10.1175/JAMC-D-16-0098.1. Source of +:func:`depolarization_ratio`. + +Zrnic, D. S., A. V. Ryzhkov, J. M. Straka, Y. Liu, and J. Vivekanandan, 2001: +Testing a procedure for automatic classification of hydrometeor types. +*J. Atmos. Oceanic Technol.*, 18, 892-913. Establishes that Z and ZDR combined +carry the strongest discriminating power. +""" + +import numpy as np + +# Integer codes are part of the file format; append, never renumber. +CLASSES = { + 0: "unclassified", + 1: "no_scatter", + 2: "clutter", + 3: "biological", + 4: "multi_trip", + 5: "light_rain", + 6: "moderate_rain", + 7: "heavy_rain", + 8: "melting_wet", + 9: "ice_snow", + 10: "graupel_hail", +} +NAME_TO_CODE = {v: k for k, v in CLASSES.items()} + +NON_MET = ("no_scatter", "clutter", "biological", "multi_trip") +LIQUID = ("light_rain", "moderate_rain", "heavy_rain") +FROZEN = ("ice_snow", "graupel_hail") + + +def trapmf(x, pts): + """Trapezoidal membership, matching ``skfuzzy.trapmf``. + + 1 on [b, c], linear ramps on [a, b) and (c, d], 0 elsewhere. NaN maps to 0 + so a missing field contributes no evidence rather than propagating NaN. + """ + a, b, c, d = (float(v) for v in pts) + x = np.asarray(x, dtype="f8") + y = np.zeros_like(x) + if b > a: + m = (x > a) & (x < b) + y[m] = (x[m] - a) / (b - a) + if d > c: + m = (x > c) & (x < d) + y[m] = (d - x[m]) / (d - c) + y[(x >= b) & (x <= c)] = 1.0 + y[~np.isfinite(x)] = 0.0 + return y + + +def depolarization_ratio(zdr_db, rhohv): + """Depolarization ratio (dB) from SHV moments -- Ryzhkov et al. 2017, Eq. 6. + + A radar transmitting horizontal and vertical polarizations simultaneously + cannot measure circular depolarization ratio directly, but + + DR = (1 + Zdr^-1 - 2 rho_hv Zdr^-1/2) + / (1 + Zdr^-1 + 2 rho_hv Zdr^-1/2) + + with ``Zdr`` in linear units is a proxy for CDR computable from the + standard moments. It behaves like LDR without needing a cross-polar + channel: it rises sharply for irregular, randomly oriented, wet or + tumbling scatterers. + + Reference: Ryzhkov, A., S. Y. Matrosov, V. Melnikov, D. Zrnic, P. Zhang, + Q. Cao, M. Knight, C. Simmer and S. Troemel, 2017: Estimation of + Depolarization Ratio Using Weather Radars with Simultaneous + Transmission/Reception. *J. Appl. Meteor. Climatol.*, 56, 1797-1816, + doi:10.1175/JAMC-D-16-0098.1. + + Measured on a BNF C-SAPR2 RHI, restricted to coherent gates with real + signal (median values): + + biology (ZDR > 3 dB, rho_hv < 0.9, T > 5 C) -5.3 dB + near-range low-velocity clutter -9.6 dB + precipitation (Z >= 25 dBZ, rho_hv > 0.97) -19.6 dB + anvil ice (T < -15 C, rho_hv > 0.97) -19.4 dB + + A ~14 dB separation between biology and hydrometeors, which is why this + is weighted heavily in the biological membership function. + """ + zdr_db = np.asarray(zdr_db, dtype="f8") + rho_in = np.asarray(rhohv, dtype="f8") + rho = np.clip(rho_in, 0.0, 1.0) + zl = np.power(10.0, zdr_db / 10.0) + with np.errstate(divide="ignore", invalid="ignore"): + inv = 1.0 / zl + s = np.sqrt(inv) + num = 1.0 + inv - 2.0 * rho * s + den = 1.0 + inv + 2.0 * rho * s + out = 10.0 * np.log10(np.clip(num / den, 1e-12, None)) + # np.where rather than boolean assignment, so scalar inputs work too. + return np.where(np.isfinite(zdr_db) & np.isfinite(rho_in), out, np.nan) + + +def texture(field, window=4): + """NaN-aware standard-deviation texture over a ``window`` square footprint. + + Py-ART offers :func:`pyart.util.texture` and + :func:`pyart.util.texture_along_ray`, and neither fits here: both take a + :class:`~pyart.core.Radar` rather than an array, both compute a 1-D + standard deviation along the ray rather than over a 2-D footprint, and + ``pyart.util.texture`` hardcodes an 11-point window and prints to stdout. + This classifier needs a 2-D footprint on a bare array, sized to match the + velocity texture it is compared against. + + The NaN handling is the substantive difference. Gates with fewer than three + valid neighbours return NaN rather than a value computed from almost + nothing, so an isolated sample cannot manufacture texture. + """ + from scipy import ndimage + + data = np.asarray(field, dtype="f8") + valid = np.isfinite(data).astype("f8") + filled = np.where(np.isfinite(data), data, 0.0) + n = ndimage.uniform_filter(valid, size=window, mode="nearest") * window**2 + s1 = ndimage.uniform_filter(filled, size=window, mode="nearest") * window**2 + s2 = ndimage.uniform_filter(filled**2, size=window, mode="nearest") * window**2 + with np.errstate(invalid="ignore", divide="ignore"): + var = s2 / n - (s1 / n) ** 2 + out = np.sqrt(np.clip(var, 0.0, None)) + out[n < 3] = np.nan + return out + + +def angular_texture(vel, nyquist, window=4): + """Folding-aware velocity texture, delegating to Py-ART. + + A plain standard deviation across the +/-Nyquist branch cut reports a huge + spread for perfectly coherent flow, so the statistic must be computed on + the unit circle. :func:`pyart.util.angular_texture_2d` does exactly that, + and is used here rather than reimplemented: on a real BNF sweep the two + agree to a median absolute difference of 7e-15. + + The only thing added is NaN propagation. Py-ART convolves with + ``boundary="symm"``, which lets a NaN spread through the whole window and, + on the same sweep, leaves 2% of gates finite that should not be. Here a + gate is NaN if it has fewer than three valid neighbours, matching + :func:`texture` so the two are comparable. + """ + import pyart + + v = np.asarray(vel, dtype="f8") + if not np.isfinite(nyquist) or nyquist <= 0: + return texture(v, window) + + size = (window, window) if isinstance(window, int) else tuple(window) + with np.errstate(invalid="ignore", divide="ignore"): + out = np.asarray( + pyart.util.angular_texture_2d( + np.nan_to_num(v, nan=0.0), size, float(nyquist) + ), + dtype="f8", + ) + + from scipy import ndimage + + valid = np.isfinite(v).astype("f8") + n = ndimage.uniform_filter(valid, size=size, mode="nearest") * float(np.prod(size)) + out[n < 3] = np.nan + out[~np.isfinite(v)] = np.nan + return out + + +# Membership functions over uncorrected moments and derived textures. +# key -> [[a, b, c, d], weight] +MEMBERSHIP = { + "no_scatter": { + "snr": [[-100, -100, 2, 5], 3.0], + "z": [[-100, -100, -8, -2], 2.0], + "rhohv": [[0, 0, 0.5, 0.7], 1.0], + # Receiver noise is phase-incoherent for the same reason second trip + # is, so it also sits at the uniform-phase texture limit. What + # separates the two is returned power, handled by the SNR terms. + "vel_texture_norm": [[0.22, 0.40, 1.2, 1.2], 2.0], + }, + "clutter": { + "dr": [[-16.0, -12.0, 0.0, 0.0], 1.5], + "vel_norm": [[0, 0, 0.03, 0.08], 3.0], + "sw": [[0, 0, 1.0, 1.8], 1.5], + "rhohv": [[0.2, 0.4, 0.85, 0.92], 1.5], + "range_km": [[0, 0, 12.0, 25.0], 1.5], + "rhohv_texture": [[0.03, 0.08, 1.0, 1.0], 1.0], + }, + "biological": { + # Depolarization ratio (Ryzhkov et al. 2017) separates biology from + # hydrometeors by ~14 dB on BNF data -- a wider margin than any single + # standard moment -- so it carries the most weight here. + "dr": [[-14.0, -10.0, 0.0, 0.0], 3.5], + "zdr": [[2.0, 3.5, 12.0, 15.0], 3.0], + "rhohv": [[0.3, 0.5, 0.85, 0.93], 2.0], + "rhohv_texture": [[0.03, 0.08, 1.0, 1.0], 2.5], + "z": [[-15, -8, 18, 25], 2.0], + "sw": [[1.0, 1.8, 6.0, 8.0], 1.0], + "phidp_texture": [[20, 40, 200, 200], 1.5], + "zdr_texture": [[0.5, 1.0, 6.0, 8.0], 1.5], + "z_texture": [[1.5, 3.0, 20.0, 25.0], 1.0], + # Insects and birds do not fly in subfreezing air. This term plus the + # hard constraint below keeps biology out of the cold cloud, where its + # noisy-rhohv/high-ZDR signature would otherwise mimic ice. + "temp": [[3.0, 6.0, 100, 100], 2.5], + }, + "multi_trip": { + # Second-trip echo from a magnetron radar is *incoherent*: the + # transmitter's starting phase is random pulse to pulse, so a target + # beyond the unambiguous range is phase-scrambled and its apparent + # radial velocity is uniformly distributed over +/-Nyquist. The + # standard deviation of a uniform distribution on that interval is + # v_nyq/sqrt(3) -- 9.4 m/s at BNF's 16.3 m/s Nyquist -- and the + # measured noise/second-trip population sits exactly there (median + # 8.96 m/s). First-trip weather is phase-coherent and measures + # 0.74 m/s. That is a 12x separation and the single most reliable + # discriminant available, so it dominates this class. + # + # The membership therefore ramps in from ~40% of the uniform-phase + # limit and saturates at it. ``vel_texture_norm`` (texture divided by + # Nyquist) is used where available so the same numbers transfer to a + # radar with a different PRF; plain ``vel_texture`` is the fallback. + "vel_texture_norm": [[0.22, 0.40, 1.2, 1.2], 4.0], + # Real returned power is the ONLY thing separating second trip from + # receiver noise -- both are phase-incoherent and sit at the same + # texture limit. The ramp starts above the noise floor and this term + # carries weight comparable to the coherence term, so a gate without + # power cannot be called second trip. + "snr": [[3.0, 8.0, 1000, 1000], 3.5], + "ncp": [[0, 0, 0.5, 0.65], 1.5], + "rhohv": [[0, 0, 0.7, 0.85], 1.5], + "rhohv_texture": [[0.05, 0.10, 1.0, 1.0], 1.0], + "phidp_texture": [[40, 60, 200, 200], 1.0], + }, + "light_rain": { + "dr": [[-60.0, -60.0, -14.0, -10.0], 1.0], + "z": [[0, 5, 25, 32], 2.0], + "rhohv": [[0.95, 0.97, 1, 1], 2.0], + "rhohv_texture": [[0, 0, 0.02, 0.05], 1.5], + "zdr": [[-0.5, -0.2, 1.5, 2.5], 1.0], + "temp": [[0.0, 2.0, 100, 100], 2.0], + }, + "moderate_rain": { + "dr": [[-60.0, -60.0, -14.0, -10.0], 1.0], + "z": [[25, 30, 45, 50], 2.0], + "rhohv": [[0.95, 0.97, 1, 1], 2.0], + "rhohv_texture": [[0, 0, 0.02, 0.05], 1.5], + "zdr": [[0.2, 0.5, 3.0, 4.0], 1.0], + "temp": [[0.0, 2.0, 100, 100], 2.0], + }, + "heavy_rain": { + "dr": [[-60.0, -60.0, -14.0, -10.0], 1.0], + "z": [[42, 47, 70, 70], 2.5], + "rhohv": [[0.93, 0.96, 1, 1], 2.0], + "zdr": [[0.5, 1.0, 4.5, 6.0], 1.0], + "temp": [[0.0, 2.0, 100, 100], 2.0], + }, + "melting_wet": { + # The melting layer is thin and sits just below the 0 C level. Wet + # growth does not begin until a particle is already descending through + # near-freezing air, so the temperature window is narrow and centred + # slightly warm of 0 C. A wide window let this class claim the whole + # anvil, which is physically wrong: at -20 C nothing is melting. + "rhohv": [[0.80, 0.85, 0.95, 0.97], 3.0], + "zdr": [[0.5, 1.0, 4.0, 5.0], 1.5], + "z": [[20, 25, 50, 55], 1.0], + "temp": [[-2.0, -0.5, 2.5, 4.0], 4.0], + # Melting particles are wet and irregular: depressed rhohv WITH low + # texture. High rhohv texture means noise or biology, not a bright band. + "rhohv_texture": [[0, 0, 0.03, 0.06], 1.5], + }, + "ice_snow": { + "dr": [[-60.0, -60.0, -14.0, -10.0], 1.0], + "rhohv": [[0.93, 0.96, 1, 1], 1.5], + "zdr": [[-0.5, -0.2, 1.0, 1.8], 1.0], + "z": [[-5, 0, 35, 42], 1.0], + # Everything colder than about -1 C is ice. This is the anvil: a broad, + # confident plateau rather than a narrow window, so ice wins there + # instead of melting or hail. + "temp": [[-100, -100, -1.0, 1.0], 4.0], + "rhohv_texture": [[0, 0, 0.03, 0.06], 1.0], + }, + "graupel_hail": { + # Hail and graupel require a genuinely strong core. Without the Z floor + # this class leaked into the weak-echo anvil. + "z": [[45, 50, 70, 70], 3.0], + "zdr": [[-1.0, -0.5, 1.0, 1.5], 1.5], + "rhohv": [[0.92, 0.95, 1, 1], 1.0], + "temp": [[-100, -100, 5.0, 10.0], 1.0], + }, +} + +#: SNR above which an incoherent gate is treated as a real distant target +#: rather than receiver noise, and so is left for the classifier to label as +#: second trip instead of being gated out as no_scatter. +TRIP_SNR_MIN = 8.0 + + +# [class, feature, (lo, hi)] -> zero the score inside the interval. +HARD_CONSTRAINTS = [ + # Park et al. 2009: snow is not permitted below the melting layer, and + # rain is not permitted above its top. Big drops are the documented + # exception, lofted above the freezing level in convective updraughts; + # heavy_rain is therefore left unconstrained aloft. + # + # NOTE: an earlier revision wrote this constraint as + # ["ice_snow", "hoi0_km", (0.5, 1e6)], which forbade ice ABOVE the + # freezing level -- the exact opposite of the intent, and the reason the + # storm anvil was coming out as melting or hail. Ice is forbidden well + # BELOW the melting layer, where it has had time to melt. + ["ice_snow", "hoi0_km", (-1e6, -1.5)], + ["light_rain", "hoi0_km", (1.0, 1e6)], + ["moderate_rain", "hoi0_km", (1.5, 1e6)], + # Melting is a narrow band around the 0 C level. Nothing melts at -5 C, and + # nothing is still melting several degrees into the warm sector. + ["melting_wet", "temp", (5.0, 1e6)], + ["melting_wet", "temp", (-1e6, -5.0)], + ["melting_wet", "hoi0_km", (-1e6, -2.5)], + ["melting_wet", "hoi0_km", (1.5, 1e6)], + # Biology cannot exist in subfreezing air. Insects and birds are confined + # to the warm boundary layer; without this the class captured anvil ice, + # whose noisy rhohv and positive ZDR look superficially similar. + ["biological", "temp", (-1e6, 3.0)], + ["biological", "z", (30.0, 1e6)], + ["biological", "height_km", (6.0, 1e6)], + # Hail requires a strong core; it must not claim weak anvil echo. + ["graupel_hail", "z", (-1e6, 40.0)], + ["clutter", "z", (45.0, 1e6)], + ["clutter", "height_km", (4.0, 1e6)], + ["multi_trip", "z", (35.0, 1e6)], + # Geometry, not tuning: where second trip is impossible the class is + # forbidden outright. ``trip_possible`` is 1 where the folded beam could + # still be in the troposphere and 0 where it could not, so the constraint + # zeroes multi_trip on the impossible half. See + # :mod:`radar_palette.gateid.multitrip`. + ["multi_trip", "trip_possible", (-1e6, 0.5)], +] + + +#: Velocity texture, as a fraction of the uniform-random-phase limit +#: v_nyq/sqrt(3), above which a gate is treated as phase-incoherent. Measured +#: on BNF RHIs: receiver noise sits at 0.95 (median 8.96 of 9.41 m/s) while +#: precipitation sits at 0.08 (0.74 m/s). A cut at 0.32 rejected 100% of noise +#: while keeping 99.9% of gates with Z >= 25 dBZ and rho_hv > 0.95. +INCOHERENT_TEXTURE_FRAC = 0.32 + + +def noise_floor_mask( + features, + snr_min=3.0, + snr_taper=1.0, + vel_texture_max=None, + ncp_min=None, + incoherent_frac=INCOHERENT_TEXTURE_FRAC, + keep_dbz=None, + trip_snr_min=TRIP_SNR_MIN, +): + """Gates that are not a coherent measurement of a scatterer. + + Py-ART's noise helpers solve a different problem: + :func:`pyart.retrieve.calculate_snr_from_reflectivity` and + :func:`pyart.retrieve.compute_snr` *derive* SNR, and + :func:`pyart.correct.calc_noise_floor` estimates a receiver noise level + from the data. This consumes an SNR that already exists and decides + membership, so it composes with those rather than replacing them: if a + volume lacks an SNR field, compute one with Py-ART and pass it in. + + Speckle outside the storm is mostly receiver noise that happens to land in + some class's membership region. Rather than let the fuzzy scores arbitrate + that, such gates are forced to ``no_scatter`` up front. + + Two independent tests, because they fail on different things: + + ``snr_min`` + Power-based. Rejects gates below the minimum detectable signal. + ``incoherent_frac`` + Phase-based, and the stronger of the two. A coherent target produces a + smooth radial-velocity field; receiver noise -- and, on a magnetron + radar, second-trip echo -- has random phase, so its velocity is + uniformly distributed over +/-Nyquist and its texture approaches + v_nyq/sqrt(3). Gates above ``incoherent_frac`` of that limit are not + first-trip weather. This is what removes speckle that passes the SNR + test, which the power-based cut alone cannot do. + + ``keep_dbz`` protects strong echo from both tests, so a bright core is + never blanked by a texture artefact at its edge. + """ + n = len(next(iter(features.values()))) + mask = np.zeros(n, dtype=bool) + + if snr_min is not None and "snr" in features and features["snr"] is not None: + snr = np.asarray(features["snr"], dtype="f8") + mask |= np.isfinite(snr) & (snr < snr_min - snr_taper) + if ncp_min is not None and "ncp" in features: + ncp = np.asarray(features["ncp"], dtype="f8") + mask |= np.isfinite(ncp) & (ncp < ncp_min) + + # Explicit absolute threshold in m/s, if the caller supplied one. + if vel_texture_max is not None and "vel_texture" in features: + vt = np.asarray(features["vel_texture"], dtype="f8") + mask |= np.isfinite(vt) & (vt > vel_texture_max) + + # Normalised phase-coherence test, PRF-independent. + # + # Incoherence alone does NOT mean no scatterer. Receiver noise and + # second-trip echo are both phase-incoherent -- that is the whole reason + # the coherence test works -- and the only thing separating them is + # returned power. Gating on incoherence without excepting powered gates + # made multi_trip unreachable: it scored 1.0 and was then overwritten with + # no_scatter before the label was ever emitted. + if incoherent_frac is not None and "vel_texture_norm" in features: + vtn = np.asarray(features["vel_texture_norm"], dtype="f8") + incoherent = np.isfinite(vtn) & (vtn > incoherent_frac) + if trip_snr_min is not None and "snr" in features: + powered = np.asarray(features["snr"], dtype="f8") + incoherent &= ~(np.isfinite(powered) & (powered >= trip_snr_min)) + mask |= incoherent + + if keep_dbz is not None and "z" in features: + zz = np.asarray(features["z"], dtype="f8") + mask &= ~(np.isfinite(zz) & (zz >= keep_dbz)) + return mask + + +def despeckle(codes, shape, min_run=3, axis=1, protect=(), keep_mask=None): + """Remove runs shorter than ``min_run`` along a ray. + + Not a substitute for :func:`pyart.correct.despeckle_field`, and not a + reimplementation of it. That function thresholds a *continuous* field, + finds 2-D connected objects and returns a + :class:`~pyart.filters.GateFilter` marking small ones for exclusion. This + operates on the *categorical* class field after classification, dissolving + short runs of one label surrounded by another, and returns labels rather + than a mask. Both are useful; they answer different questions, and the + Py-ART one is the right tool for speckle in reflectivity. + + An isolated one- or two-gate island of some class inside a different + background is not a resolvable scatterer -- the pulse volume is larger than + that -- so it is noise or a classifier artefact. Runs shorter than + ``min_run`` are replaced by the neighbouring label where the neighbours + agree, and by ``no_scatter`` otherwise. + + ``keep_mask`` protects gates that must never be dissolved regardless of run + length. This matters: a two-gate 50 dBZ core at the edge of a cell is a real + scatterer, and blanket despeckling removed 20% of gates above 30 dBZ on a + BNF RHI before this guard existed. Callers should pass a reflectivity + threshold mask. + + Operating along the range axis is deliberate: it is the axis with uniform, + fine sampling in both PPI and RHI geometry, whereas the across-beam axis is + azimuth in one and elevation in the other. + """ + if min_run <= 1: + return codes + grid = codes.reshape(shape).copy() + keep = ( + np.zeros(shape, dtype=bool) + if keep_mask is None + else np.asarray(keep_mask, dtype=bool).reshape(shape) + ) + if axis == 0: + grid, keep = grid.T, keep.T + protect = {NAME_TO_CODE.get(p, p) if isinstance(p, str) else p for p in protect} + + n_rays, n_gates = grid.shape + for i in range(n_rays): + row, krow = grid[i], keep[i] + start = 0 + for j in range(1, n_gates + 1): + if j < n_gates and row[j] == row[start]: + continue + run = j - start + val = row[start] + if ( + run < min_run + and val != 0 + and val not in protect + and not krow[start:j].any() + ): + left = row[start - 1] if start > 0 else None + right = row[j] if j < n_gates else None + if left is not None and right is not None and left == right: + row[start:j] = left + else: + row[start:j] = NAME_TO_CODE["no_scatter"] + start = j + if axis == 0: + grid = grid.T + return grid.ravel() + + +#: A class must have at least this fraction of its total weight backed by +#: features that are actually present, or it is not eligible. Budget +#: normalisation alone lets a class score a perfect 1.0 on two of five terms +#: and tie a class that satisfied all of its own -- so a gate with no range or +#: spectrum width could be called ``clutter`` on rho_hv alone. +MIN_EVIDENCE_COVERAGE = 0.5 + + +def _budget(terms, available): + b = sum(w for k, (_, w) in terms.items() if k in available) + return b if b > 0 else 1.0 + + +def _coverage(terms, available): + """Fraction of a class's total weight that its available features carry.""" + total = sum(w for _, (_, w) in terms.items()) + if total <= 0: + return 1.0 + return sum(w for k, (_, w) in terms.items() if k in available) / total + + +def classify( + features, + membership=None, + hard_constraints=None, + classes=None, + return_scores=False, + shape=None, + snr_min=3.0, + vel_texture_max=None, + min_run=3, + despeckle_keep_dbz=30.0, + incoherent_frac=INCOHERENT_TEXTURE_FRAC, + min_coverage=MIN_EVIDENCE_COVERAGE, + score_adjusters=(), +): + """Classify gates from a dict of 1-D feature arrays. + + Returns ``(codes, info)``. ``codes`` are the integer codes of + :data:`CLASSES`; ``info`` carries the normalised score matrix, the winning + score, the margin over the runner-up and the class order. + + Three speckle controls, applied in this order: + + ``snr_min`` + Gates below the minimum detectable signal become ``no_scatter`` + before the fuzzy scores are consulted. Set to ``None`` to disable. + ``vel_texture_max`` + Absolute texture cut in m/s, if you want one. Normally leave this None + and use ``incoherent_frac`` instead, which is PRF-independent. + ``incoherent_frac`` + Velocity texture as a fraction of the uniform-random-phase limit + v_nyq/sqrt(3), above which a gate is phase-incoherent and cannot be + first-trip weather. This is the main speckle control; the SNR floor + alone leaves incoherent gates that happen to carry power. + ``min_run`` + After classification, runs shorter than this along the ray are + dissolved. Requires ``shape``. Set to 1 to disable. + ``despeckle_keep_dbz`` + Gates at or above this reflectivity are never dissolved, whatever the + run length. Without it, ``min_run=3`` removed 20% of gates above + 30 dBZ on a BNF RHI. + """ + membership = MEMBERSHIP if membership is None else membership + hard_constraints = ( + HARD_CONSTRAINTS if hard_constraints is None else hard_constraints + ) + order = ( + list(classes) + if classes + else [c for c in CLASSES.values() if c != "unclassified"] + ) + order = [c for c in order if c in membership] + + available = {k for k, v in features.items() if v is not None} + n = len(next(iter(features.values()))) + scores = np.zeros((n, len(order)), dtype="f8") + + covered = [] + for j, cls in enumerate(order): + terms = membership[cls] + # A class whose defining evidence is mostly absent is not eligible. + # Without this, budget normalisation lets it score 1.0 on whatever + # happens to be present and tie a fully-evidenced class. + if _coverage(terms, available) < min_coverage: + covered.append(False) + continue + covered.append(True) + s = np.zeros(n) + for key, (pts, w) in terms.items(): + if key not in available or w == 0: + continue + s += trapmf(features[key], pts) * w + # Normalise by attainable budget so a class cannot win merely by + # having more evidence terms defined for it. + scores[:, j] = s / _budget(terms, available) + + for cls, key, (lo, hi) in hard_constraints: + if cls not in order or key not in available: + continue + v = np.asarray(features[key], dtype="f8") + scores[(v >= lo) & (v <= hi), order.index(cls)] = 0.0 + + # Score adjustments (e.g. temporal priors) are applied HERE: after the + # hard constraints, so a prior can never resurrect a class that physics + # forbids, and before the argmax, so everything downstream -- the + # unclassified rule, the noise floor and despeckling -- applies to the + # adjusted answer exactly as it does to the instantaneous one. + n_adjusted = 0 + if score_adjusters: + part0 = np.partition(scores, -2, axis=1) if scores.shape[1] > 1 else scores + margin0 = scores.max(axis=1) - ( + part0[:, -2] if scores.shape[1] > 1 else np.zeros(n) + ) + before_scores = scores + for adjust in score_adjusters: + scores = adjust(scores, order, margin0) + # Re-apply the constraints: an adjuster is not permitted to lift a + # class the geometry or thermodynamics ruled out. + for cls, key, (lo, hi) in hard_constraints: + if cls not in order or key not in available: + continue + v = np.asarray(features[key], dtype="f8") + scores[(v >= lo) & (v <= hi), order.index(cls)] = 0.0 + n_adjusted = int((scores != before_scores).any(axis=1).sum()) + + best = scores.argmax(axis=1) + top = scores[np.arange(n), best] + part = np.partition(scores, -2, axis=1) if scores.shape[1] > 1 else scores + second = part[:, -2] if scores.shape[1] > 1 else np.zeros(n) + + codes = np.array([NAME_TO_CODE[order[i]] for i in best], dtype="i2") + codes[top <= 0] = 0 # explicit unclassified, never a silent tie-break + + n_noise = n_speck = 0 + if snr_min is not None or vel_texture_max is not None or incoherent_frac: + noise = noise_floor_mask( + features, + snr_min=snr_min, + vel_texture_max=vel_texture_max, + incoherent_frac=incoherent_frac, + keep_dbz=despeckle_keep_dbz, + ) + n_noise = int(noise.sum()) + codes[noise] = NAME_TO_CODE["no_scatter"] + + if min_run and min_run > 1 and shape is not None: + before = codes.copy() + # Never dissolve a strong core, however short the run: a two-gate + # 50 dBZ return at a cell edge is a real scatterer, not speckle. + keep = None + if despeckle_keep_dbz is not None and "z" in features: + zz = np.asarray(features["z"], dtype="f8") + keep = np.isfinite(zz) & (zz >= despeckle_keep_dbz) + codes = despeckle( + codes, shape, min_run=min_run, protect=("no_scatter",), keep_mask=keep + ) + n_speck = int((codes != before).sum()) + elif min_run and min_run > 1: + # Despeckling is a spatial operation and needs the sweep geometry. A + # bare feature dict has none, so skip it and say so rather than either + # failing or silently pretending it ran. + n_speck = -1 + + info = { + "scores": scores, + "order": order, + "top_score": top, + "margin": top - second, + "n_noise_floor": n_noise, + "n_despeckled": n_speck, + "n_score_adjusted": n_adjusted, + "ineligible": [c for c, ok in zip(order, covered, strict=True) if not ok], + } + return codes, info diff --git a/cmac/radar_palette/io/__init__.py b/cmac/radar_palette/io/__init__.py new file mode 100644 index 0000000..878284c --- /dev/null +++ b/cmac/radar_palette/io/__init__.py @@ -0,0 +1,49 @@ +"""Object-flavour interoperability for radar and grid data. + +Two object families describe the same radar data, and users arrive with either: + +*Py-ART* --- :class:`pyart.core.Radar` for volumes, :class:`pyart.core.Grid` for +Cartesian output. + +*xradar / xarray* --- :class:`xarray.DataTree` for volumes (one group per sweep), +:class:`xarray.Dataset` for Cartesian output. + +Operators in this package compute on the Py-ART surface, because that is what the +underlying geometry helpers expect. Rather than requiring callers to convert, the +entry points accept either family and return the family the caller expects. The +default is to *mirror the input*: a ``DataTree`` in gives a ``DataTree`` back, a +``Radar`` in gives a ``Radar`` back, and nobody is silently handed an object of a +different type than they passed. + +Conversions delegate to Py-ART's own interoperability layer +(:mod:`pyart.xradar`, :meth:`pyart.core.Grid.to_xarray`, and xradar's cfradial +readers) rather than reimplementing the mapping between the two data models. That +layer is maintained alongside the formats and handles metadata this package has +no business duplicating. +""" + +from .flavors import ( + GridFlavor, + RadarFlavor, + detect_grid_flavor, + detect_radar_flavor, + resolve_output_flavor, + to_grid_flavor, + to_pyart_grid, + to_pyart_radar, + to_radar_flavor, + write_ray_times_to_datatree, +) + +__all__ = [ + "GridFlavor", + "RadarFlavor", + "detect_grid_flavor", + "detect_radar_flavor", + "resolve_output_flavor", + "to_grid_flavor", + "to_pyart_grid", + "to_pyart_radar", + "to_radar_flavor", + "write_ray_times_to_datatree", +] diff --git a/cmac/radar_palette/io/flavors.py b/cmac/radar_palette/io/flavors.py new file mode 100644 index 0000000..7806e16 --- /dev/null +++ b/cmac/radar_palette/io/flavors.py @@ -0,0 +1,505 @@ +"""Detection of, and conversion between, radar/grid object flavours. + +The functions here are the single place this package decides "which object family +is this, and which should I hand back". Keeping that in one module means an +operator's own code stays free of isinstance ladders, and the conversion sharp +edges are documented and tested in one place rather than rediscovered per call +site. + +Conversion strategy +------------------- +Py-ART owns the mapping between its data model and xradar's, so conversions +delegate to it: + +- ``DataTree`` to a Py-ART surface: ``DataTree.pyart.to_radar()``, which returns a + :class:`pyart.xradar.Xradar` wrapper presenting the ``Radar`` API over the live + tree. Py-ART operators accept it directly. +- ``Radar`` to ``DataTree``: written as cfradial1 and reopened with + :func:`xradar.io.open_cfradial1_datatree`. This goes through a temporary file + because Py-ART exposes no in-memory equivalent; the file format is the + interchange contract, so the round trip is exactly what a user would get by + writing and reloading. +- ``Grid`` to ``Dataset``: :meth:`pyart.core.Grid.to_xarray`. +- ``Dataset`` to a Py-ART grid surface: :class:`pyart.xradar.Xgrid`. + +Sharp edges these functions absorb +---------------------------------- +``Xgrid`` rejects a dataset whose ``time`` has been decoded to datetimes, but +``Grid.to_xarray`` produces exactly that --- so the dataset is re-encoded to CF +offsets before wrapping. + +Assigning ``Xradar.time`` mutates only the wrapper: the change does not reach the +underlying ``DataTree``, so a caller who updates times through the Py-ART surface +and then reads the tree silently sees the old clock. Time is therefore written +back explicitly, per sweep, by :func:`write_ray_times_to_datatree`. (Field +assignment via ``add_field`` *does* write through --- the asymmetry is why this is +easy to get wrong.) +""" + +import tempfile +from enum import Enum +from pathlib import Path + +import numpy as np + +__all__ = [ + "GridFlavor", + "RadarFlavor", + "detect_grid_flavor", + "detect_radar_flavor", + "resolve_output_flavor", + "to_grid_flavor", + "to_pyart_grid", + "to_pyart_radar", + "to_radar_flavor", + "write_ray_times_to_datatree", +] + + +class RadarFlavor(Enum): + """Object family of a radar volume. + + Attributes + ---------- + PYART + :class:`pyart.core.Radar`. + XRADAR + :class:`xarray.DataTree` following the xradar/CfRadial2 layout, or the + :class:`pyart.xradar.Xradar` wrapper around one. + """ + + PYART = "pyart" + XRADAR = "xradar" + + +class GridFlavor(Enum): + """Object family of a Cartesian grid. + + Attributes + ---------- + PYART + :class:`pyart.core.Grid`. + XARRAY + :class:`xarray.Dataset`. + """ + + PYART = "pyart" + XARRAY = "xarray" + + +_GRID_FLAVOR_ALIASES = {"xradar": GridFlavor.XARRAY} + + +def _import_pyart(): + """Import Py-ART, raising a clear error if it is unavailable.""" + try: + import pyart + except ImportError as error: # pragma: no cover - pyart is a core dependency + raise ImportError( + "Py-ART is required for radar/grid conversions; install arm_pyart" + ) from error + return pyart + + +def _import_xradar(): + """Import xradar and xarray, raising a clear error if either is unavailable.""" + try: + import xarray + import xradar + except ImportError as error: # pragma: no cover - both are core dependencies + raise ImportError( + "xradar and xarray are required for DataTree conversions" + ) from error + return xradar, xarray + + +def detect_radar_flavor(radar_like): + """Identify which object family a radar volume belongs to. + + Parameters + ---------- + radar_like : object + A :class:`pyart.core.Radar`, an :class:`xarray.DataTree`, or a + :class:`pyart.xradar.Xradar` wrapper. + + Returns + ------- + RadarFlavor + The family ``radar_like`` belongs to. An ``Xradar`` wrapper reports + ``XRADAR``: it presents a Py-ART surface, but its home representation is + the tree it wraps, and that is what a caller passing one expects back. + + Raises + ------ + TypeError + If the object is not a recognised radar volume. + """ + pyart = _import_pyart() + _, xarray = _import_xradar() + + if isinstance(radar_like, pyart.xradar.Xradar): + return RadarFlavor.XRADAR + if isinstance(radar_like, xarray.DataTree): + return RadarFlavor.XRADAR + if isinstance(radar_like, pyart.core.Radar): + return RadarFlavor.PYART + raise TypeError( + "expected a Py-ART Radar, an xradar DataTree, or a pyart.xradar.Xradar " + f"wrapper, got {type(radar_like).__name__}" + ) + + +def detect_grid_flavor(grid_like): + """Identify which object family a Cartesian grid belongs to. + + Parameters + ---------- + grid_like : object + A :class:`pyart.core.Grid`, an :class:`xarray.Dataset`, or a + :class:`pyart.xradar.Xgrid` wrapper. + + Returns + ------- + GridFlavor + The family ``grid_like`` belongs to. + + Raises + ------ + TypeError + If the object is not a recognised Cartesian grid. + """ + pyart = _import_pyart() + _, xarray = _import_xradar() + + if isinstance(grid_like, xarray.Dataset): + return GridFlavor.XARRAY + if isinstance(grid_like, pyart.core.Grid | pyart.xradar.Xgrid): + return GridFlavor.PYART + raise TypeError( + f"expected a Py-ART Grid or an xarray Dataset, got {type(grid_like).__name__}" + ) + + +def resolve_output_flavor(requested, input_flavor): + """Decide the output flavour from an optional request and the input flavour. + + Parameters + ---------- + requested : RadarFlavor or GridFlavor or str or None + Explicitly requested output flavour. ``None`` mirrors the input, which is + the default behaviour of every entry point in this package. Strings are + matched case-insensitively; ``"xradar"`` is accepted as an alias for + ``GridFlavor.XARRAY`` when resolving a grid flavour, since a caller + working in xradar naturally names the whole ecosystem. + input_flavor : RadarFlavor or GridFlavor + Flavour of the object the caller supplied. Determines which enumeration + the result is drawn from. + + Returns + ------- + RadarFlavor or GridFlavor + The flavour to return to the caller, in the same family as + ``input_flavor``. + + Raises + ------ + ValueError + If ``requested`` names an unknown flavour, or belongs to a different + family than ``input_flavor`` (asking for a grid flavour on a radar + conversion is a programming error, not a coercion to perform). + """ + flavor_family = type(input_flavor) + if requested is None: + return input_flavor + if isinstance(requested, flavor_family): + return requested + if isinstance(requested, RadarFlavor | GridFlavor): + raise ValueError( + f"expected a {flavor_family.__name__}, got {type(requested).__name__}" + ) + + normalised = str(requested).strip().lower() + if flavor_family is GridFlavor and normalised in _GRID_FLAVOR_ALIASES: + return _GRID_FLAVOR_ALIASES[normalised] + for candidate in flavor_family: + if candidate.value == normalised: + return candidate + valid = ", ".join(sorted(member.value for member in flavor_family)) + raise ValueError(f"unknown output flavour {requested!r}; expected one of {valid}") + + +def to_pyart_radar(radar_like): + """Present any supported radar volume as a Py-ART ``Radar`` surface. + + Parameters + ---------- + radar_like : pyart.core.Radar or xarray.DataTree or pyart.xradar.Xradar + Volume to adapt. Not copied: the returned object reads through to the + caller's data. + + Returns + ------- + radar : pyart.core.Radar or pyart.xradar.Xradar + An object exposing the Py-ART ``Radar`` API. For xradar input this is an + ``Xradar`` wrapper over the live tree, which Py-ART operators accept + directly. + input_flavor : RadarFlavor + Flavour of the supplied object, for use with + :func:`resolve_output_flavor`. + + Notes + ----- + Updating times through the returned wrapper does **not** propagate to the + underlying ``DataTree``; use :func:`write_ray_times_to_datatree` for that. + """ + input_flavor = detect_radar_flavor(radar_like) + pyart = _import_pyart() + + if isinstance(radar_like, pyart.xradar.Xradar) or isinstance( + radar_like, pyart.core.Radar + ): + return radar_like, input_flavor + return radar_like.pyart.to_radar(), input_flavor + + +def _pyart_radar_to_datatree(radar): + """Convert a Py-ART ``Radar`` to an xradar ``DataTree`` via cfradial1. + + Py-ART exposes no in-memory ``Radar`` to ``DataTree`` conversion, so this + round-trips through a temporary cfradial1 file --- the same interchange path a + user gets by writing and reloading, which keeps the conversion honest about + what survives the format. + """ + pyart = _import_pyart() + xradar, _ = _import_xradar() + + with tempfile.TemporaryDirectory(dir=Path.cwd()) as staging_directory: + cfradial_path = str(Path(staging_directory) / "volume.nc") + pyart.io.write_cfradial(cfradial_path, radar) + datatree = xradar.io.open_cfradial1_datatree(cfradial_path) + return datatree.load() + + +def to_radar_flavor(radar, output_flavor): + """Return a radar volume in the requested object family. + + Parameters + ---------- + radar : pyart.core.Radar or pyart.xradar.Xradar or xarray.DataTree + Volume to convert. + output_flavor : RadarFlavor or str + Family to return. Strings are matched case-insensitively. + + Returns + ------- + pyart.core.Radar or xarray.DataTree + The volume in the requested family. + + Raises + ------ + ValueError + If ``output_flavor`` names an unknown flavour. + """ + pyart = _import_pyart() + output_flavor = resolve_output_flavor(output_flavor, detect_radar_flavor(radar)) + + if output_flavor is RadarFlavor.XRADAR: + if isinstance(radar, pyart.xradar.Xradar): + return radar.xradar + _, xarray = _import_xradar() + if isinstance(radar, xarray.DataTree): + return radar + return _pyart_radar_to_datatree(radar) + + if isinstance(radar, pyart.core.Radar): + return radar + return _datatree_to_pyart_radar(radar) + + +def _datatree_to_pyart_radar(radar_like): + """Convert xradar input to a true :class:`pyart.core.Radar` via cfradial1. + + ``DataTree.pyart.to_radar()`` yields an ``Xradar`` wrapper rather than a + ``Radar``. A caller who explicitly asks for the Py-ART flavour wants the real + class, so this writes cfradial2-compatible output and reads it back with + Py-ART. + """ + pyart = _import_pyart() + xradar, xarray = _import_xradar() + + # Unwrap an Xradar wrapper to the tree it holds. The isinstance check is + # deliberate: xradar registers an accessor named ``xradar`` on DataTree, so a + # hasattr("xradar") probe is true for a bare tree as well and would hand back + # the accessor instead of the data. + if isinstance(radar_like, pyart.xradar.Xradar): + datatree = radar_like.xradar + elif isinstance(radar_like, xarray.DataTree): + datatree = radar_like + else: # pragma: no cover - guarded by detect_radar_flavor + raise TypeError(f"cannot convert {type(radar_like).__name__} to a Radar") + + with tempfile.TemporaryDirectory(dir=Path.cwd()) as staging_directory: + cfradial_path = str(Path(staging_directory) / "volume.nc") + xradar.io.to_cfradial1(datatree.copy(), cfradial_path) + return pyart.io.read_cfradial(cfradial_path) + + +def write_ray_times_to_datatree(datatree, time_dict): + """Write per-ray times into a ``DataTree``, distributing them across sweeps. + + Assigning ``Xradar.time`` updates only the wrapper --- the underlying tree keeps + its original clock, so a caller who sets times through the Py-ART surface and + then reads the tree silently gets stale values. This writes them back for real. + + Parameters + ---------- + datatree : xarray.DataTree + Tree to update, in place. One group per sweep. + time_dict : dict + Py-ART style time dictionary: ``data`` holds per-ray offsets for the whole + volume, concatenated in sweep order, and ``units`` the CF reference-time + string. Offsets are decoded and stored as datetime64, matching how xradar + represents sweep time. + + Returns + ------- + xarray.DataTree + ``datatree``, to allow use as an expression. + + Raises + ------ + ValueError + If the number of offsets does not match the tree's total ray count. + """ + _, xarray = _import_xradar() + from netCDF4 import num2date + + sweep_group_names = sorted( + (name for name in datatree.groups if name.startswith("/sweep_")), + key=lambda name: int(name.rsplit("_", 1)[1]), + ) + rays_per_sweep = [ + datatree[name].ds.sizes[datatree[name].ds["time"].dims[0]] + for name in sweep_group_names + ] + total_rays = sum(rays_per_sweep) + + offsets = np.asarray(time_dict["data"], dtype="float64").ravel() + if offsets.size != total_rays: + raise ValueError( + f"got {offsets.size} ray times for a volume with {total_rays} rays" + ) + + absolute_times = np.asarray( + num2date( + offsets, + time_dict["units"], + calendar=time_dict.get("calendar", "gregorian"), + only_use_cftime_datetimes=False, + only_use_python_datetimes=True, + ), + dtype="datetime64[ns]", + ) + + first_ray_index = 0 + for group_name, ray_count in zip(sweep_group_names, rays_per_sweep, strict=True): + sweep_dataset = datatree[group_name].ds + time_dimension = sweep_dataset["time"].dims[0] + sweep_times = absolute_times[first_ray_index : first_ray_index + ray_count] + datatree[group_name].ds = sweep_dataset.assign_coords( + time=xarray.DataArray( + sweep_times, + dims=(time_dimension,), + attrs=dict(sweep_dataset["time"].attrs), + ) + ) + first_ray_index += ray_count + + return datatree + + +def to_pyart_grid(grid_like): + """Present any supported Cartesian grid as a Py-ART ``Grid`` surface. + + Parameters + ---------- + grid_like : pyart.core.Grid or xarray.Dataset + Grid to adapt. + + Returns + ------- + pyart.core.Grid or pyart.xradar.Xgrid + An object exposing the Py-ART ``Grid`` API. + + Notes + ----- + :class:`pyart.xradar.Xgrid` requires ``time`` to be *undecoded* (CF offsets + with a ``units`` attribute), while :meth:`pyart.core.Grid.to_xarray` produces + decoded datetimes. This re-encodes before wrapping, so datasets from either + origin work. + """ + pyart = _import_pyart() + if detect_grid_flavor(grid_like) is GridFlavor.PYART: + return grid_like + return pyart.xradar.Xgrid(_encode_dataset_time(grid_like)) + + +def _encode_dataset_time(dataset): + """Return ``dataset`` with ``time`` as CF offsets carrying a ``units`` attribute. + + ``Xgrid`` detects decoded times by the absence of ``units`` and refuses to + wrap such a dataset. Datasets already carrying ``units`` are returned + untouched. + """ + if "units" in dataset["time"].attrs: + return dataset + + _, xarray = _import_xradar() + time_values = np.atleast_1d(dataset["time"].values) + epoch = np.datetime64(time_values[0], "s") + offsets = ( + time_values.astype("datetime64[ns]") - epoch.astype("datetime64[ns]") + ) / np.timedelta64(1, "s") + units = f"seconds since {str(epoch).replace('T', ' ')}" + + encoded = dataset.assign_coords( + time=xarray.DataArray( + np.asarray(offsets, dtype="float64"), + dims=dataset["time"].dims, + attrs={**dataset["time"].attrs, "units": units}, + ) + ) + return encoded + + +def to_grid_flavor(grid, output_flavor): + """Return a Cartesian grid in the requested object family. + + Parameters + ---------- + grid : pyart.core.Grid or xarray.Dataset + Grid to convert. + output_flavor : GridFlavor or str + Family to return. ``"xradar"`` is accepted as an alias for ``"xarray"``. + + Returns + ------- + pyart.core.Grid or xarray.Dataset + The grid in the requested family. + + Raises + ------ + ValueError + If ``output_flavor`` names an unknown flavour. + """ + _, xarray = _import_xradar() + output_flavor = resolve_output_flavor(output_flavor, detect_grid_flavor(grid)) + + if output_flavor is GridFlavor.XARRAY: + if isinstance(grid, xarray.Dataset): + return grid + return grid.to_xarray() + + if isinstance(grid, xarray.Dataset): + return to_pyart_grid(grid) + return grid diff --git a/cmac/tests/test_gate_id_backends.py b/cmac/tests/test_gate_id_backends.py index 7ab4e37..4a5de70 100644 --- a/cmac/tests/test_gate_id_backends.py +++ b/cmac/tests/test_gate_id_backends.py @@ -1,9 +1,8 @@ """Unit tests for the alternative gate-ID backends. These build their radars with ``pyart.testing`` rather than downloading a -volume, so the whole module runs offline. The tests that need the classifier -itself skip when ``radar_palette`` is not installed, since CMAC does not -depend on it. +volume, so the whole module runs offline. The classifier is included in CMAC +and therefore does not require a separately installed ``radar_palette``. """ import sys @@ -352,11 +351,12 @@ def test_sounding_freezing_level_ignores_the_fill_value(radar): assert sounding_freezing_level(radar) is None -def test_missing_radar_palette_gives_an_actionable_error(monkeypatch): - """Selecting the backend without the package must explain the fix.""" +def test_included_radar_palette_is_independent_of_external_package(monkeypatch): + """The backend does not depend on a separately installed package.""" monkeypatch.setitem(sys.modules, 'radar_palette.gateid', None) - with pytest.raises(ImportError, match="gate_id_method='cmac_fuzzy'"): - _import_radar_palette() + entry_points, single = _import_radar_palette() + assert callable(entry_points.gate_id) + assert single.NAME_TO_CODE['no_scatter'] == 1 @pytest.mark.skipif(not radar_palette_available(), diff --git a/documents/cmac_config_reference.md b/documents/cmac_config_reference.md index cd61242..a307ede 100644 --- a/documents/cmac_config_reference.md +++ b/documents/cmac_config_reference.md @@ -324,17 +324,9 @@ a gap in the positional `notes` contract or duplicate a label. Both classes are excluded from meteorological gates either way, and both remain readable in `scatterer_classification`. -`radar_palette` is not a CMAC dependency — it is imported only when a -configuration selects this backend. The classifier lives in the package's -`gateid` module, which is not yet in radar-palette's released main branch; -install it from the development branch: - -``` -pip install 'radar-palette @ git+https://github.com/scollis/radar-palette@gateid' -``` - -Selecting the backend without it installed raises an `ImportError` naming -both that install and the `cmac_fuzzy` fallback. +The gate-ID implementation from radar-palette is included in CMAC, so this +backend does not require a separate radar-palette installation. The `gateid` +classifier and its Py-ART adapter live in the `cmac.radar_palette` package. Two behaviours of this backend are CMAC's own and worth knowing about: