diff --git a/examples/plot_denver_hail_terrain.py b/examples/plot_denver_hail_terrain.py new file mode 100644 index 00000000..fa977a5e --- /dev/null +++ b/examples/plot_denver_hail_terrain.py @@ -0,0 +1,122 @@ +""" +Denver hail case: flat versus DEM immersed-boundary retrieval +-------------------------------------------------------------- + +This example shows the complete terrain-boundary workflow used for the +18 June 2018 Denver hail case. The selected volume scans are close to +23:30 UTC: + +* KFTG: ``KFTG20180618_233053_V06`` +* KCYS: ``KCYS20180618_233041_V06`` + +The Level-II files are available from the public Unidata NEXRAD archive. +The DEM is an SRTM v3 (approximately EGM96 orthometric) mosaic. The large +input files are intentionally not shipped with PyDDA. Run the case-data +preparation workflow first, or set ``CASE_DIR`` to a directory containing the +two Py-ART grids and the DEM NetCDF file. + +The example compares the default flat lower boundary with the DEM-driven +immersed-boundary condition. On sloping terrain, the physical condition is +``u dot n = 0``; ``w = 0`` is the special case for a horizontal surface. +""" + +# sphinx_gallery_thumbnail_number = 1 + +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import pydda +import xarray as xr + + +# The case-data directory is created by the reproducible workflow scripts in +# scripts/run_denver_hail_20180618_2330.py. Change this path when running the +# example from another checkout. +CASE_DIR = Path("data/denver_hail_20180618_2330") +GRID_DIR = CASE_DIR / "grid" +TERRAIN_DIR = CASE_DIR / "retrieval_terrain" +DEM_PATH = Path("data/denver_hail_20180619/dem/denver_srtm_3arcsec.nc") +RUN_RETRIEVAL = False + + +def retrieve(grids, terrain=None): + """Run a default PyDDA retrieval, optionally with terrain IBM enabled.""" + initialized = [ + pydda.initialization.make_constant_wind_field( + grid, (0.0, 0.0, 0.0), vel_field="velocity" + ) + for grid in grids + ] + kwargs = {"vel_name": "velocity", "engine": "scipy"} + if terrain is not None: + kwargs.update(Cterrain=1.0, terrain=terrain) + return pydda.retrieval.get_dd_wind_field(initialized, **kwargs)[0] + + +def plot_3km_difference(flat, terrain, terrain_geometry): + """Plot terrain-aware minus flat winds at 3 km.""" + flat_u = flat["u"].values.squeeze() + flat_v = flat["v"].values.squeeze() + flat_w = flat["w"].values.squeeze() + terrain_u = terrain["u"].values.squeeze() + terrain_v = terrain["v"].values.squeeze() + terrain_w = terrain["w"].values.squeeze() + + delta_w = terrain_w[6] - flat_w[6] + delta_speed = np.hypot(terrain_u[6], terrain_v[6]) - np.hypot(flat_u[6], flat_v[6]) + x = flat.x.values / 1000.0 + y = flat.y.values / 1000.0 + + fig, axes = plt.subplots(1, 2, figsize=(14, 6), constrained_layout=True) + for ax, field, title, label in ( + (axes[0], delta_w, r"$\Delta w$ [m/s]", "Updraft difference"), + ( + axes[1], + delta_speed, + r"$\Delta |V|$ [m/s]", + "Horizontal wind-speed difference", + ), + ): + limit = np.nanpercentile(np.abs(field), 98) + mesh = ax.pcolormesh( + x, y, field, cmap="RdBu_r", vmin=-limit, vmax=limit, shading="auto" + ) + ax.contour(x, y, field, levels=[0.0], colors="k", linewidths=0.7) + ax.set(xlim=(-75, 75), ylim=(-75, 75), xlabel="X [km]", ylabel="Y [km]") + ax.set_title(title) + fig.colorbar(mesh, ax=ax, label=label) + fig.suptitle("Denver hail case: terrain-aware minus flat retrieval at 3 km") + return fig + + +if GRID_DIR.joinpath("KFTG_grid.nc").exists() and DEM_PATH.exists(): + radar_grids = [ + pydda.io.read_grid(GRID_DIR / "KFTG_grid.nc"), + pydda.io.read_grid(GRID_DIR / "KCYS_grid.nc"), + ] + dem = xr.open_dataarray(DEM_PATH) + terrain_geometry = pydda.terrain.terrain_ghost_cells(radar_grids[0], dem) + + if RUN_RETRIEVAL: + flat_grids = retrieve(radar_grids) + terrain_grids = retrieve(radar_grids, terrain=terrain_geometry) + elif TERRAIN_DIR.joinpath("KFTG_denver_hail_20180618_2330_retrieval.nc").exists(): + # Use precomputed fields for a fast gallery build. Set RUN_RETRIEVAL=True + # above to execute both solves from the gridded radar data. + flat_grids = pydda.io.read_grid( + CASE_DIR / "retrieval/KFTG_denver_hail_20180618_2330_retrieval.nc" + ) + terrain_grids = pydda.io.read_grid( + TERRAIN_DIR / "KFTG_denver_hail_20180618_2330_retrieval.nc" + ) + else: + flat_grids = terrain_grids = None + + if flat_grids is not None: + plot_3km_difference(flat_grids, terrain_grids, terrain_geometry) +else: + print( + "Denver case data not found; prepare the case data first, then rerun " + "this example." + ) diff --git a/pydda/__init__.py b/pydda/__init__.py index 0e6784fe..34955719 100644 --- a/pydda/__init__.py +++ b/pydda/__init__.py @@ -11,6 +11,7 @@ from . import tests from . import constraints from . import io +from . import terrain __version__ = "2.5.0" diff --git a/pydda/terrain.py b/pydda/terrain.py new file mode 100644 index 00000000..94c6bedf --- /dev/null +++ b/pydda/terrain.py @@ -0,0 +1,341 @@ +""" +======================================== +pydda.terrain (pydda.terrain) +======================================== + +.. currentmodule:: pydda.terrain + +Terrain handling for wind retrievals in complex terrain. + +PyDDA's default lower boundary condition is flat impermeability: ``w = 0`` on +the bottom plane of the analysis domain. Over terrain the correct statement is +no-penetration through the terrain surface, + +.. math:: + + \\vec{u} \\cdot \\hat{n} = 0, \\qquad + \\hat{n} = \\frac{(-h_x, -h_y, 1)}{\\sqrt{1 + h_x^2 + h_y^2}}, + +which is what makes air climb a ridge. This module builds the geometry needed +to impose that condition with a ghost-cell immersed boundary method (IBM) from +a digital elevation model, for use with the ``Cterrain`` term of +:py:func:`pydda.retrieval.get_dd_wind_field`. + +Vertical datum +-------------- + +Analysis heights in PyDDA are **orthometric** (above mean sea level): +``point_altitude = origin_altitude + point_z``, and ``origin_altitude`` comes +from the radar metadata, which for NEXRAD, ARM and NASA ground radars is +antenna height above MSL. + +The DEM must therefore also be orthometric. SRTM v3 (EGM96), ASTER GDEM +(EGM96), Copernicus GLO-30 (EGM2008) and USGS 3DEP (NAVD88) are all safe at +PyDDA's vertical grid spacing. An **ellipsoidal** DEM (WGS84/ITRF heights, as +in some TanDEM-X and ICESat-2 products) is not: the geoid-ellipsoid separation +runs from about -107 m to +85 m globally, and applies as a systematic offset to +every column. Convert to orthometric heights first. + +:py:func:`check_datum` turns this into a diagnostic rather than an assumption. + +.. autosummary:: + :toctree: generated/ + + check_datum + terrain_ghost_cells + +""" + +import numpy as np +import pyart + +from scipy.interpolate import RegularGridInterpolator +from scipy.ndimage import gaussian_filter + + +def check_datum(Grid, dem, tower_max=50.0, verbose=True): + """ + Verify that a DEM and the radar altitudes share a vertical datum. + + Samples the DEM at each radar position in *Grid* and compares it with the + reported radar altitude. A consistent pair leaves a small **positive** + residual, which is the antenna tower height. A large residual of either + sign means the DEM is on a different vertical datum -- most often + ellipsoidal rather than orthometric -- and must be converted before it is + used to build a terrain boundary condition. + + Parameters + ---------- + Grid: xarray.Dataset + A PyDDA grid, as returned by :func:`pydda.io.read_grid`. + dem: xarray.DataArray + Elevation in m, with 1D "lat" and "lon" coordinates. + tower_max: float + Residuals larger than this in magnitude, in m, are reported as a + likely datum mismatch. + verbose: bool + Print a line per radar. + + Returns + ------- + residuals: 1D float array + ``radar_altitude - dem_elevation`` at each radar, in m. + """ + interp = _dem_interpolator(dem) + lat = np.atleast_1d(np.asarray(Grid["radar_latitude"].values).ravel()) + lon = np.atleast_1d(np.asarray(Grid["radar_longitude"].values).ravel()) + alt = np.atleast_1d(np.asarray(Grid["radar_altitude"].values).ravel()) + + resid = alt - interp(np.stack([lat, lon], axis=-1)) + + if verbose: + for n, r in enumerate(resid): + if np.abs(r) > tower_max: + print( + " WARNING radar %d: radar_altitude - DEM = %+.1f m. A small " + "positive tower height was expected. Check the DEM's vertical " + "datum; see the notes in pydda.terrain." % (n, r) + ) + else: + print(" radar %d: tower height %+.1f m, datum consistent." % (n, r)) + return resid + + +def terrain_ghost_cells( + Grid, dem, refine=4, smooth_cells=1.0, image_dist=1.75, verbose=True +): + """ + Build the ghost-cell immersed-boundary structure for an analysis grid. + + Each solid cell adjacent to the fluid is reflected across the terrain + surface to an image point, where the velocity is interpolated trilinearly + from the surrounding fluid cells. The two are related by + + .. math:: + + \\vec{u}_G = M_G \\vec{u}_I, \\qquad + M_G = I - (1 + \\beta_G)\\hat{n}\\hat{n}^T, \\qquad + \\beta_G = d_G / d_I + + with :math:`d_G` the distance from the ghost cell down to the surface and + :math:`d_I` the distance from the surface up to the image point. The normal + velocity varies linearly across the wall and so vanishes on it; the + tangential components have zero normal derivative. When :math:`d_I = d_G` + this is the Householder reflection :math:`I - 2\\hat{n}\\hat{n}^T`; larger + :math:`d_I` lets the image point stand off far enough for a + well-conditioned interpolation stencil. + + The resulting relation is *linear* in the wind field, so it enters the + retrieval as a quadratic penalty via the ``Cterrain`` weight of + :py:func:`pydda.retrieval.get_dd_wind_field`. + + Parameters + ---------- + Grid: xarray.Dataset + A PyDDA grid, as returned by :func:`pydda.io.read_grid`. + dem: xarray.DataArray + Orthometric (above-MSL) elevation in m, with 1D "lat" and "lon" + coordinates covering the grid domain. Run :func:`check_datum` first. + refine: int + DEM sub-samples per analysis cell along each horizontal axis. The + samples are block-averaged to the grid scale, so a DEM much finer than + the analysis grid is used properly rather than point-sampled. + smooth_cells: float + Gaussian smoothing of the grid-scale terrain, in analysis cells, + applied before the slopes are computed. Raw 30 m DEM slopes are far + steeper than the resolved flow can respond to, and differencing them + injects vertical velocity that no radar observed. + image_dist: float + Minimum image-point standoff from the terrain surface, in units of the + characteristic cell size ``(dx dy dz)**(1/3)``. + verbose: bool + Print a summary of the resulting geometry. + + Returns + ------- + terrain: dict + Dictionary with the following entries, suitable for passing to the + ``terrain`` keyword of :py:func:`pydda.retrieval.get_dd_wind_field`: + + h: 2D float array + Grid-scale terrain elevation in m above MSL. + fluid: 3D bool array + True where the cell centre is above the terrain surface. + phi: 3D float array + Signed distance to the terrain surface in m, positive in the fluid. + ghost_flat: 1D int array + Flat indices of the ghost cells. + stencil_flat: 2D int array + (n_ghost, 8) flat indices of each image point's stencil. + stencil_w: 2D float array + (n_ghost, 8) trilinear weights, rows summing to one. + M: 3D float array + (n_ghost, 3, 3) wall operators. + solid_free_flat: 1D int array + Flat indices of solid cells that are not ghost cells. These carry + no boundary information and are damped towards zero so that they + do not wander during the optimization. + n_dropped: int + Ghost cells discarded because no fluid cell was available to + interpolate from. A large count means the grid is too coarse for + the terrain. + """ + x = Grid["x"].values + y = Grid["y"].values + dx = float(x[1] - x[0]) + dy = float(y[1] - y[0]) + nx, ny = len(x), len(y) + + origin_alt = float(np.asarray(Grid["origin_altitude"].values).ravel()[0]) + z_msl = Grid["z"].values + origin_alt + dz = float(z_msl[1] - z_msl[0]) + nz = len(z_msl) + delta = (dx * dy * dz) ** (1.0 / 3.0) + + # Terrain height on the analysis columns, and its slopes. The slopes come + # from the smoothed field; see the smooth_cells docstring. + h = _sample_dem(Grid, dem, refine, x, y, dx, dy, nx, ny) + h = gaussian_filter(h, smooth_cells, mode="nearest") + hy, hx = np.gradient(h, dy, dx) + + # Signed distance to a height field. Exact for planar terrain; see the + # limitations note in the module documentation of pydda.cost_functions. + slope_norm = np.sqrt(1.0 + hx**2 + hy**2) + phi = (z_msl[:, None, None] - h[None, ...]) / slope_norm[None, ...] + fluid = phi > 0.0 + solid = ~fluid + + # A ghost cell is a solid cell with at least one face neighbour in the fluid + nbr = np.zeros_like(fluid) + for ax in (0, 1, 2): + for shift in (-1, 1): + nbr |= np.roll(fluid, shift, axis=ax) + ghost = solid & nbr + gk, gj, gi = np.nonzero(ghost) + + nvec = np.stack( + [ + -hx[gj, gi] / slope_norm[gj, gi], + -hy[gj, gi] / slope_norm[gj, gi], + 1.0 / slope_norm[gj, gi], + ], + axis=1, + ) + d_G = -phi[gk, gj, gi] + d_I = np.maximum(d_G, image_dist * delta) + beta = d_G / d_I + + x_ghost = np.stack([x[gi], y[gj], z_msl[gk]], axis=1) + x_image = x_ghost + (d_G + d_I)[:, None] * nvec + + stencil_flat, stencil_w = _trilinear_stencil( + x_image, x, y, z_msl, dx, dy, dz, nx, ny, nz + ) + + # Drop non-fluid corners from the stencil and renormalize. Ghost cells left + # with no fluid corner at all are removed from the constraint set entirely, + # and counted rather than silently ignored. + stencil_w = np.where(fluid.ravel()[stencil_flat], stencil_w, 0.0) + wsum = stencil_w.sum(axis=1) + keep = wsum > 1e-8 + n_dropped = int((~keep).sum()) + + stencil_w = stencil_w[keep] / wsum[keep, None] + stencil_flat = stencil_flat[keep] + nvec = nvec[keep] + beta = beta[keep] + ghost_flat = (gk * ny * nx + gj * nx + gi)[keep] + + M = ( + np.eye(3)[None, ...] + - (1.0 + beta)[:, None, None] * nvec[:, :, None] * nvec[:, None, :] + ) + + # Solid cells that carry no boundary information + solid_flat = np.nonzero(solid.ravel())[0] + solid_free_flat = np.setdiff1d(solid_flat, ghost_flat, assume_unique=False) + + if verbose: + print( + "Terrain: %d ghost cells, %d dropped, %d damped solid cells, " + "%.1f%% of the domain is fluid." + % ( + len(ghost_flat), + n_dropped, + len(solid_free_flat), + 100.0 * fluid.mean(), + ) + ) + if n_dropped > 0.05 * max(len(ghost_flat), 1): + print( + " WARNING: more than 5%% of ghost cells lacked a fluid stencil. " + "The analysis grid may be too coarse to resolve this terrain." + ) + + return { + "h": h, + "fluid": fluid, + "phi": phi, + "ghost_flat": ghost_flat, + "stencil_flat": stencil_flat, + "stencil_w": stencil_w, + "M": M, + "solid_free_flat": solid_free_flat, + "n_dropped": n_dropped, + } + + +def _dem_interpolator(dem): + """Build a RegularGridInterpolator over a DEM with ascending lat/lon.""" + lat = dem["lat"].values + lon = dem["lon"].values + if lat[1] < lat[0]: + lat = lat[::-1] + dem = dem.isel(lat=slice(None, None, -1)) + if lon[1] < lon[0]: + lon = lon[::-1] + dem = dem.isel(lon=slice(None, None, -1)) + return RegularGridInterpolator( + (lat, lon), + np.asarray(dem.values, dtype=np.float64), + bounds_error=False, + fill_value=None, + ) + + +def _sample_dem(Grid, dem, refine, x, y, dx, dy, nx, ny): + """Block-average a DEM onto the analysis columns.""" + off = (np.arange(refine) + 0.5) / refine - 0.5 + xf = (x[:, None] + off[None, :] * dx).ravel() + yf = (y[:, None] + off[None, :] * dy).ravel() + Xf, Yf = np.meshgrid(xf, yf) + + projparams = { + "proj": "pyart_aeqd", + "lon_0": float(np.asarray(Grid["origin_longitude"].values).ravel()[0]), + "lat_0": float(np.asarray(Grid["origin_latitude"].values).ravel()[0]), + } + lon, lat = pyart.core.cartesian_to_geographic(Xf, Yf, projparams) + + h_fine = _dem_interpolator(dem)(np.stack([lat, lon], axis=-1)) + return h_fine.reshape(ny, refine, nx, refine).mean(axis=(1, 3)) + + +def _trilinear_stencil(pts, x, y, z, dx, dy, dz, nx, ny, nz): + """Flat indices and trilinear weights of the 8 cells around each point.""" + fi = np.clip((pts[:, 0] - x[0]) / dx, 0, nx - 1.001) + fj = np.clip((pts[:, 1] - y[0]) / dy, 0, ny - 1.001) + fk = np.clip((pts[:, 2] - z[0]) / dz, 0, nz - 1.001) + i0 = fi.astype(np.int64) + j0 = fj.astype(np.int64) + k0 = fk.astype(np.int64) + ti, tj, tk = fi - i0, fj - j0, fk - k0 + + idx = np.empty((len(pts), 8), dtype=np.int64) + wts = np.empty((len(pts), 8), dtype=np.float64) + for c, (a, b, d) in enumerate(np.ndindex(2, 2, 2)): + idx[:, c] = (k0 + d) * ny * nx + (j0 + b) * nx + (i0 + a) + wts[:, c] = ( + (ti if a else 1.0 - ti) * (tj if b else 1.0 - tj) * (tk if d else 1.0 - tk) + ) + return idx, wts diff --git a/pydda/tests/test_terrain.py b/pydda/tests/test_terrain.py new file mode 100644 index 00000000..5b33c6b9 --- /dev/null +++ b/pydda/tests/test_terrain.py @@ -0,0 +1,109 @@ +"""Unit tests for the terrain immersed-boundary constraint.""" + +import numpy as np +import xarray as xr + +import pydda +from pydda.cost_functions import _cost_functions_numpy + + +def _make_grid(): + """Return a small grid whose lower level intersects a flat DEM.""" + return xr.Dataset( + { + "x": ("x", np.array([-1000.0, 0.0, 1000.0])), + "y": ("y", np.array([-1000.0, 0.0, 1000.0])), + "z": ("z", np.array([0.0, 500.0, 1000.0])), + "origin_latitude": ("nradar", np.array([40.0])), + "origin_longitude": ("nradar", np.array([-105.0])), + "origin_altitude": ("nradar", np.array([0.0])), + } + ) + + +def _make_flat_dem(): + """Return a constant orthometric DEM covering the test grid.""" + return xr.DataArray( + np.full((5, 5), 250.0), + coords={ + "lat": np.linspace(39.98, 40.02, 5), + "lon": np.linspace(-105.02, -104.98, 5), + }, + dims=("lat", "lon"), + ) + + +def test_terrain_ghost_cells_identifies_fluid_and_ghost_cells(): + terrain = pydda.terrain.terrain_ghost_cells( + _make_grid(), _make_flat_dem(), smooth_cells=0.0, verbose=False + ) + + assert terrain["h"].shape == (3, 3) + np.testing.assert_allclose(terrain["h"], 250.0) + assert not np.any(terrain["fluid"][0]) + assert np.all(terrain["fluid"][1:]) + assert terrain["n_dropped"] == 0 + assert terrain["ghost_flat"].size > 0 + np.testing.assert_allclose(terrain["stencil_w"].sum(axis=1), 1.0) + + # A flat wall has a vertical normal, so the IBM operator leaves the + # horizontal components unchanged and reverses the normal component. + np.testing.assert_allclose(terrain["M"][:, 0, 0], 1.0) + np.testing.assert_allclose(terrain["M"][:, 1, 1], 1.0) + assert np.all(terrain["M"][:, 2, 2] < 0.0) + + +def test_terrain_cost_and_gradient_match_finite_difference(): + shape = (2, 2, 2) + terrain = { + "ghost_flat": np.array([0]), + "stencil_flat": np.array([[7] * 8]), + "stencil_w": np.full((1, 8), 1.0 / 8.0), + "M": np.array([np.diag([1.0, 1.0, -1.0])]), + "solid_free_flat": np.array([1, 2]), + } + rng = np.random.default_rng(7) + fields = rng.normal(size=(3,) + shape) + coeff = 2.5 + + cost = _cost_functions_numpy.calculate_terrain_cost(*fields, terrain, coeff) + gradient = _cost_functions_numpy.calculate_terrain_gradient(*fields, terrain, coeff) + winds = fields.ravel() + numerical = np.empty_like(winds) + epsilon = 1.0e-6 + for index in range(winds.size): + plus = winds.copy() + minus = winds.copy() + plus[index] += epsilon + minus[index] -= epsilon + plus_fields = plus.reshape((3,) + shape) + minus_fields = minus.reshape((3,) + shape) + numerical[index] = ( + _cost_functions_numpy.calculate_terrain_cost(*plus_fields, terrain, coeff) + - _cost_functions_numpy.calculate_terrain_cost( + *minus_fields, terrain, coeff + ) + ) / (2.0 * epsilon) + + assert cost > 0.0 + np.testing.assert_allclose(gradient, numerical, rtol=1.0e-5, atol=1.0e-7) + + +def test_terrain_cost_is_zero_for_a_reflected_ghost_value(): + terrain = { + "ghost_flat": np.array([0]), + "stencil_flat": np.array([[7] * 8]), + "stencil_w": np.full((1, 8), 1.0 / 8.0), + "M": np.array([np.diag([1.0, 1.0, -1.0])]), + "solid_free_flat": np.array([], dtype=int), + } + u = np.zeros((2, 2, 2)) + v = np.zeros((2, 2, 2)) + w = np.zeros((2, 2, 2)) + u.ravel()[0], v.ravel()[0], w.ravel()[0] = 1.0, 2.0, -3.0 + u.ravel()[7], v.ravel()[7], w.ravel()[7] = 1.0, 2.0, 3.0 + + assert _cost_functions_numpy.calculate_terrain_cost(u, v, w, terrain) == 0.0 + np.testing.assert_allclose( + _cost_functions_numpy.calculate_terrain_gradient(u, v, w, terrain), 0.0 + )