Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Adapt

**Real-time detection, projection, and tracking of convective storms in weather radar data, built to support adaptive radar scanning at the DOE ARM User Facility.**
**Real-time detection, projection, and tracking of convective storms in weather radar data — built for operational adaptive scanning at the DOE ARM User Facility, and for large-scale generation of value-added products and AI-ready storm datasets.**

![Status](https://img.shields.io/badge/STATUS-ACTIVE%20DEVELOPMENT-orange?style=for-the-badge&logo=github)
![API](https://img.shields.io/badge/API-BREAKING%20CHANGES-red?style=for-the-badge&logo=dependabot)
Expand Down Expand Up @@ -60,6 +60,12 @@ applications such as storm lifecycle studies, radar–lightning correlation anal
of analysis-ready, multi-sensor datasets combining radar, satellite, and Lightning Mapping Array
observations.

## From operations to large-scale analysis

The pipeline that selects targets in real time also reprocesses years of archived data, with identical,
traceable results. Its tracked cells, lifecycle events and decision logs make ready-made value-added
products and consistent datasets for AI-driven research.

## Current status

Adapt is at an alpha stage of development.
Expand Down
56 changes: 56 additions & 0 deletions environment-dev.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
# Development environment for Adapt.
#
# mamba env create -f environment-dev.yml # creates `adapt_dev`
# mamba activate adapt_dev
# pip install -e . --no-deps --no-build-isolation # Adapt itself, editable
#
# Every dependency comes from conda-forge; Adapt is the only pip install and it
# is editable so source edits take effect immediately. --no-deps keeps pip from
# re-resolving anything conda already provided; --no-build-isolation reuses the
# setuptools/setuptools_scm in the env instead of fetching a build sandbox.
#
# Verified 2026-09-21: 1437 tests pass (python 3.12, arm_pyart 2.3.0, opencv 5.0).
# The 4 conv_strat_raut tests that fail under arm_pyart 2.2.0 pass here.
name: adapt_dev
channels:
- conda-forge
dependencies:
- python=3.12
- pip
# build backend for the editable install (pyproject [build-system])
- setuptools>=77
- wheel
- setuptools_scm>=8
- git # setuptools_scm reads the version from git
# runtime (pyproject [project.dependencies])
- numpy
- xarray
- pandas
- scipy
- scikit-image
- matplotlib
- networkx
- h5py
- netcdf4
- pydantic>=2
- pyyaml
- arm_pyart
- s3fs>=2021.11 # pyart requires it; the solver otherwise picks 0.4.2
- py-opencv # conda-forge name for opencv-python
- boto3
- cmweather
- pyproj
# optional extras
- contextily # [maps]
- ffmpeg # movie consumer; its tests skip without it
- pyarrow # parquet I/O for analysis scripts
# dev (pyproject [project.optional-dependencies].dev)
- pytest>=7
- pytest-cov
- import-linter>=2
- pre-commit>=4
# docs (pyproject [project.optional-dependencies].docs)
- sphinx>=7
- pydata-sphinx-theme>=0.15
- sphinx-autodoc-typehints
- myst-parser>=2
210 changes: 210 additions & 0 deletions src/adapt/configuration/schemas/collection_decisions.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
-- Analysis-only decision log: collections/<RADAR>/decisions.db
--
-- Written by the output router from the tracking and detection modules'
-- decision records (adapt.contracts.decisions); never read by the pipeline or
-- the public API. Rows mirror the frozen record fields plus run/scan stamps.
-- Config thresholds are not repeated here: join run_id to the run registry.

-- user_version 3: tracker v2 (grown overlaps, curved gate, additive cost with a
-- no-link ceiling, crossing test, latent tracks, identity rule). Its records go
-- to tables with new names — tracking_scans, _cells, _pairs, _lineage,
-- _identity, _latent — so a decisions.db written by the previous tracker keeps
-- its tracking_frames/_candidates/_unmatched/_split_merge_tests rows untouched
-- and still takes new runs. CREATE TABLE IF NOT EXISTS does not migrate an
-- existing table: a changed column set needs a new table name or a new base_dir.
PRAGMA user_version = 3;

CREATE TABLE IF NOT EXISTS tracking_scans (
run_id TEXT NOT NULL,
scan_id TEXT NOT NULL,
scan_time TEXT NOT NULL,
dt_s REAL,
reset_code TEXT,
n_prev INTEGER NOT NULL,
n_latent INTEGER NOT NULL,
n_curr INTEGER NOT NULL,
n_pairs INTEGER NOT NULL,
n_pass_overlap INTEGER NOT NULL,
n_pass_speed INTEGER NOT NULL,
n_pass_cost INTEGER NOT NULL,
n_components INTEGER NOT NULL,
n_continue INTEGER NOT NULL,
n_crossing_excluded INTEGER NOT NULL,
n_split INTEGER NOT NULL,
n_merge INTEGER NOT NULL,
n_resumed INTEGER NOT NULL,
n_identity_transfer INTEGER NOT NULL,
n_initiation INTEGER NOT NULL,
n_latent_created INTEGER NOT NULL,
n_termination INTEGER NOT NULL,
PRIMARY KEY (run_id, scan_id)
);

CREATE TABLE IF NOT EXISTS tracking_cells (
run_id TEXT NOT NULL,
scan_id TEXT NOT NULL,
scan_time TEXT NOT NULL,
cell_label INTEGER NOT NULL,
cell_uid TEXT NOT NULL,
area_km2 REAL NOT NULL,
mass REAL NOT NULL, -- Σ e, e = excess above the cell threshold
mean_excess REAL NOT NULL, -- ē = mass / area (per pixel)
centre_x REAL NOT NULL, -- e²-weighted centre, metres
centre_y REAL NOT NULL,
score REAL NOT NULL, -- identity score S = log A + γ log ē (−inf: no excess)
core_area_km2 REAL NOT NULL, -- diagnostic, above tracker.core_field_threshold
fate TEXT NOT NULL, -- CONTINUE | RESUMED | SPLIT_CHILD | INITIATION
PRIMARY KEY (run_id, scan_id, cell_label)
);
CREATE INDEX IF NOT EXISTS idx_tracking_cells_uid ON tracking_cells (run_id, cell_uid);

CREATE TABLE IF NOT EXISTS tracking_pairs (
run_id TEXT NOT NULL,
scan_id TEXT NOT NULL,
scan_time TEXT NOT NULL,
prev_cell_uid TEXT NOT NULL,
prev_cell_label INTEGER NOT NULL,
curr_cell_label INTEGER NOT NULL,
source TEXT NOT NULL, -- LIVE footprint or LATENT track
steps INTEGER NOT NULL, -- scan intervals spanned
growth_radius_km REAL NOT NULL,
footprint_area_px INTEGER NOT NULL,
footprint_grown_px INTEGER NOT NULL,
cell_grown_px INTEGER NOT NULL,
intersection_px INTEGER NOT NULL,
o_c REAL NOT NULL, -- |H+ ∩ M+| / |M+|
o_h REAL NOT NULL, -- |H+ ∩ M+| / |H+|
u REAL NOT NULL, -- 1 − √(o_c o_h)
predicted_x REAL NOT NULL,
predicted_y REAL NOT NULL,
curr_centre_x REAL NOT NULL,
curr_centre_y REAL NOT NULL,
d REAL NOT NULL, -- shape-aware residual √(δᵀ Σ⁻¹ δ)
speed_ms REAL NOT NULL,
prev_speed_ms REAL,
speed_limit_ms REAL, -- v_prev + Δv_max; NULL on a track's first step
heading_change_deg REAL, -- NULL when a step is below two grid lengths
h REAL NOT NULL,
cost REAL NOT NULL, -- u + w_d d + h
gate TEXT NOT NULL, -- PASS | OVERLAP | SPEED | SPEED_CHANGE | COST
component_id INTEGER,
cost_rank INTEGER,
margin REAL,
crossing_with_uid TEXT,
outcome TEXT NOT NULL, -- CONTINUE | RESUMED | LOST | CROSSING | REJECTED
PRIMARY KEY (run_id, scan_id, prev_cell_uid, curr_cell_label)
);
CREATE INDEX IF NOT EXISTS idx_tracking_pairs_prev ON tracking_pairs (run_id, prev_cell_uid);

CREATE TABLE IF NOT EXISTS tracking_lineage (
run_id TEXT NOT NULL,
scan_id TEXT NOT NULL,
scan_time TEXT NOT NULL,
seq INTEGER NOT NULL,
side TEXT NOT NULL, -- curr orphan | prev unlinked cell
cell_uid TEXT NOT NULL,
cell_label INTEGER NOT NULL,
hypothesis TEXT NOT NULL, -- RESUME | SPLIT | INITIATION | MERGE | LATENT
partner_uid TEXT,
partner_label INTEGER,
fraction REAL,
cost REAL,
threshold REAL,
passed INTEGER NOT NULL,
chosen INTEGER NOT NULL,
reason TEXT,
PRIMARY KEY (run_id, scan_id, seq)
);

CREATE TABLE IF NOT EXISTS tracking_identity (
run_id TEXT NOT NULL,
scan_id TEXT NOT NULL,
scan_time TEXT NOT NULL,
kind TEXT NOT NULL, -- SPLIT | MERGE
identity_uid TEXT NOT NULL,
candidate_side TEXT NOT NULL,
candidate_label INTEGER NOT NULL,
score REAL NOT NULL,
distance REAL NOT NULL,
winner INTEGER NOT NULL,
rule TEXT NOT NULL, -- SCORE | NEAREST
transferred INTEGER NOT NULL,
PRIMARY KEY (run_id, scan_id, kind, identity_uid, candidate_side, candidate_label)
);

CREATE TABLE IF NOT EXISTS tracking_latent (
run_id TEXT NOT NULL,
scan_id TEXT NOT NULL,
scan_time TEXT NOT NULL,
cell_uid TEXT NOT NULL,
last_label INTEGER NOT NULL,
last_scan_id TEXT NOT NULL,
age INTEGER NOT NULL,
origin TEXT NOT NULL, -- MERGED | TERMINATED
merged_into_uid TEXT,
status TEXT NOT NULL, -- CREATED | CARRIED | RESUMED | EXPIRED
footprint_px INTEGER NOT NULL,
predicted_x REAL NOT NULL,
predicted_y REAL NOT NULL,
PRIMARY KEY (run_id, scan_id, cell_uid)
);

CREATE TABLE IF NOT EXISTS segmentation_frames (
run_id TEXT NOT NULL,
scan_id TEXT NOT NULL,
scan_time TEXT NOT NULL,
mask_px INTEGER NOT NULL,
closed_mask_px INTEGER NOT NULL,
n_components INTEGER NOT NULL,
n_independent_seeds INTEGER NOT NULL,
n_carried_claims INTEGER NOT NULL,
n_carried_admitted INTEGER NOT NULL,
n_basins INTEGER NOT NULL,
n_dropped_small INTEGER NOT NULL,
n_dropped_large INTEGER NOT NULL,
n_cells INTEGER NOT NULL,
n_size_exempt INTEGER,
PRIMARY KEY (run_id, scan_id)
);

CREATE TABLE IF NOT EXISTS segmentation_seeds (
run_id TEXT NOT NULL,
scan_id TEXT NOT NULL,
scan_time TEXT NOT NULL,
seq INTEGER NOT NULL,
marker_id INTEGER,
origin TEXT NOT NULL,
row INTEGER NOT NULL,
col INTEGER NOT NULL,
peak_value REAL,
plateau_px INTEGER,
component_id INTEGER,
prior_label INTEGER,
prior_age INTEGER,
footprint_px INTEGER,
clearance_px REAL,
footprint_holds_seed INTEGER,
component_claims INTEGER,
component_seeds INTEGER,
component_deficit INTEGER,
decision TEXT NOT NULL,
basin_px INTEGER,
final_label INTEGER,
PRIMARY KEY (run_id, scan_id, seq)
);

CREATE TABLE IF NOT EXISTS segmentation_components (
run_id TEXT NOT NULL,
scan_id TEXT NOT NULL,
scan_time TEXT NOT NULL,
component_id INTEGER NOT NULL,
area_px INTEGER NOT NULL,
field_max REAL NOT NULL,
field_min REAL NOT NULL,
n_seeds INTEGER NOT NULL,
n_claims INTEGER NOT NULL,
deficit INTEGER NOT NULL,
n_admitted INTEGER NOT NULL,
n_final_labels INTEGER NOT NULL,
PRIMARY KEY (run_id, scan_id, component_id)
);
63 changes: 50 additions & 13 deletions src/adapt/configuration/schemas/internal.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ class InternalDownloaderConfig(AdaptBaseModel):
start_time: str | None
end_time: str | None
min_file_size: int
max_queue_size: int = 100
queue_resume_fraction: float = 0.10


class InternalRegridderConfig(AdaptBaseModel):
Expand All @@ -73,9 +75,12 @@ class InternalSegmenterConfig(AdaptBaseModel):
]
min_cellsize_gridpoint: int
max_cellsize_gridpoint: int | None
closing_kernel: tuple[int, int]
closing_radius: int
filter_by_size: bool
h_maxima: float
seed_carry_frames: int
seed_carry_min_separation: int
carried_exempt_size_filter: bool = False
threshold_params: ThresholdParams
conv_strat_raut_params: ConvStratRautParams
conv_strat_yuter_params: ConvStratYuterParams
Expand Down Expand Up @@ -118,6 +123,7 @@ class InternalGlobalConfig(AdaptBaseModel):
"""Runtime global settings."""

z_level: float
max_scan_gap_minutes: float = Field(default=10.0, gt=0.0)
tracking_field: str
coord_names: InternalCoordNamesConfig

Expand All @@ -138,7 +144,6 @@ class InternalProjectorConfig(AdaptBaseModel):
"""Runtime projection configuration."""

method: str
max_time_interval_minutes: int
max_projection_steps: int = Field(ge=1, le=10) # Capped at 10
nan_fill_value: float
flow_params: InternalFlowParamsConfig
Expand All @@ -156,26 +161,58 @@ class InternalAnalyzerConfig(AdaptBaseModel):
adjacency_min_touching_boundary_pixels: int = Field(ge=1)


# Tracker v1 keys → the v2 parameter that replaced them. A config written for
# the previous tracker must fail loudly, not run with silently ignored values.
_REMOVED_TRACKER_KEYS = {
"minimum_candidate_overlap": "max_overlap_mismatch",
"minimum_projected_overlap": "max_overlap_mismatch",
"projected_hull_buffer_km": "growth_max_km / growth_size_km",
"length_scale": "residual_weight",
"geometry_length_scale_km": "residual_weight",
"max_speed_multiplier": "max_acceleration_ms2",
"acceleration_floor_ms": "max_acceleration_ms2",
"heading_change_penalty_weight": "heading_weight",
}


class InternalTrackerConfig(AdaptBaseModel):
"""Runtime tracking configuration."""

@model_validator(mode="before")
@classmethod
def _reject_removed_keys(cls, data):
if isinstance(data, dict):
removed = sorted(k for k in data if k in _REMOVED_TRACKER_KEYS)
if removed:
hints = ", ".join(f"{k} → {_REMOVED_TRACKER_KEYS[k]}" for k in removed)
raise ValueError(
f"tracker: {hints} — these keys belong to the previous tracker; "
"replace them with the new parameters (see `adapt config`)"
)
return data

class InternalCellUidConfig(AdaptBaseModel):
"""Runtime cell UID configuration."""

width: int = Field(ge=1)
alphabet: Literal["base36_upper"]

split_overlap_threshold: float = Field(default=0.8, ge=0.0, le=1.0)
core_field_threshold: float = Field(default=40.0, ge=0.0)
max_tracking_gap_minutes: float = Field(default=20.0, gt=0.0)
max_speed_ms: float = Field(default=40.0, gt=0.0)
max_speed_multiplier: float = Field(default=3.0, gt=0.0)
heading_change_penalty_weight: float = Field(default=0.0, ge=0.0)
projected_hull_buffer_km: float = Field(default=1.0, gt=0.0)
minimum_candidate_overlap: float = Field(default=0.20, ge=0.0, le=1.0)
minimum_projected_overlap: float = Field(default=0.20, ge=0.0, le=1.0)
length_scale: Literal["hull_equiv_diameter", "sum_radii", "fixed_km"] = "hull_equiv_diameter"
geometry_length_scale_km: float = Field(default=5.0, gt=0.0)
cell_threshold: float
polarity: Literal[1, -1]
growth_max_km: float = Field(ge=0.0)
growth_size_km: float = Field(gt=0.0)
max_overlap_mismatch: float = Field(gt=0.0, le=1.0)
max_link_cost: float = Field(gt=0.0)
residual_weight: float = Field(ge=0.0)
heading_weight: float = Field(ge=0.0)
max_speed_ms: float = Field(gt=0.0)
max_acceleration_ms2: float = Field(ge=0.0)
split_overlap_threshold: float = Field(gt=0.0, le=1.0)
merge_overlap_threshold: float = Field(gt=0.0, le=1.0)
latent_scans: int = Field(ge=0)
identity_intensity_weight: float = Field(ge=0.0)
identity_score_margin: float = Field(ge=0.0)
core_field_threshold: float
cell_uid: InternalCellUidConfig


Expand Down
Loading
Loading